X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~yarrgweb/git?a=blobdiff_plain;f=yoweb-scrape;h=ba828f51c0fb1a125e6ef729f82caacf0f7919b4;hb=a732139b9b5d7a0a7d132f3020101f52ceea00e2;hp=a705f6792f05ad5d017086d54e98c717b9b79131;hpb=1da7a29fdca76778d028146370e212307a6358f5;p=ypp-sc-tools.web-live.git diff --git a/yoweb-scrape b/yoweb-scrape index a705f67..ba828f5 100755 --- a/yoweb-scrape +++ b/yoweb-scrape @@ -44,6 +44,7 @@ import random import curses import termios import random +import subprocess from optparse import OptionParser from StringIO import StringIO @@ -98,6 +99,14 @@ def format_time_interval(ti): if ti < 86400: return '%dh' % (ti / 3600) return '%dd' % (ti / 86400) +def yppsc_dir(): + lib = os.getenv("YPPSC_YARRG_SRCBASE") + if lib is not None: return lib + lib = sys.argv[0] + lib = regexp.sub('/[^/]+$', '', lib) + os.environ["YPPSC_YARRG_SRCBASE"] = lib + return lib + #---------- caching and rate-limiting data fetcher ---------- class Fetcher: @@ -377,6 +386,203 @@ class CrewInfo(SomethingSoupInfo): def __str__(self): return `(self.crew, self.msgs)` +class FlagInfo(SomethingSoupInfo): + # Public data members (after init): + # + # name # string + # + # relations[n] = (otherflagname, otherflagid, [stringfromyoweb], + # thisdeclaring, otherdeclaringmin, otherdeclaringmax) + # # where {this,other}declaring{,min,max} are: + # # -1 {this,other} is declaring war + # # 0 {this,other} is not doing either + # # +1 {this,other} is allying + # relation_byname[otherflagname] = relations[some_n] + # relation_byid[otherflagname] = relations[some_n] + # + # islands[n] = (islandname, islandid) + # + def __init__(self, flagid, max_age=600): + SomethingSoupInfo.__init__(self, + 'flag/info.wm?flagid=', flagid, max_age) + self._find_flag() + + def _find_flag(self): + font2 = self._soup.find('font',{'size':'+2'}) + self.name = font2.find('b').contents[0] + + self.relations = [ ] + self.relation_byname = { } + self.relation_byid = { } + self.islands = [ ] + + magnate = self._soup.find('img',{'src': + '/yoweb/images/repute-MAGNATE.png'}) + warinfo = (magnate.findParent('table').findParent('tr'). + findNextSibling('tr').findNext('td',{'align':'left'})) + + def warn(m): + print >>sys.stderr, 'WARNING: '+m + + def wi_warn(head, waritem): + warn('unknown warmap item: %s: %s' % + (`head`, ``waritem``)) + + def wihelp_item(waritem, thing): + url = waritem.get('href', None) + if url is None: + return ('no url for '+thing,None,None) + m = regexp.search('\?'+thing+'id=(\d+)$', url) + if not m: return ('no '+thing+'id',None,None) + tid = m.group(1) + tname = waritem.string + if tname is None: + return (thing+' name not just string',None,None) + return (None,tid,tname) + + def wi_alwar(head, waritem, thisdecl, othermin, othermax): + (err,flagid,flagname) = wihelp_item(waritem,'flag') + if err: return err + rel = self.relation_byid.get(flagid, None) + if rel: return 'flag id twice!' + if flagname in self.relation_byname: + return 'flag name twice!' + rel = (flagname,flagid,head, thisdecl,othermin,othermax) + self.relations.append(rel) + self.relation_byid[flagid] = rel + self.relation_byname[flagid] = rel + + def wi_isle(head, waritem): + (err,isleid,islename) = wihelp_item(waritem,'island') + if err: return err + self.islands.append((isleid,islename)) + + warmap = { + 'Allied with': (wi_alwar,+1,+1,+1), + 'Declaring war against': (wi_alwar,-1, 0,+1), + 'At war with': (wi_alwar,-1,-1,-1), + 'Trying to form an alliance with': (wi_alwar,+1,-1,0), + 'Islands controlled by this flag': (wi_isle,), + } + + how = (wi_warn, None) + + for waritem in warinfo.findAll(['font','a']): + if waritem is None: break + if waritem.name == 'font': + colour = waritem.get('color',None) + if colour.lstrip('#') != '958A5F': + warn('strange colour %s in %s' % + (colour,``waritem``)) + continue + head = waritem.string + if head is None: + warn('no head string in '+``waritem``) + continue + head = regexp.sub('\\s+', ' ', head).strip() + head = head.rstrip(':') + how = (head,) + warmap.get(head, (wi_warn,)) + continue + assert(waritem.name == 'a') + + debug('WARHOW %s(%s, waritem, *%s)' % + (how[1], `how[0]`, `how[2:]`)) + bad = how[1](how[0], waritem, *how[2:]) + if bad: + warn('bad waritem %s: %s: %s' % (`how[0]`, + bad, ``waritem``)) + + def __str__(self): + return `(self.name, self.islands, self.relations)` + +#---------- scraper for ocean info incl. embargoes etc. ---------- + +class IslandInfo(): + def __init__(self, ocean, islename): + self.ocean = ocean + self.name = islename + def collect(self): + pass + def yppedia_dataf(self): + def q(x): return urllib.quote(x.replace(' ','_')) + url_rhs = q(self.name) + '_(' + q(self.ocean) + ')' + if opts.localhtml is None: + url = 'http://yppedia.puzzlepirates.com/' + url_rhs + debug('IslandInfo retrieving YPP '+url); + return urllib.urlopen(url) + else: + return file(opts.localhtml + '/' + url_rhs, 'r') + def yoweb_url(self): + soup = BeautifulSoup(self.yppedia_dataf()) + content = soup.find('div', attrs = {'id': 'content'}) + yoweb_re = regexp.compile('^http://\w+\.puzzlepirates\.com/'+ + 'yoweb/island/info\.wm\?islandid=\d+$') + a = soup.find('a', attrs = { 'href': yoweb_re }) + if a is None: return None + return a['href'] + def ruling_flag_id(self): + yo = self.yoweb_url() + if yo is None: return None + dataf = fetcher.fetch(yo, 600) + soup = BeautifulSoup(dataf) + ruler_re = regexp.compile('http://\w+\.puzzlepirates\.com/'+ + 'yoweb/flag/info\.wm\?flagid=(\d+)$') + ruler = soup.find('a', attrs = { 'href': ruler_re }) + if not ruler: return None + m = ruler_re.find(ruler['href']) + return m.group(1) + +class OceanInfo(): + # Public data attributes (valid after collect()): + # oi.islands[islename] = IslandInfo(...) + # oi.arches[archname][islename] = IslandInfo(...) + def __init__(self): + self.isleclass = IslandInfo + self.ocean = fetcher.ocean.lower().capitalize() + def collect(self): + cmdl = ['./yppedia-ocean-scraper'] + if opts.localhtml is not None: + cmdl += ['--local-html-dir',opts.localhtml] + cmdl += [self.ocean] + debug('OceanInfo collect running ' + `cmdl`) + oscraper = subprocess.Popen( + cmdl, + stdout = subprocess.PIPE, + cwd = yppsc_dir()+'/yarrg', + shell=False, stderr=None, + ) + h = oscraper.stdout.readline() + debug('OceanInfo collect h '+`h`) + assert(regexp.match('^ocean ', h)) + arch_re = regexp.compile('^ (\S.*)') + island_re = regexp.compile('^ (\S.*)') + + self.islands = { } + self.arches = { } + archname = None + + for l in oscraper.stdout: + debug('OceanInfo collect l '+`l`) + l = l.rstrip('\n') + m = island_re.match(l) + if m: + assert(archname is not None) + islename = m.group(1) + isle = self.isleclass(self.ocean, islename) + isle.arch = archname + self.islands[islename] = isle + self.arches[archname][islename] = isle + continue + m = arch_re.match(l) + if m: + archname = m.group(1) + assert(archname not in self.arches) + self.arches[archname] = { } + continue + assert(False) + oscraper.wait() + assert(oscraper.returncode == 0) + #---------- pretty-printer for tables of pirate puzzle standings ---------- class StandingsTable: @@ -1036,6 +1242,14 @@ def do_crew_of(args, bu): ci = prep_crew_of(args, bu) print ci +def do_flag_of(args, bu): + if len(args) != 1: bu('flag-of takes one pirate name') + max_age = 300 + pi = PirateInfo(args[0], max_age) + if pi.flag is None: fi = None + else: fi = FlagInfo(pi.flag[0], max_age) + print fi + def do_standings_crew_of(args, bu): ci = prep_crew_of(args, bu, 60) tab = StandingsTable(sys.stdout) @@ -1048,6 +1262,18 @@ def do_standings_crew_of(args, bu): pi = PirateInfo(p, random.randint(900,1800)) tab.pirate(pi) +def do_ocean(args, bu): + if (len(args)): bu('ocean takes no further arguments') + fetcher.default_ocean() + oi = OceanInfo() + oi.collect() + for islename in sorted(oi.islands.keys()): + isle = oi.islands[islename] + yoweb_url = isle.yoweb_url() + print " %s -- %s" % (islename, yoweb_url) + +#----- modes which use the chat log parser are quite complex ----- + class ProgressPrintPercentage: def __init__(self, f=sys.stdout): self._f = f @@ -1063,8 +1289,6 @@ class ProgressPrintPercentage: self._f.write(' \r') self._f.flush() -#----- modes which use the chat log parser are quite complex ----- - def prep_chat_log(args, bu, progress=ProgressPrintPercentage(), max_myself_age=3600): @@ -1398,6 +1622,9 @@ display modes (for --display) apply to ship-aid: ao('--display', action='store', dest='display', type='choice', choices=['dumb','overwrite'], help='how to display ship aid') + ao('--local-ypp-dir', action='store', dest='localhtml', + help='get yppedia pages from local directory LOCALHTML'+ + ' instead of via HTTP') ao_jt = lambda wh, t: ao( '--timeout-sa-'+wh, action='store', dest='timeout_'+wh,