From 46fafd1982761ccb564d29a625528692abb4cd7c Mon Sep 17 00:00:00 2001 From: Ian Jackson Date: Fri, 15 May 2009 19:24:38 +0100 Subject: [PATCH] duty table --- yoweb-scrape | 148 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 102 insertions(+), 46 deletions(-) diff --git a/yoweb-scrape b/yoweb-scrape index 54ef0fe..493ef53 100755 --- a/yoweb-scrape +++ b/yoweb-scrape @@ -16,16 +16,20 @@ from BeautifulSoup import BeautifulSoup opts = None -duties = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+ + +puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+ '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+ '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+ '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/') -standingvals = ('Able/Distinguished/Respected/Master/Renowned'+ - '/Grand-Master/Legendary/Ultimate').split('/') +standingvals = ('Able/Distinguished/Respected/Master'+ + '/Renowned/Grand-Master/Legendary/Ultimate').split('/') pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm') +max_pirate_namelen = 20 + + def debug(m): if opts.debug: print >>sys.stderr, m @@ -52,7 +56,7 @@ class Fetcher: if oe.errno != errno.ENOENT: raise continue age = now - s.st_mtime - if age > opts.max_age: + if age > opts.expire_age: debug('Fetcher expire %d %s' % (age, path)) try: os.remove(path) except (OSError,IOError), oe: @@ -72,13 +76,13 @@ class Fetcher: debug('Fetcher morewait min=%d age=%d' % (min_age, age)) need_wait = max(need_wait, min_age - age) - min_age *= 2 min_age += 1 + min_age *= 1.5 if need_wait > 0: debug('Fetcher wait %d' % need_wait) time.sleep(need_wait) - def fetch(self, url): + def fetch(self, url, max_age): debug('Fetcher fetch %s' % url) cache_corename = urllib.quote_plus(url) cache_item = "%s/#%s#" % (self.cachedir, cache_corename) @@ -87,15 +91,17 @@ class Fetcher: if oe.errno != errno.ENOENT: raise f = None now = time.time() + max_age = max(opts.min_max_age, max(max_age, opts.expire_age)) if f is not None: s = os.fstat(f.fileno()) - if now > s.st_mtime + opts.max_age: + age = now - s.st_mtime + if age > max_age: debug('Fetcher stale') f = None if f is not None: data = f.read() f.close() - debug('Fetcher cached') + debug('Fetcher cached %d > %d' % (max_age, age)) return data debug('Fetcher fetch') @@ -112,10 +118,10 @@ class Fetcher: debug('Fetcher stored') return data - def yoweb(self, kind, tail): + def yoweb(self, kind, tail, max_age): url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % ( self.ocean, kind, tail) - return self.fetch(url) + return self.fetch(url, max_age) class SoupLog: def __init__(self): @@ -133,10 +139,10 @@ def soup_text(obj): return str.strip() class SomethingSoupInfo(SoupLog): - def __init__(self, kind, tail): + def __init__(self, kind, tail, max_age): SoupLog.__init__(self) - html = fetcher.yoweb(kind, tail) - self.soup = BeautifulSoup(html, + html = fetcher.yoweb(kind, tail, max_age) + self._soup = BeautifulSoup(html, convertEntities=BeautifulSoup.HTML_ENTITIES ) @@ -147,9 +153,9 @@ class PirateInfo(SomethingSoupInfo): # pi.flag = (id, name) # pi.msgs = [ 'message describing problem with scrape' ] - def __init__(self, pirate): + def __init__(self, pirate, max_age=300): SomethingSoupInfo.__init__(self, - 'pirate.wm?target=', pirate) + 'pirate.wm?target=', pirate, max_age) self._find_standings() self.crew = self._find_crewflag('crew', '^/yoweb/crew/info\\.wm') @@ -157,69 +163,69 @@ class PirateInfo(SomethingSoupInfo): '^/yoweb/flag/info\\.wm') def _find_standings(self): - imgs = self.soup.findAll('img', + imgs = self._soup.findAll('img', src=regexp.compile('/yoweb/images/stat.*')) re = regexp.compile( u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$' ) standings = { } - for skill in duties: + for skill in puzzles: standings[skill] = [ ] skl = SoupLog() for img in imgs: - try: duty = img['alt'] + try: puzzle = img['alt'] except KeyError: continue - if not duty in duties: - skl.soupm(img, 'unknown duty: "%s"' % duty) + if not puzzle in puzzles: + skl.soupm(img, 'unknown puzzle: "%s"' % puzzle) continue key = img.findParent('td') if key is None: - skl.soupm(img, 'duty at root! "%s"' % duty) + skl.soupm(img, 'puzzle at root! "%s"' % puzzle) continue valelem = key.findNextSibling('td') if valelem is None: - skl.soupm(key, 'duty missing sibling "%s"' - % duty) + skl.soupm(key, 'puzzle missing sibling "%s"' + % puzzle) continue valstr = soup_text(valelem) match = re.match(valstr) if match is None: - skl.soupm(key, ('duty "%s" unparseable'+ - ' standing "%s"') % (duty, valstr)) + skl.soupm(key, ('puzzle "%s" unparseable'+ + ' standing "%s"') % (puzzle, valstr)) continue standing = match.group(match.lastindex) - standings[duty].append(standing) + standings[puzzle].append(standing) self.standings = { } - for duty in duties: - sl = standings[duty] + for puzzle in puzzles: + sl = standings[puzzle] if len(sl) > 1: - skl.msg('duty "%s" multiple standings %s' % - (duty, `sl`)) + skl.msg('puzzle "%s" multiple standings %s' % + (puzzle, `sl`)) continue if not len(sl): - skl.msg('duty "%s" no standing found' % duty) + skl.msg('puzzle "%s" no standing found' % puzzle) continue standing = sl[0] for i in range(0, len(standingvals)-1): if standing == standingvals[i]: - self.standings[duty] = i - if not duty in self.standings: - skl.msg('duty "%s" unknown standing "%s"' % - (duty, standing)) + self.standings[puzzle] = i + if not puzzle in self.standings: + skl.msg('puzzle "%s" unknown standing "%s"' % + (puzzle, standing)) all_standings_ok = True - for duty in duties: - if not duty in self.standings: + for puzzle in puzzles: + if not puzzle in self.standings: self.needs_msgs(skl) def _find_crewflag(self, cf, yoweb_re): - things = self.soup.findAll('a', href=regexp.compile(yoweb_re)) + things = self._soup.findAll('a', href=regexp.compile(yoweb_re)) if len(things) != 1: self.msg('zero or several %s id references found' % cf) return None @@ -244,14 +250,14 @@ class CrewInfo(SomethingSoupInfo): # ... ] # pi.msgs = [ 'message describing problem with scrape' ] - def __init__(self, crewid): + def __init__(self, crewid, max_age=300): SomethingSoupInfo.__init__(self, - 'crew/info.wm?crewid=', crewid) + 'crew/info.wm?crewid=', crewid, max_age) self._find_crew() def _find_crew(self): self.crew = [] - capts = self.soup.findAll('img', + capts = self._soup.findAll('img', src='/yoweb/images/crew-captain.png') if len(capts) != 1: self.msg('crew members: no. of captain images != 1') @@ -285,6 +291,47 @@ class CrewInfo(SomethingSoupInfo): def __str__(self): return `(self.crew, self.msgs)` +class StandingsTable: + def __init__(self, use_puzzles=puzzles): + self._puzzles = use_puzzles + self.s = '' + + def _pline(self, pirate, puzstrs): + self.s += '%-*s' % (max_pirate_namelen, pirate) + for v in puzstrs: + self.s += ' %-*.*s' % (5,5, v) + self.s += '\n' + + def _puzstr(self, pi, puzzle): + if not isinstance(puzzle,list): puzzle = [puzzle] + try: standing = max([pi.standings[p] for p in puzzle]) + except KeyError: return '?' + c1 = standingvals[standing][0] + if standing < 3: c1 = c1.lower() # 3 = Master + hashes = '*' * (standing / 2) + equals = '+' * (standing % 2) + return c1 + hashes + equals + + def headings(self): + def puzn_redact(name): + if isinstance(name,list): + return '/'.join( + ["%.2s" % puzn_redact(n) + for n in name]) + spc = name.find(' ') + if spc < 0: return name + return name[0:min(4,spc)] + name[spc+1:] + self._pline('', map(puzn_redact, self._puzzles)) + def literalline(self, line): + self.s += line + '\n' + def pirate(self, pirate): + pi = PirateInfo(pirate, 600) + puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles] + self._pline(pirate, puzstrs) + + def results(self): + return self.s + def do_pirate(pirates, bu): print '{' for pirate in pirates: @@ -292,7 +339,7 @@ def do_pirate(pirates, bu): print '%s: %s,' % (`pirate`, info) print '}' -def prep_crew_of(args, bu): +def prep_crew_of(args, bu, max_age=300): if len(args) != 1: bu('crew-of takes one pirate name') pi = PirateInfo(args[0]) return CrewInfo(pi.crew[0]) @@ -301,8 +348,16 @@ def do_crew_of(args, bu): ci = prep_crew_of(args, bu) print ci -#def do_dutytab_crew_of(pirates, badusage): -# if len(pirates) != 1: badusage('dutytab-crew-of takes one pirate name') +def do_standings_crew_of(args, bu): + ci = prep_crew_of(args, bu, 60) + tab = StandingsTable() + tab.headings() + for (rank, members) in ci.crew: + if not members: continue + tab.literalline('%s:' % rank) + for p in members: + tab.pirate(p) + print tab.results() def main(): global opts, fetcher @@ -312,7 +367,7 @@ def main(): actions: yoweb-scrape [--ocean OCEAN ...] pirate PIRATE yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE - yoweb-scrape [--ocean OCEAN ...] dutytab-crew-of PIRATE + yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE ''') ao = pa.add_option ao('-O','--ocean',dest='ocean', metavar='OCEAN', @@ -336,7 +391,8 @@ actions: except KeyError: pa.error('unknown mode "%s"' % mode) # fixed parameters - opts.max_age = 240 + opts.min_max_age = 60 + opts.expire_age = 3600 if opts.cache_dir.startswith('~/'): opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:] -- 2.30.2