chiark / gitweb /
duty table
authorIan Jackson <ian@liberator.relativity.greenend.org.uk>
Fri, 15 May 2009 18:24:38 +0000 (19:24 +0100)
committerIan Jackson <ian@liberator.relativity.greenend.org.uk>
Fri, 15 May 2009 18:24:38 +0000 (19:24 +0100)
yoweb-scrape

index 54ef0fe..493ef53 100755 (executable)
@@ -16,16 +16,20 @@ from BeautifulSoup import BeautifulSoup
 
 opts = None
 
-duties = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
+
+puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
        '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
        '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
        '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
 
-standingvals = ('Able/Distinguished/Respected/Master/Renowned'+
-               '/Grand-Master/Legendary/Ultimate').split('/')
+standingvals = ('Able/Distinguished/Respected/Master'+
+               '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
 
 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
 
+max_pirate_namelen = 20
+
+
 def debug(m):
        if opts.debug:
                print >>sys.stderr, m
@@ -52,7 +56,7 @@ class Fetcher:
                                if oe.errno != errno.ENOENT: raise
                                continue
                        age = now - s.st_mtime
-                       if age > opts.max_age:
+                       if age > opts.expire_age:
                                debug('Fetcher    expire %d %s' % (age, path))
                                try: os.remove(path)
                                except (OSError,IOError), oe:
@@ -72,13 +76,13 @@ class Fetcher:
                                debug('Fetcher   morewait min=%d age=%d' %
                                        (min_age, age))
                                need_wait = max(need_wait, min_age - age)
-                       min_age *= 2
                        min_age += 1
+                       min_age *= 1.5
                if need_wait > 0:
                        debug('Fetcher   wait %d' % need_wait)
                        time.sleep(need_wait)
 
-       def fetch(self, url):
+       def fetch(self, url, max_age):
                debug('Fetcher fetch %s' % url)
                cache_corename = urllib.quote_plus(url)
                cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
@@ -87,15 +91,17 @@ class Fetcher:
                        if oe.errno != errno.ENOENT: raise
                        f = None
                now = time.time()
+               max_age = max(opts.min_max_age, max(max_age, opts.expire_age))
                if f is not None:
                        s = os.fstat(f.fileno())
-                       if now > s.st_mtime + opts.max_age:
+                       age = now - s.st_mtime
+                       if age > max_age:
                                debug('Fetcher  stale')
                                f = None
                if f is not None:
                        data = f.read()
                        f.close()
-                       debug('Fetcher  cached')
+                       debug('Fetcher  cached %d > %d' % (max_age, age))
                        return data
 
                debug('Fetcher  fetch')
@@ -112,10 +118,10 @@ class Fetcher:
                debug('Fetcher  stored')
                return data
 
-       def yoweb(self, kind, tail):
+       def yoweb(self, kind, tail, max_age):
                url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
                        self.ocean, kind, tail)
-               return self.fetch(url)
+               return self.fetch(url, max_age)
 
 class SoupLog:
        def __init__(self):
@@ -133,10 +139,10 @@ def soup_text(obj):
        return str.strip()
 
 class SomethingSoupInfo(SoupLog):
-       def __init__(self, kind, tail):
+       def __init__(self, kind, tail, max_age):
                SoupLog.__init__(self)
-               html = fetcher.yoweb(kind, tail)
-               self.soup = BeautifulSoup(html,
+               html = fetcher.yoweb(kind, tail, max_age)
+               self._soup = BeautifulSoup(html,
                        convertEntities=BeautifulSoup.HTML_ENTITIES
                        )
 
@@ -147,9 +153,9 @@ class PirateInfo(SomethingSoupInfo):
        #  pi.flag = (id, name)
        #  pi.msgs = [ 'message describing problem with scrape' ]
                
-       def __init__(self, pirate):
+       def __init__(self, pirate, max_age=300):
                SomethingSoupInfo.__init__(self,
-                       'pirate.wm?target=', pirate)
+                       'pirate.wm?target=', pirate, max_age)
                self._find_standings()
                self.crew = self._find_crewflag('crew',
                        '^/yoweb/crew/info\\.wm')
@@ -157,69 +163,69 @@ class PirateInfo(SomethingSoupInfo):
                        '^/yoweb/flag/info\\.wm')
 
        def _find_standings(self):
-               imgs = self.soup.findAll('img',
+               imgs = self._soup.findAll('img',
                        src=regexp.compile('/yoweb/images/stat.*'))
                re = regexp.compile(
 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
                        )
                standings = { }
 
-               for skill in duties:
+               for skill in puzzles:
                        standings[skill] = [ ]
 
                skl = SoupLog()
 
                for img in imgs:
-                       try: duty = img['alt']
+                       try: puzzle = img['alt']
                        except KeyError: continue
 
-                       if not duty in duties:
-                               skl.soupm(img, 'unknown duty: "%s"' % duty)
+                       if not puzzle in puzzles:
+                               skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
                                continue
                        key = img.findParent('td')
                        if key is None:
-                               skl.soupm(img, 'duty at root! "%s"' % duty)
+                               skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
                                continue
                        valelem = key.findNextSibling('td')
                        if valelem is None:
-                               skl.soupm(key, 'duty missing sibling "%s"'
-                                       % duty)
+                               skl.soupm(key, 'puzzle missing sibling "%s"'
+                                       % puzzle)
                                continue
                        valstr = soup_text(valelem)
                        match = re.match(valstr)
                        if match is None:
-                               skl.soupm(key, ('duty "%s" unparseable'+
-                                       ' standing "%s"') % (duty, valstr))
+                               skl.soupm(key, ('puzzle "%s" unparseable'+
+                                       ' standing "%s"') % (puzzle, valstr))
                                continue
                        standing = match.group(match.lastindex)
-                       standings[duty].append(standing)
+                       standings[puzzle].append(standing)
 
                self.standings = { }
 
-               for duty in duties:
-                       sl = standings[duty]
+               for puzzle in puzzles:
+                       sl = standings[puzzle]
                        if len(sl) > 1:
-                               skl.msg('duty "%s" multiple standings %s' %
-                                               (duty, `sl`))
+                               skl.msg('puzzle "%s" multiple standings %s' %
+                                               (puzzle, `sl`))
                                continue
                        if not len(sl):
-                               skl.msg('duty "%s" no standing found' % duty)
+                               skl.msg('puzzle "%s" no standing found' % puzzle)
                                continue
                        standing = sl[0]
                        for i in range(0, len(standingvals)-1):
                                if standing == standingvals[i]:
-                                       self.standings[duty] = i
-                       if not duty in self.standings:
-                               skl.msg('duty "%s" unknown standing "%s"' %
-                                       (duty, standing))
+                                       self.standings[puzzle] = i
+                       if not puzzle in self.standings:
+                               skl.msg('puzzle "%s" unknown standing "%s"' %
+                                       (puzzle, standing))
 
                all_standings_ok = True
-               for duty in duties:
-                       if not duty in self.standings:
+               for puzzle in puzzles:
+                       if not puzzle in self.standings:
                                self.needs_msgs(skl)
 
        def _find_crewflag(self, cf, yoweb_re):
-               things = self.soup.findAll('a', href=regexp.compile(yoweb_re))
+               things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
                if len(things) != 1:
                        self.msg('zero or several %s id references found' % cf)
                        return None
@@ -244,14 +250,14 @@ class CrewInfo(SomethingSoupInfo):
        #               ... ]
        #  pi.msgs = [ 'message describing problem with scrape' ]
 
-       def __init__(self, crewid):
+       def __init__(self, crewid, max_age=300):
                SomethingSoupInfo.__init__(self,
-                       'crew/info.wm?crewid=', crewid)
+                       'crew/info.wm?crewid=', crewid, max_age)
                self._find_crew()
 
        def _find_crew(self):
                self.crew = []
-               capts = self.soup.findAll('img',
+               capts = self._soup.findAll('img',
                        src='/yoweb/images/crew-captain.png')
                if len(capts) != 1:
                        self.msg('crew members: no. of captain images != 1')
@@ -285,6 +291,47 @@ class CrewInfo(SomethingSoupInfo):
        def __str__(self):
                return `(self.crew, self.msgs)`
 
+class StandingsTable:
+       def __init__(self, use_puzzles=puzzles):
+               self._puzzles = use_puzzles
+               self.s = ''
+
+       def _pline(self, pirate, puzstrs):
+               self.s += '%-*s' % (max_pirate_namelen, pirate)
+               for v in puzstrs:
+                       self.s += ' %-*.*s' % (5,5, v)
+               self.s += '\n'
+
+       def _puzstr(self, pi, puzzle):
+               if not isinstance(puzzle,list): puzzle = [puzzle]
+               try: standing = max([pi.standings[p] for p in puzzle])
+               except KeyError: return '?'
+               c1 = standingvals[standing][0]
+               if standing < 3: c1 = c1.lower() # 3 = Master
+               hashes = '*' * (standing / 2)
+               equals = '+' * (standing % 2)
+               return c1 + hashes + equals
+
+       def headings(self):
+               def puzn_redact(name):
+                       if isinstance(name,list):
+                               return '/'.join(
+                                       ["%.2s" % puzn_redact(n)
+                                        for n in name])
+                       spc = name.find(' ')
+                       if spc < 0: return name
+                       return name[0:min(4,spc)] + name[spc+1:]
+               self._pline('', map(puzn_redact, self._puzzles))
+       def literalline(self, line):
+               self.s += line + '\n'
+       def pirate(self, pirate):
+               pi = PirateInfo(pirate, 600)
+               puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
+               self._pline(pirate, puzstrs)
+
+       def results(self):
+               return self.s
+
 def do_pirate(pirates, bu):
        print '{'
        for pirate in pirates:
@@ -292,7 +339,7 @@ def do_pirate(pirates, bu):
                print '%s: %s,' % (`pirate`, info)
        print '}'
 
-def prep_crew_of(args, bu):
+def prep_crew_of(args, bu, max_age=300):
        if len(args) != 1: bu('crew-of takes one pirate name')
        pi = PirateInfo(args[0])
        return CrewInfo(pi.crew[0])
@@ -301,8 +348,16 @@ def do_crew_of(args, bu):
        ci = prep_crew_of(args, bu)
        print ci
 
-#def do_dutytab_crew_of(pirates, badusage):
-#      if len(pirates) != 1: badusage('dutytab-crew-of takes one pirate name')
+def do_standings_crew_of(args, bu):
+       ci = prep_crew_of(args, bu, 60)
+       tab = StandingsTable()
+       tab.headings()
+       for (rank, members) in ci.crew:
+               if not members: continue
+               tab.literalline('%s:' % rank)
+               for p in members:
+                       tab.pirate(p)
+       print tab.results()
 
 def main():
        global opts, fetcher
@@ -312,7 +367,7 @@ def main():
 actions:
  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
- yoweb-scrape [--ocean OCEAN ...] dutytab-crew-of PIRATE
+ yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
 ''')
        ao = pa.add_option
        ao('-O','--ocean',dest='ocean', metavar='OCEAN',
@@ -336,7 +391,8 @@ actions:
        except KeyError: pa.error('unknown mode "%s"' % mode)
 
        # fixed parameters
-       opts.max_age = 240
+       opts.min_max_age = 60
+       opts.expire_age = 3600
 
        if opts.cache_dir.startswith('~/'):
                opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]