chiark / gitweb /
Allow downloading of database tree source code via dictup CGI scripts
[ypp-sc-tools.db-test.git] / yoweb-scrape
index c3e6d73..8f0bf56 100755 (executable)
@@ -1,4 +1,32 @@
 #!/usr/bin/python
+# This is part of ypp-sc-tools, a set of third-party tools for assisting
+# players of Yohoho Puzzle Pirates.
+#
+# Copyright (C) 2009 Ian Jackson <ijackson@chiark.greenend.org.uk>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Yohoho and Puzzle Pirates are probably trademarks of Three Rings and
+# are used without permission.  This program is not endorsed or
+# sponsored by Three Rings.
+
+copyright_info = '''
+yoweb-scrape is part of ypp-sc-tools  Copyright (C) 2009 Ian Jackson
+This program comes with ABSOLUTELY NO WARRANTY; this is free software,
+and you are welcome to redistribute it under certain conditions.
+For details, read the top of the yoweb-scrape file.
+'''
 
 #---------- setup ----------
 
@@ -14,7 +42,9 @@ import sys
 import re as regexp
 import random
 import curses
+import termios
 from optparse import OptionParser
+from StringIO import StringIO
 
 from BeautifulSoup import BeautifulSoup
 
@@ -27,8 +57,20 @@ puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
        '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
        '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
 
+core_duty_puzzles = [
+               'Gunning',
+               ['Sailing','Rigging'],
+               'Bilging',
+               'Carpentry',
+               ]
+
+duty_puzzles = ([ 'Navigating', 'Battle Navigation' ] +
+               core_duty_puzzles +
+               [ 'Treasure Haul' ])
+
 standingvals = ('Able/Distinguished/Respected/Master'+
                '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
+standing_limit = len(standingvals)
 
 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
 
@@ -41,9 +83,12 @@ def debug(m):
        if opts.debug > 0:
                print >>opts.debug_file, m
 
-def sleep(seconds):
+def debug_flush():
        if opts.debug > 0:
-               opts.debug_file.flush()
+               opts.debug_file.flush() 
+
+def sleep(seconds):
+       debug_flush()
        time.sleep(seconds)
 
 def format_time_interval(ti):
@@ -89,8 +134,9 @@ class Fetcher:
                        ages.append(age)
                return ages
 
-       def need_wait(self, now):
+       def need_wait(self, now, imaginary=[]):
                ages = self._cache_scan(now)
+               ages += imaginary
                ages.sort()
                debug('Fetcher   ages ' + `ages`)
                min_age = 1
@@ -247,7 +293,7 @@ u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-
                                skl.msg('puzzle "%s" no standing found' % puzzle)
                                continue
                        standing = sl[0]
-                       for i in range(0, len(standingvals)-1):
+                       for i in range(0, standing_limit):
                                if standing == standingvals[i]:
                                        self.standings[puzzle] = i
                        if not puzzle in self.standings:
@@ -331,30 +377,33 @@ class CrewInfo(SomethingSoupInfo):
 #---------- pretty-printer for tables of pirate puzzle standings ----------
 
 class StandingsTable:
-       def __init__(self, use_puzzles=None, col_width=6):
+       def __init__(self, f, use_puzzles=None, col_width=6, gap_every=5):
                if use_puzzles is None:
                        if opts.ship_duty:
-                               use_puzzles=[
-                                       'Navigating','Battle Navigation',
-                                       'Gunning',
-                                       ['Sailing','Rigging'],
-                                       'Bilging',
-                                       'Carpentry',
-                                       'Treasure Haul'
-                               ]
+                               use_puzzles=duty_puzzles
                        else:
                                use_puzzles=puzzles
                self._puzzles = use_puzzles
-               self.s = ''
+               self.f = f
                self._cw = col_width-1
+               self._gap_every = gap_every
+               self._linecount = 0
+               self._o = f.write
+
+       def _nl(self): self._o('\n')
 
        def _pline(self, pirate, puzstrs, extra):
-               self.s += ' %-*s' % (max(max_pirate_namelen, 14), pirate)
+               if (self._linecount > 0
+                   and self._gap_every is not None
+                   and not (self._linecount % self._gap_every)):
+                       self._nl()
+               self._o(' %-*s' % (max(max_pirate_namelen, 14), pirate))
                for v in puzstrs:
-                       self.s += ' %-*.*s' % (self._cw,self._cw, v)
+                       self._o(' %-*.*s' % (self._cw,self._cw, v))
                if extra:
-                       self.s += ' ' + extra
-               self.s += '\n'
+                       self._o(' ' + extra)
+               self._nl()
+               self._linecount += 1
 
        def _puzstr(self, pi, puzzle):
                if not isinstance(puzzle,list): puzzle = [puzzle]
@@ -372,7 +421,7 @@ class StandingsTable:
                s += '+' * (standing % 2)
                return s
 
-       def headings(self):
+       def headings(self, lhs='', rhs=None):
                def puzn_redact(name):
                        if isinstance(name,list):
                                return '/'.join(
@@ -381,17 +430,19 @@ class StandingsTable:
                        spc = name.find(' ')
                        if spc < 0: return name
                        return name[0:min(4,spc)] + name[spc+1:]
-               self._pline('', map(puzn_redact, self._puzzles), None)
+               self._linecount = -2
+               self._pline(lhs, map(puzn_redact, self._puzzles), rhs)
+               self._linecount = 0
        def literalline(self, line):
-               self.s += line + '\n'
+               self._o(line)
+               self._nl()
+               self._linecount = 0
        def pirate_dummy(self, name, standingstring, extra=None):
                self._pline(name, standingstring * len(self._puzzles), extra)
        def pirate(self, pi, extra=None):
                puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
                self._pline(pi.name, puzstrs, extra)
 
-       def results(self):
-               return self.s
 
 #---------- chat log parser ----------
 
@@ -417,8 +468,23 @@ class PirateAboard:
                pa.pi = None
 
        def pirate_info(pa):
-               if not pa.pi and not fetcher.need_wait(time.time()):
-                       pa.pi = PirateInfo(pa.name, 3600)
+               now = time.time()
+               if pa.pi:
+                       age = now - pa.pi_fetched
+                       guide = random.randint(120,240)
+                       if age <= guide:
+                               return pa.pi
+                       debug('PirateAboard refresh %d > %d  %s' % (
+                               age, guide, pa.name))
+                       imaginary = [2,6]
+               else:
+                       imaginary = [1]
+               wait = fetcher.need_wait(now, imaginary)
+               if wait:
+                       debug('PirateAboard fetcher not ready %d' % wait)
+                       return pa.pi
+               pa.pi = PirateInfo(pa.name, 600)
+               pa.pi_fetched = now
                return pa.pi
 
 class ChatLogTracker:
@@ -438,6 +504,7 @@ class ChatLogTracker:
                self._progress = [0, os.fstat(self._f.fileno()).st_size]
                self._disembark_myself()
                self._need_redisplay = False
+               self._lastvessel = None
 
        def _disembark_myself(self):
                self._v = None
@@ -520,7 +587,7 @@ class ChatLogTracker:
                if pattern is None:
                        pattern_check = lambda vn: True
                else:
-                       re = '(?:.* )?%s$' % pattern.lower().replace('*','.*')
+                       re = '(?:.* )?%s$' % pattern.lower().replace('*','.+')
                        pattern_check = regexp.compile(re, regexp.I).match
 
                tries = []
@@ -610,13 +677,17 @@ class ChatLogTracker:
                        del v[pirate]
                        del self._pl[pirate]
 
+               def disembark_me(why):
+                       self._disembark_myself()
+                       return d('disembark-me '+why)
+
                m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
                if m:
                        dm = ['boarding']
                        pn = self._myself.name
                        vn = m.group(1)
                        v = self._vessel_lookup(vn, timestamp, dm, create=True)
-                       self._vessel = vn
+                       self._lastvessel = self._vessel = vn
                        self._v = v
                        ob_x(pn, 'we boarded')
                        self.expire_garbage(timestamp)
@@ -706,19 +777,35 @@ class ChatLogTracker:
                m = rm('(\\w+) officer chats, "(.*)"$')
                if m: return chat_metacmd('officer')
 
+               m = rm('Ye accepted the offer to job with ')
+               if m: return disembark_me('jobbing')
+
+               m = rm('Ye hop on the ferry and are whisked away ')
+               if m: return disembark_me('ferry')
+
+               m = rm('Whisking away to yer home on the magical winds')
+               if m: return disembark_me('home')
+
                m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
                if m:
                        pl = m.group(1).split(', ')
                        if not self._myself.name in pl:
-                               return d('lost boarding battle')
+                               return d('lost melee')
                        for pn in pl:
                                if ' ' in pn: continue
-                               ob_x(pn,'won boarding battle')
-                       return d('won boarding battle')
+                               ob_x(pn,'won melee')
+                       return d('won melee')
 
                m = rm('(\\w+) is eliminated\\!')
                if m: return ob1('eliminated in fray');
 
+               m = rm('(\\w+) has driven \w+ from the ship\\!')
+               if m: return ob1('boarder repelled');
+
+               m = rm('\w+ has bested (\\w+), and turns'+
+                       ' to the rest of the ship\\.')
+               if m: return ob1('boarder unrepelled');
+
                m = rm('(\\w+) has left the vessel\.')
                if m:
                        pirate = m.group(1)
@@ -787,14 +874,24 @@ class ChatLogTracker:
        def myname(self):
                # returns our pirate name
                return self._myself.name
-       def vessel(self):
-               # returns the vessel we're aboard or None
+       def vesselname(self):
+               # returns the vessel name we're aboard or None
                return self._vessel
-       def aboard(self):
-               # returns a list of PirateAboard sorted by name
-               if self._v is None: return []
-               return [ self._v[pn]
-                        for pn in sorted(self._v.keys())
+       def lastvesselname(self):
+               # returns the last vessel name we were aboard or None
+               return self._lastvessel
+       def aboard(self, vesselname=True):
+               # returns a list of PirateAboard the vessel
+               #  sorted by pirate name
+               #  you can pass this None and you'll get []
+               #  or True for the current vessel (which is the default)
+               #  the returned value is a fresh list of persistent
+               #  PirateAboard objects
+               if vesselname is True: v = self._v
+               else: v = self._vl.get(vesselname.title())
+               if v is None: return []
+               return [ v[pn]
+                        for pn in sorted(v.keys())
                         if not pn.startswith('#') ]
 
 #---------- implementations of actual operation modes ----------
@@ -818,15 +915,15 @@ def do_crew_of(args, bu):
 
 def do_standings_crew_of(args, bu):
        ci = prep_crew_of(args, bu, 60)
-       tab = StandingsTable()
+       tab = StandingsTable(sys.stdout)
        tab.headings()
        for (rank, members) in ci.crew:
                if not members: continue
+               tab.literalline('')
                tab.literalline('%s:' % rank)
                for p in members:
                        pi = PirateInfo(p, random.randint(900,1800))
                        tab.pirate(pi)
-       print tab.results()
 
 class ProgressPrintPercentage:
        def __init__(self, f=sys.stdout):
@@ -855,9 +952,9 @@ def prep_chat_log(args, bu,
        if not match: bu('chat log filename is not in expected format')
        (pirate, ocean) = match.groups()
        fetcher.default_ocean(ocean)
-       
-       myself = PirateInfo(pirate,max_myself_age)
+
        progress.show_init(pirate, fetcher.ocean)
+       myself = PirateInfo(pirate,max_myself_age)
        track = ChatLogTracker(myself, logfn)
 
        opts.debug -= 2
@@ -944,31 +1041,101 @@ def do_ship_aid(args, bu):
        if opts.ship_duty is None: opts.ship_duty = True
 
        displayer = globals()['Display_'+opts.display]()
-       rotate_nya = '/-\\'
 
        (myself, track) = prep_chat_log(args, bu, progress=displayer)
 
+       displayer.realstart()
+
+       if os.isatty(0): kr_create = KeystrokeReader
+       else: kr_create = DummyKeystrokeReader
+
+       try:
+               kreader = kr_create(0, 10)
+               ship_aid_core(myself, track, displayer, kreader)
+       finally:
+               kreader.stop()
+               print '\n'
+
+class KeyBasedSorter:
+       def compar_key_pa(self, pa):
+               pi = pa.pirate_info()
+               if pi is None: return None
+               return self.compar_key(pi)
+       def lsort_pa(self, l):
+               l.sort(key = self.compar_key_pa)
+
+class NameSorter(KeyBasedSorter):
+       def compar_key(self, pi): return pi.name
+       def desc(self): return 'name'
+
+class SkillSorter(NameSorter):
+       def __init__(self, relevant):
+               self._want = frozenset(relevant.split('/'))
+               self._avoid = set()
+               for p in core_duty_puzzles:
+                       if isinstance(p,basestring): self._avoid.add(p)
+                       else: self._avoid |= set(p)
+               self._avoid -= self._want
+               self._desc = '%s' % relevant
+       
+       def desc(self): return self._desc
+
+       def compar_key(self, pi):
+               best_want = max([
+                       pi.standings.get(puz,-1)
+                       for puz in self._want
+                       ])
+               best_avoid = [
+                       -pi.standings.get(puz,standing_limit)
+                       for puz in self._avoid
+                       ]
+               best_avoid.sort()
+               def negate(x): return -x
+               debug('compar_key %s bw=%s ba=%s' % (pi.name, `best_want`,
+                       `best_avoid`))
+               return (-best_want, map(negate, best_avoid), pi.name)
+
+def ship_aid_core(myself, track, displayer, kreader):
+
+       def find_vessel():
+               vn = track.vesselname()
+               if vn: return (vn, " on board the %s" % vn)
+               vn = track.lastvesselname()
+               if vn: return (vn, " ashore from the %s" % vn)
+               return (None, " not on a vessel")
+
        def timeevent(t,e):
                if t is None: return ' ' * 22
                return " %-4s %-16s" % (format_time_interval(now - t),e)
 
-       displayer.realstart()
+       displayer.show(track.myname() + find_vessel()[1] + '...')
+
+       rotate_nya = '/-\\'
+
+       sort = NameSorter()
 
        while True:
                track.catchup()
                now = time.time()
 
-               s = "%s" % track.myname()
+               (vn, s) = find_vessel()
+               s = track.myname() + s
+               s += " at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
+               s += kreader.info()
+               s += '\n'
+
+               aboard = track.aboard(vn)
+               sort.lsort_pa(aboard)
+
+               tbl_s = StringIO()
+               tbl = StandingsTable(tbl_s)
 
-               vn = track.vessel()
-               if vn is None: s += " not on a vessel?!"
-               else: s += " on board the %s" % vn
-               s += " at %s\n" % time.strftime("%Y-%m-%d %H:%M:%S")
+               if track.vesselname(): howmany = ' %d aboard' % len(aboard)
+               else: howmany = ''
 
-               tbl = StandingsTable()
-               tbl.headings()
+               tbl.headings(howmany, '  sorted by '+sort.desc())
 
-               for pa in track.aboard():
+               for pa in aboard:
                        pi = pa.pirate_info()
 
                        xs = ''
@@ -982,11 +1149,53 @@ def do_ship_aid(args, bu):
                        else:
                                tbl.pirate(pi, xs)
 
-               s += tbl.results()
-
+               s += tbl_s.getvalue()
                displayer.show(s)
-               sleep(1)
-               rotate_nya = rotate_nya[1:3] + rotate_nya[0]
+               tbl_s.close()
+
+               k = kreader.getch()
+               if k is None:
+                       rotate_nya = rotate_nya[1:3] + rotate_nya[0]
+                       continue
+
+               if k == 'q': break
+               elif k == 'g': sort = SkillSorter('Gunning')
+               elif k == 'c': sort = SkillSorter('Carpentry')
+               elif k == 's': sort = SkillSorter('Sailing/Rigging')
+               elif k == 'b': sort = SkillSorter('Bilging')
+               elif k == 'n': sort = SkillSorter('Navigating')
+               elif k == 'd': sort = SkillSorter('Battle Navigation')
+               elif k == 't': sort = SkillSorter('Treasure Haul')
+               elif k == 'a': sort = NameSorter()
+               else: pass # unknown key command
+
+#---------- individual keystroke input ----------
+
+class DummyKeystrokeReader:
+       def __init__(self,fd,timeout_dummy): pass
+       def stop(self): pass
+       def getch(self): sleep(1); return None
+       def info(self): return ' [noninteractive]'
+
+class KeystrokeReader(DummyKeystrokeReader):
+       def __init__(self, fd, timeout_decisec=0):
+               self._fd = fd
+               self._saved = termios.tcgetattr(fd)
+               a = termios.tcgetattr(fd)
+               a[3] &= ~(termios.ECHO | termios.ECHONL |
+                         termios.ICANON | termios.IEXTEN)
+               a[6][termios.VMIN] = 0
+               a[6][termios.VTIME] = timeout_decisec
+               termios.tcsetattr(fd, termios.TCSANOW, a)
+       def stop(self):
+               termios.tcsetattr(self._fd, termios.TCSANOW, self._saved)
+       def getch(self):
+               debug_flush()
+               byte = os.read(self._fd, 1)
+               if not len(byte): return None
+               return byte
+       def info(self):
+               return ''
 
 #---------- main program ----------
 
@@ -1004,8 +1213,7 @@ actions:
 
 display modes (for --display) apply to ship-aid:
  --display=dumb       just print new information, scrolling the screen
- --display=overwrite  use cursor motion, selective clear, etc. to redraw at top
-''')
+ --display=overwrite  use cursor motion, selective clear, etc. to redraw at top''')
        ao = pa.add_option
        ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
                help='select ocean OCEAN')
@@ -1027,10 +1235,15 @@ display modes (for --display) apply to ship-aid:
        ao('--all-puzzles', action='store_false', dest='ship_duty',
                help='show all puzzles, not just ship duty stations')
 
+       ao('--min-cache-reuse', type='int', dest='min_max_age',
+               metavar='SECONDS', default=60,
+               help='always reuse cache yoweb data if no older than this')
+
        (opts,args) = pa.parse_args()
        random.seed()
 
        if len(args) < 1:
+               print >>sys.stderr, copyright_info
                pa.error('need a mode argument')
 
        if opts.debug_fd is not None:
@@ -1044,8 +1257,8 @@ display modes (for --display) apply to ship-aid:
        except KeyError: pa.error('unknown mode "%s"' % mode)
 
        # fixed parameters
-       opts.min_max_age = 60
-       opts.expire_age = 3600
+       opts.expire_age = max(3600, opts.min_max_age)
+
        opts.ship_reboard_clearout = 3600
 
        if opts.cache_dir.startswith('~/'):