X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~yarrgweb/git?p=ypp-sc-tools.db-test.git;a=blobdiff_plain;f=yoweb-scrape;h=840a6d913582b2aa6d32f783cecc756afce3bffb;hp=d2d194c213e9039b941a1465c464239654ca14ae;hb=790d9953b97d1e1f9e342832855e01a38a813b86;hpb=9cec9bede40ddd31329097b6952d7a72ec005bbc diff --git a/yoweb-scrape b/yoweb-scrape index d2d194c..840a6d9 100755 --- a/yoweb-scrape +++ b/yoweb-scrape @@ -1,4 +1,32 @@ #!/usr/bin/python +# This is part of ypp-sc-tools, a set of third-party tools for assisting +# players of Yohoho Puzzle Pirates. +# +# Copyright (C) 2009 Ian Jackson +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Yohoho and Puzzle Pirates are probably trademarks of Three Rings and +# are used without permission. This program is not endorsed or +# sponsored by Three Rings. + +copyright_info = ''' +yoweb-scrape is part of ypp-sc-tools Copyright (C) 2009 Ian Jackson +This program comes with ABSOLUTELY NO WARRANTY; this is free software, +and you are welcome to redistribute it under certain conditions. +For details, read the top of the yoweb-scrape file. +''' #---------- setup ---------- @@ -15,7 +43,9 @@ import re as regexp import random import curses import termios +import random from optparse import OptionParser +from StringIO import StringIO from BeautifulSoup import BeautifulSoup @@ -39,7 +69,7 @@ duty_puzzles = ([ 'Navigating', 'Battle Navigation' ] + core_duty_puzzles + [ 'Treasure Haul' ]) -standingvals = ('Able/Distinguished/Respected/Master'+ +standingvals = ('Able/Proficient/Distinguished/Respected/Master'+ '/Renowned/Grand-Master/Legendary/Ultimate').split('/') standing_limit = len(standingvals) @@ -113,12 +143,14 @@ class Fetcher: min_age = 1 need_wait = 0 for age in ages: - if age < min_age and age < 300: + if age < min_age and age <= 5: debug('Fetcher morewait min=%d age=%d' % (min_age, age)) need_wait = max(need_wait, min_age - age) min_age += 3 min_age *= 1.25 + if need_wait > 0: + need_wait += random.random() - 0.5 return need_wait def _rate_limit_cache_clean(self, now): @@ -348,23 +380,33 @@ class CrewInfo(SomethingSoupInfo): #---------- pretty-printer for tables of pirate puzzle standings ---------- class StandingsTable: - def __init__(self, use_puzzles=None, col_width=6): + def __init__(self, f, use_puzzles=None, col_width=6, gap_every=5): if use_puzzles is None: if opts.ship_duty: use_puzzles=duty_puzzles else: use_puzzles=puzzles self._puzzles = use_puzzles - self.s = '' + self.f = f self._cw = col_width-1 - - def _pline(self, pirate, puzstrs, extra): - self.s += ' %-*s' % (max(max_pirate_namelen, 14), pirate) + self._gap_every = gap_every + self._linecount = 0 + self._o = f.write + + def _nl(self): self._o('\n') + + def _pline(self, lhs, puzstrs, extra): + if (self._linecount > 0 + and self._gap_every is not None + and not (self._linecount % self._gap_every)): + self._nl() + self._o('%-*s' % (max(max_pirate_namelen+1, 15), lhs)) for v in puzstrs: - self.s += ' %-*.*s' % (self._cw,self._cw, v) + self._o(' %-*.*s' % (self._cw,self._cw, v)) if extra: - self.s += ' ' + extra - self.s += '\n' + self._o(' ' + extra) + self._nl() + self._linecount += 1 def _puzstr(self, pi, puzzle): if not isinstance(puzzle,list): puzzle = [puzzle] @@ -382,7 +424,7 @@ class StandingsTable: s += '+' * (standing % 2) return s - def headings(self): + def headings(self, lhs='', rhs=None): def puzn_redact(name): if isinstance(name,list): return '/'.join( @@ -391,23 +433,26 @@ class StandingsTable: spc = name.find(' ') if spc < 0: return name return name[0:min(4,spc)] + name[spc+1:] - self._pline('', map(puzn_redact, self._puzzles), None) + self._linecount = -2 + self._pline(lhs, map(puzn_redact, self._puzzles), rhs) + self._linecount = 0 def literalline(self, line): - self.s += line + '\n' + self._o(line) + self._nl() + self._linecount = 0 def pirate_dummy(self, name, standingstring, extra=None): - self._pline(name, standingstring * len(self._puzzles), extra) + standings = standingstring * len(self._puzzles) + self._pline(' '+name, standings, extra) def pirate(self, pi, extra=None): puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles] - self._pline(pi.name, puzstrs, extra) + self._pline(' '+pi.name, puzstrs, extra) - def results(self): - return self.s #---------- chat log parser ---------- class PirateAboard: # This is essentially a transparent, dumb, data class. - # pa.v + # pa.v may be None # pa.name # pa.last_time # pa.last_event @@ -416,6 +461,18 @@ class PirateAboard: # pa.last_chat_chan # pa.pi + # Also used for jobbing applicants: + # happens when expires (to "-") + # - disembark, leaves crew no + # aboard evidence of them being aboard no + # applied "has applied for the job" 120s, configurable + # ashore "has taken a job" 30min, configurable + # declined "declined the job offer" 30s, configurable + # invited "has been invited to job" 120s, configurable + # + # pa.jobber None, 'ashore', 'applied', 'invited', 'declined' + # pa.expires expiry time time + def __init__(pa, pn, v, time, event): pa.name = pn pa.v = v @@ -425,6 +482,8 @@ class PirateAboard: pa.last_chat_chan = None pa.gunner = False pa.pi = None + pa.jobber = None + pa.expires = None def pirate_info(pa): now = time.time() @@ -435,7 +494,7 @@ class PirateAboard: return pa.pi debug('PirateAboard refresh %d > %d %s' % ( age, guide, pa.name)) - imaginary = [2,6] + imaginary = [2,4] else: imaginary = [1] wait = fetcher.need_wait(now, imaginary) @@ -458,9 +517,15 @@ class ChatLogTracker: # self._v = self._vl[self._vessel] self._date = None self._myself = myself_pi - self._f = file(logfn) self._lbuf = '' - self._progress = [0, os.fstat(self._f.fileno()).st_size] + self._f = file(logfn) + flen = os.fstat(self._f.fileno()).st_size + max_backlog = 500000 + if flen > max_backlog: + startpos = flen - max_backlog + self._f.seek(startpos) + self._f.readline() + self._progress = [0, flen - self._f.tell()] self._disembark_myself() self._need_redisplay = False self._lastvessel = None @@ -474,22 +539,39 @@ class ChatLogTracker: self._need_redisplay = True def _vessel_updated(self, v, timestamp): + if v is None: return v['#lastinfo'] = timestamp self.force_redisplay() - def _onboard_event(self,v,timestamp,pirate,event): + def _onboard_event(self,v,timestamp,pirate,event,jobber=None): pa = self._pl.get(pirate, None) if pa is not None and pa.v is v: pa.last_time = timestamp pa.last_event = event else: - if pa is not None: del pa.v[pirate] + if pa is not None and pa.v is not None: + del pa.v[pirate] pa = PirateAboard(pirate, v, timestamp, event) self._pl[pirate] = pa - v[pirate] = pa + if v is not None: v[pirate] = pa + pa.jobber = jobber + + if jobber is None: timeout = None + else: timeout = getattr(opts, 'timeout_'+jobber) + if timeout is None: pa.expires = None + else: pa.expires = timestamp + timeout self._vessel_updated(v, timestamp) return pa + def _expire_jobbers(self, now): + for pa in self._pl.values(): + if pa.expires is None: continue + if pa.expires >= now: continue + v = pa.v + del self._pl[pa.name] + if v is not None: del v[pa.name] + self.force_redisplay() + def _trash_vessel(self, v): for pn in v: if pn.startswith('#'): continue @@ -592,9 +674,17 @@ class ChatLogTracker: def _debug_line_disposition(self,timestamp,l,m): debug('CLT %13s %-40s %s' % (timestamp,m,l)) + def _rm_crew_l(self,re,l): + m = regexp.match(re,l) + if m and m.group(2) == self._myself.crew[1]: + return m.group(1) + else: + return None + def chatline(self,l): rm = lambda re: regexp.match(re,l) d = lambda m: self._debug_line_disposition(timestamp,l,m) + rm_crew = lambda re: self._rm_crew_l(re,l) timestamp = None m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$') @@ -630,6 +720,13 @@ class ChatLogTracker: def ob1(did): ob_x(m.group(1), did); return d(did) def oba(did): return ob1('%s %s' % (did, m.group(2))) + def jb(pirate,jobber): + return self._onboard_event( + None, timestamp, pirate, + ("jobber %s" % jobber), + jobber=jobber + ) + def disembark(v, timestamp, pirate, event): self._onboard_event( v, timestamp, pirate, 'leaving '+event) @@ -672,7 +769,8 @@ class ChatLogTracker: def chat_core(speaker, chan): try: pa = self._pl[speaker] except KeyError: return 'mystery' - if pa.v is not self._v: return 'elsewhere' + if pa.v is not None and pa.v is not self._v: + return 'elsewhere' pa.last_chat_time = timestamp pa.last_chat_chan = chan self.force_redisplay() @@ -687,15 +785,17 @@ class ChatLogTracker: (cmdr, metacmd) = m.groups() metacmd = regexp.sub('\\s+', ' ', metacmd).strip() m2 = regexp.match( - '/([ad]) (?:([A-Za-z* ]+)\\s*:)?([A-Za-z ]+)$', - metacmd) + '/([adj]) (?:([A-Za-z* ]+)\\s*:)?([A-Za-z ]+)$', + metacmd) if not m2: return chat(chan) (cmd, pattern, targets) = m2.groups() dml = ['cmd', chan, cmd] if cmd == 'a': each = self._onboard_event - else: each = disembark + elif cmd == 'd': each = disembark + else: each = lambda *l: self._onboard_event(*l, + **{'jobber':'applied'}) if cmdr == self._myself.name: dml.append('self') @@ -704,10 +804,16 @@ class ChatLogTracker: dml.append('other') how = 'cmd: %s %s' % (cmd,cmdr) - v = self._find_matching_vessel( - pattern, timestamp, cmdr, dml, create=True) + if cmd == 'j': + if pattern is not None: + return chat(chan) + v = None + else: + v = self._find_matching_vessel( + pattern, timestamp, cmdr, + dml, create=True) - if v is not None: + if cmd == 'j' or v is not None: targets = targets.strip().split(' ') dml.append(`len(targets)`) for target in targets: @@ -765,6 +871,23 @@ class ChatLogTracker: ' to the rest of the ship\\.') if m: return ob1('boarder unrepelled'); + pirate = rm_crew("(\\w+) has taken a job with '(.*)'\\.") + if pirate: return jb(pirate, 'ashore') + + pirate = rm_crew("(\\w+) has left '(.*)'\\.") + if pirate: + disembark(self._v, timestamp, pirate, 'left crew') + return d('left crew') + + m = rm('(\w+) has applied for the posted job\.') + if m: return jb(m.group(1), 'applied') + + pirate= rm_crew("(\\w+) has been invited to job for '(.*)'\\.") + if pirate: return jb(pirate, 'invited') + + pirate = rm_crew("(\\w+) declined the job offer for '(.*)'\\.") + if pirate: return jb(pirate, 'declined') + m = rm('(\\w+) has left the vessel\.') if m: pirate = m.group(1) @@ -773,6 +896,19 @@ class ChatLogTracker: return d('not-matched') + def _str_pa(self, pn, pa): + assert self._pl[pn] == pa + s = ' '*20 + "%s %-*s %13s %-30s %13s %-20s %13s" % ( + (' ','G')[pa.gunner], + max_pirate_namelen, pn, + pa.last_time, pa.last_event, + pa.last_chat_time, pa.last_chat_chan, + pa.jobber) + if pa.expires is not None: + s += " %-5d" % (pa.expires - pa.last_time) + s += "\n" + return s + def _str_vessel(self, vn, v): s = ' vessel %s\n' % vn s += ' '*20 + "%-*s %13s\n" % ( @@ -783,12 +919,7 @@ class ChatLogTracker: if pn.startswith('#'): continue pa = v[pn] assert pa.v == v - assert self._pl[pn] == pa - s += ' '*20 + "%s %-*s %13s %-30s %13s %s\n" % ( - (' ','G')[pa.gunner], - max_pirate_namelen, pn, - pa.last_time, pa.last_event, - pa.last_chat_time, pa.last_chat_chan) + s += self._str_pa(pn,pa) return s def __str__(self): @@ -803,10 +934,14 @@ class ChatLogTracker: for vn in sorted(self._vl.keys()): if vn == self._vessel: continue s += self._str_vessel(vn, self._vl[vn]) + s += " elsewhere\n" for p in self._pl: pa = self._pl[p] - assert pa.v[p] is pa - assert pa.v in self._vl.values() + if pa.v is not None: + assert pa.v[p] is pa + assert pa.v in self._vl.values() + else: + s += self._str_pa(pa.name, pa) s += '>\n' return s @@ -824,6 +959,8 @@ class ChatLogTracker: self._lbuf = '' if opts.debug >= 2: debug(self.__str__()) + self._expire_jobbers(time.time()) + if progress: progress.caughtup() def changed(self): @@ -852,6 +989,17 @@ class ChatLogTracker: return [ v[pn] for pn in sorted(v.keys()) if not pn.startswith('#') ] + def jobbers(self): + # returns a the jobbers' PirateAboards, + # sorted by jobber class and reverse of expiry time + l = [ pa + for pa in self._pl.values() + if pa.jobber is not None + ] + def compar_key(pa): + return (pa.jobber, -pa.expires) + l.sort(key = compar_key) + return l #---------- implementations of actual operation modes ---------- @@ -874,15 +1022,15 @@ def do_crew_of(args, bu): def do_standings_crew_of(args, bu): ci = prep_crew_of(args, bu, 60) - tab = StandingsTable() + tab = StandingsTable(sys.stdout) tab.headings() for (rank, members) in ci.crew: if not members: continue + tab.literalline('') tab.literalline('%s:' % rank) for p in members: pi = PirateInfo(p, random.randint(900,1800)) tab.pirate(pi) - print tab.results() class ProgressPrintPercentage: def __init__(self, f=sys.stdout): @@ -930,7 +1078,7 @@ def do_track_chat_log(args, bu): track.catchup() if track.changed(): print track - sleep(1) + sleep(0.5 + 0.5 * random.random()) #----- ship management aid ----- @@ -1017,12 +1165,15 @@ def do_ship_aid(args, bu): class KeyBasedSorter: def compar_key_pa(self, pa): - return self.compar_key(pa.pirate_info()) + pi = pa.pirate_info() + if pi is None: return None + return self.compar_key(pi) def lsort_pa(self, l): l.sort(key = self.compar_key_pa) class NameSorter(KeyBasedSorter): def compar_key(self, pi): return pi.name + def desc(self): return 'name' class SkillSorter(NameSorter): def __init__(self, relevant): @@ -1032,7 +1183,10 @@ class SkillSorter(NameSorter): if isinstance(p,basestring): self._avoid.add(p) else: self._avoid |= set(p) self._avoid -= self._want + self._desc = '%s' % relevant + def desc(self): return self._desc + def compar_key(self, pi): best_want = max([ pi.standings.get(puz,-1) @@ -1077,14 +1231,27 @@ def ship_aid_core(myself, track, displayer, kreader): s += kreader.info() s += '\n' - tbl = StandingsTable() - tbl.headings() + tbl_s = StringIO() + tbl = StandingsTable(tbl_s) aboard = track.aboard(vn) - sort.lsort_pa(aboard) - for pa in aboard: + jobbers = track.jobbers() + + if track.vesselname(): howmany = 'aboard: %2d' % len(aboard) + else: howmany = '' + + tbl.headings(howmany, ' sorted by '+sort.desc()) + + last_jobber = None + + for pa in aboard + jobbers: + if pa.jobber != last_jobber: + last_jobber = pa.jobber + tbl.literalline('') + tbl.literalline('jobbers '+last_jobber) + pi = pa.pirate_info() xs = '' @@ -1098,8 +1265,9 @@ def ship_aid_core(myself, track, displayer, kreader): else: tbl.pirate(pi, xs) - s += tbl.results() + s += tbl_s.getvalue() displayer.show(s) + tbl_s.close() k = kreader.getch() if k is None: @@ -1161,8 +1329,7 @@ actions: display modes (for --display) apply to ship-aid: --display=dumb just print new information, scrolling the screen - --display=overwrite use cursor motion, selective clear, etc. to redraw at top -''') + --display=overwrite use cursor motion, selective clear, etc. to redraw at top''') ao = pa.add_option ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None, help='select ocean OCEAN') @@ -1179,15 +1346,28 @@ display modes (for --display) apply to ship-aid: type='choice', choices=['dumb','overwrite'], help='how to display ship aid') + ao_jt = lambda wh, t: ao( + '--timeout-sa-'+wh, action='store', dest='timeout_'+wh, + default=t, help=('set timeout for expiring %s jobbers' % wh)) + ao_jt('applied', 120) + ao_jt('invited', 120) + ao_jt('declined', 30) + ao_jt('ashore', 1800) + ao('--ship-duty', action='store_true', dest='ship_duty', help='show ship duty station puzzles') ao('--all-puzzles', action='store_false', dest='ship_duty', help='show all puzzles, not just ship duty stations') + ao('--min-cache-reuse', type='int', dest='min_max_age', + metavar='SECONDS', default=60, + help='always reuse cache yoweb data if no older than this') + (opts,args) = pa.parse_args() random.seed() if len(args) < 1: + print >>sys.stderr, copyright_info pa.error('need a mode argument') if opts.debug_fd is not None: @@ -1201,8 +1381,8 @@ display modes (for --display) apply to ship-aid: except KeyError: pa.error('unknown mode "%s"' % mode) # fixed parameters - opts.min_max_age = 60 - opts.expire_age = 3600 + opts.expire_age = max(3600, opts.min_max_age) + opts.ship_reboard_clearout = 3600 if opts.cache_dir.startswith('~/'):