X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~yarrgweb/git?p=ypp-sc-tools.db-test.git;a=blobdiff_plain;f=yoweb-scrape;h=0ceeaa13c35226ee898c64011b9295845de2b520;hp=1eeb8a36e0dace67cd9bfb08d3e110a627c691ae;hb=4d3174a853d79e278d81345057e7cc89dd152048;hpb=964db588289ad72ff165ca569456a56213ba26cc diff --git a/yoweb-scrape b/yoweb-scrape index 1eeb8a3..0ceeaa1 100755 --- a/yoweb-scrape +++ b/yoweb-scrape @@ -39,7 +39,7 @@ max_pirate_namelen = 12 def debug(m): if opts.debug > 0: - print m + print >>opts.debug_file, m def format_time_interval(ti): if ti < 120: return '%d:%02d' % (ti / 60, ti % 60) @@ -423,7 +423,7 @@ class ChatLogTracker: def __init__(self, myself_pi, logfn): self._pl = {} # self._pl['Pirate'] = self._vl = {} # self._vl['Vessel']['Pirate'] = PirateAboard - # self._vl['Vessel']['#lastaboard'] + # self._vl['Vessel']['#lastinfo'] self._v = None # self._v = self._vessel = None # self._vl[self._vessel] self._date = None @@ -433,44 +433,54 @@ class ChatLogTracker: self._lbuf = '' self._progress = [0, os.fstat(self._f.fileno()).st_size] - def _refresh(self): + def force_redisplay(self): self._need_redisplay = True - def _onboard_event(self,timestamp,pirate,event): - try: pa = self._pl[pirate] - except KeyError: pa = None - if pa is not None and pa.v is self._v: + def _onboard_event(self,v,timestamp,pirate,event): + pa = self._pl.get(pirate, None) + if pa is not None and pa.v is v: pa.last_time = timestamp pa.last_event = event else: if pa is not None: del pa.v[pirate] - pa = PirateAboard(pirate, self._v, timestamp, event) + pa = PirateAboard(pirate, v, timestamp, event) self._pl[pirate] = pa - self._v[pirate] = pa - self._v['#lastaboard'] = timestamp - self._refresh() + v[pirate] = pa + v['#lastinfo'] = timestamp + self.force_redisplay() return pa def _trash_vessel(self, v): for pn in v: if pn.startswith('#'): continue del self._pl[pn] - self._refresh() + self.force_redisplay() def expire_garbage(self, timestamp): for (vn,v) in list(self._vl.iteritems()): - la = v['#lastaboard'] + la = v['#lastinfo'] if timestamp - la > opts.ship_reboard_clearout: self._debug_line_disposition(timestamp,'', 'stale reset '+vn) self._trash_vessel(v) del self._vl[vn] - def clear_vessel(self, timestamp): - if self._v is not None: - self._trash_vessel(self._v) - self._v = {'#lastaboard': timestamp} - self._vl[self._vessel] = self._v + def _create_vessel(self, vn, timestamp): + self._vl[vn] = v = { '#lastinfo': timestamp } + return v + + def _update_vessel_lookup(self, vn, timestamp, dml): + v = self._vl.get(vn, None) + if v is None: + dml.append('new') + v = self._create_vessel(vn, timestamp) + elif timestamp - v['#lastinfo'] > opts.ship_reboard_clearout: + dml.append('stale') + self._trash_vessel(v) + v = self._create_vessel(vn, timestamp) + else: + dml.append('current') + return v def _debug_line_disposition(self,timestamp,l,m): debug('CLT %13s %-30s %s' % (timestamp,m,l)) @@ -482,7 +492,8 @@ class ChatLogTracker: m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$') if m: - self._date = m.groups() + self._date = [int(x) for x in m.groups()] + self._previous_timestamp = None return d('date '+`self._date`) if self._date is None: @@ -492,35 +503,42 @@ class ChatLogTracker: if not m: return d('no timestamp') - time_tuple = [int(x) for x in self._date + m.groups()] - time_tuple += (-1,-1,-1) - timestamp = time.mktime(time_tuple) + while True: + time_tuple = (self._date + + [int(x) for x in m.groups()] + + [-1,-1,-1]) + timestamp = time.mktime(time_tuple) + if timestamp >= self._previous_timestamp: break + self._date[2] += 1 + self._debug_line_disposition(timestamp,'', + 'new date '+`self._date`) + + self._previous_timestamp = timestamp + l = l[l.find(' ')+1:] - def ob_x(who,event): - return self._onboard_event(timestamp, who, event) + def ob_x(pirate,event): + return self._onboard_event( + self._v, timestamp, pirate, event) def ob1(did): ob_x(m.group(1), did); return d(did) def oba(did): return ob1('%s %s' % (did, m.group(2))) + def disembark(v, timestamp, pirate, event): + self._onboard_event( + v, timestamp, pirate, 'leaving '+event) + del v[pirate] + del self._pl[pirate] + m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$') if m: + dm = ['boarding'] pn = self._myself.name - self._vessel = m.group(1) - dm = 'boarding' - - try: self._v = self._vl[self._vessel] - except KeyError: self._v = None; dm += ' new' - - if self._v is not None: la = self._v['#lastaboard'] - else: la = 0; dm += ' ?la' - - if timestamp - la > opts.ship_reboard_clearout: - self.clear_vessel(timestamp) - dm += ' stale' + self._vessel = vn = m.group(1) + self._v = self._update_vessel_lookup(vn, timestamp, dm) ob_x(pn, 'we boarded') self.expire_garbage(timestamp) - return d(dm) + return d(' '.join(dm)) if self._v is None: return d('no vessel') @@ -546,8 +564,36 @@ class ChatLogTracker: if pa.v is self._v: pa.last_chat_time = timestamp pa.last_chat_chan = what - self._refresh() - return d(what+' chat') + self.force_redisplay() + return d('chat '+what) + + def chat_metacmd(what): + (cmdr, metacmd) = m.groups() + metacmd = regexp.sub('\\s+', ' ', metacmd).strip() + m2 = regexp.match( + '/([ad]) (?:([A-Za-z ]+)\\s*:)?([A-Za-z ]+)$', + metacmd) + if not m2: return chat(what) + + (cmd, vn, targets) = m2.groups() + + if cmdr == self._myself.name: how = 'manual: /%s' % cmd + else: how = '/%s %s' % (cmd,cmdr) + if cmd == 'a': each = ob_x + else: each = lambda who,how: disembark( + self._v, timestamp, who, how) + + if vn is not None: + vn = vn.title() + if not regexp.match( + '(?:.* )?' + vn + '$', + self._vessel): + return chat('/%s %s:' % (cmd,vn)) + + for target in targets.split(' '): + if not target: continue + each(target.title(), how) + return d('/%s' % cmd) m = rm('(\\w+) (?:issued an order|ordered everyone) "') if m: return ob1('general order'); @@ -558,11 +604,14 @@ class ChatLogTracker: m = rm('(\\w+) tells ye, "') if m: return chat('private') + m = rm('Ye told (\\w+), "(.*)"$') + if m: return chat_metacmd('private') + m = rm('(\\w+) flag officer chats, "') if m: return chat('flag officer') - m = rm('(\\w+) officer chats, "') - if m: return chat('officer') + m = rm('(\\w+) officer chats, "(.*)"$') + if m: return chat_metacmd('officer') m = rm('Game over\\. Winners: ([A-Za-z, ]+)\\.$') if m: @@ -579,10 +628,8 @@ class ChatLogTracker: m = rm('(\\w+) has left the vessel\.') if m: - who = m.group(1) - ob_x(who, 'disembarked') - del self._v[who] - del self._pl[who] + pirate = m.group(1) + disembark(self._v, timestamp, pirate, 'disembarked') return d('disembarked') return d('not matched') @@ -590,8 +637,8 @@ class ChatLogTracker: def _str_vessel(self, vn, v): s = ' vessel %s\n' % vn s += ' '*20 + "%-*s %13s\n" % ( - max_pirate_namelen, '#lastaboard', - v['#lastaboard']) + max_pirate_namelen, '#lastinfo', + v['#lastinfo']) for pn in sorted(v.keys()): if pn.startswith('#'): continue pa = v[pn] @@ -635,6 +682,8 @@ class ChatLogTracker: if self._lbuf.endswith('\n'): self.chatline(self._lbuf.rstrip()) self._lbuf = '' + if opts.debug >= 2: + debug(self.__str__()) if progress: progress.caughtup() def changed(self): @@ -649,6 +698,7 @@ class ChatLogTracker: return self._vessel def aboard(self): # returns a list of PirateAboard sorted by name + if self._v is None: return [] return [ self._v[pn] for pn in sorted(self._v.keys()) if not pn.startswith('#') ] @@ -706,9 +756,9 @@ def prep_chat_log(args, bu, max_myself_age=3600): if len(args) != 1: bu('this action takes only chat log filename') logfn = args[0] - logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_chat-log-\\w+$' + logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_' match = regexp.match(logfn_re, logfn) - if not match: bu('chat log filename is not in default format') + if not match: bu('chat log filename is not in expected format') (pirate, ocean) = match.groups() fetcher.default_ocean(ocean) @@ -716,9 +766,11 @@ def prep_chat_log(args, bu, progress.show_init(pirate, fetcher.ocean) track = ChatLogTracker(myself, logfn) - opts.debug -= 1 + opts.debug -= 2 track.catchup(progress) - opts.debug += 1 + opts.debug += 2 + + track.force_redisplay() return (myself, track) @@ -815,11 +867,9 @@ def do_ship_aid(args, bu): s = "%s" % track.myname() vn = track.vessel() - if vn is None: - s += " not on a vessel?!" - else: - s += " on board the %s at %s\n" % ( - vn, time.strftime("%Y-%m-%d %H:%M:%S")) + if vn is None: s += " not on a vessel?!" + else: s += " on board the %s" % vn + s += " at %s\n" % time.strftime("%Y-%m-%d %H:%M:%S") tbl = StandingsTable() tbl.headings() @@ -842,7 +892,7 @@ def do_ship_aid(args, bu): displayer.show(s) time.sleep(1) - rotate_nya = rotate_nya[1:2] + rotate_nya[0] + rotate_nya = rotate_nya[1:3] + rotate_nya[0] #---------- main program ---------- @@ -870,7 +920,7 @@ display modes (for --display) apply to ship-aid: help='cache yoweb pages in DIR') ao('-D','--debug', action='count', dest='debug', default=0, help='enable debugging output') - ao('--debug-fd', action='count', dest='debug_fd', + ao('--debug-fd', type='int', dest='debug_fd', help='write any debugging output to specified fd') ao('-q','--quiet', action='store_true', dest='quiet', help='suppress warning output') @@ -890,7 +940,9 @@ display modes (for --display) apply to ship-aid: pa.error('need a mode argument') if opts.debug_fd is not None: - opts.debug_file = fdopen(opts.debug_fd, 'w') + opts.debug_file = os.fdopen(opts.debug_fd, 'w') + else: + opts.debug_file = sys.stdout mode = args[0] mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')