X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~yarrgweb/git?a=blobdiff_plain;f=yoweb-scrape;h=2a3559009ef23e941e17bed9fc35298634251c4d;hb=6024b8b7e54143104a1f29d3976c5194c889f5ba;hp=3e77ffdec575e95e48368ffbbb195ca6711d6bbe;hpb=11373a544443dbaf6ffccf5559df07b959548922;p=ypp-sc-tools.db-test.git diff --git a/yoweb-scrape b/yoweb-scrape index 3e77ffd..2a35590 100755 --- a/yoweb-scrape +++ b/yoweb-scrape @@ -1,5 +1,7 @@ #!/usr/bin/python +#---------- setup ---------- + import signal signal.signal(signal.SIGINT, signal.SIG_DFL) @@ -17,6 +19,7 @@ from BeautifulSoup import BeautifulSoup opts = None +#---------- YPP parameters and arrays ---------- puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+ '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+ @@ -31,10 +34,20 @@ pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm') max_pirate_namelen = 12 +#---------- general utilities ---------- + def debug(m): if opts.debug > 0: print m +def format_time_interval(ti): + if ti < 120: return '%d:%02d' % (ti / 60, ti % 60) + if ti < 7200: return '%2dm' % (ti / 60) + if ti < 86400: return '%dh' % (ti / 3600) + return '%dd' % (ti / 86400) + +#---------- caching and rate-limiting data fetcher ---------- + class Fetcher: def __init__(self, ocean, cachedir): debug('Fetcher init %s' % cachedir) @@ -133,6 +146,8 @@ class Fetcher: self.ocean, kind, tail) return self.fetch(url, max_age) +#---------- logging assistance for troubled screenscrapers ---------- + class SoupLog: def __init__(self): self.msgs = [ ] @@ -156,6 +171,8 @@ class SomethingSoupInfo(SoupLog): convertEntities=BeautifulSoup.HTML_ENTITIES ) +#---------- scraper for pirate pages ---------- + class PirateInfo(SomethingSoupInfo): # Public data members: # pi.standings = { 'Treasure Haul': 'Able' ... } @@ -255,6 +272,8 @@ u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A- def __str__(self): return `(self.crew, self.flag, self.standings, self.msgs)` +#---------- scraper for crew pages ---------- + class CrewInfo(SomethingSoupInfo): # Public data members: # ci.crew = [ ('Captain', ['Pirate', ...]), @@ -303,6 +322,8 @@ class CrewInfo(SomethingSoupInfo): def __str__(self): return `(self.crew, self.msgs)` +#---------- pretty-printer for tables of pirate puzzle standings ---------- + class StandingsTable: def __init__(self, use_puzzles=None, col_width=6): if use_puzzles is None: @@ -366,43 +387,18 @@ class StandingsTable: def results(self): return self.s -def do_pirate(pirates, bu): - print '{' - for pirate in pirates: - info = PirateInfo(pirate) - print '%s: %s,' % (`pirate`, info) - print '}' - -def prep_crew_of(args, bu, max_age=300): - if len(args) != 1: bu('crew-of takes one pirate name') - pi = PirateInfo(args[0], max_age) - return CrewInfo(pi.crew[0], max_age) - -def do_crew_of(args, bu): - ci = prep_crew_of(args, bu) - print ci - -def do_standings_crew_of(args, bu): - ci = prep_crew_of(args, bu, 60) - tab = StandingsTable() - tab.headings() - for (rank, members) in ci.crew: - if not members: continue - tab.literalline('%s:' % rank) - for p in members: - pi = PirateInfo(p, random.randint(900,1800)) - tab.pirate(pi) - print tab.results() +#---------- chat log parser ---------- class PirateAboard: - # pa.v - # pa.name - # pa.last_time - # pa.last_event - # pa.gunner - # pa.last_chat_time - # pa.last_chat_chan - # pa.pi + # This is essentially a transparent, dumb, data class. + # pa.v + # pa.name + # pa.last_time + # pa.last_event + # pa.gunner + # pa.last_chat_time + # pa.last_chat_chan + # pa.pi def __init__(pa, pn, v, time, event): pa.name = pn @@ -420,6 +416,9 @@ class PirateAboard: return pa.pi class ChatLogTracker: + # This is quite complex so we make it opaque. Use the + # official invokers, accessors etc. + def __init__(self, myself_pi, logfn): self._pl = {} # self._pl['Pirate'] = self._vl = {} # self._vl['Vessel']['Pirate'] = PirateAboard @@ -653,15 +652,51 @@ class ChatLogTracker: for pn in sorted(self._v.keys()) if not pn.startswith('#') ] +#---------- implementations of actual operation modes ---------- + +def do_pirate(pirates, bu): + print '{' + for pirate in pirates: + info = PirateInfo(pirate) + print '%s: %s,' % (`pirate`, info) + print '}' + +def prep_crew_of(args, bu, max_age=300): + if len(args) != 1: bu('crew-of takes one pirate name') + pi = PirateInfo(args[0], max_age) + if pi.crew is None: return None + return CrewInfo(pi.crew[0], max_age) + +def do_crew_of(args, bu): + ci = prep_crew_of(args, bu) + print ci + +def do_standings_crew_of(args, bu): + ci = prep_crew_of(args, bu, 60) + tab = StandingsTable() + tab.headings() + for (rank, members) in ci.crew: + if not members: continue + tab.literalline('%s:' % rank) + for p in members: + pi = PirateInfo(p, random.randint(900,1800)) + tab.pirate(pi) + print tab.results() + class ProgressPrintPercentage: - def __init__(self, f=sys.stdout): self._f = f - def progress(self,done,total): - self._f.write("scan chat logs %3d%%\r" % ((done*100) / total)) + def __init__(self, f=sys.stdout): + self._f = f + def progress_string(self,done,total): + return "scan chat logs %3d%%\r" % ((done*100) / total) + def progress(self,*a): + self._f.write(self.progress_string(*a)) self._f.flush() def caughtup(self): self._f.write(' \r') self._f.flush() +#----- modes which use the chat log parser are quite complex ----- + def prep_chat_log(args, bu, progress=ProgressPrintPercentage(), max_myself_age=3600): @@ -689,19 +724,22 @@ def do_track_chat_log(args, bu): print track time.sleep(1) -def format_time_interval(ti): - if ti < 120: return '%d:%02d' % (ti / 60, ti % 60) - if ti < 7200: return '%2dm' % (ti / 60) - if ti < 86400: return '%dh' % (ti / 3600) - return '%dd' % (ti / 86400) +#----- ship management aid ----- + +class Display_dumb(ProgressPrintPercentage): + def __init__(self): + ProgressPrintPercentage.__init__(self) + def show(self, s): + print '\n\n', s; def do_ship_aid(args, bu): if opts.ship_duty is None: opts.ship_duty = True - (myself, track) = prep_chat_log(args, bu) - + displayer = globals()['Display_'+opts.display]() rotate_nya = '/-\\' + (myself, track) = prep_chat_log(args, bu, progress=displayer) + def timeevent(t,e): if t is None: return ' ' * 22 return " %-4s %-16s" % (format_time_interval(now - t),e) @@ -713,10 +751,11 @@ def do_ship_aid(args, bu): s = "%s" % track.myname() vn = track.vessel() - if vn is None: print s + " ...?"; return - - s += " on board the %s at %s\n" % ( - vn, time.strftime("%Y-%m-%d %H:%M:%S")) + if vn is None: + s += " not on a vessel?!" + else: + s += " on board the %s at %s\n" % ( + vn, time.strftime("%Y-%m-%d %H:%M:%S")) tbl = StandingsTable() tbl.headings() @@ -725,6 +764,8 @@ def do_ship_aid(args, bu): pi = pa.pirate_info() xs = '' + if pa.gunner: xs += 'G ' + else: xs += ' ' xs += timeevent(pa.last_time, pa.last_event) xs += timeevent(pa.last_chat_time, pa.last_chat_chan) @@ -735,11 +776,12 @@ def do_ship_aid(args, bu): s += tbl.results() - print '\n\n', s; - + displayer.show(s) time.sleep(1) rotate_nya = rotate_nya[1:2] + rotate_nya[0] +#---------- main program ---------- + def main(): global opts, fetcher @@ -750,7 +792,11 @@ actions: yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG - yoweb-scrape [--ocean OCEAN ...] ship-aid CHAT-LOG + yoweb-scrape [options] ship-aid CHAT-LOG (must be .../PIRATE_OCEAN_chat-log*) + +display modes (for --display) apply to ship-aid: + --display=dumb just print new information, scrolling the screen + --display=overwrite use cursor motion, selective clear, etc. to redraw at top ''') ao = pa.add_option ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None, @@ -760,8 +806,13 @@ actions: help='cache yoweb pages in DIR') ao('-D','--debug', action='count', dest='debug', default=0, help='enable debugging output') + ao('--debug-fd', action='count', dest='debug_fd', + help='write any debugging output to specified fd') ao('-q','--quiet', action='store_true', dest='quiet', help='suppress warning output') + ao('--display', action='store', dest='display', + type='choice', choices=['dumb','overwrite'], + help='how to display ship aid') ao('--ship-duty', action='store_true', dest='ship_duty', help='show ship duty station puzzles') @@ -774,6 +825,9 @@ actions: if len(args) < 1: pa.error('need a mode argument') + if opts.debug_fd is not None: + opts.debug_file = fdopen(opts.debug_fd, 'w') + mode = args[0] mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_') try: mode_fn = globals()[mode_fn_name] @@ -787,6 +841,13 @@ actions: if opts.cache_dir.startswith('~/'): opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:] + if opts.display is None: + if ((opts.debug > 0 and opts.debug_fd is None) + or not os.isatty(sys.stdout.fileno())): + opts.display = 'dumb' + else: + opts.display = 'overwrite' + fetcher = Fetcher(opts.ocean, opts.cache_dir) mode_fn(args[1:], pa.error)