yoweb-scrape

   1 #!/usr/bin/python
   2
   3 #---------- setup ----------
   4
   5 import signal
   6 signal.signal(signal.SIGINT, signal.SIG_DFL)
   7
   8 import os
   9 import time
  10 import urllib
  11 import urllib2
  12 import errno
  13 import sys
  14 import re as regexp
  15 import random
  16 import curses
  17 from optparse import OptionParser
  18
  19 from BeautifulSoup import BeautifulSoup
  20
  21 opts = None
  22
  23 #---------- YPP parameters and arrays ----------
  24
  25 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
  26         '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
  27         '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
  28         '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
  29
  30 standingvals = ('Able/Distinguished/Respected/Master'+
  31                 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
  32
  33 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
  34
  35 max_pirate_namelen = 12
  36
  37
  38 #---------- general utilities ----------
  39
  40 def debug(m):
  41         if opts.debug > 0:
  42                 print >>opts.debug_file, m
  43
  44 def format_time_interval(ti):
  45         if ti < 120: return '%d:%02d' % (ti / 60, ti % 60)
  46         if ti < 7200: return '%2dm' % (ti / 60)
  47         if ti < 86400: return '%dh' % (ti / 3600)
  48         return '%dd' % (ti / 86400)
  49
  50 #---------- caching and rate-limiting data fetcher ----------
  51
  52 class Fetcher:
  53         def __init__(self, ocean, cachedir):
  54                 debug('Fetcher init %s' % cachedir)
  55                 self.ocean = ocean
  56                 self.cachedir = cachedir
  57                 try: os.mkdir(cachedir)
  58                 except (OSError,IOError), oe:
  59                         if oe.errno != errno.EEXIST: raise
  60                 self._cache_scan(time.time())
  61
  62         def default_ocean(self, ocean='ice'):
  63                 if self.ocean is None:
  64                         self.ocean = ocean
  65
  66         def _cache_scan(self, now):
  67                 # returns list of ages, unsorted
  68                 ages = []
  69                 debug('Fetcher   scan_cache')
  70                 for leaf in os.listdir(self.cachedir):
  71                         if not leaf.startswith('#'): continue
  72                         path = self.cachedir + '/' + leaf
  73                         try: s = os.stat(path)
  74                         except (OSError,IOError), oe:
  75                                 if oe.errno != errno.ENOENT: raise
  76                                 continue
  77                         age = now - s.st_mtime
  78                         if age > opts.expire_age:
  79                                 debug('Fetcher    expire %d %s' % (age, path))
  80                                 try: os.remove(path)
  81                                 except (OSError,IOError), oe:
  82                                         if oe.errno != errno.ENOENT: raise
  83                                 continue
  84                         ages.append(age)
  85                 return ages
  86
  87         def need_wait(self, now):
  88                 ages = self._cache_scan(now)
  89                 ages.sort()
  90                 debug('Fetcher   ages ' + `ages`)
  91                 min_age = 1
  92                 need_wait = 0
  93                 for age in ages:
  94                         if age < min_age and age < 300:
  95                                 debug('Fetcher   morewait min=%d age=%d' %
  96                                         (min_age, age))
  97                                 need_wait = max(need_wait, min_age - age)
  98                         min_age += 3
  99                         min_age *= 1.25
 100                 return need_wait
 101
 102         def _rate_limit_cache_clean(self, now):
 103                 need_wait = self.need_wait(now)
 104                 if need_wait > 0:
 105                         debug('Fetcher   wait %d' % need_wait)
 106                         time.sleep(need_wait)
 107
 108         def fetch(self, url, max_age):
 109                 debug('Fetcher fetch %s' % url)
 110                 cache_corename = urllib.quote_plus(url)
 111                 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
 112                 try: f = file(cache_item, 'r')
 113                 except (OSError,IOError), oe:
 114                         if oe.errno != errno.ENOENT: raise
 115                         f = None
 116                 now = time.time()
 117                 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
 118                 if f is not None:
 119                         s = os.fstat(f.fileno())
 120                         age = now - s.st_mtime
 121                         if age > max_age:
 122                                 debug('Fetcher  stale %d < %d'% (max_age, age))
 123                                 f = None
 124                 if f is not None:
 125                         data = f.read()
 126                         f.close()
 127                         debug('Fetcher  cached %d > %d' % (max_age, age))
 128                         return data
 129
 130                 debug('Fetcher  fetch')
 131                 self._rate_limit_cache_clean(now)
 132
 133                 stream = urllib2.urlopen(url)
 134                 data = stream.read()
 135                 cache_tmp = "%s/#%s~%d#" % (
 136                         self.cachedir, cache_corename, os.getpid())
 137                 f = file(cache_tmp, 'w')
 138                 f.write(data)
 139                 f.close()
 140                 os.rename(cache_tmp, cache_item)
 141                 debug('Fetcher  stored')
 142                 return data
 143
 144         def yoweb(self, kind, tail, max_age):
 145                 self.default_ocean()
 146                 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
 147                         self.ocean, kind, tail)
 148                 return self.fetch(url, max_age)
 149
 150 #---------- logging assistance for troubled screenscrapers ----------
 151
 152 class SoupLog:
 153         def __init__(self):
 154                 self.msgs = [ ]
 155         def msg(self, m):
 156                 self.msgs.append(m)
 157         def soupm(self, obj, m):
 158                 self.msg(m + '; in ' + `obj`)
 159         def needs_msgs(self, child_souplog):
 160                 self.msgs += child_souplog.msgs
 161                 child_souplog.msgs = [ ]
 162
 163 def soup_text(obj):
 164         str = ''.join(obj.findAll(text=True))
 165         return str.strip()
 166
 167 class SomethingSoupInfo(SoupLog):
 168         def __init__(self, kind, tail, max_age):
 169                 SoupLog.__init__(self)
 170                 html = fetcher.yoweb(kind, tail, max_age)
 171                 self._soup = BeautifulSoup(html,
 172                         convertEntities=BeautifulSoup.HTML_ENTITIES
 173                         )
 174
 175 #---------- scraper for pirate pages ----------
 176
 177 class PirateInfo(SomethingSoupInfo):
 178         # Public data members:
 179         #  pi.standings = { 'Treasure Haul': 'Able' ... }
 180         #  pi.name = name
 181         #  pi.crew = (id, name)
 182         #  pi.flag = (id, name)
 183         #  pi.msgs = [ 'message describing problem with scrape' ]
 184
 185         def __init__(self, pirate, max_age=300):
 186                 SomethingSoupInfo.__init__(self,
 187                         'pirate.wm?target=', pirate, max_age)
 188                 self.name = pirate
 189                 self._find_standings()
 190                 self.crew = self._find_crewflag('crew',
 191                         '^/yoweb/crew/info\\.wm')
 192                 self.flag = self._find_crewflag('flag',
 193                         '^/yoweb/flag/info\\.wm')
 194
 195         def _find_standings(self):
 196                 imgs = self._soup.findAll('img',
 197                         src=regexp.compile('/yoweb/images/stat.*'))
 198                 re = regexp.compile(
 199 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
 200                         )
 201                 standings = { }
 202
 203                 for skill in puzzles:
 204                         standings[skill] = [ ]
 205
 206                 skl = SoupLog()
 207
 208                 for img in imgs:
 209                         try: puzzle = img['alt']
 210                         except KeyError: continue
 211
 212                         if not puzzle in puzzles:
 213                                 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
 214                                 continue
 215                         key = img.findParent('td')
 216                         if key is None:
 217                                 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
 218                                 continue
 219                         valelem = key.findNextSibling('td')
 220                         if valelem is None:
 221                                 skl.soupm(key, 'puzzle missing sibling "%s"'
 222                                         % puzzle)
 223                                 continue
 224                         valstr = soup_text(valelem)
 225                         match = re.match(valstr)
 226                         if match is None:
 227                                 skl.soupm(key, ('puzzle "%s" unparseable'+
 228                                         ' standing "%s"') % (puzzle, valstr))
 229                                 continue
 230                         standing = match.group(match.lastindex)
 231                         standings[puzzle].append(standing)
 232
 233                 self.standings = { }
 234
 235                 for puzzle in puzzles:
 236                         sl = standings[puzzle]
 237                         if len(sl) > 1:
 238                                 skl.msg('puzzle "%s" multiple standings %s' %
 239                                                 (puzzle, `sl`))
 240                                 continue
 241                         if not sl:
 242                                 skl.msg('puzzle "%s" no standing found' % puzzle)
 243                                 continue
 244                         standing = sl[0]
 245                         for i in range(0, len(standingvals)-1):
 246                                 if standing == standingvals[i]:
 247                                         self.standings[puzzle] = i
 248                         if not puzzle in self.standings:
 249                                 skl.msg('puzzle "%s" unknown standing "%s"' %
 250                                         (puzzle, standing))
 251
 252                 all_standings_ok = True
 253                 for puzzle in puzzles:
 254                         if not puzzle in self.standings:
 255                                 self.needs_msgs(skl)
 256
 257         def _find_crewflag(self, cf, yoweb_re):
 258                 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
 259                 if len(things) != 1:
 260                         self.msg('zero or several %s id references found' % cf)
 261                         return None
 262                 thing = things[0]
 263                 id_re = '\\b%sid\\=(\\w+)$' % cf
 264                 id_haystack = thing['href']
 265                 match = regexp.compile(id_re).search(id_haystack)
 266                 if match is None:
 267                         self.soupm(thing, ('incomprehensible %s id ref'+
 268                                 ' (%s in %s)') % (cf, id_re, id_haystack))
 269                         return None
 270                 name = soup_text(thing)
 271                 return (match.group(1), name)
 272
 273         def __str__(self):
 274                 return `(self.crew, self.flag, self.standings, self.msgs)`
 275
 276 #---------- scraper for crew pages ----------
 277
 278 class CrewInfo(SomethingSoupInfo):
 279         # Public data members:
 280         #  ci.crew = [ ('Captain',        ['Pirate', ...]),
 281         #              ('Senior Officer', [...]),
 282         #               ... ]
 283         #  pi.msgs = [ 'message describing problem with scrape' ]
 284
 285         def __init__(self, crewid, max_age=300):
 286                 SomethingSoupInfo.__init__(self,
 287                         'crew/info.wm?crewid=', crewid, max_age)
 288                 self._find_crew()
 289
 290         def _find_crew(self):
 291                 self.crew = []
 292                 capts = self._soup.findAll('img',
 293                         src='/yoweb/images/crew-captain.png')
 294                 if len(capts) != 1:
 295                         self.msg('crew members: no. of captain images != 1')
 296                         return
 297                 tbl = capts[0]
 298                 while not tbl.find('a', href=pirate_ref_re):
 299                         tbl = tbl.findParent('table')
 300                         if not tbl:
 301                                 self.msg('crew members: cannot find table')
 302                                 return
 303                 current_rank_crew = None
 304                 crew_rank_re = regexp.compile('/yoweb/images/crew')
 305                 for row in tbl.contents:
 306                         # findAll(recurse=False)
 307                         if isinstance(row,basestring):
 308                                 continue
 309
 310                         is_rank = row.find('img', attrs={'src': crew_rank_re})
 311                         if is_rank:
 312                                 rank = soup_text(row)
 313                                 current_rank_crew = []
 314                                 self.crew.append((rank, current_rank_crew))
 315                                 continue
 316                         for cell in row.findAll('a', href=pirate_ref_re):
 317                                 if current_rank_crew is None:
 318                                         self.soupm(cell, 'crew members: crew'
 319                                                 ' before rank')
 320                                         continue
 321                                 current_rank_crew.append(soup_text(cell))
 322
 323         def __str__(self):
 324                 return `(self.crew, self.msgs)`
 325
 326 #---------- pretty-printer for tables of pirate puzzle standings ----------
 327
 328 class StandingsTable:
 329         def __init__(self, use_puzzles=None, col_width=6):
 330                 if use_puzzles is None:
 331                         if opts.ship_duty:
 332                                 use_puzzles=[
 333                                         'Navigating','Battle Navigation',
 334                                         'Gunning',
 335                                         ['Sailing','Rigging'],
 336                                         'Bilging',
 337                                         'Carpentry',
 338                                         'Treasure Haul'
 339                                 ]
 340                         else:
 341                                 use_puzzles=puzzles
 342                 self._puzzles = use_puzzles
 343                 self.s = ''
 344                 self._cw = col_width-1
 345
 346         def _pline(self, pirate, puzstrs, extra):
 347                 self.s += ' %-*s' % (max(max_pirate_namelen, 14), pirate)
 348                 for v in puzstrs:
 349                         self.s += ' %-*.*s' % (self._cw,self._cw, v)
 350                 if extra:
 351                         self.s += ' ' + extra
 352                 self.s += '\n'
 353
 354         def _puzstr(self, pi, puzzle):
 355                 if not isinstance(puzzle,list): puzzle = [puzzle]
 356                 try: standing = max([pi.standings[p] for p in puzzle])
 357                 except KeyError: return '?'
 358                 if not standing: return ''
 359                 s = ''
 360                 if self._cw > 4:
 361                         c1 = standingvals[standing][0]
 362                         if standing < 3: c1 = c1.lower() # 3 = Master
 363                         s += `standing`
 364                 if self._cw > 5:
 365                         s += ' '
 366                 s += '*' * (standing / 2)
 367                 s += '+' * (standing % 2)
 368                 return s
 369
 370         def headings(self):
 371                 def puzn_redact(name):
 372                         if isinstance(name,list):
 373                                 return '/'.join(
 374                                         ["%.*s" % (self._cw/2, puzn_redact(n))
 375                                          for n in name])
 376                         spc = name.find(' ')
 377                         if spc < 0: return name
 378                         return name[0:min(4,spc)] + name[spc+1:]
 379                 self._pline('', map(puzn_redact, self._puzzles), None)
 380         def literalline(self, line):
 381                 self.s += line + '\n'
 382         def pirate_dummy(self, name, standingstring, extra=None):
 383                 self._pline(name, standingstring * len(self._puzzles), extra)
 384         def pirate(self, pi, extra=None):
 385                 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
 386                 self._pline(pi.name, puzstrs, extra)
 387
 388         def results(self):
 389                 return self.s
 390
 391 #---------- chat log parser ----------
 392
 393 class PirateAboard:
 394         # This is essentially a transparent, dumb, data class.
 395         #  pa.v
 396         #  pa.name
 397         #  pa.last_time
 398         #  pa.last_event
 399         #  pa.gunner
 400         #  pa.last_chat_time
 401         #  pa.last_chat_chan
 402         #  pa.pi
 403
 404         def __init__(pa, pn, v, time, event):
 405                 pa.name = pn
 406                 pa.v = v
 407                 pa.last_time = time
 408                 pa.last_event = event
 409                 pa.last_chat_time = None
 410                 pa.last_chat_chan = None
 411                 pa.gunner = False
 412                 pa.pi = None
 413
 414         def pirate_info(pa):
 415                 if not pa.pi and not fetcher.need_wait(time.time()):
 416                         pa.pi = PirateInfo(pa.name, 3600)
 417                 return pa.pi
 418
 419 class ChatLogTracker:
 420         # This is quite complex so we make it opaque.  Use the
 421         # official invokers, accessors etc.
 422
 423         def __init__(self, myself_pi, logfn):
 424                 self._pl = {}   # self._pl['Pirate'] =
 425                 self._vl = {}   #   self._vl['Vessel']['Pirate'] = PirateAboard
 426                                 # self._vl['Vessel']['#lastinfo']
 427                 self._v = None          # self._v =
 428                 self._vessel = None     #       self._vl[self._vessel]
 429                 self._date = None
 430                 self._myself = myself_pi
 431                 self._need_redisplay = False
 432                 self._f = file(logfn)
 433                 self._lbuf = ''
 434                 self._progress = [0, os.fstat(self._f.fileno()).st_size]
 435
 436         def force_redisplay(self):
 437                 self._need_redisplay = True
 438
 439         def _onboard_event(self,v,timestamp,pirate,event):
 440                 pa = self._pl.get(pirate, None)
 441                 if pa is not None and pa.v is v:
 442                         pa.last_time = timestamp
 443                         pa.last_event = event
 444                 else:
 445                         if pa is not None: del pa.v[pirate]
 446                         pa = PirateAboard(pirate, v, timestamp, event)
 447                         self._pl[pirate] = pa
 448                         v[pirate] = pa
 449                 v['#lastinfo'] = timestamp
 450                 self.force_redisplay()
 451                 return pa
 452
 453         def _trash_vessel(self, v):
 454                 for pn in v:
 455                         if pn.startswith('#'): continue
 456                         del self._pl[pn]
 457                 self.force_redisplay()
 458
 459         def expire_garbage(self, timestamp):
 460                 for (vn,v) in list(self._vl.iteritems()):
 461                         la = v['#lastinfo']
 462                         if timestamp - la > opts.ship_reboard_clearout:
 463                                 self._debug_line_disposition(timestamp,'',
 464                                         'stale reset '+vn)
 465                                 self._trash_vessel(v)
 466                                 del self._vl[vn]
 467
 468         def _create_vessel(self, vn, timestamp):
 469                 self._vl[vn] = v = { '#lastinfo': timestamp }
 470                 return v
 471
 472         def _update_vessel_lookup(self, vn, timestamp, dml):
 473                 v = self._vl.get(vn, None)
 474                 if v is None:
 475                         dml.append('new')
 476                         v = self._create_vessel(vn, timestamp)
 477                 elif timestamp - v['#lastinfo'] > opts.ship_reboard_clearout:
 478                         dml.append('stale')
 479                         self._trash_vessel(v)
 480                         v = self._create_vessel(vn, timestamp)
 481                 else:
 482                         dml.append('current')
 483                 return v
 484
 485         def _debug_line_disposition(self,timestamp,l,m):
 486                 debug('CLT %13s %-30s %s' % (timestamp,m,l))
 487
 488         def chatline(self,l):
 489                 rm = lambda re: regexp.match(re,l)
 490                 d = lambda m: self._debug_line_disposition(timestamp,l,m)
 491                 timestamp = None
 492
 493                 m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$')
 494                 if m:
 495                         self._date = [int(x) for x in m.groups()]
 496                         self._previous_timestamp = None
 497                         return d('date '+`self._date`)
 498
 499                 if self._date is None:
 500                         return d('date unset')
 501
 502                 m = rm('\\[(\d\d):(\d\d):(\d\d)\\] ')
 503                 if not m:
 504                         return d('no timestamp')
 505
 506                 while True:
 507                         time_tuple = (self._date +
 508                                       [int(x) for x in m.groups()] +
 509                                       [-1,-1,-1])
 510                         timestamp = time.mktime(time_tuple)
 511                         if timestamp >= self._previous_timestamp: break
 512                         self._date[2] += 1
 513                         self._debug_line_disposition(timestamp,'',
 514                                 'new date '+`self._date`)
 515
 516                 self._previous_timestamp = timestamp
 517
 518                 l = l[l.find(' ')+1:]
 519
 520                 def ob_x(pirate,event):
 521                         return self._onboard_event(
 522                                         self._v, timestamp, pirate, event)
 523                 def ob1(did): ob_x(m.group(1), did); return d(did)
 524                 def oba(did): return ob1('%s %s' % (did, m.group(2)))
 525
 526                 def disembark(v, timestamp, pirate, event):
 527                         self._onboard_event(
 528                                         v, timestamp, pirate, 'leaving '+event)
 529                         del v[pirate]
 530                         del self._pl[pirate]
 531
 532                 m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
 533                 if m:
 534                         dm = ['boarding']
 535                         pn = self._myself.name
 536                         self._vessel = vn = m.group(1)
 537                         self._v = self._update_vessel_lookup(vn, timestamp, dm)
 538
 539                         ob_x(pn, 'we boarded')
 540                         self.expire_garbage(timestamp)
 541                         return d(' '.join(dm))
 542
 543                 if self._v is None:
 544                         return d('no vessel')
 545
 546                 m = rm('(\\w+) has come aboard\\.$')
 547                 if m: return ob1('boarded');
 548
 549                 m = rm('You have ordered (\\w+) to do some (\\S.*\\S)\\.$')
 550                 if m:
 551                         (who,what) = m.groups()
 552                         pa = ob_x(who,'ord '+what)
 553                         if what == 'Gunning':
 554                                 pa.gunner = True
 555                         return d('duty order')
 556
 557                 m = rm('(\\w+) abandoned a (\\S.*\\S) station\\.$')
 558                 if m: oba('stopped'); return d("end")
 559
 560                 def chat(what):
 561                         who = m.group(1)
 562                         try: pa = self._pl[who]
 563                         except KeyError: return d('chat mystery')
 564                         if pa.v is self._v:
 565                                 pa.last_chat_time = timestamp
 566                                 pa.last_chat_chan = what
 567                                 self.force_redisplay()
 568                                 return d('chat '+what)
 569
 570                 def chat_metacmd(what):
 571                         (cmdr, metacmd) = m.groups()
 572                         metacmd = regexp.sub('\\s+', ' ', metacmd).strip()
 573                         m2 = regexp.match(
 574                                 '/([ad]) (?:([A-Za-z ]+)\\s*:)?([A-Za-z ]+)$',
 575                                 metacmd)
 576                         if not m2: return chat(what)
 577
 578                         (cmd, vn, targets) = m2.groups()
 579
 580                         if cmdr == self._myself.name: how = 'manual: /%s' % cmd
 581                         else: how = '/%s %s' % (cmd,cmdr)
 582                         if cmd == 'a': each = ob_x
 583                         else: each = lambda who,how: disembark(
 584                                 self._v, timestamp, who, how)
 585
 586                         if vn is not None:
 587                                 vn = vn.title()
 588                                 if not regexp.match(
 589                                                 '(?:.* )?' + vn + '$',
 590                                                 self._vessel):
 591                                         return chat('/%s %s:' % (cmd,vn))
 592
 593                         for target in targets.split(' '):
 594                                 if not target: continue
 595                                 each(target.title(), how)
 596                         return d('/%s' % cmd)
 597
 598                 m = rm('(\\w+) (?:issued an order|ordered everyone) "')
 599                 if m: return ob1('general order');
 600
 601                 m = rm('(\\w+) says, "')
 602                 if m: return chat('public')
 603
 604                 m = rm('(\\w+) tells ye, "')
 605                 if m: return chat('private')
 606
 607                 m = rm('Ye told (\\w+), "(.*)"$')
 608                 if m: return chat_metacmd('private')
 609
 610                 m = rm('(\\w+) flag officer chats, "')
 611                 if m: return chat('flag officer')
 612
 613                 m = rm('(\\w+) officer chats, "(.*)"$')
 614                 if m: return chat_metacmd('officer')
 615
 616                 m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
 617                 if m:
 618                         pl = m.group(1).split(', ')
 619                         if not self._myself.name in pl:
 620                                 return d('lost boarding battle')
 621                         for pn in pl:
 622                                 if ' ' in pn: continue
 623                                 ob_x(pn,'won boarding battle')
 624                         return d('won boarding battle')
 625
 626                 m = rm('(\\w+) is eliminated\\!')
 627                 if m: return ob1('eliminated in fray');
 628
 629                 m = rm('(\\w+) has left the vessel\.')
 630                 if m:
 631                         pirate = m.group(1)
 632                         disembark(self._v, timestamp, pirate, 'disembarked')
 633                         return d('disembarked')
 634
 635                 return d('not matched')
 636
 637         def _str_vessel(self, vn, v):
 638                 s = ' vessel %s\n' % vn
 639                 s += ' '*20 + "%-*s   %13s\n" % (
 640                                 max_pirate_namelen, '#lastinfo',
 641                                 v['#lastinfo'])
 642                 for pn in sorted(v.keys()):
 643                         if pn.startswith('#'): continue
 644                         pa = v[pn]
 645                         assert pa.v == v
 646                         assert self._pl[pn] == pa
 647                         s += ' '*20 + "%s %-*s %13s %-30s %13s %s\n" % (
 648                                 (' ','G')[pa.gunner],
 649                                 max_pirate_namelen, pn,
 650                                 pa.last_time, pa.last_event,
 651                                 pa.last_chat_time, pa.last_chat_chan)
 652                 return s
 653
 654         def __str__(self):
 655                 s = '''<ChatLogTracker
 656  myself %s
 657  vessel %s
 658 '''                     % (self._myself.name, self._vessel)
 659                 assert ((self._v is None and self._vessel is None) or
 660                         (self._v is self._vl[self._vessel]))
 661                 if self._vessel is not None:
 662                         s += self._str_vessel(self._vessel, self._v)
 663                 for vn in sorted(self._vl.keys()):
 664                         if vn == self._vessel: continue
 665                         s += self._str_vessel(vn, self._vl[vn])
 666                 for p in self._pl:
 667                         pa = self._pl[p]
 668                         assert pa.v[p] is pa
 669                         assert pa.v in self._vl.values()
 670                 s += '>\n'
 671                 return s
 672
 673         def catchup(self, progress=None):
 674                 while True:
 675                         more = self._f.readline()
 676                         if not more: break
 677
 678                         self._progress[0] += len(more)
 679                         if progress: progress.progress(*self._progress)
 680
 681                         self._lbuf += more
 682                         if self._lbuf.endswith('\n'):
 683                                 self.chatline(self._lbuf.rstrip())
 684                                 self._lbuf = ''
 685                                 if opts.debug >= 2:
 686                                         debug(self.__str__())
 687                 if progress: progress.caughtup()
 688
 689         def changed(self):
 690                 rv = self._need_redisplay
 691                 self._need_redisplay = False
 692                 return rv
 693         def myname(self):
 694                 # returns our pirate name
 695                 return self._myself.name
 696         def vessel(self):
 697                 # returns the vessel we're aboard or None
 698                 return self._vessel
 699         def aboard(self):
 700                 # returns a list of PirateAboard sorted by name
 701                 if self._v is None: return []
 702                 return [ self._v[pn]
 703                          for pn in sorted(self._v.keys())
 704                          if not pn.startswith('#') ]
 705
 706 #---------- implementations of actual operation modes ----------
 707
 708 def do_pirate(pirates, bu):
 709         print '{'
 710         for pirate in pirates:
 711                 info = PirateInfo(pirate)
 712                 print '%s: %s,' % (`pirate`, info)
 713         print '}'
 714
 715 def prep_crew_of(args, bu, max_age=300):
 716         if len(args) != 1: bu('crew-of takes one pirate name')
 717         pi = PirateInfo(args[0], max_age)
 718         if pi.crew is None: return None
 719         return CrewInfo(pi.crew[0], max_age)
 720
 721 def do_crew_of(args, bu):
 722         ci = prep_crew_of(args, bu)
 723         print ci
 724
 725 def do_standings_crew_of(args, bu):
 726         ci = prep_crew_of(args, bu, 60)
 727         tab = StandingsTable()
 728         tab.headings()
 729         for (rank, members) in ci.crew:
 730                 if not members: continue
 731                 tab.literalline('%s:' % rank)
 732                 for p in members:
 733                         pi = PirateInfo(p, random.randint(900,1800))
 734                         tab.pirate(pi)
 735         print tab.results()
 736
 737 class ProgressPrintPercentage:
 738         def __init__(self, f=sys.stdout):
 739                 self._f = f
 740         def progress_string(self,done,total):
 741                 return "scan chat logs %3d%%\r" % ((done*100) / total)
 742         def progress(self,*a):
 743                 self._f.write(self.progress_string(*a))
 744                 self._f.flush()
 745         def show_init(self, pirate, ocean):
 746                 print >>self._f, 'Starting up, %s on the %s ocean' % (
 747                         pirate, ocean)
 748         def caughtup(self):
 749                 self._f.write('                   \r')
 750                 self._f.flush()
 751
 752 #----- modes which use the chat log parser are quite complex -----
 753
 754 def prep_chat_log(args, bu,
 755                 progress=ProgressPrintPercentage(),
 756                 max_myself_age=3600):
 757         if len(args) != 1: bu('this action takes only chat log filename')
 758         logfn = args[0]
 759         logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_'
 760         match = regexp.match(logfn_re, logfn)
 761         if not match: bu('chat log filename is not in expected format')
 762         (pirate, ocean) = match.groups()
 763         fetcher.default_ocean(ocean)
 764
 765         myself = PirateInfo(pirate,max_myself_age)
 766         progress.show_init(pirate, fetcher.ocean)
 767         track = ChatLogTracker(myself, logfn)
 768
 769         opts.debug -= 2
 770         track.catchup(progress)
 771         opts.debug += 2
 772
 773         track.force_redisplay()
 774
 775         return (myself, track)
 776
 777 def do_track_chat_log(args, bu):
 778         (myself, track) = prep_chat_log(args, bu)
 779         while True:
 780                 track.catchup()
 781                 if track.changed():
 782                         print track
 783                 time.sleep(1)
 784
 785 #----- ship management aid -----
 786
 787 class Display_dumb(ProgressPrintPercentage):
 788         def __init__(self):
 789                 ProgressPrintPercentage.__init__(self)
 790         def show(self, s):
 791                 print '\n\n', s;
 792         def realstart(self):
 793                 pass
 794
 795 class Display_overwrite(ProgressPrintPercentage):
 796         def __init__(self):
 797                 ProgressPrintPercentage.__init__(self)
 798
 799                 null = file('/dev/null','w')
 800                 curses.setupterm(fd=null.fileno())
 801
 802                 self._clear = curses.tigetstr('clear')
 803                 if not self._clear:
 804                         self._debug('missing clear!')
 805                         self.show = Display_dumb.show
 806                         return
 807
 808                 self._t = {'el':'', 'ed':''}
 809                 if not self._init_sophisticated():
 810                         for k in self._t.keys(): self._t[k] = ''
 811                         self._t['ho'] = self._clear
 812
 813         def _debug(self,m): debug('display overwrite: '+m)
 814
 815         def _init_sophisticated(self):
 816                 for k in self._t.keys():
 817                         s = curses.tigetstr(k)
 818                         self._t[k] = s
 819                 self._t['ho'] = curses.tigetstr('ho')
 820                 if not self._t['ho']:
 821                         cup = curses.tigetstr('cup')
 822                         self._t['ho'] = curses.tparm(cup,0,0)
 823                 missing = [k for k in self._t.keys() if not self._t[k]]
 824                 if missing:
 825                         self.debug('missing '+(' '.join(missing)))
 826                         return 0
 827                 return 1
 828
 829         def show(self, s):
 830                 w = sys.stdout.write
 831                 def wti(k): w(self._t[k])
 832
 833                 wti('ho')
 834                 nl = ''
 835                 for l in s.rstrip().split('\n'):
 836                         w(nl)
 837                         w(l)
 838                         wti('el')
 839                         nl = '\r\n'
 840                 wti('ed')
 841                 w(' ')
 842                 sys.stdout.flush()
 843
 844         def realstart(self):
 845                 sys.stdout.write(self._clear)
 846                 sys.stdout.flush()
 847
 848
 849 def do_ship_aid(args, bu):
 850         if opts.ship_duty is None: opts.ship_duty = True
 851
 852         displayer = globals()['Display_'+opts.display]()
 853         rotate_nya = '/-\\'
 854
 855         (myself, track) = prep_chat_log(args, bu, progress=displayer)
 856
 857         def timeevent(t,e):
 858                 if t is None: return ' ' * 22
 859                 return " %-4s %-16s" % (format_time_interval(now - t),e)
 860
 861         displayer.realstart()
 862
 863         while True:
 864                 track.catchup()
 865                 now = time.time()
 866
 867                 s = "%s" % track.myname()
 868
 869                 vn = track.vessel()
 870                 if vn is None: s += " not on a vessel?!"
 871                 else: s += " on board the %s" % vn
 872                 s += " at %s\n" % time.strftime("%Y-%m-%d %H:%M:%S")
 873
 874                 tbl = StandingsTable()
 875                 tbl.headings()
 876
 877                 for pa in track.aboard():
 878                         pi = pa.pirate_info()
 879
 880                         xs = ''
 881                         if pa.gunner: xs += 'G '
 882                         else: xs += '  '
 883                         xs += timeevent(pa.last_time, pa.last_event)
 884                         xs += timeevent(pa.last_chat_time, pa.last_chat_chan)
 885
 886                         if pi is None:
 887                                 tbl.pirate_dummy(pa.name, rotate_nya[0], xs)
 888                         else:
 889                                 tbl.pirate(pi, xs)
 890
 891                 s += tbl.results()
 892
 893                 displayer.show(s)
 894                 time.sleep(1)
 895                 rotate_nya = rotate_nya[1:3] + rotate_nya[0]
 896
 897 #---------- main program ----------
 898
 899 def main():
 900         global opts, fetcher
 901
 902         pa = OptionParser(
 903 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
 904 actions:
 905  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
 906  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
 907  yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
 908  yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG
 909  yoweb-scrape [options] ship-aid CHAT-LOG  (must be .../PIRATE_OCEAN_chat-log*)
 910
 911 display modes (for --display) apply to ship-aid:
 912  --display=dumb       just print new information, scrolling the screen
 913  --display=overwrite  use cursor motion, selective clear, etc. to redraw at top
 914 ''')
 915         ao = pa.add_option
 916         ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
 917                 help='select ocean OCEAN')
 918         ao('--cache-dir', dest='cache_dir', metavar='DIR',
 919                 default='~/.yoweb-scrape-cache',
 920                 help='cache yoweb pages in DIR')
 921         ao('-D','--debug', action='count', dest='debug', default=0,
 922                 help='enable debugging output')
 923         ao('--debug-fd', type='int', dest='debug_fd',
 924                 help='write any debugging output to specified fd')
 925         ao('-q','--quiet', action='store_true', dest='quiet',
 926                 help='suppress warning output')
 927         ao('--display', action='store', dest='display',
 928                 type='choice', choices=['dumb','overwrite'],
 929                 help='how to display ship aid')
 930
 931         ao('--ship-duty', action='store_true', dest='ship_duty',
 932                 help='show ship duty station puzzles')
 933         ao('--all-puzzles', action='store_false', dest='ship_duty',
 934                 help='show all puzzles, not just ship duty stations')
 935
 936         (opts,args) = pa.parse_args()
 937         random.seed()
 938
 939         if len(args) < 1:
 940                 pa.error('need a mode argument')
 941
 942         if opts.debug_fd is not None:
 943                 opts.debug_file = os.fdopen(opts.debug_fd, 'w')
 944         else:
 945                 opts.debug_file = sys.stdout
 946
 947         mode = args[0]
 948         mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
 949         try: mode_fn = globals()[mode_fn_name]
 950         except KeyError: pa.error('unknown mode "%s"' % mode)
 951
 952         # fixed parameters
 953         opts.min_max_age = 60
 954         opts.expire_age = 3600
 955         opts.ship_reboard_clearout = 3600
 956
 957         if opts.cache_dir.startswith('~/'):
 958                 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
 959
 960         if opts.display is None:
 961                 if ((opts.debug > 0 and opts.debug_fd is None)
 962                     or not os.isatty(sys.stdout.fileno())):
 963                         opts.display = 'dumb'
 964                 else:
 965                         opts.display = 'overwrite'
 966
 967         fetcher = Fetcher(opts.ocean, opts.cache_dir)
 968
 969         mode_fn(args[1:], pa.error)
 970
 971 main()