yoweb-scrape

   1 #!/usr/bin/python
   2 # This is part of ypp-sc-tools, a set of third-party tools for assisting
   3 # players of Yohoho Puzzle Pirates.
   4 #
   5 # Copyright (C) 2009 Ian Jackson <ijackson@chiark.greenend.org.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 #
  20 # Yohoho and Puzzle Pirates are probably trademarks of Three Rings and
  21 # are used without permission.  This program is not endorsed or
  22 # sponsored by Three Rings.
  23
  24 copyright_info = '''
  25 yoweb-scrape is part of ypp-sc-tools  Copyright (C) 2009 Ian Jackson
  26 This program comes with ABSOLUTELY NO WARRANTY; this is free software,
  27 and you are welcome to redistribute it under certain conditions.
  28 For details, read the top of the yoweb-scrape file.
  29 '''
  30
  31 #---------- setup ----------
  32
  33 import signal
  34 signal.signal(signal.SIGINT, signal.SIG_DFL)
  35
  36 import os
  37 import time
  38 import urllib
  39 import urllib2
  40 import errno
  41 import sys
  42 import re as regexp
  43 import random
  44 import curses
  45 import termios
  46 import random
  47 from optparse import OptionParser
  48 from StringIO import StringIO
  49
  50 from BeautifulSoup import BeautifulSoup
  51
  52 opts = None
  53
  54 #---------- YPP parameters and arrays ----------
  55
  56 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
  57         '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
  58         '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
  59         '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
  60
  61 core_duty_puzzles = [
  62                 'Gunning',
  63                 ['Sailing','Rigging'],
  64                 'Bilging',
  65                 'Carpentry',
  66                 ]
  67
  68 duty_puzzles = ([ 'Navigating', 'Battle Navigation' ] +
  69                 core_duty_puzzles +
  70                 [ 'Treasure Haul' ])
  71
  72 standingvals = ('Able/Proficient/Distinguished/Respected/Master'+
  73                 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
  74 standing_limit = len(standingvals)
  75
  76 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
  77
  78 max_pirate_namelen = 12
  79
  80
  81 #---------- general utilities ----------
  82
  83 def debug(m):
  84         if opts.debug > 0:
  85                 print >>opts.debug_file, m
  86
  87 def debug_flush():
  88         if opts.debug > 0:
  89                 opts.debug_file.flush()
  90
  91 def sleep(seconds):
  92         debug_flush()
  93         time.sleep(seconds)
  94
  95 def format_time_interval(ti):
  96         if ti < 120: return '%d:%02d' % (ti / 60, ti % 60)
  97         if ti < 7200: return '%2dm' % (ti / 60)
  98         if ti < 86400: return '%dh' % (ti / 3600)
  99         return '%dd' % (ti / 86400)
 100
 101 #---------- caching and rate-limiting data fetcher ----------
 102
 103 class Fetcher:
 104         def __init__(self, ocean, cachedir):
 105                 debug('Fetcher init %s' % cachedir)
 106                 self.ocean = ocean
 107                 self.cachedir = cachedir
 108                 try: os.mkdir(cachedir)
 109                 except (OSError,IOError), oe:
 110                         if oe.errno != errno.EEXIST: raise
 111                 self._cache_scan(time.time())
 112
 113         def default_ocean(self, ocean='ice'):
 114                 if self.ocean is None:
 115                         self.ocean = ocean
 116
 117         def _cache_scan(self, now):
 118                 # returns list of ages, unsorted
 119                 ages = []
 120                 debug('Fetcher   scan_cache')
 121                 for leaf in os.listdir(self.cachedir):
 122                         if not leaf.startswith('#'): continue
 123                         path = self.cachedir + '/' + leaf
 124                         try: s = os.stat(path)
 125                         except (OSError,IOError), oe:
 126                                 if oe.errno != errno.ENOENT: raise
 127                                 continue
 128                         age = now - s.st_mtime
 129                         if age > opts.expire_age:
 130                                 debug('Fetcher    expire %d %s' % (age, path))
 131                                 try: os.remove(path)
 132                                 except (OSError,IOError), oe:
 133                                         if oe.errno != errno.ENOENT: raise
 134                                 continue
 135                         ages.append(age)
 136                 return ages
 137
 138         def need_wait(self, now, imaginary=[]):
 139                 ages = self._cache_scan(now)
 140                 ages += imaginary
 141                 ages.sort()
 142                 debug('Fetcher   ages ' + `ages`)
 143                 min_age = 1
 144                 need_wait = 0
 145                 for age in ages:
 146                         if age < min_age and age <= 5:
 147                                 debug('Fetcher   morewait min=%d age=%d' %
 148                                         (min_age, age))
 149                                 need_wait = max(need_wait, min_age - age)
 150                         min_age += 3
 151                         min_age *= 1.25
 152                 if need_wait > 0:
 153                         need_wait += random.random() - 0.5
 154                 return need_wait
 155
 156         def _rate_limit_cache_clean(self, now):
 157                 need_wait = self.need_wait(now)
 158                 if need_wait > 0:
 159                         debug('Fetcher   wait %d' % need_wait)
 160                         sleep(need_wait)
 161
 162         def fetch(self, url, max_age):
 163                 debug('Fetcher fetch %s' % url)
 164                 cache_corename = urllib.quote_plus(url)
 165                 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
 166                 try: f = file(cache_item, 'r')
 167                 except (OSError,IOError), oe:
 168                         if oe.errno != errno.ENOENT: raise
 169                         f = None
 170                 now = time.time()
 171                 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
 172                 if f is not None:
 173                         s = os.fstat(f.fileno())
 174                         age = now - s.st_mtime
 175                         if age > max_age:
 176                                 debug('Fetcher  stale %d < %d'% (max_age, age))
 177                                 f = None
 178                 if f is not None:
 179                         data = f.read()
 180                         f.close()
 181                         debug('Fetcher  cached %d > %d' % (max_age, age))
 182                         return data
 183
 184                 debug('Fetcher  fetch')
 185                 self._rate_limit_cache_clean(now)
 186
 187                 stream = urllib2.urlopen(url)
 188                 data = stream.read()
 189                 cache_tmp = "%s/#%s~%d#" % (
 190                         self.cachedir, cache_corename, os.getpid())
 191                 f = file(cache_tmp, 'w')
 192                 f.write(data)
 193                 f.close()
 194                 os.rename(cache_tmp, cache_item)
 195                 debug('Fetcher  stored')
 196                 return data
 197
 198         def yoweb(self, kind, tail, max_age):
 199                 self.default_ocean()
 200                 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
 201                         self.ocean, kind, tail)
 202                 return self.fetch(url, max_age)
 203
 204 #---------- logging assistance for troubled screenscrapers ----------
 205
 206 class SoupLog:
 207         def __init__(self):
 208                 self.msgs = [ ]
 209         def msg(self, m):
 210                 self.msgs.append(m)
 211         def soupm(self, obj, m):
 212                 self.msg(m + '; in ' + `obj`)
 213         def needs_msgs(self, child_souplog):
 214                 self.msgs += child_souplog.msgs
 215                 child_souplog.msgs = [ ]
 216
 217 def soup_text(obj):
 218         str = ''.join(obj.findAll(text=True))
 219         return str.strip()
 220
 221 class SomethingSoupInfo(SoupLog):
 222         def __init__(self, kind, tail, max_age):
 223                 SoupLog.__init__(self)
 224                 html = fetcher.yoweb(kind, tail, max_age)
 225                 self._soup = BeautifulSoup(html,
 226                         convertEntities=BeautifulSoup.HTML_ENTITIES
 227                         )
 228
 229 #---------- scraper for pirate pages ----------
 230
 231 class PirateInfo(SomethingSoupInfo):
 232         # Public data members:
 233         #  pi.standings = { 'Treasure Haul': 'Able' ... }
 234         #  pi.name = name
 235         #  pi.crew = (id, name)
 236         #  pi.flag = (id, name)
 237         #  pi.msgs = [ 'message describing problem with scrape' ]
 238
 239         def __init__(self, pirate, max_age=300):
 240                 SomethingSoupInfo.__init__(self,
 241                         'pirate.wm?target=', pirate, max_age)
 242                 self.name = pirate
 243                 self._find_standings()
 244                 self.crew = self._find_crewflag('crew',
 245                         '^/yoweb/crew/info\\.wm')
 246                 self.flag = self._find_crewflag('flag',
 247                         '^/yoweb/flag/info\\.wm')
 248
 249         def _find_standings(self):
 250                 imgs = self._soup.findAll('img',
 251                         src=regexp.compile('/yoweb/images/stat.*'))
 252                 re = regexp.compile(
 253 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
 254                         )
 255                 standings = { }
 256
 257                 for skill in puzzles:
 258                         standings[skill] = [ ]
 259
 260                 skl = SoupLog()
 261
 262                 for img in imgs:
 263                         try: puzzle = img['alt']
 264                         except KeyError: continue
 265
 266                         if not puzzle in puzzles:
 267                                 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
 268                                 continue
 269                         key = img.findParent('td')
 270                         if key is None:
 271                                 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
 272                                 continue
 273                         valelem = key.findNextSibling('td')
 274                         if valelem is None:
 275                                 skl.soupm(key, 'puzzle missing sibling "%s"'
 276                                         % puzzle)
 277                                 continue
 278                         valstr = soup_text(valelem)
 279                         match = re.match(valstr)
 280                         if match is None:
 281                                 skl.soupm(key, ('puzzle "%s" unparseable'+
 282                                         ' standing "%s"') % (puzzle, valstr))
 283                                 continue
 284                         standing = match.group(match.lastindex)
 285                         standings[puzzle].append(standing)
 286
 287                 self.standings = { }
 288
 289                 for puzzle in puzzles:
 290                         sl = standings[puzzle]
 291                         if len(sl) > 1:
 292                                 skl.msg('puzzle "%s" multiple standings %s' %
 293                                                 (puzzle, `sl`))
 294                                 continue
 295                         if not sl:
 296                                 skl.msg('puzzle "%s" no standing found' % puzzle)
 297                                 continue
 298                         standing = sl[0]
 299                         for i in range(0, standing_limit):
 300                                 if standing == standingvals[i]:
 301                                         self.standings[puzzle] = i
 302                         if not puzzle in self.standings:
 303                                 skl.msg('puzzle "%s" unknown standing "%s"' %
 304                                         (puzzle, standing))
 305
 306                 all_standings_ok = True
 307                 for puzzle in puzzles:
 308                         if not puzzle in self.standings:
 309                                 self.needs_msgs(skl)
 310
 311         def _find_crewflag(self, cf, yoweb_re):
 312                 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
 313                 if len(things) != 1:
 314                         self.msg('zero or several %s id references found' % cf)
 315                         return None
 316                 thing = things[0]
 317                 id_re = '\\b%sid\\=(\\w+)$' % cf
 318                 id_haystack = thing['href']
 319                 match = regexp.compile(id_re).search(id_haystack)
 320                 if match is None:
 321                         self.soupm(thing, ('incomprehensible %s id ref'+
 322                                 ' (%s in %s)') % (cf, id_re, id_haystack))
 323                         return None
 324                 name = soup_text(thing)
 325                 return (match.group(1), name)
 326
 327         def __str__(self):
 328                 return `(self.crew, self.flag, self.standings, self.msgs)`
 329
 330 #---------- scraper for crew pages ----------
 331
 332 class CrewInfo(SomethingSoupInfo):
 333         # Public data members:
 334         #  ci.crew = [ ('Captain',        ['Pirate', ...]),
 335         #              ('Senior Officer', [...]),
 336         #               ... ]
 337         #  pi.msgs = [ 'message describing problem with scrape' ]
 338
 339         def __init__(self, crewid, max_age=300):
 340                 SomethingSoupInfo.__init__(self,
 341                         'crew/info.wm?crewid=', crewid, max_age)
 342                 self._find_crew()
 343
 344         def _find_crew(self):
 345                 self.crew = []
 346                 capts = self._soup.findAll('img',
 347                         src='/yoweb/images/crew-captain.png')
 348                 if len(capts) != 1:
 349                         self.msg('crew members: no. of captain images != 1')
 350                         return
 351                 tbl = capts[0]
 352                 while not tbl.find('a', href=pirate_ref_re):
 353                         tbl = tbl.findParent('table')
 354                         if not tbl:
 355                                 self.msg('crew members: cannot find table')
 356                                 return
 357                 current_rank_crew = None
 358                 crew_rank_re = regexp.compile('/yoweb/images/crew')
 359                 for row in tbl.contents:
 360                         # findAll(recurse=False)
 361                         if isinstance(row,basestring):
 362                                 continue
 363
 364                         is_rank = row.find('img', attrs={'src': crew_rank_re})
 365                         if is_rank:
 366                                 rank = soup_text(row)
 367                                 current_rank_crew = []
 368                                 self.crew.append((rank, current_rank_crew))
 369                                 continue
 370                         for cell in row.findAll('a', href=pirate_ref_re):
 371                                 if current_rank_crew is None:
 372                                         self.soupm(cell, 'crew members: crew'
 373                                                 ' before rank')
 374                                         continue
 375                                 current_rank_crew.append(soup_text(cell))
 376
 377         def __str__(self):
 378                 return `(self.crew, self.msgs)`
 379
 380 #---------- pretty-printer for tables of pirate puzzle standings ----------
 381
 382 class StandingsTable:
 383         def __init__(self, f, use_puzzles=None, col_width=6, gap_every=5):
 384                 if use_puzzles is None:
 385                         if opts.ship_duty:
 386                                 use_puzzles=duty_puzzles
 387                         else:
 388                                 use_puzzles=puzzles
 389                 self._puzzles = use_puzzles
 390                 self.f = f
 391                 self._cw = col_width-1
 392                 self._gap_every = gap_every
 393                 self._linecount = 0
 394                 self._o = f.write
 395
 396         def _nl(self): self._o('\n')
 397
 398         def _pline(self, lhs, puzstrs, extra):
 399                 if (self._linecount > 0
 400                     and self._gap_every is not None
 401                     and not (self._linecount % self._gap_every)):
 402                         self._nl()
 403                 self._o('%-*s' % (max(max_pirate_namelen+1, 15), lhs))
 404                 for v in puzstrs:
 405                         self._o(' %-*.*s' % (self._cw,self._cw, v))
 406                 if extra:
 407                         self._o(' ' + extra)
 408                 self._nl()
 409                 self._linecount += 1
 410
 411         def _puzstr(self, pi, puzzle):
 412                 if not isinstance(puzzle,list): puzzle = [puzzle]
 413                 try: standing = max([pi.standings[p] for p in puzzle])
 414                 except KeyError: return '?'
 415                 if not standing: return ''
 416                 s = ''
 417                 if self._cw > 4:
 418                         c1 = standingvals[standing][0]
 419                         if standing < 3: c1 = c1.lower() # 3 = Master
 420                         s += `standing`
 421                 if self._cw > 5:
 422                         s += ' '
 423                 s += '*' * (standing / 2)
 424                 s += '+' * (standing % 2)
 425                 return s
 426
 427         def headings(self, lhs='', rhs=None):
 428                 def puzn_redact(name):
 429                         if isinstance(name,list):
 430                                 return '/'.join(
 431                                         ["%.*s" % (self._cw/2, puzn_redact(n))
 432                                          for n in name])
 433                         spc = name.find(' ')
 434                         if spc < 0: return name
 435                         return name[0:min(4,spc)] + name[spc+1:]
 436                 self._linecount = -2
 437                 self._pline(lhs, map(puzn_redact, self._puzzles), rhs)
 438                 self._linecount = 0
 439         def literalline(self, line):
 440                 self._o(line)
 441                 self._nl()
 442                 self._linecount = 0
 443         def pirate_dummy(self, name, standingstring, extra=None):
 444                 standings = standingstring * len(self._puzzles)
 445                 self._pline(' '+name, standings, extra)
 446         def pirate(self, pi, extra=None):
 447                 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
 448                 self._pline(' '+pi.name, puzstrs, extra)
 449
 450
 451 #---------- chat log parser ----------
 452
 453 class PirateAboard:
 454         # This is essentially a transparent, dumb, data class.
 455         #  pa.v                 may be None
 456         #  pa.name
 457         #  pa.last_time
 458         #  pa.last_event
 459         #  pa.gunner
 460         #  pa.last_chat_time
 461         #  pa.last_chat_chan
 462         #  pa.pi
 463
 464         # Also used for jobbing applicants:
 465         #               happens when                    expires (to "-")
 466         #   -            disembark, leaves crew          no
 467         #   aboard       evidence of them being aboard   no
 468         #   applied      "has applied for the job"       120s, configurable
 469         #   ashore       "has taken a job"               30min, configurable
 470         #   declined     "declined the job offer"        30s, configurable
 471         #   invited      "has been invited to job"       120s, configurable
 472         #
 473         #  pa.jobber    None, 'ashore', 'applied', 'invited', 'declined'
 474         #  pa.expires   expiry time time
 475
 476         def __init__(pa, pn, v, time, event):
 477                 pa.name = pn
 478                 pa.v = v
 479                 pa.last_time = time
 480                 pa.last_event = event
 481                 pa.last_chat_time = None
 482                 pa.last_chat_chan = None
 483                 pa.gunner = False
 484                 pa.pi = None
 485                 pa.jobber = None
 486                 pa.expires = None
 487
 488         def pirate_info(pa):
 489                 now = time.time()
 490                 if pa.pi:
 491                         age = now - pa.pi_fetched
 492                         guide = random.randint(120,240)
 493                         if age <= guide:
 494                                 return pa.pi
 495                         debug('PirateAboard refresh %d > %d  %s' % (
 496                                 age, guide, pa.name))
 497                         imaginary = [2,4]
 498                 else:
 499                         imaginary = [1]
 500                 wait = fetcher.need_wait(now, imaginary)
 501                 if wait:
 502                         debug('PirateAboard fetcher not ready %d' % wait)
 503                         return pa.pi
 504                 pa.pi = PirateInfo(pa.name, 600)
 505                 pa.pi_fetched = now
 506                 return pa.pi
 507
 508 class ChatLogTracker:
 509         # This is quite complex so we make it opaque.  Use the
 510         # official invokers, accessors etc.
 511
 512         def __init__(self, myself_pi, logfn):
 513                 self._pl = {}   # self._pl['Pirate'] =
 514                 self._vl = {}   #   self._vl['Vessel']['Pirate'] = PirateAboard
 515                                 # self._vl['Vessel']['#lastinfo']
 516                                 # self._vl['Vessel']['#name']
 517                                 # self._v = self._vl[self._vessel]
 518                 self._date = None
 519                 self._myself = myself_pi
 520                 self._lbuf = ''
 521                 self._f = file(logfn)
 522                 flen = os.fstat(self._f.fileno()).st_size
 523                 max_backlog = 500000
 524                 if flen > max_backlog:
 525                         startpos = flen - max_backlog
 526                         self._f.seek(startpos)
 527                         self._f.readline()
 528                 self._progress = [0, flen - self._f.tell()]
 529                 self._disembark_myself()
 530                 self._need_redisplay = False
 531                 self._lastvessel = None
 532
 533         def _disembark_myself(self):
 534                 self._v = None
 535                 self._vessel = None
 536                 self.force_redisplay()
 537
 538         def force_redisplay(self):
 539                 self._need_redisplay = True
 540
 541         def _vessel_updated(self, v, timestamp):
 542                 if v is None: return
 543                 v['#lastinfo'] = timestamp
 544                 self.force_redisplay()
 545
 546         def _onboard_event(self,v,timestamp,pirate,event,jobber=None):
 547                 pa = self._pl.get(pirate, None)
 548                 if pa is not None and pa.v is v:
 549                         pa.last_time = timestamp
 550                         pa.last_event = event
 551                 else:
 552                         if pa is not None and pa.v is not None:
 553                                 del pa.v[pirate]
 554                         pa = PirateAboard(pirate, v, timestamp, event)
 555                         self._pl[pirate] = pa
 556                         if v is not None: v[pirate] = pa
 557                 pa.jobber = jobber
 558
 559                 if jobber is None: timeout = None
 560                 else: timeout = getattr(opts, 'timeout_'+jobber)
 561                 if timeout is None: pa.expires = None
 562                 else: pa.expires = timestamp + timeout
 563                 self._vessel_updated(v, timestamp)
 564                 return pa
 565
 566         def _expire_jobbers(self, now):
 567                 for pa in self._pl.values():
 568                         if pa.expires is None: continue
 569                         if pa.expires >= now: continue
 570                         v = pa.v
 571                         del self._pl[pa.name]
 572                         if v is not None: del v[pa.name]
 573                         self.force_redisplay()
 574
 575         def _trash_vessel(self, v):
 576                 for pn in v:
 577                         if pn.startswith('#'): continue
 578                         del self._pl[pn]
 579                 vn = v['#name']
 580                 del self._vl[vn]
 581                 if v is self._v: self._disembark_myself()
 582                 self.force_redisplay()
 583
 584         def _vessel_stale(self, v, timestamp):
 585                 return timestamp - v['#lastinfo'] > opts.ship_reboard_clearout
 586
 587         def _vessel_check_expire(self, v, timestamp):
 588                 if not self._vessel_stale(v, timestamp):
 589                         return v
 590                 self._debug_line_disposition(timestamp,'',
 591                         'stale-reset ' + v['#name'])
 592                 self._trash_vessel(v)
 593                 return None
 594
 595         def expire_garbage(self, timestamp):
 596                 for v in self._vl.values():
 597                         self._vessel_check_expire(v, timestamp)
 598
 599         def _vessel_lookup(self, vn, timestamp, dml=[], create=False):
 600                 v = self._vl.get(vn, None)
 601                 if v is not None:
 602                         v = self._vessel_check_expire(v, timestamp)
 603                 if v is not None:
 604                         dml.append('found')
 605                         return v
 606                 if not create:
 607                         dml.append('no')
 608                 dml.append('new')
 609                 self._vl[vn] = v = { '#name': vn }
 610                 self._vessel_updated(v, timestamp)
 611                 return v
 612
 613         def _find_matching_vessel(self, pattern, timestamp, cmdr,
 614                                         dml=[], create=False):
 615                 # use when a commander pirate `cmdr' specified a vessel
 616                 #  by name `pattern' (either may be None)
 617                 # if create is true, will create the vessel
 618                 #  record if an exact name is specified
 619
 620                 if (pattern is not None and
 621                     not '*' in pattern
 622                     and len(pattern.split(' ')) == 2):
 623                         vn = pattern.title()
 624                         dml.append('exact')
 625                         return self._vessel_lookup(
 626                                 vn, timestamp, dml=dml, create=create)
 627
 628                 if pattern is None:
 629                         pattern_check = lambda vn: True
 630                 else:
 631                         re = '(?:.* )?%s$' % pattern.lower().replace('*','.+')
 632                         pattern_check = regexp.compile(re, regexp.I).match
 633
 634                 tries = []
 635
 636                 cmdr_pa = self._pl.get(cmdr, None)
 637                 if cmdr_pa: tries.append((cmdr_pa.v, 'cmdr'))
 638
 639                 tries.append((self._v, 'here'))
 640                 tried_vns = []
 641
 642                 for (v, dm) in tries:
 643                         if v is None: dml.append(dm+'?'); continue
 644
 645                         vn = v['#name']
 646                         if not pattern_check(vn):
 647                                 tried_vns.append(vn)
 648                                 dml.append(dm+'#')
 649                                 continue
 650
 651                         dml.append(dm+'!')
 652                         return v
 653
 654                 if pattern is not None and '*' in pattern:
 655                         search = [
 656                                 (vn,v)
 657                                 for (vn,v) in self._vl.iteritems()
 658                                 if not self._vessel_stale(v, timestamp)
 659                                 if pattern_check(vn)
 660                                 ]
 661                         #debug('CLT-RE /%s/ wanted (%s) searched (%s)' % (
 662                         #       re,
 663                         #       '/'.join(tried_vns),
 664                         #       '/'.join([vn for (vn,v) in search])))
 665
 666                         if len(search)==1:
 667                                 dml.append('one')
 668                                 return search[0][1]
 669                         elif search:
 670                                 dml.append('many')
 671                         else:
 672                                 dml.append('none')
 673
 674         def _debug_line_disposition(self,timestamp,l,m):
 675                 debug('CLT %13s %-40s %s' % (timestamp,m,l))
 676
 677         def _rm_crew_l(self,re,l):
 678                 m = regexp.match(re,l)
 679                 if m and m.group(2) == self._myself.crew[1]:
 680                         return m.group(1)
 681                 else:
 682                         return None
 683
 684         def chatline(self,l):
 685                 rm = lambda re: regexp.match(re,l)
 686                 d = lambda m: self._debug_line_disposition(timestamp,l,m)
 687                 rm_crew = lambda re: self._rm_crew_l(re,l)
 688                 timestamp = None
 689
 690                 m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$')
 691                 if m:
 692                         self._date = [int(x) for x in m.groups()]
 693                         self._previous_timestamp = None
 694                         return d('date '+`self._date`)
 695
 696                 if self._date is None:
 697                         return d('date unset')
 698
 699                 m = rm('\\[(\d\d):(\d\d):(\d\d)\\] ')
 700                 if not m:
 701                         return d('no timestamp')
 702
 703                 while True:
 704                         time_tuple = (self._date +
 705                                       [int(x) for x in m.groups()] +
 706                                       [-1,-1,-1])
 707                         timestamp = time.mktime(time_tuple)
 708                         if timestamp >= self._previous_timestamp: break
 709                         self._date[2] += 1
 710                         self._debug_line_disposition(timestamp,'',
 711                                 'new date '+`self._date`)
 712
 713                 self._previous_timestamp = timestamp
 714
 715                 l = l[l.find(' ')+1:]
 716
 717                 def ob_x(pirate,event):
 718                         return self._onboard_event(
 719                                         self._v, timestamp, pirate, event)
 720                 def ob1(did): ob_x(m.group(1), did); return d(did)
 721                 def oba(did): return ob1('%s %s' % (did, m.group(2)))
 722
 723                 def jb(pirate,jobber):
 724                         return self._onboard_event(
 725                                 None, timestamp, pirate,
 726                                 ("jobber %s" % jobber),
 727                                 jobber=jobber
 728                                 )
 729
 730                 def disembark(v, timestamp, pirate, event):
 731                         self._onboard_event(
 732                                         v, timestamp, pirate, 'leaving '+event)
 733                         del v[pirate]
 734                         del self._pl[pirate]
 735
 736                 def disembark_me(why):
 737                         self._disembark_myself()
 738                         return d('disembark-me '+why)
 739
 740                 m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
 741                 if m:
 742                         dm = ['boarding']
 743                         pn = self._myself.name
 744                         vn = m.group(1)
 745                         v = self._vessel_lookup(vn, timestamp, dm, create=True)
 746                         self._lastvessel = self._vessel = vn
 747                         self._v = v
 748                         ob_x(pn, 'we boarded')
 749                         self.expire_garbage(timestamp)
 750                         return d(' '.join(dm))
 751
 752                 if self._v is None:
 753                         return d('no vessel')
 754
 755                 m = rm('(\\w+) has come aboard\\.$')
 756                 if m: return ob1('boarded');
 757
 758                 m = rm('You have ordered (\\w+) to do some (\\S.*\\S)\\.$')
 759                 if m:
 760                         (who,what) = m.groups()
 761                         pa = ob_x(who,'ord '+what)
 762                         if what == 'Gunning':
 763                                 pa.gunner = True
 764                         return d('duty order')
 765
 766                 m = rm('(\\w+) abandoned a (\\S.*\\S) station\\.$')
 767                 if m: oba('stopped'); return d("end")
 768
 769                 def chat_core(speaker, chan):
 770                         try: pa = self._pl[speaker]
 771                         except KeyError: return 'mystery'
 772                         if pa.v is not None and pa.v is not self._v:
 773                                 return 'elsewhere'
 774                         pa.last_chat_time = timestamp
 775                         pa.last_chat_chan = chan
 776                         self.force_redisplay()
 777                         return 'here'
 778
 779                 def chat(chan):
 780                         speaker = m.group(1)
 781                         dm = chat_core(speaker, chan)
 782                         return d('chat %s %s' % (chan, dm))
 783
 784                 def chat_metacmd(chan):
 785                         (cmdr, metacmd) = m.groups()
 786                         metacmd = regexp.sub('\\s+', ' ', metacmd).strip()
 787                         m2 = regexp.match(
 788                             '/([adj]) (?:([A-Za-z* ]+)\\s*:)?([A-Za-z ]+)$',
 789                             metacmd)
 790                         if not m2: return chat(chan)
 791
 792                         (cmd, pattern, targets) = m2.groups()
 793                         dml = ['cmd', chan, cmd]
 794
 795                         if cmd == 'a': each = self._onboard_event
 796                         elif cmd == 'd': each = disembark
 797                         else: each = lambda v,ts,tt,h: self._onboard_event(
 798                                         None,ts,tt,h, jobber='applied')
 799
 800                         if cmdr == self._myself.name:
 801                                 dml.append('self')
 802                                 how = 'cmd: %s' % cmd
 803                         else:
 804                                 dml.append('other')
 805                                 how = 'cmd: %s %s' % (cmd,cmdr)
 806
 807                         v = self._find_matching_vessel(
 808                                 pattern, timestamp, cmdr, dml, create=True)
 809
 810                         if v is not None:
 811                                 targets = targets.strip().split(' ')
 812                                 dml.append(`len(targets)`)
 813                                 for target in targets:
 814                                         each(v, timestamp, target.title(), how)
 815                                 self._vessel_updated(v, timestamp)
 816
 817                         dm = ' '.join(dml)
 818                         chat_core(cmdr, 'cmd '+chan)
 819                         return d(dm)
 820
 821                 m = rm('(\\w+) (?:issued an order|ordered everyone) "')
 822                 if m: return ob1('general order');
 823
 824                 m = rm('(\\w+) says, "')
 825                 if m: return chat('public')
 826
 827                 m = rm('(\\w+) tells ye, "')
 828                 if m: return chat('private')
 829
 830                 m = rm('Ye told (\\w+), "(.*)"$')
 831                 if m: return chat_metacmd('private')
 832
 833                 m = rm('(\\w+) flag officer chats, "')
 834                 if m: return chat('flag officer')
 835
 836                 m = rm('(\\w+) officer chats, "(.*)"$')
 837                 if m: return chat_metacmd('officer')
 838
 839                 m = rm('Ye accepted the offer to job with ')
 840                 if m: return disembark_me('jobbing')
 841
 842                 m = rm('Ye hop on the ferry and are whisked away ')
 843                 if m: return disembark_me('ferry')
 844
 845                 m = rm('Whisking away to yer home on the magical winds')
 846                 if m: return disembark_me('home')
 847
 848                 m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
 849                 if m:
 850                         pl = m.group(1).split(', ')
 851                         if not self._myself.name in pl:
 852                                 return d('lost melee')
 853                         for pn in pl:
 854                                 if ' ' in pn: continue
 855                                 ob_x(pn,'won melee')
 856                         return d('won melee')
 857
 858                 m = rm('(\\w+) is eliminated\\!')
 859                 if m: return ob1('eliminated in fray');
 860
 861                 m = rm('(\\w+) has driven \w+ from the ship\\!')
 862                 if m: return ob1('boarder repelled');
 863
 864                 m = rm('\w+ has bested (\\w+), and turns'+
 865                         ' to the rest of the ship\\.')
 866                 if m: return ob1('boarder unrepelled');
 867
 868                 pirate = rm_crew("(\\w+) has taken a job with '(.*)'\\.")
 869                 if pirate: return jb(pirate, 'ashore')
 870
 871                 pirate = rm_crew("(\\w+) has left '(.*)'\\.")
 872                 if pirate:
 873                         disembark(self._v, timestamp, pirate, 'left crew')
 874                         return d('left crew')
 875
 876                 m = rm('(\w+) has applied for the posted job\.')
 877                 if m: return jb(m.group(1), 'applied')
 878
 879                 pirate= rm_crew("(\\w+) has been invited to job for '(.*)'\\.")
 880                 if pirate: return jb(pirate, 'invited')
 881
 882                 pirate = rm_crew("(\\w+) declined the job offer for '(.*)'\\.")
 883                 if pirate: return jb(pirate, 'declined')
 884
 885                 m = rm('(\\w+) has left the vessel\.')
 886                 if m:
 887                         pirate = m.group(1)
 888                         disembark(self._v, timestamp, pirate, 'disembarked')
 889                         return d('disembarked')
 890
 891                 return d('not-matched')
 892
 893         def _str_pa(self, pn, pa):
 894                 assert self._pl[pn] == pa
 895                 s = ' '*20 + "%s %-*s %13s %-30s %13s %-20s %13s" % (
 896                         (' ','G')[pa.gunner],
 897                         max_pirate_namelen, pn,
 898                         pa.last_time, pa.last_event,
 899                         pa.last_chat_time, pa.last_chat_chan,
 900                         pa.jobber)
 901                 if pa.expires is not None:
 902                         s += " %-5d" % (pa.expires - pa.last_time)
 903                 s += "\n"
 904                 return s
 905
 906         def _str_vessel(self, vn, v):
 907                 s = ' vessel %s\n' % vn
 908                 s += ' '*20 + "%-*s   %13s\n" % (
 909                                 max_pirate_namelen, '#lastinfo',
 910                                 v['#lastinfo'])
 911                 assert v['#name'] == vn
 912                 for pn in sorted(v.keys()):
 913                         if pn.startswith('#'): continue
 914                         pa = v[pn]
 915                         assert pa.v == v
 916                         s += self._str_pa(pn,pa)
 917                 return s
 918
 919         def __str__(self):
 920                 s = '''<ChatLogTracker
 921  myself %s
 922  vessel %s
 923 '''                     % (self._myself.name, self._vessel)
 924                 assert ((self._v is None and self._vessel is None) or
 925                         (self._v is self._vl[self._vessel]))
 926                 if self._vessel is not None:
 927                         s += self._str_vessel(self._vessel, self._v)
 928                 for vn in sorted(self._vl.keys()):
 929                         if vn == self._vessel: continue
 930                         s += self._str_vessel(vn, self._vl[vn])
 931                 s += " elsewhere\n"
 932                 for p in self._pl:
 933                         pa = self._pl[p]
 934                         if pa.v is not None:
 935                                 assert pa.v[p] is pa
 936                                 assert pa.v in self._vl.values()
 937                         else:
 938                                 s += self._str_pa(pa.name, pa)
 939                 s += '>\n'
 940                 return s
 941
 942         def catchup(self, progress=None):
 943                 while True:
 944                         more = self._f.readline()
 945                         if not more: break
 946
 947                         self._progress[0] += len(more)
 948                         if progress: progress.progress(*self._progress)
 949
 950                         self._lbuf += more
 951                         if self._lbuf.endswith('\n'):
 952                                 self.chatline(self._lbuf.rstrip())
 953                                 self._lbuf = ''
 954                                 if opts.debug >= 2:
 955                                         debug(self.__str__())
 956                 self._expire_jobbers(time.time())
 957
 958                 if progress: progress.caughtup()
 959
 960         def changed(self):
 961                 rv = self._need_redisplay
 962                 self._need_redisplay = False
 963                 return rv
 964         def myname(self):
 965                 # returns our pirate name
 966                 return self._myself.name
 967         def vesselname(self):
 968                 # returns the vessel name we're aboard or None
 969                 return self._vessel
 970         def lastvesselname(self):
 971                 # returns the last vessel name we were aboard or None
 972                 return self._lastvessel
 973         def aboard(self, vesselname=True):
 974                 # returns a list of PirateAboard the vessel
 975                 #  sorted by pirate name
 976                 #  you can pass this None and you'll get []
 977                 #  or True for the current vessel (which is the default)
 978                 #  the returned value is a fresh list of persistent
 979                 #  PirateAboard objects
 980                 if vesselname is True: v = self._v
 981                 else: v = self._vl.get(vesselname.title())
 982                 if v is None: return []
 983                 return [ v[pn]
 984                          for pn in sorted(v.keys())
 985                          if not pn.startswith('#') ]
 986         def jobbers(self):
 987                 # returns a the jobbers' PirateAboards,
 988                 # sorted by jobber class and reverse of expiry time
 989                 l = [ pa
 990                       for pa in self._pl.values()
 991                       if pa.jobber is not None
 992                     ]
 993                 def compar_key(pa):
 994                         return (pa.jobber, -pa.expires)
 995                 l.sort(key = compar_key)
 996                 return l
 997
 998 #---------- implementations of actual operation modes ----------
 999
1000 def do_pirate(pirates, bu):
1001         print '{'
1002         for pirate in pirates:
1003                 info = PirateInfo(pirate)
1004                 print '%s: %s,' % (`pirate`, info)
1005         print '}'
1006
1007 def prep_crew_of(args, bu, max_age=300):
1008         if len(args) != 1: bu('crew-of takes one pirate name')
1009         pi = PirateInfo(args[0], max_age)
1010         if pi.crew is None: return None
1011         return CrewInfo(pi.crew[0], max_age)
1012
1013 def do_crew_of(args, bu):
1014         ci = prep_crew_of(args, bu)
1015         print ci
1016
1017 def do_standings_crew_of(args, bu):
1018         ci = prep_crew_of(args, bu, 60)
1019         tab = StandingsTable(sys.stdout)
1020         tab.headings()
1021         for (rank, members) in ci.crew:
1022                 if not members: continue
1023                 tab.literalline('')
1024                 tab.literalline('%s:' % rank)
1025                 for p in members:
1026                         pi = PirateInfo(p, random.randint(900,1800))
1027                         tab.pirate(pi)
1028
1029 class ProgressPrintPercentage:
1030         def __init__(self, f=sys.stdout):
1031                 self._f = f
1032         def progress_string(self,done,total):
1033                 return "scan chat logs %3d%%\r" % ((done*100) / total)
1034         def progress(self,*a):
1035                 self._f.write(self.progress_string(*a))
1036                 self._f.flush()
1037         def show_init(self, pirate, ocean):
1038                 print >>self._f, 'Starting up, %s on the %s ocean' % (
1039                         pirate, ocean)
1040         def caughtup(self):
1041                 self._f.write('                   \r')
1042                 self._f.flush()
1043
1044 #----- modes which use the chat log parser are quite complex -----
1045
1046 def prep_chat_log(args, bu,
1047                 progress=ProgressPrintPercentage(),
1048                 max_myself_age=3600):
1049         if len(args) != 1: bu('this action takes only chat log filename')
1050         logfn = args[0]
1051         logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_'
1052         match = regexp.match(logfn_re, logfn)
1053         if not match: bu('chat log filename is not in expected format')
1054         (pirate, ocean) = match.groups()
1055         fetcher.default_ocean(ocean)
1056
1057         progress.show_init(pirate, fetcher.ocean)
1058         myself = PirateInfo(pirate,max_myself_age)
1059         track = ChatLogTracker(myself, logfn)
1060
1061         opts.debug -= 2
1062         track.catchup(progress)
1063         opts.debug += 2
1064
1065         track.force_redisplay()
1066
1067         return (myself, track)
1068
1069 def do_track_chat_log(args, bu):
1070         (myself, track) = prep_chat_log(args, bu)
1071         while True:
1072                 track.catchup()
1073                 if track.changed():
1074                         print track
1075                 sleep(0.5 + 0.5 * random.random())
1076
1077 #----- ship management aid -----
1078
1079 class Display_dumb(ProgressPrintPercentage):
1080         def __init__(self):
1081                 ProgressPrintPercentage.__init__(self)
1082         def show(self, s):
1083                 print '\n\n', s;
1084         def realstart(self):
1085                 pass
1086
1087 class Display_overwrite(ProgressPrintPercentage):
1088         def __init__(self):
1089                 ProgressPrintPercentage.__init__(self)
1090
1091                 null = file('/dev/null','w')
1092                 curses.setupterm(fd=null.fileno())
1093
1094                 self._clear = curses.tigetstr('clear')
1095                 if not self._clear:
1096                         self._debug('missing clear!')
1097                         self.show = Display_dumb.show
1098                         return
1099
1100                 self._t = {'el':'', 'ed':''}
1101                 if not self._init_sophisticated():
1102                         for k in self._t.keys(): self._t[k] = ''
1103                         self._t['ho'] = self._clear
1104
1105         def _debug(self,m): debug('display overwrite: '+m)
1106
1107         def _init_sophisticated(self):
1108                 for k in self._t.keys():
1109                         s = curses.tigetstr(k)
1110                         self._t[k] = s
1111                 self._t['ho'] = curses.tigetstr('ho')
1112                 if not self._t['ho']:
1113                         cup = curses.tigetstr('cup')
1114                         self._t['ho'] = curses.tparm(cup,0,0)
1115                 missing = [k for k in self._t.keys() if not self._t[k]]
1116                 if missing:
1117                         self.debug('missing '+(' '.join(missing)))
1118                         return 0
1119                 return 1
1120
1121         def show(self, s):
1122                 w = sys.stdout.write
1123                 def wti(k): w(self._t[k])
1124
1125                 wti('ho')
1126                 nl = ''
1127                 for l in s.rstrip().split('\n'):
1128                         w(nl)
1129                         w(l)
1130                         wti('el')
1131                         nl = '\r\n'
1132                 wti('ed')
1133                 w(' ')
1134                 sys.stdout.flush()
1135
1136         def realstart(self):
1137                 sys.stdout.write(self._clear)
1138                 sys.stdout.flush()
1139
1140
1141 def do_ship_aid(args, bu):
1142         if opts.ship_duty is None: opts.ship_duty = True
1143
1144         displayer = globals()['Display_'+opts.display]()
1145
1146         (myself, track) = prep_chat_log(args, bu, progress=displayer)
1147
1148         displayer.realstart()
1149
1150         if os.isatty(0): kr_create = KeystrokeReader
1151         else: kr_create = DummyKeystrokeReader
1152
1153         try:
1154                 kreader = kr_create(0, 10)
1155                 ship_aid_core(myself, track, displayer, kreader)
1156         finally:
1157                 kreader.stop()
1158                 print '\n'
1159
1160 class KeyBasedSorter:
1161         def compar_key_pa(self, pa):
1162                 pi = pa.pirate_info()
1163                 if pi is None: return None
1164                 return self.compar_key(pi)
1165         def lsort_pa(self, l):
1166                 l.sort(key = self.compar_key_pa)
1167
1168 class NameSorter(KeyBasedSorter):
1169         def compar_key(self, pi): return pi.name
1170         def desc(self): return 'name'
1171
1172 class SkillSorter(NameSorter):
1173         def __init__(self, relevant):
1174                 self._want = frozenset(relevant.split('/'))
1175                 self._avoid = set()
1176                 for p in core_duty_puzzles:
1177                         if isinstance(p,basestring): self._avoid.add(p)
1178                         else: self._avoid |= set(p)
1179                 self._avoid -= self._want
1180                 self._desc = '%s' % relevant
1181
1182         def desc(self): return self._desc
1183
1184         def compar_key(self, pi):
1185                 best_want = max([
1186                         pi.standings.get(puz,-1)
1187                         for puz in self._want
1188                         ])
1189                 best_avoid = [
1190                         -pi.standings.get(puz,standing_limit)
1191                         for puz in self._avoid
1192                         ]
1193                 best_avoid.sort()
1194                 def negate(x): return -x
1195                 debug('compar_key %s bw=%s ba=%s' % (pi.name, `best_want`,
1196                         `best_avoid`))
1197                 return (-best_want, map(negate, best_avoid), pi.name)
1198
1199 def ship_aid_core(myself, track, displayer, kreader):
1200
1201         def find_vessel():
1202                 vn = track.vesselname()
1203                 if vn: return (vn, " on board the %s" % vn)
1204                 vn = track.lastvesselname()
1205                 if vn: return (vn, " ashore from the %s" % vn)
1206                 return (None, " not on a vessel")
1207
1208         def timeevent(t,e):
1209                 if t is None: return ' ' * 22
1210                 return " %-4s %-16s" % (format_time_interval(now - t),e)
1211
1212         displayer.show(track.myname() + find_vessel()[1] + '...')
1213
1214         rotate_nya = '/-\\'
1215
1216         sort = NameSorter()
1217
1218         while True:
1219                 track.catchup()
1220                 now = time.time()
1221
1222                 (vn, s) = find_vessel()
1223                 s = track.myname() + s
1224                 s += " at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
1225                 s += kreader.info()
1226                 s += '\n'
1227
1228                 tbl_s = StringIO()
1229                 tbl = StandingsTable(tbl_s)
1230
1231                 aboard = track.aboard(vn)
1232                 sort.lsort_pa(aboard)
1233
1234                 jobbers = track.jobbers()
1235
1236                 if track.vesselname(): howmany = 'aboard: %2d' % len(aboard)
1237                 else: howmany = ''
1238
1239                 tbl.headings(howmany, '  sorted by '+sort.desc())
1240
1241                 last_jobber = None
1242
1243                 for pa in aboard + jobbers:
1244                         if pa.jobber != last_jobber:
1245                                 last_jobber = pa.jobber
1246                                 tbl.literalline('')
1247                                 tbl.literalline('jobbers '+last_jobber)
1248
1249                         pi = pa.pirate_info()
1250
1251                         xs = ''
1252                         if pa.gunner: xs += 'G '
1253                         else: xs += '  '
1254                         xs += timeevent(pa.last_time, pa.last_event)
1255                         xs += timeevent(pa.last_chat_time, pa.last_chat_chan)
1256
1257                         if pi is None:
1258                                 tbl.pirate_dummy(pa.name, rotate_nya[0], xs)
1259                         else:
1260                                 tbl.pirate(pi, xs)
1261
1262                 s += tbl_s.getvalue()
1263                 displayer.show(s)
1264                 tbl_s.close()
1265
1266                 k = kreader.getch()
1267                 if k is None:
1268                         rotate_nya = rotate_nya[1:3] + rotate_nya[0]
1269                         continue
1270
1271                 if k == 'q': break
1272                 elif k == 'g': sort = SkillSorter('Gunning')
1273                 elif k == 'c': sort = SkillSorter('Carpentry')
1274                 elif k == 's': sort = SkillSorter('Sailing/Rigging')
1275                 elif k == 'b': sort = SkillSorter('Bilging')
1276                 elif k == 'n': sort = SkillSorter('Navigating')
1277                 elif k == 'd': sort = SkillSorter('Battle Navigation')
1278                 elif k == 't': sort = SkillSorter('Treasure Haul')
1279                 elif k == 'a': sort = NameSorter()
1280                 else: pass # unknown key command
1281
1282 #---------- individual keystroke input ----------
1283
1284 class DummyKeystrokeReader:
1285         def __init__(self,fd,timeout_dummy): pass
1286         def stop(self): pass
1287         def getch(self): sleep(1); return None
1288         def info(self): return ' [noninteractive]'
1289
1290 class KeystrokeReader(DummyKeystrokeReader):
1291         def __init__(self, fd, timeout_decisec=0):
1292                 self._fd = fd
1293                 self._saved = termios.tcgetattr(fd)
1294                 a = termios.tcgetattr(fd)
1295                 a[3] &= ~(termios.ECHO | termios.ECHONL |
1296                           termios.ICANON | termios.IEXTEN)
1297                 a[6][termios.VMIN] = 0
1298                 a[6][termios.VTIME] = timeout_decisec
1299                 termios.tcsetattr(fd, termios.TCSANOW, a)
1300         def stop(self):
1301                 termios.tcsetattr(self._fd, termios.TCSANOW, self._saved)
1302         def getch(self):
1303                 debug_flush()
1304                 byte = os.read(self._fd, 1)
1305                 if not len(byte): return None
1306                 return byte
1307         def info(self):
1308                 return ''
1309
1310 #---------- main program ----------
1311
1312 def main():
1313         global opts, fetcher
1314
1315         pa = OptionParser(
1316 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
1317 actions:
1318  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
1319  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
1320  yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
1321  yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG
1322  yoweb-scrape [options] ship-aid CHAT-LOG  (must be .../PIRATE_OCEAN_chat-log*)
1323
1324 display modes (for --display) apply to ship-aid:
1325  --display=dumb       just print new information, scrolling the screen
1326  --display=overwrite  use cursor motion, selective clear, etc. to redraw at top''')
1327         ao = pa.add_option
1328         ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
1329                 help='select ocean OCEAN')
1330         ao('--cache-dir', dest='cache_dir', metavar='DIR',
1331                 default='~/.yoweb-scrape-cache',
1332                 help='cache yoweb pages in DIR')
1333         ao('-D','--debug', action='count', dest='debug', default=0,
1334                 help='enable debugging output')
1335         ao('--debug-fd', type='int', dest='debug_fd',
1336                 help='write any debugging output to specified fd')
1337         ao('-q','--quiet', action='store_true', dest='quiet',
1338                 help='suppress warning output')
1339         ao('--display', action='store', dest='display',
1340                 type='choice', choices=['dumb','overwrite'],
1341                 help='how to display ship aid')
1342
1343         ao_jt = lambda wh, t: ao(
1344                 '--timeout-sa-'+wh, action='store', dest='timeout_'+wh,
1345                 default=t, help=('set timeout for expiring %s jobbers' % wh))
1346         ao_jt('applied',      120)
1347         ao_jt('invited',      120)
1348         ao_jt('declined',      30)
1349         ao_jt('ashore',      1800)
1350
1351         ao('--ship-duty', action='store_true', dest='ship_duty',
1352                 help='show ship duty station puzzles')
1353         ao('--all-puzzles', action='store_false', dest='ship_duty',
1354                 help='show all puzzles, not just ship duty stations')
1355
1356         ao('--min-cache-reuse', type='int', dest='min_max_age',
1357                 metavar='SECONDS', default=60,
1358                 help='always reuse cache yoweb data if no older than this')
1359
1360         (opts,args) = pa.parse_args()
1361         random.seed()
1362
1363         if len(args) < 1:
1364                 print >>sys.stderr, copyright_info
1365                 pa.error('need a mode argument')
1366
1367         if opts.debug_fd is not None:
1368                 opts.debug_file = os.fdopen(opts.debug_fd, 'w')
1369         else:
1370                 opts.debug_file = sys.stdout
1371
1372         mode = args[0]
1373         mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
1374         try: mode_fn = globals()[mode_fn_name]
1375         except KeyError: pa.error('unknown mode "%s"' % mode)
1376
1377         # fixed parameters
1378         opts.expire_age = max(3600, opts.min_max_age)
1379
1380         opts.ship_reboard_clearout = 3600
1381
1382         if opts.cache_dir.startswith('~/'):
1383                 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
1384
1385         if opts.display is None:
1386                 if ((opts.debug > 0 and opts.debug_fd is None)
1387                     or not os.isatty(sys.stdout.fileno())):
1388                         opts.display = 'dumb'
1389                 else:
1390                         opts.display = 'overwrite'
1391
1392         fetcher = Fetcher(opts.ocean, opts.cache_dir)
1393
1394         mode_fn(args[1:], pa.error)
1395
1396 main()