yoweb-scrape

   1 #!/usr/bin/python
   2 # This is part of ypp-sc-tools, a set of third-party tools for assisting
   3 # players of Yohoho Puzzle Pirates.
   4 #
   5 # Copyright (C) 2009 Ian Jackson <ijackson@chiark.greenend.org.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 #
  20 # Yohoho and Puzzle Pirates are probably trademarks of Three Rings and
  21 # are used without permission.  This program is not endorsed or
  22 # sponsored by Three Rings.
  23
  24 copyright_info = '''
  25 yoweb-scrape is part of ypp-sc-tools  Copyright (C) 2009 Ian Jackson
  26 This program comes with ABSOLUTELY NO WARRANTY; this is free software,
  27 and you are welcome to redistribute it under certain conditions.
  28 For details, read the top of the yoweb-scrape file.
  29 '''
  30
  31 #---------- setup ----------
  32
  33 import signal
  34 signal.signal(signal.SIGINT, signal.SIG_DFL)
  35
  36 import os
  37 import time
  38 import urllib
  39 import urllib2
  40 import errno
  41 import sys
  42 import re as regexp
  43 import random
  44 import curses
  45 import termios
  46 import random
  47 from optparse import OptionParser
  48 from StringIO import StringIO
  49
  50 from BeautifulSoup import BeautifulSoup
  51
  52 opts = None
  53
  54 #---------- YPP parameters and arrays ----------
  55
  56 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
  57         '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
  58         '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
  59         '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
  60
  61 core_duty_puzzles = [
  62                 'Gunning',
  63                 ['Sailing','Rigging'],
  64                 'Bilging',
  65                 'Carpentry',
  66                 ]
  67
  68 duty_puzzles = ([ 'Navigating', 'Battle Navigation' ] +
  69                 core_duty_puzzles +
  70                 [ 'Treasure Haul' ])
  71
  72 standingvals = ('Able/Proficient/Distinguished/Respected/Master'+
  73                 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
  74 standing_limit = len(standingvals)
  75
  76 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
  77
  78 max_pirate_namelen = 12
  79
  80
  81 #---------- general utilities ----------
  82
  83 def debug(m):
  84         if opts.debug > 0:
  85                 print >>opts.debug_file, m
  86
  87 def debug_flush():
  88         if opts.debug > 0:
  89                 opts.debug_file.flush()
  90
  91 def sleep(seconds):
  92         debug_flush()
  93         time.sleep(seconds)
  94
  95 def format_time_interval(ti):
  96         if ti < 120: return '%d:%02d' % (ti / 60, ti % 60)
  97         if ti < 7200: return '%2dm' % (ti / 60)
  98         if ti < 86400: return '%dh' % (ti / 3600)
  99         return '%dd' % (ti / 86400)
 100
 101 #---------- caching and rate-limiting data fetcher ----------
 102
 103 class Fetcher:
 104         def __init__(self, ocean, cachedir):
 105                 debug('Fetcher init %s' % cachedir)
 106                 self.ocean = ocean
 107                 self.cachedir = cachedir
 108                 try: os.mkdir(cachedir)
 109                 except (OSError,IOError), oe:
 110                         if oe.errno != errno.EEXIST: raise
 111                 self._cache_scan(time.time())
 112
 113         def default_ocean(self, ocean='ice'):
 114                 if self.ocean is None:
 115                         self.ocean = ocean
 116
 117         def _cache_scan(self, now):
 118                 # returns list of ages, unsorted
 119                 ages = []
 120                 debug('Fetcher   scan_cache')
 121                 for leaf in os.listdir(self.cachedir):
 122                         if not leaf.startswith('#'): continue
 123                         path = self.cachedir + '/' + leaf
 124                         try: s = os.stat(path)
 125                         except (OSError,IOError), oe:
 126                                 if oe.errno != errno.ENOENT: raise
 127                                 continue
 128                         age = now - s.st_mtime
 129                         if age > opts.expire_age:
 130                                 debug('Fetcher    expire %d %s' % (age, path))
 131                                 try: os.remove(path)
 132                                 except (OSError,IOError), oe:
 133                                         if oe.errno != errno.ENOENT: raise
 134                                 continue
 135                         ages.append(age)
 136                 return ages
 137
 138         def need_wait(self, now, imaginary=[]):
 139                 ages = self._cache_scan(now)
 140                 ages += imaginary
 141                 ages.sort()
 142                 debug('Fetcher   ages ' + `ages`)
 143                 min_age = 1
 144                 need_wait = 0
 145                 for age in ages:
 146                         if age < min_age and age <= 5:
 147                                 debug('Fetcher   morewait min=%d age=%d' %
 148                                         (min_age, age))
 149                                 need_wait = max(need_wait, min_age - age)
 150                         min_age += 3
 151                         min_age *= 1.25
 152                 if need_wait > 0:
 153                         need_wait += random.random() - 0.5
 154                 return need_wait
 155
 156         def _rate_limit_cache_clean(self, now):
 157                 need_wait = self.need_wait(now)
 158                 if need_wait > 0:
 159                         debug('Fetcher   wait %d' % need_wait)
 160                         sleep(need_wait)
 161
 162         def fetch(self, url, max_age):
 163                 debug('Fetcher fetch %s' % url)
 164                 cache_corename = urllib.quote_plus(url)
 165                 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
 166                 try: f = file(cache_item, 'r')
 167                 except (OSError,IOError), oe:
 168                         if oe.errno != errno.ENOENT: raise
 169                         f = None
 170                 now = time.time()
 171                 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
 172                 if f is not None:
 173                         s = os.fstat(f.fileno())
 174                         age = now - s.st_mtime
 175                         if age > max_age:
 176                                 debug('Fetcher  stale %d < %d'% (max_age, age))
 177                                 f = None
 178                 if f is not None:
 179                         data = f.read()
 180                         f.close()
 181                         debug('Fetcher  cached %d > %d' % (max_age, age))
 182                         return data
 183
 184                 debug('Fetcher  fetch')
 185                 self._rate_limit_cache_clean(now)
 186
 187                 stream = urllib2.urlopen(url)
 188                 data = stream.read()
 189                 cache_tmp = "%s/#%s~%d#" % (
 190                         self.cachedir, cache_corename, os.getpid())
 191                 f = file(cache_tmp, 'w')
 192                 f.write(data)
 193                 f.close()
 194                 os.rename(cache_tmp, cache_item)
 195                 debug('Fetcher  stored')
 196                 return data
 197
 198         def yoweb(self, kind, tail, max_age):
 199                 self.default_ocean()
 200                 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
 201                         self.ocean, kind, tail)
 202                 return self.fetch(url, max_age)
 203
 204 #---------- logging assistance for troubled screenscrapers ----------
 205
 206 class SoupLog:
 207         def __init__(self):
 208                 self.msgs = [ ]
 209         def msg(self, m):
 210                 self.msgs.append(m)
 211         def soupm(self, obj, m):
 212                 self.msg(m + '; in ' + `obj`)
 213         def needs_msgs(self, child_souplog):
 214                 self.msgs += child_souplog.msgs
 215                 child_souplog.msgs = [ ]
 216
 217 def soup_text(obj):
 218         str = ''.join(obj.findAll(text=True))
 219         return str.strip()
 220
 221 class SomethingSoupInfo(SoupLog):
 222         def __init__(self, kind, tail, max_age):
 223                 SoupLog.__init__(self)
 224                 html = fetcher.yoweb(kind, tail, max_age)
 225                 self._soup = BeautifulSoup(html,
 226                         convertEntities=BeautifulSoup.HTML_ENTITIES
 227                         )
 228
 229 #---------- scraper for pirate pages ----------
 230
 231 class PirateInfo(SomethingSoupInfo):
 232         # Public data members:
 233         #  pi.standings = { 'Treasure Haul': 'Able' ... }
 234         #  pi.name = name
 235         #  pi.crew = (id, name)
 236         #  pi.flag = (id, name)
 237         #  pi.msgs = [ 'message describing problem with scrape' ]
 238
 239         def __init__(self, pirate, max_age=300):
 240                 SomethingSoupInfo.__init__(self,
 241                         'pirate.wm?target=', pirate, max_age)
 242                 self.name = pirate
 243                 self._find_standings()
 244                 self.crew = self._find_crewflag('crew',
 245                         '^/yoweb/crew/info\\.wm')
 246                 self.flag = self._find_crewflag('flag',
 247                         '^/yoweb/flag/info\\.wm')
 248
 249         def _find_standings(self):
 250                 imgs = self._soup.findAll('img',
 251                         src=regexp.compile('/yoweb/images/stat.*'))
 252                 re = regexp.compile(
 253 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
 254                         )
 255                 standings = { }
 256
 257                 for skill in puzzles:
 258                         standings[skill] = [ ]
 259
 260                 skl = SoupLog()
 261
 262                 for img in imgs:
 263                         try: puzzle = img['alt']
 264                         except KeyError: continue
 265
 266                         if not puzzle in puzzles:
 267                                 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
 268                                 continue
 269                         key = img.findParent('td')
 270                         if key is None:
 271                                 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
 272                                 continue
 273                         valelem = key.findNextSibling('td')
 274                         if valelem is None:
 275                                 skl.soupm(key, 'puzzle missing sibling "%s"'
 276                                         % puzzle)
 277                                 continue
 278                         valstr = soup_text(valelem)
 279                         match = re.match(valstr)
 280                         if match is None:
 281                                 skl.soupm(key, ('puzzle "%s" unparseable'+
 282                                         ' standing "%s"') % (puzzle, valstr))
 283                                 continue
 284                         standing = match.group(match.lastindex)
 285                         standings[puzzle].append(standing)
 286
 287                 self.standings = { }
 288
 289                 for puzzle in puzzles:
 290                         sl = standings[puzzle]
 291                         if len(sl) > 1:
 292                                 skl.msg('puzzle "%s" multiple standings %s' %
 293                                                 (puzzle, `sl`))
 294                                 continue
 295                         if not sl:
 296                                 skl.msg('puzzle "%s" no standing found' % puzzle)
 297                                 continue
 298                         standing = sl[0]
 299                         for i in range(0, standing_limit):
 300                                 if standing == standingvals[i]:
 301                                         self.standings[puzzle] = i
 302                         if not puzzle in self.standings:
 303                                 skl.msg('puzzle "%s" unknown standing "%s"' %
 304                                         (puzzle, standing))
 305
 306                 all_standings_ok = True
 307                 for puzzle in puzzles:
 308                         if not puzzle in self.standings:
 309                                 self.needs_msgs(skl)
 310
 311         def _find_crewflag(self, cf, yoweb_re):
 312                 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
 313                 if len(things) != 1:
 314                         self.msg('zero or several %s id references found' % cf)
 315                         return None
 316                 thing = things[0]
 317                 id_re = '\\b%sid\\=(\\w+)$' % cf
 318                 id_haystack = thing['href']
 319                 match = regexp.compile(id_re).search(id_haystack)
 320                 if match is None:
 321                         self.soupm(thing, ('incomprehensible %s id ref'+
 322                                 ' (%s in %s)') % (cf, id_re, id_haystack))
 323                         return None
 324                 name = soup_text(thing)
 325                 return (match.group(1), name)
 326
 327         def __str__(self):
 328                 return `(self.crew, self.flag, self.standings, self.msgs)`
 329
 330 #---------- scraper for crew pages ----------
 331
 332 class CrewInfo(SomethingSoupInfo):
 333         # Public data members:
 334         #  ci.crew = [ ('Captain',        ['Pirate', ...]),
 335         #              ('Senior Officer', [...]),
 336         #               ... ]
 337         #  pi.msgs = [ 'message describing problem with scrape' ]
 338
 339         def __init__(self, crewid, max_age=300):
 340                 SomethingSoupInfo.__init__(self,
 341                         'crew/info.wm?crewid=', crewid, max_age)
 342                 self._find_crew()
 343
 344         def _find_crew(self):
 345                 self.crew = []
 346                 capts = self._soup.findAll('img',
 347                         src='/yoweb/images/crew-captain.png')
 348                 if len(capts) != 1:
 349                         self.msg('crew members: no. of captain images != 1')
 350                         return
 351                 tbl = capts[0]
 352                 while not tbl.find('a', href=pirate_ref_re):
 353                         tbl = tbl.findParent('table')
 354                         if not tbl:
 355                                 self.msg('crew members: cannot find table')
 356                                 return
 357                 current_rank_crew = None
 358                 crew_rank_re = regexp.compile('/yoweb/images/crew')
 359                 for row in tbl.contents:
 360                         # findAll(recurse=False)
 361                         if isinstance(row,basestring):
 362                                 continue
 363
 364                         is_rank = row.find('img', attrs={'src': crew_rank_re})
 365                         if is_rank:
 366                                 rank = soup_text(row)
 367                                 current_rank_crew = []
 368                                 self.crew.append((rank, current_rank_crew))
 369                                 continue
 370                         for cell in row.findAll('a', href=pirate_ref_re):
 371                                 if current_rank_crew is None:
 372                                         self.soupm(cell, 'crew members: crew'
 373                                                 ' before rank')
 374                                         continue
 375                                 current_rank_crew.append(soup_text(cell))
 376
 377         def __str__(self):
 378                 return `(self.crew, self.msgs)`
 379
 380 #---------- pretty-printer for tables of pirate puzzle standings ----------
 381
 382 class StandingsTable:
 383         def __init__(self, f, use_puzzles=None, col_width=6, gap_every=5):
 384                 if use_puzzles is None:
 385                         if opts.ship_duty:
 386                                 use_puzzles=duty_puzzles
 387                         else:
 388                                 use_puzzles=puzzles
 389                 self._puzzles = use_puzzles
 390                 self.f = f
 391                 self._cw = col_width-1
 392                 self._gap_every = gap_every
 393                 self._linecount = 0
 394                 self._o = f.write
 395
 396         def _nl(self): self._o('\n')
 397
 398         def _pline(self, lhs, puzstrs, extra):
 399                 if (self._linecount > 0
 400                     and self._gap_every is not None
 401                     and not (self._linecount % self._gap_every)):
 402                         self._nl()
 403                 self._o('%-*s' % (max(max_pirate_namelen+1, 15), lhs))
 404                 for v in puzstrs:
 405                         self._o(' %-*.*s' % (self._cw,self._cw, v))
 406                 if extra:
 407                         self._o(' ' + extra)
 408                 self._nl()
 409                 self._linecount += 1
 410
 411         def _puzstr(self, pi, puzzle):
 412                 if not isinstance(puzzle,list): puzzle = [puzzle]
 413                 try: standing = max([pi.standings[p] for p in puzzle])
 414                 except KeyError: return '?'
 415                 if not standing: return ''
 416                 s = ''
 417                 if self._cw > 4:
 418                         c1 = standingvals[standing][0]
 419                         if standing < 3: c1 = c1.lower() # 3 = Master
 420                         s += `standing`
 421                 if self._cw > 5:
 422                         s += ' '
 423                 s += '*' * (standing / 2)
 424                 s += '+' * (standing % 2)
 425                 return s
 426
 427         def headings(self, lhs='', rhs=None):
 428                 def puzn_redact(name):
 429                         if isinstance(name,list):
 430                                 return '/'.join(
 431                                         ["%.*s" % (self._cw/2, puzn_redact(n))
 432                                          for n in name])
 433                         spc = name.find(' ')
 434                         if spc < 0: return name
 435                         return name[0:min(4,spc)] + name[spc+1:]
 436                 self._linecount = -2
 437                 self._pline(lhs, map(puzn_redact, self._puzzles), rhs)
 438                 self._linecount = 0
 439         def literalline(self, line):
 440                 self._o(line)
 441                 self._nl()
 442                 self._linecount = 0
 443         def pirate_dummy(self, name, standingstring, extra=None):
 444                 standings = standingstring * len(self._puzzles)
 445                 self._pline(' '+name, standings, extra)
 446         def pirate(self, pi, extra=None):
 447                 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
 448                 self._pline(' '+pi.name, puzstrs, extra)
 449
 450
 451 #---------- chat log parser ----------
 452
 453 class PirateAboard:
 454         # This is essentially a transparent, dumb, data class.
 455         #  pa.v                 may be None
 456         #  pa.name
 457         #  pa.last_time
 458         #  pa.last_event
 459         #  pa.gunner
 460         #  pa.last_chat_time
 461         #  pa.last_chat_chan
 462         #  pa.pi
 463
 464         # Also used for jobbing applicants:
 465         #               happens when                    expires (to "-")
 466         #   -            disembark, leaves crew          no
 467         #   aboard       evidence of them being aboard   no
 468         #   applied      "has applied for the job"       120s, configurable
 469         #   ashore       "has taken a job"               30min, configurable
 470         #   declined     "declined the job offer"        30s, configurable
 471         #   invited      "has been invited to job"       120s, configurable
 472         #
 473         #  pa.jobber    None, 'ashore', 'applied', 'invited', 'declined'
 474         #  pa.expires   expiry time time
 475
 476         def __init__(pa, pn, v, time, event):
 477                 pa.name = pn
 478                 pa.v = v
 479                 pa.last_time = time
 480                 pa.last_event = event
 481                 pa.last_chat_time = None
 482                 pa.last_chat_chan = None
 483                 pa.gunner = False
 484                 pa.pi = None
 485                 pa.jobber = None
 486                 pa.expires = None
 487
 488         def pirate_info(pa):
 489                 now = time.time()
 490                 if pa.pi:
 491                         age = now - pa.pi_fetched
 492                         guide = random.randint(120,240)
 493                         if age <= guide:
 494                                 return pa.pi
 495                         debug('PirateAboard refresh %d > %d  %s' % (
 496                                 age, guide, pa.name))
 497                         imaginary = [2,4]
 498                 else:
 499                         imaginary = [1]
 500                 wait = fetcher.need_wait(now, imaginary)
 501                 if wait:
 502                         debug('PirateAboard fetcher not ready %d' % wait)
 503                         return pa.pi
 504                 pa.pi = PirateInfo(pa.name, 600)
 505                 pa.pi_fetched = now
 506                 return pa.pi
 507
 508 class ChatLogTracker:
 509         # This is quite complex so we make it opaque.  Use the
 510         # official invokers, accessors etc.
 511
 512         def __init__(self, myself_pi, logfn):
 513                 self._pl = {}   # self._pl['Pirate'] =
 514                 self._vl = {}   #   self._vl['Vessel']['Pirate'] = PirateAboard
 515                                 # self._vl['Vessel']['#lastinfo']
 516                                 # self._vl['Vessel']['#name']
 517                                 # self._v = self._vl[self._vessel]
 518                 self._date = None
 519                 self._myself = myself_pi
 520                 self._lbuf = ''
 521                 self._f = file(logfn)
 522                 flen = os.fstat(self._f.fileno()).st_size
 523                 max_backlog = 500000
 524                 if flen > max_backlog:
 525                         startpos = flen - max_backlog
 526                         self._f.seek(startpos)
 527                         self._f.readline()
 528                 self._progress = [0, flen - self._f.tell()]
 529                 self._disembark_myself()
 530                 self._need_redisplay = False
 531                 self._lastvessel = None
 532
 533         def _disembark_myself(self):
 534                 self._v = None
 535                 self._vessel = None
 536                 self.force_redisplay()
 537
 538         def force_redisplay(self):
 539                 self._need_redisplay = True
 540
 541         def _vessel_updated(self, v, timestamp):
 542                 if v is None: return
 543                 v['#lastinfo'] = timestamp
 544                 self.force_redisplay()
 545
 546         def _onboard_event(self,v,timestamp,pirate,event,jobber=None):
 547                 pa = self._pl.get(pirate, None)
 548                 if pa is not None and pa.v is v:
 549                         pa.last_time = timestamp
 550                         pa.last_event = event
 551                 else:
 552                         if pa is not None and pa.v is not None:
 553                                 del pa.v[pirate]
 554                         pa = PirateAboard(pirate, v, timestamp, event)
 555                         self._pl[pirate] = pa
 556                         if v is not None: v[pirate] = pa
 557                 pa.jobber = jobber
 558
 559                 if jobber is None: timeout = None
 560                 else: timeout = getattr(opts, 'timeout_'+jobber)
 561                 if timeout is None: pa.expires = None
 562                 else: pa.expires = timestamp + timeout
 563                 self._vessel_updated(v, timestamp)
 564                 return pa
 565
 566         def _expire_jobbers(self, now):
 567                 for pa in self._pl.values():
 568                         if pa.expires is None: continue
 569                         if pa.expires >= now: continue
 570                         v = pa.v
 571                         del self._pl[pa.name]
 572                         if v is not None: del v[pa.name]
 573                         self.force_redisplay()
 574
 575         def _trash_vessel(self, v):
 576                 for pn in v:
 577                         if pn.startswith('#'): continue
 578                         del self._pl[pn]
 579                 vn = v['#name']
 580                 del self._vl[vn]
 581                 if v is self._v: self._disembark_myself()
 582                 self.force_redisplay()
 583
 584         def _vessel_stale(self, v, timestamp):
 585                 return timestamp - v['#lastinfo'] > opts.ship_reboard_clearout
 586
 587         def _vessel_check_expire(self, v, timestamp):
 588                 if not self._vessel_stale(v, timestamp):
 589                         return v
 590                 self._debug_line_disposition(timestamp,'',
 591                         'stale-reset ' + v['#name'])
 592                 self._trash_vessel(v)
 593                 return None
 594
 595         def expire_garbage(self, timestamp):
 596                 for v in self._vl.values():
 597                         self._vessel_check_expire(v, timestamp)
 598
 599         def _vessel_lookup(self, vn, timestamp, dml=[], create=False):
 600                 v = self._vl.get(vn, None)
 601                 if v is not None:
 602                         v = self._vessel_check_expire(v, timestamp)
 603                 if v is not None:
 604                         dml.append('found')
 605                         return v
 606                 if not create:
 607                         dml.append('no')
 608                 dml.append('new')
 609                 self._vl[vn] = v = { '#name': vn }
 610                 self._vessel_updated(v, timestamp)
 611                 return v
 612
 613         def _find_matching_vessel(self, pattern, timestamp, cmdr,
 614                                         dml=[], create=False):
 615                 # use when a commander pirate `cmdr' specified a vessel
 616                 #  by name `pattern' (either may be None)
 617                 # if create is true, will create the vessel
 618                 #  record if an exact name is specified
 619
 620                 if (pattern is not None and
 621                     not '*' in pattern
 622                     and len(pattern.split(' ')) == 2):
 623                         vn = pattern.title()
 624                         dml.append('exact')
 625                         return self._vessel_lookup(
 626                                 vn, timestamp, dml=dml, create=create)
 627
 628                 if pattern is None:
 629                         pattern_check = lambda vn: True
 630                 else:
 631                         re = '(?:.* )?%s$' % pattern.lower().replace('*','.+')
 632                         pattern_check = regexp.compile(re, regexp.I).match
 633
 634                 tries = []
 635
 636                 cmdr_pa = self._pl.get(cmdr, None)
 637                 if cmdr_pa: tries.append((cmdr_pa.v, 'cmdr'))
 638
 639                 tries.append((self._v, 'here'))
 640                 tried_vns = []
 641
 642                 for (v, dm) in tries:
 643                         if v is None: dml.append(dm+'?'); continue
 644
 645                         vn = v['#name']
 646                         if not pattern_check(vn):
 647                                 tried_vns.append(vn)
 648                                 dml.append(dm+'#')
 649                                 continue
 650
 651                         dml.append(dm+'!')
 652                         return v
 653
 654                 if pattern is not None and '*' in pattern:
 655                         search = [
 656                                 (vn,v)
 657                                 for (vn,v) in self._vl.iteritems()
 658                                 if not self._vessel_stale(v, timestamp)
 659                                 if pattern_check(vn)
 660                                 ]
 661                         #debug('CLT-RE /%s/ wanted (%s) searched (%s)' % (
 662                         #       re,
 663                         #       '/'.join(tried_vns),
 664                         #       '/'.join([vn for (vn,v) in search])))
 665
 666                         if len(search)==1:
 667                                 dml.append('one')
 668                                 return search[0][1]
 669                         elif search:
 670                                 dml.append('many')
 671                         else:
 672                                 dml.append('none')
 673
 674         def _debug_line_disposition(self,timestamp,l,m):
 675                 debug('CLT %13s %-40s %s' % (timestamp,m,l))
 676
 677         def _rm_crew_l(self,re,l):
 678                 m = regexp.match(re,l)
 679                 if m and m.group(2) == self._myself.crew[1]:
 680                         return m.group(1)
 681                 else:
 682                         return None
 683
 684         def chatline(self,l):
 685                 rm = lambda re: regexp.match(re,l)
 686                 d = lambda m: self._debug_line_disposition(timestamp,l,m)
 687                 rm_crew = lambda re: self._rm_crew_l(re,l)
 688                 timestamp = None
 689
 690                 m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$')
 691                 if m:
 692                         self._date = [int(x) for x in m.groups()]
 693                         self._previous_timestamp = None
 694                         return d('date '+`self._date`)
 695
 696                 if self._date is None:
 697                         return d('date unset')
 698
 699                 m = rm('\\[(\d\d):(\d\d):(\d\d)\\] ')
 700                 if not m:
 701                         return d('no timestamp')
 702
 703                 while True:
 704                         time_tuple = (self._date +
 705                                       [int(x) for x in m.groups()] +
 706                                       [-1,-1,-1])
 707                         timestamp = time.mktime(time_tuple)
 708                         if timestamp >= self._previous_timestamp: break
 709                         self._date[2] += 1
 710                         self._debug_line_disposition(timestamp,'',
 711                                 'new date '+`self._date`)
 712
 713                 self._previous_timestamp = timestamp
 714
 715                 l = l[l.find(' ')+1:]
 716
 717                 def ob_x(pirate,event):
 718                         return self._onboard_event(
 719                                         self._v, timestamp, pirate, event)
 720                 def ob1(did): ob_x(m.group(1), did); return d(did)
 721                 def oba(did): return ob1('%s %s' % (did, m.group(2)))
 722
 723                 def jb(pirate,jobber):
 724                         return self._onboard_event(
 725                                 None, timestamp, pirate,
 726                                 ("jobber %s" % jobber),
 727                                 jobber=jobber
 728                                 )
 729
 730                 def disembark(v, timestamp, pirate, event):
 731                         self._onboard_event(
 732                                         v, timestamp, pirate, 'leaving '+event)
 733                         del v[pirate]
 734                         del self._pl[pirate]
 735
 736                 def disembark_me(why):
 737                         self._disembark_myself()
 738                         return d('disembark-me '+why)
 739
 740                 m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
 741                 if m:
 742                         dm = ['boarding']
 743                         pn = self._myself.name
 744                         vn = m.group(1)
 745                         v = self._vessel_lookup(vn, timestamp, dm, create=True)
 746                         self._lastvessel = self._vessel = vn
 747                         self._v = v
 748                         ob_x(pn, 'we boarded')
 749                         self.expire_garbage(timestamp)
 750                         return d(' '.join(dm))
 751
 752                 if self._v is None:
 753                         return d('no vessel')
 754
 755                 m = rm('(\\w+) has come aboard\\.$')
 756                 if m: return ob1('boarded');
 757
 758                 m = rm('You have ordered (\\w+) to do some (\\S.*\\S)\\.$')
 759                 if m:
 760                         (who,what) = m.groups()
 761                         pa = ob_x(who,'ord '+what)
 762                         if what == 'Gunning':
 763                                 pa.gunner = True
 764                         return d('duty order')
 765
 766                 m = rm('(\\w+) abandoned a (\\S.*\\S) station\\.$')
 767                 if m: oba('stopped'); return d("end")
 768
 769                 def chat_core(speaker, chan):
 770                         try: pa = self._pl[speaker]
 771                         except KeyError: return 'mystery'
 772                         if pa.v is not None and pa.v is not self._v:
 773                                 return 'elsewhere'
 774                         pa.last_chat_time = timestamp
 775                         pa.last_chat_chan = chan
 776                         self.force_redisplay()
 777                         return 'here'
 778
 779                 def chat(chan):
 780                         speaker = m.group(1)
 781                         dm = chat_core(speaker, chan)
 782                         return d('chat %s %s' % (chan, dm))
 783
 784                 def chat_metacmd(chan):
 785                         (cmdr, metacmd) = m.groups()
 786                         metacmd = regexp.sub('\\s+', ' ', metacmd).strip()
 787                         m2 = regexp.match(
 788                             '/([adj]) (?:([A-Za-z* ]+)\\s*:)?([A-Za-z ]+)$',
 789                             metacmd)
 790                         if not m2: return chat(chan)
 791
 792                         (cmd, pattern, targets) = m2.groups()
 793                         dml = ['cmd', chan, cmd]
 794
 795                         if cmd == 'a': each = self._onboard_event
 796                         elif cmd == 'd': each = disembark
 797                         else: each = lambda *l: self._onboard_event(*l,
 798                                         **{'jobber':'applied'})
 799
 800                         if cmdr == self._myself.name:
 801                                 dml.append('self')
 802                                 how = 'cmd: %s' % cmd
 803                         else:
 804                                 dml.append('other')
 805                                 how = 'cmd: %s %s' % (cmd,cmdr)
 806
 807                         if cmd == 'j':
 808                                 if pattern is not None:
 809                                         return chat(chan)
 810                                 v = None
 811                         else:
 812                                 v = self._find_matching_vessel(
 813                                         pattern, timestamp, cmdr,
 814                                         dml, create=True)
 815
 816                         if cmd == 'j' or v is not None:
 817                                 targets = targets.strip().split(' ')
 818                                 dml.append(`len(targets)`)
 819                                 for target in targets:
 820                                         each(v, timestamp, target.title(), how)
 821                                 self._vessel_updated(v, timestamp)
 822
 823                         dm = ' '.join(dml)
 824                         chat_core(cmdr, 'cmd '+chan)
 825                         return d(dm)
 826
 827                 m = rm('(\\w+) (?:issued an order|ordered everyone) "')
 828                 if m: return ob1('general order');
 829
 830                 m = rm('(\\w+) says, "')
 831                 if m: return chat('public')
 832
 833                 m = rm('(\\w+) tells ye, "')
 834                 if m: return chat('private')
 835
 836                 m = rm('Ye told (\\w+), "(.*)"$')
 837                 if m: return chat_metacmd('private')
 838
 839                 m = rm('(\\w+) flag officer chats, "')
 840                 if m: return chat('flag officer')
 841
 842                 m = rm('(\\w+) officer chats, "(.*)"$')
 843                 if m: return chat_metacmd('officer')
 844
 845                 m = rm('Ye accepted the offer to job with ')
 846                 if m: return disembark_me('jobbing')
 847
 848                 m = rm('Ye hop on the ferry and are whisked away ')
 849                 if m: return disembark_me('ferry')
 850
 851                 m = rm('Whisking away to yer home on the magical winds')
 852                 if m: return disembark_me('home')
 853
 854                 m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
 855                 if m:
 856                         pl = m.group(1).split(', ')
 857                         if not self._myself.name in pl:
 858                                 return d('lost melee')
 859                         for pn in pl:
 860                                 if ' ' in pn: continue
 861                                 ob_x(pn,'won melee')
 862                         return d('won melee')
 863
 864                 m = rm('(\\w+) is eliminated\\!')
 865                 if m: return ob1('eliminated in fray');
 866
 867                 m = rm('(\\w+) has driven \w+ from the ship\\!')
 868                 if m: return ob1('boarder repelled');
 869
 870                 m = rm('\w+ has bested (\\w+), and turns'+
 871                         ' to the rest of the ship\\.')
 872                 if m: return ob1('boarder unrepelled');
 873
 874                 pirate = rm_crew("(\\w+) has taken a job with '(.*)'\\.")
 875                 if pirate: return jb(pirate, 'ashore')
 876
 877                 pirate = rm_crew("(\\w+) has left '(.*)'\\.")
 878                 if pirate:
 879                         disembark(self._v, timestamp, pirate, 'left crew')
 880                         return d('left crew')
 881
 882                 m = rm('(\w+) has applied for the posted job\.')
 883                 if m: return jb(m.group(1), 'applied')
 884
 885                 pirate= rm_crew("(\\w+) has been invited to job for '(.*)'\\.")
 886                 if pirate: return jb(pirate, 'invited')
 887
 888                 pirate = rm_crew("(\\w+) declined the job offer for '(.*)'\\.")
 889                 if pirate: return jb(pirate, 'declined')
 890
 891                 m = rm('(\\w+) has left the vessel\.')
 892                 if m:
 893                         pirate = m.group(1)
 894                         disembark(self._v, timestamp, pirate, 'disembarked')
 895                         return d('disembarked')
 896
 897                 return d('not-matched')
 898
 899         def _str_pa(self, pn, pa):
 900                 assert self._pl[pn] == pa
 901                 s = ' '*20 + "%s %-*s %13s %-30s %13s %-20s %13s" % (
 902                         (' ','G')[pa.gunner],
 903                         max_pirate_namelen, pn,
 904                         pa.last_time, pa.last_event,
 905                         pa.last_chat_time, pa.last_chat_chan,
 906                         pa.jobber)
 907                 if pa.expires is not None:
 908                         s += " %-5d" % (pa.expires - pa.last_time)
 909                 s += "\n"
 910                 return s
 911
 912         def _str_vessel(self, vn, v):
 913                 s = ' vessel %s\n' % vn
 914                 s += ' '*20 + "%-*s   %13s\n" % (
 915                                 max_pirate_namelen, '#lastinfo',
 916                                 v['#lastinfo'])
 917                 assert v['#name'] == vn
 918                 for pn in sorted(v.keys()):
 919                         if pn.startswith('#'): continue
 920                         pa = v[pn]
 921                         assert pa.v == v
 922                         s += self._str_pa(pn,pa)
 923                 return s
 924
 925         def __str__(self):
 926                 s = '''<ChatLogTracker
 927  myself %s
 928  vessel %s
 929 '''                     % (self._myself.name, self._vessel)
 930                 assert ((self._v is None and self._vessel is None) or
 931                         (self._v is self._vl[self._vessel]))
 932                 if self._vessel is not None:
 933                         s += self._str_vessel(self._vessel, self._v)
 934                 for vn in sorted(self._vl.keys()):
 935                         if vn == self._vessel: continue
 936                         s += self._str_vessel(vn, self._vl[vn])
 937                 s += " elsewhere\n"
 938                 for p in self._pl:
 939                         pa = self._pl[p]
 940                         if pa.v is not None:
 941                                 assert pa.v[p] is pa
 942                                 assert pa.v in self._vl.values()
 943                         else:
 944                                 s += self._str_pa(pa.name, pa)
 945                 s += '>\n'
 946                 return s
 947
 948         def catchup(self, progress=None):
 949                 while True:
 950                         more = self._f.readline()
 951                         if not more: break
 952
 953                         self._progress[0] += len(more)
 954                         if progress: progress.progress(*self._progress)
 955
 956                         self._lbuf += more
 957                         if self._lbuf.endswith('\n'):
 958                                 self.chatline(self._lbuf.rstrip())
 959                                 self._lbuf = ''
 960                                 if opts.debug >= 2:
 961                                         debug(self.__str__())
 962                 self._expire_jobbers(time.time())
 963
 964                 if progress: progress.caughtup()
 965
 966         def changed(self):
 967                 rv = self._need_redisplay
 968                 self._need_redisplay = False
 969                 return rv
 970         def myname(self):
 971                 # returns our pirate name
 972                 return self._myself.name
 973         def vesselname(self):
 974                 # returns the vessel name we're aboard or None
 975                 return self._vessel
 976         def lastvesselname(self):
 977                 # returns the last vessel name we were aboard or None
 978                 return self._lastvessel
 979         def aboard(self, vesselname=True):
 980                 # returns a list of PirateAboard the vessel
 981                 #  sorted by pirate name
 982                 #  you can pass this None and you'll get []
 983                 #  or True for the current vessel (which is the default)
 984                 #  the returned value is a fresh list of persistent
 985                 #  PirateAboard objects
 986                 if vesselname is True: v = self._v
 987                 else: v = self._vl.get(vesselname.title())
 988                 if v is None: return []
 989                 return [ v[pn]
 990                          for pn in sorted(v.keys())
 991                          if not pn.startswith('#') ]
 992         def jobbers(self):
 993                 # returns a the jobbers' PirateAboards,
 994                 # sorted by jobber class and reverse of expiry time
 995                 l = [ pa
 996                       for pa in self._pl.values()
 997                       if pa.jobber is not None
 998                     ]
 999                 def compar_key(pa):
1000                         return (pa.jobber, -pa.expires)
1001                 l.sort(key = compar_key)
1002                 return l
1003
1004 #---------- implementations of actual operation modes ----------
1005
1006 def do_pirate(pirates, bu):
1007         print '{'
1008         for pirate in pirates:
1009                 info = PirateInfo(pirate)
1010                 print '%s: %s,' % (`pirate`, info)
1011         print '}'
1012
1013 def prep_crew_of(args, bu, max_age=300):
1014         if len(args) != 1: bu('crew-of takes one pirate name')
1015         pi = PirateInfo(args[0], max_age)
1016         if pi.crew is None: return None
1017         return CrewInfo(pi.crew[0], max_age)
1018
1019 def do_crew_of(args, bu):
1020         ci = prep_crew_of(args, bu)
1021         print ci
1022
1023 def do_standings_crew_of(args, bu):
1024         ci = prep_crew_of(args, bu, 60)
1025         tab = StandingsTable(sys.stdout)
1026         tab.headings()
1027         for (rank, members) in ci.crew:
1028                 if not members: continue
1029                 tab.literalline('')
1030                 tab.literalline('%s:' % rank)
1031                 for p in members:
1032                         pi = PirateInfo(p, random.randint(900,1800))
1033                         tab.pirate(pi)
1034
1035 class ProgressPrintPercentage:
1036         def __init__(self, f=sys.stdout):
1037                 self._f = f
1038         def progress_string(self,done,total):
1039                 return "scan chat logs %3d%%\r" % ((done*100) / total)
1040         def progress(self,*a):
1041                 self._f.write(self.progress_string(*a))
1042                 self._f.flush()
1043         def show_init(self, pirate, ocean):
1044                 print >>self._f, 'Starting up, %s on the %s ocean' % (
1045                         pirate, ocean)
1046         def caughtup(self):
1047                 self._f.write('                   \r')
1048                 self._f.flush()
1049
1050 #----- modes which use the chat log parser are quite complex -----
1051
1052 def prep_chat_log(args, bu,
1053                 progress=ProgressPrintPercentage(),
1054                 max_myself_age=3600):
1055         if len(args) != 1: bu('this action takes only chat log filename')
1056         logfn = args[0]
1057         logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_'
1058         match = regexp.match(logfn_re, logfn)
1059         if not match: bu('chat log filename is not in expected format')
1060         (pirate, ocean) = match.groups()
1061         fetcher.default_ocean(ocean)
1062
1063         progress.show_init(pirate, fetcher.ocean)
1064         myself = PirateInfo(pirate,max_myself_age)
1065         track = ChatLogTracker(myself, logfn)
1066
1067         opts.debug -= 2
1068         track.catchup(progress)
1069         opts.debug += 2
1070
1071         track.force_redisplay()
1072
1073         return (myself, track)
1074
1075 def do_track_chat_log(args, bu):
1076         (myself, track) = prep_chat_log(args, bu)
1077         while True:
1078                 track.catchup()
1079                 if track.changed():
1080                         print track
1081                 sleep(0.5 + 0.5 * random.random())
1082
1083 #----- ship management aid -----
1084
1085 class Display_dumb(ProgressPrintPercentage):
1086         def __init__(self):
1087                 ProgressPrintPercentage.__init__(self)
1088         def show(self, s):
1089                 print '\n\n', s;
1090         def realstart(self):
1091                 pass
1092
1093 class Display_overwrite(ProgressPrintPercentage):
1094         def __init__(self):
1095                 ProgressPrintPercentage.__init__(self)
1096
1097                 null = file('/dev/null','w')
1098                 curses.setupterm(fd=null.fileno())
1099
1100                 self._clear = curses.tigetstr('clear')
1101                 if not self._clear:
1102                         self._debug('missing clear!')
1103                         self.show = Display_dumb.show
1104                         return
1105
1106                 self._t = {'el':'', 'ed':''}
1107                 if not self._init_sophisticated():
1108                         for k in self._t.keys(): self._t[k] = ''
1109                         self._t['ho'] = self._clear
1110
1111         def _debug(self,m): debug('display overwrite: '+m)
1112
1113         def _init_sophisticated(self):
1114                 for k in self._t.keys():
1115                         s = curses.tigetstr(k)
1116                         self._t[k] = s
1117                 self._t['ho'] = curses.tigetstr('ho')
1118                 if not self._t['ho']:
1119                         cup = curses.tigetstr('cup')
1120                         self._t['ho'] = curses.tparm(cup,0,0)
1121                 missing = [k for k in self._t.keys() if not self._t[k]]
1122                 if missing:
1123                         self.debug('missing '+(' '.join(missing)))
1124                         return 0
1125                 return 1
1126
1127         def show(self, s):
1128                 w = sys.stdout.write
1129                 def wti(k): w(self._t[k])
1130
1131                 wti('ho')
1132                 nl = ''
1133                 for l in s.rstrip().split('\n'):
1134                         w(nl)
1135                         w(l)
1136                         wti('el')
1137                         nl = '\r\n'
1138                 wti('ed')
1139                 w(' ')
1140                 sys.stdout.flush()
1141
1142         def realstart(self):
1143                 sys.stdout.write(self._clear)
1144                 sys.stdout.flush()
1145
1146
1147 def do_ship_aid(args, bu):
1148         if opts.ship_duty is None: opts.ship_duty = True
1149
1150         displayer = globals()['Display_'+opts.display]()
1151
1152         (myself, track) = prep_chat_log(args, bu, progress=displayer)
1153
1154         displayer.realstart()
1155
1156         if os.isatty(0): kr_create = KeystrokeReader
1157         else: kr_create = DummyKeystrokeReader
1158
1159         try:
1160                 kreader = kr_create(0, 10)
1161                 ship_aid_core(myself, track, displayer, kreader)
1162         finally:
1163                 kreader.stop()
1164                 print '\n'
1165
1166 class KeyBasedSorter:
1167         def compar_key_pa(self, pa):
1168                 pi = pa.pirate_info()
1169                 if pi is None: return None
1170                 return self.compar_key(pi)
1171         def lsort_pa(self, l):
1172                 l.sort(key = self.compar_key_pa)
1173
1174 class NameSorter(KeyBasedSorter):
1175         def compar_key(self, pi): return pi.name
1176         def desc(self): return 'name'
1177
1178 class SkillSorter(NameSorter):
1179         def __init__(self, relevant):
1180                 self._want = frozenset(relevant.split('/'))
1181                 self._avoid = set()
1182                 for p in core_duty_puzzles:
1183                         if isinstance(p,basestring): self._avoid.add(p)
1184                         else: self._avoid |= set(p)
1185                 self._avoid -= self._want
1186                 self._desc = '%s' % relevant
1187
1188         def desc(self): return self._desc
1189
1190         def compar_key(self, pi):
1191                 best_want = max([
1192                         pi.standings.get(puz,-1)
1193                         for puz in self._want
1194                         ])
1195                 best_avoid = [
1196                         -pi.standings.get(puz,standing_limit)
1197                         for puz in self._avoid
1198                         ]
1199                 best_avoid.sort()
1200                 def negate(x): return -x
1201                 debug('compar_key %s bw=%s ba=%s' % (pi.name, `best_want`,
1202                         `best_avoid`))
1203                 return (-best_want, map(negate, best_avoid), pi.name)
1204
1205 def ship_aid_core(myself, track, displayer, kreader):
1206
1207         def find_vessel():
1208                 vn = track.vesselname()
1209                 if vn: return (vn, " on board the %s" % vn)
1210                 vn = track.lastvesselname()
1211                 if vn: return (vn, " ashore from the %s" % vn)
1212                 return (None, " not on a vessel")
1213
1214         def timeevent(t,e):
1215                 if t is None: return ' ' * 22
1216                 return " %-4s %-16s" % (format_time_interval(now - t),e)
1217
1218         displayer.show(track.myname() + find_vessel()[1] + '...')
1219
1220         rotate_nya = '/-\\'
1221
1222         sort = NameSorter()
1223
1224         while True:
1225                 track.catchup()
1226                 now = time.time()
1227
1228                 (vn, s) = find_vessel()
1229                 s = track.myname() + s
1230                 s += " at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
1231                 s += kreader.info()
1232                 s += '\n'
1233
1234                 tbl_s = StringIO()
1235                 tbl = StandingsTable(tbl_s)
1236
1237                 aboard = track.aboard(vn)
1238                 sort.lsort_pa(aboard)
1239
1240                 jobbers = track.jobbers()
1241
1242                 if track.vesselname(): howmany = 'aboard: %2d' % len(aboard)
1243                 else: howmany = ''
1244
1245                 tbl.headings(howmany, '  sorted by '+sort.desc())
1246
1247                 last_jobber = None
1248
1249                 for pa in aboard + jobbers:
1250                         if pa.jobber != last_jobber:
1251                                 last_jobber = pa.jobber
1252                                 tbl.literalline('')
1253                                 tbl.literalline('jobbers '+last_jobber)
1254
1255                         pi = pa.pirate_info()
1256
1257                         xs = ''
1258                         if pa.gunner: xs += 'G '
1259                         else: xs += '  '
1260                         xs += timeevent(pa.last_time, pa.last_event)
1261                         xs += timeevent(pa.last_chat_time, pa.last_chat_chan)
1262
1263                         if pi is None:
1264                                 tbl.pirate_dummy(pa.name, rotate_nya[0], xs)
1265                         else:
1266                                 tbl.pirate(pi, xs)
1267
1268                 s += tbl_s.getvalue()
1269                 displayer.show(s)
1270                 tbl_s.close()
1271
1272                 k = kreader.getch()
1273                 if k is None:
1274                         rotate_nya = rotate_nya[1:3] + rotate_nya[0]
1275                         continue
1276
1277                 if k == 'q': break
1278                 elif k == 'g': sort = SkillSorter('Gunning')
1279                 elif k == 'c': sort = SkillSorter('Carpentry')
1280                 elif k == 's': sort = SkillSorter('Sailing/Rigging')
1281                 elif k == 'b': sort = SkillSorter('Bilging')
1282                 elif k == 'n': sort = SkillSorter('Navigating')
1283                 elif k == 'd': sort = SkillSorter('Battle Navigation')
1284                 elif k == 't': sort = SkillSorter('Treasure Haul')
1285                 elif k == 'a': sort = NameSorter()
1286                 else: pass # unknown key command
1287
1288 #---------- individual keystroke input ----------
1289
1290 class DummyKeystrokeReader:
1291         def __init__(self,fd,timeout_dummy): pass
1292         def stop(self): pass
1293         def getch(self): sleep(1); return None
1294         def info(self): return ' [noninteractive]'
1295
1296 class KeystrokeReader(DummyKeystrokeReader):
1297         def __init__(self, fd, timeout_decisec=0):
1298                 self._fd = fd
1299                 self._saved = termios.tcgetattr(fd)
1300                 a = termios.tcgetattr(fd)
1301                 a[3] &= ~(termios.ECHO | termios.ECHONL |
1302                           termios.ICANON | termios.IEXTEN)
1303                 a[6][termios.VMIN] = 0
1304                 a[6][termios.VTIME] = timeout_decisec
1305                 termios.tcsetattr(fd, termios.TCSANOW, a)
1306         def stop(self):
1307                 termios.tcsetattr(self._fd, termios.TCSANOW, self._saved)
1308         def getch(self):
1309                 debug_flush()
1310                 byte = os.read(self._fd, 1)
1311                 if not len(byte): return None
1312                 return byte
1313         def info(self):
1314                 return ''
1315
1316 #---------- main program ----------
1317
1318 def main():
1319         global opts, fetcher
1320
1321         pa = OptionParser(
1322 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
1323 actions:
1324  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
1325  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
1326  yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
1327  yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG
1328  yoweb-scrape [options] ship-aid CHAT-LOG  (must be .../PIRATE_OCEAN_chat-log*)
1329
1330 display modes (for --display) apply to ship-aid:
1331  --display=dumb       just print new information, scrolling the screen
1332  --display=overwrite  use cursor motion, selective clear, etc. to redraw at top''')
1333         ao = pa.add_option
1334         ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
1335                 help='select ocean OCEAN')
1336         ao('--cache-dir', dest='cache_dir', metavar='DIR',
1337                 default='~/.yoweb-scrape-cache',
1338                 help='cache yoweb pages in DIR')
1339         ao('-D','--debug', action='count', dest='debug', default=0,
1340                 help='enable debugging output')
1341         ao('--debug-fd', type='int', dest='debug_fd',
1342                 help='write any debugging output to specified fd')
1343         ao('-q','--quiet', action='store_true', dest='quiet',
1344                 help='suppress warning output')
1345         ao('--display', action='store', dest='display',
1346                 type='choice', choices=['dumb','overwrite'],
1347                 help='how to display ship aid')
1348
1349         ao_jt = lambda wh, t: ao(
1350                 '--timeout-sa-'+wh, action='store', dest='timeout_'+wh,
1351                 default=t, help=('set timeout for expiring %s jobbers' % wh))
1352         ao_jt('applied',      120)
1353         ao_jt('invited',      120)
1354         ao_jt('declined',      30)
1355         ao_jt('ashore',      1800)
1356
1357         ao('--ship-duty', action='store_true', dest='ship_duty',
1358                 help='show ship duty station puzzles')
1359         ao('--all-puzzles', action='store_false', dest='ship_duty',
1360                 help='show all puzzles, not just ship duty stations')
1361
1362         ao('--min-cache-reuse', type='int', dest='min_max_age',
1363                 metavar='SECONDS', default=60,
1364                 help='always reuse cache yoweb data if no older than this')
1365
1366         (opts,args) = pa.parse_args()
1367         random.seed()
1368
1369         if len(args) < 1:
1370                 print >>sys.stderr, copyright_info
1371                 pa.error('need a mode argument')
1372
1373         if opts.debug_fd is not None:
1374                 opts.debug_file = os.fdopen(opts.debug_fd, 'w')
1375         else:
1376                 opts.debug_file = sys.stdout
1377
1378         mode = args[0]
1379         mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
1380         try: mode_fn = globals()[mode_fn_name]
1381         except KeyError: pa.error('unknown mode "%s"' % mode)
1382
1383         # fixed parameters
1384         opts.expire_age = max(3600, opts.min_max_age)
1385
1386         opts.ship_reboard_clearout = 3600
1387
1388         if opts.cache_dir.startswith('~/'):
1389                 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
1390
1391         if opts.display is None:
1392                 if ((opts.debug > 0 and opts.debug_fd is None)
1393                     or not os.isatty(sys.stdout.fileno())):
1394                         opts.display = 'dumb'
1395                 else:
1396                         opts.display = 'overwrite'
1397
1398         fetcher = Fetcher(opts.ocean, opts.cache_dir)
1399
1400         mode_fn(args[1:], pa.error)
1401
1402 main()