yoweb-scrape

   1 #!/usr/bin/python
   2 # This is part of ypp-sc-tools, a set of third-party tools for assisting
   3 # players of Yohoho Puzzle Pirates.
   4 #
   5 # Copyright (C) 2009 Ian Jackson <ijackson@chiark.greenend.org.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 #
  20 # Yohoho and Puzzle Pirates are probably trademarks of Three Rings and
  21 # are used without permission.  This program is not endorsed or
  22 # sponsored by Three Rings.
  23
  24 copyright_info = '''
  25 yoweb-scrape is part of ypp-sc-tools  Copyright (C) 2009 Ian Jackson
  26 This program comes with ABSOLUTELY NO WARRANTY; this is free software,
  27 and you are welcome to redistribute it under certain conditions.
  28 For details, read the top of the yoweb-scrape file.
  29 '''
  30
  31 #---------- setup ----------
  32
  33 import signal
  34 signal.signal(signal.SIGINT, signal.SIG_DFL)
  35
  36 import os
  37 import time
  38 import urllib
  39 import urllib2
  40 import errno
  41 import sys
  42 import re as regexp
  43 import random
  44 import curses
  45 import termios
  46 import random
  47 from optparse import OptionParser
  48 from StringIO import StringIO
  49
  50 from BeautifulSoup import BeautifulSoup
  51
  52 opts = None
  53
  54 #---------- YPP parameters and arrays ----------
  55
  56 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
  57         '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
  58         '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
  59         '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
  60
  61 core_duty_puzzles = [
  62                 'Gunning',
  63                 ['Sailing','Rigging'],
  64                 'Bilging',
  65                 'Carpentry',
  66                 ]
  67
  68 duty_puzzles = ([ 'Navigating', 'Battle Navigation' ] +
  69                 core_duty_puzzles +
  70                 [ 'Treasure Haul' ])
  71
  72 standingvals = ('Able/Proficient/Distinguished/Respected/Master'+
  73                 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
  74 standing_limit = len(standingvals)
  75
  76 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
  77
  78 max_pirate_namelen = 12
  79
  80
  81 #---------- general utilities ----------
  82
  83 def debug(m):
  84         if opts.debug > 0:
  85                 print >>opts.debug_file, m
  86
  87 def debug_flush():
  88         if opts.debug > 0:
  89                 opts.debug_file.flush()
  90
  91 def sleep(seconds):
  92         debug_flush()
  93         time.sleep(seconds)
  94
  95 def format_time_interval(ti):
  96         if ti < 120: return '%d:%02d' % (ti / 60, ti % 60)
  97         if ti < 7200: return '%2dm' % (ti / 60)
  98         if ti < 86400: return '%dh' % (ti / 3600)
  99         return '%dd' % (ti / 86400)
 100
 101 #---------- caching and rate-limiting data fetcher ----------
 102
 103 class Fetcher:
 104         def __init__(self, ocean, cachedir):
 105                 debug('Fetcher init %s' % cachedir)
 106                 self.ocean = ocean
 107                 self.cachedir = cachedir
 108                 try: os.mkdir(cachedir)
 109                 except (OSError,IOError), oe:
 110                         if oe.errno != errno.EEXIST: raise
 111                 self._cache_scan(time.time())
 112
 113         def default_ocean(self, ocean='ice'):
 114                 if self.ocean is None:
 115                         self.ocean = ocean
 116
 117         def _cache_scan(self, now):
 118                 # returns list of ages, unsorted
 119                 ages = []
 120                 debug('Fetcher   scan_cache')
 121                 for leaf in os.listdir(self.cachedir):
 122                         if not leaf.startswith('#'): continue
 123                         path = self.cachedir + '/' + leaf
 124                         try: s = os.stat(path)
 125                         except (OSError,IOError), oe:
 126                                 if oe.errno != errno.ENOENT: raise
 127                                 continue
 128                         age = now - s.st_mtime
 129                         if age > opts.expire_age:
 130                                 debug('Fetcher    expire %d %s' % (age, path))
 131                                 try: os.remove(path)
 132                                 except (OSError,IOError), oe:
 133                                         if oe.errno != errno.ENOENT: raise
 134                                 continue
 135                         ages.append(age)
 136                 return ages
 137
 138         def need_wait(self, now, imaginary=[]):
 139                 ages = self._cache_scan(now)
 140                 ages += imaginary
 141                 ages.sort()
 142                 debug('Fetcher   ages ' + `ages`)
 143                 min_age = 1
 144                 need_wait = 0
 145                 for age in ages:
 146                         if age < min_age and age <= 5:
 147                                 debug('Fetcher   morewait min=%d age=%d' %
 148                                         (min_age, age))
 149                                 need_wait = max(need_wait, min_age - age)
 150                         min_age += 3
 151                         min_age *= 1.25
 152                 if need_wait > 0:
 153                         need_wait += random.random() - 0.5
 154                 return need_wait
 155
 156         def _rate_limit_cache_clean(self, now):
 157                 need_wait = self.need_wait(now)
 158                 if need_wait > 0:
 159                         debug('Fetcher   wait %d' % need_wait)
 160                         sleep(need_wait)
 161
 162         def fetch(self, url, max_age):
 163                 debug('Fetcher fetch %s' % url)
 164                 cache_corename = urllib.quote_plus(url)
 165                 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
 166                 try: f = file(cache_item, 'r')
 167                 except (OSError,IOError), oe:
 168                         if oe.errno != errno.ENOENT: raise
 169                         f = None
 170                 now = time.time()
 171                 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
 172                 if f is not None:
 173                         s = os.fstat(f.fileno())
 174                         age = now - s.st_mtime
 175                         if age > max_age:
 176                                 debug('Fetcher  stale %d < %d'% (max_age, age))
 177                                 f = None
 178                 if f is not None:
 179                         data = f.read()
 180                         f.close()
 181                         debug('Fetcher  cached %d > %d' % (max_age, age))
 182                         return data
 183
 184                 debug('Fetcher  fetch')
 185                 self._rate_limit_cache_clean(now)
 186
 187                 stream = urllib2.urlopen(url)
 188                 data = stream.read()
 189                 cache_tmp = "%s/#%s~%d#" % (
 190                         self.cachedir, cache_corename, os.getpid())
 191                 f = file(cache_tmp, 'w')
 192                 f.write(data)
 193                 f.close()
 194                 os.rename(cache_tmp, cache_item)
 195                 debug('Fetcher  stored')
 196                 return data
 197
 198         def yoweb(self, kind, tail, max_age):
 199                 self.default_ocean()
 200                 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
 201                         self.ocean, kind, tail)
 202                 return self.fetch(url, max_age)
 203
 204 #---------- logging assistance for troubled screenscrapers ----------
 205
 206 class SoupLog:
 207         def __init__(self):
 208                 self.msgs = [ ]
 209         def msg(self, m):
 210                 self.msgs.append(m)
 211         def soupm(self, obj, m):
 212                 self.msg(m + '; in ' + `obj`)
 213         def needs_msgs(self, child_souplog):
 214                 self.msgs += child_souplog.msgs
 215                 child_souplog.msgs = [ ]
 216
 217 def soup_text(obj):
 218         str = ''.join(obj.findAll(text=True))
 219         return str.strip()
 220
 221 class SomethingSoupInfo(SoupLog):
 222         def __init__(self, kind, tail, max_age):
 223                 SoupLog.__init__(self)
 224                 html = fetcher.yoweb(kind, tail, max_age)
 225                 self._soup = BeautifulSoup(html,
 226                         convertEntities=BeautifulSoup.HTML_ENTITIES
 227                         )
 228
 229 #---------- scraper for pirate pages ----------
 230
 231 class PirateInfo(SomethingSoupInfo):
 232         # Public data members:
 233         #  pi.standings = { 'Treasure Haul': 'Able' ... }
 234         #  pi.name = name
 235         #  pi.crew = (id, name)
 236         #  pi.flag = (id, name)
 237         #  pi.msgs = [ 'message describing problem with scrape' ]
 238
 239         def __init__(self, pirate, max_age=300):
 240                 SomethingSoupInfo.__init__(self,
 241                         'pirate.wm?target=', pirate, max_age)
 242                 self.name = pirate
 243                 self._find_standings()
 244                 self.crew = self._find_crewflag('crew',
 245                         '^/yoweb/crew/info\\.wm')
 246                 self.flag = self._find_crewflag('flag',
 247                         '^/yoweb/flag/info\\.wm')
 248
 249         def _find_standings(self):
 250                 imgs = self._soup.findAll('img',
 251                         src=regexp.compile('/yoweb/images/stat.*'))
 252                 re = regexp.compile(
 253 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
 254                         )
 255                 standings = { }
 256
 257                 for skill in puzzles:
 258                         standings[skill] = [ ]
 259
 260                 skl = SoupLog()
 261
 262                 for img in imgs:
 263                         try: puzzle = img['alt']
 264                         except KeyError: continue
 265
 266                         if not puzzle in puzzles:
 267                                 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
 268                                 continue
 269                         key = img.findParent('td')
 270                         if key is None:
 271                                 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
 272                                 continue
 273                         valelem = key.findNextSibling('td')
 274                         if valelem is None:
 275                                 skl.soupm(key, 'puzzle missing sibling "%s"'
 276                                         % puzzle)
 277                                 continue
 278                         valstr = soup_text(valelem)
 279                         match = re.match(valstr)
 280                         if match is None:
 281                                 skl.soupm(key, ('puzzle "%s" unparseable'+
 282                                         ' standing "%s"') % (puzzle, valstr))
 283                                 continue
 284                         standing = match.group(match.lastindex)
 285                         standings[puzzle].append(standing)
 286
 287                 self.standings = { }
 288
 289                 for puzzle in puzzles:
 290                         sl = standings[puzzle]
 291                         if len(sl) > 1:
 292                                 skl.msg('puzzle "%s" multiple standings %s' %
 293                                                 (puzzle, `sl`))
 294                                 continue
 295                         if not sl:
 296                                 skl.msg('puzzle "%s" no standing found' % puzzle)
 297                                 continue
 298                         standing = sl[0]
 299                         for i in range(0, standing_limit):
 300                                 if standing == standingvals[i]:
 301                                         self.standings[puzzle] = i
 302                         if not puzzle in self.standings:
 303                                 skl.msg('puzzle "%s" unknown standing "%s"' %
 304                                         (puzzle, standing))
 305
 306                 all_standings_ok = True
 307                 for puzzle in puzzles:
 308                         if not puzzle in self.standings:
 309                                 self.needs_msgs(skl)
 310
 311         def _find_crewflag(self, cf, yoweb_re):
 312                 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
 313                 if len(things) != 1:
 314                         self.msg('zero or several %s id references found' % cf)
 315                         return None
 316                 thing = things[0]
 317                 id_re = '\\b%sid\\=(\\w+)$' % cf
 318                 id_haystack = thing['href']
 319                 match = regexp.compile(id_re).search(id_haystack)
 320                 if match is None:
 321                         self.soupm(thing, ('incomprehensible %s id ref'+
 322                                 ' (%s in %s)') % (cf, id_re, id_haystack))
 323                         return None
 324                 name = soup_text(thing)
 325                 return (match.group(1), name)
 326
 327         def __str__(self):
 328                 return `(self.crew, self.flag, self.standings, self.msgs)`
 329
 330 #---------- scraper for crew pages ----------
 331
 332 class CrewInfo(SomethingSoupInfo):
 333         # Public data members:
 334         #  ci.crew = [ ('Captain',        ['Pirate', ...]),
 335         #              ('Senior Officer', [...]),
 336         #               ... ]
 337         #  pi.msgs = [ 'message describing problem with scrape' ]
 338
 339         def __init__(self, crewid, max_age=300):
 340                 SomethingSoupInfo.__init__(self,
 341                         'crew/info.wm?crewid=', crewid, max_age)
 342                 self._find_crew()
 343
 344         def _find_crew(self):
 345                 self.crew = []
 346                 capts = self._soup.findAll('img',
 347                         src='/yoweb/images/crew-captain.png')
 348                 if len(capts) != 1:
 349                         self.msg('crew members: no. of captain images != 1')
 350                         return
 351                 tbl = capts[0]
 352                 while not tbl.find('a', href=pirate_ref_re):
 353                         tbl = tbl.findParent('table')
 354                         if not tbl:
 355                                 self.msg('crew members: cannot find table')
 356                                 return
 357                 current_rank_crew = None
 358                 crew_rank_re = regexp.compile('/yoweb/images/crew')
 359                 for row in tbl.contents:
 360                         # findAll(recurse=False)
 361                         if isinstance(row,basestring):
 362                                 continue
 363
 364                         is_rank = row.find('img', attrs={'src': crew_rank_re})
 365                         if is_rank:
 366                                 rank = soup_text(row)
 367                                 current_rank_crew = []
 368                                 self.crew.append((rank, current_rank_crew))
 369                                 continue
 370                         for cell in row.findAll('a', href=pirate_ref_re):
 371                                 if current_rank_crew is None:
 372                                         self.soupm(cell, 'crew members: crew'
 373                                                 ' before rank')
 374                                         continue
 375                                 current_rank_crew.append(soup_text(cell))
 376
 377         def __str__(self):
 378                 return `(self.crew, self.msgs)`
 379
 380 #---------- pretty-printer for tables of pirate puzzle standings ----------
 381
 382 class StandingsTable:
 383         def __init__(self, f, use_puzzles=None, col_width=6, gap_every=5):
 384                 if use_puzzles is None:
 385                         if opts.ship_duty:
 386                                 use_puzzles=duty_puzzles
 387                         else:
 388                                 use_puzzles=puzzles
 389                 self._puzzles = use_puzzles
 390                 self.f = f
 391                 self._cw = col_width-1
 392                 self._gap_every = gap_every
 393                 self._linecount = 0
 394                 self._o = f.write
 395
 396         def _nl(self): self._o('\n')
 397
 398         def _pline(self, lhs, puzstrs, extra):
 399                 if (self._linecount > 0
 400                     and self._gap_every is not None
 401                     and not (self._linecount % self._gap_every)):
 402                         self._nl()
 403                 self._o('%-*s' % (max(max_pirate_namelen+1, 15), lhs))
 404                 for v in puzstrs:
 405                         self._o(' %-*.*s' % (self._cw,self._cw, v))
 406                 if extra:
 407                         self._o(' ' + extra)
 408                 self._nl()
 409                 self._linecount += 1
 410
 411         def _puzstr(self, pi, puzzle):
 412                 if not isinstance(puzzle,list): puzzle = [puzzle]
 413                 try: standing = max([pi.standings[p] for p in puzzle])
 414                 except KeyError: return '?'
 415                 if not standing: return ''
 416                 s = ''
 417                 if self._cw > 4:
 418                         c1 = standingvals[standing][0]
 419                         if standing < 3: c1 = c1.lower() # 3 = Master
 420                         s += `standing`
 421                 if self._cw > 5:
 422                         s += ' '
 423                 s += '*' * (standing / 2)
 424                 s += '+' * (standing % 2)
 425                 return s
 426
 427         def headings(self, lhs='', rhs=None):
 428                 def puzn_redact(name):
 429                         if isinstance(name,list):
 430                                 return '/'.join(
 431                                         ["%.*s" % (self._cw/2, puzn_redact(n))
 432                                          for n in name])
 433                         spc = name.find(' ')
 434                         if spc < 0: return name
 435                         return name[0:min(4,spc)] + name[spc+1:]
 436                 self._linecount = -2
 437                 self._pline(lhs, map(puzn_redact, self._puzzles), rhs)
 438                 self._linecount = 0
 439         def literalline(self, line):
 440                 self._o(line)
 441                 self._nl()
 442                 self._linecount = 0
 443         def pirate_dummy(self, name, standingstring, extra=None):
 444                 standings = standingstring * len(self._puzzles)
 445                 self._pline(' '+name, standings, extra)
 446         def pirate(self, pi, extra=None):
 447                 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
 448                 self._pline(' '+pi.name, puzstrs, extra)
 449
 450
 451 #---------- chat log parser ----------
 452
 453 class PirateAboard:
 454         # This is essentially a transparent, dumb, data class.
 455         #  pa.v                 may be None
 456         #  pa.name
 457         #  pa.last_time
 458         #  pa.last_event
 459         #  pa.gunner
 460         #  pa.last_chat_time
 461         #  pa.last_chat_chan
 462         #  pa.pi
 463
 464         # Also used for jobbing applicants:
 465         #               happens when                    expires (to "-")
 466         #   -            disembark, leaves crew          no
 467         #   aboard       evidence of them being aboard   no
 468         #   applied      "has applied for the job"       120s, configurable
 469         #   ashore       "has taken a job"               30min, configurable
 470         #   declined     "declined the job offer"        30s, configurable
 471         #   invited      "has been invited to job"       120s, configurable
 472         #
 473         #  pa.jobber    None, 'ashore', 'applied', 'invited', 'declined'
 474         #  pa.expires   expiry time time
 475
 476         def __init__(pa, pn, v, time, event):
 477                 pa.name = pn
 478                 pa.v = v
 479                 pa.last_time = time
 480                 pa.last_event = event
 481                 pa.last_chat_time = None
 482                 pa.last_chat_chan = None
 483                 pa.gunner = False
 484                 pa.pi = None
 485                 pa.jobber = None
 486                 pa.expires = None
 487
 488         def pirate_info(pa):
 489                 now = time.time()
 490                 if pa.pi:
 491                         age = now - pa.pi_fetched
 492                         guide = random.randint(120,240)
 493                         if age <= guide:
 494                                 return pa.pi
 495                         debug('PirateAboard refresh %d > %d  %s' % (
 496                                 age, guide, pa.name))
 497                         imaginary = [2,4]
 498                 else:
 499                         imaginary = [1]
 500                 wait = fetcher.need_wait(now, imaginary)
 501                 if wait:
 502                         debug('PirateAboard fetcher not ready %d' % wait)
 503                         return pa.pi
 504                 pa.pi = PirateInfo(pa.name, 600)
 505                 pa.pi_fetched = now
 506                 return pa.pi
 507
 508 class ChatLogTracker:
 509         # This is quite complex so we make it opaque.  Use the
 510         # official invokers, accessors etc.
 511
 512         def __init__(self, myself_pi, logfn):
 513                 self._pl = {}   # self._pl['Pirate'] =
 514                 self._vl = {}   #   self._vl['Vessel']['Pirate'] = PirateAboard
 515                                 # self._vl['Vessel']['#lastinfo']
 516                                 # self._vl['Vessel']['#name']
 517                                 # self._v = self._vl[self._vessel]
 518                 self._date = None
 519                 self._myself = myself_pi
 520                 self._lbuf = ''
 521                 self._f = file(logfn)
 522                 flen = os.fstat(self._f.fileno()).st_size
 523                 max_backlog = 500000
 524                 if flen > max_backlog:
 525                         startpos = flen - max_backlog
 526                         self._f.seek(startpos)
 527                         self._f.readline()
 528                 self._progress = [0, flen - self._f.tell()]
 529                 self._disembark_myself()
 530                 self._need_redisplay = False
 531                 self._lastvessel = None
 532
 533         def _disembark_myself(self):
 534                 self._v = None
 535                 self._vessel = None
 536                 self.force_redisplay()
 537
 538         def force_redisplay(self):
 539                 self._need_redisplay = True
 540
 541         def _vessel_updated(self, v, timestamp):
 542                 if v is None: return
 543                 v['#lastinfo'] = timestamp
 544                 self.force_redisplay()
 545
 546         def _onboard_event(self,v,timestamp,pirate,event,jobber=None):
 547                 pa = self._pl.get(pirate, None)
 548                 if pa is not None and pa.v is v:
 549                         pa.last_time = timestamp
 550                         pa.last_event = event
 551                 else:
 552                         if pa is not None and pa.v is not None:
 553                                 del pa.v[pirate]
 554                         pa = PirateAboard(pirate, v, timestamp, event)
 555                         self._pl[pirate] = pa
 556                         if v is not None: v[pirate] = pa
 557                 pa.jobber = jobber
 558
 559                 if jobber is None: timeout = None
 560                 else: timeout = getattr(opts, 'timeout_'+jobber)
 561                 if timeout is None: pa.expires = None
 562                 else: pa.expires = timestamp + timeout
 563                 self._vessel_updated(v, timestamp)
 564                 return pa
 565
 566         def _expire_jobbers(self, now):
 567                 for pa in self._pl.values():
 568                         if pa.expires is None: continue
 569                         if pa.expires >= now: continue
 570                         v = pa.v
 571                         del self._pl[pa.name]
 572                         if v is not None: del v[pa.name]
 573                         self.force_redisplay()
 574
 575         def _trash_vessel(self, v):
 576                 for pn in v:
 577                         if pn.startswith('#'): continue
 578                         del self._pl[pn]
 579                 vn = v['#name']
 580                 del self._vl[vn]
 581                 if v is self._v: self._disembark_myself()
 582                 self.force_redisplay()
 583
 584         def _vessel_stale(self, v, timestamp):
 585                 return timestamp - v['#lastinfo'] > opts.ship_reboard_clearout
 586
 587         def _vessel_check_expire(self, v, timestamp):
 588                 if not self._vessel_stale(v, timestamp):
 589                         return v
 590                 self._debug_line_disposition(timestamp,'',
 591                         'stale-reset ' + v['#name'])
 592                 self._trash_vessel(v)
 593                 return None
 594
 595         def expire_garbage(self, timestamp):
 596                 for v in self._vl.values():
 597                         self._vessel_check_expire(v, timestamp)
 598
 599         def _vessel_lookup(self, vn, timestamp, dml=[], create=False):
 600                 v = self._vl.get(vn, None)
 601                 if v is not None:
 602                         v = self._vessel_check_expire(v, timestamp)
 603                 if v is not None:
 604                         dml.append('found')
 605                         return v
 606                 if not create:
 607                         dml.append('no')
 608                 dml.append('new')
 609                 self._vl[vn] = v = { '#name': vn }
 610                 self._vessel_updated(v, timestamp)
 611                 return v
 612
 613         def _find_matching_vessel(self, pattern, timestamp, cmdr,
 614                                         dml=[], create=False):
 615                 # use when a commander pirate `cmdr' specified a vessel
 616                 #  by name `pattern' (either may be None)
 617                 # if create is true, will create the vessel
 618                 #  record if an exact name is specified
 619
 620                 if (pattern is not None and
 621                     not '*' in pattern
 622                     and len(pattern.split(' ')) == 2):
 623                         vn = pattern.title()
 624                         dml.append('exact')
 625                         return self._vessel_lookup(
 626                                 vn, timestamp, dml=dml, create=create)
 627
 628                 if pattern is None:
 629                         pattern_check = lambda vn: True
 630                 else:
 631                         re = '(?:.* )?%s$' % pattern.lower().replace('*','.+')
 632                         pattern_check = regexp.compile(re, regexp.I).match
 633
 634                 tries = []
 635
 636                 cmdr_pa = self._pl.get(cmdr, None)
 637                 if cmdr_pa: tries.append((cmdr_pa.v, 'cmdr'))
 638
 639                 tries.append((self._v, 'here'))
 640                 tried_vns = []
 641
 642                 for (v, dm) in tries:
 643                         if v is None: dml.append(dm+'?'); continue
 644
 645                         vn = v['#name']
 646                         if not pattern_check(vn):
 647                                 tried_vns.append(vn)
 648                                 dml.append(dm+'#')
 649                                 continue
 650
 651                         dml.append(dm+'!')
 652                         return v
 653
 654                 if pattern is not None and '*' in pattern:
 655                         search = [
 656                                 (vn,v)
 657                                 for (vn,v) in self._vl.iteritems()
 658                                 if not self._vessel_stale(v, timestamp)
 659                                 if pattern_check(vn)
 660                                 ]
 661                         #debug('CLT-RE /%s/ wanted (%s) searched (%s)' % (
 662                         #       re,
 663                         #       '/'.join(tried_vns),
 664                         #       '/'.join([vn for (vn,v) in search])))
 665
 666                         if len(search)==1:
 667                                 dml.append('one')
 668                                 return search[0][1]
 669                         elif search:
 670                                 dml.append('many')
 671                         else:
 672                                 dml.append('none')
 673
 674         def _debug_line_disposition(self,timestamp,l,m):
 675                 debug('CLT %13s %-40s %s' % (timestamp,m,l))
 676
 677         def _rm_crew_l(self,re,l):
 678                 m = regexp.match(re,l)
 679                 if m and m.group(2) == self._myself.crew[1]:
 680                         return m.group(1)
 681                 else:
 682                         return None
 683
 684         def local_command(self, metacmd):
 685                 # returns None if all went well, or problem message
 686                 return self._command(self._myself.name, metacmd,
 687                         "local", time.time(),
 688                         (lambda m: debug('CMD %s' % metacmd)))
 689
 690         def _command(self, cmdr, metacmd, chan, timestamp, d):
 691                 # returns None if all went well, or problem message
 692                 metacmd = regexp.sub('\\s+', ' ', metacmd).strip()
 693                 m2 = regexp.match(
 694                     '/([adj]) (?:([A-Za-z* ]+)\\s*:)?([A-Za-z ]+)$',
 695                     metacmd)
 696                 if not m2: return "unknown syntax or command"
 697
 698                 (cmd, pattern, targets) = m2.groups()
 699                 dml = ['cmd', chan, cmd]
 700
 701                 if cmd == 'a': each = self._onboard_event
 702                 elif cmd == 'd': each = disembark
 703                 else: each = lambda *l: self._onboard_event(*l,
 704                                 **{'jobber':'applied'})
 705
 706                 if cmdr == self._myself.name:
 707                         dml.append('self')
 708                         how = 'cmd: %s' % cmd
 709                 else:
 710                         dml.append('other')
 711                         how = 'cmd: %s %s' % (cmd,cmdr)
 712
 713                 if cmd == 'j':
 714                         if pattern is not None:
 715                                 return "/j command does not take a vessel"
 716                         v = None
 717                 else:
 718                         v = self._find_matching_vessel(
 719                                 pattern, timestamp, cmdr,
 720                                 dml, create=True)
 721
 722                 if cmd == 'j' or v is not None:
 723                         targets = targets.strip().split(' ')
 724                         dml.append(`len(targets)`)
 725                         for target in targets:
 726                                 each(v, timestamp, target.title(), how)
 727                         self._vessel_updated(v, timestamp)
 728
 729                 dm = ' '.join(dml)
 730                 return d(dm)
 731
 732                 return None
 733
 734         def chatline(self,l):
 735                 rm = lambda re: regexp.match(re,l)
 736                 d = lambda m: self._debug_line_disposition(timestamp,l,m)
 737                 rm_crew = lambda re: self._rm_crew_l(re,l)
 738                 timestamp = None
 739
 740                 m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$')
 741                 if m:
 742                         self._date = [int(x) for x in m.groups()]
 743                         self._previous_timestamp = None
 744                         return d('date '+`self._date`)
 745
 746                 if self._date is None:
 747                         return d('date unset')
 748
 749                 m = rm('\\[(\d\d):(\d\d):(\d\d)\\] ')
 750                 if not m:
 751                         return d('no timestamp')
 752
 753                 while True:
 754                         time_tuple = (self._date +
 755                                       [int(x) for x in m.groups()] +
 756                                       [-1,-1,-1])
 757                         timestamp = time.mktime(time_tuple)
 758                         if timestamp >= self._previous_timestamp: break
 759                         self._date[2] += 1
 760                         self._debug_line_disposition(timestamp,'',
 761                                 'new date '+`self._date`)
 762
 763                 self._previous_timestamp = timestamp
 764
 765                 l = l[l.find(' ')+1:]
 766
 767                 def ob_x(pirate,event):
 768                         return self._onboard_event(
 769                                         self._v, timestamp, pirate, event)
 770                 def ob1(did): ob_x(m.group(1), did); return d(did)
 771                 def oba(did): return ob1('%s %s' % (did, m.group(2)))
 772
 773                 def jb(pirate,jobber):
 774                         return self._onboard_event(
 775                                 None, timestamp, pirate,
 776                                 ("jobber %s" % jobber),
 777                                 jobber=jobber
 778                                 )
 779
 780                 def disembark(v, timestamp, pirate, event):
 781                         self._onboard_event(
 782                                         v, timestamp, pirate, 'leaving '+event)
 783                         del v[pirate]
 784                         del self._pl[pirate]
 785
 786                 def disembark_me(why):
 787                         self._disembark_myself()
 788                         return d('disembark-me '+why)
 789
 790                 m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
 791                 if m:
 792                         dm = ['boarding']
 793                         pn = self._myself.name
 794                         vn = m.group(1)
 795                         v = self._vessel_lookup(vn, timestamp, dm, create=True)
 796                         self._lastvessel = self._vessel = vn
 797                         self._v = v
 798                         ob_x(pn, 'we boarded')
 799                         self.expire_garbage(timestamp)
 800                         return d(' '.join(dm))
 801
 802                 if self._v is None:
 803                         return d('no vessel')
 804
 805                 m = rm('(\\w+) has come aboard\\.$')
 806                 if m: return ob1('boarded');
 807
 808                 m = rm('You have ordered (\\w+) to do some (\\S.*\\S)\\.$')
 809                 if m:
 810                         (who,what) = m.groups()
 811                         pa = ob_x(who,'ord '+what)
 812                         if what == 'Gunning':
 813                                 pa.gunner = True
 814                         return d('duty order')
 815
 816                 m = rm('(\\w+) abandoned a (\\S.*\\S) station\\.$')
 817                 if m: oba('stopped'); return d("end")
 818
 819                 def chat_core(speaker, chan):
 820                         try: pa = self._pl[speaker]
 821                         except KeyError: return 'mystery'
 822                         if pa.v is not None and pa.v is not self._v:
 823                                 return 'elsewhere'
 824                         pa.last_chat_time = timestamp
 825                         pa.last_chat_chan = chan
 826                         self.force_redisplay()
 827                         return 'here'
 828
 829                 def chat(chan):
 830                         speaker = m.group(1)
 831                         dm = chat_core(speaker, chan)
 832                         return d('chat %s %s' % (chan, dm))
 833
 834                 def chat_metacmd(chan):
 835                         (cmdr, metacmd) = m.groups()
 836                         whynot = self._command(
 837                                 cmdr, metacmd, chan, timestamp, d)
 838                         if whynot is not None:
 839                                 return chat(chan)
 840                         else:
 841                                 chat_core(cmdr, 'cmd '+chan)
 842
 843                 m = rm('(\\w+) (?:issued an order|ordered everyone) "')
 844                 if m: return ob1('general order');
 845
 846                 m = rm('(\\w+) says, "')
 847                 if m: return chat('public')
 848
 849                 m = rm('(\\w+) tells ye, "')
 850                 if m: return chat('private')
 851
 852                 m = rm('Ye told (\\w+), "(.*)"$')
 853                 if m: return chat_metacmd('private')
 854
 855                 m = rm('(\\w+) flag officer chats, "')
 856                 if m: return chat('flag officer')
 857
 858                 m = rm('(\\w+) officer chats, "(.*)"$')
 859                 if m: return chat_metacmd('officer')
 860
 861                 m = rm('Ye accepted the offer to job with ')
 862                 if m: return disembark_me('jobbing')
 863
 864                 m = rm('Ye hop on the ferry and are whisked away ')
 865                 if m: return disembark_me('ferry')
 866
 867                 m = rm('Whisking away to yer home on the magical winds')
 868                 if m: return disembark_me('home')
 869
 870                 m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
 871                 if m:
 872                         pl = m.group(1).split(', ')
 873                         if not self._myself.name in pl:
 874                                 return d('lost melee')
 875                         for pn in pl:
 876                                 if ' ' in pn: continue
 877                                 ob_x(pn,'won melee')
 878                         return d('won melee')
 879
 880                 m = rm('(\\w+) is eliminated\\!')
 881                 if m: return ob1('eliminated in fray');
 882
 883                 m = rm('(\\w+) has driven \w+ from the ship\\!')
 884                 if m: return ob1('boarder repelled');
 885
 886                 m = rm('\w+ has bested (\\w+), and turns'+
 887                         ' to the rest of the ship\\.')
 888                 if m: return ob1('boarder unrepelled');
 889
 890                 pirate = rm_crew("(\\w+) has taken a job with '(.*)'\\.")
 891                 if pirate: return jb(pirate, 'ashore')
 892
 893                 pirate = rm_crew("(\\w+) has left '(.*)'\\.")
 894                 if pirate:
 895                         disembark(self._v, timestamp, pirate, 'left crew')
 896                         return d('left crew')
 897
 898                 m = rm('(\w+) has applied for the posted job\.')
 899                 if m: return jb(m.group(1), 'applied')
 900
 901                 pirate= rm_crew("(\\w+) has been invited to job for '(.*)'\\.")
 902                 if pirate: return jb(pirate, 'invited')
 903
 904                 pirate = rm_crew("(\\w+) declined the job offer for '(.*)'\\.")
 905                 if pirate: return jb(pirate, 'declined')
 906
 907                 m = rm('(\\w+) has left the vessel\.')
 908                 if m:
 909                         pirate = m.group(1)
 910                         disembark(self._v, timestamp, pirate, 'disembarked')
 911                         return d('disembarked')
 912
 913                 return d('not-matched')
 914
 915         def _str_pa(self, pn, pa):
 916                 assert self._pl[pn] == pa
 917                 s = ' '*20 + "%s %-*s %13s %-30s %13s %-20s %13s" % (
 918                         (' ','G')[pa.gunner],
 919                         max_pirate_namelen, pn,
 920                         pa.last_time, pa.last_event,
 921                         pa.last_chat_time, pa.last_chat_chan,
 922                         pa.jobber)
 923                 if pa.expires is not None:
 924                         s += " %-5d" % (pa.expires - pa.last_time)
 925                 s += "\n"
 926                 return s
 927
 928         def _str_vessel(self, vn, v):
 929                 s = ' vessel %s\n' % vn
 930                 s += ' '*20 + "%-*s   %13s\n" % (
 931                                 max_pirate_namelen, '#lastinfo',
 932                                 v['#lastinfo'])
 933                 assert v['#name'] == vn
 934                 for pn in sorted(v.keys()):
 935                         if pn.startswith('#'): continue
 936                         pa = v[pn]
 937                         assert pa.v == v
 938                         s += self._str_pa(pn,pa)
 939                 return s
 940
 941         def __str__(self):
 942                 s = '''<ChatLogTracker
 943  myself %s
 944  vessel %s
 945 '''                     % (self._myself.name, self._vessel)
 946                 assert ((self._v is None and self._vessel is None) or
 947                         (self._v is self._vl[self._vessel]))
 948                 if self._vessel is not None:
 949                         s += self._str_vessel(self._vessel, self._v)
 950                 for vn in sorted(self._vl.keys()):
 951                         if vn == self._vessel: continue
 952                         s += self._str_vessel(vn, self._vl[vn])
 953                 s += " elsewhere\n"
 954                 for p in self._pl:
 955                         pa = self._pl[p]
 956                         if pa.v is not None:
 957                                 assert pa.v[p] is pa
 958                                 assert pa.v in self._vl.values()
 959                         else:
 960                                 s += self._str_pa(pa.name, pa)
 961                 s += '>\n'
 962                 return s
 963
 964         def catchup(self, progress=None):
 965                 while True:
 966                         more = self._f.readline()
 967                         if not more: break
 968
 969                         self._progress[0] += len(more)
 970                         if progress: progress.progress(*self._progress)
 971
 972                         self._lbuf += more
 973                         if self._lbuf.endswith('\n'):
 974                                 self.chatline(self._lbuf.rstrip())
 975                                 self._lbuf = ''
 976                                 if opts.debug >= 2:
 977                                         debug(self.__str__())
 978                 self._expire_jobbers(time.time())
 979
 980                 if progress: progress.caughtup()
 981
 982         def changed(self):
 983                 rv = self._need_redisplay
 984                 self._need_redisplay = False
 985                 return rv
 986         def myname(self):
 987                 # returns our pirate name
 988                 return self._myself.name
 989         def vesselname(self):
 990                 # returns the vessel name we're aboard or None
 991                 return self._vessel
 992         def lastvesselname(self):
 993                 # returns the last vessel name we were aboard or None
 994                 return self._lastvessel
 995         def aboard(self, vesselname=True):
 996                 # returns a list of PirateAboard the vessel
 997                 #  sorted by pirate name
 998                 #  you can pass this None and you'll get []
 999                 #  or True for the current vessel (which is the default)
1000                 #  the returned value is a fresh list of persistent
1001                 #  PirateAboard objects
1002                 if vesselname is True: v = self._v
1003                 else: v = self._vl.get(vesselname.title())
1004                 if v is None: return []
1005                 return [ v[pn]
1006                          for pn in sorted(v.keys())
1007                          if not pn.startswith('#') ]
1008         def jobbers(self):
1009                 # returns a the jobbers' PirateAboards,
1010                 # sorted by jobber class and reverse of expiry time
1011                 l = [ pa
1012                       for pa in self._pl.values()
1013                       if pa.jobber is not None
1014                     ]
1015                 def compar_key(pa):
1016                         return (pa.jobber, -pa.expires)
1017                 l.sort(key = compar_key)
1018                 return l
1019
1020 #---------- implementations of actual operation modes ----------
1021
1022 def do_pirate(pirates, bu):
1023         print '{'
1024         for pirate in pirates:
1025                 info = PirateInfo(pirate)
1026                 print '%s: %s,' % (`pirate`, info)
1027         print '}'
1028
1029 def prep_crew_of(args, bu, max_age=300):
1030         if len(args) != 1: bu('crew-of takes one pirate name')
1031         pi = PirateInfo(args[0], max_age)
1032         if pi.crew is None: return None
1033         return CrewInfo(pi.crew[0], max_age)
1034
1035 def do_crew_of(args, bu):
1036         ci = prep_crew_of(args, bu)
1037         print ci
1038
1039 def do_standings_crew_of(args, bu):
1040         ci = prep_crew_of(args, bu, 60)
1041         tab = StandingsTable(sys.stdout)
1042         tab.headings()
1043         for (rank, members) in ci.crew:
1044                 if not members: continue
1045                 tab.literalline('')
1046                 tab.literalline('%s:' % rank)
1047                 for p in members:
1048                         pi = PirateInfo(p, random.randint(900,1800))
1049                         tab.pirate(pi)
1050
1051 class ProgressPrintPercentage:
1052         def __init__(self, f=sys.stdout):
1053                 self._f = f
1054         def progress_string(self,done,total):
1055                 return "scan chat logs %3d%%\r" % ((done*100) / total)
1056         def progress(self,*a):
1057                 self._f.write(self.progress_string(*a))
1058                 self._f.flush()
1059         def show_init(self, pirate, ocean):
1060                 print >>self._f, 'Starting up, %s on the %s ocean' % (
1061                         pirate, ocean)
1062         def caughtup(self):
1063                 self._f.write('                   \r')
1064                 self._f.flush()
1065
1066 #----- modes which use the chat log parser are quite complex -----
1067
1068 def prep_chat_log(args, bu,
1069                 progress=ProgressPrintPercentage(),
1070                 max_myself_age=3600):
1071         if len(args) != 1: bu('this action takes only chat log filename')
1072         logfn = args[0]
1073         logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_'
1074         match = regexp.match(logfn_re, logfn)
1075         if not match: bu('chat log filename is not in expected format')
1076         (pirate, ocean) = match.groups()
1077         fetcher.default_ocean(ocean)
1078
1079         progress.show_init(pirate, fetcher.ocean)
1080         myself = PirateInfo(pirate,max_myself_age)
1081         track = ChatLogTracker(myself, logfn)
1082
1083         opts.debug -= 2
1084         track.catchup(progress)
1085         opts.debug += 2
1086
1087         track.force_redisplay()
1088
1089         return (myself, track)
1090
1091 def do_track_chat_log(args, bu):
1092         (myself, track) = prep_chat_log(args, bu)
1093         while True:
1094                 track.catchup()
1095                 if track.changed():
1096                         print track
1097                 sleep(0.5 + 0.5 * random.random())
1098
1099 #----- ship management aid -----
1100
1101 class Display_dumb(ProgressPrintPercentage):
1102         def __init__(self):
1103                 ProgressPrintPercentage.__init__(self)
1104         def show(self, s):
1105                 print '\n\n', s;
1106         def realstart(self):
1107                 pass
1108
1109 class Display_overwrite(ProgressPrintPercentage):
1110         def __init__(self):
1111                 ProgressPrintPercentage.__init__(self)
1112
1113                 null = file('/dev/null','w')
1114                 curses.setupterm(fd=null.fileno())
1115
1116                 self._clear = curses.tigetstr('clear')
1117                 if not self._clear:
1118                         self._debug('missing clear!')
1119                         self.show = Display_dumb.show
1120                         return
1121
1122                 self._t = {'el':'', 'ed':''}
1123                 if not self._init_sophisticated():
1124                         for k in self._t.keys(): self._t[k] = ''
1125                         self._t['ho'] = self._clear
1126
1127         def _debug(self,m): debug('display overwrite: '+m)
1128
1129         def _init_sophisticated(self):
1130                 for k in self._t.keys():
1131                         s = curses.tigetstr(k)
1132                         self._t[k] = s
1133                 self._t['ho'] = curses.tigetstr('ho')
1134                 if not self._t['ho']:
1135                         cup = curses.tigetstr('cup')
1136                         self._t['ho'] = curses.tparm(cup,0,0)
1137                 missing = [k for k in self._t.keys() if not self._t[k]]
1138                 if missing:
1139                         self.debug('missing '+(' '.join(missing)))
1140                         return 0
1141                 return 1
1142
1143         def show(self, s):
1144                 w = sys.stdout.write
1145                 def wti(k): w(self._t[k])
1146
1147                 wti('ho')
1148                 nl = ''
1149                 for l in s.rstrip().split('\n'):
1150                         w(nl)
1151                         w(l)
1152                         wti('el')
1153                         nl = '\r\n'
1154                 wti('ed')
1155                 w(' ')
1156                 sys.stdout.flush()
1157
1158         def realstart(self):
1159                 sys.stdout.write(self._clear)
1160                 sys.stdout.flush()
1161
1162
1163 def do_ship_aid(args, bu):
1164         if opts.ship_duty is None: opts.ship_duty = True
1165
1166         displayer = globals()['Display_'+opts.display]()
1167
1168         (myself, track) = prep_chat_log(args, bu, progress=displayer)
1169
1170         displayer.realstart()
1171
1172         if os.isatty(0): kr_create = KeystrokeReader
1173         else: kr_create = DummyKeystrokeReader
1174
1175         try:
1176                 kreader = kr_create(0, 10)
1177                 ship_aid_core(myself, track, displayer, kreader)
1178         finally:
1179                 kreader.stop()
1180                 print '\n'
1181
1182 class KeyBasedSorter:
1183         def compar_key_pa(self, pa):
1184                 pi = pa.pirate_info()
1185                 if pi is None: return None
1186                 return self.compar_key(pi)
1187         def lsort_pa(self, l):
1188                 l.sort(key = self.compar_key_pa)
1189
1190 class NameSorter(KeyBasedSorter):
1191         def compar_key(self, pi): return pi.name
1192         def desc(self): return 'name'
1193
1194 class SkillSorter(NameSorter):
1195         def __init__(self, relevant):
1196                 self._want = frozenset(relevant.split('/'))
1197                 self._avoid = set()
1198                 for p in core_duty_puzzles:
1199                         if isinstance(p,basestring): self._avoid.add(p)
1200                         else: self._avoid |= set(p)
1201                 self._avoid -= self._want
1202                 self._desc = '%s' % relevant
1203
1204         def desc(self): return self._desc
1205
1206         def compar_key(self, pi):
1207                 best_want = max([
1208                         pi.standings.get(puz,-1)
1209                         for puz in self._want
1210                         ])
1211                 best_avoid = [
1212                         -pi.standings.get(puz,standing_limit)
1213                         for puz in self._avoid
1214                         ]
1215                 best_avoid.sort()
1216                 def negate(x): return -x
1217                 debug('compar_key %s bw=%s ba=%s' % (pi.name, `best_want`,
1218                         `best_avoid`))
1219                 return (-best_want, map(negate, best_avoid), pi.name)
1220
1221 def ship_aid_core(myself, track, displayer, kreader):
1222
1223         def find_vessel():
1224                 vn = track.vesselname()
1225                 if vn: return (vn, " on board the %s" % vn)
1226                 vn = track.lastvesselname()
1227                 if vn: return (vn, " ashore from the %s" % vn)
1228                 return (None, " not on a vessel")
1229
1230         def timeevent(t,e):
1231                 if t is None: return ' ' * 22
1232                 return " %-4s %-16s" % (format_time_interval(now - t),e)
1233
1234         displayer.show(track.myname() + find_vessel()[1] + '...')
1235
1236         rotate_nya = '/-\\'
1237
1238         sort = NameSorter()
1239         clicmd = None
1240         clierr = None
1241         cliexec = None
1242
1243         while True:
1244                 track.catchup()
1245                 now = time.time()
1246
1247                 (vn, vs) = find_vessel()
1248
1249                 s = ''
1250                 if cliexec is not None:
1251                         s += '...'
1252                 elif clierr is not None:
1253                         s += 'Error: '+clierr
1254                 elif clicmd is not None:
1255                         s += '/' + clicmd
1256                 else:
1257                         s = track.myname() + vs
1258                         s += " at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
1259                         s += kreader.info()
1260                 s += '\n'
1261
1262                 tbl_s = StringIO()
1263                 tbl = StandingsTable(tbl_s)
1264
1265                 aboard = track.aboard(vn)
1266                 sort.lsort_pa(aboard)
1267
1268                 jobbers = track.jobbers()
1269
1270                 if track.vesselname(): howmany = 'aboard: %2d' % len(aboard)
1271                 else: howmany = ''
1272
1273                 tbl.headings(howmany, '  sorted by '+sort.desc())
1274
1275                 last_jobber = None
1276
1277                 for pa in aboard + jobbers:
1278                         if pa.jobber != last_jobber:
1279                                 last_jobber = pa.jobber
1280                                 tbl.literalline('')
1281                                 tbl.literalline('jobbers '+last_jobber)
1282
1283                         pi = pa.pirate_info()
1284
1285                         xs = ''
1286                         if pa.gunner: xs += 'G '
1287                         else: xs += '  '
1288                         xs += timeevent(pa.last_time, pa.last_event)
1289                         xs += timeevent(pa.last_chat_time, pa.last_chat_chan)
1290
1291                         if pi is None:
1292                                 tbl.pirate_dummy(pa.name, rotate_nya[0], xs)
1293                         else:
1294                                 tbl.pirate(pi, xs)
1295
1296                 s += tbl_s.getvalue()
1297                 displayer.show(s)
1298                 tbl_s.close()
1299
1300                 if cliexec is not None:
1301                         clierr = track.local_command("/"+cliexec.strip())
1302                         cliexec = None
1303                         continue
1304
1305                 k = kreader.getch()
1306                 if k is None:
1307                         rotate_nya = rotate_nya[1:3] + rotate_nya[0]
1308                         continue
1309
1310                 if clierr is not None:
1311                         clierr = None
1312                         continue
1313
1314                 if clicmd is not None:
1315                         if k == '\r' or k == '\n':
1316                                 cliexec = clicmd
1317                                 clicmd = clicmdbase
1318                         elif k == '\e' and clicmd != "":
1319                                 clicmd = clicmdbase
1320                         elif k == '\33':
1321                                 clicmd = None
1322                         elif k == '\b' or k == '\177':
1323                                 clicmd = clicmd[ 0 : len(clicmd)-1 ]
1324                         else:
1325                                 clicmd += k
1326                         continue
1327
1328                 if k == 'q': break
1329                 elif k == 'g': sort = SkillSorter('Gunning')
1330                 elif k == 'c': sort = SkillSorter('Carpentry')
1331                 elif k == 's': sort = SkillSorter('Sailing/Rigging')
1332                 elif k == 'b': sort = SkillSorter('Bilging')
1333                 elif k == 'n': sort = SkillSorter('Navigating')
1334                 elif k == 'd': sort = SkillSorter('Battle Navigation')
1335                 elif k == 't': sort = SkillSorter('Treasure Haul')
1336                 elif k == 'a': sort = NameSorter()
1337                 elif k == '/': clicmdbase = ""; clicmd = clicmdbase
1338                 elif k == '+': clicmdbase = "a "; clicmd = clicmdbase
1339                 else: pass # unknown key command
1340
1341 #---------- individual keystroke input ----------
1342
1343 class DummyKeystrokeReader:
1344         def __init__(self,fd,timeout_dummy): pass
1345         def stop(self): pass
1346         def getch(self): sleep(1); return None
1347         def info(self): return ' [noninteractive]'
1348
1349 class KeystrokeReader(DummyKeystrokeReader):
1350         def __init__(self, fd, timeout_decisec=0):
1351                 self._fd = fd
1352                 self._saved = termios.tcgetattr(fd)
1353                 a = termios.tcgetattr(fd)
1354                 a[3] &= ~(termios.ECHO | termios.ECHONL |
1355                           termios.ICANON | termios.IEXTEN)
1356                 a[6][termios.VMIN] = 0
1357                 a[6][termios.VTIME] = timeout_decisec
1358                 termios.tcsetattr(fd, termios.TCSANOW, a)
1359         def stop(self):
1360                 termios.tcsetattr(self._fd, termios.TCSANOW, self._saved)
1361         def getch(self):
1362                 debug_flush()
1363                 byte = os.read(self._fd, 1)
1364                 if not len(byte): return None
1365                 return byte
1366         def info(self):
1367                 return ''
1368
1369 #---------- main program ----------
1370
1371 def main():
1372         global opts, fetcher
1373
1374         pa = OptionParser(
1375 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
1376 actions:
1377  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
1378  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
1379  yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
1380  yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG
1381  yoweb-scrape [options] ship-aid CHAT-LOG  (must be .../PIRATE_OCEAN_chat-log*)
1382
1383 display modes (for --display) apply to ship-aid:
1384  --display=dumb       just print new information, scrolling the screen
1385  --display=overwrite  use cursor motion, selective clear, etc. to redraw at top''')
1386         ao = pa.add_option
1387         ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
1388                 help='select ocean OCEAN')
1389         ao('--cache-dir', dest='cache_dir', metavar='DIR',
1390                 default='~/.yoweb-scrape-cache',
1391                 help='cache yoweb pages in DIR')
1392         ao('-D','--debug', action='count', dest='debug', default=0,
1393                 help='enable debugging output')
1394         ao('--debug-fd', type='int', dest='debug_fd',
1395                 help='write any debugging output to specified fd')
1396         ao('-q','--quiet', action='store_true', dest='quiet',
1397                 help='suppress warning output')
1398         ao('--display', action='store', dest='display',
1399                 type='choice', choices=['dumb','overwrite'],
1400                 help='how to display ship aid')
1401
1402         ao_jt = lambda wh, t: ao(
1403                 '--timeout-sa-'+wh, action='store', dest='timeout_'+wh,
1404                 default=t, help=('set timeout for expiring %s jobbers' % wh))
1405         ao_jt('applied',      120)
1406         ao_jt('invited',      120)
1407         ao_jt('declined',      30)
1408         ao_jt('ashore',      1800)
1409
1410         ao('--ship-duty', action='store_true', dest='ship_duty',
1411                 help='show ship duty station puzzles')
1412         ao('--all-puzzles', action='store_false', dest='ship_duty',
1413                 help='show all puzzles, not just ship duty stations')
1414
1415         ao('--min-cache-reuse', type='int', dest='min_max_age',
1416                 metavar='SECONDS', default=60,
1417                 help='always reuse cache yoweb data if no older than this')
1418
1419         (opts,args) = pa.parse_args()
1420         random.seed()
1421
1422         if len(args) < 1:
1423                 print >>sys.stderr, copyright_info
1424                 pa.error('need a mode argument')
1425
1426         if opts.debug_fd is not None:
1427                 opts.debug_file = os.fdopen(opts.debug_fd, 'w')
1428         else:
1429                 opts.debug_file = sys.stdout
1430
1431         mode = args[0]
1432         mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
1433         try: mode_fn = globals()[mode_fn_name]
1434         except KeyError: pa.error('unknown mode "%s"' % mode)
1435
1436         # fixed parameters
1437         opts.expire_age = max(3600, opts.min_max_age)
1438
1439         opts.ship_reboard_clearout = 3600
1440
1441         if opts.cache_dir.startswith('~/'):
1442                 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
1443
1444         if opts.display is None:
1445                 if ((opts.debug > 0 and opts.debug_fd is None)
1446                     or not os.isatty(sys.stdout.fileno())):
1447                         opts.display = 'dumb'
1448                 else:
1449                         opts.display = 'overwrite'
1450
1451         fetcher = Fetcher(opts.ocean, opts.cache_dir)
1452
1453         mode_fn(args[1:], pa.error)
1454
1455 main()