yoweb-scrape

   1 #!/usr/bin/python
   2 # This is part of ypp-sc-tools, a set of third-party tools for assisting
   3 # players of Yohoho Puzzle Pirates.
   4 #
   5 # Copyright (C) 2009 Ian Jackson <ijackson@chiark.greenend.org.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 #
  20 # Yohoho and Puzzle Pirates are probably trademarks of Three Rings and
  21 # are used without permission.  This program is not endorsed or
  22 # sponsored by Three Rings.
  23
  24 copyright_info = '''
  25 yoweb-scrape is part of ypp-sc-tools  Copyright (C) 2009 Ian Jackson
  26 This program comes with ABSOLUTELY NO WARRANTY; this is free software,
  27 and you are welcome to redistribute it under certain conditions.
  28 For details, read the top of the yoweb-scrape file.
  29 '''
  30
  31 #---------- setup ----------
  32
  33 import signal
  34 signal.signal(signal.SIGINT, signal.SIG_DFL)
  35
  36 import os
  37 import time
  38 import urllib
  39 import urllib2
  40 import errno
  41 import sys
  42 import re as regexp
  43 import random
  44 import curses
  45 import termios
  46 import random
  47 from optparse import OptionParser
  48 from StringIO import StringIO
  49
  50 from BeautifulSoup import BeautifulSoup
  51
  52 opts = None
  53
  54 #---------- YPP parameters and arrays ----------
  55
  56 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
  57         '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
  58         '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
  59         '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
  60
  61 core_duty_puzzles = [
  62                 'Gunning',
  63                 ['Sailing','Rigging'],
  64                 'Bilging',
  65                 'Carpentry',
  66                 ]
  67
  68 duty_puzzles = ([ 'Navigating', 'Battle Navigation' ] +
  69                 core_duty_puzzles +
  70                 [ 'Treasure Haul' ])
  71
  72 standingvals = ('Able/Proficient/Distinguished/Respected/Master'+
  73                 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
  74 standing_limit = len(standingvals)
  75
  76 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
  77
  78 max_pirate_namelen = 12
  79
  80
  81 #---------- general utilities ----------
  82
  83 def debug(m):
  84         if opts.debug > 0:
  85                 print >>opts.debug_file, m
  86
  87 def debug_flush():
  88         if opts.debug > 0:
  89                 opts.debug_file.flush()
  90
  91 def sleep(seconds):
  92         debug_flush()
  93         time.sleep(seconds)
  94
  95 def format_time_interval(ti):
  96         if ti < 120: return '%d:%02d' % (ti / 60, ti % 60)
  97         if ti < 7200: return '%2dm' % (ti / 60)
  98         if ti < 86400: return '%dh' % (ti / 3600)
  99         return '%dd' % (ti / 86400)
 100
 101 #---------- caching and rate-limiting data fetcher ----------
 102
 103 class Fetcher:
 104         def __init__(self, ocean, cachedir):
 105                 debug('Fetcher init %s' % cachedir)
 106                 self.ocean = ocean
 107                 self.cachedir = cachedir
 108                 try: os.mkdir(cachedir)
 109                 except (OSError,IOError), oe:
 110                         if oe.errno != errno.EEXIST: raise
 111                 self._cache_scan(time.time())
 112
 113         def default_ocean(self, ocean='ice'):
 114                 if self.ocean is None:
 115                         self.ocean = ocean
 116
 117         def _cache_scan(self, now):
 118                 # returns list of ages, unsorted
 119                 ages = []
 120                 debug('Fetcher   scan_cache')
 121                 for leaf in os.listdir(self.cachedir):
 122                         if not leaf.startswith('#'): continue
 123                         path = self.cachedir + '/' + leaf
 124                         try: s = os.stat(path)
 125                         except (OSError,IOError), oe:
 126                                 if oe.errno != errno.ENOENT: raise
 127                                 continue
 128                         age = now - s.st_mtime
 129                         if age > opts.expire_age:
 130                                 debug('Fetcher    expire %d %s' % (age, path))
 131                                 try: os.remove(path)
 132                                 except (OSError,IOError), oe:
 133                                         if oe.errno != errno.ENOENT: raise
 134                                 continue
 135                         ages.append(age)
 136                 return ages
 137
 138         def need_wait(self, now, imaginary=[]):
 139                 ages = self._cache_scan(now)
 140                 ages += imaginary
 141                 ages.sort()
 142                 debug('Fetcher   ages ' + `ages`)
 143                 min_age = 1
 144                 need_wait = 0
 145                 for age in ages:
 146                         if age < min_age and age <= 5:
 147                                 debug('Fetcher   morewait min=%d age=%d' %
 148                                         (min_age, age))
 149                                 need_wait = max(need_wait, min_age - age)
 150                         min_age += 3
 151                         min_age *= 1.25
 152                 if need_wait > 0:
 153                         need_wait += random.random() - 0.5
 154                 return need_wait
 155
 156         def _rate_limit_cache_clean(self, now):
 157                 need_wait = self.need_wait(now)
 158                 if need_wait > 0:
 159                         debug('Fetcher   wait %d' % need_wait)
 160                         sleep(need_wait)
 161
 162         def fetch(self, url, max_age):
 163                 debug('Fetcher fetch %s' % url)
 164                 cache_corename = urllib.quote_plus(url)
 165                 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
 166                 try: f = file(cache_item, 'r')
 167                 except (OSError,IOError), oe:
 168                         if oe.errno != errno.ENOENT: raise
 169                         f = None
 170                 now = time.time()
 171                 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
 172                 if f is not None:
 173                         s = os.fstat(f.fileno())
 174                         age = now - s.st_mtime
 175                         if age > max_age:
 176                                 debug('Fetcher  stale %d < %d'% (max_age, age))
 177                                 f = None
 178                 if f is not None:
 179                         data = f.read()
 180                         f.close()
 181                         debug('Fetcher  cached %d > %d' % (max_age, age))
 182                         return data
 183
 184                 debug('Fetcher  fetch')
 185                 self._rate_limit_cache_clean(now)
 186
 187                 stream = urllib2.urlopen(url)
 188                 data = stream.read()
 189                 cache_tmp = "%s/#%s~%d#" % (
 190                         self.cachedir, cache_corename, os.getpid())
 191                 f = file(cache_tmp, 'w')
 192                 f.write(data)
 193                 f.close()
 194                 os.rename(cache_tmp, cache_item)
 195                 debug('Fetcher  stored')
 196                 return data
 197
 198         def yoweb(self, kind, tail, max_age):
 199                 self.default_ocean()
 200                 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
 201                         self.ocean, kind, tail)
 202                 return self.fetch(url, max_age)
 203
 204 #---------- logging assistance for troubled screenscrapers ----------
 205
 206 class SoupLog:
 207         def __init__(self):
 208                 self.msgs = [ ]
 209         def msg(self, m):
 210                 self.msgs.append(m)
 211         def soupm(self, obj, m):
 212                 self.msg(m + '; in ' + `obj`)
 213         def needs_msgs(self, child_souplog):
 214                 self.msgs += child_souplog.msgs
 215                 child_souplog.msgs = [ ]
 216
 217 def soup_text(obj):
 218         str = ''.join(obj.findAll(text=True))
 219         return str.strip()
 220
 221 class SomethingSoupInfo(SoupLog):
 222         def __init__(self, kind, tail, max_age):
 223                 SoupLog.__init__(self)
 224                 html = fetcher.yoweb(kind, tail, max_age)
 225                 self._soup = BeautifulSoup(html,
 226                         convertEntities=BeautifulSoup.HTML_ENTITIES
 227                         )
 228
 229 #---------- scraper for pirate pages ----------
 230
 231 class PirateInfo(SomethingSoupInfo):
 232         # Public data members:
 233         #  pi.standings = { 'Treasure Haul': 'Able' ... }
 234         #  pi.name = name
 235         #  pi.crew = (id, name)
 236         #  pi.flag = (id, name)
 237         #  pi.msgs = [ 'message describing problem with scrape' ]
 238
 239         def __init__(self, pirate, max_age=300):
 240                 SomethingSoupInfo.__init__(self,
 241                         'pirate.wm?target=', pirate, max_age)
 242                 self.name = pirate
 243                 self._find_standings()
 244                 self.crew = self._find_crewflag('crew',
 245                         '^/yoweb/crew/info\\.wm')
 246                 self.flag = self._find_crewflag('flag',
 247                         '^/yoweb/flag/info\\.wm')
 248
 249         def _find_standings(self):
 250                 imgs = self._soup.findAll('img',
 251                         src=regexp.compile('/yoweb/images/stat.*'))
 252                 re = regexp.compile(
 253 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
 254                         )
 255                 standings = { }
 256
 257                 for skill in puzzles:
 258                         standings[skill] = [ ]
 259
 260                 skl = SoupLog()
 261
 262                 for img in imgs:
 263                         try: puzzle = img['alt']
 264                         except KeyError: continue
 265
 266                         if not puzzle in puzzles:
 267                                 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
 268                                 continue
 269                         key = img.findParent('td')
 270                         if key is None:
 271                                 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
 272                                 continue
 273                         valelem = key.findNextSibling('td')
 274                         if valelem is None:
 275                                 skl.soupm(key, 'puzzle missing sibling "%s"'
 276                                         % puzzle)
 277                                 continue
 278                         valstr = soup_text(valelem)
 279                         match = re.match(valstr)
 280                         if match is None:
 281                                 skl.soupm(key, ('puzzle "%s" unparseable'+
 282                                         ' standing "%s"') % (puzzle, valstr))
 283                                 continue
 284                         standing = match.group(match.lastindex)
 285                         standings[puzzle].append(standing)
 286
 287                 self.standings = { }
 288
 289                 for puzzle in puzzles:
 290                         sl = standings[puzzle]
 291                         if len(sl) > 1:
 292                                 skl.msg('puzzle "%s" multiple standings %s' %
 293                                                 (puzzle, `sl`))
 294                                 continue
 295                         if not sl:
 296                                 skl.msg('puzzle "%s" no standing found' % puzzle)
 297                                 continue
 298                         standing = sl[0]
 299                         for i in range(0, standing_limit):
 300                                 if standing == standingvals[i]:
 301                                         self.standings[puzzle] = i
 302                         if not puzzle in self.standings:
 303                                 skl.msg('puzzle "%s" unknown standing "%s"' %
 304                                         (puzzle, standing))
 305
 306                 all_standings_ok = True
 307                 for puzzle in puzzles:
 308                         if not puzzle in self.standings:
 309                                 self.needs_msgs(skl)
 310
 311         def _find_crewflag(self, cf, yoweb_re):
 312                 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
 313                 if len(things) != 1:
 314                         self.msg('zero or several %s id references found' % cf)
 315                         return None
 316                 thing = things[0]
 317                 id_re = '\\b%sid\\=(\\w+)$' % cf
 318                 id_haystack = thing['href']
 319                 match = regexp.compile(id_re).search(id_haystack)
 320                 if match is None:
 321                         self.soupm(thing, ('incomprehensible %s id ref'+
 322                                 ' (%s in %s)') % (cf, id_re, id_haystack))
 323                         return None
 324                 name = soup_text(thing)
 325                 return (match.group(1), name)
 326
 327         def __str__(self):
 328                 return `(self.crew, self.flag, self.standings, self.msgs)`
 329
 330 #---------- scraper for crew pages ----------
 331
 332 class CrewInfo(SomethingSoupInfo):
 333         # Public data members:
 334         #  ci.crew = [ ('Captain',        ['Pirate', ...]),
 335         #              ('Senior Officer', [...]),
 336         #               ... ]
 337         #  pi.msgs = [ 'message describing problem with scrape' ]
 338
 339         def __init__(self, crewid, max_age=300):
 340                 SomethingSoupInfo.__init__(self,
 341                         'crew/info.wm?crewid=', crewid, max_age)
 342                 self._find_crew()
 343
 344         def _find_crew(self):
 345                 self.crew = []
 346                 capts = self._soup.findAll('img',
 347                         src='/yoweb/images/crew-captain.png')
 348                 if len(capts) != 1:
 349                         self.msg('crew members: no. of captain images != 1')
 350                         return
 351                 tbl = capts[0]
 352                 while not tbl.find('a', href=pirate_ref_re):
 353                         tbl = tbl.findParent('table')
 354                         if not tbl:
 355                                 self.msg('crew members: cannot find table')
 356                                 return
 357                 current_rank_crew = None
 358                 crew_rank_re = regexp.compile('/yoweb/images/crew')
 359                 for row in tbl.contents:
 360                         # findAll(recurse=False)
 361                         if isinstance(row,basestring):
 362                                 continue
 363
 364                         is_rank = row.find('img', attrs={'src': crew_rank_re})
 365                         if is_rank:
 366                                 rank = soup_text(row)
 367                                 current_rank_crew = []
 368                                 self.crew.append((rank, current_rank_crew))
 369                                 continue
 370                         for cell in row.findAll('a', href=pirate_ref_re):
 371                                 if current_rank_crew is None:
 372                                         self.soupm(cell, 'crew members: crew'
 373                                                 ' before rank')
 374                                         continue
 375                                 current_rank_crew.append(soup_text(cell))
 376
 377         def __str__(self):
 378                 return `(self.crew, self.msgs)`
 379
 380 #---------- pretty-printer for tables of pirate puzzle standings ----------
 381
 382 class StandingsTable:
 383         def __init__(self, f, use_puzzles=None, col_width=6, gap_every=5):
 384                 if use_puzzles is None:
 385                         if opts.ship_duty:
 386                                 use_puzzles=duty_puzzles
 387                         else:
 388                                 use_puzzles=puzzles
 389                 self._puzzles = use_puzzles
 390                 self.f = f
 391                 self._cw = col_width-1
 392                 self._gap_every = gap_every
 393                 self._linecount = 0
 394                 self._o = f.write
 395
 396         def _nl(self): self._o('\n')
 397
 398         def _pline(self, lhs, puzstrs, extra):
 399                 if (self._linecount > 0
 400                     and self._gap_every is not None
 401                     and not (self._linecount % self._gap_every)):
 402                         self._nl()
 403                 self._o('%-*s' % (max(max_pirate_namelen+1, 15), lhs))
 404                 for v in puzstrs:
 405                         self._o(' %-*.*s' % (self._cw,self._cw, v))
 406                 if extra:
 407                         self._o(' ' + extra)
 408                 self._nl()
 409                 self._linecount += 1
 410
 411         def _puzstr(self, pi, puzzle):
 412                 if not isinstance(puzzle,list): puzzle = [puzzle]
 413                 try: standing = max([pi.standings[p] for p in puzzle])
 414                 except KeyError: return '?'
 415                 if not standing: return ''
 416                 s = ''
 417                 if self._cw > 4:
 418                         c1 = standingvals[standing][0]
 419                         if standing < 3: c1 = c1.lower() # 3 = Master
 420                         s += `standing`
 421                 if self._cw > 5:
 422                         s += ' '
 423                 s += '*' * (standing / 2)
 424                 s += '+' * (standing % 2)
 425                 return s
 426
 427         def headings(self, lhs='', rhs=None):
 428                 def puzn_redact(name):
 429                         if isinstance(name,list):
 430                                 return '/'.join(
 431                                         ["%.*s" % (self._cw/2, puzn_redact(n))
 432                                          for n in name])
 433                         spc = name.find(' ')
 434                         if spc < 0: return name
 435                         return name[0:min(4,spc)] + name[spc+1:]
 436                 self._linecount = -2
 437                 self._pline(lhs, map(puzn_redact, self._puzzles), rhs)
 438                 self._linecount = 0
 439         def literalline(self, line):
 440                 self._o(line)
 441                 self._nl()
 442                 self._linecount = 0
 443         def pirate_dummy(self, name, standingstring, extra=None):
 444                 standings = standingstring * len(self._puzzles)
 445                 self._pline(' '+name, standings, extra)
 446         def pirate(self, pi, extra=None):
 447                 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
 448                 self._pline(' '+pi.name, puzstrs, extra)
 449
 450
 451 #---------- chat log parser ----------
 452
 453 class PirateAboard:
 454         # This is essentially a transparent, dumb, data class.
 455         #  pa.v                 may be None
 456         #  pa.name
 457         #  pa.last_time
 458         #  pa.last_event
 459         #  pa.gunner
 460         #  pa.last_chat_time
 461         #  pa.last_chat_chan
 462         #  pa.pi
 463
 464         # Also used for jobbing applicants:
 465         #               happens when                    expires (to "-")
 466         #   -            disembark, leaves crew          no
 467         #   aboard       evidence of them being aboard   no
 468         #   applied      "has applied for the job"       120s, configurable
 469         #   ashore       "has taken a job"               30min, configurable
 470         #   declined     "declined the job offer"        30s, configurable
 471         #   invited      "has been invited to job"       120s, configurable
 472         #
 473         #  pa.jobber    None, 'ashore', 'applied', 'invited', 'declined'
 474         #  pa.expires   expiry time time
 475
 476         def __init__(pa, pn, v, time, event):
 477                 pa.name = pn
 478                 pa.v = v
 479                 pa.last_time = time
 480                 pa.last_event = event
 481                 pa.last_chat_time = None
 482                 pa.last_chat_chan = None
 483                 pa.gunner = False
 484                 pa.pi = None
 485                 pa.jobber = None
 486                 pa.expires = None
 487
 488         def pirate_info(pa):
 489                 now = time.time()
 490                 if pa.pi:
 491                         age = now - pa.pi_fetched
 492                         guide = random.randint(120,240)
 493                         if age <= guide:
 494                                 return pa.pi
 495                         debug('PirateAboard refresh %d > %d  %s' % (
 496                                 age, guide, pa.name))
 497                         imaginary = [2,4]
 498                 else:
 499                         imaginary = [1]
 500                 wait = fetcher.need_wait(now, imaginary)
 501                 if wait:
 502                         debug('PirateAboard fetcher not ready %d' % wait)
 503                         return pa.pi
 504                 pa.pi = PirateInfo(pa.name, 600)
 505                 pa.pi_fetched = now
 506                 return pa.pi
 507
 508 class ChatLogTracker:
 509         # This is quite complex so we make it opaque.  Use the
 510         # official invokers, accessors etc.
 511
 512         def __init__(self, myself_pi, logfn):
 513                 self._pl = {}   # self._pl['Pirate'] =
 514                 self._vl = {}   #   self._vl['Vessel']['Pirate'] = PirateAboard
 515                                 # self._vl['Vessel']['#lastinfo']
 516                                 # self._vl['Vessel']['#name']
 517                                 # self._v = self._vl[self._vessel]
 518                 self._date = None
 519                 self._myself = myself_pi
 520                 self._lbuf = ''
 521                 self._f = file(logfn)
 522                 flen = os.fstat(self._f.fileno()).st_size
 523                 max_backlog = 500000
 524                 if flen > max_backlog:
 525                         startpos = flen - max_backlog
 526                         self._f.seek(startpos)
 527                         self._f.readline()
 528                 self._progress = [0, flen - self._f.tell()]
 529                 self._disembark_myself()
 530                 self._need_redisplay = False
 531                 self._lastvessel = None
 532
 533         def _disembark_myself(self):
 534                 self._v = None
 535                 self._vessel = None
 536                 self.force_redisplay()
 537
 538         def force_redisplay(self):
 539                 self._need_redisplay = True
 540
 541         def _vessel_updated(self, v, timestamp):
 542                 if v is None: return
 543                 v['#lastinfo'] = timestamp
 544                 self.force_redisplay()
 545
 546         def _onboard_event(self,v,timestamp,pirate,event,
 547                         jobber=None,timeout=None):
 548                 pa = self._pl.get(pirate, None)
 549                 if pa is not None and pa.v is v:
 550                         pa.last_time = timestamp
 551                         pa.last_event = event
 552                 else:
 553                         if pa is not None and pa.v is not None:
 554                                 del pa.v[pirate]
 555                         pa = PirateAboard(pirate, v, timestamp, event)
 556                         self._pl[pirate] = pa
 557                         if v is not None: v[pirate] = pa
 558                 pa.jobber = jobber
 559                 if timeout is None: pa.expires = None
 560                 else: pa.expires = timestamp + timeout
 561                 self._vessel_updated(v, timestamp)
 562                 return pa
 563
 564         def _expire_jobbers(self, now):
 565                 for pa in self._pl.values():
 566                         if pa.expires is None: continue
 567                         if pa.expires >= now: continue
 568                         v = pa.v
 569                         del self._pl[pa.name]
 570                         if v is not None: del v[pa.name]
 571                         self.force_redisplay()
 572
 573         def _trash_vessel(self, v):
 574                 for pn in v:
 575                         if pn.startswith('#'): continue
 576                         del self._pl[pn]
 577                 vn = v['#name']
 578                 del self._vl[vn]
 579                 if v is self._v: self._disembark_myself()
 580                 self.force_redisplay()
 581
 582         def _vessel_stale(self, v, timestamp):
 583                 return timestamp - v['#lastinfo'] > opts.ship_reboard_clearout
 584
 585         def _vessel_check_expire(self, v, timestamp):
 586                 if not self._vessel_stale(v, timestamp):
 587                         return v
 588                 self._debug_line_disposition(timestamp,'',
 589                         'stale-reset ' + v['#name'])
 590                 self._trash_vessel(v)
 591                 return None
 592
 593         def expire_garbage(self, timestamp):
 594                 for v in self._vl.values():
 595                         self._vessel_check_expire(v, timestamp)
 596
 597         def _vessel_lookup(self, vn, timestamp, dml=[], create=False):
 598                 v = self._vl.get(vn, None)
 599                 if v is not None:
 600                         v = self._vessel_check_expire(v, timestamp)
 601                 if v is not None:
 602                         dml.append('found')
 603                         return v
 604                 if not create:
 605                         dml.append('no')
 606                 dml.append('new')
 607                 self._vl[vn] = v = { '#name': vn }
 608                 self._vessel_updated(v, timestamp)
 609                 return v
 610
 611         def _find_matching_vessel(self, pattern, timestamp, cmdr,
 612                                         dml=[], create=False):
 613                 # use when a commander pirate `cmdr' specified a vessel
 614                 #  by name `pattern' (either may be None)
 615                 # if create is true, will create the vessel
 616                 #  record if an exact name is specified
 617
 618                 if (pattern is not None and
 619                     not '*' in pattern
 620                     and len(pattern.split(' ')) == 2):
 621                         vn = pattern.title()
 622                         dml.append('exact')
 623                         return self._vessel_lookup(
 624                                 vn, timestamp, dml=dml, create=create)
 625
 626                 if pattern is None:
 627                         pattern_check = lambda vn: True
 628                 else:
 629                         re = '(?:.* )?%s$' % pattern.lower().replace('*','.+')
 630                         pattern_check = regexp.compile(re, regexp.I).match
 631
 632                 tries = []
 633
 634                 cmdr_pa = self._pl.get(cmdr, None)
 635                 if cmdr_pa: tries.append((cmdr_pa.v, 'cmdr'))
 636
 637                 tries.append((self._v, 'here'))
 638                 tried_vns = []
 639
 640                 for (v, dm) in tries:
 641                         if v is None: dml.append(dm+'?'); continue
 642
 643                         vn = v['#name']
 644                         if not pattern_check(vn):
 645                                 tried_vns.append(vn)
 646                                 dml.append(dm+'#')
 647                                 continue
 648
 649                         dml.append(dm+'!')
 650                         return v
 651
 652                 if pattern is not None and '*' in pattern:
 653                         search = [
 654                                 (vn,v)
 655                                 for (vn,v) in self._vl.iteritems()
 656                                 if not self._vessel_stale(v, timestamp)
 657                                 if pattern_check(vn)
 658                                 ]
 659                         #debug('CLT-RE /%s/ wanted (%s) searched (%s)' % (
 660                         #       re,
 661                         #       '/'.join(tried_vns),
 662                         #       '/'.join([vn for (vn,v) in search])))
 663
 664                         if len(search)==1:
 665                                 dml.append('one')
 666                                 return search[0][1]
 667                         elif search:
 668                                 dml.append('many')
 669                         else:
 670                                 dml.append('none')
 671
 672         def _debug_line_disposition(self,timestamp,l,m):
 673                 debug('CLT %13s %-40s %s' % (timestamp,m,l))
 674
 675         def _rm_crew_l(self,re,l):
 676                 m = regexp.match(re,l)
 677                 if m and m.group(2) == self._myself.crew[1]:
 678                         return m.group(1)
 679                 else:
 680                         return None
 681
 682         def chatline(self,l):
 683                 rm = lambda re: regexp.match(re,l)
 684                 d = lambda m: self._debug_line_disposition(timestamp,l,m)
 685                 rm_crew = lambda re: self._rm_crew_l(re,l)
 686                 timestamp = None
 687
 688                 m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$')
 689                 if m:
 690                         self._date = [int(x) for x in m.groups()]
 691                         self._previous_timestamp = None
 692                         return d('date '+`self._date`)
 693
 694                 if self._date is None:
 695                         return d('date unset')
 696
 697                 m = rm('\\[(\d\d):(\d\d):(\d\d)\\] ')
 698                 if not m:
 699                         return d('no timestamp')
 700
 701                 while True:
 702                         time_tuple = (self._date +
 703                                       [int(x) for x in m.groups()] +
 704                                       [-1,-1,-1])
 705                         timestamp = time.mktime(time_tuple)
 706                         if timestamp >= self._previous_timestamp: break
 707                         self._date[2] += 1
 708                         self._debug_line_disposition(timestamp,'',
 709                                 'new date '+`self._date`)
 710
 711                 self._previous_timestamp = timestamp
 712
 713                 l = l[l.find(' ')+1:]
 714
 715                 def ob_x(pirate,event):
 716                         return self._onboard_event(
 717                                         self._v, timestamp, pirate, event)
 718                 def ob1(did): ob_x(m.group(1), did); return d(did)
 719                 def oba(did): return ob1('%s %s' % (did, m.group(2)))
 720
 721                 def jb(pirate,jobber):
 722                         return self._onboard_event(
 723                                 None, timestamp, pirate,
 724                                 ("jobber %s" % jobber),
 725                                 jobber=jobber,
 726                                 timeout=getattr(opts, 'timeout_'+jobber)
 727                                 )
 728
 729                 def disembark(v, timestamp, pirate, event):
 730                         self._onboard_event(
 731                                         v, timestamp, pirate, 'leaving '+event)
 732                         del v[pirate]
 733                         del self._pl[pirate]
 734
 735                 def disembark_me(why):
 736                         self._disembark_myself()
 737                         return d('disembark-me '+why)
 738
 739                 m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
 740                 if m:
 741                         dm = ['boarding']
 742                         pn = self._myself.name
 743                         vn = m.group(1)
 744                         v = self._vessel_lookup(vn, timestamp, dm, create=True)
 745                         self._lastvessel = self._vessel = vn
 746                         self._v = v
 747                         ob_x(pn, 'we boarded')
 748                         self.expire_garbage(timestamp)
 749                         return d(' '.join(dm))
 750
 751                 if self._v is None:
 752                         return d('no vessel')
 753
 754                 m = rm('(\\w+) has come aboard\\.$')
 755                 if m: return ob1('boarded');
 756
 757                 m = rm('You have ordered (\\w+) to do some (\\S.*\\S)\\.$')
 758                 if m:
 759                         (who,what) = m.groups()
 760                         pa = ob_x(who,'ord '+what)
 761                         if what == 'Gunning':
 762                                 pa.gunner = True
 763                         return d('duty order')
 764
 765                 m = rm('(\\w+) abandoned a (\\S.*\\S) station\\.$')
 766                 if m: oba('stopped'); return d("end")
 767
 768                 def chat_core(speaker, chan):
 769                         try: pa = self._pl[speaker]
 770                         except KeyError: return 'mystery'
 771                         if pa.v is not None and pa.v is not self._v:
 772                                 return 'elsewhere'
 773                         pa.last_chat_time = timestamp
 774                         pa.last_chat_chan = chan
 775                         self.force_redisplay()
 776                         return 'here'
 777
 778                 def chat(chan):
 779                         speaker = m.group(1)
 780                         dm = chat_core(speaker, chan)
 781                         return d('chat %s %s' % (chan, dm))
 782
 783                 def chat_metacmd(chan):
 784                         (cmdr, metacmd) = m.groups()
 785                         metacmd = regexp.sub('\\s+', ' ', metacmd).strip()
 786                         m2 = regexp.match(
 787                                 '/([ad]) (?:([A-Za-z* ]+)\\s*:)?([A-Za-z ]+)$',
 788                                 metacmd)
 789                         if not m2: return chat(chan)
 790
 791                         (cmd, pattern, targets) = m2.groups()
 792                         dml = ['cmd', chan, cmd]
 793
 794                         if cmd == 'a': each = self._onboard_event
 795                         else: each = disembark
 796
 797                         if cmdr == self._myself.name:
 798                                 dml.append('self')
 799                                 how = 'cmd: %s' % cmd
 800                         else:
 801                                 dml.append('other')
 802                                 how = 'cmd: %s %s' % (cmd,cmdr)
 803
 804                         v = self._find_matching_vessel(
 805                                 pattern, timestamp, cmdr, dml, create=True)
 806
 807                         if v is not None:
 808                                 targets = targets.strip().split(' ')
 809                                 dml.append(`len(targets)`)
 810                                 for target in targets:
 811                                         each(v, timestamp, target.title(), how)
 812                                 self._vessel_updated(v, timestamp)
 813
 814                         dm = ' '.join(dml)
 815                         chat_core(cmdr, 'cmd '+chan)
 816                         return d(dm)
 817
 818                 m = rm('(\\w+) (?:issued an order|ordered everyone) "')
 819                 if m: return ob1('general order');
 820
 821                 m = rm('(\\w+) says, "')
 822                 if m: return chat('public')
 823
 824                 m = rm('(\\w+) tells ye, "')
 825                 if m: return chat('private')
 826
 827                 m = rm('Ye told (\\w+), "(.*)"$')
 828                 if m: return chat_metacmd('private')
 829
 830                 m = rm('(\\w+) flag officer chats, "')
 831                 if m: return chat('flag officer')
 832
 833                 m = rm('(\\w+) officer chats, "(.*)"$')
 834                 if m: return chat_metacmd('officer')
 835
 836                 m = rm('Ye accepted the offer to job with ')
 837                 if m: return disembark_me('jobbing')
 838
 839                 m = rm('Ye hop on the ferry and are whisked away ')
 840                 if m: return disembark_me('ferry')
 841
 842                 m = rm('Whisking away to yer home on the magical winds')
 843                 if m: return disembark_me('home')
 844
 845                 m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
 846                 if m:
 847                         pl = m.group(1).split(', ')
 848                         if not self._myself.name in pl:
 849                                 return d('lost melee')
 850                         for pn in pl:
 851                                 if ' ' in pn: continue
 852                                 ob_x(pn,'won melee')
 853                         return d('won melee')
 854
 855                 m = rm('(\\w+) is eliminated\\!')
 856                 if m: return ob1('eliminated in fray');
 857
 858                 m = rm('(\\w+) has driven \w+ from the ship\\!')
 859                 if m: return ob1('boarder repelled');
 860
 861                 m = rm('\w+ has bested (\\w+), and turns'+
 862                         ' to the rest of the ship\\.')
 863                 if m: return ob1('boarder unrepelled');
 864
 865                 pirate = rm_crew("(\\w+) has taken a job with '(.*)'\\.")
 866                 if pirate: return jb(pirate, 'ashore')
 867
 868                 pirate = rm_crew("(\\w+) has left '(.*)'\\.")
 869                 if pirate:
 870                         disembark(self._v, timestamp, pirate, 'left crew')
 871                         return d('left crew')
 872
 873                 m = rm('\w+ has applied for the posted job\.')
 874                 if m: return jb(m.group(1), 'applied')
 875
 876                 pirate= rm_crew("(\\w+) has been invited to job for '(.*)'\\.")
 877                 if pirate: return jb(pirate, 'invited')
 878
 879                 pirate = rm_crew("(\\w+) declined the job offer for '(.*)'\\.")
 880                 if pirate: return jb(pirate, 'declined')
 881
 882                 m = rm('(\\w+) has left the vessel\.')
 883                 if m:
 884                         pirate = m.group(1)
 885                         disembark(self._v, timestamp, pirate, 'disembarked')
 886                         return d('disembarked')
 887
 888                 return d('not-matched')
 889
 890         def _str_pa(self, pn, pa):
 891                 assert self._pl[pn] == pa
 892                 s = ' '*20 + "%s %-*s %13s %-30s %13s %-20s %13s" % (
 893                         (' ','G')[pa.gunner],
 894                         max_pirate_namelen, pn,
 895                         pa.last_time, pa.last_event,
 896                         pa.last_chat_time, pa.last_chat_chan,
 897                         pa.jobber)
 898                 if pa.expires is not None:
 899                         s += " %-5d" % (pa.expires - pa.last_time)
 900                 s += "\n"
 901                 return s
 902
 903         def _str_vessel(self, vn, v):
 904                 s = ' vessel %s\n' % vn
 905                 s += ' '*20 + "%-*s   %13s\n" % (
 906                                 max_pirate_namelen, '#lastinfo',
 907                                 v['#lastinfo'])
 908                 assert v['#name'] == vn
 909                 for pn in sorted(v.keys()):
 910                         if pn.startswith('#'): continue
 911                         pa = v[pn]
 912                         assert pa.v == v
 913                         s += self._str_pa(pn,pa)
 914                 return s
 915
 916         def __str__(self):
 917                 s = '''<ChatLogTracker
 918  myself %s
 919  vessel %s
 920 '''                     % (self._myself.name, self._vessel)
 921                 assert ((self._v is None and self._vessel is None) or
 922                         (self._v is self._vl[self._vessel]))
 923                 if self._vessel is not None:
 924                         s += self._str_vessel(self._vessel, self._v)
 925                 for vn in sorted(self._vl.keys()):
 926                         if vn == self._vessel: continue
 927                         s += self._str_vessel(vn, self._vl[vn])
 928                 s += " elsewhere\n"
 929                 for p in self._pl:
 930                         pa = self._pl[p]
 931                         if pa.v is not None:
 932                                 assert pa.v[p] is pa
 933                                 assert pa.v in self._vl.values()
 934                         else:
 935                                 s += self._str_pa(pa.name, pa)
 936                 s += '>\n'
 937                 return s
 938
 939         def catchup(self, progress=None):
 940                 while True:
 941                         more = self._f.readline()
 942                         if not more: break
 943
 944                         self._progress[0] += len(more)
 945                         if progress: progress.progress(*self._progress)
 946
 947                         self._lbuf += more
 948                         if self._lbuf.endswith('\n'):
 949                                 self.chatline(self._lbuf.rstrip())
 950                                 self._lbuf = ''
 951                                 if opts.debug >= 2:
 952                                         debug(self.__str__())
 953                 self._expire_jobbers(time.time())
 954
 955                 if progress: progress.caughtup()
 956
 957         def changed(self):
 958                 rv = self._need_redisplay
 959                 self._need_redisplay = False
 960                 return rv
 961         def myname(self):
 962                 # returns our pirate name
 963                 return self._myself.name
 964         def vesselname(self):
 965                 # returns the vessel name we're aboard or None
 966                 return self._vessel
 967         def lastvesselname(self):
 968                 # returns the last vessel name we were aboard or None
 969                 return self._lastvessel
 970         def aboard(self, vesselname=True):
 971                 # returns a list of PirateAboard the vessel
 972                 #  sorted by pirate name
 973                 #  you can pass this None and you'll get []
 974                 #  or True for the current vessel (which is the default)
 975                 #  the returned value is a fresh list of persistent
 976                 #  PirateAboard objects
 977                 if vesselname is True: v = self._v
 978                 else: v = self._vl.get(vesselname.title())
 979                 if v is None: return []
 980                 return [ v[pn]
 981                          for pn in sorted(v.keys())
 982                          if not pn.startswith('#') ]
 983         def jobbers(self):
 984                 # returns a the jobbers' PirateAboards,
 985                 # sorted by jobber class and reverse of expiry time
 986                 l = [ pa
 987                       for pa in self._pl.values()
 988                       if pa.jobber is not None
 989                     ]
 990                 def compar_key(pa):
 991                         return (pa.jobber, -pa.expires)
 992                 l.sort(key = compar_key)
 993                 return l
 994
 995 #---------- implementations of actual operation modes ----------
 996
 997 def do_pirate(pirates, bu):
 998         print '{'
 999         for pirate in pirates:
1000                 info = PirateInfo(pirate)
1001                 print '%s: %s,' % (`pirate`, info)
1002         print '}'
1003
1004 def prep_crew_of(args, bu, max_age=300):
1005         if len(args) != 1: bu('crew-of takes one pirate name')
1006         pi = PirateInfo(args[0], max_age)
1007         if pi.crew is None: return None
1008         return CrewInfo(pi.crew[0], max_age)
1009
1010 def do_crew_of(args, bu):
1011         ci = prep_crew_of(args, bu)
1012         print ci
1013
1014 def do_standings_crew_of(args, bu):
1015         ci = prep_crew_of(args, bu, 60)
1016         tab = StandingsTable(sys.stdout)
1017         tab.headings()
1018         for (rank, members) in ci.crew:
1019                 if not members: continue
1020                 tab.literalline('')
1021                 tab.literalline('%s:' % rank)
1022                 for p in members:
1023                         pi = PirateInfo(p, random.randint(900,1800))
1024                         tab.pirate(pi)
1025
1026 class ProgressPrintPercentage:
1027         def __init__(self, f=sys.stdout):
1028                 self._f = f
1029         def progress_string(self,done,total):
1030                 return "scan chat logs %3d%%\r" % ((done*100) / total)
1031         def progress(self,*a):
1032                 self._f.write(self.progress_string(*a))
1033                 self._f.flush()
1034         def show_init(self, pirate, ocean):
1035                 print >>self._f, 'Starting up, %s on the %s ocean' % (
1036                         pirate, ocean)
1037         def caughtup(self):
1038                 self._f.write('                   \r')
1039                 self._f.flush()
1040
1041 #----- modes which use the chat log parser are quite complex -----
1042
1043 def prep_chat_log(args, bu,
1044                 progress=ProgressPrintPercentage(),
1045                 max_myself_age=3600):
1046         if len(args) != 1: bu('this action takes only chat log filename')
1047         logfn = args[0]
1048         logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_'
1049         match = regexp.match(logfn_re, logfn)
1050         if not match: bu('chat log filename is not in expected format')
1051         (pirate, ocean) = match.groups()
1052         fetcher.default_ocean(ocean)
1053
1054         progress.show_init(pirate, fetcher.ocean)
1055         myself = PirateInfo(pirate,max_myself_age)
1056         track = ChatLogTracker(myself, logfn)
1057
1058         opts.debug -= 2
1059         track.catchup(progress)
1060         opts.debug += 2
1061
1062         track.force_redisplay()
1063
1064         return (myself, track)
1065
1066 def do_track_chat_log(args, bu):
1067         (myself, track) = prep_chat_log(args, bu)
1068         while True:
1069                 track.catchup()
1070                 if track.changed():
1071                         print track
1072                 sleep(0.5 + 0.5 * random.random())
1073
1074 #----- ship management aid -----
1075
1076 class Display_dumb(ProgressPrintPercentage):
1077         def __init__(self):
1078                 ProgressPrintPercentage.__init__(self)
1079         def show(self, s):
1080                 print '\n\n', s;
1081         def realstart(self):
1082                 pass
1083
1084 class Display_overwrite(ProgressPrintPercentage):
1085         def __init__(self):
1086                 ProgressPrintPercentage.__init__(self)
1087
1088                 null = file('/dev/null','w')
1089                 curses.setupterm(fd=null.fileno())
1090
1091                 self._clear = curses.tigetstr('clear')
1092                 if not self._clear:
1093                         self._debug('missing clear!')
1094                         self.show = Display_dumb.show
1095                         return
1096
1097                 self._t = {'el':'', 'ed':''}
1098                 if not self._init_sophisticated():
1099                         for k in self._t.keys(): self._t[k] = ''
1100                         self._t['ho'] = self._clear
1101
1102         def _debug(self,m): debug('display overwrite: '+m)
1103
1104         def _init_sophisticated(self):
1105                 for k in self._t.keys():
1106                         s = curses.tigetstr(k)
1107                         self._t[k] = s
1108                 self._t['ho'] = curses.tigetstr('ho')
1109                 if not self._t['ho']:
1110                         cup = curses.tigetstr('cup')
1111                         self._t['ho'] = curses.tparm(cup,0,0)
1112                 missing = [k for k in self._t.keys() if not self._t[k]]
1113                 if missing:
1114                         self.debug('missing '+(' '.join(missing)))
1115                         return 0
1116                 return 1
1117
1118         def show(self, s):
1119                 w = sys.stdout.write
1120                 def wti(k): w(self._t[k])
1121
1122                 wti('ho')
1123                 nl = ''
1124                 for l in s.rstrip().split('\n'):
1125                         w(nl)
1126                         w(l)
1127                         wti('el')
1128                         nl = '\r\n'
1129                 wti('ed')
1130                 w(' ')
1131                 sys.stdout.flush()
1132
1133         def realstart(self):
1134                 sys.stdout.write(self._clear)
1135                 sys.stdout.flush()
1136
1137
1138 def do_ship_aid(args, bu):
1139         if opts.ship_duty is None: opts.ship_duty = True
1140
1141         displayer = globals()['Display_'+opts.display]()
1142
1143         (myself, track) = prep_chat_log(args, bu, progress=displayer)
1144
1145         displayer.realstart()
1146
1147         if os.isatty(0): kr_create = KeystrokeReader
1148         else: kr_create = DummyKeystrokeReader
1149
1150         try:
1151                 kreader = kr_create(0, 10)
1152                 ship_aid_core(myself, track, displayer, kreader)
1153         finally:
1154                 kreader.stop()
1155                 print '\n'
1156
1157 class KeyBasedSorter:
1158         def compar_key_pa(self, pa):
1159                 pi = pa.pirate_info()
1160                 if pi is None: return None
1161                 return self.compar_key(pi)
1162         def lsort_pa(self, l):
1163                 l.sort(key = self.compar_key_pa)
1164
1165 class NameSorter(KeyBasedSorter):
1166         def compar_key(self, pi): return pi.name
1167         def desc(self): return 'name'
1168
1169 class SkillSorter(NameSorter):
1170         def __init__(self, relevant):
1171                 self._want = frozenset(relevant.split('/'))
1172                 self._avoid = set()
1173                 for p in core_duty_puzzles:
1174                         if isinstance(p,basestring): self._avoid.add(p)
1175                         else: self._avoid |= set(p)
1176                 self._avoid -= self._want
1177                 self._desc = '%s' % relevant
1178
1179         def desc(self): return self._desc
1180
1181         def compar_key(self, pi):
1182                 best_want = max([
1183                         pi.standings.get(puz,-1)
1184                         for puz in self._want
1185                         ])
1186                 best_avoid = [
1187                         -pi.standings.get(puz,standing_limit)
1188                         for puz in self._avoid
1189                         ]
1190                 best_avoid.sort()
1191                 def negate(x): return -x
1192                 debug('compar_key %s bw=%s ba=%s' % (pi.name, `best_want`,
1193                         `best_avoid`))
1194                 return (-best_want, map(negate, best_avoid), pi.name)
1195
1196 def ship_aid_core(myself, track, displayer, kreader):
1197
1198         def find_vessel():
1199                 vn = track.vesselname()
1200                 if vn: return (vn, " on board the %s" % vn)
1201                 vn = track.lastvesselname()
1202                 if vn: return (vn, " ashore from the %s" % vn)
1203                 return (None, " not on a vessel")
1204
1205         def timeevent(t,e):
1206                 if t is None: return ' ' * 22
1207                 return " %-4s %-16s" % (format_time_interval(now - t),e)
1208
1209         displayer.show(track.myname() + find_vessel()[1] + '...')
1210
1211         rotate_nya = '/-\\'
1212
1213         sort = NameSorter()
1214
1215         while True:
1216                 track.catchup()
1217                 now = time.time()
1218
1219                 (vn, s) = find_vessel()
1220                 s = track.myname() + s
1221                 s += " at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
1222                 s += kreader.info()
1223                 s += '\n'
1224
1225                 tbl_s = StringIO()
1226                 tbl = StandingsTable(tbl_s)
1227
1228                 aboard = track.aboard(vn)
1229                 sort.lsort_pa(aboard)
1230
1231                 jobbers = track.jobbers()
1232
1233                 if track.vesselname(): howmany = 'aboard: %2d' % len(aboard)
1234                 else: howmany = ''
1235
1236                 tbl.headings(howmany, '  sorted by '+sort.desc())
1237
1238                 last_jobber = None
1239
1240                 for pa in aboard + jobbers:
1241                         if pa.jobber != last_jobber:
1242                                 last_jobber = pa.jobber
1243                                 tbl.literalline('')
1244                                 tbl.literalline('jobbers '+last_jobber)
1245
1246                         pi = pa.pirate_info()
1247
1248                         xs = ''
1249                         if pa.gunner: xs += 'G '
1250                         else: xs += '  '
1251                         xs += timeevent(pa.last_time, pa.last_event)
1252                         xs += timeevent(pa.last_chat_time, pa.last_chat_chan)
1253
1254                         if pi is None:
1255                                 tbl.pirate_dummy(pa.name, rotate_nya[0], xs)
1256                         else:
1257                                 tbl.pirate(pi, xs)
1258
1259                 s += tbl_s.getvalue()
1260                 displayer.show(s)
1261                 tbl_s.close()
1262
1263                 k = kreader.getch()
1264                 if k is None:
1265                         rotate_nya = rotate_nya[1:3] + rotate_nya[0]
1266                         continue
1267
1268                 if k == 'q': break
1269                 elif k == 'g': sort = SkillSorter('Gunning')
1270                 elif k == 'c': sort = SkillSorter('Carpentry')
1271                 elif k == 's': sort = SkillSorter('Sailing/Rigging')
1272                 elif k == 'b': sort = SkillSorter('Bilging')
1273                 elif k == 'n': sort = SkillSorter('Navigating')
1274                 elif k == 'd': sort = SkillSorter('Battle Navigation')
1275                 elif k == 't': sort = SkillSorter('Treasure Haul')
1276                 elif k == 'a': sort = NameSorter()
1277                 else: pass # unknown key command
1278
1279 #---------- individual keystroke input ----------
1280
1281 class DummyKeystrokeReader:
1282         def __init__(self,fd,timeout_dummy): pass
1283         def stop(self): pass
1284         def getch(self): sleep(1); return None
1285         def info(self): return ' [noninteractive]'
1286
1287 class KeystrokeReader(DummyKeystrokeReader):
1288         def __init__(self, fd, timeout_decisec=0):
1289                 self._fd = fd
1290                 self._saved = termios.tcgetattr(fd)
1291                 a = termios.tcgetattr(fd)
1292                 a[3] &= ~(termios.ECHO | termios.ECHONL |
1293                           termios.ICANON | termios.IEXTEN)
1294                 a[6][termios.VMIN] = 0
1295                 a[6][termios.VTIME] = timeout_decisec
1296                 termios.tcsetattr(fd, termios.TCSANOW, a)
1297         def stop(self):
1298                 termios.tcsetattr(self._fd, termios.TCSANOW, self._saved)
1299         def getch(self):
1300                 debug_flush()
1301                 byte = os.read(self._fd, 1)
1302                 if not len(byte): return None
1303                 return byte
1304         def info(self):
1305                 return ''
1306
1307 #---------- main program ----------
1308
1309 def main():
1310         global opts, fetcher
1311
1312         pa = OptionParser(
1313 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
1314 actions:
1315  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
1316  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
1317  yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
1318  yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG
1319  yoweb-scrape [options] ship-aid CHAT-LOG  (must be .../PIRATE_OCEAN_chat-log*)
1320
1321 display modes (for --display) apply to ship-aid:
1322  --display=dumb       just print new information, scrolling the screen
1323  --display=overwrite  use cursor motion, selective clear, etc. to redraw at top''')
1324         ao = pa.add_option
1325         ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
1326                 help='select ocean OCEAN')
1327         ao('--cache-dir', dest='cache_dir', metavar='DIR',
1328                 default='~/.yoweb-scrape-cache',
1329                 help='cache yoweb pages in DIR')
1330         ao('-D','--debug', action='count', dest='debug', default=0,
1331                 help='enable debugging output')
1332         ao('--debug-fd', type='int', dest='debug_fd',
1333                 help='write any debugging output to specified fd')
1334         ao('-q','--quiet', action='store_true', dest='quiet',
1335                 help='suppress warning output')
1336         ao('--display', action='store', dest='display',
1337                 type='choice', choices=['dumb','overwrite'],
1338                 help='how to display ship aid')
1339
1340         ao_jt = lambda wh, t: ao(
1341                 '--timeout-sa-'+wh, action='store', dest='timeout_'+wh,
1342                 default=t, help=('set timeout for expiring %s jobbers' % wh))
1343         ao_jt('applied',      120)
1344         ao_jt('invited',      120)
1345         ao_jt('declined',      30)
1346         ao_jt('ashore',      1800)
1347
1348         ao('--ship-duty', action='store_true', dest='ship_duty',
1349                 help='show ship duty station puzzles')
1350         ao('--all-puzzles', action='store_false', dest='ship_duty',
1351                 help='show all puzzles, not just ship duty stations')
1352
1353         ao('--min-cache-reuse', type='int', dest='min_max_age',
1354                 metavar='SECONDS', default=60,
1355                 help='always reuse cache yoweb data if no older than this')
1356
1357         (opts,args) = pa.parse_args()
1358         random.seed()
1359
1360         if len(args) < 1:
1361                 print >>sys.stderr, copyright_info
1362                 pa.error('need a mode argument')
1363
1364         if opts.debug_fd is not None:
1365                 opts.debug_file = os.fdopen(opts.debug_fd, 'w')
1366         else:
1367                 opts.debug_file = sys.stdout
1368
1369         mode = args[0]
1370         mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
1371         try: mode_fn = globals()[mode_fn_name]
1372         except KeyError: pa.error('unknown mode "%s"' % mode)
1373
1374         # fixed parameters
1375         opts.expire_age = max(3600, opts.min_max_age)
1376
1377         opts.ship_reboard_clearout = 3600
1378
1379         if opts.cache_dir.startswith('~/'):
1380                 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
1381
1382         if opts.display is None:
1383                 if ((opts.debug > 0 and opts.debug_fd is None)
1384                     or not os.isatty(sys.stdout.fileno())):
1385                         opts.display = 'dumb'
1386                 else:
1387                         opts.display = 'overwrite'
1388
1389         fetcher = Fetcher(opts.ocean, opts.cache_dir)
1390
1391         mode_fn(args[1:], pa.error)
1392
1393 main()