yoweb-scrape

   1 #!/usr/bin/python
   2 # This is part of ypp-sc-tools, a set of third-party tools for assisting
   3 # players of Yohoho Puzzle Pirates.
   4 #
   5 # Copyright (C) 2009 Ian Jackson <ijackson@chiark.greenend.org.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 #
  20 # Yohoho and Puzzle Pirates are probably trademarks of Three Rings and
  21 # are used without permission.  This program is not endorsed or
  22 # sponsored by Three Rings.
  23
  24 copyright_info = '''
  25 yoweb-scrape is part of ypp-sc-tools  Copyright (C) 2009 Ian Jackson
  26 This program comes with ABSOLUTELY NO WARRANTY; this is free software,
  27 and you are welcome to redistribute it under certain conditions.
  28 For details, read the top of the yoweb-scrape file.
  29 '''
  30
  31 #---------- setup ----------
  32
  33 import signal
  34 signal.signal(signal.SIGINT, signal.SIG_DFL)
  35
  36 import os
  37 import time
  38 import urllib
  39 import urllib2
  40 import errno
  41 import sys
  42 import re as regexp
  43 import random
  44 import curses
  45 import termios
  46 import random
  47 import subprocess
  48 from optparse import OptionParser
  49 from StringIO import StringIO
  50
  51 from BeautifulSoup import BeautifulSoup
  52
  53 opts = None
  54
  55 #---------- YPP parameters and arrays ----------
  56
  57 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
  58         '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
  59         '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
  60         '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
  61
  62 core_duty_puzzles = [
  63                 'Gunning',
  64                 ['Sailing','Rigging'],
  65                 'Bilging',
  66                 'Carpentry',
  67                 ]
  68
  69 duty_puzzles = ([ 'Navigating', 'Battle Navigation' ] +
  70                 core_duty_puzzles +
  71                 [ 'Treasure Haul' ])
  72
  73 standingvals = ('Able/Proficient/Distinguished/Respected/Master'+
  74                 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
  75 standing_limit = len(standingvals)
  76
  77 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
  78
  79 max_pirate_namelen = 12
  80
  81
  82 #---------- general utilities ----------
  83
  84 def debug(m):
  85         if opts.debug > 0:
  86                 print >>opts.debug_file, m
  87
  88 def debug_flush():
  89         if opts.debug > 0:
  90                 opts.debug_file.flush()
  91
  92 def sleep(seconds):
  93         debug_flush()
  94         time.sleep(seconds)
  95
  96 def format_time_interval(ti):
  97         if ti < 120: return '%d:%02d' % (ti / 60, ti % 60)
  98         if ti < 7200: return '%2dm' % (ti / 60)
  99         if ti < 86400: return '%dh' % (ti / 3600)
 100         return '%dd' % (ti / 86400)
 101
 102 def yppsc_dir():
 103         lib = os.getenv("YPPSC_YARRG_SRCBASE")
 104         if lib is not None: return lib
 105         lib = sys.argv[0]
 106         lib = regexp.sub('/[^/]+$', '', lib)
 107         os.environ["YPPSC_YARRG_SRCBASE"] = lib
 108         return lib
 109
 110 #---------- caching and rate-limiting data fetcher ----------
 111
 112 class Fetcher:
 113         def __init__(self, ocean, cachedir):
 114                 debug('Fetcher init %s' % cachedir)
 115                 self.ocean = ocean
 116                 self.cachedir = cachedir
 117                 try: os.mkdir(cachedir)
 118                 except (OSError,IOError), oe:
 119                         if oe.errno != errno.EEXIST: raise
 120                 self._cache_scan(time.time())
 121
 122         def default_ocean(self, ocean='ice'):
 123                 if self.ocean is None:
 124                         self.ocean = ocean
 125
 126         def _cache_scan(self, now):
 127                 # returns list of ages, unsorted
 128                 ages = []
 129                 debug('Fetcher   scan_cache')
 130                 for leaf in os.listdir(self.cachedir):
 131                         if not leaf.startswith('#'): continue
 132                         path = self.cachedir + '/' + leaf
 133                         try: s = os.stat(path)
 134                         except (OSError,IOError), oe:
 135                                 if oe.errno != errno.ENOENT: raise
 136                                 continue
 137                         age = now - s.st_mtime
 138                         if age > opts.expire_age:
 139                                 debug('Fetcher    expire %d %s' % (age, path))
 140                                 try: os.remove(path)
 141                                 except (OSError,IOError), oe:
 142                                         if oe.errno != errno.ENOENT: raise
 143                                 continue
 144                         ages.append(age)
 145                 return ages
 146
 147         def need_wait(self, now, imaginary=[]):
 148                 ages = self._cache_scan(now)
 149                 ages += imaginary
 150                 ages.sort()
 151                 debug('Fetcher   ages ' + `ages`)
 152                 min_age = 1
 153                 need_wait = 0
 154                 for age in ages:
 155                         if age < min_age and age <= 5:
 156                                 debug('Fetcher   morewait min=%d age=%d' %
 157                                         (min_age, age))
 158                                 need_wait = max(need_wait, min_age - age)
 159                         min_age += 3
 160                         min_age *= 1.25
 161                 if need_wait > 0:
 162                         need_wait += random.random() - 0.5
 163                 return need_wait
 164
 165         def _rate_limit_cache_clean(self, now):
 166                 need_wait = self.need_wait(now)
 167                 if need_wait > 0:
 168                         debug('Fetcher   wait %d' % need_wait)
 169                         sleep(need_wait)
 170
 171         def fetch(self, url, max_age):
 172                 debug('Fetcher fetch %s' % url)
 173                 cache_corename = urllib.quote_plus(url)
 174                 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
 175                 try: f = file(cache_item, 'r')
 176                 except (OSError,IOError), oe:
 177                         if oe.errno != errno.ENOENT: raise
 178                         f = None
 179                 now = time.time()
 180                 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
 181                 if f is not None:
 182                         s = os.fstat(f.fileno())
 183                         age = now - s.st_mtime
 184                         if age > max_age:
 185                                 debug('Fetcher  stale %d < %d'% (max_age, age))
 186                                 f = None
 187                 if f is not None:
 188                         data = f.read()
 189                         f.close()
 190                         debug('Fetcher  cached %d > %d' % (max_age, age))
 191                         return data
 192
 193                 debug('Fetcher  fetch')
 194                 self._rate_limit_cache_clean(now)
 195
 196                 stream = urllib2.urlopen(url)
 197                 data = stream.read()
 198                 cache_tmp = "%s/#%s~%d#" % (
 199                         self.cachedir, cache_corename, os.getpid())
 200                 f = file(cache_tmp, 'w')
 201                 f.write(data)
 202                 f.close()
 203                 os.rename(cache_tmp, cache_item)
 204                 debug('Fetcher  stored')
 205                 return data
 206
 207         def yoweb(self, kind, tail, max_age):
 208                 self.default_ocean()
 209                 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
 210                         self.ocean, kind, tail)
 211                 return self.fetch(url, max_age)
 212
 213 #---------- logging assistance for troubled screenscrapers ----------
 214
 215 class SoupLog:
 216         def __init__(self):
 217                 self.msgs = [ ]
 218         def msg(self, m):
 219                 self.msgs.append(m)
 220         def soupm(self, obj, m):
 221                 self.msg(m + '; in ' + `obj`)
 222         def needs_msgs(self, child_souplog):
 223                 self.msgs += child_souplog.msgs
 224                 child_souplog.msgs = [ ]
 225
 226 def soup_text(obj):
 227         str = ''.join(obj.findAll(text=True))
 228         return str.strip()
 229
 230 class SomethingSoupInfo(SoupLog):
 231         def __init__(self, kind, tail, max_age):
 232                 SoupLog.__init__(self)
 233                 html = fetcher.yoweb(kind, tail, max_age)
 234                 self._soup = BeautifulSoup(html,
 235                         convertEntities=BeautifulSoup.HTML_ENTITIES
 236                         )
 237
 238 #---------- scraper for pirate pages ----------
 239
 240 class PirateInfo(SomethingSoupInfo):
 241         # Public data members:
 242         #  pi.standings = { 'Treasure Haul': 'Able' ... }
 243         #  pi.name = name
 244         #  pi.crew = (id, name)
 245         #  pi.flag = (id, name)
 246         #  pi.msgs = [ 'message describing problem with scrape' ]
 247
 248         def __init__(self, pirate, max_age=300):
 249                 SomethingSoupInfo.__init__(self,
 250                         'pirate.wm?target=', pirate, max_age)
 251                 self.name = pirate
 252                 self._find_standings()
 253                 self.crew = self._find_crewflag('crew',
 254                         '^/yoweb/crew/info\\.wm')
 255                 self.flag = self._find_crewflag('flag',
 256                         '^/yoweb/flag/info\\.wm')
 257
 258         def _find_standings(self):
 259                 imgs = self._soup.findAll('img',
 260                         src=regexp.compile('/yoweb/images/stat.*'))
 261                 re = regexp.compile(
 262 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
 263                         )
 264                 standings = { }
 265
 266                 for skill in puzzles:
 267                         standings[skill] = [ ]
 268
 269                 skl = SoupLog()
 270
 271                 for img in imgs:
 272                         try: puzzle = img['alt']
 273                         except KeyError: continue
 274
 275                         if not puzzle in puzzles:
 276                                 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
 277                                 continue
 278                         key = img.findParent('td')
 279                         if key is None:
 280                                 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
 281                                 continue
 282                         valelem = key.findNextSibling('td')
 283                         if valelem is None:
 284                                 skl.soupm(key, 'puzzle missing sibling "%s"'
 285                                         % puzzle)
 286                                 continue
 287                         valstr = soup_text(valelem)
 288                         match = re.match(valstr)
 289                         if match is None:
 290                                 skl.soupm(key, ('puzzle "%s" unparseable'+
 291                                         ' standing "%s"') % (puzzle, valstr))
 292                                 continue
 293                         standing = match.group(match.lastindex)
 294                         standings[puzzle].append(standing)
 295
 296                 self.standings = { }
 297
 298                 for puzzle in puzzles:
 299                         sl = standings[puzzle]
 300                         if len(sl) > 1:
 301                                 skl.msg('puzzle "%s" multiple standings %s' %
 302                                                 (puzzle, `sl`))
 303                                 continue
 304                         if not sl:
 305                                 skl.msg('puzzle "%s" no standing found' % puzzle)
 306                                 continue
 307                         standing = sl[0]
 308                         for i in range(0, standing_limit):
 309                                 if standing == standingvals[i]:
 310                                         self.standings[puzzle] = i
 311                         if not puzzle in self.standings:
 312                                 skl.msg('puzzle "%s" unknown standing "%s"' %
 313                                         (puzzle, standing))
 314
 315                 all_standings_ok = True
 316                 for puzzle in puzzles:
 317                         if not puzzle in self.standings:
 318                                 self.needs_msgs(skl)
 319
 320         def _find_crewflag(self, cf, yoweb_re):
 321                 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
 322                 if len(things) != 1:
 323                         self.msg('zero or several %s id references found' % cf)
 324                         return None
 325                 thing = things[0]
 326                 id_re = '\\b%sid\\=(\\w+)$' % cf
 327                 id_haystack = thing['href']
 328                 match = regexp.compile(id_re).search(id_haystack)
 329                 if match is None:
 330                         self.soupm(thing, ('incomprehensible %s id ref'+
 331                                 ' (%s in %s)') % (cf, id_re, id_haystack))
 332                         return None
 333                 name = soup_text(thing)
 334                 return (match.group(1), name)
 335
 336         def __str__(self):
 337                 return `(self.crew, self.flag, self.standings, self.msgs)`
 338
 339 #---------- scraper for crew pages ----------
 340
 341 class CrewInfo(SomethingSoupInfo):
 342         # Public data members:
 343         #  ci.crew = [ ('Captain',        ['Pirate', ...]),
 344         #              ('Senior Officer', [...]),
 345         #               ... ]
 346         #  pi.msgs = [ 'message describing problem with scrape' ]
 347
 348         def __init__(self, crewid, max_age=300):
 349                 SomethingSoupInfo.__init__(self,
 350                         'crew/info.wm?crewid=', crewid, max_age)
 351                 self._find_crew()
 352
 353         def _find_crew(self):
 354                 self.crew = []
 355                 capts = self._soup.findAll('img',
 356                         src='/yoweb/images/crew-captain.png')
 357                 if len(capts) != 1:
 358                         self.msg('crew members: no. of captain images != 1')
 359                         return
 360                 tbl = capts[0]
 361                 while not tbl.find('a', href=pirate_ref_re):
 362                         tbl = tbl.findParent('table')
 363                         if not tbl:
 364                                 self.msg('crew members: cannot find table')
 365                                 return
 366                 current_rank_crew = None
 367                 crew_rank_re = regexp.compile('/yoweb/images/crew')
 368                 for row in tbl.contents:
 369                         # findAll(recurse=False)
 370                         if isinstance(row,basestring):
 371                                 continue
 372
 373                         is_rank = row.find('img', attrs={'src': crew_rank_re})
 374                         if is_rank:
 375                                 rank = soup_text(row)
 376                                 current_rank_crew = []
 377                                 self.crew.append((rank, current_rank_crew))
 378                                 continue
 379                         for cell in row.findAll('a', href=pirate_ref_re):
 380                                 if current_rank_crew is None:
 381                                         self.soupm(cell, 'crew members: crew'
 382                                                 ' before rank')
 383                                         continue
 384                                 current_rank_crew.append(soup_text(cell))
 385
 386         def __str__(self):
 387                 return `(self.crew, self.msgs)`
 388
 389 class FlagInfo(SomethingSoupInfo):
 390         def __init__(self, flagid, max_age=600):
 391                 SomethingSoupInfo.__init__(self,
 392                         'flag/info.wm?flagid=', flagid, max_age)
 393                 self._find_flag()
 394
 395         def _find_flag(self):
 396                 font2 = self._soup.find('font',{'size':'+2'})
 397                 self.flag = font2.find('b').contents[0]
 398                 magnate = self._soup.find('img',{'src':
 399                         '/yoweb/images/repute-MAGNATE.png'})
 400                 warinfo = (magnate.findParent('table').findParent('tr').
 401                         findNextSibling('tr').findNext('td',{'align':'left'}))
 402                 for waritem in warinfo.contents:
 403                         print 'ITEM ',`waritem`
 404
 405 #---------- scraper for ocean info incl. embargoes etc. ----------
 406
 407 class IslandInfo():
 408         def __init__(self, ocean, islename):
 409                 self.ocean = ocean
 410                 self.name = islename
 411         def collect(self):
 412                 pass
 413         def yppedia_dataf(self):
 414                 def q(x): return urllib.quote(x.replace(' ','_'))
 415                 url_rhs = q(self.name) + '_(' + q(self.ocean) + ')'
 416                 if opts.localhtml is None:
 417                         url = 'http://yppedia.puzzlepirates.com/' + url_rhs
 418                         debug('IslandInfo retrieving YPP '+url);
 419                         return urllib.urlopen(url)
 420                 else:
 421                         return file(opts.localhtml + '/' + url_rhs, 'r')
 422         def yoweb_url(self):
 423                 soup = BeautifulSoup(self.yppedia_dataf())
 424                 content = soup.find('div', attrs = {'id': 'content'})
 425                 yoweb_re = regexp.compile('^http://\w+\.puzzlepirates\.com/'+
 426                         'yoweb/island/info\.wm\?islandid=\d+$')
 427                 a = soup.find('a', attrs = { 'href': yoweb_re })
 428                 if a is None: return None
 429                 return a['href']
 430         def ruling_flag_id(self):
 431                 yo = self.yoweb_url()
 432                 if yo is None: return None
 433                 dataf = fetcher.fetch(yo, 600)
 434                 soup = BeautifulSoup(dataf)
 435                 ruler_re = regexp.compile('http://\w+\.puzzlepirates\.com/'+
 436                         'yoweb/flag/info\.wm\?flagid=(\d+)$')
 437                 ruler = soup.find('a', attrs = { 'href': ruler_re })
 438                 if not ruler: return None
 439                 m = ruler_re.find(ruler['href'])
 440                 return m.group(1)
 441
 442 class OceanInfo():
 443         # Public data attributes (valid after collect()):
 444         #   oi.islands[islename] = IslandInfo(...)
 445         #   oi.arches[archname][islename] = IslandInfo(...)
 446         def __init__(self):
 447                 self.isleclass = IslandInfo
 448                 self.ocean = fetcher.ocean.lower().capitalize()
 449         def collect(self):
 450                 cmdl = ['./yppedia-ocean-scraper']
 451                 if opts.localhtml is not None:
 452                         cmdl += ['--local-html-dir',opts.localhtml]
 453                 cmdl += [self.ocean]
 454                 debug('OceanInfo collect running ' + `cmdl`)
 455                 oscraper = subprocess.Popen(
 456                         cmdl,
 457                         stdout = subprocess.PIPE,
 458                         cwd = yppsc_dir()+'/yarrg',
 459                         shell=False, stderr=None,
 460                         )
 461                 h = oscraper.stdout.readline()
 462                 debug('OceanInfo collect h '+`h`)
 463                 assert(regexp.match('^ocean ', h))
 464                 arch_re = regexp.compile('^ (\S.*)')
 465                 island_re = regexp.compile('^  (\S.*)')
 466
 467                 self.islands = { }
 468                 self.arches = { }
 469                 archname = None
 470
 471                 for l in oscraper.stdout:
 472                         debug('OceanInfo collect l '+`l`)
 473                         l = l.rstrip('\n')
 474                         m = island_re.match(l)
 475                         if m:
 476                                 assert(archname is not None)
 477                                 islename = m.group(1)
 478                                 isle = self.isleclass(self.ocean, islename)
 479                                 isle.arch = archname
 480                                 self.islands[islename] = isle
 481                                 self.arches[archname][islename] = isle
 482                                 continue
 483                         m = arch_re.match(l)
 484                         if m:
 485                                 archname = m.group(1)
 486                                 assert(archname not in self.arches)
 487                                 self.arches[archname] = { }
 488                                 continue
 489                         assert(False)
 490                 oscraper.wait()
 491                 assert(oscraper.returncode == 0)
 492
 493 #---------- pretty-printer for tables of pirate puzzle standings ----------
 494
 495 class StandingsTable:
 496         def __init__(self, f, use_puzzles=None, col_width=6, gap_every=5):
 497                 if use_puzzles is None:
 498                         if opts.ship_duty:
 499                                 use_puzzles=duty_puzzles
 500                         else:
 501                                 use_puzzles=puzzles
 502                 self._puzzles = use_puzzles
 503                 self.f = f
 504                 self._cw = col_width-1
 505                 self._gap_every = gap_every
 506                 self._linecount = 0
 507                 self._o = f.write
 508
 509         def _nl(self): self._o('\n')
 510
 511         def _pline(self, lhs, puzstrs, extra):
 512                 if (self._linecount > 0
 513                     and self._gap_every is not None
 514                     and not (self._linecount % self._gap_every)):
 515                         self._nl()
 516                 self._o('%-*s' % (max(max_pirate_namelen+1, 15), lhs))
 517                 for v in puzstrs:
 518                         self._o(' %-*.*s' % (self._cw,self._cw, v))
 519                 if extra:
 520                         self._o(' ' + extra)
 521                 self._nl()
 522                 self._linecount += 1
 523
 524         def _puzstr(self, pi, puzzle):
 525                 if not isinstance(puzzle,list): puzzle = [puzzle]
 526                 try: standing = max([pi.standings[p] for p in puzzle])
 527                 except KeyError: return '?'
 528                 if not standing: return ''
 529                 s = ''
 530                 if self._cw > 4:
 531                         c1 = standingvals[standing][0]
 532                         if standing < 3: c1 = c1.lower() # 3 = Master
 533                         s += `standing`
 534                 if self._cw > 5:
 535                         s += ' '
 536                 s += '*' * (standing / 2)
 537                 s += '+' * (standing % 2)
 538                 return s
 539
 540         def headings(self, lhs='', rhs=None):
 541                 def puzn_redact(name):
 542                         if isinstance(name,list):
 543                                 return '/'.join(
 544                                         ["%.*s" % (self._cw/2, puzn_redact(n))
 545                                          for n in name])
 546                         spc = name.find(' ')
 547                         if spc < 0: return name
 548                         return name[0:min(4,spc)] + name[spc+1:]
 549                 self._linecount = -2
 550                 self._pline(lhs, map(puzn_redact, self._puzzles), rhs)
 551                 self._linecount = 0
 552         def literalline(self, line):
 553                 self._o(line)
 554                 self._nl()
 555                 self._linecount = 0
 556         def pirate_dummy(self, name, standingstring, extra=None):
 557                 standings = standingstring * len(self._puzzles)
 558                 self._pline(' '+name, standings, extra)
 559         def pirate(self, pi, extra=None):
 560                 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
 561                 self._pline(' '+pi.name, puzstrs, extra)
 562
 563
 564 #---------- chat log parser ----------
 565
 566 class PirateAboard:
 567         # This is essentially a transparent, dumb, data class.
 568         #  pa.v                 may be None
 569         #  pa.name
 570         #  pa.last_time
 571         #  pa.last_event
 572         #  pa.gunner
 573         #  pa.last_chat_time
 574         #  pa.last_chat_chan
 575         #  pa.pi
 576
 577         # Also used for jobbing applicants:
 578         #               happens when                    expires (to "-")
 579         #   -            disembark, leaves crew          no
 580         #   aboard       evidence of them being aboard   no
 581         #   applied      "has applied for the job"       120s, configurable
 582         #   ashore       "has taken a job"               30min, configurable
 583         #   declined     "declined the job offer"        30s, configurable
 584         #   invited      "has been invited to job"       120s, configurable
 585         #
 586         #  pa.jobber    None, 'ashore', 'applied', 'invited', 'declined'
 587         #  pa.expires   expiry time time
 588
 589         def __init__(pa, pn, v, time, event):
 590                 pa.name = pn
 591                 pa.v = v
 592                 pa.last_time = time
 593                 pa.last_event = event
 594                 pa.last_chat_time = None
 595                 pa.last_chat_chan = None
 596                 pa.gunner = False
 597                 pa.pi = None
 598                 pa.jobber = None
 599                 pa.expires = None
 600
 601         def pirate_info(pa):
 602                 now = time.time()
 603                 if pa.pi:
 604                         age = now - pa.pi_fetched
 605                         guide = random.randint(120,240)
 606                         if age <= guide:
 607                                 return pa.pi
 608                         debug('PirateAboard refresh %d > %d  %s' % (
 609                                 age, guide, pa.name))
 610                         imaginary = [2,4]
 611                 else:
 612                         imaginary = [1]
 613                 wait = fetcher.need_wait(now, imaginary)
 614                 if wait:
 615                         debug('PirateAboard fetcher not ready %d' % wait)
 616                         return pa.pi
 617                 pa.pi = PirateInfo(pa.name, 600)
 618                 pa.pi_fetched = now
 619                 return pa.pi
 620
 621 class ChatLogTracker:
 622         # This is quite complex so we make it opaque.  Use the
 623         # official invokers, accessors etc.
 624
 625         def __init__(self, myself_pi, logfn):
 626                 self._pl = {}   # self._pl['Pirate'] =
 627                 self._vl = {}   #   self._vl['Vessel']['Pirate'] = PirateAboard
 628                                 # self._vl['Vessel']['#lastinfo']
 629                                 # self._vl['Vessel']['#name']
 630                                 # self._v = self._vl[self._vessel]
 631                 self._date = None
 632                 self._myself = myself_pi
 633                 self._lbuf = ''
 634                 self._f = file(logfn)
 635                 flen = os.fstat(self._f.fileno()).st_size
 636                 max_backlog = 500000
 637                 if flen > max_backlog:
 638                         startpos = flen - max_backlog
 639                         self._f.seek(startpos)
 640                         self._f.readline()
 641                 self._progress = [0, flen - self._f.tell()]
 642                 self._disembark_myself()
 643                 self._need_redisplay = False
 644                 self._lastvessel = None
 645
 646         def _disembark_myself(self):
 647                 self._v = None
 648                 self._vessel = None
 649                 self.force_redisplay()
 650
 651         def force_redisplay(self):
 652                 self._need_redisplay = True
 653
 654         def _vessel_updated(self, v, timestamp):
 655                 if v is None: return
 656                 v['#lastinfo'] = timestamp
 657                 self.force_redisplay()
 658
 659         def _onboard_event(self,v,timestamp,pirate,event,jobber=None):
 660                 pa = self._pl.get(pirate, None)
 661                 if pa is not None and pa.v is v:
 662                         pa.last_time = timestamp
 663                         pa.last_event = event
 664                 else:
 665                         if pa is not None and pa.v is not None:
 666                                 del pa.v[pirate]
 667                         pa = PirateAboard(pirate, v, timestamp, event)
 668                         self._pl[pirate] = pa
 669                         if v is not None: v[pirate] = pa
 670                 pa.jobber = jobber
 671
 672                 if jobber is None: timeout = None
 673                 else: timeout = getattr(opts, 'timeout_'+jobber)
 674                 if timeout is None: pa.expires = None
 675                 else: pa.expires = timestamp + timeout
 676                 self._vessel_updated(v, timestamp)
 677                 return pa
 678
 679         def _expire_jobbers(self, now):
 680                 for pa in self._pl.values():
 681                         if pa.expires is None: continue
 682                         if pa.expires >= now: continue
 683                         v = pa.v
 684                         del self._pl[pa.name]
 685                         if v is not None: del v[pa.name]
 686                         self.force_redisplay()
 687
 688         def _trash_vessel(self, v):
 689                 for pn in v:
 690                         if pn.startswith('#'): continue
 691                         del self._pl[pn]
 692                 vn = v['#name']
 693                 del self._vl[vn]
 694                 if v is self._v: self._disembark_myself()
 695                 self.force_redisplay()
 696
 697         def _vessel_stale(self, v, timestamp):
 698                 return timestamp - v['#lastinfo'] > opts.ship_reboard_clearout
 699
 700         def _vessel_check_expire(self, v, timestamp):
 701                 if not self._vessel_stale(v, timestamp):
 702                         return v
 703                 self._debug_line_disposition(timestamp,'',
 704                         'stale-reset ' + v['#name'])
 705                 self._trash_vessel(v)
 706                 return None
 707
 708         def expire_garbage(self, timestamp):
 709                 for v in self._vl.values():
 710                         self._vessel_check_expire(v, timestamp)
 711
 712         def _vessel_lookup(self, vn, timestamp, dml=[], create=False):
 713                 v = self._vl.get(vn, None)
 714                 if v is not None:
 715                         v = self._vessel_check_expire(v, timestamp)
 716                 if v is not None:
 717                         dml.append('found')
 718                         return v
 719                 if not create:
 720                         dml.append('no')
 721                 dml.append('new')
 722                 self._vl[vn] = v = { '#name': vn }
 723                 self._vessel_updated(v, timestamp)
 724                 return v
 725
 726         def _find_matching_vessel(self, pattern, timestamp, cmdr,
 727                                         dml=[], create=False):
 728                 # use when a commander pirate `cmdr' specified a vessel
 729                 #  by name `pattern' (either may be None)
 730                 # if create is true, will create the vessel
 731                 #  record if an exact name is specified
 732
 733                 if (pattern is not None and
 734                     not '*' in pattern
 735                     and len(pattern.split(' ')) == 2):
 736                         vn = pattern.title()
 737                         dml.append('exact')
 738                         return self._vessel_lookup(
 739                                 vn, timestamp, dml=dml, create=create)
 740
 741                 if pattern is None:
 742                         pattern_check = lambda vn: True
 743                 else:
 744                         re = '(?:.* )?%s$' % pattern.lower().replace('*','.+')
 745                         pattern_check = regexp.compile(re, regexp.I).match
 746
 747                 tries = []
 748
 749                 cmdr_pa = self._pl.get(cmdr, None)
 750                 if cmdr_pa: tries.append((cmdr_pa.v, 'cmdr'))
 751
 752                 tries.append((self._v, 'here'))
 753                 tried_vns = []
 754
 755                 for (v, dm) in tries:
 756                         if v is None: dml.append(dm+'?'); continue
 757
 758                         vn = v['#name']
 759                         if not pattern_check(vn):
 760                                 tried_vns.append(vn)
 761                                 dml.append(dm+'#')
 762                                 continue
 763
 764                         dml.append(dm+'!')
 765                         return v
 766
 767                 if pattern is not None and '*' in pattern:
 768                         search = [
 769                                 (vn,v)
 770                                 for (vn,v) in self._vl.iteritems()
 771                                 if not self._vessel_stale(v, timestamp)
 772                                 if pattern_check(vn)
 773                                 ]
 774                         #debug('CLT-RE /%s/ wanted (%s) searched (%s)' % (
 775                         #       re,
 776                         #       '/'.join(tried_vns),
 777                         #       '/'.join([vn for (vn,v) in search])))
 778
 779                         if len(search)==1:
 780                                 dml.append('one')
 781                                 return search[0][1]
 782                         elif search:
 783                                 dml.append('many')
 784                         else:
 785                                 dml.append('none')
 786
 787         def _debug_line_disposition(self,timestamp,l,m):
 788                 debug('CLT %13s %-40s %s' % (timestamp,m,l))
 789
 790         def _rm_crew_l(self,re,l):
 791                 m = regexp.match(re,l)
 792                 if m and m.group(2) == self._myself.crew[1]:
 793                         return m.group(1)
 794                 else:
 795                         return None
 796
 797         def local_command(self, metacmd):
 798                 # returns None if all went well, or problem message
 799                 return self._command(self._myself.name, metacmd,
 800                         "local", time.time(),
 801                         (lambda m: debug('CMD %s' % metacmd)))
 802
 803         def _command(self, cmdr, metacmd, chan, timestamp, d):
 804                 # returns None if all went well, or problem message
 805                 metacmd = regexp.sub('\\s+', ' ', metacmd).strip()
 806                 m2 = regexp.match(
 807                     '/([adj]) (?:([A-Za-z* ]+)\\s*:)?([A-Za-z ]+)$',
 808                     metacmd)
 809                 if not m2: return "unknown syntax or command"
 810
 811                 (cmd, pattern, targets) = m2.groups()
 812                 dml = ['cmd', chan, cmd]
 813
 814                 if cmd == 'a': each = self._onboard_event
 815                 elif cmd == 'd': each = disembark
 816                 else: each = lambda *l: self._onboard_event(*l,
 817                                 **{'jobber':'applied'})
 818
 819                 if cmdr == self._myself.name:
 820                         dml.append('self')
 821                         how = 'cmd: %s' % cmd
 822                 else:
 823                         dml.append('other')
 824                         how = 'cmd: %s %s' % (cmd,cmdr)
 825
 826                 if cmd == 'j':
 827                         if pattern is not None:
 828                                 return "/j command does not take a vessel"
 829                         v = None
 830                 else:
 831                         v = self._find_matching_vessel(
 832                                 pattern, timestamp, cmdr,
 833                                 dml, create=True)
 834
 835                 if cmd == 'j' or v is not None:
 836                         targets = targets.strip().split(' ')
 837                         dml.append(`len(targets)`)
 838                         for target in targets:
 839                                 each(v, timestamp, target.title(), how)
 840                         self._vessel_updated(v, timestamp)
 841
 842                 dm = ' '.join(dml)
 843                 return d(dm)
 844
 845                 return None
 846
 847         def chatline(self,l):
 848                 rm = lambda re: regexp.match(re,l)
 849                 d = lambda m: self._debug_line_disposition(timestamp,l,m)
 850                 rm_crew = lambda re: self._rm_crew_l(re,l)
 851                 timestamp = None
 852
 853                 m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$')
 854                 if m:
 855                         self._date = [int(x) for x in m.groups()]
 856                         self._previous_timestamp = None
 857                         return d('date '+`self._date`)
 858
 859                 if self._date is None:
 860                         return d('date unset')
 861
 862                 m = rm('\\[(\d\d):(\d\d):(\d\d)\\] ')
 863                 if not m:
 864                         return d('no timestamp')
 865
 866                 while True:
 867                         time_tuple = (self._date +
 868                                       [int(x) for x in m.groups()] +
 869                                       [-1,-1,-1])
 870                         timestamp = time.mktime(time_tuple)
 871                         if timestamp >= self._previous_timestamp: break
 872                         self._date[2] += 1
 873                         self._debug_line_disposition(timestamp,'',
 874                                 'new date '+`self._date`)
 875
 876                 self._previous_timestamp = timestamp
 877
 878                 l = l[l.find(' ')+1:]
 879
 880                 def ob_x(pirate,event):
 881                         return self._onboard_event(
 882                                         self._v, timestamp, pirate, event)
 883                 def ob1(did): ob_x(m.group(1), did); return d(did)
 884                 def oba(did): return ob1('%s %s' % (did, m.group(2)))
 885
 886                 def jb(pirate,jobber):
 887                         return self._onboard_event(
 888                                 None, timestamp, pirate,
 889                                 ("jobber %s" % jobber),
 890                                 jobber=jobber
 891                                 )
 892
 893                 def disembark(v, timestamp, pirate, event):
 894                         self._onboard_event(
 895                                         v, timestamp, pirate, 'leaving '+event)
 896                         del v[pirate]
 897                         del self._pl[pirate]
 898
 899                 def disembark_me(why):
 900                         self._disembark_myself()
 901                         return d('disembark-me '+why)
 902
 903                 m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
 904                 if m:
 905                         dm = ['boarding']
 906                         pn = self._myself.name
 907                         vn = m.group(1)
 908                         v = self._vessel_lookup(vn, timestamp, dm, create=True)
 909                         self._lastvessel = self._vessel = vn
 910                         self._v = v
 911                         ob_x(pn, 'we boarded')
 912                         self.expire_garbage(timestamp)
 913                         return d(' '.join(dm))
 914
 915                 if self._v is None:
 916                         return d('no vessel')
 917
 918                 m = rm('(\\w+) has come aboard\\.$')
 919                 if m: return ob1('boarded');
 920
 921                 m = rm('You have ordered (\\w+) to do some (\\S.*\\S)\\.$')
 922                 if m:
 923                         (who,what) = m.groups()
 924                         pa = ob_x(who,'ord '+what)
 925                         if what == 'Gunning':
 926                                 pa.gunner = True
 927                         return d('duty order')
 928
 929                 m = rm('(\\w+) abandoned a (\\S.*\\S) station\\.$')
 930                 if m: oba('stopped'); return d("end")
 931
 932                 def chat_core(speaker, chan):
 933                         try: pa = self._pl[speaker]
 934                         except KeyError: return 'mystery'
 935                         if pa.v is not None and pa.v is not self._v:
 936                                 return 'elsewhere'
 937                         pa.last_chat_time = timestamp
 938                         pa.last_chat_chan = chan
 939                         self.force_redisplay()
 940                         return 'here'
 941
 942                 def chat(chan):
 943                         speaker = m.group(1)
 944                         dm = chat_core(speaker, chan)
 945                         return d('chat %s %s' % (chan, dm))
 946
 947                 def chat_metacmd(chan):
 948                         (cmdr, metacmd) = m.groups()
 949                         whynot = self._command(
 950                                 cmdr, metacmd, chan, timestamp, d)
 951                         if whynot is not None:
 952                                 return chat(chan)
 953                         else:
 954                                 chat_core(cmdr, 'cmd '+chan)
 955
 956                 m = rm('(\\w+) (?:issued an order|ordered everyone) "')
 957                 if m: return ob1('general order');
 958
 959                 m = rm('(\\w+) says, "')
 960                 if m: return chat('public')
 961
 962                 m = rm('(\\w+) tells ye, "')
 963                 if m: return chat('private')
 964
 965                 m = rm('Ye told (\\w+), "(.*)"$')
 966                 if m: return chat_metacmd('private')
 967
 968                 m = rm('(\\w+) flag officer chats, "')
 969                 if m: return chat('flag officer')
 970
 971                 m = rm('(\\w+) officer chats, "(.*)"$')
 972                 if m: return chat_metacmd('officer')
 973
 974                 m = rm('Ye accepted the offer to job with ')
 975                 if m: return disembark_me('jobbing')
 976
 977                 m = rm('Ye hop on the ferry and are whisked away ')
 978                 if m: return disembark_me('ferry')
 979
 980                 m = rm('Whisking away to yer home on the magical winds')
 981                 if m: return disembark_me('home')
 982
 983                 m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
 984                 if m:
 985                         pl = m.group(1).split(', ')
 986                         if not self._myself.name in pl:
 987                                 return d('lost melee')
 988                         for pn in pl:
 989                                 if ' ' in pn: continue
 990                                 ob_x(pn,'won melee')
 991                         return d('won melee')
 992
 993                 m = rm('(\\w+) is eliminated\\!')
 994                 if m: return ob1('eliminated in fray');
 995
 996                 m = rm('(\\w+) has driven \w+ from the ship\\!')
 997                 if m: return ob1('boarder repelled');
 998
 999                 m = rm('\w+ has bested (\\w+), and turns'+
1000                         ' to the rest of the ship\\.')
1001                 if m: return ob1('boarder unrepelled');
1002
1003                 pirate = rm_crew("(\\w+) has taken a job with '(.*)'\\.")
1004                 if pirate: return jb(pirate, 'ashore')
1005
1006                 pirate = rm_crew("(\\w+) has left '(.*)'\\.")
1007                 if pirate:
1008                         disembark(self._v, timestamp, pirate, 'left crew')
1009                         return d('left crew')
1010
1011                 m = rm('(\w+) has applied for the posted job\.')
1012                 if m: return jb(m.group(1), 'applied')
1013
1014                 pirate= rm_crew("(\\w+) has been invited to job for '(.*)'\\.")
1015                 if pirate: return jb(pirate, 'invited')
1016
1017                 pirate = rm_crew("(\\w+) declined the job offer for '(.*)'\\.")
1018                 if pirate: return jb(pirate, 'declined')
1019
1020                 m = rm('(\\w+) has left the vessel\.')
1021                 if m:
1022                         pirate = m.group(1)
1023                         disembark(self._v, timestamp, pirate, 'disembarked')
1024                         return d('disembarked')
1025
1026                 return d('not-matched')
1027
1028         def _str_pa(self, pn, pa):
1029                 assert self._pl[pn] == pa
1030                 s = ' '*20 + "%s %-*s %13s %-30s %13s %-20s %13s" % (
1031                         (' ','G')[pa.gunner],
1032                         max_pirate_namelen, pn,
1033                         pa.last_time, pa.last_event,
1034                         pa.last_chat_time, pa.last_chat_chan,
1035                         pa.jobber)
1036                 if pa.expires is not None:
1037                         s += " %-5d" % (pa.expires - pa.last_time)
1038                 s += "\n"
1039                 return s
1040
1041         def _str_vessel(self, vn, v):
1042                 s = ' vessel %s\n' % vn
1043                 s += ' '*20 + "%-*s   %13s\n" % (
1044                                 max_pirate_namelen, '#lastinfo',
1045                                 v['#lastinfo'])
1046                 assert v['#name'] == vn
1047                 for pn in sorted(v.keys()):
1048                         if pn.startswith('#'): continue
1049                         pa = v[pn]
1050                         assert pa.v == v
1051                         s += self._str_pa(pn,pa)
1052                 return s
1053
1054         def __str__(self):
1055                 s = '''<ChatLogTracker
1056  myself %s
1057  vessel %s
1058 '''                     % (self._myself.name, self._vessel)
1059                 assert ((self._v is None and self._vessel is None) or
1060                         (self._v is self._vl[self._vessel]))
1061                 if self._vessel is not None:
1062                         s += self._str_vessel(self._vessel, self._v)
1063                 for vn in sorted(self._vl.keys()):
1064                         if vn == self._vessel: continue
1065                         s += self._str_vessel(vn, self._vl[vn])
1066                 s += " elsewhere\n"
1067                 for p in self._pl:
1068                         pa = self._pl[p]
1069                         if pa.v is not None:
1070                                 assert pa.v[p] is pa
1071                                 assert pa.v in self._vl.values()
1072                         else:
1073                                 s += self._str_pa(pa.name, pa)
1074                 s += '>\n'
1075                 return s
1076
1077         def catchup(self, progress=None):
1078                 while True:
1079                         more = self._f.readline()
1080                         if not more: break
1081
1082                         self._progress[0] += len(more)
1083                         if progress: progress.progress(*self._progress)
1084
1085                         self._lbuf += more
1086                         if self._lbuf.endswith('\n'):
1087                                 self.chatline(self._lbuf.rstrip())
1088                                 self._lbuf = ''
1089                                 if opts.debug >= 2:
1090                                         debug(self.__str__())
1091                 self._expire_jobbers(time.time())
1092
1093                 if progress: progress.caughtup()
1094
1095         def changed(self):
1096                 rv = self._need_redisplay
1097                 self._need_redisplay = False
1098                 return rv
1099         def myname(self):
1100                 # returns our pirate name
1101                 return self._myself.name
1102         def vesselname(self):
1103                 # returns the vessel name we're aboard or None
1104                 return self._vessel
1105         def lastvesselname(self):
1106                 # returns the last vessel name we were aboard or None
1107                 return self._lastvessel
1108         def aboard(self, vesselname=True):
1109                 # returns a list of PirateAboard the vessel
1110                 #  sorted by pirate name
1111                 #  you can pass this None and you'll get []
1112                 #  or True for the current vessel (which is the default)
1113                 #  the returned value is a fresh list of persistent
1114                 #  PirateAboard objects
1115                 if vesselname is True: v = self._v
1116                 else: v = self._vl.get(vesselname.title())
1117                 if v is None: return []
1118                 return [ v[pn]
1119                          for pn in sorted(v.keys())
1120                          if not pn.startswith('#') ]
1121         def jobbers(self):
1122                 # returns a the jobbers' PirateAboards,
1123                 # sorted by jobber class and reverse of expiry time
1124                 l = [ pa
1125                       for pa in self._pl.values()
1126                       if pa.jobber is not None
1127                     ]
1128                 def compar_key(pa):
1129                         return (pa.jobber, -pa.expires)
1130                 l.sort(key = compar_key)
1131                 return l
1132
1133 #---------- implementations of actual operation modes ----------
1134
1135 def do_pirate(pirates, bu):
1136         print '{'
1137         for pirate in pirates:
1138                 info = PirateInfo(pirate)
1139                 print '%s: %s,' % (`pirate`, info)
1140         print '}'
1141
1142 def prep_crew_of(args, bu, max_age=300):
1143         if len(args) != 1: bu('crew-of takes one pirate name')
1144         pi = PirateInfo(args[0], max_age)
1145         if pi.crew is None: return None
1146         return CrewInfo(pi.crew[0], max_age)
1147
1148 def do_crew_of(args, bu):
1149         ci = prep_crew_of(args, bu)
1150         print ci
1151
1152 def do_flag_of(args, bu):
1153         if len(args) != 1: bu('flag-of takes one pirate name')
1154         max_age = 300
1155         pi = PirateInfo(args[0], max_age)
1156         if pi.flag is None: fi = None
1157         else: fi = FlagInfo(pi.flag[0], max_age)
1158         print `fi`
1159
1160 def do_standings_crew_of(args, bu):
1161         ci = prep_crew_of(args, bu, 60)
1162         tab = StandingsTable(sys.stdout)
1163         tab.headings()
1164         for (rank, members) in ci.crew:
1165                 if not members: continue
1166                 tab.literalline('')
1167                 tab.literalline('%s:' % rank)
1168                 for p in members:
1169                         pi = PirateInfo(p, random.randint(900,1800))
1170                         tab.pirate(pi)
1171
1172 def do_ocean(args, bu):
1173         if (len(args)): bu('ocean takes no further arguments')
1174         fetcher.default_ocean()
1175         oi = OceanInfo()
1176         oi.collect()
1177         for islename in sorted(oi.islands.keys()):
1178                 isle = oi.islands[islename]
1179                 yoweb_url = isle.yoweb_url()
1180                 print " %s -- %s" % (islename, yoweb_url)
1181
1182 #----- modes which use the chat log parser are quite complex -----
1183
1184 class ProgressPrintPercentage:
1185         def __init__(self, f=sys.stdout):
1186                 self._f = f
1187         def progress_string(self,done,total):
1188                 return "scan chat logs %3d%%\r" % ((done*100) / total)
1189         def progress(self,*a):
1190                 self._f.write(self.progress_string(*a))
1191                 self._f.flush()
1192         def show_init(self, pirate, ocean):
1193                 print >>self._f, 'Starting up, %s on the %s ocean' % (
1194                         pirate, ocean)
1195         def caughtup(self):
1196                 self._f.write('                   \r')
1197                 self._f.flush()
1198
1199 def prep_chat_log(args, bu,
1200                 progress=ProgressPrintPercentage(),
1201                 max_myself_age=3600):
1202         if len(args) != 1: bu('this action takes only chat log filename')
1203         logfn = args[0]
1204         logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_'
1205         match = regexp.match(logfn_re, logfn)
1206         if not match: bu('chat log filename is not in expected format')
1207         (pirate, ocean) = match.groups()
1208         fetcher.default_ocean(ocean)
1209
1210         progress.show_init(pirate, fetcher.ocean)
1211         myself = PirateInfo(pirate,max_myself_age)
1212         track = ChatLogTracker(myself, logfn)
1213
1214         opts.debug -= 2
1215         track.catchup(progress)
1216         opts.debug += 2
1217
1218         track.force_redisplay()
1219
1220         return (myself, track)
1221
1222 def do_track_chat_log(args, bu):
1223         (myself, track) = prep_chat_log(args, bu)
1224         while True:
1225                 track.catchup()
1226                 if track.changed():
1227                         print track
1228                 sleep(0.5 + 0.5 * random.random())
1229
1230 #----- ship management aid -----
1231
1232 class Display_dumb(ProgressPrintPercentage):
1233         def __init__(self):
1234                 ProgressPrintPercentage.__init__(self)
1235         def show(self, s):
1236                 print '\n\n', s;
1237         def realstart(self):
1238                 pass
1239
1240 class Display_overwrite(ProgressPrintPercentage):
1241         def __init__(self):
1242                 ProgressPrintPercentage.__init__(self)
1243
1244                 null = file('/dev/null','w')
1245                 curses.setupterm(fd=null.fileno())
1246
1247                 self._clear = curses.tigetstr('clear')
1248                 if not self._clear:
1249                         self._debug('missing clear!')
1250                         self.show = Display_dumb.show
1251                         return
1252
1253                 self._t = {'el':'', 'ed':''}
1254                 if not self._init_sophisticated():
1255                         for k in self._t.keys(): self._t[k] = ''
1256                         self._t['ho'] = self._clear
1257
1258         def _debug(self,m): debug('display overwrite: '+m)
1259
1260         def _init_sophisticated(self):
1261                 for k in self._t.keys():
1262                         s = curses.tigetstr(k)
1263                         self._t[k] = s
1264                 self._t['ho'] = curses.tigetstr('ho')
1265                 if not self._t['ho']:
1266                         cup = curses.tigetstr('cup')
1267                         self._t['ho'] = curses.tparm(cup,0,0)
1268                 missing = [k for k in self._t.keys() if not self._t[k]]
1269                 if missing:
1270                         self.debug('missing '+(' '.join(missing)))
1271                         return 0
1272                 return 1
1273
1274         def show(self, s):
1275                 w = sys.stdout.write
1276                 def wti(k): w(self._t[k])
1277
1278                 wti('ho')
1279                 nl = ''
1280                 for l in s.rstrip().split('\n'):
1281                         w(nl)
1282                         w(l)
1283                         wti('el')
1284                         nl = '\r\n'
1285                 wti('ed')
1286                 w(' ')
1287                 sys.stdout.flush()
1288
1289         def realstart(self):
1290                 sys.stdout.write(self._clear)
1291                 sys.stdout.flush()
1292
1293
1294 def do_ship_aid(args, bu):
1295         if opts.ship_duty is None: opts.ship_duty = True
1296
1297         displayer = globals()['Display_'+opts.display]()
1298
1299         (myself, track) = prep_chat_log(args, bu, progress=displayer)
1300
1301         displayer.realstart()
1302
1303         if os.isatty(0): kr_create = KeystrokeReader
1304         else: kr_create = DummyKeystrokeReader
1305
1306         try:
1307                 kreader = kr_create(0, 10)
1308                 ship_aid_core(myself, track, displayer, kreader)
1309         finally:
1310                 kreader.stop()
1311                 print '\n'
1312
1313 class KeyBasedSorter:
1314         def compar_key_pa(self, pa):
1315                 pi = pa.pirate_info()
1316                 if pi is None: return None
1317                 return self.compar_key(pi)
1318         def lsort_pa(self, l):
1319                 l.sort(key = self.compar_key_pa)
1320
1321 class NameSorter(KeyBasedSorter):
1322         def compar_key(self, pi): return pi.name
1323         def desc(self): return 'name'
1324
1325 class SkillSorter(NameSorter):
1326         def __init__(self, relevant):
1327                 self._want = frozenset(relevant.split('/'))
1328                 self._avoid = set()
1329                 for p in core_duty_puzzles:
1330                         if isinstance(p,basestring): self._avoid.add(p)
1331                         else: self._avoid |= set(p)
1332                 self._avoid -= self._want
1333                 self._desc = '%s' % relevant
1334
1335         def desc(self): return self._desc
1336
1337         def compar_key(self, pi):
1338                 best_want = max([
1339                         pi.standings.get(puz,-1)
1340                         for puz in self._want
1341                         ])
1342                 best_avoid = [
1343                         -pi.standings.get(puz,standing_limit)
1344                         for puz in self._avoid
1345                         ]
1346                 best_avoid.sort()
1347                 def negate(x): return -x
1348                 debug('compar_key %s bw=%s ba=%s' % (pi.name, `best_want`,
1349                         `best_avoid`))
1350                 return (-best_want, map(negate, best_avoid), pi.name)
1351
1352 def ship_aid_core(myself, track, displayer, kreader):
1353
1354         def find_vessel():
1355                 vn = track.vesselname()
1356                 if vn: return (vn, " on board the %s" % vn)
1357                 vn = track.lastvesselname()
1358                 if vn: return (vn, " ashore from the %s" % vn)
1359                 return (None, " not on a vessel")
1360
1361         def timeevent(t,e):
1362                 if t is None: return ' ' * 22
1363                 return " %-4s %-16s" % (format_time_interval(now - t),e)
1364
1365         displayer.show(track.myname() + find_vessel()[1] + '...')
1366
1367         rotate_nya = '/-\\'
1368
1369         sort = NameSorter()
1370         clicmd = None
1371         clierr = None
1372         cliexec = None
1373
1374         while True:
1375                 track.catchup()
1376                 now = time.time()
1377
1378                 (vn, vs) = find_vessel()
1379
1380                 s = ''
1381                 if cliexec is not None:
1382                         s += '...'
1383                 elif clierr is not None:
1384                         s += 'Error: '+clierr
1385                 elif clicmd is not None:
1386                         s += '/' + clicmd
1387                 else:
1388                         s = track.myname() + vs
1389                         s += " at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
1390                         s += kreader.info()
1391                 s += '\n'
1392
1393                 tbl_s = StringIO()
1394                 tbl = StandingsTable(tbl_s)
1395
1396                 aboard = track.aboard(vn)
1397                 sort.lsort_pa(aboard)
1398
1399                 jobbers = track.jobbers()
1400
1401                 if track.vesselname(): howmany = 'aboard: %2d' % len(aboard)
1402                 else: howmany = ''
1403
1404                 tbl.headings(howmany, '  sorted by '+sort.desc())
1405
1406                 last_jobber = None
1407
1408                 for pa in aboard + jobbers:
1409                         if pa.jobber != last_jobber:
1410                                 last_jobber = pa.jobber
1411                                 tbl.literalline('')
1412                                 tbl.literalline('jobbers '+last_jobber)
1413
1414                         pi = pa.pirate_info()
1415
1416                         xs = ''
1417                         if pa.gunner: xs += 'G '
1418                         else: xs += '  '
1419                         xs += timeevent(pa.last_time, pa.last_event)
1420                         xs += timeevent(pa.last_chat_time, pa.last_chat_chan)
1421
1422                         if pi is None:
1423                                 tbl.pirate_dummy(pa.name, rotate_nya[0], xs)
1424                         else:
1425                                 tbl.pirate(pi, xs)
1426
1427                 s += tbl_s.getvalue()
1428                 displayer.show(s)
1429                 tbl_s.close()
1430
1431                 if cliexec is not None:
1432                         clierr = track.local_command("/"+cliexec.strip())
1433                         cliexec = None
1434                         continue
1435
1436                 k = kreader.getch()
1437                 if k is None:
1438                         rotate_nya = rotate_nya[1:3] + rotate_nya[0]
1439                         continue
1440
1441                 if clierr is not None:
1442                         clierr = None
1443                         continue
1444
1445                 if clicmd is not None:
1446                         if k == '\r' or k == '\n':
1447                                 cliexec = clicmd
1448                                 clicmd = clicmdbase
1449                         elif k == '\e' and clicmd != "":
1450                                 clicmd = clicmdbase
1451                         elif k == '\33':
1452                                 clicmd = None
1453                         elif k == '\b' or k == '\177':
1454                                 clicmd = clicmd[ 0 : len(clicmd)-1 ]
1455                         else:
1456                                 clicmd += k
1457                         continue
1458
1459                 if k == 'q': break
1460                 elif k == 'g': sort = SkillSorter('Gunning')
1461                 elif k == 'c': sort = SkillSorter('Carpentry')
1462                 elif k == 's': sort = SkillSorter('Sailing/Rigging')
1463                 elif k == 'b': sort = SkillSorter('Bilging')
1464                 elif k == 'n': sort = SkillSorter('Navigating')
1465                 elif k == 'd': sort = SkillSorter('Battle Navigation')
1466                 elif k == 't': sort = SkillSorter('Treasure Haul')
1467                 elif k == 'a': sort = NameSorter()
1468                 elif k == '/': clicmdbase = ""; clicmd = clicmdbase
1469                 elif k == '+': clicmdbase = "a "; clicmd = clicmdbase
1470                 else: pass # unknown key command
1471
1472 #---------- individual keystroke input ----------
1473
1474 class DummyKeystrokeReader:
1475         def __init__(self,fd,timeout_dummy): pass
1476         def stop(self): pass
1477         def getch(self): sleep(1); return None
1478         def info(self): return ' [noninteractive]'
1479
1480 class KeystrokeReader(DummyKeystrokeReader):
1481         def __init__(self, fd, timeout_decisec=0):
1482                 self._fd = fd
1483                 self._saved = termios.tcgetattr(fd)
1484                 a = termios.tcgetattr(fd)
1485                 a[3] &= ~(termios.ECHO | termios.ECHONL |
1486                           termios.ICANON | termios.IEXTEN)
1487                 a[6][termios.VMIN] = 0
1488                 a[6][termios.VTIME] = timeout_decisec
1489                 termios.tcsetattr(fd, termios.TCSANOW, a)
1490         def stop(self):
1491                 termios.tcsetattr(self._fd, termios.TCSANOW, self._saved)
1492         def getch(self):
1493                 debug_flush()
1494                 byte = os.read(self._fd, 1)
1495                 if not len(byte): return None
1496                 return byte
1497         def info(self):
1498                 return ''
1499
1500 #---------- main program ----------
1501
1502 def main():
1503         global opts, fetcher
1504
1505         pa = OptionParser(
1506 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
1507 actions:
1508  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
1509  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
1510  yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
1511  yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG
1512  yoweb-scrape [options] ship-aid CHAT-LOG  (must be .../PIRATE_OCEAN_chat-log*)
1513
1514 display modes (for --display) apply to ship-aid:
1515  --display=dumb       just print new information, scrolling the screen
1516  --display=overwrite  use cursor motion, selective clear, etc. to redraw at top''')
1517         ao = pa.add_option
1518         ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
1519                 help='select ocean OCEAN')
1520         ao('--cache-dir', dest='cache_dir', metavar='DIR',
1521                 default='~/.yoweb-scrape-cache',
1522                 help='cache yoweb pages in DIR')
1523         ao('-D','--debug', action='count', dest='debug', default=0,
1524                 help='enable debugging output')
1525         ao('--debug-fd', type='int', dest='debug_fd',
1526                 help='write any debugging output to specified fd')
1527         ao('-q','--quiet', action='store_true', dest='quiet',
1528                 help='suppress warning output')
1529         ao('--display', action='store', dest='display',
1530                 type='choice', choices=['dumb','overwrite'],
1531                 help='how to display ship aid')
1532         ao('--local-ypp-dir', action='store', dest='localhtml',
1533                 help='get yppedia pages from local directory LOCALHTML'+
1534                         ' instead of via HTTP')
1535
1536         ao_jt = lambda wh, t: ao(
1537                 '--timeout-sa-'+wh, action='store', dest='timeout_'+wh,
1538                 default=t, help=('set timeout for expiring %s jobbers' % wh))
1539         ao_jt('applied',      120)
1540         ao_jt('invited',      120)
1541         ao_jt('declined',      30)
1542         ao_jt('ashore',      1800)
1543
1544         ao('--ship-duty', action='store_true', dest='ship_duty',
1545                 help='show ship duty station puzzles')
1546         ao('--all-puzzles', action='store_false', dest='ship_duty',
1547                 help='show all puzzles, not just ship duty stations')
1548
1549         ao('--min-cache-reuse', type='int', dest='min_max_age',
1550                 metavar='SECONDS', default=60,
1551                 help='always reuse cache yoweb data if no older than this')
1552
1553         (opts,args) = pa.parse_args()
1554         random.seed()
1555
1556         if len(args) < 1:
1557                 print >>sys.stderr, copyright_info
1558                 pa.error('need a mode argument')
1559
1560         if opts.debug_fd is not None:
1561                 opts.debug_file = os.fdopen(opts.debug_fd, 'w')
1562         else:
1563                 opts.debug_file = sys.stdout
1564
1565         mode = args[0]
1566         mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
1567         try: mode_fn = globals()[mode_fn_name]
1568         except KeyError: pa.error('unknown mode "%s"' % mode)
1569
1570         # fixed parameters
1571         opts.expire_age = max(3600, opts.min_max_age)
1572
1573         opts.ship_reboard_clearout = 3600
1574
1575         if opts.cache_dir.startswith('~/'):
1576                 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
1577
1578         if opts.display is None:
1579                 if ((opts.debug > 0 and opts.debug_fd is None)
1580                     or not os.isatty(sys.stdout.fileno())):
1581                         opts.display = 'dumb'
1582                 else:
1583                         opts.display = 'overwrite'
1584
1585         fetcher = Fetcher(opts.ocean, opts.cache_dir)
1586
1587         mode_fn(args[1:], pa.error)
1588
1589 main()