yoweb-scrape

   1 #!/usr/bin/python
   2 # This is part of ypp-sc-tools, a set of third-party tools for assisting
   3 # players of Yohoho Puzzle Pirates.
   4 #
   5 # Copyright (C) 2009 Ian Jackson <ijackson@chiark.greenend.org.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 #
  20 # Yohoho and Puzzle Pirates are probably trademarks of Three Rings and
  21 # are used without permission.  This program is not endorsed or
  22 # sponsored by Three Rings.
  23
  24 copyright_info = '''
  25 yoweb-scrape is part of ypp-sc-tools  Copyright (C) 2009 Ian Jackson
  26 This program comes with ABSOLUTELY NO WARRANTY; this is free software,
  27 and you are welcome to redistribute it under certain conditions.
  28 For details, read the top of the yoweb-scrape file.
  29 '''
  30
  31 #---------- setup ----------
  32
  33 import signal
  34 signal.signal(signal.SIGINT, signal.SIG_DFL)
  35
  36 import os
  37 import time
  38 import urllib
  39 import urllib2
  40 import errno
  41 import sys
  42 import re as regexp
  43 import random
  44 import curses
  45 import termios
  46 import random
  47 import subprocess
  48 from optparse import OptionParser
  49 from StringIO import StringIO
  50
  51 from BeautifulSoup import BeautifulSoup
  52
  53 opts = None
  54
  55 #---------- YPP parameters and arrays ----------
  56
  57 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
  58         '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
  59         '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
  60         '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
  61
  62 core_duty_puzzles = [
  63                 'Gunning',
  64                 ['Sailing','Rigging'],
  65                 'Bilging',
  66                 'Carpentry',
  67                 ]
  68
  69 duty_puzzles = ([ 'Navigating', 'Battle Navigation' ] +
  70                 core_duty_puzzles +
  71                 [ 'Treasure Haul' ])
  72
  73 standingvals = ('Able/Proficient/Distinguished/Respected/Master'+
  74                 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
  75 standing_limit = len(standingvals)
  76
  77 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
  78
  79 max_pirate_namelen = 12
  80
  81
  82 #---------- general utilities ----------
  83
  84 def debug(m):
  85         if opts.debug > 0:
  86                 print >>opts.debug_file, m
  87
  88 def debug_flush():
  89         if opts.debug > 0:
  90                 opts.debug_file.flush()
  91
  92 def sleep(seconds):
  93         debug_flush()
  94         time.sleep(seconds)
  95
  96 def format_time_interval(ti):
  97         if ti < 120: return '%d:%02d' % (ti / 60, ti % 60)
  98         if ti < 7200: return '%2dm' % (ti / 60)
  99         if ti < 86400: return '%dh' % (ti / 3600)
 100         return '%dd' % (ti / 86400)
 101
 102 def yppsc_dir():
 103         lib = os.getenv("YPPSC_YARRG_SRCBASE")
 104         if lib is not None: return lib
 105         lib = sys.argv[0]
 106         lib = regexp.sub('/[^/]+$', '', lib)
 107         os.environ["YPPSC_YARRG_SRCBASE"] = lib
 108         return lib
 109
 110 #---------- caching and rate-limiting data fetcher ----------
 111
 112 class Fetcher:
 113         def __init__(self, ocean, cachedir):
 114                 debug('Fetcher init %s' % cachedir)
 115                 self.ocean = ocean
 116                 self.cachedir = cachedir
 117                 try: os.mkdir(cachedir)
 118                 except (OSError,IOError), oe:
 119                         if oe.errno != errno.EEXIST: raise
 120                 self._cache_scan(time.time())
 121
 122         def default_ocean(self, ocean='ice'):
 123                 if self.ocean is None:
 124                         self.ocean = ocean
 125
 126         def _cache_scan(self, now):
 127                 # returns list of ages, unsorted
 128                 ages = []
 129                 debug('Fetcher   scan_cache')
 130                 for leaf in os.listdir(self.cachedir):
 131                         if not leaf.startswith('#'): continue
 132                         path = self.cachedir + '/' + leaf
 133                         try: s = os.stat(path)
 134                         except (OSError,IOError), oe:
 135                                 if oe.errno != errno.ENOENT: raise
 136                                 continue
 137                         age = now - s.st_mtime
 138                         if age > opts.expire_age:
 139                                 debug('Fetcher    expire %d %s' % (age, path))
 140                                 try: os.remove(path)
 141                                 except (OSError,IOError), oe:
 142                                         if oe.errno != errno.ENOENT: raise
 143                                 continue
 144                         ages.append(age)
 145                 return ages
 146
 147         def need_wait(self, now, imaginary=[]):
 148                 ages = self._cache_scan(now)
 149                 ages += imaginary
 150                 ages.sort()
 151                 debug('Fetcher   ages ' + `ages`)
 152                 min_age = 1
 153                 need_wait = 0
 154                 for age in ages:
 155                         if age < min_age and age <= 5:
 156                                 debug('Fetcher   morewait min=%d age=%d' %
 157                                         (min_age, age))
 158                                 need_wait = max(need_wait, min_age - age)
 159                         min_age += 3
 160                         min_age *= 1.25
 161                 if need_wait > 0:
 162                         need_wait += random.random() - 0.5
 163                 return need_wait
 164
 165         def _rate_limit_cache_clean(self, now):
 166                 need_wait = self.need_wait(now)
 167                 if need_wait > 0:
 168                         debug('Fetcher   wait %d' % need_wait)
 169                         sleep(need_wait)
 170
 171         def fetch(self, url, max_age):
 172                 debug('Fetcher fetch %s' % url)
 173                 cache_corename = urllib.quote_plus(url)
 174                 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
 175                 try: f = file(cache_item, 'r')
 176                 except (OSError,IOError), oe:
 177                         if oe.errno != errno.ENOENT: raise
 178                         f = None
 179                 now = time.time()
 180                 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
 181                 if f is not None:
 182                         s = os.fstat(f.fileno())
 183                         age = now - s.st_mtime
 184                         if age > max_age:
 185                                 debug('Fetcher  stale %d < %d'% (max_age, age))
 186                                 f = None
 187                 if f is not None:
 188                         data = f.read()
 189                         f.close()
 190                         debug('Fetcher  cached %d > %d' % (max_age, age))
 191                         return data
 192
 193                 debug('Fetcher  fetch')
 194                 self._rate_limit_cache_clean(now)
 195
 196                 stream = urllib2.urlopen(url)
 197                 data = stream.read()
 198                 cache_tmp = "%s/#%s~%d#" % (
 199                         self.cachedir, cache_corename, os.getpid())
 200                 f = file(cache_tmp, 'w')
 201                 f.write(data)
 202                 f.close()
 203                 os.rename(cache_tmp, cache_item)
 204                 debug('Fetcher  stored')
 205                 return data
 206
 207         def yoweb(self, kind, tail, max_age):
 208                 self.default_ocean()
 209                 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
 210                         self.ocean, kind, tail)
 211                 return self.fetch(url, max_age)
 212
 213 #---------- logging assistance for troubled screenscrapers ----------
 214
 215 class SoupLog:
 216         def __init__(self):
 217                 self.msgs = [ ]
 218         def msg(self, m):
 219                 self.msgs.append(m)
 220         def soupm(self, obj, m):
 221                 self.msg(m + '; in ' + `obj`)
 222         def needs_msgs(self, child_souplog):
 223                 self.msgs += child_souplog.msgs
 224                 child_souplog.msgs = [ ]
 225
 226 def soup_text(obj):
 227         str = ''.join(obj.findAll(text=True))
 228         return str.strip()
 229
 230 class SomethingSoupInfo(SoupLog):
 231         def __init__(self, kind, tail, max_age):
 232                 SoupLog.__init__(self)
 233                 html = fetcher.yoweb(kind, tail, max_age)
 234                 self._soup = BeautifulSoup(html,
 235                         convertEntities=BeautifulSoup.HTML_ENTITIES
 236                         )
 237
 238 #---------- scraper for pirate pages ----------
 239
 240 class PirateInfo(SomethingSoupInfo):
 241         # Public data members:
 242         #  pi.standings = { 'Treasure Haul': 'Able' ... }
 243         #  pi.name = name
 244         #  pi.crew = (id, name)
 245         #  pi.flag = (id, name)
 246         #  pi.msgs = [ 'message describing problem with scrape' ]
 247
 248         def __init__(self, pirate, max_age=300):
 249                 SomethingSoupInfo.__init__(self,
 250                         'pirate.wm?target=', pirate, max_age)
 251                 self.name = pirate
 252                 self._find_standings()
 253                 self.crew = self._find_crewflag('crew',
 254                         '^/yoweb/crew/info\\.wm')
 255                 self.flag = self._find_crewflag('flag',
 256                         '^/yoweb/flag/info\\.wm')
 257
 258         def _find_standings(self):
 259                 imgs = self._soup.findAll('img',
 260                         src=regexp.compile('/yoweb/images/stat.*'))
 261                 re = regexp.compile(
 262 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
 263                         )
 264                 standings = { }
 265
 266                 for skill in puzzles:
 267                         standings[skill] = [ ]
 268
 269                 skl = SoupLog()
 270
 271                 for img in imgs:
 272                         try: puzzle = img['alt']
 273                         except KeyError: continue
 274
 275                         if not puzzle in puzzles:
 276                                 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
 277                                 continue
 278                         key = img.findParent('td')
 279                         if key is None:
 280                                 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
 281                                 continue
 282                         valelem = key.findNextSibling('td')
 283                         if valelem is None:
 284                                 skl.soupm(key, 'puzzle missing sibling "%s"'
 285                                         % puzzle)
 286                                 continue
 287                         valstr = soup_text(valelem)
 288                         match = re.match(valstr)
 289                         if match is None:
 290                                 skl.soupm(key, ('puzzle "%s" unparseable'+
 291                                         ' standing "%s"') % (puzzle, valstr))
 292                                 continue
 293                         standing = match.group(match.lastindex)
 294                         standings[puzzle].append(standing)
 295
 296                 self.standings = { }
 297
 298                 for puzzle in puzzles:
 299                         sl = standings[puzzle]
 300                         if len(sl) > 1:
 301                                 skl.msg('puzzle "%s" multiple standings %s' %
 302                                                 (puzzle, `sl`))
 303                                 continue
 304                         if not sl:
 305                                 skl.msg('puzzle "%s" no standing found' % puzzle)
 306                                 continue
 307                         standing = sl[0]
 308                         for i in range(0, standing_limit):
 309                                 if standing == standingvals[i]:
 310                                         self.standings[puzzle] = i
 311                         if not puzzle in self.standings:
 312                                 skl.msg('puzzle "%s" unknown standing "%s"' %
 313                                         (puzzle, standing))
 314
 315                 all_standings_ok = True
 316                 for puzzle in puzzles:
 317                         if not puzzle in self.standings:
 318                                 self.needs_msgs(skl)
 319
 320         def _find_crewflag(self, cf, yoweb_re):
 321                 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
 322                 if len(things) != 1:
 323                         self.msg('zero or several %s id references found' % cf)
 324                         return None
 325                 thing = things[0]
 326                 id_re = '\\b%sid\\=(\\w+)$' % cf
 327                 id_haystack = thing['href']
 328                 match = regexp.compile(id_re).search(id_haystack)
 329                 if match is None:
 330                         self.soupm(thing, ('incomprehensible %s id ref'+
 331                                 ' (%s in %s)') % (cf, id_re, id_haystack))
 332                         return None
 333                 name = soup_text(thing)
 334                 return (match.group(1), name)
 335
 336         def __str__(self):
 337                 return `(self.crew, self.flag, self.standings, self.msgs)`
 338
 339 #---------- scraper for crew pages ----------
 340
 341 class CrewInfo(SomethingSoupInfo):
 342         # Public data members:
 343         #  ci.crew = [ ('Captain',        ['Pirate', ...]),
 344         #              ('Senior Officer', [...]),
 345         #               ... ]
 346         #  pi.msgs = [ 'message describing problem with scrape' ]
 347
 348         def __init__(self, crewid, max_age=300):
 349                 SomethingSoupInfo.__init__(self,
 350                         'crew/info.wm?crewid=', crewid, max_age)
 351                 self._find_crew()
 352
 353         def _find_crew(self):
 354                 self.crew = []
 355                 capts = self._soup.findAll('img',
 356                         src='/yoweb/images/crew-captain.png')
 357                 if len(capts) != 1:
 358                         self.msg('crew members: no. of captain images != 1')
 359                         return
 360                 tbl = capts[0]
 361                 while not tbl.find('a', href=pirate_ref_re):
 362                         tbl = tbl.findParent('table')
 363                         if not tbl:
 364                                 self.msg('crew members: cannot find table')
 365                                 return
 366                 current_rank_crew = None
 367                 crew_rank_re = regexp.compile('/yoweb/images/crew')
 368                 for row in tbl.contents:
 369                         # findAll(recurse=False)
 370                         if isinstance(row,basestring):
 371                                 continue
 372
 373                         is_rank = row.find('img', attrs={'src': crew_rank_re})
 374                         if is_rank:
 375                                 rank = soup_text(row)
 376                                 current_rank_crew = []
 377                                 self.crew.append((rank, current_rank_crew))
 378                                 continue
 379                         for cell in row.findAll('a', href=pirate_ref_re):
 380                                 if current_rank_crew is None:
 381                                         self.soupm(cell, 'crew members: crew'
 382                                                 ' before rank')
 383                                         continue
 384                                 current_rank_crew.append(soup_text(cell))
 385
 386         def __str__(self):
 387                 return `(self.crew, self.msgs)`
 388
 389 class FlagInfo(SomethingSoupInfo):
 390         # Public data members (after init):
 391         #
 392         #   name        #               string
 393         #
 394         #   relations[n] = (otherflagname, otherflagid, [stringfromyoweb],
 395         #               thisdeclaring, otherdeclaringmin, otherdeclaringmax)
 396         #               # where {this,other}declaring{,min,max} are:
 397         #               #       -1      {this,other} is declaring war
 398         #               #        0      {this,other} is not doing either
 399         #               #       +1      {this,other} is allying
 400         #   relation_byname[otherflagname] = relations[some_n]
 401         #   relation_byid[otherflagname] = relations[some_n]
 402         #
 403         #   islands[n] = (islandname, islandid)
 404         #
 405         def __init__(self, flagid, max_age=600):
 406                 SomethingSoupInfo.__init__(self,
 407                         'flag/info.wm?flagid=', flagid, max_age)
 408                 self._find_flag()
 409
 410         def _find_flag(self):
 411                 font2 = self._soup.find('font',{'size':'+2'})
 412                 self.name = font2.find('b').contents[0]
 413
 414                 self.relations = [ ]
 415                 self.relation_byname = { }
 416                 self.relation_byid = { }
 417                 self.islands = [ ]
 418
 419                 magnate = self._soup.find('img',{'src':
 420                         '/yoweb/images/repute-MAGNATE.png'})
 421                 warinfo = (magnate.findParent('table').findParent('tr').
 422                         findNextSibling('tr').findNext('td',{'align':'left'}))
 423
 424                 def warn(m):
 425                         print >>sys.stderr, 'WARNING: '+m
 426
 427                 def wi_warn(head, waritem):
 428                         warn('unknown warmap item: %s: %s' %
 429                                 (`head`, ``waritem``))
 430
 431                 def wihelp_item(waritem, thing):
 432                         if waritem.name == 'a':
 433                                 url = waritem.get('href', None)
 434                                 if url is None:
 435                                         return ('no url for '+thing,None,None)
 436                         else:
 437                                 hr = waritem.find('a',{'href':True})
 438                                 if not hr: return ('no a for '+thing,None,None)
 439                                 url = hr['href']
 440                         m = regexp.search('\?'+thing+'id=(\d+)$', url)
 441                         if not m: return ('no '+thing+'id',None,None)
 442                         tid = m.group(1)
 443                         tname = m.string
 444                         if tname is None:
 445                                 return (thing+' name not just string',None,None)
 446                         return (None,tid,tname)
 447
 448                 def wi_alwar(head, waritem, thisdecl, othermin, othermax):
 449                         (err,flagid,flagname) = wihelp_item(waritem,'flag')
 450                         if err: return err
 451                         rel = self.relation_byid.get(flagid, None)
 452                         if rel: return 'flag id twice!'
 453                         if flagname in self.relation_byname:
 454                                 return 'flag name twice!'
 455                         rel = (flagname,flagid,[], thisdecl,othermin,othermax)
 456                         self.relations.append(rel)
 457                         self.relation_byid[flagid] = rel
 458                         self.relation_byname[flagid] = rel
 459
 460                 def wi_isle(head, waritem):
 461                         (err,isleid,islename) = wihelp_item(waritem,'island')
 462                         if err: return err
 463                         self.islands.append((isleid,islename))
 464
 465                 warmap = {
 466                         'Allied with':                  (wi_alwar,+1,+1,+1),
 467                         'Declaring war against':        (wi_alwar,-1, 0,+1),
 468                         'At war with':                  (wi_alwar,-1,-1,-1),
 469                         'Trying to form an alliance with': (wi_alwar,+1,-1,0),
 470                         'Islands controlled by this flag': (wi_isle,),
 471                         }
 472
 473                 how = (wi_warn, None)
 474
 475                 for waritem in warinfo.contents:
 476                         debug('WARITEM '+``waritem``)
 477                         if isinstance(waritem, unicode):
 478                                 waritem = waritem.strip()
 479                                 if waritem: warn('unknown waritem '+``waritem``)
 480                                 continue
 481                         if waritem.name == 'br':
 482                                 continue
 483                         if waritem.name == 'b':
 484                                 head = ''.join(waritem.findAll(text=True))
 485                                 head = regexp.sub('\\s+', ' ', head).strip()
 486                                 head = head.rstrip(':')
 487                                 how = (head,) + warmap.get(head, (wi_warn,))
 488                                 continue
 489                         debug('WARHOW %s(%s, waritem, *%s)' %
 490                                 (how[1], `how[0]`, `how[2:]`))
 491                         bad = how[1](how[0], waritem, *how[2:])
 492                         if bad:
 493                                 warn('bad waritem %s: %s: %s' % (`how[0]`,
 494                                         bad, ``waritem``))
 495
 496         def __str__(self):
 497                 return `(self.name, self.islands, self.relations)`
 498
 499 #---------- scraper for ocean info incl. embargoes etc. ----------
 500
 501 class IslandInfo():
 502         def __init__(self, ocean, islename):
 503                 self.ocean = ocean
 504                 self.name = islename
 505         def collect(self):
 506                 pass
 507         def yppedia_dataf(self):
 508                 def q(x): return urllib.quote(x.replace(' ','_'))
 509                 url_rhs = q(self.name) + '_(' + q(self.ocean) + ')'
 510                 if opts.localhtml is None:
 511                         url = 'http://yppedia.puzzlepirates.com/' + url_rhs
 512                         debug('IslandInfo retrieving YPP '+url);
 513                         return urllib.urlopen(url)
 514                 else:
 515                         return file(opts.localhtml + '/' + url_rhs, 'r')
 516         def yoweb_url(self):
 517                 soup = BeautifulSoup(self.yppedia_dataf())
 518                 content = soup.find('div', attrs = {'id': 'content'})
 519                 yoweb_re = regexp.compile('^http://\w+\.puzzlepirates\.com/'+
 520                         'yoweb/island/info\.wm\?islandid=\d+$')
 521                 a = soup.find('a', attrs = { 'href': yoweb_re })
 522                 if a is None: return None
 523                 return a['href']
 524         def ruling_flag_id(self):
 525                 yo = self.yoweb_url()
 526                 if yo is None: return None
 527                 dataf = fetcher.fetch(yo, 600)
 528                 soup = BeautifulSoup(dataf)
 529                 ruler_re = regexp.compile('http://\w+\.puzzlepirates\.com/'+
 530                         'yoweb/flag/info\.wm\?flagid=(\d+)$')
 531                 ruler = soup.find('a', attrs = { 'href': ruler_re })
 532                 if not ruler: return None
 533                 m = ruler_re.find(ruler['href'])
 534                 return m.group(1)
 535
 536 class OceanInfo():
 537         # Public data attributes (valid after collect()):
 538         #   oi.islands[islename] = IslandInfo(...)
 539         #   oi.arches[archname][islename] = IslandInfo(...)
 540         def __init__(self):
 541                 self.isleclass = IslandInfo
 542                 self.ocean = fetcher.ocean.lower().capitalize()
 543         def collect(self):
 544                 cmdl = ['./yppedia-ocean-scraper']
 545                 if opts.localhtml is not None:
 546                         cmdl += ['--local-html-dir',opts.localhtml]
 547                 cmdl += [self.ocean]
 548                 debug('OceanInfo collect running ' + `cmdl`)
 549                 oscraper = subprocess.Popen(
 550                         cmdl,
 551                         stdout = subprocess.PIPE,
 552                         cwd = yppsc_dir()+'/yarrg',
 553                         shell=False, stderr=None,
 554                         )
 555                 h = oscraper.stdout.readline()
 556                 debug('OceanInfo collect h '+`h`)
 557                 assert(regexp.match('^ocean ', h))
 558                 arch_re = regexp.compile('^ (\S.*)')
 559                 island_re = regexp.compile('^  (\S.*)')
 560
 561                 self.islands = { }
 562                 self.arches = { }
 563                 archname = None
 564
 565                 for l in oscraper.stdout:
 566                         debug('OceanInfo collect l '+`l`)
 567                         l = l.rstrip('\n')
 568                         m = island_re.match(l)
 569                         if m:
 570                                 assert(archname is not None)
 571                                 islename = m.group(1)
 572                                 isle = self.isleclass(self.ocean, islename)
 573                                 isle.arch = archname
 574                                 self.islands[islename] = isle
 575                                 self.arches[archname][islename] = isle
 576                                 continue
 577                         m = arch_re.match(l)
 578                         if m:
 579                                 archname = m.group(1)
 580                                 assert(archname not in self.arches)
 581                                 self.arches[archname] = { }
 582                                 continue
 583                         assert(False)
 584                 oscraper.wait()
 585                 assert(oscraper.returncode == 0)
 586
 587 #---------- pretty-printer for tables of pirate puzzle standings ----------
 588
 589 class StandingsTable:
 590         def __init__(self, f, use_puzzles=None, col_width=6, gap_every=5):
 591                 if use_puzzles is None:
 592                         if opts.ship_duty:
 593                                 use_puzzles=duty_puzzles
 594                         else:
 595                                 use_puzzles=puzzles
 596                 self._puzzles = use_puzzles
 597                 self.f = f
 598                 self._cw = col_width-1
 599                 self._gap_every = gap_every
 600                 self._linecount = 0
 601                 self._o = f.write
 602
 603         def _nl(self): self._o('\n')
 604
 605         def _pline(self, lhs, puzstrs, extra):
 606                 if (self._linecount > 0
 607                     and self._gap_every is not None
 608                     and not (self._linecount % self._gap_every)):
 609                         self._nl()
 610                 self._o('%-*s' % (max(max_pirate_namelen+1, 15), lhs))
 611                 for v in puzstrs:
 612                         self._o(' %-*.*s' % (self._cw,self._cw, v))
 613                 if extra:
 614                         self._o(' ' + extra)
 615                 self._nl()
 616                 self._linecount += 1
 617
 618         def _puzstr(self, pi, puzzle):
 619                 if not isinstance(puzzle,list): puzzle = [puzzle]
 620                 try: standing = max([pi.standings[p] for p in puzzle])
 621                 except KeyError: return '?'
 622                 if not standing: return ''
 623                 s = ''
 624                 if self._cw > 4:
 625                         c1 = standingvals[standing][0]
 626                         if standing < 3: c1 = c1.lower() # 3 = Master
 627                         s += `standing`
 628                 if self._cw > 5:
 629                         s += ' '
 630                 s += '*' * (standing / 2)
 631                 s += '+' * (standing % 2)
 632                 return s
 633
 634         def headings(self, lhs='', rhs=None):
 635                 def puzn_redact(name):
 636                         if isinstance(name,list):
 637                                 return '/'.join(
 638                                         ["%.*s" % (self._cw/2, puzn_redact(n))
 639                                          for n in name])
 640                         spc = name.find(' ')
 641                         if spc < 0: return name
 642                         return name[0:min(4,spc)] + name[spc+1:]
 643                 self._linecount = -2
 644                 self._pline(lhs, map(puzn_redact, self._puzzles), rhs)
 645                 self._linecount = 0
 646         def literalline(self, line):
 647                 self._o(line)
 648                 self._nl()
 649                 self._linecount = 0
 650         def pirate_dummy(self, name, standingstring, extra=None):
 651                 standings = standingstring * len(self._puzzles)
 652                 self._pline(' '+name, standings, extra)
 653         def pirate(self, pi, extra=None):
 654                 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
 655                 self._pline(' '+pi.name, puzstrs, extra)
 656
 657
 658 #---------- chat log parser ----------
 659
 660 class PirateAboard:
 661         # This is essentially a transparent, dumb, data class.
 662         #  pa.v                 may be None
 663         #  pa.name
 664         #  pa.last_time
 665         #  pa.last_event
 666         #  pa.gunner
 667         #  pa.last_chat_time
 668         #  pa.last_chat_chan
 669         #  pa.pi
 670
 671         # Also used for jobbing applicants:
 672         #               happens when                    expires (to "-")
 673         #   -            disembark, leaves crew          no
 674         #   aboard       evidence of them being aboard   no
 675         #   applied      "has applied for the job"       120s, configurable
 676         #   ashore       "has taken a job"               30min, configurable
 677         #   declined     "declined the job offer"        30s, configurable
 678         #   invited      "has been invited to job"       120s, configurable
 679         #
 680         #  pa.jobber    None, 'ashore', 'applied', 'invited', 'declined'
 681         #  pa.expires   expiry time time
 682
 683         def __init__(pa, pn, v, time, event):
 684                 pa.name = pn
 685                 pa.v = v
 686                 pa.last_time = time
 687                 pa.last_event = event
 688                 pa.last_chat_time = None
 689                 pa.last_chat_chan = None
 690                 pa.gunner = False
 691                 pa.pi = None
 692                 pa.jobber = None
 693                 pa.expires = None
 694
 695         def pirate_info(pa):
 696                 now = time.time()
 697                 if pa.pi:
 698                         age = now - pa.pi_fetched
 699                         guide = random.randint(120,240)
 700                         if age <= guide:
 701                                 return pa.pi
 702                         debug('PirateAboard refresh %d > %d  %s' % (
 703                                 age, guide, pa.name))
 704                         imaginary = [2,4]
 705                 else:
 706                         imaginary = [1]
 707                 wait = fetcher.need_wait(now, imaginary)
 708                 if wait:
 709                         debug('PirateAboard fetcher not ready %d' % wait)
 710                         return pa.pi
 711                 pa.pi = PirateInfo(pa.name, 600)
 712                 pa.pi_fetched = now
 713                 return pa.pi
 714
 715 class ChatLogTracker:
 716         # This is quite complex so we make it opaque.  Use the
 717         # official invokers, accessors etc.
 718
 719         def __init__(self, myself_pi, logfn):
 720                 self._pl = {}   # self._pl['Pirate'] =
 721                 self._vl = {}   #   self._vl['Vessel']['Pirate'] = PirateAboard
 722                                 # self._vl['Vessel']['#lastinfo']
 723                                 # self._vl['Vessel']['#name']
 724                                 # self._v = self._vl[self._vessel]
 725                 self._date = None
 726                 self._myself = myself_pi
 727                 self._lbuf = ''
 728                 self._f = file(logfn)
 729                 flen = os.fstat(self._f.fileno()).st_size
 730                 max_backlog = 500000
 731                 if flen > max_backlog:
 732                         startpos = flen - max_backlog
 733                         self._f.seek(startpos)
 734                         self._f.readline()
 735                 self._progress = [0, flen - self._f.tell()]
 736                 self._disembark_myself()
 737                 self._need_redisplay = False
 738                 self._lastvessel = None
 739
 740         def _disembark_myself(self):
 741                 self._v = None
 742                 self._vessel = None
 743                 self.force_redisplay()
 744
 745         def force_redisplay(self):
 746                 self._need_redisplay = True
 747
 748         def _vessel_updated(self, v, timestamp):
 749                 if v is None: return
 750                 v['#lastinfo'] = timestamp
 751                 self.force_redisplay()
 752
 753         def _onboard_event(self,v,timestamp,pirate,event,jobber=None):
 754                 pa = self._pl.get(pirate, None)
 755                 if pa is not None and pa.v is v:
 756                         pa.last_time = timestamp
 757                         pa.last_event = event
 758                 else:
 759                         if pa is not None and pa.v is not None:
 760                                 del pa.v[pirate]
 761                         pa = PirateAboard(pirate, v, timestamp, event)
 762                         self._pl[pirate] = pa
 763                         if v is not None: v[pirate] = pa
 764                 pa.jobber = jobber
 765
 766                 if jobber is None: timeout = None
 767                 else: timeout = getattr(opts, 'timeout_'+jobber)
 768                 if timeout is None: pa.expires = None
 769                 else: pa.expires = timestamp + timeout
 770                 self._vessel_updated(v, timestamp)
 771                 return pa
 772
 773         def _expire_jobbers(self, now):
 774                 for pa in self._pl.values():
 775                         if pa.expires is None: continue
 776                         if pa.expires >= now: continue
 777                         v = pa.v
 778                         del self._pl[pa.name]
 779                         if v is not None: del v[pa.name]
 780                         self.force_redisplay()
 781
 782         def _trash_vessel(self, v):
 783                 for pn in v:
 784                         if pn.startswith('#'): continue
 785                         del self._pl[pn]
 786                 vn = v['#name']
 787                 del self._vl[vn]
 788                 if v is self._v: self._disembark_myself()
 789                 self.force_redisplay()
 790
 791         def _vessel_stale(self, v, timestamp):
 792                 return timestamp - v['#lastinfo'] > opts.ship_reboard_clearout
 793
 794         def _vessel_check_expire(self, v, timestamp):
 795                 if not self._vessel_stale(v, timestamp):
 796                         return v
 797                 self._debug_line_disposition(timestamp,'',
 798                         'stale-reset ' + v['#name'])
 799                 self._trash_vessel(v)
 800                 return None
 801
 802         def expire_garbage(self, timestamp):
 803                 for v in self._vl.values():
 804                         self._vessel_check_expire(v, timestamp)
 805
 806         def _vessel_lookup(self, vn, timestamp, dml=[], create=False):
 807                 v = self._vl.get(vn, None)
 808                 if v is not None:
 809                         v = self._vessel_check_expire(v, timestamp)
 810                 if v is not None:
 811                         dml.append('found')
 812                         return v
 813                 if not create:
 814                         dml.append('no')
 815                 dml.append('new')
 816                 self._vl[vn] = v = { '#name': vn }
 817                 self._vessel_updated(v, timestamp)
 818                 return v
 819
 820         def _find_matching_vessel(self, pattern, timestamp, cmdr,
 821                                         dml=[], create=False):
 822                 # use when a commander pirate `cmdr' specified a vessel
 823                 #  by name `pattern' (either may be None)
 824                 # if create is true, will create the vessel
 825                 #  record if an exact name is specified
 826
 827                 if (pattern is not None and
 828                     not '*' in pattern
 829                     and len(pattern.split(' ')) == 2):
 830                         vn = pattern.title()
 831                         dml.append('exact')
 832                         return self._vessel_lookup(
 833                                 vn, timestamp, dml=dml, create=create)
 834
 835                 if pattern is None:
 836                         pattern_check = lambda vn: True
 837                 else:
 838                         re = '(?:.* )?%s$' % pattern.lower().replace('*','.+')
 839                         pattern_check = regexp.compile(re, regexp.I).match
 840
 841                 tries = []
 842
 843                 cmdr_pa = self._pl.get(cmdr, None)
 844                 if cmdr_pa: tries.append((cmdr_pa.v, 'cmdr'))
 845
 846                 tries.append((self._v, 'here'))
 847                 tried_vns = []
 848
 849                 for (v, dm) in tries:
 850                         if v is None: dml.append(dm+'?'); continue
 851
 852                         vn = v['#name']
 853                         if not pattern_check(vn):
 854                                 tried_vns.append(vn)
 855                                 dml.append(dm+'#')
 856                                 continue
 857
 858                         dml.append(dm+'!')
 859                         return v
 860
 861                 if pattern is not None and '*' in pattern:
 862                         search = [
 863                                 (vn,v)
 864                                 for (vn,v) in self._vl.iteritems()
 865                                 if not self._vessel_stale(v, timestamp)
 866                                 if pattern_check(vn)
 867                                 ]
 868                         #debug('CLT-RE /%s/ wanted (%s) searched (%s)' % (
 869                         #       re,
 870                         #       '/'.join(tried_vns),
 871                         #       '/'.join([vn for (vn,v) in search])))
 872
 873                         if len(search)==1:
 874                                 dml.append('one')
 875                                 return search[0][1]
 876                         elif search:
 877                                 dml.append('many')
 878                         else:
 879                                 dml.append('none')
 880
 881         def _debug_line_disposition(self,timestamp,l,m):
 882                 debug('CLT %13s %-40s %s' % (timestamp,m,l))
 883
 884         def _rm_crew_l(self,re,l):
 885                 m = regexp.match(re,l)
 886                 if m and m.group(2) == self._myself.crew[1]:
 887                         return m.group(1)
 888                 else:
 889                         return None
 890
 891         def local_command(self, metacmd):
 892                 # returns None if all went well, or problem message
 893                 return self._command(self._myself.name, metacmd,
 894                         "local", time.time(),
 895                         (lambda m: debug('CMD %s' % metacmd)))
 896
 897         def _command(self, cmdr, metacmd, chan, timestamp, d):
 898                 # returns None if all went well, or problem message
 899                 metacmd = regexp.sub('\\s+', ' ', metacmd).strip()
 900                 m2 = regexp.match(
 901                     '/([adj]) (?:([A-Za-z* ]+)\\s*:)?([A-Za-z ]+)$',
 902                     metacmd)
 903                 if not m2: return "unknown syntax or command"
 904
 905                 (cmd, pattern, targets) = m2.groups()
 906                 dml = ['cmd', chan, cmd]
 907
 908                 if cmd == 'a': each = self._onboard_event
 909                 elif cmd == 'd': each = disembark
 910                 else: each = lambda *l: self._onboard_event(*l,
 911                                 **{'jobber':'applied'})
 912
 913                 if cmdr == self._myself.name:
 914                         dml.append('self')
 915                         how = 'cmd: %s' % cmd
 916                 else:
 917                         dml.append('other')
 918                         how = 'cmd: %s %s' % (cmd,cmdr)
 919
 920                 if cmd == 'j':
 921                         if pattern is not None:
 922                                 return "/j command does not take a vessel"
 923                         v = None
 924                 else:
 925                         v = self._find_matching_vessel(
 926                                 pattern, timestamp, cmdr,
 927                                 dml, create=True)
 928
 929                 if cmd == 'j' or v is not None:
 930                         targets = targets.strip().split(' ')
 931                         dml.append(`len(targets)`)
 932                         for target in targets:
 933                                 each(v, timestamp, target.title(), how)
 934                         self._vessel_updated(v, timestamp)
 935
 936                 dm = ' '.join(dml)
 937                 return d(dm)
 938
 939                 return None
 940
 941         def chatline(self,l):
 942                 rm = lambda re: regexp.match(re,l)
 943                 d = lambda m: self._debug_line_disposition(timestamp,l,m)
 944                 rm_crew = lambda re: self._rm_crew_l(re,l)
 945                 timestamp = None
 946
 947                 m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$')
 948                 if m:
 949                         self._date = [int(x) for x in m.groups()]
 950                         self._previous_timestamp = None
 951                         return d('date '+`self._date`)
 952
 953                 if self._date is None:
 954                         return d('date unset')
 955
 956                 m = rm('\\[(\d\d):(\d\d):(\d\d)\\] ')
 957                 if not m:
 958                         return d('no timestamp')
 959
 960                 while True:
 961                         time_tuple = (self._date +
 962                                       [int(x) for x in m.groups()] +
 963                                       [-1,-1,-1])
 964                         timestamp = time.mktime(time_tuple)
 965                         if timestamp >= self._previous_timestamp: break
 966                         self._date[2] += 1
 967                         self._debug_line_disposition(timestamp,'',
 968                                 'new date '+`self._date`)
 969
 970                 self._previous_timestamp = timestamp
 971
 972                 l = l[l.find(' ')+1:]
 973
 974                 def ob_x(pirate,event):
 975                         return self._onboard_event(
 976                                         self._v, timestamp, pirate, event)
 977                 def ob1(did): ob_x(m.group(1), did); return d(did)
 978                 def oba(did): return ob1('%s %s' % (did, m.group(2)))
 979
 980                 def jb(pirate,jobber):
 981                         return self._onboard_event(
 982                                 None, timestamp, pirate,
 983                                 ("jobber %s" % jobber),
 984                                 jobber=jobber
 985                                 )
 986
 987                 def disembark(v, timestamp, pirate, event):
 988                         self._onboard_event(
 989                                         v, timestamp, pirate, 'leaving '+event)
 990                         del v[pirate]
 991                         del self._pl[pirate]
 992
 993                 def disembark_me(why):
 994                         self._disembark_myself()
 995                         return d('disembark-me '+why)
 996
 997                 m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
 998                 if m:
 999                         dm = ['boarding']
1000                         pn = self._myself.name
1001                         vn = m.group(1)
1002                         v = self._vessel_lookup(vn, timestamp, dm, create=True)
1003                         self._lastvessel = self._vessel = vn
1004                         self._v = v
1005                         ob_x(pn, 'we boarded')
1006                         self.expire_garbage(timestamp)
1007                         return d(' '.join(dm))
1008
1009                 if self._v is None:
1010                         return d('no vessel')
1011
1012                 m = rm('(\\w+) has come aboard\\.$')
1013                 if m: return ob1('boarded');
1014
1015                 m = rm('You have ordered (\\w+) to do some (\\S.*\\S)\\.$')
1016                 if m:
1017                         (who,what) = m.groups()
1018                         pa = ob_x(who,'ord '+what)
1019                         if what == 'Gunning':
1020                                 pa.gunner = True
1021                         return d('duty order')
1022
1023                 m = rm('(\\w+) abandoned a (\\S.*\\S) station\\.$')
1024                 if m: oba('stopped'); return d("end")
1025
1026                 def chat_core(speaker, chan):
1027                         try: pa = self._pl[speaker]
1028                         except KeyError: return 'mystery'
1029                         if pa.v is not None and pa.v is not self._v:
1030                                 return 'elsewhere'
1031                         pa.last_chat_time = timestamp
1032                         pa.last_chat_chan = chan
1033                         self.force_redisplay()
1034                         return 'here'
1035
1036                 def chat(chan):
1037                         speaker = m.group(1)
1038                         dm = chat_core(speaker, chan)
1039                         return d('chat %s %s' % (chan, dm))
1040
1041                 def chat_metacmd(chan):
1042                         (cmdr, metacmd) = m.groups()
1043                         whynot = self._command(
1044                                 cmdr, metacmd, chan, timestamp, d)
1045                         if whynot is not None:
1046                                 return chat(chan)
1047                         else:
1048                                 chat_core(cmdr, 'cmd '+chan)
1049
1050                 m = rm('(\\w+) (?:issued an order|ordered everyone) "')
1051                 if m: return ob1('general order');
1052
1053                 m = rm('(\\w+) says, "')
1054                 if m: return chat('public')
1055
1056                 m = rm('(\\w+) tells ye, "')
1057                 if m: return chat('private')
1058
1059                 m = rm('Ye told (\\w+), "(.*)"$')
1060                 if m: return chat_metacmd('private')
1061
1062                 m = rm('(\\w+) flag officer chats, "')
1063                 if m: return chat('flag officer')
1064
1065                 m = rm('(\\w+) officer chats, "(.*)"$')
1066                 if m: return chat_metacmd('officer')
1067
1068                 m = rm('Ye accepted the offer to job with ')
1069                 if m: return disembark_me('jobbing')
1070
1071                 m = rm('Ye hop on the ferry and are whisked away ')
1072                 if m: return disembark_me('ferry')
1073
1074                 m = rm('Whisking away to yer home on the magical winds')
1075                 if m: return disembark_me('home')
1076
1077                 m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
1078                 if m:
1079                         pl = m.group(1).split(', ')
1080                         if not self._myself.name in pl:
1081                                 return d('lost melee')
1082                         for pn in pl:
1083                                 if ' ' in pn: continue
1084                                 ob_x(pn,'won melee')
1085                         return d('won melee')
1086
1087                 m = rm('(\\w+) is eliminated\\!')
1088                 if m: return ob1('eliminated in fray');
1089
1090                 m = rm('(\\w+) has driven \w+ from the ship\\!')
1091                 if m: return ob1('boarder repelled');
1092
1093                 m = rm('\w+ has bested (\\w+), and turns'+
1094                         ' to the rest of the ship\\.')
1095                 if m: return ob1('boarder unrepelled');
1096
1097                 pirate = rm_crew("(\\w+) has taken a job with '(.*)'\\.")
1098                 if pirate: return jb(pirate, 'ashore')
1099
1100                 pirate = rm_crew("(\\w+) has left '(.*)'\\.")
1101                 if pirate:
1102                         disembark(self._v, timestamp, pirate, 'left crew')
1103                         return d('left crew')
1104
1105                 m = rm('(\w+) has applied for the posted job\.')
1106                 if m: return jb(m.group(1), 'applied')
1107
1108                 pirate= rm_crew("(\\w+) has been invited to job for '(.*)'\\.")
1109                 if pirate: return jb(pirate, 'invited')
1110
1111                 pirate = rm_crew("(\\w+) declined the job offer for '(.*)'\\.")
1112                 if pirate: return jb(pirate, 'declined')
1113
1114                 m = rm('(\\w+) has left the vessel\.')
1115                 if m:
1116                         pirate = m.group(1)
1117                         disembark(self._v, timestamp, pirate, 'disembarked')
1118                         return d('disembarked')
1119
1120                 return d('not-matched')
1121
1122         def _str_pa(self, pn, pa):
1123                 assert self._pl[pn] == pa
1124                 s = ' '*20 + "%s %-*s %13s %-30s %13s %-20s %13s" % (
1125                         (' ','G')[pa.gunner],
1126                         max_pirate_namelen, pn,
1127                         pa.last_time, pa.last_event,
1128                         pa.last_chat_time, pa.last_chat_chan,
1129                         pa.jobber)
1130                 if pa.expires is not None:
1131                         s += " %-5d" % (pa.expires - pa.last_time)
1132                 s += "\n"
1133                 return s
1134
1135         def _str_vessel(self, vn, v):
1136                 s = ' vessel %s\n' % vn
1137                 s += ' '*20 + "%-*s   %13s\n" % (
1138                                 max_pirate_namelen, '#lastinfo',
1139                                 v['#lastinfo'])
1140                 assert v['#name'] == vn
1141                 for pn in sorted(v.keys()):
1142                         if pn.startswith('#'): continue
1143                         pa = v[pn]
1144                         assert pa.v == v
1145                         s += self._str_pa(pn,pa)
1146                 return s
1147
1148         def __str__(self):
1149                 s = '''<ChatLogTracker
1150  myself %s
1151  vessel %s
1152 '''                     % (self._myself.name, self._vessel)
1153                 assert ((self._v is None and self._vessel is None) or
1154                         (self._v is self._vl[self._vessel]))
1155                 if self._vessel is not None:
1156                         s += self._str_vessel(self._vessel, self._v)
1157                 for vn in sorted(self._vl.keys()):
1158                         if vn == self._vessel: continue
1159                         s += self._str_vessel(vn, self._vl[vn])
1160                 s += " elsewhere\n"
1161                 for p in self._pl:
1162                         pa = self._pl[p]
1163                         if pa.v is not None:
1164                                 assert pa.v[p] is pa
1165                                 assert pa.v in self._vl.values()
1166                         else:
1167                                 s += self._str_pa(pa.name, pa)
1168                 s += '>\n'
1169                 return s
1170
1171         def catchup(self, progress=None):
1172                 while True:
1173                         more = self._f.readline()
1174                         if not more: break
1175
1176                         self._progress[0] += len(more)
1177                         if progress: progress.progress(*self._progress)
1178
1179                         self._lbuf += more
1180                         if self._lbuf.endswith('\n'):
1181                                 self.chatline(self._lbuf.rstrip())
1182                                 self._lbuf = ''
1183                                 if opts.debug >= 2:
1184                                         debug(self.__str__())
1185                 self._expire_jobbers(time.time())
1186
1187                 if progress: progress.caughtup()
1188
1189         def changed(self):
1190                 rv = self._need_redisplay
1191                 self._need_redisplay = False
1192                 return rv
1193         def myname(self):
1194                 # returns our pirate name
1195                 return self._myself.name
1196         def vesselname(self):
1197                 # returns the vessel name we're aboard or None
1198                 return self._vessel
1199         def lastvesselname(self):
1200                 # returns the last vessel name we were aboard or None
1201                 return self._lastvessel
1202         def aboard(self, vesselname=True):
1203                 # returns a list of PirateAboard the vessel
1204                 #  sorted by pirate name
1205                 #  you can pass this None and you'll get []
1206                 #  or True for the current vessel (which is the default)
1207                 #  the returned value is a fresh list of persistent
1208                 #  PirateAboard objects
1209                 if vesselname is True: v = self._v
1210                 else: v = self._vl.get(vesselname.title())
1211                 if v is None: return []
1212                 return [ v[pn]
1213                          for pn in sorted(v.keys())
1214                          if not pn.startswith('#') ]
1215         def jobbers(self):
1216                 # returns a the jobbers' PirateAboards,
1217                 # sorted by jobber class and reverse of expiry time
1218                 l = [ pa
1219                       for pa in self._pl.values()
1220                       if pa.jobber is not None
1221                     ]
1222                 def compar_key(pa):
1223                         return (pa.jobber, -pa.expires)
1224                 l.sort(key = compar_key)
1225                 return l
1226
1227 #---------- implementations of actual operation modes ----------
1228
1229 def do_pirate(pirates, bu):
1230         print '{'
1231         for pirate in pirates:
1232                 info = PirateInfo(pirate)
1233                 print '%s: %s,' % (`pirate`, info)
1234         print '}'
1235
1236 def prep_crew_of(args, bu, max_age=300):
1237         if len(args) != 1: bu('crew-of takes one pirate name')
1238         pi = PirateInfo(args[0], max_age)
1239         if pi.crew is None: return None
1240         return CrewInfo(pi.crew[0], max_age)
1241
1242 def do_crew_of(args, bu):
1243         ci = prep_crew_of(args, bu)
1244         print ci
1245
1246 def do_flag_of(args, bu):
1247         if len(args) != 1: bu('flag-of takes one pirate name')
1248         max_age = 300
1249         pi = PirateInfo(args[0], max_age)
1250         if pi.flag is None: fi = None
1251         else: fi = FlagInfo(pi.flag[0], max_age)
1252         print `fi`
1253
1254 def do_standings_crew_of(args, bu):
1255         ci = prep_crew_of(args, bu, 60)
1256         tab = StandingsTable(sys.stdout)
1257         tab.headings()
1258         for (rank, members) in ci.crew:
1259                 if not members: continue
1260                 tab.literalline('')
1261                 tab.literalline('%s:' % rank)
1262                 for p in members:
1263                         pi = PirateInfo(p, random.randint(900,1800))
1264                         tab.pirate(pi)
1265
1266 def do_ocean(args, bu):
1267         if (len(args)): bu('ocean takes no further arguments')
1268         fetcher.default_ocean()
1269         oi = OceanInfo()
1270         oi.collect()
1271         for islename in sorted(oi.islands.keys()):
1272                 isle = oi.islands[islename]
1273                 yoweb_url = isle.yoweb_url()
1274                 print " %s -- %s" % (islename, yoweb_url)
1275
1276 #----- modes which use the chat log parser are quite complex -----
1277
1278 class ProgressPrintPercentage:
1279         def __init__(self, f=sys.stdout):
1280                 self._f = f
1281         def progress_string(self,done,total):
1282                 return "scan chat logs %3d%%\r" % ((done*100) / total)
1283         def progress(self,*a):
1284                 self._f.write(self.progress_string(*a))
1285                 self._f.flush()
1286         def show_init(self, pirate, ocean):
1287                 print >>self._f, 'Starting up, %s on the %s ocean' % (
1288                         pirate, ocean)
1289         def caughtup(self):
1290                 self._f.write('                   \r')
1291                 self._f.flush()
1292
1293 def prep_chat_log(args, bu,
1294                 progress=ProgressPrintPercentage(),
1295                 max_myself_age=3600):
1296         if len(args) != 1: bu('this action takes only chat log filename')
1297         logfn = args[0]
1298         logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_'
1299         match = regexp.match(logfn_re, logfn)
1300         if not match: bu('chat log filename is not in expected format')
1301         (pirate, ocean) = match.groups()
1302         fetcher.default_ocean(ocean)
1303
1304         progress.show_init(pirate, fetcher.ocean)
1305         myself = PirateInfo(pirate,max_myself_age)
1306         track = ChatLogTracker(myself, logfn)
1307
1308         opts.debug -= 2
1309         track.catchup(progress)
1310         opts.debug += 2
1311
1312         track.force_redisplay()
1313
1314         return (myself, track)
1315
1316 def do_track_chat_log(args, bu):
1317         (myself, track) = prep_chat_log(args, bu)
1318         while True:
1319                 track.catchup()
1320                 if track.changed():
1321                         print track
1322                 sleep(0.5 + 0.5 * random.random())
1323
1324 #----- ship management aid -----
1325
1326 class Display_dumb(ProgressPrintPercentage):
1327         def __init__(self):
1328                 ProgressPrintPercentage.__init__(self)
1329         def show(self, s):
1330                 print '\n\n', s;
1331         def realstart(self):
1332                 pass
1333
1334 class Display_overwrite(ProgressPrintPercentage):
1335         def __init__(self):
1336                 ProgressPrintPercentage.__init__(self)
1337
1338                 null = file('/dev/null','w')
1339                 curses.setupterm(fd=null.fileno())
1340
1341                 self._clear = curses.tigetstr('clear')
1342                 if not self._clear:
1343                         self._debug('missing clear!')
1344                         self.show = Display_dumb.show
1345                         return
1346
1347                 self._t = {'el':'', 'ed':''}
1348                 if not self._init_sophisticated():
1349                         for k in self._t.keys(): self._t[k] = ''
1350                         self._t['ho'] = self._clear
1351
1352         def _debug(self,m): debug('display overwrite: '+m)
1353
1354         def _init_sophisticated(self):
1355                 for k in self._t.keys():
1356                         s = curses.tigetstr(k)
1357                         self._t[k] = s
1358                 self._t['ho'] = curses.tigetstr('ho')
1359                 if not self._t['ho']:
1360                         cup = curses.tigetstr('cup')
1361                         self._t['ho'] = curses.tparm(cup,0,0)
1362                 missing = [k for k in self._t.keys() if not self._t[k]]
1363                 if missing:
1364                         self.debug('missing '+(' '.join(missing)))
1365                         return 0
1366                 return 1
1367
1368         def show(self, s):
1369                 w = sys.stdout.write
1370                 def wti(k): w(self._t[k])
1371
1372                 wti('ho')
1373                 nl = ''
1374                 for l in s.rstrip().split('\n'):
1375                         w(nl)
1376                         w(l)
1377                         wti('el')
1378                         nl = '\r\n'
1379                 wti('ed')
1380                 w(' ')
1381                 sys.stdout.flush()
1382
1383         def realstart(self):
1384                 sys.stdout.write(self._clear)
1385                 sys.stdout.flush()
1386
1387
1388 def do_ship_aid(args, bu):
1389         if opts.ship_duty is None: opts.ship_duty = True
1390
1391         displayer = globals()['Display_'+opts.display]()
1392
1393         (myself, track) = prep_chat_log(args, bu, progress=displayer)
1394
1395         displayer.realstart()
1396
1397         if os.isatty(0): kr_create = KeystrokeReader
1398         else: kr_create = DummyKeystrokeReader
1399
1400         try:
1401                 kreader = kr_create(0, 10)
1402                 ship_aid_core(myself, track, displayer, kreader)
1403         finally:
1404                 kreader.stop()
1405                 print '\n'
1406
1407 class KeyBasedSorter:
1408         def compar_key_pa(self, pa):
1409                 pi = pa.pirate_info()
1410                 if pi is None: return None
1411                 return self.compar_key(pi)
1412         def lsort_pa(self, l):
1413                 l.sort(key = self.compar_key_pa)
1414
1415 class NameSorter(KeyBasedSorter):
1416         def compar_key(self, pi): return pi.name
1417         def desc(self): return 'name'
1418
1419 class SkillSorter(NameSorter):
1420         def __init__(self, relevant):
1421                 self._want = frozenset(relevant.split('/'))
1422                 self._avoid = set()
1423                 for p in core_duty_puzzles:
1424                         if isinstance(p,basestring): self._avoid.add(p)
1425                         else: self._avoid |= set(p)
1426                 self._avoid -= self._want
1427                 self._desc = '%s' % relevant
1428
1429         def desc(self): return self._desc
1430
1431         def compar_key(self, pi):
1432                 best_want = max([
1433                         pi.standings.get(puz,-1)
1434                         for puz in self._want
1435                         ])
1436                 best_avoid = [
1437                         -pi.standings.get(puz,standing_limit)
1438                         for puz in self._avoid
1439                         ]
1440                 best_avoid.sort()
1441                 def negate(x): return -x
1442                 debug('compar_key %s bw=%s ba=%s' % (pi.name, `best_want`,
1443                         `best_avoid`))
1444                 return (-best_want, map(negate, best_avoid), pi.name)
1445
1446 def ship_aid_core(myself, track, displayer, kreader):
1447
1448         def find_vessel():
1449                 vn = track.vesselname()
1450                 if vn: return (vn, " on board the %s" % vn)
1451                 vn = track.lastvesselname()
1452                 if vn: return (vn, " ashore from the %s" % vn)
1453                 return (None, " not on a vessel")
1454
1455         def timeevent(t,e):
1456                 if t is None: return ' ' * 22
1457                 return " %-4s %-16s" % (format_time_interval(now - t),e)
1458
1459         displayer.show(track.myname() + find_vessel()[1] + '...')
1460
1461         rotate_nya = '/-\\'
1462
1463         sort = NameSorter()
1464         clicmd = None
1465         clierr = None
1466         cliexec = None
1467
1468         while True:
1469                 track.catchup()
1470                 now = time.time()
1471
1472                 (vn, vs) = find_vessel()
1473
1474                 s = ''
1475                 if cliexec is not None:
1476                         s += '...'
1477                 elif clierr is not None:
1478                         s += 'Error: '+clierr
1479                 elif clicmd is not None:
1480                         s += '/' + clicmd
1481                 else:
1482                         s = track.myname() + vs
1483                         s += " at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
1484                         s += kreader.info()
1485                 s += '\n'
1486
1487                 tbl_s = StringIO()
1488                 tbl = StandingsTable(tbl_s)
1489
1490                 aboard = track.aboard(vn)
1491                 sort.lsort_pa(aboard)
1492
1493                 jobbers = track.jobbers()
1494
1495                 if track.vesselname(): howmany = 'aboard: %2d' % len(aboard)
1496                 else: howmany = ''
1497
1498                 tbl.headings(howmany, '  sorted by '+sort.desc())
1499
1500                 last_jobber = None
1501
1502                 for pa in aboard + jobbers:
1503                         if pa.jobber != last_jobber:
1504                                 last_jobber = pa.jobber
1505                                 tbl.literalline('')
1506                                 tbl.literalline('jobbers '+last_jobber)
1507
1508                         pi = pa.pirate_info()
1509
1510                         xs = ''
1511                         if pa.gunner: xs += 'G '
1512                         else: xs += '  '
1513                         xs += timeevent(pa.last_time, pa.last_event)
1514                         xs += timeevent(pa.last_chat_time, pa.last_chat_chan)
1515
1516                         if pi is None:
1517                                 tbl.pirate_dummy(pa.name, rotate_nya[0], xs)
1518                         else:
1519                                 tbl.pirate(pi, xs)
1520
1521                 s += tbl_s.getvalue()
1522                 displayer.show(s)
1523                 tbl_s.close()
1524
1525                 if cliexec is not None:
1526                         clierr = track.local_command("/"+cliexec.strip())
1527                         cliexec = None
1528                         continue
1529
1530                 k = kreader.getch()
1531                 if k is None:
1532                         rotate_nya = rotate_nya[1:3] + rotate_nya[0]
1533                         continue
1534
1535                 if clierr is not None:
1536                         clierr = None
1537                         continue
1538
1539                 if clicmd is not None:
1540                         if k == '\r' or k == '\n':
1541                                 cliexec = clicmd
1542                                 clicmd = clicmdbase
1543                         elif k == '\e' and clicmd != "":
1544                                 clicmd = clicmdbase
1545                         elif k == '\33':
1546                                 clicmd = None
1547                         elif k == '\b' or k == '\177':
1548                                 clicmd = clicmd[ 0 : len(clicmd)-1 ]
1549                         else:
1550                                 clicmd += k
1551                         continue
1552
1553                 if k == 'q': break
1554                 elif k == 'g': sort = SkillSorter('Gunning')
1555                 elif k == 'c': sort = SkillSorter('Carpentry')
1556                 elif k == 's': sort = SkillSorter('Sailing/Rigging')
1557                 elif k == 'b': sort = SkillSorter('Bilging')
1558                 elif k == 'n': sort = SkillSorter('Navigating')
1559                 elif k == 'd': sort = SkillSorter('Battle Navigation')
1560                 elif k == 't': sort = SkillSorter('Treasure Haul')
1561                 elif k == 'a': sort = NameSorter()
1562                 elif k == '/': clicmdbase = ""; clicmd = clicmdbase
1563                 elif k == '+': clicmdbase = "a "; clicmd = clicmdbase
1564                 else: pass # unknown key command
1565
1566 #---------- individual keystroke input ----------
1567
1568 class DummyKeystrokeReader:
1569         def __init__(self,fd,timeout_dummy): pass
1570         def stop(self): pass
1571         def getch(self): sleep(1); return None
1572         def info(self): return ' [noninteractive]'
1573
1574 class KeystrokeReader(DummyKeystrokeReader):
1575         def __init__(self, fd, timeout_decisec=0):
1576                 self._fd = fd
1577                 self._saved = termios.tcgetattr(fd)
1578                 a = termios.tcgetattr(fd)
1579                 a[3] &= ~(termios.ECHO | termios.ECHONL |
1580                           termios.ICANON | termios.IEXTEN)
1581                 a[6][termios.VMIN] = 0
1582                 a[6][termios.VTIME] = timeout_decisec
1583                 termios.tcsetattr(fd, termios.TCSANOW, a)
1584         def stop(self):
1585                 termios.tcsetattr(self._fd, termios.TCSANOW, self._saved)
1586         def getch(self):
1587                 debug_flush()
1588                 byte = os.read(self._fd, 1)
1589                 if not len(byte): return None
1590                 return byte
1591         def info(self):
1592                 return ''
1593
1594 #---------- main program ----------
1595
1596 def main():
1597         global opts, fetcher
1598
1599         pa = OptionParser(
1600 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
1601 actions:
1602  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
1603  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
1604  yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
1605  yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG
1606  yoweb-scrape [options] ship-aid CHAT-LOG  (must be .../PIRATE_OCEAN_chat-log*)
1607
1608 display modes (for --display) apply to ship-aid:
1609  --display=dumb       just print new information, scrolling the screen
1610  --display=overwrite  use cursor motion, selective clear, etc. to redraw at top''')
1611         ao = pa.add_option
1612         ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
1613                 help='select ocean OCEAN')
1614         ao('--cache-dir', dest='cache_dir', metavar='DIR',
1615                 default='~/.yoweb-scrape-cache',
1616                 help='cache yoweb pages in DIR')
1617         ao('-D','--debug', action='count', dest='debug', default=0,
1618                 help='enable debugging output')
1619         ao('--debug-fd', type='int', dest='debug_fd',
1620                 help='write any debugging output to specified fd')
1621         ao('-q','--quiet', action='store_true', dest='quiet',
1622                 help='suppress warning output')
1623         ao('--display', action='store', dest='display',
1624                 type='choice', choices=['dumb','overwrite'],
1625                 help='how to display ship aid')
1626         ao('--local-ypp-dir', action='store', dest='localhtml',
1627                 help='get yppedia pages from local directory LOCALHTML'+
1628                         ' instead of via HTTP')
1629
1630         ao_jt = lambda wh, t: ao(
1631                 '--timeout-sa-'+wh, action='store', dest='timeout_'+wh,
1632                 default=t, help=('set timeout for expiring %s jobbers' % wh))
1633         ao_jt('applied',      120)
1634         ao_jt('invited',      120)
1635         ao_jt('declined',      30)
1636         ao_jt('ashore',      1800)
1637
1638         ao('--ship-duty', action='store_true', dest='ship_duty',
1639                 help='show ship duty station puzzles')
1640         ao('--all-puzzles', action='store_false', dest='ship_duty',
1641                 help='show all puzzles, not just ship duty stations')
1642
1643         ao('--min-cache-reuse', type='int', dest='min_max_age',
1644                 metavar='SECONDS', default=60,
1645                 help='always reuse cache yoweb data if no older than this')
1646
1647         (opts,args) = pa.parse_args()
1648         random.seed()
1649
1650         if len(args) < 1:
1651                 print >>sys.stderr, copyright_info
1652                 pa.error('need a mode argument')
1653
1654         if opts.debug_fd is not None:
1655                 opts.debug_file = os.fdopen(opts.debug_fd, 'w')
1656         else:
1657                 opts.debug_file = sys.stdout
1658
1659         mode = args[0]
1660         mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
1661         try: mode_fn = globals()[mode_fn_name]
1662         except KeyError: pa.error('unknown mode "%s"' % mode)
1663
1664         # fixed parameters
1665         opts.expire_age = max(3600, opts.min_max_age)
1666
1667         opts.ship_reboard_clearout = 3600
1668
1669         if opts.cache_dir.startswith('~/'):
1670                 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
1671
1672         if opts.display is None:
1673                 if ((opts.debug > 0 and opts.debug_fd is None)
1674                     or not os.isatty(sys.stdout.fileno())):
1675                         opts.display = 'dumb'
1676                 else:
1677                         opts.display = 'overwrite'
1678
1679         fetcher = Fetcher(opts.ocean, opts.cache_dir)
1680
1681         mode_fn(args[1:], pa.error)
1682
1683 main()