yoweb-scrape

   1 #!/usr/bin/python
   2 # This is part of ypp-sc-tools, a set of third-party tools for assisting
   3 # players of Yohoho Puzzle Pirates.
   4 #
   5 # Copyright (C) 2009 Ian Jackson <ijackson@chiark.greenend.org.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 #
  20 # Yohoho and Puzzle Pirates are probably trademarks of Three Rings and
  21 # are used without permission.  This program is not endorsed or
  22 # sponsored by Three Rings.
  23
  24 copyright_info = '''
  25 yoweb-scrape is part of ypp-sc-tools  Copyright (C) 2009 Ian Jackson
  26 This program comes with ABSOLUTELY NO WARRANTY; this is free software,
  27 and you are welcome to redistribute it under certain conditions.
  28 For details, read the top of the yoweb-scrape file.
  29 '''
  30
  31 #---------- setup ----------
  32
  33 import signal
  34 signal.signal(signal.SIGINT, signal.SIG_DFL)
  35
  36 import os
  37 import time
  38 import urllib
  39 import urllib2
  40 import errno
  41 import sys
  42 import re as regexp
  43 import random
  44 import curses
  45 import termios
  46 import random
  47 import subprocess
  48 from optparse import OptionParser
  49 from StringIO import StringIO
  50
  51 from BeautifulSoup import BeautifulSoup
  52
  53 opts = None
  54
  55 #---------- YPP parameters and arrays ----------
  56
  57 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
  58         '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
  59         '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
  60         '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
  61
  62 core_duty_puzzles = [
  63                 'Gunning',
  64                 ['Sailing','Rigging'],
  65                 'Bilging',
  66                 'Carpentry',
  67                 ]
  68
  69 duty_puzzles = ([ 'Navigating', 'Battle Navigation' ] +
  70                 core_duty_puzzles +
  71                 [ 'Treasure Haul' ])
  72
  73 standingvals = ('Able/Proficient/Distinguished/Respected/Master'+
  74                 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
  75 standing_limit = len(standingvals)
  76
  77 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
  78
  79 max_pirate_namelen = 12
  80
  81
  82 #---------- general utilities ----------
  83
  84 def debug(m):
  85         if opts.debug > 0:
  86                 print >>opts.debug_file, m
  87
  88 def debug_flush():
  89         if opts.debug > 0:
  90                 opts.debug_file.flush()
  91
  92 def sleep(seconds):
  93         debug_flush()
  94         time.sleep(seconds)
  95
  96 def format_time_interval(ti):
  97         if ti < 120: return '%d:%02d' % (ti / 60, ti % 60)
  98         if ti < 7200: return '%2dm' % (ti / 60)
  99         if ti < 86400: return '%dh' % (ti / 3600)
 100         return '%dd' % (ti / 86400)
 101
 102 def yppsc_dir():
 103         lib = os.getenv("YPPSC_YARRG_SRCBASE")
 104         if lib is not None: return lib
 105         lib = sys.argv[0]
 106         lib = regexp.sub('/[^/]+$', '', lib)
 107         os.environ["YPPSC_YARRG_SRCBASE"] = lib
 108         return lib
 109
 110 #---------- caching and rate-limiting data fetcher ----------
 111
 112 class Fetcher:
 113         def __init__(self, cachedir):
 114                 debug('Fetcher init %s' % cachedir)
 115                 self.cachedir = cachedir
 116                 try: os.mkdir(cachedir)
 117                 except (OSError,IOError), oe:
 118                         if oe.errno != errno.EEXIST: raise
 119                 self._cache_scan(time.time())
 120
 121         def _cache_scan(self, now):
 122                 # returns list of ages, unsorted
 123                 ages = []
 124                 debug('Fetcher   scan_cache')
 125                 for leaf in os.listdir(self.cachedir):
 126                         if not leaf.startswith('#'): continue
 127                         path = self.cachedir + '/' + leaf
 128                         try: s = os.stat(path)
 129                         except (OSError,IOError), oe:
 130                                 if oe.errno != errno.ENOENT: raise
 131                                 continue
 132                         age = now - s.st_mtime
 133                         if age > opts.expire_age:
 134                                 debug('Fetcher    expire %d %s' % (age, path))
 135                                 try: os.remove(path)
 136                                 except (OSError,IOError), oe:
 137                                         if oe.errno != errno.ENOENT: raise
 138                                 continue
 139                         ages.append(age)
 140                 return ages
 141
 142         def need_wait(self, now, imaginary=[]):
 143                 ages = self._cache_scan(now)
 144                 ages += imaginary
 145                 ages.sort()
 146                 debug('Fetcher   ages ' + `ages`)
 147                 min_age = 1
 148                 need_wait = 0
 149                 for age in ages:
 150                         if age < min_age and age <= 5:
 151                                 debug('Fetcher   morewait min=%d age=%d' %
 152                                         (min_age, age))
 153                                 need_wait = max(need_wait, min_age - age)
 154                         min_age += 3
 155                         min_age *= 1.25
 156                 if need_wait > 0:
 157                         need_wait += random.random() - 0.5
 158                 return need_wait
 159
 160         def _rate_limit_cache_clean(self, now):
 161                 need_wait = self.need_wait(now)
 162                 if need_wait > 0:
 163                         debug('Fetcher   wait %d' % need_wait)
 164                         sleep(need_wait)
 165
 166         def fetch(self, url, max_age):
 167                 debug('Fetcher fetch %s' % url)
 168                 cache_corename = urllib.quote_plus(url)
 169                 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
 170                 try: f = file(cache_item, 'r')
 171                 except (OSError,IOError), oe:
 172                         if oe.errno != errno.ENOENT: raise
 173                         f = None
 174                 now = time.time()
 175                 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
 176                 if f is not None:
 177                         s = os.fstat(f.fileno())
 178                         age = now - s.st_mtime
 179                         if age > max_age:
 180                                 debug('Fetcher  stale %d < %d'% (max_age, age))
 181                                 f = None
 182                 if f is not None:
 183                         data = f.read()
 184                         f.close()
 185                         debug('Fetcher  cached %d > %d' % (max_age, age))
 186                         return data
 187
 188                 debug('Fetcher  fetch')
 189                 self._rate_limit_cache_clean(now)
 190
 191                 stream = urllib2.urlopen(url)
 192                 data = stream.read()
 193                 cache_tmp = "%s/#%s~%d#" % (
 194                         self.cachedir, cache_corename, os.getpid())
 195                 f = file(cache_tmp, 'w')
 196                 f.write(data)
 197                 f.close()
 198                 os.rename(cache_tmp, cache_item)
 199                 debug('Fetcher  stored')
 200                 return data
 201
 202 class Yoweb(Fetcher):
 203         def __init__(self, ocean, cachedir):
 204                 debug('Yoweb init %s' % cachedir)
 205                 self.ocean = ocean
 206                 Fetcher.__init__(self, cachedir)
 207
 208         def default_ocean(self, ocean='ice'):
 209                 if self.ocean is None:
 210                         self.ocean = ocean
 211
 212         def yoweb(self, kind, tail, max_age):
 213                 self.default_ocean()
 214                 assert(self.ocean)
 215                 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
 216                         self.ocean, kind, tail)
 217                 return self.fetch(url, max_age)
 218
 219 class Yppedia(Fetcher):
 220         def __init__(self, cachedir):
 221                 debug('Yoweb init %s' % cachedir)
 222                 self.base = 'http://yppedia.puzzlepirates.com/'
 223                 self.localhtml = opts.localhtml
 224                 Fetcher.__init__(self, cachedir)
 225
 226         def __call__(self, rhs):
 227                 if self.localhtml is None:
 228                         url = self.base + rhs
 229                         debug('Yppedia retrieving YPP '+url);
 230                         return self.fetch(url, 3000)
 231                 else:
 232                         return file(opts.localhtml + '/' + rhs, 'r')
 233
 234 #---------- logging assistance for troubled screenscrapers ----------
 235
 236 class SoupLog:
 237         def __init__(self):
 238                 self.msgs = [ ]
 239         def msg(self, m):
 240                 self.msgs.append(m)
 241         def soupm(self, obj, m):
 242                 self.msg(m + '; in ' + `obj`)
 243         def needs_msgs(self, child_souplog):
 244                 self.msgs += child_souplog.msgs
 245                 child_souplog.msgs = [ ]
 246
 247 def soup_text(obj):
 248         str = ''.join(obj.findAll(text=True))
 249         return str.strip()
 250
 251 class SomethingSoupInfo(SoupLog):
 252         def __init__(self, kind, tail, max_age):
 253                 SoupLog.__init__(self)
 254                 html = fetcher.yoweb(kind, tail, max_age)
 255                 self._soup = BeautifulSoup(html,
 256                         convertEntities=BeautifulSoup.HTML_ENTITIES
 257                         )
 258
 259 #---------- scraper for pirate pages ----------
 260
 261 class PirateInfo(SomethingSoupInfo):
 262         # Public data members:
 263         #  pi.standings = { 'Treasure Haul': 'Able' ... }
 264         #  pi.name = name
 265         #  pi.crew = (id, name)
 266         #  pi.flag = (id, name)
 267         #  pi.msgs = [ 'message describing problem with scrape' ]
 268
 269         def __init__(self, pirate, max_age=300):
 270                 SomethingSoupInfo.__init__(self,
 271                         'pirate.wm?target=', pirate, max_age)
 272                 self.name = pirate
 273                 self._find_standings()
 274                 self.crew = self._find_crewflag('crew',
 275                         '^/yoweb/crew/info\\.wm')
 276                 self.flag = self._find_crewflag('flag',
 277                         '^/yoweb/flag/info\\.wm')
 278
 279         def _find_standings(self):
 280                 imgs = self._soup.findAll('img',
 281                         src=regexp.compile('/yoweb/images/stat.*'))
 282                 re = regexp.compile(
 283 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
 284                         )
 285                 standings = { }
 286
 287                 for skill in puzzles:
 288                         standings[skill] = [ ]
 289
 290                 skl = SoupLog()
 291
 292                 for img in imgs:
 293                         try: puzzle = img['alt']
 294                         except KeyError: continue
 295
 296                         if not puzzle in puzzles:
 297                                 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
 298                                 continue
 299                         key = img.findParent('td')
 300                         if key is None:
 301                                 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
 302                                 continue
 303                         valelem = key.findNextSibling('td')
 304                         if valelem is None:
 305                                 skl.soupm(key, 'puzzle missing sibling "%s"'
 306                                         % puzzle)
 307                                 continue
 308                         valstr = soup_text(valelem)
 309                         match = re.match(valstr)
 310                         if match is None:
 311                                 skl.soupm(key, ('puzzle "%s" unparseable'+
 312                                         ' standing "%s"') % (puzzle, valstr))
 313                                 continue
 314                         standing = match.group(match.lastindex)
 315                         standings[puzzle].append(standing)
 316
 317                 self.standings = { }
 318
 319                 for puzzle in puzzles:
 320                         sl = standings[puzzle]
 321                         if len(sl) > 1:
 322                                 skl.msg('puzzle "%s" multiple standings %s' %
 323                                                 (puzzle, `sl`))
 324                                 continue
 325                         if not sl:
 326                                 skl.msg('puzzle "%s" no standing found' % puzzle)
 327                                 continue
 328                         standing = sl[0]
 329                         for i in range(0, standing_limit):
 330                                 if standing == standingvals[i]:
 331                                         self.standings[puzzle] = i
 332                         if not puzzle in self.standings:
 333                                 skl.msg('puzzle "%s" unknown standing "%s"' %
 334                                         (puzzle, standing))
 335
 336                 all_standings_ok = True
 337                 for puzzle in puzzles:
 338                         if not puzzle in self.standings:
 339                                 self.needs_msgs(skl)
 340
 341         def _find_crewflag(self, cf, yoweb_re):
 342                 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
 343                 if len(things) != 1:
 344                         self.msg('zero or several %s id references found' % cf)
 345                         return None
 346                 thing = things[0]
 347                 id_re = '\\b%sid\\=(\\w+)$' % cf
 348                 id_haystack = thing['href']
 349                 match = regexp.compile(id_re).search(id_haystack)
 350                 if match is None:
 351                         self.soupm(thing, ('incomprehensible %s id ref'+
 352                                 ' (%s in %s)') % (cf, id_re, id_haystack))
 353                         return None
 354                 name = soup_text(thing)
 355                 return (match.group(1), name)
 356
 357         def __str__(self):
 358                 return `(self.crew, self.flag, self.standings, self.msgs)`
 359
 360 #---------- scraper for crew pages ----------
 361
 362 class CrewInfo(SomethingSoupInfo):
 363         # Public data members:
 364         #  ci.crew = [ ('Captain',        ['Pirate', ...]),
 365         #              ('Senior Officer', [...]),
 366         #               ... ]
 367         #  pi.msgs = [ 'message describing problem with scrape' ]
 368
 369         def __init__(self, crewid, max_age=300):
 370                 SomethingSoupInfo.__init__(self,
 371                         'crew/info.wm?crewid=', crewid, max_age)
 372                 self._find_crew()
 373
 374         def _find_crew(self):
 375                 self.crew = []
 376                 capts = self._soup.findAll('img',
 377                         src='/yoweb/images/crew-captain.png')
 378                 if len(capts) != 1:
 379                         self.msg('crew members: no. of captain images != 1')
 380                         return
 381                 tbl = capts[0]
 382                 while not tbl.find('a', href=pirate_ref_re):
 383                         tbl = tbl.findParent('table')
 384                         if not tbl:
 385                                 self.msg('crew members: cannot find table')
 386                                 return
 387                 current_rank_crew = None
 388                 crew_rank_re = regexp.compile('/yoweb/images/crew')
 389                 for row in tbl.contents:
 390                         # findAll(recurse=False)
 391                         if isinstance(row,basestring):
 392                                 continue
 393
 394                         is_rank = row.find('img', attrs={'src': crew_rank_re})
 395                         if is_rank:
 396                                 rank = soup_text(row)
 397                                 current_rank_crew = []
 398                                 self.crew.append((rank, current_rank_crew))
 399                                 continue
 400                         for cell in row.findAll('a', href=pirate_ref_re):
 401                                 if current_rank_crew is None:
 402                                         self.soupm(cell, 'crew members: crew'
 403                                                 ' before rank')
 404                                         continue
 405                                 current_rank_crew.append(soup_text(cell))
 406
 407         def __str__(self):
 408                 return `(self.crew, self.msgs)`
 409
 410 class FlagInfo(SomethingSoupInfo):
 411         # Public data members (after init):
 412         #
 413         #   name        #               string
 414         #
 415         #   relations[n] = (otherflagname, otherflagid, [stringfromyoweb],
 416         #               thisdeclaring, otherdeclaringmin, otherdeclaringmax)
 417         #               # where {this,other}declaring{,min,max} are:
 418         #               #       -1      {this,other} is declaring war
 419         #               #        0      {this,other} is not doing either
 420         #               #       +1      {this,other} is allying
 421         #   relation_byname[otherflagname] = relations[some_n]
 422         #   relation_byid[otherflagname] = relations[some_n]
 423         #
 424         #   islands[n] = (islandname, islandid)
 425         #
 426         def __init__(self, flagid, max_age=600):
 427                 SomethingSoupInfo.__init__(self,
 428                         'flag/info.wm?flagid=', flagid, max_age)
 429                 self._find_flag()
 430
 431         def _find_flag(self):
 432                 font2 = self._soup.find('font',{'size':'+2'})
 433                 self.name = font2.find('b').contents[0]
 434
 435                 self.relations = [ ]
 436                 self.relation_byname = { }
 437                 self.relation_byid = { }
 438                 self.islands = [ ]
 439
 440                 magnate = self._soup.find('img',{'src':
 441                         '/yoweb/images/repute-MAGNATE.png'})
 442                 warinfo = (magnate.findParent('table').findParent('tr').
 443                         findNextSibling('tr').findNext('td',{'align':'left'}))
 444
 445                 def warn(m):
 446                         print >>sys.stderr, 'WARNING: '+m
 447
 448                 def wi_warn(head, waritem):
 449                         warn('unknown warmap item: %s: %s' %
 450                                 (`head`, ``waritem``))
 451
 452                 def wihelp_item(waritem, thing):
 453                         url = waritem.get('href', None)
 454                         if url is None:
 455                                 return ('no url for '+thing,None,None)
 456                         m = regexp.search('\?'+thing+'id=(\d+)$', url)
 457                         if not m: return ('no '+thing+'id',None,None)
 458                         tid = m.group(1)
 459                         tname = waritem.string
 460                         if tname is None:
 461                                 return (thing+' name not just string',None,None)
 462                         return (None,tid,tname)
 463
 464                 def wi_alwar(head, waritem, thisdecl, othermin, othermax):
 465                         (err,flagid,flagname) = wihelp_item(waritem,'flag')
 466                         if err: return err
 467                         rel = self.relation_byid.get(flagid, None)
 468                         if rel: return 'flag id twice!'
 469                         if flagname in self.relation_byname:
 470                                 return 'flag name twice!'
 471                         rel = (flagname,flagid,head, thisdecl,othermin,othermax)
 472                         self.relations.append(rel)
 473                         self.relation_byid[flagid] = rel
 474                         self.relation_byname[flagid] = rel
 475
 476                 def wi_isle(head, waritem):
 477                         (err,isleid,islename) = wihelp_item(waritem,'island')
 478                         if err: return err
 479                         self.islands.append((isleid,islename))
 480
 481                 warmap = {
 482                         'Allied with':                  (wi_alwar,+1,+1,+1),
 483                         'Declaring war against':        (wi_alwar,-1, 0,+1),
 484                         'At war with':                  (wi_alwar,-1,-1,-1),
 485                         'Trying to form an alliance with': (wi_alwar,+1,-1,0),
 486                         'Islands controlled by this flag': (wi_isle,),
 487                         }
 488
 489                 how = (wi_warn, None)
 490
 491                 for waritem in warinfo.findAll(['font','a']):
 492                         if waritem is None: break
 493                         if waritem.name == 'font':
 494                                 colour = waritem.get('color',None)
 495                                 if colour.lstrip('#') != '958A5F':
 496                                         warn('strange colour %s in %s' %
 497                                                 (colour,``waritem``))
 498                                         continue
 499                                 head = waritem.string
 500                                 if head is None:
 501                                         warn('no head string in '+``waritem``)
 502                                         continue
 503                                 head = regexp.sub('\\s+', ' ', head).strip()
 504                                 head = head.rstrip(':')
 505                                 how = (head,) + warmap.get(head, (wi_warn,))
 506                                 continue
 507                         assert(waritem.name == 'a')
 508
 509                         debug('WARHOW %s(%s, waritem, *%s)' %
 510                                 (how[1], `how[0]`, `how[2:]`))
 511                         bad = how[1](how[0], waritem, *how[2:])
 512                         if bad:
 513                                 warn('bad waritem %s: %s: %s' % (`how[0]`,
 514                                         bad, ``waritem``))
 515
 516         def __str__(self):
 517                 return `(self.name, self.islands, self.relations)`
 518
 519 #---------- scraper for ocean info incl. embargoes etc. ----------
 520
 521 class IslandBasicInfo():
 522         # Public data attributes:
 523         #  ocean
 524         #  name
 525         # Public data attributes maybe set by caller:
 526         #  arch
 527         def __init__(self, ocean, islename):
 528                 self.ocean = ocean
 529                 self.name = islename
 530         def yppedia(self):
 531                 def q(x): return urllib.quote(x.replace(' ','_'))
 532                 url_rhs = q(self.name) + '_(' + q(self.ocean) + ')'
 533                 return yppedia(url_rhs)
 534         def __str__(self):
 535                 return `(self.ocean, self.name)`
 536
 537 class IslandExtendedInfo(IslandBasicInfo):
 538         # Public data attributes (inherited):
 539         #  ocean
 540         #  name
 541         # Public data attributes (additional):
 542         #  islandid
 543         #  yoweb_url
 544         #  flagid
 545         def __init__(self, ocean, islename):
 546                 IslandBasicInfo.__init__(self, ocean, islename)
 547                 self.islandid = None
 548                 self.yoweb_url = None
 549                 self._collect_yoweb()
 550                 self._collect_flagid()
 551
 552         def _collect_yoweb(self):
 553                 debug('IEI COLLECT YOWEB '+`self.name`)
 554                 self.islandid = None
 555                 self.yoweb_url = None
 556
 557                 soup = BeautifulSoup(self.yppedia())
 558                 content = soup.find('div', attrs = {'id': 'content'})
 559                 yoweb_re = regexp.compile('^http://\w+\.puzzlepirates\.com/'+
 560                         'yoweb/island/info\.wm\?islandid=(\d+)$')
 561                 a = soup.find('a', attrs = { 'href': yoweb_re })
 562                 if a is None:
 563                         debug('IEI COLLECT YOWEB '+`self.name`+' NONE')
 564                         return
 565
 566                 debug('IEI COLLECT YOWEB '+`self.name`+' GOT '+``a``)
 567                 self.yoweb_url = a['href']
 568                 m = yoweb_re.search(self.yoweb_url)
 569                 self.islandid = m.group(1)
 570
 571         def _collect_flagid(self):
 572                 self.flagid = None
 573
 574                 yo = self.yoweb_url
 575                 debug('IEI COLLECT FLAGID '+`self.name`+' URL '+`yo`)
 576                 if yo is None: return None
 577                 dataf = fetcher.fetch(yo, 1800)
 578                 soup = BeautifulSoup(dataf)
 579                 ruler_re = regexp.compile(
 580                         '/yoweb/flag/info\.wm\?flagid=(\d+)$')
 581                 ruler = soup.find('a', attrs = { 'href': ruler_re })
 582                 if not ruler:
 583                         debug('IEI COLLECT FLAGID '+`self.name`+' NONE')
 584                         return
 585                 debug('IEI COLLECT FLAGID '+`self.name`+' GOT '+``ruler``)
 586                 m = ruler_re.search(ruler['href'])
 587                 self.flagid = m.group(1)
 588
 589         def __str__(self):
 590                 return `(self.ocean, self.islandid, self.name,
 591                         self.yoweb_url, self.flagid)`
 592
 593 class OceanInfo():
 594         # Public data attributes:
 595         #   oi.islands[islename] = IslandInfo(...)
 596         #   oi.arches[archname][islename] = IslandInfo(...)
 597         def __init__(self, isleclass=IslandBasicInfo):
 598                 self.isleclass = isleclass
 599                 self.ocean = fetcher.ocean.lower().capitalize()
 600
 601                 cmdl = ['./yppedia-ocean-scraper']
 602                 if opts.localhtml is not None:
 603                         cmdl += ['--local-html-dir',opts.localhtml]
 604                 cmdl += [self.ocean]
 605                 debug('OceanInfo collect running ' + `cmdl`)
 606                 oscraper = subprocess.Popen(
 607                         cmdl,
 608                         stdout = subprocess.PIPE,
 609                         cwd = yppsc_dir()+'/yarrg',
 610                         shell=False, stderr=None,
 611                         )
 612                 h = oscraper.stdout.readline()
 613                 debug('OceanInfo collect h '+`h`)
 614                 assert(regexp.match('^ocean ', h))
 615                 arch_re = regexp.compile('^ (\S.*)')
 616                 island_re = regexp.compile('^  (\S.*)')
 617
 618                 self.islands = { }
 619                 self.arches = { }
 620                 archname = None
 621
 622                 for l in oscraper.stdout:
 623                         debug('OceanInfo collect l '+`l`)
 624                         l = l.rstrip('\n')
 625                         m = island_re.match(l)
 626                         if m:
 627                                 assert(archname is not None)
 628                                 islename = m.group(1)
 629                                 isle = self.isleclass(self.ocean, islename)
 630                                 isle.arch = archname
 631                                 self.islands[islename] = isle
 632                                 self.arches[archname][islename] = isle
 633                                 continue
 634                         m = arch_re.match(l)
 635                         if m:
 636                                 archname = m.group(1)
 637                                 assert(archname not in self.arches)
 638                                 self.arches[archname] = { }
 639                                 continue
 640                         assert(False)
 641                 oscraper.wait()
 642                 assert(oscraper.returncode == 0)
 643
 644         def __str__(self):
 645                 return `(self.islands, self.arches)`
 646
 647 #---------- pretty-printer for tables of pirate puzzle standings ----------
 648
 649 class StandingsTable:
 650         def __init__(self, f, use_puzzles=None, col_width=6, gap_every=5):
 651                 if use_puzzles is None:
 652                         if opts.ship_duty:
 653                                 use_puzzles=duty_puzzles
 654                         else:
 655                                 use_puzzles=puzzles
 656                 self._puzzles = use_puzzles
 657                 self.f = f
 658                 self._cw = col_width-1
 659                 self._gap_every = gap_every
 660                 self._linecount = 0
 661                 self._o = f.write
 662
 663         def _nl(self): self._o('\n')
 664
 665         def _pline(self, lhs, puzstrs, extra):
 666                 if (self._linecount > 0
 667                     and self._gap_every is not None
 668                     and not (self._linecount % self._gap_every)):
 669                         self._nl()
 670                 self._o('%-*s' % (max(max_pirate_namelen+1, 15), lhs))
 671                 for v in puzstrs:
 672                         self._o(' %-*.*s' % (self._cw,self._cw, v))
 673                 if extra:
 674                         self._o(' ' + extra)
 675                 self._nl()
 676                 self._linecount += 1
 677
 678         def _puzstr(self, pi, puzzle):
 679                 if not isinstance(puzzle,list): puzzle = [puzzle]
 680                 try: standing = max([pi.standings[p] for p in puzzle])
 681                 except KeyError: return '?'
 682                 if not standing: return ''
 683                 s = ''
 684                 if self._cw > 4:
 685                         c1 = standingvals[standing][0]
 686                         if standing < 3: c1 = c1.lower() # 3 = Master
 687                         s += `standing`
 688                 if self._cw > 5:
 689                         s += ' '
 690                 s += '*' * (standing / 2)
 691                 s += '+' * (standing % 2)
 692                 return s
 693
 694         def headings(self, lhs='', rhs=None):
 695                 def puzn_redact(name):
 696                         if isinstance(name,list):
 697                                 return '/'.join(
 698                                         ["%.*s" % (self._cw/2, puzn_redact(n))
 699                                          for n in name])
 700                         spc = name.find(' ')
 701                         if spc < 0: return name
 702                         return name[0:min(4,spc)] + name[spc+1:]
 703                 self._linecount = -2
 704                 self._pline(lhs, map(puzn_redact, self._puzzles), rhs)
 705                 self._linecount = 0
 706         def literalline(self, line):
 707                 self._o(line)
 708                 self._nl()
 709                 self._linecount = 0
 710         def pirate_dummy(self, name, standingstring, extra=None):
 711                 standings = standingstring * len(self._puzzles)
 712                 self._pline(' '+name, standings, extra)
 713         def pirate(self, pi, extra=None):
 714                 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
 715                 self._pline(' '+pi.name, puzstrs, extra)
 716
 717
 718 #---------- chat log parser ----------
 719
 720 class PirateAboard:
 721         # This is essentially a transparent, dumb, data class.
 722         #  pa.v                 may be None
 723         #  pa.name
 724         #  pa.last_time
 725         #  pa.last_event
 726         #  pa.gunner
 727         #  pa.last_chat_time
 728         #  pa.last_chat_chan
 729         #  pa.pi
 730
 731         # Also used for jobbing applicants:
 732         #               happens when                    expires (to "-")
 733         #   -            disembark, leaves crew          no
 734         #   aboard       evidence of them being aboard   no
 735         #   applied      "has applied for the job"       120s, configurable
 736         #   ashore       "has taken a job"               30min, configurable
 737         #   declined     "declined the job offer"        30s, configurable
 738         #   invited      "has been invited to job"       120s, configurable
 739         #
 740         #  pa.jobber    None, 'ashore', 'applied', 'invited', 'declined'
 741         #  pa.expires   expiry time time
 742
 743         def __init__(pa, pn, v, time, event):
 744                 pa.name = pn
 745                 pa.v = v
 746                 pa.last_time = time
 747                 pa.last_event = event
 748                 pa.last_chat_time = None
 749                 pa.last_chat_chan = None
 750                 pa.gunner = False
 751                 pa.pi = None
 752                 pa.jobber = None
 753                 pa.expires = None
 754
 755         def pirate_info(pa):
 756                 now = time.time()
 757                 if pa.pi:
 758                         age = now - pa.pi_fetched
 759                         guide = random.randint(120,240)
 760                         if age <= guide:
 761                                 return pa.pi
 762                         debug('PirateAboard refresh %d > %d  %s' % (
 763                                 age, guide, pa.name))
 764                         imaginary = [2,4]
 765                 else:
 766                         imaginary = [1]
 767                 wait = fetcher.need_wait(now, imaginary)
 768                 if wait:
 769                         debug('PirateAboard fetcher not ready %d' % wait)
 770                         return pa.pi
 771                 pa.pi = PirateInfo(pa.name, 600)
 772                 pa.pi_fetched = now
 773                 return pa.pi
 774
 775 class ChatLogTracker:
 776         # This is quite complex so we make it opaque.  Use the
 777         # official invokers, accessors etc.
 778
 779         def __init__(self, myself_pi, logfn):
 780                 self._pl = {}   # self._pl['Pirate'] =
 781                 self._vl = {}   #   self._vl['Vessel']['Pirate'] = PirateAboard
 782                                 # self._vl['Vessel']['#lastinfo']
 783                                 # self._vl['Vessel']['#name']
 784                                 # self._v = self._vl[self._vessel]
 785                 self._date = None
 786                 self._myself = myself_pi
 787                 self._lbuf = ''
 788                 self._f = file(logfn)
 789                 flen = os.fstat(self._f.fileno()).st_size
 790                 max_backlog = 500000
 791                 if flen > max_backlog:
 792                         startpos = flen - max_backlog
 793                         self._f.seek(startpos)
 794                         self._f.readline()
 795                 self._progress = [0, flen - self._f.tell()]
 796                 self._disembark_myself()
 797                 self._need_redisplay = False
 798                 self._lastvessel = None
 799
 800         def _disembark_myself(self):
 801                 self._v = None
 802                 self._vessel = None
 803                 self.force_redisplay()
 804
 805         def force_redisplay(self):
 806                 self._need_redisplay = True
 807
 808         def _vessel_updated(self, v, timestamp):
 809                 if v is None: return
 810                 v['#lastinfo'] = timestamp
 811                 self.force_redisplay()
 812
 813         def _onboard_event(self,v,timestamp,pirate,event,jobber=None):
 814                 pa = self._pl.get(pirate, None)
 815                 if pa is not None and pa.v is v:
 816                         pa.last_time = timestamp
 817                         pa.last_event = event
 818                 else:
 819                         if pa is not None and pa.v is not None:
 820                                 del pa.v[pirate]
 821                         pa = PirateAboard(pirate, v, timestamp, event)
 822                         self._pl[pirate] = pa
 823                         if v is not None: v[pirate] = pa
 824                 pa.jobber = jobber
 825
 826                 if jobber is None: timeout = None
 827                 else: timeout = getattr(opts, 'timeout_'+jobber)
 828                 if timeout is None: pa.expires = None
 829                 else: pa.expires = timestamp + timeout
 830                 self._vessel_updated(v, timestamp)
 831                 return pa
 832
 833         def _expire_jobbers(self, now):
 834                 for pa in self._pl.values():
 835                         if pa.expires is None: continue
 836                         if pa.expires >= now: continue
 837                         v = pa.v
 838                         del self._pl[pa.name]
 839                         if v is not None: del v[pa.name]
 840                         self.force_redisplay()
 841
 842         def _trash_vessel(self, v):
 843                 for pn in v:
 844                         if pn.startswith('#'): continue
 845                         del self._pl[pn]
 846                 vn = v['#name']
 847                 del self._vl[vn]
 848                 if v is self._v: self._disembark_myself()
 849                 self.force_redisplay()
 850
 851         def _vessel_stale(self, v, timestamp):
 852                 return timestamp - v['#lastinfo'] > opts.ship_reboard_clearout
 853
 854         def _vessel_check_expire(self, v, timestamp):
 855                 if not self._vessel_stale(v, timestamp):
 856                         return v
 857                 self._debug_line_disposition(timestamp,'',
 858                         'stale-reset ' + v['#name'])
 859                 self._trash_vessel(v)
 860                 return None
 861
 862         def expire_garbage(self, timestamp):
 863                 for v in self._vl.values():
 864                         self._vessel_check_expire(v, timestamp)
 865
 866         def _vessel_lookup(self, vn, timestamp, dml=[], create=False):
 867                 v = self._vl.get(vn, None)
 868                 if v is not None:
 869                         v = self._vessel_check_expire(v, timestamp)
 870                 if v is not None:
 871                         dml.append('found')
 872                         return v
 873                 if not create:
 874                         dml.append('no')
 875                 dml.append('new')
 876                 self._vl[vn] = v = { '#name': vn }
 877                 self._vessel_updated(v, timestamp)
 878                 return v
 879
 880         def _find_matching_vessel(self, pattern, timestamp, cmdr,
 881                                         dml=[], create=False):
 882                 # use when a commander pirate `cmdr' specified a vessel
 883                 #  by name `pattern' (either may be None)
 884                 # if create is true, will create the vessel
 885                 #  record if an exact name is specified
 886
 887                 if (pattern is not None and
 888                     not '*' in pattern
 889                     and len(pattern.split(' ')) == 2):
 890                         vn = pattern.title()
 891                         dml.append('exact')
 892                         return self._vessel_lookup(
 893                                 vn, timestamp, dml=dml, create=create)
 894
 895                 if pattern is None:
 896                         pattern_check = lambda vn: True
 897                 else:
 898                         re = '(?:.* )?%s$' % pattern.lower().replace('*','.+')
 899                         pattern_check = regexp.compile(re, regexp.I).match
 900
 901                 tries = []
 902
 903                 cmdr_pa = self._pl.get(cmdr, None)
 904                 if cmdr_pa: tries.append((cmdr_pa.v, 'cmdr'))
 905
 906                 tries.append((self._v, 'here'))
 907                 tried_vns = []
 908
 909                 for (v, dm) in tries:
 910                         if v is None: dml.append(dm+'?'); continue
 911
 912                         vn = v['#name']
 913                         if not pattern_check(vn):
 914                                 tried_vns.append(vn)
 915                                 dml.append(dm+'#')
 916                                 continue
 917
 918                         dml.append(dm+'!')
 919                         return v
 920
 921                 if pattern is not None and '*' in pattern:
 922                         search = [
 923                                 (vn,v)
 924                                 for (vn,v) in self._vl.iteritems()
 925                                 if not self._vessel_stale(v, timestamp)
 926                                 if pattern_check(vn)
 927                                 ]
 928                         #debug('CLT-RE /%s/ wanted (%s) searched (%s)' % (
 929                         #       re,
 930                         #       '/'.join(tried_vns),
 931                         #       '/'.join([vn for (vn,v) in search])))
 932
 933                         if len(search)==1:
 934                                 dml.append('one')
 935                                 return search[0][1]
 936                         elif search:
 937                                 dml.append('many')
 938                         else:
 939                                 dml.append('none')
 940
 941         def _debug_line_disposition(self,timestamp,l,m):
 942                 debug('CLT %13s %-40s %s' % (timestamp,m,l))
 943
 944         def _rm_crew_l(self,re,l):
 945                 m = regexp.match(re,l)
 946                 if m and m.group(2) == self._myself.crew[1]:
 947                         return m.group(1)
 948                 else:
 949                         return None
 950
 951         def local_command(self, metacmd):
 952                 # returns None if all went well, or problem message
 953                 return self._command(self._myself.name, metacmd,
 954                         "local", time.time(),
 955                         (lambda m: debug('CMD %s' % metacmd)))
 956
 957         def _command(self, cmdr, metacmd, chan, timestamp, d):
 958                 # returns None if all went well, or problem message
 959                 metacmd = regexp.sub('\\s+', ' ', metacmd).strip()
 960                 m2 = regexp.match(
 961                     '/([adj]) (?:([A-Za-z* ]+)\\s*:)?([A-Za-z ]+)$',
 962                     metacmd)
 963                 if not m2: return "unknown syntax or command"
 964
 965                 (cmd, pattern, targets) = m2.groups()
 966                 dml = ['cmd', chan, cmd]
 967
 968                 if cmd == 'a': each = self._onboard_event
 969                 elif cmd == 'd': each = disembark
 970                 else: each = lambda *l: self._onboard_event(*l,
 971                                 **{'jobber':'applied'})
 972
 973                 if cmdr == self._myself.name:
 974                         dml.append('self')
 975                         how = 'cmd: %s' % cmd
 976                 else:
 977                         dml.append('other')
 978                         how = 'cmd: %s %s' % (cmd,cmdr)
 979
 980                 if cmd == 'j':
 981                         if pattern is not None:
 982                                 return "/j command does not take a vessel"
 983                         v = None
 984                 else:
 985                         v = self._find_matching_vessel(
 986                                 pattern, timestamp, cmdr,
 987                                 dml, create=True)
 988
 989                 if cmd == 'j' or v is not None:
 990                         targets = targets.strip().split(' ')
 991                         dml.append(`len(targets)`)
 992                         for target in targets:
 993                                 each(v, timestamp, target.title(), how)
 994                         self._vessel_updated(v, timestamp)
 995
 996                 dm = ' '.join(dml)
 997                 return d(dm)
 998
 999                 return None
1000
1001         def chatline(self,l):
1002                 rm = lambda re: regexp.match(re,l)
1003                 d = lambda m: self._debug_line_disposition(timestamp,l,m)
1004                 rm_crew = lambda re: self._rm_crew_l(re,l)
1005                 timestamp = None
1006
1007                 m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$')
1008                 if m:
1009                         self._date = [int(x) for x in m.groups()]
1010                         self._previous_timestamp = None
1011                         return d('date '+`self._date`)
1012
1013                 if self._date is None:
1014                         return d('date unset')
1015
1016                 m = rm('\\[(\d\d):(\d\d):(\d\d)\\] ')
1017                 if not m:
1018                         return d('no timestamp')
1019
1020                 while True:
1021                         time_tuple = (self._date +
1022                                       [int(x) for x in m.groups()] +
1023                                       [-1,-1,-1])
1024                         timestamp = time.mktime(time_tuple)
1025                         if timestamp >= self._previous_timestamp: break
1026                         self._date[2] += 1
1027                         self._debug_line_disposition(timestamp,'',
1028                                 'new date '+`self._date`)
1029
1030                 self._previous_timestamp = timestamp
1031
1032                 l = l[l.find(' ')+1:]
1033
1034                 def ob_x(pirate,event):
1035                         return self._onboard_event(
1036                                         self._v, timestamp, pirate, event)
1037                 def ob1(did): ob_x(m.group(1), did); return d(did)
1038                 def oba(did): return ob1('%s %s' % (did, m.group(2)))
1039
1040                 def jb(pirate,jobber):
1041                         return self._onboard_event(
1042                                 None, timestamp, pirate,
1043                                 ("jobber %s" % jobber),
1044                                 jobber=jobber
1045                                 )
1046
1047                 def disembark(v, timestamp, pirate, event):
1048                         self._onboard_event(
1049                                         v, timestamp, pirate, 'leaving '+event)
1050                         del v[pirate]
1051                         del self._pl[pirate]
1052
1053                 def disembark_me(why):
1054                         self._disembark_myself()
1055                         return d('disembark-me '+why)
1056
1057                 m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
1058                 if m:
1059                         dm = ['boarding']
1060                         pn = self._myself.name
1061                         vn = m.group(1)
1062                         v = self._vessel_lookup(vn, timestamp, dm, create=True)
1063                         self._lastvessel = self._vessel = vn
1064                         self._v = v
1065                         ob_x(pn, 'we boarded')
1066                         self.expire_garbage(timestamp)
1067                         return d(' '.join(dm))
1068
1069                 if self._v is None:
1070                         return d('no vessel')
1071
1072                 m = rm('(\\w+) has come aboard\\.$')
1073                 if m: return ob1('boarded');
1074
1075                 m = rm('You have ordered (\\w+) to do some (\\S.*\\S)\\.$')
1076                 if m:
1077                         (who,what) = m.groups()
1078                         pa = ob_x(who,'ord '+what)
1079                         if what == 'Gunning':
1080                                 pa.gunner = True
1081                         return d('duty order')
1082
1083                 m = rm('(\\w+) abandoned a (\\S.*\\S) station\\.$')
1084                 if m: oba('stopped'); return d("end")
1085
1086                 def chat_core(speaker, chan):
1087                         try: pa = self._pl[speaker]
1088                         except KeyError: return 'mystery'
1089                         if pa.v is not None and pa.v is not self._v:
1090                                 return 'elsewhere'
1091                         pa.last_chat_time = timestamp
1092                         pa.last_chat_chan = chan
1093                         self.force_redisplay()
1094                         return 'here'
1095
1096                 def chat(chan):
1097                         speaker = m.group(1)
1098                         dm = chat_core(speaker, chan)
1099                         return d('chat %s %s' % (chan, dm))
1100
1101                 def chat_metacmd(chan):
1102                         (cmdr, metacmd) = m.groups()
1103                         whynot = self._command(
1104                                 cmdr, metacmd, chan, timestamp, d)
1105                         if whynot is not None:
1106                                 return chat(chan)
1107                         else:
1108                                 chat_core(cmdr, 'cmd '+chan)
1109
1110                 m = rm('(\\w+) (?:issued an order|ordered everyone) "')
1111                 if m: return ob1('general order');
1112
1113                 m = rm('(\\w+) says, "')
1114                 if m: return chat('public')
1115
1116                 m = rm('(\\w+) tells ye, "')
1117                 if m: return chat('private')
1118
1119                 m = rm('Ye told (\\w+), "(.*)"$')
1120                 if m: return chat_metacmd('private')
1121
1122                 m = rm('(\\w+) flag officer chats, "')
1123                 if m: return chat('flag officer')
1124
1125                 m = rm('(\\w+) officer chats, "(.*)"$')
1126                 if m: return chat_metacmd('officer')
1127
1128                 m = rm('Ye accepted the offer to job with ')
1129                 if m: return disembark_me('jobbing')
1130
1131                 m = rm('Ye hop on the ferry and are whisked away ')
1132                 if m: return disembark_me('ferry')
1133
1134                 m = rm('Whisking away to yer home on the magical winds')
1135                 if m: return disembark_me('home')
1136
1137                 m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
1138                 if m:
1139                         pl = m.group(1).split(', ')
1140                         if not self._myself.name in pl:
1141                                 return d('lost melee')
1142                         for pn in pl:
1143                                 if ' ' in pn: continue
1144                                 ob_x(pn,'won melee')
1145                         return d('won melee')
1146
1147                 m = rm('(\\w+) is eliminated\\!')
1148                 if m: return ob1('eliminated in fray');
1149
1150                 m = rm('(\\w+) has driven \w+ from the ship\\!')
1151                 if m: return ob1('boarder repelled');
1152
1153                 m = rm('\w+ has bested (\\w+), and turns'+
1154                         ' to the rest of the ship\\.')
1155                 if m: return ob1('boarder unrepelled');
1156
1157                 pirate = rm_crew("(\\w+) has taken a job with '(.*)'\\.")
1158                 if pirate: return jb(pirate, 'ashore')
1159
1160                 pirate = rm_crew("(\\w+) has left '(.*)'\\.")
1161                 if pirate:
1162                         disembark(self._v, timestamp, pirate, 'left crew')
1163                         return d('left crew')
1164
1165                 m = rm('(\w+) has applied for the posted job\.')
1166                 if m: return jb(m.group(1), 'applied')
1167
1168                 pirate= rm_crew("(\\w+) has been invited to job for '(.*)'\\.")
1169                 if pirate: return jb(pirate, 'invited')
1170
1171                 pirate = rm_crew("(\\w+) declined the job offer for '(.*)'\\.")
1172                 if pirate: return jb(pirate, 'declined')
1173
1174                 m = rm('(\\w+) has left the vessel\.')
1175                 if m:
1176                         pirate = m.group(1)
1177                         disembark(self._v, timestamp, pirate, 'disembarked')
1178                         return d('disembarked')
1179
1180                 return d('not-matched')
1181
1182         def _str_pa(self, pn, pa):
1183                 assert self._pl[pn] == pa
1184                 s = ' '*20 + "%s %-*s %13s %-30s %13s %-20s %13s" % (
1185                         (' ','G')[pa.gunner],
1186                         max_pirate_namelen, pn,
1187                         pa.last_time, pa.last_event,
1188                         pa.last_chat_time, pa.last_chat_chan,
1189                         pa.jobber)
1190                 if pa.expires is not None:
1191                         s += " %-5d" % (pa.expires - pa.last_time)
1192                 s += "\n"
1193                 return s
1194
1195         def _str_vessel(self, vn, v):
1196                 s = ' vessel %s\n' % vn
1197                 s += ' '*20 + "%-*s   %13s\n" % (
1198                                 max_pirate_namelen, '#lastinfo',
1199                                 v['#lastinfo'])
1200                 assert v['#name'] == vn
1201                 for pn in sorted(v.keys()):
1202                         if pn.startswith('#'): continue
1203                         pa = v[pn]
1204                         assert pa.v == v
1205                         s += self._str_pa(pn,pa)
1206                 return s
1207
1208         def __str__(self):
1209                 s = '''<ChatLogTracker
1210  myself %s
1211  vessel %s
1212 '''                     % (self._myself.name, self._vessel)
1213                 assert ((self._v is None and self._vessel is None) or
1214                         (self._v is self._vl[self._vessel]))
1215                 if self._vessel is not None:
1216                         s += self._str_vessel(self._vessel, self._v)
1217                 for vn in sorted(self._vl.keys()):
1218                         if vn == self._vessel: continue
1219                         s += self._str_vessel(vn, self._vl[vn])
1220                 s += " elsewhere\n"
1221                 for p in self._pl:
1222                         pa = self._pl[p]
1223                         if pa.v is not None:
1224                                 assert pa.v[p] is pa
1225                                 assert pa.v in self._vl.values()
1226                         else:
1227                                 s += self._str_pa(pa.name, pa)
1228                 s += '>\n'
1229                 return s
1230
1231         def catchup(self, progress=None):
1232                 while True:
1233                         more = self._f.readline()
1234                         if not more: break
1235
1236                         self._progress[0] += len(more)
1237                         if progress: progress.progress(*self._progress)
1238
1239                         self._lbuf += more
1240                         if self._lbuf.endswith('\n'):
1241                                 self.chatline(self._lbuf.rstrip())
1242                                 self._lbuf = ''
1243                                 if opts.debug >= 2:
1244                                         debug(self.__str__())
1245                 self._expire_jobbers(time.time())
1246
1247                 if progress: progress.caughtup()
1248
1249         def changed(self):
1250                 rv = self._need_redisplay
1251                 self._need_redisplay = False
1252                 return rv
1253         def myname(self):
1254                 # returns our pirate name
1255                 return self._myself.name
1256         def vesselname(self):
1257                 # returns the vessel name we're aboard or None
1258                 return self._vessel
1259         def lastvesselname(self):
1260                 # returns the last vessel name we were aboard or None
1261                 return self._lastvessel
1262         def aboard(self, vesselname=True):
1263                 # returns a list of PirateAboard the vessel
1264                 #  sorted by pirate name
1265                 #  you can pass this None and you'll get []
1266                 #  or True for the current vessel (which is the default)
1267                 #  the returned value is a fresh list of persistent
1268                 #  PirateAboard objects
1269                 if vesselname is True: v = self._v
1270                 else: v = self._vl.get(vesselname.title())
1271                 if v is None: return []
1272                 return [ v[pn]
1273                          for pn in sorted(v.keys())
1274                          if not pn.startswith('#') ]
1275         def jobbers(self):
1276                 # returns a the jobbers' PirateAboards,
1277                 # sorted by jobber class and reverse of expiry time
1278                 l = [ pa
1279                       for pa in self._pl.values()
1280                       if pa.jobber is not None
1281                     ]
1282                 def compar_key(pa):
1283                         return (pa.jobber, -pa.expires)
1284                 l.sort(key = compar_key)
1285                 return l
1286
1287 #---------- implementations of actual operation modes ----------
1288
1289 def do_pirate(pirates, bu):
1290         print '{'
1291         for pirate in pirates:
1292                 info = PirateInfo(pirate)
1293                 print '%s: %s,' % (`pirate`, info)
1294         print '}'
1295
1296 def prep_crew_of(args, bu, max_age=300):
1297         if len(args) != 1: bu('crew-of takes one pirate name')
1298         pi = PirateInfo(args[0], max_age)
1299         if pi.crew is None: return None
1300         return CrewInfo(pi.crew[0], max_age)
1301
1302 def do_crew_of(args, bu):
1303         ci = prep_crew_of(args, bu)
1304         print ci
1305
1306 def do_flag_of(args, bu):
1307         if len(args) != 1: bu('flag-of takes one pirate name')
1308         max_age = 300
1309         pi = PirateInfo(args[0], max_age)
1310         if pi.flag is None: fi = None
1311         else: fi = FlagInfo(pi.flag[0], max_age)
1312         print fi
1313
1314 def do_standings_crew_of(args, bu):
1315         ci = prep_crew_of(args, bu, 60)
1316         tab = StandingsTable(sys.stdout)
1317         tab.headings()
1318         for (rank, members) in ci.crew:
1319                 if not members: continue
1320                 tab.literalline('')
1321                 tab.literalline('%s:' % rank)
1322                 for p in members:
1323                         pi = PirateInfo(p, random.randint(900,1800))
1324                         tab.pirate(pi)
1325
1326 def do_ocean(args, bu):
1327         if (len(args)): bu('ocean takes no further arguments')
1328         fetcher.default_ocean()
1329         oi = OceanInfo(IslandExtendedInfo)
1330         print oi
1331         for islename in sorted(oi.islands.keys()):
1332                 isle = oi.islands[islename]
1333                 print isle
1334
1335 #----- modes which use the chat log parser are quite complex -----
1336
1337 class ProgressPrintPercentage:
1338         def __init__(self, f=sys.stdout):
1339                 self._f = f
1340         def progress_string(self,done,total):
1341                 return "scan chat logs %3d%%\r" % ((done*100) / total)
1342         def progress(self,*a):
1343                 self._f.write(self.progress_string(*a))
1344                 self._f.flush()
1345         def show_init(self, pirate, ocean):
1346                 print >>self._f, 'Starting up, %s on the %s ocean' % (
1347                         pirate, ocean)
1348         def caughtup(self):
1349                 self._f.write('                   \r')
1350                 self._f.flush()
1351
1352 def prep_chat_log(args, bu,
1353                 progress=ProgressPrintPercentage(),
1354                 max_myself_age=3600):
1355         if len(args) != 1: bu('this action takes only chat log filename')
1356         logfn = args[0]
1357         logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_'
1358         match = regexp.match(logfn_re, logfn)
1359         if not match: bu('chat log filename is not in expected format')
1360         (pirate, ocean) = match.groups()
1361         fetcher.default_ocean(ocean)
1362
1363         progress.show_init(pirate, fetcher.ocean)
1364         myself = PirateInfo(pirate,max_myself_age)
1365         track = ChatLogTracker(myself, logfn)
1366
1367         opts.debug -= 2
1368         track.catchup(progress)
1369         opts.debug += 2
1370
1371         track.force_redisplay()
1372
1373         return (myself, track)
1374
1375 def do_track_chat_log(args, bu):
1376         (myself, track) = prep_chat_log(args, bu)
1377         while True:
1378                 track.catchup()
1379                 if track.changed():
1380                         print track
1381                 sleep(0.5 + 0.5 * random.random())
1382
1383 #----- ship management aid -----
1384
1385 class Display_dumb(ProgressPrintPercentage):
1386         def __init__(self):
1387                 ProgressPrintPercentage.__init__(self)
1388         def show(self, s):
1389                 print '\n\n', s;
1390         def realstart(self):
1391                 pass
1392
1393 class Display_overwrite(ProgressPrintPercentage):
1394         def __init__(self):
1395                 ProgressPrintPercentage.__init__(self)
1396
1397                 null = file('/dev/null','w')
1398                 curses.setupterm(fd=null.fileno())
1399
1400                 self._clear = curses.tigetstr('clear')
1401                 if not self._clear:
1402                         self._debug('missing clear!')
1403                         self.show = Display_dumb.show
1404                         return
1405
1406                 self._t = {'el':'', 'ed':''}
1407                 if not self._init_sophisticated():
1408                         for k in self._t.keys(): self._t[k] = ''
1409                         self._t['ho'] = self._clear
1410
1411         def _debug(self,m): debug('display overwrite: '+m)
1412
1413         def _init_sophisticated(self):
1414                 for k in self._t.keys():
1415                         s = curses.tigetstr(k)
1416                         self._t[k] = s
1417                 self._t['ho'] = curses.tigetstr('ho')
1418                 if not self._t['ho']:
1419                         cup = curses.tigetstr('cup')
1420                         self._t['ho'] = curses.tparm(cup,0,0)
1421                 missing = [k for k in self._t.keys() if not self._t[k]]
1422                 if missing:
1423                         self.debug('missing '+(' '.join(missing)))
1424                         return 0
1425                 return 1
1426
1427         def show(self, s):
1428                 w = sys.stdout.write
1429                 def wti(k): w(self._t[k])
1430
1431                 wti('ho')
1432                 nl = ''
1433                 for l in s.rstrip().split('\n'):
1434                         w(nl)
1435                         w(l)
1436                         wti('el')
1437                         nl = '\r\n'
1438                 wti('ed')
1439                 w(' ')
1440                 sys.stdout.flush()
1441
1442         def realstart(self):
1443                 sys.stdout.write(self._clear)
1444                 sys.stdout.flush()
1445
1446
1447 def do_ship_aid(args, bu):
1448         if opts.ship_duty is None: opts.ship_duty = True
1449
1450         displayer = globals()['Display_'+opts.display]()
1451
1452         (myself, track) = prep_chat_log(args, bu, progress=displayer)
1453
1454         displayer.realstart()
1455
1456         if os.isatty(0): kr_create = KeystrokeReader
1457         else: kr_create = DummyKeystrokeReader
1458
1459         try:
1460                 kreader = kr_create(0, 10)
1461                 ship_aid_core(myself, track, displayer, kreader)
1462         finally:
1463                 kreader.stop()
1464                 print '\n'
1465
1466 class KeyBasedSorter:
1467         def compar_key_pa(self, pa):
1468                 pi = pa.pirate_info()
1469                 if pi is None: return None
1470                 return self.compar_key(pi)
1471         def lsort_pa(self, l):
1472                 l.sort(key = self.compar_key_pa)
1473
1474 class NameSorter(KeyBasedSorter):
1475         def compar_key(self, pi): return pi.name
1476         def desc(self): return 'name'
1477
1478 class SkillSorter(NameSorter):
1479         def __init__(self, relevant):
1480                 self._want = frozenset(relevant.split('/'))
1481                 self._avoid = set()
1482                 for p in core_duty_puzzles:
1483                         if isinstance(p,basestring): self._avoid.add(p)
1484                         else: self._avoid |= set(p)
1485                 self._avoid -= self._want
1486                 self._desc = '%s' % relevant
1487
1488         def desc(self): return self._desc
1489
1490         def compar_key(self, pi):
1491                 best_want = max([
1492                         pi.standings.get(puz,-1)
1493                         for puz in self._want
1494                         ])
1495                 best_avoid = [
1496                         -pi.standings.get(puz,standing_limit)
1497                         for puz in self._avoid
1498                         ]
1499                 best_avoid.sort()
1500                 def negate(x): return -x
1501                 debug('compar_key %s bw=%s ba=%s' % (pi.name, `best_want`,
1502                         `best_avoid`))
1503                 return (-best_want, map(negate, best_avoid), pi.name)
1504
1505 def ship_aid_core(myself, track, displayer, kreader):
1506
1507         def find_vessel():
1508                 vn = track.vesselname()
1509                 if vn: return (vn, " on board the %s" % vn)
1510                 vn = track.lastvesselname()
1511                 if vn: return (vn, " ashore from the %s" % vn)
1512                 return (None, " not on a vessel")
1513
1514         def timeevent(t,e):
1515                 if t is None: return ' ' * 22
1516                 return " %-4s %-16s" % (format_time_interval(now - t),e)
1517
1518         displayer.show(track.myname() + find_vessel()[1] + '...')
1519
1520         rotate_nya = '/-\\'
1521
1522         sort = NameSorter()
1523         clicmd = None
1524         clierr = None
1525         cliexec = None
1526
1527         while True:
1528                 track.catchup()
1529                 now = time.time()
1530
1531                 (vn, vs) = find_vessel()
1532
1533                 s = ''
1534                 if cliexec is not None:
1535                         s += '...'
1536                 elif clierr is not None:
1537                         s += 'Error: '+clierr
1538                 elif clicmd is not None:
1539                         s += '/' + clicmd
1540                 else:
1541                         s = track.myname() + vs
1542                         s += " at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
1543                         s += kreader.info()
1544                 s += '\n'
1545
1546                 tbl_s = StringIO()
1547                 tbl = StandingsTable(tbl_s)
1548
1549                 aboard = track.aboard(vn)
1550                 sort.lsort_pa(aboard)
1551
1552                 jobbers = track.jobbers()
1553
1554                 if track.vesselname(): howmany = 'aboard: %2d' % len(aboard)
1555                 else: howmany = ''
1556
1557                 tbl.headings(howmany, '  sorted by '+sort.desc())
1558
1559                 last_jobber = None
1560
1561                 for pa in aboard + jobbers:
1562                         if pa.jobber != last_jobber:
1563                                 last_jobber = pa.jobber
1564                                 tbl.literalline('')
1565                                 tbl.literalline('jobbers '+last_jobber)
1566
1567                         pi = pa.pirate_info()
1568
1569                         xs = ''
1570                         if pa.gunner: xs += 'G '
1571                         else: xs += '  '
1572                         xs += timeevent(pa.last_time, pa.last_event)
1573                         xs += timeevent(pa.last_chat_time, pa.last_chat_chan)
1574
1575                         if pi is None:
1576                                 tbl.pirate_dummy(pa.name, rotate_nya[0], xs)
1577                         else:
1578                                 tbl.pirate(pi, xs)
1579
1580                 s += tbl_s.getvalue()
1581                 displayer.show(s)
1582                 tbl_s.close()
1583
1584                 if cliexec is not None:
1585                         clierr = track.local_command("/"+cliexec.strip())
1586                         cliexec = None
1587                         continue
1588
1589                 k = kreader.getch()
1590                 if k is None:
1591                         rotate_nya = rotate_nya[1:3] + rotate_nya[0]
1592                         continue
1593
1594                 if clierr is not None:
1595                         clierr = None
1596                         continue
1597
1598                 if clicmd is not None:
1599                         if k == '\r' or k == '\n':
1600                                 cliexec = clicmd
1601                                 clicmd = clicmdbase
1602                         elif k == '\e' and clicmd != "":
1603                                 clicmd = clicmdbase
1604                         elif k == '\33':
1605                                 clicmd = None
1606                         elif k == '\b' or k == '\177':
1607                                 clicmd = clicmd[ 0 : len(clicmd)-1 ]
1608                         else:
1609                                 clicmd += k
1610                         continue
1611
1612                 if k == 'q': break
1613                 elif k == 'g': sort = SkillSorter('Gunning')
1614                 elif k == 'c': sort = SkillSorter('Carpentry')
1615                 elif k == 's': sort = SkillSorter('Sailing/Rigging')
1616                 elif k == 'b': sort = SkillSorter('Bilging')
1617                 elif k == 'n': sort = SkillSorter('Navigating')
1618                 elif k == 'd': sort = SkillSorter('Battle Navigation')
1619                 elif k == 't': sort = SkillSorter('Treasure Haul')
1620                 elif k == 'a': sort = NameSorter()
1621                 elif k == '/': clicmdbase = ""; clicmd = clicmdbase
1622                 elif k == '+': clicmdbase = "a "; clicmd = clicmdbase
1623                 else: pass # unknown key command
1624
1625 #---------- individual keystroke input ----------
1626
1627 class DummyKeystrokeReader:
1628         def __init__(self,fd,timeout_dummy): pass
1629         def stop(self): pass
1630         def getch(self): sleep(1); return None
1631         def info(self): return ' [noninteractive]'
1632
1633 class KeystrokeReader(DummyKeystrokeReader):
1634         def __init__(self, fd, timeout_decisec=0):
1635                 self._fd = fd
1636                 self._saved = termios.tcgetattr(fd)
1637                 a = termios.tcgetattr(fd)
1638                 a[3] &= ~(termios.ECHO | termios.ECHONL |
1639                           termios.ICANON | termios.IEXTEN)
1640                 a[6][termios.VMIN] = 0
1641                 a[6][termios.VTIME] = timeout_decisec
1642                 termios.tcsetattr(fd, termios.TCSANOW, a)
1643         def stop(self):
1644                 termios.tcsetattr(self._fd, termios.TCSANOW, self._saved)
1645         def getch(self):
1646                 debug_flush()
1647                 byte = os.read(self._fd, 1)
1648                 if not len(byte): return None
1649                 return byte
1650         def info(self):
1651                 return ''
1652
1653 #---------- main program ----------
1654
1655 def main():
1656         global opts, fetcher, yppedia
1657
1658         pa = OptionParser(
1659 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
1660 actions:
1661  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
1662  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
1663  yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
1664  yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG
1665  yoweb-scrape [options] ship-aid CHAT-LOG  (must be .../PIRATE_OCEAN_chat-log*)
1666
1667 display modes (for --display) apply to ship-aid:
1668  --display=dumb       just print new information, scrolling the screen
1669  --display=overwrite  use cursor motion, selective clear, etc. to redraw at top''')
1670         ao = pa.add_option
1671         ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
1672                 help='select ocean OCEAN')
1673         ao('--cache-dir', dest='cache_dir', metavar='DIR',
1674                 default='~/.yoweb-scrape-cache',
1675                 help='cache yoweb pages in DIR')
1676         ao('-D','--debug', action='count', dest='debug', default=0,
1677                 help='enable debugging output')
1678         ao('--debug-fd', type='int', dest='debug_fd',
1679                 help='write any debugging output to specified fd')
1680         ao('-q','--quiet', action='store_true', dest='quiet',
1681                 help='suppress warning output')
1682         ao('--display', action='store', dest='display',
1683                 type='choice', choices=['dumb','overwrite'],
1684                 help='how to display ship aid')
1685         ao('--local-ypp-dir', action='store', dest='localhtml',
1686                 help='get yppedia pages from local directory LOCALHTML'+
1687                         ' instead of via HTTP')
1688
1689         ao_jt = lambda wh, t: ao(
1690                 '--timeout-sa-'+wh, action='store', dest='timeout_'+wh,
1691                 default=t, help=('set timeout for expiring %s jobbers' % wh))
1692         ao_jt('applied',      120)
1693         ao_jt('invited',      120)
1694         ao_jt('declined',      30)
1695         ao_jt('ashore',      1800)
1696
1697         ao('--ship-duty', action='store_true', dest='ship_duty',
1698                 help='show ship duty station puzzles')
1699         ao('--all-puzzles', action='store_false', dest='ship_duty',
1700                 help='show all puzzles, not just ship duty stations')
1701
1702         ao('--min-cache-reuse', type='int', dest='min_max_age',
1703                 metavar='SECONDS', default=60,
1704                 help='always reuse cache yoweb data if no older than this')
1705
1706         (opts,args) = pa.parse_args()
1707         random.seed()
1708
1709         if len(args) < 1:
1710                 print >>sys.stderr, copyright_info
1711                 pa.error('need a mode argument')
1712
1713         if opts.debug_fd is not None:
1714                 opts.debug_file = os.fdopen(opts.debug_fd, 'w')
1715         else:
1716                 opts.debug_file = sys.stdout
1717
1718         mode = args[0]
1719         mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
1720         try: mode_fn = globals()[mode_fn_name]
1721         except KeyError: pa.error('unknown mode "%s"' % mode)
1722
1723         # fixed parameters
1724         opts.expire_age = max(3600, opts.min_max_age)
1725
1726         opts.ship_reboard_clearout = 3600
1727
1728         if opts.cache_dir.startswith('~/'):
1729                 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
1730
1731         if opts.display is None:
1732                 if ((opts.debug > 0 and opts.debug_fd is None)
1733                     or not os.isatty(sys.stdout.fileno())):
1734                         opts.display = 'dumb'
1735                 else:
1736                         opts.display = 'overwrite'
1737
1738         fetcher = Yoweb(opts.ocean, opts.cache_dir)
1739         yppedia = Yppedia(opts.cache_dir)
1740
1741         mode_fn(args[1:], pa.error)
1742
1743 main()