yoweb-scrape

   1 #!/usr/bin/python
   2 # This is part of ypp-sc-tools, a set of third-party tools for assisting
   3 # players of Yohoho Puzzle Pirates.
   4 #
   5 # Copyright (C) 2009 Ian Jackson <ijackson@chiark.greenend.org.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 #
  20 # Yohoho and Puzzle Pirates are probably trademarks of Three Rings and
  21 # are used without permission.  This program is not endorsed or
  22 # sponsored by Three Rings.
  23
  24 copyright_info = '''
  25 yoweb-scrape is part of ypp-sc-tools  Copyright (C) 2009 Ian Jackson
  26 This program comes with ABSOLUTELY NO WARRANTY; this is free software,
  27 and you are welcome to redistribute it under certain conditions.
  28 For details, read the top of the yoweb-scrape file.
  29 '''
  30
  31 #---------- setup ----------
  32
  33 import signal
  34 signal.signal(signal.SIGINT, signal.SIG_DFL)
  35
  36 import os
  37 import time
  38 import urllib
  39 import urllib2
  40 import errno
  41 import sys
  42 import re as regexp
  43 import random
  44 import curses
  45 import termios
  46 import random
  47 import subprocess
  48 from optparse import OptionParser
  49 from StringIO import StringIO
  50
  51 from BeautifulSoup import BeautifulSoup
  52
  53 opts = None
  54
  55 #---------- YPP parameters and arrays ----------
  56
  57 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
  58         '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
  59         '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
  60         '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
  61
  62 core_duty_puzzles = [
  63                 'Gunning',
  64                 ['Sailing','Rigging'],
  65                 'Bilging',
  66                 'Carpentry',
  67                 ]
  68
  69 duty_puzzles = ([ 'Navigating', 'Battle Navigation' ] +
  70                 core_duty_puzzles +
  71                 [ 'Treasure Haul' ])
  72
  73 standingvals = ('Able/Proficient/Distinguished/Respected/Master'+
  74                 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
  75 standing_limit = len(standingvals)
  76
  77 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
  78
  79 max_pirate_namelen = 12
  80
  81
  82 #---------- general utilities ----------
  83
  84 def debug(m):
  85         if opts.debug > 0:
  86                 print >>opts.debug_file, m
  87
  88 def debug_flush():
  89         if opts.debug > 0:
  90                 opts.debug_file.flush()
  91
  92 def sleep(seconds):
  93         debug_flush()
  94         time.sleep(seconds)
  95
  96 def format_time_interval(ti):
  97         if ti < 120: return '%d:%02d' % (ti / 60, ti % 60)
  98         if ti < 7200: return '%2dm' % (ti / 60)
  99         if ti < 86400: return '%dh' % (ti / 3600)
 100         return '%dd' % (ti / 86400)
 101
 102 def yppsc_dir():
 103         lib = os.getenv("YPPSC_YARRG_SRCBASE")
 104         if lib is not None: return lib
 105         lib = sys.argv[0]
 106         lib = regexp.sub('/[^/]+$', '', lib)
 107         os.environ["YPPSC_YARRG_SRCBASE"] = lib
 108         return lib
 109
 110 #---------- caching and rate-limiting data fetcher ----------
 111
 112 class Fetcher:
 113         def __init__(self, cachedir):
 114                 debug('Fetcher init %s' % cachedir)
 115                 self.cachedir = cachedir
 116                 try: os.mkdir(cachedir)
 117                 except (OSError,IOError), oe:
 118                         if oe.errno != errno.EEXIST: raise
 119                 self._cache_scan(time.time())
 120
 121         def _cache_scan(self, now):
 122                 # returns list of ages, unsorted
 123                 ages = []
 124                 debug('Fetcher   scan_cache')
 125                 for leaf in os.listdir(self.cachedir):
 126                         if not leaf.startswith('#'): continue
 127                         path = self.cachedir + '/' + leaf
 128                         try: s = os.stat(path)
 129                         except (OSError,IOError), oe:
 130                                 if oe.errno != errno.ENOENT: raise
 131                                 continue
 132                         age = now - s.st_mtime
 133                         if age > opts.expire_age:
 134                                 debug('Fetcher    expire %d %s' % (age, path))
 135                                 try: os.remove(path)
 136                                 except (OSError,IOError), oe:
 137                                         if oe.errno != errno.ENOENT: raise
 138                                 continue
 139                         ages.append(age)
 140                 return ages
 141
 142         def need_wait(self, now, imaginary=[]):
 143                 ages = self._cache_scan(now)
 144                 ages += imaginary
 145                 ages.sort()
 146                 debug('Fetcher   ages ' + `ages`)
 147                 min_age = 1
 148                 need_wait = 0
 149                 for age in ages:
 150                         if age < min_age and age <= 5:
 151                                 debug('Fetcher   morewait min=%d age=%d' %
 152                                         (min_age, age))
 153                                 need_wait = max(need_wait, min_age - age)
 154                         min_age += 3
 155                         min_age *= 1.25
 156                 if need_wait > 0:
 157                         need_wait += random.random() - 0.5
 158                 return need_wait
 159
 160         def _rate_limit_cache_clean(self, now):
 161                 need_wait = self.need_wait(now)
 162                 if need_wait > 0:
 163                         debug('Fetcher   wait %d' % need_wait)
 164                         sleep(need_wait)
 165
 166         def fetch(self, url, max_age):
 167                 debug('Fetcher fetch %s' % url)
 168                 cache_corename = urllib.quote_plus(url)
 169                 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
 170                 try: f = file(cache_item, 'r')
 171                 except (OSError,IOError), oe:
 172                         if oe.errno != errno.ENOENT: raise
 173                         f = None
 174                 now = time.time()
 175                 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
 176                 if f is not None:
 177                         s = os.fstat(f.fileno())
 178                         age = now - s.st_mtime
 179                         if age > max_age:
 180                                 debug('Fetcher  stale %d < %d'% (max_age, age))
 181                                 f = None
 182                 if f is not None:
 183                         data = f.read()
 184                         f.close()
 185                         debug('Fetcher  cached %d > %d' % (max_age, age))
 186                         return data
 187
 188                 debug('Fetcher  fetch')
 189                 self._rate_limit_cache_clean(now)
 190
 191                 stream = urllib2.urlopen(url)
 192                 data = stream.read()
 193                 cache_tmp = "%s/#%s~%d#" % (
 194                         self.cachedir, cache_corename, os.getpid())
 195                 f = file(cache_tmp, 'w')
 196                 f.write(data)
 197                 f.close()
 198                 os.rename(cache_tmp, cache_item)
 199                 debug('Fetcher  stored')
 200                 return data
 201
 202 class Yoweb(Fetcher):
 203         def __init__(self, ocean, cachedir):
 204                 debug('Yoweb init %s' % cachedir)
 205                 self.ocean = ocean
 206                 Fetcher.__init__(self, cachedir)
 207
 208         def default_ocean(self, ocean='ice'):
 209                 if self.ocean is None:
 210                         self.ocean = ocean
 211
 212         def yoweb(self, kind, tail, max_age):
 213                 self.default_ocean()
 214                 assert(self.ocean)
 215                 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
 216                         self.ocean, kind, tail)
 217                 return self.fetch(url, max_age)
 218
 219 class Yppedia(Fetcher):
 220         def __init__(self, cachedir):
 221                 debug('Yoweb init %s' % cachedir)
 222                 self.base = 'http://yppedia.puzzlepirates.com/'
 223                 self.localhtml = opts.localhtml
 224                 Fetcher.__init__(self, cachedir)
 225
 226         def __call__(self, rhs):
 227                 if self.localhtml is None:
 228                         url = self.base + rhs
 229                         debug('Yppedia retrieving YPP '+url);
 230                         return self.fetch(url, 3000)
 231                 else:
 232                         return file(opts.localhtml + '/' + rhs, 'r')
 233
 234 #---------- logging assistance for troubled screenscrapers ----------
 235
 236 class SoupLog:
 237         def __init__(self):
 238                 self.msgs = [ ]
 239         def msg(self, m):
 240                 self.msgs.append(m)
 241         def soupm(self, obj, m):
 242                 self.msg(m + '; in ' + `obj`)
 243         def needs_msgs(self, child_souplog):
 244                 self.msgs += child_souplog.msgs
 245                 child_souplog.msgs = [ ]
 246
 247 def soup_text(obj):
 248         str = ''.join(obj.findAll(text=True))
 249         return str.strip()
 250
 251 class SomethingSoupInfo(SoupLog):
 252         def __init__(self, kind, tail, max_age):
 253                 SoupLog.__init__(self)
 254                 html = fetcher.yoweb(kind, tail, max_age)
 255                 self._soup = BeautifulSoup(html,
 256                         convertEntities=BeautifulSoup.HTML_ENTITIES
 257                         )
 258
 259 #---------- scraper for pirate pages ----------
 260
 261 class PirateInfo(SomethingSoupInfo):
 262         # Public data members:
 263         #  pi.standings = { 'Treasure Haul': 'Able' ... }
 264         #  pi.name = name
 265         #  pi.crew = (id, name)
 266         #  pi.flag = (id, name)
 267         #  pi.msgs = [ 'message describing problem with scrape' ]
 268
 269         def __init__(self, pirate, max_age=300):
 270                 SomethingSoupInfo.__init__(self,
 271                         'pirate.wm?target=', pirate, max_age)
 272                 self.name = pirate
 273                 self._find_standings()
 274                 self.crew = self._find_crewflag('crew',
 275                         '^/yoweb/crew/info\\.wm')
 276                 self.flag = self._find_crewflag('flag',
 277                         '^/yoweb/flag/info\\.wm')
 278
 279         def _find_standings(self):
 280                 imgs = self._soup.findAll('img',
 281                         src=regexp.compile('/yoweb/images/stat.*'))
 282                 re = regexp.compile(
 283 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
 284                         )
 285                 standings = { }
 286
 287                 for skill in puzzles:
 288                         standings[skill] = [ ]
 289
 290                 skl = SoupLog()
 291
 292                 for img in imgs:
 293                         try: puzzle = img['alt']
 294                         except KeyError: continue
 295
 296                         if not puzzle in puzzles:
 297                                 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
 298                                 continue
 299                         key = img.findParent('td')
 300                         if key is None:
 301                                 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
 302                                 continue
 303                         valelem = key.findNextSibling('td')
 304                         if valelem is None:
 305                                 skl.soupm(key, 'puzzle missing sibling "%s"'
 306                                         % puzzle)
 307                                 continue
 308                         valstr = soup_text(valelem)
 309                         match = re.match(valstr)
 310                         if match is None:
 311                                 skl.soupm(key, ('puzzle "%s" unparseable'+
 312                                         ' standing "%s"') % (puzzle, valstr))
 313                                 continue
 314                         standing = match.group(match.lastindex)
 315                         standings[puzzle].append(standing)
 316
 317                 self.standings = { }
 318
 319                 for puzzle in puzzles:
 320                         sl = standings[puzzle]
 321                         if len(sl) > 1:
 322                                 skl.msg('puzzle "%s" multiple standings %s' %
 323                                                 (puzzle, `sl`))
 324                                 continue
 325                         if not sl:
 326                                 skl.msg('puzzle "%s" no standing found' % puzzle)
 327                                 continue
 328                         standing = sl[0]
 329                         for i in range(0, standing_limit):
 330                                 if standing == standingvals[i]:
 331                                         self.standings[puzzle] = i
 332                         if not puzzle in self.standings:
 333                                 skl.msg('puzzle "%s" unknown standing "%s"' %
 334                                         (puzzle, standing))
 335
 336                 all_standings_ok = True
 337                 for puzzle in puzzles:
 338                         if not puzzle in self.standings:
 339                                 self.needs_msgs(skl)
 340
 341         def _find_crewflag(self, cf, yoweb_re):
 342                 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
 343                 if len(things) != 1:
 344                         self.msg('zero or several %s id references found' % cf)
 345                         return None
 346                 thing = things[0]
 347                 id_re = '\\b%sid\\=(\\w+)$' % cf
 348                 id_haystack = thing['href']
 349                 match = regexp.compile(id_re).search(id_haystack)
 350                 if match is None:
 351                         self.soupm(thing, ('incomprehensible %s id ref'+
 352                                 ' (%s in %s)') % (cf, id_re, id_haystack))
 353                         return None
 354                 name = soup_text(thing)
 355                 return (match.group(1), name)
 356
 357         def __str__(self):
 358                 return `(self.crew, self.flag, self.standings, self.msgs)`
 359
 360 #---------- scraper for crew pages ----------
 361
 362 class CrewInfo(SomethingSoupInfo):
 363         # Public data members:
 364         #  ci.crewid
 365         #  ci.crew = [ ('Captain',        ['Pirate', ...]),
 366         #              ('Senior Officer', [...]),
 367         #               ... ]
 368         #  pi.msgs = [ 'message describing problem with scrape' ]
 369
 370         def __init__(self, crewid, max_age=300):
 371                 self.crewid = crewid
 372                 SomethingSoupInfo.__init__(self,
 373                         'crew/info.wm?crewid=', crewid, max_age)
 374                 self._find_crew()
 375
 376         def _find_crew(self):
 377                 self.crew = []
 378                 capts = self._soup.findAll('img',
 379                         src='/yoweb/images/crew-captain.png')
 380                 if len(capts) != 1:
 381                         self.msg('crew members: no. of captain images != 1')
 382                         return
 383                 tbl = capts[0]
 384                 while not tbl.find('a', href=pirate_ref_re):
 385                         tbl = tbl.findParent('table')
 386                         if not tbl:
 387                                 self.msg('crew members: cannot find table')
 388                                 return
 389                 current_rank_crew = None
 390                 crew_rank_re = regexp.compile('/yoweb/images/crew')
 391                 for row in tbl.contents:
 392                         # findAll(recurse=False)
 393                         if isinstance(row,basestring):
 394                                 continue
 395
 396                         is_rank = row.find('img', attrs={'src': crew_rank_re})
 397                         if is_rank:
 398                                 rank = soup_text(row)
 399                                 current_rank_crew = []
 400                                 self.crew.append((rank, current_rank_crew))
 401                                 continue
 402                         for cell in row.findAll('a', href=pirate_ref_re):
 403                                 if current_rank_crew is None:
 404                                         self.soupm(cell, 'crew members: crew'
 405                                                 ' before rank')
 406                                         continue
 407                                 current_rank_crew.append(soup_text(cell))
 408
 409         def __str__(self):
 410                 return `(self.crew, self.msgs)`
 411
 412 class FlagRelation():
 413         # Public data members (put there by hand by creater)
 414         #       other_flagname
 415         #       other_flagid
 416         #       yoweb_heading
 417         #       this_declaring
 418         #       other_declaring_min
 419         #       other_declaring_max
 420         # where {this,other}_declaring{,_min,_max} are:
 421         #       -1      {this,other} is declaring war
 422         #        0      {this,other} is not doing either
 423         #       +1      {this,other} is allying
 424         def __repr__(self):
 425                 return '<FlagRelation %s %d/%d..%d %s %s>' % (
 426                         self.yoweb_heading, self.this_declaring,
 427                         self.other_declaring_min, self.other_declaring_max,
 428                         self.other_flagname, self.other_flagid)
 429
 430 class FlagInfo(SomethingSoupInfo):
 431         # Public data members (after init):
 432         #
 433         #   flagid
 434         #   name        #               string
 435         #
 436         #   relations[n] = FlagRelation
 437         #   relation_byname[otherflagname] = relations[some_n]
 438         #   relation_byid[otherflagname] = relations[some_n]
 439         #
 440         #   islands[n] = (islandname, islandid)
 441         #
 442         def __init__(self, flagid, max_age=600):
 443                 self.flagid = flagid
 444                 SomethingSoupInfo.__init__(self,
 445                         'flag/info.wm?flagid=', flagid, max_age)
 446                 self._find_flag()
 447
 448         def _find_flag(self):
 449                 font2 = self._soup.find('font',{'size':'+2'})
 450                 self.name = font2.find('b').contents[0]
 451
 452                 self.relations = [ ]
 453                 self.relation_byname = { }
 454                 self.relation_byid = { }
 455                 self.islands = [ ]
 456
 457                 magnate = self._soup.find('img',{'src':
 458                         '/yoweb/images/repute-MAGNATE.png'})
 459                 warinfo = (magnate.findParent('table').findParent('tr').
 460                         findNextSibling('tr').findNext('td',{'align':'left'}))
 461
 462                 def warn(m):
 463                         print >>sys.stderr, 'WARNING: '+m
 464
 465                 def wi_warn(head, waritem):
 466                         warn('unknown warmap item: %s: %s' %
 467                                 (`head`, ``waritem``))
 468
 469                 def wihelp_item(waritem, thing):
 470                         url = waritem.get('href', None)
 471                         if url is None:
 472                                 return ('no url for '+thing,None,None)
 473                         m = regexp.search('\?'+thing+'id=(\d+)$', url)
 474                         if not m: return ('no '+thing+'id',None,None)
 475                         tid = m.group(1)
 476                         tname = waritem.string
 477                         if tname is None:
 478                                 return (thing+' name not just string',None,None)
 479                         return (None,tid,tname)
 480
 481                 def wi_alwar(head, waritem, thisdecl, othermin, othermax):
 482                         (err,flagid,flagname) = wihelp_item(waritem,'flag')
 483                         if err: return err
 484                         rel = self.relation_byid.get(flagid, None)
 485                         if rel: return 'flag id twice!'
 486                         if flagname in self.relation_byname:
 487                                 return 'flag name twice!'
 488                         rel = FlagRelation()
 489                         rel.other_flagname = flagname
 490                         rel.other_flagid = flagid
 491                         rel.yoweb_heading = head
 492                         rel.this_declaring = thisdecl
 493                         rel.other_declaring_min = othermin
 494                         rel.other_declaring_max = othermax
 495                         self.relations.append(rel)
 496                         self.relation_byid[flagid] = rel
 497                         self.relation_byname[flagid] = rel
 498
 499                 def wi_isle(head, waritem):
 500                         (err,isleid,islename) = wihelp_item(waritem,'island')
 501                         if err: return err
 502                         self.islands.append((isleid,islename))
 503
 504                 warmap = {
 505                         'Allied with':                  (wi_alwar,+1,+1,+1),
 506                         'Declaring war against':        (wi_alwar,-1, 0,+1),
 507                         'At war with':                  (wi_alwar,-1,-1,-1),
 508                         'Trying to form an alliance with': (wi_alwar,+1,-1,0),
 509                         'Islands controlled by this flag': (wi_isle,),
 510                         }
 511
 512                 how = (wi_warn, None)
 513
 514                 for waritem in warinfo.findAll(['font','a']):
 515                         if waritem is None: break
 516                         if waritem.name == 'font':
 517                                 colour = waritem.get('color',None)
 518                                 if colour.lstrip('#') != '958A5F':
 519                                         warn('strange colour %s in %s' %
 520                                                 (colour,``waritem``))
 521                                         continue
 522                                 head = waritem.string
 523                                 if head is None:
 524                                         warn('no head string in '+``waritem``)
 525                                         continue
 526                                 head = regexp.sub('\\s+', ' ', head).strip()
 527                                 head = head.rstrip(':')
 528                                 how = (head,) + warmap.get(head, (wi_warn,))
 529                                 continue
 530                         assert(waritem.name == 'a')
 531
 532                         debug('WARHOW %s(%s, waritem, *%s)' %
 533                                 (how[1], `how[0]`, `how[2:]`))
 534                         bad = how[1](how[0], waritem, *how[2:])
 535                         if bad:
 536                                 warn('bad waritem %s: %s: %s' % (`how[0]`,
 537                                         bad, ``waritem``))
 538
 539         def __str__(self):
 540                 return `(self.name, self.islands, self.relations)`
 541
 542 #---------- scraper for ocean info incl. embargoes etc. ----------
 543
 544 class IslandBasicInfo():
 545         # Public data attributes:
 546         #  ocean
 547         #  name
 548         # Public data attributes maybe set by caller:
 549         #  arch
 550         def __init__(self, ocean, islename):
 551                 self.ocean = ocean
 552                 self.name = islename
 553         def yppedia(self):
 554                 def q(x): return urllib.quote(x.replace(' ','_'))
 555                 url_rhs = q(self.name) + '_(' + q(self.ocean) + ')'
 556                 return yppedia(url_rhs)
 557         def __str__(self):
 558                 return `(self.ocean, self.name)`
 559
 560 class IslandExtendedInfo(IslandBasicInfo):
 561         # Public data attributes (inherited):
 562         #  ocean
 563         #  name
 564         # Public data attributes (additional):
 565         #  islandid
 566         #  yoweb_url
 567         #  flagid
 568         def __init__(self, ocean, islename):
 569                 IslandBasicInfo.__init__(self, ocean, islename)
 570                 self.islandid = None
 571                 self.yoweb_url = None
 572                 self._collect_yoweb()
 573                 self._collect_flagid()
 574
 575         def _collect_yoweb(self):
 576                 debug('IEI COLLECT YOWEB '+`self.name`)
 577                 self.islandid = None
 578                 self.yoweb_url = None
 579
 580                 soup = BeautifulSoup(self.yppedia())
 581                 content = soup.find('div', attrs = {'id': 'content'})
 582                 yoweb_re = regexp.compile('^http://\w+\.puzzlepirates\.com/'+
 583                         'yoweb/island/info\.wm\?islandid=(\d+)$')
 584                 a = soup.find('a', attrs = { 'href': yoweb_re })
 585                 if a is None:
 586                         debug('IEI COLLECT YOWEB '+`self.name`+' NONE')
 587                         return
 588
 589                 debug('IEI COLLECT YOWEB '+`self.name`+' GOT '+``a``)
 590                 self.yoweb_url = a['href']
 591                 m = yoweb_re.search(self.yoweb_url)
 592                 self.islandid = m.group(1)
 593
 594         def _collect_flagid(self):
 595                 self.flagid = None
 596
 597                 yo = self.yoweb_url
 598                 debug('IEI COLLECT FLAGID '+`self.name`+' URL '+`yo`)
 599                 if yo is None: return None
 600                 dataf = fetcher.fetch(yo, 1800)
 601                 soup = BeautifulSoup(dataf)
 602                 ruler_re = regexp.compile(
 603                         '/yoweb/flag/info\.wm\?flagid=(\d+)$')
 604                 ruler = soup.find('a', attrs = { 'href': ruler_re })
 605                 if not ruler:
 606                         debug('IEI COLLECT FLAGID '+`self.name`+' NONE')
 607                         return
 608                 debug('IEI COLLECT FLAGID '+`self.name`+' GOT '+``ruler``)
 609                 m = ruler_re.search(ruler['href'])
 610                 self.flagid = m.group(1)
 611
 612         def __str__(self):
 613                 return `(self.ocean, self.islandid, self.name,
 614                         self.yoweb_url, self.flagid)`
 615
 616 class IslandFlagInfo(IslandExtendedInfo):
 617         # Public data attributes (inherited):
 618         #  ocean
 619         #  name
 620         #  islandid
 621         #  yoweb_url
 622         #  flagid
 623         # Public data attributes (additional):
 624         #  flag
 625         def __init__(self, ocean, islename):
 626                 IslandExtendedInfo.__init__(self, ocean, islename)
 627                 self.flag = None
 628                 self._collect_flag()
 629
 630         def _collect_flag(self):
 631                 if self.flagid is None: return
 632                 self.flag = FlagInfo(self.flagid, 1800)
 633
 634         def __str__(self):
 635                 return IslandExtendedInfo.__str__(self) + '; ' + str(self.flag)
 636
 637 class NullProgressReporter():
 638         def doing(self, msg): pass
 639         def stop(self): pass
 640
 641 class TypewriterProgressReporter():
 642         def __init__(self):
 643                 self._l = 0
 644         def doing(self,m):
 645                 self._doing(m + '...')
 646         def _doing(self,m):
 647                 self._write('\r')
 648                 self._write(m)
 649                 less = self._l - len(m)
 650                 if less > 0:
 651                         self._write(' ' * less)
 652                         self._write('\b' * less)
 653                 self._l = len(m)
 654                 sys.stdout.flush()
 655         def stop(self):
 656                 self._doing('')
 657                 self._l = 0
 658         def _write(self,t):
 659                 sys.stdout.write(t)
 660
 661 class OceanInfo():
 662         # Public data attributes:
 663         #   oi.islands[islename] = IslandInfo(...)
 664         #   oi.arches[archname][islename] = IslandInfo(...)
 665         def __init__(self, isleclass=IslandBasicInfo):
 666                 self.isleclass = isleclass
 667                 self.ocean = fetcher.ocean.lower().capitalize()
 668
 669                 progressreporter.doing('fetching ocean info')
 670
 671                 cmdl = ['./yppedia-ocean-scraper']
 672                 if opts.localhtml is not None:
 673                         cmdl += ['--local-html-dir',opts.localhtml]
 674                 cmdl += [self.ocean]
 675                 debug('OceanInfo collect running ' + `cmdl`)
 676                 oscraper = subprocess.Popen(
 677                         cmdl,
 678                         stdout = subprocess.PIPE,
 679                         cwd = yppsc_dir()+'/yarrg',
 680                         shell=False, stderr=None,
 681                         )
 682                 h = oscraper.stdout.readline()
 683                 debug('OceanInfo collect h '+`h`)
 684                 assert(regexp.match('^ocean ', h))
 685                 arch_re = regexp.compile('^ (\S.*)')
 686                 island_re = regexp.compile('^  (\S.*)')
 687
 688                 oscraper.wait()
 689                 assert(oscraper.returncode == 0)
 690
 691                 self.islands = { }
 692                 self.arches = { }
 693                 archname = None
 694
 695                 isles = [ ]
 696                 progressreporter.doing('parsing ocean info')
 697
 698                 for l in oscraper.stdout:
 699                         debug('OceanInfo collect l '+`l`)
 700                         l = l.rstrip('\n')
 701                         m = island_re.match(l)
 702                         if m:
 703                                 assert(archname is not None)
 704                                 islename = m.group(1)
 705                                 isles.append((archname, islename))
 706                                 continue
 707                         m = arch_re.match(l)
 708                         if m:
 709                                 archname = m.group(1)
 710                                 assert(archname not in self.arches)
 711                                 self.arches[archname] = { }
 712                                 continue
 713                         assert(False)
 714
 715                 for i in xrange(0, len(isles)-1):
 716                         (archname, islename) = isles[i]
 717                         progressreporter.doing(
 718                                 'fetching isle info %2d/%d (%s: %s)'
 719                                 % (i, len(isles), archname, islename))
 720                         isle = self.isleclass(self.ocean, islename)
 721                         isle.arch = archname
 722                         self.islands[islename] = isle
 723                         self.arches[archname][islename] = isle
 724
 725         def __str__(self):
 726                 return `(self.islands, self.arches)`
 727
 728 #---------- pretty-printer for tables of pirate puzzle standings ----------
 729
 730 class StandingsTable:
 731         def __init__(self, f, use_puzzles=None, col_width=6, gap_every=5):
 732                 if use_puzzles is None:
 733                         if opts.ship_duty:
 734                                 use_puzzles=duty_puzzles
 735                         else:
 736                                 use_puzzles=puzzles
 737                 self._puzzles = use_puzzles
 738                 self.f = f
 739                 self._cw = col_width-1
 740                 self._gap_every = gap_every
 741                 self._linecount = 0
 742                 self._o = f.write
 743
 744         def _nl(self): self._o('\n')
 745
 746         def _pline(self, lhs, puzstrs, extra):
 747                 if (self._linecount > 0
 748                     and self._gap_every is not None
 749                     and not (self._linecount % self._gap_every)):
 750                         self._nl()
 751                 self._o('%-*s' % (max(max_pirate_namelen+1, 15), lhs))
 752                 for v in puzstrs:
 753                         self._o(' %-*.*s' % (self._cw,self._cw, v))
 754                 if extra:
 755                         self._o(' ' + extra)
 756                 self._nl()
 757                 self._linecount += 1
 758
 759         def _puzstr(self, pi, puzzle):
 760                 if not isinstance(puzzle,list): puzzle = [puzzle]
 761                 try: standing = max([pi.standings[p] for p in puzzle])
 762                 except KeyError: return '?'
 763                 if not standing: return ''
 764                 s = ''
 765                 if self._cw > 4:
 766                         c1 = standingvals[standing][0]
 767                         if standing < 3: c1 = c1.lower() # 3 = Master
 768                         s += `standing`
 769                 if self._cw > 5:
 770                         s += ' '
 771                 s += '*' * (standing / 2)
 772                 s += '+' * (standing % 2)
 773                 return s
 774
 775         def headings(self, lhs='', rhs=None):
 776                 def puzn_redact(name):
 777                         if isinstance(name,list):
 778                                 return '/'.join(
 779                                         ["%.*s" % (self._cw/2, puzn_redact(n))
 780                                          for n in name])
 781                         spc = name.find(' ')
 782                         if spc < 0: return name
 783                         return name[0:min(4,spc)] + name[spc+1:]
 784                 self._linecount = -2
 785                 self._pline(lhs, map(puzn_redact, self._puzzles), rhs)
 786                 self._linecount = 0
 787         def literalline(self, line):
 788                 self._o(line)
 789                 self._nl()
 790                 self._linecount = 0
 791         def pirate_dummy(self, name, standingstring, extra=None):
 792                 standings = standingstring * len(self._puzzles)
 793                 self._pline(' '+name, standings, extra)
 794         def pirate(self, pi, extra=None):
 795                 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
 796                 self._pline(' '+pi.name, puzstrs, extra)
 797
 798
 799 #---------- chat log parser ----------
 800
 801 class PirateAboard:
 802         # This is essentially a transparent, dumb, data class.
 803         #  pa.v                 may be None
 804         #  pa.name
 805         #  pa.last_time
 806         #  pa.last_event
 807         #  pa.gunner
 808         #  pa.last_chat_time
 809         #  pa.last_chat_chan
 810         #  pa.pi
 811
 812         # Also used for jobbing applicants:
 813         #               happens when                    expires (to "-")
 814         #   -            disembark, leaves crew          no
 815         #   aboard       evidence of them being aboard   no
 816         #   applied      "has applied for the job"       120s, configurable
 817         #   ashore       "has taken a job"               30min, configurable
 818         #   declined     "declined the job offer"        30s, configurable
 819         #   invited      "has been invited to job"       120s, configurable
 820         #
 821         #  pa.jobber    None, 'ashore', 'applied', 'invited', 'declined'
 822         #  pa.expires   expiry time time
 823
 824         def __init__(pa, pn, v, time, event):
 825                 pa.name = pn
 826                 pa.v = v
 827                 pa.last_time = time
 828                 pa.last_event = event
 829                 pa.last_chat_time = None
 830                 pa.last_chat_chan = None
 831                 pa.gunner = False
 832                 pa.pi = None
 833                 pa.jobber = None
 834                 pa.expires = None
 835
 836         def pirate_info(pa):
 837                 now = time.time()
 838                 if pa.pi:
 839                         age = now - pa.pi_fetched
 840                         guide = random.randint(120,240)
 841                         if age <= guide:
 842                                 return pa.pi
 843                         debug('PirateAboard refresh %d > %d  %s' % (
 844                                 age, guide, pa.name))
 845                         imaginary = [2,4]
 846                 else:
 847                         imaginary = [1]
 848                 wait = fetcher.need_wait(now, imaginary)
 849                 if wait:
 850                         debug('PirateAboard fetcher not ready %d' % wait)
 851                         return pa.pi
 852                 pa.pi = PirateInfo(pa.name, 600)
 853                 pa.pi_fetched = now
 854                 return pa.pi
 855
 856 class ChatLogTracker:
 857         # This is quite complex so we make it opaque.  Use the
 858         # official invokers, accessors etc.
 859
 860         def __init__(self, myself_pi, logfn):
 861                 self._pl = {}   # self._pl['Pirate'] =
 862                 self._vl = {}   #   self._vl['Vessel']['Pirate'] = PirateAboard
 863                                 # self._vl['Vessel']['#lastinfo']
 864                                 # self._vl['Vessel']['#name']
 865                                 # self._v = self._vl[self._vessel]
 866                 self._date = None
 867                 self._myself = myself_pi
 868                 self._lbuf = ''
 869                 self._f = file(logfn)
 870                 flen = os.fstat(self._f.fileno()).st_size
 871                 max_backlog = 500000
 872                 if flen > max_backlog:
 873                         startpos = flen - max_backlog
 874                         self._f.seek(startpos)
 875                         self._f.readline()
 876                 self._progress = [0, flen - self._f.tell()]
 877                 self._disembark_myself()
 878                 self._need_redisplay = False
 879                 self._lastvessel = None
 880
 881         def _disembark_myself(self):
 882                 self._v = None
 883                 self._vessel = None
 884                 self.force_redisplay()
 885
 886         def force_redisplay(self):
 887                 self._need_redisplay = True
 888
 889         def _vessel_updated(self, v, timestamp):
 890                 if v is None: return
 891                 v['#lastinfo'] = timestamp
 892                 self.force_redisplay()
 893
 894         def _onboard_event(self,v,timestamp,pirate,event,jobber=None):
 895                 pa = self._pl.get(pirate, None)
 896                 if pa is not None and pa.v is v:
 897                         pa.last_time = timestamp
 898                         pa.last_event = event
 899                 else:
 900                         if pa is not None and pa.v is not None:
 901                                 del pa.v[pirate]
 902                         pa = PirateAboard(pirate, v, timestamp, event)
 903                         self._pl[pirate] = pa
 904                         if v is not None: v[pirate] = pa
 905                 pa.jobber = jobber
 906
 907                 if jobber is None: timeout = None
 908                 else: timeout = getattr(opts, 'timeout_'+jobber)
 909                 if timeout is None: pa.expires = None
 910                 else: pa.expires = timestamp + timeout
 911                 self._vessel_updated(v, timestamp)
 912                 return pa
 913
 914         def _expire_jobbers(self, now):
 915                 for pa in self._pl.values():
 916                         if pa.expires is None: continue
 917                         if pa.expires >= now: continue
 918                         v = pa.v
 919                         del self._pl[pa.name]
 920                         if v is not None: del v[pa.name]
 921                         self.force_redisplay()
 922
 923         def _trash_vessel(self, v):
 924                 for pn in v:
 925                         if pn.startswith('#'): continue
 926                         del self._pl[pn]
 927                 vn = v['#name']
 928                 del self._vl[vn]
 929                 if v is self._v: self._disembark_myself()
 930                 self.force_redisplay()
 931
 932         def _vessel_stale(self, v, timestamp):
 933                 return timestamp - v['#lastinfo'] > opts.ship_reboard_clearout
 934
 935         def _vessel_check_expire(self, v, timestamp):
 936                 if not self._vessel_stale(v, timestamp):
 937                         return v
 938                 self._debug_line_disposition(timestamp,'',
 939                         'stale-reset ' + v['#name'])
 940                 self._trash_vessel(v)
 941                 return None
 942
 943         def expire_garbage(self, timestamp):
 944                 for v in self._vl.values():
 945                         self._vessel_check_expire(v, timestamp)
 946
 947         def _vessel_lookup(self, vn, timestamp, dml=[], create=False):
 948                 v = self._vl.get(vn, None)
 949                 if v is not None:
 950                         v = self._vessel_check_expire(v, timestamp)
 951                 if v is not None:
 952                         dml.append('found')
 953                         return v
 954                 if not create:
 955                         dml.append('no')
 956                 dml.append('new')
 957                 self._vl[vn] = v = { '#name': vn }
 958                 self._vessel_updated(v, timestamp)
 959                 return v
 960
 961         def _find_matching_vessel(self, pattern, timestamp, cmdr,
 962                                         dml=[], create=False):
 963                 # use when a commander pirate `cmdr' specified a vessel
 964                 #  by name `pattern' (either may be None)
 965                 # if create is true, will create the vessel
 966                 #  record if an exact name is specified
 967
 968                 if (pattern is not None and
 969                     not '*' in pattern
 970                     and len(pattern.split(' ')) == 2):
 971                         vn = pattern.title()
 972                         dml.append('exact')
 973                         return self._vessel_lookup(
 974                                 vn, timestamp, dml=dml, create=create)
 975
 976                 if pattern is None:
 977                         pattern_check = lambda vn: True
 978                 else:
 979                         re = '(?:.* )?%s$' % pattern.lower().replace('*','.+')
 980                         pattern_check = regexp.compile(re, regexp.I).match
 981
 982                 tries = []
 983
 984                 cmdr_pa = self._pl.get(cmdr, None)
 985                 if cmdr_pa: tries.append((cmdr_pa.v, 'cmdr'))
 986
 987                 tries.append((self._v, 'here'))
 988                 tried_vns = []
 989
 990                 for (v, dm) in tries:
 991                         if v is None: dml.append(dm+'?'); continue
 992
 993                         vn = v['#name']
 994                         if not pattern_check(vn):
 995                                 tried_vns.append(vn)
 996                                 dml.append(dm+'#')
 997                                 continue
 998
 999                         dml.append(dm+'!')
1000                         return v
1001
1002                 if pattern is not None and '*' in pattern:
1003                         search = [
1004                                 (vn,v)
1005                                 for (vn,v) in self._vl.iteritems()
1006                                 if not self._vessel_stale(v, timestamp)
1007                                 if pattern_check(vn)
1008                                 ]
1009                         #debug('CLT-RE /%s/ wanted (%s) searched (%s)' % (
1010                         #       re,
1011                         #       '/'.join(tried_vns),
1012                         #       '/'.join([vn for (vn,v) in search])))
1013
1014                         if len(search)==1:
1015                                 dml.append('one')
1016                                 return search[0][1]
1017                         elif search:
1018                                 dml.append('many')
1019                         else:
1020                                 dml.append('none')
1021
1022         def _debug_line_disposition(self,timestamp,l,m):
1023                 debug('CLT %13s %-40s %s' % (timestamp,m,l))
1024
1025         def _rm_crew_l(self,re,l):
1026                 m = regexp.match(re,l)
1027                 if m and m.group(2) == self._myself.crew[1]:
1028                         return m.group(1)
1029                 else:
1030                         return None
1031
1032         def local_command(self, metacmd):
1033                 # returns None if all went well, or problem message
1034                 return self._command(self._myself.name, metacmd,
1035                         "local", time.time(),
1036                         (lambda m: debug('CMD %s' % metacmd)))
1037
1038         def _command(self, cmdr, metacmd, chan, timestamp, d):
1039                 # returns None if all went well, or problem message
1040                 metacmd = regexp.sub('\\s+', ' ', metacmd).strip()
1041                 m2 = regexp.match(
1042                     '/([adj]) (?:([A-Za-z* ]+)\\s*:)?([A-Za-z ]+)$',
1043                     metacmd)
1044                 if not m2: return "unknown syntax or command"
1045
1046                 (cmd, pattern, targets) = m2.groups()
1047                 dml = ['cmd', chan, cmd]
1048
1049                 if cmd == 'a': each = self._onboard_event
1050                 elif cmd == 'd': each = disembark
1051                 else: each = lambda *l: self._onboard_event(*l,
1052                                 **{'jobber':'applied'})
1053
1054                 if cmdr == self._myself.name:
1055                         dml.append('self')
1056                         how = 'cmd: %s' % cmd
1057                 else:
1058                         dml.append('other')
1059                         how = 'cmd: %s %s' % (cmd,cmdr)
1060
1061                 if cmd == 'j':
1062                         if pattern is not None:
1063                                 return "/j command does not take a vessel"
1064                         v = None
1065                 else:
1066                         v = self._find_matching_vessel(
1067                                 pattern, timestamp, cmdr,
1068                                 dml, create=True)
1069
1070                 if cmd == 'j' or v is not None:
1071                         targets = targets.strip().split(' ')
1072                         dml.append(`len(targets)`)
1073                         for target in targets:
1074                                 each(v, timestamp, target.title(), how)
1075                         self._vessel_updated(v, timestamp)
1076
1077                 dm = ' '.join(dml)
1078                 return d(dm)
1079
1080                 return None
1081
1082         def chatline(self,l):
1083                 rm = lambda re: regexp.match(re,l)
1084                 d = lambda m: self._debug_line_disposition(timestamp,l,m)
1085                 rm_crew = lambda re: self._rm_crew_l(re,l)
1086                 timestamp = None
1087
1088                 m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$')
1089                 if m:
1090                         self._date = [int(x) for x in m.groups()]
1091                         self._previous_timestamp = None
1092                         return d('date '+`self._date`)
1093
1094                 if self._date is None:
1095                         return d('date unset')
1096
1097                 m = rm('\\[(\d\d):(\d\d):(\d\d)\\] ')
1098                 if not m:
1099                         return d('no timestamp')
1100
1101                 while True:
1102                         time_tuple = (self._date +
1103                                       [int(x) for x in m.groups()] +
1104                                       [-1,-1,-1])
1105                         timestamp = time.mktime(time_tuple)
1106                         if timestamp >= self._previous_timestamp: break
1107                         self._date[2] += 1
1108                         self._debug_line_disposition(timestamp,'',
1109                                 'new date '+`self._date`)
1110
1111                 self._previous_timestamp = timestamp
1112
1113                 l = l[l.find(' ')+1:]
1114
1115                 def ob_x(pirate,event):
1116                         return self._onboard_event(
1117                                         self._v, timestamp, pirate, event)
1118                 def ob1(did): ob_x(m.group(1), did); return d(did)
1119                 def oba(did): return ob1('%s %s' % (did, m.group(2)))
1120
1121                 def jb(pirate,jobber):
1122                         return self._onboard_event(
1123                                 None, timestamp, pirate,
1124                                 ("jobber %s" % jobber),
1125                                 jobber=jobber
1126                                 )
1127
1128                 def disembark(v, timestamp, pirate, event):
1129                         self._onboard_event(
1130                                         v, timestamp, pirate, 'leaving '+event)
1131                         del v[pirate]
1132                         del self._pl[pirate]
1133
1134                 def disembark_me(why):
1135                         self._disembark_myself()
1136                         return d('disembark-me '+why)
1137
1138                 m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
1139                 if m:
1140                         dm = ['boarding']
1141                         pn = self._myself.name
1142                         vn = m.group(1)
1143                         v = self._vessel_lookup(vn, timestamp, dm, create=True)
1144                         self._lastvessel = self._vessel = vn
1145                         self._v = v
1146                         ob_x(pn, 'we boarded')
1147                         self.expire_garbage(timestamp)
1148                         return d(' '.join(dm))
1149
1150                 if self._v is None:
1151                         return d('no vessel')
1152
1153                 m = rm('(\\w+) has come aboard\\.$')
1154                 if m: return ob1('boarded');
1155
1156                 m = rm('You have ordered (\\w+) to do some (\\S.*\\S)\\.$')
1157                 if m:
1158                         (who,what) = m.groups()
1159                         pa = ob_x(who,'ord '+what)
1160                         if what == 'Gunning':
1161                                 pa.gunner = True
1162                         return d('duty order')
1163
1164                 m = rm('(\\w+) abandoned a (\\S.*\\S) station\\.$')
1165                 if m: oba('stopped'); return d("end")
1166
1167                 def chat_core(speaker, chan):
1168                         try: pa = self._pl[speaker]
1169                         except KeyError: return 'mystery'
1170                         if pa.v is not None and pa.v is not self._v:
1171                                 return 'elsewhere'
1172                         pa.last_chat_time = timestamp
1173                         pa.last_chat_chan = chan
1174                         self.force_redisplay()
1175                         return 'here'
1176
1177                 def chat(chan):
1178                         speaker = m.group(1)
1179                         dm = chat_core(speaker, chan)
1180                         return d('chat %s %s' % (chan, dm))
1181
1182                 def chat_metacmd(chan):
1183                         (cmdr, metacmd) = m.groups()
1184                         whynot = self._command(
1185                                 cmdr, metacmd, chan, timestamp, d)
1186                         if whynot is not None:
1187                                 return chat(chan)
1188                         else:
1189                                 chat_core(cmdr, 'cmd '+chan)
1190
1191                 m = rm('(\\w+) (?:issued an order|ordered everyone) "')
1192                 if m: return ob1('general order');
1193
1194                 m = rm('(\\w+) says, "')
1195                 if m: return chat('public')
1196
1197                 m = rm('(\\w+) tells ye, "')
1198                 if m: return chat('private')
1199
1200                 m = rm('Ye told (\\w+), "(.*)"$')
1201                 if m: return chat_metacmd('private')
1202
1203                 m = rm('(\\w+) flag officer chats, "')
1204                 if m: return chat('flag officer')
1205
1206                 m = rm('(\\w+) officer chats, "(.*)"$')
1207                 if m: return chat_metacmd('officer')
1208
1209                 m = rm('Ye accepted the offer to job with ')
1210                 if m: return disembark_me('jobbing')
1211
1212                 m = rm('Ye hop on the ferry and are whisked away ')
1213                 if m: return disembark_me('ferry')
1214
1215                 m = rm('Whisking away to yer home on the magical winds')
1216                 if m: return disembark_me('home')
1217
1218                 m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
1219                 if m:
1220                         pl = m.group(1).split(', ')
1221                         if not self._myself.name in pl:
1222                                 return d('lost melee')
1223                         for pn in pl:
1224                                 if ' ' in pn: continue
1225                                 ob_x(pn,'won melee')
1226                         return d('won melee')
1227
1228                 m = rm('(\\w+) is eliminated\\!')
1229                 if m: return ob1('eliminated in fray');
1230
1231                 m = rm('(\\w+) has driven \w+ from the ship\\!')
1232                 if m: return ob1('boarder repelled');
1233
1234                 m = rm('\w+ has bested (\\w+), and turns'+
1235                         ' to the rest of the ship\\.')
1236                 if m: return ob1('boarder unrepelled');
1237
1238                 pirate = rm_crew("(\\w+) has taken a job with '(.*)'\\.")
1239                 if pirate: return jb(pirate, 'ashore')
1240
1241                 pirate = rm_crew("(\\w+) has left '(.*)'\\.")
1242                 if pirate:
1243                         disembark(self._v, timestamp, pirate, 'left crew')
1244                         return d('left crew')
1245
1246                 m = rm('(\w+) has applied for the posted job\.')
1247                 if m: return jb(m.group(1), 'applied')
1248
1249                 pirate= rm_crew("(\\w+) has been invited to job for '(.*)'\\.")
1250                 if pirate: return jb(pirate, 'invited')
1251
1252                 pirate = rm_crew("(\\w+) declined the job offer for '(.*)'\\.")
1253                 if pirate: return jb(pirate, 'declined')
1254
1255                 m = rm('(\\w+) has left the vessel\.')
1256                 if m:
1257                         pirate = m.group(1)
1258                         disembark(self._v, timestamp, pirate, 'disembarked')
1259                         return d('disembarked')
1260
1261                 return d('not-matched')
1262
1263         def _str_pa(self, pn, pa):
1264                 assert self._pl[pn] == pa
1265                 s = ' '*20 + "%s %-*s %13s %-30s %13s %-20s %13s" % (
1266                         (' ','G')[pa.gunner],
1267                         max_pirate_namelen, pn,
1268                         pa.last_time, pa.last_event,
1269                         pa.last_chat_time, pa.last_chat_chan,
1270                         pa.jobber)
1271                 if pa.expires is not None:
1272                         s += " %-5d" % (pa.expires - pa.last_time)
1273                 s += "\n"
1274                 return s
1275
1276         def _str_vessel(self, vn, v):
1277                 s = ' vessel %s\n' % vn
1278                 s += ' '*20 + "%-*s   %13s\n" % (
1279                                 max_pirate_namelen, '#lastinfo',
1280                                 v['#lastinfo'])
1281                 assert v['#name'] == vn
1282                 for pn in sorted(v.keys()):
1283                         if pn.startswith('#'): continue
1284                         pa = v[pn]
1285                         assert pa.v == v
1286                         s += self._str_pa(pn,pa)
1287                 return s
1288
1289         def __str__(self):
1290                 s = '''<ChatLogTracker
1291  myself %s
1292  vessel %s
1293 '''                     % (self._myself.name, self._vessel)
1294                 assert ((self._v is None and self._vessel is None) or
1295                         (self._v is self._vl[self._vessel]))
1296                 if self._vessel is not None:
1297                         s += self._str_vessel(self._vessel, self._v)
1298                 for vn in sorted(self._vl.keys()):
1299                         if vn == self._vessel: continue
1300                         s += self._str_vessel(vn, self._vl[vn])
1301                 s += " elsewhere\n"
1302                 for p in self._pl:
1303                         pa = self._pl[p]
1304                         if pa.v is not None:
1305                                 assert pa.v[p] is pa
1306                                 assert pa.v in self._vl.values()
1307                         else:
1308                                 s += self._str_pa(pa.name, pa)
1309                 s += '>\n'
1310                 return s
1311
1312         def catchup(self, progress=None):
1313                 while True:
1314                         more = self._f.readline()
1315                         if not more: break
1316
1317                         self._progress[0] += len(more)
1318                         if progress: progress.progress(*self._progress)
1319
1320                         self._lbuf += more
1321                         if self._lbuf.endswith('\n'):
1322                                 self.chatline(self._lbuf.rstrip())
1323                                 self._lbuf = ''
1324                                 if opts.debug >= 2:
1325                                         debug(self.__str__())
1326                 self._expire_jobbers(time.time())
1327
1328                 if progress: progress.caughtup()
1329
1330         def changed(self):
1331                 rv = self._need_redisplay
1332                 self._need_redisplay = False
1333                 return rv
1334         def myname(self):
1335                 # returns our pirate name
1336                 return self._myself.name
1337         def vesselname(self):
1338                 # returns the vessel name we're aboard or None
1339                 return self._vessel
1340         def lastvesselname(self):
1341                 # returns the last vessel name we were aboard or None
1342                 return self._lastvessel
1343         def aboard(self, vesselname=True):
1344                 # returns a list of PirateAboard the vessel
1345                 #  sorted by pirate name
1346                 #  you can pass this None and you'll get []
1347                 #  or True for the current vessel (which is the default)
1348                 #  the returned value is a fresh list of persistent
1349                 #  PirateAboard objects
1350                 if vesselname is True: v = self._v
1351                 else: v = self._vl.get(vesselname.title())
1352                 if v is None: return []
1353                 return [ v[pn]
1354                          for pn in sorted(v.keys())
1355                          if not pn.startswith('#') ]
1356         def jobbers(self):
1357                 # returns a the jobbers' PirateAboards,
1358                 # sorted by jobber class and reverse of expiry time
1359                 l = [ pa
1360                       for pa in self._pl.values()
1361                       if pa.jobber is not None
1362                     ]
1363                 def compar_key(pa):
1364                         return (pa.jobber, -pa.expires)
1365                 l.sort(key = compar_key)
1366                 return l
1367
1368 #---------- implementations of actual operation modes ----------
1369
1370 def do_pirate(pirates, bu):
1371         print '{'
1372         for pirate in pirates:
1373                 info = PirateInfo(pirate)
1374                 print '%s: %s,' % (`pirate`, info)
1375         print '}'
1376
1377 def prep_crewflag_of(args, bu, max_age, selector, constructor):
1378         if len(args) != 1: bu('crew-of etc. take one pirate name')
1379         pi = PirateInfo(args[0], max_age)
1380         cf = selector(pi)
1381         if cf is None: return None
1382         return constructor(cf[0], max_age)
1383
1384 def prep_crew_of(args, bu, max_age=300):
1385         return prep_crewflag_of(args, bu, max_age,
1386                 (lambda pi: pi.crew), CrewInfo)
1387
1388 def prep_flag_of(args, bu, max_age=300):
1389         return prep_crewflag_of(args, bu, max_age,
1390                 (lambda pi: pi.flag), FlagInfo)
1391
1392 def do_crew_of(args, bu):
1393         ci = prep_crew_of(args, bu)
1394         print ci
1395
1396 def do_flag_of(args, bu):
1397         fi = prep_flag_of(args, bu)
1398         print fi
1399
1400 def do_standings_crew_of(args, bu):
1401         ci = prep_crew_of(args, bu, 60)
1402         tab = StandingsTable(sys.stdout)
1403         tab.headings()
1404         for (rank, members) in ci.crew:
1405                 if not members: continue
1406                 tab.literalline('')
1407                 tab.literalline('%s:' % rank)
1408                 for p in members:
1409                         pi = PirateInfo(p, random.randint(900,1800))
1410                         tab.pirate(pi)
1411
1412 def do_ocean(args, bu):
1413         if (len(args)): bu('ocean takes no further arguments')
1414         fetcher.default_ocean()
1415         oi = OceanInfo(IslandFlagInfo)
1416         print oi
1417         for islename in sorted(oi.islands.keys()):
1418                 isle = oi.islands[islename]
1419                 print isle
1420
1421 def do_embargoes(args, bu):
1422         if (len(args)): bu('ocean takes no further arguments')
1423         fetcher.default_ocean()
1424         oi = OceanInfo(IslandFlagInfo)
1425         wr = sys.stdout.write
1426         print ('EMBARGOES:  Island    | Owning flag'+
1427                 '                    | Embargoed flags')
1428
1429         def getflname(isle):
1430                 if isle.islandid is None: return 'uncolonisable'
1431                 if isle.flag is None: return 'uncolonised'
1432                 return isle.flag.name
1433
1434         progressreporter.stop()
1435
1436         for archname in sorted(oi.arches.keys()):
1437                 print 'ARCHIPELAGO: ',archname
1438                 for islename in sorted(oi.arches[archname].keys()):
1439                         isle = oi.islands[islename]
1440                         wr(' %-20s | ' % isle.name)
1441                         flname = getflname(isle)
1442                         wr('%-30s | ' % flname)
1443                         flag = isle.flag
1444                         if flag is None: print ''; continue
1445                         delim = ''
1446                         for rel in flag.relations:
1447                                 if rel.this_declaring >= 0: continue
1448                                 wr(delim)
1449                                 wr(rel.other_flagname)
1450                                 delim = '; '
1451                         print ''
1452
1453 def do_embargoes_flag_of(args, bu):
1454         progressreporter.doing('fetching flag info')
1455         fi = prep_flag_of(args, bu)
1456         if fi is None:
1457                 progressreporter.stop()
1458                 print 'Pirate is not in a flag.'
1459                 return
1460
1461         oi = OceanInfo(IslandFlagInfo)
1462
1463         progressreporter.stop()
1464         print ''
1465
1466         any = False
1467         for islename in sorted(oi.islands.keys()):
1468                 isle = oi.islands[islename]
1469                 flag = isle.flag
1470                 if flag is None: continue
1471                 for rel in flag.relations:
1472                         if rel.this_declaring >= 0: continue
1473                         if rel.other_flagid != fi.flagid: continue
1474                         if not any: print 'EMBARGOED:'
1475                         any = True
1476                         print "  %-30s (%s)" % (islename, flag.name)
1477         if not any:
1478                 print 'No embargoes.'
1479         print ''
1480
1481         war_flag(fi)
1482         print ''
1483
1484 def do_war_flag_of(args, bu):
1485         fi = prep_flag_of(args, bu)
1486         war_flag(fi)
1487
1488 def war_flag(fi):
1489         any = False
1490         for certain in [True, False]:
1491                 anythis = False
1492                 for rel in fi.relations:
1493                         if rel.this_declaring >= 0: continue
1494                         if (rel.other_declaring_max < 0) != certain: continue
1495                         if not anythis:
1496                                 if certain: m = 'SINKING PvP'
1497                                 else: m = 'RISK OF SINKING PvP'
1498                                 print '%s (%s):' % (m, rel.yoweb_heading)
1499                         anythis = True
1500                         any = True
1501                         print " ", rel.other_flagname
1502         if not any:
1503                 print 'No sinking PvP.'
1504
1505 #----- modes which use the chat log parser are quite complex -----
1506
1507 class ProgressPrintPercentage:
1508         def __init__(self, f=sys.stdout):
1509                 self._f = f
1510         def progress_string(self,done,total):
1511                 return "scan chat logs %3d%%\r" % ((done*100) / total)
1512         def progress(self,*a):
1513                 self._f.write(self.progress_string(*a))
1514                 self._f.flush()
1515         def show_init(self, pirate, ocean):
1516                 print >>self._f, 'Starting up, %s on the %s ocean' % (
1517                         pirate, ocean)
1518         def caughtup(self):
1519                 self._f.write('                   \r')
1520                 self._f.flush()
1521
1522 def prep_chat_log(args, bu,
1523                 progress=ProgressPrintPercentage(),
1524                 max_myself_age=3600):
1525         if len(args) != 1: bu('this action takes only chat log filename')
1526         logfn = args[0]
1527         logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_'
1528         match = regexp.match(logfn_re, logfn)
1529         if not match: bu('chat log filename is not in expected format')
1530         (pirate, ocean) = match.groups()
1531         fetcher.default_ocean(ocean)
1532
1533         progress.show_init(pirate, fetcher.ocean)
1534         myself = PirateInfo(pirate,max_myself_age)
1535         track = ChatLogTracker(myself, logfn)
1536
1537         opts.debug -= 2
1538         track.catchup(progress)
1539         opts.debug += 2
1540
1541         track.force_redisplay()
1542
1543         return (myself, track)
1544
1545 def do_track_chat_log(args, bu):
1546         (myself, track) = prep_chat_log(args, bu)
1547         while True:
1548                 track.catchup()
1549                 if track.changed():
1550                         print track
1551                 sleep(0.5 + 0.5 * random.random())
1552
1553 #----- ship management aid -----
1554
1555 class Display_dumb(ProgressPrintPercentage):
1556         def __init__(self):
1557                 ProgressPrintPercentage.__init__(self)
1558         def show(self, s):
1559                 print '\n\n', s;
1560         def realstart(self):
1561                 pass
1562
1563 class Display_overwrite(ProgressPrintPercentage):
1564         def __init__(self):
1565                 ProgressPrintPercentage.__init__(self)
1566
1567                 null = file('/dev/null','w')
1568                 curses.setupterm(fd=null.fileno())
1569
1570                 self._clear = curses.tigetstr('clear')
1571                 if not self._clear:
1572                         self._debug('missing clear!')
1573                         self.show = Display_dumb.show
1574                         return
1575
1576                 self._t = {'el':'', 'ed':''}
1577                 if not self._init_sophisticated():
1578                         for k in self._t.keys(): self._t[k] = ''
1579                         self._t['ho'] = self._clear
1580
1581         def _debug(self,m): debug('display overwrite: '+m)
1582
1583         def _init_sophisticated(self):
1584                 for k in self._t.keys():
1585                         s = curses.tigetstr(k)
1586                         self._t[k] = s
1587                 self._t['ho'] = curses.tigetstr('ho')
1588                 if not self._t['ho']:
1589                         cup = curses.tigetstr('cup')
1590                         self._t['ho'] = curses.tparm(cup,0,0)
1591                 missing = [k for k in self._t.keys() if not self._t[k]]
1592                 if missing:
1593                         self.debug('missing '+(' '.join(missing)))
1594                         return 0
1595                 return 1
1596
1597         def show(self, s):
1598                 w = sys.stdout.write
1599                 def wti(k): w(self._t[k])
1600
1601                 wti('ho')
1602                 nl = ''
1603                 for l in s.rstrip().split('\n'):
1604                         w(nl)
1605                         w(l)
1606                         wti('el')
1607                         nl = '\r\n'
1608                 wti('ed')
1609                 w(' ')
1610                 sys.stdout.flush()
1611
1612         def realstart(self):
1613                 sys.stdout.write(self._clear)
1614                 sys.stdout.flush()
1615
1616
1617 def do_ship_aid(args, bu):
1618         if opts.ship_duty is None: opts.ship_duty = True
1619
1620         displayer = globals()['Display_'+opts.display]()
1621
1622         (myself, track) = prep_chat_log(args, bu, progress=displayer)
1623
1624         displayer.realstart()
1625
1626         if os.isatty(0): kr_create = KeystrokeReader
1627         else: kr_create = DummyKeystrokeReader
1628
1629         try:
1630                 kreader = kr_create(0, 10)
1631                 ship_aid_core(myself, track, displayer, kreader)
1632         finally:
1633                 kreader.stop()
1634                 print '\n'
1635
1636 class KeyBasedSorter:
1637         def compar_key_pa(self, pa):
1638                 pi = pa.pirate_info()
1639                 if pi is None: return None
1640                 return self.compar_key(pi)
1641         def lsort_pa(self, l):
1642                 l.sort(key = self.compar_key_pa)
1643
1644 class NameSorter(KeyBasedSorter):
1645         def compar_key(self, pi): return pi.name
1646         def desc(self): return 'name'
1647
1648 class SkillSorter(NameSorter):
1649         def __init__(self, relevant):
1650                 self._want = frozenset(relevant.split('/'))
1651                 self._avoid = set()
1652                 for p in core_duty_puzzles:
1653                         if isinstance(p,basestring): self._avoid.add(p)
1654                         else: self._avoid |= set(p)
1655                 self._avoid -= self._want
1656                 self._desc = '%s' % relevant
1657
1658         def desc(self): return self._desc
1659
1660         def compar_key(self, pi):
1661                 best_want = max([
1662                         pi.standings.get(puz,-1)
1663                         for puz in self._want
1664                         ])
1665                 best_avoid = [
1666                         -pi.standings.get(puz,standing_limit)
1667                         for puz in self._avoid
1668                         ]
1669                 best_avoid.sort()
1670                 def negate(x): return -x
1671                 debug('compar_key %s bw=%s ba=%s' % (pi.name, `best_want`,
1672                         `best_avoid`))
1673                 return (-best_want, map(negate, best_avoid), pi.name)
1674
1675 def ship_aid_core(myself, track, displayer, kreader):
1676
1677         def find_vessel():
1678                 vn = track.vesselname()
1679                 if vn: return (vn, " on board the %s" % vn)
1680                 vn = track.lastvesselname()
1681                 if vn: return (vn, " ashore from the %s" % vn)
1682                 return (None, " not on a vessel")
1683
1684         def timeevent(t,e):
1685                 if t is None: return ' ' * 22
1686                 return " %-4s %-16s" % (format_time_interval(now - t),e)
1687
1688         displayer.show(track.myname() + find_vessel()[1] + '...')
1689
1690         rotate_nya = '/-\\'
1691
1692         sort = NameSorter()
1693         clicmd = None
1694         clierr = None
1695         cliexec = None
1696
1697         while True:
1698                 track.catchup()
1699                 now = time.time()
1700
1701                 (vn, vs) = find_vessel()
1702
1703                 s = ''
1704                 if cliexec is not None:
1705                         s += '...'
1706                 elif clierr is not None:
1707                         s += 'Error: '+clierr
1708                 elif clicmd is not None:
1709                         s += '/' + clicmd
1710                 else:
1711                         s = track.myname() + vs
1712                         s += " at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
1713                         s += kreader.info()
1714                 s += '\n'
1715
1716                 tbl_s = StringIO()
1717                 tbl = StandingsTable(tbl_s)
1718
1719                 aboard = track.aboard(vn)
1720                 sort.lsort_pa(aboard)
1721
1722                 jobbers = track.jobbers()
1723
1724                 if track.vesselname(): howmany = 'aboard: %2d' % len(aboard)
1725                 else: howmany = ''
1726
1727                 tbl.headings(howmany, '  sorted by '+sort.desc())
1728
1729                 last_jobber = None
1730
1731                 for pa in aboard + jobbers:
1732                         if pa.jobber != last_jobber:
1733                                 last_jobber = pa.jobber
1734                                 tbl.literalline('')
1735                                 tbl.literalline('jobbers '+last_jobber)
1736
1737                         pi = pa.pirate_info()
1738
1739                         xs = ''
1740                         if pa.gunner: xs += 'G '
1741                         else: xs += '  '
1742                         xs += timeevent(pa.last_time, pa.last_event)
1743                         xs += timeevent(pa.last_chat_time, pa.last_chat_chan)
1744
1745                         if pi is None:
1746                                 tbl.pirate_dummy(pa.name, rotate_nya[0], xs)
1747                         else:
1748                                 tbl.pirate(pi, xs)
1749
1750                 s += tbl_s.getvalue()
1751                 displayer.show(s)
1752                 tbl_s.close()
1753
1754                 if cliexec is not None:
1755                         clierr = track.local_command("/"+cliexec.strip())
1756                         cliexec = None
1757                         continue
1758
1759                 k = kreader.getch()
1760                 if k is None:
1761                         rotate_nya = rotate_nya[1:3] + rotate_nya[0]
1762                         continue
1763
1764                 if clierr is not None:
1765                         clierr = None
1766                         continue
1767
1768                 if clicmd is not None:
1769                         if k == '\r' or k == '\n':
1770                                 cliexec = clicmd
1771                                 clicmd = clicmdbase
1772                         elif k == '\e' and clicmd != "":
1773                                 clicmd = clicmdbase
1774                         elif k == '\33':
1775                                 clicmd = None
1776                         elif k == '\b' or k == '\177':
1777                                 clicmd = clicmd[ 0 : len(clicmd)-1 ]
1778                         else:
1779                                 clicmd += k
1780                         continue
1781
1782                 if k == 'q': break
1783                 elif k == 'g': sort = SkillSorter('Gunning')
1784                 elif k == 'c': sort = SkillSorter('Carpentry')
1785                 elif k == 's': sort = SkillSorter('Sailing/Rigging')
1786                 elif k == 'b': sort = SkillSorter('Bilging')
1787                 elif k == 'n': sort = SkillSorter('Navigating')
1788                 elif k == 'd': sort = SkillSorter('Battle Navigation')
1789                 elif k == 't': sort = SkillSorter('Treasure Haul')
1790                 elif k == 'a': sort = NameSorter()
1791                 elif k == '/': clicmdbase = ""; clicmd = clicmdbase
1792                 elif k == '+': clicmdbase = "a "; clicmd = clicmdbase
1793                 else: pass # unknown key command
1794
1795 #---------- individual keystroke input ----------
1796
1797 class DummyKeystrokeReader:
1798         def __init__(self,fd,timeout_dummy): pass
1799         def stop(self): pass
1800         def getch(self): sleep(1); return None
1801         def info(self): return ' [noninteractive]'
1802
1803 class KeystrokeReader(DummyKeystrokeReader):
1804         def __init__(self, fd, timeout_decisec=0):
1805                 self._fd = fd
1806                 self._saved = termios.tcgetattr(fd)
1807                 a = termios.tcgetattr(fd)
1808                 a[3] &= ~(termios.ECHO | termios.ECHONL |
1809                           termios.ICANON | termios.IEXTEN)
1810                 a[6][termios.VMIN] = 0
1811                 a[6][termios.VTIME] = timeout_decisec
1812                 termios.tcsetattr(fd, termios.TCSANOW, a)
1813         def stop(self):
1814                 termios.tcsetattr(self._fd, termios.TCSANOW, self._saved)
1815         def getch(self):
1816                 debug_flush()
1817                 byte = os.read(self._fd, 1)
1818                 if not len(byte): return None
1819                 return byte
1820         def info(self):
1821                 return ''
1822
1823 #---------- main program ----------
1824
1825 def main():
1826         global opts, fetcher, yppedia, progressreporter
1827
1828         pa = OptionParser(
1829 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
1830 actions:
1831  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
1832  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
1833  yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
1834  yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG
1835  yoweb-scrape [--ocean OCEAN ...] ocean|embargoes
1836  yoweb-scrape [--ocean OCEAN ...] war-flag-of|embargoes-flag-of PIRATE
1837  yoweb-scrape [options] ship-aid CHAT-LOG  (must be .../PIRATE_OCEAN_chat-log*)
1838
1839 display modes (for --display) apply to ship-aid:
1840  --display=dumb       just print new information, scrolling the screen
1841  --display=overwrite  use cursor motion, selective clear, etc. to redraw at top''')
1842         ao = pa.add_option
1843         ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
1844                 help='select ocean OCEAN')
1845         ao('--cache-dir', dest='cache_dir', metavar='DIR',
1846                 default='~/.yoweb-scrape-cache',
1847                 help='cache yoweb pages in DIR')
1848         ao('-D','--debug', action='count', dest='debug', default=0,
1849                 help='enable debugging output')
1850         ao('--debug-fd', type='int', dest='debug_fd',
1851                 help='write any debugging output to specified fd')
1852         ao('-q','--quiet', action='store_true', dest='quiet',
1853                 help='suppress warning output')
1854         ao('--display', action='store', dest='display',
1855                 type='choice', choices=['dumb','overwrite'],
1856                 help='how to display ship aid')
1857         ao('--local-ypp-dir', action='store', dest='localhtml',
1858                 help='get yppedia pages from local directory LOCALHTML'+
1859                         ' instead of via HTTP')
1860
1861         ao_jt = lambda wh, t: ao(
1862                 '--timeout-sa-'+wh, action='store', dest='timeout_'+wh,
1863                 default=t, help=('set timeout for expiring %s jobbers' % wh))
1864         ao_jt('applied',      120)
1865         ao_jt('invited',      120)
1866         ao_jt('declined',      30)
1867         ao_jt('ashore',      1800)
1868
1869         ao('--ship-duty', action='store_true', dest='ship_duty',
1870                 help='show ship duty station puzzles')
1871         ao('--all-puzzles', action='store_false', dest='ship_duty',
1872                 help='show all puzzles, not just ship duty stations')
1873
1874         ao('--min-cache-reuse', type='int', dest='min_max_age',
1875                 metavar='SECONDS', default=60,
1876                 help='always reuse cache yoweb data if no older than this')
1877
1878         (opts,args) = pa.parse_args()
1879         random.seed()
1880
1881         if len(args) < 1:
1882                 print >>sys.stderr, copyright_info
1883                 pa.error('need a mode argument')
1884
1885         if opts.debug_fd is not None:
1886                 opts.debug_file = os.fdopen(opts.debug_fd, 'w')
1887         else:
1888                 opts.debug_file = sys.stdout
1889
1890         mode = args[0]
1891         mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
1892         try: mode_fn = globals()[mode_fn_name]
1893         except KeyError: pa.error('unknown mode "%s"' % mode)
1894
1895         # fixed parameters
1896         opts.expire_age = max(3600, opts.min_max_age)
1897
1898         opts.ship_reboard_clearout = 3600
1899
1900         if opts.cache_dir.startswith('~/'):
1901                 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
1902
1903         if opts.display is None:
1904                 if ((opts.debug > 0 and opts.debug_fd is None)
1905                     or not os.isatty(sys.stdout.fileno())):
1906                         opts.display = 'dumb'
1907                 else:
1908                         opts.display = 'overwrite'
1909
1910         fetcher = Yoweb(opts.ocean, opts.cache_dir)
1911         yppedia = Yppedia(opts.cache_dir)
1912
1913         if opts.debug or not os.isatty(0):
1914                  progressreporter = NullProgressReporter()
1915         else:
1916                 progressreporter = TypewriterProgressReporter()
1917
1918         mode_fn(args[1:], pa.error)
1919
1920 main()