yoweb-scrape

   1 #!/usr/bin/python
   2 # This is part of ypp-sc-tools, a set of third-party tools for assisting
   3 # players of Yohoho Puzzle Pirates.
   4 #
   5 # Copyright (C) 2009 Ian Jackson <ijackson@chiark.greenend.org.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 #
  20 # Yohoho and Puzzle Pirates are probably trademarks of Three Rings and
  21 # are used without permission.  This program is not endorsed or
  22 # sponsored by Three Rings.
  23
  24 copyright_info = '''
  25 yoweb-scrape is part of ypp-sc-tools  Copyright (C) 2009 Ian Jackson
  26 This program comes with ABSOLUTELY NO WARRANTY; this is free software,
  27 and you are welcome to redistribute it under certain conditions.
  28 For details, read the top of the yoweb-scrape file.
  29 '''
  30
  31 #---------- setup ----------
  32
  33 import signal
  34 signal.signal(signal.SIGINT, signal.SIG_DFL)
  35
  36 import os
  37 import time
  38 import urllib
  39 import urllib2
  40 import errno
  41 import sys
  42 import re as regexp
  43 import random
  44 import curses
  45 import termios
  46 import random
  47 import subprocess
  48 from optparse import OptionParser
  49 from StringIO import StringIO
  50
  51 from BeautifulSoup import BeautifulSoup
  52
  53 opts = None
  54
  55 #---------- YPP parameters and arrays ----------
  56
  57 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
  58         '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
  59         '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
  60         '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
  61
  62 core_duty_puzzles = [
  63                 'Gunning',
  64                 ['Sailing','Rigging'],
  65                 'Bilging',
  66                 'Carpentry',
  67                 ]
  68
  69 duty_puzzles = ([ 'Navigating', 'Battle Navigation' ] +
  70                 core_duty_puzzles +
  71                 [ 'Treasure Haul' ])
  72
  73 standingvals = ('Able/Proficient/Distinguished/Respected/Master'+
  74                 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
  75 standing_limit = len(standingvals)
  76
  77 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
  78
  79 max_pirate_namelen = 12
  80
  81
  82 #---------- general utilities ----------
  83
  84 def debug(m):
  85         if opts.debug > 0:
  86                 print >>opts.debug_file, m
  87
  88 def debug_flush():
  89         if opts.debug > 0:
  90                 opts.debug_file.flush()
  91
  92 def sleep(seconds):
  93         debug_flush()
  94         time.sleep(seconds)
  95
  96 def format_time_interval(ti):
  97         if ti < 120: return '%d:%02d' % (ti / 60, ti % 60)
  98         if ti < 7200: return '%2dm' % (ti / 60)
  99         if ti < 86400: return '%dh' % (ti / 3600)
 100         return '%dd' % (ti / 86400)
 101
 102 def yppsc_dir():
 103         lib = os.getenv("YPPSC_YARRG_SRCBASE")
 104         if lib is not None: return lib
 105         lib = sys.argv[0]
 106         lib = regexp.sub('/[^/]+$', '', lib)
 107         os.environ["YPPSC_YARRG_SRCBASE"] = lib
 108         return lib
 109
 110 #---------- caching and rate-limiting data fetcher ----------
 111
 112 class Fetcher:
 113         def __init__(self, cachedir):
 114                 debug('Fetcher init %s' % cachedir)
 115                 self.cachedir = cachedir
 116                 try: os.mkdir(cachedir)
 117                 except (OSError,IOError), oe:
 118                         if oe.errno != errno.EEXIST: raise
 119                 self._cache_scan(time.time())
 120
 121         def _cache_scan(self, now):
 122                 # returns list of ages, unsorted
 123                 ages = []
 124                 debug('Fetcher   scan_cache')
 125                 for leaf in os.listdir(self.cachedir):
 126                         if not leaf.startswith('#'): continue
 127                         path = self.cachedir + '/' + leaf
 128                         try: s = os.stat(path)
 129                         except (OSError,IOError), oe:
 130                                 if oe.errno != errno.ENOENT: raise
 131                                 continue
 132                         age = now - s.st_mtime
 133                         if age > opts.expire_age:
 134                                 debug('Fetcher    expire %d %s' % (age, path))
 135                                 try: os.remove(path)
 136                                 except (OSError,IOError), oe:
 137                                         if oe.errno != errno.ENOENT: raise
 138                                 continue
 139                         ages.append(age)
 140                 return ages
 141
 142         def need_wait(self, now, imaginary=[]):
 143                 ages = self._cache_scan(now)
 144                 ages += imaginary
 145                 ages.sort()
 146                 debug('Fetcher   ages ' + `ages`)
 147                 min_age = 1
 148                 need_wait = 0
 149                 for age in ages:
 150                         if age < min_age and age <= 5:
 151                                 debug('Fetcher   morewait min=%d age=%d' %
 152                                         (min_age, age))
 153                                 need_wait = max(need_wait, min_age - age)
 154                         min_age += 3
 155                         min_age *= 1.25
 156                 if need_wait > 0:
 157                         need_wait += random.random() - 0.5
 158                 return need_wait
 159
 160         def _rate_limit_cache_clean(self, now):
 161                 need_wait = self.need_wait(now)
 162                 if need_wait > 0:
 163                         debug('Fetcher   wait %d' % need_wait)
 164                         sleep(need_wait)
 165
 166         def fetch(self, url, max_age):
 167                 debug('Fetcher fetch %s' % url)
 168                 cache_corename = urllib.quote_plus(url)
 169                 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
 170                 try: f = file(cache_item, 'r')
 171                 except (OSError,IOError), oe:
 172                         if oe.errno != errno.ENOENT: raise
 173                         f = None
 174                 now = time.time()
 175                 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
 176                 if f is not None:
 177                         s = os.fstat(f.fileno())
 178                         age = now - s.st_mtime
 179                         if age > max_age:
 180                                 debug('Fetcher  stale %d < %d'% (max_age, age))
 181                                 f = None
 182                 if f is not None:
 183                         data = f.read()
 184                         f.close()
 185                         debug('Fetcher  cached %d > %d' % (max_age, age))
 186                         return data
 187
 188                 debug('Fetcher  fetch')
 189                 self._rate_limit_cache_clean(now)
 190
 191                 stream = urllib2.urlopen(url)
 192                 data = stream.read()
 193                 cache_tmp = "%s/#%s~%d#" % (
 194                         self.cachedir, cache_corename, os.getpid())
 195                 f = file(cache_tmp, 'w')
 196                 f.write(data)
 197                 f.close()
 198                 os.rename(cache_tmp, cache_item)
 199                 debug('Fetcher  stored')
 200                 return data
 201
 202 class Yoweb(Fetcher):
 203         def __init__(self, ocean, cachedir):
 204                 debug('Yoweb init %s' % cachedir)
 205                 self.ocean = ocean
 206                 Fetcher.__init__(self, cachedir)
 207
 208         def default_ocean(self, ocean='ice'):
 209                 if self.ocean is None:
 210                         self.ocean = ocean
 211
 212         def yoweb(self, kind, tail, max_age):
 213                 self.default_ocean()
 214                 assert(self.ocean)
 215                 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
 216                         self.ocean, kind, tail)
 217                 return self.fetch(url, max_age)
 218
 219 class Yppedia(Fetcher):
 220         def __init__(self, cachedir):
 221                 debug('Yoweb init %s' % cachedir)
 222                 self.base = 'http://yppedia.puzzlepirates.com/'
 223                 self.localhtml = opts.localhtml
 224                 Fetcher.__init__(self, cachedir)
 225
 226         def __call__(self, rhs):
 227                 if self.localhtml is None:
 228                         url = self.base + rhs
 229                         debug('Yppedia retrieving YPP '+url);
 230                         return self.fetch(url, 3000)
 231                 else:
 232                         return file(opts.localhtml + '/' + rhs, 'r')
 233
 234 #---------- logging assistance for troubled screenscrapers ----------
 235
 236 class SoupLog:
 237         def __init__(self):
 238                 self.msgs = [ ]
 239         def msg(self, m):
 240                 self.msgs.append(m)
 241         def soupm(self, obj, m):
 242                 self.msg(m + '; in ' + `obj`)
 243         def needs_msgs(self, child_souplog):
 244                 self.msgs += child_souplog.msgs
 245                 child_souplog.msgs = [ ]
 246
 247 def soup_text(obj):
 248         str = ''.join(obj.findAll(text=True))
 249         return str.strip()
 250
 251 class SomethingSoupInfo(SoupLog):
 252         def __init__(self, kind, tail, max_age):
 253                 SoupLog.__init__(self)
 254                 html = fetcher.yoweb(kind, tail, max_age)
 255                 self._soup = BeautifulSoup(html,
 256                         convertEntities=BeautifulSoup.HTML_ENTITIES
 257                         )
 258
 259 #---------- scraper for pirate pages ----------
 260
 261 class PirateInfo(SomethingSoupInfo):
 262         # Public data members:
 263         #  pi.standings = { 'Treasure Haul': 'Able' ... }
 264         #  pi.name = name
 265         #  pi.crew = (id, name)
 266         #  pi.flag = (id, name)
 267         #  pi.msgs = [ 'message describing problem with scrape' ]
 268
 269         def __init__(self, pirate, max_age=300):
 270                 SomethingSoupInfo.__init__(self,
 271                         'pirate.wm?target=', pirate, max_age)
 272                 self.name = pirate
 273                 self._find_standings()
 274                 self.crew = self._find_crewflag('crew',
 275                         '^/yoweb/crew/info\\.wm')
 276                 self.flag = self._find_crewflag('flag',
 277                         '^/yoweb/flag/info\\.wm')
 278
 279         def _find_standings(self):
 280                 imgs = self._soup.findAll('img',
 281                         src=regexp.compile('/yoweb/images/stat.*'))
 282                 re = regexp.compile(
 283 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
 284                         )
 285                 standings = { }
 286
 287                 for skill in puzzles:
 288                         standings[skill] = [ ]
 289
 290                 skl = SoupLog()
 291
 292                 for img in imgs:
 293                         try: puzzle = img['alt']
 294                         except KeyError: continue
 295
 296                         if not puzzle in puzzles:
 297                                 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
 298                                 continue
 299                         key = img.findParent('td')
 300                         if key is None:
 301                                 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
 302                                 continue
 303                         valelem = key.findNextSibling('td')
 304                         if valelem is None:
 305                                 skl.soupm(key, 'puzzle missing sibling "%s"'
 306                                         % puzzle)
 307                                 continue
 308                         valstr = soup_text(valelem)
 309                         match = re.match(valstr)
 310                         if match is None:
 311                                 skl.soupm(key, ('puzzle "%s" unparseable'+
 312                                         ' standing "%s"') % (puzzle, valstr))
 313                                 continue
 314                         standing = match.group(match.lastindex)
 315                         standings[puzzle].append(standing)
 316
 317                 self.standings = { }
 318
 319                 for puzzle in puzzles:
 320                         sl = standings[puzzle]
 321                         if len(sl) > 1:
 322                                 skl.msg('puzzle "%s" multiple standings %s' %
 323                                                 (puzzle, `sl`))
 324                                 continue
 325                         if not sl:
 326                                 skl.msg('puzzle "%s" no standing found' % puzzle)
 327                                 continue
 328                         standing = sl[0]
 329                         for i in range(0, standing_limit):
 330                                 if standing == standingvals[i]:
 331                                         self.standings[puzzle] = i
 332                         if not puzzle in self.standings:
 333                                 skl.msg('puzzle "%s" unknown standing "%s"' %
 334                                         (puzzle, standing))
 335
 336                 all_standings_ok = True
 337                 for puzzle in puzzles:
 338                         if not puzzle in self.standings:
 339                                 self.needs_msgs(skl)
 340
 341         def _find_crewflag(self, cf, yoweb_re):
 342                 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
 343                 if len(things) != 1:
 344                         self.msg('zero or several %s id references found' % cf)
 345                         return None
 346                 thing = things[0]
 347                 id_re = '\\b%sid\\=(\\w+)$' % cf
 348                 id_haystack = thing['href']
 349                 match = regexp.compile(id_re).search(id_haystack)
 350                 if match is None:
 351                         self.soupm(thing, ('incomprehensible %s id ref'+
 352                                 ' (%s in %s)') % (cf, id_re, id_haystack))
 353                         return None
 354                 name = soup_text(thing)
 355                 return (match.group(1), name)
 356
 357         def __str__(self):
 358                 return `(self.crew, self.flag, self.standings, self.msgs)`
 359
 360 #---------- scraper for crew pages ----------
 361
 362 class CrewInfo(SomethingSoupInfo):
 363         # Public data members:
 364         #  ci.crew = [ ('Captain',        ['Pirate', ...]),
 365         #              ('Senior Officer', [...]),
 366         #               ... ]
 367         #  pi.msgs = [ 'message describing problem with scrape' ]
 368
 369         def __init__(self, crewid, max_age=300):
 370                 SomethingSoupInfo.__init__(self,
 371                         'crew/info.wm?crewid=', crewid, max_age)
 372                 self._find_crew()
 373
 374         def _find_crew(self):
 375                 self.crew = []
 376                 capts = self._soup.findAll('img',
 377                         src='/yoweb/images/crew-captain.png')
 378                 if len(capts) != 1:
 379                         self.msg('crew members: no. of captain images != 1')
 380                         return
 381                 tbl = capts[0]
 382                 while not tbl.find('a', href=pirate_ref_re):
 383                         tbl = tbl.findParent('table')
 384                         if not tbl:
 385                                 self.msg('crew members: cannot find table')
 386                                 return
 387                 current_rank_crew = None
 388                 crew_rank_re = regexp.compile('/yoweb/images/crew')
 389                 for row in tbl.contents:
 390                         # findAll(recurse=False)
 391                         if isinstance(row,basestring):
 392                                 continue
 393
 394                         is_rank = row.find('img', attrs={'src': crew_rank_re})
 395                         if is_rank:
 396                                 rank = soup_text(row)
 397                                 current_rank_crew = []
 398                                 self.crew.append((rank, current_rank_crew))
 399                                 continue
 400                         for cell in row.findAll('a', href=pirate_ref_re):
 401                                 if current_rank_crew is None:
 402                                         self.soupm(cell, 'crew members: crew'
 403                                                 ' before rank')
 404                                         continue
 405                                 current_rank_crew.append(soup_text(cell))
 406
 407         def __str__(self):
 408                 return `(self.crew, self.msgs)`
 409
 410 class FlagInfo(SomethingSoupInfo):
 411         # Public data members (after init):
 412         #
 413         #   name        #               string
 414         #
 415         #   relations[n] = (otherflagname, otherflagid, [stringfromyoweb],
 416         #               thisdeclaring, otherdeclaringmin, otherdeclaringmax)
 417         #               # where {this,other}declaring{,min,max} are:
 418         #               #       -1      {this,other} is declaring war
 419         #               #        0      {this,other} is not doing either
 420         #               #       +1      {this,other} is allying
 421         #   relation_byname[otherflagname] = relations[some_n]
 422         #   relation_byid[otherflagname] = relations[some_n]
 423         #
 424         #   islands[n] = (islandname, islandid)
 425         #
 426         def __init__(self, flagid, max_age=600):
 427                 SomethingSoupInfo.__init__(self,
 428                         'flag/info.wm?flagid=', flagid, max_age)
 429                 self._find_flag()
 430
 431         def _find_flag(self):
 432                 font2 = self._soup.find('font',{'size':'+2'})
 433                 self.name = font2.find('b').contents[0]
 434
 435                 self.relations = [ ]
 436                 self.relation_byname = { }
 437                 self.relation_byid = { }
 438                 self.islands = [ ]
 439
 440                 magnate = self._soup.find('img',{'src':
 441                         '/yoweb/images/repute-MAGNATE.png'})
 442                 warinfo = (magnate.findParent('table').findParent('tr').
 443                         findNextSibling('tr').findNext('td',{'align':'left'}))
 444
 445                 def warn(m):
 446                         print >>sys.stderr, 'WARNING: '+m
 447
 448                 def wi_warn(head, waritem):
 449                         warn('unknown warmap item: %s: %s' %
 450                                 (`head`, ``waritem``))
 451
 452                 def wihelp_item(waritem, thing):
 453                         url = waritem.get('href', None)
 454                         if url is None:
 455                                 return ('no url for '+thing,None,None)
 456                         m = regexp.search('\?'+thing+'id=(\d+)$', url)
 457                         if not m: return ('no '+thing+'id',None,None)
 458                         tid = m.group(1)
 459                         tname = waritem.string
 460                         if tname is None:
 461                                 return (thing+' name not just string',None,None)
 462                         return (None,tid,tname)
 463
 464                 def wi_alwar(head, waritem, thisdecl, othermin, othermax):
 465                         (err,flagid,flagname) = wihelp_item(waritem,'flag')
 466                         if err: return err
 467                         rel = self.relation_byid.get(flagid, None)
 468                         if rel: return 'flag id twice!'
 469                         if flagname in self.relation_byname:
 470                                 return 'flag name twice!'
 471                         rel = (flagname,flagid,head, thisdecl,othermin,othermax)
 472                         self.relations.append(rel)
 473                         self.relation_byid[flagid] = rel
 474                         self.relation_byname[flagid] = rel
 475
 476                 def wi_isle(head, waritem):
 477                         (err,isleid,islename) = wihelp_item(waritem,'island')
 478                         if err: return err
 479                         self.islands.append((isleid,islename))
 480
 481                 warmap = {
 482                         'Allied with':                  (wi_alwar,+1,+1,+1),
 483                         'Declaring war against':        (wi_alwar,-1, 0,+1),
 484                         'At war with':                  (wi_alwar,-1,-1,-1),
 485                         'Trying to form an alliance with': (wi_alwar,+1,-1,0),
 486                         'Islands controlled by this flag': (wi_isle,),
 487                         }
 488
 489                 how = (wi_warn, None)
 490
 491                 for waritem in warinfo.findAll(['font','a']):
 492                         if waritem is None: break
 493                         if waritem.name == 'font':
 494                                 colour = waritem.get('color',None)
 495                                 if colour.lstrip('#') != '958A5F':
 496                                         warn('strange colour %s in %s' %
 497                                                 (colour,``waritem``))
 498                                         continue
 499                                 head = waritem.string
 500                                 if head is None:
 501                                         warn('no head string in '+``waritem``)
 502                                         continue
 503                                 head = regexp.sub('\\s+', ' ', head).strip()
 504                                 head = head.rstrip(':')
 505                                 how = (head,) + warmap.get(head, (wi_warn,))
 506                                 continue
 507                         assert(waritem.name == 'a')
 508
 509                         debug('WARHOW %s(%s, waritem, *%s)' %
 510                                 (how[1], `how[0]`, `how[2:]`))
 511                         bad = how[1](how[0], waritem, *how[2:])
 512                         if bad:
 513                                 warn('bad waritem %s: %s: %s' % (`how[0]`,
 514                                         bad, ``waritem``))
 515
 516         def __str__(self):
 517                 return `(self.name, self.islands, self.relations)`
 518
 519 #---------- scraper for ocean info incl. embargoes etc. ----------
 520
 521 class IslandBasicInfo():
 522         # Public members:
 523         #  ocean
 524         #  name
 525         # Public members maybe set by caller:
 526         #  arch
 527         def __init__(self, ocean, islename):
 528                 self.ocean = ocean
 529                 self.name = islename
 530         def collect(self):
 531                 pass
 532         def yppedia(self):
 533                 def q(x): return urllib.quote(x.replace(' ','_'))
 534                 url_rhs = q(self.name) + '_(' + q(self.ocean) + ')'
 535                 return yppedia(url_rhs)
 536         def __str__(self):
 537                 return `(self.ocean, self.name)`
 538
 539 class IslandExtendedInfo(IslandBasicInfo):
 540         # Public members (inherited):
 541         #  ocean
 542         #  name
 543         # Public members (additional):
 544         #  islandid
 545         #  yoweb_url
 546         #  flagid
 547         def collect(self):
 548                 IslandBasicInfo.collect(self)
 549                 self._collect_yoweb()
 550                 self._collect_flagid()
 551
 552         def _collect_yoweb(self):
 553                 debug('IEI COLLECT YOWEB '+`self.name`)
 554                 self.islandid = None
 555                 self.yoweb_url = None
 556
 557                 soup = BeautifulSoup(self.yppedia())
 558                 content = soup.find('div', attrs = {'id': 'content'})
 559                 yoweb_re = regexp.compile('^http://\w+\.puzzlepirates\.com/'+
 560                         'yoweb/island/info\.wm\?islandid=(\d+)$')
 561                 a = soup.find('a', attrs = { 'href': yoweb_re })
 562                 if a is None:
 563                         debug('IEI COLLECT YOWEB '+`self.name`+' NONE')
 564                         return
 565
 566                 debug('IEI COLLECT YOWEB '+`self.name`+' GOT '+``a``)
 567                 self.yoweb_url = a['href']
 568                 m = yoweb_re.search(self.yoweb_url)
 569                 self.islandid = m.group(1)
 570
 571         def _collect_flagid(self):
 572                 self.flagid = None
 573
 574                 yo = self.yoweb_url
 575                 debug('IEI COLLECT FLAGID '+`self.name`+' URL '+`yo`)
 576                 if yo is None: return None
 577                 dataf = fetcher.fetch(yo, 1800)
 578                 soup = BeautifulSoup(dataf)
 579                 ruler_re = regexp.compile(
 580                         '/yoweb/flag/info\.wm\?flagid=(\d+)$')
 581                 ruler = soup.find('a', attrs = { 'href': ruler_re })
 582                 if not ruler:
 583                         debug('IEI COLLECT FLAGID '+`self.name`+' NONE')
 584                         return
 585                 debug('IEI COLLECT FLAGID '+`self.name`+' GOT '+``ruler``)
 586                 m = ruler_re.search(ruler['href'])
 587                 self.flagid = m.group(1)
 588
 589         def __str__(self):
 590                 return `(self.ocean, self.islandid, self.name,
 591                         self.yoweb_url, self.flagid)`
 592
 593 class OceanInfo():
 594         # Public data attributes (valid after collect()):
 595         #   oi.islands[islename] = IslandInfo(...)
 596         #   oi.arches[archname][islename] = IslandInfo(...)
 597         def __init__(self, isleclass=IslandBasicInfo):
 598                 self.isleclass = isleclass
 599                 self.ocean = fetcher.ocean.lower().capitalize()
 600         def collect(self):
 601                 cmdl = ['./yppedia-ocean-scraper']
 602                 if opts.localhtml is not None:
 603                         cmdl += ['--local-html-dir',opts.localhtml]
 604                 cmdl += [self.ocean]
 605                 debug('OceanInfo collect running ' + `cmdl`)
 606                 oscraper = subprocess.Popen(
 607                         cmdl,
 608                         stdout = subprocess.PIPE,
 609                         cwd = yppsc_dir()+'/yarrg',
 610                         shell=False, stderr=None,
 611                         )
 612                 h = oscraper.stdout.readline()
 613                 debug('OceanInfo collect h '+`h`)
 614                 assert(regexp.match('^ocean ', h))
 615                 arch_re = regexp.compile('^ (\S.*)')
 616                 island_re = regexp.compile('^  (\S.*)')
 617
 618                 self.islands = { }
 619                 self.arches = { }
 620                 archname = None
 621
 622                 for l in oscraper.stdout:
 623                         debug('OceanInfo collect l '+`l`)
 624                         l = l.rstrip('\n')
 625                         m = island_re.match(l)
 626                         if m:
 627                                 assert(archname is not None)
 628                                 islename = m.group(1)
 629                                 isle = self.isleclass(self.ocean, islename)
 630                                 isle.arch = archname
 631                                 isle.collect()
 632                                 self.islands[islename] = isle
 633                                 self.arches[archname][islename] = isle
 634                                 continue
 635                         m = arch_re.match(l)
 636                         if m:
 637                                 archname = m.group(1)
 638                                 assert(archname not in self.arches)
 639                                 self.arches[archname] = { }
 640                                 continue
 641                         assert(False)
 642                 oscraper.wait()
 643                 assert(oscraper.returncode == 0)
 644         def __str__(self):
 645                 return `(self.islands, self.arches)`
 646
 647 #---------- pretty-printer for tables of pirate puzzle standings ----------
 648
 649 class StandingsTable:
 650         def __init__(self, f, use_puzzles=None, col_width=6, gap_every=5):
 651                 if use_puzzles is None:
 652                         if opts.ship_duty:
 653                                 use_puzzles=duty_puzzles
 654                         else:
 655                                 use_puzzles=puzzles
 656                 self._puzzles = use_puzzles
 657                 self.f = f
 658                 self._cw = col_width-1
 659                 self._gap_every = gap_every
 660                 self._linecount = 0
 661                 self._o = f.write
 662
 663         def _nl(self): self._o('\n')
 664
 665         def _pline(self, lhs, puzstrs, extra):
 666                 if (self._linecount > 0
 667                     and self._gap_every is not None
 668                     and not (self._linecount % self._gap_every)):
 669                         self._nl()
 670                 self._o('%-*s' % (max(max_pirate_namelen+1, 15), lhs))
 671                 for v in puzstrs:
 672                         self._o(' %-*.*s' % (self._cw,self._cw, v))
 673                 if extra:
 674                         self._o(' ' + extra)
 675                 self._nl()
 676                 self._linecount += 1
 677
 678         def _puzstr(self, pi, puzzle):
 679                 if not isinstance(puzzle,list): puzzle = [puzzle]
 680                 try: standing = max([pi.standings[p] for p in puzzle])
 681                 except KeyError: return '?'
 682                 if not standing: return ''
 683                 s = ''
 684                 if self._cw > 4:
 685                         c1 = standingvals[standing][0]
 686                         if standing < 3: c1 = c1.lower() # 3 = Master
 687                         s += `standing`
 688                 if self._cw > 5:
 689                         s += ' '
 690                 s += '*' * (standing / 2)
 691                 s += '+' * (standing % 2)
 692                 return s
 693
 694         def headings(self, lhs='', rhs=None):
 695                 def puzn_redact(name):
 696                         if isinstance(name,list):
 697                                 return '/'.join(
 698                                         ["%.*s" % (self._cw/2, puzn_redact(n))
 699                                          for n in name])
 700                         spc = name.find(' ')
 701                         if spc < 0: return name
 702                         return name[0:min(4,spc)] + name[spc+1:]
 703                 self._linecount = -2
 704                 self._pline(lhs, map(puzn_redact, self._puzzles), rhs)
 705                 self._linecount = 0
 706         def literalline(self, line):
 707                 self._o(line)
 708                 self._nl()
 709                 self._linecount = 0
 710         def pirate_dummy(self, name, standingstring, extra=None):
 711                 standings = standingstring * len(self._puzzles)
 712                 self._pline(' '+name, standings, extra)
 713         def pirate(self, pi, extra=None):
 714                 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
 715                 self._pline(' '+pi.name, puzstrs, extra)
 716
 717
 718 #---------- chat log parser ----------
 719
 720 class PirateAboard:
 721         # This is essentially a transparent, dumb, data class.
 722         #  pa.v                 may be None
 723         #  pa.name
 724         #  pa.last_time
 725         #  pa.last_event
 726         #  pa.gunner
 727         #  pa.last_chat_time
 728         #  pa.last_chat_chan
 729         #  pa.pi
 730
 731         # Also used for jobbing applicants:
 732         #               happens when                    expires (to "-")
 733         #   -            disembark, leaves crew          no
 734         #   aboard       evidence of them being aboard   no
 735         #   applied      "has applied for the job"       120s, configurable
 736         #   ashore       "has taken a job"               30min, configurable
 737         #   declined     "declined the job offer"        30s, configurable
 738         #   invited      "has been invited to job"       120s, configurable
 739         #
 740         #  pa.jobber    None, 'ashore', 'applied', 'invited', 'declined'
 741         #  pa.expires   expiry time time
 742
 743         def __init__(pa, pn, v, time, event):
 744                 pa.name = pn
 745                 pa.v = v
 746                 pa.last_time = time
 747                 pa.last_event = event
 748                 pa.last_chat_time = None
 749                 pa.last_chat_chan = None
 750                 pa.gunner = False
 751                 pa.pi = None
 752                 pa.jobber = None
 753                 pa.expires = None
 754
 755         def pirate_info(pa):
 756                 now = time.time()
 757                 if pa.pi:
 758                         age = now - pa.pi_fetched
 759                         guide = random.randint(120,240)
 760                         if age <= guide:
 761                                 return pa.pi
 762                         debug('PirateAboard refresh %d > %d  %s' % (
 763                                 age, guide, pa.name))
 764                         imaginary = [2,4]
 765                 else:
 766                         imaginary = [1]
 767                 wait = fetcher.need_wait(now, imaginary)
 768                 if wait:
 769                         debug('PirateAboard fetcher not ready %d' % wait)
 770                         return pa.pi
 771                 pa.pi = PirateInfo(pa.name, 600)
 772                 pa.pi_fetched = now
 773                 return pa.pi
 774
 775 class ChatLogTracker:
 776         # This is quite complex so we make it opaque.  Use the
 777         # official invokers, accessors etc.
 778
 779         def __init__(self, myself_pi, logfn):
 780                 self._pl = {}   # self._pl['Pirate'] =
 781                 self._vl = {}   #   self._vl['Vessel']['Pirate'] = PirateAboard
 782                                 # self._vl['Vessel']['#lastinfo']
 783                                 # self._vl['Vessel']['#name']
 784                                 # self._v = self._vl[self._vessel]
 785                 self._date = None
 786                 self._myself = myself_pi
 787                 self._lbuf = ''
 788                 self._f = file(logfn)
 789                 flen = os.fstat(self._f.fileno()).st_size
 790                 max_backlog = 500000
 791                 if flen > max_backlog:
 792                         startpos = flen - max_backlog
 793                         self._f.seek(startpos)
 794                         self._f.readline()
 795                 self._progress = [0, flen - self._f.tell()]
 796                 self._disembark_myself()
 797                 self._need_redisplay = False
 798                 self._lastvessel = None
 799
 800         def _disembark_myself(self):
 801                 self._v = None
 802                 self._vessel = None
 803                 self.force_redisplay()
 804
 805         def force_redisplay(self):
 806                 self._need_redisplay = True
 807
 808         def _vessel_updated(self, v, timestamp):
 809                 if v is None: return
 810                 v['#lastinfo'] = timestamp
 811                 self.force_redisplay()
 812
 813         def _onboard_event(self,v,timestamp,pirate,event,jobber=None):
 814                 pa = self._pl.get(pirate, None)
 815                 if pa is not None and pa.v is v:
 816                         pa.last_time = timestamp
 817                         pa.last_event = event
 818                 else:
 819                         if pa is not None and pa.v is not None:
 820                                 del pa.v[pirate]
 821                         pa = PirateAboard(pirate, v, timestamp, event)
 822                         self._pl[pirate] = pa
 823                         if v is not None: v[pirate] = pa
 824                 pa.jobber = jobber
 825
 826                 if jobber is None: timeout = None
 827                 else: timeout = getattr(opts, 'timeout_'+jobber)
 828                 if timeout is None: pa.expires = None
 829                 else: pa.expires = timestamp + timeout
 830                 self._vessel_updated(v, timestamp)
 831                 return pa
 832
 833         def _expire_jobbers(self, now):
 834                 for pa in self._pl.values():
 835                         if pa.expires is None: continue
 836                         if pa.expires >= now: continue
 837                         v = pa.v
 838                         del self._pl[pa.name]
 839                         if v is not None: del v[pa.name]
 840                         self.force_redisplay()
 841
 842         def _trash_vessel(self, v):
 843                 for pn in v:
 844                         if pn.startswith('#'): continue
 845                         del self._pl[pn]
 846                 vn = v['#name']
 847                 del self._vl[vn]
 848                 if v is self._v: self._disembark_myself()
 849                 self.force_redisplay()
 850
 851         def _vessel_stale(self, v, timestamp):
 852                 return timestamp - v['#lastinfo'] > opts.ship_reboard_clearout
 853
 854         def _vessel_check_expire(self, v, timestamp):
 855                 if not self._vessel_stale(v, timestamp):
 856                         return v
 857                 self._debug_line_disposition(timestamp,'',
 858                         'stale-reset ' + v['#name'])
 859                 self._trash_vessel(v)
 860                 return None
 861
 862         def expire_garbage(self, timestamp):
 863                 for v in self._vl.values():
 864                         self._vessel_check_expire(v, timestamp)
 865
 866         def _vessel_lookup(self, vn, timestamp, dml=[], create=False):
 867                 v = self._vl.get(vn, None)
 868                 if v is not None:
 869                         v = self._vessel_check_expire(v, timestamp)
 870                 if v is not None:
 871                         dml.append('found')
 872                         return v
 873                 if not create:
 874                         dml.append('no')
 875                 dml.append('new')
 876                 self._vl[vn] = v = { '#name': vn }
 877                 self._vessel_updated(v, timestamp)
 878                 return v
 879
 880         def _find_matching_vessel(self, pattern, timestamp, cmdr,
 881                                         dml=[], create=False):
 882                 # use when a commander pirate `cmdr' specified a vessel
 883                 #  by name `pattern' (either may be None)
 884                 # if create is true, will create the vessel
 885                 #  record if an exact name is specified
 886
 887                 if (pattern is not None and
 888                     not '*' in pattern
 889                     and len(pattern.split(' ')) == 2):
 890                         vn = pattern.title()
 891                         dml.append('exact')
 892                         return self._vessel_lookup(
 893                                 vn, timestamp, dml=dml, create=create)
 894
 895                 if pattern is None:
 896                         pattern_check = lambda vn: True
 897                 else:
 898                         re = '(?:.* )?%s$' % pattern.lower().replace('*','.+')
 899                         pattern_check = regexp.compile(re, regexp.I).match
 900
 901                 tries = []
 902
 903                 cmdr_pa = self._pl.get(cmdr, None)
 904                 if cmdr_pa: tries.append((cmdr_pa.v, 'cmdr'))
 905
 906                 tries.append((self._v, 'here'))
 907                 tried_vns = []
 908
 909                 for (v, dm) in tries:
 910                         if v is None: dml.append(dm+'?'); continue
 911
 912                         vn = v['#name']
 913                         if not pattern_check(vn):
 914                                 tried_vns.append(vn)
 915                                 dml.append(dm+'#')
 916                                 continue
 917
 918                         dml.append(dm+'!')
 919                         return v
 920
 921                 if pattern is not None and '*' in pattern:
 922                         search = [
 923                                 (vn,v)
 924                                 for (vn,v) in self._vl.iteritems()
 925                                 if not self._vessel_stale(v, timestamp)
 926                                 if pattern_check(vn)
 927                                 ]
 928                         #debug('CLT-RE /%s/ wanted (%s) searched (%s)' % (
 929                         #       re,
 930                         #       '/'.join(tried_vns),
 931                         #       '/'.join([vn for (vn,v) in search])))
 932
 933                         if len(search)==1:
 934                                 dml.append('one')
 935                                 return search[0][1]
 936                         elif search:
 937                                 dml.append('many')
 938                         else:
 939                                 dml.append('none')
 940
 941         def _debug_line_disposition(self,timestamp,l,m):
 942                 debug('CLT %13s %-40s %s' % (timestamp,m,l))
 943
 944         def _rm_crew_l(self,re,l):
 945                 m = regexp.match(re,l)
 946                 if m and m.group(2) == self._myself.crew[1]:
 947                         return m.group(1)
 948                 else:
 949                         return None
 950
 951         def local_command(self, metacmd):
 952                 # returns None if all went well, or problem message
 953                 return self._command(self._myself.name, metacmd,
 954                         "local", time.time(),
 955                         (lambda m: debug('CMD %s' % metacmd)))
 956
 957         def _command(self, cmdr, metacmd, chan, timestamp, d):
 958                 # returns None if all went well, or problem message
 959                 metacmd = regexp.sub('\\s+', ' ', metacmd).strip()
 960                 m2 = regexp.match(
 961                     '/([adj]) (?:([A-Za-z* ]+)\\s*:)?([A-Za-z ]+)$',
 962                     metacmd)
 963                 if not m2: return "unknown syntax or command"
 964
 965                 (cmd, pattern, targets) = m2.groups()
 966                 dml = ['cmd', chan, cmd]
 967
 968                 if cmd == 'a': each = self._onboard_event
 969                 elif cmd == 'd': each = disembark
 970                 else: each = lambda *l: self._onboard_event(*l,
 971                                 **{'jobber':'applied'})
 972
 973                 if cmdr == self._myself.name:
 974                         dml.append('self')
 975                         how = 'cmd: %s' % cmd
 976                 else:
 977                         dml.append('other')
 978                         how = 'cmd: %s %s' % (cmd,cmdr)
 979
 980                 if cmd == 'j':
 981                         if pattern is not None:
 982                                 return "/j command does not take a vessel"
 983                         v = None
 984                 else:
 985                         v = self._find_matching_vessel(
 986                                 pattern, timestamp, cmdr,
 987                                 dml, create=True)
 988
 989                 if cmd == 'j' or v is not None:
 990                         targets = targets.strip().split(' ')
 991                         dml.append(`len(targets)`)
 992                         for target in targets:
 993                                 each(v, timestamp, target.title(), how)
 994                         self._vessel_updated(v, timestamp)
 995
 996                 dm = ' '.join(dml)
 997                 return d(dm)
 998
 999                 return None
1000
1001         def chatline(self,l):
1002                 rm = lambda re: regexp.match(re,l)
1003                 d = lambda m: self._debug_line_disposition(timestamp,l,m)
1004                 rm_crew = lambda re: self._rm_crew_l(re,l)
1005                 timestamp = None
1006
1007                 m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$')
1008                 if m:
1009                         self._date = [int(x) for x in m.groups()]
1010                         self._previous_timestamp = None
1011                         return d('date '+`self._date`)
1012
1013                 if self._date is None:
1014                         return d('date unset')
1015
1016                 m = rm('\\[(\d\d):(\d\d):(\d\d)\\] ')
1017                 if not m:
1018                         return d('no timestamp')
1019
1020                 while True:
1021                         time_tuple = (self._date +
1022                                       [int(x) for x in m.groups()] +
1023                                       [-1,-1,-1])
1024                         timestamp = time.mktime(time_tuple)
1025                         if timestamp >= self._previous_timestamp: break
1026                         self._date[2] += 1
1027                         self._debug_line_disposition(timestamp,'',
1028                                 'new date '+`self._date`)
1029
1030                 self._previous_timestamp = timestamp
1031
1032                 l = l[l.find(' ')+1:]
1033
1034                 def ob_x(pirate,event):
1035                         return self._onboard_event(
1036                                         self._v, timestamp, pirate, event)
1037                 def ob1(did): ob_x(m.group(1), did); return d(did)
1038                 def oba(did): return ob1('%s %s' % (did, m.group(2)))
1039
1040                 def jb(pirate,jobber):
1041                         return self._onboard_event(
1042                                 None, timestamp, pirate,
1043                                 ("jobber %s" % jobber),
1044                                 jobber=jobber
1045                                 )
1046
1047                 def disembark(v, timestamp, pirate, event):
1048                         self._onboard_event(
1049                                         v, timestamp, pirate, 'leaving '+event)
1050                         del v[pirate]
1051                         del self._pl[pirate]
1052
1053                 def disembark_me(why):
1054                         self._disembark_myself()
1055                         return d('disembark-me '+why)
1056
1057                 m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
1058                 if m:
1059                         dm = ['boarding']
1060                         pn = self._myself.name
1061                         vn = m.group(1)
1062                         v = self._vessel_lookup(vn, timestamp, dm, create=True)
1063                         self._lastvessel = self._vessel = vn
1064                         self._v = v
1065                         ob_x(pn, 'we boarded')
1066                         self.expire_garbage(timestamp)
1067                         return d(' '.join(dm))
1068
1069                 if self._v is None:
1070                         return d('no vessel')
1071
1072                 m = rm('(\\w+) has come aboard\\.$')
1073                 if m: return ob1('boarded');
1074
1075                 m = rm('You have ordered (\\w+) to do some (\\S.*\\S)\\.$')
1076                 if m:
1077                         (who,what) = m.groups()
1078                         pa = ob_x(who,'ord '+what)
1079                         if what == 'Gunning':
1080                                 pa.gunner = True
1081                         return d('duty order')
1082
1083                 m = rm('(\\w+) abandoned a (\\S.*\\S) station\\.$')
1084                 if m: oba('stopped'); return d("end")
1085
1086                 def chat_core(speaker, chan):
1087                         try: pa = self._pl[speaker]
1088                         except KeyError: return 'mystery'
1089                         if pa.v is not None and pa.v is not self._v:
1090                                 return 'elsewhere'
1091                         pa.last_chat_time = timestamp
1092                         pa.last_chat_chan = chan
1093                         self.force_redisplay()
1094                         return 'here'
1095
1096                 def chat(chan):
1097                         speaker = m.group(1)
1098                         dm = chat_core(speaker, chan)
1099                         return d('chat %s %s' % (chan, dm))
1100
1101                 def chat_metacmd(chan):
1102                         (cmdr, metacmd) = m.groups()
1103                         whynot = self._command(
1104                                 cmdr, metacmd, chan, timestamp, d)
1105                         if whynot is not None:
1106                                 return chat(chan)
1107                         else:
1108                                 chat_core(cmdr, 'cmd '+chan)
1109
1110                 m = rm('(\\w+) (?:issued an order|ordered everyone) "')
1111                 if m: return ob1('general order');
1112
1113                 m = rm('(\\w+) says, "')
1114                 if m: return chat('public')
1115
1116                 m = rm('(\\w+) tells ye, "')
1117                 if m: return chat('private')
1118
1119                 m = rm('Ye told (\\w+), "(.*)"$')
1120                 if m: return chat_metacmd('private')
1121
1122                 m = rm('(\\w+) flag officer chats, "')
1123                 if m: return chat('flag officer')
1124
1125                 m = rm('(\\w+) officer chats, "(.*)"$')
1126                 if m: return chat_metacmd('officer')
1127
1128                 m = rm('Ye accepted the offer to job with ')
1129                 if m: return disembark_me('jobbing')
1130
1131                 m = rm('Ye hop on the ferry and are whisked away ')
1132                 if m: return disembark_me('ferry')
1133
1134                 m = rm('Whisking away to yer home on the magical winds')
1135                 if m: return disembark_me('home')
1136
1137                 m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
1138                 if m:
1139                         pl = m.group(1).split(', ')
1140                         if not self._myself.name in pl:
1141                                 return d('lost melee')
1142                         for pn in pl:
1143                                 if ' ' in pn: continue
1144                                 ob_x(pn,'won melee')
1145                         return d('won melee')
1146
1147                 m = rm('(\\w+) is eliminated\\!')
1148                 if m: return ob1('eliminated in fray');
1149
1150                 m = rm('(\\w+) has driven \w+ from the ship\\!')
1151                 if m: return ob1('boarder repelled');
1152
1153                 m = rm('\w+ has bested (\\w+), and turns'+
1154                         ' to the rest of the ship\\.')
1155                 if m: return ob1('boarder unrepelled');
1156
1157                 pirate = rm_crew("(\\w+) has taken a job with '(.*)'\\.")
1158                 if pirate: return jb(pirate, 'ashore')
1159
1160                 pirate = rm_crew("(\\w+) has left '(.*)'\\.")
1161                 if pirate:
1162                         disembark(self._v, timestamp, pirate, 'left crew')
1163                         return d('left crew')
1164
1165                 m = rm('(\w+) has applied for the posted job\.')
1166                 if m: return jb(m.group(1), 'applied')
1167
1168                 pirate= rm_crew("(\\w+) has been invited to job for '(.*)'\\.")
1169                 if pirate: return jb(pirate, 'invited')
1170
1171                 pirate = rm_crew("(\\w+) declined the job offer for '(.*)'\\.")
1172                 if pirate: return jb(pirate, 'declined')
1173
1174                 m = rm('(\\w+) has left the vessel\.')
1175                 if m:
1176                         pirate = m.group(1)
1177                         disembark(self._v, timestamp, pirate, 'disembarked')
1178                         return d('disembarked')
1179
1180                 return d('not-matched')
1181
1182         def _str_pa(self, pn, pa):
1183                 assert self._pl[pn] == pa
1184                 s = ' '*20 + "%s %-*s %13s %-30s %13s %-20s %13s" % (
1185                         (' ','G')[pa.gunner],
1186                         max_pirate_namelen, pn,
1187                         pa.last_time, pa.last_event,
1188                         pa.last_chat_time, pa.last_chat_chan,
1189                         pa.jobber)
1190                 if pa.expires is not None:
1191                         s += " %-5d" % (pa.expires - pa.last_time)
1192                 s += "\n"
1193                 return s
1194
1195         def _str_vessel(self, vn, v):
1196                 s = ' vessel %s\n' % vn
1197                 s += ' '*20 + "%-*s   %13s\n" % (
1198                                 max_pirate_namelen, '#lastinfo',
1199                                 v['#lastinfo'])
1200                 assert v['#name'] == vn
1201                 for pn in sorted(v.keys()):
1202                         if pn.startswith('#'): continue
1203                         pa = v[pn]
1204                         assert pa.v == v
1205                         s += self._str_pa(pn,pa)
1206                 return s
1207
1208         def __str__(self):
1209                 s = '''<ChatLogTracker
1210  myself %s
1211  vessel %s
1212 '''                     % (self._myself.name, self._vessel)
1213                 assert ((self._v is None and self._vessel is None) or
1214                         (self._v is self._vl[self._vessel]))
1215                 if self._vessel is not None:
1216                         s += self._str_vessel(self._vessel, self._v)
1217                 for vn in sorted(self._vl.keys()):
1218                         if vn == self._vessel: continue
1219                         s += self._str_vessel(vn, self._vl[vn])
1220                 s += " elsewhere\n"
1221                 for p in self._pl:
1222                         pa = self._pl[p]
1223                         if pa.v is not None:
1224                                 assert pa.v[p] is pa
1225                                 assert pa.v in self._vl.values()
1226                         else:
1227                                 s += self._str_pa(pa.name, pa)
1228                 s += '>\n'
1229                 return s
1230
1231         def catchup(self, progress=None):
1232                 while True:
1233                         more = self._f.readline()
1234                         if not more: break
1235
1236                         self._progress[0] += len(more)
1237                         if progress: progress.progress(*self._progress)
1238
1239                         self._lbuf += more
1240                         if self._lbuf.endswith('\n'):
1241                                 self.chatline(self._lbuf.rstrip())
1242                                 self._lbuf = ''
1243                                 if opts.debug >= 2:
1244                                         debug(self.__str__())
1245                 self._expire_jobbers(time.time())
1246
1247                 if progress: progress.caughtup()
1248
1249         def changed(self):
1250                 rv = self._need_redisplay
1251                 self._need_redisplay = False
1252                 return rv
1253         def myname(self):
1254                 # returns our pirate name
1255                 return self._myself.name
1256         def vesselname(self):
1257                 # returns the vessel name we're aboard or None
1258                 return self._vessel
1259         def lastvesselname(self):
1260                 # returns the last vessel name we were aboard or None
1261                 return self._lastvessel
1262         def aboard(self, vesselname=True):
1263                 # returns a list of PirateAboard the vessel
1264                 #  sorted by pirate name
1265                 #  you can pass this None and you'll get []
1266                 #  or True for the current vessel (which is the default)
1267                 #  the returned value is a fresh list of persistent
1268                 #  PirateAboard objects
1269                 if vesselname is True: v = self._v
1270                 else: v = self._vl.get(vesselname.title())
1271                 if v is None: return []
1272                 return [ v[pn]
1273                          for pn in sorted(v.keys())
1274                          if not pn.startswith('#') ]
1275         def jobbers(self):
1276                 # returns a the jobbers' PirateAboards,
1277                 # sorted by jobber class and reverse of expiry time
1278                 l = [ pa
1279                       for pa in self._pl.values()
1280                       if pa.jobber is not None
1281                     ]
1282                 def compar_key(pa):
1283                         return (pa.jobber, -pa.expires)
1284                 l.sort(key = compar_key)
1285                 return l
1286
1287 #---------- implementations of actual operation modes ----------
1288
1289 def do_pirate(pirates, bu):
1290         print '{'
1291         for pirate in pirates:
1292                 info = PirateInfo(pirate)
1293                 print '%s: %s,' % (`pirate`, info)
1294         print '}'
1295
1296 def prep_crew_of(args, bu, max_age=300):
1297         if len(args) != 1: bu('crew-of takes one pirate name')
1298         pi = PirateInfo(args[0], max_age)
1299         if pi.crew is None: return None
1300         return CrewInfo(pi.crew[0], max_age)
1301
1302 def do_crew_of(args, bu):
1303         ci = prep_crew_of(args, bu)
1304         print ci
1305
1306 def do_flag_of(args, bu):
1307         if len(args) != 1: bu('flag-of takes one pirate name')
1308         max_age = 300
1309         pi = PirateInfo(args[0], max_age)
1310         if pi.flag is None: fi = None
1311         else: fi = FlagInfo(pi.flag[0], max_age)
1312         print fi
1313
1314 def do_standings_crew_of(args, bu):
1315         ci = prep_crew_of(args, bu, 60)
1316         tab = StandingsTable(sys.stdout)
1317         tab.headings()
1318         for (rank, members) in ci.crew:
1319                 if not members: continue
1320                 tab.literalline('')
1321                 tab.literalline('%s:' % rank)
1322                 for p in members:
1323                         pi = PirateInfo(p, random.randint(900,1800))
1324                         tab.pirate(pi)
1325
1326 def do_ocean(args, bu):
1327         if (len(args)): bu('ocean takes no further arguments')
1328         fetcher.default_ocean()
1329         oi = OceanInfo(IslandExtendedInfo)
1330         oi.collect()
1331         print oi
1332         for islename in sorted(oi.islands.keys()):
1333                 isle = oi.islands[islename]
1334                 print isle
1335
1336 #----- modes which use the chat log parser are quite complex -----
1337
1338 class ProgressPrintPercentage:
1339         def __init__(self, f=sys.stdout):
1340                 self._f = f
1341         def progress_string(self,done,total):
1342                 return "scan chat logs %3d%%\r" % ((done*100) / total)
1343         def progress(self,*a):
1344                 self._f.write(self.progress_string(*a))
1345                 self._f.flush()
1346         def show_init(self, pirate, ocean):
1347                 print >>self._f, 'Starting up, %s on the %s ocean' % (
1348                         pirate, ocean)
1349         def caughtup(self):
1350                 self._f.write('                   \r')
1351                 self._f.flush()
1352
1353 def prep_chat_log(args, bu,
1354                 progress=ProgressPrintPercentage(),
1355                 max_myself_age=3600):
1356         if len(args) != 1: bu('this action takes only chat log filename')
1357         logfn = args[0]
1358         logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_'
1359         match = regexp.match(logfn_re, logfn)
1360         if not match: bu('chat log filename is not in expected format')
1361         (pirate, ocean) = match.groups()
1362         fetcher.default_ocean(ocean)
1363
1364         progress.show_init(pirate, fetcher.ocean)
1365         myself = PirateInfo(pirate,max_myself_age)
1366         track = ChatLogTracker(myself, logfn)
1367
1368         opts.debug -= 2
1369         track.catchup(progress)
1370         opts.debug += 2
1371
1372         track.force_redisplay()
1373
1374         return (myself, track)
1375
1376 def do_track_chat_log(args, bu):
1377         (myself, track) = prep_chat_log(args, bu)
1378         while True:
1379                 track.catchup()
1380                 if track.changed():
1381                         print track
1382                 sleep(0.5 + 0.5 * random.random())
1383
1384 #----- ship management aid -----
1385
1386 class Display_dumb(ProgressPrintPercentage):
1387         def __init__(self):
1388                 ProgressPrintPercentage.__init__(self)
1389         def show(self, s):
1390                 print '\n\n', s;
1391         def realstart(self):
1392                 pass
1393
1394 class Display_overwrite(ProgressPrintPercentage):
1395         def __init__(self):
1396                 ProgressPrintPercentage.__init__(self)
1397
1398                 null = file('/dev/null','w')
1399                 curses.setupterm(fd=null.fileno())
1400
1401                 self._clear = curses.tigetstr('clear')
1402                 if not self._clear:
1403                         self._debug('missing clear!')
1404                         self.show = Display_dumb.show
1405                         return
1406
1407                 self._t = {'el':'', 'ed':''}
1408                 if not self._init_sophisticated():
1409                         for k in self._t.keys(): self._t[k] = ''
1410                         self._t['ho'] = self._clear
1411
1412         def _debug(self,m): debug('display overwrite: '+m)
1413
1414         def _init_sophisticated(self):
1415                 for k in self._t.keys():
1416                         s = curses.tigetstr(k)
1417                         self._t[k] = s
1418                 self._t['ho'] = curses.tigetstr('ho')
1419                 if not self._t['ho']:
1420                         cup = curses.tigetstr('cup')
1421                         self._t['ho'] = curses.tparm(cup,0,0)
1422                 missing = [k for k in self._t.keys() if not self._t[k]]
1423                 if missing:
1424                         self.debug('missing '+(' '.join(missing)))
1425                         return 0
1426                 return 1
1427
1428         def show(self, s):
1429                 w = sys.stdout.write
1430                 def wti(k): w(self._t[k])
1431
1432                 wti('ho')
1433                 nl = ''
1434                 for l in s.rstrip().split('\n'):
1435                         w(nl)
1436                         w(l)
1437                         wti('el')
1438                         nl = '\r\n'
1439                 wti('ed')
1440                 w(' ')
1441                 sys.stdout.flush()
1442
1443         def realstart(self):
1444                 sys.stdout.write(self._clear)
1445                 sys.stdout.flush()
1446
1447
1448 def do_ship_aid(args, bu):
1449         if opts.ship_duty is None: opts.ship_duty = True
1450
1451         displayer = globals()['Display_'+opts.display]()
1452
1453         (myself, track) = prep_chat_log(args, bu, progress=displayer)
1454
1455         displayer.realstart()
1456
1457         if os.isatty(0): kr_create = KeystrokeReader
1458         else: kr_create = DummyKeystrokeReader
1459
1460         try:
1461                 kreader = kr_create(0, 10)
1462                 ship_aid_core(myself, track, displayer, kreader)
1463         finally:
1464                 kreader.stop()
1465                 print '\n'
1466
1467 class KeyBasedSorter:
1468         def compar_key_pa(self, pa):
1469                 pi = pa.pirate_info()
1470                 if pi is None: return None
1471                 return self.compar_key(pi)
1472         def lsort_pa(self, l):
1473                 l.sort(key = self.compar_key_pa)
1474
1475 class NameSorter(KeyBasedSorter):
1476         def compar_key(self, pi): return pi.name
1477         def desc(self): return 'name'
1478
1479 class SkillSorter(NameSorter):
1480         def __init__(self, relevant):
1481                 self._want = frozenset(relevant.split('/'))
1482                 self._avoid = set()
1483                 for p in core_duty_puzzles:
1484                         if isinstance(p,basestring): self._avoid.add(p)
1485                         else: self._avoid |= set(p)
1486                 self._avoid -= self._want
1487                 self._desc = '%s' % relevant
1488
1489         def desc(self): return self._desc
1490
1491         def compar_key(self, pi):
1492                 best_want = max([
1493                         pi.standings.get(puz,-1)
1494                         for puz in self._want
1495                         ])
1496                 best_avoid = [
1497                         -pi.standings.get(puz,standing_limit)
1498                         for puz in self._avoid
1499                         ]
1500                 best_avoid.sort()
1501                 def negate(x): return -x
1502                 debug('compar_key %s bw=%s ba=%s' % (pi.name, `best_want`,
1503                         `best_avoid`))
1504                 return (-best_want, map(negate, best_avoid), pi.name)
1505
1506 def ship_aid_core(myself, track, displayer, kreader):
1507
1508         def find_vessel():
1509                 vn = track.vesselname()
1510                 if vn: return (vn, " on board the %s" % vn)
1511                 vn = track.lastvesselname()
1512                 if vn: return (vn, " ashore from the %s" % vn)
1513                 return (None, " not on a vessel")
1514
1515         def timeevent(t,e):
1516                 if t is None: return ' ' * 22
1517                 return " %-4s %-16s" % (format_time_interval(now - t),e)
1518
1519         displayer.show(track.myname() + find_vessel()[1] + '...')
1520
1521         rotate_nya = '/-\\'
1522
1523         sort = NameSorter()
1524         clicmd = None
1525         clierr = None
1526         cliexec = None
1527
1528         while True:
1529                 track.catchup()
1530                 now = time.time()
1531
1532                 (vn, vs) = find_vessel()
1533
1534                 s = ''
1535                 if cliexec is not None:
1536                         s += '...'
1537                 elif clierr is not None:
1538                         s += 'Error: '+clierr
1539                 elif clicmd is not None:
1540                         s += '/' + clicmd
1541                 else:
1542                         s = track.myname() + vs
1543                         s += " at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
1544                         s += kreader.info()
1545                 s += '\n'
1546
1547                 tbl_s = StringIO()
1548                 tbl = StandingsTable(tbl_s)
1549
1550                 aboard = track.aboard(vn)
1551                 sort.lsort_pa(aboard)
1552
1553                 jobbers = track.jobbers()
1554
1555                 if track.vesselname(): howmany = 'aboard: %2d' % len(aboard)
1556                 else: howmany = ''
1557
1558                 tbl.headings(howmany, '  sorted by '+sort.desc())
1559
1560                 last_jobber = None
1561
1562                 for pa in aboard + jobbers:
1563                         if pa.jobber != last_jobber:
1564                                 last_jobber = pa.jobber
1565                                 tbl.literalline('')
1566                                 tbl.literalline('jobbers '+last_jobber)
1567
1568                         pi = pa.pirate_info()
1569
1570                         xs = ''
1571                         if pa.gunner: xs += 'G '
1572                         else: xs += '  '
1573                         xs += timeevent(pa.last_time, pa.last_event)
1574                         xs += timeevent(pa.last_chat_time, pa.last_chat_chan)
1575
1576                         if pi is None:
1577                                 tbl.pirate_dummy(pa.name, rotate_nya[0], xs)
1578                         else:
1579                                 tbl.pirate(pi, xs)
1580
1581                 s += tbl_s.getvalue()
1582                 displayer.show(s)
1583                 tbl_s.close()
1584
1585                 if cliexec is not None:
1586                         clierr = track.local_command("/"+cliexec.strip())
1587                         cliexec = None
1588                         continue
1589
1590                 k = kreader.getch()
1591                 if k is None:
1592                         rotate_nya = rotate_nya[1:3] + rotate_nya[0]
1593                         continue
1594
1595                 if clierr is not None:
1596                         clierr = None
1597                         continue
1598
1599                 if clicmd is not None:
1600                         if k == '\r' or k == '\n':
1601                                 cliexec = clicmd
1602                                 clicmd = clicmdbase
1603                         elif k == '\e' and clicmd != "":
1604                                 clicmd = clicmdbase
1605                         elif k == '\33':
1606                                 clicmd = None
1607                         elif k == '\b' or k == '\177':
1608                                 clicmd = clicmd[ 0 : len(clicmd)-1 ]
1609                         else:
1610                                 clicmd += k
1611                         continue
1612
1613                 if k == 'q': break
1614                 elif k == 'g': sort = SkillSorter('Gunning')
1615                 elif k == 'c': sort = SkillSorter('Carpentry')
1616                 elif k == 's': sort = SkillSorter('Sailing/Rigging')
1617                 elif k == 'b': sort = SkillSorter('Bilging')
1618                 elif k == 'n': sort = SkillSorter('Navigating')
1619                 elif k == 'd': sort = SkillSorter('Battle Navigation')
1620                 elif k == 't': sort = SkillSorter('Treasure Haul')
1621                 elif k == 'a': sort = NameSorter()
1622                 elif k == '/': clicmdbase = ""; clicmd = clicmdbase
1623                 elif k == '+': clicmdbase = "a "; clicmd = clicmdbase
1624                 else: pass # unknown key command
1625
1626 #---------- individual keystroke input ----------
1627
1628 class DummyKeystrokeReader:
1629         def __init__(self,fd,timeout_dummy): pass
1630         def stop(self): pass
1631         def getch(self): sleep(1); return None
1632         def info(self): return ' [noninteractive]'
1633
1634 class KeystrokeReader(DummyKeystrokeReader):
1635         def __init__(self, fd, timeout_decisec=0):
1636                 self._fd = fd
1637                 self._saved = termios.tcgetattr(fd)
1638                 a = termios.tcgetattr(fd)
1639                 a[3] &= ~(termios.ECHO | termios.ECHONL |
1640                           termios.ICANON | termios.IEXTEN)
1641                 a[6][termios.VMIN] = 0
1642                 a[6][termios.VTIME] = timeout_decisec
1643                 termios.tcsetattr(fd, termios.TCSANOW, a)
1644         def stop(self):
1645                 termios.tcsetattr(self._fd, termios.TCSANOW, self._saved)
1646         def getch(self):
1647                 debug_flush()
1648                 byte = os.read(self._fd, 1)
1649                 if not len(byte): return None
1650                 return byte
1651         def info(self):
1652                 return ''
1653
1654 #---------- main program ----------
1655
1656 def main():
1657         global opts, fetcher, yppedia
1658
1659         pa = OptionParser(
1660 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
1661 actions:
1662  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
1663  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
1664  yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
1665  yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG
1666  yoweb-scrape [options] ship-aid CHAT-LOG  (must be .../PIRATE_OCEAN_chat-log*)
1667
1668 display modes (for --display) apply to ship-aid:
1669  --display=dumb       just print new information, scrolling the screen
1670  --display=overwrite  use cursor motion, selective clear, etc. to redraw at top''')
1671         ao = pa.add_option
1672         ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
1673                 help='select ocean OCEAN')
1674         ao('--cache-dir', dest='cache_dir', metavar='DIR',
1675                 default='~/.yoweb-scrape-cache',
1676                 help='cache yoweb pages in DIR')
1677         ao('-D','--debug', action='count', dest='debug', default=0,
1678                 help='enable debugging output')
1679         ao('--debug-fd', type='int', dest='debug_fd',
1680                 help='write any debugging output to specified fd')
1681         ao('-q','--quiet', action='store_true', dest='quiet',
1682                 help='suppress warning output')
1683         ao('--display', action='store', dest='display',
1684                 type='choice', choices=['dumb','overwrite'],
1685                 help='how to display ship aid')
1686         ao('--local-ypp-dir', action='store', dest='localhtml',
1687                 help='get yppedia pages from local directory LOCALHTML'+
1688                         ' instead of via HTTP')
1689
1690         ao_jt = lambda wh, t: ao(
1691                 '--timeout-sa-'+wh, action='store', dest='timeout_'+wh,
1692                 default=t, help=('set timeout for expiring %s jobbers' % wh))
1693         ao_jt('applied',      120)
1694         ao_jt('invited',      120)
1695         ao_jt('declined',      30)
1696         ao_jt('ashore',      1800)
1697
1698         ao('--ship-duty', action='store_true', dest='ship_duty',
1699                 help='show ship duty station puzzles')
1700         ao('--all-puzzles', action='store_false', dest='ship_duty',
1701                 help='show all puzzles, not just ship duty stations')
1702
1703         ao('--min-cache-reuse', type='int', dest='min_max_age',
1704                 metavar='SECONDS', default=60,
1705                 help='always reuse cache yoweb data if no older than this')
1706
1707         (opts,args) = pa.parse_args()
1708         random.seed()
1709
1710         if len(args) < 1:
1711                 print >>sys.stderr, copyright_info
1712                 pa.error('need a mode argument')
1713
1714         if opts.debug_fd is not None:
1715                 opts.debug_file = os.fdopen(opts.debug_fd, 'w')
1716         else:
1717                 opts.debug_file = sys.stdout
1718
1719         mode = args[0]
1720         mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
1721         try: mode_fn = globals()[mode_fn_name]
1722         except KeyError: pa.error('unknown mode "%s"' % mode)
1723
1724         # fixed parameters
1725         opts.expire_age = max(3600, opts.min_max_age)
1726
1727         opts.ship_reboard_clearout = 3600
1728
1729         if opts.cache_dir.startswith('~/'):
1730                 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
1731
1732         if opts.display is None:
1733                 if ((opts.debug > 0 and opts.debug_fd is None)
1734                     or not os.isatty(sys.stdout.fileno())):
1735                         opts.display = 'dumb'
1736                 else:
1737                         opts.display = 'overwrite'
1738
1739         fetcher = Yoweb(opts.ocean, opts.cache_dir)
1740         yppedia = Yppedia(opts.cache_dir)
1741
1742         mode_fn(args[1:], pa.error)
1743
1744 main()