yoweb-scrape

   1 #!/usr/bin/python
   2 # This is part of ypp-sc-tools, a set of third-party tools for assisting
   3 # players of Yohoho Puzzle Pirates.
   4 #
   5 # Copyright (C) 2009 Ian Jackson <ijackson@chiark.greenend.org.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 #
  20 # Yohoho and Puzzle Pirates are probably trademarks of Three Rings and
  21 # are used without permission.  This program is not endorsed or
  22 # sponsored by Three Rings.
  23
  24 copyright_info = '''
  25 yoweb-scrape is part of ypp-sc-tools  Copyright (C) 2009 Ian Jackson
  26 This program comes with ABSOLUTELY NO WARRANTY; this is free software,
  27 and you are welcome to redistribute it under certain conditions.
  28 For details, read the top of the yoweb-scrape file.
  29 '''
  30
  31 #---------- setup ----------
  32
  33 import signal
  34 signal.signal(signal.SIGINT, signal.SIG_DFL)
  35
  36 import os
  37 import time
  38 import urllib
  39 import urllib2
  40 import errno
  41 import sys
  42 import re as regexp
  43 import random
  44 import curses
  45 import termios
  46 import random
  47 import subprocess
  48 from optparse import OptionParser
  49 from StringIO import StringIO
  50
  51 from BeautifulSoup import BeautifulSoup
  52
  53 opts = None
  54
  55 #---------- YPP parameters and arrays ----------
  56
  57 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
  58         '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
  59         '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
  60         '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
  61
  62 core_duty_puzzles = [
  63                 'Gunning',
  64                 ['Sailing','Rigging'],
  65                 'Bilging',
  66                 'Carpentry',
  67                 ]
  68
  69 duty_puzzles = ([ 'Navigating', 'Battle Navigation' ] +
  70                 core_duty_puzzles +
  71                 [ 'Treasure Haul' ])
  72
  73 standingvals = ('Able/Proficient/Distinguished/Respected/Master'+
  74                 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
  75 standing_limit = len(standingvals)
  76
  77 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
  78
  79 max_pirate_namelen = 12
  80
  81
  82 #---------- general utilities ----------
  83
  84 def debug(m):
  85         if opts.debug > 0:
  86                 print >>opts.debug_file, m
  87
  88 def debug_flush():
  89         if opts.debug > 0:
  90                 opts.debug_file.flush()
  91
  92 def sleep(seconds):
  93         debug_flush()
  94         time.sleep(seconds)
  95
  96 def format_time_interval(ti):
  97         if ti < 120: return '%d:%02d' % (ti / 60, ti % 60)
  98         if ti < 7200: return '%2dm' % (ti / 60)
  99         if ti < 86400: return '%dh' % (ti / 3600)
 100         return '%dd' % (ti / 86400)
 101
 102 def yppsc_dir():
 103         lib = os.getenv("YPPSC_YARRG_SRCBASE")
 104         if lib is not None: return lib
 105         lib = sys.argv[0]
 106         lib = regexp.sub('/[^/]+$', '', lib)
 107         os.environ["YPPSC_YARRG_SRCBASE"] = lib
 108         return lib
 109
 110 #---------- caching and rate-limiting data fetcher ----------
 111
 112 class Fetcher:
 113         def __init__(self, cachedir):
 114                 debug('Fetcher init %s' % cachedir)
 115                 self.cachedir = cachedir
 116                 try: os.mkdir(cachedir)
 117                 except (OSError,IOError), oe:
 118                         if oe.errno != errno.EEXIST: raise
 119                 self._cache_scan(time.time())
 120
 121         def _cache_scan(self, now):
 122                 # returns list of ages, unsorted
 123                 ages = []
 124                 debug('Fetcher   scan_cache')
 125                 for leaf in os.listdir(self.cachedir):
 126                         if not leaf.startswith('#'): continue
 127                         path = self.cachedir + '/' + leaf
 128                         try: s = os.stat(path)
 129                         except (OSError,IOError), oe:
 130                                 if oe.errno != errno.ENOENT: raise
 131                                 continue
 132                         age = now - s.st_mtime
 133                         if age > opts.expire_age:
 134                                 debug('Fetcher    expire %d %s' % (age, path))
 135                                 try: os.remove(path)
 136                                 except (OSError,IOError), oe:
 137                                         if oe.errno != errno.ENOENT: raise
 138                                 continue
 139                         ages.append(age)
 140                 return ages
 141
 142         def need_wait(self, now, imaginary=[]):
 143                 ages = self._cache_scan(now)
 144                 ages += imaginary
 145                 ages.sort()
 146                 debug('Fetcher   ages ' + `ages`)
 147                 min_age = 1
 148                 need_wait = 0
 149                 for age in ages:
 150                         if age < min_age and age <= 5:
 151                                 debug('Fetcher   morewait min=%d age=%d' %
 152                                         (min_age, age))
 153                                 need_wait = max(need_wait, min_age - age)
 154                         min_age += 3
 155                         min_age *= 1.25
 156                 if need_wait > 0:
 157                         need_wait += random.random() - 0.5
 158                 return need_wait
 159
 160         def _rate_limit_cache_clean(self, now):
 161                 need_wait = self.need_wait(now)
 162                 if need_wait > 0:
 163                         debug('Fetcher   wait %d' % need_wait)
 164                         sleep(need_wait)
 165
 166         def fetch(self, url, max_age):
 167                 debug('Fetcher fetch %s' % url)
 168                 cache_corename = urllib.quote_plus(url)
 169                 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
 170                 try: f = file(cache_item, 'r')
 171                 except (OSError,IOError), oe:
 172                         if oe.errno != errno.ENOENT: raise
 173                         f = None
 174                 now = time.time()
 175                 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
 176                 if f is not None:
 177                         s = os.fstat(f.fileno())
 178                         age = now - s.st_mtime
 179                         if age > max_age:
 180                                 debug('Fetcher  stale %d < %d'% (max_age, age))
 181                                 f = None
 182                 if f is not None:
 183                         data = f.read()
 184                         f.close()
 185                         debug('Fetcher  cached %d > %d' % (max_age, age))
 186                         return data
 187
 188                 debug('Fetcher  fetch')
 189                 self._rate_limit_cache_clean(now)
 190
 191                 stream = urllib2.urlopen(url)
 192                 data = stream.read()
 193                 cache_tmp = "%s/#%s~%d#" % (
 194                         self.cachedir, cache_corename, os.getpid())
 195                 f = file(cache_tmp, 'w')
 196                 f.write(data)
 197                 f.close()
 198                 os.rename(cache_tmp, cache_item)
 199                 debug('Fetcher  stored')
 200                 return data
 201
 202 class Yoweb(Fetcher):
 203         def __init__(self, ocean, cachedir):
 204                 debug('Yoweb init %s' % cachedir)
 205                 self.ocean = ocean
 206                 Fetcher.__init__(self, cachedir)
 207
 208         def default_ocean(self, ocean='ice'):
 209                 if self.ocean is None:
 210                         self.ocean = ocean
 211
 212         def yoweb(self, kind, tail, max_age):
 213                 self.default_ocean()
 214                 assert(self.ocean)
 215                 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
 216                         self.ocean, kind, tail)
 217                 return self.fetch(url, max_age)
 218
 219 class Yppedia(Fetcher):
 220         def __init__(self, cachedir):
 221                 debug('Yoweb init %s' % cachedir)
 222                 self.base = 'http://yppedia.puzzlepirates.com/'
 223                 self.localhtml = opts.localhtml
 224                 Fetcher.__init__(self, cachedir)
 225
 226         def __call__(self, rhs):
 227                 if self.localhtml is None:
 228                         url = self.base + rhs
 229                         debug('Yppedia retrieving YPP '+url);
 230                         return self.fetch(url, 3000)
 231                 else:
 232                         return file(opts.localhtml + '/' + rhs, 'r')
 233
 234 #---------- logging assistance for troubled screenscrapers ----------
 235
 236 class SoupLog:
 237         def __init__(self):
 238                 self.msgs = [ ]
 239         def msg(self, m):
 240                 self.msgs.append(m)
 241         def soupm(self, obj, m):
 242                 self.msg(m + '; in ' + `obj`)
 243         def needs_msgs(self, child_souplog):
 244                 self.msgs += child_souplog.msgs
 245                 child_souplog.msgs = [ ]
 246
 247 def soup_text(obj):
 248         str = ''.join(obj.findAll(text=True))
 249         return str.strip()
 250
 251 class SomethingSoupInfo(SoupLog):
 252         def __init__(self, kind, tail, max_age):
 253                 SoupLog.__init__(self)
 254                 html = fetcher.yoweb(kind, tail, max_age)
 255                 self._soup = BeautifulSoup(html,
 256                         convertEntities=BeautifulSoup.HTML_ENTITIES
 257                         )
 258
 259 #---------- scraper for pirate pages ----------
 260
 261 class PirateInfo(SomethingSoupInfo):
 262         # Public data members:
 263         #  pi.standings = { 'Treasure Haul': 'Able' ... }
 264         #  pi.name = name
 265         #  pi.crew = (id, name)
 266         #  pi.flag = (id, name)
 267         #  pi.msgs = [ 'message describing problem with scrape' ]
 268
 269         def __init__(self, pirate, max_age=300):
 270                 SomethingSoupInfo.__init__(self,
 271                         'pirate.wm?target=', pirate, max_age)
 272                 self.name = pirate
 273                 self._find_standings()
 274                 self.crew = self._find_crewflag('crew',
 275                         '^/yoweb/crew/info\\.wm')
 276                 self.flag = self._find_crewflag('flag',
 277                         '^/yoweb/flag/info\\.wm')
 278
 279         def _find_standings(self):
 280                 imgs = self._soup.findAll('img',
 281                         src=regexp.compile('/yoweb/images/stat.*'))
 282                 re = regexp.compile(
 283 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
 284                         )
 285                 standings = { }
 286
 287                 for skill in puzzles:
 288                         standings[skill] = [ ]
 289
 290                 skl = SoupLog()
 291
 292                 for img in imgs:
 293                         try: puzzle = img['alt']
 294                         except KeyError: continue
 295
 296                         if not puzzle in puzzles:
 297                                 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
 298                                 continue
 299                         key = img.findParent('td')
 300                         if key is None:
 301                                 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
 302                                 continue
 303                         valelem = key.findNextSibling('td')
 304                         if valelem is None:
 305                                 skl.soupm(key, 'puzzle missing sibling "%s"'
 306                                         % puzzle)
 307                                 continue
 308                         valstr = soup_text(valelem)
 309                         match = re.match(valstr)
 310                         if match is None:
 311                                 skl.soupm(key, ('puzzle "%s" unparseable'+
 312                                         ' standing "%s"') % (puzzle, valstr))
 313                                 continue
 314                         standing = match.group(match.lastindex)
 315                         standings[puzzle].append(standing)
 316
 317                 self.standings = { }
 318
 319                 for puzzle in puzzles:
 320                         sl = standings[puzzle]
 321                         if len(sl) > 1:
 322                                 skl.msg('puzzle "%s" multiple standings %s' %
 323                                                 (puzzle, `sl`))
 324                                 continue
 325                         if not sl:
 326                                 skl.msg('puzzle "%s" no standing found' % puzzle)
 327                                 continue
 328                         standing = sl[0]
 329                         for i in range(0, standing_limit):
 330                                 if standing == standingvals[i]:
 331                                         self.standings[puzzle] = i
 332                         if not puzzle in self.standings:
 333                                 skl.msg('puzzle "%s" unknown standing "%s"' %
 334                                         (puzzle, standing))
 335
 336                 all_standings_ok = True
 337                 for puzzle in puzzles:
 338                         if not puzzle in self.standings:
 339                                 self.needs_msgs(skl)
 340
 341         def _find_crewflag(self, cf, yoweb_re):
 342                 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
 343                 if len(things) != 1:
 344                         self.msg('zero or several %s id references found' % cf)
 345                         return None
 346                 thing = things[0]
 347                 id_re = '\\b%sid\\=(\\w+)$' % cf
 348                 id_haystack = thing['href']
 349                 match = regexp.compile(id_re).search(id_haystack)
 350                 if match is None:
 351                         self.soupm(thing, ('incomprehensible %s id ref'+
 352                                 ' (%s in %s)') % (cf, id_re, id_haystack))
 353                         return None
 354                 name = soup_text(thing)
 355                 return (match.group(1), name)
 356
 357         def __str__(self):
 358                 return `(self.crew, self.flag, self.standings, self.msgs)`
 359
 360 #---------- scraper for crew pages ----------
 361
 362 class CrewInfo(SomethingSoupInfo):
 363         # Public data members:
 364         #  ci.crew = [ ('Captain',        ['Pirate', ...]),
 365         #              ('Senior Officer', [...]),
 366         #               ... ]
 367         #  pi.msgs = [ 'message describing problem with scrape' ]
 368
 369         def __init__(self, crewid, max_age=300):
 370                 SomethingSoupInfo.__init__(self,
 371                         'crew/info.wm?crewid=', crewid, max_age)
 372                 self._find_crew()
 373
 374         def _find_crew(self):
 375                 self.crew = []
 376                 capts = self._soup.findAll('img',
 377                         src='/yoweb/images/crew-captain.png')
 378                 if len(capts) != 1:
 379                         self.msg('crew members: no. of captain images != 1')
 380                         return
 381                 tbl = capts[0]
 382                 while not tbl.find('a', href=pirate_ref_re):
 383                         tbl = tbl.findParent('table')
 384                         if not tbl:
 385                                 self.msg('crew members: cannot find table')
 386                                 return
 387                 current_rank_crew = None
 388                 crew_rank_re = regexp.compile('/yoweb/images/crew')
 389                 for row in tbl.contents:
 390                         # findAll(recurse=False)
 391                         if isinstance(row,basestring):
 392                                 continue
 393
 394                         is_rank = row.find('img', attrs={'src': crew_rank_re})
 395                         if is_rank:
 396                                 rank = soup_text(row)
 397                                 current_rank_crew = []
 398                                 self.crew.append((rank, current_rank_crew))
 399                                 continue
 400                         for cell in row.findAll('a', href=pirate_ref_re):
 401                                 if current_rank_crew is None:
 402                                         self.soupm(cell, 'crew members: crew'
 403                                                 ' before rank')
 404                                         continue
 405                                 current_rank_crew.append(soup_text(cell))
 406
 407         def __str__(self):
 408                 return `(self.crew, self.msgs)`
 409
 410 class FlagInfo(SomethingSoupInfo):
 411         # Public data members (after init):
 412         #
 413         #   name        #               string
 414         #
 415         #   relations[n] = (otherflagname, otherflagid, [stringfromyoweb],
 416         #               thisdeclaring, otherdeclaringmin, otherdeclaringmax)
 417         #               # where {this,other}declaring{,min,max} are:
 418         #               #       -1      {this,other} is declaring war
 419         #               #        0      {this,other} is not doing either
 420         #               #       +1      {this,other} is allying
 421         #   relation_byname[otherflagname] = relations[some_n]
 422         #   relation_byid[otherflagname] = relations[some_n]
 423         #
 424         #   islands[n] = (islandname, islandid)
 425         #
 426         def __init__(self, flagid, max_age=600):
 427                 SomethingSoupInfo.__init__(self,
 428                         'flag/info.wm?flagid=', flagid, max_age)
 429                 self._find_flag()
 430
 431         def _find_flag(self):
 432                 font2 = self._soup.find('font',{'size':'+2'})
 433                 self.name = font2.find('b').contents[0]
 434
 435                 self.relations = [ ]
 436                 self.relation_byname = { }
 437                 self.relation_byid = { }
 438                 self.islands = [ ]
 439
 440                 magnate = self._soup.find('img',{'src':
 441                         '/yoweb/images/repute-MAGNATE.png'})
 442                 warinfo = (magnate.findParent('table').findParent('tr').
 443                         findNextSibling('tr').findNext('td',{'align':'left'}))
 444
 445                 def warn(m):
 446                         print >>sys.stderr, 'WARNING: '+m
 447
 448                 def wi_warn(head, waritem):
 449                         warn('unknown warmap item: %s: %s' %
 450                                 (`head`, ``waritem``))
 451
 452                 def wihelp_item(waritem, thing):
 453                         url = waritem.get('href', None)
 454                         if url is None:
 455                                 return ('no url for '+thing,None,None)
 456                         m = regexp.search('\?'+thing+'id=(\d+)$', url)
 457                         if not m: return ('no '+thing+'id',None,None)
 458                         tid = m.group(1)
 459                         tname = waritem.string
 460                         if tname is None:
 461                                 return (thing+' name not just string',None,None)
 462                         return (None,tid,tname)
 463
 464                 def wi_alwar(head, waritem, thisdecl, othermin, othermax):
 465                         (err,flagid,flagname) = wihelp_item(waritem,'flag')
 466                         if err: return err
 467                         rel = self.relation_byid.get(flagid, None)
 468                         if rel: return 'flag id twice!'
 469                         if flagname in self.relation_byname:
 470                                 return 'flag name twice!'
 471                         rel = (flagname,flagid,head, thisdecl,othermin,othermax)
 472                         self.relations.append(rel)
 473                         self.relation_byid[flagid] = rel
 474                         self.relation_byname[flagid] = rel
 475
 476                 def wi_isle(head, waritem):
 477                         (err,isleid,islename) = wihelp_item(waritem,'island')
 478                         if err: return err
 479                         self.islands.append((isleid,islename))
 480
 481                 warmap = {
 482                         'Allied with':                  (wi_alwar,+1,+1,+1),
 483                         'Declaring war against':        (wi_alwar,-1, 0,+1),
 484                         'At war with':                  (wi_alwar,-1,-1,-1),
 485                         'Trying to form an alliance with': (wi_alwar,+1,-1,0),
 486                         'Islands controlled by this flag': (wi_isle,),
 487                         }
 488
 489                 how = (wi_warn, None)
 490
 491                 for waritem in warinfo.findAll(['font','a']):
 492                         if waritem is None: break
 493                         if waritem.name == 'font':
 494                                 colour = waritem.get('color',None)
 495                                 if colour.lstrip('#') != '958A5F':
 496                                         warn('strange colour %s in %s' %
 497                                                 (colour,``waritem``))
 498                                         continue
 499                                 head = waritem.string
 500                                 if head is None:
 501                                         warn('no head string in '+``waritem``)
 502                                         continue
 503                                 head = regexp.sub('\\s+', ' ', head).strip()
 504                                 head = head.rstrip(':')
 505                                 how = (head,) + warmap.get(head, (wi_warn,))
 506                                 continue
 507                         assert(waritem.name == 'a')
 508
 509                         debug('WARHOW %s(%s, waritem, *%s)' %
 510                                 (how[1], `how[0]`, `how[2:]`))
 511                         bad = how[1](how[0], waritem, *how[2:])
 512                         if bad:
 513                                 warn('bad waritem %s: %s: %s' % (`how[0]`,
 514                                         bad, ``waritem``))
 515
 516         def __str__(self):
 517                 return `(self.name, self.islands, self.relations)`
 518
 519 #---------- scraper for ocean info incl. embargoes etc. ----------
 520
 521 class IslandBasicInfo():
 522         # Public data attributes:
 523         #  ocean
 524         #  name
 525         # Public data attributes maybe set by caller:
 526         #  arch
 527         def __init__(self, ocean, islename):
 528                 self.ocean = ocean
 529                 self.name = islename
 530         def yppedia(self):
 531                 def q(x): return urllib.quote(x.replace(' ','_'))
 532                 url_rhs = q(self.name) + '_(' + q(self.ocean) + ')'
 533                 return yppedia(url_rhs)
 534         def __str__(self):
 535                 return `(self.ocean, self.name)`
 536
 537 class IslandExtendedInfo(IslandBasicInfo):
 538         # Public data attributes (inherited):
 539         #  ocean
 540         #  name
 541         # Public data attributes (additional):
 542         #  islandid
 543         #  yoweb_url
 544         #  flagid
 545         def __init__(self, ocean, islename):
 546                 IslandBasicInfo.__init__(self, ocean, islename)
 547                 self.islandid = None
 548                 self.yoweb_url = None
 549                 self._collect_yoweb()
 550                 self._collect_flagid()
 551
 552         def _collect_yoweb(self):
 553                 debug('IEI COLLECT YOWEB '+`self.name`)
 554                 self.islandid = None
 555                 self.yoweb_url = None
 556
 557                 soup = BeautifulSoup(self.yppedia())
 558                 content = soup.find('div', attrs = {'id': 'content'})
 559                 yoweb_re = regexp.compile('^http://\w+\.puzzlepirates\.com/'+
 560                         'yoweb/island/info\.wm\?islandid=(\d+)$')
 561                 a = soup.find('a', attrs = { 'href': yoweb_re })
 562                 if a is None:
 563                         debug('IEI COLLECT YOWEB '+`self.name`+' NONE')
 564                         return
 565
 566                 debug('IEI COLLECT YOWEB '+`self.name`+' GOT '+``a``)
 567                 self.yoweb_url = a['href']
 568                 m = yoweb_re.search(self.yoweb_url)
 569                 self.islandid = m.group(1)
 570
 571         def _collect_flagid(self):
 572                 self.flagid = None
 573
 574                 yo = self.yoweb_url
 575                 debug('IEI COLLECT FLAGID '+`self.name`+' URL '+`yo`)
 576                 if yo is None: return None
 577                 dataf = fetcher.fetch(yo, 1800)
 578                 soup = BeautifulSoup(dataf)
 579                 ruler_re = regexp.compile(
 580                         '/yoweb/flag/info\.wm\?flagid=(\d+)$')
 581                 ruler = soup.find('a', attrs = { 'href': ruler_re })
 582                 if not ruler:
 583                         debug('IEI COLLECT FLAGID '+`self.name`+' NONE')
 584                         return
 585                 debug('IEI COLLECT FLAGID '+`self.name`+' GOT '+``ruler``)
 586                 m = ruler_re.search(ruler['href'])
 587                 self.flagid = m.group(1)
 588
 589         def __str__(self):
 590                 return `(self.ocean, self.islandid, self.name,
 591                         self.yoweb_url, self.flagid)`
 592
 593 class IslandFlagInfo(IslandExtendedInfo):
 594         # Public data attributes (inherited):
 595         #  ocean
 596         #  name
 597         #  islandid
 598         #  yoweb_url
 599         #  flagid
 600         # Public data attributes (additional):
 601         #  flag
 602         def __init__(self, ocean, islename):
 603                 IslandExtendedInfo.__init__(self, ocean, islename)
 604                 self.flag = None
 605                 self._collect_flag()
 606
 607         def _collect_flag(self):
 608                 if self.flagid is None: return
 609                 self.flag = FlagInfo(self.flagid, 1800)
 610
 611         def __str__(self):
 612                 return IslandExtendedInfo.__str__(self) + '; ' + str(self.flag)
 613
 614 class NullProgressReporter():
 615         def start(self): pass
 616         def doing(self, msg): pass
 617         def stop(self): pass
 618
 619 class TypewriterProgressReporter():
 620         def start(self):
 621                 self._l = 0
 622         def doing(self,m):
 623                 self._doing(m + '...')
 624         def _doing(self,m):
 625                 self._write('\r')
 626                 self._write(m)
 627                 less = self._l - len(m)
 628                 if less > 0:
 629                         self._write(' ' * less)
 630                         self._write('\b' * less)
 631                 self._l = len(m)
 632                 sys.stdout.flush()
 633         def stop(self):
 634                 self._doing('')
 635                 self._l = 0
 636         def _write(self,t):
 637                 sys.stdout.write(t)
 638
 639 class OceanInfo():
 640         # Public data attributes:
 641         #   oi.islands[islename] = IslandInfo(...)
 642         #   oi.arches[archname][islename] = IslandInfo(...)
 643         def __init__(self, isleclass=IslandBasicInfo, progressreporter=None):
 644                 if progressreporter is None:
 645                         if opts.debug: progressreporter = NullProgressReporter()
 646                         else: progressreporter = TypewriterProgressReporter()
 647
 648                 self.isleclass = isleclass
 649                 self.ocean = fetcher.ocean.lower().capitalize()
 650
 651                 progressreporter.start()
 652                 progressreporter.doing('fetching ocean info')
 653
 654                 cmdl = ['./yppedia-ocean-scraper']
 655                 if opts.localhtml is not None:
 656                         cmdl += ['--local-html-dir',opts.localhtml]
 657                 cmdl += [self.ocean]
 658                 debug('OceanInfo collect running ' + `cmdl`)
 659                 oscraper = subprocess.Popen(
 660                         cmdl,
 661                         stdout = subprocess.PIPE,
 662                         cwd = yppsc_dir()+'/yarrg',
 663                         shell=False, stderr=None,
 664                         )
 665                 h = oscraper.stdout.readline()
 666                 debug('OceanInfo collect h '+`h`)
 667                 assert(regexp.match('^ocean ', h))
 668                 arch_re = regexp.compile('^ (\S.*)')
 669                 island_re = regexp.compile('^  (\S.*)')
 670
 671                 oscraper.wait()
 672                 assert(oscraper.returncode == 0)
 673
 674                 self.islands = { }
 675                 self.arches = { }
 676                 archname = None
 677
 678                 isles = [ ]
 679                 progressreporter.doing('parsing ocean info')
 680
 681                 for l in oscraper.stdout:
 682                         debug('OceanInfo collect l '+`l`)
 683                         l = l.rstrip('\n')
 684                         m = island_re.match(l)
 685                         if m:
 686                                 assert(archname is not None)
 687                                 islename = m.group(1)
 688                                 isles.append((archname, islename))
 689                                 continue
 690                         m = arch_re.match(l)
 691                         if m:
 692                                 archname = m.group(1)
 693                                 assert(archname not in self.arches)
 694                                 self.arches[archname] = { }
 695                                 continue
 696                         assert(False)
 697
 698                 for i in xrange(0, len(isles)-1):
 699                         (archname, islename) = isles[i]
 700                         progressreporter.doing(
 701                                 'fetching isle info %2d/%d (%s: %s)'
 702                                 % (i, len(isles), archname, islename))
 703                         isle = self.isleclass(self.ocean, islename)
 704                         isle.arch = archname
 705                         self.islands[islename] = isle
 706                         self.arches[archname][islename] = isle
 707
 708                 progressreporter.stop()
 709
 710         def __str__(self):
 711                 return `(self.islands, self.arches)`
 712
 713 #---------- pretty-printer for tables of pirate puzzle standings ----------
 714
 715 class StandingsTable:
 716         def __init__(self, f, use_puzzles=None, col_width=6, gap_every=5):
 717                 if use_puzzles is None:
 718                         if opts.ship_duty:
 719                                 use_puzzles=duty_puzzles
 720                         else:
 721                                 use_puzzles=puzzles
 722                 self._puzzles = use_puzzles
 723                 self.f = f
 724                 self._cw = col_width-1
 725                 self._gap_every = gap_every
 726                 self._linecount = 0
 727                 self._o = f.write
 728
 729         def _nl(self): self._o('\n')
 730
 731         def _pline(self, lhs, puzstrs, extra):
 732                 if (self._linecount > 0
 733                     and self._gap_every is not None
 734                     and not (self._linecount % self._gap_every)):
 735                         self._nl()
 736                 self._o('%-*s' % (max(max_pirate_namelen+1, 15), lhs))
 737                 for v in puzstrs:
 738                         self._o(' %-*.*s' % (self._cw,self._cw, v))
 739                 if extra:
 740                         self._o(' ' + extra)
 741                 self._nl()
 742                 self._linecount += 1
 743
 744         def _puzstr(self, pi, puzzle):
 745                 if not isinstance(puzzle,list): puzzle = [puzzle]
 746                 try: standing = max([pi.standings[p] for p in puzzle])
 747                 except KeyError: return '?'
 748                 if not standing: return ''
 749                 s = ''
 750                 if self._cw > 4:
 751                         c1 = standingvals[standing][0]
 752                         if standing < 3: c1 = c1.lower() # 3 = Master
 753                         s += `standing`
 754                 if self._cw > 5:
 755                         s += ' '
 756                 s += '*' * (standing / 2)
 757                 s += '+' * (standing % 2)
 758                 return s
 759
 760         def headings(self, lhs='', rhs=None):
 761                 def puzn_redact(name):
 762                         if isinstance(name,list):
 763                                 return '/'.join(
 764                                         ["%.*s" % (self._cw/2, puzn_redact(n))
 765                                          for n in name])
 766                         spc = name.find(' ')
 767                         if spc < 0: return name
 768                         return name[0:min(4,spc)] + name[spc+1:]
 769                 self._linecount = -2
 770                 self._pline(lhs, map(puzn_redact, self._puzzles), rhs)
 771                 self._linecount = 0
 772         def literalline(self, line):
 773                 self._o(line)
 774                 self._nl()
 775                 self._linecount = 0
 776         def pirate_dummy(self, name, standingstring, extra=None):
 777                 standings = standingstring * len(self._puzzles)
 778                 self._pline(' '+name, standings, extra)
 779         def pirate(self, pi, extra=None):
 780                 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
 781                 self._pline(' '+pi.name, puzstrs, extra)
 782
 783
 784 #---------- chat log parser ----------
 785
 786 class PirateAboard:
 787         # This is essentially a transparent, dumb, data class.
 788         #  pa.v                 may be None
 789         #  pa.name
 790         #  pa.last_time
 791         #  pa.last_event
 792         #  pa.gunner
 793         #  pa.last_chat_time
 794         #  pa.last_chat_chan
 795         #  pa.pi
 796
 797         # Also used for jobbing applicants:
 798         #               happens when                    expires (to "-")
 799         #   -            disembark, leaves crew          no
 800         #   aboard       evidence of them being aboard   no
 801         #   applied      "has applied for the job"       120s, configurable
 802         #   ashore       "has taken a job"               30min, configurable
 803         #   declined     "declined the job offer"        30s, configurable
 804         #   invited      "has been invited to job"       120s, configurable
 805         #
 806         #  pa.jobber    None, 'ashore', 'applied', 'invited', 'declined'
 807         #  pa.expires   expiry time time
 808
 809         def __init__(pa, pn, v, time, event):
 810                 pa.name = pn
 811                 pa.v = v
 812                 pa.last_time = time
 813                 pa.last_event = event
 814                 pa.last_chat_time = None
 815                 pa.last_chat_chan = None
 816                 pa.gunner = False
 817                 pa.pi = None
 818                 pa.jobber = None
 819                 pa.expires = None
 820
 821         def pirate_info(pa):
 822                 now = time.time()
 823                 if pa.pi:
 824                         age = now - pa.pi_fetched
 825                         guide = random.randint(120,240)
 826                         if age <= guide:
 827                                 return pa.pi
 828                         debug('PirateAboard refresh %d > %d  %s' % (
 829                                 age, guide, pa.name))
 830                         imaginary = [2,4]
 831                 else:
 832                         imaginary = [1]
 833                 wait = fetcher.need_wait(now, imaginary)
 834                 if wait:
 835                         debug('PirateAboard fetcher not ready %d' % wait)
 836                         return pa.pi
 837                 pa.pi = PirateInfo(pa.name, 600)
 838                 pa.pi_fetched = now
 839                 return pa.pi
 840
 841 class ChatLogTracker:
 842         # This is quite complex so we make it opaque.  Use the
 843         # official invokers, accessors etc.
 844
 845         def __init__(self, myself_pi, logfn):
 846                 self._pl = {}   # self._pl['Pirate'] =
 847                 self._vl = {}   #   self._vl['Vessel']['Pirate'] = PirateAboard
 848                                 # self._vl['Vessel']['#lastinfo']
 849                                 # self._vl['Vessel']['#name']
 850                                 # self._v = self._vl[self._vessel]
 851                 self._date = None
 852                 self._myself = myself_pi
 853                 self._lbuf = ''
 854                 self._f = file(logfn)
 855                 flen = os.fstat(self._f.fileno()).st_size
 856                 max_backlog = 500000
 857                 if flen > max_backlog:
 858                         startpos = flen - max_backlog
 859                         self._f.seek(startpos)
 860                         self._f.readline()
 861                 self._progress = [0, flen - self._f.tell()]
 862                 self._disembark_myself()
 863                 self._need_redisplay = False
 864                 self._lastvessel = None
 865
 866         def _disembark_myself(self):
 867                 self._v = None
 868                 self._vessel = None
 869                 self.force_redisplay()
 870
 871         def force_redisplay(self):
 872                 self._need_redisplay = True
 873
 874         def _vessel_updated(self, v, timestamp):
 875                 if v is None: return
 876                 v['#lastinfo'] = timestamp
 877                 self.force_redisplay()
 878
 879         def _onboard_event(self,v,timestamp,pirate,event,jobber=None):
 880                 pa = self._pl.get(pirate, None)
 881                 if pa is not None and pa.v is v:
 882                         pa.last_time = timestamp
 883                         pa.last_event = event
 884                 else:
 885                         if pa is not None and pa.v is not None:
 886                                 del pa.v[pirate]
 887                         pa = PirateAboard(pirate, v, timestamp, event)
 888                         self._pl[pirate] = pa
 889                         if v is not None: v[pirate] = pa
 890                 pa.jobber = jobber
 891
 892                 if jobber is None: timeout = None
 893                 else: timeout = getattr(opts, 'timeout_'+jobber)
 894                 if timeout is None: pa.expires = None
 895                 else: pa.expires = timestamp + timeout
 896                 self._vessel_updated(v, timestamp)
 897                 return pa
 898
 899         def _expire_jobbers(self, now):
 900                 for pa in self._pl.values():
 901                         if pa.expires is None: continue
 902                         if pa.expires >= now: continue
 903                         v = pa.v
 904                         del self._pl[pa.name]
 905                         if v is not None: del v[pa.name]
 906                         self.force_redisplay()
 907
 908         def _trash_vessel(self, v):
 909                 for pn in v:
 910                         if pn.startswith('#'): continue
 911                         del self._pl[pn]
 912                 vn = v['#name']
 913                 del self._vl[vn]
 914                 if v is self._v: self._disembark_myself()
 915                 self.force_redisplay()
 916
 917         def _vessel_stale(self, v, timestamp):
 918                 return timestamp - v['#lastinfo'] > opts.ship_reboard_clearout
 919
 920         def _vessel_check_expire(self, v, timestamp):
 921                 if not self._vessel_stale(v, timestamp):
 922                         return v
 923                 self._debug_line_disposition(timestamp,'',
 924                         'stale-reset ' + v['#name'])
 925                 self._trash_vessel(v)
 926                 return None
 927
 928         def expire_garbage(self, timestamp):
 929                 for v in self._vl.values():
 930                         self._vessel_check_expire(v, timestamp)
 931
 932         def _vessel_lookup(self, vn, timestamp, dml=[], create=False):
 933                 v = self._vl.get(vn, None)
 934                 if v is not None:
 935                         v = self._vessel_check_expire(v, timestamp)
 936                 if v is not None:
 937                         dml.append('found')
 938                         return v
 939                 if not create:
 940                         dml.append('no')
 941                 dml.append('new')
 942                 self._vl[vn] = v = { '#name': vn }
 943                 self._vessel_updated(v, timestamp)
 944                 return v
 945
 946         def _find_matching_vessel(self, pattern, timestamp, cmdr,
 947                                         dml=[], create=False):
 948                 # use when a commander pirate `cmdr' specified a vessel
 949                 #  by name `pattern' (either may be None)
 950                 # if create is true, will create the vessel
 951                 #  record if an exact name is specified
 952
 953                 if (pattern is not None and
 954                     not '*' in pattern
 955                     and len(pattern.split(' ')) == 2):
 956                         vn = pattern.title()
 957                         dml.append('exact')
 958                         return self._vessel_lookup(
 959                                 vn, timestamp, dml=dml, create=create)
 960
 961                 if pattern is None:
 962                         pattern_check = lambda vn: True
 963                 else:
 964                         re = '(?:.* )?%s$' % pattern.lower().replace('*','.+')
 965                         pattern_check = regexp.compile(re, regexp.I).match
 966
 967                 tries = []
 968
 969                 cmdr_pa = self._pl.get(cmdr, None)
 970                 if cmdr_pa: tries.append((cmdr_pa.v, 'cmdr'))
 971
 972                 tries.append((self._v, 'here'))
 973                 tried_vns = []
 974
 975                 for (v, dm) in tries:
 976                         if v is None: dml.append(dm+'?'); continue
 977
 978                         vn = v['#name']
 979                         if not pattern_check(vn):
 980                                 tried_vns.append(vn)
 981                                 dml.append(dm+'#')
 982                                 continue
 983
 984                         dml.append(dm+'!')
 985                         return v
 986
 987                 if pattern is not None and '*' in pattern:
 988                         search = [
 989                                 (vn,v)
 990                                 for (vn,v) in self._vl.iteritems()
 991                                 if not self._vessel_stale(v, timestamp)
 992                                 if pattern_check(vn)
 993                                 ]
 994                         #debug('CLT-RE /%s/ wanted (%s) searched (%s)' % (
 995                         #       re,
 996                         #       '/'.join(tried_vns),
 997                         #       '/'.join([vn for (vn,v) in search])))
 998
 999                         if len(search)==1:
1000                                 dml.append('one')
1001                                 return search[0][1]
1002                         elif search:
1003                                 dml.append('many')
1004                         else:
1005                                 dml.append('none')
1006
1007         def _debug_line_disposition(self,timestamp,l,m):
1008                 debug('CLT %13s %-40s %s' % (timestamp,m,l))
1009
1010         def _rm_crew_l(self,re,l):
1011                 m = regexp.match(re,l)
1012                 if m and m.group(2) == self._myself.crew[1]:
1013                         return m.group(1)
1014                 else:
1015                         return None
1016
1017         def local_command(self, metacmd):
1018                 # returns None if all went well, or problem message
1019                 return self._command(self._myself.name, metacmd,
1020                         "local", time.time(),
1021                         (lambda m: debug('CMD %s' % metacmd)))
1022
1023         def _command(self, cmdr, metacmd, chan, timestamp, d):
1024                 # returns None if all went well, or problem message
1025                 metacmd = regexp.sub('\\s+', ' ', metacmd).strip()
1026                 m2 = regexp.match(
1027                     '/([adj]) (?:([A-Za-z* ]+)\\s*:)?([A-Za-z ]+)$',
1028                     metacmd)
1029                 if not m2: return "unknown syntax or command"
1030
1031                 (cmd, pattern, targets) = m2.groups()
1032                 dml = ['cmd', chan, cmd]
1033
1034                 if cmd == 'a': each = self._onboard_event
1035                 elif cmd == 'd': each = disembark
1036                 else: each = lambda *l: self._onboard_event(*l,
1037                                 **{'jobber':'applied'})
1038
1039                 if cmdr == self._myself.name:
1040                         dml.append('self')
1041                         how = 'cmd: %s' % cmd
1042                 else:
1043                         dml.append('other')
1044                         how = 'cmd: %s %s' % (cmd,cmdr)
1045
1046                 if cmd == 'j':
1047                         if pattern is not None:
1048                                 return "/j command does not take a vessel"
1049                         v = None
1050                 else:
1051                         v = self._find_matching_vessel(
1052                                 pattern, timestamp, cmdr,
1053                                 dml, create=True)
1054
1055                 if cmd == 'j' or v is not None:
1056                         targets = targets.strip().split(' ')
1057                         dml.append(`len(targets)`)
1058                         for target in targets:
1059                                 each(v, timestamp, target.title(), how)
1060                         self._vessel_updated(v, timestamp)
1061
1062                 dm = ' '.join(dml)
1063                 return d(dm)
1064
1065                 return None
1066
1067         def chatline(self,l):
1068                 rm = lambda re: regexp.match(re,l)
1069                 d = lambda m: self._debug_line_disposition(timestamp,l,m)
1070                 rm_crew = lambda re: self._rm_crew_l(re,l)
1071                 timestamp = None
1072
1073                 m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$')
1074                 if m:
1075                         self._date = [int(x) for x in m.groups()]
1076                         self._previous_timestamp = None
1077                         return d('date '+`self._date`)
1078
1079                 if self._date is None:
1080                         return d('date unset')
1081
1082                 m = rm('\\[(\d\d):(\d\d):(\d\d)\\] ')
1083                 if not m:
1084                         return d('no timestamp')
1085
1086                 while True:
1087                         time_tuple = (self._date +
1088                                       [int(x) for x in m.groups()] +
1089                                       [-1,-1,-1])
1090                         timestamp = time.mktime(time_tuple)
1091                         if timestamp >= self._previous_timestamp: break
1092                         self._date[2] += 1
1093                         self._debug_line_disposition(timestamp,'',
1094                                 'new date '+`self._date`)
1095
1096                 self._previous_timestamp = timestamp
1097
1098                 l = l[l.find(' ')+1:]
1099
1100                 def ob_x(pirate,event):
1101                         return self._onboard_event(
1102                                         self._v, timestamp, pirate, event)
1103                 def ob1(did): ob_x(m.group(1), did); return d(did)
1104                 def oba(did): return ob1('%s %s' % (did, m.group(2)))
1105
1106                 def jb(pirate,jobber):
1107                         return self._onboard_event(
1108                                 None, timestamp, pirate,
1109                                 ("jobber %s" % jobber),
1110                                 jobber=jobber
1111                                 )
1112
1113                 def disembark(v, timestamp, pirate, event):
1114                         self._onboard_event(
1115                                         v, timestamp, pirate, 'leaving '+event)
1116                         del v[pirate]
1117                         del self._pl[pirate]
1118
1119                 def disembark_me(why):
1120                         self._disembark_myself()
1121                         return d('disembark-me '+why)
1122
1123                 m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
1124                 if m:
1125                         dm = ['boarding']
1126                         pn = self._myself.name
1127                         vn = m.group(1)
1128                         v = self._vessel_lookup(vn, timestamp, dm, create=True)
1129                         self._lastvessel = self._vessel = vn
1130                         self._v = v
1131                         ob_x(pn, 'we boarded')
1132                         self.expire_garbage(timestamp)
1133                         return d(' '.join(dm))
1134
1135                 if self._v is None:
1136                         return d('no vessel')
1137
1138                 m = rm('(\\w+) has come aboard\\.$')
1139                 if m: return ob1('boarded');
1140
1141                 m = rm('You have ordered (\\w+) to do some (\\S.*\\S)\\.$')
1142                 if m:
1143                         (who,what) = m.groups()
1144                         pa = ob_x(who,'ord '+what)
1145                         if what == 'Gunning':
1146                                 pa.gunner = True
1147                         return d('duty order')
1148
1149                 m = rm('(\\w+) abandoned a (\\S.*\\S) station\\.$')
1150                 if m: oba('stopped'); return d("end")
1151
1152                 def chat_core(speaker, chan):
1153                         try: pa = self._pl[speaker]
1154                         except KeyError: return 'mystery'
1155                         if pa.v is not None and pa.v is not self._v:
1156                                 return 'elsewhere'
1157                         pa.last_chat_time = timestamp
1158                         pa.last_chat_chan = chan
1159                         self.force_redisplay()
1160                         return 'here'
1161
1162                 def chat(chan):
1163                         speaker = m.group(1)
1164                         dm = chat_core(speaker, chan)
1165                         return d('chat %s %s' % (chan, dm))
1166
1167                 def chat_metacmd(chan):
1168                         (cmdr, metacmd) = m.groups()
1169                         whynot = self._command(
1170                                 cmdr, metacmd, chan, timestamp, d)
1171                         if whynot is not None:
1172                                 return chat(chan)
1173                         else:
1174                                 chat_core(cmdr, 'cmd '+chan)
1175
1176                 m = rm('(\\w+) (?:issued an order|ordered everyone) "')
1177                 if m: return ob1('general order');
1178
1179                 m = rm('(\\w+) says, "')
1180                 if m: return chat('public')
1181
1182                 m = rm('(\\w+) tells ye, "')
1183                 if m: return chat('private')
1184
1185                 m = rm('Ye told (\\w+), "(.*)"$')
1186                 if m: return chat_metacmd('private')
1187
1188                 m = rm('(\\w+) flag officer chats, "')
1189                 if m: return chat('flag officer')
1190
1191                 m = rm('(\\w+) officer chats, "(.*)"$')
1192                 if m: return chat_metacmd('officer')
1193
1194                 m = rm('Ye accepted the offer to job with ')
1195                 if m: return disembark_me('jobbing')
1196
1197                 m = rm('Ye hop on the ferry and are whisked away ')
1198                 if m: return disembark_me('ferry')
1199
1200                 m = rm('Whisking away to yer home on the magical winds')
1201                 if m: return disembark_me('home')
1202
1203                 m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
1204                 if m:
1205                         pl = m.group(1).split(', ')
1206                         if not self._myself.name in pl:
1207                                 return d('lost melee')
1208                         for pn in pl:
1209                                 if ' ' in pn: continue
1210                                 ob_x(pn,'won melee')
1211                         return d('won melee')
1212
1213                 m = rm('(\\w+) is eliminated\\!')
1214                 if m: return ob1('eliminated in fray');
1215
1216                 m = rm('(\\w+) has driven \w+ from the ship\\!')
1217                 if m: return ob1('boarder repelled');
1218
1219                 m = rm('\w+ has bested (\\w+), and turns'+
1220                         ' to the rest of the ship\\.')
1221                 if m: return ob1('boarder unrepelled');
1222
1223                 pirate = rm_crew("(\\w+) has taken a job with '(.*)'\\.")
1224                 if pirate: return jb(pirate, 'ashore')
1225
1226                 pirate = rm_crew("(\\w+) has left '(.*)'\\.")
1227                 if pirate:
1228                         disembark(self._v, timestamp, pirate, 'left crew')
1229                         return d('left crew')
1230
1231                 m = rm('(\w+) has applied for the posted job\.')
1232                 if m: return jb(m.group(1), 'applied')
1233
1234                 pirate= rm_crew("(\\w+) has been invited to job for '(.*)'\\.")
1235                 if pirate: return jb(pirate, 'invited')
1236
1237                 pirate = rm_crew("(\\w+) declined the job offer for '(.*)'\\.")
1238                 if pirate: return jb(pirate, 'declined')
1239
1240                 m = rm('(\\w+) has left the vessel\.')
1241                 if m:
1242                         pirate = m.group(1)
1243                         disembark(self._v, timestamp, pirate, 'disembarked')
1244                         return d('disembarked')
1245
1246                 return d('not-matched')
1247
1248         def _str_pa(self, pn, pa):
1249                 assert self._pl[pn] == pa
1250                 s = ' '*20 + "%s %-*s %13s %-30s %13s %-20s %13s" % (
1251                         (' ','G')[pa.gunner],
1252                         max_pirate_namelen, pn,
1253                         pa.last_time, pa.last_event,
1254                         pa.last_chat_time, pa.last_chat_chan,
1255                         pa.jobber)
1256                 if pa.expires is not None:
1257                         s += " %-5d" % (pa.expires - pa.last_time)
1258                 s += "\n"
1259                 return s
1260
1261         def _str_vessel(self, vn, v):
1262                 s = ' vessel %s\n' % vn
1263                 s += ' '*20 + "%-*s   %13s\n" % (
1264                                 max_pirate_namelen, '#lastinfo',
1265                                 v['#lastinfo'])
1266                 assert v['#name'] == vn
1267                 for pn in sorted(v.keys()):
1268                         if pn.startswith('#'): continue
1269                         pa = v[pn]
1270                         assert pa.v == v
1271                         s += self._str_pa(pn,pa)
1272                 return s
1273
1274         def __str__(self):
1275                 s = '''<ChatLogTracker
1276  myself %s
1277  vessel %s
1278 '''                     % (self._myself.name, self._vessel)
1279                 assert ((self._v is None and self._vessel is None) or
1280                         (self._v is self._vl[self._vessel]))
1281                 if self._vessel is not None:
1282                         s += self._str_vessel(self._vessel, self._v)
1283                 for vn in sorted(self._vl.keys()):
1284                         if vn == self._vessel: continue
1285                         s += self._str_vessel(vn, self._vl[vn])
1286                 s += " elsewhere\n"
1287                 for p in self._pl:
1288                         pa = self._pl[p]
1289                         if pa.v is not None:
1290                                 assert pa.v[p] is pa
1291                                 assert pa.v in self._vl.values()
1292                         else:
1293                                 s += self._str_pa(pa.name, pa)
1294                 s += '>\n'
1295                 return s
1296
1297         def catchup(self, progress=None):
1298                 while True:
1299                         more = self._f.readline()
1300                         if not more: break
1301
1302                         self._progress[0] += len(more)
1303                         if progress: progress.progress(*self._progress)
1304
1305                         self._lbuf += more
1306                         if self._lbuf.endswith('\n'):
1307                                 self.chatline(self._lbuf.rstrip())
1308                                 self._lbuf = ''
1309                                 if opts.debug >= 2:
1310                                         debug(self.__str__())
1311                 self._expire_jobbers(time.time())
1312
1313                 if progress: progress.caughtup()
1314
1315         def changed(self):
1316                 rv = self._need_redisplay
1317                 self._need_redisplay = False
1318                 return rv
1319         def myname(self):
1320                 # returns our pirate name
1321                 return self._myself.name
1322         def vesselname(self):
1323                 # returns the vessel name we're aboard or None
1324                 return self._vessel
1325         def lastvesselname(self):
1326                 # returns the last vessel name we were aboard or None
1327                 return self._lastvessel
1328         def aboard(self, vesselname=True):
1329                 # returns a list of PirateAboard the vessel
1330                 #  sorted by pirate name
1331                 #  you can pass this None and you'll get []
1332                 #  or True for the current vessel (which is the default)
1333                 #  the returned value is a fresh list of persistent
1334                 #  PirateAboard objects
1335                 if vesselname is True: v = self._v
1336                 else: v = self._vl.get(vesselname.title())
1337                 if v is None: return []
1338                 return [ v[pn]
1339                          for pn in sorted(v.keys())
1340                          if not pn.startswith('#') ]
1341         def jobbers(self):
1342                 # returns a the jobbers' PirateAboards,
1343                 # sorted by jobber class and reverse of expiry time
1344                 l = [ pa
1345                       for pa in self._pl.values()
1346                       if pa.jobber is not None
1347                     ]
1348                 def compar_key(pa):
1349                         return (pa.jobber, -pa.expires)
1350                 l.sort(key = compar_key)
1351                 return l
1352
1353 #---------- implementations of actual operation modes ----------
1354
1355 def do_pirate(pirates, bu):
1356         print '{'
1357         for pirate in pirates:
1358                 info = PirateInfo(pirate)
1359                 print '%s: %s,' % (`pirate`, info)
1360         print '}'
1361
1362 def prep_crew_of(args, bu, max_age=300):
1363         if len(args) != 1: bu('crew-of takes one pirate name')
1364         pi = PirateInfo(args[0], max_age)
1365         if pi.crew is None: return None
1366         return CrewInfo(pi.crew[0], max_age)
1367
1368 def do_crew_of(args, bu):
1369         ci = prep_crew_of(args, bu)
1370         print ci
1371
1372 def do_flag_of(args, bu):
1373         if len(args) != 1: bu('flag-of takes one pirate name')
1374         max_age = 300
1375         pi = PirateInfo(args[0], max_age)
1376         if pi.flag is None: fi = None
1377         else: fi = FlagInfo(pi.flag[0], max_age)
1378         print fi
1379
1380 def do_standings_crew_of(args, bu):
1381         ci = prep_crew_of(args, bu, 60)
1382         tab = StandingsTable(sys.stdout)
1383         tab.headings()
1384         for (rank, members) in ci.crew:
1385                 if not members: continue
1386                 tab.literalline('')
1387                 tab.literalline('%s:' % rank)
1388                 for p in members:
1389                         pi = PirateInfo(p, random.randint(900,1800))
1390                         tab.pirate(pi)
1391
1392 def do_ocean(args, bu):
1393         if (len(args)): bu('ocean takes no further arguments')
1394         fetcher.default_ocean()
1395         oi = OceanInfo(IslandFlagInfo)
1396         print oi
1397         for islename in sorted(oi.islands.keys()):
1398                 isle = oi.islands[islename]
1399                 print isle
1400
1401 def do_embargoes(args, bu):
1402         if (len(args)): bu('ocean takes no further arguments')
1403         fetcher.default_ocean()
1404         oi = OceanInfo(IslandFlagInfo)
1405         wr = sys.stdout.write
1406         print ('EMBARGOES:  Island    | Owning flag'+
1407                 '                    | Embargoed flags')
1408
1409         def getflname(isle):
1410                 if isle.islandid is None: return 'uncolonisable'
1411                 if isle.flag is None: return 'uncolonised'
1412                 return isle.flag.name
1413
1414         for archname in sorted(oi.arches.keys()):
1415                 print 'ARCHIPELAGO: ',archname
1416                 for islename in sorted(oi.arches[archname].keys()):
1417                         isle = oi.islands[islename]
1418                         wr(' %-20s | ' % isle.name)
1419                         flname = getflname(isle)
1420                         wr('%-30s | ' % flname)
1421                         flag = isle.flag
1422                         if flag is None: print ''; continue
1423                         delim = ''
1424                         for rel in flag.relations:
1425                                 (oname, oid, dummy, thisdeclaring,
1426                                         odeclaringmin,odeclaringmax) = rel
1427                                 if thisdeclaring >= 0: continue
1428                                 wr(delim)
1429                                 wr(oname)
1430                                 delim = '; '
1431                         print ''
1432
1433 #----- modes which use the chat log parser are quite complex -----
1434
1435 class ProgressPrintPercentage:
1436         def __init__(self, f=sys.stdout):
1437                 self._f = f
1438         def progress_string(self,done,total):
1439                 return "scan chat logs %3d%%\r" % ((done*100) / total)
1440         def progress(self,*a):
1441                 self._f.write(self.progress_string(*a))
1442                 self._f.flush()
1443         def show_init(self, pirate, ocean):
1444                 print >>self._f, 'Starting up, %s on the %s ocean' % (
1445                         pirate, ocean)
1446         def caughtup(self):
1447                 self._f.write('                   \r')
1448                 self._f.flush()
1449
1450 def prep_chat_log(args, bu,
1451                 progress=ProgressPrintPercentage(),
1452                 max_myself_age=3600):
1453         if len(args) != 1: bu('this action takes only chat log filename')
1454         logfn = args[0]
1455         logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_'
1456         match = regexp.match(logfn_re, logfn)
1457         if not match: bu('chat log filename is not in expected format')
1458         (pirate, ocean) = match.groups()
1459         fetcher.default_ocean(ocean)
1460
1461         progress.show_init(pirate, fetcher.ocean)
1462         myself = PirateInfo(pirate,max_myself_age)
1463         track = ChatLogTracker(myself, logfn)
1464
1465         opts.debug -= 2
1466         track.catchup(progress)
1467         opts.debug += 2
1468
1469         track.force_redisplay()
1470
1471         return (myself, track)
1472
1473 def do_track_chat_log(args, bu):
1474         (myself, track) = prep_chat_log(args, bu)
1475         while True:
1476                 track.catchup()
1477                 if track.changed():
1478                         print track
1479                 sleep(0.5 + 0.5 * random.random())
1480
1481 #----- ship management aid -----
1482
1483 class Display_dumb(ProgressPrintPercentage):
1484         def __init__(self):
1485                 ProgressPrintPercentage.__init__(self)
1486         def show(self, s):
1487                 print '\n\n', s;
1488         def realstart(self):
1489                 pass
1490
1491 class Display_overwrite(ProgressPrintPercentage):
1492         def __init__(self):
1493                 ProgressPrintPercentage.__init__(self)
1494
1495                 null = file('/dev/null','w')
1496                 curses.setupterm(fd=null.fileno())
1497
1498                 self._clear = curses.tigetstr('clear')
1499                 if not self._clear:
1500                         self._debug('missing clear!')
1501                         self.show = Display_dumb.show
1502                         return
1503
1504                 self._t = {'el':'', 'ed':''}
1505                 if not self._init_sophisticated():
1506                         for k in self._t.keys(): self._t[k] = ''
1507                         self._t['ho'] = self._clear
1508
1509         def _debug(self,m): debug('display overwrite: '+m)
1510
1511         def _init_sophisticated(self):
1512                 for k in self._t.keys():
1513                         s = curses.tigetstr(k)
1514                         self._t[k] = s
1515                 self._t['ho'] = curses.tigetstr('ho')
1516                 if not self._t['ho']:
1517                         cup = curses.tigetstr('cup')
1518                         self._t['ho'] = curses.tparm(cup,0,0)
1519                 missing = [k for k in self._t.keys() if not self._t[k]]
1520                 if missing:
1521                         self.debug('missing '+(' '.join(missing)))
1522                         return 0
1523                 return 1
1524
1525         def show(self, s):
1526                 w = sys.stdout.write
1527                 def wti(k): w(self._t[k])
1528
1529                 wti('ho')
1530                 nl = ''
1531                 for l in s.rstrip().split('\n'):
1532                         w(nl)
1533                         w(l)
1534                         wti('el')
1535                         nl = '\r\n'
1536                 wti('ed')
1537                 w(' ')
1538                 sys.stdout.flush()
1539
1540         def realstart(self):
1541                 sys.stdout.write(self._clear)
1542                 sys.stdout.flush()
1543
1544
1545 def do_ship_aid(args, bu):
1546         if opts.ship_duty is None: opts.ship_duty = True
1547
1548         displayer = globals()['Display_'+opts.display]()
1549
1550         (myself, track) = prep_chat_log(args, bu, progress=displayer)
1551
1552         displayer.realstart()
1553
1554         if os.isatty(0): kr_create = KeystrokeReader
1555         else: kr_create = DummyKeystrokeReader
1556
1557         try:
1558                 kreader = kr_create(0, 10)
1559                 ship_aid_core(myself, track, displayer, kreader)
1560         finally:
1561                 kreader.stop()
1562                 print '\n'
1563
1564 class KeyBasedSorter:
1565         def compar_key_pa(self, pa):
1566                 pi = pa.pirate_info()
1567                 if pi is None: return None
1568                 return self.compar_key(pi)
1569         def lsort_pa(self, l):
1570                 l.sort(key = self.compar_key_pa)
1571
1572 class NameSorter(KeyBasedSorter):
1573         def compar_key(self, pi): return pi.name
1574         def desc(self): return 'name'
1575
1576 class SkillSorter(NameSorter):
1577         def __init__(self, relevant):
1578                 self._want = frozenset(relevant.split('/'))
1579                 self._avoid = set()
1580                 for p in core_duty_puzzles:
1581                         if isinstance(p,basestring): self._avoid.add(p)
1582                         else: self._avoid |= set(p)
1583                 self._avoid -= self._want
1584                 self._desc = '%s' % relevant
1585
1586         def desc(self): return self._desc
1587
1588         def compar_key(self, pi):
1589                 best_want = max([
1590                         pi.standings.get(puz,-1)
1591                         for puz in self._want
1592                         ])
1593                 best_avoid = [
1594                         -pi.standings.get(puz,standing_limit)
1595                         for puz in self._avoid
1596                         ]
1597                 best_avoid.sort()
1598                 def negate(x): return -x
1599                 debug('compar_key %s bw=%s ba=%s' % (pi.name, `best_want`,
1600                         `best_avoid`))
1601                 return (-best_want, map(negate, best_avoid), pi.name)
1602
1603 def ship_aid_core(myself, track, displayer, kreader):
1604
1605         def find_vessel():
1606                 vn = track.vesselname()
1607                 if vn: return (vn, " on board the %s" % vn)
1608                 vn = track.lastvesselname()
1609                 if vn: return (vn, " ashore from the %s" % vn)
1610                 return (None, " not on a vessel")
1611
1612         def timeevent(t,e):
1613                 if t is None: return ' ' * 22
1614                 return " %-4s %-16s" % (format_time_interval(now - t),e)
1615
1616         displayer.show(track.myname() + find_vessel()[1] + '...')
1617
1618         rotate_nya = '/-\\'
1619
1620         sort = NameSorter()
1621         clicmd = None
1622         clierr = None
1623         cliexec = None
1624
1625         while True:
1626                 track.catchup()
1627                 now = time.time()
1628
1629                 (vn, vs) = find_vessel()
1630
1631                 s = ''
1632                 if cliexec is not None:
1633                         s += '...'
1634                 elif clierr is not None:
1635                         s += 'Error: '+clierr
1636                 elif clicmd is not None:
1637                         s += '/' + clicmd
1638                 else:
1639                         s = track.myname() + vs
1640                         s += " at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
1641                         s += kreader.info()
1642                 s += '\n'
1643
1644                 tbl_s = StringIO()
1645                 tbl = StandingsTable(tbl_s)
1646
1647                 aboard = track.aboard(vn)
1648                 sort.lsort_pa(aboard)
1649
1650                 jobbers = track.jobbers()
1651
1652                 if track.vesselname(): howmany = 'aboard: %2d' % len(aboard)
1653                 else: howmany = ''
1654
1655                 tbl.headings(howmany, '  sorted by '+sort.desc())
1656
1657                 last_jobber = None
1658
1659                 for pa in aboard + jobbers:
1660                         if pa.jobber != last_jobber:
1661                                 last_jobber = pa.jobber
1662                                 tbl.literalline('')
1663                                 tbl.literalline('jobbers '+last_jobber)
1664
1665                         pi = pa.pirate_info()
1666
1667                         xs = ''
1668                         if pa.gunner: xs += 'G '
1669                         else: xs += '  '
1670                         xs += timeevent(pa.last_time, pa.last_event)
1671                         xs += timeevent(pa.last_chat_time, pa.last_chat_chan)
1672
1673                         if pi is None:
1674                                 tbl.pirate_dummy(pa.name, rotate_nya[0], xs)
1675                         else:
1676                                 tbl.pirate(pi, xs)
1677
1678                 s += tbl_s.getvalue()
1679                 displayer.show(s)
1680                 tbl_s.close()
1681
1682                 if cliexec is not None:
1683                         clierr = track.local_command("/"+cliexec.strip())
1684                         cliexec = None
1685                         continue
1686
1687                 k = kreader.getch()
1688                 if k is None:
1689                         rotate_nya = rotate_nya[1:3] + rotate_nya[0]
1690                         continue
1691
1692                 if clierr is not None:
1693                         clierr = None
1694                         continue
1695
1696                 if clicmd is not None:
1697                         if k == '\r' or k == '\n':
1698                                 cliexec = clicmd
1699                                 clicmd = clicmdbase
1700                         elif k == '\e' and clicmd != "":
1701                                 clicmd = clicmdbase
1702                         elif k == '\33':
1703                                 clicmd = None
1704                         elif k == '\b' or k == '\177':
1705                                 clicmd = clicmd[ 0 : len(clicmd)-1 ]
1706                         else:
1707                                 clicmd += k
1708                         continue
1709
1710                 if k == 'q': break
1711                 elif k == 'g': sort = SkillSorter('Gunning')
1712                 elif k == 'c': sort = SkillSorter('Carpentry')
1713                 elif k == 's': sort = SkillSorter('Sailing/Rigging')
1714                 elif k == 'b': sort = SkillSorter('Bilging')
1715                 elif k == 'n': sort = SkillSorter('Navigating')
1716                 elif k == 'd': sort = SkillSorter('Battle Navigation')
1717                 elif k == 't': sort = SkillSorter('Treasure Haul')
1718                 elif k == 'a': sort = NameSorter()
1719                 elif k == '/': clicmdbase = ""; clicmd = clicmdbase
1720                 elif k == '+': clicmdbase = "a "; clicmd = clicmdbase
1721                 else: pass # unknown key command
1722
1723 #---------- individual keystroke input ----------
1724
1725 class DummyKeystrokeReader:
1726         def __init__(self,fd,timeout_dummy): pass
1727         def stop(self): pass
1728         def getch(self): sleep(1); return None
1729         def info(self): return ' [noninteractive]'
1730
1731 class KeystrokeReader(DummyKeystrokeReader):
1732         def __init__(self, fd, timeout_decisec=0):
1733                 self._fd = fd
1734                 self._saved = termios.tcgetattr(fd)
1735                 a = termios.tcgetattr(fd)
1736                 a[3] &= ~(termios.ECHO | termios.ECHONL |
1737                           termios.ICANON | termios.IEXTEN)
1738                 a[6][termios.VMIN] = 0
1739                 a[6][termios.VTIME] = timeout_decisec
1740                 termios.tcsetattr(fd, termios.TCSANOW, a)
1741         def stop(self):
1742                 termios.tcsetattr(self._fd, termios.TCSANOW, self._saved)
1743         def getch(self):
1744                 debug_flush()
1745                 byte = os.read(self._fd, 1)
1746                 if not len(byte): return None
1747                 return byte
1748         def info(self):
1749                 return ''
1750
1751 #---------- main program ----------
1752
1753 def main():
1754         global opts, fetcher, yppedia
1755
1756         pa = OptionParser(
1757 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
1758 actions:
1759  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
1760  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
1761  yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
1762  yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG
1763  yoweb-scrape [options] ship-aid CHAT-LOG  (must be .../PIRATE_OCEAN_chat-log*)
1764
1765 display modes (for --display) apply to ship-aid:
1766  --display=dumb       just print new information, scrolling the screen
1767  --display=overwrite  use cursor motion, selective clear, etc. to redraw at top''')
1768         ao = pa.add_option
1769         ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
1770                 help='select ocean OCEAN')
1771         ao('--cache-dir', dest='cache_dir', metavar='DIR',
1772                 default='~/.yoweb-scrape-cache',
1773                 help='cache yoweb pages in DIR')
1774         ao('-D','--debug', action='count', dest='debug', default=0,
1775                 help='enable debugging output')
1776         ao('--debug-fd', type='int', dest='debug_fd',
1777                 help='write any debugging output to specified fd')
1778         ao('-q','--quiet', action='store_true', dest='quiet',
1779                 help='suppress warning output')
1780         ao('--display', action='store', dest='display',
1781                 type='choice', choices=['dumb','overwrite'],
1782                 help='how to display ship aid')
1783         ao('--local-ypp-dir', action='store', dest='localhtml',
1784                 help='get yppedia pages from local directory LOCALHTML'+
1785                         ' instead of via HTTP')
1786
1787         ao_jt = lambda wh, t: ao(
1788                 '--timeout-sa-'+wh, action='store', dest='timeout_'+wh,
1789                 default=t, help=('set timeout for expiring %s jobbers' % wh))
1790         ao_jt('applied',      120)
1791         ao_jt('invited',      120)
1792         ao_jt('declined',      30)
1793         ao_jt('ashore',      1800)
1794
1795         ao('--ship-duty', action='store_true', dest='ship_duty',
1796                 help='show ship duty station puzzles')
1797         ao('--all-puzzles', action='store_false', dest='ship_duty',
1798                 help='show all puzzles, not just ship duty stations')
1799
1800         ao('--min-cache-reuse', type='int', dest='min_max_age',
1801                 metavar='SECONDS', default=60,
1802                 help='always reuse cache yoweb data if no older than this')
1803
1804         (opts,args) = pa.parse_args()
1805         random.seed()
1806
1807         if len(args) < 1:
1808                 print >>sys.stderr, copyright_info
1809                 pa.error('need a mode argument')
1810
1811         if opts.debug_fd is not None:
1812                 opts.debug_file = os.fdopen(opts.debug_fd, 'w')
1813         else:
1814                 opts.debug_file = sys.stdout
1815
1816         mode = args[0]
1817         mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
1818         try: mode_fn = globals()[mode_fn_name]
1819         except KeyError: pa.error('unknown mode "%s"' % mode)
1820
1821         # fixed parameters
1822         opts.expire_age = max(3600, opts.min_max_age)
1823
1824         opts.ship_reboard_clearout = 3600
1825
1826         if opts.cache_dir.startswith('~/'):
1827                 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
1828
1829         if opts.display is None:
1830                 if ((opts.debug > 0 and opts.debug_fd is None)
1831                     or not os.isatty(sys.stdout.fileno())):
1832                         opts.display = 'dumb'
1833                 else:
1834                         opts.display = 'overwrite'
1835
1836         fetcher = Yoweb(opts.ocean, opts.cache_dir)
1837         yppedia = Yppedia(opts.cache_dir)
1838
1839         mode_fn(args[1:], pa.error)
1840
1841 main()