yoweb-scrape

   1 #!/usr/bin/python
   2 # This is part of ypp-sc-tools, a set of third-party tools for assisting
   3 # players of Yohoho Puzzle Pirates.
   4 #
   5 # Copyright (C) 2009 Ian Jackson <ijackson@chiark.greenend.org.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 #
  20 # Yohoho and Puzzle Pirates are probably trademarks of Three Rings and
  21 # are used without permission.  This program is not endorsed or
  22 # sponsored by Three Rings.
  23
  24 copyright_info = '''
  25 yoweb-scrape is part of ypp-sc-tools  Copyright (C) 2009 Ian Jackson
  26 This program comes with ABSOLUTELY NO WARRANTY; this is free software,
  27 and you are welcome to redistribute it under certain conditions.
  28 For details, read the top of the yoweb-scrape file.
  29 '''
  30
  31 #---------- setup ----------
  32
  33 import signal
  34 signal.signal(signal.SIGINT, signal.SIG_DFL)
  35
  36 import os
  37 import time
  38 import urllib
  39 import urllib2
  40 import errno
  41 import sys
  42 import re as regexp
  43 import random
  44 import curses
  45 import termios
  46 import random
  47 import subprocess
  48 from optparse import OptionParser
  49 from StringIO import StringIO
  50
  51 from BeautifulSoup import BeautifulSoup
  52
  53 opts = None
  54
  55 #---------- YPP parameters and arrays ----------
  56
  57 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
  58         '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
  59         '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
  60         '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
  61
  62 core_duty_puzzles = [
  63                 'Gunning',
  64                 ['Sailing','Rigging'],
  65                 'Bilging',
  66                 'Carpentry',
  67                 ]
  68
  69 duty_puzzles = ([ 'Navigating', 'Battle Navigation' ] +
  70                 core_duty_puzzles +
  71                 [ 'Treasure Haul' ])
  72
  73 standingvals = ('Able/Proficient/Distinguished/Respected/Master'+
  74                 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
  75 standing_limit = len(standingvals)
  76
  77 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
  78
  79 max_pirate_namelen = 12
  80
  81
  82 #---------- general utilities ----------
  83
  84 def debug(m):
  85         if opts.debug > 0:
  86                 print >>opts.debug_file, m
  87
  88 def debug_flush():
  89         if opts.debug > 0:
  90                 opts.debug_file.flush()
  91
  92 def sleep(seconds):
  93         debug_flush()
  94         time.sleep(seconds)
  95
  96 def format_time_interval(ti):
  97         if ti < 120: return '%d:%02d' % (ti / 60, ti % 60)
  98         if ti < 7200: return '%2dm' % (ti / 60)
  99         if ti < 86400: return '%dh' % (ti / 3600)
 100         return '%dd' % (ti / 86400)
 101
 102 def yppsc_dir():
 103         lib = os.getenv("YPPSC_YARRG_SRCBASE")
 104         if lib is not None: return lib
 105         lib = sys.argv[0]
 106         lib = regexp.sub('/[^/]+$', '', lib)
 107         os.environ["YPPSC_YARRG_SRCBASE"] = lib
 108         return lib
 109
 110 #---------- caching and rate-limiting data fetcher ----------
 111
 112 class Fetcher:
 113         def __init__(self, cachedir):
 114                 debug('Fetcher init %s' % cachedir)
 115                 self.cachedir = cachedir
 116                 try: os.mkdir(cachedir)
 117                 except (OSError,IOError), oe:
 118                         if oe.errno != errno.EEXIST: raise
 119                 self._cache_scan(time.time())
 120
 121         def _cache_scan(self, now):
 122                 # returns list of ages, unsorted
 123                 ages = []
 124                 debug('Fetcher   scan_cache')
 125                 for leaf in os.listdir(self.cachedir):
 126                         if not leaf.startswith('#'): continue
 127                         path = self.cachedir + '/' + leaf
 128                         try: s = os.stat(path)
 129                         except (OSError,IOError), oe:
 130                                 if oe.errno != errno.ENOENT: raise
 131                                 continue
 132                         age = now - s.st_mtime
 133                         if age > opts.expire_age:
 134                                 debug('Fetcher    expire %d %s' % (age, path))
 135                                 try: os.remove(path)
 136                                 except (OSError,IOError), oe:
 137                                         if oe.errno != errno.ENOENT: raise
 138                                 continue
 139                         ages.append(age)
 140                 return ages
 141
 142         def need_wait(self, now, imaginary=[]):
 143                 ages = self._cache_scan(now)
 144                 ages += imaginary
 145                 ages.sort()
 146                 debug('Fetcher   ages ' + `ages`)
 147                 min_age = 1
 148                 need_wait = 0
 149                 for age in ages:
 150                         if age < min_age and age <= 5:
 151                                 debug('Fetcher   morewait min=%d age=%d' %
 152                                         (min_age, age))
 153                                 need_wait = max(need_wait, min_age - age)
 154                         min_age += 3
 155                         min_age *= 1.25
 156                 if need_wait > 0:
 157                         need_wait += random.random() - 0.5
 158                 return need_wait
 159
 160         def _rate_limit_cache_clean(self, now):
 161                 need_wait = self.need_wait(now)
 162                 if need_wait > 0:
 163                         debug('Fetcher   wait %d' % need_wait)
 164                         sleep(need_wait)
 165
 166         def fetch(self, url, max_age):
 167                 debug('Fetcher fetch %s' % url)
 168                 cache_corename = urllib.quote_plus(url)
 169                 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
 170                 try: f = file(cache_item, 'r')
 171                 except (OSError,IOError), oe:
 172                         if oe.errno != errno.ENOENT: raise
 173                         f = None
 174                 now = time.time()
 175                 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
 176                 if f is not None:
 177                         s = os.fstat(f.fileno())
 178                         age = now - s.st_mtime
 179                         if age > max_age:
 180                                 debug('Fetcher  stale %d < %d'% (max_age, age))
 181                                 f = None
 182                 if f is not None:
 183                         data = f.read()
 184                         f.close()
 185                         debug('Fetcher  cached %d > %d' % (max_age, age))
 186                         return data
 187
 188                 debug('Fetcher  fetch')
 189                 self._rate_limit_cache_clean(now)
 190
 191                 stream = urllib2.urlopen(url)
 192                 data = stream.read()
 193                 cache_tmp = "%s/#%s~%d#" % (
 194                         self.cachedir, cache_corename, os.getpid())
 195                 f = file(cache_tmp, 'w')
 196                 f.write(data)
 197                 f.close()
 198                 os.rename(cache_tmp, cache_item)
 199                 debug('Fetcher  stored')
 200                 return data
 201
 202 class Yoweb(Fetcher):
 203         def __init__(self, ocean, cachedir):
 204                 debug('Yoweb init %s' % cachedir)
 205                 self.ocean = ocean
 206                 Fetcher.__init__(self, cachedir)
 207
 208         def default_ocean(self, ocean='ice'):
 209                 if self.ocean is None:
 210                         self.ocean = ocean
 211
 212         def yoweb(self, kind, tail, max_age):
 213                 self.default_ocean()
 214                 assert(self.ocean)
 215                 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
 216                         self.ocean, kind, tail)
 217                 return self.fetch(url, max_age)
 218
 219 class Yppedia(Fetcher):
 220         def __init__(self, cachedir):
 221                 debug('Yoweb init %s' % cachedir)
 222                 self.base = 'http://yppedia.puzzlepirates.com/'
 223                 self.localhtml = opts.localhtml
 224                 Fetcher.__init__(self, cachedir)
 225
 226         def __call__(self, rhs):
 227                 if self.localhtml is None:
 228                         url = self.base + rhs
 229                         debug('Yppedia retrieving YPP '+url);
 230                         return self.fetch(url, 3000)
 231                 else:
 232                         return file(opts.localhtml + '/' + rhs, 'r')
 233
 234 #---------- logging assistance for troubled screenscrapers ----------
 235
 236 class SoupLog:
 237         def __init__(self):
 238                 self.msgs = [ ]
 239         def msg(self, m):
 240                 self.msgs.append(m)
 241         def soupm(self, obj, m):
 242                 self.msg(m + '; in ' + `obj`)
 243         def needs_msgs(self, child_souplog):
 244                 self.msgs += child_souplog.msgs
 245                 child_souplog.msgs = [ ]
 246
 247 def soup_text(obj):
 248         str = ''.join(obj.findAll(text=True))
 249         return str.strip()
 250
 251 class SomethingSoupInfo(SoupLog):
 252         def __init__(self, kind, tail, max_age):
 253                 SoupLog.__init__(self)
 254                 html = fetcher.yoweb(kind, tail, max_age)
 255                 self._soup = BeautifulSoup(html,
 256                         convertEntities=BeautifulSoup.HTML_ENTITIES
 257                         )
 258
 259 #---------- scraper for pirate pages ----------
 260
 261 class PirateInfo(SomethingSoupInfo):
 262         # Public data members:
 263         #  pi.standings = { 'Treasure Haul': 'Able' ... }
 264         #  pi.name = name
 265         #  pi.crew = (id, name)
 266         #  pi.flag = (id, name)
 267         #  pi.msgs = [ 'message describing problem with scrape' ]
 268
 269         def __init__(self, pirate, max_age=300):
 270                 SomethingSoupInfo.__init__(self,
 271                         'pirate.wm?target=', pirate, max_age)
 272                 self.name = pirate
 273                 self._find_standings()
 274                 self.crew = self._find_crewflag('crew',
 275                         '^/yoweb/crew/info\\.wm')
 276                 self.flag = self._find_crewflag('flag',
 277                         '^/yoweb/flag/info\\.wm')
 278
 279         def _find_standings(self):
 280                 imgs = self._soup.findAll('img',
 281                         src=regexp.compile('/yoweb/images/stat.*'))
 282                 re = regexp.compile(
 283 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
 284                         )
 285                 standings = { }
 286
 287                 for skill in puzzles:
 288                         standings[skill] = [ ]
 289
 290                 skl = SoupLog()
 291
 292                 for img in imgs:
 293                         try: puzzle = img['alt']
 294                         except KeyError: continue
 295
 296                         if not puzzle in puzzles:
 297                                 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
 298                                 continue
 299                         key = img.findParent('td')
 300                         if key is None:
 301                                 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
 302                                 continue
 303                         valelem = key.findNextSibling('td')
 304                         if valelem is None:
 305                                 skl.soupm(key, 'puzzle missing sibling "%s"'
 306                                         % puzzle)
 307                                 continue
 308                         valstr = soup_text(valelem)
 309                         match = re.match(valstr)
 310                         if match is None:
 311                                 skl.soupm(key, ('puzzle "%s" unparseable'+
 312                                         ' standing "%s"') % (puzzle, valstr))
 313                                 continue
 314                         standing = match.group(match.lastindex)
 315                         standings[puzzle].append(standing)
 316
 317                 self.standings = { }
 318
 319                 for puzzle in puzzles:
 320                         sl = standings[puzzle]
 321                         if len(sl) > 1:
 322                                 skl.msg('puzzle "%s" multiple standings %s' %
 323                                                 (puzzle, `sl`))
 324                                 continue
 325                         if not sl:
 326                                 skl.msg('puzzle "%s" no standing found' % puzzle)
 327                                 continue
 328                         standing = sl[0]
 329                         for i in range(0, standing_limit):
 330                                 if standing == standingvals[i]:
 331                                         self.standings[puzzle] = i
 332                         if not puzzle in self.standings:
 333                                 skl.msg('puzzle "%s" unknown standing "%s"' %
 334                                         (puzzle, standing))
 335
 336                 all_standings_ok = True
 337                 for puzzle in puzzles:
 338                         if not puzzle in self.standings:
 339                                 self.needs_msgs(skl)
 340
 341         def _find_crewflag(self, cf, yoweb_re):
 342                 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
 343                 if len(things) != 1:
 344                         self.msg('zero or several %s id references found' % cf)
 345                         return None
 346                 thing = things[0]
 347                 id_re = '\\b%sid\\=(\\w+)$' % cf
 348                 id_haystack = thing['href']
 349                 match = regexp.compile(id_re).search(id_haystack)
 350                 if match is None:
 351                         self.soupm(thing, ('incomprehensible %s id ref'+
 352                                 ' (%s in %s)') % (cf, id_re, id_haystack))
 353                         return None
 354                 name = soup_text(thing)
 355                 return (match.group(1), name)
 356
 357         def __str__(self):
 358                 return `(self.crew, self.flag, self.standings, self.msgs)`
 359
 360 #---------- scraper for crew pages ----------
 361
 362 class CrewInfo(SomethingSoupInfo):
 363         # Public data members:
 364         #  ci.crewid
 365         #  ci.crew = [ ('Captain',        ['Pirate', ...]),
 366         #              ('Senior Officer', [...]),
 367         #               ... ]
 368         #  pi.msgs = [ 'message describing problem with scrape' ]
 369
 370         def __init__(self, crewid, max_age=300):
 371                 self.crewid = crewid
 372                 SomethingSoupInfo.__init__(self,
 373                         'crew/info.wm?crewid=', crewid, max_age)
 374                 self._find_crew()
 375
 376         def _find_crew(self):
 377                 self.crew = []
 378                 capts = self._soup.findAll('img',
 379                         src='/yoweb/images/crew-captain.png')
 380                 if len(capts) != 1:
 381                         self.msg('crew members: no. of captain images != 1')
 382                         return
 383                 tbl = capts[0]
 384                 while not tbl.find('a', href=pirate_ref_re):
 385                         tbl = tbl.findParent('table')
 386                         if not tbl:
 387                                 self.msg('crew members: cannot find table')
 388                                 return
 389                 current_rank_crew = None
 390                 crew_rank_re = regexp.compile('/yoweb/images/crew')
 391                 for row in tbl.contents:
 392                         # findAll(recurse=False)
 393                         if isinstance(row,basestring):
 394                                 continue
 395
 396                         is_rank = row.find('img', attrs={'src': crew_rank_re})
 397                         if is_rank:
 398                                 rank = soup_text(row)
 399                                 current_rank_crew = []
 400                                 self.crew.append((rank, current_rank_crew))
 401                                 continue
 402                         for cell in row.findAll('a', href=pirate_ref_re):
 403                                 if current_rank_crew is None:
 404                                         self.soupm(cell, 'crew members: crew'
 405                                                 ' before rank')
 406                                         continue
 407                                 current_rank_crew.append(soup_text(cell))
 408
 409         def __str__(self):
 410                 return `(self.crew, self.msgs)`
 411
 412 class FlagInfo(SomethingSoupInfo):
 413         # Public data members (after init):
 414         #
 415         #   flagid
 416         #   name        #               string
 417         #
 418         #   relations[n] = (otherflagname, otherflagid, [stringfromyoweb],
 419         #               thisdeclaring, otherdeclaringmin, otherdeclaringmax)
 420         #               # where {this,other}declaring{,min,max} are:
 421         #               #       -1      {this,other} is declaring war
 422         #               #        0      {this,other} is not doing either
 423         #               #       +1      {this,other} is allying
 424         #   relation_byname[otherflagname] = relations[some_n]
 425         #   relation_byid[otherflagname] = relations[some_n]
 426         #
 427         #   islands[n] = (islandname, islandid)
 428         #
 429         def __init__(self, flagid, max_age=600):
 430                 self.flagid = flagid
 431                 SomethingSoupInfo.__init__(self,
 432                         'flag/info.wm?flagid=', flagid, max_age)
 433                 self._find_flag()
 434
 435         def _find_flag(self):
 436                 font2 = self._soup.find('font',{'size':'+2'})
 437                 self.name = font2.find('b').contents[0]
 438
 439                 self.relations = [ ]
 440                 self.relation_byname = { }
 441                 self.relation_byid = { }
 442                 self.islands = [ ]
 443
 444                 magnate = self._soup.find('img',{'src':
 445                         '/yoweb/images/repute-MAGNATE.png'})
 446                 warinfo = (magnate.findParent('table').findParent('tr').
 447                         findNextSibling('tr').findNext('td',{'align':'left'}))
 448
 449                 def warn(m):
 450                         print >>sys.stderr, 'WARNING: '+m
 451
 452                 def wi_warn(head, waritem):
 453                         warn('unknown warmap item: %s: %s' %
 454                                 (`head`, ``waritem``))
 455
 456                 def wihelp_item(waritem, thing):
 457                         url = waritem.get('href', None)
 458                         if url is None:
 459                                 return ('no url for '+thing,None,None)
 460                         m = regexp.search('\?'+thing+'id=(\d+)$', url)
 461                         if not m: return ('no '+thing+'id',None,None)
 462                         tid = m.group(1)
 463                         tname = waritem.string
 464                         if tname is None:
 465                                 return (thing+' name not just string',None,None)
 466                         return (None,tid,tname)
 467
 468                 def wi_alwar(head, waritem, thisdecl, othermin, othermax):
 469                         (err,flagid,flagname) = wihelp_item(waritem,'flag')
 470                         if err: return err
 471                         rel = self.relation_byid.get(flagid, None)
 472                         if rel: return 'flag id twice!'
 473                         if flagname in self.relation_byname:
 474                                 return 'flag name twice!'
 475                         rel = (flagname,flagid,head, thisdecl,othermin,othermax)
 476                         self.relations.append(rel)
 477                         self.relation_byid[flagid] = rel
 478                         self.relation_byname[flagid] = rel
 479
 480                 def wi_isle(head, waritem):
 481                         (err,isleid,islename) = wihelp_item(waritem,'island')
 482                         if err: return err
 483                         self.islands.append((isleid,islename))
 484
 485                 warmap = {
 486                         'Allied with':                  (wi_alwar,+1,+1,+1),
 487                         'Declaring war against':        (wi_alwar,-1, 0,+1),
 488                         'At war with':                  (wi_alwar,-1,-1,-1),
 489                         'Trying to form an alliance with': (wi_alwar,+1,-1,0),
 490                         'Islands controlled by this flag': (wi_isle,),
 491                         }
 492
 493                 how = (wi_warn, None)
 494
 495                 for waritem in warinfo.findAll(['font','a']):
 496                         if waritem is None: break
 497                         if waritem.name == 'font':
 498                                 colour = waritem.get('color',None)
 499                                 if colour.lstrip('#') != '958A5F':
 500                                         warn('strange colour %s in %s' %
 501                                                 (colour,``waritem``))
 502                                         continue
 503                                 head = waritem.string
 504                                 if head is None:
 505                                         warn('no head string in '+``waritem``)
 506                                         continue
 507                                 head = regexp.sub('\\s+', ' ', head).strip()
 508                                 head = head.rstrip(':')
 509                                 how = (head,) + warmap.get(head, (wi_warn,))
 510                                 continue
 511                         assert(waritem.name == 'a')
 512
 513                         debug('WARHOW %s(%s, waritem, *%s)' %
 514                                 (how[1], `how[0]`, `how[2:]`))
 515                         bad = how[1](how[0], waritem, *how[2:])
 516                         if bad:
 517                                 warn('bad waritem %s: %s: %s' % (`how[0]`,
 518                                         bad, ``waritem``))
 519
 520         def __str__(self):
 521                 return `(self.name, self.islands, self.relations)`
 522
 523 #---------- scraper for ocean info incl. embargoes etc. ----------
 524
 525 class IslandBasicInfo():
 526         # Public data attributes:
 527         #  ocean
 528         #  name
 529         # Public data attributes maybe set by caller:
 530         #  arch
 531         def __init__(self, ocean, islename):
 532                 self.ocean = ocean
 533                 self.name = islename
 534         def yppedia(self):
 535                 def q(x): return urllib.quote(x.replace(' ','_'))
 536                 url_rhs = q(self.name) + '_(' + q(self.ocean) + ')'
 537                 return yppedia(url_rhs)
 538         def __str__(self):
 539                 return `(self.ocean, self.name)`
 540
 541 class IslandExtendedInfo(IslandBasicInfo):
 542         # Public data attributes (inherited):
 543         #  ocean
 544         #  name
 545         # Public data attributes (additional):
 546         #  islandid
 547         #  yoweb_url
 548         #  flagid
 549         def __init__(self, ocean, islename):
 550                 IslandBasicInfo.__init__(self, ocean, islename)
 551                 self.islandid = None
 552                 self.yoweb_url = None
 553                 self._collect_yoweb()
 554                 self._collect_flagid()
 555
 556         def _collect_yoweb(self):
 557                 debug('IEI COLLECT YOWEB '+`self.name`)
 558                 self.islandid = None
 559                 self.yoweb_url = None
 560
 561                 soup = BeautifulSoup(self.yppedia())
 562                 content = soup.find('div', attrs = {'id': 'content'})
 563                 yoweb_re = regexp.compile('^http://\w+\.puzzlepirates\.com/'+
 564                         'yoweb/island/info\.wm\?islandid=(\d+)$')
 565                 a = soup.find('a', attrs = { 'href': yoweb_re })
 566                 if a is None:
 567                         debug('IEI COLLECT YOWEB '+`self.name`+' NONE')
 568                         return
 569
 570                 debug('IEI COLLECT YOWEB '+`self.name`+' GOT '+``a``)
 571                 self.yoweb_url = a['href']
 572                 m = yoweb_re.search(self.yoweb_url)
 573                 self.islandid = m.group(1)
 574
 575         def _collect_flagid(self):
 576                 self.flagid = None
 577
 578                 yo = self.yoweb_url
 579                 debug('IEI COLLECT FLAGID '+`self.name`+' URL '+`yo`)
 580                 if yo is None: return None
 581                 dataf = fetcher.fetch(yo, 1800)
 582                 soup = BeautifulSoup(dataf)
 583                 ruler_re = regexp.compile(
 584                         '/yoweb/flag/info\.wm\?flagid=(\d+)$')
 585                 ruler = soup.find('a', attrs = { 'href': ruler_re })
 586                 if not ruler:
 587                         debug('IEI COLLECT FLAGID '+`self.name`+' NONE')
 588                         return
 589                 debug('IEI COLLECT FLAGID '+`self.name`+' GOT '+``ruler``)
 590                 m = ruler_re.search(ruler['href'])
 591                 self.flagid = m.group(1)
 592
 593         def __str__(self):
 594                 return `(self.ocean, self.islandid, self.name,
 595                         self.yoweb_url, self.flagid)`
 596
 597 class IslandFlagInfo(IslandExtendedInfo):
 598         # Public data attributes (inherited):
 599         #  ocean
 600         #  name
 601         #  islandid
 602         #  yoweb_url
 603         #  flagid
 604         # Public data attributes (additional):
 605         #  flag
 606         def __init__(self, ocean, islename):
 607                 IslandExtendedInfo.__init__(self, ocean, islename)
 608                 self.flag = None
 609                 self._collect_flag()
 610
 611         def _collect_flag(self):
 612                 if self.flagid is None: return
 613                 self.flag = FlagInfo(self.flagid, 1800)
 614
 615         def __str__(self):
 616                 return IslandExtendedInfo.__str__(self) + '; ' + str(self.flag)
 617
 618 class NullProgressReporter():
 619         def doing(self, msg): pass
 620         def stop(self): pass
 621
 622 class TypewriterProgressReporter():
 623         def __init__(self):
 624                 self._l = 0
 625         def doing(self,m):
 626                 self._doing(m + '...')
 627         def _doing(self,m):
 628                 self._write('\r')
 629                 self._write(m)
 630                 less = self._l - len(m)
 631                 if less > 0:
 632                         self._write(' ' * less)
 633                         self._write('\b' * less)
 634                 self._l = len(m)
 635                 sys.stdout.flush()
 636         def stop(self):
 637                 self._doing('')
 638                 self._l = 0
 639         def _write(self,t):
 640                 sys.stdout.write(t)
 641
 642 class OceanInfo():
 643         # Public data attributes:
 644         #   oi.islands[islename] = IslandInfo(...)
 645         #   oi.arches[archname][islename] = IslandInfo(...)
 646         def __init__(self, isleclass=IslandBasicInfo):
 647                 self.isleclass = isleclass
 648                 self.ocean = fetcher.ocean.lower().capitalize()
 649
 650                 progressreporter.doing('fetching ocean info')
 651
 652                 cmdl = ['./yppedia-ocean-scraper']
 653                 if opts.localhtml is not None:
 654                         cmdl += ['--local-html-dir',opts.localhtml]
 655                 cmdl += [self.ocean]
 656                 debug('OceanInfo collect running ' + `cmdl`)
 657                 oscraper = subprocess.Popen(
 658                         cmdl,
 659                         stdout = subprocess.PIPE,
 660                         cwd = yppsc_dir()+'/yarrg',
 661                         shell=False, stderr=None,
 662                         )
 663                 h = oscraper.stdout.readline()
 664                 debug('OceanInfo collect h '+`h`)
 665                 assert(regexp.match('^ocean ', h))
 666                 arch_re = regexp.compile('^ (\S.*)')
 667                 island_re = regexp.compile('^  (\S.*)')
 668
 669                 oscraper.wait()
 670                 assert(oscraper.returncode == 0)
 671
 672                 self.islands = { }
 673                 self.arches = { }
 674                 archname = None
 675
 676                 isles = [ ]
 677                 progressreporter.doing('parsing ocean info')
 678
 679                 for l in oscraper.stdout:
 680                         debug('OceanInfo collect l '+`l`)
 681                         l = l.rstrip('\n')
 682                         m = island_re.match(l)
 683                         if m:
 684                                 assert(archname is not None)
 685                                 islename = m.group(1)
 686                                 isles.append((archname, islename))
 687                                 continue
 688                         m = arch_re.match(l)
 689                         if m:
 690                                 archname = m.group(1)
 691                                 assert(archname not in self.arches)
 692                                 self.arches[archname] = { }
 693                                 continue
 694                         assert(False)
 695
 696                 for i in xrange(0, len(isles)-1):
 697                         (archname, islename) = isles[i]
 698                         progressreporter.doing(
 699                                 'fetching isle info %2d/%d (%s: %s)'
 700                                 % (i, len(isles), archname, islename))
 701                         isle = self.isleclass(self.ocean, islename)
 702                         isle.arch = archname
 703                         self.islands[islename] = isle
 704                         self.arches[archname][islename] = isle
 705
 706         def __str__(self):
 707                 return `(self.islands, self.arches)`
 708
 709 #---------- pretty-printer for tables of pirate puzzle standings ----------
 710
 711 class StandingsTable:
 712         def __init__(self, f, use_puzzles=None, col_width=6, gap_every=5):
 713                 if use_puzzles is None:
 714                         if opts.ship_duty:
 715                                 use_puzzles=duty_puzzles
 716                         else:
 717                                 use_puzzles=puzzles
 718                 self._puzzles = use_puzzles
 719                 self.f = f
 720                 self._cw = col_width-1
 721                 self._gap_every = gap_every
 722                 self._linecount = 0
 723                 self._o = f.write
 724
 725         def _nl(self): self._o('\n')
 726
 727         def _pline(self, lhs, puzstrs, extra):
 728                 if (self._linecount > 0
 729                     and self._gap_every is not None
 730                     and not (self._linecount % self._gap_every)):
 731                         self._nl()
 732                 self._o('%-*s' % (max(max_pirate_namelen+1, 15), lhs))
 733                 for v in puzstrs:
 734                         self._o(' %-*.*s' % (self._cw,self._cw, v))
 735                 if extra:
 736                         self._o(' ' + extra)
 737                 self._nl()
 738                 self._linecount += 1
 739
 740         def _puzstr(self, pi, puzzle):
 741                 if not isinstance(puzzle,list): puzzle = [puzzle]
 742                 try: standing = max([pi.standings[p] for p in puzzle])
 743                 except KeyError: return '?'
 744                 if not standing: return ''
 745                 s = ''
 746                 if self._cw > 4:
 747                         c1 = standingvals[standing][0]
 748                         if standing < 3: c1 = c1.lower() # 3 = Master
 749                         s += `standing`
 750                 if self._cw > 5:
 751                         s += ' '
 752                 s += '*' * (standing / 2)
 753                 s += '+' * (standing % 2)
 754                 return s
 755
 756         def headings(self, lhs='', rhs=None):
 757                 def puzn_redact(name):
 758                         if isinstance(name,list):
 759                                 return '/'.join(
 760                                         ["%.*s" % (self._cw/2, puzn_redact(n))
 761                                          for n in name])
 762                         spc = name.find(' ')
 763                         if spc < 0: return name
 764                         return name[0:min(4,spc)] + name[spc+1:]
 765                 self._linecount = -2
 766                 self._pline(lhs, map(puzn_redact, self._puzzles), rhs)
 767                 self._linecount = 0
 768         def literalline(self, line):
 769                 self._o(line)
 770                 self._nl()
 771                 self._linecount = 0
 772         def pirate_dummy(self, name, standingstring, extra=None):
 773                 standings = standingstring * len(self._puzzles)
 774                 self._pline(' '+name, standings, extra)
 775         def pirate(self, pi, extra=None):
 776                 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
 777                 self._pline(' '+pi.name, puzstrs, extra)
 778
 779
 780 #---------- chat log parser ----------
 781
 782 class PirateAboard:
 783         # This is essentially a transparent, dumb, data class.
 784         #  pa.v                 may be None
 785         #  pa.name
 786         #  pa.last_time
 787         #  pa.last_event
 788         #  pa.gunner
 789         #  pa.last_chat_time
 790         #  pa.last_chat_chan
 791         #  pa.pi
 792
 793         # Also used for jobbing applicants:
 794         #               happens when                    expires (to "-")
 795         #   -            disembark, leaves crew          no
 796         #   aboard       evidence of them being aboard   no
 797         #   applied      "has applied for the job"       120s, configurable
 798         #   ashore       "has taken a job"               30min, configurable
 799         #   declined     "declined the job offer"        30s, configurable
 800         #   invited      "has been invited to job"       120s, configurable
 801         #
 802         #  pa.jobber    None, 'ashore', 'applied', 'invited', 'declined'
 803         #  pa.expires   expiry time time
 804
 805         def __init__(pa, pn, v, time, event):
 806                 pa.name = pn
 807                 pa.v = v
 808                 pa.last_time = time
 809                 pa.last_event = event
 810                 pa.last_chat_time = None
 811                 pa.last_chat_chan = None
 812                 pa.gunner = False
 813                 pa.pi = None
 814                 pa.jobber = None
 815                 pa.expires = None
 816
 817         def pirate_info(pa):
 818                 now = time.time()
 819                 if pa.pi:
 820                         age = now - pa.pi_fetched
 821                         guide = random.randint(120,240)
 822                         if age <= guide:
 823                                 return pa.pi
 824                         debug('PirateAboard refresh %d > %d  %s' % (
 825                                 age, guide, pa.name))
 826                         imaginary = [2,4]
 827                 else:
 828                         imaginary = [1]
 829                 wait = fetcher.need_wait(now, imaginary)
 830                 if wait:
 831                         debug('PirateAboard fetcher not ready %d' % wait)
 832                         return pa.pi
 833                 pa.pi = PirateInfo(pa.name, 600)
 834                 pa.pi_fetched = now
 835                 return pa.pi
 836
 837 class ChatLogTracker:
 838         # This is quite complex so we make it opaque.  Use the
 839         # official invokers, accessors etc.
 840
 841         def __init__(self, myself_pi, logfn):
 842                 self._pl = {}   # self._pl['Pirate'] =
 843                 self._vl = {}   #   self._vl['Vessel']['Pirate'] = PirateAboard
 844                                 # self._vl['Vessel']['#lastinfo']
 845                                 # self._vl['Vessel']['#name']
 846                                 # self._v = self._vl[self._vessel]
 847                 self._date = None
 848                 self._myself = myself_pi
 849                 self._lbuf = ''
 850                 self._f = file(logfn)
 851                 flen = os.fstat(self._f.fileno()).st_size
 852                 max_backlog = 500000
 853                 if flen > max_backlog:
 854                         startpos = flen - max_backlog
 855                         self._f.seek(startpos)
 856                         self._f.readline()
 857                 self._progress = [0, flen - self._f.tell()]
 858                 self._disembark_myself()
 859                 self._need_redisplay = False
 860                 self._lastvessel = None
 861
 862         def _disembark_myself(self):
 863                 self._v = None
 864                 self._vessel = None
 865                 self.force_redisplay()
 866
 867         def force_redisplay(self):
 868                 self._need_redisplay = True
 869
 870         def _vessel_updated(self, v, timestamp):
 871                 if v is None: return
 872                 v['#lastinfo'] = timestamp
 873                 self.force_redisplay()
 874
 875         def _onboard_event(self,v,timestamp,pirate,event,jobber=None):
 876                 pa = self._pl.get(pirate, None)
 877                 if pa is not None and pa.v is v:
 878                         pa.last_time = timestamp
 879                         pa.last_event = event
 880                 else:
 881                         if pa is not None and pa.v is not None:
 882                                 del pa.v[pirate]
 883                         pa = PirateAboard(pirate, v, timestamp, event)
 884                         self._pl[pirate] = pa
 885                         if v is not None: v[pirate] = pa
 886                 pa.jobber = jobber
 887
 888                 if jobber is None: timeout = None
 889                 else: timeout = getattr(opts, 'timeout_'+jobber)
 890                 if timeout is None: pa.expires = None
 891                 else: pa.expires = timestamp + timeout
 892                 self._vessel_updated(v, timestamp)
 893                 return pa
 894
 895         def _expire_jobbers(self, now):
 896                 for pa in self._pl.values():
 897                         if pa.expires is None: continue
 898                         if pa.expires >= now: continue
 899                         v = pa.v
 900                         del self._pl[pa.name]
 901                         if v is not None: del v[pa.name]
 902                         self.force_redisplay()
 903
 904         def _trash_vessel(self, v):
 905                 for pn in v:
 906                         if pn.startswith('#'): continue
 907                         del self._pl[pn]
 908                 vn = v['#name']
 909                 del self._vl[vn]
 910                 if v is self._v: self._disembark_myself()
 911                 self.force_redisplay()
 912
 913         def _vessel_stale(self, v, timestamp):
 914                 return timestamp - v['#lastinfo'] > opts.ship_reboard_clearout
 915
 916         def _vessel_check_expire(self, v, timestamp):
 917                 if not self._vessel_stale(v, timestamp):
 918                         return v
 919                 self._debug_line_disposition(timestamp,'',
 920                         'stale-reset ' + v['#name'])
 921                 self._trash_vessel(v)
 922                 return None
 923
 924         def expire_garbage(self, timestamp):
 925                 for v in self._vl.values():
 926                         self._vessel_check_expire(v, timestamp)
 927
 928         def _vessel_lookup(self, vn, timestamp, dml=[], create=False):
 929                 v = self._vl.get(vn, None)
 930                 if v is not None:
 931                         v = self._vessel_check_expire(v, timestamp)
 932                 if v is not None:
 933                         dml.append('found')
 934                         return v
 935                 if not create:
 936                         dml.append('no')
 937                 dml.append('new')
 938                 self._vl[vn] = v = { '#name': vn }
 939                 self._vessel_updated(v, timestamp)
 940                 return v
 941
 942         def _find_matching_vessel(self, pattern, timestamp, cmdr,
 943                                         dml=[], create=False):
 944                 # use when a commander pirate `cmdr' specified a vessel
 945                 #  by name `pattern' (either may be None)
 946                 # if create is true, will create the vessel
 947                 #  record if an exact name is specified
 948
 949                 if (pattern is not None and
 950                     not '*' in pattern
 951                     and len(pattern.split(' ')) == 2):
 952                         vn = pattern.title()
 953                         dml.append('exact')
 954                         return self._vessel_lookup(
 955                                 vn, timestamp, dml=dml, create=create)
 956
 957                 if pattern is None:
 958                         pattern_check = lambda vn: True
 959                 else:
 960                         re = '(?:.* )?%s$' % pattern.lower().replace('*','.+')
 961                         pattern_check = regexp.compile(re, regexp.I).match
 962
 963                 tries = []
 964
 965                 cmdr_pa = self._pl.get(cmdr, None)
 966                 if cmdr_pa: tries.append((cmdr_pa.v, 'cmdr'))
 967
 968                 tries.append((self._v, 'here'))
 969                 tried_vns = []
 970
 971                 for (v, dm) in tries:
 972                         if v is None: dml.append(dm+'?'); continue
 973
 974                         vn = v['#name']
 975                         if not pattern_check(vn):
 976                                 tried_vns.append(vn)
 977                                 dml.append(dm+'#')
 978                                 continue
 979
 980                         dml.append(dm+'!')
 981                         return v
 982
 983                 if pattern is not None and '*' in pattern:
 984                         search = [
 985                                 (vn,v)
 986                                 for (vn,v) in self._vl.iteritems()
 987                                 if not self._vessel_stale(v, timestamp)
 988                                 if pattern_check(vn)
 989                                 ]
 990                         #debug('CLT-RE /%s/ wanted (%s) searched (%s)' % (
 991                         #       re,
 992                         #       '/'.join(tried_vns),
 993                         #       '/'.join([vn for (vn,v) in search])))
 994
 995                         if len(search)==1:
 996                                 dml.append('one')
 997                                 return search[0][1]
 998                         elif search:
 999                                 dml.append('many')
1000                         else:
1001                                 dml.append('none')
1002
1003         def _debug_line_disposition(self,timestamp,l,m):
1004                 debug('CLT %13s %-40s %s' % (timestamp,m,l))
1005
1006         def _rm_crew_l(self,re,l):
1007                 m = regexp.match(re,l)
1008                 if m and m.group(2) == self._myself.crew[1]:
1009                         return m.group(1)
1010                 else:
1011                         return None
1012
1013         def local_command(self, metacmd):
1014                 # returns None if all went well, or problem message
1015                 return self._command(self._myself.name, metacmd,
1016                         "local", time.time(),
1017                         (lambda m: debug('CMD %s' % metacmd)))
1018
1019         def _command(self, cmdr, metacmd, chan, timestamp, d):
1020                 # returns None if all went well, or problem message
1021                 metacmd = regexp.sub('\\s+', ' ', metacmd).strip()
1022                 m2 = regexp.match(
1023                     '/([adj]) (?:([A-Za-z* ]+)\\s*:)?([A-Za-z ]+)$',
1024                     metacmd)
1025                 if not m2: return "unknown syntax or command"
1026
1027                 (cmd, pattern, targets) = m2.groups()
1028                 dml = ['cmd', chan, cmd]
1029
1030                 if cmd == 'a': each = self._onboard_event
1031                 elif cmd == 'd': each = disembark
1032                 else: each = lambda *l: self._onboard_event(*l,
1033                                 **{'jobber':'applied'})
1034
1035                 if cmdr == self._myself.name:
1036                         dml.append('self')
1037                         how = 'cmd: %s' % cmd
1038                 else:
1039                         dml.append('other')
1040                         how = 'cmd: %s %s' % (cmd,cmdr)
1041
1042                 if cmd == 'j':
1043                         if pattern is not None:
1044                                 return "/j command does not take a vessel"
1045                         v = None
1046                 else:
1047                         v = self._find_matching_vessel(
1048                                 pattern, timestamp, cmdr,
1049                                 dml, create=True)
1050
1051                 if cmd == 'j' or v is not None:
1052                         targets = targets.strip().split(' ')
1053                         dml.append(`len(targets)`)
1054                         for target in targets:
1055                                 each(v, timestamp, target.title(), how)
1056                         self._vessel_updated(v, timestamp)
1057
1058                 dm = ' '.join(dml)
1059                 return d(dm)
1060
1061                 return None
1062
1063         def chatline(self,l):
1064                 rm = lambda re: regexp.match(re,l)
1065                 d = lambda m: self._debug_line_disposition(timestamp,l,m)
1066                 rm_crew = lambda re: self._rm_crew_l(re,l)
1067                 timestamp = None
1068
1069                 m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$')
1070                 if m:
1071                         self._date = [int(x) for x in m.groups()]
1072                         self._previous_timestamp = None
1073                         return d('date '+`self._date`)
1074
1075                 if self._date is None:
1076                         return d('date unset')
1077
1078                 m = rm('\\[(\d\d):(\d\d):(\d\d)\\] ')
1079                 if not m:
1080                         return d('no timestamp')
1081
1082                 while True:
1083                         time_tuple = (self._date +
1084                                       [int(x) for x in m.groups()] +
1085                                       [-1,-1,-1])
1086                         timestamp = time.mktime(time_tuple)
1087                         if timestamp >= self._previous_timestamp: break
1088                         self._date[2] += 1
1089                         self._debug_line_disposition(timestamp,'',
1090                                 'new date '+`self._date`)
1091
1092                 self._previous_timestamp = timestamp
1093
1094                 l = l[l.find(' ')+1:]
1095
1096                 def ob_x(pirate,event):
1097                         return self._onboard_event(
1098                                         self._v, timestamp, pirate, event)
1099                 def ob1(did): ob_x(m.group(1), did); return d(did)
1100                 def oba(did): return ob1('%s %s' % (did, m.group(2)))
1101
1102                 def jb(pirate,jobber):
1103                         return self._onboard_event(
1104                                 None, timestamp, pirate,
1105                                 ("jobber %s" % jobber),
1106                                 jobber=jobber
1107                                 )
1108
1109                 def disembark(v, timestamp, pirate, event):
1110                         self._onboard_event(
1111                                         v, timestamp, pirate, 'leaving '+event)
1112                         del v[pirate]
1113                         del self._pl[pirate]
1114
1115                 def disembark_me(why):
1116                         self._disembark_myself()
1117                         return d('disembark-me '+why)
1118
1119                 m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
1120                 if m:
1121                         dm = ['boarding']
1122                         pn = self._myself.name
1123                         vn = m.group(1)
1124                         v = self._vessel_lookup(vn, timestamp, dm, create=True)
1125                         self._lastvessel = self._vessel = vn
1126                         self._v = v
1127                         ob_x(pn, 'we boarded')
1128                         self.expire_garbage(timestamp)
1129                         return d(' '.join(dm))
1130
1131                 if self._v is None:
1132                         return d('no vessel')
1133
1134                 m = rm('(\\w+) has come aboard\\.$')
1135                 if m: return ob1('boarded');
1136
1137                 m = rm('You have ordered (\\w+) to do some (\\S.*\\S)\\.$')
1138                 if m:
1139                         (who,what) = m.groups()
1140                         pa = ob_x(who,'ord '+what)
1141                         if what == 'Gunning':
1142                                 pa.gunner = True
1143                         return d('duty order')
1144
1145                 m = rm('(\\w+) abandoned a (\\S.*\\S) station\\.$')
1146                 if m: oba('stopped'); return d("end")
1147
1148                 def chat_core(speaker, chan):
1149                         try: pa = self._pl[speaker]
1150                         except KeyError: return 'mystery'
1151                         if pa.v is not None and pa.v is not self._v:
1152                                 return 'elsewhere'
1153                         pa.last_chat_time = timestamp
1154                         pa.last_chat_chan = chan
1155                         self.force_redisplay()
1156                         return 'here'
1157
1158                 def chat(chan):
1159                         speaker = m.group(1)
1160                         dm = chat_core(speaker, chan)
1161                         return d('chat %s %s' % (chan, dm))
1162
1163                 def chat_metacmd(chan):
1164                         (cmdr, metacmd) = m.groups()
1165                         whynot = self._command(
1166                                 cmdr, metacmd, chan, timestamp, d)
1167                         if whynot is not None:
1168                                 return chat(chan)
1169                         else:
1170                                 chat_core(cmdr, 'cmd '+chan)
1171
1172                 m = rm('(\\w+) (?:issued an order|ordered everyone) "')
1173                 if m: return ob1('general order');
1174
1175                 m = rm('(\\w+) says, "')
1176                 if m: return chat('public')
1177
1178                 m = rm('(\\w+) tells ye, "')
1179                 if m: return chat('private')
1180
1181                 m = rm('Ye told (\\w+), "(.*)"$')
1182                 if m: return chat_metacmd('private')
1183
1184                 m = rm('(\\w+) flag officer chats, "')
1185                 if m: return chat('flag officer')
1186
1187                 m = rm('(\\w+) officer chats, "(.*)"$')
1188                 if m: return chat_metacmd('officer')
1189
1190                 m = rm('Ye accepted the offer to job with ')
1191                 if m: return disembark_me('jobbing')
1192
1193                 m = rm('Ye hop on the ferry and are whisked away ')
1194                 if m: return disembark_me('ferry')
1195
1196                 m = rm('Whisking away to yer home on the magical winds')
1197                 if m: return disembark_me('home')
1198
1199                 m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
1200                 if m:
1201                         pl = m.group(1).split(', ')
1202                         if not self._myself.name in pl:
1203                                 return d('lost melee')
1204                         for pn in pl:
1205                                 if ' ' in pn: continue
1206                                 ob_x(pn,'won melee')
1207                         return d('won melee')
1208
1209                 m = rm('(\\w+) is eliminated\\!')
1210                 if m: return ob1('eliminated in fray');
1211
1212                 m = rm('(\\w+) has driven \w+ from the ship\\!')
1213                 if m: return ob1('boarder repelled');
1214
1215                 m = rm('\w+ has bested (\\w+), and turns'+
1216                         ' to the rest of the ship\\.')
1217                 if m: return ob1('boarder unrepelled');
1218
1219                 pirate = rm_crew("(\\w+) has taken a job with '(.*)'\\.")
1220                 if pirate: return jb(pirate, 'ashore')
1221
1222                 pirate = rm_crew("(\\w+) has left '(.*)'\\.")
1223                 if pirate:
1224                         disembark(self._v, timestamp, pirate, 'left crew')
1225                         return d('left crew')
1226
1227                 m = rm('(\w+) has applied for the posted job\.')
1228                 if m: return jb(m.group(1), 'applied')
1229
1230                 pirate= rm_crew("(\\w+) has been invited to job for '(.*)'\\.")
1231                 if pirate: return jb(pirate, 'invited')
1232
1233                 pirate = rm_crew("(\\w+) declined the job offer for '(.*)'\\.")
1234                 if pirate: return jb(pirate, 'declined')
1235
1236                 m = rm('(\\w+) has left the vessel\.')
1237                 if m:
1238                         pirate = m.group(1)
1239                         disembark(self._v, timestamp, pirate, 'disembarked')
1240                         return d('disembarked')
1241
1242                 return d('not-matched')
1243
1244         def _str_pa(self, pn, pa):
1245                 assert self._pl[pn] == pa
1246                 s = ' '*20 + "%s %-*s %13s %-30s %13s %-20s %13s" % (
1247                         (' ','G')[pa.gunner],
1248                         max_pirate_namelen, pn,
1249                         pa.last_time, pa.last_event,
1250                         pa.last_chat_time, pa.last_chat_chan,
1251                         pa.jobber)
1252                 if pa.expires is not None:
1253                         s += " %-5d" % (pa.expires - pa.last_time)
1254                 s += "\n"
1255                 return s
1256
1257         def _str_vessel(self, vn, v):
1258                 s = ' vessel %s\n' % vn
1259                 s += ' '*20 + "%-*s   %13s\n" % (
1260                                 max_pirate_namelen, '#lastinfo',
1261                                 v['#lastinfo'])
1262                 assert v['#name'] == vn
1263                 for pn in sorted(v.keys()):
1264                         if pn.startswith('#'): continue
1265                         pa = v[pn]
1266                         assert pa.v == v
1267                         s += self._str_pa(pn,pa)
1268                 return s
1269
1270         def __str__(self):
1271                 s = '''<ChatLogTracker
1272  myself %s
1273  vessel %s
1274 '''                     % (self._myself.name, self._vessel)
1275                 assert ((self._v is None and self._vessel is None) or
1276                         (self._v is self._vl[self._vessel]))
1277                 if self._vessel is not None:
1278                         s += self._str_vessel(self._vessel, self._v)
1279                 for vn in sorted(self._vl.keys()):
1280                         if vn == self._vessel: continue
1281                         s += self._str_vessel(vn, self._vl[vn])
1282                 s += " elsewhere\n"
1283                 for p in self._pl:
1284                         pa = self._pl[p]
1285                         if pa.v is not None:
1286                                 assert pa.v[p] is pa
1287                                 assert pa.v in self._vl.values()
1288                         else:
1289                                 s += self._str_pa(pa.name, pa)
1290                 s += '>\n'
1291                 return s
1292
1293         def catchup(self, progress=None):
1294                 while True:
1295                         more = self._f.readline()
1296                         if not more: break
1297
1298                         self._progress[0] += len(more)
1299                         if progress: progress.progress(*self._progress)
1300
1301                         self._lbuf += more
1302                         if self._lbuf.endswith('\n'):
1303                                 self.chatline(self._lbuf.rstrip())
1304                                 self._lbuf = ''
1305                                 if opts.debug >= 2:
1306                                         debug(self.__str__())
1307                 self._expire_jobbers(time.time())
1308
1309                 if progress: progress.caughtup()
1310
1311         def changed(self):
1312                 rv = self._need_redisplay
1313                 self._need_redisplay = False
1314                 return rv
1315         def myname(self):
1316                 # returns our pirate name
1317                 return self._myself.name
1318         def vesselname(self):
1319                 # returns the vessel name we're aboard or None
1320                 return self._vessel
1321         def lastvesselname(self):
1322                 # returns the last vessel name we were aboard or None
1323                 return self._lastvessel
1324         def aboard(self, vesselname=True):
1325                 # returns a list of PirateAboard the vessel
1326                 #  sorted by pirate name
1327                 #  you can pass this None and you'll get []
1328                 #  or True for the current vessel (which is the default)
1329                 #  the returned value is a fresh list of persistent
1330                 #  PirateAboard objects
1331                 if vesselname is True: v = self._v
1332                 else: v = self._vl.get(vesselname.title())
1333                 if v is None: return []
1334                 return [ v[pn]
1335                          for pn in sorted(v.keys())
1336                          if not pn.startswith('#') ]
1337         def jobbers(self):
1338                 # returns a the jobbers' PirateAboards,
1339                 # sorted by jobber class and reverse of expiry time
1340                 l = [ pa
1341                       for pa in self._pl.values()
1342                       if pa.jobber is not None
1343                     ]
1344                 def compar_key(pa):
1345                         return (pa.jobber, -pa.expires)
1346                 l.sort(key = compar_key)
1347                 return l
1348
1349 #---------- implementations of actual operation modes ----------
1350
1351 def do_pirate(pirates, bu):
1352         print '{'
1353         for pirate in pirates:
1354                 info = PirateInfo(pirate)
1355                 print '%s: %s,' % (`pirate`, info)
1356         print '}'
1357
1358 def prep_crewflag_of(args, bu, max_age, selector, constructor):
1359         if len(args) != 1: bu('crew-of etc. take one pirate name')
1360         pi = PirateInfo(args[0], max_age)
1361         cf = selector(pi)
1362         if cf is None: return None
1363         return constructor(cf[0], max_age)
1364
1365 def prep_crew_of(args, bu, max_age=300):
1366         return prep_crewflag_of(args, bu, max_age,
1367                 (lambda pi: pi.crew), CrewInfo)
1368
1369 def prep_flag_of(args, bu, max_age=300):
1370         return prep_crewflag_of(args, bu, max_age,
1371                 (lambda pi: pi.flag), FlagInfo)
1372
1373 def do_crew_of(args, bu):
1374         ci = prep_crew_of(args, bu)
1375         print ci
1376
1377 def do_flag_of(args, bu):
1378         fi = prep_flag_of(args, bu)
1379
1380 def do_standings_crew_of(args, bu):
1381         ci = prep_crew_of(args, bu, 60)
1382         tab = StandingsTable(sys.stdout)
1383         tab.headings()
1384         for (rank, members) in ci.crew:
1385                 if not members: continue
1386                 tab.literalline('')
1387                 tab.literalline('%s:' % rank)
1388                 for p in members:
1389                         pi = PirateInfo(p, random.randint(900,1800))
1390                         tab.pirate(pi)
1391
1392 def do_ocean(args, bu):
1393         if (len(args)): bu('ocean takes no further arguments')
1394         fetcher.default_ocean()
1395         oi = OceanInfo(IslandFlagInfo)
1396         print oi
1397         for islename in sorted(oi.islands.keys()):
1398                 isle = oi.islands[islename]
1399                 print isle
1400
1401 def do_embargoes(args, bu):
1402         if (len(args)): bu('ocean takes no further arguments')
1403         fetcher.default_ocean()
1404         oi = OceanInfo(IslandFlagInfo)
1405         wr = sys.stdout.write
1406         print ('EMBARGOES:  Island    | Owning flag'+
1407                 '                    | Embargoed flags')
1408
1409         def getflname(isle):
1410                 if isle.islandid is None: return 'uncolonisable'
1411                 if isle.flag is None: return 'uncolonised'
1412                 return isle.flag.name
1413
1414         progressreporter.stop()
1415
1416         for archname in sorted(oi.arches.keys()):
1417                 print 'ARCHIPELAGO: ',archname
1418                 for islename in sorted(oi.arches[archname].keys()):
1419                         isle = oi.islands[islename]
1420                         wr(' %-20s | ' % isle.name)
1421                         flname = getflname(isle)
1422                         wr('%-30s | ' % flname)
1423                         flag = isle.flag
1424                         if flag is None: print ''; continue
1425                         delim = ''
1426                         for rel in flag.relations:
1427                                 (oname, oid, dummy, thisdeclaring,
1428                                         odeclaringmin,odeclaringmax) = rel
1429                                 if thisdeclaring >= 0: continue
1430                                 wr(delim)
1431                                 wr(oname)
1432                                 delim = '; '
1433                         print ''
1434
1435 def do_embargoes_flag_of(args, bu):
1436         progressreporter.doing('fetching flag info')
1437         fi = prep_flag_of(args, bu)
1438         oi = OceanInfo(IslandFlagInfo)
1439         progressreporter.stop()
1440         any = False
1441         for islename in sorted(oi.islands.keys()):
1442                 isle = oi.islands[islename]
1443                 flag = isle.flag
1444                 if flag is None: continue
1445                 for rel in flag.relations:
1446                         (oname, oid, dummy, thisdeclaring,
1447                                 odeclaringmin,odeclaringmax) = rel
1448                         if thisdeclaring >= 0: continue
1449                         if oid != fi.flagid: continue
1450                         if not any: print 'EMBARGOED:'
1451                         any = True
1452                         print " %-30s (%s)" % (islename, flag.name)
1453         if not any:
1454                 print 'No embargoes.'
1455
1456 #----- modes which use the chat log parser are quite complex -----
1457
1458 class ProgressPrintPercentage:
1459         def __init__(self, f=sys.stdout):
1460                 self._f = f
1461         def progress_string(self,done,total):
1462                 return "scan chat logs %3d%%\r" % ((done*100) / total)
1463         def progress(self,*a):
1464                 self._f.write(self.progress_string(*a))
1465                 self._f.flush()
1466         def show_init(self, pirate, ocean):
1467                 print >>self._f, 'Starting up, %s on the %s ocean' % (
1468                         pirate, ocean)
1469         def caughtup(self):
1470                 self._f.write('                   \r')
1471                 self._f.flush()
1472
1473 def prep_chat_log(args, bu,
1474                 progress=ProgressPrintPercentage(),
1475                 max_myself_age=3600):
1476         if len(args) != 1: bu('this action takes only chat log filename')
1477         logfn = args[0]
1478         logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_'
1479         match = regexp.match(logfn_re, logfn)
1480         if not match: bu('chat log filename is not in expected format')
1481         (pirate, ocean) = match.groups()
1482         fetcher.default_ocean(ocean)
1483
1484         progress.show_init(pirate, fetcher.ocean)
1485         myself = PirateInfo(pirate,max_myself_age)
1486         track = ChatLogTracker(myself, logfn)
1487
1488         opts.debug -= 2
1489         track.catchup(progress)
1490         opts.debug += 2
1491
1492         track.force_redisplay()
1493
1494         return (myself, track)
1495
1496 def do_track_chat_log(args, bu):
1497         (myself, track) = prep_chat_log(args, bu)
1498         while True:
1499                 track.catchup()
1500                 if track.changed():
1501                         print track
1502                 sleep(0.5 + 0.5 * random.random())
1503
1504 #----- ship management aid -----
1505
1506 class Display_dumb(ProgressPrintPercentage):
1507         def __init__(self):
1508                 ProgressPrintPercentage.__init__(self)
1509         def show(self, s):
1510                 print '\n\n', s;
1511         def realstart(self):
1512                 pass
1513
1514 class Display_overwrite(ProgressPrintPercentage):
1515         def __init__(self):
1516                 ProgressPrintPercentage.__init__(self)
1517
1518                 null = file('/dev/null','w')
1519                 curses.setupterm(fd=null.fileno())
1520
1521                 self._clear = curses.tigetstr('clear')
1522                 if not self._clear:
1523                         self._debug('missing clear!')
1524                         self.show = Display_dumb.show
1525                         return
1526
1527                 self._t = {'el':'', 'ed':''}
1528                 if not self._init_sophisticated():
1529                         for k in self._t.keys(): self._t[k] = ''
1530                         self._t['ho'] = self._clear
1531
1532         def _debug(self,m): debug('display overwrite: '+m)
1533
1534         def _init_sophisticated(self):
1535                 for k in self._t.keys():
1536                         s = curses.tigetstr(k)
1537                         self._t[k] = s
1538                 self._t['ho'] = curses.tigetstr('ho')
1539                 if not self._t['ho']:
1540                         cup = curses.tigetstr('cup')
1541                         self._t['ho'] = curses.tparm(cup,0,0)
1542                 missing = [k for k in self._t.keys() if not self._t[k]]
1543                 if missing:
1544                         self.debug('missing '+(' '.join(missing)))
1545                         return 0
1546                 return 1
1547
1548         def show(self, s):
1549                 w = sys.stdout.write
1550                 def wti(k): w(self._t[k])
1551
1552                 wti('ho')
1553                 nl = ''
1554                 for l in s.rstrip().split('\n'):
1555                         w(nl)
1556                         w(l)
1557                         wti('el')
1558                         nl = '\r\n'
1559                 wti('ed')
1560                 w(' ')
1561                 sys.stdout.flush()
1562
1563         def realstart(self):
1564                 sys.stdout.write(self._clear)
1565                 sys.stdout.flush()
1566
1567
1568 def do_ship_aid(args, bu):
1569         if opts.ship_duty is None: opts.ship_duty = True
1570
1571         displayer = globals()['Display_'+opts.display]()
1572
1573         (myself, track) = prep_chat_log(args, bu, progress=displayer)
1574
1575         displayer.realstart()
1576
1577         if os.isatty(0): kr_create = KeystrokeReader
1578         else: kr_create = DummyKeystrokeReader
1579
1580         try:
1581                 kreader = kr_create(0, 10)
1582                 ship_aid_core(myself, track, displayer, kreader)
1583         finally:
1584                 kreader.stop()
1585                 print '\n'
1586
1587 class KeyBasedSorter:
1588         def compar_key_pa(self, pa):
1589                 pi = pa.pirate_info()
1590                 if pi is None: return None
1591                 return self.compar_key(pi)
1592         def lsort_pa(self, l):
1593                 l.sort(key = self.compar_key_pa)
1594
1595 class NameSorter(KeyBasedSorter):
1596         def compar_key(self, pi): return pi.name
1597         def desc(self): return 'name'
1598
1599 class SkillSorter(NameSorter):
1600         def __init__(self, relevant):
1601                 self._want = frozenset(relevant.split('/'))
1602                 self._avoid = set()
1603                 for p in core_duty_puzzles:
1604                         if isinstance(p,basestring): self._avoid.add(p)
1605                         else: self._avoid |= set(p)
1606                 self._avoid -= self._want
1607                 self._desc = '%s' % relevant
1608
1609         def desc(self): return self._desc
1610
1611         def compar_key(self, pi):
1612                 best_want = max([
1613                         pi.standings.get(puz,-1)
1614                         for puz in self._want
1615                         ])
1616                 best_avoid = [
1617                         -pi.standings.get(puz,standing_limit)
1618                         for puz in self._avoid
1619                         ]
1620                 best_avoid.sort()
1621                 def negate(x): return -x
1622                 debug('compar_key %s bw=%s ba=%s' % (pi.name, `best_want`,
1623                         `best_avoid`))
1624                 return (-best_want, map(negate, best_avoid), pi.name)
1625
1626 def ship_aid_core(myself, track, displayer, kreader):
1627
1628         def find_vessel():
1629                 vn = track.vesselname()
1630                 if vn: return (vn, " on board the %s" % vn)
1631                 vn = track.lastvesselname()
1632                 if vn: return (vn, " ashore from the %s" % vn)
1633                 return (None, " not on a vessel")
1634
1635         def timeevent(t,e):
1636                 if t is None: return ' ' * 22
1637                 return " %-4s %-16s" % (format_time_interval(now - t),e)
1638
1639         displayer.show(track.myname() + find_vessel()[1] + '...')
1640
1641         rotate_nya = '/-\\'
1642
1643         sort = NameSorter()
1644         clicmd = None
1645         clierr = None
1646         cliexec = None
1647
1648         while True:
1649                 track.catchup()
1650                 now = time.time()
1651
1652                 (vn, vs) = find_vessel()
1653
1654                 s = ''
1655                 if cliexec is not None:
1656                         s += '...'
1657                 elif clierr is not None:
1658                         s += 'Error: '+clierr
1659                 elif clicmd is not None:
1660                         s += '/' + clicmd
1661                 else:
1662                         s = track.myname() + vs
1663                         s += " at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
1664                         s += kreader.info()
1665                 s += '\n'
1666
1667                 tbl_s = StringIO()
1668                 tbl = StandingsTable(tbl_s)
1669
1670                 aboard = track.aboard(vn)
1671                 sort.lsort_pa(aboard)
1672
1673                 jobbers = track.jobbers()
1674
1675                 if track.vesselname(): howmany = 'aboard: %2d' % len(aboard)
1676                 else: howmany = ''
1677
1678                 tbl.headings(howmany, '  sorted by '+sort.desc())
1679
1680                 last_jobber = None
1681
1682                 for pa in aboard + jobbers:
1683                         if pa.jobber != last_jobber:
1684                                 last_jobber = pa.jobber
1685                                 tbl.literalline('')
1686                                 tbl.literalline('jobbers '+last_jobber)
1687
1688                         pi = pa.pirate_info()
1689
1690                         xs = ''
1691                         if pa.gunner: xs += 'G '
1692                         else: xs += '  '
1693                         xs += timeevent(pa.last_time, pa.last_event)
1694                         xs += timeevent(pa.last_chat_time, pa.last_chat_chan)
1695
1696                         if pi is None:
1697                                 tbl.pirate_dummy(pa.name, rotate_nya[0], xs)
1698                         else:
1699                                 tbl.pirate(pi, xs)
1700
1701                 s += tbl_s.getvalue()
1702                 displayer.show(s)
1703                 tbl_s.close()
1704
1705                 if cliexec is not None:
1706                         clierr = track.local_command("/"+cliexec.strip())
1707                         cliexec = None
1708                         continue
1709
1710                 k = kreader.getch()
1711                 if k is None:
1712                         rotate_nya = rotate_nya[1:3] + rotate_nya[0]
1713                         continue
1714
1715                 if clierr is not None:
1716                         clierr = None
1717                         continue
1718
1719                 if clicmd is not None:
1720                         if k == '\r' or k == '\n':
1721                                 cliexec = clicmd
1722                                 clicmd = clicmdbase
1723                         elif k == '\e' and clicmd != "":
1724                                 clicmd = clicmdbase
1725                         elif k == '\33':
1726                                 clicmd = None
1727                         elif k == '\b' or k == '\177':
1728                                 clicmd = clicmd[ 0 : len(clicmd)-1 ]
1729                         else:
1730                                 clicmd += k
1731                         continue
1732
1733                 if k == 'q': break
1734                 elif k == 'g': sort = SkillSorter('Gunning')
1735                 elif k == 'c': sort = SkillSorter('Carpentry')
1736                 elif k == 's': sort = SkillSorter('Sailing/Rigging')
1737                 elif k == 'b': sort = SkillSorter('Bilging')
1738                 elif k == 'n': sort = SkillSorter('Navigating')
1739                 elif k == 'd': sort = SkillSorter('Battle Navigation')
1740                 elif k == 't': sort = SkillSorter('Treasure Haul')
1741                 elif k == 'a': sort = NameSorter()
1742                 elif k == '/': clicmdbase = ""; clicmd = clicmdbase
1743                 elif k == '+': clicmdbase = "a "; clicmd = clicmdbase
1744                 else: pass # unknown key command
1745
1746 #---------- individual keystroke input ----------
1747
1748 class DummyKeystrokeReader:
1749         def __init__(self,fd,timeout_dummy): pass
1750         def stop(self): pass
1751         def getch(self): sleep(1); return None
1752         def info(self): return ' [noninteractive]'
1753
1754 class KeystrokeReader(DummyKeystrokeReader):
1755         def __init__(self, fd, timeout_decisec=0):
1756                 self._fd = fd
1757                 self._saved = termios.tcgetattr(fd)
1758                 a = termios.tcgetattr(fd)
1759                 a[3] &= ~(termios.ECHO | termios.ECHONL |
1760                           termios.ICANON | termios.IEXTEN)
1761                 a[6][termios.VMIN] = 0
1762                 a[6][termios.VTIME] = timeout_decisec
1763                 termios.tcsetattr(fd, termios.TCSANOW, a)
1764         def stop(self):
1765                 termios.tcsetattr(self._fd, termios.TCSANOW, self._saved)
1766         def getch(self):
1767                 debug_flush()
1768                 byte = os.read(self._fd, 1)
1769                 if not len(byte): return None
1770                 return byte
1771         def info(self):
1772                 return ''
1773
1774 #---------- main program ----------
1775
1776 def main():
1777         global opts, fetcher, yppedia, progressreporter
1778
1779         pa = OptionParser(
1780 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
1781 actions:
1782  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
1783  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
1784  yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
1785  yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG
1786  yoweb-scrape [--ocean OCEAN ...] ocean|embargoes
1787  yoweb-scrape [--ocean OCEAN ...] embargoes-flag-of PIRATE
1788  yoweb-scrape [options] ship-aid CHAT-LOG  (must be .../PIRATE_OCEAN_chat-log*)
1789
1790 display modes (for --display) apply to ship-aid:
1791  --display=dumb       just print new information, scrolling the screen
1792  --display=overwrite  use cursor motion, selective clear, etc. to redraw at top''')
1793         ao = pa.add_option
1794         ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
1795                 help='select ocean OCEAN')
1796         ao('--cache-dir', dest='cache_dir', metavar='DIR',
1797                 default='~/.yoweb-scrape-cache',
1798                 help='cache yoweb pages in DIR')
1799         ao('-D','--debug', action='count', dest='debug', default=0,
1800                 help='enable debugging output')
1801         ao('--debug-fd', type='int', dest='debug_fd',
1802                 help='write any debugging output to specified fd')
1803         ao('-q','--quiet', action='store_true', dest='quiet',
1804                 help='suppress warning output')
1805         ao('--display', action='store', dest='display',
1806                 type='choice', choices=['dumb','overwrite'],
1807                 help='how to display ship aid')
1808         ao('--local-ypp-dir', action='store', dest='localhtml',
1809                 help='get yppedia pages from local directory LOCALHTML'+
1810                         ' instead of via HTTP')
1811
1812         ao_jt = lambda wh, t: ao(
1813                 '--timeout-sa-'+wh, action='store', dest='timeout_'+wh,
1814                 default=t, help=('set timeout for expiring %s jobbers' % wh))
1815         ao_jt('applied',      120)
1816         ao_jt('invited',      120)
1817         ao_jt('declined',      30)
1818         ao_jt('ashore',      1800)
1819
1820         ao('--ship-duty', action='store_true', dest='ship_duty',
1821                 help='show ship duty station puzzles')
1822         ao('--all-puzzles', action='store_false', dest='ship_duty',
1823                 help='show all puzzles, not just ship duty stations')
1824
1825         ao('--min-cache-reuse', type='int', dest='min_max_age',
1826                 metavar='SECONDS', default=60,
1827                 help='always reuse cache yoweb data if no older than this')
1828
1829         (opts,args) = pa.parse_args()
1830         random.seed()
1831
1832         if len(args) < 1:
1833                 print >>sys.stderr, copyright_info
1834                 pa.error('need a mode argument')
1835
1836         if opts.debug_fd is not None:
1837                 opts.debug_file = os.fdopen(opts.debug_fd, 'w')
1838         else:
1839                 opts.debug_file = sys.stdout
1840
1841         mode = args[0]
1842         mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
1843         try: mode_fn = globals()[mode_fn_name]
1844         except KeyError: pa.error('unknown mode "%s"' % mode)
1845
1846         # fixed parameters
1847         opts.expire_age = max(3600, opts.min_max_age)
1848
1849         opts.ship_reboard_clearout = 3600
1850
1851         if opts.cache_dir.startswith('~/'):
1852                 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
1853
1854         if opts.display is None:
1855                 if ((opts.debug > 0 and opts.debug_fd is None)
1856                     or not os.isatty(sys.stdout.fileno())):
1857                         opts.display = 'dumb'
1858                 else:
1859                         opts.display = 'overwrite'
1860
1861         fetcher = Yoweb(opts.ocean, opts.cache_dir)
1862         yppedia = Yppedia(opts.cache_dir)
1863
1864         if opts.debug or not os.isatty(0):
1865                  progressreporter = NullProgressReporter()
1866         else:
1867                 progressreporter = TypewriterProgressReporter()
1868
1869         mode_fn(args[1:], pa.error)
1870
1871 main()