yoweb-scrape

   1 #!/usr/bin/python
   2 # This is part of ypp-sc-tools, a set of third-party tools for assisting
   3 # players of Yohoho Puzzle Pirates.
   4 #
   5 # Copyright (C) 2009 Ian Jackson <ijackson@chiark.greenend.org.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 #
  20 # Yohoho and Puzzle Pirates are probably trademarks of Three Rings and
  21 # are used without permission.  This program is not endorsed or
  22 # sponsored by Three Rings.
  23
  24 copyright_info = '''
  25 yoweb-scrape is part of ypp-sc-tools  Copyright (C) 2009 Ian Jackson
  26 This program comes with ABSOLUTELY NO WARRANTY; this is free software,
  27 and you are welcome to redistribute it under certain conditions.
  28 For details, read the top of the yoweb-scrape file.
  29 '''
  30
  31 #---------- setup ----------
  32
  33 import signal
  34 signal.signal(signal.SIGINT, signal.SIG_DFL)
  35
  36 import os
  37 import time
  38 import urllib
  39 import urllib2
  40 import errno
  41 import sys
  42 import re as regexp
  43 import random
  44 import curses
  45 import termios
  46 import random
  47 import subprocess
  48 from optparse import OptionParser
  49 from StringIO import StringIO
  50
  51 from BeautifulSoup import BeautifulSoup
  52
  53 opts = None
  54
  55 #---------- YPP parameters and arrays ----------
  56
  57 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
  58         '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
  59         '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
  60         '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
  61
  62 core_duty_puzzles = [
  63                 'Gunning',
  64                 ['Sailing','Rigging'],
  65                 'Bilging',
  66                 'Carpentry',
  67                 ]
  68
  69 duty_puzzles = ([ 'Navigating', 'Battle Navigation' ] +
  70                 core_duty_puzzles +
  71                 [ 'Treasure Haul' ])
  72
  73 standingvals = ('Able/Proficient/Distinguished/Respected/Master'+
  74                 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
  75 standing_limit = len(standingvals)
  76
  77 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
  78
  79 max_pirate_namelen = 12
  80
  81
  82 #---------- general utilities ----------
  83
  84 def debug(m):
  85         if opts.debug > 0:
  86                 print >>opts.debug_file, m
  87
  88 def debug_flush():
  89         if opts.debug > 0:
  90                 opts.debug_file.flush()
  91
  92 def sleep(seconds):
  93         debug_flush()
  94         time.sleep(seconds)
  95
  96 def format_time_interval(ti):
  97         if ti < 120: return '%d:%02d' % (ti / 60, ti % 60)
  98         if ti < 7200: return '%2dm' % (ti / 60)
  99         if ti < 86400: return '%dh' % (ti / 3600)
 100         return '%dd' % (ti / 86400)
 101
 102 def yppsc_dir():
 103         lib = os.getenv("YPPSC_YARRG_SRCBASE")
 104         if lib is not None: return lib
 105         lib = sys.argv[0]
 106         lib = regexp.sub('/[^/]+$', '', lib)
 107         os.environ["YPPSC_YARRG_SRCBASE"] = lib
 108         return lib
 109
 110 #---------- caching and rate-limiting data fetcher ----------
 111
 112 class Fetcher:
 113         def __init__(self, ocean, cachedir):
 114                 debug('Fetcher init %s' % cachedir)
 115                 self.ocean = ocean
 116                 self.cachedir = cachedir
 117                 try: os.mkdir(cachedir)
 118                 except (OSError,IOError), oe:
 119                         if oe.errno != errno.EEXIST: raise
 120                 self._cache_scan(time.time())
 121
 122         def default_ocean(self, ocean='ice'):
 123                 if self.ocean is None:
 124                         self.ocean = ocean
 125
 126         def _cache_scan(self, now):
 127                 # returns list of ages, unsorted
 128                 ages = []
 129                 debug('Fetcher   scan_cache')
 130                 for leaf in os.listdir(self.cachedir):
 131                         if not leaf.startswith('#'): continue
 132                         path = self.cachedir + '/' + leaf
 133                         try: s = os.stat(path)
 134                         except (OSError,IOError), oe:
 135                                 if oe.errno != errno.ENOENT: raise
 136                                 continue
 137                         age = now - s.st_mtime
 138                         if age > opts.expire_age:
 139                                 debug('Fetcher    expire %d %s' % (age, path))
 140                                 try: os.remove(path)
 141                                 except (OSError,IOError), oe:
 142                                         if oe.errno != errno.ENOENT: raise
 143                                 continue
 144                         ages.append(age)
 145                 return ages
 146
 147         def need_wait(self, now, imaginary=[]):
 148                 ages = self._cache_scan(now)
 149                 ages += imaginary
 150                 ages.sort()
 151                 debug('Fetcher   ages ' + `ages`)
 152                 min_age = 1
 153                 need_wait = 0
 154                 for age in ages:
 155                         if age < min_age and age <= 5:
 156                                 debug('Fetcher   morewait min=%d age=%d' %
 157                                         (min_age, age))
 158                                 need_wait = max(need_wait, min_age - age)
 159                         min_age += 3
 160                         min_age *= 1.25
 161                 if need_wait > 0:
 162                         need_wait += random.random() - 0.5
 163                 return need_wait
 164
 165         def _rate_limit_cache_clean(self, now):
 166                 need_wait = self.need_wait(now)
 167                 if need_wait > 0:
 168                         debug('Fetcher   wait %d' % need_wait)
 169                         sleep(need_wait)
 170
 171         def fetch(self, url, max_age):
 172                 debug('Fetcher fetch %s' % url)
 173                 cache_corename = urllib.quote_plus(url)
 174                 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
 175                 try: f = file(cache_item, 'r')
 176                 except (OSError,IOError), oe:
 177                         if oe.errno != errno.ENOENT: raise
 178                         f = None
 179                 now = time.time()
 180                 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
 181                 if f is not None:
 182                         s = os.fstat(f.fileno())
 183                         age = now - s.st_mtime
 184                         if age > max_age:
 185                                 debug('Fetcher  stale %d < %d'% (max_age, age))
 186                                 f = None
 187                 if f is not None:
 188                         data = f.read()
 189                         f.close()
 190                         debug('Fetcher  cached %d > %d' % (max_age, age))
 191                         return data
 192
 193                 debug('Fetcher  fetch')
 194                 self._rate_limit_cache_clean(now)
 195
 196                 stream = urllib2.urlopen(url)
 197                 data = stream.read()
 198                 cache_tmp = "%s/#%s~%d#" % (
 199                         self.cachedir, cache_corename, os.getpid())
 200                 f = file(cache_tmp, 'w')
 201                 f.write(data)
 202                 f.close()
 203                 os.rename(cache_tmp, cache_item)
 204                 debug('Fetcher  stored')
 205                 return data
 206
 207         def yoweb(self, kind, tail, max_age):
 208                 self.default_ocean()
 209                 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
 210                         self.ocean, kind, tail)
 211                 return self.fetch(url, max_age)
 212
 213 #---------- logging assistance for troubled screenscrapers ----------
 214
 215 class SoupLog:
 216         def __init__(self):
 217                 self.msgs = [ ]
 218         def msg(self, m):
 219                 self.msgs.append(m)
 220         def soupm(self, obj, m):
 221                 self.msg(m + '; in ' + `obj`)
 222         def needs_msgs(self, child_souplog):
 223                 self.msgs += child_souplog.msgs
 224                 child_souplog.msgs = [ ]
 225
 226 def soup_text(obj):
 227         str = ''.join(obj.findAll(text=True))
 228         return str.strip()
 229
 230 class SomethingSoupInfo(SoupLog):
 231         def __init__(self, kind, tail, max_age):
 232                 SoupLog.__init__(self)
 233                 html = fetcher.yoweb(kind, tail, max_age)
 234                 self._soup = BeautifulSoup(html,
 235                         convertEntities=BeautifulSoup.HTML_ENTITIES
 236                         )
 237
 238 #---------- scraper for pirate pages ----------
 239
 240 class PirateInfo(SomethingSoupInfo):
 241         # Public data members:
 242         #  pi.standings = { 'Treasure Haul': 'Able' ... }
 243         #  pi.name = name
 244         #  pi.crew = (id, name)
 245         #  pi.flag = (id, name)
 246         #  pi.msgs = [ 'message describing problem with scrape' ]
 247
 248         def __init__(self, pirate, max_age=300):
 249                 SomethingSoupInfo.__init__(self,
 250                         'pirate.wm?target=', pirate, max_age)
 251                 self.name = pirate
 252                 self._find_standings()
 253                 self.crew = self._find_crewflag('crew',
 254                         '^/yoweb/crew/info\\.wm')
 255                 self.flag = self._find_crewflag('flag',
 256                         '^/yoweb/flag/info\\.wm')
 257
 258         def _find_standings(self):
 259                 imgs = self._soup.findAll('img',
 260                         src=regexp.compile('/yoweb/images/stat.*'))
 261                 re = regexp.compile(
 262 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
 263                         )
 264                 standings = { }
 265
 266                 for skill in puzzles:
 267                         standings[skill] = [ ]
 268
 269                 skl = SoupLog()
 270
 271                 for img in imgs:
 272                         try: puzzle = img['alt']
 273                         except KeyError: continue
 274
 275                         if not puzzle in puzzles:
 276                                 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
 277                                 continue
 278                         key = img.findParent('td')
 279                         if key is None:
 280                                 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
 281                                 continue
 282                         valelem = key.findNextSibling('td')
 283                         if valelem is None:
 284                                 skl.soupm(key, 'puzzle missing sibling "%s"'
 285                                         % puzzle)
 286                                 continue
 287                         valstr = soup_text(valelem)
 288                         match = re.match(valstr)
 289                         if match is None:
 290                                 skl.soupm(key, ('puzzle "%s" unparseable'+
 291                                         ' standing "%s"') % (puzzle, valstr))
 292                                 continue
 293                         standing = match.group(match.lastindex)
 294                         standings[puzzle].append(standing)
 295
 296                 self.standings = { }
 297
 298                 for puzzle in puzzles:
 299                         sl = standings[puzzle]
 300                         if len(sl) > 1:
 301                                 skl.msg('puzzle "%s" multiple standings %s' %
 302                                                 (puzzle, `sl`))
 303                                 continue
 304                         if not sl:
 305                                 skl.msg('puzzle "%s" no standing found' % puzzle)
 306                                 continue
 307                         standing = sl[0]
 308                         for i in range(0, standing_limit):
 309                                 if standing == standingvals[i]:
 310                                         self.standings[puzzle] = i
 311                         if not puzzle in self.standings:
 312                                 skl.msg('puzzle "%s" unknown standing "%s"' %
 313                                         (puzzle, standing))
 314
 315                 all_standings_ok = True
 316                 for puzzle in puzzles:
 317                         if not puzzle in self.standings:
 318                                 self.needs_msgs(skl)
 319
 320         def _find_crewflag(self, cf, yoweb_re):
 321                 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
 322                 if len(things) != 1:
 323                         self.msg('zero or several %s id references found' % cf)
 324                         return None
 325                 thing = things[0]
 326                 id_re = '\\b%sid\\=(\\w+)$' % cf
 327                 id_haystack = thing['href']
 328                 match = regexp.compile(id_re).search(id_haystack)
 329                 if match is None:
 330                         self.soupm(thing, ('incomprehensible %s id ref'+
 331                                 ' (%s in %s)') % (cf, id_re, id_haystack))
 332                         return None
 333                 name = soup_text(thing)
 334                 return (match.group(1), name)
 335
 336         def __str__(self):
 337                 return `(self.crew, self.flag, self.standings, self.msgs)`
 338
 339 #---------- scraper for crew pages ----------
 340
 341 class CrewInfo(SomethingSoupInfo):
 342         # Public data members:
 343         #  ci.crew = [ ('Captain',        ['Pirate', ...]),
 344         #              ('Senior Officer', [...]),
 345         #               ... ]
 346         #  pi.msgs = [ 'message describing problem with scrape' ]
 347
 348         def __init__(self, crewid, max_age=300):
 349                 SomethingSoupInfo.__init__(self,
 350                         'crew/info.wm?crewid=', crewid, max_age)
 351                 self._find_crew()
 352
 353         def _find_crew(self):
 354                 self.crew = []
 355                 capts = self._soup.findAll('img',
 356                         src='/yoweb/images/crew-captain.png')
 357                 if len(capts) != 1:
 358                         self.msg('crew members: no. of captain images != 1')
 359                         return
 360                 tbl = capts[0]
 361                 while not tbl.find('a', href=pirate_ref_re):
 362                         tbl = tbl.findParent('table')
 363                         if not tbl:
 364                                 self.msg('crew members: cannot find table')
 365                                 return
 366                 current_rank_crew = None
 367                 crew_rank_re = regexp.compile('/yoweb/images/crew')
 368                 for row in tbl.contents:
 369                         # findAll(recurse=False)
 370                         if isinstance(row,basestring):
 371                                 continue
 372
 373                         is_rank = row.find('img', attrs={'src': crew_rank_re})
 374                         if is_rank:
 375                                 rank = soup_text(row)
 376                                 current_rank_crew = []
 377                                 self.crew.append((rank, current_rank_crew))
 378                                 continue
 379                         for cell in row.findAll('a', href=pirate_ref_re):
 380                                 if current_rank_crew is None:
 381                                         self.soupm(cell, 'crew members: crew'
 382                                                 ' before rank')
 383                                         continue
 384                                 current_rank_crew.append(soup_text(cell))
 385
 386         def __str__(self):
 387                 return `(self.crew, self.msgs)`
 388
 389 class FlagInfo(SomethingSoupInfo):
 390         # Public data members (after init):
 391         #
 392         #   name        #               string
 393         #
 394         #   relations[n] = (otherflagname, otherflagid, [stringfromyoweb],
 395         #               thisdeclaring, otherdeclaringmin, otherdeclaringmax)
 396         #               # where {this,other}declaring{,min,max} are:
 397         #               #       -1      {this,other} is declaring war
 398         #               #        0      {this,other} is not doing either
 399         #               #       +1      {this,other} is allying
 400         #   relation_byname[otherflagname] = relations[some_n]
 401         #   relation_byid[otherflagname] = relations[some_n]
 402         #
 403         #   islands[n] = (islandname, islandid)
 404         #
 405         def __init__(self, flagid, max_age=600):
 406                 SomethingSoupInfo.__init__(self,
 407                         'flag/info.wm?flagid=', flagid, max_age)
 408                 self._find_flag()
 409
 410         def _find_flag(self):
 411                 font2 = self._soup.find('font',{'size':'+2'})
 412                 self.name = font2.find('b').contents[0]
 413
 414                 self.relations = [ ]
 415                 self.relation_byname = { }
 416                 self.relation_byid = { }
 417                 self.islands = [ ]
 418
 419                 magnate = self._soup.find('img',{'src':
 420                         '/yoweb/images/repute-MAGNATE.png'})
 421                 warinfo = (magnate.findParent('table').findParent('tr').
 422                         findNextSibling('tr').findNext('td',{'align':'left'}))
 423
 424                 def warn(m):
 425                         print >>sys.stderr, 'WARNING: '+m
 426
 427                 def wi_warn(head, waritem):
 428                         warn('unknown warmap item: %s: %s' %
 429                                 (`head`, ``waritem``))
 430
 431                 def wihelp_item(waritem, thing):
 432                         url = waritem.get('href', None)
 433                         if url is None:
 434                                 return ('no url for '+thing,None,None)
 435                         m = regexp.search('\?'+thing+'id=(\d+)$', url)
 436                         if not m: return ('no '+thing+'id',None,None)
 437                         tid = m.group(1)
 438                         tname = waritem.string
 439                         if tname is None:
 440                                 return (thing+' name not just string',None,None)
 441                         return (None,tid,tname)
 442
 443                 def wi_alwar(head, waritem, thisdecl, othermin, othermax):
 444                         (err,flagid,flagname) = wihelp_item(waritem,'flag')
 445                         if err: return err
 446                         rel = self.relation_byid.get(flagid, None)
 447                         if rel: return 'flag id twice!'
 448                         if flagname in self.relation_byname:
 449                                 return 'flag name twice!'
 450                         rel = (flagname,flagid,head, thisdecl,othermin,othermax)
 451                         self.relations.append(rel)
 452                         self.relation_byid[flagid] = rel
 453                         self.relation_byname[flagid] = rel
 454
 455                 def wi_isle(head, waritem):
 456                         (err,isleid,islename) = wihelp_item(waritem,'island')
 457                         if err: return err
 458                         self.islands.append((isleid,islename))
 459
 460                 warmap = {
 461                         'Allied with':                  (wi_alwar,+1,+1,+1),
 462                         'Declaring war against':        (wi_alwar,-1, 0,+1),
 463                         'At war with':                  (wi_alwar,-1,-1,-1),
 464                         'Trying to form an alliance with': (wi_alwar,+1,-1,0),
 465                         'Islands controlled by this flag': (wi_isle,),
 466                         }
 467
 468                 how = (wi_warn, None)
 469
 470                 for waritem in warinfo.findAll(['font','a']):
 471                         if waritem is None: break
 472                         if waritem.name == 'font':
 473                                 colour = waritem.get('color',None)
 474                                 if colour.lstrip('#') != '958A5F':
 475                                         warn('strange colour %s in %s' %
 476                                                 (colour,``waritem``))
 477                                         continue
 478                                 head = waritem.string
 479                                 if head is None:
 480                                         warn('no head string in '+``waritem``)
 481                                         continue
 482                                 head = regexp.sub('\\s+', ' ', head).strip()
 483                                 head = head.rstrip(':')
 484                                 how = (head,) + warmap.get(head, (wi_warn,))
 485                                 continue
 486                         assert(waritem.name == 'a')
 487
 488                         debug('WARHOW %s(%s, waritem, *%s)' %
 489                                 (how[1], `how[0]`, `how[2:]`))
 490                         bad = how[1](how[0], waritem, *how[2:])
 491                         if bad:
 492                                 warn('bad waritem %s: %s: %s' % (`how[0]`,
 493                                         bad, ``waritem``))
 494
 495         def __str__(self):
 496                 return `(self.name, self.islands, self.relations)`
 497
 498 #---------- scraper for ocean info incl. embargoes etc. ----------
 499
 500 class IslandInfo():
 501         def __init__(self, ocean, islename):
 502                 self.ocean = ocean
 503                 self.name = islename
 504         def collect(self):
 505                 pass
 506         def yppedia_dataf(self):
 507                 def q(x): return urllib.quote(x.replace(' ','_'))
 508                 url_rhs = q(self.name) + '_(' + q(self.ocean) + ')'
 509                 if opts.localhtml is None:
 510                         url = 'http://yppedia.puzzlepirates.com/' + url_rhs
 511                         debug('IslandInfo retrieving YPP '+url);
 512                         return urllib.urlopen(url)
 513                 else:
 514                         return file(opts.localhtml + '/' + url_rhs, 'r')
 515         def yoweb_url(self):
 516                 soup = BeautifulSoup(self.yppedia_dataf())
 517                 content = soup.find('div', attrs = {'id': 'content'})
 518                 yoweb_re = regexp.compile('^http://\w+\.puzzlepirates\.com/'+
 519                         'yoweb/island/info\.wm\?islandid=\d+$')
 520                 a = soup.find('a', attrs = { 'href': yoweb_re })
 521                 if a is None: return None
 522                 return a['href']
 523         def ruling_flag_id(self):
 524                 yo = self.yoweb_url()
 525                 if yo is None: return None
 526                 dataf = fetcher.fetch(yo, 600)
 527                 soup = BeautifulSoup(dataf)
 528                 ruler_re = regexp.compile('http://\w+\.puzzlepirates\.com/'+
 529                         'yoweb/flag/info\.wm\?flagid=(\d+)$')
 530                 ruler = soup.find('a', attrs = { 'href': ruler_re })
 531                 if not ruler: return None
 532                 m = ruler_re.find(ruler['href'])
 533                 return m.group(1)
 534
 535 class OceanInfo():
 536         # Public data attributes (valid after collect()):
 537         #   oi.islands[islename] = IslandInfo(...)
 538         #   oi.arches[archname][islename] = IslandInfo(...)
 539         def __init__(self):
 540                 self.isleclass = IslandInfo
 541                 self.ocean = fetcher.ocean.lower().capitalize()
 542         def collect(self):
 543                 cmdl = ['./yppedia-ocean-scraper']
 544                 if opts.localhtml is not None:
 545                         cmdl += ['--local-html-dir',opts.localhtml]
 546                 cmdl += [self.ocean]
 547                 debug('OceanInfo collect running ' + `cmdl`)
 548                 oscraper = subprocess.Popen(
 549                         cmdl,
 550                         stdout = subprocess.PIPE,
 551                         cwd = yppsc_dir()+'/yarrg',
 552                         shell=False, stderr=None,
 553                         )
 554                 h = oscraper.stdout.readline()
 555                 debug('OceanInfo collect h '+`h`)
 556                 assert(regexp.match('^ocean ', h))
 557                 arch_re = regexp.compile('^ (\S.*)')
 558                 island_re = regexp.compile('^  (\S.*)')
 559
 560                 self.islands = { }
 561                 self.arches = { }
 562                 archname = None
 563
 564                 for l in oscraper.stdout:
 565                         debug('OceanInfo collect l '+`l`)
 566                         l = l.rstrip('\n')
 567                         m = island_re.match(l)
 568                         if m:
 569                                 assert(archname is not None)
 570                                 islename = m.group(1)
 571                                 isle = self.isleclass(self.ocean, islename)
 572                                 isle.arch = archname
 573                                 self.islands[islename] = isle
 574                                 self.arches[archname][islename] = isle
 575                                 continue
 576                         m = arch_re.match(l)
 577                         if m:
 578                                 archname = m.group(1)
 579                                 assert(archname not in self.arches)
 580                                 self.arches[archname] = { }
 581                                 continue
 582                         assert(False)
 583                 oscraper.wait()
 584                 assert(oscraper.returncode == 0)
 585
 586 #---------- pretty-printer for tables of pirate puzzle standings ----------
 587
 588 class StandingsTable:
 589         def __init__(self, f, use_puzzles=None, col_width=6, gap_every=5):
 590                 if use_puzzles is None:
 591                         if opts.ship_duty:
 592                                 use_puzzles=duty_puzzles
 593                         else:
 594                                 use_puzzles=puzzles
 595                 self._puzzles = use_puzzles
 596                 self.f = f
 597                 self._cw = col_width-1
 598                 self._gap_every = gap_every
 599                 self._linecount = 0
 600                 self._o = f.write
 601
 602         def _nl(self): self._o('\n')
 603
 604         def _pline(self, lhs, puzstrs, extra):
 605                 if (self._linecount > 0
 606                     and self._gap_every is not None
 607                     and not (self._linecount % self._gap_every)):
 608                         self._nl()
 609                 self._o('%-*s' % (max(max_pirate_namelen+1, 15), lhs))
 610                 for v in puzstrs:
 611                         self._o(' %-*.*s' % (self._cw,self._cw, v))
 612                 if extra:
 613                         self._o(' ' + extra)
 614                 self._nl()
 615                 self._linecount += 1
 616
 617         def _puzstr(self, pi, puzzle):
 618                 if not isinstance(puzzle,list): puzzle = [puzzle]
 619                 try: standing = max([pi.standings[p] for p in puzzle])
 620                 except KeyError: return '?'
 621                 if not standing: return ''
 622                 s = ''
 623                 if self._cw > 4:
 624                         c1 = standingvals[standing][0]
 625                         if standing < 3: c1 = c1.lower() # 3 = Master
 626                         s += `standing`
 627                 if self._cw > 5:
 628                         s += ' '
 629                 s += '*' * (standing / 2)
 630                 s += '+' * (standing % 2)
 631                 return s
 632
 633         def headings(self, lhs='', rhs=None):
 634                 def puzn_redact(name):
 635                         if isinstance(name,list):
 636                                 return '/'.join(
 637                                         ["%.*s" % (self._cw/2, puzn_redact(n))
 638                                          for n in name])
 639                         spc = name.find(' ')
 640                         if spc < 0: return name
 641                         return name[0:min(4,spc)] + name[spc+1:]
 642                 self._linecount = -2
 643                 self._pline(lhs, map(puzn_redact, self._puzzles), rhs)
 644                 self._linecount = 0
 645         def literalline(self, line):
 646                 self._o(line)
 647                 self._nl()
 648                 self._linecount = 0
 649         def pirate_dummy(self, name, standingstring, extra=None):
 650                 standings = standingstring * len(self._puzzles)
 651                 self._pline(' '+name, standings, extra)
 652         def pirate(self, pi, extra=None):
 653                 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
 654                 self._pline(' '+pi.name, puzstrs, extra)
 655
 656
 657 #---------- chat log parser ----------
 658
 659 class PirateAboard:
 660         # This is essentially a transparent, dumb, data class.
 661         #  pa.v                 may be None
 662         #  pa.name
 663         #  pa.last_time
 664         #  pa.last_event
 665         #  pa.gunner
 666         #  pa.last_chat_time
 667         #  pa.last_chat_chan
 668         #  pa.pi
 669
 670         # Also used for jobbing applicants:
 671         #               happens when                    expires (to "-")
 672         #   -            disembark, leaves crew          no
 673         #   aboard       evidence of them being aboard   no
 674         #   applied      "has applied for the job"       120s, configurable
 675         #   ashore       "has taken a job"               30min, configurable
 676         #   declined     "declined the job offer"        30s, configurable
 677         #   invited      "has been invited to job"       120s, configurable
 678         #
 679         #  pa.jobber    None, 'ashore', 'applied', 'invited', 'declined'
 680         #  pa.expires   expiry time time
 681
 682         def __init__(pa, pn, v, time, event):
 683                 pa.name = pn
 684                 pa.v = v
 685                 pa.last_time = time
 686                 pa.last_event = event
 687                 pa.last_chat_time = None
 688                 pa.last_chat_chan = None
 689                 pa.gunner = False
 690                 pa.pi = None
 691                 pa.jobber = None
 692                 pa.expires = None
 693
 694         def pirate_info(pa):
 695                 now = time.time()
 696                 if pa.pi:
 697                         age = now - pa.pi_fetched
 698                         guide = random.randint(120,240)
 699                         if age <= guide:
 700                                 return pa.pi
 701                         debug('PirateAboard refresh %d > %d  %s' % (
 702                                 age, guide, pa.name))
 703                         imaginary = [2,4]
 704                 else:
 705                         imaginary = [1]
 706                 wait = fetcher.need_wait(now, imaginary)
 707                 if wait:
 708                         debug('PirateAboard fetcher not ready %d' % wait)
 709                         return pa.pi
 710                 pa.pi = PirateInfo(pa.name, 600)
 711                 pa.pi_fetched = now
 712                 return pa.pi
 713
 714 class ChatLogTracker:
 715         # This is quite complex so we make it opaque.  Use the
 716         # official invokers, accessors etc.
 717
 718         def __init__(self, myself_pi, logfn):
 719                 self._pl = {}   # self._pl['Pirate'] =
 720                 self._vl = {}   #   self._vl['Vessel']['Pirate'] = PirateAboard
 721                                 # self._vl['Vessel']['#lastinfo']
 722                                 # self._vl['Vessel']['#name']
 723                                 # self._v = self._vl[self._vessel]
 724                 self._date = None
 725                 self._myself = myself_pi
 726                 self._lbuf = ''
 727                 self._f = file(logfn)
 728                 flen = os.fstat(self._f.fileno()).st_size
 729                 max_backlog = 500000
 730                 if flen > max_backlog:
 731                         startpos = flen - max_backlog
 732                         self._f.seek(startpos)
 733                         self._f.readline()
 734                 self._progress = [0, flen - self._f.tell()]
 735                 self._disembark_myself()
 736                 self._need_redisplay = False
 737                 self._lastvessel = None
 738
 739         def _disembark_myself(self):
 740                 self._v = None
 741                 self._vessel = None
 742                 self.force_redisplay()
 743
 744         def force_redisplay(self):
 745                 self._need_redisplay = True
 746
 747         def _vessel_updated(self, v, timestamp):
 748                 if v is None: return
 749                 v['#lastinfo'] = timestamp
 750                 self.force_redisplay()
 751
 752         def _onboard_event(self,v,timestamp,pirate,event,jobber=None):
 753                 pa = self._pl.get(pirate, None)
 754                 if pa is not None and pa.v is v:
 755                         pa.last_time = timestamp
 756                         pa.last_event = event
 757                 else:
 758                         if pa is not None and pa.v is not None:
 759                                 del pa.v[pirate]
 760                         pa = PirateAboard(pirate, v, timestamp, event)
 761                         self._pl[pirate] = pa
 762                         if v is not None: v[pirate] = pa
 763                 pa.jobber = jobber
 764
 765                 if jobber is None: timeout = None
 766                 else: timeout = getattr(opts, 'timeout_'+jobber)
 767                 if timeout is None: pa.expires = None
 768                 else: pa.expires = timestamp + timeout
 769                 self._vessel_updated(v, timestamp)
 770                 return pa
 771
 772         def _expire_jobbers(self, now):
 773                 for pa in self._pl.values():
 774                         if pa.expires is None: continue
 775                         if pa.expires >= now: continue
 776                         v = pa.v
 777                         del self._pl[pa.name]
 778                         if v is not None: del v[pa.name]
 779                         self.force_redisplay()
 780
 781         def _trash_vessel(self, v):
 782                 for pn in v:
 783                         if pn.startswith('#'): continue
 784                         del self._pl[pn]
 785                 vn = v['#name']
 786                 del self._vl[vn]
 787                 if v is self._v: self._disembark_myself()
 788                 self.force_redisplay()
 789
 790         def _vessel_stale(self, v, timestamp):
 791                 return timestamp - v['#lastinfo'] > opts.ship_reboard_clearout
 792
 793         def _vessel_check_expire(self, v, timestamp):
 794                 if not self._vessel_stale(v, timestamp):
 795                         return v
 796                 self._debug_line_disposition(timestamp,'',
 797                         'stale-reset ' + v['#name'])
 798                 self._trash_vessel(v)
 799                 return None
 800
 801         def expire_garbage(self, timestamp):
 802                 for v in self._vl.values():
 803                         self._vessel_check_expire(v, timestamp)
 804
 805         def _vessel_lookup(self, vn, timestamp, dml=[], create=False):
 806                 v = self._vl.get(vn, None)
 807                 if v is not None:
 808                         v = self._vessel_check_expire(v, timestamp)
 809                 if v is not None:
 810                         dml.append('found')
 811                         return v
 812                 if not create:
 813                         dml.append('no')
 814                 dml.append('new')
 815                 self._vl[vn] = v = { '#name': vn }
 816                 self._vessel_updated(v, timestamp)
 817                 return v
 818
 819         def _find_matching_vessel(self, pattern, timestamp, cmdr,
 820                                         dml=[], create=False):
 821                 # use when a commander pirate `cmdr' specified a vessel
 822                 #  by name `pattern' (either may be None)
 823                 # if create is true, will create the vessel
 824                 #  record if an exact name is specified
 825
 826                 if (pattern is not None and
 827                     not '*' in pattern
 828                     and len(pattern.split(' ')) == 2):
 829                         vn = pattern.title()
 830                         dml.append('exact')
 831                         return self._vessel_lookup(
 832                                 vn, timestamp, dml=dml, create=create)
 833
 834                 if pattern is None:
 835                         pattern_check = lambda vn: True
 836                 else:
 837                         re = '(?:.* )?%s$' % pattern.lower().replace('*','.+')
 838                         pattern_check = regexp.compile(re, regexp.I).match
 839
 840                 tries = []
 841
 842                 cmdr_pa = self._pl.get(cmdr, None)
 843                 if cmdr_pa: tries.append((cmdr_pa.v, 'cmdr'))
 844
 845                 tries.append((self._v, 'here'))
 846                 tried_vns = []
 847
 848                 for (v, dm) in tries:
 849                         if v is None: dml.append(dm+'?'); continue
 850
 851                         vn = v['#name']
 852                         if not pattern_check(vn):
 853                                 tried_vns.append(vn)
 854                                 dml.append(dm+'#')
 855                                 continue
 856
 857                         dml.append(dm+'!')
 858                         return v
 859
 860                 if pattern is not None and '*' in pattern:
 861                         search = [
 862                                 (vn,v)
 863                                 for (vn,v) in self._vl.iteritems()
 864                                 if not self._vessel_stale(v, timestamp)
 865                                 if pattern_check(vn)
 866                                 ]
 867                         #debug('CLT-RE /%s/ wanted (%s) searched (%s)' % (
 868                         #       re,
 869                         #       '/'.join(tried_vns),
 870                         #       '/'.join([vn for (vn,v) in search])))
 871
 872                         if len(search)==1:
 873                                 dml.append('one')
 874                                 return search[0][1]
 875                         elif search:
 876                                 dml.append('many')
 877                         else:
 878                                 dml.append('none')
 879
 880         def _debug_line_disposition(self,timestamp,l,m):
 881                 debug('CLT %13s %-40s %s' % (timestamp,m,l))
 882
 883         def _rm_crew_l(self,re,l):
 884                 m = regexp.match(re,l)
 885                 if m and m.group(2) == self._myself.crew[1]:
 886                         return m.group(1)
 887                 else:
 888                         return None
 889
 890         def local_command(self, metacmd):
 891                 # returns None if all went well, or problem message
 892                 return self._command(self._myself.name, metacmd,
 893                         "local", time.time(),
 894                         (lambda m: debug('CMD %s' % metacmd)))
 895
 896         def _command(self, cmdr, metacmd, chan, timestamp, d):
 897                 # returns None if all went well, or problem message
 898                 metacmd = regexp.sub('\\s+', ' ', metacmd).strip()
 899                 m2 = regexp.match(
 900                     '/([adj]) (?:([A-Za-z* ]+)\\s*:)?([A-Za-z ]+)$',
 901                     metacmd)
 902                 if not m2: return "unknown syntax or command"
 903
 904                 (cmd, pattern, targets) = m2.groups()
 905                 dml = ['cmd', chan, cmd]
 906
 907                 if cmd == 'a': each = self._onboard_event
 908                 elif cmd == 'd': each = disembark
 909                 else: each = lambda *l: self._onboard_event(*l,
 910                                 **{'jobber':'applied'})
 911
 912                 if cmdr == self._myself.name:
 913                         dml.append('self')
 914                         how = 'cmd: %s' % cmd
 915                 else:
 916                         dml.append('other')
 917                         how = 'cmd: %s %s' % (cmd,cmdr)
 918
 919                 if cmd == 'j':
 920                         if pattern is not None:
 921                                 return "/j command does not take a vessel"
 922                         v = None
 923                 else:
 924                         v = self._find_matching_vessel(
 925                                 pattern, timestamp, cmdr,
 926                                 dml, create=True)
 927
 928                 if cmd == 'j' or v is not None:
 929                         targets = targets.strip().split(' ')
 930                         dml.append(`len(targets)`)
 931                         for target in targets:
 932                                 each(v, timestamp, target.title(), how)
 933                         self._vessel_updated(v, timestamp)
 934
 935                 dm = ' '.join(dml)
 936                 return d(dm)
 937
 938                 return None
 939
 940         def chatline(self,l):
 941                 rm = lambda re: regexp.match(re,l)
 942                 d = lambda m: self._debug_line_disposition(timestamp,l,m)
 943                 rm_crew = lambda re: self._rm_crew_l(re,l)
 944                 timestamp = None
 945
 946                 m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$')
 947                 if m:
 948                         self._date = [int(x) for x in m.groups()]
 949                         self._previous_timestamp = None
 950                         return d('date '+`self._date`)
 951
 952                 if self._date is None:
 953                         return d('date unset')
 954
 955                 m = rm('\\[(\d\d):(\d\d):(\d\d)\\] ')
 956                 if not m:
 957                         return d('no timestamp')
 958
 959                 while True:
 960                         time_tuple = (self._date +
 961                                       [int(x) for x in m.groups()] +
 962                                       [-1,-1,-1])
 963                         timestamp = time.mktime(time_tuple)
 964                         if timestamp >= self._previous_timestamp: break
 965                         self._date[2] += 1
 966                         self._debug_line_disposition(timestamp,'',
 967                                 'new date '+`self._date`)
 968
 969                 self._previous_timestamp = timestamp
 970
 971                 l = l[l.find(' ')+1:]
 972
 973                 def ob_x(pirate,event):
 974                         return self._onboard_event(
 975                                         self._v, timestamp, pirate, event)
 976                 def ob1(did): ob_x(m.group(1), did); return d(did)
 977                 def oba(did): return ob1('%s %s' % (did, m.group(2)))
 978
 979                 def jb(pirate,jobber):
 980                         return self._onboard_event(
 981                                 None, timestamp, pirate,
 982                                 ("jobber %s" % jobber),
 983                                 jobber=jobber
 984                                 )
 985
 986                 def disembark(v, timestamp, pirate, event):
 987                         self._onboard_event(
 988                                         v, timestamp, pirate, 'leaving '+event)
 989                         del v[pirate]
 990                         del self._pl[pirate]
 991
 992                 def disembark_me(why):
 993                         self._disembark_myself()
 994                         return d('disembark-me '+why)
 995
 996                 m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
 997                 if m:
 998                         dm = ['boarding']
 999                         pn = self._myself.name
1000                         vn = m.group(1)
1001                         v = self._vessel_lookup(vn, timestamp, dm, create=True)
1002                         self._lastvessel = self._vessel = vn
1003                         self._v = v
1004                         ob_x(pn, 'we boarded')
1005                         self.expire_garbage(timestamp)
1006                         return d(' '.join(dm))
1007
1008                 if self._v is None:
1009                         return d('no vessel')
1010
1011                 m = rm('(\\w+) has come aboard\\.$')
1012                 if m: return ob1('boarded');
1013
1014                 m = rm('You have ordered (\\w+) to do some (\\S.*\\S)\\.$')
1015                 if m:
1016                         (who,what) = m.groups()
1017                         pa = ob_x(who,'ord '+what)
1018                         if what == 'Gunning':
1019                                 pa.gunner = True
1020                         return d('duty order')
1021
1022                 m = rm('(\\w+) abandoned a (\\S.*\\S) station\\.$')
1023                 if m: oba('stopped'); return d("end")
1024
1025                 def chat_core(speaker, chan):
1026                         try: pa = self._pl[speaker]
1027                         except KeyError: return 'mystery'
1028                         if pa.v is not None and pa.v is not self._v:
1029                                 return 'elsewhere'
1030                         pa.last_chat_time = timestamp
1031                         pa.last_chat_chan = chan
1032                         self.force_redisplay()
1033                         return 'here'
1034
1035                 def chat(chan):
1036                         speaker = m.group(1)
1037                         dm = chat_core(speaker, chan)
1038                         return d('chat %s %s' % (chan, dm))
1039
1040                 def chat_metacmd(chan):
1041                         (cmdr, metacmd) = m.groups()
1042                         whynot = self._command(
1043                                 cmdr, metacmd, chan, timestamp, d)
1044                         if whynot is not None:
1045                                 return chat(chan)
1046                         else:
1047                                 chat_core(cmdr, 'cmd '+chan)
1048
1049                 m = rm('(\\w+) (?:issued an order|ordered everyone) "')
1050                 if m: return ob1('general order');
1051
1052                 m = rm('(\\w+) says, "')
1053                 if m: return chat('public')
1054
1055                 m = rm('(\\w+) tells ye, "')
1056                 if m: return chat('private')
1057
1058                 m = rm('Ye told (\\w+), "(.*)"$')
1059                 if m: return chat_metacmd('private')
1060
1061                 m = rm('(\\w+) flag officer chats, "')
1062                 if m: return chat('flag officer')
1063
1064                 m = rm('(\\w+) officer chats, "(.*)"$')
1065                 if m: return chat_metacmd('officer')
1066
1067                 m = rm('Ye accepted the offer to job with ')
1068                 if m: return disembark_me('jobbing')
1069
1070                 m = rm('Ye hop on the ferry and are whisked away ')
1071                 if m: return disembark_me('ferry')
1072
1073                 m = rm('Whisking away to yer home on the magical winds')
1074                 if m: return disembark_me('home')
1075
1076                 m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
1077                 if m:
1078                         pl = m.group(1).split(', ')
1079                         if not self._myself.name in pl:
1080                                 return d('lost melee')
1081                         for pn in pl:
1082                                 if ' ' in pn: continue
1083                                 ob_x(pn,'won melee')
1084                         return d('won melee')
1085
1086                 m = rm('(\\w+) is eliminated\\!')
1087                 if m: return ob1('eliminated in fray');
1088
1089                 m = rm('(\\w+) has driven \w+ from the ship\\!')
1090                 if m: return ob1('boarder repelled');
1091
1092                 m = rm('\w+ has bested (\\w+), and turns'+
1093                         ' to the rest of the ship\\.')
1094                 if m: return ob1('boarder unrepelled');
1095
1096                 pirate = rm_crew("(\\w+) has taken a job with '(.*)'\\.")
1097                 if pirate: return jb(pirate, 'ashore')
1098
1099                 pirate = rm_crew("(\\w+) has left '(.*)'\\.")
1100                 if pirate:
1101                         disembark(self._v, timestamp, pirate, 'left crew')
1102                         return d('left crew')
1103
1104                 m = rm('(\w+) has applied for the posted job\.')
1105                 if m: return jb(m.group(1), 'applied')
1106
1107                 pirate= rm_crew("(\\w+) has been invited to job for '(.*)'\\.")
1108                 if pirate: return jb(pirate, 'invited')
1109
1110                 pirate = rm_crew("(\\w+) declined the job offer for '(.*)'\\.")
1111                 if pirate: return jb(pirate, 'declined')
1112
1113                 m = rm('(\\w+) has left the vessel\.')
1114                 if m:
1115                         pirate = m.group(1)
1116                         disembark(self._v, timestamp, pirate, 'disembarked')
1117                         return d('disembarked')
1118
1119                 return d('not-matched')
1120
1121         def _str_pa(self, pn, pa):
1122                 assert self._pl[pn] == pa
1123                 s = ' '*20 + "%s %-*s %13s %-30s %13s %-20s %13s" % (
1124                         (' ','G')[pa.gunner],
1125                         max_pirate_namelen, pn,
1126                         pa.last_time, pa.last_event,
1127                         pa.last_chat_time, pa.last_chat_chan,
1128                         pa.jobber)
1129                 if pa.expires is not None:
1130                         s += " %-5d" % (pa.expires - pa.last_time)
1131                 s += "\n"
1132                 return s
1133
1134         def _str_vessel(self, vn, v):
1135                 s = ' vessel %s\n' % vn
1136                 s += ' '*20 + "%-*s   %13s\n" % (
1137                                 max_pirate_namelen, '#lastinfo',
1138                                 v['#lastinfo'])
1139                 assert v['#name'] == vn
1140                 for pn in sorted(v.keys()):
1141                         if pn.startswith('#'): continue
1142                         pa = v[pn]
1143                         assert pa.v == v
1144                         s += self._str_pa(pn,pa)
1145                 return s
1146
1147         def __str__(self):
1148                 s = '''<ChatLogTracker
1149  myself %s
1150  vessel %s
1151 '''                     % (self._myself.name, self._vessel)
1152                 assert ((self._v is None and self._vessel is None) or
1153                         (self._v is self._vl[self._vessel]))
1154                 if self._vessel is not None:
1155                         s += self._str_vessel(self._vessel, self._v)
1156                 for vn in sorted(self._vl.keys()):
1157                         if vn == self._vessel: continue
1158                         s += self._str_vessel(vn, self._vl[vn])
1159                 s += " elsewhere\n"
1160                 for p in self._pl:
1161                         pa = self._pl[p]
1162                         if pa.v is not None:
1163                                 assert pa.v[p] is pa
1164                                 assert pa.v in self._vl.values()
1165                         else:
1166                                 s += self._str_pa(pa.name, pa)
1167                 s += '>\n'
1168                 return s
1169
1170         def catchup(self, progress=None):
1171                 while True:
1172                         more = self._f.readline()
1173                         if not more: break
1174
1175                         self._progress[0] += len(more)
1176                         if progress: progress.progress(*self._progress)
1177
1178                         self._lbuf += more
1179                         if self._lbuf.endswith('\n'):
1180                                 self.chatline(self._lbuf.rstrip())
1181                                 self._lbuf = ''
1182                                 if opts.debug >= 2:
1183                                         debug(self.__str__())
1184                 self._expire_jobbers(time.time())
1185
1186                 if progress: progress.caughtup()
1187
1188         def changed(self):
1189                 rv = self._need_redisplay
1190                 self._need_redisplay = False
1191                 return rv
1192         def myname(self):
1193                 # returns our pirate name
1194                 return self._myself.name
1195         def vesselname(self):
1196                 # returns the vessel name we're aboard or None
1197                 return self._vessel
1198         def lastvesselname(self):
1199                 # returns the last vessel name we were aboard or None
1200                 return self._lastvessel
1201         def aboard(self, vesselname=True):
1202                 # returns a list of PirateAboard the vessel
1203                 #  sorted by pirate name
1204                 #  you can pass this None and you'll get []
1205                 #  or True for the current vessel (which is the default)
1206                 #  the returned value is a fresh list of persistent
1207                 #  PirateAboard objects
1208                 if vesselname is True: v = self._v
1209                 else: v = self._vl.get(vesselname.title())
1210                 if v is None: return []
1211                 return [ v[pn]
1212                          for pn in sorted(v.keys())
1213                          if not pn.startswith('#') ]
1214         def jobbers(self):
1215                 # returns a the jobbers' PirateAboards,
1216                 # sorted by jobber class and reverse of expiry time
1217                 l = [ pa
1218                       for pa in self._pl.values()
1219                       if pa.jobber is not None
1220                     ]
1221                 def compar_key(pa):
1222                         return (pa.jobber, -pa.expires)
1223                 l.sort(key = compar_key)
1224                 return l
1225
1226 #---------- implementations of actual operation modes ----------
1227
1228 def do_pirate(pirates, bu):
1229         print '{'
1230         for pirate in pirates:
1231                 info = PirateInfo(pirate)
1232                 print '%s: %s,' % (`pirate`, info)
1233         print '}'
1234
1235 def prep_crew_of(args, bu, max_age=300):
1236         if len(args) != 1: bu('crew-of takes one pirate name')
1237         pi = PirateInfo(args[0], max_age)
1238         if pi.crew is None: return None
1239         return CrewInfo(pi.crew[0], max_age)
1240
1241 def do_crew_of(args, bu):
1242         ci = prep_crew_of(args, bu)
1243         print ci
1244
1245 def do_flag_of(args, bu):
1246         if len(args) != 1: bu('flag-of takes one pirate name')
1247         max_age = 300
1248         pi = PirateInfo(args[0], max_age)
1249         if pi.flag is None: fi = None
1250         else: fi = FlagInfo(pi.flag[0], max_age)
1251         print fi
1252
1253 def do_standings_crew_of(args, bu):
1254         ci = prep_crew_of(args, bu, 60)
1255         tab = StandingsTable(sys.stdout)
1256         tab.headings()
1257         for (rank, members) in ci.crew:
1258                 if not members: continue
1259                 tab.literalline('')
1260                 tab.literalline('%s:' % rank)
1261                 for p in members:
1262                         pi = PirateInfo(p, random.randint(900,1800))
1263                         tab.pirate(pi)
1264
1265 def do_ocean(args, bu):
1266         if (len(args)): bu('ocean takes no further arguments')
1267         fetcher.default_ocean()
1268         oi = OceanInfo()
1269         oi.collect()
1270         for islename in sorted(oi.islands.keys()):
1271                 isle = oi.islands[islename]
1272                 yoweb_url = isle.yoweb_url()
1273                 print " %s -- %s" % (islename, yoweb_url)
1274
1275 #----- modes which use the chat log parser are quite complex -----
1276
1277 class ProgressPrintPercentage:
1278         def __init__(self, f=sys.stdout):
1279                 self._f = f
1280         def progress_string(self,done,total):
1281                 return "scan chat logs %3d%%\r" % ((done*100) / total)
1282         def progress(self,*a):
1283                 self._f.write(self.progress_string(*a))
1284                 self._f.flush()
1285         def show_init(self, pirate, ocean):
1286                 print >>self._f, 'Starting up, %s on the %s ocean' % (
1287                         pirate, ocean)
1288         def caughtup(self):
1289                 self._f.write('                   \r')
1290                 self._f.flush()
1291
1292 def prep_chat_log(args, bu,
1293                 progress=ProgressPrintPercentage(),
1294                 max_myself_age=3600):
1295         if len(args) != 1: bu('this action takes only chat log filename')
1296         logfn = args[0]
1297         logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_'
1298         match = regexp.match(logfn_re, logfn)
1299         if not match: bu('chat log filename is not in expected format')
1300         (pirate, ocean) = match.groups()
1301         fetcher.default_ocean(ocean)
1302
1303         progress.show_init(pirate, fetcher.ocean)
1304         myself = PirateInfo(pirate,max_myself_age)
1305         track = ChatLogTracker(myself, logfn)
1306
1307         opts.debug -= 2
1308         track.catchup(progress)
1309         opts.debug += 2
1310
1311         track.force_redisplay()
1312
1313         return (myself, track)
1314
1315 def do_track_chat_log(args, bu):
1316         (myself, track) = prep_chat_log(args, bu)
1317         while True:
1318                 track.catchup()
1319                 if track.changed():
1320                         print track
1321                 sleep(0.5 + 0.5 * random.random())
1322
1323 #----- ship management aid -----
1324
1325 class Display_dumb(ProgressPrintPercentage):
1326         def __init__(self):
1327                 ProgressPrintPercentage.__init__(self)
1328         def show(self, s):
1329                 print '\n\n', s;
1330         def realstart(self):
1331                 pass
1332
1333 class Display_overwrite(ProgressPrintPercentage):
1334         def __init__(self):
1335                 ProgressPrintPercentage.__init__(self)
1336
1337                 null = file('/dev/null','w')
1338                 curses.setupterm(fd=null.fileno())
1339
1340                 self._clear = curses.tigetstr('clear')
1341                 if not self._clear:
1342                         self._debug('missing clear!')
1343                         self.show = Display_dumb.show
1344                         return
1345
1346                 self._t = {'el':'', 'ed':''}
1347                 if not self._init_sophisticated():
1348                         for k in self._t.keys(): self._t[k] = ''
1349                         self._t['ho'] = self._clear
1350
1351         def _debug(self,m): debug('display overwrite: '+m)
1352
1353         def _init_sophisticated(self):
1354                 for k in self._t.keys():
1355                         s = curses.tigetstr(k)
1356                         self._t[k] = s
1357                 self._t['ho'] = curses.tigetstr('ho')
1358                 if not self._t['ho']:
1359                         cup = curses.tigetstr('cup')
1360                         self._t['ho'] = curses.tparm(cup,0,0)
1361                 missing = [k for k in self._t.keys() if not self._t[k]]
1362                 if missing:
1363                         self.debug('missing '+(' '.join(missing)))
1364                         return 0
1365                 return 1
1366
1367         def show(self, s):
1368                 w = sys.stdout.write
1369                 def wti(k): w(self._t[k])
1370
1371                 wti('ho')
1372                 nl = ''
1373                 for l in s.rstrip().split('\n'):
1374                         w(nl)
1375                         w(l)
1376                         wti('el')
1377                         nl = '\r\n'
1378                 wti('ed')
1379                 w(' ')
1380                 sys.stdout.flush()
1381
1382         def realstart(self):
1383                 sys.stdout.write(self._clear)
1384                 sys.stdout.flush()
1385
1386
1387 def do_ship_aid(args, bu):
1388         if opts.ship_duty is None: opts.ship_duty = True
1389
1390         displayer = globals()['Display_'+opts.display]()
1391
1392         (myself, track) = prep_chat_log(args, bu, progress=displayer)
1393
1394         displayer.realstart()
1395
1396         if os.isatty(0): kr_create = KeystrokeReader
1397         else: kr_create = DummyKeystrokeReader
1398
1399         try:
1400                 kreader = kr_create(0, 10)
1401                 ship_aid_core(myself, track, displayer, kreader)
1402         finally:
1403                 kreader.stop()
1404                 print '\n'
1405
1406 class KeyBasedSorter:
1407         def compar_key_pa(self, pa):
1408                 pi = pa.pirate_info()
1409                 if pi is None: return None
1410                 return self.compar_key(pi)
1411         def lsort_pa(self, l):
1412                 l.sort(key = self.compar_key_pa)
1413
1414 class NameSorter(KeyBasedSorter):
1415         def compar_key(self, pi): return pi.name
1416         def desc(self): return 'name'
1417
1418 class SkillSorter(NameSorter):
1419         def __init__(self, relevant):
1420                 self._want = frozenset(relevant.split('/'))
1421                 self._avoid = set()
1422                 for p in core_duty_puzzles:
1423                         if isinstance(p,basestring): self._avoid.add(p)
1424                         else: self._avoid |= set(p)
1425                 self._avoid -= self._want
1426                 self._desc = '%s' % relevant
1427
1428         def desc(self): return self._desc
1429
1430         def compar_key(self, pi):
1431                 best_want = max([
1432                         pi.standings.get(puz,-1)
1433                         for puz in self._want
1434                         ])
1435                 best_avoid = [
1436                         -pi.standings.get(puz,standing_limit)
1437                         for puz in self._avoid
1438                         ]
1439                 best_avoid.sort()
1440                 def negate(x): return -x
1441                 debug('compar_key %s bw=%s ba=%s' % (pi.name, `best_want`,
1442                         `best_avoid`))
1443                 return (-best_want, map(negate, best_avoid), pi.name)
1444
1445 def ship_aid_core(myself, track, displayer, kreader):
1446
1447         def find_vessel():
1448                 vn = track.vesselname()
1449                 if vn: return (vn, " on board the %s" % vn)
1450                 vn = track.lastvesselname()
1451                 if vn: return (vn, " ashore from the %s" % vn)
1452                 return (None, " not on a vessel")
1453
1454         def timeevent(t,e):
1455                 if t is None: return ' ' * 22
1456                 return " %-4s %-16s" % (format_time_interval(now - t),e)
1457
1458         displayer.show(track.myname() + find_vessel()[1] + '...')
1459
1460         rotate_nya = '/-\\'
1461
1462         sort = NameSorter()
1463         clicmd = None
1464         clierr = None
1465         cliexec = None
1466
1467         while True:
1468                 track.catchup()
1469                 now = time.time()
1470
1471                 (vn, vs) = find_vessel()
1472
1473                 s = ''
1474                 if cliexec is not None:
1475                         s += '...'
1476                 elif clierr is not None:
1477                         s += 'Error: '+clierr
1478                 elif clicmd is not None:
1479                         s += '/' + clicmd
1480                 else:
1481                         s = track.myname() + vs
1482                         s += " at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
1483                         s += kreader.info()
1484                 s += '\n'
1485
1486                 tbl_s = StringIO()
1487                 tbl = StandingsTable(tbl_s)
1488
1489                 aboard = track.aboard(vn)
1490                 sort.lsort_pa(aboard)
1491
1492                 jobbers = track.jobbers()
1493
1494                 if track.vesselname(): howmany = 'aboard: %2d' % len(aboard)
1495                 else: howmany = ''
1496
1497                 tbl.headings(howmany, '  sorted by '+sort.desc())
1498
1499                 last_jobber = None
1500
1501                 for pa in aboard + jobbers:
1502                         if pa.jobber != last_jobber:
1503                                 last_jobber = pa.jobber
1504                                 tbl.literalline('')
1505                                 tbl.literalline('jobbers '+last_jobber)
1506
1507                         pi = pa.pirate_info()
1508
1509                         xs = ''
1510                         if pa.gunner: xs += 'G '
1511                         else: xs += '  '
1512                         xs += timeevent(pa.last_time, pa.last_event)
1513                         xs += timeevent(pa.last_chat_time, pa.last_chat_chan)
1514
1515                         if pi is None:
1516                                 tbl.pirate_dummy(pa.name, rotate_nya[0], xs)
1517                         else:
1518                                 tbl.pirate(pi, xs)
1519
1520                 s += tbl_s.getvalue()
1521                 displayer.show(s)
1522                 tbl_s.close()
1523
1524                 if cliexec is not None:
1525                         clierr = track.local_command("/"+cliexec.strip())
1526                         cliexec = None
1527                         continue
1528
1529                 k = kreader.getch()
1530                 if k is None:
1531                         rotate_nya = rotate_nya[1:3] + rotate_nya[0]
1532                         continue
1533
1534                 if clierr is not None:
1535                         clierr = None
1536                         continue
1537
1538                 if clicmd is not None:
1539                         if k == '\r' or k == '\n':
1540                                 cliexec = clicmd
1541                                 clicmd = clicmdbase
1542                         elif k == '\e' and clicmd != "":
1543                                 clicmd = clicmdbase
1544                         elif k == '\33':
1545                                 clicmd = None
1546                         elif k == '\b' or k == '\177':
1547                                 clicmd = clicmd[ 0 : len(clicmd)-1 ]
1548                         else:
1549                                 clicmd += k
1550                         continue
1551
1552                 if k == 'q': break
1553                 elif k == 'g': sort = SkillSorter('Gunning')
1554                 elif k == 'c': sort = SkillSorter('Carpentry')
1555                 elif k == 's': sort = SkillSorter('Sailing/Rigging')
1556                 elif k == 'b': sort = SkillSorter('Bilging')
1557                 elif k == 'n': sort = SkillSorter('Navigating')
1558                 elif k == 'd': sort = SkillSorter('Battle Navigation')
1559                 elif k == 't': sort = SkillSorter('Treasure Haul')
1560                 elif k == 'a': sort = NameSorter()
1561                 elif k == '/': clicmdbase = ""; clicmd = clicmdbase
1562                 elif k == '+': clicmdbase = "a "; clicmd = clicmdbase
1563                 else: pass # unknown key command
1564
1565 #---------- individual keystroke input ----------
1566
1567 class DummyKeystrokeReader:
1568         def __init__(self,fd,timeout_dummy): pass
1569         def stop(self): pass
1570         def getch(self): sleep(1); return None
1571         def info(self): return ' [noninteractive]'
1572
1573 class KeystrokeReader(DummyKeystrokeReader):
1574         def __init__(self, fd, timeout_decisec=0):
1575                 self._fd = fd
1576                 self._saved = termios.tcgetattr(fd)
1577                 a = termios.tcgetattr(fd)
1578                 a[3] &= ~(termios.ECHO | termios.ECHONL |
1579                           termios.ICANON | termios.IEXTEN)
1580                 a[6][termios.VMIN] = 0
1581                 a[6][termios.VTIME] = timeout_decisec
1582                 termios.tcsetattr(fd, termios.TCSANOW, a)
1583         def stop(self):
1584                 termios.tcsetattr(self._fd, termios.TCSANOW, self._saved)
1585         def getch(self):
1586                 debug_flush()
1587                 byte = os.read(self._fd, 1)
1588                 if not len(byte): return None
1589                 return byte
1590         def info(self):
1591                 return ''
1592
1593 #---------- main program ----------
1594
1595 def main():
1596         global opts, fetcher
1597
1598         pa = OptionParser(
1599 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
1600 actions:
1601  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
1602  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
1603  yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
1604  yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG
1605  yoweb-scrape [options] ship-aid CHAT-LOG  (must be .../PIRATE_OCEAN_chat-log*)
1606
1607 display modes (for --display) apply to ship-aid:
1608  --display=dumb       just print new information, scrolling the screen
1609  --display=overwrite  use cursor motion, selective clear, etc. to redraw at top''')
1610         ao = pa.add_option
1611         ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
1612                 help='select ocean OCEAN')
1613         ao('--cache-dir', dest='cache_dir', metavar='DIR',
1614                 default='~/.yoweb-scrape-cache',
1615                 help='cache yoweb pages in DIR')
1616         ao('-D','--debug', action='count', dest='debug', default=0,
1617                 help='enable debugging output')
1618         ao('--debug-fd', type='int', dest='debug_fd',
1619                 help='write any debugging output to specified fd')
1620         ao('-q','--quiet', action='store_true', dest='quiet',
1621                 help='suppress warning output')
1622         ao('--display', action='store', dest='display',
1623                 type='choice', choices=['dumb','overwrite'],
1624                 help='how to display ship aid')
1625         ao('--local-ypp-dir', action='store', dest='localhtml',
1626                 help='get yppedia pages from local directory LOCALHTML'+
1627                         ' instead of via HTTP')
1628
1629         ao_jt = lambda wh, t: ao(
1630                 '--timeout-sa-'+wh, action='store', dest='timeout_'+wh,
1631                 default=t, help=('set timeout for expiring %s jobbers' % wh))
1632         ao_jt('applied',      120)
1633         ao_jt('invited',      120)
1634         ao_jt('declined',      30)
1635         ao_jt('ashore',      1800)
1636
1637         ao('--ship-duty', action='store_true', dest='ship_duty',
1638                 help='show ship duty station puzzles')
1639         ao('--all-puzzles', action='store_false', dest='ship_duty',
1640                 help='show all puzzles, not just ship duty stations')
1641
1642         ao('--min-cache-reuse', type='int', dest='min_max_age',
1643                 metavar='SECONDS', default=60,
1644                 help='always reuse cache yoweb data if no older than this')
1645
1646         (opts,args) = pa.parse_args()
1647         random.seed()
1648
1649         if len(args) < 1:
1650                 print >>sys.stderr, copyright_info
1651                 pa.error('need a mode argument')
1652
1653         if opts.debug_fd is not None:
1654                 opts.debug_file = os.fdopen(opts.debug_fd, 'w')
1655         else:
1656                 opts.debug_file = sys.stdout
1657
1658         mode = args[0]
1659         mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
1660         try: mode_fn = globals()[mode_fn_name]
1661         except KeyError: pa.error('unknown mode "%s"' % mode)
1662
1663         # fixed parameters
1664         opts.expire_age = max(3600, opts.min_max_age)
1665
1666         opts.ship_reboard_clearout = 3600
1667
1668         if opts.cache_dir.startswith('~/'):
1669                 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
1670
1671         if opts.display is None:
1672                 if ((opts.debug > 0 and opts.debug_fd is None)
1673                     or not os.isatty(sys.stdout.fileno())):
1674                         opts.display = 'dumb'
1675                 else:
1676                         opts.display = 'overwrite'
1677
1678         fetcher = Fetcher(opts.ocean, opts.cache_dir)
1679
1680         mode_fn(args[1:], pa.error)
1681
1682 main()