yoweb-scrape

   1 #!/usr/bin/python
   2 # This is part of ypp-sc-tools, a set of third-party tools for assisting
   3 # players of Yohoho Puzzle Pirates.
   4 #
   5 # Copyright (C) 2009 Ian Jackson <ijackson@chiark.greenend.org.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 #
  20 # Yohoho and Puzzle Pirates are probably trademarks of Three Rings and
  21 # are used without permission.  This program is not endorsed or
  22 # sponsored by Three Rings.
  23
  24 copyright_info = '''
  25 yoweb-scrape is part of ypp-sc-tools  Copyright (C) 2009 Ian Jackson
  26 This program comes with ABSOLUTELY NO WARRANTY; this is free software,
  27 and you are welcome to redistribute it under certain conditions.
  28 For details, read the top of the yoweb-scrape file.
  29 '''
  30
  31 #---------- setup ----------
  32
  33 import signal
  34 signal.signal(signal.SIGINT, signal.SIG_DFL)
  35
  36 import os
  37 import time
  38 import urllib
  39 import urllib2
  40 import errno
  41 import sys
  42 import re as regexp
  43 import random
  44 import curses
  45 import termios
  46 import random
  47 import subprocess
  48 import copy
  49 from optparse import OptionParser
  50 from StringIO import StringIO
  51
  52 from BeautifulSoup import BeautifulSoup
  53
  54 opts = None
  55
  56 #---------- YPP parameters and arrays ----------
  57
  58 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
  59         '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
  60         '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
  61         '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
  62
  63 core_duty_puzzles = [
  64                 'Gunning',
  65                 ['Sailing','Rigging'],
  66                 'Bilging',
  67                 'Carpentry',
  68                 ]
  69
  70 duty_puzzles = ([ 'Navigating', 'Battle Navigation' ] +
  71                 core_duty_puzzles +
  72                 [ 'Treasure Haul' ])
  73
  74 standingvals = ('Able/Proficient/Distinguished/Respected/Master'+
  75                 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
  76 standing_limit = len(standingvals)
  77
  78 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
  79
  80 max_pirate_namelen = 12
  81
  82
  83 #---------- general utilities ----------
  84
  85 def debug(m):
  86         if opts.debug > 0:
  87                 print >>opts.debug_file, m
  88
  89 def debug_flush():
  90         if opts.debug > 0:
  91                 opts.debug_file.flush()
  92
  93 def sleep(seconds):
  94         debug_flush()
  95         time.sleep(seconds)
  96
  97 def format_time_interval(ti):
  98         if ti < 120: return '%d:%02d' % (ti / 60, ti % 60)
  99         if ti < 7200: return '%2dm' % (ti / 60)
 100         if ti < 86400: return '%dh' % (ti / 3600)
 101         return '%dd' % (ti / 86400)
 102
 103 def yppsc_dir():
 104         lib = os.getenv("YPPSC_YARRG_SRCBASE")
 105         if lib is not None: return lib
 106         lib = sys.argv[0]
 107         lib = regexp.sub('/[^/]+$', '', lib)
 108         os.environ["YPPSC_YARRG_SRCBASE"] = lib
 109         return lib
 110
 111 soup_massage = copy.copy(BeautifulSoup.MARKUP_MASSAGE)
 112 soup_massage.append(
 113                 (regexp.compile('(\<td.*") ("center")'),
 114                  lambda m: m.group(1)+' align='+m.group(2))
 115         )
 116
 117 def make_soup(*args, **kwargs):
 118         return BeautifulSoup(*args,
 119                 convertEntities=BeautifulSoup.HTML_ENTITIES,
 120                 markupMassage=soup_massage,
 121                          **kwargs)
 122
 123 #---------- caching and rate-limiting data fetcher ----------
 124
 125 class Fetcher:
 126         def __init__(self, cachedir):
 127                 debug('Fetcher init %s' % cachedir)
 128                 self.cachedir = cachedir
 129                 try: os.mkdir(cachedir)
 130                 except (OSError,IOError), oe:
 131                         if oe.errno != errno.EEXIST: raise
 132                 self._cache_scan(time.time())
 133
 134         def _cache_scan(self, now):
 135                 # returns list of ages, unsorted
 136                 ages = []
 137                 debug('Fetcher   scan_cache')
 138                 for leaf in os.listdir(self.cachedir):
 139                         if not leaf.startswith('#'): continue
 140                         path = self.cachedir + '/' + leaf
 141                         try: s = os.stat(path)
 142                         except (OSError,IOError), oe:
 143                                 if oe.errno != errno.ENOENT: raise
 144                                 continue
 145                         age = now - s.st_mtime
 146                         if age > opts.expire_age:
 147                                 debug('Fetcher    expire %d %s' % (age, path))
 148                                 try: os.remove(path)
 149                                 except (OSError,IOError), oe:
 150                                         if oe.errno != errno.ENOENT: raise
 151                                 continue
 152                         ages.append(age)
 153                 return ages
 154
 155         def need_wait(self, now, imaginary=[]):
 156                 ages = self._cache_scan(now)
 157                 ages += imaginary
 158                 ages.sort()
 159                 debug('Fetcher   ages ' + `ages`)
 160                 min_age = 1
 161                 need_wait = 0
 162                 for age in ages:
 163                         if age < min_age and age <= 5:
 164                                 debug('Fetcher   morewait min=%d age=%d' %
 165                                         (min_age, age))
 166                                 need_wait = max(need_wait, min_age - age)
 167                         min_age += 3
 168                 if need_wait > 0:
 169                         need_wait += random.random() - 0.5
 170                 return need_wait
 171
 172         def _rate_limit_cache_clean(self, now):
 173                 need_wait = self.need_wait(now)
 174                 if need_wait > 0:
 175                         debug('Fetcher   wait %f' % need_wait)
 176                         sleep(need_wait)
 177
 178         def fetch(self, url, max_age):
 179                 debug('Fetcher fetch %s' % url)
 180                 cache_corename = urllib.quote_plus(url)
 181                 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
 182                 try: f = file(cache_item, 'r')
 183                 except (OSError,IOError), oe:
 184                         if oe.errno != errno.ENOENT: raise
 185                         f = None
 186                 now = time.time()
 187                 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
 188                 if f is not None:
 189                         s = os.fstat(f.fileno())
 190                         age = now - s.st_mtime
 191                         if age > max_age:
 192                                 debug('Fetcher  stale %d < %d'% (max_age, age))
 193                                 f = None
 194                 if f is not None:
 195                         data = f.read()
 196                         f.close()
 197                         debug('Fetcher  cached %d > %d' % (max_age, age))
 198                         return data
 199
 200                 debug('Fetcher  fetch')
 201                 self._rate_limit_cache_clean(now)
 202
 203                 stream = urllib2.urlopen(url)
 204                 data = stream.read()
 205                 cache_tmp = "%s/#%s~%d#" % (
 206                         self.cachedir, cache_corename, os.getpid())
 207                 f = file(cache_tmp, 'w')
 208                 f.write(data)
 209                 f.close()
 210                 os.rename(cache_tmp, cache_item)
 211                 debug('Fetcher  stored')
 212                 return data
 213
 214 class Yoweb(Fetcher):
 215         def __init__(self, ocean, cachedir):
 216                 debug('Yoweb init %s' % cachedir)
 217                 self.ocean = ocean
 218                 Fetcher.__init__(self, cachedir)
 219
 220         def default_ocean(self, ocean='ice'):
 221                 if self.ocean is None:
 222                         self.ocean = ocean
 223
 224         def yoweb(self, kind, tail, max_age):
 225                 self.default_ocean()
 226                 assert(self.ocean)
 227                 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
 228                         self.ocean, kind, tail)
 229                 return self.fetch(url, max_age)
 230
 231 class Yppedia(Fetcher):
 232         def __init__(self, cachedir):
 233                 debug('Yoweb init %s' % cachedir)
 234                 self.base = 'http://yppedia.puzzlepirates.com/'
 235                 self.localhtml = opts.localhtml
 236                 Fetcher.__init__(self, cachedir)
 237
 238         def __call__(self, rhs):
 239                 if self.localhtml is None:
 240                         url = self.base + rhs
 241                         debug('Yppedia retrieving YPP '+url);
 242                         return self.fetch(url, 3000)
 243                 else:
 244                         return file(opts.localhtml + '/' + rhs, 'r')
 245
 246 #---------- logging assistance for troubled screenscrapers ----------
 247
 248 class SoupLog:
 249         def __init__(self):
 250                 self.msgs = [ ]
 251         def msg(self, m):
 252                 self.msgs.append(m)
 253         def soupm(self, obj, m):
 254                 self.msg(m + '; in ' + `obj`)
 255         def needs_msgs(self, child_souplog):
 256                 self.msgs += child_souplog.msgs
 257                 child_souplog.msgs = [ ]
 258
 259 def soup_text(obj):
 260         str = ''.join(obj.findAll(text=True))
 261         return str.strip()
 262
 263 class SomethingSoupInfo(SoupLog):
 264         def __init__(self, kind, tail, max_age):
 265                 SoupLog.__init__(self)
 266                 html = fetcher.yoweb(kind, tail, max_age)
 267                 self._soup = make_soup(html)
 268
 269 #---------- scraper for pirate pages ----------
 270
 271 class PirateInfo(SomethingSoupInfo):
 272         # Public data members:
 273         #  pi.standings = { 'Treasure Haul': 'Able' ... }
 274         #  pi.name = name
 275         #  pi.crew = (id, name)
 276         #  pi.flag = (id, name)
 277         #  pi.msgs = [ 'message describing problem with scrape' ]
 278
 279         def __init__(self, pirate, max_age=300):
 280                 SomethingSoupInfo.__init__(self,
 281                         'pirate.wm?target=', pirate, max_age)
 282                 self.name = pirate
 283                 self._find_standings()
 284                 self.crew = self._find_crewflag('crew',
 285                         '^/yoweb/crew/info\\.wm')
 286                 self.flag = self._find_crewflag('flag',
 287                         '^/yoweb/flag/info\\.wm')
 288
 289         def _find_standings(self):
 290                 imgs = self._soup.findAll('img',
 291                         src=regexp.compile('/yoweb/images/stat.*'))
 292                 re = regexp.compile(
 293 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
 294                         )
 295                 standings = { }
 296
 297                 for skill in puzzles:
 298                         standings[skill] = [ ]
 299
 300                 skl = SoupLog()
 301
 302                 for img in imgs:
 303                         try: puzzle = img['alt']
 304                         except KeyError: continue
 305
 306                         if not puzzle in puzzles:
 307                                 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
 308                                 continue
 309                         key = img.findParent('td')
 310                         if key is None:
 311                                 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
 312                                 continue
 313                         valelem = key.findNextSibling('td')
 314                         if valelem is None:
 315                                 skl.soupm(key, 'puzzle missing sibling "%s"'
 316                                         % puzzle)
 317                                 continue
 318                         valstr = soup_text(valelem)
 319                         match = re.match(valstr)
 320                         if match is None:
 321                                 skl.soupm(key, ('puzzle "%s" unparseable'+
 322                                         ' standing "%s"') % (puzzle, valstr))
 323                                 continue
 324                         standing = match.group(match.lastindex)
 325                         standings[puzzle].append(standing)
 326
 327                 self.standings = { }
 328
 329                 for puzzle in puzzles:
 330                         sl = standings[puzzle]
 331                         if len(sl) > 1:
 332                                 skl.msg('puzzle "%s" multiple standings %s' %
 333                                                 (puzzle, `sl`))
 334                                 continue
 335                         if not sl:
 336                                 skl.msg('puzzle "%s" no standing found' % puzzle)
 337                                 continue
 338                         standing = sl[0]
 339                         for i in range(0, standing_limit):
 340                                 if standing == standingvals[i]:
 341                                         self.standings[puzzle] = i
 342                         if not puzzle in self.standings:
 343                                 skl.msg('puzzle "%s" unknown standing "%s"' %
 344                                         (puzzle, standing))
 345
 346                 all_standings_ok = True
 347                 for puzzle in puzzles:
 348                         if not puzzle in self.standings:
 349                                 self.needs_msgs(skl)
 350
 351         def _find_crewflag(self, cf, yoweb_re):
 352                 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
 353                 if len(things) != 1:
 354                         self.msg('zero or several %s id references found' % cf)
 355                         return None
 356                 thing = things[0]
 357                 id_re = '\\b%sid\\=(\\w+)$' % cf
 358                 id_haystack = thing['href']
 359                 match = regexp.compile(id_re).search(id_haystack)
 360                 if match is None:
 361                         self.soupm(thing, ('incomprehensible %s id ref'+
 362                                 ' (%s in %s)') % (cf, id_re, id_haystack))
 363                         return None
 364                 name = soup_text(thing)
 365                 return (match.group(1), name)
 366
 367         def __str__(self):
 368                 return `(self.crew, self.flag, self.standings, self.msgs)`
 369
 370 #---------- scraper for crew pages ----------
 371
 372 class CrewInfo(SomethingSoupInfo):
 373         # Public data members:
 374         #  ci.crewid
 375         #  ci.crew = [ ('Captain',        ['Pirate', ...]),
 376         #              ('Senior Officer', [...]),
 377         #               ... ]
 378         #  pi.msgs = [ 'message describing problem with scrape' ]
 379
 380         def __init__(self, crewid, max_age=300):
 381                 self.crewid = crewid
 382                 SomethingSoupInfo.__init__(self,
 383                         'crew/info.wm?crewid=', crewid, max_age)
 384                 self._find_crew()
 385
 386         def _find_crew(self):
 387                 self.crew = []
 388                 capts = self._soup.findAll('img',
 389                         src='/yoweb/images/crew-captain.png')
 390                 if len(capts) != 1:
 391                         self.msg('crew members: no. of captain images != 1')
 392                         return
 393                 tbl = capts[0]
 394                 while not tbl.find('a', href=pirate_ref_re):
 395                         tbl = tbl.findParent('table')
 396                         if not tbl:
 397                                 self.msg('crew members: cannot find table')
 398                                 return
 399                 current_rank_crew = None
 400                 crew_rank_re = regexp.compile('/yoweb/images/crew')
 401                 for row in tbl.contents:
 402                         # findAll(recurse=False)
 403                         if isinstance(row,basestring):
 404                                 continue
 405
 406                         is_rank = row.find('img', attrs={'src': crew_rank_re})
 407                         if is_rank:
 408                                 rank = soup_text(row)
 409                                 current_rank_crew = []
 410                                 self.crew.append((rank, current_rank_crew))
 411                                 continue
 412                         for cell in row.findAll('a', href=pirate_ref_re):
 413                                 if current_rank_crew is None:
 414                                         self.soupm(cell, 'crew members: crew'
 415                                                 ' before rank')
 416                                         continue
 417                                 current_rank_crew.append(soup_text(cell))
 418
 419         def __str__(self):
 420                 return `(self.crew, self.msgs)`
 421
 422 class FlagRelation():
 423         # Public data members (put there by hand by creater)
 424         #       other_flagname
 425         #       other_flagid
 426         #       yoweb_heading
 427         #       this_declaring
 428         #       other_declaring_min
 429         #       other_declaring_max
 430         # where {this,other}_declaring{,_min,_max} are:
 431         #       -1      {this,other} is declaring war
 432         #        0      {this,other} is not doing either
 433         #       +1      {this,other} is allying
 434         def __repr__(self):
 435                 return '<FlagRelation %s %d/%d..%d %s %s>' % (
 436                         self.yoweb_heading, self.this_declaring,
 437                         self.other_declaring_min, self.other_declaring_max,
 438                         self.other_flagname, self.other_flagid)
 439
 440 class FlagInfo(SomethingSoupInfo):
 441         # Public data members (after init):
 442         #
 443         #   flagid
 444         #   name        #               string
 445         #
 446         #   relations[n] = FlagRelation
 447         #   relation_byname[otherflagname] = relations[some_n]
 448         #   relation_byid[otherflagname] = relations[some_n]
 449         #
 450         #   islands[n] = (islandname, islandid)
 451         #
 452         def __init__(self, flagid, max_age=600):
 453                 self.flagid = flagid
 454                 SomethingSoupInfo.__init__(self,
 455                         'flag/info.wm?flagid=', flagid, max_age)
 456                 self._find_flag()
 457
 458         def _find_flag(self):
 459                 font2 = self._soup.find('font',{'size':'+2'})
 460                 self.name = font2.find('b').contents[0]
 461
 462                 self.relations = [ ]
 463                 self.relation_byname = { }
 464                 self.relation_byid = { }
 465                 self.islands = [ ]
 466
 467                 magnate = self._soup.find('img',{'src':
 468                         '/yoweb/images/repute-MAGNATE.png'})
 469                 warinfo = (magnate.findParent('table').findParent('tr').
 470                         findNextSibling('tr').findNext('td',{'align':'left'}))
 471
 472                 def warn(m):
 473                         print >>sys.stderr, 'WARNING: '+m
 474
 475                 def wi_warn(head, waritem):
 476                         warn('unknown warmap item: %s: %s' %
 477                                 (`head`, ``waritem``))
 478
 479                 def wihelp_item(waritem, thing):
 480                         url = waritem.get('href', None)
 481                         if url is None:
 482                                 return ('no url for '+thing,None,None)
 483                         m = regexp.search('\?'+thing+'id=(\d+)$', url)
 484                         if not m: return ('no '+thing+'id',None,None)
 485                         tid = m.group(1)
 486                         tname = waritem.string
 487                         if tname is None:
 488                                 return (thing+' name not just string',None,None)
 489                         return (None,tid,tname)
 490
 491                 def wi_alwar(head, waritem, thisdecl, othermin, othermax):
 492                         (err,flagid,flagname) = wihelp_item(waritem,'flag')
 493                         if err: return err
 494                         rel = self.relation_byid.get(flagid, None)
 495                         if rel: return 'flag id twice!'
 496                         if flagname in self.relation_byname:
 497                                 return 'flag name twice!'
 498                         rel = FlagRelation()
 499                         rel.other_flagname = flagname
 500                         rel.other_flagid = flagid
 501                         rel.yoweb_heading = head
 502                         rel.this_declaring = thisdecl
 503                         rel.other_declaring_min = othermin
 504                         rel.other_declaring_max = othermax
 505                         self.relations.append(rel)
 506                         self.relation_byid[flagid] = rel
 507                         self.relation_byname[flagid] = rel
 508
 509                 def wi_isle(head, waritem):
 510                         (err,isleid,islename) = wihelp_item(waritem,'island')
 511                         if err: return err
 512                         self.islands.append((isleid,islename))
 513
 514                 warmap = {
 515                         'Allied with':                  (wi_alwar,+1,+1,+1),
 516                         'Declaring war against':        (wi_alwar,-1, 0,+1),
 517                         'At war with':                  (wi_alwar,-1,-1,-1),
 518                         'Trying to form an alliance with': (wi_alwar,+1,-1,0),
 519                         'Islands controlled by this flag': (wi_isle,),
 520                         }
 521
 522                 how = (wi_warn, None)
 523
 524                 for waritem in warinfo.findAll(['font','a']):
 525                         if waritem is None: break
 526                         if waritem.name == 'font':
 527                                 colour = waritem.get('color',None)
 528                                 if colour.lstrip('#') != '958A5F':
 529                                         warn('strange colour %s in %s' %
 530                                                 (colour,``waritem``))
 531                                         continue
 532                                 head = waritem.string
 533                                 if head is None:
 534                                         warn('no head string in '+``waritem``)
 535                                         continue
 536                                 head = regexp.sub('\\s+', ' ', head).strip()
 537                                 head = head.rstrip(':')
 538                                 how = (head,) + warmap.get(head, (wi_warn,))
 539                                 continue
 540                         assert(waritem.name == 'a')
 541
 542                         debug('WARHOW %s(%s, waritem, *%s)' %
 543                                 (how[1], `how[0]`, `how[2:]`))
 544                         bad = how[1](how[0], waritem, *how[2:])
 545                         if bad:
 546                                 warn('bad waritem %s: %s: %s' % (`how[0]`,
 547                                         bad, ``waritem``))
 548
 549         def __str__(self):
 550                 return `(self.name, self.islands, self.relations)`
 551
 552 #---------- scraper for ocean info incl. embargoes etc. ----------
 553
 554 class IslandBasicInfo():
 555         # Public data attributes:
 556         #  ocean
 557         #  name
 558         # Public data attributes maybe set by caller:
 559         #  arch
 560         def __init__(self, ocean, islename):
 561                 self.ocean = ocean
 562                 self.name = islename
 563         def yppedia(self):
 564                 def q(x): return urllib.quote(x.replace(' ','_'))
 565                 url_rhs = q(self.name) + '_(' + q(self.ocean) + ')'
 566                 return yppedia(url_rhs)
 567         def __str__(self):
 568                 return `(self.ocean, self.name)`
 569
 570 class IslandExtendedInfo(IslandBasicInfo):
 571         # Public data attributes (inherited):
 572         #  ocean
 573         #  name
 574         # Public data attributes (additional):
 575         #  islandid
 576         #  yoweb_url
 577         #  flagid
 578         def __init__(self, ocean, islename):
 579                 IslandBasicInfo.__init__(self, ocean, islename)
 580                 self.islandid = None
 581                 self.yoweb_url = None
 582                 self._collect_yoweb()
 583                 self._collect_flagid()
 584
 585         def _collect_yoweb(self):
 586                 debug('IEI COLLECT YOWEB '+`self.name`)
 587                 self.islandid = None
 588                 self.yoweb_url = None
 589
 590                 soup = make_soup(self.yppedia())
 591                 content = soup.find('div', attrs = {'id': 'content'})
 592                 yoweb_re = regexp.compile('^http://\w+\.puzzlepirates\.com/'+
 593                         'yoweb/island/info\.wm\?islandid=(\d+)$')
 594                 a = soup.find('a', attrs = { 'href': yoweb_re })
 595                 if a is None:
 596                         debug('IEI COLLECT YOWEB '+`self.name`+' NONE')
 597                         return
 598
 599                 debug('IEI COLLECT YOWEB '+`self.name`+' GOT '+``a``)
 600                 self.yoweb_url = a['href']
 601                 m = yoweb_re.search(self.yoweb_url)
 602                 self.islandid = m.group(1)
 603
 604         def _collect_flagid(self):
 605                 self.flagid = None
 606
 607                 yo = self.yoweb_url
 608                 debug('IEI COLLECT FLAGID '+`self.name`+' URL '+`yo`)
 609                 if yo is None: return None
 610                 dataf = fetcher.fetch(yo, 1800)
 611                 soup = make_soup(dataf)
 612                 ruler_re = regexp.compile(
 613                         '/yoweb/flag/info\.wm\?flagid=(\d+)$')
 614                 ruler = soup.find('a', attrs = { 'href': ruler_re })
 615                 if not ruler:
 616                         debug('IEI COLLECT FLAGID '+`self.name`+' NONE')
 617                         return
 618                 debug('IEI COLLECT FLAGID '+`self.name`+' GOT '+``ruler``)
 619                 m = ruler_re.search(ruler['href'])
 620                 self.flagid = m.group(1)
 621
 622         def __str__(self):
 623                 return `(self.ocean, self.islandid, self.name,
 624                         self.yoweb_url, self.flagid)`
 625
 626 class IslandFlagInfo(IslandExtendedInfo):
 627         # Public data attributes (inherited):
 628         #  ocean
 629         #  name
 630         #  islandid
 631         #  yoweb_url
 632         #  flagid
 633         # Public data attributes (additional):
 634         #  flag
 635         def __init__(self, ocean, islename):
 636                 IslandExtendedInfo.__init__(self, ocean, islename)
 637                 self.flag = None
 638                 self._collect_flag()
 639
 640         def _collect_flag(self):
 641                 if self.flagid is None: return
 642                 self.flag = FlagInfo(self.flagid, 1800)
 643
 644         def __str__(self):
 645                 return IslandExtendedInfo.__str__(self) + '; ' + str(self.flag)
 646
 647 class NullProgressReporter():
 648         def doing(self, msg): pass
 649         def stop(self): pass
 650
 651 class TypewriterProgressReporter():
 652         def __init__(self):
 653                 self._l = 0
 654         def doing(self,m):
 655                 self._doing(m + '...')
 656         def _doing(self,m):
 657                 self._write('\r')
 658                 self._write(m)
 659                 less = self._l - len(m)
 660                 if less > 0:
 661                         self._write(' ' * less)
 662                         self._write('\b' * less)
 663                 self._l = len(m)
 664                 sys.stdout.flush()
 665         def stop(self):
 666                 self._doing('')
 667                 self._l = 0
 668         def _write(self,t):
 669                 sys.stdout.write(t)
 670
 671 class OceanInfo():
 672         # Public data attributes:
 673         #   oi.islands[islename] = IslandInfo(...)
 674         #   oi.arches[archname][islename] = IslandInfo(...)
 675         def __init__(self, isleclass=IslandBasicInfo):
 676                 self.isleclass = isleclass
 677                 self.ocean = fetcher.ocean.lower().capitalize()
 678
 679                 progressreporter.doing('fetching ocean info')
 680
 681                 cmdl = ['./yppedia-ocean-scraper']
 682                 if opts.localhtml is not None:
 683                         cmdl += ['--local-html-dir',opts.localhtml]
 684                 cmdl += [self.ocean]
 685                 debug('OceanInfo collect running ' + `cmdl`)
 686                 oscraper = subprocess.Popen(
 687                         cmdl,
 688                         stdout = subprocess.PIPE,
 689                         cwd = yppsc_dir()+'/yarrg',
 690                         shell=False, stderr=None,
 691                         )
 692                 h = oscraper.stdout.readline()
 693                 debug('OceanInfo collect h '+`h`)
 694                 assert(regexp.match('^ocean ', h))
 695                 arch_re = regexp.compile('^ (\S.*)')
 696                 island_re = regexp.compile('^  (\S.*)')
 697
 698                 oscraper.wait()
 699                 assert(oscraper.returncode == 0)
 700
 701                 self.islands = { }
 702                 self.arches = { }
 703                 archname = None
 704
 705                 isles = [ ]
 706                 progressreporter.doing('parsing ocean info')
 707
 708                 for l in oscraper.stdout:
 709                         debug('OceanInfo collect l '+`l`)
 710                         l = l.rstrip('\n')
 711                         m = island_re.match(l)
 712                         if m:
 713                                 assert(archname is not None)
 714                                 islename = m.group(1)
 715                                 isles.append((archname, islename))
 716                                 continue
 717                         m = arch_re.match(l)
 718                         if m:
 719                                 archname = m.group(1)
 720                                 assert(archname not in self.arches)
 721                                 self.arches[archname] = { }
 722                                 continue
 723                         assert(False)
 724
 725                 for i in xrange(0, len(isles)-1):
 726                         (archname, islename) = isles[i]
 727                         progressreporter.doing(
 728                                 'fetching isle info %2d/%d (%s: %s)'
 729                                 % (i, len(isles), archname, islename))
 730                         isle = self.isleclass(self.ocean, islename)
 731                         isle.arch = archname
 732                         self.islands[islename] = isle
 733                         self.arches[archname][islename] = isle
 734
 735         def __str__(self):
 736                 return `(self.islands, self.arches)`
 737
 738 #---------- pretty-printer for tables of pirate puzzle standings ----------
 739
 740 class StandingsTable:
 741         def __init__(self, f, use_puzzles=None, col_width=6, gap_every=5):
 742                 if use_puzzles is None:
 743                         if opts.ship_duty:
 744                                 use_puzzles=duty_puzzles
 745                         else:
 746                                 use_puzzles=puzzles
 747                 self._puzzles = use_puzzles
 748                 self.f = f
 749                 self._cw = col_width-1
 750                 self._gap_every = gap_every
 751                 self._linecount = 0
 752                 self._o = f.write
 753
 754         def _nl(self): self._o('\n')
 755
 756         def _pline(self, lhs, puzstrs, extra):
 757                 if (self._linecount > 0
 758                     and self._gap_every is not None
 759                     and not (self._linecount % self._gap_every)):
 760                         self._nl()
 761                 self._o('%-*s' % (max(max_pirate_namelen+1, 15), lhs))
 762                 for v in puzstrs:
 763                         self._o(' %-*.*s' % (self._cw,self._cw, v))
 764                 if extra:
 765                         self._o(' ' + extra)
 766                 self._nl()
 767                 self._linecount += 1
 768
 769         def _puzstr(self, pi, puzzle):
 770                 if not isinstance(puzzle,list): puzzle = [puzzle]
 771                 try: standing = max([pi.standings[p] for p in puzzle])
 772                 except KeyError: return '?'
 773                 if not standing: return ''
 774                 s = ''
 775                 if self._cw > 4:
 776                         c1 = standingvals[standing][0]
 777                         if standing < 3: c1 = c1.lower() # 3 = Master
 778                         s += `standing`
 779                 if self._cw > 5:
 780                         s += ' '
 781                 s += '*' * (standing / 2)
 782                 s += '+' * (standing % 2)
 783                 return s
 784
 785         def headings(self, lhs='', rhs=None):
 786                 def puzn_redact(name):
 787                         if isinstance(name,list):
 788                                 return '/'.join(
 789                                         ["%.*s" % (self._cw/2, puzn_redact(n))
 790                                          for n in name])
 791                         spc = name.find(' ')
 792                         if spc < 0: return name
 793                         return name[0:min(4,spc)] + name[spc+1:]
 794                 self._linecount = -2
 795                 self._pline(lhs, map(puzn_redact, self._puzzles), rhs)
 796                 self._linecount = 0
 797         def literalline(self, line):
 798                 self._o(line)
 799                 self._nl()
 800                 self._linecount = 0
 801         def pirate_dummy(self, name, standingstring, extra=None):
 802                 standings = standingstring * len(self._puzzles)
 803                 self._pline(' '+name, standings, extra)
 804         def pirate(self, pi, extra=None):
 805                 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
 806                 self._pline(' '+pi.name, puzstrs, extra)
 807
 808
 809 #---------- chat log parser ----------
 810
 811 class PirateAboard:
 812         # This is essentially a transparent, dumb, data class.
 813         #  pa.v                 may be None
 814         #  pa.name
 815         #  pa.last_time
 816         #  pa.last_event
 817         #  pa.gunner
 818         #  pa.last_chat_time
 819         #  pa.last_chat_chan
 820         #  pa.pi
 821
 822         # Also used for jobbing applicants:
 823         #               happens when                    expires (to "-")
 824         #   -            disembark, leaves crew          no
 825         #   aboard       evidence of them being aboard   no
 826         #   applied      "has applied for the job"       120s, configurable
 827         #   ashore       "has taken a job"               30min, configurable
 828         #   declined     "declined the job offer"        30s, configurable
 829         #   invited      "has been invited to job"       120s, configurable
 830         #
 831         #  pa.jobber    None, 'ashore', 'applied', 'invited', 'declined'
 832         #  pa.expires   expiry time time
 833
 834         def __init__(pa, pn, v, time, event):
 835                 pa.name = pn
 836                 pa.v = v
 837                 pa.last_time = time
 838                 pa.last_event = event
 839                 pa.last_chat_time = None
 840                 pa.last_chat_chan = None
 841                 pa.gunner = False
 842                 pa.pi = None
 843                 pa.jobber = None
 844                 pa.expires = None
 845
 846         def pirate_info(pa):
 847                 now = time.time()
 848                 if pa.pi:
 849                         age = now - pa.pi_fetched
 850                         guide = random.randint(120,240)
 851                         if age <= guide:
 852                                 return pa.pi
 853                         debug('PirateAboard refresh %d > %d  %s' % (
 854                                 age, guide, pa.name))
 855                         imaginary = [2,4]
 856                 else:
 857                         imaginary = [1]
 858                 wait = fetcher.need_wait(now, imaginary)
 859                 if wait:
 860                         debug('PirateAboard fetcher not ready %d' % wait)
 861                         return pa.pi
 862                 pa.pi = PirateInfo(pa.name, 600)
 863                 pa.pi_fetched = now
 864                 return pa.pi
 865
 866 class ChatLogTracker:
 867         # This is quite complex so we make it opaque.  Use the
 868         # official invokers, accessors etc.
 869
 870         def __init__(self, myself_pi, logfn):
 871                 self._pl = {}   # self._pl['Pirate'] =
 872                 self._vl = {}   #   self._vl['Vessel']['Pirate'] = PirateAboard
 873                                 # self._vl['Vessel']['#lastinfo']
 874                                 # self._vl['Vessel']['#name']
 875                                 # self._v = self._vl[self._vessel]
 876                 self._date = None
 877                 self._myself = myself_pi
 878                 self._lbuf = ''
 879                 self._f = file(logfn)
 880                 flen = os.fstat(self._f.fileno()).st_size
 881                 max_backlog = 500000
 882                 if flen > max_backlog:
 883                         startpos = flen - max_backlog
 884                         self._f.seek(startpos)
 885                         self._f.readline()
 886                 self._progress = [0, flen - self._f.tell()]
 887                 self._disembark_myself()
 888                 self._need_redisplay = False
 889                 self._lastvessel = None
 890
 891         def _disembark_myself(self):
 892                 self._v = None
 893                 self._vessel = None
 894                 self.force_redisplay()
 895
 896         def force_redisplay(self):
 897                 self._need_redisplay = True
 898
 899         def _vessel_updated(self, v, timestamp):
 900                 if v is None: return
 901                 v['#lastinfo'] = timestamp
 902                 self.force_redisplay()
 903
 904         def _onboard_event(self,v,timestamp,pirate,event,jobber=None):
 905                 pa = self._pl.get(pirate, None)
 906                 if pa is not None and pa.v is v:
 907                         pa.last_time = timestamp
 908                         pa.last_event = event
 909                 else:
 910                         if pa is not None and pa.v is not None:
 911                                 del pa.v[pirate]
 912                         pa = PirateAboard(pirate, v, timestamp, event)
 913                         self._pl[pirate] = pa
 914                         if v is not None: v[pirate] = pa
 915                 pa.jobber = jobber
 916
 917                 if jobber is None: timeout = None
 918                 else: timeout = getattr(opts, 'timeout_'+jobber)
 919                 if timeout is None: pa.expires = None
 920                 else: pa.expires = timestamp + timeout
 921                 self._vessel_updated(v, timestamp)
 922                 return pa
 923
 924         def _expire_jobbers(self, now):
 925                 for pa in self._pl.values():
 926                         if pa.expires is None: continue
 927                         if pa.expires >= now: continue
 928                         v = pa.v
 929                         del self._pl[pa.name]
 930                         if v is not None: del v[pa.name]
 931                         self.force_redisplay()
 932
 933         def _trash_vessel(self, v):
 934                 for pn in v:
 935                         if pn.startswith('#'): continue
 936                         del self._pl[pn]
 937                 vn = v['#name']
 938                 del self._vl[vn]
 939                 if v is self._v: self._disembark_myself()
 940                 self.force_redisplay()
 941
 942         def _vessel_stale(self, v, timestamp):
 943                 return timestamp - v['#lastinfo'] > opts.ship_reboard_clearout
 944
 945         def _vessel_check_expire(self, v, timestamp):
 946                 if not self._vessel_stale(v, timestamp):
 947                         return v
 948                 self._debug_line_disposition(timestamp,'',
 949                         'stale-reset ' + v['#name'])
 950                 self._trash_vessel(v)
 951                 return None
 952
 953         def expire_garbage(self, timestamp):
 954                 for v in self._vl.values():
 955                         self._vessel_check_expire(v, timestamp)
 956
 957         def _vessel_lookup(self, vn, timestamp, dml=[], create=False):
 958                 v = self._vl.get(vn, None)
 959                 if v is not None:
 960                         v = self._vessel_check_expire(v, timestamp)
 961                 if v is not None:
 962                         dml.append('found')
 963                         return v
 964                 if not create:
 965                         dml.append('no')
 966                 dml.append('new')
 967                 self._vl[vn] = v = { '#name': vn }
 968                 self._vessel_updated(v, timestamp)
 969                 return v
 970
 971         def _find_matching_vessel(self, pattern, timestamp, cmdr,
 972                                         dml=[], create=False):
 973                 # use when a commander pirate `cmdr' specified a vessel
 974                 #  by name `pattern' (either may be None)
 975                 # if create is true, will create the vessel
 976                 #  record if an exact name is specified
 977
 978                 if (pattern is not None and
 979                     not '*' in pattern
 980                     and len(pattern.split(' ')) == 2):
 981                         vn = pattern.title()
 982                         dml.append('exact')
 983                         return self._vessel_lookup(
 984                                 vn, timestamp, dml=dml, create=create)
 985
 986                 if pattern is None:
 987                         pattern_check = lambda vn: True
 988                 else:
 989                         re = '(?:.* )?%s$' % pattern.lower().replace('*','.+')
 990                         pattern_check = regexp.compile(re, regexp.I).match
 991
 992                 tries = []
 993
 994                 cmdr_pa = self._pl.get(cmdr, None)
 995                 if cmdr_pa: tries.append((cmdr_pa.v, 'cmdr'))
 996
 997                 tries.append((self._v, 'here'))
 998                 tried_vns = []
 999
1000                 for (v, dm) in tries:
1001                         if v is None: dml.append(dm+'?'); continue
1002
1003                         vn = v['#name']
1004                         if not pattern_check(vn):
1005                                 tried_vns.append(vn)
1006                                 dml.append(dm+'#')
1007                                 continue
1008
1009                         dml.append(dm+'!')
1010                         return v
1011
1012                 if pattern is not None and '*' in pattern:
1013                         search = [
1014                                 (vn,v)
1015                                 for (vn,v) in self._vl.iteritems()
1016                                 if not self._vessel_stale(v, timestamp)
1017                                 if pattern_check(vn)
1018                                 ]
1019                         #debug('CLT-RE /%s/ wanted (%s) searched (%s)' % (
1020                         #       re,
1021                         #       '/'.join(tried_vns),
1022                         #       '/'.join([vn for (vn,v) in search])))
1023
1024                         if len(search)==1:
1025                                 dml.append('one')
1026                                 return search[0][1]
1027                         elif search:
1028                                 dml.append('many')
1029                         else:
1030                                 dml.append('none')
1031
1032         def _debug_line_disposition(self,timestamp,l,m):
1033                 debug('CLT %13s %-40s %s' % (timestamp,m,l))
1034
1035         def _rm_crew_l(self,re,l):
1036                 m = regexp.match(re,l)
1037                 if m and m.group(2) == self._myself.crew[1]:
1038                         return m.group(1)
1039                 else:
1040                         return None
1041
1042         def local_command(self, metacmd):
1043                 # returns None if all went well, or problem message
1044                 return self._command(self._myself.name, metacmd,
1045                         "local", time.time(),
1046                         (lambda m: debug('CMD %s' % metacmd)))
1047
1048         def _command(self, cmdr, metacmd, chan, timestamp, d):
1049                 # returns None if all went well, or problem message
1050                 metacmd = regexp.sub('\\s+', ' ', metacmd).strip()
1051                 m2 = regexp.match(
1052                     '/([adj]) (?:([A-Za-z* ]+)\\s*:)?([A-Za-z ]+)$',
1053                     metacmd)
1054                 if not m2: return "unknown syntax or command"
1055
1056                 (cmd, pattern, targets) = m2.groups()
1057                 dml = ['cmd', chan, cmd]
1058
1059                 if cmd == 'a': each = self._onboard_event
1060                 elif cmd == 'd': each = disembark
1061                 else: each = lambda *l: self._onboard_event(*l,
1062                                 **{'jobber':'applied'})
1063
1064                 if cmdr == self._myself.name:
1065                         dml.append('self')
1066                         how = 'cmd: %s' % cmd
1067                 else:
1068                         dml.append('other')
1069                         how = 'cmd: %s %s' % (cmd,cmdr)
1070
1071                 if cmd == 'j':
1072                         if pattern is not None:
1073                                 return "/j command does not take a vessel"
1074                         v = None
1075                 else:
1076                         v = self._find_matching_vessel(
1077                                 pattern, timestamp, cmdr,
1078                                 dml, create=True)
1079
1080                 if cmd == 'j' or v is not None:
1081                         targets = targets.strip().split(' ')
1082                         dml.append(`len(targets)`)
1083                         for target in targets:
1084                                 each(v, timestamp, target.title(), how)
1085                         self._vessel_updated(v, timestamp)
1086
1087                 dm = ' '.join(dml)
1088                 return d(dm)
1089
1090                 return None
1091
1092         def chatline(self,l):
1093                 rm = lambda re: regexp.match(re,l)
1094                 d = lambda m: self._debug_line_disposition(timestamp,l,m)
1095                 rm_crew = lambda re: self._rm_crew_l(re,l)
1096                 timestamp = None
1097
1098                 m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$')
1099                 if m:
1100                         self._date = [int(x) for x in m.groups()]
1101                         self._previous_timestamp = None
1102                         return d('date '+`self._date`)
1103
1104                 if self._date is None:
1105                         return d('date unset')
1106
1107                 m = rm('\\[(\d\d):(\d\d):(\d\d)\\] ')
1108                 if not m:
1109                         return d('no timestamp')
1110
1111                 while True:
1112                         time_tuple = (self._date +
1113                                       [int(x) for x in m.groups()] +
1114                                       [-1,-1,-1])
1115                         timestamp = time.mktime(time_tuple)
1116                         if timestamp >= self._previous_timestamp: break
1117                         self._date[2] += 1
1118                         self._debug_line_disposition(timestamp,'',
1119                                 'new date '+`self._date`)
1120
1121                 self._previous_timestamp = timestamp
1122
1123                 l = l[l.find(' ')+1:]
1124
1125                 def ob_x(pirate,event):
1126                         return self._onboard_event(
1127                                         self._v, timestamp, pirate, event)
1128                 def ob1(did): ob_x(m.group(1), did); return d(did)
1129                 def oba(did): return ob1('%s %s' % (did, m.group(2)))
1130
1131                 def jb(pirate,jobber):
1132                         return self._onboard_event(
1133                                 None, timestamp, pirate,
1134                                 ("jobber %s" % jobber),
1135                                 jobber=jobber
1136                                 )
1137
1138                 def disembark(v, timestamp, pirate, event):
1139                         self._onboard_event(
1140                                         v, timestamp, pirate, 'leaving '+event)
1141                         del v[pirate]
1142                         del self._pl[pirate]
1143
1144                 def disembark_me(why):
1145                         self._disembark_myself()
1146                         return d('disembark-me '+why)
1147
1148                 m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
1149                 if m:
1150                         dm = ['boarding']
1151                         pn = self._myself.name
1152                         vn = m.group(1)
1153                         v = self._vessel_lookup(vn, timestamp, dm, create=True)
1154                         self._lastvessel = self._vessel = vn
1155                         self._v = v
1156                         ob_x(pn, 'we boarded')
1157                         self.expire_garbage(timestamp)
1158                         return d(' '.join(dm))
1159
1160                 if self._v is None:
1161                         return d('no vessel')
1162
1163                 m = rm('(\\w+) has come aboard\\.$')
1164                 if m: return ob1('boarded');
1165
1166                 m = rm('You have ordered (\\w+) to do some (\\S.*\\S)\\.$')
1167                 if m:
1168                         (who,what) = m.groups()
1169                         pa = ob_x(who,'ord '+what)
1170                         if what == 'Gunning':
1171                                 pa.gunner = True
1172                         return d('duty order')
1173
1174                 m = rm('(\\w+) abandoned a (\\S.*\\S) station\\.$')
1175                 if m: oba('stopped'); return d("end")
1176
1177                 def chat_core(speaker, chan):
1178                         try: pa = self._pl[speaker]
1179                         except KeyError: return 'mystery'
1180                         if pa.v is not None and pa.v is not self._v:
1181                                 return 'elsewhere'
1182                         pa.last_chat_time = timestamp
1183                         pa.last_chat_chan = chan
1184                         self.force_redisplay()
1185                         return 'here'
1186
1187                 def chat(chan):
1188                         speaker = m.group(1)
1189                         dm = chat_core(speaker, chan)
1190                         return d('chat %s %s' % (chan, dm))
1191
1192                 def chat_metacmd(chan):
1193                         (cmdr, metacmd) = m.groups()
1194                         whynot = self._command(
1195                                 cmdr, metacmd, chan, timestamp, d)
1196                         if whynot is not None:
1197                                 return chat(chan)
1198                         else:
1199                                 chat_core(cmdr, 'cmd '+chan)
1200
1201                 m = rm('(\\w+) (?:issued an order|ordered everyone) "')
1202                 if m: return ob1('general order');
1203
1204                 m = rm('(\\w+) says, "')
1205                 if m: return chat('public')
1206
1207                 m = rm('(\\w+) tells ye, "')
1208                 if m: return chat('private')
1209
1210                 m = rm('Ye told (\\w+), "(.*)"$')
1211                 if m: return chat_metacmd('private')
1212
1213                 m = rm('(\\w+) flag officer chats, "')
1214                 if m: return chat('flag officer')
1215
1216                 m = rm('(\\w+) officer chats, "(.*)"$')
1217                 if m: return chat_metacmd('officer')
1218
1219                 m = rm('Ye accepted the offer to job with ')
1220                 if m: return disembark_me('jobbing')
1221
1222                 m = rm('Ye hop on the ferry and are whisked away ')
1223                 if m: return disembark_me('ferry')
1224
1225                 m = rm('Whisking away to yer home on the magical winds')
1226                 if m: return disembark_me('home')
1227
1228                 m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
1229                 if m:
1230                         pl = m.group(1).split(', ')
1231                         if not self._myself.name in pl:
1232                                 return d('lost melee')
1233                         for pn in pl:
1234                                 if ' ' in pn: continue
1235                                 ob_x(pn,'won melee')
1236                         return d('won melee')
1237
1238                 m = rm('(\\w+) is eliminated\\!')
1239                 if m: return ob1('eliminated in fray');
1240
1241                 m = rm('(\\w+) has driven \w+ from the ship\\!')
1242                 if m: return ob1('boarder repelled');
1243
1244                 m = rm('\w+ has bested (\\w+), and turns'+
1245                         ' to the rest of the ship\\.')
1246                 if m: return ob1('boarder unrepelled');
1247
1248                 pirate = rm_crew("(\\w+) has taken a job with '(.*)'\\.")
1249                 if pirate: return jb(pirate, 'ashore')
1250
1251                 pirate = rm_crew("(\\w+) has left '(.*)'\\.")
1252                 if pirate:
1253                         disembark(self._v, timestamp, pirate, 'left crew')
1254                         return d('left crew')
1255
1256                 m = rm('(\w+) has applied for the posted job\.')
1257                 if m: return jb(m.group(1), 'applied')
1258
1259                 pirate= rm_crew("(\\w+) has been invited to job for '(.*)'\\.")
1260                 if pirate: return jb(pirate, 'invited')
1261
1262                 pirate = rm_crew("(\\w+) declined the job offer for '(.*)'\\.")
1263                 if pirate: return jb(pirate, 'declined')
1264
1265                 m = rm('(\\w+) has left the vessel\.')
1266                 if m:
1267                         pirate = m.group(1)
1268                         disembark(self._v, timestamp, pirate, 'disembarked')
1269                         return d('disembarked')
1270
1271                 return d('not-matched')
1272
1273         def _str_pa(self, pn, pa):
1274                 assert self._pl[pn] == pa
1275                 s = ' '*20 + "%s %-*s %13s %-30s %13s %-20s %13s" % (
1276                         (' ','G')[pa.gunner],
1277                         max_pirate_namelen, pn,
1278                         pa.last_time, pa.last_event,
1279                         pa.last_chat_time, pa.last_chat_chan,
1280                         pa.jobber)
1281                 if pa.expires is not None:
1282                         s += " %-5d" % (pa.expires - pa.last_time)
1283                 s += "\n"
1284                 return s
1285
1286         def _str_vessel(self, vn, v):
1287                 s = ' vessel %s\n' % vn
1288                 s += ' '*20 + "%-*s   %13s\n" % (
1289                                 max_pirate_namelen, '#lastinfo',
1290                                 v['#lastinfo'])
1291                 assert v['#name'] == vn
1292                 for pn in sorted(v.keys()):
1293                         if pn.startswith('#'): continue
1294                         pa = v[pn]
1295                         assert pa.v == v
1296                         s += self._str_pa(pn,pa)
1297                 return s
1298
1299         def __str__(self):
1300                 s = '''<ChatLogTracker
1301  myself %s
1302  vessel %s
1303 '''                     % (self._myself.name, self._vessel)
1304                 assert ((self._v is None and self._vessel is None) or
1305                         (self._v is self._vl[self._vessel]))
1306                 if self._vessel is not None:
1307                         s += self._str_vessel(self._vessel, self._v)
1308                 for vn in sorted(self._vl.keys()):
1309                         if vn == self._vessel: continue
1310                         s += self._str_vessel(vn, self._vl[vn])
1311                 s += " elsewhere\n"
1312                 for p in self._pl:
1313                         pa = self._pl[p]
1314                         if pa.v is not None:
1315                                 assert pa.v[p] is pa
1316                                 assert pa.v in self._vl.values()
1317                         else:
1318                                 s += self._str_pa(pa.name, pa)
1319                 s += '>\n'
1320                 return s
1321
1322         def catchup(self, progress=None):
1323                 while True:
1324                         more = self._f.readline()
1325                         if not more: break
1326
1327                         self._progress[0] += len(more)
1328                         if progress: progress.progress(*self._progress)
1329
1330                         self._lbuf += more
1331                         if self._lbuf.endswith('\n'):
1332                                 self.chatline(self._lbuf.rstrip())
1333                                 self._lbuf = ''
1334                                 if opts.debug >= 2:
1335                                         debug(self.__str__())
1336                 self._expire_jobbers(time.time())
1337
1338                 if progress: progress.caughtup()
1339
1340         def changed(self):
1341                 rv = self._need_redisplay
1342                 self._need_redisplay = False
1343                 return rv
1344         def myname(self):
1345                 # returns our pirate name
1346                 return self._myself.name
1347         def vesselname(self):
1348                 # returns the vessel name we're aboard or None
1349                 return self._vessel
1350         def lastvesselname(self):
1351                 # returns the last vessel name we were aboard or None
1352                 return self._lastvessel
1353         def aboard(self, vesselname=True):
1354                 # returns a list of PirateAboard the vessel
1355                 #  sorted by pirate name
1356                 #  you can pass this None and you'll get []
1357                 #  or True for the current vessel (which is the default)
1358                 #  the returned value is a fresh list of persistent
1359                 #  PirateAboard objects
1360                 if vesselname is True: v = self._v
1361                 else: v = self._vl.get(vesselname.title())
1362                 if v is None: return []
1363                 return [ v[pn]
1364                          for pn in sorted(v.keys())
1365                          if not pn.startswith('#') ]
1366         def jobbers(self):
1367                 # returns a the jobbers' PirateAboards,
1368                 # sorted by jobber class and reverse of expiry time
1369                 l = [ pa
1370                       for pa in self._pl.values()
1371                       if pa.jobber is not None
1372                     ]
1373                 def compar_key(pa):
1374                         return (pa.jobber, -pa.expires)
1375                 l.sort(key = compar_key)
1376                 return l
1377
1378 #---------- implementations of actual operation modes ----------
1379
1380 def do_pirate(pirates, bu):
1381         print '{'
1382         for pirate in pirates:
1383                 info = PirateInfo(pirate)
1384                 print '%s: %s,' % (`pirate`, info)
1385         print '}'
1386
1387 def prep_crewflag_of(args, bu, max_age, selector, constructor):
1388         if len(args) != 1: bu('crew-of etc. take one pirate name')
1389         pi = PirateInfo(args[0], max_age)
1390         cf = selector(pi)
1391         if cf is None: return None
1392         return constructor(cf[0], max_age)
1393
1394 def prep_crew_of(args, bu, max_age=300):
1395         return prep_crewflag_of(args, bu, max_age,
1396                 (lambda pi: pi.crew), CrewInfo)
1397
1398 def prep_flag_of(args, bu, max_age=300):
1399         return prep_crewflag_of(args, bu, max_age,
1400                 (lambda pi: pi.flag), FlagInfo)
1401
1402 def do_crew_of(args, bu):
1403         ci = prep_crew_of(args, bu)
1404         print ci
1405
1406 def do_flag_of(args, bu):
1407         fi = prep_flag_of(args, bu)
1408         print fi
1409
1410 def do_standings_crew_of(args, bu):
1411         ci = prep_crew_of(args, bu, 60)
1412         tab = StandingsTable(sys.stdout)
1413         tab.headings()
1414         for (rank, members) in ci.crew:
1415                 if not members: continue
1416                 tab.literalline('')
1417                 tab.literalline('%s:' % rank)
1418                 for p in members:
1419                         pi = PirateInfo(p, random.randint(900,1800))
1420                         tab.pirate(pi)
1421
1422 def do_ocean(args, bu):
1423         if (len(args)): bu('ocean takes no further arguments')
1424         fetcher.default_ocean()
1425         oi = OceanInfo(IslandFlagInfo)
1426         print oi
1427         for islename in sorted(oi.islands.keys()):
1428                 isle = oi.islands[islename]
1429                 print isle
1430
1431 def do_embargoes(args, bu):
1432         if (len(args)): bu('ocean takes no further arguments')
1433         fetcher.default_ocean()
1434         oi = OceanInfo(IslandFlagInfo)
1435         wr = sys.stdout.write
1436         print ('EMBARGOES:  Island    | Owning flag'+
1437                 '                    | Embargoed flags')
1438
1439         def getflname(isle):
1440                 if isle.islandid is None: return 'uncolonisable'
1441                 if isle.flag is None: return 'uncolonised'
1442                 return isle.flag.name
1443
1444         progressreporter.stop()
1445
1446         for archname in sorted(oi.arches.keys()):
1447                 print 'ARCHIPELAGO: ',archname
1448                 for islename in sorted(oi.arches[archname].keys()):
1449                         isle = oi.islands[islename]
1450                         wr(' %-20s | ' % isle.name)
1451                         flname = getflname(isle)
1452                         wr('%-30s | ' % flname)
1453                         flag = isle.flag
1454                         if flag is None: print ''; continue
1455                         delim = ''
1456                         for rel in flag.relations:
1457                                 if rel.this_declaring >= 0: continue
1458                                 wr(delim)
1459                                 wr(rel.other_flagname)
1460                                 delim = '; '
1461                         print ''
1462
1463 def do_embargoes_flag_of(args, bu):
1464         progressreporter.doing('fetching flag info')
1465         fi = prep_flag_of(args, bu)
1466         if fi is None:
1467                 progressreporter.stop()
1468                 print 'Pirate is not in a flag.'
1469                 return
1470
1471         oi = OceanInfo(IslandFlagInfo)
1472
1473         progressreporter.stop()
1474         print ''
1475
1476         any = False
1477         for islename in sorted(oi.islands.keys()):
1478                 isle = oi.islands[islename]
1479                 flag = isle.flag
1480                 if flag is None: continue
1481                 for rel in flag.relations:
1482                         if rel.this_declaring >= 0: continue
1483                         if rel.other_flagid != fi.flagid: continue
1484                         if not any: print 'EMBARGOED:'
1485                         any = True
1486                         print "  %-30s (%s)" % (islename, flag.name)
1487         if not any:
1488                 print 'No embargoes.'
1489         print ''
1490
1491         war_flag(fi)
1492         print ''
1493
1494 def do_war_flag_of(args, bu):
1495         fi = prep_flag_of(args, bu)
1496         war_flag(fi)
1497
1498 def war_flag(fi):
1499         any = False
1500         for certain in [True, False]:
1501                 anythis = False
1502                 for rel in fi.relations:
1503                         if rel.this_declaring >= 0: continue
1504                         if (rel.other_declaring_max < 0) != certain: continue
1505                         if not anythis:
1506                                 if certain: m = 'SINKING PvP'
1507                                 else: m = 'RISK OF SINKING PvP'
1508                                 print '%s (%s):' % (m, rel.yoweb_heading)
1509                         anythis = True
1510                         any = True
1511                         print " ", rel.other_flagname
1512         if not any:
1513                 print 'No sinking PvP.'
1514
1515 #----- modes which use the chat log parser are quite complex -----
1516
1517 class ProgressPrintPercentage:
1518         def __init__(self, f=sys.stdout):
1519                 self._f = f
1520         def progress_string(self,done,total):
1521                 return "scan chat logs %3d%%\r" % ((done*100) / total)
1522         def progress(self,*a):
1523                 self._f.write(self.progress_string(*a))
1524                 self._f.flush()
1525         def show_init(self, pirate, ocean):
1526                 print >>self._f, 'Starting up, %s on the %s ocean' % (
1527                         pirate, ocean)
1528         def caughtup(self):
1529                 self._f.write('                   \r')
1530                 self._f.flush()
1531
1532 def prep_chat_log(args, bu,
1533                 progress=ProgressPrintPercentage(),
1534                 max_myself_age=3600):
1535         if len(args) != 1: bu('this action takes only chat log filename')
1536         logfn = args[0]
1537         logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_'
1538         match = regexp.match(logfn_re, logfn)
1539         if not match: bu('chat log filename is not in expected format')
1540         (pirate, ocean) = match.groups()
1541         fetcher.default_ocean(ocean)
1542
1543         progress.show_init(pirate, fetcher.ocean)
1544         myself = PirateInfo(pirate,max_myself_age)
1545         track = ChatLogTracker(myself, logfn)
1546
1547         opts.debug -= 2
1548         track.catchup(progress)
1549         opts.debug += 2
1550
1551         track.force_redisplay()
1552
1553         return (myself, track)
1554
1555 def do_track_chat_log(args, bu):
1556         (myself, track) = prep_chat_log(args, bu)
1557         while True:
1558                 track.catchup()
1559                 if track.changed():
1560                         print track
1561                 sleep(0.5 + 0.5 * random.random())
1562
1563 #----- ship management aid -----
1564
1565 class Display_dumb(ProgressPrintPercentage):
1566         def __init__(self):
1567                 ProgressPrintPercentage.__init__(self)
1568         def show(self, s):
1569                 print '\n\n', s;
1570         def realstart(self):
1571                 pass
1572
1573 class Display_overwrite(ProgressPrintPercentage):
1574         def __init__(self):
1575                 ProgressPrintPercentage.__init__(self)
1576
1577                 null = file('/dev/null','w')
1578                 curses.setupterm(fd=null.fileno())
1579
1580                 self._clear = curses.tigetstr('clear')
1581                 if not self._clear:
1582                         self._debug('missing clear!')
1583                         self.show = Display_dumb.show
1584                         return
1585
1586                 self._t = {'el':'', 'ed':''}
1587                 if not self._init_sophisticated():
1588                         for k in self._t.keys(): self._t[k] = ''
1589                         self._t['ho'] = self._clear
1590
1591         def _debug(self,m): debug('display overwrite: '+m)
1592
1593         def _init_sophisticated(self):
1594                 for k in self._t.keys():
1595                         s = curses.tigetstr(k)
1596                         self._t[k] = s
1597                 self._t['ho'] = curses.tigetstr('ho')
1598                 if not self._t['ho']:
1599                         cup = curses.tigetstr('cup')
1600                         self._t['ho'] = curses.tparm(cup,0,0)
1601                 missing = [k for k in self._t.keys() if not self._t[k]]
1602                 if missing:
1603                         self.debug('missing '+(' '.join(missing)))
1604                         return 0
1605                 return 1
1606
1607         def show(self, s):
1608                 w = sys.stdout.write
1609                 def wti(k): w(self._t[k])
1610
1611                 wti('ho')
1612                 nl = ''
1613                 for l in s.rstrip().split('\n'):
1614                         w(nl)
1615                         w(l)
1616                         wti('el')
1617                         nl = '\r\n'
1618                 wti('ed')
1619                 w(' ')
1620                 sys.stdout.flush()
1621
1622         def realstart(self):
1623                 sys.stdout.write(self._clear)
1624                 sys.stdout.flush()
1625
1626
1627 def do_ship_aid(args, bu):
1628         if opts.ship_duty is None: opts.ship_duty = True
1629
1630         displayer = globals()['Display_'+opts.display]()
1631
1632         (myself, track) = prep_chat_log(args, bu, progress=displayer)
1633
1634         displayer.realstart()
1635
1636         if os.isatty(0): kr_create = KeystrokeReader
1637         else: kr_create = DummyKeystrokeReader
1638
1639         try:
1640                 kreader = kr_create(0, 10)
1641                 ship_aid_core(myself, track, displayer, kreader)
1642         finally:
1643                 kreader.stop()
1644                 print '\n'
1645
1646 class KeyBasedSorter:
1647         def compar_key_pa(self, pa):
1648                 pi = pa.pirate_info()
1649                 if pi is None: return None
1650                 return self.compar_key(pi)
1651         def lsort_pa(self, l):
1652                 l.sort(key = self.compar_key_pa)
1653
1654 class NameSorter(KeyBasedSorter):
1655         def compar_key(self, pi): return pi.name
1656         def desc(self): return 'name'
1657
1658 class SkillSorter(NameSorter):
1659         def __init__(self, relevant):
1660                 self._want = frozenset(relevant.split('/'))
1661                 self._avoid = set()
1662                 for p in core_duty_puzzles:
1663                         if isinstance(p,basestring): self._avoid.add(p)
1664                         else: self._avoid |= set(p)
1665                 self._avoid -= self._want
1666                 self._desc = '%s' % relevant
1667
1668         def desc(self): return self._desc
1669
1670         def compar_key(self, pi):
1671                 best_want = max([
1672                         pi.standings.get(puz,-1)
1673                         for puz in self._want
1674                         ])
1675                 best_avoid = [
1676                         -pi.standings.get(puz,standing_limit)
1677                         for puz in self._avoid
1678                         ]
1679                 best_avoid.sort()
1680                 def negate(x): return -x
1681                 debug('compar_key %s bw=%s ba=%s' % (pi.name, `best_want`,
1682                         `best_avoid`))
1683                 return (-best_want, map(negate, best_avoid), pi.name)
1684
1685 def ship_aid_core(myself, track, displayer, kreader):
1686
1687         def find_vessel():
1688                 vn = track.vesselname()
1689                 if vn: return (vn, " on board the %s" % vn)
1690                 vn = track.lastvesselname()
1691                 if vn: return (vn, " ashore from the %s" % vn)
1692                 return (None, " not on a vessel")
1693
1694         def timeevent(t,e):
1695                 if t is None: return ' ' * 22
1696                 return " %-4s %-16s" % (format_time_interval(now - t),e)
1697
1698         displayer.show(track.myname() + find_vessel()[1] + '...')
1699
1700         rotate_nya = '/-\\'
1701
1702         sort = NameSorter()
1703         clicmd = None
1704         clierr = None
1705         cliexec = None
1706
1707         while True:
1708                 track.catchup()
1709                 now = time.time()
1710
1711                 (vn, vs) = find_vessel()
1712
1713                 s = ''
1714                 if cliexec is not None:
1715                         s += '...'
1716                 elif clierr is not None:
1717                         s += 'Error: '+clierr
1718                 elif clicmd is not None:
1719                         s += '/' + clicmd
1720                 else:
1721                         s = track.myname() + vs
1722                         s += " at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
1723                         s += kreader.info()
1724                 s += '\n'
1725
1726                 tbl_s = StringIO()
1727                 tbl = StandingsTable(tbl_s)
1728
1729                 aboard = track.aboard(vn)
1730                 sort.lsort_pa(aboard)
1731
1732                 jobbers = track.jobbers()
1733
1734                 if track.vesselname(): howmany = 'aboard: %2d' % len(aboard)
1735                 else: howmany = ''
1736
1737                 tbl.headings(howmany, '  sorted by '+sort.desc())
1738
1739                 last_jobber = None
1740
1741                 for pa in aboard + jobbers:
1742                         if pa.jobber != last_jobber:
1743                                 last_jobber = pa.jobber
1744                                 tbl.literalline('')
1745                                 tbl.literalline('jobbers '+last_jobber)
1746
1747                         pi = pa.pirate_info()
1748
1749                         xs = ''
1750                         if pa.gunner: xs += 'G '
1751                         else: xs += '  '
1752                         xs += timeevent(pa.last_time, pa.last_event)
1753                         xs += timeevent(pa.last_chat_time, pa.last_chat_chan)
1754
1755                         if pi is None:
1756                                 tbl.pirate_dummy(pa.name, rotate_nya[0], xs)
1757                         else:
1758                                 tbl.pirate(pi, xs)
1759
1760                 s += tbl_s.getvalue()
1761                 displayer.show(s)
1762                 tbl_s.close()
1763
1764                 if cliexec is not None:
1765                         clierr = track.local_command("/"+cliexec.strip())
1766                         cliexec = None
1767                         continue
1768
1769                 k = kreader.getch()
1770                 if k is None:
1771                         rotate_nya = rotate_nya[1:3] + rotate_nya[0]
1772                         continue
1773
1774                 if clierr is not None:
1775                         clierr = None
1776                         continue
1777
1778                 if clicmd is not None:
1779                         if k == '\r' or k == '\n':
1780                                 cliexec = clicmd
1781                                 clicmd = clicmdbase
1782                         elif k == '\e' and clicmd != "":
1783                                 clicmd = clicmdbase
1784                         elif k == '\33':
1785                                 clicmd = None
1786                         elif k == '\b' or k == '\177':
1787                                 clicmd = clicmd[ 0 : len(clicmd)-1 ]
1788                         else:
1789                                 clicmd += k
1790                         continue
1791
1792                 if k == 'q': break
1793                 elif k == 'g': sort = SkillSorter('Gunning')
1794                 elif k == 'c': sort = SkillSorter('Carpentry')
1795                 elif k == 's': sort = SkillSorter('Sailing/Rigging')
1796                 elif k == 'b': sort = SkillSorter('Bilging')
1797                 elif k == 'n': sort = SkillSorter('Navigating')
1798                 elif k == 'd': sort = SkillSorter('Battle Navigation')
1799                 elif k == 't': sort = SkillSorter('Treasure Haul')
1800                 elif k == 'a': sort = NameSorter()
1801                 elif k == '/': clicmdbase = ""; clicmd = clicmdbase
1802                 elif k == '+': clicmdbase = "a "; clicmd = clicmdbase
1803                 else: pass # unknown key command
1804
1805 #---------- individual keystroke input ----------
1806
1807 class DummyKeystrokeReader:
1808         def __init__(self,fd,timeout_dummy): pass
1809         def stop(self): pass
1810         def getch(self): sleep(1); return None
1811         def info(self): return ' [noninteractive]'
1812
1813 class KeystrokeReader(DummyKeystrokeReader):
1814         def __init__(self, fd, timeout_decisec=0):
1815                 self._fd = fd
1816                 self._saved = termios.tcgetattr(fd)
1817                 a = termios.tcgetattr(fd)
1818                 a[3] &= ~(termios.ECHO | termios.ECHONL |
1819                           termios.ICANON | termios.IEXTEN)
1820                 a[6][termios.VMIN] = 0
1821                 a[6][termios.VTIME] = timeout_decisec
1822                 termios.tcsetattr(fd, termios.TCSANOW, a)
1823         def stop(self):
1824                 termios.tcsetattr(self._fd, termios.TCSANOW, self._saved)
1825         def getch(self):
1826                 debug_flush()
1827                 byte = os.read(self._fd, 1)
1828                 if not len(byte): return None
1829                 return byte
1830         def info(self):
1831                 return ''
1832
1833 #---------- main program ----------
1834
1835 def main():
1836         global opts, fetcher, yppedia, progressreporter
1837
1838         pa = OptionParser(
1839 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
1840 actions:
1841  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
1842  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
1843  yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
1844  yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG
1845  yoweb-scrape [--ocean OCEAN ...] ocean|embargoes
1846  yoweb-scrape [--ocean OCEAN ...] war-flag-of|embargoes-flag-of PIRATE
1847  yoweb-scrape [options] ship-aid CHAT-LOG  (must be .../PIRATE_OCEAN_chat-log*)
1848
1849 display modes (for --display) apply to ship-aid:
1850  --display=dumb       just print new information, scrolling the screen
1851  --display=overwrite  use cursor motion, selective clear, etc. to redraw at top''')
1852         ao = pa.add_option
1853         ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
1854                 help='select ocean OCEAN')
1855         ao('--cache-dir', dest='cache_dir', metavar='DIR',
1856                 default='~/.yoweb-scrape-cache',
1857                 help='cache yoweb pages in DIR')
1858         ao('-D','--debug', action='count', dest='debug', default=0,
1859                 help='enable debugging output')
1860         ao('--debug-fd', type='int', dest='debug_fd',
1861                 help='write any debugging output to specified fd')
1862         ao('-q','--quiet', action='store_true', dest='quiet',
1863                 help='suppress warning output')
1864         ao('--display', action='store', dest='display',
1865                 type='choice', choices=['dumb','overwrite'],
1866                 help='how to display ship aid')
1867         ao('--local-ypp-dir', action='store', dest='localhtml',
1868                 help='get yppedia pages from local directory LOCALHTML'+
1869                         ' instead of via HTTP')
1870
1871         ao_jt = lambda wh, t: ao(
1872                 '--timeout-sa-'+wh, action='store', dest='timeout_'+wh,
1873                 default=t, help=('set timeout for expiring %s jobbers' % wh))
1874         ao_jt('applied',      120)
1875         ao_jt('invited',      120)
1876         ao_jt('declined',      30)
1877         ao_jt('ashore',      1800)
1878
1879         ao('--ship-duty', action='store_true', dest='ship_duty',
1880                 help='show ship duty station puzzles')
1881         ao('--all-puzzles', action='store_false', dest='ship_duty',
1882                 help='show all puzzles, not just ship duty stations')
1883
1884         ao('--min-cache-reuse', type='int', dest='min_max_age',
1885                 metavar='SECONDS', default=60,
1886                 help='always reuse cache yoweb data if no older than this')
1887
1888         (opts,args) = pa.parse_args()
1889         random.seed()
1890
1891         if len(args) < 1:
1892                 print >>sys.stderr, copyright_info
1893                 pa.error('need a mode argument')
1894
1895         if opts.debug_fd is not None:
1896                 opts.debug_file = os.fdopen(opts.debug_fd, 'w')
1897         else:
1898                 opts.debug_file = sys.stdout
1899
1900         mode = args[0]
1901         mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
1902         try: mode_fn = globals()[mode_fn_name]
1903         except KeyError: pa.error('unknown mode "%s"' % mode)
1904
1905         # fixed parameters
1906         opts.expire_age = max(3600, opts.min_max_age)
1907
1908         opts.ship_reboard_clearout = 3600
1909
1910         if opts.cache_dir.startswith('~/'):
1911                 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
1912
1913         if opts.display is None:
1914                 if ((opts.debug > 0 and opts.debug_fd is None)
1915                     or not os.isatty(sys.stdout.fileno())):
1916                         opts.display = 'dumb'
1917                 else:
1918                         opts.display = 'overwrite'
1919
1920         fetcher = Yoweb(opts.ocean, opts.cache_dir)
1921         yppedia = Yppedia(opts.cache_dir)
1922
1923         if opts.debug or not os.isatty(0):
1924                  progressreporter = NullProgressReporter()
1925         else:
1926                 progressreporter = TypewriterProgressReporter()
1927
1928         mode_fn(args[1:], pa.error)
1929
1930 main()