yoweb-scrape

   1 #!/usr/bin/python
   2 # This is part of ypp-sc-tools, a set of third-party tools for assisting
   3 # players of Yohoho Puzzle Pirates.
   4 #
   5 # Copyright (C) 2009 Ian Jackson <ijackson@chiark.greenend.org.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 #
  20 # Yohoho and Puzzle Pirates are probably trademarks of Three Rings and
  21 # are used without permission.  This program is not endorsed or
  22 # sponsored by Three Rings.
  23
  24 copyright_info = '''
  25 yoweb-scrape is part of ypp-sc-tools  Copyright (C) 2009 Ian Jackson
  26 This program comes with ABSOLUTELY NO WARRANTY; this is free software,
  27 and you are welcome to redistribute it under certain conditions.
  28 For details, read the top of the yoweb-scrape file.
  29 '''
  30
  31 #---------- setup ----------
  32
  33 import signal
  34 signal.signal(signal.SIGINT, signal.SIG_DFL)
  35
  36 import os
  37 import time
  38 import urllib
  39 import urllib2
  40 import errno
  41 import sys
  42 import re as regexp
  43 import random
  44 import curses
  45 import termios
  46 import random
  47 import subprocess
  48 import copy
  49 from optparse import OptionParser
  50 from StringIO import StringIO
  51
  52 from BeautifulSoup import BeautifulSoup
  53
  54 opts = None
  55
  56 #---------- YPP parameters and arrays ----------
  57
  58 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
  59         '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
  60         '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
  61         '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
  62
  63 core_duty_puzzles = [
  64                 'Gunning',
  65                 ['Sailing','Rigging'],
  66                 'Bilging',
  67                 'Carpentry',
  68                 ]
  69
  70 duty_puzzles = ([ 'Navigating', 'Battle Navigation' ] +
  71                 core_duty_puzzles +
  72                 [ 'Treasure Haul' ])
  73
  74 standingvals = ('Able/Proficient/Distinguished/Respected/Master'+
  75                 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
  76 standing_limit = len(standingvals)
  77
  78 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
  79
  80 max_pirate_namelen = 12
  81
  82
  83 #---------- general utilities ----------
  84
  85 def debug(m):
  86         if opts.debug > 0:
  87                 print >>opts.debug_file, m
  88
  89 def debug_flush():
  90         if opts.debug > 0:
  91                 opts.debug_file.flush()
  92
  93 def sleep(seconds):
  94         debug_flush()
  95         time.sleep(seconds)
  96
  97 def format_time_interval(ti):
  98         if ti < 120: return '%d:%02d' % (ti / 60, ti % 60)
  99         if ti < 7200: return '%2dm' % (ti / 60)
 100         if ti < 86400: return '%dh' % (ti / 3600)
 101         return '%dd' % (ti / 86400)
 102
 103 def yppsc_dir():
 104         lib = os.getenv("YPPSC_YARRG_SRCBASE")
 105         if lib is not None: return lib
 106         lib = sys.argv[0]
 107         lib = regexp.sub('/[^/]+$', '', lib)
 108         os.environ["YPPSC_YARRG_SRCBASE"] = lib
 109         return lib
 110
 111 soup_massage = copy.copy(BeautifulSoup.MARKUP_MASSAGE)
 112 soup_massage.append(
 113                 (regexp.compile('(\<td.*") ("center")'),
 114                  lambda m: m.group(1)+' align='+m.group(2))
 115         )
 116
 117 def make_soup(*args, **kwargs):
 118         return BeautifulSoup(*args,
 119                 convertEntities=BeautifulSoup.HTML_ENTITIES,
 120                 markupMassage=soup_massage,
 121                          **kwargs)
 122
 123 #---------- caching and rate-limiting data fetcher ----------
 124
 125 class Fetcher:
 126         def __init__(self, cachedir):
 127                 debug('Fetcher init %s' % cachedir)
 128                 self.cachedir = cachedir
 129                 try: os.mkdir(cachedir)
 130                 except (OSError,IOError), oe:
 131                         if oe.errno != errno.EEXIST: raise
 132                 self._cache_scan(time.time())
 133
 134         def _match_url_normalise(self, url):
 135                 without_scheme = regexp.sub('^[a-z]+://', '', url)
 136                 without_tail = regexp.sub('/.*', '', without_scheme)
 137                 return without_tail
 138
 139         def _cache_scan(self, now, match_url=None):
 140                 # returns list of ages, unsorted
 141                 if match_url is not None:
 142                         match_url = self._match_url_normalise(match_url)
 143                 ages = []
 144                 debug('Fetcher   scan_cache')
 145                 for leaf in os.listdir(self.cachedir):
 146                         if not leaf.startswith('#'): continue
 147                         if match_url is not None:
 148                                 leaf_url = urllib.unquote_plus(leaf.strip('#'))
 149                                 leaf_url = self._match_url_normalise(leaf_url)
 150                                 if leaf_url != match_url:
 151                                         continue
 152                         path = self.cachedir + '/' + leaf
 153                         try: s = os.stat(path)
 154                         except (OSError,IOError), oe:
 155                                 if oe.errno != errno.ENOENT: raise
 156                                 continue
 157                         age = now - s.st_mtime
 158                         if age > opts.expire_age:
 159                                 debug('Fetcher    expire %d %s' % (age, path))
 160                                 try: os.remove(path)
 161                                 except (OSError,IOError), oe:
 162                                         if oe.errno != errno.ENOENT: raise
 163                                 continue
 164                         ages.append(age)
 165                 return ages
 166
 167         def need_wait(self, now, imaginary=[], next_url=None):
 168                 ages = self._cache_scan(now, match_url=next_url)
 169                 ages += imaginary
 170                 ages.sort()
 171                 debug('Fetcher   ages ' + `ages`)
 172                 min_age = 1
 173                 need_wait = 0
 174                 for age in ages:
 175                         if age < min_age and age <= 5:
 176                                 debug('Fetcher   morewait min=%d age=%d' %
 177                                         (min_age, age))
 178                                 need_wait = max(need_wait, min_age - age)
 179                         min_age += 3
 180                 if need_wait > 0:
 181                         need_wait += random.random() - 0.5
 182                 return need_wait
 183
 184         def _rate_limit_cache_clean(self, now, next_url=None):
 185                 need_wait = self.need_wait(now, next_url=next_url)
 186                 if need_wait > 0:
 187                         debug('Fetcher   wait %f' % need_wait)
 188                         sleep(need_wait)
 189
 190         def fetch(self, url, max_age):
 191                 debug('Fetcher fetch %s' % url)
 192                 cache_corename = urllib.quote_plus(url)
 193                 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
 194                 try: f = file(cache_item, 'r')
 195                 except (OSError,IOError), oe:
 196                         if oe.errno != errno.ENOENT: raise
 197                         f = None
 198                 now = time.time()
 199                 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
 200                 if f is not None:
 201                         s = os.fstat(f.fileno())
 202                         age = now - s.st_mtime
 203                         if age > max_age:
 204                                 debug('Fetcher  stale %d < %d'% (max_age, age))
 205                                 f = None
 206                 if f is not None:
 207                         data = f.read()
 208                         f.close()
 209                         debug('Fetcher  cached %d > %d' % (max_age, age))
 210                         return data
 211
 212                 debug('Fetcher  fetch')
 213                 self._rate_limit_cache_clean(now, next_url=url)
 214
 215                 stream = urllib2.urlopen(url)
 216                 data = stream.read()
 217                 cache_tmp = "%s/#%s~%d#" % (
 218                         self.cachedir, cache_corename, os.getpid())
 219                 f = file(cache_tmp, 'w')
 220                 f.write(data)
 221                 f.close()
 222                 os.rename(cache_tmp, cache_item)
 223                 debug('Fetcher  stored')
 224                 return data
 225
 226 class Yoweb(Fetcher):
 227         def __init__(self, ocean, cachedir):
 228                 debug('Yoweb init %s' % cachedir)
 229                 self.ocean = ocean
 230                 Fetcher.__init__(self, cachedir)
 231
 232         def default_ocean(self, ocean='ice'):
 233                 if self.ocean is None:
 234                         self.ocean = ocean
 235
 236         def yoweb(self, kind, tail, max_age):
 237                 self.default_ocean()
 238                 assert(self.ocean)
 239                 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
 240                         self.ocean, kind, tail)
 241                 return self.fetch(url, max_age)
 242
 243 class Yppedia(Fetcher):
 244         def __init__(self, cachedir):
 245                 debug('Yoweb init %s' % cachedir)
 246                 self.base = 'http://yppedia.puzzlepirates.com/'
 247                 self.localhtml = opts.localhtml
 248                 Fetcher.__init__(self, cachedir)
 249
 250         def __call__(self, rhs):
 251                 if self.localhtml is None:
 252                         url = self.base + rhs
 253                         debug('Yppedia retrieving YPP '+url);
 254                         return self.fetch(url, 3000)
 255                 else:
 256                         return file(opts.localhtml + '/' + rhs, 'r')
 257
 258 #---------- logging assistance for troubled screenscrapers ----------
 259
 260 class SoupLog:
 261         def __init__(self):
 262                 self.msgs = [ ]
 263         def msg(self, m):
 264                 self.msgs.append(m)
 265         def soupm(self, obj, m):
 266                 self.msg(m + '; in ' + `obj`)
 267         def needs_msgs(self, child_souplog):
 268                 self.msgs += child_souplog.msgs
 269                 child_souplog.msgs = [ ]
 270
 271 def soup_text(obj):
 272         str = ''.join(obj.findAll(text=True))
 273         return str.strip()
 274
 275 class SomethingSoupInfo(SoupLog):
 276         def __init__(self, kind, tail, max_age):
 277                 SoupLog.__init__(self)
 278                 html = fetcher.yoweb(kind, tail, max_age)
 279                 self._soup = make_soup(html)
 280
 281 #---------- scraper for pirate pages ----------
 282
 283 class PirateInfo(SomethingSoupInfo):
 284         # Public data members:
 285         #  pi.standings = { 'Treasure Haul': 'Able' ... }
 286         #  pi.name = name
 287         #  pi.crew = (id, name)
 288         #  pi.flag = (id, name)
 289         #  pi.msgs = [ 'message describing problem with scrape' ]
 290
 291         def __init__(self, pirate, max_age=300):
 292                 SomethingSoupInfo.__init__(self,
 293                         'pirate.wm?target=', pirate, max_age)
 294                 self.name = pirate
 295                 self._find_standings()
 296                 self.crew = self._find_crewflag('crew',
 297                         '^/yoweb/crew/info\\.wm')
 298                 self.flag = self._find_crewflag('flag',
 299                         '^/yoweb/flag/info\\.wm')
 300
 301         def _find_standings(self):
 302                 imgs = self._soup.findAll('img',
 303                         src=regexp.compile('/yoweb/images/stat.*'))
 304                 re = regexp.compile(
 305 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
 306                         )
 307                 standings = { }
 308
 309                 for skill in puzzles:
 310                         standings[skill] = [ ]
 311
 312                 skl = SoupLog()
 313
 314                 for img in imgs:
 315                         try: puzzle = img['alt']
 316                         except KeyError: continue
 317
 318                         if not puzzle in puzzles:
 319                                 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
 320                                 continue
 321                         key = img.findParent('td')
 322                         if key is None:
 323                                 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
 324                                 continue
 325                         valelem = key.findNextSibling('td')
 326                         if valelem is None:
 327                                 skl.soupm(key, 'puzzle missing sibling "%s"'
 328                                         % puzzle)
 329                                 continue
 330                         valstr = soup_text(valelem)
 331                         match = re.match(valstr)
 332                         if match is None:
 333                                 skl.soupm(key, ('puzzle "%s" unparseable'+
 334                                         ' standing "%s"') % (puzzle, valstr))
 335                                 continue
 336                         standing = match.group(match.lastindex)
 337                         standings[puzzle].append(standing)
 338
 339                 self.standings = { }
 340
 341                 for puzzle in puzzles:
 342                         sl = standings[puzzle]
 343                         if len(sl) > 1:
 344                                 skl.msg('puzzle "%s" multiple standings %s' %
 345                                                 (puzzle, `sl`))
 346                                 continue
 347                         if not sl:
 348                                 skl.msg('puzzle "%s" no standing found' % puzzle)
 349                                 continue
 350                         standing = sl[0]
 351                         for i in range(0, standing_limit):
 352                                 if standing == standingvals[i]:
 353                                         self.standings[puzzle] = i
 354                         if not puzzle in self.standings:
 355                                 skl.msg('puzzle "%s" unknown standing "%s"' %
 356                                         (puzzle, standing))
 357
 358                 all_standings_ok = True
 359                 for puzzle in puzzles:
 360                         if not puzzle in self.standings:
 361                                 self.needs_msgs(skl)
 362
 363         def _find_crewflag(self, cf, yoweb_re):
 364                 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
 365                 if len(things) != 1:
 366                         self.msg('zero or several %s id references found' % cf)
 367                         return None
 368                 thing = things[0]
 369                 id_re = '\\b%sid\\=(\\w+)$' % cf
 370                 id_haystack = thing['href']
 371                 match = regexp.compile(id_re).search(id_haystack)
 372                 if match is None:
 373                         self.soupm(thing, ('incomprehensible %s id ref'+
 374                                 ' (%s in %s)') % (cf, id_re, id_haystack))
 375                         return None
 376                 name = soup_text(thing)
 377                 return (match.group(1), name)
 378
 379         def __str__(self):
 380                 return `(self.crew, self.flag, self.standings, self.msgs)`
 381
 382 #---------- scraper for crew pages ----------
 383
 384 class CrewInfo(SomethingSoupInfo):
 385         # Public data members:
 386         #  ci.crewid
 387         #  ci.crew = [ ('Captain',        ['Pirate', ...]),
 388         #              ('Senior Officer', [...]),
 389         #               ... ]
 390         #  pi.msgs = [ 'message describing problem with scrape' ]
 391
 392         def __init__(self, crewid, max_age=300):
 393                 self.crewid = crewid
 394                 SomethingSoupInfo.__init__(self,
 395                         'crew/info.wm?crewid=', crewid, max_age)
 396                 self._find_crew()
 397
 398         def _find_crew(self):
 399                 self.crew = []
 400                 capts = self._soup.findAll('img',
 401                         src='/yoweb/images/crew-captain.png')
 402                 if len(capts) != 1:
 403                         self.msg('crew members: no. of captain images != 1')
 404                         return
 405                 tbl = capts[0]
 406                 while not tbl.find('a', href=pirate_ref_re):
 407                         tbl = tbl.findParent('table')
 408                         if not tbl:
 409                                 self.msg('crew members: cannot find table')
 410                                 return
 411                 current_rank_crew = None
 412                 crew_rank_re = regexp.compile('/yoweb/images/crew')
 413                 for row in tbl.contents:
 414                         # findAll(recurse=False)
 415                         if isinstance(row,basestring):
 416                                 continue
 417
 418                         is_rank = row.find('img', attrs={'src': crew_rank_re})
 419                         if is_rank:
 420                                 rank = soup_text(row)
 421                                 current_rank_crew = []
 422                                 self.crew.append((rank, current_rank_crew))
 423                                 continue
 424                         for cell in row.findAll('a', href=pirate_ref_re):
 425                                 if current_rank_crew is None:
 426                                         self.soupm(cell, 'crew members: crew'
 427                                                 ' before rank')
 428                                         continue
 429                                 current_rank_crew.append(soup_text(cell))
 430
 431         def __str__(self):
 432                 return `(self.crew, self.msgs)`
 433
 434 class FlagRelation():
 435         # Public data members (put there by hand by creater)
 436         #       other_flagname
 437         #       other_flagid
 438         #       yoweb_heading
 439         #       this_declaring
 440         #       other_declaring_min
 441         #       other_declaring_max
 442         # where {this,other}_declaring{,_min,_max} are:
 443         #       -1      {this,other} is declaring war
 444         #        0      {this,other} is not doing either
 445         #       +1      {this,other} is allying
 446         def __repr__(self):
 447                 return '<FlagRelation %s %d/%d..%d %s %s>' % (
 448                         self.yoweb_heading, self.this_declaring,
 449                         self.other_declaring_min, self.other_declaring_max,
 450                         self.other_flagname, self.other_flagid)
 451
 452 class FlagInfo(SomethingSoupInfo):
 453         # Public data members (after init):
 454         #
 455         #   flagid
 456         #   name        #               string
 457         #
 458         #   relations[n] = FlagRelation
 459         #   relation_byname[otherflagname] = relations[some_n]
 460         #   relation_byid[otherflagname] = relations[some_n]
 461         #
 462         #   islands[n] = (islandname, islandid)
 463         #
 464         def __init__(self, flagid, max_age=600):
 465                 self.flagid = flagid
 466                 SomethingSoupInfo.__init__(self,
 467                         'flag/info.wm?flagid=', flagid, max_age)
 468                 self._find_flag()
 469
 470         def _find_flag(self):
 471                 font2 = self._soup.find('font',{'size':'+2'})
 472                 self.name = font2.find('b').contents[0]
 473
 474                 self.relations = [ ]
 475                 self.relation_byname = { }
 476                 self.relation_byid = { }
 477                 self.islands = [ ]
 478
 479                 magnate = self._soup.find('img',{'src':
 480                         '/yoweb/images/repute-MAGNATE.png'})
 481                 warinfo = (magnate.findParent('table').findParent('tr').
 482                         findNextSibling('tr').findNext('td',{'align':'left'}))
 483
 484                 def warn(m):
 485                         print >>sys.stderr, 'WARNING: '+m
 486
 487                 def wi_warn(head, waritem):
 488                         warn('unknown warmap item: %s: %s' %
 489                                 (`head`, ``waritem``))
 490
 491                 def wihelp_item(waritem, thing):
 492                         url = waritem.get('href', None)
 493                         if url is None:
 494                                 return ('no url for '+thing,None,None)
 495                         m = regexp.search('\?'+thing+'id=(\d+)$', url)
 496                         if not m: return ('no '+thing+'id',None,None)
 497                         tid = m.group(1)
 498                         tname = waritem.string
 499                         if tname is None:
 500                                 return (thing+' name not just string',None,None)
 501                         return (None,tid,tname)
 502
 503                 def wi_alwar(head, waritem, thisdecl, othermin, othermax):
 504                         (err,flagid,flagname) = wihelp_item(waritem,'flag')
 505                         if err: return err
 506                         rel = self.relation_byid.get(flagid, None)
 507                         if rel: return 'flag id twice!'
 508                         if flagname in self.relation_byname:
 509                                 return 'flag name twice!'
 510                         rel = FlagRelation()
 511                         rel.other_flagname = flagname
 512                         rel.other_flagid = flagid
 513                         rel.yoweb_heading = head
 514                         rel.this_declaring = thisdecl
 515                         rel.other_declaring_min = othermin
 516                         rel.other_declaring_max = othermax
 517                         self.relations.append(rel)
 518                         self.relation_byid[flagid] = rel
 519                         self.relation_byname[flagid] = rel
 520
 521                 def wi_isle(head, waritem):
 522                         (err,isleid,islename) = wihelp_item(waritem,'island')
 523                         if err: return err
 524                         self.islands.append((isleid,islename))
 525
 526                 warmap = {
 527                         'Allied with':                  (wi_alwar,+1,+1,+1),
 528                         'Declaring war against':        (wi_alwar,-1, 0,+1),
 529                         'At war with':                  (wi_alwar,-1,-1,-1),
 530                         'Trying to form an alliance with': (wi_alwar,+1,-1,0),
 531                         'Islands controlled by this flag': (wi_isle,),
 532                         }
 533
 534                 how = (wi_warn, None)
 535
 536                 for waritem in warinfo.findAll(['font','a']):
 537                         if waritem is None: break
 538                         if waritem.name == 'font':
 539                                 colour = waritem.get('color',None)
 540                                 if colour.lstrip('#') != '958A5F':
 541                                         warn('strange colour %s in %s' %
 542                                                 (colour,``waritem``))
 543                                         continue
 544                                 head = waritem.string
 545                                 if head is None:
 546                                         warn('no head string in '+``waritem``)
 547                                         continue
 548                                 head = regexp.sub('\\s+', ' ', head).strip()
 549                                 head = head.rstrip(':')
 550                                 how = (head,) + warmap.get(head, (wi_warn,))
 551                                 continue
 552                         assert(waritem.name == 'a')
 553
 554                         debug('WARHOW %s(%s, waritem, *%s)' %
 555                                 (how[1], `how[0]`, `how[2:]`))
 556                         bad = how[1](how[0], waritem, *how[2:])
 557                         if bad:
 558                                 warn('bad waritem %s: %s: %s' % (`how[0]`,
 559                                         bad, ``waritem``))
 560
 561         def __str__(self):
 562                 return `(self.name, self.islands, self.relations)`
 563
 564 #---------- scraper for ocean info incl. embargoes etc. ----------
 565
 566 class IslandBasicInfo():
 567         # Public data attributes:
 568         #  ocean
 569         #  name
 570         # Public data attributes maybe set by caller:
 571         #  arch
 572         def __init__(self, ocean, islename):
 573                 self.ocean = ocean
 574                 self.name = islename
 575         def yppedia(self):
 576                 def q(x): return urllib.quote(x.replace(' ','_'))
 577                 url_rhs = q(self.name) + '_(' + q(self.ocean) + ')'
 578                 return yppedia(url_rhs)
 579         def __str__(self):
 580                 return `(self.ocean, self.name)`
 581
 582 class IslandExtendedInfo(IslandBasicInfo):
 583         # Public data attributes (inherited):
 584         #  ocean
 585         #  name
 586         # Public data attributes (additional):
 587         #  islandid
 588         #  yoweb_url
 589         #  flagid
 590         def __init__(self, ocean, islename):
 591                 IslandBasicInfo.__init__(self, ocean, islename)
 592                 self.islandid = None
 593                 self.yoweb_url = None
 594                 self._collect_yoweb()
 595                 self._collect_flagid()
 596
 597         def _collect_yoweb(self):
 598                 debug('IEI COLLECT YOWEB '+`self.name`)
 599                 self.islandid = None
 600                 self.yoweb_url = None
 601
 602                 soup = make_soup(self.yppedia())
 603                 content = soup.find('div', attrs = {'id': 'content'})
 604                 yoweb_re = regexp.compile('^http://\w+\.puzzlepirates\.com/'+
 605                         'yoweb/island/info\.wm\?islandid=(\d+)$')
 606                 a = soup.find('a', attrs = { 'href': yoweb_re })
 607                 if a is None:
 608                         debug('IEI COLLECT YOWEB '+`self.name`+' NONE')
 609                         return
 610
 611                 debug('IEI COLLECT YOWEB '+`self.name`+' GOT '+``a``)
 612                 self.yoweb_url = a['href']
 613                 m = yoweb_re.search(self.yoweb_url)
 614                 self.islandid = m.group(1)
 615
 616         def _collect_flagid(self):
 617                 self.flagid = None
 618
 619                 yo = self.yoweb_url
 620                 debug('IEI COLLECT FLAGID '+`self.name`+' URL '+`yo`)
 621                 if yo is None: return None
 622                 dataf = fetcher.fetch(yo, 1800)
 623                 soup = make_soup(dataf)
 624                 ruler_re = regexp.compile(
 625                         '/yoweb/flag/info\.wm\?flagid=(\d+)$')
 626                 ruler = soup.find('a', attrs = { 'href': ruler_re })
 627                 if not ruler:
 628                         debug('IEI COLLECT FLAGID '+`self.name`+' NONE')
 629                         return
 630                 debug('IEI COLLECT FLAGID '+`self.name`+' GOT '+``ruler``)
 631                 m = ruler_re.search(ruler['href'])
 632                 self.flagid = m.group(1)
 633
 634         def __str__(self):
 635                 return `(self.ocean, self.islandid, self.name,
 636                         self.yoweb_url, self.flagid)`
 637
 638 class IslandFlagInfo(IslandExtendedInfo):
 639         # Public data attributes (inherited):
 640         #  ocean
 641         #  name
 642         #  islandid
 643         #  yoweb_url
 644         #  flagid
 645         # Public data attributes (additional):
 646         #  flag
 647         def __init__(self, ocean, islename):
 648                 IslandExtendedInfo.__init__(self, ocean, islename)
 649                 self.flag = None
 650                 self._collect_flag()
 651
 652         def _collect_flag(self):
 653                 if self.flagid is None: return
 654                 self.flag = FlagInfo(self.flagid, 1800)
 655
 656         def __str__(self):
 657                 return IslandExtendedInfo.__str__(self) + '; ' + str(self.flag)
 658
 659 class NullProgressReporter():
 660         def doing(self, msg): pass
 661         def stop(self): pass
 662
 663 class TypewriterProgressReporter():
 664         def __init__(self):
 665                 self._l = 0
 666         def doing(self,m):
 667                 self._doing(m + '...')
 668         def _doing(self,m):
 669                 self._write('\r')
 670                 self._write(m)
 671                 less = self._l - len(m)
 672                 if less > 0:
 673                         self._write(' ' * less)
 674                         self._write('\b' * less)
 675                 self._l = len(m)
 676                 sys.stdout.flush()
 677         def stop(self):
 678                 self._doing('')
 679                 self._l = 0
 680         def _write(self,t):
 681                 sys.stdout.write(t)
 682
 683 class OceanInfo():
 684         # Public data attributes:
 685         #   oi.islands[islename] = IslandInfo(...)
 686         #   oi.arches[archname][islename] = IslandInfo(...)
 687         def __init__(self, isleclass=IslandBasicInfo):
 688                 self.isleclass = isleclass
 689                 self.ocean = fetcher.ocean.lower().capitalize()
 690
 691                 progressreporter.doing('fetching ocean info')
 692
 693                 cmdl = ['./yppedia-ocean-scraper']
 694                 if opts.localhtml is not None:
 695                         cmdl += ['--local-html-dir',opts.localhtml]
 696                 cmdl += [self.ocean]
 697                 debug('OceanInfo collect running ' + `cmdl`)
 698                 oscraper = subprocess.Popen(
 699                         cmdl,
 700                         stdout = subprocess.PIPE,
 701                         cwd = yppsc_dir()+'/yarrg',
 702                         shell=False, stderr=None,
 703                         )
 704                 h = oscraper.stdout.readline()
 705                 debug('OceanInfo collect h '+`h`)
 706                 assert(regexp.match('^ocean ', h))
 707                 arch_re = regexp.compile('^ (\S.*)')
 708                 island_re = regexp.compile('^  (\S.*)')
 709
 710                 oscraper.wait()
 711                 assert(oscraper.returncode == 0)
 712
 713                 self.islands = { }
 714                 self.arches = { }
 715                 archname = None
 716
 717                 isles = [ ]
 718                 progressreporter.doing('parsing ocean info')
 719
 720                 for l in oscraper.stdout:
 721                         debug('OceanInfo collect l '+`l`)
 722                         l = l.rstrip('\n')
 723                         m = island_re.match(l)
 724                         if m:
 725                                 assert(archname is not None)
 726                                 islename = m.group(1)
 727                                 isles.append((archname, islename))
 728                                 continue
 729                         m = arch_re.match(l)
 730                         if m:
 731                                 archname = m.group(1)
 732                                 assert(archname not in self.arches)
 733                                 self.arches[archname] = { }
 734                                 continue
 735                         assert(False)
 736
 737                 for i in xrange(0, len(isles)-1):
 738                         (archname, islename) = isles[i]
 739                         progressreporter.doing(
 740                                 'fetching isle info %2d/%d (%s: %s)'
 741                                 % (i, len(isles), archname, islename))
 742                         isle = self.isleclass(self.ocean, islename)
 743                         isle.arch = archname
 744                         self.islands[islename] = isle
 745                         self.arches[archname][islename] = isle
 746
 747         def __str__(self):
 748                 return `(self.islands, self.arches)`
 749
 750 #---------- pretty-printer for tables of pirate puzzle standings ----------
 751
 752 class StandingsTable:
 753         def __init__(self, f, use_puzzles=None, col_width=6, gap_every=5):
 754                 if use_puzzles is None:
 755                         if opts.ship_duty:
 756                                 use_puzzles=duty_puzzles
 757                         else:
 758                                 use_puzzles=puzzles
 759                 self._puzzles = use_puzzles
 760                 self.f = f
 761                 self._cw = col_width-1
 762                 self._gap_every = gap_every
 763                 self._linecount = 0
 764                 self._o = f.write
 765
 766         def _nl(self): self._o('\n')
 767
 768         def _pline(self, lhs, puzstrs, extra):
 769                 if (self._linecount > 0
 770                     and self._gap_every is not None
 771                     and not (self._linecount % self._gap_every)):
 772                         self._nl()
 773                 self._o('%-*s' % (max(max_pirate_namelen+1, 15), lhs))
 774                 for v in puzstrs:
 775                         self._o(' %-*.*s' % (self._cw,self._cw, v))
 776                 if extra:
 777                         self._o(' ' + extra)
 778                 self._nl()
 779                 self._linecount += 1
 780
 781         def _puzstr(self, pi, puzzle):
 782                 if not isinstance(puzzle,list): puzzle = [puzzle]
 783                 try: standing = max([pi.standings[p] for p in puzzle])
 784                 except KeyError: return '?'
 785                 if not standing: return ''
 786                 s = ''
 787                 if self._cw > 4:
 788                         c1 = standingvals[standing][0]
 789                         if standing < 3: c1 = c1.lower() # 3 = Master
 790                         s += `standing`
 791                 if self._cw > 5:
 792                         s += ' '
 793                 s += '*' * (standing / 2)
 794                 s += '+' * (standing % 2)
 795                 return s
 796
 797         def headings(self, lhs='', rhs=None):
 798                 def puzn_redact(name):
 799                         if isinstance(name,list):
 800                                 return '/'.join(
 801                                         ["%.*s" % (self._cw/2, puzn_redact(n))
 802                                          for n in name])
 803                         spc = name.find(' ')
 804                         if spc < 0: return name
 805                         return name[0:min(4,spc)] + name[spc+1:]
 806                 self._linecount = -2
 807                 self._pline(lhs, map(puzn_redact, self._puzzles), rhs)
 808                 self._linecount = 0
 809         def literalline(self, line):
 810                 self._o(line)
 811                 self._nl()
 812                 self._linecount = 0
 813         def pirate_dummy(self, name, standingstring, extra=None):
 814                 standings = standingstring * len(self._puzzles)
 815                 self._pline(' '+name, standings, extra)
 816         def pirate(self, pi, extra=None):
 817                 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
 818                 self._pline(' '+pi.name, puzstrs, extra)
 819
 820
 821 #---------- chat log parser ----------
 822
 823 class PirateAboard:
 824         # This is essentially a transparent, dumb, data class.
 825         #  pa.v                 may be None
 826         #  pa.name
 827         #  pa.last_time
 828         #  pa.last_event
 829         #  pa.gunner
 830         #  pa.last_chat_time
 831         #  pa.last_chat_chan
 832         #  pa.pi
 833
 834         # Also used for jobbing applicants:
 835         #               happens when                    expires (to "-")
 836         #   -            disembark, leaves crew          no
 837         #   aboard       evidence of them being aboard   no
 838         #   applied      "has applied for the job"       120s, configurable
 839         #   ashore       "has taken a job"               30min, configurable
 840         #   declined     "declined the job offer"        30s, configurable
 841         #   invited      "has been invited to job"       120s, configurable
 842         #
 843         #  pa.jobber    None, 'ashore', 'applied', 'invited', 'declined'
 844         #  pa.expires   expiry time time
 845
 846         def __init__(pa, pn, v, time, event):
 847                 pa.name = pn
 848                 pa.v = v
 849                 pa.last_time = time
 850                 pa.last_event = event
 851                 pa.last_chat_time = None
 852                 pa.last_chat_chan = None
 853                 pa.gunner = False
 854                 pa.pi = None
 855                 pa.jobber = None
 856                 pa.expires = None
 857
 858         def pirate_info(pa):
 859                 now = time.time()
 860                 if pa.pi:
 861                         age = now - pa.pi_fetched
 862                         guide = random.randint(120,240)
 863                         if age <= guide:
 864                                 return pa.pi
 865                         debug('PirateAboard refresh %d > %d  %s' % (
 866                                 age, guide, pa.name))
 867                         imaginary = [2,4]
 868                 else:
 869                         imaginary = [1]
 870                 wait = fetcher.need_wait(now, imaginary)
 871                 if wait:
 872                         debug('PirateAboard fetcher not ready %d' % wait)
 873                         return pa.pi
 874                 pa.pi = PirateInfo(pa.name, 600)
 875                 pa.pi_fetched = now
 876                 return pa.pi
 877
 878 class ChatLogTracker:
 879         # This is quite complex so we make it opaque.  Use the
 880         # official invokers, accessors etc.
 881
 882         def __init__(self, myself_pi, logfn):
 883                 self._pl = {}   # self._pl['Pirate'] =
 884                 self._vl = {}   #   self._vl['Vessel']['Pirate'] = PirateAboard
 885                                 # self._vl['Vessel']['#lastinfo']
 886                                 # self._vl['Vessel']['#name']
 887                                 # self._v = self._vl[self._vessel]
 888                 self._date = None
 889                 self._myself = myself_pi
 890                 self._lbuf = ''
 891                 self._f = file(logfn)
 892                 flen = os.fstat(self._f.fileno()).st_size
 893                 max_backlog = 500000
 894                 if flen > max_backlog:
 895                         startpos = flen - max_backlog
 896                         self._f.seek(startpos)
 897                         self._f.readline()
 898                 self._progress = [0, flen - self._f.tell()]
 899                 self._disembark_myself()
 900                 self._need_redisplay = False
 901                 self._lastvessel = None
 902
 903         def _disembark_myself(self):
 904                 self._v = None
 905                 self._vessel = None
 906                 self.force_redisplay()
 907
 908         def force_redisplay(self):
 909                 self._need_redisplay = True
 910
 911         def _vessel_updated(self, v, timestamp):
 912                 if v is None: return
 913                 v['#lastinfo'] = timestamp
 914                 self.force_redisplay()
 915
 916         def _onboard_event(self,v,timestamp,pirate,event,jobber=None):
 917                 pa = self._pl.get(pirate, None)
 918                 if pa is not None and pa.v is v:
 919                         pa.last_time = timestamp
 920                         pa.last_event = event
 921                 else:
 922                         if pa is not None and pa.v is not None:
 923                                 del pa.v[pirate]
 924                         pa = PirateAboard(pirate, v, timestamp, event)
 925                         self._pl[pirate] = pa
 926                         if v is not None: v[pirate] = pa
 927                 pa.jobber = jobber
 928
 929                 if jobber is None: timeout = None
 930                 else: timeout = getattr(opts, 'timeout_'+jobber)
 931                 if timeout is None: pa.expires = None
 932                 else: pa.expires = timestamp + timeout
 933                 self._vessel_updated(v, timestamp)
 934                 return pa
 935
 936         def _expire_jobbers(self, now):
 937                 for pa in self._pl.values():
 938                         if pa.expires is None: continue
 939                         if pa.expires >= now: continue
 940                         v = pa.v
 941                         del self._pl[pa.name]
 942                         if v is not None: del v[pa.name]
 943                         self.force_redisplay()
 944
 945         def _trash_vessel(self, v):
 946                 for pn in v:
 947                         if pn.startswith('#'): continue
 948                         del self._pl[pn]
 949                 vn = v['#name']
 950                 del self._vl[vn]
 951                 if v is self._v: self._disembark_myself()
 952                 self.force_redisplay()
 953
 954         def _vessel_stale(self, v, timestamp):
 955                 return timestamp - v['#lastinfo'] > opts.ship_reboard_clearout
 956
 957         def _vessel_check_expire(self, v, timestamp):
 958                 if not self._vessel_stale(v, timestamp):
 959                         return v
 960                 self._debug_line_disposition(timestamp,'',
 961                         'stale-reset ' + v['#name'])
 962                 self._trash_vessel(v)
 963                 return None
 964
 965         def expire_garbage(self, timestamp):
 966                 for v in self._vl.values():
 967                         self._vessel_check_expire(v, timestamp)
 968
 969         def _vessel_lookup(self, vn, timestamp, dml=[], create=False):
 970                 v = self._vl.get(vn, None)
 971                 if v is not None:
 972                         v = self._vessel_check_expire(v, timestamp)
 973                 if v is not None:
 974                         dml.append('found')
 975                         return v
 976                 if not create:
 977                         dml.append('no')
 978                 dml.append('new')
 979                 self._vl[vn] = v = { '#name': vn }
 980                 self._vessel_updated(v, timestamp)
 981                 return v
 982
 983         def _find_matching_vessel(self, pattern, timestamp, cmdr,
 984                                         dml=[], create=False):
 985                 # use when a commander pirate `cmdr' specified a vessel
 986                 #  by name `pattern' (either may be None)
 987                 # if create is true, will create the vessel
 988                 #  record if an exact name is specified
 989
 990                 if (pattern is not None and
 991                     not '*' in pattern
 992                     and len(pattern.split(' ')) == 2):
 993                         vn = pattern.title()
 994                         dml.append('exact')
 995                         return self._vessel_lookup(
 996                                 vn, timestamp, dml=dml, create=create)
 997
 998                 if pattern is None:
 999                         pattern_check = lambda vn: True
1000                 else:
1001                         re = '(?:.* )?%s$' % pattern.lower().replace('*','.+')
1002                         pattern_check = regexp.compile(re, regexp.I).match
1003
1004                 tries = []
1005
1006                 cmdr_pa = self._pl.get(cmdr, None)
1007                 if cmdr_pa: tries.append((cmdr_pa.v, 'cmdr'))
1008
1009                 tries.append((self._v, 'here'))
1010                 tried_vns = []
1011
1012                 for (v, dm) in tries:
1013                         if v is None: dml.append(dm+'?'); continue
1014
1015                         vn = v['#name']
1016                         if not pattern_check(vn):
1017                                 tried_vns.append(vn)
1018                                 dml.append(dm+'#')
1019                                 continue
1020
1021                         dml.append(dm+'!')
1022                         return v
1023
1024                 if pattern is not None and '*' in pattern:
1025                         search = [
1026                                 (vn,v)
1027                                 for (vn,v) in self._vl.iteritems()
1028                                 if not self._vessel_stale(v, timestamp)
1029                                 if pattern_check(vn)
1030                                 ]
1031                         #debug('CLT-RE /%s/ wanted (%s) searched (%s)' % (
1032                         #       re,
1033                         #       '/'.join(tried_vns),
1034                         #       '/'.join([vn for (vn,v) in search])))
1035
1036                         if len(search)==1:
1037                                 dml.append('one')
1038                                 return search[0][1]
1039                         elif search:
1040                                 dml.append('many')
1041                         else:
1042                                 dml.append('none')
1043
1044         def _debug_line_disposition(self,timestamp,l,m):
1045                 debug('CLT %13s %-40s %s' % (timestamp,m,l))
1046
1047         def _rm_crew_l(self,re,l):
1048                 m = regexp.match(re,l)
1049                 if m and m.group(2) == self._myself.crew[1]:
1050                         return m.group(1)
1051                 else:
1052                         return None
1053
1054         def local_command(self, metacmd):
1055                 # returns None if all went well, or problem message
1056                 return self._command(self._myself.name, metacmd,
1057                         "local", time.time(),
1058                         (lambda m: debug('CMD %s' % metacmd)))
1059
1060         def _command(self, cmdr, metacmd, chan, timestamp, d):
1061                 # returns None if all went well, or problem message
1062                 metacmd = regexp.sub('\\s+', ' ', metacmd).strip()
1063                 m2 = regexp.match(
1064                     '/([adj]) (?:([A-Za-z* ]+)\\s*:)?([A-Za-z ]+)$',
1065                     metacmd)
1066                 if not m2: return "unknown syntax or command"
1067
1068                 (cmd, pattern, targets) = m2.groups()
1069                 dml = ['cmd', chan, cmd]
1070
1071                 if cmd == 'a': each = self._onboard_event
1072                 elif cmd == 'd': each = disembark
1073                 else: each = lambda *l: self._onboard_event(*l,
1074                                 **{'jobber':'applied'})
1075
1076                 if cmdr == self._myself.name:
1077                         dml.append('self')
1078                         how = 'cmd: %s' % cmd
1079                 else:
1080                         dml.append('other')
1081                         how = 'cmd: %s %s' % (cmd,cmdr)
1082
1083                 if cmd == 'j':
1084                         if pattern is not None:
1085                                 return "/j command does not take a vessel"
1086                         v = None
1087                 else:
1088                         v = self._find_matching_vessel(
1089                                 pattern, timestamp, cmdr,
1090                                 dml, create=True)
1091
1092                 if cmd == 'j' or v is not None:
1093                         targets = targets.strip().split(' ')
1094                         dml.append(`len(targets)`)
1095                         for target in targets:
1096                                 each(v, timestamp, target.title(), how)
1097                         self._vessel_updated(v, timestamp)
1098
1099                 dm = ' '.join(dml)
1100                 return d(dm)
1101
1102                 return None
1103
1104         def chatline(self,l):
1105                 rm = lambda re: regexp.match(re,l)
1106                 d = lambda m: self._debug_line_disposition(timestamp,l,m)
1107                 rm_crew = lambda re: self._rm_crew_l(re,l)
1108                 timestamp = None
1109
1110                 m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$')
1111                 if m:
1112                         self._date = [int(x) for x in m.groups()]
1113                         self._previous_timestamp = None
1114                         return d('date '+`self._date`)
1115
1116                 if self._date is None:
1117                         return d('date unset')
1118
1119                 m = rm('\\[(\d\d):(\d\d):(\d\d)\\] ')
1120                 if not m:
1121                         return d('no timestamp')
1122
1123                 while True:
1124                         time_tuple = (self._date +
1125                                       [int(x) for x in m.groups()] +
1126                                       [-1,-1,-1])
1127                         timestamp = time.mktime(time_tuple)
1128                         if timestamp >= self._previous_timestamp: break
1129                         self._date[2] += 1
1130                         self._debug_line_disposition(timestamp,'',
1131                                 'new date '+`self._date`)
1132
1133                 self._previous_timestamp = timestamp
1134
1135                 l = l[l.find(' ')+1:]
1136
1137                 def ob_x(pirate,event):
1138                         return self._onboard_event(
1139                                         self._v, timestamp, pirate, event)
1140                 def ob1(did): ob_x(m.group(1), did); return d(did)
1141                 def oba(did): return ob1('%s %s' % (did, m.group(2)))
1142
1143                 def jb(pirate,jobber):
1144                         return self._onboard_event(
1145                                 None, timestamp, pirate,
1146                                 ("jobber %s" % jobber),
1147                                 jobber=jobber
1148                                 )
1149
1150                 def disembark(v, timestamp, pirate, event):
1151                         self._onboard_event(
1152                                         v, timestamp, pirate, 'leaving '+event)
1153                         del v[pirate]
1154                         del self._pl[pirate]
1155
1156                 def disembark_me(why):
1157                         self._disembark_myself()
1158                         return d('disembark-me '+why)
1159
1160                 m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
1161                 if m:
1162                         dm = ['boarding']
1163                         pn = self._myself.name
1164                         vn = m.group(1)
1165                         v = self._vessel_lookup(vn, timestamp, dm, create=True)
1166                         self._lastvessel = self._vessel = vn
1167                         self._v = v
1168                         ob_x(pn, 'we boarded')
1169                         self.expire_garbage(timestamp)
1170                         return d(' '.join(dm))
1171
1172                 if self._v is None:
1173                         return d('no vessel')
1174
1175                 m = rm('(\\w+) has come aboard\\.$')
1176                 if m: return ob1('boarded');
1177
1178                 m = rm('You have ordered (\\w+) to do some (\\S.*\\S)\\.$')
1179                 if m:
1180                         (who,what) = m.groups()
1181                         pa = ob_x(who,'ord '+what)
1182                         if what == 'Gunning':
1183                                 pa.gunner = True
1184                         return d('duty order')
1185
1186                 m = rm('(\\w+) abandoned a (\\S.*\\S) station\\.$')
1187                 if m: oba('stopped'); return d("end")
1188
1189                 def chat_core(speaker, chan):
1190                         try: pa = self._pl[speaker]
1191                         except KeyError: return 'mystery'
1192                         if pa.v is not None and pa.v is not self._v:
1193                                 return 'elsewhere'
1194                         pa.last_chat_time = timestamp
1195                         pa.last_chat_chan = chan
1196                         self.force_redisplay()
1197                         return 'here'
1198
1199                 def chat(chan):
1200                         speaker = m.group(1)
1201                         dm = chat_core(speaker, chan)
1202                         return d('chat %s %s' % (chan, dm))
1203
1204                 def chat_metacmd(chan):
1205                         (cmdr, metacmd) = m.groups()
1206                         whynot = self._command(
1207                                 cmdr, metacmd, chan, timestamp, d)
1208                         if whynot is not None:
1209                                 return chat(chan)
1210                         else:
1211                                 chat_core(cmdr, 'cmd '+chan)
1212
1213                 m = rm('(\\w+) (?:issued an order|ordered everyone) "')
1214                 if m: return ob1('general order');
1215
1216                 m = rm('(\\w+) says, "')
1217                 if m: return chat('public')
1218
1219                 m = rm('(\\w+) tells ye, "')
1220                 if m: return chat('private')
1221
1222                 m = rm('Ye told (\\w+), "(.*)"$')
1223                 if m: return chat_metacmd('private')
1224
1225                 m = rm('(\\w+) flag officer chats, "')
1226                 if m: return chat('flag officer')
1227
1228                 m = rm('(\\w+) officer chats, "(.*)"$')
1229                 if m: return chat_metacmd('officer')
1230
1231                 m = rm('Ye accepted the offer to job with ')
1232                 if m: return disembark_me('jobbing')
1233
1234                 m = rm('Ye hop on the ferry and are whisked away ')
1235                 if m: return disembark_me('ferry')
1236
1237                 m = rm('Whisking away to yer home on the magical winds')
1238                 if m: return disembark_me('home')
1239
1240                 m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
1241                 if m:
1242                         pl = m.group(1).split(', ')
1243                         if not self._myself.name in pl:
1244                                 return d('lost melee')
1245                         for pn in pl:
1246                                 if ' ' in pn: continue
1247                                 ob_x(pn,'won melee')
1248                         return d('won melee')
1249
1250                 m = rm('(\\w+) is eliminated\\!')
1251                 if m: return ob1('eliminated in fray');
1252
1253                 m = rm('(\\w+) has driven \w+ from the ship\\!')
1254                 if m: return ob1('boarder repelled');
1255
1256                 m = rm('\w+ has bested (\\w+), and turns'+
1257                         ' to the rest of the ship\\.')
1258                 if m: return ob1('boarder unrepelled');
1259
1260                 pirate = rm_crew("(\\w+) has taken a job with '(.*)'\\.")
1261                 if pirate: return jb(pirate, 'ashore')
1262
1263                 pirate = rm_crew("(\\w+) has left '(.*)'\\.")
1264                 if pirate:
1265                         disembark(self._v, timestamp, pirate, 'left crew')
1266                         return d('left crew')
1267
1268                 m = rm('(\w+) has applied for the posted job\.')
1269                 if m: return jb(m.group(1), 'applied')
1270
1271                 pirate= rm_crew("(\\w+) has been invited to job for '(.*)'\\.")
1272                 if pirate: return jb(pirate, 'invited')
1273
1274                 pirate = rm_crew("(\\w+) declined the job offer for '(.*)'\\.")
1275                 if pirate: return jb(pirate, 'declined')
1276
1277                 m = rm('(\\w+) has left the vessel\.')
1278                 if m:
1279                         pirate = m.group(1)
1280                         disembark(self._v, timestamp, pirate, 'disembarked')
1281                         return d('disembarked')
1282
1283                 return d('not-matched')
1284
1285         def _str_pa(self, pn, pa):
1286                 assert self._pl[pn] == pa
1287                 s = ' '*20 + "%s %-*s %13s %-30s %13s %-20s %13s" % (
1288                         (' ','G')[pa.gunner],
1289                         max_pirate_namelen, pn,
1290                         pa.last_time, pa.last_event,
1291                         pa.last_chat_time, pa.last_chat_chan,
1292                         pa.jobber)
1293                 if pa.expires is not None:
1294                         s += " %-5d" % (pa.expires - pa.last_time)
1295                 s += "\n"
1296                 return s
1297
1298         def _str_vessel(self, vn, v):
1299                 s = ' vessel %s\n' % vn
1300                 s += ' '*20 + "%-*s   %13s\n" % (
1301                                 max_pirate_namelen, '#lastinfo',
1302                                 v['#lastinfo'])
1303                 assert v['#name'] == vn
1304                 for pn in sorted(v.keys()):
1305                         if pn.startswith('#'): continue
1306                         pa = v[pn]
1307                         assert pa.v == v
1308                         s += self._str_pa(pn,pa)
1309                 return s
1310
1311         def __str__(self):
1312                 s = '''<ChatLogTracker
1313  myself %s
1314  vessel %s
1315 '''                     % (self._myself.name, self._vessel)
1316                 assert ((self._v is None and self._vessel is None) or
1317                         (self._v is self._vl[self._vessel]))
1318                 if self._vessel is not None:
1319                         s += self._str_vessel(self._vessel, self._v)
1320                 for vn in sorted(self._vl.keys()):
1321                         if vn == self._vessel: continue
1322                         s += self._str_vessel(vn, self._vl[vn])
1323                 s += " elsewhere\n"
1324                 for p in self._pl:
1325                         pa = self._pl[p]
1326                         if pa.v is not None:
1327                                 assert pa.v[p] is pa
1328                                 assert pa.v in self._vl.values()
1329                         else:
1330                                 s += self._str_pa(pa.name, pa)
1331                 s += '>\n'
1332                 return s
1333
1334         def catchup(self, progress=None):
1335                 while True:
1336                         more = self._f.readline()
1337                         if not more: break
1338
1339                         self._progress[0] += len(more)
1340                         if progress: progress.progress(*self._progress)
1341
1342                         self._lbuf += more
1343                         if self._lbuf.endswith('\n'):
1344                                 self.chatline(self._lbuf.rstrip())
1345                                 self._lbuf = ''
1346                                 if opts.debug >= 2:
1347                                         debug(self.__str__())
1348                 self._expire_jobbers(time.time())
1349
1350                 if progress: progress.caughtup()
1351
1352         def changed(self):
1353                 rv = self._need_redisplay
1354                 self._need_redisplay = False
1355                 return rv
1356         def myname(self):
1357                 # returns our pirate name
1358                 return self._myself.name
1359         def vesselname(self):
1360                 # returns the vessel name we're aboard or None
1361                 return self._vessel
1362         def lastvesselname(self):
1363                 # returns the last vessel name we were aboard or None
1364                 return self._lastvessel
1365         def aboard(self, vesselname=True):
1366                 # returns a list of PirateAboard the vessel
1367                 #  sorted by pirate name
1368                 #  you can pass this None and you'll get []
1369                 #  or True for the current vessel (which is the default)
1370                 #  the returned value is a fresh list of persistent
1371                 #  PirateAboard objects
1372                 if vesselname is True: v = self._v
1373                 else: v = self._vl.get(vesselname.title())
1374                 if v is None: return []
1375                 return [ v[pn]
1376                          for pn in sorted(v.keys())
1377                          if not pn.startswith('#') ]
1378         def jobbers(self):
1379                 # returns a the jobbers' PirateAboards,
1380                 # sorted by jobber class and reverse of expiry time
1381                 l = [ pa
1382                       for pa in self._pl.values()
1383                       if pa.jobber is not None
1384                     ]
1385                 def compar_key(pa):
1386                         return (pa.jobber, -pa.expires)
1387                 l.sort(key = compar_key)
1388                 return l
1389
1390 #---------- implementations of actual operation modes ----------
1391
1392 def do_pirate(pirates, bu):
1393         print '{'
1394         for pirate in pirates:
1395                 info = PirateInfo(pirate)
1396                 print '%s: %s,' % (`pirate`, info)
1397         print '}'
1398
1399 def prep_crewflag_of(args, bu, max_age, selector, constructor):
1400         if len(args) != 1: bu('crew-of etc. take one pirate name')
1401         pi = PirateInfo(args[0], max_age)
1402         cf = selector(pi)
1403         if cf is None: return None
1404         return constructor(cf[0], max_age)
1405
1406 def prep_crew_of(args, bu, max_age=300):
1407         return prep_crewflag_of(args, bu, max_age,
1408                 (lambda pi: pi.crew), CrewInfo)
1409
1410 def prep_flag_of(args, bu, max_age=300):
1411         return prep_crewflag_of(args, bu, max_age,
1412                 (lambda pi: pi.flag), FlagInfo)
1413
1414 def do_crew_of(args, bu):
1415         ci = prep_crew_of(args, bu)
1416         print ci
1417
1418 def do_flag_of(args, bu):
1419         fi = prep_flag_of(args, bu)
1420         print fi
1421
1422 def do_standings_crew_of(args, bu):
1423         ci = prep_crew_of(args, bu, 60)
1424         tab = StandingsTable(sys.stdout)
1425         tab.headings()
1426         for (rank, members) in ci.crew:
1427                 if not members: continue
1428                 tab.literalline('')
1429                 tab.literalline('%s:' % rank)
1430                 for p in members:
1431                         pi = PirateInfo(p, random.randint(900,1800))
1432                         tab.pirate(pi)
1433
1434 def do_ocean(args, bu):
1435         if (len(args)): bu('ocean takes no further arguments')
1436         fetcher.default_ocean()
1437         oi = OceanInfo(IslandFlagInfo)
1438         print oi
1439         for islename in sorted(oi.islands.keys()):
1440                 isle = oi.islands[islename]
1441                 print isle
1442
1443 def do_embargoes(args, bu):
1444         if (len(args)): bu('ocean takes no further arguments')
1445         fetcher.default_ocean()
1446         oi = OceanInfo(IslandFlagInfo)
1447         wr = sys.stdout.write
1448         print ('EMBARGOES:  Island    | Owning flag'+
1449                 '                    | Embargoed flags')
1450
1451         def getflname(isle):
1452                 if isle.islandid is None: return 'uncolonisable'
1453                 if isle.flag is None: return 'uncolonised'
1454                 return isle.flag.name
1455
1456         progressreporter.stop()
1457
1458         for archname in sorted(oi.arches.keys()):
1459                 print 'ARCHIPELAGO: ',archname
1460                 for islename in sorted(oi.arches[archname].keys()):
1461                         isle = oi.islands[islename]
1462                         wr(' %-20s | ' % isle.name)
1463                         flname = getflname(isle)
1464                         wr('%-30s | ' % flname)
1465                         flag = isle.flag
1466                         if flag is None: print ''; continue
1467                         delim = ''
1468                         for rel in flag.relations:
1469                                 if rel.this_declaring >= 0: continue
1470                                 wr(delim)
1471                                 wr(rel.other_flagname)
1472                                 delim = '; '
1473                         print ''
1474
1475 def do_embargoes_flag_of(args, bu):
1476         progressreporter.doing('fetching flag info')
1477         fi = prep_flag_of(args, bu)
1478         if fi is None:
1479                 progressreporter.stop()
1480                 print 'Pirate is not in a flag.'
1481                 return
1482
1483         oi = OceanInfo(IslandFlagInfo)
1484
1485         progressreporter.stop()
1486         print ''
1487
1488         any = False
1489         for islename in sorted(oi.islands.keys()):
1490                 isle = oi.islands[islename]
1491                 flag = isle.flag
1492                 if flag is None: continue
1493                 for rel in flag.relations:
1494                         if rel.this_declaring >= 0: continue
1495                         if rel.other_flagid != fi.flagid: continue
1496                         if not any: print 'EMBARGOED:'
1497                         any = True
1498                         print "  %-30s (%s)" % (islename, flag.name)
1499         if not any:
1500                 print 'No embargoes.'
1501         print ''
1502
1503         war_flag(fi)
1504         print ''
1505
1506 def do_war_flag_of(args, bu):
1507         fi = prep_flag_of(args, bu)
1508         war_flag(fi)
1509
1510 def war_flag(fi):
1511         any = False
1512         for certain in [True, False]:
1513                 anythis = False
1514                 for rel in fi.relations:
1515                         if rel.this_declaring >= 0: continue
1516                         if (rel.other_declaring_max < 0) != certain: continue
1517                         if not anythis:
1518                                 if certain: m = 'SINKING PvP'
1519                                 else: m = 'RISK OF SINKING PvP'
1520                                 print '%s (%s):' % (m, rel.yoweb_heading)
1521                         anythis = True
1522                         any = True
1523                         print " ", rel.other_flagname
1524         if not any:
1525                 print 'No sinking PvP.'
1526
1527 #----- modes which use the chat log parser are quite complex -----
1528
1529 class ProgressPrintPercentage:
1530         def __init__(self, f=sys.stdout):
1531                 self._f = f
1532         def progress_string(self,done,total):
1533                 return "scan chat logs %3d%%\r" % ((done*100) / total)
1534         def progress(self,*a):
1535                 self._f.write(self.progress_string(*a))
1536                 self._f.flush()
1537         def show_init(self, pirate, ocean):
1538                 print >>self._f, 'Starting up, %s on the %s ocean' % (
1539                         pirate, ocean)
1540         def caughtup(self):
1541                 self._f.write('                   \r')
1542                 self._f.flush()
1543
1544 def prep_chat_log(args, bu,
1545                 progress=ProgressPrintPercentage(),
1546                 max_myself_age=3600):
1547         if len(args) != 1: bu('this action takes only chat log filename')
1548         logfn = args[0]
1549         logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_'
1550         match = regexp.match(logfn_re, logfn)
1551         if not match: bu('chat log filename is not in expected format')
1552         (pirate, ocean) = match.groups()
1553         fetcher.default_ocean(ocean)
1554
1555         progress.show_init(pirate, fetcher.ocean)
1556         myself = PirateInfo(pirate,max_myself_age)
1557         track = ChatLogTracker(myself, logfn)
1558
1559         opts.debug -= 2
1560         track.catchup(progress)
1561         opts.debug += 2
1562
1563         track.force_redisplay()
1564
1565         return (myself, track)
1566
1567 def do_track_chat_log(args, bu):
1568         (myself, track) = prep_chat_log(args, bu)
1569         while True:
1570                 track.catchup()
1571                 if track.changed():
1572                         print track
1573                 sleep(0.5 + 0.5 * random.random())
1574
1575 #----- ship management aid -----
1576
1577 class Display_dumb(ProgressPrintPercentage):
1578         def __init__(self):
1579                 ProgressPrintPercentage.__init__(self)
1580         def show(self, s):
1581                 print '\n\n', s;
1582         def realstart(self):
1583                 pass
1584
1585 class Display_overwrite(ProgressPrintPercentage):
1586         def __init__(self):
1587                 ProgressPrintPercentage.__init__(self)
1588
1589                 null = file('/dev/null','w')
1590                 curses.setupterm(fd=null.fileno())
1591
1592                 self._clear = curses.tigetstr('clear')
1593                 if not self._clear:
1594                         self._debug('missing clear!')
1595                         self.show = Display_dumb.show
1596                         return
1597
1598                 self._t = {'el':'', 'ed':''}
1599                 if not self._init_sophisticated():
1600                         for k in self._t.keys(): self._t[k] = ''
1601                         self._t['ho'] = self._clear
1602
1603         def _debug(self,m): debug('display overwrite: '+m)
1604
1605         def _init_sophisticated(self):
1606                 for k in self._t.keys():
1607                         s = curses.tigetstr(k)
1608                         self._t[k] = s
1609                 self._t['ho'] = curses.tigetstr('ho')
1610                 if not self._t['ho']:
1611                         cup = curses.tigetstr('cup')
1612                         self._t['ho'] = curses.tparm(cup,0,0)
1613                 missing = [k for k in self._t.keys() if not self._t[k]]
1614                 if missing:
1615                         self.debug('missing '+(' '.join(missing)))
1616                         return 0
1617                 return 1
1618
1619         def show(self, s):
1620                 w = sys.stdout.write
1621                 def wti(k): w(self._t[k])
1622
1623                 wti('ho')
1624                 nl = ''
1625                 for l in s.rstrip().split('\n'):
1626                         w(nl)
1627                         w(l)
1628                         wti('el')
1629                         nl = '\r\n'
1630                 wti('ed')
1631                 w(' ')
1632                 sys.stdout.flush()
1633
1634         def realstart(self):
1635                 sys.stdout.write(self._clear)
1636                 sys.stdout.flush()
1637
1638
1639 def do_ship_aid(args, bu):
1640         if opts.ship_duty is None: opts.ship_duty = True
1641
1642         displayer = globals()['Display_'+opts.display]()
1643
1644         (myself, track) = prep_chat_log(args, bu, progress=displayer)
1645
1646         displayer.realstart()
1647
1648         if os.isatty(0): kr_create = KeystrokeReader
1649         else: kr_create = DummyKeystrokeReader
1650
1651         try:
1652                 kreader = kr_create(0, 10)
1653                 ship_aid_core(myself, track, displayer, kreader)
1654         finally:
1655                 kreader.stop()
1656                 print '\n'
1657
1658 class KeyBasedSorter:
1659         def compar_key_pa(self, pa):
1660                 pi = pa.pirate_info()
1661                 if pi is None: return None
1662                 return self.compar_key(pi)
1663         def lsort_pa(self, l):
1664                 l.sort(key = self.compar_key_pa)
1665
1666 class NameSorter(KeyBasedSorter):
1667         def compar_key(self, pi): return pi.name
1668         def desc(self): return 'name'
1669
1670 class SkillSorter(NameSorter):
1671         def __init__(self, relevant):
1672                 self._want = frozenset(relevant.split('/'))
1673                 self._avoid = set()
1674                 for p in core_duty_puzzles:
1675                         if isinstance(p,basestring): self._avoid.add(p)
1676                         else: self._avoid |= set(p)
1677                 self._avoid -= self._want
1678                 self._desc = '%s' % relevant
1679
1680         def desc(self): return self._desc
1681
1682         def compar_key(self, pi):
1683                 best_want = max([
1684                         pi.standings.get(puz,-1)
1685                         for puz in self._want
1686                         ])
1687                 best_avoid = [
1688                         -pi.standings.get(puz,standing_limit)
1689                         for puz in self._avoid
1690                         ]
1691                 best_avoid.sort()
1692                 def negate(x): return -x
1693                 debug('compar_key %s bw=%s ba=%s' % (pi.name, `best_want`,
1694                         `best_avoid`))
1695                 return (-best_want, map(negate, best_avoid), pi.name)
1696
1697 def ship_aid_core(myself, track, displayer, kreader):
1698
1699         def find_vessel():
1700                 vn = track.vesselname()
1701                 if vn: return (vn, " on board the %s" % vn)
1702                 vn = track.lastvesselname()
1703                 if vn: return (vn, " ashore from the %s" % vn)
1704                 return (None, " not on a vessel")
1705
1706         def timeevent(t,e):
1707                 if t is None: return ' ' * 22
1708                 return " %-4s %-16s" % (format_time_interval(now - t),e)
1709
1710         displayer.show(track.myname() + find_vessel()[1] + '...')
1711
1712         rotate_nya = '/-\\'
1713
1714         sort = NameSorter()
1715         clicmd = None
1716         clierr = None
1717         cliexec = None
1718
1719         while True:
1720                 track.catchup()
1721                 now = time.time()
1722
1723                 (vn, vs) = find_vessel()
1724
1725                 s = ''
1726                 if cliexec is not None:
1727                         s += '...'
1728                 elif clierr is not None:
1729                         s += 'Error: '+clierr
1730                 elif clicmd is not None:
1731                         s += '/' + clicmd
1732                 else:
1733                         s = track.myname() + vs
1734                         s += " at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
1735                         s += kreader.info()
1736                 s += '\n'
1737
1738                 tbl_s = StringIO()
1739                 tbl = StandingsTable(tbl_s)
1740
1741                 aboard = track.aboard(vn)
1742                 sort.lsort_pa(aboard)
1743
1744                 jobbers = track.jobbers()
1745
1746                 if track.vesselname(): howmany = 'aboard: %2d' % len(aboard)
1747                 else: howmany = ''
1748
1749                 tbl.headings(howmany, '  sorted by '+sort.desc())
1750
1751                 last_jobber = None
1752
1753                 for pa in aboard + jobbers:
1754                         if pa.jobber != last_jobber:
1755                                 last_jobber = pa.jobber
1756                                 tbl.literalline('')
1757                                 tbl.literalline('jobbers '+last_jobber)
1758
1759                         pi = pa.pirate_info()
1760
1761                         xs = ''
1762                         if pa.gunner: xs += 'G '
1763                         else: xs += '  '
1764                         xs += timeevent(pa.last_time, pa.last_event)
1765                         xs += timeevent(pa.last_chat_time, pa.last_chat_chan)
1766
1767                         if pi is None:
1768                                 tbl.pirate_dummy(pa.name, rotate_nya[0], xs)
1769                         else:
1770                                 tbl.pirate(pi, xs)
1771
1772                 s += tbl_s.getvalue()
1773                 displayer.show(s)
1774                 tbl_s.close()
1775
1776                 if cliexec is not None:
1777                         clierr = track.local_command("/"+cliexec.strip())
1778                         cliexec = None
1779                         continue
1780
1781                 k = kreader.getch()
1782                 if k is None:
1783                         rotate_nya = rotate_nya[1:3] + rotate_nya[0]
1784                         continue
1785
1786                 if clierr is not None:
1787                         clierr = None
1788                         continue
1789
1790                 if clicmd is not None:
1791                         if k == '\r' or k == '\n':
1792                                 cliexec = clicmd
1793                                 clicmd = clicmdbase
1794                         elif k == '\e' and clicmd != "":
1795                                 clicmd = clicmdbase
1796                         elif k == '\33':
1797                                 clicmd = None
1798                         elif k == '\b' or k == '\177':
1799                                 clicmd = clicmd[ 0 : len(clicmd)-1 ]
1800                         else:
1801                                 clicmd += k
1802                         continue
1803
1804                 if k == 'q': break
1805                 elif k == 'g': sort = SkillSorter('Gunning')
1806                 elif k == 'c': sort = SkillSorter('Carpentry')
1807                 elif k == 's': sort = SkillSorter('Sailing/Rigging')
1808                 elif k == 'b': sort = SkillSorter('Bilging')
1809                 elif k == 'n': sort = SkillSorter('Navigating')
1810                 elif k == 'd': sort = SkillSorter('Battle Navigation')
1811                 elif k == 't': sort = SkillSorter('Treasure Haul')
1812                 elif k == 'a': sort = NameSorter()
1813                 elif k == '/': clicmdbase = ""; clicmd = clicmdbase
1814                 elif k == '+': clicmdbase = "a "; clicmd = clicmdbase
1815                 else: pass # unknown key command
1816
1817 #---------- individual keystroke input ----------
1818
1819 class DummyKeystrokeReader:
1820         def __init__(self,fd,timeout_dummy): pass
1821         def stop(self): pass
1822         def getch(self): sleep(1); return None
1823         def info(self): return ' [noninteractive]'
1824
1825 class KeystrokeReader(DummyKeystrokeReader):
1826         def __init__(self, fd, timeout_decisec=0):
1827                 self._fd = fd
1828                 self._saved = termios.tcgetattr(fd)
1829                 a = termios.tcgetattr(fd)
1830                 a[3] &= ~(termios.ECHO | termios.ECHONL |
1831                           termios.ICANON | termios.IEXTEN)
1832                 a[6][termios.VMIN] = 0
1833                 a[6][termios.VTIME] = timeout_decisec
1834                 termios.tcsetattr(fd, termios.TCSANOW, a)
1835         def stop(self):
1836                 termios.tcsetattr(self._fd, termios.TCSANOW, self._saved)
1837         def getch(self):
1838                 debug_flush()
1839                 byte = os.read(self._fd, 1)
1840                 if not len(byte): return None
1841                 return byte
1842         def info(self):
1843                 return ''
1844
1845 #---------- main program ----------
1846
1847 def main():
1848         global opts, fetcher, yppedia, progressreporter
1849
1850         pa = OptionParser(
1851 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
1852 actions:
1853  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
1854  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
1855  yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
1856  yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG
1857  yoweb-scrape [--ocean OCEAN ...] ocean|embargoes
1858  yoweb-scrape [--ocean OCEAN ...] war-flag-of|embargoes-flag-of PIRATE
1859  yoweb-scrape [options] ship-aid CHAT-LOG  (must be .../PIRATE_OCEAN_chat-log*)
1860
1861 display modes (for --display) apply to ship-aid:
1862  --display=dumb       just print new information, scrolling the screen
1863  --display=overwrite  use cursor motion, selective clear, etc. to redraw at top''')
1864         ao = pa.add_option
1865         ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
1866                 help='select ocean OCEAN')
1867         ao('--cache-dir', dest='cache_dir', metavar='DIR',
1868                 default='~/.yoweb-scrape-cache',
1869                 help='cache yoweb pages in DIR')
1870         ao('-D','--debug', action='count', dest='debug', default=0,
1871                 help='enable debugging output')
1872         ao('--debug-fd', type='int', dest='debug_fd',
1873                 help='write any debugging output to specified fd')
1874         ao('-q','--quiet', action='store_true', dest='quiet',
1875                 help='suppress warning output')
1876         ao('--display', action='store', dest='display',
1877                 type='choice', choices=['dumb','overwrite'],
1878                 help='how to display ship aid')
1879         ao('--local-ypp-dir', action='store', dest='localhtml',
1880                 help='get yppedia pages from local directory LOCALHTML'+
1881                         ' instead of via HTTP')
1882
1883         ao_jt = lambda wh, t: ao(
1884                 '--timeout-sa-'+wh, action='store', dest='timeout_'+wh,
1885                 default=t, help=('set timeout for expiring %s jobbers' % wh))
1886         ao_jt('applied',      120)
1887         ao_jt('invited',      120)
1888         ao_jt('declined',      30)
1889         ao_jt('ashore',      1800)
1890
1891         ao('--ship-duty', action='store_true', dest='ship_duty',
1892                 help='show ship duty station puzzles')
1893         ao('--all-puzzles', action='store_false', dest='ship_duty',
1894                 help='show all puzzles, not just ship duty stations')
1895
1896         ao('--min-cache-reuse', type='int', dest='min_max_age',
1897                 metavar='SECONDS', default=60,
1898                 help='always reuse cache yoweb data if no older than this')
1899
1900         (opts,args) = pa.parse_args()
1901         random.seed()
1902
1903         if len(args) < 1:
1904                 print >>sys.stderr, copyright_info
1905                 pa.error('need a mode argument')
1906
1907         if opts.debug_fd is not None:
1908                 opts.debug_file = os.fdopen(opts.debug_fd, 'w')
1909         else:
1910                 opts.debug_file = sys.stdout
1911
1912         mode = args[0]
1913         mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
1914         try: mode_fn = globals()[mode_fn_name]
1915         except KeyError: pa.error('unknown mode "%s"' % mode)
1916
1917         # fixed parameters
1918         opts.expire_age = max(3600, opts.min_max_age)
1919
1920         opts.ship_reboard_clearout = 3600
1921
1922         if opts.cache_dir.startswith('~/'):
1923                 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
1924
1925         if opts.display is None:
1926                 if ((opts.debug > 0 and opts.debug_fd is None)
1927                     or not os.isatty(sys.stdout.fileno())):
1928                         opts.display = 'dumb'
1929                 else:
1930                         opts.display = 'overwrite'
1931
1932         fetcher = Yoweb(opts.ocean, opts.cache_dir)
1933         yppedia = Yppedia(opts.cache_dir)
1934
1935         if opts.debug or not os.isatty(0):
1936                  progressreporter = NullProgressReporter()
1937         else:
1938                 progressreporter = TypewriterProgressReporter()
1939
1940         mode_fn(args[1:], pa.error)
1941
1942 main()