#!/usr/bin/python
# This is part of ypp-sc-tools, a set of third-party tools for assisting
# players of Yohoho Puzzle Pirates.
#
# Copyright (C) 2009 Ian Jackson <ijackson@chiark.greenend.org.uk>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Yohoho and Puzzle Pirates are probably trademarks of Three Rings and
# are used without permission.  This program is not endorsed or
# sponsored by Three Rings.

copyright_info = '''
yoweb-scrape is part of ypp-sc-tools  Copyright (C) 2009 Ian Jackson
This program comes with ABSOLUTELY NO WARRANTY; this is free software,
and you are welcome to redistribute it under certain conditions.
For details, read the top of the yoweb-scrape file.
'''

#---------- setup ----------

import signal
signal.signal(signal.SIGINT, signal.SIG_DFL)

import os
import time
import urllib
import urllib2
import errno
import sys
import re as regexp
import random
import curses
import termios
import random
import subprocess
import copy
from optparse import OptionParser
from StringIO import StringIO

from BeautifulSoup import BeautifulSoup

opts = None

#---------- YPP parameters and arrays ----------

puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
	'/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
	'/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
	'/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')

core_duty_puzzles = [
		'Gunning',
		['Sailing','Rigging'],
		'Bilging',
		'Carpentry',
		]

duty_puzzles = ([ 'Navigating', 'Battle Navigation' ] +
		core_duty_puzzles +
		[ 'Treasure Haul' ])

standingvals = ('Able/Proficient/Distinguished/Respected/Master'+
		'/Renowned/Grand-Master/Legendary/Ultimate').split('/')
standing_limit = len(standingvals)

pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')

max_pirate_namelen = 12


#---------- general utilities ----------

def debug(m):
	if opts.debug > 0:
		print >>opts.debug_file, m

def debug_flush():
	if opts.debug > 0:
		opts.debug_file.flush()	

def sleep(seconds):
	debug_flush()
	time.sleep(seconds)

def format_time_interval(ti):
	if ti < 120: return '%d:%02d' % (ti / 60, ti % 60)
	if ti < 7200: return '%2dm' % (ti / 60)
	if ti < 86400: return '%dh' % (ti / 3600)
	return '%dd' % (ti / 86400)

def yppsc_dir():
	lib = os.getenv("YPPSC_YARRG_SRCBASE")
	if lib is not None: return lib
	lib = sys.argv[0] 
	lib = regexp.sub('/[^/]+$', '', lib)
	os.environ["YPPSC_YARRG_SRCBASE"] = lib
	return lib

soup_massage = copy.copy(BeautifulSoup.MARKUP_MASSAGE)
soup_massage.append(
		(regexp.compile('(\<td.*") ("center")'),
		 lambda m: m.group(1)+' align='+m.group(2))
	)

def make_soup(*args, **kwargs):
	return BeautifulSoup(*args,
		convertEntities=BeautifulSoup.HTML_ENTITIES,
		markupMassage=soup_massage,
			 **kwargs)

#---------- caching and rate-limiting data fetcher ----------

class Fetcher:
	def __init__(self, cachedir):
		debug('Fetcher init %s' % cachedir)
		self.cachedir = cachedir
		try: os.mkdir(cachedir)
		except (OSError,IOError), oe:
			if oe.errno != errno.EEXIST: raise
		self._cache_scan(time.time())

	def _match_url_normalise(self, url):
		without_scheme = regexp.sub('^[a-z]+://', '', url)
		without_tail = regexp.sub('/.*', '', without_scheme)
		return without_tail

	def _cache_scan(self, now, match_url=None):
		# returns list of ages, unsorted
		if match_url is not None:
			match_url = self._match_url_normalise(match_url)
		ages = []
		debug('Fetcher   scan_cache')
		for leaf in os.listdir(self.cachedir):
			if not leaf.startswith('#'): continue
			if match_url is not None:
				leaf_url = urllib.unquote_plus(leaf.strip('#'))
				leaf_url = self._match_url_normalise(leaf_url)
				if leaf_url != match_url:
					continue
			path = self.cachedir + '/' + leaf
			try: s = os.stat(path)
			except (OSError,IOError), oe:
				if oe.errno != errno.ENOENT: raise
				continue
			age = now - s.st_mtime
			if age > opts.expire_age:
				debug('Fetcher    expire %d %s' % (age, path))
				try: os.remove(path)
				except (OSError,IOError), oe:
					if oe.errno != errno.ENOENT: raise
				continue
			ages.append(age)
		return ages

	def need_wait(self, now, imaginary=[], next_url=None):
		ages = self._cache_scan(now, match_url=next_url)
		ages += imaginary
		ages.sort()
		debug('Fetcher   ages ' + `ages`)
		min_age = 1
		need_wait = 0
		for age in ages:
			if age < min_age and age <= 5:
				debug('Fetcher   morewait min=%d age=%d' %
					(min_age, age))
				need_wait = max(need_wait, min_age - age)
			min_age += 3
		if need_wait > 0:
			need_wait += random.random() - 0.5
		return need_wait

	def _rate_limit_cache_clean(self, now, next_url=None):
		need_wait = self.need_wait(now, next_url=next_url)
		if need_wait > 0:
			debug('Fetcher   wait %f' % need_wait)
			sleep(need_wait)

	def fetch(self, url, max_age):
		debug('Fetcher fetch %s' % url)
		cache_corename = urllib.quote_plus(url)
		cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
		try: f = file(cache_item, 'r')
		except (OSError,IOError), oe:
			if oe.errno != errno.ENOENT: raise
			f = None
		now = time.time()
		max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
		if f is not None:
			s = os.fstat(f.fileno())
			age = now - s.st_mtime
			if age > max_age:
				debug('Fetcher  stale %d < %d'% (max_age, age))
				f = None
		if f is not None:
			data = f.read()
			f.close()
			debug('Fetcher  cached %d > %d' % (max_age, age))
			return data

		debug('Fetcher  fetch')
		self._rate_limit_cache_clean(now, next_url=url)

		stream = urllib2.urlopen(url)
		data = stream.read()
		cache_tmp = "%s/#%s~%d#" % (
			self.cachedir, cache_corename, os.getpid())
		f = file(cache_tmp, 'w')
		f.write(data)
		f.close()
		os.rename(cache_tmp, cache_item)
		debug('Fetcher  stored')
		return data

class Yoweb(Fetcher):
	def __init__(self, ocean, cachedir):
		debug('Yoweb init %s' % cachedir)
		self.ocean = ocean
		Fetcher.__init__(self, cachedir)

	def default_ocean(self, ocean='ice'):
		if self.ocean is None:
			self.ocean = ocean

	def yoweb(self, kind, tail, max_age):
		self.default_ocean()
		assert(self.ocean)
		url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
			self.ocean, kind, tail)
		return self.fetch(url, max_age)

class Yppedia(Fetcher):
	def __init__(self, cachedir):
		debug('Yoweb init %s' % cachedir)
		self.base = 'http://yppedia.puzzlepirates.com/'
		self.localhtml = opts.localhtml
		Fetcher.__init__(self, cachedir)

	def __call__(self, rhs):
		if self.localhtml is None:
			url = self.base + rhs
			debug('Yppedia retrieving YPP '+url);
			return self.fetch(url, 3000)
		else:
			return file(opts.localhtml + '/' + rhs, 'r')

#---------- logging assistance for troubled screenscrapers ----------

class SoupLog:
	def __init__(self):
		self.msgs = [ ]
	def msg(self, m):
		self.msgs.append(m)
	def soupm(self, obj, m):
		self.msg(m + '; in ' + `obj`)
	def needs_msgs(self, child_souplog):
		self.msgs += child_souplog.msgs
		child_souplog.msgs = [ ]

def soup_text(obj):
	str = ''.join(obj.findAll(text=True))
	return str.strip()

class SomethingSoupInfo(SoupLog):
	def __init__(self, kind, tail, max_age):
		SoupLog.__init__(self)
		html = fetcher.yoweb(kind, tail, max_age)
		self._soup = make_soup(html)

#---------- scraper for pirate pages ----------

class PirateInfo(SomethingSoupInfo):
	# Public data members:
	#  pi.standings = { 'Treasure Haul': 'Able' ... }
	#  pi.name = name
	#  pi.crew = (id, name)
	#  pi.flag = (id, name)
	#  pi.msgs = [ 'message describing problem with scrape' ]
		
	def __init__(self, pirate, max_age=300):
		SomethingSoupInfo.__init__(self,
			'pirate.wm?target=', pirate, max_age)
		self.name = pirate
		self._find_standings()
		self.crew = self._find_crewflag('crew',
			'^/yoweb/crew/info\\.wm')
		self.flag = self._find_crewflag('flag',
			'^/yoweb/flag/info\\.wm')

	def _find_standings(self):
		imgs = self._soup.findAll('img',
			src=regexp.compile('/yoweb/images/stat.*'))
		re = regexp.compile(
u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
			)
		standings = { }

		for skill in puzzles:
			standings[skill] = [ ]

		skl = SoupLog()

		for img in imgs:
			try: puzzle = img['alt']
			except KeyError: continue

			if not puzzle in puzzles:
				skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
				continue
			key = img.findParent('td')
			if key is None:
				skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
				continue
			valelem = key.findNextSibling('td')
			if valelem is None:
				skl.soupm(key, 'puzzle missing sibling "%s"'
					% puzzle)
				continue
			valstr = soup_text(valelem)
			match = re.match(valstr)
			if match is None:
				skl.soupm(key, ('puzzle "%s" unparseable'+
					' standing "%s"') % (puzzle, valstr))
				continue
			standing = match.group(match.lastindex)
			standings[puzzle].append(standing)

		self.standings = { }

		for puzzle in puzzles:
			sl = standings[puzzle]
			if len(sl) > 1:
				skl.msg('puzzle "%s" multiple standings %s' %
						(puzzle, `sl`))
				continue
			if not sl:
				skl.msg('puzzle "%s" no standing found' % puzzle)
				continue
			standing = sl[0]
			for i in range(0, standing_limit):
				if standing == standingvals[i]:
					self.standings[puzzle] = i
			if not puzzle in self.standings:
				skl.msg('puzzle "%s" unknown standing "%s"' %
					(puzzle, standing))

		all_standings_ok = True
		for puzzle in puzzles:
			if not puzzle in self.standings:
				self.needs_msgs(skl)

	def _find_crewflag(self, cf, yoweb_re):
		things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
		if len(things) != 1:
			self.msg('zero or several %s id references found' % cf)
			return None
		thing = things[0]
		id_re = '\\b%sid\\=(\\w+)$' % cf
		id_haystack = thing['href']
		match = regexp.compile(id_re).search(id_haystack)
		if match is None:
			self.soupm(thing, ('incomprehensible %s id ref'+
				' (%s in %s)') % (cf, id_re, id_haystack))
			return None
		name = soup_text(thing)
		return (match.group(1), name)

	def __str__(self):
		return `(self.crew, self.flag, self.standings, self.msgs)`

#---------- scraper for crew pages ----------

class CrewInfo(SomethingSoupInfo):
	# Public data members:
	#  ci.crewid
	#  ci.crew = [ ('Captain',        ['Pirate', ...]),
	#	       ('Senior Officer', [...]),
	#		... ]
	#  pi.msgs = [ 'message describing problem with scrape' ]

	def __init__(self, crewid, max_age=300):
		self.crewid = crewid
		SomethingSoupInfo.__init__(self,
			'crew/info.wm?crewid=', crewid, max_age)
		self._find_crew()

	def _find_crew(self):
		self.crew = []
		capts = self._soup.findAll('img',
			src='/yoweb/images/crew-captain.png')
		if len(capts) != 1:
			self.msg('crew members: no. of captain images != 1')
			return
		tbl = capts[0]
		while not tbl.find('a', href=pirate_ref_re):
			tbl = tbl.findParent('table')
			if not tbl:
				self.msg('crew members: cannot find table')
				return
		current_rank_crew = None
		crew_rank_re = regexp.compile('/yoweb/images/crew')
		for row in tbl.contents:
			# findAll(recurse=False)
			if isinstance(row,basestring):
				continue

			is_rank = row.find('img', attrs={'src': crew_rank_re})
			if is_rank:
				rank = soup_text(row)
				current_rank_crew = []
				self.crew.append((rank, current_rank_crew))
				continue
			for cell in row.findAll('a', href=pirate_ref_re):
				if current_rank_crew is None:
					self.soupm(cell, 'crew members: crew'
						' before rank')
					continue
				current_rank_crew.append(soup_text(cell))

	def __str__(self):
		return `(self.crew, self.msgs)`

class FlagRelation():
	# Public data members (put there by hand by creater)
	#	other_flagname
	#	other_flagid
	#	yoweb_heading
	#	this_declaring
	#	other_declaring_min
	#	other_declaring_max
	# where {this,other}_declaring{,_min,_max} are:
	#	-1	{this,other} is declaring war
	#	 0	{this,other} is not doing either
	#	+1	{this,other} is allying
	def __repr__(self):
		return '<FlagRelation %s %d/%d..%d %s %s>' % (
			self.yoweb_heading, self.this_declaring,
			self.other_declaring_min, self.other_declaring_max,
			self.other_flagname, self.other_flagid)

class FlagInfo(SomethingSoupInfo):
	# Public data members (after init):
	#
	#   flagid
	#   name	#		string
	#
	#   relations[n] = FlagRelation
	#   relation_byname[otherflagname] = relations[some_n]
	#   relation_byid[otherflagname] = relations[some_n]
	#
	#   islands[n] = (islandname, islandid)
	#
	def __init__(self, flagid, max_age=600):
		self.flagid = flagid
		SomethingSoupInfo.__init__(self,
			'flag/info.wm?flagid=', flagid, max_age)
		self._find_flag()

	def _find_flag(self):
		font2 = self._soup.find('font',{'size':'+2'})
		self.name = font2.find('b').contents[0]

		self.relations = [ ]
		self.relation_byname = { }
		self.relation_byid = { }
		self.islands = [ ]

		magnate = self._soup.find('img',{'src':
			'/yoweb/images/repute-MAGNATE.png'})
		warinfo = (magnate.findParent('table').findParent('tr').
			findNextSibling('tr').findNext('td',{'align':'left'}))

		def warn(m):
			print >>sys.stderr, 'WARNING: '+m

		def wi_warn(head, waritem):
			warn('unknown warmap item: %s: %s' % 
				(`head`, ``waritem``))

		def wihelp_item(waritem, thing):
			url = waritem.get('href', None)
			if url is None:
				return ('no url for '+thing,None,None)
			m = regexp.search('\?'+thing+'id=(\d+)$', url)
			if not m: return ('no '+thing+'id',None,None)
			tid = m.group(1)
			tname = waritem.string
			if tname is None:
				return (thing+' name not just string',None,None)
			return (None,tid,tname)

		def wi_alwar(head, waritem, thisdecl, othermin, othermax):
			(err,flagid,flagname) = wihelp_item(waritem,'flag')
			if err: return err
			rel = self.relation_byid.get(flagid, None)
			if rel: return 'flag id twice!'
			if flagname in self.relation_byname:
				return 'flag name twice!'
			rel = FlagRelation()
			rel.other_flagname = flagname
			rel.other_flagid = flagid
			rel.yoweb_heading = head
			rel.this_declaring = thisdecl
			rel.other_declaring_min = othermin
			rel.other_declaring_max = othermax
			self.relations.append(rel)
			self.relation_byid[flagid] = rel
			self.relation_byname[flagid] = rel

		def wi_isle(head, waritem):
			(err,isleid,islename) = wihelp_item(waritem,'island')
			if err: return err
			self.islands.append((isleid,islename))

		warmap = {
			'Allied with':			(wi_alwar,+1,+1,+1),
			'Declaring war against':	(wi_alwar,-1, 0,+1),
			'At war with':			(wi_alwar,-1,-1,-1),
			'Trying to form an alliance with': (wi_alwar,+1,-1,0),
			'Islands controlled by this flag': (wi_isle,),
			}

		how = (wi_warn, None)

		for waritem in warinfo.findAll(['font','a']):
			if waritem is None: break
			if waritem.name == 'font':
				colour = waritem.get('color',None)
				if colour.lstrip('#') != '958A5F':
					warn('strange colour %s in %s' %
						(colour,``waritem``))
					continue
				head = waritem.string
				if head is None:
					warn('no head string in '+``waritem``)
					continue
				head = regexp.sub('\\s+', ' ', head).strip()
				head = head.rstrip(':')
				how = (head,) + warmap.get(head, (wi_warn,))
				continue
			assert(waritem.name == 'a')				

			debug('WARHOW %s(%s, waritem, *%s)' %
				(how[1], `how[0]`, `how[2:]`))
			bad = how[1](how[0], waritem, *how[2:])
			if bad:
				warn('bad waritem %s: %s: %s' % (`how[0]`,
					bad, ``waritem``))

	def __str__(self):
		return `(self.name, self.islands, self.relations)`

#---------- scraper for ocean info incl. embargoes etc. ----------

class IslandBasicInfo():
	# Public data attributes:
	#  ocean
	#  name
	# Public data attributes maybe set by caller:
	#  arch
	def __init__(self, ocean, islename):
		self.ocean = ocean
		self.name = islename
	def yppedia(self):
		def q(x): return urllib.quote(x.replace(' ','_'))
		url_rhs = q(self.name) + '_(' + q(self.ocean) + ')'
		return yppedia(url_rhs)
	def __str__(self):
		return `(self.ocean, self.name)`

class IslandExtendedInfo(IslandBasicInfo):
	# Public data attributes (inherited):
	#  ocean
	#  name
	# Public data attributes (additional):
	#  islandid
	#  yoweb_url
	#  flagid
	def __init__(self, ocean, islename):
		IslandBasicInfo.__init__(self, ocean, islename)
		self.islandid = None
		self.yoweb_url = None
		self._collect_yoweb()
		self._collect_flagid()

	def _collect_yoweb(self):
		debug('IEI COLLECT YOWEB '+`self.name`)
		self.islandid = None
		self.yoweb_url = None

		soup = make_soup(self.yppedia())
		content = soup.find('div', attrs = {'id': 'content'})
		yoweb_re = regexp.compile('^http://\w+\.puzzlepirates\.com/'+
			'yoweb/island/info\.wm\?islandid=(\d+)$')
		a = soup.find('a', attrs = { 'href': yoweb_re })
		if a is None:
			debug('IEI COLLECT YOWEB '+`self.name`+' NONE')
			return

		debug('IEI COLLECT YOWEB '+`self.name`+' GOT '+``a``)
		self.yoweb_url = a['href']
		m = yoweb_re.search(self.yoweb_url)
		self.islandid = m.group(1)

	def _collect_flagid(self):
		self.flagid = None

		yo = self.yoweb_url
		debug('IEI COLLECT FLAGID '+`self.name`+' URL '+`yo`)
		if yo is None: return None
		dataf = fetcher.fetch(yo, 1800)
		soup = make_soup(dataf)
		ruler_re = regexp.compile(
			'/yoweb/flag/info\.wm\?flagid=(\d+)$')
		ruler = soup.find('a', attrs = { 'href': ruler_re })
		if not ruler: 
			debug('IEI COLLECT FLAGID '+`self.name`+' NONE')
			return
		debug('IEI COLLECT FLAGID '+`self.name`+' GOT '+``ruler``)
		m = ruler_re.search(ruler['href'])
		self.flagid = m.group(1)

	def __str__(self):
		return `(self.ocean, self.islandid, self.name,
			self.yoweb_url, self.flagid)`

class IslandFlagInfo(IslandExtendedInfo):
	# Public data attributes (inherited):
	#  ocean
	#  name
	#  islandid
	#  yoweb_url
	#  flagid
	# Public data attributes (additional):
	#  flag
	def __init__(self, ocean, islename):
		IslandExtendedInfo.__init__(self, ocean, islename)
		self.flag = None
		self._collect_flag()

	def _collect_flag(self):
		if self.flagid is None: return
		self.flag = FlagInfo(self.flagid, 1800)

	def __str__(self):
		return IslandExtendedInfo.__str__(self) + '; ' + str(self.flag)

class NullProgressReporter():
	def doing(self, msg): pass
	def stop(self): pass

class TypewriterProgressReporter():
	def __init__(self):
		self._l = 0
	def doing(self,m):
		self._doing(m + '...')
	def _doing(self,m):
		self._write('\r')
		self._write(m)
		less = self._l - len(m)
		if less > 0:
			self._write(' ' * less)
			self._write('\b' * less)
		self._l = len(m)
		sys.stdout.flush()
	def stop(self):
		self._doing('')
		self._l = 0
	def _write(self,t):
		sys.stdout.write(t)

class OceanInfo():
	# Public data attributes:
	#   oi.islands[islename] = IslandInfo(...)
	#   oi.arches[archname][islename] = IslandInfo(...)
	def __init__(self, isleclass=IslandBasicInfo):
		self.isleclass = isleclass
		self.ocean = fetcher.ocean.lower().capitalize()

		progressreporter.doing('fetching ocean info')

		cmdl = ['./yppedia-ocean-scraper']
		if opts.localhtml is not None:
			cmdl += ['--local-html-dir',opts.localhtml]
		cmdl += [self.ocean]
		debug('OceanInfo collect running ' + `cmdl`)
		oscraper = subprocess.Popen(
			cmdl,
			stdout = subprocess.PIPE,
			cwd = yppsc_dir()+'/yarrg',
			shell=False, stderr=None,
			)
		h = oscraper.stdout.readline()
		debug('OceanInfo collect h '+`h`)
		assert(regexp.match('^ocean ', h))
		arch_re = regexp.compile('^ (\S.*)')
		island_re = regexp.compile('^  (\S.*)')

		oscraper.wait()
		assert(oscraper.returncode == 0)

		self.islands = { }
		self.arches = { }
		archname = None

		isles = [ ]
		progressreporter.doing('parsing ocean info')

		for l in oscraper.stdout:
			debug('OceanInfo collect l '+`l`)
			l = l.rstrip('\n')
			m = island_re.match(l)
			if m:
				assert(archname is not None)
				islename = m.group(1)
				isles.append((archname, islename))
				continue
			m = arch_re.match(l)
			if m:
				archname = m.group(1)
				assert(archname not in self.arches)
				self.arches[archname] = { }
				continue
			assert(False)

		for i in xrange(0, len(isles)-1):
			(archname, islename) = isles[i]
			progressreporter.doing(
				'fetching isle info %2d/%d (%s: %s)'
				% (i, len(isles), archname, islename))
			isle = self.isleclass(self.ocean, islename)
			isle.arch = archname
			self.islands[islename] = isle
			self.arches[archname][islename] = isle

	def __str__(self):
		return `(self.islands, self.arches)`

#---------- pretty-printer for tables of pirate puzzle standings ----------

class StandingsTable:
	def __init__(self, f, use_puzzles=None, col_width=6, gap_every=5):
		if use_puzzles is None:
			if opts.ship_duty:
				use_puzzles=duty_puzzles
			else:
				use_puzzles=puzzles
		self._puzzles = use_puzzles
		self.f = f
		self._cw = col_width-1
		self._gap_every = gap_every
		self._linecount = 0
		self._o = f.write

	def _nl(self): self._o('\n')

	def _pline(self, lhs, puzstrs, extra):
		if (self._linecount > 0
		    and self._gap_every is not None
		    and not (self._linecount % self._gap_every)):
			self._nl()
		self._o('%-*s' % (max(max_pirate_namelen+1, 15), lhs))
		for v in puzstrs:
			self._o(' %-*.*s' % (self._cw,self._cw, v))
		if extra:
			self._o(' ' + extra)
		self._nl()
		self._linecount += 1

	def _puzstr(self, pi, puzzle):
		if not isinstance(puzzle,list): puzzle = [puzzle]
		try: standing = max([pi.standings[p] for p in puzzle])
		except KeyError: return '?'
		if not standing: return ''
		s = ''
		if self._cw > 4:
			c1 = standingvals[standing][0]
			if standing < 3: c1 = c1.lower() # 3 = Master
			s += `standing`
		if self._cw > 5:
			s += ' '
		s += '*' * (standing / 2)
		s += '+' * (standing % 2)
		return s

	def headings(self, lhs='', rhs=None):
		def puzn_redact(name):
			if isinstance(name,list):
				return '/'.join(
					["%.*s" % (self._cw/2, puzn_redact(n))
					 for n in name])
			spc = name.find(' ')
			if spc < 0: return name
			return name[0:min(4,spc)] + name[spc+1:]
		self._linecount = -2
		self._pline(lhs, map(puzn_redact, self._puzzles), rhs)
		self._linecount = 0
	def literalline(self, line):
		self._o(line)
		self._nl()
		self._linecount = 0
	def pirate_dummy(self, name, standingstring, extra=None):
		standings = standingstring * len(self._puzzles)
		self._pline(' '+name, standings, extra)
	def pirate(self, pi, extra=None):
		puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
		self._pline(' '+pi.name, puzstrs, extra)


#---------- chat log parser ----------

class PirateAboard:
	# This is essentially a transparent, dumb, data class.
	#  pa.v			may be None
	#  pa.name
	#  pa.last_time
	#  pa.last_event
	#  pa.gunner
	#  pa.last_chat_time
	#  pa.last_chat_chan
	#  pa.pi

	# Also used for jobbing applicants:
	#		happens when			expires (to "-")
	#   -		 disembark, leaves crew		 no
	#   aboard	 evidence of them being aboard	 no
	#   applied	 "has applied for the job"	 120s, configurable
	#   ashore	 "has taken a job"		 30min, configurable
	#   declined	 "declined the job offer"	 30s, configurable
	#   invited	 "has been invited to job"	 120s, configurable
	#
	#  pa.jobber	None, 'ashore', 'applied', 'invited', 'declined'
	#  pa.expires	expiry time time

	def __init__(pa, pn, v, time, event):
		pa.name = pn
		pa.v = v
		pa.last_time = time
		pa.last_event = event
		pa.last_chat_time = None
		pa.last_chat_chan = None
		pa.gunner = False
		pa.pi = None
		pa.jobber = None
		pa.expires = None

	def pirate_info(pa):
		now = time.time()
		if pa.pi:
			age = now - pa.pi_fetched
			guide = random.randint(120,240)
			if age <= guide:
				return pa.pi
			debug('PirateAboard refresh %d > %d  %s' % (
				age, guide, pa.name))
			imaginary = [2,4]
		else:
			imaginary = [1]
		wait = fetcher.need_wait(now, imaginary)
		if wait:
			debug('PirateAboard fetcher not ready %d' % wait)
			return pa.pi
		pa.pi = PirateInfo(pa.name, 600)
		pa.pi_fetched = now
		return pa.pi

class ChatLogTracker:
	# This is quite complex so we make it opaque.  Use the
	# official invokers, accessors etc.

	def __init__(self, myself_pi, logfn):
		self._pl = {}	# self._pl['Pirate'] =
		self._vl = {}	#   self._vl['Vessel']['Pirate'] = PirateAboard
				# self._vl['Vessel']['#lastinfo']
				# self._vl['Vessel']['#name']
				# self._v = self._vl[self._vessel]
		self._date = None
		self._myself = myself_pi
		self._lbuf = ''
		self._f	= file(logfn)
		flen = os.fstat(self._f.fileno()).st_size
		max_backlog = 500000
		if flen > max_backlog:
			startpos = flen - max_backlog
			self._f.seek(startpos)
			self._f.readline()
		self._progress = [0, flen - self._f.tell()]
		self._disembark_myself()
		self._need_redisplay = False
		self._lastvessel = None

	def _disembark_myself(self):
		self._v = None
		self._vessel = None
		self.force_redisplay()

	def force_redisplay(self):
		self._need_redisplay = True

	def _vessel_updated(self, v, timestamp):
		if v is None: return
		v['#lastinfo'] = timestamp
		self.force_redisplay()

	def _onboard_event(self,v,timestamp,pirate,event,jobber=None):
		pa = self._pl.get(pirate, None)
		if pa is not None and pa.v is v:
			pa.last_time = timestamp
			pa.last_event = event
		else:
			if pa is not None and pa.v is not None:
				del pa.v[pirate]
			pa = PirateAboard(pirate, v, timestamp, event)
			self._pl[pirate] = pa
			if v is not None: v[pirate] = pa
		pa.jobber = jobber

		if jobber is None: timeout = None
		else: timeout = getattr(opts, 'timeout_'+jobber)
		if timeout is None: pa.expires = None
		else: pa.expires = timestamp + timeout
		self._vessel_updated(v, timestamp)
		return pa

	def _expire_jobbers(self, now):
		for pa in self._pl.values():
			if pa.expires is None: continue
			if pa.expires >= now: continue
			v = pa.v
			del self._pl[pa.name]
			if v is not None: del v[pa.name]
			self.force_redisplay()

	def _trash_vessel(self, v):
		for pn in v:
			if pn.startswith('#'): continue
			del self._pl[pn]
		vn = v['#name']
		del self._vl[vn]
		if v is self._v: self._disembark_myself()
		self.force_redisplay()

	def _vessel_stale(self, v, timestamp):
		return timestamp - v['#lastinfo'] > opts.ship_reboard_clearout

	def _vessel_check_expire(self, v, timestamp):
		if not self._vessel_stale(v, timestamp):
			return v
		self._debug_line_disposition(timestamp,'',
			'stale-reset ' + v['#name'])
		self._trash_vessel(v)
		return None

	def expire_garbage(self, timestamp):
		for v in self._vl.values():
			self._vessel_check_expire(v, timestamp)

	def _vessel_lookup(self, vn, timestamp, dml=[], create=False):
		v = self._vl.get(vn, None)
		if v is not None:
			v = self._vessel_check_expire(v, timestamp)
		if v is not None:
			dml.append('found')
			return v
		if not create:
			dml.append('no')
		dml.append('new')
		self._vl[vn] = v = { '#name': vn }
		self._vessel_updated(v, timestamp)
		return v

	def _find_matching_vessel(self, pattern, timestamp, cmdr,
					dml=[], create=False):
		# use when a commander pirate `cmdr' specified a vessel
		#  by name `pattern' (either may be None)
		# if create is true, will create the vessel
		#  record if an exact name is specified

		if (pattern is not None and
		    not '*' in pattern
		    and len(pattern.split(' ')) == 2):
			vn = pattern.title()
			dml.append('exact')
			return self._vessel_lookup(
				vn, timestamp, dml=dml, create=create)

		if pattern is None:
			pattern_check = lambda vn: True
		else:
			re = '(?:.* )?%s$' % pattern.lower().replace('*','.+')
			pattern_check = regexp.compile(re, regexp.I).match

		tries = []

		cmdr_pa = self._pl.get(cmdr, None)
		if cmdr_pa: tries.append((cmdr_pa.v, 'cmdr'))

		tries.append((self._v, 'here'))
		tried_vns = []

		for (v, dm) in tries:
			if v is None: dml.append(dm+'?'); continue
			
			vn = v['#name']
			if not pattern_check(vn):
				tried_vns.append(vn)
				dml.append(dm+'#')
				continue

			dml.append(dm+'!')
			return v

		if pattern is not None and '*' in pattern:
			search = [
				(vn,v)
				for (vn,v) in self._vl.iteritems()
				if not self._vessel_stale(v, timestamp)
				if pattern_check(vn)
				]
			#debug('CLT-RE /%s/ wanted (%s) searched (%s)' % (
			#	re,
			#	'/'.join(tried_vns),
			#	'/'.join([vn for (vn,v) in search])))

			if len(search)==1:
				dml.append('one')
				return search[0][1]
			elif search:
				dml.append('many')
			else:
				dml.append('none')

	def _debug_line_disposition(self,timestamp,l,m):
		debug('CLT %13s %-40s %s' % (timestamp,m,l))

	def _rm_crew_l(self,re,l):
		m = regexp.match(re,l)
		if m and m.group(2) == self._myself.crew[1]:
			return m.group(1)
		else:
			return None

	def local_command(self, metacmd):
		# returns None if all went well, or problem message
		return self._command(self._myself.name, metacmd,
			"local", time.time(), 
			(lambda m: debug('CMD %s' % metacmd)))

	def _command(self, cmdr, metacmd, chan, timestamp, d):
		# returns None if all went well, or problem message
		metacmd = regexp.sub('\\s+', ' ', metacmd).strip()
		m2 = regexp.match(
		    '/([adj]) (?:([A-Za-z* ]+)\\s*:)?([A-Za-z ]+)$',
		    metacmd)
		if not m2: return "unknown syntax or command"

		(cmd, pattern, targets) = m2.groups()
		dml = ['cmd', chan, cmd]

		if cmd == 'a': each = self._onboard_event
		elif cmd == 'd': each = disembark
		else: each = lambda *l: self._onboard_event(*l,
				**{'jobber':'applied'})

		if cmdr == self._myself.name:
			dml.append('self')
			how = 'cmd: %s' % cmd
		else:
			dml.append('other')
			how = 'cmd: %s %s' % (cmd,cmdr)

		if cmd == 'j':
			if pattern is not None:
				return "/j command does not take a vessel"
			v = None
		else:
			v = self._find_matching_vessel(
				pattern, timestamp, cmdr,
				dml, create=True)

		if cmd == 'j' or v is not None:
			targets = targets.strip().split(' ')
			dml.append(`len(targets)`)
			for target in targets:
				each(v, timestamp, target.title(), how)
			self._vessel_updated(v, timestamp)

		dm = ' '.join(dml)
		return d(dm)

		return None

	def chatline(self,l):
		rm = lambda re: regexp.match(re,l)
		d = lambda m: self._debug_line_disposition(timestamp,l,m)
		rm_crew = lambda re: self._rm_crew_l(re,l)
		timestamp = None

		m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$')
		if m:
			self._date = [int(x) for x in m.groups()]
			self._previous_timestamp = None
			return d('date '+`self._date`)

		if self._date is None:
			return d('date unset')

		m = rm('\\[(\d\d):(\d\d):(\d\d)\\] ')
		if not m:
			return d('no timestamp')

		while True:
			time_tuple = (self._date +
				      [int(x) for x in m.groups()] +
				      [-1,-1,-1])
			timestamp = time.mktime(time_tuple)
			if timestamp >= self._previous_timestamp: break
			self._date[2] += 1
			self._debug_line_disposition(timestamp,'',
				'new date '+`self._date`)

		self._previous_timestamp = timestamp

		l = l[l.find(' ')+1:]

		def ob_x(pirate,event):
			return self._onboard_event(
					self._v, timestamp, pirate, event)
		def ob1(did): ob_x(m.group(1), did); return d(did)
		def oba(did): return ob1('%s %s' % (did, m.group(2)))

		def jb(pirate,jobber):
			return self._onboard_event(
				None, timestamp, pirate,
				("jobber %s" % jobber),
				jobber=jobber
				)

		def disembark(v, timestamp, pirate, event):
			self._onboard_event(
					v, timestamp, pirate, 'leaving '+event)
			del v[pirate]
			del self._pl[pirate]

		def disembark_me(why):
			self._disembark_myself()
			return d('disembark-me '+why)

		m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
		if m:
			dm = ['boarding']
			pn = self._myself.name
			vn = m.group(1)
			v = self._vessel_lookup(vn, timestamp, dm, create=True)
			self._lastvessel = self._vessel = vn
			self._v = v
			ob_x(pn, 'we boarded')
			self.expire_garbage(timestamp)
			return d(' '.join(dm))

		if self._v is None:
			return d('no vessel')

		m = rm('(\\w+) has come aboard\\.$')
		if m: return ob1('boarded');

		m = rm('You have ordered (\\w+) to do some (\\S.*\\S)\\.$')
		if m:
			(who,what) = m.groups()
			pa = ob_x(who,'ord '+what)
			if what == 'Gunning':
				pa.gunner = True
			return d('duty order')

		m = rm('(\\w+) abandoned a (\\S.*\\S) station\\.$')
		if m: oba('stopped'); return d("end")

		def chat_core(speaker, chan):
			try: pa = self._pl[speaker]
			except KeyError: return 'mystery'
			if pa.v is not None and pa.v is not self._v:
				return 'elsewhere'
			pa.last_chat_time = timestamp
			pa.last_chat_chan = chan
			self.force_redisplay()
			return 'here'

		def chat(chan):
			speaker = m.group(1)
			dm = chat_core(speaker, chan)
			return d('chat %s %s' % (chan, dm))

		def chat_metacmd(chan):
			(cmdr, metacmd) = m.groups()
			whynot = self._command(
				cmdr, metacmd, chan, timestamp, d)
			if whynot is not None:
				return chat(chan)
			else:
				chat_core(cmdr, 'cmd '+chan)

		m = rm('(\\w+) (?:issued an order|ordered everyone) "')
		if m: return ob1('general order');

		m = rm('(\\w+) says, "')
		if m: return chat('public')

		m = rm('(\\w+) tells ye, "')
		if m: return chat('private')

		m = rm('Ye told (\\w+), "(.*)"$')
		if m: return chat_metacmd('private')

		m = rm('(\\w+) flag officer chats, "')
		if m: return chat('flag officer')

		m = rm('(\\w+) officer chats, "(.*)"$')
		if m: return chat_metacmd('officer')

		m = rm('Ye accepted the offer to job with ')
		if m: return disembark_me('jobbing')

		m = rm('Ye hop on the ferry and are whisked away ')
		if m: return disembark_me('ferry')

		m = rm('Whisking away to yer home on the magical winds')
		if m: return disembark_me('home')

		m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
		if m:
			pl = m.group(1).split(', ')
			if not self._myself.name in pl:
				return d('lost melee')
			for pn in pl:
				if ' ' in pn: continue
				ob_x(pn,'won melee')
			return d('won melee')

		m = rm('(\\w+) is eliminated\\!')
		if m: return ob1('eliminated in fray');

		m = rm('(\\w+) has driven \w+ from the ship\\!')
		if m: return ob1('boarder repelled');

		m = rm('\w+ has bested (\\w+), and turns'+
			' to the rest of the ship\\.')
		if m: return ob1('boarder unrepelled');

		pirate = rm_crew("(\\w+) has taken a job with '(.*)'\\.")
		if pirate: return jb(pirate, 'ashore')

		pirate = rm_crew("(\\w+) has left '(.*)'\\.")
		if pirate:
			disembark(self._v, timestamp, pirate, 'left crew')
			return d('left crew')

		m = rm('(\w+) has applied for the posted job\.')
		if m: return jb(m.group(1), 'applied')

		pirate= rm_crew("(\\w+) has been invited to job for '(.*)'\\.")
		if pirate: return jb(pirate, 'invited')

		pirate = rm_crew("(\\w+) declined the job offer for '(.*)'\\.")
		if pirate: return jb(pirate, 'declined')

		m = rm('(\\w+) has left the vessel\.')
		if m:
			pirate = m.group(1)
			disembark(self._v, timestamp, pirate, 'disembarked')
			return d('disembarked')

		return d('not-matched')

	def _str_pa(self, pn, pa):
		assert self._pl[pn] == pa
		s = ' '*20 + "%s %-*s %13s %-30s %13s %-20s %13s" % (
			(' ','G')[pa.gunner],
			max_pirate_namelen, pn,
			pa.last_time, pa.last_event,
			pa.last_chat_time, pa.last_chat_chan,
			pa.jobber)
		if pa.expires is not None:
			s += " %-5d" % (pa.expires - pa.last_time)
		s += "\n"
		return s

	def _str_vessel(self, vn, v):
		s = ' vessel %s\n' % vn
		s += ' '*20 + "%-*s   %13s\n" % (
				max_pirate_namelen, '#lastinfo',
				v['#lastinfo'])
		assert v['#name'] == vn
		for pn in sorted(v.keys()):
			if pn.startswith('#'): continue
			pa = v[pn]
			assert pa.v == v
			s += self._str_pa(pn,pa)
		return s

	def __str__(self):
		s = '''<ChatLogTracker
 myself %s
 vessel %s
'''			% (self._myself.name, self._vessel)
		assert ((self._v is None and self._vessel is None) or
			(self._v is self._vl[self._vessel]))
		if self._vessel is not None:
			s += self._str_vessel(self._vessel, self._v)
		for vn in sorted(self._vl.keys()):
			if vn == self._vessel: continue
			s += self._str_vessel(vn, self._vl[vn])
		s += " elsewhere\n"
		for p in self._pl:
			pa = self._pl[p]
			if pa.v is not None:
				assert pa.v[p] is pa
				assert pa.v in self._vl.values()
			else:
				s += self._str_pa(pa.name, pa)
		s += '>\n'
		return s

	def catchup(self, progress=None):
		while True:
			more = self._f.readline()
			if not more: break

			self._progress[0] += len(more)
			if progress: progress.progress(*self._progress)

			self._lbuf += more
			if self._lbuf.endswith('\n'):
				self.chatline(self._lbuf.rstrip())
				self._lbuf = ''
				if opts.debug >= 2:
					debug(self.__str__())
		self._expire_jobbers(time.time())

		if progress: progress.caughtup()

	def changed(self):
		rv = self._need_redisplay
		self._need_redisplay = False
		return rv
	def myname(self):
		# returns our pirate name
		return self._myself.name
	def vesselname(self):
		# returns the vessel name we're aboard or None
		return self._vessel
	def lastvesselname(self):
		# returns the last vessel name we were aboard or None
		return self._lastvessel
	def aboard(self, vesselname=True):
		# returns a list of PirateAboard the vessel
		#  sorted by pirate name
		#  you can pass this None and you'll get []
		#  or True for the current vessel (which is the default)
		#  the returned value is a fresh list of persistent
		#  PirateAboard objects
		if vesselname is True: v = self._v
		else: v = self._vl.get(vesselname.title())
		if v is None: return []
		return [ v[pn]
			 for pn in sorted(v.keys())
			 if not pn.startswith('#') ]
	def jobbers(self):
		# returns a the jobbers' PirateAboards,
		# sorted by jobber class and reverse of expiry time
		l = [ pa
		      for pa in self._pl.values()
		      if pa.jobber is not None
		    ]
		def compar_key(pa):
			return (pa.jobber, -pa.expires)
		l.sort(key = compar_key)
		return l

#---------- implementations of actual operation modes ----------

def do_pirate(pirates, bu):
	print '{'
	for pirate in pirates:
		info = PirateInfo(pirate)
		print '%s: %s,' % (`pirate`, info)
	print '}'

def prep_crewflag_of(args, bu, max_age, selector, constructor):
	if len(args) != 1: bu('crew-of etc. take one pirate name')
	pi = PirateInfo(args[0], max_age)
	cf = selector(pi)
	if cf is None: return None
	return constructor(cf[0], max_age)

def prep_crew_of(args, bu, max_age=300):
	return prep_crewflag_of(args, bu, max_age,
		(lambda pi: pi.crew), CrewInfo)

def prep_flag_of(args, bu, max_age=300):
	return prep_crewflag_of(args, bu, max_age,
		(lambda pi: pi.flag), FlagInfo)

def do_crew_of(args, bu):
	ci = prep_crew_of(args, bu)
	print ci

def do_flag_of(args, bu):
	fi = prep_flag_of(args, bu)
	print fi

def do_standings_crew_of(args, bu):
	ci = prep_crew_of(args, bu, 60)
	tab = StandingsTable(sys.stdout)
	tab.headings()
	for (rank, members) in ci.crew:
		if not members: continue
		tab.literalline('')
		tab.literalline('%s:' % rank)
		for p in members:
			pi = PirateInfo(p, random.randint(900,1800))
			tab.pirate(pi)

def do_ocean(args, bu):
	if (len(args)): bu('ocean takes no further arguments')
	fetcher.default_ocean()
	oi = OceanInfo(IslandFlagInfo)
	print oi
	for islename in sorted(oi.islands.keys()):
		isle = oi.islands[islename]
		print isle

def do_embargoes(args, bu):
	if (len(args)): bu('ocean takes no further arguments')
	fetcher.default_ocean()
	oi = OceanInfo(IslandFlagInfo)
	wr = sys.stdout.write
	print ('EMBARGOES:  Island    | Owning flag'+
		'                    | Embargoed flags')

	def getflname(isle):
		if isle.islandid is None: return 'uncolonisable'
		if isle.flag is None: return 'uncolonised'
		return isle.flag.name

	progressreporter.stop()

	for archname in sorted(oi.arches.keys()):
		print 'ARCHIPELAGO: ',archname
		for islename in sorted(oi.arches[archname].keys()):
			isle = oi.islands[islename]
			wr(' %-20s | ' % isle.name)
			flname = getflname(isle)
			wr('%-30s | ' % flname)
			flag = isle.flag
			if flag is None: print ''; continue
			delim = ''
			for rel in flag.relations:
				if rel.this_declaring >= 0: continue
				wr(delim)
				wr(rel.other_flagname)
				delim = '; '
			print ''

def do_embargoes_flag_of(args, bu):
	progressreporter.doing('fetching flag info')
	fi = prep_flag_of(args, bu)
	if fi is None:
		progressreporter.stop()
		print 'Pirate is not in a flag.'
		return

	oi = OceanInfo(IslandFlagInfo)

	progressreporter.stop()
	print ''

	any = False
	for islename in sorted(oi.islands.keys()):
		isle = oi.islands[islename]
		flag = isle.flag
		if flag is None: continue
		for rel in flag.relations:
			if rel.this_declaring >= 0: continue
			if rel.other_flagid != fi.flagid: continue
			if not any: print 'EMBARGOED:'
			any = True
			print "  %-30s (%s)" % (islename, flag.name)
	if not any:
		print 'No embargoes.'
	print ''

	war_flag(fi)
	print ''

def do_war_flag_of(args, bu):
	fi = prep_flag_of(args, bu)
	war_flag(fi)

def war_flag(fi):
	any = False
	for certain in [True, False]:
		anythis = False
		for rel in fi.relations:
			if rel.this_declaring >= 0: continue
			if (rel.other_declaring_max < 0) != certain: continue
			if not anythis:
				if certain: m = 'SINKING PvP'
				else: m = 'RISK OF SINKING PvP'
				print '%s (%s):' % (m, rel.yoweb_heading)
			anythis = True
			any = True
			print " ", rel.other_flagname
	if not any:
		print 'No sinking PvP.'

#----- modes which use the chat log parser are quite complex -----

class ProgressPrintPercentage:
	def __init__(self, f=sys.stdout):
		self._f = f
	def progress_string(self,done,total):
		return "scan chat logs %3d%%\r" % ((done*100) / total)
	def progress(self,*a):
		self._f.write(self.progress_string(*a))
		self._f.flush()
	def show_init(self, pirate, ocean):
		print >>self._f, 'Starting up, %s on the %s ocean' % (
			pirate, ocean)
	def caughtup(self):
		self._f.write('                   \r')
		self._f.flush()

def prep_chat_log(args, bu,
		progress=ProgressPrintPercentage(),
		max_myself_age=3600):
	if len(args) != 1: bu('this action takes only chat log filename')
	logfn = args[0]
	logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_'
	match = regexp.match(logfn_re, logfn)
	if not match: bu('chat log filename is not in expected format')
	(pirate, ocean) = match.groups()
	fetcher.default_ocean(ocean)

	progress.show_init(pirate, fetcher.ocean)
	myself = PirateInfo(pirate,max_myself_age)
	track = ChatLogTracker(myself, logfn)

	opts.debug -= 2
	track.catchup(progress)
	opts.debug += 2

	track.force_redisplay()

	return (myself, track)

def do_track_chat_log(args, bu):
	(myself, track) = prep_chat_log(args, bu)
	while True:
		track.catchup()
		if track.changed():
			print track
		sleep(0.5 + 0.5 * random.random())

#----- ship management aid -----

class Display_dumb(ProgressPrintPercentage):
	def __init__(self):
		ProgressPrintPercentage.__init__(self)
	def show(self, s):
		print '\n\n', s;
	def realstart(self):
		pass

class Display_overwrite(ProgressPrintPercentage):
	def __init__(self):
		ProgressPrintPercentage.__init__(self)

		null = file('/dev/null','w')
		curses.setupterm(fd=null.fileno())

		self._clear = curses.tigetstr('clear')
		if not self._clear:
			self._debug('missing clear!')
			self.show = Display_dumb.show
			return

		self._t = {'el':'', 'ed':''}
		if not self._init_sophisticated():
			for k in self._t.keys(): self._t[k] = ''
			self._t['ho'] = self._clear

	def _debug(self,m): debug('display overwrite: '+m)

	def _init_sophisticated(self):
		for k in self._t.keys():
			s = curses.tigetstr(k)
			self._t[k] = s
		self._t['ho'] = curses.tigetstr('ho')
		if not self._t['ho']:
			cup = curses.tigetstr('cup')
			self._t['ho'] = curses.tparm(cup,0,0)
		missing = [k for k in self._t.keys() if not self._t[k]]
		if missing:
			self.debug('missing '+(' '.join(missing)))
			return 0
		return 1

	def show(self, s):
		w = sys.stdout.write
		def wti(k): w(self._t[k])

		wti('ho')
		nl = ''
		for l in s.rstrip().split('\n'):
			w(nl)
			w(l)
			wti('el')
			nl = '\r\n'
		wti('ed')
		w(' ')
		sys.stdout.flush()

	def realstart(self):
		sys.stdout.write(self._clear)
		sys.stdout.flush()
			

def do_ship_aid(args, bu):
	if opts.ship_duty is None: opts.ship_duty = True

	displayer = globals()['Display_'+opts.display]()

	(myself, track) = prep_chat_log(args, bu, progress=displayer)

	displayer.realstart()

	if os.isatty(0): kr_create = KeystrokeReader
	else: kr_create = DummyKeystrokeReader

	try:
		kreader = kr_create(0, 10)
		ship_aid_core(myself, track, displayer, kreader)
	finally:
		kreader.stop()
		print '\n'

class KeyBasedSorter:
	def compar_key_pa(self, pa):
		pi = pa.pirate_info()
		if pi is None: return None
		return self.compar_key(pi)
	def lsort_pa(self, l):
		l.sort(key = self.compar_key_pa)

class NameSorter(KeyBasedSorter):
	def compar_key(self, pi): return pi.name
	def desc(self): return 'name'

class SkillSorter(NameSorter):
	def __init__(self, relevant):
		self._want = frozenset(relevant.split('/'))
		self._avoid = set()
		for p in core_duty_puzzles:
			if isinstance(p,basestring): self._avoid.add(p)
			else: self._avoid |= set(p)
		self._avoid -= self._want
		self._desc = '%s' % relevant
	
	def desc(self): return self._desc

	def compar_key(self, pi):
		best_want = max([
			pi.standings.get(puz,-1)
			for puz in self._want
			])
		best_avoid = [
			-pi.standings.get(puz,standing_limit)
			for puz in self._avoid
			]
		best_avoid.sort()
		def negate(x): return -x
		debug('compar_key %s bw=%s ba=%s' % (pi.name, `best_want`,
			`best_avoid`))
		return (-best_want, map(negate, best_avoid), pi.name)

def ship_aid_core(myself, track, displayer, kreader):

	def find_vessel():
		vn = track.vesselname()
		if vn: return (vn, " on board the %s" % vn)
		vn = track.lastvesselname()
		if vn: return (vn, " ashore from the %s" % vn)
		return (None, " not on a vessel")

	def timeevent(t,e):
		if t is None: return ' ' * 22
		return " %-4s %-16s" % (format_time_interval(now - t),e)

	displayer.show(track.myname() + find_vessel()[1] + '...')

	rotate_nya = '/-\\'

	sort = NameSorter()
	clicmd = None
	clierr = None
	cliexec = None

	while True:
		track.catchup()
		now = time.time()

		(vn, vs) = find_vessel()

		s = ''
		if cliexec is not None:
			s += '...'
		elif clierr is not None:
			s += 'Error: '+clierr
		elif clicmd is not None:
			s += '/' + clicmd
		else:
			s = track.myname() + vs
			s += " at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
			s += kreader.info()
		s += '\n'

		tbl_s = StringIO()
		tbl = StandingsTable(tbl_s)

		aboard = track.aboard(vn)
		sort.lsort_pa(aboard)

		jobbers = track.jobbers()

		if track.vesselname(): howmany = 'aboard: %2d' % len(aboard)
		else: howmany = ''

		tbl.headings(howmany, '  sorted by '+sort.desc())

		last_jobber = None

		for pa in aboard + jobbers:
			if pa.jobber != last_jobber:
				last_jobber = pa.jobber
				tbl.literalline('')
				tbl.literalline('jobbers '+last_jobber)

			pi = pa.pirate_info()

			xs = ''
			if pa.gunner: xs += 'G '
			else: xs += '  '
			xs += timeevent(pa.last_time, pa.last_event)
			xs += timeevent(pa.last_chat_time, pa.last_chat_chan)

			if pi is None:
				tbl.pirate_dummy(pa.name, rotate_nya[0], xs)
			else:
				tbl.pirate(pi, xs)

		s += tbl_s.getvalue()
		displayer.show(s)
		tbl_s.close()

		if cliexec is not None:
			clierr = track.local_command("/"+cliexec.strip())
			cliexec = None
			continue

		k = kreader.getch()
		if k is None:
			rotate_nya = rotate_nya[1:3] + rotate_nya[0]
			continue

		if clierr is not None:
			clierr = None
			continue

		if clicmd is not None:
			if k == '\r' or k == '\n':
				cliexec = clicmd
				clicmd = clicmdbase
			elif k == '\e' and clicmd != "":
				clicmd = clicmdbase
			elif k == '\33':
				clicmd = None
			elif k == '\b' or k == '\177':
				clicmd = clicmd[ 0 : len(clicmd)-1 ]
			else:
				clicmd += k
			continue

		if k == 'q': break
		elif k == 'g': sort = SkillSorter('Gunning')
		elif k == 'c': sort = SkillSorter('Carpentry')
		elif k == 's': sort = SkillSorter('Sailing/Rigging')
		elif k == 'b': sort = SkillSorter('Bilging')
		elif k == 'n': sort = SkillSorter('Navigating')
		elif k == 'd': sort = SkillSorter('Battle Navigation')
		elif k == 't': sort = SkillSorter('Treasure Haul')
		elif k == 'a': sort = NameSorter()
		elif k == '/': clicmdbase = ""; clicmd = clicmdbase
		elif k == '+': clicmdbase = "a "; clicmd = clicmdbase
		else: pass # unknown key command

#---------- individual keystroke input ----------

class DummyKeystrokeReader:
	def __init__(self,fd,timeout_dummy): pass
	def stop(self): pass
	def getch(self): sleep(1); return None
	def info(self): return ' [noninteractive]'

class KeystrokeReader(DummyKeystrokeReader):
	def __init__(self, fd, timeout_decisec=0):
		self._fd = fd
		self._saved = termios.tcgetattr(fd)
		a = termios.tcgetattr(fd)
		a[3] &= ~(termios.ECHO | termios.ECHONL |
			  termios.ICANON | termios.IEXTEN)
		a[6][termios.VMIN] = 0
		a[6][termios.VTIME] = timeout_decisec
		termios.tcsetattr(fd, termios.TCSANOW, a)
	def stop(self):
		termios.tcsetattr(self._fd, termios.TCSANOW, self._saved)
	def getch(self):
		debug_flush()
		byte = os.read(self._fd, 1)
		if not len(byte): return None
		return byte
	def info(self):
		return ''

#---------- main program ----------

def main():
	global opts, fetcher, yppedia, progressreporter

	pa = OptionParser(
'''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
actions:
 yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
 yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
 yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
 yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG
 yoweb-scrape [--ocean OCEAN ...] ocean|embargoes
 yoweb-scrape [--ocean OCEAN ...] war-flag-of|embargoes-flag-of PIRATE
 yoweb-scrape [options] ship-aid CHAT-LOG  (must be .../PIRATE_OCEAN_chat-log*)

display modes (for --display) apply to ship-aid:
 --display=dumb       just print new information, scrolling the screen
 --display=overwrite  use cursor motion, selective clear, etc. to redraw at top''')
	ao = pa.add_option
	ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
		help='select ocean OCEAN')
	ao('--cache-dir', dest='cache_dir', metavar='DIR',
		default='~/.yoweb-scrape-cache',
		help='cache yoweb pages in DIR')
	ao('-D','--debug', action='count', dest='debug', default=0,
		help='enable debugging output')
	ao('--debug-fd', type='int', dest='debug_fd',
		help='write any debugging output to specified fd')
	ao('-q','--quiet', action='store_true', dest='quiet',
		help='suppress warning output')
	ao('--display', action='store', dest='display',
		type='choice', choices=['dumb','overwrite'],
		help='how to display ship aid')
	ao('--local-ypp-dir', action='store', dest='localhtml',
		help='get yppedia pages from local directory LOCALHTML'+
			' instead of via HTTP')

	ao_jt = lambda wh, t: ao(
		'--timeout-sa-'+wh, action='store', dest='timeout_'+wh,
		default=t, help=('set timeout for expiring %s jobbers' % wh))
	ao_jt('applied',      120)
	ao_jt('invited',      120)
	ao_jt('declined',      30)
	ao_jt('ashore',      1800)

	ao('--ship-duty', action='store_true', dest='ship_duty',
		help='show ship duty station puzzles')
	ao('--all-puzzles', action='store_false', dest='ship_duty',
		help='show all puzzles, not just ship duty stations')

	ao('--min-cache-reuse', type='int', dest='min_max_age',
		metavar='SECONDS', default=60,
		help='always reuse cache yoweb data if no older than this')

	(opts,args) = pa.parse_args()
	random.seed()

	if len(args) < 1:
		print >>sys.stderr, copyright_info
		pa.error('need a mode argument')

	if opts.debug_fd is not None:
		opts.debug_file = os.fdopen(opts.debug_fd, 'w')
	else:
		opts.debug_file = sys.stdout

	mode = args[0]
	mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
	try: mode_fn = globals()[mode_fn_name]
	except KeyError: pa.error('unknown mode "%s"' % mode)

	# fixed parameters
	opts.expire_age = max(3600, opts.min_max_age)

	opts.ship_reboard_clearout = 3600

	if opts.cache_dir.startswith('~/'):
		opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]

	if opts.display is None:
		if ((opts.debug > 0 and opts.debug_fd is None)
		    or not os.isatty(sys.stdout.fileno())):
			opts.display = 'dumb'
		else:
			opts.display = 'overwrite'

	fetcher = Yoweb(opts.ocean, opts.cache_dir)
	yppedia = Yppedia(opts.cache_dir)

	if opts.debug or not os.isatty(0):
		 progressreporter = NullProgressReporter()
	else:
		progressreporter = TypewriterProgressReporter()

	mode_fn(args[1:], pa.error)

main()