#!/usr/bin/python

#---------- setup ----------

import signal
signal.signal(signal.SIGINT, signal.SIG_DFL)

import os
import time
import urllib
import urllib2
import errno
import sys
import re as regexp
import random
import curses
import termios
from optparse import OptionParser

from BeautifulSoup import BeautifulSoup

opts = None

#---------- YPP parameters and arrays ----------

puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
	'/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
	'/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
	'/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')

standingvals = ('Able/Distinguished/Respected/Master'+
		'/Renowned/Grand-Master/Legendary/Ultimate').split('/')

pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')

max_pirate_namelen = 12


#---------- general utilities ----------

def debug(m):
	if opts.debug > 0:
		print >>opts.debug_file, m

def debug_flush():
	if opts.debug > 0:
		opts.debug_file.flush()	

def sleep(seconds):
	debug_flush()
	time.sleep(seconds)

def format_time_interval(ti):
	if ti < 120: return '%d:%02d' % (ti / 60, ti % 60)
	if ti < 7200: return '%2dm' % (ti / 60)
	if ti < 86400: return '%dh' % (ti / 3600)
	return '%dd' % (ti / 86400)

#---------- caching and rate-limiting data fetcher ----------

class Fetcher:
	def __init__(self, ocean, cachedir):
		debug('Fetcher init %s' % cachedir)
		self.ocean = ocean
		self.cachedir = cachedir
		try: os.mkdir(cachedir)
		except (OSError,IOError), oe:
			if oe.errno != errno.EEXIST: raise
		self._cache_scan(time.time())

	def default_ocean(self, ocean='ice'):
		if self.ocean is None:
			self.ocean = ocean

	def _cache_scan(self, now):
		# returns list of ages, unsorted
		ages = []
		debug('Fetcher   scan_cache')
		for leaf in os.listdir(self.cachedir):
			if not leaf.startswith('#'): continue
			path = self.cachedir + '/' + leaf
			try: s = os.stat(path)
			except (OSError,IOError), oe:
				if oe.errno != errno.ENOENT: raise
				continue
			age = now - s.st_mtime
			if age > opts.expire_age:
				debug('Fetcher    expire %d %s' % (age, path))
				try: os.remove(path)
				except (OSError,IOError), oe:
					if oe.errno != errno.ENOENT: raise
				continue
			ages.append(age)
		return ages

	def need_wait(self, now, imaginary=[]):
		ages = self._cache_scan(now)
		ages += imaginary
		ages.sort()
		debug('Fetcher   ages ' + `ages`)
		min_age = 1
		need_wait = 0
		for age in ages:
			if age < min_age and age < 300:
				debug('Fetcher   morewait min=%d age=%d' %
					(min_age, age))
				need_wait = max(need_wait, min_age - age)
			min_age += 3
			min_age *= 1.25
		return need_wait

	def _rate_limit_cache_clean(self, now):
		need_wait = self.need_wait(now)
		if need_wait > 0:
			debug('Fetcher   wait %d' % need_wait)
			sleep(need_wait)

	def fetch(self, url, max_age):
		debug('Fetcher fetch %s' % url)
		cache_corename = urllib.quote_plus(url)
		cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
		try: f = file(cache_item, 'r')
		except (OSError,IOError), oe:
			if oe.errno != errno.ENOENT: raise
			f = None
		now = time.time()
		max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
		if f is not None:
			s = os.fstat(f.fileno())
			age = now - s.st_mtime
			if age > max_age:
				debug('Fetcher  stale %d < %d'% (max_age, age))
				f = None
		if f is not None:
			data = f.read()
			f.close()
			debug('Fetcher  cached %d > %d' % (max_age, age))
			return data

		debug('Fetcher  fetch')
		self._rate_limit_cache_clean(now)

		stream = urllib2.urlopen(url)
		data = stream.read()
		cache_tmp = "%s/#%s~%d#" % (
			self.cachedir, cache_corename, os.getpid())
		f = file(cache_tmp, 'w')
		f.write(data)
		f.close()
		os.rename(cache_tmp, cache_item)
		debug('Fetcher  stored')
		return data

	def yoweb(self, kind, tail, max_age):
		self.default_ocean()
		url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
			self.ocean, kind, tail)
		return self.fetch(url, max_age)

#---------- logging assistance for troubled screenscrapers ----------

class SoupLog:
	def __init__(self):
		self.msgs = [ ]
	def msg(self, m):
		self.msgs.append(m)
	def soupm(self, obj, m):
		self.msg(m + '; in ' + `obj`)
	def needs_msgs(self, child_souplog):
		self.msgs += child_souplog.msgs
		child_souplog.msgs = [ ]

def soup_text(obj):
	str = ''.join(obj.findAll(text=True))
	return str.strip()

class SomethingSoupInfo(SoupLog):
	def __init__(self, kind, tail, max_age):
		SoupLog.__init__(self)
		html = fetcher.yoweb(kind, tail, max_age)
		self._soup = BeautifulSoup(html,
			convertEntities=BeautifulSoup.HTML_ENTITIES
			)

#---------- scraper for pirate pages ----------

class PirateInfo(SomethingSoupInfo):
	# Public data members:
	#  pi.standings = { 'Treasure Haul': 'Able' ... }
	#  pi.name = name
	#  pi.crew = (id, name)
	#  pi.flag = (id, name)
	#  pi.msgs = [ 'message describing problem with scrape' ]
		
	def __init__(self, pirate, max_age=300):
		SomethingSoupInfo.__init__(self,
			'pirate.wm?target=', pirate, max_age)
		self.name = pirate
		self._find_standings()
		self.crew = self._find_crewflag('crew',
			'^/yoweb/crew/info\\.wm')
		self.flag = self._find_crewflag('flag',
			'^/yoweb/flag/info\\.wm')

	def _find_standings(self):
		imgs = self._soup.findAll('img',
			src=regexp.compile('/yoweb/images/stat.*'))
		re = regexp.compile(
u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
			)
		standings = { }

		for skill in puzzles:
			standings[skill] = [ ]

		skl = SoupLog()

		for img in imgs:
			try: puzzle = img['alt']
			except KeyError: continue

			if not puzzle in puzzles:
				skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
				continue
			key = img.findParent('td')
			if key is None:
				skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
				continue
			valelem = key.findNextSibling('td')
			if valelem is None:
				skl.soupm(key, 'puzzle missing sibling "%s"'
					% puzzle)
				continue
			valstr = soup_text(valelem)
			match = re.match(valstr)
			if match is None:
				skl.soupm(key, ('puzzle "%s" unparseable'+
					' standing "%s"') % (puzzle, valstr))
				continue
			standing = match.group(match.lastindex)
			standings[puzzle].append(standing)

		self.standings = { }

		for puzzle in puzzles:
			sl = standings[puzzle]
			if len(sl) > 1:
				skl.msg('puzzle "%s" multiple standings %s' %
						(puzzle, `sl`))
				continue
			if not sl:
				skl.msg('puzzle "%s" no standing found' % puzzle)
				continue
			standing = sl[0]
			for i in range(0, len(standingvals)):
				if standing == standingvals[i]:
					self.standings[puzzle] = i
			if not puzzle in self.standings:
				skl.msg('puzzle "%s" unknown standing "%s"' %
					(puzzle, standing))

		all_standings_ok = True
		for puzzle in puzzles:
			if not puzzle in self.standings:
				self.needs_msgs(skl)

	def _find_crewflag(self, cf, yoweb_re):
		things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
		if len(things) != 1:
			self.msg('zero or several %s id references found' % cf)
			return None
		thing = things[0]
		id_re = '\\b%sid\\=(\\w+)$' % cf
		id_haystack = thing['href']
		match = regexp.compile(id_re).search(id_haystack)
		if match is None:
			self.soupm(thing, ('incomprehensible %s id ref'+
				' (%s in %s)') % (cf, id_re, id_haystack))
			return None
		name = soup_text(thing)
		return (match.group(1), name)

	def __str__(self):
		return `(self.crew, self.flag, self.standings, self.msgs)`

#---------- scraper for crew pages ----------

class CrewInfo(SomethingSoupInfo):
	# Public data members:
	#  ci.crew = [ ('Captain',        ['Pirate', ...]),
	#	       ('Senior Officer', [...]),
	#		... ]
	#  pi.msgs = [ 'message describing problem with scrape' ]

	def __init__(self, crewid, max_age=300):
		SomethingSoupInfo.__init__(self,
			'crew/info.wm?crewid=', crewid, max_age)
		self._find_crew()

	def _find_crew(self):
		self.crew = []
		capts = self._soup.findAll('img',
			src='/yoweb/images/crew-captain.png')
		if len(capts) != 1:
			self.msg('crew members: no. of captain images != 1')
			return
		tbl = capts[0]
		while not tbl.find('a', href=pirate_ref_re):
			tbl = tbl.findParent('table')
			if not tbl:
				self.msg('crew members: cannot find table')
				return
		current_rank_crew = None
		crew_rank_re = regexp.compile('/yoweb/images/crew')
		for row in tbl.contents:
			# findAll(recurse=False)
			if isinstance(row,basestring):
				continue

			is_rank = row.find('img', attrs={'src': crew_rank_re})
			if is_rank:
				rank = soup_text(row)
				current_rank_crew = []
				self.crew.append((rank, current_rank_crew))
				continue
			for cell in row.findAll('a', href=pirate_ref_re):
				if current_rank_crew is None:
					self.soupm(cell, 'crew members: crew'
						' before rank')
					continue
				current_rank_crew.append(soup_text(cell))

	def __str__(self):
		return `(self.crew, self.msgs)`

#---------- pretty-printer for tables of pirate puzzle standings ----------

class StandingsTable:
	def __init__(self, use_puzzles=None, col_width=6):
		if use_puzzles is None:
			if opts.ship_duty:
				use_puzzles=[
					'Navigating','Battle Navigation',
					'Gunning',
					['Sailing','Rigging'],
					'Bilging',
					'Carpentry',
					'Treasure Haul'
				]
			else:
				use_puzzles=puzzles
		self._puzzles = use_puzzles
		self.s = ''
		self._cw = col_width-1

	def _pline(self, pirate, puzstrs, extra):
		self.s += ' %-*s' % (max(max_pirate_namelen, 14), pirate)
		for v in puzstrs:
			self.s += ' %-*.*s' % (self._cw,self._cw, v)
		if extra:
			self.s += ' ' + extra
		self.s += '\n'

	def _puzstr(self, pi, puzzle):
		if not isinstance(puzzle,list): puzzle = [puzzle]
		try: standing = max([pi.standings[p] for p in puzzle])
		except KeyError: return '?'
		if not standing: return ''
		s = ''
		if self._cw > 4:
			c1 = standingvals[standing][0]
			if standing < 3: c1 = c1.lower() # 3 = Master
			s += `standing`
		if self._cw > 5:
			s += ' '
		s += '*' * (standing / 2)
		s += '+' * (standing % 2)
		return s

	def headings(self):
		def puzn_redact(name):
			if isinstance(name,list):
				return '/'.join(
					["%.*s" % (self._cw/2, puzn_redact(n))
					 for n in name])
			spc = name.find(' ')
			if spc < 0: return name
			return name[0:min(4,spc)] + name[spc+1:]
		self._pline('', map(puzn_redact, self._puzzles), None)
	def literalline(self, line):
		self.s += line + '\n'
	def pirate_dummy(self, name, standingstring, extra=None):
		self._pline(name, standingstring * len(self._puzzles), extra)
	def pirate(self, pi, extra=None):
		puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
		self._pline(pi.name, puzstrs, extra)

	def results(self):
		return self.s

#---------- chat log parser ----------

class PirateAboard:
	# This is essentially a transparent, dumb, data class.
	#  pa.v
	#  pa.name
	#  pa.last_time
	#  pa.last_event
	#  pa.gunner
	#  pa.last_chat_time
	#  pa.last_chat_chan
	#  pa.pi

	def __init__(pa, pn, v, time, event):
		pa.name = pn
		pa.v = v
		pa.last_time = time
		pa.last_event = event
		pa.last_chat_time = None
		pa.last_chat_chan = None
		pa.gunner = False
		pa.pi = None

	def pirate_info(pa):
		now = time.time()
		if pa.pi:
			age = now - pa.pi_fetched
			guide = random.randint(120,240)
			if age <= guide:
				return pa.pi
			debug('PirateAboard refresh %d > %d  %s' % (
				age, guide, pa.name))
			imaginary = [2,6]
		else:
			imaginary = [1]
		wait = fetcher.need_wait(now, imaginary)
		if wait:
			debug('PirateAboard fetcher not ready %d' % wait)
			return pa.pi
		pa.pi = PirateInfo(pa.name, 600)
		pa.pi_fetched = now
		return pa.pi

class ChatLogTracker:
	# This is quite complex so we make it opaque.  Use the
	# official invokers, accessors etc.

	def __init__(self, myself_pi, logfn):
		self._pl = {}	# self._pl['Pirate'] =
		self._vl = {}	#   self._vl['Vessel']['Pirate'] = PirateAboard
				# self._vl['Vessel']['#lastinfo']
				# self._vl['Vessel']['#name']
				# self._v = self._vl[self._vessel]
		self._date = None
		self._myself = myself_pi
		self._f	= file(logfn)
		self._lbuf = ''
		self._progress = [0, os.fstat(self._f.fileno()).st_size]
		self._disembark_myself()
		self._need_redisplay = False
		self._lastvessel = None

	def _disembark_myself(self):
		self._v = None
		self._vessel = None
		self.force_redisplay()

	def force_redisplay(self):
		self._need_redisplay = True

	def _vessel_updated(self, v, timestamp):
		v['#lastinfo'] = timestamp
		self.force_redisplay()

	def _onboard_event(self,v,timestamp,pirate,event):
		pa = self._pl.get(pirate, None)
		if pa is not None and pa.v is v:
			pa.last_time = timestamp
			pa.last_event = event
		else:
			if pa is not None: del pa.v[pirate]
			pa = PirateAboard(pirate, v, timestamp, event)
			self._pl[pirate] = pa
			v[pirate] = pa
		self._vessel_updated(v, timestamp)
		return pa

	def _trash_vessel(self, v):
		for pn in v:
			if pn.startswith('#'): continue
			del self._pl[pn]
		vn = v['#name']
		del self._vl[vn]
		if v is self._v: self._disembark_myself()
		self.force_redisplay()

	def _vessel_stale(self, v, timestamp):
		return timestamp - v['#lastinfo'] > opts.ship_reboard_clearout

	def _vessel_check_expire(self, v, timestamp):
		if not self._vessel_stale(v, timestamp):
			return v
		self._debug_line_disposition(timestamp,'',
			'stale-reset ' + v['#name'])
		self._trash_vessel(v)
		return None

	def expire_garbage(self, timestamp):
		for v in self._vl.values():
			self._vessel_check_expire(v, timestamp)

	def _vessel_lookup(self, vn, timestamp, dml=[], create=False):
		v = self._vl.get(vn, None)
		if v is not None:
			v = self._vessel_check_expire(v, timestamp)
		if v is not None:
			dml.append('found')
			return v
		if not create:
			dml.append('no')
		dml.append('new')
		self._vl[vn] = v = { '#name': vn }
		self._vessel_updated(v, timestamp)
		return v

	def _find_matching_vessel(self, pattern, timestamp, cmdr,
					dml=[], create=False):
		# use when a commander pirate `cmdr' specified a vessel
		#  by name `pattern' (either may be None)
		# if create is true, will create the vessel
		#  record if an exact name is specified

		if (pattern is not None and
		    not '*' in pattern
		    and len(pattern.split(' ')) == 2):
			vn = pattern.title()
			dml.append('exact')
			return self._vessel_lookup(
				vn, timestamp, dml=dml, create=create)

		if pattern is None:
			pattern_check = lambda vn: True
		else:
			re = '(?:.* )?%s$' % pattern.lower().replace('*','.+')
			pattern_check = regexp.compile(re, regexp.I).match

		tries = []

		cmdr_pa = self._pl.get(cmdr, None)
		if cmdr_pa: tries.append((cmdr_pa.v, 'cmdr'))

		tries.append((self._v, 'here'))
		tried_vns = []

		for (v, dm) in tries:
			if v is None: dml.append(dm+'?'); continue
			
			vn = v['#name']
			if not pattern_check(vn):
				tried_vns.append(vn)
				dml.append(dm+'#')
				continue

			dml.append(dm+'!')
			return v

		if pattern is not None and '*' in pattern:
			search = [
				(vn,v)
				for (vn,v) in self._vl.iteritems()
				if not self._vessel_stale(v, timestamp)
				if pattern_check(vn)
				]
			#debug('CLT-RE /%s/ wanted (%s) searched (%s)' % (
			#	re,
			#	'/'.join(tried_vns),
			#	'/'.join([vn for (vn,v) in search])))

			if len(search)==1:
				dml.append('one')
				return search[0][1]
			elif search:
				dml.append('many')
			else:
				dml.append('none')

	def _debug_line_disposition(self,timestamp,l,m):
		debug('CLT %13s %-40s %s' % (timestamp,m,l))

	def chatline(self,l):
		rm = lambda re: regexp.match(re,l)
		d = lambda m: self._debug_line_disposition(timestamp,l,m)
		timestamp = None

		m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$')
		if m:
			self._date = [int(x) for x in m.groups()]
			self._previous_timestamp = None
			return d('date '+`self._date`)

		if self._date is None:
			return d('date unset')

		m = rm('\\[(\d\d):(\d\d):(\d\d)\\] ')
		if not m:
			return d('no timestamp')

		while True:
			time_tuple = (self._date +
				      [int(x) for x in m.groups()] +
				      [-1,-1,-1])
			timestamp = time.mktime(time_tuple)
			if timestamp >= self._previous_timestamp: break
			self._date[2] += 1
			self._debug_line_disposition(timestamp,'',
				'new date '+`self._date`)

		self._previous_timestamp = timestamp

		l = l[l.find(' ')+1:]

		def ob_x(pirate,event):
			return self._onboard_event(
					self._v, timestamp, pirate, event)
		def ob1(did): ob_x(m.group(1), did); return d(did)
		def oba(did): return ob1('%s %s' % (did, m.group(2)))

		def disembark(v, timestamp, pirate, event):
			self._onboard_event(
					v, timestamp, pirate, 'leaving '+event)
			del v[pirate]
			del self._pl[pirate]

		def disembark_me(why):
			self._disembark_myself()
			return d('disembark-me '+why)

		m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
		if m:
			dm = ['boarding']
			pn = self._myself.name
			vn = m.group(1)
			v = self._vessel_lookup(vn, timestamp, dm, create=True)
			self._lastvessel = self._vessel = vn
			self._v = v
			ob_x(pn, 'we boarded')
			self.expire_garbage(timestamp)
			return d(' '.join(dm))

		if self._v is None:
			return d('no vessel')

		m = rm('(\\w+) has come aboard\\.$')
		if m: return ob1('boarded');

		m = rm('You have ordered (\\w+) to do some (\\S.*\\S)\\.$')
		if m:
			(who,what) = m.groups()
			pa = ob_x(who,'ord '+what)
			if what == 'Gunning':
				pa.gunner = True
			return d('duty order')

		m = rm('(\\w+) abandoned a (\\S.*\\S) station\\.$')
		if m: oba('stopped'); return d("end")

		def chat_core(speaker, chan):
			try: pa = self._pl[speaker]
			except KeyError: return 'mystery'
			if pa.v is not self._v: return 'elsewhere'
			pa.last_chat_time = timestamp
			pa.last_chat_chan = chan
			self.force_redisplay()
			return 'here'

		def chat(chan):
			speaker = m.group(1)
			dm = chat_core(speaker, chan)
			return d('chat %s %s' % (chan, dm))

		def chat_metacmd(chan):
			(cmdr, metacmd) = m.groups()
			metacmd = regexp.sub('\\s+', ' ', metacmd).strip()
			m2 = regexp.match(
				'/([ad]) (?:([A-Za-z* ]+)\\s*:)?([A-Za-z ]+)$',
				metacmd)
			if not m2: return chat(chan)

			(cmd, pattern, targets) = m2.groups()
			dml = ['cmd', chan, cmd]

			if cmd == 'a': each = self._onboard_event
			else: each = disembark

			if cmdr == self._myself.name:
				dml.append('self')
				how = 'cmd: %s' % cmd
			else:
				dml.append('other')
				how = 'cmd: %s %s' % (cmd,cmdr)

			v = self._find_matching_vessel(
				pattern, timestamp, cmdr, dml, create=True)

			if v is not None:
				targets = targets.strip().split(' ')
				dml.append(`len(targets)`)
				for target in targets:
					each(v, timestamp, target.title(), how)
				self._vessel_updated(v, timestamp)

			dm = ' '.join(dml)
			chat_core(cmdr, 'cmd '+chan)
			return d(dm)

		m = rm('(\\w+) (?:issued an order|ordered everyone) "')
		if m: return ob1('general order');

		m = rm('(\\w+) says, "')
		if m: return chat('public')

		m = rm('(\\w+) tells ye, "')
		if m: return chat('private')

		m = rm('Ye told (\\w+), "(.*)"$')
		if m: return chat_metacmd('private')

		m = rm('(\\w+) flag officer chats, "')
		if m: return chat('flag officer')

		m = rm('(\\w+) officer chats, "(.*)"$')
		if m: return chat_metacmd('officer')

		m = rm('Ye accepted the offer to job with ')
		if m: return disembark_me('jobbing')

		m = rm('Ye hop on the ferry and are whisked away ')
		if m: return disembark_me('ferry')

		m = rm('Whisking away to yer home on the magical winds')
		if m: return disembark_me('home')

		m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
		if m:
			pl = m.group(1).split(', ')
			if not self._myself.name in pl:
				return d('lost melee')
			for pn in pl:
				if ' ' in pn: continue
				ob_x(pn,'won melee')
			return d('won melee')

		m = rm('(\\w+) is eliminated\\!')
		if m: return ob1('eliminated in fray');

		m = rm('(\\w+) has driven \w+ from the ship\\!')
		if m: return ob1('boarder repelled');

		m = rm('\w+ has bested (\\w+), and turns'+
			' to the rest of the ship\\.')
		if m: return ob1('boarder unrepelled');

		m = rm('(\\w+) has left the vessel\.')
		if m:
			pirate = m.group(1)
			disembark(self._v, timestamp, pirate, 'disembarked')
			return d('disembarked')

		return d('not-matched')

	def _str_vessel(self, vn, v):
		s = ' vessel %s\n' % vn
		s += ' '*20 + "%-*s   %13s\n" % (
				max_pirate_namelen, '#lastinfo',
				v['#lastinfo'])
		assert v['#name'] == vn
		for pn in sorted(v.keys()):
			if pn.startswith('#'): continue
			pa = v[pn]
			assert pa.v == v
			assert self._pl[pn] == pa
			s += ' '*20 + "%s %-*s %13s %-30s %13s %s\n" % (
				(' ','G')[pa.gunner],
				max_pirate_namelen, pn,
				pa.last_time, pa.last_event,
				pa.last_chat_time, pa.last_chat_chan)
		return s

	def __str__(self):
		s = '''<ChatLogTracker
 myself %s
 vessel %s
'''			% (self._myself.name, self._vessel)
		assert ((self._v is None and self._vessel is None) or
			(self._v is self._vl[self._vessel]))
		if self._vessel is not None:
			s += self._str_vessel(self._vessel, self._v)
		for vn in sorted(self._vl.keys()):
			if vn == self._vessel: continue
			s += self._str_vessel(vn, self._vl[vn])
		for p in self._pl:
			pa = self._pl[p]
			assert pa.v[p] is pa
			assert pa.v in self._vl.values()
		s += '>\n'
		return s

	def catchup(self, progress=None):
		while True:
			more = self._f.readline()
			if not more: break

			self._progress[0] += len(more)
			if progress: progress.progress(*self._progress)

			self._lbuf += more
			if self._lbuf.endswith('\n'):
				self.chatline(self._lbuf.rstrip())
				self._lbuf = ''
				if opts.debug >= 2:
					debug(self.__str__())
		if progress: progress.caughtup()

	def changed(self):
		rv = self._need_redisplay
		self._need_redisplay = False
		return rv
	def myname(self):
		# returns our pirate name
		return self._myself.name
	def vesselname(self):
		# returns the vessel name we're aboard or None
		return self._vessel
	def lastvesselname(self):
		# returns the last vessel name we were aboard or None
		return self._lastvessel
	def aboard(self, vesselname=True):
		# returns a list of PirateAboard the vessel
		#  sorted by pirate name
		#  you can pass this None and you'll get []
		#  or True for the current vessel (which is the default)
		if vesselname is True: v = self._v
		else: v = self._vl.get(vesselname.title())
		if v is None: return []
		return [ v[pn]
			 for pn in sorted(v.keys())
			 if not pn.startswith('#') ]

#---------- implementations of actual operation modes ----------

def do_pirate(pirates, bu):
	print '{'
	for pirate in pirates:
		info = PirateInfo(pirate)
		print '%s: %s,' % (`pirate`, info)
	print '}'

def prep_crew_of(args, bu, max_age=300):
	if len(args) != 1: bu('crew-of takes one pirate name')
	pi = PirateInfo(args[0], max_age)
	if pi.crew is None: return None
	return CrewInfo(pi.crew[0], max_age)

def do_crew_of(args, bu):
	ci = prep_crew_of(args, bu)
	print ci

def do_standings_crew_of(args, bu):
	ci = prep_crew_of(args, bu, 60)
	tab = StandingsTable()
	tab.headings()
	for (rank, members) in ci.crew:
		if not members: continue
		tab.literalline('%s:' % rank)
		for p in members:
			pi = PirateInfo(p, random.randint(900,1800))
			tab.pirate(pi)
	print tab.results()

class ProgressPrintPercentage:
	def __init__(self, f=sys.stdout):
		self._f = f
	def progress_string(self,done,total):
		return "scan chat logs %3d%%\r" % ((done*100) / total)
	def progress(self,*a):
		self._f.write(self.progress_string(*a))
		self._f.flush()
	def show_init(self, pirate, ocean):
		print >>self._f, 'Starting up, %s on the %s ocean' % (
			pirate, ocean)
	def caughtup(self):
		self._f.write('                   \r')
		self._f.flush()

#----- modes which use the chat log parser are quite complex -----

def prep_chat_log(args, bu,
		progress=ProgressPrintPercentage(),
		max_myself_age=3600):
	if len(args) != 1: bu('this action takes only chat log filename')
	logfn = args[0]
	logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_'
	match = regexp.match(logfn_re, logfn)
	if not match: bu('chat log filename is not in expected format')
	(pirate, ocean) = match.groups()
	fetcher.default_ocean(ocean)

	progress.show_init(pirate, fetcher.ocean)
	myself = PirateInfo(pirate,max_myself_age)
	track = ChatLogTracker(myself, logfn)

	opts.debug -= 2
	track.catchup(progress)
	opts.debug += 2

	track.force_redisplay()

	return (myself, track)

def do_track_chat_log(args, bu):
	(myself, track) = prep_chat_log(args, bu)
	while True:
		track.catchup()
		if track.changed():
			print track
		sleep(1)

#----- ship management aid -----

class Display_dumb(ProgressPrintPercentage):
	def __init__(self):
		ProgressPrintPercentage.__init__(self)
	def show(self, s):
		print '\n\n', s;
	def realstart(self):
		pass

class Display_overwrite(ProgressPrintPercentage):
	def __init__(self):
		ProgressPrintPercentage.__init__(self)

		null = file('/dev/null','w')
		curses.setupterm(fd=null.fileno())

		self._clear = curses.tigetstr('clear')
		if not self._clear:
			self._debug('missing clear!')
			self.show = Display_dumb.show
			return

		self._t = {'el':'', 'ed':''}
		if not self._init_sophisticated():
			for k in self._t.keys(): self._t[k] = ''
			self._t['ho'] = self._clear

	def _debug(self,m): debug('display overwrite: '+m)

	def _init_sophisticated(self):
		for k in self._t.keys():
			s = curses.tigetstr(k)
			self._t[k] = s
		self._t['ho'] = curses.tigetstr('ho')
		if not self._t['ho']:
			cup = curses.tigetstr('cup')
			self._t['ho'] = curses.tparm(cup,0,0)
		missing = [k for k in self._t.keys() if not self._t[k]]
		if missing:
			self.debug('missing '+(' '.join(missing)))
			return 0
		return 1

	def show(self, s):
		w = sys.stdout.write
		def wti(k): w(self._t[k])

		wti('ho')
		nl = ''
		for l in s.rstrip().split('\n'):
			w(nl)
			w(l)
			wti('el')
			nl = '\r\n'
		wti('ed')
		w(' ')
		sys.stdout.flush()

	def realstart(self):
		sys.stdout.write(self._clear)
		sys.stdout.flush()
			

def do_ship_aid(args, bu):
	if opts.ship_duty is None: opts.ship_duty = True

	displayer = globals()['Display_'+opts.display]()

	(myself, track) = prep_chat_log(args, bu, progress=displayer)

	displayer.realstart()

	if os.isatty(0): kr_create = KeystrokeReader
	else: kr_create = DummyKeystrokeReader

	try:
		kreader = kr_create(0, 10)
		ship_aid_core(myself, track, displayer, kreader)
	finally:
		kreader.stop()
		print '\n'

def ship_aid_core(myself, track, displayer, kreader):

	def find_vessel():
		vn = track.vesselname()
		if vn: return (vn, " on board the %s" % vn)
		vn = track.lastvesselname()
		if vn: return (vn, " ashore from the %s" % vn)
		return (None, " not on a vessel")

	def timeevent(t,e):
		if t is None: return ' ' * 22
		return " %-4s %-16s" % (format_time_interval(now - t),e)

	displayer.show(track.myname() + find_vessel()[1] + '...')

	rotate_nya = '/-\\'

	while True:
		track.catchup()
		now = time.time()

		(vn, s) = find_vessel()
		s = track.myname() + s
		s += " at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
		s += kreader.info()
		s += '\n'

		tbl = StandingsTable()
		tbl.headings()

		aboard = track.aboard(vn)

		for pa in aboard:
			pi = pa.pirate_info()

			xs = ''
			if pa.gunner: xs += 'G '
			else: xs += '  '
			xs += timeevent(pa.last_time, pa.last_event)
			xs += timeevent(pa.last_chat_time, pa.last_chat_chan)

			if pi is None:
				tbl.pirate_dummy(pa.name, rotate_nya[0], xs)
			else:
				tbl.pirate(pi, xs)

		s += tbl.results()
		displayer.show(s)

		k = kreader.getch()
		if k is None:
			rotate_nya = rotate_nya[1:3] + rotate_nya[0]
			continue

		if k == 'q':
			break

#---------- individual keystroke input ----------

class DummyKeystrokeReader:
	def __init__(self,fd,timeout_dummy): pass
	def stop(self): pass
	def getch(self): sleep(1); return None
	def info(self): return ' [noninteractive]'

class KeystrokeReader(DummyKeystrokeReader):
	def __init__(self, fd, timeout_decisec=0):
		self._fd = fd
		self._saved = termios.tcgetattr(fd)
		a = termios.tcgetattr(fd)
		a[3] &= ~(termios.ECHO | termios.ECHONL |
			  termios.ICANON | termios.IEXTEN)
		a[6][termios.VMIN] = 0
		a[6][termios.VTIME] = timeout_decisec
		termios.tcsetattr(fd, termios.TCSANOW, a)
	def stop(self):
		termios.tcsetattr(self._fd, termios.TCSANOW, self._saved)
	def getch(self):
		debug_flush()
		byte = os.read(self._fd, 1)
		if not len(byte): return None
		return byte
	def info(self):
		return ''

#---------- main program ----------

def main():
	global opts, fetcher

	pa = OptionParser(
'''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
actions:
 yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
 yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
 yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
 yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG
 yoweb-scrape [options] ship-aid CHAT-LOG  (must be .../PIRATE_OCEAN_chat-log*)

display modes (for --display) apply to ship-aid:
 --display=dumb       just print new information, scrolling the screen
 --display=overwrite  use cursor motion, selective clear, etc. to redraw at top
''')
	ao = pa.add_option
	ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
		help='select ocean OCEAN')
	ao('--cache-dir', dest='cache_dir', metavar='DIR',
		default='~/.yoweb-scrape-cache',
		help='cache yoweb pages in DIR')
	ao('-D','--debug', action='count', dest='debug', default=0,
		help='enable debugging output')
	ao('--debug-fd', type='int', dest='debug_fd',
		help='write any debugging output to specified fd')
	ao('-q','--quiet', action='store_true', dest='quiet',
		help='suppress warning output')
	ao('--display', action='store', dest='display',
		type='choice', choices=['dumb','overwrite'],
		help='how to display ship aid')

	ao('--ship-duty', action='store_true', dest='ship_duty',
		help='show ship duty station puzzles')
	ao('--all-puzzles', action='store_false', dest='ship_duty',
		help='show all puzzles, not just ship duty stations')

	(opts,args) = pa.parse_args()
	random.seed()

	if len(args) < 1:
		pa.error('need a mode argument')

	if opts.debug_fd is not None:
		opts.debug_file = os.fdopen(opts.debug_fd, 'w')
	else:
		opts.debug_file = sys.stdout

	mode = args[0]
	mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
	try: mode_fn = globals()[mode_fn_name]
	except KeyError: pa.error('unknown mode "%s"' % mode)

	# fixed parameters
	opts.min_max_age = 60
	opts.expire_age = 3600
	opts.ship_reboard_clearout = 3600

	if opts.cache_dir.startswith('~/'):
		opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]

	if opts.display is None:
		if ((opts.debug > 0 and opts.debug_fd is None)
		    or not os.isatty(sys.stdout.fileno())):
			opts.display = 'dumb'
		else:
			opts.display = 'overwrite'

	fetcher = Fetcher(opts.ocean, opts.cache_dir)

	mode_fn(args[1:], pa.error)

main()