chiark / gitweb /
cc98ea3b21b869154ed0a46ca5aec941d7f71a71
[ypp-sc-tools.main.git] / yoweb-scrape
1 #!/usr/bin/python
2
3 #---------- setup ----------
4
5 import signal
6 signal.signal(signal.SIGINT, signal.SIG_DFL)
7
8 import os
9 import time
10 import urllib
11 import urllib2
12 import errno
13 import sys
14 import re as regexp
15 import random
16 import curses
17 from optparse import OptionParser
18
19 from BeautifulSoup import BeautifulSoup
20
21 opts = None
22
23 #---------- YPP parameters and arrays ----------
24
25 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
26         '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
27         '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
28         '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
29
30 standingvals = ('Able/Distinguished/Respected/Master'+
31                 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
32
33 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
34
35 max_pirate_namelen = 12
36
37
38 #---------- general utilities ----------
39
40 def debug(m):
41         if opts.debug > 0:
42                 print m
43
44 def format_time_interval(ti):
45         if ti < 120: return '%d:%02d' % (ti / 60, ti % 60)
46         if ti < 7200: return '%2dm' % (ti / 60)
47         if ti < 86400: return '%dh' % (ti / 3600)
48         return '%dd' % (ti / 86400)
49
50 #---------- caching and rate-limiting data fetcher ----------
51
52 class Fetcher:
53         def __init__(self, ocean, cachedir):
54                 debug('Fetcher init %s' % cachedir)
55                 self.ocean = ocean
56                 self.cachedir = cachedir
57                 try: os.mkdir(cachedir)
58                 except (OSError,IOError), oe:
59                         if oe.errno != errno.EEXIST: raise
60                 self._cache_scan(time.time())
61
62         def default_ocean(self, ocean='ice'):
63                 if self.ocean is None:
64                         self.ocean = ocean
65
66         def _cache_scan(self, now):
67                 # returns list of ages, unsorted
68                 ages = []
69                 debug('Fetcher   scan_cache')
70                 for leaf in os.listdir(self.cachedir):
71                         if not leaf.startswith('#'): continue
72                         path = self.cachedir + '/' + leaf
73                         try: s = os.stat(path)
74                         except (OSError,IOError), oe:
75                                 if oe.errno != errno.ENOENT: raise
76                                 continue
77                         age = now - s.st_mtime
78                         if age > opts.expire_age:
79                                 debug('Fetcher    expire %d %s' % (age, path))
80                                 try: os.remove(path)
81                                 except (OSError,IOError), oe:
82                                         if oe.errno != errno.ENOENT: raise
83                                 continue
84                         ages.append(age)
85                 return ages
86
87         def need_wait(self, now):
88                 ages = self._cache_scan(now)
89                 ages.sort()
90                 debug('Fetcher   ages ' + `ages`)
91                 min_age = 1
92                 need_wait = 0
93                 for age in ages:
94                         if age < min_age and age < 300:
95                                 debug('Fetcher   morewait min=%d age=%d' %
96                                         (min_age, age))
97                                 need_wait = max(need_wait, min_age - age)
98                         min_age += 3
99                         min_age *= 1.25
100                 return need_wait
101
102         def _rate_limit_cache_clean(self, now):
103                 need_wait = self.need_wait(now)
104                 if need_wait > 0:
105                         debug('Fetcher   wait %d' % need_wait)
106                         time.sleep(need_wait)
107
108         def fetch(self, url, max_age):
109                 debug('Fetcher fetch %s' % url)
110                 cache_corename = urllib.quote_plus(url)
111                 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
112                 try: f = file(cache_item, 'r')
113                 except (OSError,IOError), oe:
114                         if oe.errno != errno.ENOENT: raise
115                         f = None
116                 now = time.time()
117                 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
118                 if f is not None:
119                         s = os.fstat(f.fileno())
120                         age = now - s.st_mtime
121                         if age > max_age:
122                                 debug('Fetcher  stale %d < %d'% (max_age, age))
123                                 f = None
124                 if f is not None:
125                         data = f.read()
126                         f.close()
127                         debug('Fetcher  cached %d > %d' % (max_age, age))
128                         return data
129
130                 debug('Fetcher  fetch')
131                 self._rate_limit_cache_clean(now)
132
133                 stream = urllib2.urlopen(url)
134                 data = stream.read()
135                 cache_tmp = "%s/#%s~%d#" % (
136                         self.cachedir, cache_corename, os.getpid())
137                 f = file(cache_tmp, 'w')
138                 f.write(data)
139                 f.close()
140                 os.rename(cache_tmp, cache_item)
141                 debug('Fetcher  stored')
142                 return data
143
144         def yoweb(self, kind, tail, max_age):
145                 self.default_ocean()
146                 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
147                         self.ocean, kind, tail)
148                 return self.fetch(url, max_age)
149
150 #---------- logging assistance for troubled screenscrapers ----------
151
152 class SoupLog:
153         def __init__(self):
154                 self.msgs = [ ]
155         def msg(self, m):
156                 self.msgs.append(m)
157         def soupm(self, obj, m):
158                 self.msg(m + '; in ' + `obj`)
159         def needs_msgs(self, child_souplog):
160                 self.msgs += child_souplog.msgs
161                 child_souplog.msgs = [ ]
162
163 def soup_text(obj):
164         str = ''.join(obj.findAll(text=True))
165         return str.strip()
166
167 class SomethingSoupInfo(SoupLog):
168         def __init__(self, kind, tail, max_age):
169                 SoupLog.__init__(self)
170                 html = fetcher.yoweb(kind, tail, max_age)
171                 self._soup = BeautifulSoup(html,
172                         convertEntities=BeautifulSoup.HTML_ENTITIES
173                         )
174
175 #---------- scraper for pirate pages ----------
176
177 class PirateInfo(SomethingSoupInfo):
178         # Public data members:
179         #  pi.standings = { 'Treasure Haul': 'Able' ... }
180         #  pi.name = name
181         #  pi.crew = (id, name)
182         #  pi.flag = (id, name)
183         #  pi.msgs = [ 'message describing problem with scrape' ]
184                 
185         def __init__(self, pirate, max_age=300):
186                 SomethingSoupInfo.__init__(self,
187                         'pirate.wm?target=', pirate, max_age)
188                 self.name = pirate
189                 self._find_standings()
190                 self.crew = self._find_crewflag('crew',
191                         '^/yoweb/crew/info\\.wm')
192                 self.flag = self._find_crewflag('flag',
193                         '^/yoweb/flag/info\\.wm')
194
195         def _find_standings(self):
196                 imgs = self._soup.findAll('img',
197                         src=regexp.compile('/yoweb/images/stat.*'))
198                 re = regexp.compile(
199 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
200                         )
201                 standings = { }
202
203                 for skill in puzzles:
204                         standings[skill] = [ ]
205
206                 skl = SoupLog()
207
208                 for img in imgs:
209                         try: puzzle = img['alt']
210                         except KeyError: continue
211
212                         if not puzzle in puzzles:
213                                 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
214                                 continue
215                         key = img.findParent('td')
216                         if key is None:
217                                 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
218                                 continue
219                         valelem = key.findNextSibling('td')
220                         if valelem is None:
221                                 skl.soupm(key, 'puzzle missing sibling "%s"'
222                                         % puzzle)
223                                 continue
224                         valstr = soup_text(valelem)
225                         match = re.match(valstr)
226                         if match is None:
227                                 skl.soupm(key, ('puzzle "%s" unparseable'+
228                                         ' standing "%s"') % (puzzle, valstr))
229                                 continue
230                         standing = match.group(match.lastindex)
231                         standings[puzzle].append(standing)
232
233                 self.standings = { }
234
235                 for puzzle in puzzles:
236                         sl = standings[puzzle]
237                         if len(sl) > 1:
238                                 skl.msg('puzzle "%s" multiple standings %s' %
239                                                 (puzzle, `sl`))
240                                 continue
241                         if not sl:
242                                 skl.msg('puzzle "%s" no standing found' % puzzle)
243                                 continue
244                         standing = sl[0]
245                         for i in range(0, len(standingvals)-1):
246                                 if standing == standingvals[i]:
247                                         self.standings[puzzle] = i
248                         if not puzzle in self.standings:
249                                 skl.msg('puzzle "%s" unknown standing "%s"' %
250                                         (puzzle, standing))
251
252                 all_standings_ok = True
253                 for puzzle in puzzles:
254                         if not puzzle in self.standings:
255                                 self.needs_msgs(skl)
256
257         def _find_crewflag(self, cf, yoweb_re):
258                 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
259                 if len(things) != 1:
260                         self.msg('zero or several %s id references found' % cf)
261                         return None
262                 thing = things[0]
263                 id_re = '\\b%sid\\=(\\w+)$' % cf
264                 id_haystack = thing['href']
265                 match = regexp.compile(id_re).search(id_haystack)
266                 if match is None:
267                         self.soupm(thing, ('incomprehensible %s id ref'+
268                                 ' (%s in %s)') % (cf, id_re, id_haystack))
269                         return None
270                 name = soup_text(thing)
271                 return (match.group(1), name)
272
273         def __str__(self):
274                 return `(self.crew, self.flag, self.standings, self.msgs)`
275
276 #---------- scraper for crew pages ----------
277
278 class CrewInfo(SomethingSoupInfo):
279         # Public data members:
280         #  ci.crew = [ ('Captain',        ['Pirate', ...]),
281         #              ('Senior Officer', [...]),
282         #               ... ]
283         #  pi.msgs = [ 'message describing problem with scrape' ]
284
285         def __init__(self, crewid, max_age=300):
286                 SomethingSoupInfo.__init__(self,
287                         'crew/info.wm?crewid=', crewid, max_age)
288                 self._find_crew()
289
290         def _find_crew(self):
291                 self.crew = []
292                 capts = self._soup.findAll('img',
293                         src='/yoweb/images/crew-captain.png')
294                 if len(capts) != 1:
295                         self.msg('crew members: no. of captain images != 1')
296                         return
297                 tbl = capts[0]
298                 while not tbl.find('a', href=pirate_ref_re):
299                         tbl = tbl.findParent('table')
300                         if not tbl:
301                                 self.msg('crew members: cannot find table')
302                                 return
303                 current_rank_crew = None
304                 crew_rank_re = regexp.compile('/yoweb/images/crew')
305                 for row in tbl.contents:
306                         # findAll(recurse=False)
307                         if isinstance(row,basestring):
308                                 continue
309
310                         is_rank = row.find('img', attrs={'src': crew_rank_re})
311                         if is_rank:
312                                 rank = soup_text(row)
313                                 current_rank_crew = []
314                                 self.crew.append((rank, current_rank_crew))
315                                 continue
316                         for cell in row.findAll('a', href=pirate_ref_re):
317                                 if current_rank_crew is None:
318                                         self.soupm(cell, 'crew members: crew'
319                                                 ' before rank')
320                                         continue
321                                 current_rank_crew.append(soup_text(cell))
322
323         def __str__(self):
324                 return `(self.crew, self.msgs)`
325
326 #---------- pretty-printer for tables of pirate puzzle standings ----------
327
328 class StandingsTable:
329         def __init__(self, use_puzzles=None, col_width=6):
330                 if use_puzzles is None:
331                         if opts.ship_duty:
332                                 use_puzzles=[
333                                         'Navigating','Battle Navigation',
334                                         'Gunning',
335                                         ['Sailing','Rigging'],
336                                         'Bilging',
337                                         'Carpentry',
338                                         'Treasure Haul'
339                                 ]
340                         else:
341                                 use_puzzles=puzzles
342                 self._puzzles = use_puzzles
343                 self.s = ''
344                 self._cw = col_width-1
345
346         def _pline(self, pirate, puzstrs, extra):
347                 self.s += ' %-*s' % (max(max_pirate_namelen, 14), pirate)
348                 for v in puzstrs:
349                         self.s += ' %-*.*s' % (self._cw,self._cw, v)
350                 if extra:
351                         self.s += ' ' + extra
352                 self.s += '\n'
353
354         def _puzstr(self, pi, puzzle):
355                 if not isinstance(puzzle,list): puzzle = [puzzle]
356                 try: standing = max([pi.standings[p] for p in puzzle])
357                 except KeyError: return '?'
358                 if not standing: return ''
359                 s = ''
360                 if self._cw > 4:
361                         c1 = standingvals[standing][0]
362                         if standing < 3: c1 = c1.lower() # 3 = Master
363                         s += `standing`
364                 if self._cw > 5:
365                         s += ' '
366                 s += '*' * (standing / 2)
367                 s += '+' * (standing % 2)
368                 return s
369
370         def headings(self):
371                 def puzn_redact(name):
372                         if isinstance(name,list):
373                                 return '/'.join(
374                                         ["%.*s" % (self._cw/2, puzn_redact(n))
375                                          for n in name])
376                         spc = name.find(' ')
377                         if spc < 0: return name
378                         return name[0:min(4,spc)] + name[spc+1:]
379                 self._pline('', map(puzn_redact, self._puzzles), None)
380         def literalline(self, line):
381                 self.s += line + '\n'
382         def pirate_dummy(self, name, standingstring, extra=None):
383                 self._pline(name, standingstring * len(self._puzzles), extra)
384         def pirate(self, pi, extra=None):
385                 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
386                 self._pline(pi.name, puzstrs, extra)
387
388         def results(self):
389                 return self.s
390
391 #---------- chat log parser ----------
392
393 class PirateAboard:
394         # This is essentially a transparent, dumb, data class.
395         #  pa.v
396         #  pa.name
397         #  pa.last_time
398         #  pa.last_event
399         #  pa.gunner
400         #  pa.last_chat_time
401         #  pa.last_chat_chan
402         #  pa.pi
403
404         def __init__(pa, pn, v, time, event):
405                 pa.name = pn
406                 pa.v = v
407                 pa.last_time = time
408                 pa.last_event = event
409                 pa.last_chat_time = None
410                 pa.last_chat_chan = None
411                 pa.gunner = False
412                 pa.pi = None
413
414         def pirate_info(pa):
415                 if not pa.pi and not fetcher.need_wait(time.time()):
416                         pa.pi = PirateInfo(pa.name, 3600)
417                 return pa.pi
418
419 class ChatLogTracker:
420         # This is quite complex so we make it opaque.  Use the
421         # official invokers, accessors etc.
422
423         def __init__(self, myself_pi, logfn):
424                 self._pl = {}   # self._pl['Pirate'] =
425                 self._vl = {}   #   self._vl['Vessel']['Pirate'] = PirateAboard
426                                 # self._vl['Vessel']['#lastaboard']
427                 self._v = None          # self._v =
428                 self._vessel = None     #       self._vl[self._vessel]
429                 self._date = None
430                 self._myself = myself_pi
431                 self._need_redisplay = False
432                 self._f = file(logfn)
433                 self._lbuf = ''
434                 self._progress = [0, os.fstat(self._f.fileno()).st_size]
435
436         def force_redisplay(self):
437                 self._need_redisplay = True
438
439         def _onboard_event(self,timestamp,pirate,event):
440                 try: pa = self._pl[pirate]
441                 except KeyError: pa = None
442                 if pa is not None and pa.v is self._v:
443                         pa.last_time = timestamp
444                         pa.last_event = event
445                 else:
446                         if pa is not None: del pa.v[pirate]
447                         pa = PirateAboard(pirate, self._v, timestamp, event)
448                         self._pl[pirate] = pa
449                         self._v[pirate] = pa
450                 self._v['#lastaboard'] = timestamp
451                 self.force_redisplay()
452                 return pa
453
454         def _trash_vessel(self, v):
455                 for pn in v:
456                         if pn.startswith('#'): continue
457                         del self._pl[pn]
458                 self.force_redisplay()
459
460         def expire_garbage(self, timestamp):
461                 for (vn,v) in list(self._vl.iteritems()):
462                         la = v['#lastaboard']
463                         if timestamp - la > opts.ship_reboard_clearout:
464                                 self._debug_line_disposition(timestamp,'',
465                                         'stale reset '+vn)
466                                 self._trash_vessel(v)
467                                 del self._vl[vn]
468
469         def clear_vessel(self, timestamp):
470                 if self._v is not None:
471                         self._trash_vessel(self._v)
472                 self._v = {'#lastaboard': timestamp}
473                 self._vl[self._vessel] = self._v
474
475         def _debug_line_disposition(self,timestamp,l,m):
476                 debug('CLT %13s %-30s %s' % (timestamp,m,l))
477
478         def chatline(self,l):
479                 rm = lambda re: regexp.match(re,l)
480                 d = lambda m: self._debug_line_disposition(timestamp,l,m)
481                 timestamp = None
482
483                 m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$')
484                 if m:
485                         self._date = [int(x) for x in m.groups()]
486                         self._previous_timestamp = None
487                         return d('date '+`self._date`)
488
489                 if self._date is None:
490                         return d('date unset')
491
492                 m = rm('\\[(\d\d):(\d\d):(\d\d)\\] ')
493                 if not m:
494                         return d('no timestamp')
495
496                 while True:
497                         time_tuple = (self._date +
498                                       [int(x) for x in m.groups()] +
499                                       [-1,-1,-1])
500                         timestamp = time.mktime(time_tuple)
501                         if timestamp >= self._previous_timestamp: break
502                         self._date[2] += 1
503                         self._debug_line_disposition(timestamp,'',
504                                 'new date '+`self._date`)
505
506                 self._previous_timestamp = timestamp
507
508                 l = l[l.find(' ')+1:]
509
510                 def ob_x(who,event):
511                         return self._onboard_event(timestamp, who, event)
512                 def ob1(did): ob_x(m.group(1), did); return d(did)
513                 def oba(did): return ob1('%s %s' % (did, m.group(2)))
514
515                 m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
516                 if m:
517                         pn = self._myself.name
518                         self._vessel = m.group(1)
519                         dm = 'boarding'
520
521                         try:             self._v = self._vl[self._vessel]
522                         except KeyError: self._v = None; dm += ' new'
523                         
524                         if self._v is not None:  la = self._v['#lastaboard']
525                         else:                    la = 0; dm += ' ?la'
526
527                         if timestamp - la > opts.ship_reboard_clearout:
528                                 self.clear_vessel(timestamp)
529                                 dm += ' stale'
530
531                         ob_x(pn, 'we boarded')
532                         self.expire_garbage(timestamp)
533                         return d(dm)
534
535                 if self._v is None:
536                         return d('no vessel')
537
538                 m = rm('(\\w+) has come aboard\\.$')
539                 if m: return ob1('boarded');
540
541                 m = rm('You have ordered (\\w+) to do some (\\S.*\\S)\\.$')
542                 if m:
543                         (who,what) = m.groups()
544                         pa = ob_x(who,'ord '+what)
545                         if what == 'Gunning':
546                                 pa.gunner = True
547                         return d('duty order')
548
549                 m = rm('(\\w+) abandoned a (\\S.*\\S) station\\.$')
550                 if m: oba('stopped'); return d("end")
551
552                 def chat(what):
553                         who = m.group(1)
554                         try: pa = self._pl[who]
555                         except KeyError: return d('chat mystery')
556                         if pa.v is self._v:
557                                 pa.last_chat_time = timestamp
558                                 pa.last_chat_chan = what
559                                 self.force_redisplay()
560                                 return d(what+' chat')
561
562                 m = rm('(\\w+) (?:issued an order|ordered everyone) "')
563                 if m: return ob1('general order');
564
565                 m = rm('(\\w+) says, "')
566                 if m: return chat('public')
567
568                 m = rm('(\\w+) tells ye, "')
569                 if m: return chat('private')
570
571                 m = rm('(\\w+) flag officer chats, "')
572                 if m: return chat('flag officer')
573
574                 m = rm('(\\w+) officer chats, "')
575                 if m: return chat('officer')
576
577                 m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
578                 if m:
579                         pl = m.group(1).split(', ')
580                         if not self._myself.name in pl:
581                                 return d('lost boarding battle')
582                         for pn in pl:
583                                 if ' ' in pn: continue
584                                 ob_x(pn,'won boarding battle')
585                         return d('won boarding battle')
586
587                 m = rm('(\\w+) is eliminated\\!')
588                 if m: return ob1('eliminated in fray');
589
590                 m = rm('(\\w+) has left the vessel\.')
591                 if m:
592                         who = m.group(1)
593                         ob_x(who, 'disembarked')
594                         del self._v[who]
595                         del self._pl[who]
596                         return d('disembarked')
597
598                 return d('not matched')
599
600         def _str_vessel(self, vn, v):
601                 s = ' vessel %s\n' % vn
602                 s += ' '*20 + "%-*s   %13s\n" % (
603                                 max_pirate_namelen, '#lastaboard',
604                                 v['#lastaboard'])
605                 for pn in sorted(v.keys()):
606                         if pn.startswith('#'): continue
607                         pa = v[pn]
608                         assert pa.v == v
609                         assert self._pl[pn] == pa
610                         s += ' '*20 + "%s %-*s %13s %-30s %13s %s\n" % (
611                                 (' ','G')[pa.gunner],
612                                 max_pirate_namelen, pn,
613                                 pa.last_time, pa.last_event,
614                                 pa.last_chat_time, pa.last_chat_chan)
615                 return s
616
617         def __str__(self):
618                 s = '''<ChatLogTracker
619  myself %s
620  vessel %s
621 '''                     % (self._myself.name, self._vessel)
622                 assert ((self._v is None and self._vessel is None) or
623                         (self._v is self._vl[self._vessel]))
624                 if self._vessel is not None:
625                         s += self._str_vessel(self._vessel, self._v)
626                 for vn in sorted(self._vl.keys()):
627                         if vn == self._vessel: continue
628                         s += self._str_vessel(vn, self._vl[vn])
629                 for p in self._pl:
630                         pa = self._pl[p]
631                         assert pa.v[p] is pa
632                         assert pa.v in self._vl.values()
633                 s += '>\n'
634                 return s
635
636         def catchup(self, progress=None):
637                 while True:
638                         more = self._f.readline()
639                         if not more: break
640
641                         self._progress[0] += len(more)
642                         if progress: progress.progress(*self._progress)
643
644                         self._lbuf += more
645                         if self._lbuf.endswith('\n'):
646                                 self.chatline(self._lbuf.rstrip())
647                                 self._lbuf = ''
648                 if progress: progress.caughtup()
649
650         def changed(self):
651                 rv = self._need_redisplay
652                 self._need_redisplay = False
653                 return rv
654         def myname(self):
655                 # returns our pirate name
656                 return self._myself.name
657         def vessel(self):
658                 # returns the vessel we're aboard or None
659                 return self._vessel
660         def aboard(self):
661                 # returns a list of PirateAboard sorted by name
662                 if self._v is None: return []
663                 return [ self._v[pn]
664                          for pn in sorted(self._v.keys())
665                          if not pn.startswith('#') ]
666
667 #---------- implementations of actual operation modes ----------
668
669 def do_pirate(pirates, bu):
670         print '{'
671         for pirate in pirates:
672                 info = PirateInfo(pirate)
673                 print '%s: %s,' % (`pirate`, info)
674         print '}'
675
676 def prep_crew_of(args, bu, max_age=300):
677         if len(args) != 1: bu('crew-of takes one pirate name')
678         pi = PirateInfo(args[0], max_age)
679         if pi.crew is None: return None
680         return CrewInfo(pi.crew[0], max_age)
681
682 def do_crew_of(args, bu):
683         ci = prep_crew_of(args, bu)
684         print ci
685
686 def do_standings_crew_of(args, bu):
687         ci = prep_crew_of(args, bu, 60)
688         tab = StandingsTable()
689         tab.headings()
690         for (rank, members) in ci.crew:
691                 if not members: continue
692                 tab.literalline('%s:' % rank)
693                 for p in members:
694                         pi = PirateInfo(p, random.randint(900,1800))
695                         tab.pirate(pi)
696         print tab.results()
697
698 class ProgressPrintPercentage:
699         def __init__(self, f=sys.stdout):
700                 self._f = f
701         def progress_string(self,done,total):
702                 return "scan chat logs %3d%%\r" % ((done*100) / total)
703         def progress(self,*a):
704                 self._f.write(self.progress_string(*a))
705                 self._f.flush()
706         def show_init(self, pirate, ocean):
707                 print >>self._f, 'Starting up, %s on the %s ocean' % (
708                         pirate, ocean)
709         def caughtup(self):
710                 self._f.write('                   \r')
711                 self._f.flush()
712
713 #----- modes which use the chat log parser are quite complex -----
714
715 def prep_chat_log(args, bu,
716                 progress=ProgressPrintPercentage(),
717                 max_myself_age=3600):
718         if len(args) != 1: bu('this action takes only chat log filename')
719         logfn = args[0]
720         logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_'
721         match = regexp.match(logfn_re, logfn)
722         if not match: bu('chat log filename is not in expected format')
723         (pirate, ocean) = match.groups()
724         fetcher.default_ocean(ocean)
725         
726         myself = PirateInfo(pirate,max_myself_age)
727         progress.show_init(pirate, fetcher.ocean)
728         track = ChatLogTracker(myself, logfn)
729
730         opts.debug -= 1
731         track.catchup(progress)
732         opts.debug += 1
733
734         track.force_redisplay()
735
736         return (myself, track)
737
738 def do_track_chat_log(args, bu):
739         (myself, track) = prep_chat_log(args, bu)
740         while True:
741                 track.catchup()
742                 if track.changed():
743                         print track
744                 time.sleep(1)
745
746 #----- ship management aid -----
747
748 class Display_dumb(ProgressPrintPercentage):
749         def __init__(self):
750                 ProgressPrintPercentage.__init__(self)
751         def show(self, s):
752                 print '\n\n', s;
753         def realstart(self):
754                 pass
755
756 class Display_overwrite(ProgressPrintPercentage):
757         def __init__(self):
758                 ProgressPrintPercentage.__init__(self)
759
760                 null = file('/dev/null','w')
761                 curses.setupterm(fd=null.fileno())
762
763                 self._clear = curses.tigetstr('clear')
764                 if not self._clear:
765                         self._debug('missing clear!')
766                         self.show = Display_dumb.show
767                         return
768
769                 self._t = {'el':'', 'ed':''}
770                 if not self._init_sophisticated():
771                         for k in self._t.keys(): self._t[k] = ''
772                         self._t['ho'] = self._clear
773
774         def _debug(self,m): debug('display overwrite: '+m)
775
776         def _init_sophisticated(self):
777                 for k in self._t.keys():
778                         s = curses.tigetstr(k)
779                         self._t[k] = s
780                 self._t['ho'] = curses.tigetstr('ho')
781                 if not self._t['ho']:
782                         cup = curses.tigetstr('cup')
783                         self._t['ho'] = curses.tparm(cup,0,0)
784                 missing = [k for k in self._t.keys() if not self._t[k]]
785                 if missing:
786                         self.debug('missing '+(' '.join(missing)))
787                         return 0
788                 return 1
789
790         def show(self, s):
791                 w = sys.stdout.write
792                 def wti(k): w(self._t[k])
793
794                 wti('ho')
795                 nl = ''
796                 for l in s.rstrip().split('\n'):
797                         w(nl)
798                         w(l)
799                         wti('el')
800                         nl = '\r\n'
801                 wti('ed')
802                 w(' ')
803                 sys.stdout.flush()
804
805         def realstart(self):
806                 sys.stdout.write(self._clear)
807                 sys.stdout.flush()
808                         
809
810 def do_ship_aid(args, bu):
811         if opts.ship_duty is None: opts.ship_duty = True
812
813         displayer = globals()['Display_'+opts.display]()
814         rotate_nya = '/-\\'
815
816         (myself, track) = prep_chat_log(args, bu, progress=displayer)
817
818         def timeevent(t,e):
819                 if t is None: return ' ' * 22
820                 return " %-4s %-16s" % (format_time_interval(now - t),e)
821
822         displayer.realstart()
823
824         while True:
825                 track.catchup()
826                 now = time.time()
827
828                 s = "%s" % track.myname()
829
830                 vn = track.vessel()
831                 if vn is None: s += " not on a vessel?!"
832                 else: s += " on board the %s" % vn
833                 s += " at %s\n" % time.strftime("%Y-%m-%d %H:%M:%S")
834
835                 tbl = StandingsTable()
836                 tbl.headings()
837
838                 for pa in track.aboard():
839                         pi = pa.pirate_info()
840
841                         xs = ''
842                         if pa.gunner: xs += 'G '
843                         else: xs += '  '
844                         xs += timeevent(pa.last_time, pa.last_event)
845                         xs += timeevent(pa.last_chat_time, pa.last_chat_chan)
846
847                         if pi is None:
848                                 tbl.pirate_dummy(pa.name, rotate_nya[0], xs)
849                         else:
850                                 tbl.pirate(pi, xs)
851
852                 s += tbl.results()
853
854                 displayer.show(s)
855                 time.sleep(1)
856                 rotate_nya = rotate_nya[1:2] + rotate_nya[0]
857
858 #---------- main program ----------
859
860 def main():
861         global opts, fetcher
862
863         pa = OptionParser(
864 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
865 actions:
866  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
867  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
868  yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
869  yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG
870  yoweb-scrape [options] ship-aid CHAT-LOG  (must be .../PIRATE_OCEAN_chat-log*)
871
872 display modes (for --display) apply to ship-aid:
873  --display=dumb       just print new information, scrolling the screen
874  --display=overwrite  use cursor motion, selective clear, etc. to redraw at top
875 ''')
876         ao = pa.add_option
877         ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
878                 help='select ocean OCEAN')
879         ao('--cache-dir', dest='cache_dir', metavar='DIR',
880                 default='~/.yoweb-scrape-cache',
881                 help='cache yoweb pages in DIR')
882         ao('-D','--debug', action='count', dest='debug', default=0,
883                 help='enable debugging output')
884         ao('--debug-fd', action='count', dest='debug_fd',
885                 help='write any debugging output to specified fd')
886         ao('-q','--quiet', action='store_true', dest='quiet',
887                 help='suppress warning output')
888         ao('--display', action='store', dest='display',
889                 type='choice', choices=['dumb','overwrite'],
890                 help='how to display ship aid')
891
892         ao('--ship-duty', action='store_true', dest='ship_duty',
893                 help='show ship duty station puzzles')
894         ao('--all-puzzles', action='store_false', dest='ship_duty',
895                 help='show all puzzles, not just ship duty stations')
896
897         (opts,args) = pa.parse_args()
898         random.seed()
899
900         if len(args) < 1:
901                 pa.error('need a mode argument')
902
903         if opts.debug_fd is not None:
904                 opts.debug_file = fdopen(opts.debug_fd, 'w')
905
906         mode = args[0]
907         mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
908         try: mode_fn = globals()[mode_fn_name]
909         except KeyError: pa.error('unknown mode "%s"' % mode)
910
911         # fixed parameters
912         opts.min_max_age = 60
913         opts.expire_age = 3600
914         opts.ship_reboard_clearout = 3600
915
916         if opts.cache_dir.startswith('~/'):
917                 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
918
919         if opts.display is None:
920                 if ((opts.debug > 0 and opts.debug_fd is None)
921                     or not os.isatty(sys.stdout.fileno())):
922                         opts.display = 'dumb'
923                 else:
924                         opts.display = 'overwrite'
925
926         fetcher = Fetcher(opts.ocean, opts.cache_dir)
927
928         mode_fn(args[1:], pa.error)
929
930 main()