chiark / gitweb /
Ability to sort pirates by ability in specific duty puzzles
[ypp-sc-tools.main.git] / yoweb-scrape
1 #!/usr/bin/python
2
3 #---------- setup ----------
4
5 import signal
6 signal.signal(signal.SIGINT, signal.SIG_DFL)
7
8 import os
9 import time
10 import urllib
11 import urllib2
12 import errno
13 import sys
14 import re as regexp
15 import random
16 import curses
17 import termios
18 from optparse import OptionParser
19
20 from BeautifulSoup import BeautifulSoup
21
22 opts = None
23
24 #---------- YPP parameters and arrays ----------
25
26 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
27         '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
28         '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
29         '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
30
31 core_duty_puzzles = [
32                 'Gunning',
33                 ['Sailing','Rigging'],
34                 'Bilging',
35                 'Carpentry',
36                 ]
37
38 duty_puzzles = ([ 'Navigating', 'Battle Navigation' ] +
39                 core_duty_puzzles +
40                 [ 'Treasure Haul' ])
41
42 standingvals = ('Able/Distinguished/Respected/Master'+
43                 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
44 standing_limit = len(standingvals)
45
46 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
47
48 max_pirate_namelen = 12
49
50
51 #---------- general utilities ----------
52
53 def debug(m):
54         if opts.debug > 0:
55                 print >>opts.debug_file, m
56
57 def debug_flush():
58         if opts.debug > 0:
59                 opts.debug_file.flush() 
60
61 def sleep(seconds):
62         debug_flush()
63         time.sleep(seconds)
64
65 def format_time_interval(ti):
66         if ti < 120: return '%d:%02d' % (ti / 60, ti % 60)
67         if ti < 7200: return '%2dm' % (ti / 60)
68         if ti < 86400: return '%dh' % (ti / 3600)
69         return '%dd' % (ti / 86400)
70
71 #---------- caching and rate-limiting data fetcher ----------
72
73 class Fetcher:
74         def __init__(self, ocean, cachedir):
75                 debug('Fetcher init %s' % cachedir)
76                 self.ocean = ocean
77                 self.cachedir = cachedir
78                 try: os.mkdir(cachedir)
79                 except (OSError,IOError), oe:
80                         if oe.errno != errno.EEXIST: raise
81                 self._cache_scan(time.time())
82
83         def default_ocean(self, ocean='ice'):
84                 if self.ocean is None:
85                         self.ocean = ocean
86
87         def _cache_scan(self, now):
88                 # returns list of ages, unsorted
89                 ages = []
90                 debug('Fetcher   scan_cache')
91                 for leaf in os.listdir(self.cachedir):
92                         if not leaf.startswith('#'): continue
93                         path = self.cachedir + '/' + leaf
94                         try: s = os.stat(path)
95                         except (OSError,IOError), oe:
96                                 if oe.errno != errno.ENOENT: raise
97                                 continue
98                         age = now - s.st_mtime
99                         if age > opts.expire_age:
100                                 debug('Fetcher    expire %d %s' % (age, path))
101                                 try: os.remove(path)
102                                 except (OSError,IOError), oe:
103                                         if oe.errno != errno.ENOENT: raise
104                                 continue
105                         ages.append(age)
106                 return ages
107
108         def need_wait(self, now, imaginary=[]):
109                 ages = self._cache_scan(now)
110                 ages += imaginary
111                 ages.sort()
112                 debug('Fetcher   ages ' + `ages`)
113                 min_age = 1
114                 need_wait = 0
115                 for age in ages:
116                         if age < min_age and age < 300:
117                                 debug('Fetcher   morewait min=%d age=%d' %
118                                         (min_age, age))
119                                 need_wait = max(need_wait, min_age - age)
120                         min_age += 3
121                         min_age *= 1.25
122                 return need_wait
123
124         def _rate_limit_cache_clean(self, now):
125                 need_wait = self.need_wait(now)
126                 if need_wait > 0:
127                         debug('Fetcher   wait %d' % need_wait)
128                         sleep(need_wait)
129
130         def fetch(self, url, max_age):
131                 debug('Fetcher fetch %s' % url)
132                 cache_corename = urllib.quote_plus(url)
133                 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
134                 try: f = file(cache_item, 'r')
135                 except (OSError,IOError), oe:
136                         if oe.errno != errno.ENOENT: raise
137                         f = None
138                 now = time.time()
139                 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
140                 if f is not None:
141                         s = os.fstat(f.fileno())
142                         age = now - s.st_mtime
143                         if age > max_age:
144                                 debug('Fetcher  stale %d < %d'% (max_age, age))
145                                 f = None
146                 if f is not None:
147                         data = f.read()
148                         f.close()
149                         debug('Fetcher  cached %d > %d' % (max_age, age))
150                         return data
151
152                 debug('Fetcher  fetch')
153                 self._rate_limit_cache_clean(now)
154
155                 stream = urllib2.urlopen(url)
156                 data = stream.read()
157                 cache_tmp = "%s/#%s~%d#" % (
158                         self.cachedir, cache_corename, os.getpid())
159                 f = file(cache_tmp, 'w')
160                 f.write(data)
161                 f.close()
162                 os.rename(cache_tmp, cache_item)
163                 debug('Fetcher  stored')
164                 return data
165
166         def yoweb(self, kind, tail, max_age):
167                 self.default_ocean()
168                 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
169                         self.ocean, kind, tail)
170                 return self.fetch(url, max_age)
171
172 #---------- logging assistance for troubled screenscrapers ----------
173
174 class SoupLog:
175         def __init__(self):
176                 self.msgs = [ ]
177         def msg(self, m):
178                 self.msgs.append(m)
179         def soupm(self, obj, m):
180                 self.msg(m + '; in ' + `obj`)
181         def needs_msgs(self, child_souplog):
182                 self.msgs += child_souplog.msgs
183                 child_souplog.msgs = [ ]
184
185 def soup_text(obj):
186         str = ''.join(obj.findAll(text=True))
187         return str.strip()
188
189 class SomethingSoupInfo(SoupLog):
190         def __init__(self, kind, tail, max_age):
191                 SoupLog.__init__(self)
192                 html = fetcher.yoweb(kind, tail, max_age)
193                 self._soup = BeautifulSoup(html,
194                         convertEntities=BeautifulSoup.HTML_ENTITIES
195                         )
196
197 #---------- scraper for pirate pages ----------
198
199 class PirateInfo(SomethingSoupInfo):
200         # Public data members:
201         #  pi.standings = { 'Treasure Haul': 'Able' ... }
202         #  pi.name = name
203         #  pi.crew = (id, name)
204         #  pi.flag = (id, name)
205         #  pi.msgs = [ 'message describing problem with scrape' ]
206                 
207         def __init__(self, pirate, max_age=300):
208                 SomethingSoupInfo.__init__(self,
209                         'pirate.wm?target=', pirate, max_age)
210                 self.name = pirate
211                 self._find_standings()
212                 self.crew = self._find_crewflag('crew',
213                         '^/yoweb/crew/info\\.wm')
214                 self.flag = self._find_crewflag('flag',
215                         '^/yoweb/flag/info\\.wm')
216
217         def _find_standings(self):
218                 imgs = self._soup.findAll('img',
219                         src=regexp.compile('/yoweb/images/stat.*'))
220                 re = regexp.compile(
221 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
222                         )
223                 standings = { }
224
225                 for skill in puzzles:
226                         standings[skill] = [ ]
227
228                 skl = SoupLog()
229
230                 for img in imgs:
231                         try: puzzle = img['alt']
232                         except KeyError: continue
233
234                         if not puzzle in puzzles:
235                                 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
236                                 continue
237                         key = img.findParent('td')
238                         if key is None:
239                                 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
240                                 continue
241                         valelem = key.findNextSibling('td')
242                         if valelem is None:
243                                 skl.soupm(key, 'puzzle missing sibling "%s"'
244                                         % puzzle)
245                                 continue
246                         valstr = soup_text(valelem)
247                         match = re.match(valstr)
248                         if match is None:
249                                 skl.soupm(key, ('puzzle "%s" unparseable'+
250                                         ' standing "%s"') % (puzzle, valstr))
251                                 continue
252                         standing = match.group(match.lastindex)
253                         standings[puzzle].append(standing)
254
255                 self.standings = { }
256
257                 for puzzle in puzzles:
258                         sl = standings[puzzle]
259                         if len(sl) > 1:
260                                 skl.msg('puzzle "%s" multiple standings %s' %
261                                                 (puzzle, `sl`))
262                                 continue
263                         if not sl:
264                                 skl.msg('puzzle "%s" no standing found' % puzzle)
265                                 continue
266                         standing = sl[0]
267                         for i in range(0, standing_limit):
268                                 if standing == standingvals[i]:
269                                         self.standings[puzzle] = i
270                         if not puzzle in self.standings:
271                                 skl.msg('puzzle "%s" unknown standing "%s"' %
272                                         (puzzle, standing))
273
274                 all_standings_ok = True
275                 for puzzle in puzzles:
276                         if not puzzle in self.standings:
277                                 self.needs_msgs(skl)
278
279         def _find_crewflag(self, cf, yoweb_re):
280                 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
281                 if len(things) != 1:
282                         self.msg('zero or several %s id references found' % cf)
283                         return None
284                 thing = things[0]
285                 id_re = '\\b%sid\\=(\\w+)$' % cf
286                 id_haystack = thing['href']
287                 match = regexp.compile(id_re).search(id_haystack)
288                 if match is None:
289                         self.soupm(thing, ('incomprehensible %s id ref'+
290                                 ' (%s in %s)') % (cf, id_re, id_haystack))
291                         return None
292                 name = soup_text(thing)
293                 return (match.group(1), name)
294
295         def __str__(self):
296                 return `(self.crew, self.flag, self.standings, self.msgs)`
297
298 #---------- scraper for crew pages ----------
299
300 class CrewInfo(SomethingSoupInfo):
301         # Public data members:
302         #  ci.crew = [ ('Captain',        ['Pirate', ...]),
303         #              ('Senior Officer', [...]),
304         #               ... ]
305         #  pi.msgs = [ 'message describing problem with scrape' ]
306
307         def __init__(self, crewid, max_age=300):
308                 SomethingSoupInfo.__init__(self,
309                         'crew/info.wm?crewid=', crewid, max_age)
310                 self._find_crew()
311
312         def _find_crew(self):
313                 self.crew = []
314                 capts = self._soup.findAll('img',
315                         src='/yoweb/images/crew-captain.png')
316                 if len(capts) != 1:
317                         self.msg('crew members: no. of captain images != 1')
318                         return
319                 tbl = capts[0]
320                 while not tbl.find('a', href=pirate_ref_re):
321                         tbl = tbl.findParent('table')
322                         if not tbl:
323                                 self.msg('crew members: cannot find table')
324                                 return
325                 current_rank_crew = None
326                 crew_rank_re = regexp.compile('/yoweb/images/crew')
327                 for row in tbl.contents:
328                         # findAll(recurse=False)
329                         if isinstance(row,basestring):
330                                 continue
331
332                         is_rank = row.find('img', attrs={'src': crew_rank_re})
333                         if is_rank:
334                                 rank = soup_text(row)
335                                 current_rank_crew = []
336                                 self.crew.append((rank, current_rank_crew))
337                                 continue
338                         for cell in row.findAll('a', href=pirate_ref_re):
339                                 if current_rank_crew is None:
340                                         self.soupm(cell, 'crew members: crew'
341                                                 ' before rank')
342                                         continue
343                                 current_rank_crew.append(soup_text(cell))
344
345         def __str__(self):
346                 return `(self.crew, self.msgs)`
347
348 #---------- pretty-printer for tables of pirate puzzle standings ----------
349
350 class StandingsTable:
351         def __init__(self, use_puzzles=None, col_width=6):
352                 if use_puzzles is None:
353                         if opts.ship_duty:
354                                 use_puzzles=duty_puzzles
355                         else:
356                                 use_puzzles=puzzles
357                 self._puzzles = use_puzzles
358                 self.s = ''
359                 self._cw = col_width-1
360
361         def _pline(self, pirate, puzstrs, extra):
362                 self.s += ' %-*s' % (max(max_pirate_namelen, 14), pirate)
363                 for v in puzstrs:
364                         self.s += ' %-*.*s' % (self._cw,self._cw, v)
365                 if extra:
366                         self.s += ' ' + extra
367                 self.s += '\n'
368
369         def _puzstr(self, pi, puzzle):
370                 if not isinstance(puzzle,list): puzzle = [puzzle]
371                 try: standing = max([pi.standings[p] for p in puzzle])
372                 except KeyError: return '?'
373                 if not standing: return ''
374                 s = ''
375                 if self._cw > 4:
376                         c1 = standingvals[standing][0]
377                         if standing < 3: c1 = c1.lower() # 3 = Master
378                         s += `standing`
379                 if self._cw > 5:
380                         s += ' '
381                 s += '*' * (standing / 2)
382                 s += '+' * (standing % 2)
383                 return s
384
385         def headings(self):
386                 def puzn_redact(name):
387                         if isinstance(name,list):
388                                 return '/'.join(
389                                         ["%.*s" % (self._cw/2, puzn_redact(n))
390                                          for n in name])
391                         spc = name.find(' ')
392                         if spc < 0: return name
393                         return name[0:min(4,spc)] + name[spc+1:]
394                 self._pline('', map(puzn_redact, self._puzzles), None)
395         def literalline(self, line):
396                 self.s += line + '\n'
397         def pirate_dummy(self, name, standingstring, extra=None):
398                 self._pline(name, standingstring * len(self._puzzles), extra)
399         def pirate(self, pi, extra=None):
400                 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
401                 self._pline(pi.name, puzstrs, extra)
402
403         def results(self):
404                 return self.s
405
406 #---------- chat log parser ----------
407
408 class PirateAboard:
409         # This is essentially a transparent, dumb, data class.
410         #  pa.v
411         #  pa.name
412         #  pa.last_time
413         #  pa.last_event
414         #  pa.gunner
415         #  pa.last_chat_time
416         #  pa.last_chat_chan
417         #  pa.pi
418
419         def __init__(pa, pn, v, time, event):
420                 pa.name = pn
421                 pa.v = v
422                 pa.last_time = time
423                 pa.last_event = event
424                 pa.last_chat_time = None
425                 pa.last_chat_chan = None
426                 pa.gunner = False
427                 pa.pi = None
428
429         def pirate_info(pa):
430                 now = time.time()
431                 if pa.pi:
432                         age = now - pa.pi_fetched
433                         guide = random.randint(120,240)
434                         if age <= guide:
435                                 return pa.pi
436                         debug('PirateAboard refresh %d > %d  %s' % (
437                                 age, guide, pa.name))
438                         imaginary = [2,6]
439                 else:
440                         imaginary = [1]
441                 wait = fetcher.need_wait(now, imaginary)
442                 if wait:
443                         debug('PirateAboard fetcher not ready %d' % wait)
444                         return pa.pi
445                 pa.pi = PirateInfo(pa.name, 600)
446                 pa.pi_fetched = now
447                 return pa.pi
448
449 class ChatLogTracker:
450         # This is quite complex so we make it opaque.  Use the
451         # official invokers, accessors etc.
452
453         def __init__(self, myself_pi, logfn):
454                 self._pl = {}   # self._pl['Pirate'] =
455                 self._vl = {}   #   self._vl['Vessel']['Pirate'] = PirateAboard
456                                 # self._vl['Vessel']['#lastinfo']
457                                 # self._vl['Vessel']['#name']
458                                 # self._v = self._vl[self._vessel]
459                 self._date = None
460                 self._myself = myself_pi
461                 self._f = file(logfn)
462                 self._lbuf = ''
463                 self._progress = [0, os.fstat(self._f.fileno()).st_size]
464                 self._disembark_myself()
465                 self._need_redisplay = False
466                 self._lastvessel = None
467
468         def _disembark_myself(self):
469                 self._v = None
470                 self._vessel = None
471                 self.force_redisplay()
472
473         def force_redisplay(self):
474                 self._need_redisplay = True
475
476         def _vessel_updated(self, v, timestamp):
477                 v['#lastinfo'] = timestamp
478                 self.force_redisplay()
479
480         def _onboard_event(self,v,timestamp,pirate,event):
481                 pa = self._pl.get(pirate, None)
482                 if pa is not None and pa.v is v:
483                         pa.last_time = timestamp
484                         pa.last_event = event
485                 else:
486                         if pa is not None: del pa.v[pirate]
487                         pa = PirateAboard(pirate, v, timestamp, event)
488                         self._pl[pirate] = pa
489                         v[pirate] = pa
490                 self._vessel_updated(v, timestamp)
491                 return pa
492
493         def _trash_vessel(self, v):
494                 for pn in v:
495                         if pn.startswith('#'): continue
496                         del self._pl[pn]
497                 vn = v['#name']
498                 del self._vl[vn]
499                 if v is self._v: self._disembark_myself()
500                 self.force_redisplay()
501
502         def _vessel_stale(self, v, timestamp):
503                 return timestamp - v['#lastinfo'] > opts.ship_reboard_clearout
504
505         def _vessel_check_expire(self, v, timestamp):
506                 if not self._vessel_stale(v, timestamp):
507                         return v
508                 self._debug_line_disposition(timestamp,'',
509                         'stale-reset ' + v['#name'])
510                 self._trash_vessel(v)
511                 return None
512
513         def expire_garbage(self, timestamp):
514                 for v in self._vl.values():
515                         self._vessel_check_expire(v, timestamp)
516
517         def _vessel_lookup(self, vn, timestamp, dml=[], create=False):
518                 v = self._vl.get(vn, None)
519                 if v is not None:
520                         v = self._vessel_check_expire(v, timestamp)
521                 if v is not None:
522                         dml.append('found')
523                         return v
524                 if not create:
525                         dml.append('no')
526                 dml.append('new')
527                 self._vl[vn] = v = { '#name': vn }
528                 self._vessel_updated(v, timestamp)
529                 return v
530
531         def _find_matching_vessel(self, pattern, timestamp, cmdr,
532                                         dml=[], create=False):
533                 # use when a commander pirate `cmdr' specified a vessel
534                 #  by name `pattern' (either may be None)
535                 # if create is true, will create the vessel
536                 #  record if an exact name is specified
537
538                 if (pattern is not None and
539                     not '*' in pattern
540                     and len(pattern.split(' ')) == 2):
541                         vn = pattern.title()
542                         dml.append('exact')
543                         return self._vessel_lookup(
544                                 vn, timestamp, dml=dml, create=create)
545
546                 if pattern is None:
547                         pattern_check = lambda vn: True
548                 else:
549                         re = '(?:.* )?%s$' % pattern.lower().replace('*','.+')
550                         pattern_check = regexp.compile(re, regexp.I).match
551
552                 tries = []
553
554                 cmdr_pa = self._pl.get(cmdr, None)
555                 if cmdr_pa: tries.append((cmdr_pa.v, 'cmdr'))
556
557                 tries.append((self._v, 'here'))
558                 tried_vns = []
559
560                 for (v, dm) in tries:
561                         if v is None: dml.append(dm+'?'); continue
562                         
563                         vn = v['#name']
564                         if not pattern_check(vn):
565                                 tried_vns.append(vn)
566                                 dml.append(dm+'#')
567                                 continue
568
569                         dml.append(dm+'!')
570                         return v
571
572                 if pattern is not None and '*' in pattern:
573                         search = [
574                                 (vn,v)
575                                 for (vn,v) in self._vl.iteritems()
576                                 if not self._vessel_stale(v, timestamp)
577                                 if pattern_check(vn)
578                                 ]
579                         #debug('CLT-RE /%s/ wanted (%s) searched (%s)' % (
580                         #       re,
581                         #       '/'.join(tried_vns),
582                         #       '/'.join([vn for (vn,v) in search])))
583
584                         if len(search)==1:
585                                 dml.append('one')
586                                 return search[0][1]
587                         elif search:
588                                 dml.append('many')
589                         else:
590                                 dml.append('none')
591
592         def _debug_line_disposition(self,timestamp,l,m):
593                 debug('CLT %13s %-40s %s' % (timestamp,m,l))
594
595         def chatline(self,l):
596                 rm = lambda re: regexp.match(re,l)
597                 d = lambda m: self._debug_line_disposition(timestamp,l,m)
598                 timestamp = None
599
600                 m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$')
601                 if m:
602                         self._date = [int(x) for x in m.groups()]
603                         self._previous_timestamp = None
604                         return d('date '+`self._date`)
605
606                 if self._date is None:
607                         return d('date unset')
608
609                 m = rm('\\[(\d\d):(\d\d):(\d\d)\\] ')
610                 if not m:
611                         return d('no timestamp')
612
613                 while True:
614                         time_tuple = (self._date +
615                                       [int(x) for x in m.groups()] +
616                                       [-1,-1,-1])
617                         timestamp = time.mktime(time_tuple)
618                         if timestamp >= self._previous_timestamp: break
619                         self._date[2] += 1
620                         self._debug_line_disposition(timestamp,'',
621                                 'new date '+`self._date`)
622
623                 self._previous_timestamp = timestamp
624
625                 l = l[l.find(' ')+1:]
626
627                 def ob_x(pirate,event):
628                         return self._onboard_event(
629                                         self._v, timestamp, pirate, event)
630                 def ob1(did): ob_x(m.group(1), did); return d(did)
631                 def oba(did): return ob1('%s %s' % (did, m.group(2)))
632
633                 def disembark(v, timestamp, pirate, event):
634                         self._onboard_event(
635                                         v, timestamp, pirate, 'leaving '+event)
636                         del v[pirate]
637                         del self._pl[pirate]
638
639                 def disembark_me(why):
640                         self._disembark_myself()
641                         return d('disembark-me '+why)
642
643                 m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
644                 if m:
645                         dm = ['boarding']
646                         pn = self._myself.name
647                         vn = m.group(1)
648                         v = self._vessel_lookup(vn, timestamp, dm, create=True)
649                         self._lastvessel = self._vessel = vn
650                         self._v = v
651                         ob_x(pn, 'we boarded')
652                         self.expire_garbage(timestamp)
653                         return d(' '.join(dm))
654
655                 if self._v is None:
656                         return d('no vessel')
657
658                 m = rm('(\\w+) has come aboard\\.$')
659                 if m: return ob1('boarded');
660
661                 m = rm('You have ordered (\\w+) to do some (\\S.*\\S)\\.$')
662                 if m:
663                         (who,what) = m.groups()
664                         pa = ob_x(who,'ord '+what)
665                         if what == 'Gunning':
666                                 pa.gunner = True
667                         return d('duty order')
668
669                 m = rm('(\\w+) abandoned a (\\S.*\\S) station\\.$')
670                 if m: oba('stopped'); return d("end")
671
672                 def chat_core(speaker, chan):
673                         try: pa = self._pl[speaker]
674                         except KeyError: return 'mystery'
675                         if pa.v is not self._v: return 'elsewhere'
676                         pa.last_chat_time = timestamp
677                         pa.last_chat_chan = chan
678                         self.force_redisplay()
679                         return 'here'
680
681                 def chat(chan):
682                         speaker = m.group(1)
683                         dm = chat_core(speaker, chan)
684                         return d('chat %s %s' % (chan, dm))
685
686                 def chat_metacmd(chan):
687                         (cmdr, metacmd) = m.groups()
688                         metacmd = regexp.sub('\\s+', ' ', metacmd).strip()
689                         m2 = regexp.match(
690                                 '/([ad]) (?:([A-Za-z* ]+)\\s*:)?([A-Za-z ]+)$',
691                                 metacmd)
692                         if not m2: return chat(chan)
693
694                         (cmd, pattern, targets) = m2.groups()
695                         dml = ['cmd', chan, cmd]
696
697                         if cmd == 'a': each = self._onboard_event
698                         else: each = disembark
699
700                         if cmdr == self._myself.name:
701                                 dml.append('self')
702                                 how = 'cmd: %s' % cmd
703                         else:
704                                 dml.append('other')
705                                 how = 'cmd: %s %s' % (cmd,cmdr)
706
707                         v = self._find_matching_vessel(
708                                 pattern, timestamp, cmdr, dml, create=True)
709
710                         if v is not None:
711                                 targets = targets.strip().split(' ')
712                                 dml.append(`len(targets)`)
713                                 for target in targets:
714                                         each(v, timestamp, target.title(), how)
715                                 self._vessel_updated(v, timestamp)
716
717                         dm = ' '.join(dml)
718                         chat_core(cmdr, 'cmd '+chan)
719                         return d(dm)
720
721                 m = rm('(\\w+) (?:issued an order|ordered everyone) "')
722                 if m: return ob1('general order');
723
724                 m = rm('(\\w+) says, "')
725                 if m: return chat('public')
726
727                 m = rm('(\\w+) tells ye, "')
728                 if m: return chat('private')
729
730                 m = rm('Ye told (\\w+), "(.*)"$')
731                 if m: return chat_metacmd('private')
732
733                 m = rm('(\\w+) flag officer chats, "')
734                 if m: return chat('flag officer')
735
736                 m = rm('(\\w+) officer chats, "(.*)"$')
737                 if m: return chat_metacmd('officer')
738
739                 m = rm('Ye accepted the offer to job with ')
740                 if m: return disembark_me('jobbing')
741
742                 m = rm('Ye hop on the ferry and are whisked away ')
743                 if m: return disembark_me('ferry')
744
745                 m = rm('Whisking away to yer home on the magical winds')
746                 if m: return disembark_me('home')
747
748                 m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
749                 if m:
750                         pl = m.group(1).split(', ')
751                         if not self._myself.name in pl:
752                                 return d('lost melee')
753                         for pn in pl:
754                                 if ' ' in pn: continue
755                                 ob_x(pn,'won melee')
756                         return d('won melee')
757
758                 m = rm('(\\w+) is eliminated\\!')
759                 if m: return ob1('eliminated in fray');
760
761                 m = rm('(\\w+) has driven \w+ from the ship\\!')
762                 if m: return ob1('boarder repelled');
763
764                 m = rm('\w+ has bested (\\w+), and turns'+
765                         ' to the rest of the ship\\.')
766                 if m: return ob1('boarder unrepelled');
767
768                 m = rm('(\\w+) has left the vessel\.')
769                 if m:
770                         pirate = m.group(1)
771                         disembark(self._v, timestamp, pirate, 'disembarked')
772                         return d('disembarked')
773
774                 return d('not-matched')
775
776         def _str_vessel(self, vn, v):
777                 s = ' vessel %s\n' % vn
778                 s += ' '*20 + "%-*s   %13s\n" % (
779                                 max_pirate_namelen, '#lastinfo',
780                                 v['#lastinfo'])
781                 assert v['#name'] == vn
782                 for pn in sorted(v.keys()):
783                         if pn.startswith('#'): continue
784                         pa = v[pn]
785                         assert pa.v == v
786                         assert self._pl[pn] == pa
787                         s += ' '*20 + "%s %-*s %13s %-30s %13s %s\n" % (
788                                 (' ','G')[pa.gunner],
789                                 max_pirate_namelen, pn,
790                                 pa.last_time, pa.last_event,
791                                 pa.last_chat_time, pa.last_chat_chan)
792                 return s
793
794         def __str__(self):
795                 s = '''<ChatLogTracker
796  myself %s
797  vessel %s
798 '''                     % (self._myself.name, self._vessel)
799                 assert ((self._v is None and self._vessel is None) or
800                         (self._v is self._vl[self._vessel]))
801                 if self._vessel is not None:
802                         s += self._str_vessel(self._vessel, self._v)
803                 for vn in sorted(self._vl.keys()):
804                         if vn == self._vessel: continue
805                         s += self._str_vessel(vn, self._vl[vn])
806                 for p in self._pl:
807                         pa = self._pl[p]
808                         assert pa.v[p] is pa
809                         assert pa.v in self._vl.values()
810                 s += '>\n'
811                 return s
812
813         def catchup(self, progress=None):
814                 while True:
815                         more = self._f.readline()
816                         if not more: break
817
818                         self._progress[0] += len(more)
819                         if progress: progress.progress(*self._progress)
820
821                         self._lbuf += more
822                         if self._lbuf.endswith('\n'):
823                                 self.chatline(self._lbuf.rstrip())
824                                 self._lbuf = ''
825                                 if opts.debug >= 2:
826                                         debug(self.__str__())
827                 if progress: progress.caughtup()
828
829         def changed(self):
830                 rv = self._need_redisplay
831                 self._need_redisplay = False
832                 return rv
833         def myname(self):
834                 # returns our pirate name
835                 return self._myself.name
836         def vesselname(self):
837                 # returns the vessel name we're aboard or None
838                 return self._vessel
839         def lastvesselname(self):
840                 # returns the last vessel name we were aboard or None
841                 return self._lastvessel
842         def aboard(self, vesselname=True):
843                 # returns a list of PirateAboard the vessel
844                 #  sorted by pirate name
845                 #  you can pass this None and you'll get []
846                 #  or True for the current vessel (which is the default)
847                 #  the returned value is a fresh list of persistent
848                 #  PirateAboard objects
849                 if vesselname is True: v = self._v
850                 else: v = self._vl.get(vesselname.title())
851                 if v is None: return []
852                 return [ v[pn]
853                          for pn in sorted(v.keys())
854                          if not pn.startswith('#') ]
855
856 #---------- implementations of actual operation modes ----------
857
858 def do_pirate(pirates, bu):
859         print '{'
860         for pirate in pirates:
861                 info = PirateInfo(pirate)
862                 print '%s: %s,' % (`pirate`, info)
863         print '}'
864
865 def prep_crew_of(args, bu, max_age=300):
866         if len(args) != 1: bu('crew-of takes one pirate name')
867         pi = PirateInfo(args[0], max_age)
868         if pi.crew is None: return None
869         return CrewInfo(pi.crew[0], max_age)
870
871 def do_crew_of(args, bu):
872         ci = prep_crew_of(args, bu)
873         print ci
874
875 def do_standings_crew_of(args, bu):
876         ci = prep_crew_of(args, bu, 60)
877         tab = StandingsTable()
878         tab.headings()
879         for (rank, members) in ci.crew:
880                 if not members: continue
881                 tab.literalline('%s:' % rank)
882                 for p in members:
883                         pi = PirateInfo(p, random.randint(900,1800))
884                         tab.pirate(pi)
885         print tab.results()
886
887 class ProgressPrintPercentage:
888         def __init__(self, f=sys.stdout):
889                 self._f = f
890         def progress_string(self,done,total):
891                 return "scan chat logs %3d%%\r" % ((done*100) / total)
892         def progress(self,*a):
893                 self._f.write(self.progress_string(*a))
894                 self._f.flush()
895         def show_init(self, pirate, ocean):
896                 print >>self._f, 'Starting up, %s on the %s ocean' % (
897                         pirate, ocean)
898         def caughtup(self):
899                 self._f.write('                   \r')
900                 self._f.flush()
901
902 #----- modes which use the chat log parser are quite complex -----
903
904 def prep_chat_log(args, bu,
905                 progress=ProgressPrintPercentage(),
906                 max_myself_age=3600):
907         if len(args) != 1: bu('this action takes only chat log filename')
908         logfn = args[0]
909         logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_'
910         match = regexp.match(logfn_re, logfn)
911         if not match: bu('chat log filename is not in expected format')
912         (pirate, ocean) = match.groups()
913         fetcher.default_ocean(ocean)
914
915         progress.show_init(pirate, fetcher.ocean)
916         myself = PirateInfo(pirate,max_myself_age)
917         track = ChatLogTracker(myself, logfn)
918
919         opts.debug -= 2
920         track.catchup(progress)
921         opts.debug += 2
922
923         track.force_redisplay()
924
925         return (myself, track)
926
927 def do_track_chat_log(args, bu):
928         (myself, track) = prep_chat_log(args, bu)
929         while True:
930                 track.catchup()
931                 if track.changed():
932                         print track
933                 sleep(1)
934
935 #----- ship management aid -----
936
937 class Display_dumb(ProgressPrintPercentage):
938         def __init__(self):
939                 ProgressPrintPercentage.__init__(self)
940         def show(self, s):
941                 print '\n\n', s;
942         def realstart(self):
943                 pass
944
945 class Display_overwrite(ProgressPrintPercentage):
946         def __init__(self):
947                 ProgressPrintPercentage.__init__(self)
948
949                 null = file('/dev/null','w')
950                 curses.setupterm(fd=null.fileno())
951
952                 self._clear = curses.tigetstr('clear')
953                 if not self._clear:
954                         self._debug('missing clear!')
955                         self.show = Display_dumb.show
956                         return
957
958                 self._t = {'el':'', 'ed':''}
959                 if not self._init_sophisticated():
960                         for k in self._t.keys(): self._t[k] = ''
961                         self._t['ho'] = self._clear
962
963         def _debug(self,m): debug('display overwrite: '+m)
964
965         def _init_sophisticated(self):
966                 for k in self._t.keys():
967                         s = curses.tigetstr(k)
968                         self._t[k] = s
969                 self._t['ho'] = curses.tigetstr('ho')
970                 if not self._t['ho']:
971                         cup = curses.tigetstr('cup')
972                         self._t['ho'] = curses.tparm(cup,0,0)
973                 missing = [k for k in self._t.keys() if not self._t[k]]
974                 if missing:
975                         self.debug('missing '+(' '.join(missing)))
976                         return 0
977                 return 1
978
979         def show(self, s):
980                 w = sys.stdout.write
981                 def wti(k): w(self._t[k])
982
983                 wti('ho')
984                 nl = ''
985                 for l in s.rstrip().split('\n'):
986                         w(nl)
987                         w(l)
988                         wti('el')
989                         nl = '\r\n'
990                 wti('ed')
991                 w(' ')
992                 sys.stdout.flush()
993
994         def realstart(self):
995                 sys.stdout.write(self._clear)
996                 sys.stdout.flush()
997                         
998
999 def do_ship_aid(args, bu):
1000         if opts.ship_duty is None: opts.ship_duty = True
1001
1002         displayer = globals()['Display_'+opts.display]()
1003
1004         (myself, track) = prep_chat_log(args, bu, progress=displayer)
1005
1006         displayer.realstart()
1007
1008         if os.isatty(0): kr_create = KeystrokeReader
1009         else: kr_create = DummyKeystrokeReader
1010
1011         try:
1012                 kreader = kr_create(0, 10)
1013                 ship_aid_core(myself, track, displayer, kreader)
1014         finally:
1015                 kreader.stop()
1016                 print '\n'
1017
1018 class KeyBasedSorter:
1019         def compar_key_pa(self, pa):
1020                 return self.compar_key(pa.pirate_info())
1021         def lsort_pa(self, l):
1022                 l.sort(key = self.compar_key_pa)
1023
1024 class NameSorter(KeyBasedSorter):
1025         def compar_key(self, pi): return pi.name
1026
1027 class SkillSorter(NameSorter):
1028         def __init__(self, relevant):
1029                 self._want = frozenset(relevant.split('/'))
1030                 self._avoid = set()
1031                 for p in core_duty_puzzles:
1032                         if isinstance(p,basestring): self._avoid.add(p)
1033                         else: self._avoid |= set(p)
1034                 self._avoid -= self._want
1035         
1036         def compar_key(self, pi):
1037                 best_want = max([
1038                         pi.standings.get(puz,-1)
1039                         for puz in self._want
1040                         ])
1041                 best_avoid = [
1042                         -pi.standings.get(puz,standing_limit)
1043                         for puz in self._avoid
1044                         ]
1045                 best_avoid.sort()
1046                 def negate(x): return -x
1047                 debug('compar_key %s bw=%s ba=%s' % (pi.name, `best_want`,
1048                         `best_avoid`))
1049                 return (-best_want, map(negate, best_avoid), pi.name)
1050
1051 def ship_aid_core(myself, track, displayer, kreader):
1052
1053         def find_vessel():
1054                 vn = track.vesselname()
1055                 if vn: return (vn, " on board the %s" % vn)
1056                 vn = track.lastvesselname()
1057                 if vn: return (vn, " ashore from the %s" % vn)
1058                 return (None, " not on a vessel")
1059
1060         def timeevent(t,e):
1061                 if t is None: return ' ' * 22
1062                 return " %-4s %-16s" % (format_time_interval(now - t),e)
1063
1064         displayer.show(track.myname() + find_vessel()[1] + '...')
1065
1066         rotate_nya = '/-\\'
1067
1068         sort = NameSorter()
1069
1070         while True:
1071                 track.catchup()
1072                 now = time.time()
1073
1074                 (vn, s) = find_vessel()
1075                 s = track.myname() + s
1076                 s += " at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
1077                 s += kreader.info()
1078                 s += '\n'
1079
1080                 tbl = StandingsTable()
1081                 tbl.headings()
1082
1083                 aboard = track.aboard(vn)
1084
1085                 sort.lsort_pa(aboard)
1086
1087                 for pa in aboard:
1088                         pi = pa.pirate_info()
1089
1090                         xs = ''
1091                         if pa.gunner: xs += 'G '
1092                         else: xs += '  '
1093                         xs += timeevent(pa.last_time, pa.last_event)
1094                         xs += timeevent(pa.last_chat_time, pa.last_chat_chan)
1095
1096                         if pi is None:
1097                                 tbl.pirate_dummy(pa.name, rotate_nya[0], xs)
1098                         else:
1099                                 tbl.pirate(pi, xs)
1100
1101                 s += tbl.results()
1102                 displayer.show(s)
1103
1104                 k = kreader.getch()
1105                 if k is None:
1106                         rotate_nya = rotate_nya[1:3] + rotate_nya[0]
1107                         continue
1108
1109                 if k == 'q': break
1110                 elif k == 'g': sort = SkillSorter('Gunning')
1111                 elif k == 'c': sort = SkillSorter('Carpentry')
1112                 elif k == 's': sort = SkillSorter('Sailing/Rigging')
1113                 elif k == 'b': sort = SkillSorter('Bilging')
1114                 elif k == 'n': sort = SkillSorter('Navigating')
1115                 elif k == 'd': sort = SkillSorter('Battle Navigation')
1116                 elif k == 't': sort = SkillSorter('Treasure Haul')
1117                 elif k == 'a': sort = NameSorter()
1118                 else: pass # unknown key command
1119
1120 #---------- individual keystroke input ----------
1121
1122 class DummyKeystrokeReader:
1123         def __init__(self,fd,timeout_dummy): pass
1124         def stop(self): pass
1125         def getch(self): sleep(1); return None
1126         def info(self): return ' [noninteractive]'
1127
1128 class KeystrokeReader(DummyKeystrokeReader):
1129         def __init__(self, fd, timeout_decisec=0):
1130                 self._fd = fd
1131                 self._saved = termios.tcgetattr(fd)
1132                 a = termios.tcgetattr(fd)
1133                 a[3] &= ~(termios.ECHO | termios.ECHONL |
1134                           termios.ICANON | termios.IEXTEN)
1135                 a[6][termios.VMIN] = 0
1136                 a[6][termios.VTIME] = timeout_decisec
1137                 termios.tcsetattr(fd, termios.TCSANOW, a)
1138         def stop(self):
1139                 termios.tcsetattr(self._fd, termios.TCSANOW, self._saved)
1140         def getch(self):
1141                 debug_flush()
1142                 byte = os.read(self._fd, 1)
1143                 if not len(byte): return None
1144                 return byte
1145         def info(self):
1146                 return ''
1147
1148 #---------- main program ----------
1149
1150 def main():
1151         global opts, fetcher
1152
1153         pa = OptionParser(
1154 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
1155 actions:
1156  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
1157  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
1158  yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
1159  yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG
1160  yoweb-scrape [options] ship-aid CHAT-LOG  (must be .../PIRATE_OCEAN_chat-log*)
1161
1162 display modes (for --display) apply to ship-aid:
1163  --display=dumb       just print new information, scrolling the screen
1164  --display=overwrite  use cursor motion, selective clear, etc. to redraw at top
1165 ''')
1166         ao = pa.add_option
1167         ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
1168                 help='select ocean OCEAN')
1169         ao('--cache-dir', dest='cache_dir', metavar='DIR',
1170                 default='~/.yoweb-scrape-cache',
1171                 help='cache yoweb pages in DIR')
1172         ao('-D','--debug', action='count', dest='debug', default=0,
1173                 help='enable debugging output')
1174         ao('--debug-fd', type='int', dest='debug_fd',
1175                 help='write any debugging output to specified fd')
1176         ao('-q','--quiet', action='store_true', dest='quiet',
1177                 help='suppress warning output')
1178         ao('--display', action='store', dest='display',
1179                 type='choice', choices=['dumb','overwrite'],
1180                 help='how to display ship aid')
1181
1182         ao('--ship-duty', action='store_true', dest='ship_duty',
1183                 help='show ship duty station puzzles')
1184         ao('--all-puzzles', action='store_false', dest='ship_duty',
1185                 help='show all puzzles, not just ship duty stations')
1186
1187         (opts,args) = pa.parse_args()
1188         random.seed()
1189
1190         if len(args) < 1:
1191                 pa.error('need a mode argument')
1192
1193         if opts.debug_fd is not None:
1194                 opts.debug_file = os.fdopen(opts.debug_fd, 'w')
1195         else:
1196                 opts.debug_file = sys.stdout
1197
1198         mode = args[0]
1199         mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
1200         try: mode_fn = globals()[mode_fn_name]
1201         except KeyError: pa.error('unknown mode "%s"' % mode)
1202
1203         # fixed parameters
1204         opts.min_max_age = 60
1205         opts.expire_age = 3600
1206         opts.ship_reboard_clearout = 3600
1207
1208         if opts.cache_dir.startswith('~/'):
1209                 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
1210
1211         if opts.display is None:
1212                 if ((opts.debug > 0 and opts.debug_fd is None)
1213                     or not os.isatty(sys.stdout.fileno())):
1214                         opts.display = 'dumb'
1215                 else:
1216                         opts.display = 'overwrite'
1217
1218         fetcher = Fetcher(opts.ocean, opts.cache_dir)
1219
1220         mode_fn(args[1:], pa.error)
1221
1222 main()