chiark / gitweb /
remove parser for ship info from notes file (@-thing)
[ypp-sc-tools.web-live.git] / yoweb-scrape
1 #!/usr/bin/python
2
3 #---------- setup ----------
4
5 import signal
6 signal.signal(signal.SIGINT, signal.SIG_DFL)
7
8 import os
9 import time
10 import urllib
11 import urllib2
12 import errno
13 import sys
14 import re as regexp
15 import random
16 import curses
17 from optparse import OptionParser
18
19 from BeautifulSoup import BeautifulSoup
20
21 opts = None
22
23 #---------- YPP parameters and arrays ----------
24
25 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
26         '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
27         '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
28         '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
29
30 standingvals = ('Able/Distinguished/Respected/Master'+
31                 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
32
33 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
34
35 max_pirate_namelen = 12
36
37
38 #---------- general utilities ----------
39
40 def debug(m):
41         if opts.debug > 0:
42                 print >>opts.debug_file, m
43
44 def sleep(seconds):
45         if opts.debug > 0:
46                 opts.debug_file.flush()
47         time.sleep(seconds)
48
49 def format_time_interval(ti):
50         if ti < 120: return '%d:%02d' % (ti / 60, ti % 60)
51         if ti < 7200: return '%2dm' % (ti / 60)
52         if ti < 86400: return '%dh' % (ti / 3600)
53         return '%dd' % (ti / 86400)
54
55 #---------- caching and rate-limiting data fetcher ----------
56
57 class Fetcher:
58         def __init__(self, ocean, cachedir):
59                 debug('Fetcher init %s' % cachedir)
60                 self.ocean = ocean
61                 self.cachedir = cachedir
62                 try: os.mkdir(cachedir)
63                 except (OSError,IOError), oe:
64                         if oe.errno != errno.EEXIST: raise
65                 self._cache_scan(time.time())
66
67         def default_ocean(self, ocean='ice'):
68                 if self.ocean is None:
69                         self.ocean = ocean
70
71         def _cache_scan(self, now):
72                 # returns list of ages, unsorted
73                 ages = []
74                 debug('Fetcher   scan_cache')
75                 for leaf in os.listdir(self.cachedir):
76                         if not leaf.startswith('#'): continue
77                         path = self.cachedir + '/' + leaf
78                         try: s = os.stat(path)
79                         except (OSError,IOError), oe:
80                                 if oe.errno != errno.ENOENT: raise
81                                 continue
82                         age = now - s.st_mtime
83                         if age > opts.expire_age:
84                                 debug('Fetcher    expire %d %s' % (age, path))
85                                 try: os.remove(path)
86                                 except (OSError,IOError), oe:
87                                         if oe.errno != errno.ENOENT: raise
88                                 continue
89                         ages.append(age)
90                 return ages
91
92         def need_wait(self, now, imaginary=[]):
93                 ages = self._cache_scan(now)
94                 ages += imaginary
95                 ages.sort()
96                 debug('Fetcher   ages ' + `ages`)
97                 min_age = 1
98                 need_wait = 0
99                 for age in ages:
100                         if age < min_age and age < 300:
101                                 debug('Fetcher   morewait min=%d age=%d' %
102                                         (min_age, age))
103                                 need_wait = max(need_wait, min_age - age)
104                         min_age += 3
105                         min_age *= 1.25
106                 return need_wait
107
108         def _rate_limit_cache_clean(self, now):
109                 need_wait = self.need_wait(now)
110                 if need_wait > 0:
111                         debug('Fetcher   wait %d' % need_wait)
112                         sleep(need_wait)
113
114         def fetch(self, url, max_age):
115                 debug('Fetcher fetch %s' % url)
116                 cache_corename = urllib.quote_plus(url)
117                 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
118                 try: f = file(cache_item, 'r')
119                 except (OSError,IOError), oe:
120                         if oe.errno != errno.ENOENT: raise
121                         f = None
122                 now = time.time()
123                 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
124                 if f is not None:
125                         s = os.fstat(f.fileno())
126                         age = now - s.st_mtime
127                         if age > max_age:
128                                 debug('Fetcher  stale %d < %d'% (max_age, age))
129                                 f = None
130                 if f is not None:
131                         data = f.read()
132                         f.close()
133                         debug('Fetcher  cached %d > %d' % (max_age, age))
134                         return data
135
136                 debug('Fetcher  fetch')
137                 self._rate_limit_cache_clean(now)
138
139                 stream = urllib2.urlopen(url)
140                 data = stream.read()
141                 cache_tmp = "%s/#%s~%d#" % (
142                         self.cachedir, cache_corename, os.getpid())
143                 f = file(cache_tmp, 'w')
144                 f.write(data)
145                 f.close()
146                 os.rename(cache_tmp, cache_item)
147                 debug('Fetcher  stored')
148                 return data
149
150         def yoweb(self, kind, tail, max_age):
151                 self.default_ocean()
152                 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
153                         self.ocean, kind, tail)
154                 return self.fetch(url, max_age)
155
156 #---------- logging assistance for troubled screenscrapers ----------
157
158 class SoupLog:
159         def __init__(self):
160                 self.msgs = [ ]
161         def msg(self, m):
162                 self.msgs.append(m)
163         def soupm(self, obj, m):
164                 self.msg(m + '; in ' + `obj`)
165         def needs_msgs(self, child_souplog):
166                 self.msgs += child_souplog.msgs
167                 child_souplog.msgs = [ ]
168
169 def soup_text(obj):
170         str = ''.join(obj.findAll(text=True))
171         return str.strip()
172
173 class SomethingSoupInfo(SoupLog):
174         def __init__(self, kind, tail, max_age):
175                 SoupLog.__init__(self)
176                 html = fetcher.yoweb(kind, tail, max_age)
177                 self._soup = BeautifulSoup(html,
178                         convertEntities=BeautifulSoup.HTML_ENTITIES
179                         )
180
181 #---------- scraper for pirate pages ----------
182
183 class PirateInfo(SomethingSoupInfo):
184         # Public data members:
185         #  pi.standings = { 'Treasure Haul': 'Able' ... }
186         #  pi.name = name
187         #  pi.crew = (id, name)
188         #  pi.flag = (id, name)
189         #  pi.msgs = [ 'message describing problem with scrape' ]
190                 
191         def __init__(self, pirate, max_age=300):
192                 SomethingSoupInfo.__init__(self,
193                         'pirate.wm?target=', pirate, max_age)
194                 self.name = pirate
195                 self._find_standings()
196                 self.crew = self._find_crewflag('crew',
197                         '^/yoweb/crew/info\\.wm')
198                 self.flag = self._find_crewflag('flag',
199                         '^/yoweb/flag/info\\.wm')
200
201         def _find_standings(self):
202                 imgs = self._soup.findAll('img',
203                         src=regexp.compile('/yoweb/images/stat.*'))
204                 re = regexp.compile(
205 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
206                         )
207                 standings = { }
208
209                 for skill in puzzles:
210                         standings[skill] = [ ]
211
212                 skl = SoupLog()
213
214                 for img in imgs:
215                         try: puzzle = img['alt']
216                         except KeyError: continue
217
218                         if not puzzle in puzzles:
219                                 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
220                                 continue
221                         key = img.findParent('td')
222                         if key is None:
223                                 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
224                                 continue
225                         valelem = key.findNextSibling('td')
226                         if valelem is None:
227                                 skl.soupm(key, 'puzzle missing sibling "%s"'
228                                         % puzzle)
229                                 continue
230                         valstr = soup_text(valelem)
231                         match = re.match(valstr)
232                         if match is None:
233                                 skl.soupm(key, ('puzzle "%s" unparseable'+
234                                         ' standing "%s"') % (puzzle, valstr))
235                                 continue
236                         standing = match.group(match.lastindex)
237                         standings[puzzle].append(standing)
238
239                 self.standings = { }
240
241                 for puzzle in puzzles:
242                         sl = standings[puzzle]
243                         if len(sl) > 1:
244                                 skl.msg('puzzle "%s" multiple standings %s' %
245                                                 (puzzle, `sl`))
246                                 continue
247                         if not sl:
248                                 skl.msg('puzzle "%s" no standing found' % puzzle)
249                                 continue
250                         standing = sl[0]
251                         for i in range(0, len(standingvals)-1):
252                                 if standing == standingvals[i]:
253                                         self.standings[puzzle] = i
254                         if not puzzle in self.standings:
255                                 skl.msg('puzzle "%s" unknown standing "%s"' %
256                                         (puzzle, standing))
257
258                 all_standings_ok = True
259                 for puzzle in puzzles:
260                         if not puzzle in self.standings:
261                                 self.needs_msgs(skl)
262
263         def _find_crewflag(self, cf, yoweb_re):
264                 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
265                 if len(things) != 1:
266                         self.msg('zero or several %s id references found' % cf)
267                         return None
268                 thing = things[0]
269                 id_re = '\\b%sid\\=(\\w+)$' % cf
270                 id_haystack = thing['href']
271                 match = regexp.compile(id_re).search(id_haystack)
272                 if match is None:
273                         self.soupm(thing, ('incomprehensible %s id ref'+
274                                 ' (%s in %s)') % (cf, id_re, id_haystack))
275                         return None
276                 name = soup_text(thing)
277                 return (match.group(1), name)
278
279         def __str__(self):
280                 return `(self.crew, self.flag, self.standings, self.msgs)`
281
282 #---------- scraper for crew pages ----------
283
284 class CrewInfo(SomethingSoupInfo):
285         # Public data members:
286         #  ci.crew = [ ('Captain',        ['Pirate', ...]),
287         #              ('Senior Officer', [...]),
288         #               ... ]
289         #  pi.msgs = [ 'message describing problem with scrape' ]
290
291         def __init__(self, crewid, max_age=300):
292                 SomethingSoupInfo.__init__(self,
293                         'crew/info.wm?crewid=', crewid, max_age)
294                 self._find_crew()
295
296         def _find_crew(self):
297                 self.crew = []
298                 capts = self._soup.findAll('img',
299                         src='/yoweb/images/crew-captain.png')
300                 if len(capts) != 1:
301                         self.msg('crew members: no. of captain images != 1')
302                         return
303                 tbl = capts[0]
304                 while not tbl.find('a', href=pirate_ref_re):
305                         tbl = tbl.findParent('table')
306                         if not tbl:
307                                 self.msg('crew members: cannot find table')
308                                 return
309                 current_rank_crew = None
310                 crew_rank_re = regexp.compile('/yoweb/images/crew')
311                 for row in tbl.contents:
312                         # findAll(recurse=False)
313                         if isinstance(row,basestring):
314                                 continue
315
316                         is_rank = row.find('img', attrs={'src': crew_rank_re})
317                         if is_rank:
318                                 rank = soup_text(row)
319                                 current_rank_crew = []
320                                 self.crew.append((rank, current_rank_crew))
321                                 continue
322                         for cell in row.findAll('a', href=pirate_ref_re):
323                                 if current_rank_crew is None:
324                                         self.soupm(cell, 'crew members: crew'
325                                                 ' before rank')
326                                         continue
327                                 current_rank_crew.append(soup_text(cell))
328
329         def __str__(self):
330                 return `(self.crew, self.msgs)`
331
332 #---------- pretty-printer for tables of pirate puzzle standings ----------
333
334 class StandingsTable:
335         def __init__(self, use_puzzles=None, col_width=6):
336                 if use_puzzles is None:
337                         if opts.ship_duty:
338                                 use_puzzles=[
339                                         'Navigating','Battle Navigation',
340                                         'Gunning',
341                                         ['Sailing','Rigging'],
342                                         'Bilging',
343                                         'Carpentry',
344                                         'Treasure Haul'
345                                 ]
346                         else:
347                                 use_puzzles=puzzles
348                 self._puzzles = use_puzzles
349                 self.s = ''
350                 self._cw = col_width-1
351
352         def _pline(self, pirate, puzstrs, extra):
353                 self.s += ' %-*s' % (max(max_pirate_namelen, 14), pirate)
354                 for v in puzstrs:
355                         self.s += ' %-*.*s' % (self._cw,self._cw, v)
356                 if extra:
357                         self.s += ' ' + extra
358                 self.s += '\n'
359
360         def _puzstr(self, pi, puzzle):
361                 if not isinstance(puzzle,list): puzzle = [puzzle]
362                 try: standing = max([pi.standings[p] for p in puzzle])
363                 except KeyError: return '?'
364                 if not standing: return ''
365                 s = ''
366                 if self._cw > 4:
367                         c1 = standingvals[standing][0]
368                         if standing < 3: c1 = c1.lower() # 3 = Master
369                         s += `standing`
370                 if self._cw > 5:
371                         s += ' '
372                 s += '*' * (standing / 2)
373                 s += '+' * (standing % 2)
374                 return s
375
376         def headings(self):
377                 def puzn_redact(name):
378                         if isinstance(name,list):
379                                 return '/'.join(
380                                         ["%.*s" % (self._cw/2, puzn_redact(n))
381                                          for n in name])
382                         spc = name.find(' ')
383                         if spc < 0: return name
384                         return name[0:min(4,spc)] + name[spc+1:]
385                 self._pline('', map(puzn_redact, self._puzzles), None)
386         def literalline(self, line):
387                 self.s += line + '\n'
388         def pirate_dummy(self, name, standingstring, extra=None):
389                 self._pline(name, standingstring * len(self._puzzles), extra)
390         def pirate(self, pi, extra=None):
391                 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
392                 self._pline(pi.name, puzstrs, extra)
393
394         def results(self):
395                 return self.s
396
397 #---------- chat log parser ----------
398
399 class PirateAboard:
400         # This is essentially a transparent, dumb, data class.
401         #  pa.v
402         #  pa.name
403         #  pa.last_time
404         #  pa.last_event
405         #  pa.gunner
406         #  pa.last_chat_time
407         #  pa.last_chat_chan
408         #  pa.pi
409
410         def __init__(pa, pn, v, time, event):
411                 pa.name = pn
412                 pa.v = v
413                 pa.last_time = time
414                 pa.last_event = event
415                 pa.last_chat_time = None
416                 pa.last_chat_chan = None
417                 pa.gunner = False
418                 pa.pi = None
419
420         def pirate_info(pa):
421                 now = time.time()
422                 if pa.pi:
423                         age = now - pa.pi_fetched
424                         guide = random.randint(120,240)
425                         if age <= guide:
426                                 return pa.pi
427                         debug('PirateAboard refresh %d > %d  %s' % (
428                                 age, guide, pa.name))
429                         imaginary = [2,6]
430                 else:
431                         imaginary = [1]
432                 wait = fetcher.need_wait(now, imaginary)
433                 if wait:
434                         debug('PirateAboard fetcher not ready %d' % wait)
435                         return pa.pi
436                 pa.pi = PirateInfo(pa.name, 600)
437                 pa.pi_fetched = now
438                 return pa.pi
439
440 class ChatLogTracker:
441         # This is quite complex so we make it opaque.  Use the
442         # official invokers, accessors etc.
443
444         def __init__(self, myself_pi, logfn):
445                 self._pl = {}   # self._pl['Pirate'] =
446                 self._vl = {}   #   self._vl['Vessel']['Pirate'] = PirateAboard
447                                 # self._vl['Vessel']['#lastinfo']
448                                 # self._vl['Vessel']['#name']
449                                 # self._v = self._vl[self._vessel]
450                 self._date = None
451                 self._myself = myself_pi
452                 self._f = file(logfn)
453                 self._lbuf = ''
454                 self._progress = [0, os.fstat(self._f.fileno()).st_size]
455                 self._disembark_myself()
456                 self._need_redisplay = False
457                 self._lastvessel = None
458
459         def _disembark_myself(self):
460                 self._v = None
461                 self._vessel = None
462                 self.force_redisplay()
463
464         def force_redisplay(self):
465                 self._need_redisplay = True
466
467         def _vessel_updated(self, v, timestamp):
468                 v['#lastinfo'] = timestamp
469                 self.force_redisplay()
470
471         def _onboard_event(self,v,timestamp,pirate,event):
472                 pa = self._pl.get(pirate, None)
473                 if pa is not None and pa.v is v:
474                         pa.last_time = timestamp
475                         pa.last_event = event
476                 else:
477                         if pa is not None: del pa.v[pirate]
478                         pa = PirateAboard(pirate, v, timestamp, event)
479                         self._pl[pirate] = pa
480                         v[pirate] = pa
481                 self._vessel_updated(v, timestamp)
482                 return pa
483
484         def _trash_vessel(self, v):
485                 for pn in v:
486                         if pn.startswith('#'): continue
487                         del self._pl[pn]
488                 vn = v['#name']
489                 del self._vl[vn]
490                 if v is self._v: self._disembark_myself()
491                 self.force_redisplay()
492
493         def _vessel_stale(self, v, timestamp):
494                 return timestamp - v['#lastinfo'] > opts.ship_reboard_clearout
495
496         def _vessel_check_expire(self, v, timestamp):
497                 if not self._vessel_stale(v, timestamp):
498                         return v
499                 self._debug_line_disposition(timestamp,'',
500                         'stale-reset ' + v['#name'])
501                 self._trash_vessel(v)
502                 return None
503
504         def expire_garbage(self, timestamp):
505                 for v in self._vl.values():
506                         self._vessel_check_expire(v, timestamp)
507
508         def _vessel_lookup(self, vn, timestamp, dml=[], create=False):
509                 v = self._vl.get(vn, None)
510                 if v is not None:
511                         v = self._vessel_check_expire(v, timestamp)
512                 if v is not None:
513                         dml.append('found')
514                         return v
515                 if not create:
516                         dml.append('no')
517                 dml.append('new')
518                 self._vl[vn] = v = { '#name': vn }
519                 self._vessel_updated(v, timestamp)
520                 return v
521
522         def _find_matching_vessel(self, pattern, timestamp, cmdr,
523                                         dml=[], create=False):
524                 # use when a commander pirate `cmdr' specified a vessel
525                 #  by name `pattern' (either may be None)
526                 # if create is true, will create the vessel
527                 #  record if an exact name is specified
528
529                 if (pattern is not None and
530                     not '*' in pattern
531                     and len(pattern.split(' ')) == 2):
532                         vn = pattern.title()
533                         dml.append('exact')
534                         return self._vessel_lookup(
535                                 vn, timestamp, dml=dml, create=create)
536
537                 if pattern is None:
538                         pattern_check = lambda vn: True
539                 else:
540                         re = '(?:.* )?%s$' % pattern.lower().replace('*','.+')
541                         pattern_check = regexp.compile(re, regexp.I).match
542
543                 tries = []
544
545                 cmdr_pa = self._pl.get(cmdr, None)
546                 if cmdr_pa: tries.append((cmdr_pa.v, 'cmdr'))
547
548                 tries.append((self._v, 'here'))
549                 tried_vns = []
550
551                 for (v, dm) in tries:
552                         if v is None: dml.append(dm+'?'); continue
553                         
554                         vn = v['#name']
555                         if not pattern_check(vn):
556                                 tried_vns.append(vn)
557                                 dml.append(dm+'#')
558                                 continue
559
560                         dml.append(dm+'!')
561                         return v
562
563                 if pattern is not None and '*' in pattern:
564                         search = [
565                                 (vn,v)
566                                 for (vn,v) in self._vl.iteritems()
567                                 if not self._vessel_stale(v, timestamp)
568                                 if pattern_check(vn)
569                                 ]
570                         #debug('CLT-RE /%s/ wanted (%s) searched (%s)' % (
571                         #       re,
572                         #       '/'.join(tried_vns),
573                         #       '/'.join([vn for (vn,v) in search])))
574
575                         if len(search)==1:
576                                 dml.append('one')
577                                 return search[0][1]
578                         elif search:
579                                 dml.append('many')
580                         else:
581                                 dml.append('none')
582
583         def _debug_line_disposition(self,timestamp,l,m):
584                 debug('CLT %13s %-40s %s' % (timestamp,m,l))
585
586         def chatline(self,l):
587                 rm = lambda re: regexp.match(re,l)
588                 d = lambda m: self._debug_line_disposition(timestamp,l,m)
589                 timestamp = None
590
591                 m = rm('=+ (\\d+)/(\\d+)/(\\d+) =+$')
592                 if m:
593                         self._date = [int(x) for x in m.groups()]
594                         self._previous_timestamp = None
595                         return d('date '+`self._date`)
596
597                 if self._date is None:
598                         return d('date unset')
599
600                 m = rm('\\[(\d\d):(\d\d):(\d\d)\\] ')
601                 if not m:
602                         return d('no timestamp')
603
604                 while True:
605                         time_tuple = (self._date +
606                                       [int(x) for x in m.groups()] +
607                                       [-1,-1,-1])
608                         timestamp = time.mktime(time_tuple)
609                         if timestamp >= self._previous_timestamp: break
610                         self._date[2] += 1
611                         self._debug_line_disposition(timestamp,'',
612                                 'new date '+`self._date`)
613
614                 self._previous_timestamp = timestamp
615
616                 l = l[l.find(' ')+1:]
617
618                 def ob_x(pirate,event):
619                         return self._onboard_event(
620                                         self._v, timestamp, pirate, event)
621                 def ob1(did): ob_x(m.group(1), did); return d(did)
622                 def oba(did): return ob1('%s %s' % (did, m.group(2)))
623
624                 def disembark(v, timestamp, pirate, event):
625                         self._onboard_event(
626                                         v, timestamp, pirate, 'leaving '+event)
627                         del v[pirate]
628                         del self._pl[pirate]
629
630                 def disembark_me(why):
631                         self._disembark_myself()
632                         return d('disembark-me '+why)
633
634                 m = rm('Going aboard the (\\S.*\\S)\\.\\.\\.$')
635                 if m:
636                         dm = ['boarding']
637                         pn = self._myself.name
638                         vn = m.group(1)
639                         v = self._vessel_lookup(vn, timestamp, dm, create=True)
640                         self._lastvessel = self._vessel = vn
641                         self._v = v
642                         ob_x(pn, 'we boarded')
643                         self.expire_garbage(timestamp)
644                         return d(' '.join(dm))
645
646                 if self._v is None:
647                         return d('no vessel')
648
649                 m = rm('(\\w+) has come aboard\\.$')
650                 if m: return ob1('boarded');
651
652                 m = rm('You have ordered (\\w+) to do some (\\S.*\\S)\\.$')
653                 if m:
654                         (who,what) = m.groups()
655                         pa = ob_x(who,'ord '+what)
656                         if what == 'Gunning':
657                                 pa.gunner = True
658                         return d('duty order')
659
660                 m = rm('(\\w+) abandoned a (\\S.*\\S) station\\.$')
661                 if m: oba('stopped'); return d("end")
662
663                 def chat_core(speaker, chan):
664                         try: pa = self._pl[speaker]
665                         except KeyError: return 'mystery'
666                         if pa.v is not self._v: return 'elsewhere'
667                         pa.last_chat_time = timestamp
668                         pa.last_chat_chan = chan
669                         self.force_redisplay()
670                         return 'here'
671
672                 def chat(chan):
673                         speaker = m.group(1)
674                         dm = chat_core(speaker, chan)
675                         return d('chat %s %s' % (chan, dm))
676
677                 def chat_metacmd(chan):
678                         (cmdr, metacmd) = m.groups()
679                         metacmd = regexp.sub('\\s+', ' ', metacmd).strip()
680                         m2 = regexp.match(
681                                 '/([ad]) (?:([A-Za-z* ]+)\\s*:)?([A-Za-z ]+)$',
682                                 metacmd)
683                         if not m2: return chat(chan)
684
685                         (cmd, pattern, targets) = m2.groups()
686                         dml = ['cmd', chan, cmd]
687
688                         if cmd == 'a': each = self._onboard_event
689                         else: each = disembark
690
691                         if cmdr == self._myself.name:
692                                 dml.append('self')
693                                 how = 'cmd: %s' % cmd
694                         else:
695                                 dml.append('other')
696                                 how = 'cmd: %s %s' % (cmd,cmdr)
697
698                         v = self._find_matching_vessel(
699                                 pattern, timestamp, cmdr, dml, create=True)
700
701                         if v is not None:
702                                 targets = targets.strip().split(' ')
703                                 dml.append(`len(targets)`)
704                                 for target in targets:
705                                         each(v, timestamp, target.title(), how)
706                                 self._vessel_updated(v, timestamp)
707
708                         dm = ' '.join(dml)
709                         chat_core(cmdr, 'cmd '+chan)
710                         return d(dm)
711
712                 m = rm('(\\w+) (?:issued an order|ordered everyone) "')
713                 if m: return ob1('general order');
714
715                 m = rm('(\\w+) says, "')
716                 if m: return chat('public')
717
718                 m = rm('(\\w+) tells ye, "')
719                 if m: return chat('private')
720
721                 m = rm('Ye told (\\w+), "(.*)"$')
722                 if m: return chat_metacmd('private')
723
724                 m = rm('(\\w+) flag officer chats, "')
725                 if m: return chat('flag officer')
726
727                 m = rm('(\\w+) officer chats, "(.*)"$')
728                 if m: return chat_metacmd('officer')
729
730                 m = rm('Ye accepted the offer to job with ')
731                 if m: return disembark_me('jobbing')
732
733                 m = rm('Ye hop on the ferry and are whisked away ')
734                 if m: return disembark_me('ferry')
735
736                 m = rm('Whisking away to yer home on the magical winds')
737                 if m: return disembark_me('home')
738
739                 m = rm('Game over\\.  Winners: ([A-Za-z, ]+)\\.$')
740                 if m:
741                         pl = m.group(1).split(', ')
742                         if not self._myself.name in pl:
743                                 return d('lost melee')
744                         for pn in pl:
745                                 if ' ' in pn: continue
746                                 ob_x(pn,'won melee')
747                         return d('won melee')
748
749                 m = rm('(\\w+) is eliminated\\!')
750                 if m: return ob1('eliminated in fray');
751
752                 m = rm('(\\w+) has driven \w+ from the ship\\!')
753                 if m: return ob1('boarder repelled');
754
755                 m = rm('\w+ has bested (\\w+), and turns'+
756                         ' to the rest of the ship\\.')
757                 if m: return ob1('boarder unrepelled');
758
759                 m = rm('(\\w+) has left the vessel\.')
760                 if m:
761                         pirate = m.group(1)
762                         disembark(self._v, timestamp, pirate, 'disembarked')
763                         return d('disembarked')
764
765                 return d('not-matched')
766
767         def _str_vessel(self, vn, v):
768                 s = ' vessel %s\n' % vn
769                 s += ' '*20 + "%-*s   %13s\n" % (
770                                 max_pirate_namelen, '#lastinfo',
771                                 v['#lastinfo'])
772                 assert v['#name'] == vn
773                 for pn in sorted(v.keys()):
774                         if pn.startswith('#'): continue
775                         pa = v[pn]
776                         assert pa.v == v
777                         assert self._pl[pn] == pa
778                         s += ' '*20 + "%s %-*s %13s %-30s %13s %s\n" % (
779                                 (' ','G')[pa.gunner],
780                                 max_pirate_namelen, pn,
781                                 pa.last_time, pa.last_event,
782                                 pa.last_chat_time, pa.last_chat_chan)
783                 return s
784
785         def __str__(self):
786                 s = '''<ChatLogTracker
787  myself %s
788  vessel %s
789 '''                     % (self._myself.name, self._vessel)
790                 assert ((self._v is None and self._vessel is None) or
791                         (self._v is self._vl[self._vessel]))
792                 if self._vessel is not None:
793                         s += self._str_vessel(self._vessel, self._v)
794                 for vn in sorted(self._vl.keys()):
795                         if vn == self._vessel: continue
796                         s += self._str_vessel(vn, self._vl[vn])
797                 for p in self._pl:
798                         pa = self._pl[p]
799                         assert pa.v[p] is pa
800                         assert pa.v in self._vl.values()
801                 s += '>\n'
802                 return s
803
804         def catchup(self, progress=None):
805                 while True:
806                         more = self._f.readline()
807                         if not more: break
808
809                         self._progress[0] += len(more)
810                         if progress: progress.progress(*self._progress)
811
812                         self._lbuf += more
813                         if self._lbuf.endswith('\n'):
814                                 self.chatline(self._lbuf.rstrip())
815                                 self._lbuf = ''
816                                 if opts.debug >= 2:
817                                         debug(self.__str__())
818                 if progress: progress.caughtup()
819
820         def changed(self):
821                 rv = self._need_redisplay
822                 self._need_redisplay = False
823                 return rv
824         def myname(self):
825                 # returns our pirate name
826                 return self._myself.name
827         def vesselname(self):
828                 # returns the vessel name we're aboard or None
829                 return self._vessel
830         def lastvesselname(self):
831                 # returns the last vessel name we were aboard or None
832                 return self._lastvessel
833         def aboard(self, vesselname=True):
834                 # returns a list of PirateAboard the vessel
835                 #  sorted by pirate name
836                 #  you can pass this None and you'll get []
837                 #  or True for the current vessel (which is the default)
838                 if vesselname is True: v = self._v
839                 else: v = self._vl.get(vesselname.title())
840                 if v is None: return []
841                 return [ v[pn]
842                          for pn in sorted(v.keys())
843                          if not pn.startswith('#') ]
844
845 #---------- implementations of actual operation modes ----------
846
847 def do_pirate(pirates, bu):
848         print '{'
849         for pirate in pirates:
850                 info = PirateInfo(pirate)
851                 print '%s: %s,' % (`pirate`, info)
852         print '}'
853
854 def prep_crew_of(args, bu, max_age=300):
855         if len(args) != 1: bu('crew-of takes one pirate name')
856         pi = PirateInfo(args[0], max_age)
857         if pi.crew is None: return None
858         return CrewInfo(pi.crew[0], max_age)
859
860 def do_crew_of(args, bu):
861         ci = prep_crew_of(args, bu)
862         print ci
863
864 def do_standings_crew_of(args, bu):
865         ci = prep_crew_of(args, bu, 60)
866         tab = StandingsTable()
867         tab.headings()
868         for (rank, members) in ci.crew:
869                 if not members: continue
870                 tab.literalline('%s:' % rank)
871                 for p in members:
872                         pi = PirateInfo(p, random.randint(900,1800))
873                         tab.pirate(pi)
874         print tab.results()
875
876 class ProgressPrintPercentage:
877         def __init__(self, f=sys.stdout):
878                 self._f = f
879         def progress_string(self,done,total):
880                 return "scan chat logs %3d%%\r" % ((done*100) / total)
881         def progress(self,*a):
882                 self._f.write(self.progress_string(*a))
883                 self._f.flush()
884         def show_init(self, pirate, ocean):
885                 print >>self._f, 'Starting up, %s on the %s ocean' % (
886                         pirate, ocean)
887         def caughtup(self):
888                 self._f.write('                   \r')
889                 self._f.flush()
890
891 #----- modes which use the chat log parser are quite complex -----
892
893 def prep_chat_log(args, bu,
894                 progress=ProgressPrintPercentage(),
895                 max_myself_age=3600):
896         if len(args) != 1: bu('this action takes only chat log filename')
897         logfn = args[0]
898         logfn_re = '(?:.*/)?([A-Z][a-z]+)_([a-z]+)_'
899         match = regexp.match(logfn_re, logfn)
900         if not match: bu('chat log filename is not in expected format')
901         (pirate, ocean) = match.groups()
902         fetcher.default_ocean(ocean)
903
904         progress.show_init(pirate, fetcher.ocean)
905         myself = PirateInfo(pirate,max_myself_age)
906         track = ChatLogTracker(myself, logfn)
907
908         opts.debug -= 2
909         track.catchup(progress)
910         opts.debug += 2
911
912         track.force_redisplay()
913
914         return (myself, track)
915
916 def do_track_chat_log(args, bu):
917         (myself, track) = prep_chat_log(args, bu)
918         while True:
919                 track.catchup()
920                 if track.changed():
921                         print track
922                 sleep(1)
923
924 #----- ship management aid -----
925
926 class Display_dumb(ProgressPrintPercentage):
927         def __init__(self):
928                 ProgressPrintPercentage.__init__(self)
929         def show(self, s):
930                 print '\n\n', s;
931         def realstart(self):
932                 pass
933
934 class Display_overwrite(ProgressPrintPercentage):
935         def __init__(self):
936                 ProgressPrintPercentage.__init__(self)
937
938                 null = file('/dev/null','w')
939                 curses.setupterm(fd=null.fileno())
940
941                 self._clear = curses.tigetstr('clear')
942                 if not self._clear:
943                         self._debug('missing clear!')
944                         self.show = Display_dumb.show
945                         return
946
947                 self._t = {'el':'', 'ed':''}
948                 if not self._init_sophisticated():
949                         for k in self._t.keys(): self._t[k] = ''
950                         self._t['ho'] = self._clear
951
952         def _debug(self,m): debug('display overwrite: '+m)
953
954         def _init_sophisticated(self):
955                 for k in self._t.keys():
956                         s = curses.tigetstr(k)
957                         self._t[k] = s
958                 self._t['ho'] = curses.tigetstr('ho')
959                 if not self._t['ho']:
960                         cup = curses.tigetstr('cup')
961                         self._t['ho'] = curses.tparm(cup,0,0)
962                 missing = [k for k in self._t.keys() if not self._t[k]]
963                 if missing:
964                         self.debug('missing '+(' '.join(missing)))
965                         return 0
966                 return 1
967
968         def show(self, s):
969                 w = sys.stdout.write
970                 def wti(k): w(self._t[k])
971
972                 wti('ho')
973                 nl = ''
974                 for l in s.rstrip().split('\n'):
975                         w(nl)
976                         w(l)
977                         wti('el')
978                         nl = '\r\n'
979                 wti('ed')
980                 w(' ')
981                 sys.stdout.flush()
982
983         def realstart(self):
984                 sys.stdout.write(self._clear)
985                 sys.stdout.flush()
986                         
987
988 def do_ship_aid(args, bu):
989         if opts.ship_duty is None: opts.ship_duty = True
990
991         displayer = globals()['Display_'+opts.display]()
992         rotate_nya = '/-\\'
993
994         (myself, track) = prep_chat_log(args, bu, progress=displayer)
995
996         def timeevent(t,e):
997                 if t is None: return ' ' * 22
998                 return " %-4s %-16s" % (format_time_interval(now - t),e)
999
1000         displayer.realstart()
1001
1002         def find_vessel():
1003                 vn = track.vesselname()
1004                 if vn: return (vn, " on board the %s" % vn)
1005                 vn = track.lastvesselname()
1006                 if vn: return (vn, " ashore from the %s" % vn)
1007                 return (None, " not on a vessel")
1008
1009         displayer.show(track.myname() + find_vessel()[1] + '...')
1010
1011         while True:
1012                 track.catchup()
1013                 now = time.time()
1014
1015                 (vn, s) = find_vessel()
1016                 s = track.myname() + s
1017                 s += " at %s\n" % time.strftime("%Y-%m-%d %H:%M:%S")
1018
1019                 tbl = StandingsTable()
1020                 tbl.headings()
1021
1022                 aboard = track.aboard(vn)
1023
1024                 for pa in aboard:
1025                         pi = pa.pirate_info()
1026
1027                         xs = ''
1028                         if pa.gunner: xs += 'G '
1029                         else: xs += '  '
1030                         xs += timeevent(pa.last_time, pa.last_event)
1031                         xs += timeevent(pa.last_chat_time, pa.last_chat_chan)
1032
1033                         if pi is None:
1034                                 tbl.pirate_dummy(pa.name, rotate_nya[0], xs)
1035                         else:
1036                                 tbl.pirate(pi, xs)
1037
1038                 s += tbl.results()
1039
1040                 displayer.show(s)
1041                 sleep(1)
1042                 rotate_nya = rotate_nya[1:3] + rotate_nya[0]
1043
1044 #---------- main program ----------
1045
1046 def main():
1047         global opts, fetcher
1048
1049         pa = OptionParser(
1050 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
1051 actions:
1052  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
1053  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
1054  yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
1055  yoweb-scrape [--ocean OCEAN ...] track-chat-log CHAT-LOG
1056  yoweb-scrape [options] ship-aid CHAT-LOG  (must be .../PIRATE_OCEAN_chat-log*)
1057
1058 display modes (for --display) apply to ship-aid:
1059  --display=dumb       just print new information, scrolling the screen
1060  --display=overwrite  use cursor motion, selective clear, etc. to redraw at top
1061 ''')
1062         ao = pa.add_option
1063         ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
1064                 help='select ocean OCEAN')
1065         ao('--cache-dir', dest='cache_dir', metavar='DIR',
1066                 default='~/.yoweb-scrape-cache',
1067                 help='cache yoweb pages in DIR')
1068         ao('-D','--debug', action='count', dest='debug', default=0,
1069                 help='enable debugging output')
1070         ao('--debug-fd', type='int', dest='debug_fd',
1071                 help='write any debugging output to specified fd')
1072         ao('-q','--quiet', action='store_true', dest='quiet',
1073                 help='suppress warning output')
1074         ao('--display', action='store', dest='display',
1075                 type='choice', choices=['dumb','overwrite'],
1076                 help='how to display ship aid')
1077
1078         ao('--ship-duty', action='store_true', dest='ship_duty',
1079                 help='show ship duty station puzzles')
1080         ao('--all-puzzles', action='store_false', dest='ship_duty',
1081                 help='show all puzzles, not just ship duty stations')
1082
1083         (opts,args) = pa.parse_args()
1084         random.seed()
1085
1086         if len(args) < 1:
1087                 pa.error('need a mode argument')
1088
1089         if opts.debug_fd is not None:
1090                 opts.debug_file = os.fdopen(opts.debug_fd, 'w')
1091         else:
1092                 opts.debug_file = sys.stdout
1093
1094         mode = args[0]
1095         mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
1096         try: mode_fn = globals()[mode_fn_name]
1097         except KeyError: pa.error('unknown mode "%s"' % mode)
1098
1099         # fixed parameters
1100         opts.min_max_age = 60
1101         opts.expire_age = 3600
1102         opts.ship_reboard_clearout = 3600
1103
1104         if opts.cache_dir.startswith('~/'):
1105                 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
1106
1107         if opts.display is None:
1108                 if ((opts.debug > 0 and opts.debug_fd is None)
1109                     or not os.isatty(sys.stdout.fileno())):
1110                         opts.display = 'dumb'
1111                 else:
1112                         opts.display = 'overwrite'
1113
1114         fetcher = Fetcher(opts.ocean, opts.cache_dir)
1115
1116         mode_fn(args[1:], pa.error)
1117
1118 main()