chiark / gitweb /
65f949b602bbb623baeb257bda618e5c07f149f8
[ypp-sc-tools.db-live.git] / yoweb-scrape
1 #!/usr/bin/python
2
3 import signal
4 signal.signal(signal.SIGINT, signal.SIG_DFL)
5
6 import os
7 import time
8 import urllib
9 import urllib2
10 import errno
11 import sys
12 import re as regexp
13 import random
14 from optparse import OptionParser
15
16 from BeautifulSoup import BeautifulSoup
17
18 opts = None
19
20
21 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
22         '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
23         '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
24         '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
25
26 standingvals = ('Able/Distinguished/Respected/Master'+
27                 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
28
29 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
30
31 max_pirate_namelen = 12
32
33
34 def debug(m):
35         if opts.debug:
36                 print >>sys.stderr, m
37
38 class Fetcher:
39         def __init__(self, ocean, cachedir):
40                 debug('Fetcher init %s' % cachedir)
41                 self.ocean = ocean
42                 self.cachedir = cachedir
43                 try: os.mkdir(cachedir)
44                 except (OSError,IOError), oe:
45                         if oe.errno != errno.EEXIST: raise
46                 self._cache_scan(time.time())
47
48         def _default_ocean(self):
49                 if self.ocean is None:
50                         self.ocean = 'ice'
51
52         def _cache_scan(self, now):
53                 # returns list of ages, unsorted
54                 ages = []
55                 debug('Fetcher   scan_cache')
56                 for leaf in os.listdir(self.cachedir):
57                         if not leaf.startswith('#'): continue
58                         path = self.cachedir + '/' + leaf
59                         try: s = os.stat(path)
60                         except (OSError,IOError), oe:
61                                 if oe.errno != errno.ENOENT: raise
62                                 continue
63                         age = now - s.st_mtime
64                         if age > opts.expire_age:
65                                 debug('Fetcher    expire %d %s' % (age, path))
66                                 try: os.remove(path)
67                                 except (OSError,IOError), oe:
68                                         if oe.errno != errno.ENOENT: raise
69                                 continue
70                         ages.append(age)
71                 return ages
72
73         def _rate_limit_cache_clean(self, now):
74                 ages = self._cache_scan(now)
75                 ages.sort()
76                 debug('Fetcher   ages ' + `ages`)
77                 min_age = 1
78                 need_wait = 0
79                 for age in ages:
80                         if age < min_age and age < 300:
81                                 debug('Fetcher   morewait min=%d age=%d' %
82                                         (min_age, age))
83                                 need_wait = max(need_wait, min_age - age)
84                         min_age += 3
85                         min_age *= 1.25
86                 if need_wait > 0:
87                         debug('Fetcher   wait %d' % need_wait)
88                         time.sleep(need_wait)
89
90         def fetch(self, url, max_age):
91                 debug('Fetcher fetch %s' % url)
92                 cache_corename = urllib.quote_plus(url)
93                 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
94                 try: f = file(cache_item, 'r')
95                 except (OSError,IOError), oe:
96                         if oe.errno != errno.ENOENT: raise
97                         f = None
98                 now = time.time()
99                 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
100                 if f is not None:
101                         s = os.fstat(f.fileno())
102                         age = now - s.st_mtime
103                         if age > max_age:
104                                 debug('Fetcher  stale %d < %d'% (max_age, age))
105                                 f = None
106                 if f is not None:
107                         data = f.read()
108                         f.close()
109                         debug('Fetcher  cached %d > %d' % (max_age, age))
110                         return data
111
112                 debug('Fetcher  fetch')
113                 self._rate_limit_cache_clean(now)
114
115                 stream = urllib2.urlopen(url)
116                 data = stream.read()
117                 cache_tmp = "%s/#%s~%d#" % (
118                         self.cachedir, cache_corename, os.getpid())
119                 f = file(cache_tmp, 'w')
120                 f.write(data)
121                 f.close()
122                 os.rename(cache_tmp, cache_item)
123                 debug('Fetcher  stored')
124                 return data
125
126         def yoweb(self, kind, tail, max_age):
127                 self._default_ocean()
128                 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
129                         self.ocean, kind, tail)
130                 return self.fetch(url, max_age)
131
132 class SoupLog:
133         def __init__(self):
134                 self.msgs = [ ]
135         def msg(self, m):
136                 self.msgs.append(m)
137         def soupm(self, obj, m):
138                 self.msg(m + '; in ' + `obj`)
139         def needs_msgs(self, child_souplog):
140                 self.msgs += child_souplog.msgs
141                 child_souplog.msgs = [ ]
142
143 def soup_text(obj):
144         str = ''.join(obj.findAll(text=True))
145         return str.strip()
146
147 class SomethingSoupInfo(SoupLog):
148         def __init__(self, kind, tail, max_age):
149                 SoupLog.__init__(self)
150                 html = fetcher.yoweb(kind, tail, max_age)
151                 self._soup = BeautifulSoup(html,
152                         convertEntities=BeautifulSoup.HTML_ENTITIES
153                         )
154
155 class PirateInfo(SomethingSoupInfo):
156         # Public data members:
157         #  pi.standings = { 'Treasure Haul': 'Able' ... }
158         #  pi.name = name
159         #  pi.crew = (id, name)
160         #  pi.flag = (id, name)
161         #  pi.msgs = [ 'message describing problem with scrape' ]
162                 
163         def __init__(self, pirate, max_age=300):
164                 SomethingSoupInfo.__init__(self,
165                         'pirate.wm?target=', pirate, max_age)
166                 self.name = pirate
167                 self._find_standings()
168                 self.crew = self._find_crewflag('crew',
169                         '^/yoweb/crew/info\\.wm')
170                 self.flag = self._find_crewflag('flag',
171                         '^/yoweb/flag/info\\.wm')
172
173         def _find_standings(self):
174                 imgs = self._soup.findAll('img',
175                         src=regexp.compile('/yoweb/images/stat.*'))
176                 re = regexp.compile(
177 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
178                         )
179                 standings = { }
180
181                 for skill in puzzles:
182                         standings[skill] = [ ]
183
184                 skl = SoupLog()
185
186                 for img in imgs:
187                         try: puzzle = img['alt']
188                         except KeyError: continue
189
190                         if not puzzle in puzzles:
191                                 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
192                                 continue
193                         key = img.findParent('td')
194                         if key is None:
195                                 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
196                                 continue
197                         valelem = key.findNextSibling('td')
198                         if valelem is None:
199                                 skl.soupm(key, 'puzzle missing sibling "%s"'
200                                         % puzzle)
201                                 continue
202                         valstr = soup_text(valelem)
203                         match = re.match(valstr)
204                         if match is None:
205                                 skl.soupm(key, ('puzzle "%s" unparseable'+
206                                         ' standing "%s"') % (puzzle, valstr))
207                                 continue
208                         standing = match.group(match.lastindex)
209                         standings[puzzle].append(standing)
210
211                 self.standings = { }
212
213                 for puzzle in puzzles:
214                         sl = standings[puzzle]
215                         if len(sl) > 1:
216                                 skl.msg('puzzle "%s" multiple standings %s' %
217                                                 (puzzle, `sl`))
218                                 continue
219                         if not len(sl):
220                                 skl.msg('puzzle "%s" no standing found' % puzzle)
221                                 continue
222                         standing = sl[0]
223                         for i in range(0, len(standingvals)-1):
224                                 if standing == standingvals[i]:
225                                         self.standings[puzzle] = i
226                         if not puzzle in self.standings:
227                                 skl.msg('puzzle "%s" unknown standing "%s"' %
228                                         (puzzle, standing))
229
230                 all_standings_ok = True
231                 for puzzle in puzzles:
232                         if not puzzle in self.standings:
233                                 self.needs_msgs(skl)
234
235         def _find_crewflag(self, cf, yoweb_re):
236                 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
237                 if len(things) != 1:
238                         self.msg('zero or several %s id references found' % cf)
239                         return None
240                 thing = things[0]
241                 id_re = '\\b%sid\\=(\\w+)$' % cf
242                 id_haystack = thing['href']
243                 match = regexp.compile(id_re).search(id_haystack)
244                 if match is None:
245                         self.soupm(thing, ('incomprehensible %s id ref'+
246                                 ' (%s in %s)') % (cf, id_re, id_haystack))
247                         return None
248                 name = soup_text(thing)
249                 return (match.group(1), name)
250
251         def __str__(self):
252                 return `(self.crew, self.flag, self.standings, self.msgs)`
253
254 class CrewInfo(SomethingSoupInfo):
255         # Public data members:
256         #  ci.crew = [ ('Captain',        ['Pirate', ...]),
257         #              ('Senior Officer', [...]),
258         #               ... ]
259         #  pi.msgs = [ 'message describing problem with scrape' ]
260
261         def __init__(self, crewid, max_age=300):
262                 SomethingSoupInfo.__init__(self,
263                         'crew/info.wm?crewid=', crewid, max_age)
264                 self._find_crew()
265
266         def _find_crew(self):
267                 self.crew = []
268                 capts = self._soup.findAll('img',
269                         src='/yoweb/images/crew-captain.png')
270                 if len(capts) != 1:
271                         self.msg('crew members: no. of captain images != 1')
272                         return
273                 tbl = capts[0]
274                 while not tbl.find('a', href=pirate_ref_re):
275                         tbl = tbl.findParent('table')
276                         if not tbl:
277                                 self.msg('crew members: cannot find table')
278                                 return
279                 current_rank_crew = None
280                 crew_rank_re = regexp.compile('/yoweb/images/crew')
281                 for row in tbl.contents:
282                         # findAll(recurse=False)
283                         if isinstance(row, unicode):
284                                 continue
285
286                         is_rank = row.find('img', attrs={'src': crew_rank_re})
287                         if is_rank:
288                                 rank = soup_text(row)
289                                 current_rank_crew = []
290                                 self.crew.append((rank, current_rank_crew))
291                                 continue
292                         for cell in row.findAll('a', href=pirate_ref_re):
293                                 if current_rank_crew is None:
294                                         self.soupm(cell, 'crew members: crew'
295                                                 ' before rank')
296                                         continue
297                                 current_rank_crew.append(soup_text(cell))
298
299         def __str__(self):
300                 return `(self.crew, self.msgs)`
301
302 class StandingsTable:
303         def __init__(self, use_puzzles=None, col_width=6):
304                 if use_puzzles is None:
305                         if opts.ship_duty:
306                                 use_puzzles=[
307                                         'Navigating','Battle Navigation',
308                                         'Gunning',
309                                         ['Sailing','Rigging'],
310                                         'Bilging',
311                                         'Carpentry',
312                                         'Treasure Haul'
313                                 ]
314                         else:
315                                 use_puzzles=puzzles
316                 self._puzzles = use_puzzles
317                 self.s = ''
318                 self._cw = col_width-1
319
320         def _pline(self, pirate, puzstrs):
321                 self.s += ' %-*s' % (max_pirate_namelen, pirate)
322                 for v in puzstrs:
323                         self.s += ' %-*.*s' % (self._cw,self._cw, v)
324                 self.s += '\n'
325
326         def _puzstr(self, pi, puzzle):
327                 if not isinstance(puzzle,list): puzzle = [puzzle]
328                 try: standing = max([pi.standings[p] for p in puzzle])
329                 except KeyError: return '?'
330                 if not standing: return ''
331                 s = ''
332                 if self._cw > 4:
333                         c1 = standingvals[standing][0]
334                         if standing < 3: c1 = c1.lower() # 3 = Master
335                         s += `standing`
336                 if self._cw > 5:
337                         s += ' '
338                 s += '*' * (standing / 2)
339                 s += '+' * (standing % 2)
340                 return s
341
342         def headings(self):
343                 def puzn_redact(name):
344                         if isinstance(name,list):
345                                 return '/'.join(
346                                         ["%.*s" % (self._cw/2, puzn_redact(n))
347                                          for n in name])
348                         spc = name.find(' ')
349                         if spc < 0: return name
350                         return name[0:min(4,spc)] + name[spc+1:]
351                 self._pline('', map(puzn_redact, self._puzzles))
352         def literalline(self, line):
353                 self.s += line + '\n'
354         def pirate(self, pi):
355                 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
356                 self._pline(pi.name, puzstrs)
357
358         def results(self):
359                 return self.s
360
361 def do_pirate(pirates, bu):
362         print '{'
363         for pirate in pirates:
364                 info = PirateInfo(pirate)
365                 print '%s: %s,' % (`pirate`, info)
366         print '}'
367
368 def prep_crew_of(args, bu, max_age=300):
369         if len(args) != 1: bu('crew-of takes one pirate name')
370         pi = PirateInfo(args[0], max_age)
371         return CrewInfo(pi.crew[0], max_age)
372
373 def do_crew_of(args, bu):
374         ci = prep_crew_of(args, bu)
375         print ci
376
377 def do_standings_crew_of(args, bu):
378         ci = prep_crew_of(args, bu, 60)
379         tab = StandingsTable()
380         tab.headings()
381         for (rank, members) in ci.crew:
382                 if not members: continue
383                 tab.literalline('%s:' % rank)
384                 for p in members:
385                         pi = PirateInfo(p, random.randint(900,1800))
386                         tab.pirate(pi)
387         print tab.results()
388
389 def main():
390         global opts, fetcher
391
392         pa = OptionParser(
393 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
394 actions:
395  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
396  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
397  yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
398  yoweb-scrape [--ocean OCEAN ...] ship-aid CHAT-LOG
399 ''')
400         ao = pa.add_option
401         ao('-O','--ocean',dest='ocean', metavar='OCEAN', default=None,
402                 help='select ocean OCEAN')
403         ao('--cache-dir', dest='cache_dir', metavar='DIR',
404                 default='~/.yoweb-scrape-cache',
405                 help='cache yoweb pages in DIR')
406         ao('-D','--debug', action='store_true', dest='debug', default=False,
407                 help='enable debugging output')
408         ao('-q','--quiet', action='store_true', dest='quiet',
409                 help='suppress warning output')
410
411         ao('--ship-duty', action='store_true', dest='ship_duty',
412                 help='show ship duty station puzzles')
413
414         (opts,args) = pa.parse_args()
415         random.seed()
416
417         if len(args) < 1:
418                 pa.error('need a mode argument')
419
420         mode = args[0]
421         mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
422         try: mode_fn = globals()[mode_fn_name]
423         except KeyError: pa.error('unknown mode "%s"' % mode)
424
425         # fixed parameters
426         opts.min_max_age = 60
427         opts.expire_age = 3600
428
429         if opts.cache_dir.startswith('~/'):
430                 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
431
432         fetcher = Fetcher(opts.ocean, opts.cache_dir)
433
434         mode_fn(args[1:], pa.error)
435
436 main()