4 signal.signal(signal.SIGINT, signal.SIG_DFL)
13 from optparse import OptionParser
15 from BeautifulSoup import BeautifulSoup
20 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
21 '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
22 '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
23 '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
25 standingvals = ('Able/Distinguished/Respected/Master'+
26 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
28 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
30 max_pirate_namelen = 20
38 def __init__(self, ocean, cachedir):
39 debug('Fetcher init %s' % cachedir)
41 self.cachedir = cachedir
42 try: os.mkdir(cachedir)
43 except (OSError,IOError), oe:
44 if oe.errno != errno.EEXIST: raise
45 self._cache_scan(time.time())
47 def _cache_scan(self, now):
48 # returns list of ages, unsorted
50 debug('Fetcher scan_cache')
51 for leaf in os.listdir(self.cachedir):
52 if not leaf.startswith('#'): continue
53 path = self.cachedir + '/' + leaf
54 try: s = os.stat(path)
55 except (OSError,IOError), oe:
56 if oe.errno != errno.ENOENT: raise
58 age = now - s.st_mtime
59 if age > opts.expire_age:
60 debug('Fetcher expire %d %s' % (age, path))
62 except (OSError,IOError), oe:
63 if oe.errno != errno.ENOENT: raise
68 def _rate_limit_cache_clean(self, now):
69 ages = self._cache_scan(now)
71 debug('Fetcher ages ' + `ages`)
75 if age < min_age and age < 300:
76 debug('Fetcher morewait min=%d age=%d' %
78 need_wait = max(need_wait, min_age - age)
82 debug('Fetcher wait %d' % need_wait)
85 def fetch(self, url, max_age):
86 debug('Fetcher fetch %s' % url)
87 cache_corename = urllib.quote_plus(url)
88 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
89 try: f = file(cache_item, 'r')
90 except (OSError,IOError), oe:
91 if oe.errno != errno.ENOENT: raise
94 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
96 s = os.fstat(f.fileno())
97 age = now - s.st_mtime
99 debug('Fetcher stale %d < %d'% (max_age, age))
104 debug('Fetcher cached %d > %d' % (max_age, age))
107 debug('Fetcher fetch')
108 self._rate_limit_cache_clean(now)
110 stream = urllib2.urlopen(url)
112 cache_tmp = "%s/#%s~%d#" % (
113 self.cachedir, cache_corename, os.getpid())
114 f = file(cache_tmp, 'w')
117 os.rename(cache_tmp, cache_item)
118 debug('Fetcher stored')
121 def yoweb(self, kind, tail, max_age):
122 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
123 self.ocean, kind, tail)
124 return self.fetch(url, max_age)
131 def soupm(self, obj, m):
132 self.msg(m + '; in ' + `obj`)
133 def needs_msgs(self, child_souplog):
134 self.msgs += child_souplog.msgs
135 child_souplog.msgs = [ ]
138 str = ''.join(obj.findAll(text=True))
141 class SomethingSoupInfo(SoupLog):
142 def __init__(self, kind, tail, max_age):
143 SoupLog.__init__(self)
144 html = fetcher.yoweb(kind, tail, max_age)
145 self._soup = BeautifulSoup(html,
146 convertEntities=BeautifulSoup.HTML_ENTITIES
149 class PirateInfo(SomethingSoupInfo):
150 # Public data members:
151 # pi.standings = { 'Treasure Haul': 'Able' ... }
153 # pi.crew = (id, name)
154 # pi.flag = (id, name)
155 # pi.msgs = [ 'message describing problem with scrape' ]
157 def __init__(self, pirate, max_age=300):
158 SomethingSoupInfo.__init__(self,
159 'pirate.wm?target=', pirate, max_age)
161 self._find_standings()
162 self.crew = self._find_crewflag('crew',
163 '^/yoweb/crew/info\\.wm')
164 self.flag = self._find_crewflag('flag',
165 '^/yoweb/flag/info\\.wm')
167 def _find_standings(self):
168 imgs = self._soup.findAll('img',
169 src=regexp.compile('/yoweb/images/stat.*'))
171 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
175 for skill in puzzles:
176 standings[skill] = [ ]
181 try: puzzle = img['alt']
182 except KeyError: continue
184 if not puzzle in puzzles:
185 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
187 key = img.findParent('td')
189 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
191 valelem = key.findNextSibling('td')
193 skl.soupm(key, 'puzzle missing sibling "%s"'
196 valstr = soup_text(valelem)
197 match = re.match(valstr)
199 skl.soupm(key, ('puzzle "%s" unparseable'+
200 ' standing "%s"') % (puzzle, valstr))
202 standing = match.group(match.lastindex)
203 standings[puzzle].append(standing)
207 for puzzle in puzzles:
208 sl = standings[puzzle]
210 skl.msg('puzzle "%s" multiple standings %s' %
214 skl.msg('puzzle "%s" no standing found' % puzzle)
217 for i in range(0, len(standingvals)-1):
218 if standing == standingvals[i]:
219 self.standings[puzzle] = i
220 if not puzzle in self.standings:
221 skl.msg('puzzle "%s" unknown standing "%s"' %
224 all_standings_ok = True
225 for puzzle in puzzles:
226 if not puzzle in self.standings:
229 def _find_crewflag(self, cf, yoweb_re):
230 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
232 self.msg('zero or several %s id references found' % cf)
235 id_re = '\\b%sid\\=(\\w+)$' % cf
236 id_haystack = thing['href']
237 match = regexp.compile(id_re).search(id_haystack)
239 self.soupm(thing, ('incomprehensible %s id ref'+
240 ' (%s in %s)') % (cf, id_re, id_haystack))
242 name = soup_text(thing)
243 return (match.group(1), name)
246 return `(self.crew, self.flag, self.standings, self.msgs)`
248 class CrewInfo(SomethingSoupInfo):
249 # Public data members:
250 # ci.crew = [ ('Captain', ['Pirate', ...]),
251 # ('Senior Officer', [...]),
253 # pi.msgs = [ 'message describing problem with scrape' ]
255 def __init__(self, crewid, max_age=300):
256 SomethingSoupInfo.__init__(self,
257 'crew/info.wm?crewid=', crewid, max_age)
260 def _find_crew(self):
262 capts = self._soup.findAll('img',
263 src='/yoweb/images/crew-captain.png')
265 self.msg('crew members: no. of captain images != 1')
268 while not tbl.find('a', href=pirate_ref_re):
269 tbl = tbl.findParent('table')
271 self.msg('crew members: cannot find table')
273 current_rank_crew = None
274 crew_rank_re = regexp.compile('/yoweb/images/crew')
275 for row in tbl.contents:
276 # findAll(recurse=False)
277 if isinstance(row, unicode):
280 is_rank = row.find('img', attrs={'src': crew_rank_re})
282 rank = soup_text(row)
283 current_rank_crew = []
284 self.crew.append((rank, current_rank_crew))
286 for cell in row.findAll('a', href=pirate_ref_re):
287 if current_rank_crew is None:
288 self.soupm(cell, 'crew members: crew'
291 current_rank_crew.append(soup_text(cell))
294 return `(self.crew, self.msgs)`
296 class StandingsTable:
297 def __init__(self, use_puzzles=None, col_width=6):
298 if use_puzzles is None:
301 'Navigating','Battle Navigation',
303 ['Sailing','Rigging'],
310 self._puzzles = use_puzzles
312 self._cw = col_width-1
314 def _pline(self, pirate, puzstrs):
315 self.s += ' %-*s' % (max_pirate_namelen, pirate)
317 self.s += ' %-*.*s' % (self._cw,self._cw, v)
320 def _puzstr(self, pi, puzzle):
321 if not isinstance(puzzle,list): puzzle = [puzzle]
322 try: standing = max([pi.standings[p] for p in puzzle])
323 except KeyError: return '?'
324 if not standing: return ''
327 c1 = standingvals[standing][0]
328 if standing < 3: c1 = c1.lower() # 3 = Master
332 s += '*' * (standing / 2)
333 s += '+' * (standing % 2)
337 def puzn_redact(name):
338 if isinstance(name,list):
340 ["%.*s" % (self._cw/2, puzn_redact(n))
343 if spc < 0: return name
344 return name[0:min(4,spc)] + name[spc+1:]
345 self._pline('', map(puzn_redact, self._puzzles))
346 def literalline(self, line):
347 self.s += line + '\n'
348 def pirate(self, pi):
349 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
350 self._pline(pi.name, puzstrs)
355 def do_pirate(pirates, bu):
357 for pirate in pirates:
358 info = PirateInfo(pirate)
359 print '%s: %s,' % (`pirate`, info)
362 def prep_crew_of(args, bu, max_age=300):
363 if len(args) != 1: bu('crew-of takes one pirate name')
364 pi = PirateInfo(args[0], max_age)
365 return CrewInfo(pi.crew[0], max_age)
367 def do_crew_of(args, bu):
368 ci = prep_crew_of(args, bu)
371 def do_standings_crew_of(args, bu):
372 ci = prep_crew_of(args, bu, 60)
373 tab = StandingsTable()
375 for (rank, members) in ci.crew:
376 if not members: continue
377 tab.literalline('%s:' % rank)
379 pi = PirateInfo(p, 1800)
387 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
389 yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
390 yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
391 yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
394 ao('-O','--ocean',dest='ocean', metavar='OCEAN',
396 help='select ocean OCEAN')
397 ao('--cache-dir', dest='cache_dir', metavar='DIR',
398 default='~/.yoweb-scrape-cache',
399 help='cache yoweb pages in DIR')
400 ao('-D','--debug', action='store_true', dest='debug', default=False,
401 help='enable debugging output')
402 ao('-q','--quiet', action='store_true', dest='quiet',
403 help='suppress warning output')
405 ao('--ship-duty', action='store_true', dest='ship_duty',
406 help='show ship duty station puzzles')
408 (opts,args) = pa.parse_args()
411 pa.error('need a mode argument')
414 mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
415 try: mode_fn = globals()[mode_fn_name]
416 except KeyError: pa.error('unknown mode "%s"' % mode)
419 opts.min_max_age = 60
420 opts.expire_age = 3600
422 if opts.cache_dir.startswith('~/'):
423 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
425 fetcher = Fetcher(opts.ocean, opts.cache_dir)
427 mode_fn(args[1:], pa.error)