yoweb-scrape

   1 #!/usr/bin/python
   2
   3 import signal
   4 signal.signal(signal.SIGINT, signal.SIG_DFL)
   5
   6 import os
   7 import time
   8 import urllib
   9 import urllib2
  10 import errno
  11 import sys
  12 import re as regexp
  13 from optparse import OptionParser
  14
  15 from BeautifulSoup import BeautifulSoup
  16
  17 opts = None
  18
  19
  20 puzzles = ('Swordfighting/Bilging/Sailing/Rigging/Navigating'+
  21         '/Battle Navigation/Gunning/Carpentry/Rumble/Treasure Haul'+
  22         '/Drinking/Spades/Hearts/Treasure Drop/Poker/Distilling'+
  23         '/Alchemistry/Shipwrightery/Blacksmithing/Foraging').split('/')
  24
  25 standingvals = ('Able/Distinguished/Respected/Master'+
  26                 '/Renowned/Grand-Master/Legendary/Ultimate').split('/')
  27
  28 pirate_ref_re = regexp.compile('^/yoweb/pirate\\.wm')
  29
  30 max_pirate_namelen = 20
  31
  32
  33 def debug(m):
  34         if opts.debug:
  35                 print >>sys.stderr, m
  36
  37 class Fetcher:
  38         def __init__(self, ocean, cachedir):
  39                 debug('Fetcher init %s' % cachedir)
  40                 self.ocean = ocean
  41                 self.cachedir = cachedir
  42                 try: os.mkdir(cachedir)
  43                 except (OSError,IOError), oe:
  44                         if oe.errno != errno.EEXIST: raise
  45                 self._cache_scan(time.time())
  46
  47         def _cache_scan(self, now):
  48                 # returns list of ages, unsorted
  49                 ages = []
  50                 debug('Fetcher   scan_cache')
  51                 for leaf in os.listdir(self.cachedir):
  52                         if not leaf.startswith('#'): continue
  53                         path = self.cachedir + '/' + leaf
  54                         try: s = os.stat(path)
  55                         except (OSError,IOError), oe:
  56                                 if oe.errno != errno.ENOENT: raise
  57                                 continue
  58                         age = now - s.st_mtime
  59                         if age > opts.expire_age:
  60                                 debug('Fetcher    expire %d %s' % (age, path))
  61                                 try: os.remove(path)
  62                                 except (OSError,IOError), oe:
  63                                         if oe.errno != errno.ENOENT: raise
  64                                 continue
  65                         ages.append(age)
  66                 return ages
  67
  68         def _rate_limit_cache_clean(self, now):
  69                 ages = self._cache_scan(now)
  70                 ages.sort()
  71                 debug('Fetcher   ages ' + `ages`)
  72                 min_age = 1
  73                 need_wait = 0
  74                 for age in ages:
  75                         if age < min_age and age < 300:
  76                                 debug('Fetcher   morewait min=%d age=%d' %
  77                                         (min_age, age))
  78                                 need_wait = max(need_wait, min_age - age)
  79                         min_age += 3
  80                         min_age *= 1.25
  81                 if need_wait > 0:
  82                         debug('Fetcher   wait %d' % need_wait)
  83                         time.sleep(need_wait)
  84
  85         def fetch(self, url, max_age):
  86                 debug('Fetcher fetch %s' % url)
  87                 cache_corename = urllib.quote_plus(url)
  88                 cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
  89                 try: f = file(cache_item, 'r')
  90                 except (OSError,IOError), oe:
  91                         if oe.errno != errno.ENOENT: raise
  92                         f = None
  93                 now = time.time()
  94                 max_age = max(opts.min_max_age, min(max_age, opts.expire_age))
  95                 if f is not None:
  96                         s = os.fstat(f.fileno())
  97                         age = now - s.st_mtime
  98                         if age > max_age:
  99                                 debug('Fetcher  stale %d < %d'% (max_age, age))
 100                                 f = None
 101                 if f is not None:
 102                         data = f.read()
 103                         f.close()
 104                         debug('Fetcher  cached %d > %d' % (max_age, age))
 105                         return data
 106
 107                 debug('Fetcher  fetch')
 108                 self._rate_limit_cache_clean(now)
 109
 110                 stream = urllib2.urlopen(url)
 111                 data = stream.read()
 112                 cache_tmp = "%s/#%s~%d#" % (
 113                         self.cachedir, cache_corename, os.getpid())
 114                 f = file(cache_tmp, 'w')
 115                 f.write(data)
 116                 f.close()
 117                 os.rename(cache_tmp, cache_item)
 118                 debug('Fetcher  stored')
 119                 return data
 120
 121         def yoweb(self, kind, tail, max_age):
 122                 url = 'http://%s.puzzlepirates.com/yoweb/%s%s' % (
 123                         self.ocean, kind, tail)
 124                 return self.fetch(url, max_age)
 125
 126 class SoupLog:
 127         def __init__(self):
 128                 self.msgs = [ ]
 129         def msg(self, m):
 130                 self.msgs.append(m)
 131         def soupm(self, obj, m):
 132                 self.msg(m + '; in ' + `obj`)
 133         def needs_msgs(self, child_souplog):
 134                 self.msgs += child_souplog.msgs
 135                 child_souplog.msgs = [ ]
 136
 137 def soup_text(obj):
 138         str = ''.join(obj.findAll(text=True))
 139         return str.strip()
 140
 141 class SomethingSoupInfo(SoupLog):
 142         def __init__(self, kind, tail, max_age):
 143                 SoupLog.__init__(self)
 144                 html = fetcher.yoweb(kind, tail, max_age)
 145                 self._soup = BeautifulSoup(html,
 146                         convertEntities=BeautifulSoup.HTML_ENTITIES
 147                         )
 148
 149 class PirateInfo(SomethingSoupInfo):
 150         # Public data members:
 151         #  pi.standings = { 'Treasure Haul': 'Able' ... }
 152         #  pi.name = name
 153         #  pi.crew = (id, name)
 154         #  pi.flag = (id, name)
 155         #  pi.msgs = [ 'message describing problem with scrape' ]
 156
 157         def __init__(self, pirate, max_age=300):
 158                 SomethingSoupInfo.__init__(self,
 159                         'pirate.wm?target=', pirate, max_age)
 160                 self.name = pirate
 161                 self._find_standings()
 162                 self.crew = self._find_crewflag('crew',
 163                         '^/yoweb/crew/info\\.wm')
 164                 self.flag = self._find_crewflag('flag',
 165                         '^/yoweb/flag/info\\.wm')
 166
 167         def _find_standings(self):
 168                 imgs = self._soup.findAll('img',
 169                         src=regexp.compile('/yoweb/images/stat.*'))
 170                 re = regexp.compile(
 171 u'\\s*\\S*/([-A-Za-z]+)\\s*$|\\s*\\S*/\\S*\\s*\\(ocean\\-wide(?:\\s|\\xa0)+([-A-Za-z]+)\\)\\s*$'
 172                         )
 173                 standings = { }
 174
 175                 for skill in puzzles:
 176                         standings[skill] = [ ]
 177
 178                 skl = SoupLog()
 179
 180                 for img in imgs:
 181                         try: puzzle = img['alt']
 182                         except KeyError: continue
 183
 184                         if not puzzle in puzzles:
 185                                 skl.soupm(img, 'unknown puzzle: "%s"' % puzzle)
 186                                 continue
 187                         key = img.findParent('td')
 188                         if key is None:
 189                                 skl.soupm(img, 'puzzle at root! "%s"' % puzzle)
 190                                 continue
 191                         valelem = key.findNextSibling('td')
 192                         if valelem is None:
 193                                 skl.soupm(key, 'puzzle missing sibling "%s"'
 194                                         % puzzle)
 195                                 continue
 196                         valstr = soup_text(valelem)
 197                         match = re.match(valstr)
 198                         if match is None:
 199                                 skl.soupm(key, ('puzzle "%s" unparseable'+
 200                                         ' standing "%s"') % (puzzle, valstr))
 201                                 continue
 202                         standing = match.group(match.lastindex)
 203                         standings[puzzle].append(standing)
 204
 205                 self.standings = { }
 206
 207                 for puzzle in puzzles:
 208                         sl = standings[puzzle]
 209                         if len(sl) > 1:
 210                                 skl.msg('puzzle "%s" multiple standings %s' %
 211                                                 (puzzle, `sl`))
 212                                 continue
 213                         if not len(sl):
 214                                 skl.msg('puzzle "%s" no standing found' % puzzle)
 215                                 continue
 216                         standing = sl[0]
 217                         for i in range(0, len(standingvals)-1):
 218                                 if standing == standingvals[i]:
 219                                         self.standings[puzzle] = i
 220                         if not puzzle in self.standings:
 221                                 skl.msg('puzzle "%s" unknown standing "%s"' %
 222                                         (puzzle, standing))
 223
 224                 all_standings_ok = True
 225                 for puzzle in puzzles:
 226                         if not puzzle in self.standings:
 227                                 self.needs_msgs(skl)
 228
 229         def _find_crewflag(self, cf, yoweb_re):
 230                 things = self._soup.findAll('a', href=regexp.compile(yoweb_re))
 231                 if len(things) != 1:
 232                         self.msg('zero or several %s id references found' % cf)
 233                         return None
 234                 thing = things[0]
 235                 id_re = '\\b%sid\\=(\\w+)$' % cf
 236                 id_haystack = thing['href']
 237                 match = regexp.compile(id_re).search(id_haystack)
 238                 if match is None:
 239                         self.soupm(thing, ('incomprehensible %s id ref'+
 240                                 ' (%s in %s)') % (cf, id_re, id_haystack))
 241                         return None
 242                 name = soup_text(thing)
 243                 return (match.group(1), name)
 244
 245         def __str__(self):
 246                 return `(self.crew, self.flag, self.standings, self.msgs)`
 247
 248 class CrewInfo(SomethingSoupInfo):
 249         # Public data members:
 250         #  ci.crew = [ ('Captain',        ['Pirate', ...]),
 251         #              ('Senior Officer', [...]),
 252         #               ... ]
 253         #  pi.msgs = [ 'message describing problem with scrape' ]
 254
 255         def __init__(self, crewid, max_age=300):
 256                 SomethingSoupInfo.__init__(self,
 257                         'crew/info.wm?crewid=', crewid, max_age)
 258                 self._find_crew()
 259
 260         def _find_crew(self):
 261                 self.crew = []
 262                 capts = self._soup.findAll('img',
 263                         src='/yoweb/images/crew-captain.png')
 264                 if len(capts) != 1:
 265                         self.msg('crew members: no. of captain images != 1')
 266                         return
 267                 tbl = capts[0]
 268                 while not tbl.find('a', href=pirate_ref_re):
 269                         tbl = tbl.findParent('table')
 270                         if not tbl:
 271                                 self.msg('crew members: cannot find table')
 272                                 return
 273                 current_rank_crew = None
 274                 crew_rank_re = regexp.compile('/yoweb/images/crew')
 275                 for row in tbl.contents:
 276                         # findAll(recurse=False)
 277                         if isinstance(row, unicode):
 278                                 continue
 279
 280                         is_rank = row.find('img', attrs={'src': crew_rank_re})
 281                         if is_rank:
 282                                 rank = soup_text(row)
 283                                 current_rank_crew = []
 284                                 self.crew.append((rank, current_rank_crew))
 285                                 continue
 286                         for cell in row.findAll('a', href=pirate_ref_re):
 287                                 if current_rank_crew is None:
 288                                         self.soupm(cell, 'crew members: crew'
 289                                                 ' before rank')
 290                                         continue
 291                                 current_rank_crew.append(soup_text(cell))
 292
 293         def __str__(self):
 294                 return `(self.crew, self.msgs)`
 295
 296 class StandingsTable:
 297         def __init__(self, use_puzzles=None, col_width=6):
 298                 if use_puzzles is None:
 299                         if opts.ship_duty:
 300                                 use_puzzles=[
 301                                         'Navigating','Battle Navigation',
 302                                         'Gunning',
 303                                         ['Sailing','Rigging'],
 304                                         'Bilging',
 305                                         'Carpentry',
 306                                         'Treasure Haul'
 307                                 ]
 308                         else:
 309                                 use_puzzles=puzzles
 310                 self._puzzles = use_puzzles
 311                 self.s = ''
 312                 self._cw = col_width-1
 313
 314         def _pline(self, pirate, puzstrs):
 315                 self.s += ' %-*s' % (max_pirate_namelen, pirate)
 316                 for v in puzstrs:
 317                         self.s += ' %-*.*s' % (self._cw,self._cw, v)
 318                 self.s += '\n'
 319
 320         def _puzstr(self, pi, puzzle):
 321                 if not isinstance(puzzle,list): puzzle = [puzzle]
 322                 try: standing = max([pi.standings[p] for p in puzzle])
 323                 except KeyError: return '?'
 324                 if not standing: return ''
 325                 s = ''
 326                 if self._cw > 4:
 327                         c1 = standingvals[standing][0]
 328                         if standing < 3: c1 = c1.lower() # 3 = Master
 329                         s += `standing`
 330                 if self._cw > 5:
 331                         s += ' '
 332                 s += '*' * (standing / 2)
 333                 s += '+' * (standing % 2)
 334                 return s
 335
 336         def headings(self):
 337                 def puzn_redact(name):
 338                         if isinstance(name,list):
 339                                 return '/'.join(
 340                                         ["%.*s" % (self._cw/2, puzn_redact(n))
 341                                          for n in name])
 342                         spc = name.find(' ')
 343                         if spc < 0: return name
 344                         return name[0:min(4,spc)] + name[spc+1:]
 345                 self._pline('', map(puzn_redact, self._puzzles))
 346         def literalline(self, line):
 347                 self.s += line + '\n'
 348         def pirate(self, pi):
 349                 puzstrs = [self._puzstr(pi,puz) for puz in self._puzzles]
 350                 self._pline(pi.name, puzstrs)
 351
 352         def results(self):
 353                 return self.s
 354
 355 def do_pirate(pirates, bu):
 356         print '{'
 357         for pirate in pirates:
 358                 info = PirateInfo(pirate)
 359                 print '%s: %s,' % (`pirate`, info)
 360         print '}'
 361
 362 def prep_crew_of(args, bu, max_age=300):
 363         if len(args) != 1: bu('crew-of takes one pirate name')
 364         pi = PirateInfo(args[0], max_age)
 365         return CrewInfo(pi.crew[0], max_age)
 366
 367 def do_crew_of(args, bu):
 368         ci = prep_crew_of(args, bu)
 369         print ci
 370
 371 def do_standings_crew_of(args, bu):
 372         ci = prep_crew_of(args, bu, 60)
 373         tab = StandingsTable()
 374         tab.headings()
 375         for (rank, members) in ci.crew:
 376                 if not members: continue
 377                 tab.literalline('%s:' % rank)
 378                 for p in members:
 379                         pi = PirateInfo(p, 1800)
 380                         tab.pirate(pi)
 381         print tab.results()
 382
 383 def main():
 384         global opts, fetcher
 385
 386         pa = OptionParser(
 387 '''usage: .../yoweb-scrape [OPTION...] ACTION [ARGS...]
 388 actions:
 389  yoweb-scrape [--ocean OCEAN ...] pirate PIRATE
 390  yoweb-scrape [--ocean OCEAN ...] crew-of PIRATE
 391  yoweb-scrape [--ocean OCEAN ...] standings-crew-of PIRATE
 392 ''')
 393         ao = pa.add_option
 394         ao('-O','--ocean',dest='ocean', metavar='OCEAN',
 395                 default='ice',
 396                 help='select ocean OCEAN')
 397         ao('--cache-dir', dest='cache_dir', metavar='DIR',
 398                 default='~/.yoweb-scrape-cache',
 399                 help='cache yoweb pages in DIR')
 400         ao('-D','--debug', action='store_true', dest='debug', default=False,
 401                 help='enable debugging output')
 402         ao('-q','--quiet', action='store_true', dest='quiet',
 403                 help='suppress warning output')
 404
 405         ao('--ship-duty', action='store_true', dest='ship_duty',
 406                 help='show ship duty station puzzles')
 407
 408         (opts,args) = pa.parse_args()
 409
 410         if len(args) < 1:
 411                 pa.error('need a mode argument')
 412
 413         mode = args[0]
 414         mode_fn_name = 'do_' + mode.replace('_','#').replace('-','_')
 415         try: mode_fn = globals()[mode_fn_name]
 416         except KeyError: pa.error('unknown mode "%s"' % mode)
 417
 418         # fixed parameters
 419         opts.min_max_age = 60
 420         opts.expire_age = 3600
 421
 422         if opts.cache_dir.startswith('~/'):
 423                 opts.cache_dir = os.getenv('HOME') + opts.cache_dir[1:]
 424
 425         fetcher = Fetcher(opts.ocean, opts.cache_dir)
 426
 427         mode_fn(args[1:], pa.error)
 428
 429 main()