chiark
/
gitweb
/
~yarrgweb
/
ypp-sc-tools.main.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
ocean scraper can fetch charts
[ypp-sc-tools.main.git]
/
yarrg
/
yppedia-ocean-scraper
diff --git
a/yarrg/yppedia-ocean-scraper
b/yarrg/yppedia-ocean-scraper
index 9105f1963998d96943c08237ad9a36f84deb5d85..5f9c5f007b46fda4265ccc7492dc2b3d58e079f0 100755
(executable)
--- a/
yarrg/yppedia-ocean-scraper
+++ b/
yarrg/yppedia-ocean-scraper
@@
-53,12
+53,17
@@
def debug(k,v):
def fetch():
global soup
def fetch():
global soup
- url = ('http://yppedia.puzzlepirates.com/%s_Ocean' %
- urllib.quote(ocean,''))
+ if opts.chart:
+ url_base = 'index.php?title=Template:Map:%s_Ocean&action=edit'
+ else:
+ url_base = '%s_Ocean'
+ url = ('http://yppedia.puzzlepirates.com/' +
+ (url_base % urllib.quote(ocean,'')))
debug('fetching',url)
dataf = urllib2.urlopen(url)
debug('fetched',dataf)
debug('fetching',url)
dataf = urllib2.urlopen(url)
debug('fetched',dataf)
- soup = BeautifulSoup(dataf)
+ soup = BeautifulSoup(dataf,
+ convertEntities=BeautifulSoup.HTML_ENTITIES)
title_arch_re = regexp.compile('(\\S.*\\S) Archipelago \\((\\S+)\\)$')
title_arch_re = regexp.compile('(\\S.*\\S) Archipelago \\((\\S+)\\)$')
@@
-78,7
+83,14
@@
def title_arch_ok(t):
if o is None: return False
return o == ocean
if o is None: return False
return o == ocean
-def parse():
+def parse_chart():
+ ta = soup.find('textarea')
+ debug('ta',ta)
+ rc = ta.renderContents()
+ debug('rc',rc)
+ return rc
+
+def parse_ocean():
firstarch = soup.find('a', attrs = {'title': title_arch_ok})
debug('fa',firstarch)
firstarch = soup.find('a', attrs = {'title': title_arch_ok})
debug('fa',firstarch)
@@
-136,8
+148,11
@@
def main():
global opts
pa = OptionParser(
global opts
pa = OptionParser(
-
'''usage: .../yppedia-ocean-scraper [--debug
] OCEAN''')
+
'''usage: .../yppedia-ocean-scraper [--debug] [--chart
] OCEAN''')
ao = pa.add_option
ao = pa.add_option
+
+ ao('--chart', action='store_true', dest='chart',
+ help='print chart source rather than arch/island info')
ao('--debug', action='count', dest='debug', default=0,
help='enable debugging output')
ao('--debug', action='count', dest='debug', default=0,
help='enable debugging output')
@@
-148,7
+163,10
@@
def main():
ocean = args[0]
fetch()
ocean = args[0]
fetch()
- parse()
- output()
+ if opts.chart:
+ print parse_chart()
+ else:
+ parse_ocean()
+ output()
main()
main()