X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~yarrgweb/git?p=ypp-sc-tools.db-test.git;a=blobdiff_plain;f=yoweb-scrape;h=90a5677c50e3ff1b1a01fb5427d98ff5ebf564ca;hp=d2557b594a86b8811c3323c375bd4f5cd14c1b57;hb=bb6f3057cff743f25c2ef95a734b9c0f05c97ff6;hpb=f0eb636cf4249dbc353f24a74287913f8c88fcc0 diff --git a/yoweb-scrape b/yoweb-scrape index d2557b5..90a5677 100755 --- a/yoweb-scrape +++ b/yoweb-scrape @@ -519,16 +519,14 @@ class FlagInfo(SomethingSoupInfo): #---------- scraper for ocean info incl. embargoes etc. ---------- class IslandBasicInfo(): - # Public members: + # Public data attributes: # ocean # name - # Public members maybe set by caller: + # Public data attributes maybe set by caller: # arch def __init__(self, ocean, islename): self.ocean = ocean self.name = islename - def collect(self): - pass def yppedia(self): def q(x): return urllib.quote(x.replace(' ','_')) url_rhs = q(self.name) + '_(' + q(self.ocean) + ')' @@ -537,15 +535,17 @@ class IslandBasicInfo(): return `(self.ocean, self.name)` class IslandExtendedInfo(IslandBasicInfo): - # Public members (inherited): + # Public data attributes (inherited): # ocean # name - # Public members (additional): + # Public data attributes (additional): # islandid # yoweb_url # flagid - def collect(self): - IslandBasicInfo.collect(self) + def __init__(self, ocean, islename): + IslandBasicInfo.__init__(self, ocean, islename) + self.islandid = None + self.yoweb_url = None self._collect_yoweb() self._collect_flagid() @@ -590,14 +590,67 @@ class IslandExtendedInfo(IslandBasicInfo): return `(self.ocean, self.islandid, self.name, self.yoweb_url, self.flagid)` +class IslandFlagInfo(IslandExtendedInfo): + # Public data attributes (inherited): + # ocean + # name + # islandid + # yoweb_url + # flagid + # Public data attributes (additional): + # flag + def __init__(self, ocean, islename): + IslandExtendedInfo.__init__(self, ocean, islename) + self.flag = None + self._collect_flag() + + def _collect_flag(self): + if self.flagid is None: return + self.flag = FlagInfo(self.flagid, 1800) + + def __str__(self): + return IslandExtendedInfo.__str__(self) + '; ' + str(self.flag) + +class NullProgressReporter(): + def start(self): pass + def doing(self, msg): pass + def stop(self): pass + +class TypewriterProgressReporter(): + def start(self): + self._l = 0 + def doing(self,m): + self._doing(m + '...') + def _doing(self,m): + self._write('\r') + self._write(m) + less = self._l - len(m) + if less > 0: + self._write(' ' * less) + self._write('\b' * less) + self._l = len(m) + sys.stdout.flush() + def stop(self): + self._doing('') + self._l = 0 + def _write(self,t): + sys.stdout.write(t) + class OceanInfo(): - # Public data attributes (valid after collect()): + # Public data attributes: # oi.islands[islename] = IslandInfo(...) # oi.arches[archname][islename] = IslandInfo(...) - def __init__(self, isleclass=IslandBasicInfo): + def __init__(self, isleclass=IslandBasicInfo, progressreporter=None): + if progressreporter is None: + if opts.debug: progressreporter = NullProgressReporter() + else: progressreporter = TypewriterProgressReporter() + self.isleclass = isleclass self.ocean = fetcher.ocean.lower().capitalize() - def collect(self): + + progressreporter.start() + progressreporter.doing('fetching ocean info') + cmdl = ['./yppedia-ocean-scraper'] if opts.localhtml is not None: cmdl += ['--local-html-dir',opts.localhtml] @@ -615,10 +668,16 @@ class OceanInfo(): arch_re = regexp.compile('^ (\S.*)') island_re = regexp.compile('^ (\S.*)') + oscraper.wait() + assert(oscraper.returncode == 0) + self.islands = { } self.arches = { } archname = None + isles = [ ] + progressreporter.doing('parsing ocean info') + for l in oscraper.stdout: debug('OceanInfo collect l '+`l`) l = l.rstrip('\n') @@ -626,11 +685,7 @@ class OceanInfo(): if m: assert(archname is not None) islename = m.group(1) - isle = self.isleclass(self.ocean, islename) - isle.arch = archname - isle.collect() - self.islands[islename] = isle - self.arches[archname][islename] = isle + isles.append((archname, islename)) continue m = arch_re.match(l) if m: @@ -639,8 +694,19 @@ class OceanInfo(): self.arches[archname] = { } continue assert(False) - oscraper.wait() - assert(oscraper.returncode == 0) + + for i in xrange(0, len(isles)-1): + (archname, islename) = isles[i] + progressreporter.doing( + 'fetching isle info %2d/%d (%s: %s)' + % (i, len(isles), archname, islename)) + isle = self.isleclass(self.ocean, islename) + isle.arch = archname + self.islands[islename] = isle + self.arches[archname][islename] = isle + + progressreporter.stop() + def __str__(self): return `(self.islands, self.arches)` @@ -1326,13 +1392,44 @@ def do_standings_crew_of(args, bu): def do_ocean(args, bu): if (len(args)): bu('ocean takes no further arguments') fetcher.default_ocean() - oi = OceanInfo(IslandExtendedInfo) - oi.collect() + oi = OceanInfo(IslandFlagInfo) print oi for islename in sorted(oi.islands.keys()): isle = oi.islands[islename] print isle +def do_embargoes(args, bu): + if (len(args)): bu('ocean takes no further arguments') + fetcher.default_ocean() + oi = OceanInfo(IslandFlagInfo) + wr = sys.stdout.write + print ('EMBARGOES: Island | Owning flag'+ + ' | Embargoed flags') + + def getflname(isle): + if isle.islandid is None: return 'uncolonisable' + if isle.flag is None: return 'uncolonised' + return isle.flag.name + + for archname in sorted(oi.arches.keys()): + print 'ARCHIPELAGO: ',archname + for islename in sorted(oi.arches[archname].keys()): + isle = oi.islands[islename] + wr(' %-20s | ' % isle.name) + flname = getflname(isle) + wr('%-30s | ' % flname) + flag = isle.flag + if flag is None: print ''; continue + delim = '' + for rel in flag.relations: + (oname, oid, dummy, thisdeclaring, + odeclaringmin,odeclaringmax) = rel + if thisdeclaring >= 0: continue + wr(delim) + wr(oname) + delim = '; ' + print '' + #----- modes which use the chat log parser are quite complex ----- class ProgressPrintPercentage: