-def fetch(url):
- cache_corename = urllib.quote_plus(url)
- cache_basename = "#%s#" % cache_corename
- try: f = file(cache_basename, 'r')
- except (OSError,IOError), oe:
- if oe.errno != errno.ENOENT: raise
- f = None
- if f is not None:
- s = os.fstat(f.fileno())
- if now > s.st_mtime + max_age:
+def debug(m):
+ if opts.debug:
+ print >>sys.stderr, m
+
+class Fetcher:
+ def __init__(self, ocean, cachedir):
+ debug('Fetcher init %s' % cachedir)
+ self.ocean = ocean
+ self.cachedir = cachedir
+ try: os.mkdir(cachedir)
+ except (OSError,IOError), oe:
+ if oe.errno != errno.EEXIST: raise
+
+ def _rate_limit_cache_clean(self, now):
+ ages = []
+ for path in os.listdir(self.cachedir):
+ if not path.startswith('#'): continue
+ try: s = os.stat(path)
+ except (OSError,IOError), oe:
+ if oe.errno != errno.ENOENT: raise
+ continue
+ age = now - s.st_mtime
+ if age > opts.max_age:
+ debug('Fetcher expire %d %s' % (age, path))
+ try: os.remove(path)
+ except (OSError,IOError), oe:
+ if oe.errno != errno.ENOENT: raise
+ continue
+ ages.append(age)
+ ages.sort()
+ debug('Fetcher ages ' + `ages`)
+ min_age = 1
+ need_wait = 0
+ for age in ages:
+ if age < min_age:
+ debug('Fetcher morewait min=%d age=%d' %
+ (min_age, age))
+ need_wait = max(need_wait, age - min_age)
+ min_age *= 2
+ min_age += 1
+ if need_wait:
+ debug('Fetcher wait %d' % need_wait)
+ os.sleep(need_wait)
+
+ def fetch(self, url):
+ debug('Fetcher fetch %s' % url)
+ cache_corename = urllib.quote_plus(url)
+ cache_item = "%s/#%s#" % (self.cachedir, cache_corename)
+ try: f = file(cache_item, 'r')
+ except (OSError,IOError), oe:
+ if oe.errno != errno.ENOENT: raise