X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~mdw/git/tripe/blobdiff_plain/cc72e4b3cc435903ac989d31037806d607f86c7d..8d513103f66617e96234d19813ab655ed9d5d13d:/peerdb/tripe-newpeers.in

diff --git a/peerdb/tripe-newpeers.in b/peerdb/tripe-newpeers.in
index 88d31549..92c1a070 100644
--- a/peerdb/tripe-newpeers.in
+++ b/peerdb/tripe-newpeers.in
@@ -32,8 +32,12 @@ import mLib as M
 from optparse import OptionParser
 import cdb as CDB
 from sys import stdin, stdout, exit, argv
+import subprocess as SUB
 import re as RX
 import os as OS
+import errno as E
+import fcntl as F
+import socket as S
 from cStringIO import StringIO
 
 ###--------------------------------------------------------------------------
@@ -71,14 +75,14 @@ class ResolvingHost (object):
   def __init__(me, name):
     """Make a new resolving-host object for the host NAME."""
     me.name = name
-    me.addr = { 'INET': [] }
+    me.addr = { 'INET': [], 'INET6': [] }
     me.failure = None
 
   def addaddr(me, af, addr):
     """
     Add the address ADDR with address family AF.
 
-    The address family must currently be `INET'.
+    The address family may be `INET' or `INET6'.
     """
     me.addr[af].append(addr)
 
@@ -93,17 +97,19 @@ class ResolvingHost (object):
     if me.failure is not None: raise ResolverFailure(me.name, me.failure)
     aa = []
     a4 = me.addr['INET']
+    a6 = me.addr['INET6']
     all, any = False, False
     for ch in flags:
       if ch == '*': all = True
       elif ch == '4': aa += a4; any = True
+      elif ch == '6': aa += a6; any = True
       else: raise ValueError("unknown address-resolution flag `%s'" % ch)
-    if not any: aa = a4
+    if not any: aa = a4 + a6
     if not aa: raise ResolverFailure(me.name, 'no matching addresses found')
     if not all: aa = [aa[0]]
     return aa
 
-class BulkResolver (object):
+class BaseBulkResolver (object):
   """
   Resolve a number of DNS names in parallel.
 
@@ -122,26 +128,49 @@ class BulkResolver (object):
   def __init__(me):
     """Initialize the resolver."""
     me._namemap = {}
-    me._noutstand = 0
 
   def prepare(me, name):
     """Prime the resolver to resolve the given host NAME."""
     if name not in me._namemap:
       me._namemap[name] = host = ResolvingHost(name)
-      host._resolv = M.SelResolveByName(
-        name,
-        lambda cname, alias, addr: me._resolved(host, cname, addr),
-        lambda: me._resolved(host, None, []))
-      me._noutstand += 1
-
-  def run(me):
-    """Run the background DNS resolver until it's finished."""
-    while me._noutstand: M.select()
+      try:
+        ailist = S.getaddrinfo(name, None, S.AF_UNSPEC, S.SOCK_DGRAM, 0,
+                               S.AI_NUMERICHOST | S.AI_NUMERICSERV)
+      except S.gaierror:
+        me._prepare(host, name)
+      else:
+        for af, skty, proto, cname, sa in ailist:
+          if af == S.AF_INET: host.addaddr('INET', sa[0])
+          elif af == S.AF_INET6: host.addaddr('INET6', sa[0])
 
   def lookup(me, name, flags):
     """Fetch the address corresponding to the host NAME."""
     return me._namemap[name].get(flags)
 
+class BresBulkResolver (BaseBulkResolver):
+  """
+  A BulkResolver using mLib's `bres' background resolver.
+
+  This is always available (and might use ADNS), but only does IPv4.
+  """
+
+  def __init__(me):
+    super(BresBulkResolver, me).__init__()
+    """Initialize the resolver."""
+    me._noutstand = 0
+
+  def _prepare(me, host, name):
+    """Arrange to resolve a NAME, reporting the results to HOST."""
+    host._resolv = M.SelResolveByName(
+      name,
+      lambda cname, alias, addr: me._resolved(host, cname, addr),
+      lambda: me._resolved(host, None, []))
+    me._noutstand += 1
+
+  def run(me):
+    """Run the background DNS resolver until it's finished."""
+    while me._noutstand: M.select()
+
   def _resolved(me, host, cname, addr):
     """Callback function: remember that ADDRs are the addresses for HOST."""
     if not addr:
@@ -152,6 +181,140 @@ class BulkResolver (object):
     host._resolv = None
     me._noutstand -= 1
 
+class AdnsBulkResolver (BaseBulkResolver):
+  """
+  A BulkResolver using ADNS, via the `adnshost' command-line tool.
+
+  This can do simultaneous IPv4 and IPv6 lookups and is quite shiny.
+  """
+
+  def __init__(me):
+    """Initialize the resolver."""
+
+    super(AdnsBulkResolver, me).__init__()
+
+    ## Start the external resolver process.
+    me._kid = SUB.Popen(['adnshost', '-afs'],
+                        stdin = SUB.PIPE, stdout = SUB.PIPE)
+
+    ## Set up the machinery for feeding input to the resolver.
+    me._in = me._kid.stdin
+    M.fdflags(me._in, fbic = OS.O_NONBLOCK, fxor = OS.O_NONBLOCK)
+    me._insel = M.SelFile(me._in.fileno(), M.SEL_WRITE, me._write)
+    me._inbuf, me._inoff, me._inlen = '', 0, 0
+    me._idmap = {}
+    me._nextid = 0
+
+    ## Set up the machinery for collecting the resolver's output.
+    me._out = me._kid.stdout
+    M.fdflags(me._out, fbic = OS.O_NONBLOCK, fxor = OS.O_NONBLOCK)
+    me._outline = M.SelLineBuffer(me._out,
+                                  lineproc = me._hostline, eofproc = me._eof)
+    me._outline.enable()
+
+    ## It's not finished yet.
+    me._done = False
+
+  def _prepare(me, host, name):
+    """Arrange for the resolver to resolve the name NAME."""
+
+    ## Work out the next job id, and associate that with the host record.
+    host.id = me._nextid; me._nextid += 1
+    me._namemap[name] = me._idmap[host.id] = host
+
+    ## Feed the name to the resolver process.
+    me._inbuf += name + '\n'
+    me._inlen += len(name) + 1
+    if not me._insel.activep: me._insel.enable()
+    while me._inoff < me._inlen: M.select()
+
+  def _write(me):
+    """Write material from `_inbuf' to the resolver when it's ready."""
+
+    ## Try to feed some more material to the resolver.
+    try: n = OS.write(me._in.fileno(), me._inbuf[me._inoff:])
+    except OSError, e:
+      if e.errno == E.EAGAIN or e.errno == E.EWOULDBLOCK: return
+      else: raise
+
+    ## If we're done, then clear the buffer.
+    me._inoff += n
+    if me._inoff >= me._inlen:
+      me._insel.disable()
+      me._inbuf, me._inoff, me._inlen = '', 0, 0
+
+  def _eof(me):
+    """Notice that the resolver has finished."""
+    me._outline.disable()
+    me._done = True
+    me._kid.wait()
+
+  def run(me):
+    """
+    Tell the resolver it has all of our input now, and wait for it to finish.
+    """
+    me._in.close()
+    while not me._done: M.select()
+    if me._idmap:
+      raise Exception('adnshost failed to process all the requests')
+
+  def _hostline(me, line):
+    """Handle a host line from the resolver."""
+
+    ## Parse the line into fields.
+    (id, nrrs, stty, stocde, stmsg, owner, cname, ststr), _ = \
+        M.split(line, quotep = True)
+    id, nrrs = int(id), int(nrrs)
+
+    ## Find the right record.
+    host = me._idmap[id]
+    if stty != 'ok': host.failed(ststr)
+
+    ## Stash away the canonical name of the host.
+    host.name = cname == '$' and owner or cname
+
+    ## If there are no record lines to come, then remove this record from the
+    ## list of outstanding jobs.  Otherwise, switch to the handler for record
+    ## lines.
+    if not nrrs:
+      del me._idmap[id]
+    else:
+      me._outline.lineproc = me._rrline
+      me._nrrs = nrrs
+      me._outhost = host
+
+  def _rrline(me, line):
+    """Handle a record line from the resolver."""
+
+    ## Parse the line into fields.
+    ww, _ = M.split(line, quotep = True)
+    owner, type, af = ww[:3]
+
+    ## If this is an address record, and it looks like an interesting address
+    ## type, then stash the address.
+    if type == 'A' and (af == 'INET' or af == 'INET6'):
+      me._outhost.addaddr(af, ww[3])
+
+    ## Update the parser state.  If there are no more records for this job
+    ## then mark the job as done and switch back to expecting a host line.
+    me._nrrs -= 1
+    if not me._nrrs:
+      me._outline.lineproc = me._hostline
+      del me._idmap[me._outhost.id]
+      me._outhost = None
+
+## Select a bulk resolver.  If `adnshost' exists then we might as well use
+## it.
+BulkResolver = BresBulkResolver
+try:
+  p = SUB.Popen(['adnshost', '--version'],
+                stdin = SUB.PIPE, stdout = SUB.PIPE, stderr = SUB.PIPE)
+  _out, _err = p.communicate()
+  st = p.wait()
+  if st == 0: BulkResolver = AdnsBulkResolver
+except OSError:
+  pass
+
 ###--------------------------------------------------------------------------
 ### The configuration parser.
 
@@ -178,7 +341,7 @@ RX_REF = RX.compile(r'(?x) \$ \( ([^)]+) \)')
 
 ## Match a $FLAGS[HOST] name resolution reference; group 1 are the flags;
 ## group 2 is the HOST.
-RX_RESOLVE = RX.compile(r'(?x) \$ ([4*]*) \[ ([^]]+) \]')
+RX_RESOLVE = RX.compile(r'(?x) \$ ([46*]*) \[ ([^]]+) \]')
 
 class ConfigSyntaxError (ExpectedError):
   def __init__(me, fname, lno, msg):
@@ -397,8 +560,9 @@ class MyConfigParser (object):
       expansion and processes them correctly.
 
     * It recognizes `$FLAGS[HOST]' name-resolver requests and handles them
-      correctly.  FLAGS consists of characters `4' (IPv4 addresses), and `*'
-      (all addresses, space-separated, rather than just the first).
+      correctly.  FLAGS consists of characters `4' (IPv4 addresses), `6'
+      (IPv6 addresses), and `*' (all, space-separated, rather than just the
+      first).
 
     * Its parsing behaviour is well-defined.