4 ### Build a CDB file from configuration file
6 ### (c) 2007 Straylight/Edgeware
9 ###----- Licensing notice ---------------------------------------------------
11 ### This file is part of Trivial IP Encryption (TrIPE).
13 ### TrIPE is free software: you can redistribute it and/or modify it under
14 ### the terms of the GNU General Public License as published by the Free
15 ### Software Foundation; either version 3 of the License, or (at your
16 ### option) any later version.
18 ### TrIPE is distributed in the hope that it will be useful, but WITHOUT
19 ### ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
20 ### FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
23 ### You should have received a copy of the GNU General Public License
24 ### along with TrIPE. If not, see <https://www.gnu.org/licenses/>.
28 ###--------------------------------------------------------------------------
29 ### External dependencies.
32 from optparse import OptionParser
34 from sys import stdin, stdout, exit, argv
35 import subprocess as SUB
41 from cStringIO import StringIO
43 ###--------------------------------------------------------------------------
46 class CDBFake (object):
47 """Like cdbmake, but just outputs data suitable for cdb-map."""
48 def __init__(me, file = stdout):
50 def add(me, key, value):
51 me.file.write('%s:%s\n' % (key, value))
55 class ExpectedError (Exception): pass
57 ###--------------------------------------------------------------------------
58 ### A bulk DNS resolver.
60 class ResolverFailure (ExpectedError):
61 def __init__(me, host, msg):
65 return "failed to resolve `%s': %s" % (me.host, me.msg)
67 class ResolvingHost (object):
69 A host name which is being looked up by a bulk-resolver instance.
71 Most notably, this is where the flag-handling logic lives for the
72 $FLAGS[HOSTNAME] syntax.
75 def __init__(me, name):
76 """Make a new resolving-host object for the host NAME."""
78 me.addr = { 'INET': [], 'INET6': [] }
81 def addaddr(me, af, addr):
83 Add the address ADDR with address family AF.
85 The address family may be `INET' or `INET6'.
87 me.addr[af].append(addr)
91 Report that resolution of this host failed, with a human-readable MSG.
96 """Return a list of addresses according to the FLAGS string."""
97 if me.failure is not None: raise ResolverFailure(me.name, me.failure)
100 a6 = me.addr['INET6']
101 all, any = False, False
103 if ch == '*': all = True
104 elif ch == '4': aa += a4; any = True
105 elif ch == '6': aa += a6; any = True
106 else: raise ValueError("unknown address-resolution flag `%s'" % ch)
107 if not any: aa = a4 + a6
108 if not aa: raise ResolverFailure(me.name, 'no matching addresses found')
109 if not all: aa = [aa[0]]
112 class BaseBulkResolver (object):
114 Resolve a number of DNS names in parallel.
116 The BulkResovler resolves a number of hostnames in parallel. Using it
117 works in three phases:
119 1. You call prepare(HOSTNAME) a number of times, to feed in the hostnames
120 you're interested in.
122 2. You call run() to actually drive the resolver.
124 3. You call lookup(HOSTNAME) to get the address you wanted. This will
125 fail with KeyError if the resolver couldn't resolve the HOSTNAME.
129 """Initialize the resolver."""
132 def prepare(me, name):
133 """Prime the resolver to resolve the given host NAME."""
134 if name not in me._namemap:
135 me._namemap[name] = host = ResolvingHost(name)
137 ailist = S.getaddrinfo(name, None, S.AF_UNSPEC, S.SOCK_DGRAM, 0,
138 S.AI_NUMERICHOST | S.AI_NUMERICSERV)
140 me._prepare(host, name)
142 for af, skty, proto, cname, sa in ailist:
143 if af == S.AF_INET: host.addaddr('INET', sa[0])
144 elif af == S.AF_INET6: host.addaddr('INET6', sa[0])
146 def lookup(me, name, flags):
147 """Fetch the address corresponding to the host NAME."""
148 return me._namemap[name].get(flags)
150 class BresBulkResolver (BaseBulkResolver):
152 A BulkResolver using mLib's `bres' background resolver.
154 This is always available (and might use ADNS), but only does IPv4.
158 super(BresBulkResolver, me).__init__()
159 """Initialize the resolver."""
162 def _prepare(me, host, name):
163 """Arrange to resolve a NAME, reporting the results to HOST."""
164 host._resolv = M.SelResolveByName(
166 lambda cname, alias, addr: me._resolved(host, cname, addr),
167 lambda: me._resolved(host, None, []))
171 """Run the background DNS resolver until it's finished."""
172 while me._noutstand: M.select()
174 def _resolved(me, host, cname, addr):
175 """Callback function: remember that ADDRs are the addresses for HOST."""
177 host.failed('(unknown failure)')
179 if cname is not None: host.name = cname
180 for a in addr: host.addaddr('INET', a)
184 class AdnsBulkResolver (BaseBulkResolver):
186 A BulkResolver using ADNS, via the `adnshost' command-line tool.
188 This can do simultaneous IPv4 and IPv6 lookups and is quite shiny.
192 """Initialize the resolver."""
194 super(AdnsBulkResolver, me).__init__()
196 ## Start the external resolver process.
197 me._kid = SUB.Popen(['adnshost', '-afs'],
198 stdin = SUB.PIPE, stdout = SUB.PIPE)
200 ## Set up the machinery for feeding input to the resolver.
201 me._in = me._kid.stdin
202 M.fdflags(me._in, fbic = OS.O_NONBLOCK, fxor = OS.O_NONBLOCK)
203 me._insel = M.SelFile(me._in.fileno(), M.SEL_WRITE, me._write)
204 me._inbuf, me._inoff, me._inlen = '', 0, 0
208 ## Set up the machinery for collecting the resolver's output.
209 me._out = me._kid.stdout
210 M.fdflags(me._out, fbic = OS.O_NONBLOCK, fxor = OS.O_NONBLOCK)
211 me._outline = M.SelLineBuffer(me._out,
212 lineproc = me._hostline, eofproc = me._eof)
215 ## It's not finished yet.
218 def _prepare(me, host, name):
219 """Arrange for the resolver to resolve the name NAME."""
221 ## Work out the next job id, and associate that with the host record.
222 host.id = me._nextid; me._nextid += 1
223 me._namemap[name] = me._idmap[host.id] = host
225 ## Feed the name to the resolver process.
226 me._inbuf += name + '\n'
227 me._inlen += len(name) + 1
228 if not me._insel.activep: me._insel.enable()
229 while me._inoff < me._inlen: M.select()
232 """Write material from `_inbuf' to the resolver when it's ready."""
234 ## Try to feed some more material to the resolver.
235 try: n = OS.write(me._in.fileno(), me._inbuf[me._inoff:])
237 if e.errno == E.EAGAIN or e.errno == E.EWOULDBLOCK: return
240 ## If we're done, then clear the buffer.
242 if me._inoff >= me._inlen:
244 me._inbuf, me._inoff, me._inlen = '', 0, 0
247 """Notice that the resolver has finished."""
248 me._outline.disable()
254 Tell the resolver it has all of our input now, and wait for it to finish.
257 while not me._done: M.select()
259 raise Exception('adnshost failed to process all the requests')
261 def _hostline(me, line):
262 """Handle a host line from the resolver."""
264 ## Parse the line into fields.
265 (id, nrrs, stty, stocde, stmsg, owner, cname, ststr), _ = \
266 M.split(line, quotep = True)
267 id, nrrs = int(id), int(nrrs)
269 ## Find the right record.
271 if stty != 'ok': host.failed(ststr)
273 ## Stash away the canonical name of the host.
274 host.name = cname == '$' and owner or cname
276 ## If there are no record lines to come, then remove this record from the
277 ## list of outstanding jobs. Otherwise, switch to the handler for record
282 me._outline.lineproc = me._rrline
286 def _rrline(me, line):
287 """Handle a record line from the resolver."""
289 ## Parse the line into fields.
290 ww, _ = M.split(line, quotep = True)
291 owner, type, af = ww[:3]
293 ## If this is an address record, and it looks like an interesting address
294 ## type, then stash the address.
295 if type == 'A' and (af == 'INET' or af == 'INET6'):
296 me._outhost.addaddr(af, ww[3])
298 ## Update the parser state. If there are no more records for this job
299 ## then mark the job as done and switch back to expecting a host line.
302 me._outline.lineproc = me._hostline
303 del me._idmap[me._outhost.id]
306 ## Select a bulk resolver. If `adnshost' exists then we might as well use
308 BulkResolver = BresBulkResolver
310 p = SUB.Popen(['adnshost', '--version'],
311 stdin = SUB.PIPE, stdout = SUB.PIPE, stderr = SUB.PIPE)
312 _out, _err = p.communicate()
314 if st == 0: BulkResolver = AdnsBulkResolver
318 ###--------------------------------------------------------------------------
319 ### The configuration parser.
321 ## Match a comment or empty line.
322 RX_COMMENT = RX.compile(r'(?x) ^ \s* (?: $ | [;#])')
324 ## Match a section group header.
325 RX_GRPHDR = RX.compile(r'(?x) ^ \s* \[ (.*) \] \s* $')
327 ## Match an assignment line.
328 RX_ASSGN = RX.compile(r'''(?x) ^
329 ([^\s:=] (?: [^:=]* [^\s:=])?)
334 ## Match a continuation line.
335 RX_CONT = RX.compile(r'''(?x) ^ \s+
339 ## Match a $(VAR) configuration variable reference; group 1 is the VAR.
340 RX_REF = RX.compile(r'(?x) \$ \( ([^)]+) \)')
342 ## Match a $FLAGS[HOST] name resolution reference; group 1 are the flags;
343 ## group 2 is the HOST.
344 RX_RESOLVE = RX.compile(r'(?x) \$ ([46*]*) \[ ([^]]+) \]')
346 class ConfigSyntaxError (ExpectedError):
347 def __init__(me, fname, lno, msg):
352 return '%s:%d: %s' % (me.fname, me.lno, me.msg)
355 return ' -> '.join(["`%s'" % hop for hop in path])
357 class AmbiguousOptionError (ExpectedError):
358 def __init__(me, key, patha, vala, pathb, valb):
360 me.patha, me.vala = patha, vala
361 me.pathb, me.valb = pathb, valb
363 return "Ambiguous answer resolving key `%s': " \
364 "path %s yields `%s' but %s yields `%s'" % \
365 (me.key, _fmt_path(me.patha), me.vala, _fmt_path(me.pathb), me.valb)
367 class InheritanceCycleError (ExpectedError):
368 def __init__(me, key, path):
372 return "Found a cycle %s looking up key `%s'" % \
373 (_fmt_path(me.path), me.key)
375 class MissingSectionException (ExpectedError):
376 def __init__(me, sec):
379 return "Section `%s' not found" % (me.sec)
381 class MissingKeyException (ExpectedError):
382 def __init__(me, sec, key):
386 return "Key `%s' not found in section `%s'" % (me.key, me.sec)
388 class ConfigSection (object):
390 A section in a configuration parser.
392 This is where a lot of the nitty-gritty stuff actually happens. The
393 `MyConfigParser' knows a lot about the internals of this class, which saves
394 on building a complicated interface.
397 def __init__(me, name, cp):
398 """Initialize a new, empty section with a given NAME and parent CP."""
400 ## The cache maps item keys to entries, which consist of a pair of
401 ## objects. There are four possible states for a cache entry:
403 ## * missing -- there is no entry at all with this key, so we must
406 ## * None, None -- we are actively trying to resolve this key, so if we
407 ## encounter this state, we have found a cycle in the inheritance
410 ## * None, [] -- we know that this key isn't reachable through any of
413 ## * VALUE, PATH -- we know that the key resolves to VALUE, along the
414 ## PATH from us (exclusive) to the defining parent (inclusive).
420 def _expand(me, string, resolvep):
422 Expands $(...) and (optionally) $FLAGS[...] placeholders in STRING.
424 RESOLVEP is a boolean switch: do we bother to tax the resolver or not?
425 This is turned off by MyConfigParser's resolve() method while it's
426 collecting hostnames to be resolved.
428 string = RX_REF.sub(lambda m: me.get(m.group(1), resolvep), string)
430 string = RX_RESOLVE.sub(
431 lambda m: ' '.join(me._cp._resolver.lookup(m.group(2), m.group(1))),
436 """Yield this section's parents."""
437 try: names = me._itemmap['@inherit']
438 except KeyError: return
439 for name in names.replace(',', ' ').split():
440 yield me._cp.section(name)
442 def _get(me, key, path = None):
444 Low-level option-fetching method.
446 Fetch the value for the named KEY in this section, or maybe (recursively)
447 a section which it inherits from.
449 Returns a pair VALUE, PATH. The value is not expanded; nor do we check
450 for the special `name' key. The caller is expected to do these things.
451 Returns None if no value could be found.
454 ## If we weren't given a path, then we'd better make one.
455 if path is None: path = []
457 ## Extend the path to cover us, but remember to remove us again when
458 ## we've finished. If we need to pass the current path back upwards,
459 ## then remember to take a copy.
463 ## If we've been this way before on another pass through then return the
464 ## value we found then. If we're still thinking about it then we've
466 try: v, p = me._cache[key]
467 except KeyError: pass
469 if p is None: raise InheritanceCycleError(key, path[:])
470 else: return v, path + p
472 ## See whether the answer is ready waiting for us.
473 try: v = me._itemmap[key]
474 except KeyError: pass
477 me._cache[key] = v, []
480 ## Initially we have no idea.
484 ## Go through our parents and ask them what they think.
485 me._cache[key] = None, None
486 for p in me._parents():
488 ## See whether we get an answer. If not, keep on going.
489 v, pp = p._get(key, path)
490 if v is None: continue
492 ## If we got an answer, check that it matches any previous ones.
497 raise AmbiguousOptionError(key, winner, value, pp, v)
499 ## That's the best we could manage.
500 me._cache[key] = value, winner[len(path):]
504 ## Remove us from the path again.
507 def get(me, key, resolvep = True):
509 Retrieve the value of KEY from this section.
512 ## Special handling for the `name' key.
514 value = me._itemmap.get('name', me.name)
515 elif key == '@inherits':
516 try: return me._itemmap['@inherits']
517 except KeyError: raise MissingKeyException(me.name, key)
519 value, _ = me._get(key)
521 raise MissingKeyException(me.name, key)
523 ## Expand the value and return it.
524 return me._expand(value, resolvep)
526 def items(me, resolvep = True):
528 Yield a list of item names in the section.
531 ## Initialize for a depth-first walk of the inheritance graph.
532 seen = { 'name': True }
533 visiting = { me.name: True }
536 ## Visit nodes, collecting their keys. Don't believe the values:
537 ## resolving inheritance is too hard to do like this.
540 for p in sec._parents():
541 if p.name not in visiting:
542 stack.append(p); visiting[p.name] = True
544 for key in sec._itemmap.iterkeys(): seen[key] = None
547 return seen.iterkeys()
549 class MyConfigParser (object):
551 A more advanced configuration parser.
553 This has four major enhancements over the standard ConfigParser which are
556 * It recognizes `@inherits' keys and follows them when expanding a
559 * It recognizes `$(VAR)' references to configuration variables during
560 expansion and processes them correctly.
562 * It recognizes `$FLAGS[HOST]' name-resolver requests and handles them
563 correctly. FLAGS consists of characters `4' (IPv4 addresses), `6'
564 (IPv6 addresses), and `*' (all, space-separated, rather than just the
567 * Its parsing behaviour is well-defined.
571 1. Call parse(FILENAME) to slurp in the configuration data.
573 2. Call resolve() to collect the hostnames which need to be resolved and
574 actually do the name resolution.
576 3. Call sections() to get a list of the configuration sections, or
577 section(NAME) to find a named section.
579 4. Call get(ITEM) on a section to collect the results, or items() to
585 Initialize a new, empty configuration parser.
588 me._resolver = BulkResolver()
592 Parse configuration from a file F.
595 ## Initial parser state.
601 ## An unpleasant hack. Python makes it hard to capture a value in a
602 ## variable and examine it in a single action, and this is the best that
605 def match(rx): m[0] = rx.match(line); return m[0]
607 ## Commit a key's value when we've determined that there are no further
608 ## continuation lines.
610 if key is not None: sect._itemmap[key] = val.getvalue()
612 ## Work through all of the input lines.
616 if match(RX_COMMENT):
617 ## A comment or a blank line. Nothing doing. (This means that we
618 ## leave out blank lines which look like they might be continuation
623 elif match(RX_GRPHDR):
624 ## A section header. Flush out any previous value and set up the new
629 try: sect = me._sectmap[name]
630 except KeyError: sect = me._sectmap[name] = ConfigSection(name, me)
633 elif match(RX_ASSGN):
634 ## A new assignment. Flush out the old one, and set up to store this
638 raise ConfigSyntaxError(f.name, lno, 'no active section to update')
641 val = StringIO(); val.write(m[0].group(2))
644 ## A continuation line. Accumulate the value.
647 raise ConfigSyntaxError(f.name, lno, 'no config value to continue')
648 val.write('\n'); val.write(m[0].group(1))
653 raise ConfigSyntaxError(f.name, lno, 'incomprehensible line')
655 ## Don't forget to commit any final value material.
658 def section(me, name):
659 """Return a ConfigSection with the given NAME."""
660 try: return me._sectmap[name]
661 except KeyError: raise MissingSectionException(name)
664 """Yield the known sections."""
665 return me._sectmap.itervalues()
669 Works out all of the hostnames which need resolving and resolves them.
671 Until you call this, attempts to fetch configuration items which need to
672 resolve hostnames will fail!
674 for sec in me.sections():
675 for key in sec.items():
676 value = sec.get(key, resolvep = False)
677 for match in RX_RESOLVE.finditer(value):
678 me._resolver.prepare(match.group(2))
681 ###--------------------------------------------------------------------------
682 ### Command-line handling.
684 def inputiter(things):
686 Iterate over command-line arguments, returning corresponding open files.
688 If none were given, or one is `-', assume standard input; if one is a
689 directory, scan it for files other than backups; otherwise return the
694 if OS.isatty(stdin.fileno()):
695 M.die('no input given, and stdin is a terminal')
701 elif OS.path.isdir(thing):
702 for item in OS.listdir(thing):
703 if item.endswith('~') or item.endswith('#'):
705 name = OS.path.join(thing, item)
706 if not OS.path.isfile(name):
712 def parse_options(argv = argv):
714 Parse command-line options, returning a pair (OPTS, ARGS).
717 op = OptionParser(usage = '%prog [-c CDB] INPUT...',
718 version = '%%prog (tripe, version %s)' % VERSION)
719 op.add_option('-c', '--cdb', metavar = 'CDB',
720 dest = 'cdbfile', default = None,
721 help = 'Compile output into a CDB file.')
722 opts, args = op.parse_args(argv)
725 ###--------------------------------------------------------------------------
730 Read the configuration files and return the accumulated result.
732 We make sure that all hostnames have been properly resolved.
734 conf = MyConfigParser()
735 for f in inputiter(args):
740 def output(conf, cdb):
742 Output the configuration information CONF to the database CDB.
744 This is where the special `user' and `auto' database entries get set.
747 for sec in sorted(conf.sections(), key = lambda sec: sec.name):
748 if sec.name.startswith('@'):
750 elif sec.name.startswith('$'):
753 label = 'P%s' % sec.name
754 try: a = sec.get('auto')
755 except MissingKeyException: pass
757 if a in ('y', 'yes', 't', 'true', '1', 'on'): auto.append(sec.name)
758 try: u = sec.get('user')
759 except MissingKeyException: pass
760 else: cdb.add('U%s' % u)
761 url = M.URLEncode(semip = True)
762 for key in sorted(sec.items()):
763 if not key.startswith('@'):
764 url.encode(key, sec.get(key))
765 cdb.add(label, url.result)
766 cdb.add('%AUTO', ' '.join(auto))
771 opts, args = parse_options()
773 cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new')
777 conf = getconf(args[1:])
779 except ExpectedError, e:
783 if __name__ == '__main__':
786 ###----- That's all, folks --------------------------------------------------