4 ### Build a CDB file from configuration file
6 ### (c) 2007 Straylight/Edgeware
9 ###----- Licensing notice ---------------------------------------------------
11 ### This file is part of Trivial IP Encryption (TrIPE).
13 ### TrIPE is free software: you can redistribute it and/or modify it under
14 ### the terms of the GNU General Public License as published by the Free
15 ### Software Foundation; either version 3 of the License, or (at your
16 ### option) any later version.
18 ### TrIPE is distributed in the hope that it will be useful, but WITHOUT
19 ### ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
20 ### FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
23 ### You should have received a copy of the GNU General Public License
24 ### along with TrIPE. If not, see <https://www.gnu.org/licenses/>.
28 ###--------------------------------------------------------------------------
29 ### External dependencies.
32 from optparse import OptionParser
34 from sys import stdin, stdout, exit, argv
38 from cStringIO import StringIO
40 ###--------------------------------------------------------------------------
43 class CDBFake (object):
44 """Like cdbmake, but just outputs data suitable for cdb-map."""
45 def __init__(me, file = stdout):
47 def add(me, key, value):
48 me.file.write('%s:%s\n' % (key, value))
52 class ExpectedError (Exception): pass
54 ###--------------------------------------------------------------------------
55 ### A bulk DNS resolver.
57 class ResolverFailure (ExpectedError):
58 def __init__(me, host, msg):
62 return "failed to resolve `%s': %s" % (me.host, me.msg)
64 class ResolvingHost (object):
66 A host name which is being looked up by a bulk-resolver instance.
68 Most notably, this is where the flag-handling logic lives for the
69 $FLAGS[HOSTNAME] syntax.
72 def __init__(me, name):
73 """Make a new resolving-host object for the host NAME."""
75 me.addr = { 'INET': [], 'INET6': [] }
78 def addaddr(me, af, addr):
80 Add the address ADDR with address family AF.
82 The address family may be `INET' or `INET6'.
84 me.addr[af].append(addr)
88 Report that resolution of this host failed, with a human-readable MSG.
93 """Return a list of addresses according to the FLAGS string."""
94 if me.failure is not None: raise ResolverFailure(me.name, me.failure)
98 all, any = False, False
100 if ch == '*': all = True
101 elif ch == '4': aa += a4; any = True
102 elif ch == '6': aa += a6; any = True
103 else: raise ValueError("unknown address-resolution flag `%s'" % ch)
104 if not any: aa = a4 + a6
105 if not aa: raise ResolverFailure(me.name, 'no matching addresses found')
106 if not all: aa = [aa[0]]
109 class BaseBulkResolver (object):
111 Resolve a number of DNS names in parallel.
113 The BulkResovler resolves a number of hostnames in parallel. Using it
114 works in three phases:
116 1. You call prepare(HOSTNAME) a number of times, to feed in the hostnames
117 you're interested in.
119 2. You call run() to actually drive the resolver.
121 3. You call lookup(HOSTNAME) to get the address you wanted. This will
122 fail with KeyError if the resolver couldn't resolve the HOSTNAME.
126 """Initialize the resolver."""
129 def prepare(me, name):
130 """Prime the resolver to resolve the given host NAME."""
131 if name not in me._namemap:
132 me._namemap[name] = host = ResolvingHost(name)
134 ailist = S.getaddrinfo(name, None, S.AF_UNSPEC, S.SOCK_DGRAM, 0,
135 S.AI_NUMERICHOST | S.AI_NUMERICSERV)
137 me._prepare(host, name)
139 for af, skty, proto, cname, sa in ailist:
140 if af == S.AF_INET: host.addaddr('INET', sa[0])
141 elif af == S.AF_INET6: host.addaddr('INET6', sa[0])
143 def lookup(me, name, flags):
144 """Fetch the address corresponding to the host NAME."""
145 return me._namemap[name].get(flags)
147 class BresBulkResolver (BaseBulkResolver):
149 A BulkResolver using mLib's `bres' background resolver.
151 This is always available (and might use ADNS), but only does IPv4.
155 super(BresBulkResolver, me).__init__()
156 """Initialize the resolver."""
159 def _prepare(me, host, name):
160 """Arrange to resolve a NAME, reporting the results to HOST."""
161 host._resolv = M.SelResolveByName(
163 lambda cname, alias, addr: me._resolved(host, cname, addr),
164 lambda: me._resolved(host, None, []))
168 """Run the background DNS resolver until it's finished."""
169 while me._noutstand: M.select()
171 def _resolved(me, host, cname, addr):
172 """Callback function: remember that ADDRs are the addresses for HOST."""
174 host.failed('(unknown failure)')
176 if cname is not None: host.name = cname
177 for a in addr: host.addaddr('INET', a)
181 ## Select a bulk resolver. Currently, there's only one choice.
182 BulkResolver = BresBulkResolver
184 ###--------------------------------------------------------------------------
185 ### The configuration parser.
187 ## Match a comment or empty line.
188 RX_COMMENT = RX.compile(r'(?x) ^ \s* (?: $ | [;#])')
190 ## Match a section group header.
191 RX_GRPHDR = RX.compile(r'(?x) ^ \s* \[ (.*) \] \s* $')
193 ## Match an assignment line.
194 RX_ASSGN = RX.compile(r'''(?x) ^
195 ([^\s:=] (?: [^:=]* [^\s:=])?)
200 ## Match a continuation line.
201 RX_CONT = RX.compile(r'''(?x) ^ \s+
205 ## Match a $(VAR) configuration variable reference; group 1 is the VAR.
206 RX_REF = RX.compile(r'(?x) \$ \( ([^)]+) \)')
208 ## Match a $FLAGS[HOST] name resolution reference; group 1 are the flags;
209 ## group 2 is the HOST.
210 RX_RESOLVE = RX.compile(r'(?x) \$ ([46*]*) \[ ([^]]+) \]')
212 class ConfigSyntaxError (ExpectedError):
213 def __init__(me, fname, lno, msg):
218 return '%s:%d: %s' % (me.fname, me.lno, me.msg)
221 return ' -> '.join(["`%s'" % hop for hop in path])
223 class AmbiguousOptionError (ExpectedError):
224 def __init__(me, key, patha, vala, pathb, valb):
226 me.patha, me.vala = patha, vala
227 me.pathb, me.valb = pathb, valb
229 return "Ambiguous answer resolving key `%s': " \
230 "path %s yields `%s' but %s yields `%s'" % \
231 (me.key, _fmt_path(me.patha), me.vala, _fmt_path(me.pathb), me.valb)
233 class InheritanceCycleError (ExpectedError):
234 def __init__(me, key, path):
238 return "Found a cycle %s looking up key `%s'" % \
239 (_fmt_path(me.path), me.key)
241 class MissingSectionException (ExpectedError):
242 def __init__(me, sec):
245 return "Section `%s' not found" % (me.sec)
247 class MissingKeyException (ExpectedError):
248 def __init__(me, sec, key):
252 return "Key `%s' not found in section `%s'" % (me.key, me.sec)
254 class ConfigSection (object):
256 A section in a configuration parser.
258 This is where a lot of the nitty-gritty stuff actually happens. The
259 `MyConfigParser' knows a lot about the internals of this class, which saves
260 on building a complicated interface.
263 def __init__(me, name, cp):
264 """Initialize a new, empty section with a given NAME and parent CP."""
266 ## The cache maps item keys to entries, which consist of a pair of
267 ## objects. There are four possible states for a cache entry:
269 ## * missing -- there is no entry at all with this key, so we must
272 ## * None, None -- we are actively trying to resolve this key, so if we
273 ## encounter this state, we have found a cycle in the inheritance
276 ## * None, [] -- we know that this key isn't reachable through any of
279 ## * VALUE, PATH -- we know that the key resolves to VALUE, along the
280 ## PATH from us (exclusive) to the defining parent (inclusive).
286 def _expand(me, string, resolvep):
288 Expands $(...) and (optionally) $FLAGS[...] placeholders in STRING.
290 RESOLVEP is a boolean switch: do we bother to tax the resolver or not?
291 This is turned off by MyConfigParser's resolve() method while it's
292 collecting hostnames to be resolved.
294 string = RX_REF.sub(lambda m: me.get(m.group(1), resolvep), string)
296 string = RX_RESOLVE.sub(
297 lambda m: ' '.join(me._cp._resolver.lookup(m.group(2), m.group(1))),
302 """Yield this section's parents."""
303 try: names = me._itemmap['@inherit']
304 except KeyError: return
305 for name in names.replace(',', ' ').split():
306 yield me._cp.section(name)
308 def _get(me, key, path = None):
310 Low-level option-fetching method.
312 Fetch the value for the named KEY in this section, or maybe (recursively)
313 a section which it inherits from.
315 Returns a pair VALUE, PATH. The value is not expanded; nor do we check
316 for the special `name' key. The caller is expected to do these things.
317 Returns None if no value could be found.
320 ## If we weren't given a path, then we'd better make one.
321 if path is None: path = []
323 ## Extend the path to cover us, but remember to remove us again when
324 ## we've finished. If we need to pass the current path back upwards,
325 ## then remember to take a copy.
329 ## If we've been this way before on another pass through then return the
330 ## value we found then. If we're still thinking about it then we've
332 try: v, p = me._cache[key]
333 except KeyError: pass
335 if p is None: raise InheritanceCycleError(key, path[:])
336 else: return v, path + p
338 ## See whether the answer is ready waiting for us.
339 try: v = me._itemmap[key]
340 except KeyError: pass
343 me._cache[key] = v, []
346 ## Initially we have no idea.
350 ## Go through our parents and ask them what they think.
351 me._cache[key] = None, None
352 for p in me._parents():
354 ## See whether we get an answer. If not, keep on going.
355 v, pp = p._get(key, path)
356 if v is None: continue
358 ## If we got an answer, check that it matches any previous ones.
363 raise AmbiguousOptionError(key, winner, value, pp, v)
365 ## That's the best we could manage.
366 me._cache[key] = value, winner[len(path):]
370 ## Remove us from the path again.
373 def get(me, key, resolvep = True):
375 Retrieve the value of KEY from this section.
378 ## Special handling for the `name' key.
380 value = me._itemmap.get('name', me.name)
381 elif key == '@inherits':
382 try: return me._itemmap['@inherits']
383 except KeyError: raise MissingKeyException(me.name, key)
385 value, _ = me._get(key)
387 raise MissingKeyException(me.name, key)
389 ## Expand the value and return it.
390 return me._expand(value, resolvep)
392 def items(me, resolvep = True):
394 Yield a list of item names in the section.
397 ## Initialize for a depth-first walk of the inheritance graph.
398 seen = { 'name': True }
399 visiting = { me.name: True }
402 ## Visit nodes, collecting their keys. Don't believe the values:
403 ## resolving inheritance is too hard to do like this.
406 for p in sec._parents():
407 if p.name not in visiting:
408 stack.append(p); visiting[p.name] = True
410 for key in sec._itemmap.iterkeys(): seen[key] = None
413 return seen.iterkeys()
415 class MyConfigParser (object):
417 A more advanced configuration parser.
419 This has four major enhancements over the standard ConfigParser which are
422 * It recognizes `@inherits' keys and follows them when expanding a
425 * It recognizes `$(VAR)' references to configuration variables during
426 expansion and processes them correctly.
428 * It recognizes `$FLAGS[HOST]' name-resolver requests and handles them
429 correctly. FLAGS consists of characters `4' (IPv4 addresses), `6'
430 (IPv6 addresses), and `*' (all, space-separated, rather than just the
433 * Its parsing behaviour is well-defined.
437 1. Call parse(FILENAME) to slurp in the configuration data.
439 2. Call resolve() to collect the hostnames which need to be resolved and
440 actually do the name resolution.
442 3. Call sections() to get a list of the configuration sections, or
443 section(NAME) to find a named section.
445 4. Call get(ITEM) on a section to collect the results, or items() to
451 Initialize a new, empty configuration parser.
454 me._resolver = BulkResolver()
458 Parse configuration from a file F.
461 ## Initial parser state.
467 ## An unpleasant hack. Python makes it hard to capture a value in a
468 ## variable and examine it in a single action, and this is the best that
471 def match(rx): m[0] = rx.match(line); return m[0]
473 ## Commit a key's value when we've determined that there are no further
474 ## continuation lines.
476 if key is not None: sect._itemmap[key] = val.getvalue()
478 ## Work through all of the input lines.
482 if match(RX_COMMENT):
483 ## A comment or a blank line. Nothing doing. (This means that we
484 ## leave out blank lines which look like they might be continuation
489 elif match(RX_GRPHDR):
490 ## A section header. Flush out any previous value and set up the new
495 try: sect = me._sectmap[name]
496 except KeyError: sect = me._sectmap[name] = ConfigSection(name, me)
499 elif match(RX_ASSGN):
500 ## A new assignment. Flush out the old one, and set up to store this
504 raise ConfigSyntaxError(f.name, lno, 'no active section to update')
507 val = StringIO(); val.write(m[0].group(2))
510 ## A continuation line. Accumulate the value.
513 raise ConfigSyntaxError(f.name, lno, 'no config value to continue')
514 val.write('\n'); val.write(m[0].group(1))
519 raise ConfigSyntaxError(f.name, lno, 'incomprehensible line')
521 ## Don't forget to commit any final value material.
524 def section(me, name):
525 """Return a ConfigSection with the given NAME."""
526 try: return me._sectmap[name]
527 except KeyError: raise MissingSectionException(name)
530 """Yield the known sections."""
531 return me._sectmap.itervalues()
535 Works out all of the hostnames which need resolving and resolves them.
537 Until you call this, attempts to fetch configuration items which need to
538 resolve hostnames will fail!
540 for sec in me.sections():
541 for key in sec.items():
542 value = sec.get(key, resolvep = False)
543 for match in RX_RESOLVE.finditer(value):
544 me._resolver.prepare(match.group(2))
547 ###--------------------------------------------------------------------------
548 ### Command-line handling.
550 def inputiter(things):
552 Iterate over command-line arguments, returning corresponding open files.
554 If none were given, or one is `-', assume standard input; if one is a
555 directory, scan it for files other than backups; otherwise return the
560 if OS.isatty(stdin.fileno()):
561 M.die('no input given, and stdin is a terminal')
567 elif OS.path.isdir(thing):
568 for item in OS.listdir(thing):
569 if item.endswith('~') or item.endswith('#'):
571 name = OS.path.join(thing, item)
572 if not OS.path.isfile(name):
578 def parse_options(argv = argv):
580 Parse command-line options, returning a pair (OPTS, ARGS).
583 op = OptionParser(usage = '%prog [-c CDB] INPUT...',
584 version = '%%prog (tripe, version %s)' % VERSION)
585 op.add_option('-c', '--cdb', metavar = 'CDB',
586 dest = 'cdbfile', default = None,
587 help = 'Compile output into a CDB file.')
588 opts, args = op.parse_args(argv)
591 ###--------------------------------------------------------------------------
596 Read the configuration files and return the accumulated result.
598 We make sure that all hostnames have been properly resolved.
600 conf = MyConfigParser()
601 for f in inputiter(args):
606 def output(conf, cdb):
608 Output the configuration information CONF to the database CDB.
610 This is where the special `user' and `auto' database entries get set.
613 for sec in sorted(conf.sections(), key = lambda sec: sec.name):
614 if sec.name.startswith('@'):
616 elif sec.name.startswith('$'):
619 label = 'P%s' % sec.name
620 try: a = sec.get('auto')
621 except MissingKeyException: pass
623 if a in ('y', 'yes', 't', 'true', '1', 'on'): auto.append(sec.name)
624 try: u = sec.get('user')
625 except MissingKeyException: pass
626 else: cdb.add('U%s' % u)
627 url = M.URLEncode(semip = True)
628 for key in sorted(sec.items()):
629 if not key.startswith('@'):
630 url.encode(key, sec.get(key))
631 cdb.add(label, url.result)
632 cdb.add('%AUTO', ' '.join(auto))
637 opts, args = parse_options()
639 cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new')
643 conf = getconf(args[1:])
645 except ExpectedError, e:
649 if __name__ == '__main__':
652 ###----- That's all, folks --------------------------------------------------