4 ### Build a CDB file from configuration file
6 ### (c) 2007 Straylight/Edgeware
9 ###----- Licensing notice ---------------------------------------------------
11 ### This file is part of Trivial IP Encryption (TrIPE).
13 ### TrIPE is free software: you can redistribute it and/or modify it under
14 ### the terms of the GNU General Public License as published by the Free
15 ### Software Foundation; either version 3 of the License, or (at your
16 ### option) any later version.
18 ### TrIPE is distributed in the hope that it will be useful, but WITHOUT
19 ### ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
20 ### FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
23 ### You should have received a copy of the GNU General Public License
24 ### along with TrIPE. If not, see <https://www.gnu.org/licenses/>.
28 ###--------------------------------------------------------------------------
29 ### External dependencies.
32 from optparse import OptionParser
34 from sys import stdin, stdout, exit, argv
38 from cStringIO import StringIO
40 ###--------------------------------------------------------------------------
43 class CDBFake (object):
44 """Like cdbmake, but just outputs data suitable for cdb-map."""
45 def __init__(me, file = stdout):
47 def add(me, key, value):
48 me.file.write('%s:%s\n' % (key, value))
52 class ExpectedError (Exception): pass
54 ###--------------------------------------------------------------------------
55 ### A bulk DNS resolver.
57 class ResolverFailure (ExpectedError):
58 def __init__(me, host, msg):
62 return "failed to resolve `%s': %s" % (me.host, me.msg)
64 class ResolvingHost (object):
66 A host name which is being looked up by a bulk-resolver instance.
68 Most notably, this is where the flag-handling logic lives for the
69 $FLAGS[HOSTNAME] syntax.
72 def __init__(me, name):
73 """Make a new resolving-host object for the host NAME."""
75 me.addr = { 'INET': [], 'INET6': [] }
78 def addaddr(me, af, addr):
80 Add the address ADDR with address family AF.
82 The address family may be `INET' or `INET6'.
84 me.addr[af].append(addr)
88 Report that resolution of this host failed, with a human-readable MSG.
93 """Return a list of addresses according to the FLAGS string."""
94 if me.failure is not None: raise ResolverFailure(me.name, me.failure)
98 all, any = False, False
100 if ch == '*': all = True
101 elif ch == '4': aa += a4; any = True
102 elif ch == '6': aa += a6; any = True
103 else: raise ValueError("unknown address-resolution flag `%s'" % ch)
104 if not any: aa = a4 + a6
105 if not aa: raise ResolverFailure(me.name, 'no matching addresses found')
106 if not all: aa = [aa[0]]
109 class BulkResolver (object):
111 Resolve a number of DNS names in parallel.
113 The BulkResovler resolves a number of hostnames in parallel. Using it
114 works in three phases:
116 1. You call prepare(HOSTNAME) a number of times, to feed in the hostnames
117 you're interested in.
119 2. You call run() to actually drive the resolver.
121 3. You call lookup(HOSTNAME) to get the address you wanted. This will
122 fail with KeyError if the resolver couldn't resolve the HOSTNAME.
126 """Initialize the resolver."""
130 def _prepare(me, host, name):
131 """Arrange to resolve a NAME, reporting the results to HOST."""
132 host._resolv = M.SelResolveByName(
134 lambda cname, alias, addr: me._resolved(host, cname, addr),
135 lambda: me._resolved(host, None, []))
138 def prepare(me, name):
139 """Prime the resolver to resolve the given host NAME."""
140 if name not in me._namemap:
141 me._namemap[name] = host = ResolvingHost(name)
143 ailist = S.getaddrinfo(name, None, S.AF_UNSPEC, S.SOCK_DGRAM, 0,
144 S.AI_NUMERICHOST | S.AI_NUMERICSERV)
146 me._prepare(host, name)
148 for af, skty, proto, cname, sa in ailist:
149 if af == S.AF_INET: host.addaddr('INET', sa[0])
150 elif af == S.AF_INET6: host.addaddr('INET6', sa[0])
153 """Run the background DNS resolver until it's finished."""
154 while me._noutstand: M.select()
156 def lookup(me, name, flags):
157 """Fetch the address corresponding to the host NAME."""
158 return me._namemap[name].get(flags)
160 def _resolved(me, host, cname, addr):
161 """Callback function: remember that ADDRs are the addresses for HOST."""
163 host.failed('(unknown failure)')
165 if cname is not None: host.name = cname
166 for a in addr: host.addaddr('INET', a)
170 ###--------------------------------------------------------------------------
171 ### The configuration parser.
173 ## Match a comment or empty line.
174 RX_COMMENT = RX.compile(r'(?x) ^ \s* (?: $ | [;#])')
176 ## Match a section group header.
177 RX_GRPHDR = RX.compile(r'(?x) ^ \s* \[ (.*) \] \s* $')
179 ## Match an assignment line.
180 RX_ASSGN = RX.compile(r'''(?x) ^
181 ([^\s:=] (?: [^:=]* [^\s:=])?)
186 ## Match a continuation line.
187 RX_CONT = RX.compile(r'''(?x) ^ \s+
191 ## Match a $(VAR) configuration variable reference; group 1 is the VAR.
192 RX_REF = RX.compile(r'(?x) \$ \( ([^)]+) \)')
194 ## Match a $FLAGS[HOST] name resolution reference; group 1 are the flags;
195 ## group 2 is the HOST.
196 RX_RESOLVE = RX.compile(r'(?x) \$ ([46*]*) \[ ([^]]+) \]')
198 class ConfigSyntaxError (ExpectedError):
199 def __init__(me, fname, lno, msg):
204 return '%s:%d: %s' % (me.fname, me.lno, me.msg)
207 return ' -> '.join(["`%s'" % hop for hop in path])
209 class AmbiguousOptionError (ExpectedError):
210 def __init__(me, key, patha, vala, pathb, valb):
212 me.patha, me.vala = patha, vala
213 me.pathb, me.valb = pathb, valb
215 return "Ambiguous answer resolving key `%s': " \
216 "path %s yields `%s' but %s yields `%s'" % \
217 (me.key, _fmt_path(me.patha), me.vala, _fmt_path(me.pathb), me.valb)
219 class InheritanceCycleError (ExpectedError):
220 def __init__(me, key, path):
224 return "Found a cycle %s looking up key `%s'" % \
225 (_fmt_path(me.path), me.key)
227 class MissingSectionException (ExpectedError):
228 def __init__(me, sec):
231 return "Section `%s' not found" % (me.sec)
233 class MissingKeyException (ExpectedError):
234 def __init__(me, sec, key):
238 return "Key `%s' not found in section `%s'" % (me.key, me.sec)
240 class ConfigSection (object):
242 A section in a configuration parser.
244 This is where a lot of the nitty-gritty stuff actually happens. The
245 `MyConfigParser' knows a lot about the internals of this class, which saves
246 on building a complicated interface.
249 def __init__(me, name, cp):
250 """Initialize a new, empty section with a given NAME and parent CP."""
252 ## The cache maps item keys to entries, which consist of a pair of
253 ## objects. There are four possible states for a cache entry:
255 ## * missing -- there is no entry at all with this key, so we must
258 ## * None, None -- we are actively trying to resolve this key, so if we
259 ## encounter this state, we have found a cycle in the inheritance
262 ## * None, [] -- we know that this key isn't reachable through any of
265 ## * VALUE, PATH -- we know that the key resolves to VALUE, along the
266 ## PATH from us (exclusive) to the defining parent (inclusive).
272 def _expand(me, string, resolvep):
274 Expands $(...) and (optionally) $FLAGS[...] placeholders in STRING.
276 RESOLVEP is a boolean switch: do we bother to tax the resolver or not?
277 This is turned off by MyConfigParser's resolve() method while it's
278 collecting hostnames to be resolved.
280 string = RX_REF.sub(lambda m: me.get(m.group(1), resolvep), string)
282 string = RX_RESOLVE.sub(
283 lambda m: ' '.join(me._cp._resolver.lookup(m.group(2), m.group(1))),
288 """Yield this section's parents."""
289 try: names = me._itemmap['@inherit']
290 except KeyError: return
291 for name in names.replace(',', ' ').split():
292 yield me._cp.section(name)
294 def _get(me, key, path = None):
296 Low-level option-fetching method.
298 Fetch the value for the named KEY in this section, or maybe (recursively)
299 a section which it inherits from.
301 Returns a pair VALUE, PATH. The value is not expanded; nor do we check
302 for the special `name' key. The caller is expected to do these things.
303 Returns None if no value could be found.
306 ## If we weren't given a path, then we'd better make one.
307 if path is None: path = []
309 ## Extend the path to cover us, but remember to remove us again when
310 ## we've finished. If we need to pass the current path back upwards,
311 ## then remember to take a copy.
315 ## If we've been this way before on another pass through then return the
316 ## value we found then. If we're still thinking about it then we've
318 try: v, p = me._cache[key]
319 except KeyError: pass
321 if p is None: raise InheritanceCycleError(key, path[:])
322 else: return v, path + p
324 ## See whether the answer is ready waiting for us.
325 try: v = me._itemmap[key]
326 except KeyError: pass
329 me._cache[key] = v, []
332 ## Initially we have no idea.
336 ## Go through our parents and ask them what they think.
337 me._cache[key] = None, None
338 for p in me._parents():
340 ## See whether we get an answer. If not, keep on going.
341 v, pp = p._get(key, path)
342 if v is None: continue
344 ## If we got an answer, check that it matches any previous ones.
349 raise AmbiguousOptionError(key, winner, value, pp, v)
351 ## That's the best we could manage.
352 me._cache[key] = value, winner[len(path):]
356 ## Remove us from the path again.
359 def get(me, key, resolvep = True):
361 Retrieve the value of KEY from this section.
364 ## Special handling for the `name' key.
366 value = me._itemmap.get('name', me.name)
367 elif key == '@inherits':
368 try: return me._itemmap['@inherits']
369 except KeyError: raise MissingKeyException(me.name, key)
371 value, _ = me._get(key)
373 raise MissingKeyException(me.name, key)
375 ## Expand the value and return it.
376 return me._expand(value, resolvep)
378 def items(me, resolvep = True):
380 Yield a list of item names in the section.
383 ## Initialize for a depth-first walk of the inheritance graph.
384 seen = { 'name': True }
385 visiting = { me.name: True }
388 ## Visit nodes, collecting their keys. Don't believe the values:
389 ## resolving inheritance is too hard to do like this.
392 for p in sec._parents():
393 if p.name not in visiting:
394 stack.append(p); visiting[p.name] = True
396 for key in sec._itemmap.iterkeys(): seen[key] = None
399 return seen.iterkeys()
401 class MyConfigParser (object):
403 A more advanced configuration parser.
405 This has four major enhancements over the standard ConfigParser which are
408 * It recognizes `@inherits' keys and follows them when expanding a
411 * It recognizes `$(VAR)' references to configuration variables during
412 expansion and processes them correctly.
414 * It recognizes `$FLAGS[HOST]' name-resolver requests and handles them
415 correctly. FLAGS consists of characters `4' (IPv4 addresses), `6'
416 (IPv6 addresses), and `*' (all, space-separated, rather than just the
419 * Its parsing behaviour is well-defined.
423 1. Call parse(FILENAME) to slurp in the configuration data.
425 2. Call resolve() to collect the hostnames which need to be resolved and
426 actually do the name resolution.
428 3. Call sections() to get a list of the configuration sections, or
429 section(NAME) to find a named section.
431 4. Call get(ITEM) on a section to collect the results, or items() to
437 Initialize a new, empty configuration parser.
440 me._resolver = BulkResolver()
444 Parse configuration from a file F.
447 ## Initial parser state.
453 ## An unpleasant hack. Python makes it hard to capture a value in a
454 ## variable and examine it in a single action, and this is the best that
457 def match(rx): m[0] = rx.match(line); return m[0]
459 ## Commit a key's value when we've determined that there are no further
460 ## continuation lines.
462 if key is not None: sect._itemmap[key] = val.getvalue()
464 ## Work through all of the input lines.
468 if match(RX_COMMENT):
469 ## A comment or a blank line. Nothing doing. (This means that we
470 ## leave out blank lines which look like they might be continuation
475 elif match(RX_GRPHDR):
476 ## A section header. Flush out any previous value and set up the new
481 try: sect = me._sectmap[name]
482 except KeyError: sect = me._sectmap[name] = ConfigSection(name, me)
485 elif match(RX_ASSGN):
486 ## A new assignment. Flush out the old one, and set up to store this
490 raise ConfigSyntaxError(f.name, lno, 'no active section to update')
493 val = StringIO(); val.write(m[0].group(2))
496 ## A continuation line. Accumulate the value.
499 raise ConfigSyntaxError(f.name, lno, 'no config value to continue')
500 val.write('\n'); val.write(m[0].group(1))
505 raise ConfigSyntaxError(f.name, lno, 'incomprehensible line')
507 ## Don't forget to commit any final value material.
510 def section(me, name):
511 """Return a ConfigSection with the given NAME."""
512 try: return me._sectmap[name]
513 except KeyError: raise MissingSectionException(name)
516 """Yield the known sections."""
517 return me._sectmap.itervalues()
521 Works out all of the hostnames which need resolving and resolves them.
523 Until you call this, attempts to fetch configuration items which need to
524 resolve hostnames will fail!
526 for sec in me.sections():
527 for key in sec.items():
528 value = sec.get(key, resolvep = False)
529 for match in RX_RESOLVE.finditer(value):
530 me._resolver.prepare(match.group(2))
533 ###--------------------------------------------------------------------------
534 ### Command-line handling.
536 def inputiter(things):
538 Iterate over command-line arguments, returning corresponding open files.
540 If none were given, or one is `-', assume standard input; if one is a
541 directory, scan it for files other than backups; otherwise return the
546 if OS.isatty(stdin.fileno()):
547 M.die('no input given, and stdin is a terminal')
553 elif OS.path.isdir(thing):
554 for item in OS.listdir(thing):
555 if item.endswith('~') or item.endswith('#'):
557 name = OS.path.join(thing, item)
558 if not OS.path.isfile(name):
564 def parse_options(argv = argv):
566 Parse command-line options, returning a pair (OPTS, ARGS).
569 op = OptionParser(usage = '%prog [-c CDB] INPUT...',
570 version = '%%prog (tripe, version %s)' % VERSION)
571 op.add_option('-c', '--cdb', metavar = 'CDB',
572 dest = 'cdbfile', default = None,
573 help = 'Compile output into a CDB file.')
574 opts, args = op.parse_args(argv)
577 ###--------------------------------------------------------------------------
582 Read the configuration files and return the accumulated result.
584 We make sure that all hostnames have been properly resolved.
586 conf = MyConfigParser()
587 for f in inputiter(args):
592 def output(conf, cdb):
594 Output the configuration information CONF to the database CDB.
596 This is where the special `user' and `auto' database entries get set.
599 for sec in sorted(conf.sections(), key = lambda sec: sec.name):
600 if sec.name.startswith('@'):
602 elif sec.name.startswith('$'):
605 label = 'P%s' % sec.name
606 try: a = sec.get('auto')
607 except MissingKeyException: pass
609 if a in ('y', 'yes', 't', 'true', '1', 'on'): auto.append(sec.name)
610 try: u = sec.get('user')
611 except MissingKeyException: pass
612 else: cdb.add('U%s' % u)
613 url = M.URLEncode(semip = True)
614 for key in sorted(sec.items()):
615 if not key.startswith('@'):
616 url.encode(key, sec.get(key))
617 cdb.add(label, url.result)
618 cdb.add('%AUTO', ' '.join(auto))
623 opts, args = parse_options()
625 cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new')
629 conf = getconf(args[1:])
631 except ExpectedError, e:
635 if __name__ == '__main__':
638 ###----- That's all, folks --------------------------------------------------