4 ### Build a CDB file from configuration file
6 ### (c) 2007 Straylight/Edgeware
9 ###----- Licensing notice ---------------------------------------------------
11 ### This file is part of Trivial IP Encryption (TrIPE).
13 ### TrIPE is free software: you can redistribute it and/or modify it under
14 ### the terms of the GNU General Public License as published by the Free
15 ### Software Foundation; either version 3 of the License, or (at your
16 ### option) any later version.
18 ### TrIPE is distributed in the hope that it will be useful, but WITHOUT
19 ### ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
20 ### FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
23 ### You should have received a copy of the GNU General Public License
24 ### along with TrIPE. If not, see <https://www.gnu.org/licenses/>.
28 ###--------------------------------------------------------------------------
29 ### External dependencies.
32 from optparse import OptionParser
34 from sys import stdin, stdout, exit, argv
37 from cStringIO import StringIO
39 ###--------------------------------------------------------------------------
42 class CDBFake (object):
43 """Like cdbmake, but just outputs data suitable for cdb-map."""
44 def __init__(me, file = stdout):
46 def add(me, key, value):
47 me.file.write('%s:%s\n' % (key, value))
51 class ExpectedError (Exception): pass
53 ###--------------------------------------------------------------------------
54 ### A bulk DNS resolver.
56 class ResolverFailure (ExpectedError):
57 def __init__(me, host, msg):
61 return "failed to resolve `%s': %s" % (me.host, me.msg)
63 class BulkResolver (object):
65 Resolve a number of DNS names in parallel.
67 The BulkResovler resolves a number of hostnames in parallel. Using it
68 works in three phases:
70 1. You call prepare(HOSTNAME) a number of times, to feed in the hostnames
73 2. You call run() to actually drive the resolver.
75 3. You call lookup(HOSTNAME) to get the address you wanted. This will
76 fail with KeyError if the resolver couldn't resolve the HOSTNAME.
80 """Initialize the resolver."""
84 def prepare(me, host):
85 """Prime the resolver to resolve the name HOST."""
86 if host not in me._resolvers:
87 me._resolvers[host] = M.SelResolveByName \
89 lambda name, alias, addr:
90 me._resolved(host, addr[0]),
91 lambda: me._resolved(host, None))
94 """Run the background DNS resolver until it's finished."""
100 Fetch the address corresponding to HOST.
102 addr = me._namemap[host]
104 raise ResolverFailure(host, '(unknown failure)')
107 def _resolved(me, host, addr):
108 """Callback function: remember that ADDR is the address for HOST."""
109 me._namemap[host] = addr
110 del me._resolvers[host]
112 ###--------------------------------------------------------------------------
113 ### The configuration parser.
115 ## Match a comment or empty line.
116 RX_COMMENT = RX.compile(r'(?x) ^ \s* (?: $ | [;#])')
118 ## Match a section group header.
119 RX_GRPHDR = RX.compile(r'(?x) ^ \s* \[ (.*) \] \s* $')
121 ## Match an assignment line.
122 RX_ASSGN = RX.compile(r'''(?x) ^
123 ([^\s:=] (?: [^:=]* [^\s:=])?)
128 ## Match a continuation line.
129 RX_CONT = RX.compile(r'''(?x) ^ \s+
133 ## Match a $(VAR) configuration variable reference; group 1 is the VAR.
134 RX_REF = RX.compile(r'(?x) \$ \( ([^)]+) \)')
136 ## Match a $[HOST] name resolution reference; group 1 is the HOST.
137 RX_RESOLVE = RX.compile(r'(?x) \$ \[ ([^]]+) \]')
139 class ConfigSyntaxError (ExpectedError):
140 def __init__(me, fname, lno, msg):
145 return '%s:%d: %s' % (me.fname, me.lno, me.msg)
148 return ' -> '.join(["`%s'" % hop for hop in path])
150 class AmbiguousOptionError (ExpectedError):
151 def __init__(me, key, patha, vala, pathb, valb):
153 me.patha, me.vala = patha, vala
154 me.pathb, me.valb = pathb, valb
156 return "Ambiguous answer resolving key `%s': " \
157 "path %s yields `%s' but %s yields `%s'" % \
158 (me.key, _fmt_path(me.patha), me.vala, _fmt_path(me.pathb), me.valb)
160 class InheritanceCycleError (ExpectedError):
161 def __init__(me, key, path):
165 return "Found a cycle %s looking up key `%s'" % \
166 (_fmt_path(me.path), me.key)
168 class MissingSectionException (ExpectedError):
169 def __init__(me, sec):
172 return "Section `%s' not found" % (me.sec)
174 class MissingKeyException (ExpectedError):
175 def __init__(me, sec, key):
179 return "Key `%s' not found in section `%s'" % (me.key, me.sec)
181 class ConfigSection (object):
183 A section in a configuration parser.
185 This is where a lot of the nitty-gritty stuff actually happens. The
186 `MyConfigParser' knows a lot about the internals of this class, which saves
187 on building a complicated interface.
190 def __init__(me, name, cp):
191 """Initialize a new, empty section with a given NAME and parent CP."""
193 ## The cache maps item keys to entries, which consist of a pair of
194 ## objects. There are four possible states for a cache entry:
196 ## * missing -- there is no entry at all with this key, so we must
199 ## * None, None -- we are actively trying to resolve this key, so if we
200 ## encounter this state, we have found a cycle in the inheritance
203 ## * None, [] -- we know that this key isn't reachable through any of
206 ## * VALUE, PATH -- we know that the key resolves to VALUE, along the
207 ## PATH from us (exclusive) to the defining parent (inclusive).
213 def _expand(me, string, resolvep):
215 Expands $(...) and (optionally) $[...] placeholders in STRING.
217 RESOLVEP is a boolean switch: do we bother to tax the resolver or not?
218 This is turned off by MyConfigParser's resolve() method while it's
219 collecting hostnames to be resolved.
221 string = RX_REF.sub \
222 (lambda m: me.get(m.group(1), resolvep), string)
224 string = RX_RESOLVE.sub(lambda m: me._cp._resolver.lookup(m.group(1)),
229 """Yield this section's parents."""
230 try: names = me._itemmap['@inherit']
231 except KeyError: return
232 for name in names.replace(',', ' ').split():
233 yield me._cp.section(name)
235 def _get(me, key, path = None):
237 Low-level option-fetching method.
239 Fetch the value for the named KEY in this section, or maybe (recursively)
240 a section which it inherits from.
242 Returns a pair VALUE, PATH. The value is not expanded; nor do we check
243 for the special `name' key. The caller is expected to do these things.
244 Returns None if no value could be found.
247 ## If we weren't given a path, then we'd better make one.
248 if path is None: path = []
250 ## Extend the path to cover us, but remember to remove us again when
251 ## we've finished. If we need to pass the current path back upwards,
252 ## then remember to take a copy.
256 ## If we've been this way before on another pass through then return the
257 ## value we found then. If we're still thinking about it then we've
259 try: v, p = me._cache[key]
260 except KeyError: pass
262 if p is None: raise InheritanceCycleError(key, path[:])
263 else: return v, path + p
265 ## See whether the answer is ready waiting for us.
266 try: v = me._itemmap[key]
267 except KeyError: pass
270 me._cache[key] = v, []
273 ## Initially we have no idea.
277 ## Go through our parents and ask them what they think.
278 me._cache[key] = None, None
279 for p in me._parents():
281 ## See whether we get an answer. If not, keep on going.
282 v, pp = p._get(key, path)
283 if v is None: continue
285 ## If we got an answer, check that it matches any previous ones.
290 raise AmbiguousOptionError(key, winner, value, pp, v)
292 ## That's the best we could manage.
293 me._cache[key] = value, winner[len(path):]
297 ## Remove us from the path again.
300 def get(me, key, resolvep = True):
302 Retrieve the value of KEY from this section.
305 ## Special handling for the `name' key.
307 value = me._itemmap.get('name', me.name)
308 elif key == '@inherits':
309 try: return me._itemmap['@inherits']
310 except KeyError: raise MissingKeyException(me.name, key)
312 value, _ = me._get(key)
314 raise MissingKeyException(me.name, key)
316 ## Expand the value and return it.
317 return me._expand(value, resolvep)
319 def items(me, resolvep = True):
321 Yield a list of item names in the section.
324 ## Initialize for a depth-first walk of the inheritance graph.
325 seen = { 'name': True }
326 visiting = { me.name: True }
329 ## Visit nodes, collecting their keys. Don't believe the values:
330 ## resolving inheritance is too hard to do like this.
333 for p in sec._parents():
334 if p.name not in visiting:
335 stack.append(p); visiting[p.name] = True
337 for key in sec._itemmap.iterkeys(): seen[key] = None
340 return seen.iterkeys()
342 class MyConfigParser (object):
344 A more advanced configuration parser.
346 This has four major enhancements over the standard ConfigParser which are
349 * It recognizes `@inherits' keys and follows them when expanding a
352 * It recognizes `$(VAR)' references to configuration variables during
353 expansion and processes them correctly.
355 * It recognizes `$[HOST]' name-resolver requests and handles them
358 * Its parsing behaviour is well-defined.
362 1. Call parse(FILENAME) to slurp in the configuration data.
364 2. Call resolve() to collect the hostnames which need to be resolved and
365 actually do the name resolution.
367 3. Call sections() to get a list of the configuration sections, or
368 section(NAME) to find a named section.
370 4. Call get(ITEM) on a section to collect the results, or items() to
376 Initialize a new, empty configuration parser.
379 me._resolver = BulkResolver()
383 Parse configuration from a file F.
386 ## Initial parser state.
392 ## An unpleasant hack. Python makes it hard to capture a value in a
393 ## variable and examine it in a single action, and this is the best that
396 def match(rx): m[0] = rx.match(line); return m[0]
398 ## Commit a key's value when we've determined that there are no further
399 ## continuation lines.
401 if key is not None: sect._itemmap[key] = val.getvalue()
403 ## Work through all of the input lines.
407 if match(RX_COMMENT):
408 ## A comment or a blank line. Nothing doing. (This means that we
409 ## leave out blank lines which look like they might be continuation
414 elif match(RX_GRPHDR):
415 ## A section header. Flush out any previous value and set up the new
420 try: sect = me._sectmap[name]
421 except KeyError: sect = me._sectmap[name] = ConfigSection(name, me)
424 elif match(RX_ASSGN):
425 ## A new assignment. Flush out the old one, and set up to store this
429 raise ConfigSyntaxError(f.name, lno, 'no active section to update')
432 val = StringIO(); val.write(m[0].group(2))
435 ## A continuation line. Accumulate the value.
438 raise ConfigSyntaxError(f.name, lno, 'no config value to continue')
439 val.write('\n'); val.write(m[0].group(1))
444 raise ConfigSyntaxError(f.name, lno, 'incomprehensible line')
446 ## Don't forget to commit any final value material.
449 def section(me, name):
450 """Return a ConfigSection with the given NAME."""
451 try: return me._sectmap[name]
452 except KeyError: raise MissingSectionException(name)
455 """Yield the known sections."""
456 return me._sectmap.itervalues()
460 Works out all of the hostnames which need resolving and resolves them.
462 Until you call this, attempts to fetch configuration items which need to
463 resolve hostnames will fail!
465 for sec in me.sections():
466 for key in sec.items():
467 value = sec.get(key, resolvep = False)
468 for match in RX_RESOLVE.finditer(value):
469 me._resolver.prepare(match.group(1))
472 ###--------------------------------------------------------------------------
473 ### Command-line handling.
475 def inputiter(things):
477 Iterate over command-line arguments, returning corresponding open files.
479 If none were given, or one is `-', assume standard input; if one is a
480 directory, scan it for files other than backups; otherwise return the
485 if OS.isatty(stdin.fileno()):
486 M.die('no input given, and stdin is a terminal')
492 elif OS.path.isdir(thing):
493 for item in OS.listdir(thing):
494 if item.endswith('~') or item.endswith('#'):
496 name = OS.path.join(thing, item)
497 if not OS.path.isfile(name):
503 def parse_options(argv = argv):
505 Parse command-line options, returning a pair (OPTS, ARGS).
508 op = OptionParser(usage = '%prog [-c CDB] INPUT...',
509 version = '%%prog (tripe, version %s)' % VERSION)
510 op.add_option('-c', '--cdb', metavar = 'CDB',
511 dest = 'cdbfile', default = None,
512 help = 'Compile output into a CDB file.')
513 opts, args = op.parse_args(argv)
516 ###--------------------------------------------------------------------------
521 Read the configuration files and return the accumulated result.
523 We make sure that all hostnames have been properly resolved.
525 conf = MyConfigParser()
526 for f in inputiter(args):
531 def output(conf, cdb):
533 Output the configuration information CONF to the database CDB.
535 This is where the special `user' and `auto' database entries get set.
538 for sec in sorted(conf.sections(), key = lambda sec: sec.name):
539 if sec.name.startswith('@'):
541 elif sec.name.startswith('$'):
544 label = 'P%s' % sec.name
545 try: a = sec.get('auto')
546 except MissingKeyException: pass
548 if a in ('y', 'yes', 't', 'true', '1', 'on'): auto.append(sec.name)
549 try: u = sec.get('user')
550 except MissingKeyException: pass
551 else: cdb.add('U%s' % u)
552 url = M.URLEncode(semip = True)
553 for key in sorted(sec.items()):
554 if not key.startswith('@'):
555 url.encode(key, sec.get(key))
556 cdb.add(label, url.result)
557 cdb.add('%AUTO', ' '.join(auto))
562 opts, args = parse_options()
564 cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new')
568 conf = getconf(args[1:])
570 except ExpectedError, e:
574 if __name__ == '__main__':
577 ###----- That's all, folks --------------------------------------------------