4 ### Build a CDB file from configuration file
6 ### (c) 2007 Straylight/Edgeware
9 ###----- Licensing notice ---------------------------------------------------
11 ### This file is part of Trivial IP Encryption (TrIPE).
13 ### TrIPE is free software: you can redistribute it and/or modify it under
14 ### the terms of the GNU General Public License as published by the Free
15 ### Software Foundation; either version 3 of the License, or (at your
16 ### option) any later version.
18 ### TrIPE is distributed in the hope that it will be useful, but WITHOUT
19 ### ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
20 ### FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
23 ### You should have received a copy of the GNU General Public License
24 ### along with TrIPE. If not, see <https://www.gnu.org/licenses/>.
28 ###--------------------------------------------------------------------------
29 ### External dependencies.
32 from optparse import OptionParser
34 from sys import stdin, stdout, exit, argv
37 from cStringIO import StringIO
39 ###--------------------------------------------------------------------------
42 class CDBFake (object):
43 """Like cdbmake, but just outputs data suitable for cdb-map."""
44 def __init__(me, file = stdout):
46 def add(me, key, value):
47 me.file.write('%s:%s\n' % (key, value))
51 ###--------------------------------------------------------------------------
52 ### A bulk DNS resolver.
54 class BulkResolver (object):
56 Resolve a number of DNS names in parallel.
58 The BulkResovler resolves a number of hostnames in parallel. Using it
59 works in three phases:
61 1. You call prepare(HOSTNAME) a number of times, to feed in the hostnames
64 2. You call run() to actually drive the resolver.
66 3. You call lookup(HOSTNAME) to get the address you wanted. This will
67 fail with KeyError if the resolver couldn't resolve the HOSTNAME.
71 """Initialize the resolver."""
75 def prepare(me, host):
76 """Prime the resolver to resolve the name HOST."""
77 if host not in me._resolvers:
78 me._resolvers[host] = M.SelResolveByName \
80 lambda name, alias, addr:
81 me._resolved(host, addr[0]),
82 lambda: me._resolved(host, None))
85 """Run the background DNS resolver until it's finished."""
91 Fetch the address corresponding to HOST.
93 addr = me._namemap[host]
98 def _resolved(me, host, addr):
99 """Callback function: remember that ADDR is the address for HOST."""
100 me._namemap[host] = addr
101 del me._resolvers[host]
103 ###--------------------------------------------------------------------------
104 ### The configuration parser.
106 ## Match a comment or empty line.
107 RX_COMMENT = RX.compile(r'(?x) ^ \s* (?: $ | [;#])')
109 ## Match a section group header.
110 RX_GRPHDR = RX.compile(r'(?x) ^ \s* \[ (.*) \] \s* $')
112 ## Match an assignment line.
113 RX_ASSGN = RX.compile(r'''(?x) ^
114 ([^\s:=] (?: [^:=]* [^\s:=])?)
119 ## Match a continuation line.
120 RX_CONT = RX.compile(r'''(?x) ^ \s+
124 ## Match a $(VAR) configuration variable reference; group 1 is the VAR.
125 RX_REF = RX.compile(r'(?x) \$ \( ([^)]+) \)')
127 ## Match a $[HOST] name resolution reference; group 1 is the HOST.
128 RX_RESOLVE = RX.compile(r'(?x) \$ \[ ([^]]+) \]')
130 class ConfigSyntaxError (Exception):
131 def __init__(me, fname, lno, msg):
136 return '%s:%d: %s' % (me.fname, me.lno, me.msg)
139 return ' -> '.join(["`%s'" % hop for hop in path])
141 class AmbiguousOptionError (Exception):
142 def __init__(me, key, patha, vala, pathb, valb):
144 me.patha, me.vala = patha, vala
145 me.pathb, me.valb = pathb, valb
147 return "Ambiguous answer resolving key `%s': " \
148 "path %s yields `%s' but %s yields `%s'" % \
149 (me.key, _fmt_path(me.patha), me.vala, _fmt_path(me.pathb), me.valb)
151 class InheritanceCycleError (Exception):
152 def __init__(me, key, path):
156 return "Found a cycle %s looking up key `%s'" % \
157 (_fmt_path(me.path), me.key)
159 class MissingSectionException (Exception):
160 def __init__(me, sec):
163 return "Section `%s' not found" % (me.sec)
165 class MissingKeyException (Exception):
166 def __init__(me, sec, key):
170 return "Key `%s' not found in section `%s'" % (me.key, me.sec)
172 class ConfigSection (object):
174 A section in a configuration parser.
176 This is where a lot of the nitty-gritty stuff actually happens. The
177 `MyConfigParser' knows a lot about the internals of this class, which saves
178 on building a complicated interface.
181 def __init__(me, name, cp):
182 """Initialize a new, empty section with a given NAME and parent CP."""
184 ## The cache maps item keys to entries, which consist of a pair of
185 ## objects. There are four possible states for a cache entry:
187 ## * missing -- there is no entry at all with this key, so we must
190 ## * None, None -- we are actively trying to resolve this key, so if we
191 ## encounter this state, we have found a cycle in the inheritance
194 ## * None, [] -- we know that this key isn't reachable through any of
197 ## * VALUE, PATH -- we know that the key resolves to VALUE, along the
198 ## PATH from us (exclusive) to the defining parent (inclusive).
204 def _expand(me, string, resolvep):
206 Expands $(...) and (optionally) $[...] placeholders in STRING.
208 RESOLVEP is a boolean switch: do we bother to tax the resolver or not?
209 This is turned off by MyConfigParser's resolve() method while it's
210 collecting hostnames to be resolved.
212 string = RX_REF.sub \
213 (lambda m: me.get(m.group(1), resolvep), string)
215 string = RX_RESOLVE.sub(lambda m: me._cp._resolver.lookup(m.group(1)),
220 """Yield this section's parents."""
221 try: names = me._itemmap['@inherit']
222 except KeyError: return
223 for name in names.replace(',', ' ').split():
224 yield me._cp.section(name)
226 def _get(me, key, path = None):
228 Low-level option-fetching method.
230 Fetch the value for the named KEY in this section, or maybe (recursively)
231 a section which it inherits from.
233 Returns a pair VALUE, PATH. The value is not expanded; nor do we check
234 for the special `name' key. The caller is expected to do these things.
235 Returns None if no value could be found.
238 ## If we weren't given a path, then we'd better make one.
239 if path is None: path = []
241 ## Extend the path to cover us, but remember to remove us again when
242 ## we've finished. If we need to pass the current path back upwards,
243 ## then remember to take a copy.
247 ## If we've been this way before on another pass through then return the
248 ## value we found then. If we're still thinking about it then we've
250 try: v, p = me._cache[key]
251 except KeyError: pass
253 if p is None: raise InheritanceCycleError(key, path[:])
254 else: return v, path + p
256 ## See whether the answer is ready waiting for us.
257 try: v = me._itemmap[key]
258 except KeyError: pass
261 me._cache[key] = v, []
264 ## Initially we have no idea.
268 ## Go through our parents and ask them what they think.
269 me._cache[key] = None, None
270 for p in me._parents():
272 ## See whether we get an answer. If not, keep on going.
273 v, pp = p._get(key, path)
274 if v is None: continue
276 ## If we got an answer, check that it matches any previous ones.
281 raise AmbiguousOptionError(key, winner, value, pp, v)
283 ## That's the best we could manage.
284 me._cache[key] = value, winner[len(path):]
288 ## Remove us from the path again.
291 def get(me, key, resolvep = True):
293 Retrieve the value of KEY from this section.
296 ## Special handling for the `name' key.
298 value = me._itemmap.get('name', me.name)
299 elif key == '@inherits':
300 try: return me._itemmap['@inherits']
301 except KeyError: raise MissingKeyException(me.name, key)
303 value, _ = me._get(key)
305 raise MissingKeyException(me.name, key)
307 ## Expand the value and return it.
308 return me._expand(value, resolvep)
310 def items(me, resolvep = True):
312 Yield a list of item names in the section.
315 ## Initialize for a depth-first walk of the inheritance graph.
316 seen = { 'name': True }
317 visiting = { me.name: True }
320 ## Visit nodes, collecting their keys. Don't believe the values:
321 ## resolving inheritance is too hard to do like this.
324 for p in sec._parents():
325 if p.name not in visiting:
326 stack.append(p); visiting[p.name] = True
328 for key in sec._itemmap.iterkeys(): seen[key] = None
331 return seen.iterkeys()
333 class MyConfigParser (object):
335 A more advanced configuration parser.
337 This has four major enhancements over the standard ConfigParser which are
340 * It recognizes `@inherits' keys and follows them when expanding a
343 * It recognizes `$(VAR)' references to configuration variables during
344 expansion and processes them correctly.
346 * It recognizes `$[HOST]' name-resolver requests and handles them
349 * Its parsing behaviour is well-defined.
353 1. Call parse(FILENAME) to slurp in the configuration data.
355 2. Call resolve() to collect the hostnames which need to be resolved and
356 actually do the name resolution.
358 3. Call sections() to get a list of the configuration sections, or
359 section(NAME) to find a named section.
361 4. Call get(ITEM) on a section to collect the results, or items() to
367 Initialize a new, empty configuration parser.
370 me._resolver = BulkResolver()
374 Parse configuration from a file F.
377 ## Initial parser state.
383 ## An unpleasant hack. Python makes it hard to capture a value in a
384 ## variable and examine it in a single action, and this is the best that
387 def match(rx): m[0] = rx.match(line); return m[0]
389 ## Commit a key's value when we've determined that there are no further
390 ## continuation lines.
392 if key is not None: sect._itemmap[key] = val.getvalue()
394 ## Work through all of the input lines.
398 if match(RX_COMMENT):
399 ## A comment or a blank line. Nothing doing. (This means that we
400 ## leave out blank lines which look like they might be continuation
405 elif match(RX_GRPHDR):
406 ## A section header. Flush out any previous value and set up the new
411 try: sect = me._sectmap[name]
412 except KeyError: sect = me._sectmap[name] = ConfigSection(name, me)
415 elif match(RX_ASSGN):
416 ## A new assignment. Flush out the old one, and set up to store this
420 raise ConfigSyntaxError(f.name, lno, 'no active section to update')
423 val = StringIO(); val.write(m[0].group(2))
426 ## A continuation line. Accumulate the value.
429 raise ConfigSyntaxError(f.name, lno, 'no config value to continue')
430 val.write('\n'); val.write(m[0].group(1))
435 raise ConfigSyntaxError(f.name, lno, 'incomprehensible line')
437 ## Don't forget to commit any final value material.
440 def section(me, name):
441 """Return a ConfigSection with the given NAME."""
442 try: return me._sectmap[name]
443 except KeyError: raise MissingSectionException(name)
446 """Yield the known sections."""
447 return me._sectmap.itervalues()
451 Works out all of the hostnames which need resolving and resolves them.
453 Until you call this, attempts to fetch configuration items which need to
454 resolve hostnames will fail!
456 for sec in me.sections():
457 for key in sec.items():
458 value = sec.get(key, resolvep = False)
459 for match in RX_RESOLVE.finditer(value):
460 me._resolver.prepare(match.group(1))
463 ###--------------------------------------------------------------------------
464 ### Command-line handling.
466 def inputiter(things):
468 Iterate over command-line arguments, returning corresponding open files.
470 If none were given, or one is `-', assume standard input; if one is a
471 directory, scan it for files other than backups; otherwise return the
476 if OS.isatty(stdin.fileno()):
477 M.die('no input given, and stdin is a terminal')
483 elif OS.path.isdir(thing):
484 for item in OS.listdir(thing):
485 if item.endswith('~') or item.endswith('#'):
487 name = OS.path.join(thing, item)
488 if not OS.path.isfile(name):
494 def parse_options(argv = argv):
496 Parse command-line options, returning a pair (OPTS, ARGS).
499 op = OptionParser(usage = '%prog [-c CDB] INPUT...',
500 version = '%%prog (tripe, version %s)' % VERSION)
501 op.add_option('-c', '--cdb', metavar = 'CDB',
502 dest = 'cdbfile', default = None,
503 help = 'Compile output into a CDB file.')
504 opts, args = op.parse_args(argv)
507 ###--------------------------------------------------------------------------
512 Read the configuration files and return the accumulated result.
514 We make sure that all hostnames have been properly resolved.
516 conf = MyConfigParser()
517 for f in inputiter(args):
522 def output(conf, cdb):
524 Output the configuration information CONF to the database CDB.
526 This is where the special `user' and `auto' database entries get set.
529 for sec in sorted(conf.sections(), key = lambda sec: sec.name):
530 if sec.name.startswith('@'):
532 elif sec.name.startswith('$'):
535 label = 'P%s' % sec.name
536 try: a = sec.get('auto')
537 except MissingKeyException: pass
539 if a in ('y', 'yes', 't', 'true', '1', 'on'): auto.append(sec.name)
540 try: u = sec.get('user')
541 except MissingKeyException: pass
542 else: cdb.add('U%s' % u)
543 url = M.URLEncode(semip = True)
544 for key in sorted(sec.items()):
545 if not key.startswith('@'):
546 url.encode(key, sec.get(key))
547 cdb.add(label, url.result)
548 cdb.add('%AUTO', ' '.join(auto))
553 opts, args = parse_options()
555 cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new')
558 conf = getconf(args[1:])
561 if __name__ == '__main__':
564 ###----- That's all, folks --------------------------------------------------