| 1 | #! @PYTHON@ |
| 2 | ### -*-python-*- |
| 3 | ### |
| 4 | ### Build a CDB file from configuration file |
| 5 | ### |
| 6 | ### (c) 2007 Straylight/Edgeware |
| 7 | ### |
| 8 | |
| 9 | ###----- Licensing notice --------------------------------------------------- |
| 10 | ### |
| 11 | ### This file is part of Trivial IP Encryption (TrIPE). |
| 12 | ### |
| 13 | ### TrIPE is free software: you can redistribute it and/or modify it under |
| 14 | ### the terms of the GNU General Public License as published by the Free |
| 15 | ### Software Foundation; either version 3 of the License, or (at your |
| 16 | ### option) any later version. |
| 17 | ### |
| 18 | ### TrIPE is distributed in the hope that it will be useful, but WITHOUT |
| 19 | ### ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 20 | ### FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| 21 | ### for more details. |
| 22 | ### |
| 23 | ### You should have received a copy of the GNU General Public License |
| 24 | ### along with TrIPE. If not, see <https://www.gnu.org/licenses/>. |
| 25 | |
| 26 | VERSION = '@VERSION@' |
| 27 | |
| 28 | ###-------------------------------------------------------------------------- |
| 29 | ### External dependencies. |
| 30 | |
| 31 | import mLib as M |
| 32 | from optparse import OptionParser |
| 33 | import cdb as CDB |
| 34 | from sys import stdin, stdout, exit, argv |
| 35 | import re as RX |
| 36 | import os as OS |
| 37 | from cStringIO import StringIO |
| 38 | |
| 39 | ###-------------------------------------------------------------------------- |
| 40 | ### Utilities. |
| 41 | |
| 42 | class CDBFake (object): |
| 43 | """Like cdbmake, but just outputs data suitable for cdb-map.""" |
| 44 | def __init__(me, file = stdout): |
| 45 | me.file = file |
| 46 | def add(me, key, value): |
| 47 | me.file.write('%s:%s\n' % (key, value)) |
| 48 | def finish(me): |
| 49 | pass |
| 50 | |
| 51 | ###-------------------------------------------------------------------------- |
| 52 | ### A bulk DNS resolver. |
| 53 | |
| 54 | class BulkResolver (object): |
| 55 | """ |
| 56 | Resolve a number of DNS names in parallel. |
| 57 | |
| 58 | The BulkResovler resolves a number of hostnames in parallel. Using it |
| 59 | works in three phases: |
| 60 | |
| 61 | 1. You call prepare(HOSTNAME) a number of times, to feed in the hostnames |
| 62 | you're interested in. |
| 63 | |
| 64 | 2. You call run() to actually drive the resolver. |
| 65 | |
| 66 | 3. You call lookup(HOSTNAME) to get the address you wanted. This will |
| 67 | fail with KeyError if the resolver couldn't resolve the HOSTNAME. |
| 68 | """ |
| 69 | |
| 70 | def __init__(me): |
| 71 | """Initialize the resolver.""" |
| 72 | me._resolvers = {} |
| 73 | me._namemap = {} |
| 74 | |
| 75 | def prepare(me, host): |
| 76 | """Prime the resolver to resolve the name HOST.""" |
| 77 | if host not in me._resolvers: |
| 78 | me._resolvers[host] = M.SelResolveByName \ |
| 79 | (host, |
| 80 | lambda name, alias, addr: |
| 81 | me._resolved(host, addr[0]), |
| 82 | lambda: me._resolved(host, None)) |
| 83 | |
| 84 | def run(me): |
| 85 | """Run the background DNS resolver until it's finished.""" |
| 86 | while me._resolvers: |
| 87 | M.select() |
| 88 | |
| 89 | def lookup(me, host): |
| 90 | """ |
| 91 | Fetch the address corresponding to HOST. |
| 92 | """ |
| 93 | addr = me._namemap[host] |
| 94 | if addr is None: |
| 95 | raise KeyError(host) |
| 96 | return addr |
| 97 | |
| 98 | def _resolved(me, host, addr): |
| 99 | """Callback function: remember that ADDR is the address for HOST.""" |
| 100 | me._namemap[host] = addr |
| 101 | del me._resolvers[host] |
| 102 | |
| 103 | ###-------------------------------------------------------------------------- |
| 104 | ### The configuration parser. |
| 105 | |
| 106 | ## Match a comment or empty line. |
| 107 | RX_COMMENT = RX.compile(r'(?x) ^ \s* (?: $ | [;#])') |
| 108 | |
| 109 | ## Match a section group header. |
| 110 | RX_GRPHDR = RX.compile(r'(?x) ^ \s* \[ (.*) \] \s* $') |
| 111 | |
| 112 | ## Match an assignment line. |
| 113 | RX_ASSGN = RX.compile(r'''(?x) ^ |
| 114 | ([^\s:=] (?: [^:=]* [^\s:=])?) |
| 115 | \s* [:=] \s* |
| 116 | (| \S | \S.*\S) |
| 117 | \s* $''') |
| 118 | |
| 119 | ## Match a continuation line. |
| 120 | RX_CONT = RX.compile(r'''(?x) ^ \s+ |
| 121 | (| \S | \S.*\S) |
| 122 | \s* $''') |
| 123 | |
| 124 | ## Match a $(VAR) configuration variable reference; group 1 is the VAR. |
| 125 | RX_REF = RX.compile(r'(?x) \$ \( ([^)]+) \)') |
| 126 | |
| 127 | ## Match a $[HOST] name resolution reference; group 1 is the HOST. |
| 128 | RX_RESOLVE = RX.compile(r'(?x) \$ \[ ([^]]+) \]') |
| 129 | |
| 130 | class ConfigSyntaxError (Exception): |
| 131 | def __init__(me, fname, lno, msg): |
| 132 | me.fname = fname |
| 133 | me.lno = lno |
| 134 | me.msg = msg |
| 135 | def __str__(me): |
| 136 | return '%s:%d: %s' % (me.fname, me.lno, me.msg) |
| 137 | |
| 138 | def _fmt_path(path): |
| 139 | return ' -> '.join(["`%s'" % hop for hop in path]) |
| 140 | |
| 141 | class AmbiguousOptionError (Exception): |
| 142 | def __init__(me, key, patha, vala, pathb, valb): |
| 143 | me.key = key |
| 144 | me.patha, me.vala = patha, vala |
| 145 | me.pathb, me.valb = pathb, valb |
| 146 | def __str__(me): |
| 147 | return "Ambiguous answer resolving key `%s': " \ |
| 148 | "path %s yields `%s' but %s yields `%s'" % \ |
| 149 | (me.key, _fmt_path(me.patha), me.vala, _fmt_path(me.pathb), me.valb) |
| 150 | |
| 151 | class InheritanceCycleError (Exception): |
| 152 | def __init__(me, key, path): |
| 153 | me.key = key |
| 154 | me.path = path |
| 155 | def __str__(me): |
| 156 | return "Found a cycle %s looking up key `%s'" % \ |
| 157 | (_fmt_path(me.path), me.key) |
| 158 | |
| 159 | class MissingSectionException (Exception): |
| 160 | def __init__(me, sec): |
| 161 | me.key = key |
| 162 | def __str__(me): |
| 163 | return "Section `%s' not found" % (me.sec) |
| 164 | |
| 165 | class MissingKeyException (Exception): |
| 166 | def __init__(me, sec, key): |
| 167 | me.sec = sec |
| 168 | me.key = key |
| 169 | def __str__(me): |
| 170 | return "Key `%s' not found in section `%s'" % (me.key, me.sec) |
| 171 | |
| 172 | class ConfigSection (object): |
| 173 | """ |
| 174 | A section in a configuration parser. |
| 175 | |
| 176 | This is where a lot of the nitty-gritty stuff actually happens. The |
| 177 | `MyConfigParser' knows a lot about the internals of this class, which saves |
| 178 | on building a complicated interface. |
| 179 | """ |
| 180 | |
| 181 | def __init__(me, name, cp): |
| 182 | """Initialize a new, empty section with a given NAME and parent CP.""" |
| 183 | |
| 184 | ## The cache maps item keys to entries, which consist of a pair of |
| 185 | ## objects. There are four possible states for a cache entry: |
| 186 | ## |
| 187 | ## * missing -- there is no entry at all with this key, so we must |
| 188 | ## search for it; |
| 189 | ## |
| 190 | ## * None, None -- we are actively trying to resolve this key, so if we |
| 191 | ## encounter this state, we have found a cycle in the inheritance |
| 192 | ## graph; |
| 193 | ## |
| 194 | ## * None, [] -- we know that this key isn't reachable through any of |
| 195 | ## our parents; |
| 196 | ## |
| 197 | ## * VALUE, PATH -- we know that the key resolves to VALUE, along the |
| 198 | ## PATH from us (exclusive) to the defining parent (inclusive). |
| 199 | me.name = name |
| 200 | me._itemmap = dict() |
| 201 | me._cache = dict() |
| 202 | me._cp = cp |
| 203 | |
| 204 | def _expand(me, string, resolvep): |
| 205 | """ |
| 206 | Expands $(...) and (optionally) $[...] placeholders in STRING. |
| 207 | |
| 208 | RESOLVEP is a boolean switch: do we bother to tax the resolver or not? |
| 209 | This is turned off by MyConfigParser's resolve() method while it's |
| 210 | collecting hostnames to be resolved. |
| 211 | """ |
| 212 | string = RX_REF.sub \ |
| 213 | (lambda m: me.get(m.group(1), resolvep), string) |
| 214 | if resolvep: |
| 215 | string = RX_RESOLVE.sub(lambda m: me._cp._resolver.lookup(m.group(1)), |
| 216 | string) |
| 217 | return string |
| 218 | |
| 219 | def _parents(me): |
| 220 | """Yield this section's parents.""" |
| 221 | try: names = me._itemmap['@inherit'] |
| 222 | except KeyError: return |
| 223 | for name in names.replace(',', ' ').split(): |
| 224 | yield me._cp.section(name) |
| 225 | |
| 226 | def _get(me, key, path = None): |
| 227 | """ |
| 228 | Low-level option-fetching method. |
| 229 | |
| 230 | Fetch the value for the named KEY in this section, or maybe (recursively) |
| 231 | a section which it inherits from. |
| 232 | |
| 233 | Returns a pair VALUE, PATH. The value is not expanded; nor do we check |
| 234 | for the special `name' key. The caller is expected to do these things. |
| 235 | Returns None if no value could be found. |
| 236 | """ |
| 237 | |
| 238 | ## If we weren't given a path, then we'd better make one. |
| 239 | if path is None: path = [] |
| 240 | |
| 241 | ## Extend the path to cover us, but remember to remove us again when |
| 242 | ## we've finished. If we need to pass the current path back upwards, |
| 243 | ## then remember to take a copy. |
| 244 | path.append(me.name) |
| 245 | try: |
| 246 | |
| 247 | ## If we've been this way before on another pass through then return the |
| 248 | ## value we found then. If we're still thinking about it then we've |
| 249 | ## found a cycle. |
| 250 | try: v, p = me._cache[key] |
| 251 | except KeyError: pass |
| 252 | else: |
| 253 | if p is None: raise InheritanceCycleError(key, path[:]) |
| 254 | else: return v, path + p |
| 255 | |
| 256 | ## See whether the answer is ready waiting for us. |
| 257 | try: v = me._itemmap[key] |
| 258 | except KeyError: pass |
| 259 | else: |
| 260 | p = path[:] |
| 261 | me._cache[key] = v, [] |
| 262 | return v, p |
| 263 | |
| 264 | ## Initially we have no idea. |
| 265 | value = None |
| 266 | winner = [] |
| 267 | |
| 268 | ## Go through our parents and ask them what they think. |
| 269 | me._cache[key] = None, None |
| 270 | for p in me._parents(): |
| 271 | |
| 272 | ## See whether we get an answer. If not, keep on going. |
| 273 | v, pp = p._get(key, path) |
| 274 | if v is None: continue |
| 275 | |
| 276 | ## If we got an answer, check that it matches any previous ones. |
| 277 | if value is None: |
| 278 | value = v |
| 279 | winner = pp |
| 280 | elif value != v: |
| 281 | raise AmbiguousOptionError(key, winner, value, pp, v) |
| 282 | |
| 283 | ## That's the best we could manage. |
| 284 | me._cache[key] = value, winner[len(path):] |
| 285 | return value, winner |
| 286 | |
| 287 | finally: |
| 288 | ## Remove us from the path again. |
| 289 | path.pop() |
| 290 | |
| 291 | def get(me, key, resolvep = True): |
| 292 | """ |
| 293 | Retrieve the value of KEY from this section. |
| 294 | """ |
| 295 | |
| 296 | ## Special handling for the `name' key. |
| 297 | if key == 'name': |
| 298 | value = me._itemmap.get('name', me.name) |
| 299 | else: |
| 300 | value, _ = me._get(key) |
| 301 | if value is None: |
| 302 | raise MissingKeyException(me.name, key) |
| 303 | |
| 304 | ## Expand the value and return it. |
| 305 | return me._expand(value, resolvep) |
| 306 | |
| 307 | def items(me, resolvep = True): |
| 308 | """ |
| 309 | Yield a list of item names in the section. |
| 310 | """ |
| 311 | |
| 312 | ## Initialize for a depth-first walk of the inheritance graph. |
| 313 | seen = {} |
| 314 | visiting = { me.name: True } |
| 315 | stack = [me] |
| 316 | |
| 317 | ## Visit nodes, collecting their keys. Don't believe the values: |
| 318 | ## resolving inheritance is too hard to do like this. |
| 319 | while stack: |
| 320 | sec = stack.pop() |
| 321 | for p in sec._parents(): |
| 322 | if p.name not in visiting: |
| 323 | stack.append(p); visiting[p.name] = True |
| 324 | |
| 325 | for key in sec._itemmap.iterkeys(): |
| 326 | if key != '@inherit': seen[key] = None |
| 327 | |
| 328 | ## And we're done. |
| 329 | return seen.iterkeys() |
| 330 | |
| 331 | class MyConfigParser (object): |
| 332 | """ |
| 333 | A more advanced configuration parser. |
| 334 | |
| 335 | This has four major enhancements over the standard ConfigParser which are |
| 336 | relevant to us. |
| 337 | |
| 338 | * It recognizes `@inherits' keys and follows them when expanding a |
| 339 | value. |
| 340 | |
| 341 | * It recognizes `$(VAR)' references to configuration variables during |
| 342 | expansion and processes them correctly. |
| 343 | |
| 344 | * It recognizes `$[HOST]' name-resolver requests and handles them |
| 345 | correctly. |
| 346 | |
| 347 | * Its parsing behaviour is well-defined. |
| 348 | |
| 349 | Use: |
| 350 | |
| 351 | 1. Call parse(FILENAME) to slurp in the configuration data. |
| 352 | |
| 353 | 2. Call resolve() to collect the hostnames which need to be resolved and |
| 354 | actually do the name resolution. |
| 355 | |
| 356 | 3. Call sections() to get a list of the configuration sections, or |
| 357 | section(NAME) to find a named section. |
| 358 | |
| 359 | 4. Call get(ITEM) on a section to collect the results, or items() to |
| 360 | iterate over them. |
| 361 | """ |
| 362 | |
| 363 | def __init__(me): |
| 364 | """ |
| 365 | Initialize a new, empty configuration parser. |
| 366 | """ |
| 367 | me._sectmap = dict() |
| 368 | me._resolver = BulkResolver() |
| 369 | |
| 370 | def parse(me, f): |
| 371 | """ |
| 372 | Parse configuration from a file F. |
| 373 | """ |
| 374 | |
| 375 | ## Initial parser state. |
| 376 | sect = None |
| 377 | key = None |
| 378 | val = None |
| 379 | lno = 0 |
| 380 | |
| 381 | ## An unpleasant hack. Python makes it hard to capture a value in a |
| 382 | ## variable and examine it in a single action, and this is the best that |
| 383 | ## I came up with. |
| 384 | m = [None] |
| 385 | def match(rx): m[0] = rx.match(line); return m[0] |
| 386 | |
| 387 | ## Commit a key's value when we've determined that there are no further |
| 388 | ## continuation lines. |
| 389 | def flush(): |
| 390 | if key is not None: sect._itemmap[key] = val.getvalue() |
| 391 | |
| 392 | ## Work through all of the input lines. |
| 393 | for line in f: |
| 394 | lno += 1 |
| 395 | |
| 396 | if match(RX_COMMENT): |
| 397 | ## A comment or a blank line. Nothing doing. (This means that we |
| 398 | ## leave out blank lines which look like they might be continuation |
| 399 | ## lines.) |
| 400 | |
| 401 | pass |
| 402 | |
| 403 | elif match(RX_GRPHDR): |
| 404 | ## A section header. Flush out any previous value and set up the new |
| 405 | ## group. |
| 406 | |
| 407 | flush() |
| 408 | name = m[0].group(1) |
| 409 | try: sect = me._sectmap[name] |
| 410 | except KeyError: sect = me._sectmap[name] = ConfigSection(name, me) |
| 411 | key = None |
| 412 | |
| 413 | elif match(RX_ASSGN): |
| 414 | ## A new assignment. Flush out the old one, and set up to store this |
| 415 | ## one. |
| 416 | |
| 417 | if sect is None: |
| 418 | raise ConfigSyntaxError(f.name, lno, 'no active section to update') |
| 419 | flush() |
| 420 | key = m[0].group(1) |
| 421 | val = StringIO(); val.write(m[0].group(2)) |
| 422 | |
| 423 | elif match(RX_CONT): |
| 424 | ## A continuation line. Accumulate the value. |
| 425 | |
| 426 | if key is None: |
| 427 | raise ConfigSyntaxError(f.name, lno, 'no config value to continue') |
| 428 | val.write('\n'); val.write(m[0].group(1)) |
| 429 | |
| 430 | else: |
| 431 | ## Something else. |
| 432 | |
| 433 | raise ConfigSyntaxError(f.name, lno, 'incomprehensible line') |
| 434 | |
| 435 | ## Don't forget to commit any final value material. |
| 436 | flush() |
| 437 | |
| 438 | def section(me, name): |
| 439 | """Return a ConfigSection with the given NAME.""" |
| 440 | try: return me._sectmap[name] |
| 441 | except KeyError: raise MissingSectionException(name) |
| 442 | |
| 443 | def sections(me): |
| 444 | """Yield the known sections.""" |
| 445 | return me._sectmap.itervalues() |
| 446 | |
| 447 | def resolve(me): |
| 448 | """ |
| 449 | Works out all of the hostnames which need resolving and resolves them. |
| 450 | |
| 451 | Until you call this, attempts to fetch configuration items which need to |
| 452 | resolve hostnames will fail! |
| 453 | """ |
| 454 | for sec in me.sections(): |
| 455 | for key in sec.items(): |
| 456 | value = sec.get(key, resolvep = False) |
| 457 | for match in RX_RESOLVE.finditer(value): |
| 458 | me._resolver.prepare(match.group(1)) |
| 459 | me._resolver.run() |
| 460 | |
| 461 | ###-------------------------------------------------------------------------- |
| 462 | ### Command-line handling. |
| 463 | |
| 464 | def inputiter(things): |
| 465 | """ |
| 466 | Iterate over command-line arguments, returning corresponding open files. |
| 467 | |
| 468 | If none were given, or one is `-', assume standard input; if one is a |
| 469 | directory, scan it for files other than backups; otherwise return the |
| 470 | opened files. |
| 471 | """ |
| 472 | |
| 473 | if not things: |
| 474 | if OS.isatty(stdin.fileno()): |
| 475 | M.die('no input given, and stdin is a terminal') |
| 476 | yield stdin |
| 477 | else: |
| 478 | for thing in things: |
| 479 | if thing == '-': |
| 480 | yield stdin |
| 481 | elif OS.path.isdir(thing): |
| 482 | for item in OS.listdir(thing): |
| 483 | if item.endswith('~') or item.endswith('#'): |
| 484 | continue |
| 485 | name = OS.path.join(thing, item) |
| 486 | if not OS.path.isfile(name): |
| 487 | continue |
| 488 | yield file(name) |
| 489 | else: |
| 490 | yield file(thing) |
| 491 | |
| 492 | def parse_options(argv = argv): |
| 493 | """ |
| 494 | Parse command-line options, returning a pair (OPTS, ARGS). |
| 495 | """ |
| 496 | M.ego(argv[0]) |
| 497 | op = OptionParser(usage = '%prog [-c CDB] INPUT...', |
| 498 | version = '%%prog (tripe, version %s)' % VERSION) |
| 499 | op.add_option('-c', '--cdb', metavar = 'CDB', |
| 500 | dest = 'cdbfile', default = None, |
| 501 | help = 'Compile output into a CDB file.') |
| 502 | opts, args = op.parse_args(argv) |
| 503 | return opts, args |
| 504 | |
| 505 | ###-------------------------------------------------------------------------- |
| 506 | ### Main code. |
| 507 | |
| 508 | def getconf(args): |
| 509 | """ |
| 510 | Read the configuration files and return the accumulated result. |
| 511 | |
| 512 | We make sure that all hostnames have been properly resolved. |
| 513 | """ |
| 514 | conf = MyConfigParser() |
| 515 | for f in inputiter(args): |
| 516 | conf.parse(f) |
| 517 | conf.resolve() |
| 518 | return conf |
| 519 | |
| 520 | def output(conf, cdb): |
| 521 | """ |
| 522 | Output the configuration information CONF to the database CDB. |
| 523 | |
| 524 | This is where the special `user' and `auto' database entries get set. |
| 525 | """ |
| 526 | auto = [] |
| 527 | for sec in sorted(conf.sections(), key = lambda sec: sec.name): |
| 528 | if sec.name.startswith('@'): |
| 529 | continue |
| 530 | elif sec.name.startswith('$'): |
| 531 | label = sec.name |
| 532 | else: |
| 533 | label = 'P%s' % sec.name |
| 534 | try: a = sec.get('auto') |
| 535 | except MissingKeyException: pass |
| 536 | else: |
| 537 | if a in ('y', 'yes', 't', 'true', '1', 'on'): auto.append(sec.name) |
| 538 | try: u = sec.get('user') |
| 539 | except MissingKeyException: pass |
| 540 | else: cdb.add('U%s' % u) |
| 541 | url = M.URLEncode(semip = True) |
| 542 | for key in sorted(sec.items()): |
| 543 | if not key.startswith('@'): |
| 544 | url.encode(key, sec.get(key)) |
| 545 | cdb.add(label, url.result) |
| 546 | cdb.add('%AUTO', ' '.join(auto)) |
| 547 | cdb.finish() |
| 548 | |
| 549 | def main(): |
| 550 | """Main program.""" |
| 551 | opts, args = parse_options() |
| 552 | if opts.cdbfile: |
| 553 | cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new') |
| 554 | else: |
| 555 | cdb = CDBFake() |
| 556 | conf = getconf(args[1:]) |
| 557 | output(conf, cdb) |
| 558 | |
| 559 | if __name__ == '__main__': |
| 560 | main() |
| 561 | |
| 562 | ###----- That's all, folks -------------------------------------------------- |