| 1 | #! @PYTHON@ |
| 2 | ### -*-python-*- |
| 3 | ### |
| 4 | ### Build a CDB file from configuration file |
| 5 | ### |
| 6 | ### (c) 2007 Straylight/Edgeware |
| 7 | ### |
| 8 | |
| 9 | ###----- Licensing notice --------------------------------------------------- |
| 10 | ### |
| 11 | ### This file is part of Trivial IP Encryption (TrIPE). |
| 12 | ### |
| 13 | ### TrIPE is free software: you can redistribute it and/or modify it under |
| 14 | ### the terms of the GNU General Public License as published by the Free |
| 15 | ### Software Foundation; either version 3 of the License, or (at your |
| 16 | ### option) any later version. |
| 17 | ### |
| 18 | ### TrIPE is distributed in the hope that it will be useful, but WITHOUT |
| 19 | ### ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 20 | ### FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| 21 | ### for more details. |
| 22 | ### |
| 23 | ### You should have received a copy of the GNU General Public License |
| 24 | ### along with TrIPE. If not, see <https://www.gnu.org/licenses/>. |
| 25 | |
| 26 | VERSION = '@VERSION@' |
| 27 | |
| 28 | ###-------------------------------------------------------------------------- |
| 29 | ### External dependencies. |
| 30 | |
| 31 | import mLib as M |
| 32 | from optparse import OptionParser |
| 33 | import cdb as CDB |
| 34 | from sys import stdin, stdout, exit, argv |
| 35 | import re as RX |
| 36 | import os as OS |
| 37 | from cStringIO import StringIO |
| 38 | |
| 39 | ###-------------------------------------------------------------------------- |
| 40 | ### Utilities. |
| 41 | |
| 42 | class CDBFake (object): |
| 43 | """Like cdbmake, but just outputs data suitable for cdb-map.""" |
| 44 | def __init__(me, file = stdout): |
| 45 | me.file = file |
| 46 | def add(me, key, value): |
| 47 | me.file.write('%s:%s\n' % (key, value)) |
| 48 | def finish(me): |
| 49 | pass |
| 50 | |
| 51 | ###-------------------------------------------------------------------------- |
| 52 | ### A bulk DNS resolver. |
| 53 | |
| 54 | class BulkResolver (object): |
| 55 | """ |
| 56 | Resolve a number of DNS names in parallel. |
| 57 | |
| 58 | The BulkResovler resolves a number of hostnames in parallel. Using it |
| 59 | works in three phases: |
| 60 | |
| 61 | 1. You call prepare(HOSTNAME) a number of times, to feed in the hostnames |
| 62 | you're interested in. |
| 63 | |
| 64 | 2. You call run() to actually drive the resolver. |
| 65 | |
| 66 | 3. You call lookup(HOSTNAME) to get the address you wanted. This will |
| 67 | fail with KeyError if the resolver couldn't resolve the HOSTNAME. |
| 68 | """ |
| 69 | |
| 70 | def __init__(me): |
| 71 | """Initialize the resolver.""" |
| 72 | me._resolvers = {} |
| 73 | me._namemap = {} |
| 74 | |
| 75 | def prepare(me, host): |
| 76 | """Prime the resolver to resolve the name HOST.""" |
| 77 | if host not in me._resolvers: |
| 78 | me._resolvers[host] = M.SelResolveByName \ |
| 79 | (host, |
| 80 | lambda name, alias, addr: |
| 81 | me._resolved(host, addr[0]), |
| 82 | lambda: me._resolved(host, None)) |
| 83 | |
| 84 | def run(me): |
| 85 | """Run the background DNS resolver until it's finished.""" |
| 86 | while me._resolvers: |
| 87 | M.select() |
| 88 | |
| 89 | def lookup(me, host): |
| 90 | """ |
| 91 | Fetch the address corresponding to HOST. |
| 92 | """ |
| 93 | addr = me._namemap[host] |
| 94 | if addr is None: |
| 95 | raise KeyError(host) |
| 96 | return addr |
| 97 | |
| 98 | def _resolved(me, host, addr): |
| 99 | """Callback function: remember that ADDR is the address for HOST.""" |
| 100 | me._namemap[host] = addr |
| 101 | del me._resolvers[host] |
| 102 | |
| 103 | ###-------------------------------------------------------------------------- |
| 104 | ### The configuration parser. |
| 105 | |
| 106 | ## Match a comment or empty line. |
| 107 | RX_COMMENT = RX.compile(r'(?x) ^ \s* (?: $ | [;#])') |
| 108 | |
| 109 | ## Match a section group header. |
| 110 | RX_GRPHDR = RX.compile(r'(?x) ^ \s* \[ (.*) \] \s* $') |
| 111 | |
| 112 | ## Match an assignment line. |
| 113 | RX_ASSGN = RX.compile(r'''(?x) ^ |
| 114 | ([^\s:=] (?: [^:=]* [^\s:=])?) |
| 115 | \s* [:=] \s* |
| 116 | (| \S | \S.*\S) |
| 117 | \s* $''') |
| 118 | |
| 119 | ## Match a continuation line. |
| 120 | RX_CONT = RX.compile(r'''(?x) ^ \s+ |
| 121 | (| \S | \S.*\S) |
| 122 | \s* $''') |
| 123 | |
| 124 | ## Match a $(VAR) configuration variable reference; group 1 is the VAR. |
| 125 | RX_REF = RX.compile(r'(?x) \$ \( ([^)]+) \)') |
| 126 | |
| 127 | ## Match a $[HOST] name resolution reference; group 1 is the HOST. |
| 128 | RX_RESOLVE = RX.compile(r'(?x) \$ \[ ([^]]+) \]') |
| 129 | |
| 130 | class ConfigSyntaxError (Exception): |
| 131 | def __init__(me, fname, lno, msg): |
| 132 | me.fname = fname |
| 133 | me.lno = lno |
| 134 | me.msg = msg |
| 135 | def __str__(me): |
| 136 | return '%s:%d: %s' % (me.fname, me.lno, me.msg) |
| 137 | |
| 138 | def _fmt_path(path): |
| 139 | return ' -> '.join(["`%s'" % hop for hop in path]) |
| 140 | |
| 141 | class AmbiguousOptionError (Exception): |
| 142 | def __init__(me, key, patha, vala, pathb, valb): |
| 143 | me.key = key |
| 144 | me.patha, me.vala = patha, vala |
| 145 | me.pathb, me.valb = pathb, valb |
| 146 | def __str__(me): |
| 147 | return "Ambiguous answer resolving key `%s': " \ |
| 148 | "path %s yields `%s' but %s yields `%s'" % \ |
| 149 | (me.key, _fmt_path(me.patha), me.vala, _fmt_path(me.pathb), me.valb) |
| 150 | |
| 151 | class InheritanceCycleError (Exception): |
| 152 | def __init__(me, key, path): |
| 153 | me.key = key |
| 154 | me.path = path |
| 155 | def __str__(me): |
| 156 | return "Found a cycle %s looking up key `%s'" % \ |
| 157 | (_fmt_path(me.path), me.key) |
| 158 | |
| 159 | class MissingSectionException (Exception): |
| 160 | def __init__(me, sec): |
| 161 | me.key = key |
| 162 | def __str__(me): |
| 163 | return "Section `%s' not found" % (me.sec) |
| 164 | |
| 165 | class MissingKeyException (Exception): |
| 166 | def __init__(me, sec, key): |
| 167 | me.sec = sec |
| 168 | me.key = key |
| 169 | def __str__(me): |
| 170 | return "Key `%s' not found in section `%s'" % (me.key, me.sec) |
| 171 | |
| 172 | class ConfigSection (object): |
| 173 | """ |
| 174 | A section in a configuration parser. |
| 175 | |
| 176 | This is where a lot of the nitty-gritty stuff actually happens. The |
| 177 | `MyConfigParser' knows a lot about the internals of this class, which saves |
| 178 | on building a complicated interface. |
| 179 | """ |
| 180 | |
| 181 | def __init__(me, name, cp): |
| 182 | """Initialize a new, empty section with a given NAME and parent CP.""" |
| 183 | me.name = name |
| 184 | me._itemmap = dict() |
| 185 | me._cp = cp |
| 186 | |
| 187 | def _expand(me, string, resolvep): |
| 188 | """ |
| 189 | Expands $(...) and (optionally) $[...] placeholders in STRING. |
| 190 | |
| 191 | RESOLVEP is a boolean switch: do we bother to tax the resolver or not? |
| 192 | This is turned off by MyConfigParser's resolve() method while it's |
| 193 | collecting hostnames to be resolved. |
| 194 | """ |
| 195 | string = RX_REF.sub \ |
| 196 | (lambda m: me.get(m.group(1), resolvep), string) |
| 197 | if resolvep: |
| 198 | string = RX_RESOLVE.sub(lambda m: me._cp._resolver.lookup(m.group(1)), |
| 199 | string) |
| 200 | return string |
| 201 | |
| 202 | def _get(me, key, map = None, path = None): |
| 203 | """ |
| 204 | Low-level option-fetching method. |
| 205 | |
| 206 | Fetch the value for the named KEY in this section, or maybe (recursively) |
| 207 | a section which it inherits from. |
| 208 | |
| 209 | Returns a pair VALUE, PATH. The value is not expanded; nor do we check |
| 210 | for the special `name' key. The caller is expected to do these things. |
| 211 | Returns None if no value could be found. |
| 212 | """ |
| 213 | |
| 214 | ## If we weren't given a memoization map or path, then we'd better make |
| 215 | ## one. |
| 216 | if map is None: map = {} |
| 217 | if path is None: path = [] |
| 218 | |
| 219 | ## Extend the path to cover us, but remember to remove us again when |
| 220 | ## we've finished. If we need to pass the current path back upwards, |
| 221 | ## then remember to take a copy. |
| 222 | path.append(me.name) |
| 223 | try: |
| 224 | |
| 225 | ## If we've been this way before on another pass through then return |
| 226 | ## the value we found then. If we're still thinking about it then |
| 227 | ## we've found a cycle. |
| 228 | try: threadp, value = map[me.name] |
| 229 | except KeyError: pass |
| 230 | else: |
| 231 | if threadp: raise InheritanceCycleError(key, path[:]) |
| 232 | |
| 233 | ## See whether the answer is ready waiting for us. |
| 234 | try: v = me._itemmap[key] |
| 235 | except KeyError: pass |
| 236 | else: return v, path[:] |
| 237 | |
| 238 | ## No, apparently, not. Find out our list of parents. |
| 239 | try: |
| 240 | parents = [me._cp.section(p) for p in |
| 241 | me._itemmap['@inherit'].replace(',', ' ').split()] |
| 242 | except KeyError: |
| 243 | parents = [] |
| 244 | |
| 245 | ## Initially we have no idea. |
| 246 | value = None |
| 247 | winner = None |
| 248 | |
| 249 | ## Go through our parents and ask them what they think. |
| 250 | map[me.name] = True, None |
| 251 | for p in parents: |
| 252 | |
| 253 | ## See whether we get an answer. If not, keep on going. |
| 254 | v, pp = p._get(key, map, path) |
| 255 | if v is None: continue |
| 256 | |
| 257 | ## If we got an answer, check that it matches any previous ones. |
| 258 | if value is None: |
| 259 | value = v |
| 260 | winner = pp |
| 261 | elif value != v: |
| 262 | raise AmbiguousOptionError(key, winner, value, pp, v) |
| 263 | |
| 264 | ## That's the best we could manage. |
| 265 | map[me.name] = False, value |
| 266 | return value, winner |
| 267 | |
| 268 | finally: |
| 269 | ## Remove us from the path again. |
| 270 | path.pop() |
| 271 | |
| 272 | def get(me, key, resolvep = True): |
| 273 | """ |
| 274 | Retrieve the value of KEY from this section. |
| 275 | """ |
| 276 | |
| 277 | ## Special handling for the `name' key. |
| 278 | if key == 'name': |
| 279 | value = me._itemmap.get('name', me.name) |
| 280 | else: |
| 281 | value, _ = me._get(key) |
| 282 | if value is None: |
| 283 | raise MissingKeyException(me.name, key) |
| 284 | |
| 285 | ## Expand the value and return it. |
| 286 | return me._expand(value, resolvep) |
| 287 | |
| 288 | def items(me, resolvep = True): |
| 289 | """ |
| 290 | Return a list of (NAME, VALUE) items in this section. |
| 291 | |
| 292 | This extends the default method by handling the inheritance chain. |
| 293 | """ |
| 294 | |
| 295 | ## Initialize for a depth-first walk of the inheritance graph. |
| 296 | d = {} |
| 297 | visited = {} |
| 298 | stack = [me.name] |
| 299 | |
| 300 | ## Visit nodes, collecting their keys. Don't believe the values: |
| 301 | ## resolving inheritance is too hard to do like this. |
| 302 | while stack: |
| 303 | sec = me._cp.section(stack.pop()) |
| 304 | if sec.name in visited: continue |
| 305 | visited[sec.name] = True |
| 306 | |
| 307 | for key, value in sec._itemmap.iteritems(): |
| 308 | if key == '@inherit': stack += value.replace(',', ' ').split() |
| 309 | else: d[key] = None |
| 310 | |
| 311 | ## Now collect the values for the known keys, one by one. |
| 312 | items = [] |
| 313 | for key in d: items.append((key, me.get(key, resolvep))) |
| 314 | |
| 315 | ## And we're done. |
| 316 | return items |
| 317 | |
| 318 | class MyConfigParser (object): |
| 319 | """ |
| 320 | A more advanced configuration parser. |
| 321 | |
| 322 | This has four major enhancements over the standard ConfigParser which are |
| 323 | relevant to us. |
| 324 | |
| 325 | * It recognizes `@inherits' keys and follows them when expanding a |
| 326 | value. |
| 327 | |
| 328 | * It recognizes `$(VAR)' references to configuration variables during |
| 329 | expansion and processes them correctly. |
| 330 | |
| 331 | * It recognizes `$[HOST]' name-resolver requests and handles them |
| 332 | correctly. |
| 333 | |
| 334 | * Its parsing behaviour is well-defined. |
| 335 | |
| 336 | Use: |
| 337 | |
| 338 | 1. Call parse(FILENAME) to slurp in the configuration data. |
| 339 | |
| 340 | 2. Call resolve() to collect the hostnames which need to be resolved and |
| 341 | actually do the name resolution. |
| 342 | |
| 343 | 3. Call sections() to get a list of the configuration sections, or |
| 344 | section(NAME) to find a named section. |
| 345 | |
| 346 | 4. Call get(ITEM) on a section to collect the results, or items() to |
| 347 | iterate over them. |
| 348 | """ |
| 349 | |
| 350 | def __init__(me): |
| 351 | """ |
| 352 | Initialize a new, empty configuration parser. |
| 353 | """ |
| 354 | me._sectmap = dict() |
| 355 | me._resolver = BulkResolver() |
| 356 | |
| 357 | def parse(me, f): |
| 358 | """ |
| 359 | Parse configuration from a file F. |
| 360 | """ |
| 361 | |
| 362 | ## Initial parser state. |
| 363 | sect = None |
| 364 | key = None |
| 365 | val = None |
| 366 | lno = 0 |
| 367 | |
| 368 | ## An unpleasant hack. Python makes it hard to capture a value in a |
| 369 | ## variable and examine it in a single action, and this is the best that |
| 370 | ## I came up with. |
| 371 | m = [None] |
| 372 | def match(rx): m[0] = rx.match(line); return m[0] |
| 373 | |
| 374 | ## Commit a key's value when we've determined that there are no further |
| 375 | ## continuation lines. |
| 376 | def flush(): |
| 377 | if key is not None: sect._itemmap[key] = val.getvalue() |
| 378 | |
| 379 | ## Work through all of the input lines. |
| 380 | for line in f: |
| 381 | lno += 1 |
| 382 | |
| 383 | if match(RX_COMMENT): |
| 384 | ## A comment or a blank line. Nothing doing. (This means that we |
| 385 | ## leave out blank lines which look like they might be continuation |
| 386 | ## lines.) |
| 387 | |
| 388 | pass |
| 389 | |
| 390 | elif match(RX_GRPHDR): |
| 391 | ## A section header. Flush out any previous value and set up the new |
| 392 | ## group. |
| 393 | |
| 394 | flush() |
| 395 | name = m[0].group(1) |
| 396 | try: sect = me._sectmap[name] |
| 397 | except KeyError: sect = me._sectmap[name] = ConfigSection(name, me) |
| 398 | key = None |
| 399 | |
| 400 | elif match(RX_ASSGN): |
| 401 | ## A new assignment. Flush out the old one, and set up to store this |
| 402 | ## one. |
| 403 | |
| 404 | if sect is None: |
| 405 | raise ConfigSyntaxError(f.name, lno, 'no active section to update') |
| 406 | flush() |
| 407 | key = m[0].group(1) |
| 408 | val = StringIO(); val.write(m[0].group(2)) |
| 409 | |
| 410 | elif match(RX_CONT): |
| 411 | ## A continuation line. Accumulate the value. |
| 412 | |
| 413 | if key is None: |
| 414 | raise ConfigSyntaxError(f.name, lno, 'no config value to continue') |
| 415 | val.write('\n'); val.write(m[0].group(1)) |
| 416 | |
| 417 | else: |
| 418 | ## Something else. |
| 419 | |
| 420 | raise ConfigSyntaxError(f.name, lno, 'incomprehensible line') |
| 421 | |
| 422 | ## Don't forget to commit any final value material. |
| 423 | flush() |
| 424 | |
| 425 | def section(me, name): |
| 426 | """Return a ConfigSection with the given NAME.""" |
| 427 | try: return me._sectmap[name] |
| 428 | except KeyError: raise MissingSectionException(name) |
| 429 | |
| 430 | def sections(me): |
| 431 | """Yield the known sections.""" |
| 432 | return me._sectmap.itervalues() |
| 433 | |
| 434 | def resolve(me): |
| 435 | """ |
| 436 | Works out all of the hostnames which need resolving and resolves them. |
| 437 | |
| 438 | Until you call this, attempts to fetch configuration items which need to |
| 439 | resolve hostnames will fail! |
| 440 | """ |
| 441 | for sec in me.sections(): |
| 442 | for key, value in sec.items(resolvep = False): |
| 443 | for match in RX_RESOLVE.finditer(value): |
| 444 | me._resolver.prepare(match.group(1)) |
| 445 | me._resolver.run() |
| 446 | |
| 447 | ###-------------------------------------------------------------------------- |
| 448 | ### Command-line handling. |
| 449 | |
| 450 | def inputiter(things): |
| 451 | """ |
| 452 | Iterate over command-line arguments, returning corresponding open files. |
| 453 | |
| 454 | If none were given, or one is `-', assume standard input; if one is a |
| 455 | directory, scan it for files other than backups; otherwise return the |
| 456 | opened files. |
| 457 | """ |
| 458 | |
| 459 | if not things: |
| 460 | if OS.isatty(stdin.fileno()): |
| 461 | M.die('no input given, and stdin is a terminal') |
| 462 | yield stdin |
| 463 | else: |
| 464 | for thing in things: |
| 465 | if thing == '-': |
| 466 | yield stdin |
| 467 | elif OS.path.isdir(thing): |
| 468 | for item in OS.listdir(thing): |
| 469 | if item.endswith('~') or item.endswith('#'): |
| 470 | continue |
| 471 | name = OS.path.join(thing, item) |
| 472 | if not OS.path.isfile(name): |
| 473 | continue |
| 474 | yield file(name) |
| 475 | else: |
| 476 | yield file(thing) |
| 477 | |
| 478 | def parse_options(argv = argv): |
| 479 | """ |
| 480 | Parse command-line options, returning a pair (OPTS, ARGS). |
| 481 | """ |
| 482 | M.ego(argv[0]) |
| 483 | op = OptionParser(usage = '%prog [-c CDB] INPUT...', |
| 484 | version = '%%prog (tripe, version %s)' % VERSION) |
| 485 | op.add_option('-c', '--cdb', metavar = 'CDB', |
| 486 | dest = 'cdbfile', default = None, |
| 487 | help = 'Compile output into a CDB file.') |
| 488 | opts, args = op.parse_args(argv) |
| 489 | return opts, args |
| 490 | |
| 491 | ###-------------------------------------------------------------------------- |
| 492 | ### Main code. |
| 493 | |
| 494 | def getconf(args): |
| 495 | """ |
| 496 | Read the configuration files and return the accumulated result. |
| 497 | |
| 498 | We make sure that all hostnames have been properly resolved. |
| 499 | """ |
| 500 | conf = MyConfigParser() |
| 501 | for f in inputiter(args): |
| 502 | conf.parse(f) |
| 503 | conf.resolve() |
| 504 | return conf |
| 505 | |
| 506 | def output(conf, cdb): |
| 507 | """ |
| 508 | Output the configuration information CONF to the database CDB. |
| 509 | |
| 510 | This is where the special `user' and `auto' database entries get set. |
| 511 | """ |
| 512 | auto = [] |
| 513 | for sec in sorted(conf.sections(), key = lambda sec: sec.name): |
| 514 | if sec.name.startswith('@'): |
| 515 | continue |
| 516 | elif sec.name.startswith('$'): |
| 517 | label = sec.name |
| 518 | else: |
| 519 | label = 'P%s' % sec.name |
| 520 | try: a = sec.get('auto') |
| 521 | except MissingKeyException: pass |
| 522 | else: |
| 523 | if a in ('y', 'yes', 't', 'true', '1', 'on'): auto.append(sec.name) |
| 524 | try: u = sec.get('user') |
| 525 | except MissingKeyException: pass |
| 526 | else: cdb.add('U%s' % u) |
| 527 | url = M.URLEncode(laxp = True, semip = True) |
| 528 | for key, value in sorted(sec.items(), key = lambda (k, v): k): |
| 529 | if not key.startswith('@'): |
| 530 | url.encode(key, ' '.join(M.split(value)[0])) |
| 531 | cdb.add(label, url.result) |
| 532 | cdb.add('%AUTO', ' '.join(auto)) |
| 533 | cdb.finish() |
| 534 | |
| 535 | def main(): |
| 536 | """Main program.""" |
| 537 | opts, args = parse_options() |
| 538 | if opts.cdbfile: |
| 539 | cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new') |
| 540 | else: |
| 541 | cdb = CDBFake() |
| 542 | conf = getconf(args[1:]) |
| 543 | output(conf, cdb) |
| 544 | |
| 545 | if __name__ == '__main__': |
| 546 | main() |
| 547 | |
| 548 | ###----- That's all, folks -------------------------------------------------- |