chiark / gitweb /
Upgrade licence to GPLv3+.
[tripe] / peerdb / tripe-newpeers.in
CommitLineData
6005ef9b
MW
1#! @PYTHON@
2### -*-python-*-
3###
4### Build a CDB file from configuration file
5###
6### (c) 2007 Straylight/Edgeware
7###
8
9###----- Licensing notice ---------------------------------------------------
10###
11### This file is part of Trivial IP Encryption (TrIPE).
12###
11ad66c2
MW
13### TrIPE is free software: you can redistribute it and/or modify it under
14### the terms of the GNU General Public License as published by the Free
15### Software Foundation; either version 3 of the License, or (at your
16### option) any later version.
6005ef9b 17###
11ad66c2
MW
18### TrIPE is distributed in the hope that it will be useful, but WITHOUT
19### ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
20### FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21### for more details.
6005ef9b
MW
22###
23### You should have received a copy of the GNU General Public License
11ad66c2 24### along with TrIPE. If not, see <https://www.gnu.org/licenses/>.
6005ef9b
MW
25
26VERSION = '@VERSION@'
27
28###--------------------------------------------------------------------------
29### External dependencies.
30
31import ConfigParser as CP
32import mLib as M
33from optparse import OptionParser
34import cdb as CDB
35from sys import stdin, stdout, exit, argv
36import re as RX
37import os as OS
38
39###--------------------------------------------------------------------------
40### Utilities.
41
42class CDBFake (object):
43 """Like cdbmake, but just outputs data suitable for cdb-map."""
44 def __init__(me, file = stdout):
45 me.file = file
46 def add(me, key, value):
47 me.file.write('%s:%s\n' % (key, value))
48 def finish(me):
49 pass
50
51###--------------------------------------------------------------------------
52### A bulk DNS resolver.
53
54class BulkResolver (object):
55 """
56 Resolve a number of DNS names in parallel.
57
58 The BulkResovler resolves a number of hostnames in parallel. Using it
59 works in three phases:
60
61 1. You call prepare(HOSTNAME) a number of times, to feed in the hostnames
62 you're interested in.
63
64 2. You call run() to actually drive the resolver.
65
66 3. You call lookup(HOSTNAME) to get the address you wanted. This will
67 fail with KeyError if the resolver couldn't resolve the HOSTNAME.
68 """
69
70 def __init__(me):
71 """Initialize the resolver."""
72 me._resolvers = {}
73 me._namemap = {}
74
75 def prepare(me, host):
76 """Prime the resolver to resolve the name HOST."""
d8310a3a
MW
77 if host not in me._resolvers:
78 me._resolvers[host] = M.SelResolveByName \
79 (host,
80 lambda name, alias, addr:
81 me._resolved(host, addr[0]),
82 lambda: me._resolved(host, None))
6005ef9b
MW
83
84 def run(me):
85 """Run the background DNS resolver until it's finished."""
86 while me._resolvers:
87 M.select()
88
89 def lookup(me, host):
90 """
91 Fetch the address corresponding to HOST.
92 """
93 addr = me._namemap[host]
94 if addr is None:
95 raise KeyError, host
96 return addr
97
98 def _resolved(me, host, addr):
99 """Callback function: remember that ADDR is the address for HOST."""
100 me._namemap[host] = addr
101 del me._resolvers[host]
102
103###--------------------------------------------------------------------------
104### The configuration parser.
105
106## Match a $(VAR) configuration variable reference; group 1 is the VAR.
107r_ref = RX.compile(r'\$\(([^)]+)\)')
108
109## Match a $[HOST] name resolution reference; group 1 is the HOST.
110r_resolve = RX.compile(r'\$\[([^]]+)\]')
111
bd3db76c
MW
112def _fmt_path(path):
113 return ' -> '.join(["`%s'" % hop for hop in path])
114
115class AmbiguousOptionError (Exception):
116 def __init__(me, key, patha, vala, pathb, valb):
117 me.key = key
118 me.patha, me.vala = patha, vala
119 me.pathb, me.valb = pathb, valb
120 def __str__(me):
121 return "Ambiguous answer resolving key `%s': " \
122 "path %s yields `%s' but %s yields `%s'" % \
123 (me.key, _fmt_path(me.patha), me.vala, _fmt_path(me.pathb), me.valb)
124
125class InheritanceCycleError (Exception):
126 def __init__(me, key, path):
127 me.key = key
128 me.path = path
129 def __str__(me):
130 return "Found a cycle %s looking up key `%s'" % \
131 (_fmt_path(me.path), me.key)
132
133class MissingKeyException (Exception):
134 def __init__(me, sec, key):
135 me.sec = sec
136 me.key = key
137 def __str__(me):
138 return "Key `%s' not found in section `%s'" % (me.key, me.sec)
139
6005ef9b
MW
140class MyConfigParser (CP.RawConfigParser):
141 """
142 A more advanced configuration parser.
143
144 This has two major enhancements over the standard ConfigParser which are
145 relevant to us.
146
147 * It recognizes `@inherits' keys and follows them when expanding a
148 value.
149
150 * It recognizes `$(VAR)' references to configuration variables during
151 expansion and processes them correctly.
152
153 * It recognizes `$[HOST]' name-resolver requests and handles them
154 correctly.
155
156 Use:
157
158 1. Call read(FILENAME) and/or read(FP, [FILENAME]) to slurp in the
159 configuration data.
160
161 2. Call resolve() to collect the hostnames which need to be resolved and
162 actually do the name resolution.
163
164 3. Call get(SECTION, ITEM) to collect the results, or items(SECTION) to
165 iterate over them.
166 """
167
168 def __init__(me):
169 """
170 Initialize a new, empty configuration parser.
171 """
172 CP.RawConfigParser.__init__(me)
173 me._resolver = BulkResolver()
174
175 def resolve(me):
176 """
177 Works out all of the hostnames which need resolving and resolves them.
178
179 Until you call this, attempts to fetch configuration items which need to
180 resolve hostnames will fail!
181 """
182 for sec in me.sections():
183 for key, value in me.items(sec, resolvep = False):
184 for match in r_resolve.finditer(value):
185 me._resolver.prepare(match.group(1))
186 me._resolver.run()
187
188 def _expand(me, sec, string, resolvep):
189 """
190 Expands $(...) and (optionally) $[...] placeholders in STRING.
191
192 The SEC is the configuration section from which to satisfy $(...)
193 requests. RESOLVEP is a boolean switch: do we bother to tax the resolver
194 or not? This is turned off by the resolve() method while it's collecting
195 hostnames to be resolved.
196 """
197 string = r_ref.sub \
198 (lambda m: me.get(sec, m.group(1), resolvep), string)
199 if resolvep:
200 string = r_resolve.sub(lambda m: me._resolver.lookup(m.group(1)),
201 string)
202 return string
203
204 def has_option(me, sec, key):
205 """
206 Decide whether section SEC has a configuration key KEY.
207
208 This version of the method properly handles the @inherit key.
209 """
bd3db76c 210 return key == 'name' or me._get(sec, key)[0] is not None
6005ef9b 211
bd3db76c 212 def _get(me, sec, key, map = None, path = None):
6005ef9b
MW
213 """
214 Low-level option-fetching method.
215
216 Fetch the value for the named KEY from section SEC, or maybe
bd3db76c 217 (recursively) a section which SEC inherits from.
6005ef9b 218
bd3db76c
MW
219 Returns a pair VALUE, PATH. The value is not expanded; nor do we check
220 for the special `name' key. The caller is expected to do these things.
221 Returns None if no value could be found.
222 """
6005ef9b 223
bd3db76c
MW
224 ## If we weren't given a memoization map or path, then we'd better make
225 ## one.
226 if map is None: map = {}
227 if path is None: path = []
6005ef9b 228
bd3db76c
MW
229 ## If we've been this way before on another pass through then return the
230 ## value we found then. If we're still thinking about it then we've
231 ## found a cycle.
232 path.append(sec)
6005ef9b 233 try:
bd3db76c
MW
234 threadp, value = map[sec]
235 except KeyError:
236 pass
237 else:
238 if threadp:
239 raise InheritanceCycleError, (key, path)
240
241 ## See whether the answer is ready waiting for us.
242 try:
243 v = CP.RawConfigParser.get(me, sec, key)
6005ef9b 244 except CP.NoOptionError:
bd3db76c
MW
245 pass
246 else:
247 p = path[:]
248 path.pop()
249 return v, p
250
251 ## No, apparently, not. Find out our list of parents.
252 try:
253 parents = CP.RawConfigParser.get(me, sec, '@inherit').\
254 replace(',', ' ').split()
255 except CP.NoOptionError:
256 parents = []
257
258 ## Initially we have no idea.
259 value = None
260 winner = None
261
262 ## Go through our parents and ask them what they think.
263 map[sec] = True, None
264 for p in parents:
265
266 ## See whether we get an answer. If not, keep on going.
267 v, pp = me._get(p, key, map, path)
268 if v is None: continue
269
270 ## If we got an answer, check that it matches any previous ones.
271 if value is None:
272 value = v
273 winner = pp
274 elif value != v:
275 raise AmbiguousOptionError, (key, winner, value, pp, v)
276
277 ## That's the best we could manage.
278 path.pop()
279 map[sec] = False, value
280 return value, winner
6005ef9b
MW
281
282 def get(me, sec, key, resolvep = True):
283 """
284 Retrieve the value of KEY from section SEC.
285 """
bd3db76c
MW
286
287 ## Special handling for the `name' key.
288 if key == 'name':
289 try: value = CP.RawConfigParser.get(me, sec, key)
290 except CP.NoOptionError: value = sec
291 else:
292 value, _ = me._get(sec, key)
293 if value is None:
294 raise MissingKeyException, (sec, key)
295
296 ## Expand the value and return it.
297 return me._expand(sec, value, resolvep)
6005ef9b
MW
298
299 def items(me, sec, resolvep = True):
300 """
301 Return a list of (NAME, VALUE) items in section SEC.
302
303 This extends the default method by handling the inheritance chain.
304 """
bd3db76c
MW
305
306 ## Initialize for a depth-first walk of the inheritance graph.
6005ef9b 307 d = {}
bd3db76c 308 visited = {}
6005ef9b 309 basesec = sec
bd3db76c
MW
310 stack = [sec]
311
312 ## Visit nodes, collecting their keys. Don't believe the values:
313 ## resolving inheritance is too hard to do like this.
314 while stack:
315 sec = stack.pop()
316 if sec in visited: continue
317 visited[sec] = True
318
6005ef9b 319 for key, value in CP.RawConfigParser.items(me, sec):
bd3db76c
MW
320 if key == '@inherit': stack += value.replace(',', ' ').split()
321 else: d[key] = None
322
323 ## Now collect the values for the known keys, one by one.
324 items = []
325 for key in d: items.append((key, me.get(basesec, key, resolvep)))
326
327 ## And we're done.
328 return items
6005ef9b
MW
329
330###--------------------------------------------------------------------------
331### Command-line handling.
332
333def inputiter(things):
334 """
335 Iterate over command-line arguments, returning corresponding open files.
336
337 If none were given, or one is `-', assume standard input; if one is a
338 directory, scan it for files other than backups; otherwise return the
339 opened files.
340 """
341
342 if not things:
343 if OS.isatty(stdin.fileno()):
344 M.die('no input given, and stdin is a terminal')
345 yield stdin
346 else:
347 for thing in things:
348 if thing == '-':
349 yield stdin
350 elif OS.path.isdir(thing):
351 for item in OS.listdir(thing):
352 if item.endswith('~') or item.endswith('#'):
353 continue
354 name = OS.path.join(thing, item)
355 if not OS.path.isfile(name):
356 continue
357 yield file(name)
358 else:
359 yield file(thing)
360
361def parse_options(argv = argv):
362 """
363 Parse command-line options, returning a pair (OPTS, ARGS).
364 """
365 M.ego(argv[0])
366 op = OptionParser(usage = '%prog [-c CDB] INPUT...',
367 version = '%%prog (tripe, version %s)' % VERSION)
368 op.add_option('-c', '--cdb', metavar = 'CDB',
369 dest = 'cdbfile', default = None,
370 help = 'Compile output into a CDB file.')
371 opts, args = op.parse_args(argv)
372 return opts, args
373
374###--------------------------------------------------------------------------
375### Main code.
376
377def getconf(args):
378 """
379 Read the configuration files and return the accumulated result.
380
381 We make sure that all hostnames have been properly resolved.
382 """
383 conf = MyConfigParser()
384 for f in inputiter(args):
385 conf.readfp(f)
386 conf.resolve()
387 return conf
388
389def output(conf, cdb):
390 """
391 Output the configuration information CONF to the database CDB.
392
393 This is where the special `user' and `auto' database entries get set.
394 """
395 auto = []
527cd49f 396 for sec in sorted(conf.sections()):
6005ef9b
MW
397 if sec.startswith('@'):
398 continue
399 elif sec.startswith('$'):
400 label = sec
401 else:
402 label = 'P%s' % sec
403 if conf.has_option(sec, 'auto') and \
404 conf.get(sec, 'auto') in ('y', 'yes', 't', 'true', '1', 'on'):
405 auto.append(sec)
406 if conf.has_option(sec, 'user'):
407 cdb.add('U%s' % conf.get(sec, 'user'), sec)
408 url = M.URLEncode(laxp = True, semip = True)
527cd49f 409 for key, value in sorted(conf.items(sec), key = lambda (k, v): k):
6005ef9b
MW
410 if not key.startswith('@'):
411 url.encode(key, ' '.join(M.split(value)[0]))
412 cdb.add(label, url.result)
413 cdb.add('%AUTO', ' '.join(auto))
414 cdb.finish()
415
416def main():
417 """Main program."""
418 opts, args = parse_options()
419 if opts.cdbfile:
420 cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new')
421 else:
422 cdb = CDBFake()
423 conf = getconf(args[1:])
424 output(conf, cdb)
425
426if __name__ == '__main__':
427 main()
428
429###----- That's all, folks --------------------------------------------------