[tripe] / peerdb / tripe-newpeers.in

#! @PYTHON@
### -*-python-*-
###
### Build a CDB file from configuration file
###
### (c) 2007 Straylight/Edgeware
###

###----- Licensing notice ---------------------------------------------------
###
### This file is part of Trivial IP Encryption (TrIPE).
###
### TrIPE is free software; you can redistribute it and/or modify
### it under the terms of the GNU General Public License as published by
### the Free Software Foundation; either version 2 of the License, or
### (at your option) any later version.
###
### TrIPE is distributed in the hope that it will be useful,
### but WITHOUT ANY WARRANTY; without even the implied warranty of
### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
### GNU General Public License for more details.
###
### You should have received a copy of the GNU General Public License
### along with TrIPE; if not, write to the Free Software Foundation,
### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

VERSION = '@VERSION@'

###--------------------------------------------------------------------------
### External dependencies.

import ConfigParser as CP
import mLib as M
from optparse import OptionParser
import cdb as CDB
from sys import stdin, stdout, exit, argv
import re as RX
import os as OS

###--------------------------------------------------------------------------
### Utilities.

class CDBFake (object):
  """Like cdbmake, but just outputs data suitable for cdb-map."""
  def __init__(me, file = stdout):
    me.file = file
  def add(me, key, value):
    me.file.write('%s:%s\n' % (key, value))
  def finish(me):
    pass

###--------------------------------------------------------------------------
### A bulk DNS resolver.

class BulkResolver (object):
  """
  Resolve a number of DNS names in parallel.

  The BulkResovler resolves a number of hostnames in parallel.  Using it
  works in three phases:

    1. You call prepare(HOSTNAME) a number of times, to feed in the hostnames
       you're interested in.

    2. You call run() to actually drive the resolver.

    3. You call lookup(HOSTNAME) to get the address you wanted.  This will
       fail with KeyError if the resolver couldn't resolve the HOSTNAME.
  """

  def __init__(me):
    """Initialize the resolver."""
    me._resolvers = {}
    me._namemap = {}

  def prepare(me, host):
    """Prime the resolver to resolve the name HOST."""
    if host not in me._resolvers:
      me._resolvers[host] = M.SelResolveByName \
                            (host,
                             lambda name, alias, addr:
                               me._resolved(host, addr[0]),
                             lambda: me._resolved(host, None))

  def run(me):
    """Run the background DNS resolver until it's finished."""
    while me._resolvers:
      M.select()

  def lookup(me, host):
    """
    Fetch the address corresponding to HOST.
    """
    addr = me._namemap[host]
    if addr is None:
      raise KeyError, host
    return addr

  def _resolved(me, host, addr):
    """Callback function: remember that ADDR is the address for HOST."""
    me._namemap[host] = addr
    del me._resolvers[host]

###--------------------------------------------------------------------------
### The configuration parser.

## Match a $(VAR) configuration variable reference; group 1 is the VAR.
r_ref = RX.compile(r'\$\(([^)]+)\)')

## Match a $[HOST] name resolution reference; group 1 is the HOST.
r_resolve = RX.compile(r'\$\[([^]]+)\]')

class MyConfigParser (CP.RawConfigParser):
  """
  A more advanced configuration parser.

  This has two major enhancements over the standard ConfigParser which are
  relevant to us.

    * It recognizes `@inherits' keys and follows them when expanding a
      value.

    * It recognizes `$(VAR)' references to configuration variables during
      expansion and processes them correctly.

    * It recognizes `$[HOST]' name-resolver requests and handles them
      correctly.

  Use:

    1. Call read(FILENAME) and/or read(FP, [FILENAME]) to slurp in the
       configuration data.

    2. Call resolve() to collect the hostnames which need to be resolved and
       actually do the name resolution.

    3. Call get(SECTION, ITEM) to collect the results, or items(SECTION) to
       iterate over them.
  """

  def __init__(me):
    """
    Initialize a new, empty configuration parser.
    """
    CP.RawConfigParser.__init__(me)
    me._resolver = BulkResolver()

  def resolve(me):
    """
    Works out all of the hostnames which need resolving and resolves them.

    Until you call this, attempts to fetch configuration items which need to
    resolve hostnames will fail!
    """
    for sec in me.sections():
      for key, value in me.items(sec, resolvep = False):
        for match in r_resolve.finditer(value):
          me._resolver.prepare(match.group(1))
    me._resolver.run()

  def _expand(me, sec, string, resolvep):
    """
    Expands $(...) and (optionally) $[...] placeholders in STRING.

    The SEC is the configuration section from which to satisfy $(...)
    requests.  RESOLVEP is a boolean switch: do we bother to tax the resolver
    or not?  This is turned off by the resolve() method while it's collecting
    hostnames to be resolved.
    """
    string = r_ref.sub \
             (lambda m: me.get(sec, m.group(1), resolvep), string)
    if resolvep:
      string = r_resolve.sub(lambda m: me._resolver.lookup(m.group(1)),
                             string)
    return string

  def has_option(me, sec, key):
    """
    Decide whether section SEC has a configuration key KEY.

    This version of the method properly handles the @inherit key.
    """
    return CP.RawConfigParser.has_option(me, sec, key) or \
           (CP.RawConfigParser.has_option(me, sec, '@inherit') and
            me.has_option(CP.RawConfigParser.get(me, sec, '@inherit'), key))

  def _get(me, basesec, sec, key, resolvep):
    """
    Low-level option-fetching method.

    Fetch the value for the named KEY from section SEC, or maybe
    (recursively) the section which SEC inherits from.

    The result is expanded, by _expend; RESOLVEP is passed to _expand to
    control whether $[...] should be expanded in the result.

    The BASESEC is the section for which the original request was made.  This
    will be different from SEC if we're recursing up the inheritance chain.

    We also provide the default value for `name' here.
    """
    try:
      raw = CP.RawConfigParser.get(me, sec, key)
    except CP.NoOptionError:
      if key == 'name':
        raw = basesec
      elif CP.RawConfigParser.has_option(me, sec, '@inherit'):
        raw = me._get(basesec,
                      CP.RawConfigParser.get(me, sec, '@inherit'),
                      key,
                      resolvep)
      else:
        raise
    return me._expand(basesec, raw, resolvep)

  def get(me, sec, key, resolvep = True):
    """
    Retrieve the value of KEY from section SEC.
    """
    return me._get(sec, sec, key, resolvep)

  def items(me, sec, resolvep = True):
    """
    Return a list of (NAME, VALUE) items in section SEC.

    This extends the default method by handling the inheritance chain.
    """
    d = {}
    basesec = sec
    while sec:
      next = None
      for key, value in CP.RawConfigParser.items(me, sec):
        if key == '@inherit':
          next = value
        elif not key.startswith('@') and key not in d:
          d[key] = me._expand(basesec, value, resolvep)
      sec = next
    return d.items()

###--------------------------------------------------------------------------
### Command-line handling.

def inputiter(things):
  """
  Iterate over command-line arguments, returning corresponding open files.

  If none were given, or one is `-', assume standard input; if one is a
  directory, scan it for files other than backups; otherwise return the
  opened files.
  """

  if not things:
    if OS.isatty(stdin.fileno()):
      M.die('no input given, and stdin is a terminal')
    yield stdin
  else:
    for thing in things:
      if thing == '-':
        yield stdin
      elif OS.path.isdir(thing):
        for item in OS.listdir(thing):
          if item.endswith('~') or item.endswith('#'):
            continue
          name = OS.path.join(thing, item)
          if not OS.path.isfile(name):
            continue
          yield file(name)
      else:
        yield file(thing)

def parse_options(argv = argv):
  """
  Parse command-line options, returning a pair (OPTS, ARGS).
  """
  M.ego(argv[0])
  op = OptionParser(usage = '%prog [-c CDB] INPUT...',
                    version = '%%prog (tripe, version %s)' % VERSION)
  op.add_option('-c', '--cdb', metavar = 'CDB',
                dest = 'cdbfile', default = None,
                help = 'Compile output into a CDB file.')
  opts, args = op.parse_args(argv)
  return opts, args

###--------------------------------------------------------------------------
### Main code.

def getconf(args):
  """
  Read the configuration files and return the accumulated result.

  We make sure that all hostnames have been properly resolved.
  """
  conf = MyConfigParser()
  for f in inputiter(args):
    conf.readfp(f)
  conf.resolve()
  return conf

def output(conf, cdb):
  """
  Output the configuration information CONF to the database CDB.

  This is where the special `user' and `auto' database entries get set.
  """
  auto = []
  for sec in conf.sections():
    if sec.startswith('@'):
      continue
    elif sec.startswith('$'):
      label = sec
    else:
      label = 'P%s' % sec
      if conf.has_option(sec, 'auto') and \
         conf.get(sec, 'auto') in ('y', 'yes', 't', 'true', '1', 'on'):
        auto.append(sec)
      if conf.has_option(sec, 'user'):
        cdb.add('U%s' % conf.get(sec, 'user'), sec)
    url = M.URLEncode(laxp = True, semip = True)
    for key, value in conf.items(sec):
      if not key.startswith('@'):
        url.encode(key, ' '.join(M.split(value)[0]))
    cdb.add(label, url.result)
  cdb.add('%AUTO', ' '.join(auto))
  cdb.finish()

def main():
  """Main program."""
  opts, args = parse_options()
  if opts.cdbfile:
    cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new')
  else:
    cdb = CDBFake()
  conf = getconf(args[1:])
  output(conf, cdb)

if __name__ == '__main__':
  main()

###----- That's all, folks --------------------------------------------------
Commit	Line	Data
6005ef9b MW	1	#! @PYTHON@
	2	### --python--
	3	###
	4	### Build a CDB file from configuration file
	5	###
	6	### (c) 2007 Straylight/Edgeware
	7	###
	8
	9	###----- Licensing notice ---------------------------------------------------
	10	###
	11	### This file is part of Trivial IP Encryption (TrIPE).
	12	###
	13	### TrIPE is free software; you can redistribute it and/or modify
	14	### it under the terms of the GNU General Public License as published by
	15	### the Free Software Foundation; either version 2 of the License, or
	16	### (at your option) any later version.
	17	###
	18	### TrIPE is distributed in the hope that it will be useful,
	19	### but WITHOUT ANY WARRANTY; without even the implied warranty of
	20	### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	21	### GNU General Public License for more details.
	22	###
	23	### You should have received a copy of the GNU General Public License
	24	### along with TrIPE; if not, write to the Free Software Foundation,
	25	### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
	26
	27	VERSION = '@VERSION@'
	28
	29	###--------------------------------------------------------------------------
	30	### External dependencies.
	31
	32	import ConfigParser as CP
	33	import mLib as M
	34	from optparse import OptionParser
	35	import cdb as CDB
	36	from sys import stdin, stdout, exit, argv
	37	import re as RX
	38	import os as OS
	39
	40	###--------------------------------------------------------------------------
	41	### Utilities.
	42
	43	class CDBFake (object):
	44	"""Like cdbmake, but just outputs data suitable for cdb-map."""
	45	def __init__(me, file = stdout):
	46	me.file = file
	47	def add(me, key, value):
	48	me.file.write('%s:%s\n' % (key, value))
	49	def finish(me):
	50	pass
	51
	52	###--------------------------------------------------------------------------
	53	### A bulk DNS resolver.
	54
	55	class BulkResolver (object):
	56	"""
	57	Resolve a number of DNS names in parallel.
	58
	59	The BulkResovler resolves a number of hostnames in parallel. Using it
	60	works in three phases:
	61
	62	1. You call prepare(HOSTNAME) a number of times, to feed in the hostnames
	63	you're interested in.
	64
65	2. You call run() to actually drive the resolver.
66
67	3. You call lookup(HOSTNAME) to get the address you wanted. This will
68	fail with KeyError if the resolver couldn't resolve the HOSTNAME.
69	"""
70
71	def __init__(me):
72	"""Initialize the resolver."""
73	me._resolvers = {}
74	me._namemap = {}
75
76	def prepare(me, host):
77	"""Prime the resolver to resolve the name HOST."""
d8310a3a MW	78	if host not in me._resolvers:
	79	me._resolvers[host] = M.SelResolveByName \
	80	(host,
	81	lambda name, alias, addr:
	82	me._resolved(host, addr[0]),
	83	lambda: me._resolved(host, None))
6005ef9b MW	84
	85	def run(me):
	86	"""Run the background DNS resolver until it's finished."""
	87	while me._resolvers:
	88	M.select()
	89
	90	def lookup(me, host):
	91	"""
	92	Fetch the address corresponding to HOST.
	93	"""
	94	addr = me._namemap[host]
	95	if addr is None:
	96	raise KeyError, host
	97	return addr
	98
	99	def _resolved(me, host, addr):
	100	"""Callback function: remember that ADDR is the address for HOST."""
	101	me._namemap[host] = addr
	102	del me._resolvers[host]
	103
	104	###--------------------------------------------------------------------------
	105	### The configuration parser.
	106
	107	## Match a $(VAR) configuration variable reference; group 1 is the VAR.
	108	r_ref = RX.compile(r'\$\(([^)]+)\)')
	109
	110	## Match a $[HOST] name resolution reference; group 1 is the HOST.
	111	r_resolve = RX.compile(r'\$\[([^]]+)\]')
	112
	113	class MyConfigParser (CP.RawConfigParser):
	114	"""
	115	A more advanced configuration parser.
	116
	117	This has two major enhancements over the standard ConfigParser which are
	118	relevant to us.
	119
	120	* It recognizes `@inherits' keys and follows them when expanding a
	121	value.
	122
	123	* It recognizes `$(VAR)' references to configuration variables during
	124	expansion and processes them correctly.
	125
	126	* It recognizes `$[HOST]' name-resolver requests and handles them
	127	correctly.
	128
	129	Use:
	130
	131	1. Call read(FILENAME) and/or read(FP, [FILENAME]) to slurp in the
	132	configuration data.
	133
	134	2. Call resolve() to collect the hostnames which need to be resolved and
	135	actually do the name resolution.
	136
	137	3. Call get(SECTION, ITEM) to collect the results, or items(SECTION) to
	138	iterate over them.
	139	"""
	140
	141	def __init__(me):
	142	"""
	143	Initialize a new, empty configuration parser.
	144	"""
	145	CP.RawConfigParser.__init__(me)
	146	me._resolver = BulkResolver()
	147
148	def resolve(me):
149	"""
150	Works out all of the hostnames which need resolving and resolves them.
151
152	Until you call this, attempts to fetch configuration items which need to
153	resolve hostnames will fail!
154	"""
155	for sec in me.sections():
156	for key, value in me.items(sec, resolvep = False):
157	for match in r_resolve.finditer(value):
158	me._resolver.prepare(match.group(1))
159	me._resolver.run()
160
161	def _expand(me, sec, string, resolvep):
162	"""
163	Expands $(...) and (optionally) $[...] placeholders in STRING.
164
165	The SEC is the configuration section from which to satisfy $(...)
166	requests. RESOLVEP is a boolean switch: do we bother to tax the resolver
167	or not? This is turned off by the resolve() method while it's collecting
168	hostnames to be resolved.
169	"""
170	string = r_ref.sub \
171	(lambda m: me.get(sec, m.group(1), resolvep), string)
172	if resolvep:
173	string = r_resolve.sub(lambda m: me._resolver.lookup(m.group(1)),
174	string)
175	return string
176
177	def has_option(me, sec, key):
178	"""
179	Decide whether section SEC has a configuration key KEY.
180
181	This version of the method properly handles the @inherit key.
182	"""
183	return CP.RawConfigParser.has_option(me, sec, key) or \
184	(CP.RawConfigParser.has_option(me, sec, '@inherit') and
185	me.has_option(CP.RawConfigParser.get(me, sec, '@inherit'), key))
186
187	def _get(me, basesec, sec, key, resolvep):
188	"""
189	Low-level option-fetching method.
190
191	Fetch the value for the named KEY from section SEC, or maybe
192	(recursively) the section which SEC inherits from.
193
194	The result is expanded, by _expend; RESOLVEP is passed to _expand to
195	control whether $[...] should be expanded in the result.
196
197	The BASESEC is the section for which the original request was made. This
198	will be different from SEC if we're recursing up the inheritance chain.
199
200	We also provide the default value for `name' here.
201	"""
202	try:
203	raw = CP.RawConfigParser.get(me, sec, key)
204	except CP.NoOptionError:
205	if key == 'name':
206	raw = basesec
207	elif CP.RawConfigParser.has_option(me, sec, '@inherit'):
208	raw = me._get(basesec,
209	CP.RawConfigParser.get(me, sec, '@inherit'),
210	key,
211	resolvep)
212	else:
213	raise
214	return me._expand(basesec, raw, resolvep)
215
216	def get(me, sec, key, resolvep = True):
217	"""
218	Retrieve the value of KEY from section SEC.
219	"""
220	return me._get(sec, sec, key, resolvep)
221
222	def items(me, sec, resolvep = True):
223	"""
224	Return a list of (NAME, VALUE) items in section SEC.
225
226	This extends the default method by handling the inheritance chain.
227	"""
228	d = {}
229	basesec = sec
230	while sec:
231	next = None
232	for key, value in CP.RawConfigParser.items(me, sec):
233	if key == '@inherit':
234	next = value
235	elif not key.startswith('@') and key not in d:
236	d[key] = me._expand(basesec, value, resolvep)
237	sec = next
238	return d.items()
239
240	###--------------------------------------------------------------------------
241	### Command-line handling.
242
243	def inputiter(things):
244	"""
245	Iterate over command-line arguments, returning corresponding open files.
246
247	If none were given, or one is `-', assume standard input; if one is a
248	directory, scan it for files other than backups; otherwise return the
249	opened files.
250	"""
251
252	if not things:
253	if OS.isatty(stdin.fileno()):
254	M.die('no input given, and stdin is a terminal')
255	yield stdin
256	else:
257	for thing in things:
258	if thing == '-':
259	yield stdin
260	elif OS.path.isdir(thing):
261	for item in OS.listdir(thing):
262	if item.endswith('~') or item.endswith('#'):
263	continue
264	name = OS.path.join(thing, item)
265	if not OS.path.isfile(name):
266	continue
267	yield file(name)
268	else:
269	yield file(thing)
270
271	def parse_options(argv = argv):
272	"""
273	Parse command-line options, returning a pair (OPTS, ARGS).
274	"""
275	M.ego(argv[0])
276	op = OptionParser(usage = '%prog [-c CDB] INPUT...',
277	version = '%%prog (tripe, version %s)' % VERSION)
278	op.add_option('-c', '--cdb', metavar = 'CDB',
279	dest = 'cdbfile', default = None,
280	help = 'Compile output into a CDB file.')
281	opts, args = op.parse_args(argv)
282	return opts, args
283
284	###--------------------------------------------------------------------------
285	### Main code.
286
287	def getconf(args):
288	"""
289	Read the configuration files and return the accumulated result.
290
291	We make sure that all hostnames have been properly resolved.
292	"""
293	conf = MyConfigParser()
294	for f in inputiter(args):
295	conf.readfp(f)
296	conf.resolve()
297	return conf
298
299	def output(conf, cdb):
300	"""
301	Output the configuration information CONF to the database CDB.
302
303	This is where the special `user' and `auto' database entries get set.
304	"""
305	auto = []
306	for sec in conf.sections():
307	if sec.startswith('@'):
308	continue
309	elif sec.startswith('$'):
310	label = sec
311	else:
312	label = 'P%s' % sec
313	if conf.has_option(sec, 'auto') and \
314	conf.get(sec, 'auto') in ('y', 'yes', 't', 'true', '1', 'on'):
315	auto.append(sec)
316	if conf.has_option(sec, 'user'):
317	cdb.add('U%s' % conf.get(sec, 'user'), sec)
318	url = M.URLEncode(laxp = True, semip = True)
319	for key, value in conf.items(sec):
320	if not key.startswith('@'):
321	url.encode(key, ' '.join(M.split(value)[0]))
322	cdb.add(label, url.result)
323	cdb.add('%AUTO', ' '.join(auto))
324	cdb.finish()
325
326	def main():
327	"""Main program."""
328	opts, args = parse_options()
329	if opts.cdbfile:
330	cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new')
331	else:
332	cdb = CDBFake()
333	conf = getconf(args[1:])
334	output(conf, cdb)
335
336	if __name__ == '__main__':
337	main()
338
339	###----- That's all, folks --------------------------------------------------