[tripe] / peerdb / tripe-newpeers.in

#! @PYTHON@
### -*-python-*-
###
### Build a CDB file from configuration file
###
### (c) 2007 Straylight/Edgeware
###

###----- Licensing notice ---------------------------------------------------
###
### This file is part of Trivial IP Encryption (TrIPE).
###
### TrIPE is free software; you can redistribute it and/or modify
### it under the terms of the GNU General Public License as published by
### the Free Software Foundation; either version 2 of the License, or
### (at your option) any later version.
###
### TrIPE is distributed in the hope that it will be useful,
### but WITHOUT ANY WARRANTY; without even the implied warranty of
### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
### GNU General Public License for more details.
###
### You should have received a copy of the GNU General Public License
### along with TrIPE; if not, write to the Free Software Foundation,
### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

VERSION = '@VERSION@'

###--------------------------------------------------------------------------
### External dependencies.

import ConfigParser as CP
import mLib as M
from optparse import OptionParser
import cdb as CDB
from sys import stdin, stdout, exit, argv
import re as RX
import os as OS

###--------------------------------------------------------------------------
### Utilities.

class CDBFake (object):
  """Like cdbmake, but just outputs data suitable for cdb-map."""
  def __init__(me, file = stdout):
    me.file = file
  def add(me, key, value):
    me.file.write('%s:%s\n' % (key, value))
  def finish(me):
    pass

###--------------------------------------------------------------------------
### A bulk DNS resolver.

class BulkResolver (object):
  """
  Resolve a number of DNS names in parallel.

  The BulkResovler resolves a number of hostnames in parallel.  Using it
  works in three phases:

    1. You call prepare(HOSTNAME) a number of times, to feed in the hostnames
       you're interested in.

    2. You call run() to actually drive the resolver.

    3. You call lookup(HOSTNAME) to get the address you wanted.  This will
       fail with KeyError if the resolver couldn't resolve the HOSTNAME.
  """

  def __init__(me):
    """Initialize the resolver."""
    me._resolvers = {}
    me._namemap = {}

  def prepare(me, host):
    """Prime the resolver to resolve the name HOST."""
    me._resolvers[host] = M.SelResolveByName \
                          (host,
                           lambda name, alias, addr:
                             me._resolved(host, addr[0]),
                           lambda: me._resolved(host, None))

  def run(me):
    """Run the background DNS resolver until it's finished."""
    while me._resolvers:
      M.select()

  def lookup(me, host):
    """
    Fetch the address corresponding to HOST.
    """
    addr = me._namemap[host]
    if addr is None:
      raise KeyError, host
    return addr

  def _resolved(me, host, addr):
    """Callback function: remember that ADDR is the address for HOST."""
    me._namemap[host] = addr
    del me._resolvers[host]

###--------------------------------------------------------------------------
### The configuration parser.

## Match a $(VAR) configuration variable reference; group 1 is the VAR.
r_ref = RX.compile(r'\$\(([^)]+)\)')

## Match a $[HOST] name resolution reference; group 1 is the HOST.
r_resolve = RX.compile(r'\$\[([^]]+)\]')

class MyConfigParser (CP.RawConfigParser):
  """
  A more advanced configuration parser.

  This has two major enhancements over the standard ConfigParser which are
  relevant to us.

    * It recognizes `@inherits' keys and follows them when expanding a
      value.

    * It recognizes `$(VAR)' references to configuration variables during
      expansion and processes them correctly.

    * It recognizes `$[HOST]' name-resolver requests and handles them
      correctly.

  Use:

    1. Call read(FILENAME) and/or read(FP, [FILENAME]) to slurp in the
       configuration data.

    2. Call resolve() to collect the hostnames which need to be resolved and
       actually do the name resolution.

    3. Call get(SECTION, ITEM) to collect the results, or items(SECTION) to
       iterate over them.
  """

  def __init__(me):
    """
    Initialize a new, empty configuration parser.
    """
    CP.RawConfigParser.__init__(me)
    me._resolver = BulkResolver()

  def resolve(me):
    """
    Works out all of the hostnames which need resolving and resolves them.

    Until you call this, attempts to fetch configuration items which need to
    resolve hostnames will fail!
    """
    for sec in me.sections():
      for key, value in me.items(sec, resolvep = False):
        for match in r_resolve.finditer(value):
          me._resolver.prepare(match.group(1))
    me._resolver.run()

  def _expand(me, sec, string, resolvep):
    """
    Expands $(...) and (optionally) $[...] placeholders in STRING.

    The SEC is the configuration section from which to satisfy $(...)
    requests.  RESOLVEP is a boolean switch: do we bother to tax the resolver
    or not?  This is turned off by the resolve() method while it's collecting
    hostnames to be resolved.
    """
    string = r_ref.sub \
             (lambda m: me.get(sec, m.group(1), resolvep), string)
    if resolvep:
      string = r_resolve.sub(lambda m: me._resolver.lookup(m.group(1)),
                             string)
    return string

  def has_option(me, sec, key):
    """
    Decide whether section SEC has a configuration key KEY.

    This version of the method properly handles the @inherit key.
    """
    return CP.RawConfigParser.has_option(me, sec, key) or \
           (CP.RawConfigParser.has_option(me, sec, '@inherit') and
            me.has_option(CP.RawConfigParser.get(me, sec, '@inherit'), key))

  def _get(me, basesec, sec, key, resolvep):
    """
    Low-level option-fetching method.

    Fetch the value for the named KEY from section SEC, or maybe
    (recursively) the section which SEC inherits from.

    The result is expanded, by _expend; RESOLVEP is passed to _expand to
    control whether $[...] should be expanded in the result.

    The BASESEC is the section for which the original request was made.  This
    will be different from SEC if we're recursing up the inheritance chain.

    We also provide the default value for `name' here.
    """
    try:
      raw = CP.RawConfigParser.get(me, sec, key)
    except CP.NoOptionError:
      if key == 'name':
        raw = basesec
      elif CP.RawConfigParser.has_option(me, sec, '@inherit'):
        raw = me._get(basesec,
                      CP.RawConfigParser.get(me, sec, '@inherit'),
                      key,
                      resolvep)
      else:
        raise
    return me._expand(basesec, raw, resolvep)

  def get(me, sec, key, resolvep = True):
    """
    Retrieve the value of KEY from section SEC.
    """
    return me._get(sec, sec, key, resolvep)

  def items(me, sec, resolvep = True):
    """
    Return a list of (NAME, VALUE) items in section SEC.

    This extends the default method by handling the inheritance chain.
    """
    d = {}
    basesec = sec
    while sec:
      next = None
      for key, value in CP.RawConfigParser.items(me, sec):
        if key == '@inherit':
          next = value
        elif not key.startswith('@') and key not in d:
          d[key] = me._expand(basesec, value, resolvep)
      sec = next
    return d.items()

###--------------------------------------------------------------------------
### Command-line handling.

def inputiter(things):
  """
  Iterate over command-line arguments, returning corresponding open files.

  If none were given, or one is `-', assume standard input; if one is a
  directory, scan it for files other than backups; otherwise return the
  opened files.
  """

  if not things:
    if OS.isatty(stdin.fileno()):
      M.die('no input given, and stdin is a terminal')
    yield stdin
  else:
    for thing in things:
      if thing == '-':
        yield stdin
      elif OS.path.isdir(thing):
        for item in OS.listdir(thing):
          if item.endswith('~') or item.endswith('#'):
            continue
          name = OS.path.join(thing, item)
          if not OS.path.isfile(name):
            continue
          yield file(name)
      else:
        yield file(thing)

def parse_options(argv = argv):
  """
  Parse command-line options, returning a pair (OPTS, ARGS).
  """
  M.ego(argv[0])
  op = OptionParser(usage = '%prog [-c CDB] INPUT...',
                    version = '%%prog (tripe, version %s)' % VERSION)
  op.add_option('-c', '--cdb', metavar = 'CDB',
                dest = 'cdbfile', default = None,
                help = 'Compile output into a CDB file.')
  opts, args = op.parse_args(argv)
  return opts, args

###--------------------------------------------------------------------------
### Main code.

def getconf(args):
  """
  Read the configuration files and return the accumulated result.

  We make sure that all hostnames have been properly resolved.
  """
  conf = MyConfigParser()
  for f in inputiter(args):
    conf.readfp(f)
  conf.resolve()
  return conf

def output(conf, cdb):
  """
  Output the configuration information CONF to the database CDB.

  This is where the special `user' and `auto' database entries get set.
  """
  auto = []
  for sec in conf.sections():
    if sec.startswith('@'):
      continue
    elif sec.startswith('$'):
      label = sec
    else:
      label = 'P%s' % sec
      if conf.has_option(sec, 'auto') and \
         conf.get(sec, 'auto') in ('y', 'yes', 't', 'true', '1', 'on'):
        auto.append(sec)
      if conf.has_option(sec, 'user'):
        cdb.add('U%s' % conf.get(sec, 'user'), sec)
    url = M.URLEncode(laxp = True, semip = True)
    for key, value in conf.items(sec):
      if not key.startswith('@'):
        url.encode(key, ' '.join(M.split(value)[0]))
    cdb.add(label, url.result)
  cdb.add('%AUTO', ' '.join(auto))
  cdb.finish()

def main():
  """Main program."""
  opts, args = parse_options()
  if opts.cdbfile:
    cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new')
  else:
    cdb = CDBFake()
  conf = getconf(args[1:])
  output(conf, cdb)

if __name__ == '__main__':
  main()

###----- That's all, folks --------------------------------------------------
Commit	Line	Data
6005ef9b MW	1	#! @PYTHON@
	2	### --python--
	3	###
	4	### Build a CDB file from configuration file
	5	###
	6	### (c) 2007 Straylight/Edgeware
	7	###
	8
	9	###----- Licensing notice ---------------------------------------------------
	10	###
	11	### This file is part of Trivial IP Encryption (TrIPE).
	12	###
	13	### TrIPE is free software; you can redistribute it and/or modify
	14	### it under the terms of the GNU General Public License as published by
	15	### the Free Software Foundation; either version 2 of the License, or
	16	### (at your option) any later version.
	17	###
	18	### TrIPE is distributed in the hope that it will be useful,
	19	### but WITHOUT ANY WARRANTY; without even the implied warranty of
	20	### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	21	### GNU General Public License for more details.
	22	###
	23	### You should have received a copy of the GNU General Public License
	24	### along with TrIPE; if not, write to the Free Software Foundation,
	25	### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
	26
	27	VERSION = '@VERSION@'
	28
	29	###--------------------------------------------------------------------------
	30	### External dependencies.
	31
	32	import ConfigParser as CP
	33	import mLib as M
	34	from optparse import OptionParser
	35	import cdb as CDB
	36	from sys import stdin, stdout, exit, argv
	37	import re as RX
	38	import os as OS
	39
	40	###--------------------------------------------------------------------------
	41	### Utilities.
	42
	43	class CDBFake (object):
	44	"""Like cdbmake, but just outputs data suitable for cdb-map."""
	45	def __init__(me, file = stdout):
	46	me.file = file
	47	def add(me, key, value):
	48	me.file.write('%s:%s\n' % (key, value))
	49	def finish(me):
	50	pass
	51
	52	###--------------------------------------------------------------------------
	53	### A bulk DNS resolver.
	54
	55	class BulkResolver (object):
	56	"""
	57	Resolve a number of DNS names in parallel.
	58
	59	The BulkResovler resolves a number of hostnames in parallel. Using it
	60	works in three phases:
	61
	62	1. You call prepare(HOSTNAME) a number of times, to feed in the hostnames
	63	you're interested in.
	64
65	2. You call run() to actually drive the resolver.
66
67	3. You call lookup(HOSTNAME) to get the address you wanted. This will
68	fail with KeyError if the resolver couldn't resolve the HOSTNAME.
69	"""
70
71	def __init__(me):
72	"""Initialize the resolver."""
73	me._resolvers = {}
74	me._namemap = {}
75
76	def prepare(me, host):
77	"""Prime the resolver to resolve the name HOST."""
78	me._resolvers[host] = M.SelResolveByName \
79	(host,
80	lambda name, alias, addr:
81	me._resolved(host, addr[0]),
82	lambda: me._resolved(host, None))
83
84	def run(me):
85	"""Run the background DNS resolver until it's finished."""
86	while me._resolvers:
87	M.select()
88
89	def lookup(me, host):
90	"""
91	Fetch the address corresponding to HOST.
92	"""
93	addr = me._namemap[host]
94	if addr is None:
95	raise KeyError, host
96	return addr
97
98	def _resolved(me, host, addr):
99	"""Callback function: remember that ADDR is the address for HOST."""
100	me._namemap[host] = addr
101	del me._resolvers[host]
102
103	###--------------------------------------------------------------------------
104	### The configuration parser.
105
106	## Match a $(VAR) configuration variable reference; group 1 is the VAR.
107	r_ref = RX.compile(r'\$\(([^)]+)\)')
108
109	## Match a $[HOST] name resolution reference; group 1 is the HOST.
110	r_resolve = RX.compile(r'\$\[([^]]+)\]')
111
112	class MyConfigParser (CP.RawConfigParser):
113	"""
114	A more advanced configuration parser.
115
116	This has two major enhancements over the standard ConfigParser which are
117	relevant to us.
118
119	* It recognizes `@inherits' keys and follows them when expanding a
120	value.
121
122	* It recognizes `$(VAR)' references to configuration variables during
123	expansion and processes them correctly.
124
125	* It recognizes `$[HOST]' name-resolver requests and handles them
126	correctly.
127
128	Use:
129
130	1. Call read(FILENAME) and/or read(FP, [FILENAME]) to slurp in the
131	configuration data.
132
133	2. Call resolve() to collect the hostnames which need to be resolved and
134	actually do the name resolution.
135
136	3. Call get(SECTION, ITEM) to collect the results, or items(SECTION) to
137	iterate over them.
138	"""
139
140	def __init__(me):
141	"""
142	Initialize a new, empty configuration parser.
143	"""
144	CP.RawConfigParser.__init__(me)
145	me._resolver = BulkResolver()
146
147	def resolve(me):
148	"""
149	Works out all of the hostnames which need resolving and resolves them.
150
151	Until you call this, attempts to fetch configuration items which need to
152	resolve hostnames will fail!
153	"""
154	for sec in me.sections():
155	for key, value in me.items(sec, resolvep = False):
156	for match in r_resolve.finditer(value):
157	me._resolver.prepare(match.group(1))
158	me._resolver.run()
159
160	def _expand(me, sec, string, resolvep):
161	"""
162	Expands $(...) and (optionally) $[...] placeholders in STRING.
163
164	The SEC is the configuration section from which to satisfy $(...)
165	requests. RESOLVEP is a boolean switch: do we bother to tax the resolver
166	or not? This is turned off by the resolve() method while it's collecting
167	hostnames to be resolved.
168	"""
169	string = r_ref.sub \
170	(lambda m: me.get(sec, m.group(1), resolvep), string)
171	if resolvep:
172	string = r_resolve.sub(lambda m: me._resolver.lookup(m.group(1)),
173	string)
174	return string
175
176	def has_option(me, sec, key):
177	"""
178	Decide whether section SEC has a configuration key KEY.
179
180	This version of the method properly handles the @inherit key.
181	"""
182	return CP.RawConfigParser.has_option(me, sec, key) or \
183	(CP.RawConfigParser.has_option(me, sec, '@inherit') and
184	me.has_option(CP.RawConfigParser.get(me, sec, '@inherit'), key))
185
186	def _get(me, basesec, sec, key, resolvep):
187	"""
188	Low-level option-fetching method.
189
190	Fetch the value for the named KEY from section SEC, or maybe
191	(recursively) the section which SEC inherits from.
192
193	The result is expanded, by _expend; RESOLVEP is passed to _expand to
194	control whether $[...] should be expanded in the result.
195
196	The BASESEC is the section for which the original request was made. This
197	will be different from SEC if we're recursing up the inheritance chain.
198
199	We also provide the default value for `name' here.
200	"""
201	try:
202	raw = CP.RawConfigParser.get(me, sec, key)
203	except CP.NoOptionError:
204	if key == 'name':
205	raw = basesec
206	elif CP.RawConfigParser.has_option(me, sec, '@inherit'):
207	raw = me._get(basesec,
208	CP.RawConfigParser.get(me, sec, '@inherit'),
209	key,
210	resolvep)
211	else:
212	raise
213	return me._expand(basesec, raw, resolvep)
214
215	def get(me, sec, key, resolvep = True):
216	"""
217	Retrieve the value of KEY from section SEC.
218	"""
219	return me._get(sec, sec, key, resolvep)
220
221	def items(me, sec, resolvep = True):
222	"""
223	Return a list of (NAME, VALUE) items in section SEC.
224
225	This extends the default method by handling the inheritance chain.
226	"""
227	d = {}
228	basesec = sec
229	while sec:
230	next = None
231	for key, value in CP.RawConfigParser.items(me, sec):
232	if key == '@inherit':
233	next = value
234	elif not key.startswith('@') and key not in d:
235	d[key] = me._expand(basesec, value, resolvep)
236	sec = next
237	return d.items()
238
239	###--------------------------------------------------------------------------
240	### Command-line handling.
241
242	def inputiter(things):
243	"""
244	Iterate over command-line arguments, returning corresponding open files.
245
246	If none were given, or one is `-', assume standard input; if one is a
247	directory, scan it for files other than backups; otherwise return the
248	opened files.
249	"""
250
251	if not things:
252	if OS.isatty(stdin.fileno()):
253	M.die('no input given, and stdin is a terminal')
254	yield stdin
255	else:
256	for thing in things:
257	if thing == '-':
258	yield stdin
259	elif OS.path.isdir(thing):
260	for item in OS.listdir(thing):
261	if item.endswith('~') or item.endswith('#'):
262	continue
263	name = OS.path.join(thing, item)
264	if not OS.path.isfile(name):
265	continue
266	yield file(name)
267	else:
268	yield file(thing)
269
270	def parse_options(argv = argv):
271	"""
272	Parse command-line options, returning a pair (OPTS, ARGS).
273	"""
274	M.ego(argv[0])
275	op = OptionParser(usage = '%prog [-c CDB] INPUT...',
276	version = '%%prog (tripe, version %s)' % VERSION)
277	op.add_option('-c', '--cdb', metavar = 'CDB',
278	dest = 'cdbfile', default = None,
279	help = 'Compile output into a CDB file.')
280	opts, args = op.parse_args(argv)
281	return opts, args
282
283	###--------------------------------------------------------------------------
284	### Main code.
285
286	def getconf(args):
287	"""
288	Read the configuration files and return the accumulated result.
289
290	We make sure that all hostnames have been properly resolved.
291	"""
292	conf = MyConfigParser()
293	for f in inputiter(args):
294	conf.readfp(f)
295	conf.resolve()
296	return conf
297
298	def output(conf, cdb):
299	"""
300	Output the configuration information CONF to the database CDB.
301
302	This is where the special `user' and `auto' database entries get set.
303	"""
304	auto = []
305	for sec in conf.sections():
306	if sec.startswith('@'):
307	continue
308	elif sec.startswith('$'):
309	label = sec
310	else:
311	label = 'P%s' % sec
312	if conf.has_option(sec, 'auto') and \
313	conf.get(sec, 'auto') in ('y', 'yes', 't', 'true', '1', 'on'):
314	auto.append(sec)
315	if conf.has_option(sec, 'user'):
316	cdb.add('U%s' % conf.get(sec, 'user'), sec)
317	url = M.URLEncode(laxp = True, semip = True)
318	for key, value in conf.items(sec):
319	if not key.startswith('@'):
320	url.encode(key, ' '.join(M.split(value)[0]))
321	cdb.add(label, url.result)
322	cdb.add('%AUTO', ' '.join(auto))
323	cdb.finish()
324
325	def main():
326	"""Main program."""
327	opts, args = parse_options()
328	if opts.cdbfile:
329	cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new')
330	else:
331	cdb = CDBFake()
332	conf = getconf(args[1:])
333	output(conf, cdb)
334
335	if __name__ == '__main__':
336	main()
337
338	###----- That's all, folks --------------------------------------------------