From 6005ef9bfba49124a25825a5b044d4f4cbf02792 Mon Sep 17 00:00:00 2001 Message-Id: <6005ef9bfba49124a25825a5b044d4f4cbf02792.1714891640.git.mdw@distorted.org.uk> From: Mark Wooding Date: Mon, 19 Apr 2010 21:11:04 +0100 Subject: [PATCH 1/1] init: Introduce the peer database. Organization: Straylight/Edgeware From: Mark Wooding Our services will require information about the various possible peers. This is held in a CDB file with an open-ended format, and constructed from a text database by a Python utility tripe-newpeers. This stuff doesn't yet have a Debian package to live in. That will appear in a few patches' time. --- Makefile.am | 1 + configure.ac | 1 + keys/tripe-keys.in | 22 ++- peerdb/Makefile.am | 57 +++++++ peerdb/peers.cdb.5.in | 119 +++++++++++++ peerdb/peers.in | 116 +++++++++++++ peerdb/peers.in.5.in | 211 +++++++++++++++++++++++ peerdb/tripe-newpeers.8.in | 111 ++++++++++++ peerdb/tripe-newpeers.in | 338 +++++++++++++++++++++++++++++++++++++ py/tripe.py.in | 1 + 10 files changed, 976 insertions(+), 1 deletion(-) create mode 100644 peerdb/Makefile.am create mode 100644 peerdb/peers.cdb.5.in create mode 100644 peerdb/peers.in create mode 100644 peerdb/peers.in.5.in create mode 100644 peerdb/tripe-newpeers.8.in create mode 100644 peerdb/tripe-newpeers.in diff --git a/Makefile.am b/Makefile.am index 6a814ef3..4446113f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -54,6 +54,7 @@ endif if HAVE_PYTHON SUBDIRS += svc SUBDIRS += py +SUBDIRS += peerdb endif ## Key-management. diff --git a/configure.ac b/configure.ac index 608298b0..ac12dbec 100644 --- a/configure.ac +++ b/configure.ac @@ -321,6 +321,7 @@ AC_CONFIG_FILES( [wireshark/Makefile] [init/Makefile] [py/Makefile] + [peerdb/Makefile] [keys/Makefile] [svc/Makefile] [mon/Makefile] diff --git a/keys/tripe-keys.in b/keys/tripe-keys.in index 6e947e52..7ed3aae7 100644 --- a/keys/tripe-keys.in +++ b/keys/tripe-keys.in @@ -59,6 +59,12 @@ rx_nonalpha = RX.compile(r'\W') ## Match the literal string "". rx_seq = RX.compile(r'\') +## Match a shell metacharacter. +rx_shmeta = RX.compile('[\\s`!"#$&*()\\[\\];\'|<>?\\\\]') + +## Match a character which needs escaping in a shell double-quoted string. +rx_shquote = RX.compile(r'["`$\\]') + ###-------------------------------------------------------------------------- ### Utility functions. @@ -97,6 +103,20 @@ def subst(s, rx, map): out.write(s[i:]) return out.getvalue() +def shell_quotify(arg): + """ + Quotify ARG to keep the shell happy. + + This isn't actually used for invoking commands, just for presentation + purposes; but correctness is still nice. + """ + if not rx_shmeta.search(arg): + return arg + elif arg.find("'") == -1: + return "'%s'" % arg + else: + return '"%s"' % rx_shquote.sub(lambda m: '\\' + m.group(0), arg) + def rmtree(path): """Delete the directory tree given by PATH.""" try: @@ -142,7 +162,7 @@ def run(args): else: nargs += a[1:].split() args = nargs - print '+ %s' % ' '.join(args) + print '+ %s' % ' '.join([shell_quotify(arg) for arg in args]) SYS.stdout.flush() rc = OS.spawnvp(OS.P_WAIT, args[0], args) if rc != 0: diff --git a/peerdb/Makefile.am b/peerdb/Makefile.am new file mode 100644 index 00000000..55477f4f --- /dev/null +++ b/peerdb/Makefile.am @@ -0,0 +1,57 @@ +### -*-makefile-*- +### +### Makefile for peer database +### +### (c) 2006 Straylight/Edgeware +### + +###----- Licensing notice --------------------------------------------------- +### +### This file is part of Trivial IP Encryption (TrIPE). +### +### TrIPE is free software; you can redistribute it and/or modify +### it under the terms of the GNU General Public License as published by +### the Free Software Foundation; either version 2 of the License, or +### (at your option) any later version. +### +### TrIPE is distributed in the hope that it will be useful, +### but WITHOUT ANY WARRANTY; without even the implied warranty of +### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +### GNU General Public License for more details. +### +### You should have received a copy of the GNU General Public License +### along with TrIPE; if not, write to the Free Software Foundation, +### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +include $(top_srcdir)/vars.am + +sbin_SCRIPTS = +man_MANS = + +###-------------------------------------------------------------------------- +### The peer database. + +sbin_SCRIPTS += tripe-newpeers +CLEANFILES += tripe-newpeers +EXTRA_DIST += tripe-newpeers.in +EXTRA_DIST += peers.in + +tripe-newpeers: tripe-newpeers.in Makefile + $(confsubst) $(srcdir)/tripe-newpeers.in >$@.new \ + $(SUBSTITUTIONS) && \ + chmod +x $@.new && mv $@.new $@ + +## Manual pages. +man_MANS += peers.cdb.5 +CLEANFILES += peers.cdb.5 +EXTRA_DIST += peers.cdb.5.in + +man_MANS += peers.in.5 +CLEANFILES += peers.in.5 +EXTRA_DIST += peers.in.5.in + +man_MANS += tripe-newpeers.8 +CLEANFILES += tripe-newpeers.8 +EXTRA_DIST += tripe-newpeers.8.in + +###----- That's all, folks -------------------------------------------------- diff --git a/peerdb/peers.cdb.5.in b/peerdb/peers.cdb.5.in new file mode 100644 index 00000000..187de0f2 --- /dev/null +++ b/peerdb/peers.cdb.5.in @@ -0,0 +1,119 @@ +.\" -*-nroff-*- +.\". +.\" Manual for the peer database file format +.\" +.\" (c) 2008 Straylight/Edgeware +.\" +. +.\"----- Licensing notice --------------------------------------------------- +.\" +.\" This file is part of Trivial IP Encryption (TrIPE). +.\" +.\" TrIPE is free software; you can redistribute it and/or modify +.\" it under the terms of the GNU General Public License as published by +.\" the Free Software Foundation; either version 2 of the License, or +.\" (at your option) any later version. +.\" +.\" TrIPE is distributed in the hope that it will be useful, +.\" but WITHOUT ANY WARRANTY; without even the implied warranty of +.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +.\" GNU General Public License for more details. +.\" +.\" You should have received a copy of the GNU General Public License +.\" along with TrIPE; if not, write to the Free Software Foundation, +.\" Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +. +.\"-------------------------------------------------------------------------- +.so ../defs.man.in \"@@@PRE@@@ +. +.\"-------------------------------------------------------------------------- +.TH peers.cdb 5 "27 March 2008" "Straylight/Edgeware" "TrIPE: Trivial IP Encryption" +. +.\"-------------------------------------------------------------------------- +.SH "NAME" +. +peers.cdb \- compiled peer database +. +.\"-------------------------------------------------------------------------- +.SH "DESCRIPTION" +. +The +.B \*(/c/peers.cdb +file is a +.BR cdb (5) +format database containing information about peers in the TrIPE network, +and how to connect to them. It is set up by the +.BR tripe-newpeers (8) +program based on input in a +.BR peers.in (5) +file or files. +.SS "Database records" +The database contains four kinds of records. The type of record can be +inferred from the first character of the record's key. +.hP \*o +.I "Peer records" +have keys of the form +.BI P name \fR. +The record consists of a collection of key-value +pairs in the +.B form-urlencoded +format specified by RFC1866, except that key-value pairs are separated +by semicolon +.RB ` ; ' +characters. Some of the keys have meaning to various tools and +services; others are available for local use. +.hP \*o +.I "User records" +have keys of the form +.BI U name \fR. +The record consists of a peer name (i.e., a name for which a +.BI P name +record exists); all characters are significant. User records are used +by the +.BR connect (8) +service to map between user names presented to its +.B PASSIVE +command and peer names. No particular relationship between TrIPE user +names and system users is necessary. +.hP \*o +.I "Local records" +have keys of the form +.BI $ key \fR. +The record consists of key-value pairs in +.B form-urlencoded +format, just as for peer records. Their meaning is currently not +defined. +.hP \*o +.I "Special records" +have keys of the form +.BI % key \fR. +The record format is idiosyncratic. The special records currently +defined are described below. +.SS "Special records" +The following special records are defined. +.TP +.B %AUTO +The record contains a list of a space-separated list of peer names +(i.e., names for which a +.BI P name +record exists). It is read by the +.BR connect (8) +service as a list of peers for which active connections should be made +automatically. +. +.\"-------------------------------------------------------------------------- +.SH "SEE ALSO" +. +.BR cdb (5), +.BR tripe (8). +.PP +.BR tripe-newpeers (8), +.BR peers.in (5), +.BR connect (8). +. +.\"-------------------------------------------------------------------------- +.SH "AUTHOR" +. +Mark Wooding, +. +.\"----- That's all, folks -------------------------------------------------- diff --git a/peerdb/peers.in b/peerdb/peers.in new file mode 100644 index 00000000..16acbc8e --- /dev/null +++ b/peerdb/peers.in @@ -0,0 +1,116 @@ +;;; -*-conf-windows-*- +;;; +;;; Peers description file +;;; +;;; You're best off not editing this file at all; instead, drop a file +;;; containing your overriden settings alongside. + +;;;-------------------------------------------------------------------------- +;;; Global defaults. +;;; +;;; The paramaters here affect all peer definitions. It mainly contains +;;; information about the local site. You will need to customize it. + +[@GLOBAL] + +;; domain: the domain name for your VPN; used to form default tunnel +;; addresses. +domain = vpn.example.com + +;; myhost: my (internal) host name; used by the default laddr. +myhost = thishost + +;; laddr: the local address for point-to-point interfaces. +laddr = $[$(myhost).$(domain)] + +;; raddr: the remote address for point-to-point interfaces. +raddr = $[$(name).$(domain)] + +;; ifname: the name to set on point-to-point interfaces. +ifname = vpn-$(name) + +;; ifup: script to set up a tunnel interface ready for use. The installed +;; script is good for Linux hosts. +ifup = /usr/sbin/tripe-ifup + +;; every: interval for checking that this connection is alive. +every = 2m + +;; timeout: how long to wait for a ping response before giving up. +timeout = 10s + +;; retries: how many ping attempts to make before declaring the connection +;; dead. +retries = 5 + +;;;-------------------------------------------------------------------------- +;;; Active-peers defaults. +;;; +;;; The parameters here affect both active and dynamic connections. The +;;; defaults should be good for most sites, though you may wish to add extra +;;; settings. + +[@ACTIVE] +@inherit = @GLOBAL + +;; port: the port on which the peer's tripe(8) daemon is running. The +;; default is the port officially allocated by IANA. +port = 4070 + +;; host: the external host name (or dotted-quad IP address) of the host +;; running tripe(8). This should be overridden explicitly in each peer +;; definition. +host = override-me + +;; peer: the address specification (see tripe-admin(5)) to use to connect to +;; the remote peer. +peer = INET $[$(host)] $(port) + +;;;-------------------------------------------------------------------------- +;;; Dynamic-peers defaults. +;;; +;;; The parameters here affect peers to whom dynamic connections are made. +;;; The user and connect parameters probably need customizing. + +[@DYNAMIC] +@inherit = @ACTIVE + +;; cork: whether to wait for a key-exchange packet from the peer before +;; sending one of our own. +cork = t + +;; ssh-user: user to connect as; used by the connect parameter. +ssh-user = tripe + +;; connect: shell command to use to wake up the remote peer and establish the +;; connection. +connect = ssh -q $(ssh-user)@$[$(host)] + +;; keepalive: how often to send NOP packets to keep the connection alive, at +;; least in the minds of intermediate stateful firewalls and NAT routers. +keepalive = 2m + +;; watch: whether to watch this connection and retry it if it drops. +watch = t + +;;;-------------------------------------------------------------------------- +;;; Passive-peers defaults. +;;; +;;; The parameters here affect passive peers, i.e., those to whom dynamic +;;; connections are made. The dynamic connection protocol establishes most +;;; of the parameters and these defaults are probably pretty good. + +[@PASSIVE] +@inherit = @GLOBAL + +;; peer: mark this entry as being a passive peer. +peer = PASSIVE + +;; user: the string which the dynamic peer's connect command will present to +;; the CONNECT service. +user = $(name) + +;; watch: whether to watch this connection and drop it if it dies. +watch = t + +;;;----- That's all, folks -------------------------------------------------- diff --git a/peerdb/peers.in.5.in b/peerdb/peers.in.5.in new file mode 100644 index 00000000..7b0127b1 --- /dev/null +++ b/peerdb/peers.in.5.in @@ -0,0 +1,211 @@ +.\" -*-nroff-*- +.\". +.\" Manual for the peer configuration file +.\" +.\" (c) 2008 Straylight/Edgeware +.\" +. +.\"----- Licensing notice --------------------------------------------------- +.\" +.\" This file is part of Trivial IP Encryption (TrIPE). +.\" +.\" TrIPE is free software; you can redistribute it and/or modify +.\" it under the terms of the GNU General Public License as published by +.\" the Free Software Foundation; either version 2 of the License, or +.\" (at your option) any later version. +.\" +.\" TrIPE is distributed in the hope that it will be useful, +.\" but WITHOUT ANY WARRANTY; without even the implied warranty of +.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +.\" GNU General Public License for more details. +.\" +.\" You should have received a copy of the GNU General Public License +.\" along with TrIPE; if not, write to the Free Software Foundation, +.\" Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +. +.\"-------------------------------------------------------------------------- +.so ../defs.man.in \"@@@PRE@@@ +. +.\"-------------------------------------------------------------------------- +.TH peers.in 5 "27 March 2008" "Straylight/Edgeware" "TrIPE: Trivial IP Encryption" +. +.\"-------------------------------------------------------------------------- +.SH "NAME" +. +peers.in \- source form for TrIPE peer database +. +.\"-------------------------------------------------------------------------- +.SH "DESCRIPTION" +. +The +.B peers.in +file is a plain text configuration file. It is read by +.BR tripe-newpeers (8) +in order to produce the +.BR tripe.cdb (8) +database used by services and other tools. +.SS "General structure" +The configuration file is line-oriented. Blank lines are ignored; lines +beginning with a hash +.RB ` # ' +or semicolon +.RB ` ; ' +are ignored. The file is divided into sections by section headers, +which are lines of the form +.IP +.BI [ name ] +.PP +Within each section are a number of assignments, of the form +.IP +.IB key " = " value +.PP +or (entirely equivalent) +.IP +.IB key ": " value +.PP +The +.I key +must start in the left hand column. The +.I value +may span multiple lines if subsequent lines begin with whitespace, in +the manner of RFC822 headers. +.PP +There is a special case to be aware of: if a section doesn't specify a +value for the key +.B name +then the section's own name is used as a default. +.PP +The following substitutions are made in the body of a value. +.hP \*o +An occurrence of +.BI $( key ) +is replaced by the value assigned to the given +.IR key . +.hP \*o +An occurrence of +.BI $[ host ] +is replaced by the IP address of the named +.IR host . +Note that +.I host +may itself contain +.BI $( key ) +substitutions. +.PP +There is a simple concept of +.I inheritance +for sections. If a section contains an assignment +.IP +.BI "@inherits = " parent +.PP +then any lookups which can't be satisfied in that section will be +satisfied instead from the +.I parent +section (and, if necessary, its parent in turn, and so on). Note that +.BI $( key ) +substitutions in the resulting value will be satisfied from the original +section (though falling back to scanning the parent section). For +example, given the sections +.VS +[parent] +detail = in parent +blurb = expand $(detail) + +.PP +Apart from its effect on lookups, as just described, the +.B @inherits +key is entirely ignored. In particular, it is never written to the +database. +.SS "Standard keys and their meanings" +The following keys have meanings to programs in the TrIPE suite. Other +keys may be used by separately distributed extensions or for local use. +The descriptions given are summaries only; see the references for +details. +.TP +.B auto +If true, include the peer in the +.B %AUTO +record. Used by +.BR tripe-newpeers (8); +described below. +.TP +.B user +Peer will make active connection as +.IR user . +Used by +.BR tripe-newpeers (8); +described below. +.SS "Conversion" +This section describes how the textual +.B peers.in +file is converted into the +.BR peers.cdb (5) +database. +.PP +The handling of each section depends on its name. +.hP \*o +Sections whose names have the form +.BI @ whatever +are ignored (though their contents may be relevant if the section is +named in another section's +.B @inherits +key). +.hP \*o +Sections whose names have the form +.BI $ whatever +are written to local-type database records with the same name. The keys +and values defined in the section (and its parent section, if it +contains an +.B @inherits +key) are stored in the record using +.B form-urlencoding +as defined in RFC1822, except that the key-value pairs are separated by +semicolons +.RB ` ; ' +rather than ampersands +.RB ` & '. +The +.B @inherits +key-value pair is not written to the database. +.hP \*o +Other sections are written to peer-type database records, named +.BI P name \fR, +in exactly the same way as for local-type records. However, two special +actions are also taken. +.IP +Firstly, if there is a key +.B auto +in the section (or in its parent, etc.), and the value is +.BR y , +.BR yes . +.BR t , +.BR true , +.BR 1 , +or +.BR on , +then the section's name is added in the special +.B %AUTO +record. +.IP +Secondly, if there is a key +.B user +in the section (or in its parent, etc.), then a user record +.BI U user +is created whose contents is the section name. +. +.\"-------------------------------------------------------------------------- +.SH "SEE ALSO" +. +.BR cdb (5), +.BR tripe (8). +.PP +.BR tripe-newpeers (8), +.BR peers.cdb (5), +.BR tripe-ifup (8). +. +.\"-------------------------------------------------------------------------- +.SH "AUTHOR" +. +Mark Wooding, +. +.\"----- That's all, folks -------------------------------------------------- diff --git a/peerdb/tripe-newpeers.8.in b/peerdb/tripe-newpeers.8.in new file mode 100644 index 00000000..17d00550 --- /dev/null +++ b/peerdb/tripe-newpeers.8.in @@ -0,0 +1,111 @@ +.\" -*-nroff-*- +.\". +.\" Manual for the peer database compiler +.\" +.\" (c) 2008 Straylight/Edgeware +.\" +. +.\"----- Licensing notice --------------------------------------------------- +.\" +.\" This file is part of Trivial IP Encryption (TrIPE). +.\" +.\" TrIPE is free software; you can redistribute it and/or modify +.\" it under the terms of the GNU General Public License as published by +.\" the Free Software Foundation; either version 2 of the License, or +.\" (at your option) any later version. +.\" +.\" TrIPE is distributed in the hope that it will be useful, +.\" but WITHOUT ANY WARRANTY; without even the implied warranty of +.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +.\" GNU General Public License for more details. +.\" +.\" You should have received a copy of the GNU General Public License +.\" along with TrIPE; if not, write to the Free Software Foundation, +.\" Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +. +.\"-------------------------------------------------------------------------- +.so ../defs.man.in \"@@@PRE@@@ +. +.\"-------------------------------------------------------------------------- +.TH tripe-newpeers 8 "11 December 2008" "Straylight/Edgeware" "TrIPE: Trivial IP Encryption" +. +.\"-------------------------------------------------------------------------- +.SH "NAME" +. +tripe-newpeers \- compile peer database from sources +. +.\"-------------------------------------------------------------------------- +.SH "SYNOPSIS" +. +.B tripe-newpeers +.RB [ \-c +.IR cdb-file ] +.IR input ... +. +.\"-------------------------------------------------------------------------- +.SH "DESCRIPTION" +. +The +.BR tripe-newpeers (8) +program reads in a number of configuration files in +.BR peers.in (5) +format, and writes a compiled version of the resulting database. The +program accepts the following options. +.TP +.B "\-h, \-\-help" +Write a help message to standard output, and exit. +.TP +.B "\-v, \-\-version" +Write the program's version number to standard output, and exit. +.TP +.BI "\-c, \-\-cdb=" cdb-file +Write a compiled database in +.BR cdb (5) +format to +.IR cdb-file . +.PP +In the absence of a +.B \-c +option, +.B tripe-newpeers +will write a textual representation of its database to standard output. +The textual representation consists of a sequence of lines of the form +.IP +.IB key : value +.PP +This is unambiguous as long as peer names don't contain colons, since +the values do not contain newlines. +.PP +The program reads all of the +.I input +configuration files listed on its command line. If an +.I input +file name is +.B \- +then standard input is read. If no filenames are given, and standard +input is not a terminal, then standard input is read. If no filenames +are given and standard input +.I is +a terminal then an error message is issued. The configuration files are +all merged together as if they had been a single file. The output +database is constructed as described in +.BR peers.in (5). +. +.\"-------------------------------------------------------------------------- +.SH "SEE ALSO" +. +.BR cdb (5), +.BR tripe (8). +.PP +.BR peers.in (5), +.BR peers.cdb (5), +.BR connect (8). +.BR watch (8). +.BR tripe-ifup (8). +. +.\"-------------------------------------------------------------------------- +.SH "AUTHOR" +. +Mark Wooding, +. +.\"----- That's all, folks -------------------------------------------------- diff --git a/peerdb/tripe-newpeers.in b/peerdb/tripe-newpeers.in new file mode 100644 index 00000000..d7205845 --- /dev/null +++ b/peerdb/tripe-newpeers.in @@ -0,0 +1,338 @@ +#! @PYTHON@ +### -*-python-*- +### +### Build a CDB file from configuration file +### +### (c) 2007 Straylight/Edgeware +### + +###----- Licensing notice --------------------------------------------------- +### +### This file is part of Trivial IP Encryption (TrIPE). +### +### TrIPE is free software; you can redistribute it and/or modify +### it under the terms of the GNU General Public License as published by +### the Free Software Foundation; either version 2 of the License, or +### (at your option) any later version. +### +### TrIPE is distributed in the hope that it will be useful, +### but WITHOUT ANY WARRANTY; without even the implied warranty of +### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +### GNU General Public License for more details. +### +### You should have received a copy of the GNU General Public License +### along with TrIPE; if not, write to the Free Software Foundation, +### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +VERSION = '@VERSION@' + +###-------------------------------------------------------------------------- +### External dependencies. + +import ConfigParser as CP +import mLib as M +from optparse import OptionParser +import cdb as CDB +from sys import stdin, stdout, exit, argv +import re as RX +import os as OS + +###-------------------------------------------------------------------------- +### Utilities. + +class CDBFake (object): + """Like cdbmake, but just outputs data suitable for cdb-map.""" + def __init__(me, file = stdout): + me.file = file + def add(me, key, value): + me.file.write('%s:%s\n' % (key, value)) + def finish(me): + pass + +###-------------------------------------------------------------------------- +### A bulk DNS resolver. + +class BulkResolver (object): + """ + Resolve a number of DNS names in parallel. + + The BulkResovler resolves a number of hostnames in parallel. Using it + works in three phases: + + 1. You call prepare(HOSTNAME) a number of times, to feed in the hostnames + you're interested in. + + 2. You call run() to actually drive the resolver. + + 3. You call lookup(HOSTNAME) to get the address you wanted. This will + fail with KeyError if the resolver couldn't resolve the HOSTNAME. + """ + + def __init__(me): + """Initialize the resolver.""" + me._resolvers = {} + me._namemap = {} + + def prepare(me, host): + """Prime the resolver to resolve the name HOST.""" + me._resolvers[host] = M.SelResolveByName \ + (host, + lambda name, alias, addr: + me._resolved(host, addr[0]), + lambda: me._resolved(host, None)) + + def run(me): + """Run the background DNS resolver until it's finished.""" + while me._resolvers: + M.select() + + def lookup(me, host): + """ + Fetch the address corresponding to HOST. + """ + addr = me._namemap[host] + if addr is None: + raise KeyError, host + return addr + + def _resolved(me, host, addr): + """Callback function: remember that ADDR is the address for HOST.""" + me._namemap[host] = addr + del me._resolvers[host] + +###-------------------------------------------------------------------------- +### The configuration parser. + +## Match a $(VAR) configuration variable reference; group 1 is the VAR. +r_ref = RX.compile(r'\$\(([^)]+)\)') + +## Match a $[HOST] name resolution reference; group 1 is the HOST. +r_resolve = RX.compile(r'\$\[([^]]+)\]') + +class MyConfigParser (CP.RawConfigParser): + """ + A more advanced configuration parser. + + This has two major enhancements over the standard ConfigParser which are + relevant to us. + + * It recognizes `@inherits' keys and follows them when expanding a + value. + + * It recognizes `$(VAR)' references to configuration variables during + expansion and processes them correctly. + + * It recognizes `$[HOST]' name-resolver requests and handles them + correctly. + + Use: + + 1. Call read(FILENAME) and/or read(FP, [FILENAME]) to slurp in the + configuration data. + + 2. Call resolve() to collect the hostnames which need to be resolved and + actually do the name resolution. + + 3. Call get(SECTION, ITEM) to collect the results, or items(SECTION) to + iterate over them. + """ + + def __init__(me): + """ + Initialize a new, empty configuration parser. + """ + CP.RawConfigParser.__init__(me) + me._resolver = BulkResolver() + + def resolve(me): + """ + Works out all of the hostnames which need resolving and resolves them. + + Until you call this, attempts to fetch configuration items which need to + resolve hostnames will fail! + """ + for sec in me.sections(): + for key, value in me.items(sec, resolvep = False): + for match in r_resolve.finditer(value): + me._resolver.prepare(match.group(1)) + me._resolver.run() + + def _expand(me, sec, string, resolvep): + """ + Expands $(...) and (optionally) $[...] placeholders in STRING. + + The SEC is the configuration section from which to satisfy $(...) + requests. RESOLVEP is a boolean switch: do we bother to tax the resolver + or not? This is turned off by the resolve() method while it's collecting + hostnames to be resolved. + """ + string = r_ref.sub \ + (lambda m: me.get(sec, m.group(1), resolvep), string) + if resolvep: + string = r_resolve.sub(lambda m: me._resolver.lookup(m.group(1)), + string) + return string + + def has_option(me, sec, key): + """ + Decide whether section SEC has a configuration key KEY. + + This version of the method properly handles the @inherit key. + """ + return CP.RawConfigParser.has_option(me, sec, key) or \ + (CP.RawConfigParser.has_option(me, sec, '@inherit') and + me.has_option(CP.RawConfigParser.get(me, sec, '@inherit'), key)) + + def _get(me, basesec, sec, key, resolvep): + """ + Low-level option-fetching method. + + Fetch the value for the named KEY from section SEC, or maybe + (recursively) the section which SEC inherits from. + + The result is expanded, by _expend; RESOLVEP is passed to _expand to + control whether $[...] should be expanded in the result. + + The BASESEC is the section for which the original request was made. This + will be different from SEC if we're recursing up the inheritance chain. + + We also provide the default value for `name' here. + """ + try: + raw = CP.RawConfigParser.get(me, sec, key) + except CP.NoOptionError: + if key == 'name': + raw = basesec + elif CP.RawConfigParser.has_option(me, sec, '@inherit'): + raw = me._get(basesec, + CP.RawConfigParser.get(me, sec, '@inherit'), + key, + resolvep) + else: + raise + return me._expand(basesec, raw, resolvep) + + def get(me, sec, key, resolvep = True): + """ + Retrieve the value of KEY from section SEC. + """ + return me._get(sec, sec, key, resolvep) + + def items(me, sec, resolvep = True): + """ + Return a list of (NAME, VALUE) items in section SEC. + + This extends the default method by handling the inheritance chain. + """ + d = {} + basesec = sec + while sec: + next = None + for key, value in CP.RawConfigParser.items(me, sec): + if key == '@inherit': + next = value + elif not key.startswith('@') and key not in d: + d[key] = me._expand(basesec, value, resolvep) + sec = next + return d.items() + +###-------------------------------------------------------------------------- +### Command-line handling. + +def inputiter(things): + """ + Iterate over command-line arguments, returning corresponding open files. + + If none were given, or one is `-', assume standard input; if one is a + directory, scan it for files other than backups; otherwise return the + opened files. + """ + + if not things: + if OS.isatty(stdin.fileno()): + M.die('no input given, and stdin is a terminal') + yield stdin + else: + for thing in things: + if thing == '-': + yield stdin + elif OS.path.isdir(thing): + for item in OS.listdir(thing): + if item.endswith('~') or item.endswith('#'): + continue + name = OS.path.join(thing, item) + if not OS.path.isfile(name): + continue + yield file(name) + else: + yield file(thing) + +def parse_options(argv = argv): + """ + Parse command-line options, returning a pair (OPTS, ARGS). + """ + M.ego(argv[0]) + op = OptionParser(usage = '%prog [-c CDB] INPUT...', + version = '%%prog (tripe, version %s)' % VERSION) + op.add_option('-c', '--cdb', metavar = 'CDB', + dest = 'cdbfile', default = None, + help = 'Compile output into a CDB file.') + opts, args = op.parse_args(argv) + return opts, args + +###-------------------------------------------------------------------------- +### Main code. + +def getconf(args): + """ + Read the configuration files and return the accumulated result. + + We make sure that all hostnames have been properly resolved. + """ + conf = MyConfigParser() + for f in inputiter(args): + conf.readfp(f) + conf.resolve() + return conf + +def output(conf, cdb): + """ + Output the configuration information CONF to the database CDB. + + This is where the special `user' and `auto' database entries get set. + """ + auto = [] + for sec in conf.sections(): + if sec.startswith('@'): + continue + elif sec.startswith('$'): + label = sec + else: + label = 'P%s' % sec + if conf.has_option(sec, 'auto') and \ + conf.get(sec, 'auto') in ('y', 'yes', 't', 'true', '1', 'on'): + auto.append(sec) + if conf.has_option(sec, 'user'): + cdb.add('U%s' % conf.get(sec, 'user'), sec) + url = M.URLEncode(laxp = True, semip = True) + for key, value in conf.items(sec): + if not key.startswith('@'): + url.encode(key, ' '.join(M.split(value)[0])) + cdb.add(label, url.result) + cdb.add('%AUTO', ' '.join(auto)) + cdb.finish() + +def main(): + """Main program.""" + opts, args = parse_options() + if opts.cdbfile: + cdb = CDB.cdbmake(opts.cdbfile, opts.cdbfile + '.new') + else: + cdb = CDBFake() + conf = getconf(args[1:]) + output(conf, cdb) + +if __name__ == '__main__': + main() + +###----- That's all, folks -------------------------------------------------- diff --git a/py/tripe.py.in b/py/tripe.py.in index d158d1f5..79060106 100644 --- a/py/tripe.py.in +++ b/py/tripe.py.in @@ -139,6 +139,7 @@ PACKAGE = "@PACKAGE@" VERSION = "@VERSION@" tripesock = OS.environ.get('TRIPESOCK', OS.path.join(socketdir, 'tripesock')) +peerdb = OS.environ.get('TRIPEPEERDB', 'peers.cdb') ###-------------------------------------------------------------------------- ### Connection to the server. -- [mdw]