X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=blobdiff_plain;f=make-secnet-sites;h=820d272cad726bb8b451780ebaef4de87daf59a0;hb=b2920262e82ea5939511ea841eb33a094cea566d;hp=682840f1415d2da1b967a5ec81b420f2f128ceb5;hpb=ff05a229397c75142725f45cad191ce4a00625ce;p=secnet.git

diff --git a/make-secnet-sites b/make-secnet-sites
index 682840f..820d272 100755
--- a/make-secnet-sites
+++ b/make-secnet-sites
@@ -1,19 +1,21 @@
-#! /usr/bin/env python
-# Copyright (C) 2001 Stephen Early <steve@greenend.org.uk>
+#! /usr/bin/env python3
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
+# This file is part of secnet.
+# See README for full list of copyright holders.
+#
+# secnet is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 # 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# secnet is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+# version 3 along with secnet; if not, see
+# https://www.gnu.org/licenses/gpl.html.
 
 """VPN sites file manipulation.
 
@@ -44,438 +46,812 @@ no-suppress-args
 cd ~/secnet/sites-test/
 execute ~/secnet/make-secnet-sites.py -u vpnheader groupfiles sites
 
-This program is part of secnet. It relies on the "ipaddr" library from
-Cendio Systems AB.
+This program is part of secnet.
 
 """
 
+from __future__ import print_function
+from __future__ import unicode_literals
+from builtins import int
+
 import string
 import time
 import sys
 import os
+import getopt
+import re
+import argparse
+import math
+
+import ipaddress
+
+# entry 0 is "near the executable", or maybe from PYTHONPATH=.,
+# which we don't want to preempt
+sys.path.insert(1,"/usr/local/share/secnet")
+sys.path.insert(1,"/usr/share/secnet")
+import ipaddrset
+import base91
+
+from argparseactionnoyes import ActionNoYes
+
+VERSION="0.1.18"
+
+max_version = 1
+
+from sys import version_info
+if version_info.major == 2:  # for python2
+    import codecs
+    sys.stdin = codecs.getreader('utf-8')(sys.stdin)
+    sys.stdout = codecs.getwriter('utf-8')(sys.stdout)
+    import io
+    open=lambda f,m='r': io.open(f,m,encoding='utf-8')
+
+max={'rsa_bits':8200,'name':33,'dh_bits':8200,'algname':127}
+
+def debugrepr(*args):
+	if debug_level > 0:
+		print(repr(args), file=sys.stderr)
+
+def base91s_encode(bindata):
+	return base91.encode(bindata).replace('"',"-")
+
+def base91s_decode(string):
+	return base91.decode(string.replace("-",'"'))
+
+class Tainted:
+	def __init__(self,s,tline=None,tfile=None):
+		self._s=s
+		self._ok=None
+		self._line=line if tline is None else tline
+		self._file=file if tfile is None else tfile
+	def __eq__(self,e):
+		return self._s==e
+	def __ne__(self,e):
+		# for Python2
+		return not self.__eq__(e)
+	def __str__(self):
+		raise RuntimeError('direct use of Tainted value')
+	def __repr__(self):
+		return 'Tainted(%s)' % repr(self._s)
+
+	def _bad(self,what,why):
+		assert(self._ok is not True)
+		self._ok=False
+		complain('bad parameter: %s: %s' % (what, why))
+		return False
+
+	def _max_ok(self,what,maxlen):
+		if len(self._s) > maxlen:
+			return self._bad(what,'too long (max %d)' % maxlen)
+		return True
+
+	def _re_ok(self,bad,what,maxlen=None):
+		if maxlen is None: maxlen=max[what]
+		self._max_ok(what,maxlen)
+		if self._ok is False: return False
+		if bad.search(self._s):
+			#print(repr(self), file=sys.stderr)
+			return self._bad(what,'bad syntax')
+		return True
+
+	def _rtnval(self, is_ok, ifgood, ifbad=''):
+		if is_ok:
+			assert(self._ok is not False)
+			self._ok=True
+			return ifgood
+		else:
+			assert(self._ok is not True)
+			self._ok=False
+			return ifbad
+
+	def _rtn(self, is_ok, ifbad=''):
+		return self._rtnval(is_ok, self._s, ifbad)
+
+	def raw(self):
+		return self._s
+	def raw_mark_ok(self):
+		# caller promises to throw if syntax was dangeorus
+		return self._rtn(True)
+
+	def output(self):
+		if self._ok is False: return ''
+		if self._ok is True: return self._s
+		print('%s:%d: unchecked/unknown additional data "%s"' %
+		      (self._file,self._line,self._s),
+		      file=sys.stderr)
+		sys.exit(1)
+
+	bad_name=re.compile(r'^[^a-zA-Z]|[^-_0-9a-zA-Z]')
+	# secnet accepts _ at start of names, but we reserve that
+	bad_name_counter=0
+	def name(self,what='name'):
+		ok=self._re_ok(Tainted.bad_name,what)
+		return self._rtn(ok,
+				 '_line%d_%s' % (self._line, id(self)))
+
+	def keyword(self):
+		ok=self._s in keywords or self._s in levels
+		if not ok:
+			complain('unknown keyword %s' % self._s)
+		return self._rtn(ok)
+
+	bad_hex=re.compile(r'[^0-9a-fA-F]')
+	def bignum_16(self,kind,what):
+		maxlen=(max[kind+'_bits']+3)/4
+		ok=self._re_ok(Tainted.bad_hex,what,maxlen)
+		return self._rtn(ok)
+
+	bad_num=re.compile(r'[^0-9]')
+	def bignum_10(self,kind,what):
+		maxlen=math.ceil(max[kind+'_bits'] / math.log10(2))
+		ok=self._re_ok(Tainted.bad_num,what,maxlen)
+		return self._rtn(ok)
+
+	def number(self,minn,maxx,what='number'):
+		# not for bignums
+		ok=self._re_ok(Tainted.bad_num,what,10)
+		if ok:
+			v=int(self._s)
+			if v<minn or v>maxx:
+				ok=self._bad(what,'out of range %d..%d'
+					     % (minn,maxx))
+		return self._rtnval(ok,v,minn)
+
+	def hexid(self,byteslen,what):
+		ok=self._re_ok(Tainted.bad_hex,what,byteslen*2)
+		if ok:
+			if len(self._s) < byteslen*2:
+				ok=self._bad(what,'too short')
+		return self._rtn(ok,ifbad='00'*byteslen)
+
+	bad_host=re.compile(r'[^-\][_.:0-9a-zA-Z]')
+	# We permit _ so we can refer to special non-host domains
+	# which have A and AAAA RRs.  This is a crude check and we may
+	# still produce config files with syntactically invalid
+	# domains or addresses, but that is OK.
+	def host(self):
+		ok=self._re_ok(Tainted.bad_host,'host/address',255)
+		return self._rtn(ok)
+
+	bad_email=re.compile(r'[^-._0-9a-z@!$%^&*=+~/]')
+	# ^ This does not accept all valid email addresses.  That's
+	# not really possible with this input syntax.  It accepts
+	# all ones that don't require quoting anywhere in email
+	# protocols (and also accepts some invalid ones).
+	def email(self):
+		ok=self._re_ok(Tainted.bad_email,'email address',1023)
+		return self._rtn(ok)
+
+	bad_groupname=re.compile(r'^[^_A-Za-z]|[^-+_0-9A-Za-z]')
+	def groupname(self):
+		ok=self._re_ok(Tainted.bad_groupname,'group name',64)
+		return self._rtn(ok)
+
+	bad_base91=re.compile(r'[^!-~]|[\'\"\\]')
+	def base91(self,what='base91'):
+		ok=self._re_ok(Tainted.bad_base91,what,4096)
+		return self._rtn(ok)
+
+class ArgActionLambda(argparse.Action):
+	def __init__(self, fn, **kwargs):
+		self.fn=fn
+		argparse.Action.__init__(self,**kwargs)
+	def __call__(self,ap,ns,values,option_string):
+		self.fn(values,ns,ap,option_string)
+
+def parse_args():
+	global service
+	global inputfile
+	global header
+	global groupfiledir
+	global sitesfile
+	global outputfile
+	global group
+	global user
+	global of
+	global prefix
+	global key_prefix
+	global debug_level
+	global output_version
+
+	ap = argparse.ArgumentParser(description='process secnet sites files')
+	ap.add_argument('--userv', '-u', action='store_true',
+			help='userv service fragment update mode')
+	ap.add_argument('--conf-key-prefix', action=ActionNoYes,
+			default=True,
+		 help='prefix conf file key names derived from sites data')
+	ap.add_argument('--output-version', nargs=1, type=int,
+			help='sites file output version',
+			default=[max_version])
+	ap.add_argument('--prefix', '-P', nargs=1,
+			help='set prefix')
+	ap.add_argument('--debug', '-D', action='count', default=0)
+	ap.add_argument('arg',nargs=argparse.REMAINDER)
+	av = ap.parse_args()
+	debug_level = av.debug
+	debugrepr('av',av)
+	service = 1 if av.userv else 0
+	prefix = '' if av.prefix is None else av.prefix[0]
+	key_prefix = av.conf_key_prefix
+	output_version = av.output_version[0]
+	if service:
+		if len(av.arg)!=4:
+			print("Wrong number of arguments")
+			sys.exit(1)
+		(header, groupfiledir, sitesfile, group) = av.arg
+		group = Tainted(group,0,'command line')
+		# untrusted argument from caller
+		if "USERV_USER" not in os.environ:
+			print("Environment variable USERV_USER not found")
+			sys.exit(1)
+		user=os.environ["USERV_USER"]
+		# Check that group is in USERV_GROUP
+		if "USERV_GROUP" not in os.environ:
+			print("Environment variable USERV_GROUP not found")
+			sys.exit(1)
+		ugs=os.environ["USERV_GROUP"]
+		ok=0
+		for i in ugs.split():
+			if group==i: ok=1
+		if not ok:
+			print("caller not in group %s"%group)
+			sys.exit(1)
+	else:
+		if len(av.arg)>3:
+			print("Too many arguments")
+			sys.exit(1)
+		(inputfile, outputfile) = (av.arg + [None]*2)[0:2]
+
+parse_args()
+
+# Classes describing possible datatypes in the configuration file
+
+class basetype:
+	"Common protocol for configuration types."
+	def add(self,obj,w):
+		complain("%s %s already has property %s defined"%
+			(obj.type,obj.name,w[0].raw()))
+	def forsites(self,version,copy,fs):
+		return copy
+
+class conflist:
+	"A list of some kind of configuration type."
+	def __init__(self,subtype,w):
+		self.subtype=subtype
+		self.list=[subtype(w)]
+	def add(self,obj,w):
+		self.list.append(self.subtype(w))
+	def __str__(self):
+		return ', '.join(map(str, self.list))
+	def forsites(self,version,copy,fs):
+		most_recent=self.list[len(self.list)-1]
+		return most_recent.forsites(version,copy,fs)
+def listof(subtype):
+	return lambda w: conflist(subtype, w)
+
+class single_ipaddr (basetype):
+	"An IP address"
+	def __init__(self,w):
+		self.addr=ipaddress.ip_address(w[1].raw_mark_ok())
+	def __str__(self):
+		return '"%s"'%self.addr
 
-sys.path.append("/usr/local/share/secnet")
-sys.path.append("/usr/share/secnet")
-import ipaddr
-
-VERSION="0.1.13"
-
-class vpn:
-	def __init__(self,name):
-		self.name=name
-		self.allow_defs=0
-		self.locations={}
-		self.defs={}
-
-class location:
-	def __init__(self,name,vpn):
-		self.group=None
-		self.name=name
-		self.allow_defs=1
-		self.vpn=vpn
-		self.sites={}
-		self.defs={}
-
-class site:
-	def __init__(self,name,location):
-		self.name=name
-		self.allow_defs=1
-		self.location=location
-		self.defs={}
-
-class nets:
+class networks (basetype):
+	"A set of IP addresses specified as a list of networks"
 	def __init__(self,w):
-		self.w=w
-		self.set=ipaddr.ip_set()
+		self.set=ipaddrset.IPAddressSet()
 		for i in w[1:]:
-			x=string.split(i,"/")
-			self.set.append(ipaddr.network(x[0],x[1],
-				ipaddr.DEMAND_NETWORK))
-	def subsetof(self,s):
-		# I'd like to do this:
-		# return self.set.is_subset(s)
-		# but there isn't an is_subset() method
-		# Instead we see if we intersect with the complement of s
-		sc=s.set.complement()
-		i=sc.intersection(self.set)
-		return i.is_empty()
-	def out(self):
-		if (self.w[0]=='restrict-nets'): pattern="# restrict-nets %s;"
-		else:
-			pattern="link netlink { routes %s; };"
-		return pattern%string.join(map(lambda x:'"%s/%s"'%(x.ip_str(),
-				x.mask.netmask_bits_str),
-				self.set.as_list_of_networks()),",")
+			x=ipaddress.ip_network(i.raw_mark_ok(),strict=True)
+			self.set.append([x])
+	def __str__(self):
+		return ",".join(map((lambda n: '"%s"'%n), self.set.networks()))
 
-class dhgroup:
+class dhgroup (basetype):
+	"A Diffie-Hellman group"
 	def __init__(self,w):
-		self.mod=w[1]
-		self.gen=w[2]
-	def out(self):
-		return 'dh diffie-hellman("%s","%s");'%(self.mod,self.gen)
+		self.mod=w[1].bignum_16('dh','dh mod')
+		self.gen=w[2].bignum_16('dh','dh gen')
+	def __str__(self):
+		return 'diffie-hellman("%s","%s")'%(self.mod,self.gen)
 
-class hash:
+class hash (basetype):
+	"A choice of hash function"
 	def __init__(self,w):
-		self.ht=w[1]
+		hname=w[1]
+		self.ht=hname.raw()
 		if (self.ht!='md5' and self.ht!='sha1'):
 			complain("unknown hash type %s"%(self.ht))
-	def out(self):
-		return 'hash %s;'%(self.ht)
+			self.ht=None
+		else:
+			hname.raw_mark_ok()
+	def __str__(self):
+		return '%s'%(self.ht)
 
-class email:
+class email (basetype):
+	"An email address"
 	def __init__(self,w):
-		self.addr=w[1]
-	def out(self):
-		return '# Contact email address: <%s>'%(self.addr)
+		self.addr=w[1].email()
+	def __str__(self):
+		return '<%s>'%(self.addr)
 
-class num:
+class boolean (basetype):
+	"A boolean"
 	def __init__(self,w):
-		self.what=w[0]
-		self.n=string.atol(w[1])
-	def out(self):
-		return '%s %d;'%(self.what,self.n)
+		v=w[1]
+		if re.match('[TtYy1]',v.raw()):
+			self.b=True
+			v.raw_mark_ok()
+		elif re.match('[FfNn0]',v.raw()):
+			self.b=False
+			v.raw_mark_ok()
+		else:
+			complain("invalid boolean value");
+	def __str__(self):
+		return ['False','True'][self.b]
+
+class num (basetype):
+	"A decimal number"
+	def __init__(self,w):
+		self.n=w[1].number(0,0x7fffffff)
+	def __str__(self):
+		return '%d'%(self.n)
 
-class address:
+class serial (basetype):
 	def __init__(self,w):
-		self.w=w
-		self.adr=w[1]
-		self.port=string.atoi(w[2])
-		if (self.port<1 or self.port>65535):
-			complain("invalid port number")
-	def out(self):
-		return 'address "%s"; port %d;'%(self.adr,self.port)
-
-class rsakey:
+		self.i=w[1].hexid(4,'serial')
+	def __str__(self):
+		return self.i
+	def forsites(self,version,copy,fs):
+		if version < 2: return []
+		return copy
+
+class address (basetype):
+	"A DNS name and UDP port number"
+	def __init__(self,w):
+		self.adr=w[1].host()
+		self.port=w[2].number(1,65536,'port')
+	def __str__(self):
+		return '"%s"; port %d'%(self.adr,self.port)
+
+class pubkey (basetype):
+	"Some kind of publie key"
 	def __init__(self,w):
-		self.l=string.atoi(w[1])
-		self.e=w[2]
-		self.n=w[3]
-	def out(self):
-		return 'key rsa-public("%s","%s");'%(self.e,self.n)
+		self.a=w[1].name('algname')
+		self.d=w[2].base91();
+	def __str__(self):
+		return 'make-public("%s","%s")'%(self.a,self.d)
 
-class mobileoption:
+class rsakey (pubkey):
+	"An RSA public key"
+	def __init__(self,w):
+		self.l=w[1].number(0,max['rsa_bits'],'rsa len')
+		self.e=w[2].bignum_10('rsa','rsa e')
+		self.n=w[3].bignum_10('rsa','rsa n')
+		if len(w) >= 5: w[4].email()
+	def __str__(self):
+		return 'rsa-public("%s","%s")'%(self.e,self.n)
+		# this specialisation means we can generate files
+		# compatible with old secnet executables
+
+def somepubkey(w):
+	if w[0]=='pubkey':
+		return rsakey(w)
+	elif w[0]=='pub':
+		return pubkey(w)
+	else:
+		assert(False)
+
+# Possible properties of configuration nodes
+keywords={
+ 'contact':(email,"Contact address"),
+ 'dh':(dhgroup,"Diffie-Hellman group"),
+ 'hash':(hash,"Hash function"),
+ 'key-lifetime':(num,"Maximum key lifetime (ms)"),
+ 'setup-timeout':(num,"Key setup timeout (ms)"),
+ 'setup-retries':(num,"Maximum key setup packet retries"),
+ 'wait-time':(num,"Time to wait after unsuccessful key setup (ms)"),
+ 'renegotiate-time':(num,"Time after key setup to begin renegotiation (ms)"),
+ 'restrict-nets':(networks,"Allowable networks"),
+ 'networks':(networks,"Claimed networks"),
+ 'pub':(listof(somepubkey),"new style public site key",'pubkey'),
+ 'pubkey':(listof(somepubkey),"RSA public site key"),
+ 'peer':(single_ipaddr,"Tunnel peer IP address"),
+ 'address':(address,"External contact address and port"),
+ 'mobile':(boolean,"Site is mobile"),
+}
+
+def sp(name,value):
+	"Simply output a property - the default case"
+	return "%s %s;\n"%(name,value)
+
+# All levels support these properties
+global_properties={
+	'contact':(lambda name,value:"# Contact email address: %s\n"%(value)),
+	'dh':sp,
+	'hash':sp,
+	'key-lifetime':sp,
+	'setup-timeout':sp,
+	'setup-retries':sp,
+	'wait-time':sp,
+	'renegotiate-time':sp,
+	'restrict-nets':(lambda name,value:"# restrict-nets %s\n"%value),
+}
+
+class level:
+	"A level in the configuration hierarchy"
+	depth=0
+	leaf=0
+	allow_properties={}
+	require_properties={}
+	def __init__(self,w):
+		self.type=w[0].keyword()
+		self.name=w[1].name()
+		self.properties={}
+		self.children={}
+	def indent(self,w,t):
+		w.write("                 "[:t])
+	def prop_out(self,n):
+		return self.allow_properties[n](n,str(self.properties[n]))
+	def output_props(self,w,ind):
+		for i in sorted(self.properties.keys()):
+			if self.allow_properties[i]:
+				self.indent(w,ind)
+				w.write("%s"%self.prop_out(i))
+	def kname(self):
+		return ((self.type[0].upper() if key_prefix else '')
+			+ self.name)
+	def output_data(self,w,path):
+		ind = 2*len(path)
+		self.indent(w,ind)
+		w.write("%s {\n"%(self.kname()))
+		self.output_props(w,ind+2)
+		if self.depth==1: w.write("\n");
+		for k in sorted(self.children.keys()):
+			c=self.children[k]
+			c.output_data(w,path+(c,))
+		self.indent(w,ind)
+		w.write("};\n")
+
+class vpnlevel(level):
+	"VPN level in the configuration hierarchy"
+	depth=1
+	leaf=0
+	type="vpn"
+	allow_properties=global_properties.copy()
+	require_properties={
+	 'contact':"VPN admin contact address"
+	}
 	def __init__(self,w):
-		self.w=w
-	def out(self):
-		return '# netlink-options "soft";'
+		level.__init__(self,w)
+	def output_vpnflat(self,w,path):
+		"Output flattened list of site names for this VPN"
+		ind=2*(len(path)+1)
+		self.indent(w,ind)
+		w.write("%s {\n"%(self.kname()))
+		for i in self.children.keys():
+			self.children[i].output_vpnflat(w,path+(self,))
+		w.write("\n")
+		self.indent(w,ind+2)
+		w.write("all-sites %s;\n"%
+			','.join(map(lambda i: i.kname(),
+				     self.children.values())))
+		self.indent(w,ind)
+		w.write("};\n")
+
+class locationlevel(level):
+	"Location level in the configuration hierarchy"
+	depth=2
+	leaf=0
+	type="location"
+	allow_properties=global_properties.copy()
+	require_properties={
+	 'contact':"Location admin contact address",
+	}
+	def __init__(self,w):
+		level.__init__(self,w)
+		self.group=w[2].groupname()
+	def output_vpnflat(self,w,path):
+		ind=2*(len(path)+1)
+		self.indent(w,ind)
+		# The "path=path,self=self" abomination below exists because
+		# Python didn't support nested_scopes until version 2.1
+		#
+		#"/"+self.name+"/"+i
+		w.write("%s %s;\n"%(self.kname(),','.join(
+			map(lambda x,path=path,self=self:
+			    '/'.join([prefix+"vpn-data"] + list(map(
+				    lambda i: i.kname(),
+				    path+(self,x)))),
+			    self.children.values()))))
+
+class sitelevel(level):
+	"Site level (i.e. a leafnode) in the configuration hierarchy"
+	depth=3
+	leaf=1
+	type="site"
+	allow_properties=global_properties.copy()
+	allow_properties.update({
+	 'address':sp,
+	 'networks':None,
+	 'peer':None,
+	 'pub':None,
+	 'pubkey':None,
+	 'mobile':sp,
+	})
+	require_properties={
+	 'dh':"Diffie-Hellman group",
+	 'contact':"Site admin contact address",
+	 'networks':"Networks claimed by the site",
+	 'hash':"hash function",
+	 'peer':"Gateway address of the site",
+	 'pubkey':"RSA public key of the site",
+	}
+	def __init__(self,w):
+		level.__init__(self,w)
+	def output_data(self,w,path):
+		ind=2*len(path)
+		np='/'.join(map(lambda i: i.name, path))
+		self.indent(w,ind)
+		w.write("%s {\n"%(self.kname()))
+		self.indent(w,ind+2)
+		w.write("name \"%s\";\n"%(np,))
+		self.indent(w,ind+2)
+		w.write("key %s;\n"%str(self.properties["pubkey"].list[0]))
+		self.output_props(w,ind+2)
+		self.indent(w,ind+2)
+		w.write("link netlink {\n");
+		self.indent(w,ind+4)
+		w.write("routes %s;\n"%str(self.properties["networks"]))
+		self.indent(w,ind+4)
+		w.write("ptp-address %s;\n"%str(self.properties["peer"]))
+		self.indent(w,ind+2)
+		w.write("};\n")
+		self.indent(w,ind)
+		w.write("};\n")
+
+# Levels in the configuration file
+# (depth,properties)
+levels={'vpn':vpnlevel, 'location':locationlevel, 'site':sitelevel}
 
 def complain(msg):
-	global complaints
-	print ("%s line %d: "%(file,line))+msg
-	complaints=complaints+1
+	"Complain about a particular input line"
+	moan(("%s line %d: "%(file,line))+msg)
 def moan(msg):
+	"Complain about something in general"
 	global complaints
-	print msg;
+	print(msg);
+	if complaints is None: sys.exit(1)
 	complaints=complaints+1
 
-# We don't allow redefinition of properties (because that would allow things
-# like restrict-nets to be redefined, which would be bad)
-def set(obj,defs,w):
-	if (obj.allow_defs | allow_defs):
-		if (obj.defs.has_key(w[0])):
-			complain("%s is already defined"%(w[0]))
-		else:
-			t=defs[w[0]]
-			obj.defs[w[0]]=t(w)
-
-# Process a line of configuration file
-def pline(i):
-	global allow_defs, group, current_vpn, current_location, current_object
-	w=string.split(i)
-	if len(w)==0: return
+class UntaintedRoot():
+	def __init__(self,s): self._s=s
+	def name(self): return self._s
+	def keyword(self): return self._s
+
+root=level([UntaintedRoot(x) for x in ['root','root']])
+# All vpns are children of this node
+obstack=[root]
+allow_defs=0   # Level above which new definitions are permitted
+
+def set_property(obj,w):
+	"Set a property on a configuration node"
+	prop=w[0]
+	propname=prop.raw_mark_ok()
+	kw=keywords[propname]
+	if len(kw) >= 3: propname=kw[2] # for aliases
+	if propname in obj.properties:
+		obj.properties[propname].add(obj,w)
+	else:
+		obj.properties[propname]=kw[0](w)
+	return obj.properties[propname]
+
+class FilterState:
+	def __init__(self):
+		self.reset()
+	def reset(self):
+		# called when we enter a new node,
+		# in particular, at the start of each site
+		pass
+
+def pline(il,filterstate,allow_include=False):
+	"Process a configuration file line"
+	global allow_defs, obstack, root
+	w=il.rstrip('\n').split()
+	if len(w)==0: return ['']
+	w=list([Tainted(x) for x in w])
 	keyword=w[0]
+	current=obstack[len(obstack)-1]
+	copyout_core=lambda: ' '.join([ww.output() for ww in w])
+	indent='    '*len(obstack)
+	copyout=lambda: [indent + copyout_core() + '\n']
 	if keyword=='end-definitions':
-		allow_defs=0
-		current_vpn=None
-		current_location=None
-		current_object=None
-		return
-	if keyword=='vpn':
-		if vpns.has_key(w[1]):
-			current_vpn=vpns[w[1]]
-			current_object=current_vpn
+		keyword.raw_mark_ok()
+		allow_defs=sitelevel.depth
+		obstack=[root]
+		return copyout()
+	if keyword=='include':
+		if not allow_include:
+			complain("include not permitted here")
+			return []
+		if len(w) != 2:
+			complain("include requires one argument")
+			return []
+		newfile=os.path.join(os.path.dirname(file),w[1].raw_mark_ok())
+		# ^ user of "include" is trusted so raw_mark_ok is good
+		return pfilepath(newfile,allow_include=allow_include)
+	if keyword.raw() in levels:
+		# We may go up any number of levels, but only down by one
+		newdepth=levels[keyword.raw_mark_ok()].depth
+		currentdepth=len(obstack) # actually +1...
+		if newdepth<=currentdepth:
+			obstack=obstack[:newdepth]
+		if newdepth>currentdepth:
+			complain("May not go from level %d to level %d"%
+				(currentdepth-1,newdepth))
+		# See if it's a new one (and whether that's permitted)
+		# or an existing one
+		current=obstack[len(obstack)-1]
+		tname=w[1].name()
+		if tname in current.children:
+			# Not new
+			current=current.children[tname]
+			if service and group and current.depth==2:
+				if group!=current.group:
+					complain("Incorrect group!")
+				w[2].groupname()
 		else:
-			if allow_defs:
-				current_vpn=vpn(w[1])
-				vpns[w[1]]=current_vpn
-				current_object=current_vpn
-			else:
-				complain("no new VPN definitions allowed")
-		return
-	if (current_vpn==None):
-		complain("no VPN defined yet")
-		return
-	# Keywords that can apply at all levels
-	if mldefs.has_key(w[0]):
-		set(current_object,mldefs,w)
-		return
-	if keyword=='location':
-		if (current_vpn.locations.has_key(w[1])):
-			current_location=current_vpn.locations[w[1]]
-			current_object=current_location
-			if (group and not allow_defs and 
-				current_location.group!=group):
-				complain(("must be group %s to access "+
-					"location %s")%(current_location.group,
-					w[1]))
-		else:
-			if allow_defs:
-				if reserved.has_key(w[1]):
-					complain("reserved location name")
-					return
-				current_location=location(w[1],current_vpn)
-				current_vpn.locations[w[1]]=current_location
-				current_object=current_location
-			else:
-				complain("no new location definitions allowed")
-		return
-	if (current_location==None):
-		complain("no locations defined yet")
-		return
-	if keyword=='group':
-		current_location.group=w[1]
-		return
-	if keyword=='site':
-		if (current_location.sites.has_key(w[1])):
-			current_object=current_location.sites[w[1]]
-		else:
-			if reserved.has_key(w[1]):
-				complain("reserved site name")
-				return
-			current_object=site(w[1],current_location)
-			current_location.sites[w[1]]=current_object
-		return
-	if keyword=='endsite':
-		if isinstance(current_object,site):
-			current_object=current_object.location
-		else:
-			complain("not currently defining a site")
-		return
-	# Keywords that can only apply to sites
-	if isinstance(current_object,site):
-		if sitedefs.has_key(w[0]):
-			set(current_object,sitedefs,w)
-			return
+			# New
+			# Ignore depth check for now
+			nl=levels[keyword.raw()](w)
+			if nl.depth<allow_defs:
+				complain("New definitions not allowed at "
+					"level %d"%nl.depth)
+				# we risk crashing if we continue
+				sys.exit(1)
+			current.children[tname]=nl
+			current=nl
+		filterstate.reset()
+		obstack.append(current)
+		return copyout()
+	if keyword.raw() not in current.allow_properties:
+		complain("Property %s not allowed at %s level"%
+			(keyword.raw(),current.type))
+		return []
+	elif current.depth == vpnlevel.depth < allow_defs:
+		complain("Not allowed to set VPN properties here")
+		return []
 	else:
-		if sitedefs.has_key(w[0]):
-			complain("keyword '%s' can only be used in the "
-				"context of a site definition"%(w[0]))
-			return
-	complain("unknown keyword '%s'"%(w[0]))
+		prop=set_property(current,w)
+		out=[copyout_core()]
+		out=prop.forsites(output_version,out,filterstate)
+		if len(out)==0: return [indent + '#', copyout_core(), '\n']
+		return [indent + ' '.join(out) + '\n']
 
-def pfile(name,lines):
+	complain("unknown keyword '%s'"%(keyword.raw()))
+
+def pfilepath(pathname,allow_include=False):
+	f=open(pathname)
+	outlines=pfile(pathname,f.readlines(),allow_include=allow_include)
+	f.close()
+	return outlines
+
+def pfile(name,lines,allow_include=False):
+	"Process a file"
 	global file,line
 	file=name
 	line=0
+	outlines=[]
+	filterstate = FilterState()
 	for i in lines:
 		line=line+1
 		if (i[0]=='#'): continue
-		if (i[len(i)-1]=='\n'):	i=i[:len(i)-1] # strip trailing LF
-		pline(i)
+		outlines += pline(i,filterstate,allow_include=allow_include)
+	return outlines
 
 def outputsites(w):
-	w.write("# secnet sites file autogenerated by make-secnet-sites.py "
+	"Output include file for secnet configuration"
+	w.write("# secnet sites file autogenerated by make-secnet-sites "
 		+"version %s\n"%VERSION)
-	w.write("# %s\n\n"%time.asctime(time.localtime(time.time())))
+	w.write("# %s\n"%time.asctime(time.localtime(time.time())))
+	w.write("# Command line: %s\n\n"%' '.join(sys.argv))
 
 	# Raw VPN data section of file
-	w.write("vpn-data {\n")
-	for i in vpns.values():
-		w.write("  %s {\n"%i.name)
-		for d in i.defs.values():
-			w.write("    %s\n"%d.out())
-		w.write("\n")
-		for l in i.locations.values():
-			w.write("    %s {\n"%l.name)
-			for d in l.defs.values():
-				w.write("      %s\n"%d.out())
-			for s in l.sites.values():
-				w.write("      %s {\n"%s.name)
-				w.write('        name "%s/%s/%s";\n'%
-					(i.name,l.name,s.name))
-				for d in s.defs.values():
-					w.write("        %s\n"%d.out())
-				w.write("      };\n")
-			w.write("    };\n")
-		w.write("  };\n")
+	w.write(prefix+"vpn-data {\n")
+	for i in root.children.values():
+		i.output_data(w,(i,))
 	w.write("};\n")
 
 	# Per-VPN flattened lists
-	w.write("vpn {\n")
-	for i in vpns.values():
-		w.write("  %s {\n"%(i.name))
-		for l in i.locations.values():
-			tmpl="vpn-data/%s/%s/%%s"%(i.name,l.name)
-			slist=[]
-			for s in l.sites.values(): slist.append(tmpl%s.name)
-			w.write("    %s %s;\n"%(l.name,string.join(slist,",")))
-		w.write("\n    all-sites %s;\n"%
-			string.join(i.locations.keys(),","))
-		w.write("  };\n")
+	w.write(prefix+"vpn {\n")
+	for i in root.children.values():
+		i.output_vpnflat(w,())
 	w.write("};\n")
 
 	# Flattened list of sites
-	w.write("all-sites %s;\n"%string.join(map(lambda x:"vpn/%s/all-sites"%
-		x,vpns.keys()),","))
-
-# Are we being invoked from userv?
-service=0
-# If we are, which group does the caller want to modify?
-group=None
-
-vpns={}
-allow_defs=1
-current_vpn=None
-current_location=None
-current_object=None
+	w.write(prefix+"all-sites %s;\n"%",".join(
+		map(lambda x:"%svpn/%s/all-sites"%(prefix,x.kname()),
+			root.children.values())))
 
 line=0
 file=None
 complaints=0
 
-# Things that can be defined at any level
-mldefs={
-	'dh':dhgroup,
-	'hash':hash,
-	'contact':email,
-	'key-lifetime':num,
-	'setup-retries':num,
-	'setup-timeout':num,
-	'wait-time':num,
-	'renegotiate-time':num,
-	'restrict-nets':nets
-	}
-
-# Things that can only be defined for sites
-sitedefs={
-	'address':address,
-	'networks':nets,
-	'pubkey':rsakey,
-	'mobile':mobileoption
-	}
-
-# Reserved vpn/location/site names
-reserved={'all-sites':None}
-reserved.update(mldefs)
-reserved.update(sitedefs)
-
-# Each site must have the following defined at some level:
-required={
-	'dh':"Diffie-Hellman group",
-	'networks':"network list",
-	'pubkey':"public key",
-	'hash':"hash function"
-	}
-
-if len(sys.argv)<2:
-	pfile("stdin",sys.stdin.readlines())
-	of=sys.stdout
-else:
-	if sys.argv[1]=='-u':
-		if len(sys.argv)!=6:
-			print "Wrong number of arguments"
-			sys.exit(1)
-		service=1
-		header=sys.argv[2]
-		groupfiledir=sys.argv[3]
-		sitesfile=sys.argv[4]
-		group=sys.argv[5]
-		if not os.environ.has_key("USERV_USER"):
-			print "Environment variable USERV_USER not found"
-			sys.exit(1)
-		user=os.environ["USERV_USER"]
-		# Check that group is in USERV_GROUP
-		if not os.environ.has_key("USERV_GROUP"):
-			print "Environment variable USERV_GROUP not found"
-			sys.exit(1)
-		ugs=os.environ["USERV_GROUP"]
-		ok=0
-		for i in string.split(ugs):
-			if group==i: ok=1
-		if not ok:
-			print "caller not in group %s"%group
-			sys.exit(1)
-		f=open(header)
-		headerinput=f.readlines()
-		f.close()
-		pfile(header,headerinput)
-		userinput=sys.stdin.readlines()
-		pfile("user input",userinput)
-	else:
-		if len(sys.argv)>3:
-			print "Too many arguments"
-			sys.exit(1)
-		f=open(sys.argv[1])
-		pfile(sys.argv[1],f.readlines())
-		f.close()
-		of=sys.stdout
-		if len(sys.argv)>2:
-			of=open(sys.argv[2],'w')
-
 # Sanity check section
+# Delete nodes where leaf=0 that have no children
+
+def live(n):
+	"Number of leafnodes below node n"
+	if n.leaf: return 1
+	for i in n.children.keys():
+		if live(n.children[i]): return 1
+	return 0
+def delempty(n):
+	"Delete nodes that have no leafnode children"
+	for i in list(n.children.keys()):
+		delempty(n.children[i])
+		if not live(n.children[i]):
+			del n.children[i]
+
+# Check that all constraints are met (as far as I can tell
+# restrict-nets/networks/peer are the only special cases)
+
+def checkconstraints(n,p,ra):
+	new_p=p.copy()
+	new_p.update(n.properties)
+	for i in n.require_properties.keys():
+		if i not in new_p:
+			moan("%s %s is missing property %s"%
+				(n.type,n.name,i))
+	for i in new_p.keys():
+		if i not in n.allow_properties:
+			moan("%s %s has forbidden property %s"%
+				(n.type,n.name,i))
+	# Check address range restrictions
+	if "restrict-nets" in n.properties:
+		new_ra=ra.intersection(n.properties["restrict-nets"].set)
+	else:
+		new_ra=ra
+	if "networks" in n.properties:
+		if not n.properties["networks"].set <= new_ra:
+			moan("%s %s networks out of bounds"%(n.type,n.name))
+		if "peer" in n.properties:
+			if not n.properties["networks"].set.contains(
+				n.properties["peer"].addr):
+				moan("%s %s peer not in networks"%(n.type,n.name))
+	for i in n.children.keys():
+		checkconstraints(n.children[i],new_p,new_ra)
 
-# Delete locations that have no sites defined
-for i in vpns.values():
-	for l in i.locations.keys():
-		if (len(i.locations[l].sites.values())==0):
-			del i.locations[l]
-
-# Delete VPNs that have no locations with sites defined
-for i in vpns.keys():
-	if (len(vpns[i].locations.values())==0):
-		del vpns[i]
-
-# Check all sites
-for i in vpns.values():
-	if i.defs.has_key('restrict-nets'):
-		vr=i.defs['restrict-nets']
+if service:
+	headerinput=pfilepath(header,allow_include=True)
+	userinput=sys.stdin.readlines()
+	pfile("user input",userinput)
+else:
+	if inputfile is None:
+		pfile("stdin",sys.stdin.readlines())
 	else:
-		vr=None
-	for l in i.locations.values():
-		if l.defs.has_key('restrict-nets'):
-			lr=l.defs['restrict-nets']
-			if (not lr.subsetof(vr)):
-				moan("location %s/%s restrict-nets is invalid"%
-					(i.name,l.name))
-		else:
-			lr=vr
-		for s in l.sites.values():
-			sn="%s/%s/%s"%(i.name,l.name,s.name)
-			for r in required.keys():
-				if (not (s.defs.has_key(r) or
-					l.defs.has_key(r) or
-					i.defs.has_key(r))):
-					moan("site %s missing parameter %s"%
-						(sn,r))
-			if s.defs.has_key('restrict-nets'):
-				sr=s.defs['restrict-nets']
-				if (not sr.subsetof(lr)):
-					moan("site %s restrict-nets not valid"%
-						sn)
-			else:
-				sr=lr
-			if not s.defs.has_key('networks'): continue
-			nets=s.defs['networks']
-			if (not nets.subsetof(sr)):
-				moan("site %s networks exceed restriction"%sn)
+		pfilepath(inputfile)
 
+delempty(root)
+checkconstraints(root,{},ipaddrset.complete_set())
 
 if complaints>0:
-	if complaints==1: print "There was 1 problem."
-	else: print "There were %d problems."%(complaints)
+	if complaints==1: print("There was 1 problem.")
+	else: print("There were %d problems."%(complaints))
 	sys.exit(1)
+complaints=None # arranges to crash if we complain later
 
 if service:
 	# Put the user's input into their group file, and rebuild the main
 	# sites file
-	f=open(groupfiledir+"/T"+group,'w')
+	f=open(groupfiledir+"/T"+group.groupname(),'w')
 	f.write("# Section submitted by user %s, %s\n"%
 		(user,time.asctime(time.localtime(time.time()))))
-	f.write("# Checked by make-secnet-sites.py version %s\n\n"%VERSION)
+	f.write("# Checked by make-secnet-sites version %s\n\n"%VERSION)
 	for i in userinput: f.write(i)
 	f.write("\n")
 	f.close()
-	os.rename(groupfiledir+"/T"+group,groupfiledir+"/R"+group)
+	os.rename(groupfiledir+"/T"+group.groupname(),
+		  groupfiledir+"/R"+group.groupname())
 	f=open(sitesfile+"-tmp",'w')
 	f.write("# sites file autogenerated by make-secnet-sites\n")
 	f.write("# generated %s, invoked by %s\n"%
@@ -493,4 +869,11 @@ if service:
 	f.close()
 	os.rename(sitesfile+"-tmp",sitesfile)
 else:
+	if outputfile is None:
+		of=sys.stdout
+	else:
+		tmp_outputfile=outputfile+'~tmp~'
+		of=open(tmp_outputfile,'w')
 	outputsites(of)
+	if outputfile is not None:
+		os.rename(tmp_outputfile,outputfile)