chiark / gitweb /
make-secnet-sites: OpBase.write_out: Tolerate sitesfile is None
[secnet.git] / make-secnet-sites
index ab125ffaed3f7de5e44c136266efdf8eef6753c1..f9c249bb7ac904dfc610e64bed99ef15fa9a5503 100755 (executable)
@@ -61,6 +61,7 @@ import os
 import getopt
 import re
 import argparse
+import math
 
 import ipaddress
 
@@ -69,9 +70,14 @@ import ipaddress
 sys.path.insert(1,"/usr/local/share/secnet")
 sys.path.insert(1,"/usr/share/secnet")
 import ipaddrset
+import base91
+
+from argparseactionnoyes import ActionNoYes
 
 VERSION="0.1.18"
 
+max_version = 2
+
 from sys import version_info
 if version_info.major == 2:  # for python2
     import codecs
@@ -80,34 +86,269 @@ if version_info.major == 2:  # for python2
     import io
     open=lambda f,m='r': io.open(f,m,encoding='utf-8')
 
-def parse_args():
-       global service
-       global inputfile
-       global header
-       global groupfiledir
-       global sitesfile
-       global group
-       global user
-       global of
+max={'rsa_bits':8200,'name':33,'dh_bits':8200,'algname':127}
 
-       ap = argparse.ArgumentParser(description='process secnet sites files')
-       ap.add_argument('--userv', '-u', action='store_true',
-                       help='userv service fragment update mode')
-       ap.add_argument('--prefix', '-P', nargs=1,
-                       help='set prefix')
-       ap.add_argument('arg',nargs=argparse.REMAINDER)
-       av = ap.parse_args()
-       #print(repr(av), file=sys.stderr)
-       service = 1 if av.userv else 0
-       if service:
+def debugrepr(*args):
+       if debug_level > 0:
+               print(repr(args), file=sys.stderr)
+
+def base91s_encode(bindata):
+       return base91.encode(bindata).replace('"',"-")
+
+def base91s_decode(string):
+       return base91.decode(string.replace("-",'"'))
+
+class Tainted:
+       def __init__(self,s,tline=None,tfile=None):
+               self._s=s
+               self._ok=None
+               self._line=line if tline is None else tline
+               self._file=file if tfile is None else tfile
+       def __eq__(self,e):
+               return self._s==e
+       def __ne__(self,e):
+               # for Python2
+               return not self.__eq__(e)
+       def __str__(self):
+               raise RuntimeError('direct use of Tainted value')
+       def __repr__(self):
+               return 'Tainted(%s)' % repr(self._s)
+
+       def _bad(self,what,why):
+               assert(self._ok is not True)
+               self._ok=False
+               complain('bad parameter: %s: %s' % (what, why))
+               return False
+
+       def _max_ok(self,what,maxlen):
+               if len(self._s) > maxlen:
+                       return self._bad(what,'too long (max %d)' % maxlen)
+               return True
+
+       def _re_ok(self,bad,what,maxlen=None):
+               if maxlen is None: maxlen=max[what]
+               self._max_ok(what,maxlen)
+               if self._ok is False: return False
+               if bad.search(self._s):
+                       #print(repr(self), file=sys.stderr)
+                       return self._bad(what,'bad syntax')
+               return True
+
+       def _rtnval(self, is_ok, ifgood, ifbad=''):
+               if is_ok:
+                       assert(self._ok is not False)
+                       self._ok=True
+                       return ifgood
+               else:
+                       assert(self._ok is not True)
+                       self._ok=False
+                       return ifbad
+
+       def _rtn(self, is_ok, ifbad=''):
+               return self._rtnval(is_ok, self._s, ifbad)
+
+       def raw(self):
+               return self._s
+       def raw_mark_ok(self):
+               # caller promises to throw if syntax was dangeorus
+               return self._rtn(True)
+
+       def output(self):
+               if self._ok is False: return ''
+               if self._ok is True: return self._s
+               print('%s:%d: unchecked/unknown additional data "%s"' %
+                     (self._file,self._line,self._s),
+                     file=sys.stderr)
+               sys.exit(1)
+
+       bad_name=re.compile(r'^[^a-zA-Z]|[^-_0-9a-zA-Z]')
+       # secnet accepts _ at start of names, but we reserve that
+       bad_name_counter=0
+       def name(self,what='name'):
+               ok=self._re_ok(Tainted.bad_name,what)
+               return self._rtn(ok,
+                                '_line%d_%s' % (self._line, id(self)))
+
+       def keyword(self):
+               ok=self._s in keywords or self._s in levels
+               if not ok:
+                       complain('unknown keyword %s' % self._s)
+               return self._rtn(ok)
+
+       bad_hex=re.compile(r'[^0-9a-fA-F]')
+       def bignum_16(self,kind,what):
+               maxlen=(max[kind+'_bits']+3)/4
+               ok=self._re_ok(Tainted.bad_hex,what,maxlen)
+               return self._rtn(ok)
+
+       bad_num=re.compile(r'[^0-9]')
+       def bignum_10(self,kind,what):
+               maxlen=math.ceil(max[kind+'_bits'] / math.log10(2))
+               ok=self._re_ok(Tainted.bad_num,what,maxlen)
+               return self._rtn(ok)
+
+       def number(self,minn,maxx,what='number'):
+               # not for bignums
+               ok=self._re_ok(Tainted.bad_num,what,10)
+               if ok:
+                       v=int(self._s)
+                       if v<minn or v>maxx:
+                               ok=self._bad(what,'out of range %d..%d'
+                                            % (minn,maxx))
+               return self._rtnval(ok,v,minn)
+
+       def hexid(self,byteslen,what):
+               ok=self._re_ok(Tainted.bad_hex,what,byteslen*2)
+               if ok:
+                       if len(self._s) < byteslen*2:
+                               ok=self._bad(what,'too short')
+               return self._rtn(ok,ifbad='00'*byteslen)
+
+       bad_host=re.compile(r'[^-\][_.:0-9a-zA-Z]')
+       # We permit _ so we can refer to special non-host domains
+       # which have A and AAAA RRs.  This is a crude check and we may
+       # still produce config files with syntactically invalid
+       # domains or addresses, but that is OK.
+       def host(self):
+               ok=self._re_ok(Tainted.bad_host,'host/address',255)
+               return self._rtn(ok)
+
+       bad_email=re.compile(r'[^-._0-9a-z@!$%^&*=+~/]')
+       # ^ This does not accept all valid email addresses.  That's
+       # not really possible with this input syntax.  It accepts
+       # all ones that don't require quoting anywhere in email
+       # protocols (and also accepts some invalid ones).
+       def email(self):
+               ok=self._re_ok(Tainted.bad_email,'email address',1023)
+               return self._rtn(ok)
+
+       bad_groupname=re.compile(r'^[^_A-Za-z]|[^-+_0-9A-Za-z]')
+       def groupname(self):
+               ok=self._re_ok(Tainted.bad_groupname,'group name',64)
+               return self._rtn(ok)
+
+       bad_base91=re.compile(r'[^!-~]|[\'\"\\]')
+       def base91(self,what='base91'):
+               ok=self._re_ok(Tainted.bad_base91,what,4096)
+               return self._rtn(ok)
+
+class ArgActionLambda(argparse.Action):
+       def __init__(self, fn, **kwargs):
+               self.fn=fn
+               argparse.Action.__init__(self,**kwargs)
+       def __call__(self,ap,ns,values,option_string):
+               self.fn(values,ns,ap,option_string)
+
+class PkmBase():
+       def site_start(self,pubkeys_path):
+               self._pa=pubkeys_path
+               self._fs = FilterState()
+       def site_serial(self,serial): pass
+       def write_key(self,k): pass
+       def site_finish(self,confw): pass
+
+class PkmSingle(PkmBase):
+       opt = 'single'
+       help = 'write one public key per site to sites.conf'
+       def site_start(self,pubkeys_path):
+               PkmBase.site_start(self,pubkeys_path)
+               self._outk = []
+       def write_key(self,k):
+               if k.okforonlykey(output_version,self._fs):
+                       self._outk.append(k)
+       def site_finish(self,confw):
+               if len(self._outk) == 0:
+                       complain("site with no public key");
+               elif len(self._outk) != 1:
+                       debugrepr('outk ', self._outk)
+                       complain(
+ "site with multiple public keys, without --pubkeys-install (maybe --output-version=1 would help"
+                       )
+               else:
+                       confw.write("key %s;\n"%str(self._outk[0]))
+
+class PkmInstall(PkmBase):
+       opt = 'install'
+       help = 'install public keys in public key directory'
+       def site_start(self,pubkeys_path):
+               PkmBase.site_start(self,pubkeys_path)
+               self._pw=open(self._pa+'~tmp','w')
+       def site_serial(self,serial):
+               self._pw.write('serial %s\n' % serial)
+       def write_key(self,k):
+               wout=k.forpub(output_version,self._fs)
+               self._pw.write(' '.join(wout))
+               self._pw.write('\n')
+       def site_finish(self,confw):
+               self._pw.close()
+               os.rename(self._pa+'~tmp',self._pa+'~update')
+               PkmElide.site_finish(self,confw)
+
+class PkmElide(PkmBase):
+       opt = 'elide'
+       help = 'no public keys in sites.conf output nor in directory'
+       def site_finish(self,confw):
+               confw.write("peer-keys \"%s\";\n"%self._pa);
+
+class OpBase():
+       # Base case is reading a sites file from self.inputfilee.
+       # And writing a sites file to self.sitesfile.
+       def read_in(self):
+               if self.inputfile is None:
+                       self.inputlines = pfile("stdin",sys.stdin.readlines())
+               else:
+                       self.inputlines = pfilepath(self.inputfile)
+       def write_out(self):
+               if self.sitesfile is None:
+                       f=sys.stdout
+               else:
+                       f=open(self.sitesfile+"-tmp",'w')
+               f.write("# sites file autogenerated by make-secnet-sites\n")
+               f.write("# generated %s, invoked by %s\n"%
+                       (time.asctime(time.localtime(time.time())),
+                        self.user))
+               f.write("# use make-secnet-sites to turn this file into a\n")
+               f.write("# valid /etc/secnet/sites.conf file\n\n")
+               self.write_out_contents(f)
+               f.write("# end of sites file\n")
+               if self.sitesfile is not None:
+                       f.close()
+                       os.rename(self.sitesfile+"-tmp",self.sitesfile)
+
+class OpConf(OpBase):
+       opts = ['--conf']
+       help = 'sites.conf generation mode (default)'
+       def positional_args(self, av):
+               if len(av.arg)>3:
+                       print("Too many arguments")
+                       sys.exit(1)
+               (self.inputfile, self.outputfile) = (av.arg + [None]*2)[0:2]
+       def check_group(self,group,w): pass
+       def write_out(self):
+               if self.outputfile is None:
+                       of=sys.stdout
+               else:
+                       tmp_outputfile=self.outputfile+'~tmp~'
+                       of=open(tmp_outputfile,'w')
+               outputsites(of)
+               if self.outputfile is not None:
+                       os.rename(tmp_outputfile,self.outputfile)
+
+class OpUserv(OpBase):
+       opts = ['--userv','-u']
+       help = 'userv service fragment update mode'
+       def positional_args(self, av):
                if len(av.arg)!=4:
                        print("Wrong number of arguments")
                        sys.exit(1)
-               (header, groupfiledir, sitesfile, group) = av.arg
+               (self.header, self.groupfiledir,
+                self.sitesfile, self.group) = av.arg
+               self.group = Tainted(self.group,0,'command line')
+               # untrusted argument from caller
                if "USERV_USER" not in os.environ:
                        print("Environment variable USERV_USER not found")
                        sys.exit(1)
-               user=os.environ["USERV_USER"]
+               self.user=os.environ["USERV_USER"]
                # Check that group is in USERV_GROUP
                if "USERV_GROUP" not in os.environ:
                        print("Environment variable USERV_GROUP not found")
@@ -115,17 +356,88 @@ def parse_args():
                ugs=os.environ["USERV_GROUP"]
                ok=0
                for i in ugs.split():
-                       if group==i: ok=1
+                       if self.group==i: ok=1
                if not ok:
                        print("caller not in group %s"%group)
                        sys.exit(1)
-       else:
-               if len(av.arg)>3:
-                       print("Too many arguments")
-                       sys.exit(1)
-               (inputfile, outputfile) = (av.arg + [None]*2)[0:2]
-               if outputfile is None: of=sys.stdout
-               else: of=open(sys.argv[2],'w')
+       def check_group(self,group,w):
+               if group!=self.group: complain("Incorrect group!")
+               w[2].groupname()
+       def read_in(self):
+               self.headerinput=pfilepath(self.header,allow_include=True)
+               self.userinput=sys.stdin.readlines()
+               pfile("user input",self.userinput)
+       def write_out(self):
+               # Put the user's input into their group file, and
+               # rebuild the main sites file
+               f=open(self.groupfiledir+"/T"+self.group.groupname(),'w')
+               f.write("# Section submitted by user %s, %s\n"%
+                       (self.user,time.asctime(time.localtime(time.time()))))
+               f.write("# Checked by make-secnet-sites version %s\n\n"
+                       %VERSION)
+               for i in self.userinput: f.write(i)
+               f.write("\n")
+               f.close()
+               os.rename(self.groupfiledir+"/T"+self.group.groupname(),
+                         self.groupfiledir+"/R"+self.group.groupname())
+               OpBase.write_out(self)
+       def write_out_contents(self,f):
+               for i in self.headerinput: f.write(i)
+               files=os.listdir(self.groupfiledir)
+               for i in files:
+                       if i[0]=='R':
+                               j=open(self.groupfiledir+"/"+i)
+                               f.write(j.read())
+                               j.close()
+
+def parse_args():
+       global opmode
+       global prefix
+       global key_prefix
+       global debug_level
+       global output_version
+       global pubkeys_dir
+       global pubkeys_mode
+
+       ap = argparse.ArgumentParser(description='process secnet sites files')
+       def add_opmode(how):
+               ap.add_argument(*how().opts, action=ArgActionLambda,
+                       nargs=0,
+                       fn=(lambda v,ns,*x: setattr(ns,'opmode',how)),
+                       help=how().help)
+       add_opmode(OpConf)
+       add_opmode(OpUserv)
+       ap.add_argument('--conf-key-prefix', action=ActionNoYes,
+                       default=True,
+                help='prefix conf file key names derived from sites data')
+       def add_pkm(how):
+               ap.add_argument('--pubkeys-'+how().opt, action=ArgActionLambda,
+                       nargs=0,
+                       fn=(lambda v,ns,*x: setattr(ns,'pkm',how)),
+                       help=how().help)
+       add_pkm(PkmInstall)
+       add_pkm(PkmSingle)
+       add_pkm(PkmElide)
+       ap.add_argument('--pubkeys-dir',  nargs=1,
+                       help='public key directory',
+                       default=['/var/lib/secnet/pubkeys'])
+       ap.add_argument('--output-version', nargs=1, type=int,
+                       help='sites file output version',
+                       default=[max_version])
+       ap.add_argument('--prefix', '-P', nargs=1,
+                       help='set prefix')
+       ap.add_argument('--debug', '-D', action='count', default=0)
+       ap.add_argument('arg',nargs=argparse.REMAINDER)
+       av = ap.parse_args()
+       debug_level = av.debug
+       debugrepr('av',av)
+       opmode = getattr(av,'opmode',OpConf)()
+       prefix = '' if av.prefix is None else av.prefix[0]
+       key_prefix = av.conf_key_prefix
+       output_version = av.output_version[0]
+       pubkeys_dir = av.pubkeys_dir[0]
+       pubkeys_mode = getattr(av,'pkm',PkmSingle)
+       opmode.positional_args(av)
 
 parse_args()
 
@@ -135,7 +447,9 @@ class basetype:
        "Common protocol for configuration types."
        def add(self,obj,w):
                complain("%s %s already has property %s defined"%
-                       (obj.type,obj.name,w[0]))
+                       (obj.type,obj.name,w[0].raw()))
+       def forsites(self,version,copy,fs):
+               return copy
 
 class conflist:
        "A list of some kind of configuration type."
@@ -146,13 +460,16 @@ class conflist:
                self.list.append(self.subtype(w))
        def __str__(self):
                return ', '.join(map(str, self.list))
+       def forsites(self,version,copy,fs):
+               most_recent=self.list[len(self.list)-1]
+               return most_recent.forsites(version,copy,fs)
 def listof(subtype):
        return lambda w: conflist(subtype, w)
 
 class single_ipaddr (basetype):
        "An IP address"
        def __init__(self,w):
-               self.addr=ipaddress.ip_address(w[1])
+               self.addr=ipaddress.ip_address(w[1].raw_mark_ok())
        def __str__(self):
                return '"%s"'%self.addr
 
@@ -161,7 +478,7 @@ class networks (basetype):
        def __init__(self,w):
                self.set=ipaddrset.IPAddressSet()
                for i in w[1:]:
-                       x=ipaddress.ip_network(i,strict=True)
+                       x=ipaddress.ip_network(i.raw_mark_ok(),strict=True)
                        self.set.append([x])
        def __str__(self):
                return ",".join(map((lambda n: '"%s"'%n), self.set.networks()))
@@ -169,8 +486,8 @@ class networks (basetype):
 class dhgroup (basetype):
        "A Diffie-Hellman group"
        def __init__(self,w):
-               self.mod=w[1]
-               self.gen=w[2]
+               self.mod=w[1].bignum_16('dh','dh mod')
+               self.gen=w[2].bignum_16('dh','dh gen')
        def __str__(self):
                return 'diffie-hellman("%s","%s")'%(self.mod,self.gen)
 
@@ -178,26 +495,32 @@ class hash (basetype):
        "A choice of hash function"
        def __init__(self,w):
                hname=w[1]
-               self.ht=hname
+               self.ht=hname.raw()
                if (self.ht!='md5' and self.ht!='sha1'):
                        complain("unknown hash type %s"%(self.ht))
+                       self.ht=None
+               else:
+                       hname.raw_mark_ok()
        def __str__(self):
                return '%s'%(self.ht)
 
 class email (basetype):
        "An email address"
        def __init__(self,w):
-               self.addr=w[1]
+               self.addr=w[1].email()
        def __str__(self):
                return '<%s>'%(self.addr)
 
 class boolean (basetype):
        "A boolean"
        def __init__(self,w):
-               if re.match('[TtYy1]',w[1]):
+               v=w[1]
+               if re.match('[TtYy1]',v.raw()):
                        self.b=True
-               elif re.match('[FfNn0]',w[1]):
+                       v.raw_mark_ok()
+               elif re.match('[FfNn0]',v.raw()):
                        self.b=False
+                       v.raw_mark_ok()
                else:
                        complain("invalid boolean value");
        def __str__(self):
@@ -206,28 +529,114 @@ class boolean (basetype):
 class num (basetype):
        "A decimal number"
        def __init__(self,w):
-               self.n=int(w[1])
+               self.n=w[1].number(0,0x7fffffff)
        def __str__(self):
                return '%d'%(self.n)
 
+class serial (basetype):
+       def __init__(self,w):
+               self.i=w[1].hexid(4,'serial')
+       def __str__(self):
+               return self.i
+       def forsites(self,version,copy,fs):
+               if version < 2: return []
+               return copy
+
 class address (basetype):
        "A DNS name and UDP port number"
        def __init__(self,w):
-               self.adr=w[1]
-               self.port=int(w[2])
-               if (self.port<1 or self.port>65535):
-                       complain("invalid port number")
+               self.adr=w[1].host()
+               self.port=w[2].number(1,65536,'port')
        def __str__(self):
                return '"%s"; port %d'%(self.adr,self.port)
 
-class rsakey (basetype):
-       "An RSA public key"
+class inpub (basetype):
+       def forsites(self,version,xcopy,fs):
+               return self.forpub(version,fs)
+
+class pubkey (inpub):
+       "Some kind of publie key"
        def __init__(self,w):
-               self.l=int(w[1])
-               self.e=w[2]
-               self.n=w[3]
+               self.a=w[1].name('algname')
+               self.d=w[2].base91();
+       def __str__(self):
+               return 'make-public("%s","%s")'%(self.a,self.d)
+       def forpub(self,version,fs):
+               if version < 2: return []
+               return ['pub', self.a, self.d]
+       def okforonlykey(self,version,fs):
+               return len(self.forpub(version,fs)) != 0
+
+class rsakey (pubkey):
+       "An old-style RSA public key"
+       def __init__(self,w):
+               self.l=w[1].number(0,max['rsa_bits'],'rsa len')
+               self.e=w[2].bignum_10('rsa','rsa e')
+               self.n=w[3].bignum_10('rsa','rsa n')
+               if len(w) >= 5: w[4].email()
+               self.a='rsa1'
+               self.d=base91s_encode(b'%d %s %s' %
+                                     (self.l,
+                                      self.e.encode('ascii'),
+                                      self.n.encode('ascii')))
+               # ^ this allows us to use the pubkey.forsites()
+               # method for output in versions>=2
        def __str__(self):
                return 'rsa-public("%s","%s")'%(self.e,self.n)
+               # this specialisation means we can generate files
+               # compatible with old secnet executables
+       def forpub(self,version,fs):
+               if version < 2:
+                       if fs.pkg != '00000000': return []
+                       return ['pubkey', str(self.l), self.e, self.n]
+               return pubkey.forpub(self,version,fs)
+
+class rsakey_newfmt(rsakey):
+       "An old-style RSA public key in new-style sites format"
+       # This is its own class simply to have its own constructor.
+       def __init__(self,w):
+               self.a=w[1].name()
+               assert(self.a == 'rsa1')
+               self.d=w[2].base91()
+               try:
+                       w_inner=list(map(Tainted,
+                                       ['X-PUB-RSA1'] +
+                                       base91s_decode(self.d)
+                                       .decode('ascii')
+                                       .split(' ')))
+               except UnicodeDecodeError:
+                       complain('rsa1 key in new format has bad base91')
+               #print(repr(w_inner), file=sys.stderr)
+               rsakey.__init__(self,w_inner)
+
+class pubkey_group(inpub):
+       "Public key group introducer"
+       # appears in the site's list of keys mixed in with the keys
+       def __init__(self,w,fallback):
+               self.i=w[1].hexid(4,'pkg-id')
+               self.fallback=fallback
+       def forpub(self,version,fs):
+               fs.pkg=self.i
+               if version < 2: return []
+               return ['pkgf' if self.fallback else 'pkg', self.i]
+       def okforonlykey(self,version,fs):
+               self.forpub(version,fs)
+               return False
+       
+def somepubkey(w):
+       #print(repr(w), file=sys.stderr)
+       if w[0]=='pubkey':
+               return rsakey(w)
+       elif w[0]=='pub' and w[1]=='rsa1':
+               return rsakey_newfmt(w)
+       elif w[0]=='pub':
+               return pubkey(w)
+       elif w[0]=='pkg':
+               return pubkey_group(w,False)
+       elif w[0]=='pkgf':
+               return pubkey_group(w,True)
+       else:
+               assert(False)
 
 # Possible properties of configuration nodes
 keywords={
@@ -241,7 +650,11 @@ keywords={
  'renegotiate-time':(num,"Time after key setup to begin renegotiation (ms)"),
  'restrict-nets':(networks,"Allowable networks"),
  'networks':(networks,"Claimed networks"),
- 'pubkey':(rsakey,"RSA public site key"),
+ 'serial':(serial,"public key set serial"),
+ 'pkg':(listof(somepubkey),"start of public key group",'pub'),
+ 'pkgf':(listof(somepubkey),"start of fallback public key group",'pub'),
+ 'pub':(listof(somepubkey),"new style public site key"),
+ 'pubkey':(listof(somepubkey),"Old-style RSA public site key",'pub'),
  'peer':(single_ipaddr,"Tunnel peer IP address"),
  'address':(address,"External contact address and port"),
  'mobile':(boolean,"Site is mobile"),
@@ -271,8 +684,8 @@ class level:
        allow_properties={}
        require_properties={}
        def __init__(self,w):
-               self.type=w[0]
-               self.name=w[1]
+               self.type=w[0].keyword()
+               self.name=w[1].name()
                self.properties={}
                self.children={}
        def indent(self,w,t):
@@ -280,17 +693,22 @@ class level:
        def prop_out(self,n):
                return self.allow_properties[n](n,str(self.properties[n]))
        def output_props(self,w,ind):
-               for i in self.properties.keys():
+               for i in sorted(self.properties.keys()):
                        if self.allow_properties[i]:
                                self.indent(w,ind)
                                w.write("%s"%self.prop_out(i))
-       def output_data(self,w,ind,np):
+       def kname(self):
+               return ((self.type[0].upper() if key_prefix else '')
+                       + self.name)
+       def output_data(self,w,path):
+               ind = 2*len(path)
                self.indent(w,ind)
-               w.write("%s {\n"%(self.name))
+               w.write("%s {\n"%(self.kname()))
                self.output_props(w,ind+2)
                if self.depth==1: w.write("\n");
-               for c in self.children.values():
-                       c.output_data(w,ind+2,np+self.name+"/")
+               for k in sorted(self.children.keys()):
+                       c=self.children[k]
+                       c.output_data(w,path+(c,))
                self.indent(w,ind)
                w.write("};\n")
 
@@ -305,17 +723,18 @@ class vpnlevel(level):
        }
        def __init__(self,w):
                level.__init__(self,w)
-       def output_vpnflat(self,w,ind,h):
+       def output_vpnflat(self,w,path):
                "Output flattened list of site names for this VPN"
+               ind=2*(len(path)+1)
                self.indent(w,ind)
-               w.write("%s {\n"%(self.name))
+               w.write("%s {\n"%(self.kname()))
                for i in self.children.keys():
-                       self.children[i].output_vpnflat(w,ind+2,
-                               h+"/"+self.name+"/"+i)
+                       self.children[i].output_vpnflat(w,path+(self,))
                w.write("\n")
                self.indent(w,ind+2)
                w.write("all-sites %s;\n"%
-                       ','.join(self.children.keys()))
+                       ','.join(map(lambda i: i.kname(),
+                                    self.children.values())))
                self.indent(w,ind)
                w.write("};\n")
 
@@ -330,14 +749,20 @@ class locationlevel(level):
        }
        def __init__(self,w):
                level.__init__(self,w)
-               self.group=w[2]
-       def output_vpnflat(self,w,ind,h):
+               self.group=w[2].groupname()
+       def output_vpnflat(self,w,path):
+               ind=2*(len(path)+1)
                self.indent(w,ind)
-               # The "h=h,self=self" abomination below exists because
+               # The "path=path,self=self" abomination below exists because
                # Python didn't support nested_scopes until version 2.1
-               w.write("%s %s;\n"%(self.name,','.join(
-                       map(lambda x,h=h,self=self:
-                               h+"/"+x,self.children.keys()))))
+               #
+               #"/"+self.name+"/"+i
+               w.write("%s %s;\n"%(self.kname(),','.join(
+                       map(lambda x,path=path,self=self:
+                           '/'.join([prefix+"vpn-data"] + list(map(
+                                   lambda i: i.kname(),
+                                   path+(self,x)))),
+                           self.children.values()))))
 
 class sitelevel(level):
        "Site level (i.e. a leafnode) in the configuration hierarchy"
@@ -349,7 +774,11 @@ class sitelevel(level):
         'address':sp,
         'networks':None,
         'peer':None,
-        'pubkey':(lambda n,v:"key %s;\n"%v),
+         'serial':None,
+        'pkg':None,
+        'pkgf':None,
+        'pub':None,
+        'pubkey':None,
         'mobile':sp,
        })
        require_properties={
@@ -358,15 +787,34 @@ class sitelevel(level):
         'networks':"Networks claimed by the site",
         'hash':"hash function",
         'peer':"Gateway address of the site",
-        'pubkey':"RSA public key of the site",
        }
+       def mangle_name(self):
+               return self.name.replace('/',',')
+       def pubkeys_path(self):
+               return pubkeys_dir + '/peer.' + self.mangle_name()
        def __init__(self,w):
                level.__init__(self,w)
-       def output_data(self,w,ind,np):
+       def output_data(self,w,path):
+               ind=2*len(path)
+               np='/'.join(map(lambda i: i.name, path))
                self.indent(w,ind)
-               w.write("%s {\n"%(self.name))
+               w.write("%s {\n"%(self.kname()))
+               self.indent(w,ind+2)
+               w.write("name \"%s\";\n"%(np,))
                self.indent(w,ind+2)
-               w.write("name \"%s\";\n"%(np+self.name))
+
+               pkm = pubkeys_mode()
+               debugrepr('pkm ',pkm)
+               pkm.site_start(self.pubkeys_path())
+               if 'serial' in self.properties:
+                       pkm.site_serial(self.properties['serial'])
+
+               for k in self.properties["pub"].list:
+                       debugrepr('pubkeys ', k)
+                       pkm.write_key(k)
+
+               pkm.site_finish(w)
+
                self.output_props(w,ind+2)
                self.indent(w,ind+2)
                w.write("link netlink {\n");
@@ -383,46 +831,62 @@ class sitelevel(level):
 # (depth,properties)
 levels={'vpn':vpnlevel, 'location':locationlevel, 'site':sitelevel}
 
-# Reserved vpn/location/site names
-reserved={'all-sites':None}
-reserved.update(keywords)
-reserved.update(levels)
-
 def complain(msg):
        "Complain about a particular input line"
-       global complaints
-       print(("%s line %d: "%(file,line))+msg)
-       complaints=complaints+1
+       moan(("%s line %d: "%(file,line))+msg)
 def moan(msg):
        "Complain about something in general"
        global complaints
        print(msg);
+       if complaints is None: sys.exit(1)
        complaints=complaints+1
 
-root=level(['root','root'])   # All vpns are children of this node
+class UntaintedRoot():
+       def __init__(self,s): self._s=s
+       def name(self): return self._s
+       def keyword(self): return self._s
+
+root=level([UntaintedRoot(x) for x in ['root','root']])
+# All vpns are children of this node
 obstack=[root]
 allow_defs=0   # Level above which new definitions are permitted
-prefix=''
 
 def set_property(obj,w):
        "Set a property on a configuration node"
        prop=w[0]
-       if prop in obj.properties:
-               obj.properties[prop].add(obj,w)
+       propname=prop.raw_mark_ok()
+       kw=keywords[propname]
+       if len(kw) >= 3: propname=kw[2] # for aliases
+       if propname in obj.properties:
+               obj.properties[propname].add(obj,w)
        else:
-               obj.properties[prop]=keywords[prop][0](w)
+               obj.properties[propname]=kw[0](w)
+       return obj.properties[propname]
+
+class FilterState:
+       def __init__(self):
+               self.reset()
+       def reset(self):
+               # called when we enter a new node,
+               # in particular, at the start of each site
+               self.pkg = '00000000'
 
-def pline(i,allow_include=False):
+def pline(il,filterstate,allow_include=False):
        "Process a configuration file line"
        global allow_defs, obstack, root
-       w=i.rstrip('\n').split()
-       if len(w)==0: return [i]
+       w=il.rstrip('\n').split()
+       if len(w)==0: return ['']
+       w=list([Tainted(x) for x in w])
        keyword=w[0]
        current=obstack[len(obstack)-1]
+       copyout_core=lambda: ' '.join([ww.output() for ww in w])
+       indent='    '*len(obstack)
+       copyout=lambda: [indent + copyout_core() + '\n']
        if keyword=='end-definitions':
+               keyword.raw_mark_ok()
                allow_defs=sitelevel.depth
                obstack=[root]
-               return [i]
+               return copyout()
        if keyword=='include':
                if not allow_include:
                        complain("include not permitted here")
@@ -430,11 +894,12 @@ def pline(i,allow_include=False):
                if len(w) != 2:
                        complain("include requires one argument")
                        return []
-               newfile=os.path.join(os.path.dirname(file),w[1])
+               newfile=os.path.join(os.path.dirname(file),w[1].raw_mark_ok())
+               # ^ user of "include" is trusted so raw_mark_ok is good
                return pfilepath(newfile,allow_include=allow_include)
-       if keyword in levels:
+       if keyword.raw() in levels:
                # We may go up any number of levels, but only down by one
-               newdepth=levels[keyword].depth
+               newdepth=levels[keyword.raw_mark_ok()].depth
                currentdepth=len(obstack) # actually +1...
                if newdepth<=currentdepth:
                        obstack=obstack[:newdepth]
@@ -444,17 +909,16 @@ def pline(i,allow_include=False):
                # See if it's a new one (and whether that's permitted)
                # or an existing one
                current=obstack[len(obstack)-1]
-               tname=w[1]
+               tname=w[1].name()
                if tname in current.children:
                        # Not new
                        current=current.children[tname]
-                       if service and group and current.depth==2:
-                               if group!=current.group:
-                                       complain("Incorrect group!")
+                       if current.depth==2:
+                               opmode.check_group(current.group, w)
                else:
                        # New
                        # Ignore depth check for now
-                       nl=levels[keyword](w)
+                       nl=levels[keyword.raw()](w)
                        if nl.depth<allow_defs:
                                complain("New definitions not allowed at "
                                        "level %d"%nl.depth)
@@ -462,20 +926,24 @@ def pline(i,allow_include=False):
                                sys.exit(1)
                        current.children[tname]=nl
                        current=nl
+               filterstate.reset()
                obstack.append(current)
-               return [i]
-       if keyword not in current.allow_properties:
+               return copyout()
+       if keyword.raw() not in current.allow_properties:
                complain("Property %s not allowed at %s level"%
-                       (keyword,current.type))
+                       (keyword.raw(),current.type))
                return []
        elif current.depth == vpnlevel.depth < allow_defs:
                complain("Not allowed to set VPN properties here")
                return []
        else:
-               set_property(current,w)
-               return [i]
+               prop=set_property(current,w)
+               out=[copyout_core()]
+               out=prop.forsites(output_version,out,filterstate)
+               if len(out)==0: return [indent + '#', copyout_core(), '\n']
+               return [indent + ' '.join(out) + '\n']
 
-       complain("unknown keyword '%s'"%(keyword))
+       complain("unknown keyword '%s'"%(keyword.raw()))
 
 def pfilepath(pathname,allow_include=False):
        f=open(pathname)
@@ -489,10 +957,11 @@ def pfile(name,lines,allow_include=False):
        file=name
        line=0
        outlines=[]
+       filterstate = FilterState()
        for i in lines:
                line=line+1
                if (i[0]=='#'): continue
-               outlines += pline(i,allow_include=allow_include)
+               outlines += pline(i,filterstate,allow_include=allow_include)
        return outlines
 
 def outputsites(w):
@@ -505,19 +974,19 @@ def outputsites(w):
        # Raw VPN data section of file
        w.write(prefix+"vpn-data {\n")
        for i in root.children.values():
-               i.output_data(w,2,"")
+               i.output_data(w,(i,))
        w.write("};\n")
 
        # Per-VPN flattened lists
        w.write(prefix+"vpn {\n")
        for i in root.children.values():
-               i.output_vpnflat(w,2,prefix+"vpn-data")
+               i.output_vpnflat(w,())
        w.write("};\n")
 
        # Flattened list of sites
        w.write(prefix+"all-sites %s;\n"%",".join(
-               map(lambda x:"%svpn/%s/all-sites"%(prefix,x),
-                       root.children.keys())))
+               map(lambda x:"%svpn/%s/all-sites"%(prefix,x.kname()),
+                       root.children.values())))
 
 line=0
 file=None
@@ -568,15 +1037,7 @@ def checkconstraints(n,p,ra):
        for i in n.children.keys():
                checkconstraints(n.children[i],new_p,new_ra)
 
-if service:
-       headerinput=pfilepath(header,allow_include=True)
-       userinput=sys.stdin.readlines()
-       pfile("user input",userinput)
-else:
-       if inputfile is None:
-               pfile("stdin",sys.stdin.readlines())
-       else:
-               pfilepath(inputfile)
+opmode.read_in()
 
 delempty(root)
 checkconstraints(root,{},ipaddrset.complete_set())
@@ -587,32 +1048,4 @@ if complaints>0:
        sys.exit(1)
 complaints=None # arranges to crash if we complain later
 
-if service:
-       # Put the user's input into their group file, and rebuild the main
-       # sites file
-       f=open(groupfiledir+"/T"+group,'w')
-       f.write("# Section submitted by user %s, %s\n"%
-               (user,time.asctime(time.localtime(time.time()))))
-       f.write("# Checked by make-secnet-sites version %s\n\n"%VERSION)
-       for i in userinput: f.write(i)
-       f.write("\n")
-       f.close()
-       os.rename(groupfiledir+"/T"+group,groupfiledir+"/R"+group)
-       f=open(sitesfile+"-tmp",'w')
-       f.write("# sites file autogenerated by make-secnet-sites\n")
-       f.write("# generated %s, invoked by %s\n"%
-               (time.asctime(time.localtime(time.time())),user))
-       f.write("# use make-secnet-sites to turn this file into a\n")
-       f.write("# valid /etc/secnet/sites.conf file\n\n")
-       for i in headerinput: f.write(i)
-       files=os.listdir(groupfiledir)
-       for i in files:
-               if i[0]=='R':
-                       j=open(groupfiledir+"/"+i)
-                       f.write(j.read())
-                       j.close()
-       f.write("# end of sites file\n")
-       f.close()
-       os.rename(sitesfile+"-tmp",sitesfile)
-else:
-       outputsites(of)
+opmode.write_out()