From: Ian Jackson Date: Sun, 2 Apr 2017 20:44:14 +0000 (+0100) Subject: reorg config - will break X-Git-Tag: hippotat/1.0.0~55^2~125 X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ian/git?p=hippotat.git;a=commitdiff_plain;h=c7fb640effd718bb66548327c4feddcc64416234 reorg config - will break --- diff --git a/README.config b/README.config index fc2bbf8..881694f 100644 --- a/README.config +++ b/README.config @@ -2,14 +2,14 @@ Sections - [ - ] + [ - ] [] [] usually [SERVER] [DEFAULT] -Keys are looked up in that order. +Keys are looked up in that order, unless otherwise specified. is the client's virtual address. - must not look like an address. + must be a valid DNS hostname and not look like an address. Exceptional settings: @@ -29,6 +29,7 @@ Exceptional settings: We define the sets of putative clients and servers, as follows: all those, for which there is any section (even an empty one) whose name is based on or (as applicable). + (LIMIT sections do not count.) The server queue packets for, and accept requests from, each putative client for which the config search yields a password. @@ -37,47 +38,56 @@ Exceptional settings: with the server, for each possible pair (putative server, putative client) for which the config search yields a password. + ipif + Command to run to create and communicate with local network + interface. Passed to sh -c. Must speak SLIP on stdin/stdout. + The following additional interpolations aare substituted: + %(local)s %(peer)s %(rnet)s + on server + on client + ["userv root ipif %(local)s,%(peer)s,%(mtu)s,slip %(rnets)s"] + + On server: applies to all clients; not looked up in + client-specific sections. + On client: may be different for different servers. + Capped settings: - Values in are a cap (maximum) on those from the - other sections (including DEFAULT): + Values in [ LIMIT] and [LIMIT] are a cap (maximum) on + those from the other sections (including DEFAULT). max_batch_down - Size limit for response payloads (server only) [65536 bytes] + Size limit for response payloads (used by server only) + [65536 bytes; LIMIT: 262144 bytes] max_queue_time - Discard downwards packets after this long (server only) [10 s] + Discard downwards packets after this long (used by server only) + [10 s; LIMIT: 121 s] http_timeout - (On server) return with empty payload any http request oustanding + On server: return with empty payload any http request oustanding for this long - (On client) give up on any http request outstanding for + On client: give up on any http request outstanding for for this long plus http_timeout_grace Client's effective timeout must be at least server's (checked). - [30 s] + [30 s; LIMIT: 121] target_requests_outstanding - (On server) whenever number of outstanding requests for + On server: whenever number of outstanding requests for a client exceeds this, return oldest with empty payload - (On client) try to keep this many requests outstanding. - Must match between client and server (checked). [3] + On client: try to keep this many requests outstanding. + Must match between client and server (checked). + [3; LIMIT: 10] -Ordinary settings, used by client and server: +Ordinary settings, used by both, not client-specific: - ipif - Command to run to create and communicate with local network - interface. Passed to sh -c. Must speak SLIP on stdin/stdout. - The following additional interpolations aare substituted: - %(local)s %(peer)s %(rnet)s - on server - on client - ["userv root ipif %(local)s,%(peer)s,%(mtu)s,slip %(rnets)s"] + These are not looked up in the client-specific config sections. addrs Public IP (v4 or v6) address(es) of the server; space-separated. - (On server) mandatory; used for bind. No default. - (On client) used only to construct default url. + On server: mandatory; used for bind. No default. + On client: used only to construct default url. vnetwork Private network range (/). Must contain all @@ -95,8 +105,8 @@ Ordinary settings, used by client and server: port Public port number of the server. [80] - (On server) used for bind. - (On client) used only to construct default url. + On server: used for bind. + On client: used only to construct default url. mtu Must match exactly. (checked) [1500 bytes] diff --git a/client b/client index e5d4e00..6080d4d 100755 --- a/client +++ b/client @@ -7,77 +7,23 @@ import twisted.web.client import io -client_cs = None - -def set_client(ci,cs,pw): - global client_cs - global password - assert(client_cs is None) - client_cs = cs - c.client = ci - c.max_outstanding = cfg.getint(cs, 'max_requests_outstanding') - c.target_outstanding = cfg.getint(cs, 'target_requests_outstanding') - password = pw - -def process_cfg(): - global url - global max_requests_outstanding - - process_cfg_common_always() - process_cfg_server() - - try: - c.url = cfg.get('server','url') - except NoOptionError: - process_cfg_saddrs() - c.url = c.saddrs[0].url() - - process_cfg_clients(set_client) - - c.routes = cfg.get('virtual','routes') - c.max_queue_time = cfg.getint(client_cs, 'max_queue_time') - c.max_batch_up = cfg.getint(client_cs, 'max_batch_up') - c.http_retry = cfg.getint(client_cs, 'http_retry') - c.http_timeout = (cfg.getint(client_cs, 'http_timeout') + - cfg.getint(client_cs, 'http_timeout_grace')) - - process_cfg_ipif(client_cs, - (('local', 'client'), - ('peer', 'server'), - ('rnets', 'routes'))) - -outstanding = { } - -def log_outstanding(): - log_debug(DBG.CTRL_DUMP, 'OS %s' % outstanding) - -def start_client(): - global queue - global agent - queue = PacketQueue('up', c.max_queue_time) - agent = twisted.web.client.Agent(reactor, connectTimeout = c.http_timeout) - -def outbound(packet, saddr, daddr): - #print('OUT ', saddr, daddr, repr(packet)) - queue.append(packet) - check_outbound() - class GeneralResponseConsumer(twisted.internet.protocol.Protocol): - def __init__(self, req, desc): + def __init__(self, cl, req, desc): + self._cl = cl self._req = req self._desc = desc def _log(self, dflag, msg, **kwargs): - log_debug(dflag, '%s: %s' % (self._desc, msg), idof=self._req, **kwargs) + self.cl.log(dflag, '%s: %s' % (self._desc, msg), idof=self._req, **kwargs) def connectionMade(self): self._log(DBG.HTTP_CTRL, 'connectionMade') class ResponseConsumer(GeneralResponseConsumer): - def __init__(self, req): - super().__init__(req, 'RC') + def __init__(self, cl, req): + super().__init__(cl, req, 'RC') ssddesc = '[%s] %s' % (id(req), self._desc) - self._ssd = SlipStreamDecoder(ssddesc, queue_inbound) + self._ssd = SlipStreamDecoder(ssddesc, cl.queue_inbound) self._log(DBG.HTTP_CTRL, '__init__') def dataReceived(self, data): @@ -95,7 +41,7 @@ class ResponseConsumer(GeneralResponseConsumer): try: self._log(DBG.HTTP, 'ResponseDone') self._ssd.flush() - req_fin(self._req) + self.cl.req_fin(self._req) except Exception as e: self._handleexception() @@ -104,11 +50,11 @@ class ResponseConsumer(GeneralResponseConsumer): def _latefailure(self, reason): self._log(DBG.HTTP_CTRL, '_latefailure ' + str(reason)) - req_err(self._req, reason) + self.cl.req_err(self._req, reason) class ErrorResponseConsumer(twisted.internet.protocol.Protocol): - def __init__(self, req, resp): - super().__init__(req, 'ERROR-RC') + def __init__(self, cl, req, resp): + super().__init__(cl, req, 'ERROR-RC') self._resp = resp self._m = b'' try: @@ -128,110 +74,167 @@ class ErrorResponseConsumer(twisted.internet.protocol.Protocol): mbody = repr(self._m) if not reason.check(twisted.web.client.ResponseDone): mbody += ' || ' + str(reason) - req_err(self._req, + self.cl.req_err(self._req, "FAILED %d %s | %s" % (self._resp.code, self._phrase, mbody)) -def req_ok(req, resp): - log_debug(DBG.HTTP_CTRL, +class Client(): + def __init__(cl, c,ss,cs): + cl.c = c + cl.outstanding = { } + cl.desc = '[%s %s] ' % (ss,cs) + + def log(cl, dflag, msg, **kwargs): + log_debug(dflag, cl.desc + msg, **kwargs) + + def log_outstanding(cl): + cl.log(DBG.CTRL_DUMP, 'OS %s' % outstanding) + + def start(cl): + cl.queue = PacketQueue('up', c.max_queue_time) + cl.agent = twisted.web.client.Agent( + reactor, connectTimeout = c.http_timeout) + + def outbound(cl, packet, saddr, daddr): + #print('OUT ', saddr, daddr, repr(packet)) + cl.queue.append(packet) + cl.check_outbound() + + def req_ok(cl, req, resp): + cl.log(DBG.HTTP_CTRL, 'req_ok %d %s %s' % (resp.code, repr(resp.phrase), str(resp)), idof=req) if resp.code == 200: - rc = ResponseConsumer(req) + rc = ResponseConsumer(cl, req) else: - rc = ErrorResponseConsumer(req, resp) + rc = ErrorResponseConsumer(cl, req, resp) resp.deliverBody(rc) # now rc is responsible for calling req_fin -def req_err(req, err): - # called when the Deferred fails, or (if it completes), - # later, by ResponsConsumer or ErrorResponsConsumer - try: - log_debug(DBG.HTTP_CTRL, 'req_err ' + str(err), idof=req) - if isinstance(err, twisted.python.failure.Failure): - err = err.getTraceback() - print('[%#x] %s' % (id(req), err), file=sys.stderr) - if not isinstance(outstanding[req], int): - raise RuntimeError('[%#x] previously %s' % (id(req), outstanding[req])) - outstanding[req] = err - log_outstanding() - reactor.callLater(c.http_retry, partial(req_fin, req)) - except Exception as e: - crash(traceback.format_exc() + '\n----- handling -----\n' + err) - -def req_fin(req): - del outstanding[req] - log_debug(DBG.HTTP_CTRL, 'req_fin OS=%d' % len(outstanding), idof=req) - check_outbound() - -class Errb: - def __init__(self, req): - self._req = req - def call(self, err): - req_err(self._req, err) - -def check_outbound(): - global outstanding - - while True: - if len(outstanding) >= c.max_outstanding : break - if not queue.nonempty() and len(outstanding) >= c.target_outstanding: break - - d = b'' - def moredata(s): nonlocal d; d += s - queue.process((lambda: len(d)), - moredata, - c.max_batch_up) - - d = mime_translate(d) - - crlf = b'\r\n' - lf = b'\n' - mime = (b'--b' + crlf + - b'Content-Type: text/plain; charset="utf-8"' + crlf + - b'Content-Disposition: form-data; name="m"' + crlf + crlf + - str(c.client) .encode('ascii') + crlf + - password + crlf + - str(c.target_outstanding) .encode('ascii') + crlf + - str(c.http_timeout) .encode('ascii') + crlf + - (( - b'--b' + crlf + - b'Content-Type: application/octet-stream' + crlf + - b'Content-Disposition: form-data; name="d"' + crlf + crlf + - d + crlf - ) if len(d) else b'') + - b'--b--' + crlf) - - #df = open('data.dump.dbg', mode='wb') - #df.write(mime) - #df.close() - # POST -use -c 'multipart/form-data; boundary="b"' http://localhost:8099/ = cl.c.max_outstanding: + break + + if (not queue.nonempty() and + len(cl.outstanding) >= cl.c.target_outstanding): + break + + d = b'' + def moredata(s): nonlocal d; d += s + queue.process((lambda: len(d)), + moredata, + cl.c.max_batch_up) + + d = mime_translate(d) + + crlf = b'\r\n' + lf = b'\n' + mime = (b'--b' + crlf + + b'Content-Type: text/plain; charset="utf-8"' + crlf + + b'Content-Disposition: form-data; name="m"' + crlf + crlf + + str(cl.c.client) .encode('ascii') + crlf + + cl.c.password + crlf + + str(cl.c.target_outstanding).encode('ascii') + crlf + + str(cl.c.http_timeout) .encode('ascii') + crlf + + (( + b'--b' + crlf + + b'Content-Type: application/octet-stream' + crlf + + b'Content-Disposition: form-data; name="d"' + crlf + crlf + + d + crlf + ) if len(d) else b'') + + b'--b--' + crlf) + + #df = open('data.dump.dbg', mode='wb') + #df.write(mime) + #df.close() + # POST -use -c 'multipart/form-data; boundary="b"' http://localhost:8099/ ] overrides ipif = userv root ipif %(local)s,%(peer)s,%(mtu)s,slip %(rnets)s @@ -112,9 +113,9 @@ ipif = userv root ipif %(local)s,%(peer)s,%(mtu)s,slip %(rnets)s # relating to virtual network mtu = 1500 -[server] +[SERVER] +server = SERVER # addrs = 127.0.0.1 ::1 -port = 80 # url # relating to virtual network @@ -128,7 +129,7 @@ vnetwork = 172.24.230.192 # [] # password = # used by both, must match -[limits] +[LIMIT] max_batch_down = 262144 max_queue_time = 121 http_timeout = 121 @@ -146,13 +147,11 @@ def mime_translate(s): return s.translate(_mimetrans) class ConfigResults: - def __init__(self, d = { }): - self.__dict__ = d + def __init__(self): + pass def __repr__(self): return 'ConfigResults('+repr(self.__dict__)+')' -c = ConfigResults() - def log_discard(packet, iface, saddr, daddr, why): log_debug(DBG.DROP, 'discarded packet [%s] %s -> %s: %s' % (iface, saddr, daddr, why), @@ -336,75 +335,163 @@ def crash_on_critical(event): #---------- config processing ---------- -def process_cfg_common_always(): - global mtu - c.mtu = cfg.get('virtual','mtu') +def _cfg_process_putatives(): + servers = { } + clients = { } + # maps from abstract object to canonical name for cs's -def process_cfg_ipif(section, varmap): - for d, s in varmap: - try: v = getattr(c, s) - except AttributeError: continue - setattr(c, d, v) + def putative(cmap, abstract, canoncs): + try: + current_canoncs = cmap[abstract] + except KeyError: + pass + else: + assert(current_canoncs == canoncs) + cmap[abstract] = canoncs + + server_pat = r'[-.0-9A-Za-z]+' + client_pat = r'[.:0-9a-f]+' + server_re = regexp.compile(server_pat) + serverclient_re = regexp.compile(server_pat + r' ' + client_pat) - #print(repr(c)) + for cs in cfg.sections(): + if cs = 'LIMIT': + # plan A "[LIMIT]" + continue - c.ipif_command = cfg.get(section,'ipif', vars=c.__dict__) + try: + # plan B "[]" part 1 + ci = ipaddr(cs) + except AddressValueError: -def process_cfg_network(): - c.network = ipnetwork(cfg.get('virtual','network')) + if server_re.fullmatch(cs): + # plan C "[]" + putative(servers, cs, cs) + continue + + if serverclient_re.fullmatch(cs): + # plan D "[ ]" part 1 + (pss,pcs) = cs.split(' ') + + if pcs = 'LIMIT': + # plan E "[ LIMIT]" + continue + + try: + # plan D "[ ]" part 2 + ci = ipaddr(pc) + except AddressValueError: + # plan F "[]" + # well, we ignore this + print('warning: ignoring config section %s' % cs, file=sys.stderr) + continue + + else: # no AddressValueError + # plan D "[ " part 2 + putative(clients, ci, cs) + continue + + return (servers, clients) + +def cfg_process_common(ss): + c.mtu = cfg.getint(ss, 'mtu') + +def cfg_process_saddrs(c, ss): + class ServerAddr(): + def __init__(self, port, addrspec): + self.port = port + # also self.addr + try: + self.addr = ipaddress.IPv4Address(addrspec) + self._endpointfactory = twisted.internet.endpoints.TCP4ServerEndpoint + self._inurl = b'%s' + except AddressValueError: + self.addr = ipaddress.IPv6Address(addrspec) + self._endpointfactory = twisted.internet.endpoints.TCP6ServerEndpoint + self._inurl = b'[%s]' + def make_endpoint(self): + return self._endpointfactory(reactor, self.port, self.addr) + def url(self): + url = b'http://' + (self._inurl % str(self.addr).encode('ascii')) + if self.port != 80: url += b':%d' % self.port + url += b'/' + return url + + c.port = cfg.getint(ss,'port') + c.saddrs = [ ] + for addrspec in cfg.get(ss, 'addrs').split(): + sa = ServerAddr(c.port, addrspec) + c.saddrs.append(sa) + +def cfg_process_vnetwork(c, ss): + c.network = ipnetwork(cfg.get(ss,'network')) if c.network.num_addresses < 3 + 2: raise ValueError('network needs at least 2^3 addresses') -def process_cfg_server(): +def cfg_process_vaddr(): try: c.server = cfg.get('virtual','server') except NoOptionError: process_cfg_network() c.server = next(c.network.hosts()) -class ServerAddr(): - def __init__(self, port, addrspec): - self.port = port - # also self.addr - try: - self.addr = ipaddress.IPv4Address(addrspec) - self._endpointfactory = twisted.internet.endpoints.TCP4ServerEndpoint - self._inurl = b'%s' - except AddressValueError: - self.addr = ipaddress.IPv6Address(addrspec) - self._endpointfactory = twisted.internet.endpoints.TCP6ServerEndpoint - self._inurl = b'[%s]' - def make_endpoint(self): - return self._endpointfactory(reactor, self.port, self.addr) - def url(self): - url = b'http://' + (self._inurl % str(self.addr).encode('ascii')) - if self.port != 80: url += b':%d' % self.port - url += b'/' - return url +def cfg_search_section(key,sections): + for section in sections: + if cfg.has_option(section, key): + return section + raise NoOptionError('missing %s %s' % (key, repr(sections))) + +def cfg_search(getter,key,sections): + section = cfg_search_section(key,sections) + return getter(section, key) + +def cfg_process_client_limited(cc,ss,sections,key): + val = cfg_search(cfg.getint, key, sections) + lim = cfg_search(cfg.getint, key, '%s LIMIT' % ss, 'LIMIT') + cc.__dict__[key] = min(val,lim) + +def cfg_process_client_common(cc,ss,cs,ci): + # returns sections to search in, iff password is defined, otherwise None + cc.ci = ci + + sections = ['%s %s' % section, + cs, + ss, + 'DEFAULT'] + + try: pwsection = cfg_search_section('password', sections) + except NoOptionError: return None -def process_cfg_saddrs(): - try: port = cfg.getint('server','port') - except NoOptionError: port = 80 + pw = cfg.get(pwsection, 'password') + pw = pw.encode('utf-8') - c.saddrs = [ ] - for addrspec in cfg.get('server','addrs').split(): - sa = ServerAddr(port, addrspec) - c.saddrs.append(sa) + cfg_process_client_limited(cc,ss,sections,'target_requests_outstanding') + cfg_process_client_limited(cc,ss,sections,'http_timeout') -def process_cfg_clients(constructor): - c.clients = [ ] - for cs in cfg.sections(): - if not (':' in cs or '.' in cs): continue - ci = ipaddr(cs) - pw = cfg.get(cs, 'password') - pw = pw.encode('utf-8') - constructor(ci,cs,pw) + return sections + +def process_cfg_ipif(c, sections, varmap): + for d, s in varmap: + try: v = getattr(c, s) + except AttributeError: continue + setattr(c, d, v) + + section = cfg_search_section('ipif', sections) + c.ipif_command = cfg.get(section,'ipif', vars=c.__dict__) #---------- startup ---------- def common_startup(process_cfg): + # calls process_cfg(putative_clients, putative_servers) + # ConfigParser hates #-comments after values - trailingcomments_re = regexp.compile('#.*') + trailingcomments_re = regexp.compile(r'#.*') cfg.read_string(trailingcomments_re.sub('', defcfg)) need_defcfg = True @@ -514,7 +601,9 @@ just `+': all DFLAGs. readconfig('/etc/hippotat/config', False) readconfig('/etc/hippotat/config.d', False) - try: process_cfg() + try: + (pss, pcs) = process_cfg_putatives() + process_cfg(pss, pcs) except (configparser.Error, ValueError): traceback.print_exc(file=sys.stderr) print('\nInvalid configuration, giving up.', file=sys.stderr) diff --git a/server b/server index bdd893c..144866f 100755 --- a/server +++ b/server @@ -52,7 +52,9 @@ class Client(): if ip not in c.network: raise ValueError('client %s not in network' % ip) - for k in ('max_batch_down','max_queue_time','http_timeout', + for k in ('max_batch_down', + 'max_queue_time', + 'http_timeout', 'target_requests_outstanding'): req = cfg.getint(cs, k) limit = cfg.getint('limits',k) @@ -220,26 +222,35 @@ def start_http(): #---------- config and setup ---------- -def process_cfg(): - process_cfg_common_always() - process_cfg_server() - process_cfg_network() +def process_cfg(putative_servers, putative_clients): + c = ConfigResults() + c.server = cfg.get('SERVER','server') + + process_cfg_common(c, c.server) + process_cfg_saddrs(c, c.server) + process_cfg_vnetwork(c, c.server) + process_cfg_vaddr(c, c.server) + + for (ci,cs) in putative_clients.items(): + cc = ConfigResults() + sections = cfg_process_client_common(cc,c.server,cs,ci): + if not sections: continue + cfg_process_client_limited(cc,c.server,sections, 'max_batch_down') + cfg_process_client_limited(cc,c.server,sections, 'max_queue_time') try: - c.relay = cfg.get('virtual','relay') + c.relay = cfg.get(c.server, 'relay') except NoOptionError: for search in c.network.hosts(): if search == c.server: continue c.relay = search break - process_cfg_saddrs() - process_cfg_clients(Client) - - process_cfg_ipif('server', - (('local','server'), - ('peer', 'relay'), - ('rnets','network'))) + process_cfg_ipif(c, + [c.server, 'DEFAULT'], + (('local','vaddr'), + ('peer', 'vrelay'), + ('rnets','vnetwork'))) common_startup(process_cfg) start_ipif(c.ipif_command, (lambda p,s,d: route(p,"[ipif]",s,d))) diff --git a/sgo-demo.cfg b/sgo-demo.cfg index 58f2eba..e7d369f 100644 --- a/sgo-demo.cfg +++ b/sgo-demo.cfg @@ -34,7 +34,7 @@ max_batch_down = 32768 [SERVER] server = chiark -# -- in passwords.d/chiark-192.0.2.4 (on zealot and chiark) +# -- in passwords.d/chiark-zealot (on zealot and chiark) [chiark 192.0.2.4] password = sesame diff --git a/test.cfg b/test.cfg index e1469c9..0c2ebbe 100644 --- a/test.cfg +++ b/test.cfg @@ -17,3 +17,6 @@ password = sesame ipif = PATH=/usr/local/sbin:/sbin:/usr/sbin:$PATH really ./fake-userv /home/ian/things/Userv/userv-utils.git/ipif/service \* -- %(local)s,%(peer)s,%(mtu)s,slip '%(rnets)s' # ./client -D -c test.cfg + +[192.0.2.4] +password = zorkmids