We don't need this any more.
Signed-off-by: Ian Jackson <ijackson@chiark.greenend.org.uk>
+++ /dev/null
-#!/usr/bin/python3
-#
-# Hippotat - Asinine IP Over HTTP program
-# ./hippotat - client main program
-#
-# Copyright 2017 Ian Jackson
-#
-# GPLv3+
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program, in the file GPLv3. If not,
-# see <http://www.gnu.org/licenses/>.
-
-#@ import sys; sys.path.append('@PYBUILD_INSTALL_DIR@')
-from hippotatlib import *
-
-import twisted.web
-import twisted.web.client
-import urllib.parse
-
-import io
-
-class GeneralResponseConsumer(twisted.internet.protocol.Protocol):
- def __init__(self, cl, req, resp, desc):
- self._cl = cl
- self._req = req
- self._resp = resp
- self._desc = desc
-
- def _log(self, dflag, msg, **kwargs):
- self._cl.log(dflag, '%s: %s' % (self._desc, msg), idof=self._req, **kwargs)
-
- def connectionMade(self):
- self._log(DBG.HTTP_CTRL, 'connectionMade')
-
- def connectionLostOK(self, reason):
- return (reason.check(twisted.web.client.ResponseDone) or
- reason.check(twisted.web.client.PotentialDataLoss))
- # twisted.web.client.PotentialDataLoss is an entirely daft
- # exception. It will occur every time if the origin server does
- # not provide a Content-Length. (hippotatd does, of course, but
- # the HTTP transaction might be proxied.)
-
-class ResponseConsumer(GeneralResponseConsumer):
- def __init__(self, cl, req, resp):
- super().__init__(cl, req, resp, 'RC')
- ssddesc = '[%s] %s' % (id(req), self._desc)
- self._ssd = SlipStreamDecoder(ssddesc, partial(queue_inbound, cl.ipif),
- cl.c.mtu)
- self._log(DBG.HTTP_CTRL, '__init__')
-
- def dataReceived(self, data):
- self._log(DBG.HTTP, 'dataReceived', d=data)
- try:
- self._ssd.inputdata(data)
- except Exception as e:
- self._handleexception()
-
- def connectionLost(self, reason):
- reason_msg = 'connectionLost ' + str(reason)
- self._log(DBG.HTTP_CTRL, reason_msg)
- if not self.connectionLostOK(reason):
- self._latefailure(reason_msg)
- return
- try:
- self._log(DBG.HTTP, 'ResponseDone')
- self._ssd.flush()
- self._cl.req_fin(self._req)
- except Exception as e:
- self._handleexception()
- self._cl.report_running()
-
- def _handleexception(self):
- self._latefailure(traceback.format_exc())
-
- def _latefailure(self, reason):
- self._log(DBG.HTTP_CTRL, '_latefailure ' + str(reason))
- self._cl.req_err(self._req, reason)
-
-class ErrorResponseConsumer(GeneralResponseConsumer):
- def __init__(self, cl, req, resp):
- super().__init__(cl, req, resp, 'ERROR-RC')
- self._m = b''
- try:
- self._phrase = resp.phrase.decode('utf-8')
- except Exception:
- self._phrase = repr(resp.phrase)
- self._log(DBG.HTTP_CTRL, '__init__ %d %s' % (resp.code, self._phrase))
-
- def dataReceived(self, data):
- self._log(DBG.HTTP_CTRL, 'dataReceived ' + repr(data))
- self._m += data
-
- def connectionLost(self, reason):
- try:
- mbody = self._m.decode('utf-8')
- except Exception:
- mbody = repr(self._m)
- if not self.connectionLostOK(reason):
- mbody += ' || ' + str(reason)
- self._cl.req_err(self._req,
- "FAILED %d %s | %s"
- % (self._resp.code, self._phrase, mbody))
-
-class Client():
- def __init__(cl, c,ss,cs):
- cl.c = c
- cl.outstanding = { }
- cl.desc = '[%s %s] ' % (ss,cs)
- cl.running_reported = False
- cl.log_info('setting up')
-
- def log_info(cl, msg):
- log.info(cl.desc + msg, dflag=False)
-
- def report_running(cl):
- if not cl.running_reported:
- cl.log_info('running OK')
- cl.running_reported = True
-
- def log(cl, dflag, msg, **kwargs):
- log_debug(dflag, cl.desc + msg, **kwargs)
-
- def log_outstanding(cl):
- cl.log(DBG.CTRL_DUMP, 'OS %s' % cl.outstanding)
-
- def start(cl):
- cl.queue = PacketQueue('up', cl.c.max_queue_time)
- cl.agent = twisted.web.client.Agent(
- reactor, connectTimeout = cl.c.http_timeout)
-
- def outbound(cl, packet, saddr, daddr):
- #print('OUT ', saddr, daddr, repr(packet))
- cl.queue.append(packet)
- cl.check_outbound()
-
- def req_ok(cl, req, resp):
- cl.log(DBG.HTTP_CTRL,
- 'req_ok %d %s %s' % (resp.code, repr(resp.phrase), str(resp)),
- idof=req)
- if resp.code == 200:
- rc = ResponseConsumer(cl, req, resp)
- else:
- rc = ErrorResponseConsumer(cl, req, resp)
-
- resp.deliverBody(rc)
- # now rc is responsible for calling req_fin
-
- def req_err(cl, req, err):
- # called when the Deferred fails, or (if it completes),
- # later, by ResponsConsumer or ErrorResponsConsumer
- try:
- cl.log(DBG.HTTP_CTRL, 'req_err ' + str(err), idof=req)
- cl.running_reported = False
- if isinstance(err, twisted.python.failure.Failure):
- err = err.getTraceback()
- print('%s[%#x] %s' % (cl.desc, id(req), err.strip('\n').replace('\n',' / ')),
- file=sys.stderr)
- if not isinstance(cl.outstanding[req], int):
- raise RuntimeError('[%#x] previously %s' %
- (id(req), cl.outstanding[req]))
- cl.outstanding[req] = err
- cl.log_outstanding()
- reactor.callLater(cl.c.http_retry, partial(cl.req_fin, req))
- except Exception as e:
- crash(traceback.format_exc() + '\n----- handling -----\n' + err)
-
- def req_fin(cl, req):
- del cl.outstanding[req]
- cl.log(DBG.HTTP_CTRL, 'req_fin OS=%d' % len(cl.outstanding), idof=req)
- cl.check_outbound()
-
- def check_outbound(cl):
- while True:
- if len(cl.outstanding) >= cl.c.max_outstanding:
- break
-
- if (not cl.queue.nonempty() and
- len(cl.outstanding) >= cl.c.target_requests_outstanding):
- break
-
- d = b''
- def moredata(s): nonlocal d; d += s
- cl.queue.process((lambda: len(d)),
- moredata,
- cl.c.max_batch_up)
-
- d = mime_translate(d)
-
- token = authtoken_make(cl.c.secret)
-
- crlf = b'\r\n'
- lf = b'\n'
- mime = (b'--b' + crlf +
- b'Content-Type: text/plain; charset="utf-8"' + crlf +
- b'Content-Disposition: form-data; name="m"' + crlf + crlf +
- str(cl.c.client) .encode('ascii') + crlf +
- token + crlf +
- str(cl.c.target_requests_outstanding)
- .encode('ascii') + crlf +
- str(cl.c.http_timeout) .encode('ascii') + crlf +
- ((
- b'--b' + crlf +
- b'Content-Type: application/octet-stream' + crlf +
- b'Content-Disposition: form-data; name="d"' + crlf + crlf +
- d + crlf
- ) if len(d) else b'') +
- b'--b--' + crlf)
-
- #df = open('data.dump.dbg', mode='wb')
- #df.write(mime)
- #df.close()
- # POST -use -c 'multipart/form-data; boundary="b"' http://localhost:8099/ <data.dump.dbg
-
- cl.log(DBG.HTTP_FULL, 'requesting: ' + str(mime))
-
- hh = { 'User-Agent': ['hippotat'],
- 'Content-Type': ['multipart/form-data; boundary="b"'] }
-
- bytesreader = io.BytesIO(mime)
- producer = twisted.web.client.FileBodyProducer(bytesreader)
-
- req = cl.agent.request(b'POST',
- cl.c.url,
- twisted.web.client.Headers(hh),
- producer)
-
- cl.outstanding[req] = len(d)
- cl.log(DBG.HTTP_CTRL,
- 'request OS=%d' % len(cl.outstanding),
- idof=req, d=d)
- req.addTimeout(cl.c.http_timeout, reactor)
- req.addCallback(partial(cl.req_ok, req))
- req.addErrback(partial(cl.req_err, req))
-
- cl.log_outstanding()
-
-clients = [ ]
-
-def encode_url(urlstr):
- # Oh, this is a disaster. We're given a URL as a `str', but the underlying
- # machinery insists on having `bytes'. Assume we've been given a sensible
- # URL, with escaping in all of the necessary places, except that it may
- # contain non-ASCII characters: then encode as UTF-8 and squash the top-
- # bit-set bytes down to percent escapes.
- #
- # This conses like it's going out of fashion, but it gets the job done.
- return b''.join(bytes([b]) if b < 128 else '%%%02X' % b
- for b in urlstr.encode('utf-8'))
-
-def process_cfg(_opts, putative_servers, putative_clients):
- global clients
-
- for ss in putative_servers.values():
- for (ci,cs) in putative_clients.items():
- c = ConfigResults()
-
- sections = cfg_process_client_common(c,ss,cs,ci)
- if not sections: continue
-
- log_debug_config('processing client [%s %s]' % (ss, cs))
-
- def srch(getter,key): return cfg_search(getter,key,sections)
-
- c.http_timeout += srch(cfg.getint, 'http_timeout_grace')
- c.max_outstanding = srch(cfg.getint, 'max_requests_outstanding')
- c.max_batch_up = srch(cfg.getint, 'max_batch_up')
- c.http_retry = srch(cfg.getint, 'http_retry')
- c.max_queue_time = srch(cfg.getint, 'max_queue_time')
- c.vroutes = srch(cfg.get, 'vroutes')
-
- try: c.ifname = srch(cfg_get_raw, 'ifname_client')
- except NoOptionError: pass
-
- try: c.url = encode_url(srch(cfg.get,'url'))
- except NoOptionError:
- cfg_process_saddrs(c, ss)
- c.url = c.saddrs[0].url()
-
- c.client = ci
-
- cfg_process_vaddr(c,ss)
-
- cfg_process_ipif(c,
- sections,
- (('local','client'),
- ('peer', 'vaddr'),
- ('rnets','vroutes')))
-
- clients.append(Client(c,ss,cs))
-
-common_startup(process_cfg)
-
-for cl in clients:
- cl.start()
- cl.ipif = start_ipif(cl.c.ipif_command, cl.outbound, cl.c.mtu)
- cl.check_outbound()
-
-common_run()
+++ /dev/null
-#!/usr/bin/python3
-#
-# Hippotat - Asinine IP Over HTTP program
-# ./hippotatd - server main program
-#
-# Copyright 2017 Ian Jackson
-#
-# AGPLv3+ + CAFv2+
-#
-# This program is free software: you can redistribute it and/or
-# modify it under the terms of the GNU Affero General Public
-# License as published by the Free Software Foundation, either
-# version 3 of the License, or (at your option) any later version,
-# with the "CAF Login Exception" as published by Ian Jackson
-# (version 2, or at your option any later version) as an Additional
-# Permission.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public
-# License and the CAF Login Exception along with this program, in
-# the file AGPLv3+CAFv2. If not, email Ian Jackson
-# <ijackson@chiark.greenend.org.uk>.
-
-#@ import sys; sys.path.append('@PYBUILD_INSTALL_DIR@')
-from hippotatlib import *
-
-import os
-import tempfile
-import atexit
-import shutil
-import subprocess
-
-import twisted.internet
-from twisted.web.server import NOT_DONE_YET
-
-import twisted.web.static
-
-import hippotatlib.ownsource
-from hippotatlib.ownsource import SourceShipmentPreparer
-
-#import twisted.web.server import Site
-#from twisted.web.resource import Resource
-
-import syslog
-
-cleanups = [ ]
-
-clients = { }
-
-#---------- "router" ----------
-
-def route(packet, iface, saddr, daddr):
- def lt(dest):
- log_debug(DBG.ROUTE, 'route: %s -> %s: %s' % (saddr,daddr,dest), d=packet)
- try: dclient = clients[daddr]
- except KeyError: dclient = None
- if dclient is not None:
- lt('client')
- dclient.queue_outbound(packet)
- elif daddr == c.vaddr or daddr not in c.vnetwork:
- lt('inbound')
- queue_inbound(ipif, packet)
- elif daddr == c.vrelay:
- lt('discard relay')
- log_discard(packet, iface, saddr, daddr, 'relay')
- else:
- lt('discard no-client')
- log_discard(packet, iface, saddr, daddr, 'no-client')
-
-#---------- client ----------
-
-class Client():
- def __init__(self, ip, cc):
- # instance data members
- self._ip = ip
- self.cc = cc
- self._rq = collections.deque() # requests
- self._pq = PacketQueue(str(ip), self.cc.max_queue_time)
-
- if ip not in c.vnetwork:
- raise ValueError('client %s not in vnetwork' % ip)
-
- if ip in clients:
- raise ValueError('multiple client cfg sections for %s' % ip)
- clients[ip] = self
-
- self._log(DBG.INIT, 'new')
-
- def _log(self, dflag, msg, **kwargs):
- log_debug(dflag, ('client %s: ' % self._ip)+msg, **kwargs)
-
- def process_arriving_data(self, d):
- self._log(DBG.FLOW, "req data (enc'd)", d=d)
- if not len(d): return
- for packet in slip.decode(d):
- (saddr, daddr) = packet_addrs(packet)
- if saddr != self._ip:
- raise ValueError('wrong source address %s' % saddr)
- route(packet, self._ip, saddr, daddr)
-
- def _req_cancel(self, request):
- self._log(DBG.HTTP_CTRL, 'cancel', idof=request)
- try: request.finish()
- except Exception: pass
-
- def _req_error(self, err, request):
- self._log(DBG.HTTP_CTRL, 'error %s' % err, idof=request)
- self._req_cancel(request)
-
- def queue_outbound(self, packet):
- self._pq.append(packet)
- self._check_outbound()
-
- def _req_fin(self, dummy, request, cl):
- self._log(DBG.HTTP_CTRL, '_req_fin ' + repr(dummy), idof=request)
- try: cl.cancel()
- except twisted.internet.error.AlreadyCalled: pass
-
- def new_request(self, request):
- request.setHeader('Content-Type','application/octet-stream')
- cl = reactor.callLater(self.cc.http_timeout, self._req_cancel, request)
- nf = request.notifyFinish()
- nf.addErrback(self._req_error, request)
- nf.addCallback(self._req_fin, request, cl)
- self._rq.append((request,nf))
- self._check_outbound()
-
- def _req_write(self, req, d):
- self._log(DBG.HTTP, 'req_write ', idof=req, d=d)
- req.write(d)
-
- def _check_outbound(self):
- log_debug(DBG.HTTP_CTRL, 'CHKO')
- while True:
- try: (request,nf) = self._rq[0]
- except IndexError: request = None
- if request and nf.called:
- self._log(DBG.HTTP_CTRL, 'CHKO req finished, discard', idof=request)
- self._rq.popleft()
- continue
-
- if not self._pq.nonempty():
- # no packets, oh well
- self._log(DBG.HTTP_CTRL, 'CHKO no packets, OUT-DONE', idof=request)
- break
-
- if request is None:
- # no request
- self._log(DBG.HTTP_CTRL, 'CHKO no request, OUT-DONE', idof=request)
- break
-
- self._log(DBG.HTTP_CTRL, 'CHKO processing', idof=request)
- # request, and also some non-expired packets
- self._pq.process((lambda: request.sentLength),
- (lambda d: self._req_write(request, d)),
- self.cc.max_batch_down)
-
- assert(request.sentLength)
- self._rq.popleft()
- request.finish()
- self._log(DBG.HTTP, 'complete', idof=request)
- # round again, looking for more to do
-
- while len(self._rq) > self.cc.target_requests_outstanding:
- (request, nf) = self._rq.popleft()
- self._log(DBG.HTTP, 'CHKO above target, returning empty', idof=request)
- request.finish()
-
-def process_request(request, desca):
- # find client, update config, etc.
- metadata = request.args[b'm'][0]
- metadata = metadata.split(b'\r\n')
- (ci_s, token, tro, cto) = metadata[0:4]
- desca['m[0,2:3]'] = [ci_s, tro, cto]
- ci_s = ci_s.decode('utf-8')
- tro = int(tro); desca['tro']= tro
- cto = int(cto); desca['cto']= cto
- ci = ipaddr(ci_s)
- desca['ci'] = ci
- cl = clients[ci]
- authtoken_check(cl.cc.secret, token, cl.cc.max_clock_skew)
- desca['pwok']=True
-
- if tro != cl.cc.target_requests_outstanding:
- raise ValueError('tro must be %d' % cl.cc.target_requests_outstanding)
-
- if cto < cl.cc.http_timeout:
- raise ValueError('cto must be >= %d' % cl.cc.http_timeout)
-
- try:
- d = request.args[b'd'][0]
- desca['d'] = d
- desca['dlen'] = len(d)
- except KeyError:
- d = b''
- desca['dlen'] = None
-
- log_http(desca, 'processing', idof=id(request), d=d)
-
- d = mime_translate(d)
-
- cl.process_arriving_data(d)
- cl.new_request(request)
-
-def log_http(desca, msg, **kwargs):
- try:
- kwargs['d'] = desca['d']
- del desca['d']
- except KeyError:
- pass
- log_debug(DBG.HTTP, msg + repr(desca), **kwargs)
-
-class NotStupidResource(twisted.web.resource.Resource):
- # why this is not the default is a mystery!
- def getChild(self, name, request):
- if name == b'': return self
- else: return twisted.web.resource.Resource.getChild(self, name, request)
-
-class IphttpResource(NotStupidResource):
- def render_POST(self, request):
- log_debug(DBG.HTTP_FULL,
- 'req recv: ' + repr(request) + ' ' + repr(request.args),
- idof=id(request))
- desca = {'d': None}
- try: process_request(request, desca)
- except Exception as e:
- emsg = traceback.format_exc()
- log_http(desca, 'RETURNING EXCEPTION ' + emsg)
- request.setHeader('Content-Type','text/plain; charset="utf-8"')
- request.setResponseCode(400)
- return (emsg + ' # ' + repr(desca) + '\r\n').encode('utf-8')
- log_debug(DBG.HTTP_CTRL, '...', idof=id(request))
- return NOT_DONE_YET
-
- # instantiator should set
- # self.hippotat_sources = (source_names[0], source_names[1])
- def __init__(self):
- self.hippotat_sources = [None, None]
- super().__init__()
-
- def render_GET(self, request):
- log_debug(DBG.HTTP, 'GET request')
- s = '<html><body>hippotat\n'
- (s0,s1) = self.hippotat_sources
- if s0:
- s += '<p><a href="%s">source</a>\n' % s0
- if self.hippotat_sources[1]:
- s += ('(and that of dependency <a href="%s">packages</a>)\n' % s1)
- s += 'available'
- else:
- s += 'TESTING'
- s += '</body></html>'
- return s.encode('utf-8')
-
-def start_http():
- resource = IphttpResource()
- site = twisted.web.server.Site(resource)
-
- for sa in c.saddrs:
- ep = sa.make_endpoint()
- crash_on_defer(ep.listen(site))
- log_debug(DBG.INIT, 'listening on %s' % sa)
-
- td = tempfile.mkdtemp()
-
- def cleanup():
- try: shutil.rmtree(td)
- except FileNotFoundError: pass
-
- cleanups.append(cleanup)
-
- ssp = SourceShipmentPreparer(td)
- ssp.logger = partial(log_debug, DBG.OWNSOURCE)
- if DBG.OWNSOURCE in debug_set: ssp.stream_debug = sys.stdout
- ssp.download_packages = opts.ownsource >= 2
- if opts.ownsource >= 1: ssp.generate()
-
- for ix in (0,1):
- bn = ssp.output_names[ix]
- op = ssp.output_paths[ix]
- if op is None: continue
- resource.hippotat_sources[ix] = bn
- subresource =twisted.web.static.File(op)
- resource.putChild(bn.encode('utf-8'), subresource)
-
- reactor.callLater(0.1, (lambda: log.info('hippotatd started', dflag=False)))
-
-#---------- config and setup ----------
-
-def process_cfg(_opts, putative_servers, putative_clients):
- global opts
- opts = _opts
-
- global c
- c = ConfigResults()
- try: c.server = cfg1get('SERVER','server')
- except NoOptionError: c.server = 'SERVER'
-
- cfg_process_general(c, c.server)
- cfg_process_saddrs(c, c.server)
- cfg_process_vnetwork(c, c.server)
- cfg_process_vaddr(c, c.server)
-
- for (ci,cs) in putative_clients.items():
- cc = ConfigResults()
- sections = cfg_process_client_common(cc,c.server,cs,ci)
- if not sections: continue
- cfg_process_client_limited(cc,c.server,sections, 'max_batch_down')
- cfg_process_client_limited(cc,c.server,sections, 'max_queue_time')
- cc.max_clock_skew = cfg_search(cfg.getint, 'max_clock_skew', sections)
- Client(ci, cc)
-
- try:
- c.vrelay = cfg1get(c.server, 'vrelay')
- except NoOptionError:
- for search in c.vnetwork.hosts():
- if search == c.vaddr: continue
- c.vrelay = search
- break
-
- try: c.ifname = cfg1get(c.server, 'ifname_server', raw=True)
- except NoOptionError: pass
-
- cfg_process_ipif(c,
- [c.server, 'COMMON'],
- (('local','vaddr'),
- ('peer', 'vrelay'),
- ('rnets','vnetwork')))
-
- if opts.printconfig is not None:
- try: val = cfg1get(c.server, opts.printconfig)
- except NoOptionError: pass
- else: print(val)
- sys.exit(0)
-
-def catch_termination():
- def run_cleanups():
- for cleanup in cleanups:
- cleanup()
-
- atexit.register(run_cleanups)
-
- def signal_handler(name, sig, *args):
- signal.signal(sig, signal.SIG_DFL)
- print('exiting due to %s' % name, file=sys.stderr)
- run_cleanups()
- os.kill(os.getpid(), sig)
- raise RuntimeError('did not die due to signal %s !' % name)
-
- for sig in (signal.SIGINT, signal.SIGTERM):
- try: signame = sig.name
- except AttributeError: signame = "signal %d" % sig
- signal.signal(sig, partial(signal_handler, signame))
-
-def daemonise():
- global syslogfacility
- if opts.daemon and opts.syslogfacility is None:
- opts.syslogfacility = 'daemon'
-
- if opts.syslogfacility is not None:
- facilnum = syslog.__dict__['LOG_' + opts.syslogfacility.upper()]
- syslog.openlog('hippotatd',
- facility=facilnum,
- logoption=syslog.LOG_PID)
- def emit(event):
- if logevent_is_boringtwisted(event): return
- m = twisted.logger.formatEvent(event)
- #print(repr(event), m, file=org_stderr)
- level = event.get('log_level')
- if event.get('dflag',None) is not None: sl = syslog.LOG_DEBUG
- elif level == LogLevel.critical : sl = syslog.LOG_CRIT
- elif level == LogLevel.error : sl = syslog.LOG_ERR
- elif level == LogLevel.warn : sl = syslog.LOG_WARNING
- else : sl = syslog.LOG_INFO
- syslog.syslog(sl,m)
- failure = event.get('log_failure')
- if failure is not None:
- for l in failure.getTraceback().split('\n'):
- syslog.syslog(sl,l)
- glp = twisted.logger.globalLogPublisher
- glp.addObserver(emit)
- log_debug(DBG.INIT, 'starting to log to syslog')
-
- #log.crit('daemonic hippotatd crashed', dflag=False)
- if opts.daemon:
- daemonic_reactor = (twisted.internet.interfaces.IReactorDaemonize
- .providedBy(reactor))
- if daemonic_reactor: reactor.beforeDaemonize()
- if opts.pidfile is not None:
- pidfile_h = open(opts.pidfile, 'w')
- rfd, wfd = os.pipe()
- childpid = os.fork()
- if childpid:
- # we are the parent
- os.close(wfd)
- st = os.read(rfd, 1)
- try:
- st = st[0]
- except IndexError:
- st = 127
- log.critical('daemonic hippotatd crashed', dflag=False)
- os._exit(st)
- os.close(rfd)
- os.setsid()
- grandchildpid = os.fork()
- if grandchildpid:
- # we are the intermediate child
- if opts.pidfile is not None:
- print(grandchildpid, file=pidfile_h)
- pidfile_h.close()
- os._exit(0)
-
- if opts.pidfile is not None:
- pidfile_h.close()
-
- logger = subprocess.Popen(['logger','-d',
- '-t','hippotat[%d](stderr)' % os.getpid(),
- '-p',opts.syslogfacility + '.err'],
- stdin=subprocess.PIPE,
- stdout=subprocess.DEVNULL,
- stderr=subprocess.DEVNULL,
- restore_signals=True)
-
- nullfd = os.open('/dev/null', os.O_RDWR)
- os.dup2(nullfd, 0)
- os.dup2(nullfd, 1)
- os.dup2(logger.stdin.fileno(), 2)
- os.close(nullfd)
- if daemonic_reactor: reactor.afterDaemonize()
- log_debug(DBG.INIT, 'daemonised')
- os.write(wfd, b'\0')
- os.close(wfd)
-
- if opts.syslogfacility is not None:
- glp.removeObserver(hippotatlib.file_log_observer)
-
-optparser.add_option('--ownsource', default=2,
- action='store_const', dest='ownsource', const=2,
- help='source download fully enabled (default)')
-
-optparser.add_option('--ownsource-local',
- action='store_const', dest='ownsource', const=1,
- help='source download is local source code only')
-
-optparser.add_option('--no-ownsource',
- action='store_const', dest='ownsource', const=0,
- help='source download disabled (for testing only)')
-
-optparser.add_option('--daemon',
- action='store_true', dest='daemon', default=False,
- help='daemonize (and log to syslog)')
-
-optparser.add_option('--pidfile',
- nargs=1, type='string',
- action='store', dest='pidfile', default=None,
- help='write pid to this file')
-
-optparser.add_option('--syslog-facility',
- nargs=1, type='string',action='store',
- metavar='FACILITY', dest='syslogfacility',
- default=None,
- help='log to syslog, with specified facility')
-
-optparser.add_option('--print-config',
- nargs=1, type='string',action='store',
- metavar='OPTION', dest='printconfig',
- default=None,
- help='print one config option value and exit')
-
-common_startup(process_cfg)
-catch_termination()
-start_http()
-daemonise()
-ipif = start_ipif(c.ipif_command,
- (lambda p,s,d: route(p,"[ipif]",s,d)),
- c.mtu)
-common_run()
+++ /dev/null
-# -*- python -*-
-#
-# Hippotat - Asinine IP Over HTTP program
-# hippotatlib/__init__.py - common library code
-#
-# Copyright 2017 Ian Jackson
-#
-# GPLv3+
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program, in the file GPLv3. If not,
-# see <http://www.gnu.org/licenses/>.
-
-
-import signal
-signal.signal(signal.SIGINT, signal.SIG_DFL)
-
-import sys
-import os
-
-from zope.interface import implementer
-
-import twisted
-from twisted.internet import reactor
-import twisted.internet.endpoints
-import twisted.logger
-from twisted.logger import LogLevel
-import twisted.python.constants
-from twisted.python.constants import NamedConstant
-
-import ipaddress
-from ipaddress import AddressValueError
-
-from optparse import OptionParser
-import configparser
-from configparser import ConfigParser
-from configparser import NoOptionError
-
-from functools import partial
-
-import collections
-import time
-import hmac
-import hashlib
-import base64
-import codecs
-import traceback
-
-import re as regexp
-
-import hippotatlib.slip as slip
-
-class DBG(twisted.python.constants.Names):
- INIT = NamedConstant()
- CONFIG = NamedConstant()
- ROUTE = NamedConstant()
- DROP = NamedConstant()
- OWNSOURCE = NamedConstant()
- FLOW = NamedConstant()
- HTTP = NamedConstant()
- TWISTED = NamedConstant()
- QUEUE = NamedConstant()
- HTTP_CTRL = NamedConstant()
- QUEUE_CTRL = NamedConstant()
- HTTP_FULL = NamedConstant()
- CTRL_DUMP = NamedConstant()
- SLIP_FULL = NamedConstant()
- DATA_COMPLETE = NamedConstant()
-
-_hex_codec = codecs.getencoder('hex_codec')
-
-#---------- logging ----------
-
-org_stderr = sys.stderr
-
-log = twisted.logger.Logger()
-
-debug_set = set()
-debug_def_detail = DBG.HTTP
-
-def log_debug(dflag, msg, idof=None, d=None):
- if dflag not in debug_set: return
- #print('---------------->',repr((dflag, msg, idof, d)), file=sys.stderr)
- if idof is not None:
- msg = '[%#x] %s' % (id(idof), msg)
- if d is not None:
- trunc = ''
- if not DBG.DATA_COMPLETE in debug_set:
- if len(d) > 64:
- d = d[0:64]
- trunc = '...'
- d = _hex_codec(d)[0].decode('ascii')
- msg += ' ' + d + trunc
- log.info('{dflag} {msgcore}', dflag=dflag, msgcore=msg)
-
-def logevent_is_boringtwisted(event):
- try:
- if event.get('log_level') != LogLevel.info:
- return False
- dflag = event.get('dflag')
- if dflag is False : return False
- if dflag in debug_set: return False
- if dflag is None and DBG.TWISTED in debug_set: return False
- return True
- except Exception:
- print('EXCEPTION (IN BORINGTWISTED CHECK)',
- traceback.format_exc(), file=org_stderr)
- return False
-
-@implementer(twisted.logger.ILogFilterPredicate)
-class LogNotBoringTwisted:
- def __call__(self, event):
- return (
- twisted.logger.PredicateResult.no
- if logevent_is_boringtwisted(event) else
- twisted.logger.PredicateResult.yes
- )
-
-#---------- default config ----------
-
-defcfg = '''
-[COMMON]
-max_batch_down = 65536
-max_queue_time = 10
-target_requests_outstanding = 3
-http_timeout = 30
-http_timeout_grace = 5
-max_requests_outstanding = 6
-max_batch_up = 4000
-http_retry = 5
-port = 80
-vroutes = ''
-ifname_client = hippo%%d
-ifname_server = shippo%%d
-max_clock_skew = 300
-
-#[server] or [<client>] overrides
-ipif = userv root ipif %(local)s,%(peer)s,%(mtu)s,slip,%(ifname)s %(rnets)s
-
-# relating to virtual network
-mtu = 1500
-
-# addrs = 127.0.0.1 ::1
-# url
-
-# relating to virtual network
-vvnetwork = 172.24.230.192
-# vnetwork = <prefix>/<len>
-# vaddr = <ipaddr>
-# vrelay = <ipaddr>
-
-
-# [<client-ip4-or-ipv6-address>]
-# secret = <secret> # used by both, must match
-
-[LIMIT]
-max_batch_down = 262144
-max_queue_time = 121
-http_timeout = 121
-target_requests_outstanding = 10
-'''
-
-# these need to be defined here so that they can be imported by import *
-cfg = ConfigParser(strict=False)
-optparser = OptionParser()
-
-_mimetrans = bytes.maketrans(b'-'+slip.esc, slip.esc+b'-')
-def mime_translate(s):
- # SLIP-encoded packets cannot contain ESC ESC.
- # Swap `-' and ESC. The result cannot contain `--'
- return s.translate(_mimetrans)
-
-class ConfigResults:
- def __init__(self):
- pass
- def __repr__(self):
- return 'ConfigResults('+repr(self.__dict__)+')'
-
-def log_discard(packet, iface, saddr, daddr, why):
- log_debug(DBG.DROP,
- 'discarded packet [%s] %s -> %s: %s' % (iface, saddr, daddr, why),
- d=packet)
-
-#---------- packet parsing ----------
-
-def packet_addrs(packet):
- version = packet[0] >> 4
- if version == 4:
- addrlen = 4
- saddroff = 3*4
- factory = ipaddress.IPv4Address
- elif version == 6:
- addrlen = 16
- saddroff = 2*4
- factory = ipaddress.IPv6Address
- else:
- raise ValueError('unsupported IP version %d' % version)
- saddr = factory(packet[ saddroff : saddroff + addrlen ])
- daddr = factory(packet[ saddroff + addrlen : saddroff + addrlen*2 ])
- return (saddr, daddr)
-
-#---------- address handling ----------
-
-def ipaddr(input):
- try:
- r = ipaddress.IPv4Address(input)
- except AddressValueError:
- r = ipaddress.IPv6Address(input)
- return r
-
-def ipnetwork(input):
- try:
- r = ipaddress.IPv4Network(input)
- except NetworkValueError:
- r = ipaddress.IPv6Network(input)
- return r
-
-#---------- ipif (SLIP) subprocess ----------
-
-class SlipStreamDecoder():
- def __init__(self, desc, on_packet, mtu):
- self._buffer = b''
- self._on_packet = on_packet
- self._desc = desc
- self._mtu = mtu
- self._log('__init__')
-
- def _log(self, msg, **kwargs):
- log_debug(DBG.SLIP_FULL, 'slip %s: %s' % (self._desc, msg), **kwargs)
-
- def inputdata(self, data):
- self._log('inputdata', d=data)
- data = self._buffer + data
- self._buffer = b''
- packets = slip.decode(data, True)
- self._buffer = packets.pop()
- for packet in packets:
- self._maybe_packet(packet)
- self._log('bufremain', d=self._buffer)
-
- def _maybe_packet(self, packet):
- self._log('maybepacket', d=packet)
- if len(packet) and len(packet) <= self._mtu:
- self._on_packet(packet)
-
- def flush(self):
- self._log('flush')
- data = self._buffer
- self._buffer = b''
- packets = slip.decode(data)
- assert(len(packets) == 1)
- self._maybe_packet(packets[0])
-
-class _IpifProcessProtocol(twisted.internet.protocol.ProcessProtocol):
- def __init__(self, router, mtu):
- self._router = router
- self._decoder = SlipStreamDecoder('ipif', self.slip_on_packet, mtu)
- def connectionMade(self): pass
- def outReceived(self, data):
- self._decoder.inputdata(data)
- def slip_on_packet(self, packet):
- (saddr, daddr) = packet_addrs(packet)
- if saddr.is_link_local or daddr.is_link_local:
- log_discard(packet, 'ipif', saddr, daddr, 'link-local')
- return
- self._router(packet, saddr, daddr)
- def processEnded(self, status):
- status.raiseException()
-
-def start_ipif(command, router, mtu):
- ipif = _IpifProcessProtocol(router, mtu)
- reactor.spawnProcess(ipif,
- '/bin/sh',['sh','-xc', command],
- childFDs={0:'w', 1:'r', 2:2},
- env=None)
- return ipif
-
-def queue_inbound(ipif, packet):
- log_debug(DBG.FLOW, "queue_inbound", d=packet)
- ipif.transport.write(slip.delimiter)
- ipif.transport.write(slip.encode(packet))
- ipif.transport.write(slip.delimiter)
-
-#---------- packet queue ----------
-
-class PacketQueue():
- def __init__(self, desc, max_queue_time):
- self._desc = desc
- assert(desc + '')
- self._max_queue_time = max_queue_time
- self._pq = collections.deque() # packets
-
- def _log(self, dflag, msg, **kwargs):
- log_debug(dflag, self._desc+' pq: '+msg, **kwargs)
-
- def append(self, packet):
- self._log(DBG.QUEUE, 'append', d=packet)
- self._pq.append((time.monotonic(), packet))
-
- def nonempty(self):
- self._log(DBG.QUEUE, 'nonempty ?')
- while True:
- try: (queuetime, packet) = self._pq[0]
- except IndexError:
- self._log(DBG.QUEUE, 'nonempty ? empty.')
- return False
-
- age = time.monotonic() - queuetime
- if age > self._max_queue_time:
- # strip old packets off the front
- self._log(DBG.QUEUE, 'dropping (old)', d=packet)
- self._pq.popleft()
- continue
-
- self._log(DBG.QUEUE, 'nonempty ? nonempty.')
- return True
-
- def process(self, sizequery, moredata, max_batch):
- # sizequery() should return size of batch so far
- # moredata(s) should add s to batch
- self._log(DBG.QUEUE, 'process...')
- while True:
- try: (dummy, packet) = self._pq[0]
- except IndexError:
- self._log(DBG.QUEUE, 'process... empty')
- break
-
- self._log(DBG.QUEUE_CTRL, 'process... packet', d=packet)
-
- encoded = slip.encode(packet)
- sofar = sizequery()
-
- self._log(DBG.QUEUE_CTRL,
- 'process... (sofar=%d, max=%d) encoded' % (sofar, max_batch),
- d=encoded)
-
- if sofar > 0:
- if sofar + len(slip.delimiter) + len(encoded) > max_batch:
- self._log(DBG.QUEUE_CTRL, 'process... overflow')
- break
- moredata(slip.delimiter)
-
- moredata(encoded)
- self._pq.popleft()
-
-#---------- error handling ----------
-
-_crashing = False
-
-def crash(err):
- global _crashing
- _crashing = True
- print('========== CRASH ==========', err,
- '===========================', file=sys.stderr)
- try: reactor.stop()
- except twisted.internet.error.ReactorNotRunning: pass
-
-def crash_on_defer(defer):
- defer.addErrback(lambda err: crash(err))
-
-def crash_on_critical(event):
- if event.get('log_level') >= LogLevel.critical:
- crash(twisted.logger.formatEvent(event))
-
-#---------- authentication tokens ----------
-
-_authtoken_digest = hashlib.sha256
-
-def _authtoken_time():
- return int(time.time())
-
-def _authtoken_hmac(secret, hextime):
- return hmac.new(secret, hextime, _authtoken_digest).digest()
-
-def authtoken_make(secret):
- hextime = ('%x' % _authtoken_time()).encode('ascii')
- mac = _authtoken_hmac(secret, hextime)
- return hextime + b' ' + base64.b64encode(mac)
-
-def authtoken_check(secret, token, maxskew):
- (hextime, theirmac64) = token.split(b' ')
- now = _authtoken_time()
- then = int(hextime, 16)
- skew = then - now;
- if (abs(skew) > maxskew):
- raise ValueError('too much clock skew (client %ds ahead)' % skew)
- theirmac = base64.b64decode(theirmac64)
- ourmac = _authtoken_hmac(secret, hextime)
- if not hmac.compare_digest(theirmac, ourmac):
- raise ValueError('invalid token (wrong secret?)')
- pass
-
-#---------- config processing ----------
-
-def _cfg_process_putatives():
- servers = { }
- clients = { }
- # maps from abstract object to canonical name for cs's
-
- def putative(cmap, abstract, canoncs):
- try:
- current_canoncs = cmap[abstract]
- except KeyError:
- pass
- else:
- assert(current_canoncs == canoncs)
- cmap[abstract] = canoncs
-
- server_pat = r'[-.0-9A-Za-z]+'
- client_pat = r'[.:0-9a-f]+'
- server_re = regexp.compile(server_pat)
- serverclient_re = regexp.compile(
- server_pat + r' ' + '(?:' + client_pat + '|LIMIT)')
-
- for cs in cfg.sections():
- def dbg(m):
- log_debug_config('putatives: section [%s] %s' % (cs, m))
-
- def log_ignore(why):
- dbg('X ignore: %s' % (why))
- print('warning: ignoring config section [%s] (%s)' % (cs, why),
- file=sys.stderr)
-
- if cs == 'LIMIT' or cs == 'COMMON':
- # plan A "[LIMIT]" or "[COMMON]"
- dbg('A ignore')
- continue
-
- try:
- # plan B "[<client>]" part 1
- ci = ipaddr(cs)
- except AddressValueError:
-
- if server_re.fullmatch(cs):
- # plan C "[<servername>]"
- dbg('C <server>')
- putative(servers, cs, cs)
- continue
-
- if serverclient_re.fullmatch(cs):
- # plan D "[<servername> <client>]" part 1
- (pss,pcs) = cs.split(' ')
-
- if pcs == 'LIMIT':
- # plan E "[<servername> LIMIT]"
- dbg('E <server> LIMIT')
- continue
-
- try:
- # plan D "[<servername> <client>]" part 2
- ci = ipaddr(pcs)
- except AddressValueError:
- # plan F branch 1 "[<some thing we do not understand>]"
- log_ignore('bad-addr')
- continue
-
- else: # no AddressValueError
- # plan D "[<servername> <client>]" part 3
- dbg('D <server> <client>')
- putative(clients, ci, pcs)
- putative(servers, pss, pss)
- continue
- else:
- # plan F branch 2 "[<some thing we do not understand>]"
- log_ignore('nomatch '+ repr(serverclient_re))
-
- else: # no AddressValueError
- # plan B "[<client>" part 2
- dbg('B <client>')
- putative(clients, ci, cs)
- continue
-
- return (servers, clients)
-
-def cfg_process_general(c, ss):
- c.mtu = cfg1getint(ss, 'mtu')
-
-def cfg_process_saddrs(c, ss):
- class ServerAddr():
- def __init__(self, port, addrspec):
- self.port = port
- # also self.addr
- try:
- self.addr = ipaddress.IPv4Address(addrspec)
- self._endpointfactory = twisted.internet.endpoints.TCP4ServerEndpoint
- self._inurl = b'%s'
- except AddressValueError:
- self.addr = ipaddress.IPv6Address(addrspec)
- self._endpointfactory = twisted.internet.endpoints.TCP6ServerEndpoint
- self._inurl = b'[%s]'
- def make_endpoint(self):
- return self._endpointfactory(reactor, self.port,
- interface= '%s' % self.addr)
- def url(self):
- url = b'http://' + (self._inurl % str(self.addr).encode('ascii'))
- if self.port != 80: url += b':%d' % self.port
- url += b'/'
- return url
- def __repr__(self):
- return 'ServerAddr'+repr((self.port,self.addr))
-
- c.port = cfg1getint(ss,'port')
- c.saddrs = [ ]
- for addrspec in cfg1get(ss, 'addrs').split():
- sa = ServerAddr(c.port, addrspec)
- c.saddrs.append(sa)
-
-def cfg_process_vnetwork(c, ss):
- c.vnetwork = ipnetwork(cfg1get(ss,'vnetwork'))
- if c.vnetwork.num_addresses < 3 + 2:
- raise ValueError('vnetwork needs at least 2^3 addresses')
-
-def cfg_process_vaddr(c, ss):
- try:
- c.vaddr = ipaddr(cfg1get(ss,'vaddr'))
- except NoOptionError:
- cfg_process_vnetwork(c, ss)
- c.vaddr = next(c.vnetwork.hosts())
-
-def cfg_search_section(key,sections):
- for section in sections:
- if cfg.has_option(section, key):
- return section
- raise NoOptionError(key, repr(sections))
-
-def cfg_get_raw(*args, **kwargs):
- # for passing to cfg_search
- return cfg.get(*args, raw=True, **kwargs)
-
-def cfg_search(getter,key,sections):
- section = cfg_search_section(key,sections)
- return getter(section, key)
-
-def cfg1get(section,key, getter=cfg.get,**kwargs):
- section = cfg_search_section(key,[section,'COMMON'])
- return getter(section,key,**kwargs)
-
-def cfg1getint(section,key, **kwargs):
- return cfg1get(section,key, getter=cfg.getint,**kwargs);
-
-def cfg_process_client_limited(cc,ss,sections,key):
- val = cfg_search(cfg1getint, key, sections)
- lim = cfg_search(cfg1getint, key, ['%s LIMIT' % ss, 'LIMIT'])
- cc.__dict__[key] = min(val,lim)
-
-def cfg_process_client_common(cc,ss,cs,ci):
- # returns sections to search in, iff secret is defined, otherwise None
- cc.ci = ci
-
- sections = ['%s %s' % (ss,cs),
- cs,
- ss,
- 'COMMON']
-
- try: pwsection = cfg_search_section('secret', sections)
- except NoOptionError: return None
-
- pw = cfg1get(pwsection, 'secret')
- cc.secret = pw.encode('utf-8')
-
- cfg_process_client_limited(cc,ss,sections,'target_requests_outstanding')
- cfg_process_client_limited(cc,ss,sections,'http_timeout')
-
- return sections
-
-def cfg_process_ipif(c, sections, varmap):
- for d, s in varmap:
- try: v = getattr(c, s)
- except AttributeError: continue
- setattr(c, d, v)
- for d in ('mtu',):
- v = cfg_search(cfg1getint, d, sections)
- setattr(c, d, v)
-
- #print('CFGIPIF',repr((varmap, sections, c.__dict__)),file=sys.stderr)
-
- section = cfg_search_section('ipif', sections)
- c.ipif_command = cfg1get(section,'ipif', vars=c.__dict__)
-
-#---------- startup ----------
-
-def log_debug_config(m):
- if not DBG.CONFIG in debug_set: return
- print('DBG.CONFIG:', m)
-
-def common_startup(process_cfg):
- # calls process_cfg(putative_clients, putative_servers)
-
- # ConfigParser hates #-comments after values
- trailingcomments_re = regexp.compile(r'#.*')
- cfg.read_string(trailingcomments_re.sub('', defcfg))
- need_defcfg = True
-
- def readconfig(pathname, mandatory=True):
- def log(m, p=pathname):
- if not DBG.CONFIG in debug_set: return
- log_debug_config('%s: %s' % (m, p))
-
- try:
- files = os.listdir(pathname)
-
- except FileNotFoundError:
- if mandatory: raise
- log('skipped')
- return
-
- except NotADirectoryError:
- cfg.read(pathname)
- log('read file')
- return
-
- # is a directory
- log('directory')
- re = regexp.compile('[^-A-Za-z0-9_]')
- for f in os.listdir(pathname):
- if re.search(f): continue
- subpath = pathname + '/' + f
- try:
- os.stat(subpath)
- except FileNotFoundError:
- log('entry skipped', subpath)
- continue
- cfg.read(subpath)
- log('entry read', subpath)
-
- def oc_config(od,os, value, op):
- nonlocal need_defcfg
- need_defcfg = False
- readconfig(value)
-
- def oc_extra_config(od,os, value, op):
- readconfig(value)
-
- def read_defconfig():
- readconfig('/etc/hippotat/config.d', False)
- readconfig('/etc/hippotat/secrets.d', False)
- readconfig('/etc/hippotat/master.cfg', False)
-
- def oc_defconfig(od,os, value, op):
- nonlocal need_defcfg
- need_defcfg = False
- read_defconfig(value)
-
- def dfs_less_detailed(dl):
- return [df for df in DBG.iterconstants() if df <= dl]
-
- def ds_default(od,os,dl,op):
- global debug_set
- debug_set.clear
- debug_set |= set(dfs_less_detailed(debug_def_detail))
-
- def ds_select(od,os, spec, op):
- for it in spec.split(','):
-
- if it.startswith('-'):
- mutator = debug_set.discard
- it = it[1:]
- else:
- mutator = debug_set.add
-
- if it == '+':
- dfs = DBG.iterconstants()
-
- else:
- if it.endswith('+'):
- mapper = dfs_less_detailed
- it = it[0:len(it)-1]
- else:
- mapper = lambda x: [x]
-
- try:
- dfspec = DBG.lookupByName(it)
- except ValueError:
- optparser.error('unknown debug flag %s in --debug-select' % it)
-
- dfs = mapper(dfspec)
-
- for df in dfs:
- mutator(df)
-
- optparser.add_option('-D', '--debug',
- nargs=0,
- action='callback',
- help='enable default debug (to stdout)',
- callback= ds_default)
-
- optparser.add_option('--debug-select',
- nargs=1,
- type='string',
- metavar='[-]DFLAG[+]|[-]+,...',
- help=
-'''enable (`-': disable) each specified DFLAG;
-`+': do same for all "more interesting" DFLAGSs;
-just `+': all DFLAGs.
- DFLAGS: ''' + ' '.join([df.name for df in DBG.iterconstants()]),
- action='callback',
- callback= ds_select)
-
- optparser.add_option('-c', '--config',
- nargs=1,
- type='string',
- metavar='CONFIGFILE',
- dest='configfile',
- action='callback',
- callback= oc_config)
-
- optparser.add_option('--extra-config',
- nargs=1,
- type='string',
- metavar='CONFIGFILE',
- dest='configfile',
- action='callback',
- callback= oc_extra_config)
-
- optparser.add_option('--default-config',
- action='callback',
- callback= oc_defconfig)
-
- (opts, args) = optparser.parse_args()
- if len(args): optparser.error('no non-option arguments please')
-
- if need_defcfg:
- read_defconfig()
-
- try:
- (pss, pcs) = _cfg_process_putatives()
- process_cfg(opts, pss, pcs)
- except (configparser.Error, ValueError):
- traceback.print_exc(file=sys.stderr)
- print('\nInvalid configuration, giving up.', file=sys.stderr)
- sys.exit(12)
-
-
- #print('X', debug_set, file=sys.stderr)
-
- log_formatter = twisted.logger.formatEventAsClassicLogText
- stdout_obs = twisted.logger.FileLogObserver(sys.stdout, log_formatter)
- stderr_obs = twisted.logger.FileLogObserver(sys.stderr, log_formatter)
- pred = twisted.logger.LogLevelFilterPredicate(LogLevel.error)
- stdsomething_obs = twisted.logger.FilteringLogObserver(
- stderr_obs, [pred], stdout_obs
- )
- global file_log_observer
- file_log_observer = twisted.logger.FilteringLogObserver(
- stdsomething_obs, [LogNotBoringTwisted()]
- )
- #log_observer = stdsomething_obs
- twisted.logger.globalLogBeginner.beginLoggingTo(
- [ file_log_observer, crash_on_critical ]
- )
-
-def common_run():
- log_debug(DBG.INIT, 'entering reactor')
- if not _crashing: reactor.run()
- print('ENDED', file=sys.stderr)
- sys.exit(16)
+++ /dev/null
-# -*- python -*-
-#
-# Hippotat - Asinine IP Over HTTP program
-# hippotatlib/ownsource.py - Automatic source code provision (AGPL compliance)
-#
-# Copyright 2017 Ian Jackson
-#
-# AGPLv3+ + CAFv2+
-#
-# This program is free software: you can redistribute it and/or
-# modify it under the terms of the GNU Affero General Public
-# License as published by the Free Software Foundation, either
-# version 3 of the License, or (at your option) any later version,
-# with the "CAF Login Exception" as published by Ian Jackson
-# (version 2, or at your option any later version) as an Additional
-# Permission.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public
-# License and the CAF Login Exception along with this program, in
-# the file AGPLv3+CAFv2. If not, email Ian Jackson
-# <ijackson@chiark.greenend.org.uk>.
-
-
-import os
-import sys
-import fnmatch
-import stat
-import subprocess
-import tempfile
-import shutil
-
-try: import debian.deb822
-except ImportError: pass
-
-class SourceShipmentPreparer():
- def __init__(s, destdir):
- # caller may modify, and should read after calling generate()
- s.output_names = ['srcbomb.tar.gz', 'srcpkgsbomb.tar']
- s.output_paths = [None,None] # alternatively caller may read this
- # defaults, caller can modify after creation
- s.logger = lambda m: print('SourceShipmentPreparer',m)
- s.src_filter = s.src_filter_glob
- s.src_package_globs = ['!/usr/local/*', '/usr*']
- s.src_filter_globs = ['!/etc/*']
- s.src_likeparent = s.src_likeparent_git
- s.src_direxcludes = s.src_direxcludes_git
- s.report_from_packages = s.report_from_packages_debian
- s.cwd = os.getcwd()
- s.find_rune_base = "find -type f -perm -004 \! -path '*/tmp/*'"
- s.ignores = ['*~', '*.bak', '*.tmp', '#*#', '__pycache__',
- '[0-9][0-9][0-9][0-9]-src.tar']
- s.rune_shell = ['/bin/bash', '-ec']
- s.show_pathnames = True
- s.download_packages = True
- s.stream_stderr = sys.stderr
- s.stream_debug = open('/dev/null','w')
- s.rune_cpio = r'''
- set -o pipefail
- (
- %s
- # ^ by default, is find ... -print0
- ) | (
- cpio -Hustar -o --quiet -0 -R 1000:1000 || \
- cpio -Hustar -o --quiet -0
- )
- '''
- s.rune_portmanteau = r'''
- GZIP=-1 tar zcf - "$@"
- '''
- s.rune_portmanteau_uncompressed = r'''
- tar cf - "$@"
- '''
- s.manifest_name='0000-MANIFEST.txt'
- # private
- s._destdir = destdir
- s._outcounter = 0
- s._manifest = []
- s._dirmap = { }
- s._package_files = { } # map filename => infol
- s._packages_path = os.path.join(s._destdir, 'packages')
- s._package_sources = []
-
- def thing_matches_globs(s, thing, globs):
- for pat in globs:
- negate = pat.startswith('!')
- if negate: pat = pat[1:]
- if fnmatch.fnmatch(thing, pat):
- return not negate
- return negate
-
- def src_filter_glob(s, src): # default s.src_filter
- return s.thing_matches_globs(src, s.src_filter_globs)
-
- def src_direxcludes_git(s, d):
- try:
- excl = open(os.path.join(d, '.gitignore'))
- except FileNotFoundError:
- return []
- r = []
- for l in excl:
- l = l.strip()
- if l.startswith('#'): next
- if not len(l): next
- r.append(l)
- return r
-
- def src_likeparent_git(s, src):
- try:
- os.stat(os.path.join(src, '.git/.'))
- except FileNotFoundError:
- return False
- else:
- return True
-
- def src_parentfinder(s, src, infol): # callers may monkey-patch away
- for deref in (False,True):
- xinfo = []
-
- search = src
- if deref:
- search = os.path.realpath(search)
-
- def ascend():
- nonlocal search
- xinfo.append(os.path.basename(search))
- search = os.path.dirname(search)
-
- try:
- stab = os.lstat(search)
- except FileNotFoundError:
- return
- if stat.S_ISREG(stab.st_mode):
- ascend()
-
- while not os.path.ismount(search):
- if s.src_likeparent(search):
- xinfo.reverse()
- if len(xinfo): infol.append('want=' + os.path.join(*xinfo))
- return search
-
- ascend()
-
- # no .git found anywhere
- return src
-
- def path_prenormaliser(s, d, infol): # callers may monkey-patch away
- return os.path.join(s.cwd, os.path.abspath(d))
-
- def srcdir_find_rune(s, d):
- script = s.find_rune_base
- ignores = s.ignores + s.output_names + [s.manifest_name]
- ignores += s.src_direxcludes(d)
- for excl in ignores:
- assert("'" not in excl)
- script += r" \! -name '%s'" % excl
- script += r" \! -path '*/%s/*'" % excl
- script += ' -print0'
- return script
-
- def manifest_append(s, name, infol):
- s._manifest.append({ 'file':name, 'info':' '.join(infol) })
-
- def manifest_append_absentfile(s, name, infol):
- s._manifest.append({ 'file_print':name, 'info':' '.join(infol) })
-
- def new_output_name(s, nametail, infol):
- s._outcounter += 1
- name = '%04d-%s' % (s._outcounter, nametail)
- s.manifest_append(name, infol)
- return name
-
- def open_output_fh(s, name, mode):
- return open(os.path.join(s._destdir, name), mode)
-
- def src_dir(s, d, infol):
- try: name = s._dirmap[d]
- except KeyError: pass
- else:
- s.manifest_append(name, infol)
- return
-
- if s.show_pathnames: infol.append(d)
- find_rune = s.srcdir_find_rune(d)
- total_rune = s.rune_cpio % find_rune
-
- name = s.new_output_name('src.tar', infol)
- s._dirmap[d] = name
- fh = s.open_output_fh(name, 'wb')
-
- s.logger('packing up into %s: %s (because %s)' %
- (name, d, ' '.join(infol)))
-
- subprocess.run(s.rune_shell + [total_rune],
- cwd=d,
- stdin=subprocess.DEVNULL,
- stdout=fh,
- restore_signals=True,
- check=True)
- fh.close()
-
- def src_indir(s, d, infol):
- d = s.path_prenormaliser(d, infol)
- if not s.src_filter(d): return
-
- d = s.src_parentfinder(d, infol)
- if d is None: return
- s.src_dir(d, infol)
-
- def report_from_packages_debian(s, files):
- dpkg_S_in = tempfile.TemporaryFile(mode='w+')
- for (file, infols) in files.items():
- assert('\n' not in file)
- dpkg_S_in.write(file)
- dpkg_S_in.write('\0')
- dpkg_S_in.seek(0)
- cmdl = ['xargs','-0r','dpkg','-S','--']
- dpkg_S = subprocess.Popen(cmdl,
- cwd='/',
- stdin=dpkg_S_in,
- stdout=subprocess.PIPE,
- stderr=sys.stderr,
- close_fds=False)
- dpkg_show_in = tempfile.TemporaryFile(mode='w+')
- pkginfos = { }
- for l in dpkg_S.stdout:
- l = l.strip(b'\n').decode('utf-8')
- (pkgs, fname) = l.split(': ',1)
- pks = pkgs.split(', ')
- for pk in pks:
- pkginfos.setdefault(pk,{'files':[]})['files'].append(fname)
- print(pk, file=dpkg_show_in)
- assert(dpkg_S.wait() == 0)
- dpkg_show_in.seek(0)
- cmdl = ['xargs','-r','dpkg-query',
- r'-f${binary:Package}\t${Package}\t${Architecture}\t${Version}\t${source:Package}\t${source:Version}\t${source:Upstream-Version}\n',
- '--show','--']
- dpkg_show = subprocess.Popen(cmdl,
- cwd='/',
- stdin=dpkg_show_in,
- stdout=subprocess.PIPE,
- stderr=sys.stderr,
- close_fds=False)
- for l in dpkg_show.stdout:
- l = l.strip(b'\n').decode('utf-8')
- (pk,p,a,v,sp,sv,suv) = l.split('\t')
- pkginfos[pk]['binary'] = p
- pkginfos[pk]['arch'] = a
- pkginfos[pk]['version'] = v
- pkginfos[pk]['source'] = sp
- pkginfos[pk]['sourceversion'] = sv
- pkginfos[pk]['sourceupstreamversion'] = sv
- assert(dpkg_show.wait() == 0)
- for pk in sorted(pkginfos.keys()):
- pi = pkginfos[pk]
- debfname = '%s_%s_%s.deb' % (pi['binary'], pi['version'], pi['arch'])
- dscfname = '%s_%s.dsc' % (pi['source'], pi['sourceversion'])
- s.manifest_append_absentfile(dscfname, [debfname])
- s.logger('mentioning %s and %s because %s' %
- (dscfname, debfname, pi['files'][0]))
- for fname in pi['files']:
- infol = files[fname]
- if s.show_pathnames: infol = infol + ['loaded='+fname]
- s.manifest_append_absentfile(' \t' + debfname, infol)
-
- if s.download_packages:
- try: os.mkdir(s._packages_path)
- except FileExistsError: pass
-
- cmdl = ['apt-get','--download-only','source',
- '%s=%s' % (pi['source'], pi['sourceversion'])]
- subprocess.run(cmdl,
- cwd=s._packages_path,
- stdin=subprocess.DEVNULL,
- stdout=s.stream_debug,
- stderr=s.stream_stderr,
- restore_signals=True,
- check=True)
-
- s._package_sources.append(dscfname)
- dsc = debian.deb822.Dsc(open(s._packages_path + '/' + dscfname))
- for indsc in dsc['Files']:
- s._package_sources.append(indsc['name'])
-
- def thing_ought_packaged(s, fname):
- return s.thing_matches_globs(fname, s.src_package_globs)
-
- def src_file_packaged(s, fname, infol):
- s._package_files.setdefault(fname,[]).extend(infol)
-
- def src_file(s, fname, infol):
- def fngens():
- yield (infol, fname)
- infol_copy = infol.copy()
- yield (infol_copy, s.path_prenormaliser(fname, infol_copy))
- yield (infol, os.path.realpath(fname))
-
- for (tinfol, tfname) in fngens():
- if s.thing_ought_packaged(tfname):
- s.src_file_packaged(tfname, tinfol)
- return
-
- s.src_indir(fname, infol)
-
- def src_argv0(s, program, infol):
- s.src_file(program, infol)
-
- def src_syspath(s, fname, infol):
- if s.thing_ought_packaged(fname): return
- s.src_indir(fname, infol)
-
- def src_module(s, m, infol):
- try: fname = m.__file__
- except AttributeError: return
- infol.append('module='+m.__name__)
-
- if s.thing_ought_packaged(fname):
- s.src_file_packaged(fname, infol)
- else:
- s.src_indir(fname, infol)
-
- def srcs_allitems(s, dirs=sys.path):
- s.logger('allitems')
- s.src_argv0(sys.argv[0], ['argv[0]'])
- for d in sys.path:
- s.src_syspath(d, ['sys.path'])
- for m in sys.modules.values():
- s.src_module(m, ['sys.modules'])
- s.report_from_packages(s._package_files)
- s.logger('allitems done')
-
- def _mk_portmanteau(s, ix, rune, cwd, files):
- output_name = s.output_names[ix]
- s.logger('making portmanteau %s' % output_name)
- output_path = os.path.join(s._destdir, output_name)
- subprocess.run(s.rune_shell + [ rune, 'x' ] + files,
- cwd=cwd,
- stdin=subprocess.DEVNULL,
- stdout=open(output_path, 'wb'),
- restore_signals=True,
- check=True)
- s.output_paths[ix] = output_path
-
- def mk_inner_portmanteau(s):
- outputs = [s.manifest_name]
- outputs_done = { }
- mfh = s.open_output_fh(s.manifest_name,'w')
- for me in s._manifest:
- try: fname = me['file']
- except KeyError: fname = me.get('file_print','')
- else:
- try: outputs_done[fname]
- except KeyError:
- outputs.append(fname)
- outputs_done[fname] = 1
- print('%s\t%s' % (fname, me['info']), file=mfh)
- mfh.close()
-
- s._mk_portmanteau(0, s.rune_portmanteau,
- s._destdir, outputs)
-
- def mk_packages_portmanteau(s):
- if not s.download_packages: return
- s._mk_portmanteau(1, s.rune_portmanteau_uncompressed,
- s._packages_path, s._package_sources)
-
- def generate(s):
- s.srcs_allitems()
- s.mk_inner_portmanteau()
- s.mk_packages_portmanteau()
- s.logger('portmanteau ready in %s %s' % tuple(s.output_paths))
+++ /dev/null
-# -*- python -*-
-#
-# Hippotat - Asinine IP Over HTTP program
-# hippotatlib/slip.py - SLIP handling
-#
-# Copyright 2017 Ian Jackson
-#
-# GPLv3+
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program, in the file GPLv3. If not,
-# see <http://www.gnu.org/licenses/>.
-
-
-end = b'\300'
-esc = b'\333'
-esc_end = b'\334'
-esc_esc = b'\335'
-delimiter = end
-
-def encode(packet):
- return (packet
- .replace(esc, esc + esc_esc)
- .replace(end, esc + esc_end))
-
-def decode(data, keep_tail=False):
- #print('DECODE ', repr(data))
- out = []
- inp = data.split(end)
- tail = []
- if keep_tail:
- tail.append(inp.pop())
- for packet in inp:
- pdata = b''
- while True:
- eix = packet.find(esc)
- if eix == -1:
- pdata += packet
- break
- #print('ESC ', repr((pdata, packet, eix)))
- pdata += packet[0 : eix]
- ck = packet[eix+1]
- #print('ESC... %o' % ck)
- if ck == esc_esc[0]: pdata += esc
- elif ck == esc_end[0]: pdata += end
- else: raise ValueError('invalid SLIP escape 0%o %#x' % (ck, ck))
- packet = packet[eix+2 : ]
- out.append(pdata)
- #print('DECODED ', repr(out))
- out += tail
- return out
-# -*- python -*-
-
~ian / hippotat.git / commitdiff