From: Ian Jackson Date: Sun, 25 Sep 2022 15:52:02 +0000 (+0100) Subject: old-python: Delete python code X-Git-Tag: hippotat/1.0.0~53 X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~ian/git?a=commitdiff_plain;h=19394d5b5419e4c26e6b6fbb51a58f798ba7c095;p=hippotat.git old-python: Delete python code We don't need this any more. Signed-off-by: Ian Jackson --- diff --git a/old-python/hippotat b/old-python/hippotat deleted file mode 100755 index d7a4daa..0000000 --- a/old-python/hippotat +++ /dev/null @@ -1,309 +0,0 @@ -#!/usr/bin/python3 -# -# Hippotat - Asinine IP Over HTTP program -# ./hippotat - client main program -# -# Copyright 2017 Ian Jackson -# -# GPLv3+ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program, in the file GPLv3. If not, -# see . - -#@ import sys; sys.path.append('@PYBUILD_INSTALL_DIR@') -from hippotatlib import * - -import twisted.web -import twisted.web.client -import urllib.parse - -import io - -class GeneralResponseConsumer(twisted.internet.protocol.Protocol): - def __init__(self, cl, req, resp, desc): - self._cl = cl - self._req = req - self._resp = resp - self._desc = desc - - def _log(self, dflag, msg, **kwargs): - self._cl.log(dflag, '%s: %s' % (self._desc, msg), idof=self._req, **kwargs) - - def connectionMade(self): - self._log(DBG.HTTP_CTRL, 'connectionMade') - - def connectionLostOK(self, reason): - return (reason.check(twisted.web.client.ResponseDone) or - reason.check(twisted.web.client.PotentialDataLoss)) - # twisted.web.client.PotentialDataLoss is an entirely daft - # exception. It will occur every time if the origin server does - # not provide a Content-Length. (hippotatd does, of course, but - # the HTTP transaction might be proxied.) - -class ResponseConsumer(GeneralResponseConsumer): - def __init__(self, cl, req, resp): - super().__init__(cl, req, resp, 'RC') - ssddesc = '[%s] %s' % (id(req), self._desc) - self._ssd = SlipStreamDecoder(ssddesc, partial(queue_inbound, cl.ipif), - cl.c.mtu) - self._log(DBG.HTTP_CTRL, '__init__') - - def dataReceived(self, data): - self._log(DBG.HTTP, 'dataReceived', d=data) - try: - self._ssd.inputdata(data) - except Exception as e: - self._handleexception() - - def connectionLost(self, reason): - reason_msg = 'connectionLost ' + str(reason) - self._log(DBG.HTTP_CTRL, reason_msg) - if not self.connectionLostOK(reason): - self._latefailure(reason_msg) - return - try: - self._log(DBG.HTTP, 'ResponseDone') - self._ssd.flush() - self._cl.req_fin(self._req) - except Exception as e: - self._handleexception() - self._cl.report_running() - - def _handleexception(self): - self._latefailure(traceback.format_exc()) - - def _latefailure(self, reason): - self._log(DBG.HTTP_CTRL, '_latefailure ' + str(reason)) - self._cl.req_err(self._req, reason) - -class ErrorResponseConsumer(GeneralResponseConsumer): - def __init__(self, cl, req, resp): - super().__init__(cl, req, resp, 'ERROR-RC') - self._m = b'' - try: - self._phrase = resp.phrase.decode('utf-8') - except Exception: - self._phrase = repr(resp.phrase) - self._log(DBG.HTTP_CTRL, '__init__ %d %s' % (resp.code, self._phrase)) - - def dataReceived(self, data): - self._log(DBG.HTTP_CTRL, 'dataReceived ' + repr(data)) - self._m += data - - def connectionLost(self, reason): - try: - mbody = self._m.decode('utf-8') - except Exception: - mbody = repr(self._m) - if not self.connectionLostOK(reason): - mbody += ' || ' + str(reason) - self._cl.req_err(self._req, - "FAILED %d %s | %s" - % (self._resp.code, self._phrase, mbody)) - -class Client(): - def __init__(cl, c,ss,cs): - cl.c = c - cl.outstanding = { } - cl.desc = '[%s %s] ' % (ss,cs) - cl.running_reported = False - cl.log_info('setting up') - - def log_info(cl, msg): - log.info(cl.desc + msg, dflag=False) - - def report_running(cl): - if not cl.running_reported: - cl.log_info('running OK') - cl.running_reported = True - - def log(cl, dflag, msg, **kwargs): - log_debug(dflag, cl.desc + msg, **kwargs) - - def log_outstanding(cl): - cl.log(DBG.CTRL_DUMP, 'OS %s' % cl.outstanding) - - def start(cl): - cl.queue = PacketQueue('up', cl.c.max_queue_time) - cl.agent = twisted.web.client.Agent( - reactor, connectTimeout = cl.c.http_timeout) - - def outbound(cl, packet, saddr, daddr): - #print('OUT ', saddr, daddr, repr(packet)) - cl.queue.append(packet) - cl.check_outbound() - - def req_ok(cl, req, resp): - cl.log(DBG.HTTP_CTRL, - 'req_ok %d %s %s' % (resp.code, repr(resp.phrase), str(resp)), - idof=req) - if resp.code == 200: - rc = ResponseConsumer(cl, req, resp) - else: - rc = ErrorResponseConsumer(cl, req, resp) - - resp.deliverBody(rc) - # now rc is responsible for calling req_fin - - def req_err(cl, req, err): - # called when the Deferred fails, or (if it completes), - # later, by ResponsConsumer or ErrorResponsConsumer - try: - cl.log(DBG.HTTP_CTRL, 'req_err ' + str(err), idof=req) - cl.running_reported = False - if isinstance(err, twisted.python.failure.Failure): - err = err.getTraceback() - print('%s[%#x] %s' % (cl.desc, id(req), err.strip('\n').replace('\n',' / ')), - file=sys.stderr) - if not isinstance(cl.outstanding[req], int): - raise RuntimeError('[%#x] previously %s' % - (id(req), cl.outstanding[req])) - cl.outstanding[req] = err - cl.log_outstanding() - reactor.callLater(cl.c.http_retry, partial(cl.req_fin, req)) - except Exception as e: - crash(traceback.format_exc() + '\n----- handling -----\n' + err) - - def req_fin(cl, req): - del cl.outstanding[req] - cl.log(DBG.HTTP_CTRL, 'req_fin OS=%d' % len(cl.outstanding), idof=req) - cl.check_outbound() - - def check_outbound(cl): - while True: - if len(cl.outstanding) >= cl.c.max_outstanding: - break - - if (not cl.queue.nonempty() and - len(cl.outstanding) >= cl.c.target_requests_outstanding): - break - - d = b'' - def moredata(s): nonlocal d; d += s - cl.queue.process((lambda: len(d)), - moredata, - cl.c.max_batch_up) - - d = mime_translate(d) - - token = authtoken_make(cl.c.secret) - - crlf = b'\r\n' - lf = b'\n' - mime = (b'--b' + crlf + - b'Content-Type: text/plain; charset="utf-8"' + crlf + - b'Content-Disposition: form-data; name="m"' + crlf + crlf + - str(cl.c.client) .encode('ascii') + crlf + - token + crlf + - str(cl.c.target_requests_outstanding) - .encode('ascii') + crlf + - str(cl.c.http_timeout) .encode('ascii') + crlf + - (( - b'--b' + crlf + - b'Content-Type: application/octet-stream' + crlf + - b'Content-Disposition: form-data; name="d"' + crlf + crlf + - d + crlf - ) if len(d) else b'') + - b'--b--' + crlf) - - #df = open('data.dump.dbg', mode='wb') - #df.write(mime) - #df.close() - # POST -use -c 'multipart/form-data; boundary="b"' http://localhost:8099/ . - -#@ import sys; sys.path.append('@PYBUILD_INSTALL_DIR@') -from hippotatlib import * - -import os -import tempfile -import atexit -import shutil -import subprocess - -import twisted.internet -from twisted.web.server import NOT_DONE_YET - -import twisted.web.static - -import hippotatlib.ownsource -from hippotatlib.ownsource import SourceShipmentPreparer - -#import twisted.web.server import Site -#from twisted.web.resource import Resource - -import syslog - -cleanups = [ ] - -clients = { } - -#---------- "router" ---------- - -def route(packet, iface, saddr, daddr): - def lt(dest): - log_debug(DBG.ROUTE, 'route: %s -> %s: %s' % (saddr,daddr,dest), d=packet) - try: dclient = clients[daddr] - except KeyError: dclient = None - if dclient is not None: - lt('client') - dclient.queue_outbound(packet) - elif daddr == c.vaddr or daddr not in c.vnetwork: - lt('inbound') - queue_inbound(ipif, packet) - elif daddr == c.vrelay: - lt('discard relay') - log_discard(packet, iface, saddr, daddr, 'relay') - else: - lt('discard no-client') - log_discard(packet, iface, saddr, daddr, 'no-client') - -#---------- client ---------- - -class Client(): - def __init__(self, ip, cc): - # instance data members - self._ip = ip - self.cc = cc - self._rq = collections.deque() # requests - self._pq = PacketQueue(str(ip), self.cc.max_queue_time) - - if ip not in c.vnetwork: - raise ValueError('client %s not in vnetwork' % ip) - - if ip in clients: - raise ValueError('multiple client cfg sections for %s' % ip) - clients[ip] = self - - self._log(DBG.INIT, 'new') - - def _log(self, dflag, msg, **kwargs): - log_debug(dflag, ('client %s: ' % self._ip)+msg, **kwargs) - - def process_arriving_data(self, d): - self._log(DBG.FLOW, "req data (enc'd)", d=d) - if not len(d): return - for packet in slip.decode(d): - (saddr, daddr) = packet_addrs(packet) - if saddr != self._ip: - raise ValueError('wrong source address %s' % saddr) - route(packet, self._ip, saddr, daddr) - - def _req_cancel(self, request): - self._log(DBG.HTTP_CTRL, 'cancel', idof=request) - try: request.finish() - except Exception: pass - - def _req_error(self, err, request): - self._log(DBG.HTTP_CTRL, 'error %s' % err, idof=request) - self._req_cancel(request) - - def queue_outbound(self, packet): - self._pq.append(packet) - self._check_outbound() - - def _req_fin(self, dummy, request, cl): - self._log(DBG.HTTP_CTRL, '_req_fin ' + repr(dummy), idof=request) - try: cl.cancel() - except twisted.internet.error.AlreadyCalled: pass - - def new_request(self, request): - request.setHeader('Content-Type','application/octet-stream') - cl = reactor.callLater(self.cc.http_timeout, self._req_cancel, request) - nf = request.notifyFinish() - nf.addErrback(self._req_error, request) - nf.addCallback(self._req_fin, request, cl) - self._rq.append((request,nf)) - self._check_outbound() - - def _req_write(self, req, d): - self._log(DBG.HTTP, 'req_write ', idof=req, d=d) - req.write(d) - - def _check_outbound(self): - log_debug(DBG.HTTP_CTRL, 'CHKO') - while True: - try: (request,nf) = self._rq[0] - except IndexError: request = None - if request and nf.called: - self._log(DBG.HTTP_CTRL, 'CHKO req finished, discard', idof=request) - self._rq.popleft() - continue - - if not self._pq.nonempty(): - # no packets, oh well - self._log(DBG.HTTP_CTRL, 'CHKO no packets, OUT-DONE', idof=request) - break - - if request is None: - # no request - self._log(DBG.HTTP_CTRL, 'CHKO no request, OUT-DONE', idof=request) - break - - self._log(DBG.HTTP_CTRL, 'CHKO processing', idof=request) - # request, and also some non-expired packets - self._pq.process((lambda: request.sentLength), - (lambda d: self._req_write(request, d)), - self.cc.max_batch_down) - - assert(request.sentLength) - self._rq.popleft() - request.finish() - self._log(DBG.HTTP, 'complete', idof=request) - # round again, looking for more to do - - while len(self._rq) > self.cc.target_requests_outstanding: - (request, nf) = self._rq.popleft() - self._log(DBG.HTTP, 'CHKO above target, returning empty', idof=request) - request.finish() - -def process_request(request, desca): - # find client, update config, etc. - metadata = request.args[b'm'][0] - metadata = metadata.split(b'\r\n') - (ci_s, token, tro, cto) = metadata[0:4] - desca['m[0,2:3]'] = [ci_s, tro, cto] - ci_s = ci_s.decode('utf-8') - tro = int(tro); desca['tro']= tro - cto = int(cto); desca['cto']= cto - ci = ipaddr(ci_s) - desca['ci'] = ci - cl = clients[ci] - authtoken_check(cl.cc.secret, token, cl.cc.max_clock_skew) - desca['pwok']=True - - if tro != cl.cc.target_requests_outstanding: - raise ValueError('tro must be %d' % cl.cc.target_requests_outstanding) - - if cto < cl.cc.http_timeout: - raise ValueError('cto must be >= %d' % cl.cc.http_timeout) - - try: - d = request.args[b'd'][0] - desca['d'] = d - desca['dlen'] = len(d) - except KeyError: - d = b'' - desca['dlen'] = None - - log_http(desca, 'processing', idof=id(request), d=d) - - d = mime_translate(d) - - cl.process_arriving_data(d) - cl.new_request(request) - -def log_http(desca, msg, **kwargs): - try: - kwargs['d'] = desca['d'] - del desca['d'] - except KeyError: - pass - log_debug(DBG.HTTP, msg + repr(desca), **kwargs) - -class NotStupidResource(twisted.web.resource.Resource): - # why this is not the default is a mystery! - def getChild(self, name, request): - if name == b'': return self - else: return twisted.web.resource.Resource.getChild(self, name, request) - -class IphttpResource(NotStupidResource): - def render_POST(self, request): - log_debug(DBG.HTTP_FULL, - 'req recv: ' + repr(request) + ' ' + repr(request.args), - idof=id(request)) - desca = {'d': None} - try: process_request(request, desca) - except Exception as e: - emsg = traceback.format_exc() - log_http(desca, 'RETURNING EXCEPTION ' + emsg) - request.setHeader('Content-Type','text/plain; charset="utf-8"') - request.setResponseCode(400) - return (emsg + ' # ' + repr(desca) + '\r\n').encode('utf-8') - log_debug(DBG.HTTP_CTRL, '...', idof=id(request)) - return NOT_DONE_YET - - # instantiator should set - # self.hippotat_sources = (source_names[0], source_names[1]) - def __init__(self): - self.hippotat_sources = [None, None] - super().__init__() - - def render_GET(self, request): - log_debug(DBG.HTTP, 'GET request') - s = 'hippotat\n' - (s0,s1) = self.hippotat_sources - if s0: - s += '

source\n' % s0 - if self.hippotat_sources[1]: - s += ('(and that of dependency packages)\n' % s1) - s += 'available' - else: - s += 'TESTING' - s += '' - return s.encode('utf-8') - -def start_http(): - resource = IphttpResource() - site = twisted.web.server.Site(resource) - - for sa in c.saddrs: - ep = sa.make_endpoint() - crash_on_defer(ep.listen(site)) - log_debug(DBG.INIT, 'listening on %s' % sa) - - td = tempfile.mkdtemp() - - def cleanup(): - try: shutil.rmtree(td) - except FileNotFoundError: pass - - cleanups.append(cleanup) - - ssp = SourceShipmentPreparer(td) - ssp.logger = partial(log_debug, DBG.OWNSOURCE) - if DBG.OWNSOURCE in debug_set: ssp.stream_debug = sys.stdout - ssp.download_packages = opts.ownsource >= 2 - if opts.ownsource >= 1: ssp.generate() - - for ix in (0,1): - bn = ssp.output_names[ix] - op = ssp.output_paths[ix] - if op is None: continue - resource.hippotat_sources[ix] = bn - subresource =twisted.web.static.File(op) - resource.putChild(bn.encode('utf-8'), subresource) - - reactor.callLater(0.1, (lambda: log.info('hippotatd started', dflag=False))) - -#---------- config and setup ---------- - -def process_cfg(_opts, putative_servers, putative_clients): - global opts - opts = _opts - - global c - c = ConfigResults() - try: c.server = cfg1get('SERVER','server') - except NoOptionError: c.server = 'SERVER' - - cfg_process_general(c, c.server) - cfg_process_saddrs(c, c.server) - cfg_process_vnetwork(c, c.server) - cfg_process_vaddr(c, c.server) - - for (ci,cs) in putative_clients.items(): - cc = ConfigResults() - sections = cfg_process_client_common(cc,c.server,cs,ci) - if not sections: continue - cfg_process_client_limited(cc,c.server,sections, 'max_batch_down') - cfg_process_client_limited(cc,c.server,sections, 'max_queue_time') - cc.max_clock_skew = cfg_search(cfg.getint, 'max_clock_skew', sections) - Client(ci, cc) - - try: - c.vrelay = cfg1get(c.server, 'vrelay') - except NoOptionError: - for search in c.vnetwork.hosts(): - if search == c.vaddr: continue - c.vrelay = search - break - - try: c.ifname = cfg1get(c.server, 'ifname_server', raw=True) - except NoOptionError: pass - - cfg_process_ipif(c, - [c.server, 'COMMON'], - (('local','vaddr'), - ('peer', 'vrelay'), - ('rnets','vnetwork'))) - - if opts.printconfig is not None: - try: val = cfg1get(c.server, opts.printconfig) - except NoOptionError: pass - else: print(val) - sys.exit(0) - -def catch_termination(): - def run_cleanups(): - for cleanup in cleanups: - cleanup() - - atexit.register(run_cleanups) - - def signal_handler(name, sig, *args): - signal.signal(sig, signal.SIG_DFL) - print('exiting due to %s' % name, file=sys.stderr) - run_cleanups() - os.kill(os.getpid(), sig) - raise RuntimeError('did not die due to signal %s !' % name) - - for sig in (signal.SIGINT, signal.SIGTERM): - try: signame = sig.name - except AttributeError: signame = "signal %d" % sig - signal.signal(sig, partial(signal_handler, signame)) - -def daemonise(): - global syslogfacility - if opts.daemon and opts.syslogfacility is None: - opts.syslogfacility = 'daemon' - - if opts.syslogfacility is not None: - facilnum = syslog.__dict__['LOG_' + opts.syslogfacility.upper()] - syslog.openlog('hippotatd', - facility=facilnum, - logoption=syslog.LOG_PID) - def emit(event): - if logevent_is_boringtwisted(event): return - m = twisted.logger.formatEvent(event) - #print(repr(event), m, file=org_stderr) - level = event.get('log_level') - if event.get('dflag',None) is not None: sl = syslog.LOG_DEBUG - elif level == LogLevel.critical : sl = syslog.LOG_CRIT - elif level == LogLevel.error : sl = syslog.LOG_ERR - elif level == LogLevel.warn : sl = syslog.LOG_WARNING - else : sl = syslog.LOG_INFO - syslog.syslog(sl,m) - failure = event.get('log_failure') - if failure is not None: - for l in failure.getTraceback().split('\n'): - syslog.syslog(sl,l) - glp = twisted.logger.globalLogPublisher - glp.addObserver(emit) - log_debug(DBG.INIT, 'starting to log to syslog') - - #log.crit('daemonic hippotatd crashed', dflag=False) - if opts.daemon: - daemonic_reactor = (twisted.internet.interfaces.IReactorDaemonize - .providedBy(reactor)) - if daemonic_reactor: reactor.beforeDaemonize() - if opts.pidfile is not None: - pidfile_h = open(opts.pidfile, 'w') - rfd, wfd = os.pipe() - childpid = os.fork() - if childpid: - # we are the parent - os.close(wfd) - st = os.read(rfd, 1) - try: - st = st[0] - except IndexError: - st = 127 - log.critical('daemonic hippotatd crashed', dflag=False) - os._exit(st) - os.close(rfd) - os.setsid() - grandchildpid = os.fork() - if grandchildpid: - # we are the intermediate child - if opts.pidfile is not None: - print(grandchildpid, file=pidfile_h) - pidfile_h.close() - os._exit(0) - - if opts.pidfile is not None: - pidfile_h.close() - - logger = subprocess.Popen(['logger','-d', - '-t','hippotat[%d](stderr)' % os.getpid(), - '-p',opts.syslogfacility + '.err'], - stdin=subprocess.PIPE, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - restore_signals=True) - - nullfd = os.open('/dev/null', os.O_RDWR) - os.dup2(nullfd, 0) - os.dup2(nullfd, 1) - os.dup2(logger.stdin.fileno(), 2) - os.close(nullfd) - if daemonic_reactor: reactor.afterDaemonize() - log_debug(DBG.INIT, 'daemonised') - os.write(wfd, b'\0') - os.close(wfd) - - if opts.syslogfacility is not None: - glp.removeObserver(hippotatlib.file_log_observer) - -optparser.add_option('--ownsource', default=2, - action='store_const', dest='ownsource', const=2, - help='source download fully enabled (default)') - -optparser.add_option('--ownsource-local', - action='store_const', dest='ownsource', const=1, - help='source download is local source code only') - -optparser.add_option('--no-ownsource', - action='store_const', dest='ownsource', const=0, - help='source download disabled (for testing only)') - -optparser.add_option('--daemon', - action='store_true', dest='daemon', default=False, - help='daemonize (and log to syslog)') - -optparser.add_option('--pidfile', - nargs=1, type='string', - action='store', dest='pidfile', default=None, - help='write pid to this file') - -optparser.add_option('--syslog-facility', - nargs=1, type='string',action='store', - metavar='FACILITY', dest='syslogfacility', - default=None, - help='log to syslog, with specified facility') - -optparser.add_option('--print-config', - nargs=1, type='string',action='store', - metavar='OPTION', dest='printconfig', - default=None, - help='print one config option value and exit') - -common_startup(process_cfg) -catch_termination() -start_http() -daemonise() -ipif = start_ipif(c.ipif_command, - (lambda p,s,d: route(p,"[ipif]",s,d)), - c.mtu) -common_run() diff --git a/old-python/hippotatlib/__init__.py b/old-python/hippotatlib/__init__.py deleted file mode 100644 index 91696b1..0000000 --- a/old-python/hippotatlib/__init__.py +++ /dev/null @@ -1,766 +0,0 @@ -# -*- python -*- -# -# Hippotat - Asinine IP Over HTTP program -# hippotatlib/__init__.py - common library code -# -# Copyright 2017 Ian Jackson -# -# GPLv3+ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program, in the file GPLv3. If not, -# see . - - -import signal -signal.signal(signal.SIGINT, signal.SIG_DFL) - -import sys -import os - -from zope.interface import implementer - -import twisted -from twisted.internet import reactor -import twisted.internet.endpoints -import twisted.logger -from twisted.logger import LogLevel -import twisted.python.constants -from twisted.python.constants import NamedConstant - -import ipaddress -from ipaddress import AddressValueError - -from optparse import OptionParser -import configparser -from configparser import ConfigParser -from configparser import NoOptionError - -from functools import partial - -import collections -import time -import hmac -import hashlib -import base64 -import codecs -import traceback - -import re as regexp - -import hippotatlib.slip as slip - -class DBG(twisted.python.constants.Names): - INIT = NamedConstant() - CONFIG = NamedConstant() - ROUTE = NamedConstant() - DROP = NamedConstant() - OWNSOURCE = NamedConstant() - FLOW = NamedConstant() - HTTP = NamedConstant() - TWISTED = NamedConstant() - QUEUE = NamedConstant() - HTTP_CTRL = NamedConstant() - QUEUE_CTRL = NamedConstant() - HTTP_FULL = NamedConstant() - CTRL_DUMP = NamedConstant() - SLIP_FULL = NamedConstant() - DATA_COMPLETE = NamedConstant() - -_hex_codec = codecs.getencoder('hex_codec') - -#---------- logging ---------- - -org_stderr = sys.stderr - -log = twisted.logger.Logger() - -debug_set = set() -debug_def_detail = DBG.HTTP - -def log_debug(dflag, msg, idof=None, d=None): - if dflag not in debug_set: return - #print('---------------->',repr((dflag, msg, idof, d)), file=sys.stderr) - if idof is not None: - msg = '[%#x] %s' % (id(idof), msg) - if d is not None: - trunc = '' - if not DBG.DATA_COMPLETE in debug_set: - if len(d) > 64: - d = d[0:64] - trunc = '...' - d = _hex_codec(d)[0].decode('ascii') - msg += ' ' + d + trunc - log.info('{dflag} {msgcore}', dflag=dflag, msgcore=msg) - -def logevent_is_boringtwisted(event): - try: - if event.get('log_level') != LogLevel.info: - return False - dflag = event.get('dflag') - if dflag is False : return False - if dflag in debug_set: return False - if dflag is None and DBG.TWISTED in debug_set: return False - return True - except Exception: - print('EXCEPTION (IN BORINGTWISTED CHECK)', - traceback.format_exc(), file=org_stderr) - return False - -@implementer(twisted.logger.ILogFilterPredicate) -class LogNotBoringTwisted: - def __call__(self, event): - return ( - twisted.logger.PredicateResult.no - if logevent_is_boringtwisted(event) else - twisted.logger.PredicateResult.yes - ) - -#---------- default config ---------- - -defcfg = ''' -[COMMON] -max_batch_down = 65536 -max_queue_time = 10 -target_requests_outstanding = 3 -http_timeout = 30 -http_timeout_grace = 5 -max_requests_outstanding = 6 -max_batch_up = 4000 -http_retry = 5 -port = 80 -vroutes = '' -ifname_client = hippo%%d -ifname_server = shippo%%d -max_clock_skew = 300 - -#[server] or [] overrides -ipif = userv root ipif %(local)s,%(peer)s,%(mtu)s,slip,%(ifname)s %(rnets)s - -# relating to virtual network -mtu = 1500 - -# addrs = 127.0.0.1 ::1 -# url - -# relating to virtual network -vvnetwork = 172.24.230.192 -# vnetwork = / -# vaddr = -# vrelay = - - -# [] -# secret = # used by both, must match - -[LIMIT] -max_batch_down = 262144 -max_queue_time = 121 -http_timeout = 121 -target_requests_outstanding = 10 -''' - -# these need to be defined here so that they can be imported by import * -cfg = ConfigParser(strict=False) -optparser = OptionParser() - -_mimetrans = bytes.maketrans(b'-'+slip.esc, slip.esc+b'-') -def mime_translate(s): - # SLIP-encoded packets cannot contain ESC ESC. - # Swap `-' and ESC. The result cannot contain `--' - return s.translate(_mimetrans) - -class ConfigResults: - def __init__(self): - pass - def __repr__(self): - return 'ConfigResults('+repr(self.__dict__)+')' - -def log_discard(packet, iface, saddr, daddr, why): - log_debug(DBG.DROP, - 'discarded packet [%s] %s -> %s: %s' % (iface, saddr, daddr, why), - d=packet) - -#---------- packet parsing ---------- - -def packet_addrs(packet): - version = packet[0] >> 4 - if version == 4: - addrlen = 4 - saddroff = 3*4 - factory = ipaddress.IPv4Address - elif version == 6: - addrlen = 16 - saddroff = 2*4 - factory = ipaddress.IPv6Address - else: - raise ValueError('unsupported IP version %d' % version) - saddr = factory(packet[ saddroff : saddroff + addrlen ]) - daddr = factory(packet[ saddroff + addrlen : saddroff + addrlen*2 ]) - return (saddr, daddr) - -#---------- address handling ---------- - -def ipaddr(input): - try: - r = ipaddress.IPv4Address(input) - except AddressValueError: - r = ipaddress.IPv6Address(input) - return r - -def ipnetwork(input): - try: - r = ipaddress.IPv4Network(input) - except NetworkValueError: - r = ipaddress.IPv6Network(input) - return r - -#---------- ipif (SLIP) subprocess ---------- - -class SlipStreamDecoder(): - def __init__(self, desc, on_packet, mtu): - self._buffer = b'' - self._on_packet = on_packet - self._desc = desc - self._mtu = mtu - self._log('__init__') - - def _log(self, msg, **kwargs): - log_debug(DBG.SLIP_FULL, 'slip %s: %s' % (self._desc, msg), **kwargs) - - def inputdata(self, data): - self._log('inputdata', d=data) - data = self._buffer + data - self._buffer = b'' - packets = slip.decode(data, True) - self._buffer = packets.pop() - for packet in packets: - self._maybe_packet(packet) - self._log('bufremain', d=self._buffer) - - def _maybe_packet(self, packet): - self._log('maybepacket', d=packet) - if len(packet) and len(packet) <= self._mtu: - self._on_packet(packet) - - def flush(self): - self._log('flush') - data = self._buffer - self._buffer = b'' - packets = slip.decode(data) - assert(len(packets) == 1) - self._maybe_packet(packets[0]) - -class _IpifProcessProtocol(twisted.internet.protocol.ProcessProtocol): - def __init__(self, router, mtu): - self._router = router - self._decoder = SlipStreamDecoder('ipif', self.slip_on_packet, mtu) - def connectionMade(self): pass - def outReceived(self, data): - self._decoder.inputdata(data) - def slip_on_packet(self, packet): - (saddr, daddr) = packet_addrs(packet) - if saddr.is_link_local or daddr.is_link_local: - log_discard(packet, 'ipif', saddr, daddr, 'link-local') - return - self._router(packet, saddr, daddr) - def processEnded(self, status): - status.raiseException() - -def start_ipif(command, router, mtu): - ipif = _IpifProcessProtocol(router, mtu) - reactor.spawnProcess(ipif, - '/bin/sh',['sh','-xc', command], - childFDs={0:'w', 1:'r', 2:2}, - env=None) - return ipif - -def queue_inbound(ipif, packet): - log_debug(DBG.FLOW, "queue_inbound", d=packet) - ipif.transport.write(slip.delimiter) - ipif.transport.write(slip.encode(packet)) - ipif.transport.write(slip.delimiter) - -#---------- packet queue ---------- - -class PacketQueue(): - def __init__(self, desc, max_queue_time): - self._desc = desc - assert(desc + '') - self._max_queue_time = max_queue_time - self._pq = collections.deque() # packets - - def _log(self, dflag, msg, **kwargs): - log_debug(dflag, self._desc+' pq: '+msg, **kwargs) - - def append(self, packet): - self._log(DBG.QUEUE, 'append', d=packet) - self._pq.append((time.monotonic(), packet)) - - def nonempty(self): - self._log(DBG.QUEUE, 'nonempty ?') - while True: - try: (queuetime, packet) = self._pq[0] - except IndexError: - self._log(DBG.QUEUE, 'nonempty ? empty.') - return False - - age = time.monotonic() - queuetime - if age > self._max_queue_time: - # strip old packets off the front - self._log(DBG.QUEUE, 'dropping (old)', d=packet) - self._pq.popleft() - continue - - self._log(DBG.QUEUE, 'nonempty ? nonempty.') - return True - - def process(self, sizequery, moredata, max_batch): - # sizequery() should return size of batch so far - # moredata(s) should add s to batch - self._log(DBG.QUEUE, 'process...') - while True: - try: (dummy, packet) = self._pq[0] - except IndexError: - self._log(DBG.QUEUE, 'process... empty') - break - - self._log(DBG.QUEUE_CTRL, 'process... packet', d=packet) - - encoded = slip.encode(packet) - sofar = sizequery() - - self._log(DBG.QUEUE_CTRL, - 'process... (sofar=%d, max=%d) encoded' % (sofar, max_batch), - d=encoded) - - if sofar > 0: - if sofar + len(slip.delimiter) + len(encoded) > max_batch: - self._log(DBG.QUEUE_CTRL, 'process... overflow') - break - moredata(slip.delimiter) - - moredata(encoded) - self._pq.popleft() - -#---------- error handling ---------- - -_crashing = False - -def crash(err): - global _crashing - _crashing = True - print('========== CRASH ==========', err, - '===========================', file=sys.stderr) - try: reactor.stop() - except twisted.internet.error.ReactorNotRunning: pass - -def crash_on_defer(defer): - defer.addErrback(lambda err: crash(err)) - -def crash_on_critical(event): - if event.get('log_level') >= LogLevel.critical: - crash(twisted.logger.formatEvent(event)) - -#---------- authentication tokens ---------- - -_authtoken_digest = hashlib.sha256 - -def _authtoken_time(): - return int(time.time()) - -def _authtoken_hmac(secret, hextime): - return hmac.new(secret, hextime, _authtoken_digest).digest() - -def authtoken_make(secret): - hextime = ('%x' % _authtoken_time()).encode('ascii') - mac = _authtoken_hmac(secret, hextime) - return hextime + b' ' + base64.b64encode(mac) - -def authtoken_check(secret, token, maxskew): - (hextime, theirmac64) = token.split(b' ') - now = _authtoken_time() - then = int(hextime, 16) - skew = then - now; - if (abs(skew) > maxskew): - raise ValueError('too much clock skew (client %ds ahead)' % skew) - theirmac = base64.b64decode(theirmac64) - ourmac = _authtoken_hmac(secret, hextime) - if not hmac.compare_digest(theirmac, ourmac): - raise ValueError('invalid token (wrong secret?)') - pass - -#---------- config processing ---------- - -def _cfg_process_putatives(): - servers = { } - clients = { } - # maps from abstract object to canonical name for cs's - - def putative(cmap, abstract, canoncs): - try: - current_canoncs = cmap[abstract] - except KeyError: - pass - else: - assert(current_canoncs == canoncs) - cmap[abstract] = canoncs - - server_pat = r'[-.0-9A-Za-z]+' - client_pat = r'[.:0-9a-f]+' - server_re = regexp.compile(server_pat) - serverclient_re = regexp.compile( - server_pat + r' ' + '(?:' + client_pat + '|LIMIT)') - - for cs in cfg.sections(): - def dbg(m): - log_debug_config('putatives: section [%s] %s' % (cs, m)) - - def log_ignore(why): - dbg('X ignore: %s' % (why)) - print('warning: ignoring config section [%s] (%s)' % (cs, why), - file=sys.stderr) - - if cs == 'LIMIT' or cs == 'COMMON': - # plan A "[LIMIT]" or "[COMMON]" - dbg('A ignore') - continue - - try: - # plan B "[]" part 1 - ci = ipaddr(cs) - except AddressValueError: - - if server_re.fullmatch(cs): - # plan C "[]" - dbg('C ') - putative(servers, cs, cs) - continue - - if serverclient_re.fullmatch(cs): - # plan D "[ ]" part 1 - (pss,pcs) = cs.split(' ') - - if pcs == 'LIMIT': - # plan E "[ LIMIT]" - dbg('E LIMIT') - continue - - try: - # plan D "[ ]" part 2 - ci = ipaddr(pcs) - except AddressValueError: - # plan F branch 1 "[]" - log_ignore('bad-addr') - continue - - else: # no AddressValueError - # plan D "[ ]" part 3 - dbg('D ') - putative(clients, ci, pcs) - putative(servers, pss, pss) - continue - else: - # plan F branch 2 "[]" - log_ignore('nomatch '+ repr(serverclient_re)) - - else: # no AddressValueError - # plan B "[" part 2 - dbg('B ') - putative(clients, ci, cs) - continue - - return (servers, clients) - -def cfg_process_general(c, ss): - c.mtu = cfg1getint(ss, 'mtu') - -def cfg_process_saddrs(c, ss): - class ServerAddr(): - def __init__(self, port, addrspec): - self.port = port - # also self.addr - try: - self.addr = ipaddress.IPv4Address(addrspec) - self._endpointfactory = twisted.internet.endpoints.TCP4ServerEndpoint - self._inurl = b'%s' - except AddressValueError: - self.addr = ipaddress.IPv6Address(addrspec) - self._endpointfactory = twisted.internet.endpoints.TCP6ServerEndpoint - self._inurl = b'[%s]' - def make_endpoint(self): - return self._endpointfactory(reactor, self.port, - interface= '%s' % self.addr) - def url(self): - url = b'http://' + (self._inurl % str(self.addr).encode('ascii')) - if self.port != 80: url += b':%d' % self.port - url += b'/' - return url - def __repr__(self): - return 'ServerAddr'+repr((self.port,self.addr)) - - c.port = cfg1getint(ss,'port') - c.saddrs = [ ] - for addrspec in cfg1get(ss, 'addrs').split(): - sa = ServerAddr(c.port, addrspec) - c.saddrs.append(sa) - -def cfg_process_vnetwork(c, ss): - c.vnetwork = ipnetwork(cfg1get(ss,'vnetwork')) - if c.vnetwork.num_addresses < 3 + 2: - raise ValueError('vnetwork needs at least 2^3 addresses') - -def cfg_process_vaddr(c, ss): - try: - c.vaddr = ipaddr(cfg1get(ss,'vaddr')) - except NoOptionError: - cfg_process_vnetwork(c, ss) - c.vaddr = next(c.vnetwork.hosts()) - -def cfg_search_section(key,sections): - for section in sections: - if cfg.has_option(section, key): - return section - raise NoOptionError(key, repr(sections)) - -def cfg_get_raw(*args, **kwargs): - # for passing to cfg_search - return cfg.get(*args, raw=True, **kwargs) - -def cfg_search(getter,key,sections): - section = cfg_search_section(key,sections) - return getter(section, key) - -def cfg1get(section,key, getter=cfg.get,**kwargs): - section = cfg_search_section(key,[section,'COMMON']) - return getter(section,key,**kwargs) - -def cfg1getint(section,key, **kwargs): - return cfg1get(section,key, getter=cfg.getint,**kwargs); - -def cfg_process_client_limited(cc,ss,sections,key): - val = cfg_search(cfg1getint, key, sections) - lim = cfg_search(cfg1getint, key, ['%s LIMIT' % ss, 'LIMIT']) - cc.__dict__[key] = min(val,lim) - -def cfg_process_client_common(cc,ss,cs,ci): - # returns sections to search in, iff secret is defined, otherwise None - cc.ci = ci - - sections = ['%s %s' % (ss,cs), - cs, - ss, - 'COMMON'] - - try: pwsection = cfg_search_section('secret', sections) - except NoOptionError: return None - - pw = cfg1get(pwsection, 'secret') - cc.secret = pw.encode('utf-8') - - cfg_process_client_limited(cc,ss,sections,'target_requests_outstanding') - cfg_process_client_limited(cc,ss,sections,'http_timeout') - - return sections - -def cfg_process_ipif(c, sections, varmap): - for d, s in varmap: - try: v = getattr(c, s) - except AttributeError: continue - setattr(c, d, v) - for d in ('mtu',): - v = cfg_search(cfg1getint, d, sections) - setattr(c, d, v) - - #print('CFGIPIF',repr((varmap, sections, c.__dict__)),file=sys.stderr) - - section = cfg_search_section('ipif', sections) - c.ipif_command = cfg1get(section,'ipif', vars=c.__dict__) - -#---------- startup ---------- - -def log_debug_config(m): - if not DBG.CONFIG in debug_set: return - print('DBG.CONFIG:', m) - -def common_startup(process_cfg): - # calls process_cfg(putative_clients, putative_servers) - - # ConfigParser hates #-comments after values - trailingcomments_re = regexp.compile(r'#.*') - cfg.read_string(trailingcomments_re.sub('', defcfg)) - need_defcfg = True - - def readconfig(pathname, mandatory=True): - def log(m, p=pathname): - if not DBG.CONFIG in debug_set: return - log_debug_config('%s: %s' % (m, p)) - - try: - files = os.listdir(pathname) - - except FileNotFoundError: - if mandatory: raise - log('skipped') - return - - except NotADirectoryError: - cfg.read(pathname) - log('read file') - return - - # is a directory - log('directory') - re = regexp.compile('[^-A-Za-z0-9_]') - for f in os.listdir(pathname): - if re.search(f): continue - subpath = pathname + '/' + f - try: - os.stat(subpath) - except FileNotFoundError: - log('entry skipped', subpath) - continue - cfg.read(subpath) - log('entry read', subpath) - - def oc_config(od,os, value, op): - nonlocal need_defcfg - need_defcfg = False - readconfig(value) - - def oc_extra_config(od,os, value, op): - readconfig(value) - - def read_defconfig(): - readconfig('/etc/hippotat/config.d', False) - readconfig('/etc/hippotat/secrets.d', False) - readconfig('/etc/hippotat/master.cfg', False) - - def oc_defconfig(od,os, value, op): - nonlocal need_defcfg - need_defcfg = False - read_defconfig(value) - - def dfs_less_detailed(dl): - return [df for df in DBG.iterconstants() if df <= dl] - - def ds_default(od,os,dl,op): - global debug_set - debug_set.clear - debug_set |= set(dfs_less_detailed(debug_def_detail)) - - def ds_select(od,os, spec, op): - for it in spec.split(','): - - if it.startswith('-'): - mutator = debug_set.discard - it = it[1:] - else: - mutator = debug_set.add - - if it == '+': - dfs = DBG.iterconstants() - - else: - if it.endswith('+'): - mapper = dfs_less_detailed - it = it[0:len(it)-1] - else: - mapper = lambda x: [x] - - try: - dfspec = DBG.lookupByName(it) - except ValueError: - optparser.error('unknown debug flag %s in --debug-select' % it) - - dfs = mapper(dfspec) - - for df in dfs: - mutator(df) - - optparser.add_option('-D', '--debug', - nargs=0, - action='callback', - help='enable default debug (to stdout)', - callback= ds_default) - - optparser.add_option('--debug-select', - nargs=1, - type='string', - metavar='[-]DFLAG[+]|[-]+,...', - help= -'''enable (`-': disable) each specified DFLAG; -`+': do same for all "more interesting" DFLAGSs; -just `+': all DFLAGs. - DFLAGS: ''' + ' '.join([df.name for df in DBG.iterconstants()]), - action='callback', - callback= ds_select) - - optparser.add_option('-c', '--config', - nargs=1, - type='string', - metavar='CONFIGFILE', - dest='configfile', - action='callback', - callback= oc_config) - - optparser.add_option('--extra-config', - nargs=1, - type='string', - metavar='CONFIGFILE', - dest='configfile', - action='callback', - callback= oc_extra_config) - - optparser.add_option('--default-config', - action='callback', - callback= oc_defconfig) - - (opts, args) = optparser.parse_args() - if len(args): optparser.error('no non-option arguments please') - - if need_defcfg: - read_defconfig() - - try: - (pss, pcs) = _cfg_process_putatives() - process_cfg(opts, pss, pcs) - except (configparser.Error, ValueError): - traceback.print_exc(file=sys.stderr) - print('\nInvalid configuration, giving up.', file=sys.stderr) - sys.exit(12) - - - #print('X', debug_set, file=sys.stderr) - - log_formatter = twisted.logger.formatEventAsClassicLogText - stdout_obs = twisted.logger.FileLogObserver(sys.stdout, log_formatter) - stderr_obs = twisted.logger.FileLogObserver(sys.stderr, log_formatter) - pred = twisted.logger.LogLevelFilterPredicate(LogLevel.error) - stdsomething_obs = twisted.logger.FilteringLogObserver( - stderr_obs, [pred], stdout_obs - ) - global file_log_observer - file_log_observer = twisted.logger.FilteringLogObserver( - stdsomething_obs, [LogNotBoringTwisted()] - ) - #log_observer = stdsomething_obs - twisted.logger.globalLogBeginner.beginLoggingTo( - [ file_log_observer, crash_on_critical ] - ) - -def common_run(): - log_debug(DBG.INIT, 'entering reactor') - if not _crashing: reactor.run() - print('ENDED', file=sys.stderr) - sys.exit(16) diff --git a/old-python/hippotatlib/ownsource.py b/old-python/hippotatlib/ownsource.py deleted file mode 100644 index 16ccdbd..0000000 --- a/old-python/hippotatlib/ownsource.py +++ /dev/null @@ -1,375 +0,0 @@ -# -*- python -*- -# -# Hippotat - Asinine IP Over HTTP program -# hippotatlib/ownsource.py - Automatic source code provision (AGPL compliance) -# -# Copyright 2017 Ian Jackson -# -# AGPLv3+ + CAFv2+ -# -# This program is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public -# License as published by the Free Software Foundation, either -# version 3 of the License, or (at your option) any later version, -# with the "CAF Login Exception" as published by Ian Jackson -# (version 2, or at your option any later version) as an Additional -# Permission. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public -# License and the CAF Login Exception along with this program, in -# the file AGPLv3+CAFv2. If not, email Ian Jackson -# . - - -import os -import sys -import fnmatch -import stat -import subprocess -import tempfile -import shutil - -try: import debian.deb822 -except ImportError: pass - -class SourceShipmentPreparer(): - def __init__(s, destdir): - # caller may modify, and should read after calling generate() - s.output_names = ['srcbomb.tar.gz', 'srcpkgsbomb.tar'] - s.output_paths = [None,None] # alternatively caller may read this - # defaults, caller can modify after creation - s.logger = lambda m: print('SourceShipmentPreparer',m) - s.src_filter = s.src_filter_glob - s.src_package_globs = ['!/usr/local/*', '/usr*'] - s.src_filter_globs = ['!/etc/*'] - s.src_likeparent = s.src_likeparent_git - s.src_direxcludes = s.src_direxcludes_git - s.report_from_packages = s.report_from_packages_debian - s.cwd = os.getcwd() - s.find_rune_base = "find -type f -perm -004 \! -path '*/tmp/*'" - s.ignores = ['*~', '*.bak', '*.tmp', '#*#', '__pycache__', - '[0-9][0-9][0-9][0-9]-src.tar'] - s.rune_shell = ['/bin/bash', '-ec'] - s.show_pathnames = True - s.download_packages = True - s.stream_stderr = sys.stderr - s.stream_debug = open('/dev/null','w') - s.rune_cpio = r''' - set -o pipefail - ( - %s - # ^ by default, is find ... -print0 - ) | ( - cpio -Hustar -o --quiet -0 -R 1000:1000 || \ - cpio -Hustar -o --quiet -0 - ) - ''' - s.rune_portmanteau = r''' - GZIP=-1 tar zcf - "$@" - ''' - s.rune_portmanteau_uncompressed = r''' - tar cf - "$@" - ''' - s.manifest_name='0000-MANIFEST.txt' - # private - s._destdir = destdir - s._outcounter = 0 - s._manifest = [] - s._dirmap = { } - s._package_files = { } # map filename => infol - s._packages_path = os.path.join(s._destdir, 'packages') - s._package_sources = [] - - def thing_matches_globs(s, thing, globs): - for pat in globs: - negate = pat.startswith('!') - if negate: pat = pat[1:] - if fnmatch.fnmatch(thing, pat): - return not negate - return negate - - def src_filter_glob(s, src): # default s.src_filter - return s.thing_matches_globs(src, s.src_filter_globs) - - def src_direxcludes_git(s, d): - try: - excl = open(os.path.join(d, '.gitignore')) - except FileNotFoundError: - return [] - r = [] - for l in excl: - l = l.strip() - if l.startswith('#'): next - if not len(l): next - r.append(l) - return r - - def src_likeparent_git(s, src): - try: - os.stat(os.path.join(src, '.git/.')) - except FileNotFoundError: - return False - else: - return True - - def src_parentfinder(s, src, infol): # callers may monkey-patch away - for deref in (False,True): - xinfo = [] - - search = src - if deref: - search = os.path.realpath(search) - - def ascend(): - nonlocal search - xinfo.append(os.path.basename(search)) - search = os.path.dirname(search) - - try: - stab = os.lstat(search) - except FileNotFoundError: - return - if stat.S_ISREG(stab.st_mode): - ascend() - - while not os.path.ismount(search): - if s.src_likeparent(search): - xinfo.reverse() - if len(xinfo): infol.append('want=' + os.path.join(*xinfo)) - return search - - ascend() - - # no .git found anywhere - return src - - def path_prenormaliser(s, d, infol): # callers may monkey-patch away - return os.path.join(s.cwd, os.path.abspath(d)) - - def srcdir_find_rune(s, d): - script = s.find_rune_base - ignores = s.ignores + s.output_names + [s.manifest_name] - ignores += s.src_direxcludes(d) - for excl in ignores: - assert("'" not in excl) - script += r" \! -name '%s'" % excl - script += r" \! -path '*/%s/*'" % excl - script += ' -print0' - return script - - def manifest_append(s, name, infol): - s._manifest.append({ 'file':name, 'info':' '.join(infol) }) - - def manifest_append_absentfile(s, name, infol): - s._manifest.append({ 'file_print':name, 'info':' '.join(infol) }) - - def new_output_name(s, nametail, infol): - s._outcounter += 1 - name = '%04d-%s' % (s._outcounter, nametail) - s.manifest_append(name, infol) - return name - - def open_output_fh(s, name, mode): - return open(os.path.join(s._destdir, name), mode) - - def src_dir(s, d, infol): - try: name = s._dirmap[d] - except KeyError: pass - else: - s.manifest_append(name, infol) - return - - if s.show_pathnames: infol.append(d) - find_rune = s.srcdir_find_rune(d) - total_rune = s.rune_cpio % find_rune - - name = s.new_output_name('src.tar', infol) - s._dirmap[d] = name - fh = s.open_output_fh(name, 'wb') - - s.logger('packing up into %s: %s (because %s)' % - (name, d, ' '.join(infol))) - - subprocess.run(s.rune_shell + [total_rune], - cwd=d, - stdin=subprocess.DEVNULL, - stdout=fh, - restore_signals=True, - check=True) - fh.close() - - def src_indir(s, d, infol): - d = s.path_prenormaliser(d, infol) - if not s.src_filter(d): return - - d = s.src_parentfinder(d, infol) - if d is None: return - s.src_dir(d, infol) - - def report_from_packages_debian(s, files): - dpkg_S_in = tempfile.TemporaryFile(mode='w+') - for (file, infols) in files.items(): - assert('\n' not in file) - dpkg_S_in.write(file) - dpkg_S_in.write('\0') - dpkg_S_in.seek(0) - cmdl = ['xargs','-0r','dpkg','-S','--'] - dpkg_S = subprocess.Popen(cmdl, - cwd='/', - stdin=dpkg_S_in, - stdout=subprocess.PIPE, - stderr=sys.stderr, - close_fds=False) - dpkg_show_in = tempfile.TemporaryFile(mode='w+') - pkginfos = { } - for l in dpkg_S.stdout: - l = l.strip(b'\n').decode('utf-8') - (pkgs, fname) = l.split(': ',1) - pks = pkgs.split(', ') - for pk in pks: - pkginfos.setdefault(pk,{'files':[]})['files'].append(fname) - print(pk, file=dpkg_show_in) - assert(dpkg_S.wait() == 0) - dpkg_show_in.seek(0) - cmdl = ['xargs','-r','dpkg-query', - r'-f${binary:Package}\t${Package}\t${Architecture}\t${Version}\t${source:Package}\t${source:Version}\t${source:Upstream-Version}\n', - '--show','--'] - dpkg_show = subprocess.Popen(cmdl, - cwd='/', - stdin=dpkg_show_in, - stdout=subprocess.PIPE, - stderr=sys.stderr, - close_fds=False) - for l in dpkg_show.stdout: - l = l.strip(b'\n').decode('utf-8') - (pk,p,a,v,sp,sv,suv) = l.split('\t') - pkginfos[pk]['binary'] = p - pkginfos[pk]['arch'] = a - pkginfos[pk]['version'] = v - pkginfos[pk]['source'] = sp - pkginfos[pk]['sourceversion'] = sv - pkginfos[pk]['sourceupstreamversion'] = sv - assert(dpkg_show.wait() == 0) - for pk in sorted(pkginfos.keys()): - pi = pkginfos[pk] - debfname = '%s_%s_%s.deb' % (pi['binary'], pi['version'], pi['arch']) - dscfname = '%s_%s.dsc' % (pi['source'], pi['sourceversion']) - s.manifest_append_absentfile(dscfname, [debfname]) - s.logger('mentioning %s and %s because %s' % - (dscfname, debfname, pi['files'][0])) - for fname in pi['files']: - infol = files[fname] - if s.show_pathnames: infol = infol + ['loaded='+fname] - s.manifest_append_absentfile(' \t' + debfname, infol) - - if s.download_packages: - try: os.mkdir(s._packages_path) - except FileExistsError: pass - - cmdl = ['apt-get','--download-only','source', - '%s=%s' % (pi['source'], pi['sourceversion'])] - subprocess.run(cmdl, - cwd=s._packages_path, - stdin=subprocess.DEVNULL, - stdout=s.stream_debug, - stderr=s.stream_stderr, - restore_signals=True, - check=True) - - s._package_sources.append(dscfname) - dsc = debian.deb822.Dsc(open(s._packages_path + '/' + dscfname)) - for indsc in dsc['Files']: - s._package_sources.append(indsc['name']) - - def thing_ought_packaged(s, fname): - return s.thing_matches_globs(fname, s.src_package_globs) - - def src_file_packaged(s, fname, infol): - s._package_files.setdefault(fname,[]).extend(infol) - - def src_file(s, fname, infol): - def fngens(): - yield (infol, fname) - infol_copy = infol.copy() - yield (infol_copy, s.path_prenormaliser(fname, infol_copy)) - yield (infol, os.path.realpath(fname)) - - for (tinfol, tfname) in fngens(): - if s.thing_ought_packaged(tfname): - s.src_file_packaged(tfname, tinfol) - return - - s.src_indir(fname, infol) - - def src_argv0(s, program, infol): - s.src_file(program, infol) - - def src_syspath(s, fname, infol): - if s.thing_ought_packaged(fname): return - s.src_indir(fname, infol) - - def src_module(s, m, infol): - try: fname = m.__file__ - except AttributeError: return - infol.append('module='+m.__name__) - - if s.thing_ought_packaged(fname): - s.src_file_packaged(fname, infol) - else: - s.src_indir(fname, infol) - - def srcs_allitems(s, dirs=sys.path): - s.logger('allitems') - s.src_argv0(sys.argv[0], ['argv[0]']) - for d in sys.path: - s.src_syspath(d, ['sys.path']) - for m in sys.modules.values(): - s.src_module(m, ['sys.modules']) - s.report_from_packages(s._package_files) - s.logger('allitems done') - - def _mk_portmanteau(s, ix, rune, cwd, files): - output_name = s.output_names[ix] - s.logger('making portmanteau %s' % output_name) - output_path = os.path.join(s._destdir, output_name) - subprocess.run(s.rune_shell + [ rune, 'x' ] + files, - cwd=cwd, - stdin=subprocess.DEVNULL, - stdout=open(output_path, 'wb'), - restore_signals=True, - check=True) - s.output_paths[ix] = output_path - - def mk_inner_portmanteau(s): - outputs = [s.manifest_name] - outputs_done = { } - mfh = s.open_output_fh(s.manifest_name,'w') - for me in s._manifest: - try: fname = me['file'] - except KeyError: fname = me.get('file_print','') - else: - try: outputs_done[fname] - except KeyError: - outputs.append(fname) - outputs_done[fname] = 1 - print('%s\t%s' % (fname, me['info']), file=mfh) - mfh.close() - - s._mk_portmanteau(0, s.rune_portmanteau, - s._destdir, outputs) - - def mk_packages_portmanteau(s): - if not s.download_packages: return - s._mk_portmanteau(1, s.rune_portmanteau_uncompressed, - s._packages_path, s._package_sources) - - def generate(s): - s.srcs_allitems() - s.mk_inner_portmanteau() - s.mk_packages_portmanteau() - s.logger('portmanteau ready in %s %s' % tuple(s.output_paths)) diff --git a/old-python/hippotatlib/slip.py b/old-python/hippotatlib/slip.py deleted file mode 100644 index 7a9ba60..0000000 --- a/old-python/hippotatlib/slip.py +++ /dev/null @@ -1,63 +0,0 @@ -# -*- python -*- -# -# Hippotat - Asinine IP Over HTTP program -# hippotatlib/slip.py - SLIP handling -# -# Copyright 2017 Ian Jackson -# -# GPLv3+ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program, in the file GPLv3. If not, -# see . - - -end = b'\300' -esc = b'\333' -esc_end = b'\334' -esc_esc = b'\335' -delimiter = end - -def encode(packet): - return (packet - .replace(esc, esc + esc_esc) - .replace(end, esc + esc_end)) - -def decode(data, keep_tail=False): - #print('DECODE ', repr(data)) - out = [] - inp = data.split(end) - tail = [] - if keep_tail: - tail.append(inp.pop()) - for packet in inp: - pdata = b'' - while True: - eix = packet.find(esc) - if eix == -1: - pdata += packet - break - #print('ESC ', repr((pdata, packet, eix))) - pdata += packet[0 : eix] - ck = packet[eix+1] - #print('ESC... %o' % ck) - if ck == esc_esc[0]: pdata += esc - elif ck == esc_end[0]: pdata += end - else: raise ValueError('invalid SLIP escape 0%o %#x' % (ck, ck)) - packet = packet[eix+2 : ] - out.append(pdata) - #print('DECODED ', repr(out)) - out += tail - return out -# -*- python -*- -