fixes

[hippotat.git] / hippotat / __init__.py

diff --git a/hippotat/__init__.py b/hippotat/__init__.py
index 32e685418635f9297fc02cc34586ddc6bd3782c3..1e8a23f11e7b717f0a17534835b65f1debe4eb33 100644 (file)
--- a/hippotat/__init__.py
+++ b/hippotat/__init__.py
@@ -20,9 +20,12 @@ from optparse import OptionParser
 from configparser import ConfigParser
 from configparser import NoOptionError
 
+from functools import partial
+
 import collections
 import time
 import codecs
+import traceback
 
 import re as regexp
 
@@ -30,23 +33,28 @@ import hippotat.slip as slip
 
 class DBG(twisted.python.constants.Names):
   ROUTE = NamedConstant()
+  DROP = NamedConstant()
   FLOW = NamedConstant()
   HTTP = NamedConstant()
   HTTP_CTRL = NamedConstant()
   INIT = NamedConstant()
   QUEUE = NamedConstant()
   QUEUE_CTRL = NamedConstant()
+  HTTP_FULL = NamedConstant()
+  SLIP_FULL = NamedConstant()
+  CTRL_DUMP = NamedConstant()
 
-_hexcodec = codecs.getencoder('hex_codec')
+_hex_codec = codecs.getencoder('hex_codec')
 
 log = twisted.logger.Logger()
 
 def log_debug(dflag, msg, idof=None, d=None):
+  #print('---------------->',repr((dflag, msg, idof, d)), file=sys.stderr)
   if idof is not None:
-    msg = '[%d] %s' % (id(idof), msg)
+    msg = '[%#x] %s' % (id(idof), msg)
   if d is not None:
-    d = d[0:64]
-    d = _hex_codec(d)[0]
+    #d = d[0:64]
+    d = _hex_codec(d)[0].decode('ascii')
     msg += ' ' + d
   log.info('{dflag} {msgcore}', dflag=dflag, msgcore=msg)
 
@@ -55,11 +63,11 @@ defcfg = '''
 #[<client>] overrides
 max_batch_down = 65536           # used by server, subject to [limits]
 max_queue_time = 10              # used by server, subject to [limits]
-max_request_time = 54            # used by server, subject to [limits]
 target_requests_outstanding = 3  # must match; subject to [limits] on server
+http_timeout = 30                # used by both } must be
+http_timeout_grace = 5           # used by both }  compatible
 max_requests_outstanding = 4     # used by client
 max_batch_up = 4000              # used by client
-http_timeout = 30                # used by client
 http_retry = 5                   # used by client
 
 #[server] or [<client>] overrides
@@ -88,7 +96,7 @@ port = 80                  # used by server
 [limits]
 max_batch_down = 262144           # used by server
 max_queue_time = 121              # used by server
-max_request_time = 121            # used by server
+http_timeout = 121                # used by server
 target_requests_outstanding = 10  # used by server
 '''
 
@@ -110,11 +118,10 @@ class ConfigResults:
 
 c = ConfigResults()
 
-def log_discard(packet, saddr, daddr, why):
-  
-  Print('Drop ', Saddr, Daddr, why)
-#  syslog.syslog(syslog.LOG_DEBUG,
-#                'discarded packet %s -> %s (%s)' % (saddr, daddr, why))
+def log_discard(packet, iface, saddr, daddr, why):
+  log_debug(DBG.DROP,
+            'discarded packet [%s] %s -> %s: %s' % (iface, saddr, daddr, why),
+            d=packet)
 
 #---------- packet parsing ----------
 
@@ -153,38 +160,45 @@ def ipnetwork(input):
 #---------- ipif (SLIP) subprocess ----------
 
 class SlipStreamDecoder():
-  def __init__(self, on_packet):
-    # we will call packet(<packet>)
+  def __init__(self, desc, on_packet):
     self._buffer = b''
     self._on_packet = on_packet
+    self._desc = desc
+    self._log('__init__')
+
+  def _log(self, msg, **kwargs):
+    log_debug(DBG.SLIP_FULL, 'slip %s: %s' % (self._desc, msg), **kwargs)
 
   def inputdata(self, data):
-    #print('SLIP-GOT ', repr(data))
-    self._buffer += data
-    packets = slip.decode(self._buffer)
+    self._log('inputdata', d=data)
+    packets = slip.decode(data)
+    packets[0] = self._buffer + packets[0]
     self._buffer = packets.pop()
     for packet in packets:
       self._maybe_packet(packet)
+    self._log('bufremain', d=self._buffer)
 
   def _maybe_packet(self, packet):
-      if len(packet):
-        self._on_packet(packet)
+    self._log('maybepacket', d=packet)
+    if len(packet):
+      self._on_packet(packet)
 
   def flush(self):
+    self._log('flush')
     self._maybe_packet(self._buffer)
     self._buffer = b''
 
 class _IpifProcessProtocol(twisted.internet.protocol.ProcessProtocol):
   def __init__(self, router):
     self._router = router
-    self._decoder = SlipStreamDecoder(self.slip_on_packet)
+    self._decoder = SlipStreamDecoder('ipif', self.slip_on_packet)
   def connectionMade(self): pass
   def outReceived(self, data):
     self._decoder.inputdata(data)
   def slip_on_packet(self, packet):
     (saddr, daddr) = packet_addrs(packet)
     if saddr.is_link_local or daddr.is_link_local:
-      log_discard(packet, saddr, daddr, 'link-local')
+      log_discard(packet, 'ipif', saddr, daddr, 'link-local')
       return
     self._router(packet, saddr, daddr)
   def processEnded(self, status):
@@ -199,6 +213,7 @@ def start_ipif(command, router):
                        env=None)
 
 def queue_inbound(packet):
+  log_debug(DBG.FLOW, "queue_inbound", d=packet)
   ipif.transport.write(slip.delimiter)
   ipif.transport.write(slip.encode(packet))
   ipif.transport.write(slip.delimiter)
@@ -208,10 +223,11 @@ def queue_inbound(packet):
 class PacketQueue():
   def __init__(self, desc, max_queue_time):
     self._desc = desc
+    assert(desc + '')
     self._max_queue_time = max_queue_time
     self._pq = collections.deque() # packets
 
-  def _log(self, dflag, msg, **kwargs)
+  def _log(self, dflag, msg, **kwargs):
     log_debug(dflag, self._desc+' pq: '+msg, **kwargs)
 
   def append(self, packet):
@@ -253,7 +269,7 @@ class PacketQueue():
 
       self._log(DBG.QUEUE_CTRL,
                 'process... (sofar=%d, max=%d) encoded' % (sofar, max_batch),
-                d=encoded
+                d=encoded)
 
       if sofar > 0:
         if sofar + len(slip.delimiter) + len(encoded) > max_batch:
@@ -266,8 +282,13 @@ class PacketQueue():
 
 #---------- error handling ----------
 
+_crashing = False
+
 def crash(err):
-  print('CRASH ', err, file=sys.stderr)
+  global _crashing
+  _crashing = True
+  print('========== CRASH ==========', err,
+        '===========================', file=sys.stderr)
   try: reactor.stop()
   except twisted.internet.error.ReactorNotRunning: pass
 
@@ -290,7 +311,7 @@ def process_cfg_ipif(section, varmap):
     except AttributeError: continue
     setattr(c, d, v)
 
-  print(repr(c))
+  #print(repr(c))
 
   c.ipif_command = cfg.get(section,'ipif', vars=c.__dict__)
 
@@ -363,6 +384,6 @@ def common_startup():
   cfg.read(opts.configfile)
 
 def common_run():
-  log_debug(DBG.INIT, 'ready')
-  reactor.run()
+  log_debug(DBG.INIT, 'entering reactor')
+  if not _crashing: reactor.run()
   print('CRASHED (end)', file=sys.stderr)