X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ian/git?p=secnet.git;a=blobdiff_plain;f=site.c;h=2fafd915b4beeacfa069c9efe8e83f516799bced;hp=6101b2bd32f54b5f0d9d842cb14446762d9a87e0;hb=040ee979539e88ef71fb21a80d4a05d24961ae70;hpb=9d3a4132788b198345116624761c12ed7bc936b6

diff --git a/site.c b/site.c
index 6101b2b..2fafd91 100644
--- a/site.c
+++ b/site.c
@@ -1,21 +1,36 @@
 /* site.c - manage communication with a remote network site */
 
+/* The 'site' code doesn't know anything about the structure of the
+   packets it's transmitting.  In fact, under the new netlink
+   configuration scheme it doesn't need to know anything at all about
+   IP addresses, except how to contact its peer.  This means it could
+   potentially be used to tunnel other protocols too (IPv6, IPX, plain
+   old Ethernet frames) if appropriate netlink code can be written
+   (and that ought not to be too hard, eg. using the TUN/TAP device to
+   pretend to be an Ethernet interface).  */
+
+/* At some point in the future the netlink code will be asked for
+   configuration information to go in the PING/PONG packets at the end
+   of the key exchange. */
+
 #include "secnet.h"
 #include <stdio.h>
-/* MBM asserts the next one is needed for compilation under BSD. */
+#include <string.h>
+#include <limits.h>
 #include <sys/socket.h>
 
 #include <sys/mman.h>
 #include "util.h"
 #include "unaligned.h"
+#include "magic.h"
 
 #define SETUP_BUFFER_LEN 2048
 
 #define DEFAULT_KEY_LIFETIME 3600000 /* One hour */
 #define DEFAULT_KEY_RENEGOTIATE_GAP 300000 /* Five minutes */
 #define DEFAULT_SETUP_RETRIES 5
-#define DEFAULT_SETUP_TIMEOUT 1000
-#define DEFAULT_WAIT_TIME 10000
+#define DEFAULT_SETUP_TIMEOUT 2000
+#define DEFAULT_WAIT_TIME 20000
 
 /* Each site can be in one of several possible states. */
 
@@ -64,34 +79,21 @@
 #define SITE_SENTMSG5 7
 #define SITE_WAIT     8
 
-#if 0
-static string_t state_name(uint32_t state)
+static cstring_t state_name(uint32_t state)
 {
     switch (state) {
-    case 0: return "SITE_STOP";
-    case 1: return "SITE_RUN";
-    case 2: return "SITE_RESOLVE";
-    case 3: return "SITE_SENTMSG1";
-    case 4: return "SITE_SENTMSG2";
-    case 5: return "SITE_SENTMSG3";
-    case 6: return "SITE_SENTMSG4";
-    case 7: return "SITE_SENTMSG5";
-    case 8: return "SITE_WAIT";
+    case 0: return "STOP";
+    case 1: return "RUN";
+    case 2: return "RESOLVE";
+    case 3: return "SENTMSG1";
+    case 4: return "SENTMSG2";
+    case 5: return "SENTMSG3";
+    case 6: return "SENTMSG4";
+    case 7: return "SENTMSG5";
+    case 8: return "WAIT";
     default: return "*bad state*";
     }
 }
-#endif /* 0 */
-
-#define LABEL_MSG0 0x00020200
-#define LABEL_MSG1 0x01010101
-#define LABEL_MSG2 0x02020202
-#define LABEL_MSG3 0x03030303
-#define LABEL_MSG4 0x04040404
-#define LABEL_MSG5 0x05050505
-#define LABEL_MSG6 0x06060606
-#define LABEL_MSG7 0x07070707
-#define LABEL_MSG8 0x08080808
-#define LABEL_MSG9 0x09090909
 
 #define NONCELEN 8
 
@@ -117,6 +119,8 @@ static struct flagstr log_event_table[]={
     { "packet-drop", LOG_DROP },
     { "dump-packets", LOG_DUMP },
     { "errors", LOG_ERROR },
+    { "default", LOG_SETUP_INIT|LOG_SETUP_TIMEOUT|
+      LOG_ACTIVATE_KEY|LOG_TIMEOUT_KEY|LOG_SEC|LOG_ERROR },
     { "all", 0xffffffff },
     { NULL, 0 }
 };
@@ -127,16 +131,15 @@ struct site {
 /* configuration information */
     string_t localname;
     string_t remotename;
-    string_t tunname; /* localname<->remotename by default */
+    string_t tunname; /* localname<->remotename by default, used in logs */
     string_t address; /* DNS name for bootstrapping, optional */
-    int remoteport;
+    int remoteport; /* Port for bootstrapping, optional */
     struct netlink_if *netlink;
     struct comm_if *comm;
     struct resolver_if *resolver;
     struct log_if *log;
     struct random_if *random;
     struct rsaprivkey_if *privkey;
-    struct subnet_list remotenets;
     struct rsapubkey_if *pubkey;
     struct transform_if *transform;
     struct dh_if *dh;
@@ -149,8 +152,8 @@ struct site {
     uint32_t key_renegotiate_time; /* If we see traffic (or a keepalive)
 				      after this time, initiate a new
 				      key exchange */
-
-    bool_t keepalive; /* Always keep the tunnel up */
+    bool_t keepalive; /* Send keepalives to detect peer failure (not yet
+			 implemented) */
 
     uint8_t *setupsig; /* Expected signature of incoming MSG1 packets */
     uint32_t setupsiglen; /* Allows us to discard packets quickly if
@@ -163,16 +166,22 @@ struct site {
     uint32_t state;
     uint64_t now; /* Most recently seen time */
 
-    void *netlink_cid;
-
+    /* The currently established session */
     uint32_t remote_session_id;
     struct transform_inst_if *current_transform;
     bool_t current_valid;
     uint64_t current_key_timeout; /* End of life of current key */
+    uint64_t renegotiate_key_time; /* When we can negotiate a new key */
     struct sockaddr_in peer; /* Current address of peer */
     bool_t peer_valid; /* Peer address becomes invalid when key times out,
 			  but only if we have a DNS name for our peer */
 
+    /* The current key setup protocol exchange.  We can only be
+       involved in one of these at a time.  There's a potential for
+       denial of service here (the attacker keeps sending a setup
+       packet; we keep trying to continue the exchange, and have to
+       timeout before we can listen for another setup packet); perhaps
+       we should keep a list of 'bad' sources for setup packets. */
     uint32_t setup_session_id;
     struct sockaddr_in setup_peer;
     uint8_t localN[NONCELEN]; /* Nonces for key exchange */
@@ -182,33 +191,44 @@ struct site {
     uint64_t timeout; /* Timeout for current state */
     uint8_t *dhsecret;
     uint8_t *sharedsecret;
-
     struct transform_inst_if *new_transform; /* For key setup/verify */
 };
 
-static void slog(struct site *st, uint32_t event, string_t msg, ...)
+static void slog(struct site *st, uint32_t event, cstring_t msg, ...)
 {
     va_list ap;
     uint8_t buf[240];
+    uint32_t class;
 
     va_start(ap,msg);
 
     if (event&st->log_events) {
+	switch(event) {
+	case LOG_UNEXPECTED: class=M_INFO; break;
+	case LOG_SETUP_INIT: class=M_INFO; break;
+	case LOG_SETUP_TIMEOUT: class=M_NOTICE; break;
+	case LOG_ACTIVATE_KEY: class=M_INFO; break;
+	case LOG_TIMEOUT_KEY: class=M_INFO; break;
+	case LOG_SEC: class=M_SECURITY; break;
+	case LOG_STATE: class=M_DEBUG; break;
+	case LOG_DROP: class=M_DEBUG; break;
+	case LOG_DUMP: class=M_DEBUG; break;
+	case LOG_ERROR: class=M_ERR; break;
+	default: class=M_ERR; break;
+	}
+
 	vsnprintf(buf,240,msg,ap);
-	st->log->log(st->log->st,0,"%s: %s",st->tunname,buf);
+	st->log->log(st->log->st,class,"%s: %s",st->tunname,buf);
     }
     va_end(ap);
 }
 
 static void set_link_quality(struct site *st);
-static bool_t initiate_key_setup(struct site *st);
+static void delete_key(struct site *st, cstring_t reason, uint32_t loglevel);
+static bool_t initiate_key_setup(struct site *st, cstring_t reason);
 static void enter_state_run(struct site *st);
 static bool_t enter_state_resolve(struct site *st);
-static bool_t enter_state_sentmsg1(struct site *st);
-static bool_t enter_state_sentmsg2(struct site *st);
-static bool_t enter_state_sentmsg3(struct site *st);
-static bool_t enter_state_sentmsg4(struct site *st);
-static bool_t enter_state_sentmsg5(struct site *st);
+static bool_t enter_new_state(struct site *st,uint32_t next);
 static void enter_state_wait(struct site *st);
 
 #define CHECK_AVAIL(b,l) do { if ((b)->size<(l)) return False; } while(0)
@@ -235,12 +255,12 @@ struct msg {
     uint8_t *sig;
 };
 
-/* Build any of msg1 to msg4. msg5 and msg6 are built from the inside out
-   using a transform. */
-static bool_t generate_msg(struct site *st, uint32_t type, string_t what)
+/* Build any of msg1 to msg4. msg5 and msg6 are built from the inside
+   out using a transform of config data supplied by netlink */
+static bool_t generate_msg(struct site *st, uint32_t type, cstring_t what)
 {
     void *hst;
-    uint8_t *hash=alloca(st->hash->len);
+    uint8_t *hash;
     string_t dhpub, sig;
 
     st->retries=st->setup_retries;
@@ -256,15 +276,20 @@ static bool_t generate_msg(struct site *st, uint32_t type, string_t what)
     if (type==LABEL_MSG1) return True;
     memcpy(buf_append(&st->buffer,NONCELEN),st->remoteN,NONCELEN);
     if (type==LABEL_MSG2) return True;
+
+    if (hacky_par_mid_failnow()) return False;
+
     dhpub=st->dh->makepublic(st->dh->st,st->dhsecret,st->dh->len);
     buf_append_string(&st->buffer,dhpub);
     free(dhpub);
+    hash=safe_malloc(st->hash->len, "generate_msg");
     hst=st->hash->init();
     st->hash->update(hst,st->buffer.start,st->buffer.size);
     st->hash->final(hst,hash);
     sig=st->privkey->sign(st->privkey->st,hash,st->hash->len);
     buf_append_string(&st->buffer,sig);
     free(sig);
+    free(hash);
     return True;
 }
 
@@ -310,6 +335,36 @@ static bool_t unpick_msg(struct site *st, uint32_t type,
     return True;
 }
 
+static bool_t check_msg(struct site *st, uint32_t type, struct msg *m,
+			cstring_t *error)
+{
+    if (type==LABEL_MSG1) return True;
+
+    /* Check that the site names and our nonce have been sent
+       back correctly, and then store our peer's nonce. */ 
+    if (memcmp(m->remote,st->remotename,strlen(st->remotename)!=0)) {
+	*error="wrong remote site name";
+	return False;
+    }
+    if (memcmp(m->local,st->localname,strlen(st->localname)!=0)) {
+	*error="wrong local site name";
+	return False;
+    }
+    if (memcmp(m->nL,st->localN,NONCELEN)!=0) {
+	*error="wrong locally-generated nonce";
+	return False;
+    }
+    if (type==LABEL_MSG2) return True;
+    if (memcmp(m->nR,st->remoteN,NONCELEN)!=0) {
+	*error="wrong remotely-generated nonce";
+	return False;
+    }
+    if (type==LABEL_MSG3) return True;
+    if (type==LABEL_MSG4) return True;
+    *error="unknown message type";
+    return False;
+}
+
 static bool_t generate_msg1(struct site *st)
 {
     st->random->generate(st->random->st,NONCELEN,st->localN);
@@ -327,9 +382,7 @@ static bool_t process_msg1(struct site *st, struct buffer_if *msg1,
 
     if (!unpick_msg(st,LABEL_MSG1,msg1,&m)) return False;
 
-    /* XXX save src as our peer address here? */
     st->setup_peer=*src;
-
     st->setup_session_id=m.source;
     memcpy(st->remoteN,m.nR,NONCELEN);
     return True;
@@ -345,21 +398,11 @@ static bool_t process_msg2(struct site *st, struct buffer_if *msg2,
 			   struct sockaddr_in *src)
 {
     struct msg m;
+    cstring_t err;
 
     if (!unpick_msg(st,LABEL_MSG2,msg2,&m)) return False;
-
-    /* Check that the site names and our nonce have been sent
-       back correctly, and then store our peer's nonce. */ 
-    if (memcmp(m.remote,st->remotename,strlen(st->remotename)!=0)) {
-	slog(st,LOG_SEC,"msg2: bad B (remote site name)");
-	return False;
-    }
-    if (memcmp(m.local,st->localname,strlen(st->localname)!=0)) {
-	slog(st,LOG_SEC,"msg2: bad A (local site name)");
-	return False;
-    }
-    if (memcmp(m.nL,st->localN,NONCELEN)!=0) {
-	slog(st,LOG_SEC,"msg2: bad nA (locally generated nonce)");
+    if (!check_msg(st,LABEL_MSG2,&m,&err)) {
+	slog(st,LOG_SEC,"msg2: %s",err);
 	return False;
     }
     st->setup_session_id=m.source;
@@ -379,31 +422,18 @@ static bool_t process_msg3(struct site *st, struct buffer_if *msg3,
 			   struct sockaddr_in *src)
 {
     struct msg m;
-    uint8_t *hash=alloca(st->hash->len);
+    uint8_t *hash;
     void *hst;
+    cstring_t err;
 
     if (!unpick_msg(st,LABEL_MSG3,msg3,&m)) return False;
-
-    /* Check that the site names and nonces have been sent back
-       correctly */
-    if (memcmp(m.remote,st->remotename,strlen(st->remotename)!=0)) {
-	slog(st,LOG_SEC,"msg3: bad A (remote site name)");
-	return False;
-    }
-    if (memcmp(m.local,st->localname,strlen(st->localname)!=0)) {
-	slog(st,LOG_SEC,"msg3: bad B (local site name)");
-	return False;
-    }
-    if (memcmp(m.nR,st->remoteN,NONCELEN)!=0) {
-	slog(st,LOG_SEC,"msg3: bad nA (remotely generated nonce)");
+    if (!check_msg(st,LABEL_MSG3,&m,&err)) {
+	slog(st,LOG_SEC,"msg3: %s",err);
 	return False;
     }
-    if (memcmp(m.nL,st->localN,NONCELEN)!=0) {
-	slog(st,LOG_SEC,"msg3: bad nB (locally generated nonce)");
-	return False;
-    }
-    
+
     /* Check signature and store g^x mod m */
+    hash=safe_malloc(st->hash->len, "process_msg3");
     hst=st->hash->init();
     st->hash->update(hst,m.hashstart,m.hashlen);
     st->hash->final(hst,hash);
@@ -411,8 +441,10 @@ static bool_t process_msg3(struct site *st, struct buffer_if *msg3,
     m.sig[m.siglen]=0;
     if (!st->pubkey->check(st->pubkey->st,hash,st->hash->len,m.sig)) {
 	slog(st,LOG_SEC,"msg3 signature failed check!");
+	free(hash);
 	return False;
     }
+    free(hash);
 
     /* Terminate their DH public key with a '0' */
     m.pk[m.pklen]=0;
@@ -441,31 +473,18 @@ static bool_t process_msg4(struct site *st, struct buffer_if *msg4,
 			   struct sockaddr_in *src)
 {
     struct msg m;
-    uint8_t *hash=alloca(st->hash->len);
+    uint8_t *hash;
     void *hst;
+    cstring_t err;
 
     if (!unpick_msg(st,LABEL_MSG4,msg4,&m)) return False;
-
-    /* Check that the site names and nonces have been sent back
-       correctly */
-    if (memcmp(m.remote,st->remotename,strlen(st->remotename)!=0)) {
-	slog(st,LOG_SEC,"msg4: bad B (remote site name)");
-	return False;
-    }
-    if (memcmp(m.local,st->localname,strlen(st->localname)!=0)) {
-	slog(st,LOG_SEC,"msg4: bad A (local site name)");
-	return False;
-    }
-    if (memcmp(m.nR,st->remoteN,NONCELEN)!=0) {
-	slog(st,LOG_SEC,"msg4: bad nB (remotely generated nonce)");
-	return False;
-    }
-    if (memcmp(m.nL,st->localN,NONCELEN)!=0) {
-	slog(st,LOG_SEC,"msg4: bad nA (locally generated nonce)");
+    if (!check_msg(st,LABEL_MSG4,&m,&err)) {
+	slog(st,LOG_SEC,"msg4: %s",err);
 	return False;
     }
     
     /* Check signature and store g^x mod m */
+    hash=safe_malloc(st->hash->len, "process_msg4");
     hst=st->hash->init();
     st->hash->update(hst,m.hashstart,m.hashlen);
     st->hash->final(hst,hash);
@@ -473,8 +492,10 @@ static bool_t process_msg4(struct site *st, struct buffer_if *msg4,
     m.sig[m.siglen]=0;
     if (!st->pubkey->check(st->pubkey->st,hash,st->hash->len,m.sig)) {
 	slog(st,LOG_SEC,"msg4 signature failed check!");
+	free(hash);
 	return False;
     }
+    free(hash);
 
     /* Terminate their DH public key with a '0' */
     m.pk[m.pklen]=0;
@@ -488,24 +509,6 @@ static bool_t process_msg4(struct site *st, struct buffer_if *msg4,
     return True;
 }
 
-static bool_t generate_msg5(struct site *st)
-{
-    string_t transform_err;
-
-    BUF_ALLOC(&st->buffer,"site:MSG5");
-    /* We are going to add three words to the transformed message */
-    buffer_init(&st->buffer,st->transform->max_start_pad+(4*3));
-    buf_append_uint32(&st->buffer,LABEL_MSG5);
-    st->new_transform->forwards(st->new_transform->st,&st->buffer,
-				&transform_err);
-    buf_prepend_uint32(&st->buffer,LABEL_MSG5);
-    buf_prepend_uint32(&st->buffer,(uint32_t)st);
-    buf_prepend_uint32(&st->buffer,st->setup_session_id);
-
-    st->retries=st->setup_retries;
-    return True;
-}
-
 struct msg0 {
     uint32_t dest;
     uint32_t source;
@@ -525,11 +528,32 @@ static bool_t unpick_msg0(struct site *st, struct buffer_if *msg0,
     /* Leaves transformed part of buffer untouched */
 }
 
+static bool_t generate_msg5(struct site *st)
+{
+    cstring_t transform_err;
+
+    BUF_ALLOC(&st->buffer,"site:MSG5");
+    /* We are going to add four words to the message */
+    buffer_init(&st->buffer,st->transform->max_start_pad+(4*4));
+    /* Give the netlink code an opportunity to put its own stuff in the
+       message (configuration information, etc.) */
+    st->netlink->output_config(st->netlink->st,&st->buffer);
+    buf_prepend_uint32(&st->buffer,LABEL_MSG5);
+    st->new_transform->forwards(st->new_transform->st,&st->buffer,
+				&transform_err);
+    buf_prepend_uint32(&st->buffer,LABEL_MSG5);
+    buf_prepend_uint32(&st->buffer,(uint32_t)st);
+    buf_prepend_uint32(&st->buffer,st->setup_session_id);
+
+    st->retries=st->setup_retries;
+    return True;
+}
+
 static bool_t process_msg5(struct site *st, struct buffer_if *msg5,
 			   struct sockaddr_in *src)
 {
     struct msg0 m;
-    string_t transform_err;
+    cstring_t transform_err;
 
     if (!unpick_msg0(st,msg5,&m)) return False;
 
@@ -542,7 +566,11 @@ static bool_t process_msg5(struct site *st, struct buffer_if *msg5,
     /* Buffer should now contain untransformed PING packet data */
     CHECK_AVAIL(msg5,4);
     if (buf_unprepend_uint32(msg5)!=LABEL_MSG5) {
-	slog(st,LOG_SEC,"MSG5/PING packet contained invalid data");
+	slog(st,LOG_SEC,"MSG5/PING packet contained wrong label");
+	return False;
+    }
+    if (!st->netlink->check_config(st->netlink->st,msg5)) {
+	slog(st,LOG_SEC,"MSG5/PING packet contained bad netlink config");
 	return False;
     }
     CHECK_EMPTY(msg5);
@@ -551,19 +579,22 @@ static bool_t process_msg5(struct site *st, struct buffer_if *msg5,
 
 static bool_t generate_msg6(struct site *st)
 {
-    string_t transform_err;
+    cstring_t transform_err;
 
     BUF_ALLOC(&st->buffer,"site:MSG6");
-    /* We are going to add three words to the transformed message */
-    buffer_init(&st->buffer,st->transform->max_start_pad+(4*3));
-    buf_append_uint32(&st->buffer,LABEL_MSG6);
+    /* We are going to add four words to the message */
+    buffer_init(&st->buffer,st->transform->max_start_pad+(4*4));
+    /* Give the netlink code an opportunity to put its own stuff in the
+       message (configuration information, etc.) */
+    st->netlink->output_config(st->netlink->st,&st->buffer);
+    buf_prepend_uint32(&st->buffer,LABEL_MSG6);
     st->new_transform->forwards(st->new_transform->st,&st->buffer,
 				&transform_err);
     buf_prepend_uint32(&st->buffer,LABEL_MSG6);
     buf_prepend_uint32(&st->buffer,(uint32_t)st);
     buf_prepend_uint32(&st->buffer,st->setup_session_id);
 
-    st->retries=1; /* Peer will retransmit MSG5 if necessary */
+    st->retries=1; /* Peer will retransmit MSG5 if this packet gets lost */
     return True;
 }
 
@@ -571,7 +602,7 @@ static bool_t process_msg6(struct site *st, struct buffer_if *msg6,
 			   struct sockaddr_in *src)
 {
     struct msg0 m;
-    string_t transform_err;
+    cstring_t transform_err;
 
     if (!unpick_msg0(st,msg6,&m)) return False;
 
@@ -587,6 +618,10 @@ static bool_t process_msg6(struct site *st, struct buffer_if *msg6,
 	slog(st,LOG_SEC,"MSG6/PONG packet contained invalid data");
 	return False;
     }
+    if (!st->netlink->check_config(st->netlink->st,msg6)) {
+	slog(st,LOG_SEC,"MSG6/PONG packet contained bad netlink config");
+	return False;
+    }
     CHECK_EMPTY(msg6);
     return True;
 }
@@ -595,12 +630,12 @@ static bool_t process_msg0(struct site *st, struct buffer_if *msg0,
 			   struct sockaddr_in *src)
 {
     struct msg0 m;
-    string_t transform_err;
+    cstring_t transform_err;
     uint32_t type;
 
     if (!st->current_valid) {
 	slog(st,LOG_DROP,"incoming message but no current key -> dropping");
-	return initiate_key_setup(st);
+	return initiate_key_setup(st,"incoming message but no current key");
     }
 
     if (!unpick_msg0(st,msg0,&m)) return False;
@@ -609,18 +644,24 @@ static bool_t process_msg0(struct site *st, struct buffer_if *msg0,
 				       msg0,&transform_err)) {
 	/* There's a problem */
 	slog(st,LOG_SEC,"transform: %s",transform_err);
-	return False;
+	return initiate_key_setup(st,"incoming message would not decrypt");
     }
     CHECK_AVAIL(msg0,4);
     type=buf_unprepend_uint32(msg0);
     switch(type) {
+    case LABEL_MSG7:
+	/* We must forget about the current session. */
+	delete_key(st,"request from peer",LOG_SEC);
+	return True;
+	break;
     case LABEL_MSG9:
 	/* Deliver to netlink layer */
-	st->netlink->deliver(st->netlink->st,st->netlink_cid,msg0);
+	st->netlink->deliver(st->netlink->st,msg0);
 	return True;
 	break;
     default:
-	slog(st,LOG_SEC,"incoming message of type %08x (unknown)",type);
+	slog(st,LOG_SEC,"incoming encrypted message of type %08x "
+	     "(unknown)",type);
 	break;
     }
     return False;
@@ -634,9 +675,9 @@ static void dump_packet(struct site *st, struct buffer_if *buf,
     uint32_t msgtype=ntohl(*(uint32_t *)(buf->start+8));
 
     if (st->log_events & LOG_DUMP)
-	log(st->log,0,"%s: %s: %08x<-%08x: %08x:",
-	    st->tunname,incoming?"incoming":"outgoing",
-	    dest,source,msgtype);
+	slilog(st->log,M_DEBUG,"%s: %s: %08x<-%08x: %08x:",
+	       st->tunname,incoming?"incoming":"outgoing",
+	       dest,source,msgtype);
 }
 
 static uint32_t site_status(void *st)
@@ -653,7 +694,8 @@ static bool_t send_msg(struct site *st)
 	st->retries--;
 	return True;
     } else {
-	slog(st,LOG_SETUP_TIMEOUT,"timed out sending key setup packet");
+	slog(st,LOG_SETUP_TIMEOUT,"timed out sending key setup packet "
+	    "(in state %s)",state_name(st->state));
 	enter_state_wait(st);
 	return False;
     }
@@ -672,7 +714,7 @@ static void site_resolve_callback(void *sst, struct in_addr *address)
 	st->setup_peer.sin_family=AF_INET;
 	st->setup_peer.sin_port=htons(st->remoteport);
 	st->setup_peer.sin_addr=*address;
-	enter_state_sentmsg1(st);
+	enter_new_state(st,SITE_SENTMSG1);
     } else {
 	/* Resolution failed */
 	slog(st,LOG_ERROR,"resolution of %s failed",st->address);
@@ -680,18 +722,19 @@ static void site_resolve_callback(void *sst, struct in_addr *address)
     }
 }
 
-static bool_t initiate_key_setup(struct site *st)
+static bool_t initiate_key_setup(struct site *st, cstring_t reason)
 {
     if (st->state!=SITE_RUN) return False;
+    slog(st,LOG_SETUP_INIT,"initiating key exchange (%s)",reason);
     if (st->address) {
-	slog(st,LOG_SETUP_INIT,"initiating key exchange; resolving address");
+	slog(st,LOG_SETUP_INIT,"resolving peer address");
 	return enter_state_resolve(st);
     } else if (st->peer_valid) {
-	slog(st,LOG_SETUP_INIT,"initiating key exchange using old "
-	     "peer address");
+	slog(st,LOG_SETUP_INIT,"using old peer address");
 	st->setup_peer=st->peer;
-	return enter_state_sentmsg1(st);
+	return enter_new_state(st,SITE_SENTMSG1);
     }
+    slog(st,LOG_SETUP_INIT,"key exchange failed: no address for peer");
     return False;
 }
 
@@ -699,6 +742,8 @@ static void activate_new_key(struct site *st)
 {
     struct transform_inst_if *t;
 
+    /* We have two transform instances, which we swap between active
+       and setup */
     t=st->current_transform;
     st->current_transform=st->new_transform;
     st->new_transform=t;
@@ -707,6 +752,7 @@ static void activate_new_key(struct site *st)
     st->timeout=0;
     st->current_valid=True;
     st->current_key_timeout=st->now+st->key_lifetime;
+    st->renegotiate_key_time=st->now+st->key_renegotiate_time;
     st->peer=st->setup_peer;
     st->peer_valid=True;
     st->remote_session_id=st->setup_session_id;
@@ -715,23 +761,28 @@ static void activate_new_key(struct site *st)
     enter_state_run(st);
 }
 
+static void delete_key(struct site *st, cstring_t reason, uint32_t loglevel)
+{
+    if (st->current_valid) {
+	slog(st,loglevel,"session closed (%s)",reason);
+
+	st->current_valid=False;
+	st->current_transform->delkey(st->current_transform->st);
+	st->current_key_timeout=0;
+	set_link_quality(st);
+    }
+}
+
 static void state_assert(struct site *st, bool_t ok)
 {
-    if (!ok) fatal("state_assert\n");
+    if (!ok) fatal("site:state_assert");
 }
 
 static void enter_state_stop(struct site *st)
 {
     st->state=SITE_STOP;
     st->timeout=0;
-    st->current_transform->delkey(st->current_transform->st);
-    st->current_valid=False;
-    st->current_key_timeout=0;
-    
-    st->peer_valid=False;
-
-    set_link_quality(st);
-    
+    delete_key(st,"entering state STOP",LOG_TIMEOUT_KEY);
     st->new_transform->delkey(st->new_transform->st);
 }
 
@@ -749,7 +800,7 @@ static void set_link_quality(struct site *st)
     else
 	quality=LINK_QUALITY_DOWN;
 
-    st->netlink->set_quality(st->netlink->st,st->netlink_cid,quality);
+    st->netlink->set_quality(st->netlink->st,quality);
 }
 
 static void enter_state_run(struct site *st)
@@ -758,8 +809,14 @@ static void enter_state_run(struct site *st)
     st->state=SITE_RUN;
     st->timeout=0;
 
+    st->setup_session_id=0;
+    memset(&st->setup_peer,0,sizeof(st->setup_peer));
+    memset(st->localN,0,NONCELEN);
+    memset(st->remoteN,0,NONCELEN);
+    st->new_transform->delkey(st->new_transform->st);
+    memset(st->dhsecret,0,st->dh->len);
+    memset(st->sharedsecret,0,st->transform->keylen);
     set_link_quality(st);
-    /* XXX get rid of key setup data */
 }
 
 static bool_t enter_state_resolve(struct site *st)
@@ -772,97 +829,92 @@ static bool_t enter_state_resolve(struct site *st)
     return True;
 }
 
-static bool_t enter_state_sentmsg1(struct site *st)
-{
-    state_assert(st,st->state==SITE_RUN || st->state==SITE_RESOLVE);
-    slog(st,LOG_STATE,"entering state SENTMSG1");
-    if (generate_msg1(st) && send_msg(st)) {
-	st->state=SITE_SENTMSG1;
-	return True;
-    }
-    slog(st,LOG_ERROR,"error entering state SENTMSG1");
-    st->buffer.free=False; /* Can't tell which it was, but enter_state_wait()
-			      will do a BUF_FREE() */
-    enter_state_wait(st);
-    return False;
-}
-
-static bool_t enter_state_sentmsg2(struct site *st)
-{
-    state_assert(st,st->state==SITE_RUN || st->state==SITE_RESOLVE ||
-		 st->state==SITE_SENTMSG1 || st->state==SITE_WAIT);
-    slog(st,LOG_STATE,"entering state SENTMSG2");
-    if (generate_msg2(st) && send_msg(st)) {
-	st->state=SITE_SENTMSG2;
-	return True;
-    }
-    slog(st,LOG_ERROR,"error entering state SENTMSG2");
-    st->buffer.free=False;
-    enter_state_wait(st);
-    return False;
-}
-
-static bool_t enter_state_sentmsg3(struct site *st)
+static bool_t enter_new_state(struct site *st, uint32_t next)
 {
-    state_assert(st,st->state==SITE_SENTMSG1);
-    slog(st,LOG_STATE,"entering state SENTMSG3");
-    BUF_FREE(&st->buffer); /* Free message 1 */
-    if (generate_msg3(st) && send_msg(st)) {
-	st->state=SITE_SENTMSG3;
-	return True;
+    bool_t (*gen)(struct site *st);
+    int r;
+
+    slog(st,LOG_STATE,"entering state %s",state_name(next));
+    switch(next) {
+    case SITE_SENTMSG1:
+	state_assert(st,st->state==SITE_RUN || st->state==SITE_RESOLVE);
+	gen=generate_msg1;
+	break;
+    case SITE_SENTMSG2:
+	state_assert(st,st->state==SITE_RUN || st->state==SITE_RESOLVE ||
+		     st->state==SITE_SENTMSG1 || st->state==SITE_WAIT);
+	gen=generate_msg2;
+	break;
+    case SITE_SENTMSG3:
+	state_assert(st,st->state==SITE_SENTMSG1);
+	BUF_FREE(&st->buffer);
+	gen=generate_msg3;
+	break;
+    case SITE_SENTMSG4:
+	state_assert(st,st->state==SITE_SENTMSG2);
+	BUF_FREE(&st->buffer);
+	gen=generate_msg4;
+	break;
+    case SITE_SENTMSG5:
+	state_assert(st,st->state==SITE_SENTMSG3);
+	BUF_FREE(&st->buffer);
+	gen=generate_msg5;
+	break;
+    case SITE_RUN:
+	state_assert(st,st->state==SITE_SENTMSG4);
+	BUF_FREE(&st->buffer);
+	gen=generate_msg6;
+	break;
+    default:
+	gen=NULL;
+	fatal("enter_new_state(%s): invalid new state",state_name(next));
+	break;
     }
-    slog(st,LOG_ERROR,"error entering state SENTMSG3");
-    st->buffer.free=False;
-    enter_state_wait(st);
-    return False;
-}
 
-static bool_t enter_state_sentmsg4(struct site *st)
-{
-    state_assert(st,st->state==SITE_SENTMSG2);
-    slog(st,LOG_STATE,"entering state SENTMSG4");
-    BUF_FREE(&st->buffer); /* Free message 2 */
-    if (generate_msg4(st) && send_msg(st)) {
-	st->state=SITE_SENTMSG4;
-	return True;
-    }
-    slog(st,LOG_ERROR,"error entering state SENTMSG4");
-    st->buffer.free=False;
-    enter_state_wait(st);
-    return False;
-}
+    if (hacky_par_start_failnow()) return False;
 
-static bool_t enter_state_sentmsg5(struct site *st)
-{
-    state_assert(st,st->state==SITE_SENTMSG3);
-    slog(st,LOG_STATE,"entering state SENTMSG5");
-    BUF_FREE(&st->buffer); /* Free message 3 */
+    r= gen(st) && send_msg(st);
 
-    if (generate_msg5(st) && send_msg(st)) {
-	st->state=SITE_SENTMSG5;
+    hacky_par_end(&r,
+		  st->setup_retries, st->setup_timeout,
+		  send_msg, st);
+    
+    if (r) {
+	st->state=next;
+	if (next==SITE_RUN) {
+	    BUF_FREE(&st->buffer); /* Never reused */
+	    st->timeout=0; /* Never retransmit */
+	    activate_new_key(st);
+	}
 	return True;
     }
-    slog(st,LOG_ERROR,"error entering state SENTMSG5");
-    st->buffer.free=False;
+    slog(st,LOG_ERROR,"error entering state %s",state_name(next));
+    st->buffer.free=False; /* Unconditionally use the buffer; it may be
+			      in either state, and enter_state_wait() will
+			      do a BUF_FREE() */
     enter_state_wait(st);
-    
     return False;
 }
 
-static bool_t send_msg6(struct site *st)
+/* msg7 tells our peer that we're about to forget our key */
+static bool_t send_msg7(struct site *st, cstring_t reason)
 {
-    state_assert(st,st->state==SITE_SENTMSG4);
-    slog(st,LOG_STATE,"entering state RUN after sending msg6");
-    BUF_FREE(&st->buffer); /* Free message 4 */
-    if (generate_msg6(st) && send_msg(st)) {
-	BUF_FREE(&st->buffer); /* Never reused */
-	st->timeout=0; /* Never retransmit */
-	activate_new_key(st);
+    cstring_t transform_err;
+
+    if (st->current_valid && st->peer_valid && st->buffer.free) {
+	BUF_ALLOC(&st->buffer,"site:MSG7");
+	buffer_init(&st->buffer,st->transform->max_start_pad+(4*3));
+	buf_append_uint32(&st->buffer,LABEL_MSG7);
+	buf_append_string(&st->buffer,reason);
+	st->current_transform->forwards(st->current_transform->st,
+					&st->buffer, &transform_err);
+	buf_prepend_uint32(&st->buffer,LABEL_MSG0);
+	buf_prepend_uint32(&st->buffer,(uint32_t)st);
+	buf_prepend_uint32(&st->buffer,st->remote_session_id);
+	st->comm->sendmsg(st->comm->st,&st->buffer,&st->peer);
+	BUF_FREE(&st->buffer);
 	return True;
     }
-    slog(st,LOG_ERROR,"error entering state RUN after sending msg6");
-    st->buffer.free=False;
-    enter_state_wait(st);
     return False;
 }
 
@@ -880,6 +932,17 @@ static void enter_state_wait(struct site *st)
     /* XXX Erase keys etc. */
 }
 
+static inline void site_settimeout(uint64_t timeout, uint64_t *now,
+				   int *timeout_io)
+{
+    if (timeout) {
+	uint64_t offset=timeout-*now;
+	if (offset>INT_MAX) offset=INT_MAX;
+	if (*timeout_io<0 || offset<*timeout_io)
+	    *timeout_io=offset;
+    }
+}
+
 static int site_beforepoll(void *sst, struct pollfd *fds, int *nfds_io,
 			   int *timeout_io, const struct timeval *tv_now,
 			   uint64_t *now)
@@ -892,12 +955,8 @@ static int site_beforepoll(void *sst, struct pollfd *fds, int *nfds_io,
     /* Work out when our next timeout is. The earlier of 'timeout' or
        'current_key_timeout'. A stored value of '0' indicates no timeout
        active. */
-    if (st->timeout && st->timeout-*now < *timeout_io) {
-	*timeout_io=st->timeout-*now;
-    }
-
-    if (st->current_key_timeout && st->current_key_timeout-*now < *timeout_io)
-	*timeout_io=st->current_key_timeout-*now;
+    site_settimeout(st->timeout, now, timeout_io);
+    site_settimeout(st->current_key_timeout, now, timeout_io);
 
     return 0; /* success */
 }
@@ -910,11 +969,11 @@ static void site_afterpoll(void *sst, struct pollfd *fds, int nfds,
 
     st->now=*now;
     if (st->timeout && *now>st->timeout) {
-	/* Do stuff */
 	st->timeout=0;
-	if (st->state>=SITE_SENTMSG1 && st->state<=SITE_SENTMSG5)
-	    send_msg(st);
-	else if (st->state==SITE_WAIT) {
+	if (st->state>=SITE_SENTMSG1 && st->state<=SITE_SENTMSG5) {
+	    if (!hacky_par_start_failnow())
+	        send_msg(st);
+	} else if (st->state==SITE_WAIT) {
 	    enter_state_run(st);
 	} else {
 	    slog(st,LOG_ERROR,"site_afterpoll: unexpected timeout, state=%d",
@@ -922,21 +981,17 @@ static void site_afterpoll(void *sst, struct pollfd *fds, int nfds,
 	}
     }
     if (st->current_key_timeout && *now>st->current_key_timeout) {
-	slog(st,LOG_TIMEOUT_KEY,"maximum key life exceeded; session closed");
-	st->current_valid=False;
-	st->current_transform->delkey(st->current_transform->st);
-	st->current_key_timeout=0;
-	set_link_quality(st);
+	delete_key(st,"maximum key life exceeded",LOG_TIMEOUT_KEY);
     }
 }
 
 /* This function is called by the netlink device to deliver packets
    intended for the remote network. The packet is in "raw" wire
    format, but is guaranteed to be word-aligned. */
-static void site_outgoing(void *sst, void *cid, struct buffer_if *buf)
+static void site_outgoing(void *sst, struct buffer_if *buf)
 {
     struct site *st=sst;
-    string_t transform_err;
+    cstring_t transform_err;
     
     if (st->state==SITE_STOP) {
 	BUF_FREE(buf);
@@ -957,12 +1012,15 @@ static void site_outgoing(void *sst, void *cid, struct buffer_if *buf)
 	    st->comm->sendmsg(st->comm->st,buf,&st->peer);
 	}
 	BUF_FREE(buf);
+	/* See whether we should start negotiating a new key */
+	if (st->now > st->renegotiate_key_time)
+	    initiate_key_setup(st,"outgoing packet in renegotiation window");
 	return;
     }
 
     slog(st,LOG_DROP,"discarding outgoing packet of size %d",buf->size);
     BUF_FREE(buf);
-    initiate_key_setup(st);
+    initiate_key_setup(st,"outgoing packet");
 }
 
 /* This function is called by the communication device to deliver
@@ -974,25 +1032,24 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
     uint32_t dest=ntohl(*(uint32_t *)buf->start);
 
     if (dest==0) {
-	if (buf->size<(st->setupsiglen+8+NONCELEN)) return False;
 	/* It could be for any site - it should have LABEL_MSG1 and
 	   might have our name and our peer's name in it */
+	if (buf->size<(st->setupsiglen+8+NONCELEN)) return False;
 	if (memcmp(buf->start+8,st->setupsig,st->setupsiglen)==0) {
-	    dump_packet(st,buf,source,True);
 	    /* It's addressed to us. Decide what to do about it. */
+	    dump_packet(st,buf,source,True);
 	    if (st->state==SITE_RUN || st->state==SITE_RESOLVE ||
 		st->state==SITE_WAIT) {
 		/* We should definitely process it */
 		if (process_msg1(st,buf,source)) {
 		    slog(st,LOG_SETUP_INIT,"key setup initiated by peer");
-		    enter_state_sentmsg2(st);
+		    enter_new_state(st,SITE_SENTMSG2);
 		} else {
 		    slog(st,LOG_ERROR,"failed to process incoming msg1");
 		}
 		BUF_FREE(buf);
 		return True;
-	    }
-	    if (st->state==SITE_SENTMSG1) {
+	    } else if (st->state==SITE_SENTMSG1) {
 		/* We've just sent a message 1! They may have crossed on
 		   the wire. If we have priority then we ignore the
 		   incoming one, otherwise we process it as usual. */
@@ -1006,7 +1063,7 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
 			 "priority => use incoming msg1");
 		    if (process_msg1(st,buf,source)) {
 			BUF_FREE(&st->buffer); /* Free our old message 1 */
-			enter_state_sentmsg2(st);
+			enter_new_state(st,SITE_SENTMSG2);
 		    } else {
 			slog(st,LOG_ERROR,"failed to process an incoming "
 			     "crossed msg1 (we have low priority)");
@@ -1024,10 +1081,19 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
 	return False; /* Not for us. */
     }
     if (dest==(uint32_t)st) {
-	uint32_t msgtype=ntohl(*(uint32_t *)(buf->start+8));
 	/* Explicitly addressed to us */
+	uint32_t msgtype=ntohl(get_uint32(buf->start+8));
 	if (msgtype!=LABEL_MSG0) dump_packet(st,buf,source,True);
 	switch (msgtype) {
+	case 0: /* NAK */
+	    /* If the source is our current peer then initiate a key setup,
+	       because our peer's forgotten the key */
+	    if (get_uint32(buf->start+4)==st->remote_session_id) {
+		initiate_key_setup(st,"received a NAK");
+	    } else {
+		slog(st,LOG_SEC,"bad incoming NAK");
+	    }
+	    break;
 	case LABEL_MSG0:
 	    process_msg0(st,buf,source);
 	    break;
@@ -1041,7 +1107,7 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
 	    if (st->state!=SITE_SENTMSG1) {
 		slog(st,LOG_UNEXPECTED,"unexpected MSG2");
 	    } else if (process_msg2(st,buf,source))
-		enter_state_sentmsg3(st);
+		enter_new_state(st,SITE_SENTMSG3);
 	    else {
 		slog(st,LOG_SEC,"invalid MSG2");
 	    }
@@ -1051,7 +1117,7 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
 	    if (st->state!=SITE_SENTMSG2) {
 		slog(st,LOG_UNEXPECTED,"unexpected MSG3");
 	    } else if (process_msg3(st,buf,source))
-		enter_state_sentmsg4(st);
+		enter_new_state(st,SITE_SENTMSG4);
 	    else {
 		slog(st,LOG_SEC,"invalid MSG3");
 	    }
@@ -1061,7 +1127,7 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
 	    if (st->state!=SITE_SENTMSG3) {
 		slog(st,LOG_UNEXPECTED,"unexpected MSG4");
 	    } else if (process_msg4(st,buf,source))
-		enter_state_sentmsg5(st);
+		enter_new_state(st,SITE_SENTMSG5);
 	    else {
 		slog(st,LOG_SEC,"invalid MSG4");
 	    }
@@ -1072,11 +1138,11 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
 	       and the new key has already been activated. In that
 	       case we should treat it as an ordinary PING packet. We
 	       can't pass it to process_msg5() because the
-	       new_transform will now be null. XXX) */
+	       new_transform will now be unkeyed. XXX) */
 	    if (st->state!=SITE_SENTMSG4) {
 		slog(st,LOG_UNEXPECTED,"unexpected MSG5");
 	    } else if (process_msg5(st,buf,source)) {
-		send_msg6(st);
+		enter_new_state(st,SITE_RUN);
 	    } else {
 		slog(st,LOG_SEC,"invalid MSG5");
 	    }
@@ -1092,10 +1158,6 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
 		slog(st,LOG_SEC,"invalid MSG6");
 	    }
 	    break;
-	case LABEL_MSG8:
-	    /* NAK packet: enter state where we ping and check for response */
-	    slog(st,LOG_ERROR,"received a NAK");
-	    break;
 	default:
 	    slog(st,LOG_SEC,"received message of unknown type 0x%08x",
 		 msgtype);
@@ -1115,6 +1177,14 @@ static void site_control(void *vst, bool_t run)
     else enter_state_stop(st);
 }
 
+static void site_phase_hook(void *sst, uint32_t newphase)
+{
+    struct site *st=sst;
+
+    /* The program is shutting down; tell our peer */
+    send_msg7(st,"shutting down");
+}
+
 static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
 			  list_t *args)
 {
@@ -1144,23 +1214,22 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
        site() closures for all sites including our own): refuse to
        talk to ourselves */
     if (strcmp(st->localname,st->remotename)==0) {
-	Message(M_INFO,"site %s: local-name==name -> ignoring this site\n",
+	Message(M_DEBUG,"site %s: local-name==name -> ignoring this site\n",
 		st->localname);
 	free(st);
 	return NULL;
     }
-    st->netlink=find_cl_if(dict,"netlink",CL_NETLINK,True,"site",loc);
+    st->netlink=find_cl_if(dict,"link",CL_NETLINK,True,"site",loc);
     st->comm=find_cl_if(dict,"comm",CL_COMM,True,"site",loc);
     st->resolver=find_cl_if(dict,"resolver",CL_RESOLVER,True,"site",loc);
     st->log=find_cl_if(dict,"log",CL_LOG,True,"site",loc);
     st->random=find_cl_if(dict,"random",CL_RANDOMSRC,True,"site",loc);
 
     st->privkey=find_cl_if(dict,"local-key",CL_RSAPRIVKEY,True,"site",loc);
-    st->remoteport=dict_read_number(dict,"port",True,"site",loc,0);
-
     st->address=dict_read_string(dict, "address", False, "site", loc);
-    dict_read_subnet_list(dict, "networks", True, "site", loc,
-			  &st->remotenets);
+    if (st->address)
+	st->remoteport=dict_read_number(dict,"port",True,"site",loc,0);
+    else st->remoteport=0;
     st->pubkey=find_cl_if(dict,"key",CL_RSAPUBKEY,True,"site",loc);
 
     st->transform=
@@ -1187,7 +1256,6 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
 	cfgfatal(loc,"site",
 		 "renegotiate-time must be less than key-lifetime\n");
     }
-    /* XXX keepalive will eventually default to True */
     st->keepalive=dict_read_bool(dict,"keepalive",False,"site",loc,False);
 
     st->log_events=string_list_to_word(dict_lookup(dict,"log-events"),
@@ -1198,7 +1266,6 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     sprintf(st->tunname,"%s<->%s",st->localname,st->remotename);
 
     /* The information we expect to see in incoming messages of type 1 */
-    /* XXX fix this bit for unaligned access */
     st->setupsiglen=strlen(st->remotename)+strlen(st->localname)+8;
     st->setupsig=safe_malloc(st->setupsiglen,"site_apply");
     put_uint32(st->setupsig+0,LABEL_MSG1);
@@ -1224,18 +1291,11 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     st->sharedsecret=safe_malloc(st->transform->keylen,"site:sharedsecret");
 
     /* We need to register the remote networks with the netlink device */
-    st->netlink_cid=st->netlink->regnets(st->netlink->st, &st->remotenets,
-					 site_outgoing, st,
-					 st->transform->max_start_pad+(4*4),
-					 st->transform->max_end_pad,
-					 (st->address!=NULL), st->tunname);
-    if (!st->netlink_cid) {
-	Message(M_WARNING,"%s: netlink device did not let us register "
-		"our remote networks. This site will not start.\n",
-		st->tunname);
-	return NULL; /* XXX should free site first */
-    }
-
+    st->netlink->reg(st->netlink->st, site_outgoing, st,
+		     st->transform->max_start_pad+(4*4)+
+		     st->comm->min_start_pad,
+		     st->transform->max_end_pad+st->comm->min_end_pad);
+    
     st->comm->request_notify(st->comm->st, st, site_incoming);
 
     st->current_transform=st->transform->create(st->transform->st);
@@ -1243,6 +1303,8 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
 
     enter_state_stop(st);
 
+    add_hook(PHASE_SHUTDOWN,site_phase_hook,st);
+
     return new_closure(&st->cl);
 }