chiark / gitweb /
site: dynamically create and destroy transform instances
[secnet.git] / site.c
diff --git a/site.c b/site.c
index 49e1b14..f9a005a 100644 (file)
--- a/site.c
+++ b/site.c
@@ -17,6 +17,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <limits.h>
+#include <assert.h>
 #include <sys/socket.h>
 
 #include <sys/mman.h>
 
 #define SETUP_BUFFER_LEN 2048
 
-#define DEFAULT_KEY_LIFETIME 3600000 /* One hour */
-#define DEFAULT_KEY_RENEGOTIATE_GAP 300000 /* Five minutes */
+#define DEFAULT_KEY_LIFETIME                  (3600*1000) /* [ms] */
+#define DEFAULT_KEY_RENEGOTIATE_GAP           (5*60*1000) /* [ms] */
 #define DEFAULT_SETUP_RETRIES 5
-#define DEFAULT_SETUP_TIMEOUT 2000
-#define DEFAULT_WAIT_TIME 20000
+#define DEFAULT_SETUP_RETRY_INTERVAL             (2*1000) /* [ms] */
+#define DEFAULT_WAIT_TIME                       (20*1000) /* [ms] */
+
+#define DEFAULT_MOBILE_KEY_LIFETIME      (2*24*3600*1000) /* [ms] */
+#define DEFAULT_MOBILE_KEY_RENEGOTIATE_GAP (12*3600*1000) /* [ms] */
+#define DEFAULT_MOBILE_SETUP_RETRIES 30
+#define DEFAULT_MOBILE_SETUP_RETRY_INTERVAL      (1*1000) /* [ms] */
+#define DEFAULT_MOBILE_WAIT_TIME                (10*1000) /* [ms] */
+
+#define DEFAULT_MOBILE_PEER_EXPIRY            (2*60)      /* [s] */
+#define DEFAULT_MOBILE_PEERS_MAX 3 /* send at most this many copies (default) */
 
 /* Each site can be in one of several possible states. */
 
@@ -107,6 +117,7 @@ static cstring_t state_name(uint32_t state)
 #define LOG_DROP          0x00000080
 #define LOG_DUMP          0x00000100
 #define LOG_ERROR         0x00000400
+#define LOG_PEER_ADDRS    0x00000800
 
 static struct flagstr log_event_table[]={
     { "unexpected", LOG_UNEXPECTED },
@@ -119,23 +130,110 @@ static struct flagstr log_event_table[]={
     { "packet-drop", LOG_DROP },
     { "dump-packets", LOG_DUMP },
     { "errors", LOG_ERROR },
+    { "peer-addrs", LOG_PEER_ADDRS },
     { "default", LOG_SETUP_INIT|LOG_SETUP_TIMEOUT|
       LOG_ACTIVATE_KEY|LOG_TIMEOUT_KEY|LOG_SEC|LOG_ERROR },
     { "all", 0xffffffff },
     { NULL, 0 }
 };
 
+
+/***** TRANSPORT PEERS declarations *****/
+
+/* Details of "mobile peer" semantics:
+   
+   - We record mobile_peers_max peer address/port numbers ("peers")
+     for key setup, and separately mobile_peers_max for data
+     transfer.  If these lists fill up, we retain the newest peers.
+     (For non-mobile peers we only record one of each.)
+
+   - Outgoing packets are sent to every recorded peer in the
+     applicable list.
+
+   - Data transfer peers are straightforward: whenever we successfully
+     process a data packet, we record the peer.  Also, whenever we
+     successfully complete a key setup, we merge the key setup
+     peers into the data transfer peers.
+
+     (For "non-mobile" peers we simply copy the peer used for
+     successful key setup, and don't change the peer otherwise.)
+
+   - Key setup peers are slightly more complicated.
+
+     Whenever we receive and successfully process a key exchange
+     packet, we record the peer.
+
+     Whenever we try to initiate a key setup, we copy the list of data
+     transfer peers and use it for key setup.  But we also look to see
+     if the config supplies an address and port number and if so we
+     add that as a key setup peer (possibly evicting one of the data
+     transfer peers we just copied).
+
+     (For "non-mobile" peers, if we if we have a configured peer
+     address and port, we always use that; otherwise if we have a
+     current data peer address we use that; otherwise we do not
+     attempt to initiate a key setup for lack of a peer address.)
+
+   "Record the peer" means
+    1. expire any peers last seen >120s ("mobile-peer-expiry") ago
+    2. add the peer of the just received packet to the applicable list
+       (possibly evicting older entries)
+   NB that we do not expire peers until an incoming packet arrives.
+
+   */
+
+#define MAX_MOBILE_PEERS_MAX 5 /* send at most this many copies, compiled max */
+
+typedef struct {
+    struct timeval last;
+    struct comm_addr addr;
+} transport_peer;
+
+typedef struct {
+/* configuration information */
+/* runtime information */
+    int npeers;
+    transport_peer peers[MAX_MOBILE_PEERS_MAX];
+} transport_peers;
+
+static void transport_peers_clear(struct site *st, transport_peers *peers);
+static int transport_peers_valid(transport_peers *peers);
+static void transport_peers_copy(struct site *st, transport_peers *dst,
+                                const transport_peers *src);
+
+static void transport_setup_msgok(struct site *st, const struct comm_addr *a);
+static void transport_data_msgok(struct site *st, const struct comm_addr *a);
+static bool_t transport_compute_setupinit_peers(struct site *st,
+        const struct comm_addr *configured_addr /* 0 if none or not found */);
+static void transport_record_peer(struct site *st, transport_peers *peers,
+                                 const struct comm_addr *addr, const char *m);
+
+static void transport_xmit(struct site *st, transport_peers *peers,
+                          struct buffer_if *buf, bool_t candebug);
+
+ /***** END of transport peers declarations *****/
+
+
+struct data_key {
+    struct transform_inst_if *transform;
+    uint64_t key_timeout; /* End of life of current key */
+    uint32_t remote_session_id;
+};
+
 struct site {
     closure_t cl;
     struct site_if ops;
 /* configuration information */
     string_t localname;
     string_t remotename;
+    bool_t peer_mobile; /* Mobile client support */
+    int32_t transport_peers_max;
     string_t tunname; /* localname<->remotename by default, used in logs */
     string_t address; /* DNS name for bootstrapping, optional */
     int remoteport; /* Port for bootstrapping, optional */
     struct netlink_if *netlink;
-    struct comm_if *comm;
+    struct comm_if **comms;
+    int ncomms;
     struct resolver_if *resolver;
     struct log_if *log;
     struct random_if *random;
@@ -145,19 +243,17 @@ struct site {
     struct dh_if *dh;
     struct hash_if *hash;
 
-    uint32_t setup_retries; /* How many times to send setup packets */
-    uint32_t setup_timeout; /* Initial timeout for setup packets */
-    uint32_t wait_timeout; /* How long to wait if setup unsuccessful */
-    uint32_t key_lifetime; /* How long a key lasts once set up */
-    uint32_t key_renegotiate_time; /* If we see traffic (or a keepalive)
+    uint32_t index; /* Index of this site */
+    uint32_t local_capabilities;
+    int32_t setup_retries; /* How many times to send setup packets */
+    int32_t setup_retry_interval; /* Initial timeout for setup packets */
+    int32_t wait_timeout; /* How long to wait if setup unsuccessful */
+    int32_t mobile_peer_expiry; /* How long to remember 2ary addresses */
+    int32_t key_lifetime; /* How long a key lasts once set up */
+    int32_t key_renegotiate_time; /* If we see traffic (or a keepalive)
                                      after this time, initiate a new
                                      key exchange */
-    bool_t keepalive; /* Send keepalives to detect peer failure (not yet
-                        implemented) */
 
-    uint8_t *setupsig; /* Expected signature of incoming MSG1 packets */
-    uint32_t setupsiglen; /* Allows us to discard packets quickly if
-                            they are not for us */
     bool_t setup_priority; /* Do we have precedence if both sites emit
                              message 1 simultaneously? */
     uint32_t log_events;
@@ -167,14 +263,12 @@ struct site {
     uint64_t now; /* Most recently seen time */
 
     /* The currently established session */
-    uint32_t remote_session_id;
-    struct transform_inst_if *current_transform;
-    bool_t current_valid;
-    uint64_t current_key_timeout; /* End of life of current key */
+    struct data_key current;
+    struct data_key auxiliary_key;
+    bool_t auxiliary_is_new;
     uint64_t renegotiate_key_time; /* When we can negotiate a new key */
-    struct sockaddr_in peer; /* Current address of peer */
-    bool_t peer_valid; /* Peer address becomes invalid when key times out,
-                         but only if we have a DNS name for our peer */
+    uint64_t auxiliary_renegotiate_key_time;
+    transport_peers peers; /* Current address(es) of peer for data traffic */
 
     /* The current key setup protocol exchange.  We can only be
        involved in one of these at a time.  There's a potential for
@@ -182,22 +276,25 @@ struct site {
        packet; we keep trying to continue the exchange, and have to
        timeout before we can listen for another setup packet); perhaps
        we should keep a list of 'bad' sources for setup packets. */
+    uint32_t remote_capabilities;
     uint32_t setup_session_id;
-    struct sockaddr_in setup_peer;
+    transport_peers setup_peers;
     uint8_t localN[NONCELEN]; /* Nonces for key exchange */
     uint8_t remoteN[NONCELEN];
     struct buffer_if buffer; /* Current outgoing key exchange packet */
-    uint32_t retries; /* Number of retries remaining */
+    struct buffer_if scratch;
+    int32_t retries; /* Number of retries remaining */
     uint64_t timeout; /* Timeout for current state */
     uint8_t *dhsecret;
     uint8_t *sharedsecret;
+    uint32_t sharedsecretlen;
     struct transform_inst_if *new_transform; /* For key setup/verify */
 };
 
 static void slog(struct site *st, uint32_t event, cstring_t msg, ...)
 {
     va_list ap;
-    uint8_t buf[240];
+    char buf[240];
     uint32_t class;
 
     va_start(ap,msg);
@@ -214,22 +311,64 @@ static void slog(struct site *st, uint32_t event, cstring_t msg, ...)
        case LOG_DROP: class=M_DEBUG; break;
        case LOG_DUMP: class=M_DEBUG; break;
        case LOG_ERROR: class=M_ERR; break;
+       case LOG_PEER_ADDRS: class=M_DEBUG; break;
        default: class=M_ERR; break;
        }
 
-       vsnprintf(buf,240,msg,ap);
+       vsnprintf(buf,sizeof(buf),msg,ap);
        st->log->log(st->log->st,class,"%s: %s",st->tunname,buf);
     }
     va_end(ap);
 }
 
 static void set_link_quality(struct site *st);
-static void delete_key(struct site *st, cstring_t reason, uint32_t loglevel);
+static void delete_keys(struct site *st, cstring_t reason, uint32_t loglevel);
+static void delete_one_key(struct site *st, struct data_key *key,
+                          const char *reason /* may be 0 meaning don't log*/,
+                          const char *which /* ignored if !reasonn */,
+                          uint32_t loglevel /* ignored if !reasonn */);
 static bool_t initiate_key_setup(struct site *st, cstring_t reason);
 static void enter_state_run(struct site *st);
 static bool_t enter_state_resolve(struct site *st);
 static bool_t enter_new_state(struct site *st,uint32_t next);
 static void enter_state_wait(struct site *st);
+static void activate_new_key(struct site *st);
+
+static bool_t is_transform_valid(struct transform_inst_if *transform)
+{
+    return transform && transform->valid(transform->st);
+}
+
+static bool_t current_valid(struct site *st)
+{
+    return is_transform_valid(st->current.transform);
+}
+
+#define DEFINE_CALL_TRANSFORM(fwdrev)                                  \
+static int call_transform_##fwdrev(struct site *st,                    \
+                                  struct transform_inst_if *transform, \
+                                  struct buffer_if *buf,               \
+                                  const char **errmsg)                 \
+{                                                                      \
+    if (!is_transform_valid(transform)) {                              \
+       *errmsg="transform not set up";                                 \
+       return 1;                                                       \
+    }                                                                  \
+    return transform->fwdrev(transform->st,buf,errmsg);                        \
+}
+
+DEFINE_CALL_TRANSFORM(forwards)
+DEFINE_CALL_TRANSFORM(reverse)
+
+static void dispose_transform(struct transform_inst_if **transform_var)
+{
+    struct transform_inst_if *transform=*transform_var;
+    if (transform) {
+       transform->delkey(transform->st);
+       transform->destroy(transform->st);
+    }
+    *transform_var = 0;
+}    
 
 #define CHECK_AVAIL(b,l) do { if ((b)->size<(l)) return False; } while(0)
 #define CHECK_EMPTY(b) do { if ((b)->size!=0) return False; } while(0)
@@ -238,23 +377,66 @@ static void enter_state_wait(struct site *st);
     type=buf_unprepend_uint32((b)); \
     if (type!=(t)) return False; } while(0)
 
+struct parsedname {
+    int32_t len;
+    uint8_t *name;
+    struct buffer_if extrainfo;
+};
+
 struct msg {
     uint8_t *hashstart;
     uint32_t dest;
     uint32_t source;
-    uint32_t remlen;
-    uint8_t *remote;
-    uint32_t loclen;
-    uint8_t *local;
+    struct parsedname remote;
+    struct parsedname local;
+    uint32_t remote_capabilities;
     uint8_t *nR;
     uint8_t *nL;
-    uint32_t pklen;
-    uint8_t *pk;
-    uint32_t hashlen;
-    uint32_t siglen;
-    uint8_t *sig;
+    int32_t pklen;
+    char *pk;
+    int32_t hashlen;
+    int32_t siglen;
+    char *sig;
 };
 
+static void set_new_transform(struct site *st)
+{
+    struct transform_if *generator=st->transform;
+    struct transform_inst_if *generated=generator->create(generator->st);
+    generated->setkey(generated->st,st->sharedsecret,
+                     st->sharedsecretlen,st->setup_priority);
+    dispose_transform(&st->new_transform);
+    st->new_transform=generated;
+}
+
+struct xinfoadd {
+    int32_t lenpos, afternul;
+};
+static void append_string_xinfo_start(struct buffer_if *buf,
+                                     struct xinfoadd *xia,
+                                     const char *str)
+    /* Helps construct one of the names with additional info as found
+     * in MSG1..4.  Call this function first, then append all the
+     * desired extra info (not including the nul byte) to the buffer,
+     * then call append_string_xinfo_done. */
+{
+    xia->lenpos = buf->size;
+    buf_append_string(buf,str);
+    buf_append_uint8(buf,0);
+    xia->afternul = buf->size;
+}
+static void append_string_xinfo_done(struct buffer_if *buf,
+                                    struct xinfoadd *xia)
+{
+    /* we just need to adjust the string length */
+    if (buf->size == xia->afternul) {
+       /* no extra info, strip the nul too */
+       buf_unappend_uint8(buf);
+    } else {
+       put_uint16(buf->start+xia->lenpos, buf->size-(xia->lenpos+2));
+    }
+}
+
 /* Build any of msg1 to msg4. msg5 and msg6 are built from the inside
    out using a transform of config data supplied by netlink */
 static bool_t generate_msg(struct site *st, uint32_t type, cstring_t what)
@@ -268,9 +450,16 @@ static bool_t generate_msg(struct site *st, uint32_t type, cstring_t what)
     buffer_init(&st->buffer,0);
     buf_append_uint32(&st->buffer,
        (type==LABEL_MSG1?0:st->setup_session_id));
-    buf_append_uint32(&st->buffer,(uint32_t)st);
+    buf_append_uint32(&st->buffer,st->index);
     buf_append_uint32(&st->buffer,type);
-    buf_append_string(&st->buffer,st->localname);
+
+    struct xinfoadd xia;
+    append_string_xinfo_start(&st->buffer,&xia,st->localname);
+    if ((st->local_capabilities & CAPAB_EARLY) || (type != LABEL_MSG1)) {
+       buf_append_uint32(&st->buffer,st->local_capabilities);
+    }
+    append_string_xinfo_done(&st->buffer,&xia);
+
     buf_append_string(&st->buffer,st->remotename);
     memcpy(buf_append(&st->buffer,NONCELEN),st->localN,NONCELEN);
     if (type==LABEL_MSG1) return True;
@@ -293,6 +482,22 @@ static bool_t generate_msg(struct site *st, uint32_t type, cstring_t what)
     return True;
 }
 
+static bool_t unpick_name(struct buffer_if *msg, struct parsedname *nm)
+{
+    CHECK_AVAIL(msg,2);
+    nm->len=buf_unprepend_uint16(msg);
+    CHECK_AVAIL(msg,nm->len);
+    nm->name=buf_unprepend(msg,nm->len);
+    uint8_t *nul=memchr(nm->name,0,nm->len);
+    if (!nul) {
+       buffer_readonly_view(&nm->extrainfo,0,0);
+    } else {
+       buffer_readonly_view(&nm->extrainfo, nul+1, msg->start-(nul+1));
+       nm->len=nul-nm->name;
+    }
+    return True;
+}
+
 static bool_t unpick_msg(struct site *st, uint32_t type,
                         struct buffer_if *msg, struct msg *m)
 {
@@ -302,14 +507,13 @@ static bool_t unpick_msg(struct site *st, uint32_t type,
     CHECK_AVAIL(msg,4);
     m->source=buf_unprepend_uint32(msg);
     CHECK_TYPE(msg,type);
-    CHECK_AVAIL(msg,2);
-    m->remlen=buf_unprepend_uint16(msg);
-    CHECK_AVAIL(msg,m->remlen);
-    m->remote=buf_unprepend(msg,m->remlen);
-    CHECK_AVAIL(msg,2);
-    m->loclen=buf_unprepend_uint16(msg);
-    CHECK_AVAIL(msg,m->loclen);
-    m->local=buf_unprepend(msg,m->loclen);
+    if (!unpick_name(msg,&m->remote)) return False;
+    m->remote_capabilities=0;
+    if (m->remote.extrainfo.size) {
+       CHECK_AVAIL(&m->remote.extrainfo,4);
+       m->remote_capabilities=buf_unprepend_uint32(&m->remote.extrainfo);
+    }
+    if (!unpick_name(msg,&m->local)) return False;
     CHECK_AVAIL(msg,NONCELEN);
     m->nR=buf_unprepend(msg,NONCELEN);
     if (type==LABEL_MSG1) {
@@ -335,6 +539,14 @@ static bool_t unpick_msg(struct site *st, uint32_t type,
     return True;
 }
 
+static bool_t name_matches(const struct parsedname *nm, const char *expected)
+{
+    int expected_len=strlen(expected);
+    return
+       nm->len == expected_len &&
+       !memcmp(nm->name, expected, expected_len);
+}    
+
 static bool_t check_msg(struct site *st, uint32_t type, struct msg *m,
                        cstring_t *error)
 {
@@ -342,11 +554,11 @@ static bool_t check_msg(struct site *st, uint32_t type, struct msg *m,
 
     /* Check that the site names and our nonce have been sent
        back correctly, and then store our peer's nonce. */ 
-    if (memcmp(m->remote,st->remotename,strlen(st->remotename)!=0)) {
+    if (!name_matches(&m->remote,st->remotename)) {
        *error="wrong remote site name";
        return False;
     }
-    if (memcmp(m->local,st->localname,strlen(st->localname)!=0)) {
+    if (!name_matches(&m->local,st->localname)) {
        *error="wrong local site name";
        return False;
     }
@@ -355,11 +567,17 @@ static bool_t check_msg(struct site *st, uint32_t type, struct msg *m,
        return False;
     }
     if (type==LABEL_MSG2) return True;
-    if (memcmp(m->nR,st->remoteN,NONCELEN)!=0) {
+    if (!consttime_memeq(m->nR,st->remoteN,NONCELEN)!=0) {
        *error="wrong remotely-generated nonce";
        return False;
     }
+    /* MSG3 has complicated rules about capabilities, which are
+     * handled in process_msg3. */
     if (type==LABEL_MSG3) return True;
+    if (m->remote_capabilities!=st->remote_capabilities) {
+       *error="remote capabilities changed";
+       return False;
+    }
     if (type==LABEL_MSG4) return True;
     *error="unknown message type";
     return False;
@@ -372,19 +590,16 @@ static bool_t generate_msg1(struct site *st)
 }
 
 static bool_t process_msg1(struct site *st, struct buffer_if *msg1,
-                          struct sockaddr_in *src)
+                          const struct comm_addr *src, struct msg *m)
 {
-    struct msg m;
-
     /* We've already determined we're in an appropriate state to
        process an incoming MSG1, and that the MSG1 has correct values
        of A and B. */
 
-    if (!unpick_msg(st,LABEL_MSG1,msg1,&m)) return False;
-
-    st->setup_peer=*src;
-    st->setup_session_id=m.source;
-    memcpy(st->remoteN,m.nR,NONCELEN);
+    transport_record_peer(st,&st->setup_peers,src,"msg1");
+    st->setup_session_id=m->source;
+    st->remote_capabilities=m->remote_capabilities;
+    memcpy(st->remoteN,m->nR,NONCELEN);
     return True;
 }
 
@@ -395,7 +610,7 @@ static bool_t generate_msg2(struct site *st)
 }
 
 static bool_t process_msg2(struct site *st, struct buffer_if *msg2,
-                          struct sockaddr_in *src)
+                          const struct comm_addr *src)
 {
     struct msg m;
     cstring_t err;
@@ -406,6 +621,7 @@ static bool_t process_msg2(struct site *st, struct buffer_if *msg2,
        return False;
     }
     st->setup_session_id=m.source;
+    st->remote_capabilities=m.remote_capabilities;
     memcpy(st->remoteN,m.nR,NONCELEN);
     return True;
 }
@@ -419,7 +635,7 @@ static bool_t generate_msg3(struct site *st)
 }
 
 static bool_t process_msg3(struct site *st, struct buffer_if *msg3,
-                          struct sockaddr_in *src)
+                          const struct comm_addr *src)
 {
     struct msg m;
     uint8_t *hash;
@@ -431,6 +647,15 @@ static bool_t process_msg3(struct site *st, struct buffer_if *msg3,
        slog(st,LOG_SEC,"msg3: %s",err);
        return False;
     }
+    uint32_t capab_adv_late = m.remote_capabilities
+       & ~st->remote_capabilities & CAPAB_EARLY;
+    if (capab_adv_late) {
+       slog(st,LOG_SEC,"msg3 impermissibly adds early capability flag(s)"
+            " %#"PRIx32" (was %#"PRIx32", now %#"PRIx32")",
+            capab_adv_late, st->remote_capabilities, m.remote_capabilities);
+       return False;
+    }
+    st->remote_capabilities|=m.remote_capabilities;
 
     /* Check signature and store g^x mod m */
     hash=safe_malloc(st->hash->len, "process_msg3");
@@ -453,11 +678,10 @@ static bool_t process_msg3(struct site *st, struct buffer_if *msg3,
 
     /* Generate the shared key */
     st->dh->makeshared(st->dh->st,st->dhsecret,st->dh->len,m.pk,
-                      st->sharedsecret,st->transform->keylen);
+                      st->sharedsecret,st->sharedsecretlen);
 
     /* Set up the transform */
-    st->new_transform->setkey(st->new_transform->st,st->sharedsecret,
-                             st->transform->keylen);
+    set_new_transform(st);
 
     return True;
 }
@@ -470,7 +694,7 @@ static bool_t generate_msg4(struct site *st)
 }
 
 static bool_t process_msg4(struct site *st, struct buffer_if *msg4,
-                          struct sockaddr_in *src)
+                          const struct comm_addr *src)
 {
     struct msg m;
     uint8_t *hash;
@@ -501,10 +725,9 @@ static bool_t process_msg4(struct site *st, struct buffer_if *msg4,
     m.pk[m.pklen]=0;
     /* Generate the shared key */
     st->dh->makeshared(st->dh->st,st->dhsecret,st->dh->len,m.pk,
-                      st->sharedsecret,st->transform->keylen);
+                      st->sharedsecret,st->sharedsecretlen);
     /* Set up the transform */
-    st->new_transform->setkey(st->new_transform->st,st->sharedsecret,
-                             st->transform->keylen);
+    set_new_transform(st);
 
     return True;
 }
@@ -537,12 +760,12 @@ static bool_t generate_msg5(struct site *st)
     buffer_init(&st->buffer,st->transform->max_start_pad+(4*4));
     /* Give the netlink code an opportunity to put its own stuff in the
        message (configuration information, etc.) */
-    st->netlink->output_config(st->netlink->st,&st->buffer);
     buf_prepend_uint32(&st->buffer,LABEL_MSG5);
-    st->new_transform->forwards(st->new_transform->st,&st->buffer,
-                               &transform_err);
+    if (call_transform_forwards(st,st->new_transform,
+                               &st->buffer,&transform_err))
+       return False;
     buf_prepend_uint32(&st->buffer,LABEL_MSG5);
-    buf_prepend_uint32(&st->buffer,(uint32_t)st);
+    buf_prepend_uint32(&st->buffer,st->index);
     buf_prepend_uint32(&st->buffer,st->setup_session_id);
 
     st->retries=st->setup_retries;
@@ -550,15 +773,15 @@ static bool_t generate_msg5(struct site *st)
 }
 
 static bool_t process_msg5(struct site *st, struct buffer_if *msg5,
-                          struct sockaddr_in *src)
+                          const struct comm_addr *src,
+                          struct transform_inst_if *transform)
 {
     struct msg0 m;
     cstring_t transform_err;
 
     if (!unpick_msg0(st,msg5,&m)) return False;
 
-    if (st->new_transform->reverse(st->new_transform->st,
-                                  msg5,&transform_err)) {
+    if (call_transform_reverse(st,transform,msg5,&transform_err)) {
        /* There's a problem */
        slog(st,LOG_SEC,"process_msg5: transform: %s",transform_err);
        return False;
@@ -569,15 +792,13 @@ static bool_t process_msg5(struct site *st, struct buffer_if *msg5,
        slog(st,LOG_SEC,"MSG5/PING packet contained wrong label");
        return False;
     }
-    if (!st->netlink->check_config(st->netlink->st,msg5)) {
-       slog(st,LOG_SEC,"MSG5/PING packet contained bad netlink config");
-       return False;
-    }
-    CHECK_EMPTY(msg5);
+    /* Older versions of secnet used to write some config data here
+     * which we ignore.  So we don't CHECK_EMPTY */
     return True;
 }
 
-static bool_t generate_msg6(struct site *st)
+static void create_msg6(struct site *st, struct transform_inst_if *transform,
+                       uint32_t session_id)
 {
     cstring_t transform_err;
 
@@ -586,28 +807,33 @@ static bool_t generate_msg6(struct site *st)
     buffer_init(&st->buffer,st->transform->max_start_pad+(4*4));
     /* Give the netlink code an opportunity to put its own stuff in the
        message (configuration information, etc.) */
-    st->netlink->output_config(st->netlink->st,&st->buffer);
     buf_prepend_uint32(&st->buffer,LABEL_MSG6);
-    st->new_transform->forwards(st->new_transform->st,&st->buffer,
-                               &transform_err);
+    int problem = call_transform_forwards(st,transform,
+                                         &st->buffer,&transform_err);
+    assert(!problem);
     buf_prepend_uint32(&st->buffer,LABEL_MSG6);
-    buf_prepend_uint32(&st->buffer,(uint32_t)st);
-    buf_prepend_uint32(&st->buffer,st->setup_session_id);
+    buf_prepend_uint32(&st->buffer,st->index);
+    buf_prepend_uint32(&st->buffer,session_id);
+}
 
+static bool_t generate_msg6(struct site *st)
+{
+    if (!is_transform_valid(st->new_transform))
+       return False;
+    create_msg6(st,st->new_transform,st->setup_session_id);
     st->retries=1; /* Peer will retransmit MSG5 if this packet gets lost */
     return True;
 }
 
 static bool_t process_msg6(struct site *st, struct buffer_if *msg6,
-                          struct sockaddr_in *src)
+                          const struct comm_addr *src)
 {
     struct msg0 m;
     cstring_t transform_err;
 
     if (!unpick_msg0(st,msg6,&m)) return False;
 
-    if (st->new_transform->reverse(st->new_transform->st,
-                                  msg6,&transform_err)) {
+    if (call_transform_reverse(st,st->new_transform,msg6,&transform_err)) {
        /* There's a problem */
        slog(st,LOG_SEC,"process_msg6: transform: %s",transform_err);
        return False;
@@ -618,47 +844,110 @@ static bool_t process_msg6(struct site *st, struct buffer_if *msg6,
        slog(st,LOG_SEC,"MSG6/PONG packet contained invalid data");
        return False;
     }
-    if (!st->netlink->check_config(st->netlink->st,msg6)) {
-       slog(st,LOG_SEC,"MSG6/PONG packet contained bad netlink config");
-       return False;
-    }
-    CHECK_EMPTY(msg6);
+    /* Older versions of secnet used to write some config data here
+     * which we ignore.  So we don't CHECK_EMPTY */
     return True;
 }
 
-static bool_t process_msg0(struct site *st, struct buffer_if *msg0,
-                          struct sockaddr_in *src)
+static bool_t decrypt_msg0(struct site *st, struct buffer_if *msg0,
+                          const struct comm_addr *src)
 {
+    cstring_t transform_err, auxkey_err, newkey_err="n/a";
     struct msg0 m;
-    cstring_t transform_err;
-    uint32_t type;
-
-    if (!st->current_valid) {
-       slog(st,LOG_DROP,"incoming message but no current key -> dropping");
-       return initiate_key_setup(st,"incoming message but no current key");
-    }
+    uint32_t problem;
 
     if (!unpick_msg0(st,msg0,&m)) return False;
 
-    if (st->current_transform->reverse(st->current_transform->st,
-                                      msg0,&transform_err)) {
-       /* There's a problem */
-       slog(st,LOG_SEC,"transform: %s",transform_err);
-       return initiate_key_setup(st,"incoming message would not decrypt");
+    /* Keep a copy so we can try decrypting it with multiple keys */
+    buffer_copy(&st->scratch, msg0);
+
+    problem = call_transform_reverse(st,st->current.transform,
+                                    msg0,&transform_err);
+    if (!problem) {
+       if (!st->auxiliary_is_new)
+           delete_one_key(st,&st->auxiliary_key,
+                          "peer has used new key","auxiliary key",LOG_SEC);
+       return True;
     }
+    if (problem==2)
+       goto skew;
+
+    buffer_copy(msg0, &st->scratch);
+    problem = call_transform_reverse(st,st->auxiliary_key.transform,
+                                    msg0,&auxkey_err);
+    if (problem==0) {
+       slog(st,LOG_DROP,"processing packet which uses auxiliary key");
+       if (st->auxiliary_is_new) {
+           /* We previously timed out in state SENTMSG5 but it turns
+            * out that our peer did in fact get our MSG5 and is
+            * using the new key.  So we should switch to it too. */
+           /* This is a bit like activate_new_key. */
+           struct data_key t;
+           t=st->current;
+           st->current=st->auxiliary_key;
+           st->auxiliary_key=t;
+
+           delete_one_key(st,&st->auxiliary_key,"peer has used new key",
+                          "previous key",LOG_SEC);
+           st->auxiliary_is_new=0;
+           st->renegotiate_key_time=st->auxiliary_renegotiate_key_time;
+       }
+       return True;
+    }
+    if (problem==2)
+       goto skew;
+
+    if (st->state==SITE_SENTMSG5) {
+       buffer_copy(msg0, &st->scratch);
+       problem = call_transform_reverse(st,st->new_transform,
+                                        msg0,&newkey_err);
+       if (!problem) {
+           /* It looks like we didn't get the peer's MSG6 */
+           /* This is like a cut-down enter_new_state(SITE_RUN) */
+           slog(st,LOG_STATE,"will enter state RUN (MSG0 with new key)");
+           BUF_FREE(&st->buffer);
+           st->timeout=0;
+           activate_new_key(st);
+           return True; /* do process the data in this packet */
+       }
+       if (problem==2)
+           goto skew;
+    }
+
+    slog(st,LOG_SEC,"transform: %s (aux: %s, new: %s)",
+        transform_err,auxkey_err,newkey_err);
+    initiate_key_setup(st,"incoming message would not decrypt");
+    send_nak(src,m.dest,m.source,m.type,msg0,"message would not decrypt");
+    return False;
+
+ skew:
+    slog(st,LOG_DROP,"transform: %s (merely skew)",transform_err);
+    return False;
+}
+
+static bool_t process_msg0(struct site *st, struct buffer_if *msg0,
+                          const struct comm_addr *src)
+{
+    uint32_t type;
+
+    if (!decrypt_msg0(st,msg0,src))
+       return False;
+
     CHECK_AVAIL(msg0,4);
     type=buf_unprepend_uint32(msg0);
     switch(type) {
     case LABEL_MSG7:
        /* We must forget about the current session. */
-       delete_key(st,"request from peer",LOG_SEC);
+       delete_keys(st,"request from peer",LOG_SEC);
        return True;
-       break;
     case LABEL_MSG9:
        /* Deliver to netlink layer */
        st->netlink->deliver(st->netlink->st,msg0);
+       transport_data_msgok(st,src);
+       /* See whether we should start negotiating a new key */
+       if (st->now > st->renegotiate_key_time)
+           initiate_key_setup(st,"incoming packet in renegotiation window");
        return True;
-       break;
     default:
        slog(st,LOG_SEC,"incoming encrypted message of type %08x "
             "(unknown)",type);
@@ -668,16 +957,16 @@ static bool_t process_msg0(struct site *st, struct buffer_if *msg0,
 }
 
 static void dump_packet(struct site *st, struct buffer_if *buf,
-                       struct sockaddr_in *addr, bool_t incoming)
+                       const struct comm_addr *addr, bool_t incoming)
 {
-    uint32_t dest=ntohl(*(uint32_t *)buf->start);
-    uint32_t source=ntohl(*(uint32_t *)(buf->start+4));
-    uint32_t msgtype=ntohl(*(uint32_t *)(buf->start+8));
+    uint32_t dest=get_uint32(buf->start);
+    uint32_t source=get_uint32(buf->start+4);
+    uint32_t msgtype=get_uint32(buf->start+8);
 
     if (st->log_events & LOG_DUMP)
-       log(st->log,M_DEBUG,"%s: %s: %08x<-%08x: %08x:",
-           st->tunname,incoming?"incoming":"outgoing",
-           dest,source,msgtype);
+       slilog(st->log,M_DEBUG,"%s: %s: %08x<-%08x: %08x:",
+              st->tunname,incoming?"incoming":"outgoing",
+              dest,source,msgtype);
 }
 
 static uint32_t site_status(void *st)
@@ -688,11 +977,28 @@ static uint32_t site_status(void *st)
 static bool_t send_msg(struct site *st)
 {
     if (st->retries>0) {
-       dump_packet(st,&st->buffer,&st->setup_peer,False);
-       st->comm->sendmsg(st->comm->st,&st->buffer,&st->setup_peer);
-       st->timeout=st->now+st->setup_timeout;
+       transport_xmit(st, &st->setup_peers, &st->buffer, True);
+       st->timeout=st->now+st->setup_retry_interval;
        st->retries--;
        return True;
+    } else if (st->state==SITE_SENTMSG5) {
+       slog(st,LOG_SETUP_TIMEOUT,"timed out sending MSG5, stashing new key");
+       /* We stash the key we have produced, in case it turns out that
+        * our peer did see our MSG5 after all and starts using it. */
+       /* This is a bit like some of activate_new_key */
+       struct transform_inst_if *t;
+       t=st->auxiliary_key.transform;
+       st->auxiliary_key.transform=st->new_transform;
+       st->new_transform=t;
+       dispose_transform(&st->new_transform);
+
+       st->auxiliary_is_new=1;
+       st->auxiliary_key.key_timeout=st->now+st->key_lifetime;
+       st->auxiliary_renegotiate_key_time=st->now+st->key_renegotiate_time;
+       st->auxiliary_key.remote_session_id=st->setup_session_id;
+
+       enter_state_wait(st);
+       return False;
     } else {
        slog(st,LOG_SETUP_TIMEOUT,"timed out sending key setup packet "
            "(in state %s)",state_name(st->state));
@@ -704,20 +1010,28 @@ static bool_t send_msg(struct site *st)
 static void site_resolve_callback(void *sst, struct in_addr *address)
 {
     struct site *st=sst;
+    struct comm_addr ca_buf, *ca_use;
 
     if (st->state!=SITE_RESOLVE) {
        slog(st,LOG_UNEXPECTED,"site_resolve_callback called unexpectedly");
        return;
     }
     if (address) {
-       memset(&st->setup_peer,0,sizeof(st->setup_peer));
-       st->setup_peer.sin_family=AF_INET;
-       st->setup_peer.sin_port=htons(st->remoteport);
-       st->setup_peer.sin_addr=*address;
-       enter_new_state(st,SITE_SENTMSG1);
+       FILLZERO(ca_buf);
+       ca_buf.comm=st->comms[0];
+       ca_buf.sin.sin_family=AF_INET;
+       ca_buf.sin.sin_port=htons(st->remoteport);
+       ca_buf.sin.sin_addr=*address;
+       ca_use=&ca_buf;
     } else {
-       /* Resolution failed */
        slog(st,LOG_ERROR,"resolution of %s failed",st->address);
+       ca_use=0;
+    }
+    if (transport_compute_setupinit_peers(st,ca_use)) {
+       enter_new_state(st,SITE_SENTMSG1);
+    } else {
+       /* Can't figure out who to try to to talk to */
+       slog(st,LOG_SETUP_INIT,"key exchange failed: cannot find peer address");
        enter_state_run(st);
     }
 }
@@ -729,9 +1043,7 @@ static bool_t initiate_key_setup(struct site *st, cstring_t reason)
     if (st->address) {
        slog(st,LOG_SETUP_INIT,"resolving peer address");
        return enter_state_resolve(st);
-    } else if (st->peer_valid) {
-       slog(st,LOG_SETUP_INIT,"using old peer address");
-       st->setup_peer=st->peer;
+    } else if (transport_compute_setupinit_peers(st,0)) {
        return enter_new_state(st,SITE_SENTMSG1);
     }
     slog(st,LOG_SETUP_INIT,"key exchange failed: no address for peer");
@@ -742,35 +1054,44 @@ static void activate_new_key(struct site *st)
 {
     struct transform_inst_if *t;
 
-    /* We have two transform instances, which we swap between active
-       and setup */
-    t=st->current_transform;
-    st->current_transform=st->new_transform;
+    /* We have three transform instances, which we swap between old,
+       active and setup */
+    t=st->auxiliary_key.transform;
+    st->auxiliary_key.transform=st->current.transform;
+    st->current.transform=st->new_transform;
     st->new_transform=t;
+    dispose_transform(&st->new_transform);
 
-    t->delkey(t->st);
     st->timeout=0;
-    st->current_valid=True;
-    st->current_key_timeout=st->now+st->key_lifetime;
+    st->auxiliary_is_new=0;
+    st->auxiliary_key.key_timeout=st->current.key_timeout;
+    st->current.key_timeout=st->now+st->key_lifetime;
     st->renegotiate_key_time=st->now+st->key_renegotiate_time;
-    st->peer=st->setup_peer;
-    st->peer_valid=True;
-    st->remote_session_id=st->setup_session_id;
+    transport_peers_copy(st,&st->peers,&st->setup_peers);
+    st->current.remote_session_id=st->setup_session_id;
 
     slog(st,LOG_ACTIVATE_KEY,"new key activated");
     enter_state_run(st);
 }
 
-static void delete_key(struct site *st, cstring_t reason, uint32_t loglevel)
+static void delete_one_key(struct site *st, struct data_key *key,
+                          cstring_t reason, cstring_t which, uint32_t loglevel)
+{
+    if (!is_transform_valid(key->transform)) return;
+    if (reason) slog(st,loglevel,"%s deleted (%s)",which,reason);
+    dispose_transform(&key->transform);
+    key->key_timeout=0;
+}
+
+static void delete_keys(struct site *st, cstring_t reason, uint32_t loglevel)
 {
-    if (st->current_valid) {
+    if (current_valid(st)) {
        slog(st,loglevel,"session closed (%s)",reason);
 
-       st->current_valid=False;
-       st->current_transform->delkey(st->current_transform->st);
-       st->current_key_timeout=0;
+       delete_one_key(st,&st->current,0,0,0);
        set_link_quality(st);
     }
+    delete_one_key(st,&st->auxiliary_key,0,0,0);
 }
 
 static void state_assert(struct site *st, bool_t ok)
@@ -782,20 +1103,20 @@ static void enter_state_stop(struct site *st)
 {
     st->state=SITE_STOP;
     st->timeout=0;
-    delete_key(st,"entering state STOP",LOG_TIMEOUT_KEY);
-    st->new_transform->delkey(st->new_transform->st);
+    delete_keys(st,"entering state STOP",LOG_TIMEOUT_KEY);
+    dispose_transform(&st->new_transform);
 }
 
 static void set_link_quality(struct site *st)
 {
     uint32_t quality;
-    if (st->current_valid)
+    if (current_valid(st))
        quality=LINK_QUALITY_UP;
     else if (st->state==SITE_WAIT || st->state==SITE_STOP)
        quality=LINK_QUALITY_DOWN;
     else if (st->address)
        quality=LINK_QUALITY_DOWN_CURRENT_ADDRESS;
-    else if (st->peer_valid)
+    else if (transport_peers_valid(&st->peers))
        quality=LINK_QUALITY_DOWN_STALE_ADDRESS;
     else
        quality=LINK_QUALITY_DOWN;
@@ -810,12 +1131,12 @@ static void enter_state_run(struct site *st)
     st->timeout=0;
 
     st->setup_session_id=0;
-    memset(&st->setup_peer,0,sizeof(st->setup_peer));
+    transport_peers_clear(st,&st->setup_peers);
     memset(st->localN,0,NONCELEN);
     memset(st->remoteN,0,NONCELEN);
-    st->new_transform->delkey(st->new_transform->st);
+    dispose_transform(&st->new_transform);
     memset(st->dhsecret,0,st->dh->len);
-    memset(st->sharedsecret,0,st->transform->keylen);
+    memset(st->sharedsecret,0,st->sharedsecretlen);
     set_link_quality(st);
 }
 
@@ -876,7 +1197,7 @@ static bool_t enter_new_state(struct site *st, uint32_t next)
     r= gen(st) && send_msg(st);
 
     hacky_par_end(&r,
-                 st->setup_retries, st->setup_timeout,
+                 st->setup_retries, st->setup_retry_interval,
                  send_msg, st);
     
     if (r) {
@@ -901,18 +1222,21 @@ static bool_t send_msg7(struct site *st, cstring_t reason)
 {
     cstring_t transform_err;
 
-    if (st->current_valid && st->peer_valid && st->buffer.free) {
+    if (current_valid(st) && st->buffer.free
+       && transport_peers_valid(&st->peers)) {
        BUF_ALLOC(&st->buffer,"site:MSG7");
        buffer_init(&st->buffer,st->transform->max_start_pad+(4*3));
        buf_append_uint32(&st->buffer,LABEL_MSG7);
        buf_append_string(&st->buffer,reason);
-       st->current_transform->forwards(st->current_transform->st,
-                                       &st->buffer, &transform_err);
+       if (call_transform_forwards(st, st->current.transform,
+                                   &st->buffer, &transform_err))
+           goto free_out;
        buf_prepend_uint32(&st->buffer,LABEL_MSG0);
-       buf_prepend_uint32(&st->buffer,(uint32_t)st);
-       buf_prepend_uint32(&st->buffer,st->remote_session_id);
-       st->comm->sendmsg(st->comm->st,&st->buffer,&st->peer);
+       buf_prepend_uint32(&st->buffer,st->index);
+       buf_prepend_uint32(&st->buffer,st->current.remote_session_id);
+       transport_xmit(st,&st->peers,&st->buffer,True);
        BUF_FREE(&st->buffer);
+    free_out:
        return True;
     }
     return False;
@@ -926,17 +1250,16 @@ static void enter_state_wait(struct site *st)
     slog(st,LOG_STATE,"entering state WAIT");
     st->timeout=st->now+st->wait_timeout;
     st->state=SITE_WAIT;
-    st->peer_valid=False;
     set_link_quality(st);
     BUF_FREE(&st->buffer); /* will have had an outgoing packet in it */
     /* XXX Erase keys etc. */
 }
 
-static inline void site_settimeout(uint64_t timeout, uint64_t *now,
-                                  int *timeout_io)
+static inline void site_settimeout(uint64_t timeout, int *timeout_io)
 {
     if (timeout) {
-       uint64_t offset=timeout-*now;
+       int64_t offset=timeout-*now;
+       if (offset<0) offset=0;
        if (offset>INT_MAX) offset=INT_MAX;
        if (*timeout_io<0 || offset<*timeout_io)
            *timeout_io=offset;
@@ -944,8 +1267,7 @@ static inline void site_settimeout(uint64_t timeout, uint64_t *now,
 }
 
 static int site_beforepoll(void *sst, struct pollfd *fds, int *nfds_io,
-                          int *timeout_io, const struct timeval *tv_now,
-                          uint64_t *now)
+                          int *timeout_io)
 {
     struct site *st=sst;
 
@@ -953,17 +1275,25 @@ static int site_beforepoll(void *sst, struct pollfd *fds, int *nfds_io,
     st->now=*now;
 
     /* Work out when our next timeout is. The earlier of 'timeout' or
-       'current_key_timeout'. A stored value of '0' indicates no timeout
+       'current.key_timeout'. A stored value of '0' indicates no timeout
        active. */
-    site_settimeout(st->timeout, now, timeout_io);
-    site_settimeout(st->current_key_timeout, now, timeout_io);
+    site_settimeout(st->timeout, timeout_io);
+    site_settimeout(st->current.key_timeout, timeout_io);
+    site_settimeout(st->auxiliary_key.key_timeout, timeout_io);
 
     return 0; /* success */
 }
 
+static void check_expiry(struct site *st, struct data_key *key,
+                        const char *which)
+{
+    if (key->key_timeout && *now>key->key_timeout) {
+       delete_one_key(st,key,"maximum life exceeded",which,LOG_TIMEOUT_KEY);
+    }
+}
+
 /* NB site_afterpoll will be called before site_beforepoll is ever called */
-static void site_afterpoll(void *sst, struct pollfd *fds, int nfds,
-                          const struct timeval *tv_now, uint64_t *now)
+static void site_afterpoll(void *sst, struct pollfd *fds, int nfds)
 {
     struct site *st=sst;
 
@@ -980,9 +1310,8 @@ static void site_afterpoll(void *sst, struct pollfd *fds, int nfds,
                 st->state);
        }
     }
-    if (st->current_key_timeout && *now>st->current_key_timeout) {
-       delete_key(st,"maximum key life exceeded",LOG_TIMEOUT_KEY);
-    }
+    check_expiry(st,&st->current,"current key");
+    check_expiry(st,&st->auxiliary_key,"auxiliary key");
 }
 
 /* This function is called by the netlink device to deliver packets
@@ -1000,21 +1329,20 @@ static void site_outgoing(void *sst, struct buffer_if *buf)
 
     /* In all other states we consider delivering the packet if we have
        a valid key and a valid address to send it to. */
-    if (st->current_valid && st->peer_valid) {
+    if (current_valid(st) && transport_peers_valid(&st->peers)) {
        /* Transform it and send it */
        if (buf->size>0) {
            buf_prepend_uint32(buf,LABEL_MSG9);
-           st->current_transform->forwards(st->current_transform->st,
-                                           buf, &transform_err);
+           if (call_transform_forwards(st, st->current.transform,
+                                       buf, &transform_err))
+               goto free_out;
            buf_prepend_uint32(buf,LABEL_MSG0);
-           buf_prepend_uint32(buf,(uint32_t)st);
-           buf_prepend_uint32(buf,st->remote_session_id);
-           st->comm->sendmsg(st->comm->st,buf,&st->peer);
+           buf_prepend_uint32(buf,st->index);
+           buf_prepend_uint32(buf,st->current.remote_session_id);
+           transport_xmit(st,&st->peers,buf,False);
        }
+    free_out:
        BUF_FREE(buf);
-       /* See whether we should start negotiating a new key */
-       if (st->now > st->renegotiate_key_time)
-           initiate_key_setup(st,"outgoing packet in renegotiation window");
        return;
     }
 
@@ -1023,72 +1351,84 @@ static void site_outgoing(void *sst, struct buffer_if *buf)
     initiate_key_setup(st,"outgoing packet");
 }
 
+static bool_t named_for_us(struct site *st, const struct buffer_if *buf_in,
+                          uint32_t type, struct msg *m)
+    /* For packets which are identified by the local and remote names.
+     * If it has our name and our peer's name in it it's for us. */
+{
+    struct buffer_if buf[1];
+    buffer_readonly_clone(buf,buf_in);
+    return unpick_msg(st,type,buf,m)
+       && name_matches(&m->remote,st->remotename)
+       && name_matches(&m->local,st->localname);
+}
+
 /* This function is called by the communication device to deliver
    packets from our peers. */
 static bool_t site_incoming(void *sst, struct buffer_if *buf,
-                           struct sockaddr_in *source)
+                           const struct comm_addr *source)
 {
     struct site *st=sst;
-    uint32_t dest=ntohl(*(uint32_t *)buf->start);
-
-    if (dest==0) {
-       /* It could be for any site - it should have LABEL_MSG1 and
-          might have our name and our peer's name in it */
-       if (buf->size<(st->setupsiglen+8+NONCELEN)) return False;
-       if (memcmp(buf->start+8,st->setupsig,st->setupsiglen)==0) {
-           /* It's addressed to us. Decide what to do about it. */
-           dump_packet(st,buf,source,True);
-           if (st->state==SITE_RUN || st->state==SITE_RESOLVE ||
-               st->state==SITE_WAIT) {
-               /* We should definitely process it */
-               if (process_msg1(st,buf,source)) {
-                   slog(st,LOG_SETUP_INIT,"key setup initiated by peer");
+
+    if (buf->size < 12) return False;
+
+    uint32_t dest=get_uint32(buf->start);
+    uint32_t msgtype=get_uint32(buf->start+8);
+    struct msg named_msg;
+
+    if (msgtype==LABEL_MSG1) {
+       if (!named_for_us(st,buf,msgtype,&named_msg))
+           return False;
+       /* It's a MSG1 addressed to us. Decide what to do about it. */
+       dump_packet(st,buf,source,True);
+       if (st->state==SITE_RUN || st->state==SITE_RESOLVE ||
+           st->state==SITE_WAIT) {
+           /* We should definitely process it */
+           if (process_msg1(st,buf,source,&named_msg)) {
+               slog(st,LOG_SETUP_INIT,"key setup initiated by peer");
+               enter_new_state(st,SITE_SENTMSG2);
+           } else {
+               slog(st,LOG_ERROR,"failed to process incoming msg1");
+           }
+           BUF_FREE(buf);
+           return True;
+       } else if (st->state==SITE_SENTMSG1) {
+           /* We've just sent a message 1! They may have crossed on
+              the wire. If we have priority then we ignore the
+              incoming one, otherwise we process it as usual. */
+           if (st->setup_priority) {
+               BUF_FREE(buf);
+               slog(st,LOG_DUMP,"crossed msg1s; we are higher "
+                    "priority => ignore incoming msg1");
+               return True;
+           } else {
+               slog(st,LOG_DUMP,"crossed msg1s; we are lower "
+                    "priority => use incoming msg1");
+               if (process_msg1(st,buf,source,&named_msg)) {
+                   BUF_FREE(&st->buffer); /* Free our old message 1 */
                    enter_new_state(st,SITE_SENTMSG2);
                } else {
-                   slog(st,LOG_ERROR,"failed to process incoming msg1");
+                   slog(st,LOG_ERROR,"failed to process an incoming "
+                        "crossed msg1 (we have low priority)");
                }
                BUF_FREE(buf);
                return True;
-           } else if (st->state==SITE_SENTMSG1) {
-               /* We've just sent a message 1! They may have crossed on
-                  the wire. If we have priority then we ignore the
-                  incoming one, otherwise we process it as usual. */
-               if (st->setup_priority) {
-                   BUF_FREE(buf);
-                   slog(st,LOG_DUMP,"crossed msg1s; we are higher "
-                        "priority => ignore incoming msg1");
-                   return True;
-               } else {
-                   slog(st,LOG_DUMP,"crossed msg1s; we are lower "
-                        "priority => use incoming msg1");
-                   if (process_msg1(st,buf,source)) {
-                       BUF_FREE(&st->buffer); /* Free our old message 1 */
-                       enter_new_state(st,SITE_SENTMSG2);
-                   } else {
-                       slog(st,LOG_ERROR,"failed to process an incoming "
-                            "crossed msg1 (we have low priority)");
-                   }
-                   BUF_FREE(buf);
-                   return True;
-               }
            }
-           /* The message 1 was received at an unexpected stage of the
-              key setup. XXX POLICY - what do we do? */
-           slog(st,LOG_UNEXPECTED,"unexpected incoming message 1");
-           BUF_FREE(buf);
-           return True;
        }
-       return False; /* Not for us. */
+       /* The message 1 was received at an unexpected stage of the
+          key setup. XXX POLICY - what do we do? */
+       slog(st,LOG_UNEXPECTED,"unexpected incoming message 1");
+       BUF_FREE(buf);
+       return True;
     }
-    if (dest==(uint32_t)st) {
+    if (dest==st->index) {
        /* Explicitly addressed to us */
-       uint32_t msgtype=ntohl(get_uint32(buf->start+8));
        if (msgtype!=LABEL_MSG0) dump_packet(st,buf,source,True);
        switch (msgtype) {
-       case 0: /* NAK */
+       case LABEL_NAK:
            /* If the source is our current peer then initiate a key setup,
               because our peer's forgotten the key */
-           if (get_uint32(buf->start+4)==st->remote_session_id) {
+           if (get_uint32(buf->start+4)==st->current.remote_session_id) {
                initiate_key_setup(st,"received a NAK");
            } else {
                slog(st,LOG_SEC,"bad incoming NAK");
@@ -1106,9 +1446,10 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
            /* Setup packet: expected only in state SENTMSG1 */
            if (st->state!=SITE_SENTMSG1) {
                slog(st,LOG_UNEXPECTED,"unexpected MSG2");
-           } else if (process_msg2(st,buf,source))
+           } else if (process_msg2(st,buf,source)) {
+               transport_setup_msgok(st,source);
                enter_new_state(st,SITE_SENTMSG3);
-           else {
+           else {
                slog(st,LOG_SEC,"invalid MSG2");
            }
            break;
@@ -1116,9 +1457,10 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
            /* Setup packet: expected only in state SENTMSG2 */
            if (st->state!=SITE_SENTMSG2) {
                slog(st,LOG_UNEXPECTED,"unexpected MSG3");
-           } else if (process_msg3(st,buf,source))
+           } else if (process_msg3(st,buf,source)) {
+               transport_setup_msgok(st,source);
                enter_new_state(st,SITE_SENTMSG4);
-           else {
+           else {
                slog(st,LOG_SEC,"invalid MSG3");
            }
            break;
@@ -1126,9 +1468,10 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
            /* Setup packet: expected only in state SENTMSG3 */
            if (st->state!=SITE_SENTMSG3) {
                slog(st,LOG_UNEXPECTED,"unexpected MSG4");
-           } else if (process_msg4(st,buf,source))
+           } else if (process_msg4(st,buf,source)) {
+               transport_setup_msgok(st,source);
                enter_new_state(st,SITE_SENTMSG5);
-           else {
+           else {
                slog(st,LOG_SEC,"invalid MSG4");
            }
            break;
@@ -1136,15 +1479,29 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
            /* Setup packet: expected only in state SENTMSG4 */
            /* (may turn up in state RUN if our return MSG6 was lost
               and the new key has already been activated. In that
-              case we should treat it as an ordinary PING packet. We
-              can't pass it to process_msg5() because the
-              new_transform will now be unkeyed. XXX) */
-           if (st->state!=SITE_SENTMSG4) {
-               slog(st,LOG_UNEXPECTED,"unexpected MSG5");
-           } else if (process_msg5(st,buf,source)) {
-               enter_new_state(st,SITE_RUN);
+              case we discard it. The peer will realise that we
+              are using the new key when they see our data packets.
+              Until then the peer's data packets to us get discarded. */
+           if (st->state==SITE_SENTMSG4) {
+               if (process_msg5(st,buf,source,st->new_transform)) {
+                   transport_setup_msgok(st,source);
+                   enter_new_state(st,SITE_RUN);
+               } else {
+                   slog(st,LOG_SEC,"invalid MSG5");
+               }
+           } else if (st->state==SITE_RUN) {
+               if (process_msg5(st,buf,source,st->current.transform)) {
+                   slog(st,LOG_DROP,"got MSG5, retransmitting MSG6");
+                   transport_setup_msgok(st,source);
+                   create_msg6(st,st->current.transform,
+                               st->current.remote_session_id);
+                   transport_xmit(st,&st->peers,&st->buffer,True);
+                   BUF_FREE(&st->buffer);
+               } else {
+                   slog(st,LOG_SEC,"invalid MSG5 (in state RUN)");
+               }
            } else {
-               slog(st,LOG_SEC,"invalid MSG5");
+               slog(st,LOG_UNEXPECTED,"unexpected MSG5");
            }
            break;
        case LABEL_MSG6:
@@ -1153,6 +1510,7 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
                slog(st,LOG_UNEXPECTED,"unexpected MSG6");
            } else if (process_msg6(st,buf,source)) {
                BUF_FREE(&st->buffer); /* Free message 5 */
+               transport_setup_msgok(st,source);
                activate_new_key(st);
            } else {
                slog(st,LOG_SEC,"invalid MSG6");
@@ -1188,9 +1546,11 @@ static void site_phase_hook(void *sst, uint32_t newphase)
 static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
                          list_t *args)
 {
+    static uint32_t index_sequence;
     struct site *st;
     item_t *item;
     dict_t *dict;
+    int i;
 
     st=safe_malloc(sizeof(*st),"site_apply");
 
@@ -1210,17 +1570,50 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     dict=item->data.dict;
     st->localname=dict_read_string(dict, "local-name", True, "site", loc);
     st->remotename=dict_read_string(dict, "name", True, "site", loc);
+
+    st->peer_mobile=dict_read_bool(dict,"mobile",False,"site",loc,False);
+    bool_t local_mobile=
+       dict_read_bool(dict,"local-mobile",False,"site",loc,False);
+
     /* Sanity check (which also allows the 'sites' file to include
        site() closures for all sites including our own): refuse to
        talk to ourselves */
     if (strcmp(st->localname,st->remotename)==0) {
        Message(M_DEBUG,"site %s: local-name==name -> ignoring this site\n",
                st->localname);
+       if (st->peer_mobile != local_mobile)
+           cfgfatal(loc,"site","site %s's peer-mobile=%d"
+                   " but our local-mobile=%d\n",
+                   st->localname, st->peer_mobile, local_mobile);
+       free(st);
+       return NULL;
+    }
+    if (st->peer_mobile && local_mobile) {
+       Message(M_WARNING,"site %s: site is mobile but so are we"
+               " -> ignoring this site\n", st->remotename);
        free(st);
        return NULL;
     }
+
+    assert(index_sequence < 0xffffffffUL);
+    st->index = ++index_sequence;
+    st->local_capabilities = 0;
     st->netlink=find_cl_if(dict,"link",CL_NETLINK,True,"site",loc);
-    st->comm=find_cl_if(dict,"comm",CL_COMM,True,"site",loc);
+
+    list_t *comms_cfg=dict_lookup(dict,"comm");
+    if (!comms_cfg) cfgfatal(loc,"site","closure list \"comm\" not found\n");
+    st->ncomms=list_length(comms_cfg);
+    st->comms=safe_malloc_ary(sizeof(*st->comms),st->ncomms,"comms");
+    assert(st->ncomms);
+    for (i=0; i<st->ncomms; i++) {
+       item_t *item=list_elem(comms_cfg,i);
+       if (item->type!=t_closure)
+           cfgfatal(loc,"site","comm is not a closure\n");
+       closure_t *cl=item->data.closure;
+       if (cl->type!=CL_COMM) cfgfatal(loc,"site","comm closure wrong type\n");
+       st->comms[i]=cl->interface;
+    }
+
     st->resolver=find_cl_if(dict,"resolver",CL_RESOLVER,True,"site",loc);
     st->log=find_cl_if(dict,"log",CL_LOG,True,"site",loc);
     st->random=find_cl_if(dict,"random",CL_RANDOMSRC,True,"site",loc);
@@ -1238,25 +1631,36 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     st->dh=find_cl_if(dict,"dh",CL_DH,True,"site",loc);
     st->hash=find_cl_if(dict,"hash",CL_HASH,True,"site",loc);
 
-    st->key_lifetime=dict_read_number(
-       dict,"key-lifetime",False,"site",loc,DEFAULT_KEY_LIFETIME);
-    if (st->key_lifetime < DEFAULT_KEY_RENEGOTIATE_GAP)
+#define DEFAULT(D) (st->peer_mobile || local_mobile    \
+                    ? DEFAULT_MOBILE_##D : DEFAULT_##D)
+#define CFG_NUMBER(k,D) dict_read_number(dict,(k),False,"site",loc,DEFAULT(D));
+
+    st->key_lifetime=         CFG_NUMBER("key-lifetime",  KEY_LIFETIME);
+    st->setup_retries=        CFG_NUMBER("setup-retries", SETUP_RETRIES);
+    st->setup_retry_interval= CFG_NUMBER("setup-timeout", SETUP_RETRY_INTERVAL);
+    st->wait_timeout=         CFG_NUMBER("wait-time",     WAIT_TIME);
+
+    st->mobile_peer_expiry= dict_read_number(
+       dict,"mobile-peer-expiry",False,"site",loc,DEFAULT_MOBILE_PEER_EXPIRY);
+
+    st->transport_peers_max= !st->peer_mobile ? 1 : dict_read_number(
+       dict,"mobile-peers-max",False,"site",loc,DEFAULT_MOBILE_PEERS_MAX);
+    if (st->transport_peers_max<1 ||
+       st->transport_peers_max>=MAX_MOBILE_PEERS_MAX) {
+       cfgfatal(loc,"site","mobile-peers-max must be in range 1.."
+                STRING(MAX_MOBILE_PEERS_MAX) "\n");
+    }
+
+    if (st->key_lifetime < DEFAULT(KEY_RENEGOTIATE_GAP)*2)
        st->key_renegotiate_time=st->key_lifetime/2;
     else
-       st->key_renegotiate_time=st->key_lifetime-DEFAULT_KEY_RENEGOTIATE_GAP;
-    st->setup_retries=dict_read_number(
-       dict,"setup-retries",False,"site",loc,DEFAULT_SETUP_RETRIES);
-    st->setup_timeout=dict_read_number(
-       dict,"setup-timeout",False,"site",loc,DEFAULT_SETUP_TIMEOUT);
-    st->wait_timeout=dict_read_number(
-       dict,"wait-time",False,"site",loc,DEFAULT_WAIT_TIME);
+       st->key_renegotiate_time=st->key_lifetime-DEFAULT(KEY_RENEGOTIATE_GAP);
     st->key_renegotiate_time=dict_read_number(
-       dict,"renegotiate-time",False,"site",loc,st->key_lifetime);
+       dict,"renegotiate-time",False,"site",loc,st->key_renegotiate_time);
     if (st->key_renegotiate_time > st->key_lifetime) {
        cfgfatal(loc,"site",
                 "renegotiate-time must be less than key-lifetime\n");
     }
-    st->keepalive=dict_read_bool(dict,"keepalive",False,"site",loc,False);
 
     st->log_events=string_list_to_word(dict_lookup(dict,"log-events"),
                                       log_event_table,"site");
@@ -1266,40 +1670,54 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     sprintf(st->tunname,"%s<->%s",st->localname,st->remotename);
 
     /* The information we expect to see in incoming messages of type 1 */
-    st->setupsiglen=strlen(st->remotename)+strlen(st->localname)+8;
-    st->setupsig=safe_malloc(st->setupsiglen,"site_apply");
-    put_uint32(st->setupsig+0,LABEL_MSG1);
-    put_uint16(st->setupsig+4,strlen(st->remotename));
-    memcpy(&st->setupsig[6],st->remotename,strlen(st->remotename));
-    put_uint16(st->setupsig+(6+strlen(st->remotename)),strlen(st->localname));
-    memcpy(&st->setupsig[8+strlen(st->remotename)],st->localname,
-          strlen(st->localname));
+    /* fixme: lots of unchecked overflows here, but the results are only
+       corrupted packets rather than undefined behaviour */
     st->setup_priority=(strcmp(st->localname,st->remotename)>0);
 
     buffer_new(&st->buffer,SETUP_BUFFER_LEN);
 
+    buffer_new(&st->scratch,0);
+    BUF_ALLOC(&st->scratch,"site:scratch");
+
     /* We are interested in poll(), but only for timeouts. We don't have
        any fds of our own. */
     register_for_poll(st, site_beforepoll, site_afterpoll, 0, "site");
     st->timeout=0;
 
-    st->current_valid=False;
-    st->current_key_timeout=0;
-    st->peer_valid=False;
+    st->remote_capabilities=0;
+    st->current.key_timeout=0;
+    st->auxiliary_key.key_timeout=0;
+    transport_peers_clear(st,&st->peers);
+    transport_peers_clear(st,&st->setup_peers);
     /* XXX mlock these */
     st->dhsecret=safe_malloc(st->dh->len,"site:dhsecret");
-    st->sharedsecret=safe_malloc(st->transform->keylen,"site:sharedsecret");
+    st->sharedsecretlen=st->transform->keylen?:st->dh->ceil_len;
+    st->sharedsecret=safe_malloc(st->sharedsecretlen,"site:sharedsecret");
+
+    /* We need to compute some properties of our comms */
+#define COMPUTE_WORST(pad)                     \
+    int worst_##pad=0;                         \
+    for (i=0; i<st->ncomms; i++) {             \
+       int thispad=st->comms[i]->pad;          \
+       if (thispad > worst_##pad)              \
+           worst_##pad=thispad;                \
+    }
+    COMPUTE_WORST(min_start_pad)
+    COMPUTE_WORST(min_end_pad)
 
     /* We need to register the remote networks with the netlink device */
     st->netlink->reg(st->netlink->st, site_outgoing, st,
                     st->transform->max_start_pad+(4*4)+
-                    st->comm->min_start_pad,
-                    st->transform->max_end_pad+st->comm->min_end_pad);
+                    worst_min_start_pad,
+                    st->transform->max_end_pad+worst_min_end_pad);
     
-    st->comm->request_notify(st->comm->st, st, site_incoming);
+    for (i=0; i<st->ncomms; i++)
+       st->comms[i]->request_notify(st->comms[i]->st, st, site_incoming);
 
-    st->current_transform=st->transform->create(st->transform->st);
-    st->new_transform=st->transform->create(st->transform->st);
+    st->current.transform=0;
+    st->auxiliary_key.transform=0;
+    st->new_transform=0;
+    st->auxiliary_is_new=0;
 
     enter_state_stop(st);
 
@@ -1308,8 +1726,154 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     return new_closure(&st->cl);
 }
 
-init_module site_module;
 void site_module(dict_t *dict)
 {
     add_closure(dict,"site",site_apply);
 }
+
+
+/***** TRANSPORT PEERS definitions *****/
+
+static void transport_peers_debug(struct site *st, transport_peers *dst,
+                                 const char *didwhat,
+                                 int nargs, const struct comm_addr *args,
+                                 size_t stride) {
+    int i;
+    char *argp;
+
+    if (!(st->log_events & LOG_PEER_ADDRS))
+       return; /* an optimisation */
+
+    slog(st, LOG_PEER_ADDRS, "peers (%s) %s nargs=%d => npeers=%d",
+        (dst==&st->peers ? "data" :
+         dst==&st->setup_peers ? "setup" : "UNKNOWN"),
+        didwhat, nargs, dst->npeers);
+
+    for (i=0, argp=(void*)args;
+        i<nargs;
+        i++, (argp+=stride?stride:sizeof(*args))) {
+       const struct comm_addr *ca=(void*)argp;
+       slog(st, LOG_PEER_ADDRS, " args: addrs[%d]=%s",
+            i, ca->comm->addr_to_string(ca->comm->st,ca));
+    }
+    for (i=0; i<dst->npeers; i++) {
+       struct timeval diff;
+       timersub(tv_now,&dst->peers[i].last,&diff);
+       const struct comm_addr *ca=&dst->peers[i].addr;
+       slog(st, LOG_PEER_ADDRS, " peers: addrs[%d]=%s T-%ld.%06ld",
+            i, ca->comm->addr_to_string(ca->comm->st,ca),
+            (unsigned long)diff.tv_sec, (unsigned long)diff.tv_usec);
+    }
+}
+
+static int transport_peer_compar(const void *av, const void *bv) {
+    const transport_peer *a=av;
+    const transport_peer *b=bv;
+    /* put most recent first in the array */
+    if (timercmp(&a->last, &b->last, <)) return +1;
+    if (timercmp(&a->last, &b->last, >)) return -11;
+    return 0;
+}
+
+static void transport_peers_expire(struct site *st, transport_peers *peers) {
+    /* peers must be sorted first */
+    int previous_peers=peers->npeers;
+    struct timeval oldest;
+    oldest.tv_sec  = tv_now->tv_sec - st->mobile_peer_expiry;
+    oldest.tv_usec = tv_now->tv_usec;
+    while (peers->npeers>1 &&
+          timercmp(&peers->peers[peers->npeers-1].last, &oldest, <))
+       peers->npeers--;
+    if (peers->npeers != previous_peers)
+       transport_peers_debug(st,peers,"expire", 0,0,0);
+}
+
+static void transport_record_peer(struct site *st, transport_peers *peers,
+                                 const struct comm_addr *addr, const char *m) {
+    int slot, changed=0;
+
+    for (slot=0; slot<peers->npeers; slot++)
+       if (!memcmp(&peers->peers[slot].addr, addr, sizeof(*addr)))
+           goto found;
+
+    changed=1;
+    if (peers->npeers==st->transport_peers_max)
+       slot=st->transport_peers_max;
+    else
+       slot=peers->npeers++;
+
+ found:
+    peers->peers[slot].addr=*addr;
+    peers->peers[slot].last=*tv_now;
+
+    if (peers->npeers>1)
+       qsort(peers->peers, peers->npeers,
+             sizeof(*peers->peers), transport_peer_compar);
+
+    if (changed || peers->npeers!=1)
+       transport_peers_debug(st,peers,m, 1,addr,0);
+    transport_peers_expire(st, peers);
+}
+
+static bool_t transport_compute_setupinit_peers(struct site *st,
+        const struct comm_addr *configured_addr /* 0 if none or not found */) {
+
+    if (!configured_addr && !transport_peers_valid(&st->peers))
+       return False;
+
+    slog(st,LOG_SETUP_INIT,
+        (!configured_addr ? "using only %d old peer address(es)"
+         : "using configured address, and/or perhaps %d old peer address(es)"),
+        st->peers);
+
+    /* Non-mobile peers havve st->peers.npeers==0 or ==1, since they
+     * have transport_peers_max==1.  The effect is that this code
+     * always uses the configured address if supplied, or otherwise
+     * the existing data peer if one exists; this is as desired. */
+
+    transport_peers_copy(st,&st->setup_peers,&st->peers);
+
+    if (configured_addr)
+       transport_record_peer(st,&st->setup_peers,configured_addr,"setupinit");
+
+    assert(transport_peers_valid(&st->setup_peers));
+    return True;
+}
+
+static void transport_setup_msgok(struct site *st, const struct comm_addr *a) {
+    if (st->peer_mobile)
+       transport_record_peer(st,&st->setup_peers,a,"setupmsg");
+}
+static void transport_data_msgok(struct site *st, const struct comm_addr *a) {
+    if (st->peer_mobile)
+       transport_record_peer(st,&st->peers,a,"datamsg");
+}
+
+static int transport_peers_valid(transport_peers *peers) {
+    return peers->npeers;
+}
+static void transport_peers_clear(struct site *st, transport_peers *peers) {
+    peers->npeers= 0;
+    transport_peers_debug(st,peers,"clear",0,0,0);
+}
+static void transport_peers_copy(struct site *st, transport_peers *dst,
+                                const transport_peers *src) {
+    dst->npeers=src->npeers;
+    memcpy(dst->peers, src->peers, sizeof(*dst->peers) * dst->npeers);
+    transport_peers_debug(st,dst,"copy",
+                         src->npeers, &src->peers->addr, sizeof(*src->peers));
+}
+
+void transport_xmit(struct site *st, transport_peers *peers,
+                   struct buffer_if *buf, bool_t candebug) {
+    int slot;
+    transport_peers_expire(st, peers);
+    for (slot=0; slot<peers->npeers; slot++) {
+       transport_peer *peer=&peers->peers[slot];
+       if (candebug)
+           dump_packet(st, buf, &peer->addr, False);
+       peer->addr.comm->sendmsg(peer->addr.comm->st, buf, &peer->addr);
+    }
+}
+
+/***** END of transport peers declarations *****/