chiark / gitweb /
resolver: construct comm_addr; honour multiple addresses from the resolver
[secnet.git] / site.c
diff --git a/site.c b/site.c
index 9813de0..cea331e 100644 (file)
--- a/site.c
+++ b/site.c
@@ -89,6 +89,8 @@
 #define SITE_SENTMSG5 7
 #define SITE_WAIT     8
 
+int32_t site_max_start_pad = 4*4;
+
 static cstring_t state_name(uint32_t state)
 {
     switch (state) {
@@ -141,48 +143,84 @@ static struct flagstr log_event_table[]={
 /***** TRANSPORT PEERS declarations *****/
 
 /* Details of "mobile peer" semantics:
+
+ | Note: this comment is wishful thinking right now.  It will be
+ | implemented in subsequent commits.
+
+   - We use the same data structure for the different configurations,
+     but manage it with different algorithms.
    
-   - We record mobile_peers_max peer address/port numbers ("peers")
-     for key setup, and separately mobile_peers_max for data
-     transfer.  If these lists fill up, we retain the newest peers.
-     (For non-mobile peers we only record one of each.)
+   - We record up to mobile_peers_max peer address/port numbers
+     ("peers") for key setup, and separately up to mobile_peers_max
+     for data transfer.
+
+   - In general, we make a new set of addrs (see below) when we start
+     a new key exchange; the key setup addrs become the data transport
+     addrs when key setup complets.
+
+   If our peer is mobile:
+
+   - We send to all recent addresses of incoming packets, plus
+     initially all configured addresses (which we also expire).
+
+   - So, we record addrs of good incoming packets, as follows:
+      1. expire any peers last seen >120s ("mobile-peer-expiry") ago
+      2. add the peer of the just received packet to the applicable list
+         (possibly evicting the oldest entries to make room)
+     NB that we do not expire peers until an incoming packet arrives.
 
-   - Outgoing packets are sent to every recorded peer in the
-     applicable list.
+   - If the peer has a configured address or name, we record them the
+     same way, but only as a result of our own initiation of key
+     setup.  (We might evict some incoming packet addrs to make room.)
 
-   - Data transfer peers are straightforward: whenever we successfully
-     process a data packet, we record the peer.  Also, whenever we
-     successfully complete a key setup, we merge the key setup
+   - The default number of addrs to keep is 3, or 4 if we have a
+     configured name or address.  That's space for two configured
+     addresses (one IPv6 and one IPv4), plus two received addresses.
+
+   - Outgoing packets are sent to every recorded address in the
+     applicable list.  Any unsupported[1] addresses are deleted from
+     the list right away.  (This should only happen to configured
+     addresses, of course, but there is no need to check that.)
+
+   - When we successfully complete a key setup, we merge the key setup
      peers into the data transfer peers.
 
-     (For "non-mobile" peers we simply copy the peer used for
-     successful key setup, and don't change the peer otherwise.)
+   [1] An unsupported address is one for whose AF we don't have a
+     socket (perhaps because we got EAFNOSUPPORT or some such) or for
+     which sendto gives ENETUNREACH.
+
+   If neither end is mobile:
+
+   - When peer initiated the key exchange, we use the incoming packet
+     address.
+
+   - When we initiate the key exchange, we try configured addresses
+     until we get one which isn't unsupported then fixate on that.
+
+   - When we complete a key setup, we replace the data transport peers
+     with those from the key setup.
 
-   - Key setup peers are slightly more complicated.
+   If we are mobile:
 
-     Whenever we receive and successfully process a key exchange
-     packet, we record the peer.
+   - We can't tell when local network setup changes so we can't cache
+     the unsupported addrs and completely remove the spurious calls to
+     sendto, but we can optimise things a bit by deprioritising addrs
+     which seem to be unsupported.
 
-     Whenever we try to initiate a key setup, we copy the list of data
-     transfer peers and use it for key setup.  But we also look to see
-     if the config supplies an address and port number and if so we
-     add that as a key setup peer (possibly evicting one of the data
-     transfer peers we just copied).
+   - Use only configured addresses.  (Except, that if our peer
+     initiated a key exchange we use the incoming packet address until
+     our name resolution completes.)
 
-     (For "non-mobile" peers, if we if we have a configured peer
-     address and port, we always use that; otherwise if we have a
-     current data peer address we use that; otherwise we do not
-     attempt to initiate a key setup for lack of a peer address.)
+   - When we send a packet, try each address in turn; if addr
+     supported, put that address to the end of the list for future
+     packets, and go onto the next address.
 
-   "Record the peer" means
-    1. expire any peers last seen >120s ("mobile-peer-expiry") ago
-    2. add the peer of the just received packet to the applicable list
-       (possibly evicting older entries)
-   NB that we do not expire peers until an incoming packet arrives.
+   - When we complete a key setup, we replace the data transport peers
+     with those from the key setup.
 
    */
 
-#define MAX_MOBILE_PEERS_MAX 5 /* send at most this many copies, compiled max */
+#define MAX_MOBILE_PEERS_MAX MAX_PEER_ADDRS /* send at most this many copies */
 
 typedef struct {
     struct timeval last;
@@ -196,17 +234,36 @@ typedef struct {
     transport_peer peers[MAX_MOBILE_PEERS_MAX];
 } transport_peers;
 
+/* Basic operations on transport peer address sets */
 static void transport_peers_clear(struct site *st, transport_peers *peers);
 static int transport_peers_valid(transport_peers *peers);
 static void transport_peers_copy(struct site *st, transport_peers *dst,
                                 const transport_peers *src);
 
+/* Record address of incoming setup packet; resp. data packet. */
 static void transport_setup_msgok(struct site *st, const struct comm_addr *a);
 static void transport_data_msgok(struct site *st, const struct comm_addr *a);
+
+/* Initialise the setup addresses.  Called before we send the first
+ * packet in a key exchange.  If we are the initiator, as a result of
+ * resolve completing (or being determined not to be relevant) or an
+ * incoming PROD; if we are the responder, as a result of the MSG1. */
 static bool_t transport_compute_setupinit_peers(struct site *st,
-        const struct comm_addr *configured_addr /* 0 if none or not found */);
-static void transport_record_peer(struct site *st, transport_peers *peers,
-                                 const struct comm_addr *addr, const char *m);
+        const struct comm_addr *configured_addrs /* 0 if none or not found */,
+        int n_configured_addrs /* 0 if none or not found */,
+        const struct comm_addr *incoming_packet_addr /* 0 if none */);
+
+/* Called if we are the responder in a key setup, when the resolve
+ * completes.  transport_compute_setupinit_peers will hvae been called
+ * earlier.  If _complete is called, we are still doing the key setup
+ * (and we should use the new values for both the rest of the key
+ * setup and the ongoing data exchange); if _tardy is called, the key
+ * setup is done (either completed or not) and only the data peers are
+ * relevant */
+static void transport_resolve_complete(struct site *st,
+        const struct comm_addr *addrs, int naddrs);
+static void transport_resolve_complete_tardy(struct site *st,
+        const struct comm_addr *addrs, int naddrs);
 
 static void transport_xmit(struct site *st, transport_peers *peers,
                           struct buffer_if *buf, bool_t candebug);
@@ -226,11 +283,12 @@ struct site {
 /* configuration information */
     string_t localname;
     string_t remotename;
-    bool_t peer_mobile; /* Mobile client support */
+    bool_t local_mobile, peer_mobile; /* Mobile client support */
     int32_t transport_peers_max;
     string_t tunname; /* localname<->remotename by default, used in logs */
     string_t address; /* DNS name for bootstrapping, optional */
     int remoteport; /* Port for bootstrapping, optional */
+    uint32_t mtu_target;
     struct netlink_if *netlink;
     struct comm_if **comms;
     int ncomms;
@@ -239,7 +297,8 @@ struct site {
     struct random_if *random;
     struct rsaprivkey_if *privkey;
     struct rsapubkey_if *pubkey;
-    struct transform_if *transform;
+    struct transform_if **transforms;
+    int ntransforms;
     struct dh_if *dh;
     struct hash_if *hash;
 
@@ -261,6 +320,8 @@ struct site {
 /* runtime information */
     uint32_t state;
     uint64_t now; /* Most recently seen time */
+    bool_t allow_send_prod;
+    bool_t resolving;
 
     /* The currently established session */
     struct data_key current;
@@ -277,6 +338,8 @@ struct site {
        timeout before we can listen for another setup packet); perhaps
        we should keep a list of 'bad' sources for setup packets. */
     uint32_t remote_capabilities;
+    uint16_t remote_adv_mtu;
+    struct transform_if *chosen_transform;
     uint32_t setup_session_id;
     transport_peers setup_peers;
     uint8_t localN[NONCELEN]; /* Nonces for key exchange */
@@ -287,37 +350,79 @@ struct site {
     uint64_t timeout; /* Timeout for current state */
     uint8_t *dhsecret;
     uint8_t *sharedsecret;
-    uint32_t sharedsecretlen;
+    uint32_t sharedsecretlen, sharedsecretallocd;
     struct transform_inst_if *new_transform; /* For key setup/verify */
 };
 
-static void slog(struct site *st, uint32_t event, cstring_t msg, ...)
+static uint32_t event_log_priority(struct site *st, uint32_t event)
+{
+    if (!(event&st->log_events))
+       return 0;
+    switch(event) {
+    case LOG_UNEXPECTED:    return M_INFO;
+    case LOG_SETUP_INIT:    return M_INFO;
+    case LOG_SETUP_TIMEOUT: return M_NOTICE;
+    case LOG_ACTIVATE_KEY:  return M_INFO;
+    case LOG_TIMEOUT_KEY:   return M_INFO;
+    case LOG_SEC:           return M_SECURITY;
+    case LOG_STATE:         return M_DEBUG;
+    case LOG_DROP:          return M_DEBUG;
+    case LOG_DUMP:          return M_DEBUG;
+    case LOG_ERROR:         return M_ERR;
+    case LOG_PEER_ADDRS:    return M_DEBUG;
+    default:                return M_ERR;
+    }
+}
+
+static void vslog(struct site *st, uint32_t event, cstring_t msg, va_list ap)
+FORMAT(printf,3,0);
+static void vslog(struct site *st, uint32_t event, cstring_t msg, va_list ap)
 {
-    va_list ap;
-    char buf[240];
     uint32_t class;
 
+    class=event_log_priority(st, event);
+    if (class) {
+       slilog_part(st->log,class,"%s: ",st->tunname);
+       vslilog_part(st->log,class,msg,ap);
+       slilog_part(st->log,class,"\n");
+    }
+}
+
+static void slog(struct site *st, uint32_t event, cstring_t msg, ...)
+FORMAT(printf,3,4);
+static void slog(struct site *st, uint32_t event, cstring_t msg, ...)
+{
+    va_list ap;
     va_start(ap,msg);
+    vslog(st,event,msg,ap);
+    va_end(ap);
+}
 
-    if (event&st->log_events) {
-       switch(event) {
-       case LOG_UNEXPECTED: class=M_INFO; break;
-       case LOG_SETUP_INIT: class=M_INFO; break;
-       case LOG_SETUP_TIMEOUT: class=M_NOTICE; break;
-       case LOG_ACTIVATE_KEY: class=M_INFO; break;
-       case LOG_TIMEOUT_KEY: class=M_INFO; break;
-       case LOG_SEC: class=M_SECURITY; break;
-       case LOG_STATE: class=M_DEBUG; break;
-       case LOG_DROP: class=M_DEBUG; break;
-       case LOG_DUMP: class=M_DEBUG; break;
-       case LOG_ERROR: class=M_ERR; break;
-       case LOG_PEER_ADDRS: class=M_DEBUG; break;
-       default: class=M_ERR; break;
-       }
+static void logtimeout(struct site *st, const char *fmt, ...)
+FORMAT(printf,2,3);
+static void logtimeout(struct site *st, const char *fmt, ...)
+{
+    uint32_t class=event_log_priority(st,LOG_SETUP_TIMEOUT);
+    if (!class)
+       return;
+
+    va_list ap;
+    va_start(ap,fmt);
 
-       vsnprintf(buf,sizeof(buf),msg,ap);
-       st->log->log(st->log->st,class,"%s: %s",st->tunname,buf);
+    slilog_part(st->log,class,"%s: ",st->tunname);
+    vslilog_part(st->log,class,fmt,ap);
+
+    const char *delim;
+    int i;
+    for (i=0, delim=" (tried ";
+        i<st->setup_peers.npeers;
+        i++, delim=", ") {
+       transport_peer *peer=&st->setup_peers.peers[i];
+       const char *s=comm_addr_to_string(&peer->addr);
+       slilog_part(st->log,class,"%s%s",delim,s);
     }
+
+    slilog_part(st->log,class,")\n");
     va_end(ap);
 }
 
@@ -327,7 +432,8 @@ static void delete_one_key(struct site *st, struct data_key *key,
                           const char *reason /* may be 0 meaning don't log*/,
                           const char *which /* ignored if !reasonn */,
                           uint32_t loglevel /* ignored if !reasonn */);
-static bool_t initiate_key_setup(struct site *st, cstring_t reason);
+static bool_t initiate_key_setup(struct site *st, cstring_t reason,
+                                const struct comm_addr *prod_hint);
 static void enter_state_run(struct site *st);
 static bool_t enter_state_resolve(struct site *st);
 static bool_t enter_new_state(struct site *st,uint32_t next);
@@ -336,7 +442,7 @@ static void activate_new_key(struct site *st);
 
 static bool_t is_transform_valid(struct transform_inst_if *transform)
 {
-    return transform->valid(transform->st);
+    return transform && transform->valid(transform->st);
 }
 
 static bool_t current_valid(struct site *st)
@@ -350,6 +456,10 @@ static int call_transform_##fwdrev(struct site *st,                        \
                                   struct buffer_if *buf,               \
                                   const char **errmsg)                 \
 {                                                                      \
+    if (!is_transform_valid(transform)) {                              \
+       *errmsg="transform not set up";                                 \
+       return 1;                                                       \
+    }                                                                  \
     return transform->fwdrev(transform->st,buf,errmsg);                        \
 }
 
@@ -358,9 +468,12 @@ DEFINE_CALL_TRANSFORM(reverse)
 
 static void dispose_transform(struct transform_inst_if **transform_var)
 {
-    /* will become more sophisticated very shortly */
     struct transform_inst_if *transform=*transform_var;
-    transform->delkey(transform->st);
+    if (transform) {
+       transform->delkey(transform->st);
+       transform->destroy(transform->st);
+    }
+    *transform_var = 0;
 }    
 
 #define CHECK_AVAIL(b,l) do { if ((b)->size<(l)) return False; } while(0)
@@ -370,6 +483,14 @@ static void dispose_transform(struct transform_inst_if **transform_var)
     type=buf_unprepend_uint32((b)); \
     if (type!=(t)) return False; } while(0)
 
+static _Bool type_is_msg34(uint32_t type)
+{
+    return
+       type == LABEL_MSG3 ||
+       type == LABEL_MSG3BIS ||
+       type == LABEL_MSG4;
+}
+
 struct parsedname {
     int32_t len;
     uint8_t *name;
@@ -383,6 +504,8 @@ struct msg {
     struct parsedname remote;
     struct parsedname local;
     uint32_t remote_capabilities;
+    uint16_t remote_mtu;
+    int capab_transformnum;
     uint8_t *nR;
     uint8_t *nL;
     int32_t pklen;
@@ -392,10 +515,33 @@ struct msg {
     char *sig;
 };
 
-static void set_new_transform(struct site *st)
+static void set_new_transform(struct site *st, char *pk)
 {
-    st->new_transform->setkey(st->new_transform->st,st->sharedsecret,
-                             st->sharedsecretlen,st->setup_priority);
+    /* Make room for the shared key */
+    st->sharedsecretlen=st->chosen_transform->keylen?:st->dh->ceil_len;
+    assert(st->sharedsecretlen);
+    if (st->sharedsecretlen > st->sharedsecretallocd) {
+       st->sharedsecretallocd=st->sharedsecretlen;
+       st->sharedsecret=realloc(st->sharedsecret,st->sharedsecretallocd);
+    }
+    if (!st->sharedsecret) fatal_perror("site:sharedsecret");
+
+    /* Generate the shared key */
+    st->dh->makeshared(st->dh->st,st->dhsecret,st->dh->len,pk,
+                      st->sharedsecret,st->sharedsecretlen);
+
+    /* Set up the transform */
+    struct transform_if *generator=st->chosen_transform;
+    struct transform_inst_if *generated=generator->create(generator->st);
+    generated->setkey(generated->st,st->sharedsecret,
+                     st->sharedsecretlen,st->setup_priority);
+    dispose_transform(&st->new_transform);
+    st->new_transform=generated;
+
+    slog(st,LOG_SETUP_INIT,"key exchange negotiated transform"
+        " %d (capabilities ours=%#"PRIx32" theirs=%#"PRIx32")",
+        st->chosen_transform->capab_transformnum,
+        st->local_capabilities, st->remote_capabilities);
 }
 
 struct xinfoadd {
@@ -447,16 +593,22 @@ static bool_t generate_msg(struct site *st, uint32_t type, cstring_t what)
     if ((st->local_capabilities & CAPAB_EARLY) || (type != LABEL_MSG1)) {
        buf_append_uint32(&st->buffer,st->local_capabilities);
     }
+    if (type_is_msg34(type)) {
+       buf_append_uint16(&st->buffer,st->mtu_target);
+    }
     append_string_xinfo_done(&st->buffer,&xia);
 
     buf_append_string(&st->buffer,st->remotename);
-    memcpy(buf_append(&st->buffer,NONCELEN),st->localN,NONCELEN);
+    BUF_ADD_OBJ(append,&st->buffer,st->localN);
     if (type==LABEL_MSG1) return True;
-    memcpy(buf_append(&st->buffer,NONCELEN),st->remoteN,NONCELEN);
+    BUF_ADD_OBJ(append,&st->buffer,st->remoteN);
     if (type==LABEL_MSG2) return True;
 
     if (hacky_par_mid_failnow()) return False;
 
+    if (type==LABEL_MSG3BIS)
+       buf_append_uint8(&st->buffer,st->chosen_transform->capab_transformnum);
+
     dhpub=st->dh->makepublic(st->dh->st,st->dhsecret,st->dh->len);
     buf_append_string(&st->buffer,dhpub);
     free(dhpub);
@@ -490,6 +642,7 @@ static bool_t unpick_name(struct buffer_if *msg, struct parsedname *nm)
 static bool_t unpick_msg(struct site *st, uint32_t type,
                         struct buffer_if *msg, struct msg *m)
 {
+    m->capab_transformnum=-1;
     m->hashstart=msg->start;
     CHECK_AVAIL(msg,4);
     m->dest=buf_unprepend_uint32(msg);
@@ -498,11 +651,20 @@ static bool_t unpick_msg(struct site *st, uint32_t type,
     CHECK_TYPE(msg,type);
     if (!unpick_name(msg,&m->remote)) return False;
     m->remote_capabilities=0;
+    m->remote_mtu=0;
     if (m->remote.extrainfo.size) {
        CHECK_AVAIL(&m->remote.extrainfo,4);
        m->remote_capabilities=buf_unprepend_uint32(&m->remote.extrainfo);
     }
+    if (type_is_msg34(type) && m->remote.extrainfo.size) {
+       CHECK_AVAIL(&m->remote.extrainfo,2);
+       m->remote_mtu=buf_unprepend_uint16(&m->remote.extrainfo);
+    }
     if (!unpick_name(msg,&m->local)) return False;
+    if (type==LABEL_PROD) {
+       CHECK_EMPTY(msg);
+       return True;
+    }
     CHECK_AVAIL(msg,NONCELEN);
     m->nR=buf_unprepend(msg,NONCELEN);
     if (type==LABEL_MSG1) {
@@ -515,6 +677,12 @@ static bool_t unpick_msg(struct site *st, uint32_t type,
        CHECK_EMPTY(msg);
        return True;
     }
+    if (type==LABEL_MSG3BIS) {
+       CHECK_AVAIL(msg,1);
+       m->capab_transformnum = buf_unprepend_uint8(msg);
+    } else {
+       m->capab_transformnum = CAPAB_TRANSFORMNUM_ANCIENT;
+    }
     CHECK_AVAIL(msg,2);
     m->pklen=buf_unprepend_uint16(msg);
     CHECK_AVAIL(msg,m->pklen);
@@ -562,7 +730,7 @@ static bool_t check_msg(struct site *st, uint32_t type, struct msg *m,
     }
     /* MSG3 has complicated rules about capabilities, which are
      * handled in process_msg3. */
-    if (type==LABEL_MSG3) return True;
+    if (type==LABEL_MSG3 || type==LABEL_MSG3BIS) return True;
     if (m->remote_capabilities!=st->remote_capabilities) {
        *error="remote capabilities changed";
        return False;
@@ -585,7 +753,6 @@ static bool_t process_msg1(struct site *st, struct buffer_if *msg1,
        process an incoming MSG1, and that the MSG1 has correct values
        of A and B. */
 
-    transport_record_peer(st,&st->setup_peers,src,"msg1");
     st->setup_session_id=m->source;
     st->remote_capabilities=m->remote_capabilities;
     memcpy(st->remoteN,m->nR,NONCELEN);
@@ -611,6 +778,29 @@ static bool_t process_msg2(struct site *st, struct buffer_if *msg2,
     }
     st->setup_session_id=m.source;
     st->remote_capabilities=m.remote_capabilities;
+
+    /* Select the transform to use */
+
+    uint32_t remote_transforms = st->remote_capabilities & CAPAB_TRANSFORM_MASK;
+    if (!remote_transforms)
+       /* old secnets only had this one transform */
+       remote_transforms = 1UL << CAPAB_TRANSFORMNUM_ANCIENT;
+
+    struct transform_if *ti;
+    int i;
+    for (i=0; i<st->ntransforms; i++) {
+       ti=st->transforms[i];
+       if ((1UL << ti->capab_transformnum) & remote_transforms)
+           goto transform_found;
+    }
+    slog(st,LOG_ERROR,"no transforms in common"
+        " (us %#"PRIx32"; them: %#"PRIx32")",
+        st->local_capabilities & CAPAB_TRANSFORM_MASK,
+        remote_transforms);
+    return False;
+ transform_found:
+    st->chosen_transform=ti;
+
     memcpy(st->remoteN,m.nR,NONCELEN);
     return True;
 }
@@ -620,19 +810,46 @@ static bool_t generate_msg3(struct site *st)
     /* Now we have our nonce and their nonce. Think of a secret key,
        and create message number 3. */
     st->random->generate(st->random->st,st->dh->len,st->dhsecret);
-    return generate_msg(st,LABEL_MSG3,"site:MSG3");
+    return generate_msg(st,
+                       (st->remote_capabilities & CAPAB_TRANSFORM_MASK
+                        ? LABEL_MSG3BIS : LABEL_MSG3),
+                       "site:MSG3");
 }
 
-static bool_t process_msg3(struct site *st, struct buffer_if *msg3,
-                          const struct comm_addr *src)
+static bool_t process_msg3_msg4(struct site *st, struct msg *m)
 {
-    struct msg m;
     uint8_t *hash;
     void *hst;
+
+    /* Check signature and store g^x mod m */
+    hash=safe_malloc(st->hash->len, "process_msg3_msg4");
+    hst=st->hash->init();
+    st->hash->update(hst,m->hashstart,m->hashlen);
+    st->hash->final(hst,hash);
+    /* Terminate signature with a '0' - cheating, but should be ok */
+    m->sig[m->siglen]=0;
+    if (!st->pubkey->check(st->pubkey->st,hash,st->hash->len,m->sig)) {
+       slog(st,LOG_SEC,"msg3/msg4 signature failed check!");
+       free(hash);
+       return False;
+    }
+    free(hash);
+
+    st->remote_adv_mtu=m->remote_mtu;
+
+    return True;
+}
+
+static bool_t process_msg3(struct site *st, struct buffer_if *msg3,
+                          const struct comm_addr *src, uint32_t msgtype)
+{
+    struct msg m;
     cstring_t err;
 
-    if (!unpick_msg(st,LABEL_MSG3,msg3,&m)) return False;
-    if (!check_msg(st,LABEL_MSG3,&m,&err)) {
+    assert(msgtype==LABEL_MSG3 || msgtype==LABEL_MSG3BIS);
+
+    if (!unpick_msg(st,msgtype,msg3,&m)) return False;
+    if (!check_msg(st,msgtype,&m,&err)) {
        slog(st,LOG_SEC,"msg3: %s",err);
        return False;
     }
@@ -646,31 +863,29 @@ static bool_t process_msg3(struct site *st, struct buffer_if *msg3,
     }
     st->remote_capabilities|=m.remote_capabilities;
 
-    /* Check signature and store g^x mod m */
-    hash=safe_malloc(st->hash->len, "process_msg3");
-    hst=st->hash->init();
-    st->hash->update(hst,m.hashstart,m.hashlen);
-    st->hash->final(hst,hash);
-    /* Terminate signature with a '0' - cheating, but should be ok */
-    m.sig[m.siglen]=0;
-    if (!st->pubkey->check(st->pubkey->st,hash,st->hash->len,m.sig)) {
-       slog(st,LOG_SEC,"msg3 signature failed check!");
-       free(hash);
-       return False;
+    struct transform_if *ti;
+    int i;
+    for (i=0; i<st->ntransforms; i++) {
+       ti=st->transforms[i];
+       if (ti->capab_transformnum == m.capab_transformnum)
+           goto transform_found;
     }
-    free(hash);
+    slog(st,LOG_SEC,"peer chose unknown-to-us transform %d!",
+        m.capab_transformnum);
+    return False;
+ transform_found:
+    st->chosen_transform=ti;
+
+    if (!process_msg3_msg4(st,&m))
+       return False;
 
     /* Terminate their DH public key with a '0' */
     m.pk[m.pklen]=0;
     /* Invent our DH secret key */
     st->random->generate(st->random->st,st->dh->len,st->dhsecret);
 
-    /* Generate the shared key */
-    st->dh->makeshared(st->dh->st,st->dhsecret,st->dh->len,m.pk,
-                      st->sharedsecret,st->sharedsecretlen);
-
-    /* Set up the transform */
-    set_new_transform(st);
+    /* Generate the shared key and set up the transform */
+    set_new_transform(st,m.pk);
 
     return True;
 }
@@ -686,8 +901,6 @@ static bool_t process_msg4(struct site *st, struct buffer_if *msg4,
                           const struct comm_addr *src)
 {
     struct msg m;
-    uint8_t *hash;
-    void *hst;
     cstring_t err;
 
     if (!unpick_msg(st,LABEL_MSG4,msg4,&m)) return False;
@@ -696,27 +909,14 @@ static bool_t process_msg4(struct site *st, struct buffer_if *msg4,
        return False;
     }
     
-    /* Check signature and store g^x mod m */
-    hash=safe_malloc(st->hash->len, "process_msg4");
-    hst=st->hash->init();
-    st->hash->update(hst,m.hashstart,m.hashlen);
-    st->hash->final(hst,hash);
-    /* Terminate signature with a '0' - cheating, but should be ok */
-    m.sig[m.siglen]=0;
-    if (!st->pubkey->check(st->pubkey->st,hash,st->hash->len,m.sig)) {
-       slog(st,LOG_SEC,"msg4 signature failed check!");
-       free(hash);
+    if (!process_msg3_msg4(st,&m))
        return False;
-    }
-    free(hash);
 
     /* Terminate their DH public key with a '0' */
     m.pk[m.pklen]=0;
-    /* Generate the shared key */
-    st->dh->makeshared(st->dh->st,st->dhsecret,st->dh->len,m.pk,
-                      st->sharedsecret,st->sharedsecretlen);
-    /* Set up the transform */
-    set_new_transform(st);
+
+    /* Generate the shared key and set up the transform */
+    set_new_transform(st,m.pk);
 
     return True;
 }
@@ -746,7 +946,7 @@ static bool_t generate_msg5(struct site *st)
 
     BUF_ALLOC(&st->buffer,"site:MSG5");
     /* We are going to add four words to the message */
-    buffer_init(&st->buffer,st->transform->max_start_pad+(4*4));
+    buffer_init(&st->buffer,calculate_max_start_pad());
     /* Give the netlink code an opportunity to put its own stuff in the
        message (configuration information, etc.) */
     buf_prepend_uint32(&st->buffer,LABEL_MSG5);
@@ -793,7 +993,7 @@ static void create_msg6(struct site *st, struct transform_inst_if *transform,
 
     BUF_ALLOC(&st->buffer,"site:MSG6");
     /* We are going to add four words to the message */
-    buffer_init(&st->buffer,st->transform->max_start_pad+(4*4));
+    buffer_init(&st->buffer,calculate_max_start_pad());
     /* Give the netlink code an opportunity to put its own stuff in the
        message (configuration information, etc.) */
     buf_prepend_uint32(&st->buffer,LABEL_MSG6);
@@ -862,8 +1062,8 @@ static bool_t decrypt_msg0(struct site *st, struct buffer_if *msg0,
        goto skew;
 
     buffer_copy(msg0, &st->scratch);
-    problem = call_transform_reverse
-       (st,st->auxiliary_key.transform->st,msg0,&auxkey_err);
+    problem = call_transform_reverse(st,st->auxiliary_key.transform,
+                                    msg0,&auxkey_err);
     if (problem==0) {
        slog(st,LOG_DROP,"processing packet which uses auxiliary key");
        if (st->auxiliary_is_new) {
@@ -905,7 +1105,7 @@ static bool_t decrypt_msg0(struct site *st, struct buffer_if *msg0,
 
     slog(st,LOG_SEC,"transform: %s (aux: %s, new: %s)",
         transform_err,auxkey_err,newkey_err);
-    initiate_key_setup(st,"incoming message would not decrypt");
+    initiate_key_setup(st,"incoming message would not decrypt",0);
     send_nak(src,m.dest,m.source,m.type,msg0,"message would not decrypt");
     return False;
 
@@ -935,7 +1135,7 @@ static bool_t process_msg0(struct site *st, struct buffer_if *msg0,
        transport_data_msgok(st,src);
        /* See whether we should start negotiating a new key */
        if (st->now > st->renegotiate_key_time)
-           initiate_key_setup(st,"incoming packet in renegotiation window");
+           initiate_key_setup(st,"incoming packet in renegotiation window",0);
        return True;
     default:
        slog(st,LOG_SEC,"incoming encrypted message of type %08x "
@@ -971,7 +1171,7 @@ static bool_t send_msg(struct site *st)
        st->retries--;
        return True;
     } else if (st->state==SITE_SENTMSG5) {
-       slog(st,LOG_SETUP_TIMEOUT,"timed out sending MSG5, stashing new key");
+       logtimeout(st,"timed out sending MSG5, stashing new key");
        /* We stash the key we have produced, in case it turns out that
         * our peer did see our MSG5 after all and starts using it. */
        /* This is a bit like some of activate_new_key */
@@ -989,50 +1189,92 @@ static bool_t send_msg(struct site *st)
        enter_state_wait(st);
        return False;
     } else {
-       slog(st,LOG_SETUP_TIMEOUT,"timed out sending key setup packet "
+       logtimeout(st,"timed out sending key setup packet "
            "(in state %s)",state_name(st->state));
        enter_state_wait(st);
        return False;
     }
 }
 
-static void site_resolve_callback(void *sst, struct in_addr *address)
+static void site_resolve_callback(void *sst, const struct comm_addr *addrs,
+                                 int naddrs)
 {
     struct site *st=sst;
-    struct comm_addr ca_buf, *ca_use;
 
-    if (st->state!=SITE_RESOLVE) {
-       slog(st,LOG_UNEXPECTED,"site_resolve_callback called unexpectedly");
-       return;
-    }
-    if (address) {
-       FILLZERO(ca_buf);
-       ca_buf.comm=st->comms[0];
-       ca_buf.sin.sin_family=AF_INET;
-       ca_buf.sin.sin_port=htons(st->remoteport);
-       ca_buf.sin.sin_addr=*address;
-       ca_use=&ca_buf;
+    st->resolving=False;
+
+    if (naddrs) {
+       slog(st,LOG_STATE,"resolution of %s completed, %d addrs, eg: %s",
+            st->address, naddrs, comm_addr_to_string(&addrs[0]));;
     } else {
        slog(st,LOG_ERROR,"resolution of %s failed",st->address);
-       ca_use=0;
     }
-    if (transport_compute_setupinit_peers(st,ca_use)) {
-       enter_new_state(st,SITE_SENTMSG1);
-    } else {
-       /* Can't figure out who to try to to talk to */
-       slog(st,LOG_SETUP_INIT,"key exchange failed: cannot find peer address");
-       enter_state_run(st);
+
+    switch (st->state) {
+    case SITE_RESOLVE:
+        if (transport_compute_setupinit_peers(st,addrs,naddrs,0)) {
+           enter_new_state(st,SITE_SENTMSG1);
+       } else {
+           /* Can't figure out who to try to to talk to */
+           slog(st,LOG_SETUP_INIT,
+                "key exchange failed: cannot find peer address");
+           enter_state_run(st);
+       }
+       break;
+    case SITE_SENTMSG1: case SITE_SENTMSG2:
+    case SITE_SENTMSG3: case SITE_SENTMSG4:
+    case SITE_SENTMSG5:
+       if (naddrs) {
+           /* We start using the address immediately for data too.
+            * It's best to store it in st->peers now because we might
+            * go via SENTMSG5, WAIT, and a MSG0, straight into using
+            * the new key (without updating the data peer addrs). */
+           transport_resolve_complete(st,addrs,naddrs);
+       } else if (st->local_mobile) {
+           /* We can't let this rest because we may have a peer
+            * address which will break in the future. */
+           slog(st,LOG_SETUP_INIT,"resolution of %s failed: "
+                "abandoning key exchange",st->address);
+           enter_state_wait(st);
+       } else {
+           slog(st,LOG_SETUP_INIT,"resolution of %s failed: "
+                " continuing to use source address of peer's packets"
+                " for key exchange and ultimately data",
+                st->address);
+       }
+       break;
+    case SITE_RUN:
+       if (naddrs) {
+           slog(st,LOG_SETUP_INIT,"resolution of %s completed tardily,"
+                " updating peer address(es)",st->address);
+           transport_resolve_complete_tardy(st,addrs,naddrs);
+       } else if (st->local_mobile) {
+           /* Not very good.  We should queue (another) renegotiation
+            * so that we can update the peer address. */
+           st->key_renegotiate_time=st->now+st->wait_timeout;
+       } else {
+           slog(st,LOG_SETUP_INIT,"resolution of %s failed: "
+                " continuing to use source address of peer's packets",
+                st->address);
+       }
+       break;
+    case SITE_WAIT:
+    case SITE_STOP:
+       /* oh well */
+       break;
     }
 }
 
-static bool_t initiate_key_setup(struct site *st, cstring_t reason)
+static bool_t initiate_key_setup(struct site *st, cstring_t reason,
+                                const struct comm_addr *prod_hint)
 {
+    /* Reentrancy hazard: can call enter_new_state/enter_state_* */
     if (st->state!=SITE_RUN) return False;
     slog(st,LOG_SETUP_INIT,"initiating key exchange (%s)",reason);
     if (st->address) {
        slog(st,LOG_SETUP_INIT,"resolving peer address");
        return enter_state_resolve(st);
-    } else if (transport_compute_setupinit_peers(st,0)) {
+    } else if (transport_compute_setupinit_peers(st,0,0,prod_hint)) {
        return enter_new_state(st,SITE_SENTMSG1);
     }
     slog(st,LOG_SETUP_INIT,"key exchange failed: no address for peer");
@@ -1059,7 +1301,15 @@ static void activate_new_key(struct site *st)
     transport_peers_copy(st,&st->peers,&st->setup_peers);
     st->current.remote_session_id=st->setup_session_id;
 
-    slog(st,LOG_ACTIVATE_KEY,"new key activated");
+    /* Compute the inter-site MTU.  This is min( our_mtu, their_mtu ).
+     * But their mtu be unspecified, in which case we just use ours. */
+    uint32_t intersite_mtu=
+       MIN(st->mtu_target, st->remote_adv_mtu ?: ~(uint32_t)0);
+    st->netlink->set_mtu(st->netlink->st,intersite_mtu);
+
+    slog(st,LOG_ACTIVATE_KEY,"new key activated"
+        " (mtu ours=%"PRId32" theirs=%"PRId32" intersite=%"PRId32")",
+        st->mtu_target, st->remote_adv_mtu, intersite_mtu);
     enter_state_run(st);
 }
 
@@ -1121,22 +1371,44 @@ static void enter_state_run(struct site *st)
 
     st->setup_session_id=0;
     transport_peers_clear(st,&st->setup_peers);
-    memset(st->localN,0,NONCELEN);
-    memset(st->remoteN,0,NONCELEN);
+    FILLZERO(st->localN);
+    FILLZERO(st->remoteN);
     dispose_transform(&st->new_transform);
     memset(st->dhsecret,0,st->dh->len);
     memset(st->sharedsecret,0,st->sharedsecretlen);
     set_link_quality(st);
 }
 
+static bool_t ensure_resolving(struct site *st)
+{
+    /* Reentrancy hazard: may call site_resolve_callback and hence
+     * enter_new_state, enter_state_* and generate_msg*. */
+    if (st->resolving)
+        return True;
+
+    assert(st->address);
+
+    /* resolver->request might reentrantly call site_resolve_callback
+     * which will clear st->resolving, so we need to set it beforehand
+     * rather than afterwards; also, it might return False, in which
+     * case we have to clear ->resolving again. */
+    st->resolving=True;
+    bool_t ok = st->resolver->request(st->resolver->st,st->address,
+                                     st->remoteport,st->comms[0],
+                                     site_resolve_callback,st);
+    if (!ok)
+       st->resolving=False;
+
+    return ok;
+}
+
 static bool_t enter_state_resolve(struct site *st)
 {
+    /* Reentrancy hazard!  See ensure_resolving. */
     state_assert(st,st->state==SITE_RUN);
     slog(st,LOG_STATE,"entering state RESOLVE");
     st->state=SITE_RESOLVE;
-    st->resolver->request(st->resolver->st,st->address,
-                         site_resolve_callback,st);
-    return True;
+    return ensure_resolving(st);
 }
 
 static bool_t enter_new_state(struct site *st, uint32_t next)
@@ -1214,7 +1486,7 @@ static bool_t send_msg7(struct site *st, cstring_t reason)
     if (current_valid(st) && st->buffer.free
        && transport_peers_valid(&st->peers)) {
        BUF_ALLOC(&st->buffer,"site:MSG7");
-       buffer_init(&st->buffer,st->transform->max_start_pad+(4*3));
+       buffer_init(&st->buffer,calculate_max_start_pad());
        buf_append_uint32(&st->buffer,LABEL_MSG7);
        buf_append_string(&st->buffer,reason);
        if (call_transform_forwards(st, st->current.transform,
@@ -1244,6 +1516,30 @@ static void enter_state_wait(struct site *st)
     /* XXX Erase keys etc. */
 }
 
+static void generate_prod(struct site *st, struct buffer_if *buf)
+{
+    buffer_init(buf,0);
+    buf_append_uint32(buf,0);
+    buf_append_uint32(buf,0);
+    buf_append_uint32(buf,LABEL_PROD);
+    buf_append_string(buf,st->localname);
+    buf_append_string(buf,st->remotename);
+}
+
+static void generate_send_prod(struct site *st,
+                              const struct comm_addr *source)
+{
+    if (!st->allow_send_prod) return; /* too soon */
+    if (!(st->state==SITE_RUN || st->state==SITE_RESOLVE ||
+         st->state==SITE_WAIT)) return; /* we'd ignore peer's MSG1 */
+
+    slog(st,LOG_SETUP_INIT,"prodding peer for key exchange");
+    st->allow_send_prod=0;
+    generate_prod(st,&st->scratch);
+    dump_packet(st,&st->scratch,source,False);
+    source->comm->sendmsg(source->comm->st, &st->scratch, source);
+}
+
 static inline void site_settimeout(uint64_t timeout, int *timeout_io)
 {
     if (timeout) {
@@ -1316,6 +1612,8 @@ static void site_outgoing(void *sst, struct buffer_if *buf)
        return;
     }
 
+    st->allow_send_prod=1;
+
     /* In all other states we consider delivering the packet if we have
        a valid key and a valid address to send it to. */
     if (current_valid(st) && transport_peers_valid(&st->peers)) {
@@ -1337,7 +1635,7 @@ static void site_outgoing(void *sst, struct buffer_if *buf)
 
     slog(st,LOG_DROP,"discarding outgoing packet of size %d",buf->size);
     BUF_FREE(buf);
-    initiate_key_setup(st,"outgoing packet");
+    initiate_key_setup(st,"outgoing packet",0);
 }
 
 static bool_t named_for_us(struct site *st, const struct buffer_if *buf_in,
@@ -1353,7 +1651,10 @@ static bool_t named_for_us(struct site *st, const struct buffer_if *buf_in,
 }
 
 /* This function is called by the communication device to deliver
-   packets from our peers. */
+   packets from our peers.
+   It should return True if the packet is recognised as being for
+   this current site instance (and should therefore not be processed
+   by other sites), even if the packet was otherwise ignored. */
 static bool_t site_incoming(void *sst, struct buffer_if *buf,
                            const struct comm_addr *source)
 {
@@ -1373,9 +1674,14 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
        if (st->state==SITE_RUN || st->state==SITE_RESOLVE ||
            st->state==SITE_WAIT) {
            /* We should definitely process it */
+           transport_compute_setupinit_peers(st,0,0,source);
            if (process_msg1(st,buf,source,&named_msg)) {
                slog(st,LOG_SETUP_INIT,"key setup initiated by peer");
-               enter_new_state(st,SITE_SENTMSG2);
+               bool_t entered=enter_new_state(st,SITE_SENTMSG2);
+               if (entered && st->address && st->local_mobile)
+                   /* We must do this as the very last thing, because
+                      the resolver callback might reenter us. */
+                   ensure_resolving(st);
            } else {
                slog(st,LOG_ERROR,"failed to process incoming msg1");
            }
@@ -1395,6 +1701,7 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
                     "priority => use incoming msg1");
                if (process_msg1(st,buf,source,&named_msg)) {
                    BUF_FREE(&st->buffer); /* Free our old message 1 */
+                   transport_setup_msgok(st,source);
                    enter_new_state(st,SITE_SENTMSG2);
                } else {
                    slog(st,LOG_ERROR,"failed to process an incoming "
@@ -1410,6 +1717,20 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
        BUF_FREE(buf);
        return True;
     }
+    if (msgtype==LABEL_PROD) {
+       if (!named_for_us(st,buf,msgtype,&named_msg))
+           return False;
+       dump_packet(st,buf,source,True);
+       if (st->state!=SITE_RUN) {
+           slog(st,LOG_DROP,"ignoring PROD when not in state RUN");
+       } else if (current_valid(st)) {
+           slog(st,LOG_DROP,"ignoring PROD when we think we have a key");
+       } else {
+           initiate_key_setup(st,"peer sent PROD packet",source);
+       }
+       BUF_FREE(buf);
+       return True;
+    }
     if (dest==st->index) {
        /* Explicitly addressed to us */
        if (msgtype!=LABEL_MSG0) dump_packet(st,buf,source,True);
@@ -1418,7 +1739,9 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
            /* If the source is our current peer then initiate a key setup,
               because our peer's forgotten the key */
            if (get_uint32(buf->start+4)==st->current.remote_session_id) {
-               initiate_key_setup(st,"received a NAK");
+               bool_t initiated;
+               initiated = initiate_key_setup(st,"received a NAK",source);
+               if (!initiated) generate_send_prod(st,source);
            } else {
                slog(st,LOG_SEC,"bad incoming NAK");
            }
@@ -1443,10 +1766,11 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
            }
            break;
        case LABEL_MSG3:
+       case LABEL_MSG3BIS:
            /* Setup packet: expected only in state SENTMSG2 */
            if (st->state!=SITE_SENTMSG2) {
                slog(st,LOG_UNEXPECTED,"unexpected MSG3");
-           } else if (process_msg3(st,buf,source)) {
+           } else if (process_msg3(st,buf,source,msgtype)) {
                transport_setup_msgok(st,source);
                enter_new_state(st,SITE_SENTMSG4);
            } else {
@@ -1561,7 +1885,7 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     st->remotename=dict_read_string(dict, "name", True, "site", loc);
 
     st->peer_mobile=dict_read_bool(dict,"mobile",False,"site",loc,False);
-    bool_t local_mobile=
+    st->local_mobile=
        dict_read_bool(dict,"local-mobile",False,"site",loc,False);
 
     /* Sanity check (which also allows the 'sites' file to include
@@ -1570,14 +1894,14 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     if (strcmp(st->localname,st->remotename)==0) {
        Message(M_DEBUG,"site %s: local-name==name -> ignoring this site\n",
                st->localname);
-       if (st->peer_mobile != local_mobile)
+       if (st->peer_mobile != st->local_mobile)
            cfgfatal(loc,"site","site %s's peer-mobile=%d"
                    " but our local-mobile=%d\n",
-                   st->localname, st->peer_mobile, local_mobile);
+                   st->localname, st->peer_mobile, st->local_mobile);
        free(st);
        return NULL;
     }
-    if (st->peer_mobile && local_mobile) {
+    if (st->peer_mobile && st->local_mobile) {
        Message(M_WARNING,"site %s: site is mobile but so are we"
                " -> ignoring this site\n", st->remotename);
        free(st);
@@ -1589,19 +1913,25 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     st->local_capabilities = 0;
     st->netlink=find_cl_if(dict,"link",CL_NETLINK,True,"site",loc);
 
-    list_t *comms_cfg=dict_lookup(dict,"comm");
-    if (!comms_cfg) cfgfatal(loc,"site","closure list \"comm\" not found\n");
-    st->ncomms=list_length(comms_cfg);
-    st->comms=safe_malloc_ary(sizeof(*st->comms),st->ncomms,"comms");
-    assert(st->ncomms);
-    for (i=0; i<st->ncomms; i++) {
-       item_t *item=list_elem(comms_cfg,i);
-       if (item->type!=t_closure)
-           cfgfatal(loc,"site","comm is not a closure\n");
-       closure_t *cl=item->data.closure;
-       if (cl->type!=CL_COMM) cfgfatal(loc,"site","comm closure wrong type\n");
-       st->comms[i]=cl->interface;
-    }
+#define GET_CLOSURE_LIST(dictkey,things,nthings,CL_TYPE) do{           \
+    list_t *things##_cfg=dict_lookup(dict,dictkey);                    \
+    if (!things##_cfg)                                                 \
+       cfgfatal(loc,"site","closure list \"%s\" not found\n",dictkey); \
+    st->nthings=list_length(things##_cfg);                             \
+    st->things=safe_malloc_ary(sizeof(*st->things),st->nthings,dictkey "s"); \
+    assert(st->nthings);                                               \
+    for (i=0; i<st->nthings; i++) {                                    \
+       item_t *item=list_elem(things##_cfg,i);                         \
+       if (item->type!=t_closure)                                      \
+           cfgfatal(loc,"site","%s is not a closure\n",dictkey);       \
+       closure_t *cl=item->data.closure;                               \
+       if (cl->type!=CL_TYPE)                                          \
+           cfgfatal(loc,"site","%s closure wrong type\n",dictkey);     \
+       st->things[i]=cl->interface;                                    \
+    }                                                                  \
+}while(0)
+
+    GET_CLOSURE_LIST("comm",comms,ncomms,CL_COMM);
 
     st->resolver=find_cl_if(dict,"resolver",CL_RESOLVER,True,"site",loc);
     st->log=find_cl_if(dict,"log",CL_LOG,True,"site",loc);
@@ -1614,13 +1944,12 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     else st->remoteport=0;
     st->pubkey=find_cl_if(dict,"key",CL_RSAPUBKEY,True,"site",loc);
 
-    st->transform=
-       find_cl_if(dict,"transform",CL_TRANSFORM,True,"site",loc);
+    GET_CLOSURE_LIST("transform",transforms,ntransforms,CL_TRANSFORM);
 
     st->dh=find_cl_if(dict,"dh",CL_DH,True,"site",loc);
     st->hash=find_cl_if(dict,"hash",CL_HASH,True,"site",loc);
 
-#define DEFAULT(D) (st->peer_mobile || local_mobile    \
+#define DEFAULT(D) (st->peer_mobile || st->local_mobile        \
                     ? DEFAULT_MOBILE_##D : DEFAULT_##D)
 #define CFG_NUMBER(k,D) dict_read_number(dict,(k),False,"site",loc,DEFAULT(D));
 
@@ -1628,6 +1957,7 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     st->setup_retries=        CFG_NUMBER("setup-retries", SETUP_RETRIES);
     st->setup_retry_interval= CFG_NUMBER("setup-timeout", SETUP_RETRY_INTERVAL);
     st->wait_timeout=         CFG_NUMBER("wait-time",     WAIT_TIME);
+    st->mtu_target= dict_read_number(dict,"mtu-target",False,"site",loc,0);
 
     st->mobile_peer_expiry= dict_read_number(
        dict,"mobile-peer-expiry",False,"site",loc,DEFAULT_MOBILE_PEER_EXPIRY);
@@ -1654,6 +1984,9 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     st->log_events=string_list_to_word(dict_lookup(dict,"log-events"),
                                       log_event_table,"site");
 
+    st->resolving=False;
+    st->allow_send_prod=0;
+
     st->tunname=safe_malloc(strlen(st->localname)+strlen(st->remotename)+5,
                            "site_apply");
     sprintf(st->tunname,"%s<->%s",st->localname,st->remotename);
@@ -1665,7 +1998,7 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
 
     buffer_new(&st->buffer,SETUP_BUFFER_LEN);
 
-    buffer_new(&st->scratch,0);
+    buffer_new(&st->scratch,SETUP_BUFFER_LEN);
     BUF_ALLOC(&st->scratch,"site:scratch");
 
     /* We are interested in poll(), but only for timeouts. We don't have
@@ -1674,38 +2007,37 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     st->timeout=0;
 
     st->remote_capabilities=0;
+    st->chosen_transform=0;
     st->current.key_timeout=0;
     st->auxiliary_key.key_timeout=0;
     transport_peers_clear(st,&st->peers);
     transport_peers_clear(st,&st->setup_peers);
     /* XXX mlock these */
     st->dhsecret=safe_malloc(st->dh->len,"site:dhsecret");
-    st->sharedsecretlen=st->transform->keylen?:st->dh->ceil_len;
-    st->sharedsecret=safe_malloc(st->sharedsecretlen,"site:sharedsecret");
-
-    /* We need to compute some properties of our comms */
-#define COMPUTE_WORST(pad)                     \
-    int worst_##pad=0;                         \
-    for (i=0; i<st->ncomms; i++) {             \
-       int thispad=st->comms[i]->pad;          \
-       if (thispad > worst_##pad)              \
-           worst_##pad=thispad;                \
+    st->sharedsecretlen=st->sharedsecretallocd=0;
+    st->sharedsecret=0;
+
+    for (i=0; i<st->ntransforms; i++) {
+       struct transform_if *ti=st->transforms[i];
+       uint32_t capbit = 1UL << ti->capab_transformnum;
+       if (st->local_capabilities & capbit)
+           slog(st,LOG_ERROR,"transformnum capability bit"
+                " %d (%#"PRIx32") reused", ti->capab_transformnum, capbit);
+       st->local_capabilities |= capbit;
     }
-    COMPUTE_WORST(min_start_pad)
-    COMPUTE_WORST(min_end_pad)
 
     /* We need to register the remote networks with the netlink device */
-    st->netlink->reg(st->netlink->st, site_outgoing, st,
-                    st->transform->max_start_pad+(4*4)+
-                    worst_min_start_pad,
-                    st->transform->max_end_pad+worst_min_end_pad);
+    uint32_t netlink_mtu; /* local virtual interface mtu */
+    st->netlink->reg(st->netlink->st, site_outgoing, st, &netlink_mtu);
+    if (!st->mtu_target)
+       st->mtu_target=netlink_mtu;
     
     for (i=0; i<st->ncomms; i++)
        st->comms[i]->request_notify(st->comms[i]->st, st, site_incoming);
 
-    st->current.transform=st->transform->create(st->transform->st);
-    st->auxiliary_key.transform=st->transform->create(st->transform->st);
-    st->new_transform=st->transform->create(st->transform->st);
+    st->current.transform=0;
+    st->auxiliary_key.transform=0;
+    st->new_transform=0;
     st->auxiliary_is_new=0;
 
     enter_state_stop(st);
@@ -1743,25 +2075,21 @@ static void transport_peers_debug(struct site *st, transport_peers *dst,
         i++, (argp+=stride?stride:sizeof(*args))) {
        const struct comm_addr *ca=(void*)argp;
        slog(st, LOG_PEER_ADDRS, " args: addrs[%d]=%s",
-            i, ca->comm->addr_to_string(ca->comm->st,ca));
+            i, comm_addr_to_string(ca));
     }
     for (i=0; i<dst->npeers; i++) {
        struct timeval diff;
        timersub(tv_now,&dst->peers[i].last,&diff);
        const struct comm_addr *ca=&dst->peers[i].addr;
        slog(st, LOG_PEER_ADDRS, " peers: addrs[%d]=%s T-%ld.%06ld",
-            i, ca->comm->addr_to_string(ca->comm->st,ca),
+            i, comm_addr_to_string(ca),
             (unsigned long)diff.tv_sec, (unsigned long)diff.tv_usec);
     }
 }
 
-static int transport_peer_compar(const void *av, const void *bv) {
-    const transport_peer *a=av;
-    const transport_peer *b=bv;
-    /* put most recent first in the array */
-    if (timercmp(&a->last, &b->last, <)) return +1;
-    if (timercmp(&a->last, &b->last, >)) return -11;
-    return 0;
+static bool_t transport_addrs_equal(const struct comm_addr *a,
+                                   const struct comm_addr *b) {
+    return !memcmp(a,b,sizeof(*a));
 }
 
 static void transport_peers_expire(struct site *st, transport_peers *peers) {
@@ -1777,65 +2105,105 @@ static void transport_peers_expire(struct site *st, transport_peers *peers) {
        transport_peers_debug(st,peers,"expire", 0,0,0);
 }
 
-static void transport_record_peer(struct site *st, transport_peers *peers,
-                                 const struct comm_addr *addr, const char *m) {
-    int slot, changed=0;
+static bool_t transport_peer_record_one(struct site *st, transport_peers *peers,
+                                       const struct comm_addr *ca,
+                                       const struct timeval *tv) {
+    /* returns false if output is full */
+    int search;
 
-    for (slot=0; slot<peers->npeers; slot++)
-       if (!memcmp(&peers->peers[slot].addr, addr, sizeof(*addr)))
-           goto found;
+    if (peers->npeers > st->transport_peers_max)
+       return 0;
 
-    changed=1;
-    if (peers->npeers==st->transport_peers_max)
-       slot=st->transport_peers_max;
-    else
-       slot=peers->npeers++;
+    for (search=0; search<peers->npeers; search++)
+       if (transport_addrs_equal(&peers->peers[search].addr, ca))
+           return 1;
+
+    peers->peers[peers->npeers].addr = *ca;
+    peers->peers[peers->npeers].last = *tv;
+    peers->npeers++;
+    return 1;
+}
+
+static void transport_record_peers(struct site *st, transport_peers *peers,
+                                  const struct comm_addr *addrs, int naddrs,
+                                  const char *m) {
+    /* We add addrs into peers.  The new entries end up at the front
+     * and displace entries towards the end (perhaps even off the end).
+     * Any existing matching entries are moved up to the front.
+     * Caller must first call transport_peers_expire. */
 
- found:
-    peers->peers[slot].addr=*addr;
-    peers->peers[slot].last=*tv_now;
+    if (naddrs==1 && peers->npeers>=1 &&
+       transport_addrs_equal(&addrs[0], &peers->peers[0].addr)) {
+       /* optimisation, also avoids debug for trivial updates */
+       peers->peers[0].last = *tv_now;
+       return;
+    }
 
-    if (peers->npeers>1)
-       qsort(peers->peers, peers->npeers,
-             sizeof(*peers->peers), transport_peer_compar);
+    int old_npeers=peers->npeers;
+    transport_peer old_peers[old_npeers];
+    COPY_ARRAY(old_peers,peers->peers,old_npeers);
 
-    if (changed || peers->npeers!=1)
-       transport_peers_debug(st,peers,m, 1,addr,0);
-    transport_peers_expire(st, peers);
+    peers->npeers=0;
+    int i;
+    for (i=0; i<naddrs; i++) {
+       if (!transport_peer_record_one(st,peers, &addrs[i], tv_now))
+           break;
+    }
+    for (i=0; i<old_npeers; i++) {
+       const transport_peer *old=&old_peers[i];
+       if (!transport_peer_record_one(st,peers, &old->addr, &old->last))
+           break;
+    }
+
+    transport_peers_debug(st,peers,m, naddrs,addrs,0);
 }
 
 static bool_t transport_compute_setupinit_peers(struct site *st,
-        const struct comm_addr *configured_addr /* 0 if none or not found */) {
-
-    if (!configured_addr && !transport_peers_valid(&st->peers))
+        const struct comm_addr *configured_addrs /* 0 if none or not found */,
+        int n_configured_addrs /* 0 if none or not found */,
+        const struct comm_addr *incoming_packet_addr /* 0 if none */) {
+    if (!n_configured_addrs && !incoming_packet_addr &&
+       !transport_peers_valid(&st->peers))
        return False;
 
     slog(st,LOG_SETUP_INIT,
-        (!configured_addr ? "using only %d old peer address(es)"
-         : "using configured address, and/or perhaps %d old peer address(es)"),
-        st->peers);
+        "using: %d configured addr(s);%s %d old peer addrs(es)",
+        n_configured_addrs,
+        incoming_packet_addr ? " incoming packet address;" : "",
+        st->peers.npeers);
 
-    /* Non-mobile peers havve st->peers.npeers==0 or ==1, since they
+    /* Non-mobile peers have st->peers.npeers==0 or ==1, since they
      * have transport_peers_max==1.  The effect is that this code
      * always uses the configured address if supplied, or otherwise
-     * the existing data peer if one exists; this is as desired. */
+     * the address of the incoming PROD, or the existing data peer if
+     * one exists; this is as desired. */
 
     transport_peers_copy(st,&st->setup_peers,&st->peers);
+    transport_peers_expire(st,&st->setup_peers);
 
-    if (configured_addr)
-       transport_record_peer(st,&st->setup_peers,configured_addr,"setupinit");
+    if (incoming_packet_addr)
+       transport_record_peers(st,&st->setup_peers,
+                              incoming_packet_addr,1, "incoming");
+
+    if (n_configured_addrs)
+       transport_record_peers(st,&st->setup_peers,
+                             configured_addrs,n_configured_addrs, "setupinit");
 
     assert(transport_peers_valid(&st->setup_peers));
     return True;
 }
 
 static void transport_setup_msgok(struct site *st, const struct comm_addr *a) {
-    if (st->peer_mobile)
-       transport_record_peer(st,&st->setup_peers,a,"setupmsg");
+    if (st->peer_mobile) {
+       transport_peers_expire(st,&st->setup_peers);
+       transport_record_peers(st,&st->setup_peers,a,1,"setupmsg");
+    }
 }
 static void transport_data_msgok(struct site *st, const struct comm_addr *a) {
-    if (st->peer_mobile)
-       transport_record_peer(st,&st->peers,a,"datamsg");
+    if (st->peer_mobile) {
+       transport_peers_expire(st,&st->setup_peers);
+       transport_record_peers(st,&st->peers,a,1,"datamsg");
+    }
 }
 
 static int transport_peers_valid(transport_peers *peers) {
@@ -1848,20 +2216,84 @@ static void transport_peers_clear(struct site *st, transport_peers *peers) {
 static void transport_peers_copy(struct site *st, transport_peers *dst,
                                 const transport_peers *src) {
     dst->npeers=src->npeers;
-    memcpy(dst->peers, src->peers, sizeof(*dst->peers) * dst->npeers);
+    COPY_ARRAY(dst->peers, src->peers, dst->npeers);
     transport_peers_debug(st,dst,"copy",
                          src->npeers, &src->peers->addr, sizeof(*src->peers));
 }
 
+static void transport_resolve_complete(struct site *st,
+                                      const struct comm_addr *addrs,
+                                      int naddrs) {
+    transport_peers_expire(st,&st->peers);
+    transport_record_peers(st,&st->peers,addrs,naddrs,"resolved data");
+    transport_peers_expire(st,&st->setup_peers);
+    transport_record_peers(st,&st->setup_peers,addrs,naddrs,"resolved setup");
+}
+
+static void transport_resolve_complete_tardy(struct site *st,
+                                            const struct comm_addr *addrs,
+                                            int naddrs) {
+    transport_peers_expire(st,&st->peers);
+    transport_record_peers(st,&st->peers,addrs,naddrs,"resolved tardily");
+}
+
+static void transport_peers__copy_by_mask(transport_peer *out, int *nout_io,
+                                         unsigned mask,
+                                         const transport_peers *inp) {
+    /* out and in->peers may be the same region, or nonoverlapping */
+    const transport_peer *in=inp->peers;
+    int slot;
+    for (slot=0; slot<inp->npeers; slot++) {
+       if (!(mask & (1U << slot)))
+           continue;
+       if (!(out==in && slot==*nout_io))
+           COPY_OBJ(out[*nout_io], in[slot]);
+       (*nout_io)++;
+    }
+}
+
 void transport_xmit(struct site *st, transport_peers *peers,
                    struct buffer_if *buf, bool_t candebug) {
     int slot;
     transport_peers_expire(st, peers);
+    unsigned failed=0; /* bitmask */
+    assert(MAX_MOBILE_PEERS_MAX < sizeof(unsigned)*CHAR_BIT);
+
+    int nfailed=0;
     for (slot=0; slot<peers->npeers; slot++) {
        transport_peer *peer=&peers->peers[slot];
        if (candebug)
            dump_packet(st, buf, &peer->addr, False);
-       peer->addr.comm->sendmsg(peer->addr.comm->st, buf, &peer->addr);
+       bool_t ok =
+           peer->addr.comm->sendmsg(peer->addr.comm->st, buf, &peer->addr);
+       if (!ok) {
+           failed |= 1U << slot;
+           nfailed++;
+       }
+       if (ok && !st->peer_mobile)
+           break;
+    }
+    /* Now we need to demote/delete failing addrs: if we are mobile we
+     * merely demote them; otherwise we delete them. */
+    if (st->local_mobile) {
+       unsigned expected = ((1U << nfailed)-1) << (peers->npeers-nfailed);
+       /* `expected' has all the failures at the end already */
+       if (failed != expected) {
+           int fslot=0;
+           transport_peer failedpeers[nfailed];
+           transport_peers__copy_by_mask(failedpeers, &fslot, failed,peers);
+           assert(fslot == nfailed);
+           int wslot=0;
+           transport_peers__copy_by_mask(peers->peers,&wslot,~failed,peers);
+           assert(wslot+nfailed == peers->npeers);
+           COPY_ARRAY(peers->peers+wslot, failedpeers, nfailed);
+       }
+    } else {
+       if (failed && peers->npeers > 1) {
+           int wslot=0;
+           transport_peers__copy_by_mask(peers->peers,&wslot,~failed,peers);
+           peers->npeers=wslot;
+       }
     }
 }