chiark / gitweb /
site: Read public peer keys from key file
[secnet.git] / site.c
diff --git a/site.c b/site.c
index 1c57a8a66b6296ca305b7efab2d4cf9035838c56..002b7dd48b84b8fb90347f8cbd9d98e8c94908e7 100644 (file)
--- a/site.c
+++ b/site.c
@@ -1,5 +1,24 @@
 /* site.c - manage communication with a remote network site */
 
+/*
+ * This file is part of secnet.
+ * See README for full list of copyright holders.
+ *
+ * secnet is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ * 
+ * secnet is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * version 3 along with secnet; if not, see
+ * https://www.gnu.org/licenses/gpl.html.
+ */
+
 /* The 'site' code doesn't know anything about the structure of the
    packets it's transmitting.  In fact, under the new netlink
    configuration scheme it doesn't need to know anything at all about
@@ -24,6 +43,7 @@
 #include "util.h"
 #include "unaligned.h"
 #include "magic.h"
+#include "pubkeys.h"
 
 #define SETUP_BUFFER_LEN 2048
 
 #define SITE_SENTMSG5 7
 #define SITE_WAIT     8
 
+#define CASES_MSG3_KNOWN LABEL_MSG3: case LABEL_MSG3BIS
+
+struct msg;
+
 int32_t site_max_start_pad = 4*4;
 
 static cstring_t state_name(uint32_t state)
@@ -119,6 +143,7 @@ static cstring_t state_name(uint32_t state)
 #define LOG_DUMP          0x00000100
 #define LOG_ERROR         0x00000400
 #define LOG_PEER_ADDRS    0x00000800
+#define LOG_SIGKEYS       0x00001000
 
 static struct flagstr log_event_table[]={
     { "unexpected", LOG_UNEXPECTED },
@@ -132,8 +157,9 @@ static struct flagstr log_event_table[]={
     { "dump-packets", LOG_DUMP },
     { "errors", LOG_ERROR },
     { "peer-addrs", LOG_PEER_ADDRS },
+    { "sigkeys", LOG_SIGKEYS },
     { "default", LOG_SETUP_INIT|LOG_SETUP_TIMEOUT|
-      LOG_ACTIVATE_KEY|LOG_TIMEOUT_KEY|LOG_SEC|LOG_ERROR },
+      LOG_ACTIVATE_KEY|LOG_TIMEOUT_KEY|LOG_SEC|LOG_ERROR|LOG_SIGKEYS },
     { "all", 0xffffffff },
     { NULL, 0 }
 };
@@ -277,6 +303,7 @@ struct site {
 /* configuration information */
     string_t localname;
     string_t remotename;
+    bool_t keepalive;
     bool_t local_mobile, peer_mobile; /* Mobile client support */
     int32_t transport_peers_max;
     string_t tunname; /* localname<->remotename by default, used in logs */
@@ -285,40 +312,43 @@ struct site {
     uint32_t mtu_target;
     struct netlink_if *netlink;
     struct comm_if **comms;
+    struct comm_clientinfo **commclientinfos;
     int ncomms;
     struct resolver_if *resolver;
     struct log_if *log;
     struct random_if *random;
-    struct rsaprivkey_if *privkey;
-    struct rsapubkey_if *pubkey;
+    struct privcache_if *privkeys;
+    struct sigprivkey_if *privkey_fixed;
     struct transform_if **transforms;
     int ntransforms;
     struct dh_if *dh;
-    struct hash_if *hash;
 
     uint32_t index; /* Index of this site */
+    uint32_t early_capabilities;
     uint32_t local_capabilities;
     int32_t setup_retries; /* How many times to send setup packets */
     int32_t setup_retry_interval; /* Initial timeout for setup packets */
-    int32_t wait_timeout; /* How long to wait if setup unsuccessful */
+    int32_t wait_timeout_mean; /* How long to wait if setup unsuccessful */
     int32_t mobile_peer_expiry; /* How long to remember 2ary addresses */
     int32_t key_lifetime; /* How long a key lasts once set up */
     int32_t key_renegotiate_time; /* If we see traffic (or a keepalive)
                                      after this time, initiate a new
                                      key exchange */
 
-    bool_t setup_priority; /* Do we have precedence if both sites emit
-                             message 1 simultaneously? */
+    bool_t our_name_later; /* our name > peer name */
     uint32_t log_events;
 
 /* runtime information */
     uint32_t state;
     uint64_t now; /* Most recently seen time */
     bool_t allow_send_prod;
+    bool_t msg1_crossed_logged;
     int resolving_count;
     int resolving_n_results_all;
     int resolving_n_results_stored;
     struct comm_addr resolving_results[MAX_PEER_ADDRS];
+    const char *peerkeys_path;
+    struct peer_keyset *peerkeys_current, *peerkeys_kex;
 
     /* The currently established session */
     struct data_key current;
@@ -367,19 +397,28 @@ static uint32_t event_log_priority(struct site *st, uint32_t event)
     case LOG_DUMP:          return M_DEBUG;
     case LOG_ERROR:         return M_ERR;
     case LOG_PEER_ADDRS:    return M_DEBUG;
+    case LOG_SIGKEYS:       return M_INFO;
     default:                return M_ERR;
     }
 }
 
+static uint32_t slog_start(struct site *st, uint32_t event)
+{
+    uint32_t class=event_log_priority(st, event);
+    if (class) {
+       slilog_part(st->log,class,"%s: ",st->tunname);
+    }
+    return class;
+}
+
 static void vslog(struct site *st, uint32_t event, cstring_t msg, va_list ap)
 FORMAT(printf,3,0);
 static void vslog(struct site *st, uint32_t event, cstring_t msg, va_list ap)
 {
     uint32_t class;
 
-    class=event_log_priority(st, event);
+    class=slog_start(st,event);
     if (class) {
-       slilog_part(st->log,class,"%s: ",st->tunname);
        vslilog_part(st->log,class,msg,ap);
        slilog_part(st->log,class,"\n");
     }
@@ -434,7 +473,9 @@ static bool_t initiate_key_setup(struct site *st, cstring_t reason,
 static void enter_state_run(struct site *st);
 static bool_t enter_state_resolve(struct site *st);
 static void decrement_resolving_count(struct site *st, int by);
-static bool_t enter_new_state(struct site *st,uint32_t next);
+static bool_t enter_new_state(struct site *st,uint32_t next,
+                             const struct msg *prompt
+                             /* may be 0 for SENTMSG1 */);
 static void enter_state_wait(struct site *st);
 static void activate_new_key(struct site *st);
 
@@ -449,14 +490,15 @@ static bool_t current_valid(struct site *st)
 }
 
 #define DEFINE_CALL_TRANSFORM(fwdrev)                                  \
-static int call_transform_##fwdrev(struct site *st,                    \
+static transform_apply_return                                           \
+call_transform_##fwdrev(struct site *st,                               \
                                   struct transform_inst_if *transform, \
                                   struct buffer_if *buf,               \
                                   const char **errmsg)                 \
 {                                                                      \
     if (!is_transform_valid(transform)) {                              \
        *errmsg="transform not set up";                                 \
-       return 1;                                                       \
+       return transform_apply_err;                                     \
     }                                                                  \
     return transform->fwdrev(transform->st,buf,errmsg);                        \
 }
@@ -481,12 +523,19 @@ static void dispose_transform(struct transform_inst_if **transform_var)
     type=buf_unprepend_uint32((b)); \
     if (type!=(t)) return False; } while(0)
 
+static _Bool type_is_msg23(uint32_t type)
+{
+    switch (type) {
+       case LABEL_MSG2: case CASES_MSG3_KNOWN: return True;
+       default: return False;
+    }
+}
 static _Bool type_is_msg34(uint32_t type)
 {
-    return
-       type == LABEL_MSG3 ||
-       type == LABEL_MSG3BIS ||
-       type == LABEL_MSG4;
+    switch (type) {
+       case CASES_MSG3_KNOWN: case LABEL_MSG4: return True;
+       default: return False;
+    }
 }
 
 struct parsedname {
@@ -509,12 +558,28 @@ struct msg {
     int32_t pklen;
     char *pk;
     int32_t hashlen;
-    int32_t siglen;
-    char *sig;
+    struct alg_msg_data sig;
+    int n_pubkeys_accepted_nom; /* may be > MAX_SIG_KEYS ! */
+    const struct sigkeyid *pubkeys_accepted[MAX_SIG_KEYS];
+    int signing_key_index;
 };
 
-static void set_new_transform(struct site *st, char *pk)
+static const struct sigkeyid keyid_zero;
+
+static int32_t wait_timeout(struct site *st) {
+    int32_t t = st->wait_timeout_mean;
+    int8_t factor;
+    if (t < INT_MAX/2) {
+       st->random->generate(st->random->st,sizeof(factor),&factor);
+       t += (t / 256) * factor;
+    }
+    return t;
+}
+
+static _Bool set_new_transform(struct site *st, char *pk)
 {
+    _Bool ok;
+
     /* Make room for the shared key */
     st->sharedsecretlen=st->chosen_transform->keylen?:st->dh->ceil_len;
     assert(st->sharedsecretlen);
@@ -532,15 +597,18 @@ static void set_new_transform(struct site *st, char *pk)
     /* Set up the transform */
     struct transform_if *generator=st->chosen_transform;
     struct transform_inst_if *generated=generator->create(generator->st);
-    generated->setkey(generated->st,st->sharedsecret,
-                     st->sharedsecretlen,st->setup_priority);
+    ok = generated->setkey(generated->st,st->sharedsecret,
+                          st->sharedsecretlen,st->our_name_later);
+
     dispose_transform(&st->new_transform);
+    if (!ok) return False;
     st->new_transform=generated;
 
     slog(st,LOG_SETUP_INIT,"key exchange negotiated transform"
         " %d (capabilities ours=%#"PRIx32" theirs=%#"PRIx32")",
-        st->chosen_transform->capab_transformnum,
+        st->chosen_transform->capab_bit,
         st->local_capabilities, st->remote_capabilities);
+    return True;
 }
 
 struct xinfoadd {
@@ -573,11 +641,13 @@ static void append_string_xinfo_done(struct buffer_if *buf,
 
 /* Build any of msg1 to msg4. msg5 and msg6 are built from the inside
    out using a transform of config data supplied by netlink */
-static bool_t generate_msg(struct site *st, uint32_t type, cstring_t what)
+static bool_t generate_msg(struct site *st, uint32_t type, cstring_t what,
+                          const struct msg *prompt
+                          /* may be 0 for MSG1 */)
 {
-    void *hst;
-    uint8_t *hash;
-    string_t dhpub, sig;
+    string_t dhpub;
+    unsigned minor;
+    int ki;
 
     st->retries=st->setup_retries;
     BUF_ALLOC(&st->buffer,what);
@@ -589,12 +659,57 @@ static bool_t generate_msg(struct site *st, uint32_t type, cstring_t what)
 
     struct xinfoadd xia;
     append_string_xinfo_start(&st->buffer,&xia,st->localname);
-    if ((st->local_capabilities & CAPAB_EARLY) || (type != LABEL_MSG1)) {
+    if ((st->local_capabilities & st->early_capabilities) ||
+       (type != LABEL_MSG1)) {
        buf_append_uint32(&st->buffer,st->local_capabilities);
     }
     if (type_is_msg34(type)) {
        buf_append_uint16(&st->buffer,st->mtu_target);
     }
+    if (type_is_msg23(type)) {
+       buf_append_uint8(&st->buffer,st->peerkeys_kex->nkeys);
+       for (ki=0; ki<st->peerkeys_kex->nkeys; ki++) {
+           struct peer_pubkey *pk = &st->peerkeys_kex->keys[ki];
+           BUF_ADD_OBJ(append,&st->buffer,pk->id);
+       }
+    }
+    struct sigprivkey_if *privkey=0;
+    if (type_is_msg34(type)) {
+       assert(prompt->n_pubkeys_accepted_nom>0);
+       for (ki=0;
+            ki<prompt->n_pubkeys_accepted_nom && ki<MAX_SIG_KEYS;
+            ki++) {
+           const struct sigkeyid *kid=prompt->pubkeys_accepted[ki];
+           if (st->privkeys) {
+               privkey=st->privkeys->lookup(st->privkeys->st,kid,st->log);
+               if (privkey) goto privkey_found;
+           } else {
+               if (sigkeyid_equal(&keyid_zero,kid)) {
+                   privkey=st->privkey_fixed;
+                   goto privkey_found;
+               }
+           }
+       }
+       uint32_t class = slog_start(st,LOG_ERROR);
+       if (class) {
+           slilog_part(st->log,class,"no suitable private key, peer wanted");
+           for (ki=0;
+                ki<prompt->n_pubkeys_accepted_nom && ki<MAX_SIG_KEYS;
+                ki++) {
+               slilog_part(st->log,class, " " SIGKEYID_PR_FMT,
+                           SIGKEYID_PR_VAL(prompt->pubkeys_accepted[ki]));
+           }
+           if (prompt->n_pubkeys_accepted_nom > MAX_SIG_KEYS)
+               slilog_part(st->log,class," +%d",
+                           prompt->n_pubkeys_accepted_nom - MAX_SIG_KEYS);
+           slilog_part(st->log,class,"\n");
+       }
+       return False;
+
+    privkey_found:
+       buf_append_uint8(&st->buffer,ki);
+    }
+
     append_string_xinfo_done(&st->buffer,&xia);
 
     buf_append_string(&st->buffer,st->remotename);
@@ -605,21 +720,25 @@ static bool_t generate_msg(struct site *st, uint32_t type, cstring_t what)
 
     if (hacky_par_mid_failnow()) return False;
 
-    if (type==LABEL_MSG3BIS)
-       buf_append_uint8(&st->buffer,st->chosen_transform->capab_transformnum);
+    if (MSGMAJOR(type) == 3) do {
+       minor = MSGMINOR(type);
+       if (minor < 1) break;
+       buf_append_uint8(&st->buffer,st->chosen_transform->capab_bit);
+    } while (0);
 
     dhpub=st->dh->makepublic(st->dh->st,st->dhsecret,st->dh->len);
     buf_append_string(&st->buffer,dhpub);
     free(dhpub);
-    hash=safe_malloc(st->hash->len, "generate_msg");
-    hst=st->hash->init();
-    st->hash->update(hst,st->buffer.start,st->buffer.size);
-    st->hash->final(hst,hash);
-    sig=st->privkey->sign(st->privkey->st,hash,st->hash->len);
-    buf_append_string(&st->buffer,sig);
-    free(sig);
-    free(hash);
+
+    bool_t ok=privkey->sign(privkey->st,
+                           st->buffer.start,
+                           st->buffer.size,
+                           &st->buffer);
+    if (!ok) goto fail;
     return True;
+
+ fail:
+    return False;
 }
 
 static bool_t unpick_name(struct buffer_if *msg, struct parsedname *nm)
@@ -641,7 +760,11 @@ static bool_t unpick_name(struct buffer_if *msg, struct parsedname *nm)
 static bool_t unpick_msg(struct site *st, uint32_t type,
                         struct buffer_if *msg, struct msg *m)
 {
+    unsigned minor;
+
+    m->n_pubkeys_accepted_nom=-1;
     m->capab_transformnum=-1;
+    m->signing_key_index=-1;
     m->hashstart=msg->start;
     CHECK_AVAIL(msg,4);
     m->dest=buf_unprepend_uint32(msg);
@@ -659,6 +782,23 @@ static bool_t unpick_msg(struct site *st, uint32_t type,
        CHECK_AVAIL(&m->remote.extrainfo,2);
        m->remote_mtu=buf_unprepend_uint16(&m->remote.extrainfo);
     }
+    if (type_is_msg23(type) && m->remote.extrainfo.size) {
+       m->n_pubkeys_accepted_nom = buf_unprepend_uint8(&m->remote.extrainfo);
+       if (!m->n_pubkeys_accepted_nom) return False;
+       for (int ki_nom=0; ki_nom<m->n_pubkeys_accepted_nom; ki_nom++) {
+           CHECK_AVAIL(&m->remote.extrainfo,KEYIDSZ);
+           struct sigkeyid *kid = buf_unprepend(&m->remote.extrainfo,KEYIDSZ);
+           if (ki_nom<MAX_SIG_KEYS) m->pubkeys_accepted[ki_nom] = kid;
+       }
+    } else {
+       m->n_pubkeys_accepted_nom = 1;
+       m->pubkeys_accepted[0] = &keyid_zero;
+    }
+    if (type_is_msg34(type) && m->remote.extrainfo.size) {
+       m->signing_key_index=buf_unprepend_uint8(&m->remote.extrainfo);
+    } else {
+       m->signing_key_index=0;
+    }
     if (!unpick_name(msg,&m->local)) return False;
     if (type==LABEL_PROD) {
        CHECK_EMPTY(msg);
@@ -676,22 +816,37 @@ static bool_t unpick_msg(struct site *st, uint32_t type,
        CHECK_EMPTY(msg);
        return True;
     }
-    if (type==LABEL_MSG3BIS) {
-       CHECK_AVAIL(msg,1);
-       m->capab_transformnum = buf_unprepend_uint8(msg);
-    } else {
-       m->capab_transformnum = CAPAB_TRANSFORMNUM_ANCIENT;
-    }
+    if (MSGMAJOR(type) == 3) do {
+       minor = MSGMINOR(type);
+#define MAYBE_READ_CAP(minminor, kind, dflt) do {                      \
+    if (minor < (minminor))                                            \
+       m->capab_##kind##num = (dflt);                                  \
+    else {                                                             \
+       CHECK_AVAIL(msg, 1);                                            \
+       m->capab_##kind##num = buf_unprepend_uint8(msg);                \
+    }                                                                  \
+} while (0)
+       MAYBE_READ_CAP(1, transform, CAPAB_BIT_ANCIENTTRANSFORM);
+#undef MAYBE_READ_CAP
+    } while (0);
     CHECK_AVAIL(msg,2);
     m->pklen=buf_unprepend_uint16(msg);
     CHECK_AVAIL(msg,m->pklen);
     m->pk=buf_unprepend(msg,m->pklen);
     m->hashlen=msg->start-m->hashstart;
-    CHECK_AVAIL(msg,2);
-    m->siglen=buf_unprepend_uint16(msg);
-    CHECK_AVAIL(msg,m->siglen);
-    m->sig=buf_unprepend(msg,m->siglen);
+
+    if (m->signing_key_index < 0 ||
+       m->signing_key_index >= st->peerkeys_kex->nkeys) {
+       return False;
+    }
+    struct sigpubkey_if *pubkey=
+       st->peerkeys_kex->keys[m->signing_key_index].pubkey;
+    if (!pubkey->unpick(pubkey->st,msg,&m->sig)) {
+       return False;
+    }
+
     CHECK_EMPTY(msg);
+
     return True;
 }
 
@@ -723,13 +878,13 @@ static bool_t check_msg(struct site *st, uint32_t type, struct msg *m,
        return False;
     }
     if (type==LABEL_MSG2) return True;
-    if (!consttime_memeq(m->nR,st->remoteN,NONCELEN)!=0) {
+    if (!consttime_memeq(m->nR,st->remoteN,NONCELEN)) {
        *error="wrong remotely-generated nonce";
        return False;
     }
     /* MSG3 has complicated rules about capabilities, which are
      * handled in process_msg3. */
-    if (type==LABEL_MSG3 || type==LABEL_MSG3BIS) return True;
+    if (MSGMAJOR(type) == 3) return True;
     if (m->remote_capabilities!=st->remote_capabilities) {
        *error="remote capabilities changed";
        return False;
@@ -739,14 +894,27 @@ static bool_t check_msg(struct site *st, uint32_t type, struct msg *m,
     return False;
 }
 
-static bool_t generate_msg1(struct site *st)
+static bool_t kex_init(struct site *st)
 {
+    keyset_dispose(&st->peerkeys_kex);
+    if (!st->peerkeys_current) {
+       slog(st,LOG_SETUP_INIT,"no peer public keys, abandoning key setup");
+       return False;
+    }
+    st->peerkeys_kex = keyset_dup(st->peerkeys_current);
     st->random->generate(st->random->st,NONCELEN,st->localN);
-    return generate_msg(st,LABEL_MSG1,"site:MSG1");
+    return True;
+}
+
+static bool_t generate_msg1(struct site *st, const struct msg *prompt_maybe_0)
+{
+    return
+       generate_msg(st,LABEL_MSG1,"site:MSG1",prompt_maybe_0);
 }
 
 static bool_t process_msg1(struct site *st, struct buffer_if *msg1,
-                          const struct comm_addr *src, struct msg *m)
+                          const struct comm_addr *src,
+                          const struct msg *m)
 {
     /* We've already determined we're in an appropriate state to
        process an incoming MSG1, and that the MSG1 has correct values
@@ -758,81 +926,99 @@ static bool_t process_msg1(struct site *st, struct buffer_if *msg1,
     return True;
 }
 
-static bool_t generate_msg2(struct site *st)
+static bool_t generate_msg2(struct site *st,
+                           const struct msg *prompt_may_be_null)
 {
-    st->random->generate(st->random->st,NONCELEN,st->localN);
-    return generate_msg(st,LABEL_MSG2,"site:MSG2");
+    return
+       generate_msg(st,LABEL_MSG2,"site:MSG2",prompt_may_be_null);
 }
 
 static bool_t process_msg2(struct site *st, struct buffer_if *msg2,
-                          const struct comm_addr *src)
+                          const struct comm_addr *src,
+                          struct msg *m /* returned */)
 {
-    struct msg m;
     cstring_t err;
 
-    if (!unpick_msg(st,LABEL_MSG2,msg2,&m)) return False;
-    if (!check_msg(st,LABEL_MSG2,&m,&err)) {
+    if (!unpick_msg(st,LABEL_MSG2,msg2,m)) return False;
+    if (!check_msg(st,LABEL_MSG2,m,&err)) {
        slog(st,LOG_SEC,"msg2: %s",err);
        return False;
     }
-    st->setup_session_id=m.source;
-    st->remote_capabilities=m.remote_capabilities;
+    st->setup_session_id=m->source;
+    st->remote_capabilities=m->remote_capabilities;
 
     /* Select the transform to use */
 
-    uint32_t remote_transforms = st->remote_capabilities & CAPAB_TRANSFORM_MASK;
-    if (!remote_transforms)
+    uint32_t remote_crypto_caps = st->remote_capabilities & CAPAB_TRANSFORM_MASK;
+    if (!remote_crypto_caps)
        /* old secnets only had this one transform */
-       remote_transforms = 1UL << CAPAB_TRANSFORMNUM_ANCIENT;
+       remote_crypto_caps = 1UL << CAPAB_BIT_ANCIENTTRANSFORM;
+
+#define CHOOSE_CRYPTO(kind, whats) do {                                        \
+    struct kind##_if *iface;                                           \
+    uint32_t bit, ours = 0;                                            \
+    int i;                                                             \
+    for (i= 0; i < st->n##kind##s; i++) {                              \
+       iface=st->kind##s[i];                                           \
+       bit = 1UL << iface->capab_bit;                                  \
+       if (bit & remote_crypto_caps) goto kind##_found;                \
+       ours |= bit;                                                    \
+    }                                                                  \
+    slog(st,LOG_ERROR,"no " whats " in common"                         \
+        " (us %#"PRIx32"; them: %#"PRIx32")",                          \
+        st->local_capabilities & ours, remote_crypto_caps);            \
+    return False;                                                      \
+kind##_found:                                                          \
+    st->chosen_##kind = iface;                                         \
+} while (0)
 
-    struct transform_if *ti;
-    int i;
-    for (i=0; i<st->ntransforms; i++) {
-       ti=st->transforms[i];
-       if ((1UL << ti->capab_transformnum) & remote_transforms)
-           goto transform_found;
-    }
-    slog(st,LOG_ERROR,"no transforms in common"
-        " (us %#"PRIx32"; them: %#"PRIx32")",
-        st->local_capabilities & CAPAB_TRANSFORM_MASK,
-        remote_transforms);
-    return False;
- transform_found:
-    st->chosen_transform=ti;
+    CHOOSE_CRYPTO(transform, "transforms");
 
-    memcpy(st->remoteN,m.nR,NONCELEN);
+#undef CHOOSE_CRYPTO
+
+    memcpy(st->remoteN,m->nR,NONCELEN);
     return True;
 }
 
-static bool_t generate_msg3(struct site *st)
+static bool_t generate_msg3(struct site *st, const struct msg *prompt)
 {
     /* Now we have our nonce and their nonce. Think of a secret key,
        and create message number 3. */
     st->random->generate(st->random->st,st->dh->len,st->dhsecret);
     return generate_msg(st,
-                       (st->remote_capabilities & CAPAB_TRANSFORM_MASK
-                        ? LABEL_MSG3BIS : LABEL_MSG3),
-                       "site:MSG3");
+                       (st->remote_capabilities & CAPAB_TRANSFORM_MASK)
+                       ? LABEL_MSG3BIS
+                       : LABEL_MSG3,
+                       "site:MSG3",prompt);
 }
 
 static bool_t process_msg3_msg4(struct site *st, struct msg *m)
 {
-    uint8_t *hash;
-    void *hst;
-
     /* Check signature and store g^x mod m */
-    hash=safe_malloc(st->hash->len, "process_msg3_msg4");
-    hst=st->hash->init();
-    st->hash->update(hst,m->hashstart,m->hashlen);
-    st->hash->final(hst,hash);
-    /* Terminate signature with a '0' - cheating, but should be ok */
-    m->sig[m->siglen]=0;
-    if (!st->pubkey->check(st->pubkey->st,hash,st->hash->len,m->sig)) {
+    int ki;
+
+    if (m->signing_key_index >= 0) {
+       if (m->signing_key_index >= st->peerkeys_kex->nkeys)
+           return False;
+       ki=m->signing_key_index;
+    } else {
+       for (ki=0; ki<st->peerkeys_kex->nkeys; ki++)
+           if (sigkeyid_equal(&keyid_zero,&st->peerkeys_kex->keys[ki].id))
+               goto found;
+       /* not found */
+       slog(st,LOG_ERROR,
+            "peer signed with keyid zero, which we do not accept");
+       return False;
+    found:;
+    }
+    struct sigpubkey_if *pubkey=st->peerkeys_kex->keys[ki].pubkey;
+
+    if (!pubkey->check(pubkey->st,
+                      m->hashstart,m->hashlen,
+                      &m->sig)) {
        slog(st,LOG_SEC,"msg3/msg4 signature failed check!");
-       free(hash);
        return False;
     }
-    free(hash);
 
     st->remote_adv_mtu=m->remote_mtu;
 
@@ -840,82 +1026,100 @@ static bool_t process_msg3_msg4(struct site *st, struct msg *m)
 }
 
 static bool_t process_msg3(struct site *st, struct buffer_if *msg3,
-                          const struct comm_addr *src, uint32_t msgtype)
+                          const struct comm_addr *src, uint32_t msgtype,
+                          struct msg *m /* returned */)
 {
-    struct msg m;
     cstring_t err;
 
-    assert(msgtype==LABEL_MSG3 || msgtype==LABEL_MSG3BIS);
+    switch (msgtype) {
+       case CASES_MSG3_KNOWN: break;
+       default: assert(0);
+    }
 
-    if (!unpick_msg(st,msgtype,msg3,&m)) return False;
-    if (!check_msg(st,msgtype,&m,&err)) {
+    if (!unpick_msg(st,msgtype,msg3,m)) return False;
+    if (!check_msg(st,msgtype,m,&err)) {
        slog(st,LOG_SEC,"msg3: %s",err);
        return False;
     }
-    uint32_t capab_adv_late = m.remote_capabilities
-       & ~st->remote_capabilities & CAPAB_EARLY;
+    uint32_t capab_adv_late = m->remote_capabilities
+       & ~st->remote_capabilities & st->early_capabilities;
     if (capab_adv_late) {
        slog(st,LOG_SEC,"msg3 impermissibly adds early capability flag(s)"
             " %#"PRIx32" (was %#"PRIx32", now %#"PRIx32")",
-            capab_adv_late, st->remote_capabilities, m.remote_capabilities);
+            capab_adv_late, st->remote_capabilities, m->remote_capabilities);
        return False;
     }
-    st->remote_capabilities|=m.remote_capabilities;
 
-    struct transform_if *ti;
-    int i;
-    for (i=0; i<st->ntransforms; i++) {
-       ti=st->transforms[i];
-       if (ti->capab_transformnum == m.capab_transformnum)
-           goto transform_found;
-    }
-    slog(st,LOG_SEC,"peer chose unknown-to-us transform %d!",
-        m.capab_transformnum);
-    return False;
- transform_found:
-    st->chosen_transform=ti;
+#define CHOSE_CRYPTO(kind, what) do {                                  \
+    struct kind##_if *iface;                                           \
+    int i;                                                             \
+    for (i=0; i<st->n##kind##s; i++) {                                 \
+       iface=st->kind##s[i];                                           \
+       if (iface->capab_bit == m->capab_##kind##num)                   \
+           goto kind##_found;                                          \
+    }                                                                  \
+    slog(st,LOG_SEC,"peer chose unknown-to-us " what " %d!",           \
+        m->capab_##kind##num);                                                 \
+    return False;                                                      \
+kind##_found:                                                          \
+    st->chosen_##kind=iface;                                           \
+} while (0)
 
-    if (!process_msg3_msg4(st,&m))
+    CHOSE_CRYPTO(transform, "transform");
+
+#undef CHOSE_CRYPTO
+
+    if (!process_msg3_msg4(st,m))
        return False;
 
+    /* Update our idea of the remote site's capabilities, now that we've
+     * verified that its message was authentic.
+     *
+     * Our previous idea of the remote site's capabilities came from the
+     * unauthenticated MSG1.  We've already checked that this new message
+     * doesn't change any of the bits we relied upon in the past, but it may
+     * also have set additional capability bits.  We simply throw those away
+     * now, and use the authentic capabilities from this MSG3. */
+    st->remote_capabilities=m->remote_capabilities;
+
     /* Terminate their DH public key with a '0' */
-    m.pk[m.pklen]=0;
+    m->pk[m->pklen]=0;
     /* Invent our DH secret key */
     st->random->generate(st->random->st,st->dh->len,st->dhsecret);
 
     /* Generate the shared key and set up the transform */
-    set_new_transform(st,m.pk);
+    if (!set_new_transform(st,m->pk)) return False;
 
     return True;
 }
 
-static bool_t generate_msg4(struct site *st)
+static bool_t generate_msg4(struct site *st, const struct msg *prompt)
 {
     /* We have both nonces, their public key and our private key. Generate
        our public key, sign it and send it to them. */
-    return generate_msg(st,LABEL_MSG4,"site:MSG4");
+    return generate_msg(st,LABEL_MSG4,"site:MSG4",prompt);
 }
 
 static bool_t process_msg4(struct site *st, struct buffer_if *msg4,
-                          const struct comm_addr *src)
+                          const struct comm_addr *src,
+                          struct msg *m /* returned */)
 {
-    struct msg m;
     cstring_t err;
 
-    if (!unpick_msg(st,LABEL_MSG4,msg4,&m)) return False;
-    if (!check_msg(st,LABEL_MSG4,&m,&err)) {
+    if (!unpick_msg(st,LABEL_MSG4,msg4,m)) return False;
+    if (!check_msg(st,LABEL_MSG4,m,&err)) {
        slog(st,LOG_SEC,"msg4: %s",err);
        return False;
     }
     
-    if (!process_msg3_msg4(st,&m))
+    if (!process_msg3_msg4(st,m))
        return False;
 
     /* Terminate their DH public key with a '0' */
-    m.pk[m.pklen]=0;
+    m->pk[m->pklen]=0;
 
     /* Generate the shared key and set up the transform */
-    set_new_transform(st,m.pk);
+    if (!set_new_transform(st,m->pk)) return False;
 
     return True;
 }
@@ -939,7 +1143,7 @@ static bool_t unpick_msg0(struct site *st, struct buffer_if *msg0,
     /* Leaves transformed part of buffer untouched */
 }
 
-static bool_t generate_msg5(struct site *st)
+static bool_t generate_msg5(struct site *st, const struct msg *prompt)
 {
     cstring_t transform_err;
 
@@ -996,15 +1200,16 @@ static void create_msg6(struct site *st, struct transform_inst_if *transform,
     /* Give the netlink code an opportunity to put its own stuff in the
        message (configuration information, etc.) */
     buf_prepend_uint32(&st->buffer,LABEL_MSG6);
-    int problem = call_transform_forwards(st,transform,
-                                         &st->buffer,&transform_err);
+    transform_apply_return problem =
+       call_transform_forwards(st,transform,
+                               &st->buffer,&transform_err);
     assert(!problem);
     buf_prepend_uint32(&st->buffer,LABEL_MSG6);
     buf_prepend_uint32(&st->buffer,st->index);
     buf_prepend_uint32(&st->buffer,session_id);
 }
 
-static bool_t generate_msg6(struct site *st)
+static bool_t generate_msg6(struct site *st, const struct msg *prompt)
 {
     if (!is_transform_valid(st->new_transform))
        return False;
@@ -1037,12 +1242,13 @@ static bool_t process_msg6(struct site *st, struct buffer_if *msg6,
     return True;
 }
 
-static bool_t decrypt_msg0(struct site *st, struct buffer_if *msg0,
+static transform_apply_return
+decrypt_msg0(struct site *st, struct buffer_if *msg0,
                           const struct comm_addr *src)
 {
     cstring_t transform_err, auxkey_err, newkey_err="n/a";
     struct msg0 m;
-    uint32_t problem;
+    transform_apply_return problem;
 
     if (!unpick_msg0(st,msg0,&m)) return False;
 
@@ -1055,15 +1261,15 @@ static bool_t decrypt_msg0(struct site *st, struct buffer_if *msg0,
        if (!st->auxiliary_is_new)
            delete_one_key(st,&st->auxiliary_key,
                           "peer has used new key","auxiliary key",LOG_SEC);
-       return True;
+       return 0;
     }
-    if (problem==2)
-       goto skew;
+    if (transform_apply_return_badseq(problem))
+       goto badseq;
 
     buffer_copy(msg0, &st->scratch);
     problem = call_transform_reverse(st,st->auxiliary_key.transform,
                                     msg0,&auxkey_err);
-    if (problem==0) {
+    if (!problem) {
        slog(st,LOG_DROP,"processing packet which uses auxiliary key");
        if (st->auxiliary_is_new) {
            /* We previously timed out in state SENTMSG5 but it turns
@@ -1080,10 +1286,10 @@ static bool_t decrypt_msg0(struct site *st, struct buffer_if *msg0,
            st->auxiliary_is_new=0;
            st->renegotiate_key_time=st->auxiliary_renegotiate_key_time;
        }
-       return True;
+       return 0;
     }
-    if (problem==2)
-       goto skew;
+    if (transform_apply_return_badseq(problem))
+       goto badseq;
 
     if (st->state==SITE_SENTMSG5) {
        buffer_copy(msg0, &st->scratch);
@@ -1096,29 +1302,41 @@ static bool_t decrypt_msg0(struct site *st, struct buffer_if *msg0,
            BUF_FREE(&st->buffer);
            st->timeout=0;
            activate_new_key(st);
-           return True; /* do process the data in this packet */
+           return 0; /* do process the data in this packet */
        }
-       if (problem==2)
-           goto skew;
+       if (transform_apply_return_badseq(problem))
+           goto badseq;
     }
 
     slog(st,LOG_SEC,"transform: %s (aux: %s, new: %s)",
         transform_err,auxkey_err,newkey_err);
     initiate_key_setup(st,"incoming message would not decrypt",0);
     send_nak(src,m.dest,m.source,m.type,msg0,"message would not decrypt");
-    return False;
+    assert(problem);
+    return problem;
 
- skew:
-    slog(st,LOG_DROP,"transform: %s (merely skew)",transform_err);
-    return False;
+ badseq:
+    slog(st,LOG_DROP,"transform: %s (bad seq.)",transform_err);
+    assert(problem);
+    return problem;
 }
 
 static bool_t process_msg0(struct site *st, struct buffer_if *msg0,
                           const struct comm_addr *src)
 {
     uint32_t type;
-
-    if (!decrypt_msg0(st,msg0,src))
+    transform_apply_return problem;
+
+    problem = decrypt_msg0(st,msg0,src);
+    if (problem==transform_apply_seqdupe) {
+       /* We recently received another copy of this packet, maybe due
+        * to polypath.  That's not a problem; indeed, for the
+        * purposes of transport address management it is a success.
+        * But we don't want to process the packet. */
+       transport_data_msgok(st,src);
+       return False;
+    }
+    if (problem)
        return False;
 
     CHECK_AVAIL(msg0,4);
@@ -1127,6 +1345,10 @@ static bool_t process_msg0(struct site *st, struct buffer_if *msg0,
     case LABEL_MSG7:
        /* We must forget about the current session. */
        delete_keys(st,"request from peer",LOG_SEC);
+       /* probably, the peer is shutting down, and this is going to fail,
+        * but we need to be trying to bring the link up again */
+       if (st->keepalive)
+           initiate_key_setup(st,"peer requested key teardown",0);
        return True;
     case LABEL_MSG9:
        /* Deliver to netlink layer */
@@ -1145,16 +1367,34 @@ static bool_t process_msg0(struct site *st, struct buffer_if *msg0,
 }
 
 static void dump_packet(struct site *st, struct buffer_if *buf,
-                       const struct comm_addr *addr, bool_t incoming)
+                       const struct comm_addr *addr, bool_t incoming,
+                       bool_t ok)
 {
     uint32_t dest=get_uint32(buf->start);
     uint32_t source=get_uint32(buf->start+4);
     uint32_t msgtype=get_uint32(buf->start+8);
 
     if (st->log_events & LOG_DUMP)
-       slilog(st->log,M_DEBUG,"%s: %s: %08x<-%08x: %08x:",
+       slilog(st->log,M_DEBUG,"%s: %s: %08x<-%08x: %08x: %s%s",
               st->tunname,incoming?"incoming":"outgoing",
-              dest,source,msgtype);
+              dest,source,msgtype,comm_addr_to_string(addr),
+              ok?"":" - fail");
+}
+
+static bool_t comm_addr_sendmsg(struct site *st,
+                               const struct comm_addr *dest,
+                               struct buffer_if *buf)
+{
+    int i;
+    struct comm_clientinfo *commclientinfo = 0;
+
+    for (i=0; i < st->ncomms; i++) {
+       if (st->comms[i] == dest->comm) {
+           commclientinfo = st->commclientinfos[i];
+           break;
+       }
+    }
+    return dest->comm->sendmsg(dest->comm->st, buf, dest, commclientinfo);
 }
 
 static uint32_t site_status(void *st)
@@ -1240,13 +1480,13 @@ static void decrement_resolving_count(struct site *st, int by)
                 st->resolving_n_results_all, naddrs);
        }
        slog(st,LOG_STATE,"resolution completed, %d addrs, eg: %s",
-            naddrs, comm_addr_to_string(&addrs[0]));;
+            naddrs, iaddr_to_string(&addrs[0].ia));;
     }
 
     switch (st->state) {
     case SITE_RESOLVE:
         if (transport_compute_setupinit_peers(st,addrs,naddrs,0)) {
-           enter_new_state(st,SITE_SENTMSG1);
+           enter_new_state(st,SITE_SENTMSG1,0);
        } else {
            /* Can't figure out who to try to to talk to */
            slog(st,LOG_SETUP_INIT,
@@ -1283,7 +1523,7 @@ static void decrement_resolving_count(struct site *st, int by)
        } else if (st->local_mobile) {
            /* Not very good.  We should queue (another) renegotiation
             * so that we can update the peer address. */
-           st->key_renegotiate_time=st->now+st->wait_timeout;
+           st->key_renegotiate_time=st->now+wait_timeout(st);
        } else {
            slog(st,LOG_SETUP_INIT,"resolution failed: "
                 " continuing to use source address of peer's packets");
@@ -1306,7 +1546,7 @@ static bool_t initiate_key_setup(struct site *st, cstring_t reason,
        slog(st,LOG_SETUP_INIT,"resolving peer address(es)");
        return enter_state_resolve(st);
     } else if (transport_compute_setupinit_peers(st,0,0,prod_hint)) {
-       return enter_new_state(st,SITE_SENTMSG1);
+       return enter_new_state(st,SITE_SENTMSG1,0);
     }
     slog(st,LOG_SETUP_INIT,"key exchange failed: no address for peer");
     return False;
@@ -1396,18 +1636,23 @@ static void set_link_quality(struct site *st)
 
 static void enter_state_run(struct site *st)
 {
-    slog(st,LOG_STATE,"entering state RUN");
+    slog(st,LOG_STATE,"entering state RUN%s",
+        current_valid(st) ? " (keyed)" : " (unkeyed)");
     st->state=SITE_RUN;
     st->timeout=0;
 
     st->setup_session_id=0;
     transport_peers_clear(st,&st->setup_peers);
+    keyset_dispose(&st->peerkeys_kex);
     FILLZERO(st->localN);
     FILLZERO(st->remoteN);
     dispose_transform(&st->new_transform);
     memset(st->dhsecret,0,st->dh->len);
-    memset(st->sharedsecret,0,st->sharedsecretlen);
+    if (st->sharedsecret) memset(st->sharedsecret,0,st->sharedsecretlen);
     set_link_quality(st);
+
+    if (st->keepalive && !current_valid(st))
+       initiate_key_setup(st, "keepalive", 0);
 }
 
 static bool_t ensure_resolving(struct site *st)
@@ -1456,20 +1701,25 @@ static bool_t enter_state_resolve(struct site *st)
     return ensure_resolving(st);
 }
 
-static bool_t enter_new_state(struct site *st, uint32_t next)
+static bool_t enter_new_state(struct site *st, uint32_t next,
+                             const struct msg *prompt
+                             /* may be 0 for SENTMSG1 */)
 {
-    bool_t (*gen)(struct site *st);
+    bool_t (*gen)(struct site *st, const struct msg *prompt);
     int r;
 
     slog(st,LOG_STATE,"entering state %s",state_name(next));
     switch(next) {
     case SITE_SENTMSG1:
        state_assert(st,st->state==SITE_RUN || st->state==SITE_RESOLVE);
+       if (!kex_init(st)) return False;
        gen=generate_msg1;
+       st->msg1_crossed_logged = False;
        break;
     case SITE_SENTMSG2:
        state_assert(st,st->state==SITE_RUN || st->state==SITE_RESOLVE ||
                     st->state==SITE_SENTMSG1 || st->state==SITE_WAIT);
+       if (!kex_init(st)) return False;
        gen=generate_msg2;
        break;
     case SITE_SENTMSG3:
@@ -1500,7 +1750,7 @@ static bool_t enter_new_state(struct site *st, uint32_t next)
 
     if (hacky_par_start_failnow()) return False;
 
-    r= gen(st) && send_msg(st);
+    r= gen(st,prompt) && send_msg(st);
 
     hacky_par_end(&r,
                  st->setup_retries, st->setup_retry_interval,
@@ -1554,7 +1804,7 @@ static bool_t send_msg7(struct site *st, cstring_t reason)
 static void enter_state_wait(struct site *st)
 {
     slog(st,LOG_STATE,"entering state WAIT");
-    st->timeout=st->now+st->wait_timeout;
+    st->timeout=st->now+wait_timeout(st);
     st->state=SITE_WAIT;
     set_link_quality(st);
     BUF_FREE(&st->buffer); /* will have had an outgoing packet in it */
@@ -1581,8 +1831,8 @@ static void generate_send_prod(struct site *st,
     slog(st,LOG_SETUP_INIT,"prodding peer for key exchange");
     st->allow_send_prod=0;
     generate_prod(st,&st->scratch);
-    dump_packet(st,&st->scratch,source,False);
-    source->comm->sendmsg(source->comm->st, &st->scratch, source);
+    bool_t ok = comm_addr_sendmsg(st, source, &st->scratch);
+    dump_packet(st,&st->scratch,source,False,ok);
 }
 
 static inline void site_settimeout(uint64_t timeout, int *timeout_io)
@@ -1684,15 +1934,60 @@ static void site_outgoing(void *sst, struct buffer_if *buf)
 }
 
 static bool_t named_for_us(struct site *st, const struct buffer_if *buf_in,
-                          uint32_t type, struct msg *m)
+                          uint32_t type, struct msg *m,
+                          struct priomsg *whynot)
     /* For packets which are identified by the local and remote names.
      * If it has our name and our peer's name in it it's for us. */
 {
     struct buffer_if buf[1];
     buffer_readonly_clone(buf,buf_in);
-    return unpick_msg(st,type,buf,m)
-       && name_matches(&m->remote,st->remotename)
-       && name_matches(&m->local,st->localname);
+
+    if (!unpick_msg(st,type,buf,m)) {
+       priomsg_update_fixed(whynot, comm_notify_whynot_unpick, "malformed");
+       return False;
+    }
+#define NAME_MATCHES(lr)                                               \
+    if (!name_matches(&m->lr, st->lr##name)) {                         \
+       if (priomsg_update_fixed(whynot, comm_notify_whynot_name_##lr,  \
+                                 "unknown " #lr " name: ")) {          \
+            truncmsg_add_packet_string(&whynot->m, m->lr.len, m->lr.name); \
+        }                                                              \
+        return False;                                                  \
+    }
+    NAME_MATCHES(remote);
+    NAME_MATCHES(local );
+#undef NAME_MATCHES
+
+    return True;
+}
+
+static bool_t we_have_priority(struct site *st, const struct msg *m) {
+    if (st->local_capabilities & m->remote_capabilities &
+       CAPAB_PRIORITY_MOBILE) {
+       if (st->local_mobile) return True;
+       if (st-> peer_mobile) return False;
+    }
+    return st->our_name_later;
+}
+
+static bool_t setup_late_msg_ok(struct site *st, 
+                               const struct buffer_if *buf_in,
+                               uint32_t msgtype,
+                               const struct comm_addr *source,
+                               struct msg *m /* returned */) {
+    /* For setup packets which seem from their type like they are
+     * late.  Maybe they came via a different path.  All we do is make
+     * a note of the sending address, iff they look like they are part
+     * of the current key setup attempt. */
+    if (!named_for_us(st,buf_in,msgtype,m,0))
+       /* named_for_us calls unpick_msg which gets the nonces */
+       return False;
+    if (!consttime_memeq(m->nR,st->remoteN,NONCELEN) ||
+       !consttime_memeq(m->nL,st->localN, NONCELEN))
+       /* spoof ?  from stale run ?  who knows */
+       return False;
+    transport_setup_msgok(st,source);
+    return True;
 }
 
 /* This function is called by the communication device to deliver
@@ -1701,7 +1996,8 @@ static bool_t named_for_us(struct site *st, const struct buffer_if *buf_in,
    this current site instance (and should therefore not be processed
    by other sites), even if the packet was otherwise ignored. */
 static bool_t site_incoming(void *sst, struct buffer_if *buf,
-                           const struct comm_addr *source)
+                           const struct comm_addr *source,
+                           struct priomsg *whynot)
 {
     struct site *st=sst;
 
@@ -1709,20 +2005,21 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
 
     uint32_t dest=get_uint32(buf->start);
     uint32_t msgtype=get_uint32(buf->start+8);
-    struct msg named_msg;
+    struct msg msg;
+      /* initialised by named_for_us, or process_msgN for N!=1 */
 
     if (msgtype==LABEL_MSG1) {
-       if (!named_for_us(st,buf,msgtype,&named_msg))
+       if (!named_for_us(st,buf,msgtype,&msg,whynot))
            return False;
        /* It's a MSG1 addressed to us. Decide what to do about it. */
-       dump_packet(st,buf,source,True);
+       dump_packet(st,buf,source,True,True);
        if (st->state==SITE_RUN || st->state==SITE_RESOLVE ||
            st->state==SITE_WAIT) {
            /* We should definitely process it */
            transport_compute_setupinit_peers(st,0,0,source);
-           if (process_msg1(st,buf,source,&named_msg)) {
+           if (process_msg1(st,buf,source,&msg)) {
                slog(st,LOG_SETUP_INIT,"key setup initiated by peer");
-               bool_t entered=enter_new_state(st,SITE_SENTMSG2);
+               bool_t entered=enter_new_state(st,SITE_SENTMSG2,&msg);
                if (entered && st->addresses && st->local_mobile)
                    /* We must do this as the very last thing, because
                       the resolver callback might reenter us. */
@@ -1736,18 +2033,19 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
            /* We've just sent a message 1! They may have crossed on
               the wire. If we have priority then we ignore the
               incoming one, otherwise we process it as usual. */
-           if (st->setup_priority) {
+           if (we_have_priority(st,&msg)) {
                BUF_FREE(buf);
-               slog(st,LOG_DUMP,"crossed msg1s; we are higher "
-                    "priority => ignore incoming msg1");
+               if (!st->msg1_crossed_logged++)
+                   slog(st,LOG_SETUP_INIT,"crossed msg1s; we are higher "
+                        "priority => ignore incoming msg1");
                return True;
            } else {
-               slog(st,LOG_DUMP,"crossed msg1s; we are lower "
+               slog(st,LOG_SETUP_INIT,"crossed msg1s; we are lower "
                     "priority => use incoming msg1");
-               if (process_msg1(st,buf,source,&named_msg)) {
+               if (process_msg1(st,buf,source,&msg)) {
                    BUF_FREE(&st->buffer); /* Free our old message 1 */
                    transport_setup_msgok(st,source);
-                   enter_new_state(st,SITE_SENTMSG2);
+                   enter_new_state(st,SITE_SENTMSG2,&msg);
                } else {
                    slog(st,LOG_ERROR,"failed to process an incoming "
                         "crossed msg1 (we have low priority)");
@@ -1755,17 +2053,29 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
                BUF_FREE(buf);
                return True;
            }
+       } else if (st->state==SITE_SENTMSG2 ||
+                  st->state==SITE_SENTMSG4) {
+           if (consttime_memeq(msg.nR,st->remoteN,NONCELEN)) {
+               /* We are ahead in the protocol, but that msg1 had the
+                * peer's nonce so presumably it is from this key
+                * exchange run, via a slower route */
+               transport_setup_msgok(st,source);
+           } else {
+               slog(st,LOG_UNEXPECTED,"competing incoming message 1");
+           }
+           BUF_FREE(buf);
+           return True;
        }
        /* The message 1 was received at an unexpected stage of the
-          key setup. XXX POLICY - what do we do? */
+          key setup.  Well, they lost the race. */
        slog(st,LOG_UNEXPECTED,"unexpected incoming message 1");
        BUF_FREE(buf);
        return True;
     }
     if (msgtype==LABEL_PROD) {
-       if (!named_for_us(st,buf,msgtype,&named_msg))
+       if (!named_for_us(st,buf,msgtype,&msg,whynot))
            return False;
-       dump_packet(st,buf,source,True);
+       dump_packet(st,buf,source,True,True);
        if (st->state!=SITE_RUN) {
            slog(st,LOG_DROP,"ignoring PROD when not in state RUN");
        } else if (current_valid(st)) {
@@ -1778,7 +2088,7 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
     }
     if (dest==st->index) {
        /* Explicitly addressed to us */
-       if (msgtype!=LABEL_MSG0) dump_packet(st,buf,source,True);
+       if (msgtype!=LABEL_MSG0) dump_packet(st,buf,source,True,True);
        switch (msgtype) {
        case LABEL_NAK:
            /* If the source is our current peer then initiate a key setup,
@@ -1802,22 +2112,28 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
        case LABEL_MSG2:
            /* Setup packet: expected only in state SENTMSG1 */
            if (st->state!=SITE_SENTMSG1) {
+               if ((st->state==SITE_SENTMSG3 ||
+                    st->state==SITE_SENTMSG5) &&
+                   setup_late_msg_ok(st,buf,msgtype,source,&msg))
+                   break;
                slog(st,LOG_UNEXPECTED,"unexpected MSG2");
-           } else if (process_msg2(st,buf,source)) {
+           } else if (process_msg2(st,buf,source,&msg)) {
                transport_setup_msgok(st,source);
-               enter_new_state(st,SITE_SENTMSG3);
+               enter_new_state(st,SITE_SENTMSG3,&msg);
            } else {
                slog(st,LOG_SEC,"invalid MSG2");
            }
            break;
-       case LABEL_MSG3:
-       case LABEL_MSG3BIS:
+       case CASES_MSG3_KNOWN:
            /* Setup packet: expected only in state SENTMSG2 */
            if (st->state!=SITE_SENTMSG2) {
+               if ((st->state==SITE_SENTMSG4) &&
+                   setup_late_msg_ok(st,buf,msgtype,source,&msg))
+                   break;
                slog(st,LOG_UNEXPECTED,"unexpected MSG3");
-           } else if (process_msg3(st,buf,source,msgtype)) {
+           } else if (process_msg3(st,buf,source,msgtype,&msg)) {
                transport_setup_msgok(st,source);
-               enter_new_state(st,SITE_SENTMSG4);
+               enter_new_state(st,SITE_SENTMSG4,&msg);
            } else {
                slog(st,LOG_SEC,"invalid MSG3");
            }
@@ -1825,10 +2141,13 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
        case LABEL_MSG4:
            /* Setup packet: expected only in state SENTMSG3 */
            if (st->state!=SITE_SENTMSG3) {
+               if ((st->state==SITE_SENTMSG5) &&
+                   setup_late_msg_ok(st,buf,msgtype,source,&msg))
+                   break;
                slog(st,LOG_UNEXPECTED,"unexpected MSG4");
-           } else if (process_msg4(st,buf,source)) {
+           } else if (process_msg4(st,buf,source,&msg)) {
                transport_setup_msgok(st,source);
-               enter_new_state(st,SITE_SENTMSG5);
+               enter_new_state(st,SITE_SENTMSG5,&msg);
            } else {
                slog(st,LOG_SEC,"invalid MSG4");
            }
@@ -1843,7 +2162,7 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
            if (st->state==SITE_SENTMSG4) {
                if (process_msg5(st,buf,source,st->new_transform)) {
                    transport_setup_msgok(st,source);
-                   enter_new_state(st,SITE_RUN);
+                   enter_new_state(st,SITE_RUN,&msg);
                } else {
                    slog(st,LOG_SEC,"invalid MSG5");
                }
@@ -1883,6 +2202,8 @@ static bool_t site_incoming(void *sst, struct buffer_if *buf,
        return True;
     }
 
+    priomsg_update_fixed(whynot, comm_notify_whynot_general,
+                        "not MSG1 or PROD; unknown dest index");
     return False;
 }
 
@@ -1918,6 +2239,17 @@ static void site_childpersist_clearkeys(void *sst, uint32_t newphase)
        crypto operations, but that's a task for another day. */
 }
 
+static void setup_sethash(struct site *st, dict_t *dict,
+                         struct hash_if **hash, struct cloc loc,
+                         sig_sethash_fn *sethash, void *sigkey_st) {
+    if (!*hash) *hash=find_cl_if(dict,"hash",CL_HASH,True,"site",loc);
+    sethash(sigkey_st,*hash);
+}
+#define SETUP_SETHASH(k) do{                                           \
+    if ((k)->sethash)                                                  \
+        setup_sethash(st,dict, &hash,loc, (k)->sethash,(k)->st);       \
+}while(0)
+
 static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
                          list_t *args)
 {
@@ -1927,7 +2259,7 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     dict_t *dict;
     int i;
 
-    st=safe_malloc(sizeof(*st),"site_apply");
+    NEW(st);
 
     st->cl.description="site";
     st->cl.type=CL_SITE;
@@ -1936,6 +2268,8 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     st->ops.st=st;
     st->ops.control=site_control;
     st->ops.status=site_status;
+    st->peerkeys_path=0;
+    st->peerkeys_current=st->peerkeys_kex=0;
 
     /* First parameter must be a dict */
     item=list_elem(args,0);
@@ -1946,6 +2280,8 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     st->localname=dict_read_string(dict, "local-name", True, "site", loc);
     st->remotename=dict_read_string(dict, "name", True, "site", loc);
 
+    st->keepalive=dict_read_bool(dict,"keepalive",False,"site",loc,False);
+
     st->peer_mobile=dict_read_bool(dict,"mobile",False,"site",loc,False);
     st->local_mobile=
        dict_read_bool(dict,"local-mobile",False,"site",loc,False);
@@ -1973,6 +2309,7 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     assert(index_sequence < 0xffffffffUL);
     st->index = ++index_sequence;
     st->local_capabilities = 0;
+    st->early_capabilities = CAPAB_PRIORITY_MOBILE;
     st->netlink=find_cl_if(dict,"link",CL_NETLINK,True,"site",loc);
 
 #define GET_CLOSURE_LIST(dictkey,things,nthings,CL_TYPE) do{           \
@@ -1980,7 +2317,7 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     if (!things##_cfg)                                                 \
        cfgfatal(loc,"site","closure list \"%s\" not found\n",dictkey); \
     st->nthings=list_length(things##_cfg);                             \
-    st->things=safe_malloc_ary(sizeof(*st->things),st->nthings,dictkey "s"); \
+    NEW_ARY(st->things,st->nthings);                                   \
     assert(st->nthings);                                               \
     for (i=0; i<st->nthings; i++) {                                    \
        item_t *item=list_elem(things##_cfg,i);                         \
@@ -1995,21 +2332,57 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
 
     GET_CLOSURE_LIST("comm",comms,ncomms,CL_COMM);
 
+    NEW_ARY(st->commclientinfos, st->ncomms);
+    dict_t *comminfo = dict_read_dict(dict,"comm-info",False,"site",loc);
+    for (i=0; i<st->ncomms; i++) {
+       st->commclientinfos[i] =
+           !comminfo ? 0 :
+           st->comms[i]->clientinfo(st->comms[i],comminfo,loc);
+    }
+
     st->resolver=find_cl_if(dict,"resolver",CL_RESOLVER,True,"site",loc);
     st->log=find_cl_if(dict,"log",CL_LOG,True,"site",loc);
     st->random=find_cl_if(dict,"random",CL_RANDOMSRC,True,"site",loc);
 
-    st->privkey=find_cl_if(dict,"local-key",CL_RSAPRIVKEY,True,"site",loc);
+    struct hash_if *hash=0;
+
+    st->privkeys=find_cl_if(dict,"key-cache",CL_PRIVCACHE,False,"site",loc);
+    if (!st->privkeys) {
+       st->privkey_fixed=
+           find_cl_if(dict,"local-key",CL_SIGPRIVKEY,True,"site",loc);
+       SETUP_SETHASH(st->privkey_fixed);
+    }
+
+    struct sigpubkey_if *fixed_pubkey
+       =find_cl_if(dict,"key",CL_SIGPUBKEY,False,"site",loc);
+    st->peerkeys_path=dict_read_string(dict,"peer-keys",fixed_pubkey==0,
+                                      "site",loc);
+    if (st->peerkeys_path) {
+       st->peerkeys_current=keyset_load(st->peerkeys_path,
+                                        &st->scratch,st->log,M_ERR);
+       if (fixed_pubkey) {
+           fixed_pubkey->dispose(fixed_pubkey->st);
+       }
+    } else {
+       assert(fixed_pubkey);
+       SETUP_SETHASH(fixed_pubkey);
+       NEW(st->peerkeys_current);
+       st->peerkeys_current->refcount=1;
+       st->peerkeys_current->nkeys=1;
+       st->peerkeys_current->keys[0].id=keyid_zero;
+       st->peerkeys_current->keys[0].pubkey=fixed_pubkey;
+       slog(st,LOG_SIGKEYS,
+            "using old-style fixed peer public key (no `peer-keys')");
+    }
+
     st->addresses=dict_read_string_array(dict,"address",False,"site",loc,0);
     if (st->addresses)
        st->remoteport=dict_read_number(dict,"port",True,"site",loc,0);
     else st->remoteport=0;
-    st->pubkey=find_cl_if(dict,"key",CL_RSAPUBKEY,True,"site",loc);
 
     GET_CLOSURE_LIST("transform",transforms,ntransforms,CL_TRANSFORM);
 
     st->dh=find_cl_if(dict,"dh",CL_DH,True,"site",loc);
-    st->hash=find_cl_if(dict,"hash",CL_HASH,True,"site",loc);
 
 #define DEFAULT(D) (st->peer_mobile || st->local_mobile        \
                     ? DEFAULT_MOBILE_##D : DEFAULT_##D)
@@ -2018,7 +2391,7 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     st->key_lifetime=         CFG_NUMBER("key-lifetime",  KEY_LIFETIME);
     st->setup_retries=        CFG_NUMBER("setup-retries", SETUP_RETRIES);
     st->setup_retry_interval= CFG_NUMBER("setup-timeout", SETUP_RETRY_INTERVAL);
-    st->wait_timeout=         CFG_NUMBER("wait-time",     WAIT_TIME);
+    st->wait_timeout_mean=    CFG_NUMBER("wait-time",     WAIT_TIME);
     st->mtu_target= dict_read_number(dict,"mtu-target",False,"site",loc,0);
 
     st->mobile_peer_expiry= dict_read_number(
@@ -2058,7 +2431,7 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     /* The information we expect to see in incoming messages of type 1 */
     /* fixme: lots of unchecked overflows here, but the results are only
        corrupted packets rather than undefined behaviour */
-    st->setup_priority=(strcmp(st->localname,st->remotename)>0);
+    st->our_name_later=(strcmp(st->localname,st->remotename)>0);
 
     buffer_new(&st->buffer,SETUP_BUFFER_LEN);
 
@@ -2081,14 +2454,21 @@ static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
     st->sharedsecretlen=st->sharedsecretallocd=0;
     st->sharedsecret=0;
 
-    for (i=0; i<st->ntransforms; i++) {
-       struct transform_if *ti=st->transforms[i];
-       uint32_t capbit = 1UL << ti->capab_transformnum;
-       if (st->local_capabilities & capbit)
-           slog(st,LOG_ERROR,"transformnum capability bit"
-                " %d (%#"PRIx32") reused", ti->capab_transformnum, capbit);
-       st->local_capabilities |= capbit;
-    }
+#define SET_CAPBIT(bit) do {                                           \
+    uint32_t capflag = 1UL << (bit);                                   \
+    if (st->local_capabilities & capflag)                              \
+       slog(st,LOG_ERROR,"capability bit"                              \
+            " %d (%#"PRIx32") reused", (bit), capflag);                \
+    st->local_capabilities |= capflag;                                 \
+} while (0)
+
+    for (i=0; i<st->ntransforms; i++)
+       SET_CAPBIT(st->transforms[i]->capab_bit);
+
+#undef SET_CAPBIT
+
+    if (st->local_mobile || st->peer_mobile)
+       st->local_capabilities |= CAPAB_PRIORITY_MOBILE;
 
     /* We need to register the remote networks with the netlink device */
     uint32_t netlink_mtu; /* local virtual interface mtu */
@@ -2193,11 +2573,18 @@ static void transport_record_peers(struct site *st, transport_peers *peers,
      *
      * Caller must first call transport_peers_expire. */
 
-    if (naddrs==1 && peers->npeers>=1 &&
-       comm_addr_equal(&addrs[0], &peers->peers[0].addr)) {
-       /* optimisation, also avoids debug for trivial updates */
-       peers->peers[0].last = *tv_now;
-       return;
+    if (naddrs==1) {
+       /* avoids debug for uninteresting updates */
+       int i;
+       for (i=0; i<peers->npeers; i++) {
+           if (comm_addr_equal(&addrs[0], &peers->peers[i].addr)) {
+               memmove(peers->peers+1, peers->peers,
+                       sizeof(peers->peers[0]) * i);
+               peers->peers[0].addr = addrs[0];
+               peers->peers[0].last = *tv_now;
+               return;
+           }
+       }
     }
 
     int old_npeers=peers->npeers;
@@ -2328,10 +2715,9 @@ void transport_xmit(struct site *st, transport_peers *peers,
     int nfailed=0;
     for (slot=0; slot<peers->npeers; slot++) {
        transport_peer *peer=&peers->peers[slot];
+       bool_t ok = comm_addr_sendmsg(st, &peer->addr, buf);
        if (candebug)
-           dump_packet(st, buf, &peer->addr, False);
-       bool_t ok =
-           peer->addr.comm->sendmsg(peer->addr.comm->st, buf, &peer->addr);
+           dump_packet(st, buf, &peer->addr, False, ok);
        if (!ok) {
            failed |= 1U << slot;
            nfailed++;
@@ -2353,12 +2739,14 @@ void transport_xmit(struct site *st, transport_peers *peers,
            transport_peers__copy_by_mask(peers->peers,&wslot,~failed,peers);
            assert(wslot+nfailed == peers->npeers);
            COPY_ARRAY(peers->peers+wslot, failedpeers, nfailed);
+           transport_peers_debug(st,peers,"mobile failure reorder",0,0,0);
        }
     } else {
        if (failed && peers->npeers > 1) {
            int wslot=0;
            transport_peers__copy_by_mask(peers->peers,&wslot,~failed,peers);
            peers->npeers=wslot;
+           transport_peers_debug(st,peers,"non-mobile failure cleanup",0,0,0);
        }
     }
 }