chiark / gitweb /
site: kex_init: Have it return a boolean
[secnet.git] / site.c
1 /* site.c - manage communication with a remote network site */
2
3 /*
4  * This file is part of secnet.
5  * See README for full list of copyright holders.
6  *
7  * secnet is free software; you can redistribute it and/or modify it
8  * under the terms of the GNU General Public License as published by
9  * the Free Software Foundation; either version 3 of the License, or
10  * (at your option) any later version.
11  * 
12  * secnet is distributed in the hope that it will be useful, but
13  * WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * General Public License for more details.
16  * 
17  * You should have received a copy of the GNU General Public License
18  * version 3 along with secnet; if not, see
19  * https://www.gnu.org/licenses/gpl.html.
20  */
21
22 /* The 'site' code doesn't know anything about the structure of the
23    packets it's transmitting.  In fact, under the new netlink
24    configuration scheme it doesn't need to know anything at all about
25    IP addresses, except how to contact its peer.  This means it could
26    potentially be used to tunnel other protocols too (IPv6, IPX, plain
27    old Ethernet frames) if appropriate netlink code can be written
28    (and that ought not to be too hard, eg. using the TUN/TAP device to
29    pretend to be an Ethernet interface).  */
30
31 /* At some point in the future the netlink code will be asked for
32    configuration information to go in the PING/PONG packets at the end
33    of the key exchange. */
34
35 #include "secnet.h"
36 #include <stdio.h>
37 #include <string.h>
38 #include <limits.h>
39 #include <assert.h>
40 #include <sys/socket.h>
41
42 #include <sys/mman.h>
43 #include "util.h"
44 #include "unaligned.h"
45 #include "magic.h"
46
47 #define SETUP_BUFFER_LEN 2048
48
49 #define DEFAULT_KEY_LIFETIME                  (3600*1000) /* [ms] */
50 #define DEFAULT_KEY_RENEGOTIATE_GAP           (5*60*1000) /* [ms] */
51 #define DEFAULT_SETUP_RETRIES 5
52 #define DEFAULT_SETUP_RETRY_INTERVAL             (2*1000) /* [ms] */
53 #define DEFAULT_WAIT_TIME                       (20*1000) /* [ms] */
54
55 #define DEFAULT_MOBILE_KEY_LIFETIME      (2*24*3600*1000) /* [ms] */
56 #define DEFAULT_MOBILE_KEY_RENEGOTIATE_GAP (12*3600*1000) /* [ms] */
57 #define DEFAULT_MOBILE_SETUP_RETRIES 30
58 #define DEFAULT_MOBILE_SETUP_RETRY_INTERVAL      (1*1000) /* [ms] */
59 #define DEFAULT_MOBILE_WAIT_TIME                (10*1000) /* [ms] */
60
61 #define DEFAULT_MOBILE_PEER_EXPIRY            (2*60)      /* [s] */
62
63 /* Each site can be in one of several possible states. */
64
65 /* States:
66    SITE_STOP         - nothing is allowed to happen; tunnel is down;
67                        all session keys have been erased
68      -> SITE_RUN upon external instruction
69    SITE_RUN          - site up, maybe with valid key
70      -> SITE_RESOLVE upon outgoing packet and no valid key
71          we start name resolution for the other end of the tunnel
72      -> SITE_SENTMSG2 upon valid incoming message 1 and suitable time
73          we send an appropriate message 2
74    SITE_RESOLVE      - waiting for name resolution
75      -> SITE_SENTMSG1 upon successful resolution
76          we send an appropriate message 1
77      -> SITE_SENTMSG2 upon valid incoming message 1 (then abort resolution)
78          we abort resolution and 
79      -> SITE_WAIT on timeout or resolution failure
80    SITE_SENTMSG1
81      -> SITE_SENTMSG2 upon valid incoming message 1 from higher priority end
82      -> SITE_SENTMSG3 upon valid incoming message 2
83      -> SITE_WAIT on timeout
84    SITE_SENTMSG2
85      -> SITE_SENTMSG4 upon valid incoming message 3
86      -> SITE_WAIT on timeout
87    SITE_SENTMSG3
88      -> SITE_SENTMSG5 upon valid incoming message 4
89      -> SITE_WAIT on timeout
90    SITE_SENTMSG4
91      -> SITE_RUN upon valid incoming message 5
92      -> SITE_WAIT on timeout
93    SITE_SENTMSG5
94      -> SITE_RUN upon valid incoming message 6
95      -> SITE_WAIT on timeout
96    SITE_WAIT         - failed to establish key; do nothing for a while
97      -> SITE_RUN on timeout
98    */
99
100 #define SITE_STOP     0
101 #define SITE_RUN      1
102 #define SITE_RESOLVE  2
103 #define SITE_SENTMSG1 3
104 #define SITE_SENTMSG2 4
105 #define SITE_SENTMSG3 5
106 #define SITE_SENTMSG4 6
107 #define SITE_SENTMSG5 7
108 #define SITE_WAIT     8
109
110 #define CASES_MSG3_KNOWN LABEL_MSG3: case LABEL_MSG3BIS
111
112 struct msg;
113
114 int32_t site_max_start_pad = 4*4;
115
116 static cstring_t state_name(uint32_t state)
117 {
118     switch (state) {
119     case 0: return "STOP";
120     case 1: return "RUN";
121     case 2: return "RESOLVE";
122     case 3: return "SENTMSG1";
123     case 4: return "SENTMSG2";
124     case 5: return "SENTMSG3";
125     case 6: return "SENTMSG4";
126     case 7: return "SENTMSG5";
127     case 8: return "WAIT";
128     default: return "*bad state*";
129     }
130 }
131
132 #define NONCELEN 8
133
134 #define LOG_UNEXPECTED    0x00000001
135 #define LOG_SETUP_INIT    0x00000002
136 #define LOG_SETUP_TIMEOUT 0x00000004
137 #define LOG_ACTIVATE_KEY  0x00000008
138 #define LOG_TIMEOUT_KEY   0x00000010
139 #define LOG_SEC           0x00000020
140 #define LOG_STATE         0x00000040
141 #define LOG_DROP          0x00000080
142 #define LOG_DUMP          0x00000100
143 #define LOG_ERROR         0x00000400
144 #define LOG_PEER_ADDRS    0x00000800
145
146 static struct flagstr log_event_table[]={
147     { "unexpected", LOG_UNEXPECTED },
148     { "setup-init", LOG_SETUP_INIT },
149     { "setup-timeout", LOG_SETUP_TIMEOUT },
150     { "activate-key", LOG_ACTIVATE_KEY },
151     { "timeout-key", LOG_TIMEOUT_KEY },
152     { "security", LOG_SEC },
153     { "state-change", LOG_STATE },
154     { "packet-drop", LOG_DROP },
155     { "dump-packets", LOG_DUMP },
156     { "errors", LOG_ERROR },
157     { "peer-addrs", LOG_PEER_ADDRS },
158     { "default", LOG_SETUP_INIT|LOG_SETUP_TIMEOUT|
159       LOG_ACTIVATE_KEY|LOG_TIMEOUT_KEY|LOG_SEC|LOG_ERROR },
160     { "all", 0xffffffff },
161     { NULL, 0 }
162 };
163
164
165 /***** TRANSPORT PEERS declarations *****/
166
167 /* Details of "mobile peer" semantics:
168
169    - We use the same data structure for the different configurations,
170      but manage it with different algorithms.
171    
172    - We record up to mobile_peers_max peer address/port numbers
173      ("peers") for key setup, and separately up to mobile_peers_max
174      for data transfer.
175
176    - In general, we make a new set of addrs (see below) when we start
177      a new key exchange; the key setup addrs become the data transport
178      addrs when key setup complets.
179
180    If our peer is mobile:
181
182    - We send to all recent addresses of incoming packets, plus
183      initially all configured addresses (which we also expire).
184
185    - So, we record addrs of good incoming packets, as follows:
186       1. expire any peers last seen >120s ("mobile-peer-expiry") ago
187       2. add the peer of the just received packet to the applicable list
188          (possibly evicting the oldest entries to make room)
189      NB that we do not expire peers until an incoming packet arrives.
190
191    - If the peer has a configured address or name, we record them the
192      same way, but only as a result of our own initiation of key
193      setup.  (We might evict some incoming packet addrs to make room.)
194
195    - The default number of addrs to keep is 3, or 4 if we have a
196      configured name or address.  That's space for two configured
197      addresses (one IPv6 and one IPv4), plus two received addresses.
198
199    - Outgoing packets are sent to every recorded address in the
200      applicable list.  Any unsupported[1] addresses are deleted from
201      the list right away.  (This should only happen to configured
202      addresses, of course, but there is no need to check that.)
203
204    - When we successfully complete a key setup, we merge the key setup
205      peers into the data transfer peers.
206
207    [1] An unsupported address is one for whose AF we don't have a
208      socket (perhaps because we got EAFNOSUPPORT or some such) or for
209      which sendto gives ENETUNREACH.
210
211    If neither end is mobile:
212
213    - When peer initiated the key exchange, we use the incoming packet
214      address.
215
216    - When we initiate the key exchange, we try configured addresses
217      until we get one which isn't unsupported then fixate on that.
218
219    - When we complete a key setup, we replace the data transport peers
220      with those from the key setup.
221
222    If we are mobile:
223
224    - We can't tell when local network setup changes so we can't cache
225      the unsupported addrs and completely remove the spurious calls to
226      sendto, but we can optimise things a bit by deprioritising addrs
227      which seem to be unsupported.
228
229    - Use only configured addresses.  (Except, that if our peer
230      initiated a key exchange we use the incoming packet address until
231      our name resolution completes.)
232
233    - When we send a packet, try each address in turn; if addr
234      supported, put that address to the end of the list for future
235      packets, and go onto the next address.
236
237    - When we complete a key setup, we replace the data transport peers
238      with those from the key setup.
239
240    */
241
242 typedef struct {
243     struct timeval last;
244     struct comm_addr addr;
245 } transport_peer;
246
247 typedef struct {
248 /* configuration information */
249 /* runtime information */
250     int npeers;
251     transport_peer peers[MAX_PEER_ADDRS];
252 } transport_peers;
253
254 /* Basic operations on transport peer address sets */
255 static void transport_peers_clear(struct site *st, transport_peers *peers);
256 static int transport_peers_valid(transport_peers *peers);
257 static void transport_peers_copy(struct site *st, transport_peers *dst,
258                                  const transport_peers *src);
259
260 /* Record address of incoming setup packet; resp. data packet. */
261 static void transport_setup_msgok(struct site *st, const struct comm_addr *a);
262 static void transport_data_msgok(struct site *st, const struct comm_addr *a);
263
264 /* Initialise the setup addresses.  Called before we send the first
265  * packet in a key exchange.  If we are the initiator, as a result of
266  * resolve completing (or being determined not to be relevant) or an
267  * incoming PROD; if we are the responder, as a result of the MSG1. */
268 static bool_t transport_compute_setupinit_peers(struct site *st,
269         const struct comm_addr *configured_addrs /* 0 if none or not found */,
270         int n_configured_addrs /* 0 if none or not found */,
271         const struct comm_addr *incoming_packet_addr /* 0 if none */);
272
273 /* Called if we are the responder in a key setup, when the resolve
274  * completes.  transport_compute_setupinit_peers will hvae been called
275  * earlier.  If _complete is called, we are still doing the key setup
276  * (and we should use the new values for both the rest of the key
277  * setup and the ongoing data exchange); if _tardy is called, the key
278  * setup is done (either completed or not) and only the data peers are
279  * relevant */
280 static void transport_resolve_complete(struct site *st,
281         const struct comm_addr *addrs, int naddrs);
282 static void transport_resolve_complete_tardy(struct site *st,
283         const struct comm_addr *addrs, int naddrs);
284
285 static void transport_xmit(struct site *st, transport_peers *peers,
286                            struct buffer_if *buf, bool_t candebug);
287
288  /***** END of transport peers declarations *****/
289
290
291 struct data_key {
292     struct transform_inst_if *transform;
293     uint64_t key_timeout; /* End of life of current key */
294     uint32_t remote_session_id;
295 };
296
297 struct site {
298     closure_t cl;
299     struct site_if ops;
300 /* configuration information */
301     string_t localname;
302     string_t remotename;
303     bool_t keepalive;
304     bool_t local_mobile, peer_mobile; /* Mobile client support */
305     int32_t transport_peers_max;
306     string_t tunname; /* localname<->remotename by default, used in logs */
307     cstring_t *addresses; /* DNS name or address(es) for bootstrapping, optional */
308     int remoteport; /* Port for bootstrapping, optional */
309     uint32_t mtu_target;
310     struct netlink_if *netlink;
311     struct comm_if **comms;
312     struct comm_clientinfo **commclientinfos;
313     int ncomms;
314     struct resolver_if *resolver;
315     struct log_if *log;
316     struct random_if *random;
317     struct sigprivkey_if *privkey;
318     struct sigpubkey_if *pubkey;
319     struct transform_if **transforms;
320     int ntransforms;
321     struct dh_if *dh;
322
323     uint32_t index; /* Index of this site */
324     uint32_t early_capabilities;
325     uint32_t local_capabilities;
326     int32_t setup_retries; /* How many times to send setup packets */
327     int32_t setup_retry_interval; /* Initial timeout for setup packets */
328     int32_t wait_timeout_mean; /* How long to wait if setup unsuccessful */
329     int32_t mobile_peer_expiry; /* How long to remember 2ary addresses */
330     int32_t key_lifetime; /* How long a key lasts once set up */
331     int32_t key_renegotiate_time; /* If we see traffic (or a keepalive)
332                                       after this time, initiate a new
333                                       key exchange */
334
335     bool_t our_name_later; /* our name > peer name */
336     uint32_t log_events;
337
338 /* runtime information */
339     uint32_t state;
340     uint64_t now; /* Most recently seen time */
341     bool_t allow_send_prod;
342     bool_t msg1_crossed_logged;
343     int resolving_count;
344     int resolving_n_results_all;
345     int resolving_n_results_stored;
346     struct comm_addr resolving_results[MAX_PEER_ADDRS];
347
348     /* The currently established session */
349     struct data_key current;
350     struct data_key auxiliary_key;
351     bool_t auxiliary_is_new;
352     uint64_t renegotiate_key_time; /* When we can negotiate a new key */
353     uint64_t auxiliary_renegotiate_key_time;
354     transport_peers peers; /* Current address(es) of peer for data traffic */
355
356     /* The current key setup protocol exchange.  We can only be
357        involved in one of these at a time.  There's a potential for
358        denial of service here (the attacker keeps sending a setup
359        packet; we keep trying to continue the exchange, and have to
360        timeout before we can listen for another setup packet); perhaps
361        we should keep a list of 'bad' sources for setup packets. */
362     uint32_t remote_capabilities;
363     uint16_t remote_adv_mtu;
364     struct transform_if *chosen_transform;
365     uint32_t setup_session_id;
366     transport_peers setup_peers;
367     uint8_t localN[NONCELEN]; /* Nonces for key exchange */
368     uint8_t remoteN[NONCELEN];
369     struct buffer_if buffer; /* Current outgoing key exchange packet */
370     struct buffer_if scratch;
371     int32_t retries; /* Number of retries remaining */
372     uint64_t timeout; /* Timeout for current state */
373     uint8_t *dhsecret;
374     uint8_t *sharedsecret;
375     uint32_t sharedsecretlen, sharedsecretallocd;
376     struct transform_inst_if *new_transform; /* For key setup/verify */
377 };
378
379 static uint32_t event_log_priority(struct site *st, uint32_t event)
380 {
381     if (!(event&st->log_events))
382         return 0;
383     switch(event) {
384     case LOG_UNEXPECTED:    return M_INFO;
385     case LOG_SETUP_INIT:    return M_INFO;
386     case LOG_SETUP_TIMEOUT: return M_NOTICE;
387     case LOG_ACTIVATE_KEY:  return M_INFO;
388     case LOG_TIMEOUT_KEY:   return M_INFO;
389     case LOG_SEC:           return M_SECURITY;
390     case LOG_STATE:         return M_DEBUG;
391     case LOG_DROP:          return M_DEBUG;
392     case LOG_DUMP:          return M_DEBUG;
393     case LOG_ERROR:         return M_ERR;
394     case LOG_PEER_ADDRS:    return M_DEBUG;
395     default:                return M_ERR;
396     }
397 }
398
399 static uint32_t slog_start(struct site *st, uint32_t event)
400 {
401     uint32_t class=event_log_priority(st, event);
402     if (class) {
403         slilog_part(st->log,class,"%s: ",st->tunname);
404     }
405     return class;
406 }
407
408 static void vslog(struct site *st, uint32_t event, cstring_t msg, va_list ap)
409 FORMAT(printf,3,0);
410 static void vslog(struct site *st, uint32_t event, cstring_t msg, va_list ap)
411 {
412     uint32_t class;
413
414     class=slog_start(st,event);
415     if (class) {
416         vslilog_part(st->log,class,msg,ap);
417         slilog_part(st->log,class,"\n");
418     }
419 }
420
421 static void slog(struct site *st, uint32_t event, cstring_t msg, ...)
422 FORMAT(printf,3,4);
423 static void slog(struct site *st, uint32_t event, cstring_t msg, ...)
424 {
425     va_list ap;
426     va_start(ap,msg);
427     vslog(st,event,msg,ap);
428     va_end(ap);
429 }
430
431 static void logtimeout(struct site *st, const char *fmt, ...)
432 FORMAT(printf,2,3);
433 static void logtimeout(struct site *st, const char *fmt, ...)
434 {
435     uint32_t class=event_log_priority(st,LOG_SETUP_TIMEOUT);
436     if (!class)
437         return;
438
439     va_list ap;
440     va_start(ap,fmt);
441
442     slilog_part(st->log,class,"%s: ",st->tunname);
443     vslilog_part(st->log,class,fmt,ap);
444
445     const char *delim;
446     int i;
447     for (i=0, delim=" (tried ";
448          i<st->setup_peers.npeers;
449          i++, delim=", ") {
450         transport_peer *peer=&st->setup_peers.peers[i];
451         const char *s=comm_addr_to_string(&peer->addr);
452         slilog_part(st->log,class,"%s%s",delim,s);
453     }
454
455     slilog_part(st->log,class,")\n");
456     va_end(ap);
457 }
458
459 static void set_link_quality(struct site *st);
460 static void delete_keys(struct site *st, cstring_t reason, uint32_t loglevel);
461 static void delete_one_key(struct site *st, struct data_key *key,
462                            const char *reason /* may be 0 meaning don't log*/,
463                            const char *which /* ignored if !reasonn */,
464                            uint32_t loglevel /* ignored if !reasonn */);
465 static bool_t initiate_key_setup(struct site *st, cstring_t reason,
466                                  const struct comm_addr *prod_hint);
467 static void enter_state_run(struct site *st);
468 static bool_t enter_state_resolve(struct site *st);
469 static void decrement_resolving_count(struct site *st, int by);
470 static bool_t enter_new_state(struct site *st,uint32_t next,
471                               const struct msg *prompt
472                               /* may be 0 for SENTMSG1 */);
473 static void enter_state_wait(struct site *st);
474 static void activate_new_key(struct site *st);
475
476 static bool_t is_transform_valid(struct transform_inst_if *transform)
477 {
478     return transform && transform->valid(transform->st);
479 }
480
481 static bool_t current_valid(struct site *st)
482 {
483     return is_transform_valid(st->current.transform);
484 }
485
486 #define DEFINE_CALL_TRANSFORM(fwdrev)                                   \
487 static transform_apply_return                                           \
488 call_transform_##fwdrev(struct site *st,                                \
489                                    struct transform_inst_if *transform, \
490                                    struct buffer_if *buf,               \
491                                    const char **errmsg)                 \
492 {                                                                       \
493     if (!is_transform_valid(transform)) {                               \
494         *errmsg="transform not set up";                                 \
495         return transform_apply_err;                                     \
496     }                                                                   \
497     return transform->fwdrev(transform->st,buf,errmsg);                 \
498 }
499
500 DEFINE_CALL_TRANSFORM(forwards)
501 DEFINE_CALL_TRANSFORM(reverse)
502
503 static void dispose_transform(struct transform_inst_if **transform_var)
504 {
505     struct transform_inst_if *transform=*transform_var;
506     if (transform) {
507         transform->delkey(transform->st);
508         transform->destroy(transform->st);
509     }
510     *transform_var = 0;
511 }    
512
513 #define CHECK_AVAIL(b,l) do { if ((b)->size<(l)) return False; } while(0)
514 #define CHECK_EMPTY(b) do { if ((b)->size!=0) return False; } while(0)
515 #define CHECK_TYPE(b,t) do { uint32_t type; \
516     CHECK_AVAIL((b),4); \
517     type=buf_unprepend_uint32((b)); \
518     if (type!=(t)) return False; } while(0)
519
520 static _Bool type_is_msg34(uint32_t type)
521 {
522     switch (type) {
523         case CASES_MSG3_KNOWN: case LABEL_MSG4: return True;
524         default: return False;
525     }
526 }
527
528 struct parsedname {
529     int32_t len;
530     uint8_t *name;
531     struct buffer_if extrainfo;
532 };
533
534 struct msg {
535     uint8_t *hashstart;
536     uint32_t dest;
537     uint32_t source;
538     struct parsedname remote;
539     struct parsedname local;
540     uint32_t remote_capabilities;
541     uint16_t remote_mtu;
542     int capab_transformnum;
543     uint8_t *nR;
544     uint8_t *nL;
545     int32_t pklen;
546     char *pk;
547     int32_t hashlen;
548     struct alg_msg_data sig;
549 };
550
551 static int32_t wait_timeout(struct site *st) {
552     int32_t t = st->wait_timeout_mean;
553     int8_t factor;
554     if (t < INT_MAX/2) {
555         st->random->generate(st->random->st,sizeof(factor),&factor);
556         t += (t / 256) * factor;
557     }
558     return t;
559 }
560
561 static _Bool set_new_transform(struct site *st, char *pk)
562 {
563     _Bool ok;
564
565     /* Make room for the shared key */
566     st->sharedsecretlen=st->chosen_transform->keylen?:st->dh->ceil_len;
567     assert(st->sharedsecretlen);
568     if (st->sharedsecretlen > st->sharedsecretallocd) {
569         st->sharedsecretallocd=st->sharedsecretlen;
570         st->sharedsecret=safe_realloc_ary(st->sharedsecret,1,
571                                           st->sharedsecretallocd,
572                                           "site:sharedsecret");
573     }
574
575     /* Generate the shared key */
576     st->dh->makeshared(st->dh->st,st->dhsecret,st->dh->len,pk,
577                        st->sharedsecret,st->sharedsecretlen);
578
579     /* Set up the transform */
580     struct transform_if *generator=st->chosen_transform;
581     struct transform_inst_if *generated=generator->create(generator->st);
582     ok = generated->setkey(generated->st,st->sharedsecret,
583                            st->sharedsecretlen,st->our_name_later);
584
585     dispose_transform(&st->new_transform);
586     if (!ok) return False;
587     st->new_transform=generated;
588
589     slog(st,LOG_SETUP_INIT,"key exchange negotiated transform"
590          " %d (capabilities ours=%#"PRIx32" theirs=%#"PRIx32")",
591          st->chosen_transform->capab_bit,
592          st->local_capabilities, st->remote_capabilities);
593     return True;
594 }
595
596 struct xinfoadd {
597     int32_t lenpos, afternul;
598 };
599 static void append_string_xinfo_start(struct buffer_if *buf,
600                                       struct xinfoadd *xia,
601                                       const char *str)
602     /* Helps construct one of the names with additional info as found
603      * in MSG1..4.  Call this function first, then append all the
604      * desired extra info (not including the nul byte) to the buffer,
605      * then call append_string_xinfo_done. */
606 {
607     xia->lenpos = buf->size;
608     buf_append_string(buf,str);
609     buf_append_uint8(buf,0);
610     xia->afternul = buf->size;
611 }
612 static void append_string_xinfo_done(struct buffer_if *buf,
613                                      struct xinfoadd *xia)
614 {
615     /* we just need to adjust the string length */
616     if (buf->size == xia->afternul) {
617         /* no extra info, strip the nul too */
618         buf_unappend_uint8(buf);
619     } else {
620         put_uint16(buf->start+xia->lenpos, buf->size-(xia->lenpos+2));
621     }
622 }
623
624 /* Build any of msg1 to msg4. msg5 and msg6 are built from the inside
625    out using a transform of config data supplied by netlink */
626 static bool_t generate_msg(struct site *st, uint32_t type, cstring_t what,
627                            const struct msg *prompt
628                            /* may be 0 for MSG1 */)
629 {
630     string_t dhpub;
631     unsigned minor;
632
633     st->retries=st->setup_retries;
634     BUF_ALLOC(&st->buffer,what);
635     buffer_init(&st->buffer,0);
636     buf_append_uint32(&st->buffer,
637         (type==LABEL_MSG1?0:st->setup_session_id));
638     buf_append_uint32(&st->buffer,st->index);
639     buf_append_uint32(&st->buffer,type);
640
641     struct xinfoadd xia;
642     append_string_xinfo_start(&st->buffer,&xia,st->localname);
643     if ((st->local_capabilities & st->early_capabilities) ||
644         (type != LABEL_MSG1)) {
645         buf_append_uint32(&st->buffer,st->local_capabilities);
646     }
647     if (type_is_msg34(type)) {
648         buf_append_uint16(&st->buffer,st->mtu_target);
649     }
650     append_string_xinfo_done(&st->buffer,&xia);
651
652     buf_append_string(&st->buffer,st->remotename);
653     BUF_ADD_OBJ(append,&st->buffer,st->localN);
654     if (type==LABEL_MSG1) return True;
655     BUF_ADD_OBJ(append,&st->buffer,st->remoteN);
656     if (type==LABEL_MSG2) return True;
657
658     if (hacky_par_mid_failnow()) return False;
659
660     if (MSGMAJOR(type) == 3) do {
661         minor = MSGMINOR(type);
662         if (minor < 1) break;
663         buf_append_uint8(&st->buffer,st->chosen_transform->capab_bit);
664     } while (0);
665
666     dhpub=st->dh->makepublic(st->dh->st,st->dhsecret,st->dh->len);
667     buf_append_string(&st->buffer,dhpub);
668     free(dhpub);
669
670     bool_t ok=st->privkey->sign(st->privkey->st,
671                                 st->buffer.start,
672                                 st->buffer.size,
673                                 &st->buffer);
674     if (!ok) goto fail;
675     return True;
676
677  fail:
678     return False;
679 }
680
681 static bool_t unpick_name(struct buffer_if *msg, struct parsedname *nm)
682 {
683     CHECK_AVAIL(msg,2);
684     nm->len=buf_unprepend_uint16(msg);
685     CHECK_AVAIL(msg,nm->len);
686     nm->name=buf_unprepend(msg,nm->len);
687     uint8_t *nul=memchr(nm->name,0,nm->len);
688     if (!nul) {
689         buffer_readonly_view(&nm->extrainfo,0,0);
690     } else {
691         buffer_readonly_view(&nm->extrainfo, nul+1, msg->start-(nul+1));
692         nm->len=nul-nm->name;
693     }
694     return True;
695 }
696
697 static bool_t unpick_msg(struct site *st, uint32_t type,
698                          struct buffer_if *msg, struct msg *m)
699 {
700     unsigned minor;
701
702     m->capab_transformnum=-1;
703     m->hashstart=msg->start;
704     CHECK_AVAIL(msg,4);
705     m->dest=buf_unprepend_uint32(msg);
706     CHECK_AVAIL(msg,4);
707     m->source=buf_unprepend_uint32(msg);
708     CHECK_TYPE(msg,type);
709     if (!unpick_name(msg,&m->remote)) return False;
710     m->remote_capabilities=0;
711     m->remote_mtu=0;
712     if (m->remote.extrainfo.size) {
713         CHECK_AVAIL(&m->remote.extrainfo,4);
714         m->remote_capabilities=buf_unprepend_uint32(&m->remote.extrainfo);
715     }
716     if (type_is_msg34(type) && m->remote.extrainfo.size) {
717         CHECK_AVAIL(&m->remote.extrainfo,2);
718         m->remote_mtu=buf_unprepend_uint16(&m->remote.extrainfo);
719     }
720     if (!unpick_name(msg,&m->local)) return False;
721     if (type==LABEL_PROD) {
722         CHECK_EMPTY(msg);
723         return True;
724     }
725     CHECK_AVAIL(msg,NONCELEN);
726     m->nR=buf_unprepend(msg,NONCELEN);
727     if (type==LABEL_MSG1) {
728         CHECK_EMPTY(msg);
729         return True;
730     }
731     CHECK_AVAIL(msg,NONCELEN);
732     m->nL=buf_unprepend(msg,NONCELEN);
733     if (type==LABEL_MSG2) {
734         CHECK_EMPTY(msg);
735         return True;
736     }
737     if (MSGMAJOR(type) == 3) do {
738         minor = MSGMINOR(type);
739 #define MAYBE_READ_CAP(minminor, kind, dflt) do {                       \
740     if (minor < (minminor))                                             \
741         m->capab_##kind##num = (dflt);                                  \
742     else {                                                              \
743         CHECK_AVAIL(msg, 1);                                            \
744         m->capab_##kind##num = buf_unprepend_uint8(msg);                \
745     }                                                                   \
746 } while (0)
747         MAYBE_READ_CAP(1, transform, CAPAB_BIT_ANCIENTTRANSFORM);
748 #undef MAYBE_READ_CAP
749     } while (0);
750     CHECK_AVAIL(msg,2);
751     m->pklen=buf_unprepend_uint16(msg);
752     CHECK_AVAIL(msg,m->pklen);
753     m->pk=buf_unprepend(msg,m->pklen);
754     m->hashlen=msg->start-m->hashstart;
755
756     if (!st->pubkey->unpick(st->pubkey->st,msg,&m->sig)) {
757         return False;
758     }
759
760     CHECK_EMPTY(msg);
761
762     return True;
763 }
764
765 static bool_t name_matches(const struct parsedname *nm, const char *expected)
766 {
767     int expected_len=strlen(expected);
768     return
769         nm->len == expected_len &&
770         !memcmp(nm->name, expected, expected_len);
771 }    
772
773 static bool_t check_msg(struct site *st, uint32_t type, struct msg *m,
774                         cstring_t *error)
775 {
776     if (type==LABEL_MSG1) return True;
777
778     /* Check that the site names and our nonce have been sent
779        back correctly, and then store our peer's nonce. */ 
780     if (!name_matches(&m->remote,st->remotename)) {
781         *error="wrong remote site name";
782         return False;
783     }
784     if (!name_matches(&m->local,st->localname)) {
785         *error="wrong local site name";
786         return False;
787     }
788     if (memcmp(m->nL,st->localN,NONCELEN)!=0) {
789         *error="wrong locally-generated nonce";
790         return False;
791     }
792     if (type==LABEL_MSG2) return True;
793     if (!consttime_memeq(m->nR,st->remoteN,NONCELEN)) {
794         *error="wrong remotely-generated nonce";
795         return False;
796     }
797     /* MSG3 has complicated rules about capabilities, which are
798      * handled in process_msg3. */
799     if (MSGMAJOR(type) == 3) return True;
800     if (m->remote_capabilities!=st->remote_capabilities) {
801         *error="remote capabilities changed";
802         return False;
803     }
804     if (type==LABEL_MSG4) return True;
805     *error="unknown message type";
806     return False;
807 }
808
809 static bool_t kex_init(struct site *st)
810 {
811     st->random->generate(st->random->st,NONCELEN,st->localN);
812     return True;
813 }
814
815 static bool_t generate_msg1(struct site *st, const struct msg *prompt_maybe_0)
816 {
817     return
818         kex_init(st) &&
819         generate_msg(st,LABEL_MSG1,"site:MSG1",prompt_maybe_0);
820 }
821
822 static bool_t process_msg1(struct site *st, struct buffer_if *msg1,
823                            const struct comm_addr *src,
824                            const struct msg *m)
825 {
826     /* We've already determined we're in an appropriate state to
827        process an incoming MSG1, and that the MSG1 has correct values
828        of A and B. */
829
830     st->setup_session_id=m->source;
831     st->remote_capabilities=m->remote_capabilities;
832     memcpy(st->remoteN,m->nR,NONCELEN);
833     return True;
834 }
835
836 static bool_t generate_msg2(struct site *st,
837                             const struct msg *prompt_may_be_null)
838 {
839     return
840         kex_init(st) &&
841         generate_msg(st,LABEL_MSG2,"site:MSG2",prompt_may_be_null);
842 }
843
844 static bool_t process_msg2(struct site *st, struct buffer_if *msg2,
845                            const struct comm_addr *src,
846                            struct msg *m /* returned */)
847 {
848     cstring_t err;
849
850     if (!unpick_msg(st,LABEL_MSG2,msg2,m)) return False;
851     if (!check_msg(st,LABEL_MSG2,m,&err)) {
852         slog(st,LOG_SEC,"msg2: %s",err);
853         return False;
854     }
855     st->setup_session_id=m->source;
856     st->remote_capabilities=m->remote_capabilities;
857
858     /* Select the transform to use */
859
860     uint32_t remote_crypto_caps = st->remote_capabilities & CAPAB_TRANSFORM_MASK;
861     if (!remote_crypto_caps)
862         /* old secnets only had this one transform */
863         remote_crypto_caps = 1UL << CAPAB_BIT_ANCIENTTRANSFORM;
864
865 #define CHOOSE_CRYPTO(kind, whats) do {                                 \
866     struct kind##_if *iface;                                            \
867     uint32_t bit, ours = 0;                                             \
868     int i;                                                              \
869     for (i= 0; i < st->n##kind##s; i++) {                               \
870         iface=st->kind##s[i];                                           \
871         bit = 1UL << iface->capab_bit;                                  \
872         if (bit & remote_crypto_caps) goto kind##_found;                \
873         ours |= bit;                                                    \
874     }                                                                   \
875     slog(st,LOG_ERROR,"no " whats " in common"                          \
876          " (us %#"PRIx32"; them: %#"PRIx32")",                          \
877          st->local_capabilities & ours, remote_crypto_caps);            \
878     return False;                                                       \
879 kind##_found:                                                           \
880     st->chosen_##kind = iface;                                          \
881 } while (0)
882
883     CHOOSE_CRYPTO(transform, "transforms");
884
885 #undef CHOOSE_CRYPTO
886
887     memcpy(st->remoteN,m->nR,NONCELEN);
888     return True;
889 }
890
891 static bool_t generate_msg3(struct site *st, const struct msg *prompt)
892 {
893     /* Now we have our nonce and their nonce. Think of a secret key,
894        and create message number 3. */
895     st->random->generate(st->random->st,st->dh->len,st->dhsecret);
896     return generate_msg(st,
897                         (st->remote_capabilities & CAPAB_TRANSFORM_MASK)
898                         ? LABEL_MSG3BIS
899                         : LABEL_MSG3,
900                         "site:MSG3",prompt);
901 }
902
903 static bool_t process_msg3_msg4(struct site *st, struct msg *m)
904 {
905     /* Check signature and store g^x mod m */
906     if (!st->pubkey->check(st->pubkey->st,
907                            m->hashstart,m->hashlen,
908                            &m->sig)) {
909         slog(st,LOG_SEC,"msg3/msg4 signature failed check!");
910         return False;
911     }
912
913     st->remote_adv_mtu=m->remote_mtu;
914
915     return True;
916 }
917
918 static bool_t process_msg3(struct site *st, struct buffer_if *msg3,
919                            const struct comm_addr *src, uint32_t msgtype,
920                            struct msg *m /* returned */)
921 {
922     cstring_t err;
923
924     switch (msgtype) {
925         case CASES_MSG3_KNOWN: break;
926         default: assert(0);
927     }
928
929     if (!unpick_msg(st,msgtype,msg3,m)) return False;
930     if (!check_msg(st,msgtype,m,&err)) {
931         slog(st,LOG_SEC,"msg3: %s",err);
932         return False;
933     }
934     uint32_t capab_adv_late = m->remote_capabilities
935         & ~st->remote_capabilities & st->early_capabilities;
936     if (capab_adv_late) {
937         slog(st,LOG_SEC,"msg3 impermissibly adds early capability flag(s)"
938              " %#"PRIx32" (was %#"PRIx32", now %#"PRIx32")",
939              capab_adv_late, st->remote_capabilities, m->remote_capabilities);
940         return False;
941     }
942
943 #define CHOSE_CRYPTO(kind, what) do {                                   \
944     struct kind##_if *iface;                                            \
945     int i;                                                              \
946     for (i=0; i<st->n##kind##s; i++) {                                  \
947         iface=st->kind##s[i];                                           \
948         if (iface->capab_bit == m->capab_##kind##num)                   \
949             goto kind##_found;                                          \
950     }                                                                   \
951     slog(st,LOG_SEC,"peer chose unknown-to-us " what " %d!",            \
952          m->capab_##kind##num);                                                 \
953     return False;                                                       \
954 kind##_found:                                                           \
955     st->chosen_##kind=iface;                                            \
956 } while (0)
957
958     CHOSE_CRYPTO(transform, "transform");
959
960 #undef CHOSE_CRYPTO
961
962     if (!process_msg3_msg4(st,m))
963         return False;
964
965     /* Update our idea of the remote site's capabilities, now that we've
966      * verified that its message was authentic.
967      *
968      * Our previous idea of the remote site's capabilities came from the
969      * unauthenticated MSG1.  We've already checked that this new message
970      * doesn't change any of the bits we relied upon in the past, but it may
971      * also have set additional capability bits.  We simply throw those away
972      * now, and use the authentic capabilities from this MSG3. */
973     st->remote_capabilities=m->remote_capabilities;
974
975     /* Terminate their DH public key with a '0' */
976     m->pk[m->pklen]=0;
977     /* Invent our DH secret key */
978     st->random->generate(st->random->st,st->dh->len,st->dhsecret);
979
980     /* Generate the shared key and set up the transform */
981     if (!set_new_transform(st,m->pk)) return False;
982
983     return True;
984 }
985
986 static bool_t generate_msg4(struct site *st, const struct msg *prompt)
987 {
988     /* We have both nonces, their public key and our private key. Generate
989        our public key, sign it and send it to them. */
990     return generate_msg(st,LABEL_MSG4,"site:MSG4",prompt);
991 }
992
993 static bool_t process_msg4(struct site *st, struct buffer_if *msg4,
994                            const struct comm_addr *src,
995                            struct msg *m /* returned */)
996 {
997     cstring_t err;
998
999     if (!unpick_msg(st,LABEL_MSG4,msg4,m)) return False;
1000     if (!check_msg(st,LABEL_MSG4,m,&err)) {
1001         slog(st,LOG_SEC,"msg4: %s",err);
1002         return False;
1003     }
1004     
1005     if (!process_msg3_msg4(st,m))
1006         return False;
1007
1008     /* Terminate their DH public key with a '0' */
1009     m->pk[m->pklen]=0;
1010
1011     /* Generate the shared key and set up the transform */
1012     if (!set_new_transform(st,m->pk)) return False;
1013
1014     return True;
1015 }
1016
1017 struct msg0 {
1018     uint32_t dest;
1019     uint32_t source;
1020     uint32_t type;
1021 };
1022
1023 static bool_t unpick_msg0(struct site *st, struct buffer_if *msg0,
1024                           struct msg0 *m)
1025 {
1026     CHECK_AVAIL(msg0,4);
1027     m->dest=buf_unprepend_uint32(msg0);
1028     CHECK_AVAIL(msg0,4);
1029     m->source=buf_unprepend_uint32(msg0);
1030     CHECK_AVAIL(msg0,4);
1031     m->type=buf_unprepend_uint32(msg0);
1032     return True;
1033     /* Leaves transformed part of buffer untouched */
1034 }
1035
1036 static bool_t generate_msg5(struct site *st, const struct msg *prompt)
1037 {
1038     cstring_t transform_err;
1039
1040     BUF_ALLOC(&st->buffer,"site:MSG5");
1041     /* We are going to add four words to the message */
1042     buffer_init(&st->buffer,calculate_max_start_pad());
1043     /* Give the netlink code an opportunity to put its own stuff in the
1044        message (configuration information, etc.) */
1045     buf_prepend_uint32(&st->buffer,LABEL_MSG5);
1046     if (call_transform_forwards(st,st->new_transform,
1047                                 &st->buffer,&transform_err))
1048         return False;
1049     buf_prepend_uint32(&st->buffer,LABEL_MSG5);
1050     buf_prepend_uint32(&st->buffer,st->index);
1051     buf_prepend_uint32(&st->buffer,st->setup_session_id);
1052
1053     st->retries=st->setup_retries;
1054     return True;
1055 }
1056
1057 static bool_t process_msg5(struct site *st, struct buffer_if *msg5,
1058                            const struct comm_addr *src,
1059                            struct transform_inst_if *transform)
1060 {
1061     struct msg0 m;
1062     cstring_t transform_err;
1063
1064     if (!unpick_msg0(st,msg5,&m)) return False;
1065
1066     if (call_transform_reverse(st,transform,msg5,&transform_err)) {
1067         /* There's a problem */
1068         slog(st,LOG_SEC,"process_msg5: transform: %s",transform_err);
1069         return False;
1070     }
1071     /* Buffer should now contain untransformed PING packet data */
1072     CHECK_AVAIL(msg5,4);
1073     if (buf_unprepend_uint32(msg5)!=LABEL_MSG5) {
1074         slog(st,LOG_SEC,"MSG5/PING packet contained wrong label");
1075         return False;
1076     }
1077     /* Older versions of secnet used to write some config data here
1078      * which we ignore.  So we don't CHECK_EMPTY */
1079     return True;
1080 }
1081
1082 static void create_msg6(struct site *st, struct transform_inst_if *transform,
1083                         uint32_t session_id)
1084 {
1085     cstring_t transform_err;
1086
1087     BUF_ALLOC(&st->buffer,"site:MSG6");
1088     /* We are going to add four words to the message */
1089     buffer_init(&st->buffer,calculate_max_start_pad());
1090     /* Give the netlink code an opportunity to put its own stuff in the
1091        message (configuration information, etc.) */
1092     buf_prepend_uint32(&st->buffer,LABEL_MSG6);
1093     transform_apply_return problem =
1094         call_transform_forwards(st,transform,
1095                                 &st->buffer,&transform_err);
1096     assert(!problem);
1097     buf_prepend_uint32(&st->buffer,LABEL_MSG6);
1098     buf_prepend_uint32(&st->buffer,st->index);
1099     buf_prepend_uint32(&st->buffer,session_id);
1100 }
1101
1102 static bool_t generate_msg6(struct site *st, const struct msg *prompt)
1103 {
1104     if (!is_transform_valid(st->new_transform))
1105         return False;
1106     create_msg6(st,st->new_transform,st->setup_session_id);
1107     st->retries=1; /* Peer will retransmit MSG5 if this packet gets lost */
1108     return True;
1109 }
1110
1111 static bool_t process_msg6(struct site *st, struct buffer_if *msg6,
1112                            const struct comm_addr *src)
1113 {
1114     struct msg0 m;
1115     cstring_t transform_err;
1116
1117     if (!unpick_msg0(st,msg6,&m)) return False;
1118
1119     if (call_transform_reverse(st,st->new_transform,msg6,&transform_err)) {
1120         /* There's a problem */
1121         slog(st,LOG_SEC,"process_msg6: transform: %s",transform_err);
1122         return False;
1123     }
1124     /* Buffer should now contain untransformed PING packet data */
1125     CHECK_AVAIL(msg6,4);
1126     if (buf_unprepend_uint32(msg6)!=LABEL_MSG6) {
1127         slog(st,LOG_SEC,"MSG6/PONG packet contained invalid data");
1128         return False;
1129     }
1130     /* Older versions of secnet used to write some config data here
1131      * which we ignore.  So we don't CHECK_EMPTY */
1132     return True;
1133 }
1134
1135 static transform_apply_return
1136 decrypt_msg0(struct site *st, struct buffer_if *msg0,
1137                            const struct comm_addr *src)
1138 {
1139     cstring_t transform_err, auxkey_err, newkey_err="n/a";
1140     struct msg0 m;
1141     transform_apply_return problem;
1142
1143     if (!unpick_msg0(st,msg0,&m)) return False;
1144
1145     /* Keep a copy so we can try decrypting it with multiple keys */
1146     buffer_copy(&st->scratch, msg0);
1147
1148     problem = call_transform_reverse(st,st->current.transform,
1149                                      msg0,&transform_err);
1150     if (!problem) {
1151         if (!st->auxiliary_is_new)
1152             delete_one_key(st,&st->auxiliary_key,
1153                            "peer has used new key","auxiliary key",LOG_SEC);
1154         return 0;
1155     }
1156     if (transform_apply_return_badseq(problem))
1157         goto badseq;
1158
1159     buffer_copy(msg0, &st->scratch);
1160     problem = call_transform_reverse(st,st->auxiliary_key.transform,
1161                                      msg0,&auxkey_err);
1162     if (!problem) {
1163         slog(st,LOG_DROP,"processing packet which uses auxiliary key");
1164         if (st->auxiliary_is_new) {
1165             /* We previously timed out in state SENTMSG5 but it turns
1166              * out that our peer did in fact get our MSG5 and is
1167              * using the new key.  So we should switch to it too. */
1168             /* This is a bit like activate_new_key. */
1169             struct data_key t;
1170             t=st->current;
1171             st->current=st->auxiliary_key;
1172             st->auxiliary_key=t;
1173
1174             delete_one_key(st,&st->auxiliary_key,"peer has used new key",
1175                            "previous key",LOG_SEC);
1176             st->auxiliary_is_new=0;
1177             st->renegotiate_key_time=st->auxiliary_renegotiate_key_time;
1178         }
1179         return 0;
1180     }
1181     if (transform_apply_return_badseq(problem))
1182         goto badseq;
1183
1184     if (st->state==SITE_SENTMSG5) {
1185         buffer_copy(msg0, &st->scratch);
1186         problem = call_transform_reverse(st,st->new_transform,
1187                                          msg0,&newkey_err);
1188         if (!problem) {
1189             /* It looks like we didn't get the peer's MSG6 */
1190             /* This is like a cut-down enter_new_state(SITE_RUN) */
1191             slog(st,LOG_STATE,"will enter state RUN (MSG0 with new key)");
1192             BUF_FREE(&st->buffer);
1193             st->timeout=0;
1194             activate_new_key(st);
1195             return 0; /* do process the data in this packet */
1196         }
1197         if (transform_apply_return_badseq(problem))
1198             goto badseq;
1199     }
1200
1201     slog(st,LOG_SEC,"transform: %s (aux: %s, new: %s)",
1202          transform_err,auxkey_err,newkey_err);
1203     initiate_key_setup(st,"incoming message would not decrypt",0);
1204     send_nak(src,m.dest,m.source,m.type,msg0,"message would not decrypt");
1205     assert(problem);
1206     return problem;
1207
1208  badseq:
1209     slog(st,LOG_DROP,"transform: %s (bad seq.)",transform_err);
1210     assert(problem);
1211     return problem;
1212 }
1213
1214 static bool_t process_msg0(struct site *st, struct buffer_if *msg0,
1215                            const struct comm_addr *src)
1216 {
1217     uint32_t type;
1218     transform_apply_return problem;
1219
1220     problem = decrypt_msg0(st,msg0,src);
1221     if (problem==transform_apply_seqdupe) {
1222         /* We recently received another copy of this packet, maybe due
1223          * to polypath.  That's not a problem; indeed, for the
1224          * purposes of transport address management it is a success.
1225          * But we don't want to process the packet. */
1226         transport_data_msgok(st,src);
1227         return False;
1228     }
1229     if (problem)
1230         return False;
1231
1232     CHECK_AVAIL(msg0,4);
1233     type=buf_unprepend_uint32(msg0);
1234     switch(type) {
1235     case LABEL_MSG7:
1236         /* We must forget about the current session. */
1237         delete_keys(st,"request from peer",LOG_SEC);
1238         /* probably, the peer is shutting down, and this is going to fail,
1239          * but we need to be trying to bring the link up again */
1240         if (st->keepalive)
1241             initiate_key_setup(st,"peer requested key teardown",0);
1242         return True;
1243     case LABEL_MSG9:
1244         /* Deliver to netlink layer */
1245         st->netlink->deliver(st->netlink->st,msg0);
1246         transport_data_msgok(st,src);
1247         /* See whether we should start negotiating a new key */
1248         if (st->now > st->renegotiate_key_time)
1249             initiate_key_setup(st,"incoming packet in renegotiation window",0);
1250         return True;
1251     default:
1252         slog(st,LOG_SEC,"incoming encrypted message of type %08x "
1253              "(unknown)",type);
1254         break;
1255     }
1256     return False;
1257 }
1258
1259 static void dump_packet(struct site *st, struct buffer_if *buf,
1260                         const struct comm_addr *addr, bool_t incoming,
1261                         bool_t ok)
1262 {
1263     uint32_t dest=get_uint32(buf->start);
1264     uint32_t source=get_uint32(buf->start+4);
1265     uint32_t msgtype=get_uint32(buf->start+8);
1266
1267     if (st->log_events & LOG_DUMP)
1268         slilog(st->log,M_DEBUG,"%s: %s: %08x<-%08x: %08x: %s%s",
1269                st->tunname,incoming?"incoming":"outgoing",
1270                dest,source,msgtype,comm_addr_to_string(addr),
1271                ok?"":" - fail");
1272 }
1273
1274 static bool_t comm_addr_sendmsg(struct site *st,
1275                                 const struct comm_addr *dest,
1276                                 struct buffer_if *buf)
1277 {
1278     int i;
1279     struct comm_clientinfo *commclientinfo = 0;
1280
1281     for (i=0; i < st->ncomms; i++) {
1282         if (st->comms[i] == dest->comm) {
1283             commclientinfo = st->commclientinfos[i];
1284             break;
1285         }
1286     }
1287     return dest->comm->sendmsg(dest->comm->st, buf, dest, commclientinfo);
1288 }
1289
1290 static uint32_t site_status(void *st)
1291 {
1292     return 0;
1293 }
1294
1295 static bool_t send_msg(struct site *st)
1296 {
1297     if (st->retries>0) {
1298         transport_xmit(st, &st->setup_peers, &st->buffer, True);
1299         st->timeout=st->now+st->setup_retry_interval;
1300         st->retries--;
1301         return True;
1302     } else if (st->state==SITE_SENTMSG5) {
1303         logtimeout(st,"timed out sending MSG5, stashing new key");
1304         /* We stash the key we have produced, in case it turns out that
1305          * our peer did see our MSG5 after all and starts using it. */
1306         /* This is a bit like some of activate_new_key */
1307         struct transform_inst_if *t;
1308         t=st->auxiliary_key.transform;
1309         st->auxiliary_key.transform=st->new_transform;
1310         st->new_transform=t;
1311         dispose_transform(&st->new_transform);
1312
1313         st->auxiliary_is_new=1;
1314         st->auxiliary_key.key_timeout=st->now+st->key_lifetime;
1315         st->auxiliary_renegotiate_key_time=st->now+st->key_renegotiate_time;
1316         st->auxiliary_key.remote_session_id=st->setup_session_id;
1317
1318         enter_state_wait(st);
1319         return False;
1320     } else {
1321         logtimeout(st,"timed out sending key setup packet "
1322             "(in state %s)",state_name(st->state));
1323         enter_state_wait(st);
1324         return False;
1325     }
1326 }
1327
1328 static void site_resolve_callback(void *sst, const struct comm_addr *addrs,
1329                                   int stored_naddrs, int all_naddrs,
1330                                   const char *address, const char *failwhy)
1331 {
1332     struct site *st=sst;
1333
1334     if (!stored_naddrs) {
1335         slog(st,LOG_ERROR,"resolution of %s failed: %s",address,failwhy);
1336     } else {
1337         slog(st,LOG_PEER_ADDRS,"resolution of %s completed, %d addrs, eg: %s",
1338              address, all_naddrs, comm_addr_to_string(&addrs[0]));;
1339
1340         int space=st->transport_peers_max-st->resolving_n_results_stored;
1341         int n_tocopy=MIN(stored_naddrs,space);
1342         COPY_ARRAY(st->resolving_results + st->resolving_n_results_stored,
1343                    addrs,
1344                    n_tocopy);
1345         st->resolving_n_results_stored += n_tocopy;
1346         st->resolving_n_results_all += all_naddrs;
1347     }
1348
1349     decrement_resolving_count(st,1);
1350 }
1351
1352 static void decrement_resolving_count(struct site *st, int by)
1353 {
1354     assert(st->resolving_count>0);
1355     st->resolving_count-=by;
1356
1357     if (st->resolving_count)
1358         return;
1359
1360     /* OK, we are done with them all.  Handle combined results. */
1361
1362     const struct comm_addr *addrs=st->resolving_results;
1363     int naddrs=st->resolving_n_results_stored;
1364     assert(naddrs<=st->transport_peers_max);
1365
1366     if (naddrs) {
1367         if (naddrs != st->resolving_n_results_all) {
1368             slog(st,LOG_SETUP_INIT,"resolution of supplied addresses/names"
1369                  " yielded too many results (%d > %d), some ignored",
1370                  st->resolving_n_results_all, naddrs);
1371         }
1372         slog(st,LOG_STATE,"resolution completed, %d addrs, eg: %s",
1373              naddrs, iaddr_to_string(&addrs[0].ia));;
1374     }
1375
1376     switch (st->state) {
1377     case SITE_RESOLVE:
1378         if (transport_compute_setupinit_peers(st,addrs,naddrs,0)) {
1379             enter_new_state(st,SITE_SENTMSG1,0);
1380         } else {
1381             /* Can't figure out who to try to to talk to */
1382             slog(st,LOG_SETUP_INIT,
1383                  "key exchange failed: cannot find peer address");
1384             enter_state_run(st);
1385         }
1386         break;
1387     case SITE_SENTMSG1: case SITE_SENTMSG2:
1388     case SITE_SENTMSG3: case SITE_SENTMSG4:
1389     case SITE_SENTMSG5:
1390         if (naddrs) {
1391             /* We start using the address immediately for data too.
1392              * It's best to store it in st->peers now because we might
1393              * go via SENTMSG5, WAIT, and a MSG0, straight into using
1394              * the new key (without updating the data peer addrs). */
1395             transport_resolve_complete(st,addrs,naddrs);
1396         } else if (st->local_mobile) {
1397             /* We can't let this rest because we may have a peer
1398              * address which will break in the future. */
1399             slog(st,LOG_SETUP_INIT,"resolution failed: "
1400                  "abandoning key exchange");
1401             enter_state_wait(st);
1402         } else {
1403             slog(st,LOG_SETUP_INIT,"resolution failed: "
1404                  " continuing to use source address of peer's packets"
1405                  " for key exchange and ultimately data");
1406         }
1407         break;
1408     case SITE_RUN:
1409         if (naddrs) {
1410             slog(st,LOG_SETUP_INIT,"resolution completed tardily,"
1411                  " updating peer address(es)");
1412             transport_resolve_complete_tardy(st,addrs,naddrs);
1413         } else if (st->local_mobile) {
1414             /* Not very good.  We should queue (another) renegotiation
1415              * so that we can update the peer address. */
1416             st->key_renegotiate_time=st->now+wait_timeout(st);
1417         } else {
1418             slog(st,LOG_SETUP_INIT,"resolution failed: "
1419                  " continuing to use source address of peer's packets");
1420         }
1421         break;
1422     case SITE_WAIT:
1423     case SITE_STOP:
1424         /* oh well */
1425         break;
1426     }
1427 }
1428
1429 static bool_t initiate_key_setup(struct site *st, cstring_t reason,
1430                                  const struct comm_addr *prod_hint)
1431 {
1432     /* Reentrancy hazard: can call enter_new_state/enter_state_* */
1433     if (st->state!=SITE_RUN) return False;
1434     slog(st,LOG_SETUP_INIT,"initiating key exchange (%s)",reason);
1435     if (st->addresses) {
1436         slog(st,LOG_SETUP_INIT,"resolving peer address(es)");
1437         return enter_state_resolve(st);
1438     } else if (transport_compute_setupinit_peers(st,0,0,prod_hint)) {
1439         return enter_new_state(st,SITE_SENTMSG1,0);
1440     }
1441     slog(st,LOG_SETUP_INIT,"key exchange failed: no address for peer");
1442     return False;
1443 }
1444
1445 static void activate_new_key(struct site *st)
1446 {
1447     struct transform_inst_if *t;
1448
1449     /* We have three transform instances, which we swap between old,
1450        active and setup */
1451     t=st->auxiliary_key.transform;
1452     st->auxiliary_key.transform=st->current.transform;
1453     st->current.transform=st->new_transform;
1454     st->new_transform=t;
1455     dispose_transform(&st->new_transform);
1456
1457     st->timeout=0;
1458     st->auxiliary_is_new=0;
1459     st->auxiliary_key.key_timeout=st->current.key_timeout;
1460     st->current.key_timeout=st->now+st->key_lifetime;
1461     st->renegotiate_key_time=st->now+st->key_renegotiate_time;
1462     transport_peers_copy(st,&st->peers,&st->setup_peers);
1463     st->current.remote_session_id=st->setup_session_id;
1464
1465     /* Compute the inter-site MTU.  This is min( our_mtu, their_mtu ).
1466      * But their mtu be unspecified, in which case we just use ours. */
1467     uint32_t intersite_mtu=
1468         MIN(st->mtu_target, st->remote_adv_mtu ?: ~(uint32_t)0);
1469     st->netlink->set_mtu(st->netlink->st,intersite_mtu);
1470
1471     slog(st,LOG_ACTIVATE_KEY,"new key activated"
1472          " (mtu ours=%"PRId32" theirs=%"PRId32" intersite=%"PRId32")",
1473          st->mtu_target, st->remote_adv_mtu, intersite_mtu);
1474     enter_state_run(st);
1475 }
1476
1477 static void delete_one_key(struct site *st, struct data_key *key,
1478                            cstring_t reason, cstring_t which, uint32_t loglevel)
1479 {
1480     if (!is_transform_valid(key->transform)) return;
1481     if (reason) slog(st,loglevel,"%s deleted (%s)",which,reason);
1482     dispose_transform(&key->transform);
1483     key->key_timeout=0;
1484 }
1485
1486 static void delete_keys(struct site *st, cstring_t reason, uint32_t loglevel)
1487 {
1488     if (current_valid(st)) {
1489         slog(st,loglevel,"session closed (%s)",reason);
1490
1491         delete_one_key(st,&st->current,0,0,0);
1492         set_link_quality(st);
1493     }
1494     delete_one_key(st,&st->auxiliary_key,0,0,0);
1495 }
1496
1497 static void state_assert(struct site *st, bool_t ok)
1498 {
1499     if (!ok) fatal("site:state_assert");
1500 }
1501
1502 static void enter_state_stop(struct site *st)
1503 {
1504     st->state=SITE_STOP;
1505     st->timeout=0;
1506     delete_keys(st,"entering state STOP",LOG_TIMEOUT_KEY);
1507     dispose_transform(&st->new_transform);
1508 }
1509
1510 static void set_link_quality(struct site *st)
1511 {
1512     uint32_t quality;
1513     if (current_valid(st))
1514         quality=LINK_QUALITY_UP;
1515     else if (st->state==SITE_WAIT || st->state==SITE_STOP)
1516         quality=LINK_QUALITY_DOWN;
1517     else if (st->addresses)
1518         quality=LINK_QUALITY_DOWN_CURRENT_ADDRESS;
1519     else if (transport_peers_valid(&st->peers))
1520         quality=LINK_QUALITY_DOWN_STALE_ADDRESS;
1521     else
1522         quality=LINK_QUALITY_DOWN;
1523
1524     st->netlink->set_quality(st->netlink->st,quality);
1525 }
1526
1527 static void enter_state_run(struct site *st)
1528 {
1529     slog(st,LOG_STATE,"entering state RUN%s",
1530          current_valid(st) ? " (keyed)" : " (unkeyed)");
1531     st->state=SITE_RUN;
1532     st->timeout=0;
1533
1534     st->setup_session_id=0;
1535     transport_peers_clear(st,&st->setup_peers);
1536     FILLZERO(st->localN);
1537     FILLZERO(st->remoteN);
1538     dispose_transform(&st->new_transform);
1539     memset(st->dhsecret,0,st->dh->len);
1540     if (st->sharedsecret) memset(st->sharedsecret,0,st->sharedsecretlen);
1541     set_link_quality(st);
1542
1543     if (st->keepalive && !current_valid(st))
1544         initiate_key_setup(st, "keepalive", 0);
1545 }
1546
1547 static bool_t ensure_resolving(struct site *st)
1548 {
1549     /* Reentrancy hazard: may call site_resolve_callback and hence
1550      * enter_new_state, enter_state_* and generate_msg*. */
1551     if (st->resolving_count)
1552         return True;
1553
1554     assert(st->addresses);
1555
1556     /* resolver->request might reentrantly call site_resolve_callback
1557      * which will decrement st->resolving, so we need to increment it
1558      * twice beforehand to prevent decrement from thinking we're
1559      * finished, and decrement it ourselves.  Alternatively if
1560      * everything fails then there are no callbacks due and we simply
1561      * set it to 0 and return false.. */
1562     st->resolving_n_results_stored=0;
1563     st->resolving_n_results_all=0;
1564     st->resolving_count+=2;
1565     const char **addrp=st->addresses;
1566     const char *address;
1567     bool_t anyok=False;
1568     for (; (address=*addrp++); ) {
1569         bool_t ok = st->resolver->request(st->resolver->st,address,
1570                                           st->remoteport,st->comms[0],
1571                                           site_resolve_callback,st);
1572         if (ok)
1573             st->resolving_count++;
1574         anyok|=ok;
1575     }
1576     if (!anyok) {
1577         st->resolving_count=0;
1578         return False;
1579     }
1580     decrement_resolving_count(st,2);
1581     return True;
1582 }
1583
1584 static bool_t enter_state_resolve(struct site *st)
1585 {
1586     /* Reentrancy hazard!  See ensure_resolving. */
1587     state_assert(st,st->state==SITE_RUN);
1588     slog(st,LOG_STATE,"entering state RESOLVE");
1589     st->state=SITE_RESOLVE;
1590     return ensure_resolving(st);
1591 }
1592
1593 static bool_t enter_new_state(struct site *st, uint32_t next,
1594                               const struct msg *prompt
1595                               /* may be 0 for SENTMSG1 */)
1596 {
1597     bool_t (*gen)(struct site *st, const struct msg *prompt);
1598     int r;
1599
1600     slog(st,LOG_STATE,"entering state %s",state_name(next));
1601     switch(next) {
1602     case SITE_SENTMSG1:
1603         state_assert(st,st->state==SITE_RUN || st->state==SITE_RESOLVE);
1604         gen=generate_msg1;
1605         st->msg1_crossed_logged = False;
1606         break;
1607     case SITE_SENTMSG2:
1608         state_assert(st,st->state==SITE_RUN || st->state==SITE_RESOLVE ||
1609                      st->state==SITE_SENTMSG1 || st->state==SITE_WAIT);
1610         gen=generate_msg2;
1611         break;
1612     case SITE_SENTMSG3:
1613         state_assert(st,st->state==SITE_SENTMSG1);
1614         BUF_FREE(&st->buffer);
1615         gen=generate_msg3;
1616         break;
1617     case SITE_SENTMSG4:
1618         state_assert(st,st->state==SITE_SENTMSG2);
1619         BUF_FREE(&st->buffer);
1620         gen=generate_msg4;
1621         break;
1622     case SITE_SENTMSG5:
1623         state_assert(st,st->state==SITE_SENTMSG3);
1624         BUF_FREE(&st->buffer);
1625         gen=generate_msg5;
1626         break;
1627     case SITE_RUN:
1628         state_assert(st,st->state==SITE_SENTMSG4);
1629         BUF_FREE(&st->buffer);
1630         gen=generate_msg6;
1631         break;
1632     default:
1633         gen=NULL;
1634         fatal("enter_new_state(%s): invalid new state",state_name(next));
1635         break;
1636     }
1637
1638     if (hacky_par_start_failnow()) return False;
1639
1640     r= gen(st,prompt) && send_msg(st);
1641
1642     hacky_par_end(&r,
1643                   st->setup_retries, st->setup_retry_interval,
1644                   send_msg, st);
1645     
1646     if (r) {
1647         st->state=next;
1648         if (next==SITE_RUN) {
1649             BUF_FREE(&st->buffer); /* Never reused */
1650             st->timeout=0; /* Never retransmit */
1651             activate_new_key(st);
1652         }
1653         return True;
1654     }
1655     slog(st,LOG_ERROR,"error entering state %s",state_name(next));
1656     st->buffer.free=False; /* Unconditionally use the buffer; it may be
1657                               in either state, and enter_state_wait() will
1658                               do a BUF_FREE() */
1659     enter_state_wait(st);
1660     return False;
1661 }
1662
1663 /* msg7 tells our peer that we're about to forget our key */
1664 static bool_t send_msg7(struct site *st, cstring_t reason)
1665 {
1666     cstring_t transform_err;
1667
1668     if (current_valid(st) && st->buffer.free
1669         && transport_peers_valid(&st->peers)) {
1670         BUF_ALLOC(&st->buffer,"site:MSG7");
1671         buffer_init(&st->buffer,calculate_max_start_pad());
1672         buf_append_uint32(&st->buffer,LABEL_MSG7);
1673         buf_append_string(&st->buffer,reason);
1674         if (call_transform_forwards(st, st->current.transform,
1675                                     &st->buffer, &transform_err))
1676             goto free_out;
1677         buf_prepend_uint32(&st->buffer,LABEL_MSG0);
1678         buf_prepend_uint32(&st->buffer,st->index);
1679         buf_prepend_uint32(&st->buffer,st->current.remote_session_id);
1680         transport_xmit(st,&st->peers,&st->buffer,True);
1681         BUF_FREE(&st->buffer);
1682     free_out:
1683         return True;
1684     }
1685     return False;
1686 }
1687
1688 /* We go into this state if our peer becomes uncommunicative. Similar to
1689    the "stop" state, we forget all session keys for a while, before
1690    re-entering the "run" state. */
1691 static void enter_state_wait(struct site *st)
1692 {
1693     slog(st,LOG_STATE,"entering state WAIT");
1694     st->timeout=st->now+wait_timeout(st);
1695     st->state=SITE_WAIT;
1696     set_link_quality(st);
1697     BUF_FREE(&st->buffer); /* will have had an outgoing packet in it */
1698     /* XXX Erase keys etc. */
1699 }
1700
1701 static void generate_prod(struct site *st, struct buffer_if *buf)
1702 {
1703     buffer_init(buf,0);
1704     buf_append_uint32(buf,0);
1705     buf_append_uint32(buf,0);
1706     buf_append_uint32(buf,LABEL_PROD);
1707     buf_append_string(buf,st->localname);
1708     buf_append_string(buf,st->remotename);
1709 }
1710
1711 static void generate_send_prod(struct site *st,
1712                                const struct comm_addr *source)
1713 {
1714     if (!st->allow_send_prod) return; /* too soon */
1715     if (!(st->state==SITE_RUN || st->state==SITE_RESOLVE ||
1716           st->state==SITE_WAIT)) return; /* we'd ignore peer's MSG1 */
1717
1718     slog(st,LOG_SETUP_INIT,"prodding peer for key exchange");
1719     st->allow_send_prod=0;
1720     generate_prod(st,&st->scratch);
1721     bool_t ok = comm_addr_sendmsg(st, source, &st->scratch);
1722     dump_packet(st,&st->scratch,source,False,ok);
1723 }
1724
1725 static inline void site_settimeout(uint64_t timeout, int *timeout_io)
1726 {
1727     if (timeout) {
1728         int64_t offset=timeout-*now;
1729         if (offset<0) offset=0;
1730         if (offset>INT_MAX) offset=INT_MAX;
1731         if (*timeout_io<0 || offset<*timeout_io)
1732             *timeout_io=offset;
1733     }
1734 }
1735
1736 static int site_beforepoll(void *sst, struct pollfd *fds, int *nfds_io,
1737                            int *timeout_io)
1738 {
1739     struct site *st=sst;
1740
1741     BEFOREPOLL_WANT_FDS(0); /* We don't use any file descriptors */
1742     st->now=*now;
1743
1744     /* Work out when our next timeout is. The earlier of 'timeout' or
1745        'current.key_timeout'. A stored value of '0' indicates no timeout
1746        active. */
1747     site_settimeout(st->timeout, timeout_io);
1748     site_settimeout(st->current.key_timeout, timeout_io);
1749     site_settimeout(st->auxiliary_key.key_timeout, timeout_io);
1750
1751     return 0; /* success */
1752 }
1753
1754 static void check_expiry(struct site *st, struct data_key *key,
1755                          const char *which)
1756 {
1757     if (key->key_timeout && *now>key->key_timeout) {
1758         delete_one_key(st,key,"maximum life exceeded",which,LOG_TIMEOUT_KEY);
1759     }
1760 }
1761
1762 /* NB site_afterpoll will be called before site_beforepoll is ever called */
1763 static void site_afterpoll(void *sst, struct pollfd *fds, int nfds)
1764 {
1765     struct site *st=sst;
1766
1767     st->now=*now;
1768     if (st->timeout && *now>st->timeout) {
1769         st->timeout=0;
1770         if (st->state>=SITE_SENTMSG1 && st->state<=SITE_SENTMSG5) {
1771             if (!hacky_par_start_failnow())
1772                 send_msg(st);
1773         } else if (st->state==SITE_WAIT) {
1774             enter_state_run(st);
1775         } else {
1776             slog(st,LOG_ERROR,"site_afterpoll: unexpected timeout, state=%d",
1777                  st->state);
1778         }
1779     }
1780     check_expiry(st,&st->current,"current key");
1781     check_expiry(st,&st->auxiliary_key,"auxiliary key");
1782 }
1783
1784 /* This function is called by the netlink device to deliver packets
1785    intended for the remote network. The packet is in "raw" wire
1786    format, but is guaranteed to be word-aligned. */
1787 static void site_outgoing(void *sst, struct buffer_if *buf)
1788 {
1789     struct site *st=sst;
1790     cstring_t transform_err;
1791     
1792     if (st->state==SITE_STOP) {
1793         BUF_FREE(buf);
1794         return;
1795     }
1796
1797     st->allow_send_prod=1;
1798
1799     /* In all other states we consider delivering the packet if we have
1800        a valid key and a valid address to send it to. */
1801     if (current_valid(st) && transport_peers_valid(&st->peers)) {
1802         /* Transform it and send it */
1803         if (buf->size>0) {
1804             buf_prepend_uint32(buf,LABEL_MSG9);
1805             if (call_transform_forwards(st, st->current.transform,
1806                                         buf, &transform_err))
1807                 goto free_out;
1808             buf_prepend_uint32(buf,LABEL_MSG0);
1809             buf_prepend_uint32(buf,st->index);
1810             buf_prepend_uint32(buf,st->current.remote_session_id);
1811             transport_xmit(st,&st->peers,buf,False);
1812         }
1813     free_out:
1814         BUF_FREE(buf);
1815         return;
1816     }
1817
1818     slog(st,LOG_DROP,"discarding outgoing packet of size %d",buf->size);
1819     BUF_FREE(buf);
1820     initiate_key_setup(st,"outgoing packet",0);
1821 }
1822
1823 static bool_t named_for_us(struct site *st, const struct buffer_if *buf_in,
1824                            uint32_t type, struct msg *m)
1825     /* For packets which are identified by the local and remote names.
1826      * If it has our name and our peer's name in it it's for us. */
1827 {
1828     struct buffer_if buf[1];
1829     buffer_readonly_clone(buf,buf_in);
1830     return unpick_msg(st,type,buf,m)
1831         && name_matches(&m->remote,st->remotename)
1832         && name_matches(&m->local,st->localname);
1833 }
1834
1835 static bool_t we_have_priority(struct site *st, const struct msg *m) {
1836     if (st->local_capabilities & m->remote_capabilities &
1837         CAPAB_PRIORITY_MOBILE) {
1838         if (st->local_mobile) return True;
1839         if (st-> peer_mobile) return False;
1840     }
1841     return st->our_name_later;
1842 }
1843
1844 static bool_t setup_late_msg_ok(struct site *st, 
1845                                 const struct buffer_if *buf_in,
1846                                 uint32_t msgtype,
1847                                 const struct comm_addr *source,
1848                                 struct msg *m /* returned */) {
1849     /* For setup packets which seem from their type like they are
1850      * late.  Maybe they came via a different path.  All we do is make
1851      * a note of the sending address, iff they look like they are part
1852      * of the current key setup attempt. */
1853     if (!named_for_us(st,buf_in,msgtype,m))
1854         /* named_for_us calls unpick_msg which gets the nonces */
1855         return False;
1856     if (!consttime_memeq(m->nR,st->remoteN,NONCELEN) ||
1857         !consttime_memeq(m->nL,st->localN, NONCELEN))
1858         /* spoof ?  from stale run ?  who knows */
1859         return False;
1860     transport_setup_msgok(st,source);
1861     return True;
1862 }
1863
1864 /* This function is called by the communication device to deliver
1865    packets from our peers.
1866    It should return True if the packet is recognised as being for
1867    this current site instance (and should therefore not be processed
1868    by other sites), even if the packet was otherwise ignored. */
1869 static bool_t site_incoming(void *sst, struct buffer_if *buf,
1870                             const struct comm_addr *source)
1871 {
1872     struct site *st=sst;
1873
1874     if (buf->size < 12) return False;
1875
1876     uint32_t dest=get_uint32(buf->start);
1877     uint32_t msgtype=get_uint32(buf->start+8);
1878     struct msg msg;
1879       /* initialised by named_for_us, or process_msgN for N!=1 */
1880
1881     if (msgtype==LABEL_MSG1) {
1882         if (!named_for_us(st,buf,msgtype,&msg))
1883             return False;
1884         /* It's a MSG1 addressed to us. Decide what to do about it. */
1885         dump_packet(st,buf,source,True,True);
1886         if (st->state==SITE_RUN || st->state==SITE_RESOLVE ||
1887             st->state==SITE_WAIT) {
1888             /* We should definitely process it */
1889             transport_compute_setupinit_peers(st,0,0,source);
1890             if (process_msg1(st,buf,source,&msg)) {
1891                 slog(st,LOG_SETUP_INIT,"key setup initiated by peer");
1892                 bool_t entered=enter_new_state(st,SITE_SENTMSG2,&msg);
1893                 if (entered && st->addresses && st->local_mobile)
1894                     /* We must do this as the very last thing, because
1895                        the resolver callback might reenter us. */
1896                     ensure_resolving(st);
1897             } else {
1898                 slog(st,LOG_ERROR,"failed to process incoming msg1");
1899             }
1900             BUF_FREE(buf);
1901             return True;
1902         } else if (st->state==SITE_SENTMSG1) {
1903             /* We've just sent a message 1! They may have crossed on
1904                the wire. If we have priority then we ignore the
1905                incoming one, otherwise we process it as usual. */
1906             if (we_have_priority(st,&msg)) {
1907                 BUF_FREE(buf);
1908                 if (!st->msg1_crossed_logged++)
1909                     slog(st,LOG_SETUP_INIT,"crossed msg1s; we are higher "
1910                          "priority => ignore incoming msg1");
1911                 return True;
1912             } else {
1913                 slog(st,LOG_SETUP_INIT,"crossed msg1s; we are lower "
1914                      "priority => use incoming msg1");
1915                 if (process_msg1(st,buf,source,&msg)) {
1916                     BUF_FREE(&st->buffer); /* Free our old message 1 */
1917                     transport_setup_msgok(st,source);
1918                     enter_new_state(st,SITE_SENTMSG2,&msg);
1919                 } else {
1920                     slog(st,LOG_ERROR,"failed to process an incoming "
1921                          "crossed msg1 (we have low priority)");
1922                 }
1923                 BUF_FREE(buf);
1924                 return True;
1925             }
1926         } else if (st->state==SITE_SENTMSG2 ||
1927                    st->state==SITE_SENTMSG4) {
1928             if (consttime_memeq(msg.nR,st->remoteN,NONCELEN)) {
1929                 /* We are ahead in the protocol, but that msg1 had the
1930                  * peer's nonce so presumably it is from this key
1931                  * exchange run, via a slower route */
1932                 transport_setup_msgok(st,source);
1933             } else {
1934                 slog(st,LOG_UNEXPECTED,"competing incoming message 1");
1935             }
1936             BUF_FREE(buf);
1937             return True;
1938         }
1939         /* The message 1 was received at an unexpected stage of the
1940            key setup.  Well, they lost the race. */
1941         slog(st,LOG_UNEXPECTED,"unexpected incoming message 1");
1942         BUF_FREE(buf);
1943         return True;
1944     }
1945     if (msgtype==LABEL_PROD) {
1946         if (!named_for_us(st,buf,msgtype,&msg))
1947             return False;
1948         dump_packet(st,buf,source,True,True);
1949         if (st->state!=SITE_RUN) {
1950             slog(st,LOG_DROP,"ignoring PROD when not in state RUN");
1951         } else if (current_valid(st)) {
1952             slog(st,LOG_DROP,"ignoring PROD when we think we have a key");
1953         } else {
1954             initiate_key_setup(st,"peer sent PROD packet",source);
1955         }
1956         BUF_FREE(buf);
1957         return True;
1958     }
1959     if (dest==st->index) {
1960         /* Explicitly addressed to us */
1961         if (msgtype!=LABEL_MSG0) dump_packet(st,buf,source,True,True);
1962         switch (msgtype) {
1963         case LABEL_NAK:
1964             /* If the source is our current peer then initiate a key setup,
1965                because our peer's forgotten the key */
1966             if (get_uint32(buf->start+4)==st->current.remote_session_id) {
1967                 bool_t initiated;
1968                 initiated = initiate_key_setup(st,"received a NAK",source);
1969                 if (!initiated) generate_send_prod(st,source);
1970             } else {
1971                 slog(st,LOG_SEC,"bad incoming NAK");
1972             }
1973             break;
1974         case LABEL_MSG0:
1975             process_msg0(st,buf,source);
1976             break;
1977         case LABEL_MSG1:
1978             /* Setup packet: should not have been explicitly addressed
1979                to us */
1980             slog(st,LOG_SEC,"incoming explicitly addressed msg1");
1981             break;
1982         case LABEL_MSG2:
1983             /* Setup packet: expected only in state SENTMSG1 */
1984             if (st->state!=SITE_SENTMSG1) {
1985                 if ((st->state==SITE_SENTMSG3 ||
1986                      st->state==SITE_SENTMSG5) &&
1987                     setup_late_msg_ok(st,buf,msgtype,source,&msg))
1988                     break;
1989                 slog(st,LOG_UNEXPECTED,"unexpected MSG2");
1990             } else if (process_msg2(st,buf,source,&msg)) {
1991                 transport_setup_msgok(st,source);
1992                 enter_new_state(st,SITE_SENTMSG3,&msg);
1993             } else {
1994                 slog(st,LOG_SEC,"invalid MSG2");
1995             }
1996             break;
1997         case CASES_MSG3_KNOWN:
1998             /* Setup packet: expected only in state SENTMSG2 */
1999             if (st->state!=SITE_SENTMSG2) {
2000                 if ((st->state==SITE_SENTMSG4) &&
2001                     setup_late_msg_ok(st,buf,msgtype,source,&msg))
2002                     break;
2003                 slog(st,LOG_UNEXPECTED,"unexpected MSG3");
2004             } else if (process_msg3(st,buf,source,msgtype,&msg)) {
2005                 transport_setup_msgok(st,source);
2006                 enter_new_state(st,SITE_SENTMSG4,&msg);
2007             } else {
2008                 slog(st,LOG_SEC,"invalid MSG3");
2009             }
2010             break;
2011         case LABEL_MSG4:
2012             /* Setup packet: expected only in state SENTMSG3 */
2013             if (st->state!=SITE_SENTMSG3) {
2014                 if ((st->state==SITE_SENTMSG5) &&
2015                     setup_late_msg_ok(st,buf,msgtype,source,&msg))
2016                     break;
2017                 slog(st,LOG_UNEXPECTED,"unexpected MSG4");
2018             } else if (process_msg4(st,buf,source,&msg)) {
2019                 transport_setup_msgok(st,source);
2020                 enter_new_state(st,SITE_SENTMSG5,&msg);
2021             } else {
2022                 slog(st,LOG_SEC,"invalid MSG4");
2023             }
2024             break;
2025         case LABEL_MSG5:
2026             /* Setup packet: expected only in state SENTMSG4 */
2027             /* (may turn up in state RUN if our return MSG6 was lost
2028                and the new key has already been activated. In that
2029                case we discard it. The peer will realise that we
2030                are using the new key when they see our data packets.
2031                Until then the peer's data packets to us get discarded. */
2032             if (st->state==SITE_SENTMSG4) {
2033                 if (process_msg5(st,buf,source,st->new_transform)) {
2034                     transport_setup_msgok(st,source);
2035                     enter_new_state(st,SITE_RUN,&msg);
2036                 } else {
2037                     slog(st,LOG_SEC,"invalid MSG5");
2038                 }
2039             } else if (st->state==SITE_RUN) {
2040                 if (process_msg5(st,buf,source,st->current.transform)) {
2041                     slog(st,LOG_DROP,"got MSG5, retransmitting MSG6");
2042                     transport_setup_msgok(st,source);
2043                     create_msg6(st,st->current.transform,
2044                                 st->current.remote_session_id);
2045                     transport_xmit(st,&st->peers,&st->buffer,True);
2046                     BUF_FREE(&st->buffer);
2047                 } else {
2048                     slog(st,LOG_SEC,"invalid MSG5 (in state RUN)");
2049                 }
2050             } else {
2051                 slog(st,LOG_UNEXPECTED,"unexpected MSG5");
2052             }
2053             break;
2054         case LABEL_MSG6:
2055             /* Setup packet: expected only in state SENTMSG5 */
2056             if (st->state!=SITE_SENTMSG5) {
2057                 slog(st,LOG_UNEXPECTED,"unexpected MSG6");
2058             } else if (process_msg6(st,buf,source)) {
2059                 BUF_FREE(&st->buffer); /* Free message 5 */
2060                 transport_setup_msgok(st,source);
2061                 activate_new_key(st);
2062             } else {
2063                 slog(st,LOG_SEC,"invalid MSG6");
2064             }
2065             break;
2066         default:
2067             slog(st,LOG_SEC,"received message of unknown type 0x%08x",
2068                  msgtype);
2069             break;
2070         }
2071         BUF_FREE(buf);
2072         return True;
2073     }
2074
2075     return False;
2076 }
2077
2078 static void site_control(void *vst, bool_t run)
2079 {
2080     struct site *st=vst;
2081     if (run) enter_state_run(st);
2082     else enter_state_stop(st);
2083 }
2084
2085 static void site_phase_hook(void *sst, uint32_t newphase)
2086 {
2087     struct site *st=sst;
2088
2089     /* The program is shutting down; tell our peer */
2090     send_msg7(st,"shutting down");
2091 }
2092
2093 static void site_childpersist_clearkeys(void *sst, uint32_t newphase)
2094 {
2095     struct site *st=sst;
2096     dispose_transform(&st->current.transform);
2097     dispose_transform(&st->auxiliary_key.transform);
2098     dispose_transform(&st->new_transform);
2099     /* Not much point overwiting the signing key, since we loaded it
2100        from disk, and it is only valid prospectively if at all,
2101        anyway. */
2102     /* XXX it would be best to overwrite the DH state, because that
2103        _is_ relevant to forward secrecy.  However we have no
2104        convenient interface for doing that and in practice gmp has
2105        probably dribbled droppings all over the malloc arena.  A good
2106        way to fix this would be to have a privsep child for asymmetric
2107        crypto operations, but that's a task for another day. */
2108 }
2109
2110 static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
2111                           list_t *args)
2112 {
2113     static uint32_t index_sequence;
2114     struct site *st;
2115     item_t *item;
2116     dict_t *dict;
2117     int i;
2118
2119     NEW(st);
2120
2121     st->cl.description="site";
2122     st->cl.type=CL_SITE;
2123     st->cl.apply=NULL;
2124     st->cl.interface=&st->ops;
2125     st->ops.st=st;
2126     st->ops.control=site_control;
2127     st->ops.status=site_status;
2128
2129     /* First parameter must be a dict */
2130     item=list_elem(args,0);
2131     if (!item || item->type!=t_dict)
2132         cfgfatal(loc,"site","parameter must be a dictionary\n");
2133     
2134     dict=item->data.dict;
2135     st->localname=dict_read_string(dict, "local-name", True, "site", loc);
2136     st->remotename=dict_read_string(dict, "name", True, "site", loc);
2137
2138     st->keepalive=dict_read_bool(dict,"keepalive",False,"site",loc,False);
2139
2140     st->peer_mobile=dict_read_bool(dict,"mobile",False,"site",loc,False);
2141     st->local_mobile=
2142         dict_read_bool(dict,"local-mobile",False,"site",loc,False);
2143
2144     /* Sanity check (which also allows the 'sites' file to include
2145        site() closures for all sites including our own): refuse to
2146        talk to ourselves */
2147     if (strcmp(st->localname,st->remotename)==0) {
2148         Message(M_DEBUG,"site %s: local-name==name -> ignoring this site\n",
2149                 st->localname);
2150         if (st->peer_mobile != st->local_mobile)
2151             cfgfatal(loc,"site","site %s's peer-mobile=%d"
2152                     " but our local-mobile=%d\n",
2153                     st->localname, st->peer_mobile, st->local_mobile);
2154         free(st);
2155         return NULL;
2156     }
2157     if (st->peer_mobile && st->local_mobile) {
2158         Message(M_WARNING,"site %s: site is mobile but so are we"
2159                 " -> ignoring this site\n", st->remotename);
2160         free(st);
2161         return NULL;
2162     }
2163
2164     assert(index_sequence < 0xffffffffUL);
2165     st->index = ++index_sequence;
2166     st->local_capabilities = 0;
2167     st->early_capabilities = CAPAB_PRIORITY_MOBILE;
2168     st->netlink=find_cl_if(dict,"link",CL_NETLINK,True,"site",loc);
2169
2170 #define GET_CLOSURE_LIST(dictkey,things,nthings,CL_TYPE) do{            \
2171     list_t *things##_cfg=dict_lookup(dict,dictkey);                     \
2172     if (!things##_cfg)                                                  \
2173         cfgfatal(loc,"site","closure list \"%s\" not found\n",dictkey); \
2174     st->nthings=list_length(things##_cfg);                              \
2175     NEW_ARY(st->things,st->nthings);                                    \
2176     assert(st->nthings);                                                \
2177     for (i=0; i<st->nthings; i++) {                                     \
2178         item_t *item=list_elem(things##_cfg,i);                         \
2179         if (item->type!=t_closure)                                      \
2180             cfgfatal(loc,"site","%s is not a closure\n",dictkey);       \
2181         closure_t *cl=item->data.closure;                               \
2182         if (cl->type!=CL_TYPE)                                          \
2183             cfgfatal(loc,"site","%s closure wrong type\n",dictkey);     \
2184         st->things[i]=cl->interface;                                    \
2185     }                                                                   \
2186 }while(0)
2187
2188     GET_CLOSURE_LIST("comm",comms,ncomms,CL_COMM);
2189
2190     NEW_ARY(st->commclientinfos, st->ncomms);
2191     dict_t *comminfo = dict_read_dict(dict,"comm-info",False,"site",loc);
2192     for (i=0; i<st->ncomms; i++) {
2193         st->commclientinfos[i] =
2194             !comminfo ? 0 :
2195             st->comms[i]->clientinfo(st->comms[i],comminfo,loc);
2196     }
2197
2198     st->resolver=find_cl_if(dict,"resolver",CL_RESOLVER,True,"site",loc);
2199     st->log=find_cl_if(dict,"log",CL_LOG,True,"site",loc);
2200     st->random=find_cl_if(dict,"random",CL_RANDOMSRC,True,"site",loc);
2201
2202     st->privkey=find_cl_if(dict,"local-key",CL_SIGPRIVKEY,True,"site",loc);
2203     st->addresses=dict_read_string_array(dict,"address",False,"site",loc,0);
2204     if (st->addresses)
2205         st->remoteport=dict_read_number(dict,"port",True,"site",loc,0);
2206     else st->remoteport=0;
2207     st->pubkey=find_cl_if(dict,"key",CL_SIGPUBKEY,True,"site",loc);
2208
2209     GET_CLOSURE_LIST("transform",transforms,ntransforms,CL_TRANSFORM);
2210
2211     st->dh=find_cl_if(dict,"dh",CL_DH,True,"site",loc);
2212
2213     if (st->privkey->sethash || st->pubkey->sethash) {
2214         struct hash_if *hash=find_cl_if(dict,"hash",CL_HASH,True,"site",loc);
2215         if (st->privkey->sethash) st->privkey->sethash(st->privkey->st,hash);
2216         if (st->pubkey->sethash) st->pubkey->sethash(st->pubkey->st,hash);
2217     }
2218
2219 #define DEFAULT(D) (st->peer_mobile || st->local_mobile \
2220                     ? DEFAULT_MOBILE_##D : DEFAULT_##D)
2221 #define CFG_NUMBER(k,D) dict_read_number(dict,(k),False,"site",loc,DEFAULT(D));
2222
2223     st->key_lifetime=         CFG_NUMBER("key-lifetime",  KEY_LIFETIME);
2224     st->setup_retries=        CFG_NUMBER("setup-retries", SETUP_RETRIES);
2225     st->setup_retry_interval= CFG_NUMBER("setup-timeout", SETUP_RETRY_INTERVAL);
2226     st->wait_timeout_mean=    CFG_NUMBER("wait-time",     WAIT_TIME);
2227     st->mtu_target= dict_read_number(dict,"mtu-target",False,"site",loc,0);
2228
2229     st->mobile_peer_expiry= dict_read_number(
2230        dict,"mobile-peer-expiry",False,"site",loc,DEFAULT_MOBILE_PEER_EXPIRY);
2231
2232     const char *peerskey= st->peer_mobile
2233         ? "mobile-peers-max" : "static-peers-max";
2234     st->transport_peers_max= dict_read_number(
2235         dict,peerskey,False,"site",loc, st->addresses ? 4 : 3);
2236     if (st->transport_peers_max<1 ||
2237         st->transport_peers_max>MAX_PEER_ADDRS) {
2238         cfgfatal(loc,"site", "%s must be in range 1.."
2239                  STRING(MAX_PEER_ADDRS) "\n", peerskey);
2240     }
2241
2242     if (st->key_lifetime < DEFAULT(KEY_RENEGOTIATE_GAP)*2)
2243         st->key_renegotiate_time=st->key_lifetime/2;
2244     else
2245         st->key_renegotiate_time=st->key_lifetime-DEFAULT(KEY_RENEGOTIATE_GAP);
2246     st->key_renegotiate_time=dict_read_number(
2247         dict,"renegotiate-time",False,"site",loc,st->key_renegotiate_time);
2248     if (st->key_renegotiate_time > st->key_lifetime) {
2249         cfgfatal(loc,"site",
2250                  "renegotiate-time must be less than key-lifetime\n");
2251     }
2252
2253     st->log_events=string_list_to_word(dict_lookup(dict,"log-events"),
2254                                        log_event_table,"site");
2255
2256     st->resolving_count=0;
2257     st->allow_send_prod=0;
2258
2259     st->tunname=safe_malloc(strlen(st->localname)+strlen(st->remotename)+5,
2260                             "site_apply");
2261     sprintf(st->tunname,"%s<->%s",st->localname,st->remotename);
2262
2263     /* The information we expect to see in incoming messages of type 1 */
2264     /* fixme: lots of unchecked overflows here, but the results are only
2265        corrupted packets rather than undefined behaviour */
2266     st->our_name_later=(strcmp(st->localname,st->remotename)>0);
2267
2268     buffer_new(&st->buffer,SETUP_BUFFER_LEN);
2269
2270     buffer_new(&st->scratch,SETUP_BUFFER_LEN);
2271     BUF_ALLOC(&st->scratch,"site:scratch");
2272
2273     /* We are interested in poll(), but only for timeouts. We don't have
2274        any fds of our own. */
2275     register_for_poll(st, site_beforepoll, site_afterpoll, "site");
2276     st->timeout=0;
2277
2278     st->remote_capabilities=0;
2279     st->chosen_transform=0;
2280     st->current.key_timeout=0;
2281     st->auxiliary_key.key_timeout=0;
2282     transport_peers_clear(st,&st->peers);
2283     transport_peers_clear(st,&st->setup_peers);
2284     /* XXX mlock these */
2285     st->dhsecret=safe_malloc(st->dh->len,"site:dhsecret");
2286     st->sharedsecretlen=st->sharedsecretallocd=0;
2287     st->sharedsecret=0;
2288
2289 #define SET_CAPBIT(bit) do {                                            \
2290     uint32_t capflag = 1UL << (bit);                                    \
2291     if (st->local_capabilities & capflag)                               \
2292         slog(st,LOG_ERROR,"capability bit"                              \
2293              " %d (%#"PRIx32") reused", (bit), capflag);                \
2294     st->local_capabilities |= capflag;                                  \
2295 } while (0)
2296
2297     for (i=0; i<st->ntransforms; i++)
2298         SET_CAPBIT(st->transforms[i]->capab_bit);
2299
2300 #undef SET_CAPBIT
2301
2302     if (st->local_mobile || st->peer_mobile)
2303         st->local_capabilities |= CAPAB_PRIORITY_MOBILE;
2304
2305     /* We need to register the remote networks with the netlink device */
2306     uint32_t netlink_mtu; /* local virtual interface mtu */
2307     st->netlink->reg(st->netlink->st, site_outgoing, st, &netlink_mtu);
2308     if (!st->mtu_target)
2309         st->mtu_target=netlink_mtu;
2310     
2311     for (i=0; i<st->ncomms; i++)
2312         st->comms[i]->request_notify(st->comms[i]->st, st, site_incoming);
2313
2314     st->current.transform=0;
2315     st->auxiliary_key.transform=0;
2316     st->new_transform=0;
2317     st->auxiliary_is_new=0;
2318
2319     enter_state_stop(st);
2320
2321     add_hook(PHASE_SHUTDOWN,site_phase_hook,st);
2322     add_hook(PHASE_CHILDPERSIST,site_childpersist_clearkeys,st);
2323
2324     return new_closure(&st->cl);
2325 }
2326
2327 void site_module(dict_t *dict)
2328 {
2329     add_closure(dict,"site",site_apply);
2330 }
2331
2332
2333 /***** TRANSPORT PEERS definitions *****/
2334
2335 static void transport_peers_debug(struct site *st, transport_peers *dst,
2336                                   const char *didwhat,
2337                                   int nargs, const struct comm_addr *args,
2338                                   size_t stride) {
2339     int i;
2340     char *argp;
2341
2342     if (!(st->log_events & LOG_PEER_ADDRS))
2343         return; /* an optimisation */
2344
2345     slog(st, LOG_PEER_ADDRS, "peers (%s) %s nargs=%d => npeers=%d",
2346          (dst==&st->peers ? "data" :
2347           dst==&st->setup_peers ? "setup" : "UNKNOWN"),
2348          didwhat, nargs, dst->npeers);
2349
2350     for (i=0, argp=(void*)args;
2351          i<nargs;
2352          i++, (argp+=stride?stride:sizeof(*args))) {
2353         const struct comm_addr *ca=(void*)argp;
2354         slog(st, LOG_PEER_ADDRS, " args: addrs[%d]=%s",
2355              i, comm_addr_to_string(ca));
2356     }
2357     for (i=0; i<dst->npeers; i++) {
2358         struct timeval diff;
2359         timersub(tv_now,&dst->peers[i].last,&diff);
2360         const struct comm_addr *ca=&dst->peers[i].addr;
2361         slog(st, LOG_PEER_ADDRS, " peers: addrs[%d]=%s T-%ld.%06ld",
2362              i, comm_addr_to_string(ca),
2363              (unsigned long)diff.tv_sec, (unsigned long)diff.tv_usec);
2364     }
2365 }
2366
2367 static void transport_peers_expire(struct site *st, transport_peers *peers) {
2368     /* peers must be sorted first */
2369     int previous_peers=peers->npeers;
2370     struct timeval oldest;
2371     oldest.tv_sec  = tv_now->tv_sec - st->mobile_peer_expiry;
2372     oldest.tv_usec = tv_now->tv_usec;
2373     while (peers->npeers>1 &&
2374            timercmp(&peers->peers[peers->npeers-1].last, &oldest, <))
2375         peers->npeers--;
2376     if (peers->npeers != previous_peers)
2377         transport_peers_debug(st,peers,"expire", 0,0,0);
2378 }
2379
2380 static bool_t transport_peer_record_one(struct site *st, transport_peers *peers,
2381                                         const struct comm_addr *ca,
2382                                         const struct timeval *tv) {
2383     /* returns false if output is full */
2384     int search;
2385
2386     if (peers->npeers >= st->transport_peers_max)
2387         return 0;
2388
2389     for (search=0; search<peers->npeers; search++)
2390         if (comm_addr_equal(&peers->peers[search].addr, ca))
2391             return 1;
2392
2393     peers->peers[peers->npeers].addr = *ca;
2394     peers->peers[peers->npeers].last = *tv;
2395     peers->npeers++;
2396     return 1;
2397 }
2398
2399 static void transport_record_peers(struct site *st, transport_peers *peers,
2400                                    const struct comm_addr *addrs, int naddrs,
2401                                    const char *m) {
2402     /* We add addrs into peers.  The new entries end up at the front
2403      * and displace entries towards the end (perhaps even off the
2404      * end).  Any existing matching entries are moved up to the front.
2405      *
2406      * Caller must first call transport_peers_expire. */
2407
2408     if (naddrs==1) {
2409         /* avoids debug for uninteresting updates */
2410         int i;
2411         for (i=0; i<peers->npeers; i++) {
2412             if (comm_addr_equal(&addrs[0], &peers->peers[i].addr)) {
2413                 memmove(peers->peers+1, peers->peers,
2414                         sizeof(peers->peers[0]) * i);
2415                 peers->peers[0].addr = addrs[0];
2416                 peers->peers[0].last = *tv_now;
2417                 return;
2418             }
2419         }
2420     }
2421
2422     int old_npeers=peers->npeers;
2423     transport_peer old_peers[old_npeers];
2424     COPY_ARRAY(old_peers,peers->peers,old_npeers);
2425
2426     peers->npeers=0;
2427     int i;
2428     for (i=0; i<naddrs; i++) {
2429         if (!transport_peer_record_one(st,peers, &addrs[i], tv_now))
2430             break;
2431     }
2432     for (i=0; i<old_npeers; i++) {
2433         const transport_peer *old=&old_peers[i];
2434         if (!transport_peer_record_one(st,peers, &old->addr, &old->last))
2435             break;
2436     }
2437
2438     transport_peers_debug(st,peers,m, naddrs,addrs,0);
2439 }
2440
2441 static void transport_expire_record_peers(struct site *st,
2442                                           transport_peers *peers,
2443                                           const struct comm_addr *addrs,
2444                                           int naddrs, const char *m) {
2445     /* Convenience function */
2446     transport_peers_expire(st,peers);
2447     transport_record_peers(st,peers,addrs,naddrs,m);
2448 }
2449
2450 static bool_t transport_compute_setupinit_peers(struct site *st,
2451         const struct comm_addr *configured_addrs /* 0 if none or not found */,
2452         int n_configured_addrs /* 0 if none or not found */,
2453         const struct comm_addr *incoming_packet_addr /* 0 if none */) {
2454     if (!n_configured_addrs && !incoming_packet_addr &&
2455         !transport_peers_valid(&st->peers))
2456         return False;
2457
2458     slog(st,LOG_SETUP_INIT,
2459          "using: %d configured addr(s);%s %d old peer addrs(es)",
2460          n_configured_addrs,
2461          incoming_packet_addr ? " incoming packet address;" : "",
2462          st->peers.npeers);
2463
2464     /* Non-mobile peers try addresses until one is plausible.  The
2465      * effect is that this code always tries first the configured
2466      * address if supplied, or otherwise the address of the incoming
2467      * PROD, or finally the existing data peer if one exists; this is
2468      * as desired. */
2469
2470     transport_peers_copy(st,&st->setup_peers,&st->peers);
2471     transport_peers_expire(st,&st->setup_peers);
2472
2473     if (incoming_packet_addr)
2474         transport_record_peers(st,&st->setup_peers,
2475                                incoming_packet_addr,1, "incoming");
2476
2477     if (n_configured_addrs)
2478         transport_record_peers(st,&st->setup_peers,
2479                               configured_addrs,n_configured_addrs, "setupinit");
2480
2481     assert(transport_peers_valid(&st->setup_peers));
2482     return True;
2483 }
2484
2485 static void transport_setup_msgok(struct site *st, const struct comm_addr *a) {
2486     if (st->peer_mobile)
2487         transport_expire_record_peers(st,&st->setup_peers,a,1,"setupmsg");
2488 }
2489 static void transport_data_msgok(struct site *st, const struct comm_addr *a) {
2490     if (st->peer_mobile)
2491         transport_expire_record_peers(st,&st->peers,a,1,"datamsg");
2492 }
2493
2494 static int transport_peers_valid(transport_peers *peers) {
2495     return peers->npeers;
2496 }
2497 static void transport_peers_clear(struct site *st, transport_peers *peers) {
2498     peers->npeers= 0;
2499     transport_peers_debug(st,peers,"clear",0,0,0);
2500 }
2501 static void transport_peers_copy(struct site *st, transport_peers *dst,
2502                                  const transport_peers *src) {
2503     dst->npeers=src->npeers;
2504     COPY_ARRAY(dst->peers, src->peers, dst->npeers);
2505     transport_peers_debug(st,dst,"copy",
2506                           src->npeers, &src->peers->addr, sizeof(*src->peers));
2507 }
2508
2509 static void transport_resolve_complete(struct site *st,
2510                                        const struct comm_addr *addrs,
2511                                        int naddrs) {
2512     transport_expire_record_peers(st,&st->peers,addrs,naddrs,
2513                                   "resolved data");
2514     transport_expire_record_peers(st,&st->setup_peers,addrs,naddrs,
2515                                   "resolved setup");
2516 }
2517
2518 static void transport_resolve_complete_tardy(struct site *st,
2519                                              const struct comm_addr *addrs,
2520                                              int naddrs) {
2521     transport_expire_record_peers(st,&st->peers,addrs,naddrs,
2522                                   "resolved tardily");
2523 }
2524
2525 static void transport_peers__copy_by_mask(transport_peer *out, int *nout_io,
2526                                           unsigned mask,
2527                                           const transport_peers *inp) {
2528     /* out and in->peers may be the same region, or nonoverlapping */
2529     const transport_peer *in=inp->peers;
2530     int slot;
2531     for (slot=0; slot<inp->npeers; slot++) {
2532         if (!(mask & (1U << slot)))
2533             continue;
2534         if (!(out==in && slot==*nout_io))
2535             COPY_OBJ(out[*nout_io], in[slot]);
2536         (*nout_io)++;
2537     }
2538 }
2539
2540 void transport_xmit(struct site *st, transport_peers *peers,
2541                     struct buffer_if *buf, bool_t candebug) {
2542     int slot;
2543     transport_peers_expire(st, peers);
2544     unsigned failed=0; /* bitmask */
2545     assert(MAX_PEER_ADDRS < sizeof(unsigned)*CHAR_BIT);
2546
2547     int nfailed=0;
2548     for (slot=0; slot<peers->npeers; slot++) {
2549         transport_peer *peer=&peers->peers[slot];
2550         bool_t ok = comm_addr_sendmsg(st, &peer->addr, buf);
2551         if (candebug)
2552             dump_packet(st, buf, &peer->addr, False, ok);
2553         if (!ok) {
2554             failed |= 1U << slot;
2555             nfailed++;
2556         }
2557         if (ok && !st->peer_mobile)
2558             break;
2559     }
2560     /* Now we need to demote/delete failing addrs: if we are mobile we
2561      * merely demote them; otherwise we delete them. */
2562     if (st->local_mobile) {
2563         unsigned expected = ((1U << nfailed)-1) << (peers->npeers-nfailed);
2564         /* `expected' has all the failures at the end already */
2565         if (failed != expected) {
2566             int fslot=0;
2567             transport_peer failedpeers[nfailed];
2568             transport_peers__copy_by_mask(failedpeers, &fslot, failed,peers);
2569             assert(fslot == nfailed);
2570             int wslot=0;
2571             transport_peers__copy_by_mask(peers->peers,&wslot,~failed,peers);
2572             assert(wslot+nfailed == peers->npeers);
2573             COPY_ARRAY(peers->peers+wslot, failedpeers, nfailed);
2574             transport_peers_debug(st,peers,"mobile failure reorder",0,0,0);
2575         }
2576     } else {
2577         if (failed && peers->npeers > 1) {
2578             int wslot=0;
2579             transport_peers__copy_by_mask(peers->peers,&wslot,~failed,peers);
2580             peers->npeers=wslot;
2581             transport_peers_debug(st,peers,"non-mobile failure cleanup",0,0,0);
2582         }
2583     }
2584 }
2585
2586 /***** END of transport peers declarations *****/