chiark / gitweb /
poll: Abolish max_nfds
[secnet.git] / site.c
1 /* site.c - manage communication with a remote network site */
2
3 /* The 'site' code doesn't know anything about the structure of the
4    packets it's transmitting.  In fact, under the new netlink
5    configuration scheme it doesn't need to know anything at all about
6    IP addresses, except how to contact its peer.  This means it could
7    potentially be used to tunnel other protocols too (IPv6, IPX, plain
8    old Ethernet frames) if appropriate netlink code can be written
9    (and that ought not to be too hard, eg. using the TUN/TAP device to
10    pretend to be an Ethernet interface).  */
11
12 /* At some point in the future the netlink code will be asked for
13    configuration information to go in the PING/PONG packets at the end
14    of the key exchange. */
15
16 #include "secnet.h"
17 #include <stdio.h>
18 #include <string.h>
19 #include <limits.h>
20 #include <assert.h>
21 #include <sys/socket.h>
22
23 #include <sys/mman.h>
24 #include "util.h"
25 #include "unaligned.h"
26 #include "magic.h"
27
28 #define SETUP_BUFFER_LEN 2048
29
30 #define DEFAULT_KEY_LIFETIME                  (3600*1000) /* [ms] */
31 #define DEFAULT_KEY_RENEGOTIATE_GAP           (5*60*1000) /* [ms] */
32 #define DEFAULT_SETUP_RETRIES 5
33 #define DEFAULT_SETUP_RETRY_INTERVAL             (2*1000) /* [ms] */
34 #define DEFAULT_WAIT_TIME                       (20*1000) /* [ms] */
35
36 #define DEFAULT_MOBILE_KEY_LIFETIME      (2*24*3600*1000) /* [ms] */
37 #define DEFAULT_MOBILE_KEY_RENEGOTIATE_GAP (12*3600*1000) /* [ms] */
38 #define DEFAULT_MOBILE_SETUP_RETRIES 30
39 #define DEFAULT_MOBILE_SETUP_RETRY_INTERVAL      (1*1000) /* [ms] */
40 #define DEFAULT_MOBILE_WAIT_TIME                (10*1000) /* [ms] */
41
42 #define DEFAULT_MOBILE_PEER_EXPIRY            (2*60)      /* [s] */
43
44 /* Each site can be in one of several possible states. */
45
46 /* States:
47    SITE_STOP         - nothing is allowed to happen; tunnel is down;
48                        all session keys have been erased
49      -> SITE_RUN upon external instruction
50    SITE_RUN          - site up, maybe with valid key
51      -> SITE_RESOLVE upon outgoing packet and no valid key
52          we start name resolution for the other end of the tunnel
53      -> SITE_SENTMSG2 upon valid incoming message 1 and suitable time
54          we send an appropriate message 2
55    SITE_RESOLVE      - waiting for name resolution
56      -> SITE_SENTMSG1 upon successful resolution
57          we send an appropriate message 1
58      -> SITE_SENTMSG2 upon valid incoming message 1 (then abort resolution)
59          we abort resolution and 
60      -> SITE_WAIT on timeout or resolution failure
61    SITE_SENTMSG1
62      -> SITE_SENTMSG2 upon valid incoming message 1 from higher priority end
63      -> SITE_SENTMSG3 upon valid incoming message 2
64      -> SITE_WAIT on timeout
65    SITE_SENTMSG2
66      -> SITE_SENTMSG4 upon valid incoming message 3
67      -> SITE_WAIT on timeout
68    SITE_SENTMSG3
69      -> SITE_SENTMSG5 upon valid incoming message 4
70      -> SITE_WAIT on timeout
71    SITE_SENTMSG4
72      -> SITE_RUN upon valid incoming message 5
73      -> SITE_WAIT on timeout
74    SITE_SENTMSG5
75      -> SITE_RUN upon valid incoming message 6
76      -> SITE_WAIT on timeout
77    SITE_WAIT         - failed to establish key; do nothing for a while
78      -> SITE_RUN on timeout
79    */
80
81 #define SITE_STOP     0
82 #define SITE_RUN      1
83 #define SITE_RESOLVE  2
84 #define SITE_SENTMSG1 3
85 #define SITE_SENTMSG2 4
86 #define SITE_SENTMSG3 5
87 #define SITE_SENTMSG4 6
88 #define SITE_SENTMSG5 7
89 #define SITE_WAIT     8
90
91 int32_t site_max_start_pad = 4*4;
92
93 static cstring_t state_name(uint32_t state)
94 {
95     switch (state) {
96     case 0: return "STOP";
97     case 1: return "RUN";
98     case 2: return "RESOLVE";
99     case 3: return "SENTMSG1";
100     case 4: return "SENTMSG2";
101     case 5: return "SENTMSG3";
102     case 6: return "SENTMSG4";
103     case 7: return "SENTMSG5";
104     case 8: return "WAIT";
105     default: return "*bad state*";
106     }
107 }
108
109 #define NONCELEN 8
110
111 #define LOG_UNEXPECTED    0x00000001
112 #define LOG_SETUP_INIT    0x00000002
113 #define LOG_SETUP_TIMEOUT 0x00000004
114 #define LOG_ACTIVATE_KEY  0x00000008
115 #define LOG_TIMEOUT_KEY   0x00000010
116 #define LOG_SEC           0x00000020
117 #define LOG_STATE         0x00000040
118 #define LOG_DROP          0x00000080
119 #define LOG_DUMP          0x00000100
120 #define LOG_ERROR         0x00000400
121 #define LOG_PEER_ADDRS    0x00000800
122
123 static struct flagstr log_event_table[]={
124     { "unexpected", LOG_UNEXPECTED },
125     { "setup-init", LOG_SETUP_INIT },
126     { "setup-timeout", LOG_SETUP_TIMEOUT },
127     { "activate-key", LOG_ACTIVATE_KEY },
128     { "timeout-key", LOG_TIMEOUT_KEY },
129     { "security", LOG_SEC },
130     { "state-change", LOG_STATE },
131     { "packet-drop", LOG_DROP },
132     { "dump-packets", LOG_DUMP },
133     { "errors", LOG_ERROR },
134     { "peer-addrs", LOG_PEER_ADDRS },
135     { "default", LOG_SETUP_INIT|LOG_SETUP_TIMEOUT|
136       LOG_ACTIVATE_KEY|LOG_TIMEOUT_KEY|LOG_SEC|LOG_ERROR },
137     { "all", 0xffffffff },
138     { NULL, 0 }
139 };
140
141
142 /***** TRANSPORT PEERS declarations *****/
143
144 /* Details of "mobile peer" semantics:
145
146    - We use the same data structure for the different configurations,
147      but manage it with different algorithms.
148    
149    - We record up to mobile_peers_max peer address/port numbers
150      ("peers") for key setup, and separately up to mobile_peers_max
151      for data transfer.
152
153    - In general, we make a new set of addrs (see below) when we start
154      a new key exchange; the key setup addrs become the data transport
155      addrs when key setup complets.
156
157    If our peer is mobile:
158
159    - We send to all recent addresses of incoming packets, plus
160      initially all configured addresses (which we also expire).
161
162    - So, we record addrs of good incoming packets, as follows:
163       1. expire any peers last seen >120s ("mobile-peer-expiry") ago
164       2. add the peer of the just received packet to the applicable list
165          (possibly evicting the oldest entries to make room)
166      NB that we do not expire peers until an incoming packet arrives.
167
168    - If the peer has a configured address or name, we record them the
169      same way, but only as a result of our own initiation of key
170      setup.  (We might evict some incoming packet addrs to make room.)
171
172    - The default number of addrs to keep is 3, or 4 if we have a
173      configured name or address.  That's space for two configured
174      addresses (one IPv6 and one IPv4), plus two received addresses.
175
176    - Outgoing packets are sent to every recorded address in the
177      applicable list.  Any unsupported[1] addresses are deleted from
178      the list right away.  (This should only happen to configured
179      addresses, of course, but there is no need to check that.)
180
181    - When we successfully complete a key setup, we merge the key setup
182      peers into the data transfer peers.
183
184    [1] An unsupported address is one for whose AF we don't have a
185      socket (perhaps because we got EAFNOSUPPORT or some such) or for
186      which sendto gives ENETUNREACH.
187
188    If neither end is mobile:
189
190    - When peer initiated the key exchange, we use the incoming packet
191      address.
192
193    - When we initiate the key exchange, we try configured addresses
194      until we get one which isn't unsupported then fixate on that.
195
196    - When we complete a key setup, we replace the data transport peers
197      with those from the key setup.
198
199    If we are mobile:
200
201    - We can't tell when local network setup changes so we can't cache
202      the unsupported addrs and completely remove the spurious calls to
203      sendto, but we can optimise things a bit by deprioritising addrs
204      which seem to be unsupported.
205
206    - Use only configured addresses.  (Except, that if our peer
207      initiated a key exchange we use the incoming packet address until
208      our name resolution completes.)
209
210    - When we send a packet, try each address in turn; if addr
211      supported, put that address to the end of the list for future
212      packets, and go onto the next address.
213
214    - When we complete a key setup, we replace the data transport peers
215      with those from the key setup.
216
217    */
218
219 typedef struct {
220     struct timeval last;
221     struct comm_addr addr;
222 } transport_peer;
223
224 typedef struct {
225 /* configuration information */
226 /* runtime information */
227     int npeers;
228     transport_peer peers[MAX_PEER_ADDRS];
229 } transport_peers;
230
231 /* Basic operations on transport peer address sets */
232 static void transport_peers_clear(struct site *st, transport_peers *peers);
233 static int transport_peers_valid(transport_peers *peers);
234 static void transport_peers_copy(struct site *st, transport_peers *dst,
235                                  const transport_peers *src);
236
237 /* Record address of incoming setup packet; resp. data packet. */
238 static void transport_setup_msgok(struct site *st, const struct comm_addr *a);
239 static void transport_data_msgok(struct site *st, const struct comm_addr *a);
240
241 /* Initialise the setup addresses.  Called before we send the first
242  * packet in a key exchange.  If we are the initiator, as a result of
243  * resolve completing (or being determined not to be relevant) or an
244  * incoming PROD; if we are the responder, as a result of the MSG1. */
245 static bool_t transport_compute_setupinit_peers(struct site *st,
246         const struct comm_addr *configured_addrs /* 0 if none or not found */,
247         int n_configured_addrs /* 0 if none or not found */,
248         const struct comm_addr *incoming_packet_addr /* 0 if none */);
249
250 /* Called if we are the responder in a key setup, when the resolve
251  * completes.  transport_compute_setupinit_peers will hvae been called
252  * earlier.  If _complete is called, we are still doing the key setup
253  * (and we should use the new values for both the rest of the key
254  * setup and the ongoing data exchange); if _tardy is called, the key
255  * setup is done (either completed or not) and only the data peers are
256  * relevant */
257 static void transport_resolve_complete(struct site *st,
258         const struct comm_addr *addrs, int naddrs);
259 static void transport_resolve_complete_tardy(struct site *st,
260         const struct comm_addr *addrs, int naddrs);
261
262 static void transport_xmit(struct site *st, transport_peers *peers,
263                            struct buffer_if *buf, bool_t candebug);
264
265  /***** END of transport peers declarations *****/
266
267
268 struct data_key {
269     struct transform_inst_if *transform;
270     uint64_t key_timeout; /* End of life of current key */
271     uint32_t remote_session_id;
272 };
273
274 struct site {
275     closure_t cl;
276     struct site_if ops;
277 /* configuration information */
278     string_t localname;
279     string_t remotename;
280     bool_t local_mobile, peer_mobile; /* Mobile client support */
281     int32_t transport_peers_max;
282     string_t tunname; /* localname<->remotename by default, used in logs */
283     cstring_t *addresses; /* DNS name or address(es) for bootstrapping, optional */
284     int remoteport; /* Port for bootstrapping, optional */
285     uint32_t mtu_target;
286     struct netlink_if *netlink;
287     struct comm_if **comms;
288     int ncomms;
289     struct resolver_if *resolver;
290     struct log_if *log;
291     struct random_if *random;
292     struct rsaprivkey_if *privkey;
293     struct rsapubkey_if *pubkey;
294     struct transform_if **transforms;
295     int ntransforms;
296     struct dh_if *dh;
297     struct hash_if *hash;
298
299     uint32_t index; /* Index of this site */
300     uint32_t local_capabilities;
301     int32_t setup_retries; /* How many times to send setup packets */
302     int32_t setup_retry_interval; /* Initial timeout for setup packets */
303     int32_t wait_timeout; /* How long to wait if setup unsuccessful */
304     int32_t mobile_peer_expiry; /* How long to remember 2ary addresses */
305     int32_t key_lifetime; /* How long a key lasts once set up */
306     int32_t key_renegotiate_time; /* If we see traffic (or a keepalive)
307                                       after this time, initiate a new
308                                       key exchange */
309
310     bool_t setup_priority; /* Do we have precedence if both sites emit
311                               message 1 simultaneously? */
312     uint32_t log_events;
313
314 /* runtime information */
315     uint32_t state;
316     uint64_t now; /* Most recently seen time */
317     bool_t allow_send_prod;
318     int resolving_count;
319     int resolving_n_results_all;
320     int resolving_n_results_stored;
321     struct comm_addr resolving_results[MAX_PEER_ADDRS];
322
323     /* The currently established session */
324     struct data_key current;
325     struct data_key auxiliary_key;
326     bool_t auxiliary_is_new;
327     uint64_t renegotiate_key_time; /* When we can negotiate a new key */
328     uint64_t auxiliary_renegotiate_key_time;
329     transport_peers peers; /* Current address(es) of peer for data traffic */
330
331     /* The current key setup protocol exchange.  We can only be
332        involved in one of these at a time.  There's a potential for
333        denial of service here (the attacker keeps sending a setup
334        packet; we keep trying to continue the exchange, and have to
335        timeout before we can listen for another setup packet); perhaps
336        we should keep a list of 'bad' sources for setup packets. */
337     uint32_t remote_capabilities;
338     uint16_t remote_adv_mtu;
339     struct transform_if *chosen_transform;
340     uint32_t setup_session_id;
341     transport_peers setup_peers;
342     uint8_t localN[NONCELEN]; /* Nonces for key exchange */
343     uint8_t remoteN[NONCELEN];
344     struct buffer_if buffer; /* Current outgoing key exchange packet */
345     struct buffer_if scratch;
346     int32_t retries; /* Number of retries remaining */
347     uint64_t timeout; /* Timeout for current state */
348     uint8_t *dhsecret;
349     uint8_t *sharedsecret;
350     uint32_t sharedsecretlen, sharedsecretallocd;
351     struct transform_inst_if *new_transform; /* For key setup/verify */
352 };
353
354 static uint32_t event_log_priority(struct site *st, uint32_t event)
355 {
356     if (!(event&st->log_events))
357         return 0;
358     switch(event) {
359     case LOG_UNEXPECTED:    return M_INFO;
360     case LOG_SETUP_INIT:    return M_INFO;
361     case LOG_SETUP_TIMEOUT: return M_NOTICE;
362     case LOG_ACTIVATE_KEY:  return M_INFO;
363     case LOG_TIMEOUT_KEY:   return M_INFO;
364     case LOG_SEC:           return M_SECURITY;
365     case LOG_STATE:         return M_DEBUG;
366     case LOG_DROP:          return M_DEBUG;
367     case LOG_DUMP:          return M_DEBUG;
368     case LOG_ERROR:         return M_ERR;
369     case LOG_PEER_ADDRS:    return M_DEBUG;
370     default:                return M_ERR;
371     }
372 }
373
374 static void vslog(struct site *st, uint32_t event, cstring_t msg, va_list ap)
375 FORMAT(printf,3,0);
376 static void vslog(struct site *st, uint32_t event, cstring_t msg, va_list ap)
377 {
378     uint32_t class;
379
380     class=event_log_priority(st, event);
381     if (class) {
382         slilog_part(st->log,class,"%s: ",st->tunname);
383         vslilog_part(st->log,class,msg,ap);
384         slilog_part(st->log,class,"\n");
385     }
386 }
387
388 static void slog(struct site *st, uint32_t event, cstring_t msg, ...)
389 FORMAT(printf,3,4);
390 static void slog(struct site *st, uint32_t event, cstring_t msg, ...)
391 {
392     va_list ap;
393     va_start(ap,msg);
394     vslog(st,event,msg,ap);
395     va_end(ap);
396 }
397
398 static void logtimeout(struct site *st, const char *fmt, ...)
399 FORMAT(printf,2,3);
400 static void logtimeout(struct site *st, const char *fmt, ...)
401 {
402     uint32_t class=event_log_priority(st,LOG_SETUP_TIMEOUT);
403     if (!class)
404         return;
405
406     va_list ap;
407     va_start(ap,fmt);
408
409     slilog_part(st->log,class,"%s: ",st->tunname);
410     vslilog_part(st->log,class,fmt,ap);
411
412     const char *delim;
413     int i;
414     for (i=0, delim=" (tried ";
415          i<st->setup_peers.npeers;
416          i++, delim=", ") {
417         transport_peer *peer=&st->setup_peers.peers[i];
418         const char *s=comm_addr_to_string(&peer->addr);
419         slilog_part(st->log,class,"%s%s",delim,s);
420     }
421
422     slilog_part(st->log,class,")\n");
423     va_end(ap);
424 }
425
426 static void set_link_quality(struct site *st);
427 static void delete_keys(struct site *st, cstring_t reason, uint32_t loglevel);
428 static void delete_one_key(struct site *st, struct data_key *key,
429                            const char *reason /* may be 0 meaning don't log*/,
430                            const char *which /* ignored if !reasonn */,
431                            uint32_t loglevel /* ignored if !reasonn */);
432 static bool_t initiate_key_setup(struct site *st, cstring_t reason,
433                                  const struct comm_addr *prod_hint);
434 static void enter_state_run(struct site *st);
435 static bool_t enter_state_resolve(struct site *st);
436 static void decrement_resolving_count(struct site *st, int by);
437 static bool_t enter_new_state(struct site *st,uint32_t next);
438 static void enter_state_wait(struct site *st);
439 static void activate_new_key(struct site *st);
440
441 static bool_t is_transform_valid(struct transform_inst_if *transform)
442 {
443     return transform && transform->valid(transform->st);
444 }
445
446 static bool_t current_valid(struct site *st)
447 {
448     return is_transform_valid(st->current.transform);
449 }
450
451 #define DEFINE_CALL_TRANSFORM(fwdrev)                                   \
452 static int call_transform_##fwdrev(struct site *st,                     \
453                                    struct transform_inst_if *transform, \
454                                    struct buffer_if *buf,               \
455                                    const char **errmsg)                 \
456 {                                                                       \
457     if (!is_transform_valid(transform)) {                               \
458         *errmsg="transform not set up";                                 \
459         return 1;                                                       \
460     }                                                                   \
461     return transform->fwdrev(transform->st,buf,errmsg);                 \
462 }
463
464 DEFINE_CALL_TRANSFORM(forwards)
465 DEFINE_CALL_TRANSFORM(reverse)
466
467 static void dispose_transform(struct transform_inst_if **transform_var)
468 {
469     struct transform_inst_if *transform=*transform_var;
470     if (transform) {
471         transform->delkey(transform->st);
472         transform->destroy(transform->st);
473     }
474     *transform_var = 0;
475 }    
476
477 #define CHECK_AVAIL(b,l) do { if ((b)->size<(l)) return False; } while(0)
478 #define CHECK_EMPTY(b) do { if ((b)->size!=0) return False; } while(0)
479 #define CHECK_TYPE(b,t) do { uint32_t type; \
480     CHECK_AVAIL((b),4); \
481     type=buf_unprepend_uint32((b)); \
482     if (type!=(t)) return False; } while(0)
483
484 static _Bool type_is_msg34(uint32_t type)
485 {
486     return
487         type == LABEL_MSG3 ||
488         type == LABEL_MSG3BIS ||
489         type == LABEL_MSG4;
490 }
491
492 struct parsedname {
493     int32_t len;
494     uint8_t *name;
495     struct buffer_if extrainfo;
496 };
497
498 struct msg {
499     uint8_t *hashstart;
500     uint32_t dest;
501     uint32_t source;
502     struct parsedname remote;
503     struct parsedname local;
504     uint32_t remote_capabilities;
505     uint16_t remote_mtu;
506     int capab_transformnum;
507     uint8_t *nR;
508     uint8_t *nL;
509     int32_t pklen;
510     char *pk;
511     int32_t hashlen;
512     int32_t siglen;
513     char *sig;
514 };
515
516 static void set_new_transform(struct site *st, char *pk)
517 {
518     /* Make room for the shared key */
519     st->sharedsecretlen=st->chosen_transform->keylen?:st->dh->ceil_len;
520     assert(st->sharedsecretlen);
521     if (st->sharedsecretlen > st->sharedsecretallocd) {
522         st->sharedsecretallocd=st->sharedsecretlen;
523         st->sharedsecret=realloc(st->sharedsecret,st->sharedsecretallocd);
524     }
525     if (!st->sharedsecret) fatal_perror("site:sharedsecret");
526
527     /* Generate the shared key */
528     st->dh->makeshared(st->dh->st,st->dhsecret,st->dh->len,pk,
529                        st->sharedsecret,st->sharedsecretlen);
530
531     /* Set up the transform */
532     struct transform_if *generator=st->chosen_transform;
533     struct transform_inst_if *generated=generator->create(generator->st);
534     generated->setkey(generated->st,st->sharedsecret,
535                       st->sharedsecretlen,st->setup_priority);
536     dispose_transform(&st->new_transform);
537     st->new_transform=generated;
538
539     slog(st,LOG_SETUP_INIT,"key exchange negotiated transform"
540          " %d (capabilities ours=%#"PRIx32" theirs=%#"PRIx32")",
541          st->chosen_transform->capab_transformnum,
542          st->local_capabilities, st->remote_capabilities);
543 }
544
545 struct xinfoadd {
546     int32_t lenpos, afternul;
547 };
548 static void append_string_xinfo_start(struct buffer_if *buf,
549                                       struct xinfoadd *xia,
550                                       const char *str)
551     /* Helps construct one of the names with additional info as found
552      * in MSG1..4.  Call this function first, then append all the
553      * desired extra info (not including the nul byte) to the buffer,
554      * then call append_string_xinfo_done. */
555 {
556     xia->lenpos = buf->size;
557     buf_append_string(buf,str);
558     buf_append_uint8(buf,0);
559     xia->afternul = buf->size;
560 }
561 static void append_string_xinfo_done(struct buffer_if *buf,
562                                      struct xinfoadd *xia)
563 {
564     /* we just need to adjust the string length */
565     if (buf->size == xia->afternul) {
566         /* no extra info, strip the nul too */
567         buf_unappend_uint8(buf);
568     } else {
569         put_uint16(buf->start+xia->lenpos, buf->size-(xia->lenpos+2));
570     }
571 }
572
573 /* Build any of msg1 to msg4. msg5 and msg6 are built from the inside
574    out using a transform of config data supplied by netlink */
575 static bool_t generate_msg(struct site *st, uint32_t type, cstring_t what)
576 {
577     void *hst;
578     uint8_t *hash;
579     string_t dhpub, sig;
580
581     st->retries=st->setup_retries;
582     BUF_ALLOC(&st->buffer,what);
583     buffer_init(&st->buffer,0);
584     buf_append_uint32(&st->buffer,
585         (type==LABEL_MSG1?0:st->setup_session_id));
586     buf_append_uint32(&st->buffer,st->index);
587     buf_append_uint32(&st->buffer,type);
588
589     struct xinfoadd xia;
590     append_string_xinfo_start(&st->buffer,&xia,st->localname);
591     if ((st->local_capabilities & CAPAB_EARLY) || (type != LABEL_MSG1)) {
592         buf_append_uint32(&st->buffer,st->local_capabilities);
593     }
594     if (type_is_msg34(type)) {
595         buf_append_uint16(&st->buffer,st->mtu_target);
596     }
597     append_string_xinfo_done(&st->buffer,&xia);
598
599     buf_append_string(&st->buffer,st->remotename);
600     BUF_ADD_OBJ(append,&st->buffer,st->localN);
601     if (type==LABEL_MSG1) return True;
602     BUF_ADD_OBJ(append,&st->buffer,st->remoteN);
603     if (type==LABEL_MSG2) return True;
604
605     if (hacky_par_mid_failnow()) return False;
606
607     if (type==LABEL_MSG3BIS)
608         buf_append_uint8(&st->buffer,st->chosen_transform->capab_transformnum);
609
610     dhpub=st->dh->makepublic(st->dh->st,st->dhsecret,st->dh->len);
611     buf_append_string(&st->buffer,dhpub);
612     free(dhpub);
613     hash=safe_malloc(st->hash->len, "generate_msg");
614     hst=st->hash->init();
615     st->hash->update(hst,st->buffer.start,st->buffer.size);
616     st->hash->final(hst,hash);
617     sig=st->privkey->sign(st->privkey->st,hash,st->hash->len);
618     buf_append_string(&st->buffer,sig);
619     free(sig);
620     free(hash);
621     return True;
622 }
623
624 static bool_t unpick_name(struct buffer_if *msg, struct parsedname *nm)
625 {
626     CHECK_AVAIL(msg,2);
627     nm->len=buf_unprepend_uint16(msg);
628     CHECK_AVAIL(msg,nm->len);
629     nm->name=buf_unprepend(msg,nm->len);
630     uint8_t *nul=memchr(nm->name,0,nm->len);
631     if (!nul) {
632         buffer_readonly_view(&nm->extrainfo,0,0);
633     } else {
634         buffer_readonly_view(&nm->extrainfo, nul+1, msg->start-(nul+1));
635         nm->len=nul-nm->name;
636     }
637     return True;
638 }
639
640 static bool_t unpick_msg(struct site *st, uint32_t type,
641                          struct buffer_if *msg, struct msg *m)
642 {
643     m->capab_transformnum=-1;
644     m->hashstart=msg->start;
645     CHECK_AVAIL(msg,4);
646     m->dest=buf_unprepend_uint32(msg);
647     CHECK_AVAIL(msg,4);
648     m->source=buf_unprepend_uint32(msg);
649     CHECK_TYPE(msg,type);
650     if (!unpick_name(msg,&m->remote)) return False;
651     m->remote_capabilities=0;
652     m->remote_mtu=0;
653     if (m->remote.extrainfo.size) {
654         CHECK_AVAIL(&m->remote.extrainfo,4);
655         m->remote_capabilities=buf_unprepend_uint32(&m->remote.extrainfo);
656     }
657     if (type_is_msg34(type) && m->remote.extrainfo.size) {
658         CHECK_AVAIL(&m->remote.extrainfo,2);
659         m->remote_mtu=buf_unprepend_uint16(&m->remote.extrainfo);
660     }
661     if (!unpick_name(msg,&m->local)) return False;
662     if (type==LABEL_PROD) {
663         CHECK_EMPTY(msg);
664         return True;
665     }
666     CHECK_AVAIL(msg,NONCELEN);
667     m->nR=buf_unprepend(msg,NONCELEN);
668     if (type==LABEL_MSG1) {
669         CHECK_EMPTY(msg);
670         return True;
671     }
672     CHECK_AVAIL(msg,NONCELEN);
673     m->nL=buf_unprepend(msg,NONCELEN);
674     if (type==LABEL_MSG2) {
675         CHECK_EMPTY(msg);
676         return True;
677     }
678     if (type==LABEL_MSG3BIS) {
679         CHECK_AVAIL(msg,1);
680         m->capab_transformnum = buf_unprepend_uint8(msg);
681     } else {
682         m->capab_transformnum = CAPAB_TRANSFORMNUM_ANCIENT;
683     }
684     CHECK_AVAIL(msg,2);
685     m->pklen=buf_unprepend_uint16(msg);
686     CHECK_AVAIL(msg,m->pklen);
687     m->pk=buf_unprepend(msg,m->pklen);
688     m->hashlen=msg->start-m->hashstart;
689     CHECK_AVAIL(msg,2);
690     m->siglen=buf_unprepend_uint16(msg);
691     CHECK_AVAIL(msg,m->siglen);
692     m->sig=buf_unprepend(msg,m->siglen);
693     CHECK_EMPTY(msg);
694     return True;
695 }
696
697 static bool_t name_matches(const struct parsedname *nm, const char *expected)
698 {
699     int expected_len=strlen(expected);
700     return
701         nm->len == expected_len &&
702         !memcmp(nm->name, expected, expected_len);
703 }    
704
705 static bool_t check_msg(struct site *st, uint32_t type, struct msg *m,
706                         cstring_t *error)
707 {
708     if (type==LABEL_MSG1) return True;
709
710     /* Check that the site names and our nonce have been sent
711        back correctly, and then store our peer's nonce. */ 
712     if (!name_matches(&m->remote,st->remotename)) {
713         *error="wrong remote site name";
714         return False;
715     }
716     if (!name_matches(&m->local,st->localname)) {
717         *error="wrong local site name";
718         return False;
719     }
720     if (memcmp(m->nL,st->localN,NONCELEN)!=0) {
721         *error="wrong locally-generated nonce";
722         return False;
723     }
724     if (type==LABEL_MSG2) return True;
725     if (!consttime_memeq(m->nR,st->remoteN,NONCELEN)!=0) {
726         *error="wrong remotely-generated nonce";
727         return False;
728     }
729     /* MSG3 has complicated rules about capabilities, which are
730      * handled in process_msg3. */
731     if (type==LABEL_MSG3 || type==LABEL_MSG3BIS) return True;
732     if (m->remote_capabilities!=st->remote_capabilities) {
733         *error="remote capabilities changed";
734         return False;
735     }
736     if (type==LABEL_MSG4) return True;
737     *error="unknown message type";
738     return False;
739 }
740
741 static bool_t generate_msg1(struct site *st)
742 {
743     st->random->generate(st->random->st,NONCELEN,st->localN);
744     return generate_msg(st,LABEL_MSG1,"site:MSG1");
745 }
746
747 static bool_t process_msg1(struct site *st, struct buffer_if *msg1,
748                            const struct comm_addr *src, struct msg *m)
749 {
750     /* We've already determined we're in an appropriate state to
751        process an incoming MSG1, and that the MSG1 has correct values
752        of A and B. */
753
754     st->setup_session_id=m->source;
755     st->remote_capabilities=m->remote_capabilities;
756     memcpy(st->remoteN,m->nR,NONCELEN);
757     return True;
758 }
759
760 static bool_t generate_msg2(struct site *st)
761 {
762     st->random->generate(st->random->st,NONCELEN,st->localN);
763     return generate_msg(st,LABEL_MSG2,"site:MSG2");
764 }
765
766 static bool_t process_msg2(struct site *st, struct buffer_if *msg2,
767                            const struct comm_addr *src)
768 {
769     struct msg m;
770     cstring_t err;
771
772     if (!unpick_msg(st,LABEL_MSG2,msg2,&m)) return False;
773     if (!check_msg(st,LABEL_MSG2,&m,&err)) {
774         slog(st,LOG_SEC,"msg2: %s",err);
775         return False;
776     }
777     st->setup_session_id=m.source;
778     st->remote_capabilities=m.remote_capabilities;
779
780     /* Select the transform to use */
781
782     uint32_t remote_transforms = st->remote_capabilities & CAPAB_TRANSFORM_MASK;
783     if (!remote_transforms)
784         /* old secnets only had this one transform */
785         remote_transforms = 1UL << CAPAB_TRANSFORMNUM_ANCIENT;
786
787     struct transform_if *ti;
788     int i;
789     for (i=0; i<st->ntransforms; i++) {
790         ti=st->transforms[i];
791         if ((1UL << ti->capab_transformnum) & remote_transforms)
792             goto transform_found;
793     }
794     slog(st,LOG_ERROR,"no transforms in common"
795          " (us %#"PRIx32"; them: %#"PRIx32")",
796          st->local_capabilities & CAPAB_TRANSFORM_MASK,
797          remote_transforms);
798     return False;
799  transform_found:
800     st->chosen_transform=ti;
801
802     memcpy(st->remoteN,m.nR,NONCELEN);
803     return True;
804 }
805
806 static bool_t generate_msg3(struct site *st)
807 {
808     /* Now we have our nonce and their nonce. Think of a secret key,
809        and create message number 3. */
810     st->random->generate(st->random->st,st->dh->len,st->dhsecret);
811     return generate_msg(st,
812                         (st->remote_capabilities & CAPAB_TRANSFORM_MASK
813                          ? LABEL_MSG3BIS : LABEL_MSG3),
814                         "site:MSG3");
815 }
816
817 static bool_t process_msg3_msg4(struct site *st, struct msg *m)
818 {
819     uint8_t *hash;
820     void *hst;
821
822     /* Check signature and store g^x mod m */
823     hash=safe_malloc(st->hash->len, "process_msg3_msg4");
824     hst=st->hash->init();
825     st->hash->update(hst,m->hashstart,m->hashlen);
826     st->hash->final(hst,hash);
827     /* Terminate signature with a '0' - cheating, but should be ok */
828     m->sig[m->siglen]=0;
829     if (!st->pubkey->check(st->pubkey->st,hash,st->hash->len,m->sig)) {
830         slog(st,LOG_SEC,"msg3/msg4 signature failed check!");
831         free(hash);
832         return False;
833     }
834     free(hash);
835
836     st->remote_adv_mtu=m->remote_mtu;
837
838     return True;
839 }
840
841 static bool_t process_msg3(struct site *st, struct buffer_if *msg3,
842                            const struct comm_addr *src, uint32_t msgtype)
843 {
844     struct msg m;
845     cstring_t err;
846
847     assert(msgtype==LABEL_MSG3 || msgtype==LABEL_MSG3BIS);
848
849     if (!unpick_msg(st,msgtype,msg3,&m)) return False;
850     if (!check_msg(st,msgtype,&m,&err)) {
851         slog(st,LOG_SEC,"msg3: %s",err);
852         return False;
853     }
854     uint32_t capab_adv_late = m.remote_capabilities
855         & ~st->remote_capabilities & CAPAB_EARLY;
856     if (capab_adv_late) {
857         slog(st,LOG_SEC,"msg3 impermissibly adds early capability flag(s)"
858              " %#"PRIx32" (was %#"PRIx32", now %#"PRIx32")",
859              capab_adv_late, st->remote_capabilities, m.remote_capabilities);
860         return False;
861     }
862     st->remote_capabilities|=m.remote_capabilities;
863
864     struct transform_if *ti;
865     int i;
866     for (i=0; i<st->ntransforms; i++) {
867         ti=st->transforms[i];
868         if (ti->capab_transformnum == m.capab_transformnum)
869             goto transform_found;
870     }
871     slog(st,LOG_SEC,"peer chose unknown-to-us transform %d!",
872          m.capab_transformnum);
873     return False;
874  transform_found:
875     st->chosen_transform=ti;
876
877     if (!process_msg3_msg4(st,&m))
878         return False;
879
880     /* Terminate their DH public key with a '0' */
881     m.pk[m.pklen]=0;
882     /* Invent our DH secret key */
883     st->random->generate(st->random->st,st->dh->len,st->dhsecret);
884
885     /* Generate the shared key and set up the transform */
886     set_new_transform(st,m.pk);
887
888     return True;
889 }
890
891 static bool_t generate_msg4(struct site *st)
892 {
893     /* We have both nonces, their public key and our private key. Generate
894        our public key, sign it and send it to them. */
895     return generate_msg(st,LABEL_MSG4,"site:MSG4");
896 }
897
898 static bool_t process_msg4(struct site *st, struct buffer_if *msg4,
899                            const struct comm_addr *src)
900 {
901     struct msg m;
902     cstring_t err;
903
904     if (!unpick_msg(st,LABEL_MSG4,msg4,&m)) return False;
905     if (!check_msg(st,LABEL_MSG4,&m,&err)) {
906         slog(st,LOG_SEC,"msg4: %s",err);
907         return False;
908     }
909     
910     if (!process_msg3_msg4(st,&m))
911         return False;
912
913     /* Terminate their DH public key with a '0' */
914     m.pk[m.pklen]=0;
915
916     /* Generate the shared key and set up the transform */
917     set_new_transform(st,m.pk);
918
919     return True;
920 }
921
922 struct msg0 {
923     uint32_t dest;
924     uint32_t source;
925     uint32_t type;
926 };
927
928 static bool_t unpick_msg0(struct site *st, struct buffer_if *msg0,
929                           struct msg0 *m)
930 {
931     CHECK_AVAIL(msg0,4);
932     m->dest=buf_unprepend_uint32(msg0);
933     CHECK_AVAIL(msg0,4);
934     m->source=buf_unprepend_uint32(msg0);
935     CHECK_AVAIL(msg0,4);
936     m->type=buf_unprepend_uint32(msg0);
937     return True;
938     /* Leaves transformed part of buffer untouched */
939 }
940
941 static bool_t generate_msg5(struct site *st)
942 {
943     cstring_t transform_err;
944
945     BUF_ALLOC(&st->buffer,"site:MSG5");
946     /* We are going to add four words to the message */
947     buffer_init(&st->buffer,calculate_max_start_pad());
948     /* Give the netlink code an opportunity to put its own stuff in the
949        message (configuration information, etc.) */
950     buf_prepend_uint32(&st->buffer,LABEL_MSG5);
951     if (call_transform_forwards(st,st->new_transform,
952                                 &st->buffer,&transform_err))
953         return False;
954     buf_prepend_uint32(&st->buffer,LABEL_MSG5);
955     buf_prepend_uint32(&st->buffer,st->index);
956     buf_prepend_uint32(&st->buffer,st->setup_session_id);
957
958     st->retries=st->setup_retries;
959     return True;
960 }
961
962 static bool_t process_msg5(struct site *st, struct buffer_if *msg5,
963                            const struct comm_addr *src,
964                            struct transform_inst_if *transform)
965 {
966     struct msg0 m;
967     cstring_t transform_err;
968
969     if (!unpick_msg0(st,msg5,&m)) return False;
970
971     if (call_transform_reverse(st,transform,msg5,&transform_err)) {
972         /* There's a problem */
973         slog(st,LOG_SEC,"process_msg5: transform: %s",transform_err);
974         return False;
975     }
976     /* Buffer should now contain untransformed PING packet data */
977     CHECK_AVAIL(msg5,4);
978     if (buf_unprepend_uint32(msg5)!=LABEL_MSG5) {
979         slog(st,LOG_SEC,"MSG5/PING packet contained wrong label");
980         return False;
981     }
982     /* Older versions of secnet used to write some config data here
983      * which we ignore.  So we don't CHECK_EMPTY */
984     return True;
985 }
986
987 static void create_msg6(struct site *st, struct transform_inst_if *transform,
988                         uint32_t session_id)
989 {
990     cstring_t transform_err;
991
992     BUF_ALLOC(&st->buffer,"site:MSG6");
993     /* We are going to add four words to the message */
994     buffer_init(&st->buffer,calculate_max_start_pad());
995     /* Give the netlink code an opportunity to put its own stuff in the
996        message (configuration information, etc.) */
997     buf_prepend_uint32(&st->buffer,LABEL_MSG6);
998     int problem = call_transform_forwards(st,transform,
999                                           &st->buffer,&transform_err);
1000     assert(!problem);
1001     buf_prepend_uint32(&st->buffer,LABEL_MSG6);
1002     buf_prepend_uint32(&st->buffer,st->index);
1003     buf_prepend_uint32(&st->buffer,session_id);
1004 }
1005
1006 static bool_t generate_msg6(struct site *st)
1007 {
1008     if (!is_transform_valid(st->new_transform))
1009         return False;
1010     create_msg6(st,st->new_transform,st->setup_session_id);
1011     st->retries=1; /* Peer will retransmit MSG5 if this packet gets lost */
1012     return True;
1013 }
1014
1015 static bool_t process_msg6(struct site *st, struct buffer_if *msg6,
1016                            const struct comm_addr *src)
1017 {
1018     struct msg0 m;
1019     cstring_t transform_err;
1020
1021     if (!unpick_msg0(st,msg6,&m)) return False;
1022
1023     if (call_transform_reverse(st,st->new_transform,msg6,&transform_err)) {
1024         /* There's a problem */
1025         slog(st,LOG_SEC,"process_msg6: transform: %s",transform_err);
1026         return False;
1027     }
1028     /* Buffer should now contain untransformed PING packet data */
1029     CHECK_AVAIL(msg6,4);
1030     if (buf_unprepend_uint32(msg6)!=LABEL_MSG6) {
1031         slog(st,LOG_SEC,"MSG6/PONG packet contained invalid data");
1032         return False;
1033     }
1034     /* Older versions of secnet used to write some config data here
1035      * which we ignore.  So we don't CHECK_EMPTY */
1036     return True;
1037 }
1038
1039 static bool_t decrypt_msg0(struct site *st, struct buffer_if *msg0,
1040                            const struct comm_addr *src)
1041 {
1042     cstring_t transform_err, auxkey_err, newkey_err="n/a";
1043     struct msg0 m;
1044     uint32_t problem;
1045
1046     if (!unpick_msg0(st,msg0,&m)) return False;
1047
1048     /* Keep a copy so we can try decrypting it with multiple keys */
1049     buffer_copy(&st->scratch, msg0);
1050
1051     problem = call_transform_reverse(st,st->current.transform,
1052                                      msg0,&transform_err);
1053     if (!problem) {
1054         if (!st->auxiliary_is_new)
1055             delete_one_key(st,&st->auxiliary_key,
1056                            "peer has used new key","auxiliary key",LOG_SEC);
1057         return True;
1058     }
1059     if (problem==2)
1060         goto skew;
1061
1062     buffer_copy(msg0, &st->scratch);
1063     problem = call_transform_reverse(st,st->auxiliary_key.transform,
1064                                      msg0,&auxkey_err);
1065     if (problem==0) {
1066         slog(st,LOG_DROP,"processing packet which uses auxiliary key");
1067         if (st->auxiliary_is_new) {
1068             /* We previously timed out in state SENTMSG5 but it turns
1069              * out that our peer did in fact get our MSG5 and is
1070              * using the new key.  So we should switch to it too. */
1071             /* This is a bit like activate_new_key. */
1072             struct data_key t;
1073             t=st->current;
1074             st->current=st->auxiliary_key;
1075             st->auxiliary_key=t;
1076
1077             delete_one_key(st,&st->auxiliary_key,"peer has used new key",
1078                            "previous key",LOG_SEC);
1079             st->auxiliary_is_new=0;
1080             st->renegotiate_key_time=st->auxiliary_renegotiate_key_time;
1081         }
1082         return True;
1083     }
1084     if (problem==2)
1085         goto skew;
1086
1087     if (st->state==SITE_SENTMSG5) {
1088         buffer_copy(msg0, &st->scratch);
1089         problem = call_transform_reverse(st,st->new_transform,
1090                                          msg0,&newkey_err);
1091         if (!problem) {
1092             /* It looks like we didn't get the peer's MSG6 */
1093             /* This is like a cut-down enter_new_state(SITE_RUN) */
1094             slog(st,LOG_STATE,"will enter state RUN (MSG0 with new key)");
1095             BUF_FREE(&st->buffer);
1096             st->timeout=0;
1097             activate_new_key(st);
1098             return True; /* do process the data in this packet */
1099         }
1100         if (problem==2)
1101             goto skew;
1102     }
1103
1104     slog(st,LOG_SEC,"transform: %s (aux: %s, new: %s)",
1105          transform_err,auxkey_err,newkey_err);
1106     initiate_key_setup(st,"incoming message would not decrypt",0);
1107     send_nak(src,m.dest,m.source,m.type,msg0,"message would not decrypt");
1108     return False;
1109
1110  skew:
1111     slog(st,LOG_DROP,"transform: %s (merely skew)",transform_err);
1112     return False;
1113 }
1114
1115 static bool_t process_msg0(struct site *st, struct buffer_if *msg0,
1116                            const struct comm_addr *src)
1117 {
1118     uint32_t type;
1119
1120     if (!decrypt_msg0(st,msg0,src))
1121         return False;
1122
1123     CHECK_AVAIL(msg0,4);
1124     type=buf_unprepend_uint32(msg0);
1125     switch(type) {
1126     case LABEL_MSG7:
1127         /* We must forget about the current session. */
1128         delete_keys(st,"request from peer",LOG_SEC);
1129         return True;
1130     case LABEL_MSG9:
1131         /* Deliver to netlink layer */
1132         st->netlink->deliver(st->netlink->st,msg0);
1133         transport_data_msgok(st,src);
1134         /* See whether we should start negotiating a new key */
1135         if (st->now > st->renegotiate_key_time)
1136             initiate_key_setup(st,"incoming packet in renegotiation window",0);
1137         return True;
1138     default:
1139         slog(st,LOG_SEC,"incoming encrypted message of type %08x "
1140              "(unknown)",type);
1141         break;
1142     }
1143     return False;
1144 }
1145
1146 static void dump_packet(struct site *st, struct buffer_if *buf,
1147                         const struct comm_addr *addr, bool_t incoming)
1148 {
1149     uint32_t dest=get_uint32(buf->start);
1150     uint32_t source=get_uint32(buf->start+4);
1151     uint32_t msgtype=get_uint32(buf->start+8);
1152
1153     if (st->log_events & LOG_DUMP)
1154         slilog(st->log,M_DEBUG,"%s: %s: %08x<-%08x: %08x:",
1155                st->tunname,incoming?"incoming":"outgoing",
1156                dest,source,msgtype);
1157 }
1158
1159 static uint32_t site_status(void *st)
1160 {
1161     return 0;
1162 }
1163
1164 static bool_t send_msg(struct site *st)
1165 {
1166     if (st->retries>0) {
1167         transport_xmit(st, &st->setup_peers, &st->buffer, True);
1168         st->timeout=st->now+st->setup_retry_interval;
1169         st->retries--;
1170         return True;
1171     } else if (st->state==SITE_SENTMSG5) {
1172         logtimeout(st,"timed out sending MSG5, stashing new key");
1173         /* We stash the key we have produced, in case it turns out that
1174          * our peer did see our MSG5 after all and starts using it. */
1175         /* This is a bit like some of activate_new_key */
1176         struct transform_inst_if *t;
1177         t=st->auxiliary_key.transform;
1178         st->auxiliary_key.transform=st->new_transform;
1179         st->new_transform=t;
1180         dispose_transform(&st->new_transform);
1181
1182         st->auxiliary_is_new=1;
1183         st->auxiliary_key.key_timeout=st->now+st->key_lifetime;
1184         st->auxiliary_renegotiate_key_time=st->now+st->key_renegotiate_time;
1185         st->auxiliary_key.remote_session_id=st->setup_session_id;
1186
1187         enter_state_wait(st);
1188         return False;
1189     } else {
1190         logtimeout(st,"timed out sending key setup packet "
1191             "(in state %s)",state_name(st->state));
1192         enter_state_wait(st);
1193         return False;
1194     }
1195 }
1196
1197 static void site_resolve_callback(void *sst, const struct comm_addr *addrs,
1198                                   int stored_naddrs, int all_naddrs,
1199                                   const char *address, const char *failwhy)
1200 {
1201     struct site *st=sst;
1202
1203     if (!stored_naddrs) {
1204         slog(st,LOG_ERROR,"resolution of %s failed: %s",address,failwhy);
1205     } else {
1206         slog(st,LOG_PEER_ADDRS,"resolution of %s completed, %d addrs, eg: %s",
1207              address, all_naddrs, comm_addr_to_string(&addrs[0]));;
1208
1209         int space=st->transport_peers_max-st->resolving_n_results_stored;
1210         int n_tocopy=MIN(stored_naddrs,space);
1211         COPY_ARRAY(st->resolving_results + st->resolving_n_results_stored,
1212                    addrs,
1213                    n_tocopy);
1214         st->resolving_n_results_stored += n_tocopy;
1215         st->resolving_n_results_all += all_naddrs;
1216     }
1217
1218     decrement_resolving_count(st,1);
1219 }
1220
1221 static void decrement_resolving_count(struct site *st, int by)
1222 {
1223     assert(st->resolving_count>0);
1224     st->resolving_count-=by;
1225
1226     if (st->resolving_count)
1227         return;
1228
1229     /* OK, we are done with them all.  Handle combined results. */
1230
1231     const struct comm_addr *addrs=st->resolving_results;
1232     int naddrs=st->resolving_n_results_stored;
1233     assert(naddrs<=st->transport_peers_max);
1234
1235     if (naddrs) {
1236         if (naddrs != st->resolving_n_results_all) {
1237             slog(st,LOG_SETUP_INIT,"resolution of supplied addresses/names"
1238                  " yielded too many results (%d > %d), some ignored",
1239                  st->resolving_n_results_all, naddrs);
1240         }
1241         slog(st,LOG_STATE,"resolution completed, %d addrs, eg: %s",
1242              naddrs, comm_addr_to_string(&addrs[0]));;
1243     }
1244
1245     switch (st->state) {
1246     case SITE_RESOLVE:
1247         if (transport_compute_setupinit_peers(st,addrs,naddrs,0)) {
1248             enter_new_state(st,SITE_SENTMSG1);
1249         } else {
1250             /* Can't figure out who to try to to talk to */
1251             slog(st,LOG_SETUP_INIT,
1252                  "key exchange failed: cannot find peer address");
1253             enter_state_run(st);
1254         }
1255         break;
1256     case SITE_SENTMSG1: case SITE_SENTMSG2:
1257     case SITE_SENTMSG3: case SITE_SENTMSG4:
1258     case SITE_SENTMSG5:
1259         if (naddrs) {
1260             /* We start using the address immediately for data too.
1261              * It's best to store it in st->peers now because we might
1262              * go via SENTMSG5, WAIT, and a MSG0, straight into using
1263              * the new key (without updating the data peer addrs). */
1264             transport_resolve_complete(st,addrs,naddrs);
1265         } else if (st->local_mobile) {
1266             /* We can't let this rest because we may have a peer
1267              * address which will break in the future. */
1268             slog(st,LOG_SETUP_INIT,"resolution failed: "
1269                  "abandoning key exchange");
1270             enter_state_wait(st);
1271         } else {
1272             slog(st,LOG_SETUP_INIT,"resolution failed: "
1273                  " continuing to use source address of peer's packets"
1274                  " for key exchange and ultimately data");
1275         }
1276         break;
1277     case SITE_RUN:
1278         if (naddrs) {
1279             slog(st,LOG_SETUP_INIT,"resolution completed tardily,"
1280                  " updating peer address(es)");
1281             transport_resolve_complete_tardy(st,addrs,naddrs);
1282         } else if (st->local_mobile) {
1283             /* Not very good.  We should queue (another) renegotiation
1284              * so that we can update the peer address. */
1285             st->key_renegotiate_time=st->now+st->wait_timeout;
1286         } else {
1287             slog(st,LOG_SETUP_INIT,"resolution failed: "
1288                  " continuing to use source address of peer's packets");
1289         }
1290         break;
1291     case SITE_WAIT:
1292     case SITE_STOP:
1293         /* oh well */
1294         break;
1295     }
1296 }
1297
1298 static bool_t initiate_key_setup(struct site *st, cstring_t reason,
1299                                  const struct comm_addr *prod_hint)
1300 {
1301     /* Reentrancy hazard: can call enter_new_state/enter_state_* */
1302     if (st->state!=SITE_RUN) return False;
1303     slog(st,LOG_SETUP_INIT,"initiating key exchange (%s)",reason);
1304     if (st->addresses) {
1305         slog(st,LOG_SETUP_INIT,"resolving peer address(es)");
1306         return enter_state_resolve(st);
1307     } else if (transport_compute_setupinit_peers(st,0,0,prod_hint)) {
1308         return enter_new_state(st,SITE_SENTMSG1);
1309     }
1310     slog(st,LOG_SETUP_INIT,"key exchange failed: no address for peer");
1311     return False;
1312 }
1313
1314 static void activate_new_key(struct site *st)
1315 {
1316     struct transform_inst_if *t;
1317
1318     /* We have three transform instances, which we swap between old,
1319        active and setup */
1320     t=st->auxiliary_key.transform;
1321     st->auxiliary_key.transform=st->current.transform;
1322     st->current.transform=st->new_transform;
1323     st->new_transform=t;
1324     dispose_transform(&st->new_transform);
1325
1326     st->timeout=0;
1327     st->auxiliary_is_new=0;
1328     st->auxiliary_key.key_timeout=st->current.key_timeout;
1329     st->current.key_timeout=st->now+st->key_lifetime;
1330     st->renegotiate_key_time=st->now+st->key_renegotiate_time;
1331     transport_peers_copy(st,&st->peers,&st->setup_peers);
1332     st->current.remote_session_id=st->setup_session_id;
1333
1334     /* Compute the inter-site MTU.  This is min( our_mtu, their_mtu ).
1335      * But their mtu be unspecified, in which case we just use ours. */
1336     uint32_t intersite_mtu=
1337         MIN(st->mtu_target, st->remote_adv_mtu ?: ~(uint32_t)0);
1338     st->netlink->set_mtu(st->netlink->st,intersite_mtu);
1339
1340     slog(st,LOG_ACTIVATE_KEY,"new key activated"
1341          " (mtu ours=%"PRId32" theirs=%"PRId32" intersite=%"PRId32")",
1342          st->mtu_target, st->remote_adv_mtu, intersite_mtu);
1343     enter_state_run(st);
1344 }
1345
1346 static void delete_one_key(struct site *st, struct data_key *key,
1347                            cstring_t reason, cstring_t which, uint32_t loglevel)
1348 {
1349     if (!is_transform_valid(key->transform)) return;
1350     if (reason) slog(st,loglevel,"%s deleted (%s)",which,reason);
1351     dispose_transform(&key->transform);
1352     key->key_timeout=0;
1353 }
1354
1355 static void delete_keys(struct site *st, cstring_t reason, uint32_t loglevel)
1356 {
1357     if (current_valid(st)) {
1358         slog(st,loglevel,"session closed (%s)",reason);
1359
1360         delete_one_key(st,&st->current,0,0,0);
1361         set_link_quality(st);
1362     }
1363     delete_one_key(st,&st->auxiliary_key,0,0,0);
1364 }
1365
1366 static void state_assert(struct site *st, bool_t ok)
1367 {
1368     if (!ok) fatal("site:state_assert");
1369 }
1370
1371 static void enter_state_stop(struct site *st)
1372 {
1373     st->state=SITE_STOP;
1374     st->timeout=0;
1375     delete_keys(st,"entering state STOP",LOG_TIMEOUT_KEY);
1376     dispose_transform(&st->new_transform);
1377 }
1378
1379 static void set_link_quality(struct site *st)
1380 {
1381     uint32_t quality;
1382     if (current_valid(st))
1383         quality=LINK_QUALITY_UP;
1384     else if (st->state==SITE_WAIT || st->state==SITE_STOP)
1385         quality=LINK_QUALITY_DOWN;
1386     else if (st->addresses)
1387         quality=LINK_QUALITY_DOWN_CURRENT_ADDRESS;
1388     else if (transport_peers_valid(&st->peers))
1389         quality=LINK_QUALITY_DOWN_STALE_ADDRESS;
1390     else
1391         quality=LINK_QUALITY_DOWN;
1392
1393     st->netlink->set_quality(st->netlink->st,quality);
1394 }
1395
1396 static void enter_state_run(struct site *st)
1397 {
1398     slog(st,LOG_STATE,"entering state RUN");
1399     st->state=SITE_RUN;
1400     st->timeout=0;
1401
1402     st->setup_session_id=0;
1403     transport_peers_clear(st,&st->setup_peers);
1404     FILLZERO(st->localN);
1405     FILLZERO(st->remoteN);
1406     dispose_transform(&st->new_transform);
1407     memset(st->dhsecret,0,st->dh->len);
1408     memset(st->sharedsecret,0,st->sharedsecretlen);
1409     set_link_quality(st);
1410 }
1411
1412 static bool_t ensure_resolving(struct site *st)
1413 {
1414     /* Reentrancy hazard: may call site_resolve_callback and hence
1415      * enter_new_state, enter_state_* and generate_msg*. */
1416     if (st->resolving_count)
1417         return True;
1418
1419     assert(st->addresses);
1420
1421     /* resolver->request might reentrantly call site_resolve_callback
1422      * which will decrement st->resolving, so we need to increment it
1423      * twice beforehand to prevent decrement from thinking we're
1424      * finished, and decrement it ourselves.  Alternatively if
1425      * everything fails then there are no callbacks due and we simply
1426      * set it to 0 and return false.. */
1427     st->resolving_n_results_stored=0;
1428     st->resolving_n_results_all=0;
1429     st->resolving_count+=2;
1430     const char **addrp=st->addresses;
1431     const char *address;
1432     bool_t anyok=False;
1433     for (; (address=*addrp++); ) {
1434         bool_t ok = st->resolver->request(st->resolver->st,address,
1435                                           st->remoteport,st->comms[0],
1436                                           site_resolve_callback,st);
1437         if (ok)
1438             st->resolving_count++;
1439         anyok|=ok;
1440     }
1441     if (!anyok) {
1442         st->resolving_count=0;
1443         return False;
1444     }
1445     decrement_resolving_count(st,2);
1446     return True;
1447 }
1448
1449 static bool_t enter_state_resolve(struct site *st)
1450 {
1451     /* Reentrancy hazard!  See ensure_resolving. */
1452     state_assert(st,st->state==SITE_RUN);
1453     slog(st,LOG_STATE,"entering state RESOLVE");
1454     st->state=SITE_RESOLVE;
1455     return ensure_resolving(st);
1456 }
1457
1458 static bool_t enter_new_state(struct site *st, uint32_t next)
1459 {
1460     bool_t (*gen)(struct site *st);
1461     int r;
1462
1463     slog(st,LOG_STATE,"entering state %s",state_name(next));
1464     switch(next) {
1465     case SITE_SENTMSG1:
1466         state_assert(st,st->state==SITE_RUN || st->state==SITE_RESOLVE);
1467         gen=generate_msg1;
1468         break;
1469     case SITE_SENTMSG2:
1470         state_assert(st,st->state==SITE_RUN || st->state==SITE_RESOLVE ||
1471                      st->state==SITE_SENTMSG1 || st->state==SITE_WAIT);
1472         gen=generate_msg2;
1473         break;
1474     case SITE_SENTMSG3:
1475         state_assert(st,st->state==SITE_SENTMSG1);
1476         BUF_FREE(&st->buffer);
1477         gen=generate_msg3;
1478         break;
1479     case SITE_SENTMSG4:
1480         state_assert(st,st->state==SITE_SENTMSG2);
1481         BUF_FREE(&st->buffer);
1482         gen=generate_msg4;
1483         break;
1484     case SITE_SENTMSG5:
1485         state_assert(st,st->state==SITE_SENTMSG3);
1486         BUF_FREE(&st->buffer);
1487         gen=generate_msg5;
1488         break;
1489     case SITE_RUN:
1490         state_assert(st,st->state==SITE_SENTMSG4);
1491         BUF_FREE(&st->buffer);
1492         gen=generate_msg6;
1493         break;
1494     default:
1495         gen=NULL;
1496         fatal("enter_new_state(%s): invalid new state",state_name(next));
1497         break;
1498     }
1499
1500     if (hacky_par_start_failnow()) return False;
1501
1502     r= gen(st) && send_msg(st);
1503
1504     hacky_par_end(&r,
1505                   st->setup_retries, st->setup_retry_interval,
1506                   send_msg, st);
1507     
1508     if (r) {
1509         st->state=next;
1510         if (next==SITE_RUN) {
1511             BUF_FREE(&st->buffer); /* Never reused */
1512             st->timeout=0; /* Never retransmit */
1513             activate_new_key(st);
1514         }
1515         return True;
1516     }
1517     slog(st,LOG_ERROR,"error entering state %s",state_name(next));
1518     st->buffer.free=False; /* Unconditionally use the buffer; it may be
1519                               in either state, and enter_state_wait() will
1520                               do a BUF_FREE() */
1521     enter_state_wait(st);
1522     return False;
1523 }
1524
1525 /* msg7 tells our peer that we're about to forget our key */
1526 static bool_t send_msg7(struct site *st, cstring_t reason)
1527 {
1528     cstring_t transform_err;
1529
1530     if (current_valid(st) && st->buffer.free
1531         && transport_peers_valid(&st->peers)) {
1532         BUF_ALLOC(&st->buffer,"site:MSG7");
1533         buffer_init(&st->buffer,calculate_max_start_pad());
1534         buf_append_uint32(&st->buffer,LABEL_MSG7);
1535         buf_append_string(&st->buffer,reason);
1536         if (call_transform_forwards(st, st->current.transform,
1537                                     &st->buffer, &transform_err))
1538             goto free_out;
1539         buf_prepend_uint32(&st->buffer,LABEL_MSG0);
1540         buf_prepend_uint32(&st->buffer,st->index);
1541         buf_prepend_uint32(&st->buffer,st->current.remote_session_id);
1542         transport_xmit(st,&st->peers,&st->buffer,True);
1543         BUF_FREE(&st->buffer);
1544     free_out:
1545         return True;
1546     }
1547     return False;
1548 }
1549
1550 /* We go into this state if our peer becomes uncommunicative. Similar to
1551    the "stop" state, we forget all session keys for a while, before
1552    re-entering the "run" state. */
1553 static void enter_state_wait(struct site *st)
1554 {
1555     slog(st,LOG_STATE,"entering state WAIT");
1556     st->timeout=st->now+st->wait_timeout;
1557     st->state=SITE_WAIT;
1558     set_link_quality(st);
1559     BUF_FREE(&st->buffer); /* will have had an outgoing packet in it */
1560     /* XXX Erase keys etc. */
1561 }
1562
1563 static void generate_prod(struct site *st, struct buffer_if *buf)
1564 {
1565     buffer_init(buf,0);
1566     buf_append_uint32(buf,0);
1567     buf_append_uint32(buf,0);
1568     buf_append_uint32(buf,LABEL_PROD);
1569     buf_append_string(buf,st->localname);
1570     buf_append_string(buf,st->remotename);
1571 }
1572
1573 static void generate_send_prod(struct site *st,
1574                                const struct comm_addr *source)
1575 {
1576     if (!st->allow_send_prod) return; /* too soon */
1577     if (!(st->state==SITE_RUN || st->state==SITE_RESOLVE ||
1578           st->state==SITE_WAIT)) return; /* we'd ignore peer's MSG1 */
1579
1580     slog(st,LOG_SETUP_INIT,"prodding peer for key exchange");
1581     st->allow_send_prod=0;
1582     generate_prod(st,&st->scratch);
1583     dump_packet(st,&st->scratch,source,False);
1584     source->comm->sendmsg(source->comm->st, &st->scratch, source);
1585 }
1586
1587 static inline void site_settimeout(uint64_t timeout, int *timeout_io)
1588 {
1589     if (timeout) {
1590         int64_t offset=timeout-*now;
1591         if (offset<0) offset=0;
1592         if (offset>INT_MAX) offset=INT_MAX;
1593         if (*timeout_io<0 || offset<*timeout_io)
1594             *timeout_io=offset;
1595     }
1596 }
1597
1598 static int site_beforepoll(void *sst, struct pollfd *fds, int *nfds_io,
1599                            int *timeout_io)
1600 {
1601     struct site *st=sst;
1602
1603     BEFOREPOLL_WANT_FDS(0); /* We don't use any file descriptors */
1604     st->now=*now;
1605
1606     /* Work out when our next timeout is. The earlier of 'timeout' or
1607        'current.key_timeout'. A stored value of '0' indicates no timeout
1608        active. */
1609     site_settimeout(st->timeout, timeout_io);
1610     site_settimeout(st->current.key_timeout, timeout_io);
1611     site_settimeout(st->auxiliary_key.key_timeout, timeout_io);
1612
1613     return 0; /* success */
1614 }
1615
1616 static void check_expiry(struct site *st, struct data_key *key,
1617                          const char *which)
1618 {
1619     if (key->key_timeout && *now>key->key_timeout) {
1620         delete_one_key(st,key,"maximum life exceeded",which,LOG_TIMEOUT_KEY);
1621     }
1622 }
1623
1624 /* NB site_afterpoll will be called before site_beforepoll is ever called */
1625 static void site_afterpoll(void *sst, struct pollfd *fds, int nfds)
1626 {
1627     struct site *st=sst;
1628
1629     st->now=*now;
1630     if (st->timeout && *now>st->timeout) {
1631         st->timeout=0;
1632         if (st->state>=SITE_SENTMSG1 && st->state<=SITE_SENTMSG5) {
1633             if (!hacky_par_start_failnow())
1634                 send_msg(st);
1635         } else if (st->state==SITE_WAIT) {
1636             enter_state_run(st);
1637         } else {
1638             slog(st,LOG_ERROR,"site_afterpoll: unexpected timeout, state=%d",
1639                  st->state);
1640         }
1641     }
1642     check_expiry(st,&st->current,"current key");
1643     check_expiry(st,&st->auxiliary_key,"auxiliary key");
1644 }
1645
1646 /* This function is called by the netlink device to deliver packets
1647    intended for the remote network. The packet is in "raw" wire
1648    format, but is guaranteed to be word-aligned. */
1649 static void site_outgoing(void *sst, struct buffer_if *buf)
1650 {
1651     struct site *st=sst;
1652     cstring_t transform_err;
1653     
1654     if (st->state==SITE_STOP) {
1655         BUF_FREE(buf);
1656         return;
1657     }
1658
1659     st->allow_send_prod=1;
1660
1661     /* In all other states we consider delivering the packet if we have
1662        a valid key and a valid address to send it to. */
1663     if (current_valid(st) && transport_peers_valid(&st->peers)) {
1664         /* Transform it and send it */
1665         if (buf->size>0) {
1666             buf_prepend_uint32(buf,LABEL_MSG9);
1667             if (call_transform_forwards(st, st->current.transform,
1668                                         buf, &transform_err))
1669                 goto free_out;
1670             buf_prepend_uint32(buf,LABEL_MSG0);
1671             buf_prepend_uint32(buf,st->index);
1672             buf_prepend_uint32(buf,st->current.remote_session_id);
1673             transport_xmit(st,&st->peers,buf,False);
1674         }
1675     free_out:
1676         BUF_FREE(buf);
1677         return;
1678     }
1679
1680     slog(st,LOG_DROP,"discarding outgoing packet of size %d",buf->size);
1681     BUF_FREE(buf);
1682     initiate_key_setup(st,"outgoing packet",0);
1683 }
1684
1685 static bool_t named_for_us(struct site *st, const struct buffer_if *buf_in,
1686                            uint32_t type, struct msg *m)
1687     /* For packets which are identified by the local and remote names.
1688      * If it has our name and our peer's name in it it's for us. */
1689 {
1690     struct buffer_if buf[1];
1691     buffer_readonly_clone(buf,buf_in);
1692     return unpick_msg(st,type,buf,m)
1693         && name_matches(&m->remote,st->remotename)
1694         && name_matches(&m->local,st->localname);
1695 }
1696
1697 /* This function is called by the communication device to deliver
1698    packets from our peers.
1699    It should return True if the packet is recognised as being for
1700    this current site instance (and should therefore not be processed
1701    by other sites), even if the packet was otherwise ignored. */
1702 static bool_t site_incoming(void *sst, struct buffer_if *buf,
1703                             const struct comm_addr *source)
1704 {
1705     struct site *st=sst;
1706
1707     if (buf->size < 12) return False;
1708
1709     uint32_t dest=get_uint32(buf->start);
1710     uint32_t msgtype=get_uint32(buf->start+8);
1711     struct msg named_msg;
1712
1713     if (msgtype==LABEL_MSG1) {
1714         if (!named_for_us(st,buf,msgtype,&named_msg))
1715             return False;
1716         /* It's a MSG1 addressed to us. Decide what to do about it. */
1717         dump_packet(st,buf,source,True);
1718         if (st->state==SITE_RUN || st->state==SITE_RESOLVE ||
1719             st->state==SITE_WAIT) {
1720             /* We should definitely process it */
1721             transport_compute_setupinit_peers(st,0,0,source);
1722             if (process_msg1(st,buf,source,&named_msg)) {
1723                 slog(st,LOG_SETUP_INIT,"key setup initiated by peer");
1724                 bool_t entered=enter_new_state(st,SITE_SENTMSG2);
1725                 if (entered && st->addresses && st->local_mobile)
1726                     /* We must do this as the very last thing, because
1727                        the resolver callback might reenter us. */
1728                     ensure_resolving(st);
1729             } else {
1730                 slog(st,LOG_ERROR,"failed to process incoming msg1");
1731             }
1732             BUF_FREE(buf);
1733             return True;
1734         } else if (st->state==SITE_SENTMSG1) {
1735             /* We've just sent a message 1! They may have crossed on
1736                the wire. If we have priority then we ignore the
1737                incoming one, otherwise we process it as usual. */
1738             if (st->setup_priority) {
1739                 BUF_FREE(buf);
1740                 slog(st,LOG_DUMP,"crossed msg1s; we are higher "
1741                      "priority => ignore incoming msg1");
1742                 return True;
1743             } else {
1744                 slog(st,LOG_DUMP,"crossed msg1s; we are lower "
1745                      "priority => use incoming msg1");
1746                 if (process_msg1(st,buf,source,&named_msg)) {
1747                     BUF_FREE(&st->buffer); /* Free our old message 1 */
1748                     transport_setup_msgok(st,source);
1749                     enter_new_state(st,SITE_SENTMSG2);
1750                 } else {
1751                     slog(st,LOG_ERROR,"failed to process an incoming "
1752                          "crossed msg1 (we have low priority)");
1753                 }
1754                 BUF_FREE(buf);
1755                 return True;
1756             }
1757         }
1758         /* The message 1 was received at an unexpected stage of the
1759            key setup. XXX POLICY - what do we do? */
1760         slog(st,LOG_UNEXPECTED,"unexpected incoming message 1");
1761         BUF_FREE(buf);
1762         return True;
1763     }
1764     if (msgtype==LABEL_PROD) {
1765         if (!named_for_us(st,buf,msgtype,&named_msg))
1766             return False;
1767         dump_packet(st,buf,source,True);
1768         if (st->state!=SITE_RUN) {
1769             slog(st,LOG_DROP,"ignoring PROD when not in state RUN");
1770         } else if (current_valid(st)) {
1771             slog(st,LOG_DROP,"ignoring PROD when we think we have a key");
1772         } else {
1773             initiate_key_setup(st,"peer sent PROD packet",source);
1774         }
1775         BUF_FREE(buf);
1776         return True;
1777     }
1778     if (dest==st->index) {
1779         /* Explicitly addressed to us */
1780         if (msgtype!=LABEL_MSG0) dump_packet(st,buf,source,True);
1781         switch (msgtype) {
1782         case LABEL_NAK:
1783             /* If the source is our current peer then initiate a key setup,
1784                because our peer's forgotten the key */
1785             if (get_uint32(buf->start+4)==st->current.remote_session_id) {
1786                 bool_t initiated;
1787                 initiated = initiate_key_setup(st,"received a NAK",source);
1788                 if (!initiated) generate_send_prod(st,source);
1789             } else {
1790                 slog(st,LOG_SEC,"bad incoming NAK");
1791             }
1792             break;
1793         case LABEL_MSG0:
1794             process_msg0(st,buf,source);
1795             break;
1796         case LABEL_MSG1:
1797             /* Setup packet: should not have been explicitly addressed
1798                to us */
1799             slog(st,LOG_SEC,"incoming explicitly addressed msg1");
1800             break;
1801         case LABEL_MSG2:
1802             /* Setup packet: expected only in state SENTMSG1 */
1803             if (st->state!=SITE_SENTMSG1) {
1804                 slog(st,LOG_UNEXPECTED,"unexpected MSG2");
1805             } else if (process_msg2(st,buf,source)) {
1806                 transport_setup_msgok(st,source);
1807                 enter_new_state(st,SITE_SENTMSG3);
1808             } else {
1809                 slog(st,LOG_SEC,"invalid MSG2");
1810             }
1811             break;
1812         case LABEL_MSG3:
1813         case LABEL_MSG3BIS:
1814             /* Setup packet: expected only in state SENTMSG2 */
1815             if (st->state!=SITE_SENTMSG2) {
1816                 slog(st,LOG_UNEXPECTED,"unexpected MSG3");
1817             } else if (process_msg3(st,buf,source,msgtype)) {
1818                 transport_setup_msgok(st,source);
1819                 enter_new_state(st,SITE_SENTMSG4);
1820             } else {
1821                 slog(st,LOG_SEC,"invalid MSG3");
1822             }
1823             break;
1824         case LABEL_MSG4:
1825             /* Setup packet: expected only in state SENTMSG3 */
1826             if (st->state!=SITE_SENTMSG3) {
1827                 slog(st,LOG_UNEXPECTED,"unexpected MSG4");
1828             } else if (process_msg4(st,buf,source)) {
1829                 transport_setup_msgok(st,source);
1830                 enter_new_state(st,SITE_SENTMSG5);
1831             } else {
1832                 slog(st,LOG_SEC,"invalid MSG4");
1833             }
1834             break;
1835         case LABEL_MSG5:
1836             /* Setup packet: expected only in state SENTMSG4 */
1837             /* (may turn up in state RUN if our return MSG6 was lost
1838                and the new key has already been activated. In that
1839                case we discard it. The peer will realise that we
1840                are using the new key when they see our data packets.
1841                Until then the peer's data packets to us get discarded. */
1842             if (st->state==SITE_SENTMSG4) {
1843                 if (process_msg5(st,buf,source,st->new_transform)) {
1844                     transport_setup_msgok(st,source);
1845                     enter_new_state(st,SITE_RUN);
1846                 } else {
1847                     slog(st,LOG_SEC,"invalid MSG5");
1848                 }
1849             } else if (st->state==SITE_RUN) {
1850                 if (process_msg5(st,buf,source,st->current.transform)) {
1851                     slog(st,LOG_DROP,"got MSG5, retransmitting MSG6");
1852                     transport_setup_msgok(st,source);
1853                     create_msg6(st,st->current.transform,
1854                                 st->current.remote_session_id);
1855                     transport_xmit(st,&st->peers,&st->buffer,True);
1856                     BUF_FREE(&st->buffer);
1857                 } else {
1858                     slog(st,LOG_SEC,"invalid MSG5 (in state RUN)");
1859                 }
1860             } else {
1861                 slog(st,LOG_UNEXPECTED,"unexpected MSG5");
1862             }
1863             break;
1864         case LABEL_MSG6:
1865             /* Setup packet: expected only in state SENTMSG5 */
1866             if (st->state!=SITE_SENTMSG5) {
1867                 slog(st,LOG_UNEXPECTED,"unexpected MSG6");
1868             } else if (process_msg6(st,buf,source)) {
1869                 BUF_FREE(&st->buffer); /* Free message 5 */
1870                 transport_setup_msgok(st,source);
1871                 activate_new_key(st);
1872             } else {
1873                 slog(st,LOG_SEC,"invalid MSG6");
1874             }
1875             break;
1876         default:
1877             slog(st,LOG_SEC,"received message of unknown type 0x%08x",
1878                  msgtype);
1879             break;
1880         }
1881         BUF_FREE(buf);
1882         return True;
1883     }
1884
1885     return False;
1886 }
1887
1888 static void site_control(void *vst, bool_t run)
1889 {
1890     struct site *st=vst;
1891     if (run) enter_state_run(st);
1892     else enter_state_stop(st);
1893 }
1894
1895 static void site_phase_hook(void *sst, uint32_t newphase)
1896 {
1897     struct site *st=sst;
1898
1899     /* The program is shutting down; tell our peer */
1900     send_msg7(st,"shutting down");
1901 }
1902
1903 static list_t *site_apply(closure_t *self, struct cloc loc, dict_t *context,
1904                           list_t *args)
1905 {
1906     static uint32_t index_sequence;
1907     struct site *st;
1908     item_t *item;
1909     dict_t *dict;
1910     int i;
1911
1912     st=safe_malloc(sizeof(*st),"site_apply");
1913
1914     st->cl.description="site";
1915     st->cl.type=CL_SITE;
1916     st->cl.apply=NULL;
1917     st->cl.interface=&st->ops;
1918     st->ops.st=st;
1919     st->ops.control=site_control;
1920     st->ops.status=site_status;
1921
1922     /* First parameter must be a dict */
1923     item=list_elem(args,0);
1924     if (!item || item->type!=t_dict)
1925         cfgfatal(loc,"site","parameter must be a dictionary\n");
1926     
1927     dict=item->data.dict;
1928     st->localname=dict_read_string(dict, "local-name", True, "site", loc);
1929     st->remotename=dict_read_string(dict, "name", True, "site", loc);
1930
1931     st->peer_mobile=dict_read_bool(dict,"mobile",False,"site",loc,False);
1932     st->local_mobile=
1933         dict_read_bool(dict,"local-mobile",False,"site",loc,False);
1934
1935     /* Sanity check (which also allows the 'sites' file to include
1936        site() closures for all sites including our own): refuse to
1937        talk to ourselves */
1938     if (strcmp(st->localname,st->remotename)==0) {
1939         Message(M_DEBUG,"site %s: local-name==name -> ignoring this site\n",
1940                 st->localname);
1941         if (st->peer_mobile != st->local_mobile)
1942             cfgfatal(loc,"site","site %s's peer-mobile=%d"
1943                     " but our local-mobile=%d\n",
1944                     st->localname, st->peer_mobile, st->local_mobile);
1945         free(st);
1946         return NULL;
1947     }
1948     if (st->peer_mobile && st->local_mobile) {
1949         Message(M_WARNING,"site %s: site is mobile but so are we"
1950                 " -> ignoring this site\n", st->remotename);
1951         free(st);
1952         return NULL;
1953     }
1954
1955     assert(index_sequence < 0xffffffffUL);
1956     st->index = ++index_sequence;
1957     st->local_capabilities = 0;
1958     st->netlink=find_cl_if(dict,"link",CL_NETLINK,True,"site",loc);
1959
1960 #define GET_CLOSURE_LIST(dictkey,things,nthings,CL_TYPE) do{            \
1961     list_t *things##_cfg=dict_lookup(dict,dictkey);                     \
1962     if (!things##_cfg)                                                  \
1963         cfgfatal(loc,"site","closure list \"%s\" not found\n",dictkey); \
1964     st->nthings=list_length(things##_cfg);                              \
1965     st->things=safe_malloc_ary(sizeof(*st->things),st->nthings,dictkey "s"); \
1966     assert(st->nthings);                                                \
1967     for (i=0; i<st->nthings; i++) {                                     \
1968         item_t *item=list_elem(things##_cfg,i);                         \
1969         if (item->type!=t_closure)                                      \
1970             cfgfatal(loc,"site","%s is not a closure\n",dictkey);       \
1971         closure_t *cl=item->data.closure;                               \
1972         if (cl->type!=CL_TYPE)                                          \
1973             cfgfatal(loc,"site","%s closure wrong type\n",dictkey);     \
1974         st->things[i]=cl->interface;                                    \
1975     }                                                                   \
1976 }while(0)
1977
1978     GET_CLOSURE_LIST("comm",comms,ncomms,CL_COMM);
1979
1980     st->resolver=find_cl_if(dict,"resolver",CL_RESOLVER,True,"site",loc);
1981     st->log=find_cl_if(dict,"log",CL_LOG,True,"site",loc);
1982     st->random=find_cl_if(dict,"random",CL_RANDOMSRC,True,"site",loc);
1983
1984     st->privkey=find_cl_if(dict,"local-key",CL_RSAPRIVKEY,True,"site",loc);
1985     st->addresses=dict_read_string_array(dict,"address",False,"site",loc,0);
1986     if (st->addresses)
1987         st->remoteport=dict_read_number(dict,"port",True,"site",loc,0);
1988     else st->remoteport=0;
1989     st->pubkey=find_cl_if(dict,"key",CL_RSAPUBKEY,True,"site",loc);
1990
1991     GET_CLOSURE_LIST("transform",transforms,ntransforms,CL_TRANSFORM);
1992
1993     st->dh=find_cl_if(dict,"dh",CL_DH,True,"site",loc);
1994     st->hash=find_cl_if(dict,"hash",CL_HASH,True,"site",loc);
1995
1996 #define DEFAULT(D) (st->peer_mobile || st->local_mobile \
1997                     ? DEFAULT_MOBILE_##D : DEFAULT_##D)
1998 #define CFG_NUMBER(k,D) dict_read_number(dict,(k),False,"site",loc,DEFAULT(D));
1999
2000     st->key_lifetime=         CFG_NUMBER("key-lifetime",  KEY_LIFETIME);
2001     st->setup_retries=        CFG_NUMBER("setup-retries", SETUP_RETRIES);
2002     st->setup_retry_interval= CFG_NUMBER("setup-timeout", SETUP_RETRY_INTERVAL);
2003     st->wait_timeout=         CFG_NUMBER("wait-time",     WAIT_TIME);
2004     st->mtu_target= dict_read_number(dict,"mtu-target",False,"site",loc,0);
2005
2006     st->mobile_peer_expiry= dict_read_number(
2007        dict,"mobile-peer-expiry",False,"site",loc,DEFAULT_MOBILE_PEER_EXPIRY);
2008
2009     const char *peerskey= st->peer_mobile
2010         ? "mobile-peers-max" : "static-peers-max";
2011     st->transport_peers_max= dict_read_number(
2012         dict,peerskey,False,"site",loc, st->addresses ? 4 : 3);
2013     if (st->transport_peers_max<1 ||
2014         st->transport_peers_max>MAX_PEER_ADDRS) {
2015         cfgfatal(loc,"site", "%s must be in range 1.."
2016                  STRING(MAX_PEER_ADDRS) "\n", peerskey);
2017     }
2018
2019     if (st->key_lifetime < DEFAULT(KEY_RENEGOTIATE_GAP)*2)
2020         st->key_renegotiate_time=st->key_lifetime/2;
2021     else
2022         st->key_renegotiate_time=st->key_lifetime-DEFAULT(KEY_RENEGOTIATE_GAP);
2023     st->key_renegotiate_time=dict_read_number(
2024         dict,"renegotiate-time",False,"site",loc,st->key_renegotiate_time);
2025     if (st->key_renegotiate_time > st->key_lifetime) {
2026         cfgfatal(loc,"site",
2027                  "renegotiate-time must be less than key-lifetime\n");
2028     }
2029
2030     st->log_events=string_list_to_word(dict_lookup(dict,"log-events"),
2031                                        log_event_table,"site");
2032
2033     st->resolving_count=0;
2034     st->allow_send_prod=0;
2035
2036     st->tunname=safe_malloc(strlen(st->localname)+strlen(st->remotename)+5,
2037                             "site_apply");
2038     sprintf(st->tunname,"%s<->%s",st->localname,st->remotename);
2039
2040     /* The information we expect to see in incoming messages of type 1 */
2041     /* fixme: lots of unchecked overflows here, but the results are only
2042        corrupted packets rather than undefined behaviour */
2043     st->setup_priority=(strcmp(st->localname,st->remotename)>0);
2044
2045     buffer_new(&st->buffer,SETUP_BUFFER_LEN);
2046
2047     buffer_new(&st->scratch,SETUP_BUFFER_LEN);
2048     BUF_ALLOC(&st->scratch,"site:scratch");
2049
2050     /* We are interested in poll(), but only for timeouts. We don't have
2051        any fds of our own. */
2052     register_for_poll(st, site_beforepoll, site_afterpoll, "site");
2053     st->timeout=0;
2054
2055     st->remote_capabilities=0;
2056     st->chosen_transform=0;
2057     st->current.key_timeout=0;
2058     st->auxiliary_key.key_timeout=0;
2059     transport_peers_clear(st,&st->peers);
2060     transport_peers_clear(st,&st->setup_peers);
2061     /* XXX mlock these */
2062     st->dhsecret=safe_malloc(st->dh->len,"site:dhsecret");
2063     st->sharedsecretlen=st->sharedsecretallocd=0;
2064     st->sharedsecret=0;
2065
2066     for (i=0; i<st->ntransforms; i++) {
2067         struct transform_if *ti=st->transforms[i];
2068         uint32_t capbit = 1UL << ti->capab_transformnum;
2069         if (st->local_capabilities & capbit)
2070             slog(st,LOG_ERROR,"transformnum capability bit"
2071                  " %d (%#"PRIx32") reused", ti->capab_transformnum, capbit);
2072         st->local_capabilities |= capbit;
2073     }
2074
2075     /* We need to register the remote networks with the netlink device */
2076     uint32_t netlink_mtu; /* local virtual interface mtu */
2077     st->netlink->reg(st->netlink->st, site_outgoing, st, &netlink_mtu);
2078     if (!st->mtu_target)
2079         st->mtu_target=netlink_mtu;
2080     
2081     for (i=0; i<st->ncomms; i++)
2082         st->comms[i]->request_notify(st->comms[i]->st, st, site_incoming);
2083
2084     st->current.transform=0;
2085     st->auxiliary_key.transform=0;
2086     st->new_transform=0;
2087     st->auxiliary_is_new=0;
2088
2089     enter_state_stop(st);
2090
2091     add_hook(PHASE_SHUTDOWN,site_phase_hook,st);
2092
2093     return new_closure(&st->cl);
2094 }
2095
2096 void site_module(dict_t *dict)
2097 {
2098     add_closure(dict,"site",site_apply);
2099 }
2100
2101
2102 /***** TRANSPORT PEERS definitions *****/
2103
2104 static void transport_peers_debug(struct site *st, transport_peers *dst,
2105                                   const char *didwhat,
2106                                   int nargs, const struct comm_addr *args,
2107                                   size_t stride) {
2108     int i;
2109     char *argp;
2110
2111     if (!(st->log_events & LOG_PEER_ADDRS))
2112         return; /* an optimisation */
2113
2114     slog(st, LOG_PEER_ADDRS, "peers (%s) %s nargs=%d => npeers=%d",
2115          (dst==&st->peers ? "data" :
2116           dst==&st->setup_peers ? "setup" : "UNKNOWN"),
2117          didwhat, nargs, dst->npeers);
2118
2119     for (i=0, argp=(void*)args;
2120          i<nargs;
2121          i++, (argp+=stride?stride:sizeof(*args))) {
2122         const struct comm_addr *ca=(void*)argp;
2123         slog(st, LOG_PEER_ADDRS, " args: addrs[%d]=%s",
2124              i, comm_addr_to_string(ca));
2125     }
2126     for (i=0; i<dst->npeers; i++) {
2127         struct timeval diff;
2128         timersub(tv_now,&dst->peers[i].last,&diff);
2129         const struct comm_addr *ca=&dst->peers[i].addr;
2130         slog(st, LOG_PEER_ADDRS, " peers: addrs[%d]=%s T-%ld.%06ld",
2131              i, comm_addr_to_string(ca),
2132              (unsigned long)diff.tv_sec, (unsigned long)diff.tv_usec);
2133     }
2134 }
2135
2136 static void transport_peers_expire(struct site *st, transport_peers *peers) {
2137     /* peers must be sorted first */
2138     int previous_peers=peers->npeers;
2139     struct timeval oldest;
2140     oldest.tv_sec  = tv_now->tv_sec - st->mobile_peer_expiry;
2141     oldest.tv_usec = tv_now->tv_usec;
2142     while (peers->npeers>1 &&
2143            timercmp(&peers->peers[peers->npeers-1].last, &oldest, <))
2144         peers->npeers--;
2145     if (peers->npeers != previous_peers)
2146         transport_peers_debug(st,peers,"expire", 0,0,0);
2147 }
2148
2149 static bool_t transport_peer_record_one(struct site *st, transport_peers *peers,
2150                                         const struct comm_addr *ca,
2151                                         const struct timeval *tv) {
2152     /* returns false if output is full */
2153     int search;
2154
2155     if (peers->npeers >= st->transport_peers_max)
2156         return 0;
2157
2158     for (search=0; search<peers->npeers; search++)
2159         if (comm_addr_equal(&peers->peers[search].addr, ca))
2160             return 1;
2161
2162     peers->peers[peers->npeers].addr = *ca;
2163     peers->peers[peers->npeers].last = *tv;
2164     peers->npeers++;
2165     return 1;
2166 }
2167
2168 static void transport_record_peers(struct site *st, transport_peers *peers,
2169                                    const struct comm_addr *addrs, int naddrs,
2170                                    const char *m) {
2171     /* We add addrs into peers.  The new entries end up at the front
2172      * and displace entries towards the end (perhaps even off the
2173      * end).  Any existing matching entries are moved up to the front.
2174      *
2175      * Caller must first call transport_peers_expire. */
2176
2177     if (naddrs==1 && peers->npeers>=1 &&
2178         comm_addr_equal(&addrs[0], &peers->peers[0].addr)) {
2179         /* optimisation, also avoids debug for trivial updates */
2180         peers->peers[0].last = *tv_now;
2181         return;
2182     }
2183
2184     int old_npeers=peers->npeers;
2185     transport_peer old_peers[old_npeers];
2186     COPY_ARRAY(old_peers,peers->peers,old_npeers);
2187
2188     peers->npeers=0;
2189     int i;
2190     for (i=0; i<naddrs; i++) {
2191         if (!transport_peer_record_one(st,peers, &addrs[i], tv_now))
2192             break;
2193     }
2194     for (i=0; i<old_npeers; i++) {
2195         const transport_peer *old=&old_peers[i];
2196         if (!transport_peer_record_one(st,peers, &old->addr, &old->last))
2197             break;
2198     }
2199
2200     transport_peers_debug(st,peers,m, naddrs,addrs,0);
2201 }
2202
2203 static void transport_expire_record_peers(struct site *st,
2204                                           transport_peers *peers,
2205                                           const struct comm_addr *addrs,
2206                                           int naddrs, const char *m) {
2207     /* Convenience function */
2208     transport_peers_expire(st,peers);
2209     transport_record_peers(st,peers,addrs,naddrs,m);
2210 }
2211
2212 static bool_t transport_compute_setupinit_peers(struct site *st,
2213         const struct comm_addr *configured_addrs /* 0 if none or not found */,
2214         int n_configured_addrs /* 0 if none or not found */,
2215         const struct comm_addr *incoming_packet_addr /* 0 if none */) {
2216     if (!n_configured_addrs && !incoming_packet_addr &&
2217         !transport_peers_valid(&st->peers))
2218         return False;
2219
2220     slog(st,LOG_SETUP_INIT,
2221          "using: %d configured addr(s);%s %d old peer addrs(es)",
2222          n_configured_addrs,
2223          incoming_packet_addr ? " incoming packet address;" : "",
2224          st->peers.npeers);
2225
2226     /* Non-mobile peers try addresses until one is plausible.  The
2227      * effect is that this code always tries first the configured
2228      * address if supplied, or otherwise the address of the incoming
2229      * PROD, or finally the existing data peer if one exists; this is
2230      * as desired. */
2231
2232     transport_peers_copy(st,&st->setup_peers,&st->peers);
2233     transport_peers_expire(st,&st->setup_peers);
2234
2235     if (incoming_packet_addr)
2236         transport_record_peers(st,&st->setup_peers,
2237                                incoming_packet_addr,1, "incoming");
2238
2239     if (n_configured_addrs)
2240         transport_record_peers(st,&st->setup_peers,
2241                               configured_addrs,n_configured_addrs, "setupinit");
2242
2243     assert(transport_peers_valid(&st->setup_peers));
2244     return True;
2245 }
2246
2247 static void transport_setup_msgok(struct site *st, const struct comm_addr *a) {
2248     if (st->peer_mobile)
2249         transport_expire_record_peers(st,&st->setup_peers,a,1,"setupmsg");
2250 }
2251 static void transport_data_msgok(struct site *st, const struct comm_addr *a) {
2252     if (st->peer_mobile)
2253         transport_expire_record_peers(st,&st->peers,a,1,"datamsg");
2254 }
2255
2256 static int transport_peers_valid(transport_peers *peers) {
2257     return peers->npeers;
2258 }
2259 static void transport_peers_clear(struct site *st, transport_peers *peers) {
2260     peers->npeers= 0;
2261     transport_peers_debug(st,peers,"clear",0,0,0);
2262 }
2263 static void transport_peers_copy(struct site *st, transport_peers *dst,
2264                                  const transport_peers *src) {
2265     dst->npeers=src->npeers;
2266     COPY_ARRAY(dst->peers, src->peers, dst->npeers);
2267     transport_peers_debug(st,dst,"copy",
2268                           src->npeers, &src->peers->addr, sizeof(*src->peers));
2269 }
2270
2271 static void transport_resolve_complete(struct site *st,
2272                                        const struct comm_addr *addrs,
2273                                        int naddrs) {
2274     transport_expire_record_peers(st,&st->peers,addrs,naddrs,
2275                                   "resolved data");
2276     transport_expire_record_peers(st,&st->setup_peers,addrs,naddrs,
2277                                   "resolved setup");
2278 }
2279
2280 static void transport_resolve_complete_tardy(struct site *st,
2281                                              const struct comm_addr *addrs,
2282                                              int naddrs) {
2283     transport_expire_record_peers(st,&st->peers,addrs,naddrs,
2284                                   "resolved tardily");
2285 }
2286
2287 static void transport_peers__copy_by_mask(transport_peer *out, int *nout_io,
2288                                           unsigned mask,
2289                                           const transport_peers *inp) {
2290     /* out and in->peers may be the same region, or nonoverlapping */
2291     const transport_peer *in=inp->peers;
2292     int slot;
2293     for (slot=0; slot<inp->npeers; slot++) {
2294         if (!(mask & (1U << slot)))
2295             continue;
2296         if (!(out==in && slot==*nout_io))
2297             COPY_OBJ(out[*nout_io], in[slot]);
2298         (*nout_io)++;
2299     }
2300 }
2301
2302 void transport_xmit(struct site *st, transport_peers *peers,
2303                     struct buffer_if *buf, bool_t candebug) {
2304     int slot;
2305     transport_peers_expire(st, peers);
2306     unsigned failed=0; /* bitmask */
2307     assert(MAX_PEER_ADDRS < sizeof(unsigned)*CHAR_BIT);
2308
2309     int nfailed=0;
2310     for (slot=0; slot<peers->npeers; slot++) {
2311         transport_peer *peer=&peers->peers[slot];
2312         if (candebug)
2313             dump_packet(st, buf, &peer->addr, False);
2314         bool_t ok =
2315             peer->addr.comm->sendmsg(peer->addr.comm->st, buf, &peer->addr);
2316         if (!ok) {
2317             failed |= 1U << slot;
2318             nfailed++;
2319         }
2320         if (ok && !st->peer_mobile)
2321             break;
2322     }
2323     /* Now we need to demote/delete failing addrs: if we are mobile we
2324      * merely demote them; otherwise we delete them. */
2325     if (st->local_mobile) {
2326         unsigned expected = ((1U << nfailed)-1) << (peers->npeers-nfailed);
2327         /* `expected' has all the failures at the end already */
2328         if (failed != expected) {
2329             int fslot=0;
2330             transport_peer failedpeers[nfailed];
2331             transport_peers__copy_by_mask(failedpeers, &fslot, failed,peers);
2332             assert(fslot == nfailed);
2333             int wslot=0;
2334             transport_peers__copy_by_mask(peers->peers,&wslot,~failed,peers);
2335             assert(wslot+nfailed == peers->npeers);
2336             COPY_ARRAY(peers->peers+wslot, failedpeers, nfailed);
2337         }
2338     } else {
2339         if (failed && peers->npeers > 1) {
2340             int wslot=0;
2341             transport_peers__copy_by_mask(peers->peers,&wslot,~failed,peers);
2342             peers->npeers=wslot;
2343         }
2344     }
2345 }
2346
2347 /***** END of transport peers declarations *****/