site: Make return value of transforms be an enum

We are going to need to distinguish more cases.  It was always bad to
have these hardcoded values.

transform_apply_seqrange is, right now, returned even when the problem
is that the packet is recent but is a duplicate.  This is wrong.

No functional change.

Signed-off-by: Ian Jackson <ijackson@chiark.greenend.org.uk>

diff --git a/secnet.h b/secnet.h
index fe1f484ec455983a4331d25226329d6c0da26879..361c49e4e364c21deb68574078c0a48839207018 100644 (file)
--- a/secnet.h
+++ b/secnet.h
@@ -543,13 +543,16 @@ typedef bool_t transform_setkey_fn(void *st, uint8_t *key, int32_t keylen,
 typedef bool_t transform_valid_fn(void *st); /* 0: no key; 1: ok */
 typedef void transform_delkey_fn(void *st);
 typedef void transform_destroyinstance_fn(void *st);
-/* Returns:
- *   0: all is well
- *   1: for any other problem
- *   2: message decrypted but sequence number was out of range
- */
-typedef uint32_t transform_apply_fn(void *st, struct buffer_if *buf,
-                                   const char **errmsg);
+
+typedef enum {
+    transform_apply_ok       = 0, /* all is well (everyone may assume==0) */
+    transform_apply_err      = 1, /* any other problem */
+    transform_apply_seqrange = 2,
+        /* message decrypted but sequence number was out of range */
+} transform_apply_return;
+
+typedef transform_apply_return transform_apply_fn(void *st,
+        struct buffer_if *buf, const char **errmsg);
 
 struct transform_inst_if {
     void *st;

diff --git a/site.c b/site.c
index db1452394bf67b7f173cb8133dd2793c573f24eb..0db109f239ebf9bab6ae84b163e6ce864aa3fd94 100644 (file)
--- a/site.c
+++ b/site.c
@@ -470,14 +470,15 @@ static bool_t current_valid(struct site *st)
 }
 
 #define DEFINE_CALL_TRANSFORM(fwdrev)                                  \
-static int call_transform_##fwdrev(struct site *st,                    \
+static transform_apply_return                                           \
+call_transform_##fwdrev(struct site *st,                               \
                                   struct transform_inst_if *transform, \
                                   struct buffer_if *buf,               \
                                   const char **errmsg)                 \
 {                                                                      \
     if (!is_transform_valid(transform)) {                              \
        *errmsg="transform not set up";                                 \
-       return 1;                                                       \
+       return transform_apply_err;                                     \
     }                                                                  \
     return transform->fwdrev(transform->st,buf,errmsg);                        \
 }
@@ -1029,8 +1030,9 @@ static void create_msg6(struct site *st, struct transform_inst_if *transform,
     /* Give the netlink code an opportunity to put its own stuff in the
        message (configuration information, etc.) */
     buf_prepend_uint32(&st->buffer,LABEL_MSG6);
-    int problem = call_transform_forwards(st,transform,
-                                         &st->buffer,&transform_err);
+    transform_apply_return problem =
+       call_transform_forwards(st,transform,
+                               &st->buffer,&transform_err);
     assert(!problem);
     buf_prepend_uint32(&st->buffer,LABEL_MSG6);
     buf_prepend_uint32(&st->buffer,st->index);
@@ -1075,7 +1077,7 @@ static bool_t decrypt_msg0(struct site *st, struct buffer_if *msg0,
 {
     cstring_t transform_err, auxkey_err, newkey_err="n/a";
     struct msg0 m;
-    uint32_t problem;
+    transform_apply_return problem;
 
     if (!unpick_msg0(st,msg0,&m)) return False;
 
@@ -1090,13 +1092,13 @@ static bool_t decrypt_msg0(struct site *st, struct buffer_if *msg0,
                           "peer has used new key","auxiliary key",LOG_SEC);
        return True;
     }
-    if (problem==2)
+    if (problem==transform_apply_seqrange)
        goto skew;
 
     buffer_copy(msg0, &st->scratch);
     problem = call_transform_reverse(st,st->auxiliary_key.transform,
                                     msg0,&auxkey_err);
-    if (problem==0) {
+    if (!problem) {
        slog(st,LOG_DROP,"processing packet which uses auxiliary key");
        if (st->auxiliary_is_new) {
            /* We previously timed out in state SENTMSG5 but it turns
@@ -1115,7 +1117,7 @@ static bool_t decrypt_msg0(struct site *st, struct buffer_if *msg0,
        }
        return True;
     }
-    if (problem==2)
+    if (problem==transform_apply_seqrange)
        goto skew;
 
     if (st->state==SITE_SENTMSG5) {
@@ -1131,7 +1133,7 @@ static bool_t decrypt_msg0(struct site *st, struct buffer_if *msg0,
            activate_new_key(st);
            return True; /* do process the data in this packet */
        }
-       if (problem==2)
+       if (problem==transform_apply_seqrange)
            goto skew;
     }
 

diff --git a/transform-cbcmac.c b/transform-cbcmac.c
index 7e6a1f70f6c2c1ccedd86ffac1cd7c35599afc37..5e59e212c71857a922459df8e55878299401be40 100644 (file)
--- a/transform-cbcmac.c
+++ b/transform-cbcmac.c
@@ -101,8 +101,8 @@ static void transform_delkey(void *sst)
     ti->keyed=False;
 }
 
-static uint32_t transform_forward(void *sst, struct buffer_if *buf,
-                                 const char **errmsg)
+static transform_apply_return transform_forward(void *sst,
+            struct buffer_if *buf, const char **errmsg)
 {
     struct transform_inst *ti=sst;
     uint8_t *padp;
@@ -172,8 +172,8 @@ static uint32_t transform_forward(void *sst, struct buffer_if *buf,
     return 0;
 }
 
-static uint32_t transform_reverse(void *sst, struct buffer_if *buf,
-                                 const char **errmsg)
+static transform_apply_return transform_reverse(void *sst,
+                struct buffer_if *buf, const char **errmsg)
 {
     struct transform_inst *ti=sst;
     uint8_t *padp;
@@ -191,7 +191,7 @@ static uint32_t transform_reverse(void *sst, struct buffer_if *buf,
 
     if (buf->size < 4 + 16 + 16) {
        *errmsg="msg too short";
-       return 1;
+       return transform_apply_err;
     }
 
     /* CBC */
@@ -203,7 +203,7 @@ static uint32_t transform_reverse(void *sst, struct buffer_if *buf,
     /* Assert bufsize is multiple of blocksize */
     if (buf->size&0xf) {
        *errmsg="msg not multiple of cipher blocksize";
-       return 1;
+       return transform_apply_err;
     }
     serpentbe_encrypt(&ti->cryptkey,iv,iv);
     for (n=buf->start; n<buf->start+buf->size; n+=16)
@@ -233,7 +233,7 @@ static uint32_t transform_reverse(void *sst, struct buffer_if *buf,
     serpentbe_encrypt(&ti->mackey,macacc,macacc);
     if (!consttime_memeq(macexpected,macacc,16)!=0) {
        *errmsg="invalid MAC";
-       return 1;
+       return transform_apply_err;
     }
 
     /* PKCS5, stolen from IWJ */
@@ -242,7 +242,7 @@ static uint32_t transform_reverse(void *sst, struct buffer_if *buf,
     padlen=*padp;
     if (!padlen || (padlen > PKCS5_MASK+1)) {
        *errmsg="pkcs5: invalid length";
-       return 1;
+       return transform_apply_err;
     }
 
     buf_unappend(buf,padlen-1);

diff --git a/transform-common.h b/transform-common.h
index f35b5f6fe636f0c7afc89721bfa70a41da5a574f..2e7883303a25c862d81fdb7404d8e28b3b4ad92f 100644 (file)
--- a/transform-common.h
+++ b/transform-common.h
@@ -25,7 +25,7 @@
 #define KEYED_CHECK do{                                \
        if (!ti->keyed) {                       \
            *errmsg="transform unkeyed";        \
-           return 1;                           \
+           return transform_apply_err;         \
        }                                       \
     }while(0)
 
@@ -47,13 +47,13 @@ typedef uint32_t recvbitmap_type;
        } else {                                                \
            /* Too much skew */                                 \
            *errmsg="seqnum: too much skew";                    \
-           return 2;                                           \
+           return transform_apply_seqrange;                    \
        }                                                       \
        if ((p)->dedupe) {                                      \
            recvbitmap_type recvbit=(uint32_t)1 << skew;        \
            if (ti->recvbitmap & recvbit) {                     \
                *errmsg="seqnum: duplicate";                    \
-               return 2;                                       \
+               return transform_apply_seqrange;                \
            }                                                   \
            ti->recvbitmap |= recvbit;                          \
        }                                                       \

diff --git a/transform-eax.c b/transform-eax.c
index 78f53ff65411dddd98e41a60a855c309c35bbbe3..c6da1d33d86ecf87f696c16176492eb951feb778 100644 (file)
--- a/transform-eax.c
+++ b/transform-eax.c
@@ -167,8 +167,8 @@ static void transform_delkey(void *sst)
     ti->keyed=False;
 }
 
-static uint32_t transform_forward(void *sst, struct buffer_if *buf,
-                                 const char **errmsg)
+static transform_apply_return transform_forward(void *sst,
+        struct buffer_if *buf, const char **errmsg)
 {
     struct transform_inst *ti=sst;
 
@@ -205,8 +205,8 @@ static uint32_t transform_forward(void *sst, struct buffer_if *buf,
     return 0;
 }
 
-static uint32_t transform_reverse(void *sst, struct buffer_if *buf,
-                                 const char **errmsg)
+static transform_apply_return transform_reverse(void *sst,
+        struct buffer_if *buf, const char **errmsg)
 {
     struct transform_inst *ti=sst;
 
@@ -233,7 +233,7 @@ static uint32_t transform_reverse(void *sst, struct buffer_if *buf,
     if (!ok) {
        TEAX_DEBUG(0,0);
        *errmsg="EAX decryption failed";
-       return 1;
+       return transform_apply_err;
     }
     assert(buf->size >= (int)ti->p.tag_length);
     buf->size -= ti->p.tag_length;
@@ -256,7 +256,7 @@ static uint32_t transform_reverse(void *sst, struct buffer_if *buf,
 
  too_short:
     *errmsg="ciphertext or plaintext too short";
-    return 1;
+    return transform_apply_err;
 }
 
 static struct transform_inst_if *transform_create(void *sst)