chiark - git - mdw - catacomb/blob - symm/gcm-def.h

   1 /* -*-c-*-
   2  *
   3  * The GCM authenticated encryption mode
   4  *
   5  * (c) 2018 Straylight/Edgeware
   6  */
   7
   8 /*----- Licensing notice --------------------------------------------------*
   9  *
  10  * This file is part of Catacomb.
  11  *
  12  * Catacomb is free software: you can redistribute it and/or modify it
  13  * under the terms of the GNU Library General Public License as published
  14  * by the Free Software Foundation; either version 2 of the License, or
  15  * (at your option) any later version.
  16  *
  17  * Catacomb is distributed in the hope that it will be useful, but
  18  * WITHOUT ANY WARRANTY; without even the implied warranty of
  19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  20  * Library General Public License for more details.
  21  *
  22  * You should have received a copy of the GNU Library General Public
  23  * License along with Catacomb.  If not, write to the Free Software
  24  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
  25  * USA.
  26  */
  27
  28 #ifndef CATACOMB_GCM_DEF_H
  29 #define CATACOMB_GCM_DEF_H
  30
  31 #ifdef __cplusplus
  32   extern "C" {
  33 #endif
  34
  35 /*----- Header files ------------------------------------------------------*/
  36
  37 #include <string.h>
  38
  39 #include <mLib/bits.h>
  40 #include <mLib/sub.h>
  41
  42 #ifndef CATACOMB_ARENA_H
  43 #  include "arena.h"
  44 #endif
  45
  46 #ifndef CATACOMB_BLKC_H
  47 #  include "blkc.h"
  48 #endif
  49
  50 #ifndef CATACOMB_CT_H
  51 #  include "ct.h"
  52 #endif
  53
  54 #ifndef CATACOMB_KEYSZ_H
  55 #  include "keysz.h"
  56 #endif
  57
  58 #ifndef CATACOMB_PARANOIA_H
  59 #  include "paranoia.h"
  60 #endif
  61
  62 #ifndef CATACOMB_RSVR_H
  63 #  include "rsvr.h"
  64 #endif
  65
  66 /*----- Type definitions --------------------------------------------------*/
  67
  68 typedef struct gcm_params {
  69   unsigned f;                           /* flags */
  70 #define GCMF_SWAP 1u                    /*   swap byte order? */
  71   unsigned n;                           /* number of words in block */
  72   uint32 poly;                          /* selected polynomial mask */
  73 } gcm_params;
  74
  75 /*----- Utilities ---------------------------------------------------------*/
  76
  77 /* Supported block sizes. */
  78 #define GCM_WIDTHS(_) _(64) _(96) _(128) _(192) _(256)
  79 #define GCM_NMAX 8
  80
  81 /* Polynomial tails for the supported block sizes. */
  82 #define GCM_POLY_64  0xd8000000
  83 #define GCM_POLY_96  0x82600000
  84 #define GCM_POLY_128 0xe1000000
  85 #define GCM_POLY_192 0xe1000000
  86 #define GCM_POLY_256 0xa4200000
  87
  88 /* Determine whether to set the @GCMF_SWAP@ flag. */
  89 #define GCM_SWAP_L GCMF_SWAP
  90 #define GCM_SWAP_B 0
  91
  92 /* --- @gcm_mktable@ --- *
  93  *
  94  * Arguments:   @const gcm_params *p@ = pointer to the parameters
  95  *              @uint32 *ktab@ = where to write the table; there must be
  96  *                      space for %$32 n$% $%n$%-word entries, i.e.,
  97  *                      %$32 n^2$% 32-bit words in total, where %$n$% is
  98  *                      @p->n@, the block size in words
  99  *              @const uint32 *k@ = input field element
 100  *
 101  * Returns:     ---
 102  *
 103  * Use:         Construct a table for use by @gcm_mulk_...@ below, to
 104  *              multiply (vaguely) efficiently by @k@.
 105  */
 106
 107 extern void gcm_mktable(const gcm_params */*p*/,
 108                         uint32 */*ktab*/, const uint32 */*k*/);
 109
 110 /* --- @gcm_mulk_N{b,l}@ --- *
 111  *
 112  * Arguments:   @uint32 *a@ = accumulator to multiply
 113  *              @const uint32 *ktab@ = table constructed by @gcm_mktable@
 114  *
 115  * Returns:     ---
 116  *
 117  * Use:         Multiply @a@ by @k@ (implicitly represented in @ktab@),
 118  *              updating @a@ in-place.  There are separate functions for each
 119  *              supported block size and endianness because this is the
 120  *              function whose performance actually matters.
 121  */
 122
 123 #define GCM_DECL_MULK(nbits)                                            \
 124   extern void gcm_mulk_##nbits##b(uint32 */*a*/, const uint32 */*ktab*/); \
 125   extern void gcm_mulk_##nbits##l(uint32 */*a*/, const uint32 */*ktab*/);
 126 GCM_WIDTHS(GCM_DECL_MULK)
 127 #undef GCM_DECL_MULK
 128
 129 /* Dispatch to the appropriate variant of @gcm_mulk@. */
 130 #define GCM_MULK(PRE, a, ktab)                                          \
 131   BLKC_GLUE(GCM_MULK_, BLKC_ENDIAN(PRE))(BLKC_BITS(PRE), a, ktab)
 132 #define GCM_MULK_B(nbits, a, ktab)                                      \
 133   BLKC_GLUE(BLKC_GLUE(gcm_mulk_, nbits), b)(a, ktab)
 134 #define GCM_MULK_L(nbits, a, ktab)                                      \
 135   BLKC_GLUE(BLKC_GLUE(gcm_mulk_, nbits), l)(a, ktab)
 136
 137 /* --- @gcm_ghashdone@ --- *
 138  *
 139  * Arguments:   @const gcm_params *p@ = pointer to the parameters
 140  *              @uint32 *a@ = GHASH accumulator
 141  *              @const uint32 *ktab@ = multiplication table, built by
 142  *                      @gcm_mktable@
 143  *              @unsigned long xblocks, yblocks@ = number of whole blocks in
 144  *                      the two inputs
 145  *              @unsigned xbytes, ybytes@ = number of trailing bytes in the
 146  *                      two inputs
 147  *
 148  * Returns:     ---
 149  *
 150  * Use:         Finishes a GHASH operation by appending the appropriately
 151  *              encoded lengths of the two constituent messages.
 152  */
 153
 154 extern void gcm_ghashdone(const gcm_params */*p*/,
 155                           uint32 */*a*/, const uint32 */*ktab*/,
 156                           unsigned long /*xblocks*/, unsigned /*xbytes*/,
 157                           unsigned long /*yblocks*/, unsigned /*ybytes*/);
 158
 159 /* --- @gcm_concat@ --- *
 160  *
 161  * Arguments:   @const gcm_params *p@ = pointer to the parameters
 162  *              @uint32 *z@ = GHASH accumulator for suffix, updated
 163  *              @const uint32 *x@ = GHASH accumulator for prefix
 164  *              @const uint32 *ktab@ = multiplication table, built by
 165  *                      @gcm_mktable@
 166  *              @unsigned long n@ = length of suffix in whole blocks
 167  *
 168  * Returns:     ---
 169  *
 170  * Use:         On entry, @x@ and @z@ are the results of hashing two strings
 171  *              %$a$% and %$b$%, each a whole number of blocks long; in
 172  *              particular, %$b$% is @n@ blocks long.  On exit, @z@ is
 173  *              updated to be the hash of %$a \cat b$%.
 174  */
 175
 176 extern void gcm_concat(const gcm_params */*p*/,
 177                        uint32 */*z*/, const uint32 */*x*/,
 178                        const uint32 */*ktab*/, unsigned long /*n*/);
 179
 180 /* Step the counter using GCM's strange only-the-last-32-bits convention. */
 181 #define GCM_STEP(PRE, w) BLKC_GLUE(GCM_STEP_, BLKC_ENDIAN(PRE))(PRE, w)
 182 #define GCM_STEP_B(PRE, w) GCM_STEP_X(PRE, BLKC_ID, w)
 183 #define GCM_STEP_L(PRE, w) GCM_STEP_X(PRE, ENDSWAP32, w)
 184 #define GCM_STEP_X(PRE, op, w) do {                                     \
 185   BLKC_W(w);                                                            \
 186   _w[PRE##_BLKSZ/4 - 1] = op(op(_w[PRE##_BLKSZ/4 - 1]) + 1);            \
 187 } while (0)
 188
 189 /*----- Macros ------------------------------------------------------------*/
 190
 191 /* --- @GCM_DEF@ --- *
 192  *
 193  * Arguments:   @PRE@, @pre@ = prefixes for the underlying block cipher
 194  *
 195  * Use:         Creates an implementation for the GCM authenticated-
 196  *              encryption mode.
 197  */
 198
 199 #define GCM_DEF(PRE, pre) GCM_DEFX(PRE, pre, #pre, #pre)
 200
 201 #define GCM_DEFX(PRE, pre, name, fname)                                 \
 202                                                                         \
 203 static const gcm_params pre##_gcmparams = {                             \
 204   BLKC_GLUE(GCM_SWAP_, BLKC_ENDIAN(PRE)),                               \
 205   PRE##_BLKSZ/4,                                                        \
 206   BLKC_GLUE(GCM_POLY_, BLKC_BITS(PRE))                                  \
 207 };                                                                      \
 208                                                                         \
 209 const octet                                                             \
 210   pre##_gcmnoncesz[] = { KSZ_ANY, PRE##_BLKSZ - 4 },                    \
 211   pre##_gcmtagsz[] = { KSZ_RANGE, PRE##_BLKSZ, 0, PRE##_BLKSZ, 1 };     \
 212                                                                         \
 213 static const rsvr_policy pre##_gcmpolicy = { 0, PRE##_BLKSZ, PRE##_BLKSZ }; \
 214                                                                         \
 215 /* --- @pre_gcmsetkey@ --- *                                            \
 216  *                                                                      \
 217  * Arguments:   @pre_gcmkey *key@ = pointer to key block to fill in     \
 218  *              @const void *k@ = pointer to key material               \
 219  *              @size_t ksz@ = size of key material                     \
 220  *                                                                      \
 221  * Returns:     ---                                                     \
 222  *                                                                      \
 223  * Use:         Initializes an GCM key block.                           \
 224  */                                                                     \
 225                                                                         \
 226 void pre##_gcmsetkey(pre##_gcmkey *key, const void *k, size_t ksz)      \
 227 {                                                                       \
 228   uint32 t[PRE##_BLKSZ/4];                                              \
 229                                                                         \
 230   /* Initialize the block cipher. */                                    \
 231   pre##_init(&key->ctx, k, ksz);                                        \
 232                                                                         \
 233   /* Set up the GHASH multiplication table. */                          \
 234   BLKC_ZERO(PRE, t); pre##_eblk(&key->ctx, t, t);                       \
 235   gcm_mktable(&pre##_gcmparams, key->ktab, t);                          \
 236 }                                                                       \
 237                                                                         \
 238 /* --- @pre_gcmaadinit@ --- *                                           \
 239  *                                                                      \
 240  * Arguments:   @pre_gcmaadctx *aad@ = pointer to AAD context           \
 241  *              @const pre_gcmkey *key@ = pointer to key block          \
 242  *                                                                      \
 243  * Returns:     ---                                                     \
 244  *                                                                      \
 245  * Use:         Initializes an GCM AAD (`additional authenticated       \
 246  *              data') context associated with a given key.  AAD        \
 247  *              contexts can be copied and/or reused, saving time if    \
 248  *              the AAD for a number of messages has a common prefix.   \
 249  *                                                                      \
 250  *              The @key@ doesn't need to be kept around, though        \
 251  *              usually there'll at least be another copy in some GCM   \
 252  *              operation context because the AAD on its own isn't much \
 253  *              good.                                                   \
 254  */                                                                     \
 255                                                                         \
 256 void pre##_gcmaadinit(pre##_gcmaadctx *aad, const pre##_gcmkey *key)    \
 257   { aad->k = *key; aad->off = 0; aad->len = 0; BLKC_ZERO(PRE, aad->a); } \
 258                                                                         \
 259 /* --- @pre_gcmaadhash@ --- *                                           \
 260  *                                                                      \
 261  * Arguments:   @pre_gcmaadctx *aad@ = pointer to AAD context           \
 262  *              @const void *p@ = pointer to AAD material               \
 263  *              @size_t sz@ = length of AAD material                    \
 264  *                                                                      \
 265  * Returns:     ---                                                     \
 266  *                                                                      \
 267  * Use:         Feeds AAD into the context.                             \
 268  */                                                                     \
 269                                                                         \
 270 void pre##_gcmaadhash(pre##_gcmaadctx *aad, const void *p, size_t sz)   \
 271 {                                                                       \
 272   rsvr_state st;                                                        \
 273   const octet *q;                                                       \
 274                                                                         \
 275   rsvr_setup(&st, &pre##_gcmpolicy, aad->b, &aad->off, p, sz);          \
 276   RSVR_DO(&st) while ((q = RSVR_NEXT(&st, PRE##_BLKSZ)) != 0) {         \
 277     BLKC_XLOAD(PRE, aad->a, q); GCM_MULK(PRE, aad->a, aad->k.ktab);     \
 278     aad->len++;                                                         \
 279   }                                                                     \
 280 }                                                                       \
 281                                                                         \
 282 /* --- @pre_gcminit@ --- *                                              \
 283  *                                                                      \
 284  * Arguments:   @pre_gcmctx *ctx@ = pointer to GCM context              \
 285  *              @const pre_gcmkey *key@ = pointer to key block          \
 286  *              @const void *n@ = pointer to nonce                      \
 287  *              @size_t nsz@ = size of nonce                            \
 288  *                                                                      \
 289  * Returns:     ---                                                     \
 290  *                                                                      \
 291  * Use:         Initialize an GCM operation context with a given key.   \
 292  *                                                                      \
 293  *              The original key needn't be kept around any more.       \
 294  */                                                                     \
 295                                                                         \
 296 void pre##_gcminit(pre##_gcmctx *ctx, const pre##_gcmkey *k,            \
 297                    const void *n, size_t nsz)                           \
 298   { ctx->k = *k; pre##_gcmreinit(ctx, n, nsz); }                        \
 299                                                                         \
 300 /* --- @pre_gcmreinit@ --- *                                            \
 301  *                                                                      \
 302  * Arguments:   @pre_gcmctx *ctx@ = pointer to GCM context              \
 303  *              @const void *n@ = pointer to nonce                      \
 304  *              @size_t nsz@ = size of nonce                            \
 305  *                                                                      \
 306  * Returns:     ---                                                     \
 307  *                                                                      \
 308  * Use:         Reinitialize an GCM operation context, changing the     \
 309  *              nonce.                                                  \
 310  */                                                                     \
 311                                                                         \
 312 void pre##_gcmreinit(pre##_gcmctx *ctx, const void *n, size_t nsz)      \
 313 {                                                                       \
 314   octet b[PRE##_BLKSZ];                                                 \
 315   const octet *q = n;                                                   \
 316   size_t nblocks;                                                       \
 317   unsigned i;                                                           \
 318                                                                         \
 319   /* Zero the counters. */                                              \
 320   ctx->off = 0; ctx->len = 0;                                           \
 321   BLKC_ZERO(PRE, ctx->a);                                               \
 322                                                                         \
 323   /* Calculate the initial counter from the nonce. */                   \
 324   if (nsz == PRE##_BLKSZ - 4) {                                         \
 325     /* Easy version: initialize the final word to 1 and copy the        \
 326      * remaining words from the nonce.  (The spec shows the nonce and   \
 327      * counter the other way around for 64-bit block ciphers, but I'm   \
 328      * sure this is just a mistake.)                                    \
 329      */                                                                 \
 330                                                                         \
 331     for (i = 0; i < PRE##_BLKSZ/4 - 1; i++)                             \
 332       { ctx->c0[i] = BLKC_LOAD_E(PRE)(q); q += 4; }                     \
 333     ctx->c0[PRE##_BLKSZ/4 - 1] = BLKC_BWORD(PRE, 1);                    \
 334   } else {                                                              \
 335     /* Harder version: hash the nonce down with GHASH. */               \
 336                                                                         \
 337     BLKC_ZERO(PRE, ctx->c0); nblocks = 0;                               \
 338     while (nsz >= PRE##_BLKSZ) {                                        \
 339       BLKC_XLOAD(PRE, ctx->c0, q); q += PRE##_BLKSZ;                    \
 340       GCM_MULK(PRE, ctx->c0, ctx->k.ktab);                              \
 341       nsz -= PRE##_BLKSZ; nblocks++;                                    \
 342     }                                                                   \
 343     if (nsz) {                                                          \
 344       memcpy(b, q, nsz); memset(b + nsz, 0, PRE##_BLKSZ - nsz);         \
 345       BLKC_XLOAD(PRE, ctx->c0, b);                                      \
 346       GCM_MULK(PRE, ctx->c0, ctx->k.ktab);                              \
 347     }                                                                   \
 348     gcm_ghashdone(&pre##_gcmparams, ctx->c0, ctx->k.ktab,               \
 349                   0, 0, nblocks, nsz);                                  \
 350   }                                                                     \
 351                                                                         \
 352   /* We must remember the initial counter for the final tag             \
 353    * calculation.  (I conjecture that storing the final counter instead \
 354    * would be just as secure, and require less state, but I've not      \
 355    * proven this, and anyway it wouldn't interoperate.)  Copy it to     \
 356    * make the working counter.                                          \
 357    */                                                                   \
 358   BLKC_MOVE(PRE, ctx->c, ctx->c0);                                      \
 359 }                                                                       \
 360                                                                         \
 361 /* --- @pre_gcmencrypt@ --- *                                           \
 362  *                                                                      \
 363  * Arguments:   @pre_gcmctx *ctx@ = pointer to GCM operation context    \
 364  *              @const void *src@ = pointer to plaintext message chunk  \
 365  *              @size_t sz@ = size of the plaintext                     \
 366  *              @buf *dst@ = a buffer to write the ciphertext to        \
 367  *                                                                      \
 368  * Returns:     Zero on success; @-1@ on failure.                       \
 369  *                                                                      \
 370  * Use:         Encrypts a chunk of a plaintext message, writing a      \
 371  *              chunk of ciphertext to the output buffer and updating   \
 372  *              the operation state.                                    \
 373  *                                                                      \
 374  *              For GCM, we always write a ciphertext chunk the same    \
 375  *              size as the plaintext.  The messing about with @buf@    \
 376  *              objects makes the interface consistent with other AEAD  \
 377  *              schemes which can't do this.                            \
 378  */                                                                     \
 379                                                                         \
 380 int pre##_gcmencrypt(pre##_gcmctx *ctx,                                 \
 381                      const void *src, size_t sz, buf *dst)              \
 382 {                                                                       \
 383   rsvr_plan plan;                                                       \
 384   uint32 t[PRE##_BLKSZ/4];                                              \
 385   const octet *p = src;                                                 \
 386   octet *q, *r, y;                                                      \
 387                                                                         \
 388   /* Allocate space for the ciphertext. */                              \
 389   if (sz) { q = buf_get(dst, sz); if (!q) return (-1); }                \
 390   else q = 0;                                                           \
 391                                                                         \
 392   /* Determine the buffering plan.  Our buffer is going to do double-   \
 393    * duty here.  The end portion is going to contain mask from the      \
 394    * encrypted counter which we mix into the plaintext to encrypt it;   \
 395    * the start portion, which originally mask bytes we've already used, \
 396    * will hold the output ciphertext, which will eventually be          \
 397    * collected into the GHASH state.                                    \
 398    */                                                                   \
 399   rsvr_mkplan(&plan, &pre##_gcmpolicy, ctx->off, sz);                   \
 400                                                                         \
 401   /* Initial portion, fulfilled from the buffer.  If the buffer is      \
 402    * empty, then that means that we haven't yet encrypted the current   \
 403    * counter, so we should do that and advance it.                      \
 404    */                                                                   \
 405   if (plan.head) {                                                      \
 406     if (!ctx->off) {                                                    \
 407       GCM_STEP(PRE, ctx->c); pre##_eblk(&ctx->k.ctx, ctx->c, t);        \
 408       BLKC_STORE(PRE, ctx->b, t);                                       \
 409     }                                                                   \
 410     r = ctx->b + ctx->off; ctx->off += plan.head;                       \
 411     while (plan.head--) { y = *p++ ^ *r; *r++ = *q++ = y; }             \
 412   }                                                                     \
 413                                                                         \
 414   /* If we've filled up the buffer then we need to cycle the MAC and    \
 415    * reset the offset.                                                  \
 416    */                                                                   \
 417   if (plan.from_rsvr) {                                                 \
 418     BLKC_XLOAD(PRE, ctx->a, ctx->b); GCM_MULK(PRE, ctx->a, ctx->k.ktab); \
 419     ctx->len++; ctx->off = 0;                                           \
 420   }                                                                     \
 421                                                                         \
 422   /* Now to process the main body of the input. */                      \
 423   while (plan.from_input) {                                             \
 424     GCM_STEP(PRE, ctx->c); pre##_eblk(&ctx->k.ctx, ctx->c, t);          \
 425     BLKC_XLOAD(PRE, t, p); p += PRE##_BLKSZ;                            \
 426     BLKC_STORE(PRE, q, t); q += PRE##_BLKSZ;                            \
 427     BLKC_XMOVE(PRE, ctx->a, t); GCM_MULK(PRE, ctx->a, ctx->k.ktab);     \
 428     plan.from_input -= PRE##_BLKSZ; ctx->len++;                         \
 429   }                                                                     \
 430                                                                         \
 431   /* Finally, deal with any final portion.  If there is one, we know    \
 432    * that the buffer is empty: we must have filled it above, or this    \
 433    * would all count as `initial' data.                                 \
 434    */                                                                   \
 435   if (plan.tail) {                                                      \
 436     GCM_STEP(PRE, ctx->c); pre##_eblk(&ctx->k.ctx, ctx->c, t);          \
 437     BLKC_STORE(PRE, ctx->b, t);                                         \
 438     r = ctx->b; ctx->off += plan.tail;                                  \
 439     while (plan.tail--) { y = *p++ ^ *r; *r++ = *q++ = y; }             \
 440   }                                                                     \
 441                                                                         \
 442   /* And we're done. */                                                 \
 443   return (0);                                                           \
 444 }                                                                       \
 445                                                                         \
 446 /* --- @pre_gcmdecrypt@ --- *                                           \
 447  *                                                                      \
 448  * Arguments:   @pre_gcmctx *ctx@ = pointer to GCM operation context    \
 449  *              @const void *src@ = pointer to ciphertext message chunk \
 450  *              @size_t sz@ = size of the ciphertext                    \
 451  *              @buf *dst@ = a buffer to write the plaintext to         \
 452  *                                                                      \
 453  * Returns:     Zero on success; @-1@ on failure.                       \
 454  *                                                                      \
 455  * Use:         Decrypts a chunk of a ciphertext message, writing a     \
 456  *              chunk of plaintext to the output buffer and updating    \
 457  *              the operation state.                                    \
 458  *                                                                      \
 459  *              For GCM, we always write a plaintext chunk the same     \
 460  *              size as the ciphertext.  The messing about with @buf@   \
 461  *              objects makes the interface consistent with other AEAD  \
 462  *              schemes which can't do this.                            \
 463  */                                                                     \
 464                                                                         \
 465 int pre##_gcmdecrypt(pre##_gcmctx *ctx,                                 \
 466                      const void *src, size_t sz, buf *dst)              \
 467 {                                                                       \
 468   rsvr_plan plan;                                                       \
 469   uint32 t[PRE##_BLKSZ/4], u[PRE##_BLKSZ];                              \
 470   const octet *p = src;                                                 \
 471   octet *q, *r, y;                                                      \
 472                                                                         \
 473   /* Allocate space for the plaintext. */                               \
 474   if (sz) { q = buf_get(dst, sz); if (!q) return (-1); }                \
 475   else q = 0;                                                           \
 476                                                                         \
 477   /* Determine the buffering plan.  Our buffer is going to do double-   \
 478    * duty here.  The end portion is going to contain mask from the      \
 479    * encrypted counter which we mix into the plaintext to encrypt it;   \
 480    * the start portion, which originally mask bytes we've already used, \
 481    * will hold the input ciphertext, which will eventually be           \
 482    * collected into the GHASH state.                                    \
 483    */                                                                   \
 484   rsvr_mkplan(&plan, &pre##_gcmpolicy, ctx->off, sz);                   \
 485                                                                         \
 486   /* Initial portion, fulfilled from the buffer.  If the buffer is      \
 487    * empty, then that means that we haven't yet encrypted the current   \
 488    * counter, so we should do that and advance it.                      \
 489    */                                                                   \
 490   if (plan.head) {                                                      \
 491     if (!ctx->off) {                                                    \
 492       GCM_STEP(PRE, ctx->c); pre##_eblk(&ctx->k.ctx, ctx->c, t);        \
 493       BLKC_STORE(PRE, ctx->b, t);                                       \
 494     }                                                                   \
 495     r = ctx->b + ctx->off; ctx->off += plan.head;                       \
 496     while (plan.head--) { y = *p++; *q++ = y ^ *r; *r++ = y; }          \
 497   }                                                                     \
 498                                                                         \
 499   /* If we've filled up the buffer then we need to cycle the MAC and    \
 500    * reset the offset.                                                  \
 501    */                                                                   \
 502   if (plan.from_rsvr) {                                                 \
 503     BLKC_XLOAD(PRE, ctx->a, ctx->b); GCM_MULK(PRE, ctx->a, ctx->k.ktab); \
 504     ctx->len++; ctx->off = 0;                                           \
 505   }                                                                     \
 506                                                                         \
 507   /* Now to process the main body of the input. */                      \
 508   while (plan.from_input) {                                             \
 509     GCM_STEP(PRE, ctx->c); pre##_eblk(&ctx->k.ctx, ctx->c, t);          \
 510     BLKC_LOAD(PRE, u, p); p += PRE##_BLKSZ;                             \
 511     BLKC_XSTORE(PRE, q, t, u); q += PRE##_BLKSZ;                        \
 512     BLKC_XMOVE(PRE, ctx->a, u); GCM_MULK(PRE, ctx->a, ctx->k.ktab);     \
 513     plan.from_input -= PRE##_BLKSZ; ctx->len++;                         \
 514   }                                                                     \
 515                                                                         \
 516   /* Finally, deal with any final portion.  If there is one, we know    \
 517    * that the buffer is empty: we must have filled it above, or this    \
 518    * would all count as `initial' data.                                 \
 519    */                                                                   \
 520   if (plan.tail) {                                                      \
 521     GCM_STEP(PRE, ctx->c); pre##_eblk(&ctx->k.ctx, ctx->c, t);          \
 522     BLKC_STORE(PRE, ctx->b, t);                                         \
 523     r = ctx->b; ctx->off += plan.tail;                                  \
 524     while (plan.tail--) { y = *p++; *q++ = y ^ *r; *r++ = y; }          \
 525   }                                                                     \
 526                                                                         \
 527   /* And we're done. */                                                 \
 528   return (0);                                                           \
 529 }                                                                       \
 530                                                                         \
 531 /* --- @pre_gcmtag@ --- *                                               \
 532  *                                                                      \
 533  * Arguments:   @pre_gcmctx *ctx@ = pointer to an GCM context           \
 534  *              @const pre_gcmaadctx *aad@ = pointer to AAD context, or \
 535  *                      null                                            \
 536  *              @octet *t@ = where to write a (full-length) tag         \
 537  *                                                                      \
 538  * Returns:     ---                                                     \
 539  *                                                                      \
 540  * Use:         Finishes an GCM operation, by calculating the tag.      \
 541  */                                                                     \
 542                                                                         \
 543 static void pre##_gcmtag(pre##_gcmctx *ctx,                             \
 544                          const pre##_gcmaadctx *aad, octet *t)          \
 545 {                                                                       \
 546   octet b[PRE##_BLKSZ];                                                 \
 547   uint32 u[PRE##_BLKSZ/4];                                              \
 548   unsigned long n;                                                      \
 549                                                                         \
 550   /* Finish tagging the ciphertext. */                                  \
 551   if (ctx->off) {                                                       \
 552     memcpy(b, ctx->b, ctx->off);                                        \
 553     memset(b + ctx->off, 0, PRE##_BLKSZ - ctx->off);                    \
 554     BLKC_XLOAD(PRE, ctx->a, b); GCM_MULK(PRE, ctx->a, ctx->k.ktab);     \
 555   }                                                                     \
 556                                                                         \
 557   /* If there's no AAD, because the pointer is null or no data was      \
 558    * supplied, then apply that to the GHASH state.  (Otherwise there's  \
 559    * nothing to do here.)                                               \
 560    */                                                                   \
 561   if (aad && (aad->len || aad->off)) {                                  \
 562     BLKC_MOVE(PRE, u, aad->a);                                          \
 563     if (aad->off) {                                                     \
 564       memcpy(b, aad->b, aad->off);                                      \
 565       memset(b + aad->off, 0, PRE##_BLKSZ - aad->off);                  \
 566       BLKC_XLOAD(PRE, u, b); GCM_MULK(PRE, u, ctx->k.ktab);             \
 567     }                                                                   \
 568     n = ctx->len; if (ctx->off) n++;                                    \
 569     gcm_concat(&pre##_gcmparams, ctx->a, u, ctx->k.ktab, n);            \
 570   }                                                                     \
 571                                                                         \
 572   /* Finish off the hash by appending the length. */                    \
 573   gcm_ghashdone(&pre##_gcmparams, ctx->a, ctx->k.ktab,                  \
 574                 aad ? aad->len : 0, aad ? aad->off : 0,                 \
 575                 ctx->len, ctx->off);                                    \
 576                                                                         \
 577   /* Mask the hash and store. */                                        \
 578   pre##_eblk(&ctx->k.ctx, ctx->c0, u);                                  \
 579   BLKC_XSTORE(PRE, t, ctx->a, u);                                       \
 580 }                                                                       \
 581                                                                         \
 582 /* --- @pre_gcmencryptdone@ --- *                                       \
 583  *                                                                      \
 584  * Arguments:   @pre_gcmctx *ctx@ = pointer to an GCM context           \
 585  *              @const pre_gcmaadctx *aad@ = pointer to AAD context, or \
 586  *                      null                                            \
 587  *              @buf *dst@ = buffer for remaining ciphertext            \
 588  *              @void *tag@ = where to write the tag                    \
 589  *              @size_t tsz@ = length of tag to store                   \
 590  *                                                                      \
 591  * Returns:     Zero on success; @-1@ on failure.                       \
 592  *                                                                      \
 593  * Use:         Completes an GCM encryption operation.  The @aad@       \
 594  *              pointer may be null if there is no additional           \
 595  *              authenticated data.  GCM doesn't buffer ciphertext, but \
 596  *              the output buffer is provided anyway for consistency    \
 597  *              with other AEAD schemes which don't have this property; \
 598  *              the function will fail if the output buffer is broken.  \
 599  */                                                                     \
 600                                                                         \
 601 int pre##_gcmencryptdone(pre##_gcmctx *ctx,                             \
 602                          const pre##_gcmaadctx *aad, buf *dst,          \
 603                          void *tag, size_t tsz)                         \
 604 {                                                                       \
 605   octet t[PRE##_BLKSZ];                                                 \
 606                                                                         \
 607   if (tsz > PRE##_BLKSZ) return (-1);                                   \
 608   if (!BOK(dst)) return (-1);                                           \
 609   pre##_gcmtag(ctx, aad, t); memcpy(tag, t, tsz);                       \
 610   return (0);                                                           \
 611 }                                                                       \
 612                                                                         \
 613 /* --- @pre_gcmdecryptdone@ --- *                                       \
 614  *                                                                      \
 615  * Arguments:   @pre_gcmctx *ctx@ = pointer to an GCM context           \
 616  *              @const pre_gcmaadctx *aad@ = pointer to AAD context, or \
 617  *                      null                                            \
 618  *              @buf *dst@ = buffer for remaining plaintext             \
 619  *              @const void *tag@ = tag to verify                       \
 620  *              @size_t tsz@ = length of tag                            \
 621  *                                                                      \
 622  * Returns:     @+1@ for complete success; @0@ if tag verification      \
 623  *              failed; @-1@ for other kinds of errors.                 \
 624  *                                                                      \
 625  * Use:         Completes an GCM decryption operation.  The @aad@       \
 626  *              pointer may be null if there is no additional           \
 627  *              authenticated data.  GCM doesn't buffer plaintext, but  \
 628  *              the output buffer is provided anyway for consistency    \
 629  *              with other AEAD schemes which don't have this property; \
 630  *              the function will fail if the output buffer is broken.  \
 631  */                                                                     \
 632                                                                         \
 633 int pre##_gcmdecryptdone(pre##_gcmctx *ctx,                             \
 634                          const pre##_gcmaadctx *aad, buf *dst,          \
 635                          const void *tag, size_t tsz)                   \
 636 {                                                                       \
 637   octet t[PRE##_BLKSZ];                                                 \
 638                                                                         \
 639   if (tsz > PRE##_BLKSZ) return (-1);                                   \
 640   if (!BOK(dst)) return (-1);                                           \
 641   pre##_gcmtag(ctx, aad, t);                                            \
 642   if (!ct_memeq(tag, t, tsz)) return (0);                               \
 643   else return (+1);                                                     \
 644 }                                                                       \
 645                                                                         \
 646 /* --- Generic AEAD interface --- */                                    \
 647                                                                         \
 648 typedef struct gactx {                                                  \
 649   gaead_aad a;                                                          \
 650   pre##_gcmaadctx aad;                                                  \
 651 } gactx;                                                                \
 652                                                                         \
 653 static gaead_aad *gadup(const gaead_aad *a)                             \
 654   { gactx *aad = S_CREATE(gactx); *aad = *(gactx *)a; return (&aad->a); } \
 655                                                                         \
 656 static void gahash(gaead_aad *a, const void *h, size_t hsz)             \
 657   { gactx *aad = (gactx *)a; pre##_gcmaadhash(&aad->aad, h, hsz); }     \
 658                                                                         \
 659 static void gadestroy(gaead_aad *a)                                     \
 660   { gactx *aad = (gactx *)a; BURN(*aad); S_DESTROY(aad); }              \
 661                                                                         \
 662 static const gaead_aadops gaops =                                       \
 663   { &pre##_gcm, gadup, gahash, gadestroy };                             \
 664                                                                         \
 665 static gaead_aad *gaad(const pre##_gcmkey *k)                           \
 666 {                                                                       \
 667   gactx *aad = S_CREATE(gactx);                                         \
 668   aad->a.ops = &gaops;                                                  \
 669   pre##_gcmaadinit(&aad->aad, k);                                       \
 670   return (&aad->a);                                                     \
 671 }                                                                       \
 672                                                                         \
 673 typedef struct gectx {                                                  \
 674   gaead_enc e;                                                          \
 675   pre##_gcmctx ctx;                                                     \
 676 } gectx;                                                                \
 677                                                                         \
 678 static gaead_aad *geaad(gaead_enc *e)                                   \
 679   { gectx *enc = (gectx *)e; return (gaad(&enc->ctx.k)); }              \
 680                                                                         \
 681 static int gereinit(gaead_enc *e, const void *n, size_t nsz,            \
 682                     size_t hsz, size_t msz, size_t tsz)                 \
 683 {                                                                       \
 684   gectx *enc = (gectx *)e;                                              \
 685                                                                         \
 686   if (tsz > PRE##_BLKSZ) return (-1);                                   \
 687   pre##_gcmreinit(&enc->ctx, n, nsz);                                   \
 688   return (0);                                                           \
 689 }                                                                       \
 690                                                                         \
 691 static int geenc(gaead_enc *e, const void *m, size_t msz, buf *b)       \
 692 {                                                                       \
 693   gectx *enc = (gectx *)e;                                              \
 694   return (pre##_gcmencrypt(&enc->ctx, m, msz, b));                      \
 695 }                                                                       \
 696                                                                         \
 697 static int gedone(gaead_enc *e, const gaead_aad *a,                     \
 698                   buf *b, void *t, size_t tsz)                          \
 699 {                                                                       \
 700   gectx *enc = (gectx *)e; gactx *aad = (gactx *)a;                     \
 701   assert(!a || a->ops == &gaops);                                       \
 702   return (pre##_gcmencryptdone(&enc->ctx, a ? &aad->aad : 0, b, t, tsz)); \
 703 }                                                                       \
 704                                                                         \
 705 static void gedestroy(gaead_enc *e)                                     \
 706   { gectx *enc = (gectx *)e; BURN(*enc); S_DESTROY(enc); }              \
 707                                                                         \
 708 static const gaead_encops geops =                                       \
 709   { &pre##_gcm, geaad, gereinit, geenc, gedone, gedestroy };            \
 710                                                                         \
 711 typedef struct gdctx {                                                  \
 712   gaead_dec d;                                                          \
 713   pre##_gcmctx ctx;                                                     \
 714 } gdctx;                                                                \
 715                                                                         \
 716 static gaead_aad *gdaad(gaead_dec *d)                                   \
 717   { gdctx *dec = (gdctx *)d; return (gaad(&dec->ctx.k)); }              \
 718                                                                         \
 719 static int gdreinit(gaead_dec *d, const void *n, size_t nsz,            \
 720                      size_t hsz, size_t csz, size_t tsz)                \
 721 {                                                                       \
 722   gdctx *dec = (gdctx *)d;                                              \
 723                                                                         \
 724   if (tsz > PRE##_BLKSZ) return (-1);                                   \
 725   pre##_gcmreinit(&dec->ctx, n, nsz);                                   \
 726   return (0);                                                           \
 727 }                                                                       \
 728                                                                         \
 729 static int gddec(gaead_dec *d, const void *c, size_t csz, buf *b)       \
 730 {                                                                       \
 731   gdctx *dec = (gdctx *)d;                                              \
 732   return (pre##_gcmdecrypt(&dec->ctx, c, csz, b));                      \
 733 }                                                                       \
 734                                                                         \
 735 static int gddone(gaead_dec *d, const gaead_aad *a,                     \
 736                   buf *b, const void *t, size_t tsz)                    \
 737 {                                                                       \
 738   gdctx *dec = (gdctx *)d; gactx *aad = (gactx *)a;                     \
 739   assert(!a || a->ops == &gaops);                                       \
 740   return (pre##_gcmdecryptdone(&dec->ctx, a ? &aad->aad : 0, b, t, tsz)); \
 741 }                                                                       \
 742                                                                         \
 743 static void gddestroy(gaead_dec *d)                                     \
 744   { gdctx *dec = (gdctx *)d; BURN(*dec); S_DESTROY(dec); }              \
 745                                                                         \
 746 static const gaead_decops gdops =                                       \
 747   { &pre##_gcm, gdaad, gdreinit, gddec, gddone, gddestroy };            \
 748                                                                         \
 749 typedef struct gkctx {                                                  \
 750   gaead_key k;                                                          \
 751   pre##_gcmkey key;                                                     \
 752 } gkctx;                                                                \
 753                                                                         \
 754 static gaead_aad *gkaad(const gaead_key *k)                             \
 755   { gkctx *key = (gkctx *)k; return (gaad(&key->key)); }                \
 756                                                                         \
 757 static gaead_enc *gkenc(const gaead_key *k, const void *n, size_t nsz,  \
 758                         size_t hsz, size_t msz, size_t tsz)             \
 759 {                                                                       \
 760   gkctx *key = (gkctx *)k;                                              \
 761   gectx *enc = S_CREATE(gectx);                                         \
 762                                                                         \
 763   enc->e.ops = &geops;                                                  \
 764   pre##_gcminit(&enc->ctx, &key->key, n, nsz);                          \
 765   return (&enc->e);                                                     \
 766 }                                                                       \
 767                                                                         \
 768 static gaead_dec *gkdec(const gaead_key *k, const void *n, size_t nsz,  \
 769                         size_t hsz, size_t csz, size_t tsz)             \
 770 {                                                                       \
 771   gkctx *key = (gkctx *)k;                                              \
 772   gdctx *dec = S_CREATE(gdctx);                                         \
 773                                                                         \
 774   dec->d.ops = &gdops;                                                  \
 775   pre##_gcminit(&dec->ctx, &key->key, n, nsz);                          \
 776   return (&dec->d);                                                     \
 777 }                                                                       \
 778                                                                         \
 779 static void gkdestroy(gaead_key *k)                                     \
 780   { gkctx *key = (gkctx *)k; BURN(*key); S_DESTROY(key); }              \
 781                                                                         \
 782 static const gaead_keyops gkops =                                       \
 783   { &pre##_gcm, gkaad, gkenc, gkdec, gkdestroy };                       \
 784                                                                         \
 785 static gaead_key *gckey(const void *k, size_t ksz)                      \
 786 {                                                                       \
 787   gkctx *key = S_CREATE(gkctx);                                         \
 788   key->k.ops = &gkops;                                                  \
 789   pre##_gcmsetkey(&key->key, k, ksz);                                   \
 790   return (&key->k);                                                     \
 791 }                                                                       \
 792                                                                         \
 793 const gcaead pre##_gcm = {                                              \
 794   name "-gcm",                                                          \
 795   pre##_keysz, pre##_gcmnoncesz, pre##_gcmtagsz,                        \
 796   PRE##_BLKSZ, 0, 0, 0,                                                 \
 797   gckey                                                                 \
 798 };                                                                      \
 799                                                                         \
 800 GCM_TESTX(PRE, pre, name, fname)
 801
 802 /*----- Test rig ----------------------------------------------------------*/
 803
 804 #define GCM_TEST(PRE, pre) GCM_TESTX(PRE, pre, #pre, #pre)
 805
 806 /* --- @GCM_TEST@ --- *
 807  *
 808  * Arguments:   @PRE, pre@ = prefixes for the underlying block cipher
 809  *
 810  * Use:         Standard test rig for GCM functions.
 811  */
 812
 813 #ifdef TEST_RIG
 814
 815 #include <stdio.h>
 816
 817 #include <mLib/dstr.h>
 818 #include <mLib/quis.h>
 819 #include <mLib/testrig.h>
 820
 821 #define GCM_TESTX(PRE, pre, name, fname)                                \
 822                                                                         \
 823 static int gcmverify(dstr *v)                                           \
 824 {                                                                       \
 825   pre##_gcmkey key;                                                     \
 826   pre##_gcmaadctx aad;                                                  \
 827   pre##_gcmctx ctx;                                                     \
 828   int ok = 1, win;                                                      \
 829   int i;                                                                \
 830   octet *p;                                                             \
 831   int szs[] = { 1, 7, 192, -1, 0 }, *ip;                                \
 832   size_t hsz, msz;                                                      \
 833   dstr d = DSTR_INIT, t = DSTR_INIT;                                    \
 834   buf b;                                                                \
 835                                                                         \
 836   dstr_ensure(&d, v[4].len > v[3].len ? v[4].len : v[3].len);           \
 837   dstr_ensure(&t, v[5].len); t.len = v[5].len;                          \
 838                                                                         \
 839   pre##_gcmsetkey(&key, v[0].buf, v[0].len);                            \
 840                                                                         \
 841   for (ip = szs; *ip; ip++) {                                           \
 842                                                                         \
 843     pre##_gcminit(&ctx, &key, (octet *)v[1].buf, v[1].len);             \
 844                                                                         \
 845     i = *ip;                                                            \
 846     hsz = v[2].len;                                                     \
 847     if (i == -1) i = hsz;                                               \
 848     if (i > hsz) continue;                                              \
 849     p = (octet *)v[2].buf;                                              \
 850     pre##_gcmaadinit(&aad, &key);                                       \
 851     while (hsz) {                                                       \
 852       if (i > hsz) i = hsz;                                             \
 853       pre##_gcmaadhash(&aad, p, i);                                     \
 854       p += i; hsz -= i;                                                 \
 855     }                                                                   \
 856                                                                         \
 857     buf_init(&b, d.buf, d.sz);                                          \
 858     i = *ip;                                                            \
 859     msz = v[3].len;                                                     \
 860     if (i == -1) i = msz;                                               \
 861     if (i > msz) continue;                                              \
 862     p = (octet *)v[3].buf;                                              \
 863     while (msz) {                                                       \
 864       if (i > msz) i = msz;                                             \
 865       if (pre##_gcmencrypt(&ctx, p, i, &b)) {                           \
 866         puts("!! gcmencrypt reports failure");                          \
 867         goto fail_enc;                                                  \
 868       }                                                                 \
 869       p += i; msz -= i;                                                 \
 870     }                                                                   \
 871                                                                         \
 872     if (pre##_gcmencryptdone(&ctx, &aad, &b, (octet *)t.buf, t.len)) {  \
 873       puts("!! gcmencryptdone reports failure");                        \
 874       goto fail_enc;                                                    \
 875     }                                                                   \
 876     d.len = BLEN(&b);                                                   \
 877                                                                         \
 878     if (d.len != v[4].len ||                                            \
 879         memcmp(d.buf, v[4].buf, v[4].len) != 0 ||                       \
 880         memcmp(t.buf, v[5].buf, v[5].len) != 0) {                       \
 881     fail_enc:                                                           \
 882       printf("\nfail encrypt:\n\tstep = %i", *ip);                      \
 883       fputs("\n\tkey = ", stdout); type_hex.dump(&v[0], stdout);        \
 884       fputs("\n\tnonce = ", stdout); type_hex.dump(&v[1], stdout);      \
 885       fputs("\n\theader = ", stdout); type_hex.dump(&v[2], stdout);     \
 886       fputs("\n\tmessage = ", stdout); type_hex.dump(&v[3], stdout);    \
 887       fputs("\n\texp ct = ", stdout); type_hex.dump(&v[4], stdout);     \
 888       fputs("\n\tcalc ct = ", stdout); type_hex.dump(&d, stdout);       \
 889       fputs("\n\texp tag = ", stdout); type_hex.dump(&v[5], stdout);    \
 890       fputs("\n\tcalc tag = ", stdout); type_hex.dump(&t, stdout);      \
 891       putchar('\n');                                                    \
 892       ok = 0;                                                           \
 893     }                                                                   \
 894                                                                         \
 895     pre##_gcminit(&ctx, &key, (octet *)v[1].buf, v[1].len);             \
 896                                                                         \
 897     buf_init(&b, d.buf, d.sz);                                          \
 898     i = *ip;                                                            \
 899     msz = v[4].len;                                                     \
 900     if (i == -1) i = msz;                                               \
 901     if (i > msz) continue;                                              \
 902     p = (octet *)v[4].buf;                                              \
 903     while (msz) {                                                       \
 904       if (i > msz) i = msz;                                             \
 905       if (pre##_gcmdecrypt(&ctx, p, i, &b)) {                           \
 906         puts("!! gcmdecrypt reports failure");                          \
 907         win = 0; goto fail_dec;                                         \
 908       }                                                                 \
 909       p += i; msz -= i;                                                 \
 910     }                                                                   \
 911                                                                         \
 912     win = pre##_gcmdecryptdone(&ctx, &aad, &b,                          \
 913                                (octet *)v[5].buf, v[5].len);            \
 914     if (win < 0) {                                                      \
 915       puts("!! gcmdecryptdone reports failure");                        \
 916       goto fail_dec;                                                    \
 917     }                                                                   \
 918     d.len = BLEN(&b);                                                   \
 919                                                                         \
 920     if (d.len != v[3].len || !win ||                                    \
 921         memcmp(d.buf, v[3].buf, v[3].len) != 0) {                       \
 922     fail_dec:                                                           \
 923       printf("\nfail decrypt:\n\tstep = %i", *ip);                      \
 924       fputs("\n\tkey = ", stdout); type_hex.dump(&v[0], stdout);        \
 925       fputs("\n\tnonce = ", stdout); type_hex.dump(&v[1], stdout);      \
 926       fputs("\n\theader = ", stdout); type_hex.dump(&v[2], stdout);     \
 927       fputs("\n\tciphertext = ", stdout); type_hex.dump(&v[4], stdout); \
 928       fputs("\n\texp pt = ", stdout); type_hex.dump(&v[3], stdout);     \
 929       fputs("\n\tcalc pt = ", stdout); type_hex.dump(&d, stdout);       \
 930       fputs("\n\ttag = ", stdout); type_hex.dump(&v[5], stdout);        \
 931       printf("\n\tverify %s", win ? "ok" : "FAILED");                   \
 932       putchar('\n');                                                    \
 933       ok = 0;                                                           \
 934     }                                                                   \
 935   }                                                                     \
 936                                                                         \
 937   dstr_destroy(&d); dstr_destroy(&t);                                   \
 938   return (ok);                                                          \
 939 }                                                                       \
 940                                                                         \
 941 static test_chunk aeaddefs[] = {                                        \
 942   { name "-gcm", gcmverify,                                             \
 943     { &type_hex, &type_hex, &type_hex, &type_hex,                       \
 944       &type_hex, &type_hex, 0 } },                                      \
 945   { 0, 0, { 0 } }                                                       \
 946 };                                                                      \
 947                                                                         \
 948 int main(int argc, char *argv[])                                        \
 949 {                                                                       \
 950   ego(argv[0]);                                                         \
 951   test_run(argc, argv, aeaddefs, SRCDIR"/t/" fname);                    \
 952   return (0);                                                           \
 953 }
 954
 955 #else
 956 #  define GCM_TESTX(PRE, pre, name, fname)
 957 #endif
 958
 959 /*----- That's all, folks -------------------------------------------------*/
 960
 961 #ifdef __cplusplus
 962   }
 963 #endif
 964
 965 #endif