Merge branch '2.4.x' into 2.5.x

[catacomb] / symm / gcm-arm64-pmull.S

/// -*- mode: asm; asm-comment-char: ?/ -*-
///
/// GCM acceleration for ARM64 processors
///
/// (c) 2019 Straylight/Edgeware
///

///----- Licensing notice ---------------------------------------------------
///
/// This file is part of Catacomb.
///
/// Catacomb is free software: you can redistribute it and/or modify it
/// under the terms of the GNU Library General Public License as published
/// by the Free Software Foundation; either version 2 of the License, or
/// (at your option) any later version.
///
/// Catacomb is distributed in the hope that it will be useful, but
/// WITHOUT ANY WARRANTY; without even the implied warranty of
/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
/// Library General Public License for more details.
///
/// You should have received a copy of the GNU Library General Public
/// License along with Catacomb.  If not, write to the Free Software
/// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
/// USA.

///--------------------------------------------------------------------------
/// Preliminaries.

#include "config.h"
#include "asm-common.h"

        .arch   armv8-a+crypto

        .text

///--------------------------------------------------------------------------
/// Multiplication macros.

        // The good news is that we have a fancy instruction to do the
        // multiplications.  The bad news is that it's not particularly well-
        // suited to the job.
        //
        // For one thing, it only does a 64-bit multiplication, so in general
        // we'll need to synthesize the full-width multiply by hand.  For
        // another thing, it doesn't help with the reduction, so we have to
        // do that by hand too.  And, finally, GCM has crazy bit ordering,
        // and the instruction does nothing useful for that at all.
        //
        // Focusing on that last problem first: the bits aren't in monotonic
        // significance order unless we permute them.  Fortunately, ARM64 has
        // an instruction which will just permute the bits in each byte for
        // us, so we don't have to worry about this very much.
        //
        // Our main weapons, the `pmull' and `pmull2' instructions, work on
        // 64-bit operands, in half of a vector register, and produce 128-bit
        // results.  But neither of them will multiply the high half of one
        // vector by the low half of a second one, so we have a problem,
        // which we solve by representing one of the operands redundantly:
        // rather than packing the 64-bit pieces together, we duplicate each
        // 64-bit piece across both halves of a register.
        //
        // The commentary for `mul128' is the most detailed.  The other
        // macros assume that you've already read and understood that.

.macro  mul128
        // Enter with u and v in v0 and v1/v2 respectively, and 0 in v31;
        // leave with z = u v in v0.  Clobbers v1--v6.

        // First for the double-precision multiplication.  It's tempting to
        // use Karatsuba's identity here, but I suspect that loses more in
        // the shifting, bit-twiddling, and dependency chains that it gains
        // in saving a multiplication which otherwise pipelines well.
        // v0 =                         // (u_0; u_1)
        // v1/v2 =                      // (v_0; v_1)
        pmull2  v3.1q, v0.2d, v1.2d     // u_1 v_0
        pmull   v4.1q, v0.1d, v2.1d     // u_0 v_1
        pmull2  v5.1q, v0.2d, v2.2d     // (t_1; x_3) = u_1 v_1
        pmull   v6.1q, v0.1d, v1.1d     // (x_0; t_0) = u_0 v_0

        // Arrange the pieces to form a double-precision polynomial.
        eor     v3.16b, v3.16b, v4.16b  // (m_0; m_1) = u_0 v_1 + u_1 v_0
        vshr128 v4, v3, 64              // (m_1; 0)
        vshl128 v3, v3, 64              // (0; m_0)
        eor     v1.16b, v5.16b, v4.16b  // (x_2; x_3)
        eor     v0.16b, v6.16b, v3.16b  // (x_0; x_1)

        // And now the only remaining difficulty is that the result needs to
        // be reduced modulo p(t) = t^128 + t^7 + t^2 + t + 1.  Let R = t^128
        // = t^7 + t^2 + t + 1 in our field.  So far, we've calculated z_0
        // and z_1 such that z_0 + z_1 R = u v using the identity R = t^128:
        // now we must collapse the two halves of y together using the other
        // identity R = t^7 + t^2 + t + 1.
        //
        // We do this by working on y_2 and y_3 separately, so consider y_i
        // for i = 2 or 3.  Certainly, y_i t^{64i} = y_i R t^{64(i-2) =
        // (t^7 + t^2 + t + 1) y_i t^{64(i-2)}, but we can't use that
        // directly without breaking up the 64-bit word structure.  Instead,
        // we start by considering just y_i t^7 t^{64(i-2)}, which again
        // looks tricky.  Now, split y_i = a_i + t^57 b_i, with deg a_i < 57;
        // then
        //
        //      y_i t^7 t^{64(i-2)} = a_i t^7 t^{64(i-2)} + b_i t^{64(i-1)}
        //
        // We can similarly decompose y_i t^2 and y_i t into a pair of 64-bit
        // contributions to the t^{64(i-2)} and t^{64(i-1)} words, but the
        // splits are different.  This is lovely, with one small snag: when
        // we do this to y_3, we end up with a contribution back into the
        // t^128 coefficient word.  But notice that only the low seven bits
        // of this word are affected, so there's no knock-on contribution
        // into the t^64 word.  Therefore, if we handle the high bits of each
        // word together, and then the low bits, everything will be fine.

        // First, shift the high bits down.
        ushr    v2.2d, v1.2d, #63       // the b_i for t
        ushr    v3.2d, v1.2d, #62       // the b_i for t^2
        ushr    v4.2d, v1.2d, #57       // the b_i for t^7
        eor     v2.16b, v2.16b, v3.16b  // add them all together
        eor     v2.16b, v2.16b, v4.16b
        vshr128 v3, v2, 64
        vshl128 v4, v2, 64
        eor     v1.16b, v1.16b, v3.16b  // contribution into high half
        eor     v0.16b, v0.16b, v4.16b  // and low half

        // And then shift the low bits up.
        shl     v2.2d, v1.2d, #1
        shl     v3.2d, v1.2d, #2
        shl     v4.2d, v1.2d, #7
        eor     v1.16b, v1.16b, v2.16b  // unit and t contribs
        eor     v3.16b, v3.16b, v4.16b  // t^2 and t^7 contribs
        eor     v0.16b, v0.16b, v1.16b  // mix everything together
        eor     v0.16b, v0.16b, v3.16b  // ... and we're done
.endm

.macro  mul64
        // Enter with u and v in the low halves of v0 and v1, respectively;
        // leave with z = u v in x2.  Clobbers x2--x4.

        // The multiplication is thankfully easy.
        // v0 =                                 // (u; ?)
        // v1 =                                 // (v; ?)
        pmull   v0.1q, v0.1d, v1.1d             // u v

        // Now we must reduce.  This is essentially the same as the 128-bit
        // case above, but mostly simpler because everything is smaller.  The
        // polynomial this time is p(t) = t^64 + t^4 + t^3 + t + 1.

        // Before we get stuck in, transfer the product to general-purpose
        // registers.
        mov     x3, v0.d[1]
        mov     x2, v0.d[0]

        // First, shift the high bits down.
        eor     x4, x3, x3, lsr #1      // pre-mix t^3 and t^4
        eor     x3, x3, x3, lsr #63     // mix in t contribution
        eor     x3, x3, x4, lsr #60     // shift and mix in t^3 and t^4

        // And then shift the low bits up.
        eor     x3, x3, x3, lsl #1      // mix unit and t; pre-mix t^3, t^4
        eor     x2, x2, x3              // fold them in
        eor     x2, x2, x3, lsl #3      // and t^3 and t^4
.endm

.macro  mul96
        // Enter with u in the least-significant 96 bits of v0, with zero in
        // the upper 32 bits, and with the least-significant 64 bits of v in
        // both halves of v1, and the upper 32 bits of v in the low 32 bits
        // of each half of v2, with zero in the upper 32 bits; and with zero
        // in v31.  Yes, that's a bit hairy.  Leave with the product u v in
        // the low 96 bits of v0, and /junk/ in the high 32 bits.  Clobbers
        // v1--v6.

        // This is an inconvenient size.  There's nothing for it but to do
        // four multiplications, as if for the 128-bit case.  It's possible
        // that there's cruft in the top 32 bits of the input registers, so
        // shift both of them up by four bytes before we start.  This will
        // mean that the high 64 bits of the result (from GCM's viewpoint)
        // will be zero.
        // v0 =                         // (u_0 + u_1 t^32; u_2)
        // v1 =                         // (v_0 + v_1 t^32; v_0 + v_1 t^32)
        // v2 =                         // (v_2; v_2)
        pmull2  v5.1q, v0.2d, v1.2d     // u_2 (v_0 + v_1 t^32) t^32 = e_0
        pmull   v4.1q, v0.1d, v2.1d     // v_2 (u_0 + u_1 t^32) t^32 = e_1
        pmull2  v6.1q, v0.2d, v2.2d     // u_2 v_2 = d = (d; 0)
        pmull   v3.1q, v0.1d, v1.1d     // u_0 v_0 + (u_0 v_1 + u_1 v_0) t^32
                                        //   + u_1 v_1 t^64 = f

        // Extract the high and low halves of the 192-bit result.  The answer
        // we want is d t^128 + e t^64 + f, where e = e_0 + e_1.  The low 96
        // bits of the answer will end up in v0, with junk in the top 32
        // bits; the high 96 bits will end up in v1, which must have zero in
        // its top 32 bits.
        //
        // Here, bot(x) is the low 96 bits of a 192-bit quantity x, arranged
        // in the low 96 bits of a SIMD register, with junk in the top 32
        // bits; and top(x) is the high 96 bits, also arranged in the low 96
        // bits of a register, with /zero/ in the top 32 bits.
        eor     v4.16b, v4.16b, v5.16b  // e_0 + e_1 = e
        vshl128 v6, v6, 32              // top(d t^128)
        vshr128 v5, v4, 32              // top(e t^64)
        vshl128 v4, v4, 64              // bot(e t^64)
        vshr128 v1, v3, 96              // top(f)
        eor     v6.16b, v6.16b, v5.16b  // top(d t^128 + e t^64)
        eor     v0.16b, v3.16b, v4.16b  // bot([d t^128] + e t^64 + f)
        eor     v1.16b, v1.16b, v6.16b  // top(e t^64 + d t^128 + f)

        // Finally, the reduction.  This is essentially the same as the
        // 128-bit case, except that the polynomial is p(t) = t^96 + t^10 +
        // t^9 + t^6 + 1.  The degrees are larger but not enough to cause
        // trouble for the general approach.  Unfortunately, we have to do
        // this in 32-bit pieces rather than 64.

        // First, shift the high bits down.
        ushr    v2.4s, v1.4s, #26       // the b_i for t^6
        ushr    v3.4s, v1.4s, #23       // the b_i for t^9
        ushr    v4.4s, v1.4s, #22       // the b_i for t^10
        eor     v2.16b, v2.16b, v3.16b  // add them all together
        eor     v2.16b, v2.16b, v4.16b
        vshr128 v3, v2, 64              // contribution for high half
        vshl128 v2, v2, 32              // contribution for low half
        eor     v1.16b, v1.16b, v3.16b  // apply to high half
        eor     v0.16b, v0.16b, v2.16b  // and low half

        // And then shift the low bits up.
        shl     v2.4s, v1.4s, #6
        shl     v3.4s, v1.4s, #9
        shl     v4.4s, v1.4s, #10
        eor     v1.16b, v1.16b, v2.16b  // unit and t^6 contribs
        eor     v3.16b, v3.16b, v4.16b  // t^9 and t^10 contribs
        eor     v0.16b, v0.16b, v1.16b  // mix everything together
        eor     v0.16b, v0.16b, v3.16b  // ... and we're done
.endm

.macro  mul192
        // Enter with u in v0 and the less-significant half of v1, with v
        // duplicated across both halves of v2/v3/v4, and with zero in v31.
        // Leave with the product u v in v0 and the bottom half of v1.
        // Clobbers v16--v25.

        // Start multiplying and accumulating pieces of product.
        // v0 =                         // (u_0; u_1)
        // v1 =                         // (u_2; ?)
        // v2 =                         // (v_0; v_0)
        // v3 =                         // (v_1; v_1)
        // v4 =                         // (v_2; v_2)
        pmull   v16.1q, v0.1d, v2.1d    //   a = u_0 v_0

        pmull   v19.1q, v0.1d, v3.1d    //       u_0 v_1
        pmull2  v21.1q, v0.2d, v2.2d    //       u_1 v_0

        pmull   v17.1q, v0.1d, v4.1d    //       u_0 v_2
        pmull2  v22.1q, v0.2d, v3.2d    //       u_1 v_1
        pmull   v23.1q, v1.1d, v2.1d    //       u_2 v_0
         eor    v19.16b, v19.16b, v21.16b // b = u_0 v_1 + u_1 v_0

        pmull2  v20.1q, v0.2d, v4.2d    //       u_1 v_2
        pmull   v24.1q, v1.1d, v3.1d    //       u_2 v_1
         eor    v17.16b, v17.16b, v22.16b //     u_0 v_2 + u_1 v_1

        pmull   v18.1q, v1.1d, v4.1d    //   e = u_2 v_2
         eor    v17.16b, v17.16b, v23.16b // c = u_0 v_2 + u_1 v_1 + u_2 v_1
         eor    v20.16b, v20.16b, v24.16b // d = u_1 v_2 + u_2 v_1

        // Piece the product together.
        // v16 =                        // (a_0; a_1)
        // v19 =                        // (b_0; b_1)
        // v17 =                        // (c_0; c_1)
        // v20 =                        // (d_0; d_1)
        // v18 =                        // (e_0; e_1)
        vshl128 v21, v19, 64            // (0; b_0)
        ext     v22.16b, v19.16b, v20.16b, #8 // (b_1; d_0)
        vshr128 v23, v20, 64            // (d_1; 0)
        eor     v16.16b, v16.16b, v21.16b // (x_0; x_1)
        eor     v17.16b, v17.16b, v22.16b // (x_2; x_3)
        eor     v18.16b, v18.16b, v23.16b // (x_2; x_3)

        // Next, the reduction.  Our polynomial this time is p(x) = t^192 +
        // t^7 + t^2 + t + 1.  Yes, the magic numbers are the same as the
        // 128-bit case.  I don't know why.

        // First, shift the high bits down.
        // v16 =                        // (y_0; y_1)
        // v17 =                        // (y_2; y_3)
        // v18 =                        // (y_4; y_5)
        mov     v19.d[0], v17.d[1]      // (y_3; ?)

        ushr    v23.2d, v18.2d, #63     // hi b_i for t
        ushr    d20, d19, #63           // lo b_i for t
        ushr    v24.2d, v18.2d, #62     // hi b_i for t^2
        ushr    d21, d19, #62           // lo b_i for t^2
        ushr    v25.2d, v18.2d, #57     // hi b_i for t^7
        ushr    d22, d19, #57           // lo b_i for t^7
        eor     v23.16b, v23.16b, v24.16b // mix them all together
        eor     v20.8b, v20.8b, v21.8b
        eor     v23.16b, v23.16b, v25.16b
        eor     v20.8b, v20.8b, v22.8b

        // Permute the high pieces while we fold in the b_i.
        eor     v17.16b, v17.16b, v23.16b
        vshl128 v20, v20, 64
        mov     v19.d[0], v18.d[1]      // (y_5; ?)
        ext     v18.16b, v17.16b, v18.16b, #8 // (y_3; y_4)
        eor     v16.16b, v16.16b, v20.16b

        // And finally shift the low bits up.
        // v16 =                        // (y'_0; y'_1)
        // v17 =                        // (y'_2; ?)
        // v18 =                        // (y'_3; y'_4)
        // v19 =                        // (y'_5; ?)
        shl     v20.2d, v18.2d, #1
        shl     d23, d19, #1
        shl     v21.2d, v18.2d, #2
        shl     d24, d19, #2
        shl     v22.2d, v18.2d, #7
        shl     d25, d19, #7
        eor     v18.16b, v18.16b, v20.16b // unit and t contribs
        eor     v19.8b, v19.8b, v23.8b
        eor     v21.16b, v21.16b, v22.16b // t^2 and t^7 contribs
        eor     v24.8b, v24.8b, v25.8b
        eor     v18.16b, v18.16b, v21.16b // all contribs
        eor     v19.8b, v19.8b, v24.8b
        eor     v0.16b, v16.16b, v18.16b // mix them into the low half
        eor     v1.8b, v17.8b, v19.8b
.endm

.macro  mul256
        // Enter with u in v0/v1, with v duplicated across both halves of
        // v2--v5, and with zero in v31.  Leave with the product u v in
        // v0/v1.  Clobbers ???.

        // Now it's starting to look worthwhile to do Karatsuba.  Suppose
        // u = u_0 + u_1 B and v = v_0 + v_1 B.  Then
        //
        //      u v = (u_0 v_0) + (u_0 v_1 + u_1 v_0) B + (u_1 v_1) B^2
        //
        // Name these coefficients of B^i be a, b, and c, respectively, and
        // let r = u_0 + u_1 and s = v_0 + v_1.  Then observe that
        //
        //      q = r s = (u_0 + u_1) (v_0 + v_1)
        //        = (u_0 v_0) + (u1 v_1) + (u_0 v_1 + u_1 v_0)
        //        = a + d + c
        //
        // The first two terms we've already calculated; the last is the
        // remaining one we want.  We'll set B = t^128.  We know how to do
        // 128-bit multiplications already, and Karatsuba is too annoying
        // there, so there'll be 12 multiplications altogether, rather than
        // the 16 we'd have if we did this the naïve way.
        // v0 =                         // u_0 = (u_00; u_01)
        // v1 =                         // u_1 = (u_10; u_11)
        // v2 =                         // (v_00; v_00)
        // v3 =                         // (v_01; v_01)
        // v4 =                         // (v_10; v_10)
        // v5 =                         // (v_11; v_11)

        eor     v28.16b, v0.16b, v1.16b // u_* = (u_00 + u_10; u_01 + u_11)
        eor     v29.16b, v2.16b, v4.16b // v_*0 = v_00 + v_10
        eor     v30.16b, v3.16b, v5.16b // v_*1 = v_01 + v_11

        // Start by building the cross product, q = u_* v_*.
        pmull   v24.1q, v28.1d, v30.1d  // u_*0 v_*1
        pmull2  v25.1q, v28.2d, v29.2d  // u_*1 v_*0
        pmull   v20.1q, v28.1d, v29.1d  // u_*0 v_*0
        pmull2  v21.1q, v28.2d, v30.2d  // u_*1 v_*1
        eor     v24.16b, v24.16b, v25.16b // u_*0 v_*1 + u_*1 v_*0
        vshr128 v25, v24, 64
        vshl128 v24, v24, 64
        eor     v20.16b, v20.16b, v24.16b // q_0
        eor     v21.16b, v21.16b, v25.16b // q_1

        // Next, work on the low half, a = u_0 v_0
        pmull   v24.1q, v0.1d, v3.1d    // u_00 v_01
        pmull2  v25.1q, v0.2d, v2.2d    // u_01 v_00
        pmull   v16.1q, v0.1d, v2.1d    // u_00 v_00
        pmull2  v17.1q, v0.2d, v3.2d    // u_01 v_01
        eor     v24.16b, v24.16b, v25.16b // u_00 v_01 + u_01 v_00
        vshr128 v25, v24, 64
        vshl128 v24, v24, 64
        eor     v16.16b, v16.16b, v24.16b // a_0
        eor     v17.16b, v17.16b, v25.16b // a_1

        // Mix the pieces we have so far.
        eor     v20.16b, v20.16b, v16.16b
        eor     v21.16b, v21.16b, v17.16b

        // Finally, work on the high half, c = u_1 v_1
        pmull   v24.1q, v1.1d, v5.1d    // u_10 v_11
        pmull2  v25.1q, v1.2d, v4.2d    // u_11 v_10
        pmull   v18.1q, v1.1d, v4.1d    // u_10 v_10
        pmull2  v19.1q, v1.2d, v5.2d    // u_11 v_11
        eor     v24.16b, v24.16b, v25.16b // u_10 v_11 + u_11 v_10
        vshr128 v25, v24, 64
        vshl128 v24, v24, 64
        eor     v18.16b, v18.16b, v24.16b // c_0
        eor     v19.16b, v19.16b, v25.16b // c_1

        // Finish mixing the product together.
        eor     v20.16b, v20.16b, v18.16b
        eor     v21.16b, v21.16b, v19.16b
        eor     v17.16b, v17.16b, v20.16b
        eor     v18.16b, v18.16b, v21.16b

        // Now we must reduce.  This is essentially the same as the 192-bit
        // case above, but more complicated because everything is bigger.
        // The polynomial this time is p(t) = t^256 + t^10 + t^5 + t^2 + 1.
        // v16 =                        // (y_0; y_1)
        // v17 =                        // (y_2; y_3)
        // v18 =                        // (y_4; y_5)
        // v19 =                        // (y_6; y_7)
        ushr    v24.2d, v18.2d, #62     // (y_4; y_5) b_i for t^2
        ushr    v25.2d, v19.2d, #62     // (y_6; y_7) b_i for t^2
        ushr    v26.2d, v18.2d, #59     // (y_4; y_5) b_i for t^5
        ushr    v27.2d, v19.2d, #59     // (y_6; y_7) b_i for t^5
        ushr    v28.2d, v18.2d, #54     // (y_4; y_5) b_i for t^10
        ushr    v29.2d, v19.2d, #54     // (y_6; y_7) b_i for t^10
        eor     v24.16b, v24.16b, v26.16b // mix the contributions together
        eor     v25.16b, v25.16b, v27.16b
        eor     v24.16b, v24.16b, v28.16b
        eor     v25.16b, v25.16b, v29.16b
        vshr128 v26, v25, 64            // slide contribs into position
        ext     v25.16b, v24.16b, v25.16b, #8
        vshl128 v24, v24, 64
        eor     v18.16b, v18.16b, v26.16b
        eor     v17.16b, v17.16b, v25.16b
        eor     v16.16b, v16.16b, v24.16b

        // And then shift the low bits up.
        // v16 =                        // (y'_0; y'_1)
        // v17 =                        // (y'_2; y'_3)
        // v18 =                        // (y'_4; y'_5)
        // v19 =                        // (y'_6; y'_7)
        shl     v24.2d, v18.2d, #2      // (y'_4; y_5) a_i for t^2
        shl     v25.2d, v19.2d, #2      // (y_6; y_7) a_i for t^2
        shl     v26.2d, v18.2d, #5      // (y'_4; y_5) a_i for t^5
        shl     v27.2d, v19.2d, #5      // (y_6; y_7) a_i for t^5
        shl     v28.2d, v18.2d, #10     // (y'_4; y_5) a_i for t^10
        shl     v29.2d, v19.2d, #10     // (y_6; y_7) a_i for t^10
        eor     v18.16b, v18.16b, v24.16b // mix the contributions together
        eor     v19.16b, v19.16b, v25.16b
        eor     v26.16b, v26.16b, v28.16b
        eor     v27.16b, v27.16b, v29.16b
        eor     v18.16b, v18.16b, v26.16b
        eor     v19.16b, v19.16b, v27.16b
        eor     v0.16b, v16.16b, v18.16b
        eor     v1.16b, v17.16b, v19.16b
.endm

///--------------------------------------------------------------------------
/// Main code.

// There are a number of representations of field elements in this code and
// it can be confusing.
//
//   * The `external format' consists of a sequence of contiguous bytes in
//     memory called a `block'.  The GCM spec explains how to interpret this
//     block as an element of a finite field.  As discussed extensively, this
//     representation is very annoying for a number of reasons.  On the other
//     hand, this code never actually deals with it directly.
//
//   * The `register format' consists of one or more SIMD registers,
//     depending on the block size.  The bits in each byte are reversed,
//     compared to the external format, which makes the polynomials
//     completely vanilla, unlike all of the other GCM implementations.
//
//   * The `table format' is just like the `register format', only the two
//     halves of 128-bit SIMD register are the same, so we need twice as many
//     registers.
//
//   * The `words' format consists of a sequence of bytes, as in the
//     `external format', but, according to the blockcipher in use, the bytes
//     within each 32-bit word may be reversed (`big-endian') or not
//     (`little-endian').  Accordingly, there are separate entry points for
//     each variant, identified with `b' or `l'.

FUNC(gcm_mulk_128b_arm64_pmull)
        // On entry, x0 points to a 128-bit field element A in big-endian
        // words format; x1 points to a field-element K in table format.  On
        // exit, A is updated with the product A K.

        ldr     q0, [x0]
        ldp     q1, q2, [x1]
        rev32   v0.16b, v0.16b
        vzero
        rbit    v0.16b, v0.16b
        mul128
        rbit    v0.16b, v0.16b
        rev32   v0.16b, v0.16b
        str     q0, [x0]
        ret
ENDFUNC

FUNC(gcm_mulk_128l_arm64_pmull)
        // On entry, x0 points to a 128-bit field element A in little-endian
        // words format; x1 points to a field-element K in table format.  On
        // exit, A is updated with the product A K.

        ldr     q0, [x0]
        ldp     q1, q2, [x1]
        vzero
        rbit    v0.16b, v0.16b
        mul128
        rbit    v0.16b, v0.16b
        str     q0, [x0]
        ret
ENDFUNC

FUNC(gcm_mulk_64b_arm64_pmull)
        // On entry, x0 points to a 64-bit field element A in big-endian
        // words format; x1 points to a field-element K in table format.  On
        // exit, A is updated with the product A K.

        ldr     d0, [x0]
        ldr     q1, [x1]
        rev32   v0.8b, v0.8b
        rbit    v0.8b, v0.8b
        mul64
        rbit    x2, x2
        ror     x2, x2, #32
        str     x2, [x0]
        ret
ENDFUNC

FUNC(gcm_mulk_64l_arm64_pmull)
        // On entry, x0 points to a 64-bit field element A in little-endian
        // words format; x1 points to a field-element K in table format.  On
        // exit, A is updated with the product A K.

        ldr     d0, [x0]
        ldr     q1, [x1]
        rbit    v0.8b, v0.8b
        mul64
        rbit    x2, x2
        rev     x2, x2
        str     x2, [x0]
        ret
ENDFUNC

FUNC(gcm_mulk_96b_arm64_pmull)
        // On entry, x0 points to a 96-bit field element A in big-endian
        // words format; x1 points to a field-element K in table format.  On
        // exit, A is updated with the product A K.

        ldr     w2, [x0, #8]
        ldr     d0, [x0, #0]
        mov     v0.d[1], x2
        ldp     q1, q2, [x1]
        rev32   v0.16b, v0.16b
        vzero
        rbit    v0.16b, v0.16b
        mul96
        rbit    v0.16b, v0.16b
        rev32   v0.16b, v0.16b
        mov     w2, v0.s[2]
        str     d0, [x0, #0]
        str     w2, [x0, #8]
        ret
ENDFUNC

FUNC(gcm_mulk_96l_arm64_pmull)
        // On entry, x0 points to a 96-bit field element A in little-endian
        // words format; x1 points to a field-element K in table format.  On
        // exit, A is updated with the product A K.

        ldr     d0, [x0, #0]
        ldr     w2, [x0, #8]
        mov     v0.d[1], x2
        ldp     q1, q2, [x1]
        rbit    v0.16b, v0.16b
        vzero
        mul96
        rbit    v0.16b, v0.16b
        mov     w2, v0.s[2]
        str     d0, [x0, #0]
        str     w2, [x0, #8]
        ret
ENDFUNC

FUNC(gcm_mulk_192b_arm64_pmull)
        // On entry, x0 points to a 192-bit field element A in big-endian
        // words format; x1 points to a field-element K in table format.  On
        // exit, A is updated with the product A K.

        ldr     q0, [x0, #0]
        ldr     d1, [x0, #16]
        ldp     q2, q3, [x1, #0]
        ldr     q4, [x1, #32]
        rev32   v0.16b, v0.16b
        rev32   v1.8b, v1.8b
        rbit    v0.16b, v0.16b
        rbit    v1.8b, v1.8b
        vzero
        mul192
        rev32   v0.16b, v0.16b
        rev32   v1.8b, v1.8b
        rbit    v0.16b, v0.16b
        rbit    v1.8b, v1.8b
        str     q0, [x0, #0]
        str     d1, [x0, #16]
        ret
ENDFUNC

FUNC(gcm_mulk_192l_arm64_pmull)
        // On entry, x0 points to a 192-bit field element A in little-endian
        // words format; x1 points to a field-element K in table format.  On
        // exit, A is updated with the product A K.

        ldr     q0, [x0, #0]
        ldr     d1, [x0, #16]
        ldp     q2, q3, [x1, #0]
        ldr     q4, [x1, #32]
        rbit    v0.16b, v0.16b
        rbit    v1.8b, v1.8b
        vzero
        mul192
        rbit    v0.16b, v0.16b
        rbit    v1.8b, v1.8b
        str     q0, [x0, #0]
        str     d1, [x0, #16]
        ret
ENDFUNC

FUNC(gcm_mulk_256b_arm64_pmull)
        // On entry, x0 points to a 256-bit field element A in big-endian
        // words format; x1 points to a field-element K in table format.  On
        // exit, A is updated with the product A K.

        ldp     q0, q1, [x0]
        ldp     q2, q3, [x1, #0]
        ldp     q4, q5, [x1, #32]
        rev32   v0.16b, v0.16b
        rev32   v1.16b, v1.16b
        rbit    v0.16b, v0.16b
        rbit    v1.16b, v1.16b
        vzero
        mul256
        rev32   v0.16b, v0.16b
        rev32   v1.16b, v1.16b
        rbit    v0.16b, v0.16b
        rbit    v1.16b, v1.16b
        stp     q0, q1, [x0]
        ret
ENDFUNC

FUNC(gcm_mulk_256l_arm64_pmull)
        // On entry, x0 points to a 256-bit field element A in little-endian
        // words format; x1 points to a field-element K in table format.  On
        // exit, A is updated with the product A K.

        ldp     q0, q1, [x0]
        ldp     q2, q3, [x1, #0]
        ldp     q4, q5, [x1, #32]
        rbit    v0.16b, v0.16b
        rbit    v1.16b, v1.16b
        vzero
        mul256
        rbit    v0.16b, v0.16b
        rbit    v1.16b, v1.16b
        stp     q0, q1, [x0]
        ret
ENDFUNC

///----- That's all, folks --------------------------------------------------