1 /// -*- mode: asm; asm-comment-char: ?/ -*-
3 /// Fancy SIMD implementation of ChaCha
5 /// (c) 2015 Straylight/Edgeware
8 ///----- Licensing notice ---------------------------------------------------
10 /// This file is part of Catacomb.
12 /// Catacomb is free software; you can redistribute it and/or modify
13 /// it under the terms of the GNU Library General Public License as
14 /// published by the Free Software Foundation; either version 2 of the
15 /// License, or (at your option) any later version.
17 /// Catacomb is distributed in the hope that it will be useful,
18 /// but WITHOUT ANY WARRANTY; without even the implied warranty of
19 /// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 /// GNU Library General Public License for more details.
22 /// You should have received a copy of the GNU Library General Public
23 /// License along with Catacomb; if not, write to the Free
24 /// Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
25 /// MA 02111-1307, USA.
27 ///--------------------------------------------------------------------------
28 /// External definitions.
31 #include "asm-common.h"
33 ///--------------------------------------------------------------------------
39 FUNC(chacha_core_x86ish_sse2)
44 // Arguments come in on the stack, and will need to be collected. We
45 // we can get away with just the scratch registers for integer work,
46 // but we'll run out of XMM registers and will need some properly
47 // aligned space which we'll steal from the stack. I don't trust the
48 // stack pointer's alignment, so I'll have to mask the stack pointer,
49 // which in turn means I'll need to keep track of the old value.
50 // Hence I'm making a full i386-style stack frame here.
52 // The Windows and SysV ABIs are sufficiently similar that we don't
53 // need to worry about the differences here.
72 #if CPUFAM_AMD64 && ABI_SYSV
73 // This is nice. We have plenty of XMM registers, and the arguments
74 // are in useful places. There's no need to spill anything and we
75 // can just get on with the code.
86 #if CPUFAM_AMD64 && ABI_WIN
87 // Arguments come in registers, but they're different between Windows
88 // and everyone else (and everyone else is saner).
90 // The Windows ABI insists that we preserve some of the XMM
91 // registers, but we want more than we can use as scratch space. We
92 // only need to save a copy of the input for the feedforward at the
93 // end, so we might as well use memory rather than spill extra
94 // registers. (We need an extra 8 bytes to align the stack.)
100 # define SAVE1 [rsp + 0]
101 # define SAVE2 [rsp + 16]
102 # define SAVE3 [rsp + 32]
109 // First job is to slurp the matrix into XMM registers. Be careful:
110 // the input matrix isn't likely to be properly aligned.
112 // [ 0 1 2 3] (a, xmm0)
113 // [ 4 5 6 7] (b, xmm1)
114 // [ 8 9 10 11] (c, xmm2)
115 // [12 13 14 15] (d, xmm3)
116 movdqu xmm0, [IN + 0]
117 movdqu xmm1, [IN + 16]
118 movdqu xmm2, [IN + 32]
119 movdqu xmm3, [IN + 48]
121 // Take a copy for later. This one is aligned properly, by
129 // Apply a column quarterround to each of the columns simultaneously.
130 // Alas, there doesn't seem to be a packed doubleword rotate, so we
131 // have to synthesize it.
133 // a += b; d ^= a; d <<<= 16
141 // c += d; b ^= c; b <<<= 12
149 // a += b; d ^= a; d <<<= 8
157 // c += d; b ^= c; b <<<= 7
159 pshufd xmm3, xmm3, SHUF(2, 1, 0, 3)
161 pshufd xmm2, xmm2, SHUF(1, 0, 3, 2)
167 // The not-quite-transpose conveniently only involves reordering
168 // elements of individual rows, which can be done quite easily. It
169 // doesn't involve any movement of elements between rows, or even
170 // renaming of the rows.
172 // [ 0 1 2 3] [ 0 1 2 3] (a, xmm0)
173 // [ 4 5 6 7] --> [ 5 6 7 4] (b, xmm1)
174 // [ 8 9 10 11] [10 11 8 9] (c, xmm2)
175 // [12 13 14 15] [15 12 13 14] (d, xmm3)
177 // The shuffles have quite high latency, so they've mostly been
178 // pushed upwards. The remaining one can't be moved, though.
179 pshufd xmm1, xmm1, SHUF(0, 3, 2, 1)
181 // Apply the diagonal quarterround to each of the columns
184 // a += b; d ^= a; d <<<= 16
192 // c += d; b ^= c; b <<<= 12
200 // a += b; d ^= a; d <<<= 8
208 // c += d; b ^= c; b <<<= 7
210 pshufd xmm3, xmm3, SHUF(0, 3, 2, 1)
212 pshufd xmm2, xmm2, SHUF(1, 0, 3, 2)
218 // Finally, finish off undoing the transpose, and we're done for this
219 // doubleround. Again, most of this was done above so we don't have
220 // to wait for the shuffles.
221 pshufd xmm1, xmm1, SHUF(2, 1, 0, 3)
223 // Decrement the loop counter and see if we should go round again.
227 // Almost there. Firstly, the feedforward addition.
233 // And now we write out the result. This one won't be aligned
235 movdqu [OUT + 0], xmm0
236 movdqu [OUT + 16], xmm1
237 movdqu [OUT + 32], xmm2
238 movdqu [OUT + 48], xmm3
245 #if CPUFAM_AMD64 && ABI_WIN
249 // And with that, we're done.
254 ///----- That's all, folks --------------------------------------------------