[catacomb] / symm / rijndael-arm-crypto.S

/// -*- mode: asm; asm-comment-char: ?/ -*-
///
/// ARM crypto-extension-based implementation of Rijndael
///
/// (c) 2016 Straylight/Edgeware
///

///----- Licensing notice ---------------------------------------------------
///
/// This file is part of Catacomb.
///
/// Catacomb is free software; you can redistribute it and/or modify
/// it under the terms of the GNU Library General Public License as
/// published by the Free Software Foundation; either version 2 of the
/// License, or (at your option) any later version.
///
/// Catacomb is distributed in the hope that it will be useful,
/// but WITHOUT ANY WARRANTY; without even the implied warranty of
/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
/// GNU Library General Public License for more details.
///
/// You should have received a copy of the GNU Library General Public
/// License along with Catacomb; if not, write to the Free
/// Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
/// MA 02111-1307, USA.

///--------------------------------------------------------------------------
/// External definitions.

#include "config.h"
#include "asm-common.h"

	.globl	F(abort)
	.globl	F(rijndael_rcon)

///--------------------------------------------------------------------------
/// Main code.

	.arch	armv8-a
	.fpu	crypto-neon-fp-armv8

/// The ARM crypto extension implements a little-endian version of AES
/// (though the manual doesn't actually spell this out and you have to
/// experiment), but Catacomb's internal interface presents as big-endian so
/// as to work better with things like GCM.  We therefore maintain the round
/// keys in little-endian form, and have to end-swap blocks in and out.
///
/// For added amusement, the crypto extension doesn't implement the larger-
/// block versions of Rijndael, so we have to end-swap the keys if we're
/// preparing for one of those.

	// Useful constants.
	.equ	maxrounds, 16		// maximum number of rounds
	.equ	maxblksz, 32		// maximum block size, in bytes
	.equ	kbufsz, maxblksz*(maxrounds + 1) // size of a key-schedule buffer

	// Context structure.
	.equ	nr, 0			// number of rounds
	.equ	w, nr + 4		// encryption key words
	.equ	wi, w + kbufsz		// decryption key words

///--------------------------------------------------------------------------
/// Key setup.

FUNC(rijndael_setup_arm_crypto)

	// Arguments:
	//	r0 = pointer to context
	//	r1 = block size in words
	//	r2 = pointer to key material
	//	r3 = key size in words

	stmfd	sp!, {r4-r9, r14}

	// The initial round key material is taken directly from the input
	// key, so copy it over.  Unfortunately, the key material is not
	// guaranteed to be aligned in any especially useful way, so we must
	// sort this out.
	add	r9, r0, #w
	mov	r14, r3
	ands	r6, r2, #3
	beq	1f
	mov	r6, r6, lsl #3
	rsb	r7, r6, #32
	bic	r2, r2, #3
	ldr	r4, [r2], #4

0:	ldr	r5, [r2], #4
	mov	r4, r4, lsr r6
	orr	r4, r5, lsl r7
	str	r4, [r9], #4
	subs	r14, r14, #1
	movhi	r4, r5
	bhi	0b
	b	9f

1:	ldr	r4, [r2], #4
	str	r4, [r9], #4
	subs	r14, r14, #1
	bhi	1b

	// Find out other useful things and prepare for the main loop.
9:	ldr	r7, [r0, #nr]		// number of rounds
	mla	r2, r1, r7, r1		// total key size in words
	leaextq	r5, rijndael_rcon	// round constants
	sub	r8, r2, r3		// minus what we've copied already
	veor	q1, q1			// all-zero register for the key
	add	r8, r9, r8, lsl #2	// limit of the key buffer

	// Main key expansion loop.  The first word of each key-length chunk
	// needs special treatment.
0:	ldrb	r14, [r5], #1		// next round constant
	ldr	r6, [r9, -r3, lsl #2]
	vdup.32	q0, r4
	aese.8	q0, q1			// effectively, just SubBytes
	vmov.32	r4, d0[0]
	eor	r4, r14, r4, ror #8
	eor	r4, r4, r6
	str	r4, [r9], #4
	cmp	r9, r8
	bcs	9f

	// The next three words are simple.
	ldr	r6, [r9, -r3, lsl #2]
	eor	r4, r4, r6
	str	r4, [r9], #4
	cmp	r9, r8
	bcs	9f

	// (Word 2...)
	ldr	r6, [r9, -r3, lsl #2]
	eor	r4, r4, r6
	str	r4, [r9], #4
	cmp	r9, r8
	bcs	9f

	// (Word 3...)
	ldr	r6, [r9, -r3, lsl #2]
	eor	r4, r4, r6
	str	r4, [r9], #4
	cmp	r9, r8
	bcs	9f

	// Word 4.  If the key is /more/ than 6 words long, then we must
	// apply a substitution here.
	cmp	r3, #5
	bcc	0b
	ldr	r6, [r9, -r3, lsl #2]
	cmp	r3, #7
	bcc	1f
	vdup.32	q0, r4
	aese.8	q0, q1			// effectively, just SubBytes
	vmov.32	r4, d0[0]
1:	eor	r4, r4, r6
	str	r4, [r9], #4
	cmp	r9, r8
	bcs	9f

	// (Word 5...)
	cmp	r3, #6
	bcc	0b
	ldr	r6, [r9, -r3, lsl #2]
	eor	r4, r4, r6
	str	r4, [r9], #4
	cmp	r9, r8
	bcs	9f

	// (Word 6...)
	cmp	r3, #7
	bcc	0b
	ldr	r6, [r9, -r3, lsl #2]
	eor	r4, r4, r6
	str	r4, [r9], #4
	cmp	r9, r8
	bcs	9f

	// (Word 7...)
	cmp	r3, #8
	bcc	0b
	ldr	r6, [r9, -r3, lsl #2]
	eor	r4, r4, r6
	str	r4, [r9], #4
	cmp	r9, r8
	bcs	9f

	// Must be done by now.
	b	0b

	// Next job is to construct the decryption keys.  The keys for the
	// first and last rounds don't need to be mangled, but the remaining
	// ones do -- and they all need to be reordered too.
	//
	// The plan of action, then, is to copy the final encryption round's
	// keys into place first, then to do each of the intermediate rounds
	// in reverse order, and finally do the first round.
	//
	// Do all the heavy lifting with NEON registers.  The order we're
	// doing this in means that it's OK if we read or write too much, and
	// there's easily enough buffer space for the over-enthusiastic reads
	// and writes because the context has space for 32-byte blocks, which
	// is our maximum and an exact fit for two Q-class registers.
9:	add	r5, r0, #wi
	add	r4, r0, #w
	add	r4, r4, r2, lsl #2
	sub	r4, r4, r1, lsl #2		// last round's keys

	// Copy the last encryption round's keys.
	teq	r1, #4
	vldmiaeq r4, {d0, d1}
	vldmiane r4, {d0-d3}
	vstmiaeq r5, {d0, d1}
	vstmiane r5, {d0-d3}

	// Update the loop variables and stop if we've finished.
0:	sub	r4, r4, r1, lsl #2
	add	r5, r5, r1, lsl #2
	subs	r7, r7, #1
	beq	9f

	// Do another middle round's keys...
	teq	r1, #4
	vldmiaeq r4, {d0, d1}
	vldmiane r4, {d0-d3}
	aesimc.8 q0, q0
	vstmiaeq r5, {d0, d1}
	beq	0b
	aesimc.8 q1, q1
	vstmia	r5, {d0-d3}
	b	0b

	// Finally do the first encryption round.
9:	teq	r1, #4
	vldmiaeq r4, {d0, d1}
	vldmiane r4, {d0-d3}
	vstmiaeq r5, {d0, d1}
	vstmiane r5, {d0-d3}

	// If the block size is not exactly four words then we must end-swap
	// everything.  We can use fancy NEON toys for this.
	beq	9f

	// End-swap the encryption keys.
	add	r1, r0, #w
	bl	endswap_block

	// And the decryption keys
	add	r1, r0, #wi
	bl	endswap_block

	// All done.
9:	ldmfd	sp!, {r4-r9, pc}

endswap_block:
	// End-swap R2 words starting at R1.  R1 is clobbered; R2 is not.
	// It's OK to work in 16-byte chunks.
	mov	r4, r2
0:	vldmia	r1, {d0, d1}
	vrev32.8 q0, q0
	vstmia	r1!, {d0, d1}
	subs	r4, r4, #4
	bhi	0b
	bx	r14

ENDFUNC

///--------------------------------------------------------------------------
/// Encrypting and decrypting blocks.

FUNC(rijndael_eblk_arm_crypto)

	// Arguments:
	//	r0 = pointer to context
	//	r1 = pointer to input block
	//	r2 = pointer to output block

	// Set things up ready.
	ldr	r3, [r0, #nr]
	add	r0, r0, #w
	vldmia	r1, {d0, d1}
	vrev32.8 q0, q0

	// Dispatch according to the number of rounds.
	add	r3, r3, r3, lsl #1
	rsbs	r3, r3, #3*14
	addcs	pc, pc, r3, lsl #2
	callext	F(abort)

	// The last round doesn't have MixColumns, so do it separately.
  .rept	13
	vldmia	r0!, {d2, d3}
	aese.8	q0, q1
	aesmc.8	q0, q0
  .endr

	// Final round.
	vldmia	r0!, {d2, d3}
	aese.8	q0, q1

	// Final whitening.
	vldmia	r0!, {d2, d3}
	veor	q0, q1

	// All done.
	vrev32.8 q0, q0
	vstmia	r2, {d0, d1}
	bx	r14

ENDFUNC

FUNC(rijndael_dblk_arm_crypto)

	// Arguments:
	//	r0 = pointer to context
	//	r1 = pointer to input block
	//	r2 = pointer to output block

	// Set things up ready.
	ldr	r3, [r0, #nr]
	add	r0, r0, #wi
	vldmia	r1, {d0, d1}
	vrev32.8 q0, q0

	// Dispatch according to the number of rounds.
	add	r3, r3, r3, lsl #1
	rsbs	r3, r3, #3*14
	addcs	pc, pc, r3, lsl #2
	callext	F(abort)

	// The last round doesn't have MixColumns, so do it separately.
  .rept	13
	vldmia	r0!, {d2, d3}
	aesd.8	q0, q1
	aesimc.8 q0, q0
  .endr

	// Final round.
	vldmia	r0!, {d2, d3}
	aesd.8	q0, q1

	// Final whitening.
	vldmia	r0!, {d2, d3}
	veor	q0, q1

	// All done.
	vrev32.8 q0, q0
	vstmia	r2, {d0, d1}
	bx	r14

ENDFUNC

///----- That's all, folks --------------------------------------------------
Commit	Line	Data
26e182fc MW	1	/// -- mode: asm; asm-comment-char: ?/ --
	2	///
	3	/// ARM crypto-extension-based implementation of Rijndael
	4	///
	5	/// (c) 2016 Straylight/Edgeware
	6	///
	7
	8	///----- Licensing notice ---------------------------------------------------
	9	///
	10	/// This file is part of Catacomb.
	11	///
	12	/// Catacomb is free software; you can redistribute it and/or modify
	13	/// it under the terms of the GNU Library General Public License as
	14	/// published by the Free Software Foundation; either version 2 of the
	15	/// License, or (at your option) any later version.
	16	///
	17	/// Catacomb is distributed in the hope that it will be useful,
	18	/// but WITHOUT ANY WARRANTY; without even the implied warranty of
	19	/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	20	/// GNU Library General Public License for more details.
	21	///
	22	/// You should have received a copy of the GNU Library General Public
	23	/// License along with Catacomb; if not, write to the Free
	24	/// Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
	25	/// MA 02111-1307, USA.
	26
	27	///--------------------------------------------------------------------------
	28	/// External definitions.
	29
	30	#include "config.h"
	31	#include "asm-common.h"
	32
	33	.globl F(abort)
	34	.globl F(rijndael_rcon)
	35
	36	///--------------------------------------------------------------------------
	37	/// Main code.
	38
	39	.arch armv8-a
	40	.fpu crypto-neon-fp-armv8
	41
	42	/// The ARM crypto extension implements a little-endian version of AES
	43	/// (though the manual doesn't actually spell this out and you have to
	44	/// experiment), but Catacomb's internal interface presents as big-endian so
	45	/// as to work better with things like GCM. We therefore maintain the round
	46	/// keys in little-endian form, and have to end-swap blocks in and out.
	47	///
	48	/// For added amusement, the crypto extension doesn't implement the larger-
	49	/// block versions of Rijndael, so we have to end-swap the keys if we're
	50	/// preparing for one of those.
	51
	52	// Useful constants.
	53	.equ maxrounds, 16 // maximum number of rounds
	54	.equ maxblksz, 32 // maximum block size, in bytes
	55	.equ kbufsz, maxblksz*(maxrounds + 1) // size of a key-schedule buffer
	56
	57	// Context structure.
	58	.equ nr, 0 // number of rounds
	59	.equ w, nr + 4 // encryption key words
	60	.equ wi, w + kbufsz // decryption key words
	61
	62	///--------------------------------------------------------------------------
	63	/// Key setup.
	64
65	FUNC(rijndael_setup_arm_crypto)
66
67	// Arguments:
68	// r0 = pointer to context
69	// r1 = block size in words
70	// r2 = pointer to key material
71	// r3 = key size in words
72
73	stmfd sp!, {r4-r9, r14}
74
75	// The initial round key material is taken directly from the input
76	// key, so copy it over. Unfortunately, the key material is not
77	// guaranteed to be aligned in any especially useful way, so we must
78	// sort this out.
79	add r9, r0, #w
80	mov r14, r3
c82543b7	81	ands r6, r2, #3
26e182fc	82	beq 1f
c82543b7 MW	83	mov r6, r6, lsl #3
c82543b7 MW	84	rsb r7, r6, #32
26e182fc	85	bic r2, r2, #3
c82543b7	86	ldr r4, [r2], #4
26e182fc	87
c82543b7 MW	88	0: ldr r5, [r2], #4
	89	mov r4, r4, lsr r6
	90	orr r4, r5, lsl r7
	91	str r4, [r9], #4
26e182fc	92	subs r14, r14, #1
c82543b7	93	movhi r4, r5
26e182fc MW	94	bhi 0b
	95	b 9f
	96
c82543b7 MW	97	1: ldr r4, [r2], #4
c82543b7 MW	98	str r4, [r9], #4
26e182fc MW	99	subs r14, r14, #1
	100	bhi 1b
	101
	102	// Find out other useful things and prepare for the main loop.
aec6bc36	103	9: ldr r7, [r0, #nr] // number of rounds
26e182fc	104	mla r2, r1, r7, r1 // total key size in words
26e182fc MW	105	leaextq r5, rijndael_rcon // round constants
	106	sub r8, r2, r3 // minus what we've copied already
	107	veor q1, q1 // all-zero register for the key
	108	add r8, r9, r8, lsl #2 // limit of the key buffer
	109
	110	// Main key expansion loop. The first word of each key-length chunk
	111	// needs special treatment.
f71eed58	112	0: ldrb r14, [r5], #1 // next round constant
26e182fc MW	113	ldr r6, [r9, -r3, lsl #2]
	114	vdup.32 q0, r4
	115	aese.8 q0, q1 // effectively, just SubBytes
	116	vmov.32 r4, d0[0]
	117	eor r4, r14, r4, ror #8
	118	eor r4, r4, r6
	119	str r4, [r9], #4
	120	cmp r9, r8
f71eed58	121	bcs 9f
26e182fc MW	122
	123	// The next three words are simple.
	124	ldr r6, [r9, -r3, lsl #2]
	125	eor r4, r4, r6
	126	str r4, [r9], #4
	127	cmp r9, r8
f71eed58	128	bcs 9f
26e182fc MW	129
	130	// (Word 2...)
	131	ldr r6, [r9, -r3, lsl #2]
	132	eor r4, r4, r6
	133	str r4, [r9], #4
	134	cmp r9, r8
f71eed58	135	bcs 9f
26e182fc MW	136
	137	// (Word 3...)
	138	ldr r6, [r9, -r3, lsl #2]
	139	eor r4, r4, r6
	140	str r4, [r9], #4
	141	cmp r9, r8
f71eed58	142	bcs 9f
26e182fc MW	143
	144	// Word 4. If the key is /more/ than 6 words long, then we must
	145	// apply a substitution here.
	146	cmp r3, #5
f71eed58	147	bcc 0b
26e182fc MW	148	ldr r6, [r9, -r3, lsl #2]
26e182fc MW	149	cmp r3, #7
f71eed58	150	bcc 1f
26e182fc MW	151	vdup.32 q0, r4
	152	aese.8 q0, q1 // effectively, just SubBytes
	153	vmov.32 r4, d0[0]
f71eed58	154	1: eor r4, r4, r6
26e182fc MW	155	str r4, [r9], #4
26e182fc MW	156	cmp r9, r8
f71eed58	157	bcs 9f
26e182fc MW	158
	159	// (Word 5...)
	160	cmp r3, #6
f71eed58	161	bcc 0b
26e182fc MW	162	ldr r6, [r9, -r3, lsl #2]
	163	eor r4, r4, r6
	164	str r4, [r9], #4
	165	cmp r9, r8
f71eed58	166	bcs 9f
26e182fc MW	167
	168	// (Word 6...)
	169	cmp r3, #7
f71eed58	170	bcc 0b
26e182fc MW	171	ldr r6, [r9, -r3, lsl #2]
	172	eor r4, r4, r6
	173	str r4, [r9], #4
	174	cmp r9, r8
f71eed58	175	bcs 9f
26e182fc MW	176
	177	// (Word 7...)
	178	cmp r3, #8
f71eed58	179	bcc 0b
26e182fc MW	180	ldr r6, [r9, -r3, lsl #2]
	181	eor r4, r4, r6
	182	str r4, [r9], #4
	183	cmp r9, r8
f71eed58	184	bcs 9f
26e182fc MW	185
26e182fc MW	186	// Must be done by now.
f71eed58	187	b 0b
26e182fc MW	188
	189	// Next job is to construct the decryption keys. The keys for the
	190	// first and last rounds don't need to be mangled, but the remaining
	191	// ones do -- and they all need to be reordered too.
	192	//
	193	// The plan of action, then, is to copy the final encryption round's
	194	// keys into place first, then to do each of the intermediate rounds
	195	// in reverse order, and finally do the first round.
	196	//
	197	// Do all the heavy lifting with NEON registers. The order we're
	198	// doing this in means that it's OK if we read or write too much, and
	199	// there's easily enough buffer space for the over-enthusiastic reads
	200	// and writes because the context has space for 32-byte blocks, which
	201	// is our maximum and an exact fit for two Q-class registers.
f71eed58	202	9: add r5, r0, #wi
26e182fc MW	203	add r4, r0, #w
	204	add r4, r4, r2, lsl #2
	205	sub r4, r4, r1, lsl #2 // last round's keys
	206
	207	// Copy the last encryption round's keys.
	208	teq r1, #4
	209	vldmiaeq r4, {d0, d1}
	210	vldmiane r4, {d0-d3}
	211	vstmiaeq r5, {d0, d1}
	212	vstmiane r5, {d0-d3}
	213
	214	// Update the loop variables and stop if we've finished.
f71eed58	215	0: sub r4, r4, r1, lsl #2
26e182fc MW	216	add r5, r5, r1, lsl #2
26e182fc MW	217	subs r7, r7, #1
f71eed58	218	beq 9f
26e182fc MW	219
	220	// Do another middle round's keys...
	221	teq r1, #4
	222	vldmiaeq r4, {d0, d1}
	223	vldmiane r4, {d0-d3}
	224	aesimc.8 q0, q0
	225	vstmiaeq r5, {d0, d1}
f71eed58	226	beq 0b
26e182fc MW	227	aesimc.8 q1, q1
26e182fc MW	228	vstmia r5, {d0-d3}
f71eed58	229	b 0b
26e182fc MW	230
26e182fc MW	231	// Finally do the first encryption round.
f71eed58	232	9: teq r1, #4
26e182fc MW	233	vldmiaeq r4, {d0, d1}
	234	vldmiane r4, {d0-d3}
	235	vstmiaeq r5, {d0, d1}
	236	vstmiane r5, {d0-d3}
	237
	238	// If the block size is not exactly four words then we must end-swap
	239	// everything. We can use fancy NEON toys for this.
f71eed58	240	beq 9f
26e182fc MW	241
	242	// End-swap the encryption keys.
	243	add r1, r0, #w
	244	bl endswap_block
	245
	246	// And the decryption keys
	247	add r1, r0, #wi
	248	bl endswap_block
	249
	250	// All done.
f71eed58	251	9: ldmfd sp!, {r4-r9, pc}
26e182fc MW	252
	253	endswap_block:
	254	// End-swap R2 words starting at R1. R1 is clobbered; R2 is not.
	255	// It's OK to work in 16-byte chunks.
	256	mov r4, r2
	257	0: vldmia r1, {d0, d1}
	258	vrev32.8 q0, q0
	259	vstmia r1!, {d0, d1}
	260	subs r4, r4, #4
	261	bhi 0b
	262	bx r14
	263
	264	ENDFUNC
	265
	266	///--------------------------------------------------------------------------
	267	/// Encrypting and decrypting blocks.
	268
	269	FUNC(rijndael_eblk_arm_crypto)
	270
	271	// Arguments:
	272	// r0 = pointer to context
	273	// r1 = pointer to input block
	274	// r2 = pointer to output block
	275
	276	// Set things up ready.
	277	ldr r3, [r0, #nr]
	278	add r0, r0, #w
	279	vldmia r1, {d0, d1}
	280	vrev32.8 q0, q0
	281
	282	// Dispatch according to the number of rounds.
	283	add r3, r3, r3, lsl #1
	284	rsbs r3, r3, #3*14
	285	addcs pc, pc, r3, lsl #2
	286	callext F(abort)
	287
	288	// The last round doesn't have MixColumns, so do it separately.
9ba8a1d0	289	.rept 13
26e182fc MW	290	vldmia r0!, {d2, d3}
	291	aese.8 q0, q1
	292	aesmc.8 q0, q0
9ba8a1d0	293	.endr
26e182fc MW	294
	295	// Final round.
	296	vldmia r0!, {d2, d3}
	297	aese.8 q0, q1
	298
	299	// Final whitening.
	300	vldmia r0!, {d2, d3}
	301	veor q0, q1
	302
	303	// All done.
	304	vrev32.8 q0, q0
	305	vstmia r2, {d0, d1}
	306	bx r14
	307
	308	ENDFUNC
	309
	310	FUNC(rijndael_dblk_arm_crypto)
	311
	312	// Arguments:
	313	// r0 = pointer to context
	314	// r1 = pointer to input block
	315	// r2 = pointer to output block
	316
	317	// Set things up ready.
	318	ldr r3, [r0, #nr]
	319	add r0, r0, #wi
	320	vldmia r1, {d0, d1}
	321	vrev32.8 q0, q0
	322
	323	// Dispatch according to the number of rounds.
	324	add r3, r3, r3, lsl #1
	325	rsbs r3, r3, #3*14
	326	addcs pc, pc, r3, lsl #2
	327	callext F(abort)
	328
	329	// The last round doesn't have MixColumns, so do it separately.
9ba8a1d0	330	.rept 13
26e182fc MW	331	vldmia r0!, {d2, d3}
	332	aesd.8 q0, q1
	333	aesimc.8 q0, q0
9ba8a1d0	334	.endr
26e182fc MW	335
	336	// Final round.
	337	vldmia r0!, {d2, d3}
	338	aesd.8 q0, q1
	339
	340	// Final whitening.
	341	vldmia r0!, {d2, d3}
	342	veor q0, q1
	343
	344	// All done.
	345	vrev32.8 q0, q0
	346	vstmia r2, {d0, d1}
	347	bx r14
	348
	349	ENDFUNC
	350
	351	///----- That's all, folks --------------------------------------------------