Commit bed593c0 authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Herbert Xu
Browse files

crypto: arm64/aes - add scalar implementation 



This adds a scalar implementation of AES, based on the precomputed tables
that are exposed by the generic AES code. Since rotates are cheap on arm64,
this implementation only uses the 4 core tables (of 1 KB each), and avoids
the prerotated ones, reducing the D-cache footprint by 75%.

On Cortex-A57, this code manages 13.0 cycles per byte, which is ~34% faster
than the generic C code. (Note that this is still >13x slower than the code
that uses the optional ARMv8 Crypto Extensions, which manages <1 cycles per
byte.)

Signed-off-by: default avatarArd Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent 293614ce

arch/arm64/crypto/Kconfig

+4 −0
Original line number Diff line number Diff line
@@ -41,6 +41,10 @@ config CRYPTO_CRC32_ARM64_CE 
	depends on KERNEL_MODE_NEON && CRC32

	select CRYPTO_HASH



config CRYPTO_AES_ARM64

	tristate "AES core cipher using scalar instructions"

	select CRYPTO_AES



config CRYPTO_AES_ARM64_CE

	tristate "AES core cipher using ARMv8 Crypto Extensions"

	depends on ARM64 && KERNEL_MODE_NEON

arch/arm64/crypto/Makefile

+3 −0
Original line number Diff line number Diff line
@@ -44,6 +44,9 @@ sha512-arm64-y := sha512-glue.o sha512-core.o 
obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha20-neon.o

chacha20-neon-y := chacha20-neon-core.o chacha20-neon-glue.o



obj-$(CONFIG_CRYPTO_AES_ARM64) += aes-arm64.o

aes-arm64-y := aes-cipher-core.o aes-cipher-glue.o



AFLAGS_aes-ce.o		:= -DINTERLEAVE=4

AFLAGS_aes-neon.o	:= -DINTERLEAVE=4

arch/arm64/crypto/aes-cipher-core.S

0 → 100644
+127 −0
Original line number Diff line number Diff line 
/*

 * Scalar AES core transform

 *

 * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 as

 * published by the Free Software Foundation.

 */



#include <linux/linkage.h>

#include <asm/assembler.h>



	.text



	rk		.req	x0

	out		.req	x1

	in		.req	x2

	rounds		.req	x3

	tt		.req	x4

	lt		.req	x2



	.macro		__hround, out0, out1, in0, in1, in2, in3, t0, t1, enc

	ldp		\out0, \out1, [rk], #8



	ubfx		w13, \in0, #0, #8

	ubfx		w14, \in1, #8, #8

	ldr		w13, [tt, w13, uxtw #2]

	ldr		w14, [tt, w14, uxtw #2]



	.if		\enc

	ubfx		w17, \in1, #0, #8

	ubfx		w18, \in2, #8, #8

	.else

	ubfx		w17, \in3, #0, #8

	ubfx		w18, \in0, #8, #8

	.endif

	ldr		w17, [tt, w17, uxtw #2]

	ldr		w18, [tt, w18, uxtw #2]



	ubfx		w15, \in2, #16, #8

	ubfx		w16, \in3, #24, #8

	ldr		w15, [tt, w15, uxtw #2]

	ldr		w16, [tt, w16, uxtw #2]



	.if		\enc

	ubfx		\t0, \in3, #16, #8

	ubfx		\t1, \in0, #24, #8

	.else

	ubfx		\t0, \in1, #16, #8

	ubfx		\t1, \in2, #24, #8

	.endif

	ldr		\t0, [tt, \t0, uxtw #2]

	ldr		\t1, [tt, \t1, uxtw #2]



	eor		\out0, \out0, w13

	eor		\out1, \out1, w17

	eor		\out0, \out0, w14, ror #24

	eor		\out1, \out1, w18, ror #24

	eor		\out0, \out0, w15, ror #16

	eor		\out1, \out1, \t0, ror #16

	eor		\out0, \out0, w16, ror #8

	eor		\out1, \out1, \t1, ror #8

	.endm



	.macro		fround, out0, out1, out2, out3, in0, in1, in2, in3

	__hround	\out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1

	__hround	\out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1

	.endm



	.macro		iround, out0, out1, out2, out3, in0, in1, in2, in3

	__hround	\out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0

	__hround	\out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0

	.endm



	.macro		do_crypt, round, ttab, ltab

	ldp		w5, w6, [in]

	ldp		w7, w8, [in, #8]

	ldp		w9, w10, [rk], #16

	ldp		w11, w12, [rk, #-8]



CPU_BE(	rev		w5, w5		)

CPU_BE(	rev		w6, w6		)

CPU_BE(	rev		w7, w7		)

CPU_BE(	rev		w8, w8		)



	eor		w5, w5, w9

	eor		w6, w6, w10

	eor		w7, w7, w11

	eor		w8, w8, w12



	ldr		tt, =\ttab

	ldr		lt, =\ltab



	tbnz		rounds, #1, 1f



0:	\round		w9, w10, w11, w12, w5, w6, w7, w8

	\round		w5, w6, w7, w8, w9, w10, w11, w12



1:	subs		rounds, rounds, #4

	\round		w9, w10, w11, w12, w5, w6, w7, w8

	csel		tt, tt, lt, hi

	\round		w5, w6, w7, w8, w9, w10, w11, w12

	b.hi		0b



CPU_BE(	rev		w5, w5		)

CPU_BE(	rev		w6, w6		)

CPU_BE(	rev		w7, w7		)

CPU_BE(	rev		w8, w8		)



	stp		w5, w6, [out]

	stp		w7, w8, [out, #8]

	ret



	.align		4

	.ltorg

	.endm



	.align		5

ENTRY(__aes_arm64_encrypt)

	do_crypt	fround, crypto_ft_tab, crypto_fl_tab

ENDPROC(__aes_arm64_encrypt)



	.align		5

ENTRY(__aes_arm64_decrypt)

	do_crypt	iround, crypto_it_tab, crypto_il_tab

ENDPROC(__aes_arm64_decrypt)

arch/arm64/crypto/aes-cipher-glue.c

0 → 100644
+69 −0
Original line number Diff line number Diff line 
/*

 * Scalar AES core transform

 *

 * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 as

 * published by the Free Software Foundation.

 */



#include <crypto/aes.h>

#include <linux/crypto.h>

#include <linux/module.h>



asmlinkage void __aes_arm64_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds);

EXPORT_SYMBOL(__aes_arm64_encrypt);



asmlinkage void __aes_arm64_decrypt(u32 *rk, u8 *out, const u8 *in, int rounds);

EXPORT_SYMBOL(__aes_arm64_decrypt);



static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)

{

	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);

	int rounds = 6 + ctx->key_length / 4;



	__aes_arm64_encrypt(ctx->key_enc, out, in, rounds);

}



static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)

{

	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);

	int rounds = 6 + ctx->key_length / 4;



	__aes_arm64_decrypt(ctx->key_dec, out, in, rounds);

}



static struct crypto_alg aes_alg = {

	.cra_name			= "aes",

	.cra_driver_name		= "aes-arm64",

	.cra_priority			= 200,

	.cra_flags			= CRYPTO_ALG_TYPE_CIPHER,

	.cra_blocksize			= AES_BLOCK_SIZE,

	.cra_ctxsize			= sizeof(struct crypto_aes_ctx),

	.cra_module			= THIS_MODULE,



	.cra_cipher.cia_min_keysize	= AES_MIN_KEY_SIZE,

	.cra_cipher.cia_max_keysize	= AES_MAX_KEY_SIZE,

	.cra_cipher.cia_setkey		= crypto_aes_set_key,

	.cra_cipher.cia_encrypt		= aes_encrypt,

	.cra_cipher.cia_decrypt		= aes_decrypt

};



static int __init aes_init(void)

{

	return crypto_register_alg(&aes_alg);

}



static void __exit aes_fini(void)

{

	crypto_unregister_alg(&aes_alg);

}



module_init(aes_init);

module_exit(aes_fini);



MODULE_DESCRIPTION("Scalar AES cipher for arm64");

MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");

MODULE_LICENSE("GPL v2");

MODULE_ALIAS_CRYPTO("aes");

CRA Git | Maintained and supported by SUSTech CRA and CCSE