Commit 735177ca authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Herbert Xu
Browse files

crypto: arm64/aes - implement accelerated ESSIV/CBC mode 



Add an accelerated version of the 'essiv(cbc(aes),sha256)' skcipher,
which is used by fscrypt or dm-crypt on systems where CBC mode is
signficantly more performant than XTS mode (e.g., when using a h/w
accelerator which supports the former but not the latter) This avoids
a separate call into the AES cipher for every invocation.

Signed-off-by: default avatarArd Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent 65d0042b

arch/arm64/crypto/aes-glue.c

+124 −0
Original line number Diff line number Diff line
@@ -10,6 +10,7 @@ 
#include <asm/simd.h>

#include <crypto/aes.h>

#include <crypto/ctr.h>

#include <crypto/sha.h>

#include <crypto/internal/hash.h>

#include <crypto/internal/simd.h>

#include <crypto/internal/skcipher.h>
@@ -30,6 +31,8 @@ 
#define aes_cbc_decrypt		ce_aes_cbc_decrypt

#define aes_cbc_cts_encrypt	ce_aes_cbc_cts_encrypt

#define aes_cbc_cts_decrypt	ce_aes_cbc_cts_decrypt

#define aes_essiv_cbc_encrypt	ce_aes_essiv_cbc_encrypt

#define aes_essiv_cbc_decrypt	ce_aes_essiv_cbc_decrypt

#define aes_ctr_encrypt		ce_aes_ctr_encrypt

#define aes_xts_encrypt		ce_aes_xts_encrypt

#define aes_xts_decrypt		ce_aes_xts_decrypt
@@ -44,6 +47,8 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); 
#define aes_cbc_decrypt		neon_aes_cbc_decrypt

#define aes_cbc_cts_encrypt	neon_aes_cbc_cts_encrypt

#define aes_cbc_cts_decrypt	neon_aes_cbc_cts_decrypt

#define aes_essiv_cbc_encrypt	neon_aes_essiv_cbc_encrypt

#define aes_essiv_cbc_decrypt	neon_aes_essiv_cbc_decrypt

#define aes_ctr_encrypt		neon_aes_ctr_encrypt

#define aes_xts_encrypt		neon_aes_xts_encrypt

#define aes_xts_decrypt		neon_aes_xts_decrypt
@@ -51,6 +56,7 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); 
MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON");

MODULE_ALIAS_CRYPTO("ecb(aes)");

MODULE_ALIAS_CRYPTO("cbc(aes)");

MODULE_ALIAS_CRYPTO("essiv(cbc(aes),sha256)");

MODULE_ALIAS_CRYPTO("ctr(aes)");

MODULE_ALIAS_CRYPTO("xts(aes)");

MODULE_ALIAS_CRYPTO("cmac(aes)");
@@ -87,6 +93,13 @@ asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[], 
				int rounds, int blocks, u32 const rk2[], u8 iv[],

				int first);



asmlinkage void aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[],

				      int rounds, int blocks, u8 iv[],

				      u32 const rk2[]);

asmlinkage void aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[],

				      int rounds, int blocks, u8 iv[],

				      u32 const rk2[]);



asmlinkage void aes_mac_update(u8 const in[], u32 const rk[], int rounds,

			       int blocks, u8 dg[], int enc_before,

			       int enc_after);
@@ -102,6 +115,12 @@ struct crypto_aes_xts_ctx { 
	struct crypto_aes_ctx __aligned(8) key2;

};



struct crypto_aes_essiv_cbc_ctx {

	struct crypto_aes_ctx key1;

	struct crypto_aes_ctx __aligned(8) key2;

	struct crypto_shash *hash;

};



struct mac_tfm_ctx {

	struct crypto_aes_ctx key;

	u8 __aligned(8) consts[];
@@ -146,6 +165,31 @@ static int xts_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 
	return -EINVAL;

}



static int essiv_cbc_set_key(struct crypto_skcipher *tfm, const u8 *in_key,

			     unsigned int key_len)

{

	struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);

	SHASH_DESC_ON_STACK(desc, ctx->hash);

	u8 digest[SHA256_DIGEST_SIZE];

	int ret;



	ret = aes_expandkey(&ctx->key1, in_key, key_len);

	if (ret)

		goto out;



	desc->tfm = ctx->hash;

	crypto_shash_digest(desc, in_key, key_len, digest);



	ret = aes_expandkey(&ctx->key2, digest, sizeof(digest));

	if (ret)

		goto out;



	return 0;

out:

	crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);

	return -EINVAL;

}



static int ecb_encrypt(struct skcipher_request *req)

{

	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
@@ -360,6 +404,68 @@ static int cts_cbc_decrypt(struct skcipher_request *req) 
	return skcipher_walk_done(&walk, 0);

}



static int essiv_cbc_init_tfm(struct crypto_skcipher *tfm)

{

	struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);



	ctx->hash = crypto_alloc_shash("sha256", 0, 0);

	if (IS_ERR(ctx->hash))

		return PTR_ERR(ctx->hash);



	return 0;

}



static void essiv_cbc_exit_tfm(struct crypto_skcipher *tfm)

{

	struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);



	crypto_free_shash(ctx->hash);

}



static int essiv_cbc_encrypt(struct skcipher_request *req)

{

	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);

	struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);

	int err, rounds = 6 + ctx->key1.key_length / 4;

	struct skcipher_walk walk;

	unsigned int blocks;



	err = skcipher_walk_virt(&walk, req, false);



	blocks = walk.nbytes / AES_BLOCK_SIZE;

	if (blocks) {

		kernel_neon_begin();

		aes_essiv_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,

				      ctx->key1.key_enc, rounds, blocks,

				      req->iv, ctx->key2.key_enc);

		kernel_neon_end();

		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);

	}

	return err ?: cbc_encrypt_walk(req, &walk);

}



static int essiv_cbc_decrypt(struct skcipher_request *req)

{

	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);

	struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);

	int err, rounds = 6 + ctx->key1.key_length / 4;

	struct skcipher_walk walk;

	unsigned int blocks;



	err = skcipher_walk_virt(&walk, req, false);



	blocks = walk.nbytes / AES_BLOCK_SIZE;

	if (blocks) {

		kernel_neon_begin();

		aes_essiv_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,

				      ctx->key1.key_dec, rounds, blocks,

				      req->iv, ctx->key2.key_enc);

		kernel_neon_end();

		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);

	}

	return err ?: cbc_decrypt_walk(req, &walk);

}



static int ctr_encrypt(struct skcipher_request *req)

{

	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
@@ -515,6 +621,24 @@ static struct skcipher_alg aes_algs[] = { { 
	.encrypt	= cts_cbc_encrypt,

	.decrypt	= cts_cbc_decrypt,

	.init		= cts_cbc_init_tfm,

}, {

	.base = {

		.cra_name		= "__essiv(cbc(aes),sha256)",

		.cra_driver_name	= "__essiv-cbc-aes-sha256-" MODE,

		.cra_priority		= PRIO + 1,

		.cra_flags		= CRYPTO_ALG_INTERNAL,

		.cra_blocksize		= AES_BLOCK_SIZE,

		.cra_ctxsize		= sizeof(struct crypto_aes_essiv_cbc_ctx),

		.cra_module		= THIS_MODULE,

	},

	.min_keysize	= AES_MIN_KEY_SIZE,

	.max_keysize	= AES_MAX_KEY_SIZE,

	.ivsize		= AES_BLOCK_SIZE,

	.setkey		= essiv_cbc_set_key,

	.encrypt	= essiv_cbc_encrypt,

	.decrypt	= essiv_cbc_decrypt,

	.init		= essiv_cbc_init_tfm,

	.exit		= essiv_cbc_exit_tfm,

}, {

	.base = {

		.cra_name		= "__ctr(aes)",

arch/arm64/crypto/aes-modes.S

+28 −0
Original line number Diff line number Diff line
@@ -118,8 +118,23 @@ AES_ENDPROC(aes_ecb_decrypt) 
	 *		   int blocks, u8 iv[])

	 * aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,

	 *		   int blocks, u8 iv[])

	 * aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[],

	 *			 int rounds, int blocks, u8 iv[],

	 *			 u32 const rk2[]);

	 * aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[],

	 *			 int rounds, int blocks, u8 iv[],

	 *			 u32 const rk2[]);

	 */



AES_ENTRY(aes_essiv_cbc_encrypt)

	ld1		{v4.16b}, [x5]			/* get iv */



	mov		w8, #14				/* AES-256: 14 rounds */

	enc_prepare	w8, x6, x7

	encrypt_block	v4, w8, x6, x7, w9

	enc_switch_key	w3, x2, x6

	b		.Lcbcencloop4x



AES_ENTRY(aes_cbc_encrypt)

	ld1		{v4.16b}, [x5]			/* get iv */

	enc_prepare	w3, x2, x6
@@ -153,13 +168,25 @@ AES_ENTRY(aes_cbc_encrypt) 
	st1		{v4.16b}, [x5]			/* return iv */

	ret

AES_ENDPROC(aes_cbc_encrypt)

AES_ENDPROC(aes_essiv_cbc_encrypt)



AES_ENTRY(aes_essiv_cbc_decrypt)

	stp		x29, x30, [sp, #-16]!

	mov		x29, sp



	ld1		{cbciv.16b}, [x5]		/* get iv */



	mov		w8, #14				/* AES-256: 14 rounds */

	enc_prepare	w8, x6, x7

	encrypt_block	cbciv, w8, x6, x7, w9

	b		.Lessivcbcdecstart



AES_ENTRY(aes_cbc_decrypt)

	stp		x29, x30, [sp, #-16]!

	mov		x29, sp



	ld1		{cbciv.16b}, [x5]		/* get iv */

.Lessivcbcdecstart:

	dec_prepare	w3, x2, x6



.LcbcdecloopNx:
@@ -212,6 +239,7 @@ ST5( st1 {v4.16b}, [x0], #16 ) 
	ldp		x29, x30, [sp], #16

	ret

AES_ENDPROC(aes_cbc_decrypt)

AES_ENDPROC(aes_essiv_cbc_decrypt)





	/*

CRA Git | Maintained and supported by SUSTech CRA and CCSE