Commit 5d3d9c8b authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Herbert Xu
Browse files

crypto: arm64/crc32 - merge CRC32 and PMULL instruction based drivers 



The PMULL based CRC32 implementation already contains code based on the
separate, optional CRC32 instructions to fallback to when operating on
small quantities of data. We can expose these routines directly on systems
that lack the 64x64 PMULL instructions but do implement the CRC32 ones,
which makes the driver that is based solely on those CRC32 instructions
redundant. So remove it.

Note that this aligns arm64 with ARM, whose accelerated CRC32 driver
also combines the CRC32 extension based and the PMULL based versions.

Signed-off-by: default avatarArd Biesheuvel <ard.biesheuvel@linaro.org>
Tested-by: default avatarMatthias Brugger <mbrugger@suse.com>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent 1a20b966

arch/arm64/configs/defconfig

+0 −1
Original line number Diff line number Diff line
@@ -516,4 +516,3 @@ CONFIG_CRYPTO_GHASH_ARM64_CE=y 
CONFIG_CRYPTO_AES_ARM64_CE_CCM=y

CONFIG_CRYPTO_AES_ARM64_CE_BLK=y

# CONFIG_CRYPTO_AES_ARM64_NEON_BLK is not set
 
CONFIG_CRYPTO_CRC32_ARM64=y

arch/arm64/crypto/Kconfig

+2 −7
Original line number Diff line number Diff line
@@ -37,8 +37,8 @@ config CRYPTO_CRCT10DIF_ARM64_CE 
	select CRYPTO_HASH



config CRYPTO_CRC32_ARM64_CE

	tristate "CRC32 and CRC32C digest algorithms using PMULL instructions"

	depends on KERNEL_MODE_NEON && CRC32

	tristate "CRC32 and CRC32C digest algorithms using ARMv8 extensions"

	depends on CRC32

	select CRYPTO_HASH



config CRYPTO_AES_ARM64
@@ -71,11 +71,6 @@ config CRYPTO_AES_ARM64_NEON_BLK 
	select CRYPTO_AES

	select CRYPTO_SIMD



config CRYPTO_CRC32_ARM64

	tristate "CRC32 and CRC32C using optional ARMv8 instructions"

	depends on ARM64

	select CRYPTO_HASH



config CRYPTO_CHACHA20_NEON

	tristate "NEON accelerated ChaCha20 symmetric cipher"

	depends on KERNEL_MODE_NEON

arch/arm64/crypto/Makefile

+0 −4
Original line number Diff line number Diff line
@@ -55,10 +55,6 @@ AFLAGS_aes-neon.o := -DINTERLEAVE=4 


CFLAGS_aes-glue-ce.o	:= -DUSE_V8_CRYPTO_EXTENSIONS



obj-$(CONFIG_CRYPTO_CRC32_ARM64) += crc32-arm64.o



CFLAGS_crc32-arm64.o	:= -mcpu=generic+crc



$(obj)/aes-glue-%.o: $(src)/aes-glue.c FORCE

	$(call if_changed_rule,cc_o_c)

arch/arm64/crypto/crc32-arm64.c

deleted100644 → 0
+0 −290
Original line number Diff line number Diff line 
/*

 * crc32-arm64.c - CRC32 and CRC32C using optional ARMv8 instructions

 *

 * Module based on crypto/crc32c_generic.c

 *

 * CRC32 loop taken from Ed Nevill's Hadoop CRC patch

 * http://mail-archives.apache.org/mod_mbox/hadoop-common-dev/201406.mbox/%3C1403687030.3355.19.camel%40localhost.localdomain%3E

 *

 * Using inline assembly instead of intrinsics in order to be backwards

 * compatible with older compilers.

 *

 * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 as

 * published by the Free Software Foundation.

 */



#include <linux/unaligned/access_ok.h>

#include <linux/cpufeature.h>

#include <linux/init.h>

#include <linux/kernel.h>

#include <linux/module.h>

#include <linux/string.h>



#include <crypto/internal/hash.h>



MODULE_AUTHOR("Yazen Ghannam <yazen.ghannam@linaro.org>");

MODULE_DESCRIPTION("CRC32 and CRC32C using optional ARMv8 instructions");

MODULE_LICENSE("GPL v2");



#define CRC32X(crc, value) __asm__("crc32x %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(value))

#define CRC32W(crc, value) __asm__("crc32w %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))

#define CRC32H(crc, value) __asm__("crc32h %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))

#define CRC32B(crc, value) __asm__("crc32b %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))

#define CRC32CX(crc, value) __asm__("crc32cx %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(value))

#define CRC32CW(crc, value) __asm__("crc32cw %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))

#define CRC32CH(crc, value) __asm__("crc32ch %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))

#define CRC32CB(crc, value) __asm__("crc32cb %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))



static u32 crc32_arm64_le_hw(u32 crc, const u8 *p, unsigned int len)

{

	s64 length = len;



	while ((length -= sizeof(u64)) >= 0) {

		CRC32X(crc, get_unaligned_le64(p));

		p += sizeof(u64);

	}



	/* The following is more efficient than the straight loop */

	if (length & sizeof(u32)) {

		CRC32W(crc, get_unaligned_le32(p));

		p += sizeof(u32);

	}

	if (length & sizeof(u16)) {

		CRC32H(crc, get_unaligned_le16(p));

		p += sizeof(u16);

	}

	if (length & sizeof(u8))

		CRC32B(crc, *p);



	return crc;

}



static u32 crc32c_arm64_le_hw(u32 crc, const u8 *p, unsigned int len)

{

	s64 length = len;



	while ((length -= sizeof(u64)) >= 0) {

		CRC32CX(crc, get_unaligned_le64(p));

		p += sizeof(u64);

	}



	/* The following is more efficient than the straight loop */

	if (length & sizeof(u32)) {

		CRC32CW(crc, get_unaligned_le32(p));

		p += sizeof(u32);

	}

	if (length & sizeof(u16)) {

		CRC32CH(crc, get_unaligned_le16(p));

		p += sizeof(u16);

	}

	if (length & sizeof(u8))

		CRC32CB(crc, *p);



	return crc;

}



#define CHKSUM_BLOCK_SIZE	1

#define CHKSUM_DIGEST_SIZE	4



struct chksum_ctx {

	u32 key;

};



struct chksum_desc_ctx {

	u32 crc;

};



static int chksum_init(struct shash_desc *desc)

{

	struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);

	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);



	ctx->crc = mctx->key;



	return 0;

}



/*

 * Setting the seed allows arbitrary accumulators and flexible XOR policy

 * If your algorithm starts with ~0, then XOR with ~0 before you set

 * the seed.

 */

static int chksum_setkey(struct crypto_shash *tfm, const u8 *key,

			 unsigned int keylen)

{

	struct chksum_ctx *mctx = crypto_shash_ctx(tfm);



	if (keylen != sizeof(mctx->key)) {

		crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);

		return -EINVAL;

	}

	mctx->key = get_unaligned_le32(key);

	return 0;

}



static int chksum_update(struct shash_desc *desc, const u8 *data,

			 unsigned int length)

{

	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);



	ctx->crc = crc32_arm64_le_hw(ctx->crc, data, length);

	return 0;

}



static int chksumc_update(struct shash_desc *desc, const u8 *data,

			 unsigned int length)

{

	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);



	ctx->crc = crc32c_arm64_le_hw(ctx->crc, data, length);

	return 0;

}



static int chksum_final(struct shash_desc *desc, u8 *out)

{

	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);



	put_unaligned_le32(ctx->crc, out);

	return 0;

}



static int chksumc_final(struct shash_desc *desc, u8 *out)

{

	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);



	put_unaligned_le32(~ctx->crc, out);

	return 0;

}



static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)

{

	put_unaligned_le32(crc32_arm64_le_hw(crc, data, len), out);

	return 0;

}



static int __chksumc_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)

{

	put_unaligned_le32(~crc32c_arm64_le_hw(crc, data, len), out);

	return 0;

}



static int chksum_finup(struct shash_desc *desc, const u8 *data,

			unsigned int len, u8 *out)

{

	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);



	return __chksum_finup(ctx->crc, data, len, out);

}



static int chksumc_finup(struct shash_desc *desc, const u8 *data,

			unsigned int len, u8 *out)

{

	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);



	return __chksumc_finup(ctx->crc, data, len, out);

}



static int chksum_digest(struct shash_desc *desc, const u8 *data,

			 unsigned int length, u8 *out)

{

	struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);



	return __chksum_finup(mctx->key, data, length, out);

}



static int chksumc_digest(struct shash_desc *desc, const u8 *data,

			 unsigned int length, u8 *out)

{

	struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);



	return __chksumc_finup(mctx->key, data, length, out);

}



static int crc32_cra_init(struct crypto_tfm *tfm)

{

	struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);



	mctx->key = 0;

	return 0;

}



static int crc32c_cra_init(struct crypto_tfm *tfm)

{

	struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);



	mctx->key = ~0;

	return 0;

}



static struct shash_alg crc32_alg = {

	.digestsize		=	CHKSUM_DIGEST_SIZE,

	.setkey			=	chksum_setkey,

	.init			=	chksum_init,

	.update			=	chksum_update,

	.final			=	chksum_final,

	.finup			=	chksum_finup,

	.digest			=	chksum_digest,

	.descsize		=	sizeof(struct chksum_desc_ctx),

	.base			=	{

		.cra_name		=	"crc32",

		.cra_driver_name	=	"crc32-arm64-hw",

		.cra_priority		=	300,

		.cra_blocksize		=	CHKSUM_BLOCK_SIZE,

		.cra_alignmask		=	0,

		.cra_ctxsize		=	sizeof(struct chksum_ctx),

		.cra_module		=	THIS_MODULE,

		.cra_init		=	crc32_cra_init,

	}

};



static struct shash_alg crc32c_alg = {

	.digestsize		=	CHKSUM_DIGEST_SIZE,

	.setkey			=	chksum_setkey,

	.init			=	chksum_init,

	.update			=	chksumc_update,

	.final			=	chksumc_final,

	.finup			=	chksumc_finup,

	.digest			=	chksumc_digest,

	.descsize		=	sizeof(struct chksum_desc_ctx),

	.base			=	{

		.cra_name		=	"crc32c",

		.cra_driver_name	=	"crc32c-arm64-hw",

		.cra_priority		=	300,

		.cra_blocksize		=	CHKSUM_BLOCK_SIZE,

		.cra_alignmask		=	0,

		.cra_ctxsize		=	sizeof(struct chksum_ctx),

		.cra_module		=	THIS_MODULE,

		.cra_init		=	crc32c_cra_init,

	}

};



static int __init crc32_mod_init(void)

{

	int err;



	err = crypto_register_shash(&crc32_alg);



	if (err)

		return err;



	err = crypto_register_shash(&crc32c_alg);



	if (err) {

		crypto_unregister_shash(&crc32_alg);

		return err;

	}



	return 0;

}



static void __exit crc32_mod_exit(void)

{

	crypto_unregister_shash(&crc32_alg);

	crypto_unregister_shash(&crc32c_alg);

}



module_cpu_feature_match(CRC32, crc32_mod_init);

module_exit(crc32_mod_exit);

arch/arm64/crypto/crc32-ce-glue.c

+39 −10
Original line number Diff line number Diff line
@@ -72,6 +72,24 @@ static int crc32_pmull_init(struct shash_desc *desc) 
	return 0;

}



static int crc32_update(struct shash_desc *desc, const u8 *data,

			unsigned int length)

{

	u32 *crc = shash_desc_ctx(desc);



	*crc = crc32_armv8_le(*crc, data, length);

	return 0;

}



static int crc32c_update(struct shash_desc *desc, const u8 *data,

			 unsigned int length)

{

	u32 *crc = shash_desc_ctx(desc);



	*crc = crc32c_armv8_le(*crc, data, length);

	return 0;

}



static int crc32_pmull_update(struct shash_desc *desc, const u8 *data,

			 unsigned int length)

{
@@ -156,7 +174,7 @@ static int crc32c_pmull_final(struct shash_desc *desc, u8 *out) 
static struct shash_alg crc32_pmull_algs[] = { {

	.setkey			= crc32_pmull_setkey,

	.init			= crc32_pmull_init,

	.update			= crc32_pmull_update,

	.update			= crc32_update,

	.final			= crc32_pmull_final,

	.descsize		= sizeof(u32),

	.digestsize		= sizeof(u32),
@@ -171,7 +189,7 @@ static struct shash_alg crc32_pmull_algs[] = { { 
}, {

	.setkey			= crc32_pmull_setkey,

	.init			= crc32_pmull_init,

	.update			= crc32c_pmull_update,

	.update			= crc32c_update,

	.final			= crc32c_pmull_final,

	.descsize		= sizeof(u32),

	.digestsize		= sizeof(u32),
@@ -187,6 +205,10 @@ static struct shash_alg crc32_pmull_algs[] = { { 


static int __init crc32_pmull_mod_init(void)

{

	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_PMULL)) {

		crc32_pmull_algs[0].update = crc32_pmull_update;

		crc32_pmull_algs[1].update = crc32c_pmull_update;



		if (elf_hwcap & HWCAP_CRC32) {

			fallback_crc32 = crc32_armv8_le;

			fallback_crc32c = crc32c_armv8_le;
@@ -194,7 +216,9 @@ static int __init crc32_pmull_mod_init(void) 
			fallback_crc32 = crc32_le;

			fallback_crc32c = __crc32c_le;

		}



	} else if (!(elf_hwcap & HWCAP_CRC32)) {

		return -ENODEV;

	}

	return crypto_register_shashes(crc32_pmull_algs,

				       ARRAY_SIZE(crc32_pmull_algs));

}
@@ -205,7 +229,12 @@ static void __exit crc32_pmull_mod_exit(void) 
				  ARRAY_SIZE(crc32_pmull_algs));

}



module_cpu_feature_match(PMULL, crc32_pmull_mod_init);

static const struct cpu_feature crc32_cpu_feature[] = {

	{ cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { }

};

MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature);



module_init(crc32_pmull_mod_init);

module_exit(crc32_pmull_mod_exit);



MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");

CRA Git | Maintained and supported by SUSTech CRA and CCSE