crypto: cavium/nitrox - Added rfc4106(gcm(aes)) cipher support (8a3719a5) · Commits · 戴 / test

drivers/crypto/cavium/nitrox/nitrox_aead.c

+255 −82

Original line number	Original line	Diff line number	Diff line
	@@ -18,26 +18,6 @@

	#define GCM_AES_SALT_SIZE 4		#define GCM_AES_SALT_SIZE 4

	/**
	* struct nitrox_crypt_params - Params to set nitrox crypto request.
	* @cryptlen: Encryption/Decryption data length
	* @authlen: Assoc data length + Cryptlen
	* @srclen: Input buffer length
	* @dstlen: Output buffer length
	* @iv: IV data
	* @ivsize: IV data length
	* @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT)
	*/
	struct nitrox_crypt_params {
	unsigned int cryptlen;
	unsigned int authlen;
	unsigned int srclen;
	unsigned int dstlen;
	u8 *iv;
	int ivsize;
	u8 ctrl_arg;
	};

	union gph_p3 {		union gph_p3 {
	struct {		struct {
	#ifdef __BIG_ENDIAN_BITFIELD		#ifdef __BIG_ENDIAN_BITFIELD
	@@ -94,36 +74,40 @@ static int nitrox_aead_setauthsize(struct crypto_aead *aead,
	return 0;		return 0;
	}		}

	static int alloc_src_sglist(struct aead_request areq, char iv, int ivsize,		static int alloc_src_sglist(struct nitrox_kcrypt_request *nkreq,
			struct scatterlist src, char iv, int ivsize,
	int buflen)		int buflen)
	{		{
	struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq);		int nents = sg_nents_for_len(src, buflen);
	int nents = sg_nents_for_len(areq->src, buflen) + 1;
	int ret;		int ret;

	if (nents < 0)		if (nents < 0)
	return nents;		return nents;

			/* IV entry */
			nents += 1;
	/* Allocate buffer to hold IV and input scatterlist array */		/* Allocate buffer to hold IV and input scatterlist array */
	ret = alloc_src_req_buf(nkreq, nents, ivsize);		ret = alloc_src_req_buf(nkreq, nents, ivsize);
	if (ret)		if (ret)
	return ret;		return ret;

	nitrox_creq_copy_iv(nkreq->src, iv, ivsize);		nitrox_creq_copy_iv(nkreq->src, iv, ivsize);
	nitrox_creq_set_src_sg(nkreq, nents, ivsize, areq->src, buflen);		nitrox_creq_set_src_sg(nkreq, nents, ivsize, src, buflen);

	return 0;		return 0;
	}		}

	static int alloc_dst_sglist(struct aead_request *areq, int ivsize, int buflen)		static int alloc_dst_sglist(struct nitrox_kcrypt_request *nkreq,
			struct scatterlist *dst, int ivsize, int buflen)
	{		{
	struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq);		int nents = sg_nents_for_len(dst, buflen);
	int nents = sg_nents_for_len(areq->dst, buflen) + 3;
	int ret;		int ret;

	if (nents < 0)		if (nents < 0)
	return nents;		return nents;

			/* IV, ORH, COMPLETION entries */
			nents += 3;
	/* Allocate buffer to hold ORH, COMPLETION and output scatterlist		/* Allocate buffer to hold ORH, COMPLETION and output scatterlist
	* array		* array
	*/		*/
	@@ -133,61 +117,54 @@ static int alloc_dst_sglist(struct aead_request *areq, int ivsize, int buflen)

	nitrox_creq_set_orh(nkreq);		nitrox_creq_set_orh(nkreq);
	nitrox_creq_set_comp(nkreq);		nitrox_creq_set_comp(nkreq);
	nitrox_creq_set_dst_sg(nkreq, nents, ivsize, areq->dst, buflen);		nitrox_creq_set_dst_sg(nkreq, nents, ivsize, dst, buflen);

	return 0;		return 0;
	}		}

	static void free_src_sglist(struct aead_request *areq)		static void free_src_sglist(struct nitrox_kcrypt_request *nkreq)
	{		{
	struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq);

	kfree(nkreq->src);		kfree(nkreq->src);
	}		}

	static void free_dst_sglist(struct aead_request *areq)		static void free_dst_sglist(struct nitrox_kcrypt_request *nkreq)
	{		{
	struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq);

	kfree(nkreq->dst);		kfree(nkreq->dst);
	}		}

	static int nitrox_set_creq(struct aead_request *areq,		static int nitrox_set_creq(struct nitrox_aead_rctx *rctx)
	struct nitrox_crypt_params *params)
	{		{
	struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq);		struct se_crypto_request *creq = &rctx->nkreq.creq;
	struct se_crypto_request *creq = &nkreq->creq;
	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
	union gph_p3 param3;		union gph_p3 param3;
	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
	int ret;		int ret;

	creq->flags = areq->base.flags;		creq->flags = rctx->flags;
	creq->gfp = (areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?		creq->gfp = (rctx->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
	GFP_KERNEL : GFP_ATOMIC;		GFP_ATOMIC;

	creq->ctrl.value = 0;		creq->ctrl.value = 0;
	creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC;		creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC;
	creq->ctrl.s.arg = params->ctrl_arg;		creq->ctrl.s.arg = rctx->ctrl_arg;

	creq->gph.param0 = cpu_to_be16(params->cryptlen);		creq->gph.param0 = cpu_to_be16(rctx->cryptlen);
	creq->gph.param1 = cpu_to_be16(params->authlen);		creq->gph.param1 = cpu_to_be16(rctx->cryptlen + rctx->assoclen);
	creq->gph.param2 = cpu_to_be16(params->ivsize + areq->assoclen);		creq->gph.param2 = cpu_to_be16(rctx->ivsize + rctx->assoclen);
	param3.iv_offset = 0;		param3.iv_offset = 0;
	param3.auth_offset = params->ivsize;		param3.auth_offset = rctx->ivsize;
	creq->gph.param3 = cpu_to_be16(param3.param);		creq->gph.param3 = cpu_to_be16(param3.param);

	creq->ctx_handle = nctx->u.ctx_handle;		creq->ctx_handle = rctx->ctx_handle;
	creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context);		creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context);

	ret = alloc_src_sglist(areq, params->iv, params->ivsize,		ret = alloc_src_sglist(&rctx->nkreq, rctx->src, rctx->iv, rctx->ivsize,
	params->srclen);		rctx->srclen);
	if (ret)		if (ret)
	return ret;		return ret;

	ret = alloc_dst_sglist(areq, params->ivsize, params->dstlen);		ret = alloc_dst_sglist(&rctx->nkreq, rctx->dst, rctx->ivsize,
			rctx->dstlen);
	if (ret) {		if (ret) {
	free_src_sglist(areq);		free_src_sglist(&rctx->nkreq);
	return ret;		return ret;
	}		}

	@@ -197,9 +174,10 @@ static int nitrox_set_creq(struct aead_request *areq,
	static void nitrox_aead_callback(void *arg, int err)		static void nitrox_aead_callback(void *arg, int err)
	{		{
	struct aead_request *areq = arg;		struct aead_request *areq = arg;
			struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);

	free_src_sglist(areq);		free_src_sglist(&rctx->nkreq);
	free_dst_sglist(areq);		free_dst_sglist(&rctx->nkreq);
	if (err) {		if (err) {
	pr_err_ratelimited("request failed status 0x%0x\n", err);		pr_err_ratelimited("request failed status 0x%0x\n", err);
	err = -EINVAL;		err = -EINVAL;
	@@ -212,23 +190,25 @@ static int nitrox_aes_gcm_enc(struct aead_request *areq)
	{		{
	struct crypto_aead *aead = crypto_aead_reqtfm(areq);		struct crypto_aead *aead = crypto_aead_reqtfm(areq);
	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);		struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
	struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq);		struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
	struct se_crypto_request *creq = &nkreq->creq;		struct se_crypto_request *creq = &rctx->nkreq.creq;
	struct flexi_crypto_context *fctx = nctx->u.fctx;		struct flexi_crypto_context *fctx = nctx->u.fctx;
	struct nitrox_crypt_params params;
	int ret;		int ret;

	memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);		memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);

	memset(&params, 0, sizeof(params));		rctx->cryptlen = areq->cryptlen;
	params.cryptlen = areq->cryptlen;		rctx->assoclen = areq->assoclen;
	params.authlen = areq->assoclen + params.cryptlen;		rctx->srclen = areq->assoclen + areq->cryptlen;
	params.srclen = params.authlen;		rctx->dstlen = rctx->srclen + aead->authsize;
	params.dstlen = params.srclen + aead->authsize;		rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];
	params.iv = &areq->iv[GCM_AES_SALT_SIZE];		rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
	params.ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;		rctx->flags = areq->base.flags;
	params.ctrl_arg = ENCRYPT;		rctx->ctx_handle = nctx->u.ctx_handle;
	ret = nitrox_set_creq(areq, &params);		rctx->src = areq->src;
			rctx->dst = areq->dst;
			rctx->ctrl_arg = ENCRYPT;
			ret = nitrox_set_creq(rctx);
	if (ret)		if (ret)
	return ret;		return ret;

	@@ -241,23 +221,25 @@ static int nitrox_aes_gcm_dec(struct aead_request *areq)
	{		{
	struct crypto_aead *aead = crypto_aead_reqtfm(areq);		struct crypto_aead *aead = crypto_aead_reqtfm(areq);
	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);		struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
	struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq);		struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
	struct se_crypto_request *creq = &nkreq->creq;		struct se_crypto_request *creq = &rctx->nkreq.creq;
	struct flexi_crypto_context *fctx = nctx->u.fctx;		struct flexi_crypto_context *fctx = nctx->u.fctx;
	struct nitrox_crypt_params params;
	int ret;		int ret;

	memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);		memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);

	memset(&params, 0, sizeof(params));		rctx->cryptlen = areq->cryptlen - aead->authsize;
	params.cryptlen = areq->cryptlen - aead->authsize;		rctx->assoclen = areq->assoclen;
	params.authlen = areq->assoclen + params.cryptlen;		rctx->srclen = areq->cryptlen + areq->assoclen;
	params.srclen = areq->cryptlen + areq->assoclen;		rctx->dstlen = rctx->srclen - aead->authsize;
	params.dstlen = params.srclen - aead->authsize;		rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];
	params.iv = &areq->iv[GCM_AES_SALT_SIZE];		rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
	params.ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;		rctx->flags = areq->base.flags;
	params.ctrl_arg = DECRYPT;		rctx->ctx_handle = nctx->u.ctx_handle;
	ret = nitrox_set_creq(areq, &params);		rctx->src = areq->src;
			rctx->dst = areq->dst;
			rctx->ctrl_arg = DECRYPT;
			ret = nitrox_set_creq(rctx);
	if (ret)		if (ret)
	return ret;		return ret;

	@@ -290,7 +272,7 @@ static int nitrox_aead_init(struct crypto_aead *aead)
	return 0;		return 0;
	}		}

	static int nitrox_aes_gcm_init(struct crypto_aead *aead)		static int nitrox_gcm_common_init(struct crypto_aead *aead)
	{		{
	int ret;		int ret;
	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);		struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
	@@ -308,8 +290,20 @@ static int nitrox_aes_gcm_init(struct crypto_aead *aead)
	flags->w0.auth_input_type = 1;		flags->w0.auth_input_type = 1;
	flags->f = be64_to_cpu(flags->f);		flags->f = be64_to_cpu(flags->f);

	crypto_aead_set_reqsize(aead, sizeof(struct aead_request) +		return 0;
	sizeof(struct nitrox_kcrypt_request));		}

			static int nitrox_aes_gcm_init(struct crypto_aead *aead)
			{
			int ret;

			ret = nitrox_gcm_common_init(aead);
			if (ret)
			return ret;

			crypto_aead_set_reqsize(aead,
			sizeof(struct aead_request) +
			sizeof(struct nitrox_aead_rctx));

	return 0;		return 0;
	}		}
	@@ -332,6 +326,166 @@ static void nitrox_aead_exit(struct crypto_aead *aead)
	nctx->ndev = NULL;		nctx->ndev = NULL;
	}		}

			static int nitrox_rfc4106_setkey(struct crypto_aead aead, const u8 key,
			unsigned int keylen)
			{
			struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
			struct flexi_crypto_context *fctx = nctx->u.fctx;
			int ret;

			if (keylen < GCM_AES_SALT_SIZE)
			return -EINVAL;

			keylen -= GCM_AES_SALT_SIZE;
			ret = nitrox_aes_gcm_setkey(aead, key, keylen);
			if (ret)
			return ret;

			memcpy(fctx->crypto.iv, key + keylen, GCM_AES_SALT_SIZE);
			return 0;
			}

			static int nitrox_rfc4106_setauthsize(struct crypto_aead *aead,
			unsigned int authsize)
			{
			switch (authsize) {
			case 8:
			case 12:
			case 16:
			break;
			default:
			return -EINVAL;
			}

			return nitrox_aead_setauthsize(aead, authsize);
			}

			static int nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request *areq)
			{
			struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
			struct nitrox_aead_rctx *aead_rctx = &rctx->base;
			unsigned int assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
			struct scatterlist *sg;

			if (areq->assoclen != 16 && areq->assoclen != 20)
			return -EINVAL;

			scatterwalk_map_and_copy(rctx->assoc, areq->src, 0, assoclen, 0);
			sg_init_table(rctx->src, 3);
			sg_set_buf(rctx->src, rctx->assoc, assoclen);
			sg = scatterwalk_ffwd(rctx->src + 1, areq->src, areq->assoclen);
			if (sg != rctx->src + 1)
			sg_chain(rctx->src, 2, sg);

			if (areq->src != areq->dst) {
			sg_init_table(rctx->dst, 3);
			sg_set_buf(rctx->dst, rctx->assoc, assoclen);
			sg = scatterwalk_ffwd(rctx->dst + 1, areq->dst, areq->assoclen);
			if (sg != rctx->dst + 1)
			sg_chain(rctx->dst, 2, sg);
			}

			aead_rctx->src = rctx->src;
			aead_rctx->dst = (areq->src == areq->dst) ? rctx->src : rctx->dst;

			return 0;
			}

			static void nitrox_rfc4106_callback(void *arg, int err)
			{
			struct aead_request *areq = arg;
			struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
			struct nitrox_kcrypt_request *nkreq = &rctx->base.nkreq;

			free_src_sglist(nkreq);
			free_dst_sglist(nkreq);
			if (err) {
			pr_err_ratelimited("request failed status 0x%0x\n", err);
			err = -EINVAL;
			}

			areq->base.complete(&areq->base, err);
			}

			static int nitrox_rfc4106_enc(struct aead_request *areq)
			{
			struct crypto_aead *aead = crypto_aead_reqtfm(areq);
			struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
			struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
			struct nitrox_aead_rctx *aead_rctx = &rctx->base;
			struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
			int ret;

			aead_rctx->cryptlen = areq->cryptlen;
			aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
			aead_rctx->srclen = aead_rctx->assoclen + aead_rctx->cryptlen;
			aead_rctx->dstlen = aead_rctx->srclen + aead->authsize;
			aead_rctx->iv = areq->iv;
			aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;
			aead_rctx->flags = areq->base.flags;
			aead_rctx->ctx_handle = nctx->u.ctx_handle;
			aead_rctx->ctrl_arg = ENCRYPT;

			ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);
			if (ret)
			return ret;

			ret = nitrox_set_creq(aead_rctx);
			if (ret)
			return ret;

			/* send the crypto request */
			return nitrox_process_se_request(nctx->ndev, creq,
			nitrox_rfc4106_callback, areq);
			}

			static int nitrox_rfc4106_dec(struct aead_request *areq)
			{
			struct crypto_aead *aead = crypto_aead_reqtfm(areq);
			struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
			struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
			struct nitrox_aead_rctx *aead_rctx = &rctx->base;
			struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
			int ret;

			aead_rctx->cryptlen = areq->cryptlen - aead->authsize;
			aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
			aead_rctx->srclen =
			areq->cryptlen - GCM_RFC4106_IV_SIZE + areq->assoclen;
			aead_rctx->dstlen = aead_rctx->srclen - aead->authsize;
			aead_rctx->iv = areq->iv;
			aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;
			aead_rctx->flags = areq->base.flags;
			aead_rctx->ctx_handle = nctx->u.ctx_handle;
			aead_rctx->ctrl_arg = DECRYPT;

			ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);
			if (ret)
			return ret;

			ret = nitrox_set_creq(aead_rctx);
			if (ret)
			return ret;

			/* send the crypto request */
			return nitrox_process_se_request(nctx->ndev, creq,
			nitrox_rfc4106_callback, areq);
			}

			static int nitrox_rfc4106_init(struct crypto_aead *aead)
			{
			int ret;

			ret = nitrox_gcm_common_init(aead);
			if (ret)
			return ret;

			crypto_aead_set_reqsize(aead, sizeof(struct aead_request) +
			sizeof(struct nitrox_rfc4106_rctx));

			return 0;
			}

	static struct aead_alg nitrox_aeads[] = { {		static struct aead_alg nitrox_aeads[] = { {
	.base = {		.base = {
	.cra_name = "gcm(aes)",		.cra_name = "gcm(aes)",
	@@ -351,6 +505,25 @@ static struct aead_alg nitrox_aeads[] = { {
	.exit = nitrox_aead_exit,		.exit = nitrox_aead_exit,
	.ivsize = GCM_AES_IV_SIZE,		.ivsize = GCM_AES_IV_SIZE,
	.maxauthsize = AES_BLOCK_SIZE,		.maxauthsize = AES_BLOCK_SIZE,
			}, {
			.base = {
			.cra_name = "rfc4106(gcm(aes))",
			.cra_driver_name = "n5_rfc4106",
			.cra_priority = PRIO,
			.cra_flags = CRYPTO_ALG_ASYNC,
			.cra_blocksize = AES_BLOCK_SIZE,
			.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
			.cra_alignmask = 0,
			.cra_module = THIS_MODULE,
			},
			.setkey = nitrox_rfc4106_setkey,
			.setauthsize = nitrox_rfc4106_setauthsize,
			.encrypt = nitrox_rfc4106_enc,
			.decrypt = nitrox_rfc4106_dec,
			.init = nitrox_rfc4106_init,
			.exit = nitrox_aead_exit,
			.ivsize = GCM_RFC4106_IV_SIZE,
			.maxauthsize = AES_BLOCK_SIZE,
	} };		} };

	int nitrox_register_aeads(void)		int nitrox_register_aeads(void)

drivers/crypto/cavium/nitrox/nitrox_req.h

+45 −1

Original line number	Original line	Diff line number	Diff line
	@@ -211,6 +211,50 @@ struct nitrox_kcrypt_request {
	u8 *dst;		u8 *dst;
	};		};

			/**
			* struct nitrox_aead_rctx - AEAD request context
			* @nkreq: Base request context
			* @cryptlen: Encryption/Decryption data length
			* @assoclen: AAD length
			* @srclen: Input buffer length
			* @dstlen: Output buffer length
			* @iv: IV data
			* @ivsize: IV data length
			* @flags: AEAD req flags
			* @ctx_handle: Device context handle
			* @src: Source sglist
			* @dst: Destination sglist
			* @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT)
			*/
			struct nitrox_aead_rctx {
			struct nitrox_kcrypt_request nkreq;
			unsigned int cryptlen;
			unsigned int assoclen;
			unsigned int srclen;
			unsigned int dstlen;
			u8 *iv;
			int ivsize;
			u32 flags;
			u64 ctx_handle;
			struct scatterlist *src;
			struct scatterlist *dst;
			u8 ctrl_arg;
			};

			/**
			* struct nitrox_rfc4106_rctx - rfc4106 cipher request context
			* @base: AEAD request context
			* @src: Source sglist
			* @dst: Destination sglist
			* @assoc: AAD
			*/
			struct nitrox_rfc4106_rctx {
			struct nitrox_aead_rctx base;
			struct scatterlist src[3];
			struct scatterlist dst[3];
			u8 assoc[20];
			};

	/**		/**
	* struct pkt_instr_hdr - Packet Instruction Header		* struct pkt_instr_hdr - Packet Instruction Header
	* @g: Gather used		* @g: Gather used
	@@ -512,7 +556,7 @@ static inline struct scatterlist create_multi_sg(struct scatterlist to_sg,
	struct scatterlist *sg = to_sg;		struct scatterlist *sg = to_sg;
	unsigned int sglen;		unsigned int sglen;

	for (; buflen; buflen -= sglen) {		for (; buflen && from_sg; buflen -= sglen) {
	sglen = from_sg->length;		sglen = from_sg->length;
	if (sglen > buflen)		if (sglen > buflen)
	sglen = buflen;		sglen = buflen;

Admin message