crypto: nx - add hardware 842 crypto comp alg

	  algorithm.  This supports NX hardware on the PowerNV platform.

	  If you choose 'M' here, this module will be called nx_compress_powernv.

config CRYPTO_DEV_NX_COMPRESS_CRYPTO

	tristate "Compression acceleration cryptographic interface"

	select CRYPTO_ALGAPI

	select 842_DECOMPRESS

	default y

	help

	  Support for PowerPC Nest (NX) accelerators using the cryptographic

	  API.  If you choose 'M' here, this module will be called

	  nx_compress_crypto.

endif

obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS) += nx-compress.o

obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_PSERIES) += nx-compress-pseries.o

obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_POWERNV) += nx-compress-powernv.o

obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_CRYPTO) += nx-compress-crypto.o

nx-compress-objs := nx-842.o

nx-compress-pseries-objs := nx-842-pseries.o

nx-compress-powernv-objs := nx-842-powernv.o

nx-compress-crypto-objs := nx-842-crypto.o

/*

 * Cryptographic API for the NX-842 hardware compression.

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * Copyright (C) IBM Corporation, 2011-2015

 *

 * Original Authors: Robert Jennings <rcj@linux.vnet.ibm.com>

 *                   Seth Jennings <sjenning@linux.vnet.ibm.com>

 *

 * Rewrite: Dan Streetman <ddstreet@ieee.org>

 *

 * This is an interface to the NX-842 compression hardware in PowerPC

 * processors.  Most of the complexity of this drvier is due to the fact that

 * the NX-842 compression hardware requires the input and output data buffers

 * to be specifically aligned, to be a specific multiple in length, and within

 * specific minimum and maximum lengths.  Those restrictions, provided by the

 * nx-842 driver via nx842_constraints, mean this driver must use bounce

 * buffers and headers to correct misaligned in or out buffers, and to split

 * input buffers that are too large.

 *

 * This driver will fall back to software decompression if the hardware

 * decompression fails, so this driver's decompression should never fail as

 * long as the provided compressed buffer is valid.  Any compressed buffer

 * created by this driver will have a header (except ones where the input

 * perfectly matches the constraints); so users of this driver cannot simply

 * pass a compressed buffer created by this driver over to the 842 software

 * decompression library.  Instead, users must use this driver to decompress;

 * if the hardware fails or is unavailable, the compressed buffer will be

 * parsed and the header removed, and the raw 842 buffer(s) passed to the 842

 * software decompression library.

 *

 * This does not fall back to software compression, however, since the caller

 * of this function is specifically requesting hardware compression; if the

 * hardware compression fails, the caller can fall back to software

 * compression, and the raw 842 compressed buffer that the software compressor

 * creates can be passed to this driver for hardware decompression; any

 * buffer without our specific header magic is assumed to be a raw 842 buffer

 * and passed directly to the hardware.  Note that the software compression

 * library will produce a compressed buffer that is incompatible with the

 * hardware decompressor if the original input buffer length is not a multiple

 * of 8; if such a compressed buffer is passed to this driver for

 * decompression, the hardware will reject it and this driver will then pass

 * it over to the software library for decompression.

 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/init.h>

#include <linux/module.h>

#include <linux/crypto.h>

#include <linux/vmalloc.h>

#include <linux/nx842.h>

#include <linux/sw842.h>

#include <linux/ratelimit.h>

/* The first 5 bits of this magic are 0x1f, which is an invalid 842 5-bit

 * template (see lib/842/842.h), so this magic number will never appear at

 * the start of a raw 842 compressed buffer.  That is important, as any buffer

 * passed to us without this magic is assumed to be a raw 842 compressed

 * buffer, and passed directly to the hardware to decompress.

 */

#define NX842_CRYPTO_MAGIC	(0xf842)

#define NX842_CRYPTO_GROUP_MAX	(0x20)

#define NX842_CRYPTO_HEADER_SIZE(g)				\

	(sizeof(struct nx842_crypto_header) +			\

	 sizeof(struct nx842_crypto_header_group) * (g))

#define NX842_CRYPTO_HEADER_MAX_SIZE				\

	NX842_CRYPTO_HEADER_SIZE(NX842_CRYPTO_GROUP_MAX)

/* bounce buffer size */

#define BOUNCE_BUFFER_ORDER	(2)

#define BOUNCE_BUFFER_SIZE					\

	((unsigned int)(PAGE_SIZE << BOUNCE_BUFFER_ORDER))

/* try longer on comp because we can fallback to sw decomp if hw is busy */

#define COMP_BUSY_TIMEOUT	(250) /* ms */

#define DECOMP_BUSY_TIMEOUT	(50) /* ms */

struct nx842_crypto_header_group {

	__be16 padding;			/* unused bytes at start of group */

	__be32 compressed_length;	/* compressed bytes in group */

	__be32 uncompressed_length;	/* bytes after decompression */

} __packed;

struct nx842_crypto_header {

	__be16 magic;		/* NX842_CRYPTO_MAGIC */

	__be16 ignore;		/* decompressed end bytes to ignore */

	u8 groups;		/* total groups in this header */

	struct nx842_crypto_header_group group[];

} __packed;

struct nx842_crypto_param {

	u8 *in;

	unsigned int iremain;

	u8 *out;

	unsigned int oremain;

	unsigned int ototal;

};

static int update_param(struct nx842_crypto_param *p,

			unsigned int slen, unsigned int dlen)

{

	if (p->iremain < slen)

		return -EOVERFLOW;

	if (p->oremain < dlen)

		return -ENOSPC;

	p->in += slen;

	p->iremain -= slen;

	p->out += dlen;

	p->oremain -= dlen;

	p->ototal += dlen;

	return 0;

}

struct nx842_crypto_ctx {

	u8 *wmem;

	u8 *sbounce, *dbounce;

	struct nx842_crypto_header header;

	struct nx842_crypto_header_group group[NX842_CRYPTO_GROUP_MAX];

};

static int nx842_crypto_init(struct crypto_tfm *tfm)

{

	struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);

	ctx->wmem = kmalloc(NX842_MEM_COMPRESS, GFP_KERNEL);

	ctx->sbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER);

	ctx->dbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER);

	if (!ctx->wmem || !ctx->sbounce || !ctx->dbounce) {

		kfree(ctx->wmem);

		free_page((unsigned long)ctx->sbounce);

		free_page((unsigned long)ctx->dbounce);

		return -ENOMEM;

	}

	return 0;

}

static void nx842_crypto_exit(struct crypto_tfm *tfm)

{

	struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);

	kfree(ctx->wmem);

	free_page((unsigned long)ctx->sbounce);

	free_page((unsigned long)ctx->dbounce);

}

static int read_constraints(struct nx842_constraints *c)

{

	int ret;

	ret = nx842_constraints(c);

	if (ret) {

		pr_err_ratelimited("could not get nx842 constraints : %d\n",

				   ret);

		return ret;

	}

	/* limit maximum, to always have enough bounce buffer to decompress */

	if (c->maximum > BOUNCE_BUFFER_SIZE) {

		c->maximum = BOUNCE_BUFFER_SIZE;

		pr_info_once("limiting nx842 maximum to %x\n", c->maximum);

	}

	return 0;

}

static int nx842_crypto_add_header(struct nx842_crypto_header *hdr, u8 *buf)

{

	int s = NX842_CRYPTO_HEADER_SIZE(hdr->groups);

	/* compress should have added space for header */

	if (s > be16_to_cpu(hdr->group[0].padding)) {

		pr_err("Internal error: no space for header\n");

		return -EINVAL;

	}

	memcpy(buf, hdr, s);

	print_hex_dump_debug("header ", DUMP_PREFIX_OFFSET, 16, 1, buf, s, 0);

	return 0;

}

static int compress(struct nx842_crypto_ctx *ctx,

		    struct nx842_crypto_param *p,

		    struct nx842_crypto_header_group *g,

		    struct nx842_constraints *c,

		    u16 *ignore,

		    unsigned int hdrsize)

{

	unsigned int slen = p->iremain, dlen = p->oremain, tmplen;

	unsigned int adj_slen = slen;

	u8 *src = p->in, *dst = p->out;

	int ret, dskip = 0;

	ktime_t timeout;

	if (p->iremain == 0)

		return -EOVERFLOW;

	if (p->oremain == 0 || hdrsize + c->minimum > dlen)

		return -ENOSPC;

	if (slen % c->multiple)

		adj_slen = round_up(slen, c->multiple);

	if (slen < c->minimum)

		adj_slen = c->minimum;

	if (slen > c->maximum)

		adj_slen = slen = c->maximum;

	if (adj_slen > slen || (u64)src % c->alignment) {

		adj_slen = min(adj_slen, BOUNCE_BUFFER_SIZE);

		slen = min(slen, BOUNCE_BUFFER_SIZE);

		if (adj_slen > slen)

			memset(ctx->sbounce + slen, 0, adj_slen - slen);

		memcpy(ctx->sbounce, src, slen);

		src = ctx->sbounce;

		slen = adj_slen;

		pr_debug("using comp sbounce buffer, len %x\n", slen);

	}

	dst += hdrsize;

	dlen -= hdrsize;

	if ((u64)dst % c->alignment) {

		dskip = (int)(PTR_ALIGN(dst, c->alignment) - dst);

		dst += dskip;

		dlen -= dskip;

	}

	if (dlen % c->multiple)

		dlen = round_down(dlen, c->multiple);

	if (dlen < c->minimum) {

nospc:

		dst = ctx->dbounce;

		dlen = min(p->oremain, BOUNCE_BUFFER_SIZE);

		dlen = round_down(dlen, c->multiple);

		dskip = 0;

		pr_debug("using comp dbounce buffer, len %x\n", dlen);

	}

	if (dlen > c->maximum)

		dlen = c->maximum;

	tmplen = dlen;

	timeout = ktime_add_ms(ktime_get(), COMP_BUSY_TIMEOUT);

	do {

		dlen = tmplen; /* reset dlen, if we're retrying */

		ret = nx842_compress(src, slen, dst, &dlen, ctx->wmem);

		/* possibly we should reduce the slen here, instead of

		 * retrying with the dbounce buffer?

		 */

		if (ret == -ENOSPC && dst != ctx->dbounce)

			goto nospc;

	} while (ret == -EBUSY && ktime_before(ktime_get(), timeout));

	if (ret)

		return ret;

	dskip += hdrsize;

	if (dst == ctx->dbounce)

		memcpy(p->out + dskip, dst, dlen);

	g->padding = cpu_to_be16(dskip);

	g->compressed_length = cpu_to_be32(dlen);

	g->uncompressed_length = cpu_to_be32(slen);

	if (p->iremain < slen) {

		*ignore = slen - p->iremain;

		slen = p->iremain;

	}

	pr_debug("compress slen %x ignore %x dlen %x padding %x\n",

		 slen, *ignore, dlen, dskip);

	return update_param(p, slen, dskip + dlen);

}

static int nx842_crypto_compress(struct crypto_tfm *tfm,

				 const u8 *src, unsigned int slen,

				 u8 *dst, unsigned int *dlen)

{

	struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);

	struct nx842_crypto_header *hdr = &ctx->header;

	struct nx842_crypto_param p;

	struct nx842_constraints c;

	unsigned int groups, hdrsize, h;

	int ret, n;

	bool add_header;

	u16 ignore = 0;

	if (!tfm || !src || !slen || !dst || !dlen)

		return -EINVAL;

	p.in = (u8 *)src;

	p.iremain = slen;

	p.out = dst;

	p.oremain = *dlen;

	p.ototal = 0;

	*dlen = 0;

	ret = read_constraints(&c);

	if (ret)

		return ret;

	groups = min_t(unsigned int, NX842_CRYPTO_GROUP_MAX,

		       DIV_ROUND_UP(p.iremain, c.maximum));

	hdrsize = NX842_CRYPTO_HEADER_SIZE(groups);

	/* skip adding header if the buffers meet all constraints */

	add_header = (p.iremain % c.multiple	||

		      p.iremain < c.minimum	||

		      p.iremain > c.maximum	||

		      (u64)p.in % c.alignment	||

		      p.oremain % c.multiple	||

		      p.oremain < c.minimum	||

		      p.oremain > c.maximum	||

		      (u64)p.out % c.alignment);

	hdr->magic = cpu_to_be16(NX842_CRYPTO_MAGIC);

	hdr->groups = 0;

	hdr->ignore = 0;

	while (p.iremain > 0) {

		n = hdr->groups++;

		if (hdr->groups > NX842_CRYPTO_GROUP_MAX)

			return -ENOSPC;

		/* header goes before first group */

		h = !n && add_header ? hdrsize : 0;

		if (ignore)

			pr_warn("interal error, ignore is set %x\n", ignore);

		ret = compress(ctx, &p, &hdr->group[n], &c, &ignore, h);

		if (ret)

			return ret;

	}

	if (!add_header && hdr->groups > 1) {

		pr_err("Internal error: No header but multiple groups\n");

		return -EINVAL;

	}

	/* ignore indicates the input stream needed to be padded */

	hdr->ignore = cpu_to_be16(ignore);

	if (ignore)

		pr_debug("marked %d bytes as ignore\n", ignore);

	if (add_header)

		ret = nx842_crypto_add_header(hdr, dst);

	if (ret)

		return ret;

	*dlen = p.ototal;

	pr_debug("compress total slen %x dlen %x\n", slen, *dlen);

	return 0;

}

static int decompress(struct nx842_crypto_ctx *ctx,

		      struct nx842_crypto_param *p,

		      struct nx842_crypto_header_group *g,

		      struct nx842_constraints *c,

		      u16 ignore,

		      bool usehw)

{

	unsigned int slen = be32_to_cpu(g->compressed_length);

	unsigned int required_len = be32_to_cpu(g->uncompressed_length);

	unsigned int dlen = p->oremain, tmplen;

	unsigned int adj_slen = slen;

	u8 *src = p->in, *dst = p->out;

	u16 padding = be16_to_cpu(g->padding);

	int ret, spadding = 0, dpadding = 0;

	ktime_t timeout;

	if (!slen || !required_len)

		return -EINVAL;

	if (p->iremain <= 0 || padding + slen > p->iremain)

		return -EOVERFLOW;

	if (p->oremain <= 0 || required_len - ignore > p->oremain)

		return -ENOSPC;

	src += padding;

	if (!usehw)

		goto usesw;

	if (slen % c->multiple)

		adj_slen = round_up(slen, c->multiple);

	if (slen < c->minimum)

		adj_slen = c->minimum;

	if (slen > c->maximum)

		goto usesw;

	if (slen < adj_slen || (u64)src % c->alignment) {

		/* we can append padding bytes because the 842 format defines

		 * an "end" template (see lib/842/842_decompress.c) and will

		 * ignore any bytes following it.

		 */

		if (slen < adj_slen)

			memset(ctx->sbounce + slen, 0, adj_slen - slen);

		memcpy(ctx->sbounce, src, slen);

		src = ctx->sbounce;

		spadding = adj_slen - slen;

		slen = adj_slen;

		pr_debug("using decomp sbounce buffer, len %x\n", slen);

	}

	if (dlen % c->multiple)

		dlen = round_down(dlen, c->multiple);

	if (dlen < required_len || (u64)dst % c->alignment) {

		dst = ctx->dbounce;

		dlen = min(required_len, BOUNCE_BUFFER_SIZE);

		pr_debug("using decomp dbounce buffer, len %x\n", dlen);

	}

	if (dlen < c->minimum)

		goto usesw;

	if (dlen > c->maximum)

		dlen = c->maximum;

	tmplen = dlen;

	timeout = ktime_add_ms(ktime_get(), DECOMP_BUSY_TIMEOUT);

	do {

		dlen = tmplen; /* reset dlen, if we're retrying */

		ret = nx842_decompress(src, slen, dst, &dlen, ctx->wmem);

	} while (ret == -EBUSY && ktime_before(ktime_get(), timeout));

	if (ret) {

usesw:

		/* reset everything, sw doesn't have constraints */

		src = p->in + padding;

		slen = be32_to_cpu(g->compressed_length);

		spadding = 0;

		dst = p->out;

		dlen = p->oremain;

		dpadding = 0;

		if (dlen < required_len) { /* have ignore bytes */

			dst = ctx->dbounce;

			dlen = BOUNCE_BUFFER_SIZE;

		}

		pr_info_ratelimited("using software 842 decompression\n");

		ret = sw842_decompress(src, slen, dst, &dlen);

	}

	if (ret)

		return ret;

	slen -= spadding;

	dlen -= ignore;

	if (ignore)

		pr_debug("ignoring last %x bytes\n", ignore);

	if (dst == ctx->dbounce)

		memcpy(p->out, dst, dlen);

	pr_debug("decompress slen %x padding %x dlen %x ignore %x\n",

		 slen, padding, dlen, ignore);

	return update_param(p, slen + padding, dlen);

}

static int nx842_crypto_decompress(struct crypto_tfm *tfm,

				   const u8 *src, unsigned int slen,

				   u8 *dst, unsigned int *dlen)

{

	struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);

	struct nx842_crypto_header *hdr;

	struct nx842_crypto_param p;

	struct nx842_constraints c;

	int n, ret, hdr_len;

	u16 ignore = 0;

	bool usehw = true;

	if (!tfm || !src || !slen || !dst || !dlen)

		return -EINVAL;

	p.in = (u8 *)src;

	p.iremain = slen;

	p.out = dst;

	p.oremain = *dlen;

	p.ototal = 0;

	*dlen = 0;

	if (read_constraints(&c))

		usehw = false;

	hdr = (struct nx842_crypto_header *)src;

	/* If it doesn't start with our header magic number, assume it's a raw

	 * 842 compressed buffer and pass it directly to the hardware driver

	 */

	if (be16_to_cpu(hdr->magic) != NX842_CRYPTO_MAGIC) {

		struct nx842_crypto_header_group g = {

			.padding =		0,

			.compressed_length =	cpu_to_be32(p.iremain),

			.uncompressed_length =	cpu_to_be32(p.oremain),

		};

		ret = decompress(ctx, &p, &g, &c, 0, usehw);

		if (ret)

			return ret;

		*dlen = p.ototal;

		return 0;

	}

	if (!hdr->groups) {

		pr_err("header has no groups\n");

		return -EINVAL;

	}

	if (hdr->groups > NX842_CRYPTO_GROUP_MAX) {

		pr_err("header has too many groups %x, max %x\n",

		       hdr->groups, NX842_CRYPTO_GROUP_MAX);

		return -EINVAL;

	}

	hdr_len = NX842_CRYPTO_HEADER_SIZE(hdr->groups);

	if (hdr_len > slen)

		return -EOVERFLOW;

	memcpy(&ctx->header, src, hdr_len);

	hdr = &ctx->header;

	for (n = 0; n < hdr->groups; n++) {

		/* ignore applies to last group */

		if (n + 1 == hdr->groups)

			ignore = be16_to_cpu(hdr->ignore);

		ret = decompress(ctx, &p, &hdr->group[n], &c, ignore, usehw);

		if (ret)

			return ret;

	}

	*dlen = p.ototal;

	pr_debug("decompress total slen %x dlen %x\n", slen, *dlen);

	return 0;

}

static struct crypto_alg alg = {

	.cra_name		= "842",

	.cra_driver_name	= "842-nx",

	.cra_priority		= 300,

	.cra_flags		= CRYPTO_ALG_TYPE_COMPRESS,

	.cra_ctxsize		= sizeof(struct nx842_crypto_ctx),

	.cra_module		= THIS_MODULE,

	.cra_init		= nx842_crypto_init,

	.cra_exit		= nx842_crypto_exit,

	.cra_u			= { .compress = {

	.coa_compress		= nx842_crypto_compress,

	.coa_decompress		= nx842_crypto_decompress } }

};

static int __init nx842_crypto_mod_init(void)

{

	return crypto_register_alg(&alg);

}

module_init(nx842_crypto_mod_init);

static void __exit nx842_crypto_mod_exit(void)

{

	crypto_unregister_alg(&alg);

}

module_exit(nx842_crypto_mod_exit);

MODULE_LICENSE("GPL");

MODULE_DESCRIPTION("IBM PowerPC Nest (NX) 842 Hardware Compression Interface");

MODULE_ALIAS_CRYPTO("842");

MODULE_ALIAS_CRYPTO("842-nx");

MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");