Commit cfb4b571 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 's390-5.6-2' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux 

Pull more s390 updates from Vasily Gorbik:
 "The second round of s390 fixes and features for 5.6:

   - Add KPROBES_ON_FTRACE support

   - Add EP11 AES secure keys support

   - PAES rework and prerequisites for paes-s390 ciphers selftests

   - Fix page table upgrade for hugetlbfs"

* tag 's390-5.6-2' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux:
  s390/pkey/zcrypt: Support EP11 AES secure keys
  s390/zcrypt: extend EP11 card and queue sysfs attributes
  s390/zcrypt: add new low level ep11 functions support file
  s390/zcrypt: ep11 structs rework, export zcrypt_send_ep11_cprb
  s390/zcrypt: enable card/domain autoselect on ep11 cprbs
  s390/crypto: enable clear key values for paes ciphers
  s390/pkey: Add support for key blob with clear key value
  s390/crypto: Rework on paes implementation
  s390: support KPROBES_ON_FTRACE
  s390/mm: fix dynamic pagetable upgrade for hugetlbfs
parents 6992ca0d 55d0a513

Documentation/features/debug/kprobes-on-ftrace/arch-support.txt

+1 −1
Original line number Diff line number Diff line
@@ -24,7 +24,7 @@ 
    |      parisc: |  ok  |

    |     powerpc: |  ok  |

    |       riscv: | TODO |

    |        s390: | TODO |

    |        s390: |  ok  |

    |          sh: | TODO |

    |       sparc: | TODO |

    |          um: | TODO |

arch/s390/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -156,6 +156,7 @@ config S390 
	select HAVE_KERNEL_UNCOMPRESSED

	select HAVE_KERNEL_XZ

	select HAVE_KPROBES

	select HAVE_KPROBES_ON_FTRACE

	select HAVE_KRETPROBES

	select HAVE_KVM

	select HAVE_LIVEPATCH

arch/s390/crypto/paes_s390.c

+167 −63
Original line number Diff line number Diff line
@@ -5,7 +5,7 @@ 
 * s390 implementation of the AES Cipher Algorithm with protected keys.

 *

 * s390 Version:

 *   Copyright IBM Corp. 2017,2019

 *   Copyright IBM Corp. 2017,2020

 *   Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>

 *		Harald Freudenberger <freude@de.ibm.com>

 */
@@ -20,6 +20,7 @@ 
#include <linux/module.h>

#include <linux/cpufeature.h>

#include <linux/init.h>

#include <linux/mutex.h>

#include <linux/spinlock.h>

#include <crypto/internal/skcipher.h>

#include <crypto/xts.h>
@@ -32,11 +33,11 @@ 
 * is called. As paes can handle different kinds of key blobs

 * and padding is also possible, the limits need to be generous.

 */

#define PAES_MIN_KEYSIZE 64

#define PAES_MAX_KEYSIZE 256

#define PAES_MIN_KEYSIZE 16

#define PAES_MAX_KEYSIZE 320



static u8 *ctrblk;

static DEFINE_SPINLOCK(ctrblk_lock);

static DEFINE_MUTEX(ctrblk_lock);



static cpacf_mask_t km_functions, kmc_functions, kmctr_functions;
@@ -53,10 +54,35 @@ struct key_blob { 
	unsigned int keylen;

};



static inline int _copy_key_to_kb(struct key_blob *kb,

static inline int _key_to_kb(struct key_blob *kb,

			     const u8 *key,

			     unsigned int keylen)

{

	struct clearkey_header {

		u8  type;

		u8  res0[3];

		u8  version;

		u8  res1[3];

		u32 keytype;

		u32 len;

	} __packed * h;



	switch (keylen) {

	case 16:

	case 24:

	case 32:

		/* clear key value, prepare pkey clear key token in keybuf */

		memset(kb->keybuf, 0, sizeof(kb->keybuf));

		h = (struct clearkey_header *) kb->keybuf;

		h->version = 0x02; /* TOKVER_CLEAR_KEY */

		h->keytype = (keylen - 8) >> 3;

		h->len = keylen;

		memcpy(kb->keybuf + sizeof(*h), key, keylen);

		kb->keylen = sizeof(*h) + keylen;

		kb->key = kb->keybuf;

		break;

	default:

		/* other key material, let pkey handle this */

		if (keylen <= sizeof(kb->keybuf))

			kb->key = kb->keybuf;

		else {
@@ -66,6 +92,8 @@ static inline int _copy_key_to_kb(struct key_blob *kb, 
		}

		memcpy(kb->key, key, keylen);

		kb->keylen = keylen;

		break;

	}



	return 0;

}
@@ -82,16 +110,18 @@ static inline void _free_kb_keybuf(struct key_blob *kb) 
struct s390_paes_ctx {

	struct key_blob kb;

	struct pkey_protkey pk;

	spinlock_t pk_lock;

	unsigned long fc;

};



struct s390_pxts_ctx {

	struct key_blob kb[2];

	struct pkey_protkey pk[2];

	spinlock_t pk_lock;

	unsigned long fc;

};



static inline int __paes_convert_key(struct key_blob *kb,

static inline int __paes_keyblob2pkey(struct key_blob *kb,

				     struct pkey_protkey *pk)

{

	int i, ret;
@@ -106,22 +136,18 @@ static inline int __paes_convert_key(struct key_blob *kb, 
	return ret;

}



static int __paes_set_key(struct s390_paes_ctx *ctx)

static inline int __paes_convert_key(struct s390_paes_ctx *ctx)

{

	unsigned long fc;

	struct pkey_protkey pkey;



	if (__paes_convert_key(&ctx->kb, &ctx->pk))

	if (__paes_keyblob2pkey(&ctx->kb, &pkey))

		return -EINVAL;



	/* Pick the correct function code based on the protected key type */

	fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KM_PAES_128 :

		(ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KM_PAES_192 :

		(ctx->pk.type == PKEY_KEYTYPE_AES_256) ? CPACF_KM_PAES_256 : 0;



	/* Check if the function code is available */

	ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0;

	spin_lock_bh(&ctx->pk_lock);

	memcpy(&ctx->pk, &pkey, sizeof(pkey));

	spin_unlock_bh(&ctx->pk_lock);



	return ctx->fc ? 0 : -EINVAL;

	return 0;

}



static int ecb_paes_init(struct crypto_skcipher *tfm)
@@ -129,6 +155,7 @@ static int ecb_paes_init(struct crypto_skcipher *tfm) 
	struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);



	ctx->kb.key = NULL;

	spin_lock_init(&ctx->pk_lock);



	return 0;

}
@@ -140,6 +167,24 @@ static void ecb_paes_exit(struct crypto_skcipher *tfm) 
	_free_kb_keybuf(&ctx->kb);

}



static inline int __ecb_paes_set_key(struct s390_paes_ctx *ctx)

{

	unsigned long fc;



	if (__paes_convert_key(ctx))

		return -EINVAL;



	/* Pick the correct function code based on the protected key type */

	fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KM_PAES_128 :

		(ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KM_PAES_192 :

		(ctx->pk.type == PKEY_KEYTYPE_AES_256) ? CPACF_KM_PAES_256 : 0;



	/* Check if the function code is available */

	ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0;



	return ctx->fc ? 0 : -EINVAL;

}



static int ecb_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,

			    unsigned int key_len)

{
@@ -147,11 +192,11 @@ static int ecb_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 
	struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);



	_free_kb_keybuf(&ctx->kb);

	rc = _copy_key_to_kb(&ctx->kb, in_key, key_len);

	rc = _key_to_kb(&ctx->kb, in_key, key_len);

	if (rc)

		return rc;



	return __paes_set_key(ctx);

	return __ecb_paes_set_key(ctx);

}



static int ecb_paes_crypt(struct skcipher_request *req, unsigned long modifier)
@@ -161,18 +206,31 @@ static int ecb_paes_crypt(struct skcipher_request *req, unsigned long modifier) 
	struct skcipher_walk walk;

	unsigned int nbytes, n, k;

	int ret;

	struct {

		u8 key[MAXPROTKEYSIZE];

	} param;



	ret = skcipher_walk_virt(&walk, req, false);

	if (ret)

		return ret;



	spin_lock_bh(&ctx->pk_lock);

	memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);

	spin_unlock_bh(&ctx->pk_lock);



	while ((nbytes = walk.nbytes) != 0) {

		/* only use complete blocks */

		n = nbytes & ~(AES_BLOCK_SIZE - 1);

		k = cpacf_km(ctx->fc | modifier, ctx->pk.protkey,

		k = cpacf_km(ctx->fc | modifier, &param,

			     walk.dst.virt.addr, walk.src.virt.addr, n);

		if (k)

			ret = skcipher_walk_done(&walk, nbytes - k);

		if (k < n) {

			if (__paes_set_key(ctx) != 0)

			if (__paes_convert_key(ctx))

				return skcipher_walk_done(&walk, -EIO);

			spin_lock_bh(&ctx->pk_lock);

			memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);

			spin_unlock_bh(&ctx->pk_lock);

		}

	}

	return ret;
@@ -210,6 +268,7 @@ static int cbc_paes_init(struct crypto_skcipher *tfm) 
	struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);



	ctx->kb.key = NULL;

	spin_lock_init(&ctx->pk_lock);



	return 0;

}
@@ -221,11 +280,11 @@ static void cbc_paes_exit(struct crypto_skcipher *tfm) 
	_free_kb_keybuf(&ctx->kb);

}



static int __cbc_paes_set_key(struct s390_paes_ctx *ctx)

static inline int __cbc_paes_set_key(struct s390_paes_ctx *ctx)

{

	unsigned long fc;



	if (__paes_convert_key(&ctx->kb, &ctx->pk))

	if (__paes_convert_key(ctx))

		return -EINVAL;



	/* Pick the correct function code based on the protected key type */
@@ -246,7 +305,7 @@ static int cbc_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 
	struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);



	_free_kb_keybuf(&ctx->kb);

	rc = _copy_key_to_kb(&ctx->kb, in_key, key_len);

	rc = _key_to_kb(&ctx->kb, in_key, key_len);

	if (rc)

		return rc;
@@ -268,8 +327,12 @@ static int cbc_paes_crypt(struct skcipher_request *req, unsigned long modifier) 
	ret = skcipher_walk_virt(&walk, req, false);

	if (ret)

		return ret;



	memcpy(param.iv, walk.iv, AES_BLOCK_SIZE);

	spin_lock_bh(&ctx->pk_lock);

	memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);

	spin_unlock_bh(&ctx->pk_lock);



	while ((nbytes = walk.nbytes) != 0) {

		/* only use complete blocks */

		n = nbytes & ~(AES_BLOCK_SIZE - 1);
@@ -280,9 +343,11 @@ static int cbc_paes_crypt(struct skcipher_request *req, unsigned long modifier) 
			ret = skcipher_walk_done(&walk, nbytes - k);

		}

		if (k < n) {

			if (__cbc_paes_set_key(ctx) != 0)

			if (__paes_convert_key(ctx))

				return skcipher_walk_done(&walk, -EIO);

			spin_lock_bh(&ctx->pk_lock);

			memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);

			spin_unlock_bh(&ctx->pk_lock);

		}

	}

	return ret;
@@ -322,6 +387,7 @@ static int xts_paes_init(struct crypto_skcipher *tfm) 


	ctx->kb[0].key = NULL;

	ctx->kb[1].key = NULL;

	spin_lock_init(&ctx->pk_lock);



	return 0;

}
@@ -334,12 +400,27 @@ static void xts_paes_exit(struct crypto_skcipher *tfm) 
	_free_kb_keybuf(&ctx->kb[1]);

}



static int __xts_paes_set_key(struct s390_pxts_ctx *ctx)

static inline int __xts_paes_convert_key(struct s390_pxts_ctx *ctx)

{

	struct pkey_protkey pkey0, pkey1;



	if (__paes_keyblob2pkey(&ctx->kb[0], &pkey0) ||

	    __paes_keyblob2pkey(&ctx->kb[1], &pkey1))

		return -EINVAL;



	spin_lock_bh(&ctx->pk_lock);

	memcpy(&ctx->pk[0], &pkey0, sizeof(pkey0));

	memcpy(&ctx->pk[1], &pkey1, sizeof(pkey1));

	spin_unlock_bh(&ctx->pk_lock);



	return 0;

}



static inline int __xts_paes_set_key(struct s390_pxts_ctx *ctx)

{

	unsigned long fc;



	if (__paes_convert_key(&ctx->kb[0], &ctx->pk[0]) ||

	    __paes_convert_key(&ctx->kb[1], &ctx->pk[1]))

	if (__xts_paes_convert_key(ctx))

		return -EINVAL;



	if (ctx->pk[0].type != ctx->pk[1].type)
@@ -371,10 +452,10 @@ static int xts_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 


	_free_kb_keybuf(&ctx->kb[0]);

	_free_kb_keybuf(&ctx->kb[1]);

	rc = _copy_key_to_kb(&ctx->kb[0], in_key, key_len);

	rc = _key_to_kb(&ctx->kb[0], in_key, key_len);

	if (rc)

		return rc;

	rc = _copy_key_to_kb(&ctx->kb[1], in_key + key_len, key_len);

	rc = _key_to_kb(&ctx->kb[1], in_key + key_len, key_len);

	if (rc)

		return rc;
@@ -416,15 +497,17 @@ static int xts_paes_crypt(struct skcipher_request *req, unsigned long modifier) 
	ret = skcipher_walk_virt(&walk, req, false);

	if (ret)

		return ret;



	keylen = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 48 : 64;

	offset = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 16 : 0;

retry:



	memset(&pcc_param, 0, sizeof(pcc_param));

	memcpy(pcc_param.tweak, walk.iv, sizeof(pcc_param.tweak));

	spin_lock_bh(&ctx->pk_lock);

	memcpy(pcc_param.key + offset, ctx->pk[1].protkey, keylen);

	cpacf_pcc(ctx->fc, pcc_param.key + offset);



	memcpy(xts_param.key + offset, ctx->pk[0].protkey, keylen);

	spin_unlock_bh(&ctx->pk_lock);

	cpacf_pcc(ctx->fc, pcc_param.key + offset);

	memcpy(xts_param.init, pcc_param.xts, 16);



	while ((nbytes = walk.nbytes) != 0) {
@@ -435,11 +518,15 @@ retry: 
		if (k)

			ret = skcipher_walk_done(&walk, nbytes - k);

		if (k < n) {

			if (__xts_paes_set_key(ctx) != 0)

			if (__xts_paes_convert_key(ctx))

				return skcipher_walk_done(&walk, -EIO);

			goto retry;

			spin_lock_bh(&ctx->pk_lock);

			memcpy(xts_param.key + offset,

			       ctx->pk[0].protkey, keylen);

			spin_unlock_bh(&ctx->pk_lock);

		}

	}



	return ret;

}
@@ -476,6 +563,7 @@ static int ctr_paes_init(struct crypto_skcipher *tfm) 
	struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);



	ctx->kb.key = NULL;

	spin_lock_init(&ctx->pk_lock);



	return 0;

}
@@ -487,11 +575,11 @@ static void ctr_paes_exit(struct crypto_skcipher *tfm) 
	_free_kb_keybuf(&ctx->kb);

}



static int __ctr_paes_set_key(struct s390_paes_ctx *ctx)

static inline int __ctr_paes_set_key(struct s390_paes_ctx *ctx)

{

	unsigned long fc;



	if (__paes_convert_key(&ctx->kb, &ctx->pk))

	if (__paes_convert_key(ctx))

		return -EINVAL;



	/* Pick the correct function code based on the protected key type */
@@ -513,7 +601,7 @@ static int ctr_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 
	struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);



	_free_kb_keybuf(&ctx->kb);

	rc = _copy_key_to_kb(&ctx->kb, in_key, key_len);

	rc = _key_to_kb(&ctx->kb, in_key, key_len);

	if (rc)

		return rc;
@@ -543,49 +631,65 @@ static int ctr_paes_crypt(struct skcipher_request *req) 
	struct skcipher_walk walk;

	unsigned int nbytes, n, k;

	int ret, locked;



	locked = spin_trylock(&ctrblk_lock);

	struct {

		u8 key[MAXPROTKEYSIZE];

	} param;



	ret = skcipher_walk_virt(&walk, req, false);

	if (ret)

		return ret;



	spin_lock_bh(&ctx->pk_lock);

	memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);

	spin_unlock_bh(&ctx->pk_lock);



	locked = mutex_trylock(&ctrblk_lock);



	while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {

		n = AES_BLOCK_SIZE;

		if (nbytes >= 2*AES_BLOCK_SIZE && locked)

			n = __ctrblk_init(ctrblk, walk.iv, nbytes);

		ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk.iv;

		k = cpacf_kmctr(ctx->fc, ctx->pk.protkey, walk.dst.virt.addr,

		k = cpacf_kmctr(ctx->fc, &param, walk.dst.virt.addr,

				walk.src.virt.addr, n, ctrptr);

		if (k) {

			if (ctrptr == ctrblk)

				memcpy(walk.iv, ctrptr + k - AES_BLOCK_SIZE,

				       AES_BLOCK_SIZE);

			crypto_inc(walk.iv, AES_BLOCK_SIZE);

			ret = skcipher_walk_done(&walk, nbytes - n);

			ret = skcipher_walk_done(&walk, nbytes - k);

		}

		if (k < n) {

			if (__ctr_paes_set_key(ctx) != 0) {

			if (__paes_convert_key(ctx)) {

				if (locked)

					spin_unlock(&ctrblk_lock);

					mutex_unlock(&ctrblk_lock);

				return skcipher_walk_done(&walk, -EIO);

			}

			spin_lock_bh(&ctx->pk_lock);

			memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);

			spin_unlock_bh(&ctx->pk_lock);

		}

	}

	if (locked)

		spin_unlock(&ctrblk_lock);

		mutex_unlock(&ctrblk_lock);

	/*

	 * final block may be < AES_BLOCK_SIZE, copy only nbytes

	 */

	if (nbytes) {

		while (1) {

			if (cpacf_kmctr(ctx->fc, ctx->pk.protkey, buf,

			if (cpacf_kmctr(ctx->fc, &param, buf,

					walk.src.virt.addr, AES_BLOCK_SIZE,

					walk.iv) == AES_BLOCK_SIZE)

				break;

			if (__ctr_paes_set_key(ctx) != 0)

			if (__paes_convert_key(ctx))

				return skcipher_walk_done(&walk, -EIO);

			spin_lock_bh(&ctx->pk_lock);

			memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE);

			spin_unlock_bh(&ctx->pk_lock);

		}

		memcpy(walk.dst.virt.addr, buf, nbytes);

		crypto_inc(walk.iv, AES_BLOCK_SIZE);

		ret = skcipher_walk_done(&walk, 0);

		ret = skcipher_walk_done(&walk, nbytes);

	}



	return ret;
@@ -618,12 +722,12 @@ static inline void __crypto_unregister_skcipher(struct skcipher_alg *alg) 


static void paes_s390_fini(void)

{

	if (ctrblk)

		free_page((unsigned long) ctrblk);

	__crypto_unregister_skcipher(&ctr_paes_alg);

	__crypto_unregister_skcipher(&xts_paes_alg);

	__crypto_unregister_skcipher(&cbc_paes_alg);

	__crypto_unregister_skcipher(&ecb_paes_alg);

	if (ctrblk)

		free_page((unsigned long) ctrblk);

}



static int __init paes_s390_init(void)
@@ -661,14 +765,14 @@ static int __init paes_s390_init(void) 
	if (cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_128) ||

	    cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_192) ||

	    cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_256)) {

		ret = crypto_register_skcipher(&ctr_paes_alg);

		if (ret)

			goto out_err;

		ctrblk = (u8 *) __get_free_page(GFP_KERNEL);

		if (!ctrblk) {

			ret = -ENOMEM;

			goto out_err;

		}

		ret = crypto_register_skcipher(&ctr_paes_alg);

		if (ret)

			goto out_err;

	}



	return 0;

arch/s390/include/asm/kprobes.h

+0 −1
Original line number Diff line number Diff line
@@ -54,7 +54,6 @@ typedef u16 kprobe_opcode_t; 
struct arch_specific_insn {

	/* copy of original instruction */

	kprobe_opcode_t *insn;

	unsigned int is_ftrace_insn : 1;

};



struct prev_kprobe {

arch/s390/include/asm/page.h

+2 −0
Original line number Diff line number Diff line
@@ -33,6 +33,8 @@ 
#define ARCH_HAS_PREPARE_HUGEPAGE

#define ARCH_HAS_HUGEPAGE_CLEAR_FLUSH



#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA



#include <asm/setup.h>

#ifndef __ASSEMBLY__

CRA Git | Maintained and supported by SUSTech CRA and CCSE