Commit 9de31912 authored by Vasily Gorbik's avatar Vasily Gorbik
Browse files

Merge tag 'vfio-ccw-20190621' of... 

Merge tag 'vfio-ccw-20190621' of https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/vfio-ccw into features

Refactoring of the vfio-ccw cp handling, simplifying the
code and avoiding unneeded allocating/copying.

* tag 'vfio-ccw-20190621' of https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/vfio-ccw

:
  vfio-ccw: Remove copy_ccw_from_iova()
  vfio-ccw: Factor out the ccw0-to-ccw1 transition
  vfio-ccw: Copy CCW data outside length calculation
  vfio-ccw: Skip second copy of guest cp to host
  vfio-ccw: Move guest_cp storage into common struct
  s390/cio: Combine direct and indirect CCW paths
  vfio-ccw: Rearrange IDAL allocation in direct CCW
  vfio-ccw: Remove pfn_array_table
  vfio-ccw: Adjust the first IDAW outside of the nested loops
  vfio-ccw: Rearrange pfn_array and pfn_array_table arrays
  s390/cio: Use generalized CCW handler in cp_init()
  s390/cio: Generalize the TIC handler
  s390/cio: Refactor the routine that handles TIC CCWs
  s390/cio: Squash cp_free() and cp_unpin_free()

Signed-off-by: default avatarVasily Gorbik <gor@linux.ibm.com>
parents 96e5aaf9 5223bee8

drivers/s390/cio/vfio_ccw_cp.c

+137 −278
Original line number Diff line number Diff line
@@ -16,12 +16,6 @@ 


#include "vfio_ccw_cp.h"



/*

 * Max length for ccw chain.

 * XXX: Limit to 256, need to check more?

 */

#define CCWCHAIN_LEN_MAX	256



struct pfn_array {

	/* Starting guest physical I/O address. */

	unsigned long		pa_iova;
@@ -33,11 +27,6 @@ struct pfn_array { 
	int			pa_nr;

};



struct pfn_array_table {

	struct pfn_array	*pat_pa;

	int			pat_nr;

};



struct ccwchain {

	struct list_head	next;

	struct ccw1		*ch_ccw;
@@ -46,7 +35,7 @@ struct ccwchain { 
	/* Count of the valid ccws in chain. */

	int			ch_len;

	/* Pinned PAGEs for the original data. */

	struct pfn_array_table	*ch_pat;

	struct pfn_array	*ch_pa;

};



/*
@@ -139,55 +128,23 @@ static void pfn_array_unpin_free(struct pfn_array *pa, struct device *mdev) 
	kfree(pa->pa_iova_pfn);

}



static int pfn_array_table_init(struct pfn_array_table *pat, int nr)

{

	pat->pat_pa = kcalloc(nr, sizeof(*pat->pat_pa), GFP_KERNEL);

	if (unlikely(ZERO_OR_NULL_PTR(pat->pat_pa))) {

		pat->pat_nr = 0;

		return -ENOMEM;

	}



	pat->pat_nr = nr;



	return 0;

}



static void pfn_array_table_unpin_free(struct pfn_array_table *pat,

				       struct device *mdev)

static bool pfn_array_iova_pinned(struct pfn_array *pa, unsigned long iova)

{

	int i;



	for (i = 0; i < pat->pat_nr; i++)

		pfn_array_unpin_free(pat->pat_pa + i, mdev);



	if (pat->pat_nr) {

		kfree(pat->pat_pa);

		pat->pat_pa = NULL;

		pat->pat_nr = 0;

	}

}



static bool pfn_array_table_iova_pinned(struct pfn_array_table *pat,

					unsigned long iova)

{

	struct pfn_array *pa = pat->pat_pa;

	unsigned long iova_pfn = iova >> PAGE_SHIFT;

	int i, j;

	int i;



	for (i = 0; i < pat->pat_nr; i++, pa++)

		for (j = 0; j < pa->pa_nr; j++)

			if (pa->pa_iova_pfn[j] == iova_pfn)

	for (i = 0; i < pa->pa_nr; i++)

		if (pa->pa_iova_pfn[i] == iova_pfn)

			return true;



	return false;

}

/* Create the list idal words for a pfn_array_table. */

static inline void pfn_array_table_idal_create_words(

	struct pfn_array_table *pat,

/* Create the list of IDAL words for a pfn_array. */

static inline void pfn_array_idal_create_words(

	struct pfn_array *pa,

	unsigned long *idaws)

{

	struct pfn_array *pa;

	int i, j, k;

	int i;



	/*

	 * Idal words (execept the first one) rely on the memory being 4k
@@ -196,18 +153,35 @@ static inline void pfn_array_table_idal_create_words( 
	 * there will be no problem here to simply use the phys to create an

	 * idaw.

	 */

	k = 0;

	for (i = 0; i < pat->pat_nr; i++) {

		pa = pat->pat_pa + i;

		for (j = 0; j < pa->pa_nr; j++) {

			idaws[k] = pa->pa_pfn[j] << PAGE_SHIFT;

			if (k == 0)

				idaws[k] += pa->pa_iova & (PAGE_SIZE - 1);

			k++;



	for (i = 0; i < pa->pa_nr; i++)

		idaws[i] = pa->pa_pfn[i] << PAGE_SHIFT;



	/* Adjust the first IDAW, since it may not start on a page boundary */

	idaws[0] += pa->pa_iova & (PAGE_SIZE - 1);

}



void convert_ccw0_to_ccw1(struct ccw1 *source, unsigned long len)

{

	struct ccw0 ccw0;

	struct ccw1 *pccw1 = source;

	int i;



	for (i = 0; i < len; i++) {

		ccw0 = *(struct ccw0 *)pccw1;

		if ((pccw1->cmd_code & 0x0f) == CCW_CMD_TIC) {

			pccw1->cmd_code = CCW_CMD_TIC;

			pccw1->flags = 0;

			pccw1->count = 0;

		} else {

			pccw1->cmd_code = ccw0.cmd_code;

			pccw1->flags = ccw0.flags;

			pccw1->count = ccw0.count;

		}

		pccw1->cda = ccw0.cda;

		pccw1++;

	}

}





/*

 * Within the domain (@mdev), copy @n bytes from a guest physical
@@ -254,40 +228,6 @@ static long copy_from_iova(struct device *mdev, 
	return l;

}



static long copy_ccw_from_iova(struct channel_program *cp,

			       struct ccw1 *to, u64 iova,

			       unsigned long len)

{

	struct ccw0 ccw0;

	struct ccw1 *pccw1;

	int ret;

	int i;



	ret = copy_from_iova(cp->mdev, to, iova, len * sizeof(struct ccw1));

	if (ret)

		return ret;



	if (!cp->orb.cmd.fmt) {

		pccw1 = to;

		for (i = 0; i < len; i++) {

			ccw0 = *(struct ccw0 *)pccw1;

			if ((pccw1->cmd_code & 0x0f) == CCW_CMD_TIC) {

				pccw1->cmd_code = CCW_CMD_TIC;

				pccw1->flags = 0;

				pccw1->count = 0;

			} else {

				pccw1->cmd_code = ccw0.cmd_code;

				pccw1->flags = ccw0.flags;

				pccw1->count = ccw0.count;

			}

			pccw1->cda = ccw0.cda;

			pccw1++;

		}

	}



	return ret;

}



/*

 * Helpers to operate ccwchain.

 */
@@ -377,7 +317,7 @@ static struct ccwchain *ccwchain_alloc(struct channel_program *cp, int len) 
	/* Make ccw address aligned to 8. */

	size = ((sizeof(*chain) + 7L) & -8L) +

		sizeof(*chain->ch_ccw) * len +

		sizeof(*chain->ch_pat) * len;

		sizeof(*chain->ch_pa) * len;

	chain = kzalloc(size, GFP_DMA | GFP_KERNEL);

	if (!chain)

		return NULL;
@@ -386,7 +326,7 @@ static struct ccwchain *ccwchain_alloc(struct channel_program *cp, int len) 
	chain->ch_ccw = (struct ccw1 *)data;



	data = (u8 *)(chain->ch_ccw) + sizeof(*chain->ch_ccw) * len;

	chain->ch_pat = (struct pfn_array_table *)data;

	chain->ch_pa = (struct pfn_array *)data;



	chain->ch_len = len;
@@ -412,23 +352,6 @@ static void ccwchain_cda_free(struct ccwchain *chain, int idx) 
	kfree((void *)(u64)ccw->cda);

}



/* Unpin the pages then free the memory resources. */

static void cp_unpin_free(struct channel_program *cp)

{

	struct ccwchain *chain, *temp;

	int i;



	cp->initialized = false;

	list_for_each_entry_safe(chain, temp, &cp->ccwchain_list, next) {

		for (i = 0; i < chain->ch_len; i++) {

			pfn_array_table_unpin_free(chain->ch_pat + i,

						   cp->mdev);

			ccwchain_cda_free(chain, i);

		}

		ccwchain_free(chain);

	}

}



/**

 * ccwchain_calc_length - calculate the length of the ccw chain.

 * @iova: guest physical address of the target ccw chain
@@ -444,25 +367,9 @@ static void cp_unpin_free(struct channel_program *cp) 
 */

static int ccwchain_calc_length(u64 iova, struct channel_program *cp)

{

	struct ccw1 *ccw, *p;

	int cnt;

	struct ccw1 *ccw = cp->guest_cp;

	int cnt = 0;



	/*

	 * Copy current chain from guest to host kernel.

	 * Currently the chain length is limited to CCWCHAIN_LEN_MAX (256).

	 * So copying 2K is enough (safe).

	 */

	p = ccw = kcalloc(CCWCHAIN_LEN_MAX, sizeof(*ccw), GFP_KERNEL);

	if (!ccw)

		return -ENOMEM;



	cnt = copy_ccw_from_iova(cp, ccw, iova, CCWCHAIN_LEN_MAX);

	if (cnt) {

		kfree(ccw);

		return cnt;

	}



	cnt = 0;

	do {

		cnt++;
@@ -471,10 +378,8 @@ static int ccwchain_calc_length(u64 iova, struct channel_program *cp) 
		 * orb specified one of the unsupported formats, we defer

		 * checking for IDAWs in unsupported formats to here.

		 */

		if ((!cp->orb.cmd.c64 || cp->orb.cmd.i2k) && ccw_is_idal(ccw)) {

			kfree(p);

		if ((!cp->orb.cmd.c64 || cp->orb.cmd.i2k) && ccw_is_idal(ccw))

			return -EOPNOTSUPP;

		}



		/*

		 * We want to keep counting if the current CCW has the
@@ -493,7 +398,6 @@ static int ccwchain_calc_length(u64 iova, struct channel_program *cp) 
	if (cnt == CCWCHAIN_LEN_MAX + 1)

		cnt = -EINVAL;



	kfree(p);

	return cnt;

}
@@ -514,17 +418,23 @@ static int tic_target_chain_exists(struct ccw1 *tic, struct channel_program *cp) 
static int ccwchain_loop_tic(struct ccwchain *chain,

			     struct channel_program *cp);



static int ccwchain_handle_tic(struct ccw1 *tic, struct channel_program *cp)

static int ccwchain_handle_ccw(u32 cda, struct channel_program *cp)

{

	struct ccwchain *chain;

	int len, ret;

	int len;



	/* May transfer to an existing chain. */

	if (tic_target_chain_exists(tic, cp))

		return 0;

	/* Copy 2K (the most we support today) of possible CCWs */

	len = copy_from_iova(cp->mdev, cp->guest_cp, cda,

			     CCWCHAIN_LEN_MAX * sizeof(struct ccw1));

	if (len)

		return len;



	/* Get chain length. */

	len = ccwchain_calc_length(tic->cda, cp);

	/* Convert any Format-0 CCWs to Format-1 */

	if (!cp->orb.cmd.fmt)

		convert_ccw0_to_ccw1(cp->guest_cp, len);



	/* Count the CCWs in the current chain */

	len = ccwchain_calc_length(cda, cp);

	if (len < 0)

		return len;
@@ -532,14 +442,10 @@ static int ccwchain_handle_tic(struct ccw1 *tic, struct channel_program *cp) 
	chain = ccwchain_alloc(cp, len);

	if (!chain)

		return -ENOMEM;

	chain->ch_iova = tic->cda;

	chain->ch_iova = cda;



	/* Copy the new chain from user. */

	ret = copy_ccw_from_iova(cp, chain->ch_ccw, tic->cda, len);

	if (ret) {

		ccwchain_free(chain);

		return ret;

	}

	/* Copy the actual CCWs into the new chain */

	memcpy(chain->ch_ccw, cp->guest_cp, len * sizeof(struct ccw1));



	/* Loop for tics on this new chain. */

	return ccwchain_loop_tic(chain, cp);
@@ -557,7 +463,12 @@ static int ccwchain_loop_tic(struct ccwchain *chain, struct channel_program *cp) 
		if (!ccw_is_tic(tic))

			continue;



		ret = ccwchain_handle_tic(tic, cp);

		/* May transfer to an existing chain. */

		if (tic_target_chain_exists(tic, cp))

			continue;



		/* Build a ccwchain for the next segment */

		ret = ccwchain_handle_ccw(tic->cda, cp);

		if (ret)

			return ret;

	}
@@ -590,131 +501,90 @@ static int ccwchain_fetch_direct(struct ccwchain *chain, 
				 struct channel_program *cp)

{

	struct ccw1 *ccw;

	struct pfn_array_table *pat;

	struct pfn_array *pa;

	u64 iova;

	unsigned long *idaws;

	int ret;

	int bytes = 1;

	int idaw_nr = 1;

	int idaw_nr, idal_len;

	int i;



	ccw = chain->ch_ccw + idx;



	if (ccw->count) {

	if (ccw->count)

		bytes = ccw->count;

		idaw_nr = idal_nr_words((void *)(u64)ccw->cda, ccw->count);

	}



	/*

	 * Pin data page(s) in memory.

	 * The number of pages actually is the count of the idaws which will be

	 * needed when translating a direct ccw to a idal ccw.

	 */

	pat = chain->ch_pat + idx;

	ret = pfn_array_table_init(pat, 1);

	/* Calculate size of IDAL */

	if (ccw_is_idal(ccw)) {

		/* Read first IDAW to see if it's 4K-aligned or not. */

		/* All subsequent IDAws will be 4K-aligned. */

		ret = copy_from_iova(cp->mdev, &iova, ccw->cda, sizeof(iova));

		if (ret)

		goto out_init;



	ret = pfn_array_alloc(pat->pat_pa, ccw->cda, bytes);

	if (ret < 0)

		goto out_unpin;



	if (ccw_does_data_transfer(ccw)) {

		ret = pfn_array_pin(pat->pat_pa, cp->mdev);

		if (ret < 0)

			goto out_unpin;

			return ret;

	} else {

		pat->pat_pa->pa_nr = 0;

		iova = ccw->cda;

	}

	idaw_nr = idal_nr_words((void *)iova, bytes);

	idal_len = idaw_nr * sizeof(*idaws);



	/* Translate this direct ccw to a idal ccw. */

	/* Allocate an IDAL from host storage */

	idaws = kcalloc(idaw_nr, sizeof(*idaws), GFP_DMA | GFP_KERNEL);

	if (!idaws) {

		ret = -ENOMEM;

		goto out_unpin;

	}

	ccw->cda = (__u32) virt_to_phys(idaws);

	ccw->flags |= CCW_FLAG_IDA;



	pfn_array_table_idal_create_words(pat, idaws);



	return 0;



out_unpin:

	pfn_array_table_unpin_free(pat, cp->mdev);

out_init:

	ccw->cda = 0;

	return ret;

}



static int ccwchain_fetch_idal(struct ccwchain *chain,

			       int idx,

			       struct channel_program *cp)

{

	struct ccw1 *ccw;

	struct pfn_array_table *pat;

	struct pfn_array *pa;

	unsigned long *idaws;

	u64 idaw_iova;

	unsigned int idaw_nr, idaw_len;

	int i, ret;

	int bytes = 1;



	ccw = chain->ch_ccw + idx;



	if (ccw->count)

		bytes = ccw->count;



	/* Calculate size of idaws. */

	ret = copy_from_iova(cp->mdev, &idaw_iova, ccw->cda, sizeof(idaw_iova));

	if (ret)

		return ret;

	idaw_nr = idal_nr_words((void *)(idaw_iova), bytes);

	idaw_len = idaw_nr * sizeof(*idaws);



	/* Pin data page(s) in memory. */

	pat = chain->ch_pat + idx;

	ret = pfn_array_table_init(pat, idaw_nr);

	if (ret)

		goto out_init;



	/* Translate idal ccw to use new allocated idaws. */

	idaws = kzalloc(idaw_len, GFP_DMA | GFP_KERNEL);

	if (!idaws) {

		ret = -ENOMEM;

		goto out_unpin;

	}



	ret = copy_from_iova(cp->mdev, idaws, ccw->cda, idaw_len);

	if (ret)

		goto out_free_idaws;



	ccw->cda = virt_to_phys(idaws);



	for (i = 0; i < idaw_nr; i++) {

		idaw_iova = *(idaws + i);

		pa = pat->pat_pa + i;



		ret = pfn_array_alloc(pa, idaw_iova, 1);

	/*

	 * Allocate an array of pfn's for pages to pin/translate.

	 * The number of pages is actually the count of the idaws

	 * required for the data transfer, since we only only support

	 * 4K IDAWs today.

	 */

	pa = chain->ch_pa + idx;

	ret = pfn_array_alloc(pa, iova, bytes);

	if (ret < 0)

		goto out_free_idaws;



		if (!ccw_does_data_transfer(ccw)) {

			pa->pa_nr = 0;

			continue;

	if (ccw_is_idal(ccw)) {

		/* Copy guest IDAL into host IDAL */

		ret = copy_from_iova(cp->mdev, idaws, ccw->cda, idal_len);

		if (ret)

			goto out_unpin;



		/*

		 * Copy guest IDAWs into pfn_array, in case the memory they

		 * occupy is not contiguous.

		 */

		for (i = 0; i < idaw_nr; i++)

			pa->pa_iova_pfn[i] = idaws[i] >> PAGE_SHIFT;

	} else {

		/*

		 * No action is required here; the iova addresses in pfn_array

		 * were initialized sequentially in pfn_array_alloc() beginning

		 * with the contents of ccw->cda.

		 */

	}



	if (ccw_does_data_transfer(ccw)) {

		ret = pfn_array_pin(pa, cp->mdev);

		if (ret < 0)

			goto out_free_idaws;

			goto out_unpin;

	} else {

		pa->pa_nr = 0;

	}



	pfn_array_table_idal_create_words(pat, idaws);

	ccw->cda = (__u32) virt_to_phys(idaws);

	ccw->flags |= CCW_FLAG_IDA;



	/* Populate the IDAL with pinned/translated addresses from pfn */

	pfn_array_idal_create_words(pa, idaws);



	return 0;



out_unpin:

	pfn_array_unpin_free(pa, cp->mdev);

out_free_idaws:

	kfree(idaws);

out_unpin:

	pfn_array_table_unpin_free(pat, cp->mdev);

out_init:

	ccw->cda = 0;

	return ret;
@@ -735,9 +605,6 @@ static int ccwchain_fetch_one(struct ccwchain *chain, 
	if (ccw_is_tic(ccw))

		return ccwchain_fetch_tic(chain, idx, cp);



	if (ccw_is_idal(ccw))

		return ccwchain_fetch_idal(chain, idx, cp);



	return ccwchain_fetch_direct(chain, idx, cp);

}
@@ -760,9 +627,7 @@ static int ccwchain_fetch_one(struct ccwchain *chain, 
 */

int cp_init(struct channel_program *cp, struct device *mdev, union orb *orb)

{

	u64 iova = orb->cmd.cpa;

	struct ccwchain *chain;

	int len, ret;

	int ret;



	/*

	 * XXX:
@@ -775,28 +640,11 @@ int cp_init(struct channel_program *cp, struct device *mdev, union orb *orb) 
	memcpy(&cp->orb, orb, sizeof(*orb));

	cp->mdev = mdev;



	/* Get chain length. */

	len = ccwchain_calc_length(iova, cp);

	if (len < 0)

		return len;



	/* Alloc mem for the head chain. */

	chain = ccwchain_alloc(cp, len);

	if (!chain)

		return -ENOMEM;

	chain->ch_iova = iova;



	/* Copy the head chain from guest. */

	ret = copy_ccw_from_iova(cp, chain->ch_ccw, iova, len);

	if (ret) {

		ccwchain_free(chain);

		return ret;

	}



	/* Now loop for its TICs. */

	ret = ccwchain_loop_tic(chain, cp);

	/* Build a ccwchain for the first CCW segment */

	ret = ccwchain_handle_ccw(orb->cmd.cpa, cp);

	if (ret)

		cp_unpin_free(cp);

		cp_free(cp);



	/* It is safe to force: if not set but idals used

	 * ccwchain_calc_length returns an error.

	 */
@@ -819,8 +667,20 @@ int cp_init(struct channel_program *cp, struct device *mdev, union orb *orb) 
 */

void cp_free(struct channel_program *cp)

{

	if (cp->initialized)

		cp_unpin_free(cp);

	struct ccwchain *chain, *temp;

	int i;



	if (!cp->initialized)

		return;



	cp->initialized = false;

	list_for_each_entry_safe(chain, temp, &cp->ccwchain_list, next) {

		for (i = 0; i < chain->ch_len; i++) {

			pfn_array_unpin_free(chain->ch_pa + i, cp->mdev);

			ccwchain_cda_free(chain, i);

		}

		ccwchain_free(chain);

	}

}



/**
@@ -992,8 +852,7 @@ bool cp_iova_pinned(struct channel_program *cp, u64 iova) 


	list_for_each_entry(chain, &cp->ccwchain_list, next) {

		for (i = 0; i < chain->ch_len; i++)

			if (pfn_array_table_iova_pinned(chain->ch_pat + i,

							iova))

			if (pfn_array_iova_pinned(chain->ch_pa + i, iova))

				return true;

	}

drivers/s390/cio/vfio_ccw_cp.h

+7 −0
Original line number Diff line number Diff line
@@ -16,6 +16,12 @@ 


#include "orb.h"



/*

 * Max length for ccw chain.

 * XXX: Limit to 256, need to check more?

 */

#define CCWCHAIN_LEN_MAX	256



/**

 * struct channel_program - manage information for channel program

 * @ccwchain_list: list head of ccwchains
@@ -32,6 +38,7 @@ struct channel_program { 
	union orb orb;

	struct device *mdev;

	bool initialized;

	struct ccw1 *guest_cp;

};



extern int cp_init(struct channel_program *cp, struct device *mdev,

drivers/s390/cio/vfio_ccw_drv.c

+7 −0
Original line number Diff line number Diff line
@@ -129,6 +129,11 @@ static int vfio_ccw_sch_probe(struct subchannel *sch) 
	if (!private)

		return -ENOMEM;



	private->cp.guest_cp = kcalloc(CCWCHAIN_LEN_MAX, sizeof(struct ccw1),

				       GFP_KERNEL);

	if (!private->cp.guest_cp)

		goto out_free;



	private->io_region = kmem_cache_zalloc(vfio_ccw_io_region,

					       GFP_KERNEL | GFP_DMA);

	if (!private->io_region)
@@ -169,6 +174,7 @@ out_free: 
		kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region);

	if (private->io_region)

		kmem_cache_free(vfio_ccw_io_region, private->io_region);

	kfree(private->cp.guest_cp);

	kfree(private);

	return ret;

}
@@ -185,6 +191,7 @@ static int vfio_ccw_sch_remove(struct subchannel *sch) 


	kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region);

	kmem_cache_free(vfio_ccw_io_region, private->io_region);

	kfree(private->cp.guest_cp);

	kfree(private);



	return 0;

CRA Git | Maintained and supported by SUSTech CRA and CCSE