Commit 4e13c1ac authored by Alexey Kardashevskiy's avatar Alexey Kardashevskiy Committed by Benjamin Herrenschmidt
Browse files

powerpc/vfio: Enable on PowerNV platform 



This initializes IOMMU groups based on the IOMMU configuration
discovered during the PCI scan on POWERNV (POWER non virtualized)
platform.  The IOMMU groups are to be used later by the VFIO driver,
which is used for PCI pass through.

It also implements an API for mapping/unmapping pages for
guest PCI drivers and providing DMA window properties.
This API is going to be used later by QEMU-VFIO to handle
h_put_tce hypercalls from the KVM guest.

The iommu_put_tce_user_mode() does only a single page mapping
as an API for adding many mappings at once is going to be
added later.

Although this driver has been tested only on the POWERNV
platform, it should work on any platform which supports
TCE tables.  As h_put_tce hypercall is received by the host
kernel and processed by the QEMU (what involves calling
the host kernel again), performance is not the best -
circa 220MB/s on 10Gb ethernet network.

To enable VFIO on POWER, enable SPAPR_TCE_IOMMU config
option and configure VFIO as required.

Cc: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: default avatarAlexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: default avatarPaul Mackerras <paulus@samba.org>
Signed-off-by: default avatarBenjamin Herrenschmidt <benh@kernel.crashing.org>
parent ab9a4183

arch/powerpc/include/asm/iommu.h

+26 −0
Original line number Diff line number Diff line
@@ -76,6 +76,9 @@ struct iommu_table { 
	struct iommu_pool large_pool;

	struct iommu_pool pools[IOMMU_NR_POOLS];

	unsigned long *it_map;       /* A simple allocation bitmap for now */

#ifdef CONFIG_IOMMU_API

	struct iommu_group *it_group;

#endif

};



struct scatterlist;
@@ -98,6 +101,8 @@ extern void iommu_free_table(struct iommu_table *tbl, const char *node_name); 
 */

extern struct iommu_table *iommu_init_table(struct iommu_table * tbl,

					    int nid);

extern void iommu_register_group(struct iommu_table *tbl,

				 int pci_domain_number, unsigned long pe_num);



extern int iommu_map_sg(struct device *dev, struct iommu_table *tbl,

			struct scatterlist *sglist, int nelems,
@@ -147,5 +152,26 @@ static inline void iommu_restore(void) 
}

#endif



/* The API to support IOMMU operations for VFIO */

extern int iommu_tce_clear_param_check(struct iommu_table *tbl,

		unsigned long ioba, unsigned long tce_value,

		unsigned long npages);

extern int iommu_tce_put_param_check(struct iommu_table *tbl,

		unsigned long ioba, unsigned long tce);

extern int iommu_tce_build(struct iommu_table *tbl, unsigned long entry,

		unsigned long hwaddr, enum dma_data_direction direction);

extern unsigned long iommu_clear_tce(struct iommu_table *tbl,

		unsigned long entry);

extern int iommu_clear_tces_and_put_pages(struct iommu_table *tbl,

		unsigned long entry, unsigned long pages);

extern int iommu_put_tce_user_mode(struct iommu_table *tbl,

		unsigned long entry, unsigned long tce);



extern void iommu_flush_tce(struct iommu_table *tbl);

extern int iommu_take_ownership(struct iommu_table *tbl);

extern void iommu_release_ownership(struct iommu_table *tbl);



extern enum dma_data_direction iommu_tce_direction(unsigned long tce);



#endif /* __KERNEL__ */

#endif /* _ASM_IOMMU_H */

arch/powerpc/kernel/iommu.c

+323 −0
Original line number Diff line number Diff line
@@ -36,6 +36,8 @@ 
#include <linux/hash.h>

#include <linux/fault-inject.h>

#include <linux/pci.h>

#include <linux/iommu.h>

#include <linux/sched.h>

#include <asm/io.h>

#include <asm/prom.h>

#include <asm/iommu.h>
@@ -44,6 +46,7 @@ 
#include <asm/kdump.h>

#include <asm/fadump.h>

#include <asm/vio.h>

#include <asm/tce.h>



#define DBG(...)
@@ -724,6 +727,13 @@ void iommu_free_table(struct iommu_table *tbl, const char *node_name) 
	if (tbl->it_offset == 0)

		clear_bit(0, tbl->it_map);



#ifdef CONFIG_IOMMU_API

	if (tbl->it_group) {

		iommu_group_put(tbl->it_group);

		BUG_ON(tbl->it_group);

	}

#endif



	/* verify that table contains no entries */

	if (!bitmap_empty(tbl->it_map, tbl->it_size))

		pr_warn("%s: Unexpected TCEs for %s\n", __func__, node_name);
@@ -860,3 +870,316 @@ void iommu_free_coherent(struct iommu_table *tbl, size_t size, 
		free_pages((unsigned long)vaddr, get_order(size));

	}

}



#ifdef CONFIG_IOMMU_API

/*

 * SPAPR TCE API

 */

static void group_release(void *iommu_data)

{

	struct iommu_table *tbl = iommu_data;

	tbl->it_group = NULL;

}



void iommu_register_group(struct iommu_table *tbl,

		int pci_domain_number, unsigned long pe_num)

{

	struct iommu_group *grp;

	char *name;



	grp = iommu_group_alloc();

	if (IS_ERR(grp)) {

		pr_warn("powerpc iommu api: cannot create new group, err=%ld\n",

				PTR_ERR(grp));

		return;

	}

	tbl->it_group = grp;

	iommu_group_set_iommudata(grp, tbl, group_release);

	name = kasprintf(GFP_KERNEL, "domain%d-pe%lx",

			pci_domain_number, pe_num);

	if (!name)

		return;

	iommu_group_set_name(grp, name);

	kfree(name);

}



enum dma_data_direction iommu_tce_direction(unsigned long tce)

{

	if ((tce & TCE_PCI_READ) && (tce & TCE_PCI_WRITE))

		return DMA_BIDIRECTIONAL;

	else if (tce & TCE_PCI_READ)

		return DMA_TO_DEVICE;

	else if (tce & TCE_PCI_WRITE)

		return DMA_FROM_DEVICE;

	else

		return DMA_NONE;

}

EXPORT_SYMBOL_GPL(iommu_tce_direction);



void iommu_flush_tce(struct iommu_table *tbl)

{

	/* Flush/invalidate TLB caches if necessary */

	if (ppc_md.tce_flush)

		ppc_md.tce_flush(tbl);



	/* Make sure updates are seen by hardware */

	mb();

}

EXPORT_SYMBOL_GPL(iommu_flush_tce);



int iommu_tce_clear_param_check(struct iommu_table *tbl,

		unsigned long ioba, unsigned long tce_value,

		unsigned long npages)

{

	/* ppc_md.tce_free() does not support any value but 0 */

	if (tce_value)

		return -EINVAL;



	if (ioba & ~IOMMU_PAGE_MASK)

		return -EINVAL;



	ioba >>= IOMMU_PAGE_SHIFT;

	if (ioba < tbl->it_offset)

		return -EINVAL;



	if ((ioba + npages) > (tbl->it_offset + tbl->it_size))

		return -EINVAL;



	return 0;

}

EXPORT_SYMBOL_GPL(iommu_tce_clear_param_check);



int iommu_tce_put_param_check(struct iommu_table *tbl,

		unsigned long ioba, unsigned long tce)

{

	if (!(tce & (TCE_PCI_WRITE | TCE_PCI_READ)))

		return -EINVAL;



	if (tce & ~(IOMMU_PAGE_MASK | TCE_PCI_WRITE | TCE_PCI_READ))

		return -EINVAL;



	if (ioba & ~IOMMU_PAGE_MASK)

		return -EINVAL;



	ioba >>= IOMMU_PAGE_SHIFT;

	if (ioba < tbl->it_offset)

		return -EINVAL;



	if ((ioba + 1) > (tbl->it_offset + tbl->it_size))

		return -EINVAL;



	return 0;

}

EXPORT_SYMBOL_GPL(iommu_tce_put_param_check);



unsigned long iommu_clear_tce(struct iommu_table *tbl, unsigned long entry)

{

	unsigned long oldtce;

	struct iommu_pool *pool = get_pool(tbl, entry);



	spin_lock(&(pool->lock));



	oldtce = ppc_md.tce_get(tbl, entry);

	if (oldtce & (TCE_PCI_WRITE | TCE_PCI_READ))

		ppc_md.tce_free(tbl, entry, 1);

	else

		oldtce = 0;



	spin_unlock(&(pool->lock));



	return oldtce;

}

EXPORT_SYMBOL_GPL(iommu_clear_tce);



int iommu_clear_tces_and_put_pages(struct iommu_table *tbl,

		unsigned long entry, unsigned long pages)

{

	unsigned long oldtce;

	struct page *page;



	for ( ; pages; --pages, ++entry) {

		oldtce = iommu_clear_tce(tbl, entry);

		if (!oldtce)

			continue;



		page = pfn_to_page(oldtce >> PAGE_SHIFT);

		WARN_ON(!page);

		if (page) {

			if (oldtce & TCE_PCI_WRITE)

				SetPageDirty(page);

			put_page(page);

		}

	}



	return 0;

}

EXPORT_SYMBOL_GPL(iommu_clear_tces_and_put_pages);



/*

 * hwaddr is a kernel virtual address here (0xc... bazillion),

 * tce_build converts it to a physical address.

 */

int iommu_tce_build(struct iommu_table *tbl, unsigned long entry,

		unsigned long hwaddr, enum dma_data_direction direction)

{

	int ret = -EBUSY;

	unsigned long oldtce;

	struct iommu_pool *pool = get_pool(tbl, entry);



	spin_lock(&(pool->lock));



	oldtce = ppc_md.tce_get(tbl, entry);

	/* Add new entry if it is not busy */

	if (!(oldtce & (TCE_PCI_WRITE | TCE_PCI_READ)))

		ret = ppc_md.tce_build(tbl, entry, 1, hwaddr, direction, NULL);



	spin_unlock(&(pool->lock));



	/* if (unlikely(ret))

		pr_err("iommu_tce: %s failed on hwaddr=%lx ioba=%lx kva=%lx ret=%d\n",

				__func__, hwaddr, entry << IOMMU_PAGE_SHIFT,

				hwaddr, ret); */



	return ret;

}

EXPORT_SYMBOL_GPL(iommu_tce_build);



int iommu_put_tce_user_mode(struct iommu_table *tbl, unsigned long entry,

		unsigned long tce)

{

	int ret;

	struct page *page = NULL;

	unsigned long hwaddr, offset = tce & IOMMU_PAGE_MASK & ~PAGE_MASK;

	enum dma_data_direction direction = iommu_tce_direction(tce);



	ret = get_user_pages_fast(tce & PAGE_MASK, 1,

			direction != DMA_TO_DEVICE, &page);

	if (unlikely(ret != 1)) {

		/* pr_err("iommu_tce: get_user_pages_fast failed tce=%lx ioba=%lx ret=%d\n",

				tce, entry << IOMMU_PAGE_SHIFT, ret); */

		return -EFAULT;

	}

	hwaddr = (unsigned long) page_address(page) + offset;



	ret = iommu_tce_build(tbl, entry, hwaddr, direction);

	if (ret)

		put_page(page);



	if (ret < 0)

		pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%d\n",

				__func__, entry << IOMMU_PAGE_SHIFT, tce, ret);



	return ret;

}

EXPORT_SYMBOL_GPL(iommu_put_tce_user_mode);



int iommu_take_ownership(struct iommu_table *tbl)

{

	unsigned long sz = (tbl->it_size + 7) >> 3;



	if (tbl->it_offset == 0)

		clear_bit(0, tbl->it_map);



	if (!bitmap_empty(tbl->it_map, tbl->it_size)) {

		pr_err("iommu_tce: it_map is not empty");

		return -EBUSY;

	}



	memset(tbl->it_map, 0xff, sz);

	iommu_clear_tces_and_put_pages(tbl, tbl->it_offset, tbl->it_size);



	return 0;

}

EXPORT_SYMBOL_GPL(iommu_take_ownership);



void iommu_release_ownership(struct iommu_table *tbl)

{

	unsigned long sz = (tbl->it_size + 7) >> 3;



	iommu_clear_tces_and_put_pages(tbl, tbl->it_offset, tbl->it_size);

	memset(tbl->it_map, 0, sz);



	/* Restore bit#0 set by iommu_init_table() */

	if (tbl->it_offset == 0)

		set_bit(0, tbl->it_map);

}

EXPORT_SYMBOL_GPL(iommu_release_ownership);



static int iommu_add_device(struct device *dev)

{

	struct iommu_table *tbl;

	int ret = 0;



	if (WARN_ON(dev->iommu_group)) {

		pr_warn("iommu_tce: device %s is already in iommu group %d, skipping\n",

				dev_name(dev),

				iommu_group_id(dev->iommu_group));

		return -EBUSY;

	}



	tbl = get_iommu_table_base(dev);

	if (!tbl || !tbl->it_group) {

		pr_debug("iommu_tce: skipping device %s with no tbl\n",

				dev_name(dev));

		return 0;

	}



	pr_debug("iommu_tce: adding %s to iommu group %d\n",

			dev_name(dev), iommu_group_id(tbl->it_group));



	ret = iommu_group_add_device(tbl->it_group, dev);

	if (ret < 0)

		pr_err("iommu_tce: %s has not been added, ret=%d\n",

				dev_name(dev), ret);



	return ret;

}



static void iommu_del_device(struct device *dev)

{

	iommu_group_remove_device(dev);

}



static int iommu_bus_notifier(struct notifier_block *nb,

			      unsigned long action, void *data)

{

	struct device *dev = data;



	switch (action) {

	case BUS_NOTIFY_ADD_DEVICE:

		return iommu_add_device(dev);

	case BUS_NOTIFY_DEL_DEVICE:

		iommu_del_device(dev);

		return 0;

	default:

		return 0;

	}

}



static struct notifier_block tce_iommu_bus_nb = {

	.notifier_call = iommu_bus_notifier,

};



static int __init tce_iommu_init(void)

{

	struct pci_dev *pdev = NULL;



	BUILD_BUG_ON(PAGE_SIZE < IOMMU_PAGE_SIZE);



	for_each_pci_dev(pdev)

		iommu_add_device(&pdev->dev);



	bus_register_notifier(&pci_bus_type, &tce_iommu_bus_nb);

	return 0;

}



subsys_initcall_sync(tce_iommu_init);



#else



void iommu_register_group(struct iommu_table *tbl,

		int pci_domain_number, unsigned long pe_num)

{

}



#endif /* CONFIG_IOMMU_API */

arch/powerpc/platforms/powernv/pci-ioda.c

+1 −0
Original line number Diff line number Diff line
@@ -595,6 +595,7 @@ static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb, 
			       TCE_PCI_SWINV_PAIR;

	}

	iommu_init_table(tbl, phb->hose->node);

	iommu_register_group(tbl, pci_domain_nr(pe->pbus), pe->pe_number);



	return;

 fail:

arch/powerpc/platforms/powernv/pci-p5ioc2.c

+4 −1
Original line number Diff line number Diff line
@@ -86,8 +86,11 @@ static void pnv_pci_init_p5ioc2_msis(struct pnv_phb *phb) { } 
static void pnv_pci_p5ioc2_dma_dev_setup(struct pnv_phb *phb,

					 struct pci_dev *pdev)

{

	if (phb->p5ioc2.iommu_table.it_map == NULL)

	if (phb->p5ioc2.iommu_table.it_map == NULL) {

		iommu_init_table(&phb->p5ioc2.iommu_table, phb->hose->node);

		iommu_register_group(&phb->p5ioc2.iommu_table,

				pci_domain_nr(phb->hose->bus), phb->opal_id);

	}



	set_iommu_table_base(&pdev->dev, &phb->p5ioc2.iommu_table);

}

arch/powerpc/platforms/powernv/pci.c

+2 −0
Original line number Diff line number Diff line
@@ -20,6 +20,7 @@ 
#include <linux/irq.h>

#include <linux/io.h>

#include <linux/msi.h>

#include <linux/iommu.h>



#include <asm/sections.h>

#include <asm/io.h>
@@ -412,6 +413,7 @@ static struct iommu_table *pnv_pci_setup_bml_iommu(struct pci_controller *hose) 
	pnv_pci_setup_iommu_table(tbl, __va(be64_to_cpup(basep)),

				  be32_to_cpup(sizep), 0);

	iommu_init_table(tbl, hose->node);

	iommu_register_group(tbl, pci_domain_nr(hose->bus), 0);



	/* Deal with SW invalidated TCEs when needed (BML way) */

	swinvp = of_get_property(hose->dn, "linux,tce-sw-invalidate-info",

CRA Git | Maintained and supported by SUSTech CRA and CCSE