Commit a540aa56 authored by Alexey Kardashevskiy's avatar Alexey Kardashevskiy Committed by Michael Ellerman
Browse files

powerpc/powernv/iommu: Add real mode version of iommu_table_ops::exchange() 



In real mode, TCE tables are invalidated using special
cache-inhibited store instructions which are not available in
virtual mode

This defines and implements exchange_rm() callback. This does not
define set_rm/clear_rm/flush_rm callbacks as there is no user for those -
exchange/exchange_rm are only to be used by KVM for VFIO.

The exchange_rm callback is defined for IODA1/IODA2 powernv platforms.

This replaces list_for_each_entry_rcu with its lockless version as
from now on pnv_pci_ioda2_tce_invalidate() can be called in
the real mode too.

Signed-off-by: default avatarAlexey Kardashevskiy <aik@ozlabs.ru>
Reviewed-by: default avatarDavid Gibson <david@gibson.dropbear.id.au>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
parent 6b5c19c5

arch/powerpc/include/asm/iommu.h

+7 −0
Original line number Diff line number Diff line
@@ -64,6 +64,11 @@ struct iommu_table_ops { 
			long index,

			unsigned long *hpa,

			enum dma_data_direction *direction);

	/* Real mode */

	int (*exchange_rm)(struct iommu_table *tbl,

			long index,

			unsigned long *hpa,

			enum dma_data_direction *direction);

#endif

	void (*clear)(struct iommu_table *tbl,

			long index, long npages);
@@ -208,6 +213,8 @@ extern void iommu_del_device(struct device *dev); 
extern int __init tce_iommu_bus_notifier_init(void);

extern long iommu_tce_xchg(struct iommu_table *tbl, unsigned long entry,

		unsigned long *hpa, enum dma_data_direction *direction);

extern long iommu_tce_xchg_rm(struct iommu_table *tbl, unsigned long entry,

		unsigned long *hpa, enum dma_data_direction *direction);

#else

static inline void iommu_register_group(struct iommu_table_group *table_group,

					int pci_domain_number,

arch/powerpc/kernel/iommu.c

+25 −0
Original line number Diff line number Diff line
@@ -1004,6 +1004,31 @@ long iommu_tce_xchg(struct iommu_table *tbl, unsigned long entry, 
}

EXPORT_SYMBOL_GPL(iommu_tce_xchg);



#ifdef CONFIG_PPC_BOOK3S_64

long iommu_tce_xchg_rm(struct iommu_table *tbl, unsigned long entry,

		unsigned long *hpa, enum dma_data_direction *direction)

{

	long ret;



	ret = tbl->it_ops->exchange_rm(tbl, entry, hpa, direction);



	if (!ret && ((*direction == DMA_FROM_DEVICE) ||

			(*direction == DMA_BIDIRECTIONAL))) {

		struct page *pg = realmode_pfn_to_page(*hpa >> PAGE_SHIFT);



		if (likely(pg)) {

			SetPageDirty(pg);

		} else {

			tbl->it_ops->exchange_rm(tbl, entry, hpa, direction);

			ret = -EFAULT;

		}

	}



	return ret;

}

EXPORT_SYMBOL_GPL(iommu_tce_xchg_rm);

#endif



int iommu_take_ownership(struct iommu_table *tbl)

{

	unsigned long flags, i, sz = (tbl->it_size + 7) >> 3;

arch/powerpc/platforms/powernv/pci-ioda.c

+25 −1
Original line number Diff line number Diff line
@@ -1860,6 +1860,17 @@ static int pnv_ioda1_tce_xchg(struct iommu_table *tbl, long index, 


	return ret;

}



static int pnv_ioda1_tce_xchg_rm(struct iommu_table *tbl, long index,

		unsigned long *hpa, enum dma_data_direction *direction)

{

	long ret = pnv_tce_xchg(tbl, index, hpa, direction);



	if (!ret)

		pnv_pci_p7ioc_tce_invalidate(tbl, index, 1, true);



	return ret;

}

#endif



static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index,
@@ -1874,6 +1885,7 @@ static struct iommu_table_ops pnv_ioda1_iommu_ops = { 
	.set = pnv_ioda1_tce_build,

#ifdef CONFIG_IOMMU_API

	.exchange = pnv_ioda1_tce_xchg,

	.exchange_rm = pnv_ioda1_tce_xchg_rm,

#endif

	.clear = pnv_ioda1_tce_free,

	.get = pnv_tce_get,
@@ -1948,7 +1960,7 @@ static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl, 
{

	struct iommu_table_group_link *tgl;



	list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) {

	list_for_each_entry_lockless(tgl, &tbl->it_group_list, next) {

		struct pnv_ioda_pe *pe = container_of(tgl->table_group,

				struct pnv_ioda_pe, table_group);

		struct pnv_phb *phb = pe->phb;
@@ -2004,6 +2016,17 @@ static int pnv_ioda2_tce_xchg(struct iommu_table *tbl, long index, 


	return ret;

}



static int pnv_ioda2_tce_xchg_rm(struct iommu_table *tbl, long index,

		unsigned long *hpa, enum dma_data_direction *direction)

{

	long ret = pnv_tce_xchg(tbl, index, hpa, direction);



	if (!ret)

		pnv_pci_ioda2_tce_invalidate(tbl, index, 1, true);



	return ret;

}

#endif



static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,
@@ -2024,6 +2047,7 @@ static struct iommu_table_ops pnv_ioda2_iommu_ops = { 
	.set = pnv_ioda2_tce_build,

#ifdef CONFIG_IOMMU_API

	.exchange = pnv_ioda2_tce_xchg,

	.exchange_rm = pnv_ioda2_tce_xchg_rm,

#endif

	.clear = pnv_ioda2_tce_free,

	.get = pnv_tce_get,

CRA Git | Maintained and supported by SUSTech CRA and CCSE