Merge tag 'iommu-updates-v3.16' of...

Samsung Exynos IOMMU H/W, System MMU (System Memory Management Unit)

Samsung's Exynos architecture contains System MMUs that enables scattered

physical memory chunks visible as a contiguous region to DMA-capable peripheral

devices like MFC, FIMC, FIMD, GScaler, FIMC-IS and so forth.

System MMU is an IOMMU and supports identical translation table format to

ARMv7 translation tables with minimum set of page properties including access

permissions, shareability and security protection. In addition, System MMU has

another capabilities like L2 TLB or block-fetch buffers to minimize translation

latency.

System MMUs are in many to one relation with peripheral devices, i.e. single

peripheral device might have multiple System MMUs (usually one for each bus

master), but one System MMU can handle transactions from only one peripheral

device. The relation between a System MMU and the peripheral device needs to be

defined in device node of the peripheral device.

MFC in all Exynos SoCs and FIMD, M2M Scalers and G2D in Exynos5420 has 2 System

MMUs.

* MFC has one System MMU on its left and right bus.

* FIMD in Exynos5420 has one System MMU for window 0 and 4, the other system MMU

  for window 1, 2 and 3.

* M2M Scalers and G2D in Exynos5420 has one System MMU on the read channel and

  the other System MMU on the write channel.

The drivers must consider how to handle those System MMUs. One of the idea is

to implement child devices or sub-devices which are the client devices of the

System MMU.

Note:

The current DT binding for the Exynos System MMU is incomplete.

The following properties can be removed or changed, if found incompatible with

the "Generic IOMMU Binding" support for attaching devices to the IOMMU.

Required properties:

- compatible: Should be "samsung,exynos-sysmmu"

- reg: A tuple of base address and size of System MMU registers.

- interrupt-parent: The phandle of the interrupt controller of System MMU

- interrupts: An interrupt specifier for interrupt signal of System MMU,

	      according to the format defined by a particular interrupt

	      controller.

- clock-names: Should be "sysmmu" if the System MMU is needed to gate its clock.

	       Optional "master" if the clock to the System MMU is gated by

	       another gate clock other than "sysmmu".

	       Exynos4 SoCs, there needs no "master" clock.

	       Exynos5 SoCs, some System MMUs must have "master" clocks.

- clocks: Required if the System MMU is needed to gate its clock.

- samsung,power-domain: Required if the System MMU is needed to gate its power.

	  Please refer to the following document:

	  Documentation/devicetree/bindings/arm/exynos/power_domain.txt

Examples:

	gsc_0: gsc@13e00000 {

		compatible = "samsung,exynos5-gsc";

		reg = <0x13e00000 0x1000>;

		interrupts = <0 85 0>;

		samsung,power-domain = <&pd_gsc>;

		clocks = <&clock CLK_GSCL0>;

		clock-names = "gscl";

	};

	sysmmu_gsc0: sysmmu@13E80000 {

		compatible = "samsung,exynos-sysmmu";

		reg = <0x13E80000 0x1000>;

		interrupt-parent = <&combiner>;

		interrupts = <2 0>;

		clock-names = "sysmmu", "master";

		clocks = <&clock CLK_SMMU_GSCL0>, <&clock CLK_GSCL0>;

		samsung,power-domain = <&pd_gsc>;

	};

config EXYNOS_IOMMU

	bool "Exynos IOMMU Support"

	depends on ARCH_EXYNOS && EXYNOS_DEV_SYSMMU

	depends on ARCH_EXYNOS

	select IOMMU_API

	help

	  Support for the IOMMU (System MMU) of Samsung Exynos application

	  processor family. This enables H/W multimedia accellerators to see

	  non-linear physical memory chunks as a linear memory in their

	  address spaces

	  processor family. This enables H/W multimedia accelerators to see

	  non-linear physical memory chunks as linear memory in their

	  address space.

	  If unsure, say N here.

	depends on EXYNOS_IOMMU

	help

	  Select this to see the detailed log message that shows what

	  happens in the IOMMU driver

	  happens in the IOMMU driver.

	  Say N unless you need kernel log message for IOMMU debugging

	  Say N unless you need kernel log message for IOMMU debugging.

config SHMOBILE_IPMMU

	bool

	default 256 if SHMOBILE_IOMMU_ADDRSIZE_64MB

	default 128 if SHMOBILE_IOMMU_ADDRSIZE_32MB

config IPMMU_VMSA

	bool "Renesas VMSA-compatible IPMMU"

	depends on ARM_LPAE

	depends on ARCH_SHMOBILE || COMPILE_TEST

	select IOMMU_API

	select ARM_DMA_USE_IOMMU

	help

	  Support for the Renesas VMSA-compatible IPMMU Renesas found in the

	  R-Mobile APE6 and R-Car H2/M2 SoCs.

	  If unsure, say N.

config SPAPR_TCE_IOMMU

	bool "sPAPR TCE IOMMU Support"

	depends on PPC_POWERNV || PPC_PSERIES

obj-$(CONFIG_ARM_SMMU) += arm-smmu.o

obj-$(CONFIG_DMAR_TABLE) += dmar.o

obj-$(CONFIG_INTEL_IOMMU) += iova.o intel-iommu.o

obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o

obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o

obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o

obj-$(CONFIG_OMAP_IOMMU) += omap-iommu2.o

{

	struct iommu_dev_data *dev_data;

	struct pci_dev *dev = NULL;

	struct amd_iommu *iommu;

	u16 devid;

	int ret;

	ret = alloc_passthrough_domain();

		dev_data = get_dev_data(&dev->dev);

		dev_data->passthrough = true;

		devid = get_device_id(&dev->dev);

		iommu = amd_iommu_rlookup_table[devid];

		if (!iommu)

			continue;

		attach_device(&dev->dev, pt_domain);

	}

struct pasid_state {

	struct list_head list;			/* For global state-list */

	atomic_t count;				/* Reference count */

	atomic_t mmu_notifier_count;		/* Counting nested mmu_notifier

						   calls */

	struct task_struct *task;		/* Task bound to this PASID */

	struct mm_struct *mm;			/* mm_struct for the faults */

	struct mmu_notifier mn;                 /* mmu_otifier handle */

};

struct device_state {

	struct list_head list;

	u16 devid;

	atomic_t count;

	struct pci_dev *pdev;

	struct pasid_state **states;

	u16 flags;

};

static struct device_state **state_table;

static LIST_HEAD(state_list);

static spinlock_t state_lock;

/* List and lock for all pasid_states */

static LIST_HEAD(pasid_state_list);

static DEFINE_SPINLOCK(ps_lock);

static struct workqueue_struct *iommu_wq;

/*

static void free_pasid_states(struct device_state *dev_state);

static void unbind_pasid(struct device_state *dev_state, int pasid);

static int task_exit(struct notifier_block *nb, unsigned long e, void *data);

static u16 device_id(struct pci_dev *pdev)

{

	return devid;

}

static struct device_state *__get_device_state(u16 devid)

{

	struct device_state *dev_state;

	list_for_each_entry(dev_state, &state_list, list) {

		if (dev_state->devid == devid)

			return dev_state;

	}

	return NULL;

}

static struct device_state *get_device_state(u16 devid)

{

	struct device_state *dev_state;

	unsigned long flags;

	spin_lock_irqsave(&state_lock, flags);

	dev_state = state_table[devid];

	dev_state = __get_device_state(devid);

	if (dev_state != NULL)

		atomic_inc(&dev_state->count);

	spin_unlock_irqrestore(&state_lock, flags);

	free_device_state(dev_state);

}

static struct notifier_block profile_nb = {

	.notifier_call = task_exit,

};

static void link_pasid_state(struct pasid_state *pasid_state)

{

	spin_lock(&ps_lock);

	list_add_tail(&pasid_state->list, &pasid_state_list);

	spin_unlock(&ps_lock);

}

static void __unlink_pasid_state(struct pasid_state *pasid_state)

{

	list_del(&pasid_state->list);

}

static void unlink_pasid_state(struct pasid_state *pasid_state)

{

	spin_lock(&ps_lock);

	__unlink_pasid_state(pasid_state);

	spin_unlock(&ps_lock);

}

/* Must be called under dev_state->lock */

static struct pasid_state **__get_pasid_state_ptr(struct device_state *dev_state,

						  int pasid, bool alloc)

	if (pasid_state == NULL)

		return;

	unlink_pasid_state(pasid_state);

	__unbind_pasid(pasid_state);

	put_pasid_state_wait(pasid_state); /* Reference taken in this function */

}

			continue;

		put_pasid_state(pasid_state);

		unbind_pasid(dev_state, i);

		/*

		 * This will call the mn_release function and

		 * unbind the PASID

		 */

		mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);

	}

	if (dev_state->pasid_levels == 2)

	pasid_state = mn_to_state(mn);

	dev_state   = pasid_state->device_state;

	amd_iommu_domain_set_gcr3(dev_state->domain, pasid_state->pasid,

	if (atomic_add_return(1, &pasid_state->mmu_notifier_count) == 1) {

		amd_iommu_domain_set_gcr3(dev_state->domain,

					  pasid_state->pasid,

					  __pa(empty_page_table));

	}

}

static void mn_invalidate_range_end(struct mmu_notifier *mn,

				    struct mm_struct *mm,

	pasid_state = mn_to_state(mn);

	dev_state   = pasid_state->device_state;

	amd_iommu_domain_set_gcr3(dev_state->domain, pasid_state->pasid,

	if (atomic_dec_and_test(&pasid_state->mmu_notifier_count)) {

		amd_iommu_domain_set_gcr3(dev_state->domain,

					  pasid_state->pasid,

					  __pa(pasid_state->mm->pgd));

	}

}

static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm)

{

	struct pasid_state *pasid_state;

	struct device_state *dev_state;

	might_sleep();

	pasid_state = mn_to_state(mn);

	dev_state   = pasid_state->device_state;

	if (pasid_state->device_state->inv_ctx_cb)

		dev_state->inv_ctx_cb(dev_state->pdev, pasid_state->pasid);

	unbind_pasid(dev_state, pasid_state->pasid);

}

static struct mmu_notifier_ops iommu_mn = {

	.release		= mn_release,

	.clear_flush_young      = mn_clear_flush_young,

	.change_pte             = mn_change_pte,

	.invalidate_page        = mn_invalidate_page,

	.notifier_call = ppr_notifier,

};

static int task_exit(struct notifier_block *nb, unsigned long e, void *data)

{

	struct pasid_state *pasid_state;

	struct task_struct *task;

	task = data;

	/*

	 * Using this notifier is a hack - but there is no other choice

	 * at the moment. What I really want is a sleeping notifier that

	 * is called when an MM goes down. But such a notifier doesn't

	 * exist yet. The notifier needs to sleep because it has to make

	 * sure that the device does not use the PASID and the address

	 * space anymore before it is destroyed. This includes waiting

	 * for pending PRI requests to pass the workqueue. The

	 * MMU-Notifiers would be a good fit, but they use RCU and so

	 * they are not allowed to sleep. Lets see how we can solve this

	 * in a more intelligent way in the future.

	 */

again:

	spin_lock(&ps_lock);

	list_for_each_entry(pasid_state, &pasid_state_list, list) {

		struct device_state *dev_state;

		int pasid;

		if (pasid_state->task != task)

			continue;

		/* Drop Lock and unbind */

		spin_unlock(&ps_lock);

		dev_state = pasid_state->device_state;

		pasid     = pasid_state->pasid;

		if (pasid_state->device_state->inv_ctx_cb)

			dev_state->inv_ctx_cb(dev_state->pdev, pasid);

		unbind_pasid(dev_state, pasid);

		/* Task may be in the list multiple times */

		goto again;

	}

	spin_unlock(&ps_lock);

	return NOTIFY_OK;

}

int amd_iommu_bind_pasid(struct pci_dev *pdev, int pasid,

			 struct task_struct *task)

{

		goto out;

	atomic_set(&pasid_state->count, 1);

	atomic_set(&pasid_state->mmu_notifier_count, 0);

	init_waitqueue_head(&pasid_state->wq);

	spin_lock_init(&pasid_state->lock);

	if (ret)

		goto out_clear_state;

	link_pasid_state(pasid_state);

	return 0;

out_clear_state:

void amd_iommu_unbind_pasid(struct pci_dev *pdev, int pasid)

{

	struct pasid_state *pasid_state;

	struct device_state *dev_state;

	u16 devid;

	if (pasid < 0 || pasid >= dev_state->max_pasids)

		goto out;

	unbind_pasid(dev_state, pasid);

	pasid_state = get_pasid_state(dev_state, pasid);

	if (pasid_state == NULL)

		goto out;

	/*

	 * Drop reference taken here. We are safe because we still hold

	 * the reference taken in the amd_iommu_bind_pasid function.

	 */

	put_pasid_state(pasid_state);

	/* This will call the mn_release function and unbind the PASID */

	mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);

out:

	put_device_state(dev_state);

	spin_lock_init(&dev_state->lock);

	init_waitqueue_head(&dev_state->wq);

	dev_state->pdev  = pdev;

	dev_state->devid = devid;

	tmp = pasids;

	for (dev_state->pasid_levels = 0; (tmp - 1) & ~0x1ff; tmp >>= 9)

	spin_lock_irqsave(&state_lock, flags);

	if (state_table[devid] != NULL) {

	if (__get_device_state(devid) != NULL) {

		spin_unlock_irqrestore(&state_lock, flags);

		ret = -EBUSY;

		goto out_free_domain;

	}

	state_table[devid] = dev_state;

	list_add_tail(&dev_state->list, &state_list);

	spin_unlock_irqrestore(&state_lock, flags);

	spin_lock_irqsave(&state_lock, flags);

	dev_state = state_table[devid];

	dev_state = __get_device_state(devid);

	if (dev_state == NULL) {

		spin_unlock_irqrestore(&state_lock, flags);

		return;

	}

	state_table[devid] = NULL;

	list_del(&dev_state->list);

	spin_unlock_irqrestore(&state_lock, flags);

	spin_lock_irqsave(&state_lock, flags);

	ret = -EINVAL;

	dev_state = state_table[devid];

	dev_state = __get_device_state(devid);

	if (dev_state == NULL)

		goto out_unlock;

	spin_lock_irqsave(&state_lock, flags);

	ret = -EINVAL;

	dev_state = state_table[devid];

	dev_state = __get_device_state(devid);

	if (dev_state == NULL)

		goto out_unlock;

static int __init amd_iommu_v2_init(void)

{

	size_t state_table_size;

	int ret;

	pr_info("AMD IOMMUv2 driver by Joerg Roedel <joerg.roedel@amd.com>\n");

	spin_lock_init(&state_lock);

	state_table_size = MAX_DEVICES * sizeof(struct device_state *);

	state_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,

					       get_order(state_table_size));

	if (state_table == NULL)

		return -ENOMEM;

	ret = -ENOMEM;

	iommu_wq = create_workqueue("amd_iommu_v2");

	if (iommu_wq == NULL)

		goto out_free;

		goto out;

	ret = -ENOMEM;

	empty_page_table = (u64 *)get_zeroed_page(GFP_KERNEL);

		goto out_destroy_wq;

	amd_iommu_register_ppr_notifier(&ppr_nb);

	profile_event_register(PROFILE_TASK_EXIT, &profile_nb);

	return 0;

out_destroy_wq:

	destroy_workqueue(iommu_wq);

out_free:

	free_pages((unsigned long)state_table, get_order(state_table_size));

out:

	return ret;

}

static void __exit amd_iommu_v2_exit(void)

{

	struct device_state *dev_state;

	size_t state_table_size;

	int i;

	if (!amd_iommu_v2_supported())

		return;

	profile_event_unregister(PROFILE_TASK_EXIT, &profile_nb);

	amd_iommu_unregister_ppr_notifier(&ppr_nb);

	flush_workqueue(iommu_wq);

	destroy_workqueue(iommu_wq);

	state_table_size = MAX_DEVICES * sizeof(struct device_state *);

	free_pages((unsigned long)state_table, get_order(state_table_size));

	free_page((unsigned long)empty_page_table);

}