iommu/core: Create central IOMMU group lookup/creation interface

#include <linux/idr.h>

#include <linux/notifier.h>

#include <linux/err.h>

#include <linux/pci.h>

#include <trace/events/iommu.h>

static struct kset *iommu_group_kset;

}

EXPORT_SYMBOL_GPL(iommu_group_id);

/*

 * To consider a PCI device isolated, we require ACS to support Source

 * Validation, Request Redirection, Completer Redirection, and Upstream

 * Forwarding.  This effectively means that devices cannot spoof their

 * requester ID, requests and completions cannot be redirected, and all

 * transactions are forwarded upstream, even as it passes through a

 * bridge where the target device is downstream.

 */

#define REQ_ACS_FLAGS   (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)

struct group_for_pci_data {

	struct pci_dev *pdev;

	struct iommu_group *group;

};

/*

 * DMA alias iterator callback, return the last seen device.  Stop and return

 * the IOMMU group if we find one along the way.

 */

static int get_pci_alias_or_group(struct pci_dev *pdev, u16 alias, void *opaque)

{

	struct group_for_pci_data *data = opaque;

	data->pdev = pdev;

	data->group = iommu_group_get(&pdev->dev);

	return data->group != NULL;

}

/*

 * Use standard PCI bus topology, isolation features, and DMA alias quirks

 * to find or create an IOMMU group for a device.

 */

static struct iommu_group *iommu_group_get_for_pci_dev(struct pci_dev *pdev)

{

	struct group_for_pci_data data;

	struct pci_bus *bus;

	struct iommu_group *group = NULL;

	struct pci_dev *tmp;

	/*

	 * Find the upstream DMA alias for the device.  A device must not

	 * be aliased due to topology in order to have its own IOMMU group.

	 * If we find an alias along the way that already belongs to a

	 * group, use it.

	 */

	if (pci_for_each_dma_alias(pdev, get_pci_alias_or_group, &data))

		return data.group;

	pdev = data.pdev;

	/*

	 * Continue upstream from the point of minimum IOMMU granularity

	 * due to aliases to the point where devices are protected from

	 * peer-to-peer DMA by PCI ACS.  Again, if we find an existing

	 * group, use it.

	 */

	for (bus = pdev->bus; !pci_is_root_bus(bus); bus = bus->parent) {

		if (!bus->self)

			continue;

		if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))

			break;

		pdev = bus->self;

		group = iommu_group_get(&pdev->dev);

		if (group)

			return group;

	}

	/*

	 * Next we need to consider DMA alias quirks.  If one device aliases

	 * to another, they should be grouped together.  It's theoretically

	 * possible that aliases could create chains of devices where each

	 * device aliases another device.  If we then factor in multifunction

	 * ACS grouping requirements, each alias could incorporate a new slot

	 * with multiple functions, each with aliases.  This is all extremely

	 * unlikely as DMA alias quirks are typically only used for PCIe

	 * devices where we usually have a single slot per bus.  Furthermore,

	 * the alias quirk is usually to another function within the slot

	 * (and ACS multifunction is not supported) or to a different slot

	 * that doesn't physically exist.  The likely scenario is therefore

	 * that everything on the bus gets grouped together.  To reduce the

	 * problem space, share the IOMMU group for all devices on the bus

	 * if a DMA alias quirk is present on the bus.

	 */

	tmp = NULL;

	for_each_pci_dev(tmp) {

		if (tmp->bus != pdev->bus ||

		    !(tmp->dev_flags & PCI_DEV_FLAGS_DMA_ALIAS_DEVFN))

			continue;

		pci_dev_put(tmp);

		tmp = NULL;

		/* We have an alias quirk, search for an existing group */

		for_each_pci_dev(tmp) {

			struct iommu_group *group_tmp;

			if (tmp->bus != pdev->bus)

				continue;

			group_tmp = iommu_group_get(&tmp->dev);

			if (!group) {

				group = group_tmp;

				continue;

			}

			if (group_tmp) {

				WARN_ON(group != group_tmp);

				iommu_group_put(group_tmp);

			}

		}

		return group ? group : iommu_group_alloc();

	}

	/*

	 * Non-multifunction devices or multifunction devices supporting

	 * ACS get their own group.

	 */

	if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS))

		return iommu_group_alloc();

	/*

	 * Multifunction devices not supporting ACS share a group with other

	 * similar devices in the same slot.

	 */

	tmp = NULL;

	for_each_pci_dev(tmp) {

		if (tmp == pdev || tmp->bus != pdev->bus ||

		    PCI_SLOT(tmp->devfn) !=  PCI_SLOT(pdev->devfn) ||

		    pci_acs_enabled(tmp, REQ_ACS_FLAGS))

			continue;

		group = iommu_group_get(&tmp->dev);

		if (group) {

			pci_dev_put(tmp);

			return group;

		}

	}

	/* No shared group found, allocate new */

	return iommu_group_alloc();

}

/**

 * iommu_group_get_for_dev - Find or create the IOMMU group for a device

 * @dev: target device

 *

 * This function is intended to be called by IOMMU drivers and extended to

 * support common, bus-defined algorithms when determining or creating the

 * IOMMU group for a device.  On success, the caller will hold a reference

 * to the returned IOMMU group, which will already include the provided

 * device.  The reference should be released with iommu_group_put().

 */

struct iommu_group *iommu_group_get_for_dev(struct device *dev)

{

	struct iommu_group *group = ERR_PTR(-EIO);

	int ret;

	group = iommu_group_get(dev);

	if (group)

		return group;

	if (dev_is_pci(dev))

		group = iommu_group_get_for_pci_dev(to_pci_dev(dev));

	if (IS_ERR(group))

		return group;

	ret = iommu_group_add_device(group, dev);

	if (ret) {

		iommu_group_put(group);

		return ERR_PTR(ret);

	}

	return group;

}

static int add_iommu_group(struct device *dev, void *data)

{

	struct iommu_ops *ops = data;

extern int iommu_group_unregister_notifier(struct iommu_group *group,

					   struct notifier_block *nb);

extern int iommu_group_id(struct iommu_group *group);

extern struct iommu_group *iommu_group_get_for_dev(struct device *dev);

extern int iommu_domain_get_attr(struct iommu_domain *domain, enum iommu_attr,

				 void *data);