Merge tag 'vfio-v4.17-rc1' of git://github.com/awilliam/linux-vfio (f605ba97) · Commits · 戴 / test

MAINTAINERS

+1 −1

Original line number	Diff line number	Diff line
		@@ -14797,7 +14797,7 @@ F: include/linux/mdev.h
		F: samples/vfio-mdev/

		VFIO PLATFORM DRIVER
		M: Baptiste Reynal <b.reynal@virtualopensystems.com>
		M: Eric Auger <eric.auger@redhat.com>
		L: kvm@vger.kernel.org
		S: Maintained
		F: drivers/vfio/platform/

drivers/vfio/pci/vfio_pci.c

+35 −0

Original line number	Diff line number	Diff line
		@@ -302,6 +302,7 @@ static void vfio_pci_disable(struct vfio_pci_device *vdev)
		{
		struct pci_dev *pdev = vdev->pdev;
		struct vfio_pci_dummy_resource dummy_res, tmp;
		struct vfio_pci_ioeventfd ioeventfd, ioeventfd_tmp;
		int i, bar;

		/* Stop the device from further DMA */
		@@ -311,6 +312,15 @@ static void vfio_pci_disable(struct vfio_pci_device *vdev)
		VFIO_IRQ_SET_ACTION_TRIGGER,
		vdev->irq_type, 0, 0, NULL);

		/* Device closed, don't need mutex here */
		list_for_each_entry_safe(ioeventfd, ioeventfd_tmp,
		&vdev->ioeventfds_list, next) {
		vfio_virqfd_disable(&ioeventfd->virqfd);
		list_del(&ioeventfd->next);
		kfree(ioeventfd);
		}
		vdev->ioeventfds_nr = 0;

		vdev->virq_disabled = false;

		for (i = 0; i < vdev->num_regions; i++)
		@@ -1009,6 +1019,28 @@ hot_reset_release:

		kfree(groups);
		return ret;
		} else if (cmd == VFIO_DEVICE_IOEVENTFD) {
		struct vfio_device_ioeventfd ioeventfd;
		int count;

		minsz = offsetofend(struct vfio_device_ioeventfd, fd);

		if (copy_from_user(&ioeventfd, (void __user *)arg, minsz))
		return -EFAULT;

		if (ioeventfd.argsz < minsz)
		return -EINVAL;

		if (ioeventfd.flags & ~VFIO_DEVICE_IOEVENTFD_SIZE_MASK)
		return -EINVAL;

		count = ioeventfd.flags & VFIO_DEVICE_IOEVENTFD_SIZE_MASK;

		if (hweight8(count) != 1 \|\| ioeventfd.fd < -1)
		return -EINVAL;

		return vfio_pci_ioeventfd(vdev, ioeventfd.offset,
		ioeventfd.data, count, ioeventfd.fd);
		}

		return -ENOTTY;
		@@ -1171,6 +1203,8 @@ static int vfio_pci_probe(struct pci_dev pdev, const struct pci_device_id id)
		vdev->irq_type = VFIO_PCI_NUM_IRQS;
		mutex_init(&vdev->igate);
		spin_lock_init(&vdev->irqlock);
		mutex_init(&vdev->ioeventfds_lock);
		INIT_LIST_HEAD(&vdev->ioeventfds_list);

		ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev);
		if (ret) {
		@@ -1212,6 +1246,7 @@ static void vfio_pci_remove(struct pci_dev *pdev)

		vfio_iommu_group_put(pdev->dev.iommu_group, &pdev->dev);
		kfree(vdev->region);
		mutex_destroy(&vdev->ioeventfds_lock);
		kfree(vdev);

		if (vfio_pci_is_vga(pdev)) {

drivers/vfio/pci/vfio_pci_private.h

+19 −0

Original line number	Diff line number	Diff line
		@@ -29,6 +29,19 @@
		#define PCI_CAP_ID_INVALID 0xFF /* default raw access */
		#define PCI_CAP_ID_INVALID_VIRT 0xFE /* default virt access */

		/* Cap maximum number of ioeventfds per device (arbitrary) */
		#define VFIO_PCI_IOEVENTFD_MAX 1000

		struct vfio_pci_ioeventfd {
		struct list_head next;
		struct virqfd *virqfd;
		void __iomem *addr;
		uint64_t data;
		loff_t pos;
		int bar;
		int count;
		};

		struct vfio_pci_irq_ctx {
		struct eventfd_ctx *trigger;
		struct virqfd *unmask;
		@@ -92,9 +105,12 @@ struct vfio_pci_device {
		bool nointx;
		struct pci_saved_state *pci_saved_state;
		int refcnt;
		int ioeventfds_nr;
		struct eventfd_ctx *err_trigger;
		struct eventfd_ctx *req_trigger;
		struct list_head dummy_resources_list;
		struct mutex ioeventfds_lock;
		struct list_head ioeventfds_list;
		};

		#define is_intx(vdev) (vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX)
		@@ -120,6 +136,9 @@ extern ssize_t vfio_pci_bar_rw(struct vfio_pci_device vdev, char __user buf,
		extern ssize_t vfio_pci_vga_rw(struct vfio_pci_device vdev, char __user buf,
		size_t count, loff_t *ppos, bool iswrite);

		extern long vfio_pci_ioeventfd(struct vfio_pci_device *vdev, loff_t offset,
		uint64_t data, int count, int fd);

		extern int vfio_pci_init_perm_bits(void);
		extern void vfio_pci_uninit_perm_bits(void);

drivers/vfio/pci/vfio_pci_rdwr.c

+164 −20

Original line number	Diff line number	Diff line
		@@ -17,10 +17,29 @@
		#include <linux/pci.h>
		#include <linux/uaccess.h>
		#include <linux/io.h>
		#include <linux/vfio.h>
		#include <linux/vgaarb.h>

		#include "vfio_pci_private.h"

		#ifdef __LITTLE_ENDIAN
		#define vfio_ioread64 ioread64
		#define vfio_iowrite64 iowrite64
		#define vfio_ioread32 ioread32
		#define vfio_iowrite32 iowrite32
		#define vfio_ioread16 ioread16
		#define vfio_iowrite16 iowrite16
		#else
		#define vfio_ioread64 ioread64be
		#define vfio_iowrite64 iowrite64be
		#define vfio_ioread32 ioread32be
		#define vfio_iowrite32 iowrite32be
		#define vfio_ioread16 ioread16be
		#define vfio_iowrite16 iowrite16be
		#endif
		#define vfio_ioread8 ioread8
		#define vfio_iowrite8 iowrite8

		/*
		* Read or write from an __iomem region (MMIO or I/O port) with an excluded
		* range which is inaccessible. The excluded range drops writes and fills
		@@ -44,15 +63,15 @@ static ssize_t do_io_rw(void __iomem io, char __user buf,
		fillable = 0;

		if (fillable >= 4 && !(off % 4)) {
		__le32 val;
		u32 val;

		if (iswrite) {
		if (copy_from_user(&val, buf, 4))
		return -EFAULT;

		iowrite32(le32_to_cpu(val), io + off);
		vfio_iowrite32(val, io + off);
		} else {
		val = cpu_to_le32(ioread32(io + off));
		val = vfio_ioread32(io + off);

		if (copy_to_user(buf, &val, 4))
		return -EFAULT;
		@@ -60,15 +79,15 @@ static ssize_t do_io_rw(void __iomem io, char __user buf,

		filled = 4;
		} else if (fillable >= 2 && !(off % 2)) {
		__le16 val;
		u16 val;

		if (iswrite) {
		if (copy_from_user(&val, buf, 2))
		return -EFAULT;

		iowrite16(le16_to_cpu(val), io + off);
		vfio_iowrite16(val, io + off);
		} else {
		val = cpu_to_le16(ioread16(io + off));
		val = vfio_ioread16(io + off);

		if (copy_to_user(buf, &val, 2))
		return -EFAULT;
		@@ -82,9 +101,9 @@ static ssize_t do_io_rw(void __iomem io, char __user buf,
		if (copy_from_user(&val, buf, 1))
		return -EFAULT;

		iowrite8(val, io + off);
		vfio_iowrite8(val, io + off);
		} else {
		val = ioread8(io + off);
		val = vfio_ioread8(io + off);

		if (copy_to_user(buf, &val, 1))
		return -EFAULT;
		@@ -113,6 +132,30 @@ static ssize_t do_io_rw(void __iomem io, char __user buf,
		return done;
		}

		static int vfio_pci_setup_barmap(struct vfio_pci_device *vdev, int bar)
		{
		struct pci_dev *pdev = vdev->pdev;
		int ret;
		void __iomem *io;

		if (vdev->barmap[bar])
		return 0;

		ret = pci_request_selected_regions(pdev, 1 << bar, "vfio");
		if (ret)
		return ret;

		io = pci_iomap(pdev, bar, 0);
		if (!io) {
		pci_release_selected_regions(pdev, 1 << bar);
		return -ENOMEM;
		}

		vdev->barmap[bar] = io;

		return 0;
		}

		ssize_t vfio_pci_bar_rw(struct vfio_pci_device vdev, char __user buf,
		size_t count, loff_t *ppos, bool iswrite)
		{
		@@ -147,22 +190,13 @@ ssize_t vfio_pci_bar_rw(struct vfio_pci_device vdev, char __user buf,
		if (!io)
		return -ENOMEM;
		x_end = end;
		} else if (!vdev->barmap[bar]) {
		int ret;

		ret = pci_request_selected_regions(pdev, 1 << bar, "vfio");
		} else {
		int ret = vfio_pci_setup_barmap(vdev, bar);
		if (ret)
		return ret;

		io = pci_iomap(pdev, bar, 0);
		if (!io) {
		pci_release_selected_regions(pdev, 1 << bar);
		return -ENOMEM;
		}

		vdev->barmap[bar] = io;
		} else
		io = vdev->barmap[bar];
		}

		if (bar == vdev->msix_bar) {
		x_start = vdev->msix_offset;
		@@ -242,3 +276,113 @@ ssize_t vfio_pci_vga_rw(struct vfio_pci_device vdev, char __user buf,

		return done;
		}

		static int vfio_pci_ioeventfd_handler(void opaque, void unused)
		{
		struct vfio_pci_ioeventfd *ioeventfd = opaque;

		switch (ioeventfd->count) {
		case 1:
		vfio_iowrite8(ioeventfd->data, ioeventfd->addr);
		break;
		case 2:
		vfio_iowrite16(ioeventfd->data, ioeventfd->addr);
		break;
		case 4:
		vfio_iowrite32(ioeventfd->data, ioeventfd->addr);
		break;
		#ifdef iowrite64
		case 8:
		vfio_iowrite64(ioeventfd->data, ioeventfd->addr);
		break;
		#endif
		}

		return 0;
		}

		long vfio_pci_ioeventfd(struct vfio_pci_device *vdev, loff_t offset,
		uint64_t data, int count, int fd)
		{
		struct pci_dev *pdev = vdev->pdev;
		loff_t pos = offset & VFIO_PCI_OFFSET_MASK;
		int ret, bar = VFIO_PCI_OFFSET_TO_INDEX(offset);
		struct vfio_pci_ioeventfd *ioeventfd;

		/* Only support ioeventfds into BARs */
		if (bar > VFIO_PCI_BAR5_REGION_INDEX)
		return -EINVAL;

		if (pos + count > pci_resource_len(pdev, bar))
		return -EINVAL;

		/* Disallow ioeventfds working around MSI-X table writes */
		if (bar == vdev->msix_bar &&
		!(pos + count <= vdev->msix_offset \|\|
		pos >= vdev->msix_offset + vdev->msix_size))
		return -EINVAL;

		#ifndef iowrite64
		if (count == 8)
		return -EINVAL;
		#endif

		ret = vfio_pci_setup_barmap(vdev, bar);
		if (ret)
		return ret;

		mutex_lock(&vdev->ioeventfds_lock);

		list_for_each_entry(ioeventfd, &vdev->ioeventfds_list, next) {
		if (ioeventfd->pos == pos && ioeventfd->bar == bar &&
		ioeventfd->data == data && ioeventfd->count == count) {
		if (fd == -1) {
		vfio_virqfd_disable(&ioeventfd->virqfd);
		list_del(&ioeventfd->next);
		vdev->ioeventfds_nr--;
		kfree(ioeventfd);
		ret = 0;
		} else
		ret = -EEXIST;

		goto out_unlock;
		}
		}

		if (fd < 0) {
		ret = -ENODEV;
		goto out_unlock;
		}

		if (vdev->ioeventfds_nr >= VFIO_PCI_IOEVENTFD_MAX) {
		ret = -ENOSPC;
		goto out_unlock;
		}

		ioeventfd = kzalloc(sizeof(*ioeventfd), GFP_KERNEL);
		if (!ioeventfd) {
		ret = -ENOMEM;
		goto out_unlock;
		}

		ioeventfd->addr = vdev->barmap[bar] + pos;
		ioeventfd->data = data;
		ioeventfd->pos = pos;
		ioeventfd->bar = bar;
		ioeventfd->count = count;

		ret = vfio_virqfd_enable(ioeventfd, vfio_pci_ioeventfd_handler,
		NULL, NULL, &ioeventfd->virqfd, fd);
		if (ret) {
		kfree(ioeventfd);
		goto out_unlock;
		}

		list_add(&ioeventfd->next, &vdev->ioeventfds_list);
		vdev->ioeventfds_nr++;

		out_unlock:
		mutex_unlock(&vdev->ioeventfds_lock);

		return ret;
		}

drivers/vfio/vfio_iommu_type1.c

+130 −21

Original line number	Diff line number	Diff line
		@@ -102,6 +102,13 @@ struct vfio_pfn {
		atomic_t ref_count;
		};

		struct vfio_regions {
		struct list_head list;
		dma_addr_t iova;
		phys_addr_t phys;
		size_t len;
		};

		#define IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu) \
		(!list_empty(&iommu->domain_list))

		@@ -397,7 +404,6 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
		{
		unsigned long pfn = 0;
		long ret, pinned = 0, lock_acct = 0;
		bool rsvd;
		dma_addr_t iova = vaddr - dma->vaddr + dma->iova;

		/* This code path is only user initiated */
		@@ -408,14 +414,23 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
		if (ret)
		return ret;

		if (is_invalid_reserved_pfn(*pfn_base)) {
		struct vm_area_struct *vma;

		down_read(&current->mm->mmap_sem);
		vma = find_vma_intersection(current->mm, vaddr, vaddr + 1);
		pinned = min_t(long, npage, vma_pages(vma));
		up_read(&current->mm->mmap_sem);
		return pinned;
		}

		pinned++;
		rsvd = is_invalid_reserved_pfn(*pfn_base);

		/*
		* Reserved pages aren't counted against the user, externally pinned
		* pages are already counted against the user.
		*/
		if (!rsvd && !vfio_find_vpfn(dma, iova)) {
		if (!vfio_find_vpfn(dma, iova)) {
		if (!lock_cap && current->mm->locked_vm + 1 > limit) {
		put_pfn(*pfn_base, dma->prot);
		pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__,
		@@ -435,13 +450,12 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
		if (ret)
		break;

		if (pfn != *pfn_base + pinned \|\|
		rsvd != is_invalid_reserved_pfn(pfn)) {
		if (pfn != *pfn_base + pinned) {
		put_pfn(pfn, dma->prot);
		break;
		}

		if (!rsvd && !vfio_find_vpfn(dma, iova)) {
		if (!vfio_find_vpfn(dma, iova)) {
		if (!lock_cap &&
		current->mm->locked_vm + lock_acct + 1 > limit) {
		put_pfn(pfn, dma->prot);
		@@ -459,10 +473,8 @@ out:

		unpin_out:
		if (ret) {
		if (!rsvd) {
		for (pfn = *pfn_base ; pinned ; pfn++, pinned--)
		put_pfn(pfn, dma->prot);
		}

		return ret;
		}
		@@ -660,11 +672,102 @@ unpin_exit:
		return i > npage ? npage : (i > 0 ? i : -EINVAL);
		}

		static long vfio_sync_unpin(struct vfio_dma dma, struct vfio_domain domain,
		struct list_head *regions)
		{
		long unlocked = 0;
		struct vfio_regions entry, next;

		iommu_tlb_sync(domain->domain);

		list_for_each_entry_safe(entry, next, regions, list) {
		unlocked += vfio_unpin_pages_remote(dma,
		entry->iova,
		entry->phys >> PAGE_SHIFT,
		entry->len >> PAGE_SHIFT,
		false);
		list_del(&entry->list);
		kfree(entry);
		}

		cond_resched();

		return unlocked;
		}

		/*
		* Generally, VFIO needs to unpin remote pages after each IOTLB flush.
		* Therefore, when using IOTLB flush sync interface, VFIO need to keep track
		* of these regions (currently using a list).
		*
		* This value specifies maximum number of regions for each IOTLB flush sync.
		*/
		#define VFIO_IOMMU_TLB_SYNC_MAX 512

		static size_t unmap_unpin_fast(struct vfio_domain *domain,
		struct vfio_dma dma, dma_addr_t iova,
		size_t len, phys_addr_t phys, long *unlocked,
		struct list_head *unmapped_list,
		int *unmapped_cnt)
		{
		size_t unmapped = 0;
		struct vfio_regions entry = kzalloc(sizeof(entry), GFP_KERNEL);

		if (entry) {
		unmapped = iommu_unmap_fast(domain->domain, *iova, len);

		if (!unmapped) {
		kfree(entry);
		} else {
		iommu_tlb_range_add(domain->domain, *iova, unmapped);
		entry->iova = *iova;
		entry->phys = phys;
		entry->len = unmapped;
		list_add_tail(&entry->list, unmapped_list);

		*iova += unmapped;
		(*unmapped_cnt)++;
		}
		}

		/*
		* Sync if the number of fast-unmap regions hits the limit
		* or in case of errors.
		*/
		if (*unmapped_cnt >= VFIO_IOMMU_TLB_SYNC_MAX \|\| !unmapped) {
		*unlocked += vfio_sync_unpin(dma, domain,
		unmapped_list);
		*unmapped_cnt = 0;
		}

		return unmapped;
		}

		static size_t unmap_unpin_slow(struct vfio_domain *domain,
		struct vfio_dma dma, dma_addr_t iova,
		size_t len, phys_addr_t phys,
		long *unlocked)
		{
		size_t unmapped = iommu_unmap(domain->domain, *iova, len);

		if (unmapped) {
		unlocked += vfio_unpin_pages_remote(dma, iova,
		phys >> PAGE_SHIFT,
		unmapped >> PAGE_SHIFT,
		false);
		*iova += unmapped;
		cond_resched();
		}
		return unmapped;
		}

		static long vfio_unmap_unpin(struct vfio_iommu iommu, struct vfio_dma dma,
		bool do_accounting)
		{
		dma_addr_t iova = dma->iova, end = dma->iova + dma->size;
		struct vfio_domain domain, d;
		LIST_HEAD(unmapped_region_list);
		int unmapped_region_cnt = 0;
		long unlocked = 0;

		if (!dma->size)
		@@ -710,20 +813,26 @@ static long vfio_unmap_unpin(struct vfio_iommu iommu, struct vfio_dma dma,
		break;
		}

		unmapped = iommu_unmap(domain->domain, iova, len);
		/*
		* First, try to use fast unmap/unpin. In case of failure,
		* switch to slow unmap/unpin path.
		*/
		unmapped = unmap_unpin_fast(domain, dma, &iova, len, phys,
		&unlocked, &unmapped_region_list,
		&unmapped_region_cnt);
		if (!unmapped) {
		unmapped = unmap_unpin_slow(domain, dma, &iova, len,
		phys, &unlocked);
		if (WARN_ON(!unmapped))
		break;

		unlocked += vfio_unpin_pages_remote(dma, iova,
		phys >> PAGE_SHIFT,
		unmapped >> PAGE_SHIFT,
		false);
		iova += unmapped;

		cond_resched();
		}
		}

		dma->iommu_mapped = false;

		if (unmapped_region_cnt)
		unlocked += vfio_sync_unpin(dma, domain, &unmapped_region_list);

		if (do_accounting) {
		vfio_lock_acct(dma->task, -unlocked, NULL);
		return 0;

Admin message