vfio/type1: Adopt fast IOTLB flush interface when unmap IOVAs (6bd06f5a) · Commits · 戴 / test

drivers/vfio/vfio_iommu_type1.c

+115 −11

Original line number	Diff line number	Diff line
		@@ -102,6 +102,13 @@ struct vfio_pfn {
		atomic_t ref_count;
		};

		struct vfio_regions {
		struct list_head list;
		dma_addr_t iova;
		phys_addr_t phys;
		size_t len;
		};

		#define IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu) \
		(!list_empty(&iommu->domain_list))

		@@ -660,11 +667,102 @@ unpin_exit:
		return i > npage ? npage : (i > 0 ? i : -EINVAL);
		}

		static long vfio_sync_unpin(struct vfio_dma dma, struct vfio_domain domain,
		struct list_head *regions)
		{
		long unlocked = 0;
		struct vfio_regions entry, next;

		iommu_tlb_sync(domain->domain);

		list_for_each_entry_safe(entry, next, regions, list) {
		unlocked += vfio_unpin_pages_remote(dma,
		entry->iova,
		entry->phys >> PAGE_SHIFT,
		entry->len >> PAGE_SHIFT,
		false);
		list_del(&entry->list);
		kfree(entry);
		}

		cond_resched();

		return unlocked;
		}

		/*
		* Generally, VFIO needs to unpin remote pages after each IOTLB flush.
		* Therefore, when using IOTLB flush sync interface, VFIO need to keep track
		* of these regions (currently using a list).
		*
		* This value specifies maximum number of regions for each IOTLB flush sync.
		*/
		#define VFIO_IOMMU_TLB_SYNC_MAX 512

		static size_t unmap_unpin_fast(struct vfio_domain *domain,
		struct vfio_dma dma, dma_addr_t iova,
		size_t len, phys_addr_t phys, long *unlocked,
		struct list_head *unmapped_list,
		int *unmapped_cnt)
		{
		size_t unmapped = 0;
		struct vfio_regions entry = kzalloc(sizeof(entry), GFP_KERNEL);

		if (entry) {
		unmapped = iommu_unmap_fast(domain->domain, *iova, len);

		if (!unmapped) {
		kfree(entry);
		} else {
		iommu_tlb_range_add(domain->domain, *iova, unmapped);
		entry->iova = *iova;
		entry->phys = phys;
		entry->len = unmapped;
		list_add_tail(&entry->list, unmapped_list);

		*iova += unmapped;
		(*unmapped_cnt)++;
		}
		}

		/*
		* Sync if the number of fast-unmap regions hits the limit
		* or in case of errors.
		*/
		if (*unmapped_cnt >= VFIO_IOMMU_TLB_SYNC_MAX \|\| !unmapped) {
		*unlocked += vfio_sync_unpin(dma, domain,
		unmapped_list);
		*unmapped_cnt = 0;
		}

		return unmapped;
		}

		static size_t unmap_unpin_slow(struct vfio_domain *domain,
		struct vfio_dma dma, dma_addr_t iova,
		size_t len, phys_addr_t phys,
		long *unlocked)
		{
		size_t unmapped = iommu_unmap(domain->domain, *iova, len);

		if (unmapped) {
		unlocked += vfio_unpin_pages_remote(dma, iova,
		phys >> PAGE_SHIFT,
		unmapped >> PAGE_SHIFT,
		false);
		*iova += unmapped;
		cond_resched();
		}
		return unmapped;
		}

		static long vfio_unmap_unpin(struct vfio_iommu iommu, struct vfio_dma dma,
		bool do_accounting)
		{
		dma_addr_t iova = dma->iova, end = dma->iova + dma->size;
		struct vfio_domain domain, d;
		LIST_HEAD(unmapped_region_list);
		int unmapped_region_cnt = 0;
		long unlocked = 0;

		if (!dma->size)
		@@ -710,20 +808,26 @@ static long vfio_unmap_unpin(struct vfio_iommu iommu, struct vfio_dma dma,
		break;
		}

		unmapped = iommu_unmap(domain->domain, iova, len);
		/*
		* First, try to use fast unmap/unpin. In case of failure,
		* switch to slow unmap/unpin path.
		*/
		unmapped = unmap_unpin_fast(domain, dma, &iova, len, phys,
		&unlocked, &unmapped_region_list,
		&unmapped_region_cnt);
		if (!unmapped) {
		unmapped = unmap_unpin_slow(domain, dma, &iova, len,
		phys, &unlocked);
		if (WARN_ON(!unmapped))
		break;

		unlocked += vfio_unpin_pages_remote(dma, iova,
		phys >> PAGE_SHIFT,
		unmapped >> PAGE_SHIFT,
		false);
		iova += unmapped;

		cond_resched();
		}
		}

		dma->iommu_mapped = false;

		if (unmapped_region_cnt)
		unlocked += vfio_sync_unpin(dma, domain, &unmapped_region_list);

		if (do_accounting) {
		vfio_lock_acct(dma->task, -unlocked, NULL);
		return 0;

Admin message