Merge tag 'vfio-v4.12-rc1' of git://github.com/awilliam/linux-vfio (c336bf8e) · Commits · 戴 / test

drivers/vfio/vfio_iommu_spapr_tce.c

+13 −0

Original line number	Original line	Diff line number	Diff line
	@@ -198,6 +198,11 @@ static long tce_iommu_register_pages(struct tce_container *container,
	return ret;		return ret;

	tcemem = kzalloc(sizeof(*tcemem), GFP_KERNEL);		tcemem = kzalloc(sizeof(*tcemem), GFP_KERNEL);
			if (!tcemem) {
			mm_iommu_put(container->mm, mem);
			return -ENOMEM;
			}

	tcemem->mem = mem;		tcemem->mem = mem;
	list_add(&tcemem->next, &container->prereg_list);		list_add(&tcemem->next, &container->prereg_list);

	@@ -1335,8 +1340,16 @@ static int tce_iommu_attach_group(void *iommu_data,

	if (!table_group->ops \|\| !table_group->ops->take_ownership \|\|		if (!table_group->ops \|\| !table_group->ops->take_ownership \|\|
	!table_group->ops->release_ownership) {		!table_group->ops->release_ownership) {
			if (container->v2) {
			ret = -EPERM;
			goto unlock_exit;
			}
	ret = tce_iommu_take_ownership(container, table_group);		ret = tce_iommu_take_ownership(container, table_group);
	} else {		} else {
			if (!container->v2) {
			ret = -EPERM;
			goto unlock_exit;
			}
	ret = tce_iommu_take_ownership_ddw(container, table_group);		ret = tce_iommu_take_ownership_ddw(container, table_group);
	if (!tce_groups_attached(container) && !container->tables[0])		if (!tce_groups_attached(container) && !container->tables[0])
	container->def_window_pending = true;		container->def_window_pending = true;

drivers/vfio/vfio_iommu_type1.c

+64 −86

Original line number	Original line	Diff line number	Diff line
	@@ -246,69 +246,46 @@ static int vfio_iova_put_vfio_pfn(struct vfio_dma dma, struct vfio_pfn vpfn)
	return ret;		return ret;
	}		}

	struct vwork {		static int vfio_lock_acct(struct task_struct task, long npage, bool lock_cap)
	struct mm_struct *mm;
	long npage;
	struct work_struct work;
	};

	/* delayed decrement/increment for locked_vm */
	static void vfio_lock_acct_bg(struct work_struct *work)
	{		{
	struct vwork *vwork = container_of(work, struct vwork, work);
	struct mm_struct *mm;

	mm = vwork->mm;
	down_write(&mm->mmap_sem);
	mm->locked_vm += vwork->npage;
	up_write(&mm->mmap_sem);
	mmput(mm);
	kfree(vwork);
	}

	static void vfio_lock_acct(struct task_struct *task, long npage)
	{
	struct vwork *vwork;
	struct mm_struct *mm;		struct mm_struct *mm;
	bool is_current;		bool is_current;
			int ret;

	if (!npage)		if (!npage)
	return;		return 0;

	is_current = (task->mm == current->mm);		is_current = (task->mm == current->mm);

	mm = is_current ? task->mm : get_task_mm(task);		mm = is_current ? task->mm : get_task_mm(task);
	if (!mm)		if (!mm)
	return; /* process exited */		return -ESRCH; /* process exited */

	if (down_write_trylock(&mm->mmap_sem)) {		ret = down_write_killable(&mm->mmap_sem);
	mm->locked_vm += npage;		if (!ret) {
	up_write(&mm->mmap_sem);		if (npage > 0) {
	if (!is_current)		if (lock_cap ? !*lock_cap :
	mmput(mm);		!has_capability(task, CAP_IPC_LOCK)) {
	return;		unsigned long limit;

			limit = task_rlimit(task,
			RLIMIT_MEMLOCK) >> PAGE_SHIFT;

			if (mm->locked_vm + npage > limit)
			ret = -ENOMEM;
			}
	}		}

	if (is_current) {		if (!ret)
	mm = get_task_mm(task);		mm->locked_vm += npage;
	if (!mm)
	return;		up_write(&mm->mmap_sem);
	}		}

	/*		if (!is_current)
	* Couldn't get mmap_sem lock, so must setup to update
	* mm->locked_vm later. If locked_vm were atomic, we
	* wouldn't need this silliness
	*/
	vwork = kmalloc(sizeof(struct vwork), GFP_KERNEL);
	if (WARN_ON(!vwork)) {
	mmput(mm);		mmput(mm);
	return;
	}		return ret;
	INIT_WORK(&vwork->work, vfio_lock_acct_bg);
	vwork->mm = mm;
	vwork->npage = npage;
	schedule_work(&vwork->work);
	}		}

	/*		/*
	@@ -403,10 +380,10 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
	* first page and all consecutive pages with the same locking.		* first page and all consecutive pages with the same locking.
	*/		*/
	static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,		static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
	long npage, unsigned long *pfn_base)		long npage, unsigned long *pfn_base,
			bool lock_cap, unsigned long limit)
	{		{
	unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;		unsigned long pfn = 0;
	bool lock_cap = capable(CAP_IPC_LOCK);
	long ret, pinned = 0, lock_acct = 0;		long ret, pinned = 0, lock_acct = 0;
	bool rsvd;		bool rsvd;
	dma_addr_t iova = vaddr - dma->vaddr + dma->iova;		dma_addr_t iova = vaddr - dma->vaddr + dma->iova;
	@@ -442,8 +419,6 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
	/* Lock all the consecutive pages from pfn_base */		/* Lock all the consecutive pages from pfn_base */
	for (vaddr += PAGE_SIZE, iova += PAGE_SIZE; pinned < npage;		for (vaddr += PAGE_SIZE, iova += PAGE_SIZE; pinned < npage;
	pinned++, vaddr += PAGE_SIZE, iova += PAGE_SIZE) {		pinned++, vaddr += PAGE_SIZE, iova += PAGE_SIZE) {
	unsigned long pfn = 0;

	ret = vaddr_get_pfn(current->mm, vaddr, dma->prot, &pfn);		ret = vaddr_get_pfn(current->mm, vaddr, dma->prot, &pfn);
	if (ret)		if (ret)
	break;		break;
	@@ -460,14 +435,25 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
	put_pfn(pfn, dma->prot);		put_pfn(pfn, dma->prot);
	pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n",		pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n",
	__func__, limit << PAGE_SHIFT);		__func__, limit << PAGE_SHIFT);
	break;		ret = -ENOMEM;
			goto unpin_out;
	}		}
	lock_acct++;		lock_acct++;
	}		}
	}		}

	out:		out:
	vfio_lock_acct(current, lock_acct);		ret = vfio_lock_acct(current, lock_acct, &lock_cap);

			unpin_out:
			if (ret) {
			if (!rsvd) {
			for (pfn = *pfn_base ; pinned ; pfn++, pinned--)
			put_pfn(pfn, dma->prot);
			}

			return ret;
			}

	return pinned;		return pinned;
	}		}
	@@ -488,7 +474,7 @@ static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova,
	}		}

	if (do_accounting)		if (do_accounting)
	vfio_lock_acct(dma->task, locked - unlocked);		vfio_lock_acct(dma->task, locked - unlocked, NULL);

	return unlocked;		return unlocked;
	}		}
	@@ -496,37 +482,26 @@ static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova,
	static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr,		static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr,
	unsigned long *pfn_base, bool do_accounting)		unsigned long *pfn_base, bool do_accounting)
	{		{
	unsigned long limit;
	bool lock_cap = has_capability(dma->task, CAP_IPC_LOCK);
	struct mm_struct *mm;		struct mm_struct *mm;
	int ret;		int ret;
	bool rsvd;

	mm = get_task_mm(dma->task);		mm = get_task_mm(dma->task);
	if (!mm)		if (!mm)
	return -ENODEV;		return -ENODEV;

	ret = vaddr_get_pfn(mm, vaddr, dma->prot, pfn_base);		ret = vaddr_get_pfn(mm, vaddr, dma->prot, pfn_base);
	if (ret)		if (!ret && do_accounting && !is_invalid_reserved_pfn(*pfn_base)) {
	goto pin_page_exit;		ret = vfio_lock_acct(dma->task, 1, NULL);
			if (ret) {
	rsvd = is_invalid_reserved_pfn(*pfn_base);
	limit = task_rlimit(dma->task, RLIMIT_MEMLOCK) >> PAGE_SHIFT;

	if (!rsvd && !lock_cap && mm->locked_vm + 1 > limit) {
	put_pfn(*pfn_base, dma->prot);		put_pfn(*pfn_base, dma->prot);
	pr_warn("%s: Task %s (%d) RLIMIT_MEMLOCK (%ld) exceeded\n",		if (ret == -ENOMEM)
	__func__, dma->task->comm, task_pid_nr(dma->task),		pr_warn("%s: Task %s (%d) RLIMIT_MEMLOCK "
	limit << PAGE_SHIFT);		"(%ld) exceeded\n", __func__,
	ret = -ENOMEM;		dma->task->comm, task_pid_nr(dma->task),
	goto pin_page_exit;		task_rlimit(dma->task, RLIMIT_MEMLOCK));
			}
	}		}

	if (!rsvd && do_accounting)
	vfio_lock_acct(dma->task, 1);
	ret = 1;

	pin_page_exit:
	mmput(mm);		mmput(mm);
	return ret;		return ret;
	}		}
	@@ -543,7 +518,7 @@ static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova,
	unlocked = vfio_iova_put_vfio_pfn(dma, vpfn);		unlocked = vfio_iova_put_vfio_pfn(dma, vpfn);

	if (do_accounting)		if (do_accounting)
	vfio_lock_acct(dma->task, -unlocked);		vfio_lock_acct(dma->task, -unlocked, NULL);

	return unlocked;		return unlocked;
	}		}
	@@ -606,10 +581,8 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data,
	remote_vaddr = dma->vaddr + iova - dma->iova;		remote_vaddr = dma->vaddr + iova - dma->iova;
	ret = vfio_pin_page_external(dma, remote_vaddr, &phys_pfn[i],		ret = vfio_pin_page_external(dma, remote_vaddr, &phys_pfn[i],
	do_accounting);		do_accounting);
	if (ret <= 0) {		if (ret)
	WARN_ON(!ret);
	goto pin_unwind;		goto pin_unwind;
	}

	ret = vfio_add_to_pfn_list(dma, iova, phys_pfn[i]);		ret = vfio_add_to_pfn_list(dma, iova, phys_pfn[i]);
	if (ret) {		if (ret) {
	@@ -740,7 +713,7 @@ static long vfio_unmap_unpin(struct vfio_iommu iommu, struct vfio_dma dma,

	dma->iommu_mapped = false;		dma->iommu_mapped = false;
	if (do_accounting) {		if (do_accounting) {
	vfio_lock_acct(dma->task, -unlocked);		vfio_lock_acct(dma->task, -unlocked, NULL);
	return 0;		return 0;
	}		}
	return unlocked;		return unlocked;
	@@ -951,13 +924,15 @@ static int vfio_pin_map_dma(struct vfio_iommu iommu, struct vfio_dma dma,
	unsigned long vaddr = dma->vaddr;		unsigned long vaddr = dma->vaddr;
	size_t size = map_size;		size_t size = map_size;
	long npage;		long npage;
	unsigned long pfn;		unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
			bool lock_cap = capable(CAP_IPC_LOCK);
	int ret = 0;		int ret = 0;

	while (size) {		while (size) {
	/* Pin a contiguous chunk of memory */		/* Pin a contiguous chunk of memory */
	npage = vfio_pin_pages_remote(dma, vaddr + dma->size,		npage = vfio_pin_pages_remote(dma, vaddr + dma->size,
	size >> PAGE_SHIFT, &pfn);		size >> PAGE_SHIFT, &pfn,
			lock_cap, limit);
	if (npage <= 0) {		if (npage <= 0) {
	WARN_ON(!npage);		WARN_ON(!npage);
	ret = (int)npage;		ret = (int)npage;
	@@ -1067,6 +1042,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
	{		{
	struct vfio_domain *d;		struct vfio_domain *d;
	struct rb_node *n;		struct rb_node *n;
			unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
			bool lock_cap = capable(CAP_IPC_LOCK);
	int ret;		int ret;

	/* Arbitrarily pick the first domain in the list for lookups */		/* Arbitrarily pick the first domain in the list for lookups */
	@@ -1113,7 +1090,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,

	npage = vfio_pin_pages_remote(dma, vaddr,		npage = vfio_pin_pages_remote(dma, vaddr,
	n >> PAGE_SHIFT,		n >> PAGE_SHIFT,
	&pfn);		&pfn, lock_cap,
			limit);
	if (npage <= 0) {		if (npage <= 0) {
	WARN_ON(!npage);		WARN_ON(!npage);
	ret = (int)npage;		ret = (int)npage;
	@@ -1382,7 +1360,7 @@ static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu)
	if (!is_invalid_reserved_pfn(vpfn->pfn))		if (!is_invalid_reserved_pfn(vpfn->pfn))
	locked++;		locked++;
	}		}
	vfio_lock_acct(dma->task, locked - unlocked);		vfio_lock_acct(dma->task, locked - unlocked, NULL);
	}		}
	}		}

Admin message