Commit e1605495 authored by David Woodhouse's avatar David Woodhouse
Browse files

intel-iommu: Introduce domain_sg_mapping() to speed up intel_map_sg() 



Instead of calling domain_pfn_mapping() repeatedly with single or
small numbers of pages, just pass the sglist in. It can optimise the
number of cache flushes like domain_pfn_mapping() does, and gives a huge
speedup for large scatterlists.

Signed-off-by: default avatarDavid Woodhouse <David.Woodhouse@intel.com>
parent 875764de

drivers/pci/intel-iommu.c

+62 −21
Original line number Diff line number Diff line
@@ -1635,6 +1635,56 @@ static int domain_context_mapped(struct pci_dev *pdev) 
					     tmp->devfn);

}



static int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,

			     struct scatterlist *sg, unsigned long nr_pages,

			     int prot)

{

	struct dma_pte *first_pte = NULL, *pte = NULL;

	uint64_t pteval;

	int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;

	unsigned long sg_res = 0;



	BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);



	if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)

		return -EINVAL;



	prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;



	while (nr_pages--) {

		if (!sg_res) {

			sg_res = (sg->offset + sg->length + VTD_PAGE_SIZE - 1) >> VTD_PAGE_SHIFT;

			sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;

			sg->dma_length = sg->length;

			pteval = page_to_phys(sg_page(sg)) | prot;

		}

		if (!pte) {

			first_pte = pte = pfn_to_dma_pte(domain, iov_pfn);

			if (!pte)

				return -ENOMEM;

		}

		/* We don't need lock here, nobody else

		 * touches the iova range

		 */

		BUG_ON(dma_pte_addr(pte));

		pte->val = pteval;

		pte++;

		if (!nr_pages ||

		    (unsigned long)pte >> VTD_PAGE_SHIFT !=

		    (unsigned long)first_pte >> VTD_PAGE_SHIFT) {

			domain_flush_cache(domain, first_pte,

					   (void *)pte - (void *)first_pte);

			pte = NULL;

		}

		iov_pfn++;

		pteval += VTD_PAGE_SIZE;

		sg_res--;

		if (!sg_res)

			sg = sg_next(sg);

	}

	return 0;

}



static int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,

			      unsigned long phys_pfn, unsigned long nr_pages,

			      int prot)
@@ -2758,28 +2808,19 @@ static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int ne 
		prot |= DMA_PTE_WRITE;



	start_vpfn = mm_to_dma_pfn(iova->pfn_lo);

	offset_pfn = 0;

	for_each_sg(sglist, sg, nelems, i) {

		int nr_pages = aligned_nrpages(sg->offset, sg->length);

		ret = domain_pfn_mapping(domain, start_vpfn + offset_pfn,

					 page_to_dma_pfn(sg_page(sg)),

					 nr_pages, prot);

		if (ret) {



	ret = domain_sg_mapping(domain, start_vpfn, sglist, mm_to_dma_pfn(size), prot);

	if (unlikely(ret)) {

		/*  clear the page */

		dma_pte_clear_range(domain, start_vpfn,

					    start_vpfn + offset_pfn);

				    start_vpfn + size - 1);

		/* free page tables */

		dma_pte_free_pagetable(domain, start_vpfn,

					       start_vpfn + offset_pfn);

				       start_vpfn + size - 1);

		/* free iova */

		__free_iova(&domain->iovad, iova);

		return 0;

	}

		sg->dma_address = ((dma_addr_t)(start_vpfn + offset_pfn)

				   << VTD_PAGE_SHIFT) + sg->offset;

		sg->dma_length = sg->length;

		offset_pfn += nr_pages;

	}



	/* it's a non-present to present mapping. Only flush if caching mode */

	if (cap_caching_mode(iommu->cap))

CRA Git | Maintained and supported by SUSTech CRA and CCSE