Commit b4174173 authored by Christoph Hellwig's avatar Christoph Hellwig
Browse files

dma-mapping: inline the fast path dma-direct calls 



Inline the single page map/unmap/sync dma-direct calls into the now
out of line generic wrappers.  This restores the behavior of a single
function call that we had before moving the generic calls out of line.
Besides the dma-mapping callers there are just a few callers in IOMMU
drivers that have a bypass mode, and more of those are going to be
switched to the generic bypass soon.

Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Tested-by: default avatarAlexey Kardashevskiy <aik@ozlabs.ru>
Reviewed-by: default avatarAlexey Kardashevskiy <aik@ozlabs.ru>
parent d3fa60d7

include/linux/dma-direct.h

+69 −23
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

/*

 * Internals of the DMA direct mapping implementation.  Only for use by the

 * DMA mapping code and IOMMU drivers.

 */

#ifndef _LINUX_DMA_DIRECT_H

#define _LINUX_DMA_DIRECT_H 1



#include <linux/dma-mapping.h>

#include <linux/dma-noncoherent.h>

#include <linux/memblock.h> /* for min_low_pfn */

#include <linux/mem_encrypt.h>

#include <linux/swiotlb.h>



extern unsigned int zone_dma_bits;
@@ -87,25 +93,17 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, 
		unsigned long attrs);

int dma_direct_supported(struct device *dev, u64 mask);

bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr);

dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,

		unsigned long offset, size_t size, enum dma_data_direction dir,

		unsigned long attrs);

int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,

		enum dma_data_direction dir, unsigned long attrs);

dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,

		size_t size, enum dma_data_direction dir, unsigned long attrs);

size_t dma_direct_max_mapping_size(struct device *dev);



#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \

    defined(CONFIG_SWIOTLB)

void dma_direct_sync_single_for_device(struct device *dev,

		dma_addr_t addr, size_t size, enum dma_data_direction dir);

void dma_direct_sync_sg_for_device(struct device *dev,

		struct scatterlist *sgl, int nents, enum dma_data_direction dir);

void dma_direct_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,

		int nents, enum dma_data_direction dir);

#else

static inline void dma_direct_sync_single_for_device(struct device *dev,

		dma_addr_t addr, size_t size, enum dma_data_direction dir)

{

}

static inline void dma_direct_sync_sg_for_device(struct device *dev,

		struct scatterlist *sgl, int nents, enum dma_data_direction dir)

{
@@ -115,34 +113,82 @@ static inline void dma_direct_sync_sg_for_device(struct device *dev, 
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \

    defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \

    defined(CONFIG_SWIOTLB)

void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,

		size_t size, enum dma_data_direction dir, unsigned long attrs);

void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,

		int nents, enum dma_data_direction dir, unsigned long attrs);

void dma_direct_sync_single_for_cpu(struct device *dev,

		dma_addr_t addr, size_t size, enum dma_data_direction dir);

void dma_direct_sync_sg_for_cpu(struct device *dev,

		struct scatterlist *sgl, int nents, enum dma_data_direction dir);

#else

static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,

		size_t size, enum dma_data_direction dir, unsigned long attrs)

{

}

static inline void dma_direct_unmap_sg(struct device *dev,

		struct scatterlist *sgl, int nents, enum dma_data_direction dir,

		unsigned long attrs)

{

}

static inline void dma_direct_sync_sg_for_cpu(struct device *dev,

		struct scatterlist *sgl, int nents, enum dma_data_direction dir)

{

}

#endif



static inline void dma_direct_sync_single_for_device(struct device *dev,

		dma_addr_t addr, size_t size, enum dma_data_direction dir)

{

	phys_addr_t paddr = dma_to_phys(dev, addr);



	if (unlikely(is_swiotlb_buffer(paddr)))

		swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE);



	if (!dev_is_dma_coherent(dev))

		arch_sync_dma_for_device(paddr, size, dir);

}



static inline void dma_direct_sync_single_for_cpu(struct device *dev,

		dma_addr_t addr, size_t size, enum dma_data_direction dir)

{

	phys_addr_t paddr = dma_to_phys(dev, addr);



	if (!dev_is_dma_coherent(dev)) {

		arch_sync_dma_for_cpu(paddr, size, dir);

		arch_sync_dma_for_cpu_all();

	}

static inline void dma_direct_sync_sg_for_cpu(struct device *dev,

		struct scatterlist *sgl, int nents, enum dma_data_direction dir)



	if (unlikely(is_swiotlb_buffer(paddr)))

		swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU);

}



static inline dma_addr_t dma_direct_map_page(struct device *dev,

		struct page *page, unsigned long offset, size_t size,

		enum dma_data_direction dir, unsigned long attrs)

{

	phys_addr_t phys = page_to_phys(page) + offset;

	dma_addr_t dma_addr = phys_to_dma(dev, phys);



	if (unlikely(swiotlb_force == SWIOTLB_FORCE))

		return swiotlb_map(dev, phys, size, dir, attrs);



	if (unlikely(!dma_capable(dev, dma_addr, size, true))) {

		if (swiotlb_force != SWIOTLB_NO_FORCE)

			return swiotlb_map(dev, phys, size, dir, attrs);



		dev_WARN_ONCE(dev, 1,

			     "DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",

			     &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);

		return DMA_MAPPING_ERROR;

	}

#endif



size_t dma_direct_max_mapping_size(struct device *dev);

	if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))

		arch_sync_dma_for_device(phys, size, dir);

	return dma_addr;

}



static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,

		size_t size, enum dma_data_direction dir, unsigned long attrs)

{

	phys_addr_t phys = dma_to_phys(dev, addr);



	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))

		dma_direct_sync_single_for_cpu(dev, addr, size, dir);



	if (unlikely(is_swiotlb_buffer(phys)))

		swiotlb_tbl_unmap_single(dev, phys, size, size, dir, attrs);

}

#endif /* _LINUX_DMA_DIRECT_H */

kernel/dma/direct.c

+0 −65
Original line number Diff line number Diff line
@@ -10,11 +10,9 @@ 
#include <linux/dma-direct.h>

#include <linux/scatterlist.h>

#include <linux/dma-contiguous.h>

#include <linux/dma-noncoherent.h>

#include <linux/pfn.h>

#include <linux/vmalloc.h>

#include <linux/set_memory.h>

#include <linux/swiotlb.h>



/*

 * Most architectures use ZONE_DMA for the first 16 Megabytes, but some use it
@@ -304,18 +302,6 @@ void dma_direct_free(struct device *dev, size_t size, 


#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \

    defined(CONFIG_SWIOTLB)

void dma_direct_sync_single_for_device(struct device *dev,

		dma_addr_t addr, size_t size, enum dma_data_direction dir)

{

	phys_addr_t paddr = dma_to_phys(dev, addr);



	if (unlikely(is_swiotlb_buffer(paddr)))

		swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE);



	if (!dev_is_dma_coherent(dev))

		arch_sync_dma_for_device(paddr, size, dir);

}



void dma_direct_sync_sg_for_device(struct device *dev,

		struct scatterlist *sgl, int nents, enum dma_data_direction dir)

{
@@ -339,20 +325,6 @@ void dma_direct_sync_sg_for_device(struct device *dev, 
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \

    defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \

    defined(CONFIG_SWIOTLB)

void dma_direct_sync_single_for_cpu(struct device *dev,

		dma_addr_t addr, size_t size, enum dma_data_direction dir)

{

	phys_addr_t paddr = dma_to_phys(dev, addr);



	if (!dev_is_dma_coherent(dev)) {

		arch_sync_dma_for_cpu(paddr, size, dir);

		arch_sync_dma_for_cpu_all();

	}



	if (unlikely(is_swiotlb_buffer(paddr)))

		swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU);

}



void dma_direct_sync_sg_for_cpu(struct device *dev,

		struct scatterlist *sgl, int nents, enum dma_data_direction dir)

{
@@ -374,18 +346,6 @@ void dma_direct_sync_sg_for_cpu(struct device *dev, 
		arch_sync_dma_for_cpu_all();

}



void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,

		size_t size, enum dma_data_direction dir, unsigned long attrs)

{

	phys_addr_t phys = dma_to_phys(dev, addr);



	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))

		dma_direct_sync_single_for_cpu(dev, addr, size, dir);



	if (unlikely(is_swiotlb_buffer(phys)))

		swiotlb_tbl_unmap_single(dev, phys, size, size, dir, attrs);

}



void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,

		int nents, enum dma_data_direction dir, unsigned long attrs)

{
@@ -398,31 +358,6 @@ void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl, 
}

#endif



dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,

		unsigned long offset, size_t size, enum dma_data_direction dir,

		unsigned long attrs)

{

	phys_addr_t phys = page_to_phys(page) + offset;

	dma_addr_t dma_addr = phys_to_dma(dev, phys);



	if (unlikely(swiotlb_force == SWIOTLB_FORCE))

		return swiotlb_map(dev, phys, size, dir, attrs);



	if (unlikely(!dma_capable(dev, dma_addr, size, true))) {

		if (swiotlb_force != SWIOTLB_NO_FORCE)

			return swiotlb_map(dev, phys, size, dir, attrs);



		dev_WARN_ONCE(dev, 1,

			     "DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",

			     &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);

		return DMA_MAPPING_ERROR;

	}



	if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))

		arch_sync_dma_for_device(phys, size, dir);

	return dma_addr;

}



int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,

		enum dma_data_direction dir, unsigned long attrs)

{

CRA Git | Maintained and supported by SUSTech CRA and CCSE