[PATCH] x86_64: Use function pointers to call DMA mapping functions (17a941d8) · Commits · 戴 / test

arch/x86_64/Kconfig

+5 −13

Original line number	Diff line number	Diff line
		@@ -351,32 +351,24 @@ config HPET_EMULATE_RTC
		depends on HPET_TIMER && RTC=y

		config GART_IOMMU
		bool "IOMMU support"
		bool "K8 GART IOMMU support"
		default y
		select SWIOTLB
		depends on PCI
		help
		Support the IOMMU. Needed to run systems with more than 3GB of memory
		properly with 32-bit PCI devices that do not support DAC (Double Address
		Cycle). The IOMMU can be turned off at runtime with the iommu=off parameter.
		Normally the kernel will take the right choice by itself.
		This option includes a driver for the AMD Opteron/Athlon64 IOMMU
		and a software emulation used on some other systems.
		This option includes a driver for the AMD Opteron/Athlon64 northbridge IOMMU
		and a software emulation used on other systems.
		If unsure, say Y.

		# need this always enabled with GART_IOMMU for the VIA workaround
		config SWIOTLB
		bool
		depends on GART_IOMMU
		default y

		config DUMMY_IOMMU
		bool
		depends on !GART_IOMMU && !SWIOTLB
		default y
		help
		Don't use IOMMU code. This will cause problems when you have more than 4GB
		of memory and any 32-bit devices. Don't turn on unless you know what you
		are doing.
		depends on GART_IOMMU

		config X86_MCE
		bool "Machine check support" if EMBEDDED

arch/x86_64/kernel/Makefile

+2 −2

Original line number	Diff line number	Diff line
		@@ -8,7 +8,7 @@ obj-y := process.o signal.o entry.o traps.o irq.o \
		ptrace.o time.o ioport.o ldt.o setup.o i8259.o sys_x86_64.o \
		x8664_ksyms.o i387.o syscall.o vsyscall.o \
		setup64.o bootflag.o e820.o reboot.o quirks.o i8237.o \
		dmi_scan.o
		dmi_scan.o pci-dma.o pci-nommu.o

		obj-$(CONFIG_X86_MCE) += mce.o
		obj-$(CONFIG_X86_MCE_INTEL) += mce_intel.o
		@@ -29,7 +29,7 @@ obj-$(CONFIG_SOFTWARE_SUSPEND) += suspend_asm.o
		obj-$(CONFIG_CPU_FREQ) += cpufreq/
		obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
		obj-$(CONFIG_GART_IOMMU) += pci-gart.o aperture.o
		obj-$(CONFIG_DUMMY_IOMMU) += pci-nommu.o pci-dma.o
		obj-$(CONFIG_SWIOTLB) += pci-swiotlb.o
		obj-$(CONFIG_KPROBES) += kprobes.o
		obj-$(CONFIG_X86_PM_TIMER) += pmtimer.o

arch/x86_64/kernel/pci-dma.c

+246 −40

Original line number	Diff line number	Diff line
		@@ -8,53 +8,259 @@
		#include <linux/pci.h>
		#include <linux/module.h>
		#include <asm/io.h>
		#include <asm/proto.h>

		/* Map a set of buffers described by scatterlist in streaming
		* mode for DMA. This is the scatter-gather version of the
		* above pci_map_single interface. Here the scatter gather list
		* elements are each tagged with the appropriate dma address
		* and length. They are obtained via sg_dma_{address,length}(SG).
		*
		* NOTE: An implementation may be able to use a smaller number of
		* DMA address/length pairs than there are SG table elements.
		* (for example via virtual mapping capabilities)
		* The routine returns the number of addr/length pairs actually
		* used, at most nents.
		*
		* Device ownership issues as mentioned above for pci_map_single are
		* the same here.
		int iommu_merge __read_mostly = 0;
		EXPORT_SYMBOL(iommu_merge);

		dma_addr_t bad_dma_address __read_mostly;
		EXPORT_SYMBOL(bad_dma_address);

		/* This tells the BIO block layer to assume merging. Default to off
		because we cannot guarantee merging later. */
		int iommu_bio_merge __read_mostly = 0;
		EXPORT_SYMBOL(iommu_bio_merge);

		int iommu_sac_force __read_mostly = 0;
		EXPORT_SYMBOL(iommu_sac_force);

		int no_iommu __read_mostly;
		#ifdef CONFIG_IOMMU_DEBUG
		int panic_on_overflow __read_mostly = 1;
		int force_iommu __read_mostly = 1;
		#else
		int panic_on_overflow __read_mostly = 0;
		int force_iommu __read_mostly= 0;
		#endif

		/* Dummy device used for NULL arguments (normally ISA). Better would
		be probably a smaller DMA mask, but this is bug-to-bug compatible
		to i386. */
		struct device fallback_dev = {
		.bus_id = "fallback device",
		.coherent_dma_mask = 0xffffffff,
		.dma_mask = &fallback_dev.coherent_dma_mask,
		};

		/* Allocate DMA memory on node near device */
		noinline static void *
		dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order)
		{
		struct page *page;
		int node;
		if (dev->bus == &pci_bus_type)
		node = pcibus_to_node(to_pci_dev(dev)->bus);
		else
		node = numa_node_id();
		page = alloc_pages_node(node, gfp, order);
		return page ? page_address(page) : NULL;
		}

		/*
		* Allocate memory for a coherent mapping.
		*/
		int dma_map_sg(struct device hwdev, struct scatterlist sg,
		int nents, int direction)
		void *
		dma_alloc_coherent(struct device dev, size_t size, dma_addr_t dma_handle,
		gfp_t gfp)
		{
		int i;
		void *memory;
		unsigned long dma_mask = 0;
		u64 bus;

		if (!dev)
		dev = &fallback_dev;
		dma_mask = dev->coherent_dma_mask;
		if (dma_mask == 0)
		dma_mask = 0xffffffff;

		/* Kludge to make it bug-to-bug compatible with i386. i386
		uses the normal dma_mask for alloc_coherent. */
		dma_mask &= *dev->dma_mask;

		/* Why <=? Even when the mask is smaller than 4GB it is often
		larger than 16MB and in this case we have a chance of
		finding fitting memory in the next higher zone first. If
		not retry with true GFP_DMA. -AK */
		if (dma_mask <= 0xffffffff)
		gfp \|= GFP_DMA32;

		BUG_ON(direction == DMA_NONE);
		for (i = 0; i < nents; i++ ) {
		struct scatterlist *s = &sg[i];
		BUG_ON(!s->page);
		s->dma_address = virt_to_bus(page_address(s->page) +s->offset);
		s->dma_length = s->length;
		again:
		memory = dma_alloc_pages(dev, gfp, get_order(size));
		if (memory == NULL)
		return NULL;

		{
		int high, mmu;
		bus = virt_to_bus(memory);
		high = (bus + size) >= dma_mask;
		mmu = high;
		if (force_iommu && !(gfp & GFP_DMA))
		mmu = 1;
		else if (high) {
		free_pages((unsigned long)memory,
		get_order(size));

		/* Don't use the 16MB ZONE_DMA unless absolutely
		needed. It's better to use remapping first. */
		if (dma_mask < 0xffffffff && !(gfp & GFP_DMA)) {
		gfp = (gfp & ~GFP_DMA32) \| GFP_DMA;
		goto again;
		}

		if (dma_ops->alloc_coherent)
		return dma_ops->alloc_coherent(dev, size,
		dma_handle, gfp);
		return NULL;
		}

		memset(memory, 0, size);
		if (!mmu) {
		*dma_handle = virt_to_bus(memory);
		return memory;
		}
		}

		if (dma_ops->alloc_coherent) {
		free_pages((unsigned long)memory, get_order(size));
		gfp &= ~(GFP_DMA\|GFP_DMA32);
		return dma_ops->alloc_coherent(dev, size, dma_handle, gfp);
		}
		return nents;

		if (dma_ops->map_simple) {
		*dma_handle = dma_ops->map_simple(dev, memory,
		size,
		PCI_DMA_BIDIRECTIONAL);
		if (*dma_handle != bad_dma_address)
		return memory;
		}

		EXPORT_SYMBOL(dma_map_sg);
		if (panic_on_overflow)
		panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",size);
		free_pages((unsigned long)memory, get_order(size));
		return NULL;
		}
		EXPORT_SYMBOL(dma_alloc_coherent);

		/* Unmap a set of streaming mode DMA translations.
		* Again, cpu read rules concerning calls here are the same as for
		* pci_unmap_single() above.
		/*
		* Unmap coherent memory.
		* The caller must ensure that the device has finished accessing the mapping.
		*/
		void dma_free_coherent(struct device *dev, size_t size,
		void *vaddr, dma_addr_t bus)
		{
		if (dma_ops->unmap_single)
		dma_ops->unmap_single(dev, bus, size, 0);
		free_pages((unsigned long)vaddr, get_order(size));
		}
		EXPORT_SYMBOL(dma_free_coherent);

		int dma_supported(struct device *dev, u64 mask)
		{
		if (dma_ops->dma_supported)
		return dma_ops->dma_supported(dev, mask);

		/* Copied from i386. Doesn't make much sense, because it will
		only work for pci_alloc_coherent.
		The caller just has to use GFP_DMA in this case. */
		if (mask < 0x00ffffff)
		return 0;

		/* Tell the device to use SAC when IOMMU force is on. This
		allows the driver to use cheaper accesses in some cases.

		Problem with this is that if we overflow the IOMMU area and
		return DAC as fallback address the device may not handle it
		correctly.

		As a special case some controllers have a 39bit address
		mode that is as efficient as 32bit (aic79xx). Don't force
		SAC for these. Assume all masks <= 40 bits are of this
		type. Normally this doesn't make any difference, but gives
		more gentle handling of IOMMU overflow. */
		if (iommu_sac_force && (mask >= 0xffffffffffULL)) {
		printk(KERN_INFO "%s: Force SAC with mask %Lx\n", dev->bus_id,mask);
		return 0;
		}

		return 1;
		}
		EXPORT_SYMBOL(dma_supported);

		int dma_set_mask(struct device *dev, u64 mask)
		{
		if (!dev->dma_mask \|\| !dma_supported(dev, mask))
		return -EIO;
		*dev->dma_mask = mask;
		return 0;
		}
		EXPORT_SYMBOL(dma_set_mask);

		/* iommu=[size][,noagp][,off][,force][,noforce][,leak][,memaper[=order]][,merge]
		[,forcesac][,fullflush][,nomerge][,biomerge]
		size set size of iommu (in bytes)
		noagp don't initialize the AGP driver and use full aperture.
		off don't use the IOMMU
		leak turn on simple iommu leak tracing (only when CONFIG_IOMMU_LEAK is on)
		memaper[=order] allocate an own aperture over RAM with size 32MB^order.
		noforce don't force IOMMU usage. Default.
		force Force IOMMU.
		merge Do lazy merging. This may improve performance on some block devices.
		Implies force (experimental)
		biomerge Do merging at the BIO layer. This is more efficient than merge,
		but should be only done with very big IOMMUs. Implies merge,force.
		nomerge Don't do SG merging.
		forcesac For SAC mode for masks <40bits (experimental)
		fullflush Flush IOMMU on each allocation (default)
		nofullflush Don't use IOMMU fullflush
		allowed overwrite iommu off workarounds for specific chipsets.
		soft Use software bounce buffering (default for Intel machines)
		noaperture Don't touch the aperture for AGP.
		*/
		void dma_unmap_sg(struct device dev, struct scatterlist sg,
		int nents, int dir)
		__init int iommu_setup(char *p)
		{
		int i;
		for (i = 0; i < nents; i++) {
		struct scatterlist *s = &sg[i];
		BUG_ON(s->page == NULL);
		BUG_ON(s->dma_address == 0);
		dma_unmap_single(dev, s->dma_address, s->dma_length, dir);
		iommu_merge = 1;

		while (*p) {
		if (!strncmp(p,"off",3))
		no_iommu = 1;
		/* gart_parse_options has more force support */
		if (!strncmp(p,"force",5))
		force_iommu = 1;
		if (!strncmp(p,"noforce",7)) {
		iommu_merge = 0;
		force_iommu = 0;
		}

		if (!strncmp(p, "biomerge",8)) {
		iommu_bio_merge = 4096;
		iommu_merge = 1;
		force_iommu = 1;
		}
		if (!strncmp(p, "panic",5))
		panic_on_overflow = 1;
		if (!strncmp(p, "nopanic",7))
		panic_on_overflow = 0;
		if (!strncmp(p, "merge",5)) {
		iommu_merge = 1;
		force_iommu = 1;
		}
		if (!strncmp(p, "nomerge",7))
		iommu_merge = 0;
		if (!strncmp(p, "forcesac",8))
		iommu_sac_force = 1;

		#ifdef CONFIG_SWIOTLB
		if (!strncmp(p, "soft",4))
		swiotlb = 1;
		#endif

		EXPORT_SYMBOL(dma_unmap_sg);
		#ifdef CONFIG_GART_IOMMU
		gart_parse_options(p);
		#endif

		p += strcspn(p, ",");
		if (*p == ',')
		++p;
		}
		return 1;
		}

arch/x86_64/kernel/pci-gart.c

+97 −314

File changed.

Preview size limit exceeded, changes collapsed.

arch/x86_64/kernel/pci-nommu.c

+74 −70

Original line number	Diff line number	Diff line
		@@ -6,89 +6,93 @@
		#include <linux/string.h>
		#include <asm/proto.h>
		#include <asm/processor.h>
		#include <asm/dma.h>

		int iommu_merge = 0;
		EXPORT_SYMBOL(iommu_merge);

		dma_addr_t bad_dma_address;
		EXPORT_SYMBOL(bad_dma_address);

		int iommu_bio_merge = 0;
		EXPORT_SYMBOL(iommu_bio_merge);

		int iommu_sac_force = 0;
		EXPORT_SYMBOL(iommu_sac_force);

		/*
		* Dummy IO MMU functions
		*/

		void dma_alloc_coherent(struct device hwdev, size_t size,
		dma_addr_t *dma_handle, gfp_t gfp)
		static int
		check_addr(char name, struct device hwdev, dma_addr_t bus, size_t size)
		{
		void *ret;
		u64 mask;
		int order = get_order(size);

		if (hwdev)
		mask = hwdev->coherent_dma_mask & *hwdev->dma_mask;
		else
		mask = 0xffffffff;
		for (;;) {
		ret = (void *)__get_free_pages(gfp, order);
		if (ret == NULL)
		return NULL;
		*dma_handle = virt_to_bus(ret);
		if ((*dma_handle & ~mask) == 0)
		break;
		free_pages((unsigned long)ret, order);
		if (gfp & GFP_DMA)
		return NULL;
		gfp \|= GFP_DMA;
		if (hwdev && bus + size > *hwdev->dma_mask) {
		printk(KERN_ERR
		"nommu_%s: overflow %Lx+%lu of device mask %Lx\n",
		name, (long long)bus, size, (long long)*hwdev->dma_mask);
		return 0;
		}

		memset(ret, 0, size);
		return ret;
		return 1;
		}
		EXPORT_SYMBOL(dma_alloc_coherent);

		void dma_free_coherent(struct device *hwdev, size_t size,
		void *vaddr, dma_addr_t dma_handle)
		static dma_addr_t
		nommu_map_single(struct device hwdev, void ptr, size_t size,
		int direction)
		{
		free_pages((unsigned long)vaddr, get_order(size));
		dma_addr_t bus = virt_to_bus(ptr);
		if (!check_addr("map_single", hwdev, bus, size))
		return bad_dma_address;
		return bus;
		}
		EXPORT_SYMBOL(dma_free_coherent);

		int dma_supported(struct device *hwdev, u64 mask)
		void nommu_unmap_single(struct device *dev, dma_addr_t addr,size_t size,
		int direction)
		{
		/*
		* we fall back to GFP_DMA when the mask isn't all 1s,
		* so we can't guarantee allocations that must be
		* within a tighter range than GFP_DMA..
		* RED-PEN this won't work for pci_map_single. Caller has to
		* use GFP_DMA in the first place.
		}

		/* Map a set of buffers described by scatterlist in streaming
		* mode for DMA. This is the scatter-gather version of the
		* above pci_map_single interface. Here the scatter gather list
		* elements are each tagged with the appropriate dma address
		* and length. They are obtained via sg_dma_{address,length}(SG).
		*
		* NOTE: An implementation may be able to use a smaller number of
		* DMA address/length pairs than there are SG table elements.
		* (for example via virtual mapping capabilities)
		* The routine returns the number of addr/length pairs actually
		* used, at most nents.
		*
		* Device ownership issues as mentioned above for pci_map_single are
		* the same here.
		*/
		if (mask < 0x00ffffff)
		return 0;
		int nommu_map_sg(struct device hwdev, struct scatterlist sg,
		int nents, int direction)
		{
		int i;

		return 1;
		BUG_ON(direction == DMA_NONE);
		for (i = 0; i < nents; i++ ) {
		struct scatterlist *s = &sg[i];
		BUG_ON(!s->page);
		s->dma_address = virt_to_bus(page_address(s->page) +s->offset);
		if (!check_addr("map_sg", hwdev, s->dma_address, s->length))
		return 0;
		s->dma_length = s->length;
		}
		return nents;
		}
		EXPORT_SYMBOL(dma_supported);

		int dma_get_cache_alignment(void)
		/* Unmap a set of streaming mode DMA translations.
		* Again, cpu read rules concerning calls here are the same as for
		* pci_unmap_single() above.
		*/
		void nommu_unmap_sg(struct device dev, struct scatterlist sg,
		int nents, int dir)
		{
		return boot_cpu_data.x86_clflush_size;
		}
		EXPORT_SYMBOL(dma_get_cache_alignment);

		static int __init check_ram(void)
		struct dma_mapping_ops nommu_dma_ops = {
		.map_single = nommu_map_single,
		.unmap_single = nommu_unmap_single,
		.map_sg = nommu_map_sg,
		.unmap_sg = nommu_unmap_sg,
		.is_phys = 1,
		};

		void __init no_iommu_init(void)
		{
		if (end_pfn >= 0xffffffff>>PAGE_SHIFT) {
		printk(
		KERN_ERR "WARNING more than 4GB of memory but IOMMU not compiled in.\n"
		if (dma_ops)
		return;
		printk(KERN_INFO "PCI-DMA: Disabling IOMMU.\n");
		dma_ops = &nommu_dma_ops;
		if (end_pfn > MAX_DMA32_PFN) {
		printk(KERN_ERR
		"WARNING more than 4GB of memory but IOMMU disabled.\n"
		KERN_ERR "WARNING 32bit PCI may malfunction.\n");
		}
		return 0;
		}
		__initcall(check_ram);

Admin message