Commit 65789daa authored by Barry Song's avatar Barry Song Committed by Christoph Hellwig
Browse files

dma-mapping: add benchmark support for streaming DMA APIs 



Nowadays, there are increasing requirements to benchmark the performance
of dma_map and dma_unmap particually while the device is attached to an
IOMMU.

This patch enables the support. Users can run specified number of threads
to do dma_map_page and dma_unmap_page on a specific NUMA node with the
specified duration. Then dma_map_benchmark will calculate the average
latency for map and unmap.

A difficulity for this benchmark is that dma_map/unmap APIs must run on
a particular device. Each device might have different backend of IOMMU or
non-IOMMU.

So we use the driver_override to bind dma_map_benchmark to a particual
device by:
For platform devices:
echo dma_map_benchmark > /sys/bus/platform/devices/xxx/driver_override
echo xxx > /sys/bus/platform/drivers/xxx/unbind
echo xxx > /sys/bus/platform/drivers/dma_map_benchmark/bind

For PCI devices:
echo dma_map_benchmark > /sys/bus/pci/devices/0000:00:01.0/driver_override
echo 0000:00:01.0 > /sys/bus/pci/drivers/xxx/unbind
echo 0000:00:01.0 > /sys/bus/pci/drivers/dma_map_benchmark/bind

Cc: Will Deacon <will@kernel.org>
Cc: Shuah Khan <shuah@kernel.org>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Robin Murphy <robin.murphy@arm.com>
Signed-off-by: default avatarBarry Song <song.bao.hua@hisilicon.com>
[hch: folded in two fixes from Colin Ian King <colin.king@canonical.com>]
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
parent 819b70ad

kernel/dma/Kconfig

+9 −0
Original line number Diff line number Diff line
@@ -229,3 +229,12 @@ config DMA_API_DEBUG_SG 
	  is technically out-of-spec.



	  If unsure, say N.



config DMA_MAP_BENCHMARK

	bool "Enable benchmarking of streaming DMA mapping"

	depends on DEBUG_FS

	help

	  Provides /sys/kernel/debug/dma_map_benchmark that helps with testing

	  performance of dma_(un)map_page.



	  See tools/testing/selftests/dma/dma_map_benchmark.c

kernel/dma/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -10,3 +10,4 @@ obj-$(CONFIG_DMA_API_DEBUG) += debug.o 
obj-$(CONFIG_SWIOTLB)			+= swiotlb.o

obj-$(CONFIG_DMA_COHERENT_POOL)		+= pool.o

obj-$(CONFIG_DMA_REMAP)			+= remap.o

obj-$(CONFIG_DMA_MAP_BENCHMARK)		+= map_benchmark.o

kernel/dma/map_benchmark.c

0 → 100644
+361 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-only

/*

 * Copyright (C) 2020 Hisilicon Limited.

 */



#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt



#include <linux/debugfs.h>

#include <linux/delay.h>

#include <linux/device.h>

#include <linux/dma-mapping.h>

#include <linux/kernel.h>

#include <linux/kthread.h>

#include <linux/math64.h>

#include <linux/module.h>

#include <linux/pci.h>

#include <linux/platform_device.h>

#include <linux/slab.h>

#include <linux/timekeeping.h>



#define DMA_MAP_BENCHMARK	_IOWR('d', 1, struct map_benchmark)

#define DMA_MAP_MAX_THREADS	1024

#define DMA_MAP_MAX_SECONDS	300



#define DMA_MAP_BIDIRECTIONAL	0

#define DMA_MAP_TO_DEVICE	1

#define DMA_MAP_FROM_DEVICE	2



struct map_benchmark {

	__u64 avg_map_100ns; /* average map latency in 100ns */

	__u64 map_stddev; /* standard deviation of map latency */

	__u64 avg_unmap_100ns; /* as above */

	__u64 unmap_stddev;

	__u32 threads; /* how many threads will do map/unmap in parallel */

	__u32 seconds; /* how long the test will last */

	__s32 node; /* which numa node this benchmark will run on */

	__u32 dma_bits; /* DMA addressing capability */

	__u32 dma_dir; /* DMA data direction */

	__u64 expansion[10];	/* For future use */

};



struct map_benchmark_data {

	struct map_benchmark bparam;

	struct device *dev;

	struct dentry  *debugfs;

	enum dma_data_direction dir;

	atomic64_t sum_map_100ns;

	atomic64_t sum_unmap_100ns;

	atomic64_t sum_sq_map;

	atomic64_t sum_sq_unmap;

	atomic64_t loops;

};



static int map_benchmark_thread(void *data)

{

	void *buf;

	dma_addr_t dma_addr;

	struct map_benchmark_data *map = data;

	int ret = 0;



	buf = (void *)__get_free_page(GFP_KERNEL);

	if (!buf)

		return -ENOMEM;



	while (!kthread_should_stop())  {

		u64 map_100ns, unmap_100ns, map_sq, unmap_sq;

		ktime_t map_stime, map_etime, unmap_stime, unmap_etime;

		ktime_t map_delta, unmap_delta;



		/*

		 * for a non-coherent device, if we don't stain them in the

		 * cache, this will give an underestimate of the real-world

		 * overhead of BIDIRECTIONAL or TO_DEVICE mappings;

		 * 66 means evertything goes well! 66 is lucky.

		 */

		if (map->dir != DMA_FROM_DEVICE)

			memset(buf, 0x66, PAGE_SIZE);



		map_stime = ktime_get();

		dma_addr = dma_map_single(map->dev, buf, PAGE_SIZE, map->dir);

		if (unlikely(dma_mapping_error(map->dev, dma_addr))) {

			pr_err("dma_map_single failed on %s\n",

				dev_name(map->dev));

			ret = -ENOMEM;

			goto out;

		}

		map_etime = ktime_get();

		map_delta = ktime_sub(map_etime, map_stime);



		unmap_stime = ktime_get();

		dma_unmap_single(map->dev, dma_addr, PAGE_SIZE, map->dir);

		unmap_etime = ktime_get();

		unmap_delta = ktime_sub(unmap_etime, unmap_stime);



		/* calculate sum and sum of squares */



		map_100ns = div64_ul(map_delta,  100);

		unmap_100ns = div64_ul(unmap_delta, 100);

		map_sq = map_100ns * map_100ns;

		unmap_sq = unmap_100ns * unmap_100ns;



		atomic64_add(map_100ns, &map->sum_map_100ns);

		atomic64_add(unmap_100ns, &map->sum_unmap_100ns);

		atomic64_add(map_sq, &map->sum_sq_map);

		atomic64_add(unmap_sq, &map->sum_sq_unmap);

		atomic64_inc(&map->loops);

	}



out:

	free_page((unsigned long)buf);

	return ret;

}



static int do_map_benchmark(struct map_benchmark_data *map)

{

	struct task_struct **tsk;

	int threads = map->bparam.threads;

	int node = map->bparam.node;

	const cpumask_t *cpu_mask = cpumask_of_node(node);

	u64 loops;

	int ret = 0;

	int i;



	tsk = kmalloc_array(threads, sizeof(*tsk), GFP_KERNEL);

	if (!tsk)

		return -ENOMEM;



	get_device(map->dev);



	for (i = 0; i < threads; i++) {

		tsk[i] = kthread_create_on_node(map_benchmark_thread, map,

				map->bparam.node, "dma-map-benchmark/%d", i);

		if (IS_ERR(tsk[i])) {

			pr_err("create dma_map thread failed\n");

			ret = PTR_ERR(tsk[i]);

			goto out;

		}



		if (node != NUMA_NO_NODE)

			kthread_bind_mask(tsk[i], cpu_mask);

	}



	/* clear the old value in the previous benchmark */

	atomic64_set(&map->sum_map_100ns, 0);

	atomic64_set(&map->sum_unmap_100ns, 0);

	atomic64_set(&map->sum_sq_map, 0);

	atomic64_set(&map->sum_sq_unmap, 0);

	atomic64_set(&map->loops, 0);



	for (i = 0; i < threads; i++)

		wake_up_process(tsk[i]);



	msleep_interruptible(map->bparam.seconds * 1000);



	/* wait for the completion of benchmark threads */

	for (i = 0; i < threads; i++) {

		ret = kthread_stop(tsk[i]);

		if (ret)

			goto out;

	}



	loops = atomic64_read(&map->loops);

	if (likely(loops > 0)) {

		u64 map_variance, unmap_variance;

		u64 sum_map = atomic64_read(&map->sum_map_100ns);

		u64 sum_unmap = atomic64_read(&map->sum_unmap_100ns);

		u64 sum_sq_map = atomic64_read(&map->sum_sq_map);

		u64 sum_sq_unmap = atomic64_read(&map->sum_sq_unmap);



		/* average latency */

		map->bparam.avg_map_100ns = div64_u64(sum_map, loops);

		map->bparam.avg_unmap_100ns = div64_u64(sum_unmap, loops);



		/* standard deviation of latency */

		map_variance = div64_u64(sum_sq_map, loops) -

				map->bparam.avg_map_100ns *

				map->bparam.avg_map_100ns;

		unmap_variance = div64_u64(sum_sq_unmap, loops) -

				map->bparam.avg_unmap_100ns *

				map->bparam.avg_unmap_100ns;

		map->bparam.map_stddev = int_sqrt64(map_variance);

		map->bparam.unmap_stddev = int_sqrt64(unmap_variance);

	}



out:

	put_device(map->dev);

	kfree(tsk);

	return ret;

}



static long map_benchmark_ioctl(struct file *file, unsigned int cmd,

		unsigned long arg)

{

	struct map_benchmark_data *map = file->private_data;

	void __user *argp = (void __user *)arg;

	u64 old_dma_mask;



	int ret;



	if (copy_from_user(&map->bparam, argp, sizeof(map->bparam)))

		return -EFAULT;



	switch (cmd) {

	case DMA_MAP_BENCHMARK:

		if (map->bparam.threads == 0 ||

		    map->bparam.threads > DMA_MAP_MAX_THREADS) {

			pr_err("invalid thread number\n");

			return -EINVAL;

		}



		if (map->bparam.seconds == 0 ||

		    map->bparam.seconds > DMA_MAP_MAX_SECONDS) {

			pr_err("invalid duration seconds\n");

			return -EINVAL;

		}



		if (map->bparam.node != NUMA_NO_NODE &&

		    !node_possible(map->bparam.node)) {

			pr_err("invalid numa node\n");

			return -EINVAL;

		}



		switch (map->bparam.dma_dir) {

		case DMA_MAP_BIDIRECTIONAL:

			map->dir = DMA_BIDIRECTIONAL;

			break;

		case DMA_MAP_FROM_DEVICE:

			map->dir = DMA_FROM_DEVICE;

			break;

		case DMA_MAP_TO_DEVICE:

			map->dir = DMA_TO_DEVICE;

			break;

		default:

			pr_err("invalid DMA direction\n");

			return -EINVAL;

		}



		old_dma_mask = dma_get_mask(map->dev);



		ret = dma_set_mask(map->dev,

				   DMA_BIT_MASK(map->bparam.dma_bits));

		if (ret) {

			pr_err("failed to set dma_mask on device %s\n",

				dev_name(map->dev));

			return -EINVAL;

		}



		ret = do_map_benchmark(map);



		/*

		 * restore the original dma_mask as many devices' dma_mask are

		 * set by architectures, acpi, busses. When we bind them back

		 * to their original drivers, those drivers shouldn't see

		 * dma_mask changed by benchmark

		 */

		dma_set_mask(map->dev, old_dma_mask);

		break;

	default:

		return -EINVAL;

	}



	if (copy_to_user(argp, &map->bparam, sizeof(map->bparam)))

		return -EFAULT;



	return ret;

}



static const struct file_operations map_benchmark_fops = {

	.open			= simple_open,

	.unlocked_ioctl		= map_benchmark_ioctl,

};



static void map_benchmark_remove_debugfs(void *data)

{

	struct map_benchmark_data *map = (struct map_benchmark_data *)data;



	debugfs_remove(map->debugfs);

}



static int __map_benchmark_probe(struct device *dev)

{

	struct dentry *entry;

	struct map_benchmark_data *map;

	int ret;



	map = devm_kzalloc(dev, sizeof(*map), GFP_KERNEL);

	if (!map)

		return -ENOMEM;

	map->dev = dev;



	ret = devm_add_action(dev, map_benchmark_remove_debugfs, map);

	if (ret) {

		pr_err("Can't add debugfs remove action\n");

		return ret;

	}



	/*

	 * we only permit a device bound with this driver, 2nd probe

	 * will fail

	 */

	entry = debugfs_create_file("dma_map_benchmark", 0600, NULL, map,

			&map_benchmark_fops);

	if (IS_ERR(entry))

		return PTR_ERR(entry);

	map->debugfs = entry;



	return 0;

}



static int map_benchmark_platform_probe(struct platform_device *pdev)

{

	return __map_benchmark_probe(&pdev->dev);

}



static struct platform_driver map_benchmark_platform_driver = {

	.driver		= {

		.name	= "dma_map_benchmark",

	},

	.probe = map_benchmark_platform_probe,

};



static int

map_benchmark_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)

{

	return __map_benchmark_probe(&pdev->dev);

}



static struct pci_driver map_benchmark_pci_driver = {

	.name	= "dma_map_benchmark",

	.probe	= map_benchmark_pci_probe,

};



static int __init map_benchmark_init(void)

{

	int ret;



	ret = pci_register_driver(&map_benchmark_pci_driver);

	if (ret)

		return ret;



	ret = platform_driver_register(&map_benchmark_platform_driver);

	if (ret) {

		pci_unregister_driver(&map_benchmark_pci_driver);

		return ret;

	}



	return 0;

}



static void __exit map_benchmark_cleanup(void)

{

	platform_driver_unregister(&map_benchmark_platform_driver);

	pci_unregister_driver(&map_benchmark_pci_driver);

}



module_init(map_benchmark_init);

module_exit(map_benchmark_cleanup);



MODULE_AUTHOR("Barry Song <song.bao.hua@hisilicon.com>");

MODULE_DESCRIPTION("dma_map benchmark driver");

MODULE_LICENSE("GPL");

CRA Git | Maintained and supported by SUSTech CRA and CCSE