blk-stat: convert to callback-based statistics reporting (34dbad5d) · Commits · 戴 / test

block/blk-core.c

+5 −1

Original line number	Original line	Diff line number	Diff line
	@@ -852,6 +852,10 @@ static blk_qc_t blk_queue_bio(struct request_queue q, struct bio bio);

	int blk_init_allocated_queue(struct request_queue *q)		int blk_init_allocated_queue(struct request_queue *q)
	{		{
			q->stats = blk_alloc_queue_stats();
			if (!q->stats)
			return -ENOMEM;

	q->fq = blk_alloc_flush_queue(q, NUMA_NO_NODE, q->cmd_size);		q->fq = blk_alloc_flush_queue(q, NUMA_NO_NODE, q->cmd_size);
	if (!q->fq)		if (!q->fq)
	return -ENOMEM;		return -ENOMEM;
	@@ -2698,7 +2702,7 @@ void blk_finish_request(struct request *req, int error)
	struct request_queue *q = req->q;		struct request_queue *q = req->q;

	if (req->rq_flags & RQF_STATS)		if (req->rq_flags & RQF_STATS)
	blk_stat_add(&q->rq_stats[rq_data_dir(req)], req);		blk_stat_add(req);

	if (req->rq_flags & RQF_QUEUED)		if (req->rq_flags & RQF_QUEUED)
	blk_queue_end_tag(q, req);		blk_queue_end_tag(q, req);

block/blk-mq-debugfs.c

+44 −55

Original line number	Original line	Diff line number	Diff line
	@@ -43,6 +43,42 @@ static int blk_mq_debugfs_seq_open(struct inode inode, struct file file,
	return ret;		return ret;
	}		}

			static void print_stat(struct seq_file m, struct blk_rq_stat stat)
			{
			if (stat->nr_samples) {
			seq_printf(m, "samples=%d, mean=%lld, min=%llu, max=%llu",
			stat->nr_samples, stat->mean, stat->min, stat->max);
			} else {
			seq_puts(m, "samples=0");
			}
			}

			static int queue_poll_stat_show(struct seq_file m, void v)
			{
			struct request_queue *q = m->private;

			seq_puts(m, "read: ");
			print_stat(m, &q->poll_stat[READ]);
			seq_puts(m, "\n");

			seq_puts(m, "write: ");
			print_stat(m, &q->poll_stat[WRITE]);
			seq_puts(m, "\n");
			return 0;
			}

			static int queue_poll_stat_open(struct inode inode, struct file file)
			{
			return single_open(file, queue_poll_stat_show, inode->i_private);
			}

			static const struct file_operations queue_poll_stat_fops = {
			.open = queue_poll_stat_open,
			.read = seq_read,
			.llseek = seq_lseek,
			.release = single_release,
			};

	static int hctx_state_show(struct seq_file m, void v)		static int hctx_state_show(struct seq_file m, void v)
	{		{
	struct blk_mq_hw_ctx *hctx = m->private;		struct blk_mq_hw_ctx *hctx = m->private;
	@@ -322,60 +358,6 @@ static const struct file_operations hctx_io_poll_fops = {
	.release = single_release,		.release = single_release,
	};		};

	static void print_stat(struct seq_file m, struct blk_rq_stat stat)
	{
	seq_printf(m, "samples=%d, mean=%lld, min=%llu, max=%llu",
	stat->nr_samples, stat->mean, stat->min, stat->max);
	}

	static int hctx_stats_show(struct seq_file m, void v)
	{
	struct blk_mq_hw_ctx *hctx = m->private;
	struct blk_rq_stat stat[2];

	blk_stat_init(&stat[READ]);
	blk_stat_init(&stat[WRITE]);

	blk_hctx_stat_get(hctx, stat);

	seq_puts(m, "read: ");
	print_stat(m, &stat[READ]);
	seq_puts(m, "\n");

	seq_puts(m, "write: ");
	print_stat(m, &stat[WRITE]);
	seq_puts(m, "\n");
	return 0;
	}

	static int hctx_stats_open(struct inode inode, struct file file)
	{
	return single_open(file, hctx_stats_show, inode->i_private);
	}

	static ssize_t hctx_stats_write(struct file file, const char __user buf,
	size_t count, loff_t *ppos)
	{
	struct seq_file *m = file->private_data;
	struct blk_mq_hw_ctx *hctx = m->private;
	struct blk_mq_ctx *ctx;
	int i;

	hctx_for_each_ctx(hctx, ctx, i) {
	blk_stat_init(&ctx->stat[READ]);
	blk_stat_init(&ctx->stat[WRITE]);
	}
	return count;
	}

	static const struct file_operations hctx_stats_fops = {
	.open = hctx_stats_open,
	.read = seq_read,
	.write = hctx_stats_write,
	.llseek = seq_lseek,
	.release = single_release,
	};

	static int hctx_dispatched_show(struct seq_file m, void v)		static int hctx_dispatched_show(struct seq_file m, void v)
	{		{
	struct blk_mq_hw_ctx *hctx = m->private;		struct blk_mq_hw_ctx *hctx = m->private;
	@@ -636,6 +618,11 @@ static const struct file_operations ctx_completed_fops = {
	.release = single_release,		.release = single_release,
	};		};

			static const struct blk_mq_debugfs_attr blk_mq_debugfs_queue_attrs[] = {
			{"poll_stat", 0400, &queue_poll_stat_fops},
			{},
			};

	static const struct blk_mq_debugfs_attr blk_mq_debugfs_hctx_attrs[] = {		static const struct blk_mq_debugfs_attr blk_mq_debugfs_hctx_attrs[] = {
	{"state", 0400, &hctx_state_fops},		{"state", 0400, &hctx_state_fops},
	{"flags", 0400, &hctx_flags_fops},		{"flags", 0400, &hctx_flags_fops},
	@@ -646,7 +633,6 @@ static const struct blk_mq_debugfs_attr blk_mq_debugfs_hctx_attrs[] = {
	{"sched_tags", 0400, &hctx_sched_tags_fops},		{"sched_tags", 0400, &hctx_sched_tags_fops},
	{"sched_tags_bitmap", 0400, &hctx_sched_tags_bitmap_fops},		{"sched_tags_bitmap", 0400, &hctx_sched_tags_bitmap_fops},
	{"io_poll", 0600, &hctx_io_poll_fops},		{"io_poll", 0600, &hctx_io_poll_fops},
	{"stats", 0600, &hctx_stats_fops},
	{"dispatched", 0600, &hctx_dispatched_fops},		{"dispatched", 0600, &hctx_dispatched_fops},
	{"queued", 0600, &hctx_queued_fops},		{"queued", 0600, &hctx_queued_fops},
	{"run", 0600, &hctx_run_fops},		{"run", 0600, &hctx_run_fops},
	@@ -753,6 +739,9 @@ int blk_mq_debugfs_register_hctxs(struct request_queue *q)
	if (!q->mq_debugfs_dir)		if (!q->mq_debugfs_dir)
	goto err;		goto err;

			if (!debugfs_create_files(q->mq_debugfs_dir, q, blk_mq_debugfs_queue_attrs))
			goto err;

	queue_for_each_hw_ctx(q, hctx, i) {		queue_for_each_hw_ctx(q, hctx, i) {
	if (blk_mq_debugfs_register_hctx(q, hctx))		if (blk_mq_debugfs_register_hctx(q, hctx))
	goto err;		goto err;

block/blk-mq.c

+52 −24

Original line number	Original line	Diff line number	Diff line
	@@ -39,6 +39,9 @@
	static DEFINE_MUTEX(all_q_mutex);		static DEFINE_MUTEX(all_q_mutex);
	static LIST_HEAD(all_q_list);		static LIST_HEAD(all_q_list);

			static void blk_mq_poll_stats_start(struct request_queue *q);
			static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb);

	/*		/*
	* Check if any of the ctx's have pending work in this hardware queue		* Check if any of the ctx's have pending work in this hardware queue
	*/		*/
	@@ -432,15 +435,8 @@ static void blk_mq_ipi_complete_request(struct request *rq)
	static void blk_mq_stat_add(struct request *rq)		static void blk_mq_stat_add(struct request *rq)
	{		{
	if (rq->rq_flags & RQF_STATS) {		if (rq->rq_flags & RQF_STATS) {
	/*		blk_mq_poll_stats_start(rq->q);
	* We could rq->mq_ctx here, but there's less of a risk		blk_stat_add(rq);
	* of races if we have the completion event add the stats
	* to the local software queue.
	*/
	struct blk_mq_ctx *ctx;

	ctx = __blk_mq_get_ctx(rq->q, raw_smp_processor_id());
	blk_stat_add(&ctx->stat[rq_data_dir(rq)], rq);
	}		}
	}		}

	@@ -2040,8 +2036,6 @@ static void blk_mq_init_cpu_queues(struct request_queue *q,
	spin_lock_init(&__ctx->lock);		spin_lock_init(&__ctx->lock);
	INIT_LIST_HEAD(&__ctx->rq_list);		INIT_LIST_HEAD(&__ctx->rq_list);
	__ctx->queue = q;		__ctx->queue = q;
	blk_stat_init(&__ctx->stat[READ]);
	blk_stat_init(&__ctx->stat[WRITE]);

	/* If the cpu isn't online, the cpu is mapped to first hctx */		/* If the cpu isn't online, the cpu is mapped to first hctx */
	if (!cpu_online(i))		if (!cpu_online(i))
	@@ -2339,6 +2333,15 @@ struct request_queue blk_mq_init_allocated_queue(struct blk_mq_tag_set set,
	/* mark the queue as mq asap */		/* mark the queue as mq asap */
	q->mq_ops = set->ops;		q->mq_ops = set->ops;

			q->stats = blk_alloc_queue_stats();
			if (!q->stats)
			goto err_exit;

			q->poll_cb = blk_stat_alloc_callback(blk_mq_poll_stats_fn,
			blk_stat_rq_ddir, 2, q);
			if (!q->poll_cb)
			goto err_exit;

	q->queue_ctx = alloc_percpu(struct blk_mq_ctx);		q->queue_ctx = alloc_percpu(struct blk_mq_ctx);
	if (!q->queue_ctx)		if (!q->queue_ctx)
	goto err_exit;		goto err_exit;
	@@ -2740,27 +2743,52 @@ void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
	}		}
	EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);		EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);

			/* Enable polling stats and return whether they were already enabled. */
			static bool blk_poll_stats_enable(struct request_queue *q)
			{
			if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) \|\|
			test_and_set_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags))
			return true;
			blk_stat_add_callback(q, q->poll_cb);
			return false;
			}

			static void blk_mq_poll_stats_start(struct request_queue *q)
			{
			/*
			* We don't arm the callback if polling stats are not enabled or the
			* callback is already active.
			*/
			if (!test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) \|\|
			blk_stat_is_active(q->poll_cb))
			return;

			blk_stat_activate_msecs(q->poll_cb, 100);
			}

			static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb)
			{
			struct request_queue *q = cb->data;

			if (cb->stat[READ].nr_samples)
			q->poll_stat[READ] = cb->stat[READ];
			if (cb->stat[WRITE].nr_samples)
			q->poll_stat[WRITE] = cb->stat[WRITE];
			}

	static unsigned long blk_mq_poll_nsecs(struct request_queue *q,		static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
	struct blk_mq_hw_ctx *hctx,		struct blk_mq_hw_ctx *hctx,
	struct request *rq)		struct request *rq)
	{		{
	struct blk_rq_stat stat[2];
	unsigned long ret = 0;		unsigned long ret = 0;

	/*		/*
	* If stats collection isn't on, don't sleep but turn it on for		* If stats collection isn't on, don't sleep but turn it on for
	* future users		* future users
	*/		*/
	if (!blk_stat_enable(q))		if (!blk_poll_stats_enable(q))
	return 0;		return 0;

	/*
	* We don't have to do this once per IO, should optimize this
	* to just use the current window of stats until it changes
	*/
	memset(&stat, 0, sizeof(stat));
	blk_hctx_stat_get(hctx, stat);

	/*		/*
	* As an optimistic guess, use half of the mean service time		* As an optimistic guess, use half of the mean service time
	* for this type of request. We can (and should) make this smarter.		* for this type of request. We can (and should) make this smarter.
	@@ -2769,10 +2797,10 @@ static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
	* important on devices where the completion latencies are longer		* important on devices where the completion latencies are longer
	* than ~10 usec.		* than ~10 usec.
	*/		*/
	if (req_op(rq) == REQ_OP_READ && stat[READ].nr_samples)		if (req_op(rq) == REQ_OP_READ && q->poll_stat[READ].nr_samples)
	ret = (stat[READ].mean + 1) / 2;		ret = (q->poll_stat[READ].mean + 1) / 2;
	else if (req_op(rq) == REQ_OP_WRITE && stat[WRITE].nr_samples)		else if (req_op(rq) == REQ_OP_WRITE && q->poll_stat[WRITE].nr_samples)
	ret = (stat[WRITE].mean + 1) / 2;		ret = (q->poll_stat[WRITE].mean + 1) / 2;

	return ret;		return ret;
	}		}

block/blk-mq.h

+0 −1

Original line number	Original line	Diff line number	Diff line
	@@ -20,7 +20,6 @@ struct blk_mq_ctx {

	/* incremented at completion time */		/* incremented at completion time */
	unsigned long ____cacheline_aligned_in_smp rq_completed[2];		unsigned long ____cacheline_aligned_in_smp rq_completed[2];
	struct blk_rq_stat stat[2];

	struct request_queue *queue;		struct request_queue *queue;
	struct kobject kobj;		struct kobject kobj;

block/blk-stat.c

+147 −164

Original line number	Original line	Diff line number	Diff line
	@@ -4,6 +4,7 @@
	* Copyright (C) 2016 Jens Axboe		* Copyright (C) 2016 Jens Axboe
	*/		*/
	#include <linux/kernel.h>		#include <linux/kernel.h>
			#include <linux/rculist.h>
	#include <linux/blk-mq.h>		#include <linux/blk-mq.h>

	#include "blk-stat.h"		#include "blk-stat.h"
	@@ -11,6 +12,24 @@

	#define BLK_RQ_STAT_BATCH 64		#define BLK_RQ_STAT_BATCH 64

			struct blk_queue_stats {
			struct list_head callbacks;
			spinlock_t lock;
			};

			unsigned int blk_stat_rq_ddir(const struct request *rq)
			{
			return rq_data_dir(rq);
			}
			EXPORT_SYMBOL_GPL(blk_stat_rq_ddir);

			static void blk_stat_init(struct blk_rq_stat *stat)
			{
			stat->min = -1ULL;
			stat->max = stat->nr_samples = stat->mean = 0;
			stat->batch = stat->nr_batch = 0;
			}

	static void blk_stat_flush_batch(struct blk_rq_stat *stat)		static void blk_stat_flush_batch(struct blk_rq_stat *stat)
	{		{
	const s32 nr_batch = READ_ONCE(stat->nr_batch);		const s32 nr_batch = READ_ONCE(stat->nr_batch);
	@@ -50,207 +69,171 @@ static void blk_stat_sum(struct blk_rq_stat dst, struct blk_rq_stat src)
	dst->nr_samples += src->nr_samples;		dst->nr_samples += src->nr_samples;
	}		}

	static void blk_mq_stat_get(struct request_queue q, struct blk_rq_stat dst)		static void __blk_stat_add(struct blk_rq_stat *stat, u64 value)
	{		{
	struct blk_mq_hw_ctx *hctx;		stat->min = min(stat->min, value);
	struct blk_mq_ctx *ctx;		stat->max = max(stat->max, value);
	uint64_t latest = 0;
	int i, j, nr;

	blk_stat_init(&dst[READ]);
	blk_stat_init(&dst[WRITE]);

	nr = 0;
	do {
	uint64_t newest = 0;

	queue_for_each_hw_ctx(q, hctx, i) {
	hctx_for_each_ctx(hctx, ctx, j) {
	blk_stat_flush_batch(&ctx->stat[READ]);
	blk_stat_flush_batch(&ctx->stat[WRITE]);

	if (!ctx->stat[READ].nr_samples &&
	!ctx->stat[WRITE].nr_samples)
	continue;
	if (ctx->stat[READ].time > newest)
	newest = ctx->stat[READ].time;
	if (ctx->stat[WRITE].time > newest)
	newest = ctx->stat[WRITE].time;
	}
	}

	/*		if (stat->batch + value < stat->batch \|\|
	* No samples		stat->nr_batch + 1 == BLK_RQ_STAT_BATCH)
	*/		blk_stat_flush_batch(stat);
	if (!newest)
	break;

	if (newest > latest)
	latest = newest;

	queue_for_each_hw_ctx(q, hctx, i) {
	hctx_for_each_ctx(hctx, ctx, j) {
	if (ctx->stat[READ].time == newest) {
	blk_stat_sum(&dst[READ],
	&ctx->stat[READ]);
	nr++;
	}
	if (ctx->stat[WRITE].time == newest) {
	blk_stat_sum(&dst[WRITE],
	&ctx->stat[WRITE]);
	nr++;
	}
	}
	}
	/*
	* If we race on finding an entry, just loop back again.
	* Should be very rare.
	*/
	} while (!nr);

	dst[READ].time = dst[WRITE].time = latest;		stat->batch += value;
			stat->nr_batch++;
	}		}

	void blk_queue_stat_get(struct request_queue q, struct blk_rq_stat dst)		void blk_stat_add(struct request *rq)
	{		{
	if (q->mq_ops)		struct request_queue *q = rq->q;
	blk_mq_stat_get(q, dst);		struct blk_stat_callback *cb;
	else {		struct blk_rq_stat *stat;
	blk_stat_flush_batch(&q->rq_stats[READ]);		int bucket;
	blk_stat_flush_batch(&q->rq_stats[WRITE]);		s64 now, value;
	memcpy(&dst[READ], &q->rq_stats[READ],
	sizeof(struct blk_rq_stat));		now = __blk_stat_time(ktime_to_ns(ktime_get()));
	memcpy(&dst[WRITE], &q->rq_stats[WRITE],		if (now < blk_stat_time(&rq->issue_stat))
	sizeof(struct blk_rq_stat));		return;

			value = now - blk_stat_time(&rq->issue_stat);

			rcu_read_lock();
			list_for_each_entry_rcu(cb, &q->stats->callbacks, list) {
			if (blk_stat_is_active(cb)) {
			bucket = cb->bucket_fn(rq);
			stat = &this_cpu_ptr(cb->cpu_stat)[bucket];
			__blk_stat_add(stat, value);
			}
	}		}
			rcu_read_unlock();
	}		}

	void blk_hctx_stat_get(struct blk_mq_hw_ctx hctx, struct blk_rq_stat dst)		static void blk_stat_timer_fn(unsigned long data)
	{		{
	struct blk_mq_ctx *ctx;		struct blk_stat_callback cb = (void )data;
	unsigned int i, nr;		unsigned int bucket;
			int cpu;

	nr = 0;		for (bucket = 0; bucket < cb->buckets; bucket++)
	do {		blk_stat_init(&cb->stat[bucket]);
	uint64_t newest = 0;

	hctx_for_each_ctx(hctx, ctx, i) {		for_each_online_cpu(cpu) {
	blk_stat_flush_batch(&ctx->stat[READ]);		struct blk_rq_stat *cpu_stat;
	blk_stat_flush_batch(&ctx->stat[WRITE]);

	if (!ctx->stat[READ].nr_samples &&		cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);
	!ctx->stat[WRITE].nr_samples)		for (bucket = 0; bucket < cb->buckets; bucket++) {
	continue;		blk_stat_sum(&cb->stat[bucket], &cpu_stat[bucket]);
			blk_stat_init(&cpu_stat[bucket]);
			}
			}

	if (ctx->stat[READ].time > newest)		cb->timer_fn(cb);
	newest = ctx->stat[READ].time;
	if (ctx->stat[WRITE].time > newest)
	newest = ctx->stat[WRITE].time;
	}		}

	if (!newest)		struct blk_stat_callback *
	break;		blk_stat_alloc_callback(void (timer_fn)(struct blk_stat_callback ),
			unsigned int (bucket_fn)(const struct request ),
			unsigned int buckets, void *data)
			{
			struct blk_stat_callback *cb;

	hctx_for_each_ctx(hctx, ctx, i) {		cb = kmalloc(sizeof(*cb), GFP_KERNEL);
	if (ctx->stat[READ].time == newest) {		if (!cb)
	blk_stat_sum(&dst[READ], &ctx->stat[READ]);		return NULL;
	nr++;
	}		cb->stat = kmalloc_array(buckets, sizeof(struct blk_rq_stat),
	if (ctx->stat[WRITE].time == newest) {		GFP_KERNEL);
	blk_stat_sum(&dst[WRITE], &ctx->stat[WRITE]);		if (!cb->stat) {
	nr++;		kfree(cb);
	}		return NULL;
	}		}
	/*		cb->cpu_stat = __alloc_percpu(buckets * sizeof(struct blk_rq_stat),
	* If we race on finding an entry, just loop back again.		__alignof__(struct blk_rq_stat));
	* Should be very rare, as the window is only updated		if (!cb->cpu_stat) {
	* occasionally		kfree(cb->stat);
	*/		kfree(cb);
	} while (!nr);		return NULL;
	}		}

	static void __blk_stat_init(struct blk_rq_stat *stat, s64 time_now)		cb->timer_fn = timer_fn;
	{		cb->bucket_fn = bucket_fn;
	stat->min = -1ULL;		cb->data = data;
	stat->max = stat->nr_samples = stat->mean = 0;		cb->buckets = buckets;
	stat->batch = stat->nr_batch = 0;		setup_timer(&cb->timer, blk_stat_timer_fn, (unsigned long)cb);
	stat->time = time_now & BLK_STAT_NSEC_MASK;
	}

	void blk_stat_init(struct blk_rq_stat *stat)		return cb;
	{
	__blk_stat_init(stat, ktime_to_ns(ktime_get()));
	}		}
			EXPORT_SYMBOL_GPL(blk_stat_alloc_callback);

	static bool __blk_stat_is_current(struct blk_rq_stat *stat, s64 now)		void blk_stat_add_callback(struct request_queue *q,
			struct blk_stat_callback *cb)
	{		{
	return (now & BLK_STAT_NSEC_MASK) == (stat->time & BLK_STAT_NSEC_MASK);		unsigned int bucket;
			int cpu;

			for_each_possible_cpu(cpu) {
			struct blk_rq_stat *cpu_stat;

			cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);
			for (bucket = 0; bucket < cb->buckets; bucket++)
			blk_stat_init(&cpu_stat[bucket]);
	}		}

	bool blk_stat_is_current(struct blk_rq_stat *stat)		spin_lock(&q->stats->lock);
	{		list_add_tail_rcu(&cb->list, &q->stats->callbacks);
	return __blk_stat_is_current(stat, ktime_to_ns(ktime_get()));		set_bit(QUEUE_FLAG_STATS, &q->queue_flags);
			spin_unlock(&q->stats->lock);
	}		}
			EXPORT_SYMBOL_GPL(blk_stat_add_callback);

	void blk_stat_add(struct blk_rq_stat stat, struct request rq)		void blk_stat_remove_callback(struct request_queue *q,
			struct blk_stat_callback *cb)
	{		{
	s64 now, value;		spin_lock(&q->stats->lock);
			list_del_rcu(&cb->list);
	now = __blk_stat_time(ktime_to_ns(ktime_get()));		if (list_empty(&q->stats->callbacks))
	if (now < blk_stat_time(&rq->issue_stat))		clear_bit(QUEUE_FLAG_STATS, &q->queue_flags);
	return;		spin_unlock(&q->stats->lock);

	if (!__blk_stat_is_current(stat, now))		del_timer_sync(&cb->timer);
	__blk_stat_init(stat, now);		}
			EXPORT_SYMBOL_GPL(blk_stat_remove_callback);
	value = now - blk_stat_time(&rq->issue_stat);
	if (value > stat->max)
	stat->max = value;
	if (value < stat->min)
	stat->min = value;

	if (stat->batch + value < stat->batch \|\|		static void blk_stat_free_callback_rcu(struct rcu_head *head)
	stat->nr_batch + 1 == BLK_RQ_STAT_BATCH)		{
	blk_stat_flush_batch(stat);		struct blk_stat_callback *cb;

	stat->batch += value;		cb = container_of(head, struct blk_stat_callback, rcu);
	stat->nr_batch++;		free_percpu(cb->cpu_stat);
			kfree(cb->stat);
			kfree(cb);
	}		}

	void blk_stat_clear(struct request_queue *q)		void blk_stat_free_callback(struct blk_stat_callback *cb)
	{		{
	if (q->mq_ops) {		call_rcu(&cb->rcu, blk_stat_free_callback_rcu);
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *ctx;
	int i, j;

	queue_for_each_hw_ctx(q, hctx, i) {
	hctx_for_each_ctx(hctx, ctx, j) {
	blk_stat_init(&ctx->stat[READ]);
	blk_stat_init(&ctx->stat[WRITE]);
	}
	}
	} else {
	blk_stat_init(&q->rq_stats[READ]);
	blk_stat_init(&q->rq_stats[WRITE]);
	}
	}		}
			EXPORT_SYMBOL_GPL(blk_stat_free_callback);

	void blk_stat_set_issue_time(struct blk_issue_stat *stat)		struct blk_queue_stats *blk_alloc_queue_stats(void)
	{		{
	stat->time = (stat->time & BLK_STAT_MASK) \|		struct blk_queue_stats *stats;
	(ktime_to_ns(ktime_get()) & BLK_STAT_TIME_MASK);
			stats = kmalloc(sizeof(*stats), GFP_KERNEL);
			if (!stats)
			return NULL;

			INIT_LIST_HEAD(&stats->callbacks);
			spin_lock_init(&stats->lock);

			return stats;
	}		}

	/*		void blk_free_queue_stats(struct blk_queue_stats *stats)
	* Enable stat tracking, return whether it was enabled
	*/
	bool blk_stat_enable(struct request_queue *q)
	{		{
	if (!test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {		if (!stats)
	set_bit(QUEUE_FLAG_STATS, &q->queue_flags);		return;
	return false;
	}		WARN_ON(!list_empty(&stats->callbacks));

	return true;		kfree(stats);
	}		}

Admin message