blk-iocost: use local[64]_t for percpu stat (5e124f74) · Commits · 戴 / test

block/blk-iocost.c

+27 −10

Original line number	Diff line number	Diff line
		@@ -179,6 +179,8 @@
		#include <linux/parser.h>
		#include <linux/sched/signal.h>
		#include <linux/blk-cgroup.h>
		#include <asm/local.h>
		#include <asm/local64.h>
		#include "blk-rq-qos.h"
		#include "blk-stat.h"
		#include "blk-wbt.h"
		@@ -373,8 +375,8 @@ struct ioc_params {
		};

		struct ioc_missed {
		u32 nr_met;
		u32 nr_missed;
		local_t nr_met;
		local_t nr_missed;
		u32 last_met;
		u32 last_missed;
		};
		@@ -382,7 +384,7 @@ struct ioc_missed {
		struct ioc_pcpu_stat {
		struct ioc_missed missed[2];

		u64 rq_wait_ns;
		local64_t rq_wait_ns;
		u64 last_rq_wait_ns;
		};

		@@ -1278,8 +1280,8 @@ static void ioc_lat_stat(struct ioc ioc, u32 missed_ppm_ar, u32 *rq_wait_pct_p
		u64 this_rq_wait_ns;

		for (rw = READ; rw <= WRITE; rw++) {
		u32 this_met = READ_ONCE(stat->missed[rw].nr_met);
		u32 this_missed = READ_ONCE(stat->missed[rw].nr_missed);
		u32 this_met = local_read(&stat->missed[rw].nr_met);
		u32 this_missed = local_read(&stat->missed[rw].nr_missed);

		nr_met[rw] += this_met - stat->missed[rw].last_met;
		nr_missed[rw] += this_missed - stat->missed[rw].last_missed;
		@@ -1287,7 +1289,7 @@ static void ioc_lat_stat(struct ioc ioc, u32 missed_ppm_ar, u32 *rq_wait_pct_p
		stat->missed[rw].last_missed = this_missed;
		}

		this_rq_wait_ns = READ_ONCE(stat->rq_wait_ns);
		this_rq_wait_ns = local64_read(&stat->rq_wait_ns);
		rq_wait_ns += this_rq_wait_ns - stat->last_rq_wait_ns;
		stat->last_rq_wait_ns = this_rq_wait_ns;
		}
		@@ -1908,6 +1910,7 @@ static void ioc_rqos_done_bio(struct rq_qos rqos, struct bio bio)
		static void ioc_rqos_done(struct rq_qos rqos, struct request rq)
		{
		struct ioc *ioc = rqos_to_ioc(rqos);
		struct ioc_pcpu_stat *ccs;
		u64 on_q_ns, rq_wait_ns, size_nsec;
		int pidx, rw;

		@@ -1931,13 +1934,17 @@ static void ioc_rqos_done(struct rq_qos rqos, struct request rq)
		rq_wait_ns = rq->start_time_ns - rq->alloc_time_ns;
		size_nsec = div64_u64(calc_size_vtime_cost(rq, ioc), VTIME_PER_NSEC);

		ccs = get_cpu_ptr(ioc->pcpu_stat);

		if (on_q_ns <= size_nsec \|\|
		on_q_ns - size_nsec <= ioc->params.qos[pidx] * NSEC_PER_USEC)
		this_cpu_inc(ioc->pcpu_stat->missed[rw].nr_met);
		local_inc(&ccs->missed[rw].nr_met);
		else
		this_cpu_inc(ioc->pcpu_stat->missed[rw].nr_missed);
		local_inc(&ccs->missed[rw].nr_missed);

		local64_add(rq_wait_ns, &ccs->rq_wait_ns);

		this_cpu_add(ioc->pcpu_stat->rq_wait_ns, rq_wait_ns);
		put_cpu_ptr(ccs);
		}

		static void ioc_rqos_queue_depth_changed(struct rq_qos *rqos)
		@@ -1977,7 +1984,7 @@ static int blk_iocost_init(struct request_queue *q)
		{
		struct ioc *ioc;
		struct rq_qos *rqos;
		int ret;
		int i, cpu, ret;

		ioc = kzalloc(sizeof(*ioc), GFP_KERNEL);
		if (!ioc)
		@@ -1989,6 +1996,16 @@ static int blk_iocost_init(struct request_queue *q)
		return -ENOMEM;
		}

		for_each_possible_cpu(cpu) {
		struct ioc_pcpu_stat *ccs = per_cpu_ptr(ioc->pcpu_stat, cpu);

		for (i = 0; i < ARRAY_SIZE(ccs->missed); i++) {
		local_set(&ccs->missed[i].nr_met, 0);
		local_set(&ccs->missed[i].nr_missed, 0);
		}
		local64_set(&ccs->rq_wait_ns, 0);
		}

		rqos = &ioc->rqos;
		rqos->id = RQ_QOS_COST;
		rqos->ops = &ioc_rqos_ops;

Admin message