Commit 7af6fd91 authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Jens Axboe
Browse files

blk-cgroup: introduce a new struct blkg_rwstat_sample 



When sampling the blkcg counts we don't need atomics or per-cpu
variables.  Introduce a new structure just containing plain u64
counters.

Acked-by: default avatarTejun Heo <tj@kernel.org>
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 5d0b6e48

block/bfq-cgroup.c

+4 −6
Original line number Diff line number Diff line
@@ -935,7 +935,7 @@ static u64 bfqg_prfill_stat_recursive(struct seq_file *sf, 
static u64 bfqg_prfill_rwstat_recursive(struct seq_file *sf,

					struct blkg_policy_data *pd, int off)

{

	struct blkg_rwstat sum;

	struct blkg_rwstat_sample sum;



	blkg_rwstat_recursive_sum(pd_to_blkg(pd), &blkcg_policy_bfq, off, &sum);

	return __blkg_prfill_rwstat(sf, pd, &sum);
@@ -975,15 +975,13 @@ static int bfqg_print_stat_sectors(struct seq_file *sf, void *v) 
static u64 bfqg_prfill_sectors_recursive(struct seq_file *sf,

					 struct blkg_policy_data *pd, int off)

{

	struct blkg_rwstat tmp;

	u64 sum;

	struct blkg_rwstat_sample tmp;



	blkg_rwstat_recursive_sum(pd->blkg, NULL,

			offsetof(struct blkcg_gq, stat_bytes), &tmp);

	sum = atomic64_read(&tmp.aux_cnt[BLKG_RWSTAT_READ]) +

		atomic64_read(&tmp.aux_cnt[BLKG_RWSTAT_WRITE]);



	return __blkg_prfill_u64(sf, pd, sum >> 9);

	return __blkg_prfill_u64(sf, pd,

		(tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE]) >> 9);

}



static int bfqg_print_stat_sectors_recursive(struct seq_file *sf, void *v)

block/blk-cgroup.c

+19 −20
Original line number Diff line number Diff line
@@ -544,7 +544,7 @@ EXPORT_SYMBOL_GPL(__blkg_prfill_u64); 
 * Print @rwstat to @sf for the device assocaited with @pd.

 */

u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,

			 const struct blkg_rwstat *rwstat)

			 const struct blkg_rwstat_sample *rwstat)

{

	static const char *rwstr[] = {

		[BLKG_RWSTAT_READ]	= "Read",
@@ -562,12 +562,12 @@ u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, 


	for (i = 0; i < BLKG_RWSTAT_NR; i++)

		seq_printf(sf, "%s %s %llu\n", dname, rwstr[i],

			   (unsigned long long)atomic64_read(&rwstat->aux_cnt[i]));

			   rwstat->cnt[i]);



	v = atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_READ]) +

		atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_WRITE]) +

		atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_DISCARD]);

	seq_printf(sf, "%s Total %llu\n", dname, (unsigned long long)v);

	v = rwstat->cnt[BLKG_RWSTAT_READ] +

		rwstat->cnt[BLKG_RWSTAT_WRITE] +

		rwstat->cnt[BLKG_RWSTAT_DISCARD];

	seq_printf(sf, "%s Total %llu\n", dname, v);

	return v;

}

EXPORT_SYMBOL_GPL(__blkg_prfill_rwstat);
@@ -597,7 +597,7 @@ EXPORT_SYMBOL_GPL(blkg_prfill_stat); 
u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,

		       int off)

{

	struct blkg_rwstat rwstat = { };

	struct blkg_rwstat_sample rwstat = { };



	blkg_rwstat_read((void *)pd + off, &rwstat);

	return __blkg_prfill_rwstat(sf, pd, &rwstat);
@@ -607,7 +607,7 @@ EXPORT_SYMBOL_GPL(blkg_prfill_rwstat); 
static u64 blkg_prfill_rwstat_field(struct seq_file *sf,

				    struct blkg_policy_data *pd, int off)

{

	struct blkg_rwstat rwstat = { };

	struct blkg_rwstat_sample rwstat = { };



	blkg_rwstat_read((void *)pd->blkg + off, &rwstat);

	return __blkg_prfill_rwstat(sf, pd, &rwstat);
@@ -651,7 +651,7 @@ static u64 blkg_prfill_rwstat_field_recursive(struct seq_file *sf, 
					      struct blkg_policy_data *pd,

					      int off)

{

	struct blkg_rwstat rwstat;

	struct blkg_rwstat_sample rwstat;



	blkg_rwstat_recursive_sum(pd->blkg, NULL, off, &rwstat);

	return __blkg_prfill_rwstat(sf, pd, &rwstat);
@@ -734,7 +734,7 @@ EXPORT_SYMBOL_GPL(blkg_stat_recursive_sum); 
 * @blkg: blkg of interest

 * @pol: blkcg_policy which contains the blkg_rwstat

 * @off: offset to the blkg_rwstat in blkg_policy_data or @blkg

 * @sum: blkg_rwstat structure containing the results

 * @sum: blkg_rwstat_sample structure containing the results

 *

 * Collect the blkg_rwstat specified by @blkg, @pol and @off and all its

 * online descendants and their aux counts.  The caller must be holding the
@@ -744,7 +744,7 @@ EXPORT_SYMBOL_GPL(blkg_stat_recursive_sum); 
 * is at @off bytes into @blkg's blkg_policy_data of the policy.

 */

void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol,

		int off, struct blkg_rwstat *sum)

		int off, struct blkg_rwstat_sample *sum)

{

	struct blkcg_gq *pos_blkg;

	struct cgroup_subsys_state *pos_css;
@@ -765,8 +765,7 @@ void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol, 
			rwstat = (void *)pos_blkg + off;



		for (i = 0; i < BLKG_RWSTAT_NR; i++)

			atomic64_set(&sum->aux_cnt[i],

				blkg_rwstat_read_counter(rwstat, i));

			sum->cnt[i] = blkg_rwstat_read_counter(rwstat, i);

	}

	rcu_read_unlock();

}
@@ -934,7 +933,7 @@ static int blkcg_print_stat(struct seq_file *sf, void *v) 
	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {

		const char *dname;

		char *buf;

		struct blkg_rwstat rwstat;

		struct blkg_rwstat_sample rwstat;

		u64 rbytes, wbytes, rios, wios, dbytes, dios;

		size_t size = seq_get_buf(sf, &buf), off = 0;

		int i;
@@ -956,15 +955,15 @@ static int blkcg_print_stat(struct seq_file *sf, void *v) 


		blkg_rwstat_recursive_sum(blkg, NULL,

				offsetof(struct blkcg_gq, stat_bytes), &rwstat);

		rbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);

		wbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);

		dbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_DISCARD]);

		rbytes = rwstat.cnt[BLKG_RWSTAT_READ];

		wbytes = rwstat.cnt[BLKG_RWSTAT_WRITE];

		dbytes = rwstat.cnt[BLKG_RWSTAT_DISCARD];



		blkg_rwstat_recursive_sum(blkg, NULL,

					offsetof(struct blkcg_gq, stat_ios), &rwstat);

		rios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);

		wios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);

		dios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_DISCARD]);

		rios = rwstat.cnt[BLKG_RWSTAT_READ];

		wios = rwstat.cnt[BLKG_RWSTAT_WRITE];

		dios = rwstat.cnt[BLKG_RWSTAT_DISCARD];



		spin_unlock_irq(&blkg->q->queue_lock);

include/linux/blk-cgroup.h

+12 −10
Original line number Diff line number Diff line
@@ -63,8 +63,7 @@ struct blkcg { 


/*

 * blkg_[rw]stat->aux_cnt is excluded for local stats but included for

 * recursive.  Used to carry stats of dead children, and, for blkg_rwstat,

 * to carry result values from read and sum operations.

 * recursive.  Used to carry stats of dead children.

 */

struct blkg_stat {

	struct percpu_counter		cpu_cnt;
@@ -76,6 +75,10 @@ struct blkg_rwstat { 
	atomic64_t			aux_cnt[BLKG_RWSTAT_NR];

};



struct blkg_rwstat_sample {

	u64				cnt[BLKG_RWSTAT_NR];

};



/*

 * A blkcg_gq (blkg) is association between a block cgroup (blkcg) and a

 * request_queue (q).  This is used by blkcg policies which need to track
@@ -213,7 +216,7 @@ void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg, 
		       bool show_total);

u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v);

u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,

			 const struct blkg_rwstat *rwstat);

			 const struct blkg_rwstat_sample *rwstat);

u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off);

u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,

		       int off);
@@ -225,7 +228,7 @@ int blkg_print_stat_ios_recursive(struct seq_file *sf, void *v); 
u64 blkg_stat_recursive_sum(struct blkcg_gq *blkg,

			    struct blkcg_policy *pol, int off);

void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol,

		int off, struct blkg_rwstat *sum);

		int off, struct blkg_rwstat_sample *sum);



struct blkg_conf_ctx {

	struct gendisk			*disk;
@@ -701,13 +704,13 @@ static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat, 
 * Read the current snapshot of @rwstat and return it in the aux counts.

 */

static inline void blkg_rwstat_read(struct blkg_rwstat *rwstat,

		struct blkg_rwstat *result)

		struct blkg_rwstat_sample *result)

{

	int i;



	for (i = 0; i < BLKG_RWSTAT_NR; i++)

		atomic64_set(&result->aux_cnt[i],

			     percpu_counter_sum_positive(&rwstat->cpu_cnt[i]));

		result->cnt[i] =

			percpu_counter_sum_positive(&rwstat->cpu_cnt[i]);

}



/**
@@ -720,11 +723,10 @@ static inline void blkg_rwstat_read(struct blkg_rwstat *rwstat, 
 */

static inline uint64_t blkg_rwstat_total(struct blkg_rwstat *rwstat)

{

	struct blkg_rwstat tmp = { };

	struct blkg_rwstat_sample tmp = { };



	blkg_rwstat_read(rwstat, &tmp);

	return atomic64_read(&tmp.aux_cnt[BLKG_RWSTAT_READ]) +

		atomic64_read(&tmp.aux_cnt[BLKG_RWSTAT_WRITE]);

	return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE];

}



/**

CRA Git | Maintained and supported by SUSTech CRA and CCSE