Commit 8a3d2615 authored by Tejun Heo's avatar Tejun Heo
Browse files

blkcg: move blkio_group_stats_cpu and friends to blk-throttle.c 



blkio_group_stats_cpu is used only by blk-throtl and has no reason to
be defined in blkcg core.

* Move blkio_group_stats_cpu to blk-throttle.c and rename it to
  tg_stats_cpu.

* blkg_policy_data->stats_cpu is replaced with throtl_grp->stats_cpu.
  prfill functions updated accordingly.

* All related macros / functions are renamed so that they have tg_
  prefix and the unnecessary @pol arguments are dropped.

* Per-cpu stats allocation code is also moved from blk-cgroup.c to
  blk-throttle.c and gets simplified to only deal with
  BLKIO_POLICY_THROTL.  percpu stat free is performed by the exit
  method throtl_exit_blkio_group().

* throtl_reset_group_stats() implemented for
  blkio_reset_group_stats_fn method so that tg->stats_cpu can be
  reset.

Signed-off-by: default avatarTejun Heo <tj@kernel.org>
parent 155fead9

block/blk-cgroup.c

+1 −97
Original line number Diff line number Diff line
@@ -30,13 +30,6 @@ static LIST_HEAD(blkio_list); 
static DEFINE_MUTEX(all_q_mutex);

static LIST_HEAD(all_q_list);



/* List of groups pending per cpu stats allocation */

static DEFINE_SPINLOCK(alloc_list_lock);

static LIST_HEAD(alloc_list);



static void blkio_stat_alloc_fn(struct work_struct *);

static DECLARE_DELAYED_WORK(blkio_stat_alloc_work, blkio_stat_alloc_fn);



struct blkio_cgroup blkio_root_cgroup = { .weight = 2*BLKIO_WEIGHT_DEFAULT };

EXPORT_SYMBOL_GPL(blkio_root_cgroup);
@@ -63,60 +56,6 @@ struct blkio_cgroup *bio_blkio_cgroup(struct bio *bio) 
}

EXPORT_SYMBOL_GPL(bio_blkio_cgroup);



/*

 * Worker for allocating per cpu stat for blk groups. This is scheduled on

 * the system_nrt_wq once there are some groups on the alloc_list waiting

 * for allocation.

 */

static void blkio_stat_alloc_fn(struct work_struct *work)

{

	static void *pcpu_stats[BLKIO_NR_POLICIES];

	struct delayed_work *dwork = to_delayed_work(work);

	struct blkio_group *blkg;

	int i;

	bool empty = false;



alloc_stats:

	for (i = 0; i < BLKIO_NR_POLICIES; i++) {

		if (pcpu_stats[i] != NULL)

			continue;



		pcpu_stats[i] = alloc_percpu(struct blkio_group_stats_cpu);



		/* Allocation failed. Try again after some time. */

		if (pcpu_stats[i] == NULL) {

			queue_delayed_work(system_nrt_wq, dwork,

						msecs_to_jiffies(10));

			return;

		}

	}



	spin_lock_irq(&blkio_list_lock);

	spin_lock(&alloc_list_lock);



	/* cgroup got deleted or queue exited. */

	if (!list_empty(&alloc_list)) {

		blkg = list_first_entry(&alloc_list, struct blkio_group,

						alloc_node);

		for (i = 0; i < BLKIO_NR_POLICIES; i++) {

			struct blkg_policy_data *pd = blkg->pd[i];



			if (blkio_policy[i] && pd && !pd->stats_cpu)

				swap(pd->stats_cpu, pcpu_stats[i]);

		}



		list_del_init(&blkg->alloc_node);

	}



	empty = list_empty(&alloc_list);



	spin_unlock(&alloc_list_lock);

	spin_unlock_irq(&blkio_list_lock);



	if (!empty)

		goto alloc_stats;

}



/**

 * blkg_free - free a blkg

 * @blkg: blkg to free
@@ -140,7 +79,6 @@ static void blkg_free(struct blkio_group *blkg) 
		if (pol && pol->ops.blkio_exit_group_fn)

			pol->ops.blkio_exit_group_fn(blkg);



		free_percpu(pd->stats_cpu);

		kfree(pd);

	}
@@ -167,7 +105,6 @@ static struct blkio_group *blkg_alloc(struct blkio_cgroup *blkcg, 


	blkg->q = q;

	INIT_LIST_HEAD(&blkg->q_node);

	INIT_LIST_HEAD(&blkg->alloc_node);

	blkg->blkcg = blkcg;

	blkg->refcnt = 1;

	cgroup_path(blkcg->css.cgroup, blkg->path, sizeof(blkg->path));
@@ -245,12 +182,6 @@ struct blkio_group *blkg_lookup_create(struct blkio_cgroup *blkcg, 
	hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);

	list_add(&blkg->q_node, &q->blkg_list);

	spin_unlock(&blkcg->lock);



	spin_lock(&alloc_list_lock);

	list_add(&blkg->alloc_node, &alloc_list);

	/* Queue per cpu stat allocation from worker thread. */

	queue_delayed_work(system_nrt_wq, &blkio_stat_alloc_work, 0);

	spin_unlock(&alloc_list_lock);

out:

	return blkg;

}
@@ -284,10 +215,6 @@ static void blkg_destroy(struct blkio_group *blkg) 
	list_del_init(&blkg->q_node);

	hlist_del_init_rcu(&blkg->blkcg_node);



	spin_lock(&alloc_list_lock);

	list_del_init(&blkg->alloc_node);

	spin_unlock(&alloc_list_lock);



	/*

	 * Put the reference taken at the time of creation so that when all

	 * queues are gone, group can be destroyed.
@@ -319,9 +246,6 @@ void update_root_blkg_pd(struct request_queue *q, enum blkio_policy_id plid) 
	pd = kzalloc(sizeof(*pd) + pol->pdata_size, GFP_KERNEL);

	WARN_ON_ONCE(!pd);



	pd->stats_cpu = alloc_percpu(struct blkio_group_stats_cpu);

	WARN_ON_ONCE(!pd->stats_cpu);



	blkg->pd[plid] = pd;

	pd->blkg = blkg;

	pol->ops.blkio_init_group_fn(blkg);
@@ -381,23 +305,6 @@ void __blkg_release(struct blkio_group *blkg) 
}

EXPORT_SYMBOL_GPL(__blkg_release);



static void blkio_reset_stats_cpu(struct blkio_group *blkg, int plid)

{

	struct blkg_policy_data *pd = blkg->pd[plid];

	int cpu;



	if (pd->stats_cpu == NULL)

		return;



	for_each_possible_cpu(cpu) {

		struct blkio_group_stats_cpu *sc =

			per_cpu_ptr(pd->stats_cpu, cpu);



		blkg_rwstat_reset(&sc->service_bytes);

		blkg_rwstat_reset(&sc->serviced);

	}

}



static int

blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val)

{
@@ -416,13 +323,10 @@ blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val) 
	hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {

		struct blkio_policy_type *pol;



		list_for_each_entry(pol, &blkio_list, list) {

			blkio_reset_stats_cpu(blkg, pol->plid);



		list_for_each_entry(pol, &blkio_list, list)

			if (pol->ops.blkio_reset_group_stats_fn)

				pol->ops.blkio_reset_group_stats_fn(blkg);

	}

	}



	spin_unlock_irq(&blkcg->lock);

	spin_unlock(&blkio_list_lock);

block/blk-cgroup.h

+0 −13
Original line number Diff line number Diff line
@@ -64,14 +64,6 @@ struct blkg_rwstat { 
	uint64_t			cnt[BLKG_RWSTAT_NR];

};



/* Per cpu blkio group stats */

struct blkio_group_stats_cpu {

	/* total bytes transferred */

	struct blkg_rwstat		service_bytes;

	/* total IOs serviced, post merge */

	struct blkg_rwstat		serviced;

};



struct blkio_group_conf {

	unsigned int weight;

	u64 iops[2];
@@ -86,9 +78,6 @@ struct blkg_policy_data { 
	/* Configuration */

	struct blkio_group_conf conf;



	/* Per cpu stats pointer */

	struct blkio_group_stats_cpu __percpu *stats_cpu;



	/* pol->pdata_size bytes of private data used by policy impl */

	char pdata[] __aligned(__alignof__(unsigned long long));

};
@@ -106,8 +95,6 @@ struct blkio_group { 


	struct blkg_policy_data *pd[BLKIO_NR_POLICIES];



	/* List of blkg waiting for per cpu stats memory to be allocated */

	struct list_head alloc_node;

	struct rcu_head rcu_head;

};

block/blk-throttle.c

+113 −15
Original line number Diff line number Diff line
@@ -40,6 +40,14 @@ struct throtl_rb_root { 


#define rb_entry_tg(node)	rb_entry((node), struct throtl_grp, rb_node)



/* Per-cpu group stats */

struct tg_stats_cpu {

	/* total bytes transferred */

	struct blkg_rwstat		service_bytes;

	/* total IOs serviced, post merge */

	struct blkg_rwstat		serviced;

};



struct throtl_grp {

	/* active throtl group service_tree member */

	struct rb_node rb_node;
@@ -76,6 +84,12 @@ struct throtl_grp { 


	/* Some throttle limits got updated for the group */

	int limits_changed;



	/* Per cpu stats pointer */

	struct tg_stats_cpu __percpu *stats_cpu;



	/* List of tgs waiting for per cpu stats memory to be allocated */

	struct list_head stats_alloc_node;

};



struct throtl_data
@@ -100,6 +114,13 @@ struct throtl_data 
	int limits_changed;

};



/* list and work item to allocate percpu group stats */

static DEFINE_SPINLOCK(tg_stats_alloc_lock);

static LIST_HEAD(tg_stats_alloc_list);



static void tg_stats_alloc_fn(struct work_struct *);

static DECLARE_DELAYED_WORK(tg_stats_alloc_work, tg_stats_alloc_fn);



static inline struct throtl_grp *blkg_to_tg(struct blkio_group *blkg)

{

	return blkg_to_pdata(blkg, &blkio_policy_throtl);
@@ -142,6 +163,44 @@ static inline unsigned int total_nr_queued(struct throtl_data *td) 
	return td->nr_queued[0] + td->nr_queued[1];

}



/*

 * Worker for allocating per cpu stat for tgs. This is scheduled on the

 * system_nrt_wq once there are some groups on the alloc_list waiting for

 * allocation.

 */

static void tg_stats_alloc_fn(struct work_struct *work)

{

	static struct tg_stats_cpu *stats_cpu;	/* this fn is non-reentrant */

	struct delayed_work *dwork = to_delayed_work(work);

	bool empty = false;



alloc_stats:

	if (!stats_cpu) {

		stats_cpu = alloc_percpu(struct tg_stats_cpu);

		if (!stats_cpu) {

			/* allocation failed, try again after some time */

			queue_delayed_work(system_nrt_wq, dwork,

					   msecs_to_jiffies(10));

			return;

		}

	}



	spin_lock_irq(&tg_stats_alloc_lock);



	if (!list_empty(&tg_stats_alloc_list)) {

		struct throtl_grp *tg = list_first_entry(&tg_stats_alloc_list,

							 struct throtl_grp,

							 stats_alloc_node);

		swap(tg->stats_cpu, stats_cpu);

		list_del_init(&tg->stats_alloc_node);

	}



	empty = list_empty(&tg_stats_alloc_list);

	spin_unlock_irq(&tg_stats_alloc_lock);

	if (!empty)

		goto alloc_stats;

}



static void throtl_init_blkio_group(struct blkio_group *blkg)

{

	struct throtl_grp *tg = blkg_to_tg(blkg);
@@ -155,6 +214,43 @@ static void throtl_init_blkio_group(struct blkio_group *blkg) 
	tg->bps[WRITE] = -1;

	tg->iops[READ] = -1;

	tg->iops[WRITE] = -1;



	/*

	 * Ugh... We need to perform per-cpu allocation for tg->stats_cpu

	 * but percpu allocator can't be called from IO path.  Queue tg on

	 * tg_stats_alloc_list and allocate from work item.

	 */

	spin_lock(&tg_stats_alloc_lock);

	list_add(&tg->stats_alloc_node, &tg_stats_alloc_list);

	queue_delayed_work(system_nrt_wq, &tg_stats_alloc_work, 0);

	spin_unlock(&tg_stats_alloc_lock);

}



static void throtl_exit_blkio_group(struct blkio_group *blkg)

{

	struct throtl_grp *tg = blkg_to_tg(blkg);



	spin_lock(&tg_stats_alloc_lock);

	list_del_init(&tg->stats_alloc_node);

	spin_unlock(&tg_stats_alloc_lock);



	free_percpu(tg->stats_cpu);

}



static void throtl_reset_group_stats(struct blkio_group *blkg)

{

	struct throtl_grp *tg = blkg_to_tg(blkg);

	int cpu;



	if (tg->stats_cpu == NULL)

		return;



	for_each_possible_cpu(cpu) {

		struct tg_stats_cpu *sc = per_cpu_ptr(tg->stats_cpu, cpu);



		blkg_rwstat_reset(&sc->service_bytes);

		blkg_rwstat_reset(&sc->serviced);

	}

}



static struct
@@ -565,12 +661,12 @@ static bool tg_may_dispatch(struct throtl_data *td, struct throtl_grp *tg, 
static void throtl_update_dispatch_stats(struct blkio_group *blkg, u64 bytes,

					 int rw)

{

	struct blkg_policy_data *pd = blkg->pd[BLKIO_POLICY_THROTL];

	struct blkio_group_stats_cpu *stats_cpu;

	struct throtl_grp *tg = blkg_to_tg(blkg);

	struct tg_stats_cpu *stats_cpu;

	unsigned long flags;



	/* If per cpu stats are not allocated yet, don't do any accounting. */

	if (pd->stats_cpu == NULL)

	if (tg->stats_cpu == NULL)

		return;



	/*
@@ -580,7 +676,7 @@ static void throtl_update_dispatch_stats(struct blkio_group *blkg, u64 bytes, 
	 */

	local_irq_save(flags);



	stats_cpu = this_cpu_ptr(pd->stats_cpu);

	stats_cpu = this_cpu_ptr(tg->stats_cpu);



	blkg_rwstat_add(&stats_cpu->serviced, rw, 1);

	blkg_rwstat_add(&stats_cpu->service_bytes, rw, bytes);
@@ -842,15 +938,15 @@ static void throtl_update_blkio_group_common(struct throtl_data *td, 
	throtl_schedule_delayed_work(td, 0);

}



static u64 blkg_prfill_cpu_rwstat(struct seq_file *sf,

static u64 tg_prfill_cpu_rwstat(struct seq_file *sf,

				struct blkg_policy_data *pd, int off)

{

	struct throtl_grp *tg = (void *)pd->pdata;

	struct blkg_rwstat rwstat = { }, tmp;

	int i, cpu;



	for_each_possible_cpu(cpu) {

		struct blkio_group_stats_cpu *sc =

			per_cpu_ptr(pd->stats_cpu, cpu);

		struct tg_stats_cpu *sc = per_cpu_ptr(tg->stats_cpu, cpu);



		tmp = blkg_rwstat_read((void *)sc + off);

		for (i = 0; i < BLKG_RWSTAT_NR; i++)
@@ -861,12 +957,12 @@ static u64 blkg_prfill_cpu_rwstat(struct seq_file *sf, 
}



/* print per-cpu blkg_rwstat specified by BLKCG_STAT_PRIV() */

static int blkcg_print_cpu_rwstat(struct cgroup *cgrp, struct cftype *cft,

static int tg_print_cpu_rwstat(struct cgroup *cgrp, struct cftype *cft,

			       struct seq_file *sf)

{

	struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);



	blkcg_print_blkgs(sf, blkcg, blkg_prfill_cpu_rwstat,

	blkcg_print_blkgs(sf, blkcg, tg_prfill_cpu_rwstat,

			  BLKCG_STAT_POL(cft->private),

			  BLKCG_STAT_OFF(cft->private), true);

	return 0;
@@ -1012,14 +1108,14 @@ static struct cftype throtl_files[] = { 
	{

		.name = "throttle.io_service_bytes",

		.private = BLKCG_STAT_PRIV(BLKIO_POLICY_THROTL,

				offsetof(struct blkio_group_stats_cpu, service_bytes)),

		.read_seq_string = blkcg_print_cpu_rwstat,

				offsetof(struct tg_stats_cpu, service_bytes)),

		.read_seq_string = tg_print_cpu_rwstat,

	},

	{

		.name = "throttle.io_serviced",

		.private = BLKCG_STAT_PRIV(BLKIO_POLICY_THROTL,

				offsetof(struct blkio_group_stats_cpu, serviced)),

		.read_seq_string = blkcg_print_cpu_rwstat,

				offsetof(struct tg_stats_cpu, serviced)),

		.read_seq_string = tg_print_cpu_rwstat,

	},

	{ }	/* terminate */

};
@@ -1034,6 +1130,8 @@ static void throtl_shutdown_wq(struct request_queue *q) 
static struct blkio_policy_type blkio_policy_throtl = {

	.ops = {

		.blkio_init_group_fn = throtl_init_blkio_group,

		.blkio_exit_group_fn = throtl_exit_blkio_group,

		.blkio_reset_group_stats_fn = throtl_reset_group_stats,

	},

	.plid = BLKIO_POLICY_THROTL,

	.pdata_size = sizeof(struct throtl_grp),

CRA Git | Maintained and supported by SUSTech CRA and CCSE