cgroup: make hierarchy iterators deal with cgroup_subsys_state instead of cgroup

{

	struct blkcg_policy *pol = blkcg_policy[pd->plid];

	struct blkcg_gq *pos_blkg;

	struct cgroup *pos_cgrp;

	struct cgroup_subsys_state *pos_css;

	u64 sum;

	lockdep_assert_held(pd->blkg->q->queue_lock);

	sum = blkg_stat_read((void *)pd + off);

	rcu_read_lock();

	blkg_for_each_descendant_pre(pos_blkg, pos_cgrp, pd_to_blkg(pd)) {

	blkg_for_each_descendant_pre(pos_blkg, pos_css, pd_to_blkg(pd)) {

		struct blkg_policy_data *pos_pd = blkg_to_pd(pos_blkg, pol);

		struct blkg_stat *stat = (void *)pos_pd + off;

{

	struct blkcg_policy *pol = blkcg_policy[pd->plid];

	struct blkcg_gq *pos_blkg;

	struct cgroup *pos_cgrp;

	struct cgroup_subsys_state *pos_css;

	struct blkg_rwstat sum;

	int i;

	sum = blkg_rwstat_read((void *)pd + off);

	rcu_read_lock();

	blkg_for_each_descendant_pre(pos_blkg, pos_cgrp, pd_to_blkg(pd)) {

	blkg_for_each_descendant_pre(pos_blkg, pos_css, pd_to_blkg(pd)) {

		struct blkg_policy_data *pos_pd = blkg_to_pd(pos_blkg, pol);

		struct blkg_rwstat *rwstat = (void *)pos_pd + off;

		struct blkg_rwstat tmp;

	return css ? container_of(css, struct blkcg, css) : NULL;

}

static inline struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup)

{

	return css_to_blkcg(cgroup_css(cgroup, blkio_subsys_id));

}

static inline struct blkcg *task_blkcg(struct task_struct *tsk)

{

	return css_to_blkcg(task_css(tsk, blkio_subsys_id));

/**

 * blkg_for_each_descendant_pre - pre-order walk of a blkg's descendants

 * @d_blkg: loop cursor pointing to the current descendant

 * @pos_cgrp: used for iteration

 * @pos_css: used for iteration

 * @p_blkg: target blkg to walk descendants of

 *

 * Walk @c_blkg through the descendants of @p_blkg.  Must be used with RCU

 * read locked.  If called under either blkcg or queue lock, the iteration

 * is guaranteed to include all and only online blkgs.  The caller may

 * update @pos_cgrp by calling cgroup_rightmost_descendant() to skip

 * subtree.

 * update @pos_css by calling css_rightmost_descendant() to skip subtree.

 */

#define blkg_for_each_descendant_pre(d_blkg, pos_cgrp, p_blkg)		\

	cgroup_for_each_descendant_pre((pos_cgrp), (p_blkg)->blkcg->css.cgroup) \

		if (((d_blkg) = __blkg_lookup(cgroup_to_blkcg(pos_cgrp), \

#define blkg_for_each_descendant_pre(d_blkg, pos_css, p_blkg)		\

	css_for_each_descendant_pre((pos_css), &(p_blkg)->blkcg->css)	\

		if (((d_blkg) = __blkg_lookup(css_to_blkcg(pos_css),	\

					      (p_blkg)->q, false)))

/**

 * blkg_for_each_descendant_post - post-order walk of a blkg's descendants

 * @d_blkg: loop cursor pointing to the current descendant

 * @pos_cgrp: used for iteration

 * @pos_css: used for iteration

 * @p_blkg: target blkg to walk descendants of

 *

 * Similar to blkg_for_each_descendant_pre() but performs post-order

 * traversal instead.  Synchronization rules are the same.

 */

#define blkg_for_each_descendant_post(d_blkg, pos_cgrp, p_blkg)		\

	cgroup_for_each_descendant_post((pos_cgrp), (p_blkg)->blkcg->css.cgroup) \

		if (((d_blkg) = __blkg_lookup(cgroup_to_blkcg(pos_cgrp), \

#define blkg_for_each_descendant_post(d_blkg, pos_css, p_blkg)		\

	css_for_each_descendant_post((pos_css), &(p_blkg)->blkcg->css)	\

		if (((d_blkg) = __blkg_lookup(css_to_blkcg(pos_css),	\

					      (p_blkg)->q, false)))

/**

static inline void blkcg_deactivate_policy(struct request_queue *q,

					   const struct blkcg_policy *pol) { }

static inline struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup) { return NULL; }

static inline struct blkcg *bio_blkcg(struct bio *bio) { return NULL; }

static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,

	struct throtl_grp *tg;

	struct throtl_service_queue *sq;

	struct blkcg_gq *blkg;

	struct cgroup *pos_cgrp;

	struct cgroup_subsys_state *pos_css;

	int ret;

	ret = blkg_conf_prep(blkcg, &blkcg_policy_throtl, buf, &ctx);

	 * blk-throttle.

	 */

	tg_update_has_rules(tg);

	blkg_for_each_descendant_pre(blkg, pos_cgrp, ctx.blkg)

	blkg_for_each_descendant_pre(blkg, pos_css, ctx.blkg)

		tg_update_has_rules(blkg_to_tg(blkg));

	/*

{

	struct throtl_data *td = q->td;

	struct blkcg_gq *blkg;

	struct cgroup *pos_cgrp;

	struct cgroup_subsys_state *pos_css;

	struct bio *bio;

	int rw;

	 * better to walk service_queue tree directly but blkg walk is

	 * easier.

	 */

	blkg_for_each_descendant_post(blkg, pos_cgrp, td->queue->root_blkg)

	blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg)

		tg_drain_bios(&blkg_to_tg(blkg)->service_queue);

	tg_drain_bios(&td_root_tg(td)->service_queue);

	return idr_find(&ss->root->cgroup_idr, id);

}

struct cgroup *cgroup_next_child(struct cgroup *pos, struct cgroup *cgrp);

struct cgroup_subsys_state *css_next_child(struct cgroup_subsys_state *pos,

					   struct cgroup_subsys_state *parent);

/**

 * cgroup_for_each_child - iterate through children of a cgroup

 * @pos: the cgroup * to use as the loop cursor

 * @cgrp: cgroup whose children to walk

 * css_for_each_child - iterate through children of a css

 * @pos: the css * to use as the loop cursor

 * @parent: css whose children to walk

 *

 * Walk @cgrp's children.  Must be called under rcu_read_lock().  A child

 * cgroup which hasn't finished ->css_online() or already has finished

 * Walk @parent's children.  Must be called under rcu_read_lock().  A child

 * css which hasn't finished ->css_online() or already has finished

 * ->css_offline() may show up during traversal and it's each subsystem's

 * responsibility to verify that each @pos is alive.

 *

 * If a subsystem synchronizes against the parent in its ->css_online() and

 * before starting iterating, a cgroup which finished ->css_online() is

 * before starting iterating, a css which finished ->css_online() is

 * guaranteed to be visible in the future iterations.

 *

 * It is allowed to temporarily drop RCU read lock during iteration.  The

 * caller is responsible for ensuring that @pos remains accessible until

 * the start of the next iteration by, for example, bumping the css refcnt.

 */

#define cgroup_for_each_child(pos, cgrp)				\

	for ((pos) = cgroup_next_child(NULL, (cgrp)); (pos);		\

	     (pos) = cgroup_next_child((pos), (cgrp)))

#define css_for_each_child(pos, parent)					\

	for ((pos) = css_next_child(NULL, (parent)); (pos);		\

	     (pos) = css_next_child((pos), (parent)))

struct cgroup *cgroup_next_descendant_pre(struct cgroup *pos,

					  struct cgroup *cgroup);

struct cgroup *cgroup_rightmost_descendant(struct cgroup *pos);

struct cgroup_subsys_state *

css_next_descendant_pre(struct cgroup_subsys_state *pos,

			struct cgroup_subsys_state *css);

struct cgroup_subsys_state *

css_rightmost_descendant(struct cgroup_subsys_state *pos);

/**

 * cgroup_for_each_descendant_pre - pre-order walk of a cgroup's descendants

 * @pos: the cgroup * to use as the loop cursor

 * @cgroup: cgroup whose descendants to walk

 * css_for_each_descendant_pre - pre-order walk of a css's descendants

 * @pos: the css * to use as the loop cursor

 * @root: css whose descendants to walk

 *

 * Walk @cgroup's descendants.  Must be called under rcu_read_lock().  A

 * descendant cgroup which hasn't finished ->css_online() or already has

 * Walk @root's descendants.  Must be called under rcu_read_lock().  A

 * descendant css which hasn't finished ->css_online() or already has

 * finished ->css_offline() may show up during traversal and it's each

 * subsystem's responsibility to verify that each @pos is alive.

 *

 * If a subsystem synchronizes against the parent in its ->css_online() and

 * before starting iterating, and synchronizes against @pos on each

 * iteration, any descendant cgroup which finished ->css_online() is

 * iteration, any descendant css which finished ->css_online() is

 * guaranteed to be visible in the future iterations.

 *

 * In other words, the following guarantees that a descendant can't escape

 * state updates of its ancestors.

 *

 * my_online(@cgrp)

 * my_online(@css)

 * {

 *	Lock @cgrp->parent and @cgrp;

 *	Inherit state from @cgrp->parent;

 *	Lock @css's parent and @css;

 *	Inherit state from the parent;

 *	Unlock both.

 * }

 *

 * my_update_state(@cgrp)

 * my_update_state(@css)

 * {

 *	Lock @cgrp;

 *	Update @cgrp's state;

 *	Unlock @cgrp;

 *	Lock @css;

 *	Update @css's state;

 *	Unlock @css;

 *

 *	cgroup_for_each_descendant_pre(@pos, @cgrp) {

 *	css_for_each_descendant_pre(@pos, @css) {

 *		Lock @pos;

 *		Verify @pos is alive and inherit state from @pos->parent;

 *		Verify @pos is alive and inherit state from @pos's parent;

 *		Unlock @pos;

 *	}

 * }

 * visible by walking order and, as long as inheriting operations to the

 * same @pos are atomic to each other, multiple updates racing each other

 * still result in the correct state.  It's guaranateed that at least one

 * inheritance happens for any cgroup after the latest update to its

 * parent.

 * inheritance happens for any css after the latest update to its parent.

 *

 * If checking parent's state requires locking the parent, each inheriting

 * iteration should lock and unlock both @pos->parent and @pos.

 * caller is responsible for ensuring that @pos remains accessible until

 * the start of the next iteration by, for example, bumping the css refcnt.

 */

#define cgroup_for_each_descendant_pre(pos, cgroup)			\

	for (pos = cgroup_next_descendant_pre(NULL, (cgroup)); (pos);	\

	     pos = cgroup_next_descendant_pre((pos), (cgroup)))

#define css_for_each_descendant_pre(pos, css)				\

	for ((pos) = css_next_descendant_pre(NULL, (css)); (pos);	\

	     (pos) = css_next_descendant_pre((pos), (css)))

struct cgroup *cgroup_next_descendant_post(struct cgroup *pos,

					   struct cgroup *cgroup);

struct cgroup_subsys_state *

css_next_descendant_post(struct cgroup_subsys_state *pos,

			 struct cgroup_subsys_state *css);

/**

 * cgroup_for_each_descendant_post - post-order walk of a cgroup's descendants

 * @pos: the cgroup * to use as the loop cursor

 * @cgroup: cgroup whose descendants to walk

 * css_for_each_descendant_post - post-order walk of a css's descendants

 * @pos: the css * to use as the loop cursor

 * @css: css whose descendants to walk

 *

 * Similar to cgroup_for_each_descendant_pre() but performs post-order

 * Similar to css_for_each_descendant_pre() but performs post-order

 * traversal instead.  Note that the walk visibility guarantee described in

 * pre-order walk doesn't apply the same to post-order walks.

 */

#define cgroup_for_each_descendant_post(pos, cgroup)			\

	for (pos = cgroup_next_descendant_post(NULL, (cgroup)); (pos);	\

	     pos = cgroup_next_descendant_post((pos), (cgroup)))

#define css_for_each_descendant_post(pos, css)				\

	for ((pos) = css_next_descendant_post(NULL, (css)); (pos);	\

	     (pos) = css_next_descendant_post((pos), (css)))

/* A cgroup_iter should be treated as an opaque object */

struct cgroup_iter {

	/*

	 * Thanks to the entanglement with vfs inode locking, we can't walk

	 * the existing cgroups under cgroup_mutex and create files.

	 * Instead, we use cgroup_for_each_descendant_pre() and drop RCU

	 * read lock before calling cgroup_addrm_files().

	 * Instead, we use css_for_each_descendant_pre() and drop RCU read

	 * lock before calling cgroup_addrm_files().

	 */

	mutex_lock(&cgroup_mutex);

}

{

	LIST_HEAD(pending);

	struct cgroup_subsys *ss = cfts[0].ss;

	struct cgroup *cgrp, *root = &ss->root->top_cgroup;

	struct cgroup *root = &ss->root->top_cgroup;

	struct super_block *sb = ss->root->sb;

	struct dentry *prev = NULL;

	struct inode *inode;

	struct cgroup_subsys_state *css;

	u64 update_before;

	int ret = 0;

	/* add/rm files for all cgroups created before */

	rcu_read_lock();

	cgroup_for_each_descendant_pre(cgrp, root) {

	css_for_each_descendant_pre(css, cgroup_css(root, ss->subsys_id)) {

		struct cgroup *cgrp = css->cgroup;

		if (cgroup_is_dead(cgrp))

			continue;

}

/**

 * cgroup_next_child - find the next child of a given cgroup

 * @pos: the current position (%NULL to initiate traversal)

 * @cgrp: cgroup whose descendants to walk

 * css_next_child - find the next child of a given css

 * @pos_css: the current position (%NULL to initiate traversal)

 * @parent_css: css whose children to walk

 *

 * This function returns the next child of @cgrp and should be called under

 * RCU read lock.  The only requirement is that @cgrp and @pos are

 * accessible.  The next sibling is guaranteed to be returned regardless of

 * their states.

 * This function returns the next child of @parent_css and should be called

 * under RCU read lock.  The only requirement is that @parent_css and

 * @pos_css are accessible.  The next sibling is guaranteed to be returned

 * regardless of their states.

 */

struct cgroup *cgroup_next_child(struct cgroup *pos, struct cgroup *cgrp)

struct cgroup_subsys_state *

css_next_child(struct cgroup_subsys_state *pos_css,

	       struct cgroup_subsys_state *parent_css)

{

	struct cgroup *pos = pos_css ? pos_css->cgroup : NULL;

	struct cgroup *cgrp = parent_css->cgroup;

	struct cgroup *next;

	WARN_ON_ONCE(!rcu_read_lock_held());

				break;

	}

	if (&next->sibling != &cgrp->children)

		return next;

	if (&next->sibling == &cgrp->children)

		return NULL;

	if (parent_css->ss)

		return cgroup_css(next, parent_css->ss->subsys_id);

	else

		return &next->dummy_css;

}

EXPORT_SYMBOL_GPL(cgroup_next_child);

EXPORT_SYMBOL_GPL(css_next_child);

/**

 * cgroup_next_descendant_pre - find the next descendant for pre-order walk

 * css_next_descendant_pre - find the next descendant for pre-order walk

 * @pos: the current position (%NULL to initiate traversal)

 * @cgroup: cgroup whose descendants to walk

 * @root: css whose descendants to walk

 *

 * To be used by cgroup_for_each_descendant_pre().  Find the next

 * descendant to visit for pre-order traversal of @cgroup's descendants.

 * To be used by css_for_each_descendant_pre().  Find the next descendant

 * to visit for pre-order traversal of @root's descendants.

 *

 * While this function requires RCU read locking, it doesn't require the

 * whole traversal to be contained in a single RCU critical section.  This

 * function will return the correct next descendant as long as both @pos

 * and @cgroup are accessible and @pos is a descendant of @cgroup.

 * and @root are accessible and @pos is a descendant of @root.

 */

struct cgroup *cgroup_next_descendant_pre(struct cgroup *pos,

					  struct cgroup *cgroup)

struct cgroup_subsys_state *

css_next_descendant_pre(struct cgroup_subsys_state *pos,

			struct cgroup_subsys_state *root)

{

	struct cgroup *next;

	struct cgroup_subsys_state *next;

	WARN_ON_ONCE(!rcu_read_lock_held());

	/* if first iteration, pretend we just visited @cgroup */

	/* if first iteration, pretend we just visited @root */

	if (!pos)

		pos = cgroup;

		pos = root;

	/* visit the first child if exists */

	next = cgroup_next_child(NULL, pos);

	next = css_next_child(NULL, pos);

	if (next)

		return next;

	/* no child, visit my or the closest ancestor's next sibling */

	while (pos != cgroup) {

		next = cgroup_next_child(pos, pos->parent);

	while (pos != root) {

		next = css_next_child(pos, css_parent(pos));

		if (next)

			return next;

		pos = pos->parent;

		pos = css_parent(pos);

	}

	return NULL;

}

EXPORT_SYMBOL_GPL(cgroup_next_descendant_pre);

EXPORT_SYMBOL_GPL(css_next_descendant_pre);

/**

 * cgroup_rightmost_descendant - return the rightmost descendant of a cgroup

 * @pos: cgroup of interest

 * css_rightmost_descendant - return the rightmost descendant of a css

 * @pos: css of interest

 *

 * Return the rightmost descendant of @pos.  If there's no descendant,

 * @pos is returned.  This can be used during pre-order traversal to skip

 * Return the rightmost descendant of @pos.  If there's no descendant, @pos

 * is returned.  This can be used during pre-order traversal to skip

 * subtree of @pos.

 *

 * While this function requires RCU read locking, it doesn't require the

 * function will return the correct rightmost descendant as long as @pos is

 * accessible.

 */

struct cgroup *cgroup_rightmost_descendant(struct cgroup *pos)

struct cgroup_subsys_state *

css_rightmost_descendant(struct cgroup_subsys_state *pos)

{

	struct cgroup *last, *tmp;

	struct cgroup_subsys_state *last, *tmp;

	WARN_ON_ONCE(!rcu_read_lock_held());

		last = pos;

		/* ->prev isn't RCU safe, walk ->next till the end */

		pos = NULL;

		cgroup_for_each_child(tmp, last)

		css_for_each_child(tmp, last)

			pos = tmp;

	} while (pos);

	return last;

}

EXPORT_SYMBOL_GPL(cgroup_rightmost_descendant);

EXPORT_SYMBOL_GPL(css_rightmost_descendant);

static struct cgroup *cgroup_leftmost_descendant(struct cgroup *pos)

static struct cgroup_subsys_state *

css_leftmost_descendant(struct cgroup_subsys_state *pos)

{

	struct cgroup *last;

	struct cgroup_subsys_state *last;

	do {

		last = pos;

		pos = cgroup_next_child(NULL, pos);

		pos = css_next_child(NULL, pos);

	} while (pos);

	return last;

}

/**

 * cgroup_next_descendant_post - find the next descendant for post-order walk

 * css_next_descendant_post - find the next descendant for post-order walk

 * @pos: the current position (%NULL to initiate traversal)

 * @cgroup: cgroup whose descendants to walk

 * @root: css whose descendants to walk

 *

 * To be used by cgroup_for_each_descendant_post().  Find the next

 * descendant to visit for post-order traversal of @cgroup's descendants.

 * To be used by css_for_each_descendant_post().  Find the next descendant

 * to visit for post-order traversal of @root's descendants.

 *

 * While this function requires RCU read locking, it doesn't require the

 * whole traversal to be contained in a single RCU critical section.  This

 * function will return the correct next descendant as long as both @pos

 * and @cgroup are accessible and @pos is a descendant of @cgroup.

 */

struct cgroup *cgroup_next_descendant_post(struct cgroup *pos,

					   struct cgroup *cgroup)

struct cgroup_subsys_state *

css_next_descendant_post(struct cgroup_subsys_state *pos,

			 struct cgroup_subsys_state *root)

{

	struct cgroup *next;

	struct cgroup_subsys_state *next;

	WARN_ON_ONCE(!rcu_read_lock_held());

	/* if first iteration, visit the leftmost descendant */

	if (!pos) {

		next = cgroup_leftmost_descendant(cgroup);

		return next != cgroup ? next : NULL;

		next = css_leftmost_descendant(root);

		return next != root ? next : NULL;

	}

	/* if there's an unvisited sibling, visit its leftmost descendant */

	next = cgroup_next_child(pos, pos->parent);

	next = css_next_child(pos, css_parent(pos));

	if (next)

		return cgroup_leftmost_descendant(next);

		return css_leftmost_descendant(next);

	/* no sibling left, visit parent */

	next = pos->parent;

	return next != cgroup ? next : NULL;

	next = css_parent(pos);

	return next != root ? next : NULL;

}

EXPORT_SYMBOL_GPL(cgroup_next_descendant_post);

EXPORT_SYMBOL_GPL(css_next_descendant_post);

void cgroup_iter_start(struct cgroup *cgrp, struct cgroup_iter *it)

	__acquires(css_set_lock)

	/*

	 * Mark @cgrp dead.  This prevents further task migration and child

	 * creation by disabling cgroup_lock_live_group().  Note that

	 * CGRP_DEAD assertion is depended upon by cgroup_next_child() to

	 * CGRP_DEAD assertion is depended upon by css_next_child() to

	 * resume iteration after dropping RCU read lock.  See

	 * cgroup_next_child() for details.

	 * css_next_child() for details.

	 */

	set_bit(CGRP_DEAD, &cgrp->flags);