Commit beea9da0 authored by Dennis Zhou's avatar Dennis Zhou Committed by Jens Axboe
Browse files

blkcg: convert blkg_lookup_create() to find closest blkg 



There are several scenarios where blkg_lookup_create() can fail such as
the blkcg dying, request_queue is dying, or simply being OOM. Most
handle this by simply falling back to the q->root_blkg and calling it a
day.

This patch implements the notion of closest blkg. During
blkg_lookup_create(), if it fails to create, return the closest blkg
found or the q->root_blkg. blkg_try_get_closest() is introduced and used
during association so a bio is always attached to a blkg.

Signed-off-by: default avatarDennis Zhou <dennis@kernel.org>
Acked-by: default avatarTejun Heo <tj@kernel.org>
Reviewed-by: default avatarJosef Bacik <josef@toxicpanda.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent b978962a

block/bio.c

+10 −7
Original line number Original line Diff line number Diff line
@@ -2009,21 +2009,24 @@ int bio_associate_blkcg(struct bio *bio, struct cgroup_subsys_state *blkcg_css) 
EXPORT_SYMBOL_GPL(bio_associate_blkcg);

EXPORT_SYMBOL_GPL(bio_associate_blkcg);





/**

/**

 * bio_associate_blkg - associate a bio with the specified blkg

 * bio_associate_blkg - associate a bio with the a blkg

 * @bio: target bio

 * @bio: target bio

 * @blkg: the blkg to associate

 * @blkg: the blkg to associate

 *

 *

 * Associate @bio with the blkg specified by @blkg.  This is the queue specific

 * This tries to associate @bio with the specified @blkg.  Association failure

 * blkcg information associated with the @bio, a reference will be taken on the

 * is handled by walking up the blkg tree.  Therefore, the blkg associated can

 * @blkg and will be freed when the bio is freed.

 * be anything between @blkg and the root_blkg.  This situation only happens

 * when a cgroup is dying and then the remaining bios will spill to the closest

 * alive blkg.

 *

 * A reference will be taken on the @blkg and will be released when @bio is

 * freed.

 */

 */

int bio_associate_blkg(struct bio *bio, struct blkcg_gq *blkg)

int bio_associate_blkg(struct bio *bio, struct blkcg_gq *blkg)

{

{

	if (unlikely(bio->bi_blkg))

	if (unlikely(bio->bi_blkg))

		return -EBUSY;

		return -EBUSY;

	if (!blkg_try_get(blkg))

	bio->bi_blkg = blkg_try_get_closest(blkg);

		return -ENODEV;

	bio->bi_blkg = blkg;

	return 0;

	return 0;

}

}

block/blk-cgroup.c

+16 −7
Original line number Original line Diff line number Diff line
@@ -258,9 +258,8 @@ err_free_blkg: 
 * that all non-root blkg's have access to the parent blkg.  This function

 * that all non-root blkg's have access to the parent blkg.  This function

 * should be called under RCU read lock and @q->queue_lock.

 * should be called under RCU read lock and @q->queue_lock.

 *

 *

 * Returns pointer to the looked up or created blkg on success, ERR_PTR()

 * Returns the blkg or the closest blkg if blkg_create() fails as it walks

 * value on error.  If @q is dead, returns ERR_PTR(-EINVAL).  If @q is not

 * down from root.

 * dead and bypassing, returns ERR_PTR(-EBUSY).

 */

 */

struct blkcg_gq *__blkg_lookup_create(struct blkcg *blkcg,

struct blkcg_gq *__blkg_lookup_create(struct blkcg *blkcg,

				      struct request_queue *q)

				      struct request_queue *q)
@@ -276,19 +275,29 @@ struct blkcg_gq *__blkg_lookup_create(struct blkcg *blkcg, 




	/*

	/*

	 * Create blkgs walking down from blkcg_root to @blkcg, so that all

	 * Create blkgs walking down from blkcg_root to @blkcg, so that all

	 * non-root blkgs have access to their parents.

	 * non-root blkgs have access to their parents.  Returns the closest

	 * blkg to the intended blkg should blkg_create() fail.

	 */

	 */

	while (true) {

	while (true) {

		struct blkcg *pos = blkcg;

		struct blkcg *pos = blkcg;

		struct blkcg *parent = blkcg_parent(blkcg);

		struct blkcg *parent = blkcg_parent(blkcg);

		struct blkcg_gq *ret_blkg = q->root_blkg;





		while (parent && !__blkg_lookup(parent, q, false)) {

		while (parent) {

			blkg = __blkg_lookup(parent, q, false);

			if (blkg) {

				/* remember closest blkg */

				ret_blkg = blkg;

				break;

			}

			pos = parent;

			pos = parent;

			parent = blkcg_parent(parent);

			parent = blkcg_parent(parent);

		}

		}





		blkg = blkg_create(pos, q, NULL);

		blkg = blkg_create(pos, q, NULL);

		if (pos == blkcg || IS_ERR(blkg))

		if (IS_ERR(blkg))

			return ret_blkg;

		if (pos == blkcg)

			return blkg;

			return blkg;

	}

	}

}

}

block/blk-iolatency.c

+2 −12
Original line number Original line Diff line number Diff line
@@ -483,21 +483,11 @@ static void blkcg_iolatency_throttle(struct rq_qos *rqos, struct bio *bio) 
	rcu_read_lock();

	rcu_read_lock();

	bio_associate_blkcg(bio, NULL);

	bio_associate_blkcg(bio, NULL);

	blkcg = bio_blkcg(bio);

	blkcg = bio_blkcg(bio);

	blkg = blkg_lookup(blkcg, q);

	blkg = blkg_lookup_create(blkcg, q);

	if (unlikely(!blkg)) {

		spin_lock_irq(&q->queue_lock);

		blkg = __blkg_lookup_create(blkcg, q);

		if (IS_ERR(blkg))

			blkg = NULL;

		spin_unlock_irq(&q->queue_lock);

	}

	if (!blkg)

		goto out;



	bio_issue_init(&bio->bi_issue, bio_sectors(bio));

	bio_issue_init(&bio->bi_issue, bio_sectors(bio));

	bio_associate_blkg(bio, blkg);

	bio_associate_blkg(bio, blkg);

out:

	rcu_read_unlock();

	rcu_read_unlock();



	while (blkg && blkg->parent) {

	while (blkg && blkg->parent) {

		struct iolatency_grp *iolat = blkg_to_lat(blkg);

		struct iolatency_grp *iolat = blkg_to_lat(blkg);

		if (!iolat) {

		if (!iolat) {

block/blk-throttle.c

+1 −3
Original line number Original line Diff line number Diff line
@@ -2118,9 +2118,7 @@ static inline void throtl_update_latency_buckets(struct throtl_data *td) 
static void blk_throtl_assoc_bio(struct throtl_grp *tg, struct bio *bio)

static void blk_throtl_assoc_bio(struct throtl_grp *tg, struct bio *bio)

{

{

#ifdef CONFIG_BLK_DEV_THROTTLING_LOW

#ifdef CONFIG_BLK_DEV_THROTTLING_LOW

	/* fallback to root_blkg if we fail to get a blkg ref */

	bio_associate_blkg(bio, tg_to_blkg(tg));

	if (bio->bi_css && (bio_associate_blkg(bio, tg_to_blkg(tg)) == -ENODEV))

		bio_associate_blkg(bio, bio->bi_disk->queue->root_blkg);

	bio_issue_init(&bio->bi_issue, bio_sectors(bio));

	bio_issue_init(&bio->bi_issue, bio_sectors(bio));

#endif

#endif

}

}

include/linux/blk-cgroup.h

+15 −9
Original line number Original line Diff line number Diff line
@@ -545,6 +545,20 @@ static inline struct blkcg_gq *blkg_try_get(struct blkcg_gq *blkg) 
	return NULL;

	return NULL;

}

}





/**

 * blkg_try_get_closest - try and get a blkg ref on the closet blkg

 * @blkg: blkg to get

 *

 * This walks up the blkg tree to find the closest non-dying blkg and returns

 * the blkg that it did association with as it may not be the passed in blkg.

 */

static inline struct blkcg_gq *blkg_try_get_closest(struct blkcg_gq *blkg)

{

	while (!atomic_inc_not_zero(&blkg->refcnt))

		blkg = blkg->parent;



	return blkg;

}





void __blkg_release_rcu(struct rcu_head *rcu);

void __blkg_release_rcu(struct rcu_head *rcu);
@@ -797,15 +811,7 @@ static inline bool blkcg_bio_issue_check(struct request_queue *q, 
	/* associate blkcg if bio hasn't attached one */

	/* associate blkcg if bio hasn't attached one */

	bio_associate_blkcg(bio, NULL);

	bio_associate_blkcg(bio, NULL);

	blkcg = bio_blkcg(bio);

	blkcg = bio_blkcg(bio);



	blkg = blkg_lookup_create(blkcg, q);

	blkg = blkg_lookup(blkcg, q);

	if (unlikely(!blkg)) {

		spin_lock_irq(&q->queue_lock);

		blkg = __blkg_lookup_create(blkcg, q);

		if (IS_ERR(blkg))

			blkg = NULL;

		spin_unlock_irq(&q->queue_lock);

	}





	throtl = blk_throtl_bio(q, blkg, bio);

	throtl = blk_throtl_bio(q, blkg, bio);

CRA Git | Maintained and supported by SUSTech CRA and CCSE