Commit 73f0d49a authored by Tejun Heo's avatar Tejun Heo
Browse files

blk-throttle: move bio_lists[] and friends to throtl_service_queue 



throtl_service_queues will eventually form a tree which is anchored at
throtl_data->service_queue and queue bios will climb the tree to the
top service_queue to be executed.

This patch moves bio_lists[] and nr_queued[] from throtl_grp to its
service_queue to prepare for that.  As currently only the
throtl_data->service_queue is in use, this patch just ends up moving
throtl_grp->bio_lists[] and ->nr_queued[] to
throtl_grp->service_queue.bio_lists[] and ->nr_queued[] without making
any functional differences.

Signed-off-by: default avatarTejun Heo <tj@kernel.org>
Acked-by: default avatarVivek Goyal <vgoyal@redhat.com>
parent 49a2f1e3

block/blk-throttle.c

+39 −24
Original line number Diff line number Diff line
@@ -27,6 +27,17 @@ static struct blkcg_policy blkcg_policy_throtl; 
static struct workqueue_struct *kthrotld_workqueue;



struct throtl_service_queue {

	/*

	 * Bios queued directly to this service_queue or dispatched from

	 * children throtl_grp's.

	 */

	struct bio_list		bio_lists[2];	/* queued bios [READ/WRITE] */

	unsigned int		nr_queued[2];	/* number of queued bios */



	/*

	 * RB tree of active children throtl_grp's, which are sorted by

	 * their ->disptime.

	 */

	struct rb_root		pending_tree;	/* RB tree of active tgs */

	struct rb_node		*first_pending;	/* first node in the tree */

	unsigned int		nr_pending;	/* # queued in the tree */
@@ -69,12 +80,6 @@ struct throtl_grp { 


	unsigned int flags;



	/* Two lists for READ and WRITE */

	struct bio_list bio_lists[2];



	/* Number of queued bios on READ and WRITE lists */

	unsigned int nr_queued[2];



	/* bytes per second rate limits */

	uint64_t bps[2];
@@ -193,6 +198,8 @@ alloc_stats: 
/* init a service_queue, assumes the caller zeroed it */

static void throtl_service_queue_init(struct throtl_service_queue *sq)

{

	bio_list_init(&sq->bio_lists[0]);

	bio_list_init(&sq->bio_lists[1]);

	sq->pending_tree = RB_ROOT;

}
@@ -204,8 +211,6 @@ static void throtl_pd_init(struct blkcg_gq *blkg) 
	throtl_service_queue_init(&tg->service_queue);

	RB_CLEAR_NODE(&tg->rb_node);

	tg->td = blkg->q->td;

	bio_list_init(&tg->bio_lists[0]);

	bio_list_init(&tg->bio_lists[1]);



	tg->bps[READ] = -1;

	tg->bps[WRITE] = -1;
@@ -624,7 +629,8 @@ static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio, 
	 * this function with a different bio if there are other bios

	 * queued.

	 */

	BUG_ON(tg->nr_queued[rw] && bio != bio_list_peek(&tg->bio_lists[rw]));

	BUG_ON(tg->service_queue.nr_queued[rw] &&

	       bio != bio_list_peek(&tg->service_queue.bio_lists[rw]));



	/* If tg->bps = -1, then BW is unlimited */

	if (tg->bps[rw] == -1 && tg->iops[rw] == -1) {
@@ -703,12 +709,13 @@ static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio) 
static void throtl_add_bio_tg(struct bio *bio, struct throtl_grp *tg,

			      struct throtl_service_queue *parent_sq)

{

	struct throtl_service_queue *sq = &tg->service_queue;

	bool rw = bio_data_dir(bio);



	bio_list_add(&tg->bio_lists[rw], bio);

	bio_list_add(&sq->bio_lists[rw], bio);

	/* Take a bio reference on tg */

	blkg_get(tg_to_blkg(tg));

	tg->nr_queued[rw]++;

	sq->nr_queued[rw]++;

	tg->td->nr_queued[rw]++;

	throtl_enqueue_tg(tg, parent_sq);

}
@@ -716,13 +723,14 @@ static void throtl_add_bio_tg(struct bio *bio, struct throtl_grp *tg, 
static void tg_update_disptime(struct throtl_grp *tg,

			       struct throtl_service_queue *parent_sq)

{

	struct throtl_service_queue *sq = &tg->service_queue;

	unsigned long read_wait = -1, write_wait = -1, min_wait = -1, disptime;

	struct bio *bio;



	if ((bio = bio_list_peek(&tg->bio_lists[READ])))

	if ((bio = bio_list_peek(&sq->bio_lists[READ])))

		tg_may_dispatch(tg, bio, &read_wait);



	if ((bio = bio_list_peek(&tg->bio_lists[WRITE])))

	if ((bio = bio_list_peek(&sq->bio_lists[WRITE])))

		tg_may_dispatch(tg, bio, &write_wait);



	min_wait = min(read_wait, write_wait);
@@ -737,10 +745,11 @@ static void tg_update_disptime(struct throtl_grp *tg, 
static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw,

				struct bio_list *bl)

{

	struct throtl_service_queue *sq = &tg->service_queue;

	struct bio *bio;



	bio = bio_list_pop(&tg->bio_lists[rw]);

	tg->nr_queued[rw]--;

	bio = bio_list_pop(&sq->bio_lists[rw]);

	sq->nr_queued[rw]--;

	/* Drop bio reference on blkg */

	blkg_put(tg_to_blkg(tg));
@@ -756,6 +765,7 @@ static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw, 


static int throtl_dispatch_tg(struct throtl_grp *tg, struct bio_list *bl)

{

	struct throtl_service_queue *sq = &tg->service_queue;

	unsigned int nr_reads = 0, nr_writes = 0;

	unsigned int max_nr_reads = throtl_grp_quantum*3/4;

	unsigned int max_nr_writes = throtl_grp_quantum - max_nr_reads;
@@ -763,7 +773,7 @@ static int throtl_dispatch_tg(struct throtl_grp *tg, struct bio_list *bl) 


	/* Try to dispatch 75% READS and 25% WRITES */



	while ((bio = bio_list_peek(&tg->bio_lists[READ])) &&

	while ((bio = bio_list_peek(&sq->bio_lists[READ])) &&

	       tg_may_dispatch(tg, bio, NULL)) {



		tg_dispatch_one_bio(tg, bio_data_dir(bio), bl);
@@ -773,7 +783,7 @@ static int throtl_dispatch_tg(struct throtl_grp *tg, struct bio_list *bl) 
			break;

	}



	while ((bio = bio_list_peek(&tg->bio_lists[WRITE])) &&

	while ((bio = bio_list_peek(&sq->bio_lists[WRITE])) &&

	       tg_may_dispatch(tg, bio, NULL)) {



		tg_dispatch_one_bio(tg, bio_data_dir(bio), bl);
@@ -790,10 +800,10 @@ static int throtl_select_dispatch(struct throtl_service_queue *parent_sq, 
				  struct bio_list *bl)

{

	unsigned int nr_disp = 0;

	struct throtl_grp *tg;



	while (1) {

		tg = throtl_rb_first(parent_sq);

		struct throtl_grp *tg = throtl_rb_first(parent_sq);

		struct throtl_service_queue *sq = &tg->service_queue;



		if (!tg)

			break;
@@ -805,7 +815,7 @@ static int throtl_select_dispatch(struct throtl_service_queue *parent_sq, 


		nr_disp += throtl_dispatch_tg(tg, bl);



		if (tg->nr_queued[0] || tg->nr_queued[1])

		if (sq->nr_queued[0] || sq->nr_queued[1])

			tg_update_disptime(tg, parent_sq);



		if (nr_disp >= throtl_quantum)
@@ -1043,6 +1053,7 @@ bool blk_throtl_bio(struct request_queue *q, struct bio *bio) 
{

	struct throtl_data *td = q->td;

	struct throtl_grp *tg;

	struct throtl_service_queue *sq;

	bool rw = bio_data_dir(bio), update_disptime = true;

	struct blkcg *blkcg;

	bool throttled = false;
@@ -1077,7 +1088,9 @@ bool blk_throtl_bio(struct request_queue *q, struct bio *bio) 
	if (unlikely(!tg))

		goto out_unlock;



	if (tg->nr_queued[rw]) {

	sq = &tg->service_queue;



	if (sq->nr_queued[rw]) {

		/*

		 * There is already another bio queued in same dir. No

		 * need to update dispatch time.
@@ -1112,7 +1125,7 @@ queue_bio: 
			rw == READ ? 'R' : 'W',

			tg->bytes_disp[rw], bio->bi_size, tg->bps[rw],

			tg->io_disp[rw], tg->iops[rw],

			tg->nr_queued[READ], tg->nr_queued[WRITE]);

			sq->nr_queued[READ], sq->nr_queued[WRITE]);



	bio_associate_current(bio);

	throtl_add_bio_tg(bio, tg, &q->td->service_queue);
@@ -1151,11 +1164,13 @@ void blk_throtl_drain(struct request_queue *q) 
	bio_list_init(&bl);



	while ((tg = throtl_rb_first(parent_sq))) {

		struct throtl_service_queue *sq = &tg->service_queue;



		throtl_dequeue_tg(tg, parent_sq);



		while ((bio = bio_list_peek(&tg->bio_lists[READ])))

		while ((bio = bio_list_peek(&sq->bio_lists[READ])))

			tg_dispatch_one_bio(tg, bio_data_dir(bio), &bl);

		while ((bio = bio_list_peek(&tg->bio_lists[WRITE])))

		while ((bio = bio_list_peek(&sq->bio_lists[WRITE])))

			tg_dispatch_one_bio(tg, bio_data_dir(bio), &bl);

	}

	spin_unlock_irq(q->queue_lock);

CRA Git | Maintained and supported by SUSTech CRA and CCSE