Commit 522a7775 authored by Omar Sandoval's avatar Omar Sandoval Committed by Jens Axboe
Browse files

block: consolidate struct request timestamp fields 



Currently, struct request has four timestamp fields:

- A start time, set at get_request time, in jiffies, used for iostats
- An I/O start time, set at start_request time, in ktime nanoseconds,
  used for blk-stats (i.e., wbt, kyber, hybrid polling)
- Another start time and another I/O start time, used for cfq and bfq

These can all be consolidated into one start time and one I/O start
time, both in ktime nanoseconds, shaving off up to 16 bytes from struct
request depending on the kernel config.

Signed-off-by: default avatarOmar Sandoval <osandov@fb.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 4bc6339a

block/bfq-iosched.c

+2 −2
Original line number Diff line number Diff line
@@ -4778,8 +4778,8 @@ static void bfq_finish_requeue_request(struct request *rq) 


	if (rq->rq_flags & RQF_STARTED)

		bfqg_stats_update_completion(bfqq_group(bfqq),

					     rq_start_time_ns(rq),

					     rq_io_start_time_ns(rq),

					     rq->start_time_ns,

					     rq->io_start_time_ns,

					     rq->cmd_flags);



	if (likely(rq->rq_flags & RQF_STARTED)) {

block/blk-core.c

+8 −9
Original line number Diff line number Diff line
@@ -196,8 +196,7 @@ void blk_rq_init(struct request_queue *q, struct request *rq) 
	RB_CLEAR_NODE(&rq->rb_node);

	rq->tag = -1;

	rq->internal_tag = -1;

	rq->start_time = jiffies;

	set_start_time_ns(rq);

	rq->start_time_ns = ktime_get_ns();

	rq->part = NULL;

	seqcount_init(&rq->gstate_seq);

	u64_stats_init(&rq->aborted_gstate_sync);
@@ -2726,7 +2725,7 @@ void blk_account_io_completion(struct request *req, unsigned int bytes) 
	}

}



void blk_account_io_done(struct request *req)

void blk_account_io_done(struct request *req, u64 now)

{

	/*

	 * Account IO completion.  flush_rq isn't accounted as a
@@ -2734,11 +2733,12 @@ void blk_account_io_done(struct request *req) 
	 * containing request is enough.

	 */

	if (blk_do_io_stat(req) && !(req->rq_flags & RQF_FLUSH_SEQ)) {

		unsigned long duration = jiffies - req->start_time;

		unsigned long duration;

		const int rw = rq_data_dir(req);

		struct hd_struct *part;

		int cpu;



		duration = nsecs_to_jiffies(now - req->start_time_ns);

		cpu = part_stat_lock();

		part = req->part;
@@ -2969,10 +2969,8 @@ static void blk_dequeue_request(struct request *rq) 
	 * and to it is freed is accounted as io that is in progress at

	 * the driver side.

	 */

	if (blk_account_rq(rq)) {

	if (blk_account_rq(rq))

		q->in_flight[rq_is_sync(rq)]++;

		set_io_start_time_ns(rq);

	}

}



/**
@@ -3192,12 +3190,13 @@ EXPORT_SYMBOL_GPL(blk_unprep_request); 
void blk_finish_request(struct request *req, blk_status_t error)

{

	struct request_queue *q = req->q;

	u64 now = ktime_get_ns();



	lockdep_assert_held(req->q->queue_lock);

	WARN_ON_ONCE(q->mq_ops);



	if (req->rq_flags & RQF_STATS)

		blk_stat_add(req);

		blk_stat_add(req, now);



	if (req->rq_flags & RQF_QUEUED)

		blk_queue_end_tag(q, req);
@@ -3212,7 +3211,7 @@ void blk_finish_request(struct request *req, blk_status_t error) 
	if (req->rq_flags & RQF_DONTPREP)

		blk_unprep_request(req);



	blk_account_io_done(req);

	blk_account_io_done(req, now);



	if (req->end_io) {

		wbt_done(req->q->rq_wb, req);

block/blk-merge.c

+5 −6
Original line number Diff line number Diff line
@@ -724,13 +724,12 @@ static struct request *attempt_merge(struct request_queue *q, 
	}



	/*

	 * At this point we have either done a back merge

	 * or front merge. We need the smaller start_time of

	 * the merged requests to be the current request

	 * for accounting purposes.

	 * At this point we have either done a back merge or front merge. We

	 * need the smaller start_time_ns of the merged requests to be the

	 * current request for accounting purposes.

	 */

	if (time_after(req->start_time, next->start_time))

		req->start_time = next->start_time;

	if (next->start_time_ns < req->start_time_ns)

		req->start_time_ns = next->start_time_ns;



	req->biotail->bi_next = next->bio;

	req->biotail = next->biotail;

block/blk-mq.c

+5 −5
Original line number Diff line number Diff line
@@ -309,7 +309,7 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, 
	RB_CLEAR_NODE(&rq->rb_node);

	rq->rq_disk = NULL;

	rq->part = NULL;

	rq->start_time = jiffies;

	rq->start_time_ns = ktime_get_ns();

	rq->io_start_time_ns = 0;

	rq->nr_phys_segments = 0;

#if defined(CONFIG_BLK_DEV_INTEGRITY)
@@ -329,8 +329,6 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, 


#ifdef CONFIG_BLK_CGROUP

	rq->rl = NULL;

	set_start_time_ns(rq);

	rq->cgroup_io_start_time_ns = 0;

#endif



	data->ctx->rq_dispatched[op_is_sync(op)]++;
@@ -506,12 +504,14 @@ EXPORT_SYMBOL_GPL(blk_mq_free_request); 


inline void __blk_mq_end_request(struct request *rq, blk_status_t error)

{

	u64 now = ktime_get_ns();



	if (rq->rq_flags & RQF_STATS) {

		blk_mq_poll_stats_start(rq->q);

		blk_stat_add(rq);

		blk_stat_add(rq, now);

	}



	blk_account_io_done(rq);

	blk_account_io_done(rq, now);



	if (rq->end_io) {

		wbt_done(rq->q->rq_wb, rq);

block/blk-stat.c

+2 −3
Original line number Diff line number Diff line
@@ -47,15 +47,14 @@ static void __blk_stat_add(struct blk_rq_stat *stat, u64 value) 
	stat->nr_samples++;

}



void blk_stat_add(struct request *rq)

void blk_stat_add(struct request *rq, u64 now)

{

	struct request_queue *q = rq->q;

	struct blk_stat_callback *cb;

	struct blk_rq_stat *stat;

	int bucket;

	u64 now, value;

	u64 value;



	now = ktime_get_ns();

	value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0;



	blk_throtl_stat_add(rq, value);

CRA Git | Maintained and supported by SUSTech CRA and CCSE