Commit 1226b8dd authored by Mikulas Patocka's avatar Mikulas Patocka Committed by Jens Axboe
Browse files

block: switch to per-cpu in-flight counters 



Now when part_round_stats is gone, we can switch to per-cpu in-flight
counters.

We use the local-atomic type local_t, so that if part_inc_in_flight or
part_dec_in_flight is reentrantly called from an interrupt, the value will
be correct.

The other counters could be corrupted due to reentrant interrupt, but the
corruption only results in slight counter skew - the in_flight counter
must be exact, so it needs local_t.

Signed-off-by: default avatarMikulas Patocka <mpatocka@redhat.com>
Signed-off-by: default avatarMike Snitzer <snitzer@redhat.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 5b18b5a7

block/genhd.c

+33 −10
Original line number Diff line number Diff line
@@ -50,9 +50,9 @@ void part_inc_in_flight(struct request_queue *q, struct hd_struct *part, int rw) 
	if (queue_is_mq(q))

		return;



	atomic_inc(&part->in_flight[rw]);

	part_stat_local_inc(part, in_flight[rw]);

	if (part->partno)

		atomic_inc(&part_to_disk(part)->part0.in_flight[rw]);

		part_stat_local_inc(&part_to_disk(part)->part0, in_flight[rw]);

}



void part_dec_in_flight(struct request_queue *q, struct hd_struct *part, int rw)
@@ -60,38 +60,61 @@ void part_dec_in_flight(struct request_queue *q, struct hd_struct *part, int rw) 
	if (queue_is_mq(q))

		return;



	atomic_dec(&part->in_flight[rw]);

	part_stat_local_dec(part, in_flight[rw]);

	if (part->partno)

		atomic_dec(&part_to_disk(part)->part0.in_flight[rw]);

		part_stat_local_dec(&part_to_disk(part)->part0, in_flight[rw]);

}



void part_in_flight(struct request_queue *q, struct hd_struct *part,

		    unsigned int inflight[2])

{

	int cpu;



	if (queue_is_mq(q)) {

		blk_mq_in_flight(q, part, inflight);

		return;

	}



	inflight[0] = atomic_read(&part->in_flight[0]) +

			atomic_read(&part->in_flight[1]);

	inflight[0] = 0;

	for_each_possible_cpu(cpu) {

		inflight[0] += part_stat_local_read_cpu(part, in_flight[0], cpu) +

			       part_stat_local_read_cpu(part, in_flight[1], cpu);

	}

	if ((int)inflight[0] < 0)

		inflight[0] = 0;



	if (part->partno) {

		part = &part_to_disk(part)->part0;

		inflight[1] = atomic_read(&part->in_flight[0]) +

				atomic_read(&part->in_flight[1]);

		inflight[1] = 0;

		for_each_possible_cpu(cpu) {

			inflight[1] += part_stat_local_read_cpu(part, in_flight[0], cpu) +

				       part_stat_local_read_cpu(part, in_flight[1], cpu);

		}

		if ((int)inflight[1] < 0)

			inflight[1] = 0;

	}

}



void part_in_flight_rw(struct request_queue *q, struct hd_struct *part,

		       unsigned int inflight[2])

{

	int cpu;



	if (queue_is_mq(q)) {

		blk_mq_in_flight_rw(q, part, inflight);

		return;

	}



	inflight[0] = atomic_read(&part->in_flight[0]);

	inflight[1] = atomic_read(&part->in_flight[1]);

	inflight[0] = 0;

	inflight[1] = 0;

	for_each_possible_cpu(cpu) {

		inflight[0] += part_stat_local_read_cpu(part, in_flight[0], cpu);

		inflight[1] += part_stat_local_read_cpu(part, in_flight[1], cpu);

	}

	if ((int)inflight[0] < 0)

		inflight[0] = 0;

	if ((int)inflight[1] < 0)

		inflight[1] = 0;

}



struct hd_struct *__disk_get_part(struct gendisk *disk, int partno)

include/linux/genhd.h

+22 −7
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ 
#include <linux/percpu-refcount.h>

#include <linux/uuid.h>

#include <linux/blk_types.h>

#include <asm/local.h>



#ifdef CONFIG_BLOCK
@@ -89,6 +90,7 @@ struct disk_stats { 
	unsigned long merges[NR_STAT_GROUPS];

	unsigned long io_ticks;

	unsigned long time_in_queue;

	local_t in_flight[2];

};



#define PARTITION_META_INFO_VOLNAMELTH	64
@@ -122,7 +124,6 @@ struct hd_struct { 
	int make_it_fail;

#endif

	unsigned long stamp;

	atomic_t in_flight[2];

#ifdef	CONFIG_SMP

	struct disk_stats __percpu *dkstats;

#else
@@ -295,8 +296,11 @@ extern struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, 
#define part_stat_lock()	({ rcu_read_lock(); get_cpu(); })

#define part_stat_unlock()	do { put_cpu(); rcu_read_unlock(); } while (0)



#define __part_stat_add(part, field, addnd)				\

	(per_cpu_ptr((part)->dkstats, smp_processor_id())->field += (addnd))

#define part_stat_get_cpu(part, field, cpu)					\

	(per_cpu_ptr((part)->dkstats, (cpu))->field)



#define part_stat_get(part, field)					\

	part_stat_get_cpu(part, field, smp_processor_id())



#define part_stat_read(part, field)					\

({									\
@@ -333,10 +337,9 @@ static inline void free_part_stats(struct hd_struct *part) 
#define part_stat_lock()	({ rcu_read_lock(); 0; })

#define part_stat_unlock()	rcu_read_unlock()



#define __part_stat_add(part, field, addnd)				\

	((part)->dkstats.field += addnd)



#define part_stat_read(part, field)	((part)->dkstats.field)

#define part_stat_get(part, field)		((part)->dkstats.field)

#define part_stat_get_cpu(part, field, cpu)	part_stat_get(part, field)

#define part_stat_read(part, field)		part_stat_get(part, field)



static inline void part_stat_set_all(struct hd_struct *part, int value)

{
@@ -362,6 +365,9 @@ static inline void free_part_stats(struct hd_struct *part) 
	 part_stat_read(part, field[STAT_WRITE]) +			\

	 part_stat_read(part, field[STAT_DISCARD]))



#define __part_stat_add(part, field, addnd)				\

	(part_stat_get(part, field) += (addnd))



#define part_stat_add(part, field, addnd)	do {			\

	__part_stat_add((part), field, addnd);				\

	if ((part)->partno)						\
@@ -376,6 +382,15 @@ static inline void free_part_stats(struct hd_struct *part) 
#define part_stat_sub(gendiskp, field, subnd)				\

	part_stat_add(gendiskp, field, -subnd)



#define part_stat_local_dec(gendiskp, field)				\

	local_dec(&(part_stat_get(gendiskp, field)))

#define part_stat_local_inc(gendiskp, field)				\

	local_inc(&(part_stat_get(gendiskp, field)))

#define part_stat_local_read(gendiskp, field)				\

	local_read(&(part_stat_get(gendiskp, field)))

#define part_stat_local_read_cpu(gendiskp, field, cpu)			\

	local_read(&(part_stat_get_cpu(gendiskp, field, cpu)))



void part_in_flight(struct request_queue *q, struct hd_struct *part,

		    unsigned int inflight[2]);

void part_in_flight_rw(struct request_queue *q, struct hd_struct *part,

CRA Git | Maintained and supported by SUSTech CRA and CCSE