Commit d7219312 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcutorture: Correctly handle grace-period sequence wrap 



The new ->gq_seq grace-period sequence numbers must be shifted down,
which give artifacts when these numbers wrap.  This commit therefore
enables rcutorture and rcuperf to handle grace-period sequence numbers
even if they do wrap.  It does this by allowing a special subtraction
function to be specified, and this function subtracts before shifting.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 2e3e5e55

kernel/rcu/rcu.h

+9 −0
Original line number Diff line number Diff line
@@ -142,6 +142,15 @@ static inline bool rcu_seq_new_gp(unsigned long old, unsigned long new) 
			    new);

}



/*

 * Roughly how many full grace periods have elapsed between the collection

 * of the two specified grace periods?

 */

static inline unsigned long rcu_seq_diff(unsigned long new, unsigned long old)

{

	return (new - old) >> RCU_SEQ_CTR_SHIFT;

}



/*

 * debug_rcu_head_queue()/debug_rcu_head_unqueue() are used internally

 * by call_rcu() and rcu callback execution, and are therefore not part of the

kernel/rcu/rcuperf.c

+16 −2
Original line number Diff line number Diff line
@@ -139,6 +139,7 @@ struct rcu_perf_ops { 
	int (*readlock)(void);

	void (*readunlock)(int idx);

	unsigned long (*get_gp_seq)(void);

	unsigned long (*gp_diff)(unsigned long new, unsigned long old);

	unsigned long (*exp_completed)(void);

	void (*async)(struct rcu_head *head, rcu_callback_t func);

	void (*gp_barrier)(void);
@@ -179,6 +180,7 @@ static struct rcu_perf_ops rcu_ops = { 
	.readlock	= rcu_perf_read_lock,

	.readunlock	= rcu_perf_read_unlock,

	.get_gp_seq	= rcu_get_gp_seq,

	.gp_diff	= rcu_seq_diff,

	.exp_completed	= rcu_exp_batches_completed,

	.async		= call_rcu,

	.gp_barrier	= rcu_barrier,
@@ -208,6 +210,7 @@ static struct rcu_perf_ops rcu_bh_ops = { 
	.readlock	= rcu_bh_perf_read_lock,

	.readunlock	= rcu_bh_perf_read_unlock,

	.get_gp_seq	= rcu_bh_get_gp_seq,

	.gp_diff	= rcu_seq_diff,

	.exp_completed	= rcu_exp_batches_completed_sched,

	.async		= call_rcu_bh,

	.gp_barrier	= rcu_barrier_bh,
@@ -264,6 +267,7 @@ static struct rcu_perf_ops srcu_ops = { 
	.readlock	= srcu_perf_read_lock,

	.readunlock	= srcu_perf_read_unlock,

	.get_gp_seq	= srcu_perf_completed,

	.gp_diff	= rcu_seq_diff,

	.exp_completed	= srcu_perf_completed,

	.async		= srcu_call_rcu,

	.gp_barrier	= srcu_rcu_barrier,
@@ -292,6 +296,7 @@ static struct rcu_perf_ops srcud_ops = { 
	.readlock	= srcu_perf_read_lock,

	.readunlock	= srcu_perf_read_unlock,

	.get_gp_seq	= srcu_perf_completed,

	.gp_diff	= rcu_seq_diff,

	.exp_completed	= srcu_perf_completed,

	.async		= srcu_call_rcu,

	.gp_barrier	= srcu_rcu_barrier,
@@ -321,6 +326,7 @@ static struct rcu_perf_ops sched_ops = { 
	.readlock	= sched_perf_read_lock,

	.readunlock	= sched_perf_read_unlock,

	.get_gp_seq	= rcu_sched_get_gp_seq,

	.gp_diff	= rcu_seq_diff,

	.exp_completed	= rcu_exp_batches_completed_sched,

	.async		= call_rcu_sched,

	.gp_barrier	= rcu_barrier_sched,
@@ -348,6 +354,7 @@ static struct rcu_perf_ops tasks_ops = { 
	.readlock	= tasks_perf_read_lock,

	.readunlock	= tasks_perf_read_unlock,

	.get_gp_seq	= rcu_no_completed,

	.gp_diff	= rcu_seq_diff,

	.async		= call_rcu_tasks,

	.gp_barrier	= rcu_barrier_tasks,

	.sync		= synchronize_rcu_tasks,
@@ -355,6 +362,13 @@ static struct rcu_perf_ops tasks_ops = { 
	.name		= "tasks"

};



static unsigned long rcuperf_seq_diff(unsigned long new, unsigned long old)

{

	if (!cur_ops->gp_diff)

		return new - old;

	return cur_ops->gp_diff(new, old);

}



static bool __maybe_unused torturing_tasks(void)

{

	return cur_ops == &tasks_ops;
@@ -577,8 +591,8 @@ rcu_perf_cleanup(void) 
			 t_rcu_perf_writer_finished -

			 t_rcu_perf_writer_started,

			 ngps,

			 b_rcu_perf_writer_finished -

			 b_rcu_perf_writer_started);

			 rcuperf_seq_diff(b_rcu_perf_writer_finished,

					  b_rcu_perf_writer_started));

		for (i = 0; i < nrealwriters; i++) {

			if (!writer_durations)

				break;

kernel/rcu/rcutorture.c

+13 −6
Original line number Diff line number Diff line
@@ -265,6 +265,7 @@ struct rcu_torture_ops { 
	void (*read_delay)(struct torture_random_state *rrsp);

	void (*readunlock)(int idx);

	unsigned long (*get_gp_seq)(void);

	unsigned long (*gp_diff)(unsigned long new, unsigned long old);

	void (*deferred_free)(struct rcu_torture *p);

	void (*sync)(void);

	void (*exp_sync)(void);
@@ -400,6 +401,7 @@ static struct rcu_torture_ops rcu_ops = { 
	.read_delay	= rcu_read_delay,

	.readunlock	= rcu_torture_read_unlock,

	.get_gp_seq	= rcu_get_gp_seq,

	.gp_diff	= rcu_seq_diff,

	.deferred_free	= rcu_torture_deferred_free,

	.sync		= synchronize_rcu,

	.exp_sync	= synchronize_rcu_expedited,
@@ -441,6 +443,7 @@ static struct rcu_torture_ops rcu_bh_ops = { 
	.read_delay	= rcu_read_delay,  /* just reuse rcu's version. */

	.readunlock	= rcu_bh_torture_read_unlock,

	.get_gp_seq	= rcu_bh_get_gp_seq,

	.gp_diff	= rcu_seq_diff,

	.deferred_free	= rcu_bh_torture_deferred_free,

	.sync		= synchronize_rcu_bh,

	.exp_sync	= synchronize_rcu_bh_expedited,
@@ -646,6 +649,7 @@ static struct rcu_torture_ops sched_ops = { 
	.read_delay	= rcu_read_delay,  /* just reuse rcu's version. */

	.readunlock	= sched_torture_read_unlock,

	.get_gp_seq	= rcu_sched_get_gp_seq,

	.gp_diff	= rcu_seq_diff,

	.deferred_free	= rcu_sched_torture_deferred_free,

	.sync		= synchronize_sched,

	.exp_sync	= synchronize_sched_expedited,
@@ -695,6 +699,13 @@ static struct rcu_torture_ops tasks_ops = { 
	.name		= "tasks"

};



static unsigned long rcutorture_seq_diff(unsigned long new, unsigned long old)

{

	if (!cur_ops->gp_diff)

		return new - old;

	return cur_ops->gp_diff(new, old);

}



static bool __maybe_unused torturing_tasks(void)

{

	return cur_ops == &tasks_ops;
@@ -1127,9 +1138,7 @@ static void rcu_torture_timer(struct timer_list *unused) 
		rcu_ftrace_dump(DUMP_ALL);

	}

	__this_cpu_inc(rcu_torture_count[pipe_count]);

	completed = completed - started;

	if (completed > ULONG_MAX >> 1)

		completed = 0; /* Not all gp_seq have full range. */

	completed = rcutorture_seq_diff(completed, started);

	if (completed > RCU_TORTURE_PIPE_LEN) {

		/* Should not happen, but... */

		completed = RCU_TORTURE_PIPE_LEN;
@@ -1205,9 +1214,7 @@ rcu_torture_reader(void *arg) 
			rcu_ftrace_dump(DUMP_ALL);

		}

		__this_cpu_inc(rcu_torture_count[pipe_count]);

		completed = completed - started;

		if (completed > ULONG_MAX >> 1)

			completed = 0; /* Not all gp_seq have full range. */

		completed = rcutorture_seq_diff(completed, started);

		if (completed > RCU_TORTURE_PIPE_LEN) {

			/* Should not happen, but... */

			completed = RCU_TORTURE_PIPE_LEN;

kernel/rcu/tree.c

+3 −3
Original line number Diff line number Diff line
@@ -532,7 +532,7 @@ static int rcu_pending(void); 
 */

unsigned long rcu_get_gp_seq(void)

{

	return rcu_seq_ctr(READ_ONCE(rcu_state_p->gp_seq));

	return READ_ONCE(rcu_state_p->gp_seq);

}

EXPORT_SYMBOL_GPL(rcu_get_gp_seq);
@@ -541,7 +541,7 @@ EXPORT_SYMBOL_GPL(rcu_get_gp_seq); 
 */

unsigned long rcu_sched_get_gp_seq(void)

{

	return rcu_seq_ctr(READ_ONCE(rcu_sched_state.gp_seq));

	return READ_ONCE(rcu_sched_state.gp_seq);

}

EXPORT_SYMBOL_GPL(rcu_sched_get_gp_seq);
@@ -550,7 +550,7 @@ EXPORT_SYMBOL_GPL(rcu_sched_get_gp_seq); 
 */

unsigned long rcu_bh_get_gp_seq(void)

{

	return rcu_seq_ctr(READ_ONCE(rcu_bh_state.gp_seq));

	return READ_ONCE(rcu_bh_state.gp_seq);

}

EXPORT_SYMBOL_GPL(rcu_bh_get_gp_seq);

CRA Git | Maintained and supported by SUSTech CRA and CCSE