Commit 516e5ae0 authored by Joel Fernandes (Google)'s avatar Joel Fernandes (Google) Committed by Paul E. McKenney
Browse files

rcu: Reset CPU hints when reporting a quiescent state 



In some cases, tracing shows that need_heavy_qs is still set even though
urgent_qs was cleared upon reporting of a quiescent state.  One such
case is when the softirq reports that a CPU has passed quiescent state.

Commit 671a6351 ("rcu: Avoid unnecessary softirq when system is
idle") fixed a bug where core_needs_qs was not being cleared.  In order
to avoid running into similar situations with the urgent-grace-period
flags, this commit causes rcu_disable_urgency_upon_qs(), previously
rcu_disable_tick_upon_qs(), to clear the urgency hints, ->rcu_urgent_qs
and ->rcu_need_heavy_qs.  Note that it is possible for CPUs to go
offline with these urgency hints still set.  This is handled because
rcu_disable_urgency_upon_qs() is also invoked during the online process.

Because these hints can be cleared both by the corresponding CPU and by
the grace-period kthread, this commit also adds a number of READ_ONCE()
and WRITE_ONCE() calls.

Tested overnight with rcutorture running for 60 minutes on all
configurations of RCU.

Signed-off-by: default avatar"Joel Fernandes (Google)" <joel@joelfernandes.org>
[ paulmck: Clear urgency flags in rcu_disable_urgency_upon_qs(). ]
[ paulmck: Remove ->core_needs_qs from the set cleared at quiescent state. ]
[ paulmck: Make rcu_disable_urgency_upon_qs static per kbuild test robot. ]
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent b200a048

kernel/rcu/tree.c

+10 −7
Original line number Diff line number Diff line
@@ -827,7 +827,7 @@ static __always_inline void rcu_nmi_enter_common(bool irq) 
		incby = 1;

	} else if (tick_nohz_full_cpu(rdp->cpu) &&

		   rdp->dynticks_nmi_nesting == DYNTICK_IRQ_NONIDLE &&

		   rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) {

		   READ_ONCE(rdp->rcu_urgent_qs) && !rdp->rcu_forced_tick) {

		rdp->rcu_forced_tick = true;

		tick_dep_set_cpu(rdp->cpu, TICK_DEP_BIT_RCU);

	}
@@ -892,11 +892,14 @@ void rcu_irq_enter_irqson(void) 
}



/*

 * If the scheduler-clock interrupt was enabled on a nohz_full CPU

 * in order to get to a quiescent state, disable it.

 * If any sort of urgency was applied to the current CPU (for example,

 * the scheduler-clock interrupt was enabled on a nohz_full CPU) in order

 * to get to a quiescent state, disable it.

 */

void rcu_disable_tick_upon_qs(struct rcu_data *rdp)

static void rcu_disable_urgency_upon_qs(struct rcu_data *rdp)

{

	WRITE_ONCE(rdp->rcu_urgent_qs, false);

	WRITE_ONCE(rdp->rcu_need_heavy_qs, false);

	if (tick_nohz_full_cpu(rdp->cpu) && rdp->rcu_forced_tick) {

		tick_dep_clear_cpu(rdp->cpu, TICK_DEP_BIT_RCU);

		rdp->rcu_forced_tick = false;
@@ -1997,7 +2000,7 @@ rcu_report_qs_rdp(int cpu, struct rcu_data *rdp) 
		if (!offloaded)

			needwake = rcu_accelerate_cbs(rnp, rdp);



		rcu_disable_tick_upon_qs(rdp);

		rcu_disable_urgency_upon_qs(rdp);

		rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);

		/* ^^^ Released rnp->lock */

		if (needwake)
@@ -2311,7 +2314,7 @@ static void force_qs_rnp(int (*f)(struct rcu_data *rdp)) 
				rdp = per_cpu_ptr(&rcu_data, cpu);

				if (f(rdp)) {

					mask |= bit;

					rcu_disable_tick_upon_qs(rdp);

					rcu_disable_urgency_upon_qs(rdp);

				}

			}

		}
@@ -3182,7 +3185,7 @@ void rcu_cpu_starting(unsigned int cpu) 
	rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq);

	rdp->rcu_onl_gp_flags = READ_ONCE(rcu_state.gp_flags);

	if (rnp->qsmask & mask) { /* RCU waiting on incoming CPU? */

		rcu_disable_tick_upon_qs(rdp);

		rcu_disable_urgency_upon_qs(rdp);

		/* Report QS -after- changing ->qsmaskinitnext! */

		rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);

	} else {

CRA Git | Maintained and supported by SUSTech CRA and CCSE