Commit a657f261 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Execute RCU reader shortly after rcu_core for strict GPs 



A kernel built with CONFIG_RCU_STRICT_GRACE_PERIOD=y needs a quiescent
state to appear very shortly after a CPU has noticed a new grace period.
Placing an RCU reader immediately after this point is ineffective because
this normally happens in softirq context, which acts as a big RCU reader.
This commit therefore introduces a new per-CPU work_struct, which is
used at the end of rcu_core() processing to schedule an RCU read-side
critical section from within a clean environment.

Reported-by Jann Horn <jannh@google.com>
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent 3d29aaf1

kernel/rcu/tree.c

+13 −0
Original line number Diff line number Diff line
@@ -2646,6 +2646,14 @@ void rcu_force_quiescent_state(void) 
}

EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);



// Workqueue handler for an RCU reader for kernels enforcing struct RCU

// grace periods.

static void strict_work_handler(struct work_struct *work)

{

	rcu_read_lock();

	rcu_read_unlock();

}



/* Perform RCU core processing work for the current CPU.  */

static __latent_entropy void rcu_core(void)

{
@@ -2690,6 +2698,10 @@ static __latent_entropy void rcu_core(void) 
	/* Do any needed deferred wakeups of rcuo kthreads. */

	do_nocb_deferred_wakeup(rdp);

	trace_rcu_utilization(TPS("End RCU core"));



	// If strict GPs, schedule an RCU reader in a clean environment.

	if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))

		queue_work_on(rdp->cpu, rcu_gp_wq, &rdp->strict_work);

}



static void rcu_core_si(struct softirq_action *h)
@@ -3887,6 +3899,7 @@ rcu_boot_init_percpu_data(int cpu) 


	/* Set up local state, ensuring consistent view of global state. */

	rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu);

	INIT_WORK(&rdp->strict_work, strict_work_handler);

	WARN_ON_ONCE(rdp->dynticks_nesting != 1);

	WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp)));

	rdp->rcu_ofl_gp_seq = rcu_state.gp_seq;

kernel/rcu/tree.h

+1 −0
Original line number Diff line number Diff line
@@ -164,6 +164,7 @@ struct rcu_data { 
					/* period it is aware of. */

	struct irq_work defer_qs_iw;	/* Obtain later scheduler attention. */

	bool defer_qs_iw_pending;	/* Scheduler attention pending? */

	struct work_struct strict_work;	/* Schedule readers for strict GPs. */



	/* 2) batch handling */

	struct rcu_segcblist cblist;	/* Segmented callback list, with */

CRA Git | Maintained and supported by SUSTech CRA and CCSE