Commit b11cc576 authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Paul E. McKenney
Browse files

rcu: Accelerate RCU callbacks at grace-period end 



Now that callback acceleration is idempotent, it is safe to accelerate
callbacks during grace-period cleanup on any CPUs that the kthread happens
to be running on.  This commit therefore propagates the completion
of the grace period to the per-CPU data structures, and also adds an
rcu_advance_cbs() just before the cpu_needs_another_gp() check in order
to reduce false-positive grace periods.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 5e44ce35

kernel/rcutree.c

+13 −8
Original line number Diff line number Diff line
@@ -1396,6 +1396,9 @@ static void rcu_gp_cleanup(struct rcu_state *rsp) 
	rcu_for_each_node_breadth_first(rsp, rnp) {

		raw_spin_lock_irq(&rnp->lock);

		rnp->completed = rsp->gpnum;

		rdp = this_cpu_ptr(rsp->rda);

		if (rnp == rdp->mynode)

			__rcu_process_gp_end(rsp, rnp, rdp);

		nocb += rcu_nocb_gp_cleanup(rsp, rnp);

		raw_spin_unlock_irq(&rnp->lock);

		cond_resched();
@@ -1408,6 +1411,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp) 
	trace_rcu_grace_period(rsp->name, rsp->completed, "end");

	rsp->fqs_state = RCU_GP_IDLE;

	rdp = this_cpu_ptr(rsp->rda);

	rcu_advance_cbs(rsp, rnp, rdp);  /* Reduce false positives below. */

	if (cpu_needs_another_gp(rsp, rdp))

		rsp->gp_flags = 1;

	raw_spin_unlock_irq(&rnp->lock);
@@ -1497,6 +1501,15 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags) 
	struct rcu_data *rdp = this_cpu_ptr(rsp->rda);

	struct rcu_node *rnp = rcu_get_root(rsp);



	/*

	 * If there is no grace period in progress right now, any

	 * callbacks we have up to this point will be satisfied by the

	 * next grace period.  Also, advancing the callbacks reduces the

	 * probability of false positives from cpu_needs_another_gp()

	 * resulting in pointless grace periods.  So, advance callbacks!

	 */

	rcu_advance_cbs(rsp, rnp, rdp);



	if (!rsp->gp_kthread ||

	    !cpu_needs_another_gp(rsp, rdp)) {

		/*
@@ -1509,14 +1522,6 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags) 
		return;

	}



	/*

	 * Because there is no grace period in progress right now,

	 * any callbacks we have up to this point will be satisfied

	 * by the next grace period.  So this is a good place to

	 * assign a grace period number to recently posted callbacks.

	 */

	rcu_accelerate_cbs(rsp, rnp, rdp);



	rsp->gp_flags = RCU_GP_FLAG_INIT;

	raw_spin_unlock(&rnp->lock); /* Interrupts remain disabled. */

CRA Git | Maintained and supported by SUSTech CRA and CCSE