Commit fb31340f authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Make rcu_gp_cleanup() more accurately predict need for new GP 



Currently, rcu_gp_cleanup() scans the rcu_node tree in order to reset
state to reflect the end of the grace period.  It also checks to see
whether a new grace period is needed, but in a number of cases, rather
than directly cause the new grace period to be immediately started, it
instead leaves the grace-period-needed state where various fail-safes
can find it.  This works fine, but results in higher contention on the
root rcu_node structure's ->lock, which is undesirable, and contention
on that lock has recently become noticeable.

This commit therefore makes rcu_gp_cleanup() immediately start a new
grace period if there is any need for one.

It is quite possible that it will later be necessary to throttle the
grace-period rate, but that can be dealt with when and if.

Reported-by: default avatarNicholas Piggin <npiggin@gmail.com>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Tested-by: default avatarNicholas Piggin <npiggin@gmail.com>
parent 5fe0a562

kernel/rcu/tree.c

+10 −6
Original line number Diff line number Diff line
@@ -1763,14 +1763,14 @@ out: 
 * Clean up any old requests for the just-ended grace period.  Also return

 * whether any additional grace periods have been requested.

 */

static int rcu_future_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)

static bool rcu_future_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)

{

	int c = rnp->completed;

	int needmore;

	bool needmore;

	struct rcu_data *rdp = this_cpu_ptr(rsp->rda);



	need_future_gp_element(rnp, c) = 0;

	needmore = need_future_gp_element(rnp, c + 1);

	needmore = need_any_future_gp(rnp);

	trace_rcu_future_gp(rnp, rdp, c,

			    needmore ? TPS("CleanupMore") : TPS("Cleanup"));

	return needmore;
@@ -2113,7 +2113,6 @@ static void rcu_gp_cleanup(struct rcu_state *rsp) 
{

	unsigned long gp_duration;

	bool needgp = false;

	int nocb = 0;

	struct rcu_data *rdp;

	struct rcu_node *rnp = rcu_get_root(rsp);

	struct swait_queue_head *sq;
@@ -2152,7 +2151,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp) 
		if (rnp == rdp->mynode)

			needgp = __note_gp_changes(rsp, rnp, rdp) || needgp;

		/* smp_mb() provided by prior unlock-lock pair. */

		nocb += rcu_future_gp_cleanup(rsp, rnp);

		needgp = rcu_future_gp_cleanup(rsp, rnp) || needgp;

		sq = rcu_nocb_gp_get(rnp);

		raw_spin_unlock_irq_rcu_node(rnp);

		rcu_nocb_gp_cleanup(sq);
@@ -2162,13 +2161,18 @@ static void rcu_gp_cleanup(struct rcu_state *rsp) 
	}

	rnp = rcu_get_root(rsp);

	raw_spin_lock_irq_rcu_node(rnp); /* Order GP before ->completed update. */

	rcu_nocb_gp_set(rnp, nocb);



	/* Declare grace period done. */

	WRITE_ONCE(rsp->completed, rsp->gpnum);

	trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end"));

	rsp->gp_state = RCU_GP_IDLE;

	/* Check for GP requests since above loop. */

	rdp = this_cpu_ptr(rsp->rda);

	if (need_any_future_gp(rnp)) {

		trace_rcu_future_gp(rnp, rdp, rsp->completed - 1,

				    TPS("CleanupMore"));

		needgp = true;

	}

	/* Advance CBs to reduce false positives below. */

	needgp = rcu_advance_cbs(rsp, rnp, rdp) || needgp;

	if (needgp || cpu_needs_another_gp(rsp, rdp)) {

kernel/rcu/tree.h

+0 −1
Original line number Diff line number Diff line
@@ -469,7 +469,6 @@ static void print_cpu_stall_info_end(void); 
static void zero_cpu_stall_ticks(struct rcu_data *rdp);

static void increment_cpu_stall_ticks(void);

static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu);

static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq);

static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp);

static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq);

static void rcu_init_one_nocb(struct rcu_node *rnp);

kernel/rcu/tree_plugin.h

+0 −17
Original line number Diff line number Diff line
@@ -1780,19 +1780,6 @@ static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq) 
	swake_up_all(sq);

}



/*

 * Set the root rcu_node structure's ->need_future_gp field

 * based on the sum of those of all rcu_node structures.  This does

 * double-count the root rcu_node structure's requests, but this

 * is necessary to handle the possibility of a rcu_nocb_kthread()

 * having awakened during the time that the rcu_node structures

 * were being updated for the end of the previous grace period.

 */

static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)

{

	need_future_gp_element(rnp, rnp->completed + 1) += nrq;

}



static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)

{

	return &rnp->nocb_gp_wq[rnp->completed & 0x1];
@@ -2495,10 +2482,6 @@ static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq) 
{

}



static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)

{

}



static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)

{

	return NULL;

CRA Git | Maintained and supported by SUSTech CRA and CCSE