Commit 365187fb authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Update cpu_needs_another_gp() for futures from non-NOCB CPUs 



In the old days, the only source of requests for future grace periods
was NOCB CPUs.  This has changed: CPUs routinely post requests for
future grace periods in order to promote power efficiency and reduce
OS jitter with minimal impact on grace-period latency.  This commit
therefore updates cpu_needs_another_gp() to invoke rcu_future_needs_gp()
instead of rcu_nocb_needs_gp().  The latter is no longer used, so is
now removed.  This commit also adds tracing for the irq_work_queue()
wakeup case.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: default avatarJosh Triplett <josh@joshtriplett.org>
parent 83ebe63e

kernel/rcu/tree.c

+29 −10
Original line number Diff line number Diff line
@@ -323,6 +323,28 @@ cpu_has_callbacks_ready_to_invoke(struct rcu_data *rdp) 
	       rdp->nxttail[RCU_DONE_TAIL] != NULL;

}



/*

 * Return the root node of the specified rcu_state structure.

 */

static struct rcu_node *rcu_get_root(struct rcu_state *rsp)

{

	return &rsp->node[0];

}



/*

 * Is there any need for future grace periods?

 * Interrupts must be disabled.  If the caller does not hold the root

 * rnp_node structure's ->lock, the results are advisory only.

 */

static int rcu_future_needs_gp(struct rcu_state *rsp)

{

	struct rcu_node *rnp = rcu_get_root(rsp);

	int idx = (ACCESS_ONCE(rnp->completed) + 1) & 0x1;

	int *fp = &rnp->need_future_gp[idx];



	return ACCESS_ONCE(*fp);

}



/*

 * Does the current CPU require a not-yet-started grace period?

 * The caller must have disabled interrupts to prevent races with
@@ -335,7 +357,7 @@ cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp) 


	if (rcu_gp_in_progress(rsp))

		return 0;  /* No, a grace period is already in progress. */

	if (rcu_nocb_needs_gp(rsp))

	if (rcu_future_needs_gp(rsp))

		return 1;  /* Yes, a no-CBs CPU needs one. */

	if (!rdp->nxttail[RCU_NEXT_TAIL])

		return 0;  /* No, this is a no-CBs (or offline) CPU. */
@@ -349,14 +371,6 @@ cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp) 
	return 0; /* No grace period needed. */

}



/*

 * Return the root node of the specified rcu_state structure.

 */

static struct rcu_node *rcu_get_root(struct rcu_state *rsp)

{

	return &rsp->node[0];

}



/*

 * rcu_eqs_enter_common - current CPU is moving towards extended quiescent state

 *
@@ -1672,6 +1686,8 @@ static void rsp_wakeup(struct irq_work *work) 


	/* Wake up rcu_gp_kthread() to start the grace period. */

	wake_up(&rsp->gp_wq);

	trace_rcu_grace_period(rsp->name, ACCESS_ONCE(rsp->gpnum),

			       "Workqueuewoken");

}



/*
@@ -1706,9 +1722,12 @@ rcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp, 
	 * the wakeup to interrupt context.  And don't bother waking

	 * up the running kthread.

	 */

	if (current != rsp->gp_kthread)

	if (current != rsp->gp_kthread) {

		trace_rcu_grace_period(rsp->name, ACCESS_ONCE(rsp->gpnum),

				       "Workqueuewake");

		irq_work_queue(&rsp->wakeup_work);

	}

}



/*

 * Similar to rcu_start_gp_advanced(), but also advance the calling CPU's

kernel/rcu/tree.h

+0 −1
Original line number Diff line number Diff line
@@ -547,7 +547,6 @@ static void print_cpu_stall_info(struct rcu_state *rsp, int cpu); 
static void print_cpu_stall_info_end(void);

static void zero_cpu_stall_ticks(struct rcu_data *rdp);

static void increment_cpu_stall_ticks(void);

static int rcu_nocb_needs_gp(struct rcu_state *rsp);

static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq);

static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp);

static void rcu_init_one_nocb(struct rcu_node *rnp);

kernel/rcu/tree_plugin.h

+0 −18
Original line number Diff line number Diff line
@@ -2067,19 +2067,6 @@ static int __init parse_rcu_nocb_poll(char *arg) 
}

early_param("rcu_nocb_poll", parse_rcu_nocb_poll);



/*

 * Do any no-CBs CPUs need another grace period?

 *

 * Interrupts must be disabled.  If the caller does not hold the root

 * rnp_node structure's ->lock, the results are advisory only.

 */

static int rcu_nocb_needs_gp(struct rcu_state *rsp)

{

	struct rcu_node *rnp = rcu_get_root(rsp);



	return rnp->need_future_gp[(ACCESS_ONCE(rnp->completed) + 1) & 0x1];

}



/*

 * Wake up any no-CBs CPUs' kthreads that were waiting on the just-ended

 * grace period.
@@ -2402,11 +2389,6 @@ static bool init_nocb_callback_list(struct rcu_data *rdp) 


#else /* #ifdef CONFIG_RCU_NOCB_CPU */



static int rcu_nocb_needs_gp(struct rcu_state *rsp)

{

	return 0;

}



static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)

{

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE