Commit 142d106d authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Determine expedited-GP IPI handler at build time 



Back when there could be multiple RCU flavors running in the same kernel
at the same time, it was necessary to specify the expedited grace-period
IPI handler at runtime.  Now that there is only one RCU flavor, the
IPI handler can be determined at build time.  There is therefore no
longer any reason for the RCU-preempt and RCU-sched IPI handlers to
have different names, nor is there any reason to pass these handlers in
function arguments and in the data structures enclosing workqueues.

This commit therefore makes all these changes, pushing the specification
of the expedited grace-period IPI handler down to the point of use.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.ibm.com>
parent c46f497a

Documentation/RCU/Design/Expedited-Grace-Periods/ExpSchedFlow.svg

+11 −7
Original line number Diff line number Diff line
@@ -328,13 +328,13 @@ 
     inkscape:window-height="1148"

     id="namedview90"

     showgrid="true"

     inkscape:zoom="0.80021373"

     inkscape:cx="462.49289"

     inkscape:cy="473.6718"

     inkscape:zoom="0.69092787"

     inkscape:cx="476.34085"

     inkscape:cy="712.80957"

     inkscape:window-x="770"

     inkscape:window-y="24"

     inkscape:window-maximized="0"

     inkscape:current-layer="g4114-9-3-9"

     inkscape:current-layer="g4"

     inkscape:snap-grids="false"

     fit-margin-top="5"

     fit-margin-right="5"
@@ -813,14 +813,18 @@ 
      <text

         sodipodi:linespacing="125%"

         id="text4110-5-7-6-2-4-0"

         y="841.88086"

         y="670.74316"

         x="1460.1007"

         style="font-size:267.24359131px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"

         xml:space="preserve"><tspan

           y="841.88086"

           y="670.74316"

           x="1460.1007"

           sodipodi:role="line"

           id="tspan4925-1-2-4-5">Request</tspan><tspan

           y="1004.7976"

           x="1460.1007"

           sodipodi:role="line"

           id="tspan4925-1-2-4-5">reched_cpu()</tspan></text>

           id="tspan3100">context switch</tspan></text>

    </g>

  </g>

</svg>

Documentation/RCU/Design/Expedited-Grace-Periods/Expedited-Grace-Periods.html

+13 −13
Original line number Diff line number Diff line
@@ -72,10 +72,10 @@ will ignore it because idle and offline CPUs are already residing 
in quiescent states.

Otherwise, the expedited grace period will use

<tt>smp_call_function_single()</tt> to send the CPU an IPI, which

is handled by <tt>sync_rcu_exp_handler()</tt>.

is handled by <tt>rcu_exp_handler()</tt>.



<p>

However, because this is preemptible RCU, <tt>sync_rcu_exp_handler()</tt>

However, because this is preemptible RCU, <tt>rcu_exp_handler()</tt>

can check to see if the CPU is currently running in an RCU read-side

critical section.

If not, the handler can immediately report a quiescent state.
@@ -145,19 +145,18 @@ expedited grace period is shown in the following diagram: 
<p><img src="ExpSchedFlow.svg" alt="ExpSchedFlow.svg" width="55%">



<p>

As with RCU-preempt's <tt>synchronize_rcu_expedited()</tt>,

As with RCU-preempt, RCU-sched's

<tt>synchronize_sched_expedited()</tt> ignores offline and

idle CPUs, again because they are in remotely detectable

quiescent states.

However, the <tt>synchronize_rcu_expedited()</tt> handler

is <tt>sync_sched_exp_handler()</tt>, and because the

However, because the

<tt>rcu_read_lock_sched()</tt> and <tt>rcu_read_unlock_sched()</tt>

leave no trace of their invocation, in general it is not possible to tell

whether or not the current CPU is in an RCU read-side critical section.

The best that <tt>sync_sched_exp_handler()</tt> can do is to check

The best that RCU-sched's <tt>rcu_exp_handler()</tt> can do is to check

for idle, on the off-chance that the CPU went idle while the IPI

was in flight.

If the CPU is idle, then <tt>sync_sched_exp_handler()</tt> reports

If the CPU is idle, then <tt>rcu_exp_handler()</tt> reports

the quiescent state.



<p> Otherwise, the handler forces a future context switch by setting the
@@ -298,19 +297,18 @@ Instead, the task pushing the grace period forward will include the 
idle CPUs in the mask passed to <tt>rcu_report_exp_cpu_mult()</tt>.



<p>

For RCU-sched, there is an additional check for idle in the IPI

handler, <tt>sync_sched_exp_handler()</tt>.

For RCU-sched, there is an additional check:

If the IPI has interrupted the idle loop, then

<tt>sync_sched_exp_handler()</tt> invokes <tt>rcu_report_exp_rdp()</tt>

<tt>rcu_exp_handler()</tt> invokes <tt>rcu_report_exp_rdp()</tt>

to report the corresponding quiescent state.



<p>

For RCU-preempt, there is no specific check for idle in the

IPI handler (<tt>sync_rcu_exp_handler()</tt>), but because

IPI handler (<tt>rcu_exp_handler()</tt>), but because

RCU read-side critical sections are not permitted within the

idle loop, if <tt>sync_rcu_exp_handler()</tt> sees that the CPU is within

idle loop, if <tt>rcu_exp_handler()</tt> sees that the CPU is within

RCU read-side critical section, the CPU cannot possibly be idle.

Otherwise, <tt>sync_rcu_exp_handler()</tt> invokes

Otherwise, <tt>rcu_exp_handler()</tt> invokes

<tt>rcu_report_exp_rdp()</tt> to report the corresponding quiescent

state, regardless of whether or not that quiescent state was due to

the CPU being idle.
@@ -625,6 +623,8 @@ checks, but only during the mid-boot dead zone. 
<p>

With this refinement, synchronous grace periods can now be used from

task context pretty much any time during the life of the kernel.

That is, aside from some points in the suspend, hibernate, or shutdown

code path.



<h3><a name="Summary">

Summary</a></h3>

kernel/rcu/tree.h

+0 −1
Original line number Diff line number Diff line
@@ -36,7 +36,6 @@ 


/* Communicate arguments to a workqueue handler. */

struct rcu_exp_work {

	smp_call_func_t rew_func;

	unsigned long rew_s;

	struct work_struct rew_work;

};

kernel/rcu/tree_exp.h

+14 −16
Original line number Diff line number Diff line
@@ -22,6 +22,8 @@ 


#include <linux/lockdep.h>



static void rcu_exp_handler(void *unused);



/*

 * Record the start of an expedited grace period.

 */
@@ -344,7 +346,6 @@ static void sync_rcu_exp_select_node_cpus(struct work_struct *wp) 
{

	int cpu;

	unsigned long flags;

	smp_call_func_t func;

	unsigned long mask_ofl_test;

	unsigned long mask_ofl_ipi;

	int ret;
@@ -352,7 +353,6 @@ static void sync_rcu_exp_select_node_cpus(struct work_struct *wp) 
		container_of(wp, struct rcu_exp_work, rew_work);

	struct rcu_node *rnp = container_of(rewp, struct rcu_node, rew);



	func = rewp->rew_func;

	raw_spin_lock_irqsave_rcu_node(rnp, flags);



	/* Each pass checks a CPU for identity, offline, and idle. */
@@ -396,7 +396,7 @@ retry_ipi: 
			mask_ofl_test |= mask;

			continue;

		}

		ret = smp_call_function_single(cpu, func, NULL, 0);

		ret = smp_call_function_single(cpu, rcu_exp_handler, NULL, 0);

		if (!ret) {

			mask_ofl_ipi &= ~mask;

			continue;
@@ -426,7 +426,7 @@ retry_ipi: 
 * Select the nodes that the upcoming expedited grace period needs

 * to wait for.

 */

static void sync_rcu_exp_select_cpus(smp_call_func_t func)

static void sync_rcu_exp_select_cpus(void)

{

	int cpu;

	struct rcu_node *rnp;
@@ -440,7 +440,6 @@ static void sync_rcu_exp_select_cpus(smp_call_func_t func) 
		rnp->exp_need_flush = false;

		if (!READ_ONCE(rnp->expmask))

			continue; /* Avoid early boot non-existent wq. */

		rnp->rew.rew_func = func;

		if (!READ_ONCE(rcu_par_gp_wq) ||

		    rcu_scheduler_active != RCU_SCHEDULER_RUNNING ||

		    rcu_is_last_leaf_node(rnp)) {
@@ -580,10 +579,10 @@ static void rcu_exp_wait_wake(unsigned long s) 
 * Common code to drive an expedited grace period forward, used by

 * workqueues and mid-boot-time tasks.

 */

static void rcu_exp_sel_wait_wake(smp_call_func_t func, unsigned long s)

static void rcu_exp_sel_wait_wake(unsigned long s)

{

	/* Initialize the rcu_node tree in preparation for the wait. */

	sync_rcu_exp_select_cpus(func);

	sync_rcu_exp_select_cpus();



	/* Wait and clean up, including waking everyone. */

	rcu_exp_wait_wake(s);
@@ -597,14 +596,14 @@ static void wait_rcu_exp_gp(struct work_struct *wp) 
	struct rcu_exp_work *rewp;



	rewp = container_of(wp, struct rcu_exp_work, rew_work);

	rcu_exp_sel_wait_wake(rewp->rew_func, rewp->rew_s);

	rcu_exp_sel_wait_wake(rewp->rew_s);

}



/*

 * Given a smp_call_function() handler, kick off the specified

 * implementation of expedited grace period.

 */

static void _synchronize_rcu_expedited(smp_call_func_t func)

static void _synchronize_rcu_expedited(void)

{

	struct rcu_data *rdp;

	struct rcu_exp_work rew;
@@ -625,10 +624,9 @@ static void _synchronize_rcu_expedited(smp_call_func_t func) 
	/* Ensure that load happens before action based on it. */

	if (unlikely(rcu_scheduler_active == RCU_SCHEDULER_INIT)) {

		/* Direct call during scheduler init and early_initcalls(). */

		rcu_exp_sel_wait_wake(func, s);

		rcu_exp_sel_wait_wake(s);

	} else {

		/* Marshall arguments & schedule the expedited grace period. */

		rew.rew_func = func;

		rew.rew_s = s;

		INIT_WORK_ONSTACK(&rew.rew_work, wait_rcu_exp_gp);

		queue_work(rcu_gp_wq, &rew.rew_work);
@@ -654,7 +652,7 @@ static void _synchronize_rcu_expedited(smp_call_func_t func) 
 * ->expmask fields in the rcu_node tree.  Otherwise, immediately

 * report the quiescent state.

 */

static void sync_rcu_exp_handler(void *unused)

static void rcu_exp_handler(void *unused)

{

	unsigned long flags;

	struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
@@ -760,14 +758,14 @@ void synchronize_rcu_expedited(void) 


	if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE)

		return;

	_synchronize_rcu_expedited(sync_rcu_exp_handler);

	_synchronize_rcu_expedited();

}

EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);



#else /* #ifdef CONFIG_PREEMPT_RCU */



/* Invoked on each online non-idle CPU for expedited quiescent state. */

static void sync_sched_exp_handler(void *unused)

static void rcu_exp_handler(void *unused)

{

	struct rcu_data *rdp;

	struct rcu_node *rnp;
@@ -799,7 +797,7 @@ static void sync_sched_exp_online_cleanup(int cpu) 
	rnp = rdp->mynode;

	if (!(READ_ONCE(rnp->expmask) & rdp->grpmask))

		return;

	ret = smp_call_function_single(cpu, sync_sched_exp_handler, NULL, 0);

	ret = smp_call_function_single(cpu, rcu_exp_handler, NULL, 0);

	WARN_ON_ONCE(ret);

}
@@ -835,7 +833,7 @@ void synchronize_rcu_expedited(void) 
	if (rcu_blocking_is_gp())

		return;



	_synchronize_rcu_expedited(sync_sched_exp_handler);

	_synchronize_rcu_expedited();

}

EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);

CRA Git | Maintained and supported by SUSTech CRA and CCSE