Commit 22607d66 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

srcu: Specify auto-expedite holdoff time 



On small systems, in the absence of readers, expedited SRCU grace
periods can complete in less than a microsecond.  This means that an
eight-CPU system can have all CPUs doing synchronize_srcu() in a tight
loop and almost always expedite.  This might actually be desirable in
some situations, but in general it is a good way to needlessly burn
CPU cycles.  And in those situations where it is desirable, your friend
is the function synchronize_srcu_expedited().

For other situations, this commit adds a kernel parameter that specifies
a holdoff between completing the last SRCU grace period and auto-expediting
the next.  If the next grace period starts before the holdoff expires,
auto-expediting is disabled.  The holdoff is 50 microseconds by default,
and can be tuned to the desired number of nanoseconds.  A value of zero
disables auto-expediting.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Tested-by: default avatarMike Galbraith <efault@gmx.de>
parent 2da4b2a7

Documentation/admin-guide/kernel-parameters.txt

+8 −0
Original line number Diff line number Diff line
@@ -3779,6 +3779,14 @@ 
	spia_pedr=

	spia_peddr=



	srcutree.exp_holdoff [KNL]

			Specifies how many nanoseconds must elapse

			since the end of the last SRCU grace period for

			a given srcu_struct until the next normal SRCU

			grace period will be considered for automatic

			expediting.  Set to zero to disable automatic

			expediting.



	stacktrace	[FTRACE]

			Enabled the stack tracer on boot up.

include/linux/srcutree.h

+1 −0
Original line number Diff line number Diff line
@@ -84,6 +84,7 @@ struct srcu_struct { 
	unsigned long srcu_gp_seq;		/* Grace-period seq #. */

	unsigned long srcu_gp_seq_needed;	/* Latest gp_seq needed. */

	unsigned long srcu_gp_seq_needed_exp;	/* Furthest future exp GP. */

	unsigned long srcu_last_gp_end;		/* Last GP end timestamp (ns) */

	struct srcu_data __percpu *sda;		/* Per-CPU srcu_data array. */

	unsigned long srcu_barrier_seq;		/* srcu_barrier seq #. */

	struct mutex srcu_barrier_mutex;	/* Serialize barrier ops. */

kernel/rcu/srcutree.c

+17 −1
Original line number Diff line number Diff line
@@ -34,10 +34,14 @@ 
#include <linux/sched.h>

#include <linux/smp.h>

#include <linux/delay.h>

#include <linux/module.h>

#include <linux/srcu.h>



#include "rcu.h"



ulong exp_holdoff = 50 * 1000; /* Holdoff (ns) for auto-expediting. */

module_param(exp_holdoff, ulong, 0444);



static void srcu_invoke_callbacks(struct work_struct *work);

static void srcu_reschedule(struct srcu_struct *sp, unsigned long delay);
@@ -145,6 +149,7 @@ static int init_srcu_struct_fields(struct srcu_struct *sp, bool is_static) 
		sp->sda = alloc_percpu(struct srcu_data);

	init_srcu_struct_nodes(sp, is_static);

	sp->srcu_gp_seq_needed_exp = 0;

	sp->srcu_last_gp_end = ktime_get_mono_fast_ns();

	smp_store_release(&sp->srcu_gp_seq_needed, 0); /* Init done. */

	return sp->sda ? 0 : -ENOMEM;

}
@@ -498,6 +503,7 @@ static void srcu_gp_end(struct srcu_struct *sp) 
	idx = rcu_seq_state(sp->srcu_gp_seq);

	WARN_ON_ONCE(idx != SRCU_STATE_SCAN2);

	cbdelay = srcu_get_delay(sp);

	sp->srcu_last_gp_end = ktime_get_mono_fast_ns();

	rcu_seq_end(&sp->srcu_gp_seq);

	gpseq = rcu_seq_current(&sp->srcu_gp_seq);

	if (ULONG_CMP_LT(sp->srcu_gp_seq_needed_exp, gpseq))
@@ -700,9 +706,10 @@ static void srcu_flip(struct srcu_struct *sp) 
 */

static bool srcu_might_be_idle(struct srcu_struct *sp)

{

	unsigned long curseq;

	unsigned long flags;

	struct srcu_data *sdp;

	unsigned long curseq;

	unsigned long t;



	/* If the local srcu_data structure has callbacks, not idle.  */

	local_irq_save(flags);
@@ -718,6 +725,15 @@ static bool srcu_might_be_idle(struct srcu_struct *sp) 
	 * Exact information would require acquiring locks, which would

	 * kill scalability, hence the probabalistic nature of the probe.

	 */



	/* First, see if enough time has passed since the last GP. */

	t = ktime_get_mono_fast_ns();

	if (exp_holdoff == 0 ||

	    time_in_range_open(t, sp->srcu_last_gp_end,

			       sp->srcu_last_gp_end + exp_holdoff))

		return false; /* Too soon after last GP. */



	/* Next, check for probable idleness. */

	curseq = rcu_seq_current(&sp->srcu_gp_seq);

	smp_mb(); /* Order ->srcu_gp_seq with ->srcu_gp_seq_needed. */

	if (ULONG_CMP_LT(curseq, READ_ONCE(sp->srcu_gp_seq_needed)))

CRA Git | Maintained and supported by SUSTech CRA and CCSE