Commit c0f4dfd4 authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Paul E. McKenney
Browse files

rcu: Make RCU_FAST_NO_HZ take advantage of numbered callbacks 



Because RCU callbacks are now associated with the number of the grace
period that they must wait for, CPUs can now take advance callbacks
corresponding to grace periods that ended while a given CPU was in
dyntick-idle mode.  This eliminates the need to try forcing the RCU
state machine while entering idle, thus reducing the CPU intensiveness
of RCU_FAST_NO_HZ, which should increase its energy efficiency.

Signed-off-by: default avatarPaul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent b11cc576

Documentation/kernel-parameters.txt

+19 −9
Original line number Diff line number Diff line
@@ -2490,6 +2490,17 @@ bytes respectively. Such letter suffixes can also be entirely omitted. 
			leaf rcu_node structure.  Useful for very large

			systems.



	rcutree.jiffies_till_first_fqs= [KNL,BOOT]

			Set delay from grace-period initialization to

			first attempt to force quiescent states.

			Units are jiffies, minimum value is zero,

			and maximum value is HZ.



	rcutree.jiffies_till_next_fqs= [KNL,BOOT]

			Set delay between subsequent attempts to force

			quiescent states.  Units are jiffies, minimum

			value is one, and maximum value is HZ.



	rcutree.qhimark=	[KNL,BOOT]

			Set threshold of queued

			RCU callbacks over which batch limiting is disabled.
@@ -2504,16 +2515,15 @@ bytes respectively. Such letter suffixes can also be entirely omitted. 
	rcutree.rcu_cpu_stall_timeout= [KNL,BOOT]

			Set timeout for RCU CPU stall warning messages.



	rcutree.jiffies_till_first_fqs= [KNL,BOOT]

			Set delay from grace-period initialization to

			first attempt to force quiescent states.

			Units are jiffies, minimum value is zero,

			and maximum value is HZ.

	rcutree.rcu_idle_gp_delay=	[KNL,BOOT]

			Set wakeup interval for idle CPUs that have

			RCU callbacks (RCU_FAST_NO_HZ=y).



	rcutree.jiffies_till_next_fqs= [KNL,BOOT]

			Set delay between subsequent attempts to force

			quiescent states.  Units are jiffies, minimum

			value is one, and maximum value is HZ.

	rcutree.rcu_idle_lazy_gp_delay=	[KNL,BOOT]

			Set wakeup interval for idle CPUs that have

			only "lazy" RCU callbacks (RCU_FAST_NO_HZ=y).

			Lazy RCU callbacks are those which RCU can

			prove do nothing more than free memory.



	rcutorture.fqs_duration= [KNL,BOOT]

			Set duration of force_quiescent_state bursts.

include/linux/rcupdate.h

+1 −0
Original line number Diff line number Diff line
@@ -80,6 +80,7 @@ extern void do_trace_rcu_torture_read(char *rcutorturename, 
#define UINT_CMP_LT(a, b)	(UINT_MAX / 2 < (a) - (b))

#define ULONG_CMP_GE(a, b)	(ULONG_MAX / 2 >= (a) - (b))

#define ULONG_CMP_LT(a, b)	(ULONG_MAX / 2 < (a) - (b))

#define ulong2long(a)		(*(long *)(&(a)))



/* Exported common interfaces */

init/Kconfig

+10 −7
Original line number Diff line number Diff line
@@ -582,13 +582,16 @@ config RCU_FAST_NO_HZ 
	depends on NO_HZ && SMP

	default n

	help

	  This option causes RCU to attempt to accelerate grace periods in

	  order to allow CPUs to enter dynticks-idle state more quickly.

	  On the other hand, this option increases the overhead of the

	  dynticks-idle checking, thus degrading scheduling latency.



	  Say Y if energy efficiency is critically important, and you don't

	  	care about real-time response.

	  This option permits CPUs to enter dynticks-idle state even if

	  they have RCU callbacks queued, and prevents RCU from waking

	  these CPUs up more than roughly once every four jiffies (by

	  default, you can adjust this using the rcutree.rcu_idle_gp_delay

	  parameter), thus improving energy efficiency.  On the other

	  hand, this option increases the duration of RCU grace periods,

	  for example, slowing down synchronize_rcu().



	  Say Y if energy efficiency is critically important, and you

	  	don't care about increased grace-period durations.



	  Say N if you are unsure.

kernel/rcutree.c

+17 −11
Original line number Diff line number Diff line
@@ -2640,19 +2640,27 @@ static int rcu_pending(int cpu) 
}



/*

 * Check to see if any future RCU-related work will need to be done

 * by the current CPU, even if none need be done immediately, returning

 * 1 if so.

 * Return true if the specified CPU has any callback.  If all_lazy is

 * non-NULL, store an indication of whether all callbacks are lazy.

 * (If there are no callbacks, all of them are deemed to be lazy.)

 */

static int rcu_cpu_has_callbacks(int cpu)

static int rcu_cpu_has_callbacks(int cpu, bool *all_lazy)

{

	bool al = true;

	bool hc = false;

	struct rcu_data *rdp;

	struct rcu_state *rsp;



	/* RCU callbacks either ready or pending? */

	for_each_rcu_flavor(rsp)

		if (per_cpu_ptr(rsp->rda, cpu)->nxtlist)

			return 1;

	return 0;

	for_each_rcu_flavor(rsp) {

		rdp = per_cpu_ptr(rsp->rda, cpu);

		if (rdp->qlen != rdp->qlen_lazy)

			al = false;

		if (rdp->nxtlist)

			hc = true;

	}

	if (all_lazy)

		*all_lazy = al;

	return hc;

}



/*
@@ -2871,7 +2879,6 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptible) 
	rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;

	atomic_set(&rdp->dynticks->dynticks,

		   (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);

	rcu_prepare_for_idle_init(cpu);

	raw_spin_unlock(&rnp->lock);		/* irqs remain disabled. */



	/* Add CPU to rcu_node bitmasks. */
@@ -2945,7 +2952,6 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block *self, 
		 */

		for_each_rcu_flavor(rsp)

			rcu_cleanup_dying_cpu(rsp);

		rcu_cleanup_after_idle(cpu);

		break;

	case CPU_DEAD:

	case CPU_DEAD_FROZEN:

kernel/rcutree.h

+3 −9
Original line number Diff line number Diff line
@@ -88,18 +88,13 @@ struct rcu_dynticks { 
	int dynticks_nmi_nesting;   /* Track NMI nesting level. */

	atomic_t dynticks;	    /* Even value for idle, else odd. */

#ifdef CONFIG_RCU_FAST_NO_HZ

	int dyntick_drain;	    /* Prepare-for-idle state variable. */

	unsigned long dyntick_holdoff;

				    /* No retries for the jiffy of failure. */

	struct timer_list idle_gp_timer;

				    /* Wake up CPU sleeping with callbacks. */

	unsigned long idle_gp_timer_expires;

				    /* When to wake up CPU (for repost). */

	bool idle_first_pass;	    /* First pass of attempt to go idle? */

	bool all_lazy;		    /* Are all CPU's CBs lazy? */

	unsigned long nonlazy_posted;

				    /* # times non-lazy CBs posted to CPU. */

	unsigned long nonlazy_posted_snap;

				    /* idle-period nonlazy_posted snapshot. */

	unsigned long last_accelerate;

				    /* Last jiffy CBs were accelerated. */

	int tick_nohz_enabled_snap; /* Previously seen value from sysfs. */

#endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */

};
@@ -521,7 +516,6 @@ static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp, 
						 struct rcu_node *rnp);

#endif /* #ifdef CONFIG_RCU_BOOST */

static void __cpuinit rcu_prepare_kthreads(int cpu);

static void rcu_prepare_for_idle_init(int cpu);

static void rcu_cleanup_after_idle(int cpu);

static void rcu_prepare_for_idle(int cpu);

static void rcu_idle_count_callbacks_posted(void);

CRA Git | Maintained and supported by SUSTech CRA and CCSE