Commit 1098582a authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

sched,idle,rcu: Push rcu_idle deeper into the idle path 



Lots of things take locks, due to a wee bug, rcu_lockdep didn't notice
that the locking tracepoints were using RCU.

Push rcu_idle_{enter,exit}() as deep as possible into the idle paths,
this also resolves a lot of _rcuidle()/RCU_NONIDLE() usage.

Specifically, sched_clock_idle_wakeup_event() will use ktime which
will use seqlocks which will tickle lockdep, and
stop_critical_timings() uses lock.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarSteven Rostedt (VMware) <rostedt@goodmis.org>
Reviewed-by: default avatarThomas Gleixner <tglx@linutronix.de>
Acked-by: default avatarRafael J. Wysocki <rafael.j.wysocki@intel.com>
Tested-by: default avatarMarco Elver <elver@google.com>
Link: https://lkml.kernel.org/r/20200821085348.310943801@infradead.org
parent 49d9c593

drivers/cpuidle/cpuidle.c

+8 −4
Original line number Diff line number Diff line
@@ -145,13 +145,14 @@ static void enter_s2idle_proper(struct cpuidle_driver *drv, 
	 * executing it contains RCU usage regarded as invalid in the idle

	 * context, so tell RCU about that.

	 */

	RCU_NONIDLE(tick_freeze());

	tick_freeze();

	/*

	 * The state used here cannot be a "coupled" one, because the "coupled"

	 * cpuidle mechanism enables interrupts and doing that with timekeeping

	 * suspended is generally unsafe.

	 */

	stop_critical_timings();

	rcu_idle_enter();

	drv->states[index].enter_s2idle(dev, drv, index);

	if (WARN_ON_ONCE(!irqs_disabled()))

		local_irq_disable();
@@ -160,7 +161,8 @@ static void enter_s2idle_proper(struct cpuidle_driver *drv, 
	 * first CPU executing it calls functions containing RCU read-side

	 * critical sections, so tell RCU about that.

	 */

	RCU_NONIDLE(tick_unfreeze());

	rcu_idle_exit();

	tick_unfreeze();

	start_critical_timings();



	time_end = ns_to_ktime(local_clock());
@@ -229,16 +231,18 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, 
	/* Take note of the planned idle state. */

	sched_idle_set_state(target_state);



	trace_cpu_idle_rcuidle(index, dev->cpu);

	trace_cpu_idle(index, dev->cpu);

	time_start = ns_to_ktime(local_clock());



	stop_critical_timings();

	rcu_idle_enter();

	entered_state = target_state->enter(dev, drv, index);

	rcu_idle_exit();

	start_critical_timings();



	sched_clock_idle_wakeup_event();

	time_end = ns_to_ktime(local_clock());

	trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu);

	trace_cpu_idle(PWR_EVENT_EXIT, dev->cpu);



	/* The cpu is no longer idle or about to enter idle. */

	sched_idle_set_state(NULL);

kernel/sched/idle.c

+8 −14
Original line number Diff line number Diff line
@@ -54,17 +54,18 @@ __setup("hlt", cpu_idle_nopoll_setup); 


static noinline int __cpuidle cpu_idle_poll(void)

{

	trace_cpu_idle(0, smp_processor_id());

	stop_critical_timings();

	rcu_idle_enter();

	trace_cpu_idle_rcuidle(0, smp_processor_id());

	local_irq_enable();

	stop_critical_timings();



	while (!tif_need_resched() &&

	       (cpu_idle_force_poll || tick_check_broadcast_expired()))

		cpu_relax();

	start_critical_timings();

	trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());



	rcu_idle_exit();

	start_critical_timings();

	trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());



	return 1;

}
@@ -91,7 +92,9 @@ void __cpuidle default_idle_call(void) 
		local_irq_enable();

	} else {

		stop_critical_timings();

		rcu_idle_enter();

		arch_cpu_idle();

		rcu_idle_exit();

		start_critical_timings();

	}

}
@@ -158,7 +161,6 @@ static void cpuidle_idle_call(void) 


	if (cpuidle_not_available(drv, dev)) {

		tick_nohz_idle_stop_tick();

		rcu_idle_enter();



		default_idle_call();

		goto exit_idle;
@@ -178,21 +180,17 @@ static void cpuidle_idle_call(void) 
		u64 max_latency_ns;



		if (idle_should_enter_s2idle()) {

			rcu_idle_enter();



			entered_state = call_cpuidle_s2idle(drv, dev);

			if (entered_state > 0)

				goto exit_idle;



			rcu_idle_exit();



			max_latency_ns = U64_MAX;

		} else {

			max_latency_ns = dev->forced_idle_latency_limit_ns;

		}



		tick_nohz_idle_stop_tick();

		rcu_idle_enter();



		next_state = cpuidle_find_deepest_state(drv, dev, max_latency_ns);

		call_cpuidle(drv, dev, next_state);
@@ -209,8 +207,6 @@ static void cpuidle_idle_call(void) 
		else

			tick_nohz_idle_retain_tick();



		rcu_idle_enter();



		entered_state = call_cpuidle(drv, dev, next_state);

		/*

		 * Give the governor an opportunity to reflect on the outcome
@@ -226,8 +222,6 @@ exit_idle: 
	 */

	if (WARN_ON_ONCE(irqs_disabled()))

		local_irq_enable();



	rcu_idle_exit();

}



/*

CRA Git | Maintained and supported by SUSTech CRA and CCSE