Commit 5871968d authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Tighten up affinity and check for sysidle 



If the RCU grace-period kthread invoking rcu_sysidle_check_cpu()
happens to be running on the tick_do_timer_cpu initially,
then rcu_bind_gp_kthread() won't bind it.  This kthread might
then migrate before invoking rcu_gp_fqs(), which will trigger the
WARN_ON_ONCE() in rcu_sysidle_check_cpu().  This commit therefore makes
rcu_bind_gp_kthread() do the binding even if the kthread is currently
on the same CPU.  Because this incurs added overhead, this commit also
causes each RCU grace-period kthread to invoke rcu_bind_gp_kthread()
once at boot rather than at the beginning of each grace period.
And as long as rcu_bind_gp_kthread() is being modified, this commit
eliminates its #ifdef.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 675da67f

kernel/rcu/tree.c

+1 −1
Original line number Diff line number Diff line
@@ -1707,7 +1707,6 @@ static int rcu_gp_init(struct rcu_state *rsp) 
	struct rcu_node *rnp = rcu_get_root(rsp);



	ACCESS_ONCE(rsp->gp_activity) = jiffies;

	rcu_bind_gp_kthread();

	raw_spin_lock_irq(&rnp->lock);

	smp_mb__after_unlock_lock();

	if (!ACCESS_ONCE(rsp->gp_flags)) {
@@ -1895,6 +1894,7 @@ static int __noreturn rcu_gp_kthread(void *arg) 
	struct rcu_state *rsp = arg;

	struct rcu_node *rnp = rcu_get_root(rsp);



	rcu_bind_gp_kthread();

	for (;;) {



		/* Handle grace-period start. */

kernel/rcu/tree_plugin.h

+6 −6
Original line number Diff line number Diff line
@@ -2763,7 +2763,8 @@ static void rcu_sysidle_exit(int irq) 


/*

 * Check to see if the current CPU is idle.  Note that usermode execution

 * does not count as idle.  The caller must have disabled interrupts.

 * does not count as idle.  The caller must have disabled interrupts,

 * and must be running on tick_do_timer_cpu.

 */

static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,

				  unsigned long *maxj)
@@ -2784,7 +2785,7 @@ static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle, 
	if (!*isidle || rdp->rsp != rcu_state_p ||

	    cpu_is_offline(rdp->cpu) || rdp->cpu == tick_do_timer_cpu)

		return;

	if (rcu_gp_in_progress(rdp->rsp))

	/* Verify affinity of current kthread. */

	WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu);



	/* Pick up current idle and NMI-nesting counter and check. */
@@ -3068,10 +3069,9 @@ static void rcu_bind_gp_kthread(void) 
		return;

#ifdef CONFIG_NO_HZ_FULL_SYSIDLE

	cpu = tick_do_timer_cpu;

	if (cpu >= 0 && cpu < nr_cpu_ids && raw_smp_processor_id() != cpu)

	if (cpu >= 0 && cpu < nr_cpu_ids)

		set_cpus_allowed_ptr(current, cpumask_of(cpu));

#else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */

	if (!is_housekeeping_cpu(raw_smp_processor_id()))

	housekeeping_affine(current);

#endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE