Commit 62fde6ed authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Paul E. McKenney
Browse files

rcu: Make __call_rcu() handle invocation from idle 



Although __call_rcu() is handled correctly when called from a momentary
non-idle period, if it is called on a CPU that RCU believes to be idle
on RCU_FAST_NO_HZ kernels, the callback might be indefinitely postponed.
This commit therefore ensures that RCU is aware of the new callback and
has a chance to force the CPU out of dyntick-idle mode when a new callback
is posted.

Reported-by: default avatarFrederic Weisbecker <fweisbec@gmail.com>
Signed-off-by: default avatarPaul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: default avatarJosh Triplett <josh@joshtriplett.org>
parent 28f5c693

include/linux/rcupdate.h

+4 −9
Original line number Diff line number Diff line
@@ -256,6 +256,10 @@ static inline void destroy_rcu_head_on_stack(struct rcu_head *head) 
}

#endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */



#if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_SMP)

extern int rcu_is_cpu_idle(void);

#endif /* #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_SMP) */



#if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU)

bool rcu_lockdep_current_cpu_online(void);

#else /* #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */
@@ -267,15 +271,6 @@ static inline bool rcu_lockdep_current_cpu_online(void) 


#ifdef CONFIG_DEBUG_LOCK_ALLOC



#ifdef CONFIG_PROVE_RCU

extern int rcu_is_cpu_idle(void);

#else /* !CONFIG_PROVE_RCU */

static inline int rcu_is_cpu_idle(void)

{

	return 0;

}

#endif /* else !CONFIG_PROVE_RCU */



static inline void rcu_lock_acquire(struct lockdep_map *map)

{

	lock_acquire(map, 0, 0, 2, 1, NULL, _THIS_IP_);

kernel/rcutree.c

+9 −6
Original line number Diff line number Diff line
@@ -585,8 +585,6 @@ void rcu_nmi_exit(void) 
	WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);

}



#ifdef CONFIG_PROVE_RCU



/**

 * rcu_is_cpu_idle - see if RCU thinks that the current CPU is idle

 *
@@ -604,7 +602,7 @@ int rcu_is_cpu_idle(void) 
}

EXPORT_SYMBOL(rcu_is_cpu_idle);



#ifdef CONFIG_HOTPLUG_CPU

#if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU)



/*

 * Is the current CPU online?  Disable preemption to avoid false positives
@@ -645,9 +643,7 @@ bool rcu_lockdep_current_cpu_online(void) 
}

EXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online);



#endif /* #ifdef CONFIG_HOTPLUG_CPU */



#endif /* #ifdef CONFIG_PROVE_RCU */

#endif /* #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU) */



/**

 * rcu_is_cpu_rrupt_from_idle - see if idle or immediately interrupted from idle
@@ -1904,6 +1900,13 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu), 
	else

		trace_rcu_callback(rsp->name, head, rdp->qlen_lazy, rdp->qlen);



	/*

	 * If called from an extended quiescent state, invoke the RCU

	 * core in order to force a re-evaluation of RCU's idleness.

	 */

	if (rcu_is_cpu_idle())

		invoke_rcu_core();



	/* If interrupts were disabled, don't dive into RCU core. */

	if (irqs_disabled_flags(flags)) {

		local_irq_restore(flags);

CRA Git | Maintained and supported by SUSTech CRA and CCSE