Commit c1ab99d6 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu/nocb: Use build-time no-CBs check in rcu_core() 



Currently, rcu_core() invokes rcu_segcblist_is_offloaded() each time it
needs to know whether the current CPU is a no-CBs CPU.  Given that it is
not possible to change the no-CBs status of a CPU after boot, and given
that it is not possible to even have no-CBs CPUs in CONFIG_RCU_NOCB_CPU=n
kernels, this repeated runtime invocation wastes CPU.  This commit
therefore created a const on-stack variable to allow this check to be
done only once per rcu_core() invocation.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.ibm.com>
parent ec5ef87b

kernel/rcu/tree.c

+4 −4
Original line number Diff line number Diff line
@@ -2302,6 +2302,8 @@ static __latent_entropy void rcu_core(void) 
	unsigned long flags;

	struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);

	struct rcu_node *rnp = rdp->mynode;

	const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&

			       rcu_segcblist_is_offloaded(&rdp->cblist);



	if (cpu_is_offline(smp_processor_id()))

		return;
@@ -2321,8 +2323,7 @@ static __latent_entropy void rcu_core(void) 


	/* No grace period and unregistered callbacks? */

	if (!rcu_gp_in_progress() &&

	    rcu_segcblist_is_enabled(&rdp->cblist) &&

	    !rcu_segcblist_is_offloaded(&rdp->cblist)) {

	    rcu_segcblist_is_enabled(&rdp->cblist) && !offloaded) {

		local_irq_save(flags);

		if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))

			rcu_accelerate_cbs_unlocked(rnp, rdp);
@@ -2332,8 +2333,7 @@ static __latent_entropy void rcu_core(void) 
	rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check());



	/* If there are callbacks ready, invoke them. */

	if (!rcu_segcblist_is_offloaded(&rdp->cblist) &&

	    rcu_segcblist_ready_cbs(&rdp->cblist) &&

	if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist) &&

	    likely(READ_ONCE(rcu_scheduler_fully_active)))

		rcu_do_batch(rdp);

CRA Git | Maintained and supported by SUSTech CRA and CCSE