Commit fd581a91 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Move rcu_nmi_{enter,exit}() to prepare for consolidation 



This is a code-motion-only commit that prepares to define rcu_irq_enter()
in terms of rcu_nmi_enter() and rcu_irq_exit() in terms of rcu_irq_exit().

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent a0eb22bf

kernel/rcu/tree.c

+75 −75
Original line number Diff line number Diff line
@@ -866,6 +866,44 @@ void rcu_user_enter(void) 
}

#endif /* CONFIG_NO_HZ_FULL */



/**

 * rcu_nmi_exit - inform RCU of exit from NMI context

 *

 * If we are returning from the outermost NMI handler that interrupted an

 * RCU-idle period, update rdtp->dynticks and rdtp->dynticks_nmi_nesting

 * to let the RCU grace-period handling know that the CPU is back to

 * being RCU-idle.

 *

 * If you add or remove a call to rcu_nmi_exit(), be sure to test

 * with CONFIG_RCU_EQS_DEBUG=y.

 */

void rcu_nmi_exit(void)

{

	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);



	/*

	 * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks.

	 * (We are exiting an NMI handler, so RCU better be paying attention

	 * to us!)

	 */

	WARN_ON_ONCE(rdtp->dynticks_nmi_nesting <= 0);

	WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs());



	/*

	 * If the nesting level is not 1, the CPU wasn't RCU-idle, so

	 * leave it in non-RCU-idle state.

	 */

	if (rdtp->dynticks_nmi_nesting != 1) {

		WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* No store tearing. */

			   rdtp->dynticks_nmi_nesting - 2);

		return;

	}



	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */

	WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */

	rcu_dynticks_eqs_enter();

}



/**

 * rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle

 *
@@ -1012,6 +1050,43 @@ void rcu_user_exit(void) 
}

#endif /* CONFIG_NO_HZ_FULL */



/**

 * rcu_nmi_enter - inform RCU of entry to NMI context

 *

 * If the CPU was idle from RCU's viewpoint, update rdtp->dynticks and

 * rdtp->dynticks_nmi_nesting to let the RCU grace-period handling know

 * that the CPU is active.  This implementation permits nested NMIs, as

 * long as the nesting level does not overflow an int.  (You will probably

 * run out of stack space first.)

 *

 * If you add or remove a call to rcu_nmi_enter(), be sure to test

 * with CONFIG_RCU_EQS_DEBUG=y.

 */

void rcu_nmi_enter(void)

{

	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);

	int incby = 2;



	/* Complain about underflow. */

	WARN_ON_ONCE(rdtp->dynticks_nmi_nesting < 0);



	/*

	 * If idle from RCU viewpoint, atomically increment ->dynticks

	 * to mark non-idle and increment ->dynticks_nmi_nesting by one.

	 * Otherwise, increment ->dynticks_nmi_nesting by two.  This means

	 * if ->dynticks_nmi_nesting is equal to one, we are guaranteed

	 * to be in the outermost NMI handler that interrupted an RCU-idle

	 * period (observation due to Andy Lutomirski).

	 */

	if (rcu_dynticks_curr_cpu_in_eqs()) {

		rcu_dynticks_eqs_exit();

		incby = 1;

	}

	WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* Prevent store tearing. */

		   rdtp->dynticks_nmi_nesting + incby);

	barrier();

}



/**

 * rcu_irq_enter - inform RCU that current CPU is entering irq away from idle

 *
@@ -1070,81 +1145,6 @@ void rcu_irq_enter_irqson(void) 
	local_irq_restore(flags);

}



/**

 * rcu_nmi_enter - inform RCU of entry to NMI context

 *

 * If the CPU was idle from RCU's viewpoint, update rdtp->dynticks and

 * rdtp->dynticks_nmi_nesting to let the RCU grace-period handling know

 * that the CPU is active.  This implementation permits nested NMIs, as

 * long as the nesting level does not overflow an int.  (You will probably

 * run out of stack space first.)

 *

 * If you add or remove a call to rcu_nmi_enter(), be sure to test

 * with CONFIG_RCU_EQS_DEBUG=y.

 */

void rcu_nmi_enter(void)

{

	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);

	int incby = 2;



	/* Complain about underflow. */

	WARN_ON_ONCE(rdtp->dynticks_nmi_nesting < 0);



	/*

	 * If idle from RCU viewpoint, atomically increment ->dynticks

	 * to mark non-idle and increment ->dynticks_nmi_nesting by one.

	 * Otherwise, increment ->dynticks_nmi_nesting by two.  This means

	 * if ->dynticks_nmi_nesting is equal to one, we are guaranteed

	 * to be in the outermost NMI handler that interrupted an RCU-idle

	 * period (observation due to Andy Lutomirski).

	 */

	if (rcu_dynticks_curr_cpu_in_eqs()) {

		rcu_dynticks_eqs_exit();

		incby = 1;

	}

	WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* Prevent store tearing. */

		   rdtp->dynticks_nmi_nesting + incby);

	barrier();

}



/**

 * rcu_nmi_exit - inform RCU of exit from NMI context

 *

 * If we are returning from the outermost NMI handler that interrupted an

 * RCU-idle period, update rdtp->dynticks and rdtp->dynticks_nmi_nesting

 * to let the RCU grace-period handling know that the CPU is back to

 * being RCU-idle.

 *

 * If you add or remove a call to rcu_nmi_exit(), be sure to test

 * with CONFIG_RCU_EQS_DEBUG=y.

 */

void rcu_nmi_exit(void)

{

	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);



	/*

	 * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks.

	 * (We are exiting an NMI handler, so RCU better be paying attention

	 * to us!)

	 */

	WARN_ON_ONCE(rdtp->dynticks_nmi_nesting <= 0);

	WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs());



	/*

	 * If the nesting level is not 1, the CPU wasn't RCU-idle, so

	 * leave it in non-RCU-idle state.

	 */

	if (rdtp->dynticks_nmi_nesting != 1) {

		WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* No store tearing. */

			   rdtp->dynticks_nmi_nesting - 2);

		return;

	}



	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */

	WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */

	rcu_dynticks_eqs_enter();

}



/**

 * rcu_is_watching - see if RCU thinks that the current CPU is idle

 *

CRA Git | Maintained and supported by SUSTech CRA and CCSE