Commit 281d150c authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Ingo Molnar
Browse files

rcu: Prepare for synchronization fixes: clean up for non-NO_HZ handling of ->completed counter 



Impose a clear locking design on non-NO_HZ handling of the
->completed counter.  This increases the distance between the
RCU and the CPU-hotplug mechanisms.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
Cc: <stable@kernel.org> # .32.x
LKML-Reference: <12571987491353-git-send-email->
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent 7e1a2766

kernel/rcutree.c

+30 −37
Original line number Diff line number Diff line
@@ -178,8 +178,28 @@ static struct rcu_node *rcu_get_root(struct rcu_state *rsp) 
	return &rsp->node[0];

}



/*

 * Record the specified "completed" value, which is later used to validate

 * dynticks counter manipulations and CPU-offline checks.  Specify

 * "rsp->completed - 1" to unconditionally invalidate any future dynticks

 * manipulations and CPU-offline checks.  Such invalidation is useful at

 * the beginning of a grace period.

 */

static void dyntick_record_completed(struct rcu_state *rsp, long comp)

{

	rsp->dynticks_completed = comp;

}



#ifdef CONFIG_SMP



/*

 * Recall the previously recorded value of the completion for dynticks.

 */

static long dyntick_recall_completed(struct rcu_state *rsp)

{

	return rsp->dynticks_completed;

}



/*

 * If the specified CPU is offline, tell the caller that it is in

 * a quiescent state.  Otherwise, whack it with a reschedule IPI.
@@ -337,27 +357,8 @@ void rcu_irq_exit(void) 
		set_need_resched();

}



/*

 * Record the specified "completed" value, which is later used to validate

 * dynticks counter manipulations.  Specify "rsp->completed - 1" to

 * unconditionally invalidate any future dynticks manipulations (which is

 * useful at the beginning of a grace period).

 */

static void dyntick_record_completed(struct rcu_state *rsp, long comp)

{

	rsp->dynticks_completed = comp;

}



#ifdef CONFIG_SMP



/*

 * Recall the previously recorded value of the completion for dynticks.

 */

static long dyntick_recall_completed(struct rcu_state *rsp)

{

	return rsp->dynticks_completed;

}



/*

 * Snapshot the specified CPU's dynticks counter so that we can later

 * credit them with an implicit quiescent state.  Return 1 if this CPU
@@ -421,24 +422,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) 


#else /* #ifdef CONFIG_NO_HZ */



static void dyntick_record_completed(struct rcu_state *rsp, long comp)

{

}



#ifdef CONFIG_SMP



/*

 * If there are no dynticks, then the only way that a CPU can passively

 * be in a quiescent state is to be offline.  Unlike dynticks idle, which

 * is a point in time during the prior (already finished) grace period,

 * an offline CPU is always in a quiescent state, and thus can be

 * unconditionally applied.  So just return the current value of completed.

 */

static long dyntick_recall_completed(struct rcu_state *rsp)

{

	return rsp->completed;

}



static int dyntick_save_progress_counter(struct rcu_data *rdp)

{

	return 0;
@@ -1146,6 +1131,7 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed) 
	long lastcomp;

	struct rcu_node *rnp = rcu_get_root(rsp);

	u8 signaled;

	u8 forcenow;



	if (!rcu_gp_in_progress(rsp))

		return;  /* No grace period in progress, nothing to force. */
@@ -1182,16 +1168,23 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed) 
		if (rcu_process_dyntick(rsp, lastcomp,

					dyntick_save_progress_counter))

			goto unlock_ret;

		/* fall into next case. */



	case RCU_SAVE_COMPLETED:



		/* Update state, record completion counter. */

		forcenow = 0;

		spin_lock(&rnp->lock);

		if (lastcomp == rsp->completed &&

		    rsp->signaled == RCU_SAVE_DYNTICK) {

		    rsp->signaled == signaled) {

			rsp->signaled = RCU_FORCE_QS;

			dyntick_record_completed(rsp, lastcomp);

			forcenow = signaled == RCU_SAVE_COMPLETED;

		}

		spin_unlock(&rnp->lock);

		if (!forcenow)

			break;

		/* fall into next case. */



	case RCU_FORCE_QS:

kernel/rcutree.h

+4 −4
Original line number Diff line number Diff line
@@ -204,11 +204,12 @@ struct rcu_data { 
#define RCU_GP_IDLE		0	/* No grace period in progress. */

#define RCU_GP_INIT		1	/* Grace period being initialized. */

#define RCU_SAVE_DYNTICK	2	/* Need to scan dyntick state. */

#define RCU_FORCE_QS		3	/* Need to force quiescent state. */

#define RCU_SAVE_COMPLETED	3	/* Need to save rsp->completed. */

#define RCU_FORCE_QS		4	/* Need to force quiescent state. */

#ifdef CONFIG_NO_HZ

#define RCU_SIGNAL_INIT		RCU_SAVE_DYNTICK

#else /* #ifdef CONFIG_NO_HZ */

#define RCU_SIGNAL_INIT		RCU_FORCE_QS

#define RCU_SIGNAL_INIT		RCU_SAVE_COMPLETED

#endif /* #else #ifdef CONFIG_NO_HZ */



#define RCU_JIFFIES_TILL_FORCE_QS	 3	/* for rsp->jiffies_force_qs */
@@ -274,9 +275,8 @@ struct rcu_state { 
	unsigned long jiffies_stall;		/* Time at which to check */

						/*  for CPU stalls. */

#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */

#ifdef CONFIG_NO_HZ

	long dynticks_completed;		/* Value of completed @ snap. */

#endif /* #ifdef CONFIG_NO_HZ */

						/*  Protected by fqslock. */

};



#ifdef RCU_TREE_NONCORE

CRA Git | Maintained and supported by SUSTech CRA and CCSE