Commit 77f81fe0 authored by Petr Mladek's avatar Petr Mladek Committed by Paul E. McKenney
Browse files

rcu: Finish folding ->fqs_state into ->gp_state 



Commit commit 4cdfc175 ("rcu: Move quiescent-state forcing
into kthread") started the process of folding the old ->fqs_state into
->gp_state, but did not complete it.  This situation does not cause
any malfunction, but can result in extremely confusing trace output.
This commit completes this task of eliminating ->fqs_state in favor
of ->gp_state.

The old ->fqs_state was also used to decide when to collect dyntick-idle
snapshots.  For this purpose, we add a boolean variable into the kthread,
which is set on the first call to rcu_gp_fqs() for a given grace period
and clear otherwise.

Signed-off-by: default avatarPetr Mladek <pmladek@suse.com>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: default avatarJosh Triplett <josh@joshtriplett.org>
parent 49f5903b

kernel/rcu/tree.c

+8 −10
Original line number Original line Diff line number Diff line
@@ -98,7 +98,7 @@ struct rcu_state sname##_state = { \ 
	.level = { &sname##_state.node[0] }, \

	.level = { &sname##_state.node[0] }, \

	.rda = &sname##_data, \

	.rda = &sname##_data, \

	.call = cr, \

	.call = cr, \

	.fqs_state = RCU_GP_IDLE, \

	.gp_state = RCU_GP_IDLE, \

	.gpnum = 0UL - 300UL, \

	.gpnum = 0UL - 300UL, \

	.completed = 0UL - 300UL, \

	.completed = 0UL - 300UL, \

	.orphan_lock = __RAW_SPIN_LOCK_UNLOCKED(&sname##_state.orphan_lock), \

	.orphan_lock = __RAW_SPIN_LOCK_UNLOCKED(&sname##_state.orphan_lock), \
@@ -1936,16 +1936,15 @@ static bool rcu_gp_fqs_check_wake(struct rcu_state *rsp, int *gfp) 
/*

/*

 * Do one round of quiescent-state forcing.

 * Do one round of quiescent-state forcing.

 */

 */

static int rcu_gp_fqs(struct rcu_state *rsp, int fqs_state_in)

static void rcu_gp_fqs(struct rcu_state *rsp, bool first_time)

{

{

	int fqs_state = fqs_state_in;

	bool isidle = false;

	bool isidle = false;

	unsigned long maxj;

	unsigned long maxj;

	struct rcu_node *rnp = rcu_get_root(rsp);

	struct rcu_node *rnp = rcu_get_root(rsp);





	WRITE_ONCE(rsp->gp_activity, jiffies);

	WRITE_ONCE(rsp->gp_activity, jiffies);

	rsp->n_force_qs++;

	rsp->n_force_qs++;

	if (fqs_state == RCU_SAVE_DYNTICK) {

	if (first_time) {

		/* Collect dyntick-idle snapshots. */

		/* Collect dyntick-idle snapshots. */

		if (is_sysidle_rcu_state(rsp)) {

		if (is_sysidle_rcu_state(rsp)) {

			isidle = true;

			isidle = true;
@@ -1954,7 +1953,6 @@ static int rcu_gp_fqs(struct rcu_state *rsp, int fqs_state_in) 
		force_qs_rnp(rsp, dyntick_save_progress_counter,

		force_qs_rnp(rsp, dyntick_save_progress_counter,

			     &isidle, &maxj);

			     &isidle, &maxj);

		rcu_sysidle_report_gp(rsp, isidle, maxj);

		rcu_sysidle_report_gp(rsp, isidle, maxj);

		fqs_state = RCU_FORCE_QS;

	} else {

	} else {

		/* Handle dyntick-idle and offline CPUs. */

		/* Handle dyntick-idle and offline CPUs. */

		isidle = true;

		isidle = true;
@@ -1968,7 +1966,6 @@ static int rcu_gp_fqs(struct rcu_state *rsp, int fqs_state_in) 
			   READ_ONCE(rsp->gp_flags) & ~RCU_GP_FLAG_FQS);

			   READ_ONCE(rsp->gp_flags) & ~RCU_GP_FLAG_FQS);

		raw_spin_unlock_irq(&rnp->lock);

		raw_spin_unlock_irq(&rnp->lock);

	}

	}

	return fqs_state;

}

}





/*

/*
@@ -2032,7 +2029,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp) 
	/* Declare grace period done. */

	/* Declare grace period done. */

	WRITE_ONCE(rsp->completed, rsp->gpnum);

	WRITE_ONCE(rsp->completed, rsp->gpnum);

	trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end"));

	trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end"));

	rsp->fqs_state = RCU_GP_IDLE;

	rsp->gp_state = RCU_GP_IDLE;

	rdp = this_cpu_ptr(rsp->rda);

	rdp = this_cpu_ptr(rsp->rda);

	/* Advance CBs to reduce false positives below. */

	/* Advance CBs to reduce false positives below. */

	needgp = rcu_advance_cbs(rsp, rnp, rdp) || needgp;

	needgp = rcu_advance_cbs(rsp, rnp, rdp) || needgp;
@@ -2050,7 +2047,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp) 
 */

 */

static int __noreturn rcu_gp_kthread(void *arg)

static int __noreturn rcu_gp_kthread(void *arg)

{

{

	int fqs_state;

	bool first_gp_fqs;

	int gf;

	int gf;

	unsigned long j;

	unsigned long j;

	int ret;

	int ret;
@@ -2082,7 +2079,7 @@ static int __noreturn rcu_gp_kthread(void *arg) 
		}

		}





		/* Handle quiescent-state forcing. */

		/* Handle quiescent-state forcing. */

		fqs_state = RCU_SAVE_DYNTICK;

		first_gp_fqs = true;

		j = jiffies_till_first_fqs;

		j = jiffies_till_first_fqs;

		if (j > HZ) {

		if (j > HZ) {

			j = HZ;

			j = HZ;
@@ -2110,7 +2107,8 @@ static int __noreturn rcu_gp_kthread(void *arg) 
				trace_rcu_grace_period(rsp->name,

				trace_rcu_grace_period(rsp->name,

						       READ_ONCE(rsp->gpnum),

						       READ_ONCE(rsp->gpnum),

						       TPS("fqsstart"));

						       TPS("fqsstart"));

				fqs_state = rcu_gp_fqs(rsp, fqs_state);

				rcu_gp_fqs(rsp, first_gp_fqs);

				first_gp_fqs = false;

				trace_rcu_grace_period(rsp->name,

				trace_rcu_grace_period(rsp->name,

						       READ_ONCE(rsp->gpnum),

						       READ_ONCE(rsp->gpnum),

						       TPS("fqsend"));

						       TPS("fqsend"));

kernel/rcu/tree.h

+3 −11
Original line number Original line Diff line number Diff line
@@ -412,13 +412,6 @@ struct rcu_data { 
	struct rcu_state *rsp;

	struct rcu_state *rsp;

};

};





/* Values for fqs_state field in struct rcu_state. */

#define RCU_GP_IDLE		0	/* No grace period in progress. */

#define RCU_GP_INIT		1	/* Grace period being initialized. */

#define RCU_SAVE_DYNTICK	2	/* Need to scan dyntick state. */

#define RCU_FORCE_QS		3	/* Need to force quiescent state. */

#define RCU_SIGNAL_INIT		RCU_SAVE_DYNTICK



/* Values for nocb_defer_wakeup field in struct rcu_data. */

/* Values for nocb_defer_wakeup field in struct rcu_data. */

#define RCU_NOGP_WAKE_NOT	0

#define RCU_NOGP_WAKE_NOT	0

#define RCU_NOGP_WAKE		1

#define RCU_NOGP_WAKE		1
@@ -468,9 +461,8 @@ struct rcu_state { 




	/* The following fields are guarded by the root rcu_node's lock. */

	/* The following fields are guarded by the root rcu_node's lock. */





	u8	fqs_state ____cacheline_internodealigned_in_smp;

	u8	boost ____cacheline_internodealigned_in_smp;

						/* Force QS state. */

						/* Subject to priority boost. */

	u8	boost;				/* Subject to priority boost. */

	unsigned long gpnum;			/* Current gp number. */

	unsigned long gpnum;			/* Current gp number. */

	unsigned long completed;		/* # of last completed gp. */

	unsigned long completed;		/* # of last completed gp. */

	struct task_struct *gp_kthread;		/* Task for grace periods. */

	struct task_struct *gp_kthread;		/* Task for grace periods. */
@@ -538,7 +530,7 @@ struct rcu_state { 
#define RCU_GP_FLAG_FQS  0x2	/* Need grace-period quiescent-state forcing. */

#define RCU_GP_FLAG_FQS  0x2	/* Need grace-period quiescent-state forcing. */





/* Values for rcu_state structure's gp_flags field. */

/* Values for rcu_state structure's gp_flags field. */

#define RCU_GP_WAIT_INIT 0	/* Initial state. */

#define RCU_GP_IDLE	 0	/* Initial state and no GP in progress. */

#define RCU_GP_WAIT_GPS  1	/* Wait for grace-period start. */

#define RCU_GP_WAIT_GPS  1	/* Wait for grace-period start. */

#define RCU_GP_DONE_GPS  2	/* Wait done for grace-period start. */

#define RCU_GP_DONE_GPS  2	/* Wait done for grace-period start. */

#define RCU_GP_WAIT_FQS  3	/* Wait for force-quiescent-state time. */

#define RCU_GP_WAIT_FQS  3	/* Wait for force-quiescent-state time. */

kernel/rcu/tree_trace.c

+1 −1
Original line number Original line Diff line number Diff line
@@ -268,7 +268,7 @@ static void print_one_rcu_state(struct seq_file *m, struct rcu_state *rsp) 
	gpnum = rsp->gpnum;

	gpnum = rsp->gpnum;

	seq_printf(m, "c=%ld g=%ld s=%d jfq=%ld j=%x ",

	seq_printf(m, "c=%ld g=%ld s=%d jfq=%ld j=%x ",

		   ulong2long(rsp->completed), ulong2long(gpnum),

		   ulong2long(rsp->completed), ulong2long(gpnum),

		   rsp->fqs_state,

		   rsp->gp_state,

		   (long)(rsp->jiffies_force_qs - jiffies),

		   (long)(rsp->jiffies_force_qs - jiffies),

		   (int)(jiffies & 0xffff));

		   (int)(jiffies & 0xffff));

	seq_printf(m, "nfqs=%lu/nfqsng=%lu(%lu) fqlh=%lu oqlen=%ld/%ld\n",

	seq_printf(m, "nfqs=%lu/nfqsng=%lu(%lu) fqlh=%lu oqlen=%ld/%ld\n",

CRA Git | Maintained and supported by SUSTech CRA and CCSE