Commit 477351f7 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Convert rcu_grace_period tracepoint to gp_seq 



This commit makes the rcu_grace_period tracepoint use gp_seq instead
of ->gpnum or ->completed.  It also introduces a "cpuofl-bgp" string to
less obscurely indicate when a CPU has gone offline while a grace period
is waiting on it.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent ab5e869c

include/trace/events/rcu.h

+7 −6
Original line number Diff line number Diff line
@@ -52,6 +52,7 @@ TRACE_EVENT(rcu_utilization, 
 *	"cpuqs": CPU passes through a quiescent state.

 *	"cpuonl": CPU comes online.

 *	"cpuofl": CPU goes offline.

 *	"cpuofl-bgp": CPU goes offline while blocking a grace period.

 *	"reqwait": GP kthread sleeps waiting for grace-period request.

 *	"reqwaitsig": GP kthread awakened by signal from reqwait state.

 *	"fqswait": GP kthread waiting until time to force quiescent states.
@@ -63,24 +64,24 @@ TRACE_EVENT(rcu_utilization, 
 */

TRACE_EVENT(rcu_grace_period,



	TP_PROTO(const char *rcuname, unsigned long gpnum, const char *gpevent),

	TP_PROTO(const char *rcuname, unsigned long gp_seq, const char *gpevent),



	TP_ARGS(rcuname, gpnum, gpevent),

	TP_ARGS(rcuname, gp_seq, gpevent),



	TP_STRUCT__entry(

		__field(const char *, rcuname)

		__field(unsigned long, gpnum)

		__field(unsigned long, gp_seq)

		__field(const char *, gpevent)

	),



	TP_fast_assign(

		__entry->rcuname = rcuname;

		__entry->gpnum = gpnum;

		__entry->gp_seq = gp_seq;

		__entry->gpevent = gpevent;

	),



	TP_printk("%s %lu %s",

		  __entry->rcuname, __entry->gpnum, __entry->gpevent)

		  __entry->rcuname, __entry->gp_seq, __entry->gpevent)

);



/*
@@ -753,7 +754,7 @@ TRACE_EVENT(rcu_barrier, 


#else /* #ifdef CONFIG_RCU_TRACE */



#define trace_rcu_grace_period(rcuname, gpnum, gpevent) do { } while (0)

#define trace_rcu_grace_period(rcuname, gp_seq, gpevent) do { } while (0)

#define trace_rcu_future_grace_period(rcuname, gpnum, completed, c, \

				      level, grplo, grphi, event) \

				      do { } while (0)

kernel/rcu/tree.c

+21 −22
Original line number Diff line number Diff line
@@ -234,7 +234,7 @@ void rcu_sched_qs(void) 
	if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.s))

		return;

	trace_rcu_grace_period(TPS("rcu_sched"),

			       __this_cpu_read(rcu_sched_data.gpnum),

			       __this_cpu_read(rcu_sched_data.gp_seq),

			       TPS("cpuqs"));

	__this_cpu_write(rcu_sched_data.cpu_no_qs.b.norm, false);

	if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp))
@@ -249,7 +249,7 @@ void rcu_bh_qs(void) 
	RCU_LOCKDEP_WARN(preemptible(), "rcu_bh_qs() invoked with preemption enabled!!!");

	if (__this_cpu_read(rcu_bh_data.cpu_no_qs.s)) {

		trace_rcu_grace_period(TPS("rcu_bh"),

				       __this_cpu_read(rcu_bh_data.gpnum),

				       __this_cpu_read(rcu_bh_data.gp_seq),

				       TPS("cpuqs"));

		__this_cpu_write(rcu_bh_data.cpu_no_qs.b.norm, false);

	}
@@ -1615,7 +1615,7 @@ static bool rcu_start_this_gp(struct rcu_node *rnp, struct rcu_data *rdp, 
		trace_rcu_this_gp(rnp_root, rdp, c, TPS("NoGPkthread"));

		goto unlock_out;

	}

	trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gpnum), TPS("newreq"));

	trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gp_seq), TPS("newreq"));

	ret = true;  /* Caller must wake GP kthread. */

unlock_out:

	/* Push furthest requested GP to leaf node and rcu_data structure. */
@@ -1702,9 +1702,9 @@ static bool rcu_accelerate_cbs(struct rcu_state *rsp, struct rcu_node *rnp, 


	/* Trace depending on how much we were able to accelerate. */

	if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL))

		trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccWaitCB"));

		trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("AccWaitCB"));

	else

		trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccReadyCB"));

		trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("AccReadyCB"));

	return ret;

}
@@ -1774,7 +1774,7 @@ static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp, 
		 * go looking for one.

		 */

		rdp->gpnum = rnp->gpnum;

		trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart"));

		trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("cpustart"));

		need_gp = !!(rnp->qsmask & rdp->grpmask);

		rdp->cpu_no_qs.b.norm = need_gp;

		rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_dynticks.rcu_qs_ctr);
@@ -1851,7 +1851,7 @@ static bool rcu_gp_init(struct rcu_state *rsp) 
	rcu_seq_start(&rsp->gp_seq);

	if (WARN_ON_ONCE(((rnp->completed << RCU_SEQ_CTR_SHIFT) >> RCU_SEQ_CTR_SHIFT) != rcu_seq_ctr(rnp->gp_seq))) /* Catch any ->completed/->gp_seq mismatches. */

		pr_info("%s ->completed: %#lx (%#lx) ->gp_seq %#lx (%#lx)\n", __func__, rnp->completed, (rnp->completed << RCU_SEQ_CTR_SHIFT) >> RCU_SEQ_CTR_SHIFT, rnp->gp_seq, rcu_seq_ctr(rnp->gp_seq));

	trace_rcu_grace_period(rsp->name, rsp->gpnum, TPS("start"));

	trace_rcu_grace_period(rsp->name, rsp->gp_seq, TPS("start"));

	raw_spin_unlock_irq_rcu_node(rnp);



	/*
@@ -1928,7 +1928,7 @@ static bool rcu_gp_init(struct rcu_state *rsp) 
		if (rnp == rdp->mynode)

			(void)__note_gp_changes(rsp, rnp, rdp);

		rcu_preempt_boost_start_gp(rnp);

		trace_rcu_grace_period_init(rsp->name, rnp->gpnum,

		trace_rcu_grace_period_init(rsp->name, rnp->gp_seq,

					    rnp->level, rnp->grplo,

					    rnp->grphi, rnp->qsmask);

		raw_spin_unlock_irq_rcu_node(rnp);
@@ -2048,7 +2048,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp) 
	/* Declare grace period done. */

	WRITE_ONCE(rsp->completed, rsp->gpnum);

	rcu_seq_end(&rsp->gp_seq);

	trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end"));

	trace_rcu_grace_period(rsp->name, rsp->gp_seq, TPS("end"));

	rsp->gp_state = RCU_GP_IDLE;

	/* Check for GP requests since above loop. */

	rdp = this_cpu_ptr(rsp->rda);
@@ -2061,7 +2061,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp) 
	if (!rcu_accelerate_cbs(rsp, rnp, rdp) && needgp) {

		WRITE_ONCE(rsp->gp_flags, RCU_GP_FLAG_INIT);

		rsp->gp_req_activity = jiffies;

		trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gpnum),

		trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gp_seq),

				       TPS("newreq"));

	} else {

		WRITE_ONCE(rsp->gp_flags, rsp->gp_flags & RCU_GP_FLAG_INIT);
@@ -2087,7 +2087,7 @@ static int __noreturn rcu_gp_kthread(void *arg) 
		/* Handle grace-period start. */

		for (;;) {

			trace_rcu_grace_period(rsp->name,

					       READ_ONCE(rsp->gpnum),

					       READ_ONCE(rsp->gp_seq),

					       TPS("reqwait"));

			rsp->gp_state = RCU_GP_WAIT_GPS;

			swait_event_idle(rsp->gp_wq, READ_ONCE(rsp->gp_flags) &
@@ -2100,7 +2100,7 @@ static int __noreturn rcu_gp_kthread(void *arg) 
			WRITE_ONCE(rsp->gp_activity, jiffies);

			WARN_ON(signal_pending(current));

			trace_rcu_grace_period(rsp->name,

					       READ_ONCE(rsp->gpnum),

					       READ_ONCE(rsp->gp_seq),

					       TPS("reqwaitsig"));

		}
@@ -2119,7 +2119,7 @@ static int __noreturn rcu_gp_kthread(void *arg) 
					   jiffies + 3 * j);

			}

			trace_rcu_grace_period(rsp->name,

					       READ_ONCE(rsp->gpnum),

					       READ_ONCE(rsp->gp_seq),

					       TPS("fqswait"));

			rsp->gp_state = RCU_GP_WAIT_FQS;

			ret = swait_event_idle_timeout(rsp->gp_wq,
@@ -2134,12 +2134,12 @@ static int __noreturn rcu_gp_kthread(void *arg) 
			if (ULONG_CMP_GE(jiffies, rsp->jiffies_force_qs) ||

			    (gf & RCU_GP_FLAG_FQS)) {

				trace_rcu_grace_period(rsp->name,

						       READ_ONCE(rsp->gpnum),

						       READ_ONCE(rsp->gp_seq),

						       TPS("fqsstart"));

				rcu_gp_fqs(rsp, first_gp_fqs);

				first_gp_fqs = false;

				trace_rcu_grace_period(rsp->name,

						       READ_ONCE(rsp->gpnum),

						       READ_ONCE(rsp->gp_seq),

						       TPS("fqsend"));

				cond_resched_tasks_rcu_qs();

				WRITE_ONCE(rsp->gp_activity, jiffies);
@@ -2158,7 +2158,7 @@ static int __noreturn rcu_gp_kthread(void *arg) 
				WRITE_ONCE(rsp->gp_activity, jiffies);

				WARN_ON(signal_pending(current));

				trace_rcu_grace_period(rsp->name,

						       READ_ONCE(rsp->gpnum),

						       READ_ONCE(rsp->gp_seq),

						       TPS("fqswaitsig"));

				ret = 1; /* Keep old FQS timing. */

				j = jiffies;
@@ -2388,17 +2388,16 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp) 
 */

static void rcu_cleanup_dying_cpu(struct rcu_state *rsp)

{

	RCU_TRACE(unsigned long mask;)

	RCU_TRACE(bool blkd;)

	RCU_TRACE(struct rcu_data *rdp = this_cpu_ptr(rsp->rda);)

	RCU_TRACE(struct rcu_node *rnp = rdp->mynode;)



	if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))

		return;



	RCU_TRACE(mask = rdp->grpmask;)

	trace_rcu_grace_period(rsp->name,

			       rnp->gpnum + 1 - !!(rnp->qsmask & mask),

			       TPS("cpuofl"));

	RCU_TRACE(blkd = !!(rnp->qsmask & rdp->grpmask);)

	trace_rcu_grace_period(rsp->name, rnp->gp_seq,

			       blkd ? TPS("cpuofl") : TPS("cpuofl-bgp"));

}



/*
@@ -3538,7 +3537,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp) 
	rdp->core_needs_qs = false;

	rdp->rcu_iw_pending = false;

	rdp->rcu_iw_gp_seq = rnp->gp_seq - 1;

	trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl"));

	trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("cpuonl"));

	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);

}

kernel/rcu/tree_plugin.h

+1 −1
Original line number Diff line number Diff line
@@ -301,7 +301,7 @@ static void rcu_preempt_qs(void) 
	RCU_LOCKDEP_WARN(preemptible(), "rcu_preempt_qs() invoked with preemption enabled!!!\n");

	if (__this_cpu_read(rcu_data_p->cpu_no_qs.s)) {

		trace_rcu_grace_period(TPS("rcu_preempt"),

				       __this_cpu_read(rcu_data_p->gpnum),

				       __this_cpu_read(rcu_data_p->gp_seq),

				       TPS("cpuqs"));

		__this_cpu_write(rcu_data_p->cpu_no_qs.b.norm, false);

		barrier(); /* Coordinate with rcu_preempt_check_callbacks(). */

CRA Git | Maintained and supported by SUSTech CRA and CCSE