Commit 5cd37193 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Make cond_resched_rcu_qs() apply to normal RCU flavors 



Although cond_resched_rcu_qs() only applies to TASKS_RCU, it is used
in places where it would be useful for it to apply to the normal RCU
flavors, rcu_preempt, rcu_sched, and rcu_bh.  This is especially the
case for workloads that aggressively overload the system, particularly
those that generate large numbers of RCU updates on systems running
NO_HZ_FULL CPUs.  This commit therefore communicates quiescent states
from cond_resched_rcu_qs() to the normal RCU flavors.

Note that it is unfortunately necessary to leave the old ->passed_quiesce
mechanism in place to allow quiescent states that apply to only one
flavor to be recorded.  (Yes, we could decrement ->rcu_qs_ctr_snap in
that case, but that is not so good for debugging of RCU internals.)
In addition, if one of the RCU flavor's grace period has stalled, this
will invoke rcu_momentary_dyntick_idle(), resulting in a heavy-weight
quiescent state visible from other CPUs.

Reported-by: default avatarSasha Levin <sasha.levin@oracle.com>
Reported-by: default avatarDave Jones <davej@redhat.com>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
[ paulmck: Merge commit from Sasha Levin fixing a bug where __this_cpu()
  was used in preemptible code. ]
parent a94844b2

Documentation/RCU/trace.txt

+16 −16
Original line number Diff line number Diff line
@@ -56,14 +56,14 @@ rcuboost: 


The output of "cat rcu/rcu_preempt/rcudata" looks as follows:



  0!c=30455 g=30456 pq=1 qp=1 dt=126535/140000000000000/0 df=2002 of=4 ql=0/0 qs=N... b=10 ci=74572 nci=0 co=1131 ca=716

  1!c=30719 g=30720 pq=1 qp=0 dt=132007/140000000000000/0 df=1874 of=10 ql=0/0 qs=N... b=10 ci=123209 nci=0 co=685 ca=982

  2!c=30150 g=30151 pq=1 qp=1 dt=138537/140000000000000/0 df=1707 of=8 ql=0/0 qs=N... b=10 ci=80132 nci=0 co=1328 ca=1458

  3 c=31249 g=31250 pq=1 qp=0 dt=107255/140000000000000/0 df=1749 of=6 ql=0/450 qs=NRW. b=10 ci=151700 nci=0 co=509 ca=622

  4!c=29502 g=29503 pq=1 qp=1 dt=83647/140000000000000/0 df=965 of=5 ql=0/0 qs=N... b=10 ci=65643 nci=0 co=1373 ca=1521

  5 c=31201 g=31202 pq=1 qp=1 dt=70422/0/0 df=535 of=7 ql=0/0 qs=.... b=10 ci=58500 nci=0 co=764 ca=698

  6!c=30253 g=30254 pq=1 qp=1 dt=95363/140000000000000/0 df=780 of=5 ql=0/0 qs=N... b=10 ci=100607 nci=0 co=1414 ca=1353

  7 c=31178 g=31178 pq=1 qp=0 dt=91536/0/0 df=547 of=4 ql=0/0 qs=.... b=10 ci=109819 nci=0 co=1115 ca=969

  0!c=30455 g=30456 pq=1/0 qp=1 dt=126535/140000000000000/0 df=2002 of=4 ql=0/0 qs=N... b=10 ci=74572 nci=0 co=1131 ca=716

  1!c=30719 g=30720 pq=1/0 qp=0 dt=132007/140000000000000/0 df=1874 of=10 ql=0/0 qs=N... b=10 ci=123209 nci=0 co=685 ca=982

  2!c=30150 g=30151 pq=1/1 qp=1 dt=138537/140000000000000/0 df=1707 of=8 ql=0/0 qs=N... b=10 ci=80132 nci=0 co=1328 ca=1458

  3 c=31249 g=31250 pq=1/1 qp=0 dt=107255/140000000000000/0 df=1749 of=6 ql=0/450 qs=NRW. b=10 ci=151700 nci=0 co=509 ca=622

  4!c=29502 g=29503 pq=1/0 qp=1 dt=83647/140000000000000/0 df=965 of=5 ql=0/0 qs=N... b=10 ci=65643 nci=0 co=1373 ca=1521

  5 c=31201 g=31202 pq=1/0 qp=1 dt=70422/0/0 df=535 of=7 ql=0/0 qs=.... b=10 ci=58500 nci=0 co=764 ca=698

  6!c=30253 g=30254 pq=1/0 qp=1 dt=95363/140000000000000/0 df=780 of=5 ql=0/0 qs=N... b=10 ci=100607 nci=0 co=1414 ca=1353

  7 c=31178 g=31178 pq=1/0 qp=0 dt=91536/0/0 df=547 of=4 ql=0/0 qs=.... b=10 ci=109819 nci=0 co=1115 ca=969



This file has one line per CPU, or eight for this 8-CPU system.

The fields are as follows:
@@ -188,14 +188,14 @@ o "ca" is the number of RCU callbacks that have been adopted by this 
Kernels compiled with CONFIG_RCU_BOOST=y display the following from

/debug/rcu/rcu_preempt/rcudata:



  0!c=12865 g=12866 pq=1 qp=1 dt=83113/140000000000000/0 df=288 of=11 ql=0/0 qs=N... kt=0/O ktl=944 b=10 ci=60709 nci=0 co=748 ca=871

  1 c=14407 g=14408 pq=1 qp=0 dt=100679/140000000000000/0 df=378 of=7 ql=0/119 qs=NRW. kt=0/W ktl=9b6 b=10 ci=109740 nci=0 co=589 ca=485

  2 c=14407 g=14408 pq=1 qp=0 dt=105486/0/0 df=90 of=9 ql=0/89 qs=NRW. kt=0/W ktl=c0c b=10 ci=83113 nci=0 co=533 ca=490

  3 c=14407 g=14408 pq=1 qp=0 dt=107138/0/0 df=142 of=8 ql=0/188 qs=NRW. kt=0/W ktl=b96 b=10 ci=121114 nci=0 co=426 ca=290

  4 c=14405 g=14406 pq=1 qp=1 dt=50238/0/0 df=706 of=7 ql=0/0 qs=.... kt=0/W ktl=812 b=10 ci=34929 nci=0 co=643 ca=114

  5!c=14168 g=14169 pq=1 qp=0 dt=45465/140000000000000/0 df=161 of=11 ql=0/0 qs=N... kt=0/O ktl=b4d b=10 ci=47712 nci=0 co=677 ca=722

  6 c=14404 g=14405 pq=1 qp=0 dt=59454/0/0 df=94 of=6 ql=0/0 qs=.... kt=0/W ktl=e57 b=10 ci=55597 nci=0 co=701 ca=811

  7 c=14407 g=14408 pq=1 qp=1 dt=68850/0/0 df=31 of=8 ql=0/0 qs=.... kt=0/W ktl=14bd b=10 ci=77475 nci=0 co=508 ca=1042

  0!c=12865 g=12866 pq=1/0 qp=1 dt=83113/140000000000000/0 df=288 of=11 ql=0/0 qs=N... kt=0/O ktl=944 b=10 ci=60709 nci=0 co=748 ca=871

  1 c=14407 g=14408 pq=1/0 qp=0 dt=100679/140000000000000/0 df=378 of=7 ql=0/119 qs=NRW. kt=0/W ktl=9b6 b=10 ci=109740 nci=0 co=589 ca=485

  2 c=14407 g=14408 pq=1/0 qp=0 dt=105486/0/0 df=90 of=9 ql=0/89 qs=NRW. kt=0/W ktl=c0c b=10 ci=83113 nci=0 co=533 ca=490

  3 c=14407 g=14408 pq=1/0 qp=0 dt=107138/0/0 df=142 of=8 ql=0/188 qs=NRW. kt=0/W ktl=b96 b=10 ci=121114 nci=0 co=426 ca=290

  4 c=14405 g=14406 pq=1/0 qp=1 dt=50238/0/0 df=706 of=7 ql=0/0 qs=.... kt=0/W ktl=812 b=10 ci=34929 nci=0 co=643 ca=114

  5!c=14168 g=14169 pq=1/0 qp=0 dt=45465/140000000000000/0 df=161 of=11 ql=0/0 qs=N... kt=0/O ktl=b4d b=10 ci=47712 nci=0 co=677 ca=722

  6 c=14404 g=14405 pq=1/0 qp=0 dt=59454/0/0 df=94 of=6 ql=0/0 qs=.... kt=0/W ktl=e57 b=10 ci=55597 nci=0 co=701 ca=811

  7 c=14407 g=14408 pq=1/0 qp=1 dt=68850/0/0 df=31 of=8 ql=0/0 qs=.... kt=0/W ktl=14bd b=10 ci=77475 nci=0 co=508 ca=1042



This is similar to the output discussed above, but contains the following

additional fields:

include/linux/rcupdate.h

+2 −1
Original line number Diff line number Diff line
@@ -331,12 +331,13 @@ static inline void rcu_init_nohz(void) 
extern struct srcu_struct tasks_rcu_exit_srcu;

#define rcu_note_voluntary_context_switch(t) \

	do { \

		rcu_all_qs(); \

		if (ACCESS_ONCE((t)->rcu_tasks_holdout)) \

			ACCESS_ONCE((t)->rcu_tasks_holdout) = false; \

	} while (0)

#else /* #ifdef CONFIG_TASKS_RCU */

#define TASKS_RCU(x) do { } while (0)

#define rcu_note_voluntary_context_switch(t)	do { } while (0)

#define rcu_note_voluntary_context_switch(t)	rcu_all_qs()

#endif /* #else #ifdef CONFIG_TASKS_RCU */



/**

include/linux/rcutiny.h

+4 −1
Original line number Diff line number Diff line
@@ -154,7 +154,10 @@ static inline bool rcu_is_watching(void) 
	return true;

}





#endif /* #else defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) */



static inline void rcu_all_qs(void)

{

}



#endif /* __LINUX_RCUTINY_H */

include/linux/rcutree.h

+2 −0
Original line number Diff line number Diff line
@@ -97,4 +97,6 @@ extern int rcu_scheduler_active __read_mostly; 


bool rcu_is_watching(void);



void rcu_all_qs(void);



#endif /* __LINUX_RCUTREE_H */

kernel/rcu/tree.c

+33 −5
Original line number Diff line number Diff line
@@ -219,6 +219,9 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = { 
#endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */

};



DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, rcu_qs_ctr);

EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);



/*

 * Let the RCU core know that this CPU has gone through the scheduler,

 * which is a quiescent state.  This is called when the need for a
@@ -288,6 +291,22 @@ void rcu_note_context_switch(void) 
}

EXPORT_SYMBOL_GPL(rcu_note_context_switch);



/*

 * Register a quiesecent state for all RCU flavors.  If there is an

 * emergency, invoke rcu_momentary_dyntick_idle() to do a heavy-weight

 * dyntick-idle quiescent state visible to other CPUs (but only for those

 * RCU flavors in desparate need of a quiescent state, which will normally

 * be none of them).  Either way, do a lightweight quiescent state for

 * all RCU flavors.

 */

void rcu_all_qs(void)

{

	if (unlikely(raw_cpu_read(rcu_sched_qs_mask)))

		rcu_momentary_dyntick_idle();

	this_cpu_inc(rcu_qs_ctr);

}

EXPORT_SYMBOL_GPL(rcu_all_qs);



static long blimit = 10;	/* Maximum callbacks per rcu_do_batch. */

static long qhimark = 10000;	/* If this many pending, ignore blimit. */

static long qlowmark = 100;	/* Once only this many pending, use blimit. */
@@ -1609,6 +1628,7 @@ static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp, 
		rdp->gpnum = rnp->gpnum;

		trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart"));

		rdp->passed_quiesce = 0;

		rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);

		rdp->qs_pending = !!(rnp->qsmask & rdp->grpmask);

		zero_cpu_stall_ticks(rdp);

		ACCESS_ONCE(rdp->gpwrap) = false;
@@ -2075,8 +2095,10 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp) 
	rnp = rdp->mynode;

	raw_spin_lock_irqsave(&rnp->lock, flags);

	smp_mb__after_unlock_lock();

	if (rdp->passed_quiesce == 0 || rdp->gpnum != rnp->gpnum ||

	    rnp->completed == rnp->gpnum || rdp->gpwrap) {

	if ((rdp->passed_quiesce == 0 &&

	     rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) ||

	    rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum ||

	    rdp->gpwrap) {



		/*

		 * The grace period in which this quiescent state was
@@ -2085,6 +2107,7 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp) 
		 * within the current grace period.

		 */

		rdp->passed_quiesce = 0;	/* need qs for new gp. */

		rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);

		raw_spin_unlock_irqrestore(&rnp->lock, flags);

		return;

	}
@@ -2129,7 +2152,8 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp) 
	 * Was there a quiescent state since the beginning of the grace

	 * period? If no, then exit and wait for the next call.

	 */

	if (!rdp->passed_quiesce)

	if (!rdp->passed_quiesce &&

	    rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr))

		return;



	/*
@@ -3174,9 +3198,12 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp) 


	/* Is the RCU core waiting for a quiescent state from this CPU? */

	if (rcu_scheduler_fully_active &&

	    rdp->qs_pending && !rdp->passed_quiesce) {

	    rdp->qs_pending && !rdp->passed_quiesce &&

	    rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) {

		rdp->n_rp_qs_pending++;

	} else if (rdp->qs_pending && rdp->passed_quiesce) {

	} else if (rdp->qs_pending &&

		   (rdp->passed_quiesce ||

		    rdp->rcu_qs_ctr_snap != __this_cpu_read(rcu_qs_ctr))) {

		rdp->n_rp_report_qs++;

		return 1;

	}
@@ -3510,6 +3537,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp) 
			rdp->gpnum = rnp->completed;

			rdp->completed = rnp->completed;

			rdp->passed_quiesce = 0;

			rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);

			rdp->qs_pending = 0;

			trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl"));

		}

CRA Git | Maintained and supported by SUSTech CRA and CCSE