Commit bde6c3aa authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Provide cond_resched_rcu_qs() to force quiescent states in long loops 

RCU-tasks requires the occasional voluntary context switch
from CPU-bound in-kernel tasks.  In some cases, this requires
instrumenting cond_resched().  However, there is some reluctance
to countenance unconditionally instrumenting cond_resched() (see
http://lwn.net/Articles/603252/

), so this commit creates a separate
cond_resched_rcu_qs() that may be used in place of cond_resched() in
locations prone to long-duration in-kernel looping.

This commit currently instruments only RCU-tasks.  Future possibilities
include also instrumenting RCU, RCU-bh, and RCU-sched in order to reduce
IPI usage.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 8315f422

fs/file.c

+1 −1
Original line number Diff line number Diff line
@@ -367,7 +367,7 @@ static struct fdtable *close_files(struct files_struct * files) 
				struct file * file = xchg(&fdt->fd[i], NULL);

				if (file) {

					filp_close(file, files);

					cond_resched();

					cond_resched_rcu_qs();

				}

			}

			i++;

include/linux/rcupdate.h

+13 −0
Original line number Diff line number Diff line
@@ -330,6 +330,19 @@ static inline void rcu_user_hooks_switch(struct task_struct *prev, 
#define rcu_note_voluntary_context_switch(t)	do { } while (0)

#endif /* #else #ifdef CONFIG_TASKS_RCU */



/**

 * cond_resched_rcu_qs - Report potential quiescent states to RCU

 *

 * This macro resembles cond_resched(), except that it is defined to

 * report potential quiescent states to RCU-tasks even if the cond_resched()

 * machinery were to be shut off, as some advocate for PREEMPT kernels.

 */

#define cond_resched_rcu_qs() \

do { \

	rcu_note_voluntary_context_switch(current); \

	cond_resched(); \

} while (0)



#if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) || defined(CONFIG_SMP)

bool __rcu_is_watching(void);

#endif /* #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) || defined(CONFIG_SMP) */

kernel/rcu/rcutorture.c

+2 −2
Original line number Diff line number Diff line
@@ -667,7 +667,7 @@ static int rcu_torture_boost(void *arg) 
				}

				call_rcu_time = jiffies;

			}

			cond_resched();

			cond_resched_rcu_qs();

			stutter_wait("rcu_torture_boost");

			if (torture_must_stop())

				goto checkwait;
@@ -1019,7 +1019,7 @@ rcu_torture_reader(void *arg) 
		__this_cpu_inc(rcu_torture_batch[completed]);

		preempt_enable();

		cur_ops->readunlock(idx);

		cond_resched();

		cond_resched_rcu_qs();

		stutter_wait("rcu_torture_reader");

	} while (!torture_must_stop());

	if (irqreader && cur_ops->irq_capable) {

kernel/rcu/tree.c

+6 −6
Original line number Diff line number Diff line
@@ -1647,7 +1647,7 @@ static int rcu_gp_init(struct rcu_state *rsp) 
					    rnp->level, rnp->grplo,

					    rnp->grphi, rnp->qsmask);

		raw_spin_unlock_irq(&rnp->lock);

		cond_resched();

		cond_resched_rcu_qs();

	}



	mutex_unlock(&rsp->onoff_mutex);
@@ -1736,7 +1736,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp) 
		/* smp_mb() provided by prior unlock-lock pair. */

		nocb += rcu_future_gp_cleanup(rsp, rnp);

		raw_spin_unlock_irq(&rnp->lock);

		cond_resched();

		cond_resched_rcu_qs();

	}

	rnp = rcu_get_root(rsp);

	raw_spin_lock_irq(&rnp->lock);
@@ -1785,7 +1785,7 @@ static int __noreturn rcu_gp_kthread(void *arg) 
			/* Locking provides needed memory barrier. */

			if (rcu_gp_init(rsp))

				break;

			cond_resched();

			cond_resched_rcu_qs();

			flush_signals(current);

			trace_rcu_grace_period(rsp->name,

					       ACCESS_ONCE(rsp->gpnum),
@@ -1828,10 +1828,10 @@ static int __noreturn rcu_gp_kthread(void *arg) 
				trace_rcu_grace_period(rsp->name,

						       ACCESS_ONCE(rsp->gpnum),

						       TPS("fqsend"));

				cond_resched();

				cond_resched_rcu_qs();

			} else {

				/* Deal with stray signal. */

				cond_resched();

				cond_resched_rcu_qs();

				flush_signals(current);

				trace_rcu_grace_period(rsp->name,

						       ACCESS_ONCE(rsp->gpnum),
@@ -2434,7 +2434,7 @@ static void force_qs_rnp(struct rcu_state *rsp, 
	struct rcu_node *rnp;



	rcu_for_each_leaf_node(rsp, rnp) {

		cond_resched();

		cond_resched_rcu_qs();

		mask = 0;

		raw_spin_lock_irqsave(&rnp->lock, flags);

		smp_mb__after_unlock_lock();

kernel/rcu/tree_plugin.h

+1 −1
Original line number Diff line number Diff line
@@ -1848,7 +1848,7 @@ static int rcu_oom_notify(struct notifier_block *self, 
	get_online_cpus();

	for_each_online_cpu(cpu) {

		smp_call_function_single(cpu, rcu_oom_notify_cpu, NULL, 1);

		cond_resched();

		cond_resched_rcu_qs();

	}

	put_online_cpus();

CRA Git | Maintained and supported by SUSTech CRA and CCSE