Commit 1d082fd0 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Remove local_irq_disable() in rcu_preempt_note_context_switch() 



The rcu_preempt_note_context_switch() function is on a scheduling fast
path, so it would be good to avoid disabling irqs.  The reason that irqs
are disabled is to synchronize process-level and irq-handler access to
the task_struct ->rcu_read_unlock_special bitmask.  This commit therefore
makes ->rcu_read_unlock_special instead be a union of bools with a short
allowing single-access checks in RCU's __rcu_read_unlock().  This results
in the process-level and irq-handler accesses being simple loads and
stores, so that irqs need no longer be disabled.  This commit therefore
removes the irq disabling from rcu_preempt_note_context_switch().

Reported-by: default avatarPeter Zijlstra <peterz@infradead.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 4ff475ed

include/linux/init_task.h

+1 −1
Original line number Diff line number Diff line
@@ -111,7 +111,7 @@ extern struct group_info init_groups; 
#ifdef CONFIG_PREEMPT_RCU

#define INIT_TASK_RCU_PREEMPT(tsk)					\

	.rcu_read_lock_nesting = 0,					\

	.rcu_read_unlock_special = 0,					\

	.rcu_read_unlock_special.s = 0,					\

	.rcu_node_entry = LIST_HEAD_INIT(tsk.rcu_node_entry),		\

	INIT_TASK_RCU_TREE_PREEMPT()

#else

include/linux/sched.h

+9 −7
Original line number Diff line number Diff line
@@ -1212,6 +1212,13 @@ struct sched_dl_entity { 
	struct hrtimer dl_timer;

};



union rcu_special {

	struct {

		bool blocked;

		bool need_qs;

	} b;

	short s;

};

struct rcu_node;



enum perf_event_task_context {
@@ -1264,7 +1271,7 @@ struct task_struct { 


#ifdef CONFIG_PREEMPT_RCU

	int rcu_read_lock_nesting;

	char rcu_read_unlock_special;

	union rcu_special rcu_read_unlock_special;

	struct list_head rcu_node_entry;

#endif /* #ifdef CONFIG_PREEMPT_RCU */

#ifdef CONFIG_TREE_PREEMPT_RCU
@@ -2005,16 +2012,11 @@ extern void task_clear_jobctl_trapping(struct task_struct *task); 
extern void task_clear_jobctl_pending(struct task_struct *task,

				      unsigned int mask);



#ifdef CONFIG_PREEMPT_RCU

#define RCU_READ_UNLOCK_BLOCKED (1 << 0) /* blocked while in RCU read-side. */

#define RCU_READ_UNLOCK_NEED_QS (1 << 1) /* RCU core needs CPU response. */

#endif /* #ifdef CONFIG_PREEMPT_RCU */



static inline void rcu_copy_process(struct task_struct *p)

{

#ifdef CONFIG_PREEMPT_RCU

	p->rcu_read_lock_nesting = 0;

	p->rcu_read_unlock_special = 0;

	p->rcu_read_unlock_special.s = 0;

	p->rcu_blocked_node = NULL;

	INIT_LIST_HEAD(&p->rcu_node_entry);

#endif /* #ifdef CONFIG_PREEMPT_RCU */

kernel/rcu/tree_plugin.h

+15 −17
Original line number Diff line number Diff line
@@ -155,9 +155,8 @@ EXPORT_SYMBOL_GPL(rcu_batches_completed); 
 * not in a quiescent state.  There might be any number of tasks blocked

 * while in an RCU read-side critical section.

 *

 * Unlike the other rcu_*_qs() functions, callers to this function

 * must disable irqs in order to protect the assignment to

 * ->rcu_read_unlock_special.

 * As with the other rcu_*_qs() functions, callers to this function

 * must disable preemption.

 */

static void rcu_preempt_qs(int cpu)

{
@@ -166,7 +165,7 @@ static void rcu_preempt_qs(int cpu) 
	if (rdp->passed_quiesce == 0)

		trace_rcu_grace_period(TPS("rcu_preempt"), rdp->gpnum, TPS("cpuqs"));

	rdp->passed_quiesce = 1;

	current->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;

	current->rcu_read_unlock_special.b.need_qs = false;

}



/*
@@ -190,14 +189,14 @@ static void rcu_preempt_note_context_switch(int cpu) 
	struct rcu_node *rnp;



	if (t->rcu_read_lock_nesting > 0 &&

	    (t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {

	    !t->rcu_read_unlock_special.b.blocked) {



		/* Possibly blocking in an RCU read-side critical section. */

		rdp = per_cpu_ptr(rcu_preempt_state.rda, cpu);

		rnp = rdp->mynode;

		raw_spin_lock_irqsave(&rnp->lock, flags);

		smp_mb__after_unlock_lock();

		t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED;

		t->rcu_read_unlock_special.b.blocked = true;

		t->rcu_blocked_node = rnp;



		/*
@@ -239,7 +238,7 @@ static void rcu_preempt_note_context_switch(int cpu) 
				       : rnp->gpnum + 1);

		raw_spin_unlock_irqrestore(&rnp->lock, flags);

	} else if (t->rcu_read_lock_nesting < 0 &&

		   t->rcu_read_unlock_special) {

		   t->rcu_read_unlock_special.s) {



		/*

		 * Complete exit from RCU read-side critical section on
@@ -257,9 +256,7 @@ static void rcu_preempt_note_context_switch(int cpu) 
	 * grace period, then the fact that the task has been enqueued

	 * means that we continue to block the current grace period.

	 */

	local_irq_save(flags);

	rcu_preempt_qs(cpu);

	local_irq_restore(flags);

}



/*
@@ -340,7 +337,7 @@ void rcu_read_unlock_special(struct task_struct *t) 
	bool drop_boost_mutex = false;

#endif /* #ifdef CONFIG_RCU_BOOST */

	struct rcu_node *rnp;

	int special;

	union rcu_special special;



	/* NMI handlers cannot block and cannot safely manipulate state. */

	if (in_nmi())
@@ -350,12 +347,13 @@ void rcu_read_unlock_special(struct task_struct *t) 


	/*

	 * If RCU core is waiting for this CPU to exit critical section,

	 * let it know that we have done so.

	 * let it know that we have done so.  Because irqs are disabled,

	 * t->rcu_read_unlock_special cannot change.

	 */

	special = t->rcu_read_unlock_special;

	if (special & RCU_READ_UNLOCK_NEED_QS) {

	if (special.b.need_qs) {

		rcu_preempt_qs(smp_processor_id());

		if (!t->rcu_read_unlock_special) {

		if (!t->rcu_read_unlock_special.s) {

			local_irq_restore(flags);

			return;

		}
@@ -368,8 +366,8 @@ void rcu_read_unlock_special(struct task_struct *t) 
	}



	/* Clean up if blocked during RCU read-side critical section. */

	if (special & RCU_READ_UNLOCK_BLOCKED) {

		t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_BLOCKED;

	if (special.b.blocked) {

		t->rcu_read_unlock_special.b.blocked = false;



		/*

		 * Remove this task from the list it blocked on.  The
@@ -658,7 +656,7 @@ static void rcu_preempt_check_callbacks(int cpu) 
	}

	if (t->rcu_read_lock_nesting > 0 &&

	    per_cpu(rcu_preempt_data, cpu).qs_pending)

		t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;

		t->rcu_read_unlock_special.b.need_qs = true;

}



#ifdef CONFIG_RCU_BOOST
@@ -941,7 +939,7 @@ void exit_rcu(void) 
		return;

	t->rcu_read_lock_nesting = 1;

	barrier();

	t->rcu_read_unlock_special = RCU_READ_UNLOCK_BLOCKED;

	t->rcu_read_unlock_special.b.blocked = true;

	__rcu_read_unlock();

}

kernel/rcu/update.c

+1 −1
Original line number Diff line number Diff line
@@ -93,7 +93,7 @@ void __rcu_read_unlock(void) 
		barrier();  /* critical section before exit code. */

		t->rcu_read_lock_nesting = INT_MIN;

		barrier();  /* assign before ->rcu_read_unlock_special load */

		if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))

		if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special.s)))

			rcu_read_unlock_special(t);

		barrier();  /* ->rcu_read_unlock_special load before assign */

		t->rcu_read_lock_nesting = 0;

CRA Git | Maintained and supported by SUSTech CRA and CCSE