Commit 616c310e authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Paul E. McKenney
Browse files

rcu: Move PREEMPT_RCU preemption to switch_to() invocation 



Currently, PREEMPT_RCU readers are enqueued upon entry to the scheduler.
This is inefficient because enqueuing is required only if there is a
context switch, and entry to the scheduler does not guarantee a context
switch.

The commit therefore moves the enqueuing to immediately precede the
call to switch_to() from the scheduler.

Signed-off-by: default avatarPaul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Tested-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 66f75a5d

arch/um/drivers/mconsole_kern.c

+1 −0
Original line number Diff line number Diff line
@@ -705,6 +705,7 @@ static void stack_proc(void *arg) 
	struct task_struct *from = current, *to = arg;



	to->thread.saved_task = from;

	rcu_switch_from(from);

	switch_to(from, to, from);

}

include/linux/rcupdate.h

+1 −0
Original line number Diff line number Diff line
@@ -184,6 +184,7 @@ static inline int rcu_preempt_depth(void) 
/* Internal to kernel */

extern void rcu_sched_qs(int cpu);

extern void rcu_bh_qs(int cpu);

extern void rcu_preempt_note_context_switch(void);

extern void rcu_check_callbacks(int cpu, int user);

struct notifier_block;

extern void rcu_idle_enter(void);

include/linux/rcutiny.h

+0 −6
Original line number Diff line number Diff line
@@ -87,10 +87,6 @@ static inline void kfree_call_rcu(struct rcu_head *head, 


#ifdef CONFIG_TINY_RCU



static inline void rcu_preempt_note_context_switch(void)

{

}



static inline void exit_rcu(void)

{

}
@@ -102,7 +98,6 @@ static inline int rcu_needs_cpu(int cpu) 


#else /* #ifdef CONFIG_TINY_RCU */



void rcu_preempt_note_context_switch(void);

extern void exit_rcu(void);

int rcu_preempt_needs_cpu(void);
@@ -116,7 +111,6 @@ static inline int rcu_needs_cpu(int cpu) 
static inline void rcu_note_context_switch(int cpu)

{

	rcu_sched_qs(cpu);

	rcu_preempt_note_context_switch();

}



/*

include/linux/sched.h

+10 −0
Original line number Diff line number Diff line
@@ -1905,12 +1905,22 @@ static inline void rcu_copy_process(struct task_struct *p) 
	INIT_LIST_HEAD(&p->rcu_node_entry);

}



static inline void rcu_switch_from(struct task_struct *prev)

{

	if (prev->rcu_read_lock_nesting != 0)

		rcu_preempt_note_context_switch();

}



#else



static inline void rcu_copy_process(struct task_struct *p)

{

}



static inline void rcu_switch_from(struct task_struct *prev)

{

}



#endif



#ifdef CONFIG_SMP

kernel/rcutree.c

+0 −1
Original line number Diff line number Diff line
@@ -192,7 +192,6 @@ void rcu_note_context_switch(int cpu) 
{

	trace_rcu_utilization("Start context switch");

	rcu_sched_qs(cpu);

	rcu_preempt_note_context_switch(cpu);

	trace_rcu_utilization("End context switch");

}

EXPORT_SYMBOL_GPL(rcu_note_context_switch);

CRA Git | Maintained and supported by SUSTech CRA and CCSE