Commit 9dd8fb16 authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Paul E. McKenney
Browse files

rcu: Make exit_rcu() more precise and consolidate 



When running preemptible RCU, if a task exits in an RCU read-side
critical section having blocked within that same RCU read-side critical
section, the task must be removed from the list of tasks blocking a
grace period (perhaps the current grace period, perhaps the next grace
period, depending on timing).  The exit() path invokes exit_rcu() to
do this cleanup.

However, the current implementation of exit_rcu() needlessly does the
cleanup even if the task did not block within the current RCU read-side
critical section, which wastes time and needlessly increases the size
of the state space.  Fix this by only doing the cleanup if the current
task is actually on the list of tasks blocking some grace period.

While we are at it, consolidate the two identical exit_rcu() functions
into a single function.

Signed-off-by: default avatarPaul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Tested-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>

Conflicts:

	kernel/rcupdate.c
parent 616c310e

include/linux/rcupdate.h

+1 −0
Original line number Original line Diff line number Diff line
@@ -191,6 +191,7 @@ extern void rcu_idle_enter(void); 
extern void rcu_idle_exit(void);

extern void rcu_idle_exit(void);

extern void rcu_irq_enter(void);

extern void rcu_irq_enter(void);

extern void rcu_irq_exit(void);

extern void rcu_irq_exit(void);

extern void exit_rcu(void);





/**

/**

 * RCU_NONIDLE - Indicate idle-loop code that needs RCU readers

 * RCU_NONIDLE - Indicate idle-loop code that needs RCU readers

include/linux/rcutiny.h

+0 −5
Original line number Original line Diff line number Diff line
@@ -87,10 +87,6 @@ static inline void kfree_call_rcu(struct rcu_head *head, 




#ifdef CONFIG_TINY_RCU

#ifdef CONFIG_TINY_RCU





static inline void exit_rcu(void)

{

}



static inline int rcu_needs_cpu(int cpu)

static inline int rcu_needs_cpu(int cpu)

{

{

	return 0;

	return 0;
@@ -98,7 +94,6 @@ static inline int rcu_needs_cpu(int cpu) 




#else /* #ifdef CONFIG_TINY_RCU */

#else /* #ifdef CONFIG_TINY_RCU */





extern void exit_rcu(void);

int rcu_preempt_needs_cpu(void);

int rcu_preempt_needs_cpu(void);





static inline int rcu_needs_cpu(int cpu)

static inline int rcu_needs_cpu(int cpu)

include/linux/rcutree.h

+0 −12
Original line number Original line Diff line number Diff line
@@ -45,18 +45,6 @@ static inline void rcu_virt_note_context_switch(int cpu) 
	rcu_note_context_switch(cpu);

	rcu_note_context_switch(cpu);

}

}





#ifdef CONFIG_TREE_PREEMPT_RCU



extern void exit_rcu(void);



#else /* #ifdef CONFIG_TREE_PREEMPT_RCU */



static inline void exit_rcu(void)

{

}



#endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */



extern void synchronize_rcu_bh(void);

extern void synchronize_rcu_bh(void);

extern void synchronize_sched_expedited(void);

extern void synchronize_sched_expedited(void);

extern void synchronize_rcu_expedited(void);

extern void synchronize_rcu_expedited(void);

kernel/rcupdate.c

+28 −0
Original line number Original line Diff line number Diff line
@@ -51,6 +51,34 @@ 




#include "rcu.h"

#include "rcu.h"





#ifdef CONFIG_PREEMPT_RCU



/*

 * Check for a task exiting while in a preemptible-RCU read-side

 * critical section, clean up if so.  No need to issue warnings,

 * as debug_check_no_locks_held() already does this if lockdep

 * is enabled.

 */

void exit_rcu(void)

{

	struct task_struct *t = current;



	if (likely(list_empty(&current->rcu_node_entry)))

		return;

	t->rcu_read_lock_nesting = 1;

	barrier();

	t->rcu_read_unlock_special = RCU_READ_UNLOCK_BLOCKED;

	__rcu_read_unlock();

}



#else /* #ifdef CONFIG_PREEMPT_RCU */



void exit_rcu(void)

{

}



#endif /* #else #ifdef CONFIG_PREEMPT_RCU */



#ifdef CONFIG_DEBUG_LOCK_ALLOC

#ifdef CONFIG_DEBUG_LOCK_ALLOC

static struct lock_class_key rcu_lock_key;

static struct lock_class_key rcu_lock_key;

struct lockdep_map rcu_lock_map =

struct lockdep_map rcu_lock_map =

kernel/rcutiny_plugin.h

+0 −16
Original line number Original line Diff line number Diff line
@@ -851,22 +851,6 @@ int rcu_preempt_needs_cpu(void) 
	return rcu_preempt_ctrlblk.rcb.rcucblist != NULL;

	return rcu_preempt_ctrlblk.rcb.rcucblist != NULL;

}

}





/*

 * Check for a task exiting while in a preemptible -RCU read-side

 * critical section, clean up if so.  No need to issue warnings,

 * as debug_check_no_locks_held() already does this if lockdep

 * is enabled.

 */

void exit_rcu(void)

{

	struct task_struct *t = current;



	if (t->rcu_read_lock_nesting == 0)

		return;

	t->rcu_read_lock_nesting = 1;

	__rcu_read_unlock();

}



#else /* #ifdef CONFIG_TINY_PREEMPT_RCU */

#else /* #ifdef CONFIG_TINY_PREEMPT_RCU */





#ifdef CONFIG_RCU_TRACE

#ifdef CONFIG_RCU_TRACE

CRA Git | Maintained and supported by SUSTech CRA and CCSE