Commit 8315f422 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Add call_rcu_tasks() 



This commit adds a new RCU-tasks flavor of RCU, which provides
call_rcu_tasks().  This RCU flavor's quiescent states are voluntary
context switch (not preemption!) and userspace execution (not the idle
loop -- use some sort of schedule_on_each_cpu() if you need to handle the
idle tasks.  Note that unlike other RCU flavors, these quiescent states
occur in tasks, not necessarily CPUs.  Includes fixes from Steven Rostedt.

This RCU flavor is assumed to have very infrequent latency-tolerant
updaters.  This assumption permits significant simplifications, including
a single global callback list protected by a single global lock, along
with a single task-private linked list containing all tasks that have not
yet passed through a quiescent state.  If experience shows this assumption
to be incorrect, the required additional complexity will be added.

Suggested-by: default avatarSteven Rostedt <rostedt@goodmis.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 11ed7f93

include/linux/init_task.h

+9 −0
Original line number Diff line number Diff line
@@ -117,6 +117,14 @@ extern struct group_info init_groups; 
#else

#define INIT_TASK_RCU_PREEMPT(tsk)

#endif

#ifdef CONFIG_TASKS_RCU

#define INIT_TASK_RCU_TASKS(tsk)					\

	.rcu_tasks_holdout = false,					\

	.rcu_tasks_holdout_list =					\

		LIST_HEAD_INIT(tsk.rcu_tasks_holdout_list),

#else

#define INIT_TASK_RCU_TASKS(tsk)

#endif



extern struct cred init_cred;
@@ -224,6 +232,7 @@ extern struct task_group root_task_group; 
	INIT_FTRACE_GRAPH						\

	INIT_TRACE_RECURSION						\

	INIT_TASK_RCU_PREEMPT(tsk)					\

	INIT_TASK_RCU_TASKS(tsk)					\

	INIT_CPUSET_SEQ(tsk)						\

	INIT_RT_MUTEXES(tsk)						\

	INIT_VTIME(tsk)							\

include/linux/rcupdate.h

+36 −0
Original line number Diff line number Diff line
@@ -197,6 +197,26 @@ void call_rcu_sched(struct rcu_head *head, 


void synchronize_sched(void);



/**

 * call_rcu_tasks() - Queue an RCU for invocation task-based grace period

 * @head: structure to be used for queueing the RCU updates.

 * @func: actual callback function to be invoked after the grace period

 *

 * The callback function will be invoked some time after a full grace

 * period elapses, in other words after all currently executing RCU

 * read-side critical sections have completed. call_rcu_tasks() assumes

 * that the read-side critical sections end at a voluntary context

 * switch (not a preemption!), entry into idle, or transition to usermode

 * execution.  As such, there are no read-side primitives analogous to

 * rcu_read_lock() and rcu_read_unlock() because this primitive is intended

 * to determine that all tasks have passed through a safe state, not so

 * much for data-strcuture synchronization.

 *

 * See the description of call_rcu() for more detailed information on

 * memory ordering guarantees.

 */

void call_rcu_tasks(struct rcu_head *head, void (*func)(struct rcu_head *head));



#ifdef CONFIG_PREEMPT_RCU



void __rcu_read_lock(void);
@@ -294,6 +314,22 @@ static inline void rcu_user_hooks_switch(struct task_struct *prev, 
		rcu_irq_exit(); \

	} while (0)



/*

 * Note a voluntary context switch for RCU-tasks benefit.  This is a

 * macro rather than an inline function to avoid #include hell.

 */

#ifdef CONFIG_TASKS_RCU

#define rcu_note_voluntary_context_switch(t) \

	do { \

		preempt_disable(); /* Exclude synchronize_sched(); */ \

		if (ACCESS_ONCE((t)->rcu_tasks_holdout)) \

			ACCESS_ONCE((t)->rcu_tasks_holdout) = false; \

		preempt_enable(); \

	} while (0)

#else /* #ifdef CONFIG_TASKS_RCU */

#define rcu_note_voluntary_context_switch(t)	do { } while (0)

#endif /* #else #ifdef CONFIG_TASKS_RCU */



#if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) || defined(CONFIG_SMP)

bool __rcu_is_watching(void);

#endif /* #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) || defined(CONFIG_SMP) */

include/linux/sched.h

+12 −11
Original line number Diff line number Diff line
@@ -1270,6 +1270,11 @@ struct task_struct { 
#ifdef CONFIG_TREE_PREEMPT_RCU

	struct rcu_node *rcu_blocked_node;

#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */

#ifdef CONFIG_TASKS_RCU

	unsigned long rcu_tasks_nvcsw;

	bool rcu_tasks_holdout;

	struct list_head rcu_tasks_holdout_list;

#endif /* #ifdef CONFIG_TASKS_RCU */



#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)

	struct sched_info sched_info;
@@ -2000,28 +2005,24 @@ extern void task_clear_jobctl_pending(struct task_struct *task, 
				      unsigned int mask);



#ifdef CONFIG_PREEMPT_RCU



#define RCU_READ_UNLOCK_BLOCKED (1 << 0) /* blocked while in RCU read-side. */

#define RCU_READ_UNLOCK_NEED_QS (1 << 1) /* RCU core needs CPU response. */

#endif /* #ifdef CONFIG_PREEMPT_RCU */



static inline void rcu_copy_process(struct task_struct *p)

{

#ifdef CONFIG_PREEMPT_RCU

	p->rcu_read_lock_nesting = 0;

	p->rcu_read_unlock_special = 0;

#ifdef CONFIG_TREE_PREEMPT_RCU

	p->rcu_blocked_node = NULL;

#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */

	INIT_LIST_HEAD(&p->rcu_node_entry);

#endif /* #ifdef CONFIG_PREEMPT_RCU */

#ifdef CONFIG_TASKS_RCU

	p->rcu_tasks_holdout = false;

	INIT_LIST_HEAD(&p->rcu_tasks_holdout_list);

#endif /* #ifdef CONFIG_TASKS_RCU */

}



#else



static inline void rcu_copy_process(struct task_struct *p)

{

}



#endif



static inline void tsk_restore_flags(struct task_struct *task,

				unsigned long orig_flags, unsigned long flags)

{

init/Kconfig

+10 −0
Original line number Diff line number Diff line
@@ -507,6 +507,16 @@ config PREEMPT_RCU 
	  This option enables preemptible-RCU code that is common between

	  TREE_PREEMPT_RCU and, in the old days, TINY_PREEMPT_RCU.



config TASKS_RCU

	bool "Task_based RCU implementation using voluntary context switch"

	default n

	help

	  This option enables a task-based RCU implementation that uses

	  only voluntary context switch (not preemption!), idle, and

	  user-mode execution as quiescent states.



	  If unsure, say N.



config RCU_STALL_COMMON

	def_bool ( TREE_RCU || TREE_PREEMPT_RCU || RCU_TRACE )

	help

kernel/rcu/tiny.c

+2 −0
Original line number Diff line number Diff line
@@ -254,6 +254,8 @@ void rcu_check_callbacks(int cpu, int user) 
		rcu_sched_qs(cpu);

	else if (!in_softirq())

		rcu_bh_qs(cpu);

	if (user)

		rcu_note_voluntary_context_switch(current);

}



/*

CRA Git | Maintained and supported by SUSTech CRA and CCSE