Commit 176f8f7a authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Make TASKS_RCU handle nohz_full= CPUs 



Currently TASKS_RCU would ignore a CPU running a task in nohz_full=
usermode execution.  There would be neither a context switch nor a
scheduling-clock interrupt to tell TASKS_RCU that the task in question
had passed through a quiescent state.  The grace period would therefore
extend indefinitely.  This commit therefore makes RCU's dyntick-idle
subsystem record the task_struct structure of the task that is running
in dyntick-idle mode on each CPU.  The TASKS_RCU grace period can
then access this information and record a quiescent state on
behalf of any CPU running in dyntick-idle usermode.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 84a8f446

include/linux/init_task.h

+2 −1
Original line number Diff line number Diff line
@@ -121,7 +121,8 @@ extern struct group_info init_groups; 
#define INIT_TASK_RCU_TASKS(tsk)					\

	.rcu_tasks_holdout = false,					\

	.rcu_tasks_holdout_list =					\

		LIST_HEAD_INIT(tsk.rcu_tasks_holdout_list),

		LIST_HEAD_INIT(tsk.rcu_tasks_holdout_list),		\

	.rcu_tasks_idle_cpu = -1,

#else

#define INIT_TASK_RCU_TASKS(tsk)

#endif

include/linux/sched.h

+2 −0
Original line number Diff line number Diff line
@@ -1274,6 +1274,7 @@ struct task_struct { 
	unsigned long rcu_tasks_nvcsw;

	bool rcu_tasks_holdout;

	struct list_head rcu_tasks_holdout_list;

	int rcu_tasks_idle_cpu;

#endif /* #ifdef CONFIG_TASKS_RCU */



#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
@@ -2020,6 +2021,7 @@ static inline void rcu_copy_process(struct task_struct *p) 
#ifdef CONFIG_TASKS_RCU

	p->rcu_tasks_holdout = false;

	INIT_LIST_HEAD(&p->rcu_tasks_holdout_list);

	p->rcu_tasks_idle_cpu = -1;

#endif /* #ifdef CONFIG_TASKS_RCU */

}

kernel/rcu/tree.c

+2 −0
Original line number Diff line number Diff line
@@ -526,6 +526,7 @@ static void rcu_eqs_enter_common(struct rcu_dynticks *rdtp, long long oldval, 
	atomic_inc(&rdtp->dynticks);

	smp_mb__after_atomic();  /* Force ordering with next sojourn. */

	WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);

	rcu_dynticks_task_enter();



	/*

	 * It is illegal to enter an extended quiescent state while
@@ -642,6 +643,7 @@ void rcu_irq_exit(void) 
static void rcu_eqs_exit_common(struct rcu_dynticks *rdtp, long long oldval,

			       int user)

{

	rcu_dynticks_task_exit();

	smp_mb__before_atomic();  /* Force ordering w/previous sojourn. */

	atomic_inc(&rdtp->dynticks);

	/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */

kernel/rcu/tree.h

+2 −0
Original line number Diff line number Diff line
@@ -605,6 +605,8 @@ static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle, 
static void rcu_bind_gp_kthread(void);

static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp);

static bool rcu_nohz_full_cpu(struct rcu_state *rsp);

static void rcu_dynticks_task_enter(void);

static void rcu_dynticks_task_exit(void);



#endif /* #ifndef RCU_TREE_NONCORE */

kernel/rcu/tree_plugin.h

+16 −0
Original line number Diff line number Diff line
@@ -3036,3 +3036,19 @@ static void rcu_bind_gp_kthread(void) 
		housekeeping_affine(current);

#endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */

}



/* Record the current task on dyntick-idle entry. */

static void rcu_dynticks_task_enter(void)

{

#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)

	ACCESS_ONCE(current->rcu_tasks_idle_cpu) = smp_processor_id();

#endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */

}



/* Record no current task on dyntick-idle exit. */

static void rcu_dynticks_task_exit(void)

{

#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)

	ACCESS_ONCE(current->rcu_tasks_idle_cpu) = -1;

#endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE