Commit e4fe5dd6 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu-tasks: Further refactor RCU-tasks to allow adding more variants 



This commit refactors RCU tasks to allow variants to be added.  These
variants will share the current Tasks-RCU tasklist scan and the holdout
list processing.

Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent c97d12a6

kernel/rcu/tasks.h

+108 −58
Original line number Diff line number Diff line
@@ -12,6 +12,11 @@ 


struct rcu_tasks;

typedef void (*rcu_tasks_gp_func_t)(struct rcu_tasks *rtp);

typedef void (*pregp_func_t)(void);

typedef void (*pertask_func_t)(struct task_struct *t, struct list_head *hop);

typedef void (*postscan_func_t)(void);

typedef void (*holdouts_func_t)(struct list_head *hop, bool ndrpt, bool *frptp);

typedef void (*postgp_func_t)(void);



/**

 * Definition for a Tasks-RCU-like mechanism.
@@ -21,6 +26,11 @@ typedef void (*rcu_tasks_gp_func_t)(struct rcu_tasks *rtp); 
 * @cbs_lock: Lock protecting callback list.

 * @kthread_ptr: This flavor's grace-period/callback-invocation kthread.

 * @gp_func: This flavor's grace-period-wait function.

 * @pregp_func: This flavor's pre-grace-period function (optional).

 * @pertask_func: This flavor's per-task scan function (optional).

 * @postscan_func: This flavor's post-task scan function (optional).

 * @holdout_func: This flavor's holdout-list scan function (optional).

 * @postgp_func: This flavor's post-grace-period function (optional).

 * @call_func: This flavor's call_rcu()-equivalent function.

 * @name: This flavor's textual name.

 * @kname: This flavor's kthread name.
@@ -32,6 +42,11 @@ struct rcu_tasks { 
	raw_spinlock_t cbs_lock;

	struct task_struct *kthread_ptr;

	rcu_tasks_gp_func_t gp_func;

	pregp_func_t pregp_func;

	pertask_func_t pertask_func;

	postscan_func_t postscan_func;

	holdouts_func_t holdouts_func;

	postgp_func_t postgp_func;

	call_rcu_func_t call_func;

	char *name;

	char *kname;
@@ -113,6 +128,7 @@ static int __noreturn rcu_tasks_kthread(void *arg) 


		/* Pick up any new callbacks. */

		raw_spin_lock_irqsave(&rtp->cbs_lock, flags);

		smp_mb__after_unlock_lock(); // Order updates vs. GP.

		list = rtp->cbs_head;

		rtp->cbs_head = NULL;

		rtp->cbs_tail = &rtp->cbs_head;
@@ -207,6 +223,49 @@ static void __init rcu_tasks_bootup_oddness(void) 
// rates from multiple CPUs.  If this is required, per-CPU callback lists

// will be needed.



/* Pre-grace-period preparation. */

static void rcu_tasks_pregp_step(void)

{

	/*

	 * Wait for all pre-existing t->on_rq and t->nvcsw transitions

	 * to complete.  Invoking synchronize_rcu() suffices because all

	 * these transitions occur with interrupts disabled.  Without this

	 * synchronize_rcu(), a read-side critical section that started

	 * before the grace period might be incorrectly seen as having

	 * started after the grace period.

	 *

	 * This synchronize_rcu() also dispenses with the need for a

	 * memory barrier on the first store to t->rcu_tasks_holdout,

	 * as it forces the store to happen after the beginning of the

	 * grace period.

	 */

	synchronize_rcu();

}



/* Per-task initial processing. */

static void rcu_tasks_pertask(struct task_struct *t, struct list_head *hop)

{

	if (t != current && READ_ONCE(t->on_rq) && !is_idle_task(t)) {

		get_task_struct(t);

		t->rcu_tasks_nvcsw = READ_ONCE(t->nvcsw);

		WRITE_ONCE(t->rcu_tasks_holdout, true);

		list_add(&t->rcu_tasks_holdout_list, hop);

	}

}



/* Processing between scanning taskslist and draining the holdout list. */

void rcu_tasks_postscan(void)

{

	/*

	 * Wait for tasks that are in the process of exiting.  This

	 * does only part of the job, ensuring that all tasks that were

	 * previously exiting reach the point where they have disabled

	 * preemption, allowing the later synchronize_rcu() to finish

	 * the job.

	 */

	synchronize_srcu(&tasks_rcu_exit_srcu);

}



/* See if tasks are still holding out, complain if so. */

static void check_holdout_task(struct task_struct *t,

			       bool needreport, bool *firstreport)
@@ -239,55 +298,63 @@ static void check_holdout_task(struct task_struct *t, 
	sched_show_task(t);

}



/* Scan the holdout lists for tasks no longer holding out. */

static void check_all_holdout_tasks(struct list_head *hop,

				    bool needreport, bool *firstreport)

{

	struct task_struct *t, *t1;



	list_for_each_entry_safe(t, t1, hop, rcu_tasks_holdout_list) {

		check_holdout_task(t, needreport, firstreport);

		cond_resched();

	}

}



/* Finish off the Tasks-RCU grace period. */

static void rcu_tasks_postgp(void)

{

	/*

	 * Because ->on_rq and ->nvcsw are not guaranteed to have a full

	 * memory barriers prior to them in the schedule() path, memory

	 * reordering on other CPUs could cause their RCU-tasks read-side

	 * critical sections to extend past the end of the grace period.

	 * However, because these ->nvcsw updates are carried out with

	 * interrupts disabled, we can use synchronize_rcu() to force the

	 * needed ordering on all such CPUs.

	 *

	 * This synchronize_rcu() also confines all ->rcu_tasks_holdout

	 * accesses to be within the grace period, avoiding the need for

	 * memory barriers for ->rcu_tasks_holdout accesses.

	 *

	 * In addition, this synchronize_rcu() waits for exiting tasks

	 * to complete their final preempt_disable() region of execution,

	 * cleaning up after the synchronize_srcu() above.

	 */

	synchronize_rcu();

}



/* Wait for one RCU-tasks grace period. */

static void rcu_tasks_wait_gp(struct rcu_tasks *rtp)

{

	struct task_struct *g, *t;

	unsigned long lastreport;

	LIST_HEAD(rcu_tasks_holdouts);

	LIST_HEAD(holdouts);

	int fract;



	/*

	 * Wait for all pre-existing t->on_rq and t->nvcsw transitions

	 * to complete.  Invoking synchronize_rcu() suffices because all

	 * these transitions occur with interrupts disabled.  Without this

	 * synchronize_rcu(), a read-side critical section that started

	 * before the grace period might be incorrectly seen as having

	 * started after the grace period.

	 *

	 * This synchronize_rcu() also dispenses with the need for a

	 * memory barrier on the first store to t->rcu_tasks_holdout,

	 * as it forces the store to happen after the beginning of the

	 * grace period.

	 */

	synchronize_rcu();

	rtp->pregp_func();



	/*

	 * There were callbacks, so we need to wait for an RCU-tasks

	 * grace period.  Start off by scanning the task list for tasks

	 * that are not already voluntarily blocked.  Mark these tasks

	 * and make a list of them in rcu_tasks_holdouts.

	 * and make a list of them in holdouts.

	 */

	rcu_read_lock();

	for_each_process_thread(g, t) {

		if (t != current && READ_ONCE(t->on_rq) && !is_idle_task(t)) {

			get_task_struct(t);

			t->rcu_tasks_nvcsw = READ_ONCE(t->nvcsw);

			WRITE_ONCE(t->rcu_tasks_holdout, true);

			list_add(&t->rcu_tasks_holdout_list,

				 &rcu_tasks_holdouts);

		}

	}

	for_each_process_thread(g, t)

		rtp->pertask_func(t, &holdouts);

	rcu_read_unlock();



	/*

	 * Wait for tasks that are in the process of exiting.  This

	 * does only part of the job, ensuring that all tasks that were

	 * previously exiting reach the point where they have disabled

	 * preemption, allowing the later synchronize_rcu() to finish

	 * the job.

	 */

	synchronize_srcu(&tasks_rcu_exit_srcu);

	rtp->postscan_func();



	/*

	 * Each pass through the following loop scans the list of holdout
@@ -303,9 +370,8 @@ static void rcu_tasks_wait_gp(struct rcu_tasks *rtp) 
		bool firstreport;

		bool needreport;

		int rtst;

		struct task_struct *t1;



		if (list_empty(&rcu_tasks_holdouts))

		if (list_empty(&holdouts))

			break;



		/* Slowly back off waiting for holdouts */
@@ -320,31 +386,10 @@ static void rcu_tasks_wait_gp(struct rcu_tasks *rtp) 
			lastreport = jiffies;

		firstreport = true;

		WARN_ON(signal_pending(current));

		list_for_each_entry_safe(t, t1, &rcu_tasks_holdouts,

					 rcu_tasks_holdout_list) {

			check_holdout_task(t, needreport, &firstreport);

			cond_resched();

		}

		rtp->holdouts_func(&holdouts, needreport, &firstreport);

	}



	/*

	 * Because ->on_rq and ->nvcsw are not guaranteed to have a full

	 * memory barriers prior to them in the schedule() path, memory

	 * reordering on other CPUs could cause their RCU-tasks read-side

	 * critical sections to extend past the end of the grace period.

	 * However, because these ->nvcsw updates are carried out with

	 * interrupts disabled, we can use synchronize_rcu() to force the

	 * needed ordering on all such CPUs.

	 *

	 * This synchronize_rcu() also confines all ->rcu_tasks_holdout

	 * accesses to be within the grace period, avoiding the need for

	 * memory barriers for ->rcu_tasks_holdout accesses.

	 *

	 * In addition, this synchronize_rcu() waits for exiting tasks

	 * to complete their final preempt_disable() region of execution,

	 * cleaning up after the synchronize_srcu() above.

	 */

	synchronize_rcu();

	rtp->postgp_func();

}



void call_rcu_tasks(struct rcu_head *rhp, rcu_callback_t func);
@@ -413,6 +458,11 @@ EXPORT_SYMBOL_GPL(rcu_barrier_tasks); 


static int __init rcu_spawn_tasks_kthread(void)

{

	rcu_tasks.pregp_func = rcu_tasks_pregp_step;

	rcu_tasks.pertask_func = rcu_tasks_pertask;

	rcu_tasks.postscan_func = rcu_tasks_postscan;

	rcu_tasks.holdouts_func = check_all_holdout_tasks;

	rcu_tasks.postgp_func = rcu_tasks_postgp;

	rcu_spawn_tasks_kthread_generic(&rcu_tasks);

	return 0;

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE