Commit abaa93d9 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Simplify priority boosting by putting rt_mutex in rcu_node 



RCU priority boosting currently checks for boosting via a pointer in
task_struct.  However, this is not needed: As Oleg noted, if the
rt_mutex is placed in the rcu_node instead of on the booster's stack,
the boostee can simply check it see if it owns the lock.  This commit
makes this change, shrinking task_struct by one pointer and the kernel
by thirteen lines.

Suggested-by: default avatarOleg Nesterov <oleg@redhat.com>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 48bd8e9b

include/linux/init_task.h

+1 −8
Original line number Diff line number Diff line
@@ -102,12 +102,6 @@ extern struct group_info init_groups; 
#define INIT_IDS

#endif



#ifdef CONFIG_RCU_BOOST

#define INIT_TASK_RCU_BOOST()						\

	.rcu_boost_mutex = NULL,

#else

#define INIT_TASK_RCU_BOOST()

#endif

#ifdef CONFIG_TREE_PREEMPT_RCU

#define INIT_TASK_RCU_TREE_PREEMPT()					\

	.rcu_blocked_node = NULL,
@@ -119,8 +113,7 @@ extern struct group_info init_groups; 
	.rcu_read_lock_nesting = 0,					\

	.rcu_read_unlock_special = 0,					\

	.rcu_node_entry = LIST_HEAD_INIT(tsk.rcu_node_entry),		\

	INIT_TASK_RCU_TREE_PREEMPT()					\

	INIT_TASK_RCU_BOOST()

	INIT_TASK_RCU_TREE_PREEMPT()

#else

#define INIT_TASK_RCU_PREEMPT(tsk)

#endif

include/linux/sched.h

+0 −6
Original line number Diff line number Diff line
@@ -1270,9 +1270,6 @@ struct task_struct { 
#ifdef CONFIG_TREE_PREEMPT_RCU

	struct rcu_node *rcu_blocked_node;

#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */

#ifdef CONFIG_RCU_BOOST

	struct rt_mutex *rcu_boost_mutex;

#endif /* #ifdef CONFIG_RCU_BOOST */



#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)

	struct sched_info sched_info;
@@ -2009,9 +2006,6 @@ static inline void rcu_copy_process(struct task_struct *p) 
#ifdef CONFIG_TREE_PREEMPT_RCU

	p->rcu_blocked_node = NULL;

#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */

#ifdef CONFIG_RCU_BOOST

	p->rcu_boost_mutex = NULL;

#endif /* #ifdef CONFIG_RCU_BOOST */

	INIT_LIST_HEAD(&p->rcu_node_entry);

}

kernel/rcu/tree.h

+3 −0
Original line number Diff line number Diff line
@@ -177,6 +177,9 @@ struct rcu_node { 
				/*  to carry out the boosting is fully */

				/*  released with no future boostee accesses */

				/*  before that rt_mutex is re-initialized. */

	struct rt_mutex boost_mtx;

				/* Used only for the priority-boosting */

				/*  side effect, not as a lock. */

	unsigned long boost_time;

				/* When to start boosting (jiffies). */

	struct task_struct *boost_kthread_task;

kernel/rcu/tree_plugin.h

+11 −14
Original line number Diff line number Diff line
@@ -33,6 +33,7 @@ 
#define RCU_KTHREAD_PRIO 1



#ifdef CONFIG_RCU_BOOST

#include "../locking/rtmutex_common.h"

#define RCU_BOOST_PRIO CONFIG_RCU_BOOST_PRIO

#else

#define RCU_BOOST_PRIO RCU_KTHREAD_PRIO
@@ -336,7 +337,7 @@ void rcu_read_unlock_special(struct task_struct *t) 
	unsigned long flags;

	struct list_head *np;

#ifdef CONFIG_RCU_BOOST

	struct rt_mutex *rbmp = NULL;

	bool drop_boost_mutex = false;

#endif /* #ifdef CONFIG_RCU_BOOST */

	struct rcu_node *rnp;

	int special;
@@ -398,11 +399,8 @@ void rcu_read_unlock_special(struct task_struct *t) 
#ifdef CONFIG_RCU_BOOST

		if (&t->rcu_node_entry == rnp->boost_tasks)

			rnp->boost_tasks = np;

		/* Snapshot/clear ->rcu_boost_mutex with rcu_node lock held. */

		if (t->rcu_boost_mutex) {

			rbmp = t->rcu_boost_mutex;

			t->rcu_boost_mutex = NULL;

		}

		/* Snapshot ->boost_mtx ownership with rcu_node lock held. */

		drop_boost_mutex = rt_mutex_owner(&rnp->boost_mtx) == t;

#endif /* #ifdef CONFIG_RCU_BOOST */



		/*
@@ -427,8 +425,8 @@ void rcu_read_unlock_special(struct task_struct *t) 


#ifdef CONFIG_RCU_BOOST

		/* Unboost if we were boosted. */

		if (rbmp) {

			rt_mutex_unlock(rbmp);

		if (drop_boost_mutex) {

			rt_mutex_unlock(&rnp->boost_mtx);

			complete(&rnp->boost_completion);

		}

#endif /* #ifdef CONFIG_RCU_BOOST */
@@ -1151,7 +1149,6 @@ static void rcu_wake_cond(struct task_struct *t, int status) 
static int rcu_boost(struct rcu_node *rnp)

{

	unsigned long flags;

	struct rt_mutex mtx;

	struct task_struct *t;

	struct list_head *tb;
@@ -1202,14 +1199,14 @@ static int rcu_boost(struct rcu_node *rnp) 
	 * section.

	 */

	t = container_of(tb, struct task_struct, rcu_node_entry);

	rt_mutex_init_proxy_locked(&mtx, t);

	t->rcu_boost_mutex = &mtx;

	rt_mutex_init_proxy_locked(&rnp->boost_mtx, t);

	init_completion(&rnp->boost_completion);

	raw_spin_unlock_irqrestore(&rnp->lock, flags);

	rt_mutex_lock(&mtx);  /* Side effect: boosts task t's priority. */

	rt_mutex_unlock(&mtx);  /* Keep lockdep happy. */

	/* Lock only for side effect: boosts task t's priority. */

	rt_mutex_lock(&rnp->boost_mtx);

	rt_mutex_unlock(&rnp->boost_mtx);  /* Then keep lockdep happy. */



	/* Wait until boostee is done accessing mtx before reinitializing. */

	/* Wait for boostee to be done w/boost_mtx before reinitializing. */

	wait_for_completion(&rnp->boost_completion);



	return ACCESS_ONCE(rnp->exp_tasks) != NULL ||

CRA Git | Maintained and supported by SUSTech CRA and CCSE