Commit fb00aca4 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

rtmutex: Turn the plist into an rb-tree 



Turn the pi-chains from plist to rb-tree, in the rt_mutex code,
and provide a proper comparison function for -deadline and
-priority tasks.

This is done mainly because:
 - classical prio field of the plist is just an int, which might
   not be enough for representing a deadline;
 - manipulating such a list would become O(nr_deadline_tasks),
   which might be to much, as the number of -deadline task increases.

Therefore, an rb-tree is used, and tasks are queued in it according
to the following logic:
 - among two -priority (i.e., SCHED_BATCH/OTHER/RR/FIFO) tasks, the
   one with the higher (lower, actually!) prio wins;
 - among a -priority and a -deadline task, the latter always wins;
 - among two -deadline tasks, the one with the earliest deadline
   wins.

Queueing and dequeueing functions are changed accordingly, for both
the list of a task's pi-waiters and the list of tasks blocked on
a pi-lock.

Signed-off-by: default avatarPeter Zijlstra <peterz@infradead.org>
Signed-off-by: default avatarDario Faggioli <raistlin@linux.it>
Signed-off-by: default avatarJuri Lelli <juri.lelli@gmail.com>
Signed-off-again-by: default avatarPeter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1383831828-15501-10-git-send-email-juri.lelli@gmail.com


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent af6ace76

include/linux/init_task.h

+10 −0
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ 
#include <linux/user_namespace.h>

#include <linux/securebits.h>

#include <linux/seqlock.h>

#include <linux/rbtree.h>

#include <net/net_namespace.h>

#include <linux/sched/rt.h>
@@ -154,6 +155,14 @@ extern struct task_group root_task_group; 


#define INIT_TASK_COMM "swapper"



#ifdef CONFIG_RT_MUTEXES

# define INIT_RT_MUTEXES(tsk)						\

	.pi_waiters = RB_ROOT,						\

	.pi_waiters_leftmost = NULL,

#else

# define INIT_RT_MUTEXES(tsk)

#endif



/*

 *  INIT_TASK is used to set up the first task table, touch at

 * your own risk!. Base=0, limit=0x1fffff (=2MB)
@@ -221,6 +230,7 @@ extern struct task_group root_task_group; 
	INIT_TRACE_RECURSION						\

	INIT_TASK_RCU_PREEMPT(tsk)					\

	INIT_CPUSET_SEQ(tsk)						\

	INIT_RT_MUTEXES(tsk)						\

	INIT_VTIME(tsk)							\

}

include/linux/rtmutex.h

+6 −12
Original line number Diff line number Diff line
@@ -13,7 +13,7 @@ 
#define __LINUX_RT_MUTEX_H



#include <linux/linkage.h>

#include <linux/plist.h>

#include <linux/rbtree.h>

#include <linux/spinlock_types.h>



extern int max_lock_depth; /* for sysctl */
@@ -22,12 +22,14 @@ extern int max_lock_depth; /* for sysctl */ 
 * The rt_mutex structure

 *

 * @wait_lock:	spinlock to protect the structure

 * @wait_list:	pilist head to enqueue waiters in priority order

 * @waiters:	rbtree root to enqueue waiters in priority order

 * @waiters_leftmost: top waiter

 * @owner:	the mutex owner

 */

struct rt_mutex {

	raw_spinlock_t		wait_lock;

	struct plist_head	wait_list;

	struct rb_root          waiters;

	struct rb_node          *waiters_leftmost;

	struct task_struct	*owner;

#ifdef CONFIG_DEBUG_RT_MUTEXES

	int			save_state;
@@ -66,7 +68,7 @@ struct hrtimer_sleeper; 


#define __RT_MUTEX_INITIALIZER(mutexname) \

	{ .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(mutexname.wait_lock) \

	, .wait_list = PLIST_HEAD_INIT(mutexname.wait_list) \

	, .waiters = RB_ROOT \

	, .owner = NULL \

	__DEBUG_RT_MUTEX_INITIALIZER(mutexname)}
@@ -98,12 +100,4 @@ extern int rt_mutex_trylock(struct rt_mutex *lock); 


extern void rt_mutex_unlock(struct rt_mutex *lock);



#ifdef CONFIG_RT_MUTEXES

# define INIT_RT_MUTEXES(tsk)						\

	.pi_waiters	= PLIST_HEAD_INIT(tsk.pi_waiters),	\

	INIT_RT_MUTEX_DEBUG(tsk)

#else

# define INIT_RT_MUTEXES(tsk)

#endif



#endif

include/linux/sched.h

+3 −1
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ struct sched_param { 
#include <linux/types.h>

#include <linux/timex.h>

#include <linux/jiffies.h>

#include <linux/plist.h>

#include <linux/rbtree.h>

#include <linux/thread_info.h>

#include <linux/cpumask.h>
@@ -1354,7 +1355,8 @@ struct task_struct { 


#ifdef CONFIG_RT_MUTEXES

	/* PI waiters blocked on a rt_mutex held by this task */

	struct plist_head pi_waiters;

	struct rb_root pi_waiters;

	struct rb_node *pi_waiters_leftmost;

	/* Deadlock detection and priority inheritance handling */

	struct rt_mutex_waiter *pi_blocked_on;

#endif

kernel/fork.c

+2 −1
Original line number Diff line number Diff line
@@ -1087,7 +1087,8 @@ static void rt_mutex_init_task(struct task_struct *p) 
{

	raw_spin_lock_init(&p->pi_lock);

#ifdef CONFIG_RT_MUTEXES

	plist_head_init(&p->pi_waiters);

	p->pi_waiters = RB_ROOT;

	p->pi_waiters_leftmost = NULL;

	p->pi_blocked_on = NULL;

#endif

}

kernel/futex.c

+2 −0
Original line number Diff line number Diff line
@@ -2316,6 +2316,8 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, 
	 * code while we sleep on uaddr.

	 */

	debug_rt_mutex_init_waiter(&rt_waiter);

	RB_CLEAR_NODE(&rt_waiter.pi_tree_entry);

	RB_CLEAR_NODE(&rt_waiter.tree_entry);

	rt_waiter.task = NULL;



	ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, VERIFY_WRITE);

CRA Git | Maintained and supported by SUSTech CRA and CCSE