Commit d0aa7a70 authored by Pierre Peiffer's avatar Pierre Peiffer Committed by Linus Torvalds
Browse files

futex_requeue_pi optimization 



This patch provides the futex_requeue_pi functionality, which allows some
threads waiting on a normal futex to be requeued on the wait-queue of a
PI-futex.

This provides an optimization, already used for (normal) futexes, to be used
with the PI-futexes.

This optimization is currently used by the glibc in pthread_broadcast, when
using "normal" mutexes.  With futex_requeue_pi, it can be used with
PRIO_INHERIT mutexes too.

Signed-off-by: default avatarPierre Peiffer <pierre.peiffer@bull.net>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Ulrich Drepper <drepper@redhat.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent c19384b5

include/linux/futex.h

+8 −1
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ union ktime; 
#define FUTEX_LOCK_PI		6

#define FUTEX_UNLOCK_PI		7

#define FUTEX_TRYLOCK_PI	8

#define FUTEX_CMP_REQUEUE_PI	9



/*

 * Support for robust futexes: the kernel cleans up held futexes at
@@ -84,10 +85,15 @@ struct robust_list_head { 
 */

#define FUTEX_OWNER_DIED	0x40000000



/*

 * Some processes have been requeued on this PI-futex

 */

#define FUTEX_WAITER_REQUEUED	0x20000000



/*

 * The rest of the robust-futex field is for the TID:

 */

#define FUTEX_TID_MASK		0x3fffffff

#define FUTEX_TID_MASK		0x0fffffff



/*

 * This limit protects against a deliberately circular list.
@@ -111,6 +117,7 @@ handle_futex_death(u32 __user *uaddr, struct task_struct *curr, int pi); 
 * We set bit 0 to indicate if it's an inode-based key.

 */

union futex_key {

	u32 __user *uaddr;

	struct {

		unsigned long pgoff;

		struct inode *inode;

kernel/futex.c

+487 −54

File changed.

Preview size limit exceeded, changes collapsed.

kernel/futex_compat.c

+2 −1
Original line number Diff line number Diff line
@@ -156,7 +156,8 @@ asmlinkage long compat_sys_futex(u32 __user *uaddr, int op, u32 val, 
			t = ktime_add(ktime_get(), t);

		tp = &t;

	}

	if (op == FUTEX_REQUEUE || op == FUTEX_CMP_REQUEUE)

	if (op == FUTEX_REQUEUE || op == FUTEX_CMP_REQUEUE

	    || op == FUTEX_CMP_REQUEUE_PI)

		val2 = (int) (unsigned long) utime;



	return do_futex(uaddr, op, val, tp, uaddr2, val2, val3);

kernel/rtmutex.c

+9 −32
Original line number Diff line number Diff line
@@ -56,7 +56,7 @@ 
 * state.

 */



static void

void

rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner,

		   unsigned long mask)

{
@@ -80,29 +80,6 @@ static void fixup_rt_mutex_waiters(struct rt_mutex *lock) 
		clear_rt_mutex_waiters(lock);

}



/*

 * We can speed up the acquire/release, if the architecture

 * supports cmpxchg and if there's no debugging state to be set up

 */

#if defined(__HAVE_ARCH_CMPXCHG) && !defined(CONFIG_DEBUG_RT_MUTEXES)

# define rt_mutex_cmpxchg(l,c,n)	(cmpxchg(&l->owner, c, n) == c)

static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)

{

	unsigned long owner, *p = (unsigned long *) &lock->owner;



	do {

		owner = *p;

	} while (cmpxchg(p, owner, owner | RT_MUTEX_HAS_WAITERS) != owner);

}

#else

# define rt_mutex_cmpxchg(l,c,n)	(0)

static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)

{

	lock->owner = (struct task_struct *)

			((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);

}

#endif



/*

 * Calculate task priority from the waiter list priority

 *
@@ -123,7 +100,7 @@ int rt_mutex_getprio(struct task_struct *task) 
 *

 * This can be both boosting and unboosting. task->pi_lock must be held.

 */

static void __rt_mutex_adjust_prio(struct task_struct *task)

void __rt_mutex_adjust_prio(struct task_struct *task)

{

	int prio = rt_mutex_getprio(task);
@@ -159,7 +136,7 @@ int max_lock_depth = 1024; 
 * Decreases task's usage by one - may thus free the task.

 * Returns 0 or -EDEADLK.

 */

static int rt_mutex_adjust_prio_chain(struct task_struct *task,

int rt_mutex_adjust_prio_chain(struct task_struct *task,

			       int deadlock_detect,

			       struct rt_mutex *orig_lock,

			       struct rt_mutex_waiter *orig_waiter,
@@ -524,7 +501,7 @@ static void wakeup_next_waiter(struct rt_mutex *lock) 
 *

 * Must be called with lock->wait_lock held

 */

static void remove_waiter(struct rt_mutex *lock,

void remove_waiter(struct rt_mutex *lock,

		   struct rt_mutex_waiter *waiter)

{

	int first = (waiter == rt_mutex_top_waiter(lock));

kernel/rtmutex_common.h

+34 −0
Original line number Diff line number Diff line
@@ -112,6 +112,29 @@ static inline unsigned long rt_mutex_owner_pending(struct rt_mutex *lock) 
	return (unsigned long)lock->owner & RT_MUTEX_OWNER_PENDING;

}



/*

 * We can speed up the acquire/release, if the architecture

 * supports cmpxchg and if there's no debugging state to be set up

 */

#if defined(__HAVE_ARCH_CMPXCHG) && !defined(CONFIG_DEBUG_RT_MUTEXES)

# define rt_mutex_cmpxchg(l,c,n)	(cmpxchg(&l->owner, c, n) == c)

static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)

{

	unsigned long owner, *p = (unsigned long *) &lock->owner;



	do {

		owner = *p;

	} while (cmpxchg(p, owner, owner | RT_MUTEX_HAS_WAITERS) != owner);

}

#else

# define rt_mutex_cmpxchg(l,c,n)	(0)

static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)

{

	lock->owner = (struct task_struct *)

			((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);

}

#endif



/*

 * PI-futex support (proxy locking functions, etc.):

 */
@@ -120,4 +143,15 @@ extern void rt_mutex_init_proxy_locked(struct rt_mutex *lock, 
				       struct task_struct *proxy_owner);

extern void rt_mutex_proxy_unlock(struct rt_mutex *lock,

				  struct task_struct *proxy_owner);



extern void rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner,

			       unsigned long mask);

extern void __rt_mutex_adjust_prio(struct task_struct *task);

extern int rt_mutex_adjust_prio_chain(struct task_struct *task,

				      int deadlock_detect,

				      struct rt_mutex *orig_lock,

				      struct rt_mutex_waiter *orig_waiter,

				      struct task_struct *top_task);

extern void remove_waiter(struct rt_mutex *lock,

			  struct rt_mutex_waiter *waiter);

#endif

CRA Git | Maintained and supported by SUSTech CRA and CCSE