mutex: implement adaptive spinning

	atomic_t		count;

	spinlock_t		wait_lock;

	struct list_head	wait_list;

#ifdef CONFIG_DEBUG_MUTEXES

#if defined(CONFIG_DEBUG_MUTEXES) || defined(CONFIG_SMP)

	struct thread_info	*owner;

#endif

#ifdef CONFIG_DEBUG_MUTEXES

	const char 		*name;

	void			*magic;

#endif

	struct list_head	list;

	struct task_struct	*task;

#ifdef CONFIG_DEBUG_MUTEXES

	struct mutex		*lock;

	void			*magic;

#endif

};

extern signed long schedule_timeout_uninterruptible(signed long timeout);

asmlinkage void __schedule(void);

asmlinkage void schedule(void);

extern int mutex_spin_on_owner(struct mutex *lock, struct thread_info *owner);

struct nsproxy;

struct user_namespace;

/*

 * Must be called with lock->wait_lock held.

 */

void debug_mutex_set_owner(struct mutex *lock, struct thread_info *new_owner)

{

	lock->owner = new_owner;

}

void debug_mutex_lock_common(struct mutex *lock, struct mutex_waiter *waiter)

{

	memset(waiter, MUTEX_DEBUG_INIT, sizeof(*waiter));

	/* Mark the current thread as blocked on the lock: */

	ti->task->blocked_on = waiter;

	waiter->lock = lock;

}

void mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,

	DEBUG_LOCKS_WARN_ON(lock->magic != lock);

	DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info());

	DEBUG_LOCKS_WARN_ON(!lock->wait_list.prev && !lock->wait_list.next);

	DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info());

	mutex_clear_owner(lock);

}

void debug_mutex_init(struct mutex *lock, const char *name,

	debug_check_no_locks_freed((void *)lock, sizeof(*lock));

	lockdep_init_map(&lock->dep_map, name, key, 0);

#endif

	lock->owner = NULL;

	lock->magic = lock;

}

/*

 * This must be called with lock->wait_lock held.

 */

extern void

debug_mutex_set_owner(struct mutex *lock, struct thread_info *new_owner);

static inline void debug_mutex_clear_owner(struct mutex *lock)

{

	lock->owner = NULL;

}

extern void debug_mutex_lock_common(struct mutex *lock,

				    struct mutex_waiter *waiter);

extern void debug_mutex_wake_waiter(struct mutex *lock,

extern void debug_mutex_init(struct mutex *lock, const char *name,

			     struct lock_class_key *key);

static inline void mutex_set_owner(struct mutex *lock)

{

	lock->owner = current_thread_info();

}

static inline void mutex_clear_owner(struct mutex *lock)

{

	lock->owner = NULL;

}

#define spin_lock_mutex(lock, flags)			\

	do {						\

		struct mutex *l = container_of(lock, struct mutex, wait_lock); \

 * Many thanks to Arjan van de Ven, Thomas Gleixner, Steven Rostedt and

 * David Howells for suggestions and improvements.

 *

 *  - Adaptive spinning for mutexes by Peter Zijlstra. (Ported to mainline

 *    from the -rt tree, where it was originally implemented for rtmutexes

 *    by Steven Rostedt, based on work by Gregory Haskins, Peter Morreale

 *    and Sven Dietrich.

 *

 * Also see Documentation/mutex-design.txt.

 */

#include <linux/mutex.h>

	atomic_set(&lock->count, 1);

	spin_lock_init(&lock->wait_lock);

	INIT_LIST_HEAD(&lock->wait_list);

	mutex_clear_owner(lock);

	debug_mutex_init(lock, name, key);

}

	 * 'unlocked' into 'locked' state.

	 */

	__mutex_fastpath_lock(&lock->count, __mutex_lock_slowpath);

	mutex_set_owner(lock);

}

EXPORT_SYMBOL(mutex_lock);

	 * The unlocking fastpath is the 0->1 transition from 'locked'

	 * into 'unlocked' state:

	 */

#ifndef CONFIG_DEBUG_MUTEXES

	/*

	 * When debugging is enabled we must not clear the owner before time,

	 * the slow path will always be taken, and that clears the owner field

	 * after verifying that it was indeed current.

	 */

	mutex_clear_owner(lock);

#endif

	__mutex_fastpath_unlock(&lock->count, __mutex_unlock_slowpath);

}

	unsigned long flags;

	preempt_disable();

	mutex_acquire(&lock->dep_map, subclass, 0, ip);

#if defined(CONFIG_SMP) && !defined(CONFIG_DEBUG_MUTEXES)

	/*

	 * Optimistic spinning.

	 *

	 * We try to spin for acquisition when we find that there are no

	 * pending waiters and the lock owner is currently running on a

	 * (different) CPU.

	 *

	 * The rationale is that if the lock owner is running, it is likely to

	 * release the lock soon.

	 *

	 * Since this needs the lock owner, and this mutex implementation

	 * doesn't track the owner atomically in the lock field, we need to

	 * track it non-atomically.

	 *

	 * We can't do this for DEBUG_MUTEXES because that relies on wait_lock

	 * to serialize everything.

	 */

	for (;;) {

		struct thread_info *owner;

		/*

		 * If there are pending waiters, join them.

		 */

		if (!list_empty(&lock->wait_list))

			break;

		/*

		 * If there's an owner, wait for it to either

		 * release the lock or go to sleep.

		 */

		owner = ACCESS_ONCE(lock->owner);

		if (owner && !mutex_spin_on_owner(lock, owner))

			break;

		/*

		 * When there's no owner, we might have preempted between the

		 * owner acquiring the lock and setting the owner field. If

		 * we're an RT task that will live-lock because we won't let

		 * the owner complete.

		 */

		if (!owner && (need_resched() || rt_task(task)))

			break;

		if (atomic_cmpxchg(&lock->count, 1, 0) == 1) {

			lock_acquired(&lock->dep_map, ip);

			mutex_set_owner(lock);

			preempt_enable();

			return 0;

		}

		/*

		 * The cpu_relax() call is a compiler barrier which forces

		 * everything in this loop to be re-loaded. We don't need

		 * memory barriers as we'll eventually observe the right

		 * values at the cost of a few extra spins.

		 */

		cpu_relax();

	}

#endif

	spin_lock_mutex(&lock->wait_lock, flags);

	debug_mutex_lock_common(lock, &waiter);

	mutex_acquire(&lock->dep_map, subclass, 0, ip);

	debug_mutex_add_waiter(lock, &waiter, task_thread_info(task));

	/* add waiting tasks to the end of the waitqueue (FIFO): */

done:

	lock_acquired(&lock->dep_map, ip);

	/* got the lock - rejoice! */

	mutex_remove_waiter(lock, &waiter, task_thread_info(task));

	debug_mutex_set_owner(lock, task_thread_info(task));

	mutex_remove_waiter(lock, &waiter, current_thread_info());

	mutex_set_owner(lock);

	/* set it to 0 if there are no waiters left: */

	if (likely(list_empty(&lock->wait_list)))

mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass)

{

	might_sleep();

	return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, subclass, _RET_IP_);

	return __mutex_lock_common(lock, TASK_INTERRUPTIBLE,

				   subclass, _RET_IP_);

}

EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested);

		wake_up_process(waiter->task);

	}

	debug_mutex_clear_owner(lock);

	spin_unlock_mutex(&lock->wait_lock, flags);

}

 */

int __sched mutex_lock_interruptible(struct mutex *lock)

{

	int ret;

	might_sleep();

	return __mutex_fastpath_lock_retval

	ret =  __mutex_fastpath_lock_retval

			(&lock->count, __mutex_lock_interruptible_slowpath);

	if (!ret)

		mutex_set_owner(lock);

	return ret;

}

EXPORT_SYMBOL(mutex_lock_interruptible);

int __sched mutex_lock_killable(struct mutex *lock)

{

	int ret;

	might_sleep();

	return __mutex_fastpath_lock_retval

	ret = __mutex_fastpath_lock_retval

			(&lock->count, __mutex_lock_killable_slowpath);

	if (!ret)

		mutex_set_owner(lock);

	return ret;

}

EXPORT_SYMBOL(mutex_lock_killable);

	prev = atomic_xchg(&lock->count, -1);

	if (likely(prev == 1)) {

		debug_mutex_set_owner(lock, current_thread_info());

		mutex_set_owner(lock);

		mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);

	}

	/* Set it back to 0 if there are no waiters: */

	if (likely(list_empty(&lock->wait_list)))

		atomic_set(&lock->count, 0);

 */

int __sched mutex_trylock(struct mutex *lock)

{

	return __mutex_fastpath_trylock(&lock->count,

					__mutex_trylock_slowpath);

	int ret;

	ret = __mutex_fastpath_trylock(&lock->count, __mutex_trylock_slowpath);

	if (ret)

		mutex_set_owner(lock);

	return ret;

}

EXPORT_SYMBOL(mutex_trylock);