Merge branch 'for-3.11' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu

/*

 * Percpu refcounts:

 * (C) 2012 Google, Inc.

 * Author: Kent Overstreet <koverstreet@google.com>

 *

 * This implements a refcount with similar semantics to atomic_t - atomic_inc(),

 * atomic_dec_and_test() - but percpu.

 *

 * There's one important difference between percpu refs and normal atomic_t

 * refcounts; you have to keep track of your initial refcount, and then when you

 * start shutting down you call percpu_ref_kill() _before_ dropping the initial

 * refcount.

 *

 * The refcount will have a range of 0 to ((1U << 31) - 1), i.e. one bit less

 * than an atomic_t - this is because of the way shutdown works, see

 * percpu_ref_kill()/PCPU_COUNT_BIAS.

 *

 * Before you call percpu_ref_kill(), percpu_ref_put() does not check for the

 * refcount hitting 0 - it can't, if it was in percpu mode. percpu_ref_kill()

 * puts the ref back in single atomic_t mode, collecting the per cpu refs and

 * issuing the appropriate barriers, and then marks the ref as shutting down so

 * that percpu_ref_put() will check for the ref hitting 0.  After it returns,

 * it's safe to drop the initial ref.

 *

 * USAGE:

 *

 * See fs/aio.c for some example usage; it's used there for struct kioctx, which

 * is created when userspaces calls io_setup(), and destroyed when userspace

 * calls io_destroy() or the process exits.

 *

 * In the aio code, kill_ioctx() is called when we wish to destroy a kioctx; it

 * calls percpu_ref_kill(), then hlist_del_rcu() and sychronize_rcu() to remove

 * the kioctx from the proccess's list of kioctxs - after that, there can't be

 * any new users of the kioctx (from lookup_ioctx()) and it's then safe to drop

 * the initial ref with percpu_ref_put().

 *

 * Code that does a two stage shutdown like this often needs some kind of

 * explicit synchronization to ensure the initial refcount can only be dropped

 * once - percpu_ref_kill() does this for you, it returns true once and false if

 * someone else already called it. The aio code uses it this way, but it's not

 * necessary if the code has some other mechanism to synchronize teardown.

 * around.

 */

#ifndef _LINUX_PERCPU_REFCOUNT_H

#define _LINUX_PERCPU_REFCOUNT_H

#include <linux/atomic.h>

#include <linux/kernel.h>

#include <linux/percpu.h>

#include <linux/rcupdate.h>

struct percpu_ref;

typedef void (percpu_ref_func_t)(struct percpu_ref *);

struct percpu_ref {

	atomic_t		count;

	/*

	 * The low bit of the pointer indicates whether the ref is in percpu

	 * mode; if set, then get/put will manipulate the atomic_t (this is a

	 * hack because we need to keep the pointer around for

	 * percpu_ref_kill_rcu())

	 */

	unsigned __percpu	*pcpu_count;

	percpu_ref_func_t	*release;

	percpu_ref_func_t	*confirm_kill;

	struct rcu_head		rcu;

};

int __must_check percpu_ref_init(struct percpu_ref *ref,

				 percpu_ref_func_t *release);

void percpu_ref_cancel_init(struct percpu_ref *ref);

void percpu_ref_kill_and_confirm(struct percpu_ref *ref,

				 percpu_ref_func_t *confirm_kill);

/**

 * percpu_ref_kill - drop the initial ref

 * @ref: percpu_ref to kill

 *

 * Must be used to drop the initial ref on a percpu refcount; must be called

 * precisely once before shutdown.

 *

 * Puts @ref in non percpu mode, then does a call_rcu() before gathering up the

 * percpu counters and dropping the initial ref.

 */

static inline void percpu_ref_kill(struct percpu_ref *ref)

{

	return percpu_ref_kill_and_confirm(ref, NULL);

}

#define PCPU_STATUS_BITS	2

#define PCPU_STATUS_MASK	((1 << PCPU_STATUS_BITS) - 1)

#define PCPU_REF_PTR		0

#define PCPU_REF_DEAD		1

#define REF_STATUS(count)	(((unsigned long) count) & PCPU_STATUS_MASK)

/**

 * percpu_ref_get - increment a percpu refcount

 * @ref: percpu_ref to get

 *

 * Analagous to atomic_inc().

  */

static inline void percpu_ref_get(struct percpu_ref *ref)

{

	unsigned __percpu *pcpu_count;

	rcu_read_lock_sched();

	pcpu_count = ACCESS_ONCE(ref->pcpu_count);

	if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR))

		__this_cpu_inc(*pcpu_count);

	else

		atomic_inc(&ref->count);

	rcu_read_unlock_sched();

}

/**

 * percpu_ref_tryget - try to increment a percpu refcount

 * @ref: percpu_ref to try-get

 *

 * Increment a percpu refcount unless it has already been killed.  Returns

 * %true on success; %false on failure.

 *

 * Completion of percpu_ref_kill() in itself doesn't guarantee that tryget

 * will fail.  For such guarantee, percpu_ref_kill_and_confirm() should be

 * used.  After the confirm_kill callback is invoked, it's guaranteed that

 * no new reference will be given out by percpu_ref_tryget().

 */

static inline bool percpu_ref_tryget(struct percpu_ref *ref)

{

	unsigned __percpu *pcpu_count;

	int ret = false;

	rcu_read_lock_sched();

	pcpu_count = ACCESS_ONCE(ref->pcpu_count);

	if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR)) {

		__this_cpu_inc(*pcpu_count);

		ret = true;

	}

	rcu_read_unlock_sched();

	return ret;

}

/**

 * percpu_ref_put - decrement a percpu refcount

 * @ref: percpu_ref to put

 *

 * Decrement the refcount, and if 0, call the release function (which was passed

 * to percpu_ref_init())

 */

static inline void percpu_ref_put(struct percpu_ref *ref)

{

	unsigned __percpu *pcpu_count;

	rcu_read_lock_sched();

	pcpu_count = ACCESS_ONCE(ref->pcpu_count);

	if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR))

		__this_cpu_dec(*pcpu_count);

	else if (unlikely(atomic_dec_and_test(&ref->count)))

		ref->release(ref);

	rcu_read_unlock_sched();

}

#endif

	 sha1.o md5.o irq_regs.o reciprocal_div.o argv_split.o \

	 proportions.o flex_proportions.o prio_heap.o ratelimit.o show_mem.o \

	 is_single_threaded.o plist.o decompress.o kobject_uevent.o \

	 earlycpio.o

	 earlycpio.o percpu-refcount.o

obj-$(CONFIG_ARCH_HAS_DEBUG_STRICT_USER_COPY_CHECKS) += usercopy.o

lib-$(CONFIG_MMU) += ioremap.o

#define pr_fmt(fmt) "%s: " fmt "\n", __func__

#include <linux/kernel.h>

#include <linux/percpu-refcount.h>

/*

 * Initially, a percpu refcount is just a set of percpu counters. Initially, we

 * don't try to detect the ref hitting 0 - which means that get/put can just

 * increment or decrement the local counter. Note that the counter on a

 * particular cpu can (and will) wrap - this is fine, when we go to shutdown the

 * percpu counters will all sum to the correct value

 *

 * (More precisely: because moduler arithmatic is commutative the sum of all the

 * pcpu_count vars will be equal to what it would have been if all the gets and

 * puts were done to a single integer, even if some of the percpu integers

 * overflow or underflow).

 *

 * The real trick to implementing percpu refcounts is shutdown. We can't detect

 * the ref hitting 0 on every put - this would require global synchronization

 * and defeat the whole purpose of using percpu refs.

 *

 * What we do is require the user to keep track of the initial refcount; we know

 * the ref can't hit 0 before the user drops the initial ref, so as long as we

 * convert to non percpu mode before the initial ref is dropped everything

 * works.

 *

 * Converting to non percpu mode is done with some RCUish stuff in

 * percpu_ref_kill. Additionally, we need a bias value so that the atomic_t

 * can't hit 0 before we've added up all the percpu refs.

 */

#define PCPU_COUNT_BIAS		(1U << 31)

/**

 * percpu_ref_init - initialize a percpu refcount

 * @ref: percpu_ref to initialize

 * @release: function which will be called when refcount hits 0

 *

 * Initializes the refcount in single atomic counter mode with a refcount of 1;

 * analagous to atomic_set(ref, 1).

 *

 * Note that @release must not sleep - it may potentially be called from RCU

 * callback context by percpu_ref_kill().

 */

int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release)

{

	atomic_set(&ref->count, 1 + PCPU_COUNT_BIAS);

	ref->pcpu_count = alloc_percpu(unsigned);

	if (!ref->pcpu_count)

		return -ENOMEM;

	ref->release = release;

	return 0;

}

/**

 * percpu_ref_cancel_init - cancel percpu_ref_init()

 * @ref: percpu_ref to cancel init for

 *

 * Once a percpu_ref is initialized, its destruction is initiated by

 * percpu_ref_kill() and completes asynchronously, which can be painful to

 * do when destroying a half-constructed object in init failure path.

 *

 * This function destroys @ref without invoking @ref->release and the

 * memory area containing it can be freed immediately on return.  To

 * prevent accidental misuse, it's required that @ref has finished

 * percpu_ref_init(), whether successful or not, but never used.

 *

 * The weird name and usage restriction are to prevent people from using

 * this function by mistake for normal shutdown instead of

 * percpu_ref_kill().

 */

void percpu_ref_cancel_init(struct percpu_ref *ref)

{

	unsigned __percpu *pcpu_count = ref->pcpu_count;

	int cpu;

	WARN_ON_ONCE(atomic_read(&ref->count) != 1 + PCPU_COUNT_BIAS);

	if (pcpu_count) {

		for_each_possible_cpu(cpu)

			WARN_ON_ONCE(*per_cpu_ptr(pcpu_count, cpu));

		free_percpu(ref->pcpu_count);

	}

}

static void percpu_ref_kill_rcu(struct rcu_head *rcu)

{

	struct percpu_ref *ref = container_of(rcu, struct percpu_ref, rcu);

	unsigned __percpu *pcpu_count = ref->pcpu_count;

	unsigned count = 0;

	int cpu;

	/* Mask out PCPU_REF_DEAD */

	pcpu_count = (unsigned __percpu *)

		(((unsigned long) pcpu_count) & ~PCPU_STATUS_MASK);

	for_each_possible_cpu(cpu)

		count += *per_cpu_ptr(pcpu_count, cpu);

	free_percpu(pcpu_count);

	pr_debug("global %i pcpu %i", atomic_read(&ref->count), (int) count);

	/*

	 * It's crucial that we sum the percpu counters _before_ adding the sum

	 * to &ref->count; since gets could be happening on one cpu while puts

	 * happen on another, adding a single cpu's count could cause

	 * @ref->count to hit 0 before we've got a consistent value - but the

	 * sum of all the counts will be consistent and correct.

	 *

	 * Subtracting the bias value then has to happen _after_ adding count to

	 * &ref->count; we need the bias value to prevent &ref->count from

	 * reaching 0 before we add the percpu counts. But doing it at the same

	 * time is equivalent and saves us atomic operations:

	 */

	atomic_add((int) count - PCPU_COUNT_BIAS, &ref->count);

	/* @ref is viewed as dead on all CPUs, send out kill confirmation */

	if (ref->confirm_kill)

		ref->confirm_kill(ref);

	/*

	 * Now we're in single atomic_t mode with a consistent refcount, so it's

	 * safe to drop our initial ref:

	 */

	percpu_ref_put(ref);

}

/**

 * percpu_ref_kill_and_confirm - drop the initial ref and schedule confirmation

 * @ref: percpu_ref to kill

 * @confirm_kill: optional confirmation callback

 *

 * Equivalent to percpu_ref_kill() but also schedules kill confirmation if

 * @confirm_kill is not NULL.  @confirm_kill, which may not block, will be

 * called after @ref is seen as dead from all CPUs - all further

 * invocations of percpu_ref_tryget() will fail.  See percpu_ref_tryget()

 * for more details.

 *

 * Due to the way percpu_ref is implemented, @confirm_kill will be called

 * after at least one full RCU grace period has passed but this is an

 * implementation detail and callers must not depend on it.

 */

void percpu_ref_kill_and_confirm(struct percpu_ref *ref,

				 percpu_ref_func_t *confirm_kill)

{

	WARN_ONCE(REF_STATUS(ref->pcpu_count) == PCPU_REF_DEAD,

		  "percpu_ref_kill() called more than once!\n");

	ref->pcpu_count = (unsigned __percpu *)

		(((unsigned long) ref->pcpu_count)|PCPU_REF_DEAD);

	ref->confirm_kill = confirm_kill;

	call_rcu_sched(&ref->rcu, percpu_ref_kill_rcu);

}