Commit 1cae13e7 authored by Tejun Heo's avatar Tejun Heo
Browse files

percpu_ref: make INIT_ATOMIC and switch_to_atomic() sticky 



Currently, a percpu_ref which is initialized with
PERPCU_REF_INIT_ATOMIC or switched to atomic mode via
switch_to_atomic() automatically reverts to percpu mode on the first
percpu_ref_reinit().  This makes the atomic mode difficult to use for
cases where a percpu_ref is used as a persistent on/off switch which
may be cycled multiple times.

This patch makes such atomic state sticky so that it survives through
kill/reinit cycles.  After this patch, atomic state is cleared only by
an explicit percpu_ref_switch_to_percpu() call.

Signed-off-by: default avatarTejun Heo <tj@kernel.org>
Reviewed-by: default avatarKent Overstreet <kmo@daterainc.com>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Johannes Weiner <hannes@cmpxchg.org>
parent 2aad2a86

include/linux/percpu-refcount.h

+4 −1
Original line number Diff line number Diff line
@@ -67,7 +67,9 @@ enum { 
enum {

	/*

	 * Start w/ ref == 1 in atomic mode.  Can be switched to percpu

	 * operation using percpu_ref_switch_to_percpu().

	 * operation using percpu_ref_switch_to_percpu().  If initialized

	 * with this flag, the ref will stay in atomic mode until

	 * percpu_ref_switch_to_percpu() is invoked on it.

	 */

	PERCPU_REF_INIT_ATOMIC	= 1 << 0,
@@ -87,6 +89,7 @@ struct percpu_ref { 
	unsigned long		percpu_count_ptr;

	percpu_ref_func_t	*release;

	percpu_ref_func_t	*confirm_switch;

	bool			force_atomic:1;

	struct rcu_head		rcu;

};

lib/percpu-refcount.c

+15 −5
Original line number Diff line number Diff line
@@ -67,6 +67,8 @@ int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release, 
	if (!ref->percpu_count_ptr)

		return -ENOMEM;



	ref->force_atomic = flags & PERCPU_REF_INIT_ATOMIC;



	if (flags & (PERCPU_REF_INIT_ATOMIC | PERCPU_REF_INIT_DEAD))

		ref->percpu_count_ptr |= __PERCPU_REF_ATOMIC;

	else
@@ -202,7 +204,8 @@ static void __percpu_ref_switch_to_atomic(struct percpu_ref *ref, 
 * are guaraneed to be in atomic mode, @confirm_switch, which may not

 * block, is invoked.  This function may be invoked concurrently with all

 * the get/put operations and can safely be mixed with kill and reinit

 * operations.

 * operations.  Note that @ref will stay in atomic mode across kill/reinit

 * cycles until percpu_ref_switch_to_percpu() is called.

 *

 * This function normally doesn't block and can be called from any context

 * but it may block if @confirm_kill is specified and @ref is already in
@@ -216,6 +219,7 @@ static void __percpu_ref_switch_to_atomic(struct percpu_ref *ref, 
void percpu_ref_switch_to_atomic(struct percpu_ref *ref,

				 percpu_ref_func_t *confirm_switch)

{

	ref->force_atomic = true;

	__percpu_ref_switch_to_atomic(ref, confirm_switch);

}
@@ -255,7 +259,10 @@ static void __percpu_ref_switch_to_percpu(struct percpu_ref *ref) 
 *

 * Switch @ref to percpu mode.  This function may be invoked concurrently

 * with all the get/put operations and can safely be mixed with kill and

 * reinit operations.

 * reinit operations.  This function reverses the sticky atomic state set

 * by PERCPU_REF_INIT_ATOMIC or percpu_ref_switch_to_atomic().  If @ref is

 * dying or dead, the actual switching takes place on the following

 * percpu_ref_reinit().

 *

 * This function normally doesn't block and can be called from any context

 * but it may block if @ref is in the process of switching to atomic mode
@@ -263,6 +270,8 @@ static void __percpu_ref_switch_to_percpu(struct percpu_ref *ref) 
 */

void percpu_ref_switch_to_percpu(struct percpu_ref *ref)

{

	ref->force_atomic = false;



	/* a dying or dead ref can't be switched to percpu mode w/o reinit */

	if (!(ref->percpu_count_ptr & __PERCPU_REF_DEAD))

		__percpu_ref_switch_to_percpu(ref);
@@ -304,8 +313,8 @@ EXPORT_SYMBOL_GPL(percpu_ref_kill_and_confirm); 
 * @ref: perpcu_ref to re-initialize

 *

 * Re-initialize @ref so that it's in the same state as when it finished

 * percpu_ref_init().  @ref must have been initialized successfully and

 * reached 0 but not exited.

 * percpu_ref_init() ignoring %PERCPU_REF_INIT_DEAD.  @ref must have been

 * initialized successfully and reached 0 but not exited.

 *

 * Note that percpu_ref_tryget[_live]() are safe to perform on @ref while

 * this function is in progress.
@@ -316,6 +325,7 @@ void percpu_ref_reinit(struct percpu_ref *ref) 


	ref->percpu_count_ptr &= ~__PERCPU_REF_DEAD;

	percpu_ref_get(ref);

	if (!ref->force_atomic)

		__percpu_ref_switch_to_percpu(ref);

}

EXPORT_SYMBOL_GPL(percpu_ref_reinit);

CRA Git | Maintained and supported by SUSTech CRA and CCSE