Commit f47ad457 authored by Tejun Heo's avatar Tejun Heo
Browse files

percpu_ref: decouple switching to percpu mode and reinit 



percpu_ref has treated the dropping of the base reference and
switching to atomic mode as an integral operation; however, there's
nothing inherent tying the two together.

The use cases for percpu_ref have been expanding continuously.  While
the current init/kill/reinit/exit model can cover a lot, the coupling
of kill/reinit with atomic/percpu mode switching is turning out to be
too restrictive for use cases where many percpu_refs are created and
destroyed back-to-back with only some of them reaching extended
operation.  The coupling also makes implementing always-atomic debug
mode difficult.

This patch separates out percpu mode switching into
percpu_ref_switch_to_percpu() and reimplements percpu_ref_reinit() on
top of it.

* DEAD still requires ATOMIC.  A dead ref can't be switched to percpu
  mode w/o going through reinit.

v2: __percpu_ref_switch_to_percpu() was missing static.  Fixed.
    Reported by Fengguang aka kbuild test robot.

Signed-off-by: default avatarTejun Heo <tj@kernel.org>
Reviewed-by: default avatarKent Overstreet <kmo@daterainc.com>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: kbuild test robot <fengguang.wu@intel.com>
parent 490c79a6

include/linux/percpu-refcount.h

+2 −1
Original line number Diff line number Diff line
@@ -80,9 +80,10 @@ int __must_check percpu_ref_init(struct percpu_ref *ref, 
void percpu_ref_exit(struct percpu_ref *ref);

void percpu_ref_switch_to_atomic(struct percpu_ref *ref,

				 percpu_ref_func_t *confirm_switch);

void percpu_ref_reinit(struct percpu_ref *ref);

void percpu_ref_switch_to_percpu(struct percpu_ref *ref);

void percpu_ref_kill_and_confirm(struct percpu_ref *ref,

				 percpu_ref_func_t *confirm_kill);

void percpu_ref_reinit(struct percpu_ref *ref);



/**

 * percpu_ref_kill - drop the initial ref

lib/percpu-refcount.c

+54 −19
Original line number Diff line number Diff line
@@ -206,40 +206,54 @@ void percpu_ref_switch_to_atomic(struct percpu_ref *ref, 
	__percpu_ref_switch_to_atomic(ref, confirm_switch);

}



/**

 * percpu_ref_reinit - re-initialize a percpu refcount

 * @ref: perpcu_ref to re-initialize

 *

 * Re-initialize @ref so that it's in the same state as when it finished

 * percpu_ref_init().  @ref must have been initialized successfully, killed

 * and reached 0 but not exited.

 *

 * Note that percpu_ref_tryget[_live]() are safe to perform on @ref while

 * this function is in progress.

 */

void percpu_ref_reinit(struct percpu_ref *ref)

static void __percpu_ref_switch_to_percpu(struct percpu_ref *ref)

{

	unsigned long __percpu *percpu_count = percpu_count_ptr(ref);

	int cpu;



	BUG_ON(!percpu_count);

	WARN_ON_ONCE(!percpu_ref_is_zero(ref));



	atomic_long_set(&ref->count, 1 + PERCPU_COUNT_BIAS);

	if (!(ref->percpu_count_ptr & __PERCPU_REF_ATOMIC))

		return;



	wait_event(percpu_ref_switch_waitq, !ref->confirm_switch);



	atomic_long_add(PERCPU_COUNT_BIAS, &ref->count);



	/*

	 * Restore per-cpu operation.  smp_store_release() is paired with

	 * smp_read_barrier_depends() in __ref_is_percpu() and guarantees

	 * that the zeroing is visible to all percpu accesses which can see

	 * the following __PERCPU_REF_ATOMIC_DEAD clearing.

	 * the following __PERCPU_REF_ATOMIC clearing.

	 */

	for_each_possible_cpu(cpu)

		*per_cpu_ptr(percpu_count, cpu) = 0;



	smp_store_release(&ref->percpu_count_ptr,

			  ref->percpu_count_ptr & ~__PERCPU_REF_ATOMIC_DEAD);

			  ref->percpu_count_ptr & ~__PERCPU_REF_ATOMIC);

}



/**

 * percpu_ref_switch_to_percpu - switch a percpu_ref to percpu mode

 * @ref: percpu_ref to switch to percpu mode

 *

 * There's no reason to use this function for the usual reference counting.

 * To re-use an expired ref, use percpu_ref_reinit().

 *

 * Switch @ref to percpu mode.  This function may be invoked concurrently

 * with all the get/put operations and can safely be mixed with kill and

 * reinit operations.

 *

 * This function normally doesn't block and can be called from any context

 * but it may block if @ref is in the process of switching to atomic mode

 * by percpu_ref_switch_atomic().

 */

void percpu_ref_switch_to_percpu(struct percpu_ref *ref)

{

	/* a dying or dead ref can't be switched to percpu mode w/o reinit */

	if (!(ref->percpu_count_ptr & __PERCPU_REF_DEAD))

		__percpu_ref_switch_to_percpu(ref);

}

EXPORT_SYMBOL_GPL(percpu_ref_reinit);



/**

 * percpu_ref_kill_and_confirm - drop the initial ref and schedule confirmation
@@ -253,8 +267,8 @@ EXPORT_SYMBOL_GPL(percpu_ref_reinit); 
 * percpu_ref_tryget_live() for details.

 *

 * This function normally doesn't block and can be called from any context

 * but it may block if @confirm_kill is specified and @ref is already in

 * the process of switching to atomic mode by percpu_ref_switch_atomic().

 * but it may block if @confirm_kill is specified and @ref is in the

 * process of switching to atomic mode by percpu_ref_switch_atomic().

 *

 * Due to the way percpu_ref is implemented, @confirm_switch will be called

 * after at least one full sched RCU grace period has passed but this is an
@@ -271,3 +285,24 @@ void percpu_ref_kill_and_confirm(struct percpu_ref *ref, 
	percpu_ref_put(ref);

}

EXPORT_SYMBOL_GPL(percpu_ref_kill_and_confirm);



/**

 * percpu_ref_reinit - re-initialize a percpu refcount

 * @ref: perpcu_ref to re-initialize

 *

 * Re-initialize @ref so that it's in the same state as when it finished

 * percpu_ref_init().  @ref must have been initialized successfully and

 * reached 0 but not exited.

 *

 * Note that percpu_ref_tryget[_live]() are safe to perform on @ref while

 * this function is in progress.

 */

void percpu_ref_reinit(struct percpu_ref *ref)

{

	WARN_ON_ONCE(!percpu_ref_is_zero(ref));



	ref->percpu_count_ptr &= ~__PERCPU_REF_DEAD;

	percpu_ref_get(ref);

	__percpu_ref_switch_to_percpu(ref);

}

EXPORT_SYMBOL_GPL(percpu_ref_reinit);

CRA Git | Maintained and supported by SUSTech CRA and CCSE