Commit 74de6960 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Provide functions for determining if call_rcu() has been invoked 



This commit adds rcu_head_init() and rcu_head_after_call_rcu() functions
to help RCU users detect when another CPU has passed the specified
rcu_head structure and function to call_rcu().  The rcu_head_init()
should be invoked before making the structure visible to RCU readers,
and then the rcu_head_after_call_rcu() may be invoked from within
an RCU read-side critical section on an rcu_head structure that
was obtained during a traversal of the data structure in question.
The rcu_head_after_call_rcu() function will return true if the rcu_head
structure has already been passed (with the specified function) to
call_rcu(), otherwise it will return false.

If rcu_head_init() has not been invoked on the rcu_head structure
or if the rcu_head (AKA callback) has already been invoked, then
rcu_head_after_call_rcu() will do WARN_ON_ONCE().

Reported-by: default avatarNeilBrown <neilb@suse.com>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
[ paulmck: Apply neilb naming feedback. ]
parent 7e28c5af

include/linux/rcupdate.h

+40 −0
Original line number Diff line number Diff line
@@ -857,6 +857,46 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) 
#endif /* #else #ifdef CONFIG_ARCH_WEAK_RELEASE_ACQUIRE */





/* Has the specified rcu_head structure been handed to call_rcu()? */



/*

 * rcu_head_init - Initialize rcu_head for rcu_head_after_call_rcu()

 * @rhp: The rcu_head structure to initialize.

 *

 * If you intend to invoke rcu_head_after_call_rcu() to test whether a

 * given rcu_head structure has already been passed to call_rcu(), then

 * you must also invoke this rcu_head_init() function on it just after

 * allocating that structure.  Calls to this function must not race with

 * calls to call_rcu(), rcu_head_after_call_rcu(), or callback invocation.

 */

static inline void rcu_head_init(struct rcu_head *rhp)

{

	rhp->func = (rcu_callback_t)~0L;

}



/*

 * rcu_head_after_call_rcu - Has this rcu_head been passed to call_rcu()?

 * @rhp: The rcu_head structure to test.

 * @func: The function passed to call_rcu() along with @rhp.

 *

 * Returns @true if the @rhp has been passed to call_rcu() with @func,

 * and @false otherwise.  Emits a warning in any other case, including

 * the case where @rhp has already been invoked after a grace period.

 * Calls to this function must not race with callback invocation.  One way

 * to avoid such races is to enclose the call to rcu_head_after_call_rcu()

 * in an RCU read-side critical section that includes a read-side fetch

 * of the pointer to the structure containing @rhp.

 */

static inline bool

rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f)

{

	if (READ_ONCE(rhp->func) == f)

		return true;

	WARN_ON_ONCE(READ_ONCE(rhp->func) != (rcu_callback_t)~0L);

	return false;

}





/* Transitional pre-consolidation compatibility definitions. */



static inline void synchronize_rcu_bh(void)

kernel/rcu/rcu.h

+4 −1
Original line number Diff line number Diff line
@@ -224,6 +224,7 @@ void kfree(const void *); 
 */

static inline bool __rcu_reclaim(const char *rn, struct rcu_head *head)

{

	rcu_callback_t f;

	unsigned long offset = (unsigned long)head->func;



	rcu_lock_acquire(&rcu_callback_map);
@@ -234,7 +235,9 @@ static inline bool __rcu_reclaim(const char *rn, struct rcu_head *head) 
		return true;

	} else {

		RCU_TRACE(trace_rcu_invoke_callback(rn, head);)

		head->func(head);

		f = head->func;

		WRITE_ONCE(head->func, (rcu_callback_t)0L);

		f(head);

		rcu_lock_release(&rcu_callback_map);

		return false;

	}

CRA Git | Maintained and supported by SUSTech CRA and CCSE