Commit f7194ac3 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

srcu: Add cleanup_srcu_struct_quiesced() 



The current cleanup_srcu_struct() flushes work, which prevents it
from being invoked from some workqueue contexts, as well as from
atomic (non-blocking) contexts.  This patch therefore introduced a
cleanup_srcu_struct_quiesced(), which can be invoked only after all
activity on the specified srcu_struct has completed.  This restriction
allows cleanup_srcu_struct_quiesced() to be invoked from workqueue
contexts as well as from atomic contexts.

Suggested-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Tested-by: default avatarNitzan Carmi <nitzanc@mellanox.com>
Tested-by: default avatarNicholas Piggin <npiggin@gmail.com>
parent 17672480

include/linux/srcu.h

+35 −1
Original line number Diff line number Diff line
@@ -69,11 +69,45 @@ struct srcu_struct { }; 


void call_srcu(struct srcu_struct *sp, struct rcu_head *head,

		void (*func)(struct rcu_head *head));

void cleanup_srcu_struct(struct srcu_struct *sp);

void _cleanup_srcu_struct(struct srcu_struct *sp, bool quiesced);

int __srcu_read_lock(struct srcu_struct *sp) __acquires(sp);

void __srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp);

void synchronize_srcu(struct srcu_struct *sp);



/**

 * cleanup_srcu_struct - deconstruct a sleep-RCU structure

 * @sp: structure to clean up.

 *

 * Must invoke this after you are finished using a given srcu_struct that

 * was initialized via init_srcu_struct(), else you leak memory.

 */

static inline void cleanup_srcu_struct(struct srcu_struct *sp)

{

	_cleanup_srcu_struct(sp, false);

}



/**

 * cleanup_srcu_struct_quiesced - deconstruct a quiesced sleep-RCU structure

 * @sp: structure to clean up.

 *

 * Must invoke this after you are finished using a given srcu_struct that

 * was initialized via init_srcu_struct(), else you leak memory.  Also,

 * all grace-period processing must have completed.

 *

 * "Completed" means that the last synchronize_srcu() and

 * synchronize_srcu_expedited() calls must have returned before the call

 * to cleanup_srcu_struct_quiesced().  It also means that the callback

 * from the last call_srcu() must have been invoked before the call to

 * cleanup_srcu_struct_quiesced(), but you can use srcu_barrier() to help

 * with this last.  Violating these rules will get you a WARN_ON() splat

 * (with high probability, anyway), and will also cause the srcu_struct

 * to be leaked.

 */

static inline void cleanup_srcu_struct_quiesced(struct srcu_struct *sp)

{

	_cleanup_srcu_struct(sp, true);

}



#ifdef CONFIG_DEBUG_LOCK_ALLOC



/**

kernel/rcu/rcutorture.c

+6 −1
Original line number Diff line number Diff line
@@ -593,7 +593,12 @@ static void srcu_torture_init(void) 


static void srcu_torture_cleanup(void)

{

	static DEFINE_TORTURE_RANDOM(rand);



	if (torture_random(&rand) & 0x800)

		cleanup_srcu_struct(&srcu_ctld);

	else

		cleanup_srcu_struct_quiesced(&srcu_ctld);

	srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */

}

kernel/rcu/srcutiny.c

+6 −3
Original line number Diff line number Diff line
@@ -86,16 +86,19 @@ EXPORT_SYMBOL_GPL(init_srcu_struct); 
 * Must invoke this after you are finished using a given srcu_struct that

 * was initialized via init_srcu_struct(), else you leak memory.

 */

void cleanup_srcu_struct(struct srcu_struct *sp)

void _cleanup_srcu_struct(struct srcu_struct *sp, bool quiesced)

{

	WARN_ON(sp->srcu_lock_nesting[0] || sp->srcu_lock_nesting[1]);

	if (quiesced)

		WARN_ON(work_pending(&sp->srcu_work));

	else

		flush_work(&sp->srcu_work);

	WARN_ON(sp->srcu_gp_running);

	WARN_ON(sp->srcu_gp_waiting);

	WARN_ON(sp->srcu_cb_head);

	WARN_ON(&sp->srcu_cb_head != sp->srcu_cb_tail);

}

EXPORT_SYMBOL_GPL(cleanup_srcu_struct);

EXPORT_SYMBOL_GPL(_cleanup_srcu_struct);



/*

 * Removes the count for the old reader from the appropriate element of

kernel/rcu/srcutree.c

+17 −13
Original line number Diff line number Diff line
@@ -366,24 +366,28 @@ static unsigned long srcu_get_delay(struct srcu_struct *sp) 
	return SRCU_INTERVAL;

}



/**

 * cleanup_srcu_struct - deconstruct a sleep-RCU structure

 * @sp: structure to clean up.

 *

 * Must invoke this after you are finished using a given srcu_struct that

 * was initialized via init_srcu_struct(), else you leak memory.

 */

void cleanup_srcu_struct(struct srcu_struct *sp)

/* Helper for cleanup_srcu_struct() and cleanup_srcu_struct_quiesced(). */

void _cleanup_srcu_struct(struct srcu_struct *sp, bool quiesced)

{

	int cpu;



	if (WARN_ON(!srcu_get_delay(sp)))

		return; /* Leakage unless caller handles error. */

		return; /* Just leak it! */

	if (WARN_ON(srcu_readers_active(sp)))

		return; /* Leakage unless caller handles error. */

		return; /* Just leak it! */

	if (quiesced) {

		if (WARN_ON(delayed_work_pending(&sp->work)))

			return; /* Just leak it! */

	} else {

		flush_delayed_work(&sp->work);

	}

	for_each_possible_cpu(cpu)

		if (quiesced) {

			if (WARN_ON(delayed_work_pending(&per_cpu_ptr(sp->sda, cpu)->work)))

				return; /* Just leak it! */

		} else {

			flush_delayed_work(&per_cpu_ptr(sp->sda, cpu)->work);

		}

	if (WARN_ON(rcu_seq_state(READ_ONCE(sp->srcu_gp_seq)) != SRCU_STATE_IDLE) ||

	    WARN_ON(srcu_readers_active(sp))) {

		pr_info("%s: Active srcu_struct %p state: %d\n", __func__, sp, rcu_seq_state(READ_ONCE(sp->srcu_gp_seq)));
@@ -392,7 +396,7 @@ void cleanup_srcu_struct(struct srcu_struct *sp) 
	free_percpu(sp->sda);

	sp->sda = NULL;

}

EXPORT_SYMBOL_GPL(cleanup_srcu_struct);

EXPORT_SYMBOL_GPL(_cleanup_srcu_struct);



/*

 * Counts the new reader in the appropriate per-CPU element of the

CRA Git | Maintained and supported by SUSTech CRA and CCSE