Commit 2464dd94 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

srcu: Apply trivial callback lists to shrink Tiny SRCU 



The rcu_segcblist structure provides quite a bit of functionality, and
Tiny SRCU needs almost none of it.  So this commit replaces Tiny SRCU's
uses of rcu_segcblist with a simple singly linked list with tail pointer.
This change significantly reduces Tiny SRCU's memory footprint, more
than making up for the growth caused by the creation of rcu_segcblist.c

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent 5a0465e1

include/linux/srcutiny.h

+3 −4
Original line number Diff line number Diff line
@@ -33,9 +33,8 @@ struct srcu_struct { 
	u8 srcu_gp_waiting;		/* GP waiting for readers? */

	struct swait_queue_head srcu_wq;

					/* Last srcu_read_unlock() wakes GP. */

	unsigned long srcu_gp_seq;	/* GP seq # for callback tagging. */

	struct rcu_segcblist srcu_cblist;

					/* Pending SRCU callbacks. */

	struct rcu_head *srcu_cb_head;	/* Pending callbacks: Head. */

	struct rcu_head **srcu_cb_tail;	/* Pending callbacks: Tail. */

	struct work_struct srcu_work;	/* For driving grace periods. */

#ifdef CONFIG_DEBUG_LOCK_ALLOC

	struct lockdep_map dep_map;
@@ -47,7 +46,7 @@ void srcu_drive_gp(struct work_struct *wp); 
#define __SRCU_STRUCT_INIT(name)					\

{									\

	.srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq),	\

	.srcu_cblist = RCU_SEGCBLIST_INITIALIZER(name.srcu_cblist),	\

	.srcu_cb_tail = &name.srcu_cb_head,				\

	.srcu_work = __WORK_INITIALIZER(name.srcu_work, srcu_drive_gp),	\

	__SRCU_DEP_MAP_INIT(name)					\

}

init/Kconfig

+1 −1
Original line number Diff line number Diff line
@@ -573,7 +573,7 @@ config RCU_STALL_COMMON 
	  making these warnings mandatory for the tree variants.



config RCU_NEED_SEGCBLIST

	def_bool ( TREE_RCU || PREEMPT_RCU || TINY_SRCU || TREE_SRCU )

	def_bool ( TREE_RCU || PREEMPT_RCU || TREE_SRCU )



config CONTEXT_TRACKING

       bool

kernel/rcu/rcu.h

+1 −1
Original line number Diff line number Diff line
@@ -398,7 +398,7 @@ static inline void srcutorture_get_gp_data(enum rcutorture_type test_type, 
	if (test_type != SRCU_FLAVOR)

		return;

	*flags = 0;

	*completed = sp->srcu_gp_seq;

	*completed = sp->srcu_idx;

	*gpnum = *completed;

}

kernel/rcu/srcutiny.c

+32 −38
Original line number Diff line number Diff line
@@ -38,8 +38,8 @@ static int init_srcu_struct_fields(struct srcu_struct *sp) 
	sp->srcu_lock_nesting[0] = 0;

	sp->srcu_lock_nesting[1] = 0;

	init_swait_queue_head(&sp->srcu_wq);

	sp->srcu_gp_seq = 0;

	rcu_segcblist_init(&sp->srcu_cblist);

	sp->srcu_cb_head = NULL;

	sp->srcu_cb_tail = &sp->srcu_cb_head;

	sp->srcu_gp_running = false;

	sp->srcu_gp_waiting = false;

	sp->srcu_idx = 0;
@@ -88,10 +88,10 @@ void cleanup_srcu_struct(struct srcu_struct *sp) 
{

	WARN_ON(sp->srcu_lock_nesting[0] || sp->srcu_lock_nesting[1]);

	flush_work(&sp->srcu_work);

	WARN_ON(rcu_seq_state(sp->srcu_gp_seq));

	WARN_ON(sp->srcu_gp_running);

	WARN_ON(sp->srcu_gp_waiting);

	WARN_ON(!rcu_segcblist_empty(&sp->srcu_cblist));

	WARN_ON(sp->srcu_cb_head);

	WARN_ON(&sp->srcu_cb_head != sp->srcu_cb_tail);

}

EXPORT_SYMBOL_GPL(cleanup_srcu_struct);
@@ -117,52 +117,44 @@ EXPORT_SYMBOL_GPL(__srcu_read_unlock); 
void srcu_drive_gp(struct work_struct *wp)

{

	int idx;

	struct rcu_cblist ready_cbs;

	struct srcu_struct *sp;

	struct rcu_head *lh;

	struct rcu_head *rhp;

	struct srcu_struct *sp;



	sp = container_of(wp, struct srcu_struct, srcu_work);

	if (sp->srcu_gp_running || rcu_segcblist_empty(&sp->srcu_cblist))

	if (sp->srcu_gp_running || !READ_ONCE(sp->srcu_cb_head))

		return; /* Already running or nothing to do. */



	/* Tag recently arrived callbacks and wait for readers. */

	/* Remove recently arrived callbacks and wait for readers. */

	WRITE_ONCE(sp->srcu_gp_running, true);

	rcu_segcblist_accelerate(&sp->srcu_cblist,

				 rcu_seq_snap(&sp->srcu_gp_seq));

	rcu_seq_start(&sp->srcu_gp_seq);

	local_irq_disable();

	lh = sp->srcu_cb_head;

	sp->srcu_cb_head = NULL;

	sp->srcu_cb_tail = &sp->srcu_cb_head;

	local_irq_enable();

	idx = sp->srcu_idx;

	WRITE_ONCE(sp->srcu_idx, !sp->srcu_idx);

	WRITE_ONCE(sp->srcu_gp_waiting, true);  /* srcu_read_unlock() wakes! */

	swait_event(sp->srcu_wq, !READ_ONCE(sp->srcu_lock_nesting[idx]));

	WRITE_ONCE(sp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */

	rcu_seq_end(&sp->srcu_gp_seq);



	/* Update callback list based on GP, and invoke ready callbacks. */

	rcu_segcblist_advance(&sp->srcu_cblist,

			      rcu_seq_current(&sp->srcu_gp_seq));

	if (rcu_segcblist_ready_cbs(&sp->srcu_cblist)) {

		rcu_cblist_init(&ready_cbs);

		local_irq_disable();

		rcu_segcblist_extract_done_cbs(&sp->srcu_cblist, &ready_cbs);

		local_irq_enable();

		rhp = rcu_cblist_dequeue(&ready_cbs);

		for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) {

	/* Invoke the callbacks we removed above. */

	while (lh) {

		rhp = lh;

		lh = lh->next;

		local_bh_disable();

		rhp->func(rhp);

		local_bh_enable();

	}

		local_irq_disable();

		rcu_segcblist_insert_count(&sp->srcu_cblist, &ready_cbs);

		local_irq_enable();

	}

	WRITE_ONCE(sp->srcu_gp_running, false);



	/*

	 * If more callbacks, reschedule ourselves.  This can race with

	 * a call_srcu() at interrupt level, but the ->srcu_gp_running

	 * checks will straighten that out.

	 * Enable rescheduling, and if there are more callbacks,

	 * reschedule ourselves.  This can race with a call_srcu()

	 * at interrupt level, but the ->srcu_gp_running checks will

	 * straighten that out.

	 */

	if (!rcu_segcblist_empty(&sp->srcu_cblist))

	WRITE_ONCE(sp->srcu_gp_running, false);

	if (READ_ONCE(sp->srcu_cb_head))

		schedule_work(&sp->srcu_work);

}

EXPORT_SYMBOL_GPL(srcu_drive_gp);
@@ -171,14 +163,16 @@ EXPORT_SYMBOL_GPL(srcu_drive_gp); 
 * Enqueue an SRCU callback on the specified srcu_struct structure,

 * initiating grace-period processing if it is not already running.

 */

void call_srcu(struct srcu_struct *sp, struct rcu_head *head,

void call_srcu(struct srcu_struct *sp, struct rcu_head *rhp,

	       rcu_callback_t func)

{

	unsigned long flags;



	head->func = func;

	rhp->func = func;

	rhp->next = NULL;

	local_irq_save(flags);

	rcu_segcblist_enqueue(&sp->srcu_cblist, head, false);

	*sp->srcu_cb_tail = rhp;

	sp->srcu_cb_tail = &rhp->next;

	local_irq_restore(flags);

	if (!READ_ONCE(sp->srcu_gp_running))

		schedule_work(&sp->srcu_work);

CRA Git | Maintained and supported by SUSTech CRA and CCSE