Commit 4e6ea4ef authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

srcu: Make early-boot call_srcu() reuse workqueue lists 



Allocating a list_head structure that is almost never used, and, when
used, is used only during early boot (rcu_init() and earlier), is a bit
wasteful.  This commit therefore eliminates that list_head in favor of
the one in the work_struct structure.  This is safe because the work_struct
structure cannot be used until after rcu_init() returns.

Reported-by: default avatarSteven Rostedt <rostedt@goodmis.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Lai Jiangshan <jiangshanlai@gmail.com>
Tested-by: default avatarSteven Rostedt (VMware) <rostedt@goodmis.org>
parent 55cda229

include/linux/srcutiny.h

+0 −2
Original line number Diff line number Diff line
@@ -36,7 +36,6 @@ struct srcu_struct { 
	struct rcu_head *srcu_cb_head;	/* Pending callbacks: Head. */

	struct rcu_head **srcu_cb_tail;	/* Pending callbacks: Tail. */

	struct work_struct srcu_work;	/* For driving grace periods. */

	struct list_head srcu_boot_entry; /* Early-boot callbacks. */

#ifdef CONFIG_DEBUG_LOCK_ALLOC

	struct lockdep_map dep_map;

#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
@@ -49,7 +48,6 @@ void srcu_drive_gp(struct work_struct *wp); 
	.srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq),	\

	.srcu_cb_tail = &name.srcu_cb_head,				\

	.srcu_work = __WORK_INITIALIZER(name.srcu_work, srcu_drive_gp),	\

	.srcu_boot_entry = LIST_HEAD_INIT(name.srcu_boot_entry),	\

	__SRCU_DEP_MAP_INIT(name)					\

}

include/linux/srcutree.h

+1 −2
Original line number Diff line number Diff line
@@ -94,7 +94,6 @@ struct srcu_struct { 
						/*  callback for the barrier */

						/*  operation. */

	struct delayed_work work;

	struct list_head srcu_boot_entry;	/* Early-boot callbacks. */

#ifdef CONFIG_DEBUG_LOCK_ALLOC

	struct lockdep_map dep_map;

#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
@@ -110,7 +109,7 @@ struct srcu_struct { 
	.sda = &pcpu_name,						\

	.lock = __SPIN_LOCK_UNLOCKED(name.lock),			\

	.srcu_gp_seq_needed = -1UL,					\

	.srcu_boot_entry = LIST_HEAD_INIT(name.srcu_boot_entry),	\

	.work = __DELAYED_WORK_INITIALIZER(name.work, NULL, 0),		\

	__SRCU_DEP_MAP_INIT(name)					\

}

kernel/rcu/srcutiny.c

+5 −5
Original line number Diff line number Diff line
@@ -48,7 +48,7 @@ static int init_srcu_struct_fields(struct srcu_struct *sp) 
	sp->srcu_gp_waiting = false;

	sp->srcu_idx = 0;

	INIT_WORK(&sp->srcu_work, srcu_drive_gp);

	INIT_LIST_HEAD(&sp->srcu_boot_entry);

	INIT_LIST_HEAD(&sp->srcu_work.entry);

	return 0;

}
@@ -185,8 +185,8 @@ void call_srcu(struct srcu_struct *sp, struct rcu_head *rhp, 
	if (!READ_ONCE(sp->srcu_gp_running)) {

		if (likely(srcu_init_done))

			schedule_work(&sp->srcu_work);

		else if (list_empty(&sp->srcu_boot_entry))

			list_add(&sp->srcu_boot_entry, &srcu_boot_list);

		else if (list_empty(&sp->srcu_work.entry))

			list_add(&sp->srcu_work.entry, &srcu_boot_list);

	}

}

EXPORT_SYMBOL_GPL(call_srcu);
@@ -224,8 +224,8 @@ void __init srcu_init(void) 
	srcu_init_done = true;

	while (!list_empty(&srcu_boot_list)) {

		sp = list_first_entry(&srcu_boot_list,

				      struct srcu_struct, srcu_boot_entry);

		list_del_init(&sp->srcu_boot_entry);

				      struct srcu_struct, srcu_work.entry);

		list_del_init(&sp->srcu_work.entry);

		schedule_work(&sp->srcu_work);

	}

}

kernel/rcu/srcutree.c

+5 −6
Original line number Diff line number Diff line
@@ -186,7 +186,6 @@ static int init_srcu_struct_fields(struct srcu_struct *sp, bool is_static) 
	mutex_init(&sp->srcu_barrier_mutex);

	atomic_set(&sp->srcu_barrier_cpu_cnt, 0);

	INIT_DELAYED_WORK(&sp->work, process_srcu);

	INIT_LIST_HEAD(&sp->srcu_boot_entry);

	if (!is_static)

		sp->sda = alloc_percpu(struct srcu_data);

	init_srcu_struct_nodes(sp, is_static);
@@ -708,8 +707,8 @@ static void srcu_funnel_gp_start(struct srcu_struct *sp, struct srcu_data *sdp, 
		if (likely(srcu_init_done))

			queue_delayed_work(rcu_gp_wq, &sp->work,

					   srcu_get_delay(sp));

		else if (list_empty(&sp->srcu_boot_entry))

			list_add(&sp->srcu_boot_entry, &srcu_boot_list);

		else if (list_empty(&sp->work.work.entry))

			list_add(&sp->work.work.entry, &srcu_boot_list);

	}

	spin_unlock_irqrestore_rcu_node(sp, flags);

}
@@ -1323,10 +1322,10 @@ void __init srcu_init(void) 


	srcu_init_done = true;

	while (!list_empty(&srcu_boot_list)) {

		sp = list_first_entry(&srcu_boot_list,

				      struct srcu_struct, srcu_boot_entry);

		sp = list_first_entry(&srcu_boot_list, struct srcu_struct,

				      work.work.entry);

		check_init_srcu_struct(sp);

		list_del_init(&sp->srcu_boot_entry);

		list_del_init(&sp->work.work.entry);

		queue_work(rcu_gp_wq, &sp->work.work);

	}

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE