rcu: Clean up code based on review feedback from Josh Triplett, part 2 (1eba8f84) · Commits · 戴 / test

include/linux/rcupdate.h

+4 −0

Original line number	Diff line number	Diff line
		@@ -196,6 +196,8 @@ static inline void rcu_read_lock_sched(void)
		__acquire(RCU_SCHED);
		rcu_read_acquire();
		}

		/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
		static inline notrace void rcu_read_lock_sched_notrace(void)
		{
		preempt_disable_notrace();
		@@ -213,6 +215,8 @@ static inline void rcu_read_unlock_sched(void)
		__release(RCU_SCHED);
		preempt_enable();
		}

		/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
		static inline notrace void rcu_read_unlock_sched_notrace(void)
		{
		__release(RCU_SCHED);

include/linux/rcutree.h

+1 −5

Original line number	Diff line number	Diff line
		@@ -90,10 +90,6 @@ extern long rcu_batches_completed(void);
		extern long rcu_batches_completed_bh(void);
		extern long rcu_batches_completed_sched(void);

		static inline void rcu_init_sched(void)
		{
		}

		#ifdef CONFIG_NO_HZ
		void rcu_enter_nohz(void);
		void rcu_exit_nohz(void);
		@@ -106,7 +102,7 @@ static inline void rcu_exit_nohz(void)
		}
		#endif /* CONFIG_NO_HZ */

		/* A context switch is a grace period for rcutree. */
		/* A context switch is a grace period for RCU-sched and RCU-bh. */
		static inline int rcu_blocking_is_gp(void)
		{
		return num_online_cpus() == 1;

init/main.c

+0 −1

Original line number	Diff line number	Diff line
		@@ -782,7 +782,6 @@ static void __init do_initcalls(void)
		*/
		static void __init do_basic_setup(void)
		{
		rcu_init_sched(); /* needed by module_init stage. */
		init_workqueues();
		cpuset_init_smp();
		usermodehelper_init();

kernel/rcutree.c

+33 −39

Original line number	Diff line number	Diff line
		@@ -81,24 +81,29 @@ DEFINE_PER_CPU(struct rcu_data, rcu_sched_data);
		struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh_state);
		DEFINE_PER_CPU(struct rcu_data, rcu_bh_data);

		extern long rcu_batches_completed_sched(void);
		static struct rcu_node rcu_get_root(struct rcu_state rsp);
		static void cpu_quiet_msk(unsigned long mask, struct rcu_state *rsp,
		struct rcu_node *rnp, unsigned long flags);
		static void cpu_quiet_msk_finish(struct rcu_state *rsp, unsigned long flags);
		/* Forward declarations for rcutree_plugin.h */
		static inline void rcu_bootup_announce(void);
		long rcu_batches_completed(void);
		static void rcu_preempt_note_context_switch(int cpu);
		static int rcu_preempted_readers(struct rcu_node *rnp);
		#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
		static void rcu_print_task_stall(struct rcu_node *rnp);
		#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
		static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp);
		#ifdef CONFIG_HOTPLUG_CPU
		static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp);
		#endif /* #ifdef CONFIG_HOTPLUG_CPU */
		static void __rcu_process_callbacks(struct rcu_state *rsp,
		static void rcu_preempt_offline_tasks(struct rcu_state *rsp,
		struct rcu_node *rnp,
		struct rcu_data *rdp);
		static void __call_rcu(struct rcu_head *head,
		void (func)(struct rcu_head rcu),
		struct rcu_state *rsp);
		static int __rcu_pending(struct rcu_state rsp, struct rcu_data rdp);
		static void __cpuinit rcu_init_percpu_data(int cpu, struct rcu_state *rsp,
		int preemptable);
		static void rcu_preempt_offline_cpu(int cpu);
		#endif /* #ifdef CONFIG_HOTPLUG_CPU */
		static void rcu_preempt_check_callbacks(int cpu);
		static void rcu_preempt_process_callbacks(void);
		void call_rcu(struct rcu_head head, void (func)(struct rcu_head *rcu));
		static int rcu_preempt_pending(int cpu);
		static int rcu_preempt_needs_cpu(int cpu);
		static void __cpuinit rcu_preempt_init_percpu_data(int cpu);
		static void __init __rcu_init_preempt(void);

		#include "rcutree_plugin.h"

		/*
		* Return true if an RCU grace period is in progress. The ACCESS_ONCE()s
		@@ -377,7 +382,7 @@ static long dyntick_recall_completed(struct rcu_state *rsp)
		/*
		* Snapshot the specified CPU's dynticks counter so that we can later
		* credit them with an implicit quiescent state. Return 1 if this CPU
		* is already in a quiescent state courtesy of dynticks idle mode.
		* is in dynticks idle mode, which is an extended quiescent state.
		*/
		static int dyntick_save_progress_counter(struct rcu_data *rdp)
		{
		@@ -624,9 +629,15 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
		note_new_gpnum(rsp, rdp);

		/*
		* Because we are first, we know that all our callbacks will
		* be covered by this upcoming grace period, even the ones
		* that were registered arbitrarily recently.
		* Because this CPU just now started the new grace period, we know
		* that all of its callbacks will be covered by this upcoming grace
		* period, even the ones that were registered arbitrarily recently.
		* Therefore, advance all outstanding callbacks to RCU_WAIT_TAIL.
		*
		* Other CPUs cannot be sure exactly when the grace period started.
		* Therefore, their recently registered callbacks must pass through
		* an additional RCU_NEXT_READY stage, so that they will be handled
		* by the next RCU grace period.
		*/
		rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
		rdp->nxttail[RCU_WAIT_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
		@@ -886,7 +897,7 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)

		/*
		* Move callbacks from the outgoing CPU to the running CPU.
		* Note that the outgoing CPU is now quiscent, so it is now
		* Note that the outgoing CPU is now quiescent, so it is now
		* (uncharacteristically) safe to access its rcu_data structure.
		* Note also that we must carefully retain the order of the
		* outgoing CPU's callbacks in order for rcu_barrier() to work
		@@ -1577,25 +1588,6 @@ do { \
		} \
		} while (0)

		#ifdef CONFIG_TREE_PREEMPT_RCU

		void __init __rcu_init_preempt(void)
		{
		int i; /* All used by RCU_INIT_FLAVOR(). */
		int j;
		struct rcu_node *rnp;

		RCU_INIT_FLAVOR(&rcu_preempt_state, rcu_preempt_data);
		}

		#else /* #ifdef CONFIG_TREE_PREEMPT_RCU */

		void __init __rcu_init_preempt(void)
		{
		}

		#endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */

		void __init __rcu_init(void)
		{
		int i; /* All used by RCU_INIT_FLAVOR(). */
		@@ -1612,6 +1604,8 @@ void __init __rcu_init(void)
		open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
		}

		#include "rcutree_plugin.h"

		module_param(blimit, int, 0);
		module_param(qhimark, int, 0);
		module_param(qlowmark, int, 0);

kernel/rcutree.h

+23 −8

Original line number	Diff line number	Diff line
		@@ -79,15 +79,21 @@ struct rcu_dynticks {
		* Definition for node within the RCU grace-period-detection hierarchy.
		*/
		struct rcu_node {
		spinlock_t lock;
		spinlock_t lock; /* Root rcu_node's lock protects some */
		/* rcu_state fields as well as following. */
		long gpnum; /* Current grace period for this node. */
		/* This will either be equal to or one */
		/* behind the root rcu_node's gpnum. */
		unsigned long qsmask; /* CPUs or groups that need to switch in */
		/* order for current grace period to proceed.*/
		/* In leaf rcu_node, each bit corresponds to */
		/* an rcu_data structure, otherwise, each */
		/* bit corresponds to a child rcu_node */
		/* structure. */
		unsigned long qsmaskinit;
		/* Per-GP initialization for qsmask. */
		unsigned long grpmask; /* Mask to apply to parent qsmask. */
		/* Only one bit will be set in this mask. */
		int grplo; /* lowest-numbered CPU or group here. */
		int grphi; /* highest-numbered CPU or group here. */
		u8 grpnum; /* CPU/group number for next level up. */
		@@ -95,6 +101,9 @@ struct rcu_node {
		struct rcu_node *parent;
		struct list_head blocked_tasks[2];
		/* Tasks blocked in RCU read-side critsect. */
		/* Grace period number (->gpnum) x blocked */
		/* by tasks on the (x & 0x1) element of the */
		/* blocked_tasks[] array. */
		} ____cacheline_internodealigned_in_smp;

		/* Index values for nxttail array in struct rcu_data. */
		@@ -126,19 +135,22 @@ struct rcu_data {
		* Any of the partitions might be empty, in which case the
		* pointer to that partition will be equal to the pointer for
		* the following partition. When the list is empty, all of
		* the nxttail elements point to nxtlist, which is NULL.
		* the nxttail elements point to the ->nxtlist pointer itself,
		* which in that case is NULL.
		*
		* [nxttail[RCU_NEXT_READY_TAIL], NULL = nxttail[RCU_NEXT_TAIL]):
		* Entries that might have arrived after current GP ended
		* [nxttail[RCU_WAIT_TAIL], nxttail[RCU_NEXT_READY_TAIL]):
		* Entries known to have arrived before current GP ended
		* [nxttail[RCU_DONE_TAIL], nxttail[RCU_WAIT_TAIL]):
		* Entries that batch # <= ->completed - 1: waiting for current GP
		* [nxtlist, *nxttail[RCU_DONE_TAIL]):
		* Entries that batch # <= ->completed
		* The grace period for these entries has completed, and
		* the other grace-period-completed entries may be moved
		* here temporarily in rcu_process_callbacks().
		* [nxttail[RCU_DONE_TAIL], nxttail[RCU_WAIT_TAIL]):
		* Entries that batch # <= ->completed - 1: waiting for current GP
		* [nxttail[RCU_WAIT_TAIL], nxttail[RCU_NEXT_READY_TAIL]):
		* Entries known to have arrived before current GP ended
		* [nxttail[RCU_NEXT_READY_TAIL], nxttail[RCU_NEXT_TAIL]):
		* Entries that might have arrived after current GP ended
		* Note that the value of *nxttail[RCU_NEXT_TAIL] will
		* always be NULL, as this is the end of the list.
		*/
		struct rcu_head *nxtlist;
		struct rcu_head **nxttail[RCU_NEXT_SIZE];
		@@ -216,6 +228,9 @@ struct rcu_state {
		/* Force QS state. */
		long gpnum; /* Current gp number. */
		long completed; /* # of last completed gp. */

		/* End of fields guarded by root rcu_node's lock. */

		spinlock_t onofflock; /* exclude on/offline and */
		/* starting new GP. */
		spinlock_t fqslock; /* Only one task forcing */

Admin message