rcu: Abstract rcu_cleanup_dead_rnp() from rcu_cleanup_dead_cpu() (8af3a5e7) · Commits · 戴 / test

kernel/rcu/tree.c

+48 −19

Original line number	Diff line number	Diff line
		@@ -2226,6 +2226,46 @@ static void rcu_cleanup_dying_cpu(struct rcu_state *rsp)
		TPS("cpuofl"));
		}

		/*
		* All CPUs for the specified rcu_node structure have gone offline,
		* and all tasks that were preempted within an RCU read-side critical
		* section while running on one of those CPUs have since exited their RCU
		* read-side critical section. Some other CPU is reporting this fact with
		* the specified rcu_node structure's ->lock held and interrupts disabled.
		* This function therefore goes up the tree of rcu_node structures,
		* clearing the corresponding bits in the ->qsmaskinit fields. Note that
		* the leaf rcu_node structure's ->qsmaskinit field has already been
		* updated
		*
		* This function does check that the specified rcu_node structure has
		* all CPUs offline and no blocked tasks, so it is OK to invoke it
		* prematurely. That said, invoking it after the fact will cost you
		* a needless lock acquisition. So once it has done its work, don't
		* invoke it again.
		*/
		static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf)
		{
		long mask;
		struct rcu_node *rnp = rnp_leaf;

		if (rnp->qsmaskinit \|\| rcu_preempt_has_tasks(rnp))
		return;
		for (;;) {
		mask = rnp->grpmask;
		rnp = rnp->parent;
		if (!rnp)
		break;
		raw_spin_lock(&rnp->lock); /* irqs already disabled. */
		smp_mb__after_unlock_lock(); /* GP memory ordering. */
		rnp->qsmaskinit &= ~mask;
		if (rnp->qsmaskinit) {
		raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
		return;
		}
		raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
		}
		}

		/*
		* The CPU has been completely removed, and some other CPU is reporting
		* this fact from process context. Do the remainder of the cleanup,
		@@ -2236,7 +2276,6 @@ static void rcu_cleanup_dying_cpu(struct rcu_state *rsp)
		static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp)
		{
		unsigned long flags;
		unsigned long mask;
		int need_report = 0;
		struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
		struct rcu_node rnp = rdp->mynode; / Outgoing CPU's rdp & rnp. */
		@@ -2252,24 +2291,14 @@ static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp)
		rcu_send_cbs_to_orphanage(cpu, rsp, rnp, rdp);
		rcu_adopt_orphan_cbs(rsp, flags);

		/* Remove the outgoing CPU from the masks in the rcu_node hierarchy. */
		mask = rdp->grpmask; /* rnp->grplo is constant. */
		do {
		/* Remove outgoing CPU from mask in the leaf rcu_node structure. */
		raw_spin_lock(&rnp->lock); /* irqs already disabled. */
		smp_mb__after_unlock_lock();
		rnp->qsmaskinit &= ~mask;
		if (rnp->qsmaskinit != 0) {
		if (rnp != rdp->mynode)
		raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
		break;
		}
		if (rnp == rdp->mynode)
		smp_mb__after_unlock_lock(); /* Enforce GP memory-order guarantee. */
		rnp->qsmaskinit &= ~rdp->grpmask;
		if (rnp->qsmaskinit == 0) {
		need_report = rcu_preempt_offline_tasks(rsp, rnp, rdp);
		else
		raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
		mask = rnp->grpmask;
		rnp = rnp->parent;
		} while (rnp != NULL);
		rcu_cleanup_dead_rnp(rnp);
		}

		/*
		* We still hold the leaf rcu_node structure lock here, and

kernel/rcu/tree.h

+1 −0

Original line number	Diff line number	Diff line
		@@ -552,6 +552,7 @@ static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp);
		#ifdef CONFIG_HOTPLUG_CPU
		static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp,
		unsigned long flags);
		static bool rcu_preempt_has_tasks(struct rcu_node *rnp);
		#endif /* #ifdef CONFIG_HOTPLUG_CPU */
		static void rcu_print_detail_task_stall(struct rcu_state *rsp);
		static int rcu_print_task_stall(struct rcu_node *rnp);

kernel/rcu/tree_plugin.h

+17 −0

Original line number	Diff line number	Diff line
		@@ -306,6 +306,15 @@ static struct list_head rcu_next_node_entry(struct task_struct t,
		return np;
		}

		/*
		* Return true if the specified rcu_node structure has tasks that were
		* preempted within an RCU read-side critical section.
		*/
		static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
		{
		return !list_empty(&rnp->blkd_tasks);
		}

		/*
		* Handle special cases during rcu_read_unlock(), such as needing to
		* notify RCU core processing or task having blocked during the RCU
		@@ -967,6 +976,14 @@ static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)
		raw_spin_unlock_irqrestore(&rnp->lock, flags);
		}

		/*
		* Because there is no preemptible RCU, there can be no readers blocked.
		*/
		static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
		{
		return false;
		}

		#endif /* #ifdef CONFIG_HOTPLUG_CPU */

		/*

Admin message