Commit 96fd20cf authored by Ingo Molnar's avatar Ingo Molnar
Browse files

Merge branch 'rcu/urgent' of... 

Merge branch 'rcu/urgent' of git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu

 into core/rcu

Pull RCU updates from Paul E. McKenney:

 "This series greatly reduces the performance degradation of Tree SRCU
  on a CPU-hotplug stress test.  The effect was not subtle:  Mike Galbraith
  measured Classic SRCU at 55 seconds and Tree SRCU at more than 16 -minutes-
  for this test.  Mike collected ftrace data that showed that Classic SRCU
  was auto-expediting invocations of synchronize_srcu() that found SRCU
  completely idle.  This series therefore adds this auto-expedite capability
  to Tree SRCU, bringing the performance shortfall to less than ten seconds,
  which is a great improvement over the initial shortfall of 15 minutes."

Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parents d160a727 22607d66

Documentation/admin-guide/kernel-parameters.txt

+8 −0
Original line number Diff line number Diff line
@@ -3779,6 +3779,14 @@ 
	spia_pedr=

	spia_peddr=



	srcutree.exp_holdoff [KNL]

			Specifies how many nanoseconds must elapse

			since the end of the last SRCU grace period for

			a given srcu_struct until the next normal SRCU

			grace period will be considered for automatic

			expediting.  Set to zero to disable automatic

			expediting.



	stacktrace	[FTRACE]

			Enabled the stack tracer on boot up.

include/linux/srcuclassic.h

+14 −0
Original line number Diff line number Diff line
@@ -98,4 +98,18 @@ void synchronize_srcu_expedited(struct srcu_struct *sp); 
void srcu_barrier(struct srcu_struct *sp);

unsigned long srcu_batches_completed(struct srcu_struct *sp);



static inline void srcutorture_get_gp_data(enum rcutorture_type test_type,

					   struct srcu_struct *sp, int *flags,

					   unsigned long *gpnum,

					   unsigned long *completed)

{

	if (test_type != SRCU_FLAVOR)

		return;

	*flags = 0;

	*completed = sp->completed;

	*gpnum = *completed;

	if (sp->batch_queue.head || sp->batch_check0.head || sp->batch_check0.head)

		(*gpnum)++;

}



#endif

include/linux/srcutiny.h

+12 −0
Original line number Diff line number Diff line
@@ -78,4 +78,16 @@ static inline unsigned long srcu_batches_completed(struct srcu_struct *sp) 
	return 0;

}



static inline void srcutorture_get_gp_data(enum rcutorture_type test_type,

					   struct srcu_struct *sp, int *flags,

					   unsigned long *gpnum,

					   unsigned long *completed)

{

	if (test_type != SRCU_FLAVOR)

		return;

	*flags = 0;

	*completed = sp->srcu_gp_seq;

	*gpnum = *completed;

}



#endif

include/linux/srcutree.h

+12 −1
Original line number Diff line number Diff line
@@ -43,10 +43,13 @@ struct srcu_data { 
	spinlock_t lock ____cacheline_internodealigned_in_smp;

	struct rcu_segcblist srcu_cblist;	/* List of callbacks.*/

	unsigned long srcu_gp_seq_needed;	/* Furthest future GP needed. */

	unsigned long srcu_gp_seq_needed_exp;	/* Furthest future exp GP. */

	bool srcu_cblist_invoking;		/* Invoking these CBs? */

	struct delayed_work work;		/* Context for CB invoking. */

	struct rcu_head srcu_barrier_head;	/* For srcu_barrier() use. */

	struct srcu_node *mynode;		/* Leaf srcu_node. */

	unsigned long grpmask;			/* Mask for leaf srcu_node */

						/*  ->srcu_data_have_cbs[]. */

	int cpu;

	struct srcu_struct *sp;

};
@@ -59,6 +62,9 @@ struct srcu_node { 
	unsigned long srcu_have_cbs[4];		/* GP seq for children */

						/*  having CBs, but only */

						/*  is > ->srcu_gq_seq. */

	unsigned long srcu_data_have_cbs[4];	/* Which srcu_data structs */

						/*  have CBs for given GP? */

	unsigned long srcu_gp_seq_needed_exp;	/* Furthest future exp GP. */

	struct srcu_node *srcu_parent;		/* Next up in tree. */

	int grplo;				/* Least CPU for node. */

	int grphi;				/* Biggest CPU for node. */
@@ -77,7 +83,8 @@ struct srcu_struct { 
	unsigned int srcu_idx;			/* Current rdr array element. */

	unsigned long srcu_gp_seq;		/* Grace-period seq #. */

	unsigned long srcu_gp_seq_needed;	/* Latest gp_seq needed. */

	atomic_t srcu_exp_cnt;			/* # ongoing expedited GPs. */

	unsigned long srcu_gp_seq_needed_exp;	/* Furthest future exp GP. */

	unsigned long srcu_last_gp_end;		/* Last GP end timestamp (ns) */

	struct srcu_data __percpu *sda;		/* Per-CPU srcu_data array. */

	unsigned long srcu_barrier_seq;		/* srcu_barrier seq #. */

	struct mutex srcu_barrier_mutex;	/* Serialize barrier ops. */
@@ -136,4 +143,8 @@ void synchronize_srcu_expedited(struct srcu_struct *sp); 
void srcu_barrier(struct srcu_struct *sp);

unsigned long srcu_batches_completed(struct srcu_struct *sp);



void srcutorture_get_gp_data(enum rcutorture_type test_type,

			     struct srcu_struct *sp, int *flags,

			     unsigned long *gpnum, unsigned long *completed);



#endif

kernel/rcu/rcutorture.c

+5 −3
Original line number Diff line number Diff line
@@ -1360,12 +1360,14 @@ rcu_torture_stats_print(void) 
		cur_ops->stats();

	if (rtcv_snap == rcu_torture_current_version &&

	    rcu_torture_current != NULL) {

		int __maybe_unused flags;

		unsigned long __maybe_unused gpnum;

		unsigned long __maybe_unused completed;

		int __maybe_unused flags = 0;

		unsigned long __maybe_unused gpnum = 0;

		unsigned long __maybe_unused completed = 0;



		rcutorture_get_gp_data(cur_ops->ttype,

				       &flags, &gpnum, &completed);

		srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp,

					&flags, &gpnum, &completed);

		wtp = READ_ONCE(writer_task);

		pr_alert("??? Writer stall state %s(%d) g%lu c%lu f%#x ->state %#lx\n",

			 rcu_torture_writer_state_getname(),

CRA Git | Maintained and supported by SUSTech CRA and CCSE