Commit 77a40f97 authored by Joel Fernandes (Google)'s avatar Joel Fernandes (Google) Committed by Paul E. McKenney
Browse files

rcu: Remove kfree_rcu() special casing and lazy-callback handling 



This commit removes kfree_rcu() special-casing and the lazy-callback
handling from Tree RCU.  It moves some of this special casing to Tiny RCU,
the removal of which will be the subject of later commits.

This results in a nice negative delta.

Suggested-by: default avatarPaul E. McKenney <paulmck@linux.ibm.com>
Signed-off-by: default avatarJoel Fernandes (Google) <joel@joelfernandes.org>
[ paulmck: Add slab.h #include, thanks to kbuild test robot <lkp@intel.com>. ]
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent e99637be

Documentation/RCU/stallwarn.txt

+3 −8
Original line number Diff line number Diff line
@@ -225,18 +225,13 @@ an estimate of the total number of RCU callbacks queued across all CPUs 
In kernels with CONFIG_RCU_FAST_NO_HZ, more information is printed

for each CPU:



	0: (64628 ticks this GP) idle=dd5/3fffffffffffffff/0 softirq=82/543 last_accelerate: a345/d342 Nonlazy posted: ..D

	0: (64628 ticks this GP) idle=dd5/3fffffffffffffff/0 softirq=82/543 last_accelerate: a345/d342 dyntick_enabled: 1



The "last_accelerate:" prints the low-order 16 bits (in hex) of the

jiffies counter when this CPU last invoked rcu_try_advance_all_cbs()

from rcu_needs_cpu() or last invoked rcu_accelerate_cbs() from

rcu_prepare_for_idle().  The "Nonlazy posted:" indicates lazy-callback

status, so that an "l" indicates that all callbacks were lazy at the start

of the last idle period and an "L" indicates that there are currently

no non-lazy callbacks (in both cases, "." is printed otherwise, as

shown above) and "D" indicates that dyntick-idle processing is enabled

("." is printed otherwise, for example, if disabled via the "nohz="

kernel boot parameter).

rcu_prepare_for_idle(). "dyntick_enabled: 1" indicates that dyntick-idle

processing is enabled.



If the grace period ends just as the stall warning starts printing,

there will be a spurious stall-warning message, which will include

include/linux/rcu_segcblist.h

+0 −2
Original line number Diff line number Diff line
@@ -22,7 +22,6 @@ struct rcu_cblist { 
	struct rcu_head *head;

	struct rcu_head **tail;

	long len;

	long len_lazy;

};



#define RCU_CBLIST_INITIALIZER(n) { .head = NULL, .tail = &n.head }
@@ -73,7 +72,6 @@ struct rcu_segcblist { 
#else

	long len;

#endif

	long len_lazy;

	u8 enabled;

	u8 offloaded;

};

include/trace/events/rcu.h

+12 −20
Original line number Diff line number Diff line
@@ -481,16 +481,14 @@ TRACE_EVENT_RCU(rcu_dyntick, 
 */

TRACE_EVENT_RCU(rcu_callback,



	TP_PROTO(const char *rcuname, struct rcu_head *rhp, long qlen_lazy,

		 long qlen),

	TP_PROTO(const char *rcuname, struct rcu_head *rhp, long qlen),



	TP_ARGS(rcuname, rhp, qlen_lazy, qlen),

	TP_ARGS(rcuname, rhp, qlen),



	TP_STRUCT__entry(

		__field(const char *, rcuname)

		__field(void *, rhp)

		__field(void *, func)

		__field(long, qlen_lazy)

		__field(long, qlen)

	),
@@ -498,13 +496,12 @@ TRACE_EVENT_RCU(rcu_callback, 
		__entry->rcuname = rcuname;

		__entry->rhp = rhp;

		__entry->func = rhp->func;

		__entry->qlen_lazy = qlen_lazy;

		__entry->qlen = qlen;

	),



	TP_printk("%s rhp=%p func=%ps %ld/%ld",

	TP_printk("%s rhp=%p func=%ps %ld",

		  __entry->rcuname, __entry->rhp, __entry->func,

		  __entry->qlen_lazy, __entry->qlen)

		  __entry->qlen)

);



/*
@@ -518,15 +515,14 @@ TRACE_EVENT_RCU(rcu_callback, 
TRACE_EVENT_RCU(rcu_kfree_callback,



	TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset,

		 long qlen_lazy, long qlen),

		 long qlen),



	TP_ARGS(rcuname, rhp, offset, qlen_lazy, qlen),

	TP_ARGS(rcuname, rhp, offset, qlen),



	TP_STRUCT__entry(

		__field(const char *, rcuname)

		__field(void *, rhp)

		__field(unsigned long, offset)

		__field(long, qlen_lazy)

		__field(long, qlen)

	),
@@ -534,13 +530,12 @@ TRACE_EVENT_RCU(rcu_kfree_callback, 
		__entry->rcuname = rcuname;

		__entry->rhp = rhp;

		__entry->offset = offset;

		__entry->qlen_lazy = qlen_lazy;

		__entry->qlen = qlen;

	),



	TP_printk("%s rhp=%p func=%ld %ld/%ld",

	TP_printk("%s rhp=%p func=%ld %ld",

		  __entry->rcuname, __entry->rhp, __entry->offset,

		  __entry->qlen_lazy, __entry->qlen)

		  __entry->qlen)

);



/*
@@ -552,27 +547,24 @@ TRACE_EVENT_RCU(rcu_kfree_callback, 
 */

TRACE_EVENT_RCU(rcu_batch_start,



	TP_PROTO(const char *rcuname, long qlen_lazy, long qlen, long blimit),

	TP_PROTO(const char *rcuname, long qlen, long blimit),



	TP_ARGS(rcuname, qlen_lazy, qlen, blimit),

	TP_ARGS(rcuname, qlen, blimit),



	TP_STRUCT__entry(

		__field(const char *, rcuname)

		__field(long, qlen_lazy)

		__field(long, qlen)

		__field(long, blimit)

	),



	TP_fast_assign(

		__entry->rcuname = rcuname;

		__entry->qlen_lazy = qlen_lazy;

		__entry->qlen = qlen;

		__entry->blimit = blimit;

	),



	TP_printk("%s CBs=%ld/%ld bl=%ld",

		  __entry->rcuname, __entry->qlen_lazy, __entry->qlen,

		  __entry->blimit)

	TP_printk("%s CBs=%ld bl=%ld",

		  __entry->rcuname, __entry->qlen, __entry->blimit)

);



/*

kernel/rcu/rcu.h

+0 −27
Original line number Diff line number Diff line
@@ -198,33 +198,6 @@ static inline void debug_rcu_head_unqueue(struct rcu_head *head) 
}

#endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */



void kfree(const void *);



/*

 * Reclaim the specified callback, either by invoking it (non-lazy case)

 * or freeing it directly (lazy case).  Return true if lazy, false otherwise.

 */

static inline bool __rcu_reclaim(const char *rn, struct rcu_head *head)

{

	rcu_callback_t f;

	unsigned long offset = (unsigned long)head->func;



	rcu_lock_acquire(&rcu_callback_map);

	if (__is_kfree_rcu_offset(offset)) {

		trace_rcu_invoke_kfree_callback(rn, head, offset);

		kfree((void *)head - offset);

		rcu_lock_release(&rcu_callback_map);

		return true;

	} else {

		trace_rcu_invoke_callback(rn, head);

		f = head->func;

		WRITE_ONCE(head->func, (rcu_callback_t)0L);

		f(head);

		rcu_lock_release(&rcu_callback_map);

		return false;

	}

}



#ifdef CONFIG_RCU_STALL_COMMON



extern int rcu_cpu_stall_ftrace_dump;

kernel/rcu/rcu_segcblist.c

+3 −22
Original line number Diff line number Diff line
@@ -20,14 +20,10 @@ void rcu_cblist_init(struct rcu_cblist *rclp) 
	rclp->head = NULL;

	rclp->tail = &rclp->head;

	rclp->len = 0;

	rclp->len_lazy = 0;

}



/*

 * Enqueue an rcu_head structure onto the specified callback list.

 * This function assumes that the callback is non-lazy because it

 * is intended for use by no-CBs CPUs, which do not distinguish

 * between lazy and non-lazy RCU callbacks.

 */

void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp)

{
@@ -54,7 +50,6 @@ void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp, 
	else

		drclp->tail = &drclp->head;

	drclp->len = srclp->len;

	drclp->len_lazy = srclp->len_lazy;

	if (!rhp) {

		rcu_cblist_init(srclp);

	} else {
@@ -62,16 +57,12 @@ void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp, 
		srclp->head = rhp;

		srclp->tail = &rhp->next;

		WRITE_ONCE(srclp->len, 1);

		srclp->len_lazy = 0;

	}

}



/*

 * Dequeue the oldest rcu_head structure from the specified callback

 * list.  This function assumes that the callback is non-lazy, but

 * the caller can later invoke rcu_cblist_dequeued_lazy() if it

 * finds otherwise (and if it cares about laziness).  This allows

 * different users to have different ways of determining laziness.

 * list.

 */

struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp)

{
@@ -161,7 +152,6 @@ void rcu_segcblist_init(struct rcu_segcblist *rsclp) 
	for (i = 0; i < RCU_CBLIST_NSEGS; i++)

		rsclp->tails[i] = &rsclp->head;

	rcu_segcblist_set_len(rsclp, 0);

	rsclp->len_lazy = 0;

	rsclp->enabled = 1;

}
@@ -173,7 +163,6 @@ void rcu_segcblist_disable(struct rcu_segcblist *rsclp) 
{

	WARN_ON_ONCE(!rcu_segcblist_empty(rsclp));

	WARN_ON_ONCE(rcu_segcblist_n_cbs(rsclp));

	WARN_ON_ONCE(rcu_segcblist_n_lazy_cbs(rsclp));

	rsclp->enabled = 0;

}
@@ -253,11 +242,9 @@ bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp) 
 * absolutely not OK for it to ever miss posting a callback.

 */

void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,

			   struct rcu_head *rhp, bool lazy)

			   struct rcu_head *rhp)

{

	rcu_segcblist_inc_len(rsclp);

	if (lazy)

		rsclp->len_lazy++;

	smp_mb(); /* Ensure counts are updated before callback is enqueued. */

	rhp->next = NULL;

	WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rhp);
@@ -275,15 +262,13 @@ void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp, 
 * period.  You have been warned.

 */

bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,

			   struct rcu_head *rhp, bool lazy)

			   struct rcu_head *rhp)

{

	int i;



	if (rcu_segcblist_n_cbs(rsclp) == 0)

		return false;

	rcu_segcblist_inc_len(rsclp);

	if (lazy)

		rsclp->len_lazy++;

	smp_mb(); /* Ensure counts are updated before callback is entrained. */

	rhp->next = NULL;

	for (i = RCU_NEXT_TAIL; i > RCU_DONE_TAIL; i--)
@@ -307,8 +292,6 @@ bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, 
void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp,

					       struct rcu_cblist *rclp)

{

	rclp->len_lazy += rsclp->len_lazy;

	rsclp->len_lazy = 0;

	rclp->len = rcu_segcblist_xchg_len(rsclp, 0);

}
@@ -361,9 +344,7 @@ void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp, 
void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp,

				struct rcu_cblist *rclp)

{

	rsclp->len_lazy += rclp->len_lazy;

	rcu_segcblist_add_len(rsclp, rclp->len);

	rclp->len_lazy = 0;

	rclp->len = 0;

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE