Commit 12bcbe66 authored by Steven Rostedt's avatar Steven Rostedt
Browse files

rcu: Add _notrace variation of rcu_dereference_raw() and hlist_for_each_entry_rcu() 

As rcu_dereference_raw() under RCU debug config options can add quite a
bit of checks, and that tracing uses rcu_dereference_raw(), these checks
happen with the function tracer. The function tracer also happens to trace
these debug checks too. This added overhead can livelock the system.

Add a new interface to RCU for both rcu_dereference_raw_notrace() as well
as hlist_for_each_entry_rcu_notrace() as the hlist iterator uses the
rcu_dereference_raw() as well, and is used a bit with the function tracer.

Link: http://lkml.kernel.org/r/20130528184209.304356745@goodmis.org



Acked-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: default avatarSteven Rostedt <rostedt@goodmis.org>
parent 6721cb60

include/linux/rculist.h

+20 −0
Original line number Original line Diff line number Diff line
@@ -460,6 +460,26 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev, 
		pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(\

		pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(\

			&(pos)->member)), typeof(*(pos)), member))

			&(pos)->member)), typeof(*(pos)), member))





/**

 * hlist_for_each_entry_rcu_notrace - iterate over rcu list of given type (for tracing)

 * @pos:	the type * to use as a loop cursor.

 * @head:	the head for your list.

 * @member:	the name of the hlist_node within the struct.

 *

 * This list-traversal primitive may safely run concurrently with

 * the _rcu list-mutation primitives such as hlist_add_head_rcu()

 * as long as the traversal is guarded by rcu_read_lock().

 *

 * This is the same as hlist_for_each_entry_rcu() except that it does

 * not do any RCU debugging or tracing.

 */

#define hlist_for_each_entry_rcu_notrace(pos, head, member)			\

	for (pos = hlist_entry_safe (rcu_dereference_raw_notrace(hlist_first_rcu(head)),\

			typeof(*(pos)), member);			\

		pos;							\

		pos = hlist_entry_safe(rcu_dereference_raw_notrace(hlist_next_rcu(\

			&(pos)->member)), typeof(*(pos)), member))



/**

/**

 * hlist_for_each_entry_rcu_bh - iterate over rcu list of given type

 * hlist_for_each_entry_rcu_bh - iterate over rcu list of given type

 * @pos:	the type * to use as a loop cursor.

 * @pos:	the type * to use as a loop cursor.

include/linux/rcupdate.h

+9 −0
Original line number Original line Diff line number Diff line
@@ -640,6 +640,15 @@ static inline void rcu_preempt_sleep_check(void) 




#define rcu_dereference_raw(p) rcu_dereference_check(p, 1) /*@@@ needed? @@@*/

#define rcu_dereference_raw(p) rcu_dereference_check(p, 1) /*@@@ needed? @@@*/





/*

 * The tracing infrastructure traces RCU (we want that), but unfortunately

 * some of the RCU checks causes tracing to lock up the system.

 *

 * The tracing version of rcu_dereference_raw() must not call

 * rcu_read_lock_held().

 */

#define rcu_dereference_raw_notrace(p) __rcu_dereference_check((p), 1, __rcu)



/**

/**

 * rcu_access_index() - fetch RCU index with no dereferencing

 * rcu_access_index() - fetch RCU index with no dereferencing

 * @p: The index to read

 * @p: The index to read

CRA Git | Maintained and supported by SUSTech CRA and CCSE