Commit cc216c5d authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Fix RCU list iterator use of 'rcu_dereference()' 



The RCU iterators used 'rcu_dereference()' on an already-fetched RCU
pointer value, which defeats the whole point of the exercise.

When we dereference a pointer protected by RCU, we need to make sure
that we only fetch the value _once_, because if the compiler ends up
re-loading it due to register pressure, the newly reloaded value could
be different from the previously fetched one, and you get inconsistent
results.

Cleaned-up, fixed, and the pointless list_for_each_safe_rcu #define
deleted by Paul Kenney.

Acked-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 3925e6fc

include/linux/list.h

+15 −33
Original line number Diff line number Diff line
@@ -631,31 +631,14 @@ static inline void list_splice_init_rcu(struct list_head *list, 
 * as long as the traversal is guarded by rcu_read_lock().

 */

#define list_for_each_rcu(pos, head) \

	for (pos = (head)->next; \

		prefetch(rcu_dereference(pos)->next), pos != (head); \

        	pos = pos->next)

	for (pos = rcu_dereference((head)->next); \

		prefetch(pos->next), pos != (head); \

		pos = rcu_dereference(pos->next))



#define __list_for_each_rcu(pos, head) \

	for (pos = (head)->next; \

		rcu_dereference(pos) != (head); \

        	pos = pos->next)



/**

 * list_for_each_safe_rcu

 * @pos:	the &struct list_head to use as a loop cursor.

 * @n:		another &struct list_head to use as temporary storage

 * @head:	the head for your list.

 *

 * Iterate over an rcu-protected list, safe against removal of list entry.

 *

 * This list-traversal primitive may safely run concurrently with

 * the _rcu list-mutation primitives such as list_add_rcu()

 * as long as the traversal is guarded by rcu_read_lock().

 */

#define list_for_each_safe_rcu(pos, n, head) \

	for (pos = (head)->next; \

		n = rcu_dereference(pos)->next, pos != (head); \

		pos = n)

	for (pos = rcu_dereference((head)->next); \

		pos != (head); \

		pos = rcu_dereference(pos->next))



/**

 * list_for_each_entry_rcu	-	iterate over rcu list of given type
@@ -668,10 +651,9 @@ static inline void list_splice_init_rcu(struct list_head *list, 
 * as long as the traversal is guarded by rcu_read_lock().

 */

#define list_for_each_entry_rcu(pos, head, member) \

	for (pos = list_entry((head)->next, typeof(*pos), member); \

		prefetch(rcu_dereference(pos)->member.next), \

			&pos->member != (head); \

		pos = list_entry(pos->member.next, typeof(*pos), member))

	for (pos = list_entry(rcu_dereference((head)->next), typeof(*pos), member); \

		prefetch(pos->member.next), &pos->member != (head); \

		pos = list_entry(rcu_dereference(pos->member.next), typeof(*pos), member))





/**
@@ -686,9 +668,9 @@ static inline void list_splice_init_rcu(struct list_head *list, 
 * as long as the traversal is guarded by rcu_read_lock().

 */

#define list_for_each_continue_rcu(pos, head) \

	for ((pos) = (pos)->next; \

		prefetch(rcu_dereference((pos))->next), (pos) != (head); \

        	(pos) = (pos)->next)

	for ((pos) = rcu_dereference((pos)->next); \

		prefetch((pos)->next), (pos) != (head); \

		(pos) = rcu_dereference((pos)->next))



/*

 * Double linked lists with a single pointer list head.
@@ -986,10 +968,10 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev, 
 * as long as the traversal is guarded by rcu_read_lock().

 */

#define hlist_for_each_entry_rcu(tpos, pos, head, member)		 \

	for (pos = (head)->first;					 \

	     rcu_dereference(pos) && ({ prefetch(pos->next); 1;}) &&	 \

	for (pos = rcu_dereference((head)->first);			 \

	        pos && ({ prefetch(pos->next); 1;}) &&			 \

		({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \

	     pos = pos->next)

	     pos = rcu_dereference(pos->next))



#else

#warning "don't include kernel headers in userspace"

CRA Git | Maintained and supported by SUSTech CRA and CCSE