Commit c54a2744 authored by Eric Dumazet's avatar Eric Dumazet Committed by Paul E. McKenney
Browse files

list: Add hlist_unhashed_lockless() 



We would like to use hlist_unhashed() from timer_pending(),
which runs without protection of a lock.

Note that other callers might also want to use this variant.

Instead of forcing a READ_ONCE() for all hlist_unhashed()
callers, add a new helper with an explicit _lockless suffix
in the name to better document what is going on.

Also add various WRITE_ONCE() in __hlist_del(), hlist_add_head()
and hlist_add_before()/hlist_add_behind() to pair with
the READ_ONCE().

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
[ paulmck: Also add WRITE_ONCE() to rculist.h. ]
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent f452ee09

include/linux/list.h

+21 −11
Original line number Diff line number Diff line
@@ -749,6 +749,16 @@ static inline int hlist_unhashed(const struct hlist_node *h) 
	return !h->pprev;

}



/* This variant of hlist_unhashed() must be used in lockless contexts

 * to avoid potential load-tearing.

 * The READ_ONCE() is paired with the various WRITE_ONCE() in hlist

 * helpers that are defined below.

 */

static inline int hlist_unhashed_lockless(const struct hlist_node *h)

{

	return !READ_ONCE(h->pprev);

}



static inline int hlist_empty(const struct hlist_head *h)

{

	return !READ_ONCE(h->first);
@@ -761,7 +771,7 @@ static inline void __hlist_del(struct hlist_node *n) 


	WRITE_ONCE(*pprev, next);

	if (next)

		next->pprev = pprev;

		WRITE_ONCE(next->pprev, pprev);

}



static inline void hlist_del(struct hlist_node *n)
@@ -782,32 +792,32 @@ static inline void hlist_del_init(struct hlist_node *n) 
static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h)

{

	struct hlist_node *first = h->first;

	n->next = first;

	WRITE_ONCE(n->next, first);

	if (first)

		first->pprev = &n->next;

		WRITE_ONCE(first->pprev, &n->next);

	WRITE_ONCE(h->first, n);

	n->pprev = &h->first;

	WRITE_ONCE(n->pprev, &h->first);

}



/* next must be != NULL */

static inline void hlist_add_before(struct hlist_node *n,

					struct hlist_node *next)

{

	n->pprev = next->pprev;

	n->next = next;

	next->pprev = &n->next;

	WRITE_ONCE(n->pprev, next->pprev);

	WRITE_ONCE(n->next, next);

	WRITE_ONCE(next->pprev, &n->next);

	WRITE_ONCE(*(n->pprev), n);

}



static inline void hlist_add_behind(struct hlist_node *n,

				    struct hlist_node *prev)

{

	n->next = prev->next;

	prev->next = n;

	n->pprev = &prev->next;

	WRITE_ONCE(n->next, prev->next);

	WRITE_ONCE(prev->next, n);

	WRITE_ONCE(n->pprev, &prev->next);



	if (n->next)

		n->next->pprev  = &n->next;

		WRITE_ONCE(n->next->pprev, &n->next);

}



/* after that we'll appear to be on some hlist and hlist_del will work */

include/linux/rculist.h

+12 −12
Original line number Diff line number Diff line
@@ -173,7 +173,7 @@ static inline void hlist_del_init_rcu(struct hlist_node *n) 
{

	if (!hlist_unhashed(n)) {

		__hlist_del(n);

		n->pprev = NULL;

		WRITE_ONCE(n->pprev, NULL);

	}

}
@@ -473,7 +473,7 @@ static inline void list_splice_tail_init_rcu(struct list_head *list, 
static inline void hlist_del_rcu(struct hlist_node *n)

{

	__hlist_del(n);

	n->pprev = LIST_POISON2;

	WRITE_ONCE(n->pprev, LIST_POISON2);

}



/**
@@ -489,11 +489,11 @@ static inline void hlist_replace_rcu(struct hlist_node *old, 
	struct hlist_node *next = old->next;



	new->next = next;

	new->pprev = old->pprev;

	WRITE_ONCE(new->pprev, old->pprev);

	rcu_assign_pointer(*(struct hlist_node __rcu **)new->pprev, new);

	if (next)

		new->next->pprev = &new->next;

	old->pprev = LIST_POISON2;

		WRITE_ONCE(new->next->pprev, &new->next);

	WRITE_ONCE(old->pprev, LIST_POISON2);

}



/*
@@ -528,10 +528,10 @@ static inline void hlist_add_head_rcu(struct hlist_node *n, 
	struct hlist_node *first = h->first;



	n->next = first;

	n->pprev = &h->first;

	WRITE_ONCE(n->pprev, &h->first);

	rcu_assign_pointer(hlist_first_rcu(h), n);

	if (first)

		first->pprev = &n->next;

		WRITE_ONCE(first->pprev, &n->next);

}



/**
@@ -564,7 +564,7 @@ static inline void hlist_add_tail_rcu(struct hlist_node *n, 


	if (last) {

		n->next = last->next;

		n->pprev = &last->next;

		WRITE_ONCE(n->pprev, &last->next);

		rcu_assign_pointer(hlist_next_rcu(last), n);

	} else {

		hlist_add_head_rcu(n, h);
@@ -592,10 +592,10 @@ static inline void hlist_add_tail_rcu(struct hlist_node *n, 
static inline void hlist_add_before_rcu(struct hlist_node *n,

					struct hlist_node *next)

{

	n->pprev = next->pprev;

	WRITE_ONCE(n->pprev, next->pprev);

	n->next = next;

	rcu_assign_pointer(hlist_pprev_rcu(n), n);

	next->pprev = &n->next;

	WRITE_ONCE(next->pprev, &n->next);

}



/**
@@ -620,10 +620,10 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n, 
					struct hlist_node *prev)

{

	n->next = prev->next;

	n->pprev = &prev->next;

	WRITE_ONCE(n->pprev, &prev->next);

	rcu_assign_pointer(hlist_next_rcu(prev), n);

	if (n->next)

		n->next->pprev = &n->next;

		WRITE_ONCE(n->next->pprev, &n->next);

}



#define __hlist_for_each_rcu(pos, head)				\

CRA Git | Maintained and supported by SUSTech CRA and CCSE