Commit a7ef23e5 authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'rhashtable-Fix-unprotected-RCU-dereference-in-__rht_ptr' 



Herbert Xu says:

====================
rhashtable: Fix unprotected RCU dereference in __rht_ptr

This patch series fixes an unprotected dereference in __rht_ptr.
The first patch is a minimal fix that does not use the correct
RCU markings but is suitable for backport, and the second patch
cleans up the RCU markings.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 19016d93 ce9b362b

include/linux/rhashtable.h

+31 −38
Original line number Diff line number Diff line
@@ -84,7 +84,7 @@ struct bucket_table { 


	struct lockdep_map	dep_map;



	struct rhash_lock_head *buckets[] ____cacheline_aligned_in_smp;

	struct rhash_lock_head __rcu *buckets[] ____cacheline_aligned_in_smp;

};



/*
@@ -261,13 +261,12 @@ void rhashtable_free_and_destroy(struct rhashtable *ht, 
				 void *arg);

void rhashtable_destroy(struct rhashtable *ht);



struct rhash_lock_head **rht_bucket_nested(const struct bucket_table *tbl,

					   unsigned int hash);

struct rhash_lock_head **__rht_bucket_nested(const struct bucket_table *tbl,

					     unsigned int hash);

struct rhash_lock_head **rht_bucket_nested_insert(struct rhashtable *ht,

						  struct bucket_table *tbl,

						  unsigned int hash);

struct rhash_lock_head __rcu **rht_bucket_nested(

	const struct bucket_table *tbl, unsigned int hash);

struct rhash_lock_head __rcu **__rht_bucket_nested(

	const struct bucket_table *tbl, unsigned int hash);

struct rhash_lock_head __rcu **rht_bucket_nested_insert(

	struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash);



#define rht_dereference(p, ht) \

	rcu_dereference_protected(p, lockdep_rht_mutex_is_held(ht))
@@ -284,21 +283,21 @@ struct rhash_lock_head **rht_bucket_nested_insert(struct rhashtable *ht, 
#define rht_entry(tpos, pos, member) \

	({ tpos = container_of(pos, typeof(*tpos), member); 1; })



static inline struct rhash_lock_head *const *rht_bucket(

static inline struct rhash_lock_head __rcu *const *rht_bucket(

	const struct bucket_table *tbl, unsigned int hash)

{

	return unlikely(tbl->nest) ? rht_bucket_nested(tbl, hash) :

				     &tbl->buckets[hash];

}



static inline struct rhash_lock_head **rht_bucket_var(

static inline struct rhash_lock_head __rcu **rht_bucket_var(

	struct bucket_table *tbl, unsigned int hash)

{

	return unlikely(tbl->nest) ? __rht_bucket_nested(tbl, hash) :

				     &tbl->buckets[hash];

}



static inline struct rhash_lock_head **rht_bucket_insert(

static inline struct rhash_lock_head __rcu **rht_bucket_insert(

	struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash)

{

	return unlikely(tbl->nest) ? rht_bucket_nested_insert(ht, tbl, hash) :
@@ -325,7 +324,7 @@ static inline struct rhash_lock_head **rht_bucket_insert( 
 */



static inline void rht_lock(struct bucket_table *tbl,

			    struct rhash_lock_head **bkt)

			    struct rhash_lock_head __rcu **bkt)

{

	local_bh_disable();

	bit_spin_lock(0, (unsigned long *)bkt);
@@ -333,7 +332,7 @@ static inline void rht_lock(struct bucket_table *tbl, 
}



static inline void rht_lock_nested(struct bucket_table *tbl,

				   struct rhash_lock_head **bucket,

				   struct rhash_lock_head __rcu **bucket,

				   unsigned int subclass)

{

	local_bh_disable();
@@ -342,18 +341,18 @@ static inline void rht_lock_nested(struct bucket_table *tbl, 
}



static inline void rht_unlock(struct bucket_table *tbl,

			      struct rhash_lock_head **bkt)

			      struct rhash_lock_head __rcu **bkt)

{

	lock_map_release(&tbl->dep_map);

	bit_spin_unlock(0, (unsigned long *)bkt);

	local_bh_enable();

}



static inline struct rhash_head __rcu *__rht_ptr(

	struct rhash_lock_head *const *bkt)

static inline struct rhash_head *__rht_ptr(

	struct rhash_lock_head *p, struct rhash_lock_head __rcu *const *bkt)

{

	return (struct rhash_head __rcu *)

		((unsigned long)*bkt & ~BIT(0) ?:

	return (struct rhash_head *)

		((unsigned long)p & ~BIT(0) ?:

		 (unsigned long)RHT_NULLS_MARKER(bkt));

}
@@ -365,47 +364,41 @@ static inline struct rhash_head __rcu *__rht_ptr( 
 *            access is guaranteed, such as when destroying the table.

 */

static inline struct rhash_head *rht_ptr_rcu(

	struct rhash_lock_head *const *bkt)

	struct rhash_lock_head __rcu *const *bkt)

{

	struct rhash_head __rcu *p = __rht_ptr(bkt);



	return rcu_dereference(p);

	return __rht_ptr(rcu_dereference(*bkt), bkt);

}



static inline struct rhash_head *rht_ptr(

	struct rhash_lock_head *const *bkt,

	struct rhash_lock_head __rcu *const *bkt,

	struct bucket_table *tbl,

	unsigned int hash)

{

	return rht_dereference_bucket(__rht_ptr(bkt), tbl, hash);

	return __rht_ptr(rht_dereference_bucket(*bkt, tbl, hash), bkt);

}



static inline struct rhash_head *rht_ptr_exclusive(

	struct rhash_lock_head *const *bkt)

	struct rhash_lock_head __rcu *const *bkt)

{

	return rcu_dereference_protected(__rht_ptr(bkt), 1);

	return __rht_ptr(rcu_dereference_protected(*bkt, 1), bkt);

}



static inline void rht_assign_locked(struct rhash_lock_head **bkt,

static inline void rht_assign_locked(struct rhash_lock_head __rcu **bkt,

				     struct rhash_head *obj)

{

	struct rhash_head __rcu **p = (struct rhash_head __rcu **)bkt;



	if (rht_is_a_nulls(obj))

		obj = NULL;

	rcu_assign_pointer(*p, (void *)((unsigned long)obj | BIT(0)));

	rcu_assign_pointer(*bkt, (void *)((unsigned long)obj | BIT(0)));

}



static inline void rht_assign_unlock(struct bucket_table *tbl,

				     struct rhash_lock_head **bkt,

				     struct rhash_lock_head __rcu **bkt,

				     struct rhash_head *obj)

{

	struct rhash_head __rcu **p = (struct rhash_head __rcu **)bkt;



	if (rht_is_a_nulls(obj))

		obj = NULL;

	lock_map_release(&tbl->dep_map);

	rcu_assign_pointer(*p, obj);

	rcu_assign_pointer(*bkt, (void *)obj);

	preempt_enable();

	__release(bitlock);

	local_bh_enable();
@@ -593,7 +586,7 @@ static inline struct rhash_head *__rhashtable_lookup( 
		.ht = ht,

		.key = key,

	};

	struct rhash_lock_head *const *bkt;

	struct rhash_lock_head __rcu *const *bkt;

	struct bucket_table *tbl;

	struct rhash_head *he;

	unsigned int hash;
@@ -709,7 +702,7 @@ static inline void *__rhashtable_insert_fast( 
		.ht = ht,

		.key = key,

	};

	struct rhash_lock_head **bkt;

	struct rhash_lock_head __rcu **bkt;

	struct rhash_head __rcu **pprev;

	struct bucket_table *tbl;

	struct rhash_head *head;
@@ -995,7 +988,7 @@ static inline int __rhashtable_remove_fast_one( 
	struct rhash_head *obj, const struct rhashtable_params params,

	bool rhlist)

{

	struct rhash_lock_head **bkt;

	struct rhash_lock_head __rcu **bkt;

	struct rhash_head __rcu **pprev;

	struct rhash_head *he;

	unsigned int hash;
@@ -1147,7 +1140,7 @@ static inline int __rhashtable_replace_fast( 
	struct rhash_head *obj_old, struct rhash_head *obj_new,

	const struct rhashtable_params params)

{

	struct rhash_lock_head **bkt;

	struct rhash_lock_head __rcu **bkt;

	struct rhash_head __rcu **pprev;

	struct rhash_head *he;

	unsigned int hash;

lib/rhashtable.c

+16 −19
Original line number Diff line number Diff line
@@ -31,7 +31,7 @@ 


union nested_table {

	union nested_table __rcu *table;

	struct rhash_lock_head *bucket;

	struct rhash_lock_head __rcu *bucket;

};



static u32 head_hashfn(struct rhashtable *ht,
@@ -222,7 +222,7 @@ static struct bucket_table *rhashtable_last_table(struct rhashtable *ht, 
}



static int rhashtable_rehash_one(struct rhashtable *ht,

				 struct rhash_lock_head **bkt,

				 struct rhash_lock_head __rcu **bkt,

				 unsigned int old_hash)

{

	struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);
@@ -275,7 +275,7 @@ static int rhashtable_rehash_chain(struct rhashtable *ht, 
				    unsigned int old_hash)

{

	struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);

	struct rhash_lock_head **bkt = rht_bucket_var(old_tbl, old_hash);

	struct rhash_lock_head __rcu **bkt = rht_bucket_var(old_tbl, old_hash);

	int err;



	if (!bkt)
@@ -485,7 +485,7 @@ fail: 
}



static void *rhashtable_lookup_one(struct rhashtable *ht,

				   struct rhash_lock_head **bkt,

				   struct rhash_lock_head __rcu **bkt,

				   struct bucket_table *tbl, unsigned int hash,

				   const void *key, struct rhash_head *obj)

{
@@ -535,11 +535,9 @@ static void *rhashtable_lookup_one(struct rhashtable *ht, 
	return ERR_PTR(-ENOENT);

}



static struct bucket_table *rhashtable_insert_one(struct rhashtable *ht,

						  struct rhash_lock_head **bkt,

						  struct bucket_table *tbl,

						  unsigned int hash,

						  struct rhash_head *obj,

static struct bucket_table *rhashtable_insert_one(

	struct rhashtable *ht, struct rhash_lock_head __rcu **bkt,

	struct bucket_table *tbl, unsigned int hash, struct rhash_head *obj,

	void *data)

{

	struct bucket_table *new_tbl;
@@ -591,7 +589,7 @@ static void *rhashtable_try_insert(struct rhashtable *ht, const void *key, 
{

	struct bucket_table *new_tbl;

	struct bucket_table *tbl;

	struct rhash_lock_head **bkt;

	struct rhash_lock_head __rcu **bkt;

	unsigned int hash;

	void *data;
@@ -1173,8 +1171,8 @@ void rhashtable_destroy(struct rhashtable *ht) 
}

EXPORT_SYMBOL_GPL(rhashtable_destroy);



struct rhash_lock_head **__rht_bucket_nested(const struct bucket_table *tbl,

					     unsigned int hash)

struct rhash_lock_head __rcu **__rht_bucket_nested(

	const struct bucket_table *tbl, unsigned int hash)

{

	const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *));

	unsigned int index = hash & ((1 << tbl->nest) - 1);
@@ -1202,10 +1200,10 @@ struct rhash_lock_head **__rht_bucket_nested(const struct bucket_table *tbl, 
}

EXPORT_SYMBOL_GPL(__rht_bucket_nested);



struct rhash_lock_head **rht_bucket_nested(const struct bucket_table *tbl,

					   unsigned int hash)

struct rhash_lock_head __rcu **rht_bucket_nested(

	const struct bucket_table *tbl, unsigned int hash)

{

	static struct rhash_lock_head *rhnull;

	static struct rhash_lock_head __rcu *rhnull;



	if (!rhnull)

		INIT_RHT_NULLS_HEAD(rhnull);
@@ -1213,9 +1211,8 @@ struct rhash_lock_head **rht_bucket_nested(const struct bucket_table *tbl, 
}

EXPORT_SYMBOL_GPL(rht_bucket_nested);



struct rhash_lock_head **rht_bucket_nested_insert(struct rhashtable *ht,

						  struct bucket_table *tbl,

						  unsigned int hash)

struct rhash_lock_head __rcu **rht_bucket_nested_insert(

	struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash)

{

	const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *));

	unsigned int index = hash & ((1 << tbl->nest) - 1);

CRA Git | Maintained and supported by SUSTech CRA and CCSE