Commit 3ab5aee7 authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller
Browse files

net: Convert TCP & DCCP hash tables to use RCU / hlist_nulls 



RCU was added to UDP lookups, using a fast infrastructure :
- sockets kmem_cache use SLAB_DESTROY_BY_RCU and dont pay the
  price of call_rcu() at freeing time.
- hlist_nulls permits to use few memory barriers.

This patch uses same infrastructure for TCP/DCCP established
and timewait sockets.

Thanks to SLAB_DESTROY_BY_RCU, no slowdown for applications
using short lived TCP connections. A followup patch, converting
rwlocks to spinlocks will even speedup this case.

__inet_lookup_established() is pretty fast now we dont have to
dirty a contended cache line (read_lock/read_unlock)

Only established and timewait hashtable are converted to RCU
(bind table and listen table are still using traditional locking)

Signed-off-by: default avatarEric Dumazet <dada1@cosmosbay.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 88ab1932

include/net/inet_hashtables.h

+2 −2
Original line number Diff line number Diff line
@@ -41,8 +41,8 @@ 
 * I'll experiment with dynamic table growth later.

 */

struct inet_ehash_bucket {

	struct hlist_head chain;

	struct hlist_head twchain;

	struct hlist_nulls_head chain;

	struct hlist_nulls_head twchain;

};



/* There are a few simple rules, which allow for local port reuse by

include/net/inet_timewait_sock.h

+5 −5
Original line number Diff line number Diff line
@@ -110,7 +110,7 @@ struct inet_timewait_sock { 
#define tw_state		__tw_common.skc_state

#define tw_reuse		__tw_common.skc_reuse

#define tw_bound_dev_if		__tw_common.skc_bound_dev_if

#define tw_node			__tw_common.skc_node

#define tw_node			__tw_common.skc_nulls_node

#define tw_bind_node		__tw_common.skc_bind_node

#define tw_refcnt		__tw_common.skc_refcnt

#define tw_hash			__tw_common.skc_hash
@@ -137,10 +137,10 @@ struct inet_timewait_sock { 
	struct hlist_node	tw_death_node;

};



static inline void inet_twsk_add_node(struct inet_timewait_sock *tw,

				      struct hlist_head *list)

static inline void inet_twsk_add_node_rcu(struct inet_timewait_sock *tw,

				      struct hlist_nulls_head *list)

{

	hlist_add_head(&tw->tw_node, list);

	hlist_nulls_add_head_rcu(&tw->tw_node, list);

}



static inline void inet_twsk_add_bind_node(struct inet_timewait_sock *tw,
@@ -175,7 +175,7 @@ static inline int inet_twsk_del_dead_node(struct inet_timewait_sock *tw) 
}



#define inet_twsk_for_each(tw, node, head) \

	hlist_for_each_entry(tw, node, head, tw_node)

	hlist_nulls_for_each_entry(tw, node, head, tw_node)



#define inet_twsk_for_each_inmate(tw, node, jail) \

	hlist_for_each_entry(tw, node, jail, tw_death_node)

net/core/sock.c

+3 −1
Original line number Diff line number Diff line
@@ -2082,7 +2082,9 @@ int proto_register(struct proto *prot, int alloc_slab) 
			prot->twsk_prot->twsk_slab =

				kmem_cache_create(timewait_sock_slab_name,

						  prot->twsk_prot->twsk_obj_size,

						  0, SLAB_HWCACHE_ALIGN,

						  0,

						  SLAB_HWCACHE_ALIGN |

							prot->slab_flags,

						  NULL);

			if (prot->twsk_prot->twsk_slab == NULL)

				goto out_free_timewait_sock_slab_name;

net/dccp/ipv4.c

+1 −0
Original line number Diff line number Diff line
@@ -938,6 +938,7 @@ static struct proto dccp_v4_prot = { 
	.orphan_count		= &dccp_orphan_count,

	.max_header		= MAX_DCCP_HEADER,

	.obj_size		= sizeof(struct dccp_sock),

	.slab_flags		= SLAB_DESTROY_BY_RCU,

	.rsk_prot		= &dccp_request_sock_ops,

	.twsk_prot		= &dccp_timewait_sock_ops,

	.h.hashinfo		= &dccp_hashinfo,

net/dccp/ipv6.c

+1 −0
Original line number Diff line number Diff line
@@ -1140,6 +1140,7 @@ static struct proto dccp_v6_prot = { 
	.orphan_count	   = &dccp_orphan_count,

	.max_header	   = MAX_DCCP_HEADER,

	.obj_size	   = sizeof(struct dccp6_sock),

	.slab_flags	   = SLAB_DESTROY_BY_RCU,

	.rsk_prot	   = &dccp6_request_sock_ops,

	.twsk_prot	   = &dccp6_timewait_sock_ops,

	.h.hashinfo	   = &dccp_hashinfo,

CRA Git | Maintained and supported by SUSTech CRA and CCSE