Commit 8e5eb54d authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller
Browse files

net: align sk_refcnt on 128 bytes boundary 



sk->sk_refcnt is dirtied for every TCP/UDP incoming packet.
This is a performance issue if multiple cpus hit a common socket,
or multiple sockets are chained due to SO_REUSEPORT.

By moving sk_refcnt 8 bytes further, first 128 bytes of sockets
are mostly read. As they contain the lookup keys, this has
a considerable performance impact, as cpus can cache them.

These 8 bytes are not wasted, we use them as a place holder
for various fields, depending on the socket type.

Tested:
 SYN flood hitting a 16 RX queues NIC.
 TCP listener using 16 sockets and SO_REUSEPORT
 and SO_INCOMING_CPU for proper siloing.

 Could process 6.0 Mpps SYN instead of 4.2 Mpps

 Kernel profile looked like :
    11.68%  [kernel]  [k] sha_transform
     6.51%  [kernel]  [k] __inet_lookup_listener
     5.07%  [kernel]  [k] __inet_lookup_established
     4.15%  [kernel]  [k] memcpy_erms
     3.46%  [kernel]  [k] ipt_do_table
     2.74%  [kernel]  [k] fib_table_lookup
     2.54%  [kernel]  [k] tcp_make_synack
     2.34%  [kernel]  [k] tcp_conn_request
     2.05%  [kernel]  [k] __netif_receive_skb_core
     2.03%  [kernel]  [k] kmem_cache_alloc

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 70da268b

include/net/inet_timewait_sock.h

+1 −1
Original line number Diff line number Diff line
@@ -70,6 +70,7 @@ struct inet_timewait_sock { 
#define tw_dport		__tw_common.skc_dport

#define tw_num			__tw_common.skc_num

#define tw_cookie		__tw_common.skc_cookie

#define tw_dr			__tw_common.skc_tw_dr



	int			tw_timeout;

	volatile unsigned char	tw_substate;
@@ -88,7 +89,6 @@ struct inet_timewait_sock { 
	kmemcheck_bitfield_end(flags);

	struct timer_list	tw_timer;

	struct inet_bind_bucket	*tw_tb;

	struct inet_timewait_death_row *tw_dr;

};

#define tw_tclass tw_tos

include/net/request_sock.h

+1 −1
Original line number Diff line number Diff line
@@ -50,9 +50,9 @@ struct request_sock { 
	struct sock_common		__req_common;

#define rsk_refcnt			__req_common.skc_refcnt

#define rsk_hash			__req_common.skc_hash

#define rsk_listener			__req_common.skc_listener



	struct request_sock		*dl_next;

	struct sock			*rsk_listener;

	u16				mss;

	u8				num_retrans; /* number of retransmits */

	u8				cookie_ts:1; /* syncookie: encode tcpopts in timestamp */

include/net/sock.h

+14 −3
Original line number Diff line number Diff line
@@ -150,6 +150,9 @@ typedef __u64 __bitwise __addrpair; 
 *	@skc_node: main hash linkage for various protocol lookup tables

 *	@skc_nulls_node: main hash linkage for TCP/UDP/UDP-Lite protocol

 *	@skc_tx_queue_mapping: tx queue number for this connection

 *	@skc_flags: place holder for sk_flags

 *		%SO_LINGER (l_onoff), %SO_BROADCAST, %SO_KEEPALIVE,

 *		%SO_OOBINLINE settings, %SO_TIMESTAMPING settings

 *	@skc_incoming_cpu: record/match cpu processing incoming packets

 *	@skc_refcnt: reference count

 *
@@ -201,6 +204,16 @@ struct sock_common { 


	atomic64_t		skc_cookie;



	/* following fields are padding to force

	 * offset(struct sock, sk_refcnt) == 128 on 64bit arches

	 * assuming IPV6 is enabled. We use this padding differently

	 * for different kind of 'sockets'

	 */

	union {

		unsigned long	skc_flags;

		struct sock	*skc_listener; /* request_sock */

		struct inet_timewait_death_row *skc_tw_dr; /* inet_timewait_sock */

	};

	/*

	 * fields between dontcopy_begin/dontcopy_end

	 * are not copied in sock_copy()
@@ -246,8 +259,6 @@ struct cg_proto; 
  *	@sk_pacing_rate: Pacing rate (if supported by transport/packet scheduler)

  *	@sk_max_pacing_rate: Maximum pacing rate (%SO_MAX_PACING_RATE)

  *	@sk_sndbuf: size of send buffer in bytes

  *	@sk_flags: %SO_LINGER (l_onoff), %SO_BROADCAST, %SO_KEEPALIVE,

  *		   %SO_OOBINLINE settings, %SO_TIMESTAMPING settings

  *	@sk_no_check_tx: %SO_NO_CHECK setting, set checksum in TX packets

  *	@sk_no_check_rx: allow zero checksum in RX packets

  *	@sk_route_caps: route capabilities (e.g. %NETIF_F_TSO)
@@ -334,6 +345,7 @@ struct sock { 
#define sk_v6_rcv_saddr	__sk_common.skc_v6_rcv_saddr

#define sk_cookie		__sk_common.skc_cookie

#define sk_incoming_cpu		__sk_common.skc_incoming_cpu

#define sk_flags		__sk_common.skc_flags



	socket_lock_t		sk_lock;

	struct sk_buff_head	sk_receive_queue;
@@ -371,7 +383,6 @@ struct sock { 
#ifdef CONFIG_XFRM

	struct xfrm_policy	*sk_policy[2];

#endif

	unsigned long 		sk_flags;

	struct dst_entry	*sk_rx_dst;

	struct dst_entry __rcu	*sk_dst_cache;

	spinlock_t		sk_dst_lock;

CRA Git | Maintained and supported by SUSTech CRA and CCSE