Commit efe4208f authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller
Browse files

ipv6: make lookups simpler and faster 



TCP listener refactoring, part 4 :

To speed up inet lookups, we moved IPv4 addresses from inet to struct
sock_common

Now is time to do the same for IPv6, because it permits us to have fast
lookups for all kind of sockets, including upcoming SYN_RECV.

Getting IPv6 addresses in TCP lookups currently requires two extra cache
lines, plus a dereference (and memory stall).

inet6_sk(sk) does the dereference of inet_sk(__sk)->pinet6

This patch is way bigger than its IPv4 counter part, because for IPv4,
we could add aliases (inet_daddr, inet_rcv_saddr), while on IPv6,
it's not doable easily.

inet6_sk(sk)->daddr becomes sk->sk_v6_daddr
inet6_sk(sk)->rcv_saddr becomes sk->sk_v6_rcv_saddr

And timewait socket also have tw->tw_v6_daddr & tw->tw_v6_rcv_saddr
at the same offset.

We get rid of INET6_TW_MATCH() as INET6_MATCH() is now the generic
macro.

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 05dbc7b5

include/linux/ipv6.h

+6 −40
Original line number Diff line number Diff line
@@ -141,8 +141,6 @@ struct ipv6_fl_socklist; 
 */

struct ipv6_pinfo {

	struct in6_addr 	saddr;

	struct in6_addr 	rcv_saddr;

	struct in6_addr		daddr;

	struct in6_pktinfo	sticky_pktinfo;

	const struct in6_addr		*daddr_cache;

#ifdef CONFIG_IPV6_SUBTREES
@@ -256,22 +254,10 @@ struct tcp6_sock { 


extern int inet6_sk_rebuild_header(struct sock *sk);



struct inet6_timewait_sock {

	struct in6_addr tw_v6_daddr;

	struct in6_addr	tw_v6_rcv_saddr;

};



struct tcp6_timewait_sock {

	struct tcp_timewait_sock   tcp6tw_tcp;

	struct inet6_timewait_sock tcp6tw_inet6;

};



static inline struct inet6_timewait_sock *inet6_twsk(const struct sock *sk)

{

	return (struct inet6_timewait_sock *)(((u8 *)sk) +

					      inet_twsk(sk)->tw_ipv6_offset);

}



#if IS_ENABLED(CONFIG_IPV6)

static inline struct ipv6_pinfo * inet6_sk(const struct sock *__sk)

{
@@ -321,21 +307,11 @@ static inline void inet_sk_copy_descendant(struct sock *sk_to, 
#define __ipv6_only_sock(sk)	(inet6_sk(sk)->ipv6only)

#define ipv6_only_sock(sk)	((sk)->sk_family == PF_INET6 && __ipv6_only_sock(sk))



static inline u16 inet6_tw_offset(const struct proto *prot)

static inline const struct in6_addr *inet6_rcv_saddr(const struct sock *sk)

{

	return prot->twsk_prot->twsk_obj_size -

			sizeof(struct inet6_timewait_sock);

}



static inline struct in6_addr *__inet6_rcv_saddr(const struct sock *sk)

{

	return likely(sk->sk_state != TCP_TIME_WAIT) ?

		&inet6_sk(sk)->rcv_saddr : &inet6_twsk(sk)->tw_v6_rcv_saddr;

}



static inline struct in6_addr *inet6_rcv_saddr(const struct sock *sk)

{

	return sk->sk_family == AF_INET6 ? __inet6_rcv_saddr(sk) : NULL;

	if (sk->sk_family == AF_INET6)

		return &sk->sk_v6_rcv_saddr;

	return NULL;

}



static inline int inet_v6_ipv6only(const struct sock *sk)
@@ -363,7 +339,6 @@ static inline struct raw6_sock *raw6_sk(const struct sock *sk) 
	return NULL;

}



#define __inet6_rcv_saddr(__sk)	NULL

#define inet6_rcv_saddr(__sk)	NULL

#define tcp_twsk_ipv6only(__sk)		0

#define inet_v6_ipv6only(__sk)		0
@@ -372,17 +347,8 @@ static inline struct raw6_sock *raw6_sk(const struct sock *sk) 
#define INET6_MATCH(__sk, __net, __saddr, __daddr, __ports, __dif)	\

	(((__sk)->sk_portpair == (__ports))			&&	\

	 ((__sk)->sk_family == AF_INET6)			&&	\

	 ipv6_addr_equal(&inet6_sk(__sk)->daddr, (__saddr))	&&	\

	 ipv6_addr_equal(&inet6_sk(__sk)->rcv_saddr, (__daddr))	&&	\

	 (!(__sk)->sk_bound_dev_if	||				\

	   ((__sk)->sk_bound_dev_if == (__dif))) 		&&	\

	 net_eq(sock_net(__sk), (__net)))



#define INET6_TW_MATCH(__sk, __net, __saddr, __daddr, __ports, __dif)	   \

	(((__sk)->sk_portpair == (__ports))				&& \

	 ((__sk)->sk_family == AF_INET6)				&& \

	 ipv6_addr_equal(&inet6_twsk(__sk)->tw_v6_daddr, (__saddr))	&& \

	 ipv6_addr_equal(&inet6_twsk(__sk)->tw_v6_rcv_saddr, (__daddr)) && \

	 ipv6_addr_equal(&(__sk)->sk_v6_daddr, (__saddr))		&&	\

	 ipv6_addr_equal(&(__sk)->sk_v6_rcv_saddr, (__daddr))	&&	\

	 (!(__sk)->sk_bound_dev_if	||				\

	   ((__sk)->sk_bound_dev_if == (__dif))) 		&&	\

	 net_eq(sock_net(__sk), (__net)))

include/net/inet6_hashtables.h

+2 −3
Original line number Diff line number Diff line
@@ -43,9 +43,8 @@ static inline unsigned int inet6_ehashfn(struct net *net, 
static inline int inet6_sk_ehashfn(const struct sock *sk)

{

	const struct inet_sock *inet = inet_sk(sk);

	const struct ipv6_pinfo *np = inet6_sk(sk);

	const struct in6_addr *laddr = &np->rcv_saddr;

	const struct in6_addr *faddr = &np->daddr;

	const struct in6_addr *laddr = &sk->sk_v6_rcv_saddr;

	const struct in6_addr *faddr = &sk->sk_v6_daddr;

	const __u16 lport = inet->inet_num;

	const __be16 fport = inet->inet_dport;

	struct net *net = sock_net(sk);

include/net/inet_timewait_sock.h

+3 −1
Original line number Diff line number Diff line
@@ -116,7 +116,9 @@ struct inet_timewait_sock { 
#define tw_prot			__tw_common.skc_prot

#define tw_net			__tw_common.skc_net

#define tw_daddr        	__tw_common.skc_daddr

#define tw_v6_daddr		__tw_common.skc_v6_daddr

#define tw_rcv_saddr    	__tw_common.skc_rcv_saddr

#define tw_v6_rcv_saddr    	__tw_common.skc_v6_rcv_saddr

#define tw_dport		__tw_common.skc_dport

#define tw_num			__tw_common.skc_num
@@ -133,7 +135,7 @@ struct inet_timewait_sock { 
				tw_transparent  : 1,

				tw_pad		: 6,	/* 6 bits hole */

				tw_tos		: 8,

				tw_ipv6_offset  : 16;

				tw_pad2		: 16 /* 16 bits hole */

	kmemcheck_bitfield_end(flags);

	u32			tw_ttd;

	struct inet_bind_bucket	*tw_tb;

include/net/ip.h

+1 −1
Original line number Diff line number Diff line
@@ -374,7 +374,7 @@ static __inline__ void inet_reset_saddr(struct sock *sk) 
		struct ipv6_pinfo *np = inet6_sk(sk);



		memset(&np->saddr, 0, sizeof(np->saddr));

		memset(&np->rcv_saddr, 0, sizeof(np->rcv_saddr));

		memset(&sk->sk_v6_rcv_saddr, 0, sizeof(sk->sk_v6_rcv_saddr));

	}

#endif

}

include/net/ip6_checksum.h

+1 −1
Original line number Diff line number Diff line
@@ -70,7 +70,7 @@ static inline void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb) 
{

	struct ipv6_pinfo *np = inet6_sk(sk);



	__tcp_v6_send_check(skb, &np->saddr, &np->daddr);

	__tcp_v6_send_check(skb, &np->saddr, &sk->sk_v6_daddr);

}



int udp6_csum_init(struct sk_buff *skb, struct udphdr *uh, int proto);

CRA Git | Maintained and supported by SUSTech CRA and CCSE