Commit fd1914b2 authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'tcp-fast-so_reuseport' 



Craig Gallek says:

====================
Faster SO_REUSEPORT for TCP

This patch series complements an earlier series (6a5ef90c)
which added faster SO_REUSEPORT lookup for UDP sockets by
extending the feature to TCP sockets.  It uses the same
array-based data structure which allows for socket selection
after finding the first listening socket that matches an incoming
packet.  Prior to this feature, every socket in the reuseport
group needed to be found and examined before a selection could be
made.

With this series the SO_ATTACH_REUSEPORT_CBPF and
SO_ATTACH_REUSEPORT_EBPF socket options now work for TCP sockets
as well.  The test at the end of the series includes an example of
how to use these options to select a reuseport socket based on the
cpu core id handling the incoming packet.

There are several refactoring patches that precede the feature
implementation.  Only the last two patches in this series
should result in any behavioral changes.

v4
- Fix build issue when compiling IPv6 as a module.  This required
  moving the ipv6_rcv_saddr_equal into an object that is included as a
  built-in object.  I included this change in the second patch which
  adds inet6_hash since that is where ipv6_rcv_saddr_equal will
  later be called from non-module code.

v3:
- Another warning in the first patch caught by a build bot.  Return 0 in
  the no-op UDP hash function.

v2:
- In the first patched I missed a couple of hash functions that should now be
  returning int instead of void.  I missed these the first time through as it
  only generated a warning and not an error :\
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 30c1de08 4b2a6aed

include/linux/tcp.h

+6 −1
Original line number Diff line number Diff line
@@ -29,9 +29,14 @@ static inline struct tcphdr *tcp_hdr(const struct sk_buff *skb) 
	return (struct tcphdr *)skb_transport_header(skb);

}



static inline unsigned int __tcp_hdrlen(const struct tcphdr *th)

{

	return th->doff * 4;

}



static inline unsigned int tcp_hdrlen(const struct sk_buff *skb)

{

	return tcp_hdr(skb)->doff * 4;

	return __tcp_hdrlen(tcp_hdr(skb));

}



static inline struct tcphdr *inner_tcp_hdr(const struct sk_buff *skb)

include/net/addrconf.h

+2 −0
Original line number Diff line number Diff line
@@ -87,6 +87,8 @@ int __ipv6_get_lladdr(struct inet6_dev *idev, struct in6_addr *addr, 
		      u32 banned_flags);

int ipv6_get_lladdr(struct net_device *dev, struct in6_addr *addr,

		    u32 banned_flags);

int ipv4_rcv_saddr_equal(const struct sock *sk, const struct sock *sk2,

			 bool match_wildcard);

int ipv6_rcv_saddr_equal(const struct sock *sk, const struct sock *sk2,

			 bool match_wildcard);

void addrconf_join_solict(struct net_device *dev, const struct in6_addr *addr);

include/net/inet6_hashtables.h

+9 −4
Original line number Diff line number Diff line
@@ -53,6 +53,7 @@ struct sock *__inet6_lookup_established(struct net *net, 


struct sock *inet6_lookup_listener(struct net *net,

				   struct inet_hashinfo *hashinfo,

				   struct sk_buff *skb, int doff,

				   const struct in6_addr *saddr,

				   const __be16 sport,

				   const struct in6_addr *daddr,
@@ -60,6 +61,7 @@ struct sock *inet6_lookup_listener(struct net *net, 


static inline struct sock *__inet6_lookup(struct net *net,

					  struct inet_hashinfo *hashinfo,

					  struct sk_buff *skb, int doff,

					  const struct in6_addr *saddr,

					  const __be16 sport,

					  const struct in6_addr *daddr,
@@ -71,12 +73,12 @@ static inline struct sock *__inet6_lookup(struct net *net, 
	if (sk)

		return sk;



	return inet6_lookup_listener(net, hashinfo, saddr, sport,

	return inet6_lookup_listener(net, hashinfo, skb, doff, saddr, sport,

				     daddr, hnum, dif);

}



static inline struct sock *__inet6_lookup_skb(struct inet_hashinfo *hashinfo,

					      struct sk_buff *skb,

					      struct sk_buff *skb, int doff,

					      const __be16 sport,

					      const __be16 dport,

					      int iif)
@@ -86,16 +88,19 @@ static inline struct sock *__inet6_lookup_skb(struct inet_hashinfo *hashinfo, 
	if (sk)

		return sk;



	return __inet6_lookup(dev_net(skb_dst(skb)->dev), hashinfo,

			      &ipv6_hdr(skb)->saddr, sport,

	return __inet6_lookup(dev_net(skb_dst(skb)->dev), hashinfo, skb,

			      doff, &ipv6_hdr(skb)->saddr, sport,

			      &ipv6_hdr(skb)->daddr, ntohs(dport),

			      iif);

}



struct sock *inet6_lookup(struct net *net, struct inet_hashinfo *hashinfo,

			  struct sk_buff *skb, int doff,

			  const struct in6_addr *saddr, const __be16 sport,

			  const struct in6_addr *daddr, const __be16 dport,

			  const int dif);



int inet6_hash(struct sock *sk);

#endif /* IS_ENABLED(CONFIG_IPV6) */



#define INET6_MATCH(__sk, __net, __saddr, __daddr, __ports, __dif)	\

include/net/inet_hashtables.h

+17 −8
Original line number Diff line number Diff line
@@ -207,12 +207,16 @@ void inet_hashinfo_init(struct inet_hashinfo *h); 


bool inet_ehash_insert(struct sock *sk, struct sock *osk);

bool inet_ehash_nolisten(struct sock *sk, struct sock *osk);

void __inet_hash(struct sock *sk, struct sock *osk);

void inet_hash(struct sock *sk);

int __inet_hash(struct sock *sk, struct sock *osk,

		int (*saddr_same)(const struct sock *sk1,

				  const struct sock *sk2,

				  bool match_wildcard));

int inet_hash(struct sock *sk);

void inet_unhash(struct sock *sk);



struct sock *__inet_lookup_listener(struct net *net,

				    struct inet_hashinfo *hashinfo,

				    struct sk_buff *skb, int doff,

				    const __be32 saddr, const __be16 sport,

				    const __be32 daddr,

				    const unsigned short hnum,
@@ -220,10 +224,11 @@ struct sock *__inet_lookup_listener(struct net *net, 


static inline struct sock *inet_lookup_listener(struct net *net,

		struct inet_hashinfo *hashinfo,

		struct sk_buff *skb, int doff,

		__be32 saddr, __be16 sport,

		__be32 daddr, __be16 dport, int dif)

{

	return __inet_lookup_listener(net, hashinfo, saddr, sport,

	return __inet_lookup_listener(net, hashinfo, skb, doff, saddr, sport,

				      daddr, ntohs(dport), dif);

}
@@ -299,6 +304,7 @@ static inline struct sock * 


static inline struct sock *__inet_lookup(struct net *net,

					 struct inet_hashinfo *hashinfo,

					 struct sk_buff *skb, int doff,

					 const __be32 saddr, const __be16 sport,

					 const __be32 daddr, const __be16 dport,

					 const int dif)
@@ -307,12 +313,13 @@ static inline struct sock *__inet_lookup(struct net *net, 
	struct sock *sk = __inet_lookup_established(net, hashinfo,

				saddr, sport, daddr, hnum, dif);



	return sk ? : __inet_lookup_listener(net, hashinfo, saddr, sport,

					     daddr, hnum, dif);

	return sk ? : __inet_lookup_listener(net, hashinfo, skb, doff, saddr,

					     sport, daddr, hnum, dif);

}



static inline struct sock *inet_lookup(struct net *net,

				       struct inet_hashinfo *hashinfo,

				       struct sk_buff *skb, int doff,

				       const __be32 saddr, const __be16 sport,

				       const __be32 daddr, const __be16 dport,

				       const int dif)
@@ -320,7 +327,8 @@ static inline struct sock *inet_lookup(struct net *net, 
	struct sock *sk;



	local_bh_disable();

	sk = __inet_lookup(net, hashinfo, saddr, sport, daddr, dport, dif);

	sk = __inet_lookup(net, hashinfo, skb, doff, saddr, sport, daddr,

			   dport, dif);

	local_bh_enable();



	return sk;
@@ -328,6 +336,7 @@ static inline struct sock *inet_lookup(struct net *net, 


static inline struct sock *__inet_lookup_skb(struct inet_hashinfo *hashinfo,

					     struct sk_buff *skb,

					     int doff,

					     const __be16 sport,

					     const __be16 dport)

{
@@ -337,8 +346,8 @@ static inline struct sock *__inet_lookup_skb(struct inet_hashinfo *hashinfo, 
	if (sk)

		return sk;

	else

		return __inet_lookup(dev_net(skb_dst(skb)->dev), hashinfo,

				     iph->saddr, sport,

		return __inet_lookup(dev_net(skb_dst(skb)->dev), hashinfo, skb,

				     doff, iph->saddr, sport,

				     iph->daddr, dport, inet_iif(skb));

}

include/net/phonet/phonet.h

+1 −1
Original line number Diff line number Diff line
@@ -51,7 +51,7 @@ void pn_sock_init(void); 
struct sock *pn_find_sock_by_sa(struct net *net, const struct sockaddr_pn *sa);

void pn_deliver_sock_broadcast(struct net *net, struct sk_buff *skb);

void phonet_get_local_port_range(int *min, int *max);

void pn_sock_hash(struct sock *sk);

int pn_sock_hash(struct sock *sk);

void pn_sock_unhash(struct sock *sk);

int pn_sock_get_port(struct sock *sk, unsigned short sport);

CRA Git | Maintained and supported by SUSTech CRA and CCSE