Commit 6015c71e authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller
Browse files

tcp: md5: add tcp_md5_needed jump label 



Most linux hosts never setup TCP MD5 keys. We can avoid a
cache line miss (accessing tp->md5ig_info) on RX and TX
using a jump label.

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 2f695553

include/net/tcp.h

+15 −3
Original line number Original line Diff line number Diff line
@@ -1555,9 +1555,21 @@ struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk, 
					 const struct sock *addr_sk);

					 const struct sock *addr_sk);





#ifdef CONFIG_TCP_MD5SIG

#ifdef CONFIG_TCP_MD5SIG

struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk,

#include <linux/jump_label.h>

extern struct static_key tcp_md5_needed;

struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk,

					   const union tcp_md5_addr *addr,

					   const union tcp_md5_addr *addr,

					   int family);

					   int family);

static inline struct tcp_md5sig_key *

tcp_md5_do_lookup(const struct sock *sk,

		  const union tcp_md5_addr *addr,

		  int family)

{

	if (!static_key_false(&tcp_md5_needed))

		return NULL;

	return __tcp_md5_do_lookup(sk, addr, family);

}



#define tcp_twsk_md5_key(twsk)	((twsk)->tw_md5_key)

#define tcp_twsk_md5_key(twsk)	((twsk)->tw_md5_key)

#else

#else

static inline struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk,

static inline struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk,

net/ipv4/tcp.c

+4 −1
Original line number Original line Diff line number Diff line
@@ -3656,8 +3656,11 @@ bool tcp_alloc_md5sig_pool(void) 
	if (unlikely(!tcp_md5sig_pool_populated)) {

	if (unlikely(!tcp_md5sig_pool_populated)) {

		mutex_lock(&tcp_md5sig_mutex);

		mutex_lock(&tcp_md5sig_mutex);





		if (!tcp_md5sig_pool_populated)

		if (!tcp_md5sig_pool_populated) {

			__tcp_alloc_md5sig_pool();

			__tcp_alloc_md5sig_pool();

			if (tcp_md5sig_pool_populated)

				static_key_slow_inc(&tcp_md5_needed);

		}





		mutex_unlock(&tcp_md5sig_mutex);

		mutex_unlock(&tcp_md5sig_mutex);

	}

	}

net/ipv4/tcp_ipv4.c

+7 −4
Original line number Original line Diff line number Diff line
@@ -970,8 +970,11 @@ static void tcp_v4_reqsk_destructor(struct request_sock *req) 
 * We need to maintain these in the sk structure.

 * We need to maintain these in the sk structure.

 */

 */





struct static_key tcp_md5_needed __read_mostly;

EXPORT_SYMBOL(tcp_md5_needed);



/* Find the Key structure for an address.  */

/* Find the Key structure for an address.  */

struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk,

struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk,

					   const union tcp_md5_addr *addr,

					   const union tcp_md5_addr *addr,

					   int family)

					   int family)

{

{
@@ -1011,7 +1014,7 @@ struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk, 
	}

	}

	return best_match;

	return best_match;

}

}

EXPORT_SYMBOL(tcp_md5_do_lookup);

EXPORT_SYMBOL(__tcp_md5_do_lookup);





static struct tcp_md5sig_key *tcp_md5_do_lookup_exact(const struct sock *sk,

static struct tcp_md5sig_key *tcp_md5_do_lookup_exact(const struct sock *sk,

						      const union tcp_md5_addr *addr,

						      const union tcp_md5_addr *addr,

net/ipv4/tcp_output.c

+4 −2
Original line number Original line Diff line number Diff line
@@ -594,7 +594,8 @@ static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb, 




	*md5 = NULL;

	*md5 = NULL;

#ifdef CONFIG_TCP_MD5SIG

#ifdef CONFIG_TCP_MD5SIG

	if (unlikely(rcu_access_pointer(tp->md5sig_info))) {

	if (static_key_false(&tcp_md5_needed) &&

	    rcu_access_pointer(tp->md5sig_info)) {

		*md5 = tp->af_specific->md5_lookup(sk, sk);

		*md5 = tp->af_specific->md5_lookup(sk, sk);

		if (*md5) {

		if (*md5) {

			opts->options |= OPTION_MD5;

			opts->options |= OPTION_MD5;
@@ -730,7 +731,8 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb 




	*md5 = NULL;

	*md5 = NULL;

#ifdef CONFIG_TCP_MD5SIG

#ifdef CONFIG_TCP_MD5SIG

	if (unlikely(rcu_access_pointer(tp->md5sig_info))) {

	if (static_key_false(&tcp_md5_needed) &&

	    rcu_access_pointer(tp->md5sig_info)) {

		*md5 = tp->af_specific->md5_lookup(sk, sk);

		*md5 = tp->af_specific->md5_lookup(sk, sk);

		if (*md5) {

		if (*md5) {

			opts->options |= OPTION_MD5;

			opts->options |= OPTION_MD5;

CRA Git | Maintained and supported by SUSTech CRA and CCSE