Commit dd2e0b86 authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller
Browse files

tcp: remove indirect calls for icsk->icsk_af_ops->send_check



Mitigate RETPOLINE costs in __tcp_transmit_skb()
by using INDIRECT_CALL_INET() wrapper.

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 05e22e83
Loading
Loading
Loading
Loading
+0 −9
Original line number Original line Diff line number Diff line
@@ -85,15 +85,6 @@ static inline void tcp_v6_gso_csum_prep(struct sk_buff *skb)
	th->check = ~tcp_v6_check(0, &ipv6h->saddr, &ipv6h->daddr, 0);
	th->check = ~tcp_v6_check(0, &ipv6h->saddr, &ipv6h->daddr, 0);
}
}


#if IS_ENABLED(CONFIG_IPV6)
static inline void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)
{
	struct ipv6_pinfo *np = inet6_sk(sk);

	__tcp_v6_send_check(skb, &np->saddr, &sk->sk_v6_daddr);
}
#endif

static inline __sum16 udp_v6_check(int len,
static inline __sum16 udp_v6_check(int len,
				   const struct in6_addr *saddr,
				   const struct in6_addr *saddr,
				   const struct in6_addr *daddr,
				   const struct in6_addr *daddr,
+3 −0
Original line number Original line Diff line number Diff line
@@ -932,6 +932,9 @@ static inline int tcp_v6_sdif(const struct sk_buff *skb)
#endif
#endif
	return 0;
	return 0;
}
}

INDIRECT_CALLABLE_DECLARE(void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb));

#endif
#endif


static inline bool inet_exact_dif_match(struct net *net, struct sk_buff *skb)
static inline bool inet_exact_dif_match(struct net *net, struct sk_buff *skb)
+4 −1
Original line number Original line Diff line number Diff line
@@ -1066,6 +1066,7 @@ static void tcp_update_skb_after_send(struct sock *sk, struct sk_buff *skb,


INDIRECT_CALLABLE_DECLARE(int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl));
INDIRECT_CALLABLE_DECLARE(int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl));
INDIRECT_CALLABLE_DECLARE(int inet6_csk_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl));
INDIRECT_CALLABLE_DECLARE(int inet6_csk_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl));
INDIRECT_CALLABLE_DECLARE(void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb));


/* This routine actually transmits TCP packets queued in by
/* This routine actually transmits TCP packets queued in by
 * tcp_do_sendmsg().  This is used by both the initial
 * tcp_do_sendmsg().  This is used by both the initial
@@ -1210,7 +1211,9 @@ static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb,
	}
	}
#endif
#endif


	icsk->icsk_af_ops->send_check(sk, skb);
	INDIRECT_CALL_INET(icsk->icsk_af_ops->send_check,
			   tcp_v6_send_check, tcp_v4_send_check,
			   sk, skb);


	if (likely(tcb->tcp_flags & TCPHDR_ACK))
	if (likely(tcb->tcp_flags & TCPHDR_ACK))
		tcp_event_ack_sent(sk, tcp_skb_pcount(skb), rcv_nxt);
		tcp_event_ack_sent(sk, tcp_skb_pcount(skb), rcv_nxt);
+7 −0
Original line number Original line Diff line number Diff line
@@ -1811,6 +1811,13 @@ static struct timewait_sock_ops tcp6_timewait_sock_ops = {
	.twsk_destructor = tcp_twsk_destructor,
	.twsk_destructor = tcp_twsk_destructor,
};
};


INDIRECT_CALLABLE_SCOPE void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)
{
	struct ipv6_pinfo *np = inet6_sk(sk);

	__tcp_v6_send_check(skb, &np->saddr, &sk->sk_v6_daddr);
}

const struct inet_connection_sock_af_ops ipv6_specific = {
const struct inet_connection_sock_af_ops ipv6_specific = {
	.queue_xmit	   = inet6_csk_xmit,
	.queue_xmit	   = inet6_csk_xmit,
	.send_check	   = tcp_v6_send_check,
	.send_check	   = tcp_v6_send_check,