Commit c90d1e4b authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'net-extend-indirect-calls-helper-usage' 



Paolo Abeni says:

====================
net: extend indirect calls helper usage

This series applies the indirect calls helper introduced with commit
283c16a2 ("indirect call wrappers: helpers to speed-up indirect
calls of builtin") to more hooks inside the network stack.

Overall this avoids up to 4 indirect calls for each RX packets,
giving small but measurable gain TCP_RR workloads and 5% under UDP
flood.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents b52d031b 8c3c447b

net/core/dev.c

+4 −2
Original line number Diff line number Diff line
@@ -4987,7 +4987,8 @@ static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc) 


	ret = __netif_receive_skb_core(skb, pfmemalloc, &pt_prev);

	if (pt_prev)

		ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);

		ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb,

					 skb->dev, pt_prev, orig_dev);

	return ret;

}
@@ -5033,7 +5034,8 @@ static inline void __netif_receive_skb_list_ptype(struct list_head *head, 
	else

		list_for_each_entry_safe(skb, next, head, list) {

			skb_list_del_init(skb);

			pt_prev->func(skb, skb->dev, pt_prev, orig_dev);

			INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb,

					   skb->dev, pt_prev, orig_dev);

		}

}

net/ipv4/ip_input.c

+9 −2
Original line number Diff line number Diff line
@@ -130,6 +130,7 @@ 
#include <linux/inetdevice.h>

#include <linux/netdevice.h>

#include <linux/etherdevice.h>

#include <linux/indirect_call_wrapper.h>



#include <net/snmp.h>

#include <net/ip.h>
@@ -188,6 +189,8 @@ bool ip_call_ra_chain(struct sk_buff *skb) 
	return false;

}



INDIRECT_CALLABLE_DECLARE(int udp_rcv(struct sk_buff *));

INDIRECT_CALLABLE_DECLARE(int tcp_v4_rcv(struct sk_buff *));

void ip_protocol_deliver_rcu(struct net *net, struct sk_buff *skb, int protocol)

{

	const struct net_protocol *ipprot;
@@ -205,7 +208,8 @@ resubmit: 
			}

			nf_reset(skb);

		}

		ret = ipprot->handler(skb);

		ret = INDIRECT_CALL_2(ipprot->handler, tcp_v4_rcv, udp_rcv,

				      skb);

		if (ret < 0) {

			protocol = -ret;

			goto resubmit;
@@ -305,6 +309,8 @@ drop: 
	return true;

}



INDIRECT_CALLABLE_DECLARE(int udp_v4_early_demux(struct sk_buff *));

INDIRECT_CALLABLE_DECLARE(int tcp_v4_early_demux(struct sk_buff *));

static int ip_rcv_finish_core(struct net *net, struct sock *sk,

			      struct sk_buff *skb, struct net_device *dev)

{
@@ -322,7 +328,8 @@ static int ip_rcv_finish_core(struct net *net, struct sock *sk, 


		ipprot = rcu_dereference(inet_protos[protocol]);

		if (ipprot && (edemux = READ_ONCE(ipprot->early_demux))) {

			err = edemux(skb);

			err = INDIRECT_CALL_2(edemux, tcp_v4_early_demux,

					      udp_v4_early_demux, skb);

			if (unlikely(err))

				goto drop_error;

			/* must reload iph, skb->head might have changed */

net/ipv6/ip6_input.c

+10 −2
Original line number Diff line number Diff line
@@ -29,6 +29,7 @@ 
#include <linux/icmpv6.h>

#include <linux/mroute6.h>

#include <linux/slab.h>

#include <linux/indirect_call_wrapper.h>



#include <linux/netfilter.h>

#include <linux/netfilter_ipv6.h>
@@ -47,6 +48,8 @@ 
#include <net/inet_ecn.h>

#include <net/dst_metadata.h>



INDIRECT_CALLABLE_DECLARE(void udp_v6_early_demux(struct sk_buff *));

INDIRECT_CALLABLE_DECLARE(void tcp_v6_early_demux(struct sk_buff *));

static void ip6_rcv_finish_core(struct net *net, struct sock *sk,

				struct sk_buff *skb)

{
@@ -57,7 +60,8 @@ static void ip6_rcv_finish_core(struct net *net, struct sock *sk, 


		ipprot = rcu_dereference(inet6_protos[ipv6_hdr(skb)->nexthdr]);

		if (ipprot && (edemux = READ_ONCE(ipprot->early_demux)))

			edemux(skb);

			INDIRECT_CALL_2(edemux, tcp_v6_early_demux,

					udp_v6_early_demux, skb);

	}

	if (!skb_valid_dst(skb))

		ip6_route_input(skb);
@@ -316,6 +320,9 @@ void ipv6_list_rcv(struct list_head *head, struct packet_type *pt, 
	ip6_sublist_rcv(&sublist, curr_dev, curr_net);

}



INDIRECT_CALLABLE_DECLARE(int udpv6_rcv(struct sk_buff *));

INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *));



/*

 *	Deliver the packet to the host

 */
@@ -391,7 +398,8 @@ resubmit_final: 
		    !xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb))

			goto discard;



		ret = ipprot->handler(skb);

		ret = INDIRECT_CALL_2(ipprot->handler, tcp_v6_rcv, udpv6_rcv,

				      skb);

		if (ret > 0) {

			if (ipprot->flags & INET6_PROTO_FINAL) {

				/* Not an extension header, most likely UDP

net/ipv6/tcp_ipv6.c

+3 −2
Original line number Diff line number Diff line
@@ -43,6 +43,7 @@ 
#include <linux/ipv6.h>

#include <linux/icmpv6.h>

#include <linux/random.h>

#include <linux/indirect_call_wrapper.h>



#include <net/tcp.h>

#include <net/ndisc.h>
@@ -1435,7 +1436,7 @@ static void tcp_v6_fill_cb(struct sk_buff *skb, const struct ipv6hdr *hdr, 
			skb->tstamp || skb_hwtstamps(skb)->hwtstamp;

}



static int tcp_v6_rcv(struct sk_buff *skb)

INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)

{

	struct sk_buff *skb_to_free;

	int sdif = inet6_sdif(skb);
@@ -1654,7 +1655,7 @@ do_time_wait: 
	goto discard_it;

}



static void tcp_v6_early_demux(struct sk_buff *skb)

INDIRECT_CALLABLE_SCOPE void tcp_v6_early_demux(struct sk_buff *skb)

{

	const struct ipv6hdr *hdr;

	const struct tcphdr *th;

net/ipv6/udp.c

+3 −2
Original line number Diff line number Diff line
@@ -36,6 +36,7 @@ 
#include <linux/skbuff.h>

#include <linux/slab.h>

#include <linux/uaccess.h>

#include <linux/indirect_call_wrapper.h>



#include <net/addrconf.h>

#include <net/ndisc.h>
@@ -980,7 +981,7 @@ static struct sock *__udp6_lib_demux_lookup(struct net *net, 
	return NULL;

}



static void udp_v6_early_demux(struct sk_buff *skb)

INDIRECT_CALLABLE_SCOPE void udp_v6_early_demux(struct sk_buff *skb)

{

	struct net *net = dev_net(skb->dev);

	const struct udphdr *uh;
@@ -1021,7 +1022,7 @@ static void udp_v6_early_demux(struct sk_buff *skb) 
	}

}



static __inline__ int udpv6_rcv(struct sk_buff *skb)

INDIRECT_CALLABLE_SCOPE int udpv6_rcv(struct sk_buff *skb)

{

	return __udp6_lib_rcv(skb, &udp_table, IPPROTO_UDP);

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE