Commit 55cced4f authored by Brian Vazquez's avatar Brian Vazquez Committed by David S. Miller
Browse files

ipv6: fib6: avoid indirect calls from fib6_rule_lookup 

It was reported that a considerable amount of cycles were spent on the
expensive indirect calls on fib6_rule_lookup. This patch introduces an
inline helper called pol_route_func that uses the indirect_call_wrappers
to avoid the indirect calls.

This patch saves around 50ns per call.

Performance was measured on the receiver by checking the amount of
syncookies that server was able to generate under a synflood load.

Traffic was generated using trafgen[1] which was pushing around 1Mpps on
a single queue. Receiver was using only one rx queue which help to
create a bottle neck and make the experiment rx-bounded.

These are the syncookies generated over 10s from the different runs:

Whithout the patch:
TcpExtSyncookiesSent            3553749            0.0
TcpExtSyncookiesSent            3550895            0.0
TcpExtSyncookiesSent            3553845            0.0
TcpExtSyncookiesSent            3541050            0.0
TcpExtSyncookiesSent            3539921            0.0
TcpExtSyncookiesSent            3557659            0.0
TcpExtSyncookiesSent            3526812            0.0
TcpExtSyncookiesSent            3536121            0.0
TcpExtSyncookiesSent            3529963            0.0
TcpExtSyncookiesSent            3536319            0.0

With the patch:
TcpExtSyncookiesSent            3611786            0.0
TcpExtSyncookiesSent            3596682            0.0
TcpExtSyncookiesSent            3606878            0.0
TcpExtSyncookiesSent            3599564            0.0
TcpExtSyncookiesSent            3601304            0.0
TcpExtSyncookiesSent            3609249            0.0
TcpExtSyncookiesSent            3617437            0.0
TcpExtSyncookiesSent            3608765            0.0
TcpExtSyncookiesSent            3620205            0.0
TcpExtSyncookiesSent            3601895            0.0

Without the patch the average is 354263 pkt/s or 2822 ns/pkt and with
the patch the average is 360738 pkt/s or 2772 ns/pkt which gives an
estimate of 50 ns per packet.

[1] http://netsniff-ng.org/



Changelog since v1:
 - Change ordering in the ICW (Paolo Abeni)

Cc: Luigi Rizzo <lrizzo@google.com>
Cc: Paolo Abeni <pabeni@redhat.com>
Reported-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarBrian Vazquez <brianvv@google.com>
Acked-by: default avatarPaolo Abeni <pabeni@redhat.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent e678e9dd

include/net/ip6_fib.h

+36 −0
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#include <net/netlink.h>

#include <net/inetpeer.h>

#include <net/fib_notifier.h>

#include <linux/indirect_call_wrapper.h>



#ifdef CONFIG_IPV6_MULTIPLE_TABLES

#define FIB6_TABLE_HASHSZ 256
@@ -552,6 +553,41 @@ struct bpf_iter__ipv6_route { 
};

#endif



INDIRECT_CALLABLE_DECLARE(struct rt6_info *ip6_pol_route_output(struct net *net,

					     struct fib6_table *table,

					     struct flowi6 *fl6,

					     const struct sk_buff *skb,

					     int flags));

INDIRECT_CALLABLE_DECLARE(struct rt6_info *ip6_pol_route_input(struct net *net,

					     struct fib6_table *table,

					     struct flowi6 *fl6,

					     const struct sk_buff *skb,

					     int flags));

INDIRECT_CALLABLE_DECLARE(struct rt6_info *__ip6_route_redirect(struct net *net,

					     struct fib6_table *table,

					     struct flowi6 *fl6,

					     const struct sk_buff *skb,

					     int flags));

INDIRECT_CALLABLE_DECLARE(struct rt6_info *ip6_pol_route_lookup(struct net *net,

					     struct fib6_table *table,

					     struct flowi6 *fl6,

					     const struct sk_buff *skb,

					     int flags));

static inline struct rt6_info *pol_lookup_func(pol_lookup_t lookup,

						struct net *net,

						struct fib6_table *table,

						struct flowi6 *fl6,

						const struct sk_buff *skb,

						int flags)

{

	return INDIRECT_CALL_4(lookup,

			       ip6_pol_route_output,

			       ip6_pol_route_input,

			       ip6_pol_route_lookup,

			       __ip6_route_redirect,

			       net, table, fl6, skb, flags);

}



#ifdef CONFIG_IPV6_MULTIPLE_TABLES

static inline bool fib6_has_custom_rules(const struct net *net)

{

net/ipv6/fib6_rules.c

+6 −3
Original line number Diff line number Diff line
@@ -111,11 +111,13 @@ struct dst_entry *fib6_rule_lookup(struct net *net, struct flowi6 *fl6, 
	} else {

		struct rt6_info *rt;



		rt = lookup(net, net->ipv6.fib6_local_tbl, fl6, skb, flags);

		rt = pol_lookup_func(lookup,

			     net, net->ipv6.fib6_local_tbl, fl6, skb, flags);

		if (rt != net->ipv6.ip6_null_entry && rt->dst.error != -EAGAIN)

			return &rt->dst;

		ip6_rt_put_flags(rt, flags);

		rt = lookup(net, net->ipv6.fib6_main_tbl, fl6, skb, flags);

		rt = pol_lookup_func(lookup,

			     net, net->ipv6.fib6_main_tbl, fl6, skb, flags);

		if (rt->dst.error != -EAGAIN)

			return &rt->dst;

		ip6_rt_put_flags(rt, flags);
@@ -226,7 +228,8 @@ static int __fib6_rule_action(struct fib_rule *rule, struct flowi *flp, 
		goto out;

	}



	rt = lookup(net, table, flp6, arg->lookup_data, flags);

	rt = pol_lookup_func(lookup,

			     net, table, flp6, arg->lookup_data, flags);

	if (rt != net->ipv6.ip6_null_entry) {

		err = fib6_rule_saddr(net, rule, flags, flp6,

				      ip6_dst_idev(&rt->dst)->dev);

net/ipv6/ip6_fib.c

+2 −1
Original line number Diff line number Diff line
@@ -314,7 +314,8 @@ struct dst_entry *fib6_rule_lookup(struct net *net, struct flowi6 *fl6, 
{

	struct rt6_info *rt;



	rt = lookup(net, net->ipv6.fib6_main_tbl, fl6, skb, flags);

	rt = pol_lookup_func(lookup,

			net, net->ipv6.fib6_main_tbl, fl6, skb, flags);

	if (rt->dst.error == -EAGAIN) {

		ip6_rt_put_flags(rt, flags);

		rt = net->ipv6.ip6_null_entry;

net/ipv6/route.c

+4 −4
Original line number Diff line number Diff line
@@ -1207,7 +1207,7 @@ fallback: 
	return nrt;

}



static struct rt6_info *ip6_pol_route_lookup(struct net *net,

INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_lookup(struct net *net,

					     struct fib6_table *table,

					     struct flowi6 *fl6,

					     const struct sk_buff *skb,
@@ -2274,7 +2274,7 @@ out: 
}

EXPORT_SYMBOL_GPL(ip6_pol_route);



static struct rt6_info *ip6_pol_route_input(struct net *net,

INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_input(struct net *net,

					    struct fib6_table *table,

					    struct flowi6 *fl6,

					    const struct sk_buff *skb,
@@ -2465,7 +2465,7 @@ void ip6_route_input(struct sk_buff *skb) 
						      &fl6, skb, flags));

}



static struct rt6_info *ip6_pol_route_output(struct net *net,

INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_output(struct net *net,

					     struct fib6_table *table,

					     struct flowi6 *fl6,

					     const struct sk_buff *skb,
@@ -2912,7 +2912,7 @@ struct ip6rd_flowi { 
	struct in6_addr gateway;

};



static struct rt6_info *__ip6_route_redirect(struct net *net,

INDIRECT_CALLABLE_SCOPE struct rt6_info *__ip6_route_redirect(struct net *net,

					     struct fib6_table *table,

					     struct flowi6 *fl6,

					     const struct sk_buff *skb,

CRA Git | Maintained and supported by SUSTech CRA and CCSE