Commit 13bfff25 authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller
Browse files

net: rfs: add a jump label



RFS is not commonly used, so add a jump label to avoid some conditionals
in fast path.

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Cc: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent c9fba3ed
Loading
Loading
Loading
Loading
+1 −0
Original line number Original line Diff line number Diff line
@@ -192,6 +192,7 @@ struct net_device_stats {
#ifdef CONFIG_RPS
#ifdef CONFIG_RPS
#include <linux/static_key.h>
#include <linux/static_key.h>
extern struct static_key rps_needed;
extern struct static_key rps_needed;
extern struct static_key rfs_needed;
#endif
#endif


struct neighbour;
struct neighbour;
+14 −11
Original line number Original line Diff line number Diff line
@@ -913,7 +913,9 @@ static inline void sock_rps_record_flow_hash(__u32 hash)
static inline void sock_rps_record_flow(const struct sock *sk)
static inline void sock_rps_record_flow(const struct sock *sk)
{
{
#ifdef CONFIG_RPS
#ifdef CONFIG_RPS
	/* Reading sk->sk_rxhash might incur an expensive cache line miss.
	if (static_key_false(&rfs_needed)) {
		/* Reading sk->sk_rxhash might incur an expensive cache line
		 * miss.
		 *
		 *
		 * TCP_ESTABLISHED does cover almost all states where RFS
		 * TCP_ESTABLISHED does cover almost all states where RFS
		 * might be useful, and is cheaper [1] than testing :
		 * might be useful, and is cheaper [1] than testing :
@@ -924,6 +926,7 @@ static inline void sock_rps_record_flow(const struct sock *sk)
		 */
		 */
		if (sk->sk_state == TCP_ESTABLISHED)
		if (sk->sk_state == TCP_ESTABLISHED)
			sock_rps_record_flow_hash(sk->sk_rxhash);
			sock_rps_record_flow_hash(sk->sk_rxhash);
	}
#endif
#endif
}
}


+2 −0
Original line number Original line Diff line number Diff line
@@ -3447,6 +3447,8 @@ EXPORT_SYMBOL(rps_cpu_mask);


struct static_key rps_needed __read_mostly;
struct static_key rps_needed __read_mostly;
EXPORT_SYMBOL(rps_needed);
EXPORT_SYMBOL(rps_needed);
struct static_key rfs_needed __read_mostly;
EXPORT_SYMBOL(rfs_needed);


static struct rps_dev_flow *
static struct rps_dev_flow *
set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
+4 −1
Original line number Original line Diff line number Diff line
@@ -79,10 +79,13 @@ static int rps_sock_flow_sysctl(struct ctl_table *table, int write,


		if (sock_table != orig_sock_table) {
		if (sock_table != orig_sock_table) {
			rcu_assign_pointer(rps_sock_flow_table, sock_table);
			rcu_assign_pointer(rps_sock_flow_table, sock_table);
			if (sock_table)
			if (sock_table) {
				static_key_slow_inc(&rps_needed);
				static_key_slow_inc(&rps_needed);
				static_key_slow_inc(&rfs_needed);
			}
			if (orig_sock_table) {
			if (orig_sock_table) {
				static_key_slow_dec(&rps_needed);
				static_key_slow_dec(&rps_needed);
				static_key_slow_dec(&rfs_needed);
				synchronize_rcu();
				synchronize_rcu();
				vfree(orig_sock_table);
				vfree(orig_sock_table);
			}
			}