Commit 7e550527 authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'NAPI-gro-hash' 



Convert GRO receive over to hash table.

When many parallel flows are present and being received on the same
RX queue, GRO processing can become expensive because each incoming
frame must traverse the per-NAPI GRO list at each protocol layer
of GRO receive (eth --> ipv{4,6} --> tcp).

Use the already computed hash to chain these SKBs in a hash table
instead of a simple list.

The first patch makes the GRO list a true list_head.

The second patch implements the hash table.

This series patches basic testing and I added some diagnostics
to make sure we really were aggregating GRO frames :-)

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 9ff3b40e 07d78363

drivers/net/geneve.c

+6 −5
Original line number Diff line number Diff line
@@ -418,11 +418,12 @@ static int geneve_hlen(struct genevehdr *gh) 
	return sizeof(*gh) + gh->opt_len * 4;

}



static struct sk_buff **geneve_gro_receive(struct sock *sk,

					   struct sk_buff **head,

static struct sk_buff *geneve_gro_receive(struct sock *sk,

					  struct list_head *head,

					  struct sk_buff *skb)

{

	struct sk_buff *p, **pp = NULL;

	struct sk_buff *pp = NULL;

	struct sk_buff *p;

	struct genevehdr *gh, *gh2;

	unsigned int hlen, gh_len, off_gnv;

	const struct packet_offload *ptype;
@@ -449,7 +450,7 @@ static struct sk_buff **geneve_gro_receive(struct sock *sk, 
			goto out;

	}



	for (p = *head; p; p = p->next) {

	list_for_each_entry(p, head, list) {

		if (!NAPI_GRO_CB(p)->same_flow)

			continue;

drivers/net/vxlan.c

+6 −5
Original line number Diff line number Diff line
@@ -568,11 +568,12 @@ static struct vxlanhdr *vxlan_gro_remcsum(struct sk_buff *skb, 
	return vh;

}



static struct sk_buff **vxlan_gro_receive(struct sock *sk,

					  struct sk_buff **head,

static struct sk_buff *vxlan_gro_receive(struct sock *sk,

					 struct list_head *head,

					 struct sk_buff *skb)

{

	struct sk_buff *p, **pp = NULL;

	struct sk_buff *pp = NULL;

	struct sk_buff *p;

	struct vxlanhdr *vh, *vh2;

	unsigned int hlen, off_vx;

	int flush = 1;
@@ -607,7 +608,7 @@ static struct sk_buff **vxlan_gro_receive(struct sock *sk, 


	skb_gro_pull(skb, sizeof(struct vxlanhdr)); /* pull vxlan header */



	for (p = *head; p; p = p->next) {

	list_for_each_entry(p, head, list) {

		if (!NAPI_GRO_CB(p)->same_flow)

			continue;

include/linux/etherdevice.h

+1 −2
Original line number Diff line number Diff line
@@ -59,8 +59,7 @@ struct net_device *devm_alloc_etherdev_mqs(struct device *dev, int sizeof_priv, 
					   unsigned int rxqs);

#define devm_alloc_etherdev(dev, sizeof_priv) devm_alloc_etherdev_mqs(dev, sizeof_priv, 1, 1)



struct sk_buff **eth_gro_receive(struct sk_buff **head,

				 struct sk_buff *skb);

struct sk_buff *eth_gro_receive(struct list_head *head, struct sk_buff *skb);

int eth_gro_complete(struct sk_buff *skb, int nhoff);



/* Reserved Ethernet Addresses per IEEE 802.1Q */

include/linux/netdevice.h

+17 −16
Original line number Diff line number Diff line
@@ -305,6 +305,7 @@ int __init netdev_boot_setup(char *str); 
/*

 * Structure for NAPI scheduling similar to tasklet but with weighting

 */

#define GRO_HASH_BUCKETS	8

struct napi_struct {

	/* The poll_list must only be managed by the entity which

	 * changes the state of the NAPI_STATE_SCHED bit.  This means
@@ -322,7 +323,7 @@ struct napi_struct { 
	int			poll_owner;

#endif

	struct net_device	*dev;

	struct sk_buff		*gro_list;

	struct list_head	gro_hash[GRO_HASH_BUCKETS];

	struct sk_buff		*skb;

	struct hrtimer		timer;

	struct list_head	dev_list;
@@ -2255,9 +2256,9 @@ static inline int gro_recursion_inc_test(struct sk_buff *skb) 
	return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT;

}



typedef struct sk_buff **(*gro_receive_t)(struct sk_buff **, struct sk_buff *);

static inline struct sk_buff **call_gro_receive(gro_receive_t cb,

						struct sk_buff **head,

typedef struct sk_buff *(*gro_receive_t)(struct list_head *, struct sk_buff *);

static inline struct sk_buff *call_gro_receive(gro_receive_t cb,

					       struct list_head *head,

					       struct sk_buff *skb)

{

	if (unlikely(gro_recursion_inc_test(skb))) {
@@ -2268,11 +2269,11 @@ static inline struct sk_buff **call_gro_receive(gro_receive_t cb, 
	return cb(head, skb);

}



typedef struct sk_buff **(*gro_receive_sk_t)(struct sock *, struct sk_buff **,

typedef struct sk_buff *(*gro_receive_sk_t)(struct sock *, struct list_head *,

					    struct sk_buff *);

static inline struct sk_buff **call_gro_receive_sk(gro_receive_sk_t cb,

static inline struct sk_buff *call_gro_receive_sk(gro_receive_sk_t cb,

						  struct sock *sk,

						   struct sk_buff **head,

						  struct list_head *head,

						  struct sk_buff *skb)

{

	if (unlikely(gro_recursion_inc_test(skb))) {
@@ -2299,7 +2300,7 @@ struct packet_type { 
struct offload_callbacks {

	struct sk_buff		*(*gso_segment)(struct sk_buff *skb,

						netdev_features_t features);

	struct sk_buff		**(*gro_receive)(struct sk_buff **head,

	struct sk_buff		*(*gro_receive)(struct list_head *head,

						struct sk_buff *skb);

	int			(*gro_complete)(struct sk_buff *skb, int nhoff);

};
@@ -2568,7 +2569,7 @@ struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex); 
struct net_device *dev_get_by_napi_id(unsigned int napi_id);

int netdev_get_name(struct net *net, char *name, int ifindex);

int dev_restart(struct net_device *dev);

int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb);

int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb);



static inline unsigned int skb_gro_offset(const struct sk_buff *skb)

{
@@ -2784,13 +2785,13 @@ static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb, 
}



#ifdef CONFIG_XFRM_OFFLOAD

static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff **pp, int flush)

static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)

{

	if (PTR_ERR(pp) != -EINPROGRESS)

		NAPI_GRO_CB(skb)->flush |= flush;

}

#else

static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff **pp, int flush)

static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)

{

	NAPI_GRO_CB(skb)->flush |= flush;

}

include/linux/skbuff.h

+2 −1
Original line number Diff line number Diff line
@@ -678,6 +678,7 @@ struct sk_buff { 
			};

		};

		struct rb_node		rbnode; /* used in netem & tcp stack */

		struct list_head	list;

	};

	struct sock		*sk;

CRA Git | Maintained and supported by SUSTech CRA and CCSE