Commit d9f37d01 authored by Li RongQing's avatar Li RongQing Committed by David S. Miller
Browse files

net: convert gro_count to bitmask 



gro_hash size is 192 bytes, and uses 3 cache lines, if there is few
flows, gro_hash may be not fully used, so it is unnecessary to iterate
all gro_hash in napi_gro_flush(), to occupy unnecessary cacheline.

convert gro_count to a bitmask, and rename it as gro_bitmask, each bit
represents a element of gro_hash, only flush a gro_hash element if the
related bit is set, to speed up napi_gro_flush().

and update gro_bitmask only if it will be changed, to reduce cache
update

Suggested-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarLi RongQing <lirongqing@baidu.com>
Cc: Stefano Brivio <sbrivio@redhat.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 48559af3

include/linux/netdevice.h

+7 −2
Original line number Diff line number Diff line
@@ -308,9 +308,14 @@ struct gro_list { 
};



/*

 * Structure for NAPI scheduling similar to tasklet but with weighting

 * size of gro hash buckets, must less than bit number of

 * napi_struct::gro_bitmask

 */

#define GRO_HASH_BUCKETS	8



/*

 * Structure for NAPI scheduling similar to tasklet but with weighting

 */

struct napi_struct {

	/* The poll_list must only be managed by the entity which

	 * changes the state of the NAPI_STATE_SCHED bit.  This means
@@ -322,7 +327,7 @@ struct napi_struct { 


	unsigned long		state;

	int			weight;

	unsigned int		gro_count;

	unsigned long		gro_bitmask;

	int			(*poll)(struct napi_struct *, int);

#ifdef CONFIG_NETPOLL

	int			poll_owner;

net/core/dev.c

+24 −12
Original line number Diff line number Diff line
@@ -5282,9 +5282,11 @@ static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index, 
		list_del(&skb->list);

		skb->next = NULL;

		napi_gro_complete(skb);

		napi->gro_count--;

		napi->gro_hash[index].count--;

	}



	if (!napi->gro_hash[index].count)

		__clear_bit(index, &napi->gro_bitmask);

}



/* napi->gro_hash[].list contains packets ordered by age.
@@ -5295,9 +5297,11 @@ void napi_gro_flush(struct napi_struct *napi, bool flush_old) 
{

	u32 i;



	for (i = 0; i < GRO_HASH_BUCKETS; i++)

	for (i = 0; i < GRO_HASH_BUCKETS; i++) {

		if (test_bit(i, &napi->gro_bitmask))

			__napi_gro_flush_chain(napi, i, flush_old);

	}

}

EXPORT_SYMBOL(napi_gro_flush);



static struct list_head *gro_list_prepare(struct napi_struct *napi,
@@ -5388,8 +5392,8 @@ static void gro_flush_oldest(struct list_head *head) 
	if (WARN_ON_ONCE(!oldest))

		return;



	/* Do not adjust napi->gro_count, caller is adding a new SKB to

	 * the chain.

	/* Do not adjust napi->gro_hash[].count, caller is adding a new

	 * SKB to the chain.

	 */

	list_del(&oldest->list);

	napi_gro_complete(oldest);
@@ -5464,7 +5468,6 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff 
		list_del(&pp->list);

		pp->next = NULL;

		napi_gro_complete(pp);

		napi->gro_count--;

		napi->gro_hash[hash].count--;

	}
@@ -5477,7 +5480,6 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff 
	if (unlikely(napi->gro_hash[hash].count >= MAX_GRO_SKBS)) {

		gro_flush_oldest(gro_head);

	} else {

		napi->gro_count++;

		napi->gro_hash[hash].count++;

	}

	NAPI_GRO_CB(skb)->count = 1;
@@ -5492,6 +5494,13 @@ pull: 
	if (grow > 0)

		gro_pull_from_frag0(skb, grow);

ok:

	if (napi->gro_hash[hash].count) {

		if (!test_bit(hash, &napi->gro_bitmask))

			__set_bit(hash, &napi->gro_bitmask);

	} else if (test_bit(hash, &napi->gro_bitmask)) {

		__clear_bit(hash, &napi->gro_bitmask);

	}



	return ret;



normal:
@@ -5890,7 +5899,7 @@ bool napi_complete_done(struct napi_struct *n, int work_done) 
				 NAPIF_STATE_IN_BUSY_POLL)))

		return false;



	if (n->gro_count) {

	if (n->gro_bitmask) {

		unsigned long timeout = 0;



		if (work_done)
@@ -6099,7 +6108,7 @@ static enum hrtimer_restart napi_watchdog(struct hrtimer *timer) 
	/* Note : we use a relaxed variant of napi_schedule_prep() not setting

	 * NAPI_STATE_MISSED, since we do not react to a device IRQ.

	 */

	if (napi->gro_count && !napi_disable_pending(napi) &&

	if (napi->gro_bitmask && !napi_disable_pending(napi) &&

	    !test_and_set_bit(NAPI_STATE_SCHED, &napi->state))

		__napi_schedule_irqoff(napi);
@@ -6114,7 +6123,7 @@ void netif_napi_add(struct net_device *dev, struct napi_struct *napi, 
	INIT_LIST_HEAD(&napi->poll_list);

	hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);

	napi->timer.function = napi_watchdog;

	napi->gro_count = 0;

	napi->gro_bitmask = 0;

	for (i = 0; i < GRO_HASH_BUCKETS; i++) {

		INIT_LIST_HEAD(&napi->gro_hash[i].list);

		napi->gro_hash[i].count = 0;
@@ -6174,7 +6183,7 @@ void netif_napi_del(struct napi_struct *napi) 
	napi_free_frags(napi);



	flush_gro_hash(napi);

	napi->gro_count = 0;

	napi->gro_bitmask = 0;

}

EXPORT_SYMBOL(netif_napi_del);
@@ -6216,7 +6225,7 @@ static int napi_poll(struct napi_struct *n, struct list_head *repoll) 
		goto out_unlock;

	}



	if (n->gro_count) {

	if (n->gro_bitmask) {

		/* flush too old packets

		 * If HZ < 1000, flush all packets.

		 */
@@ -9272,6 +9281,9 @@ static struct hlist_head * __net_init netdev_create_hash(void) 
/* Initialize per network namespace state */

static int __net_init netdev_init(struct net *net)

{

	BUILD_BUG_ON(GRO_HASH_BUCKETS >

			FIELD_SIZEOF(struct napi_struct, gro_bitmask));



	if (net != &init_net)

		INIT_LIST_HEAD(&net->dev_base_head);

CRA Git | Maintained and supported by SUSTech CRA and CCSE