Commit 2dc41cff authored by David Held's avatar David Held Committed by David S. Miller
Browse files

udp: Use hash2 for long hash1 chains in __udp*_lib_mcast_deliver. 



Many multicast sources can have the same port which can result in a very
large list when hashing by port only. Hash by address and port instead
if this is the case. This makes multicast more similar to unicast.

On a 24-core machine receiving from 500 multicast sockets on the same
port, before this patch 80% of system CPU was used up by spin locking
and only ~25% of packets were successfully delivered.

With this patch, all packets are delivered and kernel overhead is ~8%
system CPU on spinlocks.

Signed-off-by: default avatarDavid Held <drheld@google.com>
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 5cf3d461

include/net/sock.h

+14 −0
Original line number Diff line number Diff line
@@ -660,6 +660,20 @@ static inline void sk_add_bind_node(struct sock *sk, 
#define sk_for_each_bound(__sk, list) \

	hlist_for_each_entry(__sk, list, sk_bind_node)



/**

 * sk_nulls_for_each_entry_offset - iterate over a list at a given struct offset

 * @tpos:	the type * to use as a loop cursor.

 * @pos:	the &struct hlist_node to use as a loop cursor.

 * @head:	the head for your list.

 * @offset:	offset of hlist_node within the struct.

 *

 */

#define sk_nulls_for_each_entry_offset(tpos, pos, head, offset)		       \

	for (pos = (head)->first;					       \

	     (!is_a_nulls(pos)) &&					       \

		({ tpos = (typeof(*tpos) *)((void *)pos - offset); 1;});       \

	     pos = pos->next)



static inline struct user_namespace *sk_user_ns(struct sock *sk)

{

	/* Careful only use this in a context where these parameters

net/ipv4/udp.c

+21 −10
Original line number Diff line number Diff line
@@ -1619,6 +1619,8 @@ static void flush_stack(struct sock **stack, unsigned int count, 


		if (skb1 && udp_queue_rcv_skb(sk, skb1) <= 0)

			skb1 = NULL;



		sock_put(sk);

	}

	if (unlikely(skb1))

		kfree_skb(skb1);
@@ -1651,10 +1653,20 @@ static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb, 
	unsigned short hnum = ntohs(uh->dest);

	struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum);

	int dif = skb->dev->ifindex;

	unsigned int i, count = 0;

	unsigned int count = 0, offset = offsetof(typeof(*sk), sk_nulls_node);

	unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10);



	if (use_hash2) {

		hash2_any = udp4_portaddr_hash(net, htonl(INADDR_ANY), hnum) &

			    udp_table.mask;

		hash2 = udp4_portaddr_hash(net, daddr, hnum) & udp_table.mask;

start_lookup:

		hslot = &udp_table.hash2[hash2];

		offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);

	}



	spin_lock(&hslot->lock);

	sk_nulls_for_each(sk, node, &hslot->head) {

	sk_nulls_for_each_entry_offset(sk, node, &hslot->head, offset) {

		if (__udp_is_mcast_sock(net, sk,

					uh->dest, daddr,

					uh->source, saddr,
@@ -1664,24 +1676,23 @@ static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb, 
				count = 0;

			}

			stack[count++] = sk;

			sock_hold(sk);

		}

	}

	/*

	 * before releasing chain lock, we must take a reference on sockets

	 */

	for (i = 0; i < count; i++)

		sock_hold(stack[i]);



	spin_unlock(&hslot->lock);



	/* Also lookup *:port if we are using hash2 and haven't done so yet. */

	if (use_hash2 && hash2 != hash2_any) {

		hash2 = hash2_any;

		goto start_lookup;

	}



	/*

	 * do the slow work with no lock held

	 */

	if (count) {

		flush_stack(stack, count, skb, count - 1);



		for (i = 0; i < count; i++)

			sock_put(stack[i]);

	} else {

		kfree_skb(skb);

	}

net/ipv6/udp.c

+20 −10
Original line number Diff line number Diff line
@@ -745,6 +745,7 @@ static void flush_stack(struct sock **stack, unsigned int count, 


		if (skb1 && udpv6_queue_rcv_skb(sk, skb1) <= 0)

			skb1 = NULL;

		sock_put(sk);

	}

	if (unlikely(skb1))

		kfree_skb(skb1);
@@ -774,10 +775,20 @@ static int __udp6_lib_mcast_deliver(struct net *net, struct sk_buff *skb, 
	unsigned short hnum = ntohs(uh->dest);

	struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum);

	int dif = inet6_iif(skb);

	unsigned int i, count = 0;

	unsigned int count = 0, offset = offsetof(typeof(*sk), sk_nulls_node);

	unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10);



	if (use_hash2) {

		hash2_any = udp6_portaddr_hash(net, &in6addr_any, hnum) &

			    udp_table.mask;

		hash2 = udp6_portaddr_hash(net, daddr, hnum) & udp_table.mask;

start_lookup:

		hslot = &udp_table.hash2[hash2];

		offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);

	}



	spin_lock(&hslot->lock);

	sk_nulls_for_each(sk, node, &hslot->head) {

	sk_nulls_for_each_entry_offset(sk, node, &hslot->head, offset) {

		if (__udp_v6_is_mcast_sock(net, sk,

					   uh->dest, daddr,

					   uh->source, saddr,
@@ -791,21 +802,20 @@ static int __udp6_lib_mcast_deliver(struct net *net, struct sk_buff *skb, 
				count = 0;

			}

			stack[count++] = sk;

			sock_hold(sk);

		}

	}

	/*

	 * before releasing the lock, we must take reference on sockets

	 */

	for (i = 0; i < count; i++)

		sock_hold(stack[i]);



	spin_unlock(&hslot->lock);



	/* Also lookup *:port if we are using hash2 and haven't done so yet. */

	if (use_hash2 && hash2 != hash2_any) {

		hash2 = hash2_any;

		goto start_lookup;

	}



	if (count) {

		flush_stack(stack, count, skb, count - 1);



		for (i = 0; i < count; i++)

			sock_put(stack[i]);

	} else {

		kfree_skb(skb);

	}

CRA Git | Maintained and supported by SUSTech CRA and CCSE