Commit e227701c authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'net-ICW-sendmsg-recvmsg' 



Paolo Abeni says:

====================
net: use ICW for sk_proto->{send,recv}msg

This series extends ICW usage to one of the few remaining spots in fast-path
still hitting per packet retpoline overhead, namely the sk_proto->{send,recv}msg
calls.

The first 3 patches in this series refactor the existing code so that applying
the ICW macros is straight-forward: we demux inet_{recv,send}msg in ipv4 and
ipv6 variants so that each of them can easily select the appropriate TCP or UDP
direct call. While at it, a new helper is created to avoid excessive code
duplication, and the current ICWs for inet_{recv,send}msg are adjusted
accordingly.

The last 2 patches really introduce the new ICW use-case, respectively for the
ipv6 and the ipv4 code path.

This gives up to 5% performance improvement under UDP flood, and smaller but
measurable gains for TCP RR workloads.

v1 -> v2:
 - drop inet6_{recv,send}msg declaration from header file,
   prefer ICW macro instead
 - avoid unneeded reclaration for udp_sendmsg, as suggested by Willem
====================

Acked-by: default avatarWillem de Bruijn <willemb@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 2559d7c4 6f24080e

include/net/inet_common.h

+1 −0
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ int inet_dgram_connect(struct socket *sock, struct sockaddr *uaddr, 
		       int addr_len, int flags);

int inet_accept(struct socket *sock, struct socket *newsock, int flags,

		bool kern);

int inet_send_prepare(struct sock *sk);

int inet_sendmsg(struct socket *sock, struct msghdr *msg, size_t size);

ssize_t inet_sendpage(struct socket *sock, struct page *page, int offset,

		      size_t size, int flags);

net/ipv4/af_inet.c

+20 −11
Original line number Diff line number Diff line
@@ -784,10 +784,8 @@ int inet_getname(struct socket *sock, struct sockaddr *uaddr, 
}

EXPORT_SYMBOL(inet_getname);



int inet_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)

int inet_send_prepare(struct sock *sk)

{

	struct sock *sk = sock->sk;



	sock_rps_record_flow(sk);



	/* We may need to bind the socket. */
@@ -795,7 +793,19 @@ int inet_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) 
	    inet_autobind(sk))

		return -EAGAIN;



	return sk->sk_prot->sendmsg(sk, msg, size);

	return 0;

}

EXPORT_SYMBOL_GPL(inet_send_prepare);



int inet_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)

{

	struct sock *sk = sock->sk;



	if (unlikely(inet_send_prepare(sk)))

		return -EAGAIN;



	return INDIRECT_CALL_2(sk->sk_prot->sendmsg, tcp_sendmsg, udp_sendmsg,

			       sk, msg, size);

}

EXPORT_SYMBOL(inet_sendmsg);
@@ -804,11 +814,7 @@ ssize_t inet_sendpage(struct socket *sock, struct page *page, int offset, 
{

	struct sock *sk = sock->sk;



	sock_rps_record_flow(sk);



	/* We may need to bind the socket. */

	if (!inet_sk(sk)->inet_num && !sk->sk_prot->no_autobind &&

	    inet_autobind(sk))

	if (unlikely(inet_send_prepare(sk)))

		return -EAGAIN;



	if (sk->sk_prot->sendpage)
@@ -817,6 +823,8 @@ ssize_t inet_sendpage(struct socket *sock, struct page *page, int offset, 
}

EXPORT_SYMBOL(inet_sendpage);



INDIRECT_CALLABLE_DECLARE(int udp_recvmsg(struct sock *, struct msghdr *,

					  size_t, int, int, int *));

int inet_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,

		 int flags)

{
@@ -827,7 +835,8 @@ int inet_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, 
	if (likely(!(flags & MSG_ERRQUEUE)))

		sock_rps_record_flow(sk);



	err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT,

	err = INDIRECT_CALL_2(sk->sk_prot->recvmsg, tcp_recvmsg, udp_recvmsg,

			      sk, msg, size, flags & MSG_DONTWAIT,

			      flags & ~MSG_DONTWAIT, &addr_len);

	if (err >= 0)

		msg->msg_namelen = addr_len;

net/ipv6/af_inet6.c

+37 −4
Original line number Diff line number Diff line
@@ -564,6 +564,39 @@ int inet6_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) 
}

EXPORT_SYMBOL(inet6_ioctl);



INDIRECT_CALLABLE_DECLARE(int udpv6_sendmsg(struct sock *, struct msghdr *,

					    size_t));

int inet6_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)

{

	struct sock *sk = sock->sk;



	if (unlikely(inet_send_prepare(sk)))

		return -EAGAIN;



	return INDIRECT_CALL_2(sk->sk_prot->sendmsg, tcp_sendmsg, udpv6_sendmsg,

			       sk, msg, size);

}



INDIRECT_CALLABLE_DECLARE(int udpv6_recvmsg(struct sock *, struct msghdr *,

					    size_t, int, int, int *));

int inet6_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,

		  int flags)

{

	struct sock *sk = sock->sk;

	int addr_len = 0;

	int err;



	if (likely(!(flags & MSG_ERRQUEUE)))

		sock_rps_record_flow(sk);



	err = INDIRECT_CALL_2(sk->sk_prot->recvmsg, tcp_recvmsg, udpv6_recvmsg,

			      sk, msg, size, flags & MSG_DONTWAIT,

			      flags & ~MSG_DONTWAIT, &addr_len);

	if (err >= 0)

		msg->msg_namelen = addr_len;

	return err;

}



const struct proto_ops inet6_stream_ops = {

	.family		   = PF_INET6,

	.owner		   = THIS_MODULE,
@@ -580,8 +613,8 @@ const struct proto_ops inet6_stream_ops = { 
	.shutdown	   = inet_shutdown,		/* ok		*/

	.setsockopt	   = sock_common_setsockopt,	/* ok		*/

	.getsockopt	   = sock_common_getsockopt,	/* ok		*/

	.sendmsg	   = inet_sendmsg,		/* ok		*/

	.recvmsg	   = inet_recvmsg,		/* ok		*/

	.sendmsg	   = inet6_sendmsg,		/* retpoline's sake */

	.recvmsg	   = inet6_recvmsg,		/* retpoline's sake */

#ifdef CONFIG_MMU

	.mmap		   = tcp_mmap,

#endif
@@ -614,8 +647,8 @@ const struct proto_ops inet6_dgram_ops = { 
	.shutdown	   = inet_shutdown,		/* ok		*/

	.setsockopt	   = sock_common_setsockopt,	/* ok		*/

	.getsockopt	   = sock_common_getsockopt,	/* ok		*/

	.sendmsg	   = inet_sendmsg,		/* ok		*/

	.recvmsg	   = inet_recvmsg,		/* ok		*/

	.sendmsg	   = inet6_sendmsg,		/* retpoline's sake */

	.recvmsg	   = inet6_recvmsg,		/* retpoline's sake */

	.mmap		   = sock_no_mmap,

	.sendpage	   = sock_no_sendpage,

	.set_peek_off	   = sk_set_peek_off,

net/socket.c

+11 −12
Original line number Diff line number Diff line
@@ -103,13 +103,6 @@ 
#include <net/busy_poll.h>

#include <linux/errqueue.h>



/* proto_ops for ipv4 and ipv6 use the same {recv,send}msg function */

#if IS_ENABLED(CONFIG_INET)

#define INDIRECT_CALL_INET4(f, f1, ...) INDIRECT_CALL_1(f, f1, __VA_ARGS__)

#else

#define INDIRECT_CALL_INET4(f, f1, ...) f(__VA_ARGS__)

#endif



#ifdef CONFIG_NET_RX_BUSY_POLL

unsigned int sysctl_net_busy_read __read_mostly;

unsigned int sysctl_net_busy_poll __read_mostly;
@@ -641,10 +634,13 @@ EXPORT_SYMBOL(__sock_tx_timestamp); 


INDIRECT_CALLABLE_DECLARE(int inet_sendmsg(struct socket *, struct msghdr *,

					   size_t));

INDIRECT_CALLABLE_DECLARE(int inet6_sendmsg(struct socket *, struct msghdr *,

					    size_t));

static inline int sock_sendmsg_nosec(struct socket *sock, struct msghdr *msg)

{

	int ret = INDIRECT_CALL_INET4(sock->ops->sendmsg, inet_sendmsg, sock,

				      msg, msg_data_left(msg));

	int ret = INDIRECT_CALL_INET(sock->ops->sendmsg, inet6_sendmsg,

				     inet_sendmsg, sock, msg,

				     msg_data_left(msg));

	BUG_ON(ret == -EIOCBQUEUED);

	return ret;

}
@@ -871,11 +867,14 @@ EXPORT_SYMBOL_GPL(__sock_recv_ts_and_drops); 


INDIRECT_CALLABLE_DECLARE(int inet_recvmsg(struct socket *, struct msghdr *,

					   size_t, int));

INDIRECT_CALLABLE_DECLARE(int inet6_recvmsg(struct socket *, struct msghdr *,

					    size_t, int));

static inline int sock_recvmsg_nosec(struct socket *sock, struct msghdr *msg,

				     int flags)

{

	return INDIRECT_CALL_INET4(sock->ops->recvmsg, inet_recvmsg, sock, msg,

				   msg_data_left(msg), flags);

	return INDIRECT_CALL_INET(sock->ops->recvmsg, inet6_recvmsg,

				  inet_recvmsg, sock, msg, msg_data_left(msg),

				  flags);

}



/**

CRA Git | Maintained and supported by SUSTech CRA and CCSE