Commit 76a9ebe8 authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller
Browse files

net: extend sk_pacing_rate to unsigned long 



sk_pacing_rate has beed introduced as a u32 field in 2013,
effectively limiting per flow pacing to 34Gbit.

We believe it is time to allow TCP to pace high speed flows
on 64bit hosts, as we now can reach 100Gbit on one TCP flow.

This patch adds no cost for 32bit kernels.

The tcpi_pacing_rate and tcpi_max_pacing_rate were already
exported as 64bit, so iproute2/ss command require no changes.

Unfortunately the SO_MAX_PACING_RATE socket option will stay
32bit and we will need to add a new option to let applications
control high pacing rates.

State      Recv-Q Send-Q Local Address:Port             Peer Address:Port
ESTAB      0      1787144  10.246.9.76:49992             10.246.9.77:36741
                 timer:(on,003ms,0) ino:91863 sk:2 <->
 skmem:(r0,rb540000,t66440,tb2363904,f605944,w1822984,o0,bl0,d0)
 ts sack bbr wscale:8,8 rto:201 rtt:0.057/0.006 mss:1448
 rcvmss:536 advmss:1448
 cwnd:138 ssthresh:178 bytes_acked:256699822585 segs_out:177279177
 segs_in:3916318 data_segs_out:177279175
 bbr:(bw:31276.8Mbps,mrtt:0,pacing_gain:1.25,cwnd_gain:2)
 send 28045.5Mbps lastrcv:73333
 pacing_rate 38705.0Mbps delivery_rate 22997.6Mbps
 busy:73333ms unacked:135 retrans:0/157 rcv_space:14480
 notsent:2085120 minrtt:0.013

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 5f6188a8

include/net/sock.h

+2 −2
Original line number Diff line number Diff line
@@ -422,8 +422,8 @@ struct sock { 
	struct timer_list	sk_timer;

	__u32			sk_priority;

	__u32			sk_mark;

	u32			sk_pacing_rate; /* bytes per second */

	u32			sk_max_pacing_rate;

	unsigned long		sk_pacing_rate; /* bytes per second */

	unsigned long		sk_max_pacing_rate;

	struct page_frag	sk_frag;

	netdev_features_t	sk_route_caps;

	netdev_features_t	sk_route_nocaps;

net/core/filter.c

+2 −2
Original line number Diff line number Diff line
@@ -3927,8 +3927,8 @@ BPF_CALL_5(bpf_setsockopt, struct bpf_sock_ops_kern *, bpf_sock, 
			sk->sk_userlocks |= SOCK_SNDBUF_LOCK;

			sk->sk_sndbuf = max_t(int, val * 2, SOCK_MIN_SNDBUF);

			break;

		case SO_MAX_PACING_RATE:

			sk->sk_max_pacing_rate = val;

		case SO_MAX_PACING_RATE: /* 32bit version */

			sk->sk_max_pacing_rate = (val == ~0U) ? ~0UL : val;

			sk->sk_pacing_rate = min(sk->sk_pacing_rate,

						 sk->sk_max_pacing_rate);

			break;

net/core/sock.c

+5 −4
Original line number Diff line number Diff line
@@ -998,7 +998,7 @@ set_rcvbuf: 
			cmpxchg(&sk->sk_pacing_status,

				SK_PACING_NONE,

				SK_PACING_NEEDED);

		sk->sk_max_pacing_rate = val;

		sk->sk_max_pacing_rate = (val == ~0U) ? ~0UL : val;

		sk->sk_pacing_rate = min(sk->sk_pacing_rate,

					 sk->sk_max_pacing_rate);

		break;
@@ -1336,7 +1336,8 @@ int sock_getsockopt(struct socket *sock, int level, int optname, 
#endif



	case SO_MAX_PACING_RATE:

		v.val = sk->sk_max_pacing_rate;

		/* 32bit version */

		v.val = min_t(unsigned long, sk->sk_max_pacing_rate, ~0U);

		break;



	case SO_INCOMING_CPU:
@@ -2810,8 +2811,8 @@ void sock_init_data(struct socket *sock, struct sock *sk) 
	sk->sk_ll_usec		=	sysctl_net_busy_read;

#endif



	sk->sk_max_pacing_rate = ~0U;

	sk->sk_pacing_rate = ~0U;

	sk->sk_max_pacing_rate = ~0UL;

	sk->sk_pacing_rate = ~0UL;

	sk->sk_pacing_shift = 10;

	sk->sk_incoming_cpu = -1;

net/ipv4/tcp.c

+5 −5
Original line number Diff line number Diff line
@@ -3111,10 +3111,10 @@ void tcp_get_info(struct sock *sk, struct tcp_info *info) 
{

	const struct tcp_sock *tp = tcp_sk(sk); /* iff sk_type == SOCK_STREAM */

	const struct inet_connection_sock *icsk = inet_csk(sk);

	unsigned long rate;

	u32 now;

	u64 rate64;

	bool slow;

	u32 rate;



	memset(info, 0, sizeof(*info));

	if (sk->sk_type != SOCK_STREAM)
@@ -3124,11 +3124,11 @@ void tcp_get_info(struct sock *sk, struct tcp_info *info) 


	/* Report meaningful fields for all TCP states, including listeners */

	rate = READ_ONCE(sk->sk_pacing_rate);

	rate64 = rate != ~0U ? rate : ~0ULL;

	rate64 = (rate != ~0UL) ? rate : ~0ULL;

	info->tcpi_pacing_rate = rate64;



	rate = READ_ONCE(sk->sk_max_pacing_rate);

	rate64 = rate != ~0U ? rate : ~0ULL;

	rate64 = (rate != ~0UL) ? rate : ~0ULL;

	info->tcpi_max_pacing_rate = rate64;



	info->tcpi_reordering = tp->reordering;
@@ -3254,8 +3254,8 @@ struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk) 
	const struct tcp_sock *tp = tcp_sk(sk);

	struct sk_buff *stats;

	struct tcp_info info;

	unsigned long rate;

	u64 rate64;

	u32 rate;



	stats = alloc_skb(tcp_opt_stats_get_size(), GFP_ATOMIC);

	if (!stats)
@@ -3274,7 +3274,7 @@ struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk) 
			  tp->total_retrans, TCP_NLA_PAD);



	rate = READ_ONCE(sk->sk_pacing_rate);

	rate64 = rate != ~0U ? rate : ~0ULL;

	rate64 = (rate != ~0UL) ? rate : ~0ULL;

	nla_put_u64_64bit(stats, TCP_NLA_PACING_RATE, rate64, TCP_NLA_PAD);



	rate64 = tcp_compute_delivery_rate(tp);

net/ipv4/tcp_bbr.c

+3 −3
Original line number Diff line number Diff line
@@ -219,7 +219,7 @@ static u64 bbr_rate_bytes_per_sec(struct sock *sk, u64 rate, int gain) 
}



/* Convert a BBR bw and gain factor to a pacing rate in bytes per second. */

static u32 bbr_bw_to_pacing_rate(struct sock *sk, u32 bw, int gain)

static unsigned long bbr_bw_to_pacing_rate(struct sock *sk, u32 bw, int gain)

{

	u64 rate = bw;
@@ -258,7 +258,7 @@ static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain) 
{

	struct tcp_sock *tp = tcp_sk(sk);

	struct bbr *bbr = inet_csk_ca(sk);

	u32 rate = bbr_bw_to_pacing_rate(sk, bw, gain);

	unsigned long rate = bbr_bw_to_pacing_rate(sk, bw, gain);



	if (unlikely(!bbr->has_seen_rtt && tp->srtt_us))

		bbr_init_pacing_rate_from_rtt(sk);
@@ -280,7 +280,7 @@ static u32 bbr_tso_segs_goal(struct sock *sk) 
	/* Sort of tcp_tso_autosize() but ignoring

	 * driver provided sk_gso_max_size.

	 */

	bytes = min_t(u32, sk->sk_pacing_rate >> sk->sk_pacing_shift,

	bytes = min_t(unsigned long, sk->sk_pacing_rate >> sk->sk_pacing_shift,

		      GSO_MAX_SIZE - 1 - MAX_TCP_HEADER);

	segs = max_t(u32, bytes / tp->mss_cache, bbr_min_tso_segs(sk));

CRA Git | Maintained and supported by SUSTech CRA and CCSE