Commit ab4e846a authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller
Browse files

tcp: annotate sk->sk_wmem_queued lockless reads 



For the sake of tcp_poll(), there are few places where we fetch
sk->sk_wmem_queued while this field can change from IRQ or other cpu.

We need to add READ_ONCE() annotations, and also make sure write
sides use corresponding WRITE_ONCE() to avoid store-tearing.

sk_wmem_queued_add() helper is added so that we can in
the future convert to ADD_ONCE() or equivalent if/when
available.

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent e292f05e

include/net/sock.h

+10 −5
Original line number Diff line number Diff line
@@ -878,12 +878,17 @@ static inline bool sk_acceptq_is_full(const struct sock *sk) 
 */

static inline int sk_stream_min_wspace(const struct sock *sk)

{

	return sk->sk_wmem_queued >> 1;

	return READ_ONCE(sk->sk_wmem_queued) >> 1;

}



static inline int sk_stream_wspace(const struct sock *sk)

{

	return READ_ONCE(sk->sk_sndbuf) - sk->sk_wmem_queued;

	return READ_ONCE(sk->sk_sndbuf) - READ_ONCE(sk->sk_wmem_queued);

}



static inline void sk_wmem_queued_add(struct sock *sk, int val)

{

	WRITE_ONCE(sk->sk_wmem_queued, sk->sk_wmem_queued + val);

}



void sk_stream_write_space(struct sock *sk);
@@ -1207,7 +1212,7 @@ static inline void sk_refcnt_debug_release(const struct sock *sk) 


static inline bool __sk_stream_memory_free(const struct sock *sk, int wake)

{

	if (sk->sk_wmem_queued >= READ_ONCE(sk->sk_sndbuf))

	if (READ_ONCE(sk->sk_wmem_queued) >= READ_ONCE(sk->sk_sndbuf))

		return false;



	return sk->sk_prot->stream_memory_free ?
@@ -1467,7 +1472,7 @@ DECLARE_STATIC_KEY_FALSE(tcp_tx_skb_cache_key); 
static inline void sk_wmem_free_skb(struct sock *sk, struct sk_buff *skb)

{

	sock_set_flag(sk, SOCK_QUEUE_SHRUNK);

	sk->sk_wmem_queued -= skb->truesize;

	sk_wmem_queued_add(sk, -skb->truesize);

	sk_mem_uncharge(sk, skb->truesize);

	if (static_branch_unlikely(&tcp_tx_skb_cache_key) &&

	    !sk->sk_tx_skb_cache && !skb_cloned(skb)) {
@@ -2014,7 +2019,7 @@ static inline int skb_copy_to_page_nocache(struct sock *sk, struct iov_iter *fro 
	skb->len	     += copy;

	skb->data_len	     += copy;

	skb->truesize	     += copy;

	sk->sk_wmem_queued   += copy;

	sk_wmem_queued_add(sk, copy);

	sk_mem_charge(sk, copy);

	return 0;

}

include/trace/events/sock.h

+1 −1
Original line number Diff line number Diff line
@@ -115,7 +115,7 @@ TRACE_EVENT(sock_exceed_buf_limit, 
		__entry->rmem_alloc = atomic_read(&sk->sk_rmem_alloc);

		__entry->sysctl_wmem = sk_get_wmem0(sk, prot);

		__entry->wmem_alloc = refcount_read(&sk->sk_wmem_alloc);

		__entry->wmem_queued = sk->sk_wmem_queued;

		__entry->wmem_queued = READ_ONCE(sk->sk_wmem_queued);

		__entry->kind = kind;

	),

net/core/datagram.c

+1 −1
Original line number Diff line number Diff line
@@ -640,7 +640,7 @@ int __zerocopy_sg_from_iter(struct sock *sk, struct sk_buff *skb, 
		skb->len += copied;

		skb->truesize += truesize;

		if (sk && sk->sk_type == SOCK_STREAM) {

			sk->sk_wmem_queued += truesize;

			sk_wmem_queued_add(sk, truesize);

			sk_mem_charge(sk, truesize);

		} else {

			refcount_add(truesize, &skb->sk->sk_wmem_alloc);

net/core/sock.c

+1 −1
Original line number Diff line number Diff line
@@ -3212,7 +3212,7 @@ void sk_get_meminfo(const struct sock *sk, u32 *mem) 
	mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk);

	mem[SK_MEMINFO_SNDBUF] = READ_ONCE(sk->sk_sndbuf);

	mem[SK_MEMINFO_FWD_ALLOC] = sk->sk_forward_alloc;

	mem[SK_MEMINFO_WMEM_QUEUED] = sk->sk_wmem_queued;

	mem[SK_MEMINFO_WMEM_QUEUED] = READ_ONCE(sk->sk_wmem_queued);

	mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc);

	mem[SK_MEMINFO_BACKLOG] = READ_ONCE(sk->sk_backlog.len);

	mem[SK_MEMINFO_DROPS] = atomic_read(&sk->sk_drops);

net/ipv4/inet_diag.c

+1 −1
Original line number Diff line number Diff line
@@ -193,7 +193,7 @@ int inet_sk_diag_fill(struct sock *sk, struct inet_connection_sock *icsk, 
	if (ext & (1 << (INET_DIAG_MEMINFO - 1))) {

		struct inet_diag_meminfo minfo = {

			.idiag_rmem = sk_rmem_alloc_get(sk),

			.idiag_wmem = sk->sk_wmem_queued,

			.idiag_wmem = READ_ONCE(sk->sk_wmem_queued),

			.idiag_fmem = sk->sk_forward_alloc,

			.idiag_tmem = sk_wmem_alloc_get(sk),

		};

CRA Git | Maintained and supported by SUSTech CRA and CCSE