Commit 7db48e98 authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller
Browse files

tcp: annotate tp->copied_seq lockless reads 



There are few places where we fetch tp->copied_seq while
this field can change from IRQ or other cpu.

We need to add READ_ONCE() annotations, and also make
sure write sides use corresponding WRITE_ONCE() to avoid
store-tearing.

Note that tcp_inq_hint() was already using READ_ONCE(tp->copied_seq)

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent dba7d9b8

net/ipv4/tcp.c

+10 −10
Original line number Diff line number Diff line
@@ -477,7 +477,7 @@ static void tcp_tx_timestamp(struct sock *sk, u16 tsflags) 
static inline bool tcp_stream_is_readable(const struct tcp_sock *tp,

					  int target, struct sock *sk)

{

	return (READ_ONCE(tp->rcv_nxt) - tp->copied_seq >= target) ||

	return (READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq) >= target) ||

		(sk->sk_prot->stream_memory_read ?

		sk->sk_prot->stream_memory_read(sk) : false);

}
@@ -546,7 +546,7 @@ __poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait) 
	    (state != TCP_SYN_RECV || rcu_access_pointer(tp->fastopen_rsk))) {

		int target = sock_rcvlowat(sk, 0, INT_MAX);



		if (tp->urg_seq == tp->copied_seq &&

		if (tp->urg_seq == READ_ONCE(tp->copied_seq) &&

		    !sock_flag(sk, SOCK_URGINLINE) &&

		    tp->urg_data)

			target++;
@@ -607,7 +607,7 @@ int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg) 
		unlock_sock_fast(sk, slow);

		break;

	case SIOCATMARK:

		answ = tp->urg_data && tp->urg_seq == tp->copied_seq;

		answ = tp->urg_data && tp->urg_seq == READ_ONCE(tp->copied_seq);

		break;

	case SIOCOUTQ:

		if (sk->sk_state == TCP_LISTEN)
@@ -1668,9 +1668,9 @@ int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 
		sk_eat_skb(sk, skb);

		if (!desc->count)

			break;

		tp->copied_seq = seq;

		WRITE_ONCE(tp->copied_seq, seq);

	}

	tp->copied_seq = seq;

	WRITE_ONCE(tp->copied_seq, seq);



	tcp_rcv_space_adjust(sk);
@@ -1819,7 +1819,7 @@ static int tcp_zerocopy_receive(struct sock *sk, 
out:

	up_read(&current->mm->mmap_sem);

	if (length) {

		tp->copied_seq = seq;

		WRITE_ONCE(tp->copied_seq, seq);

		tcp_rcv_space_adjust(sk);



		/* Clean up data we have read: This will do ACK frames. */
@@ -2117,7 +2117,7 @@ int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock, 
			if (urg_offset < used) {

				if (!urg_offset) {

					if (!sock_flag(sk, SOCK_URGINLINE)) {

						++*seq;

						WRITE_ONCE(*seq, *seq + 1);

						urg_hole++;

						offset++;

						used--;
@@ -2139,7 +2139,7 @@ int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock, 
			}

		}



		*seq += used;

		WRITE_ONCE(*seq, *seq + used);

		copied += used;

		len -= used;
@@ -2166,7 +2166,7 @@ int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock, 


found_fin_ok:

		/* Process the FIN. */

		++*seq;

		WRITE_ONCE(*seq, *seq + 1);

		if (!(flags & MSG_PEEK))

			sk_eat_skb(sk, skb);

		break;
@@ -2588,7 +2588,7 @@ int tcp_disconnect(struct sock *sk, int flags) 
		__kfree_skb(sk->sk_rx_skb_cache);

		sk->sk_rx_skb_cache = NULL;

	}

	tp->copied_seq = tp->rcv_nxt;

	WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);

	tp->urg_data = 0;

	tcp_write_queue_purge(sk);

	tcp_fastopen_active_disable_ofo_check(sk);

net/ipv4/tcp_diag.c

+2 −1
Original line number Diff line number Diff line
@@ -26,7 +26,8 @@ static void tcp_diag_get_info(struct sock *sk, struct inet_diag_msg *r, 
	} else if (sk->sk_type == SOCK_STREAM) {

		const struct tcp_sock *tp = tcp_sk(sk);



		r->idiag_rqueue = max_t(int, READ_ONCE(tp->rcv_nxt) - tp->copied_seq, 0);

		r->idiag_rqueue = max_t(int, READ_ONCE(tp->rcv_nxt) -

					     READ_ONCE(tp->copied_seq), 0);

		r->idiag_wqueue = tp->write_seq - tp->snd_una;

	}

	if (info)

net/ipv4/tcp_input.c

+3 −3
Original line number Diff line number Diff line
@@ -5961,7 +5961,7 @@ static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb, 
		/* Remember, tcp_poll() does not lock socket!

		 * Change state from SYN-SENT only after copied_seq

		 * is initialized. */

		tp->copied_seq = tp->rcv_nxt;

		WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);



		smc_check_reset_syn(tp);
@@ -6036,7 +6036,7 @@ static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb, 
		}



		WRITE_ONCE(tp->rcv_nxt, TCP_SKB_CB(skb)->seq + 1);

		tp->copied_seq = tp->rcv_nxt;

		WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);

		tp->rcv_wup = TCP_SKB_CB(skb)->seq + 1;



		/* RFC1323: The window in SYN & SYN/ACK segments is
@@ -6216,7 +6216,7 @@ int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb) 
			tcp_try_undo_spurious_syn(sk);

			tp->retrans_stamp = 0;

			tcp_init_transfer(sk, BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB);

			tp->copied_seq = tp->rcv_nxt;

			WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);

		}

		smp_mb();

		tcp_set_state(sk, TCP_ESTABLISHED);

net/ipv4/tcp_ipv4.c

+1 −1
Original line number Diff line number Diff line
@@ -2456,7 +2456,7 @@ static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i) 
		 * we might find a transient negative value.

		 */

		rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -

				      tp->copied_seq, 0);

				      READ_ONCE(tp->copied_seq), 0);



	seq_printf(f, "%4d: %08X:%04X %08X:%04X %02X %08X:%08X %02X:%08lX "

			"%08X %5u %8d %lu %d %pK %lu %lu %u %u %d",

net/ipv4/tcp_minisocks.c

+1 −1
Original line number Diff line number Diff line
@@ -478,7 +478,7 @@ struct sock *tcp_create_openreq_child(const struct sock *sk, 


	seq = treq->rcv_isn + 1;

	newtp->rcv_wup = seq;

	newtp->copied_seq = seq;

	WRITE_ONCE(newtp->copied_seq, seq);

	WRITE_ONCE(newtp->rcv_nxt, seq);

	newtp->segs_in = 1;