Commit 0c3936d3 authored by Arjun Roy's avatar Arjun Roy Committed by Jakub Kicinski
Browse files

net-zerocopy: Set zerocopy hint when data is copied 



Set zerocopy hint, event when falling back to copy, so that the
pending data can be efficiently received using zerocopy when
possible.

Signed-off-by: default avatarArjun Roy <arjunroy@google.com>
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarSoheil Hassas Yeganeh <soheil@google.com>
Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent f21a3c48

net/ipv4/tcp.c

+45 −0
Original line number Diff line number Diff line
@@ -1800,6 +1800,43 @@ static int find_next_mappable_frag(const skb_frag_t *frag, 
	return offset;

}



static void tcp_zerocopy_set_hint_for_skb(struct sock *sk,

					  struct tcp_zerocopy_receive *zc,

					  struct sk_buff *skb, u32 offset)

{

	u32 frag_offset, partial_frag_remainder = 0;

	int mappable_offset;

	skb_frag_t *frag;



	/* worst case: skip to next skb. try to improve on this case below */

	zc->recv_skip_hint = skb->len - offset;



	/* Find the frag containing this offset (and how far into that frag) */

	frag = skb_advance_to_frag(skb, offset, &frag_offset);

	if (!frag)

		return;



	if (frag_offset) {

		struct skb_shared_info *info = skb_shinfo(skb);



		/* We read part of the last frag, must recvmsg() rest of skb. */

		if (frag == &info->frags[info->nr_frags - 1])

			return;



		/* Else, we must at least read the remainder in this frag. */

		partial_frag_remainder = skb_frag_size(frag) - frag_offset;

		zc->recv_skip_hint -= partial_frag_remainder;

		++frag;

	}



	/* partial_frag_remainder: If part way through a frag, must read rest.

	 * mappable_offset: Bytes till next mappable frag, *not* counting bytes

	 * in partial_frag_remainder.

	 */

	mappable_offset = find_next_mappable_frag(frag, zc->recv_skip_hint);

	zc->recv_skip_hint = mappable_offset + partial_frag_remainder;

}



static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len,

			      int nonblock, int flags,

			      struct scm_timestamping_internal *tss,
@@ -1830,6 +1867,14 @@ static int receive_fallback_to_copy(struct sock *sk, 
		return err;



	zc->copybuf_len = err;

	if (likely(zc->copybuf_len)) {

		struct sk_buff *skb;

		u32 offset;



		skb = tcp_recv_skb(sk, tcp_sk(sk)->copied_seq, &offset);

		if (skb)

			tcp_zerocopy_set_hint_for_skb(sk, zc, skb, offset);

	}

	return 0;

}