Commit a7533584 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by David S. Miller
Browse files

net/tcp: optimise io_uring zc ubuf refcounting 



io_uring keeps a reference to ubuf_info during submission, so if
tcp_sendmsg_locked() sees msghdr::msg_ubuf in can be sure the buffer
will be kept alive and doesn't need to additionally pin it.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Reviewed-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent eea96a3e

net/ipv4/tcp.c

+6 −3
Original line number Diff line number Diff line
@@ -1231,7 +1231,6 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) 
	if ((flags & MSG_ZEROCOPY) && size) {

		if (msg->msg_ubuf) {

			uarg = msg->msg_ubuf;

			net_zcopy_get(uarg);

			zc = sk->sk_route_caps & NETIF_F_SG;

		} else if (sock_flag(sk, SOCK_ZEROCOPY)) {

			skb = tcp_write_queue_tail(sk);
@@ -1458,6 +1457,8 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) 
		tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);

	}

out_nopush:

	/* msg->msg_ubuf is pinned by the caller so we don't take extra refs */

	if (uarg && !msg->msg_ubuf)

		net_zcopy_put(uarg);

	return copied + copied_syn;
@@ -1467,6 +1468,8 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) 
	if (copied + copied_syn)

		goto out;

out_err:

	/* msg->msg_ubuf is pinned by the caller so we don't take extra refs */

	if (uarg && !msg->msg_ubuf)

		net_zcopy_put_abort(uarg, true);

	err = sk_stream_error(sk, flags, err);

	/* make sure we wake any epoll edge trigger waiter */