Commit ebf2e886 authored by David Howells's avatar David Howells Committed by Jakub Kicinski
Browse files

tcp_bpf: Inline do_tcp_sendpages as it's now a wrapper around tcp_sendmsg 



do_tcp_sendpages() is now just a small wrapper around tcp_sendmsg_locked(),
so inline it.  This is part of replacing ->sendpage() with a call to
sendmsg() with MSG_SPLICE_PAGES set.

Signed-off-by: default avatarDavid Howells <dhowells@redhat.com>
cc: John Fastabend <john.fastabend@gmail.com>
cc: Jakub Sitnicki <jakub@cloudflare.com>
cc: David Ahern <dsahern@kernel.org>
cc: Jens Axboe <axboe@kernel.dk>
cc: Matthew Wilcox <willy@infradead.org>
Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent c5c37af6

net/ipv4/tcp_bpf.c

+12 −8
Original line number Diff line number Diff line
@@ -72,11 +72,13 @@ static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes, 
{

	bool apply = apply_bytes;

	struct scatterlist *sge;

	struct msghdr msghdr = { .msg_flags = flags | MSG_SPLICE_PAGES, };

	struct page *page;

	int size, ret = 0;

	u32 off;



	while (1) {

		struct bio_vec bvec;

		bool has_tx_ulp;



		sge = sk_msg_elem(msg, msg->sg.start);
@@ -88,16 +90,18 @@ static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes, 
		tcp_rate_check_app_limited(sk);

retry:

		has_tx_ulp = tls_sw_has_ctx_tx(sk);

		if (has_tx_ulp) {

			flags |= MSG_SENDPAGE_NOPOLICY;

			ret = kernel_sendpage_locked(sk,

						     page, off, size, flags);

		} else {

			ret = do_tcp_sendpages(sk, page, off, size, flags);

		}

		if (has_tx_ulp)

			msghdr.msg_flags |= MSG_SENDPAGE_NOPOLICY;



		if (flags & MSG_SENDPAGE_NOTLAST)

			msghdr.msg_flags |= MSG_MORE;



		bvec_set_page(&bvec, page, size, off);

		iov_iter_bvec(&msghdr.msg_iter, ITER_SOURCE, &bvec, 1, size);

		ret = tcp_sendmsg_locked(sk, &msghdr, size);

		if (ret <= 0)

			return ret;



		if (apply)

			apply_bytes -= ret;

		msg->sg.size -= ret;
@@ -404,7 +408,7 @@ static int tcp_bpf_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 
	long timeo;

	int flags;



	/* Don't let internal do_tcp_sendpages() flags through */

	/* Don't let internal sendpage flags through */

	flags = (msg->msg_flags & ~MSG_SENDPAGE_DECRYPTED);

	flags |= MSG_NO_SHARED_FRAGS;