Commit 51c78a4d authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'splice-net-replace-sendpage-with-sendmsg-msg_splice_pages-part-1' 

David Howells says:

====================
splice, net: Replace sendpage with sendmsg(MSG_SPLICE_PAGES), part 1

Here's the first tranche of patches towards providing a MSG_SPLICE_PAGES
internal sendmsg flag that is intended to replace the ->sendpage() op with
calls to sendmsg().  MSG_SPLICE_PAGES is a hint that tells the protocol
that it should splice the pages supplied if it can and copy them if not.

This will allow splice to pass multiple pages in a single call and allow
certain parts of higher protocols (e.g. sunrpc, iwarp) to pass an entire
message in one go rather than having to send them piecemeal.  This should
also make it easier to handle the splicing of multipage folios.

A helper, skb_splice_from_iter() is provided to do the work of splicing or
copying data from an iterator.  If a page is determined to be unspliceable
(such as being in the slab), then the helper will give an error.

Note that this facility is not made available to userspace and does not
provide any sort of callback.

This set consists of the following parts:

 (1) Define the MSG_SPLICE_PAGES flag and prevent sys_sendmsg() from being
     able to set it.

 (2) Add an extra argument to skb_append_pagefrags() so that something
     other than MAX_SKB_FRAGS can be used (sysctl_max_skb_frags for
     example).

 (3) Add the skb_splice_from_iter() helper to handle splicing pages into
     skbuffs for MSG_SPLICE_PAGES that can be shared by TCP, IP/UDP and
     AF_UNIX.

 (4) Implement MSG_SPLICE_PAGES support in TCP.

 (5) Make do_tcp_sendpages() just wrap sendmsg() and then fold it in to its
     various callers.

 (6) Implement MSG_SPLICE_PAGES support in IP and make udp_sendpage() just
     a wrapper around sendmsg().

 (7) Implement MSG_SPLICE_PAGES support in IP6/UDP6.

 (8) Implement MSG_SPLICE_PAGES support in AF_UNIX.

 (9) Make AF_UNIX copy unspliceable pages.

Link: https://lore.kernel.org/r/20230316152618.711970-1-dhowells@redhat.com/ # v1
Link: https://lore.kernel.org/r/20230329141354.516864-1-dhowells@redhat.com/ # v2
Link: https://lore.kernel.org/r/20230331160914.1608208-1-dhowells@redhat.com/ # v3
Link: https://lore.kernel.org/r/20230405165339.3468808-1-dhowells@redhat.com/ # v4
Link: https://lore.kernel.org/r/20230406094245.3633290-1-dhowells@redhat.com/ # v5
Link: https://lore.kernel.org/r/20230411160902.4134381-1-dhowells@redhat.com/ # v6
Link: https://lore.kernel.org/r/20230515093345.396978-1-dhowells@redhat.com/ # v7
Link: https://lore.kernel.org/r/20230518113453.1350757-1-dhowells@redhat.com/ # v8
====================

Link: https://lore.kernel.org/r/20230522121125.2595254-1-dhowells@redhat.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents 57910a47 57d44a35

drivers/infiniband/sw/siw/siw_qp_tx.c

+12 −5
Original line number Diff line number Diff line
@@ -312,7 +312,7 @@ static int siw_tx_ctrl(struct siw_iwarp_tx *c_tx, struct socket *s, 
}



/*

 * 0copy TCP transmit interface: Use do_tcp_sendpages.

 * 0copy TCP transmit interface: Use MSG_SPLICE_PAGES.

 *

 * Using sendpage to push page by page appears to be less efficient

 * than using sendmsg, even if data are copied.
@@ -323,20 +323,27 @@ static int siw_tx_ctrl(struct siw_iwarp_tx *c_tx, struct socket *s, 
static int siw_tcp_sendpages(struct socket *s, struct page **page, int offset,

			     size_t size)

{

	struct bio_vec bvec;

	struct msghdr msg = {

		.msg_flags = (MSG_MORE | MSG_DONTWAIT | MSG_SENDPAGE_NOTLAST |

			      MSG_SPLICE_PAGES),

	};

	struct sock *sk = s->sk;

	int i = 0, rv = 0, sent = 0,

	    flags = MSG_MORE | MSG_DONTWAIT | MSG_SENDPAGE_NOTLAST;

	int i = 0, rv = 0, sent = 0;



	while (size) {

		size_t bytes = min_t(size_t, PAGE_SIZE - offset, size);



		if (size + offset <= PAGE_SIZE)

			flags = MSG_MORE | MSG_DONTWAIT;

			msg.msg_flags &= ~MSG_SENDPAGE_NOTLAST;



		tcp_rate_check_app_limited(sk);

		bvec_set_page(&bvec, page[i], bytes, offset);

		iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, size);



try_page_again:

		lock_sock(sk);

		rv = do_tcp_sendpages(sk, page[i], offset, bytes, flags);

		rv = tcp_sendmsg_locked(sk, &msg, size);

		release_sock(sk);



		if (rv > 0) {

include/linux/skbuff.h

+4 −1
Original line number Diff line number Diff line
@@ -1383,7 +1383,7 @@ static inline int skb_pad(struct sk_buff *skb, int pad) 
#define dev_kfree_skb(a)	consume_skb(a)



int skb_append_pagefrags(struct sk_buff *skb, struct page *page,

			 int offset, size_t size);

			 int offset, size_t size, size_t max_frags);



struct skb_seq_state {

	__u32		lower_offset;
@@ -5097,5 +5097,8 @@ static inline void skb_mark_for_recycle(struct sk_buff *skb) 
#endif

}



ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter,

			     ssize_t maxsize, gfp_t gfp);



#endif	/* __KERNEL__ */

#endif	/* _LINUX_SKBUFF_H */

include/linux/socket.h

+3 −0
Original line number Diff line number Diff line
@@ -327,6 +327,7 @@ struct ucred { 
					  */



#define MSG_ZEROCOPY	0x4000000	/* Use user data in kernel path */

#define MSG_SPLICE_PAGES 0x8000000	/* Splice the pages from the iterator in sendmsg() */

#define MSG_FASTOPEN	0x20000000	/* Send data in TCP SYN */

#define MSG_CMSG_CLOEXEC 0x40000000	/* Set close_on_exec for file

					   descriptor received through
@@ -337,6 +338,8 @@ struct ucred { 
#define MSG_CMSG_COMPAT	0		/* We never have 32 bit fixups */

#endif



/* Flags to be cleared on entry by sendmsg and sendmmsg syscalls */

#define MSG_INTERNAL_SENDMSG_FLAGS (MSG_SPLICE_PAGES)



/* Setsockoptions(2) level. Thanks to BSD these must match IPPROTO_xxx */

#define SOL_IP		0

include/net/ip.h

+0 −2
Original line number Diff line number Diff line
@@ -220,8 +220,6 @@ int ip_append_data(struct sock *sk, struct flowi4 *fl4, 
		   unsigned int flags);

int ip_generic_getfrag(void *from, char *to, int offset, int len, int odd,

		       struct sk_buff *skb);

ssize_t ip_append_page(struct sock *sk, struct flowi4 *fl4, struct page *page,

		       int offset, size_t size, int flags);

struct sk_buff *__ip_make_skb(struct sock *sk, struct flowi4 *fl4,

			      struct sk_buff_head *queue,

			      struct inet_cork *cork);

include/net/tcp.h

+0 −2
Original line number Diff line number Diff line
@@ -333,8 +333,6 @@ int tcp_sendpage(struct sock *sk, struct page *page, int offset, size_t size, 
		 int flags);

int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset,

			size_t size, int flags);

ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,

		 size_t size, int flags);

int tcp_send_mss(struct sock *sk, int *size_goal, int flags);

void tcp_push(struct sock *sk, int flags, int mss_now, int nonagle,

	      int size_goal);