Commit 5df5dd03 authored by David Howells's avatar David Howells Committed by Jakub Kicinski
Browse files

sunrpc: Use sendmsg(MSG_SPLICE_PAGES) rather then sendpage 



When transmitting data, call down into TCP using sendmsg with
MSG_SPLICE_PAGES to indicate that content should be spliced rather than
performing sendpage calls to transmit header, data pages and trailer.

Signed-off-by: default avatarDavid Howells <dhowells@redhat.com>
Acked-by: default avatarChuck Lever <chuck.lever@oracle.com>
cc: Trond Myklebust <trond.myklebust@hammerspace.com>
cc: Anna Schumaker <anna@kernel.org>
cc: Jeff Layton <jlayton@kernel.org>
cc: Jens Axboe <axboe@kernel.dk>
cc: Matthew Wilcox <willy@infradead.org>
Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 345ee3e8

include/linux/sunrpc/svc.h

+5 −6
Original line number Diff line number Diff line
@@ -161,16 +161,15 @@ static inline bool svc_put_not_last(struct svc_serv *serv) 
extern u32 svc_max_payload(const struct svc_rqst *rqstp);



/*

 * RPC Requsts and replies are stored in one or more pages.

 * RPC Requests and replies are stored in one or more pages.

 * We maintain an array of pages for each server thread.

 * Requests are copied into these pages as they arrive.  Remaining

 * pages are available to write the reply into.

 *

 * Pages are sent using ->sendpage so each server thread needs to

 * allocate more to replace those used in sending.  To help keep track

 * of these pages we have a receive list where all pages initialy live,

 * and a send list where pages are moved to when there are to be part

 * of a reply.

 * Pages are sent using ->sendmsg with MSG_SPLICE_PAGES so each server thread

 * needs to allocate more to replace those used in sending.  To help keep track

 * of these pages we have a receive list where all pages initialy live, and a

 * send list where pages are moved to when there are to be part of a reply.

 *

 * We use xdr_buf for holding responses as it fits well with NFS

 * read responses (that have a header, and some data pages, and possibly

net/sunrpc/svcsock.c

+12 −26
Original line number Diff line number Diff line
@@ -1203,13 +1203,14 @@ static int svc_tcp_recvfrom(struct svc_rqst *rqstp) 
static int svc_tcp_send_kvec(struct socket *sock, const struct kvec *vec,

			      int flags)

{

	return kernel_sendpage(sock, virt_to_page(vec->iov_base),

			       offset_in_page(vec->iov_base),

			       vec->iov_len, flags);

	struct msghdr msg = { .msg_flags = MSG_SPLICE_PAGES | flags, };



	iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, vec, 1, vec->iov_len);

	return sock_sendmsg(sock, &msg);

}



/*

 * kernel_sendpage() is used exclusively to reduce the number of

 * MSG_SPLICE_PAGES is used exclusively to reduce the number of

 * copy operations in this path. Therefore the caller must ensure

 * that the pages backing @xdr are unchanging.

 *
@@ -1249,28 +1250,13 @@ static int svc_tcp_sendmsg(struct socket *sock, struct xdr_buf *xdr, 
	if (ret != head->iov_len)

		goto out;



	if (xdr->page_len) {

		unsigned int offset, len, remaining;

		struct bio_vec *bvec;



		bvec = xdr->bvec + (xdr->page_base >> PAGE_SHIFT);

		offset = offset_in_page(xdr->page_base);

		remaining = xdr->page_len;

		while (remaining > 0) {

			len = min(remaining, bvec->bv_len - offset);

			ret = kernel_sendpage(sock, bvec->bv_page,

					      bvec->bv_offset + offset,

					      len, 0);

	msg.msg_flags = MSG_SPLICE_PAGES;

	iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, xdr->bvec,

		      xdr_buf_pagecount(xdr), xdr->page_len);

	ret = sock_sendmsg(sock, &msg);

	if (ret < 0)

		return ret;

	*sentp += ret;

			if (ret != len)

				goto out;

			remaining -= len;

			offset = 0;

			bvec++;

		}

	}



	if (tail->iov_len) {

		ret = svc_tcp_send_kvec(sock, tail, 0);