Commit 62ed448c authored by Chuck Lever's avatar Chuck Lever
Browse files

SUNRPC: Optimize xdr_reserve_space() 



Transitioning between encode buffers is quite infrequent. It happens
about 1 time in 400 calls to xdr_reserve_space(), measured on NFSD
with a typical build/test workload.

Force the compiler to remove that code from xdr_reserve_space(),
which is a hot path on both the server and the client. This change
reduces the size of xdr_reserve_space() from 10 cache lines to 2
when compiled with -Os.

Signed-off-by: default avatarChuck Lever <chuck.lever@oracle.com>
Reviewed-by: default avatarJ. Bruce Fields <bfields@fieldses.org>
parent 6c254bf3

include/linux/sunrpc/xdr.h

+15 −1
Original line number Diff line number Diff line
@@ -243,7 +243,7 @@ extern void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf, 
extern __be32 *xdr_reserve_space(struct xdr_stream *xdr, size_t nbytes);

extern int xdr_reserve_space_vec(struct xdr_stream *xdr, struct kvec *vec,

		size_t nbytes);

extern void xdr_commit_encode(struct xdr_stream *xdr);

extern void __xdr_commit_encode(struct xdr_stream *xdr);

extern void xdr_truncate_encode(struct xdr_stream *xdr, size_t len);

extern int xdr_restrict_buflen(struct xdr_stream *xdr, int newbuflen);

extern void xdr_write_pages(struct xdr_stream *xdr, struct page **pages,
@@ -306,6 +306,20 @@ xdr_reset_scratch_buffer(struct xdr_stream *xdr) 
	xdr_set_scratch_buffer(xdr, NULL, 0);

}



/**

 * xdr_commit_encode - Ensure all data is written to xdr->buf

 * @xdr: pointer to xdr_stream

 *

 * Handle encoding across page boundaries by giving the caller a

 * temporary location to write to, then later copying the data into

 * place. __xdr_commit_encode() does that copying.

 */

static inline void xdr_commit_encode(struct xdr_stream *xdr)

{

	if (unlikely(xdr->scratch.iov_len))

		__xdr_commit_encode(xdr);

}



/**

 * xdr_stream_remaining - Return the number of bytes remaining in the stream

 * @xdr: pointer to struct xdr_stream

net/sunrpc/xdr.c

+10 −7
Original line number Diff line number Diff line
@@ -919,7 +919,7 @@ void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p, 
EXPORT_SYMBOL_GPL(xdr_init_encode);



/**

 * xdr_commit_encode - Ensure all data is written to buffer

 * __xdr_commit_encode - Ensure all data is written to buffer

 * @xdr: pointer to xdr_stream

 *

 * We handle encoding across page boundaries by giving the caller a
@@ -931,21 +931,24 @@ EXPORT_SYMBOL_GPL(xdr_init_encode); 
 * required at the end of encoding, or any other time when the xdr_buf

 * data might be read.

 */

inline void xdr_commit_encode(struct xdr_stream *xdr)

void __xdr_commit_encode(struct xdr_stream *xdr)

{

	int shift = xdr->scratch.iov_len;

	void *page;



	if (shift == 0)

		return;

	page = page_address(*xdr->page_ptr);

	memcpy(xdr->scratch.iov_base, page, shift);

	memmove(page, page + shift, (void *)xdr->p - page);

	xdr_reset_scratch_buffer(xdr);

}

EXPORT_SYMBOL_GPL(xdr_commit_encode);

EXPORT_SYMBOL_GPL(__xdr_commit_encode);



static __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr,

/*

 * The buffer space to be reserved crosses the boundary between

 * xdr->buf->head and xdr->buf->pages, or between two pages

 * in xdr->buf->pages.

 */

static noinline __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr,

						   size_t nbytes)

{

	__be32 *p;