Commit 2f0f88f4 authored by Chuck Lever's avatar Chuck Lever
Browse files

SUNRPC: Add svc_rqst_replace_page() API 



Replacing a page in rq_pages[] requires a get_page(), which is a
bus-locked operation, and a put_page(), which can be even more
costly.

To reduce the cost of replacing a page in rq_pages[], batch the
put_page() operations by collecting "freed" pages in a pagevec,
and then release those pages when the pagevec is full. This
pagevec is also emptied when each RPC completes.

Signed-off-by: default avatarChuck Lever <chuck.lever@oracle.com>
parent c7e0b781

include/linux/sunrpc/svc.h

+4 −0
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#include <linux/sunrpc/svcauth.h>

#include <linux/wait.h>

#include <linux/mm.h>

#include <linux/pagevec.h>



/* statistics for svc_pool structures */

struct svc_pool_stats {
@@ -256,6 +257,7 @@ struct svc_rqst { 
	struct page *		*rq_next_page; /* next reply page to use */

	struct page *		*rq_page_end;  /* one past the last page */



	struct pagevec		rq_pvec;

	struct kvec		rq_vec[RPCSVC_MAXPAGES]; /* generally useful.. */

	struct bio_vec		rq_bvec[RPCSVC_MAXPAGES];
@@ -502,6 +504,8 @@ struct svc_rqst *svc_rqst_alloc(struct svc_serv *serv, 
					struct svc_pool *pool, int node);

struct svc_rqst *svc_prepare_thread(struct svc_serv *serv,

					struct svc_pool *pool, int node);

void		   svc_rqst_replace_page(struct svc_rqst *rqstp,

					 struct page *page);

void		   svc_rqst_free(struct svc_rqst *);

void		   svc_exit_thread(struct svc_rqst *);

unsigned int	   svc_pool_map_get(void);

net/sunrpc/svc.c

+21 −0
Original line number Diff line number Diff line
@@ -838,6 +838,27 @@ svc_set_num_threads_sync(struct svc_serv *serv, struct svc_pool *pool, int nrser 
}

EXPORT_SYMBOL_GPL(svc_set_num_threads_sync);



/**

 * svc_rqst_replace_page - Replace one page in rq_pages[]

 * @rqstp: svc_rqst with pages to replace

 * @page: replacement page

 *

 * When replacing a page in rq_pages, batch the release of the

 * replaced pages to avoid hammering the page allocator.

 */

void svc_rqst_replace_page(struct svc_rqst *rqstp, struct page *page)

{

	if (*rqstp->rq_next_page) {

		if (!pagevec_space(&rqstp->rq_pvec))

			__pagevec_release(&rqstp->rq_pvec);

		pagevec_add(&rqstp->rq_pvec, *rqstp->rq_next_page);

	}



	get_page(page);

	*(rqstp->rq_next_page++) = page;

}

EXPORT_SYMBOL_GPL(svc_rqst_replace_page);



/*

 * Called from a server thread as it's exiting. Caller must hold the "service

 * mutex" for the service.

net/sunrpc/svc_xprt.c

+3 −0
Original line number Diff line number Diff line
@@ -539,6 +539,7 @@ static void svc_xprt_release(struct svc_rqst *rqstp) 
	kfree(rqstp->rq_deferred);

	rqstp->rq_deferred = NULL;



	pagevec_release(&rqstp->rq_pvec);

	svc_free_res_pages(rqstp);

	rqstp->rq_res.page_len = 0;

	rqstp->rq_res.page_base = 0;
@@ -664,6 +665,8 @@ static int svc_alloc_arg(struct svc_rqst *rqstp) 
	struct xdr_buf *arg = &rqstp->rq_arg;

	unsigned long pages, filled;



	pagevec_init(&rqstp->rq_pvec);



	pages = (serv->sv_max_mesg + 2 * PAGE_SIZE) >> PAGE_SHIFT;

	if (pages > RPCSVC_MAXPAGES) {

		pr_warn_once("svc: warning: pages=%lu > RPCSVC_MAXPAGES=%lu\n",