Commit 8d4ba034 authored by Tom Talpey's avatar Tom Talpey Committed by Trond Myklebust
Browse files

RPC/RDMA: refactor the inline memory registration code. 



Refactor the memory registration and deregistration routines.
This saves stack space, makes the code more readable and prepares
to add the new FRMR registration methods.

Signed-off-by: default avatarTom Talpey <talpey@netapp.com>
Signed-off-by: default avatarTrond Myklebust <Trond.Myklebust@netapp.com>
parent 5e2e7721

net/sunrpc/xprtrdma/verbs.c

+207 −158
Original line number Diff line number Diff line
@@ -863,6 +863,7 @@ rpcrdma_buffer_create(struct rpcrdma_buffer *buf, struct rpcrdma_ep *ep, 
	char *p;

	size_t len;

	int i, rc;

	struct rpcrdma_mw *r;



	buf->rb_max_requests = cdata->max_requests;

	spin_lock_init(&buf->rb_lock);
@@ -873,7 +874,7 @@ rpcrdma_buffer_create(struct rpcrdma_buffer *buf, struct rpcrdma_ep *ep, 
	 *   2.  arrays of struct rpcrdma_req to fill in pointers

	 *   3.  array of struct rpcrdma_rep for replies

	 *   4.  padding, if any

	 *   5.  mw's, if any

	 *   5.  mw's or fmr's, if any

	 * Send/recv buffers in req/rep need to be registered

	 */
@@ -927,15 +928,13 @@ rpcrdma_buffer_create(struct rpcrdma_buffer *buf, struct rpcrdma_ep *ep, 
	 * and also reduce unbind-to-bind collision.

	 */

	INIT_LIST_HEAD(&buf->rb_mws);

	r = (struct rpcrdma_mw *)p;

	switch (ia->ri_memreg_strategy) {

	case RPCRDMA_MTHCAFMR:

		{

		struct rpcrdma_mw *r = (struct rpcrdma_mw *)p;

		struct ib_fmr_attr fa = {

			RPCRDMA_MAX_DATA_SEGS, 1, PAGE_SHIFT

		};

		/* TBD we are perhaps overallocating here */

		for (i = (buf->rb_max_requests+1) * RPCRDMA_MAX_SEGS; i; i--) {

			static struct ib_fmr_attr fa =

				{ RPCRDMA_MAX_DATA_SEGS, 1, PAGE_SHIFT };

			r->r.fmr = ib_alloc_fmr(ia->ri_pd,

				IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ,

				&fa);
@@ -948,12 +947,9 @@ rpcrdma_buffer_create(struct rpcrdma_buffer *buf, struct rpcrdma_ep *ep, 
			list_add(&r->mw_list, &buf->rb_mws);

			++r;

		}

		}

		break;

	case RPCRDMA_MEMWINDOWS_ASYNC:

	case RPCRDMA_MEMWINDOWS:

		{

		struct rpcrdma_mw *r = (struct rpcrdma_mw *)p;

		/* Allocate one extra request's worth, for full cycling */

		for (i = (buf->rb_max_requests+1) * RPCRDMA_MAX_SEGS; i; i--) {

			r->r.mw = ib_alloc_mw(ia->ri_pd);
@@ -966,7 +962,6 @@ rpcrdma_buffer_create(struct rpcrdma_buffer *buf, struct rpcrdma_ep *ep, 
			list_add(&r->mw_list, &buf->rb_mws);

			++r;

		}

		}

		break;

	default:

		break;
@@ -1046,6 +1041,7 @@ rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf) 
{

	int rc, i;

	struct rpcrdma_ia *ia = rdmab_to_ia(buf);

	struct rpcrdma_mw *r;



	/* clean up in reverse order from create

	 *   1.  recv mr memory (mr free, then kfree)
@@ -1065,7 +1061,6 @@ rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf) 
		}

		if (buf->rb_send_bufs && buf->rb_send_bufs[i]) {

			while (!list_empty(&buf->rb_mws)) {

				struct rpcrdma_mw *r;

				r = list_entry(buf->rb_mws.next,

					struct rpcrdma_mw, mw_list);

				list_del(&r->mw_list);
@@ -1115,6 +1110,8 @@ rpcrdma_buffer_get(struct rpcrdma_buffer *buffers) 
{

	struct rpcrdma_req *req;

	unsigned long flags;

	int i;

	struct rpcrdma_mw *r;



	spin_lock_irqsave(&buffers->rb_lock, flags);

	if (buffers->rb_send_index == buffers->rb_max_requests) {
@@ -1135,9 +1132,8 @@ rpcrdma_buffer_get(struct rpcrdma_buffer *buffers) 
	}

	buffers->rb_send_bufs[buffers->rb_send_index++] = NULL;

	if (!list_empty(&buffers->rb_mws)) {

		int i = RPCRDMA_MAX_SEGS - 1;

		i = RPCRDMA_MAX_SEGS - 1;

		do {

			struct rpcrdma_mw *r;

			r = list_entry(buffers->rb_mws.next,

					struct rpcrdma_mw, mw_list);

			list_del(&r->mw_list);
@@ -1329,74 +1325,79 @@ rpcrdma_unmap_one(struct rpcrdma_ia *ia, struct rpcrdma_mr_seg *seg) 
				seg->mr_dma, seg->mr_dmalen, seg->mr_dir);

}



int

rpcrdma_register_external(struct rpcrdma_mr_seg *seg,

			int nsegs, int writing, struct rpcrdma_xprt *r_xprt)

static int

rpcrdma_register_fmr_external(struct rpcrdma_mr_seg *seg,

			int *nsegs, int writing, struct rpcrdma_ia *ia)

{

	struct rpcrdma_ia *ia = &r_xprt->rx_ia;

	int mem_priv = (writing ? IB_ACCESS_REMOTE_WRITE :

				  IB_ACCESS_REMOTE_READ);

	struct rpcrdma_mr_seg *seg1 = seg;

	int i;

	int rc = 0;



	switch (ia->ri_memreg_strategy) {



#if RPCRDMA_PERSISTENT_REGISTRATION

	case RPCRDMA_ALLPHYSICAL:

		rpcrdma_map_one(ia, seg, writing);

		seg->mr_rkey = ia->ri_bind_mem->rkey;

		seg->mr_base = seg->mr_dma;

		seg->mr_nsegs = 1;

		nsegs = 1;

		break;

#endif



	/* Registration using fast memory registration */

	case RPCRDMA_MTHCAFMR:

		{

	u64 physaddrs[RPCRDMA_MAX_DATA_SEGS];

		int len, pageoff = offset_in_page(seg->mr_offset);

	int len, pageoff, i, rc;



	pageoff = offset_in_page(seg1->mr_offset);

	seg1->mr_offset -= pageoff;	/* start of page */

	seg1->mr_len += pageoff;

	len = -pageoff;

		if (nsegs > RPCRDMA_MAX_DATA_SEGS)

			nsegs = RPCRDMA_MAX_DATA_SEGS;

		for (i = 0; i < nsegs;) {

	if (*nsegs > RPCRDMA_MAX_DATA_SEGS)

		*nsegs = RPCRDMA_MAX_DATA_SEGS;

	for (i = 0; i < *nsegs;) {

		rpcrdma_map_one(ia, seg, writing);

		physaddrs[i] = seg->mr_dma;

		len += seg->mr_len;

		++seg;

		++i;

		/* Check for holes */

			if ((i < nsegs && offset_in_page(seg->mr_offset)) ||

		if ((i < *nsegs && offset_in_page(seg->mr_offset)) ||

		    offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len))

			break;

	}

		nsegs = i;

	rc = ib_map_phys_fmr(seg1->mr_chunk.rl_mw->r.fmr,

					physaddrs, nsegs, seg1->mr_dma);

				physaddrs, i, seg1->mr_dma);

	if (rc) {

		dprintk("RPC:       %s: failed ib_map_phys_fmr "

			"%u@0x%llx+%i (%d)... status %i\n", __func__,

			len, (unsigned long long)seg1->mr_dma,

				pageoff, nsegs, rc);

			while (nsegs--)

			pageoff, i, rc);

		while (i--)

			rpcrdma_unmap_one(ia, --seg);

	} else {

		seg1->mr_rkey = seg1->mr_chunk.rl_mw->r.fmr->rkey;

		seg1->mr_base = seg1->mr_dma + pageoff;

			seg1->mr_nsegs = nsegs;

		seg1->mr_nsegs = i;

		seg1->mr_len = len;

	}

	*nsegs = i;

	return rc;

}

		break;



	/* Registration using memory windows */

	case RPCRDMA_MEMWINDOWS_ASYNC:

	case RPCRDMA_MEMWINDOWS:

static int

rpcrdma_deregister_fmr_external(struct rpcrdma_mr_seg *seg,

			struct rpcrdma_ia *ia)

{

	struct rpcrdma_mr_seg *seg1 = seg;

	LIST_HEAD(l);

	int rc;



	list_add(&seg1->mr_chunk.rl_mw->r.fmr->list, &l);

	rc = ib_unmap_fmr(&l);

	while (seg1->mr_nsegs--)

		rpcrdma_unmap_one(ia, seg++);

	if (rc)

		dprintk("RPC:       %s: failed ib_unmap_fmr,"

			" status %i\n", __func__, rc);

	return rc;

}



static int

rpcrdma_register_memwin_external(struct rpcrdma_mr_seg *seg,

			int *nsegs, int writing, struct rpcrdma_ia *ia,

			struct rpcrdma_xprt *r_xprt)

{

	int mem_priv = (writing ? IB_ACCESS_REMOTE_WRITE :

				  IB_ACCESS_REMOTE_READ);

	struct ib_mw_bind param;

	int rc;



	*nsegs = 1;

	rpcrdma_map_one(ia, seg, writing);

	param.mr = ia->ri_bind_mem;

	param.wr_id = 0ULL;	/* no send cookie */
@@ -1406,8 +1407,7 @@ rpcrdma_register_external(struct rpcrdma_mr_seg *seg, 
	param.mw_access_flags = mem_priv;



	DECR_CQCOUNT(&r_xprt->rx_ep);

		rc = ib_bind_mw(ia->ri_id->qp,

					seg->mr_chunk.rl_mw->r.mw, &param);

	rc = ib_bind_mw(ia->ri_id->qp, seg->mr_chunk.rl_mw->r.mw, &param);

	if (rc) {

		dprintk("RPC:       %s: failed ib_bind_mw "

			"%u@0x%llx status %i\n",
@@ -1418,19 +1418,56 @@ rpcrdma_register_external(struct rpcrdma_mr_seg *seg, 
		seg->mr_rkey = seg->mr_chunk.rl_mw->r.mw->rkey;

		seg->mr_base = param.addr;

		seg->mr_nsegs = 1;

			nsegs = 1;

	}

	return rc;

}

		break;



	/* Default registration each time */

	default:

static int

rpcrdma_deregister_memwin_external(struct rpcrdma_mr_seg *seg,

			struct rpcrdma_ia *ia,

			struct rpcrdma_xprt *r_xprt, void **r)

{

	struct ib_mw_bind param;

	LIST_HEAD(l);

	int rc;



	BUG_ON(seg->mr_nsegs != 1);

	param.mr = ia->ri_bind_mem;

	param.addr = 0ULL;	/* unbind */

	param.length = 0;

	param.mw_access_flags = 0;

	if (*r) {

		param.wr_id = (u64) (unsigned long) *r;

		param.send_flags = IB_SEND_SIGNALED;

		INIT_CQCOUNT(&r_xprt->rx_ep);

	} else {

		param.wr_id = 0ULL;

		param.send_flags = 0;

		DECR_CQCOUNT(&r_xprt->rx_ep);

	}

	rc = ib_bind_mw(ia->ri_id->qp, seg->mr_chunk.rl_mw->r.mw, &param);

	rpcrdma_unmap_one(ia, seg);

	if (rc)

		dprintk("RPC:       %s: failed ib_(un)bind_mw,"

			" status %i\n", __func__, rc);

	else

		*r = NULL;	/* will upcall on completion */

	return rc;

}



static int

rpcrdma_register_default_external(struct rpcrdma_mr_seg *seg,

			int *nsegs, int writing, struct rpcrdma_ia *ia)

{

	int mem_priv = (writing ? IB_ACCESS_REMOTE_WRITE :

				  IB_ACCESS_REMOTE_READ);

	struct rpcrdma_mr_seg *seg1 = seg;

	struct ib_phys_buf ipb[RPCRDMA_MAX_DATA_SEGS];

		int len = 0;

		if (nsegs > RPCRDMA_MAX_DATA_SEGS)

			nsegs = RPCRDMA_MAX_DATA_SEGS;

		for (i = 0; i < nsegs;) {

	int len, i, rc = 0;



	if (*nsegs > RPCRDMA_MAX_DATA_SEGS)

		*nsegs = RPCRDMA_MAX_DATA_SEGS;

	for (len = 0, i = 0; i < *nsegs;) {

		rpcrdma_map_one(ia, seg, writing);

		ipb[i].addr = seg->mr_dma;

		ipb[i].size = seg->mr_len;
@@ -1438,28 +1475,80 @@ rpcrdma_register_external(struct rpcrdma_mr_seg *seg, 
		++seg;

		++i;

		/* Check for holes */

			if ((i < nsegs && offset_in_page(seg->mr_offset)) ||

		if ((i < *nsegs && offset_in_page(seg->mr_offset)) ||

		    offset_in_page((seg-1)->mr_offset+(seg-1)->mr_len))

			break;

	}

		nsegs = i;

	seg1->mr_base = seg1->mr_dma;

	seg1->mr_chunk.rl_mr = ib_reg_phys_mr(ia->ri_pd,

					ipb, nsegs, mem_priv, &seg1->mr_base);

				ipb, i, mem_priv, &seg1->mr_base);

	if (IS_ERR(seg1->mr_chunk.rl_mr)) {

		rc = PTR_ERR(seg1->mr_chunk.rl_mr);

		dprintk("RPC:       %s: failed ib_reg_phys_mr "

			"%u@0x%llx (%d)... status %i\n",

			__func__, len,

				(unsigned long long)seg1->mr_dma, nsegs, rc);

			while (nsegs--)

			(unsigned long long)seg1->mr_dma, i, rc);

		while (i--)

			rpcrdma_unmap_one(ia, --seg);

	} else {

		seg1->mr_rkey = seg1->mr_chunk.rl_mr->rkey;

			seg1->mr_nsegs = nsegs;

		seg1->mr_nsegs = i;

		seg1->mr_len = len;

	}

	*nsegs = i;

	return rc;

}



static int

rpcrdma_deregister_default_external(struct rpcrdma_mr_seg *seg,

			struct rpcrdma_ia *ia)

{

	struct rpcrdma_mr_seg *seg1 = seg;

	int rc;



	rc = ib_dereg_mr(seg1->mr_chunk.rl_mr);

	seg1->mr_chunk.rl_mr = NULL;

	while (seg1->mr_nsegs--)

		rpcrdma_unmap_one(ia, seg++);

	if (rc)

		dprintk("RPC:       %s: failed ib_dereg_mr,"

			" status %i\n", __func__, rc);

	return rc;

}



int

rpcrdma_register_external(struct rpcrdma_mr_seg *seg,

			int nsegs, int writing, struct rpcrdma_xprt *r_xprt)

{

	struct rpcrdma_ia *ia = &r_xprt->rx_ia;

	int rc = 0;



	switch (ia->ri_memreg_strategy) {



#if RPCRDMA_PERSISTENT_REGISTRATION

	case RPCRDMA_ALLPHYSICAL:

		rpcrdma_map_one(ia, seg, writing);

		seg->mr_rkey = ia->ri_bind_mem->rkey;

		seg->mr_base = seg->mr_dma;

		seg->mr_nsegs = 1;

		nsegs = 1;

		break;

#endif



	/* Registration using fmr memory registration */

	case RPCRDMA_MTHCAFMR:

		rc = rpcrdma_register_fmr_external(seg, &nsegs, writing, ia);

		break;



	/* Registration using memory windows */

	case RPCRDMA_MEMWINDOWS_ASYNC:

	case RPCRDMA_MEMWINDOWS:

		rc = rpcrdma_register_memwin_external(seg, &nsegs, writing, ia, r_xprt);

		break;



	/* Default registration each time */

	default:

		rc = rpcrdma_register_default_external(seg, &nsegs, writing, ia);

		break;

	}

	if (rc)
@@ -1473,7 +1562,6 @@ rpcrdma_deregister_external(struct rpcrdma_mr_seg *seg, 
		struct rpcrdma_xprt *r_xprt, void *r)

{

	struct rpcrdma_ia *ia = &r_xprt->rx_ia;

	struct rpcrdma_mr_seg *seg1 = seg;

	int nsegs = seg->mr_nsegs, rc;



	switch (ia->ri_memreg_strategy) {
@@ -1487,55 +1575,16 @@ rpcrdma_deregister_external(struct rpcrdma_mr_seg *seg, 
#endif



	case RPCRDMA_MTHCAFMR:

		{

		LIST_HEAD(l);

		list_add(&seg->mr_chunk.rl_mw->r.fmr->list, &l);

		rc = ib_unmap_fmr(&l);

		while (seg1->mr_nsegs--)

			rpcrdma_unmap_one(ia, seg++);

		}

		if (rc)

			dprintk("RPC:       %s: failed ib_unmap_fmr,"

				" status %i\n", __func__, rc);

		rc = rpcrdma_deregister_fmr_external(seg, ia);

		break;



	case RPCRDMA_MEMWINDOWS_ASYNC:

	case RPCRDMA_MEMWINDOWS:

		{

		struct ib_mw_bind param;

		BUG_ON(nsegs != 1);

		param.mr = ia->ri_bind_mem;

		param.addr = 0ULL;	/* unbind */

		param.length = 0;

		param.mw_access_flags = 0;

		if (r) {

			param.wr_id = (u64) (unsigned long) r;

			param.send_flags = IB_SEND_SIGNALED;

			INIT_CQCOUNT(&r_xprt->rx_ep);

		} else {

			param.wr_id = 0ULL;

			param.send_flags = 0;

			DECR_CQCOUNT(&r_xprt->rx_ep);

		}

		rc = ib_bind_mw(ia->ri_id->qp,

				seg->mr_chunk.rl_mw->r.mw, &param);

		rpcrdma_unmap_one(ia, seg);

		}

		if (rc)

			dprintk("RPC:       %s: failed ib_(un)bind_mw,"

				" status %i\n", __func__, rc);

		else

			r = NULL;	/* will upcall on completion */

		rc = rpcrdma_deregister_memwin_external(seg, ia, r_xprt, &r);

		break;



	default:

		rc = ib_dereg_mr(seg1->mr_chunk.rl_mr);

		seg1->mr_chunk.rl_mr = NULL;

		while (seg1->mr_nsegs--)

			rpcrdma_unmap_one(ia, seg++);

		if (rc)

			dprintk("RPC:       %s: failed ib_dereg_mr,"

				" status %i\n", __func__, rc);

		rc = rpcrdma_deregister_default_external(seg, ia);

		break;

	}

	if (r) {