Commit 00134533 authored by Bob Pearson's avatar Bob Pearson Committed by Jason Gunthorpe
Browse files

RDMA/rxe: Separate HW and SW l/rkeys 

Separate software and simulated hardware lkeys and rkeys for MRs and MWs.
This makes struct ib_mr and struct ib_mw isolated from hardware changes
triggered by executing work requests.

This change fixes a bug seen in blktest.

Link: https://lore.kernel.org/r/20210914164206.19768-4-rpearsonhpe@gmail.com


Signed-off-by: default avatarBob Pearson <rpearsonhpe@gmail.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@nvidia.com>
parent 47b7f706

drivers/infiniband/sw/rxe/rxe_loc.h

+1 −0
Original line number Diff line number Diff line
@@ -86,6 +86,7 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key, 
int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);

int advance_dma_data(struct rxe_dma_info *dma, unsigned int length);

int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey);

int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe);

int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);

void rxe_mr_cleanup(struct rxe_pool_entry *arg);

drivers/infiniband/sw/rxe/rxe_mr.c

+58 −11
Original line number Diff line number Diff line
@@ -53,8 +53,14 @@ static void rxe_mr_init(int access, struct rxe_mr *mr) 
	u32 lkey = mr->pelem.index << 8 | rxe_get_next_key(-1);

	u32 rkey = (access & IB_ACCESS_REMOTE) ? lkey : 0;



	mr->ibmr.lkey = lkey;

	mr->ibmr.rkey = rkey;

	/* set ibmr->l/rkey and also copy into private l/rkey

	 * for user MRs these will always be the same

	 * for cases where caller 'owns' the key portion

	 * they may be different until REG_MR WQE is executed.

	 */

	mr->lkey = mr->ibmr.lkey = lkey;

	mr->rkey = mr->ibmr.rkey = rkey;



	mr->state = RXE_MR_STATE_INVALID;

	mr->map_shift = ilog2(RXE_BUF_PER_MAP);

}
@@ -195,10 +201,8 @@ int rxe_mr_init_fast(struct rxe_pd *pd, int max_pages, struct rxe_mr *mr) 
{

	int err;



	rxe_mr_init(0, mr);



	/* In fastreg, we also set the rkey */

	mr->ibmr.rkey = mr->ibmr.lkey;

	/* always allow remote access for FMRs */

	rxe_mr_init(IB_ACCESS_REMOTE, mr);



	err = rxe_mr_alloc(mr, max_pages);

	if (err)
@@ -511,8 +515,8 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key, 
	if (!mr)

		return NULL;



	if (unlikely((type == RXE_LOOKUP_LOCAL && mr_lkey(mr) != key) ||

		     (type == RXE_LOOKUP_REMOTE && mr_rkey(mr) != key) ||

	if (unlikely((type == RXE_LOOKUP_LOCAL && mr->lkey != key) ||

		     (type == RXE_LOOKUP_REMOTE && mr->rkey != key) ||

		     mr_pd(mr) != pd || (access && !(access & mr->access)) ||

		     mr->state != RXE_MR_STATE_VALID)) {

		rxe_drop_ref(mr);
@@ -535,9 +539,9 @@ int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey) 
		goto err;

	}



	if (rkey != mr->ibmr.rkey) {

		pr_err("%s: rkey (%#x) doesn't match mr->ibmr.rkey (%#x)\n",

			__func__, rkey, mr->ibmr.rkey);

	if (rkey != mr->rkey) {

		pr_err("%s: rkey (%#x) doesn't match mr->rkey (%#x)\n",

			__func__, rkey, mr->rkey);

		ret = -EINVAL;

		goto err_drop_ref;

	}
@@ -558,6 +562,49 @@ int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey) 
	return ret;

}



/* user can (re)register fast MR by executing a REG_MR WQE.

 * user is expected to hold a reference on the ib mr until the

 * WQE completes.

 * Once a fast MR is created this is the only way to change the

 * private keys. It is the responsibility of the user to maintain

 * the ib mr keys in sync with rxe mr keys.

 */

int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe)

{

	struct rxe_mr *mr = to_rmr(wqe->wr.wr.reg.mr);

	u32 key = wqe->wr.wr.reg.key;

	u32 access = wqe->wr.wr.reg.access;



	/* user can only register MR in free state */

	if (unlikely(mr->state != RXE_MR_STATE_FREE)) {

		pr_warn("%s: mr->lkey = 0x%x not free\n",

			__func__, mr->lkey);

		return -EINVAL;

	}



	/* user can only register mr with qp in same protection domain */

	if (unlikely(qp->ibqp.pd != mr->ibmr.pd)) {

		pr_warn("%s: qp->pd and mr->pd don't match\n",

			__func__);

		return -EINVAL;

	}



	/* user is only allowed to change key portion of l/rkey */

	if (unlikely((mr->lkey & ~0xff) != (key & ~0xff))) {

		pr_warn("%s: key = 0x%x has wrong index mr->lkey = 0x%x\n",

			__func__, key, mr->lkey);

		return -EINVAL;

	}



	mr->access = access;

	mr->lkey = key;

	mr->rkey = (access & IB_ACCESS_REMOTE) ? key : 0;

	mr->iova = wqe->wr.wr.reg.mr->iova;

	mr->state = RXE_MR_STATE_VALID;



	return 0;

}



int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)

{

	struct rxe_mr *mr = to_rmr(ibmr);

drivers/infiniband/sw/rxe/rxe_mw.c

+14 −16
Original line number Diff line number Diff line
@@ -21,7 +21,7 @@ int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) 
	}



	rxe_add_index(mw);

	ibmw->rkey = (mw->pelem.index << 8) | rxe_get_next_key(-1);

	mw->rkey = ibmw->rkey = (mw->pelem.index << 8) | rxe_get_next_key(-1);

	mw->state = (mw->ibmw.type == IB_MW_TYPE_2) ?

			RXE_MW_STATE_FREE : RXE_MW_STATE_VALID;

	spin_lock_init(&mw->lock);
@@ -71,6 +71,8 @@ int rxe_dealloc_mw(struct ib_mw *ibmw) 
static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,

			 struct rxe_mw *mw, struct rxe_mr *mr)

{

	u32 key = wqe->wr.wr.mw.rkey & 0xff;



	if (mw->ibmw.type == IB_MW_TYPE_1) {

		if (unlikely(mw->state != RXE_MW_STATE_VALID)) {

			pr_err_once(
@@ -108,7 +110,7 @@ static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe, 
		}

	}



	if (unlikely((wqe->wr.wr.mw.rkey & 0xff) == (mw->ibmw.rkey & 0xff))) {

	if (unlikely(key == (mw->rkey & 0xff))) {

		pr_err_once("attempt to bind MW with same key\n");

		return -EINVAL;

	}
@@ -161,13 +163,9 @@ static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe, 
static void rxe_do_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,

		      struct rxe_mw *mw, struct rxe_mr *mr)

{

	u32 rkey;

	u32 new_rkey;



	rkey = mw->ibmw.rkey;

	new_rkey = (rkey & 0xffffff00) | (wqe->wr.wr.mw.rkey & 0x000000ff);

	u32 key = wqe->wr.wr.mw.rkey & 0xff;



	mw->ibmw.rkey = new_rkey;

	mw->rkey = (mw->rkey & ~0xff) | key;

	mw->access = wqe->wr.wr.mw.access;

	mw->state = RXE_MW_STATE_VALID;

	mw->addr = wqe->wr.wr.mw.addr;
@@ -197,29 +195,29 @@ int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe) 
	struct rxe_mw *mw;

	struct rxe_mr *mr;

	struct rxe_dev *rxe = to_rdev(qp->ibqp.device);

	u32 mw_rkey = wqe->wr.wr.mw.mw_rkey;

	u32 mr_lkey = wqe->wr.wr.mw.mr_lkey;

	unsigned long flags;



	mw = rxe_pool_get_index(&rxe->mw_pool,

				wqe->wr.wr.mw.mw_rkey >> 8);

	mw = rxe_pool_get_index(&rxe->mw_pool, mw_rkey >> 8);

	if (unlikely(!mw)) {

		ret = -EINVAL;

		goto err;

	}



	if (unlikely(mw->ibmw.rkey != wqe->wr.wr.mw.mw_rkey)) {

	if (unlikely(mw->rkey != mw_rkey)) {

		ret = -EINVAL;

		goto err_drop_mw;

	}



	if (likely(wqe->wr.wr.mw.length)) {

		mr = rxe_pool_get_index(&rxe->mr_pool,

					wqe->wr.wr.mw.mr_lkey >> 8);

		mr = rxe_pool_get_index(&rxe->mr_pool, mr_lkey >> 8);

		if (unlikely(!mr)) {

			ret = -EINVAL;

			goto err_drop_mw;

		}



		if (unlikely(mr->ibmr.lkey != wqe->wr.wr.mw.mr_lkey)) {

		if (unlikely(mr->lkey != mr_lkey)) {

			ret = -EINVAL;

			goto err_drop_mr;

		}
@@ -292,7 +290,7 @@ int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey) 
		goto err;

	}



	if (rkey != mw->ibmw.rkey) {

	if (rkey != mw->rkey) {

		ret = -EINVAL;

		goto err_drop_ref;

	}
@@ -323,7 +321,7 @@ struct rxe_mw *rxe_lookup_mw(struct rxe_qp *qp, int access, u32 rkey) 
	if (!mw)

		return NULL;



	if (unlikely((rxe_mw_rkey(mw) != rkey) || rxe_mw_pd(mw) != pd ||

	if (unlikely((mw->rkey != rkey) || rxe_mw_pd(mw) != pd ||

		     (mw->ibmw.type == IB_MW_TYPE_2 && mw->qp != qp) ||

		     (mw->length == 0) ||

		     (access && !(access & mw->access)) ||

drivers/infiniband/sw/rxe/rxe_req.c

+5 −9
Original line number Diff line number Diff line
@@ -562,7 +562,6 @@ static void update_state(struct rxe_qp *qp, struct rxe_send_wqe *wqe, 
static int rxe_do_local_ops(struct rxe_qp *qp, struct rxe_send_wqe *wqe)

{

	u8 opcode = wqe->wr.opcode;

	struct rxe_mr *mr;

	u32 rkey;

	int ret;
@@ -580,14 +579,11 @@ static int rxe_do_local_ops(struct rxe_qp *qp, struct rxe_send_wqe *wqe) 
		}

		break;

	case IB_WR_REG_MR:

		mr = to_rmr(wqe->wr.wr.reg.mr);

		rxe_add_ref(mr);

		mr->state = RXE_MR_STATE_VALID;

		mr->access = wqe->wr.wr.reg.access;

		mr->ibmr.lkey = wqe->wr.wr.reg.key;

		mr->ibmr.rkey = wqe->wr.wr.reg.key;

		mr->iova = wqe->wr.wr.reg.mr->iova;

		rxe_drop_ref(mr);

		ret = rxe_reg_fast_mr(qp, wqe);

		if (unlikely(ret)) {

			wqe->status = IB_WC_LOC_QP_OP_ERR;

			return ret;

		}

		break;

	case IB_WR_BIND_MW:

		ret = rxe_bind_mw(qp, wqe);

drivers/infiniband/sw/rxe/rxe_verbs.h

+3 −15
Original line number Diff line number Diff line
@@ -305,6 +305,8 @@ struct rxe_mr { 


	struct ib_umem		*umem;



	u32			lkey;

	u32			rkey;

	enum rxe_mr_state	state;

	enum ib_mr_type		type;

	u64			va;
@@ -342,6 +344,7 @@ struct rxe_mw { 
	enum rxe_mw_state	state;

	struct rxe_qp		*qp; /* Type 2 only */

	struct rxe_mr		*mr;

	u32			rkey;

	int			access;

	u64			addr;

	u64			length;
@@ -466,26 +469,11 @@ static inline struct rxe_pd *mr_pd(struct rxe_mr *mr) 
	return to_rpd(mr->ibmr.pd);

}



static inline u32 mr_lkey(struct rxe_mr *mr)

{

	return mr->ibmr.lkey;

}



static inline u32 mr_rkey(struct rxe_mr *mr)

{

	return mr->ibmr.rkey;

}



static inline struct rxe_pd *rxe_mw_pd(struct rxe_mw *mw)

{

	return to_rpd(mw->ibmw.pd);

}



static inline u32 rxe_mw_rkey(struct rxe_mw *mw)

{

	return mw->ibmw.rkey;

}



int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name);



void rxe_mc_cleanup(struct rxe_pool_entry *arg);