Commit 6534de1f authored by Cheng Xu's avatar Cheng Xu Committed by Leon Romanovsky
Browse files

RDMA/erdma: Associate QPs/CQs with doorbells for authorization 



For the isolation requirement, each QP/CQ can only issue doorbells from the
allocated mmio space. Configure the relationship between QPs/CQs and
mmio doorbell spaces to hardware in create_qp/create_cq interfaces.

Signed-off-by: default avatarCheng Xu <chengyou@linux.alibaba.com>
Link: https://lore.kernel.org/r/20230606055005.80729-4-chengyou@linux.alibaba.com


Signed-off-by: default avatarLeon Romanovsky <leon@kernel.org>
parent 7e9a1dad

drivers/infiniband/hw/erdma/erdma_hw.h

+16 −1
Original line number Diff line number Diff line
@@ -134,7 +134,7 @@ 


/* CMDQ related. */

#define ERDMA_CMDQ_MAX_OUTSTANDING 128

#define ERDMA_CMDQ_SQE_SIZE 64

#define ERDMA_CMDQ_SQE_SIZE 128



/* cmdq sub module definition. */

enum CMDQ_WQE_SUB_MOD {
@@ -242,8 +242,12 @@ struct erdma_cmdq_ext_db_req { 
/* create_cq cfg1 */

#define ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK GENMASK(31, 16)

#define ERDMA_CMD_CREATE_CQ_MTT_TYPE_MASK BIT(15)

#define ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK BIT(11)

#define ERDMA_CMD_CREATE_CQ_EQN_MASK GENMASK(9, 0)



/* create_cq cfg2 */

#define ERDMA_CMD_CREATE_CQ_DB_CFG_MASK GENMASK(15, 0)



struct erdma_cmdq_create_cq_req {

	u64 hdr;

	u32 cfg0;
@@ -252,6 +256,7 @@ struct erdma_cmdq_create_cq_req { 
	u32 cfg1;

	u64 cq_db_info_addr;

	u32 first_page_offset;

	u32 cfg2;

};



/* regmr/deregmr cfg0 */
@@ -311,6 +316,7 @@ struct erdma_cmdq_modify_qp_req { 


/* create qp cqn_mtt_cfg */

#define ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK GENMASK(31, 28)

#define ERDMA_CMD_CREATE_QP_DB_CFG_MASK BIT(25)

#define ERDMA_CMD_CREATE_QP_CQN_MASK GENMASK(23, 0)



/* create qp mtt_cfg */
@@ -318,6 +324,10 @@ struct erdma_cmdq_modify_qp_req { 
#define ERDMA_CMD_CREATE_QP_MTT_CNT_MASK GENMASK(11, 1)

#define ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK BIT(0)



/* create qp db cfg */

#define ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK GENMASK(31, 16)

#define ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK GENMASK(15, 0)



#define ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK GENMASK_ULL(31, 0)



struct erdma_cmdq_create_qp_req {
@@ -332,6 +342,11 @@ struct erdma_cmdq_create_qp_req { 
	u32 rq_mtt_cfg;

	u64 sq_db_info_dma_addr;

	u64 rq_db_info_dma_addr;



	u64 sq_mtt_entry[3];

	u64 rq_mtt_entry[3];



	u32 db_cfg;

};



struct erdma_cmdq_destroy_qp_req {

drivers/infiniband/hw/erdma/erdma_verbs.c

+25 −6
Original line number Diff line number Diff line
@@ -19,10 +19,11 @@ 
#include "erdma_cm.h"

#include "erdma_verbs.h"



static int create_qp_cmd(struct erdma_dev *dev, struct erdma_qp *qp)

static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp)

{

	struct erdma_cmdq_create_qp_req req;

	struct erdma_dev *dev = to_edev(qp->ibqp.device);

	struct erdma_pd *pd = to_epd(qp->ibqp.pd);

	struct erdma_cmdq_create_qp_req req;

	struct erdma_uqp *user_qp;

	u64 resp0, resp1;

	int err;
@@ -93,6 +94,16 @@ static int create_qp_cmd(struct erdma_dev *dev, struct erdma_qp *qp) 


		req.sq_db_info_dma_addr = user_qp->sq_db_info_dma_addr;

		req.rq_db_info_dma_addr = user_qp->rq_db_info_dma_addr;



		if (uctx->ext_db.enable) {

			req.sq_cqn_mtt_cfg |=

				FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1);

			req.db_cfg =

				FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK,

					   uctx->ext_db.sdb_off) |

				FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK,

					   uctx->ext_db.rdb_off);

		}

	}



	err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0,
@@ -146,11 +157,12 @@ static int regmr_cmd(struct erdma_dev *dev, struct erdma_mr *mr) 
	return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);

}



static int create_cq_cmd(struct erdma_dev *dev, struct erdma_cq *cq)

static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq)

{

	struct erdma_dev *dev = to_edev(cq->ibcq.device);

	struct erdma_cmdq_create_cq_req req;

	u32 page_size;

	struct erdma_mem *mtt;

	u32 page_size;



	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,

				CMDQ_OPCODE_CREATE_CQ);
@@ -192,6 +204,13 @@ static int create_cq_cmd(struct erdma_dev *dev, struct erdma_cq *cq) 


		req.first_page_offset = mtt->page_offset;

		req.cq_db_info_addr = cq->user_cq.db_info_dma_addr;



		if (uctx->ext_db.enable) {

			req.cfg1 |= FIELD_PREP(

				ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1);

			req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK,

					      uctx->ext_db.cdb_off);

		}

	}



	return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
@@ -753,7 +772,7 @@ int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, 
	qp->attrs.state = ERDMA_QP_STATE_IDLE;

	INIT_DELAYED_WORK(&qp->reflush_dwork, erdma_flush_worker);



	ret = create_qp_cmd(dev, qp);

	ret = create_qp_cmd(uctx, qp);

	if (ret)

		goto err_out_cmd;
@@ -1517,7 +1536,7 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 
			goto err_out_xa;

	}



	ret = create_cq_cmd(dev, cq);

	ret = create_cq_cmd(ctx, cq);

	if (ret)

		goto err_free_res;