Unverified Commit c35fcf49 authored by openeuler-ci-bot's avatar openeuler-ci-bot Committed by Gitee
Browse files

!4546 RDMA/hns: Support MR management

Merge Pull Request from: @stinft 
 
Chengchang Tang (5):
   RDMA/hns: Refactor mtr find
   RDMA/hns: Refactor mtr_init_buf_cfg()
   RDMA/hns: Alloc MTR memory before alloc_mtt()
   RDMA/hns: Support flexible umem page size
   RDMA/hns: Support adaptive PBL hopnum
Yunsheng Lin (1):
   RDMA/hns: Simplify 'struct hns_roce_hem' allocation
https://gitee.com/openeuler/kernel/issues/I91AFK 
 
Link:https://gitee.com/openeuler/kernel/pulls/4546

 

Reviewed-by: default avatarChengchang Tang <tangchengchang@huawei.com>
Signed-off-by: default avatarZheng Zengkai <zhengzengkai@huawei.com>
parents 43467830 3eff4a07
Loading
Loading
Loading
Loading
+5 −6
Original line number Diff line number Diff line
@@ -133,14 +133,12 @@ static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)
	struct hns_roce_cq_table *cq_table = &hr_dev->cq_table;
	struct ib_device *ibdev = &hr_dev->ib_dev;
	u64 mtts[MTT_MIN_COUNT] = {};
	dma_addr_t dma_handle;
	int ret;

	ret = hns_roce_mtr_find(hr_dev, &hr_cq->mtr, 0, mtts, ARRAY_SIZE(mtts),
				&dma_handle);
	if (!ret) {
	ret = hns_roce_mtr_find(hr_dev, &hr_cq->mtr, 0, mtts, ARRAY_SIZE(mtts));
	if (ret) {
		ibdev_err(ibdev, "failed to find CQ mtr, ret = %d.\n", ret);
		return -EINVAL;
		return ret;
	}

	/* Get CQC memory HEM(Hardware Entry Memory) table */
@@ -157,7 +155,8 @@ static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)
		goto err_put;
	}

	ret = hns_roce_create_cqc(hr_dev, hr_cq, mtts, dma_handle);
	ret = hns_roce_create_cqc(hr_dev, hr_cq, mtts,
				  hns_roce_get_mtr_ba(&hr_cq->mtr));
	if (ret)
		goto err_xa;

+15 −1
Original line number Diff line number Diff line
@@ -179,6 +179,7 @@ enum {

#define HNS_ROCE_CMD_SUCCESS			1

#define HNS_ROCE_MAX_HOP_NUM			3
/* The minimum page size is 4K for hardware */
#define HNS_HW_PAGE_SHIFT			12
#define HNS_HW_PAGE_SIZE			(1 << HNS_HW_PAGE_SHIFT)
@@ -269,6 +270,11 @@ struct hns_roce_hem_list {
	dma_addr_t root_ba; /* pointer to the root ba table */
};

enum mtr_type {
	MTR_DEFAULT = 0,
	MTR_PBL,
};

struct hns_roce_buf_attr {
	struct {
		size_t	size;  /* region size */
@@ -277,7 +283,10 @@ struct hns_roce_buf_attr {
	unsigned int region_count; /* valid region count */
	unsigned int page_shift;  /* buffer page shift */
	unsigned int user_access; /* umem access flag */
	u64 iova;
	enum mtr_type type;
	bool mtt_only; /* only alloc buffer-required MTT memory */
	bool adaptive; /* adaptive for page_shift and hopnum */
};

struct hns_roce_hem_cfg {
@@ -1152,8 +1161,13 @@ void hns_roce_cmd_use_polling(struct hns_roce_dev *hr_dev);

/* hns roce hw need current block and next block addr from mtt */
#define MTT_MIN_COUNT	 2
static inline dma_addr_t hns_roce_get_mtr_ba(struct hns_roce_mtr *mtr)
{
	return mtr->hem_cfg.root_ba;
}

int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
		      u32 offset, u64 *mtt_buf, int mtt_max, u64 *base_addr);
		      u32 offset, u64 *mtt_buf, int mtt_max);
int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
			struct hns_roce_buf_attr *buf_attr,
			unsigned int page_shift, struct ib_udata *udata,
+19 −76
Original line number Diff line number Diff line
@@ -249,61 +249,34 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
}

static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,
					       int npages,
					       unsigned long hem_alloc_size,
					       gfp_t gfp_mask)
{
	struct hns_roce_hem_chunk *chunk = NULL;
	struct hns_roce_hem *hem;
	struct scatterlist *mem;
	int order;
	void *buf;

	WARN_ON(gfp_mask & __GFP_HIGHMEM);

	order = get_order(hem_alloc_size);
	if (PAGE_SIZE << order != hem_alloc_size) {
		dev_err(hr_dev->dev, "invalid hem_alloc_size: %lu!\n",
			hem_alloc_size);
		return NULL;
	}

	hem = kmalloc(sizeof(*hem),
		      gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
	if (!hem)
		return NULL;

	INIT_LIST_HEAD(&hem->chunk_list);

	order = get_order(hem_alloc_size);

	while (npages > 0) {
		if (!chunk) {
			chunk = kmalloc(sizeof(*chunk),
				gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
			if (!chunk)
				goto fail;

			sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN);
			chunk->npages = 0;
			chunk->nsg = 0;
			memset(chunk->buf, 0, sizeof(chunk->buf));
			list_add_tail(&chunk->list, &hem->chunk_list);
		}

		while (1 << order > npages)
			--order;

		/*
		 * Alloc memory one time. If failed, don't alloc small block
		 * memory, directly return fail.
		 */
		mem = &chunk->mem[chunk->npages];
		buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order,
				&sg_dma_address(mem), gfp_mask);
	buf = dma_alloc_coherent(hr_dev->dev, hem_alloc_size,
				 &hem->dma, gfp_mask);
	if (!buf)
		goto fail;

		chunk->buf[chunk->npages] = buf;
		sg_dma_len(mem) = PAGE_SIZE << order;

		++chunk->npages;
		++chunk->nsg;
		npages -= 1 << order;
	}
	hem->buf = buf;
	hem->size = hem_alloc_size;

	return hem;

@@ -314,20 +287,10 @@ static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,

void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)
{
	struct hns_roce_hem_chunk *chunk, *tmp;
	int i;

	if (!hem)
		return;

	list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) {
		for (i = 0; i < chunk->npages; ++i)
			dma_free_coherent(hr_dev->dev,
				   sg_dma_len(&chunk->mem[i]),
				   chunk->buf[i],
				   sg_dma_address(&chunk->mem[i]));
		kfree(chunk);
	}
	dma_free_coherent(hr_dev->dev, hem->size, hem->buf, hem->dma);

	kfree(hem);
}
@@ -415,7 +378,6 @@ static int alloc_mhop_hem(struct hns_roce_dev *hr_dev,
{
	u32 bt_size = mhop->bt_chunk_size;
	struct device *dev = hr_dev->dev;
	struct hns_roce_hem_iter iter;
	gfp_t flag;
	u64 bt_ba;
	u32 size;
@@ -456,16 +418,15 @@ static int alloc_mhop_hem(struct hns_roce_dev *hr_dev,
	 */
	size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size;
	flag = GFP_KERNEL | __GFP_NOWARN;
	table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT,
						    size, flag);
	table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size, flag);
	if (!table->hem[index->buf]) {
		ret = -ENOMEM;
		goto err_alloc_hem;
	}

	index->inited |= HEM_INDEX_BUF;
	hns_roce_hem_first(table->hem[index->buf], &iter);
	bt_ba = hns_roce_hem_addr(&iter);
	bt_ba = table->hem[index->buf]->dma;

	if (table->type < HEM_TYPE_MTT) {
		if (mhop->hop_num == 2)
			*(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba;
@@ -586,7 +547,6 @@ int hns_roce_table_get(struct hns_roce_dev *hr_dev,
	}

	table->hem[i] = hns_roce_alloc_hem(hr_dev,
				       table->table_chunk_size >> PAGE_SHIFT,
				       table->table_chunk_size,
				       GFP_KERNEL | __GFP_NOWARN);
	if (!table->hem[i]) {
@@ -725,7 +685,6 @@ void *hns_roce_table_find(struct hns_roce_dev *hr_dev,
			  struct hns_roce_hem_table *table,
			  unsigned long obj, dma_addr_t *dma_handle)
{
	struct hns_roce_hem_chunk *chunk;
	struct hns_roce_hem_mhop mhop;
	struct hns_roce_hem *hem;
	unsigned long mhop_obj = obj;
@@ -734,7 +693,6 @@ void *hns_roce_table_find(struct hns_roce_dev *hr_dev,
	int offset, dma_offset;
	void *addr = NULL;
	u32 hem_idx = 0;
	int length;
	int i, j;

	mutex_lock(&table->mutex);
@@ -767,23 +725,8 @@ void *hns_roce_table_find(struct hns_roce_dev *hr_dev,
	if (!hem)
		goto out;

	list_for_each_entry(chunk, &hem->chunk_list, list) {
		for (i = 0; i < chunk->npages; ++i) {
			length = sg_dma_len(&chunk->mem[i]);
			if (dma_handle && dma_offset >= 0) {
				if (length > (u32)dma_offset)
					*dma_handle = sg_dma_address(
						&chunk->mem[i]) + dma_offset;
				dma_offset -= length;
			}

			if (length > (u32)offset) {
				addr = chunk->buf[i] + offset;
				goto out;
			}
			offset -= length;
		}
	}
	*dma_handle = hem->dma + dma_offset;
	addr = hem->buf + offset;

out:
	mutex_unlock(&table->mutex);
+3 −53
Original line number Diff line number Diff line
@@ -56,10 +56,6 @@ enum {
	HEM_TYPE_TRRL,
};

#define HNS_ROCE_HEM_CHUNK_LEN	\
	 ((256 - sizeof(struct list_head) - 2 * sizeof(int)) /	 \
	 (sizeof(struct scatterlist) + sizeof(void *)))

#define check_whether_bt_num_3(type, hop_num) \
	(type < HEM_TYPE_MTT && hop_num == 2)

@@ -72,25 +68,13 @@ enum {
	(type >= HEM_TYPE_MTT && hop_num == 1) || \
	(type >= HEM_TYPE_MTT && hop_num == HNS_ROCE_HOP_NUM_0))

struct hns_roce_hem_chunk {
	struct list_head	 list;
	int			 npages;
	int			 nsg;
	struct scatterlist	 mem[HNS_ROCE_HEM_CHUNK_LEN];
	void			 *buf[HNS_ROCE_HEM_CHUNK_LEN];
};

struct hns_roce_hem {
	struct list_head chunk_list;
	void *buf;
	dma_addr_t dma;
	unsigned long size;
	refcount_t refcount;
};

struct hns_roce_hem_iter {
	struct hns_roce_hem		 *hem;
	struct hns_roce_hem_chunk	 *chunk;
	int				 page_idx;
};

struct hns_roce_hem_mhop {
	u32	hop_num;
	u32	buf_chunk_size;
@@ -133,38 +117,4 @@ void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev,
				 struct hns_roce_hem_list *hem_list,
				 int offset, int *mtt_cnt);

static inline void hns_roce_hem_first(struct hns_roce_hem *hem,
				      struct hns_roce_hem_iter *iter)
{
	iter->hem = hem;
	iter->chunk = list_empty(&hem->chunk_list) ? NULL :
				 list_entry(hem->chunk_list.next,
					    struct hns_roce_hem_chunk, list);
	iter->page_idx = 0;
}

static inline int hns_roce_hem_last(struct hns_roce_hem_iter *iter)
{
	return !iter->chunk;
}

static inline void hns_roce_hem_next(struct hns_roce_hem_iter *iter)
{
	if (++iter->page_idx >= iter->chunk->nsg) {
		if (iter->chunk->list.next == &iter->hem->chunk_list) {
			iter->chunk = NULL;
			return;
		}

		iter->chunk = list_entry(iter->chunk->list.next,
					 struct hns_roce_hem_chunk, list);
		iter->page_idx = 0;
	}
}

static inline dma_addr_t hns_roce_hem_addr(struct hns_roce_hem_iter *iter)
{
	return sg_dma_address(&iter->chunk->mem[iter->page_idx]);
}

#endif /* _HNS_ROCE_HEM_H */
+55 −56
Original line number Diff line number Diff line
@@ -3195,21 +3195,22 @@ static int set_mtpt_pbl(struct hns_roce_dev *hr_dev,
	u64 pages[HNS_ROCE_V2_MAX_INNER_MTPT_NUM] = { 0 };
	struct ib_device *ibdev = &hr_dev->ib_dev;
	dma_addr_t pbl_ba;
	int i, count;
	int ret;
	int i;

	count = hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, pages,
				  min_t(int, ARRAY_SIZE(pages), mr->npages),
				  &pbl_ba);
	if (count < 1) {
		ibdev_err(ibdev, "failed to find PBL mtr, count = %d.\n",
			  count);
		return -ENOBUFS;
	ret = hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, pages,
				min_t(int, ARRAY_SIZE(pages), mr->npages));
	if (ret) {
		ibdev_err(ibdev, "failed to find PBL mtr, ret = %d.\n", ret);
		return ret;
	}

	/* Aligned to the hardware address access unit */
	for (i = 0; i < count; i++)
	for (i = 0; i < ARRAY_SIZE(pages); i++)
		pages[i] >>= 6;

	pbl_ba = hns_roce_get_mtr_ba(&mr->pbl_mtr);

	mpt_entry->pbl_size = cpu_to_le32(mr->npages);
	mpt_entry->pbl_ba_l = cpu_to_le32(pbl_ba >> 3);
	hr_reg_write(mpt_entry, MPT_PBL_BA_H, upper_32_bits(pbl_ba >> 3));
@@ -3308,18 +3309,12 @@ static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,
static int hns_roce_v2_frmr_write_mtpt(struct hns_roce_dev *hr_dev,
				       void *mb_buf, struct hns_roce_mr *mr)
{
	struct ib_device *ibdev = &hr_dev->ib_dev;
	dma_addr_t pbl_ba = hns_roce_get_mtr_ba(&mr->pbl_mtr);
	struct hns_roce_v2_mpt_entry *mpt_entry;
	dma_addr_t pbl_ba = 0;

	mpt_entry = mb_buf;
	memset(mpt_entry, 0, sizeof(*mpt_entry));

	if (hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, NULL, 0, &pbl_ba) < 0) {
		ibdev_err(ibdev, "failed to find frmr mtr.\n");
		return -ENOBUFS;
	}

	hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_FREE);
	hr_reg_write(mpt_entry, MPT_PD, mr->pd);

@@ -4063,7 +4058,6 @@ static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
			       struct hns_roce_hem_table *table, int obj,
			       u32 step_idx)
{
	struct hns_roce_hem_iter iter;
	struct hns_roce_hem_mhop mhop;
	struct hns_roce_hem *hem;
	unsigned long mhop_obj = obj;
@@ -4100,12 +4094,8 @@ static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,

	if (check_whether_last_step(hop_num, step_idx)) {
		hem = table->hem[hem_idx];
		for (hns_roce_hem_first(hem, &iter);
		     !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
			bt_ba = hns_roce_hem_addr(&iter);
			ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type,
					    step_idx);
		}

		ret = set_hem_to_hw(hr_dev, obj, hem->dma, table->type, step_idx);
	} else {
		if (step_idx == 0)
			bt_ba = table->bt_l0_dma_addr[i];
@@ -4346,17 +4336,20 @@ static int config_qp_rq_buf(struct hns_roce_dev *hr_dev,
{
	u64 mtts[MTT_MIN_COUNT] = { 0 };
	u64 wqe_sge_ba;
	int count;
	int ret;

	/* Search qp buf's mtts */
	count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, mtts,
				  MTT_MIN_COUNT, &wqe_sge_ba);
	if (hr_qp->rq.wqe_cnt && count < 1) {
	ret = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, mtts,
				MTT_MIN_COUNT);
	if (hr_qp->rq.wqe_cnt && ret) {
		ibdev_err(&hr_dev->ib_dev,
			  "failed to find RQ WQE, QPN = 0x%lx.\n", hr_qp->qpn);
		return -EINVAL;
			  "failed to find QP(0x%lx) RQ WQE buf, ret = %d.\n",
			  hr_qp->qpn, ret);
		return ret;
	}

	wqe_sge_ba = hns_roce_get_mtr_ba(&hr_qp->mtr);

	context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> 3);
	qpc_mask->wqe_sge_ba = 0;

@@ -4418,23 +4411,23 @@ static int config_qp_sq_buf(struct hns_roce_dev *hr_dev,
	struct ib_device *ibdev = &hr_dev->ib_dev;
	u64 sge_cur_blk = 0;
	u64 sq_cur_blk = 0;
	int count;
	int ret;

	/* search qp buf's mtts */
	count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, &sq_cur_blk, 1, NULL);
	if (count < 1) {
		ibdev_err(ibdev, "failed to find QP(0x%lx) SQ buf.\n",
			  hr_qp->qpn);
		return -EINVAL;
	ret = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->sq.offset,
				&sq_cur_blk, 1);
	if (ret) {
		ibdev_err(ibdev, "failed to find QP(0x%lx) SQ WQE buf, ret = %d.\n",
			  hr_qp->qpn, ret);
		return ret;
	}
	if (hr_qp->sge.sge_cnt > 0) {
		count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
					  hr_qp->sge.offset,
					  &sge_cur_blk, 1, NULL);
		if (count < 1) {
			ibdev_err(ibdev, "failed to find QP(0x%lx) SGE buf.\n",
				  hr_qp->qpn);
			return -EINVAL;
		ret = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
					hr_qp->sge.offset, &sge_cur_blk, 1);
		if (ret) {
			ibdev_err(ibdev, "failed to find QP(0x%lx) SGE buf, ret = %d.\n",
				  hr_qp->qpn, ret);
			return ret;
		}
	}

@@ -5581,18 +5574,20 @@ static int hns_roce_v2_write_srqc_index_queue(struct hns_roce_srq *srq,
	struct ib_device *ibdev = srq->ibsrq.device;
	struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);
	u64 mtts_idx[MTT_MIN_COUNT] = {};
	dma_addr_t dma_handle_idx = 0;
	dma_addr_t dma_handle_idx;
	int ret;

	/* Get physical address of idx que buf */
	ret = hns_roce_mtr_find(hr_dev, &idx_que->mtr, 0, mtts_idx,
				ARRAY_SIZE(mtts_idx), &dma_handle_idx);
	if (ret < 1) {
				ARRAY_SIZE(mtts_idx));
	if (ret) {
		ibdev_err(ibdev, "failed to find mtr for SRQ idx, ret = %d.\n",
			  ret);
		return -ENOBUFS;
		return ret;
	}

	dma_handle_idx = hns_roce_get_mtr_ba(&idx_que->mtr);

	hr_reg_write(ctx, SRQC_IDX_HOP_NUM,
		     to_hr_hem_hopnum(hr_dev->caps.idx_hop_num, srq->wqe_cnt));

@@ -5624,20 +5619,22 @@ static int hns_roce_v2_write_srqc(struct hns_roce_srq *srq, void *mb_buf)
	struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);
	struct hns_roce_srq_context *ctx = mb_buf;
	u64 mtts_wqe[MTT_MIN_COUNT] = {};
	dma_addr_t dma_handle_wqe = 0;
	dma_addr_t dma_handle_wqe;
	int ret;

	memset(ctx, 0, sizeof(*ctx));

	/* Get the physical address of srq buf */
	ret = hns_roce_mtr_find(hr_dev, &srq->buf_mtr, 0, mtts_wqe,
				ARRAY_SIZE(mtts_wqe), &dma_handle_wqe);
	if (ret < 1) {
				ARRAY_SIZE(mtts_wqe));
	if (ret) {
		ibdev_err(ibdev, "failed to find mtr for SRQ WQE, ret = %d.\n",
			  ret);
		return -ENOBUFS;
		return ret;
	}

	dma_handle_wqe = hns_roce_get_mtr_ba(&srq->buf_mtr);

	hr_reg_write(ctx, SRQC_SRQ_ST, 1);
	hr_reg_write_bool(ctx, SRQC_SRQ_TYPE,
			  srq->ibsrq.srq_type == IB_SRQT_XRC);
@@ -6353,7 +6350,7 @@ static int config_eqc(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq,
	u64 eqe_ba[MTT_MIN_COUNT] = { 0 };
	struct hns_roce_eq_context *eqc;
	u64 bt_ba = 0;
	int count;
	int ret;

	eqc = mb_buf;
	memset(eqc, 0, sizeof(struct hns_roce_eq_context));
@@ -6361,13 +6358,15 @@ static int config_eqc(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq,
	init_eq_config(hr_dev, eq);

	/* if not multi-hop, eqe buffer only use one trunk */
	count = hns_roce_mtr_find(hr_dev, &eq->mtr, 0, eqe_ba, MTT_MIN_COUNT,
				  &bt_ba);
	if (count < 1) {
		dev_err(hr_dev->dev, "failed to find EQE mtr\n");
		return -ENOBUFS;
	ret = hns_roce_mtr_find(hr_dev, &eq->mtr, 0, eqe_ba,
				ARRAY_SIZE(eqe_ba));
	if (ret) {
		dev_err(hr_dev->dev, "failed to find EQE mtr, ret = %d\n", ret);
		return ret;
	}

	bt_ba = hns_roce_get_mtr_ba(&eq->mtr);

	hr_reg_write(eqc, EQC_EQ_ST, HNS_ROCE_V2_EQ_STATE_VALID);
	hr_reg_write(eqc, EQC_EQE_HOP_NUM, eq->hop_num);
	hr_reg_write(eqc, EQC_OVER_IGNORE, eq->over_ignore);
Loading