Commit 85b26a97 authored by Chengchang Tang's avatar Chengchang Tang Committed by shiyongbang
Browse files

RDMA/hns: Support STARS mode QP 

driver inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/I8C2IP



--------------------------------------------------------------------------

STARS is a HW scheduler. QP in STARS mode will be taken over by STARS's
HW.

This patch supports configuring STARS mode QP. At the same time, a
kernel API rdma_query_qp_db() is provided for querying the QP DB address
taken over by STARS.

Signed-off-by: default avatarChengchang Tang <tangchengchang@huawei.com>
parent 96df0157

drivers/infiniband/hw/hns/hns_roce_device.h

+2 −0
Original line number Diff line number Diff line
@@ -104,6 +104,8 @@ 
#define CQ_BANKID_SHIFT 2

#define CQ_BANKID_MASK GENMASK(1, 0)



#define HNS_ROCE_MEM_BAR 2



enum {

	SERV_TYPE_RC,

	SERV_TYPE_UC,

drivers/infiniband/hw/hns/hns_roce_ext.c

+33 −0
Original line number Diff line number Diff line
@@ -14,6 +14,25 @@ static bool is_hns_roce(struct ib_device *ib_dev) 
	return false;

}



static bool is_hns_roce_vf(struct hns_roce_dev *hr_dev)

{

	return hr_dev->is_vf;

}



bool rdma_support_stars(struct ib_device *ib_dev)

{

	struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);



	if (!is_hns_roce(ib_dev) || is_hns_roce_vf(hr_dev))

		return false;



	if (poe_is_supported(hr_dev))

		return true;



	return false;

}

EXPORT_SYMBOL(rdma_support_stars);



int rdma_register_poe_channel(struct ib_device *ib_dev, u8 channel,

			      u64 poe_addr)

{
@@ -37,3 +56,17 @@ int rdma_unregister_poe_channel(struct ib_device *ib_dev, u8 channel) 
}

EXPORT_SYMBOL(rdma_unregister_poe_channel);



u64 rdma_query_qp_db(struct ib_device *ib_dev, int qp_index)

{

	struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);

	u64 bar_addr;



	if (!rdma_support_stars(ib_dev))

		return 0;



	bar_addr = pci_resource_start(hr_dev->pci_dev, HNS_ROCE_MEM_BAR);

	return bar_addr + hr_dev->sdb_offset +

		DB_REG_OFFSET * hr_dev->priv_uar.index;

}

EXPORT_SYMBOL(rdma_query_qp_db);

drivers/infiniband/hw/hns/hns_roce_ext.h

+13 −0
Original line number Diff line number Diff line
@@ -19,4 +19,17 @@ 
int rdma_register_poe_channel(struct ib_device *ib_dev, u8 channel, u64 poe_addr);

int rdma_unregister_poe_channel(struct ib_device *ib_dev, u8 channel);



/**

 * rdma_support_stars - Helper function to determine whether the

 * current device supports STARS.

 */

bool rdma_support_stars(struct ib_device *ib_dev);



/**

 * rdma_query_qp_db - Helper function to get the doorbell address of this

 * device. Currently, it only supports use in STARS scenarios.

 * @qp_index - QP number.

 */

u64 rdma_query_qp_db(struct ib_device *ib_dev, int qp_index);



#endif

drivers/infiniband/hw/hns/hns_roce_qp.c

+44 −2
Original line number Diff line number Diff line
@@ -1133,6 +1133,45 @@ static void set_congest_param(struct hns_roce_dev *hr_dev, 
	default_congest_type(hr_dev, hr_qp);

}



static bool check_cq_poe_en(struct ib_cq *ib_cq)

{

	struct hns_roce_cq *hr_cq = ib_cq ? to_hr_cq(ib_cq) : NULL;



	return hr_cq && hr_cq->flags & HNS_ROCE_CQ_FLAG_POE_EN;

}



static int set_uqp_create_flag_param(struct hns_roce_dev *hr_dev,

				     struct hns_roce_qp *hr_qp,

				     struct ib_qp_init_attr *init_attr,

				     struct hns_roce_ib_create_qp *ucmd)

{

	struct ib_device *ibdev = &hr_dev->ib_dev;



	if (check_cq_poe_en(init_attr->recv_cq) ||

			check_cq_poe_en(init_attr->send_cq)) {

		if (!(ucmd->create_flags &

					HNS_ROCE_CREATE_QP_FLAGS_STARS_MODE)) {

			ibdev_err(ibdev,

				  "POE CQ only support STARS QP.\n");

			return -EINVAL;

		}

	}



	if (!(ucmd->comp_mask & HNS_ROCE_CREATE_QP_MASK_CREATE_FLAGS))

		return 0;



	if (ucmd->create_flags & HNS_ROCE_CREATE_QP_FLAGS_STARS_MODE) {

		if (!check_cq_poe_en(init_attr->send_cq)) {

			ibdev_err(ibdev,

				  "STARS QP SQ should be bound with POE CQ.\n");

			return -EINVAL;

		}



		hr_qp->en_flags |= HNS_ROCE_QP_CAP_STARS_SQ_MODE;

	}

	return 0;

}



static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,

			struct ib_qp_init_attr *init_attr,

			struct ib_udata *udata,
@@ -1173,10 +1212,13 @@ static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 
						ibucontext);

		hr_qp->config = uctx->config;

		ret = set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, ucmd);



		if (ret)

			ibdev_err(ibdev, "Failed to set user SQ size, ret = %d\n",

				  ret);



		ret = set_uqp_create_flag_param(hr_dev, hr_qp, init_attr, ucmd);

		if (ret)

			return ret;

		set_congest_param(hr_dev, hr_qp, ucmd);

	} else {

		if (init_attr->create_flags &
@@ -1209,7 +1251,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, 
{

	struct hns_roce_ib_create_qp_resp resp = {};

	struct ib_device *ibdev = &hr_dev->ib_dev;

	struct hns_roce_ib_create_qp ucmd;

	struct hns_roce_ib_create_qp ucmd = {};

	int ret;



	mutex_init(&hr_qp->mutex);

include/uapi/rdma/hns-abi.h

+8 −3
Original line number Diff line number Diff line
@@ -86,6 +86,10 @@ enum hns_roce_create_qp_comp_mask { 
	HNS_ROCE_CREATE_QP_MASK_CONGEST_TYPE = 1 << 1,

};



enum hns_roce_create_qp_flags {

	HNS_ROCE_CREATE_QP_FLAGS_STARS_MODE = 1 << 0,

};



enum hns_roce_congest_type_flags {

	HNS_ROCE_CREATE_QP_FLAGS_DCQCN = 1 << 0,

	HNS_ROCE_CREATE_QP_FLAGS_LDCP = 1 << 1,
@@ -102,8 +106,8 @@ struct hns_roce_ib_create_qp { 
	__u8    reserved[4];

	__u8    pageshift;

	__aligned_u64 sdb_addr;

	__aligned_u64 comp_mask;

	__aligned_u64 create_flags;

	__aligned_u64 comp_mask; /* Use enum hns_roce_create_qp_comp_mask */

	__aligned_u64 create_flags; /* Use enum hns_roce_create_qp_flags */

	__aligned_u64 congest_type_flags;

};
@@ -115,10 +119,11 @@ enum hns_roce_qp_cap_flags { 
	HNS_ROCE_QP_CAP_DYNAMIC_CTX_ATTACH = 1 << 4,

	HNS_ROCE_QP_CAP_DIRECT_WQE = 1 << 5,

	HNS_ROCE_QP_CAP_DYNAMIC_CTX_DETACH = 1 << 6,

	HNS_ROCE_QP_CAP_STARS_SQ_MODE = 1 << 7,

};



struct hns_roce_ib_create_qp_resp {

	__aligned_u64 cap_flags;

	__aligned_u64 cap_flags; /* Use enum hns_roce_qp_cap_flags */

	__aligned_u64 dwqe_mmap_key;

};