Commit d6d91e46 authored by Weihang Li's avatar Weihang Li Committed by Jason Gunthorpe
Browse files

RDMA/hns: Add support for configuring GMV table 

HIP09 supports to store SGID/SMAC/VLAN together in a table named GMV. The
driver needs to allocate memory for it and tell the information about this
region to hardware.

Link: https://lore.kernel.org/r/1603508836-33054-2-git-send-email-liweihang@huawei.com


Signed-off-by: default avatarWeihang Li <liweihang@huawei.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@nvidia.com>
parent 3f4e3d96

drivers/infiniband/hw/hns/hns_roce_device.h

+10 −0
Original line number Diff line number Diff line
@@ -827,6 +827,7 @@ struct hns_roce_caps { 
	u32		cqc_timer_bt_num;

	u32		mpt_bt_num;

	u32		sccc_bt_num;

	u32		gmv_bt_num;

	u32		qpc_ba_pg_sz;

	u32		qpc_buf_pg_sz;

	u32		qpc_hop_num;
@@ -866,6 +867,11 @@ struct hns_roce_caps { 
	u32		eqe_ba_pg_sz;

	u32		eqe_buf_pg_sz;

	u32		eqe_hop_num;

	u32		gmv_entry_num;

	u32		gmv_entry_sz;

	u32		gmv_ba_pg_sz;

	u32		gmv_buf_pg_sz;

	u32		gmv_hop_num;

	u32		sl_num;

	u32		tsq_buf_pg_sz;

	u32		tpq_buf_pg_sz;
@@ -1001,6 +1007,10 @@ struct hns_roce_dev { 
	struct hns_roce_eq_table  eq_table;

	struct hns_roce_hem_table  qpc_timer_table;

	struct hns_roce_hem_table  cqc_timer_table;

	/* GMV is the memory area that the driver allocates for the hardware

	 * to store SGID, SMAC and VLAN information.

	 */

	struct hns_roce_hem_table  gmv_table;



	int			cmd_mod;

	int			loop_idc;

drivers/infiniband/hw/hns/hns_roce_hem.c

+15 −0
Original line number Diff line number Diff line
@@ -75,6 +75,9 @@ bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) 
	case HEM_TYPE_CQC_TIMER:

		hop_num = hr_dev->caps.cqc_timer_hop_num;

		break;

	case HEM_TYPE_GMV:

		hop_num = hr_dev->caps.gmv_hop_num;

		break;

	default:

		return false;

	}
@@ -183,6 +186,14 @@ static int get_hem_table_config(struct hns_roce_dev *hr_dev, 
		mhop->ba_l0_num = hr_dev->caps.srqc_bt_num;

		mhop->hop_num = hr_dev->caps.srqc_hop_num;

		break;

	case HEM_TYPE_GMV:

		mhop->buf_chunk_size = 1 << (hr_dev->caps.gmv_buf_pg_sz +

					     PAGE_SHIFT);

		mhop->bt_chunk_size = 1 << (hr_dev->caps.gmv_ba_pg_sz +

					    PAGE_SHIFT);

		mhop->ba_l0_num = hr_dev->caps.gmv_bt_num;

		mhop->hop_num = hr_dev->caps.gmv_hop_num;

		break;

	default:

		dev_err(dev, "Table %d not support multi-hop addressing!\n",

			type);
@@ -1033,6 +1044,10 @@ void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev) 
	if (hr_dev->caps.trrl_entry_sz)

		hns_roce_cleanup_hem_table(hr_dev,

					   &hr_dev->qp_table.trrl_table);



	if (hr_dev->caps.gmv_entry_sz)

		hns_roce_cleanup_hem_table(hr_dev, &hr_dev->gmv_table);



	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);

	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);

	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);

drivers/infiniband/hw/hns/hns_roce_hem.h

+1 −0
Original line number Diff line number Diff line
@@ -47,6 +47,7 @@ enum { 
	HEM_TYPE_SCCC,

	HEM_TYPE_QPC_TIMER,

	HEM_TYPE_CQC_TIMER,

	HEM_TYPE_GMV,



	 /* UNMAP HEM */

	HEM_TYPE_MTT,

drivers/infiniband/hw/hns/hns_roce_hw_v2.c

+89 −26
Original line number Diff line number Diff line
@@ -1589,6 +1589,10 @@ static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev) 
					     PF_RES_DATA_4_PF_SCCC_BT_NUM_M,

					     PF_RES_DATA_4_PF_SCCC_BT_NUM_S);



	hr_dev->caps.gmv_bt_num = roce_get_field(req_b->gmv_idx_num,

						 PF_RES_DATA_5_PF_GMV_BT_NUM_M,

						 PF_RES_DATA_5_PF_GMV_BT_NUM_S);



	return 0;

}
@@ -1912,6 +1916,15 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) 
		caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE;

		caps->cqe_sz = HNS_ROCE_V3_CQE_SIZE;

		caps->qpc_sz = HNS_ROCE_V3_QPC_SZ;

		caps->sccc_sz = HNS_ROCE_V3_SCCC_SZ;

		caps->gmv_entry_sz = HNS_ROCE_V3_GMV_ENTRY_SZ;

		caps->gmv_entry_num = caps->gmv_bt_num * (PAGE_SIZE /

							  caps->gmv_entry_sz);

		caps->gmv_hop_num = HNS_ROCE_HOP_NUM_0;

		caps->gmv_ba_pg_sz = 0;

		caps->gmv_buf_pg_sz = 0;

		caps->gid_table_len[0] = caps->gmv_bt_num * (HNS_HW_PAGE_SIZE /

					 caps->gmv_entry_sz);

	}

}
@@ -2138,6 +2151,14 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) 
		caps->cqe_sz = HNS_ROCE_V3_CQE_SIZE;

		caps->qpc_sz = HNS_ROCE_V3_QPC_SZ;

		caps->sccc_sz = HNS_ROCE_V3_SCCC_SZ;

		caps->gmv_entry_sz = HNS_ROCE_V3_GMV_ENTRY_SZ;

		caps->gmv_entry_num = caps->gmv_bt_num * (PAGE_SIZE /

						    caps->gmv_entry_sz);

		caps->gmv_hop_num = HNS_ROCE_HOP_NUM_0;

		caps->gmv_ba_pg_sz = 0;

		caps->gmv_buf_pg_sz = 0;

		caps->gid_table_len[0] = caps->gmv_bt_num *

				(HNS_HW_PAGE_SIZE / caps->gmv_entry_sz);

	}



	calc_pg_sz(caps->num_qps, caps->qpc_sz, caps->qpc_hop_num,
@@ -2481,24 +2502,13 @@ static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev, 
			  link_tbl->table.map);

}



static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)

static int get_hem_table(struct hns_roce_dev *hr_dev)

{

	struct hns_roce_v2_priv *priv = hr_dev->priv;

	int qpc_count, cqc_count;

	int ret, i;



	/* TSQ includes SQ doorbell and ack doorbell */

	ret = hns_roce_init_link_table(hr_dev, TSQ_LINK_TABLE);

	if (ret) {

		dev_err(hr_dev->dev, "TSQ init failed, ret = %d.\n", ret);

		return ret;

	}



	ret = hns_roce_init_link_table(hr_dev, TPQ_LINK_TABLE);

	if (ret) {

		dev_err(hr_dev->dev, "TPQ init failed, ret = %d.\n", ret);

		goto err_tpq_init_failed;

	}

	unsigned int qpc_count;

	unsigned int cqc_count;

	unsigned int gmv_count;

	int ret;

	int i;



	/* Alloc memory for QPC Timer buffer space chunk */

	for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num;
@@ -2522,8 +2532,23 @@ static int hns_roce_v2_init(struct hns_roce_dev *hr_dev) 
		}

	}



	/* Alloc memory for GMV(GID/MAC/VLAN) table buffer space chunk */

	for (gmv_count = 0; gmv_count < hr_dev->caps.gmv_entry_num;

	     gmv_count++) {

		ret = hns_roce_table_get(hr_dev, &hr_dev->gmv_table, gmv_count);

		if (ret) {

			dev_err(hr_dev->dev,

				"failed to get gmv table, ret = %d.\n", ret);

			goto err_gmv_failed;

		}

	}



	return 0;



err_gmv_failed:

	for (i = 0; i < gmv_count; i++)

		hns_roce_table_put(hr_dev, &hr_dev->gmv_table, i);



err_cqc_timer_failed:

	for (i = 0; i < cqc_count; i++)

		hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i);
@@ -2532,6 +2557,32 @@ static int hns_roce_v2_init(struct hns_roce_dev *hr_dev) 
	for (i = 0; i < qpc_count; i++)

		hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i);



	return ret;

}



static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)

{

	struct hns_roce_v2_priv *priv = hr_dev->priv;

	int ret;



	/* TSQ includes SQ doorbell and ack doorbell */

	ret = hns_roce_init_link_table(hr_dev, TSQ_LINK_TABLE);

	if (ret) {

		dev_err(hr_dev->dev, "failed to init TSQ, ret = %d.\n", ret);

		return ret;

	}



	ret = hns_roce_init_link_table(hr_dev, TPQ_LINK_TABLE);

	if (ret) {

		dev_err(hr_dev->dev, "failed to init TPQ, ret = %d.\n", ret);

		goto err_tpq_init_failed;

	}



	ret = get_hem_table(hr_dev);

	if (ret)

		goto err_get_hem_table_failed;



err_get_hem_table_failed:

	hns_roce_free_link_table(hr_dev, &priv->tpq);



err_tpq_init_failed:
@@ -3598,9 +3649,25 @@ static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj, u64 bt_ba, 
			 u32 hem_type, int step_idx)

{

	struct hns_roce_cmd_mailbox *mailbox;

	struct hns_roce_cmq_desc desc;

	struct hns_roce_cfg_gmv_bt *gmv_bt =

				(struct hns_roce_cfg_gmv_bt *)desc.data;

	int ret;

	int op;



	if (hem_type == HEM_TYPE_GMV) {

		hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT,

					      false);



		gmv_bt->gmv_ba_l = cpu_to_le32(bt_ba >> HNS_HW_PAGE_SHIFT);

		gmv_bt->gmv_ba_h = cpu_to_le32(bt_ba >> (HNS_HW_PAGE_SHIFT +

							 32));

		gmv_bt->gmv_bt_idx = cpu_to_le32(obj /

			(HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz));



		return hns_roce_cmq_send(hr_dev, &desc, 1);

	}



	op = get_op_for_set_hem(hr_dev, hem_type, step_idx);

	if (op < 0)

		return 0;
@@ -3698,24 +3765,20 @@ static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev, 
	case HEM_TYPE_CQC:

		op = HNS_ROCE_CMD_DESTROY_CQC_BT0;

		break;

	case HEM_TYPE_SCCC:

	case HEM_TYPE_QPC_TIMER:

	case HEM_TYPE_CQC_TIMER:

		break;

	case HEM_TYPE_SRQC:

		op = HNS_ROCE_CMD_DESTROY_SRQC_BT0;

		break;

	case HEM_TYPE_SCCC:

	case HEM_TYPE_QPC_TIMER:

	case HEM_TYPE_CQC_TIMER:

	case HEM_TYPE_GMV:

		return 0;

	default:

		dev_warn(dev, "Table %d not to be destroyed by mailbox!\n",

			 table->type);

		return 0;

	}



	if (table->type == HEM_TYPE_SCCC ||

	    table->type == HEM_TYPE_QPC_TIMER ||

	    table->type == HEM_TYPE_CQC_TIMER)

		return 0;



	op += step_idx;



	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);

drivers/infiniband/hw/hns/hns_roce_hw_v2.h

+27 −2
Original line number Diff line number Diff line
@@ -44,6 +44,7 @@ 
#define HNS_ROCE_VF_SMAC_NUM			32

#define HNS_ROCE_VF_SGID_NUM			32

#define HNS_ROCE_VF_SL_NUM			8

#define HNS_ROCE_VF_GMV_BT_NUM			256



#define HNS_ROCE_V2_MAX_QP_NUM			0x100000

#define HNS_ROCE_V2_MAX_QPC_TIMER_NUM		0x200
@@ -89,6 +90,7 @@ 


#define HNS_ROCE_V2_SCCC_SZ			32

#define HNS_ROCE_V3_SCCC_SZ			64

#define HNS_ROCE_V3_GMV_ENTRY_SZ		32



#define HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ		PAGE_SIZE

#define HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ		PAGE_SIZE
@@ -241,6 +243,7 @@ enum hns_roce_opcode_type { 
	HNS_ROCE_OPC_CLR_SCCC				= 0x8509,

	HNS_ROCE_OPC_QUERY_SCCC				= 0x850a,

	HNS_ROCE_OPC_RESET_SCCC				= 0x850b,

	HNS_ROCE_OPC_CFG_GMV_BT				= 0x8510,

	HNS_SWITCH_PARAMETER_CFG			= 0x1033,

};
@@ -1334,7 +1337,7 @@ struct hns_roce_pf_res_b { 
	__le32	sgid_idx_num;

	__le32	qid_idx_sl_num;

	__le32	sccc_bt_idx_num;

	__le32	rsv;

	__le32	gmv_idx_num;

};



#define PF_RES_DATA_1_PF_SMAC_IDX_S 0
@@ -1361,6 +1364,12 @@ struct hns_roce_pf_res_b { 
#define PF_RES_DATA_4_PF_SCCC_BT_NUM_S 9

#define PF_RES_DATA_4_PF_SCCC_BT_NUM_M GENMASK(17, 9)



#define PF_RES_DATA_5_PF_GMV_BT_IDX_S 0

#define PF_RES_DATA_5_PF_GMV_BT_IDX_M GENMASK(7, 0)



#define PF_RES_DATA_5_PF_GMV_BT_NUM_S 8

#define PF_RES_DATA_5_PF_GMV_BT_NUM_M GENMASK(16, 8)



struct hns_roce_pf_timer_res_a {

	__le32	rsv0;

	__le32	qpc_timer_bt_idx_num;
@@ -1425,7 +1434,7 @@ struct hns_roce_vf_res_b { 
	__le32 vf_sgid_idx_num;

	__le32 vf_qid_idx_sl_num;

	__le32 vf_sccc_idx_num;

	__le32 rsv1;

	__le32 vf_gmv_idx_num;

};



#define VF_RES_B_DATA_0_VF_ID_S 0
@@ -1455,6 +1464,12 @@ struct hns_roce_vf_res_b { 
#define VF_RES_B_DATA_4_VF_SCCC_BT_NUM_S 9

#define VF_RES_B_DATA_4_VF_SCCC_BT_NUM_M GENMASK(17, 9)



#define VF_RES_B_DATA_5_VF_GMV_BT_IDX_S 0

#define VF_RES_B_DATA_5_VF_GMV_BT_IDX_M GENMASK(7, 0)



#define VF_RES_B_DATA_5_VF_GMV_BT_NUM_S 16

#define VF_RES_B_DATA_5_VF_GMV_BT_NUM_M GENMASK(24, 16)



struct hns_roce_vf_switch {

	__le32 rocee_sel;

	__le32 fun_id;
@@ -1577,6 +1592,16 @@ struct hns_roce_cfg_smac_tb { 
#define CFG_SMAC_TB_VF_SMAC_H_S 0

#define CFG_SMAC_TB_VF_SMAC_H_M GENMASK(15, 0)



struct hns_roce_cfg_gmv_bt {

	__le32 gmv_ba_l;

	__le32 gmv_ba_h;

	__le32 gmv_bt_idx;

	__le32 rsv[3];

};



#define CFG_GMV_BA_H_S 0

#define CFG_GMV_BA_H_M GENMASK(19, 0)



#define HNS_ROCE_QUERY_PF_CAPS_CMD_NUM 5

struct hns_roce_query_pf_caps_a {

	u8 number_ports;