Commit fc19a9b6 authored by Yu Zhang's avatar Yu Zhang Committed by JiangShui
Browse files

hns3 udma: support create and destroy jfr. 

driver inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F


CVE: NA

----------------------------------------------------------------

JFR(jetty for receive ) is a jetty which used to post work request(WR)
and receive messages delivered by the WR of sender.

Signed-off-by: default avatarYu Zhang <zhangyu709@huawei.com>
parent 0f04338d

drivers/ub/hw/hns3/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -9,6 +9,6 @@ ccflags-y += -I$(srctree)/drivers/net/ethernet/hisilicon/hns3/ \ 


$(MODULE_NAME)-objs := hns3_udma_hw.o hns3_udma_main.o hns3_udma_cmd.o \

			hns3_udma_hem.o hns3_udma_qp.o hns3_udma_eq.o \

			hns3_udma_db.o hns3_udma_jfc.o

			hns3_udma_db.o hns3_udma_jfc.o hns3_udma_jfr.o



obj-$(CONFIG_UB_UDMA_HNS3) := hns3_udma.o

drivers/ub/hw/hns3/hns3_udma_abi.h

+16 −0
Original line number Diff line number Diff line
@@ -19,6 +19,8 @@ 
#include <linux/types.h>



#define MAP_COMMAND_MASK		0xff

#define UDMA_JETTY_X_PREFIX_BIT_NUM	2

#define UDMA_JFR_QPN_PREFIX		0x1

#define UDMA_ADDR_4K_MASK		0xfffUL



enum {
@@ -40,6 +42,20 @@ enum udma_jfc_notify_mode { 
	UDMA_JFC_NOTIFY_MODE_DDR_4B_ALIGN,

};



struct udma_create_jfr_ucmd {

	uint64_t buf_addr;

	uint64_t idx_addr;

	uint64_t db_addr;

};



enum udma_jfr_cap_flags {

	UDMA_JFR_CAP_RECORD_DB = 1 << 0,

};



struct udma_create_jfr_resp {

	uint32_t jfr_caps;

};



struct udma_jfc_attr_ex {

	uint64_t	jfc_ex_mask; /* Use enum udma_jfc_init_attr_mask */

	uint64_t	create_flags; /* Use enum udma_jfc_create_flags */

drivers/ub/hw/hns3/hns3_udma_common.h

+1 −0
Original line number Diff line number Diff line
@@ -21,6 +21,7 @@ 
#define ub_read(dev, reg)	readl((dev)->reg_base + (reg))



#define UDMA_UDP_DPORT		4791

#define DMA_WQE_SHIFT		3

#define DMA_DB_RECORD_SHIFT	1



#define udma_get_field(origin, mask, shift)                                    \

drivers/ub/hw/hns3/hns3_udma_device.h

+18 −0
Original line number Diff line number Diff line
@@ -45,6 +45,8 @@ 
#define UDMA_GMV_ENTRY_SZ			32



#define UDMA_CQ_BANK_NUM			4

#define UDMA_SGE_SIZE				16

#define UDMA_IDX_QUE_ENTRY_SZ			4

/* The minimum page size is 4K for hardware */

#define UDMA_HW_PAGE_SHIFT			12

#define UDMA_PAGE_SIZE				(1 << UDMA_HW_PAGE_SHIFT)
@@ -186,6 +188,12 @@ enum { 
	TYPE_CSQ = 1

};



enum udma_qp_state {

	QPS_RESET,

	QPS_RTR = 2,

	QPS_RTS,

	QPS_ERR = 6,

};



enum {

	UDMA_BUF_DIRECT = BIT(0),
@@ -747,11 +755,21 @@ static inline struct udma_dev *to_udma_dev(const struct ubcore_device *ubcore_de 
	return container_of(ubcore_dev, struct udma_dev, ub_dev);

}



static inline uint32_t to_udma_hem_entries_size(uint32_t count,

						uint32_t buf_shift)

{

	return udma_hw_page_align(count << buf_shift);

}

static inline uint32_t to_udma_hw_page_shift(uint32_t page_shift)

{

	return page_shift - UDMA_HW_PAGE_SHIFT;

}



static inline uint64_t to_udma_hw_page_addr(uint64_t addr)

{

	return addr >> UDMA_HW_PAGE_SHIFT;

}



static inline dma_addr_t udma_buf_dma_addr(struct udma_buf *buf,

					   uint32_t offset)

{

drivers/ub/hw/hns3/hns3_udma_jfr.c

0 → 100644
+443 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0

/* Huawei UDMA Linux driver

 * Copyright (c) 2023-2023 Hisilicon Limited.

 *

 * This program is free software; you can redistribute it and/or modify it

 * under the terms and conditions of the GNU General Public License,

 * version 2, as published by the Free Software Foundation.

 *

 * This program is distributed in the hope it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License

 * for more details.

 *

 */



#include <linux/slab.h>

#include "hns3_udma_hem.h"

#include "hns3_udma_cmd.h"

#include "hns3_udma_db.h"

#include "hns3_udma_jfc.h"

#include "hns3_udma_jfr.h"



static int init_jfr_cfg(struct udma_dev *dev, struct udma_jfr *jfr,

			const struct ubcore_jfr_cfg *cfg)

{

	if (!cfg->max_sge ||

	    cfg->depth > dev->caps.max_srq_wrs ||

	    cfg->max_sge > dev->caps.max_srq_sges) {

		dev_err(dev->dev, "invalid jfr cfg, depth = %u, sge = %u.\n",

			cfg->depth, cfg->max_sge);

		return -EINVAL;

	}



	jfr->wqe_cnt = roundup_pow_of_two(cfg->depth);



	if (cfg->trans_mode == UBCORE_TP_UM)

		/* reserved for UM header */

		jfr->max_sge = roundup_pow_of_two(cfg->max_sge + 1);

	else

		jfr->max_sge = roundup_pow_of_two(cfg->max_sge);



	memcpy(&jfr->ubcore_jfr.jfr_cfg, cfg, sizeof(struct ubcore_jfr_cfg));

	return 0;

}



static int alloc_jfr_idx(struct udma_dev *dev, struct udma_jfr *jfr,

			 struct ubcore_udata *udata, uint64_t addr)

{

	struct udma_jfr_idx_que *idx_que = &jfr->idx_que;

	struct udma_buf_attr buf_attr = {};

	int ret;



	jfr->idx_que.entry_shift = ilog2(UDMA_IDX_QUE_ENTRY_SZ);



	buf_attr.page_shift = dev->caps.idx_buf_pg_sz + PAGE_SHIFT;

	buf_attr.region[0].size =

		to_udma_hem_entries_size(jfr->wqe_cnt,

					 jfr->idx_que.entry_shift);

	buf_attr.region[0].hopnum = dev->caps.idx_hop_num;

	buf_attr.region_count = 1;



	ret = udma_mtr_create(dev, &idx_que->mtr, &buf_attr,

			      dev->caps.idx_ba_pg_sz + PAGE_SHIFT,

			      addr, !!udata);

	if (ret) {

		dev_err(dev->dev,

			"failed to alloc JFR idx mtr, ret = %d.\n", ret);

		return ret;

	}



	idx_que->head = 0;

	idx_que->tail = 0;



	return 0;

}



static void free_jfr_idx(struct udma_dev *dev, struct udma_jfr *jfr)

{

	struct udma_jfr_idx_que *idx_que = &jfr->idx_que;



	return udma_mtr_destroy(dev, &idx_que->mtr);

}



static int alloc_jfr_wqe_buf(struct udma_dev *dev,

			     struct udma_jfr *jfr,

			     struct ubcore_udata *udata, uint64_t addr)

{

	struct udma_buf_attr buf_attr = {};

	int ret;



	jfr->wqe_shift = ilog2(roundup_pow_of_two(UDMA_SGE_SIZE *

						  jfr->max_sge));



	buf_attr.page_shift = dev->caps.srqwqe_buf_pg_sz + PAGE_SHIFT;

	buf_attr.region[0].size = to_udma_hem_entries_size(jfr->wqe_cnt,

							   jfr->wqe_shift);

	buf_attr.region[0].hopnum = dev->caps.srqwqe_hop_num;

	buf_attr.region_count = 1;



	ret = udma_mtr_create(dev, &jfr->buf_mtr, &buf_attr,

			      dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT,

			      addr, !!udata);

	if (ret)

		dev_err(dev->dev,

			"failed to alloc JFR buf mtr, ret = %d.\n", ret);



	return ret;

}



static void free_jfr_wqe_buf(struct udma_dev *dev, struct udma_jfr *jfr)

{

	udma_mtr_destroy(dev, &jfr->buf_mtr);

}



static int alloc_jfr_buf(struct udma_dev *dev, struct udma_jfr *jfr,

			 struct ubcore_udata *udata)

{

	struct udma_create_jfr_ucmd ucmd = {};

	struct udma_create_jfr_resp resp = {};

	int ret;



	if (udata) {

		ret = copy_from_user(&ucmd, (void *)udata->udrv_data->in_addr,

				     min(udata->udrv_data->in_len,

					 (uint32_t)sizeof(ucmd)));

		if (ret) {

			dev_err(dev->dev,

				"failed to copy JFR udata, ret = %d.\n",

				ret);

			return -EFAULT;

		}

	}



	ret = alloc_jfr_idx(dev, jfr, udata, ucmd.idx_addr);

	if (ret)

		return ret;



	ret = alloc_jfr_wqe_buf(dev, jfr, udata, ucmd.buf_addr);

	if (ret)

		goto err_idx;



	if (dev->caps.flags & UDMA_CAP_FLAG_SRQ_RECORD_DB) {

		ret = udma_db_map_user(dev, ucmd.db_addr, &jfr->db);

		if (ret) {

			dev_err(dev->dev,

				"map jfr db failed, ret = %d.\n", ret);

			goto err_db;

		}

		jfr->jfr_caps |= UDMA_JFR_CAP_RECORD_DB;

	}



	if (udata) {

		resp.jfr_caps = jfr->jfr_caps;

		ret = copy_to_user((void *)udata->udrv_data->out_addr, &resp,

				   min(udata->udrv_data->out_len,

				       (uint32_t)sizeof(resp)));

		if (ret) {

			dev_err(dev->dev,

				"failed to copy jfr resp, ret = %d.\n",

				ret);

			goto err_copy;

		}

	}

	refcount_set(&jfr->refcount, 1);

	init_completion(&jfr->free);

	return 0;



err_copy:

	if (dev->caps.flags & UDMA_CAP_FLAG_SRQ_RECORD_DB) {

		udma_db_unmap_user(dev, &jfr->db);

		jfr->jfr_caps &= ~UDMA_JFR_CAP_RECORD_DB;

	}

err_db:

	free_jfr_wqe_buf(dev, jfr);

err_idx:

	free_jfr_idx(dev, jfr);



	return ret;

}



static int udma_write_jfr_index_queue(struct udma_dev *dev,

				      struct udma_jfr *jfr,

				      struct udma_jfr_context *ctx)

{

#define DMA_IDX_SHIFT 3

	struct udma_jfr_idx_que *idx_que = &jfr->idx_que;

	uint64_t mtts_idx[MTT_MIN_COUNT] = {};

	uint64_t dma_handle_idx = 0;

	int ret;



	/* Get physical address of idx que buf */

	ret = udma_mtr_find(dev, &idx_que->mtr, 0, mtts_idx,

			    ARRAY_SIZE(mtts_idx), &dma_handle_idx);

	if (ret < 1) {

		dev_err(dev->dev, "failed to find mtr for JFR idx, ret = %d.\n",

			ret);

		return -ENOBUFS;

	}



	udma_reg_write(ctx, SRQC_IDX_HOP_NUM,

		       to_udma_hem_hopnum(dev->caps.idx_hop_num, jfr->wqe_cnt));



	udma_reg_write(ctx, SRQC_IDX_BT_BA_L, dma_handle_idx >> DMA_IDX_SHIFT);

	udma_reg_write(ctx, SRQC_IDX_BT_BA_H,

		       upper_32_bits(dma_handle_idx >> DMA_IDX_SHIFT));



	udma_reg_write(ctx, SRQC_IDX_BA_PG_SZ,

		       to_udma_hw_page_shift(idx_que->mtr.hem_cfg.ba_pg_shift));

	udma_reg_write(ctx, SRQC_IDX_BUF_PG_SZ,

		       to_udma_hw_page_shift(idx_que->mtr.hem_cfg.buf_pg_shift));



	udma_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_L,

		       to_udma_hw_page_addr(mtts_idx[0]));

	udma_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_H,

		       upper_32_bits(to_udma_hw_page_addr(mtts_idx[0])));



	udma_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_L,

		       to_udma_hw_page_addr(mtts_idx[1]));

	udma_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_H,

		       upper_32_bits(to_udma_hw_page_addr(mtts_idx[1])));



	return 0;

}



static int write_jfrc(struct udma_dev *dev, struct udma_jfr *jfr, void *mb_buf)

{

	struct udma_jfr_context *ctx = (struct udma_jfr_context *)mb_buf;

	uint64_t mtts_wqe[MTT_MIN_COUNT] = {};

	uint64_t dma_handle_wqe = 0;

	int ret;



	memset(ctx, 0, sizeof(*ctx));



	ret = udma_mtr_find(dev, &jfr->buf_mtr, 0, mtts_wqe,

			    ARRAY_SIZE(mtts_wqe), &dma_handle_wqe);

	if (ret < 1) {

		dev_err(dev->dev, "failed to find mtr for JFR WQE, ret = %d.\n",

			ret);

		return -ENOBUFS;

	}



	udma_reg_write(ctx, SRQC_SRQ_ST, 1);

	udma_reg_write(ctx, SRQC_SRQ_TYPE, 0);

	udma_reg_write(ctx, SRQC_PD, 0);

	udma_reg_write(ctx, SRQC_SRQN, jfr->jfrn);

	udma_reg_write(ctx, SRQC_XRCD, 0);

	udma_reg_write(ctx, SRQC_XRC_RSV, 0);

	udma_reg_write(ctx, SRQC_SHIFT, ilog2(jfr->wqe_cnt));

	udma_reg_write(ctx, SRQC_RQWS,

		       jfr->max_sge <= 0 ? 0 : fls(jfr->max_sge - 1));



	udma_reg_write(ctx, SRQC_WQE_HOP_NUM,

		       to_udma_hem_hopnum(dev->caps.srqwqe_hop_num,

					  jfr->wqe_cnt));



	udma_reg_write(ctx, SRQC_WQE_BT_BA_L, dma_handle_wqe >> DMA_WQE_SHIFT);

	udma_reg_write(ctx, SRQC_WQE_BT_BA_H,

		       upper_32_bits(dma_handle_wqe >> DMA_WQE_SHIFT));



	udma_reg_write(ctx, SRQC_WQE_BA_PG_SZ,

		       to_udma_hw_page_shift(jfr->buf_mtr.hem_cfg.ba_pg_shift));

	udma_reg_write(ctx, SRQC_WQE_BUF_PG_SZ,

		       to_udma_hw_page_shift(jfr->buf_mtr.hem_cfg.buf_pg_shift));

	if (jfr->jfr_caps & UDMA_JFR_CAP_RECORD_DB) {

		udma_reg_enable(ctx, SRQC_RECORD_DB_EN);

		udma_reg_write(ctx, SRQC_RECORD_DB_ADDR_L,

			       lower_32_bits(jfr->db.dma) >>

					     DMA_DB_RECORD_SHIFT);

		udma_reg_write(ctx, SRQC_RECORD_DB_ADDR_H,

			       upper_32_bits(jfr->db.dma));

	}



	return udma_write_jfr_index_queue(dev, jfr, ctx);

}



static int udma_hw_create_srq(struct udma_dev *dev,

			      struct udma_cmd_mailbox *mailbox,

			      uint64_t jfrn)

{

	struct udma_cmq_desc desc;

	struct udma_mbox *mb;



	mb = (struct udma_mbox *)desc.data;

	udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false);

	mbox_desc_init(mb, mailbox->dma, 0, jfrn, UDMA_CMD_CREATE_SRQ);



	return udma_cmd_mbox(dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0);

}



static int alloc_jfrc(struct udma_dev *dev, struct udma_jfr *jfr)

{

	struct udma_jfr_table *jfr_table = &dev->jfr_table;

	struct udma_ida *jfr_ida = &jfr_table->jfr_ida;

	struct udma_cmd_mailbox *mailbox;

	int ret;

	int id;



	id = ida_alloc_range(&jfr_ida->ida, jfr_ida->min, jfr_ida->max,

			     GFP_KERNEL);

	if (id < 0) {

		dev_err(dev->dev, "failed to alloc jfr_id(%d).\n", id);

		return id;

	}

	jfr->jfrn = (uint32_t)id;

	jfr->ubcore_jfr.id = (uint32_t)id;



	ret = udma_table_get(dev, &jfr_table->table, jfr->jfrn);

	if (ret) {

		dev_err(dev->dev, "failed to get JFRC table, ret = %d.\n", ret);

		goto err_ida;

	}



	ret = xa_err(xa_store(&jfr_table->xa, jfr->jfrn, jfr, GFP_KERNEL));

	if (ret) {

		dev_err(dev->dev, "failed to store JFRC, ret = %d.\n", ret);

		goto err_put;

	}



	mailbox = udma_alloc_cmd_mailbox(dev);

	if (IS_ERR_OR_NULL(mailbox)) {

		dev_err(dev->dev, "failed to alloc mailbox for JFRC.\n");

		ret = -ENOMEM;

		goto err_xa;

	}



	ret = write_jfrc(dev, jfr, mailbox->buf);

	if (ret) {

		dev_err(dev->dev, "failed to write JFRC.\n");

		goto err_mbox;

	}



	ret = udma_hw_create_srq(dev, mailbox, jfr->jfrn);

	if (ret) {

		dev_err(dev->dev, "failed to config JFRC, ret = %d.\n", ret);

		goto err_mbox;

	}



	udma_free_cmd_mailbox(dev, mailbox);



	return 0;



err_mbox:

	udma_free_cmd_mailbox(dev, mailbox);

err_xa:

	xa_erase(&jfr_table->xa, jfr->jfrn);

err_put:

	udma_table_put(dev, &jfr_table->table, jfr->jfrn);

err_ida:

	ida_free(&jfr_ida->ida, id);



	return ret;

}



static int udma_hw_destroy_srq(struct udma_dev *dev, uint64_t jfrn)

{

	struct udma_cmq_desc desc;

	struct udma_mbox *mb;



	mb = (struct udma_mbox *)desc.data;

	udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false);

	mbox_desc_init(mb, 0, 0, jfrn, UDMA_CMD_DESTROY_SRQ);



	return udma_cmd_mbox(dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0);

}



static void free_jfrc(struct udma_dev *dev, uint32_t jfrn)

{

	struct udma_jfr_table *jfr_table = &dev->jfr_table;

	int ret;



	ret = udma_hw_destroy_srq(dev, jfrn);

	if (ret)

		dev_err(dev->dev, "destroy failed (%d) for JFRN 0x%06x\n",

			ret, jfrn);



	xa_erase(&jfr_table->xa, jfrn);



	udma_table_put(dev, &jfr_table->table, jfrn);



	ida_free(&jfr_table->jfr_ida.ida, (int)jfrn);

}



static void free_jfr_buf(struct udma_dev *dev, struct udma_jfr *jfr)

{

	if (refcount_dec_and_test(&jfr->refcount))

		complete(&jfr->free);

	wait_for_completion(&jfr->free);



	if (dev->caps.flags & UDMA_CAP_FLAG_SRQ_RECORD_DB)

		udma_db_unmap_user(dev, &jfr->db);

	free_jfr_wqe_buf(dev, jfr);

	free_jfr_idx(dev, jfr);

}



struct ubcore_jfr *udma_create_jfr(struct ubcore_device *dev, const struct ubcore_jfr_cfg *cfg,

			      struct ubcore_udata *udata)

{

	struct udma_dev *udma_dev = to_udma_dev(dev);

	struct udma_jfr *jfr;

	int ret;



	jfr = kcalloc(1, sizeof(*jfr), GFP_KERNEL);

	if (!jfr)

		return NULL;



	ret = init_jfr_cfg(udma_dev, jfr, cfg);

	if (ret)

		goto err_alloc_jfr;



	ret = alloc_jfr_buf(udma_dev, jfr, udata);

	if (ret)

		goto err_alloc_jfr;



	ret = alloc_jfrc(udma_dev, jfr);

	if (ret)

		goto err_alloc_buf;



	xa_init(&jfr->tp_table_xa);

	init_jetty_x_qpn_bitmap(udma_dev, &jfr->qpn_map,

				udma_dev->caps.num_jfr_shift,

				UDMA_JFR_QPN_PREFIX, jfr->jfrn);

	return &jfr->ubcore_jfr;



err_alloc_buf:

	free_jfr_buf(udma_dev, jfr);

err_alloc_jfr:

	kfree(jfr);



	return NULL;

}



int udma_destroy_jfr(struct ubcore_jfr *jfr)

{

	struct udma_dev *dev = to_udma_dev(jfr->ub_dev);

	struct udma_jfr *udma_jfr = to_udma_jfr(jfr);



	clean_jetty_x_qpn_bitmap(&udma_jfr->qpn_map);

	free_jfrc(dev, udma_jfr->jfrn);

	free_jfr_buf(dev, udma_jfr);

	kfree(jfr);



	return 0;

}