Commit 1f0cdbbc authored by Weibo Zhao's avatar Weibo Zhao Committed by JiangShui
Browse files

hns3 udma: support of DCA 

driver inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F


CVE: NA

------------------------------------------------------

This patch support DCA mode. DCA(dynamic context attach)
is a feature that can reduce memory usage. Each TP must
have a sending queue to store wqe. In sparse I/O scenarios,
maintaining send queue for each TP is a waste of memory.
DCA provides a way to dynamically expand or shrink the
sending queue. DCA may cause a performance loss.

Signed-off-by: default avatarWeibo Zhao <zhaoweibo3@huawei.com>
parent c1ac9521

drivers/ub/hw/hns3/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -11,6 +11,6 @@ $(MODULE_NAME)-objs := hns3_udma_hw.o hns3_udma_main.o hns3_udma_cmd.o \ 
			hns3_udma_hem.o hns3_udma_qp.o hns3_udma_eq.o \

			hns3_udma_db.o hns3_udma_jfc.o hns3_udma_jfr.o \

			hns3_udma_segment.o  hns3_udma_tp.o hns3_udma_jfs.o \

			hns3_udma_jetty.o hns3_udma_sysfs.o

			hns3_udma_jetty.o hns3_udma_sysfs.o hns3_udma_dca.o



obj-$(CONFIG_UB_UDMA_HNS3) := hns3_udma.o

drivers/ub/hw/hns3/hns3_udma_abi.h

+76 −0
Original line number Diff line number Diff line
@@ -28,11 +28,15 @@ 
#define UDMA_JETTY_QPN_PREFIX		0x2

#define UDMA_ADDR_4K_MASK		0xfffUL

#define URMA_SEG_ACCESS_GUARD		(1UL << 5)

#define UDMA_DCA_ATTACH_FLAGS_NEW_BUFFER BIT(0)

#define UDMA_DCA_INVALID_DCA_NUM ~0U



enum {

	UDMA_MMAP_UAR_PAGE,

	UDMA_MMAP_DWQE_PAGE,

	UDMA_MMAP_DCA_PAGE,

	UDMA_MMAP_RESET_PAGE,

	UDMA_MMAP_TYPE_DCA

};



enum udma_jfc_init_attr_mask {
@@ -114,6 +118,7 @@ enum udma_qp_cap_flags { 
	UDMA_QP_CAP_RQ_RECORD_DB = 1 << 0,

	UDMA_QP_CAP_SQ_RECORD_DB = 1 << 1,

	UDMA_QP_CAP_OWNER_DB = 1 << 2,

	UDMA_QP_CAP_DYNAMIC_CTX_ATTACH = 1 << 4,

	UDMA_QP_CAP_DIRECT_WQE = 1 << 5,

};
@@ -136,6 +141,19 @@ struct udma_create_jfs_resp { 
	struct udma_create_tp_resp create_tp_resp;

};



struct udma_create_ctx_ucmd {

	uint32_t comp;

	uint32_t dca_max_qps;

	uint32_t dca_unit_size;

};



enum udma_context_comp_mask {

	UDMA_CONTEXT_MASK_DCA_PRIME_QPS = 1 << 0,

	UDMA_CONTEXT_MASK_DCA_UNIT_SIZE = 1 << 1,

	UDMA_CONTEXT_MASK_DCA_MAX_SIZE = 1 << 2,

	UDMA_CONTEXT_MASK_DCA_MIN_SIZE = 1 << 3,

};



struct udma_create_ctx_resp {

	uint32_t num_comp_vectors;

	uint32_t num_qps_shift;
@@ -150,6 +168,9 @@ struct udma_create_ctx_resp { 
	uint32_t max_jfs_sge;

	uint32_t poe_ch_num;

	uint64_t db_addr;

	uint32_t dca_qps;

	uint32_t dca_mmap_size;

	uint32_t dca_mode;

};



struct flush_cqe_param {
@@ -163,10 +184,65 @@ struct udma_poe_info { 
	uint64_t	poe_addr;

};



struct udma_dca_reg_attr {

	uintptr_t	key;

	uintptr_t	addr;

	uint32_t	size;

};



struct udma_dca_dereg_attr {

	uintptr_t	free_key;

	struct dca_mem	*mem;

};



struct udma_dca_shrink_attr {

	uint64_t reserved_size;

};



struct udma_dca_shrink_resp {

	struct dca_mem	*mem;

	uintptr_t	free_key;

	uint32_t	free_mems;

};



struct udma_dca_attach_attr {

	uint64_t	qpn;

	uint32_t	sq_offset;

	uint32_t	sge_offset;

};



struct udma_dca_attach_resp {

	uint32_t	alloc_flags;

	uint32_t	alloc_pages;

	uint32_t	dcan;

};



struct udma_dca_detach_attr {

	uint64_t	qpn;

	uint32_t	sq_idx;

};



struct udma_dca_query_attr {

	uint64_t	qpn;

	uint32_t	page_idx;

};



struct udma_dca_query_resp {

	uintptr_t	mem_key;

	uint32_t	mem_ofs;

	uint32_t	page_count;

};



enum udma_user_ctl_handlers {

	UDMA_USER_CTL_FLUSH_CQE,

	UDMA_CONFIG_POE_CHANNEL,

	UDMA_QUERY_POE_CHANNEL,

	UDMA_DCA_MEM_REG,

	UDMA_DCA_MEM_DEREG,

	UDMA_DCA_MEM_SHRINK,

	UDMA_DCA_MEM_ATTACH,

	UDMA_DCA_MEM_DETACH,

	UDMA_DCA_MEM_QUERY,

	UDMA_OPCODE_NUM,

};

drivers/ub/hw/hns3/hns3_udma_dca.c

0 → 100644
+1235 −0

File added.

Preview size limit exceeded, changes collapsed.

drivers/ub/hw/hns3/hns3_udma_dca.h

0 → 100644
+219 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

/* Huawei UDMA Linux driver

 * Copyright (c) 2023-2023 Hisilicon Limited.

 *

 * This program is free software; you can redistribute it and/or modify it

 * under the terms and conditions of the GNU General Public License,

 * version 2, as published by the Free Software Foundation.

 *

 * This program is distributed in the hope it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License

 * for more details.

 *

 */



#ifndef _UDMA_DCA_H

#define _UDMA_DCA_H



#include "hns3_udma_abi.h"

#include "hns3_udma_tp.h"

#include "hns3_udma_hem.h"



#define DCA_MEM_FLAGS_ALLOCED BIT(0)

#define DCA_MEM_FLAGS_REGISTERED BIT(1)



#define DCA_MEM_AGEING_MSES 1000 /* DCA mem ageing interval time */

#define UDMA_DCA_INVALID_BUF_ID 0U

#define DCA_MEM_STOP_ITERATE (-1)

#define DCA_MEM_NEXT_ITERATE (-2)



#define DCAN_TO_SYNC_BIT(n) ((n) * UDMA_DCA_BITS_PER_STATUS)

#define DCAN_TO_STAT_BIT(n) DCAN_TO_SYNC_BIT(n)



#define UDMA_DCA_OWN_MASK GENMASK(21, 0)



/*

 * buffer id(29b) = tag(7b) + owner(22b)

 * [28:22] tag  : indicate the QP config update times.

 * [21: 0] owner: indicate the QP to which the page belongs.

 */

#define UDMA_DCA_ID_MASK GENMASK(28, 0)

#define UDMA_DCA_TAG_MASK GENMASK(28, 22)

#define UDMA_DCA_OWN_MASK GENMASK(21, 0)



#define UDMA_DCA_BUF_ID_TO_TAG(buf_id) (((buf_id) & UDMA_DCA_TAG_MASK) >> 22)

#define UDMA_DCA_BUF_ID_TO_QPN(buf_id) ((buf_id) & UDMA_DCA_OWN_MASK)

#define UDMA_DCA_TO_BUF_ID(qpn, tag) (((qpn) & UDMA_DCA_OWN_MASK) | \

				      (((tag) << 22) & UDMA_DCA_TAG_MASK))



/* DCA page state (32 bit) */

struct dca_page_state {

	uint32_t buf_id		: 29; /* If zero, means page can be used by any buffer. */

	uint32_t lock		: 1; /* @buf_id locked this page to prepare access. */

	uint32_t active		: 1; /* @buf_id is accessing this page. */

	uint32_t head		: 1; /* This page is the head in a continuous address range. */

};



struct dca_mem {

	uint32_t		flags;

	struct list_head	list; /* link to mem list in dca context */

	spinlock_t		lock; /* protect the @flags and @list */

	uint32_t		page_count; /* page count in this mem obj */

	uint64_t		key; /* register by caller */

	uint32_t		size; /* bytes in this mem object */

	struct			dca_page_state *states; /* record each page's state */

	void			*pages; /* memory handle for getting dma address */

};



struct dca_page_free_buf_attr {

	uint32_t buf_id;

	uint32_t max_pages;

	uint32_t free_pages;

	uint32_t clean_mems;

};



struct dca_page_assign_attr {

	uint32_t buf_id;

	uint32_t unit;

	uint32_t total;

	uint32_t max;

};



struct dca_page_clear_attr {

	uint32_t buf_id;

	uint32_t max_pages;

	uint32_t clear_pages;

};



struct dca_get_alloced_pages_attr {

	uint32_t buf_id;

	dma_addr_t *pages;

	uint32_t total;

	uint32_t max;

};



struct dca_page_active_attr {

	uint32_t buf_id;

	uint32_t max_pages;

	uint32_t alloc_pages;

	uint32_t dirty_mems;

};



struct dca_page_query_active_attr {

	uint32_t buf_id;

	uint32_t curr_index;

	uint32_t start_index;

	uint32_t page_index;

	uint32_t page_count;

	uint64_t mem_key;

};



typedef int (*udma_dca_enum_callback)(struct dca_page_state *states,

				      uint32_t count, void *param);



struct dca_mem_enum_attr {

	void *param;

	udma_dca_enum_callback enum_fn;

};



static inline uint64_t umem_cal_npages(uint64_t va, uint64_t len)

{

	return (ALIGN(va + len, UDMA_PAGE_SIZE) - ALIGN_DOWN(va, UDMA_PAGE_SIZE)) /

	       UDMA_PAGE_SIZE;

}



static inline bool dca_page_is_attached(struct dca_page_state *state,

					uint32_t buf_id)

{

	/* only the own bit needs to be matched. */

	return (UDMA_DCA_OWN_MASK & buf_id) ==

	       (UDMA_DCA_OWN_MASK & state->buf_id);

}



static inline bool dca_mem_is_available(struct dca_mem *mem)

{

	return mem->flags == (DCA_MEM_FLAGS_ALLOCED | DCA_MEM_FLAGS_REGISTERED);

}



static inline void set_dca_page_to_free(struct dca_page_state *state)

{

	state->buf_id = UDMA_DCA_INVALID_BUF_ID;

	state->active = 0;

	state->lock = 0;

}



static inline bool dca_page_is_free(struct dca_page_state *state)

{

	return state->buf_id == UDMA_DCA_INVALID_BUF_ID;

}



static inline bool dca_page_is_active(struct dca_page_state *state,

				      uint32_t buf_id)

{

	/* all buf id bits must be matched */

	return (UDMA_DCA_ID_MASK & buf_id) == state->buf_id &&

		!state->lock && state->active;

}



static inline bool dca_page_is_inactive(struct dca_page_state *state)

{

	return !state->lock && !state->active;

}



static inline void lock_dca_page_to_attach(struct dca_page_state *state,

					   uint32_t buf_id)

{

	state->buf_id = UDMA_DCA_ID_MASK & buf_id;

	state->active = 0;

	state->lock = 1;

}



static inline bool dca_page_is_allocated(struct dca_page_state *state,

					 uint32_t buf_id)

{

	return dca_page_is_attached(state, buf_id) && state->lock;

}



static inline void unlock_dca_page_to_active(struct dca_page_state *state,

					     uint32_t buf_id)

{

	state->buf_id = UDMA_DCA_ID_MASK & buf_id;

	state->active = 1;

	state->lock = 0;

}



void udma_enable_dca(struct udma_dev *dev, struct udma_qp *qp);

void udma_disable_dca(struct udma_dev *dev, struct udma_qp *qp);



void udma_modify_dca(struct udma_dev *dev, struct udma_qp *qp);



int udma_register_dca_mem(struct udma_dev *dev, struct udma_ucontext *context,

			  struct udma_dca_reg_attr *attr);

void unregister_dca_mem(struct udma_dev *dev, struct udma_dca_ctx *ctx,

			struct dca_mem *mem);

int udma_deregister_dca_mem(struct udma_dev *dev,

			    struct udma_ucontext *context,

			    struct udma_dca_dereg_attr *attr, bool from_user);



void udma_shrink_dca_mem(struct udma_dev *dev, struct udma_ucontext *context,

			 struct udma_dca_shrink_attr *attr,

			 struct udma_dca_shrink_resp *resp);



int udma_query_dca_mem(struct udma_dev *dev, struct udma_dca_query_attr *attr,

		       struct udma_dca_query_resp *resp);



int udma_dca_attach(struct udma_dev *dev, struct udma_dca_attach_attr *attr,

		    struct udma_dca_attach_resp *resp);

void udma_dca_disattach(struct udma_dev *dev, struct udma_dca_attach_attr *attr);

void udma_dca_detach(struct udma_dev *dev, struct udma_dca_detach_attr *attr);



int udma_register_udca(struct udma_dev *udma_dev,

			  struct udma_ucontext *context, struct ubcore_udrv_priv *udrv_data);



void udma_unregister_udca(struct udma_dev *udma_dev,

			    struct udma_ucontext *context);



void udma_enum_dca_pool(struct udma_dca_ctx *dca_ctx, void *param,

			udma_dca_enum_callback cb);

#endif

drivers/ub/hw/hns3/hns3_udma_device.h

+31 −0
Original line number Diff line number Diff line
@@ -144,6 +144,9 @@ 


#define UDMA_MIN_JFS_DEPTH 64



#define UDMA_DCA_BITS_PER_STATUS 1

#define DCA_BITS_HALF 2



enum {

	NO_ARMED = 0x0

};
@@ -300,6 +303,8 @@ struct udma_buf_attr { 
	} region[UDMA_MAX_BT_REGION];

	uint32_t		region_count; /* valid region count */

	uint32_t		page_shift;  /* buffer page shift */

	/* only alloc buffer-required MTT memory */

	bool			mtt_only;

};



struct udma_buf_list {
@@ -389,10 +394,36 @@ struct udma_db { 
	void		*virt_addr;

};



struct udma_dca_ctx {

	struct list_head	pool; /* all DCA mems link to @pool */

	spinlock_t		pool_lock; /* protect @pool */

	uint32_t		free_mems; /* free mem num in pool */

	size_t			free_size; /* free mem size in pool */

	size_t			total_size; /* total size in pool */

	size_t			max_size; /* max size the pool can expand to */

	size_t			min_size; /* shrink if @free_size > @min_size */

	uint32_t		unit_size; /* unit size per DCA mem */



	uint32_t		max_qps;

	uint32_t		status_npage;

	struct ida		ida;



	uintptr_t		*buf_status;

	uintptr_t		*sync_status;



	bool			exit_aging;

	struct list_head	aging_proc_list;

	struct list_head	aging_new_list;

	spinlock_t		aging_lock;

	struct delayed_work	aging_dwork;

};



struct udma_ucontext {

	struct ubcore_ucontext		uctx;

	struct udma_uar			uar;

	uint64_t			pdn;

	struct udma_dca_ctx		dca_ctx;

	void				*dca_dbgfs;

};



struct udma_cmd_context {