Commit 1f0cdbbc authored by Weibo Zhao's avatar Weibo Zhao Committed by JiangShui
Browse files

hns3 udma: support of DCA

driver inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F


CVE: NA

------------------------------------------------------

This patch support DCA mode. DCA(dynamic context attach)
is a feature that can reduce memory usage. Each TP must
have a sending queue to store wqe. In sparse I/O scenarios,
maintaining send queue for each TP is a waste of memory.
DCA provides a way to dynamically expand or shrink the
sending queue. DCA may cause a performance loss.

Signed-off-by: default avatarWeibo Zhao <zhaoweibo3@huawei.com>
parent c1ac9521
Loading
Loading
Loading
Loading
+1 −1
Original line number Diff line number Diff line
@@ -11,6 +11,6 @@ $(MODULE_NAME)-objs := hns3_udma_hw.o hns3_udma_main.o hns3_udma_cmd.o \
			hns3_udma_hem.o hns3_udma_qp.o hns3_udma_eq.o \
			hns3_udma_db.o hns3_udma_jfc.o hns3_udma_jfr.o \
			hns3_udma_segment.o  hns3_udma_tp.o hns3_udma_jfs.o \
			hns3_udma_jetty.o hns3_udma_sysfs.o
			hns3_udma_jetty.o hns3_udma_sysfs.o hns3_udma_dca.o

obj-$(CONFIG_UB_UDMA_HNS3) := hns3_udma.o
+76 −0
Original line number Diff line number Diff line
@@ -28,11 +28,15 @@
#define UDMA_JETTY_QPN_PREFIX		0x2
#define UDMA_ADDR_4K_MASK		0xfffUL
#define URMA_SEG_ACCESS_GUARD		(1UL << 5)
#define UDMA_DCA_ATTACH_FLAGS_NEW_BUFFER BIT(0)
#define UDMA_DCA_INVALID_DCA_NUM ~0U

enum {
	UDMA_MMAP_UAR_PAGE,
	UDMA_MMAP_DWQE_PAGE,
	UDMA_MMAP_DCA_PAGE,
	UDMA_MMAP_RESET_PAGE,
	UDMA_MMAP_TYPE_DCA
};

enum udma_jfc_init_attr_mask {
@@ -114,6 +118,7 @@ enum udma_qp_cap_flags {
	UDMA_QP_CAP_RQ_RECORD_DB = 1 << 0,
	UDMA_QP_CAP_SQ_RECORD_DB = 1 << 1,
	UDMA_QP_CAP_OWNER_DB = 1 << 2,
	UDMA_QP_CAP_DYNAMIC_CTX_ATTACH = 1 << 4,
	UDMA_QP_CAP_DIRECT_WQE = 1 << 5,
};

@@ -136,6 +141,19 @@ struct udma_create_jfs_resp {
	struct udma_create_tp_resp create_tp_resp;
};

struct udma_create_ctx_ucmd {
	uint32_t comp;
	uint32_t dca_max_qps;
	uint32_t dca_unit_size;
};

enum udma_context_comp_mask {
	UDMA_CONTEXT_MASK_DCA_PRIME_QPS = 1 << 0,
	UDMA_CONTEXT_MASK_DCA_UNIT_SIZE = 1 << 1,
	UDMA_CONTEXT_MASK_DCA_MAX_SIZE = 1 << 2,
	UDMA_CONTEXT_MASK_DCA_MIN_SIZE = 1 << 3,
};

struct udma_create_ctx_resp {
	uint32_t num_comp_vectors;
	uint32_t num_qps_shift;
@@ -150,6 +168,9 @@ struct udma_create_ctx_resp {
	uint32_t max_jfs_sge;
	uint32_t poe_ch_num;
	uint64_t db_addr;
	uint32_t dca_qps;
	uint32_t dca_mmap_size;
	uint32_t dca_mode;
};

struct flush_cqe_param {
@@ -163,10 +184,65 @@ struct udma_poe_info {
	uint64_t	poe_addr;
};

struct udma_dca_reg_attr {
	uintptr_t	key;
	uintptr_t	addr;
	uint32_t	size;
};

struct udma_dca_dereg_attr {
	uintptr_t	free_key;
	struct dca_mem	*mem;
};

struct udma_dca_shrink_attr {
	uint64_t reserved_size;
};

struct udma_dca_shrink_resp {
	struct dca_mem	*mem;
	uintptr_t	free_key;
	uint32_t	free_mems;
};

struct udma_dca_attach_attr {
	uint64_t	qpn;
	uint32_t	sq_offset;
	uint32_t	sge_offset;
};

struct udma_dca_attach_resp {
	uint32_t	alloc_flags;
	uint32_t	alloc_pages;
	uint32_t	dcan;
};

struct udma_dca_detach_attr {
	uint64_t	qpn;
	uint32_t	sq_idx;
};

struct udma_dca_query_attr {
	uint64_t	qpn;
	uint32_t	page_idx;
};

struct udma_dca_query_resp {
	uintptr_t	mem_key;
	uint32_t	mem_ofs;
	uint32_t	page_count;
};

enum udma_user_ctl_handlers {
	UDMA_USER_CTL_FLUSH_CQE,
	UDMA_CONFIG_POE_CHANNEL,
	UDMA_QUERY_POE_CHANNEL,
	UDMA_DCA_MEM_REG,
	UDMA_DCA_MEM_DEREG,
	UDMA_DCA_MEM_SHRINK,
	UDMA_DCA_MEM_ATTACH,
	UDMA_DCA_MEM_DETACH,
	UDMA_DCA_MEM_QUERY,
	UDMA_OPCODE_NUM,
};

+1235 −0

File added.

Preview size limit exceeded, changes collapsed.

+219 −0
Original line number Diff line number Diff line
/* SPDX-License-Identifier: GPL-2.0 */
/* Huawei UDMA Linux driver
 * Copyright (c) 2023-2023 Hisilicon Limited.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
 * for more details.
 *
 */

#ifndef _UDMA_DCA_H
#define _UDMA_DCA_H

#include "hns3_udma_abi.h"
#include "hns3_udma_tp.h"
#include "hns3_udma_hem.h"

#define DCA_MEM_FLAGS_ALLOCED BIT(0)
#define DCA_MEM_FLAGS_REGISTERED BIT(1)

#define DCA_MEM_AGEING_MSES 1000 /* DCA mem ageing interval time */
#define UDMA_DCA_INVALID_BUF_ID 0U
#define DCA_MEM_STOP_ITERATE (-1)
#define DCA_MEM_NEXT_ITERATE (-2)

#define DCAN_TO_SYNC_BIT(n) ((n) * UDMA_DCA_BITS_PER_STATUS)
#define DCAN_TO_STAT_BIT(n) DCAN_TO_SYNC_BIT(n)

#define UDMA_DCA_OWN_MASK GENMASK(21, 0)

/*
 * buffer id(29b) = tag(7b) + owner(22b)
 * [28:22] tag  : indicate the QP config update times.
 * [21: 0] owner: indicate the QP to which the page belongs.
 */
#define UDMA_DCA_ID_MASK GENMASK(28, 0)
#define UDMA_DCA_TAG_MASK GENMASK(28, 22)
#define UDMA_DCA_OWN_MASK GENMASK(21, 0)

#define UDMA_DCA_BUF_ID_TO_TAG(buf_id) (((buf_id) & UDMA_DCA_TAG_MASK) >> 22)
#define UDMA_DCA_BUF_ID_TO_QPN(buf_id) ((buf_id) & UDMA_DCA_OWN_MASK)
#define UDMA_DCA_TO_BUF_ID(qpn, tag) (((qpn) & UDMA_DCA_OWN_MASK) | \
				      (((tag) << 22) & UDMA_DCA_TAG_MASK))

/* DCA page state (32 bit) */
struct dca_page_state {
	uint32_t buf_id		: 29; /* If zero, means page can be used by any buffer. */
	uint32_t lock		: 1; /* @buf_id locked this page to prepare access. */
	uint32_t active		: 1; /* @buf_id is accessing this page. */
	uint32_t head		: 1; /* This page is the head in a continuous address range. */
};

struct dca_mem {
	uint32_t		flags;
	struct list_head	list; /* link to mem list in dca context */
	spinlock_t		lock; /* protect the @flags and @list */
	uint32_t		page_count; /* page count in this mem obj */
	uint64_t		key; /* register by caller */
	uint32_t		size; /* bytes in this mem object */
	struct			dca_page_state *states; /* record each page's state */
	void			*pages; /* memory handle for getting dma address */
};

struct dca_page_free_buf_attr {
	uint32_t buf_id;
	uint32_t max_pages;
	uint32_t free_pages;
	uint32_t clean_mems;
};

struct dca_page_assign_attr {
	uint32_t buf_id;
	uint32_t unit;
	uint32_t total;
	uint32_t max;
};

struct dca_page_clear_attr {
	uint32_t buf_id;
	uint32_t max_pages;
	uint32_t clear_pages;
};

struct dca_get_alloced_pages_attr {
	uint32_t buf_id;
	dma_addr_t *pages;
	uint32_t total;
	uint32_t max;
};

struct dca_page_active_attr {
	uint32_t buf_id;
	uint32_t max_pages;
	uint32_t alloc_pages;
	uint32_t dirty_mems;
};

struct dca_page_query_active_attr {
	uint32_t buf_id;
	uint32_t curr_index;
	uint32_t start_index;
	uint32_t page_index;
	uint32_t page_count;
	uint64_t mem_key;
};

typedef int (*udma_dca_enum_callback)(struct dca_page_state *states,
				      uint32_t count, void *param);

struct dca_mem_enum_attr {
	void *param;
	udma_dca_enum_callback enum_fn;
};

static inline uint64_t umem_cal_npages(uint64_t va, uint64_t len)
{
	return (ALIGN(va + len, UDMA_PAGE_SIZE) - ALIGN_DOWN(va, UDMA_PAGE_SIZE)) /
	       UDMA_PAGE_SIZE;
}

static inline bool dca_page_is_attached(struct dca_page_state *state,
					uint32_t buf_id)
{
	/* only the own bit needs to be matched. */
	return (UDMA_DCA_OWN_MASK & buf_id) ==
	       (UDMA_DCA_OWN_MASK & state->buf_id);
}

static inline bool dca_mem_is_available(struct dca_mem *mem)
{
	return mem->flags == (DCA_MEM_FLAGS_ALLOCED | DCA_MEM_FLAGS_REGISTERED);
}

static inline void set_dca_page_to_free(struct dca_page_state *state)
{
	state->buf_id = UDMA_DCA_INVALID_BUF_ID;
	state->active = 0;
	state->lock = 0;
}

static inline bool dca_page_is_free(struct dca_page_state *state)
{
	return state->buf_id == UDMA_DCA_INVALID_BUF_ID;
}

static inline bool dca_page_is_active(struct dca_page_state *state,
				      uint32_t buf_id)
{
	/* all buf id bits must be matched */
	return (UDMA_DCA_ID_MASK & buf_id) == state->buf_id &&
		!state->lock && state->active;
}

static inline bool dca_page_is_inactive(struct dca_page_state *state)
{
	return !state->lock && !state->active;
}

static inline void lock_dca_page_to_attach(struct dca_page_state *state,
					   uint32_t buf_id)
{
	state->buf_id = UDMA_DCA_ID_MASK & buf_id;
	state->active = 0;
	state->lock = 1;
}

static inline bool dca_page_is_allocated(struct dca_page_state *state,
					 uint32_t buf_id)
{
	return dca_page_is_attached(state, buf_id) && state->lock;
}

static inline void unlock_dca_page_to_active(struct dca_page_state *state,
					     uint32_t buf_id)
{
	state->buf_id = UDMA_DCA_ID_MASK & buf_id;
	state->active = 1;
	state->lock = 0;
}

void udma_enable_dca(struct udma_dev *dev, struct udma_qp *qp);
void udma_disable_dca(struct udma_dev *dev, struct udma_qp *qp);

void udma_modify_dca(struct udma_dev *dev, struct udma_qp *qp);

int udma_register_dca_mem(struct udma_dev *dev, struct udma_ucontext *context,
			  struct udma_dca_reg_attr *attr);
void unregister_dca_mem(struct udma_dev *dev, struct udma_dca_ctx *ctx,
			struct dca_mem *mem);
int udma_deregister_dca_mem(struct udma_dev *dev,
			    struct udma_ucontext *context,
			    struct udma_dca_dereg_attr *attr, bool from_user);

void udma_shrink_dca_mem(struct udma_dev *dev, struct udma_ucontext *context,
			 struct udma_dca_shrink_attr *attr,
			 struct udma_dca_shrink_resp *resp);

int udma_query_dca_mem(struct udma_dev *dev, struct udma_dca_query_attr *attr,
		       struct udma_dca_query_resp *resp);

int udma_dca_attach(struct udma_dev *dev, struct udma_dca_attach_attr *attr,
		    struct udma_dca_attach_resp *resp);
void udma_dca_disattach(struct udma_dev *dev, struct udma_dca_attach_attr *attr);
void udma_dca_detach(struct udma_dev *dev, struct udma_dca_detach_attr *attr);

int udma_register_udca(struct udma_dev *udma_dev,
			  struct udma_ucontext *context, struct ubcore_udrv_priv *udrv_data);

void udma_unregister_udca(struct udma_dev *udma_dev,
			    struct udma_ucontext *context);

void udma_enum_dca_pool(struct udma_dca_ctx *dca_ctx, void *param,
			udma_dca_enum_callback cb);
#endif
+31 −0
Original line number Diff line number Diff line
@@ -144,6 +144,9 @@

#define UDMA_MIN_JFS_DEPTH 64

#define UDMA_DCA_BITS_PER_STATUS 1
#define DCA_BITS_HALF 2

enum {
	NO_ARMED = 0x0
};
@@ -300,6 +303,8 @@ struct udma_buf_attr {
	} region[UDMA_MAX_BT_REGION];
	uint32_t		region_count; /* valid region count */
	uint32_t		page_shift;  /* buffer page shift */
	/* only alloc buffer-required MTT memory */
	bool			mtt_only;
};

struct udma_buf_list {
@@ -389,10 +394,36 @@ struct udma_db {
	void		*virt_addr;
};

struct udma_dca_ctx {
	struct list_head	pool; /* all DCA mems link to @pool */
	spinlock_t		pool_lock; /* protect @pool */
	uint32_t		free_mems; /* free mem num in pool */
	size_t			free_size; /* free mem size in pool */
	size_t			total_size; /* total size in pool */
	size_t			max_size; /* max size the pool can expand to */
	size_t			min_size; /* shrink if @free_size > @min_size */
	uint32_t		unit_size; /* unit size per DCA mem */

	uint32_t		max_qps;
	uint32_t		status_npage;
	struct ida		ida;

	uintptr_t		*buf_status;
	uintptr_t		*sync_status;

	bool			exit_aging;
	struct list_head	aging_proc_list;
	struct list_head	aging_new_list;
	spinlock_t		aging_lock;
	struct delayed_work	aging_dwork;
};

struct udma_ucontext {
	struct ubcore_ucontext		uctx;
	struct udma_uar			uar;
	uint64_t			pdn;
	struct udma_dca_ctx		dca_ctx;
	void				*dca_dbgfs;
};

struct udma_cmd_context {
Loading