Commit 0475135f authored by David S. Miller's avatar David S. Miller
Browse files

Merge tag 'mlx5-updates-2023-04-14' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux

mlx5-updates-2023-04-14

Yevgeny Kliteynik Says:
=======================

SW Steering: Support pattern/args modify_header actions

The following patch series adds support for a new pattern/arguments type
of modify_header actions.

Starting with ConnectX-6 DX, we use a new design of modify_header FW object.
The current modify_header object allows for having only limited number of
these FW objects, which means that we are limited in the number of offloaded
flows that require modify_header action.

The new approach comprises of two types of objects: pattern and argument.
Pattern holds header modification templates, later used with corresponding
argument object to create complete header modification actions.
The pattern indicates which headers are modified, while the arguments
provide the specific values.
Therefore a single pattern can be used with different arguments in different
flows, enabling offloading of large number of modify_header flows.

 - Patch 1, 2: Add ICM pool for modify-header-pattern objects and implement
   patterns cache, allowing patterns reuse for different flows
 - Patch 3: Allow for chunk allocation separately for STEv0 and STEv1
 - Patch 4: Read related device capabilities
 - Patch 5: Add create/destroy functions for the new general object type
 - Patch 6: Add support for writing modify header argument to ICM
 - Patch 7, 8: Some required fixes to support pattern/arg - separate read
   buffer from the write buffer and fix QP continuous allocation
 - Patch 9: Add pool for modify header arg objects
 - Patch 10, 11, 12: Implement MODIFY_HEADER and TNL_L3_TO_L2 actions with
   the new patterns/args design
 - Patch 13: Optimization - set modify header action of size 1 directly on
   the STE instead of separate pattern/args combination
 - Patch 14: Adjust debug dump for patterns/args
 - Patch 15: Enable patterns and arguments for supporting devices

=======================
parents e2174b03 220ae987
Loading
Loading
Loading
Loading
+1 −1
Original line number Diff line number Diff line
@@ -113,7 +113,7 @@ mlx5_core-$(CONFIG_MLX5_SW_STEERING) += steering/dr_domain.o steering/dr_table.o
					steering/dr_cmd.o steering/dr_fw.o \
					steering/dr_action.o steering/fs_dr.o \
					steering/dr_definer.o steering/dr_ptrn.o \
					steering/dr_dbg.o lib/smfs.o
					steering/dr_arg.o steering/dr_dbg.o lib/smfs.o
#
# SF device
#
+47 −45
Original line number Diff line number Diff line
@@ -819,14 +819,34 @@ int mlx5dr_actions_build_ste_arr(struct mlx5dr_matcher *matcher,
		case DR_ACTION_TYP_TNL_L2_TO_L2:
			break;
		case DR_ACTION_TYP_TNL_L3_TO_L2:
			if (action->rewrite->ptrn && action->rewrite->arg) {
				attr.decap_index = mlx5dr_arg_get_obj_id(action->rewrite->arg);
				attr.decap_actions = action->rewrite->ptrn->num_of_actions;
				attr.decap_pat_idx = action->rewrite->ptrn->index;
			} else {
				attr.decap_index = action->rewrite->index;
				attr.decap_actions = action->rewrite->num_of_actions;
				attr.decap_with_vlan =
					attr.decap_actions == WITH_VLAN_NUM_HW_ACTIONS;
				attr.decap_pat_idx = MLX5DR_INVALID_PATTERN_INDEX;
			}
			break;
		case DR_ACTION_TYP_MODIFY_HDR:
			if (action->rewrite->single_action_opt) {
				attr.modify_actions = action->rewrite->num_of_actions;
				attr.single_modify_action = action->rewrite->data;
			} else {
				if (action->rewrite->ptrn && action->rewrite->arg) {
					attr.modify_index =
						mlx5dr_arg_get_obj_id(action->rewrite->arg);
					attr.modify_actions = action->rewrite->ptrn->num_of_actions;
					attr.modify_pat_idx = action->rewrite->ptrn->index;
				} else {
					attr.modify_index = action->rewrite->index;
					attr.modify_actions = action->rewrite->num_of_actions;
					attr.modify_pat_idx = MLX5DR_INVALID_PATTERN_INDEX;
				}
			}
			if (action->rewrite->modify_ttl)
				dr_action_modify_ttl_adjust(dmn, &attr, rx_rule,
							    &recalc_cs_required);
@@ -1365,8 +1385,6 @@ dr_action_verify_reformat_params(enum mlx5dr_action_type reformat_type,
	return -EINVAL;
}

#define ACTION_CACHE_LINE_SIZE 64

static int
dr_action_create_reformat_action(struct mlx5dr_domain *dmn,
				 u8 reformat_param_0, u8 reformat_param_1,
@@ -1403,36 +1421,25 @@ dr_action_create_reformat_action(struct mlx5dr_domain *dmn,
	}
	case DR_ACTION_TYP_TNL_L3_TO_L2:
	{
		u8 hw_actions[ACTION_CACHE_LINE_SIZE] = {};
		u8 hw_actions[DR_ACTION_CACHE_LINE_SIZE] = {};
		int ret;

		ret = mlx5dr_ste_set_action_decap_l3_list(dmn->ste_ctx,
							  data, data_sz,
							  hw_actions,
							  ACTION_CACHE_LINE_SIZE,
							  DR_ACTION_CACHE_LINE_SIZE,
							  &action->rewrite->num_of_actions);
		if (ret) {
			mlx5dr_dbg(dmn, "Failed creating decap l3 action list\n");
			return ret;
		}

		action->rewrite->chunk = mlx5dr_icm_alloc_chunk(dmn->action_icm_pool,
								DR_CHUNK_SIZE_8);
		if (!action->rewrite->chunk) {
			mlx5dr_dbg(dmn, "Failed allocating modify header chunk\n");
			return -ENOMEM;
		}

		action->rewrite->data = (void *)hw_actions;
		action->rewrite->index = (mlx5dr_icm_pool_get_chunk_icm_addr
					  (action->rewrite->chunk) -
					 dmn->info.caps.hdr_modify_icm_addr) /
					 ACTION_CACHE_LINE_SIZE;
		action->rewrite->data = hw_actions;
		action->rewrite->dmn = dmn;

		ret = mlx5dr_send_postsend_action(dmn, action);
		ret = mlx5dr_ste_alloc_modify_hdr(action);
		if (ret) {
			mlx5dr_dbg(dmn, "Writing decap l3 actions to ICM failed\n");
			mlx5dr_icm_free_chunk(action->rewrite->chunk);
			mlx5dr_dbg(dmn, "Failed preparing reformat data\n");
			return ret;
		}
		return 0;
@@ -1963,7 +1970,6 @@ static int dr_action_create_modify_action(struct mlx5dr_domain *dmn,
					  __be64 actions[],
					  struct mlx5dr_action *action)
{
	struct mlx5dr_icm_chunk *chunk;
	u32 max_hw_actions;
	u32 num_hw_actions;
	u32 num_sw_actions;
@@ -1980,15 +1986,9 @@ static int dr_action_create_modify_action(struct mlx5dr_domain *dmn,
		return -EINVAL;
	}

	chunk = mlx5dr_icm_alloc_chunk(dmn->action_icm_pool, DR_CHUNK_SIZE_16);
	if (!chunk)
		return -ENOMEM;

	hw_actions = kcalloc(1, max_hw_actions * DR_MODIFY_ACTION_SIZE, GFP_KERNEL);
	if (!hw_actions) {
		ret = -ENOMEM;
		goto free_chunk;
	}
	if (!hw_actions)
		return -ENOMEM;

	ret = dr_actions_convert_modify_header(action,
					       max_hw_actions,
@@ -2000,24 +2000,24 @@ static int dr_action_create_modify_action(struct mlx5dr_domain *dmn,
	if (ret)
		goto free_hw_actions;

	action->rewrite->chunk = chunk;
	action->rewrite->modify_ttl = modify_ttl;
	action->rewrite->data = (u8 *)hw_actions;
	action->rewrite->num_of_actions = num_hw_actions;
	action->rewrite->index = (mlx5dr_icm_pool_get_chunk_icm_addr(chunk) -
				  dmn->info.caps.hdr_modify_icm_addr) /
				  ACTION_CACHE_LINE_SIZE;

	ret = mlx5dr_send_postsend_action(dmn, action);
	if (num_hw_actions == 1 &&
	    dmn->info.caps.sw_format_ver >= MLX5_STEERING_FORMAT_CONNECTX_6DX) {
		action->rewrite->single_action_opt = true;
	} else {
		action->rewrite->single_action_opt = false;
		ret = mlx5dr_ste_alloc_modify_hdr(action);
		if (ret)
			goto free_hw_actions;
	}

	return 0;

free_hw_actions:
	kfree(hw_actions);
free_chunk:
	mlx5dr_icm_free_chunk(chunk);
	return ret;
}

@@ -2162,7 +2162,8 @@ int mlx5dr_action_destroy(struct mlx5dr_action *action)
		refcount_dec(&action->reformat->dmn->refcount);
		break;
	case DR_ACTION_TYP_TNL_L3_TO_L2:
		mlx5dr_icm_free_chunk(action->rewrite->chunk);
		mlx5dr_ste_free_modify_hdr(action);
		kfree(action->rewrite->data);
		refcount_dec(&action->rewrite->dmn->refcount);
		break;
	case DR_ACTION_TYP_L2_TO_TNL_L2:
@@ -2173,7 +2174,8 @@ int mlx5dr_action_destroy(struct mlx5dr_action *action)
		refcount_dec(&action->reformat->dmn->refcount);
		break;
	case DR_ACTION_TYP_MODIFY_HDR:
		mlx5dr_icm_free_chunk(action->rewrite->chunk);
		if (!action->rewrite->single_action_opt)
			mlx5dr_ste_free_modify_hdr(action);
		kfree(action->rewrite->data);
		refcount_dec(&action->rewrite->dmn->refcount);
		break;
+273 −0
Original line number Diff line number Diff line
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
// Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.

#include "dr_types.h"

#define DR_ICM_MODIFY_HDR_GRANULARITY_4K 12

/* modify-header arg pool */
enum dr_arg_chunk_size {
	DR_ARG_CHUNK_SIZE_1,
	DR_ARG_CHUNK_SIZE_MIN = DR_ARG_CHUNK_SIZE_1, /* keep updated when changing */
	DR_ARG_CHUNK_SIZE_2,
	DR_ARG_CHUNK_SIZE_3,
	DR_ARG_CHUNK_SIZE_4,
	DR_ARG_CHUNK_SIZE_MAX,
};

/* argument pool area */
struct dr_arg_pool {
	enum dr_arg_chunk_size log_chunk_size;
	struct mlx5dr_domain *dmn;
	struct list_head free_list;
	struct mutex mutex; /* protect arg pool */
};

struct mlx5dr_arg_mgr {
	struct mlx5dr_domain *dmn;
	struct dr_arg_pool *pools[DR_ARG_CHUNK_SIZE_MAX];
};

static int dr_arg_pool_alloc_objs(struct dr_arg_pool *pool)
{
	struct mlx5dr_arg_obj *arg_obj, *tmp_arg;
	struct list_head cur_list;
	u16 object_range;
	int num_of_objects;
	u32 obj_id = 0;
	int i, ret;

	INIT_LIST_HEAD(&cur_list);

	object_range =
		pool->dmn->info.caps.log_header_modify_argument_granularity;

	object_range =
		max_t(u32, pool->dmn->info.caps.log_header_modify_argument_granularity,
		      DR_ICM_MODIFY_HDR_GRANULARITY_4K);
	object_range =
		min_t(u32, pool->dmn->info.caps.log_header_modify_argument_max_alloc,
		      object_range);

	if (pool->log_chunk_size > object_range) {
		mlx5dr_err(pool->dmn, "Required chunk size (%d) is not supported\n",
			   pool->log_chunk_size);
		return -ENOMEM;
	}

	num_of_objects = (1 << (object_range - pool->log_chunk_size));
	/* Only one devx object per range */
	ret = mlx5dr_cmd_create_modify_header_arg(pool->dmn->mdev,
						  object_range,
						  pool->dmn->pdn,
						  &obj_id);
	if (ret) {
		mlx5dr_err(pool->dmn, "failed allocating object with range: %d:\n",
			   object_range);
		return -EAGAIN;
	}

	for (i = 0; i < num_of_objects; i++) {
		arg_obj = kzalloc(sizeof(*arg_obj), GFP_KERNEL);
		if (!arg_obj) {
			ret = -ENOMEM;
			goto clean_arg_obj;
		}

		arg_obj->log_chunk_size = pool->log_chunk_size;

		list_add_tail(&arg_obj->list_node, &cur_list);

		arg_obj->obj_id = obj_id;
		arg_obj->obj_offset = i * (1 << pool->log_chunk_size);
	}
	list_splice_tail_init(&cur_list, &pool->free_list);

	return 0;

clean_arg_obj:
	mlx5dr_cmd_destroy_modify_header_arg(pool->dmn->mdev, obj_id);
	list_for_each_entry_safe(arg_obj, tmp_arg, &cur_list, list_node) {
		list_del(&arg_obj->list_node);
		kfree(arg_obj);
	}
	return ret;
}

static struct mlx5dr_arg_obj *dr_arg_pool_get_arg_obj(struct dr_arg_pool *pool)
{
	struct mlx5dr_arg_obj *arg_obj = NULL;
	int ret;

	mutex_lock(&pool->mutex);
	if (list_empty(&pool->free_list)) {
		ret = dr_arg_pool_alloc_objs(pool);
		if (ret)
			goto out;
	}

	arg_obj = list_first_entry_or_null(&pool->free_list,
					   struct mlx5dr_arg_obj,
					   list_node);
	WARN(!arg_obj, "couldn't get dr arg obj from pool");

	if (arg_obj)
		list_del_init(&arg_obj->list_node);

out:
	mutex_unlock(&pool->mutex);
	return arg_obj;
}

static void dr_arg_pool_put_arg_obj(struct dr_arg_pool *pool,
				    struct mlx5dr_arg_obj *arg_obj)
{
	mutex_lock(&pool->mutex);
	list_add(&arg_obj->list_node, &pool->free_list);
	mutex_unlock(&pool->mutex);
}

static struct dr_arg_pool *dr_arg_pool_create(struct mlx5dr_domain *dmn,
					      enum dr_arg_chunk_size chunk_size)
{
	struct dr_arg_pool *pool;

	pool = kzalloc(sizeof(*pool), GFP_KERNEL);
	if (!pool)
		return NULL;

	pool->dmn = dmn;

	INIT_LIST_HEAD(&pool->free_list);
	mutex_init(&pool->mutex);

	pool->log_chunk_size = chunk_size;
	if (dr_arg_pool_alloc_objs(pool))
		goto free_pool;

	return pool;

free_pool:
	kfree(pool);

	return NULL;
}

static void dr_arg_pool_destroy(struct dr_arg_pool *pool)
{
	struct mlx5dr_arg_obj *arg_obj, *tmp_arg;

	list_for_each_entry_safe(arg_obj, tmp_arg, &pool->free_list, list_node) {
		list_del(&arg_obj->list_node);
		if (!arg_obj->obj_offset) /* the first in range */
			mlx5dr_cmd_destroy_modify_header_arg(pool->dmn->mdev, arg_obj->obj_id);
		kfree(arg_obj);
	}

	mutex_destroy(&pool->mutex);
	kfree(pool);
}

static enum dr_arg_chunk_size dr_arg_get_chunk_size(u16 num_of_actions)
{
	if (num_of_actions <= 8)
		return DR_ARG_CHUNK_SIZE_1;
	if (num_of_actions <= 16)
		return DR_ARG_CHUNK_SIZE_2;
	if (num_of_actions <= 32)
		return DR_ARG_CHUNK_SIZE_3;
	if (num_of_actions <= 64)
		return DR_ARG_CHUNK_SIZE_4;

	return DR_ARG_CHUNK_SIZE_MAX;
}

u32 mlx5dr_arg_get_obj_id(struct mlx5dr_arg_obj *arg_obj)
{
	return (arg_obj->obj_id + arg_obj->obj_offset);
}

struct mlx5dr_arg_obj *mlx5dr_arg_get_obj(struct mlx5dr_arg_mgr *mgr,
					  u16 num_of_actions,
					  u8 *data)
{
	u32 size = dr_arg_get_chunk_size(num_of_actions);
	struct mlx5dr_arg_obj *arg_obj;
	int ret;

	if (size >= DR_ARG_CHUNK_SIZE_MAX)
		return NULL;

	arg_obj = dr_arg_pool_get_arg_obj(mgr->pools[size]);
	if (!arg_obj) {
		mlx5dr_err(mgr->dmn, "Failed allocating args object for modify header\n");
		return NULL;
	}

	/* write it into the hw */
	ret = mlx5dr_send_postsend_args(mgr->dmn,
					mlx5dr_arg_get_obj_id(arg_obj),
					num_of_actions, data);
	if (ret) {
		mlx5dr_err(mgr->dmn, "Failed writing args object\n");
		goto put_obj;
	}

	return arg_obj;

put_obj:
	mlx5dr_arg_put_obj(mgr, arg_obj);
	return NULL;
}

void mlx5dr_arg_put_obj(struct mlx5dr_arg_mgr *mgr,
			struct mlx5dr_arg_obj *arg_obj)
{
	dr_arg_pool_put_arg_obj(mgr->pools[arg_obj->log_chunk_size], arg_obj);
}

struct mlx5dr_arg_mgr*
mlx5dr_arg_mgr_create(struct mlx5dr_domain *dmn)
{
	struct mlx5dr_arg_mgr *pool_mgr;
	int i;

	if (!mlx5dr_domain_is_support_ptrn_arg(dmn))
		return NULL;

	pool_mgr = kzalloc(sizeof(*pool_mgr), GFP_KERNEL);
	if (!pool_mgr)
		return NULL;

	pool_mgr->dmn = dmn;

	for (i = 0; i < DR_ARG_CHUNK_SIZE_MAX; i++) {
		pool_mgr->pools[i] = dr_arg_pool_create(dmn, i);
		if (!pool_mgr->pools[i])
			goto clean_pools;
	}

	return pool_mgr;

clean_pools:
	for (i--; i >= 0; i--)
		dr_arg_pool_destroy(pool_mgr->pools[i]);

	kfree(pool_mgr);
	return NULL;
}

void mlx5dr_arg_mgr_destroy(struct mlx5dr_arg_mgr *mgr)
{
	struct dr_arg_pool **pools;
	int i;

	if (!mgr)
		return;

	pools = mgr->pools;
	for (i = 0; i < DR_ARG_CHUNK_SIZE_MAX; i++)
		dr_arg_pool_destroy(pools[i]);

	kfree(mgr);
}
+54 −0
Original line number Diff line number Diff line
@@ -132,6 +132,17 @@ int mlx5dr_cmd_query_device(struct mlx5_core_dev *mdev,

	caps->isolate_vl_tc = MLX5_CAP_GEN(mdev, isolate_vl_tc_new);

	caps->support_modify_argument =
		MLX5_CAP_GEN_64(mdev, general_obj_types) &
		MLX5_GENERAL_OBJ_TYPES_CAP_HEADER_MODIFY_ARGUMENT;

	if (caps->support_modify_argument) {
		caps->log_header_modify_argument_granularity =
			MLX5_CAP_GEN(mdev, log_header_modify_argument_granularity);
		caps->log_header_modify_argument_max_alloc =
			MLX5_CAP_GEN(mdev, log_header_modify_argument_max_alloc);
	}

	/* geneve_tlv_option_0_exist is the indication of
	 * STE support for lookup type flex_parser_ok
	 */
@@ -682,6 +693,49 @@ int mlx5dr_cmd_query_gid(struct mlx5_core_dev *mdev, u8 vhca_port_num,
	return 0;
}

int mlx5dr_cmd_create_modify_header_arg(struct mlx5_core_dev *dev,
					u16 log_obj_range, u32 pd,
					u32 *obj_id)
{
	u32 in[MLX5_ST_SZ_DW(create_modify_header_arg_in)] = {};
	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
	void *attr;
	int ret;

	attr = MLX5_ADDR_OF(create_modify_header_arg_in, in, hdr);
	MLX5_SET(general_obj_in_cmd_hdr, attr, opcode,
		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
	MLX5_SET(general_obj_in_cmd_hdr, attr, obj_type,
		 MLX5_OBJ_TYPE_HEADER_MODIFY_ARGUMENT);
	MLX5_SET(general_obj_in_cmd_hdr, attr,
		 op_param.create.log_obj_range, log_obj_range);

	attr = MLX5_ADDR_OF(create_modify_header_arg_in, in, arg);
	MLX5_SET(modify_header_arg, attr, access_pd, pd);

	ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
	if (ret)
		return ret;

	*obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
	return 0;
}

void mlx5dr_cmd_destroy_modify_header_arg(struct mlx5_core_dev *dev,
					  u32 obj_id)
{
	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
	u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};

	MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
		 MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
		 MLX5_OBJ_TYPE_HEADER_MODIFY_ARGUMENT);
	MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id);

	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}

static int mlx5dr_cmd_set_extended_dest(struct mlx5_core_dev *dev,
					struct mlx5dr_cmd_fte_info *fte,
					bool *extended_dest)
+27 −3
Original line number Diff line number Diff line
@@ -140,10 +140,31 @@ dr_dump_rule_action_mem(struct seq_file *file, const u64 rule_id,
			   action->flow_tag->flow_tag);
		break;
	case DR_ACTION_TYP_MODIFY_HDR:
		seq_printf(file, "%d,0x%llx,0x%llx,0x%x\n",
	{
		struct mlx5dr_ptrn_obj *ptrn = action->rewrite->ptrn;
		struct mlx5dr_arg_obj *arg = action->rewrite->arg;
		u8 *rewrite_data = action->rewrite->data;
		bool ptrn_arg;
		int i;

		ptrn_arg = !action->rewrite->single_action_opt && ptrn && arg;

		seq_printf(file, "%d,0x%llx,0x%llx,0x%x,%d,0x%x,0x%x,0x%x",
			   DR_DUMP_REC_TYPE_ACTION_MODIFY_HDR, action_id,
			   rule_id, action->rewrite->index);
			   rule_id, action->rewrite->index,
			   action->rewrite->single_action_opt,
			   action->rewrite->num_of_actions,
			   ptrn_arg ? ptrn->index : 0,
			   ptrn_arg ? mlx5dr_arg_get_obj_id(arg) : 0);

		for (i = 0; i < action->rewrite->num_of_actions; i++) {
			seq_printf(file, ",0x%016llx",
				   be64_to_cpu(((__be64 *)rewrite_data)[i]));
		}

		seq_puts(file, "\n");
		break;
	}
	case DR_ACTION_TYP_VPORT:
		seq_printf(file, "%d,0x%llx,0x%llx,0x%x\n",
			   DR_DUMP_REC_TYPE_ACTION_VPORT, action_id, rule_id,
@@ -157,7 +178,10 @@ dr_dump_rule_action_mem(struct seq_file *file, const u64 rule_id,
	case DR_ACTION_TYP_TNL_L3_TO_L2:
		seq_printf(file, "%d,0x%llx,0x%llx,0x%x\n",
			   DR_DUMP_REC_TYPE_ACTION_DECAP_L3, action_id,
			   rule_id, action->rewrite->index);
			   rule_id,
			   (action->rewrite->ptrn && action->rewrite->arg) ?
			   mlx5dr_arg_get_obj_id(action->rewrite->arg) :
			   action->rewrite->index);
		break;
	case DR_ACTION_TYP_L2_TO_TNL_L2:
		seq_printf(file, "%d,0x%llx,0x%llx,0x%x\n",
Loading