Commit 46ae40b9 authored by Shay Drory's avatar Shay Drory Committed by Saeed Mahameed
Browse files

net/mlx5: Let user configure io_eq_size param 



Currently, each I/O EQ is taking 128KB of memory. This size
is not needed in all use cases, and is critical with large scale.
Hence, allow user to configure the size of I/O EQs.

For example, to reduce I/O EQ size to 64, execute:
$ devlink resource set pci/0000:00:0b.0 path /io_eq_size/ size 64
$ devlink dev reload pci/0000:00:0b.0

Signed-off-by: default avatarShay Drory <shayd@nvidia.com>
Reviewed-by: default avatarMoshe Shemesh <moshe@nvidia.com>
Reviewed-by: default avatarParav Pandit <parav@nvidia.com>
Signed-off-by: default avatarSaeed Mahameed <saeedm@nvidia.com>
parent 3518c83f

Documentation/networking/devlink/mlx5.rst

+12 −0
Original line number Diff line number Diff line
@@ -46,6 +46,18 @@ parameters. 


The ``mlx5`` driver supports reloading via ``DEVLINK_CMD_RELOAD``



Resources

=========



.. list-table:: Driver-specific resources implemented

   :widths: 5 5 5 85



   * - Name

     - Description

   * - ``comp_eq_size``

     - Control the size of I/O completion EQs.

       * The default value is 1024, and the range is between 64 and 4096.



Info versions

=============

drivers/net/ethernet/mellanox/mlx5/core/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -16,7 +16,7 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \ 
		transobj.o vport.o sriov.o fs_cmd.o fs_core.o pci_irq.o \

		fs_counters.o fs_ft_pool.o rl.o lag/lag.o dev.o events.o wq.o lib/gid.o \

		lib/devcom.o lib/pci_vsc.o lib/dm.o lib/fs_ttc.o diag/fs_tracepoint.o \

		diag/fw_tracer.o diag/crdump.o devlink.o diag/rsc_dump.o \

		diag/fw_tracer.o diag/crdump.o devlink.o devlink_res.o diag/rsc_dump.o \

		fw_reset.o qos.o lib/tout.o



#

drivers/net/ethernet/mellanox/mlx5/core/devlink.h

+11 −0
Original line number Diff line number Diff line
@@ -6,6 +6,13 @@ 


#include <net/devlink.h>



enum mlx5_devlink_resource_id {

	MLX5_DL_RES_COMP_EQ = 1,



	__MLX5_ID_RES_MAX,

	MLX5_ID_RES_MAX = __MLX5_ID_RES_MAX - 1,

};



enum mlx5_devlink_param_id {

	MLX5_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX,

	MLX5_DEVLINK_PARAM_ID_FLOW_STEERING_MODE,
@@ -31,6 +38,10 @@ int mlx5_devlink_trap_get_num_active(struct mlx5_core_dev *dev); 
int mlx5_devlink_traps_get_action(struct mlx5_core_dev *dev, int trap_id,

				  enum devlink_trap_action *action);



void mlx5_devlink_res_register(struct mlx5_core_dev *dev);

void mlx5_devlink_res_unregister(struct mlx5_core_dev *dev);

size_t mlx5_devlink_res_size(struct mlx5_core_dev *dev, enum mlx5_devlink_resource_id id);



struct devlink *mlx5_devlink_alloc(struct device *dev);

void mlx5_devlink_free(struct devlink *devlink);

int mlx5_devlink_register(struct devlink *devlink);

drivers/net/ethernet/mellanox/mlx5/core/devlink_res.c

0 → 100644
+56 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB

/* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. */



#include "devlink.h"

#include "mlx5_core.h"



enum {

	MLX5_EQ_MIN_SIZE = 64,

	MLX5_EQ_MAX_SIZE = 4096,

	MLX5_COMP_EQ_SIZE = 1024,

};



static int comp_eq_res_register(struct mlx5_core_dev *dev)

{

	struct devlink_resource_size_params comp_eq_size;

	struct devlink *devlink = priv_to_devlink(dev);



	devlink_resource_size_params_init(&comp_eq_size, MLX5_EQ_MIN_SIZE,

					  MLX5_EQ_MAX_SIZE, 1, DEVLINK_RESOURCE_UNIT_ENTRY);

	return devlink_resource_register(devlink, "io_eq_size", MLX5_COMP_EQ_SIZE,

					 MLX5_DL_RES_COMP_EQ,

					 DEVLINK_RESOURCE_ID_PARENT_TOP,

					 &comp_eq_size);

}



void mlx5_devlink_res_register(struct mlx5_core_dev *dev)

{

	int err;



	err = comp_eq_res_register(dev);

	if (err)

		mlx5_core_err(dev, "Failed to register resources, err = %d\n", err);

}



void mlx5_devlink_res_unregister(struct mlx5_core_dev *dev)

{

	devlink_resources_unregister(priv_to_devlink(dev), NULL);

}



static const size_t default_vals[MLX5_ID_RES_MAX + 1] = {

	[MLX5_DL_RES_COMP_EQ] = MLX5_COMP_EQ_SIZE,

};



size_t mlx5_devlink_res_size(struct mlx5_core_dev *dev, enum mlx5_devlink_resource_id id)

{

	struct devlink *devlink = priv_to_devlink(dev);

	u64 size;

	int err;



	err = devlink_resource_size_get(devlink, id, &size);

	if (!err)

		return size;

	mlx5_core_err(dev, "Failed to get param. using default. err = %d, id = %u\n",

		      err, id);

	return default_vals[id];

}

drivers/net/ethernet/mellanox/mlx5/core/eq.c

+2 −1
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#include "lib/clock.h"

#include "diag/fw_tracer.h"

#include "mlx5_irq.h"

#include "devlink.h"



enum {

	MLX5_EQE_OWNER_INIT_VAL	= 0x1,
@@ -807,7 +808,7 @@ static int create_comp_eqs(struct mlx5_core_dev *dev) 


	INIT_LIST_HEAD(&table->comp_eqs_list);

	ncomp_eqs = table->num_comp_eqs;

	nent = MLX5_COMP_EQ_SIZE;

	nent = mlx5_devlink_res_size(dev, MLX5_DL_RES_COMP_EQ);

	for (i = 0; i < ncomp_eqs; i++) {

		struct mlx5_eq_param param = {};

		int vecidx = i;