Commit 1a8e628c authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'devlink-aux-devices' 



Parav Pandit says:

====================
devlink: Control auxiliary devices

Currently, for mlx5 multi-function device, a user is not able to control
which functionality to enable/disable. For example, each PCI
PF, VF, SF function by default has netdevice, RDMA and vdpa-net
devices always enabled.

Hence, enable user to control which device functionality to enable/disable.

This is achieved by using existing devlink params [1] to
enable/disable eth, rdma and vdpa net functionality control knob.

For example user interested in only vdpa device function: performs,

$ devlink dev param set pci/0000:06:00.0 name enable_rdma value false \
                   cmode driverinit
$ devlink dev param set pci/0000:06:00.0 name enable_eth value false \
                   cmode driverinit
$ devlink dev param set pci/0000:06:00.0 name enable_vnet value true \
                   cmode driverinit

$ devlink dev reload pci/0000:06:00.0

Reload command honors parameters set, initializes the device that user
has composed using devlink dev params and resources.
Devices before reload:

            mlx5_core.sf.4
         (subfunction device)
                  /\
                 /| \
                / |  \
               /  |   \
 mlx5_core.eth.4  |  mlx5_core.rdma.4
(SF eth aux dev)  |  (SF rdma aux dev)
    |             |        |
    |             |        |
 enp6s0f0s88      |      mlx5_0
 (SF netdev)      |  (SF rdma device)
                  |
         mlx5_core.vnet.4
         (SF vnet aux dev)
                 |
                 |
        auxiliary/mlx5_core.sf.4
        (vdpa net mgmt device)

Above example reconfigures the device with only VDPA functionality.
Devices after reload:

            mlx5_core.sf.4
         (subfunction device)
                  /\
                 /  \
                /    \
               /      \
 mlx5_core.vnet.4     no eth, no rdma aux devices
 (SF vnet aux dev)

Above parameters enable user to compose the device as needed based
on the use case.

Since devlink params are done on the devlink instance, these
knobs are uniformly usable for PCI PF, VF and SF devices.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents e9c130ad 70862a5d

Documentation/networking/devlink/devlink-params.rst

+12 −0
Original line number Diff line number Diff line
@@ -97,6 +97,18 @@ own name. 
   * - ``enable_roce``

     - Boolean

     - Enable handling of RoCE traffic in the device.

   * - ``enable_eth``

     - Boolean

     - When enabled, the device driver will instantiate Ethernet specific

       auxiliary device of the devlink device.

   * - ``enable_rdma``

     - Boolean

     - When enabled, the device driver will instantiate RDMA specific

       auxiliary device of the devlink device.

   * - ``enable_vnet``

     - Boolean

     - When enabled, the device driver will instantiate VDPA networking

       specific auxiliary device of the devlink device.

   * - ``internal_err_reset``

     - Boolean

     - When enabled, the device driver will reset the device on internal

drivers/net/ethernet/mellanox/mlx5/core/dev.c

+68 −6
Original line number Diff line number Diff line
@@ -53,7 +53,7 @@ static bool is_eth_rep_supported(struct mlx5_core_dev *dev) 
	return true;

}



static bool is_eth_supported(struct mlx5_core_dev *dev)

bool mlx5_eth_supported(struct mlx5_core_dev *dev)

{

	if (!IS_ENABLED(CONFIG_MLX5_CORE_EN))

		return false;
@@ -105,7 +105,18 @@ static bool is_eth_supported(struct mlx5_core_dev *dev) 
	return true;

}



static bool is_vnet_supported(struct mlx5_core_dev *dev)

static bool is_eth_enabled(struct mlx5_core_dev *dev)

{

	union devlink_param_value val;

	int err;



	err = devlink_param_driverinit_value_get(priv_to_devlink(dev),

						 DEVLINK_PARAM_GENERIC_ID_ENABLE_ETH,

						 &val);

	return err ? false : val.vbool;

}



bool mlx5_vnet_supported(struct mlx5_core_dev *dev)

{

	if (!IS_ENABLED(CONFIG_MLX5_VDPA_NET))

		return false;
@@ -127,6 +138,17 @@ static bool is_vnet_supported(struct mlx5_core_dev *dev) 
	return true;

}



static bool is_vnet_enabled(struct mlx5_core_dev *dev)

{

	union devlink_param_value val;

	int err;



	err = devlink_param_driverinit_value_get(priv_to_devlink(dev),

						 DEVLINK_PARAM_GENERIC_ID_ENABLE_VNET,

						 &val);

	return err ? false : val.vbool;

}



static bool is_ib_rep_supported(struct mlx5_core_dev *dev)

{

	if (!IS_ENABLED(CONFIG_MLX5_INFINIBAND))
@@ -170,7 +192,7 @@ static bool is_mp_supported(struct mlx5_core_dev *dev) 
	return true;

}



static bool is_ib_supported(struct mlx5_core_dev *dev)

bool mlx5_rdma_supported(struct mlx5_core_dev *dev)

{

	if (!IS_ENABLED(CONFIG_MLX5_INFINIBAND))

		return false;
@@ -187,6 +209,17 @@ static bool is_ib_supported(struct mlx5_core_dev *dev) 
	return true;

}



static bool is_ib_enabled(struct mlx5_core_dev *dev)

{

	union devlink_param_value val;

	int err;



	err = devlink_param_driverinit_value_get(priv_to_devlink(dev),

						 DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,

						 &val);

	return err ? false : val.vbool;

}



enum {

	MLX5_INTERFACE_PROTOCOL_ETH,

	MLX5_INTERFACE_PROTOCOL_ETH_REP,
@@ -201,13 +234,17 @@ enum { 
static const struct mlx5_adev_device {

	const char *suffix;

	bool (*is_supported)(struct mlx5_core_dev *dev);

	bool (*is_enabled)(struct mlx5_core_dev *dev);

} mlx5_adev_devices[] = {

	[MLX5_INTERFACE_PROTOCOL_VNET] = { .suffix = "vnet",

					   .is_supported = &is_vnet_supported },

					   .is_supported = &mlx5_vnet_supported,

					   .is_enabled = &is_vnet_enabled },

	[MLX5_INTERFACE_PROTOCOL_IB] = { .suffix = "rdma",

					 .is_supported = &is_ib_supported },

					 .is_supported = &mlx5_rdma_supported,

					 .is_enabled = &is_ib_enabled },

	[MLX5_INTERFACE_PROTOCOL_ETH] = { .suffix = "eth",

					  .is_supported = &is_eth_supported },

					  .is_supported = &mlx5_eth_supported,

					  .is_enabled = &is_eth_enabled },

	[MLX5_INTERFACE_PROTOCOL_ETH_REP] = { .suffix = "eth-rep",

					   .is_supported = &is_eth_rep_supported },

	[MLX5_INTERFACE_PROTOCOL_IB_REP] = { .suffix = "rdma-rep",
@@ -308,6 +345,14 @@ int mlx5_attach_device(struct mlx5_core_dev *dev) 
		if (!priv->adev[i]) {

			bool is_supported = false;



			if (mlx5_adev_devices[i].is_enabled) {

				bool enabled;



				enabled = mlx5_adev_devices[i].is_enabled(dev);

				if (!enabled)

					continue;

			}



			if (mlx5_adev_devices[i].is_supported)

				is_supported = mlx5_adev_devices[i].is_supported(dev);
@@ -360,6 +405,14 @@ void mlx5_detach_device(struct mlx5_core_dev *dev) 
		if (!priv->adev[i])

			continue;



		if (mlx5_adev_devices[i].is_enabled) {

			bool enabled;



			enabled = mlx5_adev_devices[i].is_enabled(dev);

			if (!enabled)

				goto skip_suspend;

		}



		adev = &priv->adev[i]->adev;

		/* Auxiliary driver was unbind manually through sysfs */

		if (!adev->dev.driver)
@@ -447,12 +500,21 @@ static void delete_drivers(struct mlx5_core_dev *dev) 
		if (!priv->adev[i])

			continue;



		if (mlx5_adev_devices[i].is_enabled) {

			bool enabled;



			enabled = mlx5_adev_devices[i].is_enabled(dev);

			if (!enabled)

				goto del_adev;

		}



		if (mlx5_adev_devices[i].is_supported && !delete_all)

			is_supported = mlx5_adev_devices[i].is_supported(dev);



		if (is_supported)

			continue;



del_adev:

		del_adev(&priv->adev[i]->adev);

		priv->adev[i] = NULL;

	}

drivers/net/ethernet/mellanox/mlx5/core/devlink.c

+159 −0
Original line number Diff line number Diff line
@@ -596,6 +596,157 @@ static void mlx5_devlink_set_params_init_values(struct devlink *devlink) 
#endif

}



static const struct devlink_param enable_eth_param =

	DEVLINK_PARAM_GENERIC(ENABLE_ETH, BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),

			      NULL, NULL, NULL);



static int mlx5_devlink_eth_param_register(struct devlink *devlink)

{

	struct mlx5_core_dev *dev = devlink_priv(devlink);

	union devlink_param_value value;

	int err;



	if (!mlx5_eth_supported(dev))

		return 0;



	err = devlink_param_register(devlink, &enable_eth_param);

	if (err)

		return err;



	value.vbool = true;

	devlink_param_driverinit_value_set(devlink,

					   DEVLINK_PARAM_GENERIC_ID_ENABLE_ETH,

					   value);

	devlink_param_publish(devlink, &enable_eth_param);

	return 0;

}



static void mlx5_devlink_eth_param_unregister(struct devlink *devlink)

{

	struct mlx5_core_dev *dev = devlink_priv(devlink);



	if (!mlx5_eth_supported(dev))

		return;



	devlink_param_unpublish(devlink, &enable_eth_param);

	devlink_param_unregister(devlink, &enable_eth_param);

}



static int mlx5_devlink_enable_rdma_validate(struct devlink *devlink, u32 id,

					     union devlink_param_value val,

					     struct netlink_ext_ack *extack)

{

	struct mlx5_core_dev *dev = devlink_priv(devlink);

	bool new_state = val.vbool;



	if (new_state && !mlx5_rdma_supported(dev))

		return -EOPNOTSUPP;

	return 0;

}



static const struct devlink_param enable_rdma_param =

	DEVLINK_PARAM_GENERIC(ENABLE_RDMA, BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),

			      NULL, NULL, mlx5_devlink_enable_rdma_validate);



static int mlx5_devlink_rdma_param_register(struct devlink *devlink)

{

	struct mlx5_core_dev *dev = devlink_priv(devlink);

	union devlink_param_value value;

	int err;



	if (!IS_ENABLED(CONFIG_MLX5_INFINIBAND) || MLX5_ESWITCH_MANAGER(dev))

		return 0;



	err = devlink_param_register(devlink, &enable_rdma_param);

	if (err)

		return err;



	value.vbool = true;

	devlink_param_driverinit_value_set(devlink,

					   DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,

					   value);

	devlink_param_publish(devlink, &enable_rdma_param);

	return 0;

}



static void mlx5_devlink_rdma_param_unregister(struct devlink *devlink)

{

	struct mlx5_core_dev *dev = devlink_priv(devlink);



	if (!IS_ENABLED(CONFIG_MLX5_INFINIBAND) || MLX5_ESWITCH_MANAGER(dev))

		return;



	devlink_param_unpublish(devlink, &enable_rdma_param);

	devlink_param_unregister(devlink, &enable_rdma_param);

}



static const struct devlink_param enable_vnet_param =

	DEVLINK_PARAM_GENERIC(ENABLE_VNET, BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),

			      NULL, NULL, NULL);



static int mlx5_devlink_vnet_param_register(struct devlink *devlink)

{

	struct mlx5_core_dev *dev = devlink_priv(devlink);

	union devlink_param_value value;

	int err;



	if (!mlx5_vnet_supported(dev))

		return 0;



	err = devlink_param_register(devlink, &enable_vnet_param);

	if (err)

		return err;



	value.vbool = true;

	devlink_param_driverinit_value_set(devlink,

					   DEVLINK_PARAM_GENERIC_ID_ENABLE_VNET,

					   value);

	devlink_param_publish(devlink, &enable_rdma_param);

	return 0;

}



static void mlx5_devlink_vnet_param_unregister(struct devlink *devlink)

{

	struct mlx5_core_dev *dev = devlink_priv(devlink);



	if (!mlx5_vnet_supported(dev))

		return;



	devlink_param_unpublish(devlink, &enable_vnet_param);

	devlink_param_unregister(devlink, &enable_vnet_param);

}



static int mlx5_devlink_auxdev_params_register(struct devlink *devlink)

{

	int err;



	err = mlx5_devlink_eth_param_register(devlink);

	if (err)

		return err;



	err = mlx5_devlink_rdma_param_register(devlink);

	if (err)

		goto rdma_err;



	err = mlx5_devlink_vnet_param_register(devlink);

	if (err)

		goto vnet_err;

	return 0;



vnet_err:

	mlx5_devlink_rdma_param_unregister(devlink);

rdma_err:

	mlx5_devlink_eth_param_unregister(devlink);

	return err;

}



static void mlx5_devlink_auxdev_params_unregister(struct devlink *devlink)

{

	mlx5_devlink_vnet_param_unregister(devlink);

	mlx5_devlink_rdma_param_unregister(devlink);

	mlx5_devlink_eth_param_unregister(devlink);

}



#define MLX5_TRAP_DROP(_id, _group_id)					\

	DEVLINK_TRAP_GENERIC(DROP, DROP, _id,				\

			     DEVLINK_TRAP_GROUP_GENERIC_ID_##_group_id, \
@@ -654,6 +805,10 @@ int mlx5_devlink_register(struct devlink *devlink) 
	mlx5_devlink_set_params_init_values(devlink);

	devlink_params_publish(devlink);



	err = mlx5_devlink_auxdev_params_register(devlink);

	if (err)

		goto auxdev_reg_err;



	err = mlx5_devlink_traps_register(devlink);

	if (err)

		goto traps_reg_err;
@@ -661,6 +816,8 @@ int mlx5_devlink_register(struct devlink *devlink) 
	return 0;



traps_reg_err:

	mlx5_devlink_auxdev_params_unregister(devlink);

auxdev_reg_err:

	devlink_params_unregister(devlink, mlx5_devlink_params,

				  ARRAY_SIZE(mlx5_devlink_params));

params_reg_err:
@@ -671,6 +828,8 @@ int mlx5_devlink_register(struct devlink *devlink) 
void mlx5_devlink_unregister(struct devlink *devlink)

{

	mlx5_devlink_traps_unregister(devlink);

	mlx5_devlink_auxdev_params_unregister(devlink);

	devlink_params_unpublish(devlink);

	devlink_params_unregister(devlink, mlx5_devlink_params,

				  ARRAY_SIZE(mlx5_devlink_params));

	devlink_unregister(devlink);

drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h

+5 −0
Original line number Diff line number Diff line
@@ -272,4 +272,9 @@ static inline u32 mlx5_sriov_get_vf_total_msix(struct pci_dev *pdev) 


	return MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix);

}



bool mlx5_eth_supported(struct mlx5_core_dev *dev);

bool mlx5_rdma_supported(struct mlx5_core_dev *dev);

bool mlx5_vnet_supported(struct mlx5_core_dev *dev);



#endif /* __MLX5_CORE_H__ */

include/net/devlink.h

+20 −0
Original line number Diff line number Diff line
@@ -519,6 +519,9 @@ enum devlink_param_generic_id { 
	DEVLINK_PARAM_GENERIC_ID_RESET_DEV_ON_DRV_PROBE,

	DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE,

	DEVLINK_PARAM_GENERIC_ID_ENABLE_REMOTE_DEV_RESET,

	DEVLINK_PARAM_GENERIC_ID_ENABLE_ETH,

	DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,

	DEVLINK_PARAM_GENERIC_ID_ENABLE_VNET,



	/* add new param generic ids above here*/

	__DEVLINK_PARAM_GENERIC_ID_MAX,
@@ -559,6 +562,15 @@ enum devlink_param_generic_id { 
#define DEVLINK_PARAM_GENERIC_ENABLE_REMOTE_DEV_RESET_NAME "enable_remote_dev_reset"

#define DEVLINK_PARAM_GENERIC_ENABLE_REMOTE_DEV_RESET_TYPE DEVLINK_PARAM_TYPE_BOOL



#define DEVLINK_PARAM_GENERIC_ENABLE_ETH_NAME "enable_eth"

#define DEVLINK_PARAM_GENERIC_ENABLE_ETH_TYPE DEVLINK_PARAM_TYPE_BOOL



#define DEVLINK_PARAM_GENERIC_ENABLE_RDMA_NAME "enable_rdma"

#define DEVLINK_PARAM_GENERIC_ENABLE_RDMA_TYPE DEVLINK_PARAM_TYPE_BOOL



#define DEVLINK_PARAM_GENERIC_ENABLE_VNET_NAME "enable_vnet"

#define DEVLINK_PARAM_GENERIC_ENABLE_VNET_TYPE DEVLINK_PARAM_TYPE_BOOL



#define DEVLINK_PARAM_GENERIC(_id, _cmodes, _get, _set, _validate)	\

{									\

	.id = DEVLINK_PARAM_GENERIC_ID_##_id,				\
@@ -1633,8 +1645,16 @@ int devlink_params_register(struct devlink *devlink, 
void devlink_params_unregister(struct devlink *devlink,

			       const struct devlink_param *params,

			       size_t params_count);

int devlink_param_register(struct devlink *devlink,

			   const struct devlink_param *param);

void devlink_param_unregister(struct devlink *devlink,

			      const struct devlink_param *param);

void devlink_params_publish(struct devlink *devlink);

void devlink_params_unpublish(struct devlink *devlink);

void devlink_param_publish(struct devlink *devlink,

			   const struct devlink_param *param);

void devlink_param_unpublish(struct devlink *devlink,

			     const struct devlink_param *param);

int devlink_port_params_register(struct devlink_port *devlink_port,

				 const struct devlink_param *params,

				 size_t params_count);