Commit 8f010541 authored by Parav Pandit's avatar Parav Pandit Committed by Saeed Mahameed
Browse files

net/mlx5: SF, Add port add delete functionality 



To handle SF port management outside of the eswitch as independent
software layer, introduce eswitch notifier APIs so that mlx5 upper
layer who wish to support sf port management in switchdev mode can
perform its task whenever eswitch mode is set to switchdev or before
eswitch is disabled.

Initialize sf port table on such eswitch event.

Add SF port add and delete functionality in switchdev mode.
Destroy all SF ports when eswitch is disabled.
Expose SF port add and delete to user via devlink commands.

$ devlink dev eswitch set pci/0000:06:00.0 mode switchdev

$ devlink port show
pci/0000:06:00.0/65535: type eth netdev ens2f0np0 flavour physical port 0 splittable false

$ devlink port add pci/0000:06:00.0 flavour pcisf pfnum 0 sfnum 88
pci/0000:06:00.0/32768: type eth netdev eth6 flavour pcisf controller 0 pfnum 0 sfnum 88 external false splittable false
  function:
    hw_addr 00:00:00:00:00:00 state inactive opstate detached

$ devlink port show ens2f0npf0sf88
pci/0000:06:00.0/32768: type eth netdev ens2f0npf0sf88 flavour pcisf controller 0 pfnum 0 sfnum 88 external false splittable false
  function:
    hw_addr 00:00:00:00:00:00 state inactive opstate detached

or by its unique port index:
$ devlink port show pci/0000:06:00.0/32768
pci/0000:06:00.0/32768: type eth netdev ens2f0npf0sf88 flavour pcisf controller 0 pfnum 0 sfnum 88 external false splittable false
  function:
    hw_addr 00:00:00:00:00:00 state inactive opstate detached

$ devlink port show ens2f0npf0sf88 -jp
{
    "port": {
        "pci/0000:06:00.0/32768": {
            "type": "eth",
            "netdev": "ens2f0npf0sf88",
            "flavour": "pcisf",
            "controller": 0,
            "pfnum": 0,
            "sfnum": 88,
            "external": false,
            "splittable": false,
            "function": {
                "hw_addr": "00:00:00:00:00:00",
                "state": "inactive",
                "opstate": "detached"
            }
        }
    }
}

Signed-off-by: default avatarParav Pandit <parav@nvidia.com>
Reviewed-by: default avatarVu Pham <vuhuong@nvidia.com>
Signed-off-by: default avatarSaeed Mahameed <saeedm@nvidia.com>
parent d970812b

drivers/net/ethernet/mellanox/mlx5/core/Kconfig

+10 −0
Original line number Diff line number Diff line
@@ -212,3 +212,13 @@ config MLX5_SF 
	Build support for subfuction device in the NIC. A Mellanox subfunction

	device can support RDMA, netdevice and vdpa device.

	It is similar to a SRIOV VF but it doesn't require SRIOV support.



config MLX5_SF_MANAGER

	bool

	depends on MLX5_SF && MLX5_ESWITCH

	default y

	help

	Build support for subfuction port in the NIC. A Mellanox subfunction

	port is managed through devlink.  A subfunction supports RDMA, netdevice

	and vdpa device. It is similar to a SRIOV VF but it doesn't require

	SRIOV support.

drivers/net/ethernet/mellanox/mlx5/core/Makefile

+5 −0
Original line number Diff line number Diff line
@@ -90,3 +90,8 @@ mlx5_core-$(CONFIG_MLX5_SW_STEERING) += steering/dr_domain.o steering/dr_table.o 
# SF device

#

mlx5_core-$(CONFIG_MLX5_SF) += sf/vhca_event.o sf/dev/dev.o sf/dev/driver.o



#

# SF manager

#

mlx5_core-$(CONFIG_MLX5_SF_MANAGER) += sf/cmd.o sf/hw_table.o sf/devlink.o

drivers/net/ethernet/mellanox/mlx5/core/cmd.c

+4 −0
Original line number Diff line number Diff line
@@ -333,6 +333,7 @@ static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op, 
	case MLX5_CMD_OP_DEALLOC_MEMIC:

	case MLX5_CMD_OP_PAGE_FAULT_RESUME:

	case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS:

	case MLX5_CMD_OP_DEALLOC_SF:

		return MLX5_CMD_STAT_OK;



	case MLX5_CMD_OP_QUERY_HCA_CAP:
@@ -466,6 +467,7 @@ static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op, 
	case MLX5_CMD_OP_RELEASE_XRQ_ERROR:

	case MLX5_CMD_OP_QUERY_VHCA_STATE:

	case MLX5_CMD_OP_MODIFY_VHCA_STATE:

	case MLX5_CMD_OP_ALLOC_SF:

		*status = MLX5_DRIVER_STATUS_ABORTED;

		*synd = MLX5_DRIVER_SYND;

		return -EIO;
@@ -661,6 +663,8 @@ const char *mlx5_command_str(int command) 
	MLX5_COMMAND_STR_CASE(MODIFY_XRQ);

	MLX5_COMMAND_STR_CASE(QUERY_VHCA_STATE);

	MLX5_COMMAND_STR_CASE(MODIFY_VHCA_STATE);

	MLX5_COMMAND_STR_CASE(ALLOC_SF);

	MLX5_COMMAND_STR_CASE(DEALLOC_SF);

	default: return "unknown command opcode";

	}

}

drivers/net/ethernet/mellanox/mlx5/core/devlink.c

+5 −0
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ 
#include "fs_core.h"

#include "eswitch.h"

#include "sf/dev/dev.h"

#include "sf/sf.h"



static int mlx5_devlink_flash_update(struct devlink *devlink,

				     struct devlink_flash_update_params *params,
@@ -190,6 +191,10 @@ static const struct devlink_ops mlx5_devlink_ops = { 
	.eswitch_encap_mode_get = mlx5_devlink_eswitch_encap_mode_get,

	.port_function_hw_addr_get = mlx5_devlink_port_function_hw_addr_get,

	.port_function_hw_addr_set = mlx5_devlink_port_function_hw_addr_set,

#endif

#ifdef CONFIG_MLX5_SF_MANAGER

	.port_new = mlx5_devlink_sf_port_new,

	.port_del = mlx5_devlink_sf_port_del,

#endif

	.flash_update = mlx5_devlink_flash_update,

	.info_get = mlx5_devlink_info_get,

drivers/net/ethernet/mellanox/mlx5/core/eswitch.c

+25 −0
Original line number Diff line number Diff line
@@ -1599,6 +1599,15 @@ mlx5_eswitch_update_num_of_vfs(struct mlx5_eswitch *esw, int num_vfs) 
	kvfree(out);

}



static void mlx5_esw_mode_change_notify(struct mlx5_eswitch *esw, u16 mode)

{

	struct mlx5_esw_event_info info = {};



	info.new_mode = mode;



	blocking_notifier_call_chain(&esw->n_head, 0, &info);

}



/**

 * mlx5_eswitch_enable_locked - Enable eswitch

 * @esw:	Pointer to eswitch
@@ -1659,6 +1668,8 @@ int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int mode, int num_vfs) 
		 mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS",

		 esw->esw_funcs.num_vfs, esw->enabled_vports);



	mlx5_esw_mode_change_notify(esw, mode);



	return 0;



abort:
@@ -1715,6 +1726,11 @@ void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw, bool clear_vf) 
		 esw->mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS",

		 esw->esw_funcs.num_vfs, esw->enabled_vports);



	/* Notify eswitch users that it is exiting from current mode.

	 * So that it can do necessary cleanup before the eswitch is disabled.

	 */

	mlx5_esw_mode_change_notify(esw, MLX5_ESWITCH_NONE);



	mlx5_eswitch_event_handlers_unregister(esw);



	if (esw->mode == MLX5_ESWITCH_LEGACY)
@@ -1815,6 +1831,7 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev) 
	esw->offloads.inline_mode = MLX5_INLINE_MODE_NONE;



	dev->priv.eswitch = esw;

	BLOCKING_INIT_NOTIFIER_HEAD(&esw->n_head);

	return 0;

abort:

	if (esw->work_queue)
@@ -2506,4 +2523,12 @@ bool mlx5_esw_multipath_prereq(struct mlx5_core_dev *dev0, 
		dev1->priv.eswitch->mode == MLX5_ESWITCH_OFFLOADS);

}



int mlx5_esw_event_notifier_register(struct mlx5_eswitch *esw, struct notifier_block *nb)

{

	return blocking_notifier_chain_register(&esw->n_head, nb);

}



void mlx5_esw_event_notifier_unregister(struct mlx5_eswitch *esw, struct notifier_block *nb)

{

	blocking_notifier_chain_unregister(&esw->n_head, nb);

}