Commit 1958fc2f authored by Parav Pandit's avatar Parav Pandit Committed by Saeed Mahameed
Browse files

net/mlx5: SF, Add auxiliary device driver 

Add auxiliary device driver for mlx5 subfunction auxiliary device.

A mlx5 subfunction is similar to PCI PF and VF. For a subfunction
an auxiliary device is created.

As a result, when mlx5 SF auxiliary device binds to the driver,
its netdev and rdma device are created, they appear as

$ ls -l /sys/bus/auxiliary/devices/
mlx5_core.sf.4 -> ../../../devices/pci0000:00/0000:00:03.0/0000:06:00.0/mlx5_core.sf.4

$ ls -l /sys/class/net/eth1/device
/sys/class/net/eth1/device -> ../../../mlx5_core.sf.4

$ cat /sys/bus/auxiliary/devices/mlx5_core.sf.4/sfnum
88

$ devlink dev show
pci/0000:06:00.0
auxiliary/mlx5_core.sf.4

$ devlink port show auxiliary/mlx5_core.sf.4/1
auxiliary/mlx5_core.sf.4/1: type eth netdev p0sf88 flavour virtual port 0 splittable false

$ rdma link show mlx5_0/1
link mlx5_0/1 state ACTIVE physical_state LINK_UP netdev p0sf88

$ rdma dev show
8: rocep6s0f1: node_type ca fw 16.29.0550 node_guid 248a:0703:00b3:d113 sys_image_guid 248a:0703:00b3:d112
13: mlx5_0: node_type ca fw 16.29.0550 node_guid 0000:00ff:fe00:8888 sys_image_guid 248a:0703:00b3:d112

In future, devlink device instance name will adapt to have sfnum
annotation using either an alias or as devlink instance name described
in RFC [1].

[1] https://lore.kernel.org/netdev/20200519092258.GF4655@nanopsycho/



Signed-off-by: default avatarParav Pandit <parav@nvidia.com>
Reviewed-by: default avatarVu Pham <vuhuong@nvidia.com>
Signed-off-by: default avatarSaeed Mahameed <saeedm@nvidia.com>
parent 90d010b8

drivers/net/ethernet/mellanox/mlx5/core/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -89,4 +89,4 @@ mlx5_core-$(CONFIG_MLX5_SW_STEERING) += steering/dr_domain.o steering/dr_table.o 
#

# SF device

#

mlx5_core-$(CONFIG_MLX5_SF) += sf/vhca_event.o sf/dev/dev.o
 
mlx5_core-$(CONFIG_MLX5_SF) += sf/vhca_event.o sf/dev/dev.o sf/dev/driver.o

drivers/net/ethernet/mellanox/mlx5/core/devlink.c

+12 −0
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ 
#include "fw_reset.h"

#include "fs_core.h"

#include "eswitch.h"

#include "sf/dev/dev.h"



static int mlx5_devlink_flash_update(struct devlink *devlink,

				     struct devlink_flash_update_params *params,
@@ -127,6 +128,17 @@ static int mlx5_devlink_reload_down(struct devlink *devlink, bool netns_change, 
				    struct netlink_ext_ack *extack)

{

	struct mlx5_core_dev *dev = devlink_priv(devlink);

	bool sf_dev_allocated;



	sf_dev_allocated = mlx5_sf_dev_allocated(dev);

	if (sf_dev_allocated) {

		/* Reload results in deleting SF device which further results in

		 * unregistering devlink instance while holding devlink_mutext.

		 * Hence, do not support reload.

		 */

		NL_SET_ERR_MSG_MOD(extack, "reload is unsupported when SFs are allocated\n");

		return -EOPNOTSUPP;

	}



	switch (action) {

	case DEVLINK_RELOAD_ACTION_DRIVER_REINIT:

drivers/net/ethernet/mellanox/mlx5/core/eq.c

+1 −1
Original line number Diff line number Diff line
@@ -467,7 +467,7 @@ int mlx5_eq_table_init(struct mlx5_core_dev *dev) 
	for (i = 0; i < MLX5_EVENT_TYPE_MAX; i++)

		ATOMIC_INIT_NOTIFIER_HEAD(&eq_table->nh[i]);



	eq_table->irq_table = dev->priv.irq_table;

	eq_table->irq_table = mlx5_irq_table_get(dev);

	return 0;

}

drivers/net/ethernet/mellanox/mlx5/core/main.c

+9 −3
Original line number Diff line number Diff line
@@ -84,7 +84,6 @@ unsigned int mlx5_core_debug_mask; 
module_param_named(debug_mask, mlx5_core_debug_mask, uint, 0644);

MODULE_PARM_DESC(debug_mask, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0");



#define MLX5_DEFAULT_PROF	2

static unsigned int prof_sel = MLX5_DEFAULT_PROF;

module_param_named(prof_sel, prof_sel, uint, 0444);

MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2");
@@ -1303,7 +1302,7 @@ void mlx5_unload_one(struct mlx5_core_dev *dev, bool cleanup) 
	mutex_unlock(&dev->intf_state_mutex);

}



static int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx)

int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx)

{

	struct mlx5_priv *priv = &dev->priv;

	int err;
@@ -1353,7 +1352,7 @@ static int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) 
	return err;

}



static void mlx5_mdev_uninit(struct mlx5_core_dev *dev)

void mlx5_mdev_uninit(struct mlx5_core_dev *dev)

{

	struct mlx5_priv *priv = &dev->priv;
@@ -1696,6 +1695,10 @@ static int __init init(void) 
	if (err)

		goto err_debug;



	err = mlx5_sf_driver_register();

	if (err)

		goto err_sf;



#ifdef CONFIG_MLX5_CORE_EN

	err = mlx5e_init();

	if (err) {
@@ -1706,6 +1709,8 @@ static int __init init(void) 


	return 0;



err_sf:

	pci_unregister_driver(&mlx5_core_driver);

err_debug:

	mlx5_unregister_debugfs();

	return err;
@@ -1716,6 +1721,7 @@ static void __exit cleanup(void) 
#ifdef CONFIG_MLX5_CORE_EN

	mlx5e_cleanup();

#endif

	mlx5_sf_driver_unregister();

	pci_unregister_driver(&mlx5_core_driver);

	mlx5_unregister_debugfs();

}

drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h

+10 −0
Original line number Diff line number Diff line
@@ -117,6 +117,8 @@ enum mlx5_semaphore_space_address { 
	MLX5_SEMAPHORE_SW_RESET         = 0x20,

};



#define MLX5_DEFAULT_PROF       2



int mlx5_query_hca_caps(struct mlx5_core_dev *dev);

int mlx5_query_board_id(struct mlx5_core_dev *dev);

int mlx5_cmd_init(struct mlx5_core_dev *dev);
@@ -176,6 +178,7 @@ struct cpumask * 
mlx5_irq_get_affinity_mask(struct mlx5_irq_table *irq_table, int vecidx);

struct cpu_rmap *mlx5_irq_get_rmap(struct mlx5_irq_table *table);

int mlx5_irq_get_num_comp(struct mlx5_irq_table *table);

struct mlx5_irq_table *mlx5_irq_table_get(struct mlx5_core_dev *dev);



int mlx5_events_init(struct mlx5_core_dev *dev);

void mlx5_events_cleanup(struct mlx5_core_dev *dev);
@@ -257,6 +260,13 @@ enum { 
u8 mlx5_get_nic_state(struct mlx5_core_dev *dev);

void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state);



static inline bool mlx5_core_is_sf(const struct mlx5_core_dev *dev)

{

	return dev->coredev_type == MLX5_COREDEV_SF;

}



int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx);

void mlx5_mdev_uninit(struct mlx5_core_dev *dev);

void mlx5_unload_one(struct mlx5_core_dev *dev, bool cleanup);

int mlx5_load_one(struct mlx5_core_dev *dev, bool boot);