Commit 145e5637 authored by Eran Ben Elisha's avatar Eran Ben Elisha Committed by Saeed Mahameed
Browse files

net/mlx5e: Add TX PTP port object support 



Add TX PTP port object support for better TX timestamping accuracy.
Currently, driver supports CQE based TX port timestamp. Device
also offers TX port timestamp, which has less jitter and better
reflects the actual time of a packet's transmit.

Define new driver layout called ptpsq, on which driver will create
SQs that will support TX port timestamp for their transmitted packets.
Driver to identify PTP TX skbs and steer them to these dedicated SQs
as part of the select queue ndo.

Driver to hold ptpsq per TC and report them at
netif_set_real_num_tx_queues().

Add support for all needed functionality in order to xmit and poll
completions received via ptpsq.

Add ptpsq to the TX reporter recover, diagnose and dump methods.

Creation of ptpsqs is disabled by default, and can be enabled via
tx_port_ts private flag.

This patch steer all timestamp related packets to a ptpsq, but it
does not open the port timestamp support for it. The support will
be added in the following patch.

Signed-off-by: default avatarEran Ben Elisha <eranbe@nvidia.com>
Reviewed-by: default avatarTariq Toukan <tariqt@nvidia.com>
Signed-off-by: default avatarSaeed Mahameed <saeedm@nvidia.com>
parent cecaa6a7

drivers/net/ethernet/mellanox/mlx5/core/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -25,7 +25,7 @@ mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \ 
		en_tx.o en_rx.o en_dim.o en_txrx.o en/xdp.o en_stats.o \

		en_selftest.o en/port.o en/monitor_stats.o en/health.o \

		en/reporter_tx.o en/reporter_rx.o en/params.o en/xsk/pool.o \

		en/xsk/setup.o en/xsk/rx.o en/xsk/tx.o en/devlink.o

		en/xsk/setup.o en/xsk/rx.o en/xsk/tx.o en/devlink.o en/ptp.o



#

# Netdev extra

drivers/net/ethernet/mellanox/mlx5/core/en.h

+28 −1
Original line number Diff line number Diff line
@@ -227,6 +227,7 @@ enum mlx5e_priv_flag { 
	MLX5E_PFLAG_RX_NO_CSUM_COMPLETE,

	MLX5E_PFLAG_XDP_TX_MPWQE,

	MLX5E_PFLAG_SKB_TX_MPWQE,

	MLX5E_PFLAG_TX_PORT_TS,

	MLX5E_NUM_PFLAGS, /* Keep last */

};
@@ -338,6 +339,8 @@ struct mlx5e_skb_fifo { 
	u16 mask;

};



struct mlx5e_ptpsq;



struct mlx5e_txqsq {

	/* data path */
@@ -385,6 +388,7 @@ struct mlx5e_txqsq { 
	int                        txq_ix;

	u32                        rate_limit;

	struct work_struct         recover_work;

	struct mlx5e_ptpsq        *ptpsq;

} ____cacheline_aligned_in_smp;



struct mlx5e_dma_info {
@@ -692,8 +696,11 @@ struct mlx5e_channel { 
	int                        cpu;

};



struct mlx5e_port_ptp;



struct mlx5e_channels {

	struct mlx5e_channel **c;

	struct mlx5e_port_ptp  *port_ptp;

	unsigned int           num;

	struct mlx5e_params    params;

};
@@ -708,6 +715,11 @@ struct mlx5e_channel_stats { 
	struct mlx5e_xdpsq_stats xsksq;

} ____cacheline_aligned_in_smp;



struct mlx5e_port_ptp_stats {

	struct mlx5e_ch_stats ch;

	struct mlx5e_sq_stats sq[MLX5E_MAX_NUM_TC];

} ____cacheline_aligned_in_smp;



enum {

	MLX5E_STATE_OPENED,

	MLX5E_STATE_DESTROYING,
@@ -777,8 +789,10 @@ struct mlx5e_scratchpad { 


struct mlx5e_priv {

	/* priv data path fields - start */

	struct mlx5e_txqsq *txq2sq[MLX5E_MAX_NUM_CHANNELS * MLX5E_MAX_NUM_TC];

	/* +1 for port ptp ts */

	struct mlx5e_txqsq *txq2sq[(MLX5E_MAX_NUM_CHANNELS + 1) * MLX5E_MAX_NUM_TC];

	int channel_tc2realtxq[MLX5E_MAX_NUM_CHANNELS][MLX5E_MAX_NUM_TC];

	int port_ptp_tc2realtxq[MLX5E_MAX_NUM_TC];

#ifdef CONFIG_MLX5_CORE_EN_DCB

	struct mlx5e_dcbx_dp       dcbx_dp;

#endif
@@ -813,12 +827,15 @@ struct mlx5e_priv { 
	struct net_device         *netdev;

	struct mlx5e_stats         stats;

	struct mlx5e_channel_stats channel_stats[MLX5E_MAX_NUM_CHANNELS];

	struct mlx5e_port_ptp_stats port_ptp_stats;

	u16                        max_nch;

	u8                         max_opened_tc;

	bool                       port_ptp_opened;

	struct hwtstamp_config     tstamp;

	u16                        q_counter;

	u16                        drop_rq_q_counter;

	struct notifier_block      events_nb;

	int                        num_tc_x_num_ch;



	struct udp_tunnel_nic_info nic_info;

#ifdef CONFIG_MLX5_CORE_EN_DCB
@@ -993,7 +1010,17 @@ void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq); 
int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn,

		    struct mlx5e_modify_sq_param *p);

void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq);

void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq);

void mlx5e_free_txqsq(struct mlx5e_txqsq *sq);

void mlx5e_tx_disable_queue(struct netdev_queue *txq);

int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa);

void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq);

struct mlx5e_create_sq_param;

int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev,

			struct mlx5e_sq_param *param,

			struct mlx5e_create_sq_param *csp,

			u32 *sqn);

void mlx5e_tx_err_cqe_work(struct work_struct *recover_work);



static inline bool mlx5_tx_swp_supported(struct mlx5_core_dev *mdev)

{

drivers/net/ethernet/mellanox/mlx5/core/en/params.h

+8 −0
Original line number Diff line number Diff line
@@ -41,6 +41,14 @@ struct mlx5e_channel_param { 
	struct mlx5e_sq_param      async_icosq;

};



struct mlx5e_create_sq_param {

	struct mlx5_wq_ctrl        *wq_ctrl;

	u32                         cqn;

	u32                         tisn;

	u8                          tis_lst_sz;

	u8                          min_inline_mode;

};



static inline bool mlx5e_qid_get_ch_if_in_group(struct mlx5e_params *params,

						u16 qid,

						enum mlx5e_rq_group group,

drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c

0 → 100644
+360 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB

// Copyright (c) 2020 Mellanox Technologies



#include "en/ptp.h"

#include "en/txrx.h"

#include "lib/clock.h"



static int mlx5e_ptp_napi_poll(struct napi_struct *napi, int budget)

{

	struct mlx5e_port_ptp *c = container_of(napi, struct mlx5e_port_ptp,

						napi);

	struct mlx5e_ch_stats *ch_stats = c->stats;

	bool busy = false;

	int work_done = 0;

	int i;



	rcu_read_lock();



	ch_stats->poll++;



	for (i = 0; i < c->num_tc; i++)

		busy |= mlx5e_poll_tx_cq(&c->ptpsq[i].txqsq.cq, budget);



	if (busy) {

		work_done = budget;

		goto out;

	}



	if (unlikely(!napi_complete_done(napi, work_done)))

		goto out;



	ch_stats->arm++;



	for (i = 0; i < c->num_tc; i++)

		mlx5e_cq_arm(&c->ptpsq[i].txqsq.cq);



out:

	rcu_read_unlock();



	return work_done;

}



static int mlx5e_ptp_alloc_txqsq(struct mlx5e_port_ptp *c, int txq_ix,

				 struct mlx5e_params *params,

				 struct mlx5e_sq_param *param,

				 struct mlx5e_txqsq *sq, int tc,

				 struct mlx5e_ptpsq *ptpsq)

{

	void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);

	struct mlx5_core_dev *mdev = c->mdev;

	struct mlx5_wq_cyc *wq = &sq->wq;

	int err;

	int node;



	sq->pdev      = c->pdev;

	sq->tstamp    = c->tstamp;

	sq->clock     = &mdev->clock;

	sq->mkey_be   = c->mkey_be;

	sq->netdev    = c->netdev;

	sq->priv      = c->priv;

	sq->mdev      = mdev;

	sq->ch_ix     = c->ix;

	sq->txq_ix    = txq_ix;

	sq->uar_map   = mdev->mlx5e_res.bfreg.map;

	sq->min_inline_mode = params->tx_min_inline_mode;

	sq->hw_mtu    = MLX5E_SW2HW_MTU(params, params->sw_mtu);

	sq->stats     = &c->priv->port_ptp_stats.sq[tc];

	sq->ptpsq     = ptpsq;

	INIT_WORK(&sq->recover_work, mlx5e_tx_err_cqe_work);

	if (!MLX5_CAP_ETH(mdev, wqe_vlan_insert))

		set_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state);

	sq->stop_room = param->stop_room;



	node = dev_to_node(mlx5_core_dma_dev(mdev));



	param->wq.db_numa_node = node;

	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);

	if (err)

		return err;

	wq->db    = &wq->db[MLX5_SND_DBR];



	err = mlx5e_alloc_txqsq_db(sq, node);

	if (err)

		goto err_sq_wq_destroy;



	return 0;



err_sq_wq_destroy:

	mlx5_wq_destroy(&sq->wq_ctrl);



	return err;

}



static void mlx5e_ptp_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn)

{

	mlx5_core_destroy_sq(mdev, sqn);

}



static int mlx5e_ptp_open_txqsq(struct mlx5e_port_ptp *c, u32 tisn,

				int txq_ix, struct mlx5e_ptp_params *cparams,

				int tc, struct mlx5e_ptpsq *ptpsq)

{

	struct mlx5e_sq_param *sqp = &cparams->txq_sq_param;

	struct mlx5e_txqsq *txqsq = &ptpsq->txqsq;

	struct mlx5e_create_sq_param csp = {};

	int err;



	err = mlx5e_ptp_alloc_txqsq(c, txq_ix, &cparams->params, sqp,

				    txqsq, tc, ptpsq);

	if (err)

		return err;



	csp.tisn            = tisn;

	csp.tis_lst_sz      = 1;

	csp.cqn             = txqsq->cq.mcq.cqn;

	csp.wq_ctrl         = &txqsq->wq_ctrl;

	csp.min_inline_mode = txqsq->min_inline_mode;



	err = mlx5e_create_sq_rdy(c->mdev, sqp, &csp, &txqsq->sqn);

	if (err)

		goto err_free_txqsq;



	return 0;



err_free_txqsq:

	mlx5e_free_txqsq(txqsq);



	return err;

}



static void mlx5e_ptp_close_txqsq(struct mlx5e_ptpsq *ptpsq)

{

	struct mlx5e_txqsq *sq = &ptpsq->txqsq;

	struct mlx5_core_dev *mdev = sq->mdev;



	cancel_work_sync(&sq->recover_work);

	mlx5e_ptp_destroy_sq(mdev, sq->sqn);

	mlx5e_free_txqsq_descs(sq);

	mlx5e_free_txqsq(sq);

}



static int mlx5e_ptp_open_txqsqs(struct mlx5e_port_ptp *c,

				 struct mlx5e_ptp_params *cparams)

{

	struct mlx5e_params *params = &cparams->params;

	int ix_base;

	int err;

	int tc;



	ix_base = params->num_tc * params->num_channels;



	for (tc = 0; tc < params->num_tc; tc++) {

		int txq_ix = ix_base + tc;



		err = mlx5e_ptp_open_txqsq(c, c->priv->tisn[c->lag_port][tc], txq_ix,

					   cparams, tc, &c->ptpsq[tc]);

		if (err)

			goto close_txqsq;

	}



	return 0;



close_txqsq:

	for (--tc; tc >= 0; tc--)

		mlx5e_ptp_close_txqsq(&c->ptpsq[tc]);



	return err;

}



static void mlx5e_ptp_close_txqsqs(struct mlx5e_port_ptp *c)

{

	int tc;



	for (tc = 0; tc < c->num_tc; tc++)

		mlx5e_ptp_close_txqsq(&c->ptpsq[tc]);

}



static int mlx5e_ptp_open_cqs(struct mlx5e_port_ptp *c,

			      struct mlx5e_ptp_params *cparams)

{

	struct mlx5e_params *params = &cparams->params;

	struct mlx5e_create_cq_param ccp = {};

	struct dim_cq_moder ptp_moder = {};

	struct mlx5e_cq_param *cq_param;

	int err;

	int tc;



	ccp.node     = dev_to_node(mlx5_core_dma_dev(c->mdev));

	ccp.ch_stats = c->stats;

	ccp.napi     = &c->napi;

	ccp.ix       = c->ix;



	cq_param = &cparams->txq_sq_param.cqp;



	for (tc = 0; tc < params->num_tc; tc++) {

		struct mlx5e_cq *cq = &c->ptpsq[tc].txqsq.cq;



		err = mlx5e_open_cq(c->priv, ptp_moder, cq_param, &ccp, cq);

		if (err)

			goto out_err_txqsq_cq;

	}



	return 0;



out_err_txqsq_cq:

	for (--tc; tc >= 0; tc--)

		mlx5e_close_cq(&c->ptpsq[tc].txqsq.cq);



	return err;

}



static void mlx5e_ptp_close_cqs(struct mlx5e_port_ptp *c)

{

	int tc;



	for (tc = 0; tc < c->num_tc; tc++)

		mlx5e_close_cq(&c->ptpsq[tc].txqsq.cq);

}



static void mlx5e_ptp_build_sq_param(struct mlx5e_priv *priv,

				     struct mlx5e_params *params,

				     struct mlx5e_sq_param *param)

{

	void *sqc = param->sqc;

	void *wq;



	mlx5e_build_sq_param_common(priv, param);



	wq = MLX5_ADDR_OF(sqc, sqc, wq);

	MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size);

	param->stop_room = mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS);

	mlx5e_build_tx_cq_param(priv, params, &param->cqp);

}



static void mlx5e_ptp_build_params(struct mlx5e_port_ptp *c,

				   struct mlx5e_ptp_params *cparams,

				   struct mlx5e_params *orig)

{

	struct mlx5e_params *params = &cparams->params;



	params->tx_min_inline_mode = orig->tx_min_inline_mode;

	params->num_channels = orig->num_channels;

	params->hard_mtu = orig->hard_mtu;

	params->sw_mtu = orig->sw_mtu;

	params->num_tc = orig->num_tc;



	/* SQ */

	params->log_sq_size = orig->log_sq_size;



	mlx5e_ptp_build_sq_param(c->priv, params, &cparams->txq_sq_param);

}



static int mlx5e_ptp_open_queues(struct mlx5e_port_ptp *c,

				 struct mlx5e_ptp_params *cparams)

{

	int err;



	err = mlx5e_ptp_open_cqs(c, cparams);

	if (err)

		return err;



	napi_enable(&c->napi);



	err = mlx5e_ptp_open_txqsqs(c, cparams);

	if (err)

		goto disable_napi;



	return 0;



disable_napi:

	napi_disable(&c->napi);

	mlx5e_ptp_close_cqs(c);



	return err;

}



static void mlx5e_ptp_close_queues(struct mlx5e_port_ptp *c)

{

	mlx5e_ptp_close_txqsqs(c);

	napi_disable(&c->napi);

	mlx5e_ptp_close_cqs(c);

}



int mlx5e_port_ptp_open(struct mlx5e_priv *priv, struct mlx5e_params *params,

			u8 lag_port, struct mlx5e_port_ptp **cp)

{

	struct net_device *netdev = priv->netdev;

	struct mlx5_core_dev *mdev = priv->mdev;

	struct mlx5e_ptp_params *cparams;

	struct mlx5e_port_ptp *c;

	unsigned int irq;

	int err;

	int eqn;



	err = mlx5_vector2eqn(priv->mdev, 0, &eqn, &irq);

	if (err)

		return err;



	c = kvzalloc_node(sizeof(*c), GFP_KERNEL, dev_to_node(mlx5_core_dma_dev(mdev)));

	cparams = kvzalloc(sizeof(*cparams), GFP_KERNEL);

	if (!c || !cparams)

		return -ENOMEM;



	c->priv     = priv;

	c->mdev     = priv->mdev;

	c->tstamp   = &priv->tstamp;

	c->ix       = 0;

	c->pdev     = mlx5_core_dma_dev(priv->mdev);

	c->netdev   = priv->netdev;

	c->mkey_be  = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key);

	c->num_tc   = params->num_tc;

	c->stats    = &priv->port_ptp_stats.ch;

	c->irq_desc = irq_to_desc(irq);

	c->lag_port = lag_port;



	netif_napi_add(netdev, &c->napi, mlx5e_ptp_napi_poll, 64);



	mlx5e_ptp_build_params(c, cparams, params);



	err = mlx5e_ptp_open_queues(c, cparams);

	if (unlikely(err))

		goto err_napi_del;



	*cp = c;



	kvfree(cparams);



	return 0;



err_napi_del:

	netif_napi_del(&c->napi);



	kvfree(cparams);

	kvfree(c);

	return err;

}



void mlx5e_port_ptp_close(struct mlx5e_port_ptp *c)

{

	mlx5e_ptp_close_queues(c);

	netif_napi_del(&c->napi);



	kvfree(c);

}



void mlx5e_ptp_activate_channel(struct mlx5e_port_ptp *c)

{

	int tc;



	for (tc = 0; tc < c->num_tc; tc++)

		mlx5e_activate_txqsq(&c->ptpsq[tc].txqsq);

}



void mlx5e_ptp_deactivate_channel(struct mlx5e_port_ptp *c)

{

	int tc;



	for (tc = 0; tc < c->num_tc; tc++)

		mlx5e_deactivate_txqsq(&c->ptpsq[tc].txqsq);

}

drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h

0 → 100644
+48 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */

/* Copyright (c) 2020 Mellanox Technologies. */



#ifndef __MLX5_EN_PTP_H__

#define __MLX5_EN_PTP_H__



#include "en.h"

#include "en/params.h"

#include "en_stats.h"



struct mlx5e_ptpsq {

	struct mlx5e_txqsq       txqsq;

};



struct mlx5e_port_ptp {

	/* data path */

	struct mlx5e_ptpsq         ptpsq[MLX5E_MAX_NUM_TC];

	struct napi_struct         napi;

	struct device             *pdev;

	struct net_device         *netdev;

	__be32                     mkey_be;

	u8                         num_tc;

	u8                         lag_port;



	/* data path - accessed per napi poll */

	struct irq_desc *irq_desc;

	struct mlx5e_ch_stats     *stats;



	/* control */

	struct mlx5e_priv         *priv;

	struct mlx5_core_dev      *mdev;

	struct hwtstamp_config    *tstamp;

	DECLARE_BITMAP(state, MLX5E_CHANNEL_NUM_STATES);

	int                        ix;

};



struct mlx5e_ptp_params {

	struct mlx5e_params        params;

	struct mlx5e_sq_param      txq_sq_param;

};



int mlx5e_port_ptp_open(struct mlx5e_priv *priv, struct mlx5e_params *params,

			u8 lag_port, struct mlx5e_port_ptp **cp);

void mlx5e_port_ptp_close(struct mlx5e_port_ptp *c);

void mlx5e_ptp_activate_channel(struct mlx5e_port_ptp *c);

void mlx5e_ptp_deactivate_channel(struct mlx5e_port_ptp *c);



#endif /* __MLX5_EN_PTP_H__ */