Commit b89eced8 authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'mlx5-xsk-updates-part4-and-more' 

Saeed Mahameed says:

====================
mlx5 xsk updates part4 and more

1) Final part of xsk improvements,
in this series Maxim continues to improve xsk implementation
 a) XSK Busy polling support
 b) Use KLM to avoid Frame overrun in unaligned mode
 c) Optimize unaligned more for certain frame sizes
 d) Other straight forward minor optimizations.

part 1: https://lore.kernel.org/netdev/20220927203611.244301-1-saeed@kernel.org/
part 2: https://lore.kernel.org/netdev/20220929072156.93299-1-saeed@kernel.org/
part 3: https://lore.kernel.org/netdev/20220930162903.62262-1-saeed@kernel.org/

2) Oversize packets firmware counter, from Gal.

3) Set default grace period for health reporters based on function type

4) Some minor E-Switch improvements
====================

Link: https://lore.kernel.org/r/20221002045632.291612-1-saeed@kernel.org


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents a4388da5 794131c4

drivers/net/ethernet/mellanox/mlx5/core/en.h

+11 −7
Original line number Diff line number Diff line
@@ -109,12 +109,8 @@ struct page_pool; 
#define MLX5_MPWRQ_MAX_PAGES_PER_WQE \

	rounddown_pow_of_two(MLX5_UMR_MAX_MTT_SPACE / sizeof(struct mlx5_mtt))



#define MLX5_ALIGN_MTTS(mtts)		(ALIGN(mtts, 8))

#define MLX5_ALIGNED_MTTS_OCTW(mtts)	((mtts) / 2)

#define MLX5_MTT_OCTW(mtts)		(MLX5_ALIGNED_MTTS_OCTW(MLX5_ALIGN_MTTS(mtts)))

#define MLX5_KSM_OCTW(ksms)             (ksms)

#define MLX5E_MAX_RQ_NUM_MTTS	\

	(ALIGN_DOWN(U16_MAX, 4) * 2) /* So that MLX5_MTT_OCTW(num_mtts) fits into u16 */

	(ALIGN_DOWN(U16_MAX, 4) * 2) /* Fits into u16 and aligned by WQEBB. */

#define MLX5E_MAX_RQ_NUM_KSMS (U16_MAX - 1) /* So that num_ksms fits into u16. */

#define MLX5E_ORDER2_MAX_PACKET_MTU (order_base_2(10 * 1024))
@@ -681,6 +677,13 @@ struct mlx5e_hw_gro_data { 
	int second_ip_id;

};



enum mlx5e_mpwrq_umr_mode {

	MLX5E_MPWRQ_UMR_MODE_ALIGNED,

	MLX5E_MPWRQ_UMR_MODE_UNALIGNED,

	MLX5E_MPWRQ_UMR_MODE_OVERSIZED,

	MLX5E_MPWRQ_UMR_MODE_TRIPLE,

};



struct mlx5e_rq {

	/* data path */

	union {
@@ -708,7 +711,7 @@ struct mlx5e_rq { 
			u8                     pages_per_wqe;

			u8                     umr_wqebbs;

			u8                     mtts_per_wqe;

			u8                     unaligned;

			u8                     umr_mode;

			struct mlx5e_shampo_hd *shampo;

		} mpwqe;

	};
@@ -1008,7 +1011,7 @@ struct mlx5e_profile { 
void mlx5e_build_ptys2ethtool_map(void);



bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift,

					    bool unaligned);

					    enum mlx5e_mpwrq_umr_mode umr_mode);



void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq, u16 len, u16 start, bool close);

void mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats);
@@ -1036,6 +1039,7 @@ struct mlx5e_rq_param; 
int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param,

		  struct mlx5e_xsk_param *xsk, int node,

		  struct mlx5e_rq *rq);

#define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */

int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time);

void mlx5e_close_rq(struct mlx5e_rq *rq);

int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param);

drivers/net/ethernet/mellanox/mlx5/core/en/params.c

+158 −45
Original line number Diff line number Diff line
@@ -27,9 +27,80 @@ u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xs 
	return max(req_page_shift, min_page_shift);

}



u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned)

enum mlx5e_mpwrq_umr_mode

mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk)

{

	/* Different memory management schemes use different mechanisms to map

	 * user-mode memory. The stricter guarantees we have, the faster

	 * mechanisms we use:

	 * 1. MTT - direct mapping in page granularity.

	 * 2. KSM - indirect mapping to another MKey to arbitrary addresses, but

	 *    all mappings have the same size.

	 * 3. KLM - indirect mapping to another MKey to arbitrary addresses, and

	 *    mappings can have different sizes.

	 */

	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);

	bool unaligned = xsk ? xsk->unaligned : false;

	bool oversized = false;



	if (xsk) {

		oversized = xsk->chunk_size < (1 << page_shift);

		WARN_ON_ONCE(xsk->chunk_size > (1 << page_shift));

	}



	/* XSK frame size doesn't match the UMR page size, either because the

	 * frame size is not a power of two, or it's smaller than the minimal

	 * page size supported by the firmware.

	 * It's possible to receive packets bigger than MTU in certain setups.

	 * To avoid writing over the XSK frame boundary, the top region of each

	 * stride is mapped to a garbage page, resulting in two mappings of

	 * different sizes per frame.

	 */

	if (oversized) {

		/* An optimization for frame sizes equal to 3 * power_of_two.

		 * 3 KSMs point to the frame, and one KSM points to the garbage

		 * page, which works faster than KLM.

		 */

		if (xsk->chunk_size % 3 == 0 && is_power_of_2(xsk->chunk_size / 3))

			return MLX5E_MPWRQ_UMR_MODE_TRIPLE;



		return MLX5E_MPWRQ_UMR_MODE_OVERSIZED;

	}



	/* XSK frames can start at arbitrary unaligned locations, but they all

	 * have the same size which is a power of two. It allows to optimize to

	 * one KSM per frame.

	 */

	if (unaligned)

		return MLX5E_MPWRQ_UMR_MODE_UNALIGNED;



	/* XSK: frames are naturally aligned, MTT can be used.

	 * Non-XSK: Allocations happen in units of CPU pages, therefore, the

	 * mappings are naturally aligned.

	 */

	return MLX5E_MPWRQ_UMR_MODE_ALIGNED;

}



u8 mlx5e_mpwrq_umr_entry_size(enum mlx5e_mpwrq_umr_mode mode)

{

	u8 umr_entry_size = unaligned ? sizeof(struct mlx5_ksm) : sizeof(struct mlx5_mtt);

	switch (mode) {

	case MLX5E_MPWRQ_UMR_MODE_ALIGNED:

		return sizeof(struct mlx5_mtt);

	case MLX5E_MPWRQ_UMR_MODE_UNALIGNED:

		return sizeof(struct mlx5_ksm);

	case MLX5E_MPWRQ_UMR_MODE_OVERSIZED:

		return sizeof(struct mlx5_klm) * 2;

	case MLX5E_MPWRQ_UMR_MODE_TRIPLE:

		return sizeof(struct mlx5_ksm) * 4;

	}

	WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", mode);

	return 0;

}



u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift,

			  enum mlx5e_mpwrq_umr_mode umr_mode)

{

	u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(umr_mode);

	u8 max_pages_per_wqe, max_log_mpwqe_size;

	u16 max_wqe_size;
@@ -44,13 +115,21 @@ u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, bool unalig 
	return min_t(u8, max_log_mpwqe_size, MLX5_MPWRQ_MAX_LOG_WQE_SZ);

}



u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned)

u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift,

			     enum mlx5e_mpwrq_umr_mode umr_mode)

{

	u8 log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, unaligned);

	u8 log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode);

	u8 pages_per_wqe;



	pages_per_wqe = log_wqe_sz > page_shift ? (1 << (log_wqe_sz - page_shift)) : 1;



	/* Two MTTs are needed to form an octword. The number of MTTs is encoded

	 * in octwords in a UMR WQE, so we need at least two to avoid mapping

	 * garbage addresses.

	 */

	if (WARN_ON_ONCE(pages_per_wqe < 2 && umr_mode == MLX5E_MPWRQ_UMR_MODE_ALIGNED))

		pages_per_wqe = 2;



	/* Sanity check for further calculations to succeed. */

	BUILD_BUG_ON(MLX5_MPWRQ_MAX_PAGES_PER_WQE > 64);

	if (WARN_ON_ONCE(pages_per_wqe > MLX5_MPWRQ_MAX_PAGES_PER_WQE))
@@ -59,10 +138,11 @@ u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift, bool una 
	return pages_per_wqe;

}



u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned)

u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift,

			   enum mlx5e_mpwrq_umr_mode umr_mode)

{

	u8 umr_entry_size = unaligned ? sizeof(struct mlx5_ksm) : sizeof(struct mlx5_mtt);

	u8 pages_per_wqe = mlx5e_mpwrq_pages_per_wqe(mdev, page_shift, unaligned);

	u8 pages_per_wqe = mlx5e_mpwrq_pages_per_wqe(mdev, page_shift, umr_mode);

	u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(umr_mode);

	u16 umr_wqe_sz;



	umr_wqe_sz = sizeof(struct mlx5e_umr_wqe) +
@@ -73,44 +153,64 @@ u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, bool unali 
	return umr_wqe_sz;

}



u8 mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned)

u8 mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev *mdev, u8 page_shift,

			  enum mlx5e_mpwrq_umr_mode umr_mode)

{

	return DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(mdev, page_shift, unaligned),

	return DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(mdev, page_shift, umr_mode),

			    MLX5_SEND_WQE_BB);

}



u8 mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned)

u8 mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift,

			    enum mlx5e_mpwrq_umr_mode umr_mode)

{

	u8 pages_per_wqe = mlx5e_mpwrq_pages_per_wqe(mdev, page_shift, umr_mode);



	/* Add another page as a buffer between WQEs. This page will absorb

	 * write overflow by the hardware, when receiving packets larger than

	 * MTU. These oversize packets are dropped by the driver at a later

	 * stage.

	 */

	return MLX5_ALIGN_MTTS(mlx5e_mpwrq_pages_per_wqe(mdev, page_shift, unaligned) + 1);

	return ALIGN(pages_per_wqe + 1,

		     MLX5_SEND_WQE_BB / mlx5e_mpwrq_umr_entry_size(umr_mode));

}



u32 mlx5e_mpwrq_max_num_entries(struct mlx5_core_dev *mdev, bool unaligned)

u32 mlx5e_mpwrq_max_num_entries(struct mlx5_core_dev *mdev,

				enum mlx5e_mpwrq_umr_mode umr_mode)

{

	if (unaligned)

		return min(MLX5E_MAX_RQ_NUM_KSMS,

	/* Same limits apply to KSMs and KLMs. */

	u32 klm_limit = min(MLX5E_MAX_RQ_NUM_KSMS,

			    1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size));



	switch (umr_mode) {

	case MLX5E_MPWRQ_UMR_MODE_ALIGNED:

		return MLX5E_MAX_RQ_NUM_MTTS;

	case MLX5E_MPWRQ_UMR_MODE_UNALIGNED:

		return klm_limit;

	case MLX5E_MPWRQ_UMR_MODE_OVERSIZED:

		/* Each entry is two KLMs. */

		return klm_limit / 2;

	case MLX5E_MPWRQ_UMR_MODE_TRIPLE:

		/* Each entry is four KSMs. */

		return klm_limit / 4;

	}

	WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", umr_mode);

	return 0;

}



static u8 mlx5e_mpwrq_max_log_rq_size(struct mlx5_core_dev *mdev, u8 page_shift,

				      bool unaligned)

				      enum mlx5e_mpwrq_umr_mode umr_mode)

{

	u8 mtts_per_wqe = mlx5e_mpwrq_mtts_per_wqe(mdev, page_shift, unaligned);

	u32 max_entries = mlx5e_mpwrq_max_num_entries(mdev, unaligned);

	u8 mtts_per_wqe = mlx5e_mpwrq_mtts_per_wqe(mdev, page_shift, umr_mode);

	u32 max_entries = mlx5e_mpwrq_max_num_entries(mdev, umr_mode);



	return ilog2(max_entries / mtts_per_wqe);

}



u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned)

u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift,

			       enum mlx5e_mpwrq_umr_mode umr_mode)

{

	return mlx5e_mpwrq_max_log_rq_size(mdev, page_shift, unaligned) +

		mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, unaligned) -

	return mlx5e_mpwrq_max_log_rq_size(mdev, page_shift, umr_mode) +

		mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode) -

		MLX5E_ORDER2_MAX_PACKET_MTU;

}
@@ -171,10 +271,10 @@ static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5_core_dev *mdev, 
				       struct mlx5e_xsk_param *xsk)

{

	u32 linear_stride_sz = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true);

	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);

	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);

	bool unaligned = xsk ? xsk->unaligned : false;



	return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, unaligned) -

	return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode) -

		order_base_2(linear_stride_sz);

}
@@ -200,10 +300,11 @@ bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev, 


static bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev,

					  u8 log_stride_sz, u8 log_num_strides,

					  u8 page_shift, bool unaligned)

					  u8 page_shift,

					  enum mlx5e_mpwrq_umr_mode umr_mode)

{

	if (log_stride_sz + log_num_strides !=

	    mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, unaligned))

	    mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode))

		return false;



	if (log_stride_sz < MLX5_MPWQE_LOG_STRIDE_SZ_BASE ||
@@ -223,8 +324,8 @@ bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, 
				  struct mlx5e_params *params,

				  struct mlx5e_xsk_param *xsk)

{

	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);

	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);

	bool unaligned = xsk ? xsk->unaligned : false;

	u8 log_num_strides;

	u8 log_stride_sz;

	u8 log_wqe_sz;
@@ -233,7 +334,7 @@ bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, 
		return false;



	log_stride_sz = order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true));

	log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, unaligned);

	log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode);



	if (log_wqe_sz < log_stride_sz)

		return false;
@@ -242,19 +343,19 @@ bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, 


	return mlx5e_verify_rx_mpwqe_strides(mdev, log_stride_sz,

					     log_num_strides, page_shift,

					     unaligned);

					     umr_mode);

}



u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev,

			       struct mlx5e_params *params,

			       struct mlx5e_xsk_param *xsk)

{

	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);

	u8 log_pkts_per_wqe, page_shift, max_log_rq_size;

	bool unaligned = xsk ? xsk->unaligned : false;



	log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(mdev, params, xsk);

	page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);

	max_log_rq_size = mlx5e_mpwrq_max_log_rq_size(mdev, page_shift, unaligned);

	max_log_rq_size = mlx5e_mpwrq_max_log_rq_size(mdev, page_shift, umr_mode);



	/* Numbers are unsigned, don't subtract to avoid underflow. */

	if (params->log_rq_mtu_frames <
@@ -308,10 +409,10 @@ u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev, 
				   struct mlx5e_params *params,

				   struct mlx5e_xsk_param *xsk)

{

	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);

	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);

	bool unaligned = xsk ? xsk->unaligned : false;



	return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, unaligned) -

	return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode) -

		mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);

}
@@ -460,9 +561,10 @@ bool slow_pci_heuristic(struct mlx5_core_dev *mdev) 


int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params *params)

{

	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, NULL);

	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, NULL);



	if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, false))

	if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, umr_mode))

		return -EOPNOTSUPP;



	if (params->xdp_prog && !mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL))
@@ -474,11 +576,12 @@ int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params 
int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *params,

			     struct mlx5e_xsk_param *xsk)

{

	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);

	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);

	bool unaligned = xsk ? xsk->unaligned : false;

	u16 max_mtu_pkts;



	if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, xsk->unaligned))

	if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, umr_mode))

		return -EOPNOTSUPP;



	if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk))
@@ -781,16 +884,16 @@ int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, 
	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: {

		u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);

		u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);

		enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);

		u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);

		bool unaligned = xsk ? xsk->unaligned : false;



		if (!mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size,

						   log_wqe_num_of_strides,

						   page_shift, unaligned)) {

						   page_shift, umr_mode)) {

			mlx5_core_err(mdev,

				      "Bad RX MPWQE params: log_stride_size %u, log_num_strides %u, unaligned %d\n",

				      "Bad RX MPWQE params: log_stride_size %u, log_num_strides %u, umr_mode %d\n",

				      log_wqe_stride_size, log_wqe_num_of_strides,

				      unaligned);

				      umr_mode);

			return -EINVAL;

		}
@@ -974,11 +1077,11 @@ static u32 mlx5e_mpwrq_total_umr_wqebbs(struct mlx5_core_dev *mdev, 
					struct mlx5e_params *params,

					struct mlx5e_xsk_param *xsk)

{

	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);

	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);

	bool unaligned = xsk ? xsk->unaligned : false;

	u8 umr_wqebbs;



	umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, unaligned);

	umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, umr_mode);



	return umr_wqebbs * (1 << mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk));

}
@@ -1026,6 +1129,16 @@ static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev, 
			xsk.unaligned = true;

			max_xsk_wqebbs = max(max_xsk_wqebbs,

				mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));



			/* XSK unaligned mode, frame size is not equal to stride size. */

			xsk.chunk_size -= 1;

			max_xsk_wqebbs = max(max_xsk_wqebbs,

				mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));



			/* XSK unaligned mode, frame size is a triple power of two. */

			xsk.chunk_size = (1 << frame_shift) / 4 * 3;

			max_xsk_wqebbs = max(max_xsk_wqebbs,

				mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));

		}



		wqebbs += max_xsk_wqebbs;

drivers/net/ethernet/mellanox/mlx5/core/en/params.h

+17 −7
Original line number Diff line number Diff line
@@ -56,13 +56,23 @@ struct mlx5e_create_sq_param { 
/* Striding RQ dynamic parameters */



u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk);

u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned);

u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned);

u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned);

u8 mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned);

u8 mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned);

u32 mlx5e_mpwrq_max_num_entries(struct mlx5_core_dev *mdev, bool unaligned);

u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift, bool unaligned);

enum mlx5e_mpwrq_umr_mode

mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk);

u8 mlx5e_mpwrq_umr_entry_size(enum mlx5e_mpwrq_umr_mode mode);

u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift,

			  enum mlx5e_mpwrq_umr_mode umr_mode);

u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift,

			     enum mlx5e_mpwrq_umr_mode umr_mode);

u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift,

			   enum mlx5e_mpwrq_umr_mode umr_mode);

u8 mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev *mdev, u8 page_shift,

			  enum mlx5e_mpwrq_umr_mode umr_mode);

u8 mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift,

			    enum mlx5e_mpwrq_umr_mode umr_mode);

u32 mlx5e_mpwrq_max_num_entries(struct mlx5_core_dev *mdev,

				enum mlx5e_mpwrq_umr_mode umr_mode);

u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift,

			       enum mlx5e_mpwrq_umr_mode umr_mode);



/* Parameter calculations */

drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c

+18 −0
Original line number Diff line number Diff line
@@ -99,6 +99,15 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv, 


	mlx5e_build_xsk_param(pool, &xsk);



	if (priv->channels.params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&

	    mlx5e_mpwrq_umr_mode(priv->mdev, &xsk) == MLX5E_MPWRQ_UMR_MODE_OVERSIZED) {

		const char *recommendation = is_power_of_2(xsk.chunk_size) ?

			"Upgrade firmware" : "Disable striding RQ";



		mlx5_core_warn(priv->mdev, "Expected slowdown with XSK frame size %u. %s for better performance.\n",

			       xsk.chunk_size, recommendation);

	}



	if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {

		/* XSK objects will be created on open. */

		goto validate_closed;
@@ -126,6 +135,9 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv, 


	mlx5e_rx_res_xsk_update(priv->rx_res, &priv->channels, ix, true);



	mlx5e_deactivate_rq(&c->rq);

	mlx5e_flush_rq(&c->rq, MLX5_RQC_STATE_RDY);



	return 0;



err_remove_pool:
@@ -165,7 +177,13 @@ static int mlx5e_xsk_disable_locked(struct mlx5e_priv *priv, u16 ix) 
		goto remove_pool;



	c = priv->channels.c[ix];



	mlx5e_activate_rq(&c->rq);

	mlx5e_trigger_napi_icosq(c);

	mlx5e_wait_for_min_rx_wqes(&c->rq, MLX5E_RQ_WQES_TIMEOUT);



	mlx5e_rx_res_xsk_update(priv->rx_res, &priv->channels, ix, false);



	mlx5e_deactivate_xsk(c);

	mlx5e_close_xsk(c);

drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c

+52 −5
Original line number Diff line number Diff line
@@ -41,7 +41,15 @@ int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) 
	umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi);

	memcpy(umr_wqe, &rq->mpwqe.umr_wqe, sizeof(struct mlx5e_umr_wqe));



	if (unlikely(rq->mpwqe.unaligned)) {

	if (likely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_ALIGNED)) {

		for (i = 0; i < batch; i++) {

			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk);



			umr_wqe->inline_mtts[i] = (struct mlx5_mtt) {

				.ptag = cpu_to_be64(addr | MLX5_EN_WR),

			};

		}

	} else if (unlikely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_UNALIGNED)) {

		for (i = 0; i < batch; i++) {

			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk);
@@ -50,12 +58,46 @@ int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) 
				.va = cpu_to_be64(addr),

			};

		}

	} else if (likely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE)) {

		u32 mapping_size = 1 << (rq->mpwqe.page_shift - 2);



		for (i = 0; i < batch; i++) {

			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk);



			umr_wqe->inline_ksms[i << 2] = (struct mlx5_ksm) {

				.key = rq->mkey_be,

				.va = cpu_to_be64(addr),

			};

			umr_wqe->inline_ksms[(i << 2) + 1] = (struct mlx5_ksm) {

				.key = rq->mkey_be,

				.va = cpu_to_be64(addr + mapping_size),

			};

			umr_wqe->inline_ksms[(i << 2) + 2] = (struct mlx5_ksm) {

				.key = rq->mkey_be,

				.va = cpu_to_be64(addr + mapping_size * 2),

			};

			umr_wqe->inline_ksms[(i << 2) + 3] = (struct mlx5_ksm) {

				.key = rq->mkey_be,

				.va = cpu_to_be64(rq->wqe_overflow.addr),

			};

		}

	} else {

		__be32 pad_size = cpu_to_be32((1 << rq->mpwqe.page_shift) -

					      rq->xsk_pool->chunk_size);

		__be32 frame_size = cpu_to_be32(rq->xsk_pool->chunk_size);



		for (i = 0; i < batch; i++) {

			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk);



			umr_wqe->inline_mtts[i] = (struct mlx5_mtt) {

				.ptag = cpu_to_be64(addr | MLX5_EN_WR),

			umr_wqe->inline_klms[i << 1] = (struct mlx5_klm) {

				.key = rq->mkey_be,

				.va = cpu_to_be64(addr),

				.bcount = frame_size,

			};

			umr_wqe->inline_klms[(i << 1) + 1] = (struct mlx5_klm) {

				.key = rq->mkey_be,

				.va = cpu_to_be64(rq->wqe_overflow.addr),

				.bcount = pad_size,

			};

		}

	}
@@ -66,9 +108,14 @@ int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) 
	umr_wqe->ctrl.opmod_idx_opcode =

		cpu_to_be32((icosq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) | MLX5_OPCODE_UMR);



	/* Optimized for speed: keep in sync with mlx5e_mpwrq_umr_entry_size. */

	offset = ix * rq->mpwqe.mtts_per_wqe;

	if (likely(!rq->mpwqe.unaligned))

		offset = MLX5_ALIGNED_MTTS_OCTW(offset);

	if (likely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_ALIGNED))

		offset = offset * sizeof(struct mlx5_mtt) / MLX5_OCTWORD;

	else if (unlikely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_OVERSIZED))

		offset = offset * sizeof(struct mlx5_klm) * 2 / MLX5_OCTWORD;

	else if (unlikely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE))

		offset = offset * sizeof(struct mlx5_ksm) * 4 / MLX5_OCTWORD;

	umr_wqe->uctrl.xlt_offset = cpu_to_be16(offset);



	icosq->db.wqe_info[pi] = (struct mlx5e_icosq_wqe_info) {