Commit bc8d405b authored by Toke Høiland-Jørgensen's avatar Toke Høiland-Jørgensen Committed by Martin KaFai Lau
Browse files

net/mlx5e: Support RX XDP metadata 



Support RX hash and timestamp metadata kfuncs. We need to pass in the cqe
pointer to the mlx5e_skb_from* functions so it can be retrieved from the
XDP ctx to do this.

Cc: Tariq Toukan <tariqt@nvidia.com>
Cc: Saeed Mahameed <saeedm@nvidia.com>
Cc: John Fastabend <john.fastabend@gmail.com>
Cc: David Ahern <dsahern@gmail.com>
Cc: Martin KaFai Lau <martin.lau@linux.dev>
Cc: Jakub Kicinski <kuba@kernel.org>
Cc: Willem de Bruijn <willemb@google.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Anatoly Burakov <anatoly.burakov@intel.com>
Cc: Alexander Lobakin <alexandr.lobakin@intel.com>
Cc: Magnus Karlsson <magnus.karlsson@gmail.com>
Cc: Maryam Tahhan <mtahhan@redhat.com>
Cc: xdp-hints@xdp-project.net
Cc: netdev@vger.kernel.org
Signed-off-by: default avatarToke Høiland-Jørgensen <toke@redhat.com>
Signed-off-by: default avatarStanislav Fomichev <sdf@google.com>
Reviewed-by: default avatarTariq Toukan <tariqt@nvidia.com>
Link: https://lore.kernel.org/r/20230119221536.3349901-17-sdf@google.com


Signed-off-by: default avatarMartin KaFai Lau <martin.lau@kernel.org>
parent 384a13ca

drivers/net/ethernet/mellanox/mlx5/core/en.h

+3 −2
Original line number Diff line number Diff line
@@ -626,10 +626,11 @@ struct mlx5e_rq; 
typedef void (*mlx5e_fp_handle_rx_cqe)(struct mlx5e_rq*, struct mlx5_cqe64*);

typedef struct sk_buff *

(*mlx5e_fp_skb_from_cqe_mpwrq)(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,

			       u16 cqe_bcnt, u32 head_offset, u32 page_idx);

			       struct mlx5_cqe64 *cqe, u16 cqe_bcnt,

			       u32 head_offset, u32 page_idx);

typedef struct sk_buff *

(*mlx5e_fp_skb_from_cqe)(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi,

			 u32 cqe_bcnt);

			 struct mlx5_cqe64 *cqe, u32 cqe_bcnt);

typedef bool (*mlx5e_fp_post_rx_wqes)(struct mlx5e_rq *rq);

typedef void (*mlx5e_fp_dealloc_wqe)(struct mlx5e_rq*, u16);

typedef void (*mlx5e_fp_shampo_dealloc_hd)(struct mlx5e_rq*, u16, u16, bool);

drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h

+5 −0
Original line number Diff line number Diff line
@@ -73,6 +73,11 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget); 
void mlx5e_free_rx_descs(struct mlx5e_rq *rq);

void mlx5e_free_rx_in_progress_descs(struct mlx5e_rq *rq);



static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config)

{

	return config->rx_filter == HWTSTAMP_FILTER_ALL;

}



/* TX */

netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev);

bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget);

drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c

+28 −0
Original line number Diff line number Diff line
@@ -156,6 +156,34 @@ mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq, 
	return true;

}



static int mlx5e_xdp_rx_timestamp(const struct xdp_md *ctx, u64 *timestamp)

{

	const struct mlx5e_xdp_buff *_ctx = (void *)ctx;



	if (unlikely(!mlx5e_rx_hw_stamp(_ctx->rq->tstamp)))

		return -EOPNOTSUPP;



	*timestamp =  mlx5e_cqe_ts_to_ns(_ctx->rq->ptp_cyc2time,

					 _ctx->rq->clock, get_cqe_ts(_ctx->cqe));

	return 0;

}



static int mlx5e_xdp_rx_hash(const struct xdp_md *ctx, u32 *hash)

{

	const struct mlx5e_xdp_buff *_ctx = (void *)ctx;



	if (unlikely(!(_ctx->xdp.rxq->dev->features & NETIF_F_RXHASH)))

		return -EOPNOTSUPP;



	*hash = be32_to_cpu(_ctx->cqe->rss_hash_result);

	return 0;

}



const struct xdp_metadata_ops mlx5e_xdp_metadata_ops = {

	.xmo_rx_timestamp		= mlx5e_xdp_rx_timestamp,

	.xmo_rx_hash			= mlx5e_xdp_rx_hash,

};



/* returns true if packet was consumed by xdp */

bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct page *page,

		      struct bpf_prog *prog, struct mlx5e_xdp_buff *mxbuf)

drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h

+4 −0
Original line number Diff line number Diff line
@@ -46,6 +46,8 @@ 


struct mlx5e_xdp_buff {

	struct xdp_buff xdp;

	struct mlx5_cqe64 *cqe;

	struct mlx5e_rq *rq;

};



struct mlx5e_xsk_param;
@@ -60,6 +62,8 @@ void mlx5e_xdp_rx_poll_complete(struct mlx5e_rq *rq); 
int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,

		   u32 flags);



extern const struct xdp_metadata_ops mlx5e_xdp_metadata_ops;



INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq,

							  struct mlx5e_xmit_data *xdptxd,

							  struct skb_shared_info *sinfo,

drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c

+14 −0
Original line number Diff line number Diff line
@@ -52,25 +52,30 @@ int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) 


	if (likely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_ALIGNED)) {

		for (i = 0; i < batch; i++) {

			struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(wi->alloc_units[i].xsk);

			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk);



			umr_wqe->inline_mtts[i] = (struct mlx5_mtt) {

				.ptag = cpu_to_be64(addr | MLX5_EN_WR),

			};

			mxbuf->rq = rq;

		}

	} else if (unlikely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_UNALIGNED)) {

		for (i = 0; i < batch; i++) {

			struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(wi->alloc_units[i].xsk);

			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk);



			umr_wqe->inline_ksms[i] = (struct mlx5_ksm) {

				.key = rq->mkey_be,

				.va = cpu_to_be64(addr),

			};

			mxbuf->rq = rq;

		}

	} else if (likely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE)) {

		u32 mapping_size = 1 << (rq->mpwqe.page_shift - 2);



		for (i = 0; i < batch; i++) {

			struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(wi->alloc_units[i].xsk);

			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk);



			umr_wqe->inline_ksms[i << 2] = (struct mlx5_ksm) {
@@ -89,6 +94,7 @@ int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) 
				.key = rq->mkey_be,

				.va = cpu_to_be64(rq->wqe_overflow.addr),

			};

			mxbuf->rq = rq;

		}

	} else {

		__be32 pad_size = cpu_to_be32((1 << rq->mpwqe.page_shift) -
@@ -96,6 +102,7 @@ int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) 
		__be32 frame_size = cpu_to_be32(rq->xsk_pool->chunk_size);



		for (i = 0; i < batch; i++) {

			struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(wi->alloc_units[i].xsk);

			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk);



			umr_wqe->inline_klms[i << 1] = (struct mlx5_klm) {
@@ -108,6 +115,7 @@ int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) 
				.va = cpu_to_be64(rq->wqe_overflow.addr),

				.bcount = pad_size,

			};

			mxbuf->rq = rq;

		}

	}
@@ -238,6 +246,7 @@ static struct sk_buff *mlx5e_xsk_construct_skb(struct mlx5e_rq *rq, struct xdp_b 


struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,

						    struct mlx5e_mpw_info *wi,

						    struct mlx5_cqe64 *cqe,

						    u16 cqe_bcnt,

						    u32 head_offset,

						    u32 page_idx)
@@ -258,6 +267,8 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, 
	 */

	WARN_ON_ONCE(head_offset);



	/* mxbuf->rq is set on allocation, but cqe is per-packet so set it here */

	mxbuf->cqe = cqe;

	xsk_buff_set_size(&mxbuf->xdp, cqe_bcnt);

	xsk_buff_dma_sync_for_cpu(&mxbuf->xdp, rq->xsk_pool);

	net_prefetch(mxbuf->xdp.data);
@@ -292,6 +303,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, 


struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,

					      struct mlx5e_wqe_frag_info *wi,

					      struct mlx5_cqe64 *cqe,

					      u32 cqe_bcnt)

{

	struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(wi->au->xsk);
@@ -304,6 +316,8 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq, 
	 */

	WARN_ON_ONCE(wi->offset);



	/* mxbuf->rq is set on allocation, but cqe is per-packet so set it here */

	mxbuf->cqe = cqe;

	xsk_buff_set_size(&mxbuf->xdp, cqe_bcnt);

	xsk_buff_dma_sync_for_cpu(&mxbuf->xdp, rq->xsk_pool);

	net_prefetch(mxbuf->xdp.data);