Commit 62d03330 authored by Jakub Kicinski's avatar Jakub Kicinski Committed by David S. Miller
Browse files

nfp: move the fast path code to separate files 



In preparation for support for a new datapath format move all
ring and fast path logic into separate files. It is basically
a verbatim move with some wrapping functions, no new structures
and functions added.

The current data path is called NFD3 from the initial version
of the driver ABI it used. The non-fast path, but ring related
functions are moved to nfp_net_dp.c file.

Changes to Jakub's work:
* Rebase on xsk related code.
* Split the patch, move the callback changes to next commit.

Signed-off-by: default avatarJakub Kicinski <jakub.kicinski@netronome.com>
Signed-off-by: default avatarFei Qin <fei.qin@corigine.com>
Signed-off-by: default avatarSimon Horman <simon.horman@corigine.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent fc9769f6

drivers/net/ethernet/netronome/nfp/Makefile

+4 −0
Original line number Diff line number Diff line
@@ -20,12 +20,16 @@ nfp-objs := \ 
	    ccm_mbox.o \

	    devlink_param.o \

	    nfp_asm.o \

	    nfd3/dp.o \

	    nfd3/rings.o \

	    nfd3/xsk.o \

	    nfp_app.o \

	    nfp_app_nic.o \

	    nfp_devlink.o \

	    nfp_hwmon.o \

	    nfp_main.o \

	    nfp_net_common.o \

	    nfp_net_dp.o \

	    nfp_net_ctrl.o \

	    nfp_net_debugdump.o \

	    nfp_net_ethtool.o \

drivers/net/ethernet/netronome/nfp/nfd3/dp.c

0 → 100644
+1368 −0

File added.

Preview size limit exceeded, changes collapsed.

drivers/net/ethernet/netronome/nfp/nfd3/nfd3.h

0 → 100644
+126 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */

/* Copyright (C) 2015-2019 Netronome Systems, Inc. */



#ifndef _NFP_DP_NFD3_H_

#define _NFP_DP_NFD3_H_



struct sk_buff;

struct net_device;



/* TX descriptor format */



#define NFD3_DESC_TX_EOP		BIT(7)

#define NFD3_DESC_TX_OFFSET_MASK	GENMASK(6, 0)

#define NFD3_DESC_TX_MSS_MASK		GENMASK(13, 0)



/* Flags in the host TX descriptor */

#define NFD3_DESC_TX_CSUM		BIT(7)

#define NFD3_DESC_TX_IP4_CSUM		BIT(6)

#define NFD3_DESC_TX_TCP_CSUM		BIT(5)

#define NFD3_DESC_TX_UDP_CSUM		BIT(4)

#define NFD3_DESC_TX_VLAN		BIT(3)

#define NFD3_DESC_TX_LSO		BIT(2)

#define NFD3_DESC_TX_ENCAP		BIT(1)

#define NFD3_DESC_TX_O_IP4_CSUM	BIT(0)



struct nfp_nfd3_tx_desc {

	union {

		struct {

			u8 dma_addr_hi; /* High bits of host buf address */

			__le16 dma_len;	/* Length to DMA for this desc */

			u8 offset_eop;	/* Offset in buf where pkt starts +

					 * highest bit is eop flag.

					 */

			__le32 dma_addr_lo; /* Low 32bit of host buf addr */



			__le16 mss;	/* MSS to be used for LSO */

			u8 lso_hdrlen;	/* LSO, TCP payload offset */

			u8 flags;	/* TX Flags, see @NFD3_DESC_TX_* */

			union {

				struct {

					u8 l3_offset; /* L3 header offset */

					u8 l4_offset; /* L4 header offset */

				};

				__le16 vlan; /* VLAN tag to add if indicated */

			};

			__le16 data_len; /* Length of frame + meta data */

		} __packed;

		__le32 vals[4];

		__le64 vals8[2];

	};

};



/**

 * struct nfp_nfd3_tx_buf - software TX buffer descriptor

 * @skb:	normal ring, sk_buff associated with this buffer

 * @frag:	XDP ring, page frag associated with this buffer

 * @xdp:	XSK buffer pool handle (for AF_XDP)

 * @dma_addr:	DMA mapping address of the buffer

 * @fidx:	Fragment index (-1 for the head and [0..nr_frags-1] for frags)

 * @pkt_cnt:	Number of packets to be produced out of the skb associated

 *		with this buffer (valid only on the head's buffer).

 *		Will be 1 for all non-TSO packets.

 * @is_xsk_tx:	Flag if buffer is a RX buffer after a XDP_TX action and not a

 *		buffer from the TX queue (for AF_XDP).

 * @real_len:	Number of bytes which to be produced out of the skb (valid only

 *		on the head's buffer). Equal to skb->len for non-TSO packets.

 */

struct nfp_nfd3_tx_buf {

	union {

		struct sk_buff *skb;

		void *frag;

		struct xdp_buff *xdp;

	};

	dma_addr_t dma_addr;

	union {

		struct {

			short int fidx;

			u16 pkt_cnt;

		};

		struct {

			bool is_xsk_tx;

		};

	};

	u32 real_len;

};



void

nfp_nfd3_rx_csum(const struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec,

		 const struct nfp_net_rx_desc *rxd,

		 const struct nfp_meta_parsed *meta, struct sk_buff *skb);

bool

nfp_nfd3_parse_meta(struct net_device *netdev, struct nfp_meta_parsed *meta,

		    void *data, void *pkt, unsigned int pkt_len, int meta_len);

void nfp_nfd3_tx_complete(struct nfp_net_tx_ring *tx_ring, int budget);

int nfp_nfd3_poll(struct napi_struct *napi, int budget);

void nfp_nfd3_ctrl_poll(struct tasklet_struct *t);

void nfp_nfd3_rx_ring_fill_freelist(struct nfp_net_dp *dp,

				    struct nfp_net_rx_ring *rx_ring);

void nfp_nfd3_xsk_tx_free(struct nfp_nfd3_tx_buf *txbuf);

int nfp_nfd3_xsk_poll(struct napi_struct *napi, int budget);



void

nfp_nfd3_tx_ring_reset(struct nfp_net_dp *dp, struct nfp_net_tx_ring *tx_ring);

void

nfp_nfd3_rx_ring_fill_freelist(struct nfp_net_dp *dp,

			       struct nfp_net_rx_ring *rx_ring);

int

nfp_nfd3_tx_ring_alloc(struct nfp_net_dp *dp, struct nfp_net_tx_ring *tx_ring);

void

nfp_nfd3_tx_ring_free(struct nfp_net_tx_ring *tx_ring);

int

nfp_nfd3_tx_ring_bufs_alloc(struct nfp_net_dp *dp,

			    struct nfp_net_tx_ring *tx_ring);

void

nfp_nfd3_tx_ring_bufs_free(struct nfp_net_dp *dp,

			   struct nfp_net_tx_ring *tx_ring);

void

nfp_nfd3_print_tx_descs(struct seq_file *file,

			struct nfp_net_r_vector *r_vec,

			struct nfp_net_tx_ring *tx_ring,

			u32 d_rd_p, u32 d_wr_p);

netdev_tx_t nfp_nfd3_tx(struct sk_buff *skb, struct net_device *netdev);

bool nfp_nfd3_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb);

bool __nfp_nfd3_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb);



#endif

drivers/net/ethernet/netronome/nfp/nfd3/rings.c

0 → 100644
+243 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)

/* Copyright (C) 2015-2019 Netronome Systems, Inc. */



#include <linux/seq_file.h>



#include "../nfp_net.h"

#include "../nfp_net_dp.h"

#include "../nfp_net_xsk.h"

#include "nfd3.h"



static void nfp_nfd3_xsk_tx_bufs_free(struct nfp_net_tx_ring *tx_ring)

{

	struct nfp_nfd3_tx_buf *txbuf;

	unsigned int idx;



	while (tx_ring->rd_p != tx_ring->wr_p) {

		idx = D_IDX(tx_ring, tx_ring->rd_p);

		txbuf = &tx_ring->txbufs[idx];



		txbuf->real_len = 0;



		tx_ring->qcp_rd_p++;

		tx_ring->rd_p++;



		if (tx_ring->r_vec->xsk_pool) {

			if (txbuf->is_xsk_tx)

				nfp_nfd3_xsk_tx_free(txbuf);



			xsk_tx_completed(tx_ring->r_vec->xsk_pool, 1);

		}

	}

}



/**

 * nfp_nfd3_tx_ring_reset() - Free any untransmitted buffers and reset pointers

 * @dp:		NFP Net data path struct

 * @tx_ring:	TX ring structure

 *

 * Assumes that the device is stopped, must be idempotent.

 */

void

nfp_nfd3_tx_ring_reset(struct nfp_net_dp *dp, struct nfp_net_tx_ring *tx_ring)

{

	struct netdev_queue *nd_q;

	const skb_frag_t *frag;



	while (!tx_ring->is_xdp && tx_ring->rd_p != tx_ring->wr_p) {

		struct nfp_nfd3_tx_buf *tx_buf;

		struct sk_buff *skb;

		int idx, nr_frags;



		idx = D_IDX(tx_ring, tx_ring->rd_p);

		tx_buf = &tx_ring->txbufs[idx];



		skb = tx_ring->txbufs[idx].skb;

		nr_frags = skb_shinfo(skb)->nr_frags;



		if (tx_buf->fidx == -1) {

			/* unmap head */

			dma_unmap_single(dp->dev, tx_buf->dma_addr,

					 skb_headlen(skb), DMA_TO_DEVICE);

		} else {

			/* unmap fragment */

			frag = &skb_shinfo(skb)->frags[tx_buf->fidx];

			dma_unmap_page(dp->dev, tx_buf->dma_addr,

				       skb_frag_size(frag), DMA_TO_DEVICE);

		}



		/* check for last gather fragment */

		if (tx_buf->fidx == nr_frags - 1)

			dev_kfree_skb_any(skb);



		tx_buf->dma_addr = 0;

		tx_buf->skb = NULL;

		tx_buf->fidx = -2;



		tx_ring->qcp_rd_p++;

		tx_ring->rd_p++;

	}



	if (tx_ring->is_xdp)

		nfp_nfd3_xsk_tx_bufs_free(tx_ring);



	memset(tx_ring->txds, 0, tx_ring->size);

	tx_ring->wr_p = 0;

	tx_ring->rd_p = 0;

	tx_ring->qcp_rd_p = 0;

	tx_ring->wr_ptr_add = 0;



	if (tx_ring->is_xdp || !dp->netdev)

		return;



	nd_q = netdev_get_tx_queue(dp->netdev, tx_ring->idx);

	netdev_tx_reset_queue(nd_q);

}



/**

 * nfp_nfd3_tx_ring_free() - Free resources allocated to a TX ring

 * @tx_ring:   TX ring to free

 */

void nfp_nfd3_tx_ring_free(struct nfp_net_tx_ring *tx_ring)

{

	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;

	struct nfp_net_dp *dp = &r_vec->nfp_net->dp;



	kvfree(tx_ring->txbufs);



	if (tx_ring->txds)

		dma_free_coherent(dp->dev, tx_ring->size,

				  tx_ring->txds, tx_ring->dma);



	tx_ring->cnt = 0;

	tx_ring->txbufs = NULL;

	tx_ring->txds = NULL;

	tx_ring->dma = 0;

	tx_ring->size = 0;

}



/**

 * nfp_nfd3_tx_ring_alloc() - Allocate resource for a TX ring

 * @dp:        NFP Net data path struct

 * @tx_ring:   TX Ring structure to allocate

 *

 * Return: 0 on success, negative errno otherwise.

 */

int

nfp_nfd3_tx_ring_alloc(struct nfp_net_dp *dp, struct nfp_net_tx_ring *tx_ring)

{

	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;



	tx_ring->cnt = dp->txd_cnt;



	tx_ring->size = array_size(tx_ring->cnt, sizeof(*tx_ring->txds));

	tx_ring->txds = dma_alloc_coherent(dp->dev, tx_ring->size,

					   &tx_ring->dma,

					   GFP_KERNEL | __GFP_NOWARN);

	if (!tx_ring->txds) {

		netdev_warn(dp->netdev, "failed to allocate TX descriptor ring memory, requested descriptor count: %d, consider lowering descriptor count\n",

			    tx_ring->cnt);

		goto err_alloc;

	}



	tx_ring->txbufs = kvcalloc(tx_ring->cnt, sizeof(*tx_ring->txbufs),

				   GFP_KERNEL);

	if (!tx_ring->txbufs)

		goto err_alloc;



	if (!tx_ring->is_xdp && dp->netdev)

		netif_set_xps_queue(dp->netdev, &r_vec->affinity_mask,

				    tx_ring->idx);



	return 0;



err_alloc:

	nfp_nfd3_tx_ring_free(tx_ring);

	return -ENOMEM;

}



void

nfp_nfd3_tx_ring_bufs_free(struct nfp_net_dp *dp,

			   struct nfp_net_tx_ring *tx_ring)

{

	unsigned int i;



	if (!tx_ring->is_xdp)

		return;



	for (i = 0; i < tx_ring->cnt; i++) {

		if (!tx_ring->txbufs[i].frag)

			return;



		nfp_net_dma_unmap_rx(dp, tx_ring->txbufs[i].dma_addr);

		__free_page(virt_to_page(tx_ring->txbufs[i].frag));

	}

}



int

nfp_nfd3_tx_ring_bufs_alloc(struct nfp_net_dp *dp,

			    struct nfp_net_tx_ring *tx_ring)

{

	struct nfp_nfd3_tx_buf *txbufs = tx_ring->txbufs;

	unsigned int i;



	if (!tx_ring->is_xdp)

		return 0;



	for (i = 0; i < tx_ring->cnt; i++) {

		txbufs[i].frag = nfp_net_rx_alloc_one(dp, &txbufs[i].dma_addr);

		if (!txbufs[i].frag) {

			nfp_nfd3_tx_ring_bufs_free(dp, tx_ring);

			return -ENOMEM;

		}

	}



	return 0;

}



void

nfp_nfd3_print_tx_descs(struct seq_file *file,

			struct nfp_net_r_vector *r_vec,

			struct nfp_net_tx_ring *tx_ring,

			u32 d_rd_p, u32 d_wr_p)

{

	struct nfp_nfd3_tx_desc *txd;

	u32 txd_cnt = tx_ring->cnt;

	int i;



	for (i = 0; i < txd_cnt; i++) {

		struct xdp_buff *xdp;

		struct sk_buff *skb;



		txd = &tx_ring->txds[i];

		seq_printf(file, "%04d: 0x%08x 0x%08x 0x%08x 0x%08x", i,

			   txd->vals[0], txd->vals[1],

			   txd->vals[2], txd->vals[3]);



		if (!tx_ring->is_xdp) {

			skb = READ_ONCE(tx_ring->txbufs[i].skb);

			if (skb)

				seq_printf(file, " skb->head=%p skb->data=%p",

					   skb->head, skb->data);

		} else {

			xdp = READ_ONCE(tx_ring->txbufs[i].xdp);

			if (xdp)

				seq_printf(file, " xdp->data=%p", xdp->data);

		}



		if (tx_ring->txbufs[i].dma_addr)

			seq_printf(file, " dma_addr=%pad",

				   &tx_ring->txbufs[i].dma_addr);



		if (i == tx_ring->rd_p % txd_cnt)

			seq_puts(file, " H_RD");

		if (i == tx_ring->wr_p % txd_cnt)

			seq_puts(file, " H_WR");

		if (i == d_rd_p % txd_cnt)

			seq_puts(file, " D_RD");

		if (i == d_wr_p % txd_cnt)

			seq_puts(file, " D_WR");



		seq_putc(file, '\n');

	}

}

drivers/net/ethernet/netronome/nfp/nfd3/xsk.c

0 → 100644
+408 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)

/* Copyright (C) 2018 Netronome Systems, Inc */

/* Copyright (C) 2021 Corigine, Inc */



#include <linux/bpf_trace.h>

#include <linux/netdevice.h>



#include "../nfp_app.h"

#include "../nfp_net.h"

#include "../nfp_net_dp.h"

#include "../nfp_net_xsk.h"

#include "nfd3.h"



static bool

nfp_nfd3_xsk_tx_xdp(const struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec,

		    struct nfp_net_rx_ring *rx_ring,

		    struct nfp_net_tx_ring *tx_ring,

		    struct nfp_net_xsk_rx_buf *xrxbuf, unsigned int pkt_len,

		    int pkt_off)

{

	struct xsk_buff_pool *pool = r_vec->xsk_pool;

	struct nfp_nfd3_tx_buf *txbuf;

	struct nfp_nfd3_tx_desc *txd;

	unsigned int wr_idx;



	if (nfp_net_tx_space(tx_ring) < 1)

		return false;



	xsk_buff_raw_dma_sync_for_device(pool, xrxbuf->dma_addr + pkt_off,

					 pkt_len);



	wr_idx = D_IDX(tx_ring, tx_ring->wr_p);



	txbuf = &tx_ring->txbufs[wr_idx];

	txbuf->xdp = xrxbuf->xdp;

	txbuf->real_len = pkt_len;

	txbuf->is_xsk_tx = true;



	/* Build TX descriptor */

	txd = &tx_ring->txds[wr_idx];

	txd->offset_eop = NFD3_DESC_TX_EOP;

	txd->dma_len = cpu_to_le16(pkt_len);

	nfp_desc_set_dma_addr(txd, xrxbuf->dma_addr + pkt_off);

	txd->data_len = cpu_to_le16(pkt_len);



	txd->flags = 0;

	txd->mss = 0;

	txd->lso_hdrlen = 0;



	tx_ring->wr_ptr_add++;

	tx_ring->wr_p++;



	return true;

}



static void nfp_nfd3_xsk_rx_skb(struct nfp_net_rx_ring *rx_ring,

				const struct nfp_net_rx_desc *rxd,

				struct nfp_net_xsk_rx_buf *xrxbuf,

				const struct nfp_meta_parsed *meta,

				unsigned int pkt_len,

				bool meta_xdp,

				unsigned int *skbs_polled)

{

	struct nfp_net_r_vector *r_vec = rx_ring->r_vec;

	struct nfp_net_dp *dp = &r_vec->nfp_net->dp;

	struct net_device *netdev;

	struct sk_buff *skb;



	if (likely(!meta->portid)) {

		netdev = dp->netdev;

	} else {

		struct nfp_net *nn = netdev_priv(dp->netdev);



		netdev = nfp_app_dev_get(nn->app, meta->portid, NULL);

		if (unlikely(!netdev)) {

			nfp_net_xsk_rx_drop(r_vec, xrxbuf);

			return;

		}

		nfp_repr_inc_rx_stats(netdev, pkt_len);

	}



	skb = napi_alloc_skb(&r_vec->napi, pkt_len);

	if (!skb) {

		nfp_net_xsk_rx_drop(r_vec, xrxbuf);

		return;

	}

	memcpy(skb_put(skb, pkt_len), xrxbuf->xdp->data, pkt_len);



	skb->mark = meta->mark;

	skb_set_hash(skb, meta->hash, meta->hash_type);



	skb_record_rx_queue(skb, rx_ring->idx);

	skb->protocol = eth_type_trans(skb, netdev);



	nfp_nfd3_rx_csum(dp, r_vec, rxd, meta, skb);



	if (rxd->rxd.flags & PCIE_DESC_RX_VLAN)

		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),

				       le16_to_cpu(rxd->rxd.vlan));

	if (meta_xdp)

		skb_metadata_set(skb,

				 xrxbuf->xdp->data - xrxbuf->xdp->data_meta);



	napi_gro_receive(&rx_ring->r_vec->napi, skb);



	nfp_net_xsk_rx_free(xrxbuf);



	(*skbs_polled)++;

}



static unsigned int

nfp_nfd3_xsk_rx(struct nfp_net_rx_ring *rx_ring, int budget,

		unsigned int *skbs_polled)

{

	struct nfp_net_r_vector *r_vec = rx_ring->r_vec;

	struct nfp_net_dp *dp = &r_vec->nfp_net->dp;

	struct nfp_net_tx_ring *tx_ring;

	struct bpf_prog *xdp_prog;

	bool xdp_redir = false;

	int pkts_polled = 0;



	xdp_prog = READ_ONCE(dp->xdp_prog);

	tx_ring = r_vec->xdp_ring;



	while (pkts_polled < budget) {

		unsigned int meta_len, data_len, pkt_len, pkt_off;

		struct nfp_net_xsk_rx_buf *xrxbuf;

		struct nfp_net_rx_desc *rxd;

		struct nfp_meta_parsed meta;

		int idx, act;



		idx = D_IDX(rx_ring, rx_ring->rd_p);



		rxd = &rx_ring->rxds[idx];

		if (!(rxd->rxd.meta_len_dd & PCIE_DESC_RX_DD))

			break;



		rx_ring->rd_p++;

		pkts_polled++;



		xrxbuf = &rx_ring->xsk_rxbufs[idx];



		/* If starved of buffers "drop" it and scream. */

		if (rx_ring->rd_p >= rx_ring->wr_p) {

			nn_dp_warn(dp, "Starved of RX buffers\n");

			nfp_net_xsk_rx_drop(r_vec, xrxbuf);

			break;

		}



		/* Memory barrier to ensure that we won't do other reads

		 * before the DD bit.

		 */

		dma_rmb();



		memset(&meta, 0, sizeof(meta));



		/* Only supporting AF_XDP with dynamic metadata so buffer layout

		 * is always:

		 *

		 *  ---------------------------------------------------------

		 * |  off | metadata  |             packet           | XXXX  |

		 *  ---------------------------------------------------------

		 */

		meta_len = rxd->rxd.meta_len_dd & PCIE_DESC_RX_META_LEN_MASK;

		data_len = le16_to_cpu(rxd->rxd.data_len);

		pkt_len = data_len - meta_len;



		if (unlikely(meta_len > NFP_NET_MAX_PREPEND)) {

			nn_dp_warn(dp, "Oversized RX packet metadata %u\n",

				   meta_len);

			nfp_net_xsk_rx_drop(r_vec, xrxbuf);

			continue;

		}



		/* Stats update. */

		u64_stats_update_begin(&r_vec->rx_sync);

		r_vec->rx_pkts++;

		r_vec->rx_bytes += pkt_len;

		u64_stats_update_end(&r_vec->rx_sync);



		xrxbuf->xdp->data += meta_len;

		xrxbuf->xdp->data_end = xrxbuf->xdp->data + pkt_len;

		xdp_set_data_meta_invalid(xrxbuf->xdp);

		xsk_buff_dma_sync_for_cpu(xrxbuf->xdp, r_vec->xsk_pool);

		net_prefetch(xrxbuf->xdp->data);



		if (meta_len) {

			if (unlikely(nfp_nfd3_parse_meta(dp->netdev, &meta,

							 xrxbuf->xdp->data -

							 meta_len,

							 xrxbuf->xdp->data,

							 pkt_len, meta_len))) {

				nn_dp_warn(dp, "Invalid RX packet metadata\n");

				nfp_net_xsk_rx_drop(r_vec, xrxbuf);

				continue;

			}



			if (unlikely(meta.portid)) {

				struct nfp_net *nn = netdev_priv(dp->netdev);



				if (meta.portid != NFP_META_PORT_ID_CTRL) {

					nfp_nfd3_xsk_rx_skb(rx_ring, rxd,

							    xrxbuf, &meta,

							    pkt_len, false,

							    skbs_polled);

					continue;

				}



				nfp_app_ctrl_rx_raw(nn->app, xrxbuf->xdp->data,

						    pkt_len);

				nfp_net_xsk_rx_free(xrxbuf);

				continue;

			}

		}



		act = bpf_prog_run_xdp(xdp_prog, xrxbuf->xdp);



		pkt_len = xrxbuf->xdp->data_end - xrxbuf->xdp->data;

		pkt_off = xrxbuf->xdp->data - xrxbuf->xdp->data_hard_start;



		switch (act) {

		case XDP_PASS:

			nfp_nfd3_xsk_rx_skb(rx_ring, rxd, xrxbuf, &meta, pkt_len,

					    true, skbs_polled);

			break;

		case XDP_TX:

			if (!nfp_nfd3_xsk_tx_xdp(dp, r_vec, rx_ring, tx_ring,

						 xrxbuf, pkt_len, pkt_off))

				nfp_net_xsk_rx_drop(r_vec, xrxbuf);

			else

				nfp_net_xsk_rx_unstash(xrxbuf);

			break;

		case XDP_REDIRECT:

			if (xdp_do_redirect(dp->netdev, xrxbuf->xdp, xdp_prog)) {

				nfp_net_xsk_rx_drop(r_vec, xrxbuf);

			} else {

				nfp_net_xsk_rx_unstash(xrxbuf);

				xdp_redir = true;

			}

			break;

		default:

			bpf_warn_invalid_xdp_action(dp->netdev, xdp_prog, act);

			fallthrough;

		case XDP_ABORTED:

			trace_xdp_exception(dp->netdev, xdp_prog, act);

			fallthrough;

		case XDP_DROP:

			nfp_net_xsk_rx_drop(r_vec, xrxbuf);

			break;

		}

	}



	nfp_net_xsk_rx_ring_fill_freelist(r_vec->rx_ring);



	if (xdp_redir)

		xdp_do_flush_map();



	if (tx_ring->wr_ptr_add)

		nfp_net_tx_xmit_more_flush(tx_ring);



	return pkts_polled;

}



void nfp_nfd3_xsk_tx_free(struct nfp_nfd3_tx_buf *txbuf)

{

	xsk_buff_free(txbuf->xdp);



	txbuf->dma_addr = 0;

	txbuf->xdp = NULL;

}



static bool nfp_nfd3_xsk_complete(struct nfp_net_tx_ring *tx_ring)

{

	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;

	u32 done_pkts = 0, done_bytes = 0, reused = 0;

	bool done_all;

	int idx, todo;

	u32 qcp_rd_p;



	if (tx_ring->wr_p == tx_ring->rd_p)

		return true;



	/* Work out how many descriptors have been transmitted. */

	qcp_rd_p = nfp_qcp_rd_ptr_read(tx_ring->qcp_q);



	if (qcp_rd_p == tx_ring->qcp_rd_p)

		return true;



	todo = D_IDX(tx_ring, qcp_rd_p - tx_ring->qcp_rd_p);



	done_all = todo <= NFP_NET_XDP_MAX_COMPLETE;

	todo = min(todo, NFP_NET_XDP_MAX_COMPLETE);



	tx_ring->qcp_rd_p = D_IDX(tx_ring, tx_ring->qcp_rd_p + todo);



	done_pkts = todo;

	while (todo--) {

		struct nfp_nfd3_tx_buf *txbuf;



		idx = D_IDX(tx_ring, tx_ring->rd_p);

		tx_ring->rd_p++;



		txbuf = &tx_ring->txbufs[idx];

		if (unlikely(!txbuf->real_len))

			continue;



		done_bytes += txbuf->real_len;

		txbuf->real_len = 0;



		if (txbuf->is_xsk_tx) {

			nfp_nfd3_xsk_tx_free(txbuf);

			reused++;

		}

	}



	u64_stats_update_begin(&r_vec->tx_sync);

	r_vec->tx_bytes += done_bytes;

	r_vec->tx_pkts += done_pkts;

	u64_stats_update_end(&r_vec->tx_sync);



	xsk_tx_completed(r_vec->xsk_pool, done_pkts - reused);



	WARN_ONCE(tx_ring->wr_p - tx_ring->rd_p > tx_ring->cnt,

		  "XDP TX ring corruption rd_p=%u wr_p=%u cnt=%u\n",

		  tx_ring->rd_p, tx_ring->wr_p, tx_ring->cnt);



	return done_all;

}



static void nfp_nfd3_xsk_tx(struct nfp_net_tx_ring *tx_ring)

{

	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;

	struct xdp_desc desc[NFP_NET_XSK_TX_BATCH];

	struct xsk_buff_pool *xsk_pool;

	struct nfp_nfd3_tx_desc *txd;

	u32 pkts = 0, wr_idx;

	u32 i, got;



	xsk_pool = r_vec->xsk_pool;



	while (nfp_net_tx_space(tx_ring) >= NFP_NET_XSK_TX_BATCH) {

		for (i = 0; i < NFP_NET_XSK_TX_BATCH; i++)

			if (!xsk_tx_peek_desc(xsk_pool, &desc[i]))

				break;

		got = i;

		if (!got)

			break;



		wr_idx = D_IDX(tx_ring, tx_ring->wr_p + i);

		prefetchw(&tx_ring->txds[wr_idx]);



		for (i = 0; i < got; i++)

			xsk_buff_raw_dma_sync_for_device(xsk_pool, desc[i].addr,

							 desc[i].len);



		for (i = 0; i < got; i++) {

			wr_idx = D_IDX(tx_ring, tx_ring->wr_p + i);



			tx_ring->txbufs[wr_idx].real_len = desc[i].len;

			tx_ring->txbufs[wr_idx].is_xsk_tx = false;



			/* Build TX descriptor. */

			txd = &tx_ring->txds[wr_idx];

			nfp_desc_set_dma_addr(txd,

					      xsk_buff_raw_get_dma(xsk_pool,

								   desc[i].addr

								   ));

			txd->offset_eop = NFD3_DESC_TX_EOP;

			txd->dma_len = cpu_to_le16(desc[i].len);

			txd->data_len = cpu_to_le16(desc[i].len);

		}



		tx_ring->wr_p += got;

		pkts += got;

	}



	if (!pkts)

		return;



	xsk_tx_release(xsk_pool);

	/* Ensure all records are visible before incrementing write counter. */

	wmb();

	nfp_qcp_wr_ptr_add(tx_ring->qcp_q, pkts);

}



int nfp_nfd3_xsk_poll(struct napi_struct *napi, int budget)

{

	struct nfp_net_r_vector *r_vec =

		container_of(napi, struct nfp_net_r_vector, napi);

	unsigned int pkts_polled, skbs = 0;



	pkts_polled = nfp_nfd3_xsk_rx(r_vec->rx_ring, budget, &skbs);



	if (pkts_polled < budget) {

		if (r_vec->tx_ring)

			nfp_nfd3_tx_complete(r_vec->tx_ring, budget);



		if (!nfp_nfd3_xsk_complete(r_vec->xdp_ring))

			pkts_polled = budget;



		nfp_nfd3_xsk_tx(r_vec->xdp_ring);



		if (pkts_polled < budget && napi_complete_done(napi, skbs))

			nfp_net_irq_unmask(r_vec->nfp_net, r_vec->irq_entry);

	}



	return pkts_polled;

}