Commit 57939fdc authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'nfp3800' 



Simon Horman says:

====================
nfp: support for NFP-3800

Yinjun Zhan says:

This is the second of a two part series to support the NFP-3800 device.

To utilize the new hardware features of the NFP-3800, driver adds support
of a new data path NFDK. This series mainly does some refactor work to the
data path related implementations. The data path specific implementations
are now separated into nfd3 and nfdk directories respectively, and the
common part is also moved into a new file.

* The series starts with a small refinement in Patch 1/10. Patches 2/10 and
  3/10 are the main refactoring of data path implementation, which prepares
  for the adding the NFDK data path.
* Before the introduction of NFDK, there's some more preparation work
  for NFP-3800 features, such as multi-descriptor per-packet and write-back
  mechanism of TX pointer, which is done in patches 4/10, 5/10, 6/10, 7/10.
* Patch 8/10 allows the driver to select data path according
  to firmware version. Finally, patches 9/10 and 10/10 introduce the new
  NFDK data path.

Changes between v1 and v2
* Correct kdoc for nfp_nfdk_tx()
* Correct build warnings on 32-bit

Thanks to everyone who contributed to this work.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 4b45e079 d9d95049

drivers/net/ethernet/netronome/nfp/Makefile

+6 −0
Original line number Diff line number Diff line
@@ -20,12 +20,18 @@ nfp-objs := \ 
	    ccm_mbox.o \

	    devlink_param.o \

	    nfp_asm.o \

	    nfd3/dp.o \

	    nfd3/rings.o \

	    nfd3/xsk.o \

	    nfdk/dp.o \

	    nfdk/rings.o \

	    nfp_app.o \

	    nfp_app_nic.o \

	    nfp_devlink.o \

	    nfp_hwmon.o \

	    nfp_main.o \

	    nfp_net_common.o \

	    nfp_net_dp.o \

	    nfp_net_ctrl.o \

	    nfp_net_debugdump.o \

	    nfp_net_ethtool.o \

drivers/net/ethernet/netronome/nfp/nfd3/dp.c

0 → 100644
+1350 −0

File added.

Preview size limit exceeded, changes collapsed.

drivers/net/ethernet/netronome/nfp/nfd3/nfd3.h

0 → 100644
+106 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */

/* Copyright (C) 2015-2019 Netronome Systems, Inc. */



#ifndef _NFP_DP_NFD3_H_

#define _NFP_DP_NFD3_H_



struct sk_buff;

struct net_device;



/* TX descriptor format */



#define NFD3_DESC_TX_EOP		BIT(7)

#define NFD3_DESC_TX_OFFSET_MASK	GENMASK(6, 0)

#define NFD3_DESC_TX_MSS_MASK		GENMASK(13, 0)



/* Flags in the host TX descriptor */

#define NFD3_DESC_TX_CSUM		BIT(7)

#define NFD3_DESC_TX_IP4_CSUM		BIT(6)

#define NFD3_DESC_TX_TCP_CSUM		BIT(5)

#define NFD3_DESC_TX_UDP_CSUM		BIT(4)

#define NFD3_DESC_TX_VLAN		BIT(3)

#define NFD3_DESC_TX_LSO		BIT(2)

#define NFD3_DESC_TX_ENCAP		BIT(1)

#define NFD3_DESC_TX_O_IP4_CSUM	BIT(0)



struct nfp_nfd3_tx_desc {

	union {

		struct {

			u8 dma_addr_hi; /* High bits of host buf address */

			__le16 dma_len;	/* Length to DMA for this desc */

			u8 offset_eop;	/* Offset in buf where pkt starts +

					 * highest bit is eop flag.

					 */

			__le32 dma_addr_lo; /* Low 32bit of host buf addr */



			__le16 mss;	/* MSS to be used for LSO */

			u8 lso_hdrlen;	/* LSO, TCP payload offset */

			u8 flags;	/* TX Flags, see @NFD3_DESC_TX_* */

			union {

				struct {

					u8 l3_offset; /* L3 header offset */

					u8 l4_offset; /* L4 header offset */

				};

				__le16 vlan; /* VLAN tag to add if indicated */

			};

			__le16 data_len; /* Length of frame + meta data */

		} __packed;

		__le32 vals[4];

		__le64 vals8[2];

	};

};



/**

 * struct nfp_nfd3_tx_buf - software TX buffer descriptor

 * @skb:	normal ring, sk_buff associated with this buffer

 * @frag:	XDP ring, page frag associated with this buffer

 * @xdp:	XSK buffer pool handle (for AF_XDP)

 * @dma_addr:	DMA mapping address of the buffer

 * @fidx:	Fragment index (-1 for the head and [0..nr_frags-1] for frags)

 * @pkt_cnt:	Number of packets to be produced out of the skb associated

 *		with this buffer (valid only on the head's buffer).

 *		Will be 1 for all non-TSO packets.

 * @is_xsk_tx:	Flag if buffer is a RX buffer after a XDP_TX action and not a

 *		buffer from the TX queue (for AF_XDP).

 * @real_len:	Number of bytes which to be produced out of the skb (valid only

 *		on the head's buffer). Equal to skb->len for non-TSO packets.

 */

struct nfp_nfd3_tx_buf {

	union {

		struct sk_buff *skb;

		void *frag;

		struct xdp_buff *xdp;

	};

	dma_addr_t dma_addr;

	union {

		struct {

			short int fidx;

			u16 pkt_cnt;

		};

		struct {

			bool is_xsk_tx;

		};

	};

	u32 real_len;

};



void

nfp_nfd3_rx_csum(const struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec,

		 const struct nfp_net_rx_desc *rxd,

		 const struct nfp_meta_parsed *meta, struct sk_buff *skb);

bool

nfp_nfd3_parse_meta(struct net_device *netdev, struct nfp_meta_parsed *meta,

		    void *data, void *pkt, unsigned int pkt_len, int meta_len);

void nfp_nfd3_tx_complete(struct nfp_net_tx_ring *tx_ring, int budget);

int nfp_nfd3_poll(struct napi_struct *napi, int budget);

netdev_tx_t nfp_nfd3_tx(struct sk_buff *skb, struct net_device *netdev);

bool

nfp_nfd3_ctrl_tx_one(struct nfp_net *nn, struct nfp_net_r_vector *r_vec,

		     struct sk_buff *skb, bool old);

void nfp_nfd3_ctrl_poll(struct tasklet_struct *t);

void nfp_nfd3_rx_ring_fill_freelist(struct nfp_net_dp *dp,

				    struct nfp_net_rx_ring *rx_ring);

void nfp_nfd3_xsk_tx_free(struct nfp_nfd3_tx_buf *txbuf);

int nfp_nfd3_xsk_poll(struct napi_struct *napi, int budget);



#endif

drivers/net/ethernet/netronome/nfp/nfd3/rings.c

0 → 100644
+275 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)

/* Copyright (C) 2015-2019 Netronome Systems, Inc. */



#include <linux/seq_file.h>



#include "../nfp_net.h"

#include "../nfp_net_dp.h"

#include "../nfp_net_xsk.h"

#include "nfd3.h"



static void nfp_nfd3_xsk_tx_bufs_free(struct nfp_net_tx_ring *tx_ring)

{

	struct nfp_nfd3_tx_buf *txbuf;

	unsigned int idx;



	while (tx_ring->rd_p != tx_ring->wr_p) {

		idx = D_IDX(tx_ring, tx_ring->rd_p);

		txbuf = &tx_ring->txbufs[idx];



		txbuf->real_len = 0;



		tx_ring->qcp_rd_p++;

		tx_ring->rd_p++;



		if (tx_ring->r_vec->xsk_pool) {

			if (txbuf->is_xsk_tx)

				nfp_nfd3_xsk_tx_free(txbuf);



			xsk_tx_completed(tx_ring->r_vec->xsk_pool, 1);

		}

	}

}



/**

 * nfp_nfd3_tx_ring_reset() - Free any untransmitted buffers and reset pointers

 * @dp:		NFP Net data path struct

 * @tx_ring:	TX ring structure

 *

 * Assumes that the device is stopped, must be idempotent.

 */

static void

nfp_nfd3_tx_ring_reset(struct nfp_net_dp *dp, struct nfp_net_tx_ring *tx_ring)

{

	struct netdev_queue *nd_q;

	const skb_frag_t *frag;



	while (!tx_ring->is_xdp && tx_ring->rd_p != tx_ring->wr_p) {

		struct nfp_nfd3_tx_buf *tx_buf;

		struct sk_buff *skb;

		int idx, nr_frags;



		idx = D_IDX(tx_ring, tx_ring->rd_p);

		tx_buf = &tx_ring->txbufs[idx];



		skb = tx_ring->txbufs[idx].skb;

		nr_frags = skb_shinfo(skb)->nr_frags;



		if (tx_buf->fidx == -1) {

			/* unmap head */

			dma_unmap_single(dp->dev, tx_buf->dma_addr,

					 skb_headlen(skb), DMA_TO_DEVICE);

		} else {

			/* unmap fragment */

			frag = &skb_shinfo(skb)->frags[tx_buf->fidx];

			dma_unmap_page(dp->dev, tx_buf->dma_addr,

				       skb_frag_size(frag), DMA_TO_DEVICE);

		}



		/* check for last gather fragment */

		if (tx_buf->fidx == nr_frags - 1)

			dev_kfree_skb_any(skb);



		tx_buf->dma_addr = 0;

		tx_buf->skb = NULL;

		tx_buf->fidx = -2;



		tx_ring->qcp_rd_p++;

		tx_ring->rd_p++;

	}



	if (tx_ring->is_xdp)

		nfp_nfd3_xsk_tx_bufs_free(tx_ring);



	memset(tx_ring->txds, 0, tx_ring->size);

	tx_ring->wr_p = 0;

	tx_ring->rd_p = 0;

	tx_ring->qcp_rd_p = 0;

	tx_ring->wr_ptr_add = 0;



	if (tx_ring->is_xdp || !dp->netdev)

		return;



	nd_q = netdev_get_tx_queue(dp->netdev, tx_ring->idx);

	netdev_tx_reset_queue(nd_q);

}



/**

 * nfp_nfd3_tx_ring_free() - Free resources allocated to a TX ring

 * @tx_ring:   TX ring to free

 */

static void nfp_nfd3_tx_ring_free(struct nfp_net_tx_ring *tx_ring)

{

	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;

	struct nfp_net_dp *dp = &r_vec->nfp_net->dp;



	kvfree(tx_ring->txbufs);



	if (tx_ring->txds)

		dma_free_coherent(dp->dev, tx_ring->size,

				  tx_ring->txds, tx_ring->dma);



	tx_ring->cnt = 0;

	tx_ring->txbufs = NULL;

	tx_ring->txds = NULL;

	tx_ring->dma = 0;

	tx_ring->size = 0;

}



/**

 * nfp_nfd3_tx_ring_alloc() - Allocate resource for a TX ring

 * @dp:        NFP Net data path struct

 * @tx_ring:   TX Ring structure to allocate

 *

 * Return: 0 on success, negative errno otherwise.

 */

static int

nfp_nfd3_tx_ring_alloc(struct nfp_net_dp *dp, struct nfp_net_tx_ring *tx_ring)

{

	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;



	tx_ring->cnt = dp->txd_cnt;



	tx_ring->size = array_size(tx_ring->cnt, sizeof(*tx_ring->txds));

	tx_ring->txds = dma_alloc_coherent(dp->dev, tx_ring->size,

					   &tx_ring->dma,

					   GFP_KERNEL | __GFP_NOWARN);

	if (!tx_ring->txds) {

		netdev_warn(dp->netdev, "failed to allocate TX descriptor ring memory, requested descriptor count: %d, consider lowering descriptor count\n",

			    tx_ring->cnt);

		goto err_alloc;

	}



	tx_ring->txbufs = kvcalloc(tx_ring->cnt, sizeof(*tx_ring->txbufs),

				   GFP_KERNEL);

	if (!tx_ring->txbufs)

		goto err_alloc;



	if (!tx_ring->is_xdp && dp->netdev)

		netif_set_xps_queue(dp->netdev, &r_vec->affinity_mask,

				    tx_ring->idx);



	return 0;



err_alloc:

	nfp_nfd3_tx_ring_free(tx_ring);

	return -ENOMEM;

}



static void

nfp_nfd3_tx_ring_bufs_free(struct nfp_net_dp *dp,

			   struct nfp_net_tx_ring *tx_ring)

{

	unsigned int i;



	if (!tx_ring->is_xdp)

		return;



	for (i = 0; i < tx_ring->cnt; i++) {

		if (!tx_ring->txbufs[i].frag)

			return;



		nfp_net_dma_unmap_rx(dp, tx_ring->txbufs[i].dma_addr);

		__free_page(virt_to_page(tx_ring->txbufs[i].frag));

	}

}



static int

nfp_nfd3_tx_ring_bufs_alloc(struct nfp_net_dp *dp,

			    struct nfp_net_tx_ring *tx_ring)

{

	struct nfp_nfd3_tx_buf *txbufs = tx_ring->txbufs;

	unsigned int i;



	if (!tx_ring->is_xdp)

		return 0;



	for (i = 0; i < tx_ring->cnt; i++) {

		txbufs[i].frag = nfp_net_rx_alloc_one(dp, &txbufs[i].dma_addr);

		if (!txbufs[i].frag) {

			nfp_nfd3_tx_ring_bufs_free(dp, tx_ring);

			return -ENOMEM;

		}

	}



	return 0;

}



static void

nfp_nfd3_print_tx_descs(struct seq_file *file,

			struct nfp_net_r_vector *r_vec,

			struct nfp_net_tx_ring *tx_ring,

			u32 d_rd_p, u32 d_wr_p)

{

	struct nfp_nfd3_tx_desc *txd;

	u32 txd_cnt = tx_ring->cnt;

	int i;



	for (i = 0; i < txd_cnt; i++) {

		struct xdp_buff *xdp;

		struct sk_buff *skb;



		txd = &tx_ring->txds[i];

		seq_printf(file, "%04d: 0x%08x 0x%08x 0x%08x 0x%08x", i,

			   txd->vals[0], txd->vals[1],

			   txd->vals[2], txd->vals[3]);



		if (!tx_ring->is_xdp) {

			skb = READ_ONCE(tx_ring->txbufs[i].skb);

			if (skb)

				seq_printf(file, " skb->head=%p skb->data=%p",

					   skb->head, skb->data);

		} else {

			xdp = READ_ONCE(tx_ring->txbufs[i].xdp);

			if (xdp)

				seq_printf(file, " xdp->data=%p", xdp->data);

		}



		if (tx_ring->txbufs[i].dma_addr)

			seq_printf(file, " dma_addr=%pad",

				   &tx_ring->txbufs[i].dma_addr);



		if (i == tx_ring->rd_p % txd_cnt)

			seq_puts(file, " H_RD");

		if (i == tx_ring->wr_p % txd_cnt)

			seq_puts(file, " H_WR");

		if (i == d_rd_p % txd_cnt)

			seq_puts(file, " D_RD");

		if (i == d_wr_p % txd_cnt)

			seq_puts(file, " D_WR");



		seq_putc(file, '\n');

	}

}



#define NFP_NFD3_CFG_CTRL_SUPPORTED					\

	(NFP_NET_CFG_CTRL_ENABLE | NFP_NET_CFG_CTRL_PROMISC |		\

	 NFP_NET_CFG_CTRL_L2BC | NFP_NET_CFG_CTRL_L2MC |		\

	 NFP_NET_CFG_CTRL_RXCSUM | NFP_NET_CFG_CTRL_TXCSUM |		\

	 NFP_NET_CFG_CTRL_RXVLAN | NFP_NET_CFG_CTRL_TXVLAN |		\

	 NFP_NET_CFG_CTRL_GATHER | NFP_NET_CFG_CTRL_LSO |		\

	 NFP_NET_CFG_CTRL_CTAG_FILTER | NFP_NET_CFG_CTRL_CMSG_DATA |	\

	 NFP_NET_CFG_CTRL_RINGCFG | NFP_NET_CFG_CTRL_RSS |		\

	 NFP_NET_CFG_CTRL_IRQMOD | NFP_NET_CFG_CTRL_TXRWB |		\

	 NFP_NET_CFG_CTRL_VXLAN | NFP_NET_CFG_CTRL_NVGRE |		\

	 NFP_NET_CFG_CTRL_BPF | NFP_NET_CFG_CTRL_LSO2 |			\

	 NFP_NET_CFG_CTRL_RSS2 | NFP_NET_CFG_CTRL_CSUM_COMPLETE |	\

	 NFP_NET_CFG_CTRL_LIVE_ADDR)



const struct nfp_dp_ops nfp_nfd3_ops = {

	.version		= NFP_NFD_VER_NFD3,

	.tx_min_desc_per_pkt	= 1,

	.cap_mask		= NFP_NFD3_CFG_CTRL_SUPPORTED,

	.poll			= nfp_nfd3_poll,

	.xsk_poll		= nfp_nfd3_xsk_poll,

	.ctrl_poll		= nfp_nfd3_ctrl_poll,

	.xmit			= nfp_nfd3_tx,

	.ctrl_tx_one		= nfp_nfd3_ctrl_tx_one,

	.rx_ring_fill_freelist	= nfp_nfd3_rx_ring_fill_freelist,

	.tx_ring_alloc		= nfp_nfd3_tx_ring_alloc,

	.tx_ring_reset		= nfp_nfd3_tx_ring_reset,

	.tx_ring_free		= nfp_nfd3_tx_ring_free,

	.tx_ring_bufs_alloc	= nfp_nfd3_tx_ring_bufs_alloc,

	.tx_ring_bufs_free	= nfp_nfd3_tx_ring_bufs_free,

	.print_tx_descs		= nfp_nfd3_print_tx_descs

};

drivers/net/ethernet/netronome/nfp/nfd3/xsk.c

0 → 100644
+408 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)

/* Copyright (C) 2018 Netronome Systems, Inc */

/* Copyright (C) 2021 Corigine, Inc */



#include <linux/bpf_trace.h>

#include <linux/netdevice.h>



#include "../nfp_app.h"

#include "../nfp_net.h"

#include "../nfp_net_dp.h"

#include "../nfp_net_xsk.h"

#include "nfd3.h"



static bool

nfp_nfd3_xsk_tx_xdp(const struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec,

		    struct nfp_net_rx_ring *rx_ring,

		    struct nfp_net_tx_ring *tx_ring,

		    struct nfp_net_xsk_rx_buf *xrxbuf, unsigned int pkt_len,

		    int pkt_off)

{

	struct xsk_buff_pool *pool = r_vec->xsk_pool;

	struct nfp_nfd3_tx_buf *txbuf;

	struct nfp_nfd3_tx_desc *txd;

	unsigned int wr_idx;



	if (nfp_net_tx_space(tx_ring) < 1)

		return false;



	xsk_buff_raw_dma_sync_for_device(pool, xrxbuf->dma_addr + pkt_off,

					 pkt_len);



	wr_idx = D_IDX(tx_ring, tx_ring->wr_p);



	txbuf = &tx_ring->txbufs[wr_idx];

	txbuf->xdp = xrxbuf->xdp;

	txbuf->real_len = pkt_len;

	txbuf->is_xsk_tx = true;



	/* Build TX descriptor */

	txd = &tx_ring->txds[wr_idx];

	txd->offset_eop = NFD3_DESC_TX_EOP;

	txd->dma_len = cpu_to_le16(pkt_len);

	nfp_desc_set_dma_addr(txd, xrxbuf->dma_addr + pkt_off);

	txd->data_len = cpu_to_le16(pkt_len);



	txd->flags = 0;

	txd->mss = 0;

	txd->lso_hdrlen = 0;



	tx_ring->wr_ptr_add++;

	tx_ring->wr_p++;



	return true;

}



static void nfp_nfd3_xsk_rx_skb(struct nfp_net_rx_ring *rx_ring,

				const struct nfp_net_rx_desc *rxd,

				struct nfp_net_xsk_rx_buf *xrxbuf,

				const struct nfp_meta_parsed *meta,

				unsigned int pkt_len,

				bool meta_xdp,

				unsigned int *skbs_polled)

{

	struct nfp_net_r_vector *r_vec = rx_ring->r_vec;

	struct nfp_net_dp *dp = &r_vec->nfp_net->dp;

	struct net_device *netdev;

	struct sk_buff *skb;



	if (likely(!meta->portid)) {

		netdev = dp->netdev;

	} else {

		struct nfp_net *nn = netdev_priv(dp->netdev);



		netdev = nfp_app_dev_get(nn->app, meta->portid, NULL);

		if (unlikely(!netdev)) {

			nfp_net_xsk_rx_drop(r_vec, xrxbuf);

			return;

		}

		nfp_repr_inc_rx_stats(netdev, pkt_len);

	}



	skb = napi_alloc_skb(&r_vec->napi, pkt_len);

	if (!skb) {

		nfp_net_xsk_rx_drop(r_vec, xrxbuf);

		return;

	}

	memcpy(skb_put(skb, pkt_len), xrxbuf->xdp->data, pkt_len);



	skb->mark = meta->mark;

	skb_set_hash(skb, meta->hash, meta->hash_type);



	skb_record_rx_queue(skb, rx_ring->idx);

	skb->protocol = eth_type_trans(skb, netdev);



	nfp_nfd3_rx_csum(dp, r_vec, rxd, meta, skb);



	if (rxd->rxd.flags & PCIE_DESC_RX_VLAN)

		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),

				       le16_to_cpu(rxd->rxd.vlan));

	if (meta_xdp)

		skb_metadata_set(skb,

				 xrxbuf->xdp->data - xrxbuf->xdp->data_meta);



	napi_gro_receive(&rx_ring->r_vec->napi, skb);



	nfp_net_xsk_rx_free(xrxbuf);



	(*skbs_polled)++;

}



static unsigned int

nfp_nfd3_xsk_rx(struct nfp_net_rx_ring *rx_ring, int budget,

		unsigned int *skbs_polled)

{

	struct nfp_net_r_vector *r_vec = rx_ring->r_vec;

	struct nfp_net_dp *dp = &r_vec->nfp_net->dp;

	struct nfp_net_tx_ring *tx_ring;

	struct bpf_prog *xdp_prog;

	bool xdp_redir = false;

	int pkts_polled = 0;



	xdp_prog = READ_ONCE(dp->xdp_prog);

	tx_ring = r_vec->xdp_ring;



	while (pkts_polled < budget) {

		unsigned int meta_len, data_len, pkt_len, pkt_off;

		struct nfp_net_xsk_rx_buf *xrxbuf;

		struct nfp_net_rx_desc *rxd;

		struct nfp_meta_parsed meta;

		int idx, act;



		idx = D_IDX(rx_ring, rx_ring->rd_p);



		rxd = &rx_ring->rxds[idx];

		if (!(rxd->rxd.meta_len_dd & PCIE_DESC_RX_DD))

			break;



		rx_ring->rd_p++;

		pkts_polled++;



		xrxbuf = &rx_ring->xsk_rxbufs[idx];



		/* If starved of buffers "drop" it and scream. */

		if (rx_ring->rd_p >= rx_ring->wr_p) {

			nn_dp_warn(dp, "Starved of RX buffers\n");

			nfp_net_xsk_rx_drop(r_vec, xrxbuf);

			break;

		}



		/* Memory barrier to ensure that we won't do other reads

		 * before the DD bit.

		 */

		dma_rmb();



		memset(&meta, 0, sizeof(meta));



		/* Only supporting AF_XDP with dynamic metadata so buffer layout

		 * is always:

		 *

		 *  ---------------------------------------------------------

		 * |  off | metadata  |             packet           | XXXX  |

		 *  ---------------------------------------------------------

		 */

		meta_len = rxd->rxd.meta_len_dd & PCIE_DESC_RX_META_LEN_MASK;

		data_len = le16_to_cpu(rxd->rxd.data_len);

		pkt_len = data_len - meta_len;



		if (unlikely(meta_len > NFP_NET_MAX_PREPEND)) {

			nn_dp_warn(dp, "Oversized RX packet metadata %u\n",

				   meta_len);

			nfp_net_xsk_rx_drop(r_vec, xrxbuf);

			continue;

		}



		/* Stats update. */

		u64_stats_update_begin(&r_vec->rx_sync);

		r_vec->rx_pkts++;

		r_vec->rx_bytes += pkt_len;

		u64_stats_update_end(&r_vec->rx_sync);



		xrxbuf->xdp->data += meta_len;

		xrxbuf->xdp->data_end = xrxbuf->xdp->data + pkt_len;

		xdp_set_data_meta_invalid(xrxbuf->xdp);

		xsk_buff_dma_sync_for_cpu(xrxbuf->xdp, r_vec->xsk_pool);

		net_prefetch(xrxbuf->xdp->data);



		if (meta_len) {

			if (unlikely(nfp_nfd3_parse_meta(dp->netdev, &meta,

							 xrxbuf->xdp->data -

							 meta_len,

							 xrxbuf->xdp->data,

							 pkt_len, meta_len))) {

				nn_dp_warn(dp, "Invalid RX packet metadata\n");

				nfp_net_xsk_rx_drop(r_vec, xrxbuf);

				continue;

			}



			if (unlikely(meta.portid)) {

				struct nfp_net *nn = netdev_priv(dp->netdev);



				if (meta.portid != NFP_META_PORT_ID_CTRL) {

					nfp_nfd3_xsk_rx_skb(rx_ring, rxd,

							    xrxbuf, &meta,

							    pkt_len, false,

							    skbs_polled);

					continue;

				}



				nfp_app_ctrl_rx_raw(nn->app, xrxbuf->xdp->data,

						    pkt_len);

				nfp_net_xsk_rx_free(xrxbuf);

				continue;

			}

		}



		act = bpf_prog_run_xdp(xdp_prog, xrxbuf->xdp);



		pkt_len = xrxbuf->xdp->data_end - xrxbuf->xdp->data;

		pkt_off = xrxbuf->xdp->data - xrxbuf->xdp->data_hard_start;



		switch (act) {

		case XDP_PASS:

			nfp_nfd3_xsk_rx_skb(rx_ring, rxd, xrxbuf, &meta, pkt_len,

					    true, skbs_polled);

			break;

		case XDP_TX:

			if (!nfp_nfd3_xsk_tx_xdp(dp, r_vec, rx_ring, tx_ring,

						 xrxbuf, pkt_len, pkt_off))

				nfp_net_xsk_rx_drop(r_vec, xrxbuf);

			else

				nfp_net_xsk_rx_unstash(xrxbuf);

			break;

		case XDP_REDIRECT:

			if (xdp_do_redirect(dp->netdev, xrxbuf->xdp, xdp_prog)) {

				nfp_net_xsk_rx_drop(r_vec, xrxbuf);

			} else {

				nfp_net_xsk_rx_unstash(xrxbuf);

				xdp_redir = true;

			}

			break;

		default:

			bpf_warn_invalid_xdp_action(dp->netdev, xdp_prog, act);

			fallthrough;

		case XDP_ABORTED:

			trace_xdp_exception(dp->netdev, xdp_prog, act);

			fallthrough;

		case XDP_DROP:

			nfp_net_xsk_rx_drop(r_vec, xrxbuf);

			break;

		}

	}



	nfp_net_xsk_rx_ring_fill_freelist(r_vec->rx_ring);



	if (xdp_redir)

		xdp_do_flush_map();



	if (tx_ring->wr_ptr_add)

		nfp_net_tx_xmit_more_flush(tx_ring);



	return pkts_polled;

}



void nfp_nfd3_xsk_tx_free(struct nfp_nfd3_tx_buf *txbuf)

{

	xsk_buff_free(txbuf->xdp);



	txbuf->dma_addr = 0;

	txbuf->xdp = NULL;

}



static bool nfp_nfd3_xsk_complete(struct nfp_net_tx_ring *tx_ring)

{

	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;

	u32 done_pkts = 0, done_bytes = 0, reused = 0;

	bool done_all;

	int idx, todo;

	u32 qcp_rd_p;



	if (tx_ring->wr_p == tx_ring->rd_p)

		return true;



	/* Work out how many descriptors have been transmitted. */

	qcp_rd_p = nfp_qcp_rd_ptr_read(tx_ring->qcp_q);



	if (qcp_rd_p == tx_ring->qcp_rd_p)

		return true;



	todo = D_IDX(tx_ring, qcp_rd_p - tx_ring->qcp_rd_p);



	done_all = todo <= NFP_NET_XDP_MAX_COMPLETE;

	todo = min(todo, NFP_NET_XDP_MAX_COMPLETE);



	tx_ring->qcp_rd_p = D_IDX(tx_ring, tx_ring->qcp_rd_p + todo);



	done_pkts = todo;

	while (todo--) {

		struct nfp_nfd3_tx_buf *txbuf;



		idx = D_IDX(tx_ring, tx_ring->rd_p);

		tx_ring->rd_p++;



		txbuf = &tx_ring->txbufs[idx];

		if (unlikely(!txbuf->real_len))

			continue;



		done_bytes += txbuf->real_len;

		txbuf->real_len = 0;



		if (txbuf->is_xsk_tx) {

			nfp_nfd3_xsk_tx_free(txbuf);

			reused++;

		}

	}



	u64_stats_update_begin(&r_vec->tx_sync);

	r_vec->tx_bytes += done_bytes;

	r_vec->tx_pkts += done_pkts;

	u64_stats_update_end(&r_vec->tx_sync);



	xsk_tx_completed(r_vec->xsk_pool, done_pkts - reused);



	WARN_ONCE(tx_ring->wr_p - tx_ring->rd_p > tx_ring->cnt,

		  "XDP TX ring corruption rd_p=%u wr_p=%u cnt=%u\n",

		  tx_ring->rd_p, tx_ring->wr_p, tx_ring->cnt);



	return done_all;

}



static void nfp_nfd3_xsk_tx(struct nfp_net_tx_ring *tx_ring)

{

	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;

	struct xdp_desc desc[NFP_NET_XSK_TX_BATCH];

	struct xsk_buff_pool *xsk_pool;

	struct nfp_nfd3_tx_desc *txd;

	u32 pkts = 0, wr_idx;

	u32 i, got;



	xsk_pool = r_vec->xsk_pool;



	while (nfp_net_tx_space(tx_ring) >= NFP_NET_XSK_TX_BATCH) {

		for (i = 0; i < NFP_NET_XSK_TX_BATCH; i++)

			if (!xsk_tx_peek_desc(xsk_pool, &desc[i]))

				break;

		got = i;

		if (!got)

			break;



		wr_idx = D_IDX(tx_ring, tx_ring->wr_p + i);

		prefetchw(&tx_ring->txds[wr_idx]);



		for (i = 0; i < got; i++)

			xsk_buff_raw_dma_sync_for_device(xsk_pool, desc[i].addr,

							 desc[i].len);



		for (i = 0; i < got; i++) {

			wr_idx = D_IDX(tx_ring, tx_ring->wr_p + i);



			tx_ring->txbufs[wr_idx].real_len = desc[i].len;

			tx_ring->txbufs[wr_idx].is_xsk_tx = false;



			/* Build TX descriptor. */

			txd = &tx_ring->txds[wr_idx];

			nfp_desc_set_dma_addr(txd,

					      xsk_buff_raw_get_dma(xsk_pool,

								   desc[i].addr

								   ));

			txd->offset_eop = NFD3_DESC_TX_EOP;

			txd->dma_len = cpu_to_le16(desc[i].len);

			txd->data_len = cpu_to_le16(desc[i].len);

		}



		tx_ring->wr_p += got;

		pkts += got;

	}



	if (!pkts)

		return;



	xsk_tx_release(xsk_pool);

	/* Ensure all records are visible before incrementing write counter. */

	wmb();

	nfp_qcp_wr_ptr_add(tx_ring->qcp_q, pkts);

}



int nfp_nfd3_xsk_poll(struct napi_struct *napi, int budget)

{

	struct nfp_net_r_vector *r_vec =

		container_of(napi, struct nfp_net_r_vector, napi);

	unsigned int pkts_polled, skbs = 0;



	pkts_polled = nfp_nfd3_xsk_rx(r_vec->rx_ring, budget, &skbs);



	if (pkts_polled < budget) {

		if (r_vec->tx_ring)

			nfp_nfd3_tx_complete(r_vec->tx_ring, budget);



		if (!nfp_nfd3_xsk_complete(r_vec->xdp_ring))

			pkts_polled = budget;



		nfp_nfd3_xsk_tx(r_vec->xdp_ring);



		if (pkts_polled < budget && napi_complete_done(napi, skbs))

			nfp_net_irq_unmask(r_vec->nfp_net, r_vec->irq_entry);

	}



	return pkts_polled;

}