Commit c10d12e3 authored by Jakub Kicinski's avatar Jakub Kicinski Committed by David S. Miller
Browse files

nfp: add support for NFDK data path 



Add new data path.  The TX is completely different, each packet
has multiple descriptor entries (between 2 and 32).  TX ring is
divided into blocks 32 descriptor, and descritors of one packet
can't cross block bounds. The RX side is the same for now.

ABI version 5 or later is required.  There is no support for
VLAN insertion on TX. XDP_TX action and AF_XDP zero-copy is not
implemented in NFDK path.

Changes to Jakub's work:
* Move statistics of hw_csum_tx after jumbo packet's segmentation.
* Set L3_CSUM flag to enable recaculating of L3 header checksum
in ipv4 case.
* Mark the case of TSO a packet with metadata prepended as
unsupported.

Signed-off-by: default avatarJakub Kicinski <jakub.kicinski@netronome.com>
Signed-off-by: default avatarXingfeng Hu <xingfeng.hu@corigine.com>
Signed-off-by: default avatarYinjun Zhang <yinjun.zhang@corigine.com>
Signed-off-by: default avatarDianchao Wang <dianchao.wang@corigine.com>
Signed-off-by: default avatarFei Qin <fei.qin@corigine.com>
Signed-off-by: default avatarSimon Horman <simon.horman@corigine.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent d9e3c299

drivers/net/ethernet/netronome/nfp/Makefile

+2 −0
Original line number Diff line number Diff line
@@ -23,6 +23,8 @@ nfp-objs := \ 
	    nfd3/dp.o \

	    nfd3/rings.o \

	    nfd3/xsk.o \

	    nfdk/dp.o \

	    nfdk/rings.o \

	    nfp_app.o \

	    nfp_app_nic.o \

	    nfp_devlink.o \

drivers/net/ethernet/netronome/nfp/nfdk/dp.c

0 → 100644
+1338 −0

File added.

Preview size limit exceeded, changes collapsed.

drivers/net/ethernet/netronome/nfp/nfdk/nfdk.h

0 → 100644
+112 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */

/* Copyright (C) 2019 Netronome Systems, Inc. */



#ifndef _NFP_DP_NFDK_H_

#define _NFP_DP_NFDK_H_



#include <linux/bitops.h>

#include <linux/types.h>



#define NFDK_TX_DESC_PER_SIMPLE_PKT	2



#define NFDK_TX_MAX_DATA_PER_HEAD	SZ_4K

#define NFDK_TX_MAX_DATA_PER_DESC	SZ_16K

#define NFDK_TX_DESC_BLOCK_SZ		256

#define NFDK_TX_DESC_BLOCK_CNT		(NFDK_TX_DESC_BLOCK_SZ /	\

					 sizeof(struct nfp_nfdk_tx_desc))

#define NFDK_TX_DESC_STOP_CNT		(NFDK_TX_DESC_BLOCK_CNT *	\

					 NFDK_TX_DESC_PER_SIMPLE_PKT)

#define NFDK_TX_MAX_DATA_PER_BLOCK	SZ_64K

#define NFDK_TX_DESC_GATHER_MAX		17



/* TX descriptor format */



#define NFDK_DESC_TX_MSS_MASK		GENMASK(13, 0)



#define NFDK_DESC_TX_CHAIN_META		BIT(3)

#define NFDK_DESC_TX_ENCAP		BIT(2)

#define NFDK_DESC_TX_L4_CSUM		BIT(1)

#define NFDK_DESC_TX_L3_CSUM		BIT(0)



#define NFDK_DESC_TX_DMA_LEN_HEAD	GENMASK(11, 0)

#define NFDK_DESC_TX_TYPE_HEAD		GENMASK(15, 12)

#define NFDK_DESC_TX_DMA_LEN		GENMASK(13, 0)

#define NFDK_DESC_TX_TYPE_NOP		0

#define NFDK_DESC_TX_TYPE_GATHER	1

#define NFDK_DESC_TX_TYPE_TSO		2

#define NFDK_DESC_TX_TYPE_SIMPLE	8

#define NFDK_DESC_TX_EOP		BIT(14)



#define NFDK_META_LEN			GENMASK(7, 0)

#define NFDK_META_FIELDS		GENMASK(31, 8)



#define D_BLOCK_CPL(idx)		(NFDK_TX_DESC_BLOCK_CNT -	\

					 (idx) % NFDK_TX_DESC_BLOCK_CNT)



struct nfp_nfdk_tx_desc {

	union {

		struct {

			u8 dma_addr_hi;  /* High bits of host buf address */

			u8 padding;  /* Must be zero */

			__le16 dma_len_type; /* Length to DMA for this desc */

			__le32 dma_addr_lo;  /* Low 32bit of host buf addr */

		};



		struct {

			__le16 mss;	/* MSS to be used for LSO */

			u8 lso_hdrlen;  /* LSO, TCP payload offset */

			u8 lso_totsegs; /* LSO, total segments */

			u8 l3_offset;   /* L3 header offset */

			u8 l4_offset;   /* L4 header offset */

			__le16 lso_meta_res; /* Rsvd bits in TSO metadata */

		};



		struct {

			u8 flags;	/* TX Flags, see @NFDK_DESC_TX_* */

			u8 reserved[7];	/* meta byte placeholder */

		};



		__le32 vals[2];

		__le64 raw;

	};

};



struct nfp_nfdk_tx_buf {

	union {

		/* First slot */

		union {

			struct sk_buff *skb;

			void *frag;

		};



		/* 1 + nr_frags next slots */

		dma_addr_t dma_addr;



		/* TSO (optional) */

		struct {

			u32 pkt_cnt;

			u32 real_len;

		};



		u64 raw;

	};

};



static inline int nfp_nfdk_headlen_to_segs(unsigned int headlen)

{

	/* First descriptor fits less data, so adjust for that */

	return DIV_ROUND_UP(headlen +

			    NFDK_TX_MAX_DATA_PER_DESC -

			    NFDK_TX_MAX_DATA_PER_HEAD,

			    NFDK_TX_MAX_DATA_PER_DESC);

}



int nfp_nfdk_poll(struct napi_struct *napi, int budget);

netdev_tx_t nfp_nfdk_tx(struct sk_buff *skb, struct net_device *netdev);

bool

nfp_nfdk_ctrl_tx_one(struct nfp_net *nn, struct nfp_net_r_vector *r_vec,

		     struct sk_buff *skb, bool old);

void nfp_nfdk_ctrl_poll(struct tasklet_struct *t);

void nfp_nfdk_rx_ring_fill_freelist(struct nfp_net_dp *dp,

				    struct nfp_net_rx_ring *rx_ring);

#endif

drivers/net/ethernet/netronome/nfp/nfdk/rings.c

0 → 100644
+195 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)

/* Copyright (C) 2019 Netronome Systems, Inc. */



#include <linux/seq_file.h>



#include "../nfp_net.h"

#include "../nfp_net_dp.h"

#include "nfdk.h"



static void

nfp_nfdk_tx_ring_reset(struct nfp_net_dp *dp, struct nfp_net_tx_ring *tx_ring)

{

	struct device *dev = dp->dev;

	struct netdev_queue *nd_q;



	while (!tx_ring->is_xdp && tx_ring->rd_p != tx_ring->wr_p) {

		const skb_frag_t *frag, *fend;

		unsigned int size, n_descs = 1;

		struct nfp_nfdk_tx_buf *txbuf;

		int nr_frags, rd_idx;

		struct sk_buff *skb;



		rd_idx = D_IDX(tx_ring, tx_ring->rd_p);

		txbuf = &tx_ring->ktxbufs[rd_idx];



		skb = txbuf->skb;

		if (!skb) {

			n_descs = D_BLOCK_CPL(tx_ring->rd_p);

			goto next;

		}



		nr_frags = skb_shinfo(skb)->nr_frags;

		txbuf++;



		/* Unmap head */

		size = skb_headlen(skb);

		dma_unmap_single(dev, txbuf->dma_addr, size, DMA_TO_DEVICE);

		n_descs += nfp_nfdk_headlen_to_segs(size);

		txbuf++;



		frag = skb_shinfo(skb)->frags;

		fend = frag + nr_frags;

		for (; frag < fend; frag++) {

			size = skb_frag_size(frag);

			dma_unmap_page(dev, txbuf->dma_addr,

				       skb_frag_size(frag), DMA_TO_DEVICE);

			n_descs += DIV_ROUND_UP(size,

						NFDK_TX_MAX_DATA_PER_DESC);

			txbuf++;

		}



		if (skb_is_gso(skb))

			n_descs++;



		dev_kfree_skb_any(skb);

next:

		tx_ring->rd_p += n_descs;

	}



	memset(tx_ring->txds, 0, tx_ring->size);

	tx_ring->data_pending = 0;

	tx_ring->wr_p = 0;

	tx_ring->rd_p = 0;

	tx_ring->qcp_rd_p = 0;

	tx_ring->wr_ptr_add = 0;



	if (tx_ring->is_xdp || !dp->netdev)

		return;



	nd_q = netdev_get_tx_queue(dp->netdev, tx_ring->idx);

	netdev_tx_reset_queue(nd_q);

}



static void nfp_nfdk_tx_ring_free(struct nfp_net_tx_ring *tx_ring)

{

	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;

	struct nfp_net_dp *dp = &r_vec->nfp_net->dp;



	kvfree(tx_ring->ktxbufs);



	if (tx_ring->ktxds)

		dma_free_coherent(dp->dev, tx_ring->size,

				  tx_ring->ktxds, tx_ring->dma);



	tx_ring->cnt = 0;

	tx_ring->txbufs = NULL;

	tx_ring->txds = NULL;

	tx_ring->dma = 0;

	tx_ring->size = 0;

}



static int

nfp_nfdk_tx_ring_alloc(struct nfp_net_dp *dp, struct nfp_net_tx_ring *tx_ring)

{

	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;



	tx_ring->cnt = dp->txd_cnt * NFDK_TX_DESC_PER_SIMPLE_PKT;

	tx_ring->size = array_size(tx_ring->cnt, sizeof(*tx_ring->ktxds));

	tx_ring->ktxds = dma_alloc_coherent(dp->dev, tx_ring->size,

					    &tx_ring->dma,

					    GFP_KERNEL | __GFP_NOWARN);

	if (!tx_ring->ktxds) {

		netdev_warn(dp->netdev, "failed to allocate TX descriptor ring memory, requested descriptor count: %d, consider lowering descriptor count\n",

			    tx_ring->cnt);

		goto err_alloc;

	}



	tx_ring->ktxbufs = kvcalloc(tx_ring->cnt, sizeof(*tx_ring->ktxbufs),

				    GFP_KERNEL);

	if (!tx_ring->ktxbufs)

		goto err_alloc;



	if (!tx_ring->is_xdp && dp->netdev)

		netif_set_xps_queue(dp->netdev, &r_vec->affinity_mask,

				    tx_ring->idx);



	return 0;



err_alloc:

	nfp_nfdk_tx_ring_free(tx_ring);

	return -ENOMEM;

}



static void

nfp_nfdk_tx_ring_bufs_free(struct nfp_net_dp *dp,

			   struct nfp_net_tx_ring *tx_ring)

{

}



static int

nfp_nfdk_tx_ring_bufs_alloc(struct nfp_net_dp *dp,

			    struct nfp_net_tx_ring *tx_ring)

{

	return 0;

}



static void

nfp_nfdk_print_tx_descs(struct seq_file *file,

			struct nfp_net_r_vector *r_vec,

			struct nfp_net_tx_ring *tx_ring,

			u32 d_rd_p, u32 d_wr_p)

{

	struct nfp_nfdk_tx_desc *txd;

	u32 txd_cnt = tx_ring->cnt;

	int i;



	for (i = 0; i < txd_cnt; i++) {

		txd = &tx_ring->ktxds[i];



		seq_printf(file, "%04d: 0x%08x 0x%08x 0x%016llx", i,

			   txd->vals[0], txd->vals[1], tx_ring->ktxbufs[i].raw);



		if (i == tx_ring->rd_p % txd_cnt)

			seq_puts(file, " H_RD");

		if (i == tx_ring->wr_p % txd_cnt)

			seq_puts(file, " H_WR");

		if (i == d_rd_p % txd_cnt)

			seq_puts(file, " D_RD");

		if (i == d_wr_p % txd_cnt)

			seq_puts(file, " D_WR");



		seq_putc(file, '\n');

	}

}



#define NFP_NFDK_CFG_CTRL_SUPPORTED					\

	(NFP_NET_CFG_CTRL_ENABLE | NFP_NET_CFG_CTRL_PROMISC |		\

	 NFP_NET_CFG_CTRL_L2BC | NFP_NET_CFG_CTRL_L2MC |		\

	 NFP_NET_CFG_CTRL_RXCSUM | NFP_NET_CFG_CTRL_TXCSUM |		\

	 NFP_NET_CFG_CTRL_RXVLAN |					\

	 NFP_NET_CFG_CTRL_GATHER | NFP_NET_CFG_CTRL_LSO |		\

	 NFP_NET_CFG_CTRL_CTAG_FILTER | NFP_NET_CFG_CTRL_CMSG_DATA |	\

	 NFP_NET_CFG_CTRL_RINGCFG | NFP_NET_CFG_CTRL_IRQMOD |		\

	 NFP_NET_CFG_CTRL_TXRWB |					\

	 NFP_NET_CFG_CTRL_VXLAN | NFP_NET_CFG_CTRL_NVGRE |		\

	 NFP_NET_CFG_CTRL_BPF | NFP_NET_CFG_CTRL_LSO2 |			\

	 NFP_NET_CFG_CTRL_RSS2 | NFP_NET_CFG_CTRL_CSUM_COMPLETE |	\

	 NFP_NET_CFG_CTRL_LIVE_ADDR)



const struct nfp_dp_ops nfp_nfdk_ops = {

	.version		= NFP_NFD_VER_NFDK,

	.tx_min_desc_per_pkt	= NFDK_TX_DESC_PER_SIMPLE_PKT,

	.cap_mask		= NFP_NFDK_CFG_CTRL_SUPPORTED,

	.poll			= nfp_nfdk_poll,

	.ctrl_poll		= nfp_nfdk_ctrl_poll,

	.xmit			= nfp_nfdk_tx,

	.ctrl_tx_one		= nfp_nfdk_ctrl_tx_one,

	.rx_ring_fill_freelist	= nfp_nfdk_rx_ring_fill_freelist,

	.tx_ring_alloc		= nfp_nfdk_tx_ring_alloc,

	.tx_ring_reset		= nfp_nfdk_tx_ring_reset,

	.tx_ring_free		= nfp_nfdk_tx_ring_free,

	.tx_ring_bufs_alloc	= nfp_nfdk_tx_ring_bufs_alloc,

	.tx_ring_bufs_free	= nfp_nfdk_tx_ring_bufs_free,

	.print_tx_descs		= nfp_nfdk_print_tx_descs

};

drivers/net/ethernet/netronome/nfp/nfp_net.h

+21 −6
Original line number Diff line number Diff line
@@ -108,6 +108,9 @@ struct xsk_buff_pool; 
struct nfp_nfd3_tx_desc;

struct nfp_nfd3_tx_buf;



struct nfp_nfdk_tx_desc;

struct nfp_nfdk_tx_buf;



/* Convenience macro for wrapping descriptor index on ring size */

#define D_IDX(ring, idx)	((idx) & ((ring)->cnt - 1))
@@ -125,6 +128,7 @@ struct nfp_nfd3_tx_buf; 
 * struct nfp_net_tx_ring - TX ring structure

 * @r_vec:      Back pointer to ring vector structure

 * @idx:        Ring index from Linux's perspective

 * @data_pending: number of bytes added to current block (NFDK only)

 * @qcp_q:      Pointer to base of the QCP TX queue

 * @txrwb:	TX pointer write back area

 * @cnt:        Size of the queue in number of descriptors
@@ -133,8 +137,10 @@ struct nfp_nfd3_tx_buf; 
 * @qcp_rd_p:   Local copy of QCP TX queue read pointer

 * @wr_ptr_add:	Accumulated number of buffers to add to QCP write pointer

 *		(used for .xmit_more delayed kick)

 * @txbufs:     Array of transmitted TX buffers, to free on transmit

 * @txds:       Virtual address of TX ring in host memory

 * @txbufs:	Array of transmitted TX buffers, to free on transmit (NFD3)

 * @ktxbufs:	Array of transmitted TX buffers, to free on transmit (NFDK)

 * @txds:	Virtual address of TX ring in host memory (NFD3)

 * @ktxds:	Virtual address of TX ring in host memory (NFDK)

 *

 * @qcidx:      Queue Controller Peripheral (QCP) queue index for the TX queue

 * @dma:        DMA address of the TX ring
@@ -144,7 +150,8 @@ struct nfp_nfd3_tx_buf; 
struct nfp_net_tx_ring {

	struct nfp_net_r_vector *r_vec;



	u32 idx;

	u16 idx;

	u16 data_pending;

	u8 __iomem *qcp_q;

	u64 *txrwb;
@@ -155,8 +162,14 @@ struct nfp_net_tx_ring { 


	u32 wr_ptr_add;



	union {

		struct nfp_nfd3_tx_buf *txbufs;

		struct nfp_nfdk_tx_buf *ktxbufs;

	};

	union {

		struct nfp_nfd3_tx_desc *txds;

		struct nfp_nfdk_tx_desc *ktxds;

	};



	/* Cold data follows */

	int qcidx;
@@ -860,10 +873,12 @@ static inline void nn_ctrl_bar_unlock(struct nfp_net *nn) 
extern const char nfp_driver_version[];



extern const struct net_device_ops nfp_nfd3_netdev_ops;

extern const struct net_device_ops nfp_nfdk_netdev_ops;



static inline bool nfp_netdev_is_nfp_net(struct net_device *netdev)

{

	return netdev->netdev_ops == &nfp_nfd3_netdev_ops;

	return netdev->netdev_ops == &nfp_nfd3_netdev_ops ||

	       netdev->netdev_ops == &nfp_nfdk_netdev_ops;

}



static inline int nfp_net_coalesce_para_check(u32 usecs, u32 pkts)