Commit 65b28c81 authored by Gerhard Engleder's avatar Gerhard Engleder Committed by David S. Miller
Browse files

tsnep: Add XDP RX support 



If BPF program is set up, then run BPF program for every received frame
and execute the selected action.

Signed-off-by: default avatarGerhard Engleder <gerhard@engleder-embedded.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent e77832ab

drivers/net/ethernet/engleder/tsnep.h

+3 −0
Original line number Diff line number Diff line
@@ -109,6 +109,7 @@ struct tsnep_rx { 
	struct tsnep_adapter *adapter;

	void __iomem *addr;

	int queue_index;

	int tx_queue_index;



	void *page[TSNEP_RING_PAGE_COUNT];

	dma_addr_t page_dma[TSNEP_RING_PAGE_COUNT];
@@ -176,6 +177,8 @@ struct tsnep_adapter { 
	int rxnfc_count;

	int rxnfc_max;



	struct bpf_prog *xdp_prog;



	int num_tx_queues;

	struct tsnep_tx tx[TSNEP_MAX_QUEUES];

	int num_rx_queues;

drivers/net/ethernet/engleder/tsnep_main.c

+129 −2
Original line number Diff line number Diff line
@@ -27,6 +27,7 @@ 
#include <linux/phy.h>

#include <linux/iopoll.h>

#include <linux/bpf.h>

#include <linux/bpf_trace.h>



#define TSNEP_RX_OFFSET (max(NET_SKB_PAD, XDP_PACKET_HEADROOM) + NET_IP_ALIGN)

#define TSNEP_HEADROOM ALIGN(TSNEP_RX_OFFSET, 4)
@@ -49,6 +50,9 @@ 
#define TSNEP_TX_TYPE_XDP_TX	BIT(2)

#define TSNEP_TX_TYPE_XDP_NDO	BIT(3)



#define TSNEP_XDP_TX		BIT(0)

#define TSNEP_XDP_REDIRECT	BIT(1)



static void tsnep_enable_irq(struct tsnep_adapter *adapter, u32 mask)

{

	iowrite32(mask, adapter->addr + ECM_INT_ENABLE);
@@ -607,6 +611,29 @@ static void tsnep_xdp_xmit_flush(struct tsnep_tx *tx) 
	iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL);

}



static bool tsnep_xdp_xmit_back(struct tsnep_adapter *adapter,

				struct xdp_buff *xdp,

				struct netdev_queue *tx_nq, struct tsnep_tx *tx)

{

	struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);

	bool xmit;



	if (unlikely(!xdpf))

		return false;



	__netif_tx_lock(tx_nq, smp_processor_id());



	xmit = tsnep_xdp_xmit_frame_ring(xdpf, tx, TSNEP_TX_TYPE_XDP_TX);



	/* Avoid transmit queue timeout since we share it with the slow path */

	if (xmit)

		txq_trans_cond_update(tx_nq);



	__netif_tx_unlock(tx_nq);



	return xmit;

}



static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget)

{

	struct tsnep_tx_entry *entry;
@@ -938,6 +965,62 @@ static int tsnep_rx_refill(struct tsnep_rx *rx, int count, bool reuse) 
	return i;

}



static bool tsnep_xdp_run_prog(struct tsnep_rx *rx, struct bpf_prog *prog,

			       struct xdp_buff *xdp, int *status,

			       struct netdev_queue *tx_nq, struct tsnep_tx *tx)

{

	unsigned int length;

	unsigned int sync;

	u32 act;



	length = xdp->data_end - xdp->data_hard_start - XDP_PACKET_HEADROOM;



	act = bpf_prog_run_xdp(prog, xdp);



	/* Due xdp_adjust_tail: DMA sync for_device cover max len CPU touch */

	sync = xdp->data_end - xdp->data_hard_start - XDP_PACKET_HEADROOM;

	sync = max(sync, length);



	switch (act) {

	case XDP_PASS:

		return false;

	case XDP_TX:

		if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx))

			goto out_failure;

		*status |= TSNEP_XDP_TX;

		return true;

	case XDP_REDIRECT:

		if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0)

			goto out_failure;

		*status |= TSNEP_XDP_REDIRECT;

		return true;

	default:

		bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act);

		fallthrough;

	case XDP_ABORTED:

out_failure:

		trace_xdp_exception(rx->adapter->netdev, prog, act);

		fallthrough;

	case XDP_DROP:

		page_pool_put_page(rx->page_pool, virt_to_head_page(xdp->data),

				   sync, true);

		return true;

	}

}



static void tsnep_finalize_xdp(struct tsnep_adapter *adapter, int status,

			       struct netdev_queue *tx_nq, struct tsnep_tx *tx)

{

	if (status & TSNEP_XDP_TX) {

		__netif_tx_lock(tx_nq, smp_processor_id());

		tsnep_xdp_xmit_flush(tx);

		__netif_tx_unlock(tx_nq);

	}



	if (status & TSNEP_XDP_REDIRECT)

		xdp_do_flush();

}



static struct sk_buff *tsnep_build_skb(struct tsnep_rx *rx, struct page *page,

				       int length)

{
@@ -973,15 +1056,28 @@ static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi, 
			 int budget)

{

	struct device *dmadev = rx->adapter->dmadev;

	int desc_available;

	int done = 0;

	enum dma_data_direction dma_dir;

	struct tsnep_rx_entry *entry;

	struct netdev_queue *tx_nq;

	struct bpf_prog *prog;

	struct xdp_buff xdp;

	struct sk_buff *skb;

	struct tsnep_tx *tx;

	int desc_available;

	int xdp_status = 0;

	int done = 0;

	int length;



	desc_available = tsnep_rx_desc_available(rx);

	dma_dir = page_pool_get_dma_dir(rx->page_pool);

	prog = READ_ONCE(rx->adapter->xdp_prog);

	if (prog) {

		tx_nq = netdev_get_tx_queue(rx->adapter->netdev,

					    rx->tx_queue_index);

		tx = &rx->adapter->tx[rx->tx_queue_index];



		xdp_init_buff(&xdp, PAGE_SIZE, &rx->xdp_rxq);

	}



	while (likely(done < budget) && (rx->read != rx->write)) {

		entry = &rx->entry[rx->read];
@@ -1031,6 +1127,25 @@ static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi, 
		rx->read = (rx->read + 1) % TSNEP_RING_SIZE;

		desc_available++;



		if (prog) {

			bool consume;



			xdp_prepare_buff(&xdp, page_address(entry->page),

					 XDP_PACKET_HEADROOM + TSNEP_RX_INLINE_METADATA_SIZE,

					 length, false);



			consume = tsnep_xdp_run_prog(rx, prog, &xdp,

						     &xdp_status, tx_nq, tx);

			if (consume) {

				rx->packets++;

				rx->bytes += length;



				entry->page = NULL;



				continue;

			}

		}



		skb = tsnep_build_skb(rx, entry->page, length);

		if (skb) {

			page_pool_release_page(rx->page_pool, entry->page);
@@ -1049,6 +1164,9 @@ static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi, 
		entry->page = NULL;

	}



	if (xdp_status)

		tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx);



	if (desc_available)

		tsnep_rx_refill(rx, desc_available, false);
@@ -1221,6 +1339,7 @@ static int tsnep_queue_open(struct tsnep_adapter *adapter, 
			    struct tsnep_queue *queue, bool first)

{

	struct tsnep_rx *rx = queue->rx;

	struct tsnep_tx *tx = queue->tx;

	int retval;



	queue->adapter = adapter;
@@ -1228,6 +1347,14 @@ static int tsnep_queue_open(struct tsnep_adapter *adapter, 
	netif_napi_add(adapter->netdev, &queue->napi, tsnep_poll);



	if (rx) {

		/* choose TX queue for XDP_TX */

		if (tx)

			rx->tx_queue_index = tx->queue_index;

		else if (rx->queue_index < adapter->num_tx_queues)

			rx->tx_queue_index = rx->queue_index;

		else

			rx->tx_queue_index = 0;



		retval = xdp_rxq_info_reg(&rx->xdp_rxq, adapter->netdev,

					  rx->queue_index, queue->napi.napi_id);

		if (retval)