Commit 0d14657f authored by Taehee Yoo's avatar Taehee Yoo Committed by David S. Miller
Browse files

net: atlantic: Implement xdp control plane 



aq_xdp() is a xdp setup callback function for Atlantic driver.
When XDP is attached or detached, the device will be restarted because
it uses different headroom, tailroom, and page order value.

If XDP enabled, it switches default page order value from 0 to 2.
Because the default maximum frame size is still 2K and it needs
additional area for headroom and tailroom.
The total size(headroom + frame size + tailroom) is 2624.
So, 1472Bytes will be always wasted for every frame.
But when order-2 is used, these pages can be used 6 times
with flip strategy.
It means only about 106Bytes per frame will be wasted.

Also, It supports xdp fragment feature.
MTU can be 16K if xdp prog supports xdp fragment.
If not, MTU can not exceed 2K - ETH_HLEN - ETH_FCS.

And a static key is added and It will be used to call the xdp_clean
handler in ->poll(). data plane implementation will be contained
the followed patch.

Signed-off-by: default avatarTaehee Yoo <ap420073@gmail.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 8ab38ed7

drivers/net/ethernet/aquantia/atlantic/aq_cfg.h

+1 −0
Original line number Diff line number Diff line
@@ -40,6 +40,7 @@ 
#define AQ_CFG_RX_HDR_SIZE 256U



#define AQ_CFG_RX_PAGEORDER 0U

#define AQ_CFG_XDP_PAGEORDER 2U



/* LRO */

#define AQ_CFG_IS_LRO_DEF           1U

drivers/net/ethernet/aquantia/atlantic/aq_main.c

+86 −0
Original line number Diff line number Diff line
@@ -14,17 +14,22 @@ 
#include "aq_ptp.h"

#include "aq_filters.h"

#include "aq_hw_utils.h"

#include "aq_vec.h"



#include <linux/netdevice.h>

#include <linux/module.h>

#include <linux/ip.h>

#include <linux/udp.h>

#include <net/pkt_cls.h>

#include <linux/filter.h>



MODULE_LICENSE("GPL v2");

MODULE_AUTHOR(AQ_CFG_DRV_AUTHOR);

MODULE_DESCRIPTION(AQ_CFG_DRV_DESC);



DEFINE_STATIC_KEY_FALSE(aq_xdp_locking_key);

EXPORT_SYMBOL(aq_xdp_locking_key);



static const char aq_ndev_driver_name[] = AQ_CFG_DRV_NAME;



static const struct net_device_ops aq_ndev_ops;
@@ -126,9 +131,19 @@ static netdev_tx_t aq_ndev_start_xmit(struct sk_buff *skb, struct net_device *nd 


static int aq_ndev_change_mtu(struct net_device *ndev, int new_mtu)

{

	int new_frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN;

	struct aq_nic_s *aq_nic = netdev_priv(ndev);

	struct bpf_prog *prog;

	int err;



	prog = READ_ONCE(aq_nic->xdp_prog);

	if (prog && !prog->aux->xdp_has_frags &&

	    new_frame_size > AQ_CFG_RX_FRAME_MAX) {

		netdev_err(ndev, "Illegal MTU %d for XDP prog without frags\n",

			   ndev->mtu);

		return -EOPNOTSUPP;

	}



	err = aq_nic_set_mtu(aq_nic, new_mtu + ETH_HLEN);



	if (err < 0)
@@ -204,6 +219,25 @@ static int aq_ndev_set_features(struct net_device *ndev, 
	return err;

}



static netdev_features_t aq_ndev_fix_features(struct net_device *ndev,

					      netdev_features_t features)

{

	struct aq_nic_s *aq_nic = netdev_priv(ndev);

	struct bpf_prog *prog;



	if (!(features & NETIF_F_RXCSUM))

		features &= ~NETIF_F_LRO;



	prog = READ_ONCE(aq_nic->xdp_prog);

	if (prog && !prog->aux->xdp_has_frags &&

	    aq_nic->xdp_prog && features & NETIF_F_LRO) {

		netdev_err(ndev, "LRO is not supported with single buffer XDP, disabling\n");

		features &= ~NETIF_F_LRO;

	}



	return features;

}



static int aq_ndev_set_mac_address(struct net_device *ndev, void *addr)

{

	struct aq_nic_s *aq_nic = netdev_priv(ndev);
@@ -410,6 +444,56 @@ static int aq_ndo_setup_tc(struct net_device *dev, enum tc_setup_type type, 
				      mqprio->qopt.prio_tc_map);

}



static int aq_xdp_setup(struct net_device *ndev, struct bpf_prog *prog,

			struct netlink_ext_ack *extack)

{

	bool need_update, running = netif_running(ndev);

	struct aq_nic_s *aq_nic = netdev_priv(ndev);

	struct bpf_prog *old_prog;



	if (prog && !prog->aux->xdp_has_frags) {

		if (ndev->mtu > AQ_CFG_RX_FRAME_MAX) {

			NL_SET_ERR_MSG_MOD(extack,

					   "prog does not support XDP frags");

			return -EOPNOTSUPP;

		}



		if (prog && ndev->features & NETIF_F_LRO) {

			netdev_err(ndev,

				   "LRO is not supported with single buffer XDP, disabling\n");

			ndev->features &= ~NETIF_F_LRO;

		}

	}



	need_update = !!aq_nic->xdp_prog != !!prog;

	if (running && need_update)

		aq_ndev_close(ndev);



	old_prog = xchg(&aq_nic->xdp_prog, prog);

	if (old_prog)

		bpf_prog_put(old_prog);



	if (!old_prog && prog)

		static_branch_inc(&aq_xdp_locking_key);

	else if (old_prog && !prog)

		static_branch_dec(&aq_xdp_locking_key);



	if (running && need_update)

		return aq_ndev_open(ndev);



	return 0;

}



static int aq_xdp(struct net_device *dev, struct netdev_bpf *xdp)

{

	switch (xdp->command) {

	case XDP_SETUP_PROG:

		return aq_xdp_setup(dev, xdp->prog, xdp->extack);

	default:

		return -EINVAL;

	}

}



static const struct net_device_ops aq_ndev_ops = {

	.ndo_open = aq_ndev_open,

	.ndo_stop = aq_ndev_close,
@@ -418,10 +502,12 @@ static const struct net_device_ops aq_ndev_ops = { 
	.ndo_change_mtu = aq_ndev_change_mtu,

	.ndo_set_mac_address = aq_ndev_set_mac_address,

	.ndo_set_features = aq_ndev_set_features,

	.ndo_fix_features = aq_ndev_fix_features,

	.ndo_eth_ioctl = aq_ndev_ioctl,

	.ndo_vlan_rx_add_vid = aq_ndo_vlan_rx_add_vid,

	.ndo_vlan_rx_kill_vid = aq_ndo_vlan_rx_kill_vid,

	.ndo_setup_tc = aq_ndo_setup_tc,

	.ndo_bpf = aq_xdp,

};



static int __init aq_ndev_init_module(void)

drivers/net/ethernet/aquantia/atlantic/aq_main.h

+2 −0
Original line number Diff line number Diff line
@@ -12,6 +12,8 @@ 
#include "aq_common.h"

#include "aq_nic.h"



DECLARE_STATIC_KEY_FALSE(aq_xdp_locking_key);



void aq_ndev_schedule_work(struct work_struct *work);

struct net_device *aq_ndev_alloc(void);

drivers/net/ethernet/aquantia/atlantic/aq_nic.h

+3 −0
Original line number Diff line number Diff line
@@ -11,6 +11,8 @@ 
#define AQ_NIC_H



#include <linux/ethtool.h>

#include <net/xdp.h>

#include <linux/bpf.h>



#include "aq_common.h"

#include "aq_rss.h"
@@ -128,6 +130,7 @@ struct aq_nic_s { 
	struct aq_vec_s *aq_vec[AQ_CFG_VECS_MAX];

	struct aq_ring_s *aq_ring_tx[AQ_HW_QUEUES_MAX];

	struct aq_hw_s *aq_hw;

	struct bpf_prog *xdp_prog;

	struct net_device *ndev;

	unsigned int aq_vecs;

	unsigned int packet_filter;

drivers/net/ethernet/aquantia/atlantic/aq_ring.c

+42 −22
Original line number Diff line number Diff line
@@ -7,12 +7,16 @@ 


/* File aq_ring.c: Definition of functions for Rx/Tx rings. */



#include "aq_ring.h"

#include "aq_nic.h"

#include "aq_hw.h"

#include "aq_hw_utils.h"

#include "aq_ptp.h"

#include "aq_vec.h"

#include "aq_main.h"



#include <net/xdp.h>

#include <linux/filter.h>

#include <linux/bpf_trace.h>

#include <linux/netdevice.h>

#include <linux/etherdevice.h>
@@ -27,9 +31,10 @@ static inline void aq_free_rxpage(struct aq_rxpage *rxpage, struct device *dev) 
	rxpage->page = NULL;

}



static int aq_get_rxpage(struct aq_rxpage *rxpage, unsigned int order,

			 struct device *dev)

static int aq_alloc_rxpages(struct aq_rxpage *rxpage, struct aq_ring_s *rx_ring)

{

	struct device *dev = aq_nic_get_dev(rx_ring->aq_nic);

	unsigned int order = rx_ring->page_order;

	struct page *page;

	int ret = -ENOMEM;

	dma_addr_t daddr;
@@ -47,7 +52,7 @@ static int aq_get_rxpage(struct aq_rxpage *rxpage, unsigned int order, 
	rxpage->page = page;

	rxpage->daddr = daddr;

	rxpage->order = order;

	rxpage->pg_off = 0;

	rxpage->pg_off = rx_ring->page_offset;



	return 0;
@@ -58,21 +63,26 @@ static int aq_get_rxpage(struct aq_rxpage *rxpage, unsigned int order, 
	return ret;

}



static int aq_get_rxpages(struct aq_ring_s *self, struct aq_ring_buff_s *rxbuf,

			  int order)

static int aq_get_rxpages(struct aq_ring_s *self, struct aq_ring_buff_s *rxbuf)

{

	unsigned int order = self->page_order;

	u16 page_offset = self->page_offset;

	u16 frame_max = self->frame_max;

	u16 tail_size = self->tail_size;

	int ret;



	if (rxbuf->rxdata.page) {

		/* One means ring is the only user and can reuse */

		if (page_ref_count(rxbuf->rxdata.page) > 1) {

			/* Try reuse buffer */

			rxbuf->rxdata.pg_off += AQ_CFG_RX_FRAME_MAX;

			if (rxbuf->rxdata.pg_off + AQ_CFG_RX_FRAME_MAX <=

			rxbuf->rxdata.pg_off += frame_max + page_offset +

						tail_size;

			if (rxbuf->rxdata.pg_off + frame_max + tail_size <=

			    (PAGE_SIZE << order)) {

				u64_stats_update_begin(&self->stats.rx.syncp);

				self->stats.rx.pg_flips++;

				u64_stats_update_end(&self->stats.rx.syncp);



			} else {

				/* Buffer exhausted. We have other users and

				 * should release this page and realloc
@@ -84,7 +94,7 @@ static int aq_get_rxpages(struct aq_ring_s *self, struct aq_ring_buff_s *rxbuf, 
				u64_stats_update_end(&self->stats.rx.syncp);

			}

		} else {

			rxbuf->rxdata.pg_off = 0;

			rxbuf->rxdata.pg_off = page_offset;

			u64_stats_update_begin(&self->stats.rx.syncp);

			self->stats.rx.pg_reuses++;

			u64_stats_update_end(&self->stats.rx.syncp);
@@ -92,8 +102,7 @@ static int aq_get_rxpages(struct aq_ring_s *self, struct aq_ring_buff_s *rxbuf, 
	}



	if (!rxbuf->rxdata.page) {

		ret = aq_get_rxpage(&rxbuf->rxdata, order,

				    aq_nic_get_dev(self->aq_nic));

		ret = aq_alloc_rxpages(&rxbuf->rxdata, self);

		if (ret) {

			u64_stats_update_begin(&self->stats.rx.syncp);

			self->stats.rx.alloc_fails++;
@@ -117,6 +126,7 @@ static struct aq_ring_s *aq_ring_alloc(struct aq_ring_s *self, 
		err = -ENOMEM;

		goto err_exit;

	}



	self->dx_ring = dma_alloc_coherent(aq_nic_get_dev(aq_nic),

					   self->size * self->dx_size,

					   &self->dx_ring_pa, GFP_KERNEL);
@@ -172,11 +182,22 @@ struct aq_ring_s *aq_ring_rx_alloc(struct aq_ring_s *self, 
	self->idx = idx;

	self->size = aq_nic_cfg->rxds;

	self->dx_size = aq_nic_cfg->aq_hw_caps->rxd_size;

	self->page_order = fls(AQ_CFG_RX_FRAME_MAX / PAGE_SIZE +

			       (AQ_CFG_RX_FRAME_MAX % PAGE_SIZE ? 1 : 0)) - 1;



	self->xdp_prog = aq_nic->xdp_prog;

	self->frame_max = AQ_CFG_RX_FRAME_MAX;



	/* Only order-2 is allowed if XDP is enabled */

	if (READ_ONCE(self->xdp_prog)) {

		self->page_offset = AQ_XDP_HEADROOM;

		self->page_order = AQ_CFG_XDP_PAGEORDER;

		self->tail_size = AQ_XDP_TAILROOM;

	} else {

		self->page_offset = 0;

		self->page_order = fls(self->frame_max / PAGE_SIZE +

				       (self->frame_max % PAGE_SIZE ? 1 : 0)) - 1;

		if (aq_nic_cfg->rxpageorder > self->page_order)

			self->page_order = aq_nic_cfg->rxpageorder;

		self->tail_size = 0;

	}



	self = aq_ring_alloc(self, aq_nic);

	if (!self) {
@@ -449,7 +470,7 @@ int aq_ring_rx_clean(struct aq_ring_s *self, 
			skb_add_rx_frag(skb, 0, buff->rxdata.page,

					buff->rxdata.pg_off + hdr_len,

					buff->len - hdr_len,

					AQ_CFG_RX_FRAME_MAX);

					self->frame_max);

			page_ref_inc(buff->rxdata.page);

		}
@@ -469,7 +490,7 @@ int aq_ring_rx_clean(struct aq_ring_s *self, 
						buff_->rxdata.page,

						buff_->rxdata.pg_off,

						buff_->len,

						AQ_CFG_RX_FRAME_MAX);

						self->frame_max);

				page_ref_inc(buff_->rxdata.page);

				buff_->is_cleaned = 1;
@@ -529,7 +550,6 @@ void aq_ring_hwts_rx_clean(struct aq_ring_s *self, struct aq_nic_s *aq_nic) 


int aq_ring_rx_fill(struct aq_ring_s *self)

{

	unsigned int page_order = self->page_order;

	struct aq_ring_buff_s *buff = NULL;

	int err = 0;

	int i = 0;
@@ -543,9 +563,9 @@ int aq_ring_rx_fill(struct aq_ring_s *self) 
		buff = &self->buff_ring[self->sw_tail];



		buff->flags = 0U;

		buff->len = AQ_CFG_RX_FRAME_MAX;

		buff->len = self->frame_max;



		err = aq_get_rxpages(self, buff, page_order);

		err = aq_get_rxpages(self, buff);

		if (err)

			goto err_exit;