Commit e9229ffd authored by Felix Fietkau's avatar Felix Fietkau Committed by David S. Miller
Browse files

net: ethernet: mtk_eth_soc: implement dynamic interrupt moderation 



Reduces the number of interrupts under load

Signed-off-by: default avatarFelix Fietkau <nbd@nbd.name>
[Ilya: add documentation for new struct fields]
Signed-off-by: default avatarIlya Lipnitskiy <ilya.lipnitskiy@gmail.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 6b4423b2

drivers/net/ethernet/mediatek/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ config NET_MEDIATEK_SOC 
	tristate "MediaTek SoC Gigabit Ethernet support"

	depends on NET_DSA || !NET_DSA

	select PHYLINK

	select DIMLIB

	help

	  This driver supports the gigabit ethernet MACs in the

	  MediaTek SoC family.

drivers/net/ethernet/mediatek/mtk_eth_soc.c

+89 −7
Original line number Diff line number Diff line
@@ -1233,12 +1233,13 @@ static void mtk_update_rx_cpu_idx(struct mtk_eth *eth) 
static int mtk_poll_rx(struct napi_struct *napi, int budget,

		       struct mtk_eth *eth)

{

	struct dim_sample dim_sample = {};

	struct mtk_rx_ring *ring;

	int idx;

	struct sk_buff *skb;

	u8 *data, *new_data;

	struct mtk_rx_dma *rxd, trxd;

	int done = 0;

	int done = 0, bytes = 0;



	while (done < budget) {

		struct net_device *netdev;
@@ -1312,6 +1313,7 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, 
		else

			skb_checksum_none_assert(skb);

		skb->protocol = eth_type_trans(skb, netdev);

		bytes += pktlen;



		if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX &&

		    (trxd.rxd2 & RX_DMA_VTAG))
@@ -1344,6 +1346,12 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, 
		mtk_update_rx_cpu_idx(eth);

	}



	eth->rx_packets += done;

	eth->rx_bytes += bytes;

	dim_update_sample(eth->rx_events, eth->rx_packets, eth->rx_bytes,

			  &dim_sample);

	net_dim(&eth->rx_dim, dim_sample);



	return done;

}
@@ -1436,6 +1444,7 @@ static int mtk_poll_tx_pdma(struct mtk_eth *eth, int budget, 
static int mtk_poll_tx(struct mtk_eth *eth, int budget)

{

	struct mtk_tx_ring *ring = &eth->tx_ring;

	struct dim_sample dim_sample = {};

	unsigned int done[MTK_MAX_DEVS];

	unsigned int bytes[MTK_MAX_DEVS];

	int total = 0, i;
@@ -1453,8 +1462,14 @@ static int mtk_poll_tx(struct mtk_eth *eth, int budget) 
			continue;

		netdev_completed_queue(eth->netdev[i], done[i], bytes[i]);

		total += done[i];

		eth->tx_packets += done[i];

		eth->tx_bytes += bytes[i];

	}



	dim_update_sample(eth->tx_events, eth->tx_packets, eth->tx_bytes,

			  &dim_sample);

	net_dim(&eth->tx_dim, dim_sample);



	if (mtk_queue_stopped(eth) &&

	    (atomic_read(&ring->free_count) > ring->thresh))

		mtk_wake_queue(eth);
@@ -2129,6 +2144,7 @@ static irqreturn_t mtk_handle_irq_rx(int irq, void *_eth) 
{

	struct mtk_eth *eth = _eth;



	eth->rx_events++;

	if (likely(napi_schedule_prep(&eth->rx_napi))) {

		__napi_schedule(&eth->rx_napi);

		mtk_rx_irq_disable(eth, MTK_RX_DONE_INT);
@@ -2141,6 +2157,7 @@ static irqreturn_t mtk_handle_irq_tx(int irq, void *_eth) 
{

	struct mtk_eth *eth = _eth;



	eth->tx_events++;

	if (likely(napi_schedule_prep(&eth->tx_napi))) {

		__napi_schedule(&eth->tx_napi);

		mtk_tx_irq_disable(eth, MTK_TX_DONE_INT);
@@ -2325,6 +2342,9 @@ static int mtk_stop(struct net_device *dev) 
	napi_disable(&eth->tx_napi);

	napi_disable(&eth->rx_napi);



	cancel_work_sync(&eth->rx_dim.work);

	cancel_work_sync(&eth->tx_dim.work);



	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))

		mtk_stop_dma(eth, MTK_QDMA_GLO_CFG);

	mtk_stop_dma(eth, MTK_PDMA_GLO_CFG);
@@ -2377,6 +2397,64 @@ static int mtk_clk_enable(struct mtk_eth *eth) 
	return ret;

}



static void mtk_dim_rx(struct work_struct *work)

{

	struct dim *dim = container_of(work, struct dim, work);

	struct mtk_eth *eth = container_of(dim, struct mtk_eth, rx_dim);

	struct dim_cq_moder cur_profile;

	u32 val, cur;



	cur_profile = net_dim_get_rx_moderation(eth->rx_dim.mode,

						dim->profile_ix);

	spin_lock_bh(&eth->dim_lock);



	val = mtk_r32(eth, MTK_PDMA_DELAY_INT);

	val &= MTK_PDMA_DELAY_TX_MASK;

	val |= MTK_PDMA_DELAY_RX_EN;



	cur = min_t(u32, DIV_ROUND_UP(cur_profile.usec, 20), MTK_PDMA_DELAY_PTIME_MASK);

	val |= cur << MTK_PDMA_DELAY_RX_PTIME_SHIFT;



	cur = min_t(u32, cur_profile.pkts, MTK_PDMA_DELAY_PINT_MASK);

	val |= cur << MTK_PDMA_DELAY_RX_PINT_SHIFT;



	mtk_w32(eth, val, MTK_PDMA_DELAY_INT);

	mtk_w32(eth, val, MTK_QDMA_DELAY_INT);



	spin_unlock_bh(&eth->dim_lock);



	dim->state = DIM_START_MEASURE;

}



static void mtk_dim_tx(struct work_struct *work)

{

	struct dim *dim = container_of(work, struct dim, work);

	struct mtk_eth *eth = container_of(dim, struct mtk_eth, tx_dim);

	struct dim_cq_moder cur_profile;

	u32 val, cur;



	cur_profile = net_dim_get_tx_moderation(eth->tx_dim.mode,

						dim->profile_ix);

	spin_lock_bh(&eth->dim_lock);



	val = mtk_r32(eth, MTK_PDMA_DELAY_INT);

	val &= MTK_PDMA_DELAY_RX_MASK;

	val |= MTK_PDMA_DELAY_TX_EN;



	cur = min_t(u32, DIV_ROUND_UP(cur_profile.usec, 20), MTK_PDMA_DELAY_PTIME_MASK);

	val |= cur << MTK_PDMA_DELAY_TX_PTIME_SHIFT;



	cur = min_t(u32, cur_profile.pkts, MTK_PDMA_DELAY_PINT_MASK);

	val |= cur << MTK_PDMA_DELAY_TX_PINT_SHIFT;



	mtk_w32(eth, val, MTK_PDMA_DELAY_INT);

	mtk_w32(eth, val, MTK_QDMA_DELAY_INT);



	spin_unlock_bh(&eth->dim_lock);



	dim->state = DIM_START_MEASURE;

}



static int mtk_hw_init(struct mtk_eth *eth)

{

	int i, val, ret;
@@ -2398,9 +2476,6 @@ static int mtk_hw_init(struct mtk_eth *eth) 
			goto err_disable_pm;

		}



		/* enable interrupt delay for RX */

		mtk_w32(eth, MTK_PDMA_DELAY_RX_DELAY, MTK_PDMA_DELAY_INT);



		/* disable delay and normal interrupt */

		mtk_tx_irq_disable(eth, ~0);

		mtk_rx_irq_disable(eth, ~0);
@@ -2439,11 +2514,11 @@ static int mtk_hw_init(struct mtk_eth *eth) 
	/* Enable RX VLan Offloading */

	mtk_w32(eth, 1, MTK_CDMP_EG_CTRL);



	/* enable interrupt delay for RX */

	mtk_w32(eth, MTK_PDMA_DELAY_RX_DELAY, MTK_PDMA_DELAY_INT);

	/* set interrupt delays based on current Net DIM sample */

	mtk_dim_rx(&eth->rx_dim.work);

	mtk_dim_tx(&eth->tx_dim.work);



	/* disable delay and normal interrupt */

	mtk_w32(eth, 0, MTK_QDMA_DELAY_INT);

	mtk_tx_irq_disable(eth, ~0);

	mtk_rx_irq_disable(eth, ~0);
@@ -2978,6 +3053,13 @@ static int mtk_probe(struct platform_device *pdev) 
	spin_lock_init(&eth->page_lock);

	spin_lock_init(&eth->tx_irq_lock);

	spin_lock_init(&eth->rx_irq_lock);

	spin_lock_init(&eth->dim_lock);



	eth->rx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;

	INIT_WORK(&eth->rx_dim.work, mtk_dim_rx);



	eth->tx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;

	INIT_WORK(&eth->tx_dim.work, mtk_dim_tx);



	if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {

		eth->ethsys = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,

drivers/net/ethernet/mediatek/mtk_eth_soc.h

+34 −7
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ 
#include <linux/refcount.h>

#include <linux/phylink.h>

#include <linux/rhashtable.h>

#include <linux/dim.h>

#include "mtk_ppe.h"



#define MTK_QDMA_PAGE_SIZE	2048
@@ -137,13 +138,18 @@ 


/* PDMA Delay Interrupt Register */

#define MTK_PDMA_DELAY_INT		0xa0c

#define MTK_PDMA_DELAY_RX_MASK		GENMASK(15, 0)

#define MTK_PDMA_DELAY_RX_EN		BIT(15)

#define MTK_PDMA_DELAY_RX_PINT		4

#define MTK_PDMA_DELAY_RX_PINT_SHIFT	8

#define MTK_PDMA_DELAY_RX_PTIME		4

#define MTK_PDMA_DELAY_RX_DELAY		\

	(MTK_PDMA_DELAY_RX_EN | MTK_PDMA_DELAY_RX_PTIME | \

	(MTK_PDMA_DELAY_RX_PINT << MTK_PDMA_DELAY_RX_PINT_SHIFT))

#define MTK_PDMA_DELAY_RX_PTIME_SHIFT	0



#define MTK_PDMA_DELAY_TX_MASK		GENMASK(31, 16)

#define MTK_PDMA_DELAY_TX_EN		BIT(31)

#define MTK_PDMA_DELAY_TX_PINT_SHIFT	24

#define MTK_PDMA_DELAY_TX_PTIME_SHIFT	16



#define MTK_PDMA_DELAY_PINT_MASK	0x7f

#define MTK_PDMA_DELAY_PTIME_MASK	0xff



/* PDMA Interrupt Status Register */

#define MTK_PDMA_INT_STATUS	0xa20
@@ -225,6 +231,7 @@ 
/* QDMA Interrupt Status Register */

#define MTK_QDMA_INT_STATUS	0x1A18

#define MTK_RX_DONE_DLY		BIT(30)

#define MTK_TX_DONE_DLY		BIT(28)

#define MTK_RX_DONE_INT3	BIT(19)

#define MTK_RX_DONE_INT2	BIT(18)

#define MTK_RX_DONE_INT1	BIT(17)
@@ -234,8 +241,7 @@ 
#define MTK_TX_DONE_INT1	BIT(1)

#define MTK_TX_DONE_INT0	BIT(0)

#define MTK_RX_DONE_INT		MTK_RX_DONE_DLY

#define MTK_TX_DONE_INT		(MTK_TX_DONE_INT0 | MTK_TX_DONE_INT1 | \

				 MTK_TX_DONE_INT2 | MTK_TX_DONE_INT3)

#define MTK_TX_DONE_INT		MTK_TX_DONE_DLY



/* QDMA Interrupt grouping registers */

#define MTK_QDMA_INT_GRP1	0x1a20
@@ -849,6 +855,7 @@ struct mtk_sgmii { 
 * @page_lock:		Make sure that register operations are atomic

 * @tx_irq__lock:	Make sure that IRQ register operations are atomic

 * @rx_irq__lock:	Make sure that IRQ register operations are atomic

 * @dim_lock:		Make sure that Net DIM operations are atomic

 * @dummy_dev:		we run 2 netdevs on 1 physical DMA ring and need a

 *			dummy for NAPI to work

 * @netdev:		The netdev instances
@@ -867,6 +874,14 @@ struct mtk_sgmii { 
 * @rx_ring_qdma:	Pointer to the memory holding info about the QDMA RX ring

 * @tx_napi:		The TX NAPI struct

 * @rx_napi:		The RX NAPI struct

 * @rx_events:		Net DIM RX event counter

 * @rx_packets:		Net DIM RX packet counter

 * @rx_bytes:		Net DIM RX byte counter

 * @rx_dim:		Net DIM RX context

 * @tx_events:		Net DIM TX event counter

 * @tx_packets:		Net DIM TX packet counter

 * @tx_bytes:		Net DIM TX byte counter

 * @tx_dim:		Net DIM TX context

 * @scratch_ring:	Newer SoCs need memory for a second HW managed TX ring

 * @phy_scratch_ring:	physical address of scratch_ring

 * @scratch_head:	The scratch memory that scratch_ring points to.
@@ -911,6 +926,18 @@ struct mtk_eth { 


	const struct mtk_soc_data	*soc;



	spinlock_t			dim_lock;



	u32				rx_events;

	u32				rx_packets;

	u32				rx_bytes;

	struct dim			rx_dim;



	u32				tx_events;

	u32				tx_packets;

	u32				tx_bytes;

	struct dim			tx_dim;



	u32				tx_int_mask_reg;

	u32				tx_int_status_reg;

	u32				rx_dma_l4_valid;