Commit 78f31931 authored by Ciara Loftus's avatar Ciara Loftus Committed by Jakub Kicinski
Browse files

i40e: read the XDP program once per NAPI 



Similar to how it's done in the ice driver since 'eb087cd8 ("ice:
propagate xdp_ring onto rx_ring")', read the XDP program once per NAPI
instead of once per descriptor cleaned. I measured an improvement in
throughput of 2% for the AF_XDP xdpsock l2fwd benchmark for zero copy mode
and 1% for copy mode.

Signed-off-by: default avatarCiara Loftus <ciara.loftus@intel.com>
Link: https://lore.kernel.org/r/20220623100852.7867-1-ciara.loftus@intel.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 85a1c653

drivers/net/ethernet/intel/i40e/i40e_txrx.c

+6 −5
Original line number Diff line number Diff line
@@ -2289,16 +2289,14 @@ int i40e_xmit_xdp_tx_ring(struct xdp_buff *xdp, struct i40e_ring *xdp_ring) 
 * i40e_run_xdp - run an XDP program

 * @rx_ring: Rx ring being processed

 * @xdp: XDP buffer containing the frame

 * @xdp_prog: XDP program to run

 **/

static int i40e_run_xdp(struct i40e_ring *rx_ring, struct xdp_buff *xdp)

static int i40e_run_xdp(struct i40e_ring *rx_ring, struct xdp_buff *xdp, struct bpf_prog *xdp_prog)

{

	int err, result = I40E_XDP_PASS;

	struct i40e_ring *xdp_ring;

	struct bpf_prog *xdp_prog;

	u32 act;



	xdp_prog = READ_ONCE(rx_ring->xdp_prog);



	if (!xdp_prog)

		goto xdp_out;
@@ -2443,6 +2441,7 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget) 
	unsigned int offset = rx_ring->rx_offset;

	struct sk_buff *skb = rx_ring->skb;

	unsigned int xdp_xmit = 0;

	struct bpf_prog *xdp_prog;

	bool failure = false;

	struct xdp_buff xdp;

	int xdp_res = 0;
@@ -2452,6 +2451,8 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget) 
#endif

	xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq);



	xdp_prog = READ_ONCE(rx_ring->xdp_prog);



	while (likely(total_rx_packets < (unsigned int)budget)) {

		struct i40e_rx_buffer *rx_buffer;

		union i40e_rx_desc *rx_desc;
@@ -2512,7 +2513,7 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget) 
			/* At larger PAGE_SIZE, frame_sz depend on len size */

			xdp.frame_sz = i40e_rx_frame_truesize(rx_ring, size);

#endif

			xdp_res = i40e_run_xdp(rx_ring, &xdp);

			xdp_res = i40e_run_xdp(rx_ring, &xdp, xdp_prog);

		}



		if (xdp_res) {

drivers/net/ethernet/intel/i40e/i40e_xsk.c

+10 −7
Original line number Diff line number Diff line
@@ -143,20 +143,17 @@ int i40e_xsk_pool_setup(struct i40e_vsi *vsi, struct xsk_buff_pool *pool, 
 * i40e_run_xdp_zc - Executes an XDP program on an xdp_buff

 * @rx_ring: Rx ring

 * @xdp: xdp_buff used as input to the XDP program

 * @xdp_prog: XDP program to run

 *

 * Returns any of I40E_XDP_{PASS, CONSUMED, TX, REDIR}

 **/

static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp)

static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp,

			   struct bpf_prog *xdp_prog)

{

	int err, result = I40E_XDP_PASS;

	struct i40e_ring *xdp_ring;

	struct bpf_prog *xdp_prog;

	u32 act;



	/* NB! xdp_prog will always be !NULL, due to the fact that

	 * this path is enabled by setting an XDP program.

	 */

	xdp_prog = READ_ONCE(rx_ring->xdp_prog);

	act = bpf_prog_run_xdp(xdp_prog, xdp);



	if (likely(act == XDP_REDIRECT)) {
@@ -339,9 +336,15 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) 
	u16 next_to_clean = rx_ring->next_to_clean;

	u16 count_mask = rx_ring->count - 1;

	unsigned int xdp_res, xdp_xmit = 0;

	struct bpf_prog *xdp_prog;

	bool failure = false;

	u16 cleaned_count;



	/* NB! xdp_prog will always be !NULL, due to the fact that

	 * this path is enabled by setting an XDP program.

	 */

	xdp_prog = READ_ONCE(rx_ring->xdp_prog);



	while (likely(total_rx_packets < (unsigned int)budget)) {

		union i40e_rx_desc *rx_desc;

		unsigned int rx_packets;
@@ -378,7 +381,7 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) 
		xsk_buff_set_size(bi, size);

		xsk_buff_dma_sync_for_cpu(bi, rx_ring->xsk_pool);



		xdp_res = i40e_run_xdp_zc(rx_ring, bi);

		xdp_res = i40e_run_xdp_zc(rx_ring, bi, xdp_prog);

		i40e_handle_xdp_result_zc(rx_ring, bi, rx_desc, &rx_packets,

					  &rx_bytes, size, xdp_res, &failure);

		if (failure)