Commit a24b4c6e authored by Maciej Fijalkowski's avatar Maciej Fijalkowski Committed by Daniel Borkmann
Browse files

ice: xsk: Do not convert to buff to frame for XDP_TX 



Let us store pointer to xdp_buff that came from xsk_buff_pool on tx_buf
so that it will be possible to recycle it via xsk_buff_free() on Tx
cleaning side. This way it is not necessary to do expensive copy to
another xdp_buff backed by a newly allocated page.

Signed-off-by: default avatarMaciej Fijalkowski <maciej.fijalkowski@intel.com>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Reviewed-by: default avatarAlexander Lobakin <alexandr.lobakin@intel.com>
Link: https://lore.kernel.org/bpf/20230131204506.219292-14-maciej.fijalkowski@intel.com
parent f4db7b31

drivers/net/ethernet/intel/ice/ice_txrx.c

+8 −4
Original line number Diff line number Diff line
@@ -113,12 +113,16 @@ static void 
ice_unmap_and_free_tx_buf(struct ice_tx_ring *ring, struct ice_tx_buf *tx_buf)

{

	if (tx_buf->skb) {

		if (tx_buf->tx_flags & ICE_TX_FLAGS_DUMMY_PKT)

		if (tx_buf->tx_flags & ICE_TX_FLAGS_DUMMY_PKT) {

			devm_kfree(ring->dev, tx_buf->raw_buf);

		else if (ice_ring_is_xdp(ring))

			page_frag_free(tx_buf->raw_buf);

		} else if (ice_ring_is_xdp(ring)) {

			if (ring->xsk_pool)

				xsk_buff_free(tx_buf->xdp);

			else

				page_frag_free(tx_buf->raw_buf);

		} else {

			dev_kfree_skb_any(tx_buf->skb);

		}

		if (dma_unmap_len(tx_buf, len))

			dma_unmap_single(ring->dev,

					 dma_unmap_addr(tx_buf, dma),

drivers/net/ethernet/intel/ice/ice_txrx.h

+1 −0
Original line number Diff line number Diff line
@@ -157,6 +157,7 @@ struct ice_tx_buf { 
	union {

		struct sk_buff *skb;

		void *raw_buf; /* used for XDP */

		struct xdp_buff *xdp; /* used for XDP_TX ZC */

	};

	unsigned int bytecount;

	union {

drivers/net/ethernet/intel/ice/ice_txrx_lib.c

+4 −18
Original line number Diff line number Diff line
@@ -275,6 +275,9 @@ static u32 ice_clean_xdp_irq(struct ice_tx_ring *xdp_ring) 
		/* count head + frags */

		ready_frames -= frags + 1;



		if (xdp_ring->xsk_pool)

			xsk_buff_free(tx_buf->xdp);

		else

			ice_clean_xdp_tx_buf(xdp_ring, tx_buf);

		ntc++;

		if (ntc == cnt)
@@ -407,23 +410,6 @@ int ice_xmit_xdp_ring(struct xdp_frame *xdpf, struct ice_tx_ring *xdp_ring) 
	return __ice_xmit_xdp_ring(&xdp, xdp_ring);

}



/**

 * ice_xmit_xdp_buff - convert an XDP buffer to an XDP frame and send it

 * @xdp: XDP buffer

 * @xdp_ring: XDP Tx ring

 *

 * Returns negative on failure, 0 on success.

 */

int ice_xmit_xdp_buff(struct xdp_buff *xdp, struct ice_tx_ring *xdp_ring)

{

	struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);



	if (unlikely(!xdpf))

		return ICE_XDP_CONSUMED;



	return ice_xmit_xdp_ring(xdpf, xdp_ring);

}



/**

 * ice_finalize_xdp_rx - Bump XDP Tx tail and/or flush redirect map

 * @xdp_ring: XDP ring

drivers/net/ethernet/intel/ice/ice_xsk.c

+104 −72
Original line number Diff line number Diff line
@@ -597,6 +597,107 @@ ice_construct_skb_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp) 
	return skb;

}



/**

 * ice_clean_xdp_irq_zc - AF_XDP ZC specific Tx cleaning routine

 * @xdp_ring: XDP Tx ring

 */

static void ice_clean_xdp_irq_zc(struct ice_tx_ring *xdp_ring)

{

	u16 ntc = xdp_ring->next_to_clean;

	struct ice_tx_desc *tx_desc;

	u16 cnt = xdp_ring->count;

	struct ice_tx_buf *tx_buf;

	u16 xsk_frames = 0;

	u16 last_rs;

	int i;



	last_rs = xdp_ring->next_to_use ? xdp_ring->next_to_use - 1 : cnt - 1;

	tx_desc = ICE_TX_DESC(xdp_ring, last_rs);

	if (tx_desc->cmd_type_offset_bsz &

	    cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE)) {

		if (last_rs >= ntc)

			xsk_frames = last_rs - ntc + 1;

		else

			xsk_frames = last_rs + cnt - ntc + 1;

	}



	if (!xsk_frames)

		return;



	if (likely(!xdp_ring->xdp_tx_active))

		goto skip;



	ntc = xdp_ring->next_to_clean;

	for (i = 0; i < xsk_frames; i++) {

		tx_buf = &xdp_ring->tx_buf[ntc];



		if (tx_buf->xdp) {

			xsk_buff_free(tx_buf->xdp);

			xdp_ring->xdp_tx_active--;

		} else {

			xsk_frames++;

		}



		ntc++;

		if (ntc == cnt)

			ntc = 0;

	}

skip:

	tx_desc->cmd_type_offset_bsz = 0;

	xdp_ring->next_to_clean += xsk_frames;

	if (xdp_ring->next_to_clean >= cnt)

		xdp_ring->next_to_clean -= cnt;

	if (xsk_frames)

		xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames);

}



/**

 * ice_xmit_xdp_tx_zc - AF_XDP ZC handler for XDP_TX

 * @xdp: XDP buffer to xmit

 * @xdp_ring: XDP ring to produce descriptor onto

 *

 * note that this function works directly on xdp_buff, no need to convert

 * it to xdp_frame. xdp_buff pointer is stored to ice_tx_buf so that cleaning

 * side will be able to xsk_buff_free() it.

 *

 * Returns ICE_XDP_TX for successfully produced desc, ICE_XDP_CONSUMED if there

 * was not enough space on XDP ring

 */

static int ice_xmit_xdp_tx_zc(struct xdp_buff *xdp,

			      struct ice_tx_ring *xdp_ring)

{

	u32 size = xdp->data_end - xdp->data;

	u32 ntu = xdp_ring->next_to_use;

	struct ice_tx_desc *tx_desc;

	struct ice_tx_buf *tx_buf;

	dma_addr_t dma;



	if (ICE_DESC_UNUSED(xdp_ring) < ICE_RING_QUARTER(xdp_ring)) {

		ice_clean_xdp_irq_zc(xdp_ring);

		if (!ICE_DESC_UNUSED(xdp_ring)) {

			xdp_ring->ring_stats->tx_stats.tx_busy++;

			return ICE_XDP_CONSUMED;

		}

	}



	dma = xsk_buff_xdp_get_dma(xdp);

	xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_pool, dma, size);



	tx_buf = &xdp_ring->tx_buf[ntu];

	tx_buf->xdp = xdp;

	tx_desc = ICE_TX_DESC(xdp_ring, ntu);

	tx_desc->buf_addr = cpu_to_le64(dma);

	tx_desc->cmd_type_offset_bsz = ice_build_ctob(ICE_TX_DESC_CMD_EOP,

						      0, size, 0);

	xdp_ring->xdp_tx_active++;



	if (++ntu == xdp_ring->count)

		ntu = 0;

	xdp_ring->next_to_use = ntu;



	return ICE_XDP_TX;

}



/**

 * ice_run_xdp_zc - Executes an XDP program in zero-copy path

 * @rx_ring: Rx ring
@@ -630,7 +731,7 @@ ice_run_xdp_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp, 
	case XDP_PASS:

		break;

	case XDP_TX:

		result = ice_xmit_xdp_buff(xdp, xdp_ring);

		result = ice_xmit_xdp_tx_zc(xdp, xdp_ring);

		if (result == ICE_XDP_CONSUMED)

			goto out_failure;

		break;
@@ -775,75 +876,6 @@ int ice_clean_rx_irq_zc(struct ice_rx_ring *rx_ring, int budget) 
	return failure ? budget : (int)total_rx_packets;

}



/**

 * ice_clean_xdp_tx_buf - Free and unmap XDP Tx buffer

 * @xdp_ring: XDP Tx ring

 * @tx_buf: Tx buffer to clean

 */

static void

ice_clean_xdp_tx_buf(struct ice_tx_ring *xdp_ring, struct ice_tx_buf *tx_buf)

{

	page_frag_free(tx_buf->raw_buf);

	xdp_ring->xdp_tx_active--;

	dma_unmap_single(xdp_ring->dev, dma_unmap_addr(tx_buf, dma),

			 dma_unmap_len(tx_buf, len), DMA_TO_DEVICE);

	dma_unmap_len_set(tx_buf, len, 0);

}



/**

 * ice_clean_xdp_irq_zc - produce AF_XDP descriptors to CQ

 * @xdp_ring: XDP Tx ring

 */

static void ice_clean_xdp_irq_zc(struct ice_tx_ring *xdp_ring)

{

	u16 ntc = xdp_ring->next_to_clean;

	struct ice_tx_desc *tx_desc;

	u16 cnt = xdp_ring->count;

	struct ice_tx_buf *tx_buf;

	u16 xsk_frames = 0;

	u16 last_rs;

	int i;



	last_rs = xdp_ring->next_to_use ? xdp_ring->next_to_use - 1 : cnt - 1;

	tx_desc = ICE_TX_DESC(xdp_ring, last_rs);

	if ((tx_desc->cmd_type_offset_bsz &

	    cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE))) {

		if (last_rs >= ntc)

			xsk_frames = last_rs - ntc + 1;

		else

			xsk_frames = last_rs + cnt - ntc + 1;

	}



	if (!xsk_frames)

		return;



	if (likely(!xdp_ring->xdp_tx_active))

		goto skip;



	ntc = xdp_ring->next_to_clean;

	for (i = 0; i < xsk_frames; i++) {

		tx_buf = &xdp_ring->tx_buf[ntc];



		if (tx_buf->raw_buf) {

			ice_clean_xdp_tx_buf(xdp_ring, tx_buf);

			tx_buf->raw_buf = NULL;

		} else {

			xsk_frames++;

		}



		ntc++;

		if (ntc >= xdp_ring->count)

			ntc = 0;

	}

skip:

	tx_desc->cmd_type_offset_bsz = 0;

	xdp_ring->next_to_clean += xsk_frames;

	if (xdp_ring->next_to_clean >= cnt)

		xdp_ring->next_to_clean -= cnt;

	if (xsk_frames)

		xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames);

}



/**

 * ice_xmit_pkt - produce a single HW Tx descriptor out of AF_XDP descriptor

 * @xdp_ring: XDP ring to produce the HW Tx descriptor on
@@ -1051,8 +1083,8 @@ void ice_xsk_clean_xdp_ring(struct ice_tx_ring *xdp_ring) 
	while (ntc != ntu) {

		struct ice_tx_buf *tx_buf = &xdp_ring->tx_buf[ntc];



		if (tx_buf->raw_buf)

			ice_clean_xdp_tx_buf(xdp_ring, tx_buf);

		if (tx_buf->xdp)

			xsk_buff_free(tx_buf->xdp);

		else

			xsk_frames++;