Commit 7c48cb01 authored by Lorenzo Bianconi's avatar Lorenzo Bianconi Committed by Alexei Starovoitov
Browse files

xdp: add frags support to xdp_return_{buff/frame} 



Take into account if the received xdp_buff/xdp_frame is non-linear
recycling/returning the frame memory to the allocator or into
xdp_frame_bulk.

Acked-by: default avatarToke Hoiland-Jorgensen <toke@redhat.com>
Acked-by: default avatarJohn Fastabend <john.fastabend@gmail.com>
Signed-off-by: default avatarLorenzo Bianconi <lorenzo@kernel.org>
Link: https://lore.kernel.org/r/a961069febc868508ce1bdf5e53a343eb4e57cb2.1642758637.git.lorenzo@kernel.org


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent ed7a58cb

include/net/xdp.h

+16 −2
Original line number Original line Diff line number Diff line
@@ -306,9 +306,23 @@ void __xdp_release_frame(void *data, struct xdp_mem_info *mem); 
static inline void xdp_release_frame(struct xdp_frame *xdpf)

static inline void xdp_release_frame(struct xdp_frame *xdpf)

{

{

	struct xdp_mem_info *mem = &xdpf->mem;

	struct xdp_mem_info *mem = &xdpf->mem;

	struct skb_shared_info *sinfo;

	int i;





	/* Curr only page_pool needs this */

	/* Curr only page_pool needs this */

	if (mem->type == MEM_TYPE_PAGE_POOL)

	if (mem->type != MEM_TYPE_PAGE_POOL)

		return;



	if (likely(!xdp_frame_has_frags(xdpf)))

		goto out;



	sinfo = xdp_get_shared_info_from_frame(xdpf);

	for (i = 0; i < sinfo->nr_frags; i++) {

		struct page *page = skb_frag_page(&sinfo->frags[i]);



		__xdp_release_frame(page_address(page), mem);

	}

out:

	__xdp_release_frame(xdpf->data, mem);

	__xdp_release_frame(xdpf->data, mem);

}

}

net/core/xdp.c

+53 −1
Original line number Original line Diff line number Diff line
@@ -406,12 +406,38 @@ static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct, 




void xdp_return_frame(struct xdp_frame *xdpf)

void xdp_return_frame(struct xdp_frame *xdpf)

{

{

	struct skb_shared_info *sinfo;

	int i;



	if (likely(!xdp_frame_has_frags(xdpf)))

		goto out;



	sinfo = xdp_get_shared_info_from_frame(xdpf);

	for (i = 0; i < sinfo->nr_frags; i++) {

		struct page *page = skb_frag_page(&sinfo->frags[i]);



		__xdp_return(page_address(page), &xdpf->mem, false, NULL);

	}

out:

	__xdp_return(xdpf->data, &xdpf->mem, false, NULL);

	__xdp_return(xdpf->data, &xdpf->mem, false, NULL);

}

}

EXPORT_SYMBOL_GPL(xdp_return_frame);

EXPORT_SYMBOL_GPL(xdp_return_frame);





void xdp_return_frame_rx_napi(struct xdp_frame *xdpf)

void xdp_return_frame_rx_napi(struct xdp_frame *xdpf)

{

{

	struct skb_shared_info *sinfo;

	int i;



	if (likely(!xdp_frame_has_frags(xdpf)))

		goto out;



	sinfo = xdp_get_shared_info_from_frame(xdpf);

	for (i = 0; i < sinfo->nr_frags; i++) {

		struct page *page = skb_frag_page(&sinfo->frags[i]);



		__xdp_return(page_address(page), &xdpf->mem, true, NULL);

	}

out:

	__xdp_return(xdpf->data, &xdpf->mem, true, NULL);

	__xdp_return(xdpf->data, &xdpf->mem, true, NULL);

}

}

EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi);

EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi);
@@ -447,7 +473,7 @@ void xdp_return_frame_bulk(struct xdp_frame *xdpf, 
	struct xdp_mem_allocator *xa;

	struct xdp_mem_allocator *xa;





	if (mem->type != MEM_TYPE_PAGE_POOL) {

	if (mem->type != MEM_TYPE_PAGE_POOL) {

		__xdp_return(xdpf->data, &xdpf->mem, false, NULL);

		xdp_return_frame(xdpf);

		return;

		return;

	}

	}
@@ -466,12 +492,38 @@ void xdp_return_frame_bulk(struct xdp_frame *xdpf, 
		bq->xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);

		bq->xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);

	}

	}





	if (unlikely(xdp_frame_has_frags(xdpf))) {

		struct skb_shared_info *sinfo;

		int i;



		sinfo = xdp_get_shared_info_from_frame(xdpf);

		for (i = 0; i < sinfo->nr_frags; i++) {

			skb_frag_t *frag = &sinfo->frags[i];



			bq->q[bq->count++] = skb_frag_address(frag);

			if (bq->count == XDP_BULK_QUEUE_SIZE)

				xdp_flush_frame_bulk(bq);

		}

	}

	bq->q[bq->count++] = xdpf->data;

	bq->q[bq->count++] = xdpf->data;

}

}

EXPORT_SYMBOL_GPL(xdp_return_frame_bulk);

EXPORT_SYMBOL_GPL(xdp_return_frame_bulk);





void xdp_return_buff(struct xdp_buff *xdp)

void xdp_return_buff(struct xdp_buff *xdp)

{

{

	struct skb_shared_info *sinfo;

	int i;



	if (likely(!xdp_buff_has_frags(xdp)))

		goto out;



	sinfo = xdp_get_shared_info_from_buff(xdp);

	for (i = 0; i < sinfo->nr_frags; i++) {

		struct page *page = skb_frag_page(&sinfo->frags[i]);



		__xdp_return(page_address(page), &xdp->rxq->mem, true, xdp);

	}

out:

	__xdp_return(xdp->data, &xdp->rxq->mem, true, xdp);

	__xdp_return(xdp->data, &xdp->rxq->mem, true, xdp);

}

}