Commit ad4858be authored by Xuan Zhuo's avatar Xuan Zhuo Committed by Jakub Kicinski
Browse files

virtio_net: introduce mergeable_xdp_get_buf() 



Separating the logic of preparation for xdp from receive_mergeable.

The purpose of this is to simplify the logic of execution of XDP.

The main logic here is that when headroom is insufficient, we need to
allocate a new page and calculate offset. It should be noted that if
there is new page, the variable page will refer to the new page.

Signed-off-by: default avatarXuan Zhuo <xuanzhuo@linux.alibaba.com>
Acked-by: default avatarJason Wang <jasowang@redhat.com>
Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 363d8ce4

drivers/net/virtio_net.c

+79 −56
Original line number Original line Diff line number Diff line
@@ -1166,6 +1166,81 @@ static int virtnet_build_xdp_buff_mrg(struct net_device *dev, 
	return 0;

	return 0;

}

}





static void *mergeable_xdp_get_buf(struct virtnet_info *vi,

				   struct receive_queue *rq,

				   struct bpf_prog *xdp_prog,

				   void *ctx,

				   unsigned int *frame_sz,

				   int *num_buf,

				   struct page **page,

				   int offset,

				   unsigned int *len,

				   struct virtio_net_hdr_mrg_rxbuf *hdr)

{

	unsigned int truesize = mergeable_ctx_to_truesize(ctx);

	unsigned int headroom = mergeable_ctx_to_headroom(ctx);

	struct page *xdp_page;

	unsigned int xdp_room;



	/* Transient failure which in theory could occur if

	 * in-flight packets from before XDP was enabled reach

	 * the receive path after XDP is loaded.

	 */

	if (unlikely(hdr->hdr.gso_type))

		return NULL;



	/* Now XDP core assumes frag size is PAGE_SIZE, but buffers

	 * with headroom may add hole in truesize, which

	 * make their length exceed PAGE_SIZE. So we disabled the

	 * hole mechanism for xdp. See add_recvbuf_mergeable().

	 */

	*frame_sz = truesize;



	/* This happens when headroom is not enough because

	 * of the buffer was prefilled before XDP is set.

	 * This should only happen for the first several packets.

	 * In fact, vq reset can be used here to help us clean up

	 * the prefilled buffers, but many existing devices do not

	 * support it, and we don't want to bother users who are

	 * using xdp normally.

	 */

	if (!xdp_prog->aux->xdp_has_frags &&

	    (*num_buf > 1 || headroom < virtnet_get_headroom(vi))) {

		/* linearize data for XDP */

		xdp_page = xdp_linearize_page(rq, num_buf,

					      *page, offset,

					      VIRTIO_XDP_HEADROOM,

					      len);

		*frame_sz = PAGE_SIZE;



		if (!xdp_page)

			return NULL;

		offset = VIRTIO_XDP_HEADROOM;



		put_page(*page);

		*page = xdp_page;

	} else if (unlikely(headroom < virtnet_get_headroom(vi))) {

		xdp_room = SKB_DATA_ALIGN(VIRTIO_XDP_HEADROOM +

					  sizeof(struct skb_shared_info));

		if (*len + xdp_room > PAGE_SIZE)

			return NULL;



		xdp_page = alloc_page(GFP_ATOMIC);

		if (!xdp_page)

			return NULL;



		memcpy(page_address(xdp_page) + VIRTIO_XDP_HEADROOM,

		       page_address(*page) + offset, *len);

		*frame_sz = PAGE_SIZE;

		offset = VIRTIO_XDP_HEADROOM;



		put_page(*page);

		*page = xdp_page;

	}



	return page_address(*page) + offset;

}



static struct sk_buff *receive_mergeable(struct net_device *dev,

static struct sk_buff *receive_mergeable(struct net_device *dev,

					 struct virtnet_info *vi,

					 struct virtnet_info *vi,

					 struct receive_queue *rq,

					 struct receive_queue *rq,
@@ -1185,7 +1260,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, 
	unsigned int headroom = mergeable_ctx_to_headroom(ctx);

	unsigned int headroom = mergeable_ctx_to_headroom(ctx);

	unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;

	unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;

	unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);

	unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);

	unsigned int frame_sz, xdp_room;

	unsigned int frame_sz;

	int err;

	int err;





	head_skb = NULL;

	head_skb = NULL;
@@ -1215,63 +1290,11 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, 
		u32 act;

		u32 act;

		int i;

		int i;





		/* Transient failure which in theory could occur if

		data = mergeable_xdp_get_buf(vi, rq, xdp_prog, ctx, &frame_sz,

		 * in-flight packets from before XDP was enabled reach

					     &num_buf, &page, offset, &len, hdr);

		 * the receive path after XDP is loaded.

		if (unlikely(!data))

		 */

		if (unlikely(hdr->hdr.gso_type))

			goto err_xdp;

			goto err_xdp;





		/* Now XDP core assumes frag size is PAGE_SIZE, but buffers

		 * with headroom may add hole in truesize, which

		 * make their length exceed PAGE_SIZE. So we disabled the

		 * hole mechanism for xdp. See add_recvbuf_mergeable().

		 */

		frame_sz = truesize;



		/* This happens when headroom is not enough because

		 * of the buffer was prefilled before XDP is set.

		 * This should only happen for the first several packets.

		 * In fact, vq reset can be used here to help us clean up

		 * the prefilled buffers, but many existing devices do not

		 * support it, and we don't want to bother users who are

		 * using xdp normally.

		 */

		if (!xdp_prog->aux->xdp_has_frags &&

		    (num_buf > 1 || headroom < virtnet_get_headroom(vi))) {

			/* linearize data for XDP */

			xdp_page = xdp_linearize_page(rq, &num_buf,

						      page, offset,

						      VIRTIO_XDP_HEADROOM,

						      &len);

			frame_sz = PAGE_SIZE;



			if (!xdp_page)

				goto err_xdp;

			offset = VIRTIO_XDP_HEADROOM;



			put_page(page);

			page = xdp_page;

		} else if (unlikely(headroom < virtnet_get_headroom(vi))) {

			xdp_room = SKB_DATA_ALIGN(VIRTIO_XDP_HEADROOM +

						  sizeof(struct skb_shared_info));

			if (len + xdp_room > PAGE_SIZE)

				goto err_xdp;



			xdp_page = alloc_page(GFP_ATOMIC);

			if (!xdp_page)

				goto err_xdp;



			memcpy(page_address(xdp_page) + VIRTIO_XDP_HEADROOM,

			       page_address(page) + offset, len);

			frame_sz = PAGE_SIZE;

			offset = VIRTIO_XDP_HEADROOM;



			put_page(page);

			page = xdp_page;

		}



		data = page_address(page) + offset;

		err = virtnet_build_xdp_buff_mrg(dev, vi, rq, &xdp, data, len, frame_sz,

		err = virtnet_build_xdp_buff_mrg(dev, vi, rq, &xdp, data, len, frame_sz,

						 &num_buf, &xdp_frags_truesz, stats);

						 &num_buf, &xdp_frags_truesz, stats);

		if (unlikely(err))

		if (unlikely(err))