Commit 3f364222 authored by Lorenzo Bianconi's avatar Lorenzo Bianconi Committed by Alexei Starovoitov
Browse files

net: xdp: introduce bpf_xdp_pointer utility routine 



Similar to skb_header_pointer, introduce bpf_xdp_pointer utility routine
to return a pointer to a given position in the xdp_buff if the requested
area (offset + len) is contained in a contiguous memory area otherwise it
will be copied in a bounce buffer provided by the caller.
Similar to the tc counterpart, introduce the two following xdp helpers:
- bpf_xdp_load_bytes
- bpf_xdp_store_bytes

Reviewed-by: default avatarEelco Chaudron <echaudro@redhat.com>
Acked-by: default avatarToke Hoiland-Jorgensen <toke@redhat.com>
Acked-by: default avatarJohn Fastabend <john.fastabend@gmail.com>
Acked-by: default avatarJakub Kicinski <kuba@kernel.org>
Signed-off-by: default avatarLorenzo Bianconi <lorenzo@kernel.org>
Link: https://lore.kernel.org/r/ab285c1efdd5b7a9d361348b1e7d3ef49f6382b3.1642758637.git.lorenzo@kernel.org


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent f45d5b6c

include/uapi/linux/bpf.h

+18 −0
Original line number Diff line number Diff line
@@ -5060,6 +5060,22 @@ union bpf_attr { 
 *		Get the total size of a given xdp buff (linear and paged area)

 *	Return

 *		The total size of a given xdp buffer.

 *

 * long bpf_xdp_load_bytes(struct xdp_buff *xdp_md, u32 offset, void *buf, u32 len)

 *	Description

 *		This helper is provided as an easy way to load data from a

 *		xdp buffer. It can be used to load *len* bytes from *offset* from

 *		the frame associated to *xdp_md*, into the buffer pointed by

 *		*buf*.

 *	Return

 *		0 on success, or a negative error in case of failure.

 *

 * long bpf_xdp_store_bytes(struct xdp_buff *xdp_md, u32 offset, void *buf, u32 len)

 *	Description

 *		Store *len* bytes from buffer *buf* into the frame

 *		associated to *xdp_md*, at *offset*.

 *	Return

 *		0 on success, or a negative error in case of failure.

 */

#define __BPF_FUNC_MAPPER(FN)		\

	FN(unspec),			\
@@ -5251,6 +5267,8 @@ union bpf_attr { 
	FN(get_retval),			\

	FN(set_retval),			\

	FN(xdp_get_buff_len),		\

	FN(xdp_load_bytes),		\

	FN(xdp_store_bytes),		\

	/* */



/* integer value in 'imm' field of BPF_CALL instruction selects which helper

net/core/filter.c

+138 −38
Original line number Diff line number Diff line
@@ -3839,6 +3839,138 @@ static const struct bpf_func_proto bpf_xdp_adjust_head_proto = { 
	.arg2_type	= ARG_ANYTHING,

};



static void bpf_xdp_copy_buf(struct xdp_buff *xdp, unsigned long off,

			     void *buf, unsigned long len, bool flush)

{

	unsigned long ptr_len, ptr_off = 0;

	skb_frag_t *next_frag, *end_frag;

	struct skb_shared_info *sinfo;

	void *src, *dst;

	u8 *ptr_buf;



	if (likely(xdp->data_end - xdp->data >= off + len)) {

		src = flush ? buf : xdp->data + off;

		dst = flush ? xdp->data + off : buf;

		memcpy(dst, src, len);

		return;

	}



	sinfo = xdp_get_shared_info_from_buff(xdp);

	end_frag = &sinfo->frags[sinfo->nr_frags];

	next_frag = &sinfo->frags[0];



	ptr_len = xdp->data_end - xdp->data;

	ptr_buf = xdp->data;



	while (true) {

		if (off < ptr_off + ptr_len) {

			unsigned long copy_off = off - ptr_off;

			unsigned long copy_len = min(len, ptr_len - copy_off);



			src = flush ? buf : ptr_buf + copy_off;

			dst = flush ? ptr_buf + copy_off : buf;

			memcpy(dst, src, copy_len);



			off += copy_len;

			len -= copy_len;

			buf += copy_len;

		}



		if (!len || next_frag == end_frag)

			break;



		ptr_off += ptr_len;

		ptr_buf = skb_frag_address(next_frag);

		ptr_len = skb_frag_size(next_frag);

		next_frag++;

	}

}



static void *bpf_xdp_pointer(struct xdp_buff *xdp, u32 offset, u32 len)

{

	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);

	u32 size = xdp->data_end - xdp->data;

	void *addr = xdp->data;

	int i;



	if (unlikely(offset > 0xffff || len > 0xffff))

		return ERR_PTR(-EFAULT);



	if (offset + len > xdp_get_buff_len(xdp))

		return ERR_PTR(-EINVAL);



	if (offset < size) /* linear area */

		goto out;



	offset -= size;

	for (i = 0; i < sinfo->nr_frags; i++) { /* paged area */

		u32 frag_size = skb_frag_size(&sinfo->frags[i]);



		if  (offset < frag_size) {

			addr = skb_frag_address(&sinfo->frags[i]);

			size = frag_size;

			break;

		}

		offset -= frag_size;

	}

out:

	return offset + len < size ? addr + offset : NULL;

}



BPF_CALL_4(bpf_xdp_load_bytes, struct xdp_buff *, xdp, u32, offset,

	   void *, buf, u32, len)

{

	void *ptr;



	ptr = bpf_xdp_pointer(xdp, offset, len);

	if (IS_ERR(ptr))

		return PTR_ERR(ptr);



	if (!ptr)

		bpf_xdp_copy_buf(xdp, offset, buf, len, false);

	else

		memcpy(buf, ptr, len);



	return 0;

}



static const struct bpf_func_proto bpf_xdp_load_bytes_proto = {

	.func		= bpf_xdp_load_bytes,

	.gpl_only	= false,

	.ret_type	= RET_INTEGER,

	.arg1_type	= ARG_PTR_TO_CTX,

	.arg2_type	= ARG_ANYTHING,

	.arg3_type	= ARG_PTR_TO_UNINIT_MEM,

	.arg4_type	= ARG_CONST_SIZE,

};



BPF_CALL_4(bpf_xdp_store_bytes, struct xdp_buff *, xdp, u32, offset,

	   void *, buf, u32, len)

{

	void *ptr;



	ptr = bpf_xdp_pointer(xdp, offset, len);

	if (IS_ERR(ptr))

		return PTR_ERR(ptr);



	if (!ptr)

		bpf_xdp_copy_buf(xdp, offset, buf, len, true);

	else

		memcpy(ptr, buf, len);



	return 0;

}



static const struct bpf_func_proto bpf_xdp_store_bytes_proto = {

	.func		= bpf_xdp_store_bytes,

	.gpl_only	= false,

	.ret_type	= RET_INTEGER,

	.arg1_type	= ARG_PTR_TO_CTX,

	.arg2_type	= ARG_ANYTHING,

	.arg3_type	= ARG_PTR_TO_UNINIT_MEM,

	.arg4_type	= ARG_CONST_SIZE,

};



static int bpf_xdp_frags_increase_tail(struct xdp_buff *xdp, int offset)

{

	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
@@ -4677,48 +4809,12 @@ static const struct bpf_func_proto bpf_sk_ancestor_cgroup_id_proto = { 
};

#endif



static unsigned long bpf_xdp_copy(void *dst_buff, const void *ctx,

static unsigned long bpf_xdp_copy(void *dst, const void *ctx,

				  unsigned long off, unsigned long len)

{

	struct xdp_buff *xdp = (struct xdp_buff *)ctx;

	unsigned long ptr_len, ptr_off = 0;

	skb_frag_t *next_frag, *end_frag;

	struct skb_shared_info *sinfo;

	u8 *ptr_buf;



	if (likely(xdp->data_end - xdp->data >= off + len)) {

		memcpy(dst_buff, xdp->data + off, len);

		return 0;

	}



	sinfo = xdp_get_shared_info_from_buff(xdp);

	end_frag = &sinfo->frags[sinfo->nr_frags];

	next_frag = &sinfo->frags[0];



	ptr_len = xdp->data_end - xdp->data;

	ptr_buf = xdp->data;



	while (true) {

		if (off < ptr_off + ptr_len) {

			unsigned long copy_off = off - ptr_off;

			unsigned long copy_len = min(len, ptr_len - copy_off);



			memcpy(dst_buff, ptr_buf + copy_off, copy_len);



			off += copy_len;

			len -= copy_len;

			dst_buff += copy_len;

		}



		if (!len || next_frag == end_frag)

			break;



		ptr_off += ptr_len;

		ptr_buf = skb_frag_address(next_frag);

		ptr_len = skb_frag_size(next_frag);

		next_frag++;

	}



	bpf_xdp_copy_buf(xdp, off, dst, len, false);

	return 0;

}
@@ -7660,6 +7756,10 @@ xdp_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 
		return &bpf_xdp_adjust_tail_proto;

	case BPF_FUNC_xdp_get_buff_len:

		return &bpf_xdp_get_buff_len_proto;

	case BPF_FUNC_xdp_load_bytes:

		return &bpf_xdp_load_bytes_proto;

	case BPF_FUNC_xdp_store_bytes:

		return &bpf_xdp_store_bytes_proto;

	case BPF_FUNC_fib_lookup:

		return &bpf_xdp_fib_lookup_proto;

	case BPF_FUNC_check_mtu:

tools/include/uapi/linux/bpf.h

+18 −0
Original line number Diff line number Diff line
@@ -5060,6 +5060,22 @@ union bpf_attr { 
 *		Get the total size of a given xdp buff (linear and paged area)

 *	Return

 *		The total size of a given xdp buffer.

 *

 * long bpf_xdp_load_bytes(struct xdp_buff *xdp_md, u32 offset, void *buf, u32 len)

 *	Description

 *		This helper is provided as an easy way to load data from a

 *		xdp buffer. It can be used to load *len* bytes from *offset* from

 *		the frame associated to *xdp_md*, into the buffer pointed by

 *		*buf*.

 *	Return

 *		0 on success, or a negative error in case of failure.

 *

 * long bpf_xdp_store_bytes(struct xdp_buff *xdp_md, u32 offset, void *buf, u32 len)

 *	Description

 *		Store *len* bytes from buffer *buf* into the frame

 *		associated to *xdp_md*, at *offset*.

 *	Return

 *		0 on success, or a negative error in case of failure.

 */

#define __BPF_FUNC_MAPPER(FN)		\

	FN(unspec),			\
@@ -5251,6 +5267,8 @@ union bpf_attr { 
	FN(get_retval),			\

	FN(set_retval),			\

	FN(xdp_get_buff_len),		\

	FN(xdp_load_bytes),		\

	FN(xdp_store_bytes),		\

	/* */



/* integer value in 'imm' field of BPF_CALL instruction selects which helper