Commit 508362ac authored by Maxim Mikityanskiy's avatar Maxim Mikityanskiy Committed by Alexei Starovoitov
Browse files

bpf: Allow helpers to accept pointers with a fixed size 



Before this commit, the BPF verifier required ARG_PTR_TO_MEM arguments
to be followed by ARG_CONST_SIZE holding the size of the memory region.
The helpers had to check that size in runtime.

There are cases where the size expected by a helper is a compile-time
constant. Checking it in runtime is an unnecessary overhead and waste of
BPF registers.

This commit allows helpers to accept pointers to memory without the
corresponding ARG_CONST_SIZE, given that they define the memory region
size in struct bpf_func_proto and use ARG_PTR_TO_FIXED_SIZE_MEM type.

arg_size is unionized with arg_btf_id to reduce the kernel image size,
and it's valid because they are used by different argument types.

Signed-off-by: default avatarMaxim Mikityanskiy <maximmi@nvidia.com>
Reviewed-by: default avatarTariq Toukan <tariqt@nvidia.com>
Link: https://lore.kernel.org/r/20220615134847.3753567-3-maximmi@nvidia.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent ac80287a

include/linux/bpf.h

+13 −0
Original line number Diff line number Diff line
@@ -401,6 +401,9 @@ enum bpf_type_flag { 
	/* DYNPTR points to a ringbuf record. */

	DYNPTR_TYPE_RINGBUF	= BIT(9 + BPF_BASE_TYPE_BITS),



	/* Size is known at compile time. */

	MEM_FIXED_SIZE		= BIT(10 + BPF_BASE_TYPE_BITS),



	__BPF_TYPE_FLAG_MAX,

	__BPF_TYPE_LAST_FLAG	= __BPF_TYPE_FLAG_MAX - 1,

};
@@ -464,6 +467,8 @@ enum bpf_arg_type { 
	 * all bytes or clear them in error case.

	 */

	ARG_PTR_TO_UNINIT_MEM		= MEM_UNINIT | ARG_PTR_TO_MEM,

	/* Pointer to valid memory of size known at compile time. */

	ARG_PTR_TO_FIXED_SIZE_MEM	= MEM_FIXED_SIZE | ARG_PTR_TO_MEM,



	/* This must be the last entry. Its purpose is to ensure the enum is

	 * wide enough to hold the higher bits reserved for bpf_type_flag.
@@ -529,6 +534,14 @@ struct bpf_func_proto { 
			u32 *arg5_btf_id;

		};

		u32 *arg_btf_id[5];

		struct {

			size_t arg1_size;

			size_t arg2_size;

			size_t arg3_size;

			size_t arg4_size;

			size_t arg5_size;

		};

		size_t arg_size[5];

	};

	int *ret_btf_id; /* return value btf_id */

	bool (*allowed)(const struct bpf_prog *prog);

kernel/bpf/verifier.c

+32 −11
Original line number Diff line number Diff line
@@ -5848,6 +5848,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, 
	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];

	enum bpf_arg_type arg_type = fn->arg_type[arg];

	enum bpf_reg_type type = reg->type;

	u32 *arg_btf_id = NULL;

	int err = 0;



	if (arg_type == ARG_DONTCARE)
@@ -5884,7 +5885,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, 
		 */

		goto skip_type_check;



	err = check_reg_type(env, regno, arg_type, fn->arg_btf_id[arg], meta);

	/* arg_btf_id and arg_size are in a union. */

	if (base_type(arg_type) == ARG_PTR_TO_BTF_ID)

		arg_btf_id = fn->arg_btf_id[arg];



	err = check_reg_type(env, regno, arg_type, arg_btf_id, meta);

	if (err)

		return err;
@@ -6011,6 +6016,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, 
		 * next is_mem_size argument below.

		 */

		meta->raw_mode = arg_type & MEM_UNINIT;

		if (arg_type & MEM_FIXED_SIZE) {

			err = check_helper_mem_access(env, regno,

						      fn->arg_size[arg], false,

						      meta);

		}

	} else if (arg_type_is_mem_size(arg_type)) {

		bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO);
@@ -6400,11 +6410,19 @@ static bool check_raw_mode_ok(const struct bpf_func_proto *fn) 
	return count <= 1;

}



static bool check_args_pair_invalid(enum bpf_arg_type arg_curr,

				    enum bpf_arg_type arg_next)

static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg)

{

	return (base_type(arg_curr) == ARG_PTR_TO_MEM) !=

		arg_type_is_mem_size(arg_next);

	bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE;

	bool has_size = fn->arg_size[arg] != 0;

	bool is_next_size = false;



	if (arg + 1 < ARRAY_SIZE(fn->arg_type))

		is_next_size = arg_type_is_mem_size(fn->arg_type[arg + 1]);



	if (base_type(fn->arg_type[arg]) != ARG_PTR_TO_MEM)

		return is_next_size;



	return has_size == is_next_size || is_next_size == is_fixed;

}



static bool check_arg_pair_ok(const struct bpf_func_proto *fn)
@@ -6415,11 +6433,11 @@ static bool check_arg_pair_ok(const struct bpf_func_proto *fn) 
	 * helper function specification.

	 */

	if (arg_type_is_mem_size(fn->arg1_type) ||

	    base_type(fn->arg5_type) == ARG_PTR_TO_MEM ||

	    check_args_pair_invalid(fn->arg1_type, fn->arg2_type) ||

	    check_args_pair_invalid(fn->arg2_type, fn->arg3_type) ||

	    check_args_pair_invalid(fn->arg3_type, fn->arg4_type) ||

	    check_args_pair_invalid(fn->arg4_type, fn->arg5_type))

	    check_args_pair_invalid(fn, 0) ||

	    check_args_pair_invalid(fn, 1) ||

	    check_args_pair_invalid(fn, 2) ||

	    check_args_pair_invalid(fn, 3) ||

	    check_args_pair_invalid(fn, 4))

		return false;



	return true;
@@ -6460,7 +6478,10 @@ static bool check_btf_id_ok(const struct bpf_func_proto *fn) 
		if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID && !fn->arg_btf_id[i])

			return false;



		if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i])

		if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i] &&

		    /* arg_btf_id and arg_size are in a union. */

		    (base_type(fn->arg_type[i]) != ARG_PTR_TO_MEM ||

		     !(fn->arg_type[i] & MEM_FIXED_SIZE)))

			return false;

	}