Commit e94fac38 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'bpf: Add helpers to access traced function arguments' 

Jiri Olsa says:

====================
Add new helpers to access traced function arguments that
came out of the trampoline batch changes [1].

  Get n-th argument of the traced function:
    long bpf_get_func_arg(void *ctx, u32 n, u64 *value)

  Get return value of the traced function:
    long bpf_get_func_ret(void *ctx, u64 *value)

  Get arguments count of the traced funtion:
    long bpf_get_func_arg_cnt(void *ctx)

v2 changes:
  - added acks
  - updated stack diagram
  - return -EOPNOTSUPP instead of -EINVAL in bpf_get_func_ret
  - removed gpl_only for all helpers
  - added verifier fix to allow proper arguments checks,
    Andrii asked for checking also 'int *b' argument in
    bpf_modify_return_test programs and it turned out that it's currently
    not supported by verifier - we can't read argument that is int pointer,
    so I had to add verifier change to allow that + adding verifier selftest
  - checking all arguments in bpf_modify_return_test test programs
  - moved helpers proto gets in tracing_prog_func_proto with attach type check

thanks,
jirka

[1] https://lore.kernel.org/bpf/20211118112455.475349-1-jolsa@kernel.org/


====================

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 4b443bc1 006004b7

arch/x86/net/bpf_jit_comp.c

+41 −14
Original line number Diff line number Diff line
@@ -1941,7 +1941,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 
				void *orig_call)

{

	int ret, i, nr_args = m->nr_args;

	int stack_size = nr_args * 8;

	int regs_off, ip_off, args_off, stack_size = nr_args * 8;

	struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY];

	struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT];

	struct bpf_tramp_progs *fmod_ret = &tprogs[BPF_TRAMP_MODIFY_RETURN];
@@ -1956,14 +1956,39 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 
	if (!is_valid_bpf_tramp_flags(flags))

		return -EINVAL;



	/* Generated trampoline stack layout:

	 *

	 * RBP + 8         [ return address  ]

	 * RBP + 0         [ RBP             ]

	 *

	 * RBP - 8         [ return value    ]  BPF_TRAMP_F_CALL_ORIG or

	 *                                      BPF_TRAMP_F_RET_FENTRY_RET flags

	 *

	 *                 [ reg_argN        ]  always

	 *                 [ ...             ]

	 * RBP - regs_off  [ reg_arg1        ]  program's ctx pointer

	 *

	 * RBP - args_off  [ args count      ]  always

	 *

	 * RBP - ip_off    [ traced function ]  BPF_TRAMP_F_IP_ARG flag

	 */



	/* room for return value of orig_call or fentry prog */

	save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET);

	if (save_ret)

		stack_size += 8;



	regs_off = stack_size;



	/* args count  */

	stack_size += 8;

	args_off = stack_size;



	if (flags & BPF_TRAMP_F_IP_ARG)

		stack_size += 8; /* room for IP address argument */



	ip_off = stack_size;



	if (flags & BPF_TRAMP_F_SKIP_FRAME)

		/* skip patched call instruction and point orig_call to actual

		 * body of the kernel function.
@@ -1977,23 +2002,25 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 
	EMIT4(0x48, 0x83, 0xEC, stack_size); /* sub rsp, stack_size */

	EMIT1(0x53);		 /* push rbx */



	/* Store number of arguments of the traced function:

	 *   mov rax, nr_args

	 *   mov QWORD PTR [rbp - args_off], rax

	 */

	emit_mov_imm64(&prog, BPF_REG_0, 0, (u32) nr_args);

	emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -args_off);



	if (flags & BPF_TRAMP_F_IP_ARG) {

		/* Store IP address of the traced function:

		 * mov rax, QWORD PTR [rbp + 8]

		 * sub rax, X86_PATCH_SIZE

		 * mov QWORD PTR [rbp - stack_size], rax

		 * mov QWORD PTR [rbp - ip_off], rax

		 */

		emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, 8);

		EMIT4(0x48, 0x83, 0xe8, X86_PATCH_SIZE);

		emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -stack_size);



		/* Continue with stack_size for regs storage, stack will

		 * be correctly restored with 'leave' instruction.

		 */

		stack_size -= 8;

		emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -ip_off);

	}



	save_regs(m, &prog, nr_args, stack_size);

	save_regs(m, &prog, nr_args, regs_off);



	if (flags & BPF_TRAMP_F_CALL_ORIG) {

		/* arg1: mov rdi, im */
@@ -2005,7 +2032,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 
	}



	if (fentry->nr_progs)

		if (invoke_bpf(m, &prog, fentry, stack_size,

		if (invoke_bpf(m, &prog, fentry, regs_off,

			       flags & BPF_TRAMP_F_RET_FENTRY_RET))

			return -EINVAL;
@@ -2015,7 +2042,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 
		if (!branches)

			return -ENOMEM;



		if (invoke_bpf_mod_ret(m, &prog, fmod_ret, stack_size,

		if (invoke_bpf_mod_ret(m, &prog, fmod_ret, regs_off,

				       branches)) {

			ret = -EINVAL;

			goto cleanup;
@@ -2023,7 +2050,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 
	}



	if (flags & BPF_TRAMP_F_CALL_ORIG) {

		restore_regs(m, &prog, nr_args, stack_size);

		restore_regs(m, &prog, nr_args, regs_off);



		/* call original function */

		if (emit_call(&prog, orig_call, prog)) {
@@ -2053,13 +2080,13 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 
	}



	if (fexit->nr_progs)

		if (invoke_bpf(m, &prog, fexit, stack_size, false)) {

		if (invoke_bpf(m, &prog, fexit, regs_off, false)) {

			ret = -EINVAL;

			goto cleanup;

		}



	if (flags & BPF_TRAMP_F_RESTORE_REGS)

		restore_regs(m, &prog, nr_args, stack_size);

		restore_regs(m, &prog, nr_args, regs_off);



	/* This needs to be done regardless. If there were fmod_ret programs,

	 * the return value is only updated on the stack and still needs to be

include/linux/bpf.h

+5 −0
Original line number Diff line number Diff line
@@ -777,6 +777,7 @@ void bpf_ksym_add(struct bpf_ksym *ksym); 
void bpf_ksym_del(struct bpf_ksym *ksym);

int bpf_jit_charge_modmem(u32 pages);

void bpf_jit_uncharge_modmem(u32 pages);

bool bpf_prog_has_trampoline(const struct bpf_prog *prog);

#else

static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,

					   struct bpf_trampoline *tr)
@@ -805,6 +806,10 @@ static inline bool is_bpf_image_address(unsigned long address) 
{

	return false;

}

static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog)

{

	return false;

}

#endif



struct bpf_func_info_aux {

include/uapi/linux/bpf.h

+28 −0
Original line number Diff line number Diff line
@@ -4993,6 +4993,31 @@ union bpf_attr { 
 *		An integer less than, equal to, or greater than zero

 *		if the first **s1_sz** bytes of **s1** is found to be

 *		less than, to match, or be greater than **s2**.

 *

 * long bpf_get_func_arg(void *ctx, u32 n, u64 *value)

 *	Description

 *		Get **n**-th argument (zero based) of the traced function (for tracing programs)

 *		returned in **value**.

 *

 *	Return

 *		0 on success.

 *		**-EINVAL** if n >= arguments count of traced function.

 *

 * long bpf_get_func_ret(void *ctx, u64 *value)

 *	Description

 *		Get return value of the traced function (for tracing programs)

 *		in **value**.

 *

 *	Return

 *		0 on success.

 *		**-EOPNOTSUPP** for tracing programs other than BPF_TRACE_FEXIT or BPF_MODIFY_RETURN.

 *

 * long bpf_get_func_arg_cnt(void *ctx)

 *	Description

 *		Get number of arguments of the traced function (for tracing programs).

 *

 *	Return

 *		The number of arguments of the traced function.

 */

#define __BPF_FUNC_MAPPER(FN)		\

	FN(unspec),			\
@@ -5178,6 +5203,9 @@ union bpf_attr { 
	FN(find_vma),			\

	FN(loop),			\

	FN(strncmp),			\

	FN(get_func_arg),		\

	FN(get_func_ret),		\

	FN(get_func_arg_cnt),		\

	/* */



/* integer value in 'imm' field of BPF_CALL instruction selects which helper

kernel/bpf/btf.c

+3 −4
Original line number Diff line number Diff line
@@ -4826,7 +4826,7 @@ struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog) 
		return prog->aux->attach_btf;

}



static bool is_string_ptr(struct btf *btf, const struct btf_type *t)

static bool is_int_ptr(struct btf *btf, const struct btf_type *t)

{

	/* t comes in already as a pointer */

	t = btf_type_by_id(btf, t->type);
@@ -4835,8 +4835,7 @@ static bool is_string_ptr(struct btf *btf, const struct btf_type *t) 
	if (BTF_INFO_KIND(t->info) == BTF_KIND_CONST)

		t = btf_type_by_id(btf, t->type);



	/* char, signed char, unsigned char */

	return btf_type_is_int(t) && t->size == 1;

	return btf_type_is_int(t);

}



bool btf_ctx_access(int off, int size, enum bpf_access_type type,
@@ -4957,7 +4956,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, 
		 */

		return true;



	if (is_string_ptr(btf, t))

	if (is_int_ptr(btf, t))

		return true;



	/* this is a pointer to another type */

kernel/bpf/trampoline.c

+8 −0
Original line number Diff line number Diff line
@@ -27,6 +27,14 @@ static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE]; 
/* serializes access to trampoline_table */

static DEFINE_MUTEX(trampoline_mutex);



bool bpf_prog_has_trampoline(const struct bpf_prog *prog)

{

	enum bpf_attach_type eatype = prog->expected_attach_type;



	return eatype == BPF_TRACE_FENTRY || eatype == BPF_TRACE_FEXIT ||

	       eatype == BPF_MODIFY_RETURN;

}



void *bpf_jit_alloc_exec_page(void)

{

	void *image;