Commit f92c1e18 authored by Jiri Olsa's avatar Jiri Olsa Committed by Alexei Starovoitov
Browse files

bpf: Add get_func_[arg|ret|arg_cnt] helpers 



Adding following helpers for tracing programs:

Get n-th argument of the traced function:
  long bpf_get_func_arg(void *ctx, u32 n, u64 *value)

Get return value of the traced function:
  long bpf_get_func_ret(void *ctx, u64 *value)

Get arguments count of the traced function:
  long bpf_get_func_arg_cnt(void *ctx)

The trampoline now stores number of arguments on ctx-8
address, so it's easy to verify argument index and find
return value argument's position.

Moving function ip address on the trampoline stack behind
the number of functions arguments, so it's now stored on
ctx-16 address if it's needed.

All helpers above are inlined by verifier.

Also bit unrelated small change - using newly added function
bpf_prog_has_trampoline in check_get_func_ip.

Signed-off-by: default avatarJiri Olsa <jolsa@kernel.org>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/bpf/20211208193245.172141-5-jolsa@kernel.org
parent 5edf6a19

arch/x86/net/bpf_jit_comp.c

+14 −1
Original line number Diff line number Diff line
@@ -1941,7 +1941,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 
				void *orig_call)

{

	int ret, i, nr_args = m->nr_args;

	int regs_off, ip_off, stack_size = nr_args * 8;

	int regs_off, ip_off, args_off, stack_size = nr_args * 8;

	struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY];

	struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT];

	struct bpf_tramp_progs *fmod_ret = &tprogs[BPF_TRAMP_MODIFY_RETURN];
@@ -1968,6 +1968,8 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 
	 *                 [ ...             ]

	 * RBP - regs_off  [ reg_arg1        ]  program's ctx pointer

	 *

	 * RBP - args_off  [ args count      ]  always

	 *

	 * RBP - ip_off    [ traced function ]  BPF_TRAMP_F_IP_ARG flag

	 */
@@ -1978,6 +1980,10 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 


	regs_off = stack_size;



	/* args count  */

	stack_size += 8;

	args_off = stack_size;



	if (flags & BPF_TRAMP_F_IP_ARG)

		stack_size += 8; /* room for IP address argument */
@@ -1996,6 +2002,13 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 
	EMIT4(0x48, 0x83, 0xEC, stack_size); /* sub rsp, stack_size */

	EMIT1(0x53);		 /* push rbx */



	/* Store number of arguments of the traced function:

	 *   mov rax, nr_args

	 *   mov QWORD PTR [rbp - args_off], rax

	 */

	emit_mov_imm64(&prog, BPF_REG_0, 0, (u32) nr_args);

	emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -args_off);



	if (flags & BPF_TRAMP_F_IP_ARG) {

		/* Store IP address of the traced function:

		 * mov rax, QWORD PTR [rbp + 8]

include/linux/bpf.h

+5 −0
Original line number Diff line number Diff line
@@ -777,6 +777,7 @@ void bpf_ksym_add(struct bpf_ksym *ksym); 
void bpf_ksym_del(struct bpf_ksym *ksym);

int bpf_jit_charge_modmem(u32 pages);

void bpf_jit_uncharge_modmem(u32 pages);

bool bpf_prog_has_trampoline(const struct bpf_prog *prog);

#else

static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,

					   struct bpf_trampoline *tr)
@@ -805,6 +806,10 @@ static inline bool is_bpf_image_address(unsigned long address) 
{

	return false;

}

static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog)

{

	return false;

}

#endif



struct bpf_func_info_aux {

include/uapi/linux/bpf.h

+28 −0
Original line number Diff line number Diff line
@@ -4993,6 +4993,31 @@ union bpf_attr { 
 *		An integer less than, equal to, or greater than zero

 *		if the first **s1_sz** bytes of **s1** is found to be

 *		less than, to match, or be greater than **s2**.

 *

 * long bpf_get_func_arg(void *ctx, u32 n, u64 *value)

 *	Description

 *		Get **n**-th argument (zero based) of the traced function (for tracing programs)

 *		returned in **value**.

 *

 *	Return

 *		0 on success.

 *		**-EINVAL** if n >= arguments count of traced function.

 *

 * long bpf_get_func_ret(void *ctx, u64 *value)

 *	Description

 *		Get return value of the traced function (for tracing programs)

 *		in **value**.

 *

 *	Return

 *		0 on success.

 *		**-EOPNOTSUPP** for tracing programs other than BPF_TRACE_FEXIT or BPF_MODIFY_RETURN.

 *

 * long bpf_get_func_arg_cnt(void *ctx)

 *	Description

 *		Get number of arguments of the traced function (for tracing programs).

 *

 *	Return

 *		The number of arguments of the traced function.

 */

#define __BPF_FUNC_MAPPER(FN)		\

	FN(unspec),			\
@@ -5178,6 +5203,9 @@ union bpf_attr { 
	FN(find_vma),			\

	FN(loop),			\

	FN(strncmp),			\

	FN(get_func_arg),		\

	FN(get_func_ret),		\

	FN(get_func_arg_cnt),		\

	/* */



/* integer value in 'imm' field of BPF_CALL instruction selects which helper

kernel/bpf/trampoline.c

+8 −0
Original line number Diff line number Diff line
@@ -27,6 +27,14 @@ static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE]; 
/* serializes access to trampoline_table */

static DEFINE_MUTEX(trampoline_mutex);



bool bpf_prog_has_trampoline(const struct bpf_prog *prog)

{

	enum bpf_attach_type eatype = prog->expected_attach_type;



	return eatype == BPF_TRACE_FENTRY || eatype == BPF_TRACE_FEXIT ||

	       eatype == BPF_MODIFY_RETURN;

}



void *bpf_jit_alloc_exec_page(void)

{

	void *image;

kernel/bpf/verifier.c

+72 −5
Original line number Diff line number Diff line
@@ -6395,13 +6395,11 @@ static int check_bpf_snprintf_call(struct bpf_verifier_env *env, 


static int check_get_func_ip(struct bpf_verifier_env *env)

{

	enum bpf_attach_type eatype = env->prog->expected_attach_type;

	enum bpf_prog_type type = resolve_prog_type(env->prog);

	int func_id = BPF_FUNC_get_func_ip;



	if (type == BPF_PROG_TYPE_TRACING) {

		if (eatype != BPF_TRACE_FENTRY && eatype != BPF_TRACE_FEXIT &&

		    eatype != BPF_MODIFY_RETURN) {

		if (!bpf_prog_has_trampoline(env->prog)) {

			verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n",

				func_id_name(func_id), func_id);

			return -ENOTSUPP;
@@ -12997,6 +12995,7 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, 
static int do_misc_fixups(struct bpf_verifier_env *env)

{

	struct bpf_prog *prog = env->prog;

	enum bpf_attach_type eatype = prog->expected_attach_type;

	bool expect_blinding = bpf_jit_blinding_enabled(prog);

	enum bpf_prog_type prog_type = resolve_prog_type(prog);

	struct bpf_insn *insn = prog->insnsi;
@@ -13367,11 +13366,79 @@ static int do_misc_fixups(struct bpf_verifier_env *env) 
			continue;

		}



		/* Implement bpf_get_func_arg inline. */

		if (prog_type == BPF_PROG_TYPE_TRACING &&

		    insn->imm == BPF_FUNC_get_func_arg) {

			/* Load nr_args from ctx - 8 */

			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);

			insn_buf[1] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6);

			insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3);

			insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1);

			insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0);

			insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0);

			insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0);

			insn_buf[7] = BPF_JMP_A(1);

			insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL);

			cnt = 9;



			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

			if (!new_prog)

				return -ENOMEM;



			delta    += cnt - 1;

			env->prog = prog = new_prog;

			insn      = new_prog->insnsi + i + delta;

			continue;

		}



		/* Implement bpf_get_func_ret inline. */

		if (prog_type == BPF_PROG_TYPE_TRACING &&

		    insn->imm == BPF_FUNC_get_func_ret) {

			if (eatype == BPF_TRACE_FEXIT ||

			    eatype == BPF_MODIFY_RETURN) {

				/* Load nr_args from ctx - 8 */

				insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);

				insn_buf[1] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3);

				insn_buf[2] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1);

				insn_buf[3] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0);

				insn_buf[4] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0);

				insn_buf[5] = BPF_MOV64_IMM(BPF_REG_0, 0);

				cnt = 6;

			} else {

				insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP);

				cnt = 1;

			}



			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

			if (!new_prog)

				return -ENOMEM;



			delta    += cnt - 1;

			env->prog = prog = new_prog;

			insn      = new_prog->insnsi + i + delta;

			continue;

		}



		/* Implement get_func_arg_cnt inline. */

		if (prog_type == BPF_PROG_TYPE_TRACING &&

		    insn->imm == BPF_FUNC_get_func_arg_cnt) {

			/* Load nr_args from ctx - 8 */

			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);



			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);

			if (!new_prog)

				return -ENOMEM;



			env->prog = prog = new_prog;

			insn      = new_prog->insnsi + i + delta;

			continue;

		}



		/* Implement bpf_get_func_ip inline. */

		if (prog_type == BPF_PROG_TYPE_TRACING &&

		    insn->imm == BPF_FUNC_get_func_ip) {

			/* Load IP address from ctx - 8 */

			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);

			/* Load IP address from ctx - 16 */

			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16);



			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);

			if (!new_prog)