Commit 1554a080 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'Add bpf_get_func_ip helper' 

Jiri Olsa says:

====================
Add bpf_get_func_ip helper that returns IP address of the
caller function for trampoline and krobe programs.

There're 2 specific implementation of the bpf_get_func_ip
helper, one for trampoline progs and one for kprobe/kretprobe
progs.

The trampoline helper call is replaced/inlined by the verifier
with simple move instruction. The kprobe/kretprobe is actual
helper call that returns prepared caller address.

Also available at:
  https://git.kernel.org/pub/scm/linux/kernel/git/jolsa/perf.git


  bpf/get_func_ip

v4 changes:
  - dropped jit/x86 check for get_func_ip tracing check [Alexei]
  - added code to bpf_get_func_ip_tracing [Alexei]
    and tested that it works without inlining [Alexei]
  - changed has_get_func_ip to check_get_func_ip [Andrii]
  - replaced test assert loop with explicit asserts [Andrii]
  - adde bpf_program__attach_kprobe_opts function
    and use it for offset setup [Andrii]
  - used bpf_program__set_autoload(false) for test6 [Andrii]
  - added Masami's ack

v3 changes:
  - resend with Masami in cc and v3 in each patch subject

v2 changes:
  - use kprobe_running to get kprobe instead of cpu var [Masami]
  - added support to add kprobe on function+offset
    and test for that [Alan]
====================

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 76283171 8237e754

arch/x86/net/bpf_jit_comp.c

+19 −0
Original line number Diff line number Diff line
@@ -1951,6 +1951,9 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 
	if (flags & BPF_TRAMP_F_CALL_ORIG)

		stack_size += 8; /* room for return value of orig_call */



	if (flags & BPF_TRAMP_F_IP_ARG)

		stack_size += 8; /* room for IP address argument */



	if (flags & BPF_TRAMP_F_SKIP_FRAME)

		/* skip patched call instruction and point orig_call to actual

		 * body of the kernel function.
@@ -1964,6 +1967,22 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 
	EMIT4(0x48, 0x83, 0xEC, stack_size); /* sub rsp, stack_size */

	EMIT1(0x53);		 /* push rbx */



	if (flags & BPF_TRAMP_F_IP_ARG) {

		/* Store IP address of the traced function:

		 * mov rax, QWORD PTR [rbp + 8]

		 * sub rax, X86_PATCH_SIZE

		 * mov QWORD PTR [rbp - stack_size], rax

		 */

		emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, 8);

		EMIT4(0x48, 0x83, 0xe8, X86_PATCH_SIZE);

		emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -stack_size);



		/* Continue with stack_size for regs storage, stack will

		 * be correctly restored with 'leave' instruction.

		 */

		stack_size -= 8;

	}



	save_regs(m, &prog, nr_args, stack_size);



	if (flags & BPF_TRAMP_F_CALL_ORIG) {

include/linux/bpf.h

+5 −0
Original line number Diff line number Diff line
@@ -579,6 +579,11 @@ struct btf_func_model { 
 */

#define BPF_TRAMP_F_SKIP_FRAME		BIT(2)



/* Store IP address of the caller on the trampoline stack,

 * so it's available for trampoline's programs.

 */

#define BPF_TRAMP_F_IP_ARG		BIT(3)



/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50

 * bytes on x86.  Pick a number to fit into BPF_IMAGE_SIZE / 2

 */

include/linux/filter.h

+2 −1
Original line number Diff line number Diff line
@@ -559,7 +559,8 @@ struct bpf_prog { 
				kprobe_override:1, /* Do we override a kprobe? */

				has_callchain_buf:1, /* callchain buffer allocated? */

				enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */

				call_get_stack:1; /* Do we call bpf_get_stack() or bpf_get_stackid() */

				call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */

				call_get_func_ip:1; /* Do we call get_func_ip() */

	enum bpf_prog_type	type;		/* Type of BPF program */

	enum bpf_attach_type	expected_attach_type; /* For some prog types */

	u32			len;		/* Number of filter blocks */

include/uapi/linux/bpf.h

+7 −0
Original line number Diff line number Diff line
@@ -4841,6 +4841,12 @@ union bpf_attr { 
 *		**-EINVAL** if *timer* was not initialized with bpf_timer_init() earlier.

 *		**-EDEADLK** if callback_fn tried to call bpf_timer_cancel() on its

 *		own timer which would have led to a deadlock otherwise.

 *

 * u64 bpf_get_func_ip(void *ctx)

 * 	Description

 * 		Get address of the traced function (for tracing and kprobe programs).

 * 	Return

 * 		Address of the traced function.

 */

#define __BPF_FUNC_MAPPER(FN)		\

	FN(unspec),			\
@@ -5016,6 +5022,7 @@ union bpf_attr { 
	FN(timer_set_callback),		\

	FN(timer_start),		\

	FN(timer_cancel),		\

	FN(get_func_ip),		\

	/* */



/* integer value in 'imm' field of BPF_CALL instruction selects which helper

kernel/bpf/trampoline.c

+9 −3
Original line number Diff line number Diff line
@@ -172,7 +172,7 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr) 
}



static struct bpf_tramp_progs *

bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total)

bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_arg)

{

	const struct bpf_prog_aux *aux;

	struct bpf_tramp_progs *tprogs;
@@ -189,9 +189,11 @@ bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total) 
		*total += tr->progs_cnt[kind];

		progs = tprogs[kind].progs;



		hlist_for_each_entry(aux, &tr->progs_hlist[kind], tramp_hlist)

		hlist_for_each_entry(aux, &tr->progs_hlist[kind], tramp_hlist) {

			*ip_arg |= aux->prog->call_get_func_ip;

			*progs++ = aux->prog;

		}

	}

	return tprogs;

}
@@ -333,9 +335,10 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) 
	struct bpf_tramp_image *im;

	struct bpf_tramp_progs *tprogs;

	u32 flags = BPF_TRAMP_F_RESTORE_REGS;

	bool ip_arg = false;

	int err, total;



	tprogs = bpf_trampoline_get_progs(tr, &total);

	tprogs = bpf_trampoline_get_progs(tr, &total, &ip_arg);

	if (IS_ERR(tprogs))

		return PTR_ERR(tprogs);
@@ -357,6 +360,9 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) 
	    tprogs[BPF_TRAMP_MODIFY_RETURN].nr_progs)

		flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME;



	if (ip_arg)

		flags |= BPF_TRAMP_F_IP_ARG;



	err = arch_prepare_bpf_trampoline(im, im->image, im->image + PAGE_SIZE,

					  &tr->func.model, flags, tprogs,

					  tr->func.addr);