Commit dbe69b29 authored by Jiri Olsa's avatar Jiri Olsa Committed by Alexei Starovoitov
Browse files

bpf: Fix dispatcher patchable function entry to 5 bytes nop 



The patchable_function_entry(5) might output 5 single nop
instructions (depends on toolchain), which will clash with
bpf_arch_text_poke check for 5 bytes nop instruction.

Adding early init call for dispatcher that checks and change
the patchable entry into expected 5 nop instruction if needed.

There's no need to take text_mutex, because we are using it
in early init call which is called at pre-smp time.

Fixes: ceea991a ("bpf: Move bpf_dispatcher function out of ftrace locations")
Signed-off-by: default avatarJiri Olsa <jolsa@kernel.org>
Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20221018075934.574415-1-jolsa@kernel.org


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent ea68376c

arch/x86/net/bpf_jit_comp.c

+13 −0
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ 
#include <linux/bpf.h>

#include <linux/memory.h>

#include <linux/sort.h>

#include <linux/init.h>

#include <asm/extable.h>

#include <asm/set_memory.h>

#include <asm/nospec-branch.h>
@@ -388,6 +389,18 @@ static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, 
	return ret;

}



int __init bpf_arch_init_dispatcher_early(void *ip)

{

	const u8 *nop_insn = x86_nops[5];



	if (is_endbr(*(u32 *)ip))

		ip += ENDBR_INSN_SIZE;



	if (memcmp(ip, nop_insn, X86_PATCH_SIZE))

		text_poke_early(ip, nop_insn, X86_PATCH_SIZE);

	return 0;

}



int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,

		       void *old_addr, void *new_addr)

{

include/linux/bpf.h

+13 −1
Original line number Diff line number Diff line
@@ -27,6 +27,7 @@ 
#include <linux/bpfptr.h>

#include <linux/btf.h>

#include <linux/rcupdate_trace.h>

#include <linux/init.h>



struct bpf_verifier_env;

struct bpf_verifier_log;
@@ -970,6 +971,8 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key, 
					  struct bpf_attach_target_info *tgt_info);

void bpf_trampoline_put(struct bpf_trampoline *tr);

int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs);

int __init bpf_arch_init_dispatcher_early(void *ip);



#define BPF_DISPATCHER_INIT(_name) {				\

	.mutex = __MUTEX_INITIALIZER(_name.mutex),		\

	.func = &_name##_func,					\
@@ -983,6 +986,13 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func 
	},							\

}



#define BPF_DISPATCHER_INIT_CALL(_name)					\

	static int __init _name##_init(void)				\

	{								\

		return bpf_arch_init_dispatcher_early(_name##_func);	\

	}								\

	early_initcall(_name##_init)



#ifdef CONFIG_X86_64

#define BPF_DISPATCHER_ATTRIBUTES __attribute__((patchable_function_entry(5)))

#else
@@ -1000,7 +1010,9 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func 
	}								\

	EXPORT_SYMBOL(bpf_dispatcher_##name##_func);			\

	struct bpf_dispatcher bpf_dispatcher_##name =			\

		BPF_DISPATCHER_INIT(bpf_dispatcher_##name);

		BPF_DISPATCHER_INIT(bpf_dispatcher_##name);		\

	BPF_DISPATCHER_INIT_CALL(bpf_dispatcher_##name);



#define DECLARE_BPF_DISPATCHER(name)					\

	unsigned int bpf_dispatcher_##name##_func(			\

		const void *ctx,					\

kernel/bpf/dispatcher.c

+6 −0
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ 
#include <linux/hash.h>

#include <linux/bpf.h>

#include <linux/filter.h>

#include <linux/init.h>



/* The BPF dispatcher is a multiway branch code generator. The

 * dispatcher is a mechanism to avoid the performance penalty of an
@@ -90,6 +91,11 @@ int __weak arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int n 
	return -ENOTSUPP;

}



int __weak __init bpf_arch_init_dispatcher_early(void *ip)

{

	return -ENOTSUPP;

}



static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image, void *buf)

{

	s64 ips[BPF_DISPATCHER_MAX] = {}, *ipsp = &ips[0];