Commit 271de525 authored by Martin KaFai Lau's avatar Martin KaFai Lau Committed by Alexei Starovoitov
Browse files

bpf: Remove prog->active check for bpf_lsm and bpf_iter 



The commit 64696c40 ("bpf: Add __bpf_prog_{enter,exit}_struct_ops for struct_ops trampoline")
removed prog->active check for struct_ops prog.  The bpf_lsm
and bpf_iter is also using trampoline.  Like struct_ops, the bpf_lsm
and bpf_iter have fixed hooks for the prog to attach.  The
kernel does not call the same hook in a recursive way.
This patch also removes the prog->active check for
bpf_lsm and bpf_iter.

A later patch has a test to reproduce the recursion issue
for a sleepable bpf_lsm program.

This patch appends the '_recur' naming to the existing
enter and exit functions that track the prog->active counter.
New __bpf_prog_{enter,exit}[_sleepable] function are
added to skip the prog->active tracking. The '_struct_ops'
version is also removed.

It also moves the decision on picking the enter and exit function to
the new bpf_trampoline_{enter,exit}().  It returns the '_recur' ones
for all tracing progs to use.  For bpf_lsm, bpf_iter,
struct_ops (no prog->active tracking after 64696c40), and
bpf_lsm_cgroup (no prog->active tracking after 69fd337a),
it will return the functions that don't track the prog->active.

Signed-off-by: default avatarMartin KaFai Lau <martin.lau@kernel.org>
Link: https://lore.kernel.org/r/20221025184524.3526117-2-martin.lau@linux.dev


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent f3c51fe0

arch/arm64/net/bpf_jit_comp.c

+2 −7
Original line number Diff line number Diff line
@@ -1649,13 +1649,8 @@ static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l, 
	struct bpf_prog *p = l->link.prog;

	int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);



	if (p->aux->sleepable) {

		enter_prog = (u64)__bpf_prog_enter_sleepable;

		exit_prog = (u64)__bpf_prog_exit_sleepable;

	} else {

		enter_prog = (u64)__bpf_prog_enter;

		exit_prog = (u64)__bpf_prog_exit;

	}

	enter_prog = (u64)bpf_trampoline_enter(p);

	exit_prog = (u64)bpf_trampoline_exit(p);



	if (l->cookie == 0) {

		/* if cookie is zero, one instruction is enough to store it */

arch/x86/net/bpf_jit_comp.c

+2 −17
Original line number Diff line number Diff line
@@ -1894,10 +1894,6 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, 
			   struct bpf_tramp_link *l, int stack_size,

			   int run_ctx_off, bool save_ret)

{

	void (*exit)(struct bpf_prog *prog, u64 start,

		     struct bpf_tramp_run_ctx *run_ctx) = __bpf_prog_exit;

	u64 (*enter)(struct bpf_prog *prog,

		     struct bpf_tramp_run_ctx *run_ctx) = __bpf_prog_enter;

	u8 *prog = *pprog;

	u8 *jmp_insn;

	int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
@@ -1916,23 +1912,12 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, 
	 */

	emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_1, -run_ctx_off + ctx_cookie_off);



	if (p->aux->sleepable) {

		enter = __bpf_prog_enter_sleepable;

		exit = __bpf_prog_exit_sleepable;

	} else if (p->type == BPF_PROG_TYPE_STRUCT_OPS) {

		enter = __bpf_prog_enter_struct_ops;

		exit = __bpf_prog_exit_struct_ops;

	} else if (p->expected_attach_type == BPF_LSM_CGROUP) {

		enter = __bpf_prog_enter_lsm_cgroup;

		exit = __bpf_prog_exit_lsm_cgroup;

	}



	/* arg1: mov rdi, progs[i] */

	emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);

	/* arg2: lea rsi, [rbp - ctx_cookie_off] */

	EMIT4(0x48, 0x8D, 0x75, -run_ctx_off);



	if (emit_call(&prog, enter, prog))

	if (emit_call(&prog, bpf_trampoline_enter(p), prog))

		return -EINVAL;

	/* remember prog start time returned by __bpf_prog_enter */

	emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0);
@@ -1977,7 +1962,7 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, 
	emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);

	/* arg3: lea rdx, [rbp - run_ctx_off] */

	EMIT4(0x48, 0x8D, 0x55, -run_ctx_off);

	if (emit_call(&prog, exit, prog))

	if (emit_call(&prog, bpf_trampoline_exit(p), prog))

		return -EINVAL;



	*pprog = prog;

include/linux/bpf.h

+10 −14
Original line number Diff line number Diff line
@@ -854,22 +854,18 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *i 
				const struct btf_func_model *m, u32 flags,

				struct bpf_tramp_links *tlinks,

				void *orig_call);

/* these two functions are called from generated trampoline */

u64 notrace __bpf_prog_enter(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx);

void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_run_ctx *run_ctx);

u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx);

void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,

u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,

					     struct bpf_tramp_run_ctx *run_ctx);

u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,

					struct bpf_tramp_run_ctx *run_ctx);

void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start,

					struct bpf_tramp_run_ctx *run_ctx);

u64 notrace __bpf_prog_enter_struct_ops(struct bpf_prog *prog,

					struct bpf_tramp_run_ctx *run_ctx);

void notrace __bpf_prog_exit_struct_ops(struct bpf_prog *prog, u64 start,

void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start,

					     struct bpf_tramp_run_ctx *run_ctx);

void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr);

void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr);

typedef u64 (*bpf_trampoline_enter_t)(struct bpf_prog *prog,

				      struct bpf_tramp_run_ctx *run_ctx);

typedef void (*bpf_trampoline_exit_t)(struct bpf_prog *prog, u64 start,

				      struct bpf_tramp_run_ctx *run_ctx);

bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog);

bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog);



struct bpf_ksym {

	unsigned long		 start;

include/linux/bpf_verifier.h

+14 −1
Original line number Diff line number Diff line
@@ -642,10 +642,23 @@ static inline u32 type_flag(u32 type) 
}



/* only use after check_attach_btf_id() */

static inline enum bpf_prog_type resolve_prog_type(struct bpf_prog *prog)

static inline enum bpf_prog_type resolve_prog_type(const struct bpf_prog *prog)

{

	return prog->type == BPF_PROG_TYPE_EXT ?

		prog->aux->dst_prog->type : prog->type;

}



static inline bool bpf_prog_check_recur(const struct bpf_prog *prog)

{

	switch (resolve_prog_type(prog)) {

	case BPF_PROG_TYPE_TRACING:

		return prog->expected_attach_type != BPF_TRACE_ITER;

	case BPF_PROG_TYPE_STRUCT_OPS:

	case BPF_PROG_TYPE_LSM:

		return false;

	default:

		return true;

	}

}



#endif /* _LINUX_BPF_VERIFIER_H */

kernel/bpf/syscall.c

+3 −2
Original line number Diff line number Diff line
@@ -5133,13 +5133,14 @@ int kern_sys_bpf(int cmd, union bpf_attr *attr, unsigned int size) 


		run_ctx.bpf_cookie = 0;

		run_ctx.saved_run_ctx = NULL;

		if (!__bpf_prog_enter_sleepable(prog, &run_ctx)) {

		if (!__bpf_prog_enter_sleepable_recur(prog, &run_ctx)) {

			/* recursion detected */

			bpf_prog_put(prog);

			return -EBUSY;

		}

		attr->test.retval = bpf_prog_run(prog, (void *) (long) attr->test.ctx_in);

		__bpf_prog_exit_sleepable(prog, 0 /* bpf_prog_run does runtime stats */, &run_ctx);

		__bpf_prog_exit_sleepable_recur(prog, 0 /* bpf_prog_run does runtime stats */,

						&run_ctx);

		bpf_prog_put(prog);

		return 0;

#endif