Commit 93dafa92 authored by Andrii Nakryiko's avatar Andrii Nakryiko
Browse files

Merge branch 'Attach a cookie to a tracing program.' 

Kui-Feng Lee says:

====================

Allow users to attach a 64-bits cookie to a bpf_link of fentry, fexit,
or fmod_ret.

This patchset includes several major changes.

 - Define struct bpf_tramp_links to replace bpf_tramp_prog.
   struct bpf_tramp_links collects bpf_links of a trampoline

 - Generate a trampoline to call bpf_progs of given bpf_links.

 - Trampolines always set/reset bpf_run_ctx before/after
   calling/leaving a tracing program.

 - Attach a cookie to a bpf_link of fentry/fexit/fmod_ret/lsm.  The
   value will be available when running the associated bpf_prog.

Th major differences from v6:

 - bpf_link_create() can create links of BPF_LSM_MAC attach type.

 - Add a test for lsm.

 - Add function proto of bpf_get_attach_cookie() for lsm.

 - Check BPF_LSM_MAC in bpf_prog_has_trampoline().

 - Adapt to the changes of LINK_CREATE made by Andrii.

The major differences from v7:

 - Change stack_size instead of pushing/popping run_ctx.

 - Move cookie to bpf_tramp_link from bpf_tracing_link..

v1: https://lore.kernel.org/all/20220126214809.3868787-1-kuifeng@fb.com/
v2: https://lore.kernel.org/bpf/20220316004231.1103318-1-kuifeng@fb.com/
v3: https://lore.kernel.org/bpf/20220407192552.2343076-1-kuifeng@fb.com/
v4: https://lore.kernel.org/bpf/20220411173429.4139609-1-kuifeng@fb.com/
v5: https://lore.kernel.org/bpf/20220412165555.4146407-1-kuifeng@fb.com/
v6: https://lore.kernel.org/bpf/20220416042940.656344-1-kuifeng@fb.com/
v7: https://lore.kernel.org/bpf/20220508032117.2783209-1-kuifeng@fb.com/


====================

Signed-off-by: default avatarAndrii Nakryiko <andrii@kernel.org>
parents cb411545 ddc0027a

arch/x86/net/bpf_jit_comp.c

+50 −24
Original line number Diff line number Diff line
@@ -1762,13 +1762,32 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args, 
}



static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,

			   struct bpf_prog *p, int stack_size, bool save_ret)

			   struct bpf_tramp_link *l, int stack_size,

			   int run_ctx_off, bool save_ret)

{

	u8 *prog = *pprog;

	u8 *jmp_insn;

	int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);

	struct bpf_prog *p = l->link.prog;

	u64 cookie = l->cookie;



	/* mov rdi, cookie */

	emit_mov_imm64(&prog, BPF_REG_1, (long) cookie >> 32, (u32) (long) cookie);



	/* Prepare struct bpf_tramp_run_ctx.

	 *

	 * bpf_tramp_run_ctx is already preserved by

	 * arch_prepare_bpf_trampoline().

	 *

	 * mov QWORD PTR [rbp - run_ctx_off + ctx_cookie_off], rdi

	 */

	emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_1, -run_ctx_off + ctx_cookie_off);



	/* arg1: mov rdi, progs[i] */

	emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);

	/* arg2: lea rsi, [rbp - ctx_cookie_off] */

	EMIT4(0x48, 0x8D, 0x75, -run_ctx_off);



	if (emit_call(&prog,

		      p->aux->sleepable ? __bpf_prog_enter_sleepable :

		      __bpf_prog_enter, prog))
@@ -1814,6 +1833,8 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, 
	emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);

	/* arg2: mov rsi, rbx <- start time in nsec */

	emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);

	/* arg3: lea rdx, [rbp - run_ctx_off] */

	EMIT4(0x48, 0x8D, 0x55, -run_ctx_off);

	if (emit_call(&prog,

		      p->aux->sleepable ? __bpf_prog_exit_sleepable :

		      __bpf_prog_exit, prog))
@@ -1850,15 +1871,15 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond) 
}



static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,

		      struct bpf_tramp_progs *tp, int stack_size,

		      bool save_ret)

		      struct bpf_tramp_links *tl, int stack_size,

		      int run_ctx_off, bool save_ret)

{

	int i;

	u8 *prog = *pprog;



	for (i = 0; i < tp->nr_progs; i++) {

		if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size,

				    save_ret))

	for (i = 0; i < tl->nr_links; i++) {

		if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size,

				    run_ctx_off, save_ret))

			return -EINVAL;

	}

	*pprog = prog;
@@ -1866,8 +1887,8 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, 
}



static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,

			      struct bpf_tramp_progs *tp, int stack_size,

			      u8 **branches)

			      struct bpf_tramp_links *tl, int stack_size,

			      int run_ctx_off, u8 **branches)

{

	u8 *prog = *pprog;

	int i;
@@ -1877,8 +1898,8 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, 
	 */

	emit_mov_imm32(&prog, false, BPF_REG_0, 0);

	emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);

	for (i = 0; i < tp->nr_progs; i++) {

		if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size, true))

	for (i = 0; i < tl->nr_links; i++) {

		if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size, run_ctx_off, true))

			return -EINVAL;



		/* mod_ret prog stored return value into [rbp - 8]. Emit:
@@ -1980,14 +2001,14 @@ static bool is_valid_bpf_tramp_flags(unsigned int flags) 
 */

int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,

				const struct btf_func_model *m, u32 flags,

				struct bpf_tramp_progs *tprogs,

				struct bpf_tramp_links *tlinks,

				void *orig_call)

{

	int ret, i, nr_args = m->nr_args;

	int regs_off, ip_off, args_off, stack_size = nr_args * 8;

	struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY];

	struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT];

	struct bpf_tramp_progs *fmod_ret = &tprogs[BPF_TRAMP_MODIFY_RETURN];

	int regs_off, ip_off, args_off, stack_size = nr_args * 8, run_ctx_off;

	struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];

	struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];

	struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];

	u8 **branches = NULL;

	u8 *prog;

	bool save_ret;
@@ -2014,6 +2035,8 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 
	 * RBP - args_off  [ args count      ]  always

	 *

	 * RBP - ip_off    [ traced function ]  BPF_TRAMP_F_IP_ARG flag

	 *

	 * RBP - run_ctx_off [ bpf_tramp_run_ctx ]

	 */



	/* room for return value of orig_call or fentry prog */
@@ -2032,6 +2055,9 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 


	ip_off = stack_size;



	stack_size += (sizeof(struct bpf_tramp_run_ctx) + 7) & ~0x7;

	run_ctx_off = stack_size;



	if (flags & BPF_TRAMP_F_SKIP_FRAME) {

		/* skip patched call instruction and point orig_call to actual

		 * body of the kernel function.
@@ -2078,19 +2104,19 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 
		}

	}



	if (fentry->nr_progs)

		if (invoke_bpf(m, &prog, fentry, regs_off,

	if (fentry->nr_links)

		if (invoke_bpf(m, &prog, fentry, regs_off, run_ctx_off,

			       flags & BPF_TRAMP_F_RET_FENTRY_RET))

			return -EINVAL;



	if (fmod_ret->nr_progs) {

		branches = kcalloc(fmod_ret->nr_progs, sizeof(u8 *),

	if (fmod_ret->nr_links) {

		branches = kcalloc(fmod_ret->nr_links, sizeof(u8 *),

				   GFP_KERNEL);

		if (!branches)

			return -ENOMEM;



		if (invoke_bpf_mod_ret(m, &prog, fmod_ret, regs_off,

				       branches)) {

				       run_ctx_off, branches)) {

			ret = -EINVAL;

			goto cleanup;

		}
@@ -2111,7 +2137,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 
		prog += X86_PATCH_SIZE;

	}



	if (fmod_ret->nr_progs) {

	if (fmod_ret->nr_links) {

		/* From Intel 64 and IA-32 Architectures Optimization

		 * Reference Manual, 3.4.1.4 Code Alignment, Assembly/Compiler

		 * Coding Rule 11: All branch targets should be 16-byte
@@ -2121,13 +2147,13 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 
		/* Update the branches saved in invoke_bpf_mod_ret with the

		 * aligned address of do_fexit.

		 */

		for (i = 0; i < fmod_ret->nr_progs; i++)

		for (i = 0; i < fmod_ret->nr_links; i++)

			emit_cond_near_jump(&branches[i], prog, branches[i],

					    X86_JNE);

	}



	if (fexit->nr_progs)

		if (invoke_bpf(m, &prog, fexit, regs_off, false)) {

	if (fexit->nr_links)

		if (invoke_bpf(m, &prog, fexit, regs_off, run_ctx_off, false)) {

			ret = -EINVAL;

			goto cleanup;

		}

include/linux/bpf.h

+38 −16
Original line number Diff line number Diff line
@@ -723,13 +723,15 @@ struct btf_func_model { 
/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50

 * bytes on x86.  Pick a number to fit into BPF_IMAGE_SIZE / 2

 */

#define BPF_MAX_TRAMP_PROGS 38

#define BPF_MAX_TRAMP_LINKS 38



struct bpf_tramp_progs {

	struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS];

	int nr_progs;

struct bpf_tramp_links {

	struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS];

	int nr_links;

};



struct bpf_tramp_run_ctx;



/* Different use cases for BPF trampoline:

 * 1. replace nop at the function entry (kprobe equivalent)

 *    flags = BPF_TRAMP_F_RESTORE_REGS
@@ -753,13 +755,14 @@ struct bpf_tramp_progs { 
struct bpf_tramp_image;

int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end,

				const struct btf_func_model *m, u32 flags,

				struct bpf_tramp_progs *tprogs,

				struct bpf_tramp_links *tlinks,

				void *orig_call);

/* these two functions are called from generated trampoline */

u64 notrace __bpf_prog_enter(struct bpf_prog *prog);

void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start);

u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog);

void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start);

u64 notrace __bpf_prog_enter(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx);

void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_run_ctx *run_ctx);

u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx);

void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,

				       struct bpf_tramp_run_ctx *run_ctx);

void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr);

void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr);
@@ -852,9 +855,10 @@ static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( 
{

	return bpf_func(ctx, insnsi);

}



#ifdef CONFIG_BPF_JIT

int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);

int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);

int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);

int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);

struct bpf_trampoline *bpf_trampoline_get(u64 key,

					  struct bpf_attach_target_info *tgt_info);

void bpf_trampoline_put(struct bpf_trampoline *tr);
@@ -905,12 +909,12 @@ int bpf_jit_charge_modmem(u32 size); 
void bpf_jit_uncharge_modmem(u32 size);

bool bpf_prog_has_trampoline(const struct bpf_prog *prog);

#else

static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,

static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,

					   struct bpf_trampoline *tr)

{

	return -ENOTSUPP;

}

static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog,

static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,

					     struct bpf_trampoline *tr)

{

	return -ENOTSUPP;
@@ -1009,7 +1013,6 @@ struct bpf_prog_aux { 
	bool tail_call_reachable;

	bool xdp_has_frags;

	bool use_bpf_prog_pack;

	struct hlist_node tramp_hlist;

	/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */

	const struct btf_type *attach_func_proto;

	/* function name for valid attach_btf_id */
@@ -1096,6 +1099,19 @@ struct bpf_link_ops { 
			      struct bpf_link_info *info);

};



struct bpf_tramp_link {

	struct bpf_link link;

	struct hlist_node tramp_hlist;

	u64 cookie;

};



struct bpf_tracing_link {

	struct bpf_tramp_link link;

	enum bpf_attach_type attach_type;

	struct bpf_trampoline *trampoline;

	struct bpf_prog *tgt_prog;

};



struct bpf_link_primer {

	struct bpf_link *link;

	struct file *file;
@@ -1133,8 +1149,8 @@ bool bpf_struct_ops_get(const void *kdata); 
void bpf_struct_ops_put(const void *kdata);

int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,

				       void *value);

int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_progs *tprogs,

				      struct bpf_prog *prog,

int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,

				      struct bpf_tramp_link *link,

				      const struct btf_func_model *model,

				      void *image, void *image_end);

static inline bool bpf_try_module_get(const void *data, struct module *owner)
@@ -1339,6 +1355,12 @@ struct bpf_trace_run_ctx { 
	u64 bpf_cookie;

};



struct bpf_tramp_run_ctx {

	struct bpf_run_ctx run_ctx;

	u64 bpf_cookie;

	struct bpf_run_ctx *saved_run_ctx;

};



static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx)

{

	struct bpf_run_ctx *old_ctx = NULL;

include/linux/bpf_types.h

+1 −0
Original line number Diff line number Diff line
@@ -141,3 +141,4 @@ BPF_LINK_TYPE(BPF_LINK_TYPE_XDP, xdp) 
BPF_LINK_TYPE(BPF_LINK_TYPE_PERF_EVENT, perf)

#endif

BPF_LINK_TYPE(BPF_LINK_TYPE_KPROBE_MULTI, kprobe_multi)

BPF_LINK_TYPE(BPF_LINK_TYPE_STRUCT_OPS, struct_ops)

include/uapi/linux/bpf.h

+10 −0
Original line number Diff line number Diff line
@@ -1013,6 +1013,7 @@ enum bpf_link_type { 
	BPF_LINK_TYPE_XDP = 6,

	BPF_LINK_TYPE_PERF_EVENT = 7,

	BPF_LINK_TYPE_KPROBE_MULTI = 8,

	BPF_LINK_TYPE_STRUCT_OPS = 9,



	MAX_BPF_LINK_TYPE,

};
@@ -1489,6 +1490,15 @@ union bpf_attr { 
				__aligned_u64	addrs;

				__aligned_u64	cookies;

			} kprobe_multi;

			struct {

				/* this is overlaid with the target_btf_id above. */

				__u32		target_btf_id;

				/* black box user-provided value passed through

				 * to BPF program at the execution time and

				 * accessible through bpf_get_attach_cookie() BPF helper

				 */

				__u64		cookie;

			} tracing;

		};

	} link_create;

kernel/bpf/bpf_lsm.c

+17 −0
Original line number Diff line number Diff line
@@ -117,6 +117,21 @@ static const struct bpf_func_proto bpf_ima_file_hash_proto = { 
	.allowed	= bpf_ima_inode_hash_allowed,

};



BPF_CALL_1(bpf_get_attach_cookie, void *, ctx)

{

	struct bpf_trace_run_ctx *run_ctx;



	run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx);

	return run_ctx->bpf_cookie;

}



static const struct bpf_func_proto bpf_get_attach_cookie_proto = {

	.func		= bpf_get_attach_cookie,

	.gpl_only	= false,

	.ret_type	= RET_INTEGER,

	.arg1_type	= ARG_PTR_TO_CTX,

};



static const struct bpf_func_proto *

bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)

{
@@ -141,6 +156,8 @@ bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 
		return prog->aux->sleepable ? &bpf_ima_inode_hash_proto : NULL;

	case BPF_FUNC_ima_file_hash:

		return prog->aux->sleepable ? &bpf_ima_file_hash_proto : NULL;

	case BPF_FUNC_get_attach_cookie:

		return bpf_prog_has_trampoline(prog) ? &bpf_get_attach_cookie_proto : NULL;

	default:

		return tracing_prog_func_proto(func_id, prog);

	}