Commit f7e0beaf authored by Kui-Feng Lee's avatar Kui-Feng Lee Committed by Andrii Nakryiko
Browse files

bpf, x86: Generate trampolines from bpf_tramp_links 



Replace struct bpf_tramp_progs with struct bpf_tramp_links to collect
struct bpf_tramp_link(s) for a trampoline.  struct bpf_tramp_link
extends bpf_link to act as a linked list node.

arch_prepare_bpf_trampoline() accepts a struct bpf_tramp_links to
collects all bpf_tramp_link(s) that a trampoline should call.

Change BPF trampoline and bpf_struct_ops to pass bpf_tramp_links
instead of bpf_tramp_progs.

Signed-off-by: default avatarKui-Feng Lee <kuifeng@fb.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Signed-off-by: default avatarAndrii Nakryiko <andrii@kernel.org>
Acked-by: default avatarAndrii Nakryiko <andrii@kernel.org>
Link: https://lore.kernel.org/bpf/20220510205923.3206889-2-kuifeng@fb.com
parent cb411545

arch/x86/net/bpf_jit_comp.c

+19 −17
Original line number Diff line number Diff line
@@ -1762,10 +1762,12 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args, 
}



static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,

			   struct bpf_prog *p, int stack_size, bool save_ret)

			   struct bpf_tramp_link *l, int stack_size,

			   bool save_ret)

{

	u8 *prog = *pprog;

	u8 *jmp_insn;

	struct bpf_prog *p = l->link.prog;



	/* arg1: mov rdi, progs[i] */

	emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
@@ -1850,14 +1852,14 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond) 
}



static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,

		      struct bpf_tramp_progs *tp, int stack_size,

		      struct bpf_tramp_links *tl, int stack_size,

		      bool save_ret)

{

	int i;

	u8 *prog = *pprog;



	for (i = 0; i < tp->nr_progs; i++) {

		if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size,

	for (i = 0; i < tl->nr_links; i++) {

		if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size,

				    save_ret))

			return -EINVAL;

	}
@@ -1866,7 +1868,7 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, 
}



static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,

			      struct bpf_tramp_progs *tp, int stack_size,

			      struct bpf_tramp_links *tl, int stack_size,

			      u8 **branches)

{

	u8 *prog = *pprog;
@@ -1877,8 +1879,8 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, 
	 */

	emit_mov_imm32(&prog, false, BPF_REG_0, 0);

	emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);

	for (i = 0; i < tp->nr_progs; i++) {

		if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size, true))

	for (i = 0; i < tl->nr_links; i++) {

		if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size, true))

			return -EINVAL;



		/* mod_ret prog stored return value into [rbp - 8]. Emit:
@@ -1980,14 +1982,14 @@ static bool is_valid_bpf_tramp_flags(unsigned int flags) 
 */

int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,

				const struct btf_func_model *m, u32 flags,

				struct bpf_tramp_progs *tprogs,

				struct bpf_tramp_links *tlinks,

				void *orig_call)

{

	int ret, i, nr_args = m->nr_args;

	int regs_off, ip_off, args_off, stack_size = nr_args * 8;

	struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY];

	struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT];

	struct bpf_tramp_progs *fmod_ret = &tprogs[BPF_TRAMP_MODIFY_RETURN];

	struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];

	struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];

	struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];

	u8 **branches = NULL;

	u8 *prog;

	bool save_ret;
@@ -2078,13 +2080,13 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 
		}

	}



	if (fentry->nr_progs)

	if (fentry->nr_links)

		if (invoke_bpf(m, &prog, fentry, regs_off,

			       flags & BPF_TRAMP_F_RET_FENTRY_RET))

			return -EINVAL;



	if (fmod_ret->nr_progs) {

		branches = kcalloc(fmod_ret->nr_progs, sizeof(u8 *),

	if (fmod_ret->nr_links) {

		branches = kcalloc(fmod_ret->nr_links, sizeof(u8 *),

				   GFP_KERNEL);

		if (!branches)

			return -ENOMEM;
@@ -2111,7 +2113,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 
		prog += X86_PATCH_SIZE;

	}



	if (fmod_ret->nr_progs) {

	if (fmod_ret->nr_links) {

		/* From Intel 64 and IA-32 Architectures Optimization

		 * Reference Manual, 3.4.1.4 Code Alignment, Assembly/Compiler

		 * Coding Rule 11: All branch targets should be 16-byte
@@ -2121,12 +2123,12 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 
		/* Update the branches saved in invoke_bpf_mod_ret with the

		 * aligned address of do_fexit.

		 */

		for (i = 0; i < fmod_ret->nr_progs; i++)

		for (i = 0; i < fmod_ret->nr_links; i++)

			emit_cond_near_jump(&branches[i], prog, branches[i],

					    X86_JNE);

	}



	if (fexit->nr_progs)

	if (fexit->nr_links)

		if (invoke_bpf(m, &prog, fexit, regs_off, false)) {

			ret = -EINVAL;

			goto cleanup;

include/linux/bpf.h

+24 −12
Original line number Diff line number Diff line
@@ -723,11 +723,11 @@ struct btf_func_model { 
/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50

 * bytes on x86.  Pick a number to fit into BPF_IMAGE_SIZE / 2

 */

#define BPF_MAX_TRAMP_PROGS 38

#define BPF_MAX_TRAMP_LINKS 38



struct bpf_tramp_progs {

	struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS];

	int nr_progs;

struct bpf_tramp_links {

	struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS];

	int nr_links;

};



/* Different use cases for BPF trampoline:
@@ -753,7 +753,7 @@ struct bpf_tramp_progs { 
struct bpf_tramp_image;

int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end,

				const struct btf_func_model *m, u32 flags,

				struct bpf_tramp_progs *tprogs,

				struct bpf_tramp_links *tlinks,

				void *orig_call);

/* these two functions are called from generated trampoline */

u64 notrace __bpf_prog_enter(struct bpf_prog *prog);
@@ -852,9 +852,10 @@ static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( 
{

	return bpf_func(ctx, insnsi);

}



#ifdef CONFIG_BPF_JIT

int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);

int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);

int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);

int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);

struct bpf_trampoline *bpf_trampoline_get(u64 key,

					  struct bpf_attach_target_info *tgt_info);

void bpf_trampoline_put(struct bpf_trampoline *tr);
@@ -905,12 +906,12 @@ int bpf_jit_charge_modmem(u32 size); 
void bpf_jit_uncharge_modmem(u32 size);

bool bpf_prog_has_trampoline(const struct bpf_prog *prog);

#else

static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,

static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,

					   struct bpf_trampoline *tr)

{

	return -ENOTSUPP;

}

static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog,

static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,

					     struct bpf_trampoline *tr)

{

	return -ENOTSUPP;
@@ -1009,7 +1010,6 @@ struct bpf_prog_aux { 
	bool tail_call_reachable;

	bool xdp_has_frags;

	bool use_bpf_prog_pack;

	struct hlist_node tramp_hlist;

	/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */

	const struct btf_type *attach_func_proto;

	/* function name for valid attach_btf_id */
@@ -1096,6 +1096,18 @@ struct bpf_link_ops { 
			      struct bpf_link_info *info);

};



struct bpf_tramp_link {

	struct bpf_link link;

	struct hlist_node tramp_hlist;

};



struct bpf_tracing_link {

	struct bpf_tramp_link link;

	enum bpf_attach_type attach_type;

	struct bpf_trampoline *trampoline;

	struct bpf_prog *tgt_prog;

};



struct bpf_link_primer {

	struct bpf_link *link;

	struct file *file;
@@ -1133,8 +1145,8 @@ bool bpf_struct_ops_get(const void *kdata); 
void bpf_struct_ops_put(const void *kdata);

int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,

				       void *value);

int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_progs *tprogs,

				      struct bpf_prog *prog,

int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,

				      struct bpf_tramp_link *link,

				      const struct btf_func_model *model,

				      void *image, void *image_end);

static inline bool bpf_try_module_get(const void *data, struct module *owner)

include/linux/bpf_types.h

+1 −0
Original line number Diff line number Diff line
@@ -141,3 +141,4 @@ BPF_LINK_TYPE(BPF_LINK_TYPE_XDP, xdp) 
BPF_LINK_TYPE(BPF_LINK_TYPE_PERF_EVENT, perf)

#endif

BPF_LINK_TYPE(BPF_LINK_TYPE_KPROBE_MULTI, kprobe_multi)

BPF_LINK_TYPE(BPF_LINK_TYPE_STRUCT_OPS, struct_ops)

include/uapi/linux/bpf.h

+1 −0
Original line number Diff line number Diff line
@@ -1013,6 +1013,7 @@ enum bpf_link_type { 
	BPF_LINK_TYPE_XDP = 6,

	BPF_LINK_TYPE_PERF_EVENT = 7,

	BPF_LINK_TYPE_KPROBE_MULTI = 8,

	BPF_LINK_TYPE_STRUCT_OPS = 9,



	MAX_BPF_LINK_TYPE,

};

kernel/bpf/bpf_struct_ops.c

+49 −22
Original line number Diff line number Diff line
@@ -33,15 +33,15 @@ struct bpf_struct_ops_map { 
	const struct bpf_struct_ops *st_ops;

	/* protect map_update */

	struct mutex lock;

	/* progs has all the bpf_prog that is populated

	/* link has all the bpf_links that is populated

	 * to the func ptr of the kernel's struct

	 * (in kvalue.data).

	 */

	struct bpf_prog **progs;

	struct bpf_link **links;

	/* image is a page that has all the trampolines

	 * that stores the func args before calling the bpf_prog.

	 * A PAGE_SIZE "image" is enough to store all trampoline for

	 * "progs[]".

	 * "links[]".

	 */

	void *image;

	/* uvalue->data stores the kernel struct
@@ -283,9 +283,9 @@ static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map) 
	u32 i;



	for (i = 0; i < btf_type_vlen(t); i++) {

		if (st_map->progs[i]) {

			bpf_prog_put(st_map->progs[i]);

			st_map->progs[i] = NULL;

		if (st_map->links[i]) {

			bpf_link_put(st_map->links[i]);

			st_map->links[i] = NULL;

		}

	}

}
@@ -316,18 +316,34 @@ static int check_zero_holes(const struct btf_type *t, void *data) 
	return 0;

}



int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_progs *tprogs,

				      struct bpf_prog *prog,

static void bpf_struct_ops_link_release(struct bpf_link *link)

{

}



static void bpf_struct_ops_link_dealloc(struct bpf_link *link)

{

	struct bpf_tramp_link *tlink = container_of(link, struct bpf_tramp_link, link);



	kfree(tlink);

}



const struct bpf_link_ops bpf_struct_ops_link_lops = {

	.release = bpf_struct_ops_link_release,

	.dealloc = bpf_struct_ops_link_dealloc,

};



int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,

				      struct bpf_tramp_link *link,

				      const struct btf_func_model *model,

				      void *image, void *image_end)

{

	u32 flags;



	tprogs[BPF_TRAMP_FENTRY].progs[0] = prog;

	tprogs[BPF_TRAMP_FENTRY].nr_progs = 1;

	tlinks[BPF_TRAMP_FENTRY].links[0] = link;

	tlinks[BPF_TRAMP_FENTRY].nr_links = 1;

	flags = model->ret_size > 0 ? BPF_TRAMP_F_RET_FENTRY_RET : 0;

	return arch_prepare_bpf_trampoline(NULL, image, image_end,

					   model, flags, tprogs, NULL);

					   model, flags, tlinks, NULL);

}



static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
@@ -338,7 +354,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, 
	struct bpf_struct_ops_value *uvalue, *kvalue;

	const struct btf_member *member;

	const struct btf_type *t = st_ops->type;

	struct bpf_tramp_progs *tprogs = NULL;

	struct bpf_tramp_links *tlinks = NULL;

	void *udata, *kdata;

	int prog_fd, err = 0;

	void *image, *image_end;
@@ -362,8 +378,8 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, 
	if (uvalue->state || refcount_read(&uvalue->refcnt))

		return -EINVAL;



	tprogs = kcalloc(BPF_TRAMP_MAX, sizeof(*tprogs), GFP_KERNEL);

	if (!tprogs)

	tlinks = kcalloc(BPF_TRAMP_MAX, sizeof(*tlinks), GFP_KERNEL);

	if (!tlinks)

		return -ENOMEM;



	uvalue = (struct bpf_struct_ops_value *)st_map->uvalue;
@@ -386,6 +402,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, 
	for_each_member(i, t, member) {

		const struct btf_type *mtype, *ptype;

		struct bpf_prog *prog;

		struct bpf_tramp_link *link;

		u32 moff;



		moff = __btf_member_bit_offset(t, member) / 8;
@@ -439,16 +456,26 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, 
			err = PTR_ERR(prog);

			goto reset_unlock;

		}

		st_map->progs[i] = prog;



		if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||

		    prog->aux->attach_btf_id != st_ops->type_id ||

		    prog->expected_attach_type != i) {

			bpf_prog_put(prog);

			err = -EINVAL;

			goto reset_unlock;

		}



		err = bpf_struct_ops_prepare_trampoline(tprogs, prog,

		link = kzalloc(sizeof(*link), GFP_USER);

		if (!link) {

			bpf_prog_put(prog);

			err = -ENOMEM;

			goto reset_unlock;

		}

		bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS,

			      &bpf_struct_ops_link_lops, prog);

		st_map->links[i] = &link->link;



		err = bpf_struct_ops_prepare_trampoline(tlinks, link,

							&st_ops->func_models[i],

							image, image_end);

		if (err < 0)
@@ -491,7 +518,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, 
	memset(uvalue, 0, map->value_size);

	memset(kvalue, 0, map->value_size);

unlock:

	kfree(tprogs);

	kfree(tlinks);

	mutex_unlock(&st_map->lock);

	return err;

}
@@ -546,9 +573,9 @@ static void bpf_struct_ops_map_free(struct bpf_map *map) 
{

	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;



	if (st_map->progs)

	if (st_map->links)

		bpf_struct_ops_map_put_progs(st_map);

	bpf_map_area_free(st_map->progs);

	bpf_map_area_free(st_map->links);

	bpf_jit_free_exec(st_map->image);

	bpf_map_area_free(st_map->uvalue);

	bpf_map_area_free(st_map);
@@ -597,11 +624,11 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) 
	map = &st_map->map;



	st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE);

	st_map->progs =

		bpf_map_area_alloc(btf_type_vlen(t) * sizeof(struct bpf_prog *),

	st_map->links =

		bpf_map_area_alloc(btf_type_vlen(t) * sizeof(struct bpf_links *),

				   NUMA_NO_NODE);

	st_map->image = bpf_jit_alloc_exec(PAGE_SIZE);

	if (!st_map->uvalue || !st_map->progs || !st_map->image) {

	if (!st_map->uvalue || !st_map->links || !st_map->image) {

		bpf_struct_ops_map_free(map);

		return ERR_PTR(-ENOMEM);

	}