Commit af3f4134 authored by Stanislav Fomichev's avatar Stanislav Fomichev Committed by Alexei Starovoitov
Browse files

bpf: add bpf_func_t and trampoline helpers 



I'll be adding lsm cgroup specific helpers that grab
trampoline mutex.

No functional changes.

Reviewed-by: default avatarMartin KaFai Lau <kafai@fb.com>
Signed-off-by: default avatarStanislav Fomichev <sdf@google.com>
Link: https://lore.kernel.org/r/20220628174314.1216643-2-sdf@google.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent c5c7358e

include/linux/bpf.h

+5 −6
Original line number Diff line number Diff line
@@ -56,6 +56,8 @@ typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64); 
typedef int (*bpf_iter_init_seq_priv_t)(void *private_data,

					struct bpf_iter_aux_info *aux);

typedef void (*bpf_iter_fini_seq_priv_t)(void *private_data);

typedef unsigned int (*bpf_func_t)(const void *,

				   const struct bpf_insn *);

struct bpf_iter_seq_info {

	const struct seq_operations *seq_ops;

	bpf_iter_init_seq_priv_t init_seq_private;
@@ -879,8 +881,7 @@ struct bpf_dispatcher { 
static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func(

	const void *ctx,

	const struct bpf_insn *insnsi,

	unsigned int (*bpf_func)(const void *,

				 const struct bpf_insn *))

	bpf_func_t bpf_func)

{

	return bpf_func(ctx, insnsi);

}
@@ -909,8 +910,7 @@ int arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs); 
	noinline __nocfi unsigned int bpf_dispatcher_##name##_func(	\

		const void *ctx,					\

		const struct bpf_insn *insnsi,				\

		unsigned int (*bpf_func)(const void *,			\

					 const struct bpf_insn *))	\

		bpf_func_t bpf_func)					\

	{								\

		return bpf_func(ctx, insnsi);				\

	}								\
@@ -921,8 +921,7 @@ int arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs); 
	unsigned int bpf_dispatcher_##name##_func(			\

		const void *ctx,					\

		const struct bpf_insn *insnsi,				\

		unsigned int (*bpf_func)(const void *,			\

					 const struct bpf_insn *));	\

		bpf_func_t bpf_func);					\

	extern struct bpf_dispatcher bpf_dispatcher_##name;

#define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func

#define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name)

kernel/bpf/trampoline.c

+33 −30
Original line number Diff line number Diff line
@@ -410,7 +410,7 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog) 
	}

}



int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)

static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)

{

	enum bpf_tramp_prog_type kind;

	struct bpf_tramp_link *link_exiting;
@@ -418,44 +418,33 @@ int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline 
	int cnt = 0, i;



	kind = bpf_attach_type_to_tramp(link->link.prog);

	mutex_lock(&tr->mutex);

	if (tr->extension_prog) {

	if (tr->extension_prog)

		/* cannot attach fentry/fexit if extension prog is attached.

		 * cannot overwrite extension prog either.

		 */

		err = -EBUSY;

		goto out;

	}

		return -EBUSY;



	for (i = 0; i < BPF_TRAMP_MAX; i++)

		cnt += tr->progs_cnt[i];



	if (kind == BPF_TRAMP_REPLACE) {

		/* Cannot attach extension if fentry/fexit are in use. */

		if (cnt) {

			err = -EBUSY;

			goto out;

		}

		if (cnt)

			return -EBUSY;

		tr->extension_prog = link->link.prog;

		err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, NULL,

		return bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, NULL,

					  link->link.prog->bpf_func);

		goto out;

	}

	if (cnt >= BPF_MAX_TRAMP_LINKS) {

		err = -E2BIG;

		goto out;

	}

	if (!hlist_unhashed(&link->tramp_hlist)) {

	if (cnt >= BPF_MAX_TRAMP_LINKS)

		return -E2BIG;

	if (!hlist_unhashed(&link->tramp_hlist))

		/* prog already linked */

		err = -EBUSY;

		goto out;

	}

		return -EBUSY;

	hlist_for_each_entry(link_exiting, &tr->progs_hlist[kind], tramp_hlist) {

		if (link_exiting->link.prog != link->link.prog)

			continue;

		/* prog already linked */

		err = -EBUSY;

		goto out;

		return -EBUSY;

	}



	hlist_add_head(&link->tramp_hlist, &tr->progs_hlist[kind]);
@@ -465,30 +454,44 @@ int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline 
		hlist_del_init(&link->tramp_hlist);

		tr->progs_cnt[kind]--;

	}

out:

	return err;

}



int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)

{

	int err;



	mutex_lock(&tr->mutex);

	err = __bpf_trampoline_link_prog(link, tr);

	mutex_unlock(&tr->mutex);

	return err;

}



/* bpf_trampoline_unlink_prog() should never fail. */

int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)

static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)

{

	enum bpf_tramp_prog_type kind;

	int err;



	kind = bpf_attach_type_to_tramp(link->link.prog);

	mutex_lock(&tr->mutex);

	if (kind == BPF_TRAMP_REPLACE) {

		WARN_ON_ONCE(!tr->extension_prog);

		err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP,

					 tr->extension_prog->bpf_func, NULL);

		tr->extension_prog = NULL;

		goto out;

		return err;

	}

	hlist_del_init(&link->tramp_hlist);

	tr->progs_cnt[kind]--;

	err = bpf_trampoline_update(tr);

out:

	return bpf_trampoline_update(tr);

}



/* bpf_trampoline_unlink_prog() should never fail. */

int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)

{

	int err;



	mutex_lock(&tr->mutex);

	err = __bpf_trampoline_unlink_prog(link, tr);

	mutex_unlock(&tr->mutex);

	return err;

}