Commit 028a9642 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'bpf: Support struct argument for trampoline base progs'

Yonghong Song says:

====================
Currently struct arguments are not supported for trampoline based progs.
One of major reason is that struct argument may pass by value which may
use more than one registers. This breaks trampoline progs where
each argument is assumed to take one register. bcc community reported the
issue ([1]) where struct argument is not supported for fentry program.
  typedef struct {
        uid_t val;
  } kuid_t;
  typedef struct {
        gid_t val;
  } kgid_t;
  int security_path_chown(struct path *path, kuid_t uid, kgid_t gid);
Inside Meta, we also have a use case to attach to tcp_setsockopt()
  typedef struct {
        union {
                void            *kernel;
                void __user     *user;
        };
        bool            is_kernel : 1;
  } sockptr_t;
  int tcp_setsockopt(struct sock *sk, int level, int optname,
                     sockptr_t optval, unsigned int optlen);

This patch added struct value support for bpf tracing programs which
uses trampoline. Only <= 16 byte struct size is supported for now
which covers use cases in the above. For x86/arm64/bpf, <= 16
struct value will be passed in registers instead of by reference.
Only x86_64 is supported in this patch. arm64 support can be
added later.

 [1] https://github.com/iovisor/bcc/issues/3657



Changelog:
  v3 -> v4:
   - fix a test failure where no casting for
     bpf_get_func_arg() value as the value type is 'int'.
   - add tracing_struct test in DENYLIST.s390x
   - simplify macro BPF_REG_CNT for BPF_PROG2.
  v2 -> v3:
   - previously struct arguments (<= 16 bytes) are passed
     by reference for bpf programs. Suggested by Alexei,
     it is passed by value now.
   - in order to support passing <= 16 struct value, a
     new macro BPF_PROG2 is invented.
  rfc v1 -> v2:
   - changed bpf_func_model struct info fields to
     arg_flags[] to make it easy to iterate arguments
     in arch specific {save|restore}_regs() functions.
   - added fexit tests to test return values with
     struct arguments.
====================

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 1e660f7e ae63c10f
Loading
Loading
Loading
Loading
+7 −1
Original line number Diff line number Diff line
@@ -1970,7 +1970,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,
				u32 flags, struct bpf_tramp_links *tlinks,
				void *orig_call)
{
	int ret;
	int i, ret;
	int nargs = m->nr_args;
	int max_insns = ((long)image_end - (long)image) / AARCH64_INSN_SIZE;
	struct jit_ctx ctx = {
@@ -1982,6 +1982,12 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,
	if (nargs > 8)
		return -ENOTSUPP;

	/* don't support struct argument */
	for (i = 0; i < MAX_BPF_FUNC_ARGS; i++) {
		if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG)
			return -ENOTSUPP;
	}

	ret = prepare_trampoline(&ctx, im, tlinks, orig_call, nargs, flags);
	if (ret < 0)
		return ret;
+51 −17
Original line number Diff line number Diff line
@@ -1751,34 +1751,60 @@ st: if (is_imm8(insn->off))
static void save_regs(const struct btf_func_model *m, u8 **prog, int nr_args,
		      int stack_size)
{
	int i;
	int i, j, arg_size, nr_regs;
	/* Store function arguments to stack.
	 * For a function that accepts two pointers the sequence will be:
	 * mov QWORD PTR [rbp-0x10],rdi
	 * mov QWORD PTR [rbp-0x8],rsi
	 */
	for (i = 0; i < min(nr_args, 6); i++)
		emit_stx(prog, bytes_to_bpf_size(m->arg_size[i]),
	for (i = 0, j = 0; i < min(nr_args, 6); i++) {
		if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) {
			nr_regs = (m->arg_size[i] + 7) / 8;
			arg_size = 8;
		} else {
			nr_regs = 1;
			arg_size = m->arg_size[i];
		}

		while (nr_regs) {
			emit_stx(prog, bytes_to_bpf_size(arg_size),
				 BPF_REG_FP,
			 i == 5 ? X86_REG_R9 : BPF_REG_1 + i,
			 -(stack_size - i * 8));
				 j == 5 ? X86_REG_R9 : BPF_REG_1 + j,
				 -(stack_size - j * 8));
			nr_regs--;
			j++;
		}
	}
}

static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args,
			 int stack_size)
{
	int i;
	int i, j, arg_size, nr_regs;

	/* Restore function arguments from stack.
	 * For a function that accepts two pointers the sequence will be:
	 * EMIT4(0x48, 0x8B, 0x7D, 0xF0); mov rdi,QWORD PTR [rbp-0x10]
	 * EMIT4(0x48, 0x8B, 0x75, 0xF8); mov rsi,QWORD PTR [rbp-0x8]
	 */
	for (i = 0; i < min(nr_args, 6); i++)
		emit_ldx(prog, bytes_to_bpf_size(m->arg_size[i]),
			 i == 5 ? X86_REG_R9 : BPF_REG_1 + i,
	for (i = 0, j = 0; i < min(nr_args, 6); i++) {
		if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) {
			nr_regs = (m->arg_size[i] + 7) / 8;
			arg_size = 8;
		} else {
			nr_regs = 1;
			arg_size = m->arg_size[i];
		}

		while (nr_regs) {
			emit_ldx(prog, bytes_to_bpf_size(arg_size),
				 j == 5 ? X86_REG_R9 : BPF_REG_1 + j,
				 BPF_REG_FP,
			 -(stack_size - i * 8));
				 -(stack_size - j * 8));
			nr_regs--;
			j++;
		}
	}
}

static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
@@ -2015,7 +2041,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
				struct bpf_tramp_links *tlinks,
				void *orig_call)
{
	int ret, i, nr_args = m->nr_args;
	int ret, i, nr_args = m->nr_args, extra_nregs = 0;
	int regs_off, ip_off, args_off, stack_size = nr_args * 8, run_ctx_off;
	struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
	struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
@@ -2028,6 +2054,14 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
	if (nr_args > 6)
		return -ENOTSUPP;

	for (i = 0; i < MAX_BPF_FUNC_ARGS; i++) {
		if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG)
			extra_nregs += (m->arg_size[i] + 7) / 8 - 1;
	}
	if (nr_args + extra_nregs > 6)
		return -ENOTSUPP;
	stack_size += extra_nregs * 8;

	/* Generated trampoline stack layout:
	 *
	 * RBP + 8         [ return address  ]
@@ -2040,7 +2074,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
	 *                 [ ...             ]
	 * RBP - regs_off  [ reg_arg1        ]  program's ctx pointer
	 *
	 * RBP - args_off  [ args count      ]  always
	 * RBP - args_off  [ arg regs count  ]  always
	 *
	 * RBP - ip_off    [ traced function ]  BPF_TRAMP_F_IP_ARG flag
	 *
@@ -2083,11 +2117,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
	EMIT4(0x48, 0x83, 0xEC, stack_size); /* sub rsp, stack_size */
	EMIT1(0x53);		 /* push rbx */

	/* Store number of arguments of the traced function:
	 *   mov rax, nr_args
	/* Store number of argument registers of the traced function:
	 *   mov rax, nr_args + extra_nregs
	 *   mov QWORD PTR [rbp - args_off], rax
	 */
	emit_mov_imm64(&prog, BPF_REG_0, 0, (u32) nr_args);
	emit_mov_imm64(&prog, BPF_REG_0, 0, (u32) nr_args + extra_nregs);
	emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -args_off);

	if (flags & BPF_TRAMP_F_IP_ARG) {
+4 −0
Original line number Diff line number Diff line
@@ -727,10 +727,14 @@ enum bpf_cgroup_storage_type {
 */
#define MAX_BPF_FUNC_REG_ARGS 5

/* The argument is a structure. */
#define BTF_FMODEL_STRUCT_ARG		BIT(0)

struct btf_func_model {
	u8 ret_size;
	u8 nr_args;
	u8 arg_size[MAX_BPF_FUNC_ARGS];
	u8 arg_flags[MAX_BPF_FUNC_ARGS];
};

/* Restore arguments before returning from trampoline to let original function
+5 −4
Original line number Diff line number Diff line
@@ -5079,12 +5079,12 @@ union bpf_attr {
 *
 * long bpf_get_func_arg(void *ctx, u32 n, u64 *value)
 *	Description
 *		Get **n**-th argument (zero based) of the traced function (for tracing programs)
 *		Get **n**-th argument register (zero based) of the traced function (for tracing programs)
 *		returned in **value**.
 *
 *	Return
 *		0 on success.
 *		**-EINVAL** if n >= arguments count of traced function.
 *		**-EINVAL** if n >= argument register count of traced function.
 *
 * long bpf_get_func_ret(void *ctx, u64 *value)
 *	Description
@@ -5097,10 +5097,11 @@ union bpf_attr {
 *
 * long bpf_get_func_arg_cnt(void *ctx)
 *	Description
 *		Get number of arguments of the traced function (for tracing programs).
 *		Get number of registers of the traced function (for tracing programs) where
 *		function arguments are stored in these registers.
 *
 *	Return
 *		The number of arguments of the traced function.
 *		The number of argument registers of the traced function.
 *
 * int bpf_get_retval(void)
 *	Description
+39 −6
Original line number Diff line number Diff line
@@ -5328,6 +5328,34 @@ static bool is_int_ptr(struct btf *btf, const struct btf_type *t)
	return btf_type_is_int(t);
}

static u32 get_ctx_arg_idx(struct btf *btf, const struct btf_type *func_proto,
			   int off)
{
	const struct btf_param *args;
	const struct btf_type *t;
	u32 offset = 0, nr_args;
	int i;

	if (!func_proto)
		return off / 8;

	nr_args = btf_type_vlen(func_proto);
	args = (const struct btf_param *)(func_proto + 1);
	for (i = 0; i < nr_args; i++) {
		t = btf_type_skip_modifiers(btf, args[i].type, NULL);
		offset += btf_type_is_ptr(t) ? 8 : roundup(t->size, 8);
		if (off < offset)
			return i;
	}

	t = btf_type_skip_modifiers(btf, func_proto->type, NULL);
	offset += btf_type_is_ptr(t) ? 8 : roundup(t->size, 8);
	if (off < offset)
		return nr_args;

	return nr_args + 1;
}

bool btf_ctx_access(int off, int size, enum bpf_access_type type,
		    const struct bpf_prog *prog,
		    struct bpf_insn_access_aux *info)
@@ -5347,7 +5375,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
			tname, off);
		return false;
	}
	arg = off / 8;
	arg = get_ctx_arg_idx(btf, t, off);
	args = (const struct btf_param *)(t + 1);
	/* if (t == NULL) Fall back to default BPF prog with
	 * MAX_BPF_FUNC_REG_ARGS u64 arguments.
@@ -5417,7 +5445,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
	/* skip modifiers */
	while (btf_type_is_modifier(t))
		t = btf_type_by_id(btf, t->type);
	if (btf_type_is_small_int(t) || btf_is_any_enum(t))
	if (btf_type_is_small_int(t) || btf_is_any_enum(t) || __btf_type_is_struct(t))
		/* accessing a scalar */
		return true;
	if (!btf_type_is_ptr(t)) {
@@ -5881,7 +5909,7 @@ static int __get_type_size(struct btf *btf, u32 btf_id,
	if (btf_type_is_ptr(t))
		/* kernel size of pointer. Not BPF's size of pointer*/
		return sizeof(void *);
	if (btf_type_is_int(t) || btf_is_any_enum(t))
	if (btf_type_is_int(t) || btf_is_any_enum(t) || __btf_type_is_struct(t))
		return t->size;
	return -EINVAL;
}
@@ -5901,8 +5929,10 @@ int btf_distill_func_proto(struct bpf_verifier_log *log,
		/* BTF function prototype doesn't match the verifier types.
		 * Fall back to MAX_BPF_FUNC_REG_ARGS u64 args.
		 */
		for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++)
		for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) {
			m->arg_size[i] = 8;
			m->arg_flags[i] = 0;
		}
		m->ret_size = 8;
		m->nr_args = MAX_BPF_FUNC_REG_ARGS;
		return 0;
@@ -5916,7 +5946,7 @@ int btf_distill_func_proto(struct bpf_verifier_log *log,
		return -EINVAL;
	}
	ret = __get_type_size(btf, func->type, &t);
	if (ret < 0) {
	if (ret < 0 || __btf_type_is_struct(t)) {
		bpf_log(log,
			"The function %s return type %s is unsupported.\n",
			tname, btf_kind_str[BTF_INFO_KIND(t->info)]);
@@ -5932,7 +5962,9 @@ int btf_distill_func_proto(struct bpf_verifier_log *log,
			return -EINVAL;
		}
		ret = __get_type_size(btf, args[i].type, &t);
		if (ret < 0) {

		/* No support of struct argument size greater than 16 bytes */
		if (ret < 0 || ret > 16) {
			bpf_log(log,
				"The function %s arg%d type %s is unsupported.\n",
				tname, i, btf_kind_str[BTF_INFO_KIND(t->info)]);
@@ -5945,6 +5977,7 @@ int btf_distill_func_proto(struct bpf_verifier_log *log,
			return -EINVAL;
		}
		m->arg_size[i] = ret;
		m->arg_flags[i] = __btf_type_is_struct(t) ? BTF_FMODEL_STRUCT_ARG : 0;
	}
	m->nr_args = nargs;
	return 0;
Loading