Commit fbd94c7a authored by Alexei Starovoitov's avatar Alexei Starovoitov Committed by Andrii Nakryiko
Browse files

bpf: Pass a set of bpf_core_relo-s to prog_load command. 



struct bpf_core_relo is generated by llvm and processed by libbpf.
It's a de-facto uapi.
With CO-RE in the kernel the struct bpf_core_relo becomes uapi de-jure.
Add an ability to pass a set of 'struct bpf_core_relo' to prog_load command
and let the kernel perform CO-RE relocations.

Note the struct bpf_line_info and struct bpf_func_info have the same
layout when passed from LLVM to libbpf and from libbpf to the kernel
except "insn_off" fields means "byte offset" when LLVM generates it.
Then libbpf converts it to "insn index" to pass to the kernel.
The struct bpf_core_relo's "insn_off" field is always "byte offset".

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Signed-off-by: default avatarAndrii Nakryiko <andrii@kernel.org>
Acked-by: default avatarAndrii Nakryiko <andrii@kernel.org>
Link: https://lore.kernel.org/bpf/20211201181040.23337-6-alexei.starovoitov@gmail.com
parent 46334a0c

include/linux/bpf.h

+8 −0
Original line number Diff line number Diff line
@@ -1732,6 +1732,14 @@ bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog); 
const struct btf_func_model *

bpf_jit_find_kfunc_model(const struct bpf_prog *prog,

			 const struct bpf_insn *insn);

struct bpf_core_ctx {

	struct bpf_verifier_log *log;

	const struct btf *btf;

};



int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo,

		   int relo_idx, void *insn);



#else /* !CONFIG_BPF_SYSCALL */

static inline struct bpf_prog *bpf_prog_get(u32 ufd)

{

include/uapi/linux/bpf.h

+58 −1
Original line number Diff line number Diff line
@@ -1342,8 +1342,10 @@ union bpf_attr { 
			/* or valid module BTF object fd or 0 to attach to vmlinux */

			__u32		attach_btf_obj_fd;

		};

		__u32		:32;		/* pad */

		__u32		core_relo_cnt;	/* number of bpf_core_relo */

		__aligned_u64	fd_array;	/* array of FDs */

		__aligned_u64	core_relos;

		__u32		core_relo_rec_size; /* sizeof(struct bpf_core_relo) */

	};



	struct { /* anonymous struct used by BPF_OBJ_* commands */
@@ -6393,4 +6395,59 @@ enum bpf_core_relo_kind { 
	BPF_CORE_ENUMVAL_VALUE = 11,         /* enum value integer value */

};



/*

 * "struct bpf_core_relo" is used to pass relocation data form LLVM to libbpf

 * and from libbpf to the kernel.

 *

 * CO-RE relocation captures the following data:

 * - insn_off - instruction offset (in bytes) within a BPF program that needs

 *   its insn->imm field to be relocated with actual field info;

 * - type_id - BTF type ID of the "root" (containing) entity of a relocatable

 *   type or field;

 * - access_str_off - offset into corresponding .BTF string section. String

 *   interpretation depends on specific relocation kind:

 *     - for field-based relocations, string encodes an accessed field using

 *       a sequence of field and array indices, separated by colon (:). It's

 *       conceptually very close to LLVM's getelementptr ([0]) instruction's

 *       arguments for identifying offset to a field.

 *     - for type-based relocations, strings is expected to be just "0";

 *     - for enum value-based relocations, string contains an index of enum

 *       value within its enum type;

 * - kind - one of enum bpf_core_relo_kind;

 *

 * Example:

 *   struct sample {

 *       int a;

 *       struct {

 *           int b[10];

 *       };

 *   };

 *

 *   struct sample *s = ...;

 *   int *x = &s->a;     // encoded as "0:0" (a is field #0)

 *   int *y = &s->b[5];  // encoded as "0:1:0:5" (anon struct is field #1,

 *                       // b is field #0 inside anon struct, accessing elem #5)

 *   int *z = &s[10]->b; // encoded as "10:1" (ptr is used as an array)

 *

 * type_id for all relocs in this example will capture BTF type id of

 * `struct sample`.

 *

 * Such relocation is emitted when using __builtin_preserve_access_index()

 * Clang built-in, passing expression that captures field address, e.g.:

 *

 * bpf_probe_read(&dst, sizeof(dst),

 *		  __builtin_preserve_access_index(&src->a.b.c));

 *

 * In this case Clang will emit field relocation recording necessary data to

 * be able to find offset of embedded `a.b.c` field within `src` struct.

 *

 * [0] https://llvm.org/docs/LangRef.html#getelementptr-instruction

 */

struct bpf_core_relo {

	__u32 insn_off;

	__u32 type_id;

	__u32 access_str_off;

	enum bpf_core_relo_kind kind;

};



#endif /* _UAPI__LINUX_BPF_H__ */

kernel/bpf/btf.c

+6 −0
Original line number Diff line number Diff line
@@ -6439,3 +6439,9 @@ size_t bpf_core_essential_name_len(const char *name) 
	}

	return n;

}



int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo,

		   int relo_idx, void *insn)

{

	return -EOPNOTSUPP;

}

kernel/bpf/syscall.c

+1 −1
Original line number Diff line number Diff line
@@ -2184,7 +2184,7 @@ static bool is_perfmon_prog_type(enum bpf_prog_type prog_type) 
}



/* last field in 'union bpf_attr' used by this command */

#define	BPF_PROG_LOAD_LAST_FIELD fd_array

#define	BPF_PROG_LOAD_LAST_FIELD core_relo_rec_size



static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr)

{

kernel/bpf/verifier.c

+76 −0
Original line number Diff line number Diff line
@@ -10273,6 +10273,78 @@ static int check_btf_line(struct bpf_verifier_env *env, 
	return err;

}



#define MIN_CORE_RELO_SIZE	sizeof(struct bpf_core_relo)

#define MAX_CORE_RELO_SIZE	MAX_FUNCINFO_REC_SIZE



static int check_core_relo(struct bpf_verifier_env *env,

			   const union bpf_attr *attr,

			   bpfptr_t uattr)

{

	u32 i, nr_core_relo, ncopy, expected_size, rec_size;

	struct bpf_core_relo core_relo = {};

	struct bpf_prog *prog = env->prog;

	const struct btf *btf = prog->aux->btf;

	struct bpf_core_ctx ctx = {

		.log = &env->log,

		.btf = btf,

	};

	bpfptr_t u_core_relo;

	int err;



	nr_core_relo = attr->core_relo_cnt;

	if (!nr_core_relo)

		return 0;

	if (nr_core_relo > INT_MAX / sizeof(struct bpf_core_relo))

		return -EINVAL;



	rec_size = attr->core_relo_rec_size;

	if (rec_size < MIN_CORE_RELO_SIZE ||

	    rec_size > MAX_CORE_RELO_SIZE ||

	    rec_size % sizeof(u32))

		return -EINVAL;



	u_core_relo = make_bpfptr(attr->core_relos, uattr.is_kernel);

	expected_size = sizeof(struct bpf_core_relo);

	ncopy = min_t(u32, expected_size, rec_size);



	/* Unlike func_info and line_info, copy and apply each CO-RE

	 * relocation record one at a time.

	 */

	for (i = 0; i < nr_core_relo; i++) {

		/* future proofing when sizeof(bpf_core_relo) changes */

		err = bpf_check_uarg_tail_zero(u_core_relo, expected_size, rec_size);

		if (err) {

			if (err == -E2BIG) {

				verbose(env, "nonzero tailing record in core_relo");

				if (copy_to_bpfptr_offset(uattr,

							  offsetof(union bpf_attr, core_relo_rec_size),

							  &expected_size, sizeof(expected_size)))

					err = -EFAULT;

			}

			break;

		}



		if (copy_from_bpfptr(&core_relo, u_core_relo, ncopy)) {

			err = -EFAULT;

			break;

		}



		if (core_relo.insn_off % 8 || core_relo.insn_off / 8 >= prog->len) {

			verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n",

				i, core_relo.insn_off, prog->len);

			err = -EINVAL;

			break;

		}



		err = bpf_core_apply(&ctx, &core_relo, i,

				     &prog->insnsi[core_relo.insn_off / 8]);

		if (err)

			break;

		bpfptr_add(&u_core_relo, rec_size);

	}

	return err;

}



static int check_btf_info(struct bpf_verifier_env *env,

			  const union bpf_attr *attr,

			  bpfptr_t uattr)
@@ -10303,6 +10375,10 @@ static int check_btf_info(struct bpf_verifier_env *env, 
	if (err)

		return err;



	err = check_core_relo(env, attr, uattr);

	if (err)

		return err;



	return 0;

}