Commit 60a128b5 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'bpf: BTF support for ksyms' 

Hao Luo says:

====================
v3 -> v4:
 - Rebasing
 - Cast bpf_[per|this]_cpu_ptr's parameter to void __percpu * before
   passing into per_cpu_ptr.

v2 -> v3:
 - Rename functions and variables in verifier for better readability.
 - Stick to logging message convention in libbpf.
 - Move bpf_per_cpu_ptr and bpf_this_cpu_ptr from trace-specific
   helper set to base helper set.
 - More specific test in ksyms_btf.
 - Fix return type cast in bpf_*_cpu_ptr.
 - Fix btf leak in ksyms_btf selftest.
 - Fix return error code for kallsyms_find().

v1 -> v2:
 - Move check_pseudo_btf_id from check_ld_imm() to
   replace_map_fd_with_map_ptr() and rename the latter.
 - Add bpf_this_cpu_ptr().
 - Use bpf_core_types_are_compat() in libbpf.c for checking type
   compatibility.
 - Rewrite typed ksym extern type in BTF with int to save space.
 - Minor revision of bpf_per_cpu_ptr()'s comments.
 - Avoid using long in tests that use skeleton.
 - Refactored test_ksyms.c by moving kallsyms_find() to trace_helpers.c
 - Fold the patches that sync include/linux/uapi and
   tools/include/linux/uapi.

rfc -> v1:
 - Encode VAR's btf_id for PSEUDO_BTF_ID.
 - More checks in verifier. Checking the btf_id passed as
   PSEUDO_BTF_ID is valid VAR, its name and type.
 - Checks in libbpf on type compatibility of ksyms.
 - Add bpf_per_cpu_ptr() to access kernel percpu vars. Introduced
   new ARG and RET types for this helper.

This patch series extends the previously added __ksym externs with
btf support.

Right now the __ksym externs are treated as pure 64-bit scalar value.
Libbpf replaces ld_imm64 insn of __ksym by its kernel address at load
time. This patch series extend those externs with their btf info. Note
that btf support for __ksym must come with the kernel btf that has
VARs encoded to work properly. The corresponding chagnes in pahole
is available at [1] (with a fix at [2] for gcc 4.9+).

The first 3 patches in this series add support for general kernel
global variables, which include verifier checking (01/06), libpf
support (02/06) and selftests for getting typed ksym extern's kernel
address (03/06).

The next 3 patches extends that capability further by introducing
helpers bpf_per_cpu_ptr() and bpf_this_cpu_ptr(), which allows accessing
kernel percpu variables correctly (04/06 and 05/06).

The tests of this feature were performed against pahole that is extended
with [1] and [2]. For kernel BTF that does not have VARs encoded, the
selftests will be skipped.

[1] https://git.kernel.org/pub/scm/devel/pahole/pahole.git/commit/?id=f3d9054ba8ff1df0fc44e507e3a01c0964cabd42
[2] https://www.spinics.net/lists/dwarves/msg00451.html


====================

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 440c5752 00dc73e4

include/linux/bpf.h

+6 −0
Original line number Diff line number Diff line
@@ -293,6 +293,7 @@ enum bpf_arg_type { 
	ARG_PTR_TO_ALLOC_MEM_OR_NULL,	/* pointer to dynamically allocated memory or NULL */

	ARG_CONST_ALLOC_SIZE_OR_ZERO,	/* number of allocated bytes requested */

	ARG_PTR_TO_BTF_ID_SOCK_COMMON,	/* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */

	ARG_PTR_TO_PERCPU_BTF_ID,	/* pointer to in-kernel percpu type */

	__BPF_ARG_TYPE_MAX,

};
@@ -307,6 +308,8 @@ enum bpf_return_type { 
	RET_PTR_TO_SOCK_COMMON_OR_NULL,	/* returns a pointer to a sock_common or NULL */

	RET_PTR_TO_ALLOC_MEM_OR_NULL,	/* returns a pointer to dynamically allocated memory or NULL */

	RET_PTR_TO_BTF_ID_OR_NULL,	/* returns a pointer to a btf_id or NULL */

	RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL, /* returns a pointer to a valid memory or a btf_id or NULL */

	RET_PTR_TO_MEM_OR_BTF_ID,	/* returns a pointer to a valid memory or a btf_id */

};



/* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
@@ -405,6 +408,7 @@ enum bpf_reg_type { 
	PTR_TO_RDONLY_BUF_OR_NULL, /* reg points to a readonly buffer or NULL */

	PTR_TO_RDWR_BUF,	 /* reg points to a read/write buffer */

	PTR_TO_RDWR_BUF_OR_NULL, /* reg points to a read/write buffer or NULL */

	PTR_TO_PERCPU_BTF_ID,	 /* reg points to a percpu kernel variable */

};



/* The information passed from prog-specific *_is_valid_access
@@ -1828,6 +1832,8 @@ extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto; 
extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto;

extern const struct bpf_func_proto bpf_copy_from_user_proto;

extern const struct bpf_func_proto bpf_snprintf_btf_proto;

extern const struct bpf_func_proto bpf_per_cpu_ptr_proto;

extern const struct bpf_func_proto bpf_this_cpu_ptr_proto;



const struct bpf_func_proto *bpf_tracing_func_proto(

	enum bpf_func_id func_id, const struct bpf_prog *prog);

include/linux/bpf_verifier.h

+7 −0
Original line number Diff line number Diff line
@@ -308,6 +308,13 @@ struct bpf_insn_aux_data { 
			u32 map_index;		/* index into used_maps[] */

			u32 map_off;		/* offset from value base address */

		};

		struct {

			enum bpf_reg_type reg_type;	/* type of pseudo_btf_id */

			union {

				u32 btf_id;	/* btf_id for struct typed var */

				u32 mem_size;	/* mem_size for non-struct typed var */

			};

		} btf_var;

	};

	u64 map_key_state; /* constant (32 bit) key tracking for maps */

	int ctx_field_size; /* the ctx field size for load insn, maybe 0 */

include/linux/btf.h

+26 −0
Original line number Diff line number Diff line
@@ -110,6 +110,11 @@ btf_resolve_size(const struct btf *btf, const struct btf_type *type, 
	     i < btf_type_vlen(struct_type);			\

	     i++, member++)



#define for_each_vsi(i, datasec_type, member)			\

	for (i = 0, member = btf_type_var_secinfo(datasec_type);	\

	     i < btf_type_vlen(datasec_type);			\

	     i++, member++)



static inline bool btf_type_is_ptr(const struct btf_type *t)

{

	return BTF_INFO_KIND(t->info) == BTF_KIND_PTR;
@@ -145,6 +150,21 @@ static inline bool btf_type_is_func_proto(const struct btf_type *t) 
	return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC_PROTO;

}



static inline bool btf_type_is_var(const struct btf_type *t)

{

	return BTF_INFO_KIND(t->info) == BTF_KIND_VAR;

}



/* union is only a special case of struct:

 * all its offsetof(member) == 0

 */

static inline bool btf_type_is_struct(const struct btf_type *t)

{

	u8 kind = BTF_INFO_KIND(t->info);



	return kind == BTF_KIND_STRUCT || kind == BTF_KIND_UNION;

}



static inline u16 btf_type_vlen(const struct btf_type *t)

{

	return BTF_INFO_VLEN(t->info);
@@ -179,6 +199,12 @@ static inline const struct btf_member *btf_type_member(const struct btf_type *t) 
	return (const struct btf_member *)(t + 1);

}



static inline const struct btf_var_secinfo *btf_type_var_secinfo(

		const struct btf_type *t)

{

	return (const struct btf_var_secinfo *)(t + 1);

}



#ifdef CONFIG_BPF_SYSCALL

const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id);

const char *btf_name_by_offset(const struct btf *btf, u32 offset);

include/uapi/linux/bpf.h

+58 −9
Original line number Diff line number Diff line
@@ -356,18 +356,36 @@ enum bpf_link_type { 
#define BPF_F_SLEEPABLE		(1U << 4)



/* When BPF ldimm64's insn[0].src_reg != 0 then this can have

 * two extensions:

 *

 * insn[0].src_reg:  BPF_PSEUDO_MAP_FD   BPF_PSEUDO_MAP_VALUE

 * insn[0].imm:      map fd              map fd

 * insn[1].imm:      0                   offset into value

 * insn[0].off:      0                   0

 * insn[1].off:      0                   0

 * ldimm64 rewrite:  address of map      address of map[0]+offset

 * verifier type:    CONST_PTR_TO_MAP    PTR_TO_MAP_VALUE

 * the following extensions:

 *

 * insn[0].src_reg:  BPF_PSEUDO_MAP_FD

 * insn[0].imm:      map fd

 * insn[1].imm:      0

 * insn[0].off:      0

 * insn[1].off:      0

 * ldimm64 rewrite:  address of map

 * verifier type:    CONST_PTR_TO_MAP

 */

#define BPF_PSEUDO_MAP_FD	1

/* insn[0].src_reg:  BPF_PSEUDO_MAP_VALUE

 * insn[0].imm:      map fd

 * insn[1].imm:      offset into value

 * insn[0].off:      0

 * insn[1].off:      0

 * ldimm64 rewrite:  address of map[0]+offset

 * verifier type:    PTR_TO_MAP_VALUE

 */

#define BPF_PSEUDO_MAP_VALUE	2

/* insn[0].src_reg:  BPF_PSEUDO_BTF_ID

 * insn[0].imm:      kernel btd id of VAR

 * insn[1].imm:      0

 * insn[0].off:      0

 * insn[1].off:      0

 * ldimm64 rewrite:  address of the kernel variable

 * verifier type:    PTR_TO_BTF_ID or PTR_TO_MEM, depending on whether the var

 *                   is struct/union.

 */

#define BPF_PSEUDO_BTF_ID	3



/* when bpf_call->src_reg == BPF_PSEUDO_CALL, bpf_call->imm == pc-relative

 * offset to another bpf function
@@ -3668,6 +3686,35 @@ union bpf_attr { 
 * 	Return

 * 		The helper returns **TC_ACT_REDIRECT** on success or

 * 		**TC_ACT_SHOT** on error.

 *

 * void *bpf_per_cpu_ptr(const void *percpu_ptr, u32 cpu)

 *     Description

 *             Take a pointer to a percpu ksym, *percpu_ptr*, and return a

 *             pointer to the percpu kernel variable on *cpu*. A ksym is an

 *             extern variable decorated with '__ksym'. For ksym, there is a

 *             global var (either static or global) defined of the same name

 *             in the kernel. The ksym is percpu if the global var is percpu.

 *             The returned pointer points to the global percpu var on *cpu*.

 *

 *             bpf_per_cpu_ptr() has the same semantic as per_cpu_ptr() in the

 *             kernel, except that bpf_per_cpu_ptr() may return NULL. This

 *             happens if *cpu* is larger than nr_cpu_ids. The caller of

 *             bpf_per_cpu_ptr() must check the returned value.

 *     Return

 *             A pointer pointing to the kernel percpu variable on *cpu*, or

 *             NULL, if *cpu* is invalid.

 *

 * void *bpf_this_cpu_ptr(const void *percpu_ptr)

 *	Description

 *		Take a pointer to a percpu ksym, *percpu_ptr*, and return a

 *		pointer to the percpu kernel variable on this cpu. See the

 *		description of 'ksym' in **bpf_per_cpu_ptr**\ ().

 *

 *		bpf_this_cpu_ptr() has the same semantic as this_cpu_ptr() in

 *		the kernel. Different from **bpf_per_cpu_ptr**\ (), it would

 *		never return NULL.

 *	Return

 *		A pointer pointing to the kernel percpu variable on this cpu.

 */

#define __BPF_FUNC_MAPPER(FN)		\

	FN(unspec),			\
@@ -3823,6 +3870,8 @@ union bpf_attr { 
	FN(seq_printf_btf),		\

	FN(skb_cgroup_classid),		\

	FN(redirect_neigh),		\

	FN(bpf_per_cpu_ptr),            \

	FN(bpf_this_cpu_ptr),		\

	/* */



/* integer value in 'imm' field of BPF_CALL instruction selects which helper

kernel/bpf/btf.c

+0 −25
Original line number Diff line number Diff line
@@ -188,11 +188,6 @@ 
	     i < btf_type_vlen(struct_type);				\

	     i++, member++)



#define for_each_vsi(i, struct_type, member)			\

	for (i = 0, member = btf_type_var_secinfo(struct_type);	\

	     i < btf_type_vlen(struct_type);			\

	     i++, member++)



#define for_each_vsi_from(i, from, struct_type, member)				\

	for (i = from, member = btf_type_var_secinfo(struct_type) + from;	\

	     i < btf_type_vlen(struct_type);					\
@@ -440,16 +435,6 @@ static bool btf_type_nosize_or_null(const struct btf_type *t) 
	return !t || btf_type_nosize(t);

}



/* union is only a special case of struct:

 * all its offsetof(member) == 0

 */

static bool btf_type_is_struct(const struct btf_type *t)

{

	u8 kind = BTF_INFO_KIND(t->info);



	return kind == BTF_KIND_STRUCT || kind == BTF_KIND_UNION;

}



static bool __btf_type_is_struct(const struct btf_type *t)

{

	return BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT;
@@ -460,11 +445,6 @@ static bool btf_type_is_array(const struct btf_type *t) 
	return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY;

}



static bool btf_type_is_var(const struct btf_type *t)

{

	return BTF_INFO_KIND(t->info) == BTF_KIND_VAR;

}



static bool btf_type_is_datasec(const struct btf_type *t)

{

	return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC;
@@ -613,11 +593,6 @@ static const struct btf_var *btf_type_var(const struct btf_type *t) 
	return (const struct btf_var *)(t + 1);

}



static const struct btf_var_secinfo *btf_type_var_secinfo(const struct btf_type *t)

{

	return (const struct btf_var_secinfo *)(t + 1);

}



static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t)

{

	return kind_ops[BTF_INFO_KIND(t->info)];