Commit 29da17c4 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'libbpf: support custom .rodata.*/.data.* sections' 

Andrii Nakryiko says:

====================

This patch set refactors internals of libbpf to enable support for multiple
custom .rodata.* and .data.* sections. Each such section is backed by its own
BPF_MAP_TYPE_ARRAY, memory-mappable just like .rodata/.data. This is not
extended to .bss because .bss is not a great name, it is generated by compiler
with name that reflects completely irrelevant historical implementation
details. Given that users have to annotate their variables with
SEC(".data.my_sec") explicitly, standardizing on .rodata. and .data. prefixes
makes more sense and keeps things simpler.

Additionally, this patch set makes it simpler to work with those special
internal maps by allowing to look them up by their full ELF section name.

Patch #1 is a preparatory patch that deprecates one libbpf API and moves
custom logic into libbpf.c, where it's used. This code is later refactored
with the rest of libbpf.c logic to support multiple data section maps.

See individual patches for all the details.

For new custom "dot maps", their full ELF section names are used as the names
that are sent into the kernel. Object name isn't prepended like for
.data/.rodata/.bss. The reason is that with longer custom names, there isn't
much space left for object name anyways. Also, if BTF is supported,
btf_value_type_id points to DATASEC BTF type, which contains full original ELF
name of the section, so tools like bpftool could use that to recover full
name. This patch set doesn't add this logic yet, this is left for follow up
patches.

One interesting possibility that is now open by these changes is that it's
possible to do:

    bpf_trace_printk("My fmt %s", sizeof("My fmt %s"), "blah");

and it will work as expected. I haven't updated libbpf-provided helpers in
bpf_helpers.h for snprintf, seq_printf, and printk, because using
`static const char ___fmt[] = fmt;` trick is still efficient and doesn't fill
out the buffer at runtime (no copying). But we might consider updating them in
the future, especially with the array check that Kumar proposed (see [0]).

  [0] https://lore.kernel.org/bpf/20211012041524.udytbr2xs5wid6x2@apollo.localdomain/



v1->v2:
  - don't prepend object name for new dot maps;
  - add __read_mostly example in selftests (Daniel).
====================

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents b0c7663d 4f2511e1

tools/bpf/bpftool/gen.c

+105 −53
Original line number Diff line number Diff line
@@ -33,6 +33,11 @@ static void sanitize_identifier(char *name) 
			name[i] = '_';

}



static bool str_has_prefix(const char *str, const char *prefix)

{

	return strncmp(str, prefix, strlen(prefix)) == 0;

}



static bool str_has_suffix(const char *str, const char *suffix)

{

	size_t i, n1 = strlen(str), n2 = strlen(suffix);
@@ -67,23 +72,47 @@ static void get_header_guard(char *guard, const char *obj_name) 
		guard[i] = toupper(guard[i]);

}



static const char *get_map_ident(const struct bpf_map *map)

static bool get_map_ident(const struct bpf_map *map, char *buf, size_t buf_sz)

{

	static const char *sfxs[] = { ".data", ".rodata", ".bss", ".kconfig" };

	const char *name = bpf_map__name(map);

	int i, n;



	if (!bpf_map__is_internal(map))

		return name;



	if (str_has_suffix(name, ".data"))

		return "data";

	else if (str_has_suffix(name, ".rodata"))

		return "rodata";

	else if (str_has_suffix(name, ".bss"))

		return "bss";

	else if (str_has_suffix(name, ".kconfig"))

		return "kconfig";

	else

		return NULL;

	if (!bpf_map__is_internal(map)) {

		snprintf(buf, buf_sz, "%s", name);

		return true;

	}



	for  (i = 0, n = ARRAY_SIZE(sfxs); i < n; i++) {

		const char *sfx = sfxs[i], *p;



		p = strstr(name, sfx);

		if (p) {

			snprintf(buf, buf_sz, "%s", p + 1);

			sanitize_identifier(buf);

			return true;

		}

	}



	return false;

}



static bool get_datasec_ident(const char *sec_name, char *buf, size_t buf_sz)

{

	static const char *pfxs[] = { ".data", ".rodata", ".bss", ".kconfig" };

	int i, n;



	for  (i = 0, n = ARRAY_SIZE(pfxs); i < n; i++) {

		const char *pfx = pfxs[i];



		if (str_has_prefix(sec_name, pfx)) {

			snprintf(buf, buf_sz, "%s", sec_name + 1);

			sanitize_identifier(buf);

			return true;

		}

	}



	return false;

}



static void codegen_btf_dump_printf(void *ctx, const char *fmt, va_list args)
@@ -100,24 +129,14 @@ static int codegen_datasec_def(struct bpf_object *obj, 
	const char *sec_name = btf__name_by_offset(btf, sec->name_off);

	const struct btf_var_secinfo *sec_var = btf_var_secinfos(sec);

	int i, err, off = 0, pad_cnt = 0, vlen = btf_vlen(sec);

	const char *sec_ident;

	char var_ident[256];

	char var_ident[256], sec_ident[256];

	bool strip_mods = false;



	if (strcmp(sec_name, ".data") == 0) {

		sec_ident = "data";

		strip_mods = true;

	} else if (strcmp(sec_name, ".bss") == 0) {

		sec_ident = "bss";

		strip_mods = true;

	} else if (strcmp(sec_name, ".rodata") == 0) {

		sec_ident = "rodata";

		strip_mods = true;

	} else if (strcmp(sec_name, ".kconfig") == 0) {

		sec_ident = "kconfig";

	} else {

	if (!get_datasec_ident(sec_name, sec_ident, sizeof(sec_ident)))

		return 0;

	}



	if (strcmp(sec_name, ".kconfig") != 0)

		strip_mods = true;



	printf("	struct %s__%s {\n", obj_name, sec_ident);

	for (i = 0; i < vlen; i++, sec_var++) {
@@ -194,22 +213,61 @@ static int codegen_datasecs(struct bpf_object *obj, const char *obj_name) 
	struct btf *btf = bpf_object__btf(obj);

	int n = btf__get_nr_types(btf);

	struct btf_dump *d;

	struct bpf_map *map;

	const struct btf_type *sec;

	char sec_ident[256], map_ident[256];

	int i, err = 0;



	d = btf_dump__new(btf, NULL, NULL, codegen_btf_dump_printf);

	if (IS_ERR(d))

		return PTR_ERR(d);



	bpf_object__for_each_map(map, obj) {

		/* only generate definitions for memory-mapped internal maps */

		if (!bpf_map__is_internal(map))

			continue;

		if (!(bpf_map__def(map)->map_flags & BPF_F_MMAPABLE))

			continue;



		if (!get_map_ident(map, map_ident, sizeof(map_ident)))

			continue;



		sec = NULL;

		for (i = 1; i <= n; i++) {

			const struct btf_type *t = btf__type_by_id(btf, i);

			const char *name;



			if (!btf_is_datasec(t))

				continue;



		err = codegen_datasec_def(obj, btf, d, t, obj_name);

			name = btf__str_by_offset(btf, t->name_off);

			if (!get_datasec_ident(name, sec_ident, sizeof(sec_ident)))

				continue;



			if (strcmp(sec_ident, map_ident) == 0) {

				sec = t;

				break;

			}

		}



		/* In some cases (e.g., sections like .rodata.cst16 containing

		 * compiler allocated string constants only) there will be

		 * special internal maps with no corresponding DATASEC BTF

		 * type. In such case, generate empty structs for each such

		 * map. It will still be memory-mapped and its contents

		 * accessible from user-space through BPF skeleton.

		 */

		if (!sec) {

			printf("	struct %s__%s {\n", obj_name, map_ident);

			printf("	} *%s;\n", map_ident);

		} else {

			err = codegen_datasec_def(obj, btf, d, sec, obj_name);

			if (err)

				goto out;

		}

	}





out:

	btf_dump__free(d);

	return err;
@@ -385,6 +443,7 @@ static void codegen_destroy(struct bpf_object *obj, const char *obj_name) 
{

	struct bpf_program *prog;

	struct bpf_map *map;

	char ident[256];



	codegen("\

		\n\
@@ -405,10 +464,7 @@ static void codegen_destroy(struct bpf_object *obj, const char *obj_name) 
	}



	bpf_object__for_each_map(map, obj) {

		const char *ident;



		ident = get_map_ident(map);

		if (!ident)

		if (!get_map_ident(map, ident, sizeof(ident)))

			continue;

		if (bpf_map__is_internal(map) &&

		    (bpf_map__def(map)->map_flags & BPF_F_MMAPABLE))
@@ -432,6 +488,7 @@ static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *h 
	struct bpf_object_load_attr load_attr = {};

	DECLARE_LIBBPF_OPTS(gen_loader_opts, opts);

	struct bpf_map *map;

	char ident[256];

	int err = 0;



	err = bpf_object__gen_loader(obj, &opts);
@@ -477,12 +534,10 @@ static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *h 
		",

		obj_name, opts.data_sz);

	bpf_object__for_each_map(map, obj) {

		const char *ident;

		const void *mmap_data = NULL;

		size_t mmap_size = 0;



		ident = get_map_ident(map);

		if (!ident)

		if (!get_map_ident(map, ident, sizeof(ident)))

			continue;



		if (!bpf_map__is_internal(map) ||
@@ -544,15 +599,15 @@ static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *h 
				return err;				    \n\

		", obj_name);

	bpf_object__for_each_map(map, obj) {

		const char *ident, *mmap_flags;

		const char *mmap_flags;



		ident = get_map_ident(map);

		if (!ident)

		if (!get_map_ident(map, ident, sizeof(ident)))

			continue;



		if (!bpf_map__is_internal(map) ||

		    !(bpf_map__def(map)->map_flags & BPF_F_MMAPABLE))

			continue;



		if (bpf_map__def(map)->map_flags & BPF_F_RDONLY_PROG)

			mmap_flags = "PROT_READ";

		else
@@ -602,7 +657,8 @@ static int do_skeleton(int argc, char **argv) 
	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts);

	char obj_name[MAX_OBJ_NAME_LEN] = "", *obj_data;

	struct bpf_object *obj = NULL;

	const char *file, *ident;

	const char *file;

	char ident[256];

	struct bpf_program *prog;

	int fd, err = -1;

	struct bpf_map *map;
@@ -673,8 +729,7 @@ static int do_skeleton(int argc, char **argv) 
	}



	bpf_object__for_each_map(map, obj) {

		ident = get_map_ident(map);

		if (!ident) {

		if (!get_map_ident(map, ident, sizeof(ident))) {

			p_err("ignoring unrecognized internal map '%s'...",

			      bpf_map__name(map));

			continue;
@@ -727,8 +782,7 @@ static int do_skeleton(int argc, char **argv) 
	if (map_cnt) {

		printf("\tstruct {\n");

		bpf_object__for_each_map(map, obj) {

			ident = get_map_ident(map);

			if (!ident)

			if (!get_map_ident(map, ident, sizeof(ident)))

				continue;

			if (use_loader)

				printf("\t\tstruct bpf_map_desc %s;\n", ident);
@@ -897,9 +951,7 @@ static int do_skeleton(int argc, char **argv) 
		);

		i = 0;

		bpf_object__for_each_map(map, obj) {

			ident = get_map_ident(map);



			if (!ident)

			if (!get_map_ident(map, ident, sizeof(ident)))

				continue;



			codegen("\

tools/lib/bpf/btf.c

+0 −93
Original line number Diff line number Diff line
@@ -1107,99 +1107,6 @@ struct btf *btf__parse_split(const char *path, struct btf *base_btf) 
	return libbpf_ptr(btf_parse(path, base_btf, NULL));

}



static int compare_vsi_off(const void *_a, const void *_b)

{

	const struct btf_var_secinfo *a = _a;

	const struct btf_var_secinfo *b = _b;



	return a->offset - b->offset;

}



static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf,

			     struct btf_type *t)

{

	__u32 size = 0, off = 0, i, vars = btf_vlen(t);

	const char *name = btf__name_by_offset(btf, t->name_off);

	const struct btf_type *t_var;

	struct btf_var_secinfo *vsi;

	const struct btf_var *var;

	int ret;



	if (!name) {

		pr_debug("No name found in string section for DATASEC kind.\n");

		return -ENOENT;

	}



	/* .extern datasec size and var offsets were set correctly during

	 * extern collection step, so just skip straight to sorting variables

	 */

	if (t->size)

		goto sort_vars;



	ret = bpf_object__section_size(obj, name, &size);

	if (ret || !size || (t->size && t->size != size)) {

		pr_debug("Invalid size for section %s: %u bytes\n", name, size);

		return -ENOENT;

	}



	t->size = size;



	for (i = 0, vsi = btf_var_secinfos(t); i < vars; i++, vsi++) {

		t_var = btf__type_by_id(btf, vsi->type);

		var = btf_var(t_var);



		if (!btf_is_var(t_var)) {

			pr_debug("Non-VAR type seen in section %s\n", name);

			return -EINVAL;

		}



		if (var->linkage == BTF_VAR_STATIC)

			continue;



		name = btf__name_by_offset(btf, t_var->name_off);

		if (!name) {

			pr_debug("No name found in string section for VAR kind\n");

			return -ENOENT;

		}



		ret = bpf_object__variable_offset(obj, name, &off);

		if (ret) {

			pr_debug("No offset found in symbol table for VAR %s\n",

				 name);

			return -ENOENT;

		}



		vsi->offset = off;

	}



sort_vars:

	qsort(btf_var_secinfos(t), vars, sizeof(*vsi), compare_vsi_off);

	return 0;

}



int btf__finalize_data(struct bpf_object *obj, struct btf *btf)

{

	int err = 0;

	__u32 i;



	for (i = 1; i <= btf->nr_types; i++) {

		struct btf_type *t = btf_type_by_id(btf, i);



		/* Loader needs to fix up some of the things compiler

		 * couldn't get its hands on while emitting BTF. This

		 * is section size and global variable offset. We use

		 * the info from the ELF itself for this purpose.

		 */

		if (btf_is_datasec(t)) {

			err = btf_fixup_datasec(obj, btf, t);

			if (err)

				break;

		}

	}



	return libbpf_err(err);

}



static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian);



int btf__load_into_kernel(struct btf *btf)

tools/lib/bpf/btf.h

+1 −0
Original line number Diff line number Diff line
@@ -123,6 +123,7 @@ LIBBPF_API struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *b 
LIBBPF_DEPRECATED_SINCE(0, 6, "use btf__load_from_kernel_by_id instead")

LIBBPF_API int btf__get_from_id(__u32 id, struct btf **btf);



LIBBPF_DEPRECATED_SINCE(0, 6, "intended for internal libbpf use only")

LIBBPF_API int btf__finalize_data(struct bpf_object *obj, struct btf *btf);

LIBBPF_DEPRECATED_SINCE(0, 6, "use btf__load_into_kernel instead")

LIBBPF_API int btf__load(struct btf *btf);

tools/lib/bpf/libbpf.c

+562 −331

File changed.

Preview size limit exceeded, changes collapsed.

tools/lib/bpf/libbpf_internal.h

+2 −6
Original line number Diff line number Diff line
@@ -52,8 +52,8 @@ 
#endif



/* Older libelf all end up in this expression, for both 32 and 64 bit */

#ifndef GELF_ST_VISIBILITY

#define GELF_ST_VISIBILITY(o) ((o) & 0x03)

#ifndef ELF64_ST_VISIBILITY

#define ELF64_ST_VISIBILITY(o) ((o) & 0x03)

#endif



#define BTF_INFO_ENC(kind, kind_flag, vlen) \
@@ -303,10 +303,6 @@ struct bpf_prog_load_params { 


int libbpf__bpf_prog_load(const struct bpf_prog_load_params *load_attr);



int bpf_object__section_size(const struct bpf_object *obj, const char *name,

			     __u32 *size);

int bpf_object__variable_offset(const struct bpf_object *obj, const char *name,

				__u32 *off);

struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf);

void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type,

				const char **prefix, int *kind);