Commit 7d3c1077 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'BPF static linker: support externs' 

Andrii Nakryiko says:

====================

Add BPF static linker support for extern resolution of global variables,
functions, and BTF-defined maps.

This patch set consists of 4 parts:
  - few patches are extending bpftool to simplify working with BTF dump;
  - libbpf object loading logic is extended to support __hidden functions and
    overriden (unused) __weak functions; also BTF-defined map parsing logic is
    refactored to be re-used by linker;
  - the crux of the patch set is BPF static linker logic extension to perform
    extern resolution for three categories: global variables, BPF
    sub-programs, BTF-defined maps;
  - a set of selftests that validate that all the combinations of
    extern/weak/__hidden are working as expected.

See respective patches for more details.

One aspect hasn't been addressed yet and is going to be resolved in the next
patch set, but is worth mentioning. With BPF static linking of multiple .o
files, dealing with static everything becomes more problematic for BPF
skeleton and in general for any by name look up APIs. This is due to static
entities are allowed to have non-unique name. Historically this was never
a problem due to BPF programs were always confined to a single C file. That
changes now and needs to be addressed. The thinking so far is for BPF static
linker to prepend filename to each static variable and static map (which is
currently not supported by libbpf, btw), so that they can be unambiguously
resolved by (mostly) unique name. Mostly, because even filenames can be
duplicated, but that should be rare and easy to address by wiser choice of
filenames by users. Fortunately, static BPF subprograms don't suffer from this
issues, as they are not independent entities and are neither exposed in BPF
skeleton, nor is lookup-able by any of libbpf APIs (and there is little reason
to do that anyways).

This and few other things will be the topic of the next set of patches.

Some tests rely on Clang fix ([0]), so need latest Clang built from main.

  [0] https://reviews.llvm.org/D100362



v2->v3:
  - allow only STV_DEFAULT and STV_HIDDEN ELF symbol visibility (Yonghong);
  - update selftests' README for required Clang 13 fix dependency (Alexei);
  - comments, typos, slight code changes (Yonghong, Alexei);

v1->v2:
  - make map externs support full attribute list, adjust linked_maps selftest
    to demonstrate that typedef works now (though no shared header file was
    added to simplicity sake) (Alexei);
  - remove commented out parts from selftests and fix few minor code style
    issues;
  - special __weak map definition semantics not yet implemented and will be
    addressed in a follow up.
====================

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents b1b9f535 a9dab4e4

tools/bpf/bpftool/btf.c

+21 −9
Original line number Diff line number Diff line
@@ -71,7 +71,9 @@ static const char *btf_var_linkage_str(__u32 linkage) 
	case BTF_VAR_STATIC:

		return "static";

	case BTF_VAR_GLOBAL_ALLOCATED:

		return "global-alloc";

		return "global";

	case BTF_VAR_GLOBAL_EXTERN:

		return "extern";

	default:

		return "(unknown)";

	}
@@ -98,26 +100,28 @@ static const char *btf_str(const struct btf *btf, __u32 off) 
	return btf__name_by_offset(btf, off) ? : "(invalid)";

}



static int btf_kind_safe(int kind)

{

	return kind <= BTF_KIND_MAX ? kind : BTF_KIND_UNKN;

}



static int dump_btf_type(const struct btf *btf, __u32 id,

			 const struct btf_type *t)

{

	json_writer_t *w = json_wtr;

	int kind, safe_kind;



	kind = BTF_INFO_KIND(t->info);

	safe_kind = kind <= BTF_KIND_MAX ? kind : BTF_KIND_UNKN;

	int kind = btf_kind(t);



	if (json_output) {

		jsonw_start_object(w);

		jsonw_uint_field(w, "id", id);

		jsonw_string_field(w, "kind", btf_kind_str[safe_kind]);

		jsonw_string_field(w, "kind", btf_kind_str[btf_kind_safe(kind)]);

		jsonw_string_field(w, "name", btf_str(btf, t->name_off));

	} else {

		printf("[%u] %s '%s'", id, btf_kind_str[safe_kind],

		printf("[%u] %s '%s'", id, btf_kind_str[btf_kind_safe(kind)],

		       btf_str(btf, t->name_off));

	}



	switch (BTF_INFO_KIND(t->info)) {

	switch (kind) {

	case BTF_KIND_INT: {

		__u32 v = *(__u32 *)(t + 1);

		const char *enc;
@@ -301,6 +305,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, 
	}

	case BTF_KIND_DATASEC: {

		const struct btf_var_secinfo *v = (const void *)(t + 1);

		const struct btf_type *vt;

		__u16 vlen = BTF_INFO_VLEN(t->info);

		int i;
@@ -322,6 +327,13 @@ static int dump_btf_type(const struct btf *btf, __u32 id, 
			} else {

				printf("\n\ttype_id=%u offset=%u size=%u",

				       v->type, v->offset, v->size);



				if (v->type <= btf__get_nr_types(btf)) {

					vt = btf__type_by_id(btf, v->type);

					printf(" (%s '%s')",

					       btf_kind_str[btf_kind_safe(btf_kind(vt))],

					       btf_str(btf, vt->name_off));

				}

			}

		}

		if (json_output)

tools/lib/bpf/bpf_helpers.h

+17 −2
Original line number Diff line number Diff line
@@ -25,9 +25,16 @@ 
/*

 * Helper macro to place programs, maps, license in

 * different sections in elf_bpf file. Section names

 * are interpreted by elf_bpf loader

 * are interpreted by libbpf depending on the context (BPF programs, BPF maps,

 * extern variables, etc).

 * To allow use of SEC() with externs (e.g., for extern .maps declarations),

 * make sure __attribute__((unused)) doesn't trigger compilation warning.

 */

#define SEC(NAME) __attribute__((section(NAME), used))

#define SEC(name) \

	_Pragma("GCC diagnostic push")					    \

	_Pragma("GCC diagnostic ignored \"-Wignored-attributes\"")	    \

	__attribute__((section(name), used))				    \

	_Pragma("GCC diagnostic pop")					    \



/* Avoid 'linux/stddef.h' definition of '__always_inline'. */

#undef __always_inline
@@ -40,6 +47,14 @@ 
#define __weak __attribute__((weak))

#endif



/*

 * Use __hidden attribute to mark a non-static BPF subprogram effectively

 * static for BPF verifier's verification algorithm purposes, allowing more

 * extensive and permissive BPF verification process, taking into account

 * subprogram's caller context.

 */

#define __hidden __attribute__((visibility("hidden")))



/* When utilizing vmlinux.h with BPF CO-RE, user BPF programs can't include

 * any system-level headers (such as stddef.h, linux/version.h, etc), and

 * commonly-used macros like NULL and KERNEL_VERSION aren't available through

tools/lib/bpf/btf.c

+0 −5
Original line number Diff line number Diff line
@@ -1605,11 +1605,6 @@ static void *btf_add_type_mem(struct btf *btf, size_t add_sz) 
			      btf->hdr->type_len, UINT_MAX, add_sz);

}



static __u32 btf_type_info(int kind, int vlen, int kflag)

{

	return (kflag << 31) | (kind << 24) | vlen;

}



static void btf_type_inc_vlen(struct btf_type *t)

{

	t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, btf_kflag(t));

tools/lib/bpf/libbpf.c

+215 −158
Original line number Diff line number Diff line
@@ -69,8 +69,7 @@ 
#define __printf(a, b)	__attribute__((format(printf, a, b)))



static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);

static const struct btf_type *

skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id);

static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);



static int __base_pr(enum libbpf_print_level level, const char *format,

		     va_list args)
@@ -274,6 +273,7 @@ struct bpf_program { 
	bpf_program_clear_priv_t clear_priv;



	bool load;

	bool mark_btf_static;

	enum bpf_prog_type type;

	enum bpf_attach_type expected_attach_type;

	int prog_ifindex;
@@ -500,8 +500,6 @@ static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name); 
static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr);

static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn);

static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn);

static int elf_sym_by_sec_off(const struct bpf_object *obj, size_t sec_idx,

			      size_t off, __u32 sym_type, GElf_Sym *sym);



void bpf_program__unload(struct bpf_program *prog)

{
@@ -642,25 +640,29 @@ static int 
bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data,

			 const char *sec_name, int sec_idx)

{

	Elf_Data *symbols = obj->efile.symbols;

	struct bpf_program *prog, *progs;

	void *data = sec_data->d_buf;

	size_t sec_sz = sec_data->d_size, sec_off, prog_sz;

	int nr_progs, err;

	size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms;

	int nr_progs, err, i;

	const char *name;

	GElf_Sym sym;



	progs = obj->programs;

	nr_progs = obj->nr_programs;

	nr_syms = symbols->d_size / sizeof(GElf_Sym);

	sec_off = 0;



	while (sec_off < sec_sz) {

		if (elf_sym_by_sec_off(obj, sec_idx, sec_off, STT_FUNC, &sym)) {

			pr_warn("sec '%s': failed to find program symbol at offset %zu\n",

				sec_name, sec_off);

			return -LIBBPF_ERRNO__FORMAT;

		}

	for (i = 0; i < nr_syms; i++) {

		if (!gelf_getsym(symbols, i, &sym))

			continue;

		if (sym.st_shndx != sec_idx)

			continue;

		if (GELF_ST_TYPE(sym.st_info) != STT_FUNC)

			continue;



		prog_sz = sym.st_size;

		sec_off = sym.st_value;



		name = elf_sym_str(obj, sym.st_name);

		if (!name) {
@@ -698,10 +700,17 @@ bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data, 
		if (err)

			return err;



		/* if function is a global/weak symbol, but has hidden

		 * visibility (STV_HIDDEN), mark its BTF FUNC as static to

		 * enable more permissive BPF verification mode with more

		 * outside context available to BPF verifier

		 */

		if (GELF_ST_BIND(sym.st_info) != STB_LOCAL

		    && GELF_ST_VISIBILITY(sym.st_other) == STV_HIDDEN)

			prog->mark_btf_static = true;



		nr_progs++;

		obj->nr_programs = nr_progs;



		sec_off += prog_sz;

	}



	return 0;
@@ -1895,7 +1904,7 @@ static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) 
	return 0;

}



static const struct btf_type *

const struct btf_type *

skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)

{

	const struct btf_type *t = btf__type_by_id(btf, id);
@@ -1950,16 +1959,11 @@ static const char *__btf_kind_str(__u16 kind) 
	}

}



static const char *btf_kind_str(const struct btf_type *t)

const char *btf_kind_str(const struct btf_type *t)

{

	return __btf_kind_str(btf_kind(t));

}



static enum btf_func_linkage btf_func_linkage(const struct btf_type *t)

{

	return (enum btf_func_linkage)BTF_INFO_VLEN(t->info);

}



/*

 * Fetch integer attribute of BTF map definition. Such attributes are

 * represented using a pointer to an array, in which dimensionality of array
@@ -2014,255 +2018,262 @@ static int build_map_pin_path(struct bpf_map *map, const char *path) 
	return bpf_map__set_pin_path(map, buf);

}





static int parse_btf_map_def(struct bpf_object *obj,

			     struct bpf_map *map,

			     const struct btf_type *def,

			     bool strict, bool is_inner,

			     const char *pin_root_path)

int parse_btf_map_def(const char *map_name, struct btf *btf,

		      const struct btf_type *def_t, bool strict,

		      struct btf_map_def *map_def, struct btf_map_def *inner_def)

{

	const struct btf_type *t;

	const struct btf_member *m;

	bool is_inner = inner_def == NULL;

	int vlen, i;



	vlen = btf_vlen(def);

	m = btf_members(def);

	vlen = btf_vlen(def_t);

	m = btf_members(def_t);

	for (i = 0; i < vlen; i++, m++) {

		const char *name = btf__name_by_offset(obj->btf, m->name_off);

		const char *name = btf__name_by_offset(btf, m->name_off);



		if (!name) {

			pr_warn("map '%s': invalid field #%d.\n", map->name, i);

			pr_warn("map '%s': invalid field #%d.\n", map_name, i);

			return -EINVAL;

		}

		if (strcmp(name, "type") == 0) {

			if (!get_map_field_int(map->name, obj->btf, m,

					       &map->def.type))

			if (!get_map_field_int(map_name, btf, m, &map_def->map_type))

				return -EINVAL;

			pr_debug("map '%s': found type = %u.\n",

				 map->name, map->def.type);

			map_def->parts |= MAP_DEF_MAP_TYPE;

		} else if (strcmp(name, "max_entries") == 0) {

			if (!get_map_field_int(map->name, obj->btf, m,

					       &map->def.max_entries))

			if (!get_map_field_int(map_name, btf, m, &map_def->max_entries))

				return -EINVAL;

			pr_debug("map '%s': found max_entries = %u.\n",

				 map->name, map->def.max_entries);

			map_def->parts |= MAP_DEF_MAX_ENTRIES;

		} else if (strcmp(name, "map_flags") == 0) {

			if (!get_map_field_int(map->name, obj->btf, m,

					       &map->def.map_flags))

			if (!get_map_field_int(map_name, btf, m, &map_def->map_flags))

				return -EINVAL;

			pr_debug("map '%s': found map_flags = %u.\n",

				 map->name, map->def.map_flags);

			map_def->parts |= MAP_DEF_MAP_FLAGS;

		} else if (strcmp(name, "numa_node") == 0) {

			if (!get_map_field_int(map->name, obj->btf, m, &map->numa_node))

			if (!get_map_field_int(map_name, btf, m, &map_def->numa_node))

				return -EINVAL;

			pr_debug("map '%s': found numa_node = %u.\n", map->name, map->numa_node);

			map_def->parts |= MAP_DEF_NUMA_NODE;

		} else if (strcmp(name, "key_size") == 0) {

			__u32 sz;



			if (!get_map_field_int(map->name, obj->btf, m, &sz))

			if (!get_map_field_int(map_name, btf, m, &sz))

				return -EINVAL;

			pr_debug("map '%s': found key_size = %u.\n",

				 map->name, sz);

			if (map->def.key_size && map->def.key_size != sz) {

			if (map_def->key_size && map_def->key_size != sz) {

				pr_warn("map '%s': conflicting key size %u != %u.\n",

					map->name, map->def.key_size, sz);

					map_name, map_def->key_size, sz);

				return -EINVAL;

			}

			map->def.key_size = sz;

			map_def->key_size = sz;

			map_def->parts |= MAP_DEF_KEY_SIZE;

		} else if (strcmp(name, "key") == 0) {

			__s64 sz;



			t = btf__type_by_id(obj->btf, m->type);

			t = btf__type_by_id(btf, m->type);

			if (!t) {

				pr_warn("map '%s': key type [%d] not found.\n",

					map->name, m->type);

					map_name, m->type);

				return -EINVAL;

			}

			if (!btf_is_ptr(t)) {

				pr_warn("map '%s': key spec is not PTR: %s.\n",

					map->name, btf_kind_str(t));

					map_name, btf_kind_str(t));

				return -EINVAL;

			}

			sz = btf__resolve_size(obj->btf, t->type);

			sz = btf__resolve_size(btf, t->type);

			if (sz < 0) {

				pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",

					map->name, t->type, (ssize_t)sz);

					map_name, t->type, (ssize_t)sz);

				return sz;

			}

			pr_debug("map '%s': found key [%u], sz = %zd.\n",

				 map->name, t->type, (ssize_t)sz);

			if (map->def.key_size && map->def.key_size != sz) {

			if (map_def->key_size && map_def->key_size != sz) {

				pr_warn("map '%s': conflicting key size %u != %zd.\n",

					map->name, map->def.key_size, (ssize_t)sz);

					map_name, map_def->key_size, (ssize_t)sz);

				return -EINVAL;

			}

			map->def.key_size = sz;

			map->btf_key_type_id = t->type;

			map_def->key_size = sz;

			map_def->key_type_id = t->type;

			map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE;

		} else if (strcmp(name, "value_size") == 0) {

			__u32 sz;



			if (!get_map_field_int(map->name, obj->btf, m, &sz))

			if (!get_map_field_int(map_name, btf, m, &sz))

				return -EINVAL;

			pr_debug("map '%s': found value_size = %u.\n",

				 map->name, sz);

			if (map->def.value_size && map->def.value_size != sz) {

			if (map_def->value_size && map_def->value_size != sz) {

				pr_warn("map '%s': conflicting value size %u != %u.\n",

					map->name, map->def.value_size, sz);

					map_name, map_def->value_size, sz);

				return -EINVAL;

			}

			map->def.value_size = sz;

			map_def->value_size = sz;

			map_def->parts |= MAP_DEF_VALUE_SIZE;

		} else if (strcmp(name, "value") == 0) {

			__s64 sz;



			t = btf__type_by_id(obj->btf, m->type);

			t = btf__type_by_id(btf, m->type);

			if (!t) {

				pr_warn("map '%s': value type [%d] not found.\n",

					map->name, m->type);

					map_name, m->type);

				return -EINVAL;

			}

			if (!btf_is_ptr(t)) {

				pr_warn("map '%s': value spec is not PTR: %s.\n",

					map->name, btf_kind_str(t));

					map_name, btf_kind_str(t));

				return -EINVAL;

			}

			sz = btf__resolve_size(obj->btf, t->type);

			sz = btf__resolve_size(btf, t->type);

			if (sz < 0) {

				pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",

					map->name, t->type, (ssize_t)sz);

					map_name, t->type, (ssize_t)sz);

				return sz;

			}

			pr_debug("map '%s': found value [%u], sz = %zd.\n",

				 map->name, t->type, (ssize_t)sz);

			if (map->def.value_size && map->def.value_size != sz) {

			if (map_def->value_size && map_def->value_size != sz) {

				pr_warn("map '%s': conflicting value size %u != %zd.\n",

					map->name, map->def.value_size, (ssize_t)sz);

					map_name, map_def->value_size, (ssize_t)sz);

				return -EINVAL;

			}

			map->def.value_size = sz;

			map->btf_value_type_id = t->type;

			map_def->value_size = sz;

			map_def->value_type_id = t->type;

			map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE;

		}

		else if (strcmp(name, "values") == 0) {

			char inner_map_name[128];

			int err;



			if (is_inner) {

				pr_warn("map '%s': multi-level inner maps not supported.\n",

					map->name);

					map_name);

				return -ENOTSUP;

			}

			if (i != vlen - 1) {

				pr_warn("map '%s': '%s' member should be last.\n",

					map->name, name);

					map_name, name);

				return -EINVAL;

			}

			if (!bpf_map_type__is_map_in_map(map->def.type)) {

			if (!bpf_map_type__is_map_in_map(map_def->map_type)) {

				pr_warn("map '%s': should be map-in-map.\n",

					map->name);

					map_name);

				return -ENOTSUP;

			}

			if (map->def.value_size && map->def.value_size != 4) {

			if (map_def->value_size && map_def->value_size != 4) {

				pr_warn("map '%s': conflicting value size %u != 4.\n",

					map->name, map->def.value_size);

					map_name, map_def->value_size);

				return -EINVAL;

			}

			map->def.value_size = 4;

			t = btf__type_by_id(obj->btf, m->type);

			map_def->value_size = 4;

			t = btf__type_by_id(btf, m->type);

			if (!t) {

				pr_warn("map '%s': map-in-map inner type [%d] not found.\n",

					map->name, m->type);

					map_name, m->type);

				return -EINVAL;

			}

			if (!btf_is_array(t) || btf_array(t)->nelems) {

				pr_warn("map '%s': map-in-map inner spec is not a zero-sized array.\n",

					map->name);

					map_name);

				return -EINVAL;

			}

			t = skip_mods_and_typedefs(obj->btf, btf_array(t)->type,

						   NULL);

			t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL);

			if (!btf_is_ptr(t)) {

				pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",

					map->name, btf_kind_str(t));

					map_name, btf_kind_str(t));

				return -EINVAL;

			}

			t = skip_mods_and_typedefs(obj->btf, t->type, NULL);

			t = skip_mods_and_typedefs(btf, t->type, NULL);

			if (!btf_is_struct(t)) {

				pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",

					map->name, btf_kind_str(t));

					map_name, btf_kind_str(t));

				return -EINVAL;

			}



			map->inner_map = calloc(1, sizeof(*map->inner_map));

			if (!map->inner_map)

				return -ENOMEM;

			map->inner_map->fd = -1;

			map->inner_map->sec_idx = obj->efile.btf_maps_shndx;

			map->inner_map->name = malloc(strlen(map->name) +

						      sizeof(".inner") + 1);

			if (!map->inner_map->name)

				return -ENOMEM;

			sprintf(map->inner_map->name, "%s.inner", map->name);



			err = parse_btf_map_def(obj, map->inner_map, t, strict,

						true /* is_inner */, NULL);

			snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name);

			err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL);

			if (err)

				return err;



			map_def->parts |= MAP_DEF_INNER_MAP;

		} else if (strcmp(name, "pinning") == 0) {

			__u32 val;

			int err;



			if (is_inner) {

				pr_debug("map '%s': inner def can't be pinned.\n",

					 map->name);

				pr_warn("map '%s': inner def can't be pinned.\n", map_name);

				return -EINVAL;

			}

			if (!get_map_field_int(map->name, obj->btf, m, &val))

			if (!get_map_field_int(map_name, btf, m, &val))

				return -EINVAL;

			pr_debug("map '%s': found pinning = %u.\n",

				 map->name, val);



			if (val != LIBBPF_PIN_NONE &&

			    val != LIBBPF_PIN_BY_NAME) {

			if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) {

				pr_warn("map '%s': invalid pinning value %u.\n",

					map->name, val);

					map_name, val);

				return -EINVAL;

			}

			if (val == LIBBPF_PIN_BY_NAME) {

				err = build_map_pin_path(map, pin_root_path);

				if (err) {

					pr_warn("map '%s': couldn't build pin path.\n",

						map->name);

					return err;

				}

			}

			map_def->pinning = val;

			map_def->parts |= MAP_DEF_PINNING;

		} else {

			if (strict) {

				pr_warn("map '%s': unknown field '%s'.\n",

					map->name, name);

				pr_warn("map '%s': unknown field '%s'.\n", map_name, name);

				return -ENOTSUP;

			}

			pr_debug("map '%s': ignoring unknown field '%s'.\n",

				 map->name, name);

			pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name);

		}

	}



	if (map->def.type == BPF_MAP_TYPE_UNSPEC) {

		pr_warn("map '%s': map type isn't specified.\n", map->name);

	if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) {

		pr_warn("map '%s': map type isn't specified.\n", map_name);

		return -EINVAL;

	}



	return 0;

}



static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def)

{

	map->def.type = def->map_type;

	map->def.key_size = def->key_size;

	map->def.value_size = def->value_size;

	map->def.max_entries = def->max_entries;

	map->def.map_flags = def->map_flags;



	map->numa_node = def->numa_node;

	map->btf_key_type_id = def->key_type_id;

	map->btf_value_type_id = def->value_type_id;



	if (def->parts & MAP_DEF_MAP_TYPE)

		pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);



	if (def->parts & MAP_DEF_KEY_TYPE)

		pr_debug("map '%s': found key [%u], sz = %u.\n",

			 map->name, def->key_type_id, def->key_size);

	else if (def->parts & MAP_DEF_KEY_SIZE)

		pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size);



	if (def->parts & MAP_DEF_VALUE_TYPE)

		pr_debug("map '%s': found value [%u], sz = %u.\n",

			 map->name, def->value_type_id, def->value_size);

	else if (def->parts & MAP_DEF_VALUE_SIZE)

		pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size);



	if (def->parts & MAP_DEF_MAX_ENTRIES)

		pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries);

	if (def->parts & MAP_DEF_MAP_FLAGS)

		pr_debug("map '%s': found map_flags = %u.\n", map->name, def->map_flags);

	if (def->parts & MAP_DEF_PINNING)

		pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning);

	if (def->parts & MAP_DEF_NUMA_NODE)

		pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node);



	if (def->parts & MAP_DEF_INNER_MAP)

		pr_debug("map '%s': found inner map definition.\n", map->name);

}



static int bpf_object__init_user_btf_map(struct bpf_object *obj,

					 const struct btf_type *sec,

					 int var_idx, int sec_idx,

					 const Elf_Data *data, bool strict,

					 const char *pin_root_path)

{

	struct btf_map_def map_def = {}, inner_def = {};

	const struct btf_type *var, *def;

	const struct btf_var_secinfo *vi;

	const struct btf_var *var_extra;

	const char *map_name;

	struct bpf_map *map;

	int err;



	vi = btf_var_secinfos(sec) + var_idx;

	var = btf__type_by_id(obj->btf, vi->type);
@@ -2316,7 +2327,35 @@ static int bpf_object__init_user_btf_map(struct bpf_object *obj, 
	pr_debug("map '%s': at sec_idx %d, offset %zu.\n",

		 map_name, map->sec_idx, map->sec_offset);



	return parse_btf_map_def(obj, map, def, strict, false, pin_root_path);

	err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);

	if (err)

		return err;



	fill_map_from_def(map, &map_def);



	if (map_def.pinning == LIBBPF_PIN_BY_NAME) {

		err = build_map_pin_path(map, pin_root_path);

		if (err) {

			pr_warn("map '%s': couldn't build pin path.\n", map->name);

			return err;

		}

	}



	if (map_def.parts & MAP_DEF_INNER_MAP) {

		map->inner_map = calloc(1, sizeof(*map->inner_map));

		if (!map->inner_map)

			return -ENOMEM;

		map->inner_map->fd = -1;

		map->inner_map->sec_idx = sec_idx;

		map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1);

		if (!map->inner_map->name)

			return -ENOMEM;

		sprintf(map->inner_map->name, "%s.inner", map_name);



		fill_map_from_def(map->inner_map, &inner_def);

	}



	return 0;

}



static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
@@ -2618,7 +2657,7 @@ static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) 
{

	struct btf *kern_btf = obj->btf;

	bool btf_mandatory, sanitize;

	int err = 0;

	int i, err = 0;



	if (!obj->btf)

		return 0;
@@ -2632,6 +2671,38 @@ static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) 
		return 0;

	}



	/* Even though some subprogs are global/weak, user might prefer more

	 * permissive BPF verification process that BPF verifier performs for

	 * static functions, taking into account more context from the caller

	 * functions. In such case, they need to mark such subprogs with

	 * __attribute__((visibility("hidden"))) and libbpf will adjust

	 * corresponding FUNC BTF type to be marked as static and trigger more

	 * involved BPF verification process.

	 */

	for (i = 0; i < obj->nr_programs; i++) {

		struct bpf_program *prog = &obj->programs[i];

		struct btf_type *t;

		const char *name;

		int j, n;



		if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))

			continue;



		n = btf__get_nr_types(obj->btf);

		for (j = 1; j <= n; j++) {

			t = btf_type_by_id(obj->btf, j);

			if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL)

				continue;



			name = btf__str_by_offset(obj->btf, t->name_off);

			if (strcmp(name, prog->name) != 0)

				continue;



			t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0);

			break;

		}

	}



	sanitize = btf_needs_sanitization(obj);

	if (sanitize) {

		const void *raw_data;
@@ -2782,26 +2853,6 @@ static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn) 
	return data;

}



static int elf_sym_by_sec_off(const struct bpf_object *obj, size_t sec_idx,

			      size_t off, __u32 sym_type, GElf_Sym *sym)

{

	Elf_Data *symbols = obj->efile.symbols;

	size_t n = symbols->d_size / sizeof(GElf_Sym);

	int i;



	for (i = 0; i < n; i++) {

		if (!gelf_getsym(symbols, i, sym))

			continue;

		if (sym->st_shndx != sec_idx || sym->st_value != off)

			continue;

		if (GELF_ST_TYPE(sym->st_info) != sym_type)

			continue;

		return 0;

	}



	return -ENOENT;

}



static bool is_sec_name_dwarf(const char *name)

{

	/* approximation, but the actual list is too long */
@@ -3680,11 +3731,16 @@ bpf_object__collect_prog_relos(struct bpf_object *obj, GElf_Shdr *shdr, Elf_Data 
	int err, i, nrels;

	const char *sym_name;

	__u32 insn_idx;

	Elf_Scn *scn;

	Elf_Data *scn_data;

	GElf_Sym sym;

	GElf_Rel rel;



	scn = elf_sec_by_idx(obj, sec_idx);

	scn_data = elf_sec_data(obj, scn);



	relo_sec_name = elf_sec_str(obj, shdr->sh_name);

	sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));

	sec_name = elf_sec_name(obj, scn);

	if (!relo_sec_name || !sec_name)

		return -EINVAL;
@@ -3702,7 +3758,8 @@ bpf_object__collect_prog_relos(struct bpf_object *obj, GElf_Shdr *shdr, Elf_Data 
				relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i);

			return -LIBBPF_ERRNO__FORMAT;

		}

		if (rel.r_offset % BPF_INSN_SZ) {



		if (rel.r_offset % BPF_INSN_SZ || rel.r_offset >= scn_data->d_size) {

			pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n",

				relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i);

			return -LIBBPF_ERRNO__FORMAT;
@@ -3726,9 +3783,9 @@ bpf_object__collect_prog_relos(struct bpf_object *obj, GElf_Shdr *shdr, Elf_Data 


		prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);

		if (!prog) {

			pr_warn("sec '%s': relo #%d: program not found in section '%s' for insn #%u\n",

			pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n",

				relo_sec_name, i, sec_name, insn_idx);

			return -LIBBPF_ERRNO__RELOC;

			continue;

		}



		relos = libbpf_reallocarray(prog->reloc_desc,

tools/lib/bpf/libbpf_internal.h

+45 −0
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#pragma GCC poison reallocarray



#include "libbpf.h"

#include "btf.h"



#ifndef EM_BPF

#define EM_BPF 247
@@ -131,6 +132,50 @@ struct btf; 
struct btf_type;



struct btf_type *btf_type_by_id(struct btf *btf, __u32 type_id);

const char *btf_kind_str(const struct btf_type *t);

const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id);



static inline enum btf_func_linkage btf_func_linkage(const struct btf_type *t)

{

	return (enum btf_func_linkage)(int)btf_vlen(t);

}



static inline __u32 btf_type_info(int kind, int vlen, int kflag)

{

	return (kflag << 31) | (kind << 24) | vlen;

}



enum map_def_parts {

	MAP_DEF_MAP_TYPE	= 0x001,

	MAP_DEF_KEY_TYPE	= 0x002,

	MAP_DEF_KEY_SIZE	= 0x004,

	MAP_DEF_VALUE_TYPE	= 0x008,

	MAP_DEF_VALUE_SIZE	= 0x010,

	MAP_DEF_MAX_ENTRIES	= 0x020,

	MAP_DEF_MAP_FLAGS	= 0x040,

	MAP_DEF_NUMA_NODE	= 0x080,

	MAP_DEF_PINNING		= 0x100,

	MAP_DEF_INNER_MAP	= 0x200,



	MAP_DEF_ALL		= 0x3ff, /* combination of all above */

};



struct btf_map_def {

	enum map_def_parts parts;

	__u32 map_type;

	__u32 key_type_id;

	__u32 key_size;

	__u32 value_type_id;

	__u32 value_size;

	__u32 max_entries;

	__u32 map_flags;

	__u32 numa_node;

	__u32 pinning;

};



int parse_btf_map_def(const char *map_name, struct btf *btf,

		      const struct btf_type *def_t, bool strict,

		      struct btf_map_def *map_def, struct btf_map_def *inner_def);



void *libbpf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz,

		     size_t cur_cnt, size_t max_cnt, size_t add_cnt);