Commit d119e916 authored by Hou Tao's avatar Hou Tao Committed by Pu Lehui
Browse files

bpf: Optimize the free of inner map 

stable inclusion
from stable-v6.6.35
commit 2ad2f2edb944baf2735b23c7008b3dbe5b8da56c
category: bugfix
bugzilla: https://gitee.com/openeuler/kernel/issues/IAD6H2

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=2ad2f2edb944baf2735b23c7008b3dbe5b8da56c



--------------------------------

[ Upstream commit af66bfd3c8538ed21cf72af18426fc4a408665cf ]

When removing the inner map from the outer map, the inner map will be
freed after one RCU grace period and one RCU tasks trace grace
period, so it is certain that the bpf program, which may access the
inner map, has exited before the inner map is freed.

However there is no need to wait for one RCU tasks trace grace period if
the outer map is only accessed by non-sleepable program. So adding
sleepable_refcnt in bpf_map and increasing sleepable_refcnt when adding
the outer map into env->used_maps for sleepable program. Although the
max number of bpf program is INT_MAX - 1, the number of bpf programs
which are being loaded may be greater than INT_MAX, so using atomic64_t
instead of atomic_t for sleepable_refcnt. When removing the inner map
from the outer map, using sleepable_refcnt to decide whether or not a
RCU tasks trace grace period is needed before freeing the inner map.

Signed-off-by: default avatarHou Tao <houtao1@huawei.com>
Link: https://lore.kernel.org/r/20231204140425.1480317-6-houtao@huaweicloud.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Stable-dep-of: 2884dc7d08d9 ("bpf: Fix a potential use-after-free in bpf_link_free()")
Signed-off-by: default avatarSasha Levin <sashal@kernel.org>

Conflicts:
	include/linux/bpf.h
[Bpf related structures have changed in commit 6b6e3a2e ("kabi: reserve
space for bpf related structures"), causing conflicts in this patch merge]
Signed-off-by: default avatarTengda Wu <wutengda2@huawei.com>
Signed-off-by: default avatarPu Lehui <pulehui@huawei.com>
parent 597b1933

include/linux/bpf.h

+2 −0
Original line number Diff line number Diff line
@@ -302,6 +302,8 @@ struct bpf_map { 
	bool bypass_spec_v1;

	bool frozen; /* write-once; write-protected by freeze_mutex */

	bool free_after_mult_rcu_gp;

	bool free_after_rcu_gp;

	atomic64_t sleepable_refcnt;

	s64 __percpu *elem_count;



	KABI_RESERVE(1)

kernel/bpf/core.c

+4 −0
Original line number Diff line number Diff line
@@ -2677,12 +2677,16 @@ void __bpf_free_used_maps(struct bpf_prog_aux *aux, 
			  struct bpf_map **used_maps, u32 len)

{

	struct bpf_map *map;

	bool sleepable;

	u32 i;



	sleepable = aux->sleepable;

	for (i = 0; i < len; i++) {

		map = used_maps[i];

		if (map->ops->map_poke_untrack)

			map->ops->map_poke_untrack(map, aux);

		if (sleepable)

			atomic64_dec(&map->sleepable_refcnt);

		bpf_map_put(map);

	}

}

kernel/bpf/map_in_map.c

+9 −5
Original line number Diff line number Diff line
@@ -131,12 +131,16 @@ void bpf_map_fd_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) 
{

	struct bpf_map *inner_map = ptr;



	/* The inner map may still be used by both non-sleepable and sleepable

	 * bpf program, so free it after one RCU grace period and one tasks

	 * trace RCU grace period.

	/* Defer the freeing of inner map according to the sleepable attribute

	 * of bpf program which owns the outer map, so unnecessary waiting for

	 * RCU tasks trace grace period can be avoided.

	 */

	if (need_defer)

	if (need_defer) {

		if (atomic64_read(&map->sleepable_refcnt))

			WRITE_ONCE(inner_map->free_after_mult_rcu_gp, true);

		else

			WRITE_ONCE(inner_map->free_after_rcu_gp, true);

	}

	bpf_map_put(inner_map);

}

kernel/bpf/syscall.c

+8 −0
Original line number Diff line number Diff line
@@ -754,8 +754,11 @@ void bpf_map_put(struct bpf_map *map) 
		/* bpf_map_free_id() must be called first */

		bpf_map_free_id(map);



		WARN_ON_ONCE(atomic64_read(&map->sleepable_refcnt));

		if (READ_ONCE(map->free_after_mult_rcu_gp))

			call_rcu_tasks_trace(&map->rcu, bpf_map_free_mult_rcu_gp);

		else if (READ_ONCE(map->free_after_rcu_gp))

			call_rcu(&map->rcu, bpf_map_free_rcu_gp);

		else

			bpf_map_free_in_work(map);

	}
@@ -5400,6 +5403,11 @@ static int bpf_prog_bind_map(union bpf_attr *attr) 
		goto out_unlock;

	}



	/* The bpf program will not access the bpf map, but for the sake of

	 * simplicity, increase sleepable_refcnt for sleepable program as well.

	 */

	if (prog->aux->sleepable)

		atomic64_inc(&map->sleepable_refcnt);

	memcpy(used_maps_new, used_maps_old,

	       sizeof(used_maps_old[0]) * prog->aux->used_map_cnt);

	used_maps_new[prog->aux->used_map_cnt] = map;

kernel/bpf/verifier.c

+3 −1
Original line number Diff line number Diff line
@@ -17741,10 +17741,12 @@ static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) 
				return -E2BIG;

			}









			if (env->prog->aux->sleepable)









				atomic64_inc(&map->sleepable_refcnt);









			/* hold the map. If the program is rejected by verifier,









			 * the map will be released by release_maps() or it









			 * will be used by the valid program until it's unloaded









			 * and all maps are released in free_used_maps()









			 * and all maps are released in bpf_free_used_maps()









			 */









			bpf_map_inc(map);