Commit aa7881fc authored by Hou Tao's avatar Hou Tao Committed by Alexei Starovoitov
Browse files

bpf: Factor out a common helper free_all() 



Factor out a common helper free_all() to free all normal elements or
per-cpu elements on a lock-less list.

Signed-off-by: default avatarHou Tao <houtao1@huawei.com>
Link: https://lore.kernel.org/r/20230606035310.4026145-2-houtao@huaweicloud.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 09564181

kernel/bpf/memalloc.c

+16 −15
Original line number Diff line number Diff line
@@ -211,9 +211,9 @@ static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node) 
	mem_cgroup_put(memcg);

}



static void free_one(struct bpf_mem_cache *c, void *obj)

static void free_one(void *obj, bool percpu)

{

	if (c->percpu_size) {

	if (percpu) {

		free_percpu(((void **)obj)[1]);

		kfree(obj);

		return;
@@ -222,14 +222,19 @@ static void free_one(struct bpf_mem_cache *c, void *obj) 
	kfree(obj);

}



static void __free_rcu(struct rcu_head *head)

static void free_all(struct llist_node *llnode, bool percpu)

{

	struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu);

	struct llist_node *llnode = llist_del_all(&c->waiting_for_gp);

	struct llist_node *pos, *t;



	llist_for_each_safe(pos, t, llnode)

		free_one(c, pos);

		free_one(pos, percpu);

}



static void __free_rcu(struct rcu_head *head)

{

	struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu);



	free_all(llist_del_all(&c->waiting_for_gp), !!c->percpu_size);

	atomic_set(&c->call_rcu_in_progress, 0);

}
@@ -432,7 +437,7 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu) 


static void drain_mem_cache(struct bpf_mem_cache *c)

{

	struct llist_node *llnode, *t;

	bool percpu = !!c->percpu_size;



	/* No progs are using this bpf_mem_cache, but htab_map_free() called

	 * bpf_mem_cache_free() for all remaining elements and they can be in
@@ -441,14 +446,10 @@ static void drain_mem_cache(struct bpf_mem_cache *c) 
	 * Except for waiting_for_gp list, there are no concurrent operations

	 * on these lists, so it is safe to use __llist_del_all().

	 */

	llist_for_each_safe(llnode, t, __llist_del_all(&c->free_by_rcu))

		free_one(c, llnode);

	llist_for_each_safe(llnode, t, llist_del_all(&c->waiting_for_gp))

		free_one(c, llnode);

	llist_for_each_safe(llnode, t, __llist_del_all(&c->free_llist))

		free_one(c, llnode);

	llist_for_each_safe(llnode, t, __llist_del_all(&c->free_llist_extra))

		free_one(c, llnode);

	free_all(__llist_del_all(&c->free_by_rcu), percpu);

	free_all(llist_del_all(&c->waiting_for_gp), percpu);

	free_all(__llist_del_all(&c->free_llist), percpu);

	free_all(__llist_del_all(&c->free_llist_extra), percpu);

}



static void free_mem_alloc_no_barrier(struct bpf_mem_alloc *ma)