bpf: Rename few bpf_mem_alloc fields.

	int low_watermark, high_watermark, batch;

	int percpu_size;

	struct rcu_head rcu;

	struct llist_head free_by_rcu;

	struct llist_head waiting_for_gp;

	atomic_t call_rcu_in_progress;

	/* list of objects to be freed after RCU tasks trace GP */

	struct llist_head free_by_rcu_ttrace;

	struct llist_head waiting_for_gp_ttrace;

	struct rcu_head rcu_ttrace;

	atomic_t call_rcu_ttrace_in_progress;

};

struct bpf_mem_caches {

	old_memcg = set_active_memcg(memcg);

	for (i = 0; i < cnt; i++) {

		/*

		 * free_by_rcu is only manipulated by irq work refill_work().

		 * free_by_rcu_ttrace is only manipulated by irq work refill_work().

		 * IRQ works on the same CPU are called sequentially, so it is

		 * safe to use __llist_del_first() here. If alloc_bulk() is

		 * invoked by the initial prefill, there will be no running

		 * refill_work(), so __llist_del_first() is fine as well.

		 *

		 * In most cases, objects on free_by_rcu are from the same CPU.

		 * In most cases, objects on free_by_rcu_ttrace are from the same CPU.

		 * If some objects come from other CPUs, it doesn't incur any

		 * harm because NUMA_NO_NODE means the preference for current

		 * numa node and it is not a guarantee.

		 */

		obj = __llist_del_first(&c->free_by_rcu);

		obj = __llist_del_first(&c->free_by_rcu_ttrace);

		if (!obj) {

			/* Allocate, but don't deplete atomic reserves that typical

			 * GFP_ATOMIC would do. irq_work runs on this cpu and kmalloc

static void __free_rcu(struct rcu_head *head)

{

	struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu);

	struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu_ttrace);

	free_all(llist_del_all(&c->waiting_for_gp), !!c->percpu_size);

	atomic_set(&c->call_rcu_in_progress, 0);

	free_all(llist_del_all(&c->waiting_for_gp_ttrace), !!c->percpu_size);

	atomic_set(&c->call_rcu_ttrace_in_progress, 0);

}

static void __free_rcu_tasks_trace(struct rcu_head *head)

	struct llist_node *llnode = obj;

	/* bpf_mem_cache is a per-cpu object. Freeing happens in irq_work.

	 * Nothing races to add to free_by_rcu list.

	 * Nothing races to add to free_by_rcu_ttrace list.

	 */

	__llist_add(llnode, &c->free_by_rcu);

	__llist_add(llnode, &c->free_by_rcu_ttrace);

}

static void do_call_rcu(struct bpf_mem_cache *c)

static void do_call_rcu_ttrace(struct bpf_mem_cache *c)

{

	struct llist_node *llnode, *t;

	if (atomic_xchg(&c->call_rcu_in_progress, 1))

	if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 1))

		return;

	WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp));

	llist_for_each_safe(llnode, t, __llist_del_all(&c->free_by_rcu))

		/* There is no concurrent __llist_add(waiting_for_gp) access.

	WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp_ttrace));

	llist_for_each_safe(llnode, t, __llist_del_all(&c->free_by_rcu_ttrace))

		/* There is no concurrent __llist_add(waiting_for_gp_ttrace) access.

		 * It doesn't race with llist_del_all either.

		 * But there could be two concurrent llist_del_all(waiting_for_gp):

		 * But there could be two concurrent llist_del_all(waiting_for_gp_ttrace):

		 * from __free_rcu() and from drain_mem_cache().

		 */

		__llist_add(llnode, &c->waiting_for_gp);

		__llist_add(llnode, &c->waiting_for_gp_ttrace);

	/* Use call_rcu_tasks_trace() to wait for sleepable progs to finish.

	 * If RCU Tasks Trace grace period implies RCU grace period, free

	 * these elements directly, else use call_rcu() to wait for normal

	 * progs to finish and finally do free_one() on each element.

	 */

	call_rcu_tasks_trace(&c->rcu, __free_rcu_tasks_trace);

	call_rcu_tasks_trace(&c->rcu_ttrace, __free_rcu_tasks_trace);

}

static void free_bulk(struct bpf_mem_cache *c)

	/* and drain free_llist_extra */

	llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist_extra))

		enque_to_free(c, llnode);

	do_call_rcu(c);

	do_call_rcu_ttrace(c);

}

static void bpf_mem_refill(struct irq_work *work)

	/* No progs are using this bpf_mem_cache, but htab_map_free() called

	 * bpf_mem_cache_free() for all remaining elements and they can be in

	 * free_by_rcu or in waiting_for_gp lists, so drain those lists now.

	 * free_by_rcu_ttrace or in waiting_for_gp_ttrace lists, so drain those lists now.

	 *

	 * Except for waiting_for_gp list, there are no concurrent operations

	 * Except for waiting_for_gp_ttrace list, there are no concurrent operations

	 * on these lists, so it is safe to use __llist_del_all().

	 */

	free_all(__llist_del_all(&c->free_by_rcu), percpu);

	free_all(llist_del_all(&c->waiting_for_gp), percpu);

	free_all(__llist_del_all(&c->free_by_rcu_ttrace), percpu);

	free_all(llist_del_all(&c->waiting_for_gp_ttrace), percpu);

	free_all(__llist_del_all(&c->free_llist), percpu);

	free_all(__llist_del_all(&c->free_llist_extra), percpu);

}

static void free_mem_alloc(struct bpf_mem_alloc *ma)

{

	/* waiting_for_gp lists was drained, but __free_rcu might

	/* waiting_for_gp_ttrace lists was drained, but __free_rcu might

	 * still execute. Wait for it now before we freeing percpu caches.

	 *

	 * rcu_barrier_tasks_trace() doesn't imply synchronize_rcu_tasks_trace(),

			 */

			irq_work_sync(&c->refill_work);

			drain_mem_cache(c);

			rcu_in_progress += atomic_read(&c->call_rcu_in_progress);

			rcu_in_progress += atomic_read(&c->call_rcu_ttrace_in_progress);

		}

		/* objcg is the same across cpus */

		if (c->objcg)

				c = &cc->cache[i];

				irq_work_sync(&c->refill_work);

				drain_mem_cache(c);

				rcu_in_progress += atomic_read(&c->call_rcu_in_progress);

				rcu_in_progress += atomic_read(&c->call_rcu_ttrace_in_progress);

			}

		}

		if (c->objcg)