!10757 CVE-2024-41045 (64bbdff4) · Commits · EulixOS / Software / Kernel

kernel/bpf/helpers.c

+89 −41

Original line number	Diff line number	Diff line
		@@ -1083,7 +1083,10 @@ struct bpf_async_cb {
		struct bpf_prog *prog;
		void __rcu *callback_fn;
		void *value;
		union {
		struct rcu_head rcu;
		struct work_struct delete_work;
		};
		u64 flags;
		};

		@@ -1167,6 +1170,21 @@ static enum hrtimer_restart bpf_timer_cb(struct hrtimer *hrtimer)
		return HRTIMER_NORESTART;
		}

		static void bpf_timer_delete_work(struct work_struct *work)
		{
		struct bpf_hrtimer *t = container_of(work, struct bpf_hrtimer, cb.delete_work);

		/* Cancel the timer and wait for callback to complete if it was running.
		* If hrtimer_cancel() can be safely called it's safe to call
		* kfree_rcu(t) right after for both preallocated and non-preallocated
		* maps. The async->cb = NULL was already done and no code path can see
		* address 't' anymore. Timer if armed for existing bpf_hrtimer before
		* bpf_timer_cancel_and_free will have been cancelled.
		*/
		hrtimer_cancel(&t->timer);
		kfree_rcu(t, cb.rcu);
		}

		static int __bpf_async_init(struct bpf_async_kern async, struct bpf_map map, u64 flags,
		enum bpf_async_type type)
		{
		@@ -1206,6 +1224,7 @@ static int __bpf_async_init(struct bpf_async_kern async, struct bpf_map map, u
		t = (struct bpf_hrtimer *)cb;

		atomic_set(&t->cancelling, 0);
		INIT_WORK(&t->cb.delete_work, bpf_timer_delete_work);
		hrtimer_init(&t->timer, clockid, HRTIMER_MODE_REL_SOFT);
		t->timer.function = bpf_timer_cb;
		cb->value = (void *)async - map->record->timer_off;
		@@ -1263,22 +1282,23 @@ static const struct bpf_func_proto bpf_timer_init_proto = {
		.arg3_type = ARG_ANYTHING,
		};

		BPF_CALL_3(bpf_timer_set_callback, struct bpf_async_kern , timer, void , callback_fn,
		struct bpf_prog_aux *, aux)
		static int __bpf_async_set_callback(struct bpf_async_kern async, void callback_fn,
		struct bpf_prog_aux *aux, unsigned int flags,
		enum bpf_async_type type)
		{
		struct bpf_prog prev, prog = aux->prog;
		struct bpf_hrtimer *t;
		struct bpf_async_cb *cb;
		int ret = 0;

		if (in_nmi())
		return -EOPNOTSUPP;
		__bpf_spin_lock_irqsave(&timer->lock);
		t = timer->timer;
		if (!t) {
		__bpf_spin_lock_irqsave(&async->lock);
		cb = async->cb;
		if (!cb) {
		ret = -EINVAL;
		goto out;
		}
		if (!atomic64_read(&t->cb.map->usercnt)) {
		if (!atomic64_read(&cb->map->usercnt)) {
		/* maps with timers must be either held by user space
		* or pinned in bpffs. Otherwise timer might still be
		* running even when bpf prog is detached and user space
		@@ -1287,7 +1307,7 @@ BPF_CALL_3(bpf_timer_set_callback, struct bpf_async_kern , timer, void , callb
		ret = -EPERM;
		goto out;
		}
		prev = t->cb.prog;
		prev = cb->prog;
		if (prev != prog) {
		/* Bump prog refcnt once. Every bpf_timer_set_callback()
		* can pick different callback_fn-s within the same prog.
		@@ -1300,14 +1320,20 @@ BPF_CALL_3(bpf_timer_set_callback, struct bpf_async_kern , timer, void , callb
		if (prev)
		/* Drop prev prog refcnt when swapping with new prog */
		bpf_prog_put(prev);
		t->cb.prog = prog;
		cb->prog = prog;
		}
		rcu_assign_pointer(t->cb.callback_fn, callback_fn);
		rcu_assign_pointer(cb->callback_fn, callback_fn);
		out:
		__bpf_spin_unlock_irqrestore(&timer->lock);
		__bpf_spin_unlock_irqrestore(&async->lock);
		return ret;
		}

		BPF_CALL_3(bpf_timer_set_callback, struct bpf_async_kern , timer, void , callback_fn,
		struct bpf_prog_aux *, aux)
		{
		return __bpf_async_set_callback(timer, callback_fn, aux, 0, BPF_ASYNC_TYPE_TIMER);
		}

		static const struct bpf_func_proto bpf_timer_set_callback_proto = {
		.func = bpf_timer_set_callback,
		.gpl_only = true,
		@@ -1434,51 +1460,73 @@ static const struct bpf_func_proto bpf_timer_cancel_proto = {
		.arg1_type = ARG_PTR_TO_TIMER,
		};

		/* This function is called by map_delete/update_elem for individual element and
		* by ops->map_release_uref when the user space reference to a map reaches zero.
		*/
		void bpf_timer_cancel_and_free(void *val)
		static struct bpf_async_cb __bpf_async_cancel_and_free(struct bpf_async_kern async)
		{
		struct bpf_async_kern *timer = val;
		struct bpf_hrtimer *t;
		struct bpf_async_cb *cb;

		/* Performance optimization: read timer->timer without lock first. */
		if (!READ_ONCE(timer->timer))
		return;
		/* Performance optimization: read async->cb without lock first. */
		if (!READ_ONCE(async->cb))
		return NULL;

		__bpf_spin_lock_irqsave(&timer->lock);
		__bpf_spin_lock_irqsave(&async->lock);
		/* re-read it under lock */
		t = timer->timer;
		if (!t)
		cb = async->cb;
		if (!cb)
		goto out;
		drop_prog_refcnt(&t->cb);
		drop_prog_refcnt(cb);
		/* The subsequent bpf_timer_start/cancel() helpers won't be able to use
		* this timer, since it won't be initialized.
		*/
		WRITE_ONCE(timer->timer, NULL);
		WRITE_ONCE(async->cb, NULL);
		out:
		__bpf_spin_unlock_irqrestore(&timer->lock);
		__bpf_spin_unlock_irqrestore(&async->lock);
		return cb;
		}

		/* This function is called by map_delete/update_elem for individual element and
		* by ops->map_release_uref when the user space reference to a map reaches zero.
		*/
		void bpf_timer_cancel_and_free(void *val)
		{
		struct bpf_hrtimer *t;

		t = (struct bpf_hrtimer *)__bpf_async_cancel_and_free(val);

		if (!t)
		return;
		/* Cancel the timer and wait for callback to complete if it was running.
		* If hrtimer_cancel() can be safely called it's safe to call kfree(t)
		* right after for both preallocated and non-preallocated maps.
		* The timer->timer = NULL was already done and no code path can
		* see address 't' anymore.
		*
		* Check that bpf_map_delete/update_elem() wasn't called from timer
		* callback_fn. In such case don't call hrtimer_cancel() (since it will
		* deadlock) and don't call hrtimer_try_to_cancel() (since it will just
		* return -1). Though callback_fn is still running on this cpu it's
		/* We check that bpf_map_delete/update_elem() was called from timer
		* callback_fn. In such case we don't call hrtimer_cancel() (since it
		* will deadlock) and don't call hrtimer_try_to_cancel() (since it will
		* just return -1). Though callback_fn is still running on this cpu it's
		* safe to do kfree(t) because bpf_timer_cb() read everything it needed
		* from 't'. The bpf subprog callback_fn won't be able to access 't',
		* since timer->timer = NULL was already done. The timer will be
		* since async->cb = NULL was already done. The timer will be
		* effectively cancelled because bpf_timer_cb() will return
		* HRTIMER_NORESTART.
		*
		* However, it is possible the timer callback_fn calling us armed the
		* timer _before_ calling us, such that failing to cancel it here will
		* cause it to possibly use struct hrtimer after freeing bpf_hrtimer.
		* Therefore, we _need_ to cancel any outstanding timers before we do
		* kfree_rcu, even though no more timers can be armed.
		*
		* Moreover, we need to schedule work even if timer does not belong to
		* the calling callback_fn, as on two different CPUs, we can end up in a
		* situation where both sides run in parallel, try to cancel one
		* another, and we end up waiting on both sides in hrtimer_cancel
		* without making forward progress, since timer1 depends on time2
		* callback to finish, and vice versa.
		*
		* CPU 1 (timer1_cb) CPU 2 (timer2_cb)
		* bpf_timer_cancel_and_free(timer2) bpf_timer_cancel_and_free(timer1)
		*
		* To avoid these issues, punt to workqueue context when we are in a
		* timer callback.
		*/
		if (this_cpu_read(hrtimer_running) != t)
		hrtimer_cancel(&t->timer);
		kfree_rcu(t, cb.rcu);
		if (this_cpu_read(hrtimer_running))
		queue_work(system_unbound_wq, &t->cb.delete_work);
		else
		bpf_timer_delete_work(&t->cb.delete_work);
		}

		BPF_CALL_2(bpf_kptr_xchg, void , map_value, void , ptr)