Commit 727c3991 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'sched-urgent-2022-06-19' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull scheduler fix from Thomas Gleixner:
 "A single scheduler fix plugging a race between sched_setscheduler()
  and balance_push().

  sched_setscheduler() spliced the balance callbacks accross a lock
  break which makes it possible for an interleaving schedule() to
  observe an empty list"

* tag 'sched-urgent-2022-06-19' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  sched: Fix balance_push() vs __sched_setscheduler()
parents 4afb6515 04193d59

kernel/sched/core.c

+33 −3
Original line number Diff line number Diff line
@@ -4798,25 +4798,55 @@ static void do_balance_callbacks(struct rq *rq, struct callback_head *head) 


static void balance_push(struct rq *rq);



/*

 * balance_push_callback is a right abuse of the callback interface and plays

 * by significantly different rules.

 *

 * Where the normal balance_callback's purpose is to be ran in the same context

 * that queued it (only later, when it's safe to drop rq->lock again),

 * balance_push_callback is specifically targeted at __schedule().

 *

 * This abuse is tolerated because it places all the unlikely/odd cases behind

 * a single test, namely: rq->balance_callback == NULL.

 */

struct callback_head balance_push_callback = {

	.next = NULL,

	.func = (void (*)(struct callback_head *))balance_push,

};



static inline struct callback_head *splice_balance_callbacks(struct rq *rq)

static inline struct callback_head *

__splice_balance_callbacks(struct rq *rq, bool split)

{

	struct callback_head *head = rq->balance_callback;



	if (likely(!head))

		return NULL;



	lockdep_assert_rq_held(rq);

	if (head)

	/*

	 * Must not take balance_push_callback off the list when

	 * splice_balance_callbacks() and balance_callbacks() are not

	 * in the same rq->lock section.

	 *

	 * In that case it would be possible for __schedule() to interleave

	 * and observe the list empty.

	 */

	if (split && head == &balance_push_callback)

		head = NULL;

	else

		rq->balance_callback = NULL;



	return head;

}



static inline struct callback_head *splice_balance_callbacks(struct rq *rq)

{

	return __splice_balance_callbacks(rq, true);

}



static void __balance_callbacks(struct rq *rq)

{

	do_balance_callbacks(rq, splice_balance_callbacks(rq));

	do_balance_callbacks(rq, __splice_balance_callbacks(rq, false));

}



static inline void balance_callbacks(struct rq *rq, struct callback_head *head)

kernel/sched/sched.h

+5 −0
Original line number Diff line number Diff line
@@ -1693,6 +1693,11 @@ queue_balance_callback(struct rq *rq, 
{

	lockdep_assert_rq_held(rq);



	/*

	 * Don't (re)queue an already queued item; nor queue anything when

	 * balance_push() is active, see the comment with

	 * balance_push_callback.

	 */

	if (unlikely(head->next || rq->balance_callback == &balance_push_callback))

		return;