Commit 857d315f authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

sched: Simplify ttwu() 



Use guards to reduce gotos and simplify control flow.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarValentin Schneider <vschneid@redhat.com>
Link: https://lore.kernel.org/r/20230801211812.101069260@infradead.org
parent 4eb054f9

kernel/sched/core.c

+109 −112
Original line number Diff line number Diff line
@@ -3733,14 +3733,14 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags) 
		struct sched_domain *sd;



		__schedstat_inc(p->stats.nr_wakeups_remote);

		rcu_read_lock();



		guard(rcu)();

		for_each_domain(rq->cpu, sd) {

			if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {

				__schedstat_inc(sd->ttwu_wake_remote);

				break;

			}

		}

		rcu_read_unlock();

	}



	if (wake_flags & WF_MIGRATED)
@@ -4199,10 +4199,9 @@ bool ttwu_state_match(struct task_struct *p, unsigned int state, int *success) 
static int

try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)

{

	unsigned long flags;

	guard(preempt)();

	int cpu, success = 0;



	preempt_disable();

	if (p == current) {

		/*

		 * We're waking current, this means 'p->on_rq' and 'task_cpu(p)
@@ -4229,10 +4228,10 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) 
	 * reordered with p->state check below. This pairs with smp_store_mb()

	 * in set_current_state() that the waiting thread does.

	 */

	raw_spin_lock_irqsave(&p->pi_lock, flags);

	scoped_guard (raw_spinlock_irqsave, &p->pi_lock) {

		smp_mb__after_spinlock();

		if (!ttwu_state_match(p, state, &success))

		goto unlock;

			break;



		trace_sched_waking(p);
@@ -4260,7 +4259,7 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) 
		 */

		smp_rmb();

		if (READ_ONCE(p->on_rq) && ttwu_runnable(p, wake_flags))

		goto unlock;

			break;



#ifdef CONFIG_SMP

		/*
@@ -4317,7 +4316,7 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) 
		 */

		if (smp_load_acquire(&p->on_cpu) &&

		    ttwu_queue_wakelist(p, task_cpu(p), wake_flags))

		goto unlock;

			break;



		/*

		 * If the owning (remote) CPU is still in the middle of schedule() with
@@ -4346,12 +4345,10 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) 
#endif /* CONFIG_SMP */



		ttwu_queue(p, cpu, wake_flags);

unlock:

	raw_spin_unlock_irqrestore(&p->pi_lock, flags);

	}

out:

	if (success)

		ttwu_stat(p, task_cpu(p), wake_flags);

	preempt_enable();



	return success;

}