Commit c6f88654 authored by Vincent Guittot's avatar Vincent Guittot Committed by Ingo Molnar
Browse files

sched/fair: Trigger the update of blocked load on newly idle cpu 



Instead of waking up a random and already idle CPU, we can take advantage
of this_cpu being about to enter idle to run the ILB and update the
blocked load.

Signed-off-by: default avatarVincent Guittot <vincent.guittot@linaro.org>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Reviewed-by: default avatarValentin Schneider <valentin.schneider@arm.com>
Link: https://lkml.kernel.org/r/20210224133007.28644-7-vincent.guittot@linaro.org
parent 6553fc18

kernel/sched/core.c

+1 −1
Original line number Diff line number Diff line
@@ -737,7 +737,7 @@ static void nohz_csd_func(void *info) 
	/*

	 * Release the rq::nohz_csd.

	 */

	flags = atomic_fetch_andnot(NOHZ_KICK_MASK, nohz_flags(cpu));

	flags = atomic_fetch_andnot(NOHZ_KICK_MASK | NOHZ_NEWILB_KICK, nohz_flags(cpu));

	WARN_ON(!(flags & NOHZ_KICK_MASK));



	rq->idle_balance = idle_cpu(cpu);

kernel/sched/fair.c

+21 −3
Original line number Diff line number Diff line
@@ -10453,6 +10453,24 @@ static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 
	return true;

}



/*

 * Check if we need to run the ILB for updating blocked load before entering

 * idle state.

 */

void nohz_run_idle_balance(int cpu)

{

	unsigned int flags;



	flags = atomic_fetch_andnot(NOHZ_NEWILB_KICK, nohz_flags(cpu));



	/*

	 * Update the blocked load only if no SCHED_SOFTIRQ is about to happen

	 * (ie NOHZ_STATS_KICK set) and will do the same.

	 */

	if ((flags == NOHZ_NEWILB_KICK) && !need_resched())

		_nohz_idle_balance(cpu_rq(cpu), NOHZ_STATS_KICK, CPU_IDLE);

}



static void nohz_newidle_balance(struct rq *this_rq)

{

	int this_cpu = this_rq->cpu;
@@ -10474,10 +10492,10 @@ static void nohz_newidle_balance(struct rq *this_rq) 
		return;



	/*

	 * Blocked load of idle CPUs need to be updated.

	 * Kick an ILB to update statistics.

	 * Set the need to trigger ILB in order to update blocked load

	 * before entering idle state.

	 */

	kick_ilb(NOHZ_STATS_KICK);

	atomic_or(NOHZ_NEWILB_KICK, nohz_flags(this_cpu));

}



#else /* !CONFIG_NO_HZ_COMMON */

kernel/sched/idle.c

+6 −0
Original line number Diff line number Diff line
@@ -261,6 +261,12 @@ static void cpuidle_idle_call(void) 
static void do_idle(void)

{

	int cpu = smp_processor_id();



	/*

	 * Check if we need to update blocked load

	 */

	nohz_run_idle_balance(cpu);



	/*

	 * If the arch has a polling bit, we maintain an invariant:

	 *

kernel/sched/sched.h

+7 −0
Original line number Diff line number Diff line
@@ -2385,9 +2385,11 @@ extern void cfs_bandwidth_usage_dec(void); 
#ifdef CONFIG_NO_HZ_COMMON

#define NOHZ_BALANCE_KICK_BIT	0

#define NOHZ_STATS_KICK_BIT	1

#define NOHZ_NEWILB_KICK_BIT	2



#define NOHZ_BALANCE_KICK	BIT(NOHZ_BALANCE_KICK_BIT)

#define NOHZ_STATS_KICK		BIT(NOHZ_STATS_KICK_BIT)

#define NOHZ_NEWILB_KICK	BIT(NOHZ_NEWILB_KICK_BIT)



#define NOHZ_KICK_MASK	(NOHZ_BALANCE_KICK | NOHZ_STATS_KICK)
@@ -2398,6 +2400,11 @@ extern void nohz_balance_exit_idle(struct rq *rq); 
static inline void nohz_balance_exit_idle(struct rq *rq) { }

#endif



#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)

extern void nohz_run_idle_balance(int cpu);

#else

static inline void nohz_run_idle_balance(int cpu) { }

#endif



#ifdef CONFIG_SMP

static inline