Commit 4006a72b authored by Ricardo Neri's avatar Ricardo Neri Committed by Peter Zijlstra
Browse files

sched/fair: Consider SMT in ASYM_PACKING load balance 



When deciding to pull tasks in ASYM_PACKING, it is necessary not only to
check for the idle state of the destination CPU, dst_cpu, but also of
its SMT siblings.

If dst_cpu is idle but its SMT siblings are busy, performance suffers
if it pulls tasks from a medium priority CPU that does not have SMT
siblings.

Implement asym_smt_can_pull_tasks() to inspect the state of the SMT
siblings of both dst_cpu and the CPUs in the candidate busiest group.

Signed-off-by: default avatarRicardo Neri <ricardo.neri-calderon@linux.intel.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarJoel Fernandes (Google) <joel@joelfernandes.org>
Reviewed-by: default avatarLen Brown <len.brown@intel.com>
Reviewed-by: default avatarVincent Guittot <vincent.guittot@linaro.org>
Link: https://lkml.kernel.org/r/20210911011819.12184-7-ricardo.neri-calderon@linux.intel.com
parent aafc917a

kernel/sched/fair.c

+92 −0
Original line number Diff line number Diff line
@@ -8571,10 +8571,96 @@ group_type group_classify(unsigned int imbalance_pct, 
	return group_has_spare;

}



/**

 * asym_smt_can_pull_tasks - Check whether the load balancing CPU can pull tasks

 * @dst_cpu:	Destination CPU of the load balancing

 * @sds:	Load-balancing data with statistics of the local group

 * @sgs:	Load-balancing statistics of the candidate busiest group

 * @sg:		The candidate busiest group

 *

 * Check the state of the SMT siblings of both @sds::local and @sg and decide

 * if @dst_cpu can pull tasks.

 *

 * If @dst_cpu does not have SMT siblings, it can pull tasks if two or more of

 * the SMT siblings of @sg are busy. If only one CPU in @sg is busy, pull tasks

 * only if @dst_cpu has higher priority.

 *

 * If both @dst_cpu and @sg have SMT siblings, and @sg has exactly one more

 * busy CPU than @sds::local, let @dst_cpu pull tasks if it has higher priority.

 * Bigger imbalances in the number of busy CPUs will be dealt with in

 * update_sd_pick_busiest().

 *

 * If @sg does not have SMT siblings, only pull tasks if all of the SMT siblings

 * of @dst_cpu are idle and @sg has lower priority.

 */

static bool asym_smt_can_pull_tasks(int dst_cpu, struct sd_lb_stats *sds,

				    struct sg_lb_stats *sgs,

				    struct sched_group *sg)

{

#ifdef CONFIG_SCHED_SMT

	bool local_is_smt, sg_is_smt;

	int sg_busy_cpus;



	local_is_smt = sds->local->flags & SD_SHARE_CPUCAPACITY;

	sg_is_smt = sg->flags & SD_SHARE_CPUCAPACITY;



	sg_busy_cpus = sgs->group_weight - sgs->idle_cpus;



	if (!local_is_smt) {

		/*

		 * If we are here, @dst_cpu is idle and does not have SMT

		 * siblings. Pull tasks if candidate group has two or more

		 * busy CPUs.

		 */

		if (sg_busy_cpus >= 2) /* implies sg_is_smt */

			return true;



		/*

		 * @dst_cpu does not have SMT siblings. @sg may have SMT

		 * siblings and only one is busy. In such case, @dst_cpu

		 * can help if it has higher priority and is idle (i.e.,

		 * it has no running tasks).

		 */

		return sched_asym_prefer(dst_cpu, sg->asym_prefer_cpu);

	}



	/* @dst_cpu has SMT siblings. */



	if (sg_is_smt) {

		int local_busy_cpus = sds->local->group_weight -

				      sds->local_stat.idle_cpus;

		int busy_cpus_delta = sg_busy_cpus - local_busy_cpus;



		if (busy_cpus_delta == 1)

			return sched_asym_prefer(dst_cpu, sg->asym_prefer_cpu);



		return false;

	}



	/*

	 * @sg does not have SMT siblings. Ensure that @sds::local does not end

	 * up with more than one busy SMT sibling and only pull tasks if there

	 * are not busy CPUs (i.e., no CPU has running tasks).

	 */

	if (!sds->local_stat.sum_nr_running)

		return sched_asym_prefer(dst_cpu, sg->asym_prefer_cpu);



	return false;

#else

	/* Always return false so that callers deal with non-SMT cases. */

	return false;

#endif

}



static inline bool

sched_asym(struct lb_env *env, struct sd_lb_stats *sds,  struct sg_lb_stats *sgs,

	   struct sched_group *group)

{

	/* Only do SMT checks if either local or candidate have SMT siblings */

	if ((sds->local->flags & SD_SHARE_CPUCAPACITY) ||

	    (group->flags & SD_SHARE_CPUCAPACITY))

		return asym_smt_can_pull_tasks(env->dst_cpu, sds, sgs, group);



	return sched_asym_prefer(env->dst_cpu, group->asym_prefer_cpu);

}
@@ -9580,6 +9666,12 @@ static struct rq *find_busiest_queue(struct lb_env *env, 
		    nr_running == 1)

			continue;



		/* Make sure we only pull tasks from a CPU of lower priority */

		if ((env->sd->flags & SD_ASYM_PACKING) &&

		    sched_asym_prefer(i, env->dst_cpu) &&

		    nr_running == 1)

			continue;



		switch (env->migration_type) {

		case migrate_load:

			/*