Unverified Commit 6e92c8c8 authored by openeuler-ci-bot's avatar openeuler-ci-bot Committed by Gitee
Browse files

!3774 [OLK-6.6] sched/fair: Scan cluster before scanning LLC in wake-up path 

Merge Pull Request from: @liujie-248683921 
 
This is the follow-up work to support cluster scheduler. Previously
we have added cluster level in the scheduler for both ARM64[1] and
X86[2] to support load balance between clusters to bring more memory
bandwidth and decrease cache contention. This patchset, on the other
hand, takes care of wake-up path by giving CPUs within the same cluster
a try before scanning the whole LLC to benefit those tasks communicating
with each other.

[1] 778c558f ("sched: Add cluster scheduler level in core and related Kconfig for ARM64")
[2] 66558b73 ("sched: Add cluster scheduler level for x86")

Barry Song (2):
sched: Add cpus_share_resources API
sched/fair: Scan cluster before scanning LLC in wake-up path

Yicong Yang (1):
sched/fair: Use candidate prev/recent_used CPU if scanning failed for
cluster wakeup

include/linux/sched/sd_flags.h | 7 ++++
include/linux/sched/topology.h | 8 ++++-
kernel/sched/core.c | 12 +++++++
kernel/sched/fair.c | 58 +++++++++++++++++++++++++++++++---
kernel/sched/sched.h | 2 ++
kernel/sched/topology.c | 25 +++++++++++++++
6 files changed, 106 insertions(+), 6 deletions(-)

--
2.24.0 
 
Link:https://gitee.com/openeuler/kernel/pulls/3774

 

Reviewed-by: default avatarZucheng Zheng <zhengzucheng@huawei.com>
Signed-off-by: default avatarZheng Zengkai <zhengzengkai@huawei.com>
parents 52c21146 408e8cee

include/linux/sched/sd_flags.h

+7 −0
Original line number Diff line number Diff line
@@ -109,6 +109,13 @@ SD_FLAG(SD_ASYM_CPUCAPACITY_FULL, SDF_SHARED_PARENT | SDF_NEEDS_GROUPS) 
 */

SD_FLAG(SD_SHARE_CPUCAPACITY, SDF_SHARED_CHILD | SDF_NEEDS_GROUPS)



/*

 * Domain members share CPU cluster (LLC tags or L2 cache)

 *

 * NEEDS_GROUPS: Clusters are shared between groups.

 */

SD_FLAG(SD_CLUSTER, SDF_NEEDS_GROUPS)



/*

 * Domain members share CPU package resources (i.e. caches)

 *

include/linux/sched/topology.h

+7 −1
Original line number Diff line number Diff line
@@ -45,7 +45,7 @@ static inline int cpu_smt_flags(void) 
#ifdef CONFIG_SCHED_CLUSTER

static inline int cpu_cluster_flags(void)

{

	return SD_SHARE_PKG_RESOURCES;

	return SD_CLUSTER | SD_SHARE_PKG_RESOURCES;

}

#endif
@@ -182,6 +182,7 @@ cpumask_var_t *alloc_sched_domains(unsigned int ndoms); 
void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms);



bool cpus_share_cache(int this_cpu, int that_cpu);

bool cpus_share_resources(int this_cpu, int that_cpu);



typedef const struct cpumask *(*sched_domain_mask_f)(int cpu);

typedef int (*sched_domain_flags_f)(void);
@@ -235,6 +236,11 @@ static inline bool cpus_share_cache(int this_cpu, int that_cpu) 
	return true;

}



static inline bool cpus_share_resources(int this_cpu, int that_cpu)

{

	return true;

}



#endif	/* !CONFIG_SMP */



#if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)

kernel/sched/core.c

+12 −0
Original line number Diff line number Diff line
@@ -3970,6 +3970,18 @@ bool cpus_share_cache(int this_cpu, int that_cpu) 
	return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);

}



/*

 * Whether CPUs are share cache resources, which means LLC on non-cluster

 * machines and LLC tag or L2 on machines with clusters.

 */

bool cpus_share_resources(int this_cpu, int that_cpu)

{

	if (this_cpu == that_cpu)

		return true;



	return per_cpu(sd_share_id, this_cpu) == per_cpu(sd_share_id, that_cpu);

}



static inline bool ttwu_queue_cond(struct task_struct *p, int cpu)

{

	/*

kernel/sched/fair.c

+56 −10
Original line number Diff line number Diff line
@@ -8117,6 +8117,30 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool 
		}

	}



	if (static_branch_unlikely(&sched_cluster_active)) {

		struct sched_group *sg = sd->groups;



		if (sg->flags & SD_CLUSTER) {

			for_each_cpu_wrap(cpu, sched_group_span(sg), target + 1) {

				if (!cpumask_test_cpu(cpu, cpus))

					continue;



				if (has_idle_core) {

					i = select_idle_core(p, cpu, cpus, &idle_cpu);

					if ((unsigned int)i < nr_cpumask_bits)

						return i;

				} else {

					if (--nr <= 0)

						return -1;

					idle_cpu = __select_idle_cpu(cpu, p);

					if ((unsigned int)idle_cpu < nr_cpumask_bits)

						return idle_cpu;

				}

			}

			cpumask_andnot(cpus, cpus, sched_group_span(sg));

		}

	}



	for_each_cpu_wrap(cpu, cpus, target + 1) {

		if (has_idle_core) {

			i = select_idle_core(p, cpu, cpus, &idle_cpu);
@@ -8124,7 +8148,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool 
				return i;



		} else {

			if (!--nr)

			if (--nr <= 0)

				return -1;

			idle_cpu = __select_idle_cpu(cpu, p);

			if ((unsigned int)idle_cpu < nr_cpumask_bits)
@@ -8240,7 +8264,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) 
	bool has_idle_core = false;

	struct sched_domain *sd;

	unsigned long task_util, util_min, util_max;

	int i, recent_used_cpu;

	int i, recent_used_cpu, prev_aff = -1;



	/*

	 * On asymmetric system, update task utilization because we will check
@@ -8276,10 +8300,15 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) 
	    cpumask_test_cpu(prev, p->select_cpus) &&

#endif

	    asym_fits_cpu(task_util, util_min, util_max, prev)) {

		if (!static_branch_unlikely(&sched_cluster_active) ||

		    cpus_share_resources(prev, target)) {

			SET_STAT(found_idle_cpu_easy);

			return prev;

		}



		prev_aff = prev;

	}



	/*

	 * Allow a per-cpu kthread to stack with the wakee if the

	 * kworker thread and the tasks previous CPUs are the same.
@@ -8310,6 +8339,8 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) 
	    cpumask_test_cpu(recent_used_cpu, p->cpus_ptr) &&

#endif

	    asym_fits_cpu(task_util, util_min, util_max, recent_used_cpu)) {

		if (!static_branch_unlikely(&sched_cluster_active) ||

		    cpus_share_resources(recent_used_cpu, target)) {

			/*

			 * Replace recent_used_cpu with prev as it is a potential

			 * candidate for the next wake:
@@ -8317,6 +8348,9 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) 
			SET_STAT(found_idle_cpu_easy);

			return recent_used_cpu;

		}

	} else {

		recent_used_cpu = -1;

	}



	/*

	 * For asymmetric CPU capacity systems, our domain of interest is
@@ -8362,6 +8396,18 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) 
	}



	SET_STAT(nofound_idle_cpu);



	/*

	 * For cluster machines which have lower sharing cache like L2 or

	 * LLC Tag, we tend to find an idle CPU in the target's cluster

	 * first. But prev_cpu or recent_used_cpu may also be a good candidate,

	 * use them if possible when no idle CPU found in select_idle_cpu().

	 */

	if ((unsigned int)prev_aff < nr_cpumask_bits)

		return prev_aff;

	if ((unsigned int)recent_used_cpu < nr_cpumask_bits)

		return recent_used_cpu;



	return target;

}

kernel/sched/sched.h

+2 −0
Original line number Diff line number Diff line
@@ -1980,11 +1980,13 @@ static inline struct sched_domain *lowest_flag_domain(int cpu, int flag) 
DECLARE_PER_CPU(struct sched_domain __rcu *, sd_llc);

DECLARE_PER_CPU(int, sd_llc_size);

DECLARE_PER_CPU(int, sd_llc_id);

DECLARE_PER_CPU(int, sd_share_id);

DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared);

DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa);

DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing);

DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);

extern struct static_key_false sched_asym_cpucapacity;

extern struct static_key_false sched_cluster_active;



static __always_inline bool sched_asym_cpucap_active(void)

{