Commit 2a3bb3c0 authored by tanghui's avatar tanghui Committed by Zhong Jinghua
Browse files

sched: Adjust wakeup cpu range according CPU util dynamicly 

hulk inclusion
category: feature
bugzilla: 186575, https://gitee.com/openeuler/kernel/issues/I526XC



--------------------------------

Compare taskgroup 'util_avg' in perferred cpu with capacity preferred cpu,
dynamicly adjust cpu range for task wakeup process.

Signed-off-by: default avatartanghui <tanghui20@huawei.com>
Signed-off-by: default avatarZheng Zucheng <zhengzucheng@huawei.com>
Reviewed-by: default avatarZhang Qiao <zhangqiao22@huawei.com>
parent ebeb84ad

include/linux/sched.h

+2 −1
Original line number Diff line number Diff line
@@ -1424,10 +1424,11 @@ struct task_struct { 
	KABI_USE(7, void *pf_io_worker)

#if defined(CONFIG_QOS_SCHED_DYNAMIC_AFFINITY) && !defined(__GENKSYMS__)

	KABI_USE(8, cpumask_t *prefer_cpus)

	KABI_USE(9, const cpumask_t *select_cpus)

#else

	KABI_RESERVE(8)

#endif

	KABI_RESERVE(9)

#endif

	KABI_RESERVE(10)

	KABI_RESERVE(11)

	KABI_RESERVE(12)

include/linux/sched/sysctl.h

+4 −0
Original line number Diff line number Diff line
@@ -31,6 +31,10 @@ extern unsigned int sysctl_sched_min_granularity; 
extern unsigned int sysctl_sched_wakeup_granularity;

extern unsigned int sysctl_sched_child_runs_first;



#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY

extern int sysctl_sched_util_low_pct;

#endif



enum sched_tunable_scaling {

	SCHED_TUNABLESCALING_NONE,

	SCHED_TUNABLESCALING_LOG,

kernel/sched/fair.c

+131 −0
Original line number Diff line number Diff line
@@ -6092,7 +6092,11 @@ find_idlest_group_cpu(struct sched_group *group, struct task_struct *p, int this 
		return cpumask_first(sched_group_span(group));



	/* Traverse only the allowed CPUs */

#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY

	for_each_cpu_and(i, sched_group_span(group), p->select_cpus) {

#else

	for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) {

#endif

		struct rq *rq = cpu_rq(i);



		if (!sched_core_cookie_match(rq, p))
@@ -6139,7 +6143,11 @@ static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p 
{

	int new_cpu = cpu;



#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY

	if (!cpumask_intersects(sched_domain_span(sd), p->select_cpus))

#else

	if (!cpumask_intersects(sched_domain_span(sd), p->cpus_ptr))

#endif

		return prev_cpu;



	/*
@@ -6266,7 +6274,11 @@ static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpu 
		if (!available_idle_cpu(cpu)) {

			idle = false;

			if (*idle_cpu == -1) {

#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY

				if (sched_idle_cpu(cpu) && cpumask_test_cpu(cpu, p->select_cpus)) {

#else

				if (sched_idle_cpu(cpu) && cpumask_test_cpu(cpu, p->cpus_ptr)) {

#endif

					*idle_cpu = cpu;

					break;

				}
@@ -6322,7 +6334,11 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t 
	if (!this_sd)

		return -1;



#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY

	cpumask_and(cpus, sched_domain_span(sd), p->select_cpus);

#else

	cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr);

#endif



	if (sched_feat(SIS_PROP) && !smt) {

		u64 avg_cost, avg_idle, span_avg;
@@ -6460,6 +6476,9 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) 
	lockdep_assert_irqs_disabled();



	if ((available_idle_cpu(target) || sched_idle_cpu(target)) &&

#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY

	    cpumask_test_cpu(target, p->select_cpus) &&

#endif

	    asym_fits_capacity(task_util, target)) {

		SET_STAT(found_idle_cpu_easy);

		return target;
@@ -6470,6 +6489,9 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) 
	 */

	if (prev != target && cpus_share_cache(prev, target) &&

	    (available_idle_cpu(prev) || sched_idle_cpu(prev)) &&

#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY

	    cpumask_test_cpu(prev, p->select_cpus) &&

#endif

	    asym_fits_capacity(task_util, prev)) {

		SET_STAT(found_idle_cpu_easy);

		return prev;
@@ -6498,7 +6520,11 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) 
	    recent_used_cpu != target &&

	    cpus_share_cache(recent_used_cpu, target) &&

	    (available_idle_cpu(recent_used_cpu) || sched_idle_cpu(recent_used_cpu)) &&

#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY

	    cpumask_test_cpu(p->recent_used_cpu, p->select_cpus) &&

#else

	    cpumask_test_cpu(p->recent_used_cpu, p->cpus_ptr) &&

#endif

	    asym_fits_capacity(task_util, recent_used_cpu)) {

		/*

		 * Replace recent_used_cpu with prev as it is a potential
@@ -6921,6 +6947,82 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) 
	return -1;

}



#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY

/*

 * Low utilization threshold for CPU

 *

 * (default: 85%), units: percentage of CPU utilization)

 */

int sysctl_sched_util_low_pct = 85;



static inline bool prefer_cpus_valid(struct task_struct *p)

{

	return p->prefer_cpus &&

	       !cpumask_empty(p->prefer_cpus) &&

	       !cpumask_equal(p->prefer_cpus, p->cpus_ptr) &&

	       cpumask_subset(p->prefer_cpus, p->cpus_ptr);

}



/*

 * set_task_select_cpus: select the cpu range for task

 * @p: the task whose available cpu range will to set

 * @idlest_cpu: the cpu which is the idlest in prefer cpus

 *

 * If sum of 'util_avg' among 'preferred_cpus' lower than the percentage

 * 'sysctl_sched_util_low_pct' of 'preferred_cpus' capacity, select

 * 'preferred_cpus' range for task, otherwise select 'preferred_cpus' for task.

 *

 * The available cpu range set to p->select_cpus. Idlest cpu in preferred cpus

 * set to @idlest_cpu, which is set to wakeup cpu when fast path wakeup cpu

 * without p->select_cpus.

 */

static void set_task_select_cpus(struct task_struct *p, int *idlest_cpu,

				 int sd_flag)

{

	unsigned long util_avg_sum = 0;

	unsigned long tg_capacity = 0;

	long min_util = INT_MIN;

	struct task_group *tg;

	long spare;

	int cpu;



	p->select_cpus = p->cpus_ptr;

	if (!prefer_cpus_valid(p))

		return;



	rcu_read_lock();

	tg = task_group(p);

	for_each_cpu(cpu, p->prefer_cpus) {

		if (unlikely(!tg->se[cpu]))

			continue;



		if (idlest_cpu && available_idle_cpu(cpu)) {

			*idlest_cpu = cpu;

		} else if (idlest_cpu) {

			spare = (long)(capacity_of(cpu) - tg->se[cpu]->avg.util_avg);

			if (spare > min_util) {

				min_util = spare;

				*idlest_cpu = cpu;

			}

		}



		if (available_idle_cpu(cpu)) {

			rcu_read_unlock();

			p->select_cpus = p->prefer_cpus;

			return;

		}



		util_avg_sum += tg->se[cpu]->avg.util_avg;

		tg_capacity += capacity_of(cpu);

	}

	rcu_read_unlock();



	if (tg_capacity > cpumask_weight(p->prefer_cpus) &&

	    util_avg_sum * 100 <= tg_capacity * sysctl_sched_util_low_pct)

		p->select_cpus = p->prefer_cpus;

}

#endif



/*

 * select_task_rq_fair: Select target runqueue for the waking task in domains

 * that have the 'sd_flag' flag set. In practice, this is SD_BALANCE_WAKE,
@@ -6946,6 +7048,9 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f 
	cpumask_t *cpus;

	int ret;

#endif

#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY

	int idlest_cpu = 0;

#endif



	time = schedstat_start_time();
@@ -6953,6 +7058,11 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f 
	 * required for stable ->cpus_allowed

	 */

	lockdep_assert_held(&p->pi_lock);



#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY

	set_task_select_cpus(p, &idlest_cpu, sd_flag);

#endif



	if (sd_flag & SD_BALANCE_WAKE) {

		record_wakee(p);
@@ -6963,7 +7073,11 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f 
			new_cpu = prev_cpu;

		}



#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY

		want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, p->select_cpus);

#else

		want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, p->cpus_ptr);

#endif

	}



	rcu_read_lock();
@@ -7000,7 +7114,13 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f 
		 */

		if (want_affine && (tmp->flags & SD_WAKE_AFFINE) &&

		    cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) {

#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY

			new_cpu = cpu;

			if (cpu != prev_cpu &&

			    cpumask_test_cpu(prev_cpu, p->select_cpus))

#else

			if (cpu != prev_cpu)

#endif

				new_cpu = wake_affine(tmp, p, cpu, prev_cpu, sync);



			sd = NULL; /* Prefer wake_affine over balance flags */
@@ -7038,6 +7158,12 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f 
#endif



	rcu_read_unlock();



#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY

	if (!cpumask_test_cpu(new_cpu, p->select_cpus))

		new_cpu = idlest_cpu;

#endif



	schedstat_end_time(cpu_rq(cpu), time);



	return new_cpu;
@@ -9712,8 +9838,13 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu) 
		int local_group;



		/* Skip over this group if it has no CPUs allowed */

#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY

		if (!cpumask_intersects(sched_group_span(group),

					p->select_cpus))

#else

		if (!cpumask_intersects(sched_group_span(group),

					p->cpus_ptr))

#endif

			continue;



		/* Skip over this group if no cookie matched */

kernel/sysctl.c

+11 −0
Original line number Diff line number Diff line
@@ -2728,6 +2728,17 @@ static struct ctl_table kern_table[] = { 
		.extra1		= SYSCTL_ZERO,

		.extra2		= SYSCTL_ONE,

	},

#endif

#ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY

	{

		.procname       = "sched_util_low_pct",

		.data           = &sysctl_sched_util_low_pct,

		.maxlen         = sizeof(sysctl_sched_util_low_pct),

		.mode           = 0644,

		.proc_handler   = proc_dointvec_minmax,

		.extra1         = SYSCTL_ZERO,

		.extra2		= &one_hundred,

	},

#endif

	{ }

};