Commit ceea34dd authored by Guan Jing's avatar Guan Jing Committed by Xia Fukun
Browse files

sched: Implement the function of qos smt expeller 

hulk inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/I8O3MY


CVE: NA

--------------------------------

We implement the function of qos smt expeller by this following
two points：
a) when online tasks and offline tasks are running on the same
physical cpu, online tasks will send ipi to expel offline tasks
on the smt sibling cpus.
b) when online tasks are running, the smt sibling cpus will not
allow offline tasks to be selected.

Signed-off-by: default avatarGuan Jing <guanjing6@huawei.com>
Signed-off-by: default avatarXia Fukun <xiafukun@huawei.com>
parent d4614d07

include/linux/sched.h

+7 −0
Original line number Original line Diff line number Diff line
@@ -2021,9 +2021,16 @@ extern char *__get_task_comm(char *to, size_t len, struct task_struct *tsk); 
	__get_task_comm(buf, sizeof(buf), tsk);		\

	__get_task_comm(buf, sizeof(buf), tsk);		\

})

})





#ifdef CONFIG_QOS_SCHED_SMT_EXPELLER

void qos_smt_check_need_resched(void);

#endif



#ifdef CONFIG_SMP

#ifdef CONFIG_SMP

static __always_inline void scheduler_ipi(void)

static __always_inline void scheduler_ipi(void)

{

{

#ifdef CONFIG_QOS_SCHED_SMT_EXPELLER

	qos_smt_check_need_resched();

#endif

	/*

	/*

	 * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting

	 * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting

	 * TIF_NEED_RESCHED remotely (for the first time) will also send

	 * TIF_NEED_RESCHED remotely (for the first time) will also send

kernel/sched/fair.c

+187 −2
Original line number Original line Diff line number Diff line
@@ -62,6 +62,10 @@ 
#include <linux/resume_user_mode.h>

#include <linux/resume_user_mode.h>

#endif

#endif





#ifdef CONFIG_QOS_SCHED_SMT_EXPELLER

#include <trace/events/ipi.h>

#endif



#ifdef CONFIG_SCHED_STEAL

#ifdef CONFIG_SCHED_STEAL

#include "sparsemask.h"

#include "sparsemask.h"

#endif

#endif
@@ -151,6 +155,10 @@ static int hundred_thousand = 100000; 
static int unthrottle_qos_cfs_rqs(int cpu);

static int unthrottle_qos_cfs_rqs(int cpu);

#endif

#endif





#ifdef CONFIG_QOS_SCHED_SMT_EXPELLER

static DEFINE_PER_CPU(int, qos_smt_status);

#endif



#ifdef CONFIG_QOS_SCHED_PRIO_LB

#ifdef CONFIG_QOS_SCHED_PRIO_LB

unsigned int sysctl_sched_prio_load_balance_enabled;

unsigned int sysctl_sched_prio_load_balance_enabled;

#endif

#endif
@@ -8893,6 +8901,131 @@ static void qos_schedule_throttle(struct task_struct *p) 




#endif

#endif





#ifdef CONFIG_QOS_SCHED_SMT_EXPELLER

static bool qos_smt_check_siblings_status(int this_cpu)

{

	int cpu;



	if (!sched_smt_active())

		return false;



	for_each_cpu(cpu, cpu_smt_mask(this_cpu)) {

		if (cpu == this_cpu)

			continue;



		if (per_cpu(qos_smt_status, cpu) == QOS_LEVEL_ONLINE)

			return true;

	}



	return false;

}



static bool qos_smt_expelled(int this_cpu)

{

	/*

	 * The qos_smt_status of siblings cpu is online, and current cpu only has

	 * offline tasks enqueued, there is not suitable task,

	 * so pick_next_task_fair return null.

	 */

	if (qos_smt_check_siblings_status(this_cpu) && sched_idle_cpu(this_cpu))

		return true;



	return false;

}



static bool qos_smt_update_status(struct task_struct *p)

{

	int status = QOS_LEVEL_OFFLINE;



	if (p != NULL && task_group(p)->qos_level >= QOS_LEVEL_ONLINE)

		status = QOS_LEVEL_ONLINE;



	if (__this_cpu_read(qos_smt_status) == status)

		return false;



	__this_cpu_write(qos_smt_status, status);



	return true;

}



static void qos_smt_send_ipi(int this_cpu)

{

	int cpu;

	struct rq *rq = NULL;



	if (!sched_smt_active())

		return;



	for_each_cpu(cpu, cpu_smt_mask(this_cpu)) {

		if (cpu == this_cpu)

			continue;



		rq = cpu_rq(cpu);



		/*

		* There are two cases where current don't need to send scheduler_ipi:

		* a) The qos_smt_status of siblings cpu is online;

		* b) The cfs.h_nr_running of siblings cpu is 0.

		*/

		if (per_cpu(qos_smt_status, cpu) == QOS_LEVEL_ONLINE ||

		    rq->cfs.h_nr_running == 0)

			continue;



		smp_send_reschedule(cpu);

	}

}



static void qos_smt_expel(int this_cpu, struct task_struct *p)

{

	if (qos_smt_update_status(p))

		qos_smt_send_ipi(this_cpu);

}



static bool _qos_smt_check_need_resched(int this_cpu, struct rq *rq)

{

	int cpu;



	if (!sched_smt_active())

		return false;



	for_each_cpu(cpu, cpu_smt_mask(this_cpu)) {

		if (cpu == this_cpu)

			continue;



		/*

		* There are two cases rely on the set need_resched to drive away

		* offline task：

		* a) The qos_smt_status of siblings cpu is online, the task of curr cpu is offline;

		* b) The qos_smt_status of siblings cpu is offline, the task of curr cpu is idle,

		*    and current cpu only has SCHED_IDLE tasks enqueued.

		*/

		if (per_cpu(qos_smt_status, cpu) == QOS_LEVEL_ONLINE &&

		    task_group(current)->qos_level < QOS_LEVEL_ONLINE)

			return true;



		if (per_cpu(qos_smt_status, cpu) == QOS_LEVEL_OFFLINE &&

		    rq->curr == rq->idle && sched_idle_cpu(this_cpu))

			return true;

	}



	return false;

}



void qos_smt_check_need_resched(void)

{

	struct rq *rq = this_rq();

	int this_cpu = rq->cpu;



	if (test_tsk_need_resched(current))

		return;



	if (_qos_smt_check_need_resched(this_cpu, rq)) {

		set_tsk_need_resched(current);

		set_preempt_need_resched();

	}

}

#endif



#ifdef CONFIG_SMP

#ifdef CONFIG_SMP

static struct task_struct *pick_task_fair(struct rq *rq)

static struct task_struct *pick_task_fair(struct rq *rq)

{

{
@@ -8934,14 +9067,30 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf 
	struct task_struct *p;

	struct task_struct *p;

	int new_tasks;

	int new_tasks;

	unsigned long time;

	unsigned long time;

#ifdef CONFIG_QOS_SCHED_SMT_EXPELLER

	int this_cpu = rq->cpu;

#endif





again:

again:

#ifdef CONFIG_QOS_SCHED_SMT_EXPELLER

	if (qos_smt_expelled(this_cpu)) {

		__this_cpu_write(qos_smt_status, QOS_LEVEL_OFFLINE);

		return NULL;

	}

#endif



	if (!sched_fair_runnable(rq))

	if (!sched_fair_runnable(rq))

		goto idle;

		goto idle;





#ifdef CONFIG_FAIR_GROUP_SCHED

#ifdef CONFIG_FAIR_GROUP_SCHED

	if (!prev || prev->sched_class != &fair_sched_class)

	if (!prev || prev->sched_class != &fair_sched_class) {

#ifdef CONFIG_QOS_SCHED

		if (cfs_rq->idle_h_nr_running != 0 && rq->online)

			goto qos_simple;

		else

#endif

			goto simple;

			goto simple;

	}





	/*

	/*

	 * Because of the set_next_buddy() in dequeue_task_fair() it is rather

	 * Because of the set_next_buddy() in dequeue_task_fair() it is rather
@@ -9025,6 +9174,34 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf 
	}

	}





	goto done;

	goto done;



#ifdef CONFIG_QOS_SCHED

qos_simple:

	if (prev)

		put_prev_task(rq, prev);



	do {

		se = pick_next_entity(cfs_rq, NULL);

		if (check_qos_cfs_rq(group_cfs_rq(se))) {

			cfs_rq = &rq->cfs;

			if (!cfs_rq->nr_running)

				goto idle;

			continue;

		}



		cfs_rq = group_cfs_rq(se);

	} while (cfs_rq);



	p = task_of(se);



	while (se) {

		set_next_entity(cfs_rq_of(se), se);

		se = parent_entity(se);

	}



	goto done;

#endif



simple:

simple:

#endif

#endif

	if (prev)

	if (prev)
@@ -9062,6 +9239,10 @@ done: __maybe_unused; 
	qos_schedule_throttle(p);

	qos_schedule_throttle(p);

#endif

#endif





#ifdef CONFIG_QOS_SCHED_SMT_EXPELLER

	qos_smt_expel(this_cpu, p);

#endif



	return p;

	return p;





idle:

idle:
@@ -9110,6 +9291,10 @@ done: __maybe_unused; 
	 */

	 */

	update_idle_rq_clock_pelt(rq);

	update_idle_rq_clock_pelt(rq);





#ifdef CONFIG_QOS_SCHED_SMT_EXPELLER

	qos_smt_expel(this_cpu, NULL);

#endif



	return NULL;

	return NULL;

}

}

kernel/sched/sched.h

+5 −0
Original line number Original line Diff line number Diff line
@@ -1432,6 +1432,11 @@ do { \ 
} while (0)

} while (0)





#ifdef CONFIG_QOS_SCHED

#ifdef CONFIG_QOS_SCHED

enum task_qos_level {

	QOS_LEVEL_OFFLINE = -1,

	QOS_LEVEL_ONLINE = 0,

	QOS_LEVEL_MAX

};

void init_qos_hrtimer(int cpu);

void init_qos_hrtimer(int cpu);

#endif

#endif