Commit 21f56ffe authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

sched: Introduce sched_class::pick_task() 



Because sched_class::pick_next_task() also implies
sched_class::set_next_task() (and possibly put_prev_task() and
newidle_balance) it is not state invariant. This makes it unsuitable
for remote task selection.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
[Vineeth: folded fixes]
Signed-off-by: default avatarVineeth Remanan Pillai <viremana@linux.microsoft.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Tested-by: default avatarDon Hiatt <dhiatt@digitalocean.com>
Tested-by: default avatarHongyu Ning <hongyu.ning@linux.intel.com>
Tested-by: default avatarVincent Guittot <vincent.guittot@linaro.org>
Link: https://lkml.kernel.org/r/20210422123308.437092775@infradead.org
parent 875feb41

kernel/sched/deadline.c

+14 −2
Original line number Diff line number Diff line
@@ -1852,7 +1852,7 @@ static struct sched_dl_entity *pick_next_dl_entity(struct rq *rq, 
	return rb_entry(left, struct sched_dl_entity, rb_node);

}



static struct task_struct *pick_next_task_dl(struct rq *rq)

static struct task_struct *pick_task_dl(struct rq *rq)

{

	struct sched_dl_entity *dl_se;

	struct dl_rq *dl_rq = &rq->dl;
@@ -1864,7 +1864,18 @@ static struct task_struct *pick_next_task_dl(struct rq *rq) 
	dl_se = pick_next_dl_entity(rq, dl_rq);

	BUG_ON(!dl_se);

	p = dl_task_of(dl_se);



	return p;

}



static struct task_struct *pick_next_task_dl(struct rq *rq)

{

	struct task_struct *p;



	p = pick_task_dl(rq);

	if (p)

		set_next_task_dl(rq, p, true);



	return p;

}
@@ -2539,6 +2550,7 @@ DEFINE_SCHED_CLASS(dl) = { 


#ifdef CONFIG_SMP

	.balance		= balance_dl,

	.pick_task		= pick_task_dl,

	.select_task_rq		= select_task_rq_dl,

	.migrate_task_rq	= migrate_task_rq_dl,

	.set_cpus_allowed       = set_cpus_allowed_dl,

kernel/sched/fair.c

+37 −3
Original line number Diff line number Diff line
@@ -4419,6 +4419,8 @@ check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) 
static void

set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)

{

	clear_buddies(cfs_rq, se);



	/* 'current' is not kept within the tree. */

	if (se->on_rq) {

		/*
@@ -4478,7 +4480,7 @@ pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr) 
	 * Avoid running the skip buddy, if running something else can

	 * be done without getting too unfair.

	 */

	if (cfs_rq->skip == se) {

	if (cfs_rq->skip && cfs_rq->skip == se) {

		struct sched_entity *second;



		if (se == curr) {
@@ -4505,8 +4507,6 @@ pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr) 
		se = cfs_rq->last;

	}



	clear_buddies(cfs_rq, se);



	return se;

}
@@ -7095,6 +7095,39 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ 
		set_last_buddy(se);

}



#ifdef CONFIG_SMP

static struct task_struct *pick_task_fair(struct rq *rq)

{

	struct sched_entity *se;

	struct cfs_rq *cfs_rq;



again:

	cfs_rq = &rq->cfs;

	if (!cfs_rq->nr_running)

		return NULL;



	do {

		struct sched_entity *curr = cfs_rq->curr;



		/* When we pick for a remote RQ, we'll not have done put_prev_entity() */

		if (curr) {

			if (curr->on_rq)

				update_curr(cfs_rq);

			else

				curr = NULL;



			if (unlikely(check_cfs_rq_runtime(cfs_rq)))

				goto again;

		}



		se = pick_next_entity(cfs_rq, curr);

		cfs_rq = group_cfs_rq(se);

	} while (cfs_rq);



	return task_of(se);

}

#endif



struct task_struct *

pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)

{
@@ -11298,6 +11331,7 @@ DEFINE_SCHED_CLASS(fair) = { 


#ifdef CONFIG_SMP

	.balance		= balance_fair,

	.pick_task		= pick_task_fair,

	.select_task_rq		= select_task_rq_fair,

	.migrate_task_rq	= migrate_task_rq_fair,

kernel/sched/idle.c

+8 −0
Original line number Diff line number Diff line
@@ -439,6 +439,13 @@ static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool fir 
	schedstat_inc(rq->sched_goidle);

}



#ifdef CONFIG_SMP

static struct task_struct *pick_task_idle(struct rq *rq)

{

	return rq->idle;

}

#endif



struct task_struct *pick_next_task_idle(struct rq *rq)

{

	struct task_struct *next = rq->idle;
@@ -506,6 +513,7 @@ DEFINE_SCHED_CLASS(idle) = { 


#ifdef CONFIG_SMP

	.balance		= balance_idle,

	.pick_task		= pick_task_idle,

	.select_task_rq		= select_task_rq_idle,

	.set_cpus_allowed	= set_cpus_allowed_common,

#endif

kernel/sched/rt.c

+13 −2
Original line number Diff line number Diff line
@@ -1626,7 +1626,7 @@ static struct task_struct *_pick_next_task_rt(struct rq *rq) 
	return rt_task_of(rt_se);

}



static struct task_struct *pick_next_task_rt(struct rq *rq)

static struct task_struct *pick_task_rt(struct rq *rq)

{

	struct task_struct *p;
@@ -1634,7 +1634,17 @@ static struct task_struct *pick_next_task_rt(struct rq *rq) 
		return NULL;



	p = _pick_next_task_rt(rq);



	return p;

}



static struct task_struct *pick_next_task_rt(struct rq *rq)

{

	struct task_struct *p = pick_task_rt(rq);



	if (p)

		set_next_task_rt(rq, p, true);



	return p;

}
@@ -2483,6 +2493,7 @@ DEFINE_SCHED_CLASS(rt) = { 


#ifdef CONFIG_SMP

	.balance		= balance_rt,

	.pick_task		= pick_task_rt,

	.select_task_rq		= select_task_rq_rt,

	.set_cpus_allowed       = set_cpus_allowed_common,

	.rq_online              = rq_online_rt,

kernel/sched/sched.h

+3 −0
Original line number Diff line number Diff line
@@ -1953,6 +1953,9 @@ struct sched_class { 
#ifdef CONFIG_SMP

	int (*balance)(struct rq *rq, struct task_struct *prev, struct rq_flags *rf);

	int  (*select_task_rq)(struct task_struct *p, int task_cpu, int flags);



	struct task_struct * (*pick_task)(struct rq *rq);



	void (*migrate_task_rq)(struct task_struct *p, int new_cpu);



	void (*task_woken)(struct rq *this_rq, struct task_struct *task);