Unverified Commit b4a3ad1c authored by openeuler-ci-bot's avatar openeuler-ci-bot Committed by Gitee
Browse files

!5877 optimize eevdf scheduler 

Merge Pull Request from: @ci-robot 
 
PR sync from: Zhang Qiao <zhangqiao22@huawei.com>
https://mailweb.openeuler.org/hyperkitty/list/kernel@openeuler.org/message/5UCKPSHK4EETBJCU6KLKHNUC62O5ECMY/ 
Abel Wu (2):
  sched/eevdf: Sort the rbtree by virtual deadline
  sched/eevdf: O(1) fastpath for task selection

K Prateek Nayak (1):
  sched/eevdf: Skip eligibility check for current entity during wakeup
    preemption


-- 
2.18.0.huawei.25
 
https://gitee.com/openeuler/kernel/issues/I9EHKI 
 
Link:https://gitee.com/openeuler/kernel/pulls/5877

 

Reviewed-by: default avatarZucheng Zheng <zhengzucheng@huawei.com>
Signed-off-by: default avatarZheng Zengkai <zhengzengkai@huawei.com>
parents e14b59d5 0d46b757

include/linux/sched.h

+1 −1
Original line number Diff line number Diff line
@@ -572,7 +572,7 @@ struct sched_entity { 
	struct load_weight		load;

	struct rb_node			run_node;

	u64				deadline;

	u64				min_deadline;

	u64				min_vruntime;



	struct list_head		group_node;

	unsigned int			on_rq;

kernel/sched/debug.c

+8 −3
Original line number Diff line number Diff line
@@ -628,8 +628,8 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) 


void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)

{

	s64 left_vruntime = -1, min_vruntime, right_vruntime = -1, spread;

	struct sched_entity *last, *first;

	s64 left_vruntime = -1, min_vruntime, right_vruntime = -1, left_deadline = -1, spread;

	struct sched_entity *last, *first, *root;

	struct rq *rq = cpu_rq(cpu);

	unsigned long flags;
@@ -644,15 +644,20 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) 
			SPLIT_NS(cfs_rq->exec_clock));



	raw_spin_rq_lock_irqsave(rq, flags);

	root = __pick_root_entity(cfs_rq);

	if (root)

		left_vruntime = root->min_vruntime;

	first = __pick_first_entity(cfs_rq);

	if (first)

		left_vruntime = first->vruntime;

		left_deadline = first->deadline;

	last = __pick_last_entity(cfs_rq);

	if (last)

		right_vruntime = last->vruntime;

	min_vruntime = cfs_rq->min_vruntime;

	raw_spin_rq_unlock_irqrestore(rq, flags);



	SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", "left_deadline",

			SPLIT_NS(left_deadline));

	SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", "left_vruntime",

			SPLIT_NS(left_vruntime));

	SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", "min_vruntime",

kernel/sched/fair.c

+96 −106
Original line number Diff line number Diff line
@@ -708,7 +708,11 @@ static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime) 
static inline bool entity_before(const struct sched_entity *a,

				 const struct sched_entity *b)

{

	return (s64)(a->vruntime - b->vruntime) < 0;

	/*

	 * Tiebreak on vruntime seems unnecessary since it can

	 * hardly happen.

	 */

	return (s64)(a->deadline - b->deadline) < 0;

}



static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se)
@@ -877,7 +881,7 @@ static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) 
 * Note: using 'avg_vruntime() > se->vruntime' is inacurate due

 *       to the loss in precision caused by the division.

 */

int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se)

static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime)

{

	struct sched_entity *curr = cfs_rq->curr;

	s64 avg = cfs_rq->avg_vruntime;
@@ -890,7 +894,12 @@ int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se) 
		load += weight;

	}



	return avg >= entity_key(cfs_rq, se) * load;

	return avg >= (s64)(vruntime - cfs_rq->min_vruntime) * load;

}



int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se)

{

	return vruntime_eligible(cfs_rq, se->vruntime);

}



static u64 __update_min_vruntime(struct cfs_rq *cfs_rq, u64 vruntime)
@@ -909,9 +918,8 @@ static u64 __update_min_vruntime(struct cfs_rq *cfs_rq, u64 vruntime) 


static void update_min_vruntime(struct cfs_rq *cfs_rq)

{

	struct sched_entity *se = __pick_first_entity(cfs_rq);

	struct sched_entity *se = __pick_root_entity(cfs_rq);

	struct sched_entity *curr = cfs_rq->curr;



	u64 vruntime = cfs_rq->min_vruntime;



	if (curr) {
@@ -923,9 +931,9 @@ static void update_min_vruntime(struct cfs_rq *cfs_rq) 


	if (se) {

		if (!curr)

			vruntime = se->vruntime;

			vruntime = se->min_vruntime;

		else

			vruntime = min_vruntime(vruntime, se->vruntime);

			vruntime = min_vruntime(vruntime, se->min_vruntime);

	}



	/* ensure we never gain time by being placed backwards. */
@@ -938,34 +946,34 @@ static inline bool __entity_less(struct rb_node *a, const struct rb_node *b) 
	return entity_before(__node_2_se(a), __node_2_se(b));

}



#define deadline_gt(field, lse, rse) ({ (s64)((lse)->field - (rse)->field) > 0; })

#define vruntime_gt(field, lse, rse) ({ (s64)((lse)->field - (rse)->field) > 0; })



static inline void __update_min_deadline(struct sched_entity *se, struct rb_node *node)

static inline void __min_vruntime_update(struct sched_entity *se, struct rb_node *node)

{

	if (node) {

		struct sched_entity *rse = __node_2_se(node);

		if (deadline_gt(min_deadline, se, rse))

			se->min_deadline = rse->min_deadline;

		if (vruntime_gt(min_vruntime, se, rse))

			se->min_vruntime = rse->min_vruntime;

	}

}



/*

 * se->min_deadline = min(se->deadline, left->min_deadline, right->min_deadline)

 * se->min_vruntime = min(se->vruntime, {left,right}->min_vruntime)

 */

static inline bool min_deadline_update(struct sched_entity *se, bool exit)

static inline bool min_vruntime_update(struct sched_entity *se, bool exit)

{

	u64 old_min_deadline = se->min_deadline;

	u64 old_min_vruntime = se->min_vruntime;

	struct rb_node *node = &se->run_node;



	se->min_deadline = se->deadline;

	__update_min_deadline(se, node->rb_right);

	__update_min_deadline(se, node->rb_left);

	se->min_vruntime = se->vruntime;

	__min_vruntime_update(se, node->rb_right);

	__min_vruntime_update(se, node->rb_left);



	return se->min_deadline == old_min_deadline;

	return se->min_vruntime == old_min_vruntime;

}



RB_DECLARE_CALLBACKS(static, min_deadline_cb, struct sched_entity,

		     run_node, min_deadline, min_deadline_update);

RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity,

		     run_node, min_vruntime, min_vruntime_update);



/*

 * Enqueue an entity into the rb-tree:
@@ -973,18 +981,28 @@ RB_DECLARE_CALLBACKS(static, min_deadline_cb, struct sched_entity, 
static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)

{

	avg_vruntime_add(cfs_rq, se);

	se->min_deadline = se->deadline;

	se->min_vruntime = se->vruntime;

	rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline,

				__entity_less, &min_deadline_cb);

				__entity_less, &min_vruntime_cb);

}



static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)

{

	rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline,

				  &min_deadline_cb);

				  &min_vruntime_cb);

	avg_vruntime_sub(cfs_rq, se);

}



struct sched_entity *__pick_root_entity(struct cfs_rq *cfs_rq)

{

	struct rb_node *root = cfs_rq->tasks_timeline.rb_root.rb_node;



	if (!root)

		return NULL;



	return __node_2_se(root);

}



struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq)

{

	struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline);
@@ -1007,23 +1025,45 @@ struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) 
 *     with the earliest virtual deadline.

 *

 * We can do this in O(log n) time due to an augmented RB-tree. The

 * tree keeps the entries sorted on service, but also functions as a

 * heap based on the deadline by keeping:

 * tree keeps the entries sorted on deadline, but also functions as a

 * heap based on the vruntime by keeping:

 *

 *  se->min_deadline = min(se->deadline, se->{left,right}->min_deadline)

 *  se->min_vruntime = min(se->vruntime, se->{left,right}->min_vruntime)

 *

 * Which allows an EDF like search on (sub)trees.

 * Which allows tree pruning through eligibility.

 */

static struct sched_entity *__pick_eevdf(struct cfs_rq *cfs_rq)

static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq, bool wakeup_preempt)

{

	struct rb_node *node = cfs_rq->tasks_timeline.rb_root.rb_node;

	struct sched_entity *se = __pick_first_entity(cfs_rq);

	struct sched_entity *curr = cfs_rq->curr;

	struct sched_entity *best = NULL;

	struct sched_entity *best_left = NULL;



	if (curr && (!curr->on_rq || !entity_eligible(cfs_rq, curr)))

	/*

	 * We can safely skip eligibility check if there is only one entity

	 * in this cfs_rq, saving some cycles.

	 */

	if (cfs_rq->nr_running == 1)

		return curr && curr->on_rq ? curr : se;



	if (curr && !curr->on_rq)

		curr = NULL;



	/*

	 * When an entity with positive lag wakes up, it pushes the

	 * avg_vruntime of the runqueue backwards. This may causes the

	 * current entity to be ineligible soon into its run leading to

	 * wakeup preemption.

	 *

	 * To prevent such aggressive preemption of the current running

	 * entity during task wakeups, skip the eligibility check if the

	 * slice promised to the entity since its selection has not yet

	 * elapsed.

	 */

	if (curr &&

	    !(sched_feat(RUN_TO_PARITY_WAKEUP) && wakeup_preempt && curr->vlag == curr->deadline) &&

	    !entity_eligible(cfs_rq, curr))

		curr = NULL;

	best = curr;



	/*

	 * Once selected, run a task until it either becomes non-eligible or
@@ -1032,95 +1072,45 @@ static struct sched_entity *__pick_eevdf(struct cfs_rq *cfs_rq) 
	if (sched_feat(RUN_TO_PARITY) && curr && curr->vlag == curr->deadline)

		return curr;



	/* Pick the leftmost entity if it's eligible */

	if (se && entity_eligible(cfs_rq, se)) {

		best = se;

		goto found;

	}



	/* Heap search for the EEVD entity */

	while (node) {

		struct sched_entity *se = __node_2_se(node);

		struct rb_node *left = node->rb_left;



		/*

		 * If this entity is not eligible, try the left subtree.

		 * Eligible entities in left subtree are always better

		 * choices, since they have earlier deadlines.

		 */

		if (!entity_eligible(cfs_rq, se)) {

			node = node->rb_left;

		if (left && vruntime_eligible(cfs_rq,

					__node_2_se(left)->min_vruntime)) {

			node = left;

			continue;

		}



		/*

		 * Now we heap search eligible trees for the best (min_)deadline

		 */

		if (!best || deadline_gt(deadline, best, se))

			best = se;

		se = __node_2_se(node);



		/*

		 * Every se in a left branch is eligible, keep track of the

		 * branch with the best min_deadline

		 * The left subtree either is empty or has no eligible

		 * entity, so check the current node since it is the one

		 * with earliest deadline that might be eligible.

		 */

		if (node->rb_left) {

			struct sched_entity *left = __node_2_se(node->rb_left);



			if (!best_left || deadline_gt(min_deadline, best_left, left))

				best_left = left;



			/*

			 * min_deadline is in the left branch. rb_left and all

			 * descendants are eligible, so immediately switch to the second

			 * loop.

			 */

			if (left->min_deadline == se->min_deadline)

		if (entity_eligible(cfs_rq, se)) {

			best = se;

			break;

		}



		/* min_deadline is at this node, no need to look right */

		if (se->deadline == se->min_deadline)

			break;



		/* else min_deadline is in the right branch. */

		node = node->rb_right;

	}

found:

	if (!best || (curr && entity_before(curr, best)))

		best = curr;



	/*

	 * We ran into an eligible node which is itself the best.

	 * (Or nr_running == 0 and both are NULL)

	 */

	if (!best_left || (s64)(best_left->min_deadline - best->deadline) > 0)

	return best;



	/*

	 * Now best_left and all of its children are eligible, and we are just

	 * looking for deadline == min_deadline

	 */

	node = &best_left->run_node;

	while (node) {

		struct sched_entity *se = __node_2_se(node);



		/* min_deadline is the current node */

		if (se->deadline == se->min_deadline)

			return se;



		/* min_deadline is in the left branch */

		if (node->rb_left &&

		    __node_2_se(node->rb_left)->min_deadline == se->min_deadline) {

			node = node->rb_left;

			continue;

		}



		/* else min_deadline is in the right branch */

		node = node->rb_right;

	}

	return NULL;

}



static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq)

{

	struct sched_entity *se = __pick_eevdf(cfs_rq);



	if (!se) {

		struct sched_entity *left = __pick_first_entity(cfs_rq);

		if (left) {

			pr_err("EEVDF scheduling fail, picking leftmost\n");

			return left;

		}

	}



	return se;

}



#ifdef CONFIG_SCHED_DEBUG
@@ -5602,7 +5592,7 @@ pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr) 
	    cfs_rq->next && entity_eligible(cfs_rq, cfs_rq->next))

		return cfs_rq->next;



	return pick_eevdf(cfs_rq);

	return pick_eevdf(cfs_rq, false);

}



static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq);
@@ -9271,7 +9261,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ 
	/*

	 * XXX pick_eevdf(cfs_rq) != se ?

	 */

	if (pick_eevdf(cfs_rq) == pse)

	if (pick_eevdf(cfs_rq, true) == pse)

		goto preempt;



	return;

kernel/sched/features.h

+1 −0
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ 
SCHED_FEAT(PLACE_LAG, true)

SCHED_FEAT(PLACE_DEADLINE_INITIAL, true)

SCHED_FEAT(RUN_TO_PARITY, true)

SCHED_FEAT(RUN_TO_PARITY_WAKEUP, true)



/*

 * Prefer to schedule the task we woke last (assuming it failed

kernel/sched/sched.h

+1 −0
Original line number Diff line number Diff line
@@ -2995,6 +2995,7 @@ DEFINE_LOCK_GUARD_2(double_rq_lock, struct rq, 
		    double_rq_lock(_T->lock, _T->lock2),

		    double_rq_unlock(_T->lock, _T->lock2))



extern struct sched_entity *__pick_root_entity(struct cfs_rq *cfs_rq);

extern struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq);

extern struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq);