Commit 64b7a3fa authored by Chris Wilson's avatar Chris Wilson
Browse files

drm/i915/gt: Use virtual_engine during execlists_dequeue 



Rather than going back and forth between the rb_node entry and the
virtual_engine type, store the ve local and reuse it. As the
container_of conversion from rb_node to virtual_engine requires a
variable offset, performing that conversion just once shaves off a bit
of code.

v2: Keep a single virtual engine lookup, for typical use.

Signed-off-by: default avatarChris Wilson <chris@chris-wilson.co.uk>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Reviewed-by: default avatarMatthew Auld <matthew.auld@intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20201224135544.1713-2-chris@chris-wilson.co.uk
parent 16f2941a

drivers/gpu/drm/i915/gt/intel_execlists_submission.c

+105 −134
Original line number Diff line number Diff line
@@ -293,9 +293,15 @@ static int queue_prio(const struct intel_engine_execlists *execlists) 
	return ((p->priority + 1) << I915_USER_PRIORITY_SHIFT) - ffs(p->used);

}



static int virtual_prio(const struct intel_engine_execlists *el)

{

	struct rb_node *rb = rb_first_cached(&el->virtual);



	return rb ? rb_entry(rb, struct ve_node, rb)->prio : INT_MIN;

}



static inline bool need_preempt(const struct intel_engine_cs *engine,

				const struct i915_request *rq,

				struct rb_node *rb)

				const struct i915_request *rq)

{

	int last_prio;
@@ -332,25 +338,6 @@ static inline bool need_preempt(const struct intel_engine_cs *engine, 
	    rq_prio(list_next_entry(rq, sched.link)) > last_prio)

		return true;



	if (rb) {

		struct virtual_engine *ve =

			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);

		bool preempt = false;



		if (engine == ve->siblings[0]) { /* only preempt one sibling */

			struct i915_request *next;



			rcu_read_lock();

			next = READ_ONCE(ve->request);

			if (next)

				preempt = rq_prio(next) > last_prio;

			rcu_read_unlock();

		}



		if (preempt)

			return preempt;

	}



	/*

	 * If the inflight context did not trigger the preemption, then maybe

	 * it was the set of queued requests? Pick the highest priority in
@@ -361,7 +348,8 @@ static inline bool need_preempt(const struct intel_engine_cs *engine, 
	 * ELSP[0] or ELSP[1] as, thanks again to PI, if it was the same

	 * context, it's priority would not exceed ELSP[0] aka last_prio.

	 */

	return queue_prio(&engine->execlists) > last_prio;

	return max(virtual_prio(&engine->execlists),

		   queue_prio(&engine->execlists)) > last_prio;

}



__maybe_unused static inline bool
@@ -997,6 +985,35 @@ static bool virtual_matches(const struct virtual_engine *ve, 
	return true;

}



static struct virtual_engine *

first_virtual_engine(struct intel_engine_cs *engine)

{

	struct intel_engine_execlists *el = &engine->execlists;

	struct rb_node *rb = rb_first_cached(&el->virtual);



	while (rb) {

		struct virtual_engine *ve =

			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);

		struct i915_request *rq = READ_ONCE(ve->request);



		/* lazily cleanup after another engine handled rq */

		if (!rq) {

			rb_erase_cached(rb, &el->virtual);

			RB_CLEAR_NODE(rb);

			rb = rb_first_cached(&el->virtual);

			continue;

		}



		if (!virtual_matches(ve, rq, engine)) {

			rb = rb_next(rb);

			continue;

		}

		return ve;

	}



	return NULL;

}



static void virtual_xfer_context(struct virtual_engine *ve,

				 struct intel_engine_cs *engine)

{
@@ -1084,32 +1101,15 @@ static void defer_active(struct intel_engine_cs *engine) 


static bool

need_timeslice(const struct intel_engine_cs *engine,

	       const struct i915_request *rq,

	       const struct rb_node *rb)

	       const struct i915_request *rq)

{

	int hint;



	if (!intel_engine_has_timeslices(engine))

		return false;



	hint = engine->execlists.queue_priority_hint;



	if (rb) {

		const struct virtual_engine *ve =

			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);

		const struct intel_engine_cs *inflight =

			intel_context_inflight(&ve->context);



		if (!inflight || inflight == engine) {

			struct i915_request *next;



			rcu_read_lock();

			next = READ_ONCE(ve->request);

			if (next)

				hint = max(hint, rq_prio(next));

			rcu_read_unlock();

		}

	}

	hint = max(engine->execlists.queue_priority_hint,

		   virtual_prio(&engine->execlists));



	if (!list_is_last(&rq->sched.link, &engine->active.requests))

		hint = max(hint, rq_prio(list_next_entry(rq, sched.link)));
@@ -1256,6 +1256,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) 
	struct i915_request **port = execlists->pending;

	struct i915_request ** const last_port = port + execlists->port_mask;

	struct i915_request *last = *execlists->active;

	struct virtual_engine *ve;

	struct rb_node *rb;

	bool submit = false;
@@ -1283,26 +1284,6 @@ static void execlists_dequeue(struct intel_engine_cs *engine) 


	spin_lock(&engine->active.lock);



	for (rb = rb_first_cached(&execlists->virtual); rb; ) {

		struct virtual_engine *ve =

			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);

		struct i915_request *rq = READ_ONCE(ve->request);



		if (!rq) { /* lazily cleanup after another engine handled rq */

			rb_erase_cached(rb, &execlists->virtual);

			RB_CLEAR_NODE(rb);

			rb = rb_first_cached(&execlists->virtual);

			continue;

		}



		if (!virtual_matches(ve, rq, engine)) {

			rb = rb_next(rb);

			continue;

		}



		break;

	}



	/*

	 * If the queue is higher priority than the last

	 * request in the currently active context, submit afresh.
@@ -1325,7 +1306,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) 
	if (last) {

		if (i915_request_completed(last)) {

			goto check_secondary;

		} else if (need_preempt(engine, last, rb)) {

		} else if (need_preempt(engine, last)) {

			ENGINE_TRACE(engine,

				     "preempting last=%llx:%lld, prio=%d, hint=%d\n",

				     last->fence.context,
@@ -1351,7 +1332,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) 
			__unwind_incomplete_requests(engine);



			last = NULL;

		} else if (need_timeslice(engine, last, rb) &&

		} else if (need_timeslice(engine, last) &&

			   timeslice_expired(execlists, last)) {

			ENGINE_TRACE(engine,

				     "expired last=%llx:%lld, prio=%d, hint=%d, yield?=%s\n",
@@ -1402,33 +1383,26 @@ static void execlists_dequeue(struct intel_engine_cs *engine) 
		}

	}



	while (rb) { /* XXX virtual is always taking precedence */

		struct virtual_engine *ve =

			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);

	/* XXX virtual is always taking precedence */

	while ((ve = first_virtual_engine(engine))) {

		struct i915_request *rq;



		spin_lock(&ve->base.active.lock);



		rq = ve->request;

		if (unlikely(!rq)) { /* lost the race to a sibling */

			spin_unlock(&ve->base.active.lock);

			rb_erase_cached(rb, &execlists->virtual);

			RB_CLEAR_NODE(rb);

			rb = rb_first_cached(&execlists->virtual);

			continue;

		}

		if (unlikely(!rq)) /* lost the race to a sibling */

			goto unlock;



		GEM_BUG_ON(rq != ve->request);

		GEM_BUG_ON(rq->engine != &ve->base);

		GEM_BUG_ON(rq->context != &ve->context);



		if (rq_prio(rq) >= queue_prio(execlists)) {

			if (!virtual_matches(ve, rq, engine)) {

		if (unlikely(rq_prio(rq) < queue_prio(execlists))) {

			spin_unlock(&ve->base.active.lock);

				rb = rb_next(rb);

				continue;

			break;

		}



		GEM_BUG_ON(!virtual_matches(ve, rq, engine));



		if (last && !can_merge_rq(last, rq)) {

			spin_unlock(&ve->base.active.lock);

			spin_unlock(&engine->active.lock);
@@ -1446,8 +1420,9 @@ static void execlists_dequeue(struct intel_engine_cs *engine) 
			     yesno(engine != ve->siblings[0]));



		WRITE_ONCE(ve->request, NULL);

			WRITE_ONCE(ve->base.execlists.queue_priority_hint,

				   INT_MIN);

		WRITE_ONCE(ve->base.execlists.queue_priority_hint, INT_MIN);



		rb = &ve->nodes[engine->id].rb;

		rb_erase_cached(rb, &execlists->virtual);

		RB_CLEAR_NODE(rb);
@@ -1474,7 +1449,10 @@ static void execlists_dequeue(struct intel_engine_cs *engine) 
			submit = true;

			last = rq;

		}



		i915_request_put(rq);

unlock:

		spin_unlock(&ve->base.active.lock);



		/*

		 * Hmm, we have a bunch of virtual engine requests,
@@ -1483,14 +1461,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) 
		 * until we have no more relevant requests (i.e.

		 * the normal submit queue has higher priority).

		 */

			if (!submit) {

				spin_unlock(&ve->base.active.lock);

				rb = rb_first_cached(&execlists->virtual);

				continue;

			}

		}



		spin_unlock(&ve->base.active.lock);

		if (submit)

			break;

	}