Commit d8b93201 authored by Fei Yang's avatar Fei Yang Committed by Tvrtko Ursulin
Browse files

drm/i915: avoid concurrent writes to aux_inv 



GPU hangs have been observed when multiple engines write to the
same aux_inv register at the same time. To avoid this each engine
should only invalidate its own auxiliary table. The function
gen12_emit_flush_xcs() currently invalidate the auxiliary table for
all engines because the rq->engine is not necessarily the engine
eventually carrying out the request, and potentially the engine
could even be a virtual one (with engine->instance being -1).
With the MMIO remap feature, we can actually set bit 17 of MI_LRI
instruction and let the hardware to figure out the local aux_inv
register at runtime to avoid invalidating auxiliary table for all
engines.

Bspec: 45728

v2: Invalidate AUX table for indirect context as well.

Cc: Stuart Summers <stuart.summers@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Signed-off-by: default avatarChris Wilson <chris.p.wilson@intel.com>
Signed-off-by: default avatarFei Yang <fei.yang@intel.com>
Reviewed-by: default avatarTvrtko Ursulin <tvrtko.ursulin@intel.com>
Signed-off-by: default avatarTvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20220328171650.1900674-1-fei.yang@intel.com
parent be658e70

drivers/gpu/drm/i915/gt/gen8_engine_cs.c

+11 −39
Original line number Diff line number Diff line
@@ -6,7 +6,6 @@ 
#include "gen8_engine_cs.h"

#include "i915_drv.h"

#include "intel_gpu_commands.h"

#include "intel_gt_regs.h"

#include "intel_lrc.h"

#include "intel_ring.h"
@@ -165,33 +164,9 @@ static u32 preparser_disable(bool state) 
	return MI_ARB_CHECK | 1 << 8 | state;

}



static i915_reg_t aux_inv_reg(const struct intel_engine_cs *engine)

u32 *gen12_emit_aux_table_inv(u32 *cs, const i915_reg_t inv_reg)

{

	static const i915_reg_t vd[] = {

		GEN12_VD0_AUX_NV,

		GEN12_VD1_AUX_NV,

		GEN12_VD2_AUX_NV,

		GEN12_VD3_AUX_NV,

	};



	static const i915_reg_t ve[] = {

		GEN12_VE0_AUX_NV,

		GEN12_VE1_AUX_NV,

	};



	if (engine->class == VIDEO_DECODE_CLASS)

		return vd[engine->instance];



	if (engine->class == VIDEO_ENHANCEMENT_CLASS)

		return ve[engine->instance];



	GEM_BUG_ON("unknown aux_inv reg\n");

	return INVALID_MMIO_REG;

}



static u32 *gen12_emit_aux_table_inv(const i915_reg_t inv_reg, u32 *cs)

{

	*cs++ = MI_LOAD_REGISTER_IMM(1);

	*cs++ = MI_LOAD_REGISTER_IMM(1) | MI_LRI_MMIO_REMAP_EN;

	*cs++ = i915_mmio_reg_offset(inv_reg);

	*cs++ = AUX_INV;

	*cs++ = MI_NOOP;
@@ -274,7 +249,7 @@ int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode) 


		if (!HAS_FLAT_CCS(rq->engine->i915)) {

			/* hsdes: 1809175790 */

			cs = gen12_emit_aux_table_inv(GEN12_GFX_CCS_AUX_NV, cs);

			cs = gen12_emit_aux_table_inv(cs, GEN12_GFX_CCS_AUX_NV);

		}



		*cs++ = preparser_disable(false);
@@ -293,10 +268,12 @@ int gen12_emit_flush_xcs(struct i915_request *rq, u32 mode) 
	if (mode & EMIT_INVALIDATE) {

		cmd += 2;



		if (!HAS_FLAT_CCS(rq->engine->i915)) {

		if (!HAS_FLAT_CCS(rq->engine->i915) &&

		    (rq->engine->class == VIDEO_DECODE_CLASS ||

		     rq->engine->class == VIDEO_ENHANCEMENT_CLASS)) {

			aux_inv = rq->engine->mask & ~BIT(BCS0);

			if (aux_inv)

				cmd += 2 * hweight32(aux_inv) + 2;

				cmd += 4;

		}

	}
@@ -329,15 +306,10 @@ int gen12_emit_flush_xcs(struct i915_request *rq, u32 mode) 
	*cs++ = 0; /* value */



	if (aux_inv) { /* hsdes: 1809175790 */

		struct intel_engine_cs *engine;

		unsigned int tmp;



		*cs++ = MI_LOAD_REGISTER_IMM(hweight32(aux_inv));

		for_each_engine_masked(engine, rq->engine->gt, aux_inv, tmp) {

			*cs++ = i915_mmio_reg_offset(aux_inv_reg(engine));

			*cs++ = AUX_INV;

		}

		*cs++ = MI_NOOP;

		if (rq->engine->class == VIDEO_DECODE_CLASS)

			cs = gen12_emit_aux_table_inv(cs, GEN12_VD0_AUX_NV);

		else

			cs = gen12_emit_aux_table_inv(cs, GEN12_VE0_AUX_NV);

	}



	if (mode & EMIT_INVALIDATE)

drivers/gpu/drm/i915/gt/gen8_engine_cs.h

+3 −1
Original line number Diff line number Diff line
@@ -10,7 +10,7 @@ 
#include <linux/types.h>



#include "i915_gem.h" /* GEM_BUG_ON */



#include "intel_gt_regs.h"

#include "intel_gpu_commands.h"



struct i915_request;
@@ -38,6 +38,8 @@ u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs); 
u32 *gen11_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs);

u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs);



u32 *gen12_emit_aux_table_inv(u32 *cs, const i915_reg_t inv_reg);



static inline u32 *

__gen8_emit_pipe_control(u32 *batch, u32 flags0, u32 flags1, u32 offset)

{

drivers/gpu/drm/i915/gt/intel_gpu_commands.h

+1 −0
Original line number Diff line number Diff line
@@ -144,6 +144,7 @@ 
#define MI_LOAD_REGISTER_IMM(x)	MI_INSTR(0x22, 2*(x)-1)

/* Gen11+. addr = base + (ctx_restore ? offset & GENMASK(12,2) : offset) */

#define   MI_LRI_LRM_CS_MMIO		REG_BIT(19)

#define   MI_LRI_MMIO_REMAP_EN		REG_BIT(17)

#define   MI_LRI_FORCE_POSTED		(1<<12)

#define MI_LOAD_REGISTER_IMM_MAX_REGS (126)

#define MI_STORE_REGISTER_MEM        MI_INSTR(0x24, 1)

drivers/gpu/drm/i915/gt/intel_lrc.c

+12 −0
Original line number Diff line number Diff line
@@ -1208,6 +1208,10 @@ gen12_emit_indirect_ctx_rcs(const struct intel_context *ce, u32 *cs) 
	    IS_DG2_G11(ce->engine->i915))

		cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE, 0);



	/* hsdes: 1809175790 */

	if (!HAS_FLAT_CCS(ce->engine->i915))

		cs = gen12_emit_aux_table_inv(cs, GEN12_GFX_CCS_AUX_NV);



	return cs;

}
@@ -1225,6 +1229,14 @@ gen12_emit_indirect_ctx_xcs(const struct intel_context *ce, u32 *cs) 
						    PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE,

						    0);



	/* hsdes: 1809175790 */

	if (!HAS_FLAT_CCS(ce->engine->i915)) {

		if (ce->engine->class == VIDEO_DECODE_CLASS)

			cs = gen12_emit_aux_table_inv(cs, GEN12_VD0_AUX_NV);

		else if (ce->engine->class == VIDEO_ENHANCEMENT_CLASS)

			cs = gen12_emit_aux_table_inv(cs, GEN12_VE0_AUX_NV);

	}



	return cs;

}