Commit 9275277d authored by Fei Yang's avatar Fei Yang Committed by Andi Shyti
Browse files

drm/i915: use pat_index instead of cache_level



Currently the KMD is using enum i915_cache_level to set caching policy for
buffer objects. This is flaky because the PAT index which really controls
the caching behavior in PTE has far more levels than what's defined in the
enum. In addition, the PAT index is platform dependent, having to translate
between i915_cache_level and PAT index is not reliable, and makes the code
more complicated.

From UMD's perspective there is also a necessity to set caching policy for
performance fine tuning. It's much easier for the UMD to directly use PAT
index because the behavior of each PAT index is clearly defined in Bspec.
Having the abstracted i915_cache_level sitting in between would only cause
more ambiguity. PAT is expected to work much like MOCS already works today,
and by design userspace is expected to select the index that exactly
matches the desired behavior described in the hardware specification.

For these reasons this patch replaces i915_cache_level with PAT index. Also
note, the cache_level is not completely removed yet, because the KMD still
has the need of creating buffer objects with simple cache settings such as
cached, uncached, or writethrough. For kernel objects, cache_level is used
for simplicity and backward compatibility. For Pre-gen12 platforms PAT can
have 1:1 mapping to i915_cache_level, so these two are interchangeable. see
the use of LEGACY_CACHELEVEL.

One consequence of this change is that gen8_pte_encode is no longer working
for gen12 platforms due to the fact that gen12 platforms has different PAT
definitions. In the meantime the mtl_pte_encode introduced specfically for
MTL becomes generic for all gen12 platforms. This patch renames the MTL
PTE encode function into gen12_pte_encode and apply it to all gen12. Even
though this change looks unrelated, but separating them would temporarily
break gen12 PTE encoding, thus squash them in one patch.

Special note: this patch changes the way caching behavior is controlled in
the sense that some objects are left to be managed by userspace. For such
objects we need to be careful not to change the userspace settings.There
are kerneldoc and comments added around obj->cache_coherent, cache_dirty,
and how to bypass the checkings by i915_gem_object_has_cache_level. For
full understanding, these changes need to be looked at together with the
two follow-up patches, one disables the {set|get}_caching ioctl's and the
other adds set_pat extension to the GEM_CREATE uAPI.

Bspec: 63019

Cc: Chris Wilson <chris.p.wilson@linux.intel.com>
Signed-off-by: default avatarFei Yang <fei.yang@intel.com>
Reviewed-by: default avatarAndi Shyti <andi.shyti@linux.intel.com>
Reviewed-by: default avatarMatt Roper <matthew.d.roper@intel.com>
Signed-off-by: default avatarAndi Shyti <andi.shyti@linux.intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20230509165200.1740-3-fei.yang@intel.com
parent 5e352e32
Loading
Loading
Loading
Loading
+6 −6
Original line number Diff line number Diff line
@@ -43,24 +43,24 @@ static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
static void dpt_insert_page(struct i915_address_space *vm,
			    dma_addr_t addr,
			    u64 offset,
			    enum i915_cache_level level,
			    unsigned int pat_index,
			    u32 flags)
{
	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
	gen8_pte_t __iomem *base = dpt->iomem;

	gen8_set_pte(base + offset / I915_GTT_PAGE_SIZE,
		     vm->pte_encode(addr, level, flags));
		     vm->pte_encode(addr, pat_index, flags));
}

static void dpt_insert_entries(struct i915_address_space *vm,
			       struct i915_vma_resource *vma_res,
			       enum i915_cache_level level,
			       unsigned int pat_index,
			       u32 flags)
{
	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
	gen8_pte_t __iomem *base = dpt->iomem;
	const gen8_pte_t pte_encode = vm->pte_encode(0, level, flags);
	const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags);
	struct sgt_iter sgt_iter;
	dma_addr_t addr;
	int i;
@@ -83,7 +83,7 @@ static void dpt_clear_range(struct i915_address_space *vm,
static void dpt_bind_vma(struct i915_address_space *vm,
			 struct i915_vm_pt_stash *stash,
			 struct i915_vma_resource *vma_res,
			 enum i915_cache_level cache_level,
			 unsigned int pat_index,
			 u32 flags)
{
	u32 pte_flags;
@@ -98,7 +98,7 @@ static void dpt_bind_vma(struct i915_address_space *vm,
	if (vma_res->bi.lmem)
		pte_flags |= PTE_LM;

	vm->insert_entries(vm, vma_res, cache_level, pte_flags);
	vm->insert_entries(vm, vma_res, pat_index, pte_flags);

	vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE;

+40 −18
Original line number Diff line number Diff line
@@ -27,8 +27,15 @@ static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj)
	if (IS_DGFX(i915))
		return false;

	return !(obj->cache_level == I915_CACHE_NONE ||
		 obj->cache_level == I915_CACHE_WT);
	/*
	 * For objects created by userspace through GEM_CREATE with pat_index
	 * set by set_pat extension, i915_gem_object_has_cache_level() will
	 * always return true, because the coherency of such object is managed
	 * by userspace. Othereise the call here would fall back to checking
	 * whether the object is un-cached or write-through.
	 */
	return !(i915_gem_object_has_cache_level(obj, I915_CACHE_NONE) ||
		 i915_gem_object_has_cache_level(obj, I915_CACHE_WT));
}

bool i915_gem_cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
@@ -267,7 +274,13 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
{
	int ret;

	if (obj->cache_level == cache_level)
	/*
	 * For objects created by userspace through GEM_CREATE with pat_index
	 * set by set_pat extension, simply return 0 here without touching
	 * the cache setting, because such objects should have an immutable
	 * cache setting by desgin and always managed by userspace.
	 */
	if (i915_gem_object_has_cache_level(obj, cache_level))
		return 0;

	ret = i915_gem_object_wait(obj,
@@ -278,10 +291,8 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		return ret;

	/* Always invalidate stale cachelines */
	if (obj->cache_level != cache_level) {
	i915_gem_object_set_cache_coherency(obj, cache_level);
	obj->cache_dirty = true;
	}

	/* The cache-level will be applied when each vma is rebound. */
	return i915_gem_object_unbind(obj,
@@ -306,20 +317,22 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
		goto out;
	}

	switch (obj->cache_level) {
	case I915_CACHE_LLC:
	case I915_CACHE_L3_LLC:
		args->caching = I915_CACHING_CACHED;
		break;
	/*
	 * This ioctl should be disabled for the objects with pat_index
	 * set by user space.
	 */
	if (obj->pat_set_by_user) {
		err = -EOPNOTSUPP;
		goto out;
	}

	case I915_CACHE_WT:
	if (i915_gem_object_has_cache_level(obj, I915_CACHE_LLC) ||
	    i915_gem_object_has_cache_level(obj, I915_CACHE_L3_LLC))
		args->caching = I915_CACHING_CACHED;
	else if (i915_gem_object_has_cache_level(obj, I915_CACHE_WT))
		args->caching = I915_CACHING_DISPLAY;
		break;

	default:
	else
		args->caching = I915_CACHING_NONE;
		break;
	}
out:
	rcu_read_unlock();
	return err;
@@ -364,6 +377,15 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
	if (!obj)
		return -ENOENT;

	/*
	 * This ioctl should be disabled for the objects with pat_index
	 * set by user space.
	 */
	if (obj->pat_set_by_user) {
		ret = -EOPNOTSUPP;
		goto out;
	}

	/*
	 * The caching mode of proxy object is handled by its generator, and
	 * not allowed to be changed by userspace.
+12 −3
Original line number Diff line number Diff line
@@ -640,9 +640,15 @@ static inline int use_cpu_reloc(const struct reloc_cache *cache,
	if (DBG_FORCE_RELOC == FORCE_GTT_RELOC)
		return false;

	/*
	 * For objects created by userspace through GEM_CREATE with pat_index
	 * set by set_pat extension, i915_gem_object_has_cache_level() always
	 * return true, otherwise the call would fall back to checking whether
	 * the object is un-cached.
	 */
	return (cache->has_llc ||
		obj->cache_dirty ||
		obj->cache_level != I915_CACHE_NONE);
		!i915_gem_object_has_cache_level(obj, I915_CACHE_NONE));
}

static int eb_reserve_vma(struct i915_execbuffer *eb,
@@ -1324,7 +1330,10 @@ static void *reloc_iomap(struct i915_vma *batch,
	if (drm_mm_node_allocated(&cache->node)) {
		ggtt->vm.insert_page(&ggtt->vm,
				     i915_gem_object_get_dma_address(obj, page),
				     offset, I915_CACHE_NONE, 0);
				     offset,
				     i915_gem_get_pat_index(ggtt->vm.i915,
							    I915_CACHE_NONE),
				     0);
	} else {
		offset += page << PAGE_SHIFT;
	}
@@ -1464,7 +1473,7 @@ eb_relocate_entry(struct i915_execbuffer *eb,
			reloc_cache_unmap(&eb->reloc_cache);
			mutex_lock(&vma->vm->mutex);
			err = i915_vma_bind(target->vma,
					    target->vma->obj->cache_level,
					    target->vma->obj->pat_index,
					    PIN_GLOBAL, NULL, NULL);
			mutex_unlock(&vma->vm->mutex);
			reloc_cache_remap(&eb->reloc_cache, ev->vma->obj);
+10 −1
Original line number Diff line number Diff line
@@ -383,7 +383,16 @@ static vm_fault_t vm_fault_gtt(struct vm_fault *vmf)
	}

	/* Access to snoopable pages through the GTT is incoherent. */
	if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(i915)) {
	/*
	 * For objects created by userspace through GEM_CREATE with pat_index
	 * set by set_pat extension, coherency is managed by userspace, make
	 * sure we don't fail handling the vm fault by calling
	 * i915_gem_object_has_cache_level() which always return true for such
	 * objects. Otherwise this helper function would fall back to checking
	 * whether the object is un-cached.
	 */
	if (!(i915_gem_object_has_cache_level(obj, I915_CACHE_NONE) ||
	      HAS_LLC(i915))) {
		ret = -EFAULT;
		goto err_unpin;
	}
+50 −1
Original line number Diff line number Diff line
@@ -54,6 +54,24 @@ unsigned int i915_gem_get_pat_index(struct drm_i915_private *i915,
	return INTEL_INFO(i915)->cachelevel_to_pat[level];
}

bool i915_gem_object_has_cache_level(const struct drm_i915_gem_object *obj,
				     enum i915_cache_level lvl)
{
	/*
	 * In case the pat_index is set by user space, this kernel mode
	 * driver should leave the coherency to be managed by user space,
	 * simply return true here.
	 */
	if (obj->pat_set_by_user)
		return true;

	/*
	 * Otherwise the pat_index should have been converted from cache_level
	 * so that the following comparison is valid.
	 */
	return obj->pat_index == i915_gem_get_pat_index(obj_to_i915(obj), lvl);
}

struct drm_i915_gem_object *i915_gem_object_alloc(void)
{
	struct drm_i915_gem_object *obj;
@@ -133,7 +151,7 @@ void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj,
{
	struct drm_i915_private *i915 = to_i915(obj->base.dev);

	obj->cache_level = cache_level;
	obj->pat_index = i915_gem_get_pat_index(i915, cache_level);

	if (cache_level != I915_CACHE_NONE)
		obj->cache_coherent = (I915_BO_CACHE_COHERENT_FOR_READ |
@@ -148,6 +166,37 @@ void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj,
		!IS_DGFX(i915);
}

/**
 * i915_gem_object_set_pat_index - set PAT index to be used in PTE encode
 * @obj: #drm_i915_gem_object
 * @pat_index: PAT index
 *
 * This is a clone of i915_gem_object_set_cache_coherency taking pat index
 * instead of cache_level as its second argument.
 */
void i915_gem_object_set_pat_index(struct drm_i915_gem_object *obj,
				   unsigned int pat_index)
{
	struct drm_i915_private *i915 = to_i915(obj->base.dev);

	if (obj->pat_index == pat_index)
		return;

	obj->pat_index = pat_index;

	if (pat_index != i915_gem_get_pat_index(i915, I915_CACHE_NONE))
		obj->cache_coherent = (I915_BO_CACHE_COHERENT_FOR_READ |
				       I915_BO_CACHE_COHERENT_FOR_WRITE);
	else if (HAS_LLC(i915))
		obj->cache_coherent = I915_BO_CACHE_COHERENT_FOR_READ;
	else
		obj->cache_coherent = 0;

	obj->cache_dirty =
		!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE) &&
		!IS_DGFX(i915);
}

bool i915_gem_object_can_bypass_llc(struct drm_i915_gem_object *obj)
{
	struct drm_i915_private *i915 = to_i915(obj->base.dev);
Loading