Commit 529b9ec8 authored by Matthew Auld's avatar Matthew Auld
Browse files

drm/i915/gtt: map the PD up front 



We need to generalise our accessor for the page directories and tables from
using the simple kmap_atomic to support local memory, and this setup
must be done on acquisition of the backing storage prior to entering
fence execution contexts. Here we replace the kmap with the object
mapping code that for simple single page shmemfs object will return a
plain kmap, that is then kept for the lifetime of the page directory.

Note that keeping the mapping around is a potential concern here, since
while the vma is pinned the mapping remains there for the PDs
underneath, or at least until the used_count reaches zero, at which
point we can safely destroy the mapping. For 32b this will be even worse
since the address space is more limited, but since this change mostly
impacts full ppGTT platforms, the justification is that for modern
platforms we shouldn't care too much about 32b.

Signed-off-by: default avatarMatthew Auld <matthew.auld@intel.com>
Signed-off-by: default avatarChris Wilson <chris@chris-wilson.co.uk>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Reviewed-by: default avatarTvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20210427085417.120246-3-matthew.auld@intel.com
parent fa85bfd1

drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c

+2 −9
Original line number Diff line number Diff line
@@ -1740,7 +1740,6 @@ static int read_from_scratch(struct i915_gem_context *ctx, 
static int check_scratch_page(struct i915_gem_context *ctx, u32 *out)

{

	struct i915_address_space *vm;

	struct page *page;

	u32 *vaddr;

	int err = 0;
@@ -1748,24 +1747,18 @@ static int check_scratch_page(struct i915_gem_context *ctx, u32 *out) 
	if (!vm)

		return -ENODEV;



	page = __px_page(vm->scratch[0]);

	if (!page) {

	if (!vm->scratch[0]) {

		pr_err("No scratch page!\n");

		return -EINVAL;

	}



	vaddr = kmap(page);

	if (!vaddr) {

		pr_err("No (mappable) scratch page!\n");

		return -EINVAL;

	}

	vaddr = __px_vaddr(vm->scratch[0]);



	memcpy(out, vaddr, sizeof(*out));

	if (memchr_inv(vaddr, *out, PAGE_SIZE)) {

		pr_err("Inconsistent initial state of scratch page!\n");

		err = -EINVAL;

	}

	kunmap(page);



	return err;

}

drivers/gpu/drm/i915/gt/gen6_ppgtt.c

+4 −7
Original line number Diff line number Diff line
@@ -96,9 +96,8 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm, 
		 * entries back to scratch.

		 */



		vaddr = kmap_atomic_px(pt);

		vaddr = px_vaddr(pt);

		memset32(vaddr + pte, scratch_pte, count);

		kunmap_atomic(vaddr);



		pte = 0;

	}
@@ -120,7 +119,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, 


	GEM_BUG_ON(!pd->entry[act_pt]);



	vaddr = kmap_atomic_px(i915_pt_entry(pd, act_pt));

	vaddr = px_vaddr(i915_pt_entry(pd, act_pt));

	do {

		GEM_BUG_ON(sg_dma_len(iter.sg) < I915_GTT_PAGE_SIZE);

		vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma);
@@ -136,12 +135,10 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, 
		}



		if (++act_pte == GEN6_PTES) {

			kunmap_atomic(vaddr);

			vaddr = kmap_atomic_px(i915_pt_entry(pd, ++act_pt));

			vaddr = px_vaddr(i915_pt_entry(pd, ++act_pt));

			act_pte = 0;

		}

	} while (1);

	kunmap_atomic(vaddr);



	vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;

}
@@ -235,7 +232,7 @@ static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt) 
		goto err_scratch0;

	}



	ret = pin_pt_dma(vm, vm->scratch[1]);

	ret = map_pt_dma(vm, vm->scratch[1]);

	if (ret)

		goto err_scratch1;

drivers/gpu/drm/i915/gt/gen8_ppgtt.c

+10 −16
Original line number Diff line number Diff line
@@ -242,11 +242,10 @@ static u64 __gen8_ppgtt_clear(struct i915_address_space * const vm, 
			    atomic_read(&pt->used));

			GEM_BUG_ON(!count || count >= atomic_read(&pt->used));



			vaddr = kmap_atomic_px(pt);

			vaddr = px_vaddr(pt);

			memset64(vaddr + gen8_pd_index(start, 0),

				 vm->scratch[0]->encode,

				 count);

			kunmap_atomic(vaddr);



			atomic_sub(count, &pt->used);

			start += count;
@@ -375,7 +374,7 @@ gen8_ppgtt_insert_pte(struct i915_ppgtt *ppgtt, 
	gen8_pte_t *vaddr;



	pd = i915_pd_entry(pdp, gen8_pd_index(idx, 2));

	vaddr = kmap_atomic_px(i915_pt_entry(pd, gen8_pd_index(idx, 1)));

	vaddr = px_vaddr(i915_pt_entry(pd, gen8_pd_index(idx, 1)));

	do {

		GEM_BUG_ON(sg_dma_len(iter->sg) < I915_GTT_PAGE_SIZE);

		vaddr[gen8_pd_index(idx, 0)] = pte_encode | iter->dma;
@@ -402,12 +401,10 @@ gen8_ppgtt_insert_pte(struct i915_ppgtt *ppgtt, 
			}



			clflush_cache_range(vaddr, PAGE_SIZE);

			kunmap_atomic(vaddr);

			vaddr = kmap_atomic_px(i915_pt_entry(pd, gen8_pd_index(idx, 1)));

			vaddr = px_vaddr(i915_pt_entry(pd, gen8_pd_index(idx, 1)));

		}

	} while (1);

	clflush_cache_range(vaddr, PAGE_SIZE);

	kunmap_atomic(vaddr);



	return idx;

}
@@ -442,7 +439,7 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, 
			encode |= GEN8_PDE_PS_2M;

			page_size = I915_GTT_PAGE_SIZE_2M;



			vaddr = kmap_atomic_px(pd);

			vaddr = px_vaddr(pd);

		} else {

			struct i915_page_table *pt =

				i915_pt_entry(pd, __gen8_pte_index(start, 1));
@@ -457,7 +454,7 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, 
			     rem >= (I915_PDES - index) * I915_GTT_PAGE_SIZE))

				maybe_64K = __gen8_pte_index(start, 1);



			vaddr = kmap_atomic_px(pt);

			vaddr = px_vaddr(pt);

		}



		do {
@@ -491,7 +488,6 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, 
		} while (rem >= page_size && index < I915_PDES);



		clflush_cache_range(vaddr, PAGE_SIZE);

		kunmap_atomic(vaddr);



		/*

		 * Is it safe to mark the 2M block as 64K? -- Either we have
@@ -505,9 +501,8 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, 
		      !iter->sg && IS_ALIGNED(vma->node.start +

					      vma->node.size,

					      I915_GTT_PAGE_SIZE_2M)))) {

			vaddr = kmap_atomic_px(pd);

			vaddr = px_vaddr(pd);

			vaddr[maybe_64K] |= GEN8_PDE_IPS_64K;

			kunmap_atomic(vaddr);

			page_size = I915_GTT_PAGE_SIZE_64K;



			/*
@@ -523,12 +518,11 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, 
				u16 i;



				encode = vma->vm->scratch[0]->encode;

				vaddr = kmap_atomic_px(i915_pt_entry(pd, maybe_64K));

				vaddr = px_vaddr(i915_pt_entry(pd, maybe_64K));



				for (i = 1; i < index; i += 16)

					memset64(vaddr + i, encode, 15);



				kunmap_atomic(vaddr);

			}

		}
@@ -602,7 +596,7 @@ static int gen8_init_scratch(struct i915_address_space *vm) 
		if (IS_ERR(obj))

			goto free_scratch;



		ret = pin_pt_dma(vm, obj);

		ret = map_pt_dma(vm, obj);

		if (ret) {

			i915_gem_object_put(obj);

			goto free_scratch;
@@ -639,7 +633,7 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt) 
		if (IS_ERR(pde))

			return PTR_ERR(pde);



		err = pin_pt_dma(vm, pde->pt.base);

		err = map_pt_dma(vm, pde->pt.base);

		if (err) {

			free_pd(vm, pde);

			return err;
@@ -674,7 +668,7 @@ gen8_alloc_top_pd(struct i915_address_space *vm) 
		goto err_pd;

	}



	err = pin_pt_dma(vm, pd->pt.base);

	err = map_pt_dma(vm, pd->pt.base);

	if (err)

		goto err_pd;

drivers/gpu/drm/i915/gt/intel_ggtt.c

+1 −1
Original line number Diff line number Diff line
@@ -657,7 +657,7 @@ static int init_aliasing_ppgtt(struct i915_ggtt *ggtt) 
		goto err_ppgtt;



	i915_gem_object_lock(ppgtt->vm.scratch[0], NULL);

	err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash);

	err = i915_vm_map_pt_stash(&ppgtt->vm, &stash);

	i915_gem_object_unlock(ppgtt->vm.scratch[0]);

	if (err)

		goto err_stash;

drivers/gpu/drm/i915/gt/intel_gtt.c

+22 −26
Original line number Diff line number Diff line
@@ -25,27 +25,25 @@ struct drm_i915_gem_object *alloc_pt_dma(struct i915_address_space *vm, int sz) 
	return obj;

}



int pin_pt_dma(struct i915_address_space *vm, struct drm_i915_gem_object *obj)

int map_pt_dma(struct i915_address_space *vm, struct drm_i915_gem_object *obj)

{

	int err;

	void *vaddr;



	i915_gem_object_lock(obj, NULL);

	err = i915_gem_object_pin_pages(obj);

	i915_gem_object_unlock(obj);

	if (err)

		return err;

	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);

	if (IS_ERR(vaddr))

		return PTR_ERR(vaddr);



	i915_gem_object_make_unshrinkable(obj);

	return 0;

}



int pin_pt_dma_locked(struct i915_address_space *vm, struct drm_i915_gem_object *obj)

int map_pt_dma_locked(struct i915_address_space *vm, struct drm_i915_gem_object *obj)

{

	int err;

	void *vaddr;



	err = i915_gem_object_pin_pages(obj);

	if (err)

		return err;

	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);

	if (IS_ERR(vaddr))

		return PTR_ERR(vaddr);



	i915_gem_object_make_unshrinkable(obj);

	return 0;
@@ -155,6 +153,14 @@ void clear_pages(struct i915_vma *vma) 
	memset(&vma->page_sizes, 0, sizeof(vma->page_sizes));

}



void *__px_vaddr(struct drm_i915_gem_object *p)

{

	enum i915_map_type type;



	GEM_BUG_ON(!i915_gem_object_has_pages(p));

	return page_unpack_bits(p->mm.mapping, &type);

}



dma_addr_t __px_dma(struct drm_i915_gem_object *p)

{

	GEM_BUG_ON(!i915_gem_object_has_pages(p));
@@ -170,32 +176,22 @@ struct page *__px_page(struct drm_i915_gem_object *p) 
void

fill_page_dma(struct drm_i915_gem_object *p, const u64 val, unsigned int count)

{

	struct page *page = __px_page(p);

	void *vaddr;

	void *vaddr = __px_vaddr(p);



	vaddr = kmap(page);

	memset64(vaddr, val, count);

	clflush_cache_range(vaddr, PAGE_SIZE);

	kunmap(page);

}



static void poison_scratch_page(struct drm_i915_gem_object *scratch)

{

	struct sgt_iter sgt;

	struct page *page;

	void *vaddr = __px_vaddr(scratch);

	u8 val;



	val = 0;

	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))

		val = POISON_FREE;



	for_each_sgt_page(page, sgt, scratch->mm.pages) {

		void *vaddr;



		vaddr = kmap(page);

		memset(vaddr, val, PAGE_SIZE);

		kunmap(page);

	}

	memset(vaddr, val, scratch->base.size);

}



int setup_scratch_page(struct i915_address_space *vm)
@@ -225,7 +221,7 @@ int setup_scratch_page(struct i915_address_space *vm) 
		if (IS_ERR(obj))

			goto skip;



		if (pin_pt_dma(vm, obj))

		if (map_pt_dma(vm, obj))

			goto skip_obj;



		/* We need a single contiguous page for our scratch */