Commit a9abea97 authored by Chris Wilson's avatar Chris Wilson
Browse files

drm/i915/gtt: Reorder gen8 ppgtt free/clear/alloc 



In preparation for refactoring the free/clear/alloc, first move the code
around so that we can avoid forward declarations in the next set of
patches.

Signed-off-by: default avatarChris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: default avatarMika Kuoppala <mika.kuoppala@linux.intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20190712094327.24437-2-chris@chris-wilson.co.uk
parent 57a7e305

drivers/gpu/drm/i915/i915_gem_gtt.c

+337 −336
Original line number Diff line number Diff line
@@ -830,6 +830,104 @@ static void mark_tlbs_dirty(struct i915_ppgtt *ppgtt) 
	ppgtt->pd_dirty_engines = ALL_ENGINES;

}



static int gen8_ppgtt_notify_vgt(struct i915_ppgtt *ppgtt, bool create)

{

	struct i915_address_space *vm = &ppgtt->vm;

	struct drm_i915_private *dev_priv = vm->i915;

	enum vgt_g2v_type msg;

	int i;



	if (create)

		atomic_inc(px_used(ppgtt->pd)); /* never remove */

	else

		atomic_dec(px_used(ppgtt->pd));



	if (i915_vm_is_4lvl(vm)) {

		const u64 daddr = px_dma(ppgtt->pd);



		I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));

		I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr));



		msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE :

				VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY);

	} else {

		for (i = 0; i < GEN8_3LVL_PDPES; i++) {

			const u64 daddr = i915_page_dir_dma_addr(ppgtt, i);



			I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr));

			I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr));

		}



		msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE :

				VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY);

	}



	I915_WRITE(vgtif_reg(g2v_notify), msg);



	return 0;

}



static void gen8_free_page_tables(struct i915_address_space *vm,

				  struct i915_page_directory *pd)

{

	int i;



	for (i = 0; i < I915_PDES; i++) {

		if (pd->entry[i] != &vm->scratch_pt)

			free_pd(vm, pd->entry[i]);

	}

}



static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm,

				    struct i915_page_directory *pdp)

{

	const unsigned int pdpes = i915_pdpes_per_pdp(vm);

	int i;



	for (i = 0; i < pdpes; i++) {

		if (pdp->entry[i] == &vm->scratch_pd)

			continue;



		gen8_free_page_tables(vm, pdp->entry[i]);

		free_pd(vm, pdp->entry[i]);

	}



	free_px(vm, pdp);

}



static void gen8_ppgtt_cleanup_4lvl(struct i915_ppgtt *ppgtt)

{

	struct i915_page_directory * const pml4 = ppgtt->pd;

	int i;



	for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) {

		struct i915_page_directory *pdp = i915_pdp_entry(pml4, i);



		if (px_base(pdp) == &ppgtt->vm.scratch_pdp)

			continue;



		gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, pdp);

	}



	free_px(&ppgtt->vm, pml4);

}



static void gen8_ppgtt_cleanup(struct i915_address_space *vm)

{

	struct drm_i915_private *i915 = vm->i915;

	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);



	if (intel_vgpu_active(i915))

		gen8_ppgtt_notify_vgt(ppgtt, false);



	if (i915_vm_is_4lvl(vm))

		gen8_ppgtt_cleanup_4lvl(ppgtt);

	else

		gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, ppgtt->pd);



	free_scratch(vm);

}



/* Removes entries from a single page table, releasing it if it's empty.

 * Caller can use the return value to update higher-level entries.

 */
@@ -916,124 +1014,294 @@ static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm, 
	}

}



static inline struct sgt_dma {

	struct scatterlist *sg;

	dma_addr_t dma, max;

} sgt_dma(struct i915_vma *vma) {

	struct scatterlist *sg = vma->pages->sgl;

	dma_addr_t addr = sg_dma_address(sg);

	return (struct sgt_dma) { sg, addr, addr + sg->length };

}



struct gen8_insert_pte {

	u16 pml4e;

	u16 pdpe;

	u16 pde;

	u16 pte;

};



static __always_inline struct gen8_insert_pte gen8_insert_pte(u64 start)

{

	return (struct gen8_insert_pte) {

		 gen8_pml4e_index(start),

		 gen8_pdpe_index(start),

		 gen8_pde_index(start),

		 gen8_pte_index(start),

	};

}



static __always_inline bool

gen8_ppgtt_insert_pte_entries(struct i915_ppgtt *ppgtt,

			      struct i915_page_directory *pdp,

			      struct sgt_dma *iter,

			      struct gen8_insert_pte *idx,

			      enum i915_cache_level cache_level,

			      u32 flags)

static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,

			       struct i915_page_directory *pd,

			       u64 start, u64 length)

{

	struct i915_page_directory *pd;

	const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level, flags);

	gen8_pte_t *vaddr;

	bool ret;

	struct i915_page_table *pt, *alloc = NULL;

	u64 from = start;

	unsigned int pde;

	int ret = 0;



	GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));

	pd = i915_pd_entry(pdp, idx->pdpe);

	vaddr = kmap_atomic_px(i915_pt_entry(pd, idx->pde));

	do {

		vaddr[idx->pte] = pte_encode | iter->dma;

	spin_lock(&pd->lock);

	gen8_for_each_pde(pt, pd, start, length, pde) {

		const int count = gen8_pte_count(start, length);



		iter->dma += I915_GTT_PAGE_SIZE;

		if (iter->dma >= iter->max) {

			iter->sg = __sg_next(iter->sg);

			if (!iter->sg) {

				ret = false;

				break;

			}

		if (px_base(pt) == &vm->scratch_pt) {

			spin_unlock(&pd->lock);



			iter->dma = sg_dma_address(iter->sg);

			iter->max = iter->dma + iter->sg->length;

			pt = fetch_and_zero(&alloc);

			if (!pt)

				pt = alloc_pt(vm);

			if (IS_ERR(pt)) {

				ret = PTR_ERR(pt);

				goto unwind;

			}



		if (++idx->pte == GEN8_PTES) {

			idx->pte = 0;



			if (++idx->pde == I915_PDES) {

				idx->pde = 0;

			if (count < GEN8_PTES || intel_vgpu_active(vm->i915))

				fill_px(pt, vm->scratch_pte);



				/* Limited by sg length for 3lvl */

				if (++idx->pdpe == GEN8_PML4ES_PER_PML4) {

					idx->pdpe = 0;

					ret = true;

					break;

			spin_lock(&pd->lock);

			if (pd->entry[pde] == &vm->scratch_pt) {

				set_pd_entry(pd, pde, pt);

			} else {

				alloc = pt;

				pt = pd->entry[pde];

			}



				GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));

				pd = pdp->entry[idx->pdpe];

		}



			kunmap_atomic(vaddr);

			vaddr = kmap_atomic_px(i915_pt_entry(pd, idx->pde));

		atomic_add(count, &pt->used);

	}

	} while (1);

	kunmap_atomic(vaddr);

	spin_unlock(&pd->lock);

	goto out;



unwind:

	gen8_ppgtt_clear_pd(vm, pd, from, start - from);

out:

	if (alloc)

		free_px(vm, alloc);

	return ret;

}



static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm,

				   struct i915_vma *vma,

				   enum i915_cache_level cache_level,

				   u32 flags)

static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,

				struct i915_page_directory *pdp,

				u64 start, u64 length)

{

	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);

	struct sgt_dma iter = sgt_dma(vma);

	struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);

	struct i915_page_directory *pd, *alloc = NULL;

	u64 from = start;

	unsigned int pdpe;

	int ret = 0;



	gen8_ppgtt_insert_pte_entries(ppgtt, ppgtt->pd, &iter, &idx,

				      cache_level, flags);

	spin_lock(&pdp->lock);

	gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {

		if (px_base(pd) == &vm->scratch_pd) {

			spin_unlock(&pdp->lock);



	vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;

			pd = fetch_and_zero(&alloc);

			if (!pd)

				pd = alloc_pd(vm);

			if (IS_ERR(pd)) {

				ret = PTR_ERR(pd);

				goto unwind;

			}



static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,

					   struct i915_page_directory *pml4,

					   struct sgt_dma *iter,

					   enum i915_cache_level cache_level,

					   u32 flags)

{

	const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level, flags);

	u64 start = vma->node.start;

	dma_addr_t rem = iter->sg->length;

			init_pd(pd, &vm->scratch_pt);



	do {

		struct gen8_insert_pte idx = gen8_insert_pte(start);

		struct i915_page_directory *pdp =

			i915_pdp_entry(pml4, idx.pml4e);

		struct i915_page_directory *pd = i915_pd_entry(pdp, idx.pdpe);

		unsigned int page_size;

		bool maybe_64K = false;

		gen8_pte_t encode = pte_encode;

		gen8_pte_t *vaddr;

		u16 index, max;

			spin_lock(&pdp->lock);

			if (pdp->entry[pdpe] == &vm->scratch_pd) {

				set_pd_entry(pdp, pdpe, pd);

			} else {

				alloc = pd;

				pd = pdp->entry[pdpe];

			}

		}

		atomic_inc(px_used(pd));

		spin_unlock(&pdp->lock);



		ret = gen8_ppgtt_alloc_pd(vm, pd, start, length);

		if (unlikely(ret))

			goto unwind_pd;



		spin_lock(&pdp->lock);

		atomic_dec(px_used(pd));

	}

	spin_unlock(&pdp->lock);

	goto out;



unwind_pd:

	if (release_pd_entry(pdp, pdpe, &pd->pt, &vm->scratch_pd))

		free_px(vm, pd);

unwind:

	gen8_ppgtt_clear_pdp(vm, pdp, from, start - from);

out:

	if (alloc)

		free_px(vm, alloc);

	return ret;

}



static int gen8_ppgtt_alloc_3lvl(struct i915_address_space *vm,

				 u64 start, u64 length)

{

	return gen8_ppgtt_alloc_pdp(vm,

				    i915_vm_to_ppgtt(vm)->pd, start, length);

}



static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,

				 u64 start, u64 length)

{

	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);

	struct i915_page_directory * const pml4 = ppgtt->pd;

	struct i915_page_directory *pdp, *alloc = NULL;

	u64 from = start;

	int ret = 0;

	u32 pml4e;



	spin_lock(&pml4->lock);

	gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {

		if (px_base(pdp) == &vm->scratch_pdp) {

			spin_unlock(&pml4->lock);



			pdp = fetch_and_zero(&alloc);

			if (!pdp)

				pdp = alloc_pd(vm);

			if (IS_ERR(pdp)) {

				ret = PTR_ERR(pdp);

				goto unwind;

			}



			init_pd(pdp, &vm->scratch_pd);



			spin_lock(&pml4->lock);

			if (pml4->entry[pml4e] == &vm->scratch_pdp) {

				set_pd_entry(pml4, pml4e, pdp);

			} else {

				alloc = pdp;

				pdp = pml4->entry[pml4e];

			}

		}

		atomic_inc(px_used(pdp));

		spin_unlock(&pml4->lock);



		ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length);

		if (unlikely(ret))

			goto unwind_pdp;



		spin_lock(&pml4->lock);

		atomic_dec(px_used(pdp));

	}

	spin_unlock(&pml4->lock);

	goto out;



unwind_pdp:

	if (release_pd_entry(pml4, pml4e, &pdp->pt, &vm->scratch_pdp))

		free_px(vm, pdp);

unwind:

	gen8_ppgtt_clear_4lvl(vm, from, start - from);

out:

	if (alloc)

		free_px(vm, alloc);

	return ret;

}



static inline struct sgt_dma {

	struct scatterlist *sg;

	dma_addr_t dma, max;

} sgt_dma(struct i915_vma *vma) {

	struct scatterlist *sg = vma->pages->sgl;

	dma_addr_t addr = sg_dma_address(sg);

	return (struct sgt_dma) { sg, addr, addr + sg->length };

}



struct gen8_insert_pte {

	u16 pml4e;

	u16 pdpe;

	u16 pde;

	u16 pte;

};



static __always_inline struct gen8_insert_pte gen8_insert_pte(u64 start)

{

	return (struct gen8_insert_pte) {

		 gen8_pml4e_index(start),

		 gen8_pdpe_index(start),

		 gen8_pde_index(start),

		 gen8_pte_index(start),

	};

}



static __always_inline bool

gen8_ppgtt_insert_pte_entries(struct i915_ppgtt *ppgtt,

			      struct i915_page_directory *pdp,

			      struct sgt_dma *iter,

			      struct gen8_insert_pte *idx,

			      enum i915_cache_level cache_level,

			      u32 flags)

{

	struct i915_page_directory *pd;

	const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level, flags);

	gen8_pte_t *vaddr;

	bool ret;



	GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));

	pd = i915_pd_entry(pdp, idx->pdpe);

	vaddr = kmap_atomic_px(i915_pt_entry(pd, idx->pde));

	do {

		vaddr[idx->pte] = pte_encode | iter->dma;



		iter->dma += I915_GTT_PAGE_SIZE;

		if (iter->dma >= iter->max) {

			iter->sg = __sg_next(iter->sg);

			if (!iter->sg) {

				ret = false;

				break;

			}



			iter->dma = sg_dma_address(iter->sg);

			iter->max = iter->dma + iter->sg->length;

		}



		if (++idx->pte == GEN8_PTES) {

			idx->pte = 0;



			if (++idx->pde == I915_PDES) {

				idx->pde = 0;



				/* Limited by sg length for 3lvl */

				if (++idx->pdpe == GEN8_PML4ES_PER_PML4) {

					idx->pdpe = 0;

					ret = true;

					break;

				}



				GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));

				pd = pdp->entry[idx->pdpe];

			}



			kunmap_atomic(vaddr);

			vaddr = kmap_atomic_px(i915_pt_entry(pd, idx->pde));

		}

	} while (1);

	kunmap_atomic(vaddr);



	return ret;

}



static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm,

				   struct i915_vma *vma,

				   enum i915_cache_level cache_level,

				   u32 flags)

{

	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);

	struct sgt_dma iter = sgt_dma(vma);

	struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);



	gen8_ppgtt_insert_pte_entries(ppgtt, ppgtt->pd, &iter, &idx,

				      cache_level, flags);



	vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;

}



static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,

					   struct i915_page_directory *pml4,

					   struct sgt_dma *iter,

					   enum i915_cache_level cache_level,

					   u32 flags)

{

	const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level, flags);

	u64 start = vma->node.start;

	dma_addr_t rem = iter->sg->length;



	do {

		struct gen8_insert_pte idx = gen8_insert_pte(start);

		struct i915_page_directory *pdp =

			i915_pdp_entry(pml4, idx.pml4e);

		struct i915_page_directory *pd = i915_pd_entry(pdp, idx.pdpe);

		unsigned int page_size;

		bool maybe_64K = false;

		gen8_pte_t encode = pte_encode;

		gen8_pte_t *vaddr;

		u16 index, max;



		if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_2M &&

		    IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_2M) &&
@@ -1160,17 +1428,6 @@ static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm, 
	}

}



static void gen8_free_page_tables(struct i915_address_space *vm,

				  struct i915_page_directory *pd)

{

	int i;



	for (i = 0; i < I915_PDES; i++) {

		if (pd->entry[i] != &vm->scratch_pt)

			free_pd(vm, pd->entry[i]);

	}

}



static int gen8_init_scratch(struct i915_address_space *vm)

{

	int ret;
@@ -1237,262 +1494,6 @@ static int gen8_init_scratch(struct i915_address_space *vm) 
	return ret;

}



static int gen8_ppgtt_notify_vgt(struct i915_ppgtt *ppgtt, bool create)

{

	struct i915_address_space *vm = &ppgtt->vm;

	struct drm_i915_private *dev_priv = vm->i915;

	enum vgt_g2v_type msg;

	int i;



	if (create)

		atomic_inc(px_used(ppgtt->pd)); /* never remove */

	else

		atomic_dec(px_used(ppgtt->pd));



	if (i915_vm_is_4lvl(vm)) {

		const u64 daddr = px_dma(ppgtt->pd);



		I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));

		I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr));



		msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE :

				VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY);

	} else {

		for (i = 0; i < GEN8_3LVL_PDPES; i++) {

			const u64 daddr = i915_page_dir_dma_addr(ppgtt, i);



			I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr));

			I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr));

		}



		msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE :

				VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY);

	}



	I915_WRITE(vgtif_reg(g2v_notify), msg);



	return 0;

}



static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm,

				    struct i915_page_directory *pdp)

{

	const unsigned int pdpes = i915_pdpes_per_pdp(vm);

	int i;



	for (i = 0; i < pdpes; i++) {

		if (pdp->entry[i] == &vm->scratch_pd)

			continue;



		gen8_free_page_tables(vm, pdp->entry[i]);

		free_pd(vm, pdp->entry[i]);

	}



	free_px(vm, pdp);

}



static void gen8_ppgtt_cleanup_4lvl(struct i915_ppgtt *ppgtt)

{

	struct i915_page_directory * const pml4 = ppgtt->pd;

	int i;



	for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) {

		struct i915_page_directory *pdp = i915_pdp_entry(pml4, i);



		if (px_base(pdp) == &ppgtt->vm.scratch_pdp)

			continue;



		gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, pdp);

	}



	free_px(&ppgtt->vm, pml4);

}



static void gen8_ppgtt_cleanup(struct i915_address_space *vm)

{

	struct drm_i915_private *i915 = vm->i915;

	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);



	if (intel_vgpu_active(i915))

		gen8_ppgtt_notify_vgt(ppgtt, false);



	if (i915_vm_is_4lvl(vm))

		gen8_ppgtt_cleanup_4lvl(ppgtt);

	else

		gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, ppgtt->pd);



	free_scratch(vm);

}



static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,

			       struct i915_page_directory *pd,

			       u64 start, u64 length)

{

	struct i915_page_table *pt, *alloc = NULL;

	u64 from = start;

	unsigned int pde;

	int ret = 0;



	spin_lock(&pd->lock);

	gen8_for_each_pde(pt, pd, start, length, pde) {

		const int count = gen8_pte_count(start, length);



		if (px_base(pt) == &vm->scratch_pt) {

			spin_unlock(&pd->lock);



			pt = fetch_and_zero(&alloc);

			if (!pt)

				pt = alloc_pt(vm);

			if (IS_ERR(pt)) {

				ret = PTR_ERR(pt);

				goto unwind;

			}



			if (count < GEN8_PTES || intel_vgpu_active(vm->i915))

				fill_px(pt, vm->scratch_pte);



			spin_lock(&pd->lock);

			if (pd->entry[pde] == &vm->scratch_pt) {

				set_pd_entry(pd, pde, pt);

			} else {

				alloc = pt;

				pt = pd->entry[pde];

			}

		}



		atomic_add(count, &pt->used);

	}

	spin_unlock(&pd->lock);

	goto out;



unwind:

	gen8_ppgtt_clear_pd(vm, pd, from, start - from);

out:

	if (alloc)

		free_px(vm, alloc);

	return ret;

}



static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,

				struct i915_page_directory *pdp,

				u64 start, u64 length)

{

	struct i915_page_directory *pd, *alloc = NULL;

	u64 from = start;

	unsigned int pdpe;

	int ret = 0;



	spin_lock(&pdp->lock);

	gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {

		if (px_base(pd) == &vm->scratch_pd) {

			spin_unlock(&pdp->lock);



			pd = fetch_and_zero(&alloc);

			if (!pd)

				pd = alloc_pd(vm);

			if (IS_ERR(pd)) {

				ret = PTR_ERR(pd);

				goto unwind;

			}



			init_pd(pd, &vm->scratch_pt);



			spin_lock(&pdp->lock);

			if (pdp->entry[pdpe] == &vm->scratch_pd) {

				set_pd_entry(pdp, pdpe, pd);

			} else {

				alloc = pd;

				pd = pdp->entry[pdpe];

			}

		}

		atomic_inc(px_used(pd));

		spin_unlock(&pdp->lock);



		ret = gen8_ppgtt_alloc_pd(vm, pd, start, length);

		if (unlikely(ret))

			goto unwind_pd;



		spin_lock(&pdp->lock);

		atomic_dec(px_used(pd));

	}

	spin_unlock(&pdp->lock);

	goto out;



unwind_pd:

	if (release_pd_entry(pdp, pdpe, &pd->pt, &vm->scratch_pd))

		free_px(vm, pd);

unwind:

	gen8_ppgtt_clear_pdp(vm, pdp, from, start - from);

out:

	if (alloc)

		free_px(vm, alloc);

	return ret;

}



static int gen8_ppgtt_alloc_3lvl(struct i915_address_space *vm,

				 u64 start, u64 length)

{

	return gen8_ppgtt_alloc_pdp(vm,

				    i915_vm_to_ppgtt(vm)->pd, start, length);

}



static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,

				 u64 start, u64 length)

{

	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);

	struct i915_page_directory * const pml4 = ppgtt->pd;

	struct i915_page_directory *pdp, *alloc = NULL;

	u64 from = start;

	int ret = 0;

	u32 pml4e;



	spin_lock(&pml4->lock);

	gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {

		if (px_base(pdp) == &vm->scratch_pdp) {

			spin_unlock(&pml4->lock);



			pdp = fetch_and_zero(&alloc);

			if (!pdp)

				pdp = alloc_pd(vm);

			if (IS_ERR(pdp)) {

				ret = PTR_ERR(pdp);

				goto unwind;

			}



			init_pd(pdp, &vm->scratch_pd);



			spin_lock(&pml4->lock);

			if (pml4->entry[pml4e] == &vm->scratch_pdp) {

				set_pd_entry(pml4, pml4e, pdp);

			} else {

				alloc = pdp;

				pdp = pml4->entry[pml4e];

			}

		}

		atomic_inc(px_used(pdp));

		spin_unlock(&pml4->lock);



		ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length);

		if (unlikely(ret))

			goto unwind_pdp;



		spin_lock(&pml4->lock);

		atomic_dec(px_used(pdp));

	}

	spin_unlock(&pml4->lock);

	goto out;



unwind_pdp:

	if (release_pd_entry(pml4, pml4e, &pdp->pt, &vm->scratch_pdp))

		free_px(vm, pdp);

unwind:

	gen8_ppgtt_clear_4lvl(vm, from, start - from);

out:

	if (alloc)

		free_px(vm, alloc);

	return ret;

}



static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)

{

	struct i915_address_space *vm = &ppgtt->vm;