Commit d8ac30fd authored by Thomas Hellström's avatar Thomas Hellström
Browse files

drm/i915/ttm: Reorganize the ttm move code somewhat 



In order to make the code a bit more readable and to facilitate
async memcpy moves, reorganize the move code a little. Determine
at an early stage whether to copy or to clear.

v2:
- Don't set up the memcpy iterators unless we are actually going to memcpy.

Signed-off-by: default avatarThomas Hellström <thomas.hellstrom@linux.intel.com>
Reviewed-by: default avatarMatthew Auld <matthew.auld@intel.com>
Link: https://lore.kernel.org/r/20210813144331.372957-2-thomas.hellstrom@linux.intel.com
Link: https://patchwork.freedesktop.org/patch/msgid/20210813144331.372957-2-thomas.hellstrom@linux.intel.com
parent ae4b0eac

drivers/gpu/drm/i915/gem/i915_gem_ttm.c

+44 −33
Original line number Diff line number Diff line
@@ -431,6 +431,7 @@ i915_ttm_resource_get_st(struct drm_i915_gem_object *obj, 
}



static int i915_ttm_accel_move(struct ttm_buffer_object *bo,

			       bool clear,

			       struct ttm_resource *dst_mem,

			       struct sg_table *dst_st)

{
@@ -449,13 +450,10 @@ static int i915_ttm_accel_move(struct ttm_buffer_object *bo, 
		return -EINVAL;



	dst_level = i915_ttm_cache_level(i915, dst_mem, ttm);

	if (!ttm || !ttm_tt_is_populated(ttm)) {

	if (clear) {

		if (bo->type == ttm_bo_type_kernel)

			return -EINVAL;



		if (ttm && !(ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC))

			return 0;



		intel_engine_pm_get(i915->gt.migrate.context->engine);

		ret = intel_context_migrate_clear(i915->gt.migrate.context, NULL,

						  dst_st->sgl, dst_level,
@@ -489,27 +487,54 @@ static int i915_ttm_accel_move(struct ttm_buffer_object *bo, 
	return ret;

}



static int i915_ttm_move(struct ttm_buffer_object *bo, bool evict,

			 struct ttm_operation_ctx *ctx,

static void __i915_ttm_move(struct ttm_buffer_object *bo, bool clear,

			    struct ttm_resource *dst_mem,

			 struct ttm_place *hop)

			    struct sg_table *dst_st)

{

	int ret;



	ret = i915_ttm_accel_move(bo, clear, dst_mem, dst_st);

	if (ret) {

		struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);

	struct ttm_resource_manager *dst_man =

		ttm_manager_type(bo->bdev, dst_mem->mem_type);

		struct intel_memory_region *dst_reg, *src_reg;

		union {

			struct ttm_kmap_iter_tt tt;

			struct ttm_kmap_iter_iomap io;

		} _dst_iter, _src_iter;

		struct ttm_kmap_iter *dst_iter, *src_iter;

	struct sg_table *dst_st;

	int ret;



		dst_reg = i915_ttm_region(bo->bdev, dst_mem->mem_type);

		src_reg = i915_ttm_region(bo->bdev, bo->resource->mem_type);

		GEM_BUG_ON(!dst_reg || !src_reg);



		dst_iter = !cpu_maps_iomem(dst_mem) ?

			ttm_kmap_iter_tt_init(&_dst_iter.tt, bo->ttm) :

			ttm_kmap_iter_iomap_init(&_dst_iter.io, &dst_reg->iomap,

						 dst_st, dst_reg->region.start);



		src_iter = !cpu_maps_iomem(bo->resource) ?

			ttm_kmap_iter_tt_init(&_src_iter.tt, bo->ttm) :

			ttm_kmap_iter_iomap_init(&_src_iter.io, &src_reg->iomap,

						 obj->ttm.cached_io_st,

						 src_reg->region.start);



		ttm_move_memcpy(bo, dst_mem->num_pages, dst_iter, src_iter);

	}

}



static int i915_ttm_move(struct ttm_buffer_object *bo, bool evict,

			 struct ttm_operation_ctx *ctx,

			 struct ttm_resource *dst_mem,

			 struct ttm_place *hop)

{

	struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);

	struct ttm_resource_manager *dst_man =

		ttm_manager_type(bo->bdev, dst_mem->mem_type);

	struct ttm_tt *ttm = bo->ttm;

	struct sg_table *dst_st;

	bool clear;

	int ret;



	/* Sync for now. We could do the actual copy async. */

	ret = ttm_bo_wait_ctx(bo, ctx);

	if (ret)
@@ -526,9 +551,8 @@ static int i915_ttm_move(struct ttm_buffer_object *bo, bool evict, 
	}



	/* Populate ttm with pages if needed. Typically system memory. */

	if (bo->ttm && (dst_man->use_tt ||

			(bo->ttm->page_flags & TTM_PAGE_FLAG_SWAPPED))) {

		ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx);

	if (ttm && (dst_man->use_tt || (ttm->page_flags & TTM_PAGE_FLAG_SWAPPED))) {

		ret = ttm_tt_populate(bo->bdev, ttm, ctx);

		if (ret)

			return ret;

	}
@@ -537,23 +561,10 @@ static int i915_ttm_move(struct ttm_buffer_object *bo, bool evict, 
	if (IS_ERR(dst_st))

		return PTR_ERR(dst_st);



	ret = i915_ttm_accel_move(bo, dst_mem, dst_st);

	if (ret) {

		/* If we start mapping GGTT, we can no longer use man::use_tt here. */

		dst_iter = !cpu_maps_iomem(dst_mem) ?

			ttm_kmap_iter_tt_init(&_dst_iter.tt, bo->ttm) :

			ttm_kmap_iter_iomap_init(&_dst_iter.io, &dst_reg->iomap,

						 dst_st, dst_reg->region.start);

	clear = !cpu_maps_iomem(bo->resource) && (!ttm || !ttm_tt_is_populated(ttm));

	if (!(clear && ttm && !(ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC)))

		__i915_ttm_move(bo, clear, dst_mem, dst_st);



		src_iter = !cpu_maps_iomem(bo->resource) ?

			ttm_kmap_iter_tt_init(&_src_iter.tt, bo->ttm) :

			ttm_kmap_iter_iomap_init(&_src_iter.io, &src_reg->iomap,

						 obj->ttm.cached_io_st,

						 src_reg->region.start);



		ttm_move_memcpy(bo, dst_mem->num_pages, dst_iter, src_iter);

	}

	/* Below dst_mem becomes bo->resource. */

	ttm_bo_move_sync_cleanup(bo, dst_mem);

	i915_ttm_adjust_domains_after_move(obj);

	i915_ttm_free_cached_io_st(obj);