Commit 6a9b0289 authored by Christian König's avatar Christian König
Browse files

drm/ttm: move the LRU into resource handling v4 



This way we finally fix the problem that new resource are
not immediately evict-able after allocation.

That has caused numerous problems including OOM on GDS handling
and not being able to use TTM as general resource manager.

v2: stop assuming in ttm_resource_fini that res->bo is still valid.
v3: cleanup kerneldoc, add more lockdep annotation
v4: consistently use res->num_pages

Signed-off-by: default avatarChristian König <christian.koenig@amd.com>
Tested-by: default avatarBas Nieuwenhuizen <bas@basnieuwenhuizen.nl>
Reviewed-by: default avatarDaniel Vetter <daniel.vetter@ffwll.ch>
Link: https://patchwork.freedesktop.org/patch/msgid/20220321132601.2161-1-christian.koenig@amd.com
parent e795df5b

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c

+3 −5
Original line number Diff line number Diff line
@@ -683,12 +683,12 @@ void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev, 


	if (vm->bulk_moveable) {

		spin_lock(&adev->mman.bdev.lru_lock);

		ttm_bo_bulk_move_lru_tail(&vm->lru_bulk_move);

		ttm_lru_bulk_move_tail(&vm->lru_bulk_move);

		spin_unlock(&adev->mman.bdev.lru_lock);

		return;

	}



	memset(&vm->lru_bulk_move, 0, sizeof(vm->lru_bulk_move));

	ttm_lru_bulk_move_init(&vm->lru_bulk_move);



	spin_lock(&adev->mman.bdev.lru_lock);

	list_for_each_entry(bo_base, &vm->idle, vm_status) {
@@ -698,11 +698,9 @@ void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev, 
		if (!bo->parent)

			continue;



		ttm_bo_move_to_lru_tail(&bo->tbo, bo->tbo.resource,

					&vm->lru_bulk_move);

		ttm_bo_move_to_lru_tail(&bo->tbo, &vm->lru_bulk_move);

		if (shadow)

			ttm_bo_move_to_lru_tail(&shadow->tbo,

						shadow->tbo.resource,

						&vm->lru_bulk_move);

	}

	spin_unlock(&adev->mman.bdev.lru_lock);

drivers/gpu/drm/i915/gem/i915_gem_ttm.c

+1 −1
Original line number Diff line number Diff line
@@ -849,7 +849,7 @@ void i915_ttm_adjust_lru(struct drm_i915_gem_object *obj) 
			bo->priority = I915_TTM_PRIO_NO_PAGES;

	}



	ttm_bo_move_to_lru_tail(bo, bo->resource, NULL);

	ttm_bo_move_to_lru_tail(bo, NULL);

	spin_unlock(&bo->bdev->lru_lock);

}

drivers/gpu/drm/ttm/ttm_bo.c

+8 −107
Original line number Diff line number Diff line
@@ -69,108 +69,16 @@ static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, 
	}

}



static inline void ttm_bo_move_to_pinned(struct ttm_buffer_object *bo)

{

	struct ttm_device *bdev = bo->bdev;



	list_move_tail(&bo->lru, &bdev->pinned);



	if (bdev->funcs->del_from_lru_notify)

		bdev->funcs->del_from_lru_notify(bo);

}



static inline void ttm_bo_del_from_lru(struct ttm_buffer_object *bo)

{

	struct ttm_device *bdev = bo->bdev;



	list_del_init(&bo->lru);



	if (bdev->funcs->del_from_lru_notify)

		bdev->funcs->del_from_lru_notify(bo);

}



static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos,

				     struct ttm_buffer_object *bo)

{

	if (!pos->first)

		pos->first = bo;

	pos->last = bo;

}



void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo,

			     struct ttm_resource *mem,

			     struct ttm_lru_bulk_move *bulk)

{

	struct ttm_device *bdev = bo->bdev;

	struct ttm_resource_manager *man;



	if (!bo->deleted)

	dma_resv_assert_held(bo->base.resv);



	if (bo->pin_count) {

		ttm_bo_move_to_pinned(bo);

		return;

	}



	if (!mem)

		return;



	man = ttm_manager_type(bdev, mem->mem_type);

	list_move_tail(&bo->lru, &man->lru[bo->priority]);



	if (bdev->funcs->del_from_lru_notify)

		bdev->funcs->del_from_lru_notify(bo);



	if (bulk && !bo->pin_count) {

		switch (bo->resource->mem_type) {

		case TTM_PL_TT:

			ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo);

			break;



		case TTM_PL_VRAM:

			ttm_bo_bulk_move_set_pos(&bulk->vram[bo->priority], bo);

			break;

		}

	}

	if (bo->resource)

		ttm_resource_move_to_lru_tail(bo->resource, bulk);

}

EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);



void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk)

{

	unsigned i;



	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {

		struct ttm_lru_bulk_move_pos *pos = &bulk->tt[i];

		struct ttm_resource_manager *man;



		if (!pos->first)

			continue;



		dma_resv_assert_held(pos->first->base.resv);

		dma_resv_assert_held(pos->last->base.resv);



		man = ttm_manager_type(pos->first->bdev, TTM_PL_TT);

		list_bulk_move_tail(&man->lru[i], &pos->first->lru,

				    &pos->last->lru);

	}



	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {

		struct ttm_lru_bulk_move_pos *pos = &bulk->vram[i];

		struct ttm_resource_manager *man;



		if (!pos->first)

			continue;



		dma_resv_assert_held(pos->first->base.resv);

		dma_resv_assert_held(pos->last->base.resv);



		man = ttm_manager_type(pos->first->bdev, TTM_PL_VRAM);

		list_bulk_move_tail(&man->lru[i], &pos->first->lru,

				    &pos->last->lru);

	}

}

EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail);



static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,

				  struct ttm_resource *mem, bool evict,

				  struct ttm_operation_ctx *ctx,
@@ -344,7 +252,6 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, 
		return ret;

	}



	ttm_bo_move_to_pinned(bo);

	list_del_init(&bo->ddestroy);

	spin_unlock(&bo->bdev->lru_lock);

	ttm_bo_cleanup_memtype_use(bo);
@@ -445,7 +352,7 @@ static void ttm_bo_release(struct kref *kref) 
		 */

		if (bo->pin_count) {

			bo->pin_count = 0;

			ttm_bo_move_to_lru_tail(bo, bo->resource, NULL);

			ttm_resource_move_to_lru_tail(bo->resource, NULL);

		}



		kref_init(&bo->kref);
@@ -458,7 +365,6 @@ static void ttm_bo_release(struct kref *kref) 
	}



	spin_lock(&bo->bdev->lru_lock);

	ttm_bo_del_from_lru(bo);

	list_del(&bo->ddestroy);

	spin_unlock(&bo->bdev->lru_lock);
@@ -673,15 +579,17 @@ int ttm_mem_evict_first(struct ttm_device *bdev, 
			struct ww_acquire_ctx *ticket)

{

	struct ttm_buffer_object *bo = NULL, *busy_bo = NULL;

	struct ttm_resource *res;

	bool locked = false;

	unsigned i;

	int ret;



	spin_lock(&bdev->lru_lock);

	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {

		list_for_each_entry(bo, &man->lru[i], lru) {

		list_for_each_entry(res, &man->lru[i], lru) {

			bool busy;



			bo = res->bo;

			if (!ttm_bo_evict_swapout_allowable(bo, ctx, place,

							    &locked, &busy)) {

				if (busy && !busy_bo && ticket !=
@@ -699,7 +607,7 @@ int ttm_mem_evict_first(struct ttm_device *bdev, 
		}



		/* If the inner loop terminated early, we have our candidate */

		if (&bo->lru != &man->lru[i])

		if (&res->lru != &man->lru[i])

			break;



		bo = NULL;
@@ -875,9 +783,6 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, 
	}



error:

	if (bo->resource->mem_type == TTM_PL_SYSTEM && !bo->pin_count)

		ttm_bo_move_to_lru_tail_unlocked(bo);



	return ret;

}

EXPORT_SYMBOL(ttm_bo_mem_space);
@@ -971,7 +876,6 @@ int ttm_bo_init_reserved(struct ttm_device *bdev, 
	bo->destroy = destroy ? destroy : ttm_bo_default_destroy;



	kref_init(&bo->kref);

	INIT_LIST_HEAD(&bo->lru);

	INIT_LIST_HEAD(&bo->ddestroy);

	bo->bdev = bdev;

	bo->type = type;
@@ -1021,8 +925,6 @@ int ttm_bo_init_reserved(struct ttm_device *bdev, 
		return ret;

	}



	ttm_bo_move_to_lru_tail_unlocked(bo);



	return ret;

}

EXPORT_SYMBOL(ttm_bo_init_reserved);
@@ -1123,7 +1025,6 @@ int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, 
		return ret == -EBUSY ? -ENOSPC : ret;

	}



	ttm_bo_move_to_pinned(bo);

	/* TODO: Cleanup the locking */

	spin_unlock(&bo->bdev->lru_lock);

drivers/gpu/drm/ttm/ttm_bo_util.c

+0 −1
Original line number Diff line number Diff line
@@ -231,7 +231,6 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, 


	atomic_inc(&ttm_glob.bo_count);

	INIT_LIST_HEAD(&fbo->base.ddestroy);

	INIT_LIST_HEAD(&fbo->base.lru);

	fbo->base.moving = NULL;

	drm_vma_node_reset(&fbo->base.base.vma_node);

drivers/gpu/drm/ttm/ttm_device.c

+32 −32
Original line number Diff line number Diff line
@@ -144,6 +144,7 @@ int ttm_device_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx, 
{

	struct ttm_resource_manager *man;

	struct ttm_buffer_object *bo;

	struct ttm_resource *res;

	unsigned i, j;

	int ret;
@@ -154,8 +155,11 @@ int ttm_device_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx, 
			continue;



		for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {

			list_for_each_entry(bo, &man->lru[j], lru) {

				uint32_t num_pages = PFN_UP(bo->base.size);

			list_for_each_entry(res, &man->lru[j], lru) {

				uint32_t num_pages;



				bo = res->bo;

				num_pages = PFN_UP(bo->base.size);



				ret = ttm_bo_swapout(bo, ctx, gfp_flags);

				/* ttm_bo_swapout has dropped the lru_lock */
@@ -259,18 +263,20 @@ void ttm_device_fini(struct ttm_device *bdev) 
}

EXPORT_SYMBOL(ttm_device_fini);



void ttm_device_clear_dma_mappings(struct ttm_device *bdev)

static void ttm_device_clear_lru_dma_mappings(struct ttm_device *bdev,

					      struct list_head *list)

{

	struct ttm_resource_manager *man;

	struct ttm_buffer_object *bo;

	unsigned int i, j;

	struct ttm_resource *res;



	spin_lock(&bdev->lru_lock);

	while (!list_empty(&bdev->pinned)) {

		bo = list_first_entry(&bdev->pinned, struct ttm_buffer_object, lru);

	while ((res = list_first_entry_or_null(list, typeof(*res), lru))) {

		struct ttm_buffer_object *bo = res->bo;



		/* Take ref against racing releases once lru_lock is unlocked */

		if (ttm_bo_get_unless_zero(bo)) {

			list_del_init(&bo->lru);

		if (!ttm_bo_get_unless_zero(bo))

			continue;



		list_del_init(&res->lru);

		spin_unlock(&bdev->lru_lock);



		if (bo->ttm)
@@ -279,29 +285,23 @@ void ttm_device_clear_dma_mappings(struct ttm_device *bdev) 
		ttm_bo_put(bo);

		spin_lock(&bdev->lru_lock);

	}

	spin_unlock(&bdev->lru_lock);

}



void ttm_device_clear_dma_mappings(struct ttm_device *bdev)

{

	struct ttm_resource_manager *man;

	unsigned int i, j;



	ttm_device_clear_lru_dma_mappings(bdev, &bdev->pinned);



	for (i = TTM_PL_SYSTEM; i < TTM_NUM_MEM_TYPES; ++i) {

		man = ttm_manager_type(bdev, i);

		if (!man || !man->use_tt)

			continue;



		for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {

			while (!list_empty(&man->lru[j])) {

				bo = list_first_entry(&man->lru[j], struct ttm_buffer_object, lru);

				if (ttm_bo_get_unless_zero(bo)) {

					list_del_init(&bo->lru);

					spin_unlock(&bdev->lru_lock);



					if (bo->ttm)

						ttm_tt_unpopulate(bo->bdev, bo->ttm);



					ttm_bo_put(bo);

					spin_lock(&bdev->lru_lock);

				}

		for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j)

			ttm_device_clear_lru_dma_mappings(bdev, &man->lru[j]);

	}

}

	}

	spin_unlock(&bdev->lru_lock);

}

EXPORT_SYMBOL(ttm_device_clear_dma_mappings);