Commit 9bff18d1 authored by Christian König's avatar Christian König
Browse files

drm/ttm: use per BO cleanup workers 



Instead of a single worker going over the list of delete BOs in regular
intervals use a per BO worker which blocks for the resv object and
locking of the BO.

This not only simplifies the handling massively, but also results in
much better response time when cleaning up buffers.

Signed-off-by: default avatarChristian König <christian.koenig@amd.com>
Reviewed-by: default avatarFelix Kuehling <Felix.Kuehling@amd.com>
Reviewed-by: default avatarArunpravin Paneer Selvam <Arunpravin.PaneerSelvam@amd.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20221125102137.1801-3-christian.koenig@amd.com
parent cd3a8a59

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

+1 −1
Original line number Diff line number Diff line
@@ -3984,7 +3984,7 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) 
	amdgpu_fence_driver_hw_fini(adev);



	if (adev->mman.initialized)

		flush_delayed_work(&adev->mman.bdev.wq);

		drain_workqueue(adev->mman.bdev.wq);



	if (adev->pm_sysfs_en)

		amdgpu_pm_sysfs_fini(adev);

drivers/gpu/drm/i915/i915_gem.c

+1 −1
Original line number Diff line number Diff line
@@ -1099,7 +1099,7 @@ void i915_gem_drain_freed_objects(struct drm_i915_private *i915) 
{

	while (atomic_read(&i915->mm.free_count)) {

		flush_work(&i915->mm.free_work);

		flush_delayed_work(&i915->bdev.wq);

		drain_workqueue(i915->bdev.wq);

		rcu_barrier();

	}

}

drivers/gpu/drm/i915/intel_region_ttm.c

+1 −1
Original line number Diff line number Diff line
@@ -132,7 +132,7 @@ int intel_region_ttm_fini(struct intel_memory_region *mem) 
			break;



		msleep(20);

		flush_delayed_work(&mem->i915->bdev.wq);

		drain_workqueue(mem->i915->bdev.wq);

	}



	/* If we leaked objects, Don't free the region causing use after free */

drivers/gpu/drm/ttm/ttm_bo.c

+40 −72
Original line number Diff line number Diff line
@@ -280,14 +280,13 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, 
		ret = 0;

	}



	if (ret || unlikely(list_empty(&bo->ddestroy))) {

	if (ret) {

		if (unlock_resv)

			dma_resv_unlock(bo->base.resv);

		spin_unlock(&bo->bdev->lru_lock);

		return ret;

	}



	list_del_init(&bo->ddestroy);

	spin_unlock(&bo->bdev->lru_lock);

	ttm_bo_cleanup_memtype_use(bo);
@@ -300,47 +299,21 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, 
}



/*

 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all

 * encountered buffers.

 * Block for the dma_resv object to become idle, lock the buffer and clean up

 * the resource and tt object.

 */

bool ttm_bo_delayed_delete(struct ttm_device *bdev, bool remove_all)

static void ttm_bo_delayed_delete(struct work_struct *work)

{

	struct list_head removed;

	bool empty;



	INIT_LIST_HEAD(&removed);



	spin_lock(&bdev->lru_lock);

	while (!list_empty(&bdev->ddestroy)) {

	struct ttm_buffer_object *bo;



		bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,

				      ddestroy);

		list_move_tail(&bo->ddestroy, &removed);

		if (!ttm_bo_get_unless_zero(bo))

			continue;

	bo = container_of(work, typeof(*bo), delayed_delete);



		if (remove_all || bo->base.resv != &bo->base._resv) {

			spin_unlock(&bdev->lru_lock);

	dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, false,

			      MAX_SCHEDULE_TIMEOUT);

	dma_resv_lock(bo->base.resv, NULL);



			spin_lock(&bdev->lru_lock);

			ttm_bo_cleanup_refs(bo, false, !remove_all, true);



		} else if (dma_resv_trylock(bo->base.resv)) {

			ttm_bo_cleanup_refs(bo, false, !remove_all, true);

		} else {

			spin_unlock(&bdev->lru_lock);

		}



	ttm_bo_cleanup_memtype_use(bo);

	dma_resv_unlock(bo->base.resv);

	ttm_bo_put(bo);

		spin_lock(&bdev->lru_lock);

	}

	list_splice_tail(&removed, &bdev->ddestroy);

	empty = list_empty(&bdev->ddestroy);

	spin_unlock(&bdev->lru_lock);



	return empty;

}



static void ttm_bo_release(struct kref *kref)
@@ -369,9 +342,9 @@ static void ttm_bo_release(struct kref *kref) 


		drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node);

		ttm_mem_io_free(bdev, bo->resource);

	}



	if (!dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP) ||

		if (!dma_resv_test_signaled(bo->base.resv,

					    DMA_RESV_USAGE_BOOKKEEP) ||

		    !dma_resv_trylock(bo->base.resv)) {

			/* The BO is not idle, resurrect it for delayed destroy */

			ttm_bo_flush_all_fences(bo);
@@ -393,20 +366,16 @@ static void ttm_bo_release(struct kref *kref) 
			}



			kref_init(&bo->kref);

		list_add_tail(&bo->ddestroy, &bdev->ddestroy);

			spin_unlock(&bo->bdev->lru_lock);



		schedule_delayed_work(&bdev->wq,

				      ((HZ / 100) < 1) ? 1 : HZ / 100);

			INIT_WORK(&bo->delayed_delete, ttm_bo_delayed_delete);

			queue_work(bdev->wq, &bo->delayed_delete);

			return;

		}



	spin_lock(&bo->bdev->lru_lock);

	list_del(&bo->ddestroy);

	spin_unlock(&bo->bdev->lru_lock);



		ttm_bo_cleanup_memtype_use(bo);

		dma_resv_unlock(bo->base.resv);

	}



	atomic_dec(&ttm_glob.bo_count);

	bo->destroy(bo);
@@ -946,7 +915,6 @@ int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo, 
	int ret;



	kref_init(&bo->kref);

	INIT_LIST_HEAD(&bo->ddestroy);

	bo->bdev = bdev;

	bo->type = type;

	bo->page_alignment = alignment;

drivers/gpu/drm/ttm/ttm_bo_util.c

+0 −1
Original line number Diff line number Diff line
@@ -230,7 +230,6 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, 
	 */



	atomic_inc(&ttm_glob.bo_count);

	INIT_LIST_HEAD(&fbo->base.ddestroy);

	drm_vma_node_reset(&fbo->base.base.vma_node);



	kref_init(&fbo->base.kref);