Commit 85b144f8 authored by Maarten Lankhorst's avatar Maarten Lankhorst Committed by Dave Airlie
Browse files

drm/ttm: call ttm_bo_cleanup_refs with reservation and lru lock held, v3 



By removing the unlocking of lru and retaking it immediately, a race is
removed where the bo is taken off the swap list or the lru list between
the unlock and relock. As such the cleanup_refs code can be simplified,
it will attempt to call ttm_bo_wait non-blockingly, and if it fails
it will drop the locks and perform a blocking wait, or return an error
if no_wait_gpu was set.

The need for looping is also eliminated, since swapout and evict_mem_first
will always follow the destruction path, no new fence is allowed
to be attached. As far as I can see this may already have been the case,
but the unlocking / relocking required a complicated loop to deal with
re-reservation.

Changes since v1:
 - Simplify no_wait_gpu case by folding it in with empty ddestroy.
 - Hold a reservation while calling ttm_bo_cleanup_memtype_use again.
Changes since v2:
 - Do not remove bo from lru list while waiting

Signed-off-by: default avatarMaarten Lankhorst <maarten.lankhorst@canonical.com>
Reviewed-by: default avatarThomas Hellstrom <thellstrom@vmware.com>
Signed-off-by: default avatarDave Airlie <airlied@redhat.com>
parent 6ed9ccb4

drivers/gpu/drm/ttm/ttm_bo.c

+76 −47
Original line number Diff line number Diff line
@@ -488,12 +488,16 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 
	ttm_bo_mem_put(bo, &bo->mem);



	atomic_set(&bo->reserved, 0);

	wake_up_all(&bo->event_queue);



	/*

	 * Make processes trying to reserve really pick it up.

	 * Since the final reference to this bo may not be dropped by

	 * the current task we have to put a memory barrier here to make

	 * sure the changes done in this function are always visible.

	 *

	 * This function only needs protection against the final kref_put.

	 */

	smp_mb__after_atomic_dec();

	wake_up_all(&bo->event_queue);

	smp_mb__before_atomic_dec();

}



static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
@@ -543,68 +547,84 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) 
}



/**

 * function ttm_bo_cleanup_refs

 * function ttm_bo_cleanup_refs_and_unlock

 * If bo idle, remove from delayed- and lru lists, and unref.

 * If not idle, do nothing.

 *

 * Must be called with lru_lock and reservation held, this function

 * will drop both before returning.

 *

 * @interruptible         Any sleeps should occur interruptibly.

 * @no_wait_reserve       Never wait for reserve. Return -EBUSY instead.

 * @no_wait_gpu           Never wait for gpu. Return -EBUSY instead.

 */



static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,

static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo,

					  bool interruptible,

			       bool no_wait_reserve,

					  bool no_wait_gpu)

{

	struct ttm_bo_device *bdev = bo->bdev;

	struct ttm_bo_driver *driver = bdev->driver;

	struct ttm_bo_global *glob = bo->glob;

	int put_count;

	int ret = 0;

	int ret;



retry:

	spin_lock(&bdev->fence_lock);

	ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);

	spin_unlock(&bdev->fence_lock);

	ret = ttm_bo_wait(bo, false, false, true);



	if (unlikely(ret != 0))

		return ret;

	if (ret && !no_wait_gpu) {

		void *sync_obj;



retry_reserve:

	spin_lock(&glob->lru_lock);

		/*

		 * Take a reference to the fence and unreserve,

		 * at this point the buffer should be dead, so

		 * no new sync objects can be attached.

		 */

		sync_obj = driver->sync_obj_ref(&bo->sync_obj);

		spin_unlock(&bdev->fence_lock);



	if (unlikely(list_empty(&bo->ddestroy))) {

		atomic_set(&bo->reserved, 0);

		wake_up_all(&bo->event_queue);

		spin_unlock(&glob->lru_lock);

		return 0;

	}



	ret = ttm_bo_reserve_locked(bo, false, true, false, 0);



	if (unlikely(ret == -EBUSY)) {

		spin_unlock(&glob->lru_lock);

		if (likely(!no_wait_reserve))

			ret = ttm_bo_wait_unreserved(bo, interruptible);

		if (unlikely(ret != 0))

		ret = driver->sync_obj_wait(sync_obj, false, interruptible);

		driver->sync_obj_unref(&sync_obj);

		if (ret)

			return ret;



		goto retry_reserve;

	}

		/*

		 * remove sync_obj with ttm_bo_wait, the wait should be

		 * finished, and no new wait object should have been added.

		 */

		spin_lock(&bdev->fence_lock);

		ret = ttm_bo_wait(bo, false, false, true);

		WARN_ON(ret);

		spin_unlock(&bdev->fence_lock);

		if (ret)

			return ret;



	BUG_ON(ret != 0);

		spin_lock(&glob->lru_lock);

		ret = ttm_bo_reserve_locked(bo, false, true, false, 0);



	/**

	 * We can re-check for sync object without taking

	 * the bo::lock since setting the sync object requires

	 * also bo::reserved. A busy object at this point may

	 * be caused by another thread recently starting an accelerated

	 * eviction.

		/*

		 * We raced, and lost, someone else holds the reservation now,

		 * and is probably busy in ttm_bo_cleanup_memtype_use.

		 *

		 * Even if it's not the case, because we finished waiting any

		 * delayed destruction would succeed, so just return success

		 * here.

		 */

		if (ret) {

			spin_unlock(&glob->lru_lock);

			return 0;

		}

	} else

		spin_unlock(&bdev->fence_lock);



	if (unlikely(bo->sync_obj)) {

	if (ret || unlikely(list_empty(&bo->ddestroy))) {

		atomic_set(&bo->reserved, 0);

		wake_up_all(&bo->event_queue);

		spin_unlock(&glob->lru_lock);

		goto retry;

		return ret;

	}



	put_count = ttm_bo_del_from_lru(bo);
@@ -647,9 +667,13 @@ static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) 
			kref_get(&nentry->list_kref);

		}



		spin_unlock(&glob->lru_lock);

		ret = ttm_bo_cleanup_refs(entry, false, !remove_all,

		ret = ttm_bo_reserve_locked(entry, false, !remove_all, false, 0);

		if (!ret)

			ret = ttm_bo_cleanup_refs_and_unlock(entry, false,

							     !remove_all);

		else

			spin_unlock(&glob->lru_lock);



		kref_put(&entry->list_kref, ttm_bo_release_list);

		entry = nentry;
@@ -800,9 +824,13 @@ retry: 
	kref_get(&bo->list_kref);



	if (!list_empty(&bo->ddestroy)) {

		ret = ttm_bo_reserve_locked(bo, interruptible, no_wait_reserve, false, 0);

		if (!ret)

			ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible,

							     no_wait_gpu);

		else

			spin_unlock(&glob->lru_lock);

		ret = ttm_bo_cleanup_refs(bo, interruptible,

					  no_wait_reserve, no_wait_gpu);



		kref_put(&bo->list_kref, ttm_bo_release_list);



		return ret;
@@ -1796,8 +1824,9 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) 
		kref_get(&bo->list_kref);



		if (!list_empty(&bo->ddestroy)) {

			spin_unlock(&glob->lru_lock);

			(void) ttm_bo_cleanup_refs(bo, false, false, false);

			ttm_bo_reserve_locked(bo, false, false, false, 0);

			ttm_bo_cleanup_refs_and_unlock(bo, false, false);



			kref_put(&bo->list_kref, ttm_bo_release_list);

			spin_lock(&glob->lru_lock);

			continue;

CRA Git | Maintained and supported by SUSTech CRA and CCSE