Commit b9e50713 authored by Dave Airlie's avatar Dave Airlie
Browse files

Merge branch 'for-airlied' of git://people.freedesktop.org/~mlankhorst/linux into drm-next 

TTM reservations changes, preparing for new reservation mutex system.

* 'for-airlied' of git://people.freedesktop.org/~mlankhorst/linux:
  drm/ttm: unexport ttm_bo_wait_unreserved
  drm/nouveau: use ttm_bo_reserve_slowpath in validate_init, v2
  drm/ttm: use ttm_bo_reserve_slowpath_nolru in ttm_eu_reserve_buffers, v2
  drm/ttm: add ttm_bo_reserve_slowpath
  drm/ttm: cleanup ttm_eu_reserve_buffers handling
  drm/ttm: remove lru_lock around ttm_bo_reserve
  drm/nouveau: increase reservation sequence every retry
  drm/vmwgfx: always use ttm_bo_is_reserved
parents 85a7ce67 cc4c0c4d

drivers/gpu/drm/nouveau/nouveau_gem.c

+16 −4
Original line number Diff line number Diff line
@@ -318,6 +318,7 @@ validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 
	uint32_t sequence;

	int trycnt = 0;

	int ret, i;

	struct nouveau_bo *res_bo = NULL;



	sequence = atomic_add_return(1, &drm->ttm.validate_sequence);

retry:
@@ -338,6 +339,11 @@ retry: 
			return -ENOENT;

		}

		nvbo = gem->driver_private;

		if (nvbo == res_bo) {

			res_bo = NULL;

			drm_gem_object_unreference_unlocked(gem);

			continue;

		}



		if (nvbo->reserved_by && nvbo->reserved_by == file_priv) {

			NV_ERROR(drm, "multiple instances of buffer %d on "
@@ -350,15 +356,19 @@ retry: 
		ret = ttm_bo_reserve(&nvbo->bo, true, false, true, sequence);

		if (ret) {

			validate_fini(op, NULL);

			if (unlikely(ret == -EAGAIN))

				ret = ttm_bo_wait_unreserved(&nvbo->bo, true);

			drm_gem_object_unreference_unlocked(gem);

			if (unlikely(ret == -EAGAIN)) {

				sequence = atomic_add_return(1, &drm->ttm.validate_sequence);

				ret = ttm_bo_reserve_slowpath(&nvbo->bo, true,

							      sequence);

				if (!ret)

					res_bo = nvbo;

			}

			if (unlikely(ret)) {

				drm_gem_object_unreference_unlocked(gem);

				if (ret != -ERESTARTSYS)

					NV_ERROR(drm, "fail reserve\n");

				return ret;

			}

			goto retry;

		}



		b->user_priv = (uint64_t)(unsigned long)nvbo;
@@ -380,6 +390,8 @@ retry: 
			validate_fini(op, NULL);

			return -EINVAL;

		}

		if (nvbo == res_bo)

			goto retry;

	}



	return 0;

drivers/gpu/drm/ttm/ttm_bo.c

+83 −20
Original line number Diff line number Diff line
@@ -158,7 +158,8 @@ static void ttm_bo_release_list(struct kref *list_kref) 
	ttm_mem_global_free(bdev->glob->mem_glob, acc_size);

}



int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible)

static int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo,

				  bool interruptible)

{

	if (interruptible) {

		return wait_event_interruptible(bo->event_queue,
@@ -168,7 +169,6 @@ int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible) 
		return 0;

	}

}

EXPORT_SYMBOL(ttm_bo_wait_unreserved);



void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)

{
@@ -213,14 +213,13 @@ int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) 
	return put_count;

}



int ttm_bo_reserve_locked(struct ttm_buffer_object *bo,

int ttm_bo_reserve_nolru(struct ttm_buffer_object *bo,

			  bool interruptible,

			  bool no_wait, bool use_sequence, uint32_t sequence)

{

	struct ttm_bo_global *glob = bo->glob;

	int ret;



	while (unlikely(atomic_read(&bo->reserved) != 0)) {

	while (unlikely(atomic_xchg(&bo->reserved, 1) != 0)) {

		/**

		 * Deadlock avoidance for multi-bo reserving.

		 */
@@ -241,26 +240,36 @@ int ttm_bo_reserve_locked(struct ttm_buffer_object *bo, 
		if (no_wait)

			return -EBUSY;



		spin_unlock(&glob->lru_lock);

		ret = ttm_bo_wait_unreserved(bo, interruptible);

		spin_lock(&glob->lru_lock);



		if (unlikely(ret))

			return ret;

	}



	atomic_set(&bo->reserved, 1);

	if (use_sequence) {

		bool wake_up = false;

		/**

		 * Wake up waiters that may need to recheck for deadlock,

		 * if we decreased the sequence number.

		 */

		if (unlikely((bo->val_seq - sequence < (1 << 31))

			     || !bo->seq_valid))

			wake_up_all(&bo->event_queue);

			wake_up = true;



		/*

		 * In the worst case with memory ordering these values can be

		 * seen in the wrong order. However since we call wake_up_all

		 * in that case, this will hopefully not pose a problem,

		 * and the worst case would only cause someone to accidentally

		 * hit -EAGAIN in ttm_bo_reserve when they see old value of

		 * val_seq. However this would only happen if seq_valid was

		 * written before val_seq was, and just means some slightly

		 * increased cpu usage

		 */

		bo->val_seq = sequence;

		bo->seq_valid = true;

		if (wake_up)

			wake_up_all(&bo->event_queue);

	} else {

		bo->seq_valid = false;

	}
@@ -289,17 +298,64 @@ int ttm_bo_reserve(struct ttm_buffer_object *bo, 
	int put_count = 0;

	int ret;



	spin_lock(&glob->lru_lock);

	ret = ttm_bo_reserve_locked(bo, interruptible, no_wait, use_sequence,

	ret = ttm_bo_reserve_nolru(bo, interruptible, no_wait, use_sequence,

				   sequence);

	if (likely(ret == 0))

	if (likely(ret == 0)) {

		spin_lock(&glob->lru_lock);

		put_count = ttm_bo_del_from_lru(bo);

		spin_unlock(&glob->lru_lock);



		ttm_bo_list_ref_sub(bo, put_count, true);

	}



	return ret;

}



int ttm_bo_reserve_slowpath_nolru(struct ttm_buffer_object *bo,

				  bool interruptible, uint32_t sequence)

{

	bool wake_up = false;

	int ret;



	while (unlikely(atomic_xchg(&bo->reserved, 1) != 0)) {

		WARN_ON(bo->seq_valid && sequence == bo->val_seq);



		ret = ttm_bo_wait_unreserved(bo, interruptible);



		if (unlikely(ret))

			return ret;

	}



	if ((bo->val_seq - sequence < (1 << 31)) || !bo->seq_valid)

		wake_up = true;



	/**

	 * Wake up waiters that may need to recheck for deadlock,

	 * if we decreased the sequence number.

	 */

	bo->val_seq = sequence;

	bo->seq_valid = true;

	if (wake_up)

		wake_up_all(&bo->event_queue);



	return 0;

}



int ttm_bo_reserve_slowpath(struct ttm_buffer_object *bo,

			    bool interruptible, uint32_t sequence)

{

	struct ttm_bo_global *glob = bo->glob;

	int put_count, ret;



	ret = ttm_bo_reserve_slowpath_nolru(bo, interruptible, sequence);

	if (likely(!ret)) {

		spin_lock(&glob->lru_lock);

		put_count = ttm_bo_del_from_lru(bo);

		spin_unlock(&glob->lru_lock);

		ttm_bo_list_ref_sub(bo, put_count, true);

	}

	return ret;

}

EXPORT_SYMBOL(ttm_bo_reserve_slowpath);



void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo)

{
@@ -511,7 +567,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) 
	int ret;



	spin_lock(&glob->lru_lock);

	ret = ttm_bo_reserve_locked(bo, false, true, false, 0);

	ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);



	spin_lock(&bdev->fence_lock);

	(void) ttm_bo_wait(bo, false, false, true);
@@ -604,7 +660,7 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, 
			return ret;



		spin_lock(&glob->lru_lock);

		ret = ttm_bo_reserve_locked(bo, false, true, false, 0);

		ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);



		/*

		 * We raced, and lost, someone else holds the reservation now,
@@ -668,7 +724,14 @@ static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) 
			kref_get(&nentry->list_kref);

		}



		ret = ttm_bo_reserve_locked(entry, false, !remove_all, false, 0);

		ret = ttm_bo_reserve_nolru(entry, false, true, false, 0);

		if (remove_all && ret) {

			spin_unlock(&glob->lru_lock);

			ret = ttm_bo_reserve_nolru(entry, false, false,

						   false, 0);

			spin_lock(&glob->lru_lock);

		}



		if (!ret)

			ret = ttm_bo_cleanup_refs_and_unlock(entry, false,

							     !remove_all);
@@ -816,7 +879,7 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev, 


	spin_lock(&glob->lru_lock);

	list_for_each_entry(bo, &man->lru, lru) {

		ret = ttm_bo_reserve_locked(bo, false, true, false, 0);

		ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);

		if (!ret)

			break;

	}
@@ -1797,7 +1860,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) 


	spin_lock(&glob->lru_lock);

	list_for_each_entry(bo, &glob->swap_lru, swap) {

		ret = ttm_bo_reserve_locked(bo, false, true, false, 0);

		ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);

		if (!ret)

			break;

	}

drivers/gpu/drm/ttm/ttm_execbuf_util.c

+43 −35
Original line number Diff line number Diff line
@@ -82,22 +82,6 @@ static void ttm_eu_list_ref_sub(struct list_head *list) 
	}

}



static int ttm_eu_wait_unreserved_locked(struct list_head *list,

					 struct ttm_buffer_object *bo)

{

	struct ttm_bo_global *glob = bo->glob;

	int ret;



	ttm_eu_del_from_lru_locked(list);

	spin_unlock(&glob->lru_lock);

	ret = ttm_bo_wait_unreserved(bo, true);

	spin_lock(&glob->lru_lock);

	if (unlikely(ret != 0))

		ttm_eu_backoff_reservation_locked(list);

	return ret;

}





void ttm_eu_backoff_reservation(struct list_head *list)

{

	struct ttm_validate_buffer *entry;
@@ -145,47 +129,65 @@ int ttm_eu_reserve_buffers(struct list_head *list) 
	entry = list_first_entry(list, struct ttm_validate_buffer, head);

	glob = entry->bo->glob;



retry:

	spin_lock(&glob->lru_lock);

	val_seq = entry->bo->bdev->val_seq++;



retry:

	list_for_each_entry(entry, list, head) {

		struct ttm_buffer_object *bo = entry->bo;



retry_this_bo:

		ret = ttm_bo_reserve_locked(bo, true, true, true, val_seq);

		/* already slowpath reserved? */

		if (entry->reserved)

			continue;



		ret = ttm_bo_reserve_nolru(bo, true, true, true, val_seq);

		switch (ret) {

		case 0:

			break;

		case -EBUSY:

			ret = ttm_eu_wait_unreserved_locked(list, bo);

			if (unlikely(ret != 0)) {

			ttm_eu_del_from_lru_locked(list);

			spin_unlock(&glob->lru_lock);

				ttm_eu_list_ref_sub(list);

				return ret;

			}

			goto retry_this_bo;

			ret = ttm_bo_reserve_nolru(bo, true, false,

						   true, val_seq);

			spin_lock(&glob->lru_lock);

			if (!ret)

				break;



			if (unlikely(ret != -EAGAIN))

				goto err;



			/* fallthrough */

		case -EAGAIN:

			ttm_eu_backoff_reservation_locked(list);



			/*

			 * temporarily increase sequence number every retry,

			 * to prevent us from seeing our old reservation

			 * sequence when someone else reserved the buffer,

			 * but hasn't updated the seq_valid/seqno members yet.

			 */

			val_seq = entry->bo->bdev->val_seq++;



			spin_unlock(&glob->lru_lock);

			ttm_eu_list_ref_sub(list);

			ret = ttm_bo_wait_unreserved(bo, true);

			ret = ttm_bo_reserve_slowpath_nolru(bo, true, val_seq);

			if (unlikely(ret != 0))

				return ret;

			spin_lock(&glob->lru_lock);

			entry->reserved = true;

			if (unlikely(atomic_read(&bo->cpu_writers) > 0)) {

				ret = -EBUSY;

				goto err;

			}

			goto retry;

		default:

			ttm_eu_backoff_reservation_locked(list);

			spin_unlock(&glob->lru_lock);

			ttm_eu_list_ref_sub(list);

			return ret;

			goto err;

		}



		entry->reserved = true;

		if (unlikely(atomic_read(&bo->cpu_writers) > 0)) {

			ttm_eu_backoff_reservation_locked(list);

			spin_unlock(&glob->lru_lock);

			ttm_eu_list_ref_sub(list);

			return -EBUSY;

			ret = -EBUSY;

			goto err;

		}

	}
@@ -194,6 +196,12 @@ retry_this_bo: 
	ttm_eu_list_ref_sub(list);



	return 0;



err:

	ttm_eu_backoff_reservation_locked(list);

	spin_unlock(&glob->lru_lock);

	ttm_eu_list_ref_sub(list);

	return ret;

}

EXPORT_SYMBOL(ttm_eu_reserve_buffers);

drivers/gpu/drm/vmwgfx/vmwgfx_resource.c

+2 −2
Original line number Diff line number Diff line
@@ -959,13 +959,13 @@ void vmw_resource_unreserve(struct vmw_resource *res, 
	if (new_backup && new_backup != res->backup) {



		if (res->backup) {

			BUG_ON(atomic_read(&res->backup->base.reserved) == 0);

			BUG_ON(!ttm_bo_is_reserved(&res->backup->base));

			list_del_init(&res->mob_head);

			vmw_dmabuf_unreference(&res->backup);

		}



		res->backup = vmw_dmabuf_reference(new_backup);

		BUG_ON(atomic_read(&new_backup->base.reserved) == 0);

		BUG_ON(!ttm_bo_is_reserved(&new_backup->base));

		list_add_tail(&res->mob_head, &new_backup->res_list);

	}

	if (new_backup)

include/drm/ttm/ttm_bo_driver.h

+34 −27
Original line number Diff line number Diff line
@@ -790,16 +790,7 @@ extern void ttm_mem_io_unlock(struct ttm_mem_type_manager *man); 
 * to make room for a buffer already reserved. (Buffers are reserved before

 * they are evicted). The following algorithm prevents such deadlocks from

 * occurring:

 * 1) Buffers are reserved with the lru spinlock held. Upon successful

 * reservation they are removed from the lru list. This stops a reserved buffer

 * from being evicted. However the lru spinlock is released between the time

 * a buffer is selected for eviction and the time it is reserved.

 * Therefore a check is made when a buffer is reserved for eviction, that it

 * is still the first buffer in the lru list, before it is removed from the

 * list. @check_lru == 1 forces this check. If it fails, the function returns

 * -EINVAL, and the caller should then choose a new buffer to evict and repeat

 * the procedure.

 * 2) Processes attempting to reserve multiple buffers other than for eviction,

 * Processes attempting to reserve multiple buffers other than for eviction,

 * (typically execbuf), should first obtain a unique 32-bit

 * validation sequence number,

 * and call this function with @use_sequence == 1 and @sequence == the unique
@@ -830,9 +821,39 @@ extern int ttm_bo_reserve(struct ttm_buffer_object *bo, 
			  bool interruptible,

			  bool no_wait, bool use_sequence, uint32_t sequence);



/**

 * ttm_bo_reserve_slowpath_nolru:

 * @bo: A pointer to a struct ttm_buffer_object.

 * @interruptible: Sleep interruptible if waiting.

 * @sequence: Set (@bo)->sequence to this value after lock

 *

 * This is called after ttm_bo_reserve returns -EAGAIN and we backed off

 * from all our other reservations. Because there are no other reservations

 * held by us, this function cannot deadlock any more.

 *

 * Will not remove reserved buffers from the lru lists.

 * Otherwise identical to ttm_bo_reserve_slowpath.

 */

extern int ttm_bo_reserve_slowpath_nolru(struct ttm_buffer_object *bo,

					 bool interruptible,

					 uint32_t sequence);





/**

 * ttm_bo_reserve_slowpath:

 * @bo: A pointer to a struct ttm_buffer_object.

 * @interruptible: Sleep interruptible if waiting.

 * @sequence: Set (@bo)->sequence to this value after lock

 *

 * This is called after ttm_bo_reserve returns -EAGAIN and we backed off

 * from all our other reservations. Because there are no other reservations

 * held by us, this function cannot deadlock any more.

 */

extern int ttm_bo_reserve_slowpath(struct ttm_buffer_object *bo,

				   bool interruptible, uint32_t sequence);



/**

 * ttm_bo_reserve_locked:

 * ttm_bo_reserve_nolru:

 *

 * @bo: A pointer to a struct ttm_buffer_object.

 * @interruptible: Sleep interruptible if waiting.
@@ -840,9 +861,7 @@ extern int ttm_bo_reserve(struct ttm_buffer_object *bo, 
 * @use_sequence: If @bo is already reserved, Only sleep waiting for

 * it to become unreserved if @sequence < (@bo)->sequence.

 *

 * Must be called with struct ttm_bo_global::lru_lock held,

 * and will not remove reserved buffers from the lru lists.

 * The function may release the LRU spinlock if it needs to sleep.

 * Will not remove reserved buffers from the lru lists.

 * Otherwise identical to ttm_bo_reserve.

 *

 * Returns:
@@ -855,7 +874,7 @@ extern int ttm_bo_reserve(struct ttm_buffer_object *bo, 
 * -EDEADLK: Bo already reserved using @sequence. This error code will only

 * be returned if @use_sequence is set to true.

 */

extern int ttm_bo_reserve_locked(struct ttm_buffer_object *bo,

extern int ttm_bo_reserve_nolru(struct ttm_buffer_object *bo,

				 bool interruptible,

				 bool no_wait, bool use_sequence,

				 uint32_t sequence);
@@ -879,18 +898,6 @@ extern void ttm_bo_unreserve(struct ttm_buffer_object *bo); 
 */

extern void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo);



/**

 * ttm_bo_wait_unreserved

 *

 * @bo: A pointer to a struct ttm_buffer_object.

 *

 * Wait for a struct ttm_buffer_object to become unreserved.

 * This is typically used in the execbuf code to relax cpu-usage when

 * a potential deadlock condition backoff.

 */

extern int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo,

				  bool interruptible);



/*

 * ttm_bo_util.c

 */

CRA Git | Maintained and supported by SUSTech CRA and CCSE