Commit 8ba306a6 authored by Chris Wilson's avatar Chris Wilson
Browse files

drm/i915: Share per-timeline HWSP using a slab suballocator 



If we restrict ourselves to only using a cacheline for each timeline's
HWSP (we could go smaller, but want to avoid needless polluting
cachelines on different engines between different contexts), then we can
suballocate a single 4k page into 64 different timeline HWSP. By
treating each fresh allocation as a slab of 64 entries, we can keep it
around for the next 64 allocation attempts until we need to refresh the
slab cache.

John Harrison noted the issue of fragmentation leading to the same worst
case performance of one page per timeline as before, which can be
mitigated by adopting a freelist.

v2: Keep all partially allocated HWSP on a freelist

This is still without migration, so it is possible for the system to end
up with each timeline in its own page, but we ensure that no new
allocation would needless allocate a fresh page!

v3: Throw a selftest at the allocator to try and catch invalid cacheline
reuse.

Signed-off-by: default avatarChris Wilson <chris@chris-wilson.co.uk>
Cc: John Harrison <John.C.Harrison@Intel.com>
Reviewed-by: default avatarTvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20190128181812.22804-4-chris@chris-wilson.co.uk
parent 52954edd

drivers/gpu/drm/i915/i915_drv.h

+4 −0
Original line number Diff line number Diff line
@@ -1978,6 +1978,10 @@ struct drm_i915_private { 
		struct i915_gt_timelines {

			struct mutex mutex; /* protects list, tainted by GPU */

			struct list_head list;



			/* Pack multiple timelines' seqnos into the same page */

			spinlock_t hwsp_lock;

			struct list_head hwsp_free_list;

		} timelines;



		struct list_head active_rings;

drivers/gpu/drm/i915/i915_timeline.c

+103 −21
Original line number Diff line number Diff line
@@ -9,6 +9,18 @@ 
#include "i915_timeline.h"

#include "i915_syncmap.h"



struct i915_timeline_hwsp {

	struct i915_vma *vma;

	struct list_head free_link;

	u64 free_bitmap;

};



static inline struct i915_timeline_hwsp *

i915_timeline_hwsp(const struct i915_timeline *tl)

{

	return tl->hwsp_ggtt->private;

}



static struct i915_vma *__hwsp_alloc(struct drm_i915_private *i915)

{

	struct drm_i915_gem_object *obj;
@@ -27,28 +39,89 @@ static struct i915_vma *__hwsp_alloc(struct drm_i915_private *i915) 
	return vma;

}



static int hwsp_alloc(struct i915_timeline *timeline)

static struct i915_vma *

hwsp_alloc(struct i915_timeline *timeline, unsigned int *cacheline)

{

	struct drm_i915_private *i915 = timeline->i915;

	struct i915_gt_timelines *gt = &i915->gt.timelines;

	struct i915_timeline_hwsp *hwsp;



	BUILD_BUG_ON(BITS_PER_TYPE(u64) * CACHELINE_BYTES > PAGE_SIZE);



	spin_lock(&gt->hwsp_lock);



	/* hwsp_free_list only contains HWSP that have available cachelines */

	hwsp = list_first_entry_or_null(&gt->hwsp_free_list,

					typeof(*hwsp), free_link);

	if (!hwsp) {

		struct i915_vma *vma;



	vma = __hwsp_alloc(timeline->i915);

	if (IS_ERR(vma))

		return PTR_ERR(vma);

		spin_unlock(&gt->hwsp_lock);



		hwsp = kmalloc(sizeof(*hwsp), GFP_KERNEL);

		if (!hwsp)

			return ERR_PTR(-ENOMEM);



		vma = __hwsp_alloc(i915);

		if (IS_ERR(vma)) {

			kfree(hwsp);

			return vma;

		}



	timeline->hwsp_ggtt = vma;

	timeline->hwsp_offset = 0;

		vma->private = hwsp;

		hwsp->vma = vma;

		hwsp->free_bitmap = ~0ull;



	return 0;

		spin_lock(&gt->hwsp_lock);

		list_add(&hwsp->free_link, &gt->hwsp_free_list);

	}



	GEM_BUG_ON(!hwsp->free_bitmap);

	*cacheline = __ffs64(hwsp->free_bitmap);

	hwsp->free_bitmap &= ~BIT_ULL(*cacheline);

	if (!hwsp->free_bitmap)

		list_del(&hwsp->free_link);



	spin_unlock(&gt->hwsp_lock);



	GEM_BUG_ON(hwsp->vma->private != hwsp);

	return hwsp->vma;

}



static void hwsp_free(struct i915_timeline *timeline)

{

	struct i915_gt_timelines *gt = &timeline->i915->gt.timelines;

	struct i915_timeline_hwsp *hwsp;



	hwsp = i915_timeline_hwsp(timeline);

	if (!hwsp) /* leave global HWSP alone! */

		return;



	spin_lock(&gt->hwsp_lock);



	/* As a cacheline becomes available, publish the HWSP on the freelist */

	if (!hwsp->free_bitmap)

		list_add_tail(&hwsp->free_link, &gt->hwsp_free_list);



	hwsp->free_bitmap |= BIT_ULL(timeline->hwsp_offset / CACHELINE_BYTES);



	/* And if no one is left using it, give the page back to the system */

	if (hwsp->free_bitmap == ~0ull) {

		i915_vma_put(hwsp->vma);

		list_del(&hwsp->free_link);

		kfree(hwsp);

	}



	spin_unlock(&gt->hwsp_lock);

}



int i915_timeline_init(struct drm_i915_private *i915,

		       struct i915_timeline *timeline,

		       const char *name,

		       struct i915_vma *global_hwsp)

		       struct i915_vma *hwsp)

{

	struct i915_gt_timelines *gt = &i915->gt.timelines;

	void *vaddr;

	int err;



	/*

	 * Ideally we want a set of engines on a single leaf as we expect
@@ -64,18 +137,22 @@ int i915_timeline_init(struct drm_i915_private *i915, 
	timeline->name = name;

	timeline->pin_count = 0;



	if (global_hwsp) {

		timeline->hwsp_ggtt = i915_vma_get(global_hwsp);

	timeline->hwsp_offset = I915_GEM_HWS_SEQNO_ADDR;

	} else {

		err = hwsp_alloc(timeline);

		if (err)

			return err;

	if (!hwsp) {

		unsigned int cacheline;



		hwsp = hwsp_alloc(timeline, &cacheline);

		if (IS_ERR(hwsp))

			return PTR_ERR(hwsp);



		timeline->hwsp_offset = cacheline * CACHELINE_BYTES;

	}

	timeline->hwsp_ggtt = i915_vma_get(hwsp);



	vaddr = i915_gem_object_pin_map(timeline->hwsp_ggtt->obj, I915_MAP_WB);

	vaddr = i915_gem_object_pin_map(hwsp->obj, I915_MAP_WB);

	if (IS_ERR(vaddr)) {

		i915_vma_put(timeline->hwsp_ggtt);

		hwsp_free(timeline);

		i915_vma_put(hwsp);

		return PTR_ERR(vaddr);

	}
@@ -105,6 +182,9 @@ void i915_timelines_init(struct drm_i915_private *i915) 
	mutex_init(&gt->mutex);

	INIT_LIST_HEAD(&gt->list);



	spin_lock_init(&gt->hwsp_lock);

	INIT_LIST_HEAD(&gt->hwsp_free_list);



	/* via i915_gem_wait_for_idle() */

	i915_gem_shrinker_taints_mutex(i915, &gt->mutex);

}
@@ -144,12 +224,13 @@ void i915_timeline_fini(struct i915_timeline *timeline) 
	GEM_BUG_ON(timeline->pin_count);

	GEM_BUG_ON(!list_empty(&timeline->requests));



	i915_syncmap_free(&timeline->sync);



	mutex_lock(&gt->mutex);

	list_del(&timeline->link);

	mutex_unlock(&gt->mutex);



	i915_syncmap_free(&timeline->sync);

	hwsp_free(timeline);



	i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);

	i915_vma_put(timeline->hwsp_ggtt);

}
@@ -226,6 +307,7 @@ void i915_timelines_fini(struct drm_i915_private *i915) 
	struct i915_gt_timelines *gt = &i915->gt.timelines;



	GEM_BUG_ON(!list_empty(&gt->list));

	GEM_BUG_ON(!list_empty(&gt->hwsp_free_list));



	mutex_destroy(&gt->mutex);

}

drivers/gpu/drm/i915/i915_timeline.h

+1 −0
Original line number Diff line number Diff line
@@ -33,6 +33,7 @@ 
#include "i915_utils.h"



struct i915_vma;

struct i915_timeline_hwsp;



struct i915_timeline {

	u64 fence_context;

drivers/gpu/drm/i915/selftests/i915_random.c

+26 −7
Original line number Diff line number Diff line
@@ -41,18 +41,37 @@ u64 i915_prandom_u64_state(struct rnd_state *rnd) 
	return x;

}



void i915_random_reorder(unsigned int *order, unsigned int count,

void i915_prandom_shuffle(void *arr, size_t elsz, size_t count,

			  struct rnd_state *state)

{

	unsigned int i, j;

	char stack[128];



	if (WARN_ON(elsz > sizeof(stack) || count > U32_MAX))

		return;



	if (!elsz || !count)

		return;



	/* Fisher-Yates shuffle courtesy of Knuth */

	while (--count) {

		size_t swp;



		swp = i915_prandom_u32_max_state(count + 1, state);

		if (swp == count)

			continue;



	for (i = 0; i < count; i++) {

		BUILD_BUG_ON(sizeof(unsigned int) > sizeof(u32));

		j = i915_prandom_u32_max_state(count, state);

		swap(order[i], order[j]);

		memcpy(stack, arr + count * elsz, elsz);

		memcpy(arr + count * elsz, arr + swp * elsz, elsz);

		memcpy(arr + swp * elsz, stack, elsz);

	}

}



void i915_random_reorder(unsigned int *order, unsigned int count,

			 struct rnd_state *state)

{

	i915_prandom_shuffle(order, sizeof(*order), count, state);

}



unsigned int *i915_random_order(unsigned int count, struct rnd_state *state)

{

	unsigned int *order, i;

drivers/gpu/drm/i915/selftests/i915_random.h

+3 −0
Original line number Diff line number Diff line
@@ -54,4 +54,7 @@ void i915_random_reorder(unsigned int *order, 
			 unsigned int count,

			 struct rnd_state *state);



void i915_prandom_shuffle(void *arr, size_t elsz, size_t count,

			  struct rnd_state *state);



#endif /* !__I915_SELFTESTS_RANDOM_H__ */

CRA Git | Maintained and supported by SUSTech CRA and CCSE