Commit c826a6e1 authored by Eric Anholt's avatar Eric Anholt
Browse files

drm/vc4: Add a BO cache. 



We need to allocate new BOs in the kernel as part of each frame, but
the CMA allocator is way too slow for that.  As an optimization, keep
track of recently-freed BOs and reuse them, with a 1 second timeout to
fully free them back to the system.

This improves 3D performance by about 15%.

Signed-off-by: default avatarEric Anholt <eric@anholt.net>
parent 10028c5a

drivers/gpu/drm/vc4/vc4_bo.c

+330 −6
Original line number Diff line number Diff line
@@ -12,18 +12,228 @@ 
 * access to system memory with no MMU in between.  To support it, we

 * use the GEM CMA helper functions to allocate contiguous ranges of

 * physical memory for our BOs.

 *

 * Since the CMA allocator is very slow, we keep a cache of recently

 * freed BOs around so that the kernel's allocation of objects for 3D

 * rendering can return quickly.

 */



#include "vc4_drv.h"



struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t size)

static void vc4_bo_stats_dump(struct vc4_dev *vc4)

{

	DRM_INFO("num bos allocated: %d\n",

		 vc4->bo_stats.num_allocated);

	DRM_INFO("size bos allocated: %dkb\n",

		 vc4->bo_stats.size_allocated / 1024);

	DRM_INFO("num bos used: %d\n",

		 vc4->bo_stats.num_allocated - vc4->bo_stats.num_cached);

	DRM_INFO("size bos used: %dkb\n",

		 (vc4->bo_stats.size_allocated -

		  vc4->bo_stats.size_cached) / 1024);

	DRM_INFO("num bos cached: %d\n",

		 vc4->bo_stats.num_cached);

	DRM_INFO("size bos cached: %dkb\n",

		 vc4->bo_stats.size_cached / 1024);

}



#ifdef CONFIG_DEBUG_FS

int vc4_bo_stats_debugfs(struct seq_file *m, void *unused)

{

	struct drm_info_node *node = (struct drm_info_node *)m->private;

	struct drm_device *dev = node->minor->dev;

	struct vc4_dev *vc4 = to_vc4_dev(dev);

	struct vc4_bo_stats stats;



	/* Take a snapshot of the current stats with the lock held. */

	mutex_lock(&vc4->bo_lock);

	stats = vc4->bo_stats;

	mutex_unlock(&vc4->bo_lock);



	seq_printf(m, "num bos allocated: %d\n",

		   stats.num_allocated);

	seq_printf(m, "size bos allocated: %dkb\n",

		   stats.size_allocated / 1024);

	seq_printf(m, "num bos used: %d\n",

		   stats.num_allocated - stats.num_cached);

	seq_printf(m, "size bos used: %dkb\n",

		   (stats.size_allocated - stats.size_cached) / 1024);

	seq_printf(m, "num bos cached: %d\n",

		   stats.num_cached);

	seq_printf(m, "size bos cached: %dkb\n",

		   stats.size_cached / 1024);



	return 0;

}

#endif



static uint32_t bo_page_index(size_t size)

{

	return (size / PAGE_SIZE) - 1;

}



/* Must be called with bo_lock held. */

static void vc4_bo_destroy(struct vc4_bo *bo)

{

	struct drm_gem_object *obj = &bo->base.base;

	struct vc4_dev *vc4 = to_vc4_dev(obj->dev);



	vc4->bo_stats.num_allocated--;

	vc4->bo_stats.size_allocated -= obj->size;

	drm_gem_cma_free_object(obj);

}



/* Must be called with bo_lock held. */

static void vc4_bo_remove_from_cache(struct vc4_bo *bo)

{

	struct drm_gem_object *obj = &bo->base.base;

	struct vc4_dev *vc4 = to_vc4_dev(obj->dev);



	vc4->bo_stats.num_cached--;

	vc4->bo_stats.size_cached -= obj->size;



	list_del(&bo->unref_head);

	list_del(&bo->size_head);

}



static struct list_head *vc4_get_cache_list_for_size(struct drm_device *dev,

						     size_t size)

{

	struct vc4_dev *vc4 = to_vc4_dev(dev);

	uint32_t page_index = bo_page_index(size);



	if (vc4->bo_cache.size_list_size <= page_index) {

		uint32_t new_size = max(vc4->bo_cache.size_list_size * 2,

					page_index + 1);

		struct list_head *new_list;

		uint32_t i;



		new_list = kmalloc_array(new_size, sizeof(struct list_head),

					 GFP_KERNEL);

		if (!new_list)

			return NULL;



		/* Rebase the old cached BO lists to their new list

		 * head locations.

		 */

		for (i = 0; i < vc4->bo_cache.size_list_size; i++) {

			struct list_head *old_list =

				&vc4->bo_cache.size_list[i];



			if (list_empty(old_list))

				INIT_LIST_HEAD(&new_list[i]);

			else

				list_replace(old_list, &new_list[i]);

		}

		/* And initialize the brand new BO list heads. */

		for (i = vc4->bo_cache.size_list_size; i < new_size; i++)

			INIT_LIST_HEAD(&new_list[i]);



		kfree(vc4->bo_cache.size_list);

		vc4->bo_cache.size_list = new_list;

		vc4->bo_cache.size_list_size = new_size;

	}



	return &vc4->bo_cache.size_list[page_index];

}



void vc4_bo_cache_purge(struct drm_device *dev)

{

	struct vc4_dev *vc4 = to_vc4_dev(dev);



	mutex_lock(&vc4->bo_lock);

	while (!list_empty(&vc4->bo_cache.time_list)) {

		struct vc4_bo *bo = list_last_entry(&vc4->bo_cache.time_list,

						    struct vc4_bo, unref_head);

		vc4_bo_remove_from_cache(bo);

		vc4_bo_destroy(bo);

	}

	mutex_unlock(&vc4->bo_lock);

}



static struct vc4_bo *vc4_bo_get_from_cache(struct drm_device *dev,

					    uint32_t size)

{

	struct vc4_dev *vc4 = to_vc4_dev(dev);

	uint32_t page_index = bo_page_index(size);

	struct vc4_bo *bo = NULL;



	size = roundup(size, PAGE_SIZE);



	mutex_lock(&vc4->bo_lock);

	if (page_index >= vc4->bo_cache.size_list_size)

		goto out;



	if (list_empty(&vc4->bo_cache.size_list[page_index]))

		goto out;



	bo = list_first_entry(&vc4->bo_cache.size_list[page_index],

			      struct vc4_bo, size_head);

	vc4_bo_remove_from_cache(bo);

	kref_init(&bo->base.base.refcount);



out:

	mutex_unlock(&vc4->bo_lock);

	return bo;

}



/**

 * vc4_gem_create_object - Implementation of driver->gem_create_object.

 *

 * This lets the CMA helpers allocate object structs for us, and keep

 * our BO stats correct.

 */

struct drm_gem_object *vc4_create_object(struct drm_device *dev, size_t size)

{

	struct vc4_dev *vc4 = to_vc4_dev(dev);

	struct vc4_bo *bo;



	bo = kzalloc(sizeof(*bo), GFP_KERNEL);

	if (!bo)

		return ERR_PTR(-ENOMEM);



	mutex_lock(&vc4->bo_lock);

	vc4->bo_stats.num_allocated++;

	vc4->bo_stats.size_allocated += size;

	mutex_unlock(&vc4->bo_lock);



	return &bo->base.base;

}



struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t unaligned_size,

			     bool from_cache)

{

	size_t size = roundup(unaligned_size, PAGE_SIZE);

	struct vc4_dev *vc4 = to_vc4_dev(dev);

	struct drm_gem_cma_object *cma_obj;



	if (size == 0)

		return NULL;



	/* First, try to get a vc4_bo from the kernel BO cache. */

	if (from_cache) {

		struct vc4_bo *bo = vc4_bo_get_from_cache(dev, size);



		if (bo)

			return bo;

	}



	cma_obj = drm_gem_cma_create(dev, size);

	if (IS_ERR(cma_obj))

	if (IS_ERR(cma_obj)) {

		/*

		 * If we've run out of CMA memory, kill the cache of

		 * CMA allocations we've got laying around and try again.

		 */

		vc4_bo_cache_purge(dev);



		cma_obj = drm_gem_cma_create(dev, size);

		if (IS_ERR(cma_obj)) {

			DRM_ERROR("Failed to allocate from CMA:\n");

			vc4_bo_stats_dump(vc4);

			return NULL;

	else

		}

	}



	return to_vc4_bo(&cma_obj->base);

}
@@ -41,7 +251,7 @@ int vc4_dumb_create(struct drm_file *file_priv, 
	if (args->size < args->pitch * args->height)

		args->size = args->pitch * args->height;



	bo = vc4_bo_create(dev, roundup(args->size, PAGE_SIZE));

	bo = vc4_bo_create(dev, args->size, false);

	if (!bo)

		return -ENOMEM;
@@ -50,3 +260,117 @@ int vc4_dumb_create(struct drm_file *file_priv, 


	return ret;

}



/* Must be called with bo_lock held. */

static void vc4_bo_cache_free_old(struct drm_device *dev)

{

	struct vc4_dev *vc4 = to_vc4_dev(dev);

	unsigned long expire_time = jiffies - msecs_to_jiffies(1000);



	while (!list_empty(&vc4->bo_cache.time_list)) {

		struct vc4_bo *bo = list_last_entry(&vc4->bo_cache.time_list,

						    struct vc4_bo, unref_head);

		if (time_before(expire_time, bo->free_time)) {

			mod_timer(&vc4->bo_cache.time_timer,

				  round_jiffies_up(jiffies +

						   msecs_to_jiffies(1000)));

			return;

		}



		vc4_bo_remove_from_cache(bo);

		vc4_bo_destroy(bo);

	}

}



/* Called on the last userspace/kernel unreference of the BO.  Returns

 * it to the BO cache if possible, otherwise frees it.

 *

 * Note that this is called with the struct_mutex held.

 */

void vc4_free_object(struct drm_gem_object *gem_bo)

{

	struct drm_device *dev = gem_bo->dev;

	struct vc4_dev *vc4 = to_vc4_dev(dev);

	struct vc4_bo *bo = to_vc4_bo(gem_bo);

	struct list_head *cache_list;



	mutex_lock(&vc4->bo_lock);

	/* If the object references someone else's memory, we can't cache it.

	 */

	if (gem_bo->import_attach) {

		vc4_bo_destroy(bo);

		goto out;

	}



	/* Don't cache if it was publicly named. */

	if (gem_bo->name) {

		vc4_bo_destroy(bo);

		goto out;

	}



	cache_list = vc4_get_cache_list_for_size(dev, gem_bo->size);

	if (!cache_list) {

		vc4_bo_destroy(bo);

		goto out;

	}



	bo->free_time = jiffies;

	list_add(&bo->size_head, cache_list);

	list_add(&bo->unref_head, &vc4->bo_cache.time_list);



	vc4->bo_stats.num_cached++;

	vc4->bo_stats.size_cached += gem_bo->size;



	vc4_bo_cache_free_old(dev);



out:

	mutex_unlock(&vc4->bo_lock);

}



static void vc4_bo_cache_time_work(struct work_struct *work)

{

	struct vc4_dev *vc4 =

		container_of(work, struct vc4_dev, bo_cache.time_work);

	struct drm_device *dev = vc4->dev;



	mutex_lock(&vc4->bo_lock);

	vc4_bo_cache_free_old(dev);

	mutex_unlock(&vc4->bo_lock);

}



static void vc4_bo_cache_time_timer(unsigned long data)

{

	struct drm_device *dev = (struct drm_device *)data;

	struct vc4_dev *vc4 = to_vc4_dev(dev);



	schedule_work(&vc4->bo_cache.time_work);

}



void vc4_bo_cache_init(struct drm_device *dev)

{

	struct vc4_dev *vc4 = to_vc4_dev(dev);



	mutex_init(&vc4->bo_lock);



	INIT_LIST_HEAD(&vc4->bo_cache.time_list);



	INIT_WORK(&vc4->bo_cache.time_work, vc4_bo_cache_time_work);

	setup_timer(&vc4->bo_cache.time_timer,

		    vc4_bo_cache_time_timer,

		    (unsigned long)dev);

}



void vc4_bo_cache_destroy(struct drm_device *dev)

{

	struct vc4_dev *vc4 = to_vc4_dev(dev);



	del_timer(&vc4->bo_cache.time_timer);

	cancel_work_sync(&vc4->bo_cache.time_work);



	vc4_bo_cache_purge(dev);



	if (vc4->bo_stats.num_allocated) {

		DRM_ERROR("Destroying BO cache while BOs still allocated:\n");

		vc4_bo_stats_dump(vc4);

	}

}

drivers/gpu/drm/vc4/vc4_debugfs.c

+1 −0
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ 
#include "vc4_regs.h"



static const struct drm_info_list vc4_debugfs_list[] = {

	{"bo_stats", vc4_bo_stats_debugfs, 0},

	{"hdmi_regs", vc4_hdmi_debugfs_regs, 0},

	{"hvs_regs", vc4_hvs_debugfs_regs, 0},

	{"crtc0_regs", vc4_crtc_debugfs_regs, 0, (void *)(uintptr_t)0},

drivers/gpu/drm/vc4/vc4_drv.c

+5 −1
Original line number Diff line number Diff line
@@ -92,7 +92,8 @@ static struct drm_driver vc4_drm_driver = { 
	.debugfs_cleanup = vc4_debugfs_cleanup,

#endif



	.gem_free_object = drm_gem_cma_free_object,

	.gem_create_object = vc4_create_object,

	.gem_free_object = vc4_free_object,

	.gem_vm_ops = &drm_gem_cma_vm_ops,



	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
@@ -170,6 +171,8 @@ static int vc4_drm_bind(struct device *dev) 


	drm_dev_set_unique(drm, dev_name(dev));



	vc4_bo_cache_init(drm);



	drm_mode_config_init(drm);

	if (ret)

		goto unref;
@@ -202,6 +205,7 @@ unbind_all: 
	component_unbind_all(dev, drm);

unref:

	drm_dev_unref(drm);

	vc4_bo_cache_destroy(drm);

	return ret;

}

drivers/gpu/drm/vc4/vc4_drv.h

+48 −1
Original line number Diff line number Diff line
@@ -17,6 +17,37 @@ struct vc4_dev { 
	struct vc4_crtc *crtc[3];



	struct drm_fbdev_cma *fbdev;



	/* The kernel-space BO cache.  Tracks buffers that have been

	 * unreferenced by all other users (refcounts of 0!) but not

	 * yet freed, so we can do cheap allocations.

	 */

	struct vc4_bo_cache {

		/* Array of list heads for entries in the BO cache,

		 * based on number of pages, so we can do O(1) lookups

		 * in the cache when allocating.

		 */

		struct list_head *size_list;

		uint32_t size_list_size;



		/* List of all BOs in the cache, ordered by age, so we

		 * can do O(1) lookups when trying to free old

		 * buffers.

		 */

		struct list_head time_list;

		struct work_struct time_work;

		struct timer_list time_timer;

	} bo_cache;



	struct vc4_bo_stats {

		u32 num_allocated;

		u32 size_allocated;

		u32 num_cached;

		u32 size_cached;

	} bo_stats;



	/* Protects bo_cache and the BO stats. */

	struct mutex bo_lock;

};



static inline struct vc4_dev *
@@ -27,6 +58,17 @@ to_vc4_dev(struct drm_device *dev) 


struct vc4_bo {

	struct drm_gem_cma_object base;



	/* List entry for the BO's position in either

	 * vc4_exec_info->unref_list or vc4_dev->bo_cache.time_list

	 */

	struct list_head unref_head;



	/* Time in jiffies when the BO was put in vc4->bo_cache. */

	unsigned long free_time;



	/* List entry for the BO's position in vc4_dev->bo_cache.size_list */

	struct list_head size_head;

};



static inline struct vc4_bo *
@@ -104,13 +146,18 @@ to_vc4_encoder(struct drm_encoder *encoder) 
#define wait_for(COND, MS) _wait_for(COND, MS, 1)



/* vc4_bo.c */

struct drm_gem_object *vc4_create_object(struct drm_device *dev, size_t size);

void vc4_free_object(struct drm_gem_object *gem_obj);

struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t size);

struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t size,

			     bool from_cache);

int vc4_dumb_create(struct drm_file *file_priv,

		    struct drm_device *dev,

		    struct drm_mode_create_dumb *args);

struct dma_buf *vc4_prime_export(struct drm_device *dev,

				 struct drm_gem_object *obj, int flags);

void vc4_bo_cache_init(struct drm_device *dev);

void vc4_bo_cache_destroy(struct drm_device *dev);

int vc4_bo_stats_debugfs(struct seq_file *m, void *arg);



/* vc4_crtc.c */

extern struct platform_driver vc4_crtc_driver;

CRA Git | Maintained and supported by SUSTech CRA and CCSE