Commit ea7b1dd4 authored by Daniel Vetter's avatar Daniel Vetter Committed by Dave Airlie
Browse files

drm: mm: track free areas implicitly 



The idea is to track free holes implicitly by marking the allocation
immediatly preceeding a hole.

To avoid an ugly corner case add a dummy head_node to struct drm_mm
to track the hole that spans to complete allocation area when the
memory manager is empty.

To guarantee that there's always a preceeding/following node (that might
be marked as hole_follows == 1), move the mm->node_list list_head to the
head_node.

The main allocator and fair-lru scan code actually becomes simpler.
Only the debug code slightly suffers because free areas are no longer
explicit.

Also add drm_mm_for_each_node (which will be much more useful when
struct drm_mm_node is embeddable).

Signed-off-by: default avatarDaniel Vetter <daniel.vetter@ffwll.ch>
Signed-off-by: default avatarDave Airlie <airlied@redhat.com>
parent 31a5b8ce

drivers/gpu/drm/drm_mm.c

+213 −251
Original line number Diff line number Diff line
@@ -64,8 +64,8 @@ static struct drm_mm_node *drm_mm_kmalloc(struct drm_mm *mm, int atomic) 
		else {

			child =

			    list_entry(mm->unused_nodes.next,

				       struct drm_mm_node, free_stack);

			list_del(&child->free_stack);

				       struct drm_mm_node, node_list);

			list_del(&child->node_list);

			--mm->num_unused;

		}

		spin_unlock(&mm->unused_lock);
@@ -94,126 +94,123 @@ int drm_mm_pre_get(struct drm_mm *mm) 
			return ret;

		}

		++mm->num_unused;

		list_add_tail(&node->free_stack, &mm->unused_nodes);

		list_add_tail(&node->node_list, &mm->unused_nodes);

	}

	spin_unlock(&mm->unused_lock);

	return 0;

}

EXPORT_SYMBOL(drm_mm_pre_get);



static int drm_mm_create_tail_node(struct drm_mm *mm,

				   unsigned long start,

				   unsigned long size, int atomic)

static inline unsigned long drm_mm_hole_node_start(struct drm_mm_node *hole_node)

{

	struct drm_mm_node *child;



	child = drm_mm_kmalloc(mm, atomic);

	if (unlikely(child == NULL))

		return -ENOMEM;



	child->free = 1;

	child->size = size;

	child->start = start;

	child->mm = mm;

	return hole_node->start + hole_node->size;

}



	list_add_tail(&child->node_list, &mm->node_list);

	list_add_tail(&child->free_stack, &mm->free_stack);

static inline unsigned long drm_mm_hole_node_end(struct drm_mm_node *hole_node)

{

	struct drm_mm_node *next_node =

		list_entry(hole_node->node_list.next, struct drm_mm_node,

			   node_list);



	return 0;

	return next_node->start;

}



static struct drm_mm_node *drm_mm_split_at_start(struct drm_mm_node *parent,

struct drm_mm_node *drm_mm_get_block_generic(struct drm_mm_node *hole_node,

					     unsigned long size,

					     unsigned alignment,

					     int atomic)

{

	struct drm_mm_node *child;



	child = drm_mm_kmalloc(parent->mm, atomic);

	if (unlikely(child == NULL))

		return NULL;



	INIT_LIST_HEAD(&child->free_stack);

	struct drm_mm_node *node;

	struct drm_mm *mm = hole_node->mm;

	unsigned long tmp = 0, wasted = 0;

	unsigned long hole_start = drm_mm_hole_node_start(hole_node);

	unsigned long hole_end = drm_mm_hole_node_end(hole_node);



	child->size = size;

	child->start = parent->start;

	child->mm = parent->mm;

	BUG_ON(!hole_node->hole_follows);



	list_add_tail(&child->node_list, &parent->node_list);

	INIT_LIST_HEAD(&child->free_stack);

	node = drm_mm_kmalloc(mm, atomic);

	if (unlikely(node == NULL))

		return NULL;



	parent->size -= size;

	parent->start += size;

	return child;

}

	if (alignment)

		tmp = hole_start % alignment;



	if (!tmp) {

		hole_node->hole_follows = 0;

		list_del_init(&hole_node->hole_stack);

	} else

		wasted = alignment - tmp;



struct drm_mm_node *drm_mm_get_block_generic(struct drm_mm_node *node,

					     unsigned long size,

					     unsigned alignment,

					     int atomic)

{

	node->start = hole_start + wasted;

	node->size = size;

	node->mm = mm;



	struct drm_mm_node *align_splitoff = NULL;

	unsigned tmp = 0;

	INIT_LIST_HEAD(&node->hole_stack);

	list_add(&node->node_list, &hole_node->node_list);



	if (alignment)

		tmp = node->start % alignment;

	BUG_ON(node->start + node->size > hole_end);



	if (tmp) {

		align_splitoff =

		    drm_mm_split_at_start(node, alignment - tmp, atomic);

		if (unlikely(align_splitoff == NULL))

			return NULL;

	}



	if (node->size == size) {

		list_del_init(&node->free_stack);

		node->free = 0;

	if (node->start + node->size < hole_end) {

		list_add(&node->hole_stack, &mm->hole_stack);

		node->hole_follows = 1;

	} else {

		node = drm_mm_split_at_start(node, size, atomic);

		node->hole_follows = 0;

	}



	if (align_splitoff)

		drm_mm_put_block(align_splitoff);



	return node;

}

EXPORT_SYMBOL(drm_mm_get_block_generic);



struct drm_mm_node *drm_mm_get_block_range_generic(struct drm_mm_node *node,

struct drm_mm_node *drm_mm_get_block_range_generic(struct drm_mm_node *hole_node,

						unsigned long size,

						unsigned alignment,

						unsigned long start,

						unsigned long end,

						int atomic)

{

	struct drm_mm_node *align_splitoff = NULL;

	unsigned tmp = 0;

	unsigned wasted = 0;

	struct drm_mm_node *node;

	struct drm_mm *mm = hole_node->mm;

	unsigned long tmp = 0, wasted = 0;

	unsigned long hole_start = drm_mm_hole_node_start(hole_node);

	unsigned long hole_end = drm_mm_hole_node_end(hole_node);



	BUG_ON(!hole_node->hole_follows);



	if (node->start < start)

		wasted += start - node->start;

	node = drm_mm_kmalloc(mm, atomic);

	if (unlikely(node == NULL))

		return NULL;



	if (hole_start < start)

		wasted += start - hole_start;

	if (alignment)

		tmp = ((node->start + wasted) % alignment);

		tmp = (hole_start + wasted) % alignment;



	if (tmp)

		wasted += alignment - tmp;

	if (wasted) {

		align_splitoff = drm_mm_split_at_start(node, wasted, atomic);

		if (unlikely(align_splitoff == NULL))

			return NULL;



	if (!wasted) {

		hole_node->hole_follows = 0;

		list_del_init(&hole_node->hole_stack);

	}



	if (node->size == size) {

		list_del_init(&node->free_stack);

		node->free = 0;

	node->start = hole_start + wasted;

	node->size = size;

	node->mm = mm;



	INIT_LIST_HEAD(&node->hole_stack);

	list_add(&node->node_list, &hole_node->node_list);



	BUG_ON(node->start + node->size > hole_end);

	BUG_ON(node->start + node->size > end);



	if (node->start + node->size < hole_end) {

		list_add(&node->hole_stack, &mm->hole_stack);

		node->hole_follows = 1;

	} else {

		node = drm_mm_split_at_start(node, size, atomic);

		node->hole_follows = 0;

	}



	if (align_splitoff)

		drm_mm_put_block(align_splitoff);



	return node;

}

EXPORT_SYMBOL(drm_mm_get_block_range_generic);
@@ -223,66 +220,41 @@ EXPORT_SYMBOL(drm_mm_get_block_range_generic); 
 * Otherwise add to the free stack.

 */



void drm_mm_put_block(struct drm_mm_node *cur)

void drm_mm_put_block(struct drm_mm_node *node)

{



	struct drm_mm *mm = cur->mm;

	struct list_head *cur_head = &cur->node_list;

	struct list_head *root_head = &mm->node_list;

	struct drm_mm_node *prev_node = NULL;

	struct drm_mm_node *next_node;



	int merged = 0;

	struct drm_mm *mm = node->mm;

	struct drm_mm_node *prev_node;



	BUG_ON(cur->scanned_block || cur->scanned_prev_free

				  || cur->scanned_next_free);

	BUG_ON(node->scanned_block || node->scanned_prev_free

				   || node->scanned_next_free);



	if (cur_head->prev != root_head) {

	prev_node =

		    list_entry(cur_head->prev, struct drm_mm_node, node_list);

		if (prev_node->free) {

			prev_node->size += cur->size;

			merged = 1;

		}

	}

	if (cur_head->next != root_head) {

		next_node =

		    list_entry(cur_head->next, struct drm_mm_node, node_list);

		if (next_node->free) {

			if (merged) {

				prev_node->size += next_node->size;

				list_del(&next_node->node_list);

				list_del(&next_node->free_stack);

				spin_lock(&mm->unused_lock);

				if (mm->num_unused < MM_UNUSED_TARGET) {

					list_add(&next_node->free_stack,

						 &mm->unused_nodes);

					++mm->num_unused;

	    list_entry(node->node_list.prev, struct drm_mm_node, node_list);



	if (node->hole_follows) {

		BUG_ON(drm_mm_hole_node_start(node)

				== drm_mm_hole_node_end(node));

		list_del(&node->hole_stack);

	} else

					kfree(next_node);

				spin_unlock(&mm->unused_lock);

			} else {

				next_node->size += cur->size;

				next_node->start = cur->start;

				merged = 1;

			}

		}

	}

	if (!merged) {

		cur->free = 1;

		list_add(&cur->free_stack, &mm->free_stack);

	} else {

		list_del(&cur->node_list);

		BUG_ON(drm_mm_hole_node_start(node)

				!= drm_mm_hole_node_end(node));



	if (!prev_node->hole_follows) {

		prev_node->hole_follows = 1;

		list_add(&prev_node->hole_stack, &mm->hole_stack);

	} else

		list_move(&prev_node->hole_stack, &mm->hole_stack);



	list_del(&node->node_list);

	spin_lock(&mm->unused_lock);

	if (mm->num_unused < MM_UNUSED_TARGET) {

			list_add(&cur->free_stack, &mm->unused_nodes);

		list_add(&node->node_list, &mm->unused_nodes);

		++mm->num_unused;

	} else

			kfree(cur);

		kfree(node);

	spin_unlock(&mm->unused_lock);

}

}



EXPORT_SYMBOL(drm_mm_put_block);



static int check_free_hole(unsigned long start, unsigned long end,
@@ -319,8 +291,10 @@ struct drm_mm_node *drm_mm_search_free(const struct drm_mm *mm, 
	best = NULL;

	best_size = ~0UL;



	list_for_each_entry(entry, &mm->free_stack, free_stack) {

		if (!check_free_hole(entry->start, entry->start + entry->size,

	list_for_each_entry(entry, &mm->hole_stack, hole_stack) {

		BUG_ON(!entry->hole_follows);

		if (!check_free_hole(drm_mm_hole_node_start(entry),

				     drm_mm_hole_node_end(entry),

				     size, alignment))

			continue;
@@ -353,12 +327,13 @@ struct drm_mm_node *drm_mm_search_free_in_range(const struct drm_mm *mm, 
	best = NULL;

	best_size = ~0UL;



	list_for_each_entry(entry, &mm->free_stack, free_stack) {

		unsigned long adj_start = entry->start < start ?

			start : entry->start;

		unsigned long adj_end = entry->start + entry->size > end ?

			end : entry->start + entry->size;

	list_for_each_entry(entry, &mm->hole_stack, hole_stack) {

		unsigned long adj_start = drm_mm_hole_node_start(entry) < start ?

			start : drm_mm_hole_node_start(entry);

		unsigned long adj_end = drm_mm_hole_node_end(entry) > end ?

			end : drm_mm_hole_node_end(entry);



		BUG_ON(!entry->hole_follows);

		if (!check_free_hole(adj_start, adj_end, size, alignment))

			continue;
@@ -430,70 +405,40 @@ EXPORT_SYMBOL(drm_mm_init_scan_with_range); 
int drm_mm_scan_add_block(struct drm_mm_node *node)

{

	struct drm_mm *mm = node->mm;

	struct list_head *prev_free, *next_free;

	struct drm_mm_node *prev_node, *next_node;

	struct drm_mm_node *prev_node;

	unsigned long hole_start, hole_end;

	unsigned long adj_start;

	unsigned long adj_end;



	mm->scanned_blocks++;



	prev_free = next_free = NULL;



	BUG_ON(node->free);

	BUG_ON(node->scanned_block);

	node->scanned_block = 1;

	node->free = 1;



	if (node->node_list.prev != &mm->node_list) {

	prev_node = list_entry(node->node_list.prev, struct drm_mm_node,

			       node_list);



		if (prev_node->free) {

			list_del(&prev_node->node_list);



			node->start = prev_node->start;

			node->size += prev_node->size;



			prev_node->scanned_prev_free = 1;



			prev_free = &prev_node->free_stack;

		}

	}



	if (node->node_list.next != &mm->node_list) {

		next_node = list_entry(node->node_list.next, struct drm_mm_node,

				       node_list);



		if (next_node->free) {

			list_del(&next_node->node_list);



			node->size += next_node->size;



			next_node->scanned_next_free = 1;



			next_free = &next_node->free_stack;

		}

	}



	/* The free_stack list is not used for allocated objects, so these two

	 * pointers can be abused (as long as no allocations in this memory

	 * manager happens). */

	node->free_stack.prev = prev_free;

	node->free_stack.next = next_free;

	node->scanned_preceeds_hole = prev_node->hole_follows;

	prev_node->hole_follows = 1;

	list_del(&node->node_list);

	node->node_list.prev = &prev_node->node_list;



	hole_start = drm_mm_hole_node_start(prev_node);

	hole_end = drm_mm_hole_node_end(prev_node);

	if (mm->scan_check_range) {

		adj_start = node->start < mm->scan_start ?

			mm->scan_start : node->start;

		adj_end = node->start + node->size > mm->scan_end ?

			mm->scan_end : node->start + node->size;

		adj_start = hole_start < mm->scan_start ?

			mm->scan_start : hole_start;

		adj_end = hole_end > mm->scan_end ?

			mm->scan_end : hole_end;

	} else {

		adj_start = node->start;

		adj_end = node->start + node->size;

		adj_start = hole_start;

		adj_end = hole_end;

	}



	if (check_free_hole(adj_start , adj_end,

			    mm->scan_size, mm->scan_alignment)) {

		mm->scan_hit_start = node->start;

		mm->scan_hit_size = node->size;

		mm->scan_hit_start = hole_start;

		mm->scan_hit_size = hole_end;



		return 1;

	}
@@ -519,39 +464,19 @@ EXPORT_SYMBOL(drm_mm_scan_add_block); 
int drm_mm_scan_remove_block(struct drm_mm_node *node)

{

	struct drm_mm *mm = node->mm;

	struct drm_mm_node *prev_node, *next_node;

	struct drm_mm_node *prev_node;



	mm->scanned_blocks--;



	BUG_ON(!node->scanned_block);

	node->scanned_block = 0;

	node->free = 0;



	prev_node = list_entry(node->free_stack.prev, struct drm_mm_node,

			       free_stack);

	next_node = list_entry(node->free_stack.next, struct drm_mm_node,

			       free_stack);



	if (prev_node) {

		BUG_ON(!prev_node->scanned_prev_free);

		prev_node->scanned_prev_free = 0;



		list_add_tail(&prev_node->node_list, &node->node_list);



		node->start = prev_node->start + prev_node->size;

		node->size -= prev_node->size;

	}



	if (next_node) {

		BUG_ON(!next_node->scanned_next_free);

		next_node->scanned_next_free = 0;



		list_add(&next_node->node_list, &node->node_list);



		node->size -= next_node->size;

	}

	prev_node = list_entry(node->node_list.prev, struct drm_mm_node,

			       node_list);



	INIT_LIST_HEAD(&node->free_stack);

	prev_node->hole_follows = node->scanned_preceeds_hole;

	INIT_LIST_HEAD(&node->node_list);

	list_add(&node->node_list, &prev_node->node_list);



	/* Only need to check for containement because start&size for the

	 * complete resulting free block (not just the desired part) is
@@ -568,7 +493,7 @@ EXPORT_SYMBOL(drm_mm_scan_remove_block); 


int drm_mm_clean(struct drm_mm * mm)

{

	struct list_head *head = &mm->node_list;

	struct list_head *head = &mm->head_node.node_list;



	return (head->next->next == head);

}
@@ -576,38 +501,40 @@ EXPORT_SYMBOL(drm_mm_clean); 


int drm_mm_init(struct drm_mm * mm, unsigned long start, unsigned long size)

{

	INIT_LIST_HEAD(&mm->node_list);

	INIT_LIST_HEAD(&mm->free_stack);

	INIT_LIST_HEAD(&mm->hole_stack);

	INIT_LIST_HEAD(&mm->unused_nodes);

	mm->num_unused = 0;

	mm->scanned_blocks = 0;

	spin_lock_init(&mm->unused_lock);



	return drm_mm_create_tail_node(mm, start, size, 0);

	/* Clever trick to avoid a special case in the free hole tracking. */

	INIT_LIST_HEAD(&mm->head_node.node_list);

	INIT_LIST_HEAD(&mm->head_node.hole_stack);

	mm->head_node.hole_follows = 1;

	mm->head_node.scanned_block = 0;

	mm->head_node.scanned_prev_free = 0;

	mm->head_node.scanned_next_free = 0;

	mm->head_node.mm = mm;

	mm->head_node.start = start + size;

	mm->head_node.size = start - mm->head_node.start;

	list_add_tail(&mm->head_node.hole_stack, &mm->hole_stack);



	return 0;

}

EXPORT_SYMBOL(drm_mm_init);



void drm_mm_takedown(struct drm_mm * mm)

{

	struct list_head *bnode = mm->free_stack.next;

	struct drm_mm_node *entry;

	struct drm_mm_node *next;



	entry = list_entry(bnode, struct drm_mm_node, free_stack);

	struct drm_mm_node *entry, *next;



	if (entry->node_list.next != &mm->node_list ||

	    entry->free_stack.next != &mm->free_stack) {

	if (!list_empty(&mm->head_node.node_list)) {

		DRM_ERROR("Memory manager not clean. Delaying takedown\n");

		return;

	}



	list_del(&entry->free_stack);

	list_del(&entry->node_list);

	kfree(entry);



	spin_lock(&mm->unused_lock);

	list_for_each_entry_safe(entry, next, &mm->unused_nodes, free_stack) {

		list_del(&entry->free_stack);

	list_for_each_entry_safe(entry, next, &mm->unused_nodes, node_list) {

		list_del(&entry->node_list);

		kfree(entry);

		--mm->num_unused;

	}
@@ -620,19 +547,37 @@ EXPORT_SYMBOL(drm_mm_takedown); 
void drm_mm_debug_table(struct drm_mm *mm, const char *prefix)

{

	struct drm_mm_node *entry;

	int total_used = 0, total_free = 0, total = 0;



	list_for_each_entry(entry, &mm->node_list, node_list) {

		printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8ld: %s\n",

	unsigned long total_used = 0, total_free = 0, total = 0;

	unsigned long hole_start, hole_end, hole_size;



	hole_start = drm_mm_hole_node_start(&mm->head_node);

	hole_end = drm_mm_hole_node_end(&mm->head_node);

	hole_size = hole_end - hole_start;

	if (hole_size)

		printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8lu: free\n",

			prefix, hole_start, hole_end,

			hole_size);

	total_free += hole_size;



	drm_mm_for_each_node(entry, mm) {

		printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8lu: used\n",

			prefix, entry->start, entry->start + entry->size,

			entry->size, entry->free ? "free" : "used");

		total += entry->size;

		if (entry->free)

			total_free += entry->size;

		else

			entry->size);

		total_used += entry->size;



		if (entry->hole_follows) {

			hole_start = drm_mm_hole_node_start(entry);

			hole_end = drm_mm_hole_node_end(entry);

			hole_size = hole_end - hole_start;

			printk(KERN_DEBUG "%s 0x%08lx-0x%08lx: %8lu: free\n",

				prefix, hole_start, hole_end,

				hole_size);

			total_free += hole_size;

		}

	printk(KERN_DEBUG "%s total: %d, used %d free %d\n", prefix, total,

	}

	total = total_free + total_used;



	printk(KERN_DEBUG "%s total: %lu, used %lu free %lu\n", prefix, total,

		total_used, total_free);

}

EXPORT_SYMBOL(drm_mm_debug_table);
@@ -641,17 +586,34 @@ EXPORT_SYMBOL(drm_mm_debug_table); 
int drm_mm_dump_table(struct seq_file *m, struct drm_mm *mm)

{

	struct drm_mm_node *entry;

	int total_used = 0, total_free = 0, total = 0;



	list_for_each_entry(entry, &mm->node_list, node_list) {

		seq_printf(m, "0x%08lx-0x%08lx: 0x%08lx: %s\n", entry->start, entry->start + entry->size, entry->size, entry->free ? "free" : "used");

		total += entry->size;

		if (entry->free)

			total_free += entry->size;

		else

	unsigned long total_used = 0, total_free = 0, total = 0;

	unsigned long hole_start, hole_end, hole_size;



	hole_start = drm_mm_hole_node_start(&mm->head_node);

	hole_end = drm_mm_hole_node_end(&mm->head_node);

	hole_size = hole_end - hole_start;

	if (hole_size)

		seq_printf(m, "0x%08lx-0x%08lx: 0x%08lx: free\n",

				hole_start, hole_end, hole_size);

	total_free += hole_size;



	drm_mm_for_each_node(entry, mm) {

		seq_printf(m, "0x%08lx-0x%08lx: 0x%08lx: used\n",

				entry->start, entry->start + entry->size,

				entry->size);

		total_used += entry->size;

		if (entry->hole_follows) {

			hole_start = drm_mm_hole_node_start(&mm->head_node);

			hole_end = drm_mm_hole_node_end(&mm->head_node);

			hole_size = hole_end - hole_start;

			seq_printf(m, "0x%08lx-0x%08lx: 0x%08lx: free\n",

					hole_start, hole_end, hole_size);

			total_free += hole_size;

		}

	}

	seq_printf(m, "total: %d, used %d free %d\n", total, total_used, total_free);

	total = total_free + total_used;



	seq_printf(m, "total: %lu, used %lu free %lu\n", total, total_used, total_free);

	return 0;

}

EXPORT_SYMBOL(drm_mm_dump_table);

include/drm/drm_mm.h

+12 −9
Original line number Diff line number Diff line
@@ -42,23 +42,24 @@ 
#endif



struct drm_mm_node {

	struct list_head free_stack;

	struct list_head node_list;

	unsigned free : 1;

	struct list_head hole_stack;

	unsigned hole_follows : 1;

	unsigned scanned_block : 1;

	unsigned scanned_prev_free : 1;

	unsigned scanned_next_free : 1;

	unsigned scanned_preceeds_hole : 1;

	unsigned long start;

	unsigned long size;

	struct drm_mm *mm;

};



struct drm_mm {

	/* List of free memory blocks, most recently freed ordered. */

	struct list_head free_stack;

	/* List of all memory nodes, ordered according to the (increasing) start

	 * address of the memory node. */

	struct list_head node_list;

	/* List of all memory nodes that immediatly preceed a free hole. */

	struct list_head hole_stack;

	/* head_node.node_list is the list of all memory nodes, ordered

	 * according to the (increasing) start address of the memory node. */

	struct drm_mm_node head_node;

	struct list_head unused_nodes;

	int num_unused;

	spinlock_t unused_lock;
@@ -74,9 +75,11 @@ struct drm_mm { 


static inline bool drm_mm_initialized(struct drm_mm *mm)

{

	return mm->free_stack.next;

	return mm->hole_stack.next;

}



#define drm_mm_for_each_node(entry, mm) list_for_each_entry(entry, \

						&(mm)->head_node.node_list, \

						node_list);

/*

 * Basic range manager support (drm_mm.c)

 */

CRA Git | Maintained and supported by SUSTech CRA and CCSE