Commit f808c13f authored by Davidlohr Bueso's avatar Davidlohr Bueso Committed by Linus Torvalds
Browse files

lib/interval_tree: fast overlap detection 

Allow interval trees to quickly check for overlaps to avoid unnecesary
tree lookups in interval_tree_iter_first().

As of this patch, all interval tree flavors will require using a
'rb_root_cached' such that we can have the leftmost node easily
available.  While most users will make use of this feature, those with
special functions (in addition to the generic insert, delete, search
calls) will avoid using the cached option as they can do funky things
with insertions -- for example, vma_interval_tree_insert_after().

[jglisse@redhat.com: fix deadlock from typo vm_lock_anon_vma()]
  Link: http://lkml.kernel.org/r/20170808225719.20723-1-jglisse@redhat.com
Link: http://lkml.kernel.org/r/20170719014603.19029-12-dave@stgolabs.net


Signed-off-by: default avatarDavidlohr Bueso <dbueso@suse.de>
Signed-off-by: default avatarJérôme Glisse <jglisse@redhat.com>
Acked-by: default avatarChristian König <christian.koenig@amd.com>
Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: default avatarDoug Ledford <dledford@redhat.com>
Acked-by: default avatarMichael S. Tsirkin <mst@redhat.com>
Cc: David Airlie <airlied@linux.ie>
Cc: Jason Wang <jasowang@redhat.com>
Cc: Christian Benvenuti <benve@cisco.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 09663c86

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c

+4 −4
Original line number Diff line number Diff line
@@ -51,7 +51,7 @@ struct amdgpu_mn { 


	/* objects protected by lock */

	struct mutex		lock;

	struct rb_root		objects;

	struct rb_root_cached	objects;

};



struct amdgpu_mn_node {
@@ -76,8 +76,8 @@ static void amdgpu_mn_destroy(struct work_struct *work) 
	mutex_lock(&adev->mn_lock);

	mutex_lock(&rmn->lock);

	hash_del(&rmn->node);

	rbtree_postorder_for_each_entry_safe(node, next_node, &rmn->objects,

					     it.rb) {

	rbtree_postorder_for_each_entry_safe(node, next_node,

					     &rmn->objects.rb_root, it.rb) {

		list_for_each_entry_safe(bo, next_bo, &node->bos, mn_list) {

			bo->mn = NULL;

			list_del_init(&bo->mn_list);
@@ -221,7 +221,7 @@ static struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev) 
	rmn->mm = mm;

	rmn->mn.ops = &amdgpu_mn_ops;

	mutex_init(&rmn->lock);

	rmn->objects = RB_ROOT;

	rmn->objects = RB_ROOT_CACHED;



	r = __mmu_notifier_register(&rmn->mn, mm);

	if (r)

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c

+4 −3
Original line number Diff line number Diff line
@@ -2475,7 +2475,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, 
	u64 flags;

	uint64_t init_pde_value = 0;



	vm->va = RB_ROOT;

	vm->va = RB_ROOT_CACHED;

	vm->client_id = atomic64_inc_return(&adev->vm_manager.client_counter);

	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)

		vm->reserved_vmid[i] = NULL;
@@ -2596,10 +2596,11 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) 


	amd_sched_entity_fini(vm->entity.sched, &vm->entity);



	if (!RB_EMPTY_ROOT(&vm->va)) {

	if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {

		dev_err(adev->dev, "still active bo inside vm\n");

	}

	rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, rb) {

	rbtree_postorder_for_each_entry_safe(mapping, tmp,

					     &vm->va.rb_root, rb) {

		list_del(&mapping->list);

		amdgpu_vm_it_remove(mapping, &vm->va);

		kfree(mapping);

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h

+1 −1
Original line number Diff line number Diff line
@@ -118,7 +118,7 @@ struct amdgpu_vm_pt { 


struct amdgpu_vm {

	/* tree of virtual addresses mapped */

	struct rb_root		va;

	struct rb_root_cached	va;



	/* protecting invalidated */

	spinlock_t		status_lock;

drivers/gpu/drm/drm_mm.c

+11 −8
Original line number Diff line number Diff line
@@ -169,7 +169,7 @@ INTERVAL_TREE_DEFINE(struct drm_mm_node, rb, 
struct drm_mm_node *

__drm_mm_interval_first(const struct drm_mm *mm, u64 start, u64 last)

{

	return drm_mm_interval_tree_iter_first((struct rb_root *)&mm->interval_tree,

	return drm_mm_interval_tree_iter_first((struct rb_root_cached *)&mm->interval_tree,

					       start, last) ?: (struct drm_mm_node *)&mm->head_node;

}

EXPORT_SYMBOL(__drm_mm_interval_first);
@@ -180,6 +180,7 @@ static void drm_mm_interval_tree_add_node(struct drm_mm_node *hole_node, 
	struct drm_mm *mm = hole_node->mm;

	struct rb_node **link, *rb;

	struct drm_mm_node *parent;

	bool leftmost = true;



	node->__subtree_last = LAST(node);
@@ -196,9 +197,10 @@ static void drm_mm_interval_tree_add_node(struct drm_mm_node *hole_node, 


		rb = &hole_node->rb;

		link = &hole_node->rb.rb_right;

		leftmost = false;

	} else {

		rb = NULL;

		link = &mm->interval_tree.rb_node;

		link = &mm->interval_tree.rb_root.rb_node;

	}



	while (*link) {
@@ -208,13 +210,14 @@ static void drm_mm_interval_tree_add_node(struct drm_mm_node *hole_node, 
			parent->__subtree_last = node->__subtree_last;

		if (node->start < parent->start)

			link = &parent->rb.rb_left;

		else

		else {

			link = &parent->rb.rb_right;

			leftmost = true;

		}

	}



	rb_link_node(&node->rb, rb, link);

	rb_insert_augmented(&node->rb,

			    &mm->interval_tree,

	rb_insert_augmented_cached(&node->rb, &mm->interval_tree, leftmost,

				   &drm_mm_interval_tree_augment);

}
@@ -577,7 +580,7 @@ void drm_mm_replace_node(struct drm_mm_node *old, struct drm_mm_node *new) 
	*new = *old;



	list_replace(&old->node_list, &new->node_list);

	rb_replace_node(&old->rb, &new->rb, &old->mm->interval_tree);

	rb_replace_node(&old->rb, &new->rb, &old->mm->interval_tree.rb_root);



	if (drm_mm_hole_follows(old)) {

		list_replace(&old->hole_stack, &new->hole_stack);
@@ -863,7 +866,7 @@ void drm_mm_init(struct drm_mm *mm, u64 start, u64 size) 
	mm->color_adjust = NULL;



	INIT_LIST_HEAD(&mm->hole_stack);

	mm->interval_tree = RB_ROOT;

	mm->interval_tree = RB_ROOT_CACHED;

	mm->holes_size = RB_ROOT;

	mm->holes_addr = RB_ROOT;

drivers/gpu/drm/drm_vma_manager.c

+1 −1
Original line number Diff line number Diff line
@@ -147,7 +147,7 @@ struct drm_vma_offset_node *drm_vma_offset_lookup_locked(struct drm_vma_offset_m 
	struct rb_node *iter;

	unsigned long offset;



	iter = mgr->vm_addr_space_mm.interval_tree.rb_node;

	iter = mgr->vm_addr_space_mm.interval_tree.rb_root.rb_node;

	best = NULL;



	while (likely(iter)) {

CRA Git | Maintained and supported by SUSTech CRA and CCSE