Commit 268f42de authored by Matthew Wilcox's avatar Matthew Wilcox Committed by Linus Torvalds
Browse files

radix-tree: delete radix_tree_range_tag_if_tagged() 

This is an exceptionally complicated function with just one caller
(tag_pages_for_writeback).  We devote a large portion of the runtime of
the test suite to testing this one function which has one caller.  By
introducing the new function radix_tree_iter_tag_set(), we can eliminate
all of the complexity while keeping the performance.  The caller can now
use a fairly standard radix_tree_for_each() loop, and it doesn't need to
worry about tricksy things like 'start' wrapping.

The test suite continues to spend a large amount of time investigating
this function, but now it's testing the underlying primitives such as
radix_tree_iter_resume() and the radix_tree_for_each_tagged() iterator
which are also used by other parts of the kernel.

Link: http://lkml.kernel.org/r/1480369871-5271-57-git-send-email-mawilcox@linuxonhyperv.com


Signed-off-by: default avatarMatthew Wilcox <willy@infradead.org>
Tested-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Konstantin Khlebnikov <koct9i@gmail.com>
Cc: Ross Zwisler <ross.zwisler@linux.intel.com>
Cc: Matthew Wilcox <mawilcox@microsoft.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 478922e2

include/linux/radix-tree.h

+37 −37
Original line number Diff line number Diff line
@@ -120,6 +120,41 @@ static inline bool radix_tree_empty(struct radix_tree_root *root) 
	return root->rnode == NULL;

}



/**

 * struct radix_tree_iter - radix tree iterator state

 *

 * @index:	index of current slot

 * @next_index:	one beyond the last index for this chunk

 * @tags:	bit-mask for tag-iterating

 * @node:	node that contains current slot

 * @shift:	shift for the node that holds our slots

 *

 * This radix tree iterator works in terms of "chunks" of slots.  A chunk is a

 * subinterval of slots contained within one radix tree leaf node.  It is

 * described by a pointer to its first slot and a struct radix_tree_iter

 * which holds the chunk's position in the tree and its size.  For tagged

 * iteration radix_tree_iter also holds the slots' bit-mask for one chosen

 * radix tree tag.

 */

struct radix_tree_iter {

	unsigned long	index;

	unsigned long	next_index;

	unsigned long	tags;

	struct radix_tree_node *node;

#ifdef CONFIG_RADIX_TREE_MULTIORDER

	unsigned int	shift;

#endif

};



static inline unsigned int iter_shift(const struct radix_tree_iter *iter)

{

#ifdef CONFIG_RADIX_TREE_MULTIORDER

	return iter->shift;

#else

	return 0;

#endif

}



/**

 * Radix-tree synchronization

 *
@@ -283,6 +318,8 @@ void *radix_tree_tag_clear(struct radix_tree_root *root, 
			unsigned long index, unsigned int tag);

int radix_tree_tag_get(struct radix_tree_root *root,

			unsigned long index, unsigned int tag);

void radix_tree_iter_tag_set(struct radix_tree_root *root,

		const struct radix_tree_iter *iter, unsigned int tag);

unsigned int

radix_tree_gang_lookup_tag(struct radix_tree_root *root, void **results,

		unsigned long first_index, unsigned int max_items,
@@ -291,10 +328,6 @@ unsigned int 
radix_tree_gang_lookup_tag_slot(struct radix_tree_root *root, void ***results,

		unsigned long first_index, unsigned int max_items,

		unsigned int tag);

unsigned long radix_tree_range_tag_if_tagged(struct radix_tree_root *root,

		unsigned long *first_indexp, unsigned long last_index,

		unsigned long nr_to_tag,

		unsigned int fromtag, unsigned int totag);

int radix_tree_tagged(struct radix_tree_root *root, unsigned int tag);



static inline void radix_tree_preload_end(void)
@@ -302,39 +335,6 @@ static inline void radix_tree_preload_end(void) 
	preempt_enable();

}



/**

 * struct radix_tree_iter - radix tree iterator state

 *

 * @index:	index of current slot

 * @next_index:	one beyond the last index for this chunk

 * @tags:	bit-mask for tag-iterating

 * @shift:	shift for the node that holds our slots

 *

 * This radix tree iterator works in terms of "chunks" of slots.  A chunk is a

 * subinterval of slots contained within one radix tree leaf node.  It is

 * described by a pointer to its first slot and a struct radix_tree_iter

 * which holds the chunk's position in the tree and its size.  For tagged

 * iteration radix_tree_iter also holds the slots' bit-mask for one chosen

 * radix tree tag.

 */

struct radix_tree_iter {

	unsigned long	index;

	unsigned long	next_index;

	unsigned long	tags;

#ifdef CONFIG_RADIX_TREE_MULTIORDER

	unsigned int	shift;

#endif

};



static inline unsigned int iter_shift(struct radix_tree_iter *iter)

{

#ifdef CONFIG_RADIX_TREE_MULTIORDER

	return iter->shift;

#else

	return 0;

#endif

}



#define RADIX_TREE_ITER_TAG_MASK	0x00FF	/* tag index in lower byte */

#define RADIX_TREE_ITER_TAGGED		0x0100	/* lookup tagged slots */

#define RADIX_TREE_ITER_CONTIG		0x0200	/* stop at first hole */

lib/radix-tree.c

+20 −97
Original line number Diff line number Diff line
@@ -219,6 +219,11 @@ radix_tree_find_next_bit(struct radix_tree_node *node, unsigned int tag, 
	return RADIX_TREE_MAP_SIZE;

}



static unsigned int iter_offset(const struct radix_tree_iter *iter)

{

	return (iter->index >> iter_shift(iter)) & RADIX_TREE_MAP_MASK;

}



/*

 * The maximum index which can be stored in a radix tree

 */
@@ -1014,6 +1019,18 @@ static void node_tag_set(struct radix_tree_root *root, 
		root_tag_set(root, tag);

}



/**

 * radix_tree_iter_tag_set - set a tag on the current iterator entry

 * @root:	radix tree root

 * @iter:	iterator state

 * @tag:	tag to set

 */

void radix_tree_iter_tag_set(struct radix_tree_root *root,

			const struct radix_tree_iter *iter, unsigned int tag)

{

	node_tag_set(root, iter->node, tag, iter_offset(iter));

}



/**

 *	radix_tree_tag_clear - clear a tag on a radix tree node

 *	@root:		radix tree root
@@ -1164,6 +1181,7 @@ void ** __radix_tree_next_slot(void **slot, struct radix_tree_iter *iter, 
		if (node == RADIX_TREE_RETRY)

			return slot;

		node = entry_to_node(node);

		iter->node = node;

		iter->shift = node->shift;



		if (flags & RADIX_TREE_ITER_TAGGED) {
@@ -1266,6 +1284,7 @@ void **radix_tree_next_chunk(struct radix_tree_root *root, 
		iter->index = index;

		iter->next_index = maxindex + 1;

		iter->tags = 1;

		iter->node = NULL;

		__set_iter_shift(iter, 0);

		return (void **)&root->rnode;

	}
@@ -1308,6 +1327,7 @@ void **radix_tree_next_chunk(struct radix_tree_root *root, 
	/* Update the iterator state */

	iter->index = (index &~ node_maxindex(node)) | (offset << node->shift);

	iter->next_index = (index | node_maxindex(node)) + 1;

	iter->node = node;

	__set_iter_shift(iter, node->shift);



	if (flags & RADIX_TREE_ITER_TAGGED)
@@ -1317,103 +1337,6 @@ void **radix_tree_next_chunk(struct radix_tree_root *root, 
}

EXPORT_SYMBOL(radix_tree_next_chunk);



/**

 * radix_tree_range_tag_if_tagged - for each item in given range set given

 *				   tag if item has another tag set

 * @root:		radix tree root

 * @first_indexp:	pointer to a starting index of a range to scan

 * @last_index:		last index of a range to scan

 * @nr_to_tag:		maximum number items to tag

 * @iftag:		tag index to test

 * @settag:		tag index to set if tested tag is set

 *

 * This function scans range of radix tree from first_index to last_index

 * (inclusive).  For each item in the range if iftag is set, the function sets

 * also settag. The function stops either after tagging nr_to_tag items or

 * after reaching last_index.

 *

 * The tags must be set from the leaf level only and propagated back up the

 * path to the root. We must do this so that we resolve the full path before

 * setting any tags on intermediate nodes. If we set tags as we descend, then

 * we can get to the leaf node and find that the index that has the iftag

 * set is outside the range we are scanning. This reults in dangling tags and

 * can lead to problems with later tag operations (e.g. livelocks on lookups).

 *

 * The function returns the number of leaves where the tag was set and sets

 * *first_indexp to the first unscanned index.

 * WARNING! *first_indexp can wrap if last_index is ULONG_MAX. Caller must

 * be prepared to handle that.

 */

unsigned long radix_tree_range_tag_if_tagged(struct radix_tree_root *root,

		unsigned long *first_indexp, unsigned long last_index,

		unsigned long nr_to_tag,

		unsigned int iftag, unsigned int settag)

{

	struct radix_tree_node *node, *child;

	unsigned long maxindex;

	unsigned long tagged = 0;

	unsigned long index = *first_indexp;



	radix_tree_load_root(root, &child, &maxindex);

	last_index = min(last_index, maxindex);

	if (index > last_index)

		return 0;

	if (!nr_to_tag)

		return 0;

	if (!root_tag_get(root, iftag)) {

		*first_indexp = last_index + 1;

		return 0;

	}

	if (!radix_tree_is_internal_node(child)) {

		*first_indexp = last_index + 1;

		root_tag_set(root, settag);

		return 1;

	}



	node = entry_to_node(child);



	for (;;) {

		unsigned offset = radix_tree_descend(node, &child, index);

		if (!child)

			goto next;

		if (!tag_get(node, iftag, offset))

			goto next;

		/* Sibling slots never have tags set on them */

		if (radix_tree_is_internal_node(child)) {

			node = entry_to_node(child);

			continue;

		}



		tagged++;

		node_tag_set(root, node, settag, offset);

 next:

		/* Go to next entry in node */

		index = ((index >> node->shift) + 1) << node->shift;

		/* Overflow can happen when last_index is ~0UL... */

		if (index > last_index || !index)

			break;

		offset = (index >> node->shift) & RADIX_TREE_MAP_MASK;

		while (offset == 0) {

			/*

			 * We've fully scanned this node. Go up. Because

			 * last_index is guaranteed to be in the tree, what

			 * we do below cannot wander astray.

			 */

			node = node->parent;

			offset = (index >> node->shift) & RADIX_TREE_MAP_MASK;

		}

		if (is_sibling_entry(node, node->slots[offset]))

			goto next;

		if (tagged >= nr_to_tag)

			break;

	}



	*first_indexp = index;



	return tagged;

}

EXPORT_SYMBOL(radix_tree_range_tag_if_tagged);



/**

 *	radix_tree_gang_lookup - perform multiple lookup on a radix tree

 *	@root:		radix tree root

mm/page-writeback.c

+18 −10
Original line number Diff line number Diff line
@@ -2106,18 +2106,26 @@ void tag_pages_for_writeback(struct address_space *mapping, 
			     pgoff_t start, pgoff_t end)

{

#define WRITEBACK_TAG_BATCH 4096

	unsigned long tagged;

	unsigned long tagged = 0;

	struct radix_tree_iter iter;

	void **slot;



	do {

	spin_lock_irq(&mapping->tree_lock);

		tagged = radix_tree_range_tag_if_tagged(&mapping->page_tree,

				&start, end, WRITEBACK_TAG_BATCH,

				PAGECACHE_TAG_DIRTY, PAGECACHE_TAG_TOWRITE);

	radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, start,

							PAGECACHE_TAG_DIRTY) {

		if (iter.index > end)

			break;

		radix_tree_iter_tag_set(&mapping->page_tree, &iter,

							PAGECACHE_TAG_TOWRITE);

		tagged++;

		if ((tagged % WRITEBACK_TAG_BATCH) != 0)

			continue;

		slot = radix_tree_iter_resume(slot, &iter);

		spin_unlock_irq(&mapping->tree_lock);

		WARN_ON_ONCE(tagged > WRITEBACK_TAG_BATCH);

		cond_resched();

		/* We check 'start' to handle wrapping when end == ~0UL */

	} while (tagged >= WRITEBACK_TAG_BATCH && start);

		spin_lock_irq(&mapping->tree_lock);

	}

	spin_unlock_irq(&mapping->tree_lock);

}

EXPORT_SYMBOL(tag_pages_for_writeback);

tools/testing/radix-tree/main.c

+4 −8
Original line number Diff line number Diff line
@@ -205,8 +205,7 @@ void copy_tag_check(void) 
	}



//	printf("\ncopying tags...\n");

	cur = start;

	tagged = radix_tree_range_tag_if_tagged(&tree, &cur, end, ITEMS, 0, 1);

	tagged = tag_tagged_items(&tree, NULL, start, end, ITEMS, 0, 1);



//	printf("checking copied tags\n");

	assert(tagged == count);
@@ -214,16 +213,13 @@ void copy_tag_check(void) 


	/* Copy tags in several rounds */

//	printf("\ncopying tags...\n");

	cur = start;

	do {

	tmp = rand() % (count / 10 + 2);

		tagged = radix_tree_range_tag_if_tagged(&tree, &cur, end, tmp, 0, 2);

	} while (tmp == tagged);

	tagged = tag_tagged_items(&tree, NULL, start, end, tmp, 0, 2);

	assert(tagged == count);



//	printf("%lu %lu %lu\n", tagged, tmp, count);

//	printf("checking copied tags\n");

	check_copied_tags(&tree, start, end, idx, ITEMS, 0, 2);

	assert(tagged < tmp);

	verify_tag_consistency(&tree, 0);

	verify_tag_consistency(&tree, 1);

	verify_tag_consistency(&tree, 2);

tools/testing/radix-tree/multiorder.c

+5 −8
Original line number Diff line number Diff line
@@ -26,7 +26,6 @@ static void __multiorder_tag_test(int index, int order) 
{

	RADIX_TREE(tree, GFP_KERNEL);

	int base, err, i;

	unsigned long first = 0;



	/* our canonical entry */

	base = index & ~((1 << order) - 1);
@@ -60,7 +59,7 @@ static void __multiorder_tag_test(int index, int order) 
		assert(!radix_tree_tag_get(&tree, i, 1));

	}



	assert(radix_tree_range_tag_if_tagged(&tree, &first, ~0UL, 10, 0, 1) == 1);

	assert(tag_tagged_items(&tree, NULL, 0, ~0UL, 10, 0, 1) == 1);

	assert(radix_tree_tag_clear(&tree, index, 0));



	for_each_index(i, base, order) {
@@ -251,7 +250,6 @@ void multiorder_tagged_iteration(void) 
	RADIX_TREE(tree, GFP_KERNEL);

	struct radix_tree_iter iter;

	void **slot;

	unsigned long first = 0;

	int i, j;



	printf("Multiorder tagged iteration test\n");
@@ -296,8 +294,8 @@ void multiorder_tagged_iteration(void) 
		}

	}



	radix_tree_range_tag_if_tagged(&tree, &first, ~0UL,

					MT_NUM_ENTRIES, 1, 2);

	assert(tag_tagged_items(&tree, NULL, 0, ~0UL, TAG_ENTRIES, 1, 2) ==

				TAG_ENTRIES);



	for (j = 0; j < 256; j++) {

		int mask, k;
@@ -323,9 +321,8 @@ void multiorder_tagged_iteration(void) 
		}

	}



	first = 1;

	radix_tree_range_tag_if_tagged(&tree, &first, ~0UL,

					MT_NUM_ENTRIES, 1, 0);

	assert(tag_tagged_items(&tree, NULL, 1, ~0UL, MT_NUM_ENTRIES * 2, 1, 0)

			== TAG_ENTRIES);

	i = 0;

	radix_tree_for_each_tagged(slot, &tree, &iter, 0, 0) {

		assert(iter.index == tag_index[i]);

CRA Git | Maintained and supported by SUSTech CRA and CCSE