Commit 9b89a035 authored by Matthew Wilcox's avatar Matthew Wilcox
Browse files

xarray: Add XArray marks 



XArray marks are like the radix tree tags, only slightly more strongly
typed.  They are renamed in order to distinguish them from tagged
pointers.  This commit adds the basic get/set/clear operations.

Signed-off-by: default avatarMatthew Wilcox <willy@infradead.org>
parent ad3d6c72

include/linux/xarray.h

+63 −0
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ 


#include <linux/bug.h>

#include <linux/compiler.h>

#include <linux/gfp.h>

#include <linux/kconfig.h>

#include <linux/kernel.h>

#include <linux/rcupdate.h>
@@ -197,6 +198,20 @@ static inline int xa_err(void *entry) 
	return 0;

}



typedef unsigned __bitwise xa_mark_t;

#define XA_MARK_0		((__force xa_mark_t)0U)

#define XA_MARK_1		((__force xa_mark_t)1U)

#define XA_MARK_2		((__force xa_mark_t)2U)

#define XA_PRESENT		((__force xa_mark_t)8U)

#define XA_MARK_MAX		XA_MARK_2



/*

 * Values for xa_flags.  The radix tree stores its GFP flags in the xa_flags,

 * and we remain compatible with that.

 */

#define XA_FLAGS_MARK(mark)	((__force gfp_t)((1U << __GFP_BITS_SHIFT) << \

						(__force unsigned)(mark)))



/**

 * struct xarray - The anchor of the XArray.

 * @xa_lock: Lock that protects the contents of the XArray.
@@ -252,6 +267,9 @@ struct xarray { 


void xa_init_flags(struct xarray *, gfp_t flags);

void *xa_load(struct xarray *, unsigned long index);

bool xa_get_mark(struct xarray *, unsigned long index, xa_mark_t);

void xa_set_mark(struct xarray *, unsigned long index, xa_mark_t);

void xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t);



/**

 * xa_init() - Initialise an empty XArray.
@@ -278,6 +296,19 @@ static inline bool xa_empty(const struct xarray *xa) 
	return xa->xa_head == NULL;

}



/**

 * xa_marked() - Inquire whether any entry in this array has a mark set

 * @xa: Array

 * @mark: Mark value

 *

 * Context: Any context.

 * Return: %true if any entry has this mark set.

 */

static inline bool xa_marked(const struct xarray *xa, xa_mark_t mark)

{

	return xa->xa_flags & XA_FLAGS_MARK(mark);

}



#define xa_trylock(xa)		spin_trylock(&(xa)->xa_lock)

#define xa_lock(xa)		spin_lock(&(xa)->xa_lock)

#define xa_unlock(xa)		spin_unlock(&(xa)->xa_lock)
@@ -290,6 +321,12 @@ static inline bool xa_empty(const struct xarray *xa) 
#define xa_unlock_irqrestore(xa, flags) \

				spin_unlock_irqrestore(&(xa)->xa_lock, flags)



/*

 * Versions of the normal API which require the caller to hold the xa_lock.

 */

void __xa_set_mark(struct xarray *, unsigned long index, xa_mark_t);

void __xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t);



/* Everything below here is the Advanced API.  Proceed with caution. */



/*
@@ -388,6 +425,22 @@ static inline void *xa_entry_locked(const struct xarray *xa, 
						lockdep_is_held(&xa->xa_lock));

}



/* Private */

static inline struct xa_node *xa_parent(const struct xarray *xa,

					const struct xa_node *node)

{

	return rcu_dereference_check(node->parent,

						lockdep_is_held(&xa->xa_lock));

}



/* Private */

static inline struct xa_node *xa_parent_locked(const struct xarray *xa,

					const struct xa_node *node)

{

	return rcu_dereference_protected(node->parent,

						lockdep_is_held(&xa->xa_lock));

}



/* Private */

static inline struct xa_node *xa_to_node(const void *entry)

{
@@ -588,6 +641,12 @@ static inline bool xas_valid(const struct xa_state *xas) 
	return !xas_invalid(xas);

}



/* True if the pointer is something other than a node */

static inline bool xas_not_node(struct xa_node *node)

{

	return ((unsigned long)node & 3) || !node;

}



/**

 * xas_reset() - Reset an XArray operation state.

 * @xas: XArray operation state.
@@ -625,6 +684,10 @@ static inline bool xas_retry(struct xa_state *xas, const void *entry) 


void *xas_load(struct xa_state *);



bool xas_get_mark(const struct xa_state *, xa_mark_t);

void xas_set_mark(const struct xa_state *, xa_mark_t);

void xas_clear_mark(const struct xa_state *, xa_mark_t);



/**

 * xas_reload() - Refetch an entry from the xarray.

 * @xas: XArray operation state.

lib/test_xarray.c

+34 −0
Original line number Diff line number Diff line
@@ -67,11 +67,45 @@ static noinline void check_xa_load(struct xarray *xa) 
	XA_BUG_ON(xa, !xa_empty(xa));

}



static noinline void check_xa_mark_1(struct xarray *xa, unsigned long index)

{

	/* NULL elements have no marks set */

	XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0));

	xa_set_mark(xa, index, XA_MARK_0);

	XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0));



	/* Storing a pointer will not make a mark appear */

	XA_BUG_ON(xa, xa_store_index(xa, index, GFP_KERNEL) != NULL);

	XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0));

	xa_set_mark(xa, index, XA_MARK_0);

	XA_BUG_ON(xa, !xa_get_mark(xa, index, XA_MARK_0));



	/* Setting one mark will not set another mark */

	XA_BUG_ON(xa, xa_get_mark(xa, index + 1, XA_MARK_0));

	XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_1));



	/* Storing NULL clears marks, and they can't be set again */

	xa_erase_index(xa, index);

	XA_BUG_ON(xa, !xa_empty(xa));

	XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0));

	xa_set_mark(xa, index, XA_MARK_0);

	XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0));

}



static noinline void check_xa_mark(struct xarray *xa)

{

	unsigned long index;



	for (index = 0; index < 16384; index += 4)

		check_xa_mark_1(xa, index);

}



static RADIX_TREE(array, GFP_KERNEL);



static int xarray_checks(void)

{

	check_xa_load(&array);

	check_xa_mark(&array);



	printk("XArray: %u of %u tests passed\n", tests_passed, tests_run);

	return (tests_run == tests_passed) ? 0 : -EINVAL;

lib/xarray.c

+230 −2
Original line number Diff line number Diff line
@@ -5,6 +5,7 @@ 
 * Author: Matthew Wilcox <willy@infradead.org>

 */



#include <linux/bitmap.h>

#include <linux/export.h>

#include <linux/xarray.h>
@@ -24,6 +25,48 @@ 
 * @entry refers to something stored in a slot in the xarray

 */



static inline void xa_mark_set(struct xarray *xa, xa_mark_t mark)

{

	if (!(xa->xa_flags & XA_FLAGS_MARK(mark)))

		xa->xa_flags |= XA_FLAGS_MARK(mark);

}



static inline void xa_mark_clear(struct xarray *xa, xa_mark_t mark)

{

	if (xa->xa_flags & XA_FLAGS_MARK(mark))

		xa->xa_flags &= ~(XA_FLAGS_MARK(mark));

}



static inline unsigned long *node_marks(struct xa_node *node, xa_mark_t mark)

{

	return node->marks[(__force unsigned)mark];

}



static inline bool node_get_mark(struct xa_node *node,

		unsigned int offset, xa_mark_t mark)

{

	return test_bit(offset, node_marks(node, mark));

}



/* returns true if the bit was set */

static inline bool node_set_mark(struct xa_node *node, unsigned int offset,

				xa_mark_t mark)

{

	return __test_and_set_bit(offset, node_marks(node, mark));

}



/* returns true if the bit was set */

static inline bool node_clear_mark(struct xa_node *node, unsigned int offset,

				xa_mark_t mark)

{

	return __test_and_clear_bit(offset, node_marks(node, mark));

}



static inline bool node_any_mark(struct xa_node *node, xa_mark_t mark)

{

	return !bitmap_empty(node_marks(node, mark), XA_CHUNK_SIZE);

}



/* extracts the offset within this node from the index */

static unsigned int get_offset(unsigned long index, struct xa_node *node)

{
@@ -118,6 +161,85 @@ void *xas_load(struct xa_state *xas) 
}

EXPORT_SYMBOL_GPL(xas_load);



/**

 * xas_get_mark() - Returns the state of this mark.

 * @xas: XArray operation state.

 * @mark: Mark number.

 *

 * Return: true if the mark is set, false if the mark is clear or @xas

 * is in an error state.

 */

bool xas_get_mark(const struct xa_state *xas, xa_mark_t mark)

{

	if (xas_invalid(xas))

		return false;

	if (!xas->xa_node)

		return xa_marked(xas->xa, mark);

	return node_get_mark(xas->xa_node, xas->xa_offset, mark);

}

EXPORT_SYMBOL_GPL(xas_get_mark);



/**

 * xas_set_mark() - Sets the mark on this entry and its parents.

 * @xas: XArray operation state.

 * @mark: Mark number.

 *

 * Sets the specified mark on this entry, and walks up the tree setting it

 * on all the ancestor entries.  Does nothing if @xas has not been walked to

 * an entry, or is in an error state.

 */

void xas_set_mark(const struct xa_state *xas, xa_mark_t mark)

{

	struct xa_node *node = xas->xa_node;

	unsigned int offset = xas->xa_offset;



	if (xas_invalid(xas))

		return;



	while (node) {

		if (node_set_mark(node, offset, mark))

			return;

		offset = node->offset;

		node = xa_parent_locked(xas->xa, node);

	}



	if (!xa_marked(xas->xa, mark))

		xa_mark_set(xas->xa, mark);

}

EXPORT_SYMBOL_GPL(xas_set_mark);



/**

 * xas_clear_mark() - Clears the mark on this entry and its parents.

 * @xas: XArray operation state.

 * @mark: Mark number.

 *

 * Clears the specified mark on this entry, and walks back to the head

 * attempting to clear it on all the ancestor entries.  Does nothing if

 * @xas has not been walked to an entry, or is in an error state.

 */

void xas_clear_mark(const struct xa_state *xas, xa_mark_t mark)

{

	struct xa_node *node = xas->xa_node;

	unsigned int offset = xas->xa_offset;



	if (xas_invalid(xas))

		return;



	while (node) {

		if (!node_clear_mark(node, offset, mark))

			return;

		if (node_any_mark(node, mark))

			return;



		offset = node->offset;

		node = xa_parent_locked(xas->xa, node);

	}



	if (xa_marked(xas->xa, mark))

		xa_mark_clear(xas->xa, mark);

}

EXPORT_SYMBOL_GPL(xas_clear_mark);



/**

 * xa_init_flags() - Initialise an empty XArray with flags.

 * @xa: XArray.
@@ -160,6 +282,112 @@ void *xa_load(struct xarray *xa, unsigned long index) 
}

EXPORT_SYMBOL(xa_load);



/**

 * __xa_set_mark() - Set this mark on this entry while locked.

 * @xa: XArray.

 * @index: Index of entry.

 * @mark: Mark number.

 *

 * Attempting to set a mark on a NULL entry does not succeed.

 *

 * Context: Any context.  Expects xa_lock to be held on entry.

 */

void __xa_set_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)

{

	XA_STATE(xas, xa, index);

	void *entry = xas_load(&xas);



	if (entry)

		xas_set_mark(&xas, mark);

}

EXPORT_SYMBOL_GPL(__xa_set_mark);



/**

 * __xa_clear_mark() - Clear this mark on this entry while locked.

 * @xa: XArray.

 * @index: Index of entry.

 * @mark: Mark number.

 *

 * Context: Any context.  Expects xa_lock to be held on entry.

 */

void __xa_clear_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)

{

	XA_STATE(xas, xa, index);

	void *entry = xas_load(&xas);



	if (entry)

		xas_clear_mark(&xas, mark);

}

EXPORT_SYMBOL_GPL(__xa_clear_mark);



/**

 * xa_get_mark() - Inquire whether this mark is set on this entry.

 * @xa: XArray.

 * @index: Index of entry.

 * @mark: Mark number.

 *

 * This function uses the RCU read lock, so the result may be out of date

 * by the time it returns.  If you need the result to be stable, use a lock.

 *

 * Context: Any context.  Takes and releases the RCU lock.

 * Return: True if the entry at @index has this mark set, false if it doesn't.

 */

bool xa_get_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)

{

	XA_STATE(xas, xa, index);

	void *entry;



	rcu_read_lock();

	entry = xas_start(&xas);

	while (xas_get_mark(&xas, mark)) {

		if (!xa_is_node(entry))

			goto found;

		entry = xas_descend(&xas, xa_to_node(entry));

	}

	rcu_read_unlock();

	return false;

 found:

	rcu_read_unlock();

	return true;

}

EXPORT_SYMBOL(xa_get_mark);



/**

 * xa_set_mark() - Set this mark on this entry.

 * @xa: XArray.

 * @index: Index of entry.

 * @mark: Mark number.

 *

 * Attempting to set a mark on a NULL entry does not succeed.

 *

 * Context: Process context.  Takes and releases the xa_lock.

 */

void xa_set_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)

{

	xa_lock(xa);

	__xa_set_mark(xa, index, mark);

	xa_unlock(xa);

}

EXPORT_SYMBOL(xa_set_mark);



/**

 * xa_clear_mark() - Clear this mark on this entry.

 * @xa: XArray.

 * @index: Index of entry.

 * @mark: Mark number.

 *

 * Clearing a mark always succeeds.

 *

 * Context: Process context.  Takes and releases the xa_lock.

 */

void xa_clear_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)

{

	xa_lock(xa);

	__xa_clear_mark(xa, index, mark);

	xa_unlock(xa);

}

EXPORT_SYMBOL(xa_clear_mark);



#ifdef XA_DEBUG

void xa_dump_node(const struct xa_node *node)

{
@@ -230,8 +458,8 @@ void xa_dump(const struct xarray *xa) 
	unsigned int shift = 0;



	pr_info("xarray: %px head %px flags %x marks %d %d %d\n", xa, entry,

			xa->xa_flags, radix_tree_tagged(xa, 0),

			radix_tree_tagged(xa, 1), radix_tree_tagged(xa, 2));

			xa->xa_flags, xa_marked(xa, XA_MARK_0),

			xa_marked(xa, XA_MARK_1), xa_marked(xa, XA_MARK_2));

	if (xa_is_node(entry))

		shift = xa_to_node(entry)->shift + XA_CHUNK_SHIFT;

	xa_dump_entry(entry, 0, shift);

tools/include/asm-generic/bitops.h

+1 −0
Original line number Diff line number Diff line
@@ -27,5 +27,6 @@ 
#include <asm-generic/bitops/hweight.h>



#include <asm-generic/bitops/atomic.h>

#include <asm-generic/bitops/non-atomic.h>



#endif /* __TOOLS_ASM_GENERIC_BITOPS_H */

tools/include/asm-generic/bitops/atomic.h

+0 −9
Original line number Diff line number Diff line
@@ -15,13 +15,4 @@ static inline void clear_bit(int nr, unsigned long *addr) 
	addr[nr / __BITS_PER_LONG] &= ~(1UL << (nr % __BITS_PER_LONG));

}



static __always_inline int test_bit(unsigned int nr, const unsigned long *addr)

{

	return ((1UL << (nr % __BITS_PER_LONG)) &

		(((unsigned long *)addr)[nr / __BITS_PER_LONG])) != 0;

}



#define __set_bit(nr, addr)	set_bit(nr, addr)

#define __clear_bit(nr, addr)	clear_bit(nr, addr)



#endif /* _TOOLS_LINUX_ASM_GENERIC_BITOPS_ATOMIC_H_ */

CRA Git | Maintained and supported by SUSTech CRA and CCSE