Commit f379fdf1 authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'net_sched-idr' 



Chris Mi says:

====================
net/sched: Improve getting objects by indexes

Using current TC code, it is very slow to insert a lot of rules.

In order to improve the rules update rate in TC,
we introduced the following two changes:
        1) changed cls_flower to use IDR to manage the filters.
        2) changed all act_xxx modules to use IDR instead of
           a small hash table

But IDR has a limitation that it uses int. TC handle uses u32.
To make sure there is no regression, we add several new IDR APIs
to support unsigned long.

v2
==

Addressed Hannes's comment:
express idr_alloc in terms of idr_alloc_ext and most of the other functions
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents c2f8a6ce 65a206c0

include/linux/idr.h

+66 −3
Original line number Diff line number Diff line
@@ -80,19 +80,75 @@ static inline void idr_set_cursor(struct idr *idr, unsigned int val) 
 */



void idr_preload(gfp_t gfp_mask);

int idr_alloc(struct idr *, void *entry, int start, int end, gfp_t);



int idr_alloc_cmn(struct idr *idr, void *ptr, unsigned long *index,

		  unsigned long start, unsigned long end, gfp_t gfp,

		  bool ext);



/**

 * idr_alloc - allocate an id

 * @idr: idr handle

 * @ptr: pointer to be associated with the new id

 * @start: the minimum id (inclusive)

 * @end: the maximum id (exclusive)

 * @gfp: memory allocation flags

 *

 * Allocates an unused ID in the range [start, end).  Returns -ENOSPC

 * if there are no unused IDs in that range.

 *

 * Note that @end is treated as max when <= 0.  This is to always allow

 * using @start + N as @end as long as N is inside integer range.

 *

 * Simultaneous modifications to the @idr are not allowed and should be

 * prevented by the user, usually with a lock.  idr_alloc() may be called

 * concurrently with read-only accesses to the @idr, such as idr_find() and

 * idr_for_each_entry().

 */

static inline int idr_alloc(struct idr *idr, void *ptr,

			    int start, int end, gfp_t gfp)

{

	unsigned long id;

	int ret;



	if (WARN_ON_ONCE(start < 0))

		return -EINVAL;



	ret = idr_alloc_cmn(idr, ptr, &id, start, end, gfp, false);



	if (ret)

		return ret;



	return id;

}



static inline int idr_alloc_ext(struct idr *idr, void *ptr,

				unsigned long *index,

				unsigned long start,

				unsigned long end,

				gfp_t gfp)

{

	return idr_alloc_cmn(idr, ptr, index, start, end, gfp, true);

}



int idr_alloc_cyclic(struct idr *, void *entry, int start, int end, gfp_t);

int idr_for_each(const struct idr *,

		 int (*fn)(int id, void *p, void *data), void *data);

void *idr_get_next(struct idr *, int *nextid);

void *idr_get_next_ext(struct idr *idr, unsigned long *nextid);

void *idr_replace(struct idr *, void *, int id);

void *idr_replace_ext(struct idr *idr, void *ptr, unsigned long id);

void idr_destroy(struct idr *);



static inline void *idr_remove(struct idr *idr, int id)

static inline void *idr_remove_ext(struct idr *idr, unsigned long id)

{

	return radix_tree_delete_item(&idr->idr_rt, id, NULL);

}



static inline void *idr_remove(struct idr *idr, int id)

{

	return idr_remove_ext(idr, id);

}



static inline void idr_init(struct idr *idr)

{

	INIT_RADIX_TREE(&idr->idr_rt, IDR_RT_MARKER);
@@ -128,11 +184,16 @@ static inline void idr_preload_end(void) 
 * This function can be called under rcu_read_lock(), given that the leaf

 * pointers lifetimes are correctly managed.

 */

static inline void *idr_find(const struct idr *idr, int id)

static inline void *idr_find_ext(const struct idr *idr, unsigned long id)

{

	return radix_tree_lookup(&idr->idr_rt, id);

}



static inline void *idr_find(const struct idr *idr, int id)

{

	return idr_find_ext(idr, id);

}



/**

 * idr_for_each_entry - iterate over an idr's elements of a given type

 * @idr:     idr handle
@@ -145,6 +206,8 @@ static inline void *idr_find(const struct idr *idr, int id) 
 */

#define idr_for_each_entry(idr, entry, id)			\

	for (id = 0; ((entry) = idr_get_next(idr, &(id))) != NULL; ++id)

#define idr_for_each_entry_ext(idr, entry, id)			\

	for (id = 0; ((entry) = idr_get_next_ext(idr, &(id))) != NULL; ++id)



/**

 * idr_for_each_entry_continue - continue iteration over an idr's elements of a given type

include/linux/radix-tree.h

+19 −2
Original line number Diff line number Diff line
@@ -357,8 +357,25 @@ int radix_tree_split(struct radix_tree_root *, unsigned long index, 
			unsigned new_order);

int radix_tree_join(struct radix_tree_root *, unsigned long index,

			unsigned new_order, void *);

void __rcu **idr_get_free(struct radix_tree_root *, struct radix_tree_iter *,

			gfp_t, int end);



void __rcu **idr_get_free_cmn(struct radix_tree_root *root,

			      struct radix_tree_iter *iter, gfp_t gfp,

			      unsigned long max);

static inline void __rcu **idr_get_free(struct radix_tree_root *root,

					struct radix_tree_iter *iter,

					gfp_t gfp,

					int end)

{

	return idr_get_free_cmn(root, iter, gfp, end > 0 ? end - 1 : INT_MAX);

}



static inline void __rcu **idr_get_free_ext(struct radix_tree_root *root,

					    struct radix_tree_iter *iter,

					    gfp_t gfp,

					    unsigned long end)

{

	return idr_get_free_cmn(root, iter, gfp, end - 1);

}



enum {

	RADIX_TREE_ITER_TAG_MASK = 0x0f,	/* tag index in lower nybble */

include/net/act_api.h

+24 −52
Original line number Diff line number Diff line
@@ -10,12 +10,9 @@ 
#include <net/net_namespace.h>

#include <net/netns/generic.h>





struct tcf_hashinfo {

	struct hlist_head	*htab;

	unsigned int		hmask;

struct tcf_idrinfo {

	spinlock_t	lock;

	u32			index;

	struct idr	action_idr;

};



struct tc_action_ops;
@@ -25,9 +22,8 @@ struct tc_action { 
	__u32				type; /* for backward compat(TCA_OLD_COMPAT) */

	__u32				order;

	struct list_head		list;

	struct tcf_hashinfo		*hinfo;

	struct tcf_idrinfo		*idrinfo;



	struct hlist_node		tcfa_head;

	u32				tcfa_index;

	int				tcfa_refcnt;

	int				tcfa_bindcnt;
@@ -44,7 +40,6 @@ struct tc_action { 
	struct tc_cookie	*act_cookie;

	struct tcf_chain	*goto_chain;

};

#define tcf_head	common.tcfa_head

#define tcf_index	common.tcfa_index

#define tcf_refcnt	common.tcfa_refcnt

#define tcf_bindcnt	common.tcfa_bindcnt
@@ -57,27 +52,6 @@ struct tc_action { 
#define tcf_lock	common.tcfa_lock

#define tcf_rcu		common.tcfa_rcu



static inline unsigned int tcf_hash(u32 index, unsigned int hmask)

{

	return index & hmask;

}



static inline int tcf_hashinfo_init(struct tcf_hashinfo *hf, unsigned int mask)

{

	int i;



	spin_lock_init(&hf->lock);

	hf->index = 0;

	hf->hmask = mask;

	hf->htab = kzalloc((mask + 1) * sizeof(struct hlist_head),

			   GFP_KERNEL);

	if (!hf->htab)

		return -ENOMEM;

	for (i = 0; i < mask + 1; i++)

		INIT_HLIST_HEAD(&hf->htab[i]);

	return 0;

}



/* Update lastuse only if needed, to avoid dirtying a cache line.

 * We use a temp variable to avoid fetching jiffies twice.

 */
@@ -126,53 +100,51 @@ struct tc_action_ops { 
};



struct tc_action_net {

	struct tcf_hashinfo *hinfo;

	struct tcf_idrinfo *idrinfo;

	const struct tc_action_ops *ops;

};



static inline

int tc_action_net_init(struct tc_action_net *tn,

		       const struct tc_action_ops *ops, unsigned int mask)

		       const struct tc_action_ops *ops)

{

	int err = 0;



	tn->hinfo = kmalloc(sizeof(*tn->hinfo), GFP_KERNEL);

	if (!tn->hinfo)

	tn->idrinfo = kmalloc(sizeof(*tn->idrinfo), GFP_KERNEL);

	if (!tn->idrinfo)

		return -ENOMEM;

	tn->ops = ops;

	err = tcf_hashinfo_init(tn->hinfo, mask);

	if (err)

		kfree(tn->hinfo);

	spin_lock_init(&tn->idrinfo->lock);

	idr_init(&tn->idrinfo->action_idr);

	return err;

}



void tcf_hashinfo_destroy(const struct tc_action_ops *ops,

			  struct tcf_hashinfo *hinfo);

void tcf_idrinfo_destroy(const struct tc_action_ops *ops,

			 struct tcf_idrinfo *idrinfo);



static inline void tc_action_net_exit(struct tc_action_net *tn)

{

	tcf_hashinfo_destroy(tn->ops, tn->hinfo);

	kfree(tn->hinfo);

	tcf_idrinfo_destroy(tn->ops, tn->idrinfo);

	kfree(tn->idrinfo);

}



int tcf_generic_walker(struct tc_action_net *tn, struct sk_buff *skb,

		       struct netlink_callback *cb, int type,

		       const struct tc_action_ops *ops);

int tcf_hash_search(struct tc_action_net *tn, struct tc_action **a, u32 index);

u32 tcf_hash_new_index(struct tc_action_net *tn);

bool tcf_hash_check(struct tc_action_net *tn, u32 index, struct tc_action **a,

int tcf_idr_search(struct tc_action_net *tn, struct tc_action **a, u32 index);

bool tcf_idr_check(struct tc_action_net *tn, u32 index, struct tc_action **a,

		    int bind);

int tcf_hash_create(struct tc_action_net *tn, u32 index, struct nlattr *est,

		    struct tc_action **a, const struct tc_action_ops *ops, int bind,

		    bool cpustats);

void tcf_hash_cleanup(struct tc_action *a, struct nlattr *est);

void tcf_hash_insert(struct tc_action_net *tn, struct tc_action *a);

int tcf_idr_create(struct tc_action_net *tn, u32 index, struct nlattr *est,

		   struct tc_action **a, const struct tc_action_ops *ops,

		   int bind, bool cpustats);

void tcf_idr_cleanup(struct tc_action *a, struct nlattr *est);

void tcf_idr_insert(struct tc_action_net *tn, struct tc_action *a);



int __tcf_hash_release(struct tc_action *a, bool bind, bool strict);

int __tcf_idr_release(struct tc_action *a, bool bind, bool strict);



static inline int tcf_hash_release(struct tc_action *a, bool bind)

static inline int tcf_idr_release(struct tc_action *a, bool bind)

{

	return __tcf_hash_release(a, bind, false);

	return __tcf_idr_release(a, bind, false);

}



int tcf_register_action(struct tc_action_ops *a, struct pernet_operations *ops);

lib/idr.c

+37 −29
Original line number Diff line number Diff line
@@ -7,45 +7,32 @@ 
DEFINE_PER_CPU(struct ida_bitmap *, ida_bitmap);

static DEFINE_SPINLOCK(simple_ida_lock);



/**

 * idr_alloc - allocate an id

 * @idr: idr handle

 * @ptr: pointer to be associated with the new id

 * @start: the minimum id (inclusive)

 * @end: the maximum id (exclusive)

 * @gfp: memory allocation flags

 *

 * Allocates an unused ID in the range [start, end).  Returns -ENOSPC

 * if there are no unused IDs in that range.

 *

 * Note that @end is treated as max when <= 0.  This is to always allow

 * using @start + N as @end as long as N is inside integer range.

 *

 * Simultaneous modifications to the @idr are not allowed and should be

 * prevented by the user, usually with a lock.  idr_alloc() may be called

 * concurrently with read-only accesses to the @idr, such as idr_find() and

 * idr_for_each_entry().

 */

int idr_alloc(struct idr *idr, void *ptr, int start, int end, gfp_t gfp)

int idr_alloc_cmn(struct idr *idr, void *ptr, unsigned long *index,

		  unsigned long start, unsigned long end, gfp_t gfp,

		  bool ext)

{

	void __rcu **slot;

	struct radix_tree_iter iter;

	void __rcu **slot;



	if (WARN_ON_ONCE(start < 0))

		return -EINVAL;

	if (WARN_ON_ONCE(radix_tree_is_internal_node(ptr)))

		return -EINVAL;



	radix_tree_iter_init(&iter, start);

	if (ext)

		slot = idr_get_free_ext(&idr->idr_rt, &iter, gfp, end);

	else

		slot = idr_get_free(&idr->idr_rt, &iter, gfp, end);

	if (IS_ERR(slot))

		return PTR_ERR(slot);



	radix_tree_iter_replace(&idr->idr_rt, &iter, slot, ptr);

	radix_tree_iter_tag_clear(&idr->idr_rt, &iter, IDR_FREE);

	return iter.index;



	if (index)

		*index = iter.index;

	return 0;

}

EXPORT_SYMBOL_GPL(idr_alloc);

EXPORT_SYMBOL_GPL(idr_alloc_cmn);



/**

 * idr_alloc_cyclic - allocate new idr entry in a cyclical fashion
@@ -134,6 +121,20 @@ void *idr_get_next(struct idr *idr, int *nextid) 
}

EXPORT_SYMBOL(idr_get_next);



void *idr_get_next_ext(struct idr *idr, unsigned long *nextid)

{

	struct radix_tree_iter iter;

	void __rcu **slot;



	slot = radix_tree_iter_find(&idr->idr_rt, &iter, *nextid);

	if (!slot)

		return NULL;



	*nextid = iter.index;

	return rcu_dereference_raw(*slot);

}

EXPORT_SYMBOL(idr_get_next_ext);



/**

 * idr_replace - replace pointer for given id

 * @idr: idr handle
@@ -149,13 +150,20 @@ EXPORT_SYMBOL(idr_get_next); 
 * %-EINVAL indicates that @id or @ptr were not valid.

 */

void *idr_replace(struct idr *idr, void *ptr, int id)

{

	if (WARN_ON_ONCE(id < 0))

		return ERR_PTR(-EINVAL);



	return idr_replace_ext(idr, ptr, id);

}

EXPORT_SYMBOL(idr_replace);



void *idr_replace_ext(struct idr *idr, void *ptr, unsigned long id)

{

	struct radix_tree_node *node;

	void __rcu **slot = NULL;

	void *entry;



	if (WARN_ON_ONCE(id < 0))

		return ERR_PTR(-EINVAL);

	if (WARN_ON_ONCE(radix_tree_is_internal_node(ptr)))

		return ERR_PTR(-EINVAL);
@@ -167,7 +175,7 @@ void *idr_replace(struct idr *idr, void *ptr, int id) 


	return entry;

}

EXPORT_SYMBOL(idr_replace);

EXPORT_SYMBOL(idr_replace_ext);



/**

 * DOC: IDA description

lib/radix-tree.c

+3 −3
Original line number Diff line number Diff line
@@ -2137,13 +2137,13 @@ int ida_pre_get(struct ida *ida, gfp_t gfp) 
}

EXPORT_SYMBOL(ida_pre_get);



void __rcu **idr_get_free(struct radix_tree_root *root,

			struct radix_tree_iter *iter, gfp_t gfp, int end)

void __rcu **idr_get_free_cmn(struct radix_tree_root *root,

			      struct radix_tree_iter *iter, gfp_t gfp,

			      unsigned long max)

{

	struct radix_tree_node *node = NULL, *child;

	void __rcu **slot = (void __rcu **)&root->rnode;

	unsigned long maxindex, start = iter->next_index;

	unsigned long max = end > 0 ? end - 1 : INT_MAX;

	unsigned int shift, offset = 0;



 grow:

CRA Git | Maintained and supported by SUSTech CRA and CCSE