Commit f034b5d4 authored by David S. Miller's avatar David S. Miller
Browse files

[XFRM]: Dynamic xfrm_state hash table sizing. 



The grow algorithm is simple, we grow if:

1) we see a hash chain collision at insert, and
2) we haven't hit the hash size limit (currently 1*1024*1024 slots), and
3) the number of xfrm_state objects is > the current hash mask

All of this needs some tweaking.

Remove __initdata from "hashdist" so we can use it safely at run time.

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 8f126e37

include/linux/bootmem.h

+1 −1
Original line number Diff line number Diff line
@@ -114,7 +114,7 @@ extern void *__init alloc_large_system_hash(const char *tablename, 
#else

#define HASHDIST_DEFAULT 0

#endif

extern int __initdata hashdist;		/* Distribute hashes across NUMA nodes? */

extern int hashdist;		/* Distribute hashes across NUMA nodes? */





#endif /* _LINUX_BOOTMEM_H */

mm/page_alloc.c

+1 −1
Original line number Diff line number Diff line
@@ -2363,7 +2363,7 @@ int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write, 
	return 0;

}



__initdata int hashdist = HASHDIST_DEFAULT;

int hashdist = HASHDIST_DEFAULT;



#ifdef CONFIG_NUMA

static int __init set_hashdist(char *str)

net/xfrm/xfrm_state.c

+195 −52
Original line number Diff line number Diff line
@@ -18,6 +18,9 @@ 
#include <linux/pfkeyv2.h>

#include <linux/ipsec.h>

#include <linux/module.h>

#include <linux/bootmem.h>

#include <linux/vmalloc.h>

#include <linux/cache.h>

#include <asm/uaccess.h>



struct sock *xfrm_nl;
@@ -38,102 +41,230 @@ EXPORT_SYMBOL(sysctl_xfrm_aevent_rseqth); 


static DEFINE_SPINLOCK(xfrm_state_lock);



#define XFRM_DST_HSIZE		1024



/* Hash table to find appropriate SA towards given target (endpoint

 * of tunnel or destination of transport mode) allowed by selector.

 *

 * Main use is finding SA after policy selected tunnel or transport mode.

 * Also, it can be used by ah/esp icmp error handler to find offending SA.

 */

static struct hlist_head xfrm_state_bydst[XFRM_DST_HSIZE];

static struct hlist_head xfrm_state_bysrc[XFRM_DST_HSIZE];

static struct hlist_head xfrm_state_byspi[XFRM_DST_HSIZE];



static __inline__

unsigned __xfrm4_dst_hash(xfrm_address_t *addr)

static struct hlist_head *xfrm_state_bydst __read_mostly;

static struct hlist_head *xfrm_state_bysrc __read_mostly;

static struct hlist_head *xfrm_state_byspi __read_mostly;

static unsigned int xfrm_state_hmask __read_mostly;

static unsigned int xfrm_state_hashmax __read_mostly = 1 * 1024 * 1024;

static unsigned int xfrm_state_num;



static inline unsigned int __xfrm4_dst_hash(xfrm_address_t *addr, unsigned int hmask)

{

	unsigned h;

	unsigned int h;

	h = ntohl(addr->a4);

	h = (h ^ (h>>16)) % XFRM_DST_HSIZE;

	h = (h ^ (h>>16)) & hmask;

	return h;

}



static __inline__

unsigned __xfrm6_dst_hash(xfrm_address_t *addr)

static inline unsigned int __xfrm6_dst_hash(xfrm_address_t *addr, unsigned int hmask)

{

	unsigned h;

	unsigned int h;

	h = ntohl(addr->a6[2]^addr->a6[3]);

	h = (h ^ (h>>16)) % XFRM_DST_HSIZE;

	h = (h ^ (h>>16)) & hmask;

	return h;

}



static __inline__

unsigned __xfrm4_src_hash(xfrm_address_t *addr)

static inline unsigned int __xfrm4_src_hash(xfrm_address_t *addr, unsigned int hmask)

{

	return __xfrm4_dst_hash(addr);

	return __xfrm4_dst_hash(addr, hmask);

}



static __inline__

unsigned __xfrm6_src_hash(xfrm_address_t *addr)

static inline unsigned int __xfrm6_src_hash(xfrm_address_t *addr, unsigned int hmask)

{

	return __xfrm6_dst_hash(addr);

	return __xfrm6_dst_hash(addr, hmask);

}



static __inline__

unsigned xfrm_src_hash(xfrm_address_t *addr, unsigned short family)

static inline unsigned __xfrm_src_hash(xfrm_address_t *addr, unsigned short family,  unsigned int hmask)

{

	switch (family) {

	case AF_INET:

		return __xfrm4_src_hash(addr);

		return __xfrm4_src_hash(addr, hmask);

	case AF_INET6:

		return __xfrm6_src_hash(addr);

		return __xfrm6_src_hash(addr, hmask);

	}

	return 0;

}



static __inline__

unsigned xfrm_dst_hash(xfrm_address_t *addr, unsigned short family)

static inline unsigned xfrm_src_hash(xfrm_address_t *addr, unsigned short family)

{

	return __xfrm_src_hash(addr, family, xfrm_state_hmask);

}



static inline unsigned int __xfrm_dst_hash(xfrm_address_t *addr, unsigned short family, unsigned int hmask)

{

	switch (family) {

	case AF_INET:

		return __xfrm4_dst_hash(addr);

		return __xfrm4_dst_hash(addr, hmask);

	case AF_INET6:

		return __xfrm6_dst_hash(addr);

		return __xfrm6_dst_hash(addr, hmask);

	}

	return 0;

}



static __inline__

unsigned __xfrm4_spi_hash(xfrm_address_t *addr, u32 spi, u8 proto)

static inline unsigned int xfrm_dst_hash(xfrm_address_t *addr, unsigned short family)

{

	return __xfrm_dst_hash(addr, family, xfrm_state_hmask);

}



static inline unsigned int __xfrm4_spi_hash(xfrm_address_t *addr, u32 spi, u8 proto,

					unsigned int hmask)

{

	unsigned h;

	unsigned int h;

	h = ntohl(addr->a4^spi^proto);

	h = (h ^ (h>>10) ^ (h>>20)) % XFRM_DST_HSIZE;

	h = (h ^ (h>>10) ^ (h>>20)) & hmask;

	return h;

}



static __inline__

unsigned __xfrm6_spi_hash(xfrm_address_t *addr, u32 spi, u8 proto)

static inline unsigned int __xfrm6_spi_hash(xfrm_address_t *addr, u32 spi, u8 proto,

					    unsigned int hmask)

{

	unsigned h;

	unsigned int h;

	h = ntohl(addr->a6[2]^addr->a6[3]^spi^proto);

	h = (h ^ (h>>10) ^ (h>>20)) % XFRM_DST_HSIZE;

	h = (h ^ (h>>10) ^ (h>>20)) & hmask;

	return h;

}



static __inline__

unsigned xfrm_spi_hash(xfrm_address_t *addr, u32 spi, u8 proto, unsigned short family)

static inline

unsigned __xfrm_spi_hash(xfrm_address_t *addr, u32 spi, u8 proto, unsigned short family,

			 unsigned int hmask)

{

	switch (family) {

	case AF_INET:

		return __xfrm4_spi_hash(addr, spi, proto);

		return __xfrm4_spi_hash(addr, spi, proto, hmask);

	case AF_INET6:

		return __xfrm6_spi_hash(addr, spi, proto);

		return __xfrm6_spi_hash(addr, spi, proto, hmask);

	}

	return 0;	/*XXX*/

}



static inline unsigned int

xfrm_spi_hash(xfrm_address_t *addr, u32 spi, u8 proto, unsigned short family)

{

	return __xfrm_spi_hash(addr, spi, proto, family, xfrm_state_hmask);

}



static struct hlist_head *xfrm_state_hash_alloc(unsigned int sz)

{

	struct hlist_head *n;



	if (sz <= PAGE_SIZE)

		n = kmalloc(sz, GFP_KERNEL);

	else if (hashdist)

		n = __vmalloc(sz, GFP_KERNEL, PAGE_KERNEL);

	else

		n = (struct hlist_head *)

			__get_free_pages(GFP_KERNEL, get_order(sz));



	if (n)

		memset(n, 0, sz);



	return n;

}



static void xfrm_state_hash_free(struct hlist_head *n, unsigned int sz)

{

	if (sz <= PAGE_SIZE)

		kfree(n);

	else if (hashdist)

		vfree(n);

	else

		free_pages((unsigned long)n, get_order(sz));

}



static void xfrm_hash_transfer(struct hlist_head *list,

			       struct hlist_head *ndsttable,

			       struct hlist_head *nsrctable,

			       struct hlist_head *nspitable,

			       unsigned int nhashmask)

{

	struct hlist_node *entry, *tmp;

	struct xfrm_state *x;



	hlist_for_each_entry_safe(x, entry, tmp, list, bydst) {

		unsigned int h;



		h = __xfrm_dst_hash(&x->id.daddr, x->props.family, nhashmask);

		hlist_add_head(&x->bydst, ndsttable+h);



		h = __xfrm_src_hash(&x->props.saddr, x->props.family,

				    nhashmask);

		hlist_add_head(&x->bysrc, nsrctable+h);



		h = __xfrm_spi_hash(&x->id.daddr, x->id.spi, x->id.proto,

				    x->props.family, nhashmask);

		hlist_add_head(&x->byspi, nspitable+h);

	}

}



static unsigned long xfrm_hash_new_size(void)

{

	return ((xfrm_state_hmask + 1) << 1) *

		sizeof(struct hlist_head);

}



static DEFINE_MUTEX(hash_resize_mutex);



static void xfrm_hash_resize(void *__unused)

{

	struct hlist_head *ndst, *nsrc, *nspi, *odst, *osrc, *ospi;

	unsigned long nsize, osize;

	unsigned int nhashmask, ohashmask;

	int i;



	mutex_lock(&hash_resize_mutex);



	nsize = xfrm_hash_new_size();

	ndst = xfrm_state_hash_alloc(nsize);

	if (!ndst)

		goto out_unlock;

	nsrc = xfrm_state_hash_alloc(nsize);

	if (!nsrc) {

		xfrm_state_hash_free(ndst, nsize);

		goto out_unlock;

	}

	nspi = xfrm_state_hash_alloc(nsize);

	if (!nspi) {

		xfrm_state_hash_free(ndst, nsize);

		xfrm_state_hash_free(nsrc, nsize);

		goto out_unlock;

	}



	spin_lock_bh(&xfrm_state_lock);



	nhashmask = (nsize / sizeof(struct hlist_head)) - 1U;

	for (i = xfrm_state_hmask; i >= 0; i--)

		xfrm_hash_transfer(xfrm_state_bydst+i, ndst, nsrc, nspi,

				   nhashmask);



	odst = xfrm_state_bydst;

	osrc = xfrm_state_bysrc;

	ospi = xfrm_state_byspi;

	ohashmask = xfrm_state_hmask;



	xfrm_state_bydst = ndst;

	xfrm_state_bysrc = nsrc;

	xfrm_state_byspi = nspi;

	xfrm_state_hmask = nhashmask;



	spin_unlock_bh(&xfrm_state_lock);



	osize = (ohashmask + 1) * sizeof(struct hlist_head);

	xfrm_state_hash_free(odst, osize);

	xfrm_state_hash_free(osrc, osize);

	xfrm_state_hash_free(ospi, osize);



out_unlock:

	mutex_unlock(&hash_resize_mutex);

}



static DECLARE_WORK(xfrm_hash_work, xfrm_hash_resize, NULL);



DECLARE_WAIT_QUEUE_HEAD(km_waitq);

EXPORT_SYMBOL(km_waitq);
@@ -335,6 +466,7 @@ int __xfrm_state_delete(struct xfrm_state *x) 
			hlist_del(&x->byspi);

			__xfrm_state_put(x);

		}

		xfrm_state_num--;

		spin_unlock(&xfrm_state_lock);

		if (del_timer(&x->timer))

			__xfrm_state_put(x);
@@ -380,7 +512,7 @@ void xfrm_state_flush(u8 proto) 
	int i;



	spin_lock_bh(&xfrm_state_lock);

	for (i = 0; i < XFRM_DST_HSIZE; i++) {

	for (i = 0; i < xfrm_state_hmask; i++) {

		struct hlist_node *entry;

		struct xfrm_state *x;

restart:
@@ -611,7 +743,7 @@ out: 


static void __xfrm_state_insert(struct xfrm_state *x)

{

	unsigned h = xfrm_dst_hash(&x->id.daddr, x->props.family);

	unsigned int h = xfrm_dst_hash(&x->id.daddr, x->props.family);



	hlist_add_head(&x->bydst, xfrm_state_bydst+h);

	xfrm_state_hold(x);
@@ -637,6 +769,13 @@ static void __xfrm_state_insert(struct xfrm_state *x) 
		xfrm_state_hold(x);



	wake_up(&km_waitq);



	xfrm_state_num++;



	if (x->bydst.next != NULL &&

	    (xfrm_state_hmask + 1) < xfrm_state_hashmax &&

	    xfrm_state_num > xfrm_state_hmask)

		schedule_work(&xfrm_hash_work);

}



void xfrm_state_insert(struct xfrm_state *x)
@@ -984,7 +1123,7 @@ static struct xfrm_state *__xfrm_find_acq_byseq(u32 seq) 
{

	int i;



	for (i = 0; i < XFRM_DST_HSIZE; i++) {

	for (i = 0; i <= xfrm_state_hmask; i++) {

		struct hlist_node *entry;

		struct xfrm_state *x;
@@ -1026,7 +1165,7 @@ EXPORT_SYMBOL(xfrm_get_acqseq); 
void

xfrm_alloc_spi(struct xfrm_state *x, u32 minspi, u32 maxspi)

{

	u32 h;

	unsigned int h;

	struct xfrm_state *x0;



	if (x->id.spi)
@@ -1074,7 +1213,7 @@ int xfrm_state_walk(u8 proto, int (*func)(struct xfrm_state *, int, void*), 
	int err = 0;



	spin_lock_bh(&xfrm_state_lock);

	for (i = 0; i < XFRM_DST_HSIZE; i++) {

	for (i = 0; i <= xfrm_state_hmask; i++) {

		hlist_for_each_entry(x, entry, xfrm_state_bydst+i, bydst) {

			if (xfrm_id_proto_match(x->id.proto, proto))

				count++;
@@ -1085,7 +1224,7 @@ int xfrm_state_walk(u8 proto, int (*func)(struct xfrm_state *, int, void*), 
		goto out;

	}



	for (i = 0; i < XFRM_DST_HSIZE; i++) {

	for (i = 0; i <= xfrm_state_hmask; i++) {

		hlist_for_each_entry(x, entry, xfrm_state_bydst+i, bydst) {

			if (!xfrm_id_proto_match(x->id.proto, proto))

				continue;
@@ -1531,13 +1670,17 @@ EXPORT_SYMBOL(xfrm_init_state); 
 

void __init xfrm_state_init(void)

{

	int i;

	unsigned int sz;



	sz = sizeof(struct hlist_head) * 8;



	xfrm_state_bydst = xfrm_state_hash_alloc(sz);

	xfrm_state_bysrc = xfrm_state_hash_alloc(sz);

	xfrm_state_byspi = xfrm_state_hash_alloc(sz);

	if (!xfrm_state_bydst || !xfrm_state_bysrc || !xfrm_state_byspi)

		panic("XFRM: Cannot allocate bydst/bysrc/byspi hashes.");

	xfrm_state_hmask = ((sz / sizeof(struct hlist_head)) - 1);



	for (i=0; i<XFRM_DST_HSIZE; i++) {

		INIT_HLIST_HEAD(&xfrm_state_bydst[i]);

		INIT_HLIST_HEAD(&xfrm_state_bysrc[i]);

		INIT_HLIST_HEAD(&xfrm_state_byspi[i]);

	}

	INIT_WORK(&xfrm_state_gc_work, xfrm_state_gc_task, NULL);

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE