Commit 54f180d3 authored by Huang Ying's avatar Huang Ying Committed by Linus Torvalds
Browse files

mm, swap: use kvzalloc to allocate some swap data structures 

Now vzalloc() is used in swap code to allocate various data structures,
such as swap cache, swap slots cache, cluster info, etc.  Because the
size may be too large on some system, so that normal kzalloc() may fail.
But using kzalloc() has some advantages, for example, less memory
fragmentation, less TLB pressure, etc.  So change the data structure
allocation in swap code to use kvzalloc() which will try kzalloc()
firstly, and fallback to vzalloc() if kzalloc() failed.

In general, although kmalloc() will reduce the number of high-order
pages in short term, vmalloc() will cause more pain for memory
fragmentation in the long term.  And the swap data structure allocation
that is changed in this patch is expected to be long term allocation.

From Dave Hansen:
 "for example, we have a two-page data structure. vmalloc() takes two
  effectively random order-0 pages, probably from two different 2M pages
  and pins them. That "kills" two 2M pages. kmalloc(), allocating two
  *contiguous* pages, will not cross a 2M boundary. That means it will
  only "kill" the possibility of a single 2M page. More 2M pages == less
  fragmentation.

The allocation in this patch occurs during swap on time, which is
usually done during system boot, so usually we have high opportunity to
allocate the contiguous pages successfully.

The allocation for swap_map[] in struct swap_info_struct is not changed,
because that is usually quite large and vmalloc_to_page() is used for
it.  That makes it a little harder to change.

Link: http://lkml.kernel.org/r/20170407064911.25447-1-ying.huang@intel.com


Signed-off-by: default avatarHuang Ying <ying.huang@intel.com>
Acked-by: default avatarTim Chen <tim.c.chen@intel.com>
Acked-by: default avatarMichal Hocko <mhocko@suse.com>
Acked-by: default avatarRik van Riel <riel@redhat.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Shaohua Li <shli@kernel.org>
Cc: Minchan Kim <minchan@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent bc4e54f6

mm/swap_slots.c

+11 −8
Original line number Diff line number Diff line
@@ -31,6 +31,7 @@ 
#include <linux/cpumask.h>

#include <linux/vmalloc.h>

#include <linux/mutex.h>

#include <linux/mm.h>



#ifdef CONFIG_SWAP
@@ -119,16 +120,18 @@ static int alloc_swap_slot_cache(unsigned int cpu) 


	/*

	 * Do allocation outside swap_slots_cache_mutex

	 * as vzalloc could trigger reclaim and get_swap_page,

	 * as kvzalloc could trigger reclaim and get_swap_page,

	 * which can lock swap_slots_cache_mutex.

	 */

	slots = vzalloc(sizeof(swp_entry_t) * SWAP_SLOTS_CACHE_SIZE);

	slots = kvzalloc(sizeof(swp_entry_t) * SWAP_SLOTS_CACHE_SIZE,

			 GFP_KERNEL);

	if (!slots)

		return -ENOMEM;



	slots_ret = vzalloc(sizeof(swp_entry_t) * SWAP_SLOTS_CACHE_SIZE);

	slots_ret = kvzalloc(sizeof(swp_entry_t) * SWAP_SLOTS_CACHE_SIZE,

			     GFP_KERNEL);

	if (!slots_ret) {

		vfree(slots);

		kvfree(slots);

		return -ENOMEM;

	}
@@ -152,9 +155,9 @@ static int alloc_swap_slot_cache(unsigned int cpu) 
out:

	mutex_unlock(&swap_slots_cache_mutex);

	if (slots)

		vfree(slots);

		kvfree(slots);

	if (slots_ret)

		vfree(slots_ret);

		kvfree(slots_ret);

	return 0;

}
@@ -171,7 +174,7 @@ static void drain_slots_cache_cpu(unsigned int cpu, unsigned int type, 
		cache->cur = 0;

		cache->nr = 0;

		if (free_slots && cache->slots) {

			vfree(cache->slots);

			kvfree(cache->slots);

			cache->slots = NULL;

		}

		mutex_unlock(&cache->alloc_lock);
@@ -186,7 +189,7 @@ static void drain_slots_cache_cpu(unsigned int cpu, unsigned int type, 
		}

		spin_unlock_irq(&cache->free_lock);

		if (slots)

			vfree(slots);

			kvfree(slots);

	}

}

mm/swap_state.c

+1 −1
Original line number Diff line number Diff line
@@ -523,7 +523,7 @@ int init_swap_address_space(unsigned int type, unsigned long nr_pages) 
	unsigned int i, nr;



	nr = DIV_ROUND_UP(nr_pages, SWAP_ADDRESS_SPACE_PAGES);

	spaces = vzalloc(sizeof(struct address_space) * nr);

	spaces = kvzalloc(sizeof(struct address_space) * nr, GFP_KERNEL);

	if (!spaces)

		return -ENOMEM;

	for (i = 0; i < nr; i++) {

mm/swapfile.c

+6 −4
Original line number Diff line number Diff line
@@ -2270,8 +2270,8 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile) 
	free_percpu(p->percpu_cluster);

	p->percpu_cluster = NULL;

	vfree(swap_map);

	vfree(cluster_info);

	vfree(frontswap_map);

	kvfree(cluster_info);

	kvfree(frontswap_map);

	/* Destroy swap account information */

	swap_cgroup_swapoff(p->type);

	exit_swap_address_space(p->type);
@@ -2794,7 +2794,8 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) 
		p->cluster_next = 1 + (prandom_u32() % p->highest_bit);

		nr_cluster = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);



		cluster_info = vzalloc(nr_cluster * sizeof(*cluster_info));

		cluster_info = kvzalloc(nr_cluster * sizeof(*cluster_info),

					GFP_KERNEL);

		if (!cluster_info) {

			error = -ENOMEM;

			goto bad_swap;
@@ -2827,7 +2828,8 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) 
	}

	/* frontswap enabled? set up bit-per-page map for frontswap */

	if (IS_ENABLED(CONFIG_FRONTSWAP))

		frontswap_map = vzalloc(BITS_TO_LONGS(maxpages) * sizeof(long));

		frontswap_map = kvzalloc(BITS_TO_LONGS(maxpages) * sizeof(long),

					 GFP_KERNEL);



	if (p->bdev &&(swap_flags & SWAP_FLAG_DISCARD) && swap_discardable(p)) {

		/*

CRA Git | Maintained and supported by SUSTech CRA and CCSE