Commit 2f0799a0 authored by David Rientjes's avatar David Rientjes Committed by Linus Torvalds
Browse files

mm, thp: restore node-local hugepage allocations 



This is a full revert of ac5b2c18 ("mm: thp: relax __GFP_THISNODE for
MADV_HUGEPAGE mappings") and a partial revert of 89c83fb5 ("mm, thp:
consolidate THP gfp handling into alloc_hugepage_direct_gfpmask").

By not setting __GFP_THISNODE, applications can allocate remote hugepages
when the local node is fragmented or low on memory when either the thp
defrag setting is "always" or the vma has been madvised with
MADV_HUGEPAGE.

Remote access to hugepages often has much higher latency than local pages
of the native page size.  On Haswell, ac5b2c18 was shown to have a
13.9% access regression after this commit for binaries that remap their
text segment to be backed by transparent hugepages.

The intent of ac5b2c18 is to address an issue where a local node is
low on memory or fragmented such that a hugepage cannot be allocated.  In
every scenario where this was described as a fix, there is abundant and
unfragmented remote memory available to allocate from, even with a greater
access latency.

If remote memory is also low or fragmented, not setting __GFP_THISNODE was
also measured on Haswell to have a 40% regression in allocation latency.

Restore __GFP_THISNODE for thp allocations.

Fixes: ac5b2c18 ("mm: thp: relax __GFP_THISNODE for MADV_HUGEPAGE mappings")
Fixes: 89c83fb5 ("mm, thp: consolidate THP gfp handling into alloc_hugepage_direct_gfpmask")
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarDavid Rientjes <rientjes@google.com>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 4eaaa2b9

include/linux/mempolicy.h

+0 −2
Original line number Diff line number Diff line
@@ -139,8 +139,6 @@ struct mempolicy *mpol_shared_policy_lookup(struct shared_policy *sp, 
struct mempolicy *get_task_policy(struct task_struct *p);

struct mempolicy *__get_vma_policy(struct vm_area_struct *vma,

		unsigned long addr);

struct mempolicy *get_vma_policy(struct vm_area_struct *vma,

						unsigned long addr);

bool vma_policy_mof(struct vm_area_struct *vma);



extern void numa_default_policy(void);

mm/huge_memory.c

+16 −26
Original line number Diff line number Diff line
@@ -632,37 +632,27 @@ release: 
static inline gfp_t alloc_hugepage_direct_gfpmask(struct vm_area_struct *vma, unsigned long addr)

{

	const bool vma_madvised = !!(vma->vm_flags & VM_HUGEPAGE);

	gfp_t this_node = 0;



#ifdef CONFIG_NUMA

	struct mempolicy *pol;

	/*

	 * __GFP_THISNODE is used only when __GFP_DIRECT_RECLAIM is not

	 * specified, to express a general desire to stay on the current

	 * node for optimistic allocation attempts. If the defrag mode

	 * and/or madvise hint requires the direct reclaim then we prefer

	 * to fallback to other node rather than node reclaim because that

	 * can lead to excessive reclaim even though there is free memory

	 * on other nodes. We expect that NUMA preferences are specified

	 * by memory policies.

	 */

	pol = get_vma_policy(vma, addr);

	if (pol->mode != MPOL_BIND)

		this_node = __GFP_THISNODE;

	mpol_cond_put(pol);

#endif

	const gfp_t gfp_mask = GFP_TRANSHUGE_LIGHT | __GFP_THISNODE;



	/* Always do synchronous compaction */

	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))

		return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY);

		return GFP_TRANSHUGE | __GFP_THISNODE |

		       (vma_madvised ? 0 : __GFP_NORETRY);



	/* Kick kcompactd and fail quickly */

	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))

		return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM | this_node;

		return gfp_mask | __GFP_KSWAPD_RECLAIM;



	/* Synchronous compaction if madvised, otherwise kick kcompactd */

	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags))

		return GFP_TRANSHUGE_LIGHT | (vma_madvised ? __GFP_DIRECT_RECLAIM :

							     __GFP_KSWAPD_RECLAIM | this_node);

		return gfp_mask | (vma_madvised ? __GFP_DIRECT_RECLAIM :

						  __GFP_KSWAPD_RECLAIM);



	/* Only do synchronous compaction if madvised */

	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))

		return GFP_TRANSHUGE_LIGHT | (vma_madvised ? __GFP_DIRECT_RECLAIM :

							     this_node);

	return GFP_TRANSHUGE_LIGHT | this_node;

		return gfp_mask | (vma_madvised ? __GFP_DIRECT_RECLAIM : 0);



	return gfp_mask;

}



/* Caller must hold page table lock. */

mm/mempolicy.c

+1 −1
Original line number Diff line number Diff line
@@ -1662,7 +1662,7 @@ struct mempolicy *__get_vma_policy(struct vm_area_struct *vma, 
 * freeing by another task.  It is the caller's responsibility to free the

 * extra reference for shared policies.

 */

struct mempolicy *get_vma_policy(struct vm_area_struct *vma,

static struct mempolicy *get_vma_policy(struct vm_area_struct *vma,

						unsigned long addr)

{

	struct mempolicy *pol = __get_vma_policy(vma, addr);

CRA Git | Maintained and supported by SUSTech CRA and CCSE