Commit a00cc7d9 authored by Matthew Wilcox's avatar Matthew Wilcox Committed by Linus Torvalds
Browse files

mm, x86: add support for PUD-sized transparent hugepages 

The current transparent hugepage code only supports PMDs.  This patch
adds support for transparent use of PUDs with DAX.  It does not include
support for anonymous pages.  x86 support code also added.

Most of this patch simply parallels the work that was done for huge
PMDs.  The only major difference is how the new ->pud_entry method in
mm_walk works.  The ->pmd_entry method replaces the ->pte_entry method,
whereas the ->pud_entry method works along with either ->pmd_entry or
->pte_entry.  The pagewalk code takes care of locking the PUD before
calling ->pud_walk, so handlers do not need to worry whether the PUD is
stable.

[dave.jiang@intel.com: fix SMP x86 32bit build for native_pud_clear()]
  Link: http://lkml.kernel.org/r/148719066814.31111.3239231168815337012.stgit@djiang5-desk3.ch.intel.com
[dave.jiang@intel.com: native_pud_clear missing on i386 build]
  Link: http://lkml.kernel.org/r/148640375195.69754.3315433724330910314.stgit@djiang5-desk3.ch.intel.com
Link: http://lkml.kernel.org/r/148545059381.17912.8602162635537598445.stgit@djiang5-desk3.ch.intel.com


Signed-off-by: default avatarMatthew Wilcox <mawilcox@microsoft.com>
Signed-off-by: default avatarDave Jiang <dave.jiang@intel.com>
Tested-by: default avatarAlexander Kapshuk <alexander.kapshuk@gmail.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Jan Kara <jack@suse.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Ross Zwisler <ross.zwisler@linux.intel.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Nilesh Choudhury <nilesh.choudhury@oracle.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent a2d58167

arch/Kconfig

+3 −0
Original line number Diff line number Diff line
@@ -571,6 +571,9 @@ config HAVE_IRQ_TIME_ACCOUNTING 
config HAVE_ARCH_TRANSPARENT_HUGEPAGE

	bool



config HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD

	bool



config HAVE_ARCH_HUGE_VMAP

	bool

arch/x86/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -109,6 +109,7 @@ config X86 
	select HAVE_ARCH_SECCOMP_FILTER

	select HAVE_ARCH_TRACEHOOK

	select HAVE_ARCH_TRANSPARENT_HUGEPAGE

	select HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD if X86_64

	select HAVE_ARCH_VMAP_STACK		if X86_64

	select HAVE_ARCH_WITHIN_STACK_FRAMES

	select HAVE_CC_STACKPROTECTOR

arch/x86/include/asm/paravirt.h

+11 −0
Original line number Diff line number Diff line
@@ -475,6 +475,17 @@ static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, 
			    native_pmd_val(pmd));

}



static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,

			      pud_t *pudp, pud_t pud)

{

	if (sizeof(pudval_t) > sizeof(long))

		/* 5 arg words */

		pv_mmu_ops.set_pud_at(mm, addr, pudp, pud);

	else

		PVOP_VCALL4(pv_mmu_ops.set_pud_at, mm, addr, pudp,

			    native_pud_val(pud));

}



static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)

{

	pmdval_t val = native_pmd_val(pmd);

arch/x86/include/asm/paravirt_types.h

+2 −0
Original line number Diff line number Diff line
@@ -249,6 +249,8 @@ struct pv_mmu_ops { 
	void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval);

	void (*set_pmd_at)(struct mm_struct *mm, unsigned long addr,

			   pmd_t *pmdp, pmd_t pmdval);

	void (*set_pud_at)(struct mm_struct *mm, unsigned long addr,

			   pud_t *pudp, pud_t pudval);

	void (*pte_update)(struct mm_struct *mm, unsigned long addr,

			   pte_t *ptep);

arch/x86/include/asm/pgtable-2level.h

+17 −0
Original line number Diff line number Diff line
@@ -21,6 +21,10 @@ static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) 
	*pmdp = pmd;

}



static inline void native_set_pud(pud_t *pudp, pud_t pud)

{

}



static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)

{

	native_set_pte(ptep, pte);
@@ -31,6 +35,10 @@ static inline void native_pmd_clear(pmd_t *pmdp) 
	native_set_pmd(pmdp, __pmd(0));

}



static inline void native_pud_clear(pud_t *pudp)

{

}



static inline void native_pte_clear(struct mm_struct *mm,

				    unsigned long addr, pte_t *xp)

{
@@ -55,6 +63,15 @@ static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp) 
#define native_pmdp_get_and_clear(xp) native_local_pmdp_get_and_clear(xp)

#endif



#ifdef CONFIG_SMP

static inline pud_t native_pudp_get_and_clear(pud_t *xp)

{

	return __pud(xchg((pudval_t *)xp, 0));

}

#else

#define native_pudp_get_and_clear(xp) native_local_pudp_get_and_clear(xp)

#endif



/* Bit manipulation helper on pte/pgoff entry */

static inline unsigned long pte_bitop(unsigned long value, unsigned int rightshift,

				      unsigned long mask, unsigned int leftshift)

CRA Git | Maintained and supported by SUSTech CRA and CCSE