Commit 6b637835 authored by Toshi Kani's avatar Toshi Kani Committed by Linus Torvalds
Browse files

x86, mm: support huge KVA mappings on x86 



Implement huge KVA mapping interfaces on x86.

On x86, MTRRs can override PAT memory types with a 4KB granularity.  When
using a huge page, MTRRs can override the memory type of the huge page,
which may lead a performance penalty.  The processor can also behave in an
undefined manner if a huge page is mapped to a memory range that MTRRs
have mapped with multiple different memory types.  Therefore, the mapping
code falls back to use a smaller page size toward 4KB when a mapping range
is covered by non-WB type of MTRRs.  The WB type of MTRRs has no affect on
the PAT memory types.

pud_set_huge() and pmd_set_huge() call mtrr_type_lookup() to see if a
given range is covered by MTRRs.  MTRR_TYPE_WRBACK indicates that the
range is either covered by WB or not covered and the MTRR default value is
set to WB.  0xFF indicates that MTRRs are disabled.

HAVE_ARCH_HUGE_VMAP is selected when X86_64 or X86_32 with X86_PAE is set.
 X86_32 without X86_PAE is not supported since such config can unlikey be
benefited from this feature, and there was an issue found in testing.

[fengguang.wu@intel.com: ioremap_pud_capable can be static]
Signed-off-by: default avatarToshi Kani <toshi.kani@hp.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Robert Elliott <Elliott@hp.com>
Signed-off-by: default avatarFengguang Wu <fengguang.wu@intel.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 5d72b4fb

arch/x86/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -99,6 +99,7 @@ config X86 
	select IRQ_FORCED_THREADING

	select HAVE_BPF_JIT if X86_64

	select HAVE_ARCH_TRANSPARENT_HUGEPAGE

	select HAVE_ARCH_HUGE_VMAP if X86_64 || (X86_32 && X86_PAE)

	select ARCH_HAS_SG_CHAIN

	select CLKEVT_I8253

	select ARCH_HAVE_NMI_SAFE_CMPXCHG

arch/x86/mm/pgtable.c

+65 −0
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ 
#include <asm/pgtable.h>

#include <asm/tlb.h>

#include <asm/fixmap.h>

#include <asm/mtrr.h>



#define PGALLOC_GFP GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO
@@ -560,3 +561,67 @@ void native_set_fixmap(enum fixed_addresses idx, phys_addr_t phys, 
{

	__native_set_fixmap(idx, pfn_pte(phys >> PAGE_SHIFT, flags));

}



#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP

int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)

{

	u8 mtrr;



	/*

	 * Do not use a huge page when the range is covered by non-WB type

	 * of MTRRs.

	 */

	mtrr = mtrr_type_lookup(addr, addr + PUD_SIZE);

	if ((mtrr != MTRR_TYPE_WRBACK) && (mtrr != 0xFF))

		return 0;



	prot = pgprot_4k_2_large(prot);



	set_pte((pte_t *)pud, pfn_pte(

		(u64)addr >> PAGE_SHIFT,

		__pgprot(pgprot_val(prot) | _PAGE_PSE)));



	return 1;

}



int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)

{

	u8 mtrr;



	/*

	 * Do not use a huge page when the range is covered by non-WB type

	 * of MTRRs.

	 */

	mtrr = mtrr_type_lookup(addr, addr + PMD_SIZE);

	if ((mtrr != MTRR_TYPE_WRBACK) && (mtrr != 0xFF))

		return 0;



	prot = pgprot_4k_2_large(prot);



	set_pte((pte_t *)pmd, pfn_pte(

		(u64)addr >> PAGE_SHIFT,

		__pgprot(pgprot_val(prot) | _PAGE_PSE)));



	return 1;

}



int pud_clear_huge(pud_t *pud)

{

	if (pud_large(*pud)) {

		pud_clear(pud);

		return 1;

	}



	return 0;

}



int pmd_clear_huge(pmd_t *pmd)

{

	if (pmd_large(*pmd)) {

		pmd_clear(pmd);

		return 1;

	}



	return 0;

}

#endif	/* CONFIG_HAVE_ARCH_HUGE_VMAP */

lib/ioremap.c

+3 −3
Original line number Diff line number Diff line
@@ -14,9 +14,9 @@ 
#include <asm/pgtable.h>



#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP

int __read_mostly ioremap_pud_capable;

int __read_mostly ioremap_pmd_capable;

int __read_mostly ioremap_huge_disabled;

static int __read_mostly ioremap_pud_capable;

static int __read_mostly ioremap_pmd_capable;

static int __read_mostly ioremap_huge_disabled;



static int __init set_nohugeiomap(char *str)

{

CRA Git | Maintained and supported by SUSTech CRA and CCSE