Merge branch 'for-next/hugepages' of... (aa729dcc) · Commits · 戴 / test

arch/arm64/Kconfig

+17 −0

Original line number	Diff line number	Diff line
		@@ -188,6 +188,18 @@ config HW_PERF_EVENTS
		Enable hardware performance counter support for perf events. If
		disabled, perf events will use software events only.

		config SYS_SUPPORTS_HUGETLBFS
		def_bool y

		config ARCH_WANT_GENERAL_HUGETLB
		def_bool y

		config ARCH_WANT_HUGE_PMD_SHARE
		def_bool y if !ARM64_64K_PAGES

		config HAVE_ARCH_TRANSPARENT_HUGEPAGE
		def_bool y

		source "mm/Kconfig"

		config XEN_DOM0
		@@ -200,6 +212,11 @@ config XEN
		help
		Say Y if you want to run Linux in a Virtual Machine on Xen on ARM64.

		config FORCE_MAX_ZONEORDER
		int
		default "14" if (ARM64_64K_PAGES && TRANSPARENT_HUGEPAGE)
		default "11"

		endmenu

		menu "Boot options"

arch/arm64/include/asm/hugetlb.h

0 → 100644

+117 −0

Original line number	Diff line number	Diff line
		/*
		* arch/arm64/include/asm/hugetlb.h
		*
		* Copyright (C) 2013 Linaro Ltd.
		*
		* Based on arch/x86/include/asm/hugetlb.h
		*
		* This program is free software; you can redistribute it and/or modify
		* it under the terms of the GNU General Public License version 2 as
		* published by the Free Software Foundation.
		*
		* This program is distributed in the hope that it will be useful,
		* but WITHOUT ANY WARRANTY; without even the implied warranty of
		* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		* GNU General Public License for more details.
		*
		* You should have received a copy of the GNU General Public License
		* along with this program; if not, write to the Free Software
		* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
		*/

		#ifndef __ASM_HUGETLB_H
		#define __ASM_HUGETLB_H

		#include <asm-generic/hugetlb.h>
		#include <asm/page.h>

		static inline pte_t huge_ptep_get(pte_t *ptep)
		{
		return *ptep;
		}

		static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
		pte_t *ptep, pte_t pte)
		{
		set_pte_at(mm, addr, ptep, pte);
		}

		static inline void huge_ptep_clear_flush(struct vm_area_struct *vma,
		unsigned long addr, pte_t *ptep)
		{
		ptep_clear_flush(vma, addr, ptep);
		}

		static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
		unsigned long addr, pte_t *ptep)
		{
		ptep_set_wrprotect(mm, addr, ptep);
		}

		static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
		unsigned long addr, pte_t *ptep)
		{
		return ptep_get_and_clear(mm, addr, ptep);
		}

		static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma,
		unsigned long addr, pte_t *ptep,
		pte_t pte, int dirty)
		{
		return ptep_set_access_flags(vma, addr, ptep, pte, dirty);
		}

		static inline void hugetlb_free_pgd_range(struct mmu_gather *tlb,
		unsigned long addr, unsigned long end,
		unsigned long floor,
		unsigned long ceiling)
		{
		free_pgd_range(tlb, addr, end, floor, ceiling);
		}

		static inline int is_hugepage_only_range(struct mm_struct *mm,
		unsigned long addr, unsigned long len)
		{
		return 0;
		}

		static inline int prepare_hugepage_range(struct file *file,
		unsigned long addr, unsigned long len)
		{
		struct hstate *h = hstate_file(file);
		if (len & ~huge_page_mask(h))
		return -EINVAL;
		if (addr & ~huge_page_mask(h))
		return -EINVAL;
		return 0;
		}

		static inline void hugetlb_prefault_arch_hook(struct mm_struct *mm)
		{
		}

		static inline int huge_pte_none(pte_t pte)
		{
		return pte_none(pte);
		}

		static inline pte_t huge_pte_wrprotect(pte_t pte)
		{
		return pte_wrprotect(pte);
		}

		static inline int arch_prepare_hugepage(struct page *page)
		{
		return 0;
		}

		static inline void arch_release_hugepage(struct page *page)
		{
		}

		static inline void arch_clear_hugepage_flags(struct page *page)
		{
		clear_bit(PG_dcache_clean, &page->flags);
		}

		#endif /* __ASM_HUGETLB_H */

arch/arm64/include/asm/pgtable-hwdef.h

+12 −1

Original line number	Diff line number	Diff line
		@@ -25,17 +25,27 @@
		/*
		* Hardware page table definitions.
		*
		* Level 1 descriptor (PUD).
		*/

		#define PUD_TABLE_BIT (_AT(pgdval_t, 1) << 1)

		/*
		* Level 2 descriptor (PMD).
		*/
		#define PMD_TYPE_MASK (_AT(pmdval_t, 3) << 0)
		#define PMD_TYPE_FAULT (_AT(pmdval_t, 0) << 0)
		#define PMD_TYPE_TABLE (_AT(pmdval_t, 3) << 0)
		#define PMD_TYPE_SECT (_AT(pmdval_t, 1) << 0)
		#define PMD_TABLE_BIT (_AT(pmdval_t, 1) << 1)

		/*
		* Section
		*/
		#define PMD_SECT_USER (_AT(pteval_t, 1) << 6) /* AP[1] */
		#define PMD_SECT_VALID (_AT(pmdval_t, 1) << 0)
		#define PMD_SECT_PROT_NONE (_AT(pmdval_t, 1) << 2)
		#define PMD_SECT_USER (_AT(pmdval_t, 1) << 6) /* AP[1] */
		#define PMD_SECT_RDONLY (_AT(pmdval_t, 1) << 7) /* AP[2] */
		#define PMD_SECT_S (_AT(pmdval_t, 3) << 8)
		#define PMD_SECT_AF (_AT(pmdval_t, 1) << 10)
		#define PMD_SECT_NG (_AT(pmdval_t, 1) << 11)
		@@ -54,6 +64,7 @@
		#define PTE_TYPE_MASK (_AT(pteval_t, 3) << 0)
		#define PTE_TYPE_FAULT (_AT(pteval_t, 0) << 0)
		#define PTE_TYPE_PAGE (_AT(pteval_t, 3) << 0)
		#define PTE_TABLE_BIT (_AT(pteval_t, 1) << 1)
		#define PTE_USER (_AT(pteval_t, 1) << 6) /* AP[1] */
		#define PTE_RDONLY (_AT(pteval_t, 1) << 7) /* AP[2] */
		#define PTE_SHARED (_AT(pteval_t, 3) << 8) /* SH[1:0], inner shareable */

arch/arm64/include/asm/pgtable.h

+80 −16

Original line number	Diff line number	Diff line
		@@ -25,8 +25,8 @@
		* Software defined PTE bits definition.
		*/
		#define PTE_VALID (_AT(pteval_t, 1) << 0)
		#define PTE_PROT_NONE (_AT(pteval_t, 1) << 1) /* only when !PTE_VALID */
		#define PTE_FILE (_AT(pteval_t, 1) << 2) /* only when !pte_present() */
		#define PTE_PROT_NONE (_AT(pteval_t, 1) << 2) /* only when !PTE_VALID */
		#define PTE_FILE (_AT(pteval_t, 1) << 3) /* only when !pte_present() */
		#define PTE_DIRTY (_AT(pteval_t, 1) << 55)
		#define PTE_SPECIAL (_AT(pteval_t, 1) << 56)

		@@ -66,7 +66,7 @@ extern pgprot_t pgprot_default;

		#define _MOD_PROT(p, b) __pgprot_modify(p, 0, b)

		#define PAGE_NONE __pgprot_modify(pgprot_default, PTE_TYPE_MASK, PTE_PROT_NONE)
		#define PAGE_NONE __pgprot_modify(pgprot_default, PTE_TYPE_MASK, PTE_PROT_NONE \| PTE_RDONLY \| PTE_PXN \| PTE_UXN)
		#define PAGE_SHARED _MOD_PROT(pgprot_default, PTE_USER \| PTE_NG \| PTE_PXN \| PTE_UXN)
		#define PAGE_SHARED_EXEC _MOD_PROT(pgprot_default, PTE_USER \| PTE_NG \| PTE_PXN)
		#define PAGE_COPY _MOD_PROT(pgprot_default, PTE_USER \| PTE_NG \| PTE_PXN \| PTE_UXN \| PTE_RDONLY)
		@@ -82,7 +82,7 @@ extern pgprot_t pgprot_default;
		#define PAGE_S2 __pgprot_modify(pgprot_default, PTE_S2_MEMATTR_MASK, PTE_S2_MEMATTR(MT_S2_NORMAL) \| PTE_S2_RDONLY)
		#define PAGE_S2_DEVICE __pgprot(PROT_DEFAULT \| PTE_S2_MEMATTR(MT_S2_DEVICE_nGnRE) \| PTE_S2_RDWR \| PTE_UXN)

		#define __PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_TYPE_MASK) \| PTE_PROT_NONE)
		#define __PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_TYPE_MASK) \| PTE_PROT_NONE \| PTE_RDONLY \| PTE_PXN \| PTE_UXN)
		#define __PAGE_SHARED __pgprot(_PAGE_DEFAULT \| PTE_USER \| PTE_NG \| PTE_PXN \| PTE_UXN)
		#define __PAGE_SHARED_EXEC __pgprot(_PAGE_DEFAULT \| PTE_USER \| PTE_NG \| PTE_PXN)
		#define __PAGE_COPY __pgprot(_PAGE_DEFAULT \| PTE_USER \| PTE_NG \| PTE_PXN \| PTE_UXN \| PTE_RDONLY)
		@@ -179,11 +179,75 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
		/*
		* Huge pte definitions.
		*/
		#define pte_huge(pte) ((pte_val(pte) & PTE_TYPE_MASK) == PTE_TYPE_HUGEPAGE)
		#define pte_mkhuge(pte) (__pte((pte_val(pte) & ~PTE_TYPE_MASK) \| PTE_TYPE_HUGEPAGE))
		#define pte_huge(pte) (!(pte_val(pte) & PTE_TABLE_BIT))
		#define pte_mkhuge(pte) (__pte(pte_val(pte) & ~PTE_TABLE_BIT))

		/*
		* Hugetlb definitions.
		*/
		#define HUGE_MAX_HSTATE 2
		#define HPAGE_SHIFT PMD_SHIFT
		#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT)
		#define HPAGE_MASK (~(HPAGE_SIZE - 1))
		#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT)

		#define __HAVE_ARCH_PTE_SPECIAL

		/*
		* Software PMD bits for THP
		*/

		#define PMD_SECT_DIRTY (_AT(pmdval_t, 1) << 55)
		#define PMD_SECT_SPLITTING (_AT(pmdval_t, 1) << 57)

		/*
		* THP definitions.
		*/
		#define pmd_young(pmd) (pmd_val(pmd) & PMD_SECT_AF)

		#define __HAVE_ARCH_PMD_WRITE
		#define pmd_write(pmd) (!(pmd_val(pmd) & PMD_SECT_RDONLY))

		#ifdef CONFIG_TRANSPARENT_HUGEPAGE
		#define pmd_trans_huge(pmd) (pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT))
		#define pmd_trans_splitting(pmd) (pmd_val(pmd) & PMD_SECT_SPLITTING)
		#endif

		#define PMD_BIT_FUNC(fn,op) \
		static inline pmd_t pmd_##fn(pmd_t pmd) { pmd_val(pmd) op; return pmd; }

		PMD_BIT_FUNC(wrprotect, \|= PMD_SECT_RDONLY);
		PMD_BIT_FUNC(mkold, &= ~PMD_SECT_AF);
		PMD_BIT_FUNC(mksplitting, \|= PMD_SECT_SPLITTING);
		PMD_BIT_FUNC(mkwrite, &= ~PMD_SECT_RDONLY);
		PMD_BIT_FUNC(mkdirty, \|= PMD_SECT_DIRTY);
		PMD_BIT_FUNC(mkyoung, \|= PMD_SECT_AF);
		PMD_BIT_FUNC(mknotpresent, &= ~PMD_TYPE_MASK);

		#define pmd_mkhuge(pmd) (__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT))

		#define pmd_pfn(pmd) (((pmd_val(pmd) & PMD_MASK) & PHYS_MASK) >> PAGE_SHIFT)
		#define pfn_pmd(pfn,prot) (__pmd(((phys_addr_t)(pfn) << PAGE_SHIFT) \| pgprot_val(prot)))
		#define mk_pmd(page,prot) pfn_pmd(page_to_pfn(page),prot)

		#define pmd_page(pmd) pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK))

		static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
		{
		const pmdval_t mask = PMD_SECT_USER \| PMD_SECT_PXN \| PMD_SECT_UXN \|
		PMD_SECT_RDONLY \| PMD_SECT_PROT_NONE \|
		PMD_SECT_VALID;
		pmd_val(pmd) = (pmd_val(pmd) & ~mask) \| (pgprot_val(newprot) & mask);
		return pmd;
		}

		#define set_pmd_at(mm, addr, pmdp, pmd) set_pmd(pmdp, pmd)

		static inline int has_transparent_hugepage(void)
		{
		return 1;
		}

		/*
		* Mark the prot value as uncacheable and unbufferable.
		*/
		@@ -293,12 +357,12 @@ extern pgd_t idmap_pg_dir[PTRS_PER_PGD];

		/*
		* Encode and decode a swap entry:
		* bits 0-1: present (must be zero)
		* bit 2: PTE_FILE
		* bits 3-8: swap type
		* bits 0, 2: present (must both be zero)
		* bit 3: PTE_FILE
		* bits 4-8: swap type
		* bits 9-63: swap offset
		*/
		#define __SWP_TYPE_SHIFT 3
		#define __SWP_TYPE_SHIFT 4
		#define __SWP_TYPE_BITS 6
		#define __SWP_TYPE_MASK ((1 << __SWP_TYPE_BITS) - 1)
		#define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT)
		@@ -318,15 +382,15 @@ extern pgd_t idmap_pg_dir[PTRS_PER_PGD];

		/*
		* Encode and decode a file entry:
		* bits 0-1: present (must be zero)
		* bit 2: PTE_FILE
		* bits 3-63: file offset / PAGE_SIZE
		* bits 0, 2: present (must both be zero)
		* bit 3: PTE_FILE
		* bits 4-63: file offset / PAGE_SIZE
		*/
		#define pte_file(pte) (pte_val(pte) & PTE_FILE)
		#define pte_to_pgoff(x) (pte_val(x) >> 3)
		#define pgoff_to_pte(x) __pte(((x) << 3) \| PTE_FILE)
		#define pte_to_pgoff(x) (pte_val(x) >> 4)
		#define pgoff_to_pte(x) __pte(((x) << 4) \| PTE_FILE)

		#define PTE_FILE_MAX_BITS 61
		#define PTE_FILE_MAX_BITS 60

		extern int kern_addr_valid(unsigned long addr);

arch/arm64/include/asm/tlb.h

+6 −0

Original line number	Diff line number	Diff line
		@@ -187,4 +187,10 @@ static inline void __pmd_free_tlb(struct mmu_gather tlb, pmd_t pmdp,

		#define tlb_migrate_finish(mm) do { } while (0)

		static inline void
		tlb_remove_pmd_tlb_entry(struct mmu_gather tlb, pmd_t pmdp, unsigned long addr)
		{
		tlb_add_flush(tlb, addr);
		}

		#endif

Admin message