Commit 0bfc445d authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Catalin Marinas
Browse files

arm64: mm: set the contiguous bit for kernel mappings where appropriate 



Now that we no longer allow live kernel PMDs to be split, it is safe to
start using the contiguous bit for kernel mappings. So set the contiguous
bit in the kernel page mappings for regions whose size and alignment are
suitable for this.

This enables the following contiguous range sizes for the virtual mapping
of the kernel image, and for the linear mapping:

          granule size |  cont PTE  |  cont PMD  |
          -------------+------------+------------+
               4 KB    |    64 KB   |   32 MB    |
              16 KB    |     2 MB   |    1 GB*   |
              64 KB    |     2 MB   |   16 GB*   |

* Only when built for 3 or more levels of translation. This is due to the
  fact that a 2 level configuration only consists of PGDs and PTEs, and the
  added complexity of dealing with folded PMDs is not justified considering
  that 16 GB contiguous ranges are likely to be ignored by the hardware (and
  16k/2 levels is a niche configuration)

Signed-off-by: default avatarArd Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent f14c66ce

arch/arm64/mm/mmu.c

+30 −4
Original line number Diff line number Diff line
@@ -107,8 +107,10 @@ static bool pgattr_change_is_safe(u64 old, u64 new) 
static void alloc_init_pte(pmd_t *pmd, unsigned long addr,

				  unsigned long end, unsigned long pfn,

				  pgprot_t prot,

				  phys_addr_t (*pgtable_alloc)(void))

				  phys_addr_t (*pgtable_alloc)(void),

				  bool page_mappings_only)

{

	pgprot_t __prot = prot;

	pte_t *pte;



	BUG_ON(pmd_sect(*pmd));
@@ -126,7 +128,18 @@ static void alloc_init_pte(pmd_t *pmd, unsigned long addr, 
	do {

		pte_t old_pte = *pte;



		set_pte(pte, pfn_pte(pfn, prot));

		/*

		 * Set the contiguous bit for the subsequent group of PTEs if

		 * its size and alignment are appropriate.

		 */

		if (((addr | PFN_PHYS(pfn)) & ~CONT_PTE_MASK) == 0) {

			if (end - addr >= CONT_PTE_SIZE && !page_mappings_only)

				__prot = __pgprot(pgprot_val(prot) | PTE_CONT);

			else

				__prot = prot;

		}



		set_pte(pte, pfn_pte(pfn, __prot));

		pfn++;



		/*
@@ -145,6 +158,7 @@ static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end, 
				  phys_addr_t (*pgtable_alloc)(void),

				  bool page_mappings_only)

{

	pgprot_t __prot = prot;

	pmd_t *pmd;

	unsigned long next;
@@ -171,7 +185,18 @@ static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end, 
		/* try section mapping first */

		if (((addr | next | phys) & ~SECTION_MASK) == 0 &&

		      !page_mappings_only) {

			pmd_set_huge(pmd, phys, prot);

			/*

			 * Set the contiguous bit for the subsequent group of

			 * PMDs if its size and alignment are appropriate.

			 */

			if (((addr | phys) & ~CONT_PMD_MASK) == 0) {

				if (end - addr >= CONT_PMD_SIZE)

					__prot = __pgprot(pgprot_val(prot) |

							  PTE_CONT);

				else

					__prot = prot;

			}

			pmd_set_huge(pmd, phys, __prot);



			/*

			 * After the PMD entry has been populated once, we
@@ -181,7 +206,8 @@ static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end, 
						      pmd_val(*pmd)));

		} else {

			alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys),

				       prot, pgtable_alloc);

				       prot, pgtable_alloc,

				       page_mappings_only);



			BUG_ON(pmd_val(old_pmd) != 0 &&

			       pmd_val(old_pmd) != pmd_val(*pmd));

CRA Git | Maintained and supported by SUSTech CRA and CCSE