Commit fb7332a9 authored by Will Deacon's avatar Will Deacon
Browse files

mmu_gather: move minimal range calculations into generic code 



On architectures with hardware broadcasting of TLB invalidation messages
, it makes sense to reduce the range of the mmu_gather structure when
unmapping page ranges based on the dirty address information passed to
tlb_remove_tlb_entry.

arm64 already does this by directly manipulating the start/end fields
of the gather structure, but this confuses the generic code which
does not expect these fields to change and can end up calculating
invalid, negative ranges when forcing a flush in zap_pte_range.

This patch moves the minimal range calculation out of the arm64 code
and into the generic implementation, simplifying zap_pte_range in the
process (which no longer needs to care about start/end, since they will
point to the appropriate ranges already). With the range being tracked
by core code, the need_flush flag is dropped in favour of checking that
the end of the range has actually been set.

Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Russell King - ARM Linux <linux@arm.linux.org.uk>
Cc: Michal Simek <monstr@monstr.eu>
Acked-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: default avatarWill Deacon <will.deacon@arm.com>
parent 63648dd2

arch/arm64/include/asm/tlb.h

+3 −64
Original line number Diff line number Diff line
@@ -19,10 +19,6 @@ 
#ifndef __ASM_TLB_H

#define __ASM_TLB_H



#define  __tlb_remove_pmd_tlb_entry __tlb_remove_pmd_tlb_entry



#include <asm-generic/tlb.h>



#include <linux/pagemap.h>

#include <linux/swap.h>
@@ -37,71 +33,22 @@ static inline void __tlb_remove_table(void *_table) 
#define tlb_remove_entry(tlb, entry)	tlb_remove_page(tlb, entry)

#endif /* CONFIG_HAVE_RCU_TABLE_FREE */



/*

 * There's three ways the TLB shootdown code is used:

 *  1. Unmapping a range of vmas.  See zap_page_range(), unmap_region().

 *     tlb->fullmm = 0, and tlb_start_vma/tlb_end_vma will be called.

 *  2. Unmapping all vmas.  See exit_mmap().

 *     tlb->fullmm = 1, and tlb_start_vma/tlb_end_vma will be called.

 *     Page tables will be freed.

 *  3. Unmapping argument pages.  See shift_arg_pages().

 *     tlb->fullmm = 0, but tlb_start_vma/tlb_end_vma will not be called.

 */

#include <asm-generic/tlb.h>



static inline void tlb_flush(struct mmu_gather *tlb)

{

	if (tlb->fullmm) {

		flush_tlb_mm(tlb->mm);

	} else if (tlb->end > 0) {

	} else {

		struct vm_area_struct vma = { .vm_mm = tlb->mm, };

		flush_tlb_range(&vma, tlb->start, tlb->end);

		tlb->start = TASK_SIZE;

		tlb->end = 0;

	}

}



static inline void tlb_add_flush(struct mmu_gather *tlb, unsigned long addr)

{

	if (!tlb->fullmm) {

		tlb->start = min(tlb->start, addr);

		tlb->end = max(tlb->end, addr + PAGE_SIZE);

	}

}



/*

 * Memorize the range for the TLB flush.

 */

static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep,

					  unsigned long addr)

{

	tlb_add_flush(tlb, addr);

}



/*

 * In the case of tlb vma handling, we can optimise these away in the

 * case where we're doing a full MM flush.  When we're doing a munmap,

 * the vmas are adjusted to only cover the region to be torn down.

 */

static inline void tlb_start_vma(struct mmu_gather *tlb,

				 struct vm_area_struct *vma)

{

	if (!tlb->fullmm) {

		tlb->start = TASK_SIZE;

		tlb->end = 0;

	}

}



static inline void tlb_end_vma(struct mmu_gather *tlb,

			       struct vm_area_struct *vma)

{

	if (!tlb->fullmm)

		tlb_flush(tlb);

}



static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,

				  unsigned long addr)

{

	pgtable_page_dtor(pte);

	tlb_add_flush(tlb, addr);

	tlb_remove_entry(tlb, pte);

}
@@ -109,7 +56,6 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, 
static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,

				  unsigned long addr)

{

	tlb_add_flush(tlb, addr);

	tlb_remove_entry(tlb, virt_to_page(pmdp));

}

#endif
@@ -118,15 +64,8 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, 
static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp,

				  unsigned long addr)

{

	tlb_add_flush(tlb, addr);

	tlb_remove_entry(tlb, virt_to_page(pudp));

}

#endif



static inline void __tlb_remove_pmd_tlb_entry(struct mmu_gather *tlb, pmd_t *pmdp,

						unsigned long address)

{

	tlb_add_flush(tlb, address);

}



#endif

arch/microblaze/include/asm/tlb.h

+2 −1
Original line number Diff line number Diff line
@@ -14,7 +14,6 @@ 
#define tlb_flush(tlb)	flush_tlb_mm((tlb)->mm)



#include <linux/pagemap.h>

#include <asm-generic/tlb.h>



#ifdef CONFIG_MMU

#define tlb_start_vma(tlb, vma)		do { } while (0)
@@ -22,4 +21,6 @@ 
#define __tlb_remove_tlb_entry(tlb, pte, address) do { } while (0)

#endif



#include <asm-generic/tlb.h>



#endif /* _ASM_MICROBLAZE_TLB_H */

arch/powerpc/include/asm/pgalloc.h

+2 −1
Original line number Diff line number Diff line
@@ -3,7 +3,6 @@ 
#ifdef __KERNEL__



#include <linux/mm.h>

#include <asm-generic/tlb.h>



#ifdef CONFIG_PPC_BOOK3E

extern void tlb_flush_pgtable(struct mmu_gather *tlb, unsigned long address);
@@ -14,6 +13,8 @@ static inline void tlb_flush_pgtable(struct mmu_gather *tlb, 
}

#endif /* !CONFIG_PPC_BOOK3E */



extern void tlb_remove_table(struct mmu_gather *tlb, void *table);



#ifdef CONFIG_PPC64

#include <asm/pgalloc-64.h>

#else

arch/powerpc/include/asm/tlb.h

+1 −0
Original line number Diff line number Diff line
@@ -27,6 +27,7 @@ 


#define tlb_start_vma(tlb, vma)	do { } while (0)

#define tlb_end_vma(tlb, vma)	do { } while (0)

#define __tlb_remove_tlb_entry	__tlb_remove_tlb_entry



extern void tlb_flush(struct mmu_gather *tlb);

arch/powerpc/mm/hugetlbpage.c

+0 −2
Original line number Diff line number Diff line
@@ -517,8 +517,6 @@ static void free_hugepd_range(struct mmu_gather *tlb, hugepd_t *hpdp, int pdshif 
	for (i = 0; i < num_hugepd; i++, hpdp++)

		hpdp->pd = 0;



	tlb->need_flush = 1;



#ifdef CONFIG_PPC_FSL_BOOK3E

	hugepd_free(tlb, hugepte);

#else

CRA Git | Maintained and supported by SUSTech CRA and CCSE