Commit 2631ed00 authored by Peter Zijlstra (Intel)'s avatar Peter Zijlstra (Intel) Committed by Catalin Marinas
Browse files

tlb: mmu_gather: add tlb_flush_*_range APIs 



tlb_flush_{pte|pmd|pud|p4d}_range() adjust the tlb->start and
tlb->end, then set corresponding cleared_*.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarZhenyu Ye <yezhenyu2@huawei.com>
Acked-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20200625080314.230-5-yezhenyu2@huawei.com


Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent e735b98a

include/asm-generic/tlb.h

+40 −15
Original line number Diff line number Diff line
@@ -512,6 +512,38 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm 
}

#endif



/*

 * tlb_flush_{pte|pmd|pud|p4d}_range() adjust the tlb->start and tlb->end,

 * and set corresponding cleared_*.

 */

static inline void tlb_flush_pte_range(struct mmu_gather *tlb,

				     unsigned long address, unsigned long size)

{

	__tlb_adjust_range(tlb, address, size);

	tlb->cleared_ptes = 1;

}



static inline void tlb_flush_pmd_range(struct mmu_gather *tlb,

				     unsigned long address, unsigned long size)

{

	__tlb_adjust_range(tlb, address, size);

	tlb->cleared_pmds = 1;

}



static inline void tlb_flush_pud_range(struct mmu_gather *tlb,

				     unsigned long address, unsigned long size)

{

	__tlb_adjust_range(tlb, address, size);

	tlb->cleared_puds = 1;

}



static inline void tlb_flush_p4d_range(struct mmu_gather *tlb,

				     unsigned long address, unsigned long size)

{

	__tlb_adjust_range(tlb, address, size);

	tlb->cleared_p4ds = 1;

}



#ifndef __tlb_remove_tlb_entry

#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)

#endif
@@ -525,19 +557,17 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm 
 */

#define tlb_remove_tlb_entry(tlb, ptep, address)		\

	do {							\

		__tlb_adjust_range(tlb, address, PAGE_SIZE);	\

		tlb->cleared_ptes = 1;				\

		tlb_flush_pte_range(tlb, address, PAGE_SIZE);	\

		__tlb_remove_tlb_entry(tlb, ptep, address);	\

	} while (0)



#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address)	\

	do {							\

		unsigned long _sz = huge_page_size(h);		\

		__tlb_adjust_range(tlb, address, _sz);		\

		if (_sz == PMD_SIZE)				\

			tlb->cleared_pmds = 1;			\

			tlb_flush_pmd_range(tlb, address, _sz);	\

		else if (_sz == PUD_SIZE)			\

			tlb->cleared_puds = 1;			\

			tlb_flush_pud_range(tlb, address, _sz);	\

		__tlb_remove_tlb_entry(tlb, ptep, address);	\

	} while (0)
@@ -551,8 +581,7 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm 


#define tlb_remove_pmd_tlb_entry(tlb, pmdp, address)			\

	do {								\

		__tlb_adjust_range(tlb, address, HPAGE_PMD_SIZE);	\

		tlb->cleared_pmds = 1;					\

		tlb_flush_pmd_range(tlb, address, HPAGE_PMD_SIZE);	\

		__tlb_remove_pmd_tlb_entry(tlb, pmdp, address);		\

	} while (0)
@@ -566,8 +595,7 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm 


#define tlb_remove_pud_tlb_entry(tlb, pudp, address)			\

	do {								\

		__tlb_adjust_range(tlb, address, HPAGE_PUD_SIZE);	\

		tlb->cleared_puds = 1;					\

		tlb_flush_pud_range(tlb, address, HPAGE_PUD_SIZE);	\

		__tlb_remove_pud_tlb_entry(tlb, pudp, address);		\

	} while (0)
@@ -592,9 +620,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm 
#ifndef pte_free_tlb

#define pte_free_tlb(tlb, ptep, address)			\

	do {							\

		__tlb_adjust_range(tlb, address, PAGE_SIZE);	\

		tlb_flush_pmd_range(tlb, address, PAGE_SIZE);	\

		tlb->freed_tables = 1;				\

		tlb->cleared_pmds = 1;				\

		__pte_free_tlb(tlb, ptep, address);		\

	} while (0)

#endif
@@ -602,9 +629,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm 
#ifndef pmd_free_tlb

#define pmd_free_tlb(tlb, pmdp, address)			\

	do {							\

		__tlb_adjust_range(tlb, address, PAGE_SIZE);	\

		tlb_flush_pud_range(tlb, address, PAGE_SIZE);	\

		tlb->freed_tables = 1;				\

		tlb->cleared_puds = 1;				\

		__pmd_free_tlb(tlb, pmdp, address);		\

	} while (0)

#endif
@@ -612,9 +638,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm 
#ifndef pud_free_tlb

#define pud_free_tlb(tlb, pudp, address)			\

	do {							\

		__tlb_adjust_range(tlb, address, PAGE_SIZE);	\

		tlb_flush_p4d_range(tlb, address, PAGE_SIZE);	\

		tlb->freed_tables = 1;				\

		tlb->cleared_p4ds = 1;				\

		__pud_free_tlb(tlb, pudp, address);		\

	} while (0)

#endif

CRA Git | Maintained and supported by SUSTech CRA and CCSE