Commit e9d55e15 authored by Aneesh Kumar K.V's avatar Aneesh Kumar K.V Committed by Linus Torvalds
Browse files

mm: change the interface for __tlb_remove_page() 

This updates the generic and arch specific implementation to return true
if we need to do a tlb flush.  That means if a __tlb_remove_page
indicate a flush is needed, the page we try to remove need to be tracked
and added again after the flush.  We need to track it because we have
already update the pte to none and we can't just loop back.

This change is done to enable us to do a tlb_flush when we try to flush
a range that consists of different page sizes.  For architectures like
ppc64, we can do a range based tlb flush and we need to track page size
for that.  When we try to remove a huge page, we will force a tlb flush
and starts a new mmu gather.

[aneesh.kumar@linux.vnet.ibm.com: mm-change-the-interface-for-__tlb_remove_page-v3]
  Link: http://lkml.kernel.org/r/1465049193-22197-2-git-send-email-aneesh.kumar@linux.vnet.ibm.com
Link: http://lkml.kernel.org/r/1464860389-29019-2-git-send-email-aneesh.kumar@linux.vnet.ibm.com


Signed-off-by: default avatarAneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Hugh Dickins <hughd@google.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Minchan Kim <minchan.kim@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 31d49da5

arch/arm/include/asm/tlb.h

+13 −4
Original line number Diff line number Diff line
@@ -209,17 +209,26 @@ tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) 
		tlb_flush(tlb);

}



static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)

static inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page)

{

	if (tlb->nr == tlb->max)

		return true;

	tlb->pages[tlb->nr++] = page;

	VM_BUG_ON(tlb->nr > tlb->max);

	return tlb->max - tlb->nr;

	return false;

}



static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)

{

	if (!__tlb_remove_page(tlb, page))

	if (__tlb_remove_page(tlb, page)) {

		tlb_flush_mmu(tlb);

		__tlb_remove_page(tlb, page);

	}

}



static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,

					 struct page *page)

{

	return __tlb_remove_page(tlb, page);

}



static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,

arch/ia64/include/asm/tlb.h

+14 −5
Original line number Diff line number Diff line
@@ -205,17 +205,18 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) 
 * must be delayed until after the TLB has been flushed (see comments at the beginning of

 * this file).

 */

static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)

static inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page)

{

	if (tlb->nr == tlb->max)

		return true;



	tlb->need_flush = 1;



	if (!tlb->nr && tlb->pages == tlb->local)

		__tlb_alloc_page(tlb);



	tlb->pages[tlb->nr++] = page;

	VM_BUG_ON(tlb->nr > tlb->max);



	return tlb->max - tlb->nr;

	return false;

}



static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
@@ -235,8 +236,16 @@ static inline void tlb_flush_mmu(struct mmu_gather *tlb) 


static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)

{

	if (!__tlb_remove_page(tlb, page))

	if (__tlb_remove_page(tlb, page)) {

		tlb_flush_mmu(tlb);

		__tlb_remove_page(tlb, page);

	}

}



static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,

					 struct page *page)

{

	return __tlb_remove_page(tlb, page);

}



/*

arch/s390/include/asm/tlb.h

+7 −2
Original line number Diff line number Diff line
@@ -87,10 +87,10 @@ static inline void tlb_finish_mmu(struct mmu_gather *tlb, 
 * tlb_ptep_clear_flush. In both flush modes the tlb for a page cache page

 * has already been freed, so just do free_page_and_swap_cache.

 */

static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)

static inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page)

{

	free_page_and_swap_cache(page);

	return 1; /* avoid calling tlb_flush_mmu */

	return false; /* avoid calling tlb_flush_mmu */

}



static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
@@ -98,6 +98,11 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) 
	free_page_and_swap_cache(page);

}



static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,

					 struct page *page)

{

	return __tlb_remove_page(tlb, page);

}

/*

 * pte_free_tlb frees a pte table and clears the CRSTE for the

 * page table from the tlb.

arch/sh/include/asm/tlb.h

+7 −1
Original line number Diff line number Diff line
@@ -101,7 +101,7 @@ static inline void tlb_flush_mmu(struct mmu_gather *tlb) 
static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)

{

	free_page_and_swap_cache(page);

	return 1; /* avoid calling tlb_flush_mmu */

	return false; /* avoid calling tlb_flush_mmu */

}



static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
@@ -109,6 +109,12 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) 
	__tlb_remove_page(tlb, page);

}



static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,

					 struct page *page)

{

	return __tlb_remove_page(tlb, page);

}



#define pte_free_tlb(tlb, ptep, addr)	pte_free((tlb)->mm, ptep)

#define pmd_free_tlb(tlb, pmdp, addr)	pmd_free((tlb)->mm, pmdp)

#define pud_free_tlb(tlb, pudp, addr)	pud_free((tlb)->mm, pudp)

arch/um/include/asm/tlb.h

+7 −1
Original line number Diff line number Diff line
@@ -102,7 +102,7 @@ static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page) 
{

	tlb->need_flush = 1;

	free_page_and_swap_cache(page);

	return 1; /* avoid calling tlb_flush_mmu */

	return false; /* avoid calling tlb_flush_mmu */

}



static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
@@ -110,6 +110,12 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) 
	__tlb_remove_page(tlb, page);

}



static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,

					 struct page *page)

{

	return __tlb_remove_page(tlb, page);

}



/**

 * tlb_remove_tlb_entry - remember a pte unmapping for later tlb invalidation.

 *

CRA Git | Maintained and supported by SUSTech CRA and CCSE