Commit 953c66c2 authored by Aneesh Kumar K.V's avatar Aneesh Kumar K.V Committed by Linus Torvalds
Browse files

mm: THP page cache support for ppc64 

Add arch specific callback in the generic THP page cache code that will
deposit and withdarw preallocated page table.  Archs like ppc64 use this
preallocated table to store the hash pte slot information.

Testing:
kernel build of the patch series on tmpfs mounted with option huge=always

The related thp stat:
thp_fault_alloc 72939
thp_fault_fallback 60547
thp_collapse_alloc 603
thp_collapse_alloc_failed 0
thp_file_alloc 253763
thp_file_mapped 4251
thp_split_page 51518
thp_split_page_failed 1
thp_deferred_split_page 73566
thp_split_pmd 665
thp_zero_page_alloc 3
thp_zero_page_alloc_failed 0

[akpm@linux-foundation.org: remove unneeded parentheses, per Kirill]
Link: http://lkml.kernel.org/r/20161113150025.17942-2-aneesh.kumar@linux.vnet.ibm.com


Signed-off-by: default avatarAneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Acked-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Michael Neuling <mikey@neuling.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Balbir Singh <bsingharora@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 1dd38b6c

arch/powerpc/include/asm/book3s/64/pgtable.h

+10 −0
Original line number Diff line number Diff line
@@ -1021,6 +1021,16 @@ static inline int pmd_move_must_withdraw(struct spinlock *new_pmd_ptl, 
	 */

	return true;

}





#define arch_needs_pgtable_deposit arch_needs_pgtable_deposit

static inline bool arch_needs_pgtable_deposit(void)

{

	if (radix_enabled())

		return false;

	return true;

}



#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

#endif /* __ASSEMBLY__ */

#endif /* _ASM_POWERPC_BOOK3S_64_PGTABLE_H_ */

include/asm-generic/pgtable.h

+3 −0
Original line number Diff line number Diff line
@@ -652,6 +652,9 @@ static inline pmd_t pmd_read_atomic(pmd_t *pmdp) 
}

#endif



#ifndef arch_needs_pgtable_deposit

#define arch_needs_pgtable_deposit() (false)

#endif

/*

 * This function is meant to be used by sites walking pagetables with

 * the mmap_sem hold in read mode to protect against MADV_DONTNEED and

mm/Kconfig

+1 −5
Original line number Diff line number Diff line
@@ -447,13 +447,9 @@ choice 
	  benefit.

endchoice



#

# We don't deposit page tables on file THP mapping,

# but Power makes use of them to address MMU quirk.

#

config	TRANSPARENT_HUGE_PAGECACHE

	def_bool y

	depends on TRANSPARENT_HUGEPAGE && !PPC

	depends on TRANSPARENT_HUGEPAGE



#

# UP and nommu archs use km based percpu allocator

mm/huge_memory.c

+17 −0
Original line number Diff line number Diff line
@@ -1380,6 +1380,15 @@ out_unlocked: 
	return ret;

}



static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)

{

	pgtable_t pgtable;



	pgtable = pgtable_trans_huge_withdraw(mm, pmd);

	pte_free(mm, pgtable);

	atomic_long_dec(&mm->nr_ptes);

}



int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,

		 pmd_t *pmd, unsigned long addr)

{
@@ -1421,6 +1430,8 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 
			atomic_long_dec(&tlb->mm->nr_ptes);

			add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);

		} else {

			if (arch_needs_pgtable_deposit())

				zap_deposited_table(tlb->mm, pmd);

			add_mm_counter(tlb->mm, MM_FILEPAGES, -HPAGE_PMD_NR);

		}

		spin_unlock(ptl);
@@ -1607,6 +1618,12 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, 


	if (!vma_is_anonymous(vma)) {

		_pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd);

		/*

		 * We are going to unmap this huge page. So

		 * just go ahead and zap it

		 */

		if (arch_needs_pgtable_deposit())

			zap_deposited_table(mm, pmd);

		if (vma_is_dax(vma))

			return;

		page = pmd_page(_pmd);

mm/khugepaged.c

+19 −2
Original line number Diff line number Diff line
@@ -1242,6 +1242,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) 
	struct vm_area_struct *vma;

	unsigned long addr;

	pmd_t *pmd, _pmd;

	bool deposited = false;



	i_mmap_lock_write(mapping);

	vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
@@ -1266,12 +1267,28 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) 
			spinlock_t *ptl = pmd_lock(vma->vm_mm, pmd);

			/* assume page table is clear */

			_pmd = pmdp_collapse_flush(vma, addr, pmd);

			/*

			 * now deposit the pgtable for arch that need it

			 * otherwise free it.

			 */

			if (arch_needs_pgtable_deposit()) {

				/*

				 * The deposit should be visibile only after

				 * collapse is seen by others.

				 */

				smp_wmb();

				pgtable_trans_huge_deposit(vma->vm_mm, pmd,

							   pmd_pgtable(_pmd));

				deposited = true;

			}

			spin_unlock(ptl);

			up_write(&vma->vm_mm->mmap_sem);

			if (!deposited) {

				atomic_long_dec(&vma->vm_mm->nr_ptes);

				pte_free(vma->vm_mm, pmd_pgtable(_pmd));

			}

		}

	}

	i_mmap_unlock_write(mapping);

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE