Commit 29e56940 authored by Steve Capper's avatar Steve Capper Committed by Linus Torvalds
Browse files

arm64: mm: enable RCU fast_gup 



Activate the RCU fast_gup for ARM64.  We also need to force THP splits to
broadcast an IPI s.t.  we block in the fast_gup page walker.  As THP
splits are comparatively rare, this should not lead to a noticeable
performance degradation.

Some pre-requisite functions pud_write and pud_page are also added.

[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: default avatarSteve Capper <steve.capper@linaro.org>
Tested-by: default avatarDann Frazier <dann.frazier@canonical.com>
Acked-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Russell King <rmk@arm.linux.org.uk>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Christoffer Dall <christoffer.dall@linaro.org>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 5e5f6dc1

arch/arm64/Kconfig

+3 −0
Original line number Diff line number Diff line
@@ -111,6 +111,9 @@ config GENERIC_CALIBRATE_DELAY 
config ZONE_DMA

	def_bool y



config HAVE_GENERIC_RCU_GUP

	def_bool y



config ARCH_DMA_ADDR_T_64BIT

	def_bool y

arch/arm64/include/asm/pgtable.h

+20 −1
Original line number Diff line number Diff line
@@ -244,6 +244,16 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, 


#define __HAVE_ARCH_PTE_SPECIAL



static inline pte_t pud_pte(pud_t pud)

{

	return __pte(pud_val(pud));

}



static inline pmd_t pud_pmd(pud_t pud)

{

	return __pmd(pud_val(pud));

}



static inline pte_t pmd_pte(pmd_t pmd)

{

	return __pte(pmd_val(pmd));
@@ -261,7 +271,13 @@ static inline pmd_t pte_pmd(pte_t pte) 
#ifdef CONFIG_TRANSPARENT_HUGEPAGE

#define pmd_trans_huge(pmd)	(pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT))

#define pmd_trans_splitting(pmd)	pte_special(pmd_pte(pmd))

#endif

#ifdef CONFIG_HAVE_RCU_TABLE_FREE

#define __HAVE_ARCH_PMDP_SPLITTING_FLUSH

struct vm_area_struct;

void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address,

			  pmd_t *pmdp);

#endif /* CONFIG_HAVE_RCU_TABLE_FREE */

#endif /* CONFIG_TRANSPARENT_HUGEPAGE */



#define pmd_young(pmd)		pte_young(pmd_pte(pmd))

#define pmd_wrprotect(pmd)	pte_pmd(pte_wrprotect(pmd_pte(pmd)))
@@ -282,6 +298,7 @@ static inline pmd_t pte_pmd(pte_t pte) 
#define mk_pmd(page,prot)	pfn_pmd(page_to_pfn(page),prot)



#define pmd_page(pmd)           pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK))

#define pud_write(pud)		pte_write(pud_pte(pud))

#define pud_pfn(pud)		(((pud_val(pud) & PUD_MASK) & PHYS_MASK) >> PAGE_SHIFT)



#define set_pmd_at(mm, addr, pmdp, pmd)	set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd))
@@ -381,6 +398,8 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) 
	return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(addr);

}



#define pud_page(pud)           pmd_page(pud_pmd(pud))



#endif	/* CONFIG_ARM64_PGTABLE_LEVELS > 2 */



#if CONFIG_ARM64_PGTABLE_LEVELS > 3

arch/arm64/mm/flush.c

+16 −0
Original line number Diff line number Diff line
@@ -104,3 +104,19 @@ EXPORT_SYMBOL(flush_dcache_page); 
 */

EXPORT_SYMBOL(flush_cache_all);

EXPORT_SYMBOL(flush_icache_range);



#ifdef CONFIG_TRANSPARENT_HUGEPAGE

#ifdef CONFIG_HAVE_RCU_TABLE_FREE

void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address,

			  pmd_t *pmdp)

{

	pmd_t pmd = pmd_mksplitting(*pmdp);



	VM_BUG_ON(address & ~PMD_MASK);

	set_pmd_at(vma->vm_mm, address, pmdp, pmd);



	/* dummy IPI to serialise against fast_gup */

	kick_all_cpus_sync();

}

#endif /* CONFIG_HAVE_RCU_TABLE_FREE */

#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

CRA Git | Maintained and supported by SUSTech CRA and CCSE