Commit fd60775a authored by David Rientjes's avatar David Rientjes Committed by Linus Torvalds
Browse files

mm, thp: avoid unlikely branches for split_huge_pmd 

While doing MADV_DONTNEED on a large area of thp memory, I noticed we
encountered many unlikely() branches in profiles for each backing
hugepage.  This is because zap_pmd_range() would call split_huge_pmd(),
which rechecked the conditions that were already validated, but as part
of an unlikely() branch.

Avoid the unlikely() branch when in a context where pmd is known to be
good for __split_huge_pmd() directly.

Link: http://lkml.kernel.org/r/alpine.DEB.2.10.1610181600300.84525@chino.kir.corp.google.com


Signed-off-by: default avatarDavid Rientjes <rientjes@google.com>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 3f500069

include/linux/huge_mm.h

+2 −0
Original line number Diff line number Diff line
@@ -189,6 +189,8 @@ static inline void deferred_split_huge_page(struct page *page) {} 
#define split_huge_pmd(__vma, __pmd, __address)	\

	do { } while (0)



static inline void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,

		unsigned long address, bool freeze, struct page *page) {}

static inline void split_huge_pmd_address(struct vm_area_struct *vma,

		unsigned long address, bool freeze, struct page *page) {}

mm/memory.c

+2 −2
Original line number Diff line number Diff line
@@ -1240,7 +1240,7 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb, 
			if (next - addr != HPAGE_PMD_SIZE) {

				VM_BUG_ON_VMA(vma_is_anonymous(vma) &&

				    !rwsem_is_locked(&tlb->mm->mmap_sem), vma);

				split_huge_pmd(vma, pmd, addr);

				__split_huge_pmd(vma, pmd, addr, false, NULL);

			} else if (zap_huge_pmd(tlb, vma, pmd, addr))

				goto next;

			/* fall through */
@@ -3454,7 +3454,7 @@ static int wp_huge_pmd(struct fault_env *fe, pmd_t orig_pmd) 


	/* COW handled on pte level: split pmd */

	VM_BUG_ON_VMA(fe->vma->vm_flags & VM_SHARED, fe->vma);

	split_huge_pmd(fe->vma, fe->pmd, fe->address);

	__split_huge_pmd(fe->vma, fe->pmd, fe->address, false, NULL);



	return VM_FAULT_FALLBACK;

}

mm/mempolicy.c

+1 −1
Original line number Diff line number Diff line
@@ -496,7 +496,7 @@ static int queue_pages_pte_range(pmd_t *pmd, unsigned long addr, 
			page = pmd_page(*pmd);

			if (is_huge_zero_page(page)) {

				spin_unlock(ptl);

				split_huge_pmd(vma, pmd, addr);

				__split_huge_pmd(vma, pmd, addr, false, NULL);

			} else {

				get_page(page);

				spin_unlock(ptl);

mm/mprotect.c

+1 −1
Original line number Diff line number Diff line
@@ -176,7 +176,7 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma, 


		if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {

			if (next - addr != HPAGE_PMD_SIZE) {

				split_huge_pmd(vma, pmd, addr);

				__split_huge_pmd(vma, pmd, addr, false, NULL);

				if (pmd_trans_unstable(pmd))

					continue;

			} else {

CRA Git | Maintained and supported by SUSTech CRA and CCSE