Commit d616d512 authored by Minchan Kim's avatar Minchan Kim Committed by Linus Torvalds
Browse files

mm: factor out common parts between MADV_COLD and MADV_PAGEOUT 

There are many common parts between MADV_COLD and MADV_PAGEOUT.
This patch factor them out to save code duplication.

Link: http://lkml.kernel.org/r/20190726023435.214162-6-minchan@kernel.org


Signed-off-by: default avatarMinchan Kim <minchan@kernel.org>
Suggested-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Acked-by: default avatarMichal Hocko <mhocko@suse.com>
Cc: Chris Zankel <chris@zankel.net>
Cc: Daniel Colascione <dancol@google.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Hillf Danton <hdanton@sina.com>
Cc: James E.J. Bottomley <James.Bottomley@HansenPartnership.com>
Cc: Joel Fernandes (Google) <joel@joelfernandes.org>
Cc: kbuild test robot <lkp@intel.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Oleksandr Natalenko <oleksandr@redhat.com>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: Richard Henderson <rth@twiddle.net>
Cc: Shakeel Butt <shakeelb@google.com>
Cc: Sonny Rao <sonnyrao@google.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Tim Murray <timmurray@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 1a4e58cc

mm/madvise.c

+45 −147
Original line number Diff line number Diff line
@@ -32,6 +32,11 @@ 


#include "internal.h"



struct madvise_walk_private {

	struct mmu_gather *tlb;

	bool pageout;

};



/*

 * Any behaviour which results in changes to the vma->vm_flags needs to

 * take mmap_sem for writing. Others, which simply traverse vmas, need
@@ -292,15 +297,22 @@ static long madvise_willneed(struct vm_area_struct *vma, 
	return 0;

}



static int madvise_cold_pte_range(pmd_t *pmd, unsigned long addr,

				unsigned long end, struct mm_walk *walk)

static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,

				unsigned long addr, unsigned long end,

				struct mm_walk *walk)

{

	struct mmu_gather *tlb = walk->private;

	struct madvise_walk_private *private = walk->private;

	struct mmu_gather *tlb = private->tlb;

	bool pageout = private->pageout;

	struct mm_struct *mm = tlb->mm;

	struct vm_area_struct *vma = walk->vma;

	pte_t *orig_pte, *pte, ptent;

	spinlock_t *ptl;

	struct page *page;

	struct page *page = NULL;

	LIST_HEAD(page_list);



	if (fatal_signal_pending(current))

		return -EINTR;



#ifdef CONFIG_TRANSPARENT_HUGEPAGE

	if (pmd_trans_huge(*pmd)) {
@@ -348,10 +360,17 @@ static int madvise_cold_pte_range(pmd_t *pmd, unsigned long addr, 
			tlb_remove_pmd_tlb_entry(tlb, pmd, addr);

		}



		ClearPageReferenced(page);

		test_and_clear_page_young(page);

		if (pageout) {

			if (!isolate_lru_page(page))

				list_add(&page->lru, &page_list);

		} else

			deactivate_page(page);

huge_unlock:

		spin_unlock(ptl);

		if (pageout)

			reclaim_pages(&page_list);

		return 0;

	}
@@ -419,27 +438,39 @@ regular_page: 
		 * As a side effect, it makes confuse idle-page tracking

		 * because they will miss recent referenced history.

		 */

		ClearPageReferenced(page);

		test_and_clear_page_young(page);

		if (pageout) {

			if (!isolate_lru_page(page))

				list_add(&page->lru, &page_list);

		} else

			deactivate_page(page);

	}



	arch_leave_lazy_mmu_mode();

	pte_unmap_unlock(orig_pte, ptl);

	if (pageout)

		reclaim_pages(&page_list);

	cond_resched();



	return 0;

}



static const struct mm_walk_ops cold_walk_ops = {

	.pmd_entry = madvise_cold_pte_range,

	.pmd_entry = madvise_cold_or_pageout_pte_range,

};



static void madvise_cold_page_range(struct mmu_gather *tlb,

			     struct vm_area_struct *vma,

			     unsigned long addr, unsigned long end)

{

	struct madvise_walk_private walk_private = {

		.pageout = false,

		.tlb = tlb,

	};



	tlb_start_vma(tlb, vma);

	walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, NULL);

	walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private);

	tlb_end_vma(tlb, vma);

}
@@ -462,150 +493,17 @@ static long madvise_cold(struct vm_area_struct *vma, 
	return 0;

}



static int madvise_pageout_pte_range(pmd_t *pmd, unsigned long addr,

				unsigned long end, struct mm_walk *walk)

{

	struct mmu_gather *tlb = walk->private;

	struct mm_struct *mm = tlb->mm;

	struct vm_area_struct *vma = walk->vma;

	pte_t *orig_pte, *pte, ptent;

	spinlock_t *ptl;

	LIST_HEAD(page_list);

	struct page *page;



	if (fatal_signal_pending(current))

		return -EINTR;



#ifdef CONFIG_TRANSPARENT_HUGEPAGE

	if (pmd_trans_huge(*pmd)) {

		pmd_t orig_pmd;

		unsigned long next = pmd_addr_end(addr, end);



		tlb_change_page_size(tlb, HPAGE_PMD_SIZE);

		ptl = pmd_trans_huge_lock(pmd, vma);

		if (!ptl)

			return 0;



		orig_pmd = *pmd;

		if (is_huge_zero_pmd(orig_pmd))

			goto huge_unlock;



		if (unlikely(!pmd_present(orig_pmd))) {

			VM_BUG_ON(thp_migration_supported() &&

					!is_pmd_migration_entry(orig_pmd));

			goto huge_unlock;

		}



		page = pmd_page(orig_pmd);

		if (next - addr != HPAGE_PMD_SIZE) {

			int err;



			if (page_mapcount(page) != 1)

				goto huge_unlock;

			get_page(page);

			spin_unlock(ptl);

			lock_page(page);

			err = split_huge_page(page);

			unlock_page(page);

			put_page(page);

			if (!err)

				goto regular_page;

			return 0;

		}



		if (pmd_young(orig_pmd)) {

			pmdp_invalidate(vma, addr, pmd);

			orig_pmd = pmd_mkold(orig_pmd);



			set_pmd_at(mm, addr, pmd, orig_pmd);

			tlb_remove_tlb_entry(tlb, pmd, addr);

		}



		ClearPageReferenced(page);

		test_and_clear_page_young(page);



		if (!isolate_lru_page(page))

			list_add(&page->lru, &page_list);

huge_unlock:

		spin_unlock(ptl);

		reclaim_pages(&page_list);

		return 0;

	}



	if (pmd_trans_unstable(pmd))

		return 0;

regular_page:

#endif

	tlb_change_page_size(tlb, PAGE_SIZE);

	orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);

	flush_tlb_batched_pending(mm);

	arch_enter_lazy_mmu_mode();

	for (; addr < end; pte++, addr += PAGE_SIZE) {

		ptent = *pte;

		if (!pte_present(ptent))

			continue;



		page = vm_normal_page(vma, addr, ptent);

		if (!page)

			continue;



		/*

		 * creating a THP page is expensive so split it only if we

		 * are sure it's worth. Split it if we are only owner.

		 */

		if (PageTransCompound(page)) {

			if (page_mapcount(page) != 1)

				break;

			get_page(page);

			if (!trylock_page(page)) {

				put_page(page);

				break;

			}

			pte_unmap_unlock(orig_pte, ptl);

			if (split_huge_page(page)) {

				unlock_page(page);

				put_page(page);

				pte_offset_map_lock(mm, pmd, addr, &ptl);

				break;

			}

			unlock_page(page);

			put_page(page);

			pte = pte_offset_map_lock(mm, pmd, addr, &ptl);

			pte--;

			addr -= PAGE_SIZE;

			continue;

		}



		VM_BUG_ON_PAGE(PageTransCompound(page), page);



		if (pte_young(ptent)) {

			ptent = ptep_get_and_clear_full(mm, addr, pte,

							tlb->fullmm);

			ptent = pte_mkold(ptent);

			set_pte_at(mm, addr, pte, ptent);

			tlb_remove_tlb_entry(tlb, pte, addr);

		}

		ClearPageReferenced(page);

		test_and_clear_page_young(page);



		if (!isolate_lru_page(page))

			list_add(&page->lru, &page_list);

	}



	arch_leave_lazy_mmu_mode();

	pte_unmap_unlock(orig_pte, ptl);

	reclaim_pages(&page_list);

	cond_resched();



	return 0;

}



static void madvise_pageout_page_range(struct mmu_gather *tlb,

			     struct vm_area_struct *vma,

			     unsigned long addr, unsigned long end)

{

	struct madvise_walk_private walk_private = {

		.pageout = true,

		.tlb = tlb,

	};



	tlb_start_vma(tlb, vma);

	walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, NULL);

	walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private);

	tlb_end_vma(tlb, vma);

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE