Commit 89f5b7da authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Reinstate ZERO_PAGE optimization in 'get_user_pages()' and fix XIP 



KAMEZAWA Hiroyuki and Oleg Nesterov point out that since the commit
557ed1fa ("remove ZERO_PAGE") removed
the ZERO_PAGE from the VM mappings, any users of get_user_pages() will
generally now populate the VM with real empty pages needlessly.

We used to get the ZERO_PAGE when we did the "handle_mm_fault()", but
since fault handling no longer uses ZERO_PAGE for new anonymous pages,
we now need to handle that special case in follow_page() instead.

In particular, the removal of ZERO_PAGE effectively removed the core
file writing optimization where we would skip writing pages that had not
been populated at all, and increased memory pressure a lot by allocating
all those useless newly zeroed pages.

This reinstates the optimization by making the unmapped PTE case the
same as for a non-existent page table, which already did this correctly.

While at it, this also fixes the XIP case for follow_page(), where the
caller could not differentiate between the case of a page that simply
could not be used (because it had no "struct page" associated with it)
and a page that just wasn't mapped.

We do that by simply returning an error pointer for pages that could not
be turned into a "struct page *".  The error is arbitrarily picked to be
EFAULT, since that was what get_user_pages() already used for the
equivalent IO-mapped page case.

[ Also removed an impossible test for pte_offset_map_lock() failing:
  that's not how that function works ]

Acked-by: default avatarOleg Nesterov <oleg@tv-sign.ru>
Acked-by: default avatarNick Piggin <npiggin@suse.de>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Roland McGrath <roland@redhat.com>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 9bedbcb2

arch/powerpc/kernel/vdso.c

+1 −1
Original line number Original line Diff line number Diff line
@@ -142,7 +142,7 @@ static void dump_one_vdso_page(struct page *pg, struct page *upg) 
	printk("kpg: %p (c:%d,f:%08lx)", __va(page_to_pfn(pg) << PAGE_SHIFT),

	printk("kpg: %p (c:%d,f:%08lx)", __va(page_to_pfn(pg) << PAGE_SHIFT),

	       page_count(pg),

	       page_count(pg),

	       pg->flags);

	       pg->flags);

	if (upg/* && pg != upg*/) {

	if (upg && !IS_ERR(upg) /* && pg != upg*/) {

		printk(" upg: %p (c:%d,f:%08lx)", __va(page_to_pfn(upg)

		printk(" upg: %p (c:%d,f:%08lx)", __va(page_to_pfn(upg)

						       << PAGE_SHIFT),

						       << PAGE_SHIFT),

		       page_count(upg),

		       page_count(upg),

mm/memory.c

+13 −4
Original line number Original line Diff line number Diff line
@@ -999,17 +999,15 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address, 
		goto no_page_table;

		goto no_page_table;





	ptep = pte_offset_map_lock(mm, pmd, address, &ptl);

	ptep = pte_offset_map_lock(mm, pmd, address, &ptl);

	if (!ptep)

		goto out;





	pte = *ptep;

	pte = *ptep;

	if (!pte_present(pte))

	if (!pte_present(pte))

		goto unlock;

		goto no_page;

	if ((flags & FOLL_WRITE) && !pte_write(pte))

	if ((flags & FOLL_WRITE) && !pte_write(pte))

		goto unlock;

		goto unlock;

	page = vm_normal_page(vma, address, pte);

	page = vm_normal_page(vma, address, pte);

	if (unlikely(!page))

	if (unlikely(!page))

		goto unlock;

		goto bad_page;





	if (flags & FOLL_GET)

	if (flags & FOLL_GET)

		get_page(page);

		get_page(page);
@@ -1024,6 +1022,15 @@ unlock: 
out:

out:

	return page;

	return page;





bad_page:

	pte_unmap_unlock(ptep, ptl);

	return ERR_PTR(-EFAULT);



no_page:

	pte_unmap_unlock(ptep, ptl);

	if (!pte_none(pte))

		return page;

	/* Fall through to ZERO_PAGE handling */

no_page_table:

no_page_table:

	/*

	/*

	 * When core dumping an enormous anonymous area that nobody

	 * When core dumping an enormous anonymous area that nobody
@@ -1159,6 +1166,8 @@ int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 




				cond_resched();

				cond_resched();

			}

			}

			if (IS_ERR(page))

				return i ? i : PTR_ERR(page);

			if (pages) {

			if (pages) {

				pages[i] = page;

				pages[i] = page;

mm/migrate.c

+10 −0
Original line number Original line Diff line number Diff line
@@ -865,6 +865,11 @@ static int do_move_pages(struct mm_struct *mm, struct page_to_node *pm, 
			goto set_status;

			goto set_status;





		page = follow_page(vma, pp->addr, FOLL_GET);

		page = follow_page(vma, pp->addr, FOLL_GET);



		err = PTR_ERR(page);

		if (IS_ERR(page))

			goto set_status;



		err = -ENOENT;

		err = -ENOENT;

		if (!page)

		if (!page)

			goto set_status;

			goto set_status;
@@ -928,6 +933,11 @@ static int do_pages_stat(struct mm_struct *mm, struct page_to_node *pm) 
			goto set_status;

			goto set_status;





		page = follow_page(vma, pm->addr, 0);

		page = follow_page(vma, pm->addr, 0);



		err = PTR_ERR(page);

		if (IS_ERR(page))

			goto set_status;



		err = -ENOENT;

		err = -ENOENT;

		/* Use PageReserved to check for zero page */

		/* Use PageReserved to check for zero page */

		if (!page || PageReserved(page))

		if (!page || PageReserved(page))

CRA Git | Maintained and supported by SUSTech CRA and CCSE