Commit 7af75561 authored by Ira Weiny's avatar Ira Weiny Committed by Linus Torvalds
Browse files

mm/gup: add FOLL_LONGTERM capability to GUP fast 

DAX pages were previously unprotected from longterm pins when users called
get_user_pages_fast().

Use the new FOLL_LONGTERM flag to check for DEVMAP pages and fall back to
regular GUP processing if a DEVMAP page is encountered.

[ira.weiny@intel.com: v3]
  Link: http://lkml.kernel.org/r/20190328084422.29911-5-ira.weiny@intel.com
Link: http://lkml.kernel.org/r/20190328084422.29911-5-ira.weiny@intel.com
Link: http://lkml.kernel.org/r/20190317183438.2057-5-ira.weiny@intel.com


Signed-off-by: default avatarIra Weiny <ira.weiny@intel.com>
Reviewed-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: James Hogan <jhogan@kernel.org>
Cc: Jason Gunthorpe <jgg@ziepe.ca>
Cc: John Hubbard <jhubbard@nvidia.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: Rich Felker <dalias@libc.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Yoshinori Sato <ysato@users.sourceforge.jp>
Cc: Mike Marshall <hubcap@omnibond.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 73b0140b

mm/gup.c

+36 −4
Original line number Diff line number Diff line
@@ -1637,6 +1637,9 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, 
			goto pte_unmap;



		if (pte_devmap(pte)) {

			if (unlikely(flags & FOLL_LONGTERM))

				goto pte_unmap;



			pgmap = get_dev_pagemap(pte_pfn(pte), pgmap);

			if (unlikely(!pgmap)) {

				undo_dev_pagemap(nr, nr_start, pages);
@@ -1776,8 +1779,11 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, 
	if (!pmd_access_permitted(orig, flags & FOLL_WRITE))

		return 0;



	if (pmd_devmap(orig))

	if (pmd_devmap(orig)) {

		if (unlikely(flags & FOLL_LONGTERM))

			return 0;

		return __gup_device_huge_pmd(orig, pmdp, addr, end, pages, nr);

	}



	refs = 0;

	page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
@@ -1814,8 +1820,11 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, 
	if (!pud_access_permitted(orig, flags & FOLL_WRITE))

		return 0;



	if (pud_devmap(orig))

	if (pud_devmap(orig)) {

		if (unlikely(flags & FOLL_LONGTERM))

			return 0;

		return __gup_device_huge_pud(orig, pudp, addr, end, pages, nr);

	}



	refs = 0;

	page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
@@ -2058,6 +2067,29 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write, 
	return nr;

}



static int __gup_longterm_unlocked(unsigned long start, int nr_pages,

				   unsigned int gup_flags, struct page **pages)

{

	int ret;



	/*

	 * FIXME: FOLL_LONGTERM does not work with

	 * get_user_pages_unlocked() (see comments in that function)

	 */

	if (gup_flags & FOLL_LONGTERM) {

		down_read(&current->mm->mmap_sem);

		ret = __gup_longterm_locked(current, current->mm,

					    start, nr_pages,

					    pages, NULL, gup_flags);

		up_read(&current->mm->mmap_sem);

	} else {

		ret = get_user_pages_unlocked(start, nr_pages,

					      pages, gup_flags);

	}



	return ret;

}



/**

 * get_user_pages_fast() - pin user pages in memory

 * @start:	starting user address
@@ -2103,8 +2135,8 @@ int get_user_pages_fast(unsigned long start, int nr_pages, 
		start += nr << PAGE_SHIFT;

		pages += nr;



		ret = get_user_pages_unlocked(start, nr_pages - nr, pages,

					      gup_flags);

		ret = __gup_longterm_unlocked(start, nr_pages - nr,

					      gup_flags, pages);



		/* Have to be a bit careful with return values */

		if (nr > 0) {

CRA Git | Maintained and supported by SUSTech CRA and CCSE