Commit 050a9adc authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Linus Torvalds
Browse files

mm: consolidate the get_user_pages* implementations 

Always build mm/gup.c so that we don't have to provide separate nommu
stubs.  Also merge the get_user_pages_fast and __get_user_pages_fast stubs
when HAVE_FAST_GUP into the main implementations, which will never call
the fast path if HAVE_FAST_GUP is not set.

This also ensures the new put_user_pages* helpers are available for nommu,
as those are currently missing, which would create a problem as soon as we
actually grew users for it.

Link: http://lkml.kernel.org/r/20190625143715.1689-13-hch@lst.de


Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Cc: Andrey Konovalov <andreyknvl@google.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: David Miller <davem@davemloft.net>
Cc: James Hogan <jhogan@kernel.org>
Cc: Jason Gunthorpe <jgg@mellanox.com>
Cc: Khalid Aziz <khalid.aziz@oracle.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Paul Burton <paul.burton@mips.com>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: Rich Felker <dalias@libc.org>
Cc: Yoshinori Sato <ysato@users.sourceforge.jp>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent d3649f68

mm/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -133,6 +133,7 @@ config HAVE_MEMBLOCK_PHYS_MAP 
	bool



config HAVE_FAST_GUP

	depends on MMU

	bool



config ARCH_KEEP_MEMBLOCK

mm/Makefile

+2 −2
Original line number Diff line number Diff line
@@ -22,7 +22,7 @@ KCOV_INSTRUMENT_mmzone.o := n 
KCOV_INSTRUMENT_vmstat.o := n



mmu-y			:= nommu.o

mmu-$(CONFIG_MMU)	:= gup.o highmem.o memory.o mincore.o \

mmu-$(CONFIG_MMU)	:= highmem.o memory.o mincore.o \

			   mlock.o mmap.o mmu_gather.o mprotect.o mremap.o \

			   msync.o page_vma_mapped.o pagewalk.o \

			   pgtable-generic.o rmap.o vmalloc.o
@@ -39,7 +39,7 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ 
			   mm_init.o mmu_context.o percpu.o slab_common.o \

			   compaction.o vmacache.o \

			   interval_tree.o list_lru.o workingset.o \

			   debug.o $(mmu-y)

			   debug.o gup.o $(mmu-y)



# Give 'page_alloc' its own module-parameter namespace

page-alloc-y := page_alloc.o

mm/gup.c

+62 −5
Original line number Diff line number Diff line
@@ -134,6 +134,7 @@ void put_user_pages(struct page **pages, unsigned long npages) 
}

EXPORT_SYMBOL(put_user_pages);



#ifdef CONFIG_MMU

static struct page *no_page_table(struct vm_area_struct *vma,

		unsigned int flags)

{
@@ -1322,6 +1323,51 @@ struct page *get_dump_page(unsigned long addr) 
	return page;

}

#endif /* CONFIG_ELF_CORE */

#else /* CONFIG_MMU */

static long __get_user_pages_locked(struct task_struct *tsk,

		struct mm_struct *mm, unsigned long start,

		unsigned long nr_pages, struct page **pages,

		struct vm_area_struct **vmas, int *locked,

		unsigned int foll_flags)

{

	struct vm_area_struct *vma;

	unsigned long vm_flags;

	int i;



	/* calculate required read or write permissions.

	 * If FOLL_FORCE is set, we only require the "MAY" flags.

	 */

	vm_flags  = (foll_flags & FOLL_WRITE) ?

			(VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);

	vm_flags &= (foll_flags & FOLL_FORCE) ?

			(VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);



	for (i = 0; i < nr_pages; i++) {

		vma = find_vma(mm, start);

		if (!vma)

			goto finish_or_fault;



		/* protect what we can, including chardevs */

		if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) ||

		    !(vm_flags & vma->vm_flags))

			goto finish_or_fault;



		if (pages) {

			pages[i] = virt_to_page(start);

			if (pages[i])

				get_page(pages[i]);

		}

		if (vmas)

			vmas[i] = vma;

		start = (start + PAGE_SIZE) & PAGE_MASK;

	}



	return i;



finish_or_fault:

	return i ? : -EFAULT;

}

#endif /* !CONFIG_MMU */



#if defined(CONFIG_FS_DAX) || defined (CONFIG_CMA)

static bool check_dax_vmas(struct vm_area_struct **vmas, long nr_pages)
@@ -1484,7 +1530,7 @@ static long check_and_migrate_cma_pages(struct task_struct *tsk, 
{

	return nr_pages;

}

#endif

#endif /* CONFIG_CMA */



/*

 * __gup_longterm_locked() is a wrapper for __get_user_pages_locked which
@@ -2160,6 +2206,12 @@ static void gup_pgd_range(unsigned long addr, unsigned long end, 
			return;

	} while (pgdp++, addr = next, addr != end);

}

#else

static inline void gup_pgd_range(unsigned long addr, unsigned long end,

		unsigned int flags, struct page **pages, int *nr)

{

}

#endif /* CONFIG_HAVE_FAST_GUP */



#ifndef gup_fast_permitted

/*
@@ -2177,6 +2229,9 @@ static bool gup_fast_permitted(unsigned long start, unsigned long end) 
 * the regular GUP.

 * Note a difference with get_user_pages_fast: this always returns the

 * number of pages pinned, 0 if no pages were pinned.

 *

 * If the architecture does not support this function, simply return with no

 * pages pinned.

 */

int __get_user_pages_fast(unsigned long start, int nr_pages, int write,

			  struct page **pages)
@@ -2206,7 +2261,8 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write, 
	 * block IPIs that come from THPs splitting.

	 */



	if (gup_fast_permitted(start, end)) {

	if (IS_ENABLED(CONFIG_HAVE_FAST_GUP) &&

	    gup_fast_permitted(start, end)) {

		local_irq_save(flags);

		gup_pgd_range(start, end, write ? FOLL_WRITE : 0, pages, &nr);

		local_irq_restore(flags);
@@ -2214,6 +2270,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write, 


	return nr;

}

EXPORT_SYMBOL_GPL(__get_user_pages_fast);



static int __gup_longterm_unlocked(unsigned long start, int nr_pages,

				   unsigned int gup_flags, struct page **pages)
@@ -2270,7 +2327,8 @@ int get_user_pages_fast(unsigned long start, int nr_pages, 
	if (unlikely(!access_ok((void __user *)start, len)))

		return -EFAULT;



	if (gup_fast_permitted(start, end)) {

	if (IS_ENABLED(CONFIG_HAVE_FAST_GUP) &&

	    gup_fast_permitted(start, end)) {

		local_irq_disable();

		gup_pgd_range(addr, end, gup_flags, pages, &nr);

		local_irq_enable();
@@ -2296,5 +2354,4 @@ int get_user_pages_fast(unsigned long start, int nr_pages, 


	return ret;

}



#endif /* CONFIG_HAVE_GENERIC_GUP */
 
EXPORT_SYMBOL_GPL(get_user_pages_fast);

mm/nommu.c

+0 −88
Original line number Diff line number Diff line
@@ -111,94 +111,6 @@ unsigned int kobjsize(const void *objp) 
	return PAGE_SIZE << compound_order(page);

}



static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,

		      unsigned long start, unsigned long nr_pages,

		      unsigned int foll_flags, struct page **pages,

		      struct vm_area_struct **vmas, int *nonblocking)

{

	struct vm_area_struct *vma;

	unsigned long vm_flags;

	int i;



	/* calculate required read or write permissions.

	 * If FOLL_FORCE is set, we only require the "MAY" flags.

	 */

	vm_flags  = (foll_flags & FOLL_WRITE) ?

			(VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);

	vm_flags &= (foll_flags & FOLL_FORCE) ?

			(VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);



	for (i = 0; i < nr_pages; i++) {

		vma = find_vma(mm, start);

		if (!vma)

			goto finish_or_fault;



		/* protect what we can, including chardevs */

		if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) ||

		    !(vm_flags & vma->vm_flags))

			goto finish_or_fault;



		if (pages) {

			pages[i] = virt_to_page(start);

			if (pages[i])

				get_page(pages[i]);

		}

		if (vmas)

			vmas[i] = vma;

		start = (start + PAGE_SIZE) & PAGE_MASK;

	}



	return i;



finish_or_fault:

	return i ? : -EFAULT;

}



/*

 * get a list of pages in an address range belonging to the specified process

 * and indicate the VMA that covers each page

 * - this is potentially dodgy as we may end incrementing the page count of a

 *   slab page or a secondary page from a compound page

 * - don't permit access to VMAs that don't support it, such as I/O mappings

 */

long get_user_pages(unsigned long start, unsigned long nr_pages,

		    unsigned int gup_flags, struct page **pages,

		    struct vm_area_struct **vmas)

{

	return __get_user_pages(current, current->mm, start, nr_pages,

				gup_flags, pages, vmas, NULL);

}

EXPORT_SYMBOL(get_user_pages);



long get_user_pages_locked(unsigned long start, unsigned long nr_pages,

			    unsigned int gup_flags, struct page **pages,

			    int *locked)

{

	return get_user_pages(start, nr_pages, gup_flags, pages, NULL);

}

EXPORT_SYMBOL(get_user_pages_locked);



static long __get_user_pages_unlocked(struct task_struct *tsk,

			struct mm_struct *mm, unsigned long start,

			unsigned long nr_pages, struct page **pages,

			unsigned int gup_flags)

{

	long ret;

	down_read(&mm->mmap_sem);

	ret = __get_user_pages(tsk, mm, start, nr_pages, gup_flags, pages,

				NULL, NULL);

	up_read(&mm->mmap_sem);

	return ret;

}



long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,

			     struct page **pages, unsigned int gup_flags)

{

	return __get_user_pages_unlocked(current, current->mm, start, nr_pages,

					 pages, gup_flags);

}

EXPORT_SYMBOL(get_user_pages_unlocked);



/**

 * follow_pfn - look up PFN at a user virtual address

 * @vma: memory mapping

mm/util.c

+0 −47
Original line number Diff line number Diff line
@@ -300,53 +300,6 @@ void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack) 
}

#endif



/*

 * Like get_user_pages_fast() except its IRQ-safe in that it won't fall

 * back to the regular GUP.

 * Note a difference with get_user_pages_fast: this always returns the

 * number of pages pinned, 0 if no pages were pinned.

 * If the architecture does not support this function, simply return with no

 * pages pinned.

 */

int __weak __get_user_pages_fast(unsigned long start,

				 int nr_pages, int write, struct page **pages)

{

	return 0;

}

EXPORT_SYMBOL_GPL(__get_user_pages_fast);



/**

 * get_user_pages_fast() - pin user pages in memory

 * @start:	starting user address

 * @nr_pages:	number of pages from start to pin

 * @gup_flags:	flags modifying pin behaviour

 * @pages:	array that receives pointers to the pages pinned.

 *		Should be at least nr_pages long.

 *

 * get_user_pages_fast provides equivalent functionality to get_user_pages,

 * operating on current and current->mm, with force=0 and vma=NULL. However

 * unlike get_user_pages, it must be called without mmap_sem held.

 *

 * get_user_pages_fast may take mmap_sem and page table locks, so no

 * assumptions can be made about lack of locking. get_user_pages_fast is to be

 * implemented in a way that is advantageous (vs get_user_pages()) when the

 * user memory area is already faulted in and present in ptes. However if the

 * pages have to be faulted in, it may turn out to be slightly slower so

 * callers need to carefully consider what to use. On many architectures,

 * get_user_pages_fast simply falls back to get_user_pages.

 *

 * Return: number of pages pinned. This may be fewer than the number

 * requested. If nr_pages is 0 or negative, returns 0. If no pages

 * were pinned, returns -errno.

 */

int __weak get_user_pages_fast(unsigned long start,

				int nr_pages, unsigned int gup_flags,

				struct page **pages)

{

	return get_user_pages_unlocked(start, nr_pages, pages, gup_flags);

}

EXPORT_SYMBOL_GPL(get_user_pages_fast);



unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr,

	unsigned long len, unsigned long prot,

	unsigned long flag, unsigned long pgoff)

CRA Git | Maintained and supported by SUSTech CRA and CCSE