Commit cbd34da7 authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Linus Torvalds
Browse files

mm: move the powerpc hugepd code to mm/gup.c 

While only powerpc supports the hugepd case, the code is pretty generic
and I'd like to keep all GUP internals in one place.

Link: http://lkml.kernel.org/r/20190625143715.1689-15-hch@lst.de


Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Cc: Andrey Konovalov <andreyknvl@google.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: David Miller <davem@davemloft.net>
Cc: James Hogan <jhogan@kernel.org>
Cc: Jason Gunthorpe <jgg@mellanox.com>
Cc: Khalid Aziz <khalid.aziz@oracle.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Paul Burton <paul.burton@mips.com>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: Rich Felker <dalias@libc.org>
Cc: Yoshinori Sato <ysato@users.sourceforge.jp>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 817be129

arch/powerpc/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -125,6 +125,7 @@ config PPC 
	select ARCH_HAS_FORTIFY_SOURCE

	select ARCH_HAS_GCOV_PROFILE_ALL

	select ARCH_HAS_KCOV

	select ARCH_HAS_HUGEPD			if HUGETLB_PAGE

	select ARCH_HAS_MMIOWB			if PPC64

	select ARCH_HAS_PHYS_TO_DMA

	select ARCH_HAS_PMEM_API                if PPC64

arch/powerpc/mm/hugetlbpage.c

+0 −72
Original line number Diff line number Diff line
@@ -511,13 +511,6 @@ retry: 
	return page;

}



static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,

				      unsigned long sz)

{

	unsigned long __boundary = (addr + sz) & ~(sz-1);

	return (__boundary - 1 < end - 1) ? __boundary : end;

}



#ifdef CONFIG_PPC_MM_SLICES

unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,

					unsigned long len, unsigned long pgoff,
@@ -665,68 +658,3 @@ void flush_dcache_icache_hugepage(struct page *page) 
		}

	}

}
 


static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,

		       unsigned long end, int write, struct page **pages, int *nr)

{

	unsigned long pte_end;

	struct page *head, *page;

	pte_t pte;

	int refs;



	pte_end = (addr + sz) & ~(sz-1);

	if (pte_end < end)

		end = pte_end;



	pte = READ_ONCE(*ptep);



	if (!pte_access_permitted(pte, write))

		return 0;



	/* hugepages are never "special" */

	VM_BUG_ON(!pfn_valid(pte_pfn(pte)));



	refs = 0;

	head = pte_page(pte);



	page = head + ((addr & (sz-1)) >> PAGE_SHIFT);

	do {

		VM_BUG_ON(compound_head(page) != head);

		pages[*nr] = page;

		(*nr)++;

		page++;

		refs++;

	} while (addr += PAGE_SIZE, addr != end);



	if (!page_cache_add_speculative(head, refs)) {

		*nr -= refs;

		return 0;

	}



	if (unlikely(pte_val(pte) != pte_val(*ptep))) {

		/* Could be optimized better */

		*nr -= refs;

		while (refs--)

			put_page(head);

		return 0;

	}



	return 1;

}



int gup_huge_pd(hugepd_t hugepd, unsigned long addr, unsigned int pdshift,

		unsigned long end, int write, struct page **pages, int *nr)

{

	pte_t *ptep;

	unsigned long sz = 1UL << hugepd_shift(hugepd);

	unsigned long next;



	ptep = hugepte_offset(hugepd, addr, pdshift);

	do {

		next = hugepte_addr_end(addr, end, sz);

		if (!gup_hugepte(ptep, sz, addr, end, write, pages, nr))

			return 0;

	} while (ptep++, addr = next, addr != end);



	return 1;

}

include/linux/hugetlb.h

+0 −18
Original line number Diff line number Diff line
@@ -16,29 +16,11 @@ struct user_struct; 
struct mmu_gather;



#ifndef is_hugepd

/*

 * Some architectures requires a hugepage directory format that is

 * required to support multiple hugepage sizes. For example

 * a4fe3ce76 "powerpc/mm: Allow more flexible layouts for hugepage pagetables"

 * introduced the same on powerpc. This allows for a more flexible hugepage

 * pagetable layout.

 */

typedef struct { unsigned long pd; } hugepd_t;

#define is_hugepd(hugepd) (0)

#define __hugepd(x) ((hugepd_t) { (x) })

static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr,

			      unsigned pdshift, unsigned long end,

			      int write, struct page **pages, int *nr)

{

	return 0;

}

#else

extern int gup_huge_pd(hugepd_t hugepd, unsigned long addr,

		       unsigned pdshift, unsigned long end,

		       int write, struct page **pages, int *nr);

#endif





#ifdef CONFIG_HUGETLB_PAGE



#include <linux/mempolicy.h>

mm/Kconfig

+10 −0
Original line number Diff line number Diff line
@@ -769,4 +769,14 @@ config GUP_GET_PTE_LOW_HIGH 
config ARCH_HAS_PTE_SPECIAL

	bool



#

# Some architectures require a special hugepage directory format that is

# required to support multiple hugepage sizes. For example a4fe3ce76

# "powerpc/mm: Allow more flexible layouts for hugepage pagetables"

# introduced it on powerpc.  This allows for a more flexible hugepage

# pagetable layouts.

#

config ARCH_HAS_HUGEPD

	bool



endmenu

mm/gup.c

+82 −0
Original line number Diff line number Diff line
@@ -1966,6 +1966,88 @@ static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr, 
}

#endif



#ifdef CONFIG_ARCH_HAS_HUGEPD

static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,

				      unsigned long sz)

{

	unsigned long __boundary = (addr + sz) & ~(sz-1);

	return (__boundary - 1 < end - 1) ? __boundary : end;

}



static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,

		       unsigned long end, int write, struct page **pages, int *nr)

{

	unsigned long pte_end;

	struct page *head, *page;

	pte_t pte;

	int refs;



	pte_end = (addr + sz) & ~(sz-1);

	if (pte_end < end)

		end = pte_end;



	pte = READ_ONCE(*ptep);



	if (!pte_access_permitted(pte, write))

		return 0;



	/* hugepages are never "special" */

	VM_BUG_ON(!pfn_valid(pte_pfn(pte)));



	refs = 0;

	head = pte_page(pte);



	page = head + ((addr & (sz-1)) >> PAGE_SHIFT);

	do {

		VM_BUG_ON(compound_head(page) != head);

		pages[*nr] = page;

		(*nr)++;

		page++;

		refs++;

	} while (addr += PAGE_SIZE, addr != end);



	if (!page_cache_add_speculative(head, refs)) {

		*nr -= refs;

		return 0;

	}



	if (unlikely(pte_val(pte) != pte_val(*ptep))) {

		/* Could be optimized better */

		*nr -= refs;

		while (refs--)

			put_page(head);

		return 0;

	}



	return 1;

}



static int gup_huge_pd(hugepd_t hugepd, unsigned long addr,

		unsigned int pdshift, unsigned long end, int write,

		struct page **pages, int *nr)

{

	pte_t *ptep;

	unsigned long sz = 1UL << hugepd_shift(hugepd);

	unsigned long next;



	ptep = hugepte_offset(hugepd, addr, pdshift);

	do {

		next = hugepte_addr_end(addr, end, sz);

		if (!gup_hugepte(ptep, sz, addr, end, write, pages, nr))

			return 0;

	} while (ptep++, addr = next, addr != end);



	return 1;

}

#else

static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr,

		unsigned pdshift, unsigned long end, int write,

		struct page **pages, int *nr)

{

	return 0;

}

#endif /* CONFIG_ARCH_HAS_HUGEPD */



static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,

		unsigned long end, unsigned int flags, struct page **pages, int *nr)

{

CRA Git | Maintained and supported by SUSTech CRA and CCSE