Commit 63551ae0 authored by David Gibson's avatar David Gibson Committed by Linus Torvalds
Browse files

[PATCH] Hugepage consolidation 



A lot of the code in arch/*/mm/hugetlbpage.c is quite similar.  This patch
attempts to consolidate a lot of the code across the arch's, putting the
combined version in mm/hugetlb.c.  There are a couple of uglyish hacks in
order to covert all the hugepage archs, but the result is a very large
reduction in the total amount of code.  It also means things like hugepage
lazy allocation could be implemented in one place, instead of six.

Tested, at least a little, on ppc64, i386 and x86_64.

Notes:
	- this patch changes the meaning of set_huge_pte() to be more
	  analagous to set_pte()
	- does SH4 need s special huge_ptep_get_and_clear()??

Acked-by: default avatarWilliam Lee Irwin <wli@holomorphy.com>
Signed-off-by: default avatarAndrew Morton <akpm@osdl.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent 1e7e5a90

arch/i386/mm/hugetlbpage.c

+8 −162
Original line number Diff line number Diff line
@@ -18,7 +18,7 @@ 
#include <asm/tlb.h>

#include <asm/tlbflush.h>



static pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)

pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)

{

	pgd_t *pgd;

	pud_t *pud;
@@ -30,7 +30,7 @@ static pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr) 
	return (pte_t *) pmd;

}



static pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)

pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)

{

	pgd_t *pgd;

	pud_t *pud;
@@ -42,21 +42,6 @@ static pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) 
	return (pte_t *) pmd;

}



static void set_huge_pte(struct mm_struct *mm, struct vm_area_struct *vma, struct page *page, pte_t * page_table, int write_access)

{

	pte_t entry;



	add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE);

	if (write_access) {

		entry =

		    pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));

	} else

		entry = pte_wrprotect(mk_pte(page, vma->vm_page_prot));

	entry = pte_mkyoung(entry);

	mk_pte_huge(entry);

	set_pte(page_table, entry);

}



/*

 * This function checks for proper alignment of input addr and len parameters.

 */
@@ -69,77 +54,6 @@ int is_aligned_hugepage_range(unsigned long addr, unsigned long len) 
	return 0;

}



int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,

			struct vm_area_struct *vma)

{

	pte_t *src_pte, *dst_pte, entry;

	struct page *ptepage;

	unsigned long addr = vma->vm_start;

	unsigned long end = vma->vm_end;



	while (addr < end) {

		dst_pte = huge_pte_alloc(dst, addr);

		if (!dst_pte)

			goto nomem;

		src_pte = huge_pte_offset(src, addr);

		entry = *src_pte;

		ptepage = pte_page(entry);

		get_page(ptepage);

		set_pte(dst_pte, entry);

		add_mm_counter(dst, rss, HPAGE_SIZE / PAGE_SIZE);

		addr += HPAGE_SIZE;

	}

	return 0;



nomem:

	return -ENOMEM;

}



int

follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,

		    struct page **pages, struct vm_area_struct **vmas,

		    unsigned long *position, int *length, int i)

{

	unsigned long vpfn, vaddr = *position;

	int remainder = *length;



	WARN_ON(!is_vm_hugetlb_page(vma));



	vpfn = vaddr/PAGE_SIZE;

	while (vaddr < vma->vm_end && remainder) {



		if (pages) {

			pte_t *pte;

			struct page *page;



			pte = huge_pte_offset(mm, vaddr);



			/* hugetlb should be locked, and hence, prefaulted */

			WARN_ON(!pte || pte_none(*pte));



			page = &pte_page(*pte)[vpfn % (HPAGE_SIZE/PAGE_SIZE)];



			WARN_ON(!PageCompound(page));



			get_page(page);

			pages[i] = page;

		}



		if (vmas)

			vmas[i] = vma;



		vaddr += PAGE_SIZE;

		++vpfn;

		--remainder;

		++i;

	}



	*length = remainder;

	*position = vaddr;



	return i;

}



#if 0	/* This is just for testing */

struct page *

follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
@@ -204,83 +118,15 @@ follow_huge_pmd(struct mm_struct *mm, unsigned long address, 
}

#endif



void unmap_hugepage_range(struct vm_area_struct *vma,

		unsigned long start, unsigned long end)

{

	struct mm_struct *mm = vma->vm_mm;

	unsigned long address;

	pte_t pte, *ptep;

	struct page *page;



	BUG_ON(start & (HPAGE_SIZE - 1));

	BUG_ON(end & (HPAGE_SIZE - 1));



	for (address = start; address < end; address += HPAGE_SIZE) {

		ptep = huge_pte_offset(mm, address);

		if (!ptep)

			continue;

		pte = ptep_get_and_clear(mm, address, ptep);

		if (pte_none(pte))

			continue;

		page = pte_page(pte);

		put_page(page);

	}

	add_mm_counter(mm ,rss, -((end - start) >> PAGE_SHIFT));

	flush_tlb_range(vma, start, end);

}



int hugetlb_prefault(struct address_space *mapping, struct vm_area_struct *vma)

void hugetlb_clean_stale_pgtable(pte_t *pte)

{

	struct mm_struct *mm = current->mm;

	unsigned long addr;

	int ret = 0;



	BUG_ON(vma->vm_start & ~HPAGE_MASK);

	BUG_ON(vma->vm_end & ~HPAGE_MASK);



	spin_lock(&mm->page_table_lock);

	for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) {

		unsigned long idx;

		pte_t *pte = huge_pte_alloc(mm, addr);

	pmd_t *pmd = (pmd_t *) pte;

	struct page *page;



		if (!pte) {

			ret = -ENOMEM;

			goto out;

		}



		if (!pte_none(*pte))

			continue;



		idx = ((addr - vma->vm_start) >> HPAGE_SHIFT)

			+ (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT));

		page = find_get_page(mapping, idx);

		if (!page) {

			/* charge the fs quota first */

			if (hugetlb_get_quota(mapping)) {

				ret = -ENOMEM;

				goto out;

			}

			page = alloc_huge_page();

			if (!page) {

				hugetlb_put_quota(mapping);

				ret = -ENOMEM;

				goto out;

			}

			ret = add_to_page_cache(page, mapping, idx, GFP_ATOMIC);

			if (! ret) {

				unlock_page(page);

			} else {

				hugetlb_put_quota(mapping);

				free_huge_page(page);

				goto out;

			}

		}

		set_huge_pte(mm, vma, page, pte, vma->vm_flags & VM_WRITE);

	}

out:

	spin_unlock(&mm->page_table_lock);

	return ret;

	page = pmd_page(*pmd);

	pmd_clear(pmd);

	dec_page_state(nr_page_table_pages);

	page_cache_release(page);

}



/* x86_64 also uses this file */

arch/ia64/mm/hugetlbpage.c

+2 −156
Original line number Diff line number Diff line
@@ -24,7 +24,7 @@ 


unsigned int hpage_shift=HPAGE_SHIFT_DEFAULT;



static pte_t *

pte_t *

huge_pte_alloc (struct mm_struct *mm, unsigned long addr)

{

	unsigned long taddr = htlbpage_to_page(addr);
@@ -43,7 +43,7 @@ huge_pte_alloc (struct mm_struct *mm, unsigned long addr) 
	return pte;

}



static pte_t *

pte_t *

huge_pte_offset (struct mm_struct *mm, unsigned long addr)

{

	unsigned long taddr = htlbpage_to_page(addr);
@@ -67,23 +67,6 @@ huge_pte_offset (struct mm_struct *mm, unsigned long addr) 


#define mk_pte_huge(entry) { pte_val(entry) |= _PAGE_P; }



static void

set_huge_pte (struct mm_struct *mm, struct vm_area_struct *vma,

	      struct page *page, pte_t * page_table, int write_access)

{

	pte_t entry;



	add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE);

	if (write_access) {

		entry =

		    pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));

	} else

		entry = pte_wrprotect(mk_pte(page, vma->vm_page_prot));

	entry = pte_mkyoung(entry);

	mk_pte_huge(entry);

	set_pte(page_table, entry);

	return;

}

/*

 * This function checks for proper alignment of input addr and len parameters.

 */
@@ -99,68 +82,6 @@ int is_aligned_hugepage_range(unsigned long addr, unsigned long len) 
	return 0;

}



int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,

			struct vm_area_struct *vma)

{

	pte_t *src_pte, *dst_pte, entry;

	struct page *ptepage;

	unsigned long addr = vma->vm_start;

	unsigned long end = vma->vm_end;



	while (addr < end) {

		dst_pte = huge_pte_alloc(dst, addr);

		if (!dst_pte)

			goto nomem;

		src_pte = huge_pte_offset(src, addr);

		entry = *src_pte;

		ptepage = pte_page(entry);

		get_page(ptepage);

		set_pte(dst_pte, entry);

		add_mm_counter(dst, rss, HPAGE_SIZE / PAGE_SIZE);

		addr += HPAGE_SIZE;

	}

	return 0;

nomem:

	return -ENOMEM;

}



int

follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,

		    struct page **pages, struct vm_area_struct **vmas,

		    unsigned long *st, int *length, int i)

{

	pte_t *ptep, pte;

	unsigned long start = *st;

	unsigned long pstart;

	int len = *length;

	struct page *page;



	do {

		pstart = start & HPAGE_MASK;

		ptep = huge_pte_offset(mm, start);

		pte = *ptep;



back1:

		page = pte_page(pte);

		if (pages) {

			page += ((start & ~HPAGE_MASK) >> PAGE_SHIFT);

			get_page(page);

			pages[i] = page;

		}

		if (vmas)

			vmas[i] = vma;

		i++;

		len--;

		start += PAGE_SIZE;

		if (((start & HPAGE_MASK) == pstart) && len &&

				(start < vma->vm_end))

			goto back1;

	} while (len && start < vma->vm_end);

	*length = len;

	*st = start;

	return i;

}



struct page *follow_huge_addr(struct mm_struct *mm, unsigned long addr, int write)

{

	struct page *page;
@@ -212,81 +133,6 @@ void hugetlb_free_pgd_range(struct mmu_gather **tlb, 
	free_pgd_range(tlb, addr, end, floor, ceiling);

}



void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)

{

	struct mm_struct *mm = vma->vm_mm;

	unsigned long address;

	pte_t *pte;

	struct page *page;



	BUG_ON(start & (HPAGE_SIZE - 1));

	BUG_ON(end & (HPAGE_SIZE - 1));



	for (address = start; address < end; address += HPAGE_SIZE) {

		pte = huge_pte_offset(mm, address);

		if (pte_none(*pte))

			continue;

		page = pte_page(*pte);

		put_page(page);

		pte_clear(mm, address, pte);

	}

	add_mm_counter(mm, rss, - ((end - start) >> PAGE_SHIFT));

	flush_tlb_range(vma, start, end);

}



int hugetlb_prefault(struct address_space *mapping, struct vm_area_struct *vma)

{

	struct mm_struct *mm = current->mm;

	unsigned long addr;

	int ret = 0;



	BUG_ON(vma->vm_start & ~HPAGE_MASK);

	BUG_ON(vma->vm_end & ~HPAGE_MASK);



	spin_lock(&mm->page_table_lock);

	for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) {

		unsigned long idx;

		pte_t *pte = huge_pte_alloc(mm, addr);

		struct page *page;



		if (!pte) {

			ret = -ENOMEM;

			goto out;

		}

		if (!pte_none(*pte))

			continue;



		idx = ((addr - vma->vm_start) >> HPAGE_SHIFT)

			+ (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT));

		page = find_get_page(mapping, idx);

		if (!page) {

			/* charge the fs quota first */

			if (hugetlb_get_quota(mapping)) {

				ret = -ENOMEM;

				goto out;

			}

			page = alloc_huge_page();

			if (!page) {

				hugetlb_put_quota(mapping);

				ret = -ENOMEM;

				goto out;

			}

			ret = add_to_page_cache(page, mapping, idx, GFP_ATOMIC);

			if (! ret) {

				unlock_page(page);

			} else {

				hugetlb_put_quota(mapping);

				page_cache_release(page);

				goto out;

			}

		}

		set_huge_pte(mm, vma, page, pte, vma->vm_flags & VM_WRITE);

	}

out:

	spin_unlock(&mm->page_table_lock);

	return ret;

}



unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,

		unsigned long pgoff, unsigned long flags)

{

arch/ppc64/mm/hugetlbpage.c

+2 −178
Original line number Diff line number Diff line
@@ -121,7 +121,7 @@ static pte_t *hugepte_alloc(struct mm_struct *mm, pud_t *dir, unsigned long addr 
	return hugepte_offset(dir, addr);

}



static pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)

pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)

{

	pud_t *pud;
@@ -134,7 +134,7 @@ static pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) 
	return hugepte_offset(pud, addr);

}



static pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)

pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)

{

	pud_t *pud;
@@ -147,25 +147,6 @@ static pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr) 
	return hugepte_alloc(mm, pud, addr);

}



static void set_huge_pte(struct mm_struct *mm, struct vm_area_struct *vma,

			 unsigned long addr, struct page *page,

			 pte_t *ptep, int write_access)

{

	pte_t entry;



	add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE);

	if (write_access) {

		entry =

		    pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));

	} else {

		entry = pte_wrprotect(mk_pte(page, vma->vm_page_prot));

	}

	entry = pte_mkyoung(entry);

	entry = pte_mkhuge(entry);



	set_pte_at(mm, addr, ptep, entry);

}



/*

 * This function checks for proper alignment of input addr and len parameters.

 */
@@ -259,80 +240,6 @@ int prepare_hugepage_range(unsigned long addr, unsigned long len) 
	return -EINVAL;

}



int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,

			struct vm_area_struct *vma)

{

	pte_t *src_pte, *dst_pte, entry;

	struct page *ptepage;

	unsigned long addr = vma->vm_start;

	unsigned long end = vma->vm_end;

	int err = -ENOMEM;



	while (addr < end) {

		dst_pte = huge_pte_alloc(dst, addr);

		if (!dst_pte)

			goto out;



		src_pte = huge_pte_offset(src, addr);

		entry = *src_pte;

		

		ptepage = pte_page(entry);

		get_page(ptepage);

		add_mm_counter(dst, rss, HPAGE_SIZE / PAGE_SIZE);

		set_pte_at(dst, addr, dst_pte, entry);



		addr += HPAGE_SIZE;

	}



	err = 0;

 out:

	return err;

}



int

follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,

		    struct page **pages, struct vm_area_struct **vmas,

		    unsigned long *position, int *length, int i)

{

	unsigned long vpfn, vaddr = *position;

	int remainder = *length;



	WARN_ON(!is_vm_hugetlb_page(vma));



	vpfn = vaddr/PAGE_SIZE;

	while (vaddr < vma->vm_end && remainder) {

		if (pages) {

			pte_t *pte;

			struct page *page;



			pte = huge_pte_offset(mm, vaddr);



			/* hugetlb should be locked, and hence, prefaulted */

			WARN_ON(!pte || pte_none(*pte));



			page = &pte_page(*pte)[vpfn % (HPAGE_SIZE/PAGE_SIZE)];



			WARN_ON(!PageCompound(page));



			get_page(page);

			pages[i] = page;

		}



		if (vmas)

			vmas[i] = vma;



		vaddr += PAGE_SIZE;

		++vpfn;

		--remainder;

		++i;

	}



	*length = remainder;

	*position = vaddr;



	return i;

}



struct page *

follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)

{
@@ -363,89 +270,6 @@ follow_huge_pmd(struct mm_struct *mm, unsigned long address, 
	return NULL;

}



void unmap_hugepage_range(struct vm_area_struct *vma,

			  unsigned long start, unsigned long end)

{

	struct mm_struct *mm = vma->vm_mm;

	unsigned long addr;

	pte_t *ptep;

	struct page *page;



	WARN_ON(!is_vm_hugetlb_page(vma));

	BUG_ON((start % HPAGE_SIZE) != 0);

	BUG_ON((end % HPAGE_SIZE) != 0);



	for (addr = start; addr < end; addr += HPAGE_SIZE) {

		pte_t pte;



		ptep = huge_pte_offset(mm, addr);

		if (!ptep || pte_none(*ptep))

			continue;



		pte = *ptep;

		page = pte_page(pte);

		pte_clear(mm, addr, ptep);



		put_page(page);

	}

	add_mm_counter(mm, rss, -((end - start) >> PAGE_SHIFT));

	flush_tlb_pending();

}



int hugetlb_prefault(struct address_space *mapping, struct vm_area_struct *vma)

{

	struct mm_struct *mm = current->mm;

	unsigned long addr;

	int ret = 0;



	WARN_ON(!is_vm_hugetlb_page(vma));

	BUG_ON((vma->vm_start % HPAGE_SIZE) != 0);

	BUG_ON((vma->vm_end % HPAGE_SIZE) != 0);



	spin_lock(&mm->page_table_lock);

	for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) {

		unsigned long idx;

		pte_t *pte = huge_pte_alloc(mm, addr);

		struct page *page;



		if (!pte) {

			ret = -ENOMEM;

			goto out;

		}

		if (! pte_none(*pte))

			continue;



		idx = ((addr - vma->vm_start) >> HPAGE_SHIFT)

			+ (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT));

		page = find_get_page(mapping, idx);

		if (!page) {

			/* charge the fs quota first */

			if (hugetlb_get_quota(mapping)) {

				ret = -ENOMEM;

				goto out;

			}

			page = alloc_huge_page();

			if (!page) {

				hugetlb_put_quota(mapping);

				ret = -ENOMEM;

				goto out;

			}

			ret = add_to_page_cache(page, mapping, idx, GFP_ATOMIC);

			if (! ret) {

				unlock_page(page);

			} else {

				hugetlb_put_quota(mapping);

				free_huge_page(page);

				goto out;

			}

		}

		set_huge_pte(mm, vma, addr, page, pte, vma->vm_flags & VM_WRITE);

	}

out:

	spin_unlock(&mm->page_table_lock);

	return ret;

}



/* Because we have an exclusive hugepage region which lies within the

 * normal user address space, we have to take special measures to make

 * non-huge mmap()s evade the hugepage reserved regions. */

arch/sh/mm/hugetlbpage.c

+24 −172
Original line number Diff line number Diff line
@@ -24,7 +24,7 @@ 
#include <asm/tlbflush.h>

#include <asm/cacheflush.h>



static pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)

pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)

{

	pgd_t *pgd;

	pmd_t *pmd;
@@ -39,7 +39,7 @@ static pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr) 
	return pte;

}



static pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)

pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)

{

	pgd_t *pgd;

	pmd_t *pmd;
@@ -56,28 +56,34 @@ static pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) 


#define mk_pte_huge(entry) do { pte_val(entry) |= _PAGE_SZHUGE; } while (0)



static void set_huge_pte(struct mm_struct *mm, struct vm_area_struct *vma,

			 struct page *page, pte_t * page_table, int write_access)

void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,

		     pte_t *ptep, pte_t entry)

{

	unsigned long i;

	pte_t entry;

	int i;



	for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) {

		set_pte_at(mm, addr, ptep, entry);

		ptep++;

		addr += PAGE_SIZE;

		pte_val(entry) += PAGE_SIZE;

	}

}



	add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE);

pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,

			      pte_t *ptep)

{

	pte_t entry;

	int i;



	if (write_access)

		entry = pte_mkwrite(pte_mkdirty(mk_pte(page,

						       vma->vm_page_prot)));

	else

		entry = pte_wrprotect(mk_pte(page, vma->vm_page_prot));

	entry = pte_mkyoung(entry);

	mk_pte_huge(entry);

	entry = *ptep;



	for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) {

		set_pte(page_table, entry);

		page_table++;



		pte_val(entry) += PAGE_SIZE;

		pte_clear(mm, addr, ptep);

		addr += PAGE_SIZE;

		ptep++;

	}



	return entry;

}



/*
@@ -92,79 +98,6 @@ int is_aligned_hugepage_range(unsigned long addr, unsigned long len) 
	return 0;

}



int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,

			    struct vm_area_struct *vma)

{

	pte_t *src_pte, *dst_pte, entry;

	struct page *ptepage;

	unsigned long addr = vma->vm_start;

	unsigned long end = vma->vm_end;

	int i;



	while (addr < end) {

		dst_pte = huge_pte_alloc(dst, addr);

		if (!dst_pte)

			goto nomem;

		src_pte = huge_pte_offset(src, addr);

		BUG_ON(!src_pte || pte_none(*src_pte));

		entry = *src_pte;

		ptepage = pte_page(entry);

		get_page(ptepage);

		for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) {

			set_pte(dst_pte, entry);

			pte_val(entry) += PAGE_SIZE;

			dst_pte++;

		}

		add_mm_counter(dst, rss, HPAGE_SIZE / PAGE_SIZE);

		addr += HPAGE_SIZE;

	}

	return 0;



nomem:

	return -ENOMEM;

}



int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,

			struct page **pages, struct vm_area_struct **vmas,

			unsigned long *position, int *length, int i)

{

	unsigned long vaddr = *position;

	int remainder = *length;



	WARN_ON(!is_vm_hugetlb_page(vma));



	while (vaddr < vma->vm_end && remainder) {

		if (pages) {

			pte_t *pte;

			struct page *page;



			pte = huge_pte_offset(mm, vaddr);



			/* hugetlb should be locked, and hence, prefaulted */

			BUG_ON(!pte || pte_none(*pte));



			page = pte_page(*pte);



			WARN_ON(!PageCompound(page));



			get_page(page);

			pages[i] = page;

		}



		if (vmas)

			vmas[i] = vma;



		vaddr += PAGE_SIZE;

		--remainder;

		++i;

	}



	*length = remainder;

	*position = vaddr;



	return i;

}



struct page *follow_huge_addr(struct mm_struct *mm,

			      unsigned long address, int write)

{
@@ -181,84 +114,3 @@ struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address, 
{

	return NULL;

}
 


void unmap_hugepage_range(struct vm_area_struct *vma,

			  unsigned long start, unsigned long end)

{

	struct mm_struct *mm = vma->vm_mm;

	unsigned long address;

	pte_t *pte;

	struct page *page;

	int i;



	BUG_ON(start & (HPAGE_SIZE - 1));

	BUG_ON(end & (HPAGE_SIZE - 1));



	for (address = start; address < end; address += HPAGE_SIZE) {

		pte = huge_pte_offset(mm, address);

		BUG_ON(!pte);

		if (pte_none(*pte))

			continue;

		page = pte_page(*pte);

		put_page(page);

		for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) {

			pte_clear(mm, address+(i*PAGE_SIZE), pte);

			pte++;

		}

	}

	add_mm_counter(mm, rss, -((end - start) >> PAGE_SHIFT));

	flush_tlb_range(vma, start, end);

}



int hugetlb_prefault(struct address_space *mapping, struct vm_area_struct *vma)

{

	struct mm_struct *mm = current->mm;

	unsigned long addr;

	int ret = 0;



	BUG_ON(vma->vm_start & ~HPAGE_MASK);

	BUG_ON(vma->vm_end & ~HPAGE_MASK);



	spin_lock(&mm->page_table_lock);

	for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) {

		unsigned long idx;

		pte_t *pte = huge_pte_alloc(mm, addr);

		struct page *page;



		if (!pte) {

			ret = -ENOMEM;

			goto out;

		}

		if (!pte_none(*pte))

			continue;



		idx = ((addr - vma->vm_start) >> HPAGE_SHIFT)

			+ (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT));

		page = find_get_page(mapping, idx);

		if (!page) {

			/* charge the fs quota first */

			if (hugetlb_get_quota(mapping)) {

				ret = -ENOMEM;

				goto out;

			}

			page = alloc_huge_page();

			if (!page) {

				hugetlb_put_quota(mapping);

				ret = -ENOMEM;

				goto out;

			}

			ret = add_to_page_cache(page, mapping, idx, GFP_ATOMIC);

			if (! ret) {

				unlock_page(page);

			} else {

				hugetlb_put_quota(mapping);

				free_huge_page(page);

				goto out;

			}

		}

		set_huge_pte(mm, vma, page, pte, vma->vm_flags & VM_WRITE);

	}

out:

	spin_unlock(&mm->page_table_lock);

	return ret;

}

arch/sh64/mm/hugetlbpage.c

+16 −2
Original line number Diff line number Diff line
@@ -24,7 +24,7 @@ 
#include <asm/tlbflush.h>

#include <asm/cacheflush.h>



static pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)

pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)

{

	pgd_t *pgd;

	pmd_t *pmd;
@@ -39,7 +39,7 @@ static pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr) 
	return pte;

}



static pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)

pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)

{

	pgd_t *pgd;

	pmd_t *pmd;
@@ -80,6 +80,20 @@ static void set_huge_pte(struct mm_struct *mm, struct vm_area_struct *vma, 
	}

}



pte_t huge_ptep_get_and_clear(pte_t *ptep)

{

	pte_t entry;



	entry = *ptep;



	for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) {

		pte_clear(pte);

		pte++;

	}



	return entry;

}



/*

 * This function checks for proper alignment of input addr and len parameters.

 */

CRA Git | Maintained and supported by SUSTech CRA and CCSE