Commit dcd1912d authored by Nitin Gupta's avatar Nitin Gupta Committed by David S. Miller
Browse files

sparc64: Add 64K page size support 



This patch depends on:
[v6] sparc64: Multi-page size support

- Testing

Tested on Sonoma by running stream benchmark instance which allocated
48G worth of 64K pages.

boot params: default_hugepagesz=64K hugepagesz=64K hugepages=1310720

Signed-off-by: default avatarNitin Gupta <nitin.m.gupta@oracle.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent c7d9f77d

arch/sparc/include/asm/page_64.h

+2 −1
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ 
#define HPAGE_SHIFT		23

#define REAL_HPAGE_SHIFT	22

#define HPAGE_256MB_SHIFT	28

#define HPAGE_64K_SHIFT		16

#define REAL_HPAGE_SIZE		(_AC(1,UL) << REAL_HPAGE_SHIFT)



#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
@@ -26,7 +27,7 @@ 
#define HUGETLB_PAGE_ORDER	(HPAGE_SHIFT - PAGE_SHIFT)

#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA

#define REAL_HPAGE_PER_HPAGE	(_AC(1,UL) << (HPAGE_SHIFT - REAL_HPAGE_SHIFT))

#define HUGE_MAX_HSTATE		2

#define HUGE_MAX_HSTATE		3

#endif



#ifndef __ASSEMBLY__

arch/sparc/mm/hugetlbpage.c

+43 −11
Original line number Diff line number Diff line
@@ -149,6 +149,9 @@ static pte_t sun4v_hugepage_shift_to_tte(pte_t entry, unsigned int shift) 
	case HPAGE_SHIFT:

		pte_val(entry) |= _PAGE_PMD_HUGE;

		break;

	case HPAGE_64K_SHIFT:

		hugepage_size = _PAGE_SZ64K_4V;

		break;

	default:

		WARN_ONCE(1, "unsupported hugepage shift=%u\n", shift);

	}
@@ -185,6 +188,9 @@ static unsigned int sun4v_huge_tte_to_shift(pte_t entry) 
	case _PAGE_SZ4MB_4V:

		shift = REAL_HPAGE_SHIFT;

		break;

	case _PAGE_SZ64K_4V:

		shift = HPAGE_64K_SHIFT;

		break;

	default:

		shift = PAGE_SHIFT;

		break;
@@ -204,6 +210,9 @@ static unsigned int sun4u_huge_tte_to_shift(pte_t entry) 
	case _PAGE_SZ4MB_4U:

		shift = REAL_HPAGE_SHIFT;

		break;

	case _PAGE_SZ64K_4U:

		shift = HPAGE_64K_SHIFT;

		break;

	default:

		shift = PAGE_SHIFT;

		break;
@@ -241,12 +250,21 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, 
{

	pgd_t *pgd;

	pud_t *pud;

	pmd_t *pmd;

	pte_t *pte = NULL;



	pgd = pgd_offset(mm, addr);

	pud = pud_alloc(mm, pgd, addr);

	if (pud)

		pte = (pte_t *)pmd_alloc(mm, pud, addr);

	if (pud) {

		pmd = pmd_alloc(mm, pud, addr);

		if (!pmd)

			return NULL;



		if (sz == PMD_SHIFT)

			pte = (pte_t *)pmd;

		else

			pte = pte_alloc_map(mm, pmd, addr);

	}



	return pte;

}
@@ -255,42 +273,52 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) 
{

	pgd_t *pgd;

	pud_t *pud;

	pmd_t *pmd;

	pte_t *pte = NULL;



	pgd = pgd_offset(mm, addr);

	if (!pgd_none(*pgd)) {

		pud = pud_offset(pgd, addr);

		if (!pud_none(*pud))

			pte = (pte_t *)pmd_offset(pud, addr);

		if (!pud_none(*pud)) {

			pmd = pmd_offset(pud, addr);

			if (!pmd_none(*pmd)) {

				if (is_hugetlb_pmd(*pmd))

					pte = (pte_t *)pmd;

				else

					pte = pte_offset_map(pmd, addr);

			}

		}

	}



	return pte;

}



void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,

		     pte_t *ptep, pte_t entry)

{

	unsigned int i, nptes, hugepage_shift;

	unsigned int i, nptes, orig_shift, shift;

	unsigned long size;

	pte_t orig;



	size = huge_tte_to_size(entry);

	nptes = size >> PMD_SHIFT;

	shift = size >= HPAGE_SIZE ? PMD_SHIFT : PAGE_SHIFT;

	nptes = size >> shift;



	if (!pte_present(*ptep) && pte_present(entry))

		mm->context.hugetlb_pte_count += nptes;



	addr &= ~(size - 1);

	orig = *ptep;

	hugepage_shift = pte_none(orig) ? PAGE_SIZE : huge_tte_to_shift(orig);

	orig_shift = pte_none(orig) ? PAGE_SIZE : huge_tte_to_shift(orig);



	for (i = 0; i < nptes; i++)

		ptep[i] = __pte(pte_val(entry) + (i << PMD_SHIFT));

		ptep[i] = __pte(pte_val(entry) + (i << shift));



	maybe_tlb_batch_add(mm, addr, ptep, orig, 0, hugepage_shift);

	maybe_tlb_batch_add(mm, addr, ptep, orig, 0, orig_shift);

	/* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */

	if (size == HPAGE_SIZE)

		maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, orig, 0,

				    hugepage_shift);

				    orig_shift);

}



pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
@@ -302,7 +330,11 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, 


	entry = *ptep;

	size = huge_tte_to_size(entry);

	if (size >= HPAGE_SIZE)

		nptes = size >> PMD_SHIFT;

	else

		nptes = size >> PAGE_SHIFT;



	hugepage_shift = pte_none(entry) ? PAGE_SIZE : huge_tte_to_shift(entry);



	if (pte_present(entry))

arch/sparc/mm/init_64.c

+4 −0
Original line number Diff line number Diff line
@@ -345,6 +345,10 @@ static int __init setup_hugepagesz(char *string) 
		hv_pgsz_mask = HV_PGSZ_MASK_4MB;

		hv_pgsz_idx = HV_PGSZ_IDX_4MB;

		break;

	case HPAGE_64K_SHIFT:

		hv_pgsz_mask = HV_PGSZ_MASK_64K;

		hv_pgsz_idx = HV_PGSZ_IDX_64K;

		break;

	default:

		hv_pgsz_mask = 0;

	}

arch/sparc/mm/tsb.c

+3 −2
Original line number Diff line number Diff line
@@ -147,12 +147,13 @@ void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, 


	spin_lock_irqsave(&mm->context.lock, flags);



	if (hugepage_shift == PAGE_SHIFT) {

	if (hugepage_shift < HPAGE_SHIFT) {

		base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;

		nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;

		if (tlb_type == cheetah_plus || tlb_type == hypervisor)

			base = __pa(base);

		__flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, nentries);

		__flush_huge_tsb_one_entry(base, vaddr, PAGE_SHIFT, nentries,

					   hugepage_shift);

	}

#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)

	else if (mm->context.tsb_block[MM_TSB_HUGE].tsb) {

CRA Git | Maintained and supported by SUSTech CRA and CCSE