Commit bb7f3808 authored by Christophe Leroy's avatar Christophe Leroy Committed by Scott Wood
Browse files

powerpc/8xx: Don't use page table for linear memory space 



Instead of using the first level page table to define mappings for
the linear memory space, we can use direct mapping from the TLB
handling routines. This has several advantages:
* No need to read the tables at each TLB miss
* No issue in 16k pages mode where the 1st level table maps 64 Mbytes

The size of the available linear space is known at system startup.
In order to avoid data access at each TLB miss to know the memory
size, the TLB routine is patched at startup with the proper size

This patch provides a 10%-15% improvment of TLB miss handling for
kernel addresses

Signed-off-by: default avatarChristophe Leroy <christophe.leroy@c-s.fr>
Signed-off-by: default avatarScott Wood <oss@buserror.net>
parent 6264dbb9

arch/powerpc/kernel/head_8xx.S

+37 −34
Original line number Diff line number Diff line
@@ -389,52 +389,52 @@ InstructionTLBMiss: 
 * not enough space in the DataStoreTLBMiss area

 */

DTLBMissIMMR:

	mtcr	r3

	mtcr	r10

	/* Set 512k byte guarded page and mark it valid */

	li	r10, MD_PS512K | MD_GUARDED | MD_SVALID

	MTSPR_CPU6(SPRN_MD_TWC, r10, r3)

	MTSPR_CPU6(SPRN_MD_TWC, r10, r11)

	mfspr	r10, SPRN_IMMR			/* Get current IMMR */

	rlwinm	r10, r10, 0, 0xfff80000		/* Get 512 kbytes boundary */

	ori	r10, r10, 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY	| \

			  _PAGE_PRESENT | _PAGE_NO_CACHE

	MTSPR_CPU6(SPRN_MD_RPN, r10, r3)	/* Update TLB entry */

	MTSPR_CPU6(SPRN_MD_RPN, r10, r11)	/* Update TLB entry */



	li	r11, RPN_PATTERN

	mfspr	r3, SPRN_SPRG_SCRATCH2

	mtspr	SPRN_DAR, r11	/* Tag DAR */

	EXCEPTION_EPILOG_0

	rfi



	. = 0x1200

DataStoreTLBMiss:

	mtspr	SPRN_SPRG_SCRATCH2, r3

	EXCEPTION_PROLOG_0

	mfcr	r3

	mfcr	r10



	/* If we are faulting a kernel address, we have to use the

	 * kernel page tables.

	 */

	mfspr	r10, SPRN_MD_EPN

	IS_KERNEL(r11, r10)

	mfspr	r11, SPRN_M_TW	/* Get level 1 table */

	BRANCH_UNLESS_KERNEL(3f)



	rlwinm	r11, r10, 16, 0xfff8

	mfspr	r11, SPRN_MD_EPN

	rlwinm	r11, r11, 16, 0xfff8

#ifndef CONFIG_PIN_TLB

	cmpli	cr0, r11, VIRT_IMMR_BASE@h

#endif

	cmpli	cr7, r11, PAGE_OFFSET@h

#ifndef CONFIG_PIN_TLB

_ENTRY(DTLBMiss_jmp)

	beq-	DTLBMissIMMR

#endif

	bge-	cr7, 4f



	lis	r11, (swapper_pg_dir-PAGE_OFFSET)@ha

	mfspr	r11, SPRN_M_TW	/* Get level 1 table */

3:

	mtcr	r10

#ifdef CONFIG_8xx_CPU6

	mtspr	SPRN_SPRG_SCRATCH2, r3

#endif

	mfspr	r10, SPRN_MD_EPN



	/* Insert level 1 index */

	rlwimi	r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29

	lwz	r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11)	/* Get the level 1 entry */

	mtcr	r11

	bt-	28,DTLBMiss8M		/* bit 28 = Large page (8M) */

	mtcr	r3



	/* We have a pte table, so load fetch the pte from the table.

	 */
@@ -482,29 +482,30 @@ _ENTRY(DTLBMiss_jmp) 
	MTSPR_CPU6(SPRN_MD_RPN, r10, r3)	/* Update TLB entry */



	/* Restore registers */

#ifdef CONFIG_8xx_CPU6

	mfspr	r3, SPRN_SPRG_SCRATCH2

#endif

	mtspr	SPRN_DAR, r11	/* Tag DAR */

	EXCEPTION_EPILOG_0

	rfi



DTLBMiss8M:

	mtcr	r3

	ori	r11, r11, MD_SVALID

	MTSPR_CPU6(SPRN_MD_TWC, r11, r3)

#ifdef CONFIG_PPC_16K_PAGES

	/*

	 * In 16k pages mode, each PGD entry defines a 64M block.

	 * Here we select the 8M page within the block.

	 */

	rlwimi	r11, r10, 0, 0x03800000

#endif

	rlwinm	r10, r11, 0, 0xff800000

4:

_ENTRY(DTLBMiss_cmp)

	cmpli	cr0, r11, PAGE_OFFSET@h

	lis	r11, (swapper_pg_dir-PAGE_OFFSET)@ha

	bge-	3b



	mtcr	r10

	/* Set 8M byte page and mark it valid */

	li	r10, MD_PS8MEG | MD_SVALID

	MTSPR_CPU6(SPRN_MD_TWC, r10, r11)

	mfspr	r10, SPRN_MD_EPN

	rlwinm	r10, r10, 0, 0x0f800000		/* 8xx supports max 256Mb RAM */

	ori	r10, r10, 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY	| \

			  _PAGE_PRESENT

	MTSPR_CPU6(SPRN_MD_RPN, r10, r3)	/* Update TLB entry */

	MTSPR_CPU6(SPRN_MD_RPN, r10, r11)	/* Update TLB entry */



	li	r11, RPN_PATTERN

	mfspr	r3, SPRN_SPRG_SCRATCH2

	mtspr	SPRN_DAR, r11	/* Tag DAR */

	EXCEPTION_EPILOG_0

	rfi
@@ -583,12 +584,14 @@ FixupDAR:/* Entry point for dcbx workaround. */ 
	IS_KERNEL(r11, r10)

	mfspr	r11, SPRN_M_TW	/* Get level 1 table */

	BRANCH_UNLESS_KERNEL(3f)

	rlwinm	r11, r10, 16, 0xfff8

_ENTRY(FixupDAR_cmp)

	cmpli	cr7, r11, PAGE_OFFSET@h

	blt-	cr7, 200f

	lis	r11, (swapper_pg_dir-PAGE_OFFSET)@ha

	/* Insert level 1 index */

3:	rlwimi	r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29

	lwz	r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11)	/* Get the level 1 entry */

	mtcr	r11

	bt	28,200f		/* bit 28 = Large page (8M) */

	rlwinm	r11, r11,0,0,19	/* Extract page descriptor page address */

	/* Insert level 2 index */

	rlwimi	r11, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29
@@ -614,8 +617,8 @@ FixupDAR:/* Entry point for dcbx workaround. */ 
141:	mfspr	r10,SPRN_SPRG_SCRATCH2

	b	DARFixed	/* Nope, go back to normal TLB processing */



	/* concat physical page address(r11) and page offset(r10) */

200:	rlwimi	r11, r10, 0, 32 - (PAGE_SHIFT << 1), 31

	/* create physical page address from effective address */

200:	tophys(r11, r10)

	b	201b



144:	mfspr	r10, SPRN_DSISR

arch/powerpc/mm/8xx_mmu.c

+18 −38
Original line number Diff line number Diff line
@@ -58,9 +58,7 @@ void __init MMU_init_hw(void) 
	/* Nothing to do for the time being but keep it similar to other PPC */

}



#define LARGE_PAGE_SIZE_4M	(1<<22)

#define LARGE_PAGE_SIZE_8M	(1<<23)

#define LARGE_PAGE_SIZE_64M	(1<<26)



static void mmu_mapin_immr(void)

{
@@ -77,52 +75,33 @@ static void mmu_mapin_immr(void) 
#ifndef CONFIG_PIN_TLB

extern unsigned int DTLBMiss_jmp;

#endif

extern unsigned int DTLBMiss_cmp, FixupDAR_cmp;



unsigned long __init mmu_mapin_ram(unsigned long top)

void mmu_patch_cmp_limit(unsigned int *addr, unsigned long mapped)

{

	unsigned long v, s, mapped;

	phys_addr_t p;

	unsigned int instr = *addr;



	v = KERNELBASE;

	p = 0;

	s = top;

	instr &= 0xffff0000;

	instr |= (unsigned long)__va(mapped) >> 16;

	patch_instruction(addr, instr);

}



unsigned long __init mmu_mapin_ram(unsigned long top)

{

	unsigned long mapped;



	if (__map_without_ltlbs) {

		mapped = 0;

		mmu_mapin_immr();

#ifndef CONFIG_PIN_TLB

		patch_instruction(&DTLBMiss_jmp, PPC_INST_NOP);

#endif

		return 0;

	}



#ifdef CONFIG_PPC_4K_PAGES

	while (s >= LARGE_PAGE_SIZE_8M) {

		pmd_t *pmdp;

		unsigned long val = p | MD_PS8MEG;



		pmdp = pmd_offset(pud_offset(pgd_offset_k(v), v), v);

		*pmdp++ = __pmd(val);

		*pmdp++ = __pmd(val + LARGE_PAGE_SIZE_4M);



		v += LARGE_PAGE_SIZE_8M;

		p += LARGE_PAGE_SIZE_8M;

		s -= LARGE_PAGE_SIZE_8M;

	} else {

		mapped = top & ~(LARGE_PAGE_SIZE_8M - 1);

	}

#else /* CONFIG_PPC_16K_PAGES */

	while (s >= LARGE_PAGE_SIZE_64M) {

		pmd_t *pmdp;

		unsigned long val = p | MD_PS8MEG;



		pmdp = pmd_offset(pud_offset(pgd_offset_k(v), v), v);

		*pmdp++ = __pmd(val);



		v += LARGE_PAGE_SIZE_64M;

		p += LARGE_PAGE_SIZE_64M;

		s -= LARGE_PAGE_SIZE_64M;

	}

#endif



	mapped = top - s;

	mmu_patch_cmp_limit(&DTLBMiss_cmp, mapped);

	mmu_patch_cmp_limit(&FixupDAR_cmp, mapped);



	/* If the size of RAM is not an exact power of two, we may not

	 * have covered RAM in its entirety with 8 MiB
@@ -131,6 +110,7 @@ unsigned long __init mmu_mapin_ram(unsigned long top) 
	 * coverage with normal-sized pages (or other reasons) do not

	 * attempt to allocate outside the allowed range.

	 */

	if (mapped)

		memblock_set_current_limit(mapped);



	return mapped;

CRA Git | Maintained and supported by SUSTech CRA and CCSE