Commit a372acfa authored by Christophe Leroy's avatar Christophe Leroy Committed by Scott Wood
Browse files

powerpc/8xx: Map linear kernel RAM with 8M pages 



On a live running system (VoIP gateway for Air Trafic Control), over
a 10 minutes period (with 277s idle), we get 87 millions DTLB misses
and approximatly 35 secondes are spent in DTLB handler.
This represents 5.8% of the overall time and even 10.8% of the
non-idle time.
Among those 87 millions DTLB misses, 15% are on user addresses and
85% are on kernel addresses. And within the kernel addresses, 93%
are on addresses from the linear address space and only 7% are on
addresses from the virtual address space.

MPC8xx has no BATs but it has 8Mb page size. This patch implements
mapping of kernel RAM using 8Mb pages, on the same model as what is
done on the 40x.

In 4k pages mode, each PGD entry maps a 4Mb area: we map every two
entries to the same 8Mb physical page. In each second entry, we add
4Mb to the page physical address to ease life of the FixupDAR
routine. This is just ignored by HW.

In 16k pages mode, each PGD entry maps a 64Mb area: each PGD entry
will point to the first page of the area. The DTLB handler adds
the 3 bits from EPN to map the correct page.

With this patch applied, we now get only 13 millions TLB misses
during the 10 minutes period. The idle time has increased to 313s
and the overall time spent in DTLB miss handler is 6.3s, which
represents 1% of the overall time and 2.2% of non-idle time.

Signed-off-by: default avatarChristophe Leroy <christophe.leroy@c-s.fr>
Signed-off-by: default avatarScott Wood <oss@buserror.net>
parent 913a6b3d

arch/powerpc/kernel/head_8xx.S

+33 −2
Original line number Diff line number Diff line
@@ -398,11 +398,13 @@ DataStoreTLBMiss: 
	BRANCH_UNLESS_KERNEL(3f)

	lis	r11, (swapper_pg_dir-PAGE_OFFSET)@ha

3:

	mtcr	r3



	/* Insert level 1 index */

	rlwimi	r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29

	lwz	r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11)	/* Get the level 1 entry */

	mtcr	r11

	bt-	28,DTLBMiss8M		/* bit 28 = Large page (8M) */

	mtcr	r3



	/* We have a pte table, so load fetch the pte from the table.

	 */
@@ -455,6 +457,29 @@ DataStoreTLBMiss: 
	EXCEPTION_EPILOG_0

	rfi



DTLBMiss8M:

	mtcr	r3

	ori	r11, r11, MD_SVALID

	MTSPR_CPU6(SPRN_MD_TWC, r11, r3)

#ifdef CONFIG_PPC_16K_PAGES

	/*

	 * In 16k pages mode, each PGD entry defines a 64M block.

	 * Here we select the 8M page within the block.

	 */

	rlwimi	r11, r10, 0, 0x03800000

#endif

	rlwinm	r10, r11, 0, 0xff800000

	ori	r10, r10, 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY	| \

			  _PAGE_PRESENT

	MTSPR_CPU6(SPRN_MD_RPN, r10, r3)	/* Update TLB entry */



	li	r11, RPN_PATTERN

	mfspr	r3, SPRN_SPRG_SCRATCH2

	mtspr	SPRN_DAR, r11	/* Tag DAR */

	EXCEPTION_EPILOG_0

	rfi





/* This is an instruction TLB error on the MPC8xx.  This could be due

 * to many reasons, such as executing guarded memory or illegal instruction

 * addresses.  There is nothing to do but handle a big time error fault.
@@ -532,13 +557,15 @@ FixupDAR:/* Entry point for dcbx workaround. */ 
	/* Insert level 1 index */

3:	rlwimi	r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29

	lwz	r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11)	/* Get the level 1 entry */

	mtcr	r11

	bt	28,200f		/* bit 28 = Large page (8M) */

	rlwinm	r11, r11,0,0,19	/* Extract page descriptor page address */

	/* Insert level 2 index */

	rlwimi	r11, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29

	lwz	r11, 0(r11)	/* Get the pte */

	/* concat physical page address(r11) and page offset(r10) */

	rlwimi	r11, r10, 0, 32 - PAGE_SHIFT, 31

	lwz	r11,0(r11)

201:	lwz	r11,0(r11)

/* Check if it really is a dcbx instruction. */

/* dcbt and dcbtst does not generate DTLB Misses/Errors,

 * no need to include them here */
@@ -557,6 +584,10 @@ FixupDAR:/* Entry point for dcbx workaround. */ 
141:	mfspr	r10,SPRN_SPRG_SCRATCH2

	b	DARFixed	/* Nope, go back to normal TLB processing */



	/* concat physical page address(r11) and page offset(r10) */

200:	rlwimi	r11, r10, 0, 32 - (PAGE_SHIFT << 1), 31

	b	201b



144:	mfspr	r10, SPRN_DSISR

	rlwinm	r10, r10,0,7,5	/* Clear store bit for buggy dcbst insn */

	mtspr	SPRN_DSISR, r10

arch/powerpc/mm/8xx_mmu.c

0 → 100644
+83 −0
Original line number Diff line number Diff line 
/*

 * This file contains the routines for initializing the MMU

 * on the 8xx series of chips.

 *  -- christophe

 *

 *  Derived from arch/powerpc/mm/40x_mmu.c:

 *

 *  This program is free software; you can redistribute it and/or

 *  modify it under the terms of the GNU General Public License

 *  as published by the Free Software Foundation; either version

 *  2 of the License, or (at your option) any later version.

 *

 */



#include <linux/memblock.h>



#include "mmu_decl.h"



extern int __map_without_ltlbs;

/*

 * MMU_init_hw does the chip-specific initialization of the MMU hardware.

 */

void __init MMU_init_hw(void)

{

	/* Nothing to do for the time being but keep it similar to other PPC */

}



#define LARGE_PAGE_SIZE_4M	(1<<22)

#define LARGE_PAGE_SIZE_8M	(1<<23)

#define LARGE_PAGE_SIZE_64M	(1<<26)



unsigned long __init mmu_mapin_ram(unsigned long top)

{

	unsigned long v, s, mapped;

	phys_addr_t p;



	v = KERNELBASE;

	p = 0;

	s = top;



	if (__map_without_ltlbs)

		return 0;



#ifdef CONFIG_PPC_4K_PAGES

	while (s >= LARGE_PAGE_SIZE_8M) {

		pmd_t *pmdp;

		unsigned long val = p | MD_PS8MEG;



		pmdp = pmd_offset(pud_offset(pgd_offset_k(v), v), v);

		*pmdp++ = __pmd(val);

		*pmdp++ = __pmd(val + LARGE_PAGE_SIZE_4M);



		v += LARGE_PAGE_SIZE_8M;

		p += LARGE_PAGE_SIZE_8M;

		s -= LARGE_PAGE_SIZE_8M;

	}

#else /* CONFIG_PPC_16K_PAGES */

	while (s >= LARGE_PAGE_SIZE_64M) {

		pmd_t *pmdp;

		unsigned long val = p | MD_PS8MEG;



		pmdp = pmd_offset(pud_offset(pgd_offset_k(v), v), v);

		*pmdp++ = __pmd(val);



		v += LARGE_PAGE_SIZE_64M;

		p += LARGE_PAGE_SIZE_64M;

		s -= LARGE_PAGE_SIZE_64M;

	}

#endif



	mapped = top - s;



	/* If the size of RAM is not an exact power of two, we may not

	 * have covered RAM in its entirety with 8 MiB

	 * pages. Consequently, restrict the top end of RAM currently

	 * allocable so that calls to the MEMBLOCK to allocate PTEs for "tail"

	 * coverage with normal-sized pages (or other reasons) do not

	 * attempt to allocate outside the allowed range.

	 */

	memblock_set_current_limit(mapped);



	return mapped;

}

arch/powerpc/mm/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ obj-$(CONFIG_PPC_ICSWX) += icswx.o 
obj-$(CONFIG_PPC_ICSWX_PID)	+= icswx_pid.o

obj-$(CONFIG_40x)		+= 40x_mmu.o

obj-$(CONFIG_44x)		+= 44x_mmu.o

obj-$(CONFIG_PPC_8xx)		+= 8xx_mmu.o

obj-$(CONFIG_PPC_FSL_BOOK3E)	+= fsl_booke_mmu.o

obj-$(CONFIG_NEED_MULTIPLE_NODES) += numa.o

obj-$(CONFIG_PPC_SPLPAR)	+= vphn.o

arch/powerpc/mm/mmu_decl.h

+3 −12
Original line number Diff line number Diff line
@@ -133,22 +133,17 @@ extern void wii_memory_fixups(void); 
/* ...and now those things that may be slightly different between processor

 * architectures.  -- Dan

 */

#if defined(CONFIG_8xx)

#define MMU_init_hw()		do { } while(0)

#define mmu_mapin_ram(top)	(0UL)



#elif defined(CONFIG_4xx)

#ifdef CONFIG_PPC32

extern void MMU_init_hw(void);

extern unsigned long mmu_mapin_ram(unsigned long top);

#endif



#elif defined(CONFIG_PPC_FSL_BOOK3E)

#ifdef CONFIG_PPC_FSL_BOOK3E

extern unsigned long map_mem_in_cams(unsigned long ram, int max_cam_idx,

				     bool dryrun);

extern unsigned long calc_cam_sz(unsigned long ram, unsigned long virt,

				 phys_addr_t phys);

#ifdef CONFIG_PPC32

extern void MMU_init_hw(void);

extern unsigned long mmu_mapin_ram(unsigned long top);

extern void adjust_total_lowmem(void);

extern int switch_to_as1(void);

extern void restore_to_as0(int esel, int offset, void *dt_ptr, int bootcpu);
@@ -163,8 +158,4 @@ struct tlbcam { 
	u32	MAS3;

	u32	MAS7;

};

#elif defined(CONFIG_PPC32)

/* anything 32-bit except 4xx or 8xx */

extern void MMU_init_hw(void);

extern unsigned long mmu_mapin_ram(unsigned long top);

#endif

CRA Git | Maintained and supported by SUSTech CRA and CCSE