Commit 609af38f authored by Scott Wood's avatar Scott Wood
Browse files

powerpc/booke64: Critical and machine check exception support 



Add special state saving for critical and machine check exceptions.

Most of this code could be used to handle debug exceptions taken from
kernel space, but actually doing so is outside the scope of this patch.

The various critical and machine check exceptions now point to their
real handlers, rather than hanging the kernel.

Signed-off-by: default avatarScott Wood <scottwood@freescale.com>
parent 31f71248

arch/powerpc/include/asm/paca.h

+5 −2
Original line number Diff line number Diff line
@@ -116,8 +116,11 @@ struct paca_struct { 
	/* Shared by all threads of a core -- points to tcd of first thread */

	struct tlb_core_data *tcd_ptr;



	/* We can have up to 3 levels of reentrancy in the TLB miss handler */

	u64 extlb[3][EX_TLB_SIZE / sizeof(u64)];

	/*

	 * We can have up to 3 levels of reentrancy in the TLB miss handler,

	 * in each of four exception levels (normal, crit, mcheck, debug).

	 */

	u64 extlb[12][EX_TLB_SIZE / sizeof(u64)];

	u64 exmc[8];		/* used for machine checks */

	u64 excrit[8];		/* used for crit interrupts */

	u64 exdbg[8];		/* used for debug interrupts */

arch/powerpc/kernel/exceptions-64e.S

+284 −59
Original line number Diff line number Diff line
@@ -34,20 +34,250 @@ 
 *     special interrupts from within a non-standard level will probably

 *     blow you up

 */

#define	SPECIAL_EXC_FRAME_SIZE	INT_FRAME_SIZE

#define SPECIAL_EXC_SRR0	0

#define SPECIAL_EXC_SRR1	1

#define SPECIAL_EXC_SPRG_GEN	2

#define SPECIAL_EXC_SPRG_TLB	3

#define SPECIAL_EXC_MAS0	4

#define SPECIAL_EXC_MAS1	5

#define SPECIAL_EXC_MAS2	6

#define SPECIAL_EXC_MAS3	7

#define SPECIAL_EXC_MAS6	8

#define SPECIAL_EXC_MAS7	9

#define SPECIAL_EXC_MAS5	10	/* E.HV only */

#define SPECIAL_EXC_MAS8	11	/* E.HV only */

#define SPECIAL_EXC_IRQHAPPENED	12

#define SPECIAL_EXC_DEAR	13

#define SPECIAL_EXC_ESR		14

#define SPECIAL_EXC_SOFTE	15

#define SPECIAL_EXC_CSRR0	16

#define SPECIAL_EXC_CSRR1	17

/* must be even to keep 16-byte stack alignment */

#define SPECIAL_EXC_END		18



#define SPECIAL_EXC_FRAME_SIZE	(INT_FRAME_SIZE + SPECIAL_EXC_END * 8)

#define SPECIAL_EXC_FRAME_OFFS  (INT_FRAME_SIZE - 288)



#define SPECIAL_EXC_STORE(reg, name) \

	std	reg, (SPECIAL_EXC_##name * 8 + SPECIAL_EXC_FRAME_OFFS)(r1)



#define SPECIAL_EXC_LOAD(reg, name) \

	ld	reg, (SPECIAL_EXC_##name * 8 + SPECIAL_EXC_FRAME_OFFS)(r1)



special_reg_save:

	lbz	r9,PACAIRQHAPPENED(r13)

	RECONCILE_IRQ_STATE(r3,r4)



/* Now we only store something to exception thread info */

#define	EXC_LEVEL_EXCEPTION_PROLOG(type)				\

	ld	r14,PACAKSAVE(r13);					\

	CURRENT_THREAD_INFO(r14, r14);					\

	CURRENT_THREAD_INFO(r15, r1);					\

	ld	r10,TI_FLAGS(r14);		     			\

	std	r10,TI_FLAGS(r15);			     		\

	ld	r10,TI_PREEMPT(r14);		     			\

	std	r10,TI_PREEMPT(r15);		     			\

	ld	r10,TI_TASK(r14);			     		\

	std	r10,TI_TASK(r15);

	/*

	 * We only need (or have stack space) to save this stuff if

	 * we interrupted the kernel.

	 */

	ld	r3,_MSR(r1)

	andi.	r3,r3,MSR_PR

	bnelr



	/* Copy info into temporary exception thread info */

	ld	r11,PACAKSAVE(r13)

	CURRENT_THREAD_INFO(r11, r11)

	CURRENT_THREAD_INFO(r12, r1)

	ld	r10,TI_FLAGS(r11)

	std	r10,TI_FLAGS(r12)

	ld	r10,TI_PREEMPT(r11)

	std	r10,TI_PREEMPT(r12)

	ld	r10,TI_TASK(r11)

	std	r10,TI_TASK(r12)



	/*

	 * Advance to the next TLB exception frame for handler

	 * types that don't do it automatically.

	 */

	LOAD_REG_ADDR(r11,extlb_level_exc)

	lwz	r12,0(r11)

	mfspr	r10,SPRN_SPRG_TLB_EXFRAME

	add	r10,r10,r12

	mtspr	SPRN_SPRG_TLB_EXFRAME,r10



	/*

	 * Save registers needed to allow nesting of certain exceptions

	 * (such as TLB misses) inside special exception levels

	 */

	mfspr	r10,SPRN_SRR0

	SPECIAL_EXC_STORE(r10,SRR0)

	mfspr	r10,SPRN_SRR1

	SPECIAL_EXC_STORE(r10,SRR1)

	mfspr	r10,SPRN_SPRG_GEN_SCRATCH

	SPECIAL_EXC_STORE(r10,SPRG_GEN)

	mfspr	r10,SPRN_SPRG_TLB_SCRATCH

	SPECIAL_EXC_STORE(r10,SPRG_TLB)

	mfspr	r10,SPRN_MAS0

	SPECIAL_EXC_STORE(r10,MAS0)

	mfspr	r10,SPRN_MAS1

	SPECIAL_EXC_STORE(r10,MAS1)

	mfspr	r10,SPRN_MAS2

	SPECIAL_EXC_STORE(r10,MAS2)

	mfspr	r10,SPRN_MAS3

	SPECIAL_EXC_STORE(r10,MAS3)

	mfspr	r10,SPRN_MAS6

	SPECIAL_EXC_STORE(r10,MAS6)

	mfspr	r10,SPRN_MAS7

	SPECIAL_EXC_STORE(r10,MAS7)

BEGIN_FTR_SECTION

	mfspr	r10,SPRN_MAS5

	SPECIAL_EXC_STORE(r10,MAS5)

	mfspr	r10,SPRN_MAS8

	SPECIAL_EXC_STORE(r10,MAS8)



	/* MAS5/8 could have inappropriate values if we interrupted KVM code */

	li	r10,0

	mtspr	SPRN_MAS5,r10

	mtspr	SPRN_MAS8,r10

END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)

	SPECIAL_EXC_STORE(r9,IRQHAPPENED)



	mfspr	r10,SPRN_DEAR

	SPECIAL_EXC_STORE(r10,DEAR)

	mfspr	r10,SPRN_ESR

	SPECIAL_EXC_STORE(r10,ESR)



	lbz	r10,PACASOFTIRQEN(r13)

	SPECIAL_EXC_STORE(r10,SOFTE)

	ld	r10,_NIP(r1)

	SPECIAL_EXC_STORE(r10,CSRR0)

	ld	r10,_MSR(r1)

	SPECIAL_EXC_STORE(r10,CSRR1)



	blr



ret_from_level_except:

	ld	r3,_MSR(r1)

	andi.	r3,r3,MSR_PR

	beq	1f

	b	ret_from_except

1:



	LOAD_REG_ADDR(r11,extlb_level_exc)

	lwz	r12,0(r11)

	mfspr	r10,SPRN_SPRG_TLB_EXFRAME

	sub	r10,r10,r12

	mtspr	SPRN_SPRG_TLB_EXFRAME,r10



	/*

	 * It's possible that the special level exception interrupted a

	 * TLB miss handler, and inserted the same entry that the

	 * interrupted handler was about to insert.  On CPUs without TLB

	 * write conditional, this can result in a duplicate TLB entry.

	 * Wipe all non-bolted entries to be safe.

	 *

	 * Note that this doesn't protect against any TLB misses

	 * we may take accessing the stack from here to the end of

	 * the special level exception.  It's not clear how we can

	 * reasonably protect against that, but only CPUs with

	 * neither TLB write conditional nor bolted kernel memory

	 * are affected.  Do any such CPUs even exist?

	 */

	PPC_TLBILX_ALL(0,R0)



	REST_NVGPRS(r1)



	SPECIAL_EXC_LOAD(r10,SRR0)

	mtspr	SPRN_SRR0,r10

	SPECIAL_EXC_LOAD(r10,SRR1)

	mtspr	SPRN_SRR1,r10

	SPECIAL_EXC_LOAD(r10,SPRG_GEN)

	mtspr	SPRN_SPRG_GEN_SCRATCH,r10

	SPECIAL_EXC_LOAD(r10,SPRG_TLB)

	mtspr	SPRN_SPRG_TLB_SCRATCH,r10

	SPECIAL_EXC_LOAD(r10,MAS0)

	mtspr	SPRN_MAS0,r10

	SPECIAL_EXC_LOAD(r10,MAS1)

	mtspr	SPRN_MAS1,r10

	SPECIAL_EXC_LOAD(r10,MAS2)

	mtspr	SPRN_MAS2,r10

	SPECIAL_EXC_LOAD(r10,MAS3)

	mtspr	SPRN_MAS3,r10

	SPECIAL_EXC_LOAD(r10,MAS6)

	mtspr	SPRN_MAS6,r10

	SPECIAL_EXC_LOAD(r10,MAS7)

	mtspr	SPRN_MAS7,r10

BEGIN_FTR_SECTION

	SPECIAL_EXC_LOAD(r10,MAS5)

	mtspr	SPRN_MAS5,r10

	SPECIAL_EXC_LOAD(r10,MAS8)

	mtspr	SPRN_MAS8,r10

END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)



	lbz	r6,PACASOFTIRQEN(r13)

	ld	r5,SOFTE(r1)



	/* Interrupts had better not already be enabled... */

	twnei	r6,0



	cmpwi	cr0,r5,0

	beq	1f



	TRACE_ENABLE_INTS

	stb	r5,PACASOFTIRQEN(r13)

1:

	/*

	 * Restore PACAIRQHAPPENED rather than setting it based on

	 * the return MSR[EE], since we could have interrupted

	 * __check_irq_replay() or other inconsistent transitory

	 * states that must remain that way.

	 */

	SPECIAL_EXC_LOAD(r10,IRQHAPPENED)

	stb	r10,PACAIRQHAPPENED(r13)



	SPECIAL_EXC_LOAD(r10,DEAR)

	mtspr	SPRN_DEAR,r10

	SPECIAL_EXC_LOAD(r10,ESR)

	mtspr	SPRN_ESR,r10



	stdcx.	r0,0,r1		/* to clear the reservation */



	REST_4GPRS(2, r1)

	REST_4GPRS(6, r1)



	ld	r10,_CTR(r1)

	ld	r11,_XER(r1)

	mtctr	r10

	mtxer	r11



	blr



.macro ret_from_level srr0 srr1 paca_ex scratch

	bl	ret_from_level_except



	ld	r10,_LINK(r1)

	ld	r11,_CCR(r1)

	ld	r0,GPR13(r1)

	mtlr	r10

	mtcr	r11



	ld	r10,GPR10(r1)

	ld	r11,GPR11(r1)

	ld	r12,GPR12(r1)

	mtspr	\scratch,r0



	std	r10,\paca_ex+EX_R10(r13);

	std	r11,\paca_ex+EX_R11(r13);

	ld	r10,_NIP(r1)

	ld	r11,_MSR(r1)

	ld	r0,GPR0(r1)

	ld	r1,GPR1(r1)

	mtspr	\srr0,r10

	mtspr	\srr1,r11

	ld	r10,\paca_ex+EX_R10(r13)

	ld	r11,\paca_ex+EX_R11(r13)

	mfspr	r13,\scratch

.endm



ret_from_crit_except:

	ret_from_level SPRN_CSRR0 SPRN_CSRR1 PACA_EXCRIT SPRN_SPRG_CRIT_SCRATCH

	rfci



ret_from_mc_except:

	ret_from_level SPRN_MCSRR0 SPRN_MCSRR1 PACA_EXMC SPRN_SPRG_MC_SCRATCH

	rfmci



/* Exception prolog code for all exceptions */

#define EXCEPTION_PROLOG(n, intnum, type, addition)	    		    \
@@ -81,22 +311,19 @@ 


#define CRIT_SET_KSTACK						            \

	ld	r1,PACA_CRIT_STACK(r13);				    \

	subi	r1,r1,SPECIAL_EXC_FRAME_SIZE;				    \

	EXC_LEVEL_EXCEPTION_PROLOG(CRIT);

	subi	r1,r1,SPECIAL_EXC_FRAME_SIZE

#define SPRN_CRIT_SRR0	SPRN_CSRR0

#define SPRN_CRIT_SRR1	SPRN_CSRR1



#define DBG_SET_KSTACK						            \

	ld	r1,PACA_DBG_STACK(r13);					    \

	subi	r1,r1,SPECIAL_EXC_FRAME_SIZE;				    \

	EXC_LEVEL_EXCEPTION_PROLOG(DBG);

	subi	r1,r1,SPECIAL_EXC_FRAME_SIZE

#define SPRN_DBG_SRR0	SPRN_DSRR0

#define SPRN_DBG_SRR1	SPRN_DSRR1



#define MC_SET_KSTACK						            \

	ld	r1,PACA_MC_STACK(r13);					    \

	subi	r1,r1,SPECIAL_EXC_FRAME_SIZE;				    \

	EXC_LEVEL_EXCEPTION_PROLOG(MC);

	subi	r1,r1,SPECIAL_EXC_FRAME_SIZE

#define SPRN_MC_SRR0	SPRN_MCSRR0

#define SPRN_MC_SRR1	SPRN_MCSRR1
@@ -322,27 +549,25 @@ interrupt_end_book3e: 
	START_EXCEPTION(critical_input);

	CRIT_EXCEPTION_PROLOG(0x100, BOOKE_INTERRUPT_CRITICAL,

			      PROLOG_ADDITION_NONE)

//	EXCEPTION_COMMON_CRIT(0x100)

//	INTS_DISABLE

//	bl	special_reg_save_crit

//	CHECK_NAPPING();

//	addi	r3,r1,STACK_FRAME_OVERHEAD

//	bl	.critical_exception

//	b	ret_from_crit_except

	b	.

	EXCEPTION_COMMON_CRIT(0x100)

	bl	.save_nvgprs

	bl	special_reg_save

	CHECK_NAPPING();

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	.unknown_exception

	b	ret_from_crit_except



/* Machine Check Interrupt */

	START_EXCEPTION(machine_check);

	MC_EXCEPTION_PROLOG(0x000, BOOKE_INTERRUPT_MACHINE_CHECK,

			    PROLOG_ADDITION_NONE)

//	EXCEPTION_COMMON_MC(0x000)

//	INTS_DISABLE

//	bl	special_reg_save_mc

//	addi	r3,r1,STACK_FRAME_OVERHEAD

//	CHECK_NAPPING();

//	bl	.machine_check_exception

//	b	ret_from_mc_except

	b	.

	EXCEPTION_COMMON_MC(0x000)

	bl	.save_nvgprs

	bl	special_reg_save

	CHECK_NAPPING();

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	.machine_check_exception

	b	ret_from_mc_except



/* Data Storage Interrupt */

	START_EXCEPTION(data_storage)
@@ -461,14 +686,17 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 
	START_EXCEPTION(watchdog);

	CRIT_EXCEPTION_PROLOG(0x9f0, BOOKE_INTERRUPT_WATCHDOG,

			      PROLOG_ADDITION_NONE)

//	EXCEPTION_COMMON_CRIT(0x9f0)

//	INTS_DISABLE

//	bl	special_reg_save_crit

//	CHECK_NAPPING();

//	addi	r3,r1,STACK_FRAME_OVERHEAD

//	bl	.unknown_exception

//	b	ret_from_crit_except

	b	.

	EXCEPTION_COMMON_CRIT(0x9f0)

	bl	.save_nvgprs

	bl	special_reg_save

	CHECK_NAPPING();

	addi	r3,r1,STACK_FRAME_OVERHEAD

#ifdef CONFIG_BOOKE_WDT

	bl	.WatchdogException

#else

	bl	.unknown_exception

#endif

	b	ret_from_crit_except



/* System Call Interrupt */

	START_EXCEPTION(system_call)
@@ -541,7 +769,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 
	 */

	mfspr	r14,SPRN_DBSR

	EXCEPTION_COMMON_CRIT(0xd00)

	INTS_DISABLE

	std	r14,_DSISR(r1)

	addi	r3,r1,STACK_FRAME_OVERHEAD

	mr	r4,r14
@@ -634,14 +861,13 @@ kernel_dbg_exc: 
	START_EXCEPTION(doorbell_crit);

	CRIT_EXCEPTION_PROLOG(0x2a0, BOOKE_INTERRUPT_DOORBELL_CRITICAL,

			      PROLOG_ADDITION_NONE)

//	EXCEPTION_COMMON_CRIT(0x2a0)

//	INTS_DISABLE

//	bl	special_reg_save_crit

//	CHECK_NAPPING();

//	addi	r3,r1,STACK_FRAME_OVERHEAD

//	bl	.doorbell_critical_exception

//	b	ret_from_crit_except

	b	.

	EXCEPTION_COMMON_CRIT(0x2a0)

	bl	.save_nvgprs

	bl	special_reg_save

	CHECK_NAPPING();

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	.unknown_exception

	b	ret_from_crit_except



/*

 *	Guest doorbell interrupt
@@ -661,14 +887,13 @@ kernel_dbg_exc: 
	START_EXCEPTION(guest_doorbell_crit);

	CRIT_EXCEPTION_PROLOG(0x2e0, BOOKE_INTERRUPT_GUEST_DBELL_CRIT,

			      PROLOG_ADDITION_NONE)

//	EXCEPTION_COMMON_CRIT(0x2e0)

//	INTS_DISABLE

//	bl	special_reg_save_crit

//	CHECK_NAPPING();

//	addi	r3,r1,STACK_FRAME_OVERHEAD

//	bl	.guest_doorbell_critical_exception

//	b	ret_from_crit_except

	b	.

	EXCEPTION_COMMON_CRIT(0x2e0)

	bl	.save_nvgprs

	bl	special_reg_save

	CHECK_NAPPING();

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	.unknown_exception

	b	ret_from_crit_except



/* Hypervisor call */

	START_EXCEPTION(hypercall);

arch/powerpc/mm/tlb_nohash.c

+11 −0
Original line number Diff line number Diff line
@@ -144,6 +144,15 @@ int mmu_vmemmap_psize; /* Page size used for the virtual mem map */ 
int book3e_htw_mode;		/* HW tablewalk?  Value is PPC_HTW_* */

unsigned long linear_map_top;	/* Top of linear mapping */





/*

 * Number of bytes to add to SPRN_SPRG_TLB_EXFRAME on crit/mcheck/debug

 * exceptions.  This is used for bolted and e6500 TLB miss handlers which

 * do not modify this SPRG in the TLB miss code; for other TLB miss handlers,

 * this is set to zero.

 */

int extlb_level_exc;



#endif /* CONFIG_PPC64 */



#ifdef CONFIG_PPC_FSL_BOOK3E
@@ -559,6 +568,7 @@ static void setup_mmu_htw(void) 
		break;

#ifdef CONFIG_PPC_FSL_BOOK3E

	case PPC_HTW_E6500:

		extlb_level_exc = EX_TLB_SIZE;

		patch_exception(0x1c0, exc_data_tlb_miss_e6500_book3e);

		patch_exception(0x1e0, exc_instruction_tlb_miss_e6500_book3e);

		break;
@@ -652,6 +662,7 @@ static void __early_init_mmu(int boot_cpu) 
		memblock_enforce_memory_limit(linear_map_top);



		if (book3e_htw_mode == PPC_HTW_NONE) {

			extlb_level_exc = EX_TLB_SIZE;

			patch_exception(0x1c0, exc_data_tlb_miss_bolted_book3e);

			patch_exception(0x1e0,

				exc_instruction_tlb_miss_bolted_book3e);

CRA Git | Maintained and supported by SUSTech CRA and CCSE