Commit d04c56f7 authored by Paul Mackerras's avatar Paul Mackerras
Browse files

[POWERPC] Lazy interrupt disabling for 64-bit machines 



This implements a lazy strategy for disabling interrupts.  This means
that local_irq_disable() et al. just clear the 'interrupts are
enabled' flag in the paca.  If an interrupt comes along, the interrupt
entry code notices that interrupts are supposed to be disabled, and
clears the EE bit in SRR1, clears the 'interrupts are hard-enabled'
flag in the paca, and returns.  This means that interrupts only
actually get disabled in the processor when an interrupt comes along.

When interrupts are enabled by local_irq_enable() et al., the code
sets the interrupts-enabled flag in the paca, and then checks whether
interrupts got hard-disabled.  If so, it also sets the EE bit in the
MSR to hard-enable the interrupts.

This has the potential to improve performance, and also makes it
easier to make a kernel that can boot on iSeries and on other 64-bit
machines, since this lazy-disable strategy is very similar to the
soft-disable strategy that iSeries already uses.

This version renames paca->proc_enabled to paca->soft_enabled, and
changes a couple of soft-disables in the kexec code to hard-disables,
which should fix the crash that Michael Ellerman saw.  This doesn't
yet use a reserved CR field for the soft_enabled and hard_enabled
flags.  This applies on top of Stephen Rothwell's patches to make it
possible to build a combined iSeries/other kernel.

Signed-off-by: default avatarPaul Mackerras <paulus@samba.org>
parent 284a9406

arch/powerpc/kernel/asm-offsets.c

+2 −1
Original line number Diff line number Diff line
@@ -118,7 +118,8 @@ int main(void) 
	DEFINE(PACASTABRR, offsetof(struct paca_struct, stab_rr));

	DEFINE(PACAR1, offsetof(struct paca_struct, saved_r1));

	DEFINE(PACATOC, offsetof(struct paca_struct, kernel_toc));

	DEFINE(PACAPROCENABLED, offsetof(struct paca_struct, proc_enabled));

	DEFINE(PACASOFTIRQEN, offsetof(struct paca_struct, soft_enabled));

	DEFINE(PACAHARDIRQEN, offsetof(struct paca_struct, hard_enabled));

	DEFINE(PACASLBCACHE, offsetof(struct paca_struct, slb_cache));

	DEFINE(PACASLBCACHEPTR, offsetof(struct paca_struct, slb_cache_ptr));

	DEFINE(PACACONTEXTID, offsetof(struct paca_struct, context.id));

arch/powerpc/kernel/crash.c

+2 −2
Original line number Diff line number Diff line
@@ -111,7 +111,7 @@ void crash_ipi_callback(struct pt_regs *regs) 
	if (!cpu_online(cpu))

		return;



	local_irq_disable();

	hard_irq_disable();

	if (!cpu_isset(cpu, cpus_in_crash))

		crash_save_this_cpu(regs, cpu);

	cpu_set(cpu, cpus_in_crash);
@@ -289,7 +289,7 @@ void default_machine_crash_shutdown(struct pt_regs *regs) 
	 * an SMP system.

	 * The kernel is broken so disable interrupts.

	 */

	local_irq_disable();

	hard_irq_disable();



	for_each_irq(irq) {

		struct irq_desc *desc = irq_desc + irq;

arch/powerpc/kernel/entry_64.S

+18 −21
Original line number Diff line number Diff line
@@ -87,6 +87,10 @@ system_call_common: 
	addi	r9,r1,STACK_FRAME_OVERHEAD

	ld	r11,exception_marker@toc(r2)

	std	r11,-16(r9)		/* "regshere" marker */

	li	r10,1

	stb	r10,PACASOFTIRQEN(r13)

	stb	r10,PACAHARDIRQEN(r13)

	std	r10,SOFTE(r1)

#ifdef CONFIG_PPC_ISERIES

BEGIN_FW_FTR_SECTION

	/* Hack for handling interrupts when soft-enabling on iSeries */
@@ -94,8 +98,6 @@ BEGIN_FW_FTR_SECTION 
	andi.	r10,r12,MSR_PR		/* from kernel */

	crand	4*cr0+eq,4*cr1+eq,4*cr0+eq

	beq	hardware_interrupt_entry

	lbz	r10,PACAPROCENABLED(r13)

	std	r10,SOFTE(r1)

END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)

#endif

	mfmsr	r11
@@ -460,9 +462,9 @@ _GLOBAL(ret_from_except_lite) 
#endif



restore:

	ld	r5,SOFTE(r1)

#ifdef CONFIG_PPC_ISERIES

BEGIN_FW_FTR_SECTION

	ld	r5,SOFTE(r1)

	cmpdi	0,r5,0

	beq	4f

	/* Check for pending interrupts (iSeries) */
@@ -472,16 +474,16 @@ BEGIN_FW_FTR_SECTION 
	beq+	4f			/* skip do_IRQ if no interrupts */



	li	r3,0

	stb	r3,PACAPROCENABLED(r13)	/* ensure we are soft-disabled */

	stb	r3,PACASOFTIRQEN(r13)	/* ensure we are soft-disabled */

	ori	r10,r10,MSR_EE

	mtmsrd	r10			/* hard-enable again */

	addi	r3,r1,STACK_FRAME_OVERHEAD

	bl	.do_IRQ

	b	.ret_from_except_lite		/* loop back and handle more */



4:	stb	r5,PACAPROCENABLED(r13)

4:

END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)

#endif

	stb	r5,PACASOFTIRQEN(r13)



	ld	r3,_MSR(r1)

	andi.	r0,r3,MSR_RI
@@ -538,25 +540,15 @@ do_work: 
	/* Check that preempt_count() == 0 and interrupts are enabled */

	lwz	r8,TI_PREEMPT(r9)

	cmpwi	cr1,r8,0

#ifdef CONFIG_PPC_ISERIES

BEGIN_FW_FTR_SECTION

	ld	r0,SOFTE(r1)

	cmpdi	r0,0

END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)

#endif

BEGIN_FW_FTR_SECTION

	andi.	r0,r3,MSR_EE

END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)

	crandc	eq,cr1*4+eq,eq

	bne	restore

	/* here we are preempting the current task */

1:

#ifdef CONFIG_PPC_ISERIES

BEGIN_FW_FTR_SECTION

	li	r0,1

	stb	r0,PACAPROCENABLED(r13)

END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)

#endif

	stb	r0,PACASOFTIRQEN(r13)

	stb	r0,PACAHARDIRQEN(r13)

	ori	r10,r10,MSR_EE

	mtmsrd	r10,1		/* reenable interrupts */

	bl	.preempt_schedule
@@ -639,8 +631,7 @@ _GLOBAL(enter_rtas) 
	/* There is no way it is acceptable to get here with interrupts enabled,

	 * check it with the asm equivalent of WARN_ON

	 */

	mfmsr	r6

	andi.	r0,r6,MSR_EE

	lbz	r0,PACASOFTIRQEN(r13)

1:	tdnei	r0,0

.section __bug_table,"a"

	.llong	1b,__LINE__ + 0x1000000, 1f, 2f
@@ -650,6 +641,12 @@ _GLOBAL(enter_rtas) 
2:	.asciz "enter_rtas"

.previous



	/* Hard-disable interrupts */

	mfmsr	r6

	rldicl	r7,r6,48,1

	rotldi	r7,r7,16

	mtmsrd	r7,1



	/* Unfortunately, the stack pointer and the MSR are also clobbered,

	 * so they are saved in the PACA which allows us to restore

	 * our original state after RTAS returns.

arch/powerpc/kernel/head_64.S

+77 −33
Original line number Diff line number Diff line
@@ -35,9 +35,7 @@ 
#include <asm/thread_info.h>

#include <asm/firmware.h>



#ifdef CONFIG_PPC_ISERIES

#define DO_SOFT_DISABLE

#endif



/*

 * We layout physical memory as follows:
@@ -308,7 +306,9 @@ exception_marker: 
	std	r9,_LINK(r1);						   \

	mfctr	r10;			/* save CTR in stackframe	*/ \

	std	r10,_CTR(r1);						   \

	lbz	r10,PACASOFTIRQEN(r13);				   \

	mfspr	r11,SPRN_XER;		/* save XER in stackframe	*/ \

	std	r10,SOFTE(r1);						   \

	std	r11,_XER(r1);						   \

	li	r9,(n)+1;						   \

	std	r9,_TRAP(r1);		/* set trap number		*/ \
@@ -343,6 +343,34 @@ label##_pSeries: \ 
	EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common)





#define MASKABLE_EXCEPTION_PSERIES(n, label)				\

	. = n;								\

	.globl label##_pSeries;						\

label##_pSeries:							\

	HMT_MEDIUM;							\

	mtspr	SPRN_SPRG1,r13;		/* save r13 */			\

	mfspr	r13,SPRN_SPRG3;		/* get paca address into r13 */	\

	std	r9,PACA_EXGEN+EX_R9(r13);	/* save r9, r10 */	\

	std	r10,PACA_EXGEN+EX_R10(r13);				\

	lbz	r10,PACASOFTIRQEN(r13);					\

	mfcr	r9;							\

	cmpwi	r10,0;							\

	beq	masked_interrupt;					\

	mfspr	r10,SPRN_SPRG1;						\

	std	r10,PACA_EXGEN+EX_R13(r13);				\

	std	r11,PACA_EXGEN+EX_R11(r13);				\

	std	r12,PACA_EXGEN+EX_R12(r13);				\

	clrrdi	r12,r13,32;		/* get high part of &label */	\

	mfmsr	r10;							\

	mfspr	r11,SPRN_SRR0;		/* save SRR0 */			\

	LOAD_HANDLER(r12,label##_common)				\

	ori	r10,r10,MSR_IR|MSR_DR|MSR_RI;				\

	mtspr	SPRN_SRR0,r12;						\

	mfspr	r12,SPRN_SRR1;		/* and SRR1 */			\

	mtspr	SPRN_SRR1,r10;						\

	rfid;								\

	b	.	/* prevent speculative execution */



#define STD_EXCEPTION_ISERIES(n, label, area)		\

	.globl label##_iSeries;				\

label##_iSeries:					\
@@ -358,40 +386,32 @@ label##_iSeries: \ 
	HMT_MEDIUM;							\

	mtspr	SPRN_SPRG1,r13;		/* save r13 */			\

	EXCEPTION_PROLOG_ISERIES_1(PACA_EXGEN);				\

	lbz	r10,PACAPROCENABLED(r13);				\

	lbz	r10,PACASOFTIRQEN(r13);					\

	cmpwi	0,r10,0;						\

	beq-	label##_iSeries_masked;					\

	EXCEPTION_PROLOG_ISERIES_2;					\

	b	label##_common;						\



#ifdef DO_SOFT_DISABLE

#ifdef CONFIG_PPC_ISERIES

#define DISABLE_INTS				\

BEGIN_FW_FTR_SECTION;				\

	lbz	r10,PACAPROCENABLED(r13);	\

	li	r11,0;				\

	std	r10,SOFTE(r1);			\

	stb	r11,PACASOFTIRQEN(r13);		\

BEGIN_FW_FTR_SECTION;				\

	stb	r11,PACAHARDIRQEN(r13);		\

END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES);	\

BEGIN_FW_FTR_SECTION;				\

	mfmsr	r10;				\

	stb	r11,PACAPROCENABLED(r13);	\

	ori	r10,r10,MSR_EE;			\

	mtmsrd	r10,1;				\

END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)



#define ENABLE_INTS				\

BEGIN_FW_FTR_SECTION;				\

	lbz	r10,PACAPROCENABLED(r13);	\

	mfmsr	r11;				\

	std	r10,SOFTE(r1);			\

	ori	r11,r11,MSR_EE;			\

END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES);	\

BEGIN_FW_FTR_SECTION;				\

	ld	r12,_MSR(r1);			\

	mfmsr	r11;				\

	rlwimi	r11,r12,0,MSR_EE;		\

END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES);	\

	mtmsrd	r11,1

#else

#define DISABLE_INTS				\

	li	r11,0;				\

	stb	r11,PACASOFTIRQEN(r13);		\

	stb	r11,PACAHARDIRQEN(r13)



#else	/* hard enable/disable interrupts */

#define DISABLE_INTS

#endif /* CONFIG_PPC_ISERIES */



#define ENABLE_INTS				\

	ld	r12,_MSR(r1);			\
@@ -399,8 +419,6 @@ END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES); \ 
	rlwimi	r11,r12,0,MSR_EE;		\

	mtmsrd	r11,1



#endif



#define STD_EXCEPTION_COMMON(trap, label, hdlr)		\

	.align	7;					\

	.globl label##_common;				\
@@ -541,11 +559,11 @@ instruction_access_slb_pSeries: 
	mfspr	r12,SPRN_SRR1		/* and SRR1 */

	b	.slb_miss_realmode	/* Rel. branch works in real mode */



	STD_EXCEPTION_PSERIES(0x500, hardware_interrupt)

	MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt)

	STD_EXCEPTION_PSERIES(0x600, alignment)

	STD_EXCEPTION_PSERIES(0x700, program_check)

	STD_EXCEPTION_PSERIES(0x800, fp_unavailable)

	STD_EXCEPTION_PSERIES(0x900, decrementer)

	MASKABLE_EXCEPTION_PSERIES(0x900, decrementer)

	STD_EXCEPTION_PSERIES(0xa00, trap_0a)

	STD_EXCEPTION_PSERIES(0xb00, trap_0b)
@@ -597,7 +615,24 @@ system_call_pSeries: 
/*** pSeries interrupt support ***/



	/* moved from 0xf00 */

	STD_EXCEPTION_PSERIES(., performance_monitor)

	MASKABLE_EXCEPTION_PSERIES(., performance_monitor)



/*

 * An interrupt came in while soft-disabled; clear EE in SRR1,

 * clear paca->hard_enabled and return.

 */

masked_interrupt:

	stb	r10,PACAHARDIRQEN(r13)

	mtcrf	0x80,r9

	ld	r9,PACA_EXGEN+EX_R9(r13)

	mfspr	r10,SPRN_SRR1

	rldicl	r10,r10,48,1		/* clear MSR_EE */

	rotldi	r10,r10,16

	mtspr	SPRN_SRR1,r10

	ld	r10,PACA_EXGEN+EX_R10(r13)

	mfspr	r13,SPRN_SPRG1

	rfid

	b	.



	.align	7

_GLOBAL(do_stab_bolted_pSeries)
@@ -952,7 +987,8 @@ fast_exception_return: 
	REST_8GPRS(2, r1)



	mfmsr	r10

	clrrdi	r10,r10,2		/* clear RI (LE is 0 already) */

	rldicl	r10,r10,48,1		/* clear EE */

	rldicr	r10,r10,16,61		/* clear RI (LE is 0 already) */

	mtmsrd	r10,1



	mtspr	SPRN_SRR1,r12
@@ -1877,11 +1913,16 @@ _GLOBAL(__secondary_start) 
	/* enable MMU and jump to start_secondary */

	LOAD_REG_ADDR(r3, .start_secondary_prolog)

	LOAD_REG_IMMEDIATE(r4, MSR_KERNEL)

#ifdef DO_SOFT_DISABLE

#ifdef CONFIG_PPC_ISERIES

BEGIN_FW_FTR_SECTION

	ori	r4,r4,MSR_EE

END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)

#endif

BEGIN_FW_FTR_SECTION

	stb	r7,PACASOFTIRQEN(r13)

	stb	r7,PACAHARDIRQEN(r13)

END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)



	mtspr	SPRN_SRR0,r3

	mtspr	SPRN_SRR1,r4

	rfid
@@ -2019,15 +2060,18 @@ _STATIC(start_here_common) 


	/* Load up the kernel context */

5:

#ifdef DO_SOFT_DISABLE

BEGIN_FW_FTR_SECTION

	li	r5,0

	stb	r5,PACAPROCENABLED(r13)	/* Soft Disabled */

	stb	r5,PACASOFTIRQEN(r13)	/* Soft Disabled */

#ifdef CONFIG_PPC_ISERIES

BEGIN_FW_FTR_SECTION

	mfmsr	r5

	ori	r5,r5,MSR_EE		/* Hard Enabled */

	mtmsrd	r5

END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)

#endif

BEGIN_FW_FTR_SECTION

	stb	r5,PACAHARDIRQEN(r13)

END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)



	bl .start_kernel

arch/powerpc/kernel/idle_power4.S

+7 −1
Original line number Diff line number Diff line
@@ -30,6 +30,13 @@ END_FTR_SECTION_IFCLR(CPU_FTR_CAN_NAP) 
	beqlr



	/* Go to NAP now */

	mfmsr	r7

	rldicl	r0,r7,48,1

	rotldi	r0,r0,16

	mtmsrd	r0,1			/* hard-disable interrupts */

	li	r0,1

	stb	r0,PACASOFTIRQEN(r13)	/* we'll hard-enable shortly */

	stb	r0,PACAHARDIRQEN(r13)

BEGIN_FTR_SECTION

	DSSALL

	sync
@@ -38,7 +45,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 
	ld	r8,TI_LOCAL_FLAGS(r9)	/* set napping bit */

	ori	r8,r8,_TLF_NAPPING	/* so when we take an exception */

	std	r8,TI_LOCAL_FLAGS(r9)	/* it will return to our caller */

	mfmsr	r7

	ori	r7,r7,MSR_EE

	oris	r7,r7,MSR_POW@h

1:	sync

CRA Git | Maintained and supported by SUSTech CRA and CCSE