Commit 3282a3da authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Michael Ellerman
Browse files

powerpc/64: Implement soft interrupt replay in C 



When local_irq_enable() finds a pending soft-masked interrupt, it
"replays" it by setting up registers like the initial interrupt entry,
then calls into the low level handler to set up an interrupt stack
frame and process the interrupt.

This is not necessary, and uses more stack than needed. The high level
interrupt handler can be called directly from C, with just pt_regs set
up on stack. This should be faster and use less stack.

Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20200225173541.1549955-28-npiggin@gmail.com
parent 993c670a

arch/powerpc/include/asm/hw_irq.h

+0 −1
Original line number Diff line number Diff line
@@ -52,7 +52,6 @@ 
#ifndef __ASSEMBLY__



extern void replay_system_reset(void);

extern void __replay_interrupt(unsigned int vector);



extern void timer_interrupt(struct pt_regs *);

extern void timer_broadcast_interrupt(void);

arch/powerpc/kernel/exceptions-64e.S

+0 −32
Original line number Diff line number Diff line
@@ -1002,38 +1002,6 @@ masked_interrupt_book3e_0x280: 
masked_interrupt_book3e_0x2c0:

	masked_interrupt_book3e PACA_IRQ_DBELL 0



/*

 * Called from arch_local_irq_enable when an interrupt needs

 * to be resent. r3 contains either 0x500,0x900,0x260 or 0x280

 * to indicate the kind of interrupt. MSR:EE is already off.

 * We generate a stackframe like if a real interrupt had happened.

 *

 * Note: While MSR:EE is off, we need to make sure that _MSR

 * in the generated frame has EE set to 1 or the exception

 * handler will not properly re-enable them.

 */

_GLOBAL(__replay_interrupt)

	/* We are going to jump to the exception common code which

	 * will retrieve various register values from the PACA which

	 * we don't give a damn about.

	 */

	mflr	r10

	mfmsr	r11

	mfcr	r4

	mtspr	SPRN_SPRG_GEN_SCRATCH,r13;

	std	r1,PACA_EXGEN+EX_R1(r13);

	stw	r4,PACA_EXGEN+EX_CR(r13);

	ori	r11,r11,MSR_EE

	subi	r1,r1,INT_FRAME_SIZE;

	cmpwi	cr0,r3,0x500

	beq	exc_0x500_common

	cmpwi	cr0,r3,0x900

	beq	exc_0x900_common

	cmpwi	cr0,r3,0x280

	beq	exc_0x280_common

	blr





/*

 * This is called from 0x300 and 0x400 handlers after the prologs with

 * r14 and r15 containing the fault address and error code, with the

arch/powerpc/kernel/exceptions-64s.S

+0 −47
Original line number Diff line number Diff line
@@ -3165,50 +3165,3 @@ doorbell_super_common_msgclr: 
	LOAD_REG_IMMEDIATE(r3, PPC_DBELL_MSGTYPE << (63-36))

	PPC_MSGCLRP(3)

	b 	doorbell_super_common_virt
 


/*

 * Called from arch_local_irq_enable when an interrupt needs

 * to be resent. r3 contains 0x500, 0x900, 0xa00 or 0xe80 to indicate

 * which kind of interrupt. MSR:EE is already off. We generate a

 * stackframe like if a real interrupt had happened.

 *

 * Note: While MSR:EE is off, we need to make sure that _MSR

 * in the generated frame has EE set to 1 or the exception

 * handler will not properly re-enable them.

 *

 * Note that we don't specify LR as the NIP (return address) for

 * the interrupt because that would unbalance the return branch

 * predictor.

 */

_GLOBAL(__replay_interrupt)

	/* We are going to jump to the exception common code which

	 * will retrieve various register values from the PACA which

	 * we don't give a damn about, so we don't bother storing them.

	 */

	mfmsr	r12

	LOAD_REG_ADDR(r11, replay_interrupt_return)

	mfcr	r9

	ori	r12,r12,MSR_EE

	cmpwi	r3,0x900

	beq	decrementer_common_virt

	cmpwi	r3,0x500

BEGIN_FTR_SECTION

	beq	h_virt_irq_common_virt

FTR_SECTION_ELSE

	beq	hardware_interrupt_common_virt

ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_300)

	cmpwi	r3,0xf00

	beq	performance_monitor_common_virt

BEGIN_FTR_SECTION

	cmpwi	r3,0xa00

	beq	h_doorbell_common_msgclr

	cmpwi	r3,0xe60

	beq	hmi_exception_common_virt

FTR_SECTION_ELSE

	cmpwi	r3,0xa00

	beq	doorbell_super_common_msgclr

ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)

replay_interrupt_return:

	blr



_ASM_NOKPROBE_SYMBOL(__replay_interrupt)

arch/powerpc/kernel/irq.c

+130 −35
Original line number Diff line number Diff line
@@ -70,6 +70,7 @@ 
#include <asm/paca.h>

#include <asm/firmware.h>

#include <asm/lv1call.h>

#include <asm/dbell.h>

#endif

#define CREATE_TRACE_POINTS

#include <asm/trace.h>
@@ -230,10 +231,121 @@ notrace unsigned int __check_irq_replay(void) 
	return 0;

}



static void replay_soft_interrupts(void)

{

	/*

	 * We use local_paca rather than get_paca() to avoid all

	 * the debug_smp_processor_id() business in this low level

	 * function

	 */

	unsigned char happened = local_paca->irq_happened;

	struct pt_regs regs;



	ppc_save_regs(&regs);

	regs.softe = IRQS_ALL_DISABLED;



again:

	if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))

		WARN_ON_ONCE(mfmsr() & MSR_EE);



	if (happened & PACA_IRQ_HARD_DIS) {

		/*

		 * We may have missed a decrementer interrupt if hard disabled.

		 * Check the decrementer register in case we had a rollover

		 * while hard disabled.

		 */

		if (!(happened & PACA_IRQ_DEC)) {

			if (decrementer_check_overflow())

				happened |= PACA_IRQ_DEC;

		}

	}



	/*

	 * Force the delivery of pending soft-disabled interrupts on PS3.

	 * Any HV call will have this side effect.

	 */

	if (firmware_has_feature(FW_FEATURE_PS3_LV1)) {

		u64 tmp, tmp2;

		lv1_get_version_info(&tmp, &tmp2);

	}



	/*

	 * Check if an hypervisor Maintenance interrupt happened.

	 * This is a higher priority interrupt than the others, so

	 * replay it first.

	 */

	if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (happened & PACA_IRQ_HMI)) {

		local_paca->irq_happened &= ~PACA_IRQ_HMI;

		regs.trap = 0xe60;

		handle_hmi_exception(&regs);

		if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))

			hard_irq_disable();

	}



	if (happened & PACA_IRQ_DEC) {

		local_paca->irq_happened &= ~PACA_IRQ_DEC;

		regs.trap = 0x900;

		timer_interrupt(&regs);

		if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))

			hard_irq_disable();

	}



	if (happened & PACA_IRQ_EE) {

		local_paca->irq_happened &= ~PACA_IRQ_EE;

		regs.trap = 0x500;

		do_IRQ(&regs);

		if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))

			hard_irq_disable();

	}



	/*

	 * Check if an EPR external interrupt happened this bit is typically

	 * set if we need to handle another "edge" interrupt from within the

	 * MPIC "EPR" handler.

	 */

	if (IS_ENABLED(CONFIG_PPC_BOOK3E) && (happened & PACA_IRQ_EE_EDGE)) {

		local_paca->irq_happened &= ~PACA_IRQ_EE_EDGE;

		regs.trap = 0x500;

		do_IRQ(&regs);

		if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))

			hard_irq_disable();

	}



	if (IS_ENABLED(CONFIG_PPC_DOORBELL) && (happened & PACA_IRQ_DBELL)) {

		local_paca->irq_happened &= ~PACA_IRQ_DBELL;

		if (IS_ENABLED(CONFIG_PPC_BOOK3E))

			regs.trap = 0x280;

		else

			regs.trap = 0xa00;

		doorbell_exception(&regs);

		if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))

			hard_irq_disable();

	}



	/* Book3E does not support soft-masking PMI interrupts */

	if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (happened & PACA_IRQ_PMI)) {

		local_paca->irq_happened &= ~PACA_IRQ_PMI;

		regs.trap = 0xf00;

		performance_monitor_exception(&regs);

		if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))

			hard_irq_disable();

	}



	happened = local_paca->irq_happened;

	if (happened & ~PACA_IRQ_HARD_DIS) {

		/*

		 * We are responding to the next interrupt, so interrupt-off

		 * latencies should be reset here.

		 */

		trace_hardirqs_on();

		trace_hardirqs_off();

		goto again;

	}

}



notrace void arch_local_irq_restore(unsigned long mask)

{

	unsigned char irq_happened;

	unsigned int replay;



	/* Write the new soft-enabled value */

	irq_soft_mask_set(mask);
@@ -255,24 +367,16 @@ notrace void arch_local_irq_restore(unsigned long mask) 
	 */

	irq_happened = get_irq_happened();

	if (!irq_happened) {

#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG

		if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))

			WARN_ON_ONCE(!(mfmsr() & MSR_EE));

#endif

		return;

	}



	/*

	 * We need to hard disable to get a trusted value from

	 * __check_irq_replay(). We also need to soft-disable

	 * again to avoid warnings in there due to the use of

	 * per-cpu variables.

	 */

	/* We need to hard disable to replay. */

	if (!(irq_happened & PACA_IRQ_HARD_DIS)) {

#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG

		if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))

			WARN_ON_ONCE(!(mfmsr() & MSR_EE));

#endif

		__hard_irq_disable();

#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG

	} else {

		/*

		 * We should already be hard disabled here. We had bugs
@@ -280,35 +384,26 @@ notrace void arch_local_irq_restore(unsigned long mask) 
		 * warn if we are wrong. Only do that when IRQ tracing

		 * is enabled as mfmsr() can be costly.

		 */

		if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) {

			if (WARN_ON_ONCE(mfmsr() & MSR_EE))

				__hard_irq_disable();

#endif

		}



		if (irq_happened == PACA_IRQ_HARD_DIS) {

			local_paca->irq_happened = 0;

			__hard_irq_enable();

			return;

		}

	}



	irq_soft_mask_set(IRQS_ALL_DISABLED);

	trace_hardirqs_off();



	/*

	 * Check if anything needs to be re-emitted. We haven't

	 * soft-enabled yet to avoid warnings in decrementer_check_overflow

	 * accessing per-cpu variables

	 */

	replay = __check_irq_replay();

	replay_soft_interrupts();

	local_paca->irq_happened = 0;



	/* We can soft-enable now */

	trace_hardirqs_on();

	irq_soft_mask_set(IRQS_ENABLED);



	/*

	 * And replay if we have to. This will return with interrupts

	 * hard-enabled.

	 */

	if (replay) {

		__replay_interrupt(replay);

		return;

	}



	/* Finally, let's ensure we are hard enabled */

	__hard_irq_enable();

}

EXPORT_SYMBOL(arch_local_irq_restore);

CRA Git | Maintained and supported by SUSTech CRA and CCSE