Commit f5df2696 authored by James Morse's avatar James Morse Committed by Catalin Marinas
Browse files

arm64: kernel: Add arch-specific SDEI entry code and CPU masking 



The Software Delegated Exception Interface (SDEI) is an ARM standard
for registering callbacks from the platform firmware into the OS.
This is typically used to implement RAS notifications.

Such notifications enter the kernel at the registered entry-point
with the register values of the interrupted CPU context. Because this
is not a CPU exception, it cannot reuse the existing entry code.
(crucially we don't implicitly know which exception level we interrupted),

Add the entry point to entry.S to set us up for calling into C code. If
the event interrupted code that had interrupts masked, we always return
to that location. Otherwise we pretend this was an IRQ, and use SDEI's
complete_and_resume call to return to vbar_el1 + offset.

This allows the kernel to deliver signals to user space processes. For
KVM this triggers the world switch, a quick spin round vcpu_run, then
back into the guest, unless there are pending signals.

Add sdei_mask_local_cpu() calls to the smp_send_stop() code, this covers
the panic() code-path, which doesn't invoke cpuhotplug notifiers.

Because we can interrupt entry-from/exit-to another EL, we can't trust the
value in sp_el0 or x29, even if we interrupted the kernel, in this case
the code in entry.S will save/restore sp_el0 and use the value in
__entry_task.

When we have VMAP stacks we can interrupt the stack-overflow test, which
stirs x0 into sp, meaning we have to have our own VMAP stacks. For now
these are allocated when we probe the interface. Future patches will add
refcounting hooks to allow the arch code to allocate them lazily.

Signed-off-by: default avatarJames Morse <james.morse@arm.com>
Reviewed-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent e1281f56

arch/arm64/include/asm/sdei.h

+45 −2
Original line number Diff line number Diff line
@@ -3,6 +3,49 @@ 
#ifndef __ASM_SDEI_H

#define __ASM_SDEI_H



/* Later patches add the arch specific bits */

/* Values for sdei_exit_mode */

#define SDEI_EXIT_HVC  0

#define SDEI_EXIT_SMC  1



#define SDEI_STACK_SIZE		IRQ_STACK_SIZE



#ifndef __ASSEMBLY__



#include <linux/linkage.h>

#include <linux/preempt.h>

#include <linux/types.h>



#include <asm/virt.h>



extern unsigned long sdei_exit_mode;



/* Software Delegated Exception entry point from firmware*/

asmlinkage void __sdei_asm_handler(unsigned long event_num, unsigned long arg,

				   unsigned long pc, unsigned long pstate);



/*

 * The above entry point does the minimum to call C code. This function does

 * anything else, before calling the driver.

 */

struct sdei_registered_event;

asmlinkage unsigned long __sdei_handler(struct pt_regs *regs,

					struct sdei_registered_event *arg);



unsigned long sdei_arch_get_entry_point(int conduit);

#define sdei_arch_get_entry_point(x)	sdei_arch_get_entry_point(x)



bool _on_sdei_stack(unsigned long sp);

static inline bool on_sdei_stack(unsigned long sp)

{

	if (!IS_ENABLED(CONFIG_VMAP_STACK))

		return false;

	if (!IS_ENABLED(CONFIG_ARM_SDE_INTERFACE))

		return false;

	if (in_nmi())

		return _on_sdei_stack(sp);



	return false;

}



#endif /* __ASSEMBLY__ */

#endif	/* __ASM_SDEI_H */

arch/arm64/include/asm/stacktrace.h

+3 −0
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ 


#include <asm/memory.h>

#include <asm/ptrace.h>

#include <asm/sdei.h>



struct stackframe {

	unsigned long fp;
@@ -85,6 +86,8 @@ static inline bool on_accessible_stack(struct task_struct *tsk, unsigned long sp 
		return true;

	if (on_overflow_stack(sp))

		return true;

	if (on_sdei_stack(sp))

		return true;



	return false;

}

arch/arm64/kernel/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -52,6 +52,7 @@ arm64-obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o \ 
arm64-obj-$(CONFIG_ARM64_RELOC_TEST)	+= arm64-reloc-test.o

arm64-reloc-test-y := reloc_test_core.o reloc_test_syms.o

arm64-obj-$(CONFIG_CRASH_DUMP)		+= crash_dump.o

arm64-obj-$(CONFIG_ARM_SDE_INTERFACE)	+= sdei.o



ifeq ($(CONFIG_KVM),y)

arm64-obj-$(CONFIG_HARDEN_BRANCH_PREDICTOR)	+= bpi.o

arch/arm64/kernel/asm-offsets.c

+5 −0
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ 
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.

 */



#include <linux/arm_sdei.h>

#include <linux/sched.h>

#include <linux/mm.h>

#include <linux/dma-mapping.h>
@@ -157,6 +158,10 @@ int main(void) 
  BLANK();

#ifdef CONFIG_UNMAP_KERNEL_AT_EL0

  DEFINE(TRAMP_VALIAS,		TRAMP_VALIAS);

#endif

#ifdef CONFIG_ARM_SDE_INTERFACE

  DEFINE(SDEI_EVENT_INTREGS,	offsetof(struct sdei_registered_event, interrupted_regs));

  DEFINE(SDEI_EVENT_PRIORITY,	offsetof(struct sdei_registered_event, priority));

#endif

  return 0;

}

arch/arm64/kernel/entry.S

+101 −0
Original line number Diff line number Diff line
@@ -1153,3 +1153,104 @@ ENTRY(ret_from_fork) 
	b	ret_to_user

ENDPROC(ret_from_fork)

NOKPROBE(ret_from_fork)



#ifdef CONFIG_ARM_SDE_INTERFACE



#include <asm/sdei.h>

#include <uapi/linux/arm_sdei.h>



/*

 * Software Delegated Exception entry point.

 *

 * x0: Event number

 * x1: struct sdei_registered_event argument from registration time.

 * x2: interrupted PC

 * x3: interrupted PSTATE

 *

 * Firmware has preserved x0->x17 for us, we must save/restore the rest to

 * follow SMC-CC. We save (or retrieve) all the registers as the handler may

 * want them.

 */

ENTRY(__sdei_asm_handler)

	stp     x2, x3, [x1, #SDEI_EVENT_INTREGS + S_PC]

	stp     x4, x5, [x1, #SDEI_EVENT_INTREGS + 16 * 2]

	stp     x6, x7, [x1, #SDEI_EVENT_INTREGS + 16 * 3]

	stp     x8, x9, [x1, #SDEI_EVENT_INTREGS + 16 * 4]

	stp     x10, x11, [x1, #SDEI_EVENT_INTREGS + 16 * 5]

	stp     x12, x13, [x1, #SDEI_EVENT_INTREGS + 16 * 6]

	stp     x14, x15, [x1, #SDEI_EVENT_INTREGS + 16 * 7]

	stp     x16, x17, [x1, #SDEI_EVENT_INTREGS + 16 * 8]

	stp     x18, x19, [x1, #SDEI_EVENT_INTREGS + 16 * 9]

	stp     x20, x21, [x1, #SDEI_EVENT_INTREGS + 16 * 10]

	stp     x22, x23, [x1, #SDEI_EVENT_INTREGS + 16 * 11]

	stp     x24, x25, [x1, #SDEI_EVENT_INTREGS + 16 * 12]

	stp     x26, x27, [x1, #SDEI_EVENT_INTREGS + 16 * 13]

	stp     x28, x29, [x1, #SDEI_EVENT_INTREGS + 16 * 14]

	mov	x4, sp

	stp     lr, x4, [x1, #SDEI_EVENT_INTREGS + S_LR]



	mov	x19, x1



#ifdef CONFIG_VMAP_STACK

	/*

	 * entry.S may have been using sp as a scratch register, find whether

	 * this is a normal or critical event and switch to the appropriate

	 * stack for this CPU.

	 */

	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]

	cbnz	w4, 1f

	ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6

	b	2f

1:	ldr_this_cpu dst=x5, sym=sdei_stack_critical_ptr, tmp=x6

2:	mov	x6, #SDEI_STACK_SIZE

	add	x5, x5, x6

	mov	sp, x5

#endif



	/*

	 * We may have interrupted userspace, or a guest, or exit-from or

	 * return-to either of these. We can't trust sp_el0, restore it.

	 */

	mrs	x28, sp_el0

	ldr_this_cpu	dst=x0, sym=__entry_task, tmp=x1

	msr	sp_el0, x0



	/* If we interrupted the kernel point to the previous stack/frame. */

	and     x0, x3, #0xc

	mrs     x1, CurrentEL

	cmp     x0, x1

	csel	x29, x29, xzr, eq	// fp, or zero

	csel	x4, x2, xzr, eq		// elr, or zero



	stp	x29, x4, [sp, #-16]!

	mov	x29, sp



	add	x0, x19, #SDEI_EVENT_INTREGS

	mov	x1, x19

	bl	__sdei_handler



	msr	sp_el0, x28

	/* restore regs >x17 that we clobbered */

	ldp     x28, x29, [x19, #SDEI_EVENT_INTREGS + 16 * 14]

	ldp     lr, x4, [x19, #SDEI_EVENT_INTREGS + S_LR]

	mov	sp, x4

	ldp     x18, x19, [x19, #SDEI_EVENT_INTREGS + 16 * 9]



	mov	x1, x0			// address to complete_and_resume

	/* x0 = (x0 <= 1) ? EVENT_COMPLETE:EVENT_COMPLETE_AND_RESUME */

	cmp	x0, #1

	mov_q	x2, SDEI_1_0_FN_SDEI_EVENT_COMPLETE

	mov_q	x3, SDEI_1_0_FN_SDEI_EVENT_COMPLETE_AND_RESUME

	csel	x0, x2, x3, ls



	/* On success, this call never returns... */

	ldr_l	x2, sdei_exit_mode

	cmp	x2, #SDEI_EXIT_SMC

	b.ne	1f

	smc	#0

	b	.

1:	hvc	#0

	b	.

ENDPROC(__sdei_asm_handler)

NOKPROBE(__sdei_asm_handler)

#endif /* CONFIG_ARM_SDE_INTERFACE */

CRA Git | Maintained and supported by SUSTech CRA and CCSE