arm64: scs: use vmapped IRQ and SDEI shadow stacks (ac20ffbb) · Commits · 戴 / test

arch/arm64/kernel/Makefile

+0 −1

Original line number	Diff line number	Diff line
		@@ -58,7 +58,6 @@ obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
		obj-$(CONFIG_CRASH_CORE) += crash_core.o
		obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o
		obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o
		obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o
		obj-$(CONFIG_ARM64_MTE) += mte.o

		obj-y += vdso/ probes/

arch/arm64/kernel/entry.S

+3 −3

Original line number	Diff line number	Diff line
		@@ -441,7 +441,7 @@ SYM_CODE_END(__swpan_exit_el0)

		#ifdef CONFIG_SHADOW_CALL_STACK
		/* also switch to the irq shadow stack */
		adr_this_cpu scs_sp, irq_shadow_call_stack, x26
		ldr_this_cpu scs_sp, irq_shadow_call_stack_ptr, x26
		#endif

		9998:
		@@ -1097,9 +1097,9 @@ SYM_CODE_START(__sdei_asm_handler)
		#ifdef CONFIG_SHADOW_CALL_STACK
		/* Use a separate shadow call stack for normal and critical events */
		cbnz w4, 3f
		adr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_normal, tmp=x6
		ldr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_normal_ptr, tmp=x6
		b 4f
		3: adr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_critical, tmp=x6
		3: ldr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_critical_ptr, tmp=x6
		4:
		#endif

arch/arm64/kernel/irq.c

+21 −0

Original line number	Diff line number	Diff line
		@@ -17,6 +17,7 @@
		#include <linux/init.h>
		#include <linux/irqchip.h>
		#include <linux/kprobes.h>
		#include <linux/scs.h>
		#include <linux/seq_file.h>
		#include <linux/vmalloc.h>
		#include <asm/daifflags.h>
		@@ -27,6 +28,25 @@ DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts);

		DEFINE_PER_CPU(unsigned long *, irq_stack_ptr);


		DECLARE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);

		#ifdef CONFIG_SHADOW_CALL_STACK
		DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
		#endif

		static void init_irq_scs(void)
		{
		int cpu;

		if (!IS_ENABLED(CONFIG_SHADOW_CALL_STACK))
		return;

		for_each_possible_cpu(cpu)
		per_cpu(irq_shadow_call_stack_ptr, cpu) =
		scs_alloc(cpu_to_node(cpu));
		}

		#ifdef CONFIG_VMAP_STACK
		static void init_irq_stacks(void)
		{
		@@ -54,6 +74,7 @@ static void init_irq_stacks(void)
		void __init init_IRQ(void)
		{
		init_irq_stacks();
		init_irq_scs();
		irqchip_init();
		if (!handle_arch_irq)
		panic("No interrupt controller found.");

arch/arm64/kernel/scs.c

deleted100644 → 0

+0 −16

Original line number	Diff line number	Diff line
		// SPDX-License-Identifier: GPL-2.0
		/*
		* Shadow Call Stack support.
		*
		* Copyright (C) 2019 Google LLC
		*/

		#include <linux/percpu.h>
		#include <linux/scs.h>

		DEFINE_SCS(irq_shadow_call_stack);

		#ifdef CONFIG_ARM_SDE_INTERFACE
		DEFINE_SCS(sdei_shadow_call_stack_normal);
		DEFINE_SCS(sdei_shadow_call_stack_critical);
		#endif

arch/arm64/kernel/sdei.c

+70 −0

Original line number	Diff line number	Diff line
		@@ -7,6 +7,7 @@
		#include <linux/hardirq.h>
		#include <linux/irqflags.h>
		#include <linux/sched/task_stack.h>
		#include <linux/scs.h>
		#include <linux/uaccess.h>

		#include <asm/alternative.h>
		@@ -37,6 +38,14 @@ DEFINE_PER_CPU(unsigned long *, sdei_stack_normal_ptr);
		DEFINE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
		#endif

		DECLARE_PER_CPU(unsigned long *, sdei_shadow_call_stack_normal_ptr);
		DECLARE_PER_CPU(unsigned long *, sdei_shadow_call_stack_critical_ptr);

		#ifdef CONFIG_SHADOW_CALL_STACK
		DEFINE_PER_CPU(unsigned long *, sdei_shadow_call_stack_normal_ptr);
		DEFINE_PER_CPU(unsigned long *, sdei_shadow_call_stack_critical_ptr);
		#endif

		static void _free_sdei_stack(unsigned long * __percpu *ptr, int cpu)
		{
		unsigned long *p;
		@@ -90,6 +99,59 @@ static int init_sdei_stacks(void)
		return err;
		}

		static void _free_sdei_scs(unsigned long * __percpu *ptr, int cpu)
		{
		void *s;

		s = per_cpu(*ptr, cpu);
		if (s) {
		per_cpu(*ptr, cpu) = NULL;
		scs_free(s);
		}
		}

		static void free_sdei_scs(void)
		{
		int cpu;

		for_each_possible_cpu(cpu) {
		_free_sdei_scs(&sdei_shadow_call_stack_normal_ptr, cpu);
		_free_sdei_scs(&sdei_shadow_call_stack_critical_ptr, cpu);
		}
		}

		static int _init_sdei_scs(unsigned long * __percpu *ptr, int cpu)
		{
		void *s;

		s = scs_alloc(cpu_to_node(cpu));
		if (!s)
		return -ENOMEM;
		per_cpu(*ptr, cpu) = s;

		return 0;
		}

		static int init_sdei_scs(void)
		{
		int cpu;
		int err = 0;

		for_each_possible_cpu(cpu) {
		err = _init_sdei_scs(&sdei_shadow_call_stack_normal_ptr, cpu);
		if (err)
		break;
		err = _init_sdei_scs(&sdei_shadow_call_stack_critical_ptr, cpu);
		if (err)
		break;
		}

		if (err)
		free_sdei_scs();

		return err;
		}

		static bool on_sdei_normal_stack(unsigned long sp, struct stack_info *info)
		{
		unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_normal_ptr);
		@@ -138,6 +200,14 @@ unsigned long sdei_arch_get_entry_point(int conduit)
		return 0;
		}

		if (IS_ENABLED(CONFIG_SHADOW_CALL_STACK)) {
		if (init_sdei_scs()) {
		if (IS_ENABLED(CONFIG_VMAP_STACK))
		free_sdei_stacks();
		return 0;
		}
		}

		sdei_exit_mode = (conduit == SMCCC_CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;

		#ifdef CONFIG_UNMAP_KERNEL_AT_EL0

Admin message