Commit da192676 authored by Mark Rutland's avatar Mark Rutland Committed by Will Deacon
Browse files

arm64: entry: mark entry code as noinstr 



Functions in entry-common.c are marked as notrace and NOKPROBE_SYMBOL(),
but they're still subject to other instrumentation which may rely on
lockdep/rcu/context-tracking being up-to-date, and may cause nested
exceptions (e.g. for WARN/BUG or KASAN's use of BRK) which will corrupt
exceptions registers which have not yet been read.

Prevent this by marking all functions in entry-common.c as noinstr to
prevent compiler instrumentation. This also blacklists the functions for
tracing and kprobes, so we don't need to handle that separately.
Functions elsewhere will be dealt with in subsequent patches.

Signed-off-by: default avatarMark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20201130115950.22492-4-mark.rutland@arm.com


Signed-off-by: default avatarWill Deacon <will@kernel.org>
parent 114e0a68

arch/arm64/kernel/entry-common.c

+25 −50
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ 
#include <asm/mmu.h>

#include <asm/sysreg.h>



static void notrace el1_abort(struct pt_regs *regs, unsigned long esr)

static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)

{

	unsigned long far = read_sysreg(far_el1);
@@ -25,32 +25,28 @@ static void notrace el1_abort(struct pt_regs *regs, unsigned long esr) 
	far = untagged_addr(far);

	do_mem_abort(far, esr, regs);

}

NOKPROBE_SYMBOL(el1_abort);



static void notrace el1_pc(struct pt_regs *regs, unsigned long esr)

static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)

{

	unsigned long far = read_sysreg(far_el1);



	local_daif_inherit(regs);

	do_sp_pc_abort(far, esr, regs);

}

NOKPROBE_SYMBOL(el1_pc);



static void notrace el1_undef(struct pt_regs *regs)

static void noinstr el1_undef(struct pt_regs *regs)

{

	local_daif_inherit(regs);

	do_undefinstr(regs);

}

NOKPROBE_SYMBOL(el1_undef);



static void notrace el1_inv(struct pt_regs *regs, unsigned long esr)

static void noinstr el1_inv(struct pt_regs *regs, unsigned long esr)

{

	local_daif_inherit(regs);

	bad_mode(regs, 0, esr);

}

NOKPROBE_SYMBOL(el1_inv);



static void notrace el1_dbg(struct pt_regs *regs, unsigned long esr)

static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)

{

	unsigned long far = read_sysreg(far_el1);
@@ -64,16 +60,14 @@ static void notrace el1_dbg(struct pt_regs *regs, unsigned long esr) 


	do_debug_exception(far, esr, regs);

}

NOKPROBE_SYMBOL(el1_dbg);



static void notrace el1_fpac(struct pt_regs *regs, unsigned long esr)

static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)

{

	local_daif_inherit(regs);

	do_ptrauth_fault(regs, esr);

}

NOKPROBE_SYMBOL(el1_fpac);



asmlinkage void notrace el1_sync_handler(struct pt_regs *regs)

asmlinkage void noinstr el1_sync_handler(struct pt_regs *regs)

{

	unsigned long esr = read_sysreg(esr_el1);
@@ -106,9 +100,8 @@ asmlinkage void notrace el1_sync_handler(struct pt_regs *regs) 
		el1_inv(regs, esr);

	}

}

NOKPROBE_SYMBOL(el1_sync_handler);



static void notrace el0_da(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)

{

	unsigned long far = read_sysreg(far_el1);
@@ -117,9 +110,8 @@ static void notrace el0_da(struct pt_regs *regs, unsigned long esr) 
	far = untagged_addr(far);

	do_mem_abort(far, esr, regs);

}

NOKPROBE_SYMBOL(el0_da);



static void notrace el0_ia(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)

{

	unsigned long far = read_sysreg(far_el1);
@@ -135,41 +127,36 @@ static void notrace el0_ia(struct pt_regs *regs, unsigned long esr) 
	local_daif_restore(DAIF_PROCCTX);

	do_mem_abort(far, esr, regs);

}

NOKPROBE_SYMBOL(el0_ia);



static void notrace el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)

{

	user_exit_irqoff();

	local_daif_restore(DAIF_PROCCTX);

	do_fpsimd_acc(esr, regs);

}

NOKPROBE_SYMBOL(el0_fpsimd_acc);



static void notrace el0_sve_acc(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)

{

	user_exit_irqoff();

	local_daif_restore(DAIF_PROCCTX);

	do_sve_acc(esr, regs);

}

NOKPROBE_SYMBOL(el0_sve_acc);



static void notrace el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)

{

	user_exit_irqoff();

	local_daif_restore(DAIF_PROCCTX);

	do_fpsimd_exc(esr, regs);

}

NOKPROBE_SYMBOL(el0_fpsimd_exc);



static void notrace el0_sys(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)

{

	user_exit_irqoff();

	local_daif_restore(DAIF_PROCCTX);

	do_sysinstr(esr, regs);

}

NOKPROBE_SYMBOL(el0_sys);



static void notrace el0_pc(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)

{

	unsigned long far = read_sysreg(far_el1);
@@ -180,41 +167,36 @@ static void notrace el0_pc(struct pt_regs *regs, unsigned long esr) 
	local_daif_restore(DAIF_PROCCTX);

	do_sp_pc_abort(far, esr, regs);

}

NOKPROBE_SYMBOL(el0_pc);



static void notrace el0_sp(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)

{

	user_exit_irqoff();

	local_daif_restore(DAIF_PROCCTX);

	do_sp_pc_abort(regs->sp, esr, regs);

}

NOKPROBE_SYMBOL(el0_sp);



static void notrace el0_undef(struct pt_regs *regs)

static void noinstr el0_undef(struct pt_regs *regs)

{

	user_exit_irqoff();

	local_daif_restore(DAIF_PROCCTX);

	do_undefinstr(regs);

}

NOKPROBE_SYMBOL(el0_undef);



static void notrace el0_bti(struct pt_regs *regs)

static void noinstr el0_bti(struct pt_regs *regs)

{

	user_exit_irqoff();

	local_daif_restore(DAIF_PROCCTX);

	do_bti(regs);

}

NOKPROBE_SYMBOL(el0_bti);



static void notrace el0_inv(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)

{

	user_exit_irqoff();

	local_daif_restore(DAIF_PROCCTX);

	bad_el0_sync(regs, 0, esr);

}

NOKPROBE_SYMBOL(el0_inv);



static void notrace el0_dbg(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)

{

	/* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */

	unsigned long far = read_sysreg(far_el1);
@@ -226,26 +208,23 @@ static void notrace el0_dbg(struct pt_regs *regs, unsigned long esr) 
	do_debug_exception(far, esr, regs);

	local_daif_restore(DAIF_PROCCTX_NOIRQ);

}

NOKPROBE_SYMBOL(el0_dbg);



static void notrace el0_svc(struct pt_regs *regs)

static void noinstr el0_svc(struct pt_regs *regs)

{

	if (system_uses_irq_prio_masking())

		gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);



	do_el0_svc(regs);

}

NOKPROBE_SYMBOL(el0_svc);



static void notrace el0_fpac(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)

{

	user_exit_irqoff();

	local_daif_restore(DAIF_PROCCTX);

	do_ptrauth_fault(regs, esr);

}

NOKPROBE_SYMBOL(el0_fpac);



asmlinkage void notrace el0_sync_handler(struct pt_regs *regs)

asmlinkage void noinstr el0_sync_handler(struct pt_regs *regs)

{

	unsigned long esr = read_sysreg(esr_el1);
@@ -297,27 +276,24 @@ asmlinkage void notrace el0_sync_handler(struct pt_regs *regs) 
		el0_inv(regs, esr);

	}

}

NOKPROBE_SYMBOL(el0_sync_handler);



#ifdef CONFIG_COMPAT

static void notrace el0_cp15(struct pt_regs *regs, unsigned long esr)

static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)

{

	user_exit_irqoff();

	local_daif_restore(DAIF_PROCCTX);

	do_cp15instr(esr, regs);

}

NOKPROBE_SYMBOL(el0_cp15);



static void notrace el0_svc_compat(struct pt_regs *regs)

static void noinstr el0_svc_compat(struct pt_regs *regs)

{

	if (system_uses_irq_prio_masking())

		gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);



	do_el0_svc_compat(regs);

}

NOKPROBE_SYMBOL(el0_svc_compat);



asmlinkage void notrace el0_sync_compat_handler(struct pt_regs *regs)

asmlinkage void noinstr el0_sync_compat_handler(struct pt_regs *regs)

{

	unsigned long esr = read_sysreg(esr_el1);
@@ -360,5 +336,4 @@ asmlinkage void notrace el0_sync_compat_handler(struct pt_regs *regs) 
		el0_inv(regs, esr);

	}

}

NOKPROBE_SYMBOL(el0_sync_compat_handler);

#endif /* CONFIG_COMPAT */

CRA Git | Maintained and supported by SUSTech CRA and CCSE