Commit 2671828c authored by James Morse's avatar James Morse Committed by Will Deacon
Browse files

arm64: entry: Move ct_user_exit before any other exception 



When taking an SError or Debug exception from EL0, we run the C
handler for these exceptions before updating the context tracking
code and unmasking lower priority interrupts.

When booting with nohz_full lockdep tells us we got this wrong:
| =============================
| WARNING: suspicious RCU usage
| 5.3.0-rc2-00010-gb4b5e9dcb11b-dirty #11271 Not tainted
| -----------------------------
| include/linux/rcupdate.h:643 rcu_read_unlock() used illegally wh!
|
| other info that might help us debug this:
|
|
| RCU used illegally from idle CPU!
| rcu_scheduler_active = 2, debug_locks = 1
| RCU used illegally from extended quiescent state!
| 1 lock held by a.out/432:
|  #0: 00000000c7a79515 (rcu_read_lock){....}, at: brk_handler+0x00
|
| stack backtrace:
| CPU: 1 PID: 432 Comm: a.out Not tainted 5.3.0-rc2-00010-gb4b5e9d1
| Hardware name: ARM LTD ARM Juno Development Platform/ARM Juno De8
| Call trace:
|  dump_backtrace+0x0/0x140
|  show_stack+0x14/0x20
|  dump_stack+0xbc/0x104
|  lockdep_rcu_suspicious+0xf8/0x108
|  brk_handler+0x164/0x1b0
|  do_debug_exception+0x11c/0x278
|  el0_dbg+0x14/0x20

Moving the ct_user_exit calls to be before do_debug_exception() means
they are also before trace_hardirqs_off() has been updated. Add a new
ct_user_exit_irqoff macro to avoid the context-tracking code using
irqsave/restore before we've updated trace_hardirqs_off(). To be
consistent, do this everywhere.

The C helper is called enter_from_user_mode() to match x86 in the hope
we can merge them into kernel/context_tracking.c later.

Cc: Masami Hiramatsu <mhiramat@kernel.org>
Fixes: 6c81fe79 ("arm64: enable context tracking")
Signed-off-by: default avatarJames Morse <james.morse@arm.com>
Signed-off-by: default avatarWill Deacon <will@kernel.org>
parent 2d122942

arch/arm64/include/asm/exception.h

+2 −0
Original line number Diff line number Diff line
@@ -30,4 +30,6 @@ static inline u32 disr_to_esr(u64 disr) 
	return esr;

}



asmlinkage void enter_from_user_mode(void);



#endif	/* __ASM_EXCEPTION_H */

arch/arm64/kernel/entry.S

+19 −17
Original line number Diff line number Diff line
@@ -30,9 +30,9 @@ 
 * Context tracking subsystem.  Used to instrument transitions

 * between user and kernel mode.

 */

	.macro ct_user_exit

	.macro ct_user_exit_irqoff

#ifdef CONFIG_CONTEXT_TRACKING

	bl	context_tracking_user_exit

	bl	enter_from_user_mode

#endif

	.endm
@@ -792,8 +792,8 @@ el0_cp15: 
	/*

	 * Trapped CP15 (MRC, MCR, MRRC, MCRR) instructions

	 */

	ct_user_exit_irqoff

	enable_daif

	ct_user_exit

	mov	x0, x25

	mov	x1, sp

	bl	do_cp15instr
@@ -805,8 +805,8 @@ el0_da: 
	 * Data abort handling

	 */

	mrs	x26, far_el1

	ct_user_exit_irqoff

	enable_daif

	ct_user_exit

	clear_address_tag x0, x26

	mov	x1, x25

	mov	x2, sp
@@ -818,11 +818,11 @@ el0_ia: 
	 */

	mrs	x26, far_el1

	gic_prio_kentry_setup tmp=x0

	ct_user_exit_irqoff

	enable_da_f

#ifdef CONFIG_TRACE_IRQFLAGS

	bl	trace_hardirqs_off

#endif

	ct_user_exit

	mov	x0, x26

	mov	x1, x25

	mov	x2, sp
@@ -832,8 +832,8 @@ el0_fpsimd_acc: 
	/*

	 * Floating Point or Advanced SIMD access

	 */

	ct_user_exit_irqoff

	enable_daif

	ct_user_exit

	mov	x0, x25

	mov	x1, sp

	bl	do_fpsimd_acc
@@ -842,8 +842,8 @@ el0_sve_acc: 
	/*

	 * Scalable Vector Extension access

	 */

	ct_user_exit_irqoff

	enable_daif

	ct_user_exit

	mov	x0, x25

	mov	x1, sp

	bl	do_sve_acc
@@ -852,8 +852,8 @@ el0_fpsimd_exc: 
	/*

	 * Floating Point, Advanced SIMD or SVE exception

	 */

	ct_user_exit_irqoff

	enable_daif

	ct_user_exit

	mov	x0, x25

	mov	x1, sp

	bl	do_fpsimd_exc
@@ -868,11 +868,11 @@ el0_sp_pc: 
	 * Stack or PC alignment exception handling

	 */

	gic_prio_kentry_setup tmp=x0

	ct_user_exit_irqoff

	enable_da_f

#ifdef CONFIG_TRACE_IRQFLAGS

	bl	trace_hardirqs_off

#endif

	ct_user_exit

	mov	x0, x26

	mov	x1, x25

	mov	x2, sp
@@ -882,8 +882,8 @@ el0_undef: 
	/*

	 * Undefined instruction

	 */

	ct_user_exit_irqoff

	enable_daif

	ct_user_exit

	mov	x0, sp

	bl	do_undefinstr

	b	ret_to_user
@@ -891,8 +891,8 @@ el0_sys: 
	/*

	 * System instructions, for trapped cache maintenance instructions

	 */

	ct_user_exit_irqoff

	enable_daif

	ct_user_exit

	mov	x0, x25

	mov	x1, sp

	bl	do_sysinstr
@@ -902,17 +902,18 @@ el0_dbg: 
	 * Debug exception handling

	 */

	tbnz	x24, #0, el0_inv		// EL0 only

	mrs	x24, far_el1

	gic_prio_kentry_setup tmp=x3

	mrs	x0, far_el1

	ct_user_exit_irqoff

	mov	x0, x24

	mov	x1, x25

	mov	x2, sp

	bl	do_debug_exception

	enable_da_f

	ct_user_exit

	b	ret_to_user

el0_inv:

	ct_user_exit_irqoff

	enable_daif

	ct_user_exit

	mov	x0, sp

	mov	x1, #BAD_SYNC

	mov	x2, x25
@@ -925,13 +926,13 @@ el0_irq: 
	kernel_entry 0

el0_irq_naked:

	gic_prio_irq_setup pmr=x20, tmp=x0

	ct_user_exit_irqoff

	enable_da_f



#ifdef CONFIG_TRACE_IRQFLAGS

	bl	trace_hardirqs_off

#endif



	ct_user_exit

#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR

	tbz	x22, #55, 1f

	bl	do_el0_irq_bp_hardening
@@ -958,13 +959,14 @@ ENDPROC(el1_error) 
el0_error:

	kernel_entry 0

el0_error_naked:

	mrs	x1, esr_el1

	mrs	x25, esr_el1

	gic_prio_kentry_setup tmp=x2

	ct_user_exit_irqoff

	enable_dbg

	mov	x0, sp

	mov	x1, x25

	bl	do_serror

	enable_da_f

	ct_user_exit

	b	ret_to_user

ENDPROC(el0_error)

arch/arm64/kernel/traps.c

+9 −0
Original line number Diff line number Diff line
@@ -7,9 +7,11 @@ 
 */



#include <linux/bug.h>

#include <linux/context_tracking.h>

#include <linux/signal.h>

#include <linux/personality.h>

#include <linux/kallsyms.h>

#include <linux/kprobes.h>

#include <linux/spinlock.h>

#include <linux/uaccess.h>

#include <linux/hardirq.h>
@@ -900,6 +902,13 @@ asmlinkage void do_serror(struct pt_regs *regs, unsigned int esr) 
		nmi_exit();

}



asmlinkage void enter_from_user_mode(void)

{

	CT_WARN_ON(ct_state() != CONTEXT_USER);

	user_exit_irqoff();

}

NOKPROBE_SYMBOL(enter_from_user_mode);



void __pte_error(const char *file, int line, unsigned long val)

{

	pr_err("%s:%d: bad pte %016lx.\n", file, line, val);

CRA Git | Maintained and supported by SUSTech CRA and CCSE