Commit ed84b4e9 authored by Mark Rutland's avatar Mark Rutland
Browse files

arm64: move non-entry code out of .entry.text 



Currently, cpu_switch_to and ret_from_fork both live in .entry.text,
though neither form the critical path for an exception entry.

In subsequent patches, we will require that code in .entry.text is part
of the critical path for exception entry, for which we can assume
certain properties (e.g. the presence of exception regs on the stack).

Neither cpu_switch_to nor ret_from_fork will meet these requirements, so
we must move them out of .entry.text. To ensure that neither are kprobed
after being moved out of .entry.text, we must explicitly blacklist them,
requiring a new NOKPROBE() asm helper.

Signed-off-by: default avatarMark Rutland <mark.rutland@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
parent 2d0e751a

arch/arm64/include/asm/assembler.h

+11 −0
Original line number Diff line number Diff line
@@ -403,6 +403,17 @@ alternative_endif 
	.size	__pi_##x, . - x;	\

	ENDPROC(x)



/*

 * Annotate a function as being unsuitable for kprobes.

 */

#ifdef CONFIG_KPROBES

#define NOKPROBE(x)				\

	.pushsection "_kprobe_blacklist", "aw";	\

	.quad	x;				\

	.popsection;

#else

#define NOKPROBE(x)

#endif

	/*

	 * Emit a 64-bit absolute little endian symbol reference in a way that

	 * ensures that it will be resolved at build time, even when building a

arch/arm64/kernel/entry.S

+46 −44
Original line number Diff line number Diff line
@@ -710,38 +710,6 @@ el0_irq_naked: 
	b	ret_to_user

ENDPROC(el0_irq)



/*

 * Register switch for AArch64. The callee-saved registers need to be saved

 * and restored. On entry:

 *   x0 = previous task_struct (must be preserved across the switch)

 *   x1 = next task_struct

 * Previous and next are guaranteed not to be the same.

 *

 */

ENTRY(cpu_switch_to)

	mov	x10, #THREAD_CPU_CONTEXT

	add	x8, x0, x10

	mov	x9, sp

	stp	x19, x20, [x8], #16		// store callee-saved registers

	stp	x21, x22, [x8], #16

	stp	x23, x24, [x8], #16

	stp	x25, x26, [x8], #16

	stp	x27, x28, [x8], #16

	stp	x29, x9, [x8], #16

	str	lr, [x8]

	add	x8, x1, x10

	ldp	x19, x20, [x8], #16		// restore callee-saved registers

	ldp	x21, x22, [x8], #16

	ldp	x23, x24, [x8], #16

	ldp	x25, x26, [x8], #16

	ldp	x27, x28, [x8], #16

	ldp	x29, x9, [x8], #16

	ldr	lr, [x8]

	mov	sp, x9

	msr	sp_el0, x1

	ret

ENDPROC(cpu_switch_to)



/*

 * This is the fast syscall return path.  We do as little as possible here,

 * and this includes saving x0 back into the kernel stack.
@@ -784,18 +752,6 @@ finish_ret_to_user: 
	kernel_exit 0

ENDPROC(ret_to_user)



/*

 * This is how we return from a fork.

 */

ENTRY(ret_from_fork)

	bl	schedule_tail

	cbz	x19, 1f				// not a kernel thread

	mov	x0, x20

	blr	x19

1:	get_thread_info tsk

	b	ret_to_user

ENDPROC(ret_from_fork)



/*

 * SVC handler.

 */
@@ -869,3 +825,49 @@ ENTRY(sys_rt_sigreturn_wrapper) 
	mov	x0, sp

	b	sys_rt_sigreturn

ENDPROC(sys_rt_sigreturn_wrapper)



/*

 * Register switch for AArch64. The callee-saved registers need to be saved

 * and restored. On entry:

 *   x0 = previous task_struct (must be preserved across the switch)

 *   x1 = next task_struct

 * Previous and next are guaranteed not to be the same.

 *

 */

ENTRY(cpu_switch_to)

	mov	x10, #THREAD_CPU_CONTEXT

	add	x8, x0, x10

	mov	x9, sp

	stp	x19, x20, [x8], #16		// store callee-saved registers

	stp	x21, x22, [x8], #16

	stp	x23, x24, [x8], #16

	stp	x25, x26, [x8], #16

	stp	x27, x28, [x8], #16

	stp	x29, x9, [x8], #16

	str	lr, [x8]

	add	x8, x1, x10

	ldp	x19, x20, [x8], #16		// restore callee-saved registers

	ldp	x21, x22, [x8], #16

	ldp	x23, x24, [x8], #16

	ldp	x25, x26, [x8], #16

	ldp	x27, x28, [x8], #16

	ldp	x29, x9, [x8], #16

	ldr	lr, [x8]

	mov	sp, x9

	msr	sp_el0, x1

	ret

ENDPROC(cpu_switch_to)

NOKPROBE(cpu_switch_to)



/*

 * This is how we return from a fork.

 */

ENTRY(ret_from_fork)

	bl	schedule_tail

	cbz	x19, 1f				// not a kernel thread

	mov	x0, x20

	blr	x19

1:	get_thread_info tsk

	b	ret_to_user

ENDPROC(ret_from_fork)

NOKPROBE(ret_from_fork)

CRA Git | Maintained and supported by SUSTech CRA and CCSE