Commit 6c94f27a authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Catalin Marinas
Browse files

arm64: switch to relative exception tables 



Instead of using absolute addresses for both the exception location
and the fixup, use offsets relative to the exception table entry values.
Not only does this cut the size of the exception table in half, it is
also a prerequisite for KASLR, since absolute exception table entries
are subject to dynamic relocation, which is incompatible with the sorting
of the exception table that occurs at build time.

This patch also introduces the _ASM_EXTABLE preprocessor macro (which
exists on x86 as well) and its _asm_extable assembly counterpart, as
shorthands to emit exception table entries.

Acked-by: default avatarWill Deacon <will.deacon@arm.com>
Signed-off-by: default avatarArd Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent a272858a

arch/arm64/include/asm/alternative.h

+5 −14
Original line number Diff line number Diff line
@@ -157,11 +157,8 @@ void apply_alternatives(void *start, size_t length); 
			add	\addr, \addr, \post_inc;

		alternative_endif



		.section __ex_table,"a";

		.align	3;

		.quad	8888b,\l;

		.quad	8889b,\l;

		.previous;

		_asm_extable	8888b,\l;

		_asm_extable	8889b,\l;

	.endm



	.macro uao_stp l, reg1, reg2, addr, post_inc
@@ -175,11 +172,8 @@ void apply_alternatives(void *start, size_t length); 
			add	\addr, \addr, \post_inc;

		alternative_endif



		.section __ex_table,"a";

		.align	3;

		.quad	8888b,\l;

		.quad	8889b,\l;

		.previous

		_asm_extable	8888b,\l;

		_asm_extable	8889b,\l;

	.endm



	.macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
@@ -191,10 +185,7 @@ void apply_alternatives(void *start, size_t length); 
			add		\addr, \addr, \post_inc;

		alternative_endif



		.section __ex_table,"a";

		.align	3;

		.quad	8888b,\l;

		.previous

		_asm_extable	8888b,\l;

	.endm

#else

	.macro uao_ldp l, reg1, reg2, addr, post_inc

arch/arm64/include/asm/assembler.h

+11 −4
Original line number Diff line number Diff line
@@ -94,12 +94,19 @@ 
	dmb	\opt

	.endm



/*

 * Emit an entry into the exception table

 */

	.macro		_asm_extable, from, to

	.pushsection	__ex_table, "a"

	.align		3

	.long		(\from - .), (\to - .)

	.popsection

	.endm



#define USER(l, x...)				\

9999:	x;					\

	.section __ex_table,"a";		\

	.align	3;				\

	.quad	9999b,l;			\

	.previous

	_asm_extable	9999b, l



/*

 * Register aliases.

arch/arm64/include/asm/futex.h

+4 −8
Original line number Diff line number Diff line
@@ -42,10 +42,8 @@ 
"4:	mov	%w0, %w5\n"						\

"	b	3b\n"							\

"	.popsection\n"							\

"	.pushsection __ex_table,\"a\"\n"				\

"	.align	3\n"							\

"	.quad	1b, 4b, 2b, 4b\n"					\

"	.popsection\n"							\

	_ASM_EXTABLE(1b, 4b)						\

	_ASM_EXTABLE(2b, 4b)						\

	ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN,		\

		    CONFIG_ARM64_PAN)					\

	: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp)	\
@@ -134,10 +132,8 @@ ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, CONFIG_ARM64_PAN) 
"4:	mov	%w0, %w6\n"

"	b	3b\n"

"	.popsection\n"

"	.pushsection __ex_table,\"a\"\n"

"	.align	3\n"

"	.quad	1b, 4b, 2b, 4b\n"

"	.popsection\n"

	_ASM_EXTABLE(1b, 4b)

	_ASM_EXTABLE(2b, 4b)

ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, CONFIG_ARM64_PAN)

	: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp)

	: "r" (oldval), "r" (newval), "Ir" (-EFAULT)

arch/arm64/include/asm/uaccess.h

+16 −14
Original line number Diff line number Diff line
@@ -36,11 +36,11 @@ 
#define VERIFY_WRITE 1



/*

 * The exception table consists of pairs of addresses: the first is the

 * address of an instruction that is allowed to fault, and the second is

 * the address at which the program should continue.  No registers are

 * modified, so it is entirely up to the continuation code to figure out

 * what to do.

 * The exception table consists of pairs of relative offsets: the first

 * is the relative offset to an instruction that is allowed to fault,

 * and the second is the relative offset at which the program should

 * continue. No registers are modified, so it is entirely up to the

 * continuation code to figure out what to do.

 *

 * All the routines below use bits of fixup code that are out of line

 * with the main instruction path.  This means when everything is well,
@@ -50,9 +50,11 @@ 


struct exception_table_entry

{

	unsigned long insn, fixup;

	int insn, fixup;

};



#define ARCH_HAS_RELATIVE_EXTABLE



extern int fixup_exception(struct pt_regs *regs);



#define KERNEL_DS	(-1UL)
@@ -115,6 +117,12 @@ static inline void set_fs(mm_segment_t fs) 
#define access_ok(type, addr, size)	__range_ok(addr, size)

#define user_addr_max			get_fs



#define _ASM_EXTABLE(from, to)						\

	"	.pushsection	__ex_table, \"a\"\n"			\

	"	.align		3\n"					\

	"	.long		(" #from " - .), (" #to " - .)\n"	\

	"	.popsection\n"



/*

 * The "__xxx" versions of the user access functions do not verify the address

 * space - it must have been done previously with a separate "access_ok()"
@@ -134,10 +142,7 @@ static inline void set_fs(mm_segment_t fs) 
	"	mov	%1, #0\n"					\

	"	b	2b\n"						\

	"	.previous\n"						\

	"	.section __ex_table,\"a\"\n"				\

	"	.align	3\n"						\

	"	.quad	1b, 3b\n"					\

	"	.previous"						\

	_ASM_EXTABLE(1b, 3b)						\

	: "+r" (err), "=&r" (x)						\

	: "r" (addr), "i" (-EFAULT))
@@ -206,10 +211,7 @@ do { \ 
	"3:	mov	%w0, %3\n"					\

	"	b	2b\n"						\

	"	.previous\n"						\

	"	.section __ex_table,\"a\"\n"				\

	"	.align	3\n"						\

	"	.quad	1b, 3b\n"					\

	"	.previous"						\

	_ASM_EXTABLE(1b, 3b)						\

	: "+r" (err)							\

	: "r" (x), "r" (addr), "i" (-EFAULT))

arch/arm64/include/asm/word-at-a-time.h

+3 −4
Original line number Diff line number Diff line
@@ -16,6 +16,8 @@ 
#ifndef __ASM_WORD_AT_A_TIME_H

#define __ASM_WORD_AT_A_TIME_H



#include <asm/uaccess.h>



#ifndef __AARCH64EB__



#include <linux/kernel.h>
@@ -81,10 +83,7 @@ static inline unsigned long load_unaligned_zeropad(const void *addr) 
#endif

	"	b	2b\n"

	"	.popsection\n"

	"	.pushsection __ex_table,\"a\"\n"

	"	.align	3\n"

	"	.quad	1b, 3b\n"

	"	.popsection"

	_ASM_EXTABLE(1b, 3b)

	: "=&r" (ret), "=&r" (offset)

	: "r" (addr), "Q" (*(unsigned long *)addr));

CRA Git | Maintained and supported by SUSTech CRA and CCSE