Commit a257e025 authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Will Deacon
Browse files

arm64/kernel: don't ban ADRP to work around Cortex-A53 erratum #843419 



Working around Cortex-A53 erratum #843419 involves special handling of
ADRP instructions that end up in the last two instruction slots of a
4k page, or whose output register gets overwritten without having been
read. (Note that the latter instruction sequence is never emitted by
a properly functioning compiler, which is why it is disregarded by the
handling of the same erratum in the bfd.ld linker which we rely on for
the core kernel)

Normally, this gets taken care of by the linker, which can spot such
sequences at final link time, and insert a veneer if the ADRP ends up
at a vulnerable offset. However, linux kernel modules are partially
linked ELF objects, and so there is no 'final link time' other than the
runtime loading of the module, at which time all the static relocations
are resolved.

For this reason, we have implemented the #843419 workaround for modules
by avoiding ADRP instructions altogether, by using the large C model,
and by passing -mpc-relative-literal-loads to recent versions of GCC
that may emit adrp/ldr pairs to perform literal loads. However, this
workaround forces us to keep literal data mixed with the instructions
in the executable .text segment, and literal data may inadvertently
turn into an exploitable speculative gadget depending on the relative
offsets of arbitrary symbols.

So let's reimplement this workaround in a way that allows us to switch
back to the small C model, and to drop the -mpc-relative-literal-loads
GCC switch, by patching affected ADRP instructions at runtime:
- ADRP instructions that do not appear at 4k relative offset 0xff8 or
  0xffc are ignored
- ADRP instructions that are within 1 MB of their target symbol are
  converted into ADR instructions
- remaining ADRP instructions are redirected via a veneer that performs
  the load using an unaffected movn/movk sequence.

Signed-off-by: default avatarArd Biesheuvel <ard.biesheuvel@linaro.org>
[will: tidied up ADRP -> ADR instruction patching.]
[will: use ULL suffix for 64-bit immediate]
Signed-off-by: default avatarWill Deacon <will.deacon@arm.com>
parent f2b9ba87

arch/arm64/Kconfig

+4 −7
Original line number Diff line number Diff line
@@ -456,12 +456,12 @@ config ARM64_ERRATUM_845719 
config ARM64_ERRATUM_843419

	bool "Cortex-A53: 843419: A load or store might access an incorrect address"

	default y

	select ARM64_MODULE_CMODEL_LARGE if MODULES

	select ARM64_MODULE_PLTS if MODULES

	help

	  This option links the kernel with '--fix-cortex-a53-843419' and

	  builds modules using the large memory model in order to avoid the use

	  of the ADRP instruction, which can cause a subsequent memory access

	  to use an incorrect address on Cortex-A53 parts up to r0p4.

	  enables PLT support to replace certain ADRP instructions, which can

	  cause subsequent memory accesses to use an incorrect address on

	  Cortex-A53 parts up to r0p4.



	  If unsure, say Y.
@@ -1105,9 +1105,6 @@ config ARM64_SVE 


	  To enable use of this extension on CPUs that implement it, say Y.



config ARM64_MODULE_CMODEL_LARGE

	bool



config ARM64_MODULE_PLTS

	bool

	select HAVE_MOD_ARCH_SPECIFIC

arch/arm64/Makefile

+0 −5
Original line number Diff line number Diff line
@@ -51,7 +51,6 @@ endif 


KBUILD_CFLAGS	+= -mgeneral-regs-only $(lseinstr) $(brokengasinst)

KBUILD_CFLAGS	+= -fno-asynchronous-unwind-tables

KBUILD_CFLAGS	+= $(call cc-option, -mpc-relative-literal-loads)

KBUILD_AFLAGS	+= $(lseinstr) $(brokengasinst)



KBUILD_CFLAGS	+= $(call cc-option,-mabi=lp64)
@@ -77,10 +76,6 @@ endif 


CHECKFLAGS	+= -D__aarch64__ -m64



ifeq ($(CONFIG_ARM64_MODULE_CMODEL_LARGE), y)

KBUILD_CFLAGS_MODULE	+= -mcmodel=large

endif



ifeq ($(CONFIG_ARM64_MODULE_PLTS),y)

KBUILD_LDFLAGS_MODULE	+= -T $(srctree)/arch/arm64/kernel/module.lds

endif

arch/arm64/include/asm/module.h

+2 −0
Original line number Diff line number Diff line
@@ -39,6 +39,8 @@ struct mod_arch_specific { 
u64 module_emit_plt_entry(struct module *mod, void *loc, const Elf64_Rela *rela,

			  Elf64_Sym *sym);



u64 module_emit_adrp_veneer(struct module *mod, void *loc, u64 val);



#ifdef CONFIG_RANDOMIZE_BASE

extern u64 module_alloc_base;

#else

arch/arm64/kernel/module-plts.c

+83 −3
Original line number Diff line number Diff line
@@ -42,6 +42,47 @@ u64 module_emit_plt_entry(struct module *mod, void *loc, const Elf64_Rela *rela, 
	return (u64)&plt[i];

}



#ifdef CONFIG_ARM64_ERRATUM_843419

u64 module_emit_adrp_veneer(struct module *mod, void *loc, u64 val)

{

	struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core :

							  &mod->arch.init;

	struct plt_entry *plt = (struct plt_entry *)pltsec->plt->sh_addr;

	int i = pltsec->plt_num_entries++;

	u32 mov0, mov1, mov2, br;

	int rd;



	if (WARN_ON(pltsec->plt_num_entries > pltsec->plt_max_entries))

		return 0;



	/* get the destination register of the ADRP instruction */

	rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD,

					  le32_to_cpup((__le32 *)loc));



	/* generate the veneer instructions */

	mov0 = aarch64_insn_gen_movewide(rd, (u16)~val, 0,

					 AARCH64_INSN_VARIANT_64BIT,

					 AARCH64_INSN_MOVEWIDE_INVERSE);

	mov1 = aarch64_insn_gen_movewide(rd, (u16)(val >> 16), 16,

					 AARCH64_INSN_VARIANT_64BIT,

					 AARCH64_INSN_MOVEWIDE_KEEP);

	mov2 = aarch64_insn_gen_movewide(rd, (u16)(val >> 32), 32,

					 AARCH64_INSN_VARIANT_64BIT,

					 AARCH64_INSN_MOVEWIDE_KEEP);

	br = aarch64_insn_gen_branch_imm((u64)&plt[i].br, (u64)loc + 4,

					 AARCH64_INSN_BRANCH_NOLINK);



	plt[i] = (struct plt_entry){

			cpu_to_le32(mov0),

			cpu_to_le32(mov1),

			cpu_to_le32(mov2),

			cpu_to_le32(br)

		};



	return (u64)&plt[i];

}

#endif



#define cmp_3way(a,b)	((a) < (b) ? -1 : (a) > (b))



static int cmp_rela(const void *a, const void *b)
@@ -69,16 +110,21 @@ static bool duplicate_rel(const Elf64_Rela *rela, int num) 
}



static unsigned int count_plts(Elf64_Sym *syms, Elf64_Rela *rela, int num,

			       Elf64_Word dstidx)

			       Elf64_Word dstidx, Elf_Shdr *dstsec)

{

	unsigned int ret = 0;

	Elf64_Sym *s;

	int i;



	for (i = 0; i < num; i++) {

		u64 min_align;



		switch (ELF64_R_TYPE(rela[i].r_info)) {

		case R_AARCH64_JUMP26:

		case R_AARCH64_CALL26:

			if (!IS_ENABLED(CONFIG_RANDOMIZE_BASE))

				break;



			/*

			 * We only have to consider branch targets that resolve

			 * to symbols that are defined in a different section.
@@ -110,6 +156,40 @@ static unsigned int count_plts(Elf64_Sym *syms, Elf64_Rela *rela, int num, 
			if (rela[i].r_addend != 0 || !duplicate_rel(rela, i))

				ret++;

			break;

		case R_AARCH64_ADR_PREL_PG_HI21_NC:

		case R_AARCH64_ADR_PREL_PG_HI21:

			if (!IS_ENABLED(CONFIG_ARM64_ERRATUM_843419))

				break;



			/*

			 * Determine the minimal safe alignment for this ADRP

			 * instruction: the section alignment at which it is

			 * guaranteed not to appear at a vulnerable offset.

			 *

			 * This comes down to finding the least significant zero

			 * bit in bits [11:3] of the section offset, and

			 * increasing the section's alignment so that the

			 * resulting address of this instruction is guaranteed

			 * to equal the offset in that particular bit (as well

			 * as all less signficant bits). This ensures that the

			 * address modulo 4 KB != 0xfff8 or 0xfffc (which would

			 * have all ones in bits [11:3])

			 */

			min_align = 2ULL << ffz(rela[i].r_offset | 0x7);



			/*

			 * Allocate veneer space for each ADRP that may appear

			 * at a vulnerable offset nonetheless. At relocation

			 * time, some of these will remain unused since some

			 * ADRP instructions can be patched to ADR instructions

			 * instead.

			 */

			if (min_align > SZ_4K)

				ret++;

			else

				dstsec->sh_addralign = max(dstsec->sh_addralign,

							   min_align);

			break;

		}

	}

	return ret;
@@ -167,10 +247,10 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, 


		if (strncmp(secstrings + dstsec->sh_name, ".init", 5) != 0)

			core_plts += count_plts(syms, rels, numrels,

						sechdrs[i].sh_info);

						sechdrs[i].sh_info, dstsec);

		else

			init_plts += count_plts(syms, rels, numrels,

						sechdrs[i].sh_info);

						sechdrs[i].sh_info, dstsec);

	}



	mod->arch.core.plt->sh_type = SHT_NOBITS;

arch/arm64/kernel/module.c

+30 −4
Original line number Diff line number Diff line
@@ -198,6 +198,33 @@ static int reloc_insn_imm(enum aarch64_reloc_op op, __le32 *place, u64 val, 
	return 0;

}



static int reloc_insn_adrp(struct module *mod, __le32 *place, u64 val)

{

	u32 insn;



	if (!IS_ENABLED(CONFIG_ARM64_ERRATUM_843419) ||

	    ((u64)place & 0xfff) < 0xff8)

		return reloc_insn_imm(RELOC_OP_PAGE, place, val, 12, 21,

				      AARCH64_INSN_IMM_ADR);



	/* patch ADRP to ADR if it is in range */

	if (!reloc_insn_imm(RELOC_OP_PREL, place, val & ~0xfff, 0, 21,

			    AARCH64_INSN_IMM_ADR)) {

		insn = le32_to_cpu(*place);

		insn &= ~BIT(31);

	} else {

		/* out of range for ADR -> emit a veneer */

		val = module_emit_adrp_veneer(mod, place, val & ~0xfff);

		if (!val)

			return -ENOEXEC;

		insn = aarch64_insn_gen_branch_imm((u64)place, val,

						   AARCH64_INSN_BRANCH_NOLINK);

	}



	*place = cpu_to_le32(insn);

	return 0;

}



int apply_relocate_add(Elf64_Shdr *sechdrs,

		       const char *strtab,

		       unsigned int symindex,
@@ -337,14 +364,13 @@ int apply_relocate_add(Elf64_Shdr *sechdrs, 
			ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 0, 21,

					     AARCH64_INSN_IMM_ADR);

			break;

#ifndef CONFIG_ARM64_ERRATUM_843419

		case R_AARCH64_ADR_PREL_PG_HI21_NC:

			overflow_check = false;

		case R_AARCH64_ADR_PREL_PG_HI21:

			ovf = reloc_insn_imm(RELOC_OP_PAGE, loc, val, 12, 21,

					     AARCH64_INSN_IMM_ADR);

			ovf = reloc_insn_adrp(me, loc, val);

			if (ovf && ovf != -ERANGE)

				return ovf;

			break;

#endif

		case R_AARCH64_ADD_ABS_LO12_NC:

		case R_AARCH64_LDST8_ABS_LO12_NC:

			overflow_check = false;

CRA Git | Maintained and supported by SUSTech CRA and CCSE