arm64: Add support for relocating the kernel with RELR relocations

LDFLAGS_vmlinux	+= $(call ld-option, -X,)

endif

ifeq ($(CONFIG_RELR),y)

LDFLAGS_vmlinux	+= --pack-dyn-relocs=relr

endif

# insure the checker run with the right endianness

CHECKFLAGS += $(if $(CONFIG_CPU_BIG_ENDIAN),-mbig-endian,-mlittle-endian)

	  the chance of application behavior change because of timing

	  differences. The counts are reported via debugfs.

# Select if the architecture has support for applying RELR relocations.

config ARCH_HAS_RELR

	bool

config RELR

	bool "Use RELR relocation packing"

	depends on ARCH_HAS_RELR && TOOLS_SUPPORT_RELR

	default y

	help

	  Store the kernel's dynamic relocations in the RELR relocation packing

	  format. Requires a compatible linker (LLD supports this feature), as

	  well as compatible NM and OBJCOPY utilities (llvm-nm and llvm-objcopy

	  are compatible).

source "kernel/gcov/Kconfig"

source "scripts/gcc-plugins/Kconfig"

config RELOCATABLE

	bool

	select ARCH_HAS_RELR

	help

	  This builds the kernel as a Position Independent Executable (PIE),

	  which retains all relocation metadata required to relocate the

	 *  x23        stext() .. start_kernel()  physical misalignment/KASLR offset

	 *  x28        __create_page_tables()     callee preserved temp register

	 *  x19/x20    __primary_switch()         callee preserved temp registers

	 *  x24        __primary_switch() .. relocate_kernel()

	 *                                        current RELR displacement

	 */

ENTRY(stext)

	bl	preserve_boot_args

0:	cmp	x9, x10

	b.hs	1f

	ldp	x11, x12, [x9], #24

	ldr	x13, [x9, #-8]

	cmp	w12, #R_AARCH64_RELATIVE

	ldp	x12, x13, [x9], #24

	ldr	x14, [x9, #-8]

	cmp	w13, #R_AARCH64_RELATIVE

	b.ne	0b

	add	x13, x13, x23			// relocate

	str	x13, [x11, x23]

	add	x14, x14, x23			// relocate

	str	x14, [x12, x23]

	b	0b

1:	ret

1:

#ifdef CONFIG_RELR

	/*

	 * Apply RELR relocations.

	 *

	 * RELR is a compressed format for storing relative relocations. The

	 * encoded sequence of entries looks like:

	 * [ AAAAAAAA BBBBBBB1 BBBBBBB1 ... AAAAAAAA BBBBBB1 ... ]

	 *

	 * i.e. start with an address, followed by any number of bitmaps. The

	 * address entry encodes 1 relocation. The subsequent bitmap entries

	 * encode up to 63 relocations each, at subsequent offsets following

	 * the last address entry.

	 *

	 * The bitmap entries must have 1 in the least significant bit. The

	 * assumption here is that an address cannot have 1 in lsb. Odd

	 * addresses are not supported. Any odd addresses are stored in the RELA

	 * section, which is handled above.

	 *

	 * Excluding the least significant bit in the bitmap, each non-zero

	 * bit in the bitmap represents a relocation to be applied to

	 * a corresponding machine word that follows the base address

	 * word. The second least significant bit represents the machine

	 * word immediately following the initial address, and each bit

	 * that follows represents the next word, in linear order. As such,

	 * a single bitmap can encode up to 63 relocations in a 64-bit object.

	 *

	 * In this implementation we store the address of the next RELR table

	 * entry in x9, the address being relocated by the current address or

	 * bitmap entry in x13 and the address being relocated by the current

	 * bit in x14.

	 *

	 * Because addends are stored in place in the binary, RELR relocations

	 * cannot be applied idempotently. We use x24 to keep track of the

	 * currently applied displacement so that we can correctly relocate if

	 * __relocate_kernel is called twice with non-zero displacements (i.e.

	 * if there is both a physical misalignment and a KASLR displacement).

	 */

	ldr	w9, =__relr_offset		// offset to reloc table

	ldr	w10, =__relr_size		// size of reloc table

	add	x9, x9, x11			// __va(.relr)

	add	x10, x9, x10			// __va(.relr) + sizeof(.relr)

	sub	x15, x23, x24			// delta from previous offset

	cbz	x15, 7f				// nothing to do if unchanged

	mov	x24, x23			// save new offset

2:	cmp	x9, x10

	b.hs	7f

	ldr	x11, [x9], #8

	tbnz	x11, #0, 3f			// branch to handle bitmaps

	add	x13, x11, x23

	ldr	x12, [x13]			// relocate address entry

	add	x12, x12, x15

	str	x12, [x13], #8			// adjust to start of bitmap

	b	2b

3:	mov	x14, x13

4:	lsr	x11, x11, #1

	cbz	x11, 6f

	tbz	x11, #0, 5f			// skip bit if not set

	ldr	x12, [x14]			// relocate bit

	add	x12, x12, x15

	str	x12, [x14]

5:	add	x14, x14, #8			// move to next bit's address

	b	4b

6:	/*

	 * Move to the next bitmap's address. 8 is the word size, and 63 is the

	 * number of significant bits in a bitmap entry.

	 */

	add	x13, x13, #(8 * 63)

	b	2b

7:

#endif

	ret

ENDPROC(__relocate_kernel)

#endif

	adrp	x1, init_pg_dir

	bl	__enable_mmu

#ifdef CONFIG_RELOCATABLE

#ifdef CONFIG_RELR

	mov	x24, #0				// no RELR displacement yet

#endif

	bl	__relocate_kernel

#ifdef CONFIG_RANDOMIZE_BASE

	ldr	x8, =__primary_switched

	__rela_offset	= ABSOLUTE(ADDR(.rela.dyn) - KIMAGE_VADDR);

	__rela_size	= SIZEOF(.rela.dyn);

#ifdef CONFIG_RELR

	.relr.dyn : ALIGN(8) {

		*(.relr.dyn)

	}

	__relr_offset	= ABSOLUTE(ADDR(.relr.dyn) - KIMAGE_VADDR);

	__relr_size	= SIZEOF(.relr.dyn);

#endif

	. = ALIGN(SEGMENT_ALIGN);

	__initdata_end = .;

	__init_end = .;