Commit 1c9a8e87 authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Will Deacon
Browse files

arm64: entry: simplify trampoline data page 



Get rid of some clunky open coded arithmetic on section addresses, by
emitting the trampoline data variables into a separate, dedicated r/o
data section, and putting it at the next page boundary. This way, we can
access the literals via single LDR instruction.

While at it, get rid of other, implicit literals, and use ADRP/ADD or
MOVZ/MOVK sequences, as appropriate. Note that the latter are only
supported for CONFIG_RELOCATABLE=n (which is usually the case if
CONFIG_RANDOMIZE_BASE=n), so update the CPP conditionals to reflect
this.

Acked-by: default avatarMark Rutland <mark.rutland@arm.com>
Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20220622161010.3845775-1-ardb@kernel.org


Signed-off-by: default avatarWill Deacon <will@kernel.org>
parent 47546a19

arch/arm64/include/asm/fixmap.h

+3 −1
Original line number Diff line number Diff line
@@ -62,10 +62,12 @@ enum fixed_addresses { 
#endif /* CONFIG_ACPI_APEI_GHES */



#ifdef CONFIG_UNMAP_KERNEL_AT_EL0

#ifdef CONFIG_RELOCATABLE

	FIX_ENTRY_TRAMP_TEXT4,	/* one extra slot for the data page */

#endif

	FIX_ENTRY_TRAMP_TEXT3,

	FIX_ENTRY_TRAMP_TEXT2,

	FIX_ENTRY_TRAMP_TEXT1,

	FIX_ENTRY_TRAMP_DATA,

#define TRAMP_VALIAS		(__fix_to_virt(FIX_ENTRY_TRAMP_TEXT1))

#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */

	__end_of_permanent_fixed_addresses,

arch/arm64/kernel/entry.S

+22 −31
Original line number Diff line number Diff line
@@ -636,18 +636,28 @@ alternative_else_nop_endif 
	 */

	.endm



	.macro tramp_data_page	dst

	adr_l	\dst, .entry.tramp.text

	sub	\dst, \dst, PAGE_SIZE

	.endm



	.macro		tramp_data_read_var	dst, var

#ifdef CONFIG_RANDOMIZE_BASE

	tramp_data_page		\dst

	add	\dst, \dst, #:lo12:__entry_tramp_data_\var

	ldr	\dst, [\dst]

#ifdef CONFIG_RELOCATABLE

	ldr		\dst, .L__tramp_data_\var

	.ifndef		.L__tramp_data_\var

	.pushsection	".entry.tramp.rodata", "a", %progbits

	.align		3

.L__tramp_data_\var:

	.quad		\var

	.popsection

	.endif

#else

	ldr	\dst, =\var

	/*

	 * As !RELOCATABLE implies !RANDOMIZE_BASE the address is always a

	 * compile time constant (and hence not secret and not worth hiding).

	 *

	 * As statically allocated kernel code and data always live in the top

	 * 47 bits of the address space we can sign-extend bit 47 and avoid an

	 * instruction to load the upper 16 bits (which must be 0xFFFF).

	 */

	movz		\dst, :abs_g2_s:\var

	movk		\dst, :abs_g1_nc:\var

	movk		\dst, :abs_g0_nc:\var

#endif

	.endm
@@ -695,7 +705,7 @@ alternative_else_nop_endif 
	msr	vbar_el1, x30

	isb

	.else

	ldr	x30, =vectors

	adr_l	x30, vectors

	.endif // \kpti == 1



	.if	\bhb == BHB_MITIGATION_FW
@@ -764,24 +774,7 @@ SYM_CODE_END(tramp_exit_native) 
SYM_CODE_START(tramp_exit_compat)

	tramp_exit	32

SYM_CODE_END(tramp_exit_compat)



	.ltorg

	.popsection				// .entry.tramp.text

#ifdef CONFIG_RANDOMIZE_BASE

	.pushsection ".rodata", "a"

	.align PAGE_SHIFT

SYM_DATA_START(__entry_tramp_data_start)

__entry_tramp_data_vectors:

	.quad	vectors

#ifdef CONFIG_ARM_SDE_INTERFACE

__entry_tramp_data___sdei_asm_handler:

	.quad	__sdei_asm_handler

#endif /* CONFIG_ARM_SDE_INTERFACE */

__entry_tramp_data_this_cpu_vector:

	.quad	this_cpu_vector

SYM_DATA_END(__entry_tramp_data_start)

	.popsection				// .rodata

#endif /* CONFIG_RANDOMIZE_BASE */

#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */



/*
@@ -932,7 +925,6 @@ NOKPROBE(call_on_irq_stack) 
 * This clobbers x4, __sdei_handler() will restore this from firmware's

 * copy.

 */

.ltorg

.pushsection ".entry.tramp.text", "ax"

SYM_CODE_START(__sdei_asm_entry_trampoline)

	mrs	x4, ttbr1_el1
@@ -967,7 +959,6 @@ SYM_CODE_START(__sdei_asm_exit_trampoline) 
1:	sdei_handler_exit exit_mode=x2

SYM_CODE_END(__sdei_asm_exit_trampoline)

NOKPROBE(__sdei_asm_exit_trampoline)

	.ltorg

.popsection		// .entry.tramp.text

#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */

arch/arm64/kernel/vmlinux.lds.S

+2 −1
Original line number Diff line number Diff line
@@ -115,7 +115,8 @@ jiffies = jiffies_64; 
	__entry_tramp_text_start = .;			\

	*(.entry.tramp.text)				\

	. = ALIGN(PAGE_SIZE);				\

	__entry_tramp_text_end = .;

	__entry_tramp_text_end = .;			\

	*(.entry.tramp.rodata)

#else

#define TRAMP_TEXT

#endif

arch/arm64/mm/mmu.c

+3 −7
Original line number Diff line number Diff line
@@ -672,13 +672,9 @@ static int __init map_entry_trampoline(void) 
		__set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i,

			     pa_start + i * PAGE_SIZE, prot);



	if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {

		extern char __entry_tramp_data_start[];



		__set_fixmap(FIX_ENTRY_TRAMP_DATA,

			     __pa_symbol(__entry_tramp_data_start),

			     PAGE_KERNEL_RO);

	}

	if (IS_ENABLED(CONFIG_RELOCATABLE))

		__set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i,

			     pa_start + i * PAGE_SIZE, PAGE_KERNEL_RO);



	return 0;

}