Commit 211ceca3 authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Catalin Marinas
Browse files

arm64: entry: Simplify tramp_alias macro and tramp_exit routine 



The tramp_alias macro constructs the virtual alias of a symbol in the
trampoline text mapping, based on its kernel text address, and does so
in a way that is more convoluted than necessary. So let's simplify that.

Also, now that the address of the vector table is kept in a per-CPU
variable, there is no need to defer the load and the assignment of
VBAR_EL1 to tramp_exit(). This means we can use a PC-relative reference
to the per-CPU variable instead of storing its absolute address in a
global variable in the trampoline rodata.

And given that tramp_alias no longer needs a temp register, this means
we can restore X30 earlier as well, and only leave X29 for tramp_exit()
to restore.

While at it, give some related symbols static linkage, considering that
they are only referenced from the object file that defines them.

Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Reviewed-by: default avatarWill Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20230418143604.1176437-3-ardb@kernel.org


Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 0936243c

arch/arm64/kernel/entry.S

+20 −27
Original line number Diff line number Diff line
@@ -101,12 +101,11 @@ 
.org .Lventry_start\@ + 128	// Did we overflow the ventry slot?

	.endm



	.macro tramp_alias, dst, sym, tmp

	mov_q	\dst, TRAMP_VALIAS

	adr_l	\tmp, \sym

	add	\dst, \dst, \tmp

	adr_l	\tmp, .entry.tramp.text

	sub	\dst, \dst, \tmp

	.macro	tramp_alias, dst, sym

	.set	.Lalias\@, TRAMP_VALIAS + \sym - .entry.tramp.text

	movz	\dst, :abs_g2_s:.Lalias\@

	movk	\dst, :abs_g1_nc:.Lalias\@

	movk	\dst, :abs_g0_nc:.Lalias\@

	.endm



	/*
@@ -436,8 +435,13 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 
alternative_else_nop_endif

#ifdef CONFIG_UNMAP_KERNEL_AT_EL0

	msr	far_el1, x29

	tramp_alias	x30, tramp_exit_native, x29

	br	x30



	ldr_this_cpu	x30, this_cpu_vector, x29

	tramp_alias	x29, tramp_exit

	msr		vbar_el1, x30		// install vector table

	ldr		lr, [sp, #S_LR]		// restore x30

	add		sp, sp, #PT_REGS_SIZE	// restore sp

	br		x29

#endif

	.else

	ldr	lr, [sp, #S_LR]
@@ -728,20 +732,6 @@ alternative_else_nop_endif 
.org 1b + 128	// Did we overflow the ventry slot?

	.endm



	.macro tramp_exit, regsize = 64

	tramp_data_read_var	x30, this_cpu_vector

	get_this_cpu_offset x29

	ldr	x30, [x30, x29]



	msr	vbar_el1, x30

	ldr	lr, [sp, #S_LR]

	tramp_unmap_kernel	x29

	mrs	x29, far_el1

	add	sp, sp, #PT_REGS_SIZE		// restore sp

	eret

	sb

	.endm



	.macro	generate_tramp_vector,	kpti, bhb

.Lvector_start\@:

	.space	0x400
@@ -762,7 +752,7 @@ alternative_else_nop_endif 
 */

	.pushsection ".entry.tramp.text", "ax"

	.align	11

SYM_CODE_START_NOALIGN(tramp_vectors)

SYM_CODE_START_LOCAL_NOALIGN(tramp_vectors)

#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY

	generate_tramp_vector	kpti=1, bhb=BHB_MITIGATION_LOOP

	generate_tramp_vector	kpti=1, bhb=BHB_MITIGATION_FW
@@ -771,9 +761,12 @@ SYM_CODE_START_NOALIGN(tramp_vectors) 
	generate_tramp_vector	kpti=1, bhb=BHB_MITIGATION_NONE

SYM_CODE_END(tramp_vectors)



SYM_CODE_START(tramp_exit_native)

	tramp_exit

SYM_CODE_END(tramp_exit_native)

SYM_CODE_START_LOCAL(tramp_exit)

	tramp_unmap_kernel	x29

	mrs		x29, far_el1		// restore x29

	eret

	sb

SYM_CODE_END(tramp_exit)

	.popsection				// .entry.tramp.text

#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
@@ -1067,7 +1060,7 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 
alternative_else_nop_endif



#ifdef CONFIG_UNMAP_KERNEL_AT_EL0

	tramp_alias	dst=x5, sym=__sdei_asm_exit_trampoline, tmp=x3

	tramp_alias	dst=x5, sym=__sdei_asm_exit_trampoline

	br	x5

#endif

SYM_CODE_END(__sdei_asm_handler)