Commit 3dcf60bb authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Catalin Marinas
Browse files

arm64: head: Clean the ID map and the HYP text to the PoC if needed 



If we enter with the MMU and caches enabled, the bootloader may not have
performed any cache maintenance to the PoC. So clean the ID mapped page
to the PoC, to ensure that instruction and data accesses with the MMU
off see the correct data. For similar reasons, clean all the HYP text to
the PoC as well when entering at EL2 with the MMU and caches enabled.

Note that this means primary_entry() itself needs to be moved into the
ID map as well, as we will return from init_kernel_el() with the MMU and
caches off.

Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20230111102236.1430401-6-ardb@kernel.org


Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 32b135a7

arch/arm64/kernel/head.S

+27 −4
Original line number Diff line number Diff line
@@ -70,7 +70,7 @@ 


	__EFI_PE_HEADER



	__INIT

	.section ".idmap.text","awx"



	/*

	 * The following callee saved general purpose registers are used on the
@@ -90,6 +90,17 @@ SYM_CODE_START(primary_entry) 
	bl	record_mmu_state

	bl	preserve_boot_args

	bl	create_idmap



	/*

	 * If we entered with the MMU and caches on, clean the ID mapped part

	 * of the primary boot code to the PoC so we can safely execute it with

	 * the MMU off.

	 */

	cbz	x19, 0f

	adrp	x0, __idmap_text_start

	adr_l	x1, __idmap_text_end

	bl	dcache_clean_poc

0:	mov	x0, x19

	bl	init_kernel_el			// w0=cpu_boot_mode

	mov	x20, x0
@@ -111,6 +122,7 @@ SYM_CODE_START(primary_entry) 
	b	__primary_switch

SYM_CODE_END(primary_entry)



	__INIT

SYM_CODE_START_LOCAL(record_mmu_state)

	mrs	x19, CurrentEL

	cmp	x19, #CurrentEL_EL2
@@ -507,10 +519,12 @@ SYM_FUNC_END(__primary_switched) 
 * Returns either BOOT_CPU_MODE_EL1 or BOOT_CPU_MODE_EL2 in x0 if

 * booted in EL1 or EL2 respectively, with the top 32 bits containing

 * potential context flags. These flags are *not* stored in __boot_cpu_mode.

 *

 * x0: whether we are being called from the primary boot path with the MMU on

 */

SYM_FUNC_START(init_kernel_el)

	mrs	x0, CurrentEL

	cmp	x0, #CurrentEL_EL2

	mrs	x1, CurrentEL

	cmp	x1, #CurrentEL_EL2

	b.eq	init_el2



SYM_INNER_LABEL(init_el1, SYM_L_LOCAL)
@@ -525,6 +539,14 @@ SYM_INNER_LABEL(init_el1, SYM_L_LOCAL) 
	eret



SYM_INNER_LABEL(init_el2, SYM_L_LOCAL)

	msr	elr_el2, lr



	// clean all HYP code to the PoC if we booted at EL2 with the MMU on

	cbz	x0, 0f

	adrp	x0, __hyp_idmap_text_start

	adr_l	x1, __hyp_text_end

	bl	dcache_clean_poc

0:

	mov_q	x0, HCR_HOST_NVHE_FLAGS

	msr	hcr_el2, x0

	isb
@@ -558,7 +580,6 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL) 
	msr	sctlr_el1, x1

	mov	x2, xzr

2:

	msr	elr_el2, lr

	mov	w0, #BOOT_CPU_MODE_EL2

	orr	x0, x0, x2

	eret
@@ -569,6 +590,7 @@ SYM_FUNC_END(init_kernel_el) 
	 * cores are held until we're ready for them to initialise.

	 */

SYM_FUNC_START(secondary_holding_pen)

	mov	x0, xzr

	bl	init_kernel_el			// w0=cpu_boot_mode

	mrs	x2, mpidr_el1

	mov_q	x1, MPIDR_HWID_BITMASK
@@ -586,6 +608,7 @@ SYM_FUNC_END(secondary_holding_pen) 
	 * be used where CPUs are brought online dynamically by the kernel.

	 */

SYM_FUNC_START(secondary_entry)

	mov	x0, xzr

	bl	init_kernel_el			// w0=cpu_boot_mode

	b	secondary_startup

SYM_FUNC_END(secondary_entry)

arch/arm64/kernel/sleep.S

+1 −0
Original line number Diff line number Diff line
@@ -99,6 +99,7 @@ SYM_FUNC_END(__cpu_suspend_enter) 


	.pushsection ".idmap.text", "awx"

SYM_CODE_START(cpu_resume)

	mov	x0, xzr

	bl	init_kernel_el

	mov	x19, x0			// preserve boot mode

#if VA_BITS > 48