Commit 005e1267 authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Will Deacon
Browse files

arm64: head: record CPU boot mode after enabling the MMU 



In order to avoid having to touch memory with the MMU and caches
disabled, and therefore having to invalidate it from the caches
explicitly, just defer storing the value until after the MMU has been
turned on, unless we are giving up with an error.

While at it, move the associated variable definitions into C code.

Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20220624150651.1358849-19-ardb@kernel.org


Signed-off-by: default avatarWill Deacon <will@kernel.org>
parent 6495b9ba

arch/arm64/kernel/head.S

+13 −37
Original line number Diff line number Diff line
@@ -82,6 +82,7 @@ 
	 * primary lowlevel boot path:

	 *

	 *  Register   Scope                      Purpose

	 *  x20        primary_entry() .. __primary_switch()    CPU boot mode

	 *  x21        primary_entry() .. start_kernel()        FDT pointer passed at boot in x0

	 *  x22        create_idmap() .. start_kernel()         ID map VA of the DT blob

	 *  x23        primary_entry() .. start_kernel()        physical misalignment/KASLR offset
@@ -91,9 +92,9 @@ 
SYM_CODE_START(primary_entry)

	bl	preserve_boot_args

	bl	init_kernel_el			// w0=cpu_boot_mode

	mov	x20, x0

	adrp	x23, __PHYS_OFFSET

	and	x23, x23, MIN_KIMG_ALIGN - 1	// KASLR offset, defaults to 0

	bl	set_cpu_boot_mode_flag

	bl	create_idmap



	/*
@@ -429,6 +430,9 @@ SYM_FUNC_START_LOCAL(__primary_switched) 
	sub	x4, x4, x0			// the kernel virtual and

	str_l	x4, kimage_voffset, x5		// physical mappings



	mov	x0, x20

	bl	set_cpu_boot_mode_flag



	// Clear BSS

	adr_l	x0, __bss_start

	mov	x1, xzr
@@ -454,6 +458,7 @@ SYM_FUNC_START_LOCAL(__primary_switched) 
	ret					// to __primary_switch()

0:

#endif

	mov	x0, x20

	bl	switch_to_vhe			// Prefer VHE if possible

	ldp	x29, x30, [sp], #16

	bl	start_kernel
@@ -553,52 +558,21 @@ SYM_FUNC_START_LOCAL(set_cpu_boot_mode_flag) 
	b.ne	1f

	add	x1, x1, #4

1:	str	w0, [x1]			// Save CPU boot mode

	dmb	sy

	dc	ivac, x1			// Invalidate potentially stale cache line

	ret

SYM_FUNC_END(set_cpu_boot_mode_flag)



/*

 * These values are written with the MMU off, but read with the MMU on.

 * Writers will invalidate the corresponding address, discarding up to a

 * 'Cache Writeback Granule' (CWG) worth of data. The linker script ensures

 * sufficient alignment that the CWG doesn't overlap another section.

 */

	.pushsection ".mmuoff.data.write", "aw"

/*

 * We need to find out the CPU boot mode long after boot, so we need to

 * store it in a writable variable.

 *

 * This is not in .bss, because we set it sufficiently early that the boot-time

 * zeroing of .bss would clobber it.

 */

SYM_DATA_START(__boot_cpu_mode)

	.long	BOOT_CPU_MODE_EL2

	.long	BOOT_CPU_MODE_EL1

SYM_DATA_END(__boot_cpu_mode)

/*

 * The booting CPU updates the failed status @__early_cpu_boot_status,

 * with MMU turned off.

 */

SYM_DATA_START(__early_cpu_boot_status)

	.quad 	0

SYM_DATA_END(__early_cpu_boot_status)



	.popsection



	/*

	 * This provides a "holding pen" for platforms to hold all secondary

	 * cores are held until we're ready for them to initialise.

	 */

SYM_FUNC_START(secondary_holding_pen)

	bl	init_kernel_el			// w0=cpu_boot_mode

	bl	set_cpu_boot_mode_flag

	mrs	x0, mpidr_el1

	mrs	x2, mpidr_el1

	mov_q	x1, MPIDR_HWID_BITMASK

	and	x0, x0, x1

	and	x2, x2, x1

	adr_l	x3, secondary_holding_pen_release

pen:	ldr	x4, [x3]

	cmp	x4, x0

	cmp	x4, x2

	b.eq	secondary_startup

	wfe

	b	pen
@@ -610,7 +584,6 @@ SYM_FUNC_END(secondary_holding_pen) 
	 */

SYM_FUNC_START(secondary_entry)

	bl	init_kernel_el			// w0=cpu_boot_mode

	bl	set_cpu_boot_mode_flag

	b	secondary_startup

SYM_FUNC_END(secondary_entry)
@@ -618,6 +591,7 @@ SYM_FUNC_START_LOCAL(secondary_startup) 
	/*

	 * Common entry point for secondary CPUs.

	 */

	mov	x20, x0				// preserve boot mode

	bl	switch_to_vhe

	bl	__cpu_secondary_check52bitva

	bl	__cpu_setup			// initialise processor
@@ -629,6 +603,9 @@ SYM_FUNC_START_LOCAL(secondary_startup) 
SYM_FUNC_END(secondary_startup)



SYM_FUNC_START_LOCAL(__secondary_switched)

	mov	x0, x20

	bl	set_cpu_boot_mode_flag

	str_l	xzr, __early_cpu_boot_status, x3

	adr_l	x5, vectors

	msr	vbar_el1, x5

	isb
@@ -691,7 +668,6 @@ SYM_FUNC_START(__enable_mmu) 
	b.lt    __no_granule_support

	cmp     x3, #ID_AA64MMFR0_TGRAN_SUPPORTED_MAX

	b.gt    __no_granule_support

	update_early_cpu_boot_status 0, x3, x4

	phys_to_ttbr x2, x2

	msr	ttbr0_el1, x2			// load TTBR0

	load_ttbr1 x1, x1, x3

arch/arm64/kernel/hyp-stub.S

+2 −2
Original line number Diff line number Diff line
@@ -223,11 +223,11 @@ SYM_FUNC_END(__hyp_reset_vectors) 


/*

 * Entry point to switch to VHE if deemed capable

 *

 * w0: boot mode, as returned by init_kernel_el()

 */

SYM_FUNC_START(switch_to_vhe)

	// Need to have booted at EL2

	adr_l	x1, __boot_cpu_mode

	ldr	w0, [x1]

	cmp	w0, #BOOT_CPU_MODE_EL2

	b.ne	1f

arch/arm64/mm/mmu.c

+8 −0
Original line number Diff line number Diff line
@@ -56,6 +56,14 @@ EXPORT_SYMBOL(kimage_vaddr); 
u64 kimage_voffset __ro_after_init;

EXPORT_SYMBOL(kimage_voffset);



u32 __boot_cpu_mode[] = { BOOT_CPU_MODE_EL2, BOOT_CPU_MODE_EL1 };



/*

 * The booting CPU updates the failed status @__early_cpu_boot_status,

 * with MMU turned off.

 */

long __section(".mmuoff.data.write") __early_cpu_boot_status;



/*

 * Empty_zero_page is a special page that is used for zero-initialized data

 * and COW.