Commit 64ef578b authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Borislav Petkov (AMD)
Browse files

x86/decompressor: Call trampoline directly from C code 



Instead of returning to the asm calling code to invoke the trampoline,
call it straight from the C code that sets it up. That way, the struct
return type is no longer needed for returning two values, and the call
can be made conditional more cleanly in a subsequent patch.

This means that all callee save 64-bit registers need to be preserved
and restored, as their contents may not survive the legacy mode switch.

Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Signed-off-by: default avatarBorislav Petkov (AMD) <bp@alien8.de>
Acked-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Link: https://lore.kernel.org/r/20230807162720.545787-13-ardb@kernel.org
parent bd328aa0

arch/x86/boot/compressed/head_64.S

+13 −18
Original line number Diff line number Diff line
@@ -430,25 +430,14 @@ SYM_CODE_START(startup_64) 
#endif



	/*

	 * paging_prepare() sets up the trampoline and checks if we need to

	 * enable 5-level paging.

	 *

	 * paging_prepare() returns a two-quadword structure which lands

	 * into RDX:RAX:

	 *   - Address of the trampoline is returned in RAX.

	 *   - Non zero RDX means trampoline needs to enable 5-level

	 *     paging.

	 * configure_5level_paging() updates the number of paging levels using

	 * a trampoline in 32-bit addressable memory if the current number does

	 * not match the desired number.

	 *

	 * Pass the boot_params pointer as the first argument.

	 */

	movq	%r15, %rdi

	call	paging_prepare



	/* Pass the trampoline address and boolean flag as args #1 and #2 */

	movq	%rax, %rdi

	movq	%rdx, %rsi

	leaq	TRAMPOLINE_32BIT_CODE_OFFSET(%rax), %rax

	call	*%rax

	call	configure_5level_paging



	/*

	 * cleanup_trampoline() would restore trampoline memory.
@@ -543,11 +532,14 @@ SYM_FUNC_END(.Lrelocated) 
	.section ".rodata", "a", @progbits

SYM_CODE_START(trampoline_32bit_src)

	/*

	 * Preserve live 64-bit registers on the stack: this is necessary

	 * because the architecture does not guarantee that GPRs will retain

	 * their full 64-bit values across a 32-bit mode switch.

	 * Preserve callee save 64-bit registers on the stack: this is

	 * necessary because the architecture does not guarantee that GPRs will

	 * retain their full 64-bit values across a 32-bit mode switch.

	 */

	pushq	%r15

	pushq	%r14

	pushq	%r13

	pushq	%r12

	pushq	%rbp

	pushq	%rbx
@@ -574,6 +566,9 @@ SYM_CODE_START(trampoline_32bit_src) 
	/* Restore the preserved 64-bit registers */

	popq	%rbx

	popq	%rbp

	popq	%r12

	popq	%r13

	popq	%r14

	popq	%r15

	retq

arch/x86/boot/compressed/pgtable_64.c

+13 −19
Original line number Diff line number Diff line
@@ -16,11 +16,6 @@ unsigned int __section(".data") pgdir_shift = 39; 
unsigned int __section(".data") ptrs_per_p4d = 1;

#endif



struct paging_config {

	unsigned long trampoline_start;

	unsigned long l5_required;

};



/* Buffer to preserve trampoline memory */

static char trampoline_save[TRAMPOLINE_32BIT_SIZE];
@@ -29,7 +24,7 @@ static char trampoline_save[TRAMPOLINE_32BIT_SIZE]; 
 * purposes.

 *

 * Avoid putting the pointer into .bss as it will be cleared between

 * paging_prepare() and extract_kernel().

 * configure_5level_paging() and extract_kernel().

 */

unsigned long *trampoline_32bit __section(".data");
@@ -106,13 +101,13 @@ static unsigned long find_trampoline_placement(void) 
	return bios_start - TRAMPOLINE_32BIT_SIZE;

}



struct paging_config paging_prepare(void *rmode)

asmlinkage void configure_5level_paging(struct boot_params *bp)

{

	struct paging_config paging_config = {};

	void *tramp_code;

	void (*toggle_la57)(void *trampoline, bool enable_5lvl);

	bool l5_required = false;



	/* Initialize boot_params. Required for cmdline_find_option_bool(). */

	boot_params = rmode;

	boot_params = bp;



	/*

	 * Check if LA57 is desired and supported.
@@ -130,7 +125,7 @@ struct paging_config paging_prepare(void *rmode) 
			!cmdline_find_option_bool("no5lvl") &&

			native_cpuid_eax(0) >= 7 &&

			(native_cpuid_ecx(7) & (1 << (X86_FEATURE_LA57 & 31)))) {

		paging_config.l5_required = 1;

		l5_required = true;



		/* Initialize variables for 5-level paging */

		__pgtable_l5_enabled = 1;
@@ -138,9 +133,7 @@ struct paging_config paging_prepare(void *rmode) 
		ptrs_per_p4d = 512;

	}



	paging_config.trampoline_start = find_trampoline_placement();



	trampoline_32bit = (unsigned long *)paging_config.trampoline_start;

	trampoline_32bit = (unsigned long *)find_trampoline_placement();



	/* Preserve trampoline memory */

	memcpy(trampoline_save, trampoline_32bit, TRAMPOLINE_32BIT_SIZE);
@@ -149,7 +142,7 @@ struct paging_config paging_prepare(void *rmode) 
	memset(trampoline_32bit, 0, TRAMPOLINE_32BIT_SIZE);



	/* Copy trampoline code in place */

	tramp_code = memcpy(trampoline_32bit +

	toggle_la57 = memcpy(trampoline_32bit +

			TRAMPOLINE_32BIT_CODE_OFFSET / sizeof(unsigned long),

			&trampoline_32bit_src, TRAMPOLINE_32BIT_CODE_SIZE);
@@ -159,7 +152,8 @@ struct paging_config paging_prepare(void *rmode) 
	 * immediate absolute address, which needs to be adjusted based on the

	 * placement of the trampoline.

	 */

	*(u32 *)(tramp_code + trampoline_ljmp_imm_offset) += (unsigned long)tramp_code;

	*(u32 *)((u8 *)toggle_la57 + trampoline_ljmp_imm_offset) +=

						(unsigned long)toggle_la57;



	/*

	 * The code below prepares page table in trampoline memory.
@@ -175,10 +169,10 @@ struct paging_config paging_prepare(void *rmode) 
	 * We are not going to use the page table in trampoline memory if we

	 * are already in the desired paging mode.

	 */

	if (paging_config.l5_required == !!(native_read_cr4() & X86_CR4_LA57))

	if (l5_required == !!(native_read_cr4() & X86_CR4_LA57))

		goto out;



	if (paging_config.l5_required) {

	if (l5_required) {

		/*

		 * For 4- to 5-level paging transition, set up current CR3 as

		 * the first and the only entry in a new top-level page table.
@@ -201,7 +195,7 @@ struct paging_config paging_prepare(void *rmode) 
	}



out:

	return paging_config;

	toggle_la57(trampoline_32bit, l5_required);

}



void cleanup_trampoline(void *pgtable)