Merge tag 'x86_boot_for_v6.6_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

The function prototype for the handover entry point looks like this::

    efi_main(void *handle, efi_system_table_t *table, struct boot_params *bp)

    efi_stub_entry(void *handle, efi_system_table_t *table, struct boot_params *bp)

'handle' is the EFI image handle passed to the boot loader by the EFI

firmware, 'table' is the EFI system table - these are the first two

ifeq ($(CONFIG_LD_IS_BFD),y)

LDFLAGS_vmlinux += $(call ld-option,--no-warn-rwx-segments)

endif

ifeq ($(CONFIG_EFI_STUB),y)

# ensure that the static EFI stub library will be pulled in, even if it is

# never referenced explicitly from the startup code

LDFLAGS_vmlinux += -u efi_pe_entry

endif

LDFLAGS_vmlinux += -T

hostprogs	:= mkpiggy

 * When booting in 64-bit mode on 32-bit EFI firmware, startup_64_mixed_mode()

 * is the first thing that runs after switching to long mode. Depending on

 * whether the EFI handover protocol or the compat entry point was used to

 * enter the kernel, it will either branch to the 64-bit EFI handover

 * entrypoint at offset 0x390 in the image, or to the 64-bit EFI PE/COFF

 * enter the kernel, it will either branch to the common 64-bit EFI stub

 * entrypoint efi_stub_entry() directly, or via the 64-bit EFI PE/COFF

 * entrypoint efi_pe_entry(). In the former case, the bootloader must provide a

 * struct bootparams pointer as the third argument, so the presence of such a

 * pointer is used to disambiguate.

 *  | efi32_pe_entry   |---->|            |            |       +-----------+--+

 *  +------------------+     |            |     +------+----------------+  |

 *                           | startup_32 |---->| startup_64_mixed_mode |  |

 *  +------------------+     |            |     +------+----------------+  V

 *  | efi32_stub_entry |---->|            |            |     +------------------+

 *  +------------------+     +------------+            +---->| efi64_stub_entry |

 *                                                           +-------------+----+

 *                           +------------+     +----------+               |

 *                           | startup_64 |<----| efi_main |<--------------+

 *                           +------------+     +----------+

 *  +------------------+     |            |     +------+----------------+  |

 *  | efi32_stub_entry |---->|            |            |                   |

 *  +------------------+     +------------+            |                   |

 *                                                     V                   |

 *                           +------------+     +----------------+         |

 *                           | startup_64 |<----| efi_stub_entry |<--------+

 *                           +------------+     +----------------+

 */

SYM_FUNC_START(startup_64_mixed_mode)

	lea	efi32_boot_args(%rip), %rdx

	mov	0(%rdx), %edi

	mov	4(%rdx), %esi

#ifdef CONFIG_EFI_HANDOVER_PROTOCOL

	mov	8(%rdx), %edx		// saved bootparams pointer

	test	%edx, %edx

	jnz	efi64_stub_entry

	jnz	efi_stub_entry

#endif

	/*

	 * efi_pe_entry uses MS calling convention, which requires 32 bytes of

	 * shadow space on the stack even if all arguments are passed in

SYM_FUNC_END(__efi64_thunk)

	.code32

#ifdef CONFIG_EFI_HANDOVER_PROTOCOL

SYM_FUNC_START(efi32_stub_entry)

	call	1f

1:	popl	%ecx

	/* Clear BSS */

	xorl	%eax, %eax

	leal	(_bss - 1b)(%ecx), %edi

	leal	(_ebss - 1b)(%ecx), %ecx

	subl	%edi, %ecx

	shrl	$2, %ecx

	cld

	rep	stosl

	add	$0x4, %esp		/* Discard return address */

	popl	%ecx

	popl	%edx

	popl	%esi

	jmp	efi32_entry

SYM_FUNC_END(efi32_stub_entry)

#endif

/*

 * EFI service pointer must be in %edi.

 *

 * stub may still exit and return to the firmware using the Exit() EFI boot

 * service.]

 */

SYM_FUNC_START(efi32_entry)

SYM_FUNC_START_LOCAL(efi32_entry)

	call	1f

1:	pop	%ebx

	jmp	startup_32

SYM_FUNC_END(efi32_entry)

#define ST32_boottime		60 // offsetof(efi_system_table_32_t, boottime)

#define BS32_handle_protocol	88 // offsetof(efi_boot_services_32_t, handle_protocol)

#define LI32_image_base		32 // offsetof(efi_loaded_image_32_t, image_base)

/*

 * efi_status_t efi32_pe_entry(efi_handle_t image_handle,

 *			       efi_system_table_32_t *sys_table)

SYM_FUNC_START(efi32_pe_entry)

	pushl	%ebp

	movl	%esp, %ebp

	pushl	%eax				// dummy push to allocate loaded_image

	pushl	%ebx				// save callee-save registers

	pushl	%edi

	movl	$0x80000003, %eax		// EFI_UNSUPPORTED

	jnz	2f

	call	1f

1:	pop	%ebx

	/* Get the loaded image protocol pointer from the image handle */

	leal	-4(%ebp), %eax

	pushl	%eax				// &loaded_image

	leal	(loaded_image_proto - 1b)(%ebx), %eax

	pushl	%eax				// pass the GUID address

	pushl	8(%ebp)				// pass the image handle

	/*

	 * Note the alignment of the stack frame.

	 *   sys_table

	 *   handle             <-- 16-byte aligned on entry by ABI

	 *   return address

	 *   frame pointer

	 *   loaded_image       <-- local variable

	 *   saved %ebx		<-- 16-byte aligned here

	 *   saved %edi

	 *   &loaded_image

	 *   &loaded_image_proto

	 *   handle             <-- 16-byte aligned for call to handle_protocol

	 */

	movl	12(%ebp), %eax			// sys_table

	movl	ST32_boottime(%eax), %eax	// sys_table->boottime

	call	*BS32_handle_protocol(%eax)	// sys_table->boottime->handle_protocol

	addl	$12, %esp			// restore argument space

	testl	%eax, %eax

	jnz	2f

	movl	8(%ebp), %ecx			// image_handle

	movl	12(%ebp), %edx			// sys_table

	movl	-4(%ebp), %esi			// loaded_image

	movl	LI32_image_base(%esi), %esi	// loaded_image->image_base

	leal	(startup_32 - 1b)(%ebx), %ebp	// runtime address of startup_32

	/*

	 * We need to set the image_offset variable here since startup_32() will

	 * use it before we get to the 64-bit efi_pe_entry() in C code.

	 */

	subl	%esi, %ebp			// calculate image_offset

	movl	%ebp, (image_offset - 1b)(%ebx)	// save image_offset

	xorl	%esi, %esi

	jmp	efi32_entry			// pass %ecx, %edx, %esi

						// no other registers remain live

	RET

SYM_FUNC_END(efi32_pe_entry)

	.section ".rodata"

	/* EFI loaded image protocol GUID */

	.balign 4

SYM_DATA_START_LOCAL(loaded_image_proto)

	.long	0x5b1b31a1

	.word	0x9562, 0x11d2

	.byte	0x8e, 0x3f, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b

SYM_DATA_END(loaded_image_proto)

#ifdef CONFIG_EFI_HANDOVER_PROTOCOL

	.org	efi32_stub_entry + 0x200

	.code64

SYM_FUNC_START_NOALIGN(efi64_stub_entry)

	jmp	efi_handover_entry

SYM_FUNC_END(efi64_stub_entry)

#endif

	.data

	.balign	8

#ifdef CONFIG_RELOCATABLE

	leal	startup_32@GOTOFF(%edx), %ebx

#ifdef CONFIG_EFI_STUB

/*

 * If we were loaded via the EFI LoadImage service, startup_32() will be at an

 * offset to the start of the space allocated for the image. efi_pe_entry() will

 * set up image_offset to tell us where the image actually starts, so that we

 * can use the full available buffer.

 *	image_offset = startup_32 - image_base

 * Otherwise image_offset will be zero and has no effect on the calculations.

 */

	subl    image_offset@GOTOFF(%edx), %ebx

#endif

	movl	BP_kernel_alignment(%esi), %eax

	decl	%eax

	addl    %eax, %ebx

	jmp	*%eax

SYM_FUNC_END(startup_32)

#ifdef CONFIG_EFI_STUB

SYM_FUNC_START(efi32_stub_entry)

	add	$0x4, %esp

	movl	8(%esp), %esi	/* save boot_params pointer */

	call	efi_main

	/* efi_main returns the possibly relocated address of startup_32 */

	jmp	*%eax

SYM_FUNC_END(efi32_stub_entry)

SYM_FUNC_ALIAS(efi_stub_entry, efi32_stub_entry)

#endif

	.text

SYM_FUNC_START_LOCAL_NOALIGN(.Lrelocated)

 */

	/* push arguments for extract_kernel: */

	pushl	output_len@GOTOFF(%ebx)	/* decompressed length, end of relocs */

	pushl	%ebp			/* output address */

	pushl	input_len@GOTOFF(%ebx)	/* input_len */

	leal	input_data@GOTOFF(%ebx), %eax

	pushl	%eax			/* input_data */

	leal	boot_heap@GOTOFF(%ebx), %eax

	pushl	%eax			/* heap area */

	pushl	%esi			/* real mode pointer */

	call	extract_kernel		/* returns kernel entry point in %eax */

	addl	$24, %esp

 */

	.bss

	.balign 4

boot_heap:

	.fill BOOT_HEAP_SIZE, 1, 0

boot_stack:

	.fill BOOT_STACK_SIZE, 1, 0

boot_stack_end:

#ifdef CONFIG_RELOCATABLE

	movl	%ebp, %ebx

#ifdef CONFIG_EFI_STUB

/*

 * If we were loaded via the EFI LoadImage service, startup_32 will be at an

 * offset to the start of the space allocated for the image. efi_pe_entry will

 * set up image_offset to tell us where the image actually starts, so that we

 * can use the full available buffer.

 *	image_offset = startup_32 - image_base

 * Otherwise image_offset will be zero and has no effect on the calculations.

 */

	subl    rva(image_offset)(%ebp), %ebx

#endif

	movl	BP_kernel_alignment(%esi), %eax

	decl	%eax

	addl	%eax, %ebx

	lret

SYM_FUNC_END(startup_32)

#if IS_ENABLED(CONFIG_EFI_MIXED) && IS_ENABLED(CONFIG_EFI_HANDOVER_PROTOCOL)

	.org 0x190

SYM_FUNC_START(efi32_stub_entry)

	add	$0x4, %esp		/* Discard return address */

	popl	%ecx

	popl	%edx

	popl	%esi

	jmp	efi32_entry

SYM_FUNC_END(efi32_stub_entry)

#endif

	.code64

	.org 0x200

SYM_CODE_START(startup_64)

	/* Start with the delta to where the kernel will run at. */

#ifdef CONFIG_RELOCATABLE

	leaq	startup_32(%rip) /* - $startup_32 */, %rbp

#ifdef CONFIG_EFI_STUB

/*

 * If we were loaded via the EFI LoadImage service, startup_32 will be at an

 * offset to the start of the space allocated for the image. efi_pe_entry will

 * set up image_offset to tell us where the image actually starts, so that we

 * can use the full available buffer.

 *	image_offset = startup_32 - image_base

 * Otherwise image_offset will be zero and has no effect on the calculations.

 */

	movl    image_offset(%rip), %eax

	subq	%rax, %rbp

#endif

	movl	BP_kernel_alignment(%rsi), %eax

	decl	%eax

	addq	%rax, %rbp

	 * For the trampoline, we need the top page table to reside in lower

	 * memory as we don't have a way to load 64-bit values into CR3 in

	 * 32-bit mode.

	 *

	 * We go though the trampoline even if we don't have to: if we're

	 * already in a desired paging mode. This way the trampoline code gets

	 * tested on every boot.

	 */

	/* Make sure we have GDT with 32-bit code segment */

	lretq

.Lon_kernel_cs:

	/*

	 * RSI holds a pointer to a boot_params structure provided by the

	 * loader, and this needs to be preserved across C function calls. So

	 * move it into a callee saved register.

	 */

	movq	%rsi, %r15

	pushq	%rsi

	call	load_stage1_idt

	popq	%rsi

#ifdef CONFIG_AMD_MEM_ENCRYPT

	/*

	 * CPUID instructions being issued, so go ahead and do that now via

	 * sev_enable(), which will also handle the rest of the SEV-related

	 * detection/setup to ensure that has been done in advance of any dependent

	 * code.

	 * code. Pass the boot_params pointer as the first argument.

	 */

	pushq	%rsi

	movq	%rsi, %rdi		/* real mode address */

	movq	%r15, %rdi

	call	sev_enable

	popq	%rsi

#endif

	/*

	 * paging_prepare() sets up the trampoline and checks if we need to

	 * enable 5-level paging.

	 *

	 * paging_prepare() returns a two-quadword structure which lands

	 * into RDX:RAX:

	 *   - Address of the trampoline is returned in RAX.

	 *   - Non zero RDX means trampoline needs to enable 5-level

	 *     paging.

	 *

	 * RSI holds real mode data and needs to be preserved across

	 * this function call.

	 */

	pushq	%rsi

	movq	%rsi, %rdi		/* real mode address */

	call	paging_prepare

	popq	%rsi

	/* Save the trampoline address in RCX */

	movq	%rax, %rcx

	/*

	 * Load the address of trampoline_return() into RDI.

	 * It will be used by the trampoline to return to the main code.

	 */

	leaq	trampoline_return(%rip), %rdi

	/* Switch to compatibility mode (CS.L = 0 CS.D = 1) via far return */

	pushq	$__KERNEL32_CS

	leaq	TRAMPOLINE_32BIT_CODE_OFFSET(%rax), %rax

	pushq	%rax

	lretq

trampoline_return:

	/* Restore the stack, the 32-bit trampoline uses its own stack */

	leaq	rva(boot_stack_end)(%rbx), %rsp

	/*

	 * cleanup_trampoline() would restore trampoline memory.

	 * configure_5level_paging() updates the number of paging levels using

	 * a trampoline in 32-bit addressable memory if the current number does

	 * not match the desired number.

	 *

	 * RDI is address of the page table to use instead of page table

	 * in trampoline memory (if required).

	 *

	 * RSI holds real mode data and needs to be preserved across

	 * this function call.

	 * Pass the boot_params pointer as the first argument. The second

	 * argument is the relocated address of the page table to use instead

	 * of the page table in trampoline memory (if required).

	 */

	pushq	%rsi

	leaq	rva(top_pgtable)(%rbx), %rdi

	call	cleanup_trampoline

	popq	%rsi

	movq	%r15, %rdi

	leaq	rva(top_pgtable)(%rbx), %rsi

	call	configure_5level_paging

	/* Zero EFLAGS */

	pushq	$0

 * Copy the compressed kernel to the end of our buffer

 * where decompression in place becomes safe.

 */

	pushq	%rsi

	leaq	(_bss-8)(%rip), %rsi

	leaq	rva(_bss-8)(%rbx), %rdi

	movl	$(_bss - startup_32), %ecx

	std

	rep	movsq

	cld

	popq	%rsi

	/*

	 * The GDT may get overwritten either during the copy we just did or

	jmp	*%rax

SYM_CODE_END(startup_64)

#ifdef CONFIG_EFI_STUB

#ifdef CONFIG_EFI_HANDOVER_PROTOCOL

	.org 0x390

#endif

SYM_FUNC_START(efi64_stub_entry)

	and	$~0xf, %rsp			/* realign the stack */

	movq	%rdx, %rbx			/* save boot_params pointer */

	call	efi_main

	movq	%rbx,%rsi

	leaq	rva(startup_64)(%rax), %rax

	jmp	*%rax

SYM_FUNC_END(efi64_stub_entry)

SYM_FUNC_ALIAS(efi_stub_entry, efi64_stub_entry)

#endif

	.text

SYM_FUNC_START_LOCAL_NOALIGN(.Lrelocated)

	shrq	$3, %rcx

	rep	stosq

	pushq	%rsi

	call	load_stage2_idt

	/* Pass boot_params to initialize_identity_maps() */

	movq	(%rsp), %rdi

	movq	%r15, %rdi

	call	initialize_identity_maps

	popq	%rsi

/*

 * Do the extraction, and jump to the new kernel..

 */

	pushq	%rsi			/* Save the real mode argument */

	movq	%rsi, %rdi		/* real mode address */

	leaq	boot_heap(%rip), %rsi	/* malloc area for uncompression */

	leaq	input_data(%rip), %rdx  /* input_data */

	movl	input_len(%rip), %ecx	/* input_len */

	movq	%rbp, %r8		/* output target address */

	movl	output_len(%rip), %r9d	/* decompressed length, end of relocs */

	/* pass struct boot_params pointer and output target address */

	movq	%r15, %rdi

	movq	%rbp, %rsi

	call	extract_kernel		/* returns kernel entry point in %rax */

	popq	%rsi

/*

 * Jump to the decompressed kernel.

 */

	movq	%r15, %rsi

	jmp	*%rax

SYM_FUNC_END(.Lrelocated)

	.code32

/*

 * This is the 32-bit trampoline that will be copied over to low memory.

 * This is the 32-bit trampoline that will be copied over to low memory. It

 * will be called using the ordinary 64-bit calling convention from code

 * running in 64-bit mode.

 *

 * RDI contains the return address (might be above 4G).

 * ECX contains the base address of the trampoline memory.

 * Non zero RDX means trampoline needs to enable 5-level paging.

 * Return address is at the top of the stack (might be above 4G).

 * The first argument (EDI) contains the address of the temporary PGD level

 * page table in 32-bit addressable memory which will be programmed into

 * register CR3.

 */

	.section ".rodata", "a", @progbits

SYM_CODE_START(trampoline_32bit_src)

	/* Set up data and stack segments */

	movl	$__KERNEL_DS, %eax

	movl	%eax, %ds

	movl	%eax, %ss

	/*

	 * Preserve callee save 64-bit registers on the stack: this is

	 * necessary because the architecture does not guarantee that GPRs will

	 * retain their full 64-bit values across a 32-bit mode switch.

	 */

	pushq	%r15

	pushq	%r14

	pushq	%r13

	pushq	%r12

	pushq	%rbp

	pushq	%rbx

	/* Set up new stack */

	leal	TRAMPOLINE_32BIT_STACK_END(%ecx), %esp

	/* Preserve top half of RSP in a legacy mode GPR to avoid truncation */

	movq	%rsp, %rbx

	shrq	$32, %rbx

	/* Switch to compatibility mode (CS.L = 0 CS.D = 1) via far return */

	pushq	$__KERNEL32_CS

	leaq	0f(%rip), %rax

	pushq	%rax

	lretq

	/*

	 * The 32-bit code below will do a far jump back to long mode and end

	 * up here after reconfiguring the number of paging levels. First, the

	 * stack pointer needs to be restored to its full 64-bit value before

	 * the callee save register contents can be popped from the stack.

	 */

.Lret:

	shlq	$32, %rbx

	orq	%rbx, %rsp

	/* Restore the preserved 64-bit registers */

	popq	%rbx

	popq	%rbp

	popq	%r12

	popq	%r13

	popq	%r14

	popq	%r15

	retq

	.code32

0:

	/* Disable paging */

	movl	%cr0, %eax

	btrl	$X86_CR0_PG_BIT, %eax

	movl	%eax, %cr0

	/* Check what paging mode we want to be in after the trampoline */

	testl	%edx, %edx

	jz	1f

	/* We want 5-level paging: don't touch CR3 if it already points to 5-level page tables */

	movl	%cr4, %eax

	testl	$X86_CR4_LA57, %eax

	jnz	3f

	jmp	2f

1:

	/* We want 4-level paging: don't touch CR3 if it already points to 4-level page tables */

	movl	%cr4, %eax

	testl	$X86_CR4_LA57, %eax

	jz	3f

2:

	/* Point CR3 to the trampoline's new top level page table */

	leal	TRAMPOLINE_32BIT_PGTABLE_OFFSET(%ecx), %eax

	movl	%eax, %cr3

3:

	movl	%edi, %cr3

	/* Set EFER.LME=1 as a precaution in case hypervsior pulls the rug */

	pushl	%ecx

	pushl	%edx

	movl	$MSR_EFER, %ecx

	rdmsr

	btsl	$_EFER_LME, %eax

	/* Avoid writing EFER if no change was made (for TDX guest) */

	jc	1f

	wrmsr

1:	popl	%edx

	popl	%ecx