Commit 8b87d8ce authored by Peter Zijlstra's avatar Peter Zijlstra
Browse files

x86/entry,xen: Early rewrite of restore_regs_and_return_to_kernel() 



By doing an early rewrite of 'jmp native_iret` in
restore_regs_and_return_to_kernel() we can get rid of the last
INTERRUPT_RETURN user and paravirt_iret.

Suggested-by: default avatarAndrew Cooper <Andrew.Cooper3@citrix.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: default avatarJosh Poimboeuf <jpoimboe@redhat.com>
Link: https://lore.kernel.org/r/20220308154317.815039833@infradead.org
parent 6cf3e4c0

arch/x86/entry/entry_64.S

+8 −3
Original line number Diff line number Diff line
@@ -609,7 +609,7 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) 
	/* Restore RDI. */

	popq	%rdi

	swapgs

	jmp	native_iret

	jmp	.Lnative_iret





SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
@@ -626,9 +626,14 @@ SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL) 
	 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization

	 * when returning from IPI handler.

	 */

	INTERRUPT_RETURN

#ifdef CONFIG_XEN_PV

SYM_INNER_LABEL(early_xen_iret_patch, SYM_L_GLOBAL)

	ANNOTATE_NOENDBR

	.byte 0xe9

	.long .Lnative_iret - (. + 4)

#endif



SYM_INNER_LABEL_ALIGN(native_iret, SYM_L_GLOBAL)

.Lnative_iret:

	UNWIND_HINT_IRET_REGS

	/*

	 * Are we returning to a stack segment from the LDT?  Note: in

arch/x86/include/asm/irqflags.h

+0 −5
Original line number Diff line number Diff line
@@ -141,13 +141,8 @@ static __always_inline void arch_local_irq_restore(unsigned long flags) 
#ifdef CONFIG_X86_64

#ifdef CONFIG_XEN_PV

#define SWAPGS	ALTERNATIVE "swapgs", "", X86_FEATURE_XENPV

#define INTERRUPT_RETURN						\

	ANNOTATE_RETPOLINE_SAFE;					\

	ALTERNATIVE_TERNARY("jmp *paravirt_iret(%rip);",		\

		X86_FEATURE_XENPV, "jmp xen_iret;", "jmp native_iret;")

#else

#define SWAPGS	swapgs

#define INTERRUPT_RETURN	jmp native_iret

#endif

#endif

#endif /* !__ASSEMBLY__ */

arch/x86/include/asm/paravirt_types.h

+0 −1
Original line number Diff line number Diff line
@@ -272,7 +272,6 @@ struct paravirt_patch_template { 


extern struct pv_info pv_info;

extern struct paravirt_patch_template pv_ops;

extern void (*paravirt_iret)(void);



#define PARAVIRT_PATCH(x)					\

	(offsetof(struct paravirt_patch_template, x) / sizeof(void *))

arch/x86/kernel/head_64.S

+2 −1
Original line number Diff line number Diff line
@@ -345,7 +345,6 @@ SYM_CODE_START_NOALIGN(vc_boot_ghcb) 
	/* Remove Error Code */

	addq    $8, %rsp



	/* Pure iret required here - don't use INTERRUPT_RETURN */

	iretq

SYM_CODE_END(vc_boot_ghcb)

#endif
@@ -426,6 +425,8 @@ SYM_CODE_END(early_idt_handler_common) 
 * early_idt_handler_array can't be used because it returns via the

 * paravirtualized INTERRUPT_RETURN and pv-ops don't work that early.

 *

 * XXX it does, fix this.

 *

 * This handler will end up in the .init.text section and not be

 * available to boot secondary CPUs.

 */

arch/x86/kernel/paravirt.c

+0 −4
Original line number Diff line number Diff line
@@ -132,8 +132,6 @@ void paravirt_set_sched_clock(u64 (*func)(void)) 
}



/* These are in entry.S */

extern void native_iret(void);



static struct resource reserve_ioports = {

	.start = 0,

	.end = IO_SPACE_LIMIT,
@@ -397,8 +395,6 @@ struct paravirt_patch_template pv_ops = { 


#ifdef CONFIG_PARAVIRT_XXL

NOKPROBE_SYMBOL(native_load_idt);



void (*paravirt_iret)(void) = native_iret;

#endif



EXPORT_SYMBOL(pv_ops);