Commit a149180f authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Borislav Petkov
Browse files

x86: Add magic AMD return-thunk 



Note: needs to be in a section distinct from Retpolines such that the
Retpoline RET substitution cannot possibly use immediate jumps.

ORC unwinding for zen_untrain_ret() and __x86_return_thunk() is a
little tricky but works due to the fact that zen_untrain_ret() doesn't
have any stack ops and as such will emit a single ORC entry at the
start (+0x3f).

Meanwhile, unwinding an IP, including the __x86_return_thunk() one
(+0x40) will search for the largest ORC entry smaller or equal to the
IP, these will find the one ORC entry (+0x3f) and all works.

  [ Alexandre: SVM part. ]
  [ bp: Build fix, massages. ]

Suggested-by: default avatarAndrew Cooper <Andrew.Cooper3@citrix.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarBorislav Petkov <bp@suse.de>
Reviewed-by: default avatarJosh Poimboeuf <jpoimboe@kernel.org>
Signed-off-by: default avatarBorislav Petkov <bp@suse.de>
parent 951ddecf

arch/x86/entry/entry_64.S

+6 −0
Original line number Diff line number Diff line
@@ -96,6 +96,7 @@ SYM_CODE_START(entry_SYSCALL_64) 


SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL)

	ANNOTATE_NOENDBR

	UNTRAIN_RET



	/* Construct struct pt_regs on stack */

	pushq	$__USER_DS				/* pt_regs->ss */
@@ -717,6 +718,7 @@ native_irq_return_ldt: 
	pushq	%rdi				/* Stash user RDI */

	swapgs					/* to kernel GS */

	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi	/* to kernel CR3 */

	UNTRAIN_RET



	movq	PER_CPU_VAR(espfix_waddr), %rdi

	movq	%rax, (0*8)(%rdi)		/* user RAX */
@@ -911,6 +913,7 @@ SYM_CODE_START_LOCAL(paranoid_entry) 
	 * be retrieved from a kernel internal table.

	 */

	SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14

	UNTRAIN_RET



	/*

	 * Handling GSBASE depends on the availability of FSGSBASE.
@@ -1020,6 +1023,7 @@ SYM_CODE_START_LOCAL(error_entry) 
	FENCE_SWAPGS_USER_ENTRY

	/* We have user CR3.  Change to kernel CR3. */

	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax

	UNTRAIN_RET



	leaq	8(%rsp), %rdi			/* arg0 = pt_regs pointer */

.Lerror_entry_from_usermode_after_swapgs:
@@ -1072,6 +1076,7 @@ SYM_CODE_START_LOCAL(error_entry) 
	swapgs

	FENCE_SWAPGS_USER_ENTRY

	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax

	UNTRAIN_RET



	/*

	 * Pretend that the exception came from user mode: set up pt_regs
@@ -1167,6 +1172,7 @@ SYM_CODE_START(asm_exc_nmi) 
	movq	%rsp, %rdx

	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp

	UNWIND_HINT_IRET_REGS base=%rdx offset=8

	UNTRAIN_RET

	pushq	5*8(%rdx)	/* pt_regs->ss */

	pushq	4*8(%rdx)	/* pt_regs->rsp */

	pushq	3*8(%rdx)	/* pt_regs->flags */

arch/x86/entry/entry_64_compat.S

+4 −0
Original line number Diff line number Diff line
@@ -14,6 +14,7 @@ 
#include <asm/irqflags.h>

#include <asm/asm.h>

#include <asm/smap.h>

#include <asm/nospec-branch.h>

#include <linux/linkage.h>

#include <linux/err.h>
@@ -72,6 +73,7 @@ SYM_CODE_START(entry_SYSENTER_compat) 
	pushq	$__USER32_CS		/* pt_regs->cs */

	pushq	$0			/* pt_regs->ip = 0 (placeholder) */

SYM_INNER_LABEL(entry_SYSENTER_compat_after_hwframe, SYM_L_GLOBAL)

	UNTRAIN_RET



	/*

	 * User tracing code (ptrace or signal handlers) might assume that
@@ -190,6 +192,7 @@ SYM_CODE_START(entry_SYSCALL_compat) 


SYM_INNER_LABEL(entry_SYSCALL_compat_safe_stack, SYM_L_GLOBAL)

	ANNOTATE_NOENDBR

	UNTRAIN_RET



	/* Construct struct pt_regs on stack */

	pushq	$__USER32_DS		/* pt_regs->ss */
@@ -332,6 +335,7 @@ SYM_CODE_START(entry_INT80_compat) 
	pushq	0*8(%rax)		/* regs->orig_ax */

.Lint80_keep_stack:



	UNTRAIN_RET

	PUSH_AND_CLEAR_REGS rax=$-ENOSYS

	UNWIND_HINT_REGS

arch/x86/include/asm/cpufeatures.h

+1 −0
Original line number Diff line number Diff line
@@ -301,6 +301,7 @@ 
#define X86_FEATURE_RETPOLINE		(11*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */

#define X86_FEATURE_RETPOLINE_LFENCE	(11*32+13) /* "" Use LFENCE for Spectre variant 2 */

#define X86_FEATURE_RETHUNK		(11*32+14) /* "" Use REturn THUNK */

#define X86_FEATURE_UNRET		(11*32+15) /* "" AMD BTB untrain return */



/* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */

#define X86_FEATURE_AVX_VNNI		(12*32+ 4) /* AVX VNNI instructions */

arch/x86/include/asm/disabled-features.h

+2 −1
Original line number Diff line number Diff line
@@ -55,7 +55,8 @@ 
#else

# define DISABLE_RETPOLINE	((1 << (X86_FEATURE_RETPOLINE & 31)) | \

				 (1 << (X86_FEATURE_RETPOLINE_LFENCE & 31)) | \

				 (1 << (X86_FEATURE_RETHUNK & 31)))

				 (1 << (X86_FEATURE_RETHUNK & 31)) | \

				 (1 << (X86_FEATURE_UNRET & 31)))

#endif



#ifdef CONFIG_INTEL_IOMMU_SVM

arch/x86/include/asm/nospec-branch.h

+17 −0
Original line number Diff line number Diff line
@@ -112,6 +112,22 @@ 
#endif

.endm



/*

 * Mitigate RETBleed for AMD/Hygon Zen uarch. Requires KERNEL CR3 because the

 * return thunk isn't mapped into the userspace tables (then again, AMD

 * typically has NO_MELTDOWN).

 *

 * Doesn't clobber any registers but does require a stable stack.

 *

 * As such, this must be placed after every *SWITCH_TO_KERNEL_CR3 at a point

 * where we have a stack but before any RET instruction.

 */

.macro UNTRAIN_RET

#ifdef CONFIG_RETPOLINE

	ALTERNATIVE "", "call zen_untrain_ret", X86_FEATURE_UNRET

#endif

.endm



#else /* __ASSEMBLY__ */



#define ANNOTATE_RETPOLINE_SAFE					\
@@ -124,6 +140,7 @@ typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE]; 
extern retpoline_thunk_t __x86_indirect_thunk_array[];



extern void __x86_return_thunk(void);

extern void zen_untrain_ret(void);



#ifdef CONFIG_RETPOLINE