Commit 5318b987 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'x86_bugs_pbrsb' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull x86 eIBRS fixes from Borislav Petkov:
 "More from the CPU vulnerability nightmares front:

  Intel eIBRS machines do not sufficiently mitigate against RET
  mispredictions when doing a VM Exit therefore an additional RSB,
  one-entry stuffing is needed"

* tag 'x86_bugs_pbrsb' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/speculation: Add LFENCE to RSB fill sequence
  x86/speculation: Add RSB VM Exit protections
parents eb555cb5 ba6e31af

Documentation/admin-guide/hw-vuln/spectre.rst

+8 −0
Original line number Diff line number Diff line
@@ -422,6 +422,14 @@ The possible values in this file are: 
  'RSB filling'   Protection of RSB on context switch enabled

  =============   ===========================================



  - EIBRS Post-barrier Return Stack Buffer (PBRSB) protection status:



  ===========================  =======================================================

  'PBRSB-eIBRS: SW sequence'   CPU is affected and protection of RSB on VMEXIT enabled

  'PBRSB-eIBRS: Vulnerable'    CPU is vulnerable

  'PBRSB-eIBRS: Not affected'  CPU is not affected by PBRSB

  ===========================  =======================================================



Full mitigation might require a microcode update from the CPU

vendor. When the necessary microcode is not available, the kernel will

report vulnerability.

arch/x86/include/asm/cpufeatures.h

+2 −0
Original line number Diff line number Diff line
@@ -303,6 +303,7 @@ 
#define X86_FEATURE_RETHUNK		(11*32+14) /* "" Use REturn THUNK */

#define X86_FEATURE_UNRET		(11*32+15) /* "" AMD BTB untrain return */

#define X86_FEATURE_USE_IBPB_FW		(11*32+16) /* "" Use IBPB during runtime firmware calls */

#define X86_FEATURE_RSB_VMEXIT_LITE	(11*32+17) /* "" Fill RSB on VM exit when EIBRS is enabled */



/* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */

#define X86_FEATURE_AVX_VNNI		(12*32+ 4) /* AVX VNNI instructions */
@@ -457,5 +458,6 @@ 
#define X86_BUG_SRBDS			X86_BUG(24) /* CPU may leak RNG bits if not mitigated */

#define X86_BUG_MMIO_STALE_DATA		X86_BUG(25) /* CPU is affected by Processor MMIO Stale Data vulnerabilities */

#define X86_BUG_RETBLEED		X86_BUG(26) /* CPU is affected by RETBleed */

#define X86_BUG_EIBRS_PBRSB		X86_BUG(27) /* EIBRS is vulnerable to Post Barrier RSB Predictions */



#endif /* _ASM_X86_CPUFEATURES_H */

arch/x86/include/asm/msr-index.h

+4 −0
Original line number Diff line number Diff line
@@ -150,6 +150,10 @@ 
						 * are restricted to targets in

						 * kernel.

						 */

#define ARCH_CAP_PBRSB_NO		BIT(24)	/*

						 * Not susceptible to Post-Barrier

						 * Return Stack Buffer Predictions.

						 */



#define MSR_IA32_FLUSH_CMD		0x0000010b

#define L1D_FLUSH			BIT(0)	/*

arch/x86/include/asm/nospec-branch.h

+19 −2
Original line number Diff line number Diff line
@@ -60,7 +60,9 @@ 
774:						\

	add	$(BITS_PER_LONG/8) * 2, sp;	\

	dec	reg;				\

	jnz	771b;

	jnz	771b;				\

	/* barrier for jnz misprediction */	\

	lfence;



#ifdef __ASSEMBLY__
@@ -130,13 +132,28 @@ 
#endif

.endm



.macro ISSUE_UNBALANCED_RET_GUARD

	ANNOTATE_INTRA_FUNCTION_CALL

	call .Lunbalanced_ret_guard_\@

	int3

.Lunbalanced_ret_guard_\@:

	add $(BITS_PER_LONG/8), %_ASM_SP

	lfence

.endm



 /*

  * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP

  * monstrosity above, manually.

  */

.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req

.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req ftr2

.ifb \ftr2

	ALTERNATIVE "jmp .Lskip_rsb_\@", "", \ftr

.else

	ALTERNATIVE_2 "jmp .Lskip_rsb_\@", "", \ftr, "jmp .Lunbalanced_\@", \ftr2

.endif

	__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)

.Lunbalanced_\@:

	ISSUE_UNBALANCED_RET_GUARD

.Lskip_rsb_\@:

.endm

arch/x86/kernel/cpu/bugs.c

+63 −23
Original line number Diff line number Diff line
@@ -1335,6 +1335,53 @@ static void __init spec_ctrl_disable_kernel_rrsba(void) 
	}

}



static void __init spectre_v2_determine_rsb_fill_type_at_vmexit(enum spectre_v2_mitigation mode)

{

	/*

	 * Similar to context switches, there are two types of RSB attacks

	 * after VM exit:

	 *

	 * 1) RSB underflow

	 *

	 * 2) Poisoned RSB entry

	 *

	 * When retpoline is enabled, both are mitigated by filling/clearing

	 * the RSB.

	 *

	 * When IBRS is enabled, while #1 would be mitigated by the IBRS branch

	 * prediction isolation protections, RSB still needs to be cleared

	 * because of #2.  Note that SMEP provides no protection here, unlike

	 * user-space-poisoned RSB entries.

	 *

	 * eIBRS should protect against RSB poisoning, but if the EIBRS_PBRSB

	 * bug is present then a LITE version of RSB protection is required,

	 * just a single call needs to retire before a RET is executed.

	 */

	switch (mode) {

	case SPECTRE_V2_NONE:

		return;



	case SPECTRE_V2_EIBRS_LFENCE:

	case SPECTRE_V2_EIBRS:

		if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) {

			setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT_LITE);

			pr_info("Spectre v2 / PBRSB-eIBRS: Retire a single CALL on VMEXIT\n");

		}

		return;



	case SPECTRE_V2_EIBRS_RETPOLINE:

	case SPECTRE_V2_RETPOLINE:

	case SPECTRE_V2_LFENCE:

	case SPECTRE_V2_IBRS:

		setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT);

		pr_info("Spectre v2 / SpectreRSB : Filling RSB on VMEXIT\n");

		return;

	}



	pr_warn_once("Unknown Spectre v2 mode, disabling RSB mitigation at VM exit");

	dump_stack();

}



static void __init spectre_v2_select_mitigation(void)

{

	enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
@@ -1485,28 +1532,7 @@ static void __init spectre_v2_select_mitigation(void) 
	setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);

	pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");



	/*

	 * Similar to context switches, there are two types of RSB attacks

	 * after vmexit:

	 *

	 * 1) RSB underflow

	 *

	 * 2) Poisoned RSB entry

	 *

	 * When retpoline is enabled, both are mitigated by filling/clearing

	 * the RSB.

	 *

	 * When IBRS is enabled, while #1 would be mitigated by the IBRS branch

	 * prediction isolation protections, RSB still needs to be cleared

	 * because of #2.  Note that SMEP provides no protection here, unlike

	 * user-space-poisoned RSB entries.

	 *

	 * eIBRS, on the other hand, has RSB-poisoning protections, so it

	 * doesn't need RSB clearing after vmexit.

	 */

	if (boot_cpu_has(X86_FEATURE_RETPOLINE) ||

	    boot_cpu_has(X86_FEATURE_KERNEL_IBRS))

		setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT);

	spectre_v2_determine_rsb_fill_type_at_vmexit(mode);



	/*

	 * Retpoline protects the kernel, but doesn't protect firmware.  IBRS
@@ -2292,6 +2318,19 @@ static char *ibpb_state(void) 
	return "";

}



static char *pbrsb_eibrs_state(void)

{

	if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) {

		if (boot_cpu_has(X86_FEATURE_RSB_VMEXIT_LITE) ||

		    boot_cpu_has(X86_FEATURE_RSB_VMEXIT))

			return ", PBRSB-eIBRS: SW sequence";

		else

			return ", PBRSB-eIBRS: Vulnerable";

	} else {

		return ", PBRSB-eIBRS: Not affected";

	}

}



static ssize_t spectre_v2_show_state(char *buf)

{

	if (spectre_v2_enabled == SPECTRE_V2_LFENCE)
@@ -2304,12 +2343,13 @@ static ssize_t spectre_v2_show_state(char *buf) 
	    spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)

		return sprintf(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n");



	return sprintf(buf, "%s%s%s%s%s%s\n",

	return sprintf(buf, "%s%s%s%s%s%s%s\n",

		       spectre_v2_strings[spectre_v2_enabled],

		       ibpb_state(),

		       boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",

		       stibp_state(),

		       boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",

		       pbrsb_eibrs_state(),

		       spectre_v2_module_string());

}