KVM: SVM: move MSR_IA32_SPEC_CTRL save/restore to assembly (9f2febf3) · Commits · EulixOS / Software / Kernel

arch/x86/kernel/cpu/bugs.c

+3 −10

Original line number	Original line	Diff line number	Diff line
	@@ -196,22 +196,15 @@ void __init check_bugs(void)
	}		}

	/*		/*
	* NOTE: This function is only called for SVM. VMX spec_ctrl handling is		* NOTE: This function is only called for SVM, since Intel uses
	* done in vmenter.S.		* MSR_IA32_SPEC_CTRL for SSBD.
	*/		*/
	void		void
	x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)		x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
	{		{
	u64 msrval, guestval = guest_spec_ctrl, hostval = spec_ctrl_current();		u64 guestval, hostval;
	struct thread_info *ti = current_thread_info();		struct thread_info *ti = current_thread_info();

	if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
	if (hostval != guestval) {
	msrval = setguest ? guestval : hostval;
	wrmsrl(MSR_IA32_SPEC_CTRL, msrval);
	}
	}

	/*		/*
	* If SSBD is not handled in MSR_SPEC_CTRL on AMD, update		* If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
	* MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.		* MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.

arch/x86/kvm/kvm-asm-offsets.c

+1 −0

Original line number	Original line	Diff line number	Diff line
	@@ -16,6 +16,7 @@ static void __used common(void)
	BLANK();		BLANK();
	OFFSET(SVM_vcpu_arch_regs, vcpu_svm, vcpu.arch.regs);		OFFSET(SVM_vcpu_arch_regs, vcpu_svm, vcpu.arch.regs);
	OFFSET(SVM_current_vmcb, vcpu_svm, current_vmcb);		OFFSET(SVM_current_vmcb, vcpu_svm, current_vmcb);
			OFFSET(SVM_spec_ctrl, vcpu_svm, spec_ctrl);
	OFFSET(SVM_vmcb01, vcpu_svm, vmcb01);		OFFSET(SVM_vmcb01, vcpu_svm, vmcb01);
	OFFSET(KVM_VMCB_pa, kvm_vmcb_info, pa);		OFFSET(KVM_VMCB_pa, kvm_vmcb_info, pa);
	OFFSET(SD_save_area_pa, svm_cpu_data, save_area_pa);		OFFSET(SD_save_area_pa, svm_cpu_data, save_area_pa);

arch/x86/kvm/svm/svm.c

+14 −23

Original line number	Original line	Diff line number	Diff line
	@@ -720,6 +720,15 @@ static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
	u32 offset;		u32 offset;
	u32 *msrpm;		u32 *msrpm;

			/*
			* For non-nested case:
			* If the L01 MSR bitmap does not intercept the MSR, then we need to
			* save it.
			*
			* For nested case:
			* If the L02 MSR bitmap does not intercept the MSR, then we need to
			* save it.
			*/
	msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:		msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
	to_svm(vcpu)->msrpm;		to_svm(vcpu)->msrpm;

	@@ -3901,16 +3910,16 @@ static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
	return EXIT_FASTPATH_NONE;		return EXIT_FASTPATH_NONE;
	}		}

	static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu)		static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_intercepted)
	{		{
	struct vcpu_svm *svm = to_svm(vcpu);		struct vcpu_svm *svm = to_svm(vcpu);

	guest_state_enter_irqoff();		guest_state_enter_irqoff();

	if (sev_es_guest(vcpu->kvm))		if (sev_es_guest(vcpu->kvm))
	__svm_sev_es_vcpu_run(svm);		__svm_sev_es_vcpu_run(svm, spec_ctrl_intercepted);
	else		else
	__svm_vcpu_run(svm);		__svm_vcpu_run(svm, spec_ctrl_intercepted);

	guest_state_exit_irqoff();		guest_state_exit_irqoff();
	}		}
	@@ -3918,6 +3927,7 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu)
	static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu)		static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu)
	{		{
	struct vcpu_svm *svm = to_svm(vcpu);		struct vcpu_svm *svm = to_svm(vcpu);
			bool spec_ctrl_intercepted = msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL);

	trace_kvm_entry(vcpu);		trace_kvm_entry(vcpu);

	@@ -3976,26 +3986,7 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu)
	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))		if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
	x86_spec_ctrl_set_guest(svm->spec_ctrl, svm->virt_spec_ctrl);		x86_spec_ctrl_set_guest(svm->spec_ctrl, svm->virt_spec_ctrl);

	svm_vcpu_enter_exit(vcpu);		svm_vcpu_enter_exit(vcpu, spec_ctrl_intercepted);

	/*
	* We do not use IBRS in the kernel. If this vCPU has used the
	* SPEC_CTRL MSR it may have left it on; save the value and
	* turn it off. This is much more efficient than blindly adding
	* it to the atomic save/restore list. Especially as the former
	* (Saving guest MSRs on vmexit) doesn't even exist in KVM.
	*
	* For non-nested case:
	* If the L01 MSR bitmap does not intercept the MSR, then we need to
	* save it.
	*
	* For nested case:
	* If the L02 MSR bitmap does not intercept the MSR, then we need to
	* save it.
	*/
	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL) &&
	unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
	svm->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);

	if (!sev_es_guest(vcpu->kvm))		if (!sev_es_guest(vcpu->kvm))
	reload_tss(vcpu);		reload_tss(vcpu);

arch/x86/kvm/svm/svm.h

+2 −2

Original line number	Original line	Diff line number	Diff line
	@@ -682,7 +682,7 @@ void sev_es_unmap_ghcb(struct vcpu_svm *svm);

	/* vmenter.S */		/* vmenter.S */

	void __svm_sev_es_vcpu_run(struct vcpu_svm *svm);		void __svm_sev_es_vcpu_run(struct vcpu_svm *svm, bool spec_ctrl_intercepted);
	void __svm_vcpu_run(struct vcpu_svm *svm);		void __svm_vcpu_run(struct vcpu_svm *svm, bool spec_ctrl_intercepted);

	#endif		#endif

arch/x86/kvm/svm/vmenter.S

+116 −3

Original line number	Original line	Diff line number	Diff line
	@@ -32,9 +32,69 @@

	.section .noinstr.text, "ax"		.section .noinstr.text, "ax"

			.macro RESTORE_GUEST_SPEC_CTRL
			/* No need to do anything if SPEC_CTRL is unset or V_SPEC_CTRL is set */
			ALTERNATIVE_2 "", \
			"jmp 800f", X86_FEATURE_MSR_SPEC_CTRL, \
			"", X86_FEATURE_V_SPEC_CTRL
			801:
			.endm
			.macro RESTORE_GUEST_SPEC_CTRL_BODY
			800:
			/*
			* SPEC_CTRL handling: if the guest's SPEC_CTRL value differs from the
			* host's, write the MSR. This is kept out-of-line so that the common
			* case does not have to jump.
			*
			* IMPORTANT: To avoid RSB underflow attacks and any other nastiness,
			* there must not be any returns or indirect branches between this code
			* and vmentry.
			*/
			movl SVM_spec_ctrl(%_ASM_DI), %eax
			cmp PER_CPU_VAR(x86_spec_ctrl_current), %eax
			je 801b
			mov $MSR_IA32_SPEC_CTRL, %ecx
			xor %edx, %edx
			wrmsr
			jmp 801b
			.endm

			.macro RESTORE_HOST_SPEC_CTRL
			/* No need to do anything if SPEC_CTRL is unset or V_SPEC_CTRL is set */
			ALTERNATIVE_2 "", \
			"jmp 900f", X86_FEATURE_MSR_SPEC_CTRL, \
			"", X86_FEATURE_V_SPEC_CTRL
			901:
			.endm
			.macro RESTORE_HOST_SPEC_CTRL_BODY
			900:
			/* Same for after vmexit. */
			mov $MSR_IA32_SPEC_CTRL, %ecx

			/*
			* Load the value that the guest had written into MSR_IA32_SPEC_CTRL,
			* if it was not intercepted during guest execution.
			*/
			cmpb $0, (%_ASM_SP)
			jnz 998f
			rdmsr
			movl %eax, SVM_spec_ctrl(%_ASM_DI)
			998:

			/* Now restore the host value of the MSR if different from the guest's. */
			movl PER_CPU_VAR(x86_spec_ctrl_current), %eax
			cmp SVM_spec_ctrl(%_ASM_DI), %eax
			je 901b
			xor %edx, %edx
			wrmsr
			jmp 901b
			.endm


	/**		/**
	* __svm_vcpu_run - Run a vCPU via a transition to SVM guest mode		* __svm_vcpu_run - Run a vCPU via a transition to SVM guest mode
	* @svm: struct vcpu_svm *		* @svm: struct vcpu_svm *
			* @spec_ctrl_intercepted: bool
	*/		*/
	SYM_FUNC_START(__svm_vcpu_run)		SYM_FUNC_START(__svm_vcpu_run)
	push %_ASM_BP		push %_ASM_BP
	@@ -54,17 +114,26 @@ SYM_FUNC_START(__svm_vcpu_run)
	* order compared to when they are needed.		* order compared to when they are needed.
	*/		*/

			/* Accessed directly from the stack in RESTORE_HOST_SPEC_CTRL. */
			push %_ASM_ARG2

	/* Needed to restore access to percpu variables. */		/* Needed to restore access to percpu variables. */
	__ASM_SIZE(push) PER_CPU_VAR(svm_data + SD_save_area_pa)		__ASM_SIZE(push) PER_CPU_VAR(svm_data + SD_save_area_pa)

	/* Save @svm. */		/* Finally save @svm. */
	push %_ASM_ARG1		push %_ASM_ARG1

	.ifnc _ASM_ARG1, _ASM_DI		.ifnc _ASM_ARG1, _ASM_DI
	/* Move @svm to RDI. */		/*
			* Stash @svm in RDI early. On 32-bit, arguments are in RAX, RCX
			* and RDX which are clobbered by RESTORE_GUEST_SPEC_CTRL.
			*/
	mov %_ASM_ARG1, %_ASM_DI		mov %_ASM_ARG1, %_ASM_DI
	.endif		.endif

			/* Clobbers RAX, RCX, RDX. */
			RESTORE_GUEST_SPEC_CTRL

	/*		/*
	* Use a single vmcb (vmcb01 because it's always valid) for		* Use a single vmcb (vmcb01 because it's always valid) for
	* context switching guest state via VMLOAD/VMSAVE, that way		* context switching guest state via VMLOAD/VMSAVE, that way
	@@ -142,6 +211,9 @@ SYM_FUNC_START(__svm_vcpu_run)
	FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE		FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE
	#endif		#endif

			/* Clobbers RAX, RCX, RDX. */
			RESTORE_HOST_SPEC_CTRL

	/*		/*
	* Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be		* Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be
	* untrained as soon as we exit the VM and are back to the		* untrained as soon as we exit the VM and are back to the
	@@ -177,6 +249,9 @@ SYM_FUNC_START(__svm_vcpu_run)
	xor %r15d, %r15d		xor %r15d, %r15d
	#endif		#endif

			/* "Pop" @spec_ctrl_intercepted. */
			pop %_ASM_BX

	pop %_ASM_BX		pop %_ASM_BX

	#ifdef CONFIG_X86_64		#ifdef CONFIG_X86_64
	@@ -191,6 +266,9 @@ SYM_FUNC_START(__svm_vcpu_run)
	pop %_ASM_BP		pop %_ASM_BP
	RET		RET

			RESTORE_GUEST_SPEC_CTRL_BODY
			RESTORE_HOST_SPEC_CTRL_BODY

	10: cmpb $0, kvm_rebooting		10: cmpb $0, kvm_rebooting
	jne 2b		jne 2b
	ud2		ud2
	@@ -214,6 +292,7 @@ SYM_FUNC_END(__svm_vcpu_run)
	/**		/**
	* __svm_sev_es_vcpu_run - Run a SEV-ES vCPU via a transition to SVM guest mode		* __svm_sev_es_vcpu_run - Run a SEV-ES vCPU via a transition to SVM guest mode
	* @svm: struct vcpu_svm *		* @svm: struct vcpu_svm *
			* @spec_ctrl_intercepted: bool
	*/		*/
	SYM_FUNC_START(__svm_sev_es_vcpu_run)		SYM_FUNC_START(__svm_sev_es_vcpu_run)
	push %_ASM_BP		push %_ASM_BP
	@@ -228,8 +307,30 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run)
	#endif		#endif
	push %_ASM_BX		push %_ASM_BX

			/*
			* Save variables needed after vmexit on the stack, in inverse
			* order compared to when they are needed.
			*/

			/* Accessed directly from the stack in RESTORE_HOST_SPEC_CTRL. */
			push %_ASM_ARG2

			/* Save @svm. */
			push %_ASM_ARG1

			.ifnc _ASM_ARG1, _ASM_DI
			/*
			* Stash @svm in RDI early. On 32-bit, arguments are in RAX, RCX
			* and RDX which are clobbered by RESTORE_GUEST_SPEC_CTRL.
			*/
			mov %_ASM_ARG1, %_ASM_DI
			.endif

			/* Clobbers RAX, RCX, RDX. */
			RESTORE_GUEST_SPEC_CTRL

	/* Get svm->current_vmcb->pa into RAX. */		/* Get svm->current_vmcb->pa into RAX. */
	mov SVM_current_vmcb(%_ASM_ARG1), %_ASM_AX		mov SVM_current_vmcb(%_ASM_DI), %_ASM_AX
	mov KVM_VMCB_pa(%_ASM_AX), %_ASM_AX		mov KVM_VMCB_pa(%_ASM_AX), %_ASM_AX

	/* Enter guest mode */		/* Enter guest mode */
	@@ -239,11 +340,17 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run)

	2: cli		2: cli

			/* Pop @svm to RDI, guest registers have been saved already. */
			pop %_ASM_DI

	#ifdef CONFIG_RETPOLINE		#ifdef CONFIG_RETPOLINE
	/* IMPORTANT: Stuff the RSB immediately after VM-Exit, before RET! */		/* IMPORTANT: Stuff the RSB immediately after VM-Exit, before RET! */
	FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE		FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE
	#endif		#endif

			/* Clobbers RAX, RCX, RDX. */
			RESTORE_HOST_SPEC_CTRL

	/*		/*
	* Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be		* Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be
	* untrained as soon as we exit the VM and are back to the		* untrained as soon as we exit the VM and are back to the
	@@ -253,6 +360,9 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run)
	*/		*/
	UNTRAIN_RET		UNTRAIN_RET

			/* "Pop" @spec_ctrl_intercepted. */
			pop %_ASM_BX

	pop %_ASM_BX		pop %_ASM_BX

	#ifdef CONFIG_X86_64		#ifdef CONFIG_X86_64
	@@ -267,6 +377,9 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run)
	pop %_ASM_BP		pop %_ASM_BP
	RET		RET

			RESTORE_GUEST_SPEC_CTRL_BODY
			RESTORE_HOST_SPEC_CTRL_BODY

	3: cmpb $0, kvm_rebooting		3: cmpb $0, kvm_rebooting
	jne 2b		jne 2b
	ud2		ud2