Commit bc908e09 authored by Sean Christopherson's avatar Sean Christopherson Committed by Thomas Gleixner
Browse files

KVM: x86: Consolidate guest enter/exit logic to common helpers 



Move the enter/exit logic in {svm,vmx}_vcpu_enter_exit() to common
helpers.  Opportunistically update the somewhat stale comment about the
updates needing to occur immediately after VM-Exit.

No functional change intended.

Signed-off-by: default avatarSean Christopherson <seanjc@google.com>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/r/20210505002735.1684165-9-seanjc@google.com
parent 1ca0016c

arch/x86/kvm/svm/svm.c

+2 −37
Original line number Diff line number Diff line
@@ -3710,25 +3710,7 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) 
	struct vcpu_svm *svm = to_svm(vcpu);

	unsigned long vmcb_pa = svm->current_vmcb->pa;



	/*

	 * VMENTER enables interrupts (host state), but the kernel state is

	 * interrupts disabled when this is invoked. Also tell RCU about

	 * it. This is the same logic as for exit_to_user_mode().

	 *

	 * This ensures that e.g. latency analysis on the host observes

	 * guest mode as interrupt enabled.

	 *

	 * guest_enter_irqoff() informs context tracking about the

	 * transition to guest mode and if enabled adjusts RCU state

	 * accordingly.

	 */

	instrumentation_begin();

	trace_hardirqs_on_prepare();

	lockdep_hardirqs_on_prepare(CALLER_ADDR0);

	instrumentation_end();



	guest_enter_irqoff();

	lockdep_hardirqs_on(CALLER_ADDR0);

	kvm_guest_enter_irqoff();



	if (sev_es_guest(vcpu->kvm)) {

		__svm_sev_es_vcpu_run(vmcb_pa);
@@ -3748,24 +3730,7 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) 
		vmload(__sme_page_pa(sd->save_area));

	}



	/*

	 * VMEXIT disables interrupts (host state), but tracing and lockdep

	 * have them in state 'on' as recorded before entering guest mode.

	 * Same as enter_from_user_mode().

	 *

	 * context_tracking_guest_exit() restores host context and reinstates

	 * RCU if enabled and required.

	 *

	 * This needs to be done before the below as native_read_msr()

	 * contains a tracepoint and x86_spec_ctrl_restore_host() calls

	 * into world and some more.

	 */

	lockdep_hardirqs_off(CALLER_ADDR0);

	context_tracking_guest_exit();



	instrumentation_begin();

	trace_hardirqs_off_finish();

	instrumentation_end();

	kvm_guest_exit_irqoff();

}



static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu)

arch/x86/kvm/vmx/vmx.c

+2 −37
Original line number Diff line number Diff line
@@ -6664,25 +6664,7 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu) 
static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,

					struct vcpu_vmx *vmx)

{

	/*

	 * VMENTER enables interrupts (host state), but the kernel state is

	 * interrupts disabled when this is invoked. Also tell RCU about

	 * it. This is the same logic as for exit_to_user_mode().

	 *

	 * This ensures that e.g. latency analysis on the host observes

	 * guest mode as interrupt enabled.

	 *

	 * guest_enter_irqoff() informs context tracking about the

	 * transition to guest mode and if enabled adjusts RCU state

	 * accordingly.

	 */

	instrumentation_begin();

	trace_hardirqs_on_prepare();

	lockdep_hardirqs_on_prepare(CALLER_ADDR0);

	instrumentation_end();



	guest_enter_irqoff();

	lockdep_hardirqs_on(CALLER_ADDR0);

	kvm_guest_enter_irqoff();



	/* L1D Flush includes CPU buffer clear to mitigate MDS */

	if (static_branch_unlikely(&vmx_l1d_should_flush))
@@ -6698,24 +6680,7 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, 


	vcpu->arch.cr2 = native_read_cr2();



	/*

	 * VMEXIT disables interrupts (host state), but tracing and lockdep

	 * have them in state 'on' as recorded before entering guest mode.

	 * Same as enter_from_user_mode().

	 *

	 * context_tracking_guest_exit() restores host context and reinstates

	 * RCU if enabled and required.

	 *

	 * This needs to be done before the below as native_read_msr()

	 * contains a tracepoint and x86_spec_ctrl_restore_host() calls

	 * into world and some more.

	 */

	lockdep_hardirqs_off(CALLER_ADDR0);

	context_tracking_guest_exit();



	instrumentation_begin();

	trace_hardirqs_off_finish();

	instrumentation_end();

	kvm_guest_exit_irqoff();

}



static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu)

arch/x86/kvm/x86.h

+45 −0
Original line number Diff line number Diff line
@@ -8,6 +8,51 @@ 
#include "kvm_cache_regs.h"

#include "kvm_emulate.h"



static __always_inline void kvm_guest_enter_irqoff(void)

{

	/*

	 * VMENTER enables interrupts (host state), but the kernel state is

	 * interrupts disabled when this is invoked. Also tell RCU about

	 * it. This is the same logic as for exit_to_user_mode().

	 *

	 * This ensures that e.g. latency analysis on the host observes

	 * guest mode as interrupt enabled.

	 *

	 * guest_enter_irqoff() informs context tracking about the

	 * transition to guest mode and if enabled adjusts RCU state

	 * accordingly.

	 */

	instrumentation_begin();

	trace_hardirqs_on_prepare();

	lockdep_hardirqs_on_prepare(CALLER_ADDR0);

	instrumentation_end();



	guest_enter_irqoff();

	lockdep_hardirqs_on(CALLER_ADDR0);

}



static __always_inline void kvm_guest_exit_irqoff(void)

{

	/*

	 * VMEXIT disables interrupts (host state), but tracing and lockdep

	 * have them in state 'on' as recorded before entering guest mode.

	 * Same as enter_from_user_mode().

	 *

	 * context_tracking_guest_exit() restores host context and reinstates

	 * RCU if enabled and required.

	 *

	 * This needs to be done immediately after VM-Exit, before any code

	 * that might contain tracepoints or call out to the greater world,

	 * e.g. before x86_spec_ctrl_restore_host().

	 */

	lockdep_hardirqs_off(CALLER_ADDR0);

	context_tracking_guest_exit();



	instrumentation_begin();

	trace_hardirqs_off_finish();

	instrumentation_end();

}



#define KVM_NESTED_VMENTER_CONSISTENCY_CHECK(consistency_check)		\

({									\

	bool failed = (consistency_check);				\