Commit fe7f895d authored by Sean Christopherson's avatar Sean Christopherson Committed by Paolo Bonzini
Browse files

KVM: VMX: Shadow VMCS secondary execution controls 



Prepare to shadow all major control fields on a per-VMCS basis, which
allows KVM to avoid costly VMWRITEs when switching between vmcs01 and
vmcs02.

Signed-off-by: default avatarSean Christopherson <sean.j.christopherson@intel.com>
Signed-off-by: default avatarPaolo Bonzini <pbonzini@redhat.com>
parent 2183f564

arch/x86/kvm/vmx/nested.c

+6 −6
Original line number Diff line number Diff line
@@ -192,7 +192,7 @@ static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator) 


static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx)

{

	vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL, SECONDARY_EXEC_SHADOW_VMCS);

	secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_SHADOW_VMCS);

	vmcs_write64(VMCS_LINK_POINTER, -1ull);

}
@@ -287,6 +287,7 @@ static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs) 
	vm_exit_controls_reset_shadow(vmx);

	pin_controls_reset_shadow(vmx);

	exec_controls_reset_shadow(vmx);

	secondary_exec_controls_reset_shadow(vmx);

	vmx_segment_cache_clear(vmx);

}
@@ -2083,7 +2084,7 @@ static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 
			vmcs_write16(GUEST_INTR_STATUS,

				vmcs12->guest_intr_status);



		vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);

		secondary_exec_controls_init(vmx, exec_control);

	}



	/*
@@ -2853,7 +2854,7 @@ static void nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) 
			hpa = page_to_phys(vmx->nested.apic_access_page);

			vmcs_write64(APIC_ACCESS_ADDR, hpa);

		} else {

			vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL,

			secondary_exec_controls_clearbit(vmx,

				SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);

		}

	}
@@ -4667,8 +4668,7 @@ static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr) 
{

	vmx->nested.current_vmptr = vmptr;

	if (enable_shadow_vmcs) {

		vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL,

			      SECONDARY_EXEC_SHADOW_VMCS);

		secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_SHADOW_VMCS);

		vmcs_write64(VMCS_LINK_POINTER,

			     __pa(vmx->vmcs01.shadow_vmcs));

		vmx->nested.need_vmcs12_to_shadow_sync = true;

arch/x86/kvm/vmx/vmx.c

+20 −20
Original line number Diff line number Diff line
@@ -2909,6 +2909,7 @@ void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) 


int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)

{

	struct vcpu_vmx *vmx = to_vmx(vcpu);

	/*

	 * Pass through host's Machine Check Enable value to hw_cr4, which

	 * is in force while we are in guest mode.  Do not let guests control
@@ -2919,20 +2920,19 @@ int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 
	hw_cr4 = (cr4_read_shadow() & X86_CR4_MCE) | (cr4 & ~X86_CR4_MCE);

	if (enable_unrestricted_guest)

		hw_cr4 |= KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST;

	else if (to_vmx(vcpu)->rmode.vm86_active)

	else if (vmx->rmode.vm86_active)

		hw_cr4 |= KVM_RMODE_VM_CR4_ALWAYS_ON;

	else

		hw_cr4 |= KVM_PMODE_VM_CR4_ALWAYS_ON;



	if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated()) {

		if (cr4 & X86_CR4_UMIP) {

			vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL,

				SECONDARY_EXEC_DESC);

			secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_DESC);

			hw_cr4 &= ~X86_CR4_UMIP;

		} else if (!is_guest_mode(vcpu) ||

			!nested_cpu_has2(get_vmcs12(vcpu), SECONDARY_EXEC_DESC))

			vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL,

					SECONDARY_EXEC_DESC);

			!nested_cpu_has2(get_vmcs12(vcpu), SECONDARY_EXEC_DESC)) {

			secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_DESC);

		}

	}



	if (cr4 & X86_CR4_VMXE) {
@@ -2947,7 +2947,7 @@ int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 
			return 1;

	}



	if (to_vmx(vcpu)->nested.vmxon && !nested_cr4_valid(vcpu, cr4))

	if (vmx->nested.vmxon && !nested_cr4_valid(vcpu, cr4))

		return 1;



	vcpu->arch.cr4 = cr4;
@@ -3565,7 +3565,7 @@ static u8 vmx_msr_bitmap_mode(struct kvm_vcpu *vcpu) 
	u8 mode = 0;



	if (cpu_has_secondary_exec_ctrls() &&

	    (vmcs_read32(SECONDARY_VM_EXEC_CONTROL) &

	    (secondary_exec_controls_get(to_vmx(vcpu)) &

	     SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) {

		mode |= MSR_BITMAP_MODE_X2APIC;

		if (enable_apicv && kvm_vcpu_apicv_active(vcpu))
@@ -3845,11 +3845,11 @@ static void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) 
	pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx));

	if (cpu_has_secondary_exec_ctrls()) {

		if (kvm_vcpu_apicv_active(vcpu))

			vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL,

			secondary_exec_controls_setbit(vmx,

				      SECONDARY_EXEC_APIC_REGISTER_VIRT |

				      SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);

		else

			vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL,

			secondary_exec_controls_clearbit(vmx,

					SECONDARY_EXEC_APIC_REGISTER_VIRT |

					SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);

	}
@@ -4047,8 +4047,7 @@ static void vmx_vcpu_setup(struct vcpu_vmx *vmx) 


	if (cpu_has_secondary_exec_ctrls()) {

		vmx_compute_secondary_exec_control(vmx);

		vmcs_write32(SECONDARY_VM_EXEC_CONTROL,

			     vmx->secondary_exec_control);

		secondary_exec_controls_init(vmx, vmx->secondary_exec_control);

	}



	if (kvm_vcpu_apicv_active(&vmx->vcpu)) {
@@ -5969,6 +5968,7 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) 


void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu)

{

	struct vcpu_vmx *vmx = to_vmx(vcpu);

	u32 sec_exec_control;



	if (!lapic_in_kernel(vcpu))
@@ -5980,11 +5980,11 @@ void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu) 


	/* Postpone execution until vmcs01 is the current VMCS. */

	if (is_guest_mode(vcpu)) {

		to_vmx(vcpu)->nested.change_vmcs01_virtual_apic_mode = true;

		vmx->nested.change_vmcs01_virtual_apic_mode = true;

		return;

	}



	sec_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);

	sec_exec_control = secondary_exec_controls_get(vmx);

	sec_exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |

			      SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE);
@@ -6006,7 +6006,7 @@ void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu) 
				SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;

		break;

	}

	vmcs_write32(SECONDARY_VM_EXEC_CONTROL, sec_exec_control);

	secondary_exec_controls_set(vmx, sec_exec_control);



	vmx_update_msr_bitmap(vcpu);

}
@@ -6827,7 +6827,7 @@ static int vmx_get_lpage_level(void) 
		return PT_PDPE_LEVEL;

}



static void vmcs_set_secondary_exec_control(u32 new_ctl)

static void vmcs_set_secondary_exec_control(struct vcpu_vmx *vmx)

{

	/*

	 * These bits in the secondary execution controls field
@@ -6841,10 +6841,10 @@ static void vmcs_set_secondary_exec_control(u32 new_ctl) 
		SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |

		SECONDARY_EXEC_DESC;



	u32 cur_ctl = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);

	u32 new_ctl = vmx->secondary_exec_control;

	u32 cur_ctl = secondary_exec_controls_get(vmx);



	vmcs_write32(SECONDARY_VM_EXEC_CONTROL,

		     (new_ctl & ~mask) | (cur_ctl & mask));

	secondary_exec_controls_set(vmx, (new_ctl & ~mask) | (cur_ctl & mask));

}



/*
@@ -6982,7 +6982,7 @@ static void vmx_cpuid_update(struct kvm_vcpu *vcpu) 


	if (cpu_has_secondary_exec_ctrls()) {

		vmx_compute_secondary_exec_control(vmx);

		vmcs_set_secondary_exec_control(vmx->secondary_exec_control);

		vmcs_set_secondary_exec_control(vmx);

	}



	if (nested_vmx_allowed(vcpu))

arch/x86/kvm/vmx/vmx.h

+2 −0
Original line number Diff line number Diff line
@@ -90,6 +90,7 @@ struct vmx_controls_shadow { 
	u32 vm_exit;

	u32 pin;

	u32 exec;

	u32 secondary_exec;

};



/*
@@ -427,6 +428,7 @@ BUILD_CONTROLS_SHADOW(vm_entry, VM_ENTRY_CONTROLS) 
BUILD_CONTROLS_SHADOW(vm_exit, VM_EXIT_CONTROLS)

BUILD_CONTROLS_SHADOW(pin, PIN_BASED_VM_EXEC_CONTROL)

BUILD_CONTROLS_SHADOW(exec, CPU_BASED_VM_EXEC_CONTROL)

BUILD_CONTROLS_SHADOW(secondary_exec, SECONDARY_VM_EXEC_CONTROL)



static inline void vmx_segment_cache_clear(struct vcpu_vmx *vmx)

{