Commit 2fdcc1b3 authored by Paolo Bonzini's avatar Paolo Bonzini Committed by Sean Christopherson
Browse files

KVM: x86/mmu: Avoid indirect call for get_cr3 



Most of the time, calls to get_guest_pgd result in calling
kvm_read_cr3 (the exception is only nested TDP).  Hardcode
the default instead of using the get_cr3 function, avoiding
a retpoline if they are enabled.

Signed-off-by: default avatarPaolo Bonzini <pbonzini@redhat.com>
Signed-off-by: default avatarMathias Krause <minipli@grsecurity.net>
Link: https://lore.kernel.org/r/20230322013731.102955-2-minipli@grsecurity.net


Signed-off-by: default avatarSean Christopherson <seanjc@google.com>
parent f3d90f90

arch/x86/kvm/mmu/mmu.c

+20 −11
Original line number Diff line number Diff line
@@ -256,6 +256,20 @@ static struct kvm_mmu_role_regs vcpu_to_role_regs(struct kvm_vcpu *vcpu) 
	return regs;

}



static unsigned long get_guest_cr3(struct kvm_vcpu *vcpu)

{

	return kvm_read_cr3(vcpu);

}



static inline unsigned long kvm_mmu_get_guest_pgd(struct kvm_vcpu *vcpu,

						  struct kvm_mmu *mmu)

{

	if (IS_ENABLED(CONFIG_RETPOLINE) && mmu->get_guest_pgd == get_guest_cr3)

		return kvm_read_cr3(vcpu);



	return mmu->get_guest_pgd(vcpu);

}



static inline bool kvm_available_flush_tlb_with_range(void)

{

	return kvm_x86_ops.tlb_remote_flush_with_range;
@@ -3801,7 +3815,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) 
	int quadrant, i, r;

	hpa_t root;



	root_pgd = mmu->get_guest_pgd(vcpu);

	root_pgd = kvm_mmu_get_guest_pgd(vcpu, mmu);

	root_gfn = root_pgd >> PAGE_SHIFT;



	if (mmu_check_root(vcpu, root_gfn))
@@ -4251,7 +4265,7 @@ static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, 
	arch.token = alloc_apf_token(vcpu);

	arch.gfn = gfn;

	arch.direct_map = vcpu->arch.mmu->root_role.direct;

	arch.cr3 = vcpu->arch.mmu->get_guest_pgd(vcpu);

	arch.cr3 = kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu);



	return kvm_setup_async_pf(vcpu, cr2_or_gpa,

				  kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch);
@@ -4270,7 +4284,7 @@ void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work) 
		return;



	if (!vcpu->arch.mmu->root_role.direct &&

	      work->arch.cr3 != vcpu->arch.mmu->get_guest_pgd(vcpu))

	      work->arch.cr3 != kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu))

		return;



	kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, 0, true, NULL);
@@ -4673,11 +4687,6 @@ void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd) 
}

EXPORT_SYMBOL_GPL(kvm_mmu_new_pgd);



static unsigned long get_cr3(struct kvm_vcpu *vcpu)

{

	return kvm_read_cr3(vcpu);

}



static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,

			   unsigned int access)

{
@@ -5223,7 +5232,7 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu, 
	context->root_role.word = root_role.word;

	context->page_fault = kvm_tdp_page_fault;

	context->sync_spte = NULL;

	context->get_guest_pgd = get_cr3;

	context->get_guest_pgd = get_guest_cr3;

	context->get_pdptr = kvm_pdptr_read;

	context->inject_page_fault = kvm_inject_page_fault;
@@ -5372,7 +5381,7 @@ static void init_kvm_softmmu(struct kvm_vcpu *vcpu, 


	kvm_init_shadow_mmu(vcpu, cpu_role);



	context->get_guest_pgd     = get_cr3;

	context->get_guest_pgd     = get_guest_cr3;

	context->get_pdptr         = kvm_pdptr_read;

	context->inject_page_fault = kvm_inject_page_fault;

}
@@ -5386,7 +5395,7 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu, 
		return;



	g_context->cpu_role.as_u64   = new_mode.as_u64;

	g_context->get_guest_pgd     = get_cr3;

	g_context->get_guest_pgd     = get_guest_cr3;

	g_context->get_pdptr         = kvm_pdptr_read;

	g_context->inject_page_fault = kvm_inject_page_fault;

arch/x86/kvm/mmu/paging_tmpl.h

+1 −1
Original line number Diff line number Diff line
@@ -324,7 +324,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, 
	trace_kvm_mmu_pagetable_walk(addr, access);

retry_walk:

	walker->level = mmu->cpu_role.base.level;

	pte           = mmu->get_guest_pgd(vcpu);

	pte           = kvm_mmu_get_guest_pgd(vcpu, mmu);

	have_ad       = PT_HAVE_ACCESSED_DIRTY(mmu);



#if PTTYPE == 64