Commit 8a9f566a authored by David Matlack's avatar David Matlack Committed by Paolo Bonzini
Browse files

KVM: x86/mmu: Avoid memslot lookup in rmap_add 



Avoid the memslot lookup in rmap_add, by passing it down from the fault
handling code to mmu_set_spte and then to rmap_add.

No functional change intended.

Signed-off-by: default avatarDavid Matlack <dmatlack@google.com>
Message-Id: <20210813203504.2742757-6-dmatlack@google.com>
Signed-off-by: default avatarPaolo Bonzini <pbonzini@redhat.com>
parent a12f4381

arch/x86/kvm/mmu/mmu.c

+7 −20
Original line number Diff line number Diff line
@@ -1625,16 +1625,15 @@ static bool kvm_test_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, 


#define RMAP_RECYCLE_THRESHOLD 1000



static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)

static void rmap_add(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot,

		     u64 *spte, gfn_t gfn)

{

	struct kvm_memory_slot *slot;

	struct kvm_mmu_page *sp;

	struct kvm_rmap_head *rmap_head;

	int rmap_count;



	sp = sptep_to_sp(spte);

	kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);

	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);

	rmap_head = gfn_to_rmap(gfn, sp->role.level, slot);

	rmap_count = pte_list_add(vcpu, spte, rmap_head);
@@ -2674,8 +2673,8 @@ int mmu_try_to_unsync_pages(struct kvm_vcpu *vcpu, gfn_t gfn, bool can_unsync, 
	return 0;

}



static int mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,

			unsigned int pte_access, gfn_t gfn,

static int mmu_set_spte(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot,

			u64 *sptep, unsigned int pte_access, gfn_t gfn,

			kvm_pfn_t pfn, struct kvm_page_fault *fault)

{

	struct kvm_mmu_page *sp = sptep_to_sp(sptep);
@@ -2744,24 +2743,12 @@ static int mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, 
	if (!was_rmapped) {

		WARN_ON_ONCE(ret == RET_PF_SPURIOUS);

		kvm_update_page_stats(vcpu->kvm, level, 1);

		rmap_add(vcpu, sptep, gfn);

		rmap_add(vcpu, slot, sptep, gfn);

	}



	return ret;

}



static kvm_pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,

				     bool no_dirty_log)

{

	struct kvm_memory_slot *slot;



	slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log);

	if (!slot)

		return KVM_PFN_ERR_FAULT;



	return gfn_to_pfn_memslot_atomic(slot, gfn);

}



static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,

				    struct kvm_mmu_page *sp,

				    u64 *start, u64 *end)
@@ -2782,7 +2769,7 @@ static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu, 
		return -1;



	for (i = 0; i < ret; i++, gfn++, start++) {

		mmu_set_spte(vcpu, start, access, gfn,

		mmu_set_spte(vcpu, slot, start, access, gfn,

			     page_to_pfn(pages[i]), NULL);

		put_page(pages[i]);

	}
@@ -2984,7 +2971,7 @@ static int __direct_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) 
	if (WARN_ON_ONCE(it.level != fault->goal_level))

		return -EFAULT;



	ret = mmu_set_spte(vcpu, it.sptep, ACC_ALL,

	ret = mmu_set_spte(vcpu, fault->slot, it.sptep, ACC_ALL,

			   base_gfn, fault->pfn, fault);

	if (ret == RET_PF_SPURIOUS)

		return ret;

arch/x86/kvm/mmu/paging_tmpl.h

+9 −3
Original line number Diff line number Diff line
@@ -561,6 +561,7 @@ static bool 
FNAME(prefetch_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,

		     u64 *spte, pt_element_t gpte, bool no_dirty_log)

{

	struct kvm_memory_slot *slot;

	unsigned pte_access;

	gfn_t gfn;

	kvm_pfn_t pfn;
@@ -573,12 +574,17 @@ FNAME(prefetch_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, 
	gfn = gpte_to_gfn(gpte);

	pte_access = sp->role.access & FNAME(gpte_access)(gpte);

	FNAME(protect_clean_gpte)(vcpu->arch.mmu, &pte_access, gpte);

	pfn = pte_prefetch_gfn_to_pfn(vcpu, gfn,



	slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn,

			no_dirty_log && (pte_access & ACC_WRITE_MASK));

	if (!slot)

		return false;



	pfn = gfn_to_pfn_memslot_atomic(slot, gfn);

	if (is_error_pfn(pfn))

		return false;



	mmu_set_spte(vcpu, spte, pte_access, gfn, pfn, NULL);

	mmu_set_spte(vcpu, slot, spte, pte_access, gfn, pfn, NULL);

	kvm_release_pfn_clean(pfn);

	return true;

}
@@ -757,7 +763,7 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, 
	if (WARN_ON_ONCE(it.level != fault->goal_level))

		return -EFAULT;



	ret = mmu_set_spte(vcpu, it.sptep, gw->pte_access,

	ret = mmu_set_spte(vcpu, fault->slot, it.sptep, gw->pte_access,

			   base_gfn, fault->pfn, fault);

	if (ret == RET_PF_SPURIOUS)

		return ret;