Commit 2bb48074 authored by Marc Zyngier's avatar Marc Zyngier
Browse files

Merge branch kvm-arm64/mmu-rwlock into kvmarm-master/next 



* kvm-arm64/mmu-rwlock:
  : .
  : MMU locking optimisations from Jing Zhang, allowing permission
  : relaxations to occur in parallel.
  : .
  KVM: selftests: Add vgic initialization for dirty log perf test for ARM
  KVM: arm64: Add fast path to handle permission relaxation during dirty logging
  KVM: arm64: Use read/write spin lock for MMU protection

Signed-off-by: default avatarMarc Zyngier <maz@kernel.org>
parents b61fa004 c340f789

arch/arm64/include/asm/kvm_host.h

+2 −0
Original line number Original line Diff line number Diff line
@@ -50,6 +50,8 @@ 
#define KVM_DIRTY_LOG_MANUAL_CAPS   (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \

#define KVM_DIRTY_LOG_MANUAL_CAPS   (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \

				     KVM_DIRTY_LOG_INITIALLY_SET)

				     KVM_DIRTY_LOG_INITIALLY_SET)





#define KVM_HAVE_MMU_RWLOCK



/*

/*

 * Mode of operation configurable with kvm-arm.mode early param.

 * Mode of operation configurable with kvm-arm.mode early param.

 * See Documentation/admin-guide/kernel-parameters.txt for more information.

 * See Documentation/admin-guide/kernel-parameters.txt for more information.

arch/arm64/kvm/mmu.c

+31 −18
Original line number Original line Diff line number Diff line
@@ -58,7 +58,7 @@ static int stage2_apply_range(struct kvm *kvm, phys_addr_t addr, 
			break;

			break;





		if (resched && next != end)

		if (resched && next != end)

			cond_resched_lock(&kvm->mmu_lock);

			cond_resched_rwlock_write(&kvm->mmu_lock);

	} while (addr = next, addr != end);

	} while (addr = next, addr != end);





	return ret;

	return ret;
@@ -179,7 +179,7 @@ static void __unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 
	struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);

	struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);

	phys_addr_t end = start + size;

	phys_addr_t end = start + size;





	assert_spin_locked(&kvm->mmu_lock);

	lockdep_assert_held_write(&kvm->mmu_lock);

	WARN_ON(size & ~PAGE_MASK);

	WARN_ON(size & ~PAGE_MASK);

	WARN_ON(stage2_apply_range(kvm, start, end, kvm_pgtable_stage2_unmap,

	WARN_ON(stage2_apply_range(kvm, start, end, kvm_pgtable_stage2_unmap,

				   may_block));

				   may_block));
@@ -213,13 +213,13 @@ static void stage2_flush_vm(struct kvm *kvm) 
	int idx, bkt;

	int idx, bkt;





	idx = srcu_read_lock(&kvm->srcu);

	idx = srcu_read_lock(&kvm->srcu);

	spin_lock(&kvm->mmu_lock);

	write_lock(&kvm->mmu_lock);





	slots = kvm_memslots(kvm);

	slots = kvm_memslots(kvm);

	kvm_for_each_memslot(memslot, bkt, slots)

	kvm_for_each_memslot(memslot, bkt, slots)

		stage2_flush_memslot(kvm, memslot);

		stage2_flush_memslot(kvm, memslot);





	spin_unlock(&kvm->mmu_lock);

	write_unlock(&kvm->mmu_lock);

	srcu_read_unlock(&kvm->srcu, idx);

	srcu_read_unlock(&kvm->srcu, idx);

}

}
@@ -720,13 +720,13 @@ void stage2_unmap_vm(struct kvm *kvm) 




	idx = srcu_read_lock(&kvm->srcu);

	idx = srcu_read_lock(&kvm->srcu);

	mmap_read_lock(current->mm);

	mmap_read_lock(current->mm);

	spin_lock(&kvm->mmu_lock);

	write_lock(&kvm->mmu_lock);





	slots = kvm_memslots(kvm);

	slots = kvm_memslots(kvm);

	kvm_for_each_memslot(memslot, bkt, slots)

	kvm_for_each_memslot(memslot, bkt, slots)

		stage2_unmap_memslot(kvm, memslot);

		stage2_unmap_memslot(kvm, memslot);





	spin_unlock(&kvm->mmu_lock);

	write_unlock(&kvm->mmu_lock);

	mmap_read_unlock(current->mm);

	mmap_read_unlock(current->mm);

	srcu_read_unlock(&kvm->srcu, idx);

	srcu_read_unlock(&kvm->srcu, idx);

}

}
@@ -736,14 +736,14 @@ void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu) 
	struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);

	struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);

	struct kvm_pgtable *pgt = NULL;

	struct kvm_pgtable *pgt = NULL;





	spin_lock(&kvm->mmu_lock);

	write_lock(&kvm->mmu_lock);

	pgt = mmu->pgt;

	pgt = mmu->pgt;

	if (pgt) {

	if (pgt) {

		mmu->pgd_phys = 0;

		mmu->pgd_phys = 0;

		mmu->pgt = NULL;

		mmu->pgt = NULL;

		free_percpu(mmu->last_vcpu_ran);

		free_percpu(mmu->last_vcpu_ran);

	}

	}

	spin_unlock(&kvm->mmu_lock);

	write_unlock(&kvm->mmu_lock);





	if (pgt) {

	if (pgt) {

		kvm_pgtable_stage2_destroy(pgt);

		kvm_pgtable_stage2_destroy(pgt);
@@ -783,10 +783,10 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, 
		if (ret)

		if (ret)

			break;

			break;





		spin_lock(&kvm->mmu_lock);

		write_lock(&kvm->mmu_lock);

		ret = kvm_pgtable_stage2_map(pgt, addr, PAGE_SIZE, pa, prot,

		ret = kvm_pgtable_stage2_map(pgt, addr, PAGE_SIZE, pa, prot,

					     &cache);

					     &cache);

		spin_unlock(&kvm->mmu_lock);

		write_unlock(&kvm->mmu_lock);

		if (ret)

		if (ret)

			break;

			break;
@@ -834,9 +834,9 @@ static void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot) 
	start = memslot->base_gfn << PAGE_SHIFT;

	start = memslot->base_gfn << PAGE_SHIFT;

	end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT;

	end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT;





	spin_lock(&kvm->mmu_lock);

	write_lock(&kvm->mmu_lock);

	stage2_wp_range(&kvm->arch.mmu, start, end);

	stage2_wp_range(&kvm->arch.mmu, start, end);

	spin_unlock(&kvm->mmu_lock);

	write_unlock(&kvm->mmu_lock);

	kvm_flush_remote_tlbs(kvm);

	kvm_flush_remote_tlbs(kvm);

}

}
@@ -1080,6 +1080,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, 
	gfn_t gfn;

	gfn_t gfn;

	kvm_pfn_t pfn;

	kvm_pfn_t pfn;

	bool logging_active = memslot_is_logging(memslot);

	bool logging_active = memslot_is_logging(memslot);

	bool logging_perm_fault = false;

	unsigned long fault_level = kvm_vcpu_trap_get_fault_level(vcpu);

	unsigned long fault_level = kvm_vcpu_trap_get_fault_level(vcpu);

	unsigned long vma_pagesize, fault_granule;

	unsigned long vma_pagesize, fault_granule;

	enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R;

	enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R;
@@ -1114,6 +1115,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, 
	if (logging_active) {

	if (logging_active) {

		force_pte = true;

		force_pte = true;

		vma_shift = PAGE_SHIFT;

		vma_shift = PAGE_SHIFT;

		logging_perm_fault = (fault_status == FSC_PERM && write_fault);

	} else {

	} else {

		vma_shift = get_vma_page_shift(vma, hva);

		vma_shift = get_vma_page_shift(vma, hva);

	}

	}
@@ -1212,7 +1214,15 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, 
	if (exec_fault && device)

	if (exec_fault && device)

		return -ENOEXEC;

		return -ENOEXEC;





	spin_lock(&kvm->mmu_lock);

	/*

	 * To reduce MMU contentions and enhance concurrency during dirty

	 * logging dirty logging, only acquire read lock for permission

	 * relaxation.

	 */

	if (logging_perm_fault)

		read_lock(&kvm->mmu_lock);

	else

		write_lock(&kvm->mmu_lock);

	pgt = vcpu->arch.hw_mmu->pgt;

	pgt = vcpu->arch.hw_mmu->pgt;

	if (mmu_notifier_retry(kvm, mmu_seq))

	if (mmu_notifier_retry(kvm, mmu_seq))

		goto out_unlock;

		goto out_unlock;
@@ -1271,7 +1281,10 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, 
	}

	}





out_unlock:

out_unlock:

	spin_unlock(&kvm->mmu_lock);

	if (logging_perm_fault)

		read_unlock(&kvm->mmu_lock);

	else

		write_unlock(&kvm->mmu_lock);

	kvm_set_pfn_accessed(pfn);

	kvm_set_pfn_accessed(pfn);

	kvm_release_pfn_clean(pfn);

	kvm_release_pfn_clean(pfn);

	return ret != -EAGAIN ? ret : 0;

	return ret != -EAGAIN ? ret : 0;
@@ -1286,10 +1299,10 @@ static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa) 




	trace_kvm_access_fault(fault_ipa);

	trace_kvm_access_fault(fault_ipa);





	spin_lock(&vcpu->kvm->mmu_lock);

	write_lock(&vcpu->kvm->mmu_lock);

	mmu = vcpu->arch.hw_mmu;

	mmu = vcpu->arch.hw_mmu;

	kpte = kvm_pgtable_stage2_mkyoung(mmu->pgt, fault_ipa);

	kpte = kvm_pgtable_stage2_mkyoung(mmu->pgt, fault_ipa);

	spin_unlock(&vcpu->kvm->mmu_lock);

	write_unlock(&vcpu->kvm->mmu_lock);





	pte = __pte(kpte);

	pte = __pte(kpte);

	if (pte_valid(pte))

	if (pte_valid(pte))
@@ -1692,9 +1705,9 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm, 
	gpa_t gpa = slot->base_gfn << PAGE_SHIFT;

	gpa_t gpa = slot->base_gfn << PAGE_SHIFT;

	phys_addr_t size = slot->npages << PAGE_SHIFT;

	phys_addr_t size = slot->npages << PAGE_SHIFT;





	spin_lock(&kvm->mmu_lock);

	write_lock(&kvm->mmu_lock);

	unmap_stage2_range(&kvm->arch.mmu, gpa, size);

	unmap_stage2_range(&kvm->arch.mmu, gpa, size);

	spin_unlock(&kvm->mmu_lock);

	write_unlock(&kvm->mmu_lock);

}

}





/*

/*

tools/testing/selftests/kvm/dirty_log_perf_test.c

+10 −0
Original line number Original line Diff line number Diff line
@@ -18,6 +18,12 @@ 
#include "test_util.h"

#include "test_util.h"

#include "perf_test_util.h"

#include "perf_test_util.h"

#include "guest_modes.h"

#include "guest_modes.h"

#ifdef __aarch64__

#include "aarch64/vgic.h"



#define GICD_BASE_GPA			0x8000000ULL

#define GICR_BASE_GPA			0x80A0000ULL

#endif





/* How many host loops to run by default (one KVM_GET_DIRTY_LOG for each loop)*/

/* How many host loops to run by default (one KVM_GET_DIRTY_LOG for each loop)*/

#define TEST_HOST_LOOP_N		2UL

#define TEST_HOST_LOOP_N		2UL
@@ -200,6 +206,10 @@ static void run_test(enum vm_guest_mode mode, void *arg) 
		vm_enable_cap(vm, &cap);

		vm_enable_cap(vm, &cap);

	}

	}





#ifdef __aarch64__

	vgic_v3_setup(vm, nr_vcpus, 64, GICD_BASE_GPA, GICR_BASE_GPA);

#endif



	/* Start the iterations */

	/* Start the iterations */

	iteration = 0;

	iteration = 0;

	host_quit = false;

	host_quit = false;