Commit ef5f97e9 authored by Marc Zyngier's avatar Marc Zyngier
Browse files

Merge branch kvm-arm64/lock-inversion into kvmarm-master/next 



* kvm-arm64/lock-inversion:
  : .
  : vm/vcpu lock inversion fixes, courtesy of Oliver Upton, plus a few
  : extra fixes from both Oliver and Reiji Watanabe.
  :
  : From the initial cover letter:
  :
  : As it so happens, lock ordering in KVM/arm64 is completely backwards.
  : There's a significant amount of VM-wide state that needs to be accessed
  : from the context of a vCPU. Until now, this was accomplished by
  : acquiring the kvm->lock, but that cannot be nested within vcpu->mutex.
  :
  : This series fixes the issue with some fine-grained locking for MP state
  : and a new, dedicated mutex that can nest with both kvm->lock and
  : vcpu->mutex.
  : .
  KVM: arm64: Have kvm_psci_vcpu_on() use WRITE_ONCE() to update mp_state
  KVM: arm64: Acquire mp_state_lock in kvm_arch_vcpu_ioctl_vcpu_init()
  KVM: arm64: vgic: Don't acquire its_lock before config_lock
  KVM: arm64: Use config_lock to protect vgic state
  KVM: arm64: Use config_lock to protect data ordered against KVM_RUN
  KVM: arm64: Avoid lock inversion when setting the VM register width
  KVM: arm64: Avoid vcpu->mutex v. kvm->lock inversion in CPU_ON

Signed-off-by: default avatarMarc Zyngier <maz@kernel.org>
parents 197b6b60 a189884b

arch/arm64/include/asm/kvm_host.h

+4 −0
Original line number Diff line number Diff line
@@ -199,6 +199,9 @@ struct kvm_arch { 
	/* Mandated version of PSCI */

	u32 psci_version;



	/* Protects VM-scoped configuration data */

	struct mutex config_lock;



	/*

	 * If we encounter a data abort without valid instruction syndrome

	 * information, report this to user space.  User space can (and
@@ -522,6 +525,7 @@ struct kvm_vcpu_arch { 


	/* vcpu power state */

	struct kvm_mp_state mp_state;

	spinlock_t mp_state_lock;



	/* Cache some mmu pages needed inside spinlock regions */

	struct kvm_mmu_memory_cache mmu_page_cache;

arch/arm64/kvm/arm.c

+47 −12
Original line number Diff line number Diff line
@@ -128,6 +128,16 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 
{

	int ret;



	mutex_init(&kvm->arch.config_lock);



#ifdef CONFIG_LOCKDEP

	/* Clue in lockdep that the config_lock must be taken inside kvm->lock */

	mutex_lock(&kvm->lock);

	mutex_lock(&kvm->arch.config_lock);

	mutex_unlock(&kvm->arch.config_lock);

	mutex_unlock(&kvm->lock);

#endif



	ret = kvm_share_hyp(kvm, kvm + 1);

	if (ret)

		return ret;
@@ -326,6 +336,16 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 
{

	int err;



	spin_lock_init(&vcpu->arch.mp_state_lock);



#ifdef CONFIG_LOCKDEP

	/* Inform lockdep that the config_lock is acquired after vcpu->mutex */

	mutex_lock(&vcpu->mutex);

	mutex_lock(&vcpu->kvm->arch.config_lock);

	mutex_unlock(&vcpu->kvm->arch.config_lock);

	mutex_unlock(&vcpu->mutex);

#endif



	/* Force users to call KVM_ARM_VCPU_INIT */

	vcpu->arch.target = -1;

	bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
@@ -443,34 +463,41 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 
	vcpu->cpu = -1;

}



void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)

static void __kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)

{

	vcpu->arch.mp_state.mp_state = KVM_MP_STATE_STOPPED;

	WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED);

	kvm_make_request(KVM_REQ_SLEEP, vcpu);

	kvm_vcpu_kick(vcpu);

}



void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)

{

	spin_lock(&vcpu->arch.mp_state_lock);

	__kvm_arm_vcpu_power_off(vcpu);

	spin_unlock(&vcpu->arch.mp_state_lock);

}



bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu)

{

	return vcpu->arch.mp_state.mp_state == KVM_MP_STATE_STOPPED;

	return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_STOPPED;

}



static void kvm_arm_vcpu_suspend(struct kvm_vcpu *vcpu)

{

	vcpu->arch.mp_state.mp_state = KVM_MP_STATE_SUSPENDED;

	WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_SUSPENDED);

	kvm_make_request(KVM_REQ_SUSPEND, vcpu);

	kvm_vcpu_kick(vcpu);

}



static bool kvm_arm_vcpu_suspended(struct kvm_vcpu *vcpu)

{

	return vcpu->arch.mp_state.mp_state == KVM_MP_STATE_SUSPENDED;

	return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_SUSPENDED;

}



int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,

				    struct kvm_mp_state *mp_state)

{

	*mp_state = vcpu->arch.mp_state;

	*mp_state = READ_ONCE(vcpu->arch.mp_state);



	return 0;

}
@@ -480,12 +507,14 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 
{

	int ret = 0;



	spin_lock(&vcpu->arch.mp_state_lock);



	switch (mp_state->mp_state) {

	case KVM_MP_STATE_RUNNABLE:

		vcpu->arch.mp_state = *mp_state;

		WRITE_ONCE(vcpu->arch.mp_state, *mp_state);

		break;

	case KVM_MP_STATE_STOPPED:

		kvm_arm_vcpu_power_off(vcpu);

		__kvm_arm_vcpu_power_off(vcpu);

		break;

	case KVM_MP_STATE_SUSPENDED:

		kvm_arm_vcpu_suspend(vcpu);
@@ -494,6 +523,8 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 
		ret = -EINVAL;

	}



	spin_unlock(&vcpu->arch.mp_state_lock);



	return ret;

}
@@ -593,9 +624,9 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) 
	if (kvm_vm_is_protected(kvm))

		kvm_call_hyp_nvhe(__pkvm_vcpu_init_traps, vcpu);



	mutex_lock(&kvm->lock);

	mutex_lock(&kvm->arch.config_lock);

	set_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags);

	mutex_unlock(&kvm->lock);

	mutex_unlock(&kvm->arch.config_lock);



	return ret;

}
@@ -1210,10 +1241,14 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, 
	/*

	 * Handle the "start in power-off" case.

	 */

	spin_lock(&vcpu->arch.mp_state_lock);



	if (test_bit(KVM_ARM_VCPU_POWER_OFF, vcpu->arch.features))

		kvm_arm_vcpu_power_off(vcpu);

		__kvm_arm_vcpu_power_off(vcpu);

	else

		vcpu->arch.mp_state.mp_state = KVM_MP_STATE_RUNNABLE;

		WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE);



	spin_unlock(&vcpu->arch.mp_state_lock);



	return 0;

}

arch/arm64/kvm/guest.c

+2 −0
Original line number Diff line number Diff line
@@ -957,7 +957,9 @@ int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu, 


	switch (attr->group) {

	case KVM_ARM_VCPU_PMU_V3_CTRL:

		mutex_lock(&vcpu->kvm->arch.config_lock);

		ret = kvm_arm_pmu_v3_set_attr(vcpu, attr);

		mutex_unlock(&vcpu->kvm->arch.config_lock);

		break;

	case KVM_ARM_VCPU_TIMER_CTRL:

		ret = kvm_arm_timer_set_attr(vcpu, attr);

arch/arm64/kvm/hypercalls.c

+2 −2
Original line number Diff line number Diff line
@@ -377,7 +377,7 @@ static int kvm_arm_set_fw_reg_bmap(struct kvm_vcpu *vcpu, u64 reg_id, u64 val) 
	if (val & ~fw_reg_features)

		return -EINVAL;



	mutex_lock(&kvm->lock);

	mutex_lock(&kvm->arch.config_lock);



	if (test_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags) &&

	    val != *fw_reg_bmap) {
@@ -387,7 +387,7 @@ static int kvm_arm_set_fw_reg_bmap(struct kvm_vcpu *vcpu, u64 reg_id, u64 val) 


	WRITE_ONCE(*fw_reg_bmap, val);

out:

	mutex_unlock(&kvm->lock);

	mutex_unlock(&kvm->arch.config_lock);

	return ret;

}

arch/arm64/kvm/pmu-emul.c

+6 −17
Original line number Diff line number Diff line
@@ -874,7 +874,7 @@ static int kvm_arm_pmu_v3_set_pmu(struct kvm_vcpu *vcpu, int pmu_id) 
	struct arm_pmu *arm_pmu;

	int ret = -ENXIO;



	mutex_lock(&kvm->lock);

	lockdep_assert_held(&kvm->arch.config_lock);

	mutex_lock(&arm_pmus_lock);



	list_for_each_entry(entry, &arm_pmus, entry) {
@@ -894,7 +894,6 @@ static int kvm_arm_pmu_v3_set_pmu(struct kvm_vcpu *vcpu, int pmu_id) 
	}



	mutex_unlock(&arm_pmus_lock);

	mutex_unlock(&kvm->lock);

	return ret;

}
@@ -902,22 +901,20 @@ int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) 
{

	struct kvm *kvm = vcpu->kvm;



	lockdep_assert_held(&kvm->arch.config_lock);



	if (!kvm_vcpu_has_pmu(vcpu))

		return -ENODEV;



	if (vcpu->arch.pmu.created)

		return -EBUSY;



	mutex_lock(&kvm->lock);

	if (!kvm->arch.arm_pmu) {

		/* No PMU set, get the default one */

		kvm->arch.arm_pmu = kvm_pmu_probe_armpmu();

		if (!kvm->arch.arm_pmu) {

			mutex_unlock(&kvm->lock);

		if (!kvm->arch.arm_pmu)

			return -ENODEV;

	}

	}

	mutex_unlock(&kvm->lock);



	switch (attr->attr) {

	case KVM_ARM_VCPU_PMU_V3_IRQ: {
@@ -961,19 +958,13 @@ int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) 
		     filter.action != KVM_PMU_EVENT_DENY))

			return -EINVAL;



		mutex_lock(&kvm->lock);



		if (test_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags)) {

			mutex_unlock(&kvm->lock);

		if (test_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags))

			return -EBUSY;

		}



		if (!kvm->arch.pmu_filter) {

			kvm->arch.pmu_filter = bitmap_alloc(nr_events, GFP_KERNEL_ACCOUNT);

			if (!kvm->arch.pmu_filter) {

				mutex_unlock(&kvm->lock);

			if (!kvm->arch.pmu_filter)

				return -ENOMEM;

			}



			/*

			 * The default depends on the first applied filter.
@@ -992,8 +983,6 @@ int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) 
		else

			bitmap_clear(kvm->arch.pmu_filter, filter.base_event, filter.nevents);



		mutex_unlock(&kvm->lock);



		return 0;

	}

	case KVM_ARM_VCPU_PMU_V3_SET_PMU: {