Commit 699bb2e0 authored by Marc Zyngier's avatar Marc Zyngier
Browse files

KVM: arm64: Move vcpu PC/Exception flags to the input flag set 



The PC update flags (which also deal with exception injection)
is one of the most complicated use of the flag we have. Make it
more fool prof by:

- moving it over to the new accessors and assign it to the
  input flag set

- turn the combination of generic ELx flags with another flag
  indicating the target EL itself into an explicit set of
  flags for each EL and vector combination

- add a new accessor to pend the exception

This is otherwise a pretty straightformward conversion.

Reviewed-by: default avatarFuad Tabba <tabba@google.com>
Reviewed-by: default avatarReiji Watanabe <reijiw@google.com>
Signed-off-by: default avatarMarc Zyngier <maz@kernel.org>
parent 4c0680d3

arch/arm64/include/asm/kvm_emulate.h

+8 −1
Original line number Diff line number Diff line
@@ -473,9 +473,16 @@ static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu, 


static __always_inline void kvm_incr_pc(struct kvm_vcpu *vcpu)

{

	vcpu->arch.flags |= KVM_ARM64_INCREMENT_PC;

	vcpu_set_flag(vcpu, INCREMENT_PC);

}



#define kvm_pend_exception(v, e)					\

	do {								\

		vcpu_set_flag((v), PENDING_EXCEPTION);			\

		vcpu_set_flag((v), e);					\

	} while (0)





static inline bool vcpu_has_feature(struct kvm_vcpu *vcpu, int feature)

{

	return test_bit(feature, vcpu->arch.features);

arch/arm64/include/asm/kvm_host.h

+34 −24
Original line number Diff line number Diff line
@@ -474,6 +474,40 @@ struct kvm_vcpu_arch { 
/* PTRAUTH exposed to guest */

#define GUEST_HAS_PTRAUTH	__vcpu_single_flag(cflags, BIT(2))



/* Exception pending */

#define PENDING_EXCEPTION	__vcpu_single_flag(iflags, BIT(0))

/*

 * PC increment. Overlaps with EXCEPT_MASK on purpose so that it can't

 * be set together with an exception...

 */

#define INCREMENT_PC		__vcpu_single_flag(iflags, BIT(1))

/* Target EL/MODE (not a single flag, but let's abuse the macro) */

#define EXCEPT_MASK		__vcpu_single_flag(iflags, GENMASK(3, 1))



/* Helpers to encode exceptions with minimum fuss */

#define __EXCEPT_MASK_VAL	unpack_vcpu_flag(EXCEPT_MASK)

#define __EXCEPT_SHIFT		__builtin_ctzl(__EXCEPT_MASK_VAL)

#define __vcpu_except_flags(_f)	iflags, (_f << __EXCEPT_SHIFT), __EXCEPT_MASK_VAL



/*

 * When PENDING_EXCEPTION is set, EXCEPT_MASK can take the following

 * values:

 *

 * For AArch32 EL1:

 */

#define EXCEPT_AA32_UND		__vcpu_except_flags(0)

#define EXCEPT_AA32_IABT	__vcpu_except_flags(1)

#define EXCEPT_AA32_DABT	__vcpu_except_flags(2)

/* For AArch64: */

#define EXCEPT_AA64_EL1_SYNC	__vcpu_except_flags(0)

#define EXCEPT_AA64_EL1_IRQ	__vcpu_except_flags(1)

#define EXCEPT_AA64_EL1_FIQ	__vcpu_except_flags(2)

#define EXCEPT_AA64_EL1_SERR	__vcpu_except_flags(3)

/* For AArch64 with NV (one day): */

#define EXCEPT_AA64_EL2_SYNC	__vcpu_except_flags(4)

#define EXCEPT_AA64_EL2_IRQ	__vcpu_except_flags(5)

#define EXCEPT_AA64_EL2_FIQ	__vcpu_except_flags(6)

#define EXCEPT_AA64_EL2_SERR	__vcpu_except_flags(7)



/* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */

#define vcpu_sve_pffr(vcpu) (kern_hyp_va((vcpu)->arch.sve_state) +	\
@@ -498,30 +532,6 @@ struct kvm_vcpu_arch { 
/* vcpu_arch flags field values: */

#define KVM_ARM64_DEBUG_DIRTY		(1 << 0)

#define KVM_ARM64_HOST_SVE_ENABLED	(1 << 4) /* SVE enabled for EL0 */

#define KVM_ARM64_PENDING_EXCEPTION	(1 << 8) /* Exception pending */

/*

 * Overlaps with KVM_ARM64_EXCEPT_MASK on purpose so that it can't be

 * set together with an exception...

 */

#define KVM_ARM64_INCREMENT_PC		(1 << 9) /* Increment PC */

#define KVM_ARM64_EXCEPT_MASK		(7 << 9) /* Target EL/MODE */

/*

 * When KVM_ARM64_PENDING_EXCEPTION is set, KVM_ARM64_EXCEPT_MASK can

 * take the following values:

 *

 * For AArch32 EL1:

 */

#define KVM_ARM64_EXCEPT_AA32_UND	(0 << 9)

#define KVM_ARM64_EXCEPT_AA32_IABT	(1 << 9)

#define KVM_ARM64_EXCEPT_AA32_DABT	(2 << 9)

/* For AArch64: */

#define KVM_ARM64_EXCEPT_AA64_ELx_SYNC	(0 << 9)

#define KVM_ARM64_EXCEPT_AA64_ELx_IRQ	(1 << 9)

#define KVM_ARM64_EXCEPT_AA64_ELx_FIQ	(2 << 9)

#define KVM_ARM64_EXCEPT_AA64_ELx_SERR	(3 << 9)

#define KVM_ARM64_EXCEPT_AA64_EL1	(0 << 11)

#define KVM_ARM64_EXCEPT_AA64_EL2	(1 << 11)



#define KVM_ARM64_DEBUG_STATE_SAVE_SPE	(1 << 12) /* Save SPE context if active  */

#define KVM_ARM64_DEBUG_STATE_SAVE_TRBE	(1 << 13) /* Save TRBE context if active  */

#define KVM_ARM64_ON_UNSUPPORTED_CPU	(1 << 15) /* Physical CPU not in supported_cpus */

arch/arm64/kvm/arm.c

+2 −2
Original line number Diff line number Diff line
@@ -1013,8 +1013,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) 
	 * the vcpu state. Note that this relies on __kvm_adjust_pc()

	 * being preempt-safe on VHE.

	 */

	if (unlikely(vcpu->arch.flags & (KVM_ARM64_PENDING_EXCEPTION |

					 KVM_ARM64_INCREMENT_PC)))

	if (unlikely(vcpu_get_flag(vcpu, PENDING_EXCEPTION) ||

		     vcpu_get_flag(vcpu, INCREMENT_PC)))

		kvm_call_hyp(__kvm_adjust_pc, vcpu);



	vcpu_put(vcpu);

arch/arm64/kvm/hyp/exception.c

+11 −12
Original line number Diff line number Diff line
@@ -303,14 +303,14 @@ static void enter_exception32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset) 
static void kvm_inject_exception(struct kvm_vcpu *vcpu)

{

	if (vcpu_el1_is_32bit(vcpu)) {

		switch (vcpu->arch.flags & KVM_ARM64_EXCEPT_MASK) {

		case KVM_ARM64_EXCEPT_AA32_UND:

		switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) {

		case unpack_vcpu_flag(EXCEPT_AA32_UND):

			enter_exception32(vcpu, PSR_AA32_MODE_UND, 4);

			break;

		case KVM_ARM64_EXCEPT_AA32_IABT:

		case unpack_vcpu_flag(EXCEPT_AA32_IABT):

			enter_exception32(vcpu, PSR_AA32_MODE_ABT, 12);

			break;

		case KVM_ARM64_EXCEPT_AA32_DABT:

		case unpack_vcpu_flag(EXCEPT_AA32_DABT):

			enter_exception32(vcpu, PSR_AA32_MODE_ABT, 16);

			break;

		default:
@@ -318,9 +318,8 @@ static void kvm_inject_exception(struct kvm_vcpu *vcpu) 
			break;

		}

	} else {

		switch (vcpu->arch.flags & KVM_ARM64_EXCEPT_MASK) {

		case (KVM_ARM64_EXCEPT_AA64_ELx_SYNC |

		      KVM_ARM64_EXCEPT_AA64_EL1):

		switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) {

		case unpack_vcpu_flag(EXCEPT_AA64_EL1_SYNC):

			enter_exception64(vcpu, PSR_MODE_EL1h, except_type_sync);

			break;

		default:
@@ -340,12 +339,12 @@ static void kvm_inject_exception(struct kvm_vcpu *vcpu) 
 */

void __kvm_adjust_pc(struct kvm_vcpu *vcpu)

{

	if (vcpu->arch.flags & KVM_ARM64_PENDING_EXCEPTION) {

	if (vcpu_get_flag(vcpu, PENDING_EXCEPTION)) {

		kvm_inject_exception(vcpu);

		vcpu->arch.flags &= ~(KVM_ARM64_PENDING_EXCEPTION |

				      KVM_ARM64_EXCEPT_MASK);

	} else 	if (vcpu->arch.flags & KVM_ARM64_INCREMENT_PC) {

		vcpu_clear_flag(vcpu, PENDING_EXCEPTION);

		vcpu_clear_flag(vcpu, EXCEPT_MASK);

	} else if (vcpu_get_flag(vcpu, INCREMENT_PC)) {

		kvm_skip_instr(vcpu);

		vcpu->arch.flags &= ~KVM_ARM64_INCREMENT_PC;

		vcpu_clear_flag(vcpu, INCREMENT_PC);

	}

}

arch/arm64/kvm/hyp/nvhe/sys_regs.c

+1 −3
Original line number Diff line number Diff line
@@ -38,9 +38,7 @@ static void inject_undef64(struct kvm_vcpu *vcpu) 
	*vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR);

	*vcpu_cpsr(vcpu) = read_sysreg_el2(SYS_SPSR);



	vcpu->arch.flags |= (KVM_ARM64_EXCEPT_AA64_EL1 |

			     KVM_ARM64_EXCEPT_AA64_ELx_SYNC |

			     KVM_ARM64_PENDING_EXCEPTION);

	kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC);



	__kvm_adjust_pc(vcpu);