Commit 8fb20461 authored by Marc Zyngier's avatar Marc Zyngier
Browse files

KVM: arm64: Move early handlers to per-EC handlers 



Simplify the early exception handling by slicing the gigantic decoding
tree into a more manageable set of functions, similar to what we have
in handle_exit.c.

This will also make the structure reusable for pKVM's own early exit
handling.

Signed-off-by: default avatarMarc Zyngier <maz@kernel.org>
Signed-off-by: default avatarFuad Tabba <tabba@google.com>
Link: https://lore.kernel.org/r/20211010145636.1950948-4-tabba@google.com
parent cc1e6fdf

arch/arm64/kvm/hyp/include/hyp/switch.h

+92 −68
Original line number Diff line number Diff line
@@ -136,16 +136,7 @@ static inline void ___deactivate_traps(struct kvm_vcpu *vcpu) 


static inline bool __populate_fault_info(struct kvm_vcpu *vcpu)

{

	u8 ec;

	u64 esr;



	esr = vcpu->arch.fault.esr_el2;

	ec = ESR_ELx_EC(esr);



	if (ec != ESR_ELx_EC_DABT_LOW && ec != ESR_ELx_EC_IABT_LOW)

		return true;



	return __get_fault_info(esr, &vcpu->arch.fault);

	return __get_fault_info(vcpu->arch.fault.esr_el2, &vcpu->arch.fault);

}



static inline void __hyp_sve_save_host(struct kvm_vcpu *vcpu)
@@ -166,8 +157,13 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu) 
	write_sysreg_el1(__vcpu_sys_reg(vcpu, ZCR_EL1), SYS_ZCR);

}



/* Check for an FPSIMD/SVE trap and handle as appropriate */

static inline bool __hyp_handle_fpsimd(struct kvm_vcpu *vcpu)

/*

 * We trap the first access to the FP/SIMD to save the host context and

 * restore the guest context lazily.

 * If FP/SIMD is not implemented, handle the trap and inject an undefined

 * instruction exception to the guest. Similarly for trapped SVE accesses.

 */

static bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code)

{

	bool sve_guest, sve_host;

	u8 esr_ec;
@@ -185,9 +181,6 @@ static inline bool __hyp_handle_fpsimd(struct kvm_vcpu *vcpu) 
	}



	esr_ec = kvm_vcpu_trap_get_class(vcpu);

	if (esr_ec != ESR_ELx_EC_FP_ASIMD &&

	    esr_ec != ESR_ELx_EC_SVE)

		return false;



	/* Don't handle SVE traps for non-SVE vcpus here: */

	if (!sve_guest && esr_ec != ESR_ELx_EC_FP_ASIMD)
@@ -325,7 +318,7 @@ static inline bool esr_is_ptrauth_trap(u32 esr) 


DECLARE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);



static inline bool __hyp_handle_ptrauth(struct kvm_vcpu *vcpu)

static bool kvm_hyp_handle_ptrauth(struct kvm_vcpu *vcpu, u64 *exit_code)

{

	struct kvm_cpu_context *ctxt;

	u64 val;
@@ -350,6 +343,87 @@ static inline bool __hyp_handle_ptrauth(struct kvm_vcpu *vcpu) 
	return true;

}



static bool kvm_hyp_handle_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code)

{

	if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM) &&

	    handle_tx2_tvm(vcpu))

		return true;



	if (static_branch_unlikely(&vgic_v3_cpuif_trap) &&

	    __vgic_v3_perform_cpuif_access(vcpu) == 1)

		return true;



	return false;

}



static bool kvm_hyp_handle_cp15_32(struct kvm_vcpu *vcpu, u64 *exit_code)

{

	if (static_branch_unlikely(&vgic_v3_cpuif_trap) &&

	    __vgic_v3_perform_cpuif_access(vcpu) == 1)

		return true;



	return false;

}



static bool kvm_hyp_handle_iabt_low(struct kvm_vcpu *vcpu, u64 *exit_code)

{

	if (!__populate_fault_info(vcpu))

		return true;



	return false;

}



static bool kvm_hyp_handle_dabt_low(struct kvm_vcpu *vcpu, u64 *exit_code)

{

	if (!__populate_fault_info(vcpu))

		return true;



	if (static_branch_unlikely(&vgic_v2_cpuif_trap)) {

		bool valid;



		valid = kvm_vcpu_trap_get_fault_type(vcpu) == FSC_FAULT &&

			kvm_vcpu_dabt_isvalid(vcpu) &&

			!kvm_vcpu_abt_issea(vcpu) &&

			!kvm_vcpu_abt_iss1tw(vcpu);



		if (valid) {

			int ret = __vgic_v2_perform_cpuif_access(vcpu);



			if (ret == 1)

				return true;



			/* Promote an illegal access to an SError.*/

			if (ret == -1)

				*exit_code = ARM_EXCEPTION_EL1_SERROR;

		}

	}



	return false;

}



typedef bool (*exit_handler_fn)(struct kvm_vcpu *, u64 *);



static const exit_handler_fn *kvm_get_exit_handler_array(void);



/*

 * Allow the hypervisor to handle the exit with an exit handler if it has one.

 *

 * Returns true if the hypervisor handled the exit, and control should go back

 * to the guest, or false if it hasn't.

 */

static inline bool kvm_hyp_handle_exit(struct kvm_vcpu *vcpu, u64 *exit_code)

{

	const exit_handler_fn *handlers = kvm_get_exit_handler_array();

	exit_handler_fn fn;



	fn = handlers[kvm_vcpu_trap_get_class(vcpu)];



	if (fn)

		return fn(vcpu, exit_code);



	return false;

}



/*

 * Return true when we were able to fixup the guest exit and should return to

 * the guest, false when we should restore the host state and return to the
@@ -384,59 +458,9 @@ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) 
	if (*exit_code != ARM_EXCEPTION_TRAP)

		goto exit;



	if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM) &&

	    kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_SYS64 &&

	    handle_tx2_tvm(vcpu))

		goto guest;



	/*

	 * We trap the first access to the FP/SIMD to save the host context

	 * and restore the guest context lazily.

	 * If FP/SIMD is not implemented, handle the trap and inject an

	 * undefined instruction exception to the guest.

	 * Similarly for trapped SVE accesses.

	 */

	if (__hyp_handle_fpsimd(vcpu))

		goto guest;



	if (__hyp_handle_ptrauth(vcpu))

		goto guest;



	if (!__populate_fault_info(vcpu))

		goto guest;



	if (static_branch_unlikely(&vgic_v2_cpuif_trap)) {

		bool valid;



		valid = kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_DABT_LOW &&

			kvm_vcpu_trap_get_fault_type(vcpu) == FSC_FAULT &&

			kvm_vcpu_dabt_isvalid(vcpu) &&

			!kvm_vcpu_abt_issea(vcpu) &&

			!kvm_vcpu_abt_iss1tw(vcpu);



		if (valid) {

			int ret = __vgic_v2_perform_cpuif_access(vcpu);



			if (ret == 1)

				goto guest;



			/* Promote an illegal access to an SError.*/

			if (ret == -1)

				*exit_code = ARM_EXCEPTION_EL1_SERROR;



			goto exit;

		}

	}



	if (static_branch_unlikely(&vgic_v3_cpuif_trap) &&

	    (kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_SYS64 ||

	     kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_CP15_32)) {

		int ret = __vgic_v3_perform_cpuif_access(vcpu);



		if (ret == 1)

	/* Check if there's an exit handler and allow it to handle the exit. */

	if (kvm_hyp_handle_exit(vcpu, exit_code))

		goto guest;

	}



exit:

	/* Return to the host kernel and handle the exit */

	return false;

arch/arm64/kvm/hyp/nvhe/switch.c

+16 −0
Original line number Diff line number Diff line
@@ -158,6 +158,22 @@ static void __pmu_switch_to_host(struct kvm_cpu_context *host_ctxt) 
		write_sysreg(pmu->events_host, pmcntenset_el0);

}



static const exit_handler_fn hyp_exit_handlers[] = {

	[0 ... ESR_ELx_EC_MAX]		= NULL,

	[ESR_ELx_EC_CP15_32]		= kvm_hyp_handle_cp15_32,

	[ESR_ELx_EC_SYS64]		= kvm_hyp_handle_sysreg,

	[ESR_ELx_EC_SVE]		= kvm_hyp_handle_fpsimd,

	[ESR_ELx_EC_FP_ASIMD]		= kvm_hyp_handle_fpsimd,

	[ESR_ELx_EC_IABT_LOW]		= kvm_hyp_handle_iabt_low,

	[ESR_ELx_EC_DABT_LOW]		= kvm_hyp_handle_dabt_low,

	[ESR_ELx_EC_PAC]		= kvm_hyp_handle_ptrauth,

};



static const exit_handler_fn *kvm_get_exit_handler_array(void)

{

	return hyp_exit_handlers;

}



/* Switch to the guest for legacy non-VHE systems */

int __kvm_vcpu_run(struct kvm_vcpu *vcpu)

{

arch/arm64/kvm/hyp/vhe/switch.c

+16 −0
Original line number Diff line number Diff line
@@ -96,6 +96,22 @@ void deactivate_traps_vhe_put(struct kvm_vcpu *vcpu) 
	__deactivate_traps_common(vcpu);

}



static const exit_handler_fn hyp_exit_handlers[] = {

	[0 ... ESR_ELx_EC_MAX]		= NULL,

	[ESR_ELx_EC_CP15_32]		= kvm_hyp_handle_cp15_32,

	[ESR_ELx_EC_SYS64]		= kvm_hyp_handle_sysreg,

	[ESR_ELx_EC_SVE]		= kvm_hyp_handle_fpsimd,

	[ESR_ELx_EC_FP_ASIMD]		= kvm_hyp_handle_fpsimd,

	[ESR_ELx_EC_IABT_LOW]		= kvm_hyp_handle_iabt_low,

	[ESR_ELx_EC_DABT_LOW]		= kvm_hyp_handle_dabt_low,

	[ESR_ELx_EC_PAC]		= kvm_hyp_handle_ptrauth,

};



static const exit_handler_fn *kvm_get_exit_handler_array(void)

{

	return hyp_exit_handlers;

}



/* Switch to the guest for VHE systems running in EL2 */

static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)

{