Merge branch 'kvm-arm64/el2-pc' into kvmarm-master/next (37da329e) · Commits · EulixOS / Software / Kernel

arch/arm64/include/asm/kvm_emulate.h

+14 −56

Original line number	Diff line number	Diff line
		@@ -21,20 +21,25 @@
		#include <asm/cputype.h>
		#include <asm/virt.h>

		unsigned long vcpu_reg32(const struct kvm_vcpu vcpu, u8 reg_num);
		unsigned long vcpu_read_spsr32(const struct kvm_vcpu *vcpu);
		void vcpu_write_spsr32(struct kvm_vcpu *vcpu, unsigned long v);
		#define CURRENT_EL_SP_EL0_VECTOR 0x0
		#define CURRENT_EL_SP_ELx_VECTOR 0x200
		#define LOWER_EL_AArch64_VECTOR 0x400
		#define LOWER_EL_AArch32_VECTOR 0x600

		enum exception_type {
		except_type_sync = 0,
		except_type_irq = 0x80,
		except_type_fiq = 0x100,
		except_type_serror = 0x180,
		};

		bool kvm_condition_valid32(const struct kvm_vcpu *vcpu);
		void kvm_skip_instr32(struct kvm_vcpu *vcpu, bool is_wide_instr);
		void kvm_skip_instr32(struct kvm_vcpu *vcpu);

		void kvm_inject_undefined(struct kvm_vcpu *vcpu);
		void kvm_inject_vabt(struct kvm_vcpu *vcpu);
		void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr);
		void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr);
		void kvm_inject_undef32(struct kvm_vcpu *vcpu);
		void kvm_inject_dabt32(struct kvm_vcpu *vcpu, unsigned long addr);
		void kvm_inject_pabt32(struct kvm_vcpu *vcpu, unsigned long addr);

		static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu)
		{
		@@ -168,30 +173,6 @@ static __always_inline void vcpu_set_reg(struct kvm_vcpu *vcpu, u8 reg_num,
		vcpu_gp_regs(vcpu)->regs[reg_num] = val;
		}

		static inline unsigned long vcpu_read_spsr(const struct kvm_vcpu *vcpu)
		{
		if (vcpu_mode_is_32bit(vcpu))
		return vcpu_read_spsr32(vcpu);

		if (vcpu->arch.sysregs_loaded_on_cpu)
		return read_sysreg_el1(SYS_SPSR);
		else
		return __vcpu_sys_reg(vcpu, SPSR_EL1);
		}

		static inline void vcpu_write_spsr(struct kvm_vcpu *vcpu, unsigned long v)
		{
		if (vcpu_mode_is_32bit(vcpu)) {
		vcpu_write_spsr32(vcpu, v);
		return;
		}

		if (vcpu->arch.sysregs_loaded_on_cpu)
		write_sysreg_el1(v, SYS_SPSR);
		else
		__vcpu_sys_reg(vcpu, SPSR_EL1) = v;
		}

		/*
		* The layout of SPSR for an AArch32 state is different when observed from an
		* AArch64 SPSR_ELx or an AArch32 SPSR_*. This function generates the AArch32
		@@ -472,32 +453,9 @@ static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu,
		return data; /* Leave LE untouched */
		}

		static __always_inline void kvm_skip_instr(struct kvm_vcpu *vcpu, bool is_wide_instr)
		{
		if (vcpu_mode_is_32bit(vcpu)) {
		kvm_skip_instr32(vcpu, is_wide_instr);
		} else {
		*vcpu_pc(vcpu) += 4;
		*vcpu_cpsr(vcpu) &= ~PSR_BTYPE_MASK;
		}

		/* advance the singlestep state machine */
		*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
		}

		/*
		* Skip an instruction which has been emulated at hyp while most guest sysregs
		* are live.
		*/
		static __always_inline void __kvm_skip_instr(struct kvm_vcpu *vcpu)
		static __always_inline void kvm_incr_pc(struct kvm_vcpu *vcpu)
		{
		*vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR);
		vcpu_gp_regs(vcpu)->pstate = read_sysreg_el2(SYS_SPSR);

		kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));

		write_sysreg_el2(vcpu_gp_regs(vcpu)->pstate, SYS_SPSR);
		write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR);
		vcpu->arch.flags \|= KVM_ARM64_INCREMENT_PC;
		}

		#endif /* __ARM64_KVM_EMULATE_H__ */

arch/arm64/include/asm/kvm_host.h

+117 −1

Original line number	Diff line number	Diff line
		@@ -409,6 +409,31 @@ struct kvm_vcpu_arch {
		#define KVM_ARM64_GUEST_HAS_SVE (1 << 5) /* SVE exposed to guest */
		#define KVM_ARM64_VCPU_SVE_FINALIZED (1 << 6) /* SVE config completed */
		#define KVM_ARM64_GUEST_HAS_PTRAUTH (1 << 7) /* PTRAUTH exposed to guest */
		#define KVM_ARM64_PENDING_EXCEPTION (1 << 8) /* Exception pending */
		#define KVM_ARM64_EXCEPT_MASK (7 << 9) /* Target EL/MODE */

		/*
		* When KVM_ARM64_PENDING_EXCEPTION is set, KVM_ARM64_EXCEPT_MASK can
		* take the following values:
		*
		* For AArch32 EL1:
		*/
		#define KVM_ARM64_EXCEPT_AA32_UND (0 << 9)
		#define KVM_ARM64_EXCEPT_AA32_IABT (1 << 9)
		#define KVM_ARM64_EXCEPT_AA32_DABT (2 << 9)
		/* For AArch64: */
		#define KVM_ARM64_EXCEPT_AA64_ELx_SYNC (0 << 9)
		#define KVM_ARM64_EXCEPT_AA64_ELx_IRQ (1 << 9)
		#define KVM_ARM64_EXCEPT_AA64_ELx_FIQ (2 << 9)
		#define KVM_ARM64_EXCEPT_AA64_ELx_SERR (3 << 9)
		#define KVM_ARM64_EXCEPT_AA64_EL1 (0 << 11)
		#define KVM_ARM64_EXCEPT_AA64_EL2 (1 << 11)

		/*
		* Overlaps with KVM_ARM64_EXCEPT_MASK on purpose so that it can't be
		* set together with an exception...
		*/
		#define KVM_ARM64_INCREMENT_PC (1 << 9) /* Increment PC */

		#define vcpu_has_sve(vcpu) (system_supports_sve() && \
		((vcpu)->arch.flags & KVM_ARM64_GUEST_HAS_SVE))
		@@ -440,6 +465,97 @@ struct kvm_vcpu_arch {
		u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg);
		void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg);

		static inline bool __vcpu_read_sys_reg_from_cpu(int reg, u64 *val)
		{
		/*
		* * VHE ONLY *
		*
		* System registers listed in the switch are not saved on every
		* exit from the guest but are only saved on vcpu_put.
		*
		* Note that MPIDR_EL1 for the guest is set by KVM via VMPIDR_EL2 but
		* should never be listed below, because the guest cannot modify its
		* own MPIDR_EL1 and MPIDR_EL1 is accessed for VCPU A from VCPU B's
		* thread when emulating cross-VCPU communication.
		*/
		if (!has_vhe())
		return false;

		switch (reg) {
		case CSSELR_EL1: *val = read_sysreg_s(SYS_CSSELR_EL1); break;
		case SCTLR_EL1: *val = read_sysreg_s(SYS_SCTLR_EL12); break;
		case CPACR_EL1: *val = read_sysreg_s(SYS_CPACR_EL12); break;
		case TTBR0_EL1: *val = read_sysreg_s(SYS_TTBR0_EL12); break;
		case TTBR1_EL1: *val = read_sysreg_s(SYS_TTBR1_EL12); break;
		case TCR_EL1: *val = read_sysreg_s(SYS_TCR_EL12); break;
		case ESR_EL1: *val = read_sysreg_s(SYS_ESR_EL12); break;
		case AFSR0_EL1: *val = read_sysreg_s(SYS_AFSR0_EL12); break;
		case AFSR1_EL1: *val = read_sysreg_s(SYS_AFSR1_EL12); break;
		case FAR_EL1: *val = read_sysreg_s(SYS_FAR_EL12); break;
		case MAIR_EL1: *val = read_sysreg_s(SYS_MAIR_EL12); break;
		case VBAR_EL1: *val = read_sysreg_s(SYS_VBAR_EL12); break;
		case CONTEXTIDR_EL1: *val = read_sysreg_s(SYS_CONTEXTIDR_EL12);break;
		case TPIDR_EL0: *val = read_sysreg_s(SYS_TPIDR_EL0); break;
		case TPIDRRO_EL0: *val = read_sysreg_s(SYS_TPIDRRO_EL0); break;
		case TPIDR_EL1: *val = read_sysreg_s(SYS_TPIDR_EL1); break;
		case AMAIR_EL1: *val = read_sysreg_s(SYS_AMAIR_EL12); break;
		case CNTKCTL_EL1: *val = read_sysreg_s(SYS_CNTKCTL_EL12); break;
		case ELR_EL1: *val = read_sysreg_s(SYS_ELR_EL12); break;
		case PAR_EL1: *val = read_sysreg_par(); break;
		case DACR32_EL2: *val = read_sysreg_s(SYS_DACR32_EL2); break;
		case IFSR32_EL2: *val = read_sysreg_s(SYS_IFSR32_EL2); break;
		case DBGVCR32_EL2: *val = read_sysreg_s(SYS_DBGVCR32_EL2); break;
		default: return false;
		}

		return true;
		}

		static inline bool __vcpu_write_sys_reg_to_cpu(u64 val, int reg)
		{
		/*
		* * VHE ONLY *
		*
		* System registers listed in the switch are not restored on every
		* entry to the guest but are only restored on vcpu_load.
		*
		* Note that MPIDR_EL1 for the guest is set by KVM via VMPIDR_EL2 but
		* should never be listed below, because the MPIDR should only be set
		* once, before running the VCPU, and never changed later.
		*/
		if (!has_vhe())
		return false;

		switch (reg) {
		case CSSELR_EL1: write_sysreg_s(val, SYS_CSSELR_EL1); break;
		case SCTLR_EL1: write_sysreg_s(val, SYS_SCTLR_EL12); break;
		case CPACR_EL1: write_sysreg_s(val, SYS_CPACR_EL12); break;
		case TTBR0_EL1: write_sysreg_s(val, SYS_TTBR0_EL12); break;
		case TTBR1_EL1: write_sysreg_s(val, SYS_TTBR1_EL12); break;
		case TCR_EL1: write_sysreg_s(val, SYS_TCR_EL12); break;
		case ESR_EL1: write_sysreg_s(val, SYS_ESR_EL12); break;
		case AFSR0_EL1: write_sysreg_s(val, SYS_AFSR0_EL12); break;
		case AFSR1_EL1: write_sysreg_s(val, SYS_AFSR1_EL12); break;
		case FAR_EL1: write_sysreg_s(val, SYS_FAR_EL12); break;
		case MAIR_EL1: write_sysreg_s(val, SYS_MAIR_EL12); break;
		case VBAR_EL1: write_sysreg_s(val, SYS_VBAR_EL12); break;
		case CONTEXTIDR_EL1: write_sysreg_s(val, SYS_CONTEXTIDR_EL12);break;
		case TPIDR_EL0: write_sysreg_s(val, SYS_TPIDR_EL0); break;
		case TPIDRRO_EL0: write_sysreg_s(val, SYS_TPIDRRO_EL0); break;
		case TPIDR_EL1: write_sysreg_s(val, SYS_TPIDR_EL1); break;
		case AMAIR_EL1: write_sysreg_s(val, SYS_AMAIR_EL12); break;
		case CNTKCTL_EL1: write_sysreg_s(val, SYS_CNTKCTL_EL12); break;
		case ELR_EL1: write_sysreg_s(val, SYS_ELR_EL12); break;
		case PAR_EL1: write_sysreg_s(val, SYS_PAR_EL1); break;
		case DACR32_EL2: write_sysreg_s(val, SYS_DACR32_EL2); break;
		case IFSR32_EL2: write_sysreg_s(val, SYS_IFSR32_EL2); break;
		case DBGVCR32_EL2: write_sysreg_s(val, SYS_DBGVCR32_EL2); break;
		default: return false;
		}

		return true;
		}

		/*
		* CP14 and CP15 live in the same array, as they are backed by the
		* same system registers.

arch/arm64/kvm/Makefile

+2 −2

Original line number	Diff line number	Diff line
		@@ -13,10 +13,10 @@ obj-$(CONFIG_KVM) += hyp/
		kvm-y := $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o \
		$(KVM)/vfio.o $(KVM)/irqchip.o \
		arm.o mmu.o mmio.o psci.o perf.o hypercalls.o pvtime.o \
		inject_fault.o regmap.o va_layout.o handle_exit.o \
		inject_fault.o va_layout.o handle_exit.o \
		guest.o debug.o reset.o sys_regs.o \
		vgic-sys-reg-v3.o fpsimd.o pmu.o \
		aarch32.o arch_timer.o \
		arch_timer.o \
		vgic/vgic.o vgic/vgic-init.o \
		vgic/vgic-irqfd.o vgic/vgic-v2.o \
		vgic/vgic-v3.o vgic/vgic-v4.o \

arch/arm64/kvm/guest.c

+25 −3

Original line number	Diff line number	Diff line
		@@ -252,10 +252,32 @@ static int set_core_reg(struct kvm_vcpu vcpu, const struct kvm_one_reg reg)
		memcpy(addr, valp, KVM_REG_SIZE(reg->id));

		if (*vcpu_cpsr(vcpu) & PSR_MODE32_BIT) {
		int i;
		int i, nr_reg;

		for (i = 0; i < 16; i++)
		vcpu_reg32(vcpu, i) = (u32)vcpu_reg32(vcpu, i);
		switch (*vcpu_cpsr(vcpu)) {
		/*
		* Either we are dealing with user mode, and only the
		* first 15 registers (+ PC) must be narrowed to 32bit.
		* AArch32 r0-r14 conveniently map to AArch64 x0-x14.
		*/
		case PSR_AA32_MODE_USR:
		case PSR_AA32_MODE_SYS:
		nr_reg = 15;
		break;

		/*
		* Otherwide, this is a priviledged mode, and all the
		* registers must be narrowed to 32bit.
		*/
		default:
		nr_reg = 31;
		break;
		}

		for (i = 0; i < nr_reg; i++)
		vcpu_set_reg(vcpu, i, (u32)vcpu_get_reg(vcpu, i));

		vcpu_pc(vcpu) = (u32)vcpu_pc(vcpu);
		}
		out:
		return err;

arch/arm64/kvm/handle_exit.c

+3 −20

Original line number	Diff line number	Diff line
		@@ -61,7 +61,7 @@ static int handle_smc(struct kvm_vcpu *vcpu)
		* otherwise return to the same address...
		*/
		vcpu_set_reg(vcpu, 0, ~0UL);
		kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
		kvm_incr_pc(vcpu);
		return 1;
		}

		@@ -100,7 +100,7 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu)
		kvm_clear_request(KVM_REQ_UNHALT, vcpu);
		}

		kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
		kvm_incr_pc(vcpu);

		return 1;
		}
		@@ -221,7 +221,7 @@ static int handle_trap_exceptions(struct kvm_vcpu *vcpu)
		* that fail their condition code check"
		*/
		if (!kvm_condition_valid(vcpu)) {
		kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
		kvm_incr_pc(vcpu);
		handled = 1;
		} else {
		exit_handle_fn exit_handler;
		@@ -241,23 +241,6 @@ int handle_exit(struct kvm_vcpu *vcpu, int exception_index)
		{
		struct kvm_run *run = vcpu->run;

		if (ARM_SERROR_PENDING(exception_index)) {
		u8 esr_ec = ESR_ELx_EC(kvm_vcpu_get_esr(vcpu));

		/*
		* HVC/SMC already have an adjusted PC, which we need
		* to correct in order to return to after having
		* injected the SError.
		*/
		if (esr_ec == ESR_ELx_EC_HVC32 \|\| esr_ec == ESR_ELx_EC_HVC64 \|\|
		esr_ec == ESR_ELx_EC_SMC32 \|\| esr_ec == ESR_ELx_EC_SMC64) {
		u32 adj = kvm_vcpu_trap_il_is32bit(vcpu) ? 4 : 2;
		*vcpu_pc(vcpu) -= adj;
		}

		return 1;
		}

		exception_index = ARM_EXCEPTION_CODE(exception_index);

		switch (exception_index) {