Commit fdec2a9e authored by Dave Martin's avatar Dave Martin Committed by Marc Zyngier
Browse files

KVM: arm64: Migrate _elx sysreg accessors to msr_s/mrs_s 



Currently, the {read,write}_sysreg_el*() accessors for accessing
particular ELs' sysregs in the presence of VHE rely on some local
hacks and define their system register encodings in a way that is
inconsistent with the core definitions in <asm/sysreg.h>.

As a result, it is necessary to add duplicate definitions for any
system register that already needs a definition in sysreg.h for
other reasons.

This is a bit of a maintenance headache, and the reasons for the
_el*() accessors working the way they do is a bit historical.

This patch gets rid of the shadow sysreg definitions in
<asm/kvm_hyp.h>, converts the _el*() accessors to use the core
__msr_s/__mrs_s interface, and converts all call sites to use the
standard sysreg #define names (i.e., upper case, with SYS_ prefix).

This patch will conflict heavily anyway, so the opportunity
to clean up some bad whitespace in the context of the changes is
taken.

The change exposes a few system registers that have no sysreg.h
definition, due to msr_s/mrs_s being used in place of msr/mrs:
additions are made in order to fill in the gaps.

Signed-off-by: default avatarDave Martin <Dave.Martin@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christoffer Dall <christoffer.dall@arm.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Link: https://www.spinics.net/lists/kvm-arm/msg31717.html


[Rebased to v4.21-rc1]
Signed-off-by: default avatarSudeep Holla <sudeep.holla@arm.com>
[Rebased to v5.2-rc5, changelog updates]
Signed-off-by: default avatarMarc Zyngier <marc.zyngier@arm.com>
parent 49caebe9

arch/arm/include/asm/kvm_hyp.h

+7 −6
Original line number Diff line number Diff line
@@ -82,13 +82,14 @@ 
#define VFP_FPEXC	__ACCESS_VFP(FPEXC)



/* AArch64 compatibility macros, only for the timer so far */

#define read_sysreg_el0(r)		read_sysreg(r##_el0)

#define write_sysreg_el0(v, r)		write_sysreg(v, r##_el0)

#define read_sysreg_el0(r)		read_sysreg(r##_EL0)

#define write_sysreg_el0(v, r)		write_sysreg(v, r##_EL0)



#define SYS_CNTP_CTL_EL0		CNTP_CTL

#define SYS_CNTP_CVAL_EL0		CNTP_CVAL

#define SYS_CNTV_CTL_EL0		CNTV_CTL

#define SYS_CNTV_CVAL_EL0		CNTV_CVAL



#define cntp_ctl_el0			CNTP_CTL

#define cntp_cval_el0			CNTP_CVAL

#define cntv_ctl_el0			CNTV_CTL

#define cntv_cval_el0			CNTV_CVAL

#define cntvoff_el2			CNTVOFF

#define cnthctl_el2			CNTHCTL

arch/arm64/include/asm/kvm_emulate.h

+8 −8
Original line number Diff line number Diff line
@@ -126,7 +126,7 @@ static inline unsigned long *__vcpu_elr_el1(const struct kvm_vcpu *vcpu) 
static inline unsigned long vcpu_read_elr_el1(const struct kvm_vcpu *vcpu)

{

	if (vcpu->arch.sysregs_loaded_on_cpu)

		return read_sysreg_el1(elr);

		return read_sysreg_el1(SYS_ELR);

	else

		return *__vcpu_elr_el1(vcpu);

}
@@ -134,7 +134,7 @@ static inline unsigned long vcpu_read_elr_el1(const struct kvm_vcpu *vcpu) 
static inline void vcpu_write_elr_el1(const struct kvm_vcpu *vcpu, unsigned long v)

{

	if (vcpu->arch.sysregs_loaded_on_cpu)

		write_sysreg_el1(v, elr);

		write_sysreg_el1(v, SYS_ELR);

	else

		*__vcpu_elr_el1(vcpu) = v;

}
@@ -186,7 +186,7 @@ static inline unsigned long vcpu_read_spsr(const struct kvm_vcpu *vcpu) 
		return vcpu_read_spsr32(vcpu);



	if (vcpu->arch.sysregs_loaded_on_cpu)

		return read_sysreg_el1(spsr);

		return read_sysreg_el1(SYS_SPSR);

	else

		return vcpu_gp_regs(vcpu)->spsr[KVM_SPSR_EL1];

}
@@ -199,7 +199,7 @@ static inline void vcpu_write_spsr(struct kvm_vcpu *vcpu, unsigned long v) 
	}



	if (vcpu->arch.sysregs_loaded_on_cpu)

		write_sysreg_el1(v, spsr);

		write_sysreg_el1(v, SYS_SPSR);

	else

		vcpu_gp_regs(vcpu)->spsr[KVM_SPSR_EL1] = v;

}
@@ -465,13 +465,13 @@ static inline void kvm_skip_instr(struct kvm_vcpu *vcpu, bool is_wide_instr) 
 */

static inline void __hyp_text __kvm_skip_instr(struct kvm_vcpu *vcpu)

{

	*vcpu_pc(vcpu) = read_sysreg_el2(elr);

	vcpu->arch.ctxt.gp_regs.regs.pstate = read_sysreg_el2(spsr);

	*vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR);

	vcpu->arch.ctxt.gp_regs.regs.pstate = read_sysreg_el2(SYS_SPSR);



	kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));



	write_sysreg_el2(vcpu->arch.ctxt.gp_regs.regs.pstate, spsr);

	write_sysreg_el2(*vcpu_pc(vcpu), elr);

	write_sysreg_el2(vcpu->arch.ctxt.gp_regs.regs.pstate, SYS_SPSR);

	write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR);

}



#endif /* __ARM64_KVM_EMULATE_H__ */

arch/arm64/include/asm/kvm_hyp.h

+5 −45
Original line number Diff line number Diff line
@@ -18,7 +18,7 @@ 
#define read_sysreg_elx(r,nvh,vh)					\

	({								\

		u64 reg;						\

		asm volatile(ALTERNATIVE("mrs %0, " __stringify(r##nvh),\

		asm volatile(ALTERNATIVE(__mrs_s("%0", r##nvh),	\

					 __mrs_s("%0", r##vh),		\

					 ARM64_HAS_VIRT_HOST_EXTN)	\

			     : "=r" (reg));				\
@@ -28,7 +28,7 @@ 
#define write_sysreg_elx(v,r,nvh,vh)					\

	do {								\

		u64 __val = (u64)(v);					\

		asm volatile(ALTERNATIVE("msr " __stringify(r##nvh) ", %x0",\

		asm volatile(ALTERNATIVE(__msr_s(r##nvh, "%x0"),	\

					 __msr_s(r##vh, "%x0"),		\

					 ARM64_HAS_VIRT_HOST_EXTN)	\

					 : : "rZ" (__val));		\
@@ -37,55 +37,15 @@ 
/*

 * Unified accessors for registers that have a different encoding

 * between VHE and non-VHE. They must be specified without their "ELx"

 * encoding.

 * encoding, but with the SYS_ prefix, as defined in asm/sysreg.h.

 */

#define read_sysreg_el2(r)						\

	({								\

		u64 reg;						\

		asm volatile(ALTERNATIVE("mrs %0, " __stringify(r##_EL2),\

					 "mrs %0, " __stringify(r##_EL1),\

					 ARM64_HAS_VIRT_HOST_EXTN)	\

			     : "=r" (reg));				\

		reg;							\

	})



#define write_sysreg_el2(v,r)						\

	do {								\

		u64 __val = (u64)(v);					\

		asm volatile(ALTERNATIVE("msr " __stringify(r##_EL2) ", %x0",\

					 "msr " __stringify(r##_EL1) ", %x0",\

					 ARM64_HAS_VIRT_HOST_EXTN)	\

					 : : "rZ" (__val));		\

	} while (0)



#define read_sysreg_el0(r)	read_sysreg_elx(r, _EL0, _EL02)

#define write_sysreg_el0(v,r)	write_sysreg_elx(v, r, _EL0, _EL02)

#define read_sysreg_el1(r)	read_sysreg_elx(r, _EL1, _EL12)

#define write_sysreg_el1(v,r)	write_sysreg_elx(v, r, _EL1, _EL12)



/* The VHE specific system registers and their encoding */

#define sctlr_EL12              sys_reg(3, 5, 1, 0, 0)

#define cpacr_EL12              sys_reg(3, 5, 1, 0, 2)

#define ttbr0_EL12              sys_reg(3, 5, 2, 0, 0)

#define ttbr1_EL12              sys_reg(3, 5, 2, 0, 1)

#define tcr_EL12                sys_reg(3, 5, 2, 0, 2)

#define afsr0_EL12              sys_reg(3, 5, 5, 1, 0)

#define afsr1_EL12              sys_reg(3, 5, 5, 1, 1)

#define esr_EL12                sys_reg(3, 5, 5, 2, 0)

#define far_EL12                sys_reg(3, 5, 6, 0, 0)

#define mair_EL12               sys_reg(3, 5, 10, 2, 0)

#define amair_EL12              sys_reg(3, 5, 10, 3, 0)

#define vbar_EL12               sys_reg(3, 5, 12, 0, 0)

#define contextidr_EL12         sys_reg(3, 5, 13, 0, 1)

#define cntkctl_EL12            sys_reg(3, 5, 14, 1, 0)

#define cntp_tval_EL02          sys_reg(3, 5, 14, 2, 0)

#define cntp_ctl_EL02           sys_reg(3, 5, 14, 2, 1)

#define cntp_cval_EL02          sys_reg(3, 5, 14, 2, 2)

#define cntv_tval_EL02          sys_reg(3, 5, 14, 3, 0)

#define cntv_ctl_EL02           sys_reg(3, 5, 14, 3, 1)

#define cntv_cval_EL02          sys_reg(3, 5, 14, 3, 2)

#define spsr_EL12               sys_reg(3, 5, 4, 0, 0)

#define elr_EL12                sys_reg(3, 5, 4, 0, 1)

#define read_sysreg_el2(r)	read_sysreg_elx(r, _EL2, _EL1)

#define write_sysreg_el2(v,r)	write_sysreg_elx(v, r, _EL2, _EL1)



/**

 * hyp_alternate_select - Generates patchable code sequences that are

arch/arm64/include/asm/sysreg.h

+33 −2
Original line number Diff line number Diff line
@@ -191,6 +191,9 @@ 
#define SYS_APGAKEYLO_EL1		sys_reg(3, 0, 2, 3, 0)

#define SYS_APGAKEYHI_EL1		sys_reg(3, 0, 2, 3, 1)



#define SYS_SPSR_EL1			sys_reg(3, 0, 4, 0, 0)

#define SYS_ELR_EL1			sys_reg(3, 0, 4, 0, 1)



#define SYS_ICC_PMR_EL1			sys_reg(3, 0, 4, 6, 0)



#define SYS_AFSR0_EL1			sys_reg(3, 0, 5, 1, 0)
@@ -382,6 +385,9 @@ 
#define SYS_CNTP_CTL_EL0		sys_reg(3, 3, 14, 2, 1)

#define SYS_CNTP_CVAL_EL0		sys_reg(3, 3, 14, 2, 2)



#define SYS_CNTV_CTL_EL0		sys_reg(3, 3, 14, 3, 1)

#define SYS_CNTV_CVAL_EL0		sys_reg(3, 3, 14, 3, 2)



#define SYS_AARCH32_CNTP_TVAL		sys_reg(0, 0, 14, 2, 0)

#define SYS_AARCH32_CNTP_CTL		sys_reg(0, 0, 14, 2, 1)

#define SYS_AARCH32_CNTP_CVAL		sys_reg(0, 2, 0, 14, 0)
@@ -395,11 +401,14 @@ 
#define SYS_PMCCFILTR_EL0		sys_reg(3, 3, 14, 15, 7)



#define SYS_ZCR_EL2			sys_reg(3, 4, 1, 2, 0)



#define SYS_DACR32_EL2			sys_reg(3, 4, 3, 0, 0)

#define SYS_SPSR_EL2			sys_reg(3, 4, 4, 0, 0)

#define SYS_ELR_EL2			sys_reg(3, 4, 4, 0, 1)

#define SYS_IFSR32_EL2			sys_reg(3, 4, 5, 0, 1)

#define SYS_ESR_EL2			sys_reg(3, 4, 5, 2, 0)

#define SYS_VSESR_EL2			sys_reg(3, 4, 5, 2, 3)

#define SYS_FPEXC32_EL2			sys_reg(3, 4, 5, 3, 0)

#define SYS_FAR_EL2			sys_reg(3, 4, 6, 0, 0)



#define SYS_VDISR_EL2			sys_reg(3, 4, 12, 1,  1)

#define __SYS__AP0Rx_EL2(x)		sys_reg(3, 4, 12, 8, x)
@@ -444,7 +453,29 @@ 
#define SYS_ICH_LR15_EL2		__SYS__LR8_EL2(7)



/* VHE encodings for architectural EL0/1 system registers */

#define SYS_SCTLR_EL12			sys_reg(3, 5, 1, 0, 0)

#define SYS_CPACR_EL12			sys_reg(3, 5, 1, 0, 2)

#define SYS_ZCR_EL12			sys_reg(3, 5, 1, 2, 0)

#define SYS_TTBR0_EL12			sys_reg(3, 5, 2, 0, 0)

#define SYS_TTBR1_EL12			sys_reg(3, 5, 2, 0, 1)

#define SYS_TCR_EL12			sys_reg(3, 5, 2, 0, 2)

#define SYS_SPSR_EL12			sys_reg(3, 5, 4, 0, 0)

#define SYS_ELR_EL12			sys_reg(3, 5, 4, 0, 1)

#define SYS_AFSR0_EL12			sys_reg(3, 5, 5, 1, 0)

#define SYS_AFSR1_EL12			sys_reg(3, 5, 5, 1, 1)

#define SYS_ESR_EL12			sys_reg(3, 5, 5, 2, 0)

#define SYS_FAR_EL12			sys_reg(3, 5, 6, 0, 0)

#define SYS_MAIR_EL12			sys_reg(3, 5, 10, 2, 0)

#define SYS_AMAIR_EL12			sys_reg(3, 5, 10, 3, 0)

#define SYS_VBAR_EL12			sys_reg(3, 5, 12, 0, 0)

#define SYS_CONTEXTIDR_EL12		sys_reg(3, 5, 13, 0, 1)

#define SYS_CNTKCTL_EL12		sys_reg(3, 5, 14, 1, 0)

#define SYS_CNTP_TVAL_EL02		sys_reg(3, 5, 14, 2, 0)

#define SYS_CNTP_CTL_EL02		sys_reg(3, 5, 14, 2, 1)

#define SYS_CNTP_CVAL_EL02		sys_reg(3, 5, 14, 2, 2)

#define SYS_CNTV_TVAL_EL02		sys_reg(3, 5, 14, 3, 0)

#define SYS_CNTV_CTL_EL02		sys_reg(3, 5, 14, 3, 1)

#define SYS_CNTV_CVAL_EL02		sys_reg(3, 5, 14, 3, 2)



/* Common SCTLR_ELx flags. */

#define SCTLR_ELx_DSSBS	(_BITUL(44))

arch/arm64/kvm/hyp/switch.c

+7 −7
Original line number Diff line number Diff line
@@ -284,7 +284,7 @@ static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu) 
	if (ec != ESR_ELx_EC_DABT_LOW && ec != ESR_ELx_EC_IABT_LOW)

		return true;



	far = read_sysreg_el2(far);

	far = read_sysreg_el2(SYS_FAR);



	/*

	 * The HPFAR can be invalid if the stage 2 fault did not
@@ -401,7 +401,7 @@ static bool __hyp_text __hyp_handle_fpsimd(struct kvm_vcpu *vcpu) 
static bool __hyp_text fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)

{

	if (ARM_EXCEPTION_CODE(*exit_code) != ARM_EXCEPTION_IRQ)

		vcpu->arch.fault.esr_el2 = read_sysreg_el2(esr);

		vcpu->arch.fault.esr_el2 = read_sysreg_el2(SYS_ESR);



	/*

	 * We're using the raw exception code in order to only process
@@ -698,7 +698,7 @@ static void __hyp_text __hyp_call_panic_nvhe(u64 spsr, u64 elr, u64 par, 


	__hyp_do_panic(str_va,

		       spsr, elr,

		       read_sysreg(esr_el2),   read_sysreg_el2(far),

		       read_sysreg(esr_el2), read_sysreg_el2(SYS_FAR),

		       read_sysreg(hpfar_el2), par, vcpu);

}
@@ -713,15 +713,15 @@ static void __hyp_call_panic_vhe(u64 spsr, u64 elr, u64 par, 


	panic(__hyp_panic_string,

	      spsr,  elr,

	      read_sysreg_el2(esr),   read_sysreg_el2(far),

	      read_sysreg_el2(SYS_ESR),   read_sysreg_el2(SYS_FAR),

	      read_sysreg(hpfar_el2), par, vcpu);

}

NOKPROBE_SYMBOL(__hyp_call_panic_vhe);



void __hyp_text __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt)

{

	u64 spsr = read_sysreg_el2(spsr);

	u64 elr = read_sysreg_el2(elr);

	u64 spsr = read_sysreg_el2(SYS_SPSR);

	u64 elr = read_sysreg_el2(SYS_ELR);

	u64 par = read_sysreg(par_el1);



	if (!has_vhe())