Commit b710fe0d authored by Oliver Upton's avatar Oliver Upton
Browse files

Merge branch kvm-arm64/hvhe into kvmarm/next 



* kvm-arm64/hvhe:
  : Support for running split-hypervisor w/VHE, courtesy of Marc Zyngier
  :
  : From the cover letter:
  :
  : KVM (on ARMv8.0) and pKVM (on all revisions of the architecture) use
  : the split hypervisor model that makes the EL2 code more or less
  : standalone. In the later case, we totally ignore the VHE mode and
  : stick with the good old v8.0 EL2 setup.
  :
  : We introduce a new "mode" for KVM called hVHE, in reference to the
  : nVHE mode, and indicating that only the hypervisor is using VHE.
  KVM: arm64: Fix hVHE init on CPUs where HCR_EL2.E2H is not RES1
  arm64: Allow arm64_sw.hvhe on command line
  KVM: arm64: Force HCR_E2H in guest context when ARM64_KVM_HVHE is set
  KVM: arm64: Program the timer traps with VHE layout in hVHE mode
  KVM: arm64: Rework CPTR_EL2 programming for HVHE configuration
  KVM: arm64: Adjust EL2 stage-1 leaf AP bits when ARM64_KVM_HVHE is set
  KVM: arm64: Disable TTBR1_EL2 when using ARM64_KVM_HVHE
  KVM: arm64: Force HCR_EL2.E2H when ARM64_KVM_HVHE is set
  KVM: arm64: Key use of VHE instructions in nVHE code off ARM64_KVM_HVHE
  KVM: arm64: Remove alternatives from sysreg accessors in VHE hypervisor context
  arm64: Use CPACR_EL1 format to set CPTR_EL2 when E2H is set
  arm64: Allow EL1 physical timer access when running VHE
  arm64: Don't enable VHE for the kernel if OVERRIDE_HVHE is set
  arm64: Add KVM_HVHE capability and has_hvhe() predicate
  arm64: Turn kaslr_feature_override into a generic SW feature override
  arm64: Prevent the use of is_kernel_in_hyp_mode() in hypervisor code
  KVM: arm64: Drop is_kernel_in_hyp_mode() from __invalidate_icache_guest_page()

Signed-off-by: default avatarOliver Upton <oliver.upton@linux.dev>
parents 1a08f492 1700f89c

arch/arm64/include/asm/cpufeature.h

+5 −0
Original line number Diff line number Diff line
@@ -15,6 +15,9 @@ 
#define MAX_CPU_FEATURES	128

#define cpu_feature(x)		KERNEL_HWCAP_ ## x



#define ARM64_SW_FEATURE_OVERRIDE_NOKASLR	0

#define ARM64_SW_FEATURE_OVERRIDE_HVHE		4



#ifndef __ASSEMBLY__



#include <linux/bug.h>
@@ -925,6 +928,8 @@ extern struct arm64_ftr_override id_aa64smfr0_override; 
extern struct arm64_ftr_override id_aa64isar1_override;

extern struct arm64_ftr_override id_aa64isar2_override;



extern struct arm64_ftr_override arm64_sw_feature_override;



u32 get_kvm_ipa_limit(void);

void dump_cpu_features(void);

arch/arm64/include/asm/el2_setup.h

+24 −3
Original line number Diff line number Diff line
@@ -34,6 +34,11 @@ 
 */

.macro __init_el2_timers

	mov	x0, #3				// Enable EL1 physical timers

	mrs	x1, hcr_el2

	and	x1, x1, #HCR_E2H

	cbz	x1, .LnVHE_\@

	lsl	x0, x0, #10

.LnVHE_\@:

	msr	cnthctl_el2, x0

	msr	cntvoff_el2, xzr		// Clear virtual offset

.endm
@@ -124,8 +129,15 @@ 
.endm



/* Coprocessor traps */

.macro __init_el2_nvhe_cptr

.macro __init_el2_cptr

	mrs	x1, hcr_el2

	and	x1, x1, #HCR_E2H

	cbz	x1, .LnVHE_\@

	mov	x0, #(CPACR_EL1_FPEN_EL1EN | CPACR_EL1_FPEN_EL0EN)

	b	.Lset_cptr_\@

.LnVHE_\@:

	mov	x0, #0x33ff

.Lset_cptr_\@:

	msr	cptr_el2, x0			// Disable copro. traps to EL2

.endm
@@ -191,9 +203,8 @@ 
	__init_el2_gicv3

	__init_el2_hstr

	__init_el2_nvhe_idregs

	__init_el2_nvhe_cptr

	__init_el2_cptr

	__init_el2_fgt

	__init_el2_nvhe_prepare_eret

.endm



#ifndef __KVM_NVHE_HYPERVISOR__
@@ -239,7 +250,17 @@ 


.Linit_sve_\@:	/* SVE register access */

	mrs	x0, cptr_el2			// Disable SVE traps

	mrs	x1, hcr_el2

	and	x1, x1, #HCR_E2H

	cbz	x1, .Lcptr_nvhe_\@



	// VHE case

	orr	x0, x0, #(CPACR_EL1_ZEN_EL1EN | CPACR_EL1_ZEN_EL0EN)

	b	.Lset_cptr_\@



.Lcptr_nvhe_\@: // nVHE case

	bic	x0, x0, #CPTR_EL2_TZ

.Lset_cptr_\@:

	msr	cptr_el2, x0

	isb

	mov	x1, #ZCR_ELx_LEN_MASK		// SVE: Enable full vector

arch/arm64/include/asm/kvm_arm.h

+1 −3
Original line number Diff line number Diff line
@@ -285,7 +285,6 @@ 
#define CPTR_EL2_TFP	(1 << CPTR_EL2_TFP_SHIFT)

#define CPTR_EL2_TZ	(1 << 8)

#define CPTR_NVHE_EL2_RES1	0x000032ff /* known RES1 bits in CPTR_EL2 (nVHE) */

#define CPTR_EL2_DEFAULT	CPTR_NVHE_EL2_RES1

#define CPTR_NVHE_EL2_RES0	(GENMASK(63, 32) |	\

				 GENMASK(29, 21) |	\

				 GENMASK(19, 14) |	\
@@ -347,8 +346,7 @@ 
	ECN(SOFTSTP_CUR), ECN(WATCHPT_LOW), ECN(WATCHPT_CUR), \

	ECN(BKPT32), ECN(VECTOR32), ECN(BRK64), ECN(ERET)



#define CPACR_EL1_DEFAULT	(CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN |\

				 CPACR_EL1_ZEN_EL1EN)

#define CPACR_EL1_TTA		(1 << 28)



#define kvm_mode_names				\

	{ PSR_MODE_EL0t,	"EL0t" },	\

arch/arm64/include/asm/kvm_emulate.h

+32 −1
Original line number Diff line number Diff line
@@ -74,7 +74,7 @@ static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu) 
static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu)

{

	vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS;

	if (is_kernel_in_hyp_mode())

	if (has_vhe() || has_hvhe())

		vcpu->arch.hcr_el2 |= HCR_E2H;

	if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN)) {

		/* route synchronous external abort exceptions to EL2 */
@@ -570,4 +570,35 @@ static inline bool vcpu_has_feature(struct kvm_vcpu *vcpu, int feature) 
	return test_bit(feature, vcpu->arch.features);

}



static __always_inline u64 kvm_get_reset_cptr_el2(struct kvm_vcpu *vcpu)

{

	u64 val;



	if (has_vhe()) {

		val = (CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN |

		       CPACR_EL1_ZEN_EL1EN);

	} else if (has_hvhe()) {

		val = (CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN);

	} else {

		val = CPTR_NVHE_EL2_RES1;



		if (vcpu_has_sve(vcpu) &&

		    (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED))

			val |= CPTR_EL2_TZ;

		if (cpus_have_final_cap(ARM64_SME))

			val &= ~CPTR_EL2_TSM;

	}



	return val;

}



static __always_inline void kvm_reset_cptr_el2(struct kvm_vcpu *vcpu)

{

	u64 val = kvm_get_reset_cptr_el2(vcpu);



	if (has_vhe() || has_hvhe())

		write_sysreg(val, cpacr_el1);

	else

		write_sysreg(val, cptr_el2);

}

#endif /* __ARM64_KVM_EMULATE_H__ */

arch/arm64/include/asm/kvm_hyp.h

+28 −9
Original line number Diff line number Diff line
@@ -16,12 +16,35 @@ DECLARE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt); 
DECLARE_PER_CPU(unsigned long, kvm_hyp_vector);

DECLARE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);



/*

 * Unified accessors for registers that have a different encoding

 * between VHE and non-VHE. They must be specified without their "ELx"

 * encoding, but with the SYS_ prefix, as defined in asm/sysreg.h.

 */



#if defined(__KVM_VHE_HYPERVISOR__)



#define read_sysreg_el0(r)	read_sysreg_s(r##_EL02)

#define write_sysreg_el0(v,r)	write_sysreg_s(v, r##_EL02)

#define read_sysreg_el1(r)	read_sysreg_s(r##_EL12)

#define write_sysreg_el1(v,r)	write_sysreg_s(v, r##_EL12)

#define read_sysreg_el2(r)	read_sysreg_s(r##_EL1)

#define write_sysreg_el2(v,r)	write_sysreg_s(v, r##_EL1)



#else // !__KVM_VHE_HYPERVISOR__



#if defined(__KVM_NVHE_HYPERVISOR__)

#define VHE_ALT_KEY	ARM64_KVM_HVHE

#else

#define VHE_ALT_KEY	ARM64_HAS_VIRT_HOST_EXTN

#endif



#define read_sysreg_elx(r,nvh,vh)					\

	({								\

		u64 reg;						\

		asm volatile(ALTERNATIVE(__mrs_s("%0", r##nvh),		\

					 __mrs_s("%0", r##vh),		\

					 ARM64_HAS_VIRT_HOST_EXTN)	\

					 VHE_ALT_KEY)			\

			     : "=r" (reg));				\

		reg;							\

	})
@@ -31,16 +54,10 @@ DECLARE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); 
		u64 __val = (u64)(v);					\

		asm volatile(ALTERNATIVE(__msr_s(r##nvh, "%x0"),	\

					 __msr_s(r##vh, "%x0"),		\

					 ARM64_HAS_VIRT_HOST_EXTN)	\

					 VHE_ALT_KEY)			\

					 : : "rZ" (__val));		\

	} while (0)



/*

 * Unified accessors for registers that have a different encoding

 * between VHE and non-VHE. They must be specified without their "ELx"

 * encoding, but with the SYS_ prefix, as defined in asm/sysreg.h.

 */



#define read_sysreg_el0(r)	read_sysreg_elx(r, _EL0, _EL02)

#define write_sysreg_el0(v,r)	write_sysreg_elx(v, r, _EL0, _EL02)

#define read_sysreg_el1(r)	read_sysreg_elx(r, _EL1, _EL12)
@@ -48,6 +65,8 @@ DECLARE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); 
#define read_sysreg_el2(r)	read_sysreg_elx(r, _EL2, _EL1)

#define write_sysreg_el2(v,r)	write_sysreg_elx(v, r, _EL2, _EL1)



#endif	// __KVM_VHE_HYPERVISOR__



/*

 * Without an __arch_swab32(), we fall back to ___constant_swab32(), but the

 * static inline can allow the compiler to out-of-line this. KVM always wants