Commit dc94f89a authored by Marc Zyngier's avatar Marc Zyngier
Browse files

Merge branch kvm-arm64/burn-the-flags into kvmarm-master/next 



* kvm-arm64/burn-the-flags:
  : .
  : Rework the per-vcpu flags to make them more manageable,
  : splitting them in different sets that have specific
  : uses:
  :
  : - configuration flags
  : - input to the world-switch
  : - state bookkeeping for the kernel itself
  :
  : The FP tracking is also simplified and tracked outside
  : of the flags as a separate state.
  : .
  KVM: arm64: Move the handling of !FP outside of the fast path
  KVM: arm64: Document why pause cannot be turned into a flag
  KVM: arm64: Reduce the size of the vcpu flag members
  KVM: arm64: Add build-time sanity checks for flags
  KVM: arm64: Warn when PENDING_EXCEPTION and INCREMENT_PC are set together
  KVM: arm64: Convert vcpu sysregs_loaded_on_cpu to a state flag
  KVM: arm64: Kill unused vcpu flags field
  KVM: arm64: Move vcpu WFIT flag to the state flag set
  KVM: arm64: Move vcpu ON_UNSUPPORTED_CPU flag to the state flag set
  KVM: arm64: Move vcpu SVE/SME flags to the state flag set
  KVM: arm64: Move vcpu debug/SPE/TRBE flags to the input flag set
  KVM: arm64: Move vcpu PC/Exception flags to the input flag set
  KVM: arm64: Move vcpu configuration flags into their own set
  KVM: arm64: Add three sets of flags to the vcpu state
  KVM: arm64: Add helpers to manipulate vcpu flags among a set
  KVM: arm64: Move FP state ownership from flag to a tristate
  KVM: arm64: Drop FP_FOREIGN_STATE from the hypervisor code

Signed-off-by: default avatarMarc Zyngier <maz@kernel.org>
parents a111daf0 b4da9187

arch/arm64/include/asm/kvm_emulate.h

+10 −1
Original line number Diff line number Diff line
@@ -473,9 +473,18 @@ static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu, 


static __always_inline void kvm_incr_pc(struct kvm_vcpu *vcpu)

{

	vcpu->arch.flags |= KVM_ARM64_INCREMENT_PC;

	WARN_ON(vcpu_get_flag(vcpu, PENDING_EXCEPTION));

	vcpu_set_flag(vcpu, INCREMENT_PC);

}



#define kvm_pend_exception(v, e)					\

	do {								\

		WARN_ON(vcpu_get_flag((v), INCREMENT_PC));		\

		vcpu_set_flag((v), PENDING_EXCEPTION);			\

		vcpu_set_flag((v), e);					\

	} while (0)





static inline bool vcpu_has_feature(struct kvm_vcpu *vcpu, int feature)

{

	return test_bit(feature, vcpu->arch.features);

arch/arm64/include/asm/kvm_host.h

+148 −55
Original line number Diff line number Diff line
@@ -325,8 +325,30 @@ struct kvm_vcpu_arch { 
	/* Exception Information */

	struct kvm_vcpu_fault_info fault;



	/* Miscellaneous vcpu state flags */

	u64 flags;

	/* Ownership of the FP regs */

	enum {

		FP_STATE_FREE,

		FP_STATE_HOST_OWNED,

		FP_STATE_GUEST_OWNED,

	} fp_state;



	/* Configuration flags, set once and for all before the vcpu can run */

	u8 cflags;



	/* Input flags to the hypervisor code, potentially cleared after use */

	u8 iflags;



	/* State flags for kernel bookkeeping, unused by the hypervisor code */

	u8 sflags;



	/*

	 * Don't run the guest (internal implementation need).

	 *

	 * Contrary to the flags above, this is set/cleared outside of

	 * a vcpu context, and thus cannot be mixed with the flags

	 * themselves (or the flag accesses need to be made atomic).

	 */

	bool pause;



	/*

	 * We maintain more than a single set of debug registers to support
@@ -376,9 +398,6 @@ struct kvm_vcpu_arch { 
	/* vcpu power state */

	struct kvm_mp_state mp_state;



	/* Don't run the guest (internal implementation need) */

	bool pause;



	/* Cache some mmu pages needed inside spinlock regions */

	struct kvm_mmu_memory_cache mmu_page_cache;
@@ -392,10 +411,6 @@ struct kvm_vcpu_arch { 
	/* Additional reset state */

	struct vcpu_reset_state	reset_state;



	/* True when deferrable sysregs are loaded on the physical CPU,

	 * see kvm_vcpu_load_sysregs_vhe and kvm_vcpu_put_sysregs_vhe. */

	bool sysregs_loaded_on_cpu;



	/* Guest PV state */

	struct {

		u64 last_steal;
@@ -403,6 +418,124 @@ struct kvm_vcpu_arch { 
	} steal;

};



/*

 * Each 'flag' is composed of a comma-separated triplet:

 *

 * - the flag-set it belongs to in the vcpu->arch structure

 * - the value for that flag

 * - the mask for that flag

 *

 *  __vcpu_single_flag() builds such a triplet for a single-bit flag.

 * unpack_vcpu_flag() extract the flag value from the triplet for

 * direct use outside of the flag accessors.

 */

#define __vcpu_single_flag(_set, _f)	_set, (_f), (_f)



#define __unpack_flag(_set, _f, _m)	_f

#define unpack_vcpu_flag(...)		__unpack_flag(__VA_ARGS__)



#define __build_check_flag(v, flagset, f, m)			\

	do {							\

		typeof(v->arch.flagset) *_fset;			\

								\

		/* Check that the flags fit in the mask */	\

		BUILD_BUG_ON(HWEIGHT(m) != HWEIGHT((f) | (m)));	\

		/* Check that the flags fit in the type */	\

		BUILD_BUG_ON((sizeof(*_fset) * 8) <= __fls(m));	\

	} while (0)



#define __vcpu_get_flag(v, flagset, f, m)			\

	({							\

		__build_check_flag(v, flagset, f, m);		\

								\

		v->arch.flagset & (m);				\

	})



#define __vcpu_set_flag(v, flagset, f, m)			\

	do {							\

		typeof(v->arch.flagset) *fset;			\

								\

		__build_check_flag(v, flagset, f, m);		\

								\

		fset = &v->arch.flagset;			\

		if (HWEIGHT(m) > 1)				\

			*fset &= ~(m);				\

		*fset |= (f);					\

	} while (0)



#define __vcpu_clear_flag(v, flagset, f, m)			\

	do {							\

		typeof(v->arch.flagset) *fset;			\

								\

		__build_check_flag(v, flagset, f, m);		\

								\

		fset = &v->arch.flagset;			\

		*fset &= ~(m);					\

	} while (0)



#define vcpu_get_flag(v, ...)	__vcpu_get_flag((v), __VA_ARGS__)

#define vcpu_set_flag(v, ...)	__vcpu_set_flag((v), __VA_ARGS__)

#define vcpu_clear_flag(v, ...)	__vcpu_clear_flag((v), __VA_ARGS__)



/* SVE exposed to guest */

#define GUEST_HAS_SVE		__vcpu_single_flag(cflags, BIT(0))

/* SVE config completed */

#define VCPU_SVE_FINALIZED	__vcpu_single_flag(cflags, BIT(1))

/* PTRAUTH exposed to guest */

#define GUEST_HAS_PTRAUTH	__vcpu_single_flag(cflags, BIT(2))



/* Exception pending */

#define PENDING_EXCEPTION	__vcpu_single_flag(iflags, BIT(0))

/*

 * PC increment. Overlaps with EXCEPT_MASK on purpose so that it can't

 * be set together with an exception...

 */

#define INCREMENT_PC		__vcpu_single_flag(iflags, BIT(1))

/* Target EL/MODE (not a single flag, but let's abuse the macro) */

#define EXCEPT_MASK		__vcpu_single_flag(iflags, GENMASK(3, 1))



/* Helpers to encode exceptions with minimum fuss */

#define __EXCEPT_MASK_VAL	unpack_vcpu_flag(EXCEPT_MASK)

#define __EXCEPT_SHIFT		__builtin_ctzl(__EXCEPT_MASK_VAL)

#define __vcpu_except_flags(_f)	iflags, (_f << __EXCEPT_SHIFT), __EXCEPT_MASK_VAL



/*

 * When PENDING_EXCEPTION is set, EXCEPT_MASK can take the following

 * values:

 *

 * For AArch32 EL1:

 */

#define EXCEPT_AA32_UND		__vcpu_except_flags(0)

#define EXCEPT_AA32_IABT	__vcpu_except_flags(1)

#define EXCEPT_AA32_DABT	__vcpu_except_flags(2)

/* For AArch64: */

#define EXCEPT_AA64_EL1_SYNC	__vcpu_except_flags(0)

#define EXCEPT_AA64_EL1_IRQ	__vcpu_except_flags(1)

#define EXCEPT_AA64_EL1_FIQ	__vcpu_except_flags(2)

#define EXCEPT_AA64_EL1_SERR	__vcpu_except_flags(3)

/* For AArch64 with NV (one day): */

#define EXCEPT_AA64_EL2_SYNC	__vcpu_except_flags(4)

#define EXCEPT_AA64_EL2_IRQ	__vcpu_except_flags(5)

#define EXCEPT_AA64_EL2_FIQ	__vcpu_except_flags(6)

#define EXCEPT_AA64_EL2_SERR	__vcpu_except_flags(7)

/* Guest debug is live */

#define DEBUG_DIRTY		__vcpu_single_flag(iflags, BIT(4))

/* Save SPE context if active  */

#define DEBUG_STATE_SAVE_SPE	__vcpu_single_flag(iflags, BIT(5))

/* Save TRBE context if active  */

#define DEBUG_STATE_SAVE_TRBE	__vcpu_single_flag(iflags, BIT(6))



/* SVE enabled for host EL0 */

#define HOST_SVE_ENABLED	__vcpu_single_flag(sflags, BIT(0))

/* SME enabled for EL0 */

#define HOST_SME_ENABLED	__vcpu_single_flag(sflags, BIT(1))

/* Physical CPU not in supported_cpus */

#define ON_UNSUPPORTED_CPU	__vcpu_single_flag(sflags, BIT(2))

/* WFIT instruction trapped */

#define IN_WFIT			__vcpu_single_flag(sflags, BIT(3))

/* vcpu system registers loaded on physical CPU */

#define SYSREGS_ON_CPU		__vcpu_single_flag(sflags, BIT(4))



/* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */

#define vcpu_sve_pffr(vcpu) (kern_hyp_va((vcpu)->arch.sve_state) +	\

			     sve_ffr_offset((vcpu)->arch.sve_max_vl))
@@ -423,70 +556,31 @@ struct kvm_vcpu_arch { 
	__size_ret;							\

})



/* vcpu_arch flags field values: */

#define KVM_ARM64_DEBUG_DIRTY		(1 << 0)

#define KVM_ARM64_FP_ENABLED		(1 << 1) /* guest FP regs loaded */

#define KVM_ARM64_FP_HOST		(1 << 2) /* host FP regs loaded */

#define KVM_ARM64_HOST_SVE_ENABLED	(1 << 4) /* SVE enabled for EL0 */

#define KVM_ARM64_GUEST_HAS_SVE		(1 << 5) /* SVE exposed to guest */

#define KVM_ARM64_VCPU_SVE_FINALIZED	(1 << 6) /* SVE config completed */

#define KVM_ARM64_GUEST_HAS_PTRAUTH	(1 << 7) /* PTRAUTH exposed to guest */

#define KVM_ARM64_PENDING_EXCEPTION	(1 << 8) /* Exception pending */

/*

 * Overlaps with KVM_ARM64_EXCEPT_MASK on purpose so that it can't be

 * set together with an exception...

 */

#define KVM_ARM64_INCREMENT_PC		(1 << 9) /* Increment PC */

#define KVM_ARM64_EXCEPT_MASK		(7 << 9) /* Target EL/MODE */

/*

 * When KVM_ARM64_PENDING_EXCEPTION is set, KVM_ARM64_EXCEPT_MASK can

 * take the following values:

 *

 * For AArch32 EL1:

 */

#define KVM_ARM64_EXCEPT_AA32_UND	(0 << 9)

#define KVM_ARM64_EXCEPT_AA32_IABT	(1 << 9)

#define KVM_ARM64_EXCEPT_AA32_DABT	(2 << 9)

/* For AArch64: */

#define KVM_ARM64_EXCEPT_AA64_ELx_SYNC	(0 << 9)

#define KVM_ARM64_EXCEPT_AA64_ELx_IRQ	(1 << 9)

#define KVM_ARM64_EXCEPT_AA64_ELx_FIQ	(2 << 9)

#define KVM_ARM64_EXCEPT_AA64_ELx_SERR	(3 << 9)

#define KVM_ARM64_EXCEPT_AA64_EL1	(0 << 11)

#define KVM_ARM64_EXCEPT_AA64_EL2	(1 << 11)



#define KVM_ARM64_DEBUG_STATE_SAVE_SPE	(1 << 12) /* Save SPE context if active  */

#define KVM_ARM64_DEBUG_STATE_SAVE_TRBE	(1 << 13) /* Save TRBE context if active  */

#define KVM_ARM64_FP_FOREIGN_FPSTATE	(1 << 14)

#define KVM_ARM64_ON_UNSUPPORTED_CPU	(1 << 15) /* Physical CPU not in supported_cpus */

#define KVM_ARM64_HOST_SME_ENABLED	(1 << 16) /* SME enabled for EL0 */

#define KVM_ARM64_WFIT			(1 << 17) /* WFIT instruction trapped */



#define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE | \

				 KVM_GUESTDBG_USE_SW_BP | \

				 KVM_GUESTDBG_USE_HW | \

				 KVM_GUESTDBG_SINGLESTEP)



#define vcpu_has_sve(vcpu) (system_supports_sve() &&			\

			    ((vcpu)->arch.flags & KVM_ARM64_GUEST_HAS_SVE))

			    vcpu_get_flag(vcpu, GUEST_HAS_SVE))



#ifdef CONFIG_ARM64_PTR_AUTH

#define vcpu_has_ptrauth(vcpu)						\

	((cpus_have_final_cap(ARM64_HAS_ADDRESS_AUTH) ||		\

	  cpus_have_final_cap(ARM64_HAS_GENERIC_AUTH)) &&		\

	 (vcpu)->arch.flags & KVM_ARM64_GUEST_HAS_PTRAUTH)

	  vcpu_get_flag(vcpu, GUEST_HAS_PTRAUTH))

#else

#define vcpu_has_ptrauth(vcpu)		false

#endif



#define vcpu_on_unsupported_cpu(vcpu)					\

	((vcpu)->arch.flags & KVM_ARM64_ON_UNSUPPORTED_CPU)

	vcpu_get_flag(vcpu, ON_UNSUPPORTED_CPU)



#define vcpu_set_on_unsupported_cpu(vcpu)				\

	((vcpu)->arch.flags |= KVM_ARM64_ON_UNSUPPORTED_CPU)

	vcpu_set_flag(vcpu, ON_UNSUPPORTED_CPU)



#define vcpu_clear_on_unsupported_cpu(vcpu)				\

	((vcpu)->arch.flags &= ~KVM_ARM64_ON_UNSUPPORTED_CPU)

	vcpu_clear_flag(vcpu, ON_UNSUPPORTED_CPU)



#define vcpu_gp_regs(v)		(&(v)->arch.ctxt.regs)
@@ -831,8 +925,7 @@ void kvm_init_protected_traps(struct kvm_vcpu *vcpu); 
int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature);

bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu);



#define kvm_arm_vcpu_sve_finalized(vcpu) \

	((vcpu)->arch.flags & KVM_ARM64_VCPU_SVE_FINALIZED)

#define kvm_arm_vcpu_sve_finalized(vcpu) vcpu_get_flag(vcpu, VCPU_SVE_FINALIZED)



#define kvm_has_mte(kvm)					\

	(system_supports_mte() &&				\

arch/arm64/kvm/arch_timer.c

+1 −1
Original line number Diff line number Diff line
@@ -242,7 +242,7 @@ static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx) 
static bool vcpu_has_wfit_active(struct kvm_vcpu *vcpu)

{

	return (cpus_have_final_cap(ARM64_HAS_WFXT) &&

		(vcpu->arch.flags & KVM_ARM64_WFIT));

		vcpu_get_flag(vcpu, IN_WFIT));

}



static u64 wfit_delay_ns(struct kvm_vcpu *vcpu)

arch/arm64/kvm/arm.c

+9 −3
Original line number Diff line number Diff line
@@ -330,6 +330,12 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 


	vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO;



	/*

	 * Default value for the FP state, will be overloaded at load

	 * time if we support FP (pretty likely)

	 */

	vcpu->arch.fp_state = FP_STATE_FREE;



	/* Set up the timer */

	kvm_timer_vcpu_init(vcpu);
@@ -659,7 +665,7 @@ void kvm_vcpu_wfi(struct kvm_vcpu *vcpu) 
	preempt_enable();



	kvm_vcpu_halt(vcpu);

	vcpu->arch.flags &= ~KVM_ARM64_WFIT;

	vcpu_clear_flag(vcpu, IN_WFIT);

	kvm_clear_request(KVM_REQ_UNHALT, vcpu);



	preempt_disable();
@@ -1015,8 +1021,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) 
	 * the vcpu state. Note that this relies on __kvm_adjust_pc()

	 * being preempt-safe on VHE.

	 */

	if (unlikely(vcpu->arch.flags & (KVM_ARM64_PENDING_EXCEPTION |

					 KVM_ARM64_INCREMENT_PC)))

	if (unlikely(vcpu_get_flag(vcpu, PENDING_EXCEPTION) ||

		     vcpu_get_flag(vcpu, INCREMENT_PC)))

		kvm_call_hyp(__kvm_adjust_pc, vcpu);



	vcpu_put(vcpu);

arch/arm64/kvm/debug.c

+12 −13
Original line number Diff line number Diff line
@@ -104,11 +104,11 @@ static void kvm_arm_setup_mdcr_el2(struct kvm_vcpu *vcpu) 
	 * Trap debug register access when one of the following is true:

	 *  - Userspace is using the hardware to debug the guest

	 *  (KVM_GUESTDBG_USE_HW is set).

	 *  - The guest is not using debug (KVM_ARM64_DEBUG_DIRTY is clear).

	 *  - The guest is not using debug (DEBUG_DIRTY clear).

	 *  - The guest has enabled the OS Lock (debug exceptions are blocked).

	 */

	if ((vcpu->guest_debug & KVM_GUESTDBG_USE_HW) ||

	    !(vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY) ||

	    !vcpu_get_flag(vcpu, DEBUG_DIRTY) ||

	    kvm_vcpu_os_lock_enabled(vcpu))

		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
@@ -147,8 +147,8 @@ void kvm_arm_reset_debug_ptr(struct kvm_vcpu *vcpu) 
 * debug related registers.

 *

 * Additionally, KVM only traps guest accesses to the debug registers if

 * the guest is not actively using them (see the KVM_ARM64_DEBUG_DIRTY

 * flag on vcpu->arch.flags).  Since the guest must not interfere

 * the guest is not actively using them (see the DEBUG_DIRTY

 * flag on vcpu->arch.iflags).  Since the guest must not interfere

 * with the hardware state when debugging the guest, we must ensure that

 * trapping is enabled whenever we are debugging the guest using the

 * debug registers.
@@ -205,9 +205,8 @@ void kvm_arm_setup_debug(struct kvm_vcpu *vcpu) 
		 *

		 * We simply switch the debug_ptr to point to our new

		 * external_debug_state which has been populated by the

		 * debug ioctl. The existing KVM_ARM64_DEBUG_DIRTY

		 * mechanism ensures the registers are updated on the

		 * world switch.

		 * debug ioctl. The existing DEBUG_DIRTY mechanism ensures

		 * the registers are updated on the world switch.

		 */

		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW) {

			/* Enable breakpoints/watchpoints */
@@ -216,7 +215,7 @@ void kvm_arm_setup_debug(struct kvm_vcpu *vcpu) 
			vcpu_write_sys_reg(vcpu, mdscr, MDSCR_EL1);



			vcpu->arch.debug_ptr = &vcpu->arch.external_debug_state;

			vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;

			vcpu_set_flag(vcpu, DEBUG_DIRTY);



			trace_kvm_arm_set_regset("BKPTS", get_num_brps(),

						&vcpu->arch.debug_ptr->dbg_bcr[0],
@@ -246,7 +245,7 @@ void kvm_arm_setup_debug(struct kvm_vcpu *vcpu) 


	/* If KDE or MDE are set, perform a full save/restore cycle. */

	if (vcpu_read_sys_reg(vcpu, MDSCR_EL1) & (DBG_MDSCR_KDE | DBG_MDSCR_MDE))

		vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;

		vcpu_set_flag(vcpu, DEBUG_DIRTY);



	/* Write mdcr_el2 changes since vcpu_load on VHE systems */

	if (has_vhe() && orig_mdcr_el2 != vcpu->arch.mdcr_el2)
@@ -298,16 +297,16 @@ void kvm_arch_vcpu_load_debug_state_flags(struct kvm_vcpu *vcpu) 
	 */

	if (cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_PMSVER_SHIFT) &&

	    !(read_sysreg_s(SYS_PMBIDR_EL1) & BIT(SYS_PMBIDR_EL1_P_SHIFT)))

		vcpu->arch.flags |= KVM_ARM64_DEBUG_STATE_SAVE_SPE;

		vcpu_set_flag(vcpu, DEBUG_STATE_SAVE_SPE);



	/* Check if we have TRBE implemented and available at the host */

	if (cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_TRBE_SHIFT) &&

	    !(read_sysreg_s(SYS_TRBIDR_EL1) & TRBIDR_PROG))

		vcpu->arch.flags |= KVM_ARM64_DEBUG_STATE_SAVE_TRBE;

		vcpu_set_flag(vcpu, DEBUG_STATE_SAVE_TRBE);

}



void kvm_arch_vcpu_put_debug_state_flags(struct kvm_vcpu *vcpu)

{

	vcpu->arch.flags &= ~(KVM_ARM64_DEBUG_STATE_SAVE_SPE |

			      KVM_ARM64_DEBUG_STATE_SAVE_TRBE);

	vcpu_clear_flag(vcpu, DEBUG_STATE_SAVE_SPE);

	vcpu_clear_flag(vcpu, DEBUG_STATE_SAVE_TRBE);

}