Commit b42990d3 authored by Mark Brown's avatar Mark Brown Committed by Catalin Marinas
Browse files

arm64/sme: Identify supported SME vector lengths at boot 



The vector lengths used for SME are controlled through a similar set of
registers to those for SVE and enumerated using a similar algorithm with
some slight differences due to the fact that unlike SVE there are no
restrictions on which combinations of vector lengths can be supported
nor any mandatory vector lengths which must be implemented.  Add a new
vector type and implement support for enumerating it.

One slightly awkward feature is that we need to read the current vector
length using a different instruction (or enter streaming mode which
would have the same issue and be higher cost).  Rather than add an ops
structure we add special cases directly in the otherwise generic
vec_probe_vqs() function, this is a bit inelegant but it's the only
place where this is an issue.

Signed-off-by: default avatarMark Brown <broonie@kernel.org>
Reviewed-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20220419112247.711548-10-broonie@kernel.org


Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 5e64b862

arch/arm64/include/asm/cpu.h

+3 −0
Original line number Diff line number Diff line
@@ -64,6 +64,9 @@ struct cpuinfo_arm64 { 


	/* pseudo-ZCR for recording maximum ZCR_EL1 LEN value: */

	u64		reg_zcr;



	/* pseudo-SMCR for recording maximum SMCR_EL1 LEN value: */

	u64		reg_smcr;

};



DECLARE_PER_CPU(struct cpuinfo_arm64, cpu_data);

arch/arm64/include/asm/cpufeature.h

+7 −0
Original line number Diff line number Diff line
@@ -622,6 +622,13 @@ static inline bool id_aa64pfr0_sve(u64 pfr0) 
	return val > 0;

}



static inline bool id_aa64pfr1_sme(u64 pfr1)

{

	u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_SME_SHIFT);



	return val > 0;

}



static inline bool id_aa64pfr1_mte(u64 pfr1)

{

	u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_MTE_SHIFT);

arch/arm64/include/asm/fpsimd.h

+26 −0
Original line number Diff line number Diff line
@@ -78,6 +78,7 @@ extern void sme_kernel_enable(const struct arm64_cpu_capabilities *__unused); 
extern void fa64_kernel_enable(const struct arm64_cpu_capabilities *__unused);



extern u64 read_zcr_features(void);

extern u64 read_smcr_features(void);



/*

 * Helpers to translate bit indices in sve_vq_map to VQ values (and
@@ -172,6 +173,12 @@ static inline void write_vl(enum vec_type type, u64 val) 
		tmp = read_sysreg_s(SYS_ZCR_EL1) & ~ZCR_ELx_LEN_MASK;

		write_sysreg_s(tmp | val, SYS_ZCR_EL1);

		break;

#endif

#ifdef CONFIG_ARM64_SME

	case ARM64_VEC_SME:

		tmp = read_sysreg_s(SYS_SMCR_EL1) & ~SMCR_ELx_LEN_MASK;

		write_sysreg_s(tmp | val, SYS_SMCR_EL1);

		break;

#endif

	default:

		WARN_ON_ONCE(1);
@@ -268,12 +275,31 @@ static inline void sme_smstop(void) 
	asm volatile(__msr_s(SYS_SVCR_SMSTOP_SMZA_EL0, "xzr"));

}



extern void __init sme_setup(void);



static inline int sme_max_vl(void)

{

	return vec_max_vl(ARM64_VEC_SME);

}



static inline int sme_max_virtualisable_vl(void)

{

	return vec_max_virtualisable_vl(ARM64_VEC_SME);

}



extern unsigned int sme_get_vl(void);



#else



static inline void sme_smstart_sm(void) { }

static inline void sme_smstop_sm(void) { }

static inline void sme_smstop(void) { }



static inline void sme_setup(void) { }

static inline unsigned int sme_get_vl(void) { return 0; }

static inline int sme_max_vl(void) { return 0; }

static inline int sme_max_virtualisable_vl(void) { return 0; }



#endif /* ! CONFIG_ARM64_SME */



/* For use by EFI runtime services calls only */

arch/arm64/include/asm/processor.h

+1 −0
Original line number Diff line number Diff line
@@ -118,6 +118,7 @@ struct debug_info { 


enum vec_type {

	ARM64_VEC_SVE = 0,

	ARM64_VEC_SME,

	ARM64_VEC_MAX,

};

arch/arm64/kernel/cpufeature.c

+47 −0
Original line number Diff line number Diff line
@@ -581,6 +581,12 @@ static const struct arm64_ftr_bits ftr_zcr[] = { 
	ARM64_FTR_END,

};



static const struct arm64_ftr_bits ftr_smcr[] = {

	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE,

		SMCR_ELx_LEN_SHIFT, SMCR_ELx_LEN_SIZE, 0),	/* LEN */

	ARM64_FTR_END,

};



/*

 * Common ftr bits for a 32bit register with all hidden, strict

 * attributes, with 4bit feature fields and a default safe value of
@@ -687,6 +693,7 @@ static const struct __ftr_reg_entry { 


	/* Op1 = 0, CRn = 1, CRm = 2 */

	ARM64_FTR_REG(SYS_ZCR_EL1, ftr_zcr),

	ARM64_FTR_REG(SYS_SMCR_EL1, ftr_smcr),



	/* Op1 = 1, CRn = 0, CRm = 0 */

	ARM64_FTR_REG(SYS_GMID_EL1, ftr_gmid),
@@ -991,6 +998,12 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) 
		vec_init_vq_map(ARM64_VEC_SVE);

	}



	if (id_aa64pfr1_sme(info->reg_id_aa64pfr1)) {

		init_cpu_ftr_reg(SYS_SMCR_EL1, info->reg_smcr);

		if (IS_ENABLED(CONFIG_ARM64_SME))

			vec_init_vq_map(ARM64_VEC_SME);

	}



	if (id_aa64pfr1_mte(info->reg_id_aa64pfr1))

		init_cpu_ftr_reg(SYS_GMID_EL1, info->reg_gmid);
@@ -1217,6 +1230,9 @@ void update_cpu_features(int cpu, 
	taint |= check_update_ftr_reg(SYS_ID_AA64ZFR0_EL1, cpu,

				      info->reg_id_aa64zfr0, boot->reg_id_aa64zfr0);



	taint |= check_update_ftr_reg(SYS_ID_AA64SMFR0_EL1, cpu,

				      info->reg_id_aa64smfr0, boot->reg_id_aa64smfr0);



	if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {

		taint |= check_update_ftr_reg(SYS_ZCR_EL1, cpu,

					info->reg_zcr, boot->reg_zcr);
@@ -1227,6 +1243,16 @@ void update_cpu_features(int cpu, 
			vec_update_vq_map(ARM64_VEC_SVE);

	}



	if (id_aa64pfr1_sme(info->reg_id_aa64pfr1)) {

		taint |= check_update_ftr_reg(SYS_SMCR_EL1, cpu,

					info->reg_smcr, boot->reg_smcr);



		/* Probe vector lengths, unless we already gave up on SME */

		if (id_aa64pfr1_sme(read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1)) &&

		    !system_capabilities_finalized())

			vec_update_vq_map(ARM64_VEC_SME);

	}



	/*

	 * The kernel uses the LDGM/STGM instructions and the number of tags

	 * they read/write depends on the GMID_EL1.BS field. Check that the
@@ -2931,6 +2957,23 @@ static void verify_sve_features(void) 
	/* Add checks on other ZCR bits here if necessary */

}



static void verify_sme_features(void)

{

	u64 safe_smcr = read_sanitised_ftr_reg(SYS_SMCR_EL1);

	u64 smcr = read_smcr_features();



	unsigned int safe_len = safe_smcr & SMCR_ELx_LEN_MASK;

	unsigned int len = smcr & SMCR_ELx_LEN_MASK;



	if (len < safe_len || vec_verify_vq_map(ARM64_VEC_SME)) {

		pr_crit("CPU%d: SME: vector length support mismatch\n",

			smp_processor_id());

		cpu_die_early();

	}



	/* Add checks on other SMCR bits here if necessary */

}



static void verify_hyp_capabilities(void)

{

	u64 safe_mmfr1, mmfr0, mmfr1;
@@ -2983,6 +3026,9 @@ static void verify_local_cpu_capabilities(void) 
	if (system_supports_sve())

		verify_sve_features();



	if (system_supports_sme())

		verify_sme_features();



	if (is_hyp_mode_available())

		verify_hyp_capabilities();

}
@@ -3100,6 +3146,7 @@ void __init setup_cpu_features(void) 
		pr_info("emulated: Privileged Access Never (PAN) using TTBR0_EL1 switching\n");



	sve_setup();

	sme_setup();

	minsigstksz_setup();



	/* Advertise that we have computed the system capabilities */