Commit 655ee557 authored by Will Deacon's avatar Will Deacon
Browse files

Merge branch 'for-next/sve' into for-next/core 

* for-next/sve:
  arm64/sve: Fix warnings when SVE is disabled
  arm64/sve: Add stub for sve_max_virtualisable_vl()
  arm64/sve: Track vector lengths for tasks in an array
  arm64/sve: Explicitly load vector length when restoring SVE state
  arm64/sve: Put system wide vector length information into structs
  arm64/sve: Use accessor functions for vector lengths in thread_struct
  arm64/sve: Rename find_supported_vector_length()
  arm64/sve: Make access to FFR optional
  arm64/sve: Make sve_state_size() static
  arm64/sve: Remove sve_load_from_fpsimd_state()
  arm64/fp: Reindent fpsimd_save()
parents 3d9c8315 04ee53a5

arch/arm64/include/asm/fpsimd.h

+95 −23
Original line number Diff line number Diff line
@@ -62,15 +62,13 @@ static inline size_t sve_ffr_offset(int vl) 


static inline void *sve_pffr(struct thread_struct *thread)

{

	return (char *)thread->sve_state + sve_ffr_offset(thread->sve_vl);

	return (char *)thread->sve_state + sve_ffr_offset(thread_get_sve_vl(thread));

}



extern void sve_save_state(void *state, u32 *pfpsr);

extern void sve_save_state(void *state, u32 *pfpsr, int save_ffr);

extern void sve_load_state(void const *state, u32 const *pfpsr,

			   unsigned long vq_minus_1);

extern void sve_flush_live(unsigned long vq_minus_1);

extern void sve_load_from_fpsimd_state(struct user_fpsimd_state const *state,

				       unsigned long vq_minus_1);

			   int restore_ffr);

extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1);

extern unsigned int sve_get_vl(void);

extern void sve_set_vq(unsigned long vq_minus_1);
@@ -79,10 +77,6 @@ extern void sve_kernel_enable(const struct arm64_cpu_capabilities *__unused); 


extern u64 read_zcr_features(void);



extern int __ro_after_init sve_max_vl;

extern int __ro_after_init sve_max_virtualisable_vl;

extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);



/*

 * Helpers to translate bit indices in sve_vq_map to VQ values (and

 * vice versa).  This allows find_next_bit() to be used to find the
@@ -98,15 +92,29 @@ static inline unsigned int __bit_to_vq(unsigned int bit) 
	return SVE_VQ_MAX - bit;

}



/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */

static inline bool sve_vq_available(unsigned int vq)

{

	return test_bit(__vq_to_bit(vq), sve_vq_map);

}



#ifdef CONFIG_ARM64_SVE

struct vl_info {

	enum vec_type type;

	const char *name;		/* For display purposes */



	/* Minimum supported vector length across all CPUs */

	int min_vl;



extern size_t sve_state_size(struct task_struct const *task);

	/* Maximum supported vector length across all CPUs */

	int max_vl;

	int max_virtualisable_vl;



	/*

	 * Set of available vector lengths,

	 * where length vq encoded as bit __vq_to_bit(vq):

	 */

	DECLARE_BITMAP(vq_map, SVE_VQ_MAX);



	/* Set of vector lengths present on at least one cpu: */

	DECLARE_BITMAP(vq_partial_map, SVE_VQ_MAX);

};



#ifdef CONFIG_ARM64_SVE



extern void sve_alloc(struct task_struct *task);

extern void fpsimd_release_task(struct task_struct *task);
@@ -143,11 +151,63 @@ static inline void sve_user_enable(void) 
 * Probing and setup functions.

 * Calls to these functions must be serialised with one another.

 */

extern void __init sve_init_vq_map(void);

extern void sve_update_vq_map(void);

extern int sve_verify_vq_map(void);

enum vec_type;



extern void __init vec_init_vq_map(enum vec_type type);

extern void vec_update_vq_map(enum vec_type type);

extern int vec_verify_vq_map(enum vec_type type);

extern void __init sve_setup(void);



extern __ro_after_init struct vl_info vl_info[ARM64_VEC_MAX];



static inline void write_vl(enum vec_type type, u64 val)

{

	u64 tmp;



	switch (type) {

#ifdef CONFIG_ARM64_SVE

	case ARM64_VEC_SVE:

		tmp = read_sysreg_s(SYS_ZCR_EL1) & ~ZCR_ELx_LEN_MASK;

		write_sysreg_s(tmp | val, SYS_ZCR_EL1);

		break;

#endif

	default:

		WARN_ON_ONCE(1);

		break;

	}

}



static inline int vec_max_vl(enum vec_type type)

{

	return vl_info[type].max_vl;

}



static inline int vec_max_virtualisable_vl(enum vec_type type)

{

	return vl_info[type].max_virtualisable_vl;

}



static inline int sve_max_vl(void)

{

	return vec_max_vl(ARM64_VEC_SVE);

}



static inline int sve_max_virtualisable_vl(void)

{

	return vec_max_virtualisable_vl(ARM64_VEC_SVE);

}



/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */

static inline bool vq_available(enum vec_type type, unsigned int vq)

{

	return test_bit(__vq_to_bit(vq), vl_info[type].vq_map);

}



static inline bool sve_vq_available(unsigned int vq)

{

	return vq_available(ARM64_VEC_SVE, vq);

}



#else /* ! CONFIG_ARM64_SVE */



static inline void sve_alloc(struct task_struct *task) { }
@@ -155,6 +215,11 @@ static inline void fpsimd_release_task(struct task_struct *task) { } 
static inline void sve_sync_to_fpsimd(struct task_struct *task) { }

static inline void sve_sync_from_fpsimd_zeropad(struct task_struct *task) { }



static inline int sve_max_virtualisable_vl(void)

{

	return 0;

}



static inline int sve_set_current_vl(unsigned long arg)

{

	return -EINVAL;
@@ -165,14 +230,21 @@ static inline int sve_get_current_vl(void) 
	return -EINVAL;

}



static inline int sve_max_vl(void)

{

	return -EINVAL;

}



static inline bool sve_vq_available(unsigned int vq) { return false; }



static inline void sve_user_disable(void) { BUILD_BUG(); }

static inline void sve_user_enable(void) { BUILD_BUG(); }



#define sve_cond_update_zcr_vq(val, reg) do { } while (0)



static inline void sve_init_vq_map(void) { }

static inline void sve_update_vq_map(void) { }

static inline int sve_verify_vq_map(void) { return 0; }

static inline void vec_init_vq_map(enum vec_type t) { }

static inline void vec_update_vq_map(enum vec_type t) { }

static inline int vec_verify_vq_map(enum vec_type t) { return 0; }

static inline void sve_setup(void) { }



#endif /* ! CONFIG_ARM64_SVE */

arch/arm64/include/asm/fpsimdmacros.h

+12 −9
Original line number Diff line number Diff line
@@ -217,28 +217,36 @@ 
.macro sve_flush_z

 _for n, 0, 31, _sve_flush_z	\n

.endm

.macro sve_flush_p_ffr

.macro sve_flush_p

 _for n, 0, 15, _sve_pfalse	\n

.endm

.macro sve_flush_ffr

		_sve_wrffr	0

.endm



.macro sve_save nxbase, xpfpsr, nxtmp

.macro sve_save nxbase, xpfpsr, save_ffr, nxtmp

 _for n, 0, 31,	_sve_str_v	\n, \nxbase, \n - 34

 _for n, 0, 15,	_sve_str_p	\n, \nxbase, \n - 16

		cbz		\save_ffr, 921f

		_sve_rdffr	0

		_sve_str_p	0, \nxbase

		_sve_ldr_p	0, \nxbase, -16



		b		922f

921:

		str		xzr, [x\nxbase]		// Zero out FFR

922:

		mrs		x\nxtmp, fpsr

		str		w\nxtmp, [\xpfpsr]

		mrs		x\nxtmp, fpcr

		str		w\nxtmp, [\xpfpsr, #4]

.endm



.macro __sve_load nxbase, xpfpsr, nxtmp

.macro sve_load nxbase, xpfpsr, restore_ffr, nxtmp

 _for n, 0, 31,	_sve_ldr_v	\n, \nxbase, \n - 34

		cbz		\restore_ffr, 921f

		_sve_ldr_p	0, \nxbase

		_sve_wrffr	0

921:

 _for n, 0, 15,	_sve_ldr_p	\n, \nxbase, \n - 16



		ldr		w\nxtmp, [\xpfpsr]
@@ -246,8 +254,3 @@ 
		ldr		w\nxtmp, [\xpfpsr, #4]

		msr		fpcr, x\nxtmp

.endm
 


.macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp, xtmp2

		sve_load_vq	\xvqminus1, x\nxtmp, \xtmp2

		__sve_load	\nxbase, \xpfpsr, \nxtmp

.endm

arch/arm64/include/asm/processor.h

+47 −2
Original line number Diff line number Diff line
@@ -115,6 +115,11 @@ struct debug_info { 
#endif

};



enum vec_type {

	ARM64_VEC_SVE = 0,

	ARM64_VEC_MAX,

};



struct cpu_context {

	unsigned long x19;

	unsigned long x20;
@@ -147,8 +152,8 @@ struct thread_struct { 


	unsigned int		fpsimd_cpu;

	void			*sve_state;	/* SVE registers, if any */

	unsigned int		sve_vl;		/* SVE vector length */

	unsigned int		sve_vl_onexec;	/* SVE vl after next exec */

	unsigned int		vl[ARM64_VEC_MAX];	/* vector length */

	unsigned int		vl_onexec[ARM64_VEC_MAX]; /* vl after next exec */

	unsigned long		fault_address;	/* fault info */

	unsigned long		fault_code;	/* ESR_EL1 value */

	struct debug_info	debug;		/* debugging */
@@ -164,6 +169,46 @@ struct thread_struct { 
	u64			sctlr_user;

};



static inline unsigned int thread_get_vl(struct thread_struct *thread,

					 enum vec_type type)

{

	return thread->vl[type];

}



static inline unsigned int thread_get_sve_vl(struct thread_struct *thread)

{

	return thread_get_vl(thread, ARM64_VEC_SVE);

}



unsigned int task_get_vl(const struct task_struct *task, enum vec_type type);

void task_set_vl(struct task_struct *task, enum vec_type type,

		 unsigned long vl);

void task_set_vl_onexec(struct task_struct *task, enum vec_type type,

			unsigned long vl);

unsigned int task_get_vl_onexec(const struct task_struct *task,

				enum vec_type type);



static inline unsigned int task_get_sve_vl(const struct task_struct *task)

{

	return task_get_vl(task, ARM64_VEC_SVE);

}



static inline void task_set_sve_vl(struct task_struct *task, unsigned long vl)

{

	task_set_vl(task, ARM64_VEC_SVE, vl);

}



static inline unsigned int task_get_sve_vl_onexec(const struct task_struct *task)

{

	return task_get_vl_onexec(task, ARM64_VEC_SVE);

}



static inline void task_set_sve_vl_onexec(struct task_struct *task,

					  unsigned long vl)

{

	task_set_vl_onexec(task, ARM64_VEC_SVE, vl);

}



#define SCTLR_USER_MASK                                                        \

	(SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | SCTLR_ELx_ENDA | SCTLR_ELx_ENDB |   \

	 SCTLR_EL1_TCF0_MASK)

arch/arm64/include/asm/thread_info.h

+1 −1
Original line number Diff line number Diff line
@@ -78,7 +78,7 @@ int arch_dup_task_struct(struct task_struct *dst, 
#define TIF_SINGLESTEP		21

#define TIF_32BIT		22	/* 32bit process */

#define TIF_SVE			23	/* Scalable Vector Extension in use */

#define TIF_SVE_VL_INHERIT	24	/* Inherit sve_vl_onexec across exec */

#define TIF_SVE_VL_INHERIT	24	/* Inherit SVE vl_onexec across exec */

#define TIF_SSBD		25	/* Wants SSB mitigation */

#define TIF_TAGGED_ADDR		26	/* Allow tagged user addresses */

arch/arm64/kernel/cpufeature.c

+3 −3
Original line number Diff line number Diff line
@@ -941,7 +941,7 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) 


	if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {

		init_cpu_ftr_reg(SYS_ZCR_EL1, info->reg_zcr);

		sve_init_vq_map();

		vec_init_vq_map(ARM64_VEC_SVE);

	}



	if (id_aa64pfr1_mte(info->reg_id_aa64pfr1))
@@ -1175,7 +1175,7 @@ void update_cpu_features(int cpu, 
		/* Probe vector lengths, unless we already gave up on SVE */

		if (id_aa64pfr0_sve(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1)) &&

		    !system_capabilities_finalized())

			sve_update_vq_map();

			vec_update_vq_map(ARM64_VEC_SVE);

	}



	/*
@@ -2760,7 +2760,7 @@ static void verify_sve_features(void) 
	unsigned int safe_len = safe_zcr & ZCR_ELx_LEN_MASK;

	unsigned int len = zcr & ZCR_ELx_LEN_MASK;



	if (len < safe_len || sve_verify_vq_map()) {

	if (len < safe_len || vec_verify_vq_map(ARM64_VEC_SVE)) {

		pr_crit("CPU%d: SVE: vector length support mismatch\n",

			smp_processor_id());

		cpu_die_early();