Commit df95b0f1 authored by Thomas Gleixner's avatar Thomas Gleixner Committed by Borislav Petkov
Browse files

x86/fpu: Move os_xsave() and os_xrstor() to core 



Nothing outside the core code needs these.

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarBorislav Petkov <bp@suse.de>
Link: https://lkml.kernel.org/r/20211015011539.513368075@linutronix.de
parent b579d0c3

arch/x86/include/asm/fpu/internal.h

+0 −165
Original line number Diff line number Diff line
@@ -161,171 +161,6 @@ static inline void fxsave(struct fxregs_state *fx) 
		asm volatile("fxsaveq %[fx]" : [fx] "=m" (*fx));

}



/* These macros all use (%edi)/(%rdi) as the single memory argument. */

#define XSAVE		".byte " REX_PREFIX "0x0f,0xae,0x27"

#define XSAVEOPT	".byte " REX_PREFIX "0x0f,0xae,0x37"

#define XSAVES		".byte " REX_PREFIX "0x0f,0xc7,0x2f"

#define XRSTOR		".byte " REX_PREFIX "0x0f,0xae,0x2f"

#define XRSTORS		".byte " REX_PREFIX "0x0f,0xc7,0x1f"



/*

 * After this @err contains 0 on success or the trap number when the

 * operation raises an exception.

 */

#define XSTATE_OP(op, st, lmask, hmask, err)				\

	asm volatile("1:" op "\n\t"					\

		     "xor %[err], %[err]\n"				\

		     "2:\n\t"						\

		     _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_FAULT_MCE_SAFE)	\

		     : [err] "=a" (err)					\

		     : "D" (st), "m" (*st), "a" (lmask), "d" (hmask)	\

		     : "memory")



/*

 * If XSAVES is enabled, it replaces XSAVEOPT because it supports a compact

 * format and supervisor states in addition to modified optimization in

 * XSAVEOPT.

 *

 * Otherwise, if XSAVEOPT is enabled, XSAVEOPT replaces XSAVE because XSAVEOPT

 * supports modified optimization which is not supported by XSAVE.

 *

 * We use XSAVE as a fallback.

 *

 * The 661 label is defined in the ALTERNATIVE* macros as the address of the

 * original instruction which gets replaced. We need to use it here as the

 * address of the instruction where we might get an exception at.

 */

#define XSTATE_XSAVE(st, lmask, hmask, err)				\

	asm volatile(ALTERNATIVE_2(XSAVE,				\

				   XSAVEOPT, X86_FEATURE_XSAVEOPT,	\

				   XSAVES,   X86_FEATURE_XSAVES)	\

		     "\n"						\

		     "xor %[err], %[err]\n"				\

		     "3:\n"						\

		     ".pushsection .fixup,\"ax\"\n"			\

		     "4: movl $-2, %[err]\n"				\

		     "jmp 3b\n"						\

		     ".popsection\n"					\

		     _ASM_EXTABLE(661b, 4b)				\

		     : [err] "=r" (err)					\

		     : "D" (st), "m" (*st), "a" (lmask), "d" (hmask)	\

		     : "memory")



/*

 * Use XRSTORS to restore context if it is enabled. XRSTORS supports compact

 * XSAVE area format.

 */

#define XSTATE_XRESTORE(st, lmask, hmask)				\

	asm volatile(ALTERNATIVE(XRSTOR,				\

				 XRSTORS, X86_FEATURE_XSAVES)		\

		     "\n"						\

		     "3:\n"						\

		     _ASM_EXTABLE_TYPE(661b, 3b, EX_TYPE_FPU_RESTORE)	\

		     :							\

		     : "D" (st), "m" (*st), "a" (lmask), "d" (hmask)	\

		     : "memory")



/*

 * Save processor xstate to xsave area.

 *

 * Uses either XSAVE or XSAVEOPT or XSAVES depending on the CPU features

 * and command line options. The choice is permanent until the next reboot.

 */

static inline void os_xsave(struct xregs_state *xstate)

{

	u64 mask = xfeatures_mask_all;

	u32 lmask = mask;

	u32 hmask = mask >> 32;

	int err;



	WARN_ON_FPU(!alternatives_patched);



	XSTATE_XSAVE(xstate, lmask, hmask, err);



	/* We should never fault when copying to a kernel buffer: */

	WARN_ON_FPU(err);

}



/*

 * Restore processor xstate from xsave area.

 *

 * Uses XRSTORS when XSAVES is used, XRSTOR otherwise.

 */

static inline void os_xrstor(struct xregs_state *xstate, u64 mask)

{

	u32 lmask = mask;

	u32 hmask = mask >> 32;



	XSTATE_XRESTORE(xstate, lmask, hmask);

}



/*

 * Save xstate to user space xsave area.

 *

 * We don't use modified optimization because xrstor/xrstors might track

 * a different application.

 *

 * We don't use compacted format xsave area for backward compatibility for

 * old applications which don't understand the compacted format of the

 * xsave area.

 *

 * The caller has to zero buf::header before calling this because XSAVE*

 * does not touch the reserved fields in the header.

 */

static inline int xsave_to_user_sigframe(struct xregs_state __user *buf)

{

	/*

	 * Include the features which are not xsaved/rstored by the kernel

	 * internally, e.g. PKRU. That's user space ABI and also required

	 * to allow the signal handler to modify PKRU.

	 */

	u64 mask = xfeatures_mask_uabi();

	u32 lmask = mask;

	u32 hmask = mask >> 32;

	int err;



	stac();

	XSTATE_OP(XSAVE, buf, lmask, hmask, err);

	clac();



	return err;

}



/*

 * Restore xstate from user space xsave area.

 */

static inline int xrstor_from_user_sigframe(struct xregs_state __user *buf, u64 mask)

{

	struct xregs_state *xstate = ((__force struct xregs_state *)buf);

	u32 lmask = mask;

	u32 hmask = mask >> 32;

	int err;



	stac();

	XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);

	clac();



	return err;

}



/*

 * Restore xstate from kernel space xsave area, return an error code instead of

 * an exception.

 */

static inline int os_xrstor_safe(struct xregs_state *xstate, u64 mask)

{

	u32 lmask = mask;

	u32 hmask = mask >> 32;

	int err;



	if (cpu_feature_enabled(X86_FEATURE_XSAVES))

		XSTATE_OP(XRSTORS, xstate, lmask, hmask, err);

	else

		XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);



	return err;

}



extern void restore_fpregs_from_fpstate(union fpregs_state *fpstate, u64 mask);



extern bool copy_fpstate_to_sigframe(void __user *buf, void __user *fp, int size);

arch/x86/include/asm/fpu/xstate.h

+0 −6
Original line number Diff line number Diff line
@@ -78,12 +78,6 @@ 
				      XFEATURE_MASK_INDEPENDENT | \

				      XFEATURE_MASK_SUPERVISOR_UNSUPPORTED)



#ifdef CONFIG_X86_64

#define REX_PREFIX	"0x48, "

#else

#define REX_PREFIX

#endif



extern u64 xfeatures_mask_all;



static inline u64 xfeatures_mask_supervisor(void)

arch/x86/kernel/fpu/signal.c

+1 −0
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ 
#include <asm/trace/fpu.h>



#include "internal.h"

#include "xstate.h"



static struct _fpx_sw_bytes fx_sw_reserved __ro_after_init;

static struct _fpx_sw_bytes fx_sw_reserved_ia32 __ro_after_init;

arch/x86/kernel/fpu/xstate.h

+174 −0
Original line number Diff line number Diff line
@@ -18,4 +18,178 @@ static inline void xstate_init_xcomp_bv(struct xregs_state *xsave, u64 mask) 
extern void __copy_xstate_to_uabi_buf(struct membuf to, struct xregs_state *xsave,

				      u32 pkru_val, enum xstate_copy_mode copy_mode);



/* XSAVE/XRSTOR wrapper functions */



#ifdef CONFIG_X86_64

#define REX_PREFIX	"0x48, "

#else

#define REX_PREFIX

#endif



/* These macros all use (%edi)/(%rdi) as the single memory argument. */

#define XSAVE		".byte " REX_PREFIX "0x0f,0xae,0x27"

#define XSAVEOPT	".byte " REX_PREFIX "0x0f,0xae,0x37"

#define XSAVES		".byte " REX_PREFIX "0x0f,0xc7,0x2f"

#define XRSTOR		".byte " REX_PREFIX "0x0f,0xae,0x2f"

#define XRSTORS		".byte " REX_PREFIX "0x0f,0xc7,0x1f"



/*

 * After this @err contains 0 on success or the trap number when the

 * operation raises an exception.

 */

#define XSTATE_OP(op, st, lmask, hmask, err)				\

	asm volatile("1:" op "\n\t"					\

		     "xor %[err], %[err]\n"				\

		     "2:\n\t"						\

		     _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_FAULT_MCE_SAFE)	\

		     : [err] "=a" (err)					\

		     : "D" (st), "m" (*st), "a" (lmask), "d" (hmask)	\

		     : "memory")



/*

 * If XSAVES is enabled, it replaces XSAVEOPT because it supports a compact

 * format and supervisor states in addition to modified optimization in

 * XSAVEOPT.

 *

 * Otherwise, if XSAVEOPT is enabled, XSAVEOPT replaces XSAVE because XSAVEOPT

 * supports modified optimization which is not supported by XSAVE.

 *

 * We use XSAVE as a fallback.

 *

 * The 661 label is defined in the ALTERNATIVE* macros as the address of the

 * original instruction which gets replaced. We need to use it here as the

 * address of the instruction where we might get an exception at.

 */

#define XSTATE_XSAVE(st, lmask, hmask, err)				\

	asm volatile(ALTERNATIVE_2(XSAVE,				\

				   XSAVEOPT, X86_FEATURE_XSAVEOPT,	\

				   XSAVES,   X86_FEATURE_XSAVES)	\

		     "\n"						\

		     "xor %[err], %[err]\n"				\

		     "3:\n"						\

		     ".pushsection .fixup,\"ax\"\n"			\

		     "4: movl $-2, %[err]\n"				\

		     "jmp 3b\n"						\

		     ".popsection\n"					\

		     _ASM_EXTABLE(661b, 4b)				\

		     : [err] "=r" (err)					\

		     : "D" (st), "m" (*st), "a" (lmask), "d" (hmask)	\

		     : "memory")



/*

 * Use XRSTORS to restore context if it is enabled. XRSTORS supports compact

 * XSAVE area format.

 */

#define XSTATE_XRESTORE(st, lmask, hmask)				\

	asm volatile(ALTERNATIVE(XRSTOR,				\

				 XRSTORS, X86_FEATURE_XSAVES)		\

		     "\n"						\

		     "3:\n"						\

		     _ASM_EXTABLE_TYPE(661b, 3b, EX_TYPE_FPU_RESTORE)	\

		     :							\

		     : "D" (st), "m" (*st), "a" (lmask), "d" (hmask)	\

		     : "memory")



/*

 * Save processor xstate to xsave area.

 *

 * Uses either XSAVE or XSAVEOPT or XSAVES depending on the CPU features

 * and command line options. The choice is permanent until the next reboot.

 */

static inline void os_xsave(struct xregs_state *xstate)

{

	u64 mask = xfeatures_mask_all;

	u32 lmask = mask;

	u32 hmask = mask >> 32;

	int err;



	WARN_ON_FPU(!alternatives_patched);



	XSTATE_XSAVE(xstate, lmask, hmask, err);



	/* We should never fault when copying to a kernel buffer: */

	WARN_ON_FPU(err);

}



/*

 * Restore processor xstate from xsave area.

 *

 * Uses XRSTORS when XSAVES is used, XRSTOR otherwise.

 */

static inline void os_xrstor(struct xregs_state *xstate, u64 mask)

{

	u32 lmask = mask;

	u32 hmask = mask >> 32;



	XSTATE_XRESTORE(xstate, lmask, hmask);

}



/*

 * Save xstate to user space xsave area.

 *

 * We don't use modified optimization because xrstor/xrstors might track

 * a different application.

 *

 * We don't use compacted format xsave area for backward compatibility for

 * old applications which don't understand the compacted format of the

 * xsave area.

 *

 * The caller has to zero buf::header before calling this because XSAVE*

 * does not touch the reserved fields in the header.

 */

static inline int xsave_to_user_sigframe(struct xregs_state __user *buf)

{

	/*

	 * Include the features which are not xsaved/rstored by the kernel

	 * internally, e.g. PKRU. That's user space ABI and also required

	 * to allow the signal handler to modify PKRU.

	 */

	u64 mask = xfeatures_mask_uabi();

	u32 lmask = mask;

	u32 hmask = mask >> 32;

	int err;



	stac();

	XSTATE_OP(XSAVE, buf, lmask, hmask, err);

	clac();



	return err;

}



/*

 * Restore xstate from user space xsave area.

 */

static inline int xrstor_from_user_sigframe(struct xregs_state __user *buf, u64 mask)

{

	struct xregs_state *xstate = ((__force struct xregs_state *)buf);

	u32 lmask = mask;

	u32 hmask = mask >> 32;

	int err;



	stac();

	XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);

	clac();



	return err;

}



/*

 * Restore xstate from kernel space xsave area, return an error code instead of

 * an exception.

 */

static inline int os_xrstor_safe(struct xregs_state *xstate, u64 mask)

{

	u32 lmask = mask;

	u32 hmask = mask >> 32;

	int err;



	if (cpu_feature_enabled(X86_FEATURE_XSAVES))

		XSTATE_OP(XRSTORS, xstate, lmask, hmask, err);

	else

		XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);



	return err;

}





#endif