Commit a0ff0611 authored by Thomas Gleixner's avatar Thomas Gleixner Committed by Borislav Petkov
Browse files

x86/fpu: Move KVMs FPU swapping to FPU core 



Swapping the host/guest FPU is directly fiddling with FPU internals which
requires 5 exports. The upcoming support of dynamically enabled states
would even need more.

Implement a swap function in the FPU core code and export that instead.

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarBorislav Petkov <bp@suse.de>
Reviewed-by: default avatarPaolo Bonzini <pbonzini@redhat.com>
Cc: kvm@vger.kernel.org
Link: https://lkml.kernel.org/r/20211015011539.076072399@linutronix.de
parent 63cf05a1

arch/x86/include/asm/fpu/api.h

+8 −0
Original line number Diff line number Diff line
@@ -12,6 +12,8 @@ 
#define _ASM_X86_FPU_API_H

#include <linux/bottom_half.h>



#include <asm/fpu/types.h>



/*

 * Use kernel_fpu_begin/end() if you intend to use FPU in kernel context. It

 * disables preemption so be careful if you intend to use it for long periods
@@ -108,4 +110,10 @@ extern int cpu_has_xfeatures(u64 xfeatures_mask, const char **feature_name); 


static inline void update_pasid(void) { }



/* fpstate-related functions which are exported to KVM */

extern void fpu_init_fpstate_user(struct fpu *fpu);



/* KVM specific functions */

extern void fpu_swap_kvm_fpu(struct fpu *save, struct fpu *rstor, u64 restore_mask);



#endif /* _ASM_X86_FPU_API_H */

arch/x86/include/asm/fpu/internal.h

+2 −13
Original line number Diff line number Diff line
@@ -74,14 +74,8 @@ static __always_inline __pure bool use_fxsr(void) 
	return static_cpu_has(X86_FEATURE_FXSR);

}



/*

 * fpstate handling functions:

 */



extern union fpregs_state init_fpstate;



extern void fpstate_init_user(union fpregs_state *state);

extern void fpu_init_fpstate_user(struct fpu *fpu);



#ifdef CONFIG_MATH_EMULATION

extern void fpstate_init_soft(struct swregs_state *soft);
@@ -381,12 +375,7 @@ static inline int os_xrstor_safe(struct xregs_state *xstate, u64 mask) 
	return err;

}



extern void __restore_fpregs_from_fpstate(union fpregs_state *fpstate, u64 mask);



static inline void restore_fpregs_from_fpstate(union fpregs_state *fpstate)

{

	__restore_fpregs_from_fpstate(fpstate, xfeatures_mask_fpstate());

}

extern void restore_fpregs_from_fpstate(union fpregs_state *fpstate, u64 mask);



extern bool copy_fpstate_to_sigframe(void __user *buf, void __user *fp, int size);
@@ -467,7 +456,7 @@ static inline void fpregs_restore_userregs(void) 
		 */

		mask = xfeatures_mask_restore_user() |

			xfeatures_mask_supervisor();

		__restore_fpregs_from_fpstate(&fpu->state, mask);

		restore_fpregs_from_fpstate(&fpu->state, mask);



		fpregs_activate(fpu);

		fpu->last_cpu = cpu;

arch/x86/kernel/fpu/core.c

+26 −4
Original line number Diff line number Diff line
@@ -124,9 +124,8 @@ void save_fpregs_to_fpstate(struct fpu *fpu) 
	asm volatile("fnsave %[fp]; fwait" : [fp] "=m" (fpu->state.fsave));

	frstor(&fpu->state.fsave);

}

EXPORT_SYMBOL(save_fpregs_to_fpstate);



void __restore_fpregs_from_fpstate(union fpregs_state *fpstate, u64 mask)

void restore_fpregs_from_fpstate(union fpregs_state *fpstate, u64 mask)

{

	/*

	 * AMD K7/K8 and later CPUs up to Zen don't save/restore
@@ -151,7 +150,31 @@ void __restore_fpregs_from_fpstate(union fpregs_state *fpstate, u64 mask) 
			frstor(&fpstate->fsave);

	}

}

EXPORT_SYMBOL_GPL(__restore_fpregs_from_fpstate);



#if IS_ENABLED(CONFIG_KVM)

void fpu_swap_kvm_fpu(struct fpu *save, struct fpu *rstor, u64 restore_mask)

{

	fpregs_lock();



	if (save) {

		if (test_thread_flag(TIF_NEED_FPU_LOAD)) {

			memcpy(&save->state, &current->thread.fpu.state,

			       fpu_kernel_xstate_size);

		} else {

			save_fpregs_to_fpstate(save);

		}

	}



	if (rstor) {

		restore_mask &= xfeatures_mask_fpstate();

		restore_fpregs_from_fpstate(&rstor->state, restore_mask);

	}



	fpregs_mark_activate();

	fpregs_unlock();

}

EXPORT_SYMBOL_GPL(fpu_swap_kvm_fpu);

#endif



void kernel_fpu_begin_mask(unsigned int kfpu_mask)

{
@@ -457,7 +480,6 @@ void fpregs_mark_activate(void) 
	fpu->last_cpu = smp_processor_id();

	clear_thread_flag(TIF_NEED_FPU_LOAD);

}

EXPORT_SYMBOL_GPL(fpregs_mark_activate);



/*

 * x87 math exception handling:

arch/x86/kernel/fpu/init.c

+0 −1
Original line number Diff line number Diff line
@@ -136,7 +136,6 @@ static void __init fpu__init_system_generic(void) 
 * components into a single, continuous memory block:

 */

unsigned int fpu_kernel_xstate_size __ro_after_init;

EXPORT_SYMBOL_GPL(fpu_kernel_xstate_size);



/* Get alignment of the TYPE. */

#define TYPE_ALIGN(TYPE) offsetof(struct { char x; TYPE test; }, test)

arch/x86/kernel/fpu/xstate.c

+0 −1
Original line number Diff line number Diff line
@@ -65,7 +65,6 @@ static short xsave_cpuid_features[] __initdata = { 
 * XSAVE buffer, both supervisor and user xstates.

 */

u64 xfeatures_mask_all __ro_after_init;

EXPORT_SYMBOL_GPL(xfeatures_mask_all);



static unsigned int xstate_offsets[XFEATURE_MAX] __ro_after_init =

	{ [ 0 ... XFEATURE_MAX - 1] = -1};