!789 x86 FPU/AMX bug fix since kernel v5.18

		xpkru = get_xsave_addr(&kstate->regs.xsave, XFEATURE_PKRU);

		*vpkru = xpkru->pkru;

	}

	/* Ensure that XCOMP_BV is set up for XSAVES */

	xstate_init_xcomp_bv(&kstate->regs.xsave, kstate->xfeatures);

	return 0;

}

EXPORT_SYMBOL_GPL(fpu_copy_uabi_to_guest_fpstate);

	fpstate_reset(&current->thread.fpu);

}

static void __init fpu__init_init_fpstate(void)

{

	/* Bring init_fpstate size and features up to date */

	init_fpstate.size		= fpu_kernel_cfg.max_size;

	init_fpstate.xfeatures		= fpu_kernel_cfg.max_features;

}

/*

 * Called on the boot CPU once per system bootup, to set up the initial

 * FPU state that is later cloned into all processes:

	fpu__init_system_xstate_size_legacy();

	fpu__init_system_xstate(fpu_kernel_cfg.max_size);

	fpu__init_task_struct_size();

	fpu__init_init_fpstate();

}

	{ [ 0 ... XFEATURE_MAX - 1] = -1};

static unsigned int xstate_sizes[XFEATURE_MAX] __ro_after_init =

	{ [ 0 ... XFEATURE_MAX - 1] = -1};

static unsigned int xstate_comp_offsets[XFEATURE_MAX] __ro_after_init =

	{ [ 0 ... XFEATURE_MAX - 1] = -1};

static unsigned int xstate_supervisor_only_offsets[XFEATURE_MAX] __ro_after_init =

	{ [ 0 ... XFEATURE_MAX - 1] = -1};

static unsigned int xstate_flags[XFEATURE_MAX] __ro_after_init;

#define XSTATE_FLAG_SUPERVISOR	BIT(0)

#define XSTATE_FLAG_ALIGNED64	BIT(1)

/*

 * Return whether the system supports a given xfeature.

}

EXPORT_SYMBOL_GPL(cpu_has_xfeatures);

static bool xfeature_is_aligned64(int xfeature_nr)

{

	return xstate_flags[xfeature_nr] & XSTATE_FLAG_ALIGNED64;

}

static bool xfeature_is_supervisor(int xfeature_nr)

{

	return xstate_flags[xfeature_nr] & XSTATE_FLAG_SUPERVISOR;

}

static unsigned int xfeature_get_offset(u64 xcomp_bv, int xfeature)

{

	unsigned int offs, i;

	/*

	 * Extended State Enumeration Sub-leaves (EAX = 0DH, ECX = n, n > 1)

	 * returns ECX[0] set to (1) for a supervisor state, and cleared (0)

	 * for a user state.

	 * Non-compacted format and legacy features use the cached fixed

	 * offsets.

	 */

	u32 eax, ebx, ecx, edx;

	if (!cpu_feature_enabled(X86_FEATURE_XSAVES) || xfeature <= XFEATURE_SSE)

		return xstate_offsets[xfeature];

	cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);

	return ecx & 1;

	/*

	 * Compacted format offsets depend on the actual content of the

	 * compacted xsave area which is determined by the xcomp_bv header

	 * field.

	 */

	offs = FXSAVE_SIZE + XSAVE_HDR_SIZE;

	for_each_extended_xfeature(i, xcomp_bv) {

		if (xfeature_is_aligned64(i))

			offs = ALIGN(offs, 64);

		if (i == xfeature)

			break;

		offs += xstate_sizes[i];

	}

	return offs;

}

/*

 * Record the offsets and sizes of various xstates contained

 * in the XSAVE state memory layout.

 */

static void __init setup_xstate_features(void)

static void __init setup_xstate_cache(void)

{

	u32 eax, ebx, ecx, edx, i;

	/* start at the beginnning of the "extended state" */

	/* start at the beginning of the "extended state" */

	unsigned int last_good_offset = offsetof(struct xregs_state,

						 extended_state_area);

	/*

		cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);

		xstate_sizes[i] = eax;

		xstate_flags[i] = ecx;

		/*

		 * If an xfeature is supervisor state, the offset in EBX is

	WARN_ON(nr >= XFEATURE_MAX);	\

} while (0)

/*

 * We could cache this like xstate_size[], but we only use

 * it here, so it would be a waste of space.

 */

static int xfeature_is_aligned(int xfeature_nr)

{

	u32 eax, ebx, ecx, edx;

	CHECK_XFEATURE(xfeature_nr);

	if (!xfeature_enabled(xfeature_nr)) {

		WARN_ONCE(1, "Checking alignment of disabled xfeature %d\n",

			  xfeature_nr);

		return 0;

	}

	cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);

	/*

	 * The value returned by ECX[1] indicates the alignment

	 * of state component 'i' when the compacted format

	 * of the extended region of an XSAVE area is used:

	 */

	return !!(ecx & 2);

}

/*

 * This function sets up offsets and sizes of all extended states in

 * xsave area. This supports both standard format and compacted format

 * of the xsave area.

 */

static void __init setup_xstate_comp_offsets(void)

{

	unsigned int next_offset;

	int i;

	/*

	 * The FP xstates and SSE xstates are legacy states. They are always

	 * in the fixed offsets in the xsave area in either compacted form

	 * or standard form.

	 */

	xstate_comp_offsets[XFEATURE_FP] = 0;

	xstate_comp_offsets[XFEATURE_SSE] = offsetof(struct fxregs_state,

						     xmm_space);

	if (!cpu_feature_enabled(X86_FEATURE_XSAVES)) {

		for_each_extended_xfeature(i, fpu_kernel_cfg.max_features)

			xstate_comp_offsets[i] = xstate_offsets[i];

		return;

	}

	next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE;

	for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) {

		if (xfeature_is_aligned(i))

			next_offset = ALIGN(next_offset, 64);

		xstate_comp_offsets[i] = next_offset;

		next_offset += xstate_sizes[i];

	}

}

/*

 * Setup offsets of a supervisor-state-only XSAVES buffer:

 *

 * The offsets stored in xstate_comp_offsets[] only work for one specific

 * value of the Requested Feature BitMap (RFBM).  In cases where a different

 * RFBM value is used, a different set of offsets is required.  This set of

 * offsets is for when RFBM=xfeatures_mask_supervisor().

 */

static void __init setup_supervisor_only_offsets(void)

{

	unsigned int next_offset;

	int i;

	next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE;

	for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) {

		if (!xfeature_is_supervisor(i))

			continue;

		if (xfeature_is_aligned(i))

			next_offset = ALIGN(next_offset, 64);

		xstate_supervisor_only_offsets[i] = next_offset;

		next_offset += xstate_sizes[i];

	}

}

/*

 * Print out xstate component offsets and sizes

 */

	for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) {

		pr_info("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n",

			 i, xstate_comp_offsets[i], i, xstate_sizes[i]);

			i, xfeature_get_offset(fpu_kernel_cfg.max_features, i),

			i, xstate_sizes[i]);

	}

}

	if (!boot_cpu_has(X86_FEATURE_XSAVE))

		return;

	setup_xstate_features();

	print_xstate_features();

	xstate_init_xcomp_bv(&init_fpstate.regs.xsave, fpu_kernel_cfg.max_features);

	xstate_init_xcomp_bv(&init_fpstate.regs.xsave, init_fpstate.xfeatures);

	/*

	 * Init all the features state with header.xfeatures being 0x0

	fxsave(&init_fpstate.regs.fxsave);

}

static int xfeature_uncompacted_offset(int xfeature_nr)

{

	u32 eax, ebx, ecx, edx;

	/*

	 * Only XSAVES supports supervisor states and it uses compacted

	 * format. Checking a supervisor state's uncompacted offset is

	 * an error.

	 */

	if (XFEATURE_MASK_SUPERVISOR_ALL & BIT_ULL(xfeature_nr)) {

		WARN_ONCE(1, "No fixed offset for xstate %d\n", xfeature_nr);

		return -1;

	}

	CHECK_XFEATURE(xfeature_nr);

	cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);

	return ebx;

}

int xfeature_size(int xfeature_nr)

{

	u32 eax, ebx, ecx, edx;

static unsigned int xstate_calculate_size(u64 xfeatures, bool compacted)

{

	unsigned int size = FXSAVE_SIZE + XSAVE_HDR_SIZE;

	int i;

	unsigned int topmost = fls64(xfeatures) -  1;

	unsigned int offset = xstate_offsets[topmost];

	for_each_extended_xfeature(i, xfeatures) {

		/* Align from the end of the previous feature */

		if (xfeature_is_aligned(i))

			size = ALIGN(size, 64);

		/*

		 * In compacted format the enabled features are packed,

		 * i.e. disabled features do not occupy space.

		 *

		 * In non-compacted format the offsets are fixed and

		 * disabled states still occupy space in the memory buffer.

		 */

		if (!compacted)

			size = xfeature_uncompacted_offset(i);

		/*

		 * Add the feature size even for non-compacted format

		 * to make the end result correct

		 */

		size += xfeature_size(i);

	}

	return size;

	if (topmost <= XFEATURE_SSE)

		return sizeof(struct xregs_state);

	if (compacted)

		offset = xfeature_get_offset(xfeatures, topmost);

	return offset + xstate_sizes[topmost];

}

/*

	return ebx;

}

/*

 * Will the runtime-enumerated 'xstate_size' fit in the init

 * task's statically-allocated buffer?

 */

static bool __init is_supported_xstate_size(unsigned int test_xstate_size)

{

	if (test_xstate_size <= sizeof(init_fpstate.regs))

		return true;

	pr_warn("x86/fpu: xstate buffer too small (%zu < %d), disabling xsave\n",

			sizeof(init_fpstate.regs), test_xstate_size);

	return false;

}

static int __init init_xstate_size(void)

{

	/* Recompute the context size for enabled features: */

	kernel_default_size =

		xstate_calculate_size(fpu_kernel_cfg.default_features, compacted);

	/* Ensure we have the space to store all default enabled features. */

	if (!is_supported_xstate_size(kernel_default_size))

		return -EINVAL;

	if (!paranoid_xstate_size_valid(kernel_size))

		return -EINVAL;

	/* Enable xstate instructions to be able to continue with initialization: */

	fpu__init_cpu_xstate();

	/* Cache size, offset and flags for initialization */

	setup_xstate_cache();

	err = init_xstate_size();

	if (err)

		goto out_disable;

	update_regset_xstate_info(fpu_user_cfg.max_size,

				  fpu_user_cfg.max_features);

	/*

	 * init_fpstate excludes dynamic states as they are large but init

	 * state is zero.

	 */

	init_fpstate.size		= fpu_kernel_cfg.default_size;

	init_fpstate.xfeatures		= fpu_kernel_cfg.default_features;

	if (init_fpstate.size > sizeof(init_fpstate.regs)) {

		pr_warn("x86/fpu: init_fpstate buffer too small (%zu < %d), disabling XSAVE\n",

			sizeof(init_fpstate.regs), init_fpstate.size);

		goto out_disable;

	}

	setup_init_fpu_buf();

	setup_xstate_comp_offsets();

	setup_supervisor_only_offsets();

	/*

	 * Paranoia check whether something in the setup modified the

 */

static void *__raw_xsave_addr(struct xregs_state *xsave, int xfeature_nr)

{

	if (!xfeature_enabled(xfeature_nr)) {

		WARN_ON_FPU(1);

	u64 xcomp_bv = xsave->header.xcomp_bv;

	if (WARN_ON_ONCE(!xfeature_enabled(xfeature_nr)))

		return NULL;

	if (cpu_feature_enabled(X86_FEATURE_XSAVES)) {

		if (WARN_ON_ONCE(!(xcomp_bv & BIT_ULL(xfeature_nr))))

			return NULL;

	}

	return (void *)xsave + xstate_comp_offsets[xfeature_nr];

	return (void *)xsave + xfeature_get_offset(xcomp_bv, xfeature_nr);

}

/*

 * Given the xsave area and a state inside, this function returns the

 * address of the state.

	 * We should not ever be requesting features that we

	 * have not enabled.

	 */

	WARN_ONCE(!(fpu_kernel_cfg.max_features & BIT_ULL(xfeature_nr)),

		  "get of unsupported state");

	if (WARN_ON_ONCE(!xfeature_enabled(xfeature_nr)))

		return NULL;

	/*

	 * This assumes the last 'xsave*' instruction to

	 * have requested that 'xfeature_nr' be saved.

	zerofrom = offsetof(struct xregs_state, extended_state_area);

	/*

	 * The ptrace buffer is in non-compacted XSAVE format.  In

	 * non-compacted format disabled features still occupy state space,

	 * but there is no state to copy from in the compacted

	 * init_fpstate. The gap tracking will zero these states.

	 * This 'mask' indicates which states to copy from fpstate.

	 * Those extended states that are not present in fpstate are

	 * either disabled or initialized:

	 *

	 * In non-compacted format, disabled features still occupy

	 * state space but there is no state to copy from in the

	 * compacted init_fpstate. The gap tracking will zero these

	 * states.

	 *

	 * The extended features have an all zeroes init state. Thus,

	 * remove them from 'mask' to zero those features in the user

	 * buffer instead of retrieving them from init_fpstate.

	 */

	mask = fpstate->user_xfeatures;

	mask = header.xfeatures;

	for_each_extended_xfeature(i, mask) {

		/*

			pkru.pkru = pkru_val;

			membuf_write(&to, &pkru, sizeof(pkru));

		} else {

			copy_feature(header.xfeatures & BIT_ULL(i), &to,

			membuf_write(&to,

				     __raw_xsave_addr(xsave, i),

				     __raw_xsave_addr(xinit, i),

				     xstate_sizes[i]);

		}

		/*

	/* Calculate the resulting kernel state size */

	mask = permitted | requested;

	/* Take supervisor states into account on the host */

	if (!guest)

		mask |= xfeatures_mask_supervisor();

	ksize = xstate_calculate_size(mask, compacted);

	/* Calculate the resulting user state size */

#include <stdarg.h>

#include <stdio.h>

/*

 * gcc cpuid.h provides __cpuid_count() since v4.4.

 * Clang/LLVM cpuid.h provides  __cpuid_count() since v3.4.0.

 *

 * Provide local define for tests needing __cpuid_count() because

 * selftests need to work in older environments that do not yet

 * have __cpuid_count().

 */

#ifndef __cpuid_count

#define __cpuid_count(level, count, a, b, c, d)				\

	__asm__ __volatile__ ("cpuid\n\t"				\

			      : "=a" (a), "=b" (b), "=c" (c), "=d" (d)	\

			      : "0" (level), "2" (count))

#endif

/* define kselftest exit codes */

#define KSFT_PASS  0

#define KSFT_FAIL  1

#include <sys/auxv.h>

#include <sys/mman.h>

#include <sys/shm.h>

#include <sys/ptrace.h>

#include <sys/syscall.h>

#include <sys/wait.h>

#include <sys/uio.h>

#include "../kselftest.h" /* For __cpuid_count() */

#ifndef __x86_64__

# error This test is 64-bit only

	return eax + ((uint64_t)edx << 32);

}

static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx)

{

	asm volatile("cpuid;"

		     : "=a" (*eax), "=b" (*ebx), "=c" (*ecx), "=d" (*edx)

		     : "0" (*eax), "2" (*ecx));

}

static inline void xsave(struct xsave_buffer *xbuf, uint64_t rfbm)

{

	uint32_t rfbm_lo = rfbm;

	 * support for the XSAVE feature set, including

	 * XGETBV.

	 */

	eax = 1;

	ecx = 0;

	cpuid(&eax, &ebx, &ecx, &edx);

	__cpuid_count(1, 0, eax, ebx, ecx, edx);

	if (!(ecx & CPUID_LEAF1_ECX_XSAVE_MASK))

		fatal_error("cpuid: no CPU xsave support");

	if (!(ecx & CPUID_LEAF1_ECX_OSXSAVE_MASK))

{

	uint32_t eax, ebx, ecx, edx;

	eax = CPUID_LEAF_XSTATE;

	ecx = CPUID_SUBLEAF_XSTATE_USER;

	cpuid(&eax, &ebx, &ecx, &edx);

	__cpuid_count(CPUID_LEAF_XSTATE, CPUID_SUBLEAF_XSTATE_USER,

		      eax, ebx, ecx, edx);

	/*

	 * EBX enumerates the size (in bytes) required by the XSAVE

	 */

	xbuf_size = ebx;

	eax = CPUID_LEAF_XSTATE;

	ecx = XFEATURE_XTILEDATA;

	cpuid(&eax, &ebx, &ecx, &edx);

	__cpuid_count(CPUID_LEAF_XSTATE, XFEATURE_XTILEDATA,

		      eax, ebx, ecx, edx);

	/*

	 * eax: XTILEDATA state component size

	 * ebx: XTILEDATA state component offset in user buffer

	_exit(0);

}

static inline int __compare_tiledata_state(struct xsave_buffer *xbuf1, struct xsave_buffer *xbuf2)

{

	return memcmp(&xbuf1->bytes[xtiledata.xbuf_offset],

		      &xbuf2->bytes[xtiledata.xbuf_offset],

		      xtiledata.size);

}

/*

 * Save current register state and compare it to @xbuf1.'

 *

		fatal_error("failed to allocate XSAVE buffer\n");

	xsave(xbuf2, XFEATURE_MASK_XTILEDATA);

	ret = memcmp(&xbuf1->bytes[xtiledata.xbuf_offset],

		     &xbuf2->bytes[xtiledata.xbuf_offset],

		     xtiledata.size);

	ret = __compare_tiledata_state(xbuf1, xbuf2);

	free(xbuf2);

	free(finfo);

}

/* Ptrace test */

/*

 * Make sure the ptracee has the expanded kernel buffer on the first

 * use. Then, initialize the state before performing the state

 * injection from the ptracer.

 */

static inline void ptracee_firstuse_tiledata(void)

{

	load_rand_tiledata(stashed_xsave);

	init_xtiledata();

}

/*

 * Ptracer injects the randomized tile data state. It also reads

 * before and after that, which will execute the kernel's state copy

 * functions. So, the tester is advised to double-check any emitted

 * kernel messages.

 */

static void ptracer_inject_tiledata(pid_t target)

{

	struct xsave_buffer *xbuf;

	struct iovec iov;

	xbuf = alloc_xbuf();

	if (!xbuf)

		fatal_error("unable to allocate XSAVE buffer");

	printf("\tRead the init'ed tiledata via ptrace().\n");

	iov.iov_base = xbuf;

	iov.iov_len = xbuf_size;

	memset(stashed_xsave, 0, xbuf_size);

	if (ptrace(PTRACE_GETREGSET, target, (uint32_t)NT_X86_XSTATE, &iov))

		fatal_error("PTRACE_GETREGSET");

	if (!__compare_tiledata_state(stashed_xsave, xbuf))

		printf("[OK]\tThe init'ed tiledata was read from ptracee.\n");

	else

		printf("[FAIL]\tThe init'ed tiledata was not read from ptracee.\n");

	printf("\tInject tiledata via ptrace().\n");

	load_rand_tiledata(xbuf);

	memcpy(&stashed_xsave->bytes[xtiledata.xbuf_offset],

	       &xbuf->bytes[xtiledata.xbuf_offset],

	       xtiledata.size);

	if (ptrace(PTRACE_SETREGSET, target, (uint32_t)NT_X86_XSTATE, &iov))

		fatal_error("PTRACE_SETREGSET");

	if (ptrace(PTRACE_GETREGSET, target, (uint32_t)NT_X86_XSTATE, &iov))

		fatal_error("PTRACE_GETREGSET");

	if (!__compare_tiledata_state(stashed_xsave, xbuf))

		printf("[OK]\tTiledata was correctly written to ptracee.\n");

	else

		printf("[FAIL]\tTiledata was not correctly written to ptracee.\n");

}

static void test_ptrace(void)

{

	pid_t child;

	int status;

	child = fork();

	if (child < 0) {

		err(1, "fork");

	} else if (!child) {

		if (ptrace(PTRACE_TRACEME, 0, NULL, NULL))

			err(1, "PTRACE_TRACEME");

		ptracee_firstuse_tiledata();

		raise(SIGTRAP);

		_exit(0);

	}

	do {

		wait(&status);

	} while (WSTOPSIG(status) != SIGTRAP);

	ptracer_inject_tiledata(child);

	ptrace(PTRACE_DETACH, child, NULL, NULL);

	wait(&status);

	if (!WIFEXITED(status) || WEXITSTATUS(status))

		err(1, "ptrace test");

}

int main(void)

{

	/* Check hardware availability at first */

	ctxtswtest_config.num_threads = 5;

	test_context_switch();

	test_ptrace();

	clearhandler(SIGILL);

	free_stashed_xsave();