Commit 5cd0dad6 authored by James Morse's avatar James Morse Committed by Zeng Heng
Browse files

arm64: mpam: Context switch the MPAM registers 

maillist inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/I8T2RT

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/morse/linux.git/log/?h=mpam/snapshot/v6.7-rc2



---------------------------

MPAM has a system register that is used to hold the partid and pmg values
that traffic generated by EL0 will use. This can be set per-task by the
resctrl file system.

Add a helper to switch this. resctrl expects a 'default' value to be
used in preference if the default partid and pmg are selected.

struct task_struct's separate closid and rmid fields are insufficient
to implement resctrl using MPAM, as resctrl can change the partid (closid)
and pmg (sort of like the rmid) separately. On x86, the rmid is an
independent number, so a race that writes a mismatched  closid and rmid
into hardware is benign. On arm64, the pmg bits extend the partid.
(i.e. partid-5 has a pmg-0 that is not the same as partid-6's pmg-0).
In this case, mismatching the values will 'dirty' a pmg value that
resctrl believes is clean, and is not tracking with its 'limbo' code.

To avoid this, the partid and pmg are always read and written as a pair.
Instead of making struct task_struct's closid and rmid fields an
endian-unsafe union, add the value to struct thread_info and always use
READ_ONCE()/WRITE_ONCE() when accessing this field.

CC: Amit Singh Tomar <amitsinght@marvell.com>
Signed-off-by: default avatarJames Morse <james.morse@arm.com>
Signed-off-by: default avatarZeng Heng <zengheng4@huawei.com>
parent 30fae95d

arch/arm64/include/asm/mpam.h

+46 −0
Original line number Diff line number Diff line
@@ -7,6 +7,8 @@ 
#include <linux/bitops.h>

#include <linux/init.h>

#include <linux/jump_label.h>

#include <linux/percpu.h>

#include <linux/sched.h>



#include <asm/cpucaps.h>

#include <asm/cpufeature.h>
@@ -49,6 +51,9 @@ 




DECLARE_STATIC_KEY_FALSE(arm64_mpam_has_hcr);

DECLARE_STATIC_KEY_FALSE(mpam_enabled);

DECLARE_PER_CPU(u64, arm64_mpam_default);

DECLARE_PER_CPU(u64, arm64_mpam_current);



/* check whether all CPUs have MPAM support */

static __always_inline bool mpam_cpus_have_feature(void)
@@ -73,4 +78,45 @@ static inline void __init __enable_mpam_hcr(void) 
		static_branch_enable(&arm64_mpam_has_hcr);

}



/*

 * The resctrl filesystem writes to the partid/pmg values for threads and CPUs,

 * which may race with reads in __mpam_sched_in(). Ensure only one of the old

 * or new values are used. Particular care should be taken with the pmg field

 * as __mpam_sched_in() may read a partid and pmg that don't match, causing

 * this value to be stored with cache allocations, despite being considered

 * 'free' by resctrl.

 *

 * A value in struct thread_info is used instead of struct task_struct as the

 * cpu's u64 register format is used, but struct task_struct has two u32'.

 */

static inline u64 mpam_get_regval(struct task_struct *tsk)

{

#ifdef CONFIG_ARM64_MPAM

	return READ_ONCE(task_thread_info(tsk)->mpam_partid_pmg);

#else

	return 0;

#endif

}



static inline void mpam_thread_switch(struct task_struct *tsk)

{

	u64 oldregval;

	int cpu = smp_processor_id();

	u64 regval = mpam_get_regval(tsk);



	if (!IS_ENABLED(CONFIG_ARM64_MPAM) ||

	    !static_branch_likely(&mpam_enabled))

		return;



	if (!regval)

		regval = READ_ONCE(per_cpu(arm64_mpam_default, cpu));



	oldregval = READ_ONCE(per_cpu(arm64_mpam_current, cpu));

	if (oldregval == regval)

		return;



	/* Synchronising this write is left until the ERET to EL0 */

	write_sysreg_s(regval, SYS_MPAM0_EL1);

	WRITE_ONCE(per_cpu(arm64_mpam_current, cpu), regval);

}

#endif /* __ASM__MPAM_H */

arch/arm64/include/asm/thread_info.h

+3 −0
Original line number Diff line number Diff line
@@ -41,6 +41,9 @@ struct thread_info { 
#ifdef CONFIG_SHADOW_CALL_STACK

	void			*scs_base;

	void			*scs_sp;

#endif

#ifdef CONFIG_ARM64_MPAM

	u64			mpam_partid_pmg;

#endif

	u32			cpu;

};

arch/arm64/kernel/mpam.c

+5 −0
Original line number Diff line number Diff line
@@ -4,5 +4,10 @@ 
#include <asm/mpam.h>



#include <linux/jump_label.h>

#include <linux/percpu.h>



DEFINE_STATIC_KEY_FALSE(arm64_mpam_has_hcr);

DEFINE_STATIC_KEY_FALSE(mpam_enabled);

DEFINE_PER_CPU(u64, arm64_mpam_default);

DEFINE_PER_CPU(u64, arm64_mpam_current);

arch/arm64/kernel/process.c

+7 −0
Original line number Diff line number Diff line
@@ -48,6 +48,7 @@ 
#include <asm/exec.h>

#include <asm/fpsimd.h>

#include <asm/mmu_context.h>

#include <asm/mpam.h>

#include <asm/mte.h>

#include <asm/processor.h>

#include <asm/pointer_auth.h>
@@ -551,6 +552,12 @@ struct task_struct *__switch_to(struct task_struct *prev, 
	if (prev->thread.sctlr_user != next->thread.sctlr_user)

		update_sctlr_el1(next->thread.sctlr_user);



	/*

	 * MPAM thread switch happens after the DSB to ensure prev's accesses

	 * use prev's MPAM settings.

	 */

	mpam_thread_switch(next);



	/* the actual thread switch */

	last = cpu_switch_to(prev, next);