Commit 2cc029a0 authored by Vincenzo Frascino's avatar Vincenzo Frascino Committed by Andrew Morton
Browse files

arm64: mte: rename TCO routines 

The TCO related routines are used in uaccess methods and
load_unaligned_zeropad() but are unrelated to both even if the naming
suggest otherwise.

Improve the readability of the code moving the away from uaccess.h and
pre-pending them with "mte".

[andreyknvl@google.com: drop __ from mte_disable/enable_tco names]
  Link: https://lkml.kernel.org/r/74d26337b2360733956114069e96ff11c296a944.1680114854.git.andreyknvl@google.com
Link: https://lkml.kernel.org/r/a48e7adce1248c0f9603a457776d59daa0ef734b.1678491668.git.andreyknvl@google.com


Signed-off-by: default avatarVincenzo Frascino <vincenzo.frascino@arm.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Signed-off-by: default avatarAndrey Konovalov <andreyknvl@google.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Evgenii Stepanov <eugenis@google.com>
Cc: Marco Elver <elver@google.com>
Cc: Peter Collingbourne <pcc@google.com>
Cc: Weizhao Ouyang <ouyangweizhao@zeku.com>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 0eafff1c

arch/arm64/include/asm/mte-kasan.h

+81 −0
Original line number Diff line number Diff line
@@ -13,8 +13,73 @@ 


#include <linux/types.h>



#ifdef CONFIG_KASAN_HW_TAGS



/* Whether the MTE asynchronous mode is enabled. */

DECLARE_STATIC_KEY_FALSE(mte_async_or_asymm_mode);



static inline bool system_uses_mte_async_or_asymm_mode(void)

{

	return static_branch_unlikely(&mte_async_or_asymm_mode);

}



#else /* CONFIG_KASAN_HW_TAGS */



static inline bool system_uses_mte_async_or_asymm_mode(void)

{

	return false;

}



#endif /* CONFIG_KASAN_HW_TAGS */



#ifdef CONFIG_ARM64_MTE



/*

 * The Tag Check Flag (TCF) mode for MTE is per EL, hence TCF0

 * affects EL0 and TCF affects EL1 irrespective of which TTBR is

 * used.

 * The kernel accesses TTBR0 usually with LDTR/STTR instructions

 * when UAO is available, so these would act as EL0 accesses using

 * TCF0.

 * However futex.h code uses exclusives which would be executed as

 * EL1, this can potentially cause a tag check fault even if the

 * user disables TCF0.

 *

 * To address the problem we set the PSTATE.TCO bit in uaccess_enable()

 * and reset it in uaccess_disable().

 *

 * The Tag check override (TCO) bit disables temporarily the tag checking

 * preventing the issue.

 */

static inline void mte_disable_tco(void)

{

	asm volatile(ALTERNATIVE("nop", SET_PSTATE_TCO(0),

				 ARM64_MTE, CONFIG_KASAN_HW_TAGS));

}



static inline void mte_enable_tco(void)

{

	asm volatile(ALTERNATIVE("nop", SET_PSTATE_TCO(1),

				 ARM64_MTE, CONFIG_KASAN_HW_TAGS));

}



/*

 * These functions disable tag checking only if in MTE async mode

 * since the sync mode generates exceptions synchronously and the

 * nofault or load_unaligned_zeropad can handle them.

 */

static inline void __mte_disable_tco_async(void)

{

	if (system_uses_mte_async_or_asymm_mode())

		mte_disable_tco();

}



static inline void __mte_enable_tco_async(void)

{

	if (system_uses_mte_async_or_asymm_mode())

		mte_enable_tco();

}



/*

 * These functions are meant to be only used from KASAN runtime through

 * the arch_*() interface defined in asm/memory.h.
@@ -138,6 +203,22 @@ void mte_enable_kernel_asymm(void); 


#else /* CONFIG_ARM64_MTE */



static inline void mte_disable_tco(void)

{

}



static inline void mte_enable_tco(void)

{

}



static inline void __mte_disable_tco_async(void)

{

}



static inline void __mte_enable_tco_async(void)

{

}



static inline u8 mte_get_ptr_tag(void *ptr)

{

	return 0xFF;

arch/arm64/include/asm/mte.h

+0 −12
Original line number Diff line number Diff line
@@ -178,14 +178,6 @@ static inline void mte_disable_tco_entry(struct task_struct *task) 
}



#ifdef CONFIG_KASAN_HW_TAGS

/* Whether the MTE asynchronous mode is enabled. */

DECLARE_STATIC_KEY_FALSE(mte_async_or_asymm_mode);



static inline bool system_uses_mte_async_or_asymm_mode(void)

{

	return static_branch_unlikely(&mte_async_or_asymm_mode);

}



void mte_check_tfsr_el1(void);



static inline void mte_check_tfsr_entry(void)
@@ -212,10 +204,6 @@ static inline void mte_check_tfsr_exit(void) 
	mte_check_tfsr_el1();

}

#else

static inline bool system_uses_mte_async_or_asymm_mode(void)

{

	return false;

}

static inline void mte_check_tfsr_el1(void)

{

}

arch/arm64/include/asm/uaccess.h

+10 −56
Original line number Diff line number Diff line
@@ -136,55 +136,9 @@ static inline void __uaccess_enable_hw_pan(void) 
			CONFIG_ARM64_PAN));

}



/*

 * The Tag Check Flag (TCF) mode for MTE is per EL, hence TCF0

 * affects EL0 and TCF affects EL1 irrespective of which TTBR is

 * used.

 * The kernel accesses TTBR0 usually with LDTR/STTR instructions

 * when UAO is available, so these would act as EL0 accesses using

 * TCF0.

 * However futex.h code uses exclusives which would be executed as

 * EL1, this can potentially cause a tag check fault even if the

 * user disables TCF0.

 *

 * To address the problem we set the PSTATE.TCO bit in uaccess_enable()

 * and reset it in uaccess_disable().

 *

 * The Tag check override (TCO) bit disables temporarily the tag checking

 * preventing the issue.

 */

static inline void __uaccess_disable_tco(void)

{

	asm volatile(ALTERNATIVE("nop", SET_PSTATE_TCO(0),

				 ARM64_MTE, CONFIG_KASAN_HW_TAGS));

}



static inline void __uaccess_enable_tco(void)

{

	asm volatile(ALTERNATIVE("nop", SET_PSTATE_TCO(1),

				 ARM64_MTE, CONFIG_KASAN_HW_TAGS));

}



/*

 * These functions disable tag checking only if in MTE async mode

 * since the sync mode generates exceptions synchronously and the

 * nofault or load_unaligned_zeropad can handle them.

 */

static inline void __uaccess_disable_tco_async(void)

{

	if (system_uses_mte_async_or_asymm_mode())

		 __uaccess_disable_tco();

}



static inline void __uaccess_enable_tco_async(void)

{

	if (system_uses_mte_async_or_asymm_mode())

		__uaccess_enable_tco();

}



static inline void uaccess_disable_privileged(void)

{

	__uaccess_disable_tco();

	mte_disable_tco();



	if (uaccess_ttbr0_disable())

		return;
@@ -194,7 +148,7 @@ static inline void uaccess_disable_privileged(void) 


static inline void uaccess_enable_privileged(void)

{

	__uaccess_enable_tco();

	mte_enable_tco();



	if (uaccess_ttbr0_enable())

		return;
@@ -302,8 +256,8 @@ do { \ 
#define get_user	__get_user



/*

 * We must not call into the scheduler between __uaccess_enable_tco_async() and

 * __uaccess_disable_tco_async(). As `dst` and `src` may contain blocking

 * We must not call into the scheduler between __mte_enable_tco_async() and

 * __mte_disable_tco_async(). As `dst` and `src` may contain blocking

 * functions, we must evaluate these outside of the critical section.

 */

#define __get_kernel_nofault(dst, src, type, err_label)			\
@@ -312,10 +266,10 @@ do { \ 
	__typeof__(src) __gkn_src = (src);				\

	int __gkn_err = 0;						\

									\

	__uaccess_enable_tco_async();					\

	__mte_enable_tco_async();					\

	__raw_get_mem("ldr", *((type *)(__gkn_dst)),			\

		      (__force type *)(__gkn_src), __gkn_err, K);	\

	__uaccess_disable_tco_async();					\

	__mte_disable_tco_async();					\

									\

	if (unlikely(__gkn_err))					\

		goto err_label;						\
@@ -388,8 +342,8 @@ do { \ 
#define put_user	__put_user



/*

 * We must not call into the scheduler between __uaccess_enable_tco_async() and

 * __uaccess_disable_tco_async(). As `dst` and `src` may contain blocking

 * We must not call into the scheduler between __mte_enable_tco_async() and

 * __mte_disable_tco_async(). As `dst` and `src` may contain blocking

 * functions, we must evaluate these outside of the critical section.

 */

#define __put_kernel_nofault(dst, src, type, err_label)			\
@@ -398,10 +352,10 @@ do { \ 
	__typeof__(src) __pkn_src = (src);				\

	int __pkn_err = 0;						\

									\

	__uaccess_enable_tco_async();					\

	__mte_enable_tco_async();					\

	__raw_put_mem("str", *((type *)(__pkn_src)),			\

		      (__force type *)(__pkn_dst), __pkn_err, K);	\

	__uaccess_disable_tco_async();					\

	__mte_disable_tco_async();					\

									\

	if (unlikely(__pkn_err))					\

		goto err_label;						\

arch/arm64/include/asm/word-at-a-time.h

+2 −2
Original line number Diff line number Diff line
@@ -55,7 +55,7 @@ static inline unsigned long load_unaligned_zeropad(const void *addr) 
{

	unsigned long ret;



	__uaccess_enable_tco_async();

	__mte_enable_tco_async();



	/* Load word from unaligned pointer addr */

	asm(
@@ -65,7 +65,7 @@ static inline unsigned long load_unaligned_zeropad(const void *addr) 
	: "=&r" (ret)

	: "r" (addr), "Q" (*(unsigned long *)addr));



	__uaccess_disable_tco_async();

	__mte_disable_tco_async();



	return ret;

}