Commit bad1e1c6 authored by Vincenzo Frascino's avatar Vincenzo Frascino Committed by Linus Torvalds
Browse files

arm64: mte: switch GCR_EL1 in kernel entry and exit 

When MTE is present, the GCR_EL1 register contains the tags mask that
allows to exclude tags from the random generation via the IRG instruction.

With the introduction of the new Tag-Based KASAN API that provides a
mechanism to reserve tags for special reasons, the MTE implementation has
to make sure that the GCR_EL1 setting for the kernel does not affect the
userspace processes and viceversa.

Save and restore the kernel/user mask in GCR_EL1 in kernel entry and exit.

Link: https://lkml.kernel.org/r/578b03294708cc7258fad0dc9c2a2e809e5a8214.1606161801.git.andreyknvl@google.com


Signed-off-by: default avatarVincenzo Frascino <vincenzo.frascino@arm.com>
Co-developed-by: default avatarAndrey Konovalov <andreyknvl@google.com>
Signed-off-by: default avatarAndrey Konovalov <andreyknvl@google.com>
Reviewed-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Tested-by: default avatarVincenzo Frascino <vincenzo.frascino@arm.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Ryabinin <aryabinin@virtuozzo.com>
Cc: Branislav Rankov <Branislav.Rankov@arm.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Evgenii Stepanov <eugenis@google.com>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Marco Elver <elver@google.com>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 620954a6

arch/arm64/include/asm/mte-def.h

+0 −1
Original line number Diff line number Diff line
@@ -10,6 +10,5 @@ 
#define MTE_TAG_SHIFT		56

#define MTE_TAG_SIZE		4

#define MTE_TAG_MASK		GENMASK((MTE_TAG_SHIFT + (MTE_TAG_SIZE - 1)), MTE_TAG_SHIFT)

#define MTE_TAG_MAX		(MTE_TAG_MASK >> MTE_TAG_SHIFT)



#endif /* __ASM_MTE_DEF_H  */

arch/arm64/include/asm/mte-kasan.h

+5 −0
Original line number Diff line number Diff line
@@ -30,6 +30,7 @@ u8 mte_get_random_tag(void); 
void *mte_set_mem_tag_range(void *addr, size_t size, u8 tag);



void mte_enable_kernel(void);

void mte_init_tags(u64 max_tag);



#else /* CONFIG_ARM64_MTE */
@@ -55,6 +56,10 @@ static inline void mte_enable_kernel(void) 
{

}



static inline void mte_init_tags(u64 max_tag)

{

}



#endif /* CONFIG_ARM64_MTE */



#endif /* __ASSEMBLY__ */

arch/arm64/include/asm/mte.h

+2 −0
Original line number Diff line number Diff line
@@ -18,6 +18,8 @@ 


#include <asm/pgtable-types.h>



extern u64 gcr_kernel_excl;



void mte_clear_page_tags(void *addr);

unsigned long mte_copy_tags_from_user(void *to, const void __user *from,

				      unsigned long n);

arch/arm64/kernel/asm-offsets.c

+3 −0
Original line number Diff line number Diff line
@@ -46,6 +46,9 @@ int main(void) 
#ifdef CONFIG_ARM64_PTR_AUTH

  DEFINE(THREAD_KEYS_USER,	offsetof(struct task_struct, thread.keys_user));

  DEFINE(THREAD_KEYS_KERNEL,	offsetof(struct task_struct, thread.keys_kernel));

#endif

#ifdef CONFIG_ARM64_MTE

  DEFINE(THREAD_GCR_EL1_USER,	offsetof(struct task_struct, thread.gcr_user_excl));

#endif

  BLANK();

  DEFINE(S_X0,			offsetof(struct pt_regs, regs[0]));

arch/arm64/kernel/entry.S

+41 −0
Original line number Diff line number Diff line
@@ -173,6 +173,43 @@ alternative_else_nop_endif 
#endif

	.endm



	.macro mte_set_gcr, tmp, tmp2

#ifdef CONFIG_ARM64_MTE

	/*

	 * Calculate and set the exclude mask preserving

	 * the RRND (bit[16]) setting.

	 */

	mrs_s	\tmp2, SYS_GCR_EL1

	bfi	\tmp2, \tmp, #0, #16

	msr_s	SYS_GCR_EL1, \tmp2

	isb

#endif

	.endm



	.macro mte_set_kernel_gcr, tmp, tmp2

#ifdef CONFIG_KASAN_HW_TAGS

alternative_if_not ARM64_MTE

	b	1f

alternative_else_nop_endif

	ldr_l	\tmp, gcr_kernel_excl



	mte_set_gcr \tmp, \tmp2

1:

#endif

	.endm



	.macro mte_set_user_gcr, tsk, tmp, tmp2

#ifdef CONFIG_ARM64_MTE

alternative_if_not ARM64_MTE

	b	1f

alternative_else_nop_endif

	ldr	\tmp, [\tsk, #THREAD_GCR_EL1_USER]



	mte_set_gcr \tmp, \tmp2

1:

#endif

	.endm



	.macro	kernel_entry, el, regsize = 64

	.if	\regsize == 32

	mov	w0, w0				// zero upper 32 bits of x0
@@ -212,6 +249,8 @@ alternative_else_nop_endif 


	ptrauth_keys_install_kernel tsk, x20, x22, x23



	mte_set_kernel_gcr x22, x23



	scs_load tsk, x20

	.else

	add	x21, sp, #S_FRAME_SIZE
@@ -315,6 +354,8 @@ alternative_else_nop_endif 
	/* No kernel C function calls after this as user keys are set. */

	ptrauth_keys_install_user tsk, x0, x1, x2



	mte_set_user_gcr tsk, x0, x1



	apply_ssbd 0, x0, x1

	.endif