Commit 50596b75 authored by Ard Biesheuvel's avatar Ard Biesheuvel
Browse files

ARM: smp: Store current pointer in TPIDRURO register if available 



Now that the user space TLS register is assigned on every return to user
space, we can use it to keep the 'current' pointer while running in the
kernel. This removes the need to access it via thread_info, which is
located at the base of the stack, but will be moved out of there in a
subsequent patch.

Use the __builtin_thread_pointer() helper when available - this will
help GCC understand that reloading the value within the same function is
not necessary, even when using the per-task stack protector (which also
generates accesses via the TLS register). For example, the generated
code below loads TPIDRURO only once, and uses it to access both the
stack canary and the preempt_count fields.

<do_one_initcall>:
       e92d 41f0       stmdb   sp!, {r4, r5, r6, r7, r8, lr}
       ee1d 4f70       mrc     15, 0, r4, cr13, cr0, {3}
       4606            mov     r6, r0
       b094            sub     sp, #80 ; 0x50
       f8d4 34e8       ldr.w   r3, [r4, #1256] ; 0x4e8  <- stack canary
       9313            str     r3, [sp, #76]   ; 0x4c
       f8d4 8004       ldr.w   r8, [r4, #4]             <- preempt count

Co-developed-by: default avatarKeith Packard <keithpac@amazon.com>
Signed-off-by: default avatarKeith Packard <keithpac@amazon.com>
Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Reviewed-by: default avatarLinus Walleij <linus.walleij@linaro.org>
Tested-by: default avatarAmit Daniel Kachhap <amit.kachhap@arm.com>
parent 3855ab61

arch/arm/Kconfig

+5 −0
Original line number Diff line number Diff line
@@ -1157,6 +1157,11 @@ config SMP_ON_UP 


	  If you don't know what to do here, say Y.





config CURRENT_POINTER_IN_TPIDRURO

	def_bool y

	depends on SMP && CPU_32v6K && !CPU_V6



config ARM_CPU_TOPOLOGY

	bool "Support cpu topology definition"

	depends on SMP && CPU_V7

arch/arm/Makefile

+4 −0
Original line number Diff line number Diff line
@@ -113,6 +113,10 @@ ifeq ($(CONFIG_CC_IS_CLANG),y) 
CFLAGS_ABI	+= -meabi gnu

endif



ifeq ($(CONFIG_CURRENT_POINTER_IN_TPIDRURO),y)

CFLAGS_ABI	+= -mtp=cp15

endif



# Accept old syntax despite ".syntax unified"

AFLAGS_NOWARN	:=$(call as-option,-Wa$(comma)-mno-warn-deprecated,-Wa$(comma)-W)

arch/arm/include/asm/assembler.h

+24 −0
Original line number Diff line number Diff line
@@ -199,6 +199,30 @@ 
	.endm

	.endr



	.macro	get_current, rd

#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO

	mrc	p15, 0, \rd, c13, c0, 3		@ get TPIDRURO register

#else

	get_thread_info \rd

	ldr	\rd, [\rd, #TI_TASK]

#endif

	.endm



	.macro	set_current, rn

#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO

	mcr	p15, 0, \rn, c13, c0, 3		@ set TPIDRURO register

#endif

	.endm



	.macro	reload_current, t1:req, t2:req

#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO

	adr_l	\t1, __entry_task		@ get __entry_task base address

	mrc	p15, 0, \t2, c13, c0, 4		@ get per-CPU offset

	ldr	\t1, [\t1, \t2]			@ load variable

	mcr	p15, 0, \t1, c13, c0, 3		@ store in TPIDRURO

#endif

	.endm



/*

 * Get current thread_info.

 */

arch/arm/include/asm/current.h

0 → 100644
+50 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0-only */

/*

 * Copyright (c) 2021 Keith Packard <keithp@keithp.com>

 * Copyright (c) 2021 Google, LLC <ardb@kernel.org>

 */



#ifndef _ASM_ARM_CURRENT_H

#define _ASM_ARM_CURRENT_H



#ifndef __ASSEMBLY__



struct task_struct;



static inline void set_current(struct task_struct *cur)

{

	if (!IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO))

		return;



	/* Set TPIDRURO */

	asm("mcr p15, 0, %0, c13, c0, 3" :: "r"(cur) : "memory");

}



#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO



static inline struct task_struct *get_current(void)

{

	struct task_struct *cur;



#if __has_builtin(__builtin_thread_pointer)

	/*

	 * Use the __builtin helper when available - this results in better

	 * code, especially when using GCC in combination with the per-task

	 * stack protector, as the compiler will recognize that it needs to

	 * load the TLS register only once in every function.

	 */

	cur = __builtin_thread_pointer();

#else

	asm("mrc p15, 0, %0, c13, c0, 3" : "=r"(cur));

#endif

	return cur;

}



#define current get_current()

#else

#include <asm-generic/current.h>

#endif /* CONFIG_CURRENT_POINTER_IN_TPIDRURO */



#endif /* __ASSEMBLY__ */



#endif /* _ASM_ARM_CURRENT_H */

arch/arm/include/asm/switch_to.h

+2 −0
Original line number Diff line number Diff line
@@ -26,6 +26,8 @@ extern struct task_struct *__switch_to(struct task_struct *, struct thread_info 
#define switch_to(prev,next,last)					\

do {									\

	__complete_pending_tlbi();					\

	if (IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO))		\

		__this_cpu_write(__entry_task, next);			\

	last = __switch_to(prev,task_thread_info(prev), task_thread_info(next));	\

} while (0)