Commit a1c510d0 authored by Ard Biesheuvel's avatar Ard Biesheuvel
Browse files

ARM: implement support for vmap'ed stacks 



Wire up the generic support for managing task stack allocations via vmalloc,
and implement the entry code that detects whether we faulted because of a
stack overrun (or future stack overrun caused by pushing the pt_regs array)

While this adds a fair amount of tricky entry asm code, it should be
noted that it only adds a TST + branch to the svc_entry path. The code
implementing the non-trivial handling of the overflow stack is emitted
out-of-line into the .text section.

Since on ARM, we rely on do_translation_fault() to keep PMD level page
table entries that cover the vmalloc region up to date, we need to
ensure that we don't hit such a stale PMD entry when accessing the
stack. So we do a dummy read from the new stack while still running from
the old one on the context switch path, and bump the vmalloc_seq counter
when PMD level entries in the vmalloc range are modified, so that the MM
switch fetches the latest version of the entries.

Note that we need to increase the per-mode stack by 1 word, to gain some
space to stash a GPR until we know it is safe to touch the stack.
However, due to the cacheline alignment of the struct, this does not
actually increase the memory footprint of the struct stack array at all.

Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Tested-by: default avatarKeith Packard <keithpac@amazon.com>
Tested-by: default avatarMarc Zyngier <maz@kernel.org>
Tested-by: Vladimir Murzin <vladimir.murzin@arm.com> # ARMv7M
parent ae5cc07d

arch/arm/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -127,6 +127,7 @@ config ARM 
	select RTC_LIB

	select SYS_SUPPORTS_APM_EMULATION

	select THREAD_INFO_IN_TASK if CURRENT_POINTER_IN_TPIDRURO

	select HAVE_ARCH_VMAP_STACK if MMU && THREAD_INFO_IN_TASK && (!LD_IS_LLD || LLD_VERSION >= 140000)

	select TRACE_IRQFLAGS_SUPPORT if !CPU_V7M

	# Above selects are sorted alphabetically; please add new ones

	# according to that.  Thanks.

arch/arm/include/asm/page.h

+4 −0
Original line number Diff line number Diff line
@@ -149,6 +149,10 @@ extern void copy_page(void *to, const void *from); 
#include <asm/pgtable-2level-types.h>

#endif



#ifdef CONFIG_VMAP_STACK

#define ARCH_PAGE_TABLE_SYNC_MASK	PGTBL_PMD_MODIFIED

#endif



#endif /* CONFIG_MMU */



typedef struct page *pgtable_t;

arch/arm/include/asm/thread_info.h

+8 −0
Original line number Diff line number Diff line
@@ -25,6 +25,14 @@ 
#define THREAD_SIZE		(PAGE_SIZE << THREAD_SIZE_ORDER)

#define THREAD_START_SP		(THREAD_SIZE - 8)



#ifdef CONFIG_VMAP_STACK

#define THREAD_ALIGN		(2 * THREAD_SIZE)

#else

#define THREAD_ALIGN		THREAD_SIZE

#endif



#define OVERFLOW_STACK_SIZE	SZ_4K



#ifndef __ASSEMBLY__



struct task_struct;

arch/arm/kernel/entry-armv.S

+92 −5
Original line number Diff line number Diff line
@@ -57,6 +57,10 @@ UNWIND( .setfp fpreg, sp ) 
	@

	subs	r2, sp, r0		@ SP above bottom of IRQ stack?

	rsbscs	r2, r2, #THREAD_SIZE	@ ... and below the top?

#ifdef CONFIG_VMAP_STACK

	ldr_l	r2, high_memory, cc	@ End of the linear region

	cmpcc	r2, r0			@ Stack pointer was below it?

#endif

	movcs	sp, r0			@ If so, revert to incoming SP



#ifndef CONFIG_UNWINDER_ARM
@@ -188,13 +192,18 @@ ENDPROC(__und_invalid) 
#define SPFIX(code...)

#endif



	.macro	svc_entry, stack_hole=0, trace=1, uaccess=1

	.macro	svc_entry, stack_hole=0, trace=1, uaccess=1, overflow_check=1

 UNWIND(.fnstart		)

 UNWIND(.save {r0 - pc}		)

	sub	sp, sp, #(SVC_REGS_SIZE + \stack_hole)

 THUMB(	add	sp, r1		)	@ get SP in a GPR without

 THUMB(	sub	r1, sp, r1	)	@ using a temp register



	.if	\overflow_check

 UNWIND(.save	{r0 - pc}	)

	do_overflow_check (SVC_REGS_SIZE + \stack_hole)

	.endif



#ifdef CONFIG_THUMB2_KERNEL

	add	sp, r1			@ get SP in a GPR without

	sub	r1, sp, r1		@ using a temp register

	tst	r1, #4			@ test stack pointer alignment

	sub	r1, sp, r1		@ restore original R1

	sub	sp, r1			@ restore original SP
@@ -827,12 +836,20 @@ ENTRY(__switch_to) 
	str	r7, [r8]

#endif

	mov	r0, r5

#if !defined(CONFIG_THUMB2_KERNEL)

#if !defined(CONFIG_THUMB2_KERNEL) && !defined(CONFIG_VMAP_STACK)

	set_current r7

	ldmia	r4, {r4 - sl, fp, sp, pc}	@ Load all regs saved previously

#else

	mov	r1, r7

	ldmia	r4, {r4 - sl, fp, ip, lr}	@ Load all regs saved previously

#ifdef CONFIG_VMAP_STACK

	@

	@ Do a dummy read from the new stack while running from the old one so

	@ that we can rely on do_translation_fault() to fix up any stale PMD

	@ entries covering the vmalloc region.

	@

	ldr	r2, [ip]

#endif



	@ When CONFIG_THREAD_INFO_IN_TASK=n, the update of SP itself is what

	@ effectuates the task switch, as that is what causes the observable
@@ -849,6 +866,76 @@ ENTRY(__switch_to) 
 UNWIND(.fnend		)

ENDPROC(__switch_to)



#ifdef CONFIG_VMAP_STACK

	.text

	.align	2

__bad_stack:

	@

	@ We've just detected an overflow. We need to load the address of this

	@ CPU's overflow stack into the stack pointer register. We have only one

	@ scratch register so let's use a sequence of ADDs including one

	@ involving the PC, and decorate them with PC-relative group

	@ relocations. As these are ARM only, switch to ARM mode first.

	@

	@ We enter here with IP clobbered and its value stashed on the mode

	@ stack.

	@

THUMB(	bx	pc		)

THUMB(	nop			)

THUMB(	.arm			)

	mrc	p15, 0, ip, c13, c0, 4		@ Get per-CPU offset



	.globl	overflow_stack_ptr

	.reloc	0f, R_ARM_ALU_PC_G0_NC, overflow_stack_ptr

	.reloc	1f, R_ARM_ALU_PC_G1_NC, overflow_stack_ptr

	.reloc	2f, R_ARM_LDR_PC_G2, overflow_stack_ptr

	add	ip, ip, pc

0:	add	ip, ip, #-4

1:	add	ip, ip, #0

2:	ldr	ip, [ip, #4]



	str	sp, [ip, #-4]!			@ Preserve original SP value

	mov	sp, ip				@ Switch to overflow stack

	pop	{ip}				@ Original SP in IP



#if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC)

	mov	ip, ip				@ mov expected by unwinder

	push	{fp, ip, lr, pc}		@ GCC flavor frame record

#else

	str	ip, [sp, #-8]!			@ store original SP

	push	{fpreg, lr}			@ Clang flavor frame record

#endif

UNWIND( ldr	ip, [r0, #4]	)		@ load exception LR

UNWIND( str	ip, [sp, #12]	)		@ store in the frame record

	ldr	ip, [r0, #12]			@ reload IP



	@ Store the original GPRs to the new stack.

	svc_entry uaccess=0, overflow_check=0



UNWIND( .save   {sp, pc}	)

UNWIND( .save   {fpreg, lr}	)

UNWIND( .setfp  fpreg, sp	)



	ldr	fpreg, [sp, #S_SP]		@ Add our frame record

						@ to the linked list

#if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC)

	ldr	r1, [fp, #4]			@ reload SP at entry

	add	fp, fp, #12

#else

	ldr	r1, [fpreg, #8]

#endif

	str	r1, [sp, #S_SP]			@ store in pt_regs



	@ Stash the regs for handle_bad_stack

	mov	r0, sp



	@ Time to die

	bl	handle_bad_stack

	nop

UNWIND( .fnend			)

ENDPROC(__bad_stack)

#endif



	__INIT



/*

arch/arm/kernel/entry-header.S

+37 −0
Original line number Diff line number Diff line
@@ -423,3 +423,40 @@ scno .req r7 @ syscall number 
tbl	.req	r8		@ syscall table pointer

why	.req	r8		@ Linux syscall (!= 0)

tsk	.req	r9		@ current thread_info



	.macro	do_overflow_check, frame_size:req

#ifdef CONFIG_VMAP_STACK

	@

	@ Test whether the SP has overflowed. Task and IRQ stacks are aligned

	@ so that SP & BIT(THREAD_SIZE_ORDER + PAGE_SHIFT) should always be

	@ zero.

	@

ARM(	tst	sp, #1 << (THREAD_SIZE_ORDER + PAGE_SHIFT)	)

THUMB(	tst	r1, #1 << (THREAD_SIZE_ORDER + PAGE_SHIFT)	)

THUMB(	it	ne						)

	bne	.Lstack_overflow_check\@



	.pushsection	.text

.Lstack_overflow_check\@:

	@

	@ The stack pointer is not pointing to a valid vmap'ed stack, but it

	@ may be pointing into the linear map instead, which may happen if we

	@ are already running from the overflow stack. We cannot detect overflow

	@ in such cases so just carry on.

	@

	str	ip, [r0, #12]			@ Stash IP on the mode stack

	ldr_l	ip, high_memory			@ Start of VMALLOC space

ARM(	cmp	sp, ip			)	@ SP in vmalloc space?

THUMB(	cmp	r1, ip			)

THUMB(	itt	lo			)

	ldrlo	ip, [r0, #12]			@ Restore IP

	blo	.Lout\@				@ Carry on



THUMB(	sub	r1, sp, r1		)	@ Restore original R1

THUMB(	sub	sp, r1			)	@ Restore original SP

	add	sp, sp, #\frame_size		@ Undo svc_entry's SP change

	b	__bad_stack			@ Handle VMAP stack overflow

	.popsection

.Lout\@:

#endif

	.endm