Commit c063a217 authored by Thomas Gleixner's avatar Thomas Gleixner Committed by Peter Zijlstra
Browse files

x86/percpu: Move current_top_of_stack next to current_task 



Extend the struct pcpu_hot cacheline with current_top_of_stack;
another very frequently used value.

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20220915111145.493038635@infradead.org
parent 7443b296

arch/x86/entry/entry_32.S

+2 −2
Original line number Diff line number Diff line
@@ -1181,7 +1181,7 @@ SYM_CODE_START(asm_exc_nmi) 
	 * is using the thread stack right now, so it's safe for us to use it.

	 */

	movl	%esp, %ebx

	movl	PER_CPU_VAR(cpu_current_top_of_stack), %esp

	movl	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %esp

	call	exc_nmi

	movl	%ebx, %esp
@@ -1243,7 +1243,7 @@ SYM_CODE_START(rewind_stack_and_make_dead) 
	/* Prevent any naive code from trying to unwind to our caller. */

	xorl	%ebp, %ebp



	movl	PER_CPU_VAR(cpu_current_top_of_stack), %esi

	movl	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %esi

	leal	-TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%esi), %esp



	call	make_task_dead

arch/x86/entry/entry_64.S

+3 −3
Original line number Diff line number Diff line
@@ -92,7 +92,7 @@ SYM_CODE_START(entry_SYSCALL_64) 
	/* tss.sp2 is scratch space. */

	movq	%rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2)

	SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp

	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp

	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp



SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL)

	ANNOTATE_NOENDBR
@@ -1209,7 +1209,7 @@ SYM_CODE_START(asm_exc_nmi) 
	FENCE_SWAPGS_USER_ENTRY

	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx

	movq	%rsp, %rdx

	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp

	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp

	UNWIND_HINT_IRET_REGS base=%rdx offset=8

	pushq	5*8(%rdx)	/* pt_regs->ss */

	pushq	4*8(%rdx)	/* pt_regs->rsp */
@@ -1525,7 +1525,7 @@ SYM_CODE_START_NOALIGN(rewind_stack_and_make_dead) 
	/* Prevent any naive code from trying to unwind to our caller. */

	xorl	%ebp, %ebp



	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rax

	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rax

	leaq	-PTREGS_SIZE(%rax), %rsp

	UNWIND_HINT_REGS

arch/x86/entry/entry_64_compat.S

+3 −3
Original line number Diff line number Diff line
@@ -58,7 +58,7 @@ SYM_CODE_START(entry_SYSENTER_compat) 
	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax

	popq	%rax



	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp

	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp



	/* Construct struct pt_regs on stack */

	pushq	$__USER32_DS		/* pt_regs->ss */
@@ -191,7 +191,7 @@ SYM_CODE_START(entry_SYSCALL_compat) 
	SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp



	/* Switch to the kernel stack */

	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp

	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp



SYM_INNER_LABEL(entry_SYSCALL_compat_safe_stack, SYM_L_GLOBAL)

	ANNOTATE_NOENDBR
@@ -332,7 +332,7 @@ SYM_CODE_START(entry_INT80_compat) 
	ALTERNATIVE "", "jmp .Lint80_keep_stack", X86_FEATURE_XENPV



	movq	%rsp, %rax

	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp

	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp



	pushq	5*8(%rax)		/* regs->ss */

	pushq	4*8(%rax)		/* regs->rsp */

arch/x86/include/asm/current.h

+1 −0
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ struct pcpu_hot { 
			struct task_struct	*current_task;

			int			preempt_count;

			int			cpu_number;

			unsigned long		top_of_stack;

		};

		u8	pad[64];

	};

arch/x86/include/asm/processor.h

+1 −3
Original line number Diff line number Diff line
@@ -426,8 +426,6 @@ struct irq_stack { 
	char		stack[IRQ_STACK_SIZE];

} __aligned(IRQ_STACK_SIZE);



DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack);



#ifdef CONFIG_X86_64

struct fixed_percpu_data {

	/*
@@ -566,7 +564,7 @@ static __always_inline unsigned long current_top_of_stack(void) 
	 *  and around vm86 mode and sp0 on x86_64 is special because of the

	 *  entry trampoline.

	 */

	return this_cpu_read_stable(cpu_current_top_of_stack);

	return this_cpu_read_stable(pcpu_hot.top_of_stack);

}



static __always_inline bool on_thread_stack(void)