Commit d7b6d709 authored by Thomas Gleixner's avatar Thomas Gleixner Committed by Peter Zijlstra
Browse files

x86/percpu: Move irq_stack variables next to current_task 



Further extend struct pcpu_hot with the hard and soft irq stack
pointers.

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20220915111145.599170752@infradead.org
parent c063a217

arch/x86/include/asm/current.h

+6 −0
Original line number Diff line number Diff line
@@ -18,6 +18,12 @@ struct pcpu_hot { 
			int			preempt_count;

			int			cpu_number;

			unsigned long		top_of_stack;

			void			*hardirq_stack_ptr;

#ifdef CONFIG_X86_64

			bool			hardirq_stack_inuse;

#else

			void			*softirq_stack_ptr;

#endif

		};

		u8	pad[64];

	};

arch/x86/include/asm/irq_stack.h

+6 −6
Original line number Diff line number Diff line
@@ -116,7 +116,7 @@ 
	ASM_CALL_ARG2



#define call_on_irqstack(func, asm_call, argconstr...)			\

	call_on_stack(__this_cpu_read(hardirq_stack_ptr),		\

	call_on_stack(__this_cpu_read(pcpu_hot.hardirq_stack_ptr),	\

		      func, asm_call, argconstr)



/* Macros to assert type correctness for run_*_on_irqstack macros */
@@ -135,7 +135,7 @@ 
	 * User mode entry and interrupt on the irq stack do not	\

	 * switch stacks. If from user mode the task stack is empty.	\

	 */								\

	if (user_mode(regs) || __this_cpu_read(hardirq_stack_inuse)) {	\

	if (user_mode(regs) || __this_cpu_read(pcpu_hot.hardirq_stack_inuse)) { \

		irq_enter_rcu();					\

		func(c_args);						\

		irq_exit_rcu();						\
@@ -146,9 +146,9 @@ 
		 * places. Invoke the stack switch macro with the call	\

		 * sequence which matches the above direct invocation.	\

		 */							\

		__this_cpu_write(hardirq_stack_inuse, true);		\

		__this_cpu_write(pcpu_hot.hardirq_stack_inuse, true);	\

		call_on_irqstack(func, asm_call, constr);		\

		__this_cpu_write(hardirq_stack_inuse, false);		\

		__this_cpu_write(pcpu_hot.hardirq_stack_inuse, false);	\

	}								\

}
@@ -212,9 +212,9 @@ 
 */

#define do_softirq_own_stack()						\

{									\

	__this_cpu_write(hardirq_stack_inuse, true);			\

	__this_cpu_write(pcpu_hot.hardirq_stack_inuse, true);		\

	call_on_irqstack(__do_softirq, ASM_CALL_ARG0);			\

	__this_cpu_write(hardirq_stack_inuse, false);			\

	__this_cpu_write(pcpu_hot.hardirq_stack_inuse, false);		\

}



#endif

arch/x86/include/asm/processor.h

+0 −4
Original line number Diff line number Diff line
@@ -448,8 +448,6 @@ static inline unsigned long cpu_kernelmode_gs_base(int cpu) 
	return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu);

}



DECLARE_PER_CPU(void *, hardirq_stack_ptr);

DECLARE_PER_CPU(bool, hardirq_stack_inuse);

extern asmlinkage void ignore_sysret(void);



/* Save actual FS/GS selectors and bases to current->thread */
@@ -458,8 +456,6 @@ void current_save_fsgs(void); 
#ifdef CONFIG_STACKPROTECTOR

DECLARE_PER_CPU(unsigned long, __stack_chk_guard);

#endif

DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);

DECLARE_PER_CPU(struct irq_stack *, softirq_stack_ptr);

#endif	/* !X86_64 */



struct perf_event;

arch/x86/kernel/cpu/common.c

+0 −3
Original line number Diff line number Diff line
@@ -2024,9 +2024,6 @@ DEFINE_PER_CPU_FIRST(struct fixed_percpu_data, 
		     fixed_percpu_data) __aligned(PAGE_SIZE) __visible;

EXPORT_PER_CPU_SYMBOL_GPL(fixed_percpu_data);



DEFINE_PER_CPU(void *, hardirq_stack_ptr);

DEFINE_PER_CPU(bool, hardirq_stack_inuse);



static void wrmsrl_cstar(unsigned long val)

{

	/*

arch/x86/kernel/dumpstack_32.c

+2 −2
Original line number Diff line number Diff line
@@ -37,7 +37,7 @@ const char *stack_type_name(enum stack_type type) 


static bool in_hardirq_stack(unsigned long *stack, struct stack_info *info)

{

	unsigned long *begin = (unsigned long *)this_cpu_read(hardirq_stack_ptr);

	unsigned long *begin = (unsigned long *)this_cpu_read(pcpu_hot.hardirq_stack_ptr);

	unsigned long *end   = begin + (THREAD_SIZE / sizeof(long));



	/*
@@ -62,7 +62,7 @@ static bool in_hardirq_stack(unsigned long *stack, struct stack_info *info) 


static bool in_softirq_stack(unsigned long *stack, struct stack_info *info)

{

	unsigned long *begin = (unsigned long *)this_cpu_read(softirq_stack_ptr);

	unsigned long *begin = (unsigned long *)this_cpu_read(pcpu_hot.softirq_stack_ptr);

	unsigned long *end   = begin + (THREAD_SIZE / sizeof(long));



	/*