Commit 64701838 authored by Thomas Gleixner's avatar Thomas Gleixner Committed by Peter Zijlstra
Browse files

x86/percpu: Move preempt_count next to current_task 



Add preempt_count to pcpu_hot, since it is once of the most used
per-cpu variables.

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20220915111145.284170644@infradead.org
parent e57ef2ed

arch/x86/include/asm/current.h

+1 −0
Original line number Diff line number Diff line
@@ -15,6 +15,7 @@ struct pcpu_hot { 
	union {

		struct {

			struct task_struct	*current_task;

			int			preempt_count;

		};

		u8	pad[64];

	};

arch/x86/include/asm/preempt.h

+14 −13
Original line number Diff line number Diff line
@@ -4,11 +4,11 @@ 


#include <asm/rmwcc.h>

#include <asm/percpu.h>

#include <asm/current.h>



#include <linux/thread_info.h>

#include <linux/static_call_types.h>



DECLARE_PER_CPU(int, __preempt_count);



/* We use the MSB mostly because its available */

#define PREEMPT_NEED_RESCHED	0x80000000
@@ -24,7 +24,7 @@ DECLARE_PER_CPU(int, __preempt_count); 
 */

static __always_inline int preempt_count(void)

{

	return raw_cpu_read_4(__preempt_count) & ~PREEMPT_NEED_RESCHED;

	return raw_cpu_read_4(pcpu_hot.preempt_count) & ~PREEMPT_NEED_RESCHED;

}



static __always_inline void preempt_count_set(int pc)
@@ -32,10 +32,10 @@ static __always_inline void preempt_count_set(int pc) 
	int old, new;



	do {

		old = raw_cpu_read_4(__preempt_count);

		old = raw_cpu_read_4(pcpu_hot.preempt_count);

		new = (old & PREEMPT_NEED_RESCHED) |

			(pc & ~PREEMPT_NEED_RESCHED);

	} while (raw_cpu_cmpxchg_4(__preempt_count, old, new) != old);

	} while (raw_cpu_cmpxchg_4(pcpu_hot.preempt_count, old, new) != old);

}



/*
@@ -44,7 +44,7 @@ static __always_inline void preempt_count_set(int pc) 
#define init_task_preempt_count(p) do { } while (0)



#define init_idle_preempt_count(p, cpu) do { \

	per_cpu(__preempt_count, (cpu)) = PREEMPT_DISABLED; \

	per_cpu(pcpu_hot.preempt_count, (cpu)) = PREEMPT_DISABLED; \

} while (0)



/*
@@ -58,17 +58,17 @@ static __always_inline void preempt_count_set(int pc) 


static __always_inline void set_preempt_need_resched(void)

{

	raw_cpu_and_4(__preempt_count, ~PREEMPT_NEED_RESCHED);

	raw_cpu_and_4(pcpu_hot.preempt_count, ~PREEMPT_NEED_RESCHED);

}



static __always_inline void clear_preempt_need_resched(void)

{

	raw_cpu_or_4(__preempt_count, PREEMPT_NEED_RESCHED);

	raw_cpu_or_4(pcpu_hot.preempt_count, PREEMPT_NEED_RESCHED);

}



static __always_inline bool test_preempt_need_resched(void)

{

	return !(raw_cpu_read_4(__preempt_count) & PREEMPT_NEED_RESCHED);

	return !(raw_cpu_read_4(pcpu_hot.preempt_count) & PREEMPT_NEED_RESCHED);

}



/*
@@ -77,12 +77,12 @@ static __always_inline bool test_preempt_need_resched(void) 


static __always_inline void __preempt_count_add(int val)

{

	raw_cpu_add_4(__preempt_count, val);

	raw_cpu_add_4(pcpu_hot.preempt_count, val);

}



static __always_inline void __preempt_count_sub(int val)

{

	raw_cpu_add_4(__preempt_count, -val);

	raw_cpu_add_4(pcpu_hot.preempt_count, -val);

}



/*
@@ -92,7 +92,8 @@ static __always_inline void __preempt_count_sub(int val) 
 */

static __always_inline bool __preempt_count_dec_and_test(void)

{

	return GEN_UNARY_RMWcc("decl", __preempt_count, e, __percpu_arg([var]));

	return GEN_UNARY_RMWcc("decl", pcpu_hot.preempt_count, e,

			       __percpu_arg([var]));

}



/*
@@ -100,7 +101,7 @@ static __always_inline bool __preempt_count_dec_and_test(void) 
 */

static __always_inline bool should_resched(int preempt_offset)

{

	return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset);

	return unlikely(raw_cpu_read_4(pcpu_hot.preempt_count) == preempt_offset);

}



#ifdef CONFIG_PREEMPTION

arch/x86/kernel/cpu/common.c

+1 −7
Original line number Diff line number Diff line
@@ -2014,6 +2014,7 @@ __setup("clearcpuid=", setup_clearcpuid); 


DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = {

	.current_task	= &init_task,

	.preempt_count	= INIT_PREEMPT_COUNT,

};

EXPORT_PER_CPU_SYMBOL(pcpu_hot);
@@ -2022,13 +2023,9 @@ DEFINE_PER_CPU_FIRST(struct fixed_percpu_data, 
		     fixed_percpu_data) __aligned(PAGE_SIZE) __visible;

EXPORT_PER_CPU_SYMBOL_GPL(fixed_percpu_data);





DEFINE_PER_CPU(void *, hardirq_stack_ptr);

DEFINE_PER_CPU(bool, hardirq_stack_inuse);



DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;

EXPORT_PER_CPU_SYMBOL(__preempt_count);



DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) = TOP_OF_INIT_STACK;



static void wrmsrl_cstar(unsigned long val)
@@ -2081,9 +2078,6 @@ void syscall_init(void) 


#else	/* CONFIG_X86_64 */



DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;

EXPORT_PER_CPU_SYMBOL(__preempt_count);



/*

 * On x86_32, vm86 modifies tss.sp0, so sp0 isn't a reliable way to find

 * the top of the kernel stack.  Use an extra percpu variable to track the