Commit 9eb912d1 authored by Brian Gerst's avatar Brian Gerst Committed by Tejun Heo
Browse files

x86-64: Move TLB state from PDA to per-cpu and consolidate with 32-bit. 



Signed-off-by: default avatarBrian Gerst <brgerst@gmail.com>
Signed-off-by: default avatarTejun Heo <tj@kernel.org>
parent 1b437c8c

arch/x86/include/asm/mmu_context_64.h

+7 −9
Original line number Diff line number Diff line 
#ifndef _ASM_X86_MMU_CONTEXT_64_H

#define _ASM_X86_MMU_CONTEXT_64_H



#include <asm/pda.h>



static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)

{

#ifdef CONFIG_SMP

	if (read_pda(mmu_state) == TLBSTATE_OK)

		write_pda(mmu_state, TLBSTATE_LAZY);

	if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)

		percpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);

#endif

}
@@ -19,8 +17,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, 
		/* stop flush ipis for the previous mm */

		cpu_clear(cpu, prev->cpu_vm_mask);

#ifdef CONFIG_SMP

		write_pda(mmu_state, TLBSTATE_OK);

		write_pda(active_mm, next);

		percpu_write(cpu_tlbstate.state, TLBSTATE_OK);

		percpu_write(cpu_tlbstate.active_mm, next);

#endif

		cpu_set(cpu, next->cpu_vm_mask);

		load_cr3(next->pgd);
@@ -30,9 +28,9 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, 
	}

#ifdef CONFIG_SMP

	else {

		write_pda(mmu_state, TLBSTATE_OK);

		if (read_pda(active_mm) != next)

			BUG();

		percpu_write(cpu_tlbstate.state, TLBSTATE_OK);

		BUG_ON(percpu_read(cpu_tlbstate.active_mm) != next);



		if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) {

			/* We were in lazy tlb mode and leave_mm disabled

			 * tlb flush IPI delivery. We must reload CR3

arch/x86/include/asm/pda.h

+0 −2
Original line number Diff line number Diff line
@@ -25,9 +25,7 @@ struct x8664_pda { 
	char *irqstackptr;

	short nodenumber;		/* number of current node (32k max) */

	short in_bootmem;		/* pda lives in bootmem */

	short mmu_state;

	short isidle;

	struct mm_struct *active_mm;

} ____cacheline_aligned_in_smp;



DECLARE_PER_CPU(struct x8664_pda, __pda);

arch/x86/include/asm/tlbflush.h

+2 −5
Original line number Diff line number Diff line
@@ -148,20 +148,17 @@ void native_flush_tlb_others(const struct cpumask *cpumask, 
#define TLBSTATE_OK	1

#define TLBSTATE_LAZY	2



#ifdef CONFIG_X86_32

struct tlb_state {

	struct mm_struct *active_mm;

	int state;

	char __cacheline_padding[L1_CACHE_BYTES-8];

};

DECLARE_PER_CPU(struct tlb_state, cpu_tlbstate);



void reset_lazy_tlbstate(void);

#else

static inline void reset_lazy_tlbstate(void)

{

	percpu_write(cpu_tlbstate.state, 0);

	percpu_write(cpu_tlbstate.active_mm, &init_mm);

}

#endif



#endif	/* SMP */

arch/x86/kernel/cpu/common.c

+0 −2
Original line number Diff line number Diff line
@@ -897,8 +897,6 @@ void __cpuinit pda_init(int cpu) 
	pda->irqcount = -1;

	pda->kernelstack = (unsigned long)stack_thread_info() -

				 PDA_STACKOFFSET + THREAD_SIZE;

	pda->active_mm = &init_mm;

	pda->mmu_state = 0;



	if (cpu == 0) {

		/* others are initialized in smpboot.c */

arch/x86/kernel/tlb_32.c

+2 −10
Original line number Diff line number Diff line
@@ -4,8 +4,8 @@ 


#include <asm/tlbflush.h>



DEFINE_PER_CPU(struct tlb_state, cpu_tlbstate)

			____cacheline_aligned = { &init_mm, 0, };

DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)

			= { &init_mm, 0, };



/* must come after the send_IPI functions above for inlining */

#include <mach_ipi.h>
@@ -231,14 +231,6 @@ void flush_tlb_all(void) 
	on_each_cpu(do_flush_tlb_all, NULL, 1);

}



void reset_lazy_tlbstate(void)

{

	int cpu = raw_smp_processor_id();



	per_cpu(cpu_tlbstate, cpu).state = 0;

	per_cpu(cpu_tlbstate, cpu).active_mm = &init_mm;

}



static int init_flush_cpumask(void)

{

	alloc_cpumask_var(&flush_cpumask, GFP_KERNEL);