Commit 20c03576 authored by Kefeng Wang's avatar Kefeng Wang Committed by Linus Torvalds
Browse files

mm: percpu: add generic pcpu_populate_pte() function 

With NEED_PER_CPU_PAGE_FIRST_CHUNK enabled, we need a function to
populate pte, this patch adds a generic pcpu populate pte function,
pcpu_populate_pte(), which is marked __weak and used on most
architectures, but it is overridden on x86, which has its own
implementation.

Link: https://lkml.kernel.org/r/20211216112359.103822-5-wangkefeng.wang@huawei.com


Signed-off-by: default avatarKefeng Wang <wangkefeng.wang@huawei.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: "Rafael J. Wysocki" <rafael@kernel.org>
Cc: Dennis Zhou <dennis@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Christoph Lameter <cl@linux.com>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: Thomas Bogendoerfer <tsbogend@alpha.franken.de>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 23f91716

arch/powerpc/kernel/setup_64.c

+1 −46
Original line number Diff line number Diff line
@@ -787,51 +787,6 @@ static __init int pcpu_cpu_to_node(int cpu) 
unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;

EXPORT_SYMBOL(__per_cpu_offset);



static void __init pcpu_populate_pte(unsigned long addr)

{

	pgd_t *pgd = pgd_offset_k(addr);

	p4d_t *p4d;

	pud_t *pud;

	pmd_t *pmd;



	p4d = p4d_offset(pgd, addr);

	if (p4d_none(*p4d)) {

		pud_t *new;



		new = memblock_alloc(PUD_TABLE_SIZE, PUD_TABLE_SIZE);

		if (!new)

			goto err_alloc;

		p4d_populate(&init_mm, p4d, new);

	}



	pud = pud_offset(p4d, addr);

	if (pud_none(*pud)) {

		pmd_t *new;



		new = memblock_alloc(PMD_TABLE_SIZE, PMD_TABLE_SIZE);

		if (!new)

			goto err_alloc;

		pud_populate(&init_mm, pud, new);

	}



	pmd = pmd_offset(pud, addr);

	if (!pmd_present(*pmd)) {

		pte_t *new;



		new = memblock_alloc(PTE_TABLE_SIZE, PTE_TABLE_SIZE);

		if (!new)

			goto err_alloc;

		pmd_populate_kernel(&init_mm, pmd, new);

	}



	return;



err_alloc:

	panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n",

	      __func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);

}





void __init setup_per_cpu_areas(void)

{

	const size_t dyn_size = PERCPU_MODULE_RESERVE + PERCPU_DYNAMIC_RESERVE;
@@ -860,7 +815,7 @@ void __init setup_per_cpu_areas(void) 
	}



	if (rc < 0)

		rc = pcpu_page_first_chunk(0, pcpu_cpu_to_node, pcpu_populate_pte);

		rc = pcpu_page_first_chunk(0, pcpu_cpu_to_node);

	if (rc < 0)

		panic("cannot initialize percpu area (err=%d)", rc);

arch/sparc/kernel/smp_64.c

+1 −55
Original line number Diff line number Diff line
@@ -1539,59 +1539,6 @@ static int __init pcpu_cpu_to_node(int cpu) 
	return cpu_to_node(cpu);

}



static void __init pcpu_populate_pte(unsigned long addr)

{

	pgd_t *pgd = pgd_offset_k(addr);

	p4d_t *p4d;

	pud_t *pud;

	pmd_t *pmd;



	if (pgd_none(*pgd)) {

		pud_t *new;



		new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);

		if (!new)

			goto err_alloc;

		pgd_populate(&init_mm, pgd, new);

	}



	p4d = p4d_offset(pgd, addr);

	if (p4d_none(*p4d)) {

		pud_t *new;



		new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);

		if (!new)

			goto err_alloc;

		p4d_populate(&init_mm, p4d, new);

	}



	pud = pud_offset(p4d, addr);

	if (pud_none(*pud)) {

		pmd_t *new;



		new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);

		if (!new)

			goto err_alloc;

		pud_populate(&init_mm, pud, new);

	}



	pmd = pmd_offset(pud, addr);

	if (!pmd_present(*pmd)) {

		pte_t *new;



		new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);

		if (!new)

			goto err_alloc;

		pmd_populate_kernel(&init_mm, pmd, new);

	}



	return;



err_alloc:

	panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n",

	      __func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);

}



void __init setup_per_cpu_areas(void)

{

	unsigned long delta;
@@ -1610,8 +1557,7 @@ void __init setup_per_cpu_areas(void) 
	}

	if (rc < 0)

		rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE,

					   pcpu_cpu_to_node,

					   pcpu_populate_pte);

					   pcpu_cpu_to_node);

	if (rc < 0)

		panic("cannot initialize percpu area (err=%d)", rc);

arch/x86/kernel/setup_percpu.c

+2 −3
Original line number Diff line number Diff line
@@ -101,7 +101,7 @@ static int __init pcpu_cpu_to_node(int cpu) 
	return early_cpu_to_node(cpu);

}



static void __init pcpup_populate_pte(unsigned long addr)

void __init pcpu_populate_pte(unsigned long addr)

{

	populate_extra_pte(addr);

}
@@ -163,8 +163,7 @@ void __init setup_per_cpu_areas(void) 
	}

	if (rc < 0)

		rc = pcpu_page_first_chunk(PERCPU_FIRST_CHUNK_RESERVE,

					   pcpu_cpu_to_node,

					   pcpup_populate_pte);

					   pcpu_cpu_to_node);

	if (rc < 0)

		panic("cannot initialize percpu area (err=%d)", rc);

drivers/base/arch_numa.c

+1 −50
Original line number Diff line number Diff line
@@ -14,7 +14,6 @@ 
#include <linux/of.h>



#include <asm/sections.h>

#include <asm/pgalloc.h>



struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;

EXPORT_SYMBOL(node_data);
@@ -155,52 +154,6 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) 
	return node_distance(early_cpu_to_node(from), early_cpu_to_node(to));

}



#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK

static void __init pcpu_populate_pte(unsigned long addr)

{

	pgd_t *pgd = pgd_offset_k(addr);

	p4d_t *p4d;

	pud_t *pud;

	pmd_t *pmd;



	p4d = p4d_offset(pgd, addr);

	if (p4d_none(*p4d)) {

		pud_t *new;



		new = memblock_alloc(PAGE_SIZE, PAGE_SIZE);

		if (!new)

			goto err_alloc;

		p4d_populate(&init_mm, p4d, new);

	}



	pud = pud_offset(p4d, addr);

	if (pud_none(*pud)) {

		pmd_t *new;



		new = memblock_alloc(PAGE_SIZE, PAGE_SIZE);

		if (!new)

			goto err_alloc;

		pud_populate(&init_mm, pud, new);

	}



	pmd = pmd_offset(pud, addr);

	if (!pmd_present(*pmd)) {

		pte_t *new;



		new = memblock_alloc(PAGE_SIZE, PAGE_SIZE);

		if (!new)

			goto err_alloc;

		pmd_populate_kernel(&init_mm, pmd, new);

	}



	return;



err_alloc:

	panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n",

	      __func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);

}

#endif



void __init setup_per_cpu_areas(void)

{

	unsigned long delta;
@@ -225,9 +178,7 @@ void __init setup_per_cpu_areas(void) 


#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK

	if (rc < 0)

		rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE,

					   early_cpu_to_node,

					   pcpu_populate_pte);

		rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, early_cpu_to_node);

#endif

	if (rc < 0)

		panic("Failed to initialize percpu areas (err=%d).", rc);

include/linux/percpu.h

+2 −3
Original line number Diff line number Diff line
@@ -95,7 +95,6 @@ extern const char * const pcpu_fc_names[PCPU_FC_NR]; 
extern enum pcpu_fc pcpu_chosen_fc;



typedef int (pcpu_fc_cpu_to_node_fn_t)(int cpu);

typedef void (*pcpu_fc_populate_pte_fn_t)(unsigned long addr);

typedef int (pcpu_fc_cpu_distance_fn_t)(unsigned int from, unsigned int to);



extern struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups,
@@ -113,9 +112,9 @@ extern int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, 
#endif



#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK

void __init pcpu_populate_pte(unsigned long addr);

extern int __init pcpu_page_first_chunk(size_t reserved_size,

				pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn,

				pcpu_fc_populate_pte_fn_t populate_pte_fn);

				pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn);

#endif



extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align) __alloc_size(1);