Commit 23f91716 authored by Kefeng Wang's avatar Kefeng Wang Committed by Linus Torvalds
Browse files

mm: percpu: add generic pcpu_fc_alloc/free funciton 

With the previous patch, we could add a generic pcpu first chunk
allocate and free function to cleanup the duplicated definations on each
architecture.

Link: https://lkml.kernel.org/r/20211216112359.103822-4-wangkefeng.wang@huawei.com


Signed-off-by: default avatarKefeng Wang <wangkefeng.wang@huawei.com>
Cc: Thomas Bogendoerfer <tsbogend@alpha.franken.de>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Dennis Zhou <dennis@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Christoph Lameter <cl@linux.com>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: "Rafael J. Wysocki" <rafael@kernel.org>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 1ca3fb3a

arch/mips/mm/init.c

+1 −15
Original line number Diff line number Diff line
@@ -524,19 +524,6 @@ static int __init pcpu_cpu_to_node(int cpu) 
	return cpu_to_node(cpu);

}



static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align,

				   pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn)

{

	return memblock_alloc_try_nid(size, align, __pa(MAX_DMA_ADDRESS),

				      MEMBLOCK_ALLOC_ACCESSIBLE,

				      cpu_to_nd_fn(cpu));

}



static void __init pcpu_fc_free(void *ptr, size_t size)

{

	memblock_free(ptr, size);

}



void __init setup_per_cpu_areas(void)

{

	unsigned long delta;
@@ -550,8 +537,7 @@ void __init setup_per_cpu_areas(void) 
	rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,

				    PERCPU_DYNAMIC_RESERVE, PAGE_SIZE,

				    pcpu_cpu_distance,

				    pcpu_cpu_to_node,

				    pcpu_fc_alloc, pcpu_fc_free);

				    pcpu_cpu_to_node);

	if (rc < 0)

		panic("Failed to initialize percpu areas.");

arch/powerpc/kernel/setup_64.c

+2 −49
Original line number Diff line number Diff line
@@ -771,50 +771,6 @@ void __init emergency_stack_init(void) 
}



#ifdef CONFIG_SMP

/**

 * pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu

 * @cpu: cpu to allocate for

 * @size: size allocation in bytes

 * @align: alignment

 *

 * Allocate @size bytes aligned at @align for cpu @cpu.  This wrapper

 * does the right thing for NUMA regardless of the current

 * configuration.

 *

 * RETURNS:

 * Pointer to the allocated area on success, NULL on failure.

 */

static void * __init pcpu_alloc_bootmem(unsigned int cpu, size_t size, size_t align,

					pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn)

{

	const unsigned long goal = __pa(MAX_DMA_ADDRESS);

#ifdef CONFIG_NUMA

	int node = cpu_to_nd_fun(cpu);

	void *ptr;



	if (!node_online(node) || !NODE_DATA(node)) {

		ptr = memblock_alloc_from(size, align, goal);

		pr_info("cpu %d has no node %d or node-local memory\n",

			cpu, node);

		pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n",

			 cpu, size, __pa(ptr));

	} else {

		ptr = memblock_alloc_try_nid(size, align, goal,

					     MEMBLOCK_ALLOC_ACCESSIBLE, node);

		pr_debug("per cpu data for cpu%d %lu bytes on node%d at "

			 "%016lx\n", cpu, size, node, __pa(ptr));

	}

	return ptr;

#else

	return memblock_alloc_from(size, align, goal);

#endif

}



static void __init pcpu_free_bootmem(void *ptr, size_t size)

{

	memblock_free(ptr, size);

}



static int pcpu_cpu_distance(unsigned int from, unsigned int to)

{

	if (early_cpu_to_node(from) == early_cpu_to_node(to))
@@ -896,8 +852,7 @@ void __init setup_per_cpu_areas(void) 


	if (pcpu_chosen_fc != PCPU_FC_PAGE) {

		rc = pcpu_embed_first_chunk(0, dyn_size, atom_size, pcpu_cpu_distance,

					    pcpu_cpu_to_node,

					    pcpu_alloc_bootmem, pcpu_free_bootmem);

					    pcpu_cpu_to_node);

		if (rc)

			pr_warn("PERCPU: %s allocator failed (%d), "

				"falling back to page size\n",
@@ -905,9 +860,7 @@ void __init setup_per_cpu_areas(void) 
	}



	if (rc < 0)

		rc = pcpu_page_first_chunk(0, pcpu_cpu_to_node,

					   pcpu_alloc_bootmem, pcpu_free_bootmem,

					   pcpu_populate_pte);

		rc = pcpu_page_first_chunk(0, pcpu_cpu_to_node, pcpu_populate_pte);

	if (rc < 0)

		panic("cannot initialize percpu area (err=%d)", rc);

arch/sparc/kernel/smp_64.c

+1 −49
Original line number Diff line number Diff line
@@ -1526,50 +1526,6 @@ void smp_send_stop(void) 
		smp_call_function(stop_this_cpu, NULL, 0);

}



/**

 * pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu

 * @cpu: cpu to allocate for

 * @size: size allocation in bytes

 * @align: alignment

 *

 * Allocate @size bytes aligned at @align for cpu @cpu.  This wrapper

 * does the right thing for NUMA regardless of the current

 * configuration.

 *

 * RETURNS:

 * Pointer to the allocated area on success, NULL on failure.

 */

static void * __init pcpu_alloc_bootmem(unsigned int cpu, size_t size, size_t align,

					pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn)

{

	const unsigned long goal = __pa(MAX_DMA_ADDRESS);

#ifdef CONFIG_NUMA

	int node = cpu_to_nd_fn(cpu);

	void *ptr;



	if (!node_online(node) || !NODE_DATA(node)) {

		ptr = memblock_alloc_from(size, align, goal);

		pr_info("cpu %d has no node %d or node-local memory\n",

			cpu, node);

		pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n",

			 cpu, size, __pa(ptr));

	} else {

		ptr = memblock_alloc_try_nid(size, align, goal,

					     MEMBLOCK_ALLOC_ACCESSIBLE, node);

		pr_debug("per cpu data for cpu%d %lu bytes on node%d at "

			 "%016lx\n", cpu, size, node, __pa(ptr));

	}

	return ptr;

#else

	return memblock_alloc_from(size, align, goal);

#endif

}



static void __init pcpu_free_bootmem(void *ptr, size_t size)

{

	memblock_free(ptr, size);

}



static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)

{

	if (cpu_to_node(from) == cpu_to_node(to))
@@ -1646,9 +1602,7 @@ void __init setup_per_cpu_areas(void) 
		rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,

					    PERCPU_DYNAMIC_RESERVE, 4 << 20,

					    pcpu_cpu_distance,

					    pcpu_cpu_to_node,

					    pcpu_alloc_bootmem,

					    pcpu_free_bootmem);

					    pcpu_cpu_to_node);

		if (rc)

			pr_warn("PERCPU: %s allocator failed (%d), "

				"falling back to page size\n",
@@ -1657,8 +1611,6 @@ void __init setup_per_cpu_areas(void) 
	if (rc < 0)

		rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE,

					   pcpu_cpu_to_node,

					   pcpu_alloc_bootmem,

					   pcpu_free_bootmem,

					   pcpu_populate_pte);

	if (rc < 0)

		panic("cannot initialize percpu area (err=%d)", rc);

arch/x86/kernel/setup_percpu.c

+1 −58
Original line number Diff line number Diff line
@@ -84,61 +84,6 @@ static bool __init pcpu_need_numa(void) 
}

#endif



/**

 * pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu

 * @cpu: cpu to allocate for

 * @size: size allocation in bytes

 * @align: alignment

 *

 * Allocate @size bytes aligned at @align for cpu @cpu.  This wrapper

 * does the right thing for NUMA regardless of the current

 * configuration.

 *

 * RETURNS:

 * Pointer to the allocated area on success, NULL on failure.

 */

static void * __init pcpu_alloc_bootmem(unsigned int cpu, unsigned long size, unsigned long align,

					pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn)

{

	const unsigned long goal = __pa(MAX_DMA_ADDRESS);

#ifdef CONFIG_NUMA

	int node = cpu_to_nd_fn(cpu);

	void *ptr;



	if (!node_online(node) || !NODE_DATA(node)) {

		ptr = memblock_alloc_from(size, align, goal);

		pr_info("cpu %d has no node %d or node-local memory\n",

			cpu, node);

		pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n",

			 cpu, size, __pa(ptr));

	} else {

		ptr = memblock_alloc_try_nid(size, align, goal,

					     MEMBLOCK_ALLOC_ACCESSIBLE,

					     node);



		pr_debug("per cpu data for cpu%d %lu bytes on node%d at %016lx\n",

			 cpu, size, node, __pa(ptr));

	}

	return ptr;

#else

	return memblock_alloc_from(size, align, goal);

#endif

}



/*

 * Helpers for first chunk memory allocation

 */

static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align,

				   pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn)

{

	return pcpu_alloc_bootmem(cpu, size, align, cpu_to_nd_fn);

}



static void __init pcpu_fc_free(void *ptr, size_t size)

{

	memblock_free(ptr, size);

}



static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)

{

#ifdef CONFIG_NUMA
@@ -211,8 +156,7 @@ void __init setup_per_cpu_areas(void) 
		rc = pcpu_embed_first_chunk(PERCPU_FIRST_CHUNK_RESERVE,

					    dyn_size, atom_size,

					    pcpu_cpu_distance,

					    pcpu_cpu_to_node,

					    pcpu_fc_alloc, pcpu_fc_free);

					    pcpu_cpu_to_node);

		if (rc < 0)

			pr_warn("%s allocator failed (%d), falling back to page size\n",

				pcpu_fc_names[pcpu_chosen_fc], rc);
@@ -220,7 +164,6 @@ void __init setup_per_cpu_areas(void) 
	if (rc < 0)

		rc = pcpu_page_first_chunk(PERCPU_FIRST_CHUNK_RESERVE,

					   pcpu_cpu_to_node,

					   pcpu_fc_alloc, pcpu_fc_free,

					   pcpup_populate_pte);

	if (rc < 0)

		panic("cannot initialize percpu area (err=%d)", rc);

drivers/base/arch_numa.c

+1 −18
Original line number Diff line number Diff line
@@ -155,20 +155,6 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) 
	return node_distance(early_cpu_to_node(from), early_cpu_to_node(to));

}



static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align,

				   pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn)

{

	int nid = cpu_to_nd_fn(cpu);



	return  memblock_alloc_try_nid(size, align,

			__pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, nid);

}



static void __init pcpu_fc_free(void *ptr, size_t size)

{

	memblock_free(ptr, size);

}



#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK

static void __init pcpu_populate_pte(unsigned long addr)

{
@@ -229,8 +215,7 @@ void __init setup_per_cpu_areas(void) 
		rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,

					    PERCPU_DYNAMIC_RESERVE, PAGE_SIZE,

					    pcpu_cpu_distance,

					    early_cpu_to_node,

					    pcpu_fc_alloc, pcpu_fc_free);

					    early_cpu_to_node);

#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK

		if (rc < 0)

			pr_warn("PERCPU: %s allocator failed (%d), falling back to page size\n",
@@ -242,8 +227,6 @@ void __init setup_per_cpu_areas(void) 
	if (rc < 0)

		rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE,

					   early_cpu_to_node,

					   pcpu_fc_alloc,

					   pcpu_fc_free,

					   pcpu_populate_pte);

#endif

	if (rc < 0)