Commit 2045a3b8 authored by Feiyang Chen's avatar Feiyang Chen Committed by Andrew Morton
Browse files

mm/sparse-vmemmap: generalise vmemmap_populate_hugepages() 

Generalise vmemmap_populate_hugepages() so ARM64 & X86 & LoongArch can
share its implementation.

Link: https://lkml.kernel.org/r/20221027125253.3458989-4-chenhuacai@loongson.cn


Signed-off-by: default avatarFeiyang Chen <chenfeiyang@loongson.cn>
Signed-off-by: default avatarHuacai Chen <chenhuacai@loongson.cn>
Acked-by: default avatarWill Deacon <will@kernel.org>
Acked-by: default avatarDave Hansen <dave.hansen@linux.intel.com>
Reviewed-by: default avatarArnd Bergmann <arnd@arndb.de>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Dinh Nguyen <dinguyen@kernel.org>
Cc: Guo Ren <guoren@kernel.org>
Cc: Jiaxun Yang <jiaxun.yang@flygoat.com>
Cc: Min Zhou <zhoumin@loongson.cn>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Philippe Mathieu-Daudé <philmd@linaro.org>
Cc: Thomas Bogendoerfer <tsbogend@alpha.franken.de>
Cc: Xuefeng Li <lixuefeng@loongson.cn>
Cc: Xuerui Wang <kernel@xen0n.name>
Cc: Muchun Song <songmuchun@bytedance.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 7b09f5af

arch/arm64/mm/mmu.c

+15 −40
Original line number Diff line number Diff line
@@ -1137,53 +1137,28 @@ static void free_empty_tables(unsigned long addr, unsigned long end, 
}

#endif



void __meminit vmemmap_set_pmd(pmd_t *pmdp, void *p, int node,

			       unsigned long addr, unsigned long next)

{

	pmd_set_huge(pmdp, __pa(p), __pgprot(PROT_SECT_NORMAL));

}



int __meminit vmemmap_check_pmd(pmd_t *pmdp, int node,

				unsigned long addr, unsigned long next)

{

	vmemmap_verify((pte_t *)pmdp, node, addr, next);

	return 1;

}



int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,

		struct vmem_altmap *altmap)

{

	unsigned long addr = start;

	unsigned long next;

	pgd_t *pgdp;

	p4d_t *p4dp;

	pud_t *pudp;

	pmd_t *pmdp;



	WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END));



	if (!ARM64_KERNEL_USES_PMD_MAPS)

		return vmemmap_populate_basepages(start, end, node, altmap);



	do {

		next = pmd_addr_end(addr, end);



		pgdp = vmemmap_pgd_populate(addr, node);

		if (!pgdp)

			return -ENOMEM;



		p4dp = vmemmap_p4d_populate(pgdp, addr, node);

		if (!p4dp)

			return -ENOMEM;



		pudp = vmemmap_pud_populate(p4dp, addr, node);

		if (!pudp)

			return -ENOMEM;



		pmdp = pmd_offset(pudp, addr);

		if (pmd_none(READ_ONCE(*pmdp))) {

			void *p = NULL;



			p = vmemmap_alloc_block_buf(PMD_SIZE, node, altmap);

			if (!p) {

				if (vmemmap_populate_basepages(addr, next, node, altmap))

					return -ENOMEM;

				continue;

			}



			pmd_set_huge(pmdp, __pa(p), __pgprot(PROT_SECT_NORMAL));

		} else

			vmemmap_verify((pte_t *)pmdp, node, addr, next);

	} while (addr = next, addr != end);



	return 0;

	else

		return vmemmap_populate_hugepages(start, end, node, altmap);

}



#ifdef CONFIG_MEMORY_HOTPLUG

arch/loongarch/mm/init.c

+16 −43
Original line number Diff line number Diff line
@@ -153,52 +153,25 @@ EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid); 
#endif



#ifdef CONFIG_SPARSEMEM_VMEMMAP

static int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,

						int node, struct vmem_altmap *altmap)

void __meminit vmemmap_set_pmd(pmd_t *pmd, void *p, int node,

			       unsigned long addr, unsigned long next)

{

	unsigned long addr = start;

	unsigned long next;

	pgd_t *pgd;

	p4d_t *p4d;

	pud_t *pud;

	pmd_t *pmd;



	for (addr = start; addr < end; addr = next) {

		next = pmd_addr_end(addr, end);



		pgd = vmemmap_pgd_populate(addr, node);

		if (!pgd)

			return -ENOMEM;

		p4d = vmemmap_p4d_populate(pgd, addr, node);

		if (!p4d)

			return -ENOMEM;

		pud = vmemmap_pud_populate(p4d, addr, node);

		if (!pud)

			return -ENOMEM;



		pmd = pmd_offset(pud, addr);

		if (pmd_none(*pmd)) {

			void *p = NULL;



			p = vmemmap_alloc_block_buf(PMD_SIZE, node, NULL);

			if (p) {

	pmd_t entry;



	entry = pfn_pmd(virt_to_pfn(p), PAGE_KERNEL);

	pmd_val(entry) |= _PAGE_HUGE | _PAGE_HGLOBAL;

	set_pmd_at(&init_mm, addr, pmd, entry);



				continue;

}

		} else if (pmd_val(*pmd) & _PAGE_HUGE) {



int __meminit vmemmap_check_pmd(pmd_t *pmd, int node,

				unsigned long addr, unsigned long next)

{

	int huge = pmd_val(*pmd) & _PAGE_HUGE;



	if (huge)

		vmemmap_verify((pte_t *)pmd, node, addr, next);

			continue;

		}

		if (vmemmap_populate_basepages(addr, next, node, NULL))

			return -ENOMEM;

	}



	return 0;

	return huge;

}



int __meminit vmemmap_populate(unsigned long start, unsigned long end,

arch/x86/mm/init_64.c

+32 −60
Original line number Diff line number Diff line
@@ -1492,37 +1492,9 @@ static long __meminitdata addr_start, addr_end; 
static void __meminitdata *p_start, *p_end;

static int __meminitdata node_start;



static int __meminit vmemmap_populate_hugepages(unsigned long start,

		unsigned long end, int node, struct vmem_altmap *altmap)

void __meminit vmemmap_set_pmd(pmd_t *pmd, void *p, int node,

			       unsigned long addr, unsigned long next)

{

	unsigned long addr;

	unsigned long next;

	pgd_t *pgd;

	p4d_t *p4d;

	pud_t *pud;

	pmd_t *pmd;



	for (addr = start; addr < end; addr = next) {

		next = pmd_addr_end(addr, end);



		pgd = vmemmap_pgd_populate(addr, node);

		if (!pgd)

			return -ENOMEM;



		p4d = vmemmap_p4d_populate(pgd, addr, node);

		if (!p4d)

			return -ENOMEM;



		pud = vmemmap_pud_populate(p4d, addr, node);

		if (!pud)

			return -ENOMEM;



		pmd = pmd_offset(pud, addr);

		if (pmd_none(*pmd)) {

			void *p;



			p = vmemmap_alloc_block_buf(PMD_SIZE, node, altmap);

			if (p) {

	pte_t entry;



	entry = pfn_pte(__pa(p) >> PAGE_SHIFT,
@@ -1545,19 +1517,19 @@ static int __meminit vmemmap_populate_hugepages(unsigned long start, 
	if (!IS_ALIGNED(addr, PMD_SIZE) ||

		!IS_ALIGNED(next, PMD_SIZE))

		vmemmap_use_new_sub_pmd(addr, next);

}



				continue;

			} else if (altmap)

				return -ENOMEM; /* no fallback */

		} else if (pmd_large(*pmd)) {

int __meminit vmemmap_check_pmd(pmd_t *pmd, int node,

				unsigned long addr, unsigned long next)

{

	int large = pmd_large(*pmd);



	if (pmd_large(*pmd)) {

		vmemmap_verify((pte_t *)pmd, node, addr, next);

		vmemmap_use_sub_pmd(addr, next);

			continue;

	}

		if (vmemmap_populate_basepages(addr, next, node, NULL))

			return -ENOMEM;

	}

	return 0;



	return large;

}



int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,

include/linux/mm.h

+6 −0
Original line number Diff line number Diff line
@@ -3373,8 +3373,14 @@ struct vmem_altmap; 
void *vmemmap_alloc_block_buf(unsigned long size, int node,

			      struct vmem_altmap *altmap);

void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);

void vmemmap_set_pmd(pmd_t *pmd, void *p, int node,

		     unsigned long addr, unsigned long next);

int vmemmap_check_pmd(pmd_t *pmd, int node,

		      unsigned long addr, unsigned long next);

int vmemmap_populate_basepages(unsigned long start, unsigned long end,

			       int node, struct vmem_altmap *altmap);

int vmemmap_populate_hugepages(unsigned long start, unsigned long end,

			       int node, struct vmem_altmap *altmap);

int vmemmap_populate(unsigned long start, unsigned long end, int node,

		struct vmem_altmap *altmap);

void vmemmap_populate_print_last(void);

mm/sparse-vmemmap.c

+63 −0
Original line number Diff line number Diff line
@@ -295,6 +295,69 @@ int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end, 
	return vmemmap_populate_range(start, end, node, altmap, NULL);

}



void __weak __meminit vmemmap_set_pmd(pmd_t *pmd, void *p, int node,

				      unsigned long addr, unsigned long next)

{

}



int __weak __meminit vmemmap_check_pmd(pmd_t *pmd, int node,

				       unsigned long addr, unsigned long next)

{

	return 0;

}



int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,

					 int node, struct vmem_altmap *altmap)

{

	unsigned long addr;

	unsigned long next;

	pgd_t *pgd;

	p4d_t *p4d;

	pud_t *pud;

	pmd_t *pmd;



	for (addr = start; addr < end; addr = next) {

		next = pmd_addr_end(addr, end);



		pgd = vmemmap_pgd_populate(addr, node);

		if (!pgd)

			return -ENOMEM;



		p4d = vmemmap_p4d_populate(pgd, addr, node);

		if (!p4d)

			return -ENOMEM;



		pud = vmemmap_pud_populate(p4d, addr, node);

		if (!pud)

			return -ENOMEM;



		pmd = pmd_offset(pud, addr);

		if (pmd_none(READ_ONCE(*pmd))) {

			void *p;



			p = vmemmap_alloc_block_buf(PMD_SIZE, node, altmap);

			if (p) {

				vmemmap_set_pmd(pmd, p, node, addr, next);

				continue;

			} else if (altmap) {

				/*

				 * No fallback: In any case we care about, the

				 * altmap should be reasonably sized and aligned

				 * such that vmemmap_alloc_block_buf() will always

				 * succeed. For consistency with the PTE case,

				 * return an error here as failure could indicate

				 * a configuration issue with the size of the altmap.

				 */

				return -ENOMEM;

			}

		} else if (vmemmap_check_pmd(pmd, node, addr, next))

			continue;

		if (vmemmap_populate_basepages(addr, next, node, altmap))

			return -ENOMEM;

	}

	return 0;

}



/*

 * For compound pages bigger than section size (e.g. x86 1G compound

 * pages with 2M subsection size) fill the rest of sections as tail