Commit 8845ec7e authored by Nikita Panov's avatar Nikita Panov Committed by Denis Darvish
Browse files

mm: allow per-NUMA node local P4D/PUD/PMD/PTE allocation 

kunpeng inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/IBOJU2



-------------------------------------------------

Acked-by: default avatarArtem Kuzin <artem.kuzin@huawei.com>
Acked-by: default avatarAlexander Grubnikov <alexander.grubnikov@huawei.com>
Acked-by: default avatarIlya Hanov <ilya.hanov@huawei-partners.com>
Acked-by: default avatarDenis Darvish <darvish.denis@huawei.com>
Signed-off-by: default avatarNikita Panov <panov.nikita@huawei.com>
parent efffa341

include/asm-generic/pgalloc.h

+78 −0
Original line number Diff line number Diff line
@@ -76,6 +76,24 @@ static inline pgtable_t __pte_alloc_one(struct mm_struct *mm, gfp_t gfp) 
	return ptdesc_page(ptdesc);

}



#ifdef CONFIG_KERNEL_REPLICATION

static inline pgtable_t __pte_alloc_one_node(unsigned int nid,

					     struct mm_struct *mm, gfp_t gfp)

{

	struct page *pte;



	pte = alloc_pages_node(nid, gfp, 0);

	if (!pte)

		return NULL;

	if (!pagetable_pte_ctor(page_ptdesc(pte))) {

		__free_page(pte);

		return NULL;

	}



	return pte;

}

#endif



#ifndef __HAVE_ARCH_PTE_ALLOC_ONE

/**

 * pte_alloc_one - allocate a page for PTE-level user page table
@@ -89,6 +107,15 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm) 
{

	return __pte_alloc_one(mm, GFP_PGTABLE_USER);

}



#ifdef CONFIG_KERNEL_REPLICATION

static inline pgtable_t pte_alloc_one_node(unsigned int nid,

					   struct mm_struct *mm)

{

	return __pte_alloc_one_node(nid, mm, GFP_PGTABLE_USER | __GFP_THISNODE);

}

#endif



#endif



/*
@@ -140,6 +167,30 @@ static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) 
	}

	return ptdesc_address(ptdesc);

}



#ifdef CONFIG_KERNEL_REPLICATION

static inline pmd_t *pmd_alloc_one_node(unsigned int nid,

					struct mm_struct *mm,

					unsigned long addr)

{

	struct ptdesc *ptdesc;

	gfp_t gfp = GFP_PGTABLE_USER;



	if (mm == &init_mm)

		gfp = GFP_PGTABLE_KERNEL;



	gfp |= __GFP_THISNODE;



	ptdesc = pagetable_alloc_node(nid, gfp, 0);

	if (!ptdesc)

		return NULL;

	if (!pagetable_pmd_ctor(ptdesc)) {

		pagetable_free(ptdesc);

		return NULL;

	}

	return ptdesc_address(ptdesc);

}

#endif /* CONFIG_KERNEL_REPLICATION */

#endif



#ifndef __HAVE_ARCH_PMD_FREE
@@ -172,6 +223,25 @@ static inline pud_t *__pud_alloc_one(struct mm_struct *mm, unsigned long addr) 
	return ptdesc_address(ptdesc);

}



#ifdef CONFIG_KERNEL_REPLICATION

static inline pud_t *__pud_alloc_one_node(unsigned int nid,

					  struct mm_struct *mm,

					  unsigned long addr)

{

	gfp_t gfp = GFP_PGTABLE_USER;

	struct ptdesc *ptdesc;



	if (mm == &init_mm)

		gfp = GFP_PGTABLE_KERNEL;



	gfp |= __GFP_THISNODE;

	ptdesc = pagetable_alloc_node(nid, gfp, 0);

	if (!ptdesc)

		return NULL;

	return ptdesc_address(ptdesc);

}

#endif /* CONFIG_KERNEL_REPLICATION */



#ifndef __HAVE_ARCH_PUD_ALLOC_ONE

/**

 * pud_alloc_one - allocate memory for a PUD-level page table
@@ -186,6 +256,14 @@ static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) 
{

	return __pud_alloc_one(mm, addr);

}



#ifdef CONFIG_KERNEL_REPLICATION

static inline pud_t *pud_alloc_one_node(unsigned int nid,

					struct mm_struct *mm, unsigned long addr)

{

	return __pud_alloc_one_node(nid, mm, addr);

}

#endif /* CONFIG_KERNEL_REPLICATION */

#endif



static inline void __pud_free(struct mm_struct *mm, pud_t *pud)

include/asm-generic/pgtable-nop4d.h

+5 −0
Original line number Diff line number Diff line
@@ -48,6 +48,11 @@ static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) 
 * inside the pgd, so has no extra memory associated with it.

 */

#define p4d_alloc_one(mm, address)		NULL



#ifdef CONFIG_KERNEL_REPLICATION

#define p4d_alloc_one_node(nid, mm, address)	NULL

#endif



#define p4d_free(mm, x)				do { } while (0)

#define p4d_free_tlb(tlb, x, a)			do { } while (0)

include/asm-generic/pgtable-nopmd.h

+5 −0
Original line number Diff line number Diff line
@@ -60,6 +60,11 @@ static inline pmd_t * pmd_offset(pud_t * pud, unsigned long address) 
 * inside the pud, so has no extra memory associated with it.

 */

#define pmd_alloc_one(mm, address)		NULL



#ifdef CONFIG_KERNEL_REPLICATION

#define pmd_alloc_one_node(nid, mm, address)	NULL

#endif



static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)

{

}

include/asm-generic/pgtable-nopud.h

+5 −0
Original line number Diff line number Diff line
@@ -56,6 +56,11 @@ static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) 
 * inside the p4d, so has no extra memory associated with it.

 */

#define pud_alloc_one(mm, address)		NULL



#ifdef CONFIG_KERNEL_REPLICATION

#define pud_alloc_one_node(nid, mm, address)	NULL

#endif



#define pud_free(mm, x)				do { } while (0)

#define pud_free_tlb(tlb, x, a)		        do { } while (0)

include/linux/gfp.h

+5 −0
Original line number Diff line number Diff line
@@ -303,6 +303,11 @@ static inline struct page *alloc_page_vma(gfp_t gfp, 
extern unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order);

extern unsigned long get_zeroed_page(gfp_t gfp_mask);



#ifdef CONFIG_KERNEL_REPLICATION

extern unsigned long __get_free_pages_node(unsigned int nid, gfp_t gfp_mask, unsigned int order);

extern unsigned long get_zeroed_page_node(unsigned int nid, gfp_t gfp_mask);

#endif /* CONFIG_KERNEL_REPLICATION */



void *alloc_pages_exact(size_t size, gfp_t gfp_mask) __alloc_size(1);

void free_pages_exact(void *virt, size_t size);

__meminit void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) __alloc_size(2);