Commit 0bc0d0d5 authored by Liu Shixin's avatar Liu Shixin Committed by Yongqiang Liu
Browse files

dhugetlb: backport dynamic hugetlb feature 

hulk inclusion
category: feature
bugzilla: 46904, https://gitee.com/openeuler/kernel/issues/I6BDME


CVE: NA

--------------------------------

This feature has already beed supported on x86_64 and this is the origin
description:

 Dynamic hugetlb which is based on Hugetlb, supports to be splited
 dynamically in a specified cgroup. We add a hugetlb_pool in a
 mem_cgroup to manage dynamic hugetlb for corresponding cgroup.
 After dynamic hugepages are allocated for a cgroup, these hugepages
 can be used as 1G/2M/4K pages by split/merge opreation.

It is now supported on arm64. This feature will be limited to depends on
ARM64_4K_PAGES and not support cont-bits hugepage. We merge the previous
patches into one patch which is patch[1]. While merge the code ,we found
some code can be isolated by config DYNAMIC_HUGETLB, so we add patch[2] to
re-isolated them. In patch[3], we restrict the feature on mentioned limit.
The patch[4] add skip of dissolve hugepage which may conflict with memory
hotplug and memory failure. The patch[5] set DYNAMIC_HUGETLB to y in
hulk_defconfig to enable by default.

This patch includes all previous patches and the patches list is recorded
in bugzilla.

Signed-off-by: default avatarLiu Shixin <liushixin2@hauwei.com>
Reviewed-by: default avatarKefeng Wang <wangkefeng.wang@huawei.com>
Signed-off-by: default avatarYongqiang Liu <liuyongqiang13@huawei.com>
parent b8042a6b

fs/Kconfig

+9 −0
Original line number Diff line number Diff line
@@ -211,6 +211,15 @@ config TMPFS_INODE64 


	  If unsure, say N.



config DYNAMIC_HUGETLB

	bool "Dynamic HugeTLB"

	depends on HUGETLB_PAGE

	depends on MEMCG

	depends on CGROUP_HUGETLB

	help

	  Dynamic hugepage are used in memcg and can be splited into small pages

	  automatically. The tasks in the memcg prefer to alloc dynamic hugepage.



config HUGETLBFS

	bool "HugeTLB file system support"

	depends on X86 || IA64 || SPARC64 || (S390 && 64BIT) || \

fs/hugetlbfs/inode.c

+4 −0
Original line number Diff line number Diff line
@@ -1164,6 +1164,8 @@ static struct inode *hugetlbfs_alloc_inode(struct super_block *sb) 
	 * private inode.  This simplifies hugetlbfs_destroy_inode.

	 */

	mpol_shared_policy_init(&p->policy, NULL);

	/* Initialize hpool here in case of a quick call to destroy */

	p->hpool = get_dhugetlb_pool_from_task(current);



	return &p->vfs_inode;

}
@@ -1178,6 +1180,8 @@ static void hugetlbfs_destroy_inode(struct inode *inode) 
{

	hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb));

	mpol_free_shared_policy(&HUGETLBFS_I(inode)->policy);

	dhugetlb_pool_put(HUGETLBFS_I(inode)->hpool);

	HUGETLBFS_I(inode)->hpool = NULL;

	call_rcu(&inode->i_rcu, hugetlbfs_i_callback);

}

include/linux/gfp.h

+3 −1
Original line number Diff line number Diff line
@@ -501,7 +501,9 @@ static inline void arch_alloc_page(struct page *page, int order) { } 
struct page *

__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid,

							nodemask_t *nodemask);



void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,

						unsigned int alloc_flags);

bool free_pages_prepare(struct page *page, unsigned int order, bool check_free);

static inline struct page *

__alloc_pages(gfp_t gfp_mask, unsigned int order, int preferred_nid)

{

include/linux/hugetlb.h

+97 −0
Original line number Diff line number Diff line
@@ -289,6 +289,7 @@ struct hugetlbfs_inode_info { 
	struct shared_policy policy;

	struct inode vfs_inode;

	unsigned int seals;

	struct dhugetlb_pool *hpool;

};



static inline struct hugetlbfs_inode_info *HUGETLBFS_I(struct inode *inode)
@@ -655,6 +656,102 @@ static inline void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr 


#endif	/* CONFIG_HUGETLB_PAGE */



#ifdef CONFIG_DYNAMIC_HUGETLB

/* The number of small_page_pool for a dhugetlb_pool */

#define NR_SMPOOL		num_possible_cpus()

/* The max page number in a small_page_pool */

#define MAX_SMPOOL_PAGE		1024

/* number to move between list */

#define BATCH_SMPOOL_PAGE	(MAX_SMPOOL_PAGE >> 2)

/* We don't need to try 5 times, or we can't migrate the pages. */

#define HPOOL_RECLAIM_RETRIES	5



extern struct static_key_false dhugetlb_enabled_key;

#define dhugetlb_enabled (static_branch_unlikely(&dhugetlb_enabled_key))



#define DEFAULT_PAGESIZE	4096

extern rwlock_t dhugetlb_pagelist_rwlock;

struct dhugetlb_pagelist {

	unsigned long count;

	struct dhugetlb_pool *hpool[0];

};

extern struct dhugetlb_pagelist *dhugetlb_pagelist_t;



struct split_pages {

	struct list_head list;

	unsigned long start_pfn;

	unsigned long free_pages;

};



struct small_page_pool {

	spinlock_t lock;

	unsigned long free_pages;

	long used_pages;

	struct list_head head_page;

};



struct dhugetlb_pool {

	int nid;

	spinlock_t lock;

	spinlock_t reserved_lock;

	atomic_t refcnt;



	struct mem_cgroup *attach_memcg;



	struct list_head dhugetlb_1G_freelists;

	struct list_head dhugetlb_2M_freelists;

	struct list_head dhugetlb_4K_freelists;



	struct list_head split_1G_freelists;

	struct list_head split_2M_freelists;



	unsigned long total_nr_pages;



	unsigned long total_reserved_1G;

	unsigned long free_reserved_1G;

	unsigned long mmap_reserved_1G;

	unsigned long used_1G;

	unsigned long free_unreserved_1G;

	unsigned long nr_split_1G;



	unsigned long total_reserved_2M;

	unsigned long free_reserved_2M;

	unsigned long mmap_reserved_2M;

	unsigned long used_2M;

	unsigned long free_unreserved_2M;

	unsigned long nr_split_2M;



	unsigned long free_pages;

	struct small_page_pool smpool[0];

};



bool dhugetlb_pool_get(struct dhugetlb_pool *hpool);

void dhugetlb_pool_put(struct dhugetlb_pool *hpool);

struct dhugetlb_pool *hpool_alloc(unsigned long nid);

int alloc_hugepage_from_hugetlb(struct dhugetlb_pool *hpool,

				unsigned long nid, unsigned long size);

bool free_dhugetlb_pool(struct dhugetlb_pool *hpool);

int update_dhugetlb_pagelist(unsigned long idx, struct dhugetlb_pool *hpool);

struct dhugetlb_pool *get_dhugetlb_pool_from_dhugetlb_pagelist(

							struct page *page);

struct dhugetlb_pool *get_dhugetlb_pool_from_task(struct task_struct *tsk);

bool move_pages_from_hpool_to_smpool(struct dhugetlb_pool *hpool,

				     struct small_page_pool *smpool);

void move_pages_from_smpool_to_hpool(struct dhugetlb_pool *hpool,

				     struct small_page_pool *smpool);

void dhugetlb_reserve_hugepages(struct dhugetlb_pool *hpool,

				unsigned long count, bool gigantic);

#else

#define dhugetlb_enabled       0

struct dhugetlb_pool {};

static inline struct dhugetlb_pool *get_dhugetlb_pool_from_task(

						struct task_struct *tsk)

{

	return NULL;

}

static inline void dhugetlb_pool_put(struct dhugetlb_pool *hpool) { return; }

#endif /* CONFIG_DYNAMIC_HUGETLB */



static inline spinlock_t *huge_pte_lock(struct hstate *h,

					struct mm_struct *mm, pte_t *pte)

{

include/linux/memcontrol.h

+15 −0
Original line number Diff line number Diff line
@@ -326,6 +326,7 @@ struct mem_cgroup { 
};



struct mem_cgroup_extension {

	struct dhugetlb_pool *hpool;

#ifdef CONFIG_MEMCG_QOS

	/* Currently support 0 and -1.

	 * in the future it can expand to other value.
@@ -1406,4 +1407,18 @@ static inline void memcg_put_cache_ids(void) 


#endif /* CONFIG_MEMCG_KMEM */



#ifdef CONFIG_DYNAMIC_HUGETLB

struct dhugetlb_pool *get_dhugetlb_pool_from_memcg(struct mem_cgroup *memcg);

struct page *alloc_page_from_dhugetlb_pool(gfp_t gfp_mask);

void free_page_to_dhugetlb_pool(struct page *page);

int dhugetlb_pool_force_empty(struct mem_cgroup *memcg);

bool dhugetlb_pool_is_free(struct cgroup_subsys_state *css);

#else

static inline struct page *alloc_page_from_dhugetlb_pool(gfp_t gfp_mask)

{

	return NULL;

}

static inline void free_page_to_dhugetlb_pool(struct page *page) {}

#endif



#endif /* _LINUX_MEMCONTROL_H */