Commit eb315a23 authored by Liu Shixin's avatar Liu Shixin
Browse files

mm/dynamic_pool: support to flow pages between 2M and 4K pages pool 

hulk inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/I8S9BY


CVE: NA

--------------------------------

Support to demote pages in 2M pages pool to 4K size.
Support to promote pages in 4K pages pool to 2M size.

Signed-off-by: default avatarLiu Shixin <liushixin2@huawei.com>
parent fa6dcb77

mm/dynamic_pool.c

+92 −0
Original line number Diff line number Diff line
@@ -134,6 +134,34 @@ static int dpool_demote_gigantic_page(struct pages_pool *src_pool, 
	return 0;

}



static int dpool_demote_huge_page(struct pages_pool *src_pool,

				  struct pages_pool *dst_pool,

				  struct page *page)

{

	struct folio *folio = page_folio(page);

	int nr_pages = 1 << PMD_ORDER;

	struct page *subpage;

	int i;



	if (PageHWPoison(page))

		return -EHWPOISON;



	list_del(&page->lru);

	__ClearPageDpool(page);

	src_pool->free_pages--;



	clear_compound_page(page_folio(page), PMD_ORDER);

	for (i = 0; i < nr_pages; i++) {

		subpage = folio_page(folio, i);

		free_pages_prepare(subpage, 0, 0);

		__SetPageDpool(subpage);

		list_add_tail(&subpage->lru, &dst_pool->freelist);

		dst_pool->free_pages++;

	}



	return 0;

}



static int dpool_demote_pool_locked(struct dynamic_pool *dpool, int type)

{

	struct pages_pool *src_pool, *dst_pool;
@@ -161,6 +189,9 @@ static int dpool_demote_pool_locked(struct dynamic_pool *dpool, int type) 
		case PAGES_POOL_1G:

			ret = dpool_demote_gigantic_page(src_pool, dst_pool, page);

			break;

		case PAGES_POOL_2M:

			ret = dpool_demote_huge_page(src_pool, dst_pool, page);

			break;

		default:

			BUG();

		}
@@ -218,6 +249,43 @@ static int dpool_promote_gigantic_page(struct pages_pool *src_pool, 
	return 0;

}



static int dpool_promote_huge_page(struct pages_pool *src_pool,

				   struct pages_pool *dst_pool,

				   struct split_page *spage)

{

	struct hstate *h = size_to_hstate(PMD_SIZE);

	int nr_pages = 1 << PMD_ORDER;

	struct page *page, *subpage;

	int i;



	for (i = 0; i < nr_pages; i++) {

		subpage = pfn_to_page(spage->start_pfn + i);

		if (!PageDpool(subpage))

			return -EBUSY;



		if (PageHWPoison(subpage))

			return -EHWPOISON;

	}



	for (i = 0; i < nr_pages; i++) {

		subpage = pfn_to_page(spage->start_pfn + i);

		__ClearPageDpool(subpage);

		list_del(&subpage->lru);

		src_pool->free_pages--;

	}



	page = pfn_to_page(spage->start_pfn);

	prep_new_page(page, PMD_ORDER, __GFP_COMP, 0);

	set_page_count(page, 0);

	folio_change_private(page_folio(page), NULL);

	__SetPageDpool(page);

	__prep_new_hugetlb_folio(h, page_folio(page));

	list_add_tail(&page->lru, &dst_pool->freelist);

	dst_pool->free_pages++;



	return 0;

}



static int dpool_promote_pool(struct dynamic_pool *dpool, int type)

{

	struct pages_pool *src_pool, *dst_pool;
@@ -241,6 +309,18 @@ static int dpool_promote_pool(struct dynamic_pool *dpool, int type) 
		case PAGES_POOL_1G:

			ret = dpool_promote_gigantic_page(src_pool, dst_pool, spage);

			break;

		case PAGES_POOL_2M: {

			/*

			 * Since the dpool_mutex is already locked,

			 * there is no way to free spage_next, so

			 * it is safe to unlock here.

			 */

			spin_unlock(&dpool->lock);

			cond_resched();

			spin_lock(&dpool->lock);

			ret = dpool_promote_huge_page(src_pool, dst_pool, spage);

			break;

		}

		default:

			BUG();

		}
@@ -749,6 +829,18 @@ int dynamic_pool_reserve_hugepage(struct mem_cgroup *memcg, 
		while (delta > pool->free_pages &&

		       !dpool_demote_pool_locked(dpool, type - 1))

			cond_resched_lock(&dpool->lock);

		/* Only try merge pages for 2M pages */

		if (type == PAGES_POOL_2M) {

			while (delta > pool->free_pages) {

				spin_unlock(&dpool->lock);

				cond_resched();

				if (dpool_promote_pool(dpool, type)) {

					spin_lock(&dpool->lock);

					break;

				}

				spin_lock(&dpool->lock);

			}

		}

		delta = min(delta, pool->free_pages);

		pool->nr_huge_pages += delta;

		pool->free_huge_pages += delta;

mm/internal.h

+5 −0
Original line number Diff line number Diff line
@@ -434,10 +434,15 @@ static inline void prep_compound_tail(struct page *head, int tail_idx) 
	set_page_private(p, 0);

}



typedef int __bitwise fpi_t;

extern void prep_compound_page(struct page *page, unsigned int order);



extern void post_alloc_hook(struct page *page, unsigned int order,

					gfp_t gfp_flags);

extern bool free_pages_prepare(struct page *page, unsigned int order,

			       fpi_t fpi_flags);

extern void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,

			  unsigned int alloc_flags);

extern int user_min_free_kbytes;



extern void free_unref_page(struct page *page, unsigned int order);

mm/page_alloc.c

+3 −3
Original line number Diff line number Diff line
@@ -1073,7 +1073,7 @@ static void kernel_init_pages(struct page *page, int numpages) 
	kasan_enable_current();

}



static __always_inline bool free_pages_prepare(struct page *page,

__always_inline bool free_pages_prepare(struct page *page,

			unsigned int order, fpi_t fpi_flags)

{

	int bad = 0;
@@ -1538,7 +1538,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order, 
	page_table_check_alloc(page, order);

}



static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,

void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,

						unsigned int alloc_flags)

{

	post_alloc_hook(page, order, gfp_flags);