Unverified Commit 13706c95 authored by openeuler-ci-bot's avatar openeuler-ci-bot Committed by Gitee
Browse files

!11503 mm: mTHP user controls to pagecache large folio 

Merge Pull Request from: @ci-robot 
 
PR sync from: Liu Shixin <liushixin2@huawei.com>
https://mailweb.openeuler.org/hyperkitty/list/kernel@openeuler.org/message/E6P2WX7CJTP7WYDLTAT76VF4EXVQDYNN/ 
Liu Shixin (2):
  mm/huge_memory: mTHP user controls to pagecache large folio
  mm/huge_memory: allow to enable 64K anouymous mapping align alone


-- 
2.34.1
 
https://gitee.com/openeuler/kernel/issues/IAKLTF 
 
Link:https://gitee.com/openeuler/kernel/pulls/11503

 

Reviewed-by: default avatarKefeng Wang <wangkefeng.wang@huawei.com>
Signed-off-by: default avatarZhang Peng <zhangpeng362@huawei.com>
parents 63fb4139 dc801cc2

Documentation/admin-guide/mm/transhuge.rst

+10 −6
Original line number Diff line number Diff line
@@ -211,12 +211,10 @@ possible to enable/disable it by configurate the corresponding bit:: 
	echo 0x2 >/sys/kernel/mm/transparent_hugepage/thp_exec_enabled

	echo 0x3 >/sys/kernel/mm/transparent_hugepage/thp_exec_enabled



The kernel could try to enable mappings for different sizes, eg, 64K on

arm64, BIT0 for file mapping, BIT1 for anonymous mapping, and THP size

page, BIT2 for anonymous mapping, where 2M anonymous mapping for arm64

is dependent on BIT2 being turned on, the above feature are disabled by

default, and could enable the above feature by writing the corresponding

bit to 1::

The kernel could try to enable mappings for different sizes, BIT0 for

64K file mapping, BIT1 for 64K anonymous mapping, and BIT2 for PMD size

anonymous mapping, the above feature are disabled by default, and could

enable the above feature by writing the corresponding bit to 1::



	echo 0x1 >/sys/kernel/mm/transparent_hugepage/thp_mapping_align

	echo 0x4 >/sys/kernel/mm/transparent_hugepage/thp_mapping_align
@@ -232,6 +230,12 @@ it back by writing 0:: 
        echo 0 >/sys/kernel/mm/transparent_hugepage/pcp_allow_high_order

        echo 4 >/sys/kernel/mm/transparent_hugepage/pcp_allow_high_order



The kernel could enable or disable file-backed hugepages, which has no

effect on existed pagecache::



	echo always >/sys/kernel/mm/transparent_hugepage/file_enabled

	echo never >/sys/kernel/mm/transparent_hugepage/file_enabled



khugepaged will be automatically started when PMD-sized THP is enabled

(either of the per-size anon control or the top-level control are set

to "always" or "madvise"), and it'll be automatically shutdown when

include/linux/huge_mm.h

+4 −3
Original line number Diff line number Diff line
@@ -51,6 +51,7 @@ enum transparent_hugepage_flag { 
	TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,

	TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG,

	TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG,

	TRANSPARENT_HUGEPAGE_FILE_MTHP_FLAG,

	TRANSPARENT_HUGEPAGE_FILE_EXEC_THP_FLAG,

	TRANSPARENT_HUGEPAGE_FILE_EXEC_MTHP_FLAG,

	TRANSPARENT_HUGEPAGE_FILE_MAPPING_ALIGN_FLAG,
@@ -308,9 +309,9 @@ static inline void count_mthp_stat(int order, enum mthp_stat_item item) 
	(transparent_hugepage_flags &					\

	 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))



#define thp_anon_mapping_pmd_align()				\

#define file_mthp_enabled()					\

	(transparent_hugepage_flags &				\

	 (1<<TRANSPARENT_HUGEPAGE_ANON_MAPPING_PMD_ALIGN_FLAG))

	 (1<<TRANSPARENT_HUGEPAGE_FILE_MTHP_FLAG))



unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,

		unsigned long len, unsigned long pgoff, unsigned long flags);
@@ -465,7 +466,7 @@ static inline void folio_prep_large_rmappable(struct folio *folio) {} 


#define transparent_hugepage_flags 0UL



#define thp_anon_mapping_pmd_align()	NULL

#define file_mthp_enabled()	false



#define thp_get_unmapped_area	NULL

mm/filemap.c

+2 −0
Original line number Diff line number Diff line
@@ -1932,6 +1932,8 @@ struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index, 


		if (!mapping_large_folio_support(mapping))

			order = 0;

		if (order && !file_mthp_enabled())

			order = 0;

		if (order && mm_in_dynamic_pool(current->mm))

			order = 0;

		if (order > MAX_PAGECACHE_ORDER)

mm/huge_memory.c

+65 −53
Original line number Diff line number Diff line
@@ -65,7 +65,8 @@ unsigned long transparent_hugepage_flags __read_mostly = 
#endif

	(1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)|

	(1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)|

	(1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);

	(1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG)|

	(1<<TRANSPARENT_HUGEPAGE_FILE_MTHP_FLAG);



static struct shrinker deferred_split_shrinker;
@@ -489,6 +490,40 @@ static void thp_flag_set(enum transparent_hugepage_flag flag, bool enable) 
		clear_bit(flag, &transparent_hugepage_flags);

}



static ssize_t file_enabled_show(struct kobject *kobj,

				 struct kobj_attribute *attr, char *buf)

{

	const char *output;



	if (test_bit(TRANSPARENT_HUGEPAGE_FILE_MTHP_FLAG,

		     &transparent_hugepage_flags))

		output = "[always] never";

	else

		output = "always [never]";



	return sysfs_emit(buf, "%s\n", output);

}



static ssize_t file_enabled_store(struct kobject *kobj,

				  struct kobj_attribute *attr,

				  const char *buf, size_t count)

{

	ssize_t ret = count;



	if (sysfs_streq(buf, "always")) {

		set_bit(TRANSPARENT_HUGEPAGE_FILE_MTHP_FLAG,

			&transparent_hugepage_flags);

	} else if (sysfs_streq(buf, "never")) {

		clear_bit(TRANSPARENT_HUGEPAGE_FILE_MTHP_FLAG,

			  &transparent_hugepage_flags);

	} else

		ret = -EINVAL;



	return ret;

}



static struct kobj_attribute file_enabled_attr = __ATTR_RW(file_enabled);



static ssize_t thp_exec_enabled_show(struct kobject *kobj,

				     struct kobj_attribute *attr, char *buf)

{
@@ -567,8 +602,7 @@ static ssize_t thp_mapping_align_store(struct kobject *kobj, 
	ret = kstrtoul(buf, 16, &val);

	if (ret < 0)

		return ret;

	if ((val & ~THP_MAPPING_ALIGN_ALL) || (!(val & ANON_MAPPING_PMD_ALIGN) &&

	    (val & ANON_MAPPING_ALIGN)))

	if (val & ~THP_MAPPING_ALIGN_ALL)

		return -EINVAL;



	thp_flag_set(TRANSPARENT_HUGEPAGE_FILE_MAPPING_ALIGN_FLAG,
@@ -592,6 +626,7 @@ static struct attribute *hugepage_attr[] = { 
#ifdef CONFIG_SHMEM

	&shmem_enabled_attr.attr,

#endif

	&file_enabled_attr.attr,

	&thp_exec_enabled_attr.attr,

	&thp_mapping_align_attr.attr,

	NULL,
@@ -1052,64 +1087,31 @@ static unsigned long __thp_get_unmapped_area(struct file *filp, 
	return ret;

}



#define thp_file_mapping_align_enabled()			\

#define mthp_file_mapping_align()				\

	(transparent_hugepage_flags &				\

	 (1<<TRANSPARENT_HUGEPAGE_FILE_MAPPING_ALIGN_FLAG))



#define thp_anon_mapping_align_enabled()			\

#define mthp_anon_mapping_align()			\

	(transparent_hugepage_flags &				\

	 (1<<TRANSPARENT_HUGEPAGE_ANON_MAPPING_ALIGN_FLAG))



static bool file_mapping_align_enabled(struct file *filp)

#define thp_anon_mapping_align()				\

	(transparent_hugepage_flags &				\

	 (1<<TRANSPARENT_HUGEPAGE_ANON_MAPPING_PMD_ALIGN_FLAG))



static bool mthp_mapping_align_enabled(struct file *filp)

{

	struct address_space *mapping;



	if (!thp_file_mapping_align_enabled())

		return false;



	if (!filp)

		return false;

		return mthp_anon_mapping_align();



	mapping = filp->f_mapping;

	if (!mapping || !mapping_large_folio_support(mapping))

	if (!mthp_file_mapping_align())

		return false;



	return true;

}



static bool anon_mapping_align_enabled(int order)

{

	unsigned long mask;



	if (!thp_anon_mapping_align_enabled())

		return 0;



	mask = READ_ONCE(huge_anon_orders_always) |

	       READ_ONCE(huge_anon_orders_madvise);



	if (hugepage_global_enabled())

		mask |= READ_ONCE(huge_anon_orders_inherit);



	mask = BIT(order) & mask;

	if (!mask)

		return false;



	return true;

}



static unsigned long folio_get_unmapped_area(struct file *filp, unsigned long addr,

		unsigned long len, unsigned long pgoff, unsigned long flags)

{

	int order = arch_wants_exec_folio_order();



	if (order < 0)

		return 0;

	mapping = filp->f_mapping;



	if (file_mapping_align_enabled(filp) ||

		(!filp && anon_mapping_align_enabled(order)))

		return __thp_get_unmapped_area(filp, addr, len, pgoff, flags,

					       PAGE_SIZE << order);

	return 0;

	return mapping && mapping_large_folio_support(mapping);

}



unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
@@ -1118,13 +1120,23 @@ unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, 
	unsigned long ret;

	loff_t off = (loff_t)pgoff << PAGE_SHIFT;



	ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE);

	if (filp || thp_anon_mapping_align()) {

		ret = __thp_get_unmapped_area(filp, addr, len, off, flags,

					      PMD_SIZE);

		if (ret)

			return ret;

	}



	ret = folio_get_unmapped_area(filp, addr, len, off, flags);

	if (mthp_mapping_align_enabled(filp)) {

		int order = arch_wants_exec_folio_order();



		if (order >= 0) {

			ret = __thp_get_unmapped_area(filp, addr, len, off,

					flags, PAGE_SIZE << order);

			if (ret)

				return ret;

		}

	}



	return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);

}

mm/mmap.c

+1 −2
Original line number Diff line number Diff line
@@ -1859,8 +1859,7 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, 
		 * so use shmem's get_unmapped_area in case it can be huge.

		 */

		get_area = shmem_get_unmapped_area;

	} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&

		   thp_anon_mapping_pmd_align()) {

	} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {

		/* Ensures that larger anonymous mappings are THP aligned. */

		get_area = thp_get_unmapped_area;

	}