Commit 08f7407a authored by Kefeng Wang's avatar Kefeng Wang Committed by Ze Zuo
Browse files

mm: add thp anon pmd size mapping align control 

hulk inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/I9R7N8


CVE: NA

-------------------------------------------------

After committing df0ad057 ('mm: align larger anonymous mappings on THP
boundaries'), we found a number of issues, eg, a significant increase
in memory access latency on arm64, which was addressed by commit
bb713c2a (arm64: mmap: disable align larger anonymous mappings on THP
boundaries), but subsequently found on x86 that the patch also resulted in
a significant degradation of the SPECCPU test. Commit 44984691('mm:
huge_memory: add thp mapping align control') has implemented the sysfs
interface control of 64 KB aligned mapping in arm64, based on this for
thp_align_mapping to add a new bit to control pmd size page alignment, where
the 64K alignment anouymous mapping enable for arm64 will depend on the new
bits being turned on. it is disabled by default, and could be enabled by
/sys/kernel/mm/transparent_hugepage/thp_mapping_align.

Signed-off-by: default avatarKefeng Wang <wangkefeng.wang@huawei.com>
Signed-off-by: default avatarZe Zuo <zuoze1@huawei.com>
parent b51533d4

Documentation/admin-guide/mm/transhuge.rst

+9 −6
Original line number Diff line number Diff line
@@ -211,14 +211,17 @@ possible to enable/disable it by configurate the corresponding bit:: 
	echo 0x2 >/sys/kernel/mm/transparent_hugepage/thp_exec_enabled

	echo 0x3 >/sys/kernel/mm/transparent_hugepage/thp_exec_enabled



The kernel could try to enable other larger size mappings align other

than THP size, eg, 64K on arm64, BIT0 for file mapping, BIT1 for anon

mapping, it is disabled by default, and could enable this feature by

writing the corresponding bit to 1::

The kernel could try to enable mappings for different sizes, eg, 64K on

arm64, BIT0 for file mapping, BIT1 for anonymous mapping, and THP size

page, BIT3 for anonymous mapping, where 64K anonymous mapping for arm64

is dependent on BIT3 being turned on, the above feature are disabled by

default, and could enable the above feature by writing the corresponding

bit to 1::



	echo 0x1 >/sys/kernel/mm/transparent_hugepage/thp_mapping_align

	echo 0x2 >/sys/kernel/mm/transparent_hugepage/thp_mapping_align

	echo 0x3 >/sys/kernel/mm/transparent_hugepage/thp_mapping_align

	echo 0x4 >/sys/kernel/mm/transparent_hugepage/thp_mapping_align

	echo 0x6 >/sys/kernel/mm/transparent_hugepage/thp_mapping_align





The kernel could enable high-orders(greated than PAGE_ALLOC_COSTLY_ORDER, only

support order 4 for now) be stored on PCP lists(except PMD order), which could

include/linux/huge_mm.h

+7 −0
Original line number Diff line number Diff line
@@ -54,6 +54,7 @@ enum transparent_hugepage_flag { 
	TRANSPARENT_HUGEPAGE_FILE_EXEC_MTHP_FLAG,

	TRANSPARENT_HUGEPAGE_FILE_MAPPING_ALIGN_FLAG,

	TRANSPARENT_HUGEPAGE_ANON_MAPPING_ALIGN_FLAG,

	TRANSPARENT_HUGEPAGE_ANON_MAPPING_PMD_ALIGN_FLAG,

};



struct kobject;
@@ -288,6 +289,10 @@ static inline void count_mthp_stat(int order, enum mthp_stat_item item) 
	(transparent_hugepage_flags &					\

	 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))



#define thp_anon_mapping_pmd_align()				\

	(transparent_hugepage_flags &				\

	 (1<<TRANSPARENT_HUGEPAGE_ANON_MAPPING_PMD_ALIGN_FLAG))



unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,

		unsigned long len, unsigned long pgoff, unsigned long flags);
@@ -443,6 +448,8 @@ static inline void folio_prep_large_rmappable(struct folio *folio) {} 


#define transparent_hugepage_flags 0UL



#define thp_anon_mapping_pmd_align()	NULL



#define thp_get_unmapped_area	NULL



static inline bool

mm/huge_memory.c

+11 −2
Original line number Diff line number Diff line
@@ -518,7 +518,9 @@ static struct kobj_attribute thp_exec_enabled_attr = 


#define FILE_MAPPING_ALIGN	BIT(0)

#define ANON_MAPPING_ALIGN	BIT(1)

#define THP_MAPPING_ALIGN_ALL	(FILE_MAPPING_ALIGN | ANON_MAPPING_ALIGN)

#define ANON_MAPPING_PMD_ALIGN	BIT(2)

#define THP_MAPPING_ALIGN_ALL		\

	(FILE_MAPPING_ALIGN | ANON_MAPPING_ALIGN | ANON_MAPPING_PMD_ALIGN)



static ssize_t thp_mapping_align_show(struct kobject *kobj,

				      struct kobj_attribute *attr, char *buf)
@@ -533,6 +535,10 @@ static ssize_t thp_mapping_align_show(struct kobject *kobj, 
		     &transparent_hugepage_flags))

		val |= ANON_MAPPING_ALIGN;



	if (test_bit(TRANSPARENT_HUGEPAGE_ANON_MAPPING_PMD_ALIGN_FLAG,

		     &transparent_hugepage_flags))

		val |= ANON_MAPPING_PMD_ALIGN;



	return sysfs_emit(buf, "0x%lx\n", val);

}

static ssize_t thp_mapping_align_store(struct kobject *kobj,
@@ -544,13 +550,16 @@ static ssize_t thp_mapping_align_store(struct kobject *kobj, 
	ret = kstrtoul(buf, 16, &val);

	if (ret < 0)

		return ret;

	if (val & ~THP_MAPPING_ALIGN_ALL)

	if ((val & ~THP_MAPPING_ALIGN_ALL) || (!(val & ANON_MAPPING_PMD_ALIGN) &&

	    (val & ANON_MAPPING_ALIGN)))

		return -EINVAL;



	thp_flag_set(TRANSPARENT_HUGEPAGE_FILE_MAPPING_ALIGN_FLAG,

		     val & FILE_MAPPING_ALIGN);

	thp_flag_set(TRANSPARENT_HUGEPAGE_ANON_MAPPING_ALIGN_FLAG,

		     val & ANON_MAPPING_ALIGN);

	thp_flag_set(TRANSPARENT_HUGEPAGE_ANON_MAPPING_PMD_ALIGN_FLAG,

		     val & ANON_MAPPING_PMD_ALIGN);



	return count;

}

mm/mmap.c

+1 −1
Original line number Diff line number Diff line
@@ -1857,7 +1857,7 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, 
		 */

		get_area = shmem_get_unmapped_area;

	} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&

		   !IS_ENABLED(CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT)) {

		   thp_anon_mapping_pmd_align()) {

		/* Ensures that larger anonymous mappings are THP aligned. */

		get_area = thp_get_unmapped_area;

	}