Commit 5b9ed9fb authored by Zi Yan's avatar Zi Yan Committed by Tong Tiangen
Browse files

memory tiering: introduce folio_use_access_time() check 

mainline inclusion
from mainline-v6.12-rc1
commit 2a28713a67fd28eedc13b32300df6b9c5a2381f5
category: bugfix
bugzilla: https://gitee.com/openeuler/kernel/issues/IB8EOE

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=2a28713a67fd28eedc13b32300df6b9c5a2381f5

--------------------------------

If memory tiering mode is on and a folio is not in the top tier memory,
folio's cpupid field is repurposed to store page access time.  Instead of
an open coded check, use a function to encapsulate the check.

Link: https://lkml.kernel.org/r/20240724130115.793641-3-ziy@nvidia.com


Signed-off-by: default avatarZi Yan <ziy@nvidia.com>
Reviewed-by: default avatar"Huang, Ying" <ying.huang@intel.com>
Acked-by: default avatarDavid Hildenbrand <david@redhat.com>
Reviewed-by: default avatarKefeng Wang <wangkefeng.wang@huawei.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Conflicts:
	mm/huge_memory.c
[context conflicts, not affect.]
Signed-off-by: default avatarTong Tiangen <tongtiangen@huawei.com>
parent b7fb00d3

include/linux/mm.h

+6 −0
Original line number Diff line number Diff line
@@ -1774,6 +1774,8 @@ static inline void vma_set_access_pid_bit(struct vm_area_struct *vma) 
		__set_bit(pid_bit, &vma->numab_state->access_pids[1]);

	}

}



bool folio_use_access_time(struct folio *folio);

#else /* !CONFIG_NUMA_BALANCING */

static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid)

{
@@ -1827,6 +1829,10 @@ static inline bool cpupid_match_pid(struct task_struct *task, int cpupid) 
static inline void vma_set_access_pid_bit(struct vm_area_struct *vma)

{

}

static inline bool folio_use_access_time(struct folio *folio)

{

	return false;

}

#endif /* CONFIG_NUMA_BALANCING */



#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)

kernel/sched/fair.c

+1 −2
Original line number Diff line number Diff line
@@ -1999,8 +1999,7 @@ bool should_numa_migrate_memory(struct task_struct *p, struct folio *folio, 
	 * The pages in slow memory node should be migrated according

	 * to hot/cold instead of private/shared.

	 */

	if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&

	    !node_is_toptier(src_nid)) {

	if (folio_use_access_time(folio)) {

		struct pglist_data *pgdat;

		unsigned long rate_limit;

		unsigned int latency, th, def_th;

mm/huge_memory.c

+2 −4
Original line number Diff line number Diff line
@@ -2033,8 +2033,7 @@ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) 
	 * For memory tiering mode, cpupid of slow memory page is used

	 * to record page access time.  So use default value.

	 */

	if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) ||

	    node_is_toptier(nid))

	if (!folio_use_access_time(folio))

		last_cpupid = folio_last_cpupid(folio);

	target_nid = numa_migrate_prep(folio, vma, haddr, nid, &flags);

	if (target_nid == NUMA_NO_NODE) {
@@ -2384,8 +2383,7 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 
		    toptier)

			goto unlock;



		if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&

		    !toptier)

		if (folio_use_access_time(folio))

			folio_xchg_access_time(folio,

					       jiffies_to_msecs(jiffies));

	}

mm/memory-tiers.c

+19 −0
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ 
#include <linux/memory.h>

#include <linux/memory-tiers.h>

#include <linux/notifier.h>

#include <linux/sched/sysctl.h>



#include "internal.h"
@@ -44,6 +45,24 @@ static struct bus_type memory_tier_subsys = { 
	.dev_name = "memory_tier",

};



#ifdef CONFIG_NUMA_BALANCING

/**

 * folio_use_access_time - check if a folio reuses cpupid for page access time

 * @folio: folio to check

 *

 * folio's _last_cpupid field is repurposed by memory tiering. In memory

 * tiering mode, cpupid of slow memory folio (not toptier memory) is used to

 * record page access time.

 *

 * Return: the folio _last_cpupid is used to record page access time

 */

bool folio_use_access_time(struct folio *folio)

{

	return (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&

	       !node_is_toptier(folio_nid(folio));

}

#endif



#ifdef CONFIG_MIGRATION

static int top_tier_adistance;

/*

mm/memory.c

+1 −2
Original line number Diff line number Diff line
@@ -5328,8 +5328,7 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) 
	 * For memory tiering mode, cpupid of slow memory page is used

	 * to record page access time.  So use default value.

	 */

	if ((sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&

	    !node_is_toptier(nid))

	if (folio_use_access_time(folio))

		last_cpupid = (-1 & LAST_CPUPID_MASK);

	else

		last_cpupid = folio_last_cpupid(folio);