Commit aafd6500 authored by Wei Xu's avatar Wei Xu Committed by Kaixiong Yu
Browse files

mm/mglru: reset page lru tier bits when activating 

mainline inclusion
from mainline-v6.13-rc1
commit f1001f3d3b6868998cab73d10fda1a5c99ddf963
category: bugfix
bugzilla: https://gitee.com/openeuler/kernel/issues/IBET92

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=f1001f3d3b6868998cab73d10fda1a5c99ddf963

--------------------------------

When a folio is activated, lru_gen_add_folio() moves the folio to the
youngest generation.  But unlike folio_update_gen()/folio_inc_gen(),
lru_gen_add_folio() doesn't reset the folio lru tier bits (LRU_REFS_MASK |
LRU_REFS_FLAGS).  This inconsistency can affect how pages are aged via
folio_mark_accessed() (e.g.  fd accesses), though no user visible impact
related to this has been detected yet.

Note that lru_gen_add_folio() cannot clear PG_workingset if the activation
is due to workingset refault, otherwise PSI accounting will be skipped.
So fix lru_gen_add_folio() to clear the lru tier bits other than
PG_workingset when activating a folio, and also clear all the lru tier
bits when a folio is activated via folio_activate() in
lru_gen_look_around().

Link: https://lkml.kernel.org/r/20241017181528.3358821-1-weixugc@google.com


Fixes: 018ee47f ("mm: multi-gen LRU: exploit locality in rmap")
Signed-off-by: default avatarWei Xu <weixugc@google.com>
Cc: Axel Rasmussen <axelrasmussen@google.com>
Cc: Brian Geffon <bgeffon@google.com>
Cc: Jan Alexander Steffens <heftig@archlinux.org>
Cc: Suleiman Souhlal <suleiman@google.com>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarKaixiong Yu <yukaixiong@huawei.com>
parent ede039b6

include/linux/mm_inline.h

+14 −1
Original line number Diff line number Diff line
@@ -155,6 +155,11 @@ static inline int folio_lru_refs(struct folio *folio) 
	return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + workingset;

}



static inline void folio_clear_lru_refs(struct folio *folio)

{

	set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 0);

}



static inline int folio_lru_gen(struct folio *folio)

{

	unsigned long flags = READ_ONCE(folio->flags);
@@ -224,6 +229,7 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, 
{

	unsigned long seq;

	unsigned long flags;

	unsigned long mask;

	int gen = folio_lru_gen(folio);

	int type = folio_is_file_lru(folio);

	int zone = folio_zonenum(folio);
@@ -259,7 +265,14 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, 
	gen = lru_gen_from_seq(seq);

	flags = (gen + 1UL) << LRU_GEN_PGOFF;

	/* see the comment on MIN_NR_GENS about PG_active */

	set_mask_bits(&folio->flags, LRU_GEN_MASK | BIT(PG_active), flags);

	mask = LRU_GEN_MASK;

	/*

	 * Don't clear PG_workingset here because it can affect PSI accounting

	 * if the activation is due to workingset refault.

	 */

	if (folio_test_active(folio))

		mask |= LRU_REFS_MASK | BIT(PG_referenced) | BIT(PG_active);

	set_mask_bits(&folio->flags, mask, flags);



	lru_gen_update_size(lruvec, folio, -1, gen);

	/* for folio_rotate_reclaimable() */

include/linux/mmzone.h

+2 −0
Original line number Diff line number Diff line
@@ -404,6 +404,8 @@ enum { 
	NR_LRU_GEN_CAPS

};



#define LRU_REFS_FLAGS		(BIT(PG_referenced) | BIT(PG_workingset))



#define MIN_LRU_BATCH		BITS_PER_LONG

#define MAX_LRU_BATCH		(MIN_LRU_BATCH * 64)

mm/vmscan.c

+4 −4
Original line number Diff line number Diff line
@@ -3284,8 +3284,6 @@ static bool should_clear_pmd_young(void) 
 *                          shorthand helpers

 ******************************************************************************/



#define LRU_REFS_FLAGS	(BIT(PG_referenced) | BIT(PG_workingset))



#define DEFINE_MAX_SEQ(lruvec)						\

	unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq)
@@ -4787,9 +4785,11 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) 
		old_gen = folio_lru_gen(folio);

		if (old_gen < 0)

			folio_set_referenced(folio);

		else if (old_gen != new_gen)

		else if (old_gen != new_gen) {

			folio_clear_lru_refs(folio);

			folio_activate(folio);

		}

	}



	arch_leave_lazy_mmu_mode();

	mem_cgroup_unlock_pages();
@@ -5040,7 +5040,7 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca 


	/* see the comment on MAX_NR_TIERS */

	if (!folio_test_referenced(folio))

		set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 0);

		folio_clear_lru_refs(folio);



	/* for shrink_folio_list() */

	folio_clear_reclaim(folio);