Commit 6df1b221 authored by Yu Zhao's avatar Yu Zhao Committed by Andrew Morton
Browse files

mm: multi-gen LRU: rename lrugen->lists[] to lrugen->folios[] 

lru_gen_folio will be chained into per-node lists by the coming
lrugen->list.

Link: https://lkml.kernel.org/r/20221222041905.2431096-3-yuzhao@google.com


Signed-off-by: default avatarYu Zhao <yuzhao@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Michael Larabel <Michael@MichaelLarabel.com>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Suren Baghdasaryan <surenb@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 391655fe

Documentation/mm/multigen_lru.rst

+4 −4
Original line number Diff line number Diff line
@@ -89,15 +89,15 @@ variables are monotonically increasing. 


Generation numbers are truncated into ``order_base_2(MAX_NR_GENS+1)``

bits in order to fit into the gen counter in ``folio->flags``. Each

truncated generation number is an index to ``lrugen->lists[]``. The

truncated generation number is an index to ``lrugen->folios[]``. The

sliding window technique is used to track at least ``MIN_NR_GENS`` and

at most ``MAX_NR_GENS`` generations. The gen counter stores a value

within ``[1, MAX_NR_GENS]`` while a page is on one of

``lrugen->lists[]``; otherwise it stores zero.

``lrugen->folios[]``; otherwise it stores zero.



Each generation is divided into multiple tiers. A page accessed ``N``

times through file descriptors is in tier ``order_base_2(N)``. Unlike

generations, tiers do not have dedicated ``lrugen->lists[]``. In

generations, tiers do not have dedicated ``lrugen->folios[]``. In

contrast to moving across generations, which requires the LRU lock,

moving across tiers only involves atomic operations on

``folio->flags`` and therefore has a negligible cost. A feedback loop
@@ -127,7 +127,7 @@ page mapped by this PTE to ``(max_seq%MAX_NR_GENS)+1``. 
Eviction

--------

The eviction consumes old generations. Given an ``lruvec``, it

increments ``min_seq`` when ``lrugen->lists[]`` indexed by

increments ``min_seq`` when ``lrugen->folios[]`` indexed by

``min_seq%MAX_NR_GENS`` becomes empty. To select a type and a tier to

evict from, it first compares ``min_seq[]`` to select the older type.

If both types are equally old, it selects the one whose first tier has

include/linux/mm_inline.h

+2 −2
Original line number Diff line number Diff line
@@ -256,9 +256,9 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, 
	lru_gen_update_size(lruvec, folio, -1, gen);

	/* for folio_rotate_reclaimable() */

	if (reclaiming)

		list_add_tail(&folio->lru, &lrugen->lists[gen][type][zone]);

		list_add_tail(&folio->lru, &lrugen->folios[gen][type][zone]);

	else

		list_add(&folio->lru, &lrugen->lists[gen][type][zone]);

		list_add(&folio->lru, &lrugen->folios[gen][type][zone]);



	return true;

}

include/linux/mmzone.h

+4 −4
Original line number Diff line number Diff line
@@ -312,7 +312,7 @@ enum lruvec_flags { 
 * They form a sliding window of a variable size [MIN_NR_GENS, MAX_NR_GENS]. An

 * offset within MAX_NR_GENS, i.e., gen, indexes the LRU list of the

 * corresponding generation. The gen counter in folio->flags stores gen+1 while

 * a page is on one of lrugen->lists[]. Otherwise it stores 0.

 * a page is on one of lrugen->folios[]. Otherwise it stores 0.

 *

 * A page is added to the youngest generation on faulting. The aging needs to

 * check the accessed bit at least twice before handing this page over to the
@@ -324,8 +324,8 @@ enum lruvec_flags { 
 * rest of generations, if they exist, are considered inactive. See

 * lru_gen_is_active().

 *

 * PG_active is always cleared while a page is on one of lrugen->lists[] so that

 * the aging needs not to worry about it. And it's set again when a page

 * PG_active is always cleared while a page is on one of lrugen->folios[] so

 * that the aging needs not to worry about it. And it's set again when a page

 * considered active is isolated for non-reclaiming purposes, e.g., migration.

 * See lru_gen_add_folio() and lru_gen_del_folio().

 *
@@ -412,7 +412,7 @@ struct lru_gen_folio { 
	/* the birth time of each generation in jiffies */

	unsigned long timestamps[MAX_NR_GENS];

	/* the multi-gen LRU lists, lazily sorted on eviction */

	struct list_head lists[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];

	struct list_head folios[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];

	/* the multi-gen LRU sizes, eventually consistent */

	long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];

	/* the exponential moving average of refaulted */

mm/vmscan.c

+10 −10
Original line number Diff line number Diff line
@@ -4271,7 +4271,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap) 


	/* prevent cold/hot inversion if force_scan is true */

	for (zone = 0; zone < MAX_NR_ZONES; zone++) {

		struct list_head *head = &lrugen->lists[old_gen][type][zone];

		struct list_head *head = &lrugen->folios[old_gen][type][zone];



		while (!list_empty(head)) {

			struct folio *folio = lru_to_folio(head);
@@ -4282,7 +4282,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap) 
			VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);



			new_gen = folio_inc_gen(lruvec, folio, false);

			list_move_tail(&folio->lru, &lrugen->lists[new_gen][type][zone]);

			list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]);



			if (!--remaining)

				return false;
@@ -4310,7 +4310,7 @@ static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap) 
			gen = lru_gen_from_seq(min_seq[type]);



			for (zone = 0; zone < MAX_NR_ZONES; zone++) {

				if (!list_empty(&lrugen->lists[gen][type][zone]))

				if (!list_empty(&lrugen->folios[gen][type][zone]))

					goto next;

			}
@@ -4775,7 +4775,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx) 


	/* promoted */

	if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {

		list_move(&folio->lru, &lrugen->lists[gen][type][zone]);

		list_move(&folio->lru, &lrugen->folios[gen][type][zone]);

		return true;

	}
@@ -4784,7 +4784,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx) 
		int hist = lru_hist_from_seq(lrugen->min_seq[type]);



		gen = folio_inc_gen(lruvec, folio, false);

		list_move_tail(&folio->lru, &lrugen->lists[gen][type][zone]);

		list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);



		WRITE_ONCE(lrugen->protected[hist][type][tier - 1],

			   lrugen->protected[hist][type][tier - 1] + delta);
@@ -4796,7 +4796,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx) 
	if (folio_test_locked(folio) || folio_test_writeback(folio) ||

	    (type == LRU_GEN_FILE && folio_test_dirty(folio))) {

		gen = folio_inc_gen(lruvec, folio, true);

		list_move(&folio->lru, &lrugen->lists[gen][type][zone]);

		list_move(&folio->lru, &lrugen->folios[gen][type][zone]);

		return true;

	}
@@ -4863,7 +4863,7 @@ static int scan_folios(struct lruvec *lruvec, struct scan_control *sc, 
	for (zone = sc->reclaim_idx; zone >= 0; zone--) {

		LIST_HEAD(moved);

		int skipped = 0;

		struct list_head *head = &lrugen->lists[gen][type][zone];

		struct list_head *head = &lrugen->folios[gen][type][zone];



		while (!list_empty(head)) {

			struct folio *folio = lru_to_folio(head);
@@ -5264,7 +5264,7 @@ static bool __maybe_unused state_is_valid(struct lruvec *lruvec) 
		int gen, type, zone;



		for_each_gen_type_zone(gen, type, zone) {

			if (!list_empty(&lrugen->lists[gen][type][zone]))

			if (!list_empty(&lrugen->folios[gen][type][zone]))

				return false;

		}

	}
@@ -5309,7 +5309,7 @@ static bool drain_evictable(struct lruvec *lruvec) 
	int remaining = MAX_LRU_BATCH;



	for_each_gen_type_zone(gen, type, zone) {

		struct list_head *head = &lruvec->lrugen.lists[gen][type][zone];

		struct list_head *head = &lruvec->lrugen.folios[gen][type][zone];



		while (!list_empty(head)) {

			bool success;
@@ -5843,7 +5843,7 @@ void lru_gen_init_lruvec(struct lruvec *lruvec) 
		lrugen->timestamps[i] = jiffies;



	for_each_gen_type_zone(gen, type, zone)

		INIT_LIST_HEAD(&lrugen->lists[gen][type][zone]);

		INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]);



	lruvec->mm_state.seq = MIN_NR_GENS;

	init_waitqueue_head(&lruvec->mm_state.wait);