Commit 286201c8 authored by Yosry Ahmed's avatar Yosry Ahmed Committed by Liu Shixin
Browse files

mm: rmap: abstract updating per-node and per-memcg stats 

mainline inclusion
from mainline-v6.11-rc1
commit 15c0536fb57fd989e24335020a443486bac01dac
category: cleanup
bugzilla: https://gitee.com/openeuler/kernel/issues/IAJ5MT

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=15c0536fb57fd989e24335020a443486bac01dac

--------------------------------

A lot of intricacies go into updating the stats when adding or removing
mappings: which stat index to use and which function.  Abstract this away
into a new static helper in rmap.c, __folio_mod_stat().

This adds an unnecessary call to folio_test_anon() in
__folio_add_anon_rmap() and __folio_add_file_rmap().  However, the folio
struct should already be in the cache at this point, so it shouldn't cause
any noticeable overhead.

No functional change intended.

[hughd@google.com: fix /proc/meminfo]
  Link: https://lkml.kernel.org/r/49914517-dfc7-e784-fde0-0e08fafbecc2@google.com
Link: https://lkml.kernel.org/r/20240506211333.346605-1-yosryahmed@google.com


Signed-off-by: default avatarYosry Ahmed <yosryahmed@google.com>
Signed-off-by: default avatarHugh Dickins <hughd@google.com>
Reviewed-by: default avatarDavid Hildenbrand <david@redhat.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Conflicts:
	mm/rmap.c
[ Conflicts due to miss commit 4f687281012e which is bugfix for another
  feature ]
Signed-off-by: default avatarLiu Shixin <liushixin2@huawei.com>
parent d081bee4

mm/rmap.c

+28 −24
Original line number Diff line number Diff line
@@ -1255,6 +1255,27 @@ static void __page_check_anon_rmap(struct folio *folio, struct page *page, 
		       page);

}



static void __folio_mod_stat(struct folio *folio, int nr, int nr_pmdmapped)

{

	int idx;



	if (nr) {

		idx = folio_test_anon(folio) ? NR_ANON_MAPPED : NR_FILE_MAPPED;

		__lruvec_stat_mod_folio(folio, idx, nr);

	}

	if (nr_pmdmapped) {

		if (folio_test_anon(folio)) {

			idx = NR_ANON_THPS;

			__lruvec_stat_mod_folio(folio, idx, nr_pmdmapped);

		} else {

			/* NR_*_PMDMAPPED are not maintained per-memcg */

			idx = folio_test_swapbacked(folio) ?

				NR_SHMEM_PMDMAPPED : NR_FILE_PMDMAPPED;

			__lruvec_stat_mod_folio(folio, idx, nr_pmdmapped);

		}

	}

}



static __always_inline void __folio_add_anon_rmap(struct folio *folio,

		struct page *page, int nr_pages, struct vm_area_struct *vma,

		unsigned long address, rmap_t flags, enum rmap_level level)
@@ -1262,10 +1283,6 @@ static __always_inline void __folio_add_anon_rmap(struct folio *folio, 
	int i, nr, nr_pmdmapped = 0;



	nr = __folio_add_rmap(folio, page, nr_pages, level, &nr_pmdmapped);

	if (nr_pmdmapped)

		__lruvec_stat_mod_folio(folio, NR_ANON_THPS, nr_pmdmapped);

	if (nr)

		__lruvec_stat_mod_folio(folio, NR_ANON_MAPPED, nr);



	if (unlikely(!folio_test_anon(folio))) {

		VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
@@ -1283,6 +1300,8 @@ static __always_inline void __folio_add_anon_rmap(struct folio *folio, 
		__page_check_anon_rmap(folio, page, vma, address);

	}



	__folio_mod_stat(folio, nr, nr_pmdmapped);



	if (flags & RMAP_EXCLUSIVE) {

		switch (level) {

		case RMAP_LEVEL_PTE:
@@ -1379,6 +1398,7 @@ void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma, 
		unsigned long address)

{

	int nr = folio_nr_pages(folio);

	int nr_pmdmapped = 0;



	VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio);

	VM_BUG_ON_VMA(address < vma->vm_start ||
@@ -1407,10 +1427,10 @@ void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma, 
		atomic_set(&folio->_entire_mapcount, 0);

		atomic_set(&folio->_nr_pages_mapped, ENTIRELY_MAPPED);

		SetPageAnonExclusive(&folio->page);

		__lruvec_stat_mod_folio(folio, NR_ANON_THPS, nr);

		nr_pmdmapped = nr;

	}



	__lruvec_stat_mod_folio(folio, NR_ANON_MAPPED, nr);

	__folio_mod_stat(folio, nr, nr_pmdmapped);

}



static __always_inline void __folio_add_file_rmap(struct folio *folio,
@@ -1422,11 +1442,7 @@ static __always_inline void __folio_add_file_rmap(struct folio *folio, 
	VM_WARN_ON_FOLIO(folio_test_anon(folio), folio);



	nr = __folio_add_rmap(folio, page, nr_pages, level, &nr_pmdmapped);

	if (nr_pmdmapped)

		__lruvec_stat_mod_folio(folio, folio_test_swapbacked(folio) ?

			NR_SHMEM_PMDMAPPED : NR_FILE_PMDMAPPED, nr_pmdmapped);

	if (nr)

		__lruvec_stat_mod_folio(folio, NR_FILE_MAPPED, nr);

	__folio_mod_stat(folio, nr, nr_pmdmapped);



	/* See comments in folio_add_anon_rmap_*() */

	if (!folio_test_large(folio))
@@ -1477,7 +1493,6 @@ static __always_inline void __folio_remove_rmap(struct folio *folio, 
	atomic_t *mapped = &folio->_nr_pages_mapped;

	int last, nr = 0, nr_pmdmapped = 0;

	bool partially_mapped = false;

	enum node_stat_item idx;



	__folio_rmap_sanity_checks(folio, page, nr_pages, level);
@@ -1519,19 +1534,7 @@ static __always_inline void __folio_remove_rmap(struct folio *folio, 
		break;

	}



	if (nr_pmdmapped) {

		if (folio_test_anon(folio))

			idx = NR_ANON_THPS;

		else if (folio_test_swapbacked(folio))

			idx = NR_SHMEM_PMDMAPPED;

		else

			idx = NR_FILE_PMDMAPPED;

		__lruvec_stat_mod_folio(folio, idx, -nr_pmdmapped);

	}

	if (nr) {

		idx = folio_test_anon(folio) ? NR_ANON_MAPPED : NR_FILE_MAPPED;

		__lruvec_stat_mod_folio(folio, idx, -nr);



		/*

		 * Queue anon large folio for deferred split if at least one

		 * page of the folio is unmapped and at least one page
@@ -1543,6 +1546,7 @@ static __always_inline void __folio_remove_rmap(struct folio *folio, 
		    list_empty(&folio->_deferred_list))

			deferred_split_folio(folio);

	}

	__folio_mod_stat(folio, -nr, -nr_pmdmapped);



	/*

	 * It would be tidy to reset folio_test_anon mapping when fully