Commit 7f989cae authored by Ryan Roberts's avatar Ryan Roberts Committed by Liu Shixin
Browse files

mm: swap: free_swap_and_cache_nr() as batched free_swap_and_cache() 

next inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/I9OCYO
CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=a62fb92ac12ed39df4930dca599a3b427552882a

--------------------------------

Now that we no longer have a convenient flag in the cluster to determine
if a folio is large, free_swap_and_cache() will take a reference and lock
a large folio much more often, which could lead to contention and (e.g.)
failure to split large folios, etc.

Let's solve that problem by batch freeing swap and cache with a new
function, free_swap_and_cache_nr(), to free a contiguous range of swap
entries together.  This allows us to first drop a reference to each swap
slot before we try to release the cache folio.  This means we only try to
release the folio once, only taking the reference and lock once - much
better than the previous 512 times for the 2M THP case.

Contiguous swap entries are gathered in zap_pte_range() and
madvise_free_pte_range() in a similar way to how present ptes are already
gathered in zap_pte_range().

While we are at it, let's simplify by converting the return type of both
functions to void.  The return value was used only by zap_pte_range() to
print a bad pte, and was ignored by everyone else, so the extra reporting
wasn't exactly guaranteed.  We will still get the warning with most of the
information from get_swap_device().  With the batch version, we wouldn't
know which pte was bad anyway so could print the wrong one.

[ryan.roberts@arm.com: fix a build warning on parisc]
  Link: https://lkml.kernel.org/r/20240409111840.3173122-1-ryan.roberts@arm.com
Link: https://lkml.kernel.org/r/20240408183946.2991168-3-ryan.roberts@arm.com


Signed-off-by: default avatarRyan Roberts <ryan.roberts@arm.com>
Acked-by: default avatarDavid Hildenbrand <david@redhat.com>
Cc: Barry Song <21cnbao@gmail.com>
Cc: Barry Song <v-songbaohua@oppo.com>
Cc: Chris Li <chrisl@kernel.org>
Cc: Gao Xiang <xiang@kernel.org>
Cc: "Huang, Ying" <ying.huang@intel.com>
Cc: Kefeng Wang <wangkefeng.wang@huawei.com>
Cc: Lance Yang <ioworker0@gmail.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Yang Shi <shy828301@gmail.com>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLiu Shixin <liushixin2@huawei.com>
parent 3a795cdd

include/linux/pgtable.h

+29 −0
Original line number Diff line number Diff line
@@ -680,6 +680,35 @@ static inline void pte_clear_not_present_full(struct mm_struct *mm, 
}

#endif



#ifndef clear_not_present_full_ptes

/**

 * clear_not_present_full_ptes - Clear multiple not present PTEs which are

 *				 consecutive in the pgtable.

 * @mm: Address space the ptes represent.

 * @addr: Address of the first pte.

 * @ptep: Page table pointer for the first entry.

 * @nr: Number of entries to clear.

 * @full: Whether we are clearing a full mm.

 *

 * May be overridden by the architecture; otherwise, implemented as a simple

 * loop over pte_clear_not_present_full().

 *

 * Context: The caller holds the page table lock.  The PTEs are all not present.

 * The PTEs are all in the same PMD.

 */

static inline void clear_not_present_full_ptes(struct mm_struct *mm,

		unsigned long addr, pte_t *ptep, unsigned int nr, int full)

{

	for (;;) {

		pte_clear_not_present_full(mm, addr, ptep, full);

		if (--nr == 0)

			break;

		ptep++;

		addr += PAGE_SIZE;

	}

}

#endif



#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH

extern pte_t ptep_clear_flush(struct vm_area_struct *vma,

			      unsigned long address,

include/linux/swap.h

+9 −3
Original line number Diff line number Diff line
@@ -502,7 +502,7 @@ extern int swap_duplicate(swp_entry_t); 
extern int swapcache_prepare(swp_entry_t);

extern void swap_free(swp_entry_t);

extern void swapcache_free_entries(swp_entry_t *entries, int n);

extern int free_swap_and_cache(swp_entry_t);

extern void free_swap_and_cache_nr(swp_entry_t entry, int nr);

int swap_type_of(dev_t device, sector_t offset);

int find_first_swap(dev_t *device);

extern unsigned int count_swap_pages(int, int);
@@ -560,8 +560,9 @@ static inline void put_swap_device(struct swap_info_struct *si) 
#define free_pages_and_swap_cache(pages, nr) \

	release_pages((pages), (nr));



/* used to sanity check ptes in zap_pte_range when CONFIG_SWAP=0 */

#define free_swap_and_cache(e) is_pfn_swap_entry(e)

static inline void free_swap_and_cache_nr(swp_entry_t entry, int nr)

{

}



static inline void free_swap_cache(struct page *page)

{
@@ -629,6 +630,11 @@ static inline int add_swap_extent(struct swap_info_struct *sis, 
}

#endif /* CONFIG_SWAP */



static inline void free_swap_and_cache(swp_entry_t entry)

{

	free_swap_and_cache_nr(entry, 1);

}



#ifdef CONFIG_MEMCG

static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg)

{

mm/internal.h

+64 −0
Original line number Diff line number Diff line
@@ -11,6 +11,8 @@ 
#include <linux/mm.h>

#include <linux/pagemap.h>

#include <linux/rmap.h>

#include <linux/swap.h>

#include <linux/swapops.h>

#include <linux/tracepoint-defs.h>



struct folio_batch;
@@ -188,6 +190,68 @@ static inline int folio_pte_batch(struct folio *folio, unsigned long addr, 


	return min(ptep - start_ptep, max_nr);

}



/**

 * pte_next_swp_offset - Increment the swap entry offset field of a swap pte.

 * @pte: The initial pte state; is_swap_pte(pte) must be true and

 *	 non_swap_entry() must be false.

 *

 * Increments the swap offset, while maintaining all other fields, including

 * swap type, and any swp pte bits. The resulting pte is returned.

 */

static inline pte_t pte_next_swp_offset(pte_t pte)

{

	swp_entry_t entry = pte_to_swp_entry(pte);

	pte_t new = __swp_entry_to_pte(__swp_entry(swp_type(entry),

						   (swp_offset(entry) + 1)));



	if (pte_swp_soft_dirty(pte))

		new = pte_swp_mksoft_dirty(new);

	if (pte_swp_exclusive(pte))

		new = pte_swp_mkexclusive(new);

	if (pte_swp_uffd_wp(pte))

		new = pte_swp_mkuffd_wp(new);



	return new;

}



/**

 * swap_pte_batch - detect a PTE batch for a set of contiguous swap entries

 * @start_ptep: Page table pointer for the first entry.

 * @max_nr: The maximum number of table entries to consider.

 * @pte: Page table entry for the first entry.

 *

 * Detect a batch of contiguous swap entries: consecutive (non-present) PTEs

 * containing swap entries all with consecutive offsets and targeting the same

 * swap type, all with matching swp pte bits.

 *

 * max_nr must be at least one and must be limited by the caller so scanning

 * cannot exceed a single page table.

 *

 * Return: the number of table entries in the batch.

 */

static inline int swap_pte_batch(pte_t *start_ptep, int max_nr, pte_t pte)

{

	pte_t expected_pte = pte_next_swp_offset(pte);

	const pte_t *end_ptep = start_ptep + max_nr;

	pte_t *ptep = start_ptep + 1;



	VM_WARN_ON(max_nr < 1);

	VM_WARN_ON(!is_swap_pte(pte));

	VM_WARN_ON(non_swap_entry(pte_to_swp_entry(pte)));



	while (ptep < end_ptep) {

		pte = ptep_get(ptep);



		if (!pte_same(pte, expected_pte))

			break;



		expected_pte = pte_next_swp_offset(expected_pte);

		ptep++;

	}



	return ptep - start_ptep;

}

#endif /* CONFIG_MMU */



void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio,

mm/madvise.c

+8 −4
Original line number Diff line number Diff line
@@ -667,6 +667,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, 
	struct folio *folio;

	int nr_swap = 0;

	unsigned long next;

	int nr, max_nr;



	next = pmd_addr_end(addr, end);

	if (pmd_trans_huge(*pmd))
@@ -679,7 +680,8 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, 
		return 0;

	flush_tlb_batched_pending(mm);

	arch_enter_lazy_mmu_mode();

	for (; addr != end; pte++, addr += PAGE_SIZE) {

	for (; addr != end; pte += nr, addr += PAGE_SIZE * nr) {

		nr = 1;

		ptent = ptep_get(pte);



		if (pte_none(ptent))
@@ -694,9 +696,11 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, 


			entry = pte_to_swp_entry(ptent);

			if (!non_swap_entry(entry)) {

				nr_swap--;

				free_swap_and_cache(entry);

				pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);

				max_nr = (end - addr) / PAGE_SIZE;

				nr = swap_pte_batch(pte, max_nr, ptent);

				nr_swap -= nr;

				free_swap_and_cache_nr(entry, nr);

				clear_not_present_full_ptes(mm, addr, pte, nr, tlb->fullmm);

			} else if (is_hwpoison_entry(entry) ||

				   is_poisoned_swp_entry(entry)) {

				pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);

mm/memory.c

+7 −6
Original line number Diff line number Diff line
@@ -1642,12 +1642,13 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, 
				folio_remove_rmap_pte(folio, page, vma);

			folio_put(folio);

		} else if (!non_swap_entry(entry)) {

			/* Genuine swap entry, hence a private anon page */

			max_nr = (end - addr) / PAGE_SIZE;

			nr = swap_pte_batch(pte, max_nr, ptent);

			/* Genuine swap entries, hence a private anon pages */

			if (!should_zap_cows(details))

				continue;

			rss[MM_SWAPENTS]--;

			if (unlikely(!free_swap_and_cache(entry)))

				print_bad_pte(vma, addr, ptent, NULL);

			rss[MM_SWAPENTS] -= nr;

			free_swap_and_cache_nr(entry, nr);

		} else if (is_migration_entry(entry)) {

			folio = pfn_swap_entry_folio(entry);

			if (!should_zap_folio(details, folio))
@@ -1672,8 +1673,8 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, 
			/* We should have covered all the swap entry types */

			WARN_ON_ONCE(1);

		}

		pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);

		zap_install_uffd_wp_if_needed(vma, addr, pte, 1, details, ptent);

		clear_not_present_full_ptes(mm, addr, pte, nr, tlb->fullmm);

		zap_install_uffd_wp_if_needed(vma, addr, pte, nr, details, ptent);

	} while (pte += nr, addr += PAGE_SIZE * nr, addr != end);



	add_mm_rss_vec(mm, rss);