Commit cfeed8ff authored by David Hildenbrand's avatar David Hildenbrand Committed by Andrew Morton
Browse files

mm/swap: stop using page->private on tail pages for THP_SWAP 

Patch series "mm/swap: stop using page->private on tail pages for THP_SWAP
+ cleanups".

This series stops using page->private on tail pages for THP_SWAP, replaces
folio->private by folio->swap for swapcache folios, and starts using
"new_folio" for tail pages that we are splitting to remove the usage of
page->private for swapcache handling completely.


This patch (of 4):

Let's stop using page->private on tail pages, making it possible to just
unconditionally reuse that field in the tail pages of large folios.

The remaining usage of the private field for THP_SWAP is in the THP
splitting code (mm/huge_memory.c), that we'll handle separately later.

Update the THP_SWAP documentation and sanity checks in mm_types.h and
__split_huge_page_tail().

[david@redhat.com: stop using page->private on tail pages for THP_SWAP]
  Link: https://lkml.kernel.org/r/6f0a82a3-6948-20d9-580b-be1dbf415701@redhat.com
Link: https://lkml.kernel.org/r/20230821160849.531668-1-david@redhat.com
Link: https://lkml.kernel.org/r/20230821160849.531668-2-david@redhat.com


Signed-off-by: default avatarDavid Hildenbrand <david@redhat.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>	[arm64]
Reviewed-by: default avatarYosry Ahmed <yosryahmed@google.com>
Cc: Dan Streetman <ddstreet@ieee.org>
Cc: Hugh Dickins <hughd@google.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Peter Xu <peterx@redhat.com>
Cc: Seth Jennings <sjenning@redhat.com>
Cc: Vitaly Wool <vitaly.wool@konsulko.com>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent bad5a3a4

arch/arm64/mm/mteswap.c

+3 −2
Original line number Diff line number Diff line
@@ -33,8 +33,9 @@ int mte_save_tags(struct page *page) 


	mte_save_page_tags(page_address(page), tag_storage);



	/* page_private contains the swap entry.val set in do_swap_page */

	ret = xa_store(&mte_pages, page_private(page), tag_storage, GFP_KERNEL);

	/* lookup the swap entry.val from the page */

	ret = xa_store(&mte_pages, page_swap_entry(page).val, tag_storage,

		       GFP_KERNEL);

	if (WARN(xa_is_err(ret), "Failed to store MTE tags")) {

		mte_free_tag_storage(tag_storage);

		return xa_err(ret);

include/linux/mm_types.h

+1 −11
Original line number Diff line number Diff line
@@ -322,11 +322,8 @@ struct folio { 
			atomic_t _pincount;

#ifdef CONFIG_64BIT

			unsigned int _folio_nr_pages;

			/* 4 byte gap here */

	/* private: the union with struct page is transitional */

			/* Fix THP_SWAP to not use tail->private */

			unsigned long _private_1;

#endif

	/* private: the union with struct page is transitional */

		};

		struct page __page_1;

	};
@@ -347,9 +344,6 @@ struct folio { 
	/* public: */

			struct list_head _deferred_list;

	/* private: the union with struct page is transitional */

			unsigned long _avail_2a;

			/* Fix THP_SWAP to not use tail->private */

			unsigned long _private_2a;

		};

		struct page __page_2;

	};
@@ -374,9 +368,6 @@ FOLIO_MATCH(memcg_data, memcg_data); 
			offsetof(struct page, pg) + sizeof(struct page))

FOLIO_MATCH(flags, _flags_1);

FOLIO_MATCH(compound_head, _head_1);

#ifdef CONFIG_64BIT

FOLIO_MATCH(private, _private_1);

#endif

#undef FOLIO_MATCH

#define FOLIO_MATCH(pg, fl)						\

	static_assert(offsetof(struct folio, fl) ==			\
@@ -385,7 +376,6 @@ FOLIO_MATCH(flags, _flags_2); 
FOLIO_MATCH(compound_head, _head_2);

FOLIO_MATCH(flags, _flags_2a);

FOLIO_MATCH(compound_head, _head_2a);

FOLIO_MATCH(private, _private_2a);

#undef FOLIO_MATCH



/**

include/linux/swap.h

+9 −0
Original line number Diff line number Diff line
@@ -339,6 +339,15 @@ static inline swp_entry_t folio_swap_entry(struct folio *folio) 
	return entry;

}



static inline swp_entry_t page_swap_entry(struct page *page)

{

	struct folio *folio = page_folio(page);

	swp_entry_t entry = folio_swap_entry(folio);



	entry.val += folio_page_idx(folio, page);

	return entry;

}



static inline void folio_set_swap_entry(struct folio *folio, swp_entry_t entry)

{

	folio->private = (void *)entry.val;

mm/huge_memory.c

+6 −9
Original line number Diff line number Diff line
@@ -2446,18 +2446,15 @@ static void __split_huge_page_tail(struct page *head, int tail, 
	page_tail->index = head->index + tail;



	/*

	 * page->private should not be set in tail pages with the exception

	 * of swap cache pages that store the swp_entry_t in tail pages.

	 * Fix up and warn once if private is unexpectedly set.

	 *

	 * What of 32-bit systems, on which folio->_pincount overlays

	 * head[1].private?  No problem: THP_SWAP is not enabled on 32-bit, and

	 * pincount must be 0 for folio_ref_freeze() to have succeeded.

	 * page->private should not be set in tail pages. Fix up and warn once

	 * if private is unexpectedly set.

	 */

	if (!folio_test_swapcache(page_folio(head))) {

		VM_WARN_ON_ONCE_PAGE(page_tail->private != 0, page_tail);

	if (unlikely(page_tail->private)) {

		VM_WARN_ON_ONCE_PAGE(true, page_tail);

		page_tail->private = 0;

	}

	if (PageSwapCache(head))

		set_page_private(page_tail, (unsigned long)head->private + tail);



	/* Page flags must be visible before we make the page non-compound. */

	smp_wmb();

mm/memory.c

+1 −1
Original line number Diff line number Diff line
@@ -3879,7 +3879,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) 
		 * changed.

		 */

		if (unlikely(!folio_test_swapcache(folio) ||

			     page_private(page) != entry.val))

			     page_swap_entry(page).val != entry.val))

			goto out_page;



		/*