Commit e2e3fdc7 authored by Matthew Wilcox (Oracle)'s avatar Matthew Wilcox (Oracle) Committed by Andrew Morton
Browse files

swap: turn get_swap_page() into folio_alloc_swap() 

This removes an assumption that a large folio is HPAGE_PMD_NR pages
in size.

Link: https://lkml.kernel.org/r/20220504182857.4013401-8-willy@infradead.org


Signed-off-by: default avatarMatthew Wilcox (Oracle) <willy@infradead.org>
Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent d33e4e14

include/linux/swap.h

+7 −6
Original line number Diff line number Diff line
@@ -470,7 +470,7 @@ static inline long get_nr_swap_pages(void) 
}



extern void si_swapinfo(struct sysinfo *);

extern swp_entry_t get_swap_page(struct page *page);

swp_entry_t folio_alloc_swap(struct folio *folio);

extern void put_swap_page(struct page *page, swp_entry_t entry);

extern swp_entry_t get_swap_page_of_type(int);

extern int get_swap_pages(int n, swp_entry_t swp_entries[], int entry_size);
@@ -583,7 +583,7 @@ static inline int try_to_free_swap(struct page *page) 
	return 0;

}



static inline swp_entry_t get_swap_page(struct page *page)

static inline swp_entry_t folio_alloc_swap(struct folio *folio)

{

	swp_entry_t entry;

	entry.val = 0;
@@ -643,12 +643,13 @@ static inline void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask) 


#ifdef CONFIG_MEMCG_SWAP

void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry);

extern int __mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry);

static inline int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)

int __mem_cgroup_try_charge_swap(struct folio *folio, swp_entry_t entry);

static inline int mem_cgroup_try_charge_swap(struct folio *folio,

		swp_entry_t entry)

{

	if (mem_cgroup_disabled())

		return 0;

	return __mem_cgroup_try_charge_swap(page, entry);

	return __mem_cgroup_try_charge_swap(folio, entry);

}



extern void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages);
@@ -666,7 +667,7 @@ static inline void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry) 
{

}



static inline int mem_cgroup_try_charge_swap(struct page *page,

static inline int mem_cgroup_try_charge_swap(struct folio *folio,

					     swp_entry_t entry)

{

	return 0;

mm/memcontrol.c

+8 −8
Original line number Diff line number Diff line
@@ -7162,17 +7162,17 @@ void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry) 
}



/**

 * __mem_cgroup_try_charge_swap - try charging swap space for a page

 * @page: page being added to swap

 * __mem_cgroup_try_charge_swap - try charging swap space for a folio

 * @folio: folio being added to swap

 * @entry: swap entry to charge

 *

 * Try to charge @page's memcg for the swap space at @entry.

 * Try to charge @folio's memcg for the swap space at @entry.

 *

 * Returns 0 on success, -ENOMEM on failure.

 */

int __mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)

int __mem_cgroup_try_charge_swap(struct folio *folio, swp_entry_t entry)

{

	unsigned int nr_pages = thp_nr_pages(page);

	unsigned int nr_pages = folio_nr_pages(folio);

	struct page_counter *counter;

	struct mem_cgroup *memcg;

	unsigned short oldid;
@@ -7180,9 +7180,9 @@ int __mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry) 
	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))

		return 0;



	memcg = page_memcg(page);

	memcg = folio_memcg(folio);



	VM_WARN_ON_ONCE_PAGE(!memcg, page);

	VM_WARN_ON_ONCE_FOLIO(!memcg, folio);

	if (!memcg)

		return 0;
@@ -7205,7 +7205,7 @@ int __mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry) 
	if (nr_pages > 1)

		mem_cgroup_id_get_many(memcg, nr_pages - 1);

	oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);

	VM_BUG_ON_PAGE(oldid, page);

	VM_BUG_ON_FOLIO(oldid, folio);

	mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);



	return 0;

mm/shmem.c

+2 −1
Original line number Diff line number Diff line
@@ -1313,6 +1313,7 @@ int shmem_unuse(unsigned int type) 
 */

static int shmem_writepage(struct page *page, struct writeback_control *wbc)

{

	struct folio *folio = page_folio(page);

	struct shmem_inode_info *info;

	struct address_space *mapping;

	struct inode *inode;
@@ -1386,7 +1387,7 @@ static int shmem_writepage(struct page *page, struct writeback_control *wbc) 
		SetPageUptodate(page);

	}



	swap = get_swap_page(page);

	swap = folio_alloc_swap(folio);

	if (!swap.val)

		goto redirty;

mm/swap_slots.c

+7 −7
Original line number Diff line number Diff line
@@ -117,7 +117,7 @@ static int alloc_swap_slot_cache(unsigned int cpu) 


	/*

	 * Do allocation outside swap_slots_cache_mutex

	 * as kvzalloc could trigger reclaim and get_swap_page,

	 * as kvzalloc could trigger reclaim and folio_alloc_swap,

	 * which can lock swap_slots_cache_mutex.

	 */

	slots = kvcalloc(SWAP_SLOTS_CACHE_SIZE, sizeof(swp_entry_t),
@@ -213,7 +213,7 @@ static void __drain_swap_slots_cache(unsigned int type) 
	 * this function can be invoked in the cpu

	 * hot plug path:

	 * cpu_up -> lock cpu_hotplug -> cpu hotplug state callback

	 *   -> memory allocation -> direct reclaim -> get_swap_page

	 *   -> memory allocation -> direct reclaim -> folio_alloc_swap

	 *   -> drain_swap_slots_cache

	 *

	 * Hence the loop over current online cpu below could miss cpu that
@@ -301,16 +301,16 @@ int free_swap_slot(swp_entry_t entry) 
	return 0;

}



swp_entry_t get_swap_page(struct page *page)

swp_entry_t folio_alloc_swap(struct folio *folio)

{

	swp_entry_t entry;

	struct swap_slots_cache *cache;



	entry.val = 0;



	if (PageTransHuge(page)) {

	if (folio_test_large(folio)) {

		if (IS_ENABLED(CONFIG_THP_SWAP))

			get_swap_pages(1, &entry, HPAGE_PMD_NR);

			get_swap_pages(1, &entry, folio_nr_pages(folio));

		goto out;

	}
@@ -344,8 +344,8 @@ swp_entry_t get_swap_page(struct page *page) 


	get_swap_pages(1, &entry, 1);

out:

	if (mem_cgroup_try_charge_swap(page, entry)) {

		put_swap_page(page, entry);

	if (mem_cgroup_try_charge_swap(folio, entry)) {

		put_swap_page(&folio->page, entry);

		entry.val = 0;

	}

	return entry;

mm/swap_state.c

+2 −1
Original line number Diff line number Diff line
@@ -184,13 +184,14 @@ void __delete_from_swap_cache(struct page *page, 
 */

int add_to_swap(struct page *page)

{

	struct folio *folio = page_folio(page);

	swp_entry_t entry;

	int err;



	VM_BUG_ON_PAGE(!PageLocked(page), page);

	VM_BUG_ON_PAGE(!PageUptodate(page), page);



	entry = get_swap_page(page);

	entry = folio_alloc_swap(folio);

	if (!entry.val)

		return 0;