Commit 5169b844 authored by NeilBrown's avatar NeilBrown Committed by Andrew Morton
Browse files

mm: submit multipage reads for SWP_FS_OPS swap-space 

swap_readpage() is given one page at a time, but may be called repeatedly
in succession.

For block-device swap-space, the blk_plug functionality allows the
multiple pages to be combined together at lower layers.  That cannot be
used for SWP_FS_OPS as blk_plug may not exist - it is only active when
CONFIG_BLOCK=y.  Consequently all swap reads over NFS are single page
reads.

With this patch we pass in a pointer-to-pointer when swap_readpage can
store state between calls - much like the effect of blk_plug.  After
calling swap_readpage() some number of times, the state will be passed to
swap_read_unplug() which can submit the combined request.

Link: https://lkml.kernel.org/r/164859778127.29473.14059420492644907783.stgit@noble.brown


Signed-off-by: default avatarNeilBrown <neilb@suse.de>
Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Tested-by: default avatarDavid Howells <dhowells@redhat.com>
Tested-by: default avatarGeert Uytterhoeven <geert+renesas@glider.be>
Cc: Hugh Dickins <hughd@google.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Trond Myklebust <trond.myklebust@hammerspace.com>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent cba738f6

mm/madvise.c

+6 −2
Original line number Diff line number Diff line
@@ -198,6 +198,7 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start, 
	pte_t *orig_pte;

	struct vm_area_struct *vma = walk->private;

	unsigned long index;

	struct swap_iocb *splug = NULL;



	if (pmd_none_or_trans_huge_or_clear_bad(pmd))

		return 0;
@@ -219,10 +220,11 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start, 
			continue;



		page = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE,

							vma, index, false);

					     vma, index, false, &splug);

		if (page)

			put_page(page);

	}

	swap_read_unplug(splug);



	return 0;

}
@@ -238,6 +240,7 @@ static void force_shm_swapin_readahead(struct vm_area_struct *vma, 
	XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start));

	pgoff_t end_index = linear_page_index(vma, end + PAGE_SIZE - 1);

	struct page *page;

	struct swap_iocb *splug = NULL;



	rcu_read_lock();

	xas_for_each(&xas, page, end_index) {
@@ -250,13 +253,14 @@ static void force_shm_swapin_readahead(struct vm_area_struct *vma, 


		swap = radix_to_swp_entry(page);

		page = read_swap_cache_async(swap, GFP_HIGHUSER_MOVABLE,

							NULL, 0, false);

					     NULL, 0, false, &splug);

		if (page)

			put_page(page);



		rcu_read_lock();

	}

	rcu_read_unlock();

	swap_read_unplug(splug);



	lru_add_drain();	/* Push any new pages onto the LRU now */

}

mm/memory.c

+1 −1
Original line number Diff line number Diff line
@@ -3633,7 +3633,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) 


				/* To provide entry to swap_readpage() */

				set_page_private(page, entry.val);

				swap_readpage(page, true);

				swap_readpage(page, true, NULL);

				set_page_private(page, 0);

			}

		} else {

mm/page_io.c

+69 −35
Original line number Diff line number Diff line
@@ -237,7 +237,8 @@ static void bio_associate_blkg_from_page(struct bio *bio, struct page *page) 


struct swap_iocb {

	struct kiocb		iocb;

	struct bio_vec		bvec;

	struct bio_vec		bvec[SWAP_CLUSTER_MAX];

	int			pages;

};

static mempool_t *sio_pool;
@@ -257,7 +258,7 @@ int sio_pool_init(void) 
static void sio_write_complete(struct kiocb *iocb, long ret)

{

	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);

	struct page *page = sio->bvec.bv_page;

	struct page *page = sio->bvec[0].bv_page;



	if (ret != PAGE_SIZE) {

		/*
@@ -295,10 +296,10 @@ static int swap_writepage_fs(struct page *page, struct writeback_control *wbc) 
	init_sync_kiocb(&sio->iocb, swap_file);

	sio->iocb.ki_complete = sio_write_complete;

	sio->iocb.ki_pos = page_file_offset(page);

	sio->bvec.bv_page = page;

	sio->bvec.bv_len = PAGE_SIZE;

	sio->bvec.bv_offset = 0;

	iov_iter_bvec(&from, WRITE, &sio->bvec, 1, PAGE_SIZE);

	sio->bvec[0].bv_page = page;

	sio->bvec[0].bv_len = PAGE_SIZE;

	sio->bvec[0].bv_offset = 0;

	iov_iter_bvec(&from, WRITE, &sio->bvec[0], 1, PAGE_SIZE);

	ret = mapping->a_ops->swap_rw(&sio->iocb, &from);

	if (ret != -EIOCBQUEUED)

		sio_write_complete(&sio->iocb, ret);
@@ -346,46 +347,66 @@ int __swap_writepage(struct page *page, struct writeback_control *wbc, 
static void sio_read_complete(struct kiocb *iocb, long ret)

{

	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);

	struct page *page = sio->bvec.bv_page;

	int p;



	if (ret == PAGE_SIZE * sio->pages) {

		for (p = 0; p < sio->pages; p++) {

			struct page *page = sio->bvec[p].bv_page;



			SetPageUptodate(page);

			unlock_page(page);

		}

		count_vm_events(PSWPIN, sio->pages);

	} else {

		for (p = 0; p < sio->pages; p++) {

			struct page *page = sio->bvec[p].bv_page;



	if (ret != 0 && ret != PAGE_SIZE) {

			SetPageError(page);

			ClearPageUptodate(page);

			unlock_page(page);

		}

		pr_alert_ratelimited("Read-error on swap-device\n");

	} else {

		SetPageUptodate(page);

		count_vm_event(PSWPIN);

	}

	unlock_page(page);

	mempool_free(sio, sio_pool);

}



static int swap_readpage_fs(struct page *page)

static void swap_readpage_fs(struct page *page,

			     struct swap_iocb **plug)

{

	struct swap_info_struct *sis = page_swap_info(page);

	struct file *swap_file = sis->swap_file;

	struct address_space *mapping = swap_file->f_mapping;

	struct iov_iter from;

	struct swap_iocb *sio;

	struct swap_iocb *sio = NULL;

	loff_t pos = page_file_offset(page);

	int ret;



	if (plug)

		sio = *plug;

	if (sio) {

		if (sio->iocb.ki_filp != sis->swap_file ||

		    sio->iocb.ki_pos + sio->pages * PAGE_SIZE != pos) {

			swap_read_unplug(sio);

			sio = NULL;

		}

	}

	if (!sio) {

		sio = mempool_alloc(sio_pool, GFP_KERNEL);

	init_sync_kiocb(&sio->iocb, swap_file);

		init_sync_kiocb(&sio->iocb, sis->swap_file);

		sio->iocb.ki_pos = pos;

		sio->iocb.ki_complete = sio_read_complete;

	sio->bvec.bv_page = page;

	sio->bvec.bv_len = PAGE_SIZE;

	sio->bvec.bv_offset = 0;



	iov_iter_bvec(&from, READ, &sio->bvec, 1, PAGE_SIZE);

	ret = mapping->a_ops->swap_rw(&sio->iocb, &from);

	if (ret != -EIOCBQUEUED)

		sio_read_complete(&sio->iocb, ret);

	return ret;

		sio->pages = 0;

	}

	sio->bvec[sio->pages].bv_page = page;

	sio->bvec[sio->pages].bv_len = PAGE_SIZE;

	sio->bvec[sio->pages].bv_offset = 0;

	sio->pages += 1;

	if (sio->pages == ARRAY_SIZE(sio->bvec) || !plug) {

		swap_read_unplug(sio);

		sio = NULL;

	}

	if (plug)

		*plug = sio;

}



int swap_readpage(struct page *page, bool synchronous)

int swap_readpage(struct page *page, bool synchronous,

		  struct swap_iocb **plug)

{

	struct bio *bio;

	int ret = 0;
@@ -413,7 +434,7 @@ int swap_readpage(struct page *page, bool synchronous) 
	}



	if (data_race(sis->flags & SWP_FS_OPS)) {

		ret = swap_readpage_fs(page);

		swap_readpage_fs(page, plug);

		goto out;

	}
@@ -459,3 +480,16 @@ int swap_readpage(struct page *page, bool synchronous) 
	delayacct_swapin_end();

	return ret;

}



void __swap_read_unplug(struct swap_iocb *sio)

{

	struct iov_iter from;

	struct address_space *mapping = sio->iocb.ki_filp->f_mapping;

	int ret;



	iov_iter_bvec(&from, READ, sio->bvec, sio->pages,

		      PAGE_SIZE * sio->pages);

	ret = mapping->a_ops->swap_rw(&sio->iocb, &from);

	if (ret != -EIOCBQUEUED)

		sio_read_complete(&sio->iocb, ret);

}

mm/swap.h

+14 −3
Original line number Diff line number Diff line
@@ -7,7 +7,15 @@ 


/* linux/mm/page_io.c */

int sio_pool_init(void);

int swap_readpage(struct page *page, bool do_poll);

struct swap_iocb;

int swap_readpage(struct page *page, bool do_poll,

		  struct swap_iocb **plug);

void __swap_read_unplug(struct swap_iocb *plug);

static inline void swap_read_unplug(struct swap_iocb *plug)

{

	if (unlikely(plug))

		__swap_read_unplug(plug);

}

int swap_writepage(struct page *page, struct writeback_control *wbc);

void end_swap_bio_write(struct bio *bio);

int __swap_writepage(struct page *page, struct writeback_control *wbc,
@@ -41,7 +49,8 @@ struct page *find_get_incore_page(struct address_space *mapping, pgoff_t index); 
struct page *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,

				   struct vm_area_struct *vma,

				   unsigned long addr,

				   bool do_poll);

				   bool do_poll,

				   struct swap_iocb **plug);

struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,

				     struct vm_area_struct *vma,

				     unsigned long addr,
@@ -56,7 +65,9 @@ static inline unsigned int page_swap_flags(struct page *page) 
	return page_swap_info(page)->flags;

}

#else /* CONFIG_SWAP */

static inline int swap_readpage(struct page *page, bool do_poll)

struct swap_iocb;

static inline int swap_readpage(struct page *page, bool do_poll,

				struct swap_iocb **plug)

{

	return 0;

}

mm/swap_state.c

+14 −6
Original line number Diff line number Diff line
@@ -520,14 +520,16 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, 
 * the swap entry is no longer in use.

 */

struct page *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,

		struct vm_area_struct *vma, unsigned long addr, bool do_poll)

				   struct vm_area_struct *vma,

				   unsigned long addr, bool do_poll,

				   struct swap_iocb **plug)

{

	bool page_was_allocated;

	struct page *retpage = __read_swap_cache_async(entry, gfp_mask,

			vma, addr, &page_was_allocated);



	if (page_was_allocated)

		swap_readpage(retpage, do_poll);

		swap_readpage(retpage, do_poll, plug);



	return retpage;

}
@@ -621,6 +623,7 @@ struct page *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask, 
	unsigned long mask;

	struct swap_info_struct *si = swp_swap_info(entry);

	struct blk_plug plug;

	struct swap_iocb *splug = NULL;

	bool do_poll = true, page_allocated;

	struct vm_area_struct *vma = vmf->vma;

	unsigned long addr = vmf->address;
@@ -647,7 +650,7 @@ struct page *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask, 
		if (!page)

			continue;

		if (page_allocated) {

			swap_readpage(page, false);

			swap_readpage(page, false, &splug);

			if (offset != entry_offset) {

				SetPageReadahead(page);

				count_vm_event(SWAP_RA);
@@ -656,10 +659,12 @@ struct page *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask, 
		put_page(page);

	}

	blk_finish_plug(&plug);

	swap_read_unplug(splug);



	lru_add_drain();	/* Push any new pages onto the LRU now */

skip:

	return read_swap_cache_async(entry, gfp_mask, vma, addr, do_poll);

	/* The page was likely read above, so no need for plugging here */

	return read_swap_cache_async(entry, gfp_mask, vma, addr, do_poll, NULL);

}



int init_swap_address_space(unsigned int type, unsigned long nr_pages)
@@ -790,6 +795,7 @@ static struct page *swap_vma_readahead(swp_entry_t fentry, gfp_t gfp_mask, 
				       struct vm_fault *vmf)

{

	struct blk_plug plug;

	struct swap_iocb *splug = NULL;

	struct vm_area_struct *vma = vmf->vma;

	struct page *page;

	pte_t *pte, pentry;
@@ -820,7 +826,7 @@ static struct page *swap_vma_readahead(swp_entry_t fentry, gfp_t gfp_mask, 
		if (!page)

			continue;

		if (page_allocated) {

			swap_readpage(page, false);

			swap_readpage(page, false, &splug);

			if (i != ra_info.offset) {

				SetPageReadahead(page);

				count_vm_event(SWAP_RA);
@@ -829,10 +835,12 @@ static struct page *swap_vma_readahead(swp_entry_t fentry, gfp_t gfp_mask, 
		put_page(page);

	}

	blk_finish_plug(&plug);

	swap_read_unplug(splug);

	lru_add_drain();

skip:

	/* The page was likely read above, so no need for plugging here */

	return read_swap_cache_async(fentry, gfp_mask, vma, vmf->address,

				     ra_info.win == 1);

				     ra_info.win == 1, NULL);

}



/**