Commit 07173c3e authored by Ming Lei's avatar Ming Lei Committed by Jens Axboe
Browse files

block: enable multipage bvecs 



This patch pulls the trigger for multi-page bvecs.

Reviewed-by: default avatarOmar Sandoval <osandov@fb.com>
Signed-off-by: default avatarMing Lei <ming.lei@redhat.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 6dc4f100

block/bio.c

+15 −7
Original line number Diff line number Diff line
@@ -753,6 +753,8 @@ EXPORT_SYMBOL(bio_add_pc_page); 
 * @page: page to add

 * @len: length of the data to add

 * @off: offset of the data in @page

 * @same_page: if %true only merge if the new data is in the same physical

 *		page as the last segment of the bio.

 *

 * Try to add the data at @page + @off to the last bvec of @bio.  This is a

 * a useful optimisation for file systems with a block size smaller than the
@@ -761,20 +763,26 @@ EXPORT_SYMBOL(bio_add_pc_page); 
 * Return %true on success or %false on failure.

 */

bool __bio_try_merge_page(struct bio *bio, struct page *page,

		unsigned int len, unsigned int off)

		unsigned int len, unsigned int off, bool same_page)

{

	if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))

		return false;



	if (bio->bi_vcnt > 0) {

		struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];

		phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) +

			bv->bv_offset + bv->bv_len - 1;

		phys_addr_t page_addr = page_to_phys(page);



		if (vec_end_addr + 1 != page_addr + off)

			return false;

		if (same_page && (vec_end_addr & PAGE_MASK) != page_addr)

			return false;



		if (page == bv->bv_page && off == bv->bv_offset + bv->bv_len) {

		bv->bv_len += len;

		bio->bi_iter.bi_size += len;

		return true;

	}

	}

	return false;

}

EXPORT_SYMBOL_GPL(__bio_try_merge_page);
@@ -819,7 +827,7 @@ EXPORT_SYMBOL_GPL(__bio_add_page); 
int bio_add_page(struct bio *bio, struct page *page,

		 unsigned int len, unsigned int offset)

{

	if (!__bio_try_merge_page(bio, page, len, offset)) {

	if (!__bio_try_merge_page(bio, page, len, offset, false)) {

		if (bio_full(bio))

			return 0;

		__bio_add_page(bio, page, len, offset);

fs/iomap.c

+2 −2
Original line number Diff line number Diff line
@@ -318,7 +318,7 @@ iomap_readpage_actor(struct inode *inode, loff_t pos, loff_t length, void *data, 
	 */

	sector = iomap_sector(iomap, pos);

	if (ctx->bio && bio_end_sector(ctx->bio) == sector) {

		if (__bio_try_merge_page(ctx->bio, page, plen, poff))

		if (__bio_try_merge_page(ctx->bio, page, plen, poff, true))

			goto done;

		is_contig = true;

	}
@@ -349,7 +349,7 @@ iomap_readpage_actor(struct inode *inode, loff_t pos, loff_t length, void *data, 
		ctx->bio->bi_end_io = iomap_read_end_io;

	}



	__bio_add_page(ctx->bio, page, plen, poff);

	bio_add_page(ctx->bio, page, plen, poff);

done:

	/*

	 * Move the caller beyond our range so that it keeps making progress.

fs/xfs/xfs_aops.c

+2 −2
Original line number Diff line number Diff line
@@ -616,12 +616,12 @@ xfs_add_to_ioend( 
				bdev, sector);

	}



	if (!__bio_try_merge_page(wpc->ioend->io_bio, page, len, poff)) {

	if (!__bio_try_merge_page(wpc->ioend->io_bio, page, len, poff, true)) {

		if (iop)

			atomic_inc(&iop->write_count);

		if (bio_full(wpc->ioend->io_bio))

			xfs_chain_bio(wpc->ioend, wbc, bdev, sector);

		__bio_add_page(wpc->ioend->io_bio, page, len, poff);

		bio_add_page(wpc->ioend->io_bio, page, len, poff);

	}



	wpc->ioend->io_size += len;

include/linux/bio.h

+1 −1
Original line number Diff line number Diff line
@@ -441,7 +441,7 @@ extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int); 
extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,

			   unsigned int, unsigned int);

bool __bio_try_merge_page(struct bio *bio, struct page *page,

		unsigned int len, unsigned int off);

		unsigned int len, unsigned int off, bool same_page);

void __bio_add_page(struct bio *bio, struct page *page,

		unsigned int len, unsigned int off);

int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter);