Commit 1bb6b810 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

block: avoid extra iter advance with async iocb 



Nobody cares about iov iterators state if we return -EIOCBQUEUED, so as
the we now have __blkdev_direct_IO_async(), which gets pages only once,
we can skip expensive iov_iter_advance(). It's around 1-2% of all CPU
spent.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/a6158edfbfa2ae3bc24aed29a72f035df18fad2f.1635337135.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent a2247f19

block/bio.c

+1 −1
Original line number Diff line number Diff line
@@ -1046,7 +1046,7 @@ void __bio_release_pages(struct bio *bio, bool mark_dirty) 
}

EXPORT_SYMBOL_GPL(__bio_release_pages);



static void bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter)

void bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter)

{

	size_t size = iov_iter_count(iter);

block/fops.c

+15 −5
Original line number Diff line number Diff line
@@ -352,12 +352,22 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb, 
	bio->bi_end_io = blkdev_bio_end_io_async;

	bio->bi_ioprio = iocb->ki_ioprio;



	if (iov_iter_is_bvec(iter)) {

		/*

		 * Users don't rely on the iterator being in any particular

		 * state for async I/O returning -EIOCBQUEUED, hence we can

		 * avoid expensive iov_iter_advance(). Bypass

		 * bio_iov_iter_get_pages() and set the bvec directly.

		 */

		bio_iov_bvec_set(bio, iter);

	} else {

		ret = bio_iov_iter_get_pages(bio, iter);

		if (unlikely(ret)) {

			bio->bi_status = BLK_STS_IOERR;

			bio_endio(bio);

			return ret;

		}

	}

	dio->size = bio->bi_iter.bi_size;



	if (iov_iter_rw(iter) == READ) {

include/linux/bio.h

+1 −0
Original line number Diff line number Diff line
@@ -417,6 +417,7 @@ int bio_add_zone_append_page(struct bio *bio, struct page *page, 
void __bio_add_page(struct bio *bio, struct page *page,

		unsigned int len, unsigned int off);

int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter);

void bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter);

void __bio_release_pages(struct bio *bio, bool mark_dirty);

extern void bio_set_pages_dirty(struct bio *bio);

extern void bio_check_pages_dirty(struct bio *bio);