Commit be6bfe36 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

block: inline hot paths of blk_account_io_*() 



Extract hot paths of __blk_account_io_start() and
__blk_account_io_done() into inline functions, so we don't always pay
for function calls.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/b0662a636bd4cc7b4f84c9d0a41efa46a688ef13.1633781740.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 8a709512

block/blk-core.c

+9 −21
Original line number Diff line number Diff line
@@ -1180,7 +1180,6 @@ blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request * 
	if (blk_crypto_insert_cloned_request(rq))

		return BLK_STS_IOERR;



	if (blk_queue_io_stat(q))

	blk_account_io_start(rq);



	/*
@@ -1260,15 +1259,8 @@ static void blk_account_io_completion(struct request *req, unsigned int bytes) 
	}

}



void blk_account_io_done(struct request *req, u64 now)

void __blk_account_io_done(struct request *req, u64 now)

{

	/*

	 * Account IO completion.  flush_rq isn't accounted as a

	 * normal IO on queueing nor completion.  Accounting the

	 * containing request is enough.

	 */

	if (req->part && blk_do_io_stat(req) &&

	    !(req->rq_flags & RQF_FLUSH_SEQ)) {

	const int sgrp = op_stat_group(req_op(req));



	part_stat_lock();
@@ -1277,13 +1269,9 @@ void blk_account_io_done(struct request *req, u64 now) 
	part_stat_add(req->part, nsecs[sgrp], now - req->start_time_ns);

	part_stat_unlock();

}

}



void blk_account_io_start(struct request *rq)

void __blk_account_io_start(struct request *rq)

{

	if (!blk_do_io_stat(rq))

		return;



	/* passthrough requests can hold bios that do not have ->bi_bdev set */

	if (rq->bio && rq->bio->bi_bdev)

		rq->part = rq->bio->bi_bdev;

block/blk.h

+21 −3
Original line number Diff line number Diff line
@@ -221,8 +221,8 @@ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, 
bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,

			struct bio *bio, unsigned int nr_segs);



void blk_account_io_start(struct request *req);

void blk_account_io_done(struct request *req, u64 now);

void __blk_account_io_start(struct request *req);

void __blk_account_io_done(struct request *req, u64 now);



/*

 * Plug flush limits
@@ -286,7 +286,25 @@ int blk_dev_init(void); 
 */

static inline bool blk_do_io_stat(struct request *rq)

{

	return rq->rq_disk && (rq->rq_flags & RQF_IO_STAT);

	return (rq->rq_flags & RQF_IO_STAT) && rq->rq_disk;

}



static inline void blk_account_io_done(struct request *req, u64 now)

{

	/*

	 * Account IO completion.  flush_rq isn't accounted as a

	 * normal IO on queueing nor completion.  Accounting the

	 * containing request is enough.

	 */

	if (blk_do_io_stat(req) && req->part &&

	    !(req->rq_flags & RQF_FLUSH_SEQ))

		__blk_account_io_done(req, now);

}



static inline void blk_account_io_start(struct request *req)

{

	if (blk_do_io_stat(req))

		__blk_account_io_start(req);

}



static inline void req_set_nomerge(struct request_queue *q, struct request *req)