Commit 9be3e06f authored by Jens Axboe's avatar Jens Axboe
Browse files

block: move update request helpers into blk-mq.c 



For some reason we still have them in blk-core, with the rest of the
request completion being in blk-mq. That causes and out-of-line call
for each completion.

Move them into blk-mq.c instead, where they belong.

Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent c477b797

block/blk-core.c

+1 −145
Original line number Diff line number Diff line
@@ -216,7 +216,7 @@ int blk_status_to_errno(blk_status_t status) 
}

EXPORT_SYMBOL_GPL(blk_status_to_errno);



static void print_req_error(struct request *req, blk_status_t status)

void blk_print_req_error(struct request *req, blk_status_t status)

{

	int idx = (__force int)status;
@@ -234,33 +234,6 @@ static void print_req_error(struct request *req, blk_status_t status) 
		IOPRIO_PRIO_CLASS(req->ioprio));

}



static void req_bio_endio(struct request *rq, struct bio *bio,

			  unsigned int nbytes, blk_status_t error)

{

	if (error)

		bio->bi_status = error;



	if (unlikely(rq->rq_flags & RQF_QUIET))

		bio_set_flag(bio, BIO_QUIET);



	bio_advance(bio, nbytes);



	if (req_op(rq) == REQ_OP_ZONE_APPEND && error == BLK_STS_OK) {

		/*

		 * Partial zone append completions cannot be supported as the

		 * BIO fragments may end up not being written sequentially.

		 */

		if (bio->bi_iter.bi_size)

			bio->bi_status = BLK_STS_IOERR;

		else

			bio->bi_iter.bi_sector = rq->__sector;

	}



	/* don't actually finish bio if it's part of flush sequence */

	if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))

		bio_endio(bio);

}



void blk_dump_rq_flags(struct request *rq, char *msg)

{

	printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
@@ -1311,17 +1284,6 @@ static void update_io_ticks(struct block_device *part, unsigned long now, 
	}

}



static void blk_account_io_completion(struct request *req, unsigned int bytes)

{

	if (req->part && blk_do_io_stat(req)) {

		const int sgrp = op_stat_group(req_op(req));



		part_stat_lock();

		part_stat_add(req->part, sectors[sgrp], bytes >> 9);

		part_stat_unlock();

	}

}



void __blk_account_io_done(struct request *req, u64 now)

{

	const int sgrp = op_stat_group(req_op(req));
@@ -1430,112 +1392,6 @@ void blk_steal_bios(struct bio_list *list, struct request *rq) 
}

EXPORT_SYMBOL_GPL(blk_steal_bios);



/**

 * blk_update_request - Complete multiple bytes without completing the request

 * @req:      the request being processed

 * @error:    block status code

 * @nr_bytes: number of bytes to complete for @req

 *

 * Description:

 *     Ends I/O on a number of bytes attached to @req, but doesn't complete

 *     the request structure even if @req doesn't have leftover.

 *     If @req has leftover, sets it up for the next range of segments.

 *

 *     Passing the result of blk_rq_bytes() as @nr_bytes guarantees

 *     %false return from this function.

 *

 * Note:

 *	The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in this function

 *      except in the consistency check at the end of this function.

 *

 * Return:

 *     %false - this request doesn't have any more data

 *     %true  - this request has more data

 **/

bool blk_update_request(struct request *req, blk_status_t error,

		unsigned int nr_bytes)

{

	int total_bytes;



	trace_block_rq_complete(req, blk_status_to_errno(error), nr_bytes);



	if (!req->bio)

		return false;



#ifdef CONFIG_BLK_DEV_INTEGRITY

	if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ &&

	    error == BLK_STS_OK)

		req->q->integrity.profile->complete_fn(req, nr_bytes);

#endif



	if (unlikely(error && !blk_rq_is_passthrough(req) &&

		     !(req->rq_flags & RQF_QUIET)))

		print_req_error(req, error);



	blk_account_io_completion(req, nr_bytes);



	total_bytes = 0;

	while (req->bio) {

		struct bio *bio = req->bio;

		unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);



		if (bio_bytes == bio->bi_iter.bi_size)

			req->bio = bio->bi_next;



		/* Completion has already been traced */

		bio_clear_flag(bio, BIO_TRACE_COMPLETION);

		req_bio_endio(req, bio, bio_bytes, error);



		total_bytes += bio_bytes;

		nr_bytes -= bio_bytes;



		if (!nr_bytes)

			break;

	}



	/*

	 * completely done

	 */

	if (!req->bio) {

		/*

		 * Reset counters so that the request stacking driver

		 * can find how many bytes remain in the request

		 * later.

		 */

		req->__data_len = 0;

		return false;

	}



	req->__data_len -= total_bytes;



	/* update sector only for requests with clear definition of sector */

	if (!blk_rq_is_passthrough(req))

		req->__sector += total_bytes >> 9;



	/* mixed attributes always follow the first bio */

	if (req->rq_flags & RQF_MIXED_MERGE) {

		req->cmd_flags &= ~REQ_FAILFAST_MASK;

		req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;

	}



	if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {

		/*

		 * If total number of sectors is less than the first segment

		 * size, something has gone terribly wrong.

		 */

		if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {

			blk_dump_rq_flags(req, "request botched");

			req->__data_len = blk_rq_cur_bytes(req);

		}



		/* recalculate the number of segments */

		req->nr_phys_segments = blk_recalc_rq_segments(req);

	}



	return true;

}

EXPORT_SYMBOL_GPL(blk_update_request);



#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE

/**

 * rq_flush_dcache_pages - Helper function to flush all pages in a request

block/blk-mq.c

+144 −0
Original line number Diff line number Diff line
@@ -628,6 +628,150 @@ void blk_mq_free_plug_rqs(struct blk_plug *plug) 
	}

}



static void req_bio_endio(struct request *rq, struct bio *bio,

			  unsigned int nbytes, blk_status_t error)

{

	if (error)

		bio->bi_status = error;



	if (unlikely(rq->rq_flags & RQF_QUIET))

		bio_set_flag(bio, BIO_QUIET);



	bio_advance(bio, nbytes);



	if (req_op(rq) == REQ_OP_ZONE_APPEND && error == BLK_STS_OK) {

		/*

		 * Partial zone append completions cannot be supported as the

		 * BIO fragments may end up not being written sequentially.

		 */

		if (bio->bi_iter.bi_size)

			bio->bi_status = BLK_STS_IOERR;

		else

			bio->bi_iter.bi_sector = rq->__sector;

	}



	/* don't actually finish bio if it's part of flush sequence */

	if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))

		bio_endio(bio);

}



static void blk_account_io_completion(struct request *req, unsigned int bytes)

{

	if (req->part && blk_do_io_stat(req)) {

		const int sgrp = op_stat_group(req_op(req));



		part_stat_lock();

		part_stat_add(req->part, sectors[sgrp], bytes >> 9);

		part_stat_unlock();

	}

}



/**

 * blk_update_request - Complete multiple bytes without completing the request

 * @req:      the request being processed

 * @error:    block status code

 * @nr_bytes: number of bytes to complete for @req

 *

 * Description:

 *     Ends I/O on a number of bytes attached to @req, but doesn't complete

 *     the request structure even if @req doesn't have leftover.

 *     If @req has leftover, sets it up for the next range of segments.

 *

 *     Passing the result of blk_rq_bytes() as @nr_bytes guarantees

 *     %false return from this function.

 *

 * Note:

 *	The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in this function

 *      except in the consistency check at the end of this function.

 *

 * Return:

 *     %false - this request doesn't have any more data

 *     %true  - this request has more data

 **/

bool blk_update_request(struct request *req, blk_status_t error,

		unsigned int nr_bytes)

{

	int total_bytes;



	trace_block_rq_complete(req, blk_status_to_errno(error), nr_bytes);



	if (!req->bio)

		return false;



#ifdef CONFIG_BLK_DEV_INTEGRITY

	if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ &&

	    error == BLK_STS_OK)

		req->q->integrity.profile->complete_fn(req, nr_bytes);

#endif



	if (unlikely(error && !blk_rq_is_passthrough(req) &&

		     !(req->rq_flags & RQF_QUIET)))

		blk_print_req_error(req, error);



	blk_account_io_completion(req, nr_bytes);



	total_bytes = 0;

	while (req->bio) {

		struct bio *bio = req->bio;

		unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);



		if (bio_bytes == bio->bi_iter.bi_size)

			req->bio = bio->bi_next;



		/* Completion has already been traced */

		bio_clear_flag(bio, BIO_TRACE_COMPLETION);

		req_bio_endio(req, bio, bio_bytes, error);



		total_bytes += bio_bytes;

		nr_bytes -= bio_bytes;



		if (!nr_bytes)

			break;

	}



	/*

	 * completely done

	 */

	if (!req->bio) {

		/*

		 * Reset counters so that the request stacking driver

		 * can find how many bytes remain in the request

		 * later.

		 */

		req->__data_len = 0;

		return false;

	}



	req->__data_len -= total_bytes;



	/* update sector only for requests with clear definition of sector */

	if (!blk_rq_is_passthrough(req))

		req->__sector += total_bytes >> 9;



	/* mixed attributes always follow the first bio */

	if (req->rq_flags & RQF_MIXED_MERGE) {

		req->cmd_flags &= ~REQ_FAILFAST_MASK;

		req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;

	}



	if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {

		/*

		 * If total number of sectors is less than the first segment

		 * size, something has gone terribly wrong.

		 */

		if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {

			blk_dump_rq_flags(req, "request botched");

			req->__data_len = blk_rq_cur_bytes(req);

		}



		/* recalculate the number of segments */

		req->nr_phys_segments = blk_recalc_rq_segments(req);

	}



	return true;

}

EXPORT_SYMBOL_GPL(blk_update_request);



inline void __blk_mq_end_request(struct request *rq, blk_status_t error)

{

	if (blk_mq_need_time_stamp(rq)) {

block/blk.h

+1 −0
Original line number Diff line number Diff line
@@ -215,6 +215,7 @@ static inline void blk_integrity_del(struct gendisk *disk) 


unsigned long blk_rq_timeout(unsigned long timeout);

void blk_add_timer(struct request *req);

void blk_print_req_error(struct request *req, blk_status_t status);



bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,

		unsigned int nr_segs, struct request **same_queue_rq);