Commit 06c8c691 authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Jens Axboe
Browse files

block: move request based cloning helpers to blk-mq.c 



Keep all the request based code together.

Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Reviewed-by: default avatarJohannes Thumshirn <johannes.thumshirn@wdc.com>
Link: https://lore.kernel.org/r/20211117061404.331732-6-hch@lst.de


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent b84c5b50

block/blk-core.c

+1 −183
Original line number Diff line number Diff line
@@ -594,7 +594,7 @@ static int __init setup_fail_make_request(char *str) 
}

__setup("fail_make_request=", setup_fail_make_request);



static bool should_fail_request(struct block_device *part, unsigned int bytes)

bool should_fail_request(struct block_device *part, unsigned int bytes)

{

	return part->bd_make_it_fail && should_fail(&fail_make_request, bytes);

}
@@ -608,15 +608,6 @@ static int __init fail_make_request_debugfs(void) 
}



late_initcall(fail_make_request_debugfs);



#else /* CONFIG_FAIL_MAKE_REQUEST */



static inline bool should_fail_request(struct block_device *part,

					unsigned int bytes)

{

	return false;

}



#endif /* CONFIG_FAIL_MAKE_REQUEST */



static inline bool bio_check_ro(struct bio *bio)
@@ -1090,92 +1081,6 @@ int iocb_bio_iopoll(struct kiocb *kiocb, struct io_comp_batch *iob, 
}

EXPORT_SYMBOL_GPL(iocb_bio_iopoll);



/**

 * blk_cloned_rq_check_limits - Helper function to check a cloned request

 *                              for the new queue limits

 * @q:  the queue

 * @rq: the request being checked

 *

 * Description:

 *    @rq may have been made based on weaker limitations of upper-level queues

 *    in request stacking drivers, and it may violate the limitation of @q.

 *    Since the block layer and the underlying device driver trust @rq

 *    after it is inserted to @q, it should be checked against @q before

 *    the insertion using this generic function.

 *

 *    Request stacking drivers like request-based dm may change the queue

 *    limits when retrying requests on other queues. Those requests need

 *    to be checked against the new queue limits again during dispatch.

 */

static blk_status_t blk_cloned_rq_check_limits(struct request_queue *q,

				      struct request *rq)

{

	unsigned int max_sectors = blk_queue_get_max_sectors(q, req_op(rq));



	if (blk_rq_sectors(rq) > max_sectors) {

		/*

		 * SCSI device does not have a good way to return if

		 * Write Same/Zero is actually supported. If a device rejects

		 * a non-read/write command (discard, write same,etc.) the

		 * low-level device driver will set the relevant queue limit to

		 * 0 to prevent blk-lib from issuing more of the offending

		 * operations. Commands queued prior to the queue limit being

		 * reset need to be completed with BLK_STS_NOTSUPP to avoid I/O

		 * errors being propagated to upper layers.

		 */

		if (max_sectors == 0)

			return BLK_STS_NOTSUPP;



		printk(KERN_ERR "%s: over max size limit. (%u > %u)\n",

			__func__, blk_rq_sectors(rq), max_sectors);

		return BLK_STS_IOERR;

	}



	/*

	 * The queue settings related to segment counting may differ from the

	 * original queue.

	 */

	rq->nr_phys_segments = blk_recalc_rq_segments(rq);

	if (rq->nr_phys_segments > queue_max_segments(q)) {

		printk(KERN_ERR "%s: over max segments limit. (%hu > %hu)\n",

			__func__, rq->nr_phys_segments, queue_max_segments(q));

		return BLK_STS_IOERR;

	}



	return BLK_STS_OK;

}



/**

 * blk_insert_cloned_request - Helper for stacking drivers to submit a request

 * @q:  the queue to submit the request

 * @rq: the request being queued

 */

blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *rq)

{

	blk_status_t ret;



	ret = blk_cloned_rq_check_limits(q, rq);

	if (ret != BLK_STS_OK)

		return ret;



	if (rq->rq_disk &&

	    should_fail_request(rq->rq_disk->part0, blk_rq_bytes(rq)))

		return BLK_STS_IOERR;



	if (blk_crypto_insert_cloned_request(rq))

		return BLK_STS_IOERR;



	blk_account_io_start(rq);



	/*

	 * Since we have a scheduler attached on the top device,

	 * bypass a potential scheduler on the bottom device for

	 * insert.

	 */

	return blk_mq_request_issue_directly(rq, true);

}

EXPORT_SYMBOL_GPL(blk_insert_cloned_request);



static void update_io_ticks(struct block_device *part, unsigned long now,

		bool end)

{
@@ -1328,93 +1233,6 @@ int blk_lld_busy(struct request_queue *q) 
}

EXPORT_SYMBOL_GPL(blk_lld_busy);



/**

 * blk_rq_unprep_clone - Helper function to free all bios in a cloned request

 * @rq: the clone request to be cleaned up

 *

 * Description:

 *     Free all bios in @rq for a cloned request.

 */

void blk_rq_unprep_clone(struct request *rq)

{

	struct bio *bio;



	while ((bio = rq->bio) != NULL) {

		rq->bio = bio->bi_next;



		bio_put(bio);

	}

}

EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);



/**

 * blk_rq_prep_clone - Helper function to setup clone request

 * @rq: the request to be setup

 * @rq_src: original request to be cloned

 * @bs: bio_set that bios for clone are allocated from

 * @gfp_mask: memory allocation mask for bio

 * @bio_ctr: setup function to be called for each clone bio.

 *           Returns %0 for success, non %0 for failure.

 * @data: private data to be passed to @bio_ctr

 *

 * Description:

 *     Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.

 *     Also, pages which the original bios are pointing to are not copied

 *     and the cloned bios just point same pages.

 *     So cloned bios must be completed before original bios, which means

 *     the caller must complete @rq before @rq_src.

 */

int blk_rq_prep_clone(struct request *rq, struct request *rq_src,

		      struct bio_set *bs, gfp_t gfp_mask,

		      int (*bio_ctr)(struct bio *, struct bio *, void *),

		      void *data)

{

	struct bio *bio, *bio_src;



	if (!bs)

		bs = &fs_bio_set;



	__rq_for_each_bio(bio_src, rq_src) {

		bio = bio_clone_fast(bio_src, gfp_mask, bs);

		if (!bio)

			goto free_and_out;



		if (bio_ctr && bio_ctr(bio, bio_src, data))

			goto free_and_out;



		if (rq->bio) {

			rq->biotail->bi_next = bio;

			rq->biotail = bio;

		} else {

			rq->bio = rq->biotail = bio;

		}

		bio = NULL;

	}



	/* Copy attributes of the original request to the clone request. */

	rq->__sector = blk_rq_pos(rq_src);

	rq->__data_len = blk_rq_bytes(rq_src);

	if (rq_src->rq_flags & RQF_SPECIAL_PAYLOAD) {

		rq->rq_flags |= RQF_SPECIAL_PAYLOAD;

		rq->special_vec = rq_src->special_vec;

	}

	rq->nr_phys_segments = rq_src->nr_phys_segments;

	rq->ioprio = rq_src->ioprio;



	if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0)

		goto free_and_out;



	return 0;



free_and_out:

	if (bio)

		bio_put(bio);

	blk_rq_unprep_clone(rq);



	return -ENOMEM;

}

EXPORT_SYMBOL_GPL(blk_rq_prep_clone);



int kblockd_schedule_work(struct work_struct *work)

{

	return queue_work(kblockd_workqueue, work);

block/blk-mq.c

+174 −1
Original line number Diff line number Diff line
@@ -2435,7 +2435,7 @@ static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, 
	hctx_unlock(hctx, srcu_idx);

}



blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last)

static blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last)

{

	blk_status_t ret;

	int srcu_idx;
@@ -2824,6 +2824,179 @@ void blk_mq_submit_bio(struct bio *bio) 
	}

}



/**

 * blk_cloned_rq_check_limits - Helper function to check a cloned request

 *                              for the new queue limits

 * @q:  the queue

 * @rq: the request being checked

 *

 * Description:

 *    @rq may have been made based on weaker limitations of upper-level queues

 *    in request stacking drivers, and it may violate the limitation of @q.

 *    Since the block layer and the underlying device driver trust @rq

 *    after it is inserted to @q, it should be checked against @q before

 *    the insertion using this generic function.

 *

 *    Request stacking drivers like request-based dm may change the queue

 *    limits when retrying requests on other queues. Those requests need

 *    to be checked against the new queue limits again during dispatch.

 */

static blk_status_t blk_cloned_rq_check_limits(struct request_queue *q,

				      struct request *rq)

{

	unsigned int max_sectors = blk_queue_get_max_sectors(q, req_op(rq));



	if (blk_rq_sectors(rq) > max_sectors) {

		/*

		 * SCSI device does not have a good way to return if

		 * Write Same/Zero is actually supported. If a device rejects

		 * a non-read/write command (discard, write same,etc.) the

		 * low-level device driver will set the relevant queue limit to

		 * 0 to prevent blk-lib from issuing more of the offending

		 * operations. Commands queued prior to the queue limit being

		 * reset need to be completed with BLK_STS_NOTSUPP to avoid I/O

		 * errors being propagated to upper layers.

		 */

		if (max_sectors == 0)

			return BLK_STS_NOTSUPP;



		printk(KERN_ERR "%s: over max size limit. (%u > %u)\n",

			__func__, blk_rq_sectors(rq), max_sectors);

		return BLK_STS_IOERR;

	}



	/*

	 * The queue settings related to segment counting may differ from the

	 * original queue.

	 */

	rq->nr_phys_segments = blk_recalc_rq_segments(rq);

	if (rq->nr_phys_segments > queue_max_segments(q)) {

		printk(KERN_ERR "%s: over max segments limit. (%hu > %hu)\n",

			__func__, rq->nr_phys_segments, queue_max_segments(q));

		return BLK_STS_IOERR;

	}



	return BLK_STS_OK;

}



/**

 * blk_insert_cloned_request - Helper for stacking drivers to submit a request

 * @q:  the queue to submit the request

 * @rq: the request being queued

 */

blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *rq)

{

	blk_status_t ret;



	ret = blk_cloned_rq_check_limits(q, rq);

	if (ret != BLK_STS_OK)

		return ret;



	if (rq->rq_disk &&

	    should_fail_request(rq->rq_disk->part0, blk_rq_bytes(rq)))

		return BLK_STS_IOERR;



	if (blk_crypto_insert_cloned_request(rq))

		return BLK_STS_IOERR;



	blk_account_io_start(rq);



	/*

	 * Since we have a scheduler attached on the top device,

	 * bypass a potential scheduler on the bottom device for

	 * insert.

	 */

	return blk_mq_request_issue_directly(rq, true);

}

EXPORT_SYMBOL_GPL(blk_insert_cloned_request);



/**

 * blk_rq_unprep_clone - Helper function to free all bios in a cloned request

 * @rq: the clone request to be cleaned up

 *

 * Description:

 *     Free all bios in @rq for a cloned request.

 */

void blk_rq_unprep_clone(struct request *rq)

{

	struct bio *bio;



	while ((bio = rq->bio) != NULL) {

		rq->bio = bio->bi_next;



		bio_put(bio);

	}

}

EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);



/**

 * blk_rq_prep_clone - Helper function to setup clone request

 * @rq: the request to be setup

 * @rq_src: original request to be cloned

 * @bs: bio_set that bios for clone are allocated from

 * @gfp_mask: memory allocation mask for bio

 * @bio_ctr: setup function to be called for each clone bio.

 *           Returns %0 for success, non %0 for failure.

 * @data: private data to be passed to @bio_ctr

 *

 * Description:

 *     Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.

 *     Also, pages which the original bios are pointing to are not copied

 *     and the cloned bios just point same pages.

 *     So cloned bios must be completed before original bios, which means

 *     the caller must complete @rq before @rq_src.

 */

int blk_rq_prep_clone(struct request *rq, struct request *rq_src,

		      struct bio_set *bs, gfp_t gfp_mask,

		      int (*bio_ctr)(struct bio *, struct bio *, void *),

		      void *data)

{

	struct bio *bio, *bio_src;



	if (!bs)

		bs = &fs_bio_set;



	__rq_for_each_bio(bio_src, rq_src) {

		bio = bio_clone_fast(bio_src, gfp_mask, bs);

		if (!bio)

			goto free_and_out;



		if (bio_ctr && bio_ctr(bio, bio_src, data))

			goto free_and_out;



		if (rq->bio) {

			rq->biotail->bi_next = bio;

			rq->biotail = bio;

		} else {

			rq->bio = rq->biotail = bio;

		}

		bio = NULL;

	}



	/* Copy attributes of the original request to the clone request. */

	rq->__sector = blk_rq_pos(rq_src);

	rq->__data_len = blk_rq_bytes(rq_src);

	if (rq_src->rq_flags & RQF_SPECIAL_PAYLOAD) {

		rq->rq_flags |= RQF_SPECIAL_PAYLOAD;

		rq->special_vec = rq_src->special_vec;

	}

	rq->nr_phys_segments = rq_src->nr_phys_segments;

	rq->ioprio = rq_src->ioprio;



	if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0)

		goto free_and_out;



	return 0;



free_and_out:

	if (bio)

		bio_put(bio);

	blk_rq_unprep_clone(rq);



	return -ENOMEM;

}

EXPORT_SYMBOL_GPL(blk_rq_prep_clone);



static size_t order_to_size(unsigned int order)

{

	return (size_t)PAGE_SIZE << order;

block/blk-mq.h

+0 −3
Original line number Diff line number Diff line
@@ -65,9 +65,6 @@ void blk_mq_request_bypass_insert(struct request *rq, bool at_head, 
				  bool run_queue);

void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,

				struct list_head *list);



/* Used by blk_insert_cloned_request() to issue request directly */

blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last);

void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,

				    struct list_head *list);

block/blk.h

+10 −0
Original line number Diff line number Diff line
@@ -493,4 +493,14 @@ int disk_register_independent_access_ranges(struct gendisk *disk, 
				struct blk_independent_access_ranges *new_iars);

void disk_unregister_independent_access_ranges(struct gendisk *disk);



#ifdef CONFIG_FAIL_MAKE_REQUEST

bool should_fail_request(struct block_device *part, unsigned int bytes);

#else /* CONFIG_FAIL_MAKE_REQUEST */

static inline bool should_fail_request(struct block_device *part,

					unsigned int bytes)

{

	return false;

}

#endif /* CONFIG_FAIL_MAKE_REQUEST */



#endif /* BLK_INTERNAL_H */