Commit 65c76369 authored by Ming Lei's avatar Ming Lei Committed by Jens Axboe
Browse files

blk-mq: pass request queue into get/put budget callback 



blk-mq budget is abstract from scsi's device queue depth, and it is
always per-request-queue instead of hctx.

It can be quite absurd to get a budget from one hctx, then dequeue a
request from scheduler queue, and this request may not belong to this
hctx, at least for bfq and deadline.

So fix the mess and always pass request queue to get/put budget
callback.

Signed-off-by: default avatarMing Lei <ming.lei@redhat.com>
Tested-by: default avatarBaolin Wang <baolin.wang7@gmail.com>
Reviewed-by: default avatarJohannes Thumshirn <johannes.thumshirn@wdc.com>
Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Reviewed-by: default avatarDouglas Anderson <dianders@chromium.org>
Reviewed-by: default avatarSagi Grimberg <sagi@grimberg.me>
Cc: Sagi Grimberg <sagi@grimberg.me>
Cc: Baolin Wang <baolin.wang7@gmail.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Douglas Anderson <dianders@chromium.org>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 42fdc5e4

block/blk-mq-sched.c

+4 −4
Original line number Diff line number Diff line
@@ -108,12 +108,12 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx) 
			break;

		}



		if (!blk_mq_get_dispatch_budget(hctx))

		if (!blk_mq_get_dispatch_budget(q))

			break;



		rq = e->type->ops.dispatch_request(hctx);

		if (!rq) {

			blk_mq_put_dispatch_budget(hctx);

			blk_mq_put_dispatch_budget(q);

			/*

			 * We're releasing without dispatching. Holding the

			 * budget could have blocked any "hctx"s with the
@@ -173,12 +173,12 @@ static int blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx) 
		if (!sbitmap_any_bit_set(&hctx->ctx_map))

			break;



		if (!blk_mq_get_dispatch_budget(hctx))

		if (!blk_mq_get_dispatch_budget(q))

			break;



		rq = blk_mq_dequeue_from_ctx(hctx, ctx);

		if (!rq) {

			blk_mq_put_dispatch_budget(hctx);

			blk_mq_put_dispatch_budget(q);

			/*

			 * We're releasing without dispatching. Holding the

			 * budget could have blocked any "hctx"s with the

block/blk-mq.c

+4 −4
Original line number Diff line number Diff line
@@ -1284,7 +1284,7 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list, 
		rq = list_first_entry(list, struct request, queuelist);



		hctx = rq->mq_hctx;

		if (!got_budget && !blk_mq_get_dispatch_budget(hctx)) {

		if (!got_budget && !blk_mq_get_dispatch_budget(q)) {

			blk_mq_put_driver_tag(rq);

			no_budget_avail = true;

			break;
@@ -1299,7 +1299,7 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list, 
			 * we'll re-run it below.

			 */

			if (!blk_mq_mark_tag_wait(hctx, rq)) {

				blk_mq_put_dispatch_budget(hctx);

				blk_mq_put_dispatch_budget(q);

				/*

				 * For non-shared tags, the RESTART check

				 * will suffice.
@@ -1947,11 +1947,11 @@ static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, 
	if (q->elevator && !bypass_insert)

		goto insert;



	if (!blk_mq_get_dispatch_budget(hctx))

	if (!blk_mq_get_dispatch_budget(q))

		goto insert;



	if (!blk_mq_get_driver_tag(rq)) {

		blk_mq_put_dispatch_budget(hctx);

		blk_mq_put_dispatch_budget(q);

		goto insert;

	}

block/blk-mq.h

+4 −8
Original line number Diff line number Diff line
@@ -179,20 +179,16 @@ unsigned int blk_mq_in_flight(struct request_queue *q, struct hd_struct *part); 
void blk_mq_in_flight_rw(struct request_queue *q, struct hd_struct *part,

			 unsigned int inflight[2]);



static inline void blk_mq_put_dispatch_budget(struct blk_mq_hw_ctx *hctx)

static inline void blk_mq_put_dispatch_budget(struct request_queue *q)

{

	struct request_queue *q = hctx->queue;



	if (q->mq_ops->put_budget)

		q->mq_ops->put_budget(hctx);

		q->mq_ops->put_budget(q);

}



static inline bool blk_mq_get_dispatch_budget(struct blk_mq_hw_ctx *hctx)

static inline bool blk_mq_get_dispatch_budget(struct request_queue *q)

{

	struct request_queue *q = hctx->queue;



	if (q->mq_ops->get_budget)

		return q->mq_ops->get_budget(hctx);

		return q->mq_ops->get_budget(q);

	return true;

}

drivers/scsi/scsi_lib.c

+3 −5
Original line number Diff line number Diff line
@@ -1597,17 +1597,15 @@ static void scsi_mq_done(struct scsi_cmnd *cmd) 
	blk_mq_complete_request(cmd->request);

}



static void scsi_mq_put_budget(struct blk_mq_hw_ctx *hctx)

static void scsi_mq_put_budget(struct request_queue *q)

{

	struct request_queue *q = hctx->queue;

	struct scsi_device *sdev = q->queuedata;



	atomic_dec(&sdev->device_busy);

}



static bool scsi_mq_get_budget(struct blk_mq_hw_ctx *hctx)

static bool scsi_mq_get_budget(struct request_queue *q)

{

	struct request_queue *q = hctx->queue;

	struct scsi_device *sdev = q->queuedata;



	return scsi_dev_queue_ready(q, sdev);
@@ -1674,7 +1672,7 @@ static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx, 
	if (scsi_target(sdev)->can_queue > 0)

		atomic_dec(&scsi_target(sdev)->target_busy);

out_put_budget:

	scsi_mq_put_budget(hctx);

	scsi_mq_put_budget(q);

	switch (ret) {

	case BLK_STS_OK:

		break;

include/linux/blk-mq.h

+2 −2
Original line number Diff line number Diff line
@@ -270,8 +270,8 @@ struct blk_mq_queue_data { 
typedef blk_status_t (queue_rq_fn)(struct blk_mq_hw_ctx *,

		const struct blk_mq_queue_data *);

typedef void (commit_rqs_fn)(struct blk_mq_hw_ctx *);

typedef bool (get_budget_fn)(struct blk_mq_hw_ctx *);

typedef void (put_budget_fn)(struct blk_mq_hw_ctx *);

typedef bool (get_budget_fn)(struct request_queue *);

typedef void (put_budget_fn)(struct request_queue *);

typedef enum blk_eh_timer_return (timeout_fn)(struct request *, bool);

typedef int (init_hctx_fn)(struct blk_mq_hw_ctx *, void *, unsigned int);

typedef void (exit_hctx_fn)(struct blk_mq_hw_ctx *, unsigned int);