Commit dc5fc361 authored by Jens Axboe's avatar Jens Axboe
Browse files

block: attempt direct issue of plug list 



If we have just one queue type in the plug list, then we can extend our
direct issue to cover a full plug list as well. This allows sending a
batch of requests for direct issue, which is more efficient than doing
one-at-a-time kind of issue.

Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent bc490f81

block/blk-core.c

+1 −0
Original line number Original line Diff line number Diff line
@@ -1555,6 +1555,7 @@ void blk_start_plug_nr_ios(struct blk_plug *plug, unsigned short nr_ios) 
	plug->nr_ios = min_t(unsigned short, nr_ios, BLK_MAX_REQUEST_COUNT);

	plug->nr_ios = min_t(unsigned short, nr_ios, BLK_MAX_REQUEST_COUNT);

	plug->rq_count = 0;

	plug->rq_count = 0;

	plug->multiple_queues = false;

	plug->multiple_queues = false;

	plug->has_elevator = false;

	plug->nowait = false;

	plug->nowait = false;

	INIT_LIST_HEAD(&plug->cb_list);

	INIT_LIST_HEAD(&plug->cb_list);

block/blk-mq.c

+60 −0
Original line number Original line Diff line number Diff line
@@ -2149,6 +2149,58 @@ void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx, 
	spin_unlock(&ctx->lock);

	spin_unlock(&ctx->lock);

}

}





static void blk_mq_commit_rqs(struct blk_mq_hw_ctx *hctx, int *queued,

			      bool from_schedule)

{

	if (hctx->queue->mq_ops->commit_rqs) {

		trace_block_unplug(hctx->queue, *queued, !from_schedule);

		hctx->queue->mq_ops->commit_rqs(hctx);

	}

	*queued = 0;

}



static void blk_mq_plug_issue_direct(struct blk_plug *plug, bool from_schedule)

{

	struct blk_mq_hw_ctx *hctx = NULL;

	struct request *rq;

	int queued = 0;

	int errors = 0;



	while ((rq = rq_list_pop(&plug->mq_list))) {

		bool last = rq_list_empty(plug->mq_list);

		blk_status_t ret;



		if (hctx != rq->mq_hctx) {

			if (hctx)

				blk_mq_commit_rqs(hctx, &queued, from_schedule);

			hctx = rq->mq_hctx;

		}



		ret = blk_mq_request_issue_directly(rq, last);

		switch (ret) {

		case BLK_STS_OK:

			queued++;

			break;

		case BLK_STS_RESOURCE:

		case BLK_STS_DEV_RESOURCE:

			blk_mq_request_bypass_insert(rq, false, last);

			blk_mq_commit_rqs(hctx, &queued, from_schedule);

			return;

		default:

			blk_mq_end_request(rq, ret);

			errors++;

			break;

		}

	}



	/*

	 * If we didn't flush the entire list, we could have told the driver

	 * there was more coming, but that turned out to be a lie.

	 */

	if (errors)

		blk_mq_commit_rqs(hctx, &queued, from_schedule);

}



void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)

void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)

{

{

	struct blk_mq_hw_ctx *this_hctx;

	struct blk_mq_hw_ctx *this_hctx;
@@ -2160,6 +2212,12 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) 
		return;

		return;

	plug->rq_count = 0;

	plug->rq_count = 0;





	if (!plug->multiple_queues && !plug->has_elevator) {

		blk_mq_plug_issue_direct(plug, from_schedule);

		if (rq_list_empty(plug->mq_list))

			return;

	}



	this_hctx = NULL;

	this_hctx = NULL;

	this_ctx = NULL;

	this_ctx = NULL;

	depth = 0;

	depth = 0;
@@ -2376,6 +2434,8 @@ static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq) 
		if (nxt && nxt->q != rq->q)

		if (nxt && nxt->q != rq->q)

			plug->multiple_queues = true;

			plug->multiple_queues = true;

	}

	}

	if (!plug->has_elevator && (rq->rq_flags & RQF_ELV))

		plug->has_elevator = true;

	rq->rq_next = NULL;

	rq->rq_next = NULL;

	rq_list_add(&plug->mq_list, rq);

	rq_list_add(&plug->mq_list, rq);

	plug->rq_count++;

	plug->rq_count++;

include/linux/blkdev.h

+1 −0
Original line number Original line Diff line number Diff line
@@ -737,6 +737,7 @@ struct blk_plug { 
	unsigned short rq_count;

	unsigned short rq_count;





	bool multiple_queues;

	bool multiple_queues;

	bool has_elevator;

	bool nowait;

	bool nowait;





	struct list_head cb_list; /* md requires an unplug callback */

	struct list_head cb_list; /* md requires an unplug callback */