Commit 32bc15af authored by John Garry's avatar John Garry Committed by Jens Axboe
Browse files

blk-mq: Facilitate a shared sbitmap per tagset 

Some SCSI HBAs (such as HPSA, megaraid, mpt3sas, hisi_sas_v3 ..) support
multiple reply queues with single hostwide tags.

In addition, these drivers want to use interrupt assignment in
pci_alloc_irq_vectors(PCI_IRQ_AFFINITY). However, as discussed in [0],
CPU hotplug may cause in-flight IO completion to not be serviced when an
interrupt is shutdown. That problem is solved in commit bf0beec0
("blk-mq: drain I/O when all CPUs in a hctx are offline").

However, to take advantage of that blk-mq feature, the HBA HW queuess are
required to be mapped to that of the blk-mq hctx's; to do that, the HBA HW
queues need to be exposed to the upper layer.

In making that transition, the per-SCSI command request tags are no
longer unique per Scsi host - they are just unique per hctx. As such, the
HBA LLDD would have to generate this tag internally, which has a certain
performance overhead.

However another problem is that blk-mq assumes the host may accept
(Scsi_host.can_queue * #hw queue) commands. In commit 6eb045e0 ("scsi:
 core: avoid host-wide host_busy counter for scsi_mq"), the Scsi host busy
counter was removed, which would stop the LLDD being sent more than
.can_queue commands; however, it should still be ensured that the block
layer does not issue more than .can_queue commands to the Scsi host.

To solve this problem, introduce a shared sbitmap per blk_mq_tag_set,
which may be requested at init time.

New flag BLK_MQ_F_TAG_HCTX_SHARED should be set when requesting the
tagset to indicate whether the shared sbitmap should be used.

Even when BLK_MQ_F_TAG_HCTX_SHARED is set, a full set of tags and requests
are still allocated per hctx; the reason for this is that if tags and
requests were only allocated for a single hctx - like hctx0 - it may break
block drivers which expect a request be associated with a specific hctx,
i.e. not always hctx0. This will introduce extra memory usage.

This change is based on work originally from Ming Lei in [1] and from
Bart's suggestion in [2].

[0] https://lore.kernel.org/linux-block/alpine.DEB.2.21.1904051331270.1802@nanos.tec.linutronix.de/
[1] https://lore.kernel.org/linux-block/20190531022801.10003-1-ming.lei@redhat.com/
[2] https://lore.kernel.org/linux-block/ff77beff-5fd9-9f05-12b6-826922bace1f@huawei.com/T/#m3db0a602f095cbcbff27e9c884d6b4ae826144be



Signed-off-by: default avatarJohn Garry <john.garry@huawei.com>
Tested-by: Don Brace<don.brace@microsemi.com> #SCSI resv cmds patches used
Tested-by: default avatarDouglas Gilbert <dgilbert@interlog.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 222a5ae0

block/blk-mq-sched.c

+5 −3
Original line number Diff line number Diff line
@@ -560,7 +560,7 @@ static void blk_mq_sched_free_tags(struct blk_mq_tag_set *set, 
				   struct blk_mq_hw_ctx *hctx,

				   unsigned int hctx_idx)

{

	unsigned int flags = set->flags;

	unsigned int flags = set->flags & ~BLK_MQ_F_TAG_HCTX_SHARED;



	if (hctx->sched_tags) {

		blk_mq_free_rqs(set, hctx->sched_tags, hctx_idx);
@@ -574,7 +574,8 @@ static int blk_mq_sched_alloc_tags(struct request_queue *q, 
				   unsigned int hctx_idx)

{

	struct blk_mq_tag_set *set = q->tag_set;

	unsigned int flags = set->flags;

	/* Clear HCTX_SHARED so tags are init'ed */

	unsigned int flags = set->flags & ~BLK_MQ_F_TAG_HCTX_SHARED;

	int ret;



	hctx->sched_tags = blk_mq_alloc_rq_map(set, hctx_idx, q->nr_requests,
@@ -596,7 +597,8 @@ static void blk_mq_sched_tags_teardown(struct request_queue *q) 
	int i;



	queue_for_each_hw_ctx(q, hctx, i) {

		unsigned int flags = hctx->flags;

		/* Clear HCTX_SHARED so tags are freed */

		unsigned int flags = hctx->flags & ~BLK_MQ_F_TAG_HCTX_SHARED;



		if (hctx->sched_tags) {

			blk_mq_free_rq_map(hctx->sched_tags, flags);

block/blk-mq-tag.c

+47 −4
Original line number Diff line number Diff line
@@ -197,7 +197,7 @@ static bool bt_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) 
	 * We can hit rq == NULL here, because the tagging functions

	 * test and set the bit before assigning ->rqs[].

	 */

	if (rq && rq->q == hctx->queue)

	if (rq && rq->q == hctx->queue && rq->mq_hctx == hctx)

		return iter_data->fn(hctx, rq, iter_data->data, reserved);

	return true;

}
@@ -450,6 +450,38 @@ static int blk_mq_init_bitmap_tags(struct blk_mq_tags *tags, 
	return -ENOMEM;

}



int blk_mq_init_shared_sbitmap(struct blk_mq_tag_set *set, unsigned int flags)

{

	unsigned int depth = set->queue_depth - set->reserved_tags;

	int alloc_policy = BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags);

	bool round_robin = alloc_policy == BLK_TAG_ALLOC_RR;

	int i, node = set->numa_node;



	if (bt_alloc(&set->__bitmap_tags, depth, round_robin, node))

		return -ENOMEM;

	if (bt_alloc(&set->__breserved_tags, set->reserved_tags,

		     round_robin, node))

		goto free_bitmap_tags;



	for (i = 0; i < set->nr_hw_queues; i++) {

		struct blk_mq_tags *tags = set->tags[i];



		tags->bitmap_tags = &set->__bitmap_tags;

		tags->breserved_tags = &set->__breserved_tags;

	}



	return 0;

free_bitmap_tags:

	sbitmap_queue_free(&set->__bitmap_tags);

	return -ENOMEM;

}



void blk_mq_exit_shared_sbitmap(struct blk_mq_tag_set *set)

{

	sbitmap_queue_free(&set->__bitmap_tags);

	sbitmap_queue_free(&set->__breserved_tags);

}



struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags,

				     unsigned int reserved_tags,

				     int node, unsigned int flags)
@@ -469,6 +501,9 @@ struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags, 
	tags->nr_tags = total_tags;

	tags->nr_reserved_tags = reserved_tags;



	if (flags & BLK_MQ_F_TAG_HCTX_SHARED)

		return tags;



	if (blk_mq_init_bitmap_tags(tags, node, alloc_policy) < 0) {

		kfree(tags);

		return NULL;
@@ -478,8 +513,10 @@ struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags, 


void blk_mq_free_tags(struct blk_mq_tags *tags, unsigned int flags)

{

	sbitmap_queue_free(&tags->__bitmap_tags);

	sbitmap_queue_free(&tags->__breserved_tags);

	if (!(flags & BLK_MQ_F_TAG_HCTX_SHARED)) {

		sbitmap_queue_free(tags->bitmap_tags);

		sbitmap_queue_free(tags->breserved_tags);

	}

	kfree(tags);

}
@@ -498,7 +535,8 @@ int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx, 
	 */

	if (tdepth > tags->nr_tags) {

		struct blk_mq_tag_set *set = hctx->queue->tag_set;

		unsigned int flags = set->flags;

		/* Only sched tags can grow, so clear HCTX_SHARED flag  */

		unsigned int flags = set->flags & ~BLK_MQ_F_TAG_HCTX_SHARED;

		struct blk_mq_tags *new;

		bool ret;
@@ -537,6 +575,11 @@ int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx, 
	return 0;

}



void blk_mq_tag_resize_shared_sbitmap(struct blk_mq_tag_set *set, unsigned int size)

{

	sbitmap_queue_resize(&set->__bitmap_tags, size - set->reserved_tags);

}



/**

 * blk_mq_unique_tag() - return a tag that is unique queue-wide

 * @rq: request for which to compute a unique tag

block/blk-mq-tag.h

+7 −0
Original line number Diff line number Diff line
@@ -29,12 +29,19 @@ extern struct blk_mq_tags *blk_mq_init_tags(unsigned int nr_tags, 
					int node, unsigned int flags);

extern void blk_mq_free_tags(struct blk_mq_tags *tags, unsigned int flags);



extern int blk_mq_init_shared_sbitmap(struct blk_mq_tag_set *set,

				      unsigned int flags);

extern void blk_mq_exit_shared_sbitmap(struct blk_mq_tag_set *set);



extern unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data);

extern void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx,

			   unsigned int tag);

extern int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx,

					struct blk_mq_tags **tags,

					unsigned int depth, bool can_grow);

extern void blk_mq_tag_resize_shared_sbitmap(struct blk_mq_tag_set *set,

					     unsigned int size);



extern void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool);

void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,

		void *priv);

block/blk-mq.c

+15 −0
Original line number Diff line number Diff line
@@ -3441,11 +3441,21 @@ int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set) 
	if (ret)

		goto out_free_mq_map;



	if (blk_mq_is_sbitmap_shared(set->flags)) {

		if (blk_mq_init_shared_sbitmap(set, set->flags)) {

			ret = -ENOMEM;

			goto out_free_mq_rq_maps;

		}

	}



	mutex_init(&set->tag_list_lock);

	INIT_LIST_HEAD(&set->tag_list);



	return 0;



out_free_mq_rq_maps:

	for (i = 0; i < set->nr_hw_queues; i++)

		blk_mq_free_map_and_requests(set, i);

out_free_mq_map:

	for (i = 0; i < set->nr_maps; i++) {

		kfree(set->map[i].mq_map);
@@ -3464,6 +3474,9 @@ void blk_mq_free_tag_set(struct blk_mq_tag_set *set) 
	for (i = 0; i < set->nr_hw_queues; i++)

		blk_mq_free_map_and_requests(set, i);



	if (blk_mq_is_sbitmap_shared(set->flags))

		blk_mq_exit_shared_sbitmap(set);



	for (j = 0; j < set->nr_maps; j++) {

		kfree(set->map[j].mq_map);

		set->map[j].mq_map = NULL;
@@ -3500,6 +3513,8 @@ int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr) 
		if (!hctx->sched_tags) {

			ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr,

							false);

			if (!ret && blk_mq_is_sbitmap_shared(set->flags))

				blk_mq_tag_resize_shared_sbitmap(set, nr);

		} else {

			ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,

							nr, true);

block/blk-mq.h

+5 −0
Original line number Diff line number Diff line
@@ -159,6 +159,11 @@ struct blk_mq_alloc_data { 
	struct blk_mq_hw_ctx *hctx;

};



static inline bool blk_mq_is_sbitmap_shared(unsigned int flags)

{

	return flags & BLK_MQ_F_TAG_HCTX_SHARED;

}



static inline struct blk_mq_tags *blk_mq_tags_from_data(struct blk_mq_alloc_data *data)

{

	if (data->q->elevator)