Commit e028f167 authored by Jens Axboe's avatar Jens Axboe
Browse files

block: move blk_mq_tag_to_rq() inline 



This is in the fast path of driver issue or completion, and it's a single
array index operation. Move it inline to avoid a function call for it.

This does mean making struct blk_mq_tags block layer public, but there's
not really much in there.

Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent df87eb0f

block/blk-mq-tag.h

+0 −23
Original line number Diff line number Diff line
@@ -4,29 +4,6 @@ 


struct blk_mq_alloc_data;



/*

 * Tag address space map.

 */

struct blk_mq_tags {

	unsigned int nr_tags;

	unsigned int nr_reserved_tags;



	atomic_t active_queues;



	struct sbitmap_queue bitmap_tags;

	struct sbitmap_queue breserved_tags;



	struct request **rqs;

	struct request **static_rqs;

	struct list_head page_list;



	/*

	 * used to clear request reference in rqs[] before freeing one

	 * request pool

	 */

	spinlock_t lock;

};



extern struct blk_mq_tags *blk_mq_init_tags(unsigned int nr_tags,

					unsigned int reserved_tags,

					int node, int alloc_policy);

block/blk-mq.c

+0 −11
Original line number Diff line number Diff line
@@ -1120,17 +1120,6 @@ void blk_mq_delay_kick_requeue_list(struct request_queue *q, 
}

EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);



struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)

{

	if (tag < tags->nr_tags) {

		prefetch(tags->rqs[tag]);

		return tags->rqs[tag];

	}



	return NULL;

}

EXPORT_SYMBOL(blk_mq_tag_to_rq);



static bool blk_mq_rq_inflight(struct blk_mq_hw_ctx *hctx, struct request *rq,

			       void *priv, bool reserved)

{

include/linux/blk-mq.h

+35 −1
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ 
#include <linux/srcu.h>

#include <linux/lockdep.h>

#include <linux/scatterlist.h>

#include <linux/prefetch.h>



struct blk_mq_tags;

struct blk_flush_queue;
@@ -675,7 +676,40 @@ struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op, 
struct request *blk_mq_alloc_request_hctx(struct request_queue *q,

		unsigned int op, blk_mq_req_flags_t flags,

		unsigned int hctx_idx);

struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag);



/*

 * Tag address space map.

 */

struct blk_mq_tags {

	unsigned int nr_tags;

	unsigned int nr_reserved_tags;



	atomic_t active_queues;



	struct sbitmap_queue bitmap_tags;

	struct sbitmap_queue breserved_tags;



	struct request **rqs;

	struct request **static_rqs;

	struct list_head page_list;



	/*

	 * used to clear request reference in rqs[] before freeing one

	 * request pool

	 */

	spinlock_t lock;

};



static inline struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags,

					       unsigned int tag)

{

	if (tag < tags->nr_tags) {

		prefetch(tags->rqs[tag]);

		return tags->rqs[tag];

	}



	return NULL;

}



enum {

	BLK_MQ_UNIQUE_TAG_BITS = 16,