Commit faf88dde authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: don't inline __io_get_cqe() 



__io_get_cqe() is not as hot as io_get_cqe(), no need to inline it, it
sheds ~500B from the binary.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/c1ac829198a881b7af8710926f99a3559b9f24c0.1655455613.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent d245bca6

io_uring/io_uring.c

+35 −0
Original line number Diff line number Diff line
@@ -166,6 +166,11 @@ static inline void io_submit_flush_completions(struct io_ring_ctx *ctx) 
		__io_submit_flush_completions(ctx);

}



static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)

{

	return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);

}



static bool io_match_linked(struct io_kiocb *head)

{

	struct io_kiocb *req;
@@ -676,6 +681,36 @@ bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, s32 res, 
	return true;

}



/*

 * writes to the cq entry need to come after reading head; the

 * control dependency is enough as we're using WRITE_ONCE to

 * fill the cq entry

 */

struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx)

{

	struct io_rings *rings = ctx->rings;

	unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1);

	unsigned int shift = 0;

	unsigned int free, queued, len;



	if (ctx->flags & IORING_SETUP_CQE32)

		shift = 1;



	/* userspace may cheat modifying the tail, be safe and do min */

	queued = min(__io_cqring_events(ctx), ctx->cq_entries);

	free = ctx->cq_entries - queued;

	/* we need a contiguous range, limit based on the current array offset */

	len = min(free, ctx->cq_entries - off);

	if (!len)

		return NULL;



	ctx->cached_cq_tail++;

	ctx->cqe_cached = &rings->cqes[off];

	ctx->cqe_sentinel = ctx->cqe_cached + len;

	ctx->cqe_cached++;

	return &rings->cqes[off << shift];

}



static bool io_fill_cqe_aux(struct io_ring_ctx *ctx,

			    u64 user_data, s32 res, u32 cflags)

{

io_uring/io_uring.h

+1 −35
Original line number Diff line number Diff line
@@ -14,44 +14,10 @@ enum { 
	IOU_ISSUE_SKIP_COMPLETE	= -EIOCBQUEUED,

};



struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx);

bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, s32 res,

			      u32 cflags, u64 extra1, u64 extra2);



static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)

{

	return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);

}



/*

 * writes to the cq entry need to come after reading head; the

 * control dependency is enough as we're using WRITE_ONCE to

 * fill the cq entry

 */

static inline struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx)

{

	struct io_rings *rings = ctx->rings;

	unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1);

	unsigned int shift = 0;

	unsigned int free, queued, len;



	if (ctx->flags & IORING_SETUP_CQE32)

		shift = 1;



	/* userspace may cheat modifying the tail, be safe and do min */

	queued = min(__io_cqring_events(ctx), ctx->cq_entries);

	free = ctx->cq_entries - queued;

	/* we need a contiguous range, limit based on the current array offset */

	len = min(free, ctx->cq_entries - off);

	if (!len)

		return NULL;



	ctx->cached_cq_tail++;

	ctx->cqe_cached = &rings->cqes[off];

	ctx->cqe_sentinel = ctx->cqe_cached + len;

	ctx->cqe_cached++;

	return &rings->cqes[off << shift];

}



static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)

{

	if (likely(ctx->cqe_cached < ctx->cqe_sentinel)) {