Commit 0aa7aa5f authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: move multishot cqe cache in ctx 



We cache multishot CQEs before flushing them to the CQ in
submit_state.cqe. It's a 16 entry cache totalling 256 bytes in the
middle of the io_submit_state structure. Move it out of there, it
should help with CPU caches for the submission state, and shouldn't
affect cached CQEs.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/dbe1f39c043ee23da918836be44fcec252ce6711.1692916914.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent c9def23d

include/linux/io_uring_types.h

+2 −1
Original line number Diff line number Diff line
@@ -176,7 +176,6 @@ struct io_submit_state { 
	unsigned short		submit_nr;

	unsigned int		cqes_count;

	struct blk_plug		plug;

	struct io_uring_cqe	cqes[16];

};



struct io_ev_fd {
@@ -307,6 +306,8 @@ struct io_ring_ctx { 
		unsigned		cq_last_tm_flush;

	} ____cacheline_aligned_in_smp;



	struct io_uring_cqe	completion_cqes[16];



	/* IRQ completion list, under ->completion_lock */

	struct io_wq_work_list	locked_free_list;

	unsigned int		locked_free_nr;

io_uring/io_uring.c

+3 −3
Original line number Diff line number Diff line
@@ -880,7 +880,7 @@ static void __io_flush_post_cqes(struct io_ring_ctx *ctx) 


	lockdep_assert_held(&ctx->uring_lock);

	for (i = 0; i < state->cqes_count; i++) {

		struct io_uring_cqe *cqe = &state->cqes[i];

		struct io_uring_cqe *cqe = &ctx->completion_cqes[i];



		if (!io_fill_cqe_aux(ctx, cqe->user_data, cqe->res, cqe->flags)) {

			if (ctx->task_complete) {
@@ -931,7 +931,7 @@ bool io_fill_cqe_req_aux(struct io_kiocb *req, bool defer, s32 res, u32 cflags) 


	lockdep_assert_held(&ctx->uring_lock);



	if (ctx->submit_state.cqes_count == ARRAY_SIZE(ctx->submit_state.cqes)) {

	if (ctx->submit_state.cqes_count == ARRAY_SIZE(ctx->completion_cqes)) {

		__io_cq_lock(ctx);

		__io_flush_post_cqes(ctx);

		/* no need to flush - flush is deferred */
@@ -945,7 +945,7 @@ bool io_fill_cqe_req_aux(struct io_kiocb *req, bool defer, s32 res, u32 cflags) 
	if (test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq))

		return false;



	cqe = &ctx->submit_state.cqes[ctx->submit_state.cqes_count++];

	cqe = &ctx->completion_cqes[ctx->submit_state.cqes_count++];

	cqe->user_data = user_data;

	cqe->res = res;

	cqe->flags = cflags;