Commit fc86f9d3 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: remove redundant memory barrier in io_req_local_work_add 



io_cqring_wake() needs a barrier for the waitqueue_active() check.
However, in the case of io_req_local_work_add(), we call llist_add()
first, which implies an atomic. Hence we can replace smb_mb() with
smp_mb__after_atomic().

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/43983bc8bc507172adda7a0f00cab1aff09fd238.1665018309.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 3fb1bd68

io_uring/io_uring.c

+3 −2
Original line number Diff line number Diff line
@@ -1106,6 +1106,8 @@ static void io_req_local_work_add(struct io_kiocb *req) 


	if (!llist_add(&req->io_task_work.node, &ctx->work_llist))

		return;

	/* need it for the following io_cqring_wake() */

	smp_mb__after_atomic();



	if (unlikely(atomic_read(&req->task->io_uring->in_idle))) {

		io_move_task_work_from_local(ctx);
@@ -1117,8 +1119,7 @@ static void io_req_local_work_add(struct io_kiocb *req) 


	if (ctx->has_evfd)

		io_eventfd_signal(ctx);

	io_cqring_wake(ctx);



	__io_cqring_wake(ctx);

}



static inline void __io_req_task_work_add(struct io_kiocb *req, bool allow_local)

io_uring/io_uring.h

+9 −2
Original line number Diff line number Diff line
@@ -203,17 +203,24 @@ static inline void io_commit_cqring(struct io_ring_ctx *ctx) 
	smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);

}



static inline void io_cqring_wake(struct io_ring_ctx *ctx)

/* requires smb_mb() prior, see wq_has_sleeper() */

static inline void __io_cqring_wake(struct io_ring_ctx *ctx)

{

	/*

	 * wake_up_all() may seem excessive, but io_wake_function() and

	 * io_should_wake() handle the termination of the loop and only

	 * wake as many waiters as we need to.

	 */

	if (wq_has_sleeper(&ctx->cq_wait))

	if (waitqueue_active(&ctx->cq_wait))

		wake_up_all(&ctx->cq_wait);

}



static inline void io_cqring_wake(struct io_ring_ctx *ctx)

{

	smp_mb();

	__io_cqring_wake(ctx);

}



static inline bool io_sqring_full(struct io_ring_ctx *ctx)

{

	struct io_rings *r = ctx->rings;