Commit c75312dd authored by Usama Arif's avatar Usama Arif Committed by Jens Axboe
Browse files

io_uring: avoid ring quiesce while registering async eventfd 



This is done using the RCU data structure (io_ev_fd). eventfd_async is
moved from io_ring_ctx to io_ev_fd which is RCU protected hence avoiding
ring quiesce which is much more expensive than an RCU lock. The place
where eventfd_async is read is already under rcu_read_lock so there is no
extra RCU read-side critical section needed.

Signed-off-by: default avatarUsama Arif <usama.arif@bytedance.com>
Link: https://lore.kernel.org/r/20220204145117.1186568-4-usama.arif@bytedance.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 77bc59b4

fs/io_uring.c

+12 −10
Original line number Diff line number Diff line
@@ -328,6 +328,7 @@ struct io_submit_state { 


struct io_ev_fd {

	struct eventfd_ctx	*cq_ev_fd;

	unsigned int		eventfd_async: 1;

	struct rcu_head		rcu;

};
@@ -340,7 +341,6 @@ struct io_ring_ctx { 
		unsigned int		flags;

		unsigned int		compat: 1;

		unsigned int		drain_next: 1;

		unsigned int		eventfd_async: 1;

		unsigned int		restricted: 1;

		unsigned int		off_timeout_used: 1;

		unsigned int		drain_active: 1;
@@ -1756,7 +1756,7 @@ static void io_eventfd_signal(struct io_ring_ctx *ctx) 
	if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)

		goto out;



	if (!ctx->eventfd_async || io_wq_current_is_worker())

	if (!ev_fd->eventfd_async || io_wq_current_is_worker())

		eventfd_signal(ev_fd->cq_ev_fd, 1);



out:
@@ -9385,7 +9385,8 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, 
	return done ? done : err;

}



static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)

static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg,

			       unsigned int eventfd_async)

{

	struct io_ev_fd *ev_fd;

	__s32 __user *fds = arg;
@@ -9409,6 +9410,7 @@ static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg) 
		kfree(ev_fd);

		return ret;

	}

	ev_fd->eventfd_async = eventfd_async;



	rcu_assign_pointer(ctx->io_ev_fd, ev_fd);

	return ret;
@@ -11014,6 +11016,7 @@ static bool io_register_op_must_quiesce(int op) 
	case IORING_UNREGISTER_FILES:

	case IORING_REGISTER_FILES_UPDATE:

	case IORING_REGISTER_EVENTFD:

	case IORING_REGISTER_EVENTFD_ASYNC:

	case IORING_UNREGISTER_EVENTFD:

	case IORING_REGISTER_PROBE:

	case IORING_REGISTER_PERSONALITY:
@@ -11114,17 +11117,16 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, 
		ret = io_register_files_update(ctx, arg, nr_args);

		break;

	case IORING_REGISTER_EVENTFD:

	case IORING_REGISTER_EVENTFD_ASYNC:

		ret = -EINVAL;

		if (nr_args != 1)

			break;

		ret = io_eventfd_register(ctx, arg);

		if (ret)

		ret = io_eventfd_register(ctx, arg, 0);

		break;

		if (opcode == IORING_REGISTER_EVENTFD_ASYNC)

			ctx->eventfd_async = 1;

		else

			ctx->eventfd_async = 0;

	case IORING_REGISTER_EVENTFD_ASYNC:

		ret = -EINVAL;

		if (nr_args != 1)

			break;

		ret = io_eventfd_register(ctx, arg, 1);

		break;

	case IORING_UNREGISTER_EVENTFD:

		ret = -EINVAL;