Commit d245bca6 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: don't expose io_fill_cqe_aux() 



Deduplicate some code and add a helper for filling an aux CQE, locking
and notification.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/b7c6557c8f9dc5c4cfb01292116c682a0ff61081.1655455613.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent f09c8643

io_uring/io_uring.c

+16 −2
Original line number Diff line number Diff line
@@ -676,8 +676,8 @@ bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, s32 res, 
	return true;

}



bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res,

		     u32 cflags)

static bool io_fill_cqe_aux(struct io_ring_ctx *ctx,

			    u64 user_data, s32 res, u32 cflags)

{

	struct io_uring_cqe *cqe;
@@ -704,6 +704,20 @@ bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res, 
	return io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0);

}



bool io_post_aux_cqe(struct io_ring_ctx *ctx,

		     u64 user_data, s32 res, u32 cflags)

{

	bool filled;



	spin_lock(&ctx->completion_lock);

	filled = io_fill_cqe_aux(ctx, user_data, res, cflags);

	io_commit_cqring(ctx);

	spin_unlock(&ctx->completion_lock);

	if (filled)

		io_cqring_ev_posted(ctx);

	return filled;

}



static void __io_req_complete_put(struct io_kiocb *req)

{

	/*

io_uring/io_uring.h

+1 −2
Original line number Diff line number Diff line
@@ -239,8 +239,7 @@ void io_req_complete_failed(struct io_kiocb *req, s32 res); 
void __io_req_complete(struct io_kiocb *req, unsigned issue_flags);

void io_req_complete_post(struct io_kiocb *req);

void __io_req_complete_post(struct io_kiocb *req);

bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res,

		     u32 cflags);

bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags);

void io_cqring_ev_posted(struct io_ring_ctx *ctx);

void __io_commit_cqring_flush(struct io_ring_ctx *ctx);

io_uring/msg_ring.c

+1 −10
Original line number Diff line number Diff line
@@ -34,7 +34,6 @@ int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags) 
{

	struct io_msg *msg = io_kiocb_to_cmd(req);

	struct io_ring_ctx *target_ctx;

	bool filled;

	int ret;



	ret = -EBADFD;
@@ -43,16 +42,8 @@ int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags) 


	ret = -EOVERFLOW;

	target_ctx = req->file->private_data;



	spin_lock(&target_ctx->completion_lock);

	filled = io_fill_cqe_aux(target_ctx, msg->user_data, msg->len, 0);

	io_commit_cqring(target_ctx);

	spin_unlock(&target_ctx->completion_lock);



	if (filled) {

		io_cqring_ev_posted(target_ctx);

	if (io_post_aux_cqe(target_ctx, msg->user_data, msg->len, 0))

		ret = 0;

	}



done:

	if (ret < 0)

io_uring/net.c

+5 −15
Original line number Diff line number Diff line
@@ -644,22 +644,12 @@ int io_accept(struct io_kiocb *req, unsigned int issue_flags) 
		io_req_set_res(req, ret, 0);

		return IOU_OK;

	}

	if (ret >= 0) {

		bool filled;



		spin_lock(&ctx->completion_lock);

		filled = io_fill_cqe_aux(ctx, req->cqe.user_data, ret,

					 IORING_CQE_F_MORE);

		io_commit_cqring(ctx);

		spin_unlock(&ctx->completion_lock);

		if (filled) {

			io_cqring_ev_posted(ctx);

			goto retry;

		}

		ret = -ECANCELED;

	}



	if (ret < 0)

		return ret;

	if (io_post_aux_cqe(ctx, req->cqe.user_data, ret, IORING_CQE_F_MORE))

		goto retry;

	return -ECANCELED;

}



int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)

io_uring/poll.c

+8 −16
Original line number Diff line number Diff line
@@ -214,23 +214,15 @@ static int io_poll_check_events(struct io_kiocb *req, bool *locked) 
		if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {

			__poll_t mask = mangle_poll(req->cqe.res &

						    req->apoll_events);

			bool filled;



			spin_lock(&ctx->completion_lock);

			filled = io_fill_cqe_aux(ctx, req->cqe.user_data,

						 mask, IORING_CQE_F_MORE);

			io_commit_cqring(ctx);

			spin_unlock(&ctx->completion_lock);

			if (filled) {

				io_cqring_ev_posted(ctx);

				continue;

			}

			return -ECANCELED;

		}



			if (!io_post_aux_cqe(ctx, req->cqe.user_data,

					     mask, IORING_CQE_F_MORE))

				return -ECANCELED;

		} else {

			ret = io_poll_issue(req, locked);

			if (ret)

				return ret;

		}



		/*

		 * Release all references, retry if someone tried to restart