Commit 196be95c authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: allocate io_kiocb upfront 



Let io_submit_sqes() to allocate io_kiocb before fetching an sqe.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent e5eb6366

fs/io_uring.c

+15 −13
Original line number Diff line number Diff line
@@ -2551,30 +2551,23 @@ static int io_queue_link_head(struct io_ring_ctx *ctx, struct io_kiocb *req, 


#define SQE_VALID_FLAGS	(IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK)



static void io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,

			  struct io_submit_state *state, struct io_kiocb **link)

static void io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,

			  struct sqe_submit *s, struct io_submit_state *state,

			  struct io_kiocb **link)

{

	struct io_uring_sqe *sqe_copy;

	struct io_kiocb *req;

	int ret;



	/* enforce forwards compatibility on users */

	if (unlikely(s->sqe->flags & ~SQE_VALID_FLAGS)) {

		ret = -EINVAL;

		goto err;

	}



	req = io_get_req(ctx, state);

	if (unlikely(!req)) {

		ret = -EAGAIN;

		goto err;

		goto err_req;

	}



	ret = io_req_set_file(ctx, s, state, req);

	if (unlikely(ret)) {

err_req:

		io_free_req(req, NULL);

err:

		io_cqring_add_event(ctx, s->sqe->user_data, ret);

		return;

	}
@@ -2710,9 +2703,18 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr, 


	for (i = 0; i < nr; i++) {

		struct sqe_submit s;

		struct io_kiocb *req;



		if (!io_get_sqring(ctx, &s))

		req = io_get_req(ctx, statep);

		if (unlikely(!req)) {

			if (!submitted)

				submitted = -EAGAIN;

			break;

		}

		if (!io_get_sqring(ctx, &s)) {

			__io_free_req(req);

			break;

		}



		if (io_sqe_needs_user(s.sqe) && !*mm) {

			mm_fault = mm_fault || !mmget_not_zero(ctx->sqo_mm);
@@ -2740,7 +2742,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr, 
		s.in_async = async;

		s.needs_fixed_file = async;

		trace_io_uring_submit_sqe(ctx, s.sqe->user_data, true, async);

		io_submit_sqe(ctx, &s, statep, &link);

		io_submit_sqe(ctx, req, &s, statep, &link);

		submitted++;



		/*