Commit 6dd0be1e authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: replace list with array for compl batch 



Reincarnation of an old patch that replaces a list in struct
io_compl_batch with an array. It's needed to avoid hooking requests via
their compl.list, because it won't be always available in the future.

It's also nice to split io_submit_flush_completions() to avoid free
under locks and remove unlock/lock with a long comment describing when
it can be done.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 5087275d

fs/io_uring.c

+11 −24
Original line number Diff line number Diff line
@@ -265,10 +265,11 @@ struct io_sq_data { 
};



#define IO_IOPOLL_BATCH			8

#define IO_COMPL_BATCH			32



struct io_comp_state {

	unsigned int		nr;

	struct list_head	list;

	struct io_kiocb		*reqs[IO_COMPL_BATCH];

};



struct io_submit_state {
@@ -1345,7 +1346,6 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) 
	INIT_LIST_HEAD(&ctx->rsrc_ref_list);

	INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work);

	init_llist_head(&ctx->rsrc_put_llist);

	INIT_LIST_HEAD(&submit_state->comp.list);

	return ctx;

err:

	if (ctx->fallback_req)
@@ -1927,33 +1927,20 @@ static inline void io_req_complete_nostate(struct io_kiocb *req, long res, 
static void io_submit_flush_completions(struct io_comp_state *cs,

					struct io_ring_ctx *ctx)

{

	int i, nr = cs->nr;



	spin_lock_irq(&ctx->completion_lock);

	while (!list_empty(&cs->list)) {

		struct io_kiocb *req;

	for (i = 0; i < nr; i++) {

		struct io_kiocb *req = cs->reqs[i];



		req = list_first_entry(&cs->list, struct io_kiocb, compl.list);

		list_del(&req->compl.list);

		__io_cqring_fill_event(req, req->result, req->compl.cflags);



		/*

		 * io_free_req() doesn't care about completion_lock unless one

		 * of these flags is set. REQ_F_WORK_INITIALIZED is in the list

		 * because of a potential deadlock with req->work.fs->lock

		 * We defer both, completion and submission refs.

		 */

		if (req->flags & (REQ_F_FAIL_LINK|REQ_F_LINK_TIMEOUT

				 |REQ_F_WORK_INITIALIZED)) {

			spin_unlock_irq(&ctx->completion_lock);

			io_double_put_req(req);

			spin_lock_irq(&ctx->completion_lock);

		} else {

			io_double_put_req(req);

		}

	}

	io_commit_cqring(ctx);

	spin_unlock_irq(&ctx->completion_lock);



	io_cqring_ev_posted(ctx);

	for (i = 0; i < nr; i++)

		io_double_put_req(cs->reqs[i]);

	cs->nr = 0;

}
@@ -6517,8 +6504,8 @@ static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs) 
	} else if (likely(!ret)) {

		/* drop submission reference */

		if (req->flags & REQ_F_COMPLETE_INLINE) {

			list_add_tail(&req->compl.list, &cs->list);

			if (++cs->nr >= 32)

			cs->reqs[cs->nr++] = req;

			if (cs->nr == IO_COMPL_BATCH)

				io_submit_flush_completions(cs, req->ctx);

			req = NULL;

		} else {
@@ -6657,7 +6644,7 @@ static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, 
static void io_submit_state_end(struct io_submit_state *state,

				struct io_ring_ctx *ctx)

{

	if (!list_empty(&state->comp.list))

	if (state->comp.nr)

		io_submit_flush_completions(&state->comp, ctx);

	if (state->plug_started)

		blk_finish_plug(&state->plug);