Commit a38d68db authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: help inlining of io_req_complete() 



__io_req_complete() inlining is a bit weird, some compilers don't
optimise out the non-NULL branch of it even when called as
io_req_complete(). Help it a bit by extracting state and stateless
helpers out of __io_req_complete().

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 8662daec

fs/io_uring.c

+21 −15
Original line number Diff line number Diff line
@@ -1886,7 +1886,8 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res) 
	__io_cqring_fill_event(req, res, 0);

}



static void io_cqring_add_event(struct io_kiocb *req, long res, long cflags)

static void io_req_complete_nostate(struct io_kiocb *req, long res,

				    unsigned int cflags)

{

	struct io_ring_ctx *ctx = req->ctx;

	unsigned long flags;
@@ -1897,6 +1898,7 @@ static void io_cqring_add_event(struct io_kiocb *req, long res, long cflags) 
	spin_unlock_irqrestore(&ctx->completion_lock, flags);



	io_cqring_ev_posted(ctx);

	io_put_req(req);

}



static void io_submit_flush_completions(struct io_comp_state *cs)
@@ -1932,13 +1934,9 @@ static void io_submit_flush_completions(struct io_comp_state *cs) 
	cs->nr = 0;

}



static void __io_req_complete(struct io_kiocb *req, long res, unsigned cflags,

			      struct io_comp_state *cs)

static void io_req_complete_state(struct io_kiocb *req, long res,

				  unsigned int cflags, struct io_comp_state *cs)

{

	if (!cs) {

		io_cqring_add_event(req, res, cflags);

		io_put_req(req);

	} else {

	io_clean_op(req);

	req->result = res;

	req->compl.cflags = cflags;
@@ -1946,9 +1944,17 @@ static void __io_req_complete(struct io_kiocb *req, long res, unsigned cflags, 
	if (++cs->nr >= 32)

		io_submit_flush_completions(cs);

}



static inline void __io_req_complete(struct io_kiocb *req, long res,

				     unsigned cflags, struct io_comp_state *cs)

{

	if (!cs)

		io_req_complete_nostate(req, res, cflags);

	else

		io_req_complete_state(req, res, cflags, cs);

}



static void io_req_complete(struct io_kiocb *req, long res)

static inline void io_req_complete(struct io_kiocb *req, long res)

{

	__io_req_complete(req, res, 0, NULL);

}