Commit 9532b99b authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: optimise rw complete error handling 



Expect read/write to succeed and create a hot path for this case, in
particular hide all error handling with resubmission under a single
check with the desired result.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent ab454438

fs/io_uring.c

+18 −15
Original line number Diff line number Diff line
@@ -2487,12 +2487,14 @@ static void __io_complete_rw(struct io_kiocb *req, long res, long res2, 


	if (req->rw.kiocb.ki_flags & IOCB_WRITE)

		kiocb_end_write(req);

	if ((res == -EAGAIN || res == -EOPNOTSUPP) && io_rw_should_reissue(req)) {

	if (res != req->result) {

		if ((res == -EAGAIN || res == -EOPNOTSUPP) &&

		    io_rw_should_reissue(req)) {

			req->flags |= REQ_F_REISSUE;

			return;

		}

	if (res != req->result)

		req_set_fail_links(req);

	}

	if (req->flags & REQ_F_BUFFER_SELECTED)

		cflags = io_put_rw_kbuf(req);

	__io_req_complete(req, issue_flags, res, cflags);
@@ -2509,19 +2511,20 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) 
{

	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);



#ifdef CONFIG_BLOCK

	if (res == -EAGAIN && io_rw_should_reissue(req)) {

		if (!io_resubmit_prep(req))

			req->flags |= REQ_F_DONT_REISSUE;

	}

#endif



	if (kiocb->ki_flags & IOCB_WRITE)

		kiocb_end_write(req);

	if (unlikely(res != req->result)) {

		bool fail = true;



	if (res != -EAGAIN && res != req->result) {

		req->flags |= REQ_F_DONT_REISSUE;

#ifdef CONFIG_BLOCK

		if (res == -EAGAIN && io_rw_should_reissue(req) &&

		    io_resubmit_prep(req))

			fail = false;

#endif

		if (fail) {

			req_set_fail_links(req);

			req->flags |= REQ_F_DONT_REISSUE;

		}

	}



	WRITE_ONCE(req->result, res);