Commit 584b0180 authored by Jens Axboe's avatar Jens Axboe
Browse files

io_uring: move read/write file prep state into actual opcode handler 



In preparation for not necessarily having a file assigned at prep time,
defer any initialization associated with the file to when the opcode
handler is run.

Cc: stable@vger.kernel.org # v5.15+
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent a3e4bc23

fs/io_uring.c

+53 −48
Original line number Diff line number Diff line
@@ -592,7 +592,8 @@ struct io_rw { 
	/* NOTE: kiocb has the file as the first member, so don't do it here */

	struct kiocb			kiocb;

	u64				addr;

	u64				len;

	u32				len;

	u32				flags;

};



struct io_connect {
@@ -3178,42 +3179,11 @@ static inline bool io_file_supports_nowait(struct io_kiocb *req) 


static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe)

{

	struct io_ring_ctx *ctx = req->ctx;

	struct kiocb *kiocb = &req->rw.kiocb;

	struct file *file = req->file;

	unsigned ioprio;

	int ret;



	if (!io_req_ffs_set(req))

		req->flags |= io_file_get_flags(file) << REQ_F_SUPPORT_NOWAIT_BIT;



	kiocb->ki_pos = READ_ONCE(sqe->off);

	kiocb->ki_flags = iocb_flags(file);

	ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));

	if (unlikely(ret))

		return ret;



	/*

	 * If the file is marked O_NONBLOCK, still allow retry for it if it

	 * supports async. Otherwise it's impossible to use O_NONBLOCK files

	 * reliably. If not, or it IOCB_NOWAIT is set, don't retry.

	 */

	if ((kiocb->ki_flags & IOCB_NOWAIT) ||

	    ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req)))

		req->flags |= REQ_F_NOWAIT;



	if (ctx->flags & IORING_SETUP_IOPOLL) {

		if (!(kiocb->ki_flags & IOCB_DIRECT) || !file->f_op->iopoll)

			return -EOPNOTSUPP;



		kiocb->ki_flags |= IOCB_HIPRI | IOCB_ALLOC_CACHE;

		kiocb->ki_complete = io_complete_rw_iopoll;

		req->iopoll_completed = 0;

	} else {

		if (kiocb->ki_flags & IOCB_HIPRI)

			return -EINVAL;

		kiocb->ki_complete = io_complete_rw;

	}



	ioprio = READ_ONCE(sqe->ioprio);

	if (ioprio) {
@@ -3229,6 +3199,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) 
	req->imu = NULL;

	req->rw.addr = READ_ONCE(sqe->addr);

	req->rw.len = READ_ONCE(sqe->len);

	req->rw.flags = READ_ONCE(sqe->rw_flags);

	req->buf_index = READ_ONCE(sqe->buf_index);

	return 0;

}
@@ -3732,13 +3703,6 @@ static inline int io_rw_prep_async(struct io_kiocb *req, int rw) 
	return 0;

}



static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)

{

	if (unlikely(!(req->file->f_mode & FMODE_READ)))

		return -EBADF;

	return io_prep_rw(req, sqe);

}



/*

 * This is our waitqueue callback handler, registered through __folio_lock_async()

 * when we initially tried to do the IO with the iocb armed our waitqueue.
@@ -3826,6 +3790,49 @@ static bool need_read_all(struct io_kiocb *req) 
		S_ISBLK(file_inode(req->file)->i_mode);

}



static int io_rw_init_file(struct io_kiocb *req, fmode_t mode)

{

	struct kiocb *kiocb = &req->rw.kiocb;

	struct io_ring_ctx *ctx = req->ctx;

	struct file *file = req->file;

	int ret;



	if (unlikely(!file || !(file->f_mode & mode)))

		return -EBADF;



	if (!io_req_ffs_set(req))

		req->flags |= io_file_get_flags(file) << REQ_F_SUPPORT_NOWAIT_BIT;



	kiocb->ki_flags = iocb_flags(file);

	ret = kiocb_set_rw_flags(kiocb, req->rw.flags);

	if (unlikely(ret))

		return ret;



	/*

	 * If the file is marked O_NONBLOCK, still allow retry for it if it

	 * supports async. Otherwise it's impossible to use O_NONBLOCK files

	 * reliably. If not, or it IOCB_NOWAIT is set, don't retry.

	 */

	if ((kiocb->ki_flags & IOCB_NOWAIT) ||

	    ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req)))

		req->flags |= REQ_F_NOWAIT;



	if (ctx->flags & IORING_SETUP_IOPOLL) {

		if (!(kiocb->ki_flags & IOCB_DIRECT) || !file->f_op->iopoll)

			return -EOPNOTSUPP;



		kiocb->ki_flags |= IOCB_HIPRI | IOCB_ALLOC_CACHE;

		kiocb->ki_complete = io_complete_rw_iopoll;

		req->iopoll_completed = 0;

	} else {

		if (kiocb->ki_flags & IOCB_HIPRI)

			return -EINVAL;

		kiocb->ki_complete = io_complete_rw;

	}



	return 0;

}



static int io_read(struct io_kiocb *req, unsigned int issue_flags)

{

	struct io_rw_state __s, *s = &__s;
@@ -3861,6 +3868,9 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) 
		iov_iter_restore(&s->iter, &s->iter_state);

		iovec = NULL;

	}

	ret = io_rw_init_file(req, FMODE_READ);

	if (unlikely(ret))

		return ret;

	req->result = iov_iter_count(&s->iter);



	if (force_nonblock) {
@@ -3964,13 +3974,6 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) 
	return 0;

}



static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)

{

	if (unlikely(!(req->file->f_mode & FMODE_WRITE)))

		return -EBADF;

	return io_prep_rw(req, sqe);

}



static int io_write(struct io_kiocb *req, unsigned int issue_flags)

{

	struct io_rw_state __s, *s = &__s;
@@ -3991,6 +3994,9 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) 
		iov_iter_restore(&s->iter, &s->iter_state);

		iovec = NULL;

	}

	ret = io_rw_init_file(req, FMODE_WRITE);

	if (unlikely(ret))

		return ret;

	req->result = iov_iter_count(&s->iter);



	if (force_nonblock) {
@@ -6987,11 +6993,10 @@ static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 
	case IORING_OP_READV:

	case IORING_OP_READ_FIXED:

	case IORING_OP_READ:

		return io_read_prep(req, sqe);

	case IORING_OP_WRITEV:

	case IORING_OP_WRITE_FIXED:

	case IORING_OP_WRITE:

		return io_write_prep(req, sqe);

		return io_prep_rw(req, sqe);

	case IORING_OP_POLL_ADD:

		return io_poll_add_prep(req, sqe);

	case IORING_OP_POLL_REMOVE: