Commit 5853a7b5 authored by Jens Axboe's avatar Jens Axboe
Browse files

Merge branch 'for-6.1/io_uring' into for-6.1/passthrough 

* for-6.1/io_uring: (56 commits)
  io_uring/net: fix notif cqe reordering
  io_uring/net: don't update msg_name if not provided
  io_uring: don't gate task_work run on TIF_NOTIFY_SIGNAL
  io_uring/rw: defer fsnotify calls to task context
  io_uring/net: fix fast_iov assignment in io_setup_async_msg()
  io_uring/net: fix non-zc send with address
  io_uring/net: don't skip notifs for failed requests
  io_uring/rw: don't lose short results on io_setup_async_rw()
  io_uring/rw: fix unexpected link breakage
  io_uring/net: fix cleanup double free free_iov init
  io_uring: fix CQE reordering
  io_uring/net: fix UAF in io_sendrecv_fail()
  selftest/net: adjust io_uring sendzc notif handling
  io_uring: ensure local task_work marks task as running
  io_uring/net: zerocopy sendmsg
  io_uring/net: combine fail handlers
  io_uring/net: rename io_sendzc()
  io_uring/net: support non-zerocopy sendto
  io_uring/net: refactor io_setup_async_addr
  io_uring/net: don't lose partial send_zc on fail
  ...
parents 736feaa3 108893dd

block/blk-mq.c

+2 −1
Original line number Diff line number Diff line
@@ -1241,7 +1241,7 @@ static void blk_end_sync_rq(struct request *rq, blk_status_t ret) 
	complete(&wait->done);

}



static bool blk_rq_is_poll(struct request *rq)

bool blk_rq_is_poll(struct request *rq)

{

	if (!rq->mq_hctx)

		return false;
@@ -1251,6 +1251,7 @@ static bool blk_rq_is_poll(struct request *rq) 
		return false;

	return true;

}

EXPORT_SYMBOL_GPL(blk_rq_is_poll);



static void blk_rq_poll_completion(struct request *rq, struct completion *wait)

{

drivers/nvme/host/core.c

+1 −0
Original line number Diff line number Diff line
@@ -3976,6 +3976,7 @@ static const struct file_operations nvme_ns_chr_fops = { 
	.unlocked_ioctl	= nvme_ns_chr_ioctl,

	.compat_ioctl	= compat_ptr_ioctl,

	.uring_cmd	= nvme_ns_chr_uring_cmd,

	.uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll,

};



static int nvme_add_ns_cdev(struct nvme_ns *ns)

drivers/nvme/host/ioctl.c

+72 −5
Original line number Diff line number Diff line
@@ -398,10 +398,18 @@ static void nvme_uring_cmd_end_io(struct request *req, blk_status_t err) 
	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);

	/* extract bio before reusing the same field for request */

	struct bio *bio = pdu->bio;

	void *cookie = READ_ONCE(ioucmd->cookie);



	pdu->req = req;

	req->bio = bio;

	/* this takes care of moving rest of completion-work to task context */



	/*

	 * For iopoll, complete it directly.

	 * Otherwise, move the completion to task work.

	 */

	if (cookie != NULL && blk_rq_is_poll(req))

		nvme_uring_task_cb(ioucmd);

	else

		io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_cb);

}
@@ -452,7 +460,10 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, 
		rq_flags = REQ_NOWAIT;

		blk_flags = BLK_MQ_REQ_NOWAIT;

	}

	if (issue_flags & IO_URING_F_IOPOLL)

		rq_flags |= REQ_POLLED;



retry:

	req = nvme_alloc_user_request(q, &c, nvme_to_user_ptr(d.addr),

			d.data_len, nvme_to_user_ptr(d.metadata),

			d.metadata_len, 0, &meta, d.timeout_ms ?
@@ -463,6 +474,17 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, 
	req->end_io = nvme_uring_cmd_end_io;

	req->end_io_data = ioucmd;



	if (issue_flags & IO_URING_F_IOPOLL && rq_flags & REQ_POLLED) {

		if (unlikely(!req->bio)) {

			/* we can't poll this, so alloc regular req instead */

			blk_mq_free_request(req);

			rq_flags &= ~REQ_POLLED;

			goto retry;

		} else {

			WRITE_ONCE(ioucmd->cookie, req->bio);

			req->bio->bi_opf |= REQ_POLLED;

		}

	}

	/* to free bio on completion, as req->bio will be null at that time */

	pdu->bio = req->bio;

	pdu->meta = meta;
@@ -566,9 +588,6 @@ long nvme_ns_chr_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 


static int nvme_uring_cmd_checks(unsigned int issue_flags)

{

	/* IOPOLL not supported yet */

	if (issue_flags & IO_URING_F_IOPOLL)

		return -EOPNOTSUPP;



	/* NVMe passthrough requires big SQE/CQE support */

	if ((issue_flags & (IO_URING_F_SQE128|IO_URING_F_CQE32)) !=
@@ -611,6 +630,25 @@ int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags) 
	return nvme_ns_uring_cmd(ns, ioucmd, issue_flags);

}



int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,

				 struct io_comp_batch *iob,

				 unsigned int poll_flags)

{

	struct bio *bio;

	int ret = 0;

	struct nvme_ns *ns;

	struct request_queue *q;



	rcu_read_lock();

	bio = READ_ONCE(ioucmd->cookie);

	ns = container_of(file_inode(ioucmd->file)->i_cdev,

			struct nvme_ns, cdev);

	q = ns->queue;

	if (test_bit(QUEUE_FLAG_POLL, &q->queue_flags) && bio && bio->bi_bdev)

		ret = bio_poll(bio, iob, poll_flags);

	rcu_read_unlock();

	return ret;

}

#ifdef CONFIG_NVME_MULTIPATH

static int nvme_ns_head_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,

		void __user *argp, struct nvme_ns_head *head, int srcu_idx)
@@ -692,6 +730,31 @@ int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd, 
	srcu_read_unlock(&head->srcu, srcu_idx);

	return ret;

}



int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,

				      struct io_comp_batch *iob,

				      unsigned int poll_flags)

{

	struct cdev *cdev = file_inode(ioucmd->file)->i_cdev;

	struct nvme_ns_head *head = container_of(cdev, struct nvme_ns_head, cdev);

	int srcu_idx = srcu_read_lock(&head->srcu);

	struct nvme_ns *ns = nvme_find_path(head);

	struct bio *bio;

	int ret = 0;

	struct request_queue *q;



	if (ns) {

		rcu_read_lock();

		bio = READ_ONCE(ioucmd->cookie);

		q = ns->queue;

		if (test_bit(QUEUE_FLAG_POLL, &q->queue_flags) && bio

				&& bio->bi_bdev)

			ret = bio_poll(bio, iob, poll_flags);

		rcu_read_unlock();

	}

	srcu_read_unlock(&head->srcu, srcu_idx);

	return ret;

}

#endif /* CONFIG_NVME_MULTIPATH */



int nvme_dev_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags)
@@ -699,6 +762,10 @@ int nvme_dev_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags) 
	struct nvme_ctrl *ctrl = ioucmd->file->private_data;

	int ret;



	/* IOPOLL not supported yet */

	if (issue_flags & IO_URING_F_IOPOLL)

		return -EOPNOTSUPP;



	ret = nvme_uring_cmd_checks(issue_flags);

	if (ret)

		return ret;

drivers/nvme/host/multipath.c

+1 −0
Original line number Diff line number Diff line
@@ -439,6 +439,7 @@ static const struct file_operations nvme_ns_head_chr_fops = { 
	.unlocked_ioctl	= nvme_ns_head_chr_ioctl,

	.compat_ioctl	= compat_ptr_ioctl,

	.uring_cmd	= nvme_ns_head_chr_uring_cmd,

	.uring_cmd_iopoll = nvme_ns_head_chr_uring_cmd_iopoll,

};



static int nvme_add_ns_head_cdev(struct nvme_ns_head *head)

drivers/nvme/host/nvme.h

+4 −0
Original line number Diff line number Diff line
@@ -843,6 +843,10 @@ long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd, 
		unsigned long arg);

long nvme_dev_ioctl(struct file *file, unsigned int cmd,

		unsigned long arg);

int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,

		struct io_comp_batch *iob, unsigned int poll_flags);

int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,

		struct io_comp_batch *iob, unsigned int poll_flags);

int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd,

		unsigned int issue_flags);

int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd,