Commit ef060ea9 authored by Jens Axboe's avatar Jens Axboe
Browse files

io_uring: add IORING_SETUP_TASKRUN_FLAG 



If IORING_SETUP_COOP_TASKRUN is set to use cooperative scheduling for
running task_work, then IORING_SETUP_TASKRUN_FLAG can be set so the
application can tell if task_work is pending in the kernel for this
ring. This allows use cases like io_uring_peek_cqe() to still function
appropriately, or for the task to know when it would be useful to
call io_uring_wait_cqe() to run pending events.

Reviewed-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/20220426014904.60384-7-axboe@kernel.dk


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent e1169f06

fs/io_uring.c

+11 −3
Original line number Diff line number Diff line
@@ -2506,6 +2506,8 @@ static void ctx_flush_and_put(struct io_ring_ctx *ctx, bool *locked) 
{

	if (!ctx)

		return;

	if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)

		atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);

	if (*locked) {

		io_submit_flush_completions(ctx);

		mutex_unlock(&ctx->uring_lock);
@@ -2646,6 +2648,9 @@ static void io_req_task_work_add(struct io_kiocb *req, bool priority) 
	if (running)

		return;



	if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)

		atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);



	if (likely(!task_work_add(tsk, &tctx->task_work, ctx->notify_method)))

		return;
@@ -11333,12 +11338,15 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, 
	ret = -EINVAL;

	if (ctx->flags & IORING_SETUP_SQPOLL) {

		/* IPI related flags don't make sense with SQPOLL */

		if (ctx->flags & IORING_SETUP_COOP_TASKRUN)

		if (ctx->flags & (IORING_SETUP_COOP_TASKRUN |

				  IORING_SETUP_TASKRUN_FLAG))

			goto err;

		ctx->notify_method = TWA_SIGNAL_NO_IPI;

	} else if (ctx->flags & IORING_SETUP_COOP_TASKRUN) {

		ctx->notify_method = TWA_SIGNAL_NO_IPI;

	} else {

		if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)

			goto err;

		ctx->notify_method = TWA_SIGNAL;

	}
@@ -11440,7 +11448,7 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) 
			IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |

			IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |

			IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL |

			IORING_SETUP_COOP_TASKRUN))

			IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG))

		return -EINVAL;



	return io_uring_create(entries, &p, params);

include/uapi/linux/io_uring.h

+7 −0
Original line number Diff line number Diff line
@@ -110,6 +110,12 @@ enum { 
 * a task running in userspace, and saves an IPI.

 */

#define IORING_SETUP_COOP_TASKRUN	(1U << 8)

/*

 * If COOP_TASKRUN is set, get notified if task work is available for

 * running and a kernel transition would be needed to run it. This sets

 * IORING_SQ_TASKRUN in the sq ring flags. Not valid with COOP_TASKRUN.

 */

#define IORING_SETUP_TASKRUN_FLAG	(1U << 9)



enum {

	IORING_OP_NOP,
@@ -256,6 +262,7 @@ struct io_sqring_offsets { 
 */

#define IORING_SQ_NEED_WAKEUP	(1U << 0) /* needs io_uring_enter wakeup */

#define IORING_SQ_CQ_OVERFLOW	(1U << 1) /* CQ ring is overflown */

#define IORING_SQ_TASKRUN	(1U << 2) /* task should enter the kernel */



struct io_cqring_offsets {

	__u32 head;