Commit e58d498e authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: complete notifiers in tw 



We need a task context to post CQEs but using wq is too expensive.
Try to complete notifiers using task_work and fall back to wq if fails.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/089799ab665b10b78fdc614ae6d59fa7ef0d5f91.1657643355.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent eb4a299b

io_uring/notif.c

+19 −3
Original line number Diff line number Diff line
@@ -13,6 +13,11 @@ static void __io_notif_complete_tw(struct callback_head *cb) 
	struct io_notif *notif = container_of(cb, struct io_notif, task_work);

	struct io_ring_ctx *ctx = notif->ctx;



	if (likely(notif->task)) {

		io_put_task(notif->task, 1);

		notif->task = NULL;

	}



	io_cq_lock(ctx);

	io_fill_cqe_aux(ctx, notif->tag, 0, notif->seq, true);
@@ -43,6 +48,14 @@ static void io_uring_tx_zerocopy_callback(struct sk_buff *skb, 


	if (!refcount_dec_and_test(&uarg->refcnt))

		return;



	if (likely(notif->task)) {

		init_task_work(&notif->task_work, __io_notif_complete_tw);

		if (likely(!task_work_add(notif->task, &notif->task_work,

					  TWA_SIGNAL)))

			return;

	}



	INIT_WORK(&notif->commit_work, io_notif_complete_wq);

	queue_work(system_unbound_wq, &notif->commit_work);

}
@@ -134,7 +147,10 @@ __cold int io_notif_unregister(struct io_ring_ctx *ctx) 
	for (i = 0; i < ctx->nr_notif_slots; i++) {

		struct io_notif_slot *slot = &ctx->notif_slots[i];



		if (slot->notif)

		if (!slot->notif)

			continue;

		if (WARN_ON_ONCE(slot->notif->task))

			slot->notif->task = NULL;

		io_notif_slot_flush(slot);

	}

io_uring/notif.h

+3 −0
Original line number Diff line number Diff line
@@ -11,6 +11,9 @@ struct io_notif { 
	struct ubuf_info	uarg;

	struct io_ring_ctx	*ctx;



	/* complete via tw if ->task is non-NULL, fallback to wq otherwise */

	struct task_struct	*task;



	/* cqe->user_data, io_notif_slot::tag if not overridden */

	u64			tag;

	/* see struct io_notif_slot::seq */