Commit 4d244327 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'io_uring-6.1-2022-10-28' of git://git.kernel.dk/linux 

Pull io_uring fix from Jens Axboe:
 "Just a fix for a locking regression introduced with the deferred
  task_work running from this merge window"

* tag 'io_uring-6.1-2022-10-28' of git://git.kernel.dk/linux:
  io_uring: unlock if __io_run_local_work locked inside
  io_uring: use io_run_local_work_locked helper
parents 3c339dbd b3026767

io_uring/io_uring.c

+5 −6
Original line number Diff line number Diff line
@@ -1173,7 +1173,7 @@ static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx) 
	}

}



int __io_run_local_work(struct io_ring_ctx *ctx, bool locked)

int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked)

{

	struct llist_node *node;

	struct llist_node fake;
@@ -1192,7 +1192,7 @@ int __io_run_local_work(struct io_ring_ctx *ctx, bool locked) 
		struct io_kiocb *req = container_of(node, struct io_kiocb,

						    io_task_work.node);

		prefetch(container_of(next, struct io_kiocb, io_task_work.node));

		req->io_task_work.func(req, &locked);

		req->io_task_work.func(req, locked);

		ret++;

		node = next;

	}
@@ -1208,7 +1208,7 @@ int __io_run_local_work(struct io_ring_ctx *ctx, bool locked) 
		goto again;

	}



	if (locked)

	if (*locked)

		io_submit_flush_completions(ctx);

	trace_io_uring_local_work_run(ctx, ret, loops);

	return ret;
@@ -1225,7 +1225,7 @@ int io_run_local_work(struct io_ring_ctx *ctx) 


	__set_current_state(TASK_RUNNING);

	locked = mutex_trylock(&ctx->uring_lock);

	ret = __io_run_local_work(ctx, locked);

	ret = __io_run_local_work(ctx, &locked);

	if (locked)

		mutex_unlock(&ctx->uring_lock);
@@ -1446,8 +1446,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) 
		    io_task_work_pending(ctx)) {

			u32 tail = ctx->cached_cq_tail;



			if (!llist_empty(&ctx->work_llist))

				__io_run_local_work(ctx, true);

			(void) io_run_local_work_locked(ctx);



			if (task_work_pending(current) ||

			    wq_list_empty(&ctx->iopoll_list)) {

io_uring/io_uring.h

+11 −2
Original line number Diff line number Diff line
@@ -27,7 +27,7 @@ enum { 
struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow);

bool io_req_cqe_overflow(struct io_kiocb *req);

int io_run_task_work_sig(struct io_ring_ctx *ctx);

int __io_run_local_work(struct io_ring_ctx *ctx, bool locked);

int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked);

int io_run_local_work(struct io_ring_ctx *ctx);

void io_req_complete_failed(struct io_kiocb *req, s32 res);

void __io_req_complete(struct io_kiocb *req, unsigned issue_flags);
@@ -277,9 +277,18 @@ static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx) 


static inline int io_run_local_work_locked(struct io_ring_ctx *ctx)

{

	bool locked;

	int ret;



	if (llist_empty(&ctx->work_llist))

		return 0;

	return __io_run_local_work(ctx, true);



	locked = true;

	ret = __io_run_local_work(ctx, &locked);

	/* shouldn't happen! */

	if (WARN_ON_ONCE(!locked))

		mutex_lock(&ctx->uring_lock);

	return ret;

}



static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked)