Commit c1bdf8ed authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: store rsrc node in req instead of refs 



req->fixed_rsrc_refs keeps a pointer to rsrc node pcpu references, but
it's more natural just to store rsrc node directly. There were some
reasons for that in the past but not anymore.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/cee1c86ec9023f3e4f6ce8940d58c017ef8782f4.1650311386.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 772f5e00

fs/io_uring.c

+11 −11
Original line number Diff line number Diff line
@@ -931,7 +931,7 @@ struct io_kiocb { 
	struct io_ring_ctx		*ctx;

	struct task_struct		*task;



	struct percpu_ref		*fixed_rsrc_refs;

	struct io_rsrc_node		*rsrc_node;

	/* store used ubuf, so we can prevent reloading */

	struct io_mapped_ubuf		*imu;
@@ -1330,20 +1330,20 @@ static inline void io_req_put_rsrc_locked(struct io_kiocb *req, 
					  struct io_ring_ctx *ctx)

	__must_hold(&ctx->uring_lock)

{

	struct percpu_ref *ref = req->fixed_rsrc_refs;

	struct io_rsrc_node *node = req->rsrc_node;



	if (ref) {

		if (ref == &ctx->rsrc_node->refs)

	if (node) {

		if (node == ctx->rsrc_node)

			ctx->rsrc_cached_refs++;

		else

			percpu_ref_put(ref);

			percpu_ref_put(&node->refs);

	}

}



static inline void io_req_put_rsrc(struct io_kiocb *req, struct io_ring_ctx *ctx)

{

	if (req->fixed_rsrc_refs)

		percpu_ref_put(req->fixed_rsrc_refs);

	if (req->rsrc_node)

		percpu_ref_put(&req->rsrc_node->refs);

}



static __cold void io_rsrc_refs_drop(struct io_ring_ctx *ctx)
@@ -1366,8 +1366,8 @@ static inline void io_req_set_rsrc_node(struct io_kiocb *req, 
					struct io_ring_ctx *ctx,

					unsigned int issue_flags)

{

	if (!req->fixed_rsrc_refs) {

		req->fixed_rsrc_refs = &ctx->rsrc_node->refs;

	if (!req->rsrc_node) {

		req->rsrc_node = ctx->rsrc_node;



		if (!(issue_flags & IO_URING_F_UNLOCKED)) {

			lockdep_assert_held(&ctx->uring_lock);
@@ -1375,7 +1375,7 @@ static inline void io_req_set_rsrc_node(struct io_kiocb *req, 
			if (unlikely(ctx->rsrc_cached_refs < 0))

				io_rsrc_refs_refill(ctx);

		} else {

			percpu_ref_get(req->fixed_rsrc_refs);

			percpu_ref_get(&req->rsrc_node->refs);

		}

	}

}
@@ -7623,7 +7623,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, 
	req->flags = sqe_flags = READ_ONCE(sqe->flags);

	req->cqe.user_data = READ_ONCE(sqe->user_data);

	req->file = NULL;

	req->fixed_rsrc_refs = NULL;

	req->rsrc_node = NULL;

	req->task = current;



	if (unlikely(opcode >= IORING_OP_LAST)) {