Commit e70cb608 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: export io_put_task() 



Make io_put_task() available to non-core parts of io_uring, we'll need
it for notification infrastructure.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/3686807d4c03b72e389947b0e8692d4d44334ef0.1657643355.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent e02b6651

include/linux/io_uring_types.h

+25 −0
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ 
#include <linux/blkdev.h>

#include <linux/task_work.h>

#include <linux/bitmap.h>

#include <linux/llist.h>

#include <uapi/linux/io_uring.h>



struct io_wq_work_node {
@@ -43,6 +44,30 @@ struct io_hash_table { 
	unsigned		hash_bits;

};



/*

 * Arbitrary limit, can be raised if need be

 */

#define IO_RINGFD_REG_MAX 16



struct io_uring_task {

	/* submission side */

	int				cached_refs;

	const struct io_ring_ctx 	*last;

	struct io_wq			*io_wq;

	struct file			*registered_rings[IO_RINGFD_REG_MAX];



	struct xarray			xa;

	struct wait_queue_head		wait;

	atomic_t			in_idle;

	atomic_t			inflight_tracked;

	struct percpu_counter		inflight;



	struct { /* task_work */

		struct llist_head	task_list;

		struct callback_head	task_work;

	} ____cacheline_aligned_in_smp;

};



struct io_uring {

	u32 head ____cacheline_aligned_in_smp;

	u32 tail ____cacheline_aligned_in_smp;

io_uring/io_uring.c

+1 −10
Original line number Diff line number Diff line
@@ -608,7 +608,7 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx) 
	return ret;

}



static void __io_put_task(struct task_struct *task, int nr)

void __io_put_task(struct task_struct *task, int nr)

{

	struct io_uring_task *tctx = task->io_uring;
@@ -618,15 +618,6 @@ static void __io_put_task(struct task_struct *task, int nr) 
	put_task_struct_many(task, nr);

}



/* must to be called somewhat shortly after putting a request */

static inline void io_put_task(struct task_struct *task, int nr)

{

	if (likely(task == current))

		task->io_uring->cached_refs += nr;

	else

		__io_put_task(task, nr);

}



static void io_task_refs_refill(struct io_uring_task *tctx)

{

	unsigned int refill = -tctx->cached_refs + IO_TCTX_REFS_CACHE_NR;

io_uring/io_uring.h

+10 −0
Original line number Diff line number Diff line
@@ -71,6 +71,7 @@ void io_wq_submit_work(struct io_wq_work *work); 


void io_free_req(struct io_kiocb *req);

void io_queue_next(struct io_kiocb *req);

void __io_put_task(struct task_struct *task, int nr);



bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task,

			bool cancel_all);
@@ -258,4 +259,13 @@ static inline void io_commit_cqring_flush(struct io_ring_ctx *ctx) 
		__io_commit_cqring_flush(ctx);

}



/* must to be called somewhat shortly after putting a request */

static inline void io_put_task(struct task_struct *task, int nr)

{

	if (likely(task == current))

		task->io_uring->cached_refs += nr;

	else

		__io_put_task(task, nr);

}



#endif

io_uring/tctx.h

+0 −26
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0



#include <linux/llist.h>



/*

 * Arbitrary limit, can be raised if need be

 */

#define IO_RINGFD_REG_MAX 16



struct io_uring_task {

	/* submission side */

	int				cached_refs;

	const struct io_ring_ctx 	*last;

	struct io_wq			*io_wq;

	struct file			*registered_rings[IO_RINGFD_REG_MAX];



	struct xarray			xa;

	struct wait_queue_head		wait;

	atomic_t			in_idle;

	atomic_t			inflight_tracked;

	struct percpu_counter		inflight;



	struct { /* task_work */

		struct llist_head	task_list;

		struct callback_head	task_work;

	} ____cacheline_aligned_in_smp;

};



struct io_tctx_node {

	struct list_head	ctx_node;

	struct task_struct	*task;