Commit e297822b authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: order refnode recycling 



Don't recycle a refnode until we're done with all requests of nodes
ejected before.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Cc: stable@vger.kernel.org # v5.7+
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 1e5d770b

fs/io_uring.c

+23 −10
Original line number Diff line number Diff line
@@ -205,6 +205,7 @@ struct fixed_file_ref_node { 
	struct list_head		file_list;

	struct fixed_file_data		*file_data;

	struct llist_node		llist;

	bool				done;

};



struct fixed_file_data {
@@ -7323,10 +7324,6 @@ static void __io_file_put_work(struct fixed_file_ref_node *ref_node) 
		kfree(pfile);

	}



	spin_lock(&file_data->lock);

	list_del(&ref_node->node);

	spin_unlock(&file_data->lock);



	percpu_ref_exit(&ref_node->refs);

	kfree(ref_node);

	percpu_ref_put(&file_data->refs);
@@ -7353,17 +7350,32 @@ static void io_file_put_work(struct work_struct *work) 
static void io_file_data_ref_zero(struct percpu_ref *ref)

{

	struct fixed_file_ref_node *ref_node;

	struct fixed_file_data *data;

	struct io_ring_ctx *ctx;

	bool first_add;

	bool first_add = false;

	int delay = HZ;



	ref_node = container_of(ref, struct fixed_file_ref_node, refs);

	ctx = ref_node->file_data->ctx;

	data = ref_node->file_data;

	ctx = data->ctx;



	spin_lock(&data->lock);

	ref_node->done = true;



	while (!list_empty(&data->ref_list)) {

		ref_node = list_first_entry(&data->ref_list,

					struct fixed_file_ref_node, node);

		/* recycle ref nodes in order */

		if (!ref_node->done)

			break;

		list_del(&ref_node->node);

		first_add |= llist_add(&ref_node->llist, &ctx->file_put_llist);

	}

	spin_unlock(&data->lock);



	if (percpu_ref_is_dying(&ctx->file_data->refs))

	if (percpu_ref_is_dying(&data->refs))

		delay = 0;



	first_add = llist_add(&ref_node->llist, &ctx->file_put_llist);

	if (!delay)

		mod_delayed_work(system_wq, &ctx->file_put_work, 0);

	else if (first_add)
@@ -7387,6 +7399,7 @@ static struct fixed_file_ref_node *alloc_fixed_file_ref_node( 
	INIT_LIST_HEAD(&ref_node->node);

	INIT_LIST_HEAD(&ref_node->file_list);

	ref_node->file_data = ctx->file_data;

	ref_node->done = false;

	return ref_node;

}
@@ -7482,7 +7495,7 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, 


	file_data->node = ref_node;

	spin_lock(&file_data->lock);

	list_add(&ref_node->node, &file_data->ref_list);

	list_add_tail(&ref_node->node, &file_data->ref_list);

	spin_unlock(&file_data->lock);

	percpu_ref_get(&file_data->refs);

	return ret;
@@ -7641,7 +7654,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, 
	if (needs_switch) {

		percpu_ref_kill(&data->node->refs);

		spin_lock(&data->lock);

		list_add(&ref_node->node, &data->ref_list);

		list_add_tail(&ref_node->node, &data->ref_list);

		data->node = ref_node;

		spin_unlock(&data->lock);

		percpu_ref_get(&ctx->file_data->refs);