Commit 4379bf8b authored by Jens Axboe's avatar Jens Axboe
Browse files

io_uring: remove io_identity 



We are no longer grabbing state, so no need to maintain an IO identity
that we COW if there are changes.

Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 44526bed

fs/io-wq.c

+26 −0
Original line number Diff line number Diff line
@@ -53,6 +53,9 @@ struct io_worker { 
	struct io_wq_work *cur_work;

	spinlock_t lock;



	const struct cred *cur_creds;

	const struct cred *saved_creds;



	struct rcu_head rcu;

};
@@ -171,6 +174,11 @@ static void io_worker_exit(struct io_worker *worker) 
	worker->flags = 0;

	preempt_enable();



	if (worker->saved_creds) {

		revert_creds(worker->saved_creds);

		worker->cur_creds = worker->saved_creds = NULL;

	}



	raw_spin_lock_irq(&wqe->lock);

	hlist_nulls_del_rcu(&worker->nulls_node);

	list_del_rcu(&worker->all_list);
@@ -312,6 +320,10 @@ static void __io_worker_idle(struct io_wqe *wqe, struct io_worker *worker) 
		worker->flags |= IO_WORKER_F_FREE;

		hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list);

	}

	if (worker->saved_creds) {

		revert_creds(worker->saved_creds);

		worker->cur_creds = worker->saved_creds = NULL;

	}

}



static inline unsigned int io_get_work_hash(struct io_wq_work *work)
@@ -359,6 +371,18 @@ static void io_flush_signals(void) 
	}

}



static void io_wq_switch_creds(struct io_worker *worker,

			       struct io_wq_work *work)

{

	const struct cred *old_creds = override_creds(work->creds);



	worker->cur_creds = work->creds;

	if (worker->saved_creds)

		put_cred(old_creds); /* creds set by previous switch */

	else

		worker->saved_creds = old_creds;

}



static void io_assign_current_work(struct io_worker *worker,

				   struct io_wq_work *work)

{
@@ -407,6 +431,8 @@ static void io_worker_handle_work(struct io_worker *worker) 
			unsigned int hash = io_get_work_hash(work);



			next_hashed = wq_next_work(work);

			if (work->creds && worker->cur_creds != work->creds)

				io_wq_switch_creds(worker, work);

			wq->do_work(work);

			io_assign_current_work(worker, NULL);

fs/io-wq.h

+1 −1
Original line number Diff line number Diff line
@@ -78,7 +78,7 @@ static inline void wq_list_del(struct io_wq_work_list *list, 


struct io_wq_work {

	struct io_wq_work_node list;

	struct io_identity *identity;

	const struct cred *creds;

	unsigned flags;

};

fs/io_uring.c

+25 −79
Original line number Diff line number Diff line
@@ -1094,7 +1094,7 @@ static bool io_match_task(struct io_kiocb *head, 
			continue;

		if (req->file && req->file->f_op == &io_uring_fops)

			return true;

		if (req->work.identity->files == files)

		if (req->task->files == files)

			return true;

	}

	return false;
@@ -1218,31 +1218,6 @@ static inline void req_set_fail_links(struct io_kiocb *req) 
		req->flags |= REQ_F_FAIL_LINK;

}



/*

 * None of these are dereferenced, they are simply used to check if any of

 * them have changed. If we're under current and check they are still the

 * same, we're fine to grab references to them for actual out-of-line use.

 */

static void io_init_identity(struct io_identity *id)

{

	id->files = current->files;

	id->mm = current->mm;

#ifdef CONFIG_BLK_CGROUP

	rcu_read_lock();

	id->blkcg_css = blkcg_css();

	rcu_read_unlock();

#endif

	id->creds = current_cred();

	id->nsproxy = current->nsproxy;

	id->fs = current->fs;

	id->fsize = rlimit(RLIMIT_FSIZE);

#ifdef CONFIG_AUDIT

	id->loginuid = current->loginuid;

	id->sessionid = current->sessionid;

#endif

	refcount_set(&id->count, 1);

}



static inline void __io_req_init_async(struct io_kiocb *req)

{

	memset(&req->work, 0, sizeof(req->work));
@@ -1255,17 +1230,10 @@ static inline void __io_req_init_async(struct io_kiocb *req) 
 */

static inline void io_req_init_async(struct io_kiocb *req)

{

	struct io_uring_task *tctx = current->io_uring;



	if (req->flags & REQ_F_WORK_INITIALIZED)

		return;



	__io_req_init_async(req);



	/* Grab a ref if this isn't our static identity */

	req->work.identity = tctx->identity;

	if (tctx->identity != &tctx->__identity)

		refcount_inc(&req->work.identity->count);

}



static void io_ring_ctx_ref_free(struct percpu_ref *ref)
@@ -1350,19 +1318,15 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq) 
	return false;

}



static void io_put_identity(struct io_uring_task *tctx, struct io_kiocb *req)

{

	if (req->work.identity == &tctx->__identity)

		return;

	if (refcount_dec_and_test(&req->work.identity->count))

		kfree(req->work.identity);

}



static void io_req_clean_work(struct io_kiocb *req)

{

	if (!(req->flags & REQ_F_WORK_INITIALIZED))

		return;



	if (req->work.creds) {

		put_cred(req->work.creds);

		req->work.creds = NULL;

	}

	if (req->flags & REQ_F_INFLIGHT) {

		struct io_ring_ctx *ctx = req->ctx;

		struct io_uring_task *tctx = req->task->io_uring;
@@ -1377,7 +1341,6 @@ static void io_req_clean_work(struct io_kiocb *req) 
	}



	req->flags &= ~REQ_F_WORK_INITIALIZED;

	io_put_identity(req->task->io_uring, req);

}



static void io_req_track_inflight(struct io_kiocb *req)
@@ -1411,6 +1374,8 @@ static void io_prep_async_work(struct io_kiocb *req) 
		if (def->unbound_nonreg_file)

			req->work.flags |= IO_WQ_WORK_UNBOUND;

	}

	if (!req->work.creds)

		req->work.creds = get_current_cred();

}



static void io_prep_async_link(struct io_kiocb *req)
@@ -6376,9 +6341,9 @@ static void __io_queue_sqe(struct io_kiocb *req) 
	const struct cred *old_creds = NULL;

	int ret;



	if ((req->flags & REQ_F_WORK_INITIALIZED) &&

	    req->work.identity->creds != current_cred())

		old_creds = override_creds(req->work.identity->creds);

	if ((req->flags & REQ_F_WORK_INITIALIZED) && req->work.creds &&

	    req->work.creds != current_cred())

		old_creds = override_creds(req->work.creds);



	ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);
@@ -6508,16 +6473,11 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, 


	id = READ_ONCE(sqe->personality);

	if (id) {

		struct io_identity *iod;



		iod = idr_find(&ctx->personality_idr, id);

		if (unlikely(!iod))

			return -EINVAL;

		refcount_inc(&iod->count);



		__io_req_init_async(req);

		get_cred(iod->creds);

		req->work.identity = iod;

		req->work.creds = idr_find(&ctx->personality_idr, id);

		if (unlikely(!req->work.creds))

			return -EINVAL;

		get_cred(req->work.creds);

	}



	state = &ctx->submit_state;
@@ -7936,8 +7896,6 @@ static int io_uring_alloc_task_context(struct task_struct *task, 
	tctx->last = NULL;

	atomic_set(&tctx->in_idle, 0);

	tctx->sqpoll = false;

	io_init_identity(&tctx->__identity);

	tctx->identity = &tctx->__identity;

	task->io_uring = tctx;

	spin_lock_init(&tctx->task_lock);

	INIT_WQ_LIST(&tctx->task_list);
@@ -7951,9 +7909,6 @@ void __io_uring_free(struct task_struct *tsk) 
	struct io_uring_task *tctx = tsk->io_uring;



	WARN_ON_ONCE(!xa_empty(&tctx->xa));

	WARN_ON_ONCE(refcount_read(&tctx->identity->count) != 1);

	if (tctx->identity != &tctx->__identity)

		kfree(tctx->identity);

	percpu_counter_destroy(&tctx->inflight);

	kfree(tctx);

	tsk->io_uring = NULL;
@@ -8593,13 +8548,11 @@ static int io_uring_fasync(int fd, struct file *file, int on) 


static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)

{

	struct io_identity *iod;

	const struct cred *creds;



	iod = idr_remove(&ctx->personality_idr, id);

	if (iod) {

		put_cred(iod->creds);

		if (refcount_dec_and_test(&iod->count))

			kfree(iod);

	creds = idr_remove(&ctx->personality_idr, id);

	if (creds) {

		put_cred(creds);

		return 0;

	}
@@ -9300,8 +9253,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, 
#ifdef CONFIG_PROC_FS

static int io_uring_show_cred(int id, void *p, void *data)

{

	struct io_identity *iod = p;

	const struct cred *cred = iod->creds;

	const struct cred *cred = p;

	struct seq_file *m = data;

	struct user_namespace *uns = seq_user_ns(m);

	struct group_info *gi;
@@ -9732,21 +9684,15 @@ static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) 


static int io_register_personality(struct io_ring_ctx *ctx)

{

	struct io_identity *id;

	const struct cred *creds;

	int ret;



	id = kmalloc(sizeof(*id), GFP_KERNEL);

	if (unlikely(!id))

		return -ENOMEM;



	io_init_identity(id);

	id->creds = get_current_cred();

	creds = get_current_cred();



	ret = idr_alloc_cyclic(&ctx->personality_idr, id, 1, USHRT_MAX, GFP_KERNEL);

	if (ret < 0) {

		put_cred(id->creds);

		kfree(id);

	}

	ret = idr_alloc_cyclic(&ctx->personality_idr, (void *) creds, 1,

				USHRT_MAX, GFP_KERNEL);

	if (ret < 0)

		put_cred(creds);

	return ret;

}

include/linux/io_uring.h

+0 −19
Original line number Diff line number Diff line
@@ -5,23 +5,6 @@ 
#include <linux/sched.h>

#include <linux/xarray.h>



struct io_identity {

	struct files_struct		*files;

	struct mm_struct		*mm;

#ifdef CONFIG_BLK_CGROUP

	struct cgroup_subsys_state	*blkcg_css;

#endif

	const struct cred		*creds;

	struct nsproxy			*nsproxy;

	struct fs_struct		*fs;

	unsigned long			fsize;

#ifdef CONFIG_AUDIT

	kuid_t				loginuid;

	unsigned int			sessionid;

#endif

	refcount_t			count;

};



struct io_wq_work_node {

	struct io_wq_work_node *next;

};
@@ -38,8 +21,6 @@ struct io_uring_task { 
	struct file		*last;

	void			*io_wq;

	struct percpu_counter	inflight;

	struct io_identity	__identity;

	struct io_identity	*identity;

	atomic_t		in_idle;

	bool			sqpoll;