Commit 38513c46 authored by Hao Xu's avatar Hao Xu Committed by Jens Axboe
Browse files

io_uring: switch cancel_hash to use per entry spinlock 



Add a new io_hash_bucket structure so that each bucket in cancel_hash
has separate spinlock. Use per entry lock for cancel_hash, this removes
some completion lock invocation and remove contension between different
cancel_hash entries.

Signed-off-by: default avatarHao Xu <howeyxu@tencent.com>
Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/05d1e135b0c8bce9d1441e6346776589e5783e26.1655371007.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 3654ab0c

io_uring/cancel.c

+12 −2
Original line number Diff line number Diff line
@@ -93,14 +93,14 @@ int io_try_cancel(struct io_kiocb *req, struct io_cancel_data *cd) 
	if (!ret)

		return 0;



	spin_lock(&ctx->completion_lock);

	ret = io_poll_cancel(ctx, cd);

	if (ret != -ENOENT)

		goto out;

	spin_lock(&ctx->completion_lock);

	if (!(cd->flags & IORING_ASYNC_CANCEL_FD))

		ret = io_timeout_cancel(ctx, cd);

out:

	spin_unlock(&ctx->completion_lock);

out:

	return ret;

}
@@ -192,3 +192,13 @@ int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags) 
	io_req_set_res(req, ret, 0);

	return IOU_OK;

}



void init_hash_table(struct io_hash_bucket *hash_table, unsigned size)

{

	unsigned int i;



	for (i = 0; i < size; i++) {

		spin_lock_init(&hash_table[i].lock);

		INIT_HLIST_HEAD(&hash_table[i].list);

	}

}

io_uring/cancel.h

+6 −0
Original line number Diff line number Diff line
@@ -4,3 +4,9 @@ int io_async_cancel_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 
int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags);



int io_try_cancel(struct io_kiocb *req, struct io_cancel_data *cd);

void init_hash_table(struct io_hash_bucket *hash_table, unsigned size);



struct io_hash_bucket {

	spinlock_t		lock;

	struct hlist_head	list;

} ____cacheline_aligned_in_smp;

io_uring/fdinfo.c

+6 −3
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ 
#include "io_uring.h"

#include "sqpoll.h"

#include "fdinfo.h"

#include "cancel.h"



#ifdef CONFIG_PROC_FS

static __cold int io_uring_show_cred(struct seq_file *m, unsigned int id,
@@ -157,17 +158,19 @@ static __cold void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, 
		mutex_unlock(&ctx->uring_lock);



	seq_puts(m, "PollList:\n");

	spin_lock(&ctx->completion_lock);

	for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {

		struct hlist_head *list = &ctx->cancel_hash[i];

		struct io_hash_bucket *hb = &ctx->cancel_hash[i];

		struct io_kiocb *req;



		hlist_for_each_entry(req, list, hash_node)

		spin_lock(&hb->lock);

		hlist_for_each_entry(req, &hb->list, hash_node)

			seq_printf(m, "  op=%d, task_works=%d\n", req->opcode,

					task_work_pending(req->task));

		spin_unlock(&hb->lock);

	}



	seq_puts(m, "CqOverflowList:\n");

	spin_lock(&ctx->completion_lock);

	list_for_each_entry(ocqe, &ctx->cq_overflow_list, list) {

		struct io_uring_cqe *cqe = &ocqe->cqe;

io_uring/io_uring.c

+6 −3
Original line number Diff line number Diff line
@@ -89,6 +89,7 @@ 
#include "fdinfo.h"

#include "kbuf.h"

#include "rsrc.h"

#include "cancel.h"



#include "timeout.h"

#include "poll.h"
@@ -260,11 +261,13 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) 
	if (hash_bits <= 0)

		hash_bits = 1;

	ctx->cancel_hash_bits = hash_bits;

	ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head),

	ctx->cancel_hash =

		kmalloc((1U << hash_bits) * sizeof(struct io_hash_bucket),

			GFP_KERNEL);

	if (!ctx->cancel_hash)

		goto err;

	__hash_init(ctx->cancel_hash, 1U << hash_bits);



	init_hash_table(ctx->cancel_hash, 1U << hash_bits);



	ctx->dummy_ubuf = kzalloc(sizeof(*ctx->dummy_ubuf), GFP_KERNEL);

	if (!ctx->dummy_ubuf)

io_uring/io_uring_types.h

+1 −1
Original line number Diff line number Diff line
@@ -224,7 +224,7 @@ struct io_ring_ctx { 
		 * manipulate the list, hence no extra locking is needed there.

		 */

		struct io_wq_work_list	iopoll_list;

		struct hlist_head	*cancel_hash;

		struct io_hash_bucket	*cancel_hash;

		unsigned		cancel_hash_bits;

		bool			poll_multi_queue;