Commit 9b797a37 authored by Jens Axboe's avatar Jens Axboe
Browse files

io_uring: add abstraction around apoll cache 



In preparation for adding limits, and one more user, abstract out the
core bits of the allocation+free cache.

Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 9da7471e

include/linux/io_uring_types.h

+5 −1
Original line number Diff line number Diff line
@@ -158,6 +158,10 @@ struct io_ev_fd { 
	struct rcu_head		rcu;

};



struct io_alloc_cache {

	struct hlist_head	list;

};



struct io_ring_ctx {

	/* const or read-mostly hot data */

	struct {
@@ -216,7 +220,7 @@ struct io_ring_ctx { 


		struct io_hash_table	cancel_table_locked;

		struct list_head	cq_overflow_list;

		struct list_head	apoll_cache;

		struct io_alloc_cache	apoll_cache;

		struct xarray		personalities;

		u32			pers_next;

	} ____cacheline_aligned_in_smp;

io_uring/alloc_cache.h

0 → 100644
+41 −0
Original line number Diff line number Diff line 
#ifndef IOU_ALLOC_CACHE_H

#define IOU_ALLOC_CACHE_H



struct io_cache_entry {

	struct hlist_node	node;

};



static inline void io_alloc_cache_put(struct io_alloc_cache *cache,

				      struct io_cache_entry *entry)

{

	hlist_add_head(&entry->node, &cache->list);

}



static inline struct io_cache_entry *io_alloc_cache_get(struct io_alloc_cache *cache)

{

	if (!hlist_empty(&cache->list)) {

		struct hlist_node *node = cache->list.first;



		hlist_del(node);

		return container_of(node, struct io_cache_entry, node);

	}



	return NULL;

}



static inline void io_alloc_cache_init(struct io_alloc_cache *cache)

{

	INIT_HLIST_HEAD(&cache->list);

}



static inline void io_alloc_cache_free(struct io_alloc_cache *cache,

					void (*free)(struct io_cache_entry *))

{

	while (!hlist_empty(&cache->list)) {

		struct hlist_node *node = cache->list.first;



		hlist_del(node);

		free(container_of(node, struct io_cache_entry, node));

	}

}

#endif

io_uring/io_uring.c

+4 −4
Original line number Diff line number Diff line
@@ -92,6 +92,7 @@ 


#include "timeout.h"

#include "poll.h"

#include "alloc_cache.h"



#define IORING_MAX_ENTRIES	32768

#define IORING_MAX_CQ_ENTRIES	(2 * IORING_MAX_ENTRIES)
@@ -295,7 +296,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) 
	INIT_LIST_HEAD(&ctx->sqd_list);

	INIT_LIST_HEAD(&ctx->cq_overflow_list);

	INIT_LIST_HEAD(&ctx->io_buffers_cache);

	INIT_LIST_HEAD(&ctx->apoll_cache);

	io_alloc_cache_init(&ctx->apoll_cache);

	init_completion(&ctx->ref_comp);

	xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);

	mutex_init(&ctx->uring_lock);
@@ -1180,8 +1181,7 @@ void io_free_batch_list(struct io_ring_ctx *ctx, struct io_wq_work_node *node) 


				if (apoll->double_poll)

					kfree(apoll->double_poll);

				list_add(&apoll->poll.wait.entry,

						&ctx->apoll_cache);

				io_alloc_cache_put(&ctx->apoll_cache, &apoll->cache);

				req->flags &= ~REQ_F_POLLED;

			}

			if (req->flags & IO_REQ_LINK_FLAGS)
@@ -2467,7 +2467,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) 
	if (ctx->rings)

		__io_cqring_overflow_flush(ctx, true);

	io_eventfd_unregister(ctx);

	io_flush_apoll_cache(ctx);

	io_alloc_cache_free(&ctx->apoll_cache, io_apoll_cache_free);

	mutex_unlock(&ctx->uring_lock);

	io_destroy_buffers(ctx);

	if (ctx->sq_creds)

io_uring/poll.c

+5 −13
Original line number Diff line number Diff line
@@ -590,16 +590,15 @@ static struct async_poll *io_req_alloc_apoll(struct io_kiocb *req, 
					     unsigned issue_flags)

{

	struct io_ring_ctx *ctx = req->ctx;

	struct io_cache_entry *entry;

	struct async_poll *apoll;



	if (req->flags & REQ_F_POLLED) {

		apoll = req->apoll;

		kfree(apoll->double_poll);

	} else if (!(issue_flags & IO_URING_F_UNLOCKED) &&

		   !list_empty(&ctx->apoll_cache)) {

		apoll = list_first_entry(&ctx->apoll_cache, struct async_poll,

						poll.wait.entry);

		list_del_init(&apoll->poll.wait.entry);

		   (entry = io_alloc_cache_get(&ctx->apoll_cache)) != NULL) {

		apoll = container_of(entry, struct async_poll, cache);

	} else {

		apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);

		if (unlikely(!apoll))
@@ -960,14 +959,7 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) 
	return IOU_OK;

}



void io_flush_apoll_cache(struct io_ring_ctx *ctx)

void io_apoll_cache_free(struct io_cache_entry *entry)

{

	struct async_poll *apoll;



	while (!list_empty(&ctx->apoll_cache)) {

		apoll = list_first_entry(&ctx->apoll_cache, struct async_poll,

						poll.wait.entry);

		list_del(&apoll->poll.wait.entry);

		kfree(apoll);

	}

	kfree(container_of(entry, struct async_poll, cache));

}

io_uring/poll.h

+7 −2
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0



#include "alloc_cache.h"



enum {

	IO_APOLL_OK,

	IO_APOLL_ABORTED,
@@ -14,7 +16,10 @@ struct io_poll { 
};



struct async_poll {

	union {

		struct io_poll		poll;

		struct io_cache_entry	cache;

	};

	struct io_poll		*double_poll;

};
@@ -31,4 +36,4 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags); 
bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,

			bool cancel_all);



void io_flush_apoll_cache(struct io_ring_ctx *ctx);
 
void io_apoll_cache_free(struct io_cache_entry *entry);