Commit 042b0d85 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: use kvmalloc for fixed files 



Instead of hand-coded two-level tables for registered files, allocate
them with kvmalloc(). In many cases small enough tables are enough, and
so can be kmalloc()'ed removing an extra memory load and a bunch of bit
logic instructions from the hot path. If the table is larger, we trade
off all the pros with a TLB-assisted memory lookup.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/280421d3b48775dabab773006bb5588c7b2dabc0.1628471125.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 5fd46178

fs/io_uring.c

+10 −23
Original line number Diff line number Diff line
@@ -92,13 +92,8 @@ 
#define IORING_MAX_CQ_ENTRIES	(2 * IORING_MAX_ENTRIES)

#define IORING_SQPOLL_CAP_ENTRIES_VALUE 8



/*

 * Shift of 9 is 512 entries, or exactly one page on 64-bit archs

 */

#define IORING_FILE_TABLE_SHIFT	9

#define IORING_MAX_FILES_TABLE	(1U << IORING_FILE_TABLE_SHIFT)

#define IORING_FILE_TABLE_MASK	(IORING_MAX_FILES_TABLE - 1)

#define IORING_MAX_FIXED_FILES	(64 * IORING_MAX_FILES_TABLE)

/* 512 entries per page on 64-bit archs, 64 pages max */

#define IORING_MAX_FIXED_FILES	(1U << 15)

#define IORING_MAX_RESTRICTIONS	(IORING_RESTRICTION_LAST + \

				 IORING_REGISTER_LAST + IORING_OP_LAST)
@@ -235,8 +230,7 @@ struct io_rsrc_put { 
};



struct io_file_table {

	/* two level table */

	struct io_fixed_file **files;

	struct io_fixed_file *files;

};



struct io_rsrc_node {
@@ -6335,10 +6329,7 @@ static void io_wq_submit_work(struct io_wq_work *work) 
static inline struct io_fixed_file *io_fixed_file_slot(struct io_file_table *table,

						       unsigned i)

{

	struct io_fixed_file *table_l2;



	table_l2 = table->files[i >> IORING_FILE_TABLE_SHIFT];

	return &table_l2[i & IORING_FILE_TABLE_MASK];

	return &table->files[i];

}



static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
@@ -7276,17 +7267,13 @@ static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_put, 


static bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files)

{

	size_t size = nr_files * sizeof(struct io_fixed_file);



	table->files = (struct io_fixed_file **)io_alloc_page_table(size);

	table->files = kvcalloc(nr_files, sizeof(table->files[0]), GFP_KERNEL);

	return !!table->files;

}



static void io_free_file_tables(struct io_file_table *table, unsigned nr_files)

static void io_free_file_tables(struct io_file_table *table)

{

	size_t size = nr_files * sizeof(struct io_fixed_file);



	io_free_page_table((void **)table->files, size);

	kvfree(table->files);

	table->files = NULL;

}
@@ -7311,7 +7298,7 @@ static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) 
			fput(file);

	}

#endif

	io_free_file_tables(&ctx->file_table, ctx->nr_user_files);

	io_free_file_tables(&ctx->file_table);

	io_rsrc_data_free(ctx->file_data);

	ctx->file_data = NULL;

	ctx->nr_user_files = 0;
@@ -7779,7 +7766,7 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, 
		if (file)

			fput(file);

	}

	io_free_file_tables(&ctx->file_table, nr_args);

	io_free_file_tables(&ctx->file_table);

	ctx->nr_user_files = 0;

out_free:

	io_rsrc_data_free(ctx->file_data);