Commit 204f600c authored by Li Nan's avatar Li Nan
Browse files

block: support to dispatch bio asynchronously 

hulk inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/IA5AEP


CVE: NA

--------------------------------

In certain environments, specific CPUs handle a large number of tasks
and become bottlenecks, affecting overall system performance. This
commit introduces a new feature that enables asynchronous I/O dispatch
to designated CPUs, thereby relieving the pressure on the busy CPUs.

Signed-off-by: default avatarLi Nan <linan122@huawei.com>
parent 55bdd9ab

arch/arm64/configs/openeuler_defconfig

+1 −0
Original line number Diff line number Diff line
@@ -964,6 +964,7 @@ CONFIG_BLK_DEBUG_FS_ZONED=y 
# CONFIG_BLK_SED_OPAL is not set

# CONFIG_BLK_INLINE_ENCRYPTION is not set

CONFIG_BLK_DEV_DUMPINFO=y

CONFIG_BLK_BIO_DISPATCH_ASYNC=y



#

# Partition Types

arch/x86/configs/openeuler_defconfig

+1 −0
Original line number Diff line number Diff line
@@ -942,6 +942,7 @@ CONFIG_BLK_DEBUG_FS_ZONED=y 
# CONFIG_BLK_SED_OPAL is not set

# CONFIG_BLK_INLINE_ENCRYPTION is not set

CONFIG_BLK_DEV_DUMPINFO=y

CONFIG_BLK_BIO_DISPATCH_ASYNC=y



#

# Partition Types

block/Kconfig

+11 −0
Original line number Diff line number Diff line
@@ -209,6 +209,17 @@ config BLK_DEV_DUMPINFO 
	  Dump info when open an write opened block device exclusively

	  or open an exclusive opened device for write



config BLK_BIO_DISPATCH_ASYNC

	bool "Dispatch bios asynchronously on specific cpus"

	default n

	help

	In certain environments, specific CPUs handle a large number of

	tasks and become bottlenecks, affecting overall system

	performance. This commit introduces a new feature that enables

	asynchronous I/O dispatch to designated CPUs, thereby relieving

	the pressure on the busy CPUs.

	If unsure, say N.



menu "Partition Types"



source "block/partitions/Kconfig"

block/blk-core.c

+239 −1
Original line number Diff line number Diff line
@@ -87,6 +87,234 @@ struct kmem_cache *blk_requestq_cachep; 
 */

static struct workqueue_struct *kblockd_workqueue;



#ifdef CONFIG_BLK_BIO_DISPATCH_ASYNC



#define BIO_DISPATCH_MAX_LOOP 16



struct async_bio {

	struct bio_list list;

	spinlock_t lock;

} ____cacheline_aligned_in_smp;



struct bio_dispatch_async_ctl {

	/*

	 * Vector size is nr_cpu_ids, list stores bio dispatched from other cpu,

	 * such bio will be dispatched asynchronously to the cpu this structure

	 * is serviced.

	 */

	struct async_bio	*bios;

	/* kthread to handle bio dispatched from other cpu. */

	struct task_struct	*thread;

	wait_queue_head_t       wait;

};



static struct bio_dispatch_async_ctl __percpu *bio_dispatch_async_ctl;



static int blk_alloc_queue_dispatch_async(struct request_queue *q)

{

	int cpu;



	/* use the same function and parameters as alloc_cpumask_var() */

	q->dispatch_async_cpus = kmalloc_node(cpumask_size(),

					      GFP_KERNEL, q->node);

	if (!q->dispatch_async_cpus)

		return -ENOMEM;



	q->last_dispatch_cpu = alloc_percpu(int);

	if (!q->last_dispatch_cpu) {

		kfree(q->dispatch_async_cpus);

		q->dispatch_async_cpus = NULL;

		return -ENOMEM;

	}



	cpumask_setall(q->dispatch_async_cpus);

	for_each_possible_cpu(cpu)

		*per_cpu_ptr(q->last_dispatch_cpu, cpu) = cpu;



	return 0;

}



void blk_free_queue_dispatch_async(struct request_queue *q)

{

	kfree(q->dispatch_async_cpus);

	q->dispatch_async_cpus = NULL;

	free_percpu(q->last_dispatch_cpu);

	q->last_dispatch_cpu = NULL;

}



static int get_dispatch_cpu(struct request_queue *q)

{

	int cpu = cpumask_next(this_cpu_read(*q->last_dispatch_cpu),

			       q->dispatch_async_cpus);



	if (cpu >= nr_cpu_ids)

		cpu = cpumask_first(q->dispatch_async_cpus);



	return cpu;

}



static bool __submit_bio_noacct_async(struct bio *bio)

{

	struct request_queue *q = bio->bi_disk->queue;

	int current_cpu = smp_processor_id();

	int dispatch_cpu = get_dispatch_cpu(q);

	struct bio_dispatch_async_ctl *ctl;



	if (dispatch_cpu >= nr_cpu_ids)

		return false;



	this_cpu_write(*q->last_dispatch_cpu, dispatch_cpu);



	ctl = per_cpu_ptr(bio_dispatch_async_ctl, dispatch_cpu);

	spin_lock_irq(&ctl->bios[current_cpu].lock);

	bio_list_add(&ctl->bios[current_cpu].list, bio);

	spin_unlock_irq(&ctl->bios[current_cpu].lock);



	if (wq_has_sleeper(&ctl->wait))

		wake_up(&ctl->wait);



	return true;

}



static bool submit_bio_noacct_async(struct bio *bio)

{

	struct request_queue *q;



	if (bio_flagged(bio, BIO_ASYNC))

		return false;



	bio_set_flag(bio, BIO_ASYNC);

	/*

	 * Don't dispatch bio asynchronously in following cases:

	 *

	 * - QUEUE_FLAG_DISPATCH_ASYNC is not set;

	 * - current cpu is the target cpu;

	 * - bio is flagged no wait;

	 * - io polling is enabled;

	 */

	q = bio->bi_disk->queue;

	if (!test_bit(QUEUE_FLAG_DISPATCH_ASYNC, &q->queue_flags) ||

	    test_bit(QUEUE_FLAG_POLL, &q->queue_flags) ||

	    cpumask_test_cpu(smp_processor_id(), q->dispatch_async_cpus) ||

	    bio->bi_opf & REQ_NOWAIT)

		return false;



	return __submit_bio_noacct_async(bio);

}



static bool collect_bio(struct bio_dispatch_async_ctl *ctl,

			struct bio_list *list)

{

	bool has_bio = false;

	int cpu;



	for_each_possible_cpu(cpu) {

		struct async_bio *abio = &ctl->bios[cpu];



		if (bio_list_empty(&abio->list))

			continue;



		has_bio = true;



		spin_lock_irq(&abio->lock);

		bio_list_merge(list, &abio->list);

		bio_list_init(&abio->list);

		spin_unlock_irq(&abio->lock);

	}



	return has_bio;

}



static int bio_dispatch_work(void *data)

{

	int loop_count = 0;

	struct bio_list bio_list_on_stack;

	struct blk_plug plug;

	struct bio_dispatch_async_ctl *ctl;



	bio_list_init(&bio_list_on_stack);

	ctl = this_cpu_ptr(bio_dispatch_async_ctl);



	for (;; loop_count++) {

		struct bio *bio;

		bool has_bio = collect_bio(ctl, &bio_list_on_stack);



		if (!has_bio) {

			DEFINE_WAIT(wait);



			for (;;) {

				prepare_to_wait(&ctl->wait, &wait,

						TASK_INTERRUPTIBLE);

				has_bio = collect_bio(ctl, &bio_list_on_stack);

				if (has_bio)

					break;

				schedule();

				loop_count = 0;

			}

			finish_wait(&ctl->wait, &wait);

		}



		blk_start_plug(&plug);

		while ((bio = bio_list_pop(&bio_list_on_stack)))

			submit_bio_noacct(bio);

		blk_finish_plug(&plug);



		/* prevent soft lockup. */

		if (loop_count >= BIO_DISPATCH_MAX_LOOP) {

			loop_count = 0;

			cond_resched();

		}

	}



	return 0;

}



static void init_blk_queue_async_dispatch(void)

{

	int cpu;



	bio_dispatch_async_ctl = alloc_percpu(struct bio_dispatch_async_ctl);

	if (!bio_dispatch_async_ctl)

		panic("Failed to alloc bio_dispatch_async_ctl\n");



	for_each_possible_cpu(cpu) {

		int i;

		struct bio_dispatch_async_ctl *ctl =

			per_cpu_ptr(bio_dispatch_async_ctl, cpu);



		init_waitqueue_head(&ctl->wait);

		ctl->bios = kmalloc_array(nr_cpu_ids, sizeof(struct async_bio),

					  GFP_KERNEL | __GFP_NOFAIL);

		for (i = 0; i < nr_cpu_ids; ++i) {

			bio_list_init(&ctl->bios[i].list);

			spin_lock_init(&ctl->bios[i].lock);

		}



		ctl->thread =

			kthread_create_on_cpu(bio_dispatch_work, NULL, cpu,

					      "bio_dispatch_work_%u");

		if (IS_ERR_OR_NULL(ctl->thread))

			panic("Failed to create bio dispatch thread\n");



		wake_up_process(ctl->thread);

	}

}

#else

static int blk_alloc_queue_dispatch_async(struct request_queue *q)

{

	return 0;

}



static bool submit_bio_noacct_async(struct bio *bio)

{

	return false;

}



static void init_blk_queue_async_dispatch(void)

{

}

#endif



/**

 * blk_queue_flag_set - atomically set a queue flag

 * @flag: flag to be set
@@ -539,9 +767,12 @@ struct request_queue *blk_alloc_queue(int node_id) 


	q->last_merge = NULL;



	if (blk_alloc_queue_dispatch_async(q))

		goto fail_q;



	q->id = ida_simple_get(&blk_queue_ida, 0, 0, GFP_KERNEL);

	if (q->id < 0)

		goto fail_q;

		goto fail_dispatch_async;



	ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);

	if (ret)
@@ -606,6 +837,8 @@ struct request_queue *blk_alloc_queue(int node_id) 
	bioset_exit(&q->bio_split);

fail_id:

	ida_simple_remove(&blk_queue_ida, q->id);

fail_dispatch_async:

	blk_free_queue_dispatch_async(q);

fail_q:

	kmem_cache_free(blk_requestq_cachep, q);

	return NULL;
@@ -1055,6 +1288,9 @@ static blk_qc_t __submit_bio_noacct_mq(struct bio *bio) 
 */

blk_qc_t submit_bio_noacct(struct bio *bio)

{

	if (submit_bio_noacct_async(bio))

		return BLK_QC_T_NONE;



	if (!submit_bio_checks(bio))

		return BLK_QC_T_NONE;
@@ -1905,5 +2141,7 @@ int __init blk_dev_init(void) 


	blk_debugfs_root = debugfs_create_dir("block", NULL);



	init_blk_queue_async_dispatch();



	return 0;

}

block/blk-mq-debugfs.c

+1 −0
Original line number Diff line number Diff line
@@ -131,6 +131,7 @@ static const char *const blk_queue_flag_name[] = { 
	QUEUE_FLAG_NAME(RQ_ALLOC_TIME),

	QUEUE_FLAG_NAME(HCTX_ACTIVE),

	QUEUE_FLAG_NAME(NOWAIT),

	QUEUE_FLAG_NAME(DISPATCH_ASYNC),

};

#undef QUEUE_FLAG_NAME