Commit ae42c317 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'for-5.12/block-ipi-2021-02-21' of git://git.kernel.dk/linux-block 

Pull block IPI updates from Jens Axboe:
 "Avoid IRQ locking for the block IPI handling (Sebastian Andrzej
  Siewior)"

* tag 'for-5.12/block-ipi-2021-02-21' of git://git.kernel.dk/linux-block:
  blk-mq: Use llist_head for blk_cpu_done
  blk-mq: Always complete remote completions requests in softirq
  smp: Process pending softirqs in flush_smp_call_function_from_idle()
parents d652ea30 f9ab4918

block/blk-mq.c

+41 −68
Original line number Diff line number Diff line
@@ -41,7 +41,7 @@ 
#include "blk-mq-sched.h"

#include "blk-rq-qos.h"



static DEFINE_PER_CPU(struct list_head, blk_cpu_done);

static DEFINE_PER_CPU(struct llist_head, blk_cpu_done);



static void blk_mq_poll_stats_start(struct request_queue *q);

static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb);
@@ -567,80 +567,29 @@ void blk_mq_end_request(struct request *rq, blk_status_t error) 
}

EXPORT_SYMBOL(blk_mq_end_request);



/*

 * Softirq action handler - move entries to local list and loop over them

 * while passing them to the queue registered handler.

 */

static __latent_entropy void blk_done_softirq(struct softirq_action *h)

static void blk_complete_reqs(struct llist_head *list)

{

	struct list_head *cpu_list, local_list;



	local_irq_disable();

	cpu_list = this_cpu_ptr(&blk_cpu_done);

	list_replace_init(cpu_list, &local_list);

	local_irq_enable();



	while (!list_empty(&local_list)) {

		struct request *rq;

	struct llist_node *entry = llist_reverse_order(llist_del_all(list));

	struct request *rq, *next;



		rq = list_entry(local_list.next, struct request, ipi_list);

		list_del_init(&rq->ipi_list);

	llist_for_each_entry_safe(rq, next, entry, ipi_list)

		rq->q->mq_ops->complete(rq);

}

}



static void blk_mq_trigger_softirq(struct request *rq)

static __latent_entropy void blk_done_softirq(struct softirq_action *h)

{

	struct list_head *list;

	unsigned long flags;



	local_irq_save(flags);

	list = this_cpu_ptr(&blk_cpu_done);

	list_add_tail(&rq->ipi_list, list);



	/*

	 * If the list only contains our just added request, signal a raise of

	 * the softirq.  If there are already entries there, someone already

	 * raised the irq but it hasn't run yet.

	 */

	if (list->next == &rq->ipi_list)

		raise_softirq_irqoff(BLOCK_SOFTIRQ);

	local_irq_restore(flags);

	blk_complete_reqs(this_cpu_ptr(&blk_cpu_done));

}



static int blk_softirq_cpu_dead(unsigned int cpu)

{

	/*

	 * If a CPU goes away, splice its entries to the current CPU

	 * and trigger a run of the softirq

	 */

	local_irq_disable();

	list_splice_init(&per_cpu(blk_cpu_done, cpu),

			 this_cpu_ptr(&blk_cpu_done));

	raise_softirq_irqoff(BLOCK_SOFTIRQ);

	local_irq_enable();



	blk_complete_reqs(&per_cpu(blk_cpu_done, cpu));

	return 0;

}





static void __blk_mq_complete_request_remote(void *data)

{

	struct request *rq = data;



	/*

	 * For most of single queue controllers, there is only one irq vector

	 * for handling I/O completion, and the only irq's affinity is set

	 * to all possible CPUs.  On most of ARCHs, this affinity means the irq

	 * is handled on one specific CPU.

	 *

	 * So complete I/O requests in softirq context in case of single queue

	 * devices to avoid degrading I/O performance due to irqsoff latency.

	 */

	if (rq->q->nr_hw_queues == 1)

		blk_mq_trigger_softirq(rq);

	else

		rq->q->mq_ops->complete(rq);

	__raise_softirq_irqoff(BLOCK_SOFTIRQ);

}



static inline bool blk_mq_complete_need_ipi(struct request *rq)
@@ -669,6 +618,30 @@ static inline bool blk_mq_complete_need_ipi(struct request *rq) 
	return cpu_online(rq->mq_ctx->cpu);

}



static void blk_mq_complete_send_ipi(struct request *rq)

{

	struct llist_head *list;

	unsigned int cpu;



	cpu = rq->mq_ctx->cpu;

	list = &per_cpu(blk_cpu_done, cpu);

	if (llist_add(&rq->ipi_list, list)) {

		INIT_CSD(&rq->csd, __blk_mq_complete_request_remote, rq);

		smp_call_function_single_async(cpu, &rq->csd);

	}

}



static void blk_mq_raise_softirq(struct request *rq)

{

	struct llist_head *list;



	preempt_disable();

	list = this_cpu_ptr(&blk_cpu_done);

	if (llist_add(&rq->ipi_list, list))

		raise_softirq(BLOCK_SOFTIRQ);

	preempt_enable();

}



bool blk_mq_complete_request_remote(struct request *rq)

{

	WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);
@@ -681,16 +654,16 @@ bool blk_mq_complete_request_remote(struct request *rq) 
		return false;



	if (blk_mq_complete_need_ipi(rq)) {

		INIT_CSD(&rq->csd, __blk_mq_complete_request_remote, rq);

		smp_call_function_single_async(rq->mq_ctx->cpu, &rq->csd);

	} else {

		if (rq->q->nr_hw_queues > 1)

			return false;

		blk_mq_trigger_softirq(rq);

		blk_mq_complete_send_ipi(rq);

		return true;

	}



	if (rq->q->nr_hw_queues == 1) {

		blk_mq_raise_softirq(rq);

		return true;

	}

	return false;

}

EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote);



/**
@@ -3957,7 +3930,7 @@ static int __init blk_mq_init(void) 
	int i;



	for_each_possible_cpu(i)

		INIT_LIST_HEAD(&per_cpu(blk_cpu_done, i));

		init_llist_head(&per_cpu(blk_cpu_done, i));

	open_softirq(BLOCK_SOFTIRQ, blk_done_softirq);



	cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD,

include/linux/blkdev.h

+1 −1
Original line number Diff line number Diff line
@@ -153,7 +153,7 @@ struct request { 
	 */

	union {

		struct hlist_node hash;	/* merge hash */

		struct list_head ipi_list;

		struct llist_node ipi_list;

	};



	/*