Commit 8ef3fc3a authored by Paolo Valente's avatar Paolo Valente Committed by Jens Axboe
Browse files

block, bfq: make shared queues inherit wakers 



Consider a bfq_queue bfqq that is about to be merged with another
bfq_queue new_bfqq. The processes associated with bfqq are cooperators
of the processes associated with new_bfqq. So, if bfqq has a waker,
then it is reasonable (and beneficial for throughput) to assume that
all these processes will be happy to let bfqq's waker freely inject
I/O when they have no I/O. So this commit makes new_bfqq inherit
bfqq's waker.

Tested-by: default avatarJan Kara <jack@suse.cz>
Signed-off-by: default avatarPaolo Valente <paolo.valente@linaro.org>
Tested-by: default avatarOleksandr Natalenko <oleksandr@natalenko.name>
Link: https://lore.kernel.org/r/20210304174627.161-4-paolo.valente@linaro.org


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 7cc4ffc5

block/bfq-iosched.c

+39 −3
Original line number Diff line number Diff line
@@ -2822,6 +2822,29 @@ bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic, 
		bfq_mark_bfqq_IO_bound(new_bfqq);

	bfq_clear_bfqq_IO_bound(bfqq);



	/*

	 * The processes associated with bfqq are cooperators of the

	 * processes associated with new_bfqq. So, if bfqq has a

	 * waker, then assume that all these processes will be happy

	 * to let bfqq's waker freely inject I/O when they have no

	 * I/O.

	 */

	if (bfqq->waker_bfqq && !new_bfqq->waker_bfqq &&

	    bfqq->waker_bfqq != new_bfqq) {

		new_bfqq->waker_bfqq = bfqq->waker_bfqq;

		new_bfqq->tentative_waker_bfqq = NULL;



		/*

		 * If the waker queue disappears, then

		 * new_bfqq->waker_bfqq must be reset. So insert

		 * new_bfqq into the woken_list of the waker. See

		 * bfq_check_waker for details.

		 */

		hlist_add_head(&new_bfqq->woken_list_node,

			       &new_bfqq->waker_bfqq->woken_list);



	}



	/*

	 * If bfqq is weight-raised, then let new_bfqq inherit

	 * weight-raising. To reduce false positives, neglect the case
@@ -6310,7 +6333,7 @@ static struct bfq_queue *bfq_init_rq(struct request *rq) 
	if (likely(!new_queue)) {

		/* If the queue was seeky for too long, break it apart. */

		if (bfq_bfqq_coop(bfqq) && bfq_bfqq_split_coop(bfqq)) {

			bfq_log_bfqq(bfqd, bfqq, "breaking apart bfqq");

			struct bfq_queue *old_bfqq = bfqq;



			/* Update bic before losing reference to bfqq */

			if (bfq_bfqq_in_large_burst(bfqq))
@@ -6319,11 +6342,24 @@ static struct bfq_queue *bfq_init_rq(struct request *rq) 
			bfqq = bfq_split_bfqq(bic, bfqq);

			split = true;



			if (!bfqq)

			if (!bfqq) {

				bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio,

								 true, is_sync,

								 NULL);

			else

				bfqq->waker_bfqq = old_bfqq->waker_bfqq;

				bfqq->tentative_waker_bfqq = NULL;



				/*

				 * If the waker queue disappears, then

				 * new_bfqq->waker_bfqq must be

				 * reset. So insert new_bfqq into the

				 * woken_list of the waker. See

				 * bfq_check_waker for details.

				 */

				if (bfqq->waker_bfqq)

					hlist_add_head(&bfqq->woken_list_node,

						       &bfqq->waker_bfqq->woken_list);

			} else

				bfqq_already_existing = true;

		}

	}