Commit fd571df0 authored by Paolo Valente's avatar Paolo Valente Committed by Jens Axboe
Browse files

block, bfq: turn bfqq_data into an array in bfq_io_cq 



When a bfq_queue Q is merged with another queue, several pieces of
information are saved about Q. These pieces are stored in the
bfqq_data field in the bfq_io_cq data structure of the process
associated with Q.

Yet, with a multi-actuator drive, a process may get associated with
multiple bfq_queues: one queue for each of the N actuators. Each of
these queues may undergo a merge. So, the bfq_io_cq data structure
must be able to accommodate the above information for N queues.

This commit solves this problem by turning the bfqq_data scalar field
into an array of N elements (and by changing code so as to handle
this array).

This solution is written under the assumption that bfq_queues
associated with different actuators cannot be cross-merged. This
assumption holds naturally with basic queue merging: the latter is
triggered by spatial locality, and sectors for different actuators are
not close to each other (apart from the corner case of the last
sectors served by a given actuator and the first sectors served by the
next actuator). As for stable cross-merging, the assumption here is
that it is disabled.

Reviewed-by: default avatarDamien Le Moal <damien.lemoal@opensource.wdc.com>
Signed-off-by: default avatarGabriele Felici <felicigb@gmail.com>
Signed-off-by: default avatarGianmarco Lusvardi <glusvardi@posteo.net>
Signed-off-by: default avatarGiulio Barabino <giuliobarabino99@gmail.com>
Signed-off-by: default avatarEmiliano Maccaferri <inbox@emilianomaccaferri.com>
Signed-off-by: default avatarPaolo Valente <paolo.valente@linaro.org>
Link: https://lore.kernel.org/r/20230103145503.71712-5-paolo.valente@linaro.org


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent a6123047

block/bfq-iosched.c

+59 −41
Original line number Diff line number Diff line
@@ -408,7 +408,7 @@ void bic_set_bfqq(struct bfq_io_cq *bic, 
	 * we cancel the stable merge if

	 * bic->stable_merge_bfqq == bfqq.

	 */

	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;

	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[actuator_idx];



	/* Clear bic pointer if bfqq is detached from this bic */

	if (old_bfqq && old_bfqq->bic == bic)
@@ -1135,9 +1135,10 @@ static void 
bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd,

		      struct bfq_io_cq *bic, bool bfq_already_existing)

{

	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;

	unsigned int old_wr_coeff = 1;

	bool busy = bfq_already_existing && bfq_bfqq_busy(bfqq);

	unsigned int a_idx = bfqq->actuator_idx;

	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[a_idx];



	if (bfqq_data->saved_has_short_ttime)

		bfq_mark_bfqq_has_short_ttime(bfqq);
@@ -1818,7 +1819,9 @@ static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd, 
		arrived_in_time =  ktime_get_ns() <=

			bfqq->ttime.last_end_request +

			bfqd->bfq_slice_idle * 3;



	unsigned int act_idx = bfq_actuator_index(bfqd, rq->bio);

	bool bfqq_non_merged_or_stably_merged =

		bfqq->bic || RQ_BIC(rq)->bfqq_data[act_idx].stably_merged;



	/*

	 * bfqq deserves to be weight-raised if:
@@ -1852,8 +1855,7 @@ static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd, 
	 */

	wr_or_deserves_wr = bfqd->low_latency &&

		(bfqq->wr_coeff > 1 ||

		 (bfq_bfqq_sync(bfqq) &&

		  (bfqq->bic || RQ_BIC(rq)->bfqq_data.stably_merged) &&

		 (bfq_bfqq_sync(bfqq) && bfqq_non_merged_or_stably_merged &&

		  (*interactive || soft_rt)));



	/*
@@ -2828,6 +2830,35 @@ static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq, 
static bool idling_boosts_thr_without_issues(struct bfq_data *bfqd,

					     struct bfq_queue *bfqq);



static struct bfq_queue *

bfq_setup_stable_merge(struct bfq_data *bfqd, struct bfq_queue *bfqq,

		       struct bfq_queue *stable_merge_bfqq,

		       struct bfq_iocq_bfqq_data *bfqq_data)

{

	int proc_ref = min(bfqq_process_refs(bfqq),

			   bfqq_process_refs(stable_merge_bfqq));

	struct bfq_queue *new_bfqq;



	if (idling_boosts_thr_without_issues(bfqd, bfqq) ||

	    proc_ref == 0)

		return NULL;



	/* next function will take at least one ref */

	new_bfqq = bfq_setup_merge(bfqq, stable_merge_bfqq);



	if (new_bfqq) {

		bfqq_data->stably_merged = true;

		if (new_bfqq->bic) {

			unsigned int new_a_idx = new_bfqq->actuator_idx;

			struct bfq_iocq_bfqq_data *new_bfqq_data =

				&new_bfqq->bic->bfqq_data[new_a_idx];



			new_bfqq_data->stably_merged = true;

		}

	}

	return new_bfqq;

}



/*

 * Attempt to schedule a merge of bfqq with the currently in-service

 * queue or with a close queue among the scheduled queues.  Return
@@ -2853,7 +2884,8 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, 
		     void *io_struct, bool request, struct bfq_io_cq *bic)

{

	struct bfq_queue *in_service_bfqq, *new_bfqq;

	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;

	unsigned int a_idx = bfqq->actuator_idx;

	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[a_idx];



	/* if a merge has already been setup, then proceed with that first */

	if (bfqq->new_bfqq)
@@ -2883,29 +2915,15 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, 
					   msecs_to_jiffies(bfq_late_stable_merging))) {

			struct bfq_queue *stable_merge_bfqq =

				bfqq_data->stable_merge_bfqq;

			int proc_ref = min(bfqq_process_refs(bfqq),

					   bfqq_process_refs(stable_merge_bfqq));



			/* deschedule stable merge, because done or aborted here */

			bfq_put_stable_ref(stable_merge_bfqq);



			bfqq_data->stable_merge_bfqq = NULL;



			if (!idling_boosts_thr_without_issues(bfqd, bfqq) &&

			    proc_ref > 0) {

				/* next function will take at least one ref */

				struct bfq_queue *new_bfqq =

					bfq_setup_merge(bfqq, stable_merge_bfqq);



				if (new_bfqq) {

					bfqq_data->stably_merged = true;

					if (new_bfqq->bic)

						new_bfqq->bic->bfqq_data.stably_merged =

							true;

				}

				return new_bfqq;

			} else

				return NULL;

			return bfq_setup_stable_merge(bfqd, bfqq,

						      stable_merge_bfqq,

						      bfqq_data);

		}

	}
@@ -3000,7 +3018,8 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, 
static void bfq_bfqq_save_state(struct bfq_queue *bfqq)

{

	struct bfq_io_cq *bic = bfqq->bic;

	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;

	unsigned int a_idx = bfqq->actuator_idx;

	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[a_idx];



	/*

	 * If !bfqq->bic, the queue is already shared or its requests
@@ -5377,7 +5396,7 @@ static void bfq_exit_icq(struct io_cq *icq) 
	 * therefore on its unused per-actuator fields being NULL.

	 */

	unsigned int num_actuators = BFQ_MAX_ACTUATORS;

	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;

	struct bfq_iocq_bfqq_data *bfqq_data = bic->bfqq_data;



	/*

	 * bfqd is NULL if scheduler already exited, and in that case
@@ -5388,10 +5407,10 @@ static void bfq_exit_icq(struct io_cq *icq) 
		num_actuators = bfqd->num_actuators;

	}



	if (bfqq_data->stable_merge_bfqq)

		bfq_put_stable_ref(bfqq_data->stable_merge_bfqq);



	for (act_idx = 0; act_idx < num_actuators; act_idx++) {

		if (bfqq_data[act_idx].stable_merge_bfqq)

			bfq_put_stable_ref(bfqq_data[act_idx].stable_merge_bfqq);



		bfq_exit_icq_bfqq(bic, true, act_idx);

		bfq_exit_icq_bfqq(bic, false, act_idx);

	}
@@ -5578,16 +5597,16 @@ bfq_do_early_stable_merge(struct bfq_data *bfqd, struct bfq_queue *bfqq, 
			  struct bfq_io_cq *bic,

			  struct bfq_queue *last_bfqq_created)

{

	unsigned int a_idx = last_bfqq_created->actuator_idx;

	struct bfq_queue *new_bfqq =

		bfq_setup_merge(bfqq, last_bfqq_created);

	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;



	if (!new_bfqq)

		return bfqq;



	if (new_bfqq->bic)

		new_bfqq->bic->bfqq_data.stably_merged = true;

	bfqq_data->stably_merged = true;

		new_bfqq->bic->bfqq_data[a_idx].stably_merged = true;

	bic->bfqq_data[a_idx].stably_merged = true;



	/*

	 * Reusing merge functions. This implies that
@@ -5656,7 +5675,6 @@ static struct bfq_queue *bfq_do_or_sched_stable_merge(struct bfq_data *bfqd, 
		&bfqd->last_bfqq_created;



	struct bfq_queue *last_bfqq_created = *source_bfqq;

	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;



	/*

	 * If last_bfqq_created has not been set yet, then init it. If
@@ -5718,7 +5736,8 @@ static struct bfq_queue *bfq_do_or_sched_stable_merge(struct bfq_data *bfqd, 
			/*

			 * Record the bfqq to merge to.

			 */

			bfqq_data->stable_merge_bfqq = last_bfqq_created;

			bic->bfqq_data[last_bfqq_created->actuator_idx].stable_merge_bfqq =

				last_bfqq_created;

		}

	}
@@ -6639,7 +6658,7 @@ static struct bfq_queue *bfq_get_bfqq_handle_split(struct bfq_data *bfqd, 
{

	unsigned int act_idx = bfq_actuator_index(bfqd, bio);

	struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync, act_idx);

	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data;

	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[act_idx];



	if (likely(bfqq && bfqq != &bfqd->oom_bfqq))

		return bfqq;
@@ -6747,7 +6766,7 @@ static struct bfq_queue *bfq_init_rq(struct request *rq) 
	struct bfq_queue *bfqq;

	bool new_queue = false;

	bool bfqq_already_existing = false, split = false;

	struct bfq_iocq_bfqq_data *bfqq_data;

	unsigned int a_idx = bfq_actuator_index(bfqd, bio);



	if (unlikely(!rq->elv.icq))

		return NULL;
@@ -6771,17 +6790,16 @@ static struct bfq_queue *bfq_init_rq(struct request *rq) 
	bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, false, is_sync,

					 &new_queue);



	bfqq_data = &bic->bfqq_data;



	if (likely(!new_queue)) {

		/* If the queue was seeky for too long, break it apart. */

		if (bfq_bfqq_coop(bfqq) && bfq_bfqq_split_coop(bfqq) &&

			!bfqq_data->stably_merged) {

			!bic->bfqq_data[a_idx].stably_merged) {

			struct bfq_queue *old_bfqq = bfqq;



			/* Update bic before losing reference to bfqq */

			if (bfq_bfqq_in_large_burst(bfqq))

				bfqq_data->saved_in_large_burst = true;

				bic->bfqq_data[a_idx].saved_in_large_burst =

					true;



			bfqq = bfq_split_bfqq(bic, bfqq);

			split = true;

block/bfq-iosched.h

+8 −4
Original line number Diff line number Diff line
@@ -418,7 +418,7 @@ struct bfq_queue { 
struct bfq_iocq_bfqq_data {

	/*

	 * Snapshot of the has_short_time flag before merging; taken

	 * to remember its value while the queue is merged, so as to

	 * to remember its values while the queue is merged, so as to

	 * be able to restore it in case of split.

	 */

	bool saved_has_short_ttime;
@@ -432,7 +432,7 @@ struct bfq_iocq_bfqq_data { 
	u64 saved_tot_idle_time;



	/*

	 * Same purpose as the previous fields for the value of the

	 * Same purpose as the previous fields for the values of the

	 * field keeping the queue's belonging to a large burst

	 */

	bool saved_in_large_burst;
@@ -495,8 +495,12 @@ struct bfq_io_cq { 
	uint64_t blkcg_serial_nr; /* the current blkcg serial */

#endif



	/* persistent data for associated synchronous process queue */

	struct bfq_iocq_bfqq_data bfqq_data;

	/*

	 * Persistent data for associated synchronous process queues

	 * (one queue per actuator, see field bfqq above). In

	 * particular, each of these queues may undergo a merge.

	 */

	struct bfq_iocq_bfqq_data bfqq_data[BFQ_MAX_ACTUATORS];



	unsigned int requests;	/* Number of requests this process has in flight */

};