Commit 2a371f7d authored by Chengming Zhou's avatar Chengming Zhou Committed by Jens Axboe
Browse files

blk-iocost: combine local_stat and desc_stat to stat 



When we flush usage, wait, indebt stat in iocg_flush_stat(), we use
local_stat and desc_stat, which has no point since the leaf iocg
only has local_stat and the inner iocg only has desc_stat. Also
we don't need to flush percpu abs_vusage for these inner iocgs.

This patch combine local_stat and desc_stat to stat, only flush
percpu abs_vusage for active leaf iocgs, then build inner walk
list to propagate.

Signed-off-by: default avatarChengming Zhou <zhouchengming@bytedance.com>
Acked-by: default avatarTejun Heo <tj@kernel.org>
Link: https://lore.kernel.org/r/20220510034757.21761-1-zhouchengming@bytedance.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 069adbac

block/blk-iocost.c

+32 −39
Original line number Diff line number Diff line
@@ -533,8 +533,7 @@ struct ioc_gq { 


	/* statistics */

	struct iocg_pcpu_stat __percpu	*pcpu_stat;

	struct iocg_stat		local_stat;

	struct iocg_stat		desc_stat;

	struct iocg_stat		stat;

	struct iocg_stat		last_stat;

	u64				last_stat_abs_vusage;

	u64				usage_delta_us;
@@ -1371,7 +1370,7 @@ static bool iocg_kick_delay(struct ioc_gq *iocg, struct ioc_now *now) 
		return true;

	} else {

		if (iocg->indelay_since) {

			iocg->local_stat.indelay_us += now->now - iocg->indelay_since;

			iocg->stat.indelay_us += now->now - iocg->indelay_since;

			iocg->indelay_since = 0;

		}

		iocg->delay = 0;
@@ -1419,7 +1418,7 @@ static void iocg_pay_debt(struct ioc_gq *iocg, u64 abs_vpay, 


	/* if debt is paid in full, restore inuse */

	if (!iocg->abs_vdebt) {

		iocg->local_stat.indebt_us += now->now - iocg->indebt_since;

		iocg->stat.indebt_us += now->now - iocg->indebt_since;

		iocg->indebt_since = 0;



		propagate_weights(iocg, iocg->active, iocg->last_inuse,
@@ -1513,7 +1512,7 @@ static void iocg_kick_waitq(struct ioc_gq *iocg, bool pay_debt, 


	if (!waitqueue_active(&iocg->waitq)) {

		if (iocg->wait_since) {

			iocg->local_stat.wait_us += now->now - iocg->wait_since;

			iocg->stat.wait_us += now->now - iocg->wait_since;

			iocg->wait_since = 0;

		}

		return;
@@ -1641,11 +1640,30 @@ static void iocg_build_inner_walk(struct ioc_gq *iocg, 
	}

}



/* propagate the deltas to the parent */

static void iocg_flush_stat_upward(struct ioc_gq *iocg)

{

	if (iocg->level > 0) {

		struct iocg_stat *parent_stat =

			&iocg->ancestors[iocg->level - 1]->stat;



		parent_stat->usage_us +=

			iocg->stat.usage_us - iocg->last_stat.usage_us;

		parent_stat->wait_us +=

			iocg->stat.wait_us - iocg->last_stat.wait_us;

		parent_stat->indebt_us +=

			iocg->stat.indebt_us - iocg->last_stat.indebt_us;

		parent_stat->indelay_us +=

			iocg->stat.indelay_us - iocg->last_stat.indelay_us;

	}



	iocg->last_stat = iocg->stat;

}



/* collect per-cpu counters and propagate the deltas to the parent */

static void iocg_flush_stat_one(struct ioc_gq *iocg, struct ioc_now *now)

static void iocg_flush_stat_leaf(struct ioc_gq *iocg, struct ioc_now *now)

{

	struct ioc *ioc = iocg->ioc;

	struct iocg_stat new_stat;

	u64 abs_vusage = 0;

	u64 vusage_delta;

	int cpu;
@@ -1661,34 +1679,9 @@ static void iocg_flush_stat_one(struct ioc_gq *iocg, struct ioc_now *now) 
	iocg->last_stat_abs_vusage = abs_vusage;



	iocg->usage_delta_us = div64_u64(vusage_delta, ioc->vtime_base_rate);

	iocg->local_stat.usage_us += iocg->usage_delta_us;



	/* propagate upwards */

	new_stat.usage_us =

		iocg->local_stat.usage_us + iocg->desc_stat.usage_us;

	new_stat.wait_us =

		iocg->local_stat.wait_us + iocg->desc_stat.wait_us;

	new_stat.indebt_us =

		iocg->local_stat.indebt_us + iocg->desc_stat.indebt_us;

	new_stat.indelay_us =

		iocg->local_stat.indelay_us + iocg->desc_stat.indelay_us;



	/* propagate the deltas to the parent */

	if (iocg->level > 0) {

		struct iocg_stat *parent_stat =

			&iocg->ancestors[iocg->level - 1]->desc_stat;



		parent_stat->usage_us +=

			new_stat.usage_us - iocg->last_stat.usage_us;

		parent_stat->wait_us +=

			new_stat.wait_us - iocg->last_stat.wait_us;

		parent_stat->indebt_us +=

			new_stat.indebt_us - iocg->last_stat.indebt_us;

		parent_stat->indelay_us +=

			new_stat.indelay_us - iocg->last_stat.indelay_us;

	}

	iocg->stat.usage_us += iocg->usage_delta_us;



	iocg->last_stat = new_stat;

	iocg_flush_stat_upward(iocg);

}



/* get stat counters ready for reading on all active iocgs */
@@ -1699,13 +1692,13 @@ static void iocg_flush_stat(struct list_head *target_iocgs, struct ioc_now *now) 


	/* flush leaves and build inner node walk list */

	list_for_each_entry(iocg, target_iocgs, active_list) {

		iocg_flush_stat_one(iocg, now);

		iocg_flush_stat_leaf(iocg, now);

		iocg_build_inner_walk(iocg, &inner_walk);

	}



	/* keep flushing upwards by walking the inner list backwards */

	list_for_each_entry_safe_reverse(iocg, tiocg, &inner_walk, walk_list) {

		iocg_flush_stat_one(iocg, now);

		iocg_flush_stat_upward(iocg);

		list_del_init(&iocg->walk_list);

	}

}
@@ -2152,16 +2145,16 @@ static int ioc_check_iocgs(struct ioc *ioc, struct ioc_now *now) 


		/* flush wait and indebt stat deltas */

		if (iocg->wait_since) {

			iocg->local_stat.wait_us += now->now - iocg->wait_since;

			iocg->stat.wait_us += now->now - iocg->wait_since;

			iocg->wait_since = now->now;

		}

		if (iocg->indebt_since) {

			iocg->local_stat.indebt_us +=

			iocg->stat.indebt_us +=

				now->now - iocg->indebt_since;

			iocg->indebt_since = now->now;

		}

		if (iocg->indelay_since) {

			iocg->local_stat.indelay_us +=

			iocg->stat.indelay_us +=

				now->now - iocg->indelay_since;

			iocg->indelay_since = now->now;

		}