Commit b9c1133a authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'block-6.4-2023-06-15' of git://git.kernel.dk/linux 

Pull block fix from Jens Axboe:
 "Just a single fix for blk-cg stats flushing"

* tag 'block-6.4-2023-06-15' of git://git.kernel.dk/linux:
  blk-cgroup: Flush stats before releasing blkcg_gq
parents 3a12faba 20cb1c2f

block/blk-cgroup.c

+31 −9
Original line number Diff line number Diff line
@@ -34,6 +34,8 @@ 
#include "blk-ioprio.h"

#include "blk-throttle.h"



static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu);



/*

 * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.

 * blkcg_pol_register_mutex nests outside of it and synchronizes entire
@@ -56,6 +58,8 @@ static LIST_HEAD(all_blkcgs); /* protected by blkcg_pol_mutex */ 


bool blkcg_debug_stats = false;



static DEFINE_RAW_SPINLOCK(blkg_stat_lock);



#define BLKG_DESTROY_BATCH_SIZE  64



/*
@@ -163,10 +167,20 @@ static void blkg_free(struct blkcg_gq *blkg) 
static void __blkg_release(struct rcu_head *rcu)

{

	struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head);

	struct blkcg *blkcg = blkg->blkcg;

	int cpu;



#ifdef CONFIG_BLK_CGROUP_PUNT_BIO

	WARN_ON(!bio_list_empty(&blkg->async_bios));

#endif

	/*

	 * Flush all the non-empty percpu lockless lists before releasing

	 * us, given these stat belongs to us.

	 *

	 * blkg_stat_lock is for serializing blkg stat update

	 */

	for_each_possible_cpu(cpu)

		__blkcg_rstat_flush(blkcg, cpu);



	/* release the blkcg and parent blkg refs this blkg has been holding */

	css_put(&blkg->blkcg->css);
@@ -951,23 +965,26 @@ static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur, 
	u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);

}



static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)

static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu)

{

	struct blkcg *blkcg = css_to_blkcg(css);

	struct llist_head *lhead = per_cpu_ptr(blkcg->lhead, cpu);

	struct llist_node *lnode;

	struct blkg_iostat_set *bisc, *next_bisc;



	/* Root-level stats are sourced from system-wide IO stats */

	if (!cgroup_parent(css->cgroup))

		return;



	rcu_read_lock();



	lnode = llist_del_all(lhead);

	if (!lnode)

		goto out;



	/*

	 * For covering concurrent parent blkg update from blkg_release().

	 *

	 * When flushing from cgroup, cgroup_rstat_lock is always held, so

	 * this lock won't cause contention most of time.

	 */

	raw_spin_lock(&blkg_stat_lock);



	/*

	 * Iterate only the iostat_cpu's queued in the lockless list.

	 */
@@ -991,13 +1008,19 @@ static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu) 
		if (parent && parent->parent)

			blkcg_iostat_update(parent, &blkg->iostat.cur,

					    &blkg->iostat.last);

		percpu_ref_put(&blkg->refcnt);

	}



	raw_spin_unlock(&blkg_stat_lock);

out:

	rcu_read_unlock();

}



static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)

{

	/* Root-level stats are sourced from system-wide IO stats */

	if (cgroup_parent(css->cgroup))

		__blkcg_rstat_flush(css_to_blkcg(css), cpu);

}



/*

 * We source root cgroup stats from the system-wide stats to avoid

 * tracking the same information twice and incurring overhead when no
@@ -2075,7 +2098,6 @@ void blk_cgroup_bio_start(struct bio *bio) 


		llist_add(&bis->lnode, lhead);

		WRITE_ONCE(bis->lqueued, true);

		percpu_ref_get(&bis->blkg->refcnt);

	}



	u64_stats_update_end_irqrestore(&bis->sync, flags);