Commit 5e124f74 authored by Tejun Heo's avatar Tejun Heo Committed by Jens Axboe
Browse files

blk-iocost: use local[64]_t for percpu stat 



blk-iocost has been reading percpu stat counters from remote cpus which on
some archs can lead to torn reads in really rare occassions. Use local[64]_t
for those counters.

Signed-off-by: default avatarTejun Heo <tj@kernel.org>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 1f06959b

block/blk-iocost.c

+27 −10
Original line number Diff line number Diff line
@@ -179,6 +179,8 @@ 
#include <linux/parser.h>

#include <linux/sched/signal.h>

#include <linux/blk-cgroup.h>

#include <asm/local.h>

#include <asm/local64.h>

#include "blk-rq-qos.h"

#include "blk-stat.h"

#include "blk-wbt.h"
@@ -373,8 +375,8 @@ struct ioc_params { 
};



struct ioc_missed {

	u32				nr_met;

	u32				nr_missed;

	local_t				nr_met;

	local_t				nr_missed;

	u32				last_met;

	u32				last_missed;

};
@@ -382,7 +384,7 @@ struct ioc_missed { 
struct ioc_pcpu_stat {

	struct ioc_missed		missed[2];



	u64				rq_wait_ns;

	local64_t			rq_wait_ns;

	u64				last_rq_wait_ns;

};
@@ -1278,8 +1280,8 @@ static void ioc_lat_stat(struct ioc *ioc, u32 *missed_ppm_ar, u32 *rq_wait_pct_p 
		u64 this_rq_wait_ns;



		for (rw = READ; rw <= WRITE; rw++) {

			u32 this_met = READ_ONCE(stat->missed[rw].nr_met);

			u32 this_missed = READ_ONCE(stat->missed[rw].nr_missed);

			u32 this_met = local_read(&stat->missed[rw].nr_met);

			u32 this_missed = local_read(&stat->missed[rw].nr_missed);



			nr_met[rw] += this_met - stat->missed[rw].last_met;

			nr_missed[rw] += this_missed - stat->missed[rw].last_missed;
@@ -1287,7 +1289,7 @@ static void ioc_lat_stat(struct ioc *ioc, u32 *missed_ppm_ar, u32 *rq_wait_pct_p 
			stat->missed[rw].last_missed = this_missed;

		}



		this_rq_wait_ns = READ_ONCE(stat->rq_wait_ns);

		this_rq_wait_ns = local64_read(&stat->rq_wait_ns);

		rq_wait_ns += this_rq_wait_ns - stat->last_rq_wait_ns;

		stat->last_rq_wait_ns = this_rq_wait_ns;

	}
@@ -1908,6 +1910,7 @@ static void ioc_rqos_done_bio(struct rq_qos *rqos, struct bio *bio) 
static void ioc_rqos_done(struct rq_qos *rqos, struct request *rq)

{

	struct ioc *ioc = rqos_to_ioc(rqos);

	struct ioc_pcpu_stat *ccs;

	u64 on_q_ns, rq_wait_ns, size_nsec;

	int pidx, rw;
@@ -1931,13 +1934,17 @@ static void ioc_rqos_done(struct rq_qos *rqos, struct request *rq) 
	rq_wait_ns = rq->start_time_ns - rq->alloc_time_ns;

	size_nsec = div64_u64(calc_size_vtime_cost(rq, ioc), VTIME_PER_NSEC);



	ccs = get_cpu_ptr(ioc->pcpu_stat);



	if (on_q_ns <= size_nsec ||

	    on_q_ns - size_nsec <= ioc->params.qos[pidx] * NSEC_PER_USEC)

		this_cpu_inc(ioc->pcpu_stat->missed[rw].nr_met);

		local_inc(&ccs->missed[rw].nr_met);

	else

		this_cpu_inc(ioc->pcpu_stat->missed[rw].nr_missed);

		local_inc(&ccs->missed[rw].nr_missed);



	local64_add(rq_wait_ns, &ccs->rq_wait_ns);



	this_cpu_add(ioc->pcpu_stat->rq_wait_ns, rq_wait_ns);

	put_cpu_ptr(ccs);

}



static void ioc_rqos_queue_depth_changed(struct rq_qos *rqos)
@@ -1977,7 +1984,7 @@ static int blk_iocost_init(struct request_queue *q) 
{

	struct ioc *ioc;

	struct rq_qos *rqos;

	int ret;

	int i, cpu, ret;



	ioc = kzalloc(sizeof(*ioc), GFP_KERNEL);

	if (!ioc)
@@ -1989,6 +1996,16 @@ static int blk_iocost_init(struct request_queue *q) 
		return -ENOMEM;

	}



	for_each_possible_cpu(cpu) {

		struct ioc_pcpu_stat *ccs = per_cpu_ptr(ioc->pcpu_stat, cpu);



		for (i = 0; i < ARRAY_SIZE(ccs->missed); i++) {

			local_set(&ccs->missed[i].nr_met, 0);

			local_set(&ccs->missed[i].nr_missed, 0);

		}

		local64_set(&ccs->rq_wait_ns, 0);

	}



	rqos = &ioc->rqos;

	rqos->id = RQ_QOS_COST;

	rqos->ops = &ioc_rqos_ops;