Commit 61a0abae authored by Eric Dumazet's avatar Eric Dumazet Committed by Alexei Starovoitov
Browse files

bpf: Use u64_stats_t in struct bpf_prog_stats 



Commit 316580b6 ("u64_stats: provide u64_stats_t type")
fixed possible load/store tearing on 64bit arches.

For instance the following C code

stats->nsecs += sched_clock() - start;

Could be rightfully implemented like this by a compiler,
confusing concurrent readers a lot:

stats->nsecs += sched_clock();
// arbitrary delay
stats->nsecs -= start;

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/bpf/20211026214133.3114279-4-eric.dumazet@gmail.com
parent d979617a

include/linux/filter.h

+5 −5
Original line number Diff line number Diff line
@@ -553,9 +553,9 @@ struct bpf_binary_header { 
};



struct bpf_prog_stats {

	u64 cnt;

	u64 nsecs;

	u64 misses;

	u64_stats_t cnt;

	u64_stats_t nsecs;

	u64_stats_t misses;

	struct u64_stats_sync syncp;

} __aligned(2 * sizeof(u64));
@@ -617,8 +617,8 @@ static __always_inline u32 __bpf_prog_run(const struct bpf_prog *prog, 
		ret = dfunc(ctx, prog->insnsi, prog->bpf_func);

		stats = this_cpu_ptr(prog->stats);

		flags = u64_stats_update_begin_irqsave(&stats->syncp);

		stats->cnt++;

		stats->nsecs += sched_clock() - start;

		u64_stats_inc(&stats->cnt);

		u64_stats_add(&stats->nsecs, sched_clock() - start);

		u64_stats_update_end_irqrestore(&stats->syncp, flags);

	} else {

		ret = dfunc(ctx, prog->insnsi, prog->bpf_func);

kernel/bpf/syscall.c

+12 −6
Original line number Diff line number Diff line
@@ -1804,8 +1804,14 @@ static int bpf_prog_release(struct inode *inode, struct file *filp) 
	return 0;

}



struct bpf_prog_kstats {

	u64 nsecs;

	u64 cnt;

	u64 misses;

};



static void bpf_prog_get_stats(const struct bpf_prog *prog,

			       struct bpf_prog_stats *stats)

			       struct bpf_prog_kstats *stats)

{

	u64 nsecs = 0, cnt = 0, misses = 0;

	int cpu;
@@ -1818,9 +1824,9 @@ static void bpf_prog_get_stats(const struct bpf_prog *prog, 
		st = per_cpu_ptr(prog->stats, cpu);

		do {

			start = u64_stats_fetch_begin_irq(&st->syncp);

			tnsecs = st->nsecs;

			tcnt = st->cnt;

			tmisses = st->misses;

			tnsecs = u64_stats_read(&st->nsecs);

			tcnt = u64_stats_read(&st->cnt);

			tmisses = u64_stats_read(&st->misses);

		} while (u64_stats_fetch_retry_irq(&st->syncp, start));

		nsecs += tnsecs;

		cnt += tcnt;
@@ -1836,7 +1842,7 @@ static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 
{

	const struct bpf_prog *prog = filp->private_data;

	char prog_tag[sizeof(prog->tag) * 2 + 1] = { };

	struct bpf_prog_stats stats;

	struct bpf_prog_kstats stats;



	bpf_prog_get_stats(prog, &stats);

	bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
@@ -3577,7 +3583,7 @@ static int bpf_prog_get_info_by_fd(struct file *file, 
	struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);

	struct bpf_prog_info info;

	u32 info_len = attr->info.info_len;

	struct bpf_prog_stats stats;

	struct bpf_prog_kstats stats;

	char __user *uinsns;

	u32 ulen;

	int err;

kernel/bpf/trampoline.c

+3 −3
Original line number Diff line number Diff line
@@ -545,7 +545,7 @@ static void notrace inc_misses_counter(struct bpf_prog *prog) 


	stats = this_cpu_ptr(prog->stats);

	u64_stats_update_begin(&stats->syncp);

	stats->misses++;

	u64_stats_inc(&stats->misses);

	u64_stats_update_end(&stats->syncp);

}
@@ -590,8 +590,8 @@ static void notrace update_prog_stats(struct bpf_prog *prog, 


		stats = this_cpu_ptr(prog->stats);

		flags = u64_stats_update_begin_irqsave(&stats->syncp);

		stats->cnt++;

		stats->nsecs += sched_clock() - start;

		u64_stats_inc(&stats->cnt);

		u64_stats_add(&stats->nsecs, sched_clock() - start);

		u64_stats_update_end_irqrestore(&stats->syncp, flags);

	}

}