Commit f9d532fc authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'bpf: use 32bit safe version of u64_stats' 



Eric Dumazet says:

====================

From: Eric Dumazet <edumazet@google.com>

Two first patches fix bugs added in 5.1 and 5.5

Third patch replaces the u64 fields in struct bpf_prog_stats
with u64_stats_t ones to avoid possible sampling errors,
in case of load/store stearing.
====================

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 689624f0 61a0abae

include/linux/filter.h

+8 −7
Original line number Diff line number Diff line
@@ -553,9 +553,9 @@ struct bpf_binary_header { 
};



struct bpf_prog_stats {

	u64 cnt;

	u64 nsecs;

	u64 misses;

	u64_stats_t cnt;

	u64_stats_t nsecs;

	u64_stats_t misses;

	struct u64_stats_sync syncp;

} __aligned(2 * sizeof(u64));
@@ -612,13 +612,14 @@ static __always_inline u32 __bpf_prog_run(const struct bpf_prog *prog, 
	if (static_branch_unlikely(&bpf_stats_enabled_key)) {

		struct bpf_prog_stats *stats;

		u64 start = sched_clock();

		unsigned long flags;



		ret = dfunc(ctx, prog->insnsi, prog->bpf_func);

		stats = this_cpu_ptr(prog->stats);

		u64_stats_update_begin(&stats->syncp);

		stats->cnt++;

		stats->nsecs += sched_clock() - start;

		u64_stats_update_end(&stats->syncp);

		flags = u64_stats_update_begin_irqsave(&stats->syncp);

		u64_stats_inc(&stats->cnt);

		u64_stats_add(&stats->nsecs, sched_clock() - start);

		u64_stats_update_end_irqrestore(&stats->syncp, flags);

	} else {

		ret = dfunc(ctx, prog->insnsi, prog->bpf_func);

	}

kernel/bpf/syscall.c

+12 −6
Original line number Diff line number Diff line
@@ -1804,8 +1804,14 @@ static int bpf_prog_release(struct inode *inode, struct file *filp) 
	return 0;

}



struct bpf_prog_kstats {

	u64 nsecs;

	u64 cnt;

	u64 misses;

};



static void bpf_prog_get_stats(const struct bpf_prog *prog,

			       struct bpf_prog_stats *stats)

			       struct bpf_prog_kstats *stats)

{

	u64 nsecs = 0, cnt = 0, misses = 0;

	int cpu;
@@ -1818,9 +1824,9 @@ static void bpf_prog_get_stats(const struct bpf_prog *prog, 
		st = per_cpu_ptr(prog->stats, cpu);

		do {

			start = u64_stats_fetch_begin_irq(&st->syncp);

			tnsecs = st->nsecs;

			tcnt = st->cnt;

			tmisses = st->misses;

			tnsecs = u64_stats_read(&st->nsecs);

			tcnt = u64_stats_read(&st->cnt);

			tmisses = u64_stats_read(&st->misses);

		} while (u64_stats_fetch_retry_irq(&st->syncp, start));

		nsecs += tnsecs;

		cnt += tcnt;
@@ -1836,7 +1842,7 @@ static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 
{

	const struct bpf_prog *prog = filp->private_data;

	char prog_tag[sizeof(prog->tag) * 2 + 1] = { };

	struct bpf_prog_stats stats;

	struct bpf_prog_kstats stats;



	bpf_prog_get_stats(prog, &stats);

	bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
@@ -3577,7 +3583,7 @@ static int bpf_prog_get_info_by_fd(struct file *file, 
	struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);

	struct bpf_prog_info info;

	u32 info_len = attr->info.info_len;

	struct bpf_prog_stats stats;

	struct bpf_prog_kstats stats;

	char __user *uinsns;

	u32 ulen;

	int err;

kernel/bpf/trampoline.c

+7 −5
Original line number Diff line number Diff line
@@ -545,7 +545,7 @@ static void notrace inc_misses_counter(struct bpf_prog *prog) 


	stats = this_cpu_ptr(prog->stats);

	u64_stats_update_begin(&stats->syncp);

	stats->misses++;

	u64_stats_inc(&stats->misses);

	u64_stats_update_end(&stats->syncp);

}
@@ -586,11 +586,13 @@ static void notrace update_prog_stats(struct bpf_prog *prog, 
	     * Hence check that 'start' is valid.

	     */

	    start > NO_START_TIME) {

		unsigned long flags;



		stats = this_cpu_ptr(prog->stats);

		u64_stats_update_begin(&stats->syncp);

		stats->cnt++;

		stats->nsecs += sched_clock() - start;

		u64_stats_update_end(&stats->syncp);

		flags = u64_stats_update_begin_irqsave(&stats->syncp);

		u64_stats_inc(&stats->cnt);

		u64_stats_add(&stats->nsecs, sched_clock() - start);

		u64_stats_update_end_irqrestore(&stats->syncp, flags);

	}

}