Commit 255f0e14 authored by Daniel Borkmann's avatar Daniel Borkmann
Browse files

Merge branch 'bpf-verifier-log-rotation' 

Andrii Nakryiko says:

====================
This patch set changes BPF verifier log behavior to behave as a rotating log,
by default. If user-supplied log buffer is big enough to contain entire
verifier log output, there is no effective difference. But where previously
user supplied too small log buffer and would get -ENOSPC error result and the
beginning part of the verifier log, now there will be no error and user will
get ending part of verifier log filling up user-supplied log buffer.  Which
is, in absolute majority of cases, is exactly what's useful, relevant, and
what users want and need, as the ending of the verifier log is containing
details of verifier failure and relevant state that got us to that failure.
So this rotating mode is made default, but for some niche advanced debugging
scenarios it's possible to request old behavior by specifying additional
BPF_LOG_FIXED (8) flag.

This patch set adjusts libbpf to allow specifying flags beyond 1 | 2 | 4. We
also add --log-size and --log-fixed options to veristat to be able to both
test this functionality manually, but also to be used in various debugging
scenarios. We also add selftests that tries many variants of log buffer size
to stress-test correctness of internal verifier log bookkeeping code.

Further, this patch set is merged with log_size_actual v1 patchset ([0]),
which adds ability to get required log buffer size to fit entire verifier
log output.

This addresses a long-standing limitation, which causes users and BPF loader
library writers to guess and pre-size log buffer, often allocating unnecessary
extra memory for this or doing extra program verifications just to size logs
better, ultimately wasting resources. This was requested most recently by Go
BPF library maintainers ([1]).

See respective patches for details. A bunch of them some drive-by fixes
detecting during working with the code. Some other further refactor and
compratmentalize verifier log handling code into kernel/bpf/log.c, which
should also make it simpler to integrate such verbose log for other
complicated bpf() syscall commands, if necessary. The rest are actual logic
to calculate maximum log buffer size needed and return it to user-space.
Few patches wire this on libbpf side, and the rest add selftests to test
proper log truncation and log_buf==NULL handling.

This turned into a pretty sizable patch set with lots of arithmetics, but
hopefully the set of features added to verifier log in this patch set are
both useful for BPF users and are self-contained and isolated enough to not
cause troubles going forward.

v3->v4:
  - s/log_size_actual/log_true_size/ (Alexei);
  - log_buf==NULL && log_size==0 don't trigger -ENOSPC (Lorenz);
  - added WARN_ON_ONCE if we try bpf_vlog_reset() forward (Lorenz);
  - added selftests for truncation in BPF_LOG_FIXED mode;
  - fixed edge case in BPF_LOG_FIXED when log_size==1, leaving buf not zero
    terminated;
v2->v3:
  - typos and comment improvement (Lorenz);
  - merged with log_size_actual v1 ([0]) patch set (Alexei);
  - added log_buf==NULL condition allowed (Lorenz);
  - added BPF_BTF_LOAD logs tests (Lorenz);
  - more clean up and refactoring of internal verifier log API;
v1->v2:
  - return -ENOSPC even in rotating log mode for preserving backwards
    compatibility (Lorenz);

  [0] https://patchwork.kernel.org/project/netdevbpf/list/?series=735213&state=*
  [1] https://lore.kernel.org/bpf/CAN+4W8iNoEbQzQVbB_o1W0MWBDV4xCJAq7K3f6psVE-kkCfMqg@mail.gmail.com/


====================

Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
parents eafa9215 054b6c78

include/linux/bpf.h

+1 −1
Original line number Diff line number Diff line
@@ -2175,7 +2175,7 @@ int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size, 
			     size_t actual_size);



/* verify correctness of eBPF program */

int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr);

int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size);



#ifndef CONFIG_BPF_JIT_ALWAYS_ON

void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);

include/linux/bpf_verifier.h

+21 −20
Original line number Diff line number Diff line
@@ -491,39 +491,36 @@ struct bpf_insn_aux_data { 
#define BPF_VERIFIER_TMP_LOG_SIZE	1024



struct bpf_verifier_log {

	u32 level;

	char kbuf[BPF_VERIFIER_TMP_LOG_SIZE];

	/* Logical start and end positions of a "log window" of the verifier log.

	 * start_pos == 0 means we haven't truncated anything.

	 * Once truncation starts to happen, start_pos + len_total == end_pos,

	 * except during log reset situations, in which (end_pos - start_pos)

	 * might get smaller than len_total (see bpf_vlog_reset()).

	 * Generally, (end_pos - start_pos) gives number of useful data in

	 * user log buffer.

	 */

	u64 start_pos;

	u64 end_pos;

	char __user *ubuf;

	u32 len_used;

	u32 level;

	u32 len_total;

	u32 len_max;

	char kbuf[BPF_VERIFIER_TMP_LOG_SIZE];

};



static inline bool bpf_verifier_log_full(const struct bpf_verifier_log *log)

{

	return log->len_used >= log->len_total - 1;

}



#define BPF_LOG_LEVEL1	1

#define BPF_LOG_LEVEL2	2

#define BPF_LOG_STATS	4

#define BPF_LOG_FIXED	8

#define BPF_LOG_LEVEL	(BPF_LOG_LEVEL1 | BPF_LOG_LEVEL2)

#define BPF_LOG_MASK	(BPF_LOG_LEVEL | BPF_LOG_STATS)

#define BPF_LOG_MASK	(BPF_LOG_LEVEL | BPF_LOG_STATS | BPF_LOG_FIXED)

#define BPF_LOG_KERNEL	(BPF_LOG_MASK + 1) /* kernel internal flag */

#define BPF_LOG_MIN_ALIGNMENT 8U

#define BPF_LOG_ALIGNMENT 40U



static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)

{

	return log &&

		((log->level && log->ubuf && !bpf_verifier_log_full(log)) ||

		 log->level == BPF_LOG_KERNEL);

}



static inline bool

bpf_verifier_log_attr_valid(const struct bpf_verifier_log *log)

{

	return log->len_total >= 128 && log->len_total <= UINT_MAX >> 2 &&

	       log->level && log->ubuf && !(log->level & ~BPF_LOG_MASK);

	return log && log->level;

}



#define BPF_MAX_SUBPROGS 256
@@ -603,7 +600,7 @@ struct bpf_verifier_env { 
	u32 scratched_regs;

	/* Same as scratched_regs but for stack slots */

	u64 scratched_stack_slots;

	u32 prev_log_len, prev_insn_print_len;

	u64 prev_log_pos, prev_insn_print_pos;

	/* buffer used in reg_type_str() to generate reg_type string */

	char type_str_buf[TYPE_STR_BUF_LEN];

};
@@ -614,6 +611,10 @@ __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, 
					   const char *fmt, ...);

__printf(2, 3) void bpf_log(struct bpf_verifier_log *log,

			    const char *fmt, ...);

int bpf_vlog_init(struct bpf_verifier_log *log, u32 log_level,

		  char __user *log_buf, u32 log_size);

void bpf_vlog_reset(struct bpf_verifier_log *log, u64 new_pos);

int bpf_vlog_finalize(struct bpf_verifier_log *log, u32 *log_size_actual);



static inline struct bpf_func_state *cur_func(struct bpf_verifier_env *env)

{

include/linux/btf.h

+1 −1
Original line number Diff line number Diff line
@@ -125,7 +125,7 @@ extern const struct file_operations btf_fops; 


void btf_get(struct btf *btf);

void btf_put(struct btf *btf);

int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr);

int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr, u32 uattr_sz);

struct btf *btf_get_by_fd(int fd);

int btf_get_info_by_fd(const struct btf *btf,

		       const union bpf_attr *attr,

include/uapi/linux/bpf.h

+10 −0
Original line number Diff line number Diff line
@@ -1407,6 +1407,11 @@ union bpf_attr { 
		__aligned_u64	fd_array;	/* array of FDs */

		__aligned_u64	core_relos;

		__u32		core_relo_rec_size; /* sizeof(struct bpf_core_relo) */

		/* output: actual total log contents size (including termintaing zero).

		 * It could be both larger than original log_size (if log was

		 * truncated), or smaller (if log buffer wasn't filled completely).

		 */

		__u32		log_true_size;

	};



	struct { /* anonymous struct used by BPF_OBJ_* commands */
@@ -1492,6 +1497,11 @@ union bpf_attr { 
		__u32		btf_size;

		__u32		btf_log_size;

		__u32		btf_log_level;

		/* output: actual total log contents size (including termintaing zero).

		 * It could be both larger than original log_size (if log was

		 * truncated), or smaller (if log buffer wasn't filled completely).

		 */

		__u32		btf_log_true_size;

	};



	struct {

kernel/bpf/Makefile

+2 −1
Original line number Diff line number Diff line
@@ -6,7 +6,8 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse 
endif

CFLAGS_core.o += $(call cc-disable-warning, override-init) $(cflags-nogcse-yy)



obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o

obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o

obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o

obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o

obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o

obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o