Commit fbc0b025 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'Add precision propagation for subprogs and callbacks' 



Andrii Nakryiko says:

====================
As more and more real-world BPF programs become more complex
and increasingly use subprograms (both static and global), scalar precision
tracking and its (previously weak) support for BPF subprograms (and callbacks
as a special case of that) is becoming more and more of an issue and
limitation. Couple that with increasing reliance on state equivalence (BPF
open-coded iterators have a hard requirement for state equivalence to converge
and successfully validate loops), and it becomes pretty critical to address
this limitation and make precision tracking universally supported for BPF
programs of any complexity and composition.

This patch set teaches BPF verifier to support SCALAR precision
backpropagation across multiple frames (for subprogram calls and callback
simulations) and addresses most practical situations (SCALAR stack
loads/stores using registers other than r10 being the last remaining
limitation, though thankfully rarely used in practice).

Main logic is explained in details in patch #8. The rest are preliminary
preparations, refactorings, clean ups, and fixes. See respective patches for
details.

Patch #8 has also veristat comparison of results for selftests, Cilium, and
some of Meta production BPF programs before and after these changes.

v2->v3:
  - drop bitcnt and ifs from bt_xxx() helpers (Alexei);
v1->v2:
  - addressed review feedback form Alexei, adjusted commit messages, comments,
    added verbose(), WARN_ONCE(), etc;
  - re-ran all the tests and veristat on selftests, cilium, and meta-internal
    code: no new changes and no kernel warnings.
====================

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 7866fc6a c91ab90c

include/linux/bpf_verifier.h

+23 −4
Original line number Diff line number Diff line
@@ -18,8 +18,11 @@ 
 * that converting umax_value to int cannot overflow.

 */

#define BPF_MAX_VAR_SIZ	(1 << 29)

/* size of type_str_buf in bpf_verifier. */

#define TYPE_STR_BUF_LEN 128

/* size of tmp_str_buf in bpf_verifier.

 * we need at least 306 bytes to fit full stack mask representation

 * (in the "-8,-16,...,-512" form)

 */

#define TMP_STR_BUF_LEN 320



/* Liveness marks, used for registers and spilled-regs (in stack slots).

 * Read marks propagate upwards until they find a write mark; they record that
@@ -238,6 +241,10 @@ enum bpf_stack_slot_type { 


#define BPF_REG_SIZE 8	/* size of eBPF register in bytes */



#define BPF_REGMASK_ARGS ((1 << BPF_REG_1) | (1 << BPF_REG_2) | \

			  (1 << BPF_REG_3) | (1 << BPF_REG_4) | \

			  (1 << BPF_REG_5))



#define BPF_DYNPTR_SIZE		sizeof(struct bpf_dynptr_kern)

#define BPF_DYNPTR_NR_SLOTS		(BPF_DYNPTR_SIZE / BPF_REG_SIZE)
@@ -541,6 +548,15 @@ struct bpf_subprog_info { 
	bool is_async_cb;

};



struct bpf_verifier_env;



struct backtrack_state {

	struct bpf_verifier_env *env;

	u32 frame;

	u32 reg_masks[MAX_CALL_FRAMES];

	u64 stack_masks[MAX_CALL_FRAMES];

};



/* single container for all structs

 * one verifier_env per bpf_check() call

 */
@@ -578,6 +594,7 @@ struct bpf_verifier_env { 
		int *insn_stack;

		int cur_stack;

	} cfg;

	struct backtrack_state bt;

	u32 pass_cnt; /* number of times do_check() was called */

	u32 subprog_cnt;

	/* number of instructions analyzed by the verifier */
@@ -606,8 +623,10 @@ struct bpf_verifier_env { 
	/* Same as scratched_regs but for stack slots */

	u64 scratched_stack_slots;

	u64 prev_log_pos, prev_insn_print_pos;

	/* buffer used in reg_type_str() to generate reg_type string */

	char type_str_buf[TYPE_STR_BUF_LEN];

	/* buffer used to generate temporary string representations,

	 * e.g., in reg_type_str() to generate reg_type string

	 */

	char tmp_str_buf[TMP_STR_BUF_LEN];

};



__printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log,

kernel/bpf/verifier.c

+471 −151

File changed.

Preview size limit exceeded, changes collapsed.

tools/testing/selftests/bpf/prog_tests/verifier.c

+2 −0
Original line number Diff line number Diff line
@@ -55,6 +55,7 @@ 
#include "verifier_spill_fill.skel.h"

#include "verifier_spin_lock.skel.h"

#include "verifier_stack_ptr.skel.h"

#include "verifier_subprog_precision.skel.h"

#include "verifier_subreg.skel.h"

#include "verifier_uninit.skel.h"

#include "verifier_unpriv.skel.h"
@@ -154,6 +155,7 @@ void test_verifier_sock(void) { RUN(verifier_sock); } 
void test_verifier_spill_fill(void)           { RUN(verifier_spill_fill); }

void test_verifier_spin_lock(void)            { RUN(verifier_spin_lock); }

void test_verifier_stack_ptr(void)            { RUN(verifier_stack_ptr); }

void test_verifier_subprog_precision(void)    { RUN(verifier_subprog_precision); }

void test_verifier_subreg(void)               { RUN(verifier_subreg); }

void test_verifier_uninit(void)               { RUN(verifier_uninit); }

void test_verifier_unpriv(void)               { RUN(verifier_unpriv); }

tools/testing/selftests/bpf/progs/bpf_misc.h

+4 −0
Original line number Diff line number Diff line
@@ -86,6 +86,10 @@ 
#define POINTER_VALUE	0xcafe4all

#define TEST_DATA_LEN	64



#ifndef __used

#define __used __attribute__((used))

#endif



#if defined(__TARGET_ARCH_x86)

#define SYSCALL_WRAPPER 1

#define SYS_PREFIX "__x64_"

tools/testing/selftests/bpf/progs/iters.c

+11 −15
Original line number Diff line number Diff line
@@ -651,29 +651,25 @@ int iter_stack_array_loop(const void *ctx) 
	return sum;

}



#define ARR_SZ 16



static __noinline void fill(struct bpf_iter_num *it, int *arr, int mul)

static __noinline void fill(struct bpf_iter_num *it, int *arr, __u32 n, int mul)

{

	int *t;

	__u64 i;

	int *t, i;



	while ((t = bpf_iter_num_next(it))) {

		i = *t;

		if (i >= ARR_SZ)

		if (i >= n)

			break;

		arr[i] =  i * mul;

	}

}



static __noinline int sum(struct bpf_iter_num *it, int *arr)

static __noinline int sum(struct bpf_iter_num *it, int *arr, __u32 n)

{

	int *t, sum = 0;;

	__u64 i;

	int *t, i, sum = 0;;



	while ((t = bpf_iter_num_next(it))) {

		i = *t;

		if (i >= ARR_SZ)

		if (i >= n)

			break;

		sum += arr[i];

	}
@@ -685,7 +681,7 @@ SEC("raw_tp") 
__success

int iter_pass_iter_ptr_to_subprog(const void *ctx)

{

	int arr1[ARR_SZ], arr2[ARR_SZ];

	int arr1[16], arr2[32];

	struct bpf_iter_num it;

	int n, sum1, sum2;
@@ -694,25 +690,25 @@ int iter_pass_iter_ptr_to_subprog(const void *ctx) 
	/* fill arr1 */

	n = ARRAY_SIZE(arr1);

	bpf_iter_num_new(&it, 0, n);

	fill(&it, arr1, 2);

	fill(&it, arr1, n, 2);

	bpf_iter_num_destroy(&it);



	/* fill arr2 */

	n = ARRAY_SIZE(arr2);

	bpf_iter_num_new(&it, 0, n);

	fill(&it, arr2, 10);

	fill(&it, arr2, n, 10);

	bpf_iter_num_destroy(&it);



	/* sum arr1 */

	n = ARRAY_SIZE(arr1);

	bpf_iter_num_new(&it, 0, n);

	sum1 = sum(&it, arr1);

	sum1 = sum(&it, arr1, n);

	bpf_iter_num_destroy(&it);



	/* sum arr2 */

	n = ARRAY_SIZE(arr2);

	bpf_iter_num_new(&it, 0, n);

	sum2 = sum(&it, arr2);

	sum2 = sum(&it, arr2, n);

	bpf_iter_num_destroy(&it);



	bpf_printk("sum1=%d, sum2=%d", sum1, sum2);