Commit 6fae2e8a authored by Johan Almbladh's avatar Johan Almbladh Committed by Daniel Borkmann
Browse files

bpf/tests: Add exhaustive tests of BPF_ATOMIC register combinations 



This patch adds tests of all register combinations for BPF_ATOMIC
operations on both BPF_W and BPF_DW sizes.

Signed-off-by: default avatarJohan Almbladh <johan.almbladh@anyfinetworks.com>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/bpf/20211001130348.3670534-9-johan.almbladh@anyfinetworks.com
parent daed6083

lib/test_bpf.c

+422 −0
Original line number Diff line number Diff line
@@ -1805,6 +1805,246 @@ static int bpf_fill_cmpxchg32(struct bpf_test *self) 
				  &__bpf_emit_cmpxchg32);

}



/*

 * Test JITs that implement ATOMIC operations as function calls or

 * other primitives, and must re-arrange operands for argument passing.

 */

static int __bpf_fill_atomic_reg_pairs(struct bpf_test *self, u8 width, u8 op)

{

	struct bpf_insn *insn;

	int len = 2 + 34 * 10 * 10;

	u64 mem, upd, res;

	int rd, rs, i = 0;



	insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);

	if (!insn)

		return -ENOMEM;



	/* Operand and memory values */

	if (width == BPF_DW) {

		mem = 0x0123456789abcdefULL;

		upd = 0xfedcba9876543210ULL;

	} else { /* BPF_W */

		mem = 0x01234567U;

		upd = 0x76543210U;

	}



	/* Memory updated according to operation */

	switch (op) {

	case BPF_XCHG:

		res = upd;

		break;

	case BPF_CMPXCHG:

		res = mem;

		break;

	default:

		__bpf_alu_result(&res, mem, upd, BPF_OP(op));

	}



	/* Test all operand registers */

	for (rd = R0; rd <= R9; rd++) {

		for (rs = R0; rs <= R9; rs++) {

			u64 cmp, src;



			/* Initialize value in memory */

			i += __bpf_ld_imm64(&insn[i], R0, mem);

			insn[i++] = BPF_STX_MEM(width, R10, R0, -8);



			/* Initialize registers in order */

			i += __bpf_ld_imm64(&insn[i], R0, ~mem);

			i += __bpf_ld_imm64(&insn[i], rs, upd);

			insn[i++] = BPF_MOV64_REG(rd, R10);



			/* Perform atomic operation */

			insn[i++] = BPF_ATOMIC_OP(width, op, rd, rs, -8);

			if (op == BPF_CMPXCHG && width == BPF_W)

				insn[i++] = BPF_ZEXT_REG(R0);



			/* Check R0 register value */

			if (op == BPF_CMPXCHG)

				cmp = mem;  /* Expect value from memory */

			else if (R0 == rd || R0 == rs)

				cmp = 0;    /* Aliased, checked below */

			else

				cmp = ~mem; /* Expect value to be preserved */

			if (cmp) {

				insn[i++] = BPF_JMP32_IMM(BPF_JEQ, R0,

							   (u32)cmp, 2);

				insn[i++] = BPF_MOV32_IMM(R0, __LINE__);

				insn[i++] = BPF_EXIT_INSN();

				insn[i++] = BPF_ALU64_IMM(BPF_RSH, R0, 32);

				insn[i++] = BPF_JMP32_IMM(BPF_JEQ, R0,

							   cmp >> 32, 2);

				insn[i++] = BPF_MOV32_IMM(R0, __LINE__);

				insn[i++] = BPF_EXIT_INSN();

			}



			/* Check source register value */

			if (rs == R0 && op == BPF_CMPXCHG)

				src = 0;   /* Aliased with R0, checked above */

			else if (rs == rd && (op == BPF_CMPXCHG ||

					      !(op & BPF_FETCH)))

				src = 0;   /* Aliased with rd, checked below */

			else if (op == BPF_CMPXCHG)

				src = upd; /* Expect value to be preserved */

			else if (op & BPF_FETCH)

				src = mem; /* Expect fetched value from mem */

			else /* no fetch */

				src = upd; /* Expect value to be preserved */

			if (src) {

				insn[i++] = BPF_JMP32_IMM(BPF_JEQ, rs,

							   (u32)src, 2);

				insn[i++] = BPF_MOV32_IMM(R0, __LINE__);

				insn[i++] = BPF_EXIT_INSN();

				insn[i++] = BPF_ALU64_IMM(BPF_RSH, rs, 32);

				insn[i++] = BPF_JMP32_IMM(BPF_JEQ, rs,

							   src >> 32, 2);

				insn[i++] = BPF_MOV32_IMM(R0, __LINE__);

				insn[i++] = BPF_EXIT_INSN();

			}



			/* Check destination register value */

			if (!(rd == R0 && op == BPF_CMPXCHG) &&

			    !(rd == rs && (op & BPF_FETCH))) {

				insn[i++] = BPF_JMP_REG(BPF_JEQ, rd, R10, 2);

				insn[i++] = BPF_MOV32_IMM(R0, __LINE__);

				insn[i++] = BPF_EXIT_INSN();

			}



			/* Check value in memory */

			if (rs != rd) {                  /* No aliasing */

				i += __bpf_ld_imm64(&insn[i], R1, res);

			} else if (op == BPF_XCHG) {     /* Aliased, XCHG */

				insn[i++] = BPF_MOV64_REG(R1, R10);

			} else if (op == BPF_CMPXCHG) {  /* Aliased, CMPXCHG */

				i += __bpf_ld_imm64(&insn[i], R1, mem);

			} else {                        /* Aliased, ALU oper */

				i += __bpf_ld_imm64(&insn[i], R1, mem);

				insn[i++] = BPF_ALU64_REG(BPF_OP(op), R1, R10);

			}



			insn[i++] = BPF_LDX_MEM(width, R0, R10, -8);

			if (width == BPF_DW)

				insn[i++] = BPF_JMP_REG(BPF_JEQ, R0, R1, 2);

			else /* width == BPF_W */

				insn[i++] = BPF_JMP32_REG(BPF_JEQ, R0, R1, 2);

			insn[i++] = BPF_MOV32_IMM(R0, __LINE__);

			insn[i++] = BPF_EXIT_INSN();

		}

	}



	insn[i++] = BPF_MOV64_IMM(R0, 1);

	insn[i++] = BPF_EXIT_INSN();



	self->u.ptr.insns = insn;

	self->u.ptr.len = i;

	BUG_ON(i > len);



	return 0;

}



/* 64-bit atomic register tests */

static int bpf_fill_atomic64_add_reg_pairs(struct bpf_test *self)

{

	return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_ADD);

}



static int bpf_fill_atomic64_and_reg_pairs(struct bpf_test *self)

{

	return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_AND);

}



static int bpf_fill_atomic64_or_reg_pairs(struct bpf_test *self)

{

	return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_OR);

}



static int bpf_fill_atomic64_xor_reg_pairs(struct bpf_test *self)

{

	return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XOR);

}



static int bpf_fill_atomic64_add_fetch_reg_pairs(struct bpf_test *self)

{

	return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_ADD | BPF_FETCH);

}



static int bpf_fill_atomic64_and_fetch_reg_pairs(struct bpf_test *self)

{

	return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_AND | BPF_FETCH);

}



static int bpf_fill_atomic64_or_fetch_reg_pairs(struct bpf_test *self)

{

	return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_OR | BPF_FETCH);

}



static int bpf_fill_atomic64_xor_fetch_reg_pairs(struct bpf_test *self)

{

	return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XOR | BPF_FETCH);

}



static int bpf_fill_atomic64_xchg_reg_pairs(struct bpf_test *self)

{

	return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XCHG);

}



static int bpf_fill_atomic64_cmpxchg_reg_pairs(struct bpf_test *self)

{

	return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_CMPXCHG);

}



/* 32-bit atomic register tests */

static int bpf_fill_atomic32_add_reg_pairs(struct bpf_test *self)

{

	return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_ADD);

}



static int bpf_fill_atomic32_and_reg_pairs(struct bpf_test *self)

{

	return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_AND);

}



static int bpf_fill_atomic32_or_reg_pairs(struct bpf_test *self)

{

	return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_OR);

}



static int bpf_fill_atomic32_xor_reg_pairs(struct bpf_test *self)

{

	return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XOR);

}



static int bpf_fill_atomic32_add_fetch_reg_pairs(struct bpf_test *self)

{

	return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_ADD | BPF_FETCH);

}



static int bpf_fill_atomic32_and_fetch_reg_pairs(struct bpf_test *self)

{

	return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_AND | BPF_FETCH);

}



static int bpf_fill_atomic32_or_fetch_reg_pairs(struct bpf_test *self)

{

	return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_OR | BPF_FETCH);

}



static int bpf_fill_atomic32_xor_fetch_reg_pairs(struct bpf_test *self)

{

	return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XOR | BPF_FETCH);

}



static int bpf_fill_atomic32_xchg_reg_pairs(struct bpf_test *self)

{

	return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XCHG);

}



static int bpf_fill_atomic32_cmpxchg_reg_pairs(struct bpf_test *self)

{

	return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_CMPXCHG);

}



/*

 * Test the two-instruction 64-bit immediate load operation for all

 * power-of-two magnitudes of the immediate operand. For each MSB, a block
@@ -11976,6 +12216,188 @@ static struct bpf_test tests[] = { 
		{ { 0, 1 } },

		.fill_helper = bpf_fill_ld_imm64,

	},

	/* 64-bit ATOMIC register combinations */

	{

		"ATOMIC_DW_ADD: register combinations",

		{ },

		INTERNAL,

		{ },

		{ { 0, 1 } },

		.fill_helper = bpf_fill_atomic64_add_reg_pairs,

		.stack_depth = 8,

	},

	{

		"ATOMIC_DW_AND: register combinations",

		{ },

		INTERNAL,

		{ },

		{ { 0, 1 } },

		.fill_helper = bpf_fill_atomic64_and_reg_pairs,

		.stack_depth = 8,

	},

	{

		"ATOMIC_DW_OR: register combinations",

		{ },

		INTERNAL,

		{ },

		{ { 0, 1 } },

		.fill_helper = bpf_fill_atomic64_or_reg_pairs,

		.stack_depth = 8,

	},

	{

		"ATOMIC_DW_XOR: register combinations",

		{ },

		INTERNAL,

		{ },

		{ { 0, 1 } },

		.fill_helper = bpf_fill_atomic64_xor_reg_pairs,

		.stack_depth = 8,

	},

	{

		"ATOMIC_DW_ADD_FETCH: register combinations",

		{ },

		INTERNAL,

		{ },

		{ { 0, 1 } },

		.fill_helper = bpf_fill_atomic64_add_fetch_reg_pairs,

		.stack_depth = 8,

	},

	{

		"ATOMIC_DW_AND_FETCH: register combinations",

		{ },

		INTERNAL,

		{ },

		{ { 0, 1 } },

		.fill_helper = bpf_fill_atomic64_and_fetch_reg_pairs,

		.stack_depth = 8,

	},

	{

		"ATOMIC_DW_OR_FETCH: register combinations",

		{ },

		INTERNAL,

		{ },

		{ { 0, 1 } },

		.fill_helper = bpf_fill_atomic64_or_fetch_reg_pairs,

		.stack_depth = 8,

	},

	{

		"ATOMIC_DW_XOR_FETCH: register combinations",

		{ },

		INTERNAL,

		{ },

		{ { 0, 1 } },

		.fill_helper = bpf_fill_atomic64_xor_fetch_reg_pairs,

		.stack_depth = 8,

	},

	{

		"ATOMIC_DW_XCHG: register combinations",

		{ },

		INTERNAL,

		{ },

		{ { 0, 1 } },

		.fill_helper = bpf_fill_atomic64_xchg_reg_pairs,

		.stack_depth = 8,

	},

	{

		"ATOMIC_DW_CMPXCHG: register combinations",

		{ },

		INTERNAL,

		{ },

		{ { 0, 1 } },

		.fill_helper = bpf_fill_atomic64_cmpxchg_reg_pairs,

		.stack_depth = 8,

	},

	/* 32-bit ATOMIC register combinations */

	{

		"ATOMIC_W_ADD: register combinations",

		{ },

		INTERNAL,

		{ },

		{ { 0, 1 } },

		.fill_helper = bpf_fill_atomic32_add_reg_pairs,

		.stack_depth = 8,

	},

	{

		"ATOMIC_W_AND: register combinations",

		{ },

		INTERNAL,

		{ },

		{ { 0, 1 } },

		.fill_helper = bpf_fill_atomic32_and_reg_pairs,

		.stack_depth = 8,

	},

	{

		"ATOMIC_W_OR: register combinations",

		{ },

		INTERNAL,

		{ },

		{ { 0, 1 } },

		.fill_helper = bpf_fill_atomic32_or_reg_pairs,

		.stack_depth = 8,

	},

	{

		"ATOMIC_W_XOR: register combinations",

		{ },

		INTERNAL,

		{ },

		{ { 0, 1 } },

		.fill_helper = bpf_fill_atomic32_xor_reg_pairs,

		.stack_depth = 8,

	},

	{

		"ATOMIC_W_ADD_FETCH: register combinations",

		{ },

		INTERNAL,

		{ },

		{ { 0, 1 } },

		.fill_helper = bpf_fill_atomic32_add_fetch_reg_pairs,

		.stack_depth = 8,

	},

	{

		"ATOMIC_W_AND_FETCH: register combinations",

		{ },

		INTERNAL,

		{ },

		{ { 0, 1 } },

		.fill_helper = bpf_fill_atomic32_and_fetch_reg_pairs,

		.stack_depth = 8,

	},

	{

		"ATOMIC_W_OR_FETCH: register combinations",

		{ },

		INTERNAL,

		{ },

		{ { 0, 1 } },

		.fill_helper = bpf_fill_atomic32_or_fetch_reg_pairs,

		.stack_depth = 8,

	},

	{

		"ATOMIC_W_XOR_FETCH: register combinations",

		{ },

		INTERNAL,

		{ },

		{ { 0, 1 } },

		.fill_helper = bpf_fill_atomic32_xor_fetch_reg_pairs,

		.stack_depth = 8,

	},

	{

		"ATOMIC_W_XCHG: register combinations",

		{ },

		INTERNAL,

		{ },

		{ { 0, 1 } },

		.fill_helper = bpf_fill_atomic32_xchg_reg_pairs,

		.stack_depth = 8,

	},

	{

		"ATOMIC_W_CMPXCHG: register combinations",

		{ },

		INTERNAL,

		{ },

		{ { 0, 1 } },

		.fill_helper = bpf_fill_atomic32_cmpxchg_reg_pairs,

		.stack_depth = 8,

	},

	/* 64-bit ATOMIC magnitudes */

	{

		"ATOMIC_DW_ADD: all operand magnitudes",