Commit 91c960b0 authored by Brendan Jackman's avatar Brendan Jackman Committed by Alexei Starovoitov
Browse files

bpf: Rename BPF_XADD and prepare to encode other atomics in .imm 



A subsequent patch will add additional atomic operations. These new
operations will use the same opcode field as the existing XADD, with
the immediate discriminating different operations.

In preparation, rename the instruction mode BPF_ATOMIC and start
calling the zero immediate BPF_ADD.

This is possible (doesn't break existing valid BPF progs) because the
immediate field is currently reserved MBZ and BPF_ADD is zero.

All uses are removed from the tree but the BPF_XADD definition is
kept around to avoid breaking builds for people including kernel
headers.

Signed-off-by: default avatarBrendan Jackman <jackmanb@google.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Acked-by: default avatarBjörn Töpel <bjorn.topel@gmail.com>
Link: https://lore.kernel.org/bpf/20210114181751.768687-5-jackmanb@google.com
parent e5f02cac

Documentation/networking/filter.rst

+19 −11
Original line number Diff line number Diff line
@@ -1012,7 +1012,7 @@ Mode modifier is one of:: 
  BPF_MEM     0x60

  BPF_LEN     0x80  /* classic BPF only, reserved in eBPF */

  BPF_MSH     0xa0  /* classic BPF only, reserved in eBPF */

  BPF_XADD 0xc0  /* eBPF only, exclusive add */

  BPF_ATOMIC  0xc0  /* eBPF only, atomic operations */



eBPF has two non-generic instructions: (BPF_ABS | <size> | BPF_LD) and

(BPF_IND | <size> | BPF_LD) which are used to access packet data.
@@ -1044,11 +1044,19 @@ Unlike classic BPF instruction set, eBPF has generic load/store operations:: 
    BPF_MEM | <size> | BPF_STX:  *(size *) (dst_reg + off) = src_reg

    BPF_MEM | <size> | BPF_ST:   *(size *) (dst_reg + off) = imm32

    BPF_MEM | <size> | BPF_LDX:  dst_reg = *(size *) (src_reg + off)

    BPF_XADD | BPF_W  | BPF_STX: lock xadd *(u32 *)(dst_reg + off16) += src_reg

    BPF_XADD | BPF_DW | BPF_STX: lock xadd *(u64 *)(dst_reg + off16) += src_reg



Where size is one of: BPF_B or BPF_H or BPF_W or BPF_DW. Note that 1 and

2 byte atomic increments are not supported.

Where size is one of: BPF_B or BPF_H or BPF_W or BPF_DW.



It also includes atomic operations, which use the immediate field for extra

encoding.



   .imm = BPF_ADD, .code = BPF_ATOMIC | BPF_W  | BPF_STX: lock xadd *(u32 *)(dst_reg + off16) += src_reg

   .imm = BPF_ADD, .code = BPF_ATOMIC | BPF_DW | BPF_STX: lock xadd *(u64 *)(dst_reg + off16) += src_reg



Note that 1 and 2 byte atomic operations are not supported.



You may encounter BPF_XADD - this is a legacy name for BPF_ATOMIC, referring to

the exclusive-add operation encoded when the immediate field is zero.



eBPF has one 16-byte instruction: BPF_LD | BPF_DW | BPF_IMM which consists

of two consecutive ``struct bpf_insn`` 8-byte blocks and interpreted as single

arch/arm/net/bpf_jit_32.c

+3 −4
Original line number Diff line number Diff line
@@ -1620,10 +1620,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) 
		}

		emit_str_r(dst_lo, tmp2, off, ctx, BPF_SIZE(code));

		break;

	/* STX XADD: lock *(u32 *)(dst + off) += src */

	case BPF_STX | BPF_XADD | BPF_W:

	/* STX XADD: lock *(u64 *)(dst + off) += src */

	case BPF_STX | BPF_XADD | BPF_DW:

	/* Atomic ops */

	case BPF_STX | BPF_ATOMIC | BPF_W:

	case BPF_STX | BPF_ATOMIC | BPF_DW:

		goto notyet;

	/* STX: *(size *)(dst + off) = src */

	case BPF_STX | BPF_MEM | BPF_W:

arch/arm64/net/bpf_jit_comp.c

+12 −4
Original line number Diff line number Diff line
@@ -875,10 +875,18 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, 
		}

		break;



	/* STX XADD: lock *(u32 *)(dst + off) += src */

	case BPF_STX | BPF_XADD | BPF_W:

	/* STX XADD: lock *(u64 *)(dst + off) += src */

	case BPF_STX | BPF_XADD | BPF_DW:

	case BPF_STX | BPF_ATOMIC | BPF_W:

	case BPF_STX | BPF_ATOMIC | BPF_DW:

		if (insn->imm != BPF_ADD) {

			pr_err_once("unknown atomic op code %02x\n", insn->imm);

			return -EINVAL;

		}



		/* STX XADD: lock *(u32 *)(dst + off) += src

		 * and

		 * STX XADD: lock *(u64 *)(dst + off) += src

		 */



		if (!off) {

			reg = dst;

		} else {

arch/mips/net/ebpf_jit.c

+8 −3
Original line number Diff line number Diff line
@@ -1423,8 +1423,8 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, 
	case BPF_STX | BPF_H | BPF_MEM:

	case BPF_STX | BPF_W | BPF_MEM:

	case BPF_STX | BPF_DW | BPF_MEM:

	case BPF_STX | BPF_W | BPF_XADD:

	case BPF_STX | BPF_DW | BPF_XADD:

	case BPF_STX | BPF_W | BPF_ATOMIC:

	case BPF_STX | BPF_DW | BPF_ATOMIC:

		if (insn->dst_reg == BPF_REG_10) {

			ctx->flags |= EBPF_SEEN_FP;

			dst = MIPS_R_SP;
@@ -1438,7 +1438,12 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, 
		src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp);

		if (src < 0)

			return src;

		if (BPF_MODE(insn->code) == BPF_XADD) {

		if (BPF_MODE(insn->code) == BPF_ATOMIC) {

			if (insn->imm != BPF_ADD) {

				pr_err("ATOMIC OP %02x NOT HANDLED\n", insn->imm);

				return -EINVAL;

			}



			/*

			 * If mem_off does not fit within the 9 bit ll/sc

			 * instruction immediate field, use a temp reg.

arch/powerpc/net/bpf_jit_comp64.c

+20 −5
Original line number Diff line number Diff line
@@ -683,10 +683,18 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, 
			break;



		/*

		 * BPF_STX XADD (atomic_add)

		 * BPF_STX ATOMIC (atomic ops)

		 */

		case BPF_STX | BPF_ATOMIC | BPF_W:

			if (insn->imm != BPF_ADD) {

				pr_err_ratelimited(

					"eBPF filter atomic op code %02x (@%d) unsupported\n",

					code, i);

				return -ENOTSUPP;

			}



			/* *(u32 *)(dst + off) += src */

		case BPF_STX | BPF_XADD | BPF_W:



			/* Get EA into TMP_REG_1 */

			EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], dst_reg, off));

			tmp_idx = ctx->idx * 4;
@@ -699,8 +707,15 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, 
			/* we're done if this succeeded */

			PPC_BCC_SHORT(COND_NE, tmp_idx);

			break;

		case BPF_STX | BPF_ATOMIC | BPF_DW:

			if (insn->imm != BPF_ADD) {

				pr_err_ratelimited(

					"eBPF filter atomic op code %02x (@%d) unsupported\n",

					code, i);

				return -ENOTSUPP;

			}

			/* *(u64 *)(dst + off) += src */

		case BPF_STX | BPF_XADD | BPF_DW:



			EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], dst_reg, off));

			tmp_idx = ctx->idx * 4;

			EMIT(PPC_RAW_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0));