Commit 5ca419f2 authored by Brendan Jackman's avatar Brendan Jackman Committed by Alexei Starovoitov
Browse files

bpf: Add BPF_FETCH field / create atomic_fetch_add instruction 



The BPF_FETCH field can be set in bpf_insn.imm, for BPF_ATOMIC
instructions, in order to have the previous value of the
atomically-modified memory location loaded into the src register
after an atomic op is carried out.

Suggested-by: default avatarYonghong Song <yhs@fb.com>
Signed-off-by: default avatarBrendan Jackman <jackmanb@google.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Acked-by: default avatarJohn Fastabend <john.fastabend@gmail.com>
Link: https://lore.kernel.org/bpf/20210114181751.768687-7-jackmanb@google.com
parent c5bcb5eb

arch/x86/net/bpf_jit_comp.c

+4 −0
Original line number Diff line number Diff line
@@ -811,6 +811,10 @@ static int emit_atomic(u8 **pprog, u8 atomic_op, 
		/* lock *(u32/u64*)(dst_reg + off) <op>= src_reg */

		EMIT1(simple_alu_opcodes[atomic_op]);

		break;

	case BPF_ADD | BPF_FETCH:

		/* src_reg = atomic_fetch_add(dst_reg + off, src_reg); */

		EMIT2(0x0F, 0xC1);

		break;

	default:

		pr_err("bpf_jit: unknown atomic opcode %02x\n", atomic_op);

		return -EFAULT;

include/linux/filter.h

+1 −0
Original line number Diff line number Diff line
@@ -264,6 +264,7 @@ static inline bool insn_is_zext(const struct bpf_insn *insn) 
 * Atomic operations:

 *

 *   BPF_ADD                  *(uint *) (dst_reg + off16) += src_reg

 *   BPF_ADD | BPF_FETCH      src_reg = atomic_fetch_add(dst_reg + off16, src_reg);

 */



#define BPF_ATOMIC_OP(SIZE, OP, DST, SRC, OFF)			\

include/uapi/linux/bpf.h

+3 −0
Original line number Diff line number Diff line
@@ -44,6 +44,9 @@ 
#define BPF_CALL	0x80	/* function call */

#define BPF_EXIT	0x90	/* function return */



/* atomic op type fields (stored in immediate) */

#define BPF_FETCH	0x01	/* fetch previous value into src reg */



/* Register numbers */

enum {

	BPF_REG_0 = 0,

kernel/bpf/core.c

+13 −0
Original line number Diff line number Diff line
@@ -1624,16 +1624,29 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn, u64 *stack) 
			/* lock xadd *(u32 *)(dst_reg + off16) += src_reg */

			atomic_add((u32) SRC, (atomic_t *)(unsigned long)

				   (DST + insn->off));

			break;

		case BPF_ADD | BPF_FETCH:

			SRC = (u32) atomic_fetch_add(

				(u32) SRC,

				(atomic_t *)(unsigned long) (DST + insn->off));

			break;

		default:

			goto default_label;

		}

		CONT;



	STX_ATOMIC_DW:

		switch (IMM) {

		case BPF_ADD:

			/* lock xadd *(u64 *)(dst_reg + off16) += src_reg */

			atomic64_add((u64) SRC, (atomic64_t *)(unsigned long)

				     (DST + insn->off));

			break;

		case BPF_ADD | BPF_FETCH:

			SRC = (u64) atomic64_fetch_add(

				(u64) SRC,

				(atomic64_t *)(unsigned long) (DST + insn->off));

			break;

		default:

			goto default_label;

		}

kernel/bpf/disasm.c

+7 −0
Original line number Diff line number Diff line
@@ -160,6 +160,13 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, 
				bpf_ldst_string[BPF_SIZE(insn->code) >> 3],

				insn->dst_reg, insn->off,

				insn->src_reg);

		} else if (BPF_MODE(insn->code) == BPF_ATOMIC &&

			   insn->imm == (BPF_ADD | BPF_FETCH)) {

			verbose(cbs->private_data, "(%02x) r%d = atomic%s_fetch_add((%s *)(r%d %+d), r%d)\n",

				insn->code, insn->src_reg,

				BPF_SIZE(insn->code) == BPF_DW ? "64" : "",

				bpf_ldst_string[BPF_SIZE(insn->code) >> 3],

				insn->dst_reg, insn->off, insn->src_reg);

		} else {

			verbose(cbs->private_data, "BUG_%02x\n", insn->code);

		}