Commit dd642ccb authored by Pu Lehui's avatar Pu Lehui Committed by Daniel Borkmann
Browse files

riscv, bpf: Implement more atomic operations for RV64 



This patch implement more BPF atomic operations for RV64. The newly
added operations are shown below:

  atomic[64]_[fetch_]add
  atomic[64]_[fetch_]and
  atomic[64]_[fetch_]or
  atomic[64]_xchg
  atomic[64]_cmpxchg

Since riscv specification does not provide AMO instruction for CAS
operation, we use lr/sc instruction for cmpxchg operation, and AMO
instructions for the rest ops.

Tests "test_bpf.ko" and "test_progs -t atomic" have passed, as well
as "test_verifier" with no new failure cases.

Signed-off-by: default avatarPu Lehui <pulehui@huawei.com>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Acked-by: default avatarSong Liu <songliubraving@fb.com>
Acked-by: default avatarBjörn Töpel <bjorn@kernel.org>
Link: https://lore.kernel.org/bpf/20220410101246.232875-1-pulehui@huawei.com
parent 33fc250c

arch/riscv/net/bpf_jit.h

+67 −0
Original line number Original line Diff line number Diff line
@@ -535,6 +535,43 @@ static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 
	return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f);

	return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f);

}

}





static inline u32 rv_amoand_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)

{

	return rv_amo_insn(0xc, aq, rl, rs2, rs1, 2, rd, 0x2f);

}



static inline u32 rv_amoor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)

{

	return rv_amo_insn(0x8, aq, rl, rs2, rs1, 2, rd, 0x2f);

}



static inline u32 rv_amoxor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)

{

	return rv_amo_insn(0x4, aq, rl, rs2, rs1, 2, rd, 0x2f);

}



static inline u32 rv_amoswap_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)

{

	return rv_amo_insn(0x1, aq, rl, rs2, rs1, 2, rd, 0x2f);

}



static inline u32 rv_lr_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)

{

	return rv_amo_insn(0x2, aq, rl, rs2, rs1, 2, rd, 0x2f);

}



static inline u32 rv_sc_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)

{

	return rv_amo_insn(0x3, aq, rl, rs2, rs1, 2, rd, 0x2f);

}



static inline u32 rv_fence(u8 pred, u8 succ)

{

	u16 imm11_0 = pred << 4 | succ;



	return rv_i_insn(imm11_0, 0, 0, 0, 0xf);

}



/* RVC instrutions. */

/* RVC instrutions. */





static inline u16 rvc_addi4spn(u8 rd, u32 imm10)

static inline u16 rvc_addi4spn(u8 rd, u32 imm10)
@@ -753,6 +790,36 @@ static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 
	return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f);

	return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f);

}

}





static inline u32 rv_amoand_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)

{

	return rv_amo_insn(0xc, aq, rl, rs2, rs1, 3, rd, 0x2f);

}



static inline u32 rv_amoor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)

{

	return rv_amo_insn(0x8, aq, rl, rs2, rs1, 3, rd, 0x2f);

}



static inline u32 rv_amoxor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)

{

	return rv_amo_insn(0x4, aq, rl, rs2, rs1, 3, rd, 0x2f);

}



static inline u32 rv_amoswap_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)

{

	return rv_amo_insn(0x1, aq, rl, rs2, rs1, 3, rd, 0x2f);

}



static inline u32 rv_lr_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)

{

	return rv_amo_insn(0x2, aq, rl, rs2, rs1, 3, rd, 0x2f);

}



static inline u32 rv_sc_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)

{

	return rv_amo_insn(0x3, aq, rl, rs2, rs1, 3, rd, 0x2f);

}



/* RV64-only RVC instructions. */

/* RV64-only RVC instructions. */





static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1)

static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1)

arch/riscv/net/bpf_jit_comp64.c

+86 −24
Original line number Original line Diff line number Diff line
@@ -455,6 +455,90 @@ static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx) 
	return 0;

	return 0;

}

}





static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64,

			struct rv_jit_context *ctx)

{

	u8 r0;

	int jmp_offset;



	if (off) {

		if (is_12b_int(off)) {

			emit_addi(RV_REG_T1, rd, off, ctx);

		} else {

			emit_imm(RV_REG_T1, off, ctx);

			emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);

		}

		rd = RV_REG_T1;

	}



	switch (imm) {

	/* lock *(u32/u64 *)(dst_reg + off16) <op>= src_reg */

	case BPF_ADD:

		emit(is64 ? rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0) :

		     rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0), ctx);

		break;

	case BPF_AND:

		emit(is64 ? rv_amoand_d(RV_REG_ZERO, rs, rd, 0, 0) :

		     rv_amoand_w(RV_REG_ZERO, rs, rd, 0, 0), ctx);

		break;

	case BPF_OR:

		emit(is64 ? rv_amoor_d(RV_REG_ZERO, rs, rd, 0, 0) :

		     rv_amoor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx);

		break;

	case BPF_XOR:

		emit(is64 ? rv_amoxor_d(RV_REG_ZERO, rs, rd, 0, 0) :

		     rv_amoxor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx);

		break;

	/* src_reg = atomic_fetch_<op>(dst_reg + off16, src_reg) */

	case BPF_ADD | BPF_FETCH:

		emit(is64 ? rv_amoadd_d(rs, rs, rd, 0, 0) :

		     rv_amoadd_w(rs, rs, rd, 0, 0), ctx);

		if (!is64)

			emit_zext_32(rs, ctx);

		break;

	case BPF_AND | BPF_FETCH:

		emit(is64 ? rv_amoand_d(rs, rs, rd, 0, 0) :

		     rv_amoand_w(rs, rs, rd, 0, 0), ctx);

		if (!is64)

			emit_zext_32(rs, ctx);

		break;

	case BPF_OR | BPF_FETCH:

		emit(is64 ? rv_amoor_d(rs, rs, rd, 0, 0) :

		     rv_amoor_w(rs, rs, rd, 0, 0), ctx);

		if (!is64)

			emit_zext_32(rs, ctx);

		break;

	case BPF_XOR | BPF_FETCH:

		emit(is64 ? rv_amoxor_d(rs, rs, rd, 0, 0) :

		     rv_amoxor_w(rs, rs, rd, 0, 0), ctx);

		if (!is64)

			emit_zext_32(rs, ctx);

		break;

	/* src_reg = atomic_xchg(dst_reg + off16, src_reg); */

	case BPF_XCHG:

		emit(is64 ? rv_amoswap_d(rs, rs, rd, 0, 0) :

		     rv_amoswap_w(rs, rs, rd, 0, 0), ctx);

		if (!is64)

			emit_zext_32(rs, ctx);

		break;

	/* r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg); */

	case BPF_CMPXCHG:

		r0 = bpf_to_rv_reg(BPF_REG_0, ctx);

		emit(is64 ? rv_addi(RV_REG_T2, r0, 0) :

		     rv_addiw(RV_REG_T2, r0, 0), ctx);

		emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) :

		     rv_lr_w(r0, 0, rd, 0, 0), ctx);

		jmp_offset = ninsns_rvoff(8);

		emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx);

		emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 0) :

		     rv_sc_w(RV_REG_T3, rs, rd, 0, 0), ctx);

		jmp_offset = ninsns_rvoff(-6);

		emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx);

		emit(rv_fence(0x3, 0x3), ctx);

		break;

	}

}



#define BPF_FIXUP_OFFSET_MASK   GENMASK(26, 0)

#define BPF_FIXUP_OFFSET_MASK   GENMASK(26, 0)

#define BPF_FIXUP_REG_MASK      GENMASK(31, 27)

#define BPF_FIXUP_REG_MASK      GENMASK(31, 27)
@@ -1146,30 +1230,8 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
		break;

		break;

	case BPF_STX | BPF_ATOMIC | BPF_W:

	case BPF_STX | BPF_ATOMIC | BPF_W:

	case BPF_STX | BPF_ATOMIC | BPF_DW:

	case BPF_STX | BPF_ATOMIC | BPF_DW:

		if (insn->imm != BPF_ADD) {

		emit_atomic(rd, rs, off, imm,

			pr_err("bpf-jit: not supported: atomic operation %02x ***\n",

			    BPF_SIZE(code) == BPF_DW, ctx);

			       insn->imm);

			return -EINVAL;

		}



		/* atomic_add: lock *(u32 *)(dst + off) += src

		 * atomic_add: lock *(u64 *)(dst + off) += src

		 */



		if (off) {

			if (is_12b_int(off)) {

				emit_addi(RV_REG_T1, rd, off, ctx);

			} else {

				emit_imm(RV_REG_T1, off, ctx);

				emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);

			}



			rd = RV_REG_T1;

		}



		emit(BPF_SIZE(code) == BPF_W ?

		     rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0) :

		     rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0), ctx);

		break;

		break;

	default:

	default:

		pr_err("bpf-jit: unknown opcode %02x\n", code);

		pr_err("bpf-jit: unknown opcode %02x\n", code);