Commit 9e3b47ab authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'add-support-cpu-v4-insns-for-rv64' 

Pu Lehui says:

====================
Add support cpu v4 insns for RV64

Add support cpu v4 instructions for RV64. The relevant tests have passed as show bellow:

Summary: 6/166 PASSED, 0 SKIPPED, 0 FAILED

NOTE: ldsx_insn testcase uses fentry and needs to rely on ftrace direct call [0].
[0] https://lore.kernel.org/all/20230627111612.761164-1-suagrfillet@gmail.com/

v2:
- Use temporary reg to avoid clobbering the source reg in movs_8/16 insns. (Björn)
- Add Acked-by

v1:
https://lore.kernel.org/bpf/20230823231059.3363698-1-pulehui@huaweicloud.com


====================

Tested-by: default avatarBjörn Töpel <bjorn@rivosinc.com>
Link: https://lore.kernel.org/r/20230824095001.3408573-1-pulehui@huaweicloud.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 1b580c9b 0209fd51

arch/riscv/net/bpf_jit.h

+30 −0
Original line number Diff line number Diff line
@@ -431,11 +431,21 @@ static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2) 
	return rv_r_insn(1, rs2, rs1, 3, rd, 0x33);

}



static inline u32 rv_div(u8 rd, u8 rs1, u8 rs2)

{

	return rv_r_insn(1, rs2, rs1, 4, rd, 0x33);

}



static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2)

{

	return rv_r_insn(1, rs2, rs1, 5, rd, 0x33);

}



static inline u32 rv_rem(u8 rd, u8 rs1, u8 rs2)

{

	return rv_r_insn(1, rs2, rs1, 6, rd, 0x33);

}



static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2)

{

	return rv_r_insn(1, rs2, rs1, 7, rd, 0x33);
@@ -501,6 +511,16 @@ static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1) 
	return rv_bge(rs2, rs1, imm12_1);

}



static inline u32 rv_lb(u8 rd, u16 imm11_0, u8 rs1)

{

	return rv_i_insn(imm11_0, rs1, 0, rd, 0x03);

}



static inline u32 rv_lh(u8 rd, u16 imm11_0, u8 rs1)

{

	return rv_i_insn(imm11_0, rs1, 1, rd, 0x03);

}



static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1)

{

	return rv_i_insn(imm11_0, rs1, 2, rd, 0x03);
@@ -766,11 +786,21 @@ static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2) 
	return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b);

}



static inline u32 rv_divw(u8 rd, u8 rs1, u8 rs2)

{

	return rv_r_insn(1, rs2, rs1, 4, rd, 0x3b);

}



static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2)

{

	return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b);

}



static inline u32 rv_remw(u8 rd, u8 rs1, u8 rs2)

{

	return rv_r_insn(1, rs2, rs1, 6, rd, 0x3b);

}



static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2)

{

	return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b);

arch/riscv/net/bpf_jit_comp64.c

+80 −22
Original line number Diff line number Diff line
@@ -580,7 +580,8 @@ static int add_exception_handler(const struct bpf_insn *insn, 
	unsigned long pc;

	off_t offset;



	if (!ctx->insns || !ctx->prog->aux->extable || BPF_MODE(insn->code) != BPF_PROBE_MEM)

	if (!ctx->insns || !ctx->prog->aux->extable ||

	    (BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX))

		return 0;



	if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries))
@@ -1046,7 +1047,19 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
			emit_zext_32(rd, ctx);

			break;

		}

		switch (insn->off) {

		case 0:

			emit_mv(rd, rs, ctx);

			break;

		case 8:

		case 16:

			emit_slli(RV_REG_T1, rs, 64 - insn->off, ctx);

			emit_srai(rd, RV_REG_T1, 64 - insn->off, ctx);

			break;

		case 32:

			emit_addiw(rd, rs, 0, ctx);

			break;

		}

		if (!is64 && !aux->verifier_zext)

			emit_zext_32(rd, ctx);

		break;
@@ -1094,12 +1107,18 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
		break;

	case BPF_ALU | BPF_DIV | BPF_X:

	case BPF_ALU64 | BPF_DIV | BPF_X:

		if (off)

			emit(is64 ? rv_div(rd, rd, rs) : rv_divw(rd, rd, rs), ctx);

		else

			emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx);

		if (!is64 && !aux->verifier_zext)

			emit_zext_32(rd, ctx);

		break;

	case BPF_ALU | BPF_MOD | BPF_X:

	case BPF_ALU64 | BPF_MOD | BPF_X:

		if (off)

			emit(is64 ? rv_rem(rd, rd, rs) : rv_remw(rd, rd, rs), ctx);

		else

			emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx);

		if (!is64 && !aux->verifier_zext)

			emit_zext_32(rd, ctx);
@@ -1149,6 +1168,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
		break;



	case BPF_ALU | BPF_END | BPF_FROM_BE:

	case BPF_ALU64 | BPF_END | BPF_FROM_LE:

		emit_li(RV_REG_T2, 0, ctx);



		emit_andi(RV_REG_T1, rd, 0xff, ctx);
@@ -1271,6 +1291,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
	case BPF_ALU | BPF_DIV | BPF_K:

	case BPF_ALU64 | BPF_DIV | BPF_K:

		emit_imm(RV_REG_T1, imm, ctx);

		if (off)

			emit(is64 ? rv_div(rd, rd, RV_REG_T1) :

			     rv_divw(rd, rd, RV_REG_T1), ctx);

		else

			emit(is64 ? rv_divu(rd, rd, RV_REG_T1) :

			     rv_divuw(rd, rd, RV_REG_T1), ctx);

		if (!is64 && !aux->verifier_zext)
@@ -1279,6 +1303,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
	case BPF_ALU | BPF_MOD | BPF_K:

	case BPF_ALU64 | BPF_MOD | BPF_K:

		emit_imm(RV_REG_T1, imm, ctx);

		if (off)

			emit(is64 ? rv_rem(rd, rd, RV_REG_T1) :

			     rv_remw(rd, rd, RV_REG_T1), ctx);

		else

			emit(is64 ? rv_remu(rd, rd, RV_REG_T1) :

			     rv_remuw(rd, rd, RV_REG_T1), ctx);

		if (!is64 && !aux->verifier_zext)
@@ -1314,7 +1342,11 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 


	/* JUMP off */

	case BPF_JMP | BPF_JA:

	case BPF_JMP32 | BPF_JA:

		if (BPF_CLASS(code) == BPF_JMP)

			rvoff = rv_offset(i, off, ctx);

		else

			rvoff = rv_offset(i, imm, ctx);

		ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx);

		if (ret)

			return ret;
@@ -1486,7 +1518,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
		return 1;

	}



	/* LDX: dst = *(size *)(src + off) */

	/* LDX: dst = *(unsigned size *)(src + off) */

	case BPF_LDX | BPF_MEM | BPF_B:

	case BPF_LDX | BPF_MEM | BPF_H:

	case BPF_LDX | BPF_MEM | BPF_W:
@@ -1495,13 +1527,27 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
	case BPF_LDX | BPF_PROBE_MEM | BPF_H:

	case BPF_LDX | BPF_PROBE_MEM | BPF_W:

	case BPF_LDX | BPF_PROBE_MEM | BPF_DW:

	/* LDSX: dst = *(signed size *)(src + off) */

	case BPF_LDX | BPF_MEMSX | BPF_B:

	case BPF_LDX | BPF_MEMSX | BPF_H:

	case BPF_LDX | BPF_MEMSX | BPF_W:

	case BPF_LDX | BPF_PROBE_MEMSX | BPF_B:

	case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:

	case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:

	{

		int insn_len, insns_start;

		bool sign_ext;



		sign_ext = BPF_MODE(insn->code) == BPF_MEMSX ||

			   BPF_MODE(insn->code) == BPF_PROBE_MEMSX;



		switch (BPF_SIZE(code)) {

		case BPF_B:

			if (is_12b_int(off)) {

				insns_start = ctx->ninsns;

				if (sign_ext)

					emit(rv_lb(rd, off, rs), ctx);

				else

					emit(rv_lbu(rd, off, rs), ctx);

				insn_len = ctx->ninsns - insns_start;

				break;
@@ -1510,14 +1556,18 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
			emit_imm(RV_REG_T1, off, ctx);

			emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);

			insns_start = ctx->ninsns;

			if (sign_ext)

				emit(rv_lb(rd, 0, RV_REG_T1), ctx);

			else

				emit(rv_lbu(rd, 0, RV_REG_T1), ctx);

			insn_len = ctx->ninsns - insns_start;

			if (insn_is_zext(&insn[1]))

				return 1;

			break;

		case BPF_H:

			if (is_12b_int(off)) {

				insns_start = ctx->ninsns;

				if (sign_ext)

					emit(rv_lh(rd, off, rs), ctx);

				else

					emit(rv_lhu(rd, off, rs), ctx);

				insn_len = ctx->ninsns - insns_start;

				break;
@@ -1526,14 +1576,18 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
			emit_imm(RV_REG_T1, off, ctx);

			emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);

			insns_start = ctx->ninsns;

			if (sign_ext)

				emit(rv_lh(rd, 0, RV_REG_T1), ctx);

			else

				emit(rv_lhu(rd, 0, RV_REG_T1), ctx);

			insn_len = ctx->ninsns - insns_start;

			if (insn_is_zext(&insn[1]))

				return 1;

			break;

		case BPF_W:

			if (is_12b_int(off)) {

				insns_start = ctx->ninsns;

				if (sign_ext)

					emit(rv_lw(rd, off, rs), ctx);

				else

					emit(rv_lwu(rd, off, rs), ctx);

				insn_len = ctx->ninsns - insns_start;

				break;
@@ -1542,10 +1596,11 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
			emit_imm(RV_REG_T1, off, ctx);

			emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);

			insns_start = ctx->ninsns;

			if (sign_ext)

				emit(rv_lw(rd, 0, RV_REG_T1), ctx);

			else

				emit(rv_lwu(rd, 0, RV_REG_T1), ctx);

			insn_len = ctx->ninsns - insns_start;

			if (insn_is_zext(&insn[1]))

				return 1;

			break;

		case BPF_DW:

			if (is_12b_int(off)) {
@@ -1566,6 +1621,9 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
		ret = add_exception_handler(insn, ctx, rd, insn_len);

		if (ret)

			return ret;



		if (BPF_SIZE(code) != BPF_DW && insn_is_zext(&insn[1]))

			return 1;

		break;

	}

	/* speculation barrier */

tools/testing/selftests/bpf/progs/test_ldsx_insn.c

+2 −1
Original line number Diff line number Diff line
@@ -5,7 +5,8 @@ 
#include <bpf/bpf_helpers.h>

#include <bpf/bpf_tracing.h>



#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) && __clang_major__ >= 18

#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \

     (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && __clang_major__ >= 18

const volatile int skip = 0;

#else

const volatile int skip = 1;

tools/testing/selftests/bpf/progs/verifier_bswap.c

+2 −1
Original line number Diff line number Diff line
@@ -4,7 +4,8 @@ 
#include <bpf/bpf_helpers.h>

#include "bpf_misc.h"



#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) && __clang_major__ >= 18

#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \

     (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && __clang_major__ >= 18



SEC("socket")

__description("BSWAP, 16")

tools/testing/selftests/bpf/progs/verifier_gotol.c

+2 −1
Original line number Diff line number Diff line
@@ -4,7 +4,8 @@ 
#include <bpf/bpf_helpers.h>

#include "bpf_misc.h"



#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) && __clang_major__ >= 18

#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \

     (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && __clang_major__ >= 18



SEC("socket")

__description("gotol, small_imm")