Commit 1022a549 authored by Song Liu's avatar Song Liu Committed by Alexei Starovoitov
Browse files

bpf, x86_64: Use bpf_jit_binary_pack_alloc 



Use bpf_jit_binary_pack_alloc in x86_64 jit. The jit engine first writes
the program to the rw buffer. When the jit is done, the program is copied
to the final location with bpf_jit_binary_pack_finalize.

Note that we need to do bpf_tail_call_direct_fixup after finalize.
Therefore, the text_live = false logic in __bpf_arch_text_poke is no
longer needed.

Signed-off-by: default avatarSong Liu <songliubraving@fb.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/bpf/20220204185742.271030-10-song@kernel.org
parent 33c98058

arch/x86/net/bpf_jit_comp.c

+31 −27
Original line number Diff line number Diff line
@@ -330,8 +330,7 @@ static int emit_jump(u8 **pprog, void *func, void *ip) 
}



static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,

				void *old_addr, void *new_addr,

				const bool text_live)

				void *old_addr, void *new_addr)

{

	const u8 *nop_insn = x86_nops[5];

	u8 old_insn[X86_PATCH_SIZE];
@@ -365,10 +364,7 @@ static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, 
		goto out;

	ret = 1;

	if (memcmp(ip, new_insn, X86_PATCH_SIZE)) {

		if (text_live)

		text_poke_bp(ip, new_insn, X86_PATCH_SIZE, NULL);

		else

			memcpy(ip, new_insn, X86_PATCH_SIZE);

		ret = 0;

	}

out:
@@ -384,7 +380,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, 
		/* BPF poking in modules is not supported */

		return -EINVAL;



	return __bpf_arch_text_poke(ip, t, old_addr, new_addr, true);

	return __bpf_arch_text_poke(ip, t, old_addr, new_addr);

}



#define EMIT_LFENCE()	EMIT3(0x0F, 0xAE, 0xE8)
@@ -558,24 +554,15 @@ static void bpf_tail_call_direct_fixup(struct bpf_prog *prog) 
		mutex_lock(&array->aux->poke_mutex);

		target = array->ptrs[poke->tail_call.key];

		if (target) {

			/* Plain memcpy is used when image is not live yet

			 * and still not locked as read-only. Once poke

			 * location is active (poke->tailcall_target_stable),

			 * any parallel bpf_arch_text_poke() might occur

			 * still on the read-write image until we finally

			 * locked it as read-only. Both modifications on

			 * the given image are under text_mutex to avoid

			 * interference.

			 */

			ret = __bpf_arch_text_poke(poke->tailcall_target,

						   BPF_MOD_JUMP, NULL,

						   (u8 *)target->bpf_func +

						   poke->adj_off, false);

						   poke->adj_off);

			BUG_ON(ret < 0);

			ret = __bpf_arch_text_poke(poke->tailcall_bypass,

						   BPF_MOD_JUMP,

						   (u8 *)poke->tailcall_target +

						   X86_PATCH_SIZE, NULL, false);

						   X86_PATCH_SIZE, NULL);

			BUG_ON(ret < 0);

		}

		WRITE_ONCE(poke->tailcall_target_stable, true);
@@ -866,7 +853,7 @@ static void emit_nops(u8 **pprog, int len) 


#define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))



static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,

static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image,

		  int oldproglen, struct jit_context *ctx, bool jmp_padding)

{

	bool tail_call_reachable = bpf_prog->aux->tail_call_reachable;
@@ -893,8 +880,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, 
	push_callee_regs(&prog, callee_regs_used);



	ilen = prog - temp;

	if (image)

		memcpy(image + proglen, temp, ilen);

	if (rw_image)

		memcpy(rw_image + proglen, temp, ilen);

	proglen += ilen;

	addrs[0] = proglen;

	prog = temp;
@@ -1323,6 +1310,9 @@ st: if (is_imm8(insn->off)) 
					pr_err("extable->insn doesn't fit into 32-bit\n");

					return -EFAULT;

				}

				/* switch ex to rw buffer for writes */

				ex = (void *)rw_image + ((void *)ex - (void *)image);



				ex->insn = delta;



				ex->data = EX_TYPE_BPF;
@@ -1705,7 +1695,7 @@ st: if (is_imm8(insn->off)) 
				pr_err("bpf_jit: fatal error\n");

				return -EFAULT;

			}

			memcpy(image + proglen, temp, ilen);

			memcpy(rw_image + proglen, temp, ilen);

		}

		proglen += ilen;

		addrs[i] = proglen;
@@ -2246,6 +2236,7 @@ int arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs) 
}



struct x64_jit_data {

	struct bpf_binary_header *rw_header;

	struct bpf_binary_header *header;

	int *addrs;

	u8 *image;
@@ -2258,6 +2249,7 @@ struct x64_jit_data { 


struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)

{

	struct bpf_binary_header *rw_header = NULL;

	struct bpf_binary_header *header = NULL;

	struct bpf_prog *tmp, *orig_prog = prog;

	struct x64_jit_data *jit_data;
@@ -2266,6 +2258,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 
	bool tmp_blinded = false;

	bool extra_pass = false;

	bool padding = false;

	u8 *rw_image = NULL;

	u8 *image = NULL;

	int *addrs;

	int pass;
@@ -2301,6 +2294,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 
		oldproglen = jit_data->proglen;

		image = jit_data->image;

		header = jit_data->header;

		rw_header = jit_data->rw_header;

		rw_image = (void *)rw_header + ((void *)image - (void *)header);

		extra_pass = true;

		padding = true;

		goto skip_init_addrs;
@@ -2331,12 +2326,12 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 
	for (pass = 0; pass < MAX_PASSES || image; pass++) {

		if (!padding && pass >= PADDING_PASSES)

			padding = true;

		proglen = do_jit(prog, addrs, image, oldproglen, &ctx, padding);

		proglen = do_jit(prog, addrs, image, rw_image, oldproglen, &ctx, padding);

		if (proglen <= 0) {

out_image:

			image = NULL;

			if (header)

				bpf_jit_binary_free(header);

				bpf_jit_binary_pack_free(header, rw_header);

			prog = orig_prog;

			goto out_addrs;

		}
@@ -2360,8 +2355,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 
				sizeof(struct exception_table_entry);



			/* allocate module memory for x86 insns and extable */

			header = bpf_jit_binary_alloc(roundup(proglen, align) + extable_size,

						      &image, align, jit_fill_hole);

			header = bpf_jit_binary_pack_alloc(roundup(proglen, align) + extable_size,

							   &image, align, &rw_header, &rw_image,

							   jit_fill_hole);

			if (!header) {

				prog = orig_prog;

				goto out_addrs;
@@ -2377,14 +2373,22 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 


	if (image) {

		if (!prog->is_func || extra_pass) {

			/*

			 * bpf_jit_binary_pack_finalize fails in two scenarios:

			 *   1) header is not pointing to proper module memory;

			 *   2) the arch doesn't support bpf_arch_text_copy().

			 *

			 * Both cases are serious bugs that we should not continue.

			 */

			BUG_ON(bpf_jit_binary_pack_finalize(prog, header, rw_header));

			bpf_tail_call_direct_fixup(prog);

			bpf_jit_binary_lock_ro(header);

		} else {

			jit_data->addrs = addrs;

			jit_data->ctx = ctx;

			jit_data->proglen = proglen;

			jit_data->image = image;

			jit_data->header = header;

			jit_data->rw_header = rw_header;

		}

		prog->bpf_func = (void *)image;

		prog->jited = 1;