Commit 1d5f82d9 authored by Song Liu's avatar Song Liu Committed by Alexei Starovoitov
Browse files

bpf, x86: fix freeing of not-finalized bpf_prog_pack 

syzbot reported a few issues with bpf_prog_pack [1], [2]. This only happens
with multiple subprogs. In jit_subprogs(), we first call bpf_int_jit_compile()
on each sub program. And then, we call it on each sub program again. jit_data
is not freed in the first call of bpf_int_jit_compile(). Similarly we don't
call bpf_jit_binary_pack_finalize() in the first call of bpf_int_jit_compile().

If bpf_int_jit_compile() failed for one sub program, we will call
bpf_jit_binary_pack_finalize() for this sub program. However, we don't have a
chance to call it for other sub programs. Then we will hit "goto out_free" in
jit_subprogs(), and call bpf_jit_free on some subprograms that haven't got
bpf_jit_binary_pack_finalize() yet.

At this point, bpf_jit_binary_pack_free() is called and the whole 2MB page is
freed erroneously.

Fix this with a custom bpf_jit_free() for x86_64, which calls
bpf_jit_binary_pack_finalize() if necessary. Also, with custom
bpf_jit_free(), bpf_prog_aux->use_bpf_prog_pack is not needed any more,
remove it.

Fixes: 1022a549 ("bpf, x86_64: Use bpf_jit_binary_pack_alloc")
[1] https://syzkaller.appspot.com/bug?extid=2f649ec6d2eea1495a8f
[2] https://syzkaller.appspot.com/bug?extid=87f65c75f4a72db05445


Reported-by: default avatar <syzbot+2f649ec6d2eea1495a8f@syzkaller.appspotmail.com>
Reported-by: default avatar <syzbot+87f65c75f4a72db05445@syzkaller.appspotmail.com>
Signed-off-by: default avatarSong Liu <song@kernel.org>
Link: https://lore.kernel.org/r/20220706002612.4013790-1-song@kernel.org


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 4201d9ab

arch/x86/net/bpf_jit_comp.c

+25 −0
Original line number Diff line number Diff line
@@ -2486,3 +2486,28 @@ bool bpf_jit_supports_subprog_tailcalls(void) 
{

	return true;

}



void bpf_jit_free(struct bpf_prog *prog)

{

	if (prog->jited) {

		struct x64_jit_data *jit_data = prog->aux->jit_data;

		struct bpf_binary_header *hdr;



		/*

		 * If we fail the final pass of JIT (from jit_subprogs),

		 * the program may not be finalized yet. Call finalize here

		 * before freeing it.

		 */

		if (jit_data) {

			bpf_jit_binary_pack_finalize(prog, jit_data->header,

						     jit_data->rw_header);

			kvfree(jit_data->addrs);

			kfree(jit_data);

		}

		hdr = bpf_jit_binary_pack_hdr(prog);

		bpf_jit_binary_pack_free(hdr, NULL);

		WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));

	}



	bpf_prog_unlock_free(prog);

}

include/linux/bpf.h

+0 −1
Original line number Diff line number Diff line
@@ -1044,7 +1044,6 @@ struct bpf_prog_aux { 
	bool sleepable;

	bool tail_call_reachable;

	bool xdp_has_frags;

	bool use_bpf_prog_pack;

	/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */

	const struct btf_type *attach_func_proto;

	/* function name for valid attach_btf_id */

include/linux/filter.h

+8 −0
Original line number Diff line number Diff line
@@ -1027,6 +1027,14 @@ u64 bpf_jit_alloc_exec_limit(void); 
void *bpf_jit_alloc_exec(unsigned long size);

void bpf_jit_free_exec(void *addr);

void bpf_jit_free(struct bpf_prog *fp);

struct bpf_binary_header *

bpf_jit_binary_pack_hdr(const struct bpf_prog *fp);



static inline bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp)

{

	return list_empty(&fp->aux->ksym.lnode) ||

	       fp->aux->ksym.lnode.prev == LIST_POISON2;

}



struct bpf_binary_header *

bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **ro_image,

kernel/bpf/core.c

+12 −17
Original line number Diff line number Diff line
@@ -650,12 +650,6 @@ static bool bpf_prog_kallsyms_candidate(const struct bpf_prog *fp) 
	return fp->jited && !bpf_prog_was_classic(fp);

}



static bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp)

{

	return list_empty(&fp->aux->ksym.lnode) ||

	       fp->aux->ksym.lnode.prev == LIST_POISON2;

}



void bpf_prog_kallsyms_add(struct bpf_prog *fp)

{

	if (!bpf_prog_kallsyms_candidate(fp) ||
@@ -1153,7 +1147,6 @@ int bpf_jit_binary_pack_finalize(struct bpf_prog *prog, 
		bpf_prog_pack_free(ro_header);

		return PTR_ERR(ptr);

	}

	prog->aux->use_bpf_prog_pack = true;

	return 0;

}
@@ -1177,17 +1170,23 @@ void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header, 
	bpf_jit_uncharge_modmem(size);

}



struct bpf_binary_header *

bpf_jit_binary_pack_hdr(const struct bpf_prog *fp)

{

	unsigned long real_start = (unsigned long)fp->bpf_func;

	unsigned long addr;



	addr = real_start & BPF_PROG_CHUNK_MASK;

	return (void *)addr;

}



static inline struct bpf_binary_header *

bpf_jit_binary_hdr(const struct bpf_prog *fp)

{

	unsigned long real_start = (unsigned long)fp->bpf_func;

	unsigned long addr;



	if (fp->aux->use_bpf_prog_pack)

		addr = real_start & BPF_PROG_CHUNK_MASK;

	else

	addr = real_start & PAGE_MASK;



	return (void *)addr;

}
@@ -1200,11 +1199,7 @@ void __weak bpf_jit_free(struct bpf_prog *fp) 
	if (fp->jited) {

		struct bpf_binary_header *hdr = bpf_jit_binary_hdr(fp);



		if (fp->aux->use_bpf_prog_pack)

			bpf_jit_binary_pack_free(hdr, NULL /* rw_buffer */);

		else

		bpf_jit_binary_free(hdr);



		WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp));

	}