Commit 3ac1f01b authored by Roman Gushchin's avatar Roman Gushchin Committed by Alexei Starovoitov
Browse files

bpf: Eliminate rlimit-based memory accounting for bpf progs 



Do not use rlimit-based memory accounting for bpf progs. It has been
replaced with memcg-based memory accounting.

Signed-off-by: default avatarRoman Gushchin <guro@fb.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Acked-by: default avatarSong Liu <songliubraving@fb.com>
Link: https://lore.kernel.org/bpf/20201201215900.3569844-34-guro@fb.com
parent 80ee81e0

include/linux/bpf.h

+0 −11
Original line number Diff line number Diff line
@@ -1202,8 +1202,6 @@ void bpf_prog_sub(struct bpf_prog *prog, int i); 
void bpf_prog_inc(struct bpf_prog *prog);

struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog);

void bpf_prog_put(struct bpf_prog *prog);

int __bpf_prog_charge(struct user_struct *user, u32 pages);

void __bpf_prog_uncharge(struct user_struct *user, u32 pages);

void __bpf_free_used_maps(struct bpf_prog_aux *aux,

			  struct bpf_map **used_maps, u32 len);
@@ -1512,15 +1510,6 @@ bpf_prog_inc_not_zero(struct bpf_prog *prog) 
	return ERR_PTR(-EOPNOTSUPP);

}



static inline int __bpf_prog_charge(struct user_struct *user, u32 pages)

{

	return 0;

}



static inline void __bpf_prog_uncharge(struct user_struct *user, u32 pages)

{

}



static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,

				 const struct bpf_link_ops *ops,

				 struct bpf_prog *prog)

kernel/bpf/core.c

+2 −10
Original line number Diff line number Diff line
@@ -221,23 +221,15 @@ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, 
{

	gfp_t gfp_flags = GFP_KERNEL_ACCOUNT | __GFP_ZERO | gfp_extra_flags;

	struct bpf_prog *fp;

	u32 pages, delta;

	int ret;

	u32 pages;



	size = round_up(size, PAGE_SIZE);

	pages = size / PAGE_SIZE;

	if (pages <= fp_old->pages)

		return fp_old;



	delta = pages - fp_old->pages;

	ret = __bpf_prog_charge(fp_old->aux->user, delta);

	if (ret)

		return NULL;



	fp = __vmalloc(size, gfp_flags);

	if (fp == NULL) {

		__bpf_prog_uncharge(fp_old->aux->user, delta);

	} else {

	if (fp) {

		memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE);

		fp->pages = pages;

		fp->aux->prog = fp;

kernel/bpf/syscall.c

+10 −59
Original line number Diff line number Diff line
@@ -1632,51 +1632,6 @@ static void bpf_audit_prog(const struct bpf_prog *prog, unsigned int op) 
	audit_log_end(ab);

}



int __bpf_prog_charge(struct user_struct *user, u32 pages)

{

	unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;

	unsigned long user_bufs;



	if (user) {

		user_bufs = atomic_long_add_return(pages, &user->locked_vm);

		if (user_bufs > memlock_limit) {

			atomic_long_sub(pages, &user->locked_vm);

			return -EPERM;

		}

	}



	return 0;

}



void __bpf_prog_uncharge(struct user_struct *user, u32 pages)

{

	if (user)

		atomic_long_sub(pages, &user->locked_vm);

}



static int bpf_prog_charge_memlock(struct bpf_prog *prog)

{

	struct user_struct *user = get_current_user();

	int ret;



	ret = __bpf_prog_charge(user, prog->pages);

	if (ret) {

		free_uid(user);

		return ret;

	}



	prog->aux->user = user;

	return 0;

}



static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)

{

	struct user_struct *user = prog->aux->user;



	__bpf_prog_uncharge(user, prog->pages);

	free_uid(user);

}



static int bpf_prog_alloc_id(struct bpf_prog *prog)

{

	int id;
@@ -1726,7 +1681,7 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu) 


	kvfree(aux->func_info);

	kfree(aux->func_info_aux);

	bpf_prog_uncharge_memlock(aux->prog);

	free_uid(aux->user);

	security_bpf_prog_free(aux);

	bpf_prog_free(aux->prog);

}
@@ -2164,7 +2119,7 @@ static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) 
		dst_prog = bpf_prog_get(attr->attach_prog_fd);

		if (IS_ERR(dst_prog)) {

			err = PTR_ERR(dst_prog);

			goto free_prog_nouncharge;

			goto free_prog;

		}

		prog->aux->dst_prog = dst_prog;

	}
@@ -2174,18 +2129,15 @@ static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) 


	err = security_bpf_prog_alloc(prog->aux);

	if (err)

		goto free_prog_nouncharge;



	err = bpf_prog_charge_memlock(prog);

	if (err)

		goto free_prog_sec;

		goto free_prog;



	prog->aux->user = get_current_user();

	prog->len = attr->insn_cnt;



	err = -EFAULT;

	if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns),

			   bpf_prog_insn_size(prog)) != 0)

		goto free_prog;

		goto free_prog_sec;



	prog->orig_prog = NULL;

	prog->jited = 0;
@@ -2196,19 +2148,19 @@ static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) 
	if (bpf_prog_is_dev_bound(prog->aux)) {

		err = bpf_prog_offload_init(prog, attr);

		if (err)

			goto free_prog;

			goto free_prog_sec;

	}



	/* find program type: socket_filter vs tracing_filter */

	err = find_prog_type(type, prog);

	if (err < 0)

		goto free_prog;

		goto free_prog_sec;



	prog->aux->load_time = ktime_get_boottime_ns();

	err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name,

			       sizeof(attr->prog_name));

	if (err < 0)

		goto free_prog;

		goto free_prog_sec;



	/* run eBPF verifier */

	err = bpf_check(&prog, attr, uattr);
@@ -2253,11 +2205,10 @@ static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) 
	 */

	__bpf_prog_put_noref(prog, prog->aux->func_cnt);

	return err;

free_prog:

	bpf_prog_uncharge_memlock(prog);

free_prog_sec:

	free_uid(prog->aux->user);

	security_bpf_prog_free(prog->aux);

free_prog_nouncharge:

free_prog:

	bpf_prog_free(prog);

	return err;

}