Commit 65334e64 authored by Kumar Kartikeya Dwivedi's avatar Kumar Kartikeya Dwivedi Committed by Alexei Starovoitov
Browse files

bpf: Support kptrs in percpu hashmap and percpu LRU hashmap 



Enable support for kptrs in percpu BPF hashmap and percpu BPF LRU
hashmap by wiring up the freeing of these kptrs from percpu map
elements.

Signed-off-by: default avatarKumar Kartikeya Dwivedi <memxor@gmail.com>
Link: https://lore.kernel.org/r/20230225154010.391965-2-memxor@gmail.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent c4b5c5ba

kernel/bpf/hashtab.c

+37 −22
Original line number Diff line number Diff line
@@ -249,9 +249,20 @@ static void htab_free_prealloced_fields(struct bpf_htab *htab) 
		struct htab_elem *elem;



		elem = get_htab_elem(htab, i);

		if (htab_is_percpu(htab)) {

			void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size);

			int cpu;



			for_each_possible_cpu(cpu) {

				bpf_obj_free_fields(htab->map.record, per_cpu_ptr(pptr, cpu));

				cond_resched();

			}

		} else {

			bpf_obj_free_fields(htab->map.record, elem->key + round_up(htab->map.key_size, 8));

			cond_resched();

		}

		cond_resched();

	}

}



static void htab_free_elems(struct bpf_htab *htab)
@@ -759,10 +770,18 @@ static int htab_lru_map_gen_lookup(struct bpf_map *map, 
static void check_and_free_fields(struct bpf_htab *htab,

				  struct htab_elem *elem)

{

	if (htab_is_percpu(htab)) {

		void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size);

		int cpu;



		for_each_possible_cpu(cpu)

			bpf_obj_free_fields(htab->map.record, per_cpu_ptr(pptr, cpu));

	} else {

		void *map_value = elem->key + round_up(htab->map.key_size, 8);



		bpf_obj_free_fields(htab->map.record, map_value);

	}

}



/* It is called from the bpf_lru_list when the LRU needs to delete

 * older elements from the htab.
@@ -858,9 +877,9 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key) 


static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l)

{

	check_and_free_fields(htab, l);

	if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH)

		bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr);

	check_and_free_fields(htab, l);

	bpf_mem_cache_free(&htab->ma, l);

}
@@ -918,14 +937,13 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, 
{

	if (!onallcpus) {

		/* copy true value_size bytes */

		memcpy(this_cpu_ptr(pptr), value, htab->map.value_size);

		copy_map_value(&htab->map, this_cpu_ptr(pptr), value);

	} else {

		u32 size = round_up(htab->map.value_size, 8);

		int off = 0, cpu;



		for_each_possible_cpu(cpu) {

			bpf_long_memcpy(per_cpu_ptr(pptr, cpu),

					value + off, size);

			copy_map_value_long(&htab->map, per_cpu_ptr(pptr, cpu), value + off);

			off += size;

		}

	}
@@ -940,16 +958,14 @@ static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr, 
	 * (onallcpus=false always when coming from bpf prog).

	 */

	if (!onallcpus) {

		u32 size = round_up(htab->map.value_size, 8);

		int current_cpu = raw_smp_processor_id();

		int cpu;



		for_each_possible_cpu(cpu) {

			if (cpu == current_cpu)

				bpf_long_memcpy(per_cpu_ptr(pptr, cpu), value,

						size);

			else

				memset(per_cpu_ptr(pptr, cpu), 0, size);

				copy_map_value_long(&htab->map, per_cpu_ptr(pptr, cpu), value);

			else /* Since elem is preallocated, we cannot touch special fields */

				zero_map_value(&htab->map, per_cpu_ptr(pptr, cpu));

		}

	} else {

		pcpu_copy_value(htab, pptr, value, onallcpus);
@@ -1575,9 +1591,8 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, 


			pptr = htab_elem_get_ptr(l, key_size);

			for_each_possible_cpu(cpu) {

				bpf_long_memcpy(value + off,

						per_cpu_ptr(pptr, cpu),

						roundup_value_size);

				copy_map_value_long(&htab->map, value + off, per_cpu_ptr(pptr, cpu));

				check_and_init_map_value(&htab->map, value + off);

				off += roundup_value_size;

			}

		} else {
@@ -1772,8 +1787,8 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, 


			pptr = htab_elem_get_ptr(l, map->key_size);

			for_each_possible_cpu(cpu) {

				bpf_long_memcpy(dst_val + off,

						per_cpu_ptr(pptr, cpu), size);

				copy_map_value_long(&htab->map, dst_val + off, per_cpu_ptr(pptr, cpu));

				check_and_init_map_value(&htab->map, dst_val + off);

				off += size;

			}

		} else {
@@ -2046,9 +2061,9 @@ static int __bpf_hash_map_seq_show(struct seq_file *seq, struct htab_elem *elem) 
				roundup_value_size = round_up(map->value_size, 8);

				pptr = htab_elem_get_ptr(elem, map->key_size);

				for_each_possible_cpu(cpu) {

					bpf_long_memcpy(info->percpu_value_buf + off,

							per_cpu_ptr(pptr, cpu),

							roundup_value_size);

					copy_map_value_long(map, info->percpu_value_buf + off,

							    per_cpu_ptr(pptr, cpu));

					check_and_init_map_value(map, info->percpu_value_buf + off);

					off += roundup_value_size;

				}

				ctx.value = info->percpu_value_buf;
@@ -2292,8 +2307,8 @@ int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value) 
	 */

	pptr = htab_elem_get_ptr(l, map->key_size);

	for_each_possible_cpu(cpu) {

		bpf_long_memcpy(value + off,

				per_cpu_ptr(pptr, cpu), size);

		copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu));

		check_and_init_map_value(map, value + off);

		off += size;

	}

	ret = 0;

kernel/bpf/syscall.c

+2 −0
Original line number Diff line number Diff line
@@ -1059,7 +1059,9 @@ static int map_check_btf(struct bpf_map *map, const struct btf *btf, 
			case BPF_KPTR_UNREF:

			case BPF_KPTR_REF:

				if (map->map_type != BPF_MAP_TYPE_HASH &&

				    map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&

				    map->map_type != BPF_MAP_TYPE_LRU_HASH &&

				    map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH &&

				    map->map_type != BPF_MAP_TYPE_ARRAY &&

				    map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) {

					ret = -EOPNOTSUPP;