Commit 07343110 authored by Feng Zhou's avatar Feng Zhou Committed by Alexei Starovoitov
Browse files

bpf: add bpf_map_lookup_percpu_elem for percpu map 



Add new ebpf helpers bpf_map_lookup_percpu_elem.

The implementation method is relatively simple, refer to the implementation
method of map_lookup_elem of percpu map, increase the parameters of cpu, and
obtain it according to the specified cpu.

Signed-off-by: default avatarFeng Zhou <zhoufeng.zf@bytedance.com>
Link: https://lore.kernel.org/r/20220511093854.411-2-zhoufeng.zf@bytedance.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 571b8739

include/linux/bpf.h

+2 −0
Original line number Diff line number Diff line
@@ -89,6 +89,7 @@ struct bpf_map_ops { 
	int (*map_push_elem)(struct bpf_map *map, void *value, u64 flags);

	int (*map_pop_elem)(struct bpf_map *map, void *value);

	int (*map_peek_elem)(struct bpf_map *map, void *value);

	void *(*map_lookup_percpu_elem)(struct bpf_map *map, void *key, u32 cpu);



	/* funcs called by prog_array and perf_event_array map */

	void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,
@@ -2184,6 +2185,7 @@ extern const struct bpf_func_proto bpf_map_delete_elem_proto; 
extern const struct bpf_func_proto bpf_map_push_elem_proto;

extern const struct bpf_func_proto bpf_map_pop_elem_proto;

extern const struct bpf_func_proto bpf_map_peek_elem_proto;

extern const struct bpf_func_proto bpf_map_lookup_percpu_elem_proto;



extern const struct bpf_func_proto bpf_get_prandom_u32_proto;

extern const struct bpf_func_proto bpf_get_smp_processor_id_proto;

include/uapi/linux/bpf.h

+9 −0
Original line number Diff line number Diff line
@@ -5164,6 +5164,14 @@ union bpf_attr { 
 *		if not NULL, is a reference which must be released using its

 *		corresponding release function, or moved into a BPF map before

 *		program exit.

 *

 * void *bpf_map_lookup_percpu_elem(struct bpf_map *map, const void *key, u32 cpu)

 * 	Description

 * 		Perform a lookup in *percpu map* for an entry associated to

 * 		*key* on *cpu*.

 * 	Return

 * 		Map value associated to *key* on *cpu*, or **NULL** if no entry

 * 		was found or *cpu* is invalid.

 */

#define __BPF_FUNC_MAPPER(FN)		\

	FN(unspec),			\
@@ -5361,6 +5369,7 @@ union bpf_attr { 
	FN(skb_set_tstamp),		\

	FN(ima_file_hash),		\

	FN(kptr_xchg),			\

	FN(map_lookup_percpu_elem),     \

	/* */



/* integer value in 'imm' field of BPF_CALL instruction selects which helper

kernel/bpf/arraymap.c

+15 −0
Original line number Diff line number Diff line
@@ -243,6 +243,20 @@ static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key) 
	return this_cpu_ptr(array->pptrs[index & array->index_mask]);

}



static void *percpu_array_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu)

{

	struct bpf_array *array = container_of(map, struct bpf_array, map);

	u32 index = *(u32 *)key;



	if (cpu >= nr_cpu_ids)

		return NULL;



	if (unlikely(index >= array->map.max_entries))

		return NULL;



	return per_cpu_ptr(array->pptrs[index & array->index_mask], cpu);

}



int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value)

{

	struct bpf_array *array = container_of(map, struct bpf_array, map);
@@ -725,6 +739,7 @@ const struct bpf_map_ops percpu_array_map_ops = { 
	.map_lookup_elem = percpu_array_map_lookup_elem,

	.map_update_elem = array_map_update_elem,

	.map_delete_elem = array_map_delete_elem,

	.map_lookup_percpu_elem = percpu_array_map_lookup_percpu_elem,

	.map_seq_show_elem = percpu_array_map_seq_show_elem,

	.map_check_btf = array_map_check_btf,

	.map_lookup_batch = generic_map_lookup_batch,

kernel/bpf/core.c

+1 −0
Original line number Diff line number Diff line
@@ -2619,6 +2619,7 @@ const struct bpf_func_proto bpf_map_delete_elem_proto __weak; 
const struct bpf_func_proto bpf_map_push_elem_proto __weak;

const struct bpf_func_proto bpf_map_pop_elem_proto __weak;

const struct bpf_func_proto bpf_map_peek_elem_proto __weak;

const struct bpf_func_proto bpf_map_lookup_percpu_elem_proto __weak;

const struct bpf_func_proto bpf_spin_lock_proto __weak;

const struct bpf_func_proto bpf_spin_unlock_proto __weak;

const struct bpf_func_proto bpf_jiffies64_proto __weak;

kernel/bpf/hashtab.c

+32 −0
Original line number Diff line number Diff line
@@ -2199,6 +2199,20 @@ static void *htab_percpu_map_lookup_elem(struct bpf_map *map, void *key) 
		return NULL;

}



static void *htab_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu)

{

	struct htab_elem *l;



	if (cpu >= nr_cpu_ids)

		return NULL;



	l = __htab_map_lookup_elem(map, key);

	if (l)

		return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu);

	else

		return NULL;

}



static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key)

{

	struct htab_elem *l = __htab_map_lookup_elem(map, key);
@@ -2211,6 +2225,22 @@ static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key) 
	return NULL;

}



static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu)

{

	struct htab_elem *l;



	if (cpu >= nr_cpu_ids)

		return NULL;



	l = __htab_map_lookup_elem(map, key);

	if (l) {

		bpf_lru_node_set_ref(&l->lru_node);

		return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu);

	}



	return NULL;

}



int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value)

{

	struct htab_elem *l;
@@ -2300,6 +2330,7 @@ const struct bpf_map_ops htab_percpu_map_ops = { 
	.map_lookup_and_delete_elem = htab_percpu_map_lookup_and_delete_elem,

	.map_update_elem = htab_percpu_map_update_elem,

	.map_delete_elem = htab_map_delete_elem,

	.map_lookup_percpu_elem = htab_percpu_map_lookup_percpu_elem,

	.map_seq_show_elem = htab_percpu_map_seq_show_elem,

	.map_set_for_each_callback_args = map_set_for_each_callback_args,

	.map_for_each_callback = bpf_for_each_hash_elem,
@@ -2318,6 +2349,7 @@ const struct bpf_map_ops htab_lru_percpu_map_ops = { 
	.map_lookup_and_delete_elem = htab_lru_percpu_map_lookup_and_delete_elem,

	.map_update_elem = htab_lru_percpu_map_update_elem,

	.map_delete_elem = htab_lru_map_delete_elem,

	.map_lookup_percpu_elem = htab_lru_percpu_map_lookup_percpu_elem,

	.map_seq_show_elem = htab_percpu_map_seq_show_elem,

	.map_set_for_each_callback_args = map_set_for_each_callback_args,

	.map_for_each_callback = bpf_for_each_hash_elem,