98 	struct htab_elem *__percpu *extra_elems;
111 struct htab_elem {  struct
118 				struct htab_elem *batch_flink;  argument
178 static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size,  in htab_elem_set_ptr()
184 static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size)  in htab_elem_get_ptr()
189 static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l)  in fd_htab_map_get_ptr()
194 static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i)  in get_htab_elem()
196 	return (struct htab_elem *) (htab->elems + i * (u64)htab->elem_size);  in get_htab_elem()
213 		struct htab_elem *elem;  in htab_free_prealloced_timers_and_wq()
236 		struct htab_elem *elem;  in htab_free_prealloced_fields()
285 static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key,  in prealloc_lru_pop()
289 	struct htab_elem *l;  in prealloc_lru_pop()
293 		l = container_of(node, struct htab_elem, lru_node);  in prealloc_lru_pop()
334 				   offsetof(struct htab_elem, hash) -  in prealloc_init()
335 				   offsetof(struct htab_elem, lru_node),  in prealloc_init()
346 				 offsetof(struct htab_elem, lru_node),  in prealloc_init()
350 				       htab->elems + offsetof(struct htab_elem, fnode),  in prealloc_init()
372 	struct htab_elem *__percpu *pptr, *l_new;  in alloc_extra_elems()
376 	pptr = bpf_map_alloc_percpu(&htab->map, sizeof(struct htab_elem *), 8,  in alloc_extra_elems()
386 		l_new = container_of(l, struct htab_elem, fnode);  in alloc_extra_elems()
410 	BUILD_BUG_ON(offsetof(struct htab_elem, fnode.next) !=  in htab_map_alloc_check()
411 		     offsetof(struct htab_elem, hash_node.pprev));  in htab_map_alloc_check()
438 	   sizeof(struct htab_elem))  in htab_map_alloc_check()
495 	htab->elem_size = sizeof(struct htab_elem) +  in htab_map_alloc()
607 static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash,  in lookup_elem_raw()
611 	struct htab_elem *l;  in lookup_elem_raw()
624 static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head,  in lookup_nulls_elem_raw()
629 	struct htab_elem *l;  in lookup_nulls_elem_raw()
651 	struct htab_elem *l;  in __htab_map_lookup_elem()
670 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_map_lookup_elem()
699 				offsetof(struct htab_elem, key) +  in htab_map_gen_lookup()
707 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in __htab_lru_map_lookup_elem()
740 			      offsetof(struct htab_elem, lru_node) +  in htab_lru_map_gen_lookup()
744 			     offsetof(struct htab_elem, lru_node) +  in htab_lru_map_gen_lookup()
748 				offsetof(struct htab_elem, key) +  in htab_lru_map_gen_lookup()
754 				  struct htab_elem *elem)  in check_and_free_fields()
778 	struct htab_elem *l = NULL, *tgt_l;  in htab_lru_map_delete_node()
785 	tgt_l = container_of(node, struct htab_elem, lru_node);  in htab_lru_map_delete_node()
812 	struct htab_elem *l, *next_l;  in htab_map_get_next_key()
835 				  struct htab_elem, hash_node);  in htab_map_get_next_key()
854 					  struct htab_elem, hash_node);  in htab_map_get_next_key()
866 static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l)  in htab_elem_free()
875 static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l)  in htab_put_fd_value()
915 static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)  in free_htab_elem()
975 static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,  in alloc_htab_elem()
978 					 struct htab_elem *old_elem)  in alloc_htab_elem()
982 	struct htab_elem *l_new, **pl_new;  in alloc_htab_elem()
999 			l_new = container_of(l, struct htab_elem, fnode);  in alloc_htab_elem()
1056 static int check_flags(struct bpf_htab *htab, struct htab_elem *l_old,  in check_flags()
1075 	struct htab_elem *l_new = NULL, *l_old;  in htab_map_update_elem()
1183 static void htab_lru_push_free(struct bpf_htab *htab, struct htab_elem *elem)  in htab_lru_push_free()
1194 	struct htab_elem *l_new, *l_old = NULL;  in htab_lru_map_update_elem()
1263 	struct htab_elem *l_new = NULL, *l_old;  in __htab_percpu_map_update_elem()
1318 	struct htab_elem *l_new = NULL, *l_old;  in __htab_lru_percpu_map_update_elem()
1402 	struct htab_elem *l;  in htab_map_delete_elem()
1438 	struct htab_elem *l;  in htab_lru_map_delete_elem()
1479 		struct htab_elem *l;  in delete_all_elements()
1497 		struct htab_elem *l;  in htab_free_malloced_timers_and_wq()
1585 	struct htab_elem *l;  in __htab_map_lookup_and_delete_elem()
1689 	struct htab_elem *node_to_free = NULL;  in __htab_map_lookup_and_delete_batch()
1695 	struct htab_elem *l;  in __htab_map_lookup_and_delete_batch()
1984 static struct htab_elem *
1986 			   struct htab_elem *prev_elem)  in bpf_hash_map_seq_find_next()
1993 	struct htab_elem *elem;  in bpf_hash_map_seq_find_next()
2006 		elem = hlist_nulls_entry_safe(n, struct htab_elem, hash_node);  in bpf_hash_map_seq_find_next()
2043 	struct htab_elem *elem;  in bpf_hash_map_seq_start()
2063 static int __bpf_hash_map_seq_show(struct seq_file *seq, struct htab_elem *elem)  in __bpf_hash_map_seq_show()
2167 	struct htab_elem *elem;  in bpf_for_each_hash_elem()
2238 			usage += sizeof(struct htab_elem *) * num_possible_cpus();  in htab_map_mem_usage()
2301 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_percpu_map_lookup_elem()
2322 				offsetof(struct htab_elem, key) + roundup(map->key_size, 8));  in htab_percpu_map_gen_lookup()
2331 	struct htab_elem *l;  in htab_percpu_map_lookup_percpu_elem()
2345 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_lru_percpu_map_lookup_elem()
2357 	struct htab_elem *l;  in htab_lru_percpu_map_lookup_percpu_elem()
2373 	struct htab_elem *l;  in bpf_percpu_hash_copy()
2424 	struct htab_elem *l;  in htab_percpu_map_seq_show_elem()
2503 	struct htab_elem *l;  in fd_htab_map_free()
2604 				offsetof(struct htab_elem, key) +  in htab_of_map_gen_lookup()