Lines Matching refs:htab
131 static inline bool htab_is_prealloc(const struct bpf_htab *htab) in htab_is_prealloc() argument
133 return !(htab->map.map_flags & BPF_F_NO_PREALLOC); in htab_is_prealloc()
136 static void htab_init_buckets(struct bpf_htab *htab) in htab_init_buckets() argument
140 for (i = 0; i < htab->n_buckets; i++) { in htab_init_buckets()
141 INIT_HLIST_NULLS_HEAD(&htab->buckets[i].head, i); in htab_init_buckets()
142 raw_res_spin_lock_init(&htab->buckets[i].raw_lock); in htab_init_buckets()
166 static bool htab_is_lru(const struct bpf_htab *htab) in htab_is_lru() argument
168 return htab->map.map_type == BPF_MAP_TYPE_LRU_HASH || in htab_is_lru()
169 htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; in htab_is_lru()
172 static bool htab_is_percpu(const struct bpf_htab *htab) in htab_is_percpu() argument
174 return htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH || in htab_is_percpu()
175 htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; in htab_is_percpu()
194 static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i) in get_htab_elem() argument
196 return (struct htab_elem *) (htab->elems + i * (u64)htab->elem_size); in get_htab_elem()
199 static bool htab_has_extra_elems(struct bpf_htab *htab) in htab_has_extra_elems() argument
201 return !htab_is_percpu(htab) && !htab_is_lru(htab); in htab_has_extra_elems()
204 static void htab_free_prealloced_timers_and_wq(struct bpf_htab *htab) in htab_free_prealloced_timers_and_wq() argument
206 u32 num_entries = htab->map.max_entries; in htab_free_prealloced_timers_and_wq()
209 if (htab_has_extra_elems(htab)) in htab_free_prealloced_timers_and_wq()
215 elem = get_htab_elem(htab, i); in htab_free_prealloced_timers_and_wq()
216 if (btf_record_has_field(htab->map.record, BPF_TIMER)) in htab_free_prealloced_timers_and_wq()
217 bpf_obj_free_timer(htab->map.record, in htab_free_prealloced_timers_and_wq()
218 elem->key + round_up(htab->map.key_size, 8)); in htab_free_prealloced_timers_and_wq()
219 if (btf_record_has_field(htab->map.record, BPF_WORKQUEUE)) in htab_free_prealloced_timers_and_wq()
220 bpf_obj_free_workqueue(htab->map.record, in htab_free_prealloced_timers_and_wq()
221 elem->key + round_up(htab->map.key_size, 8)); in htab_free_prealloced_timers_and_wq()
226 static void htab_free_prealloced_fields(struct bpf_htab *htab) in htab_free_prealloced_fields() argument
228 u32 num_entries = htab->map.max_entries; in htab_free_prealloced_fields()
231 if (IS_ERR_OR_NULL(htab->map.record)) in htab_free_prealloced_fields()
233 if (htab_has_extra_elems(htab)) in htab_free_prealloced_fields()
238 elem = get_htab_elem(htab, i); in htab_free_prealloced_fields()
239 if (htab_is_percpu(htab)) { in htab_free_prealloced_fields()
240 void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size); in htab_free_prealloced_fields()
244 bpf_obj_free_fields(htab->map.record, per_cpu_ptr(pptr, cpu)); in htab_free_prealloced_fields()
248 bpf_obj_free_fields(htab->map.record, elem->key + round_up(htab->map.key_size, 8)); in htab_free_prealloced_fields()
255 static void htab_free_elems(struct bpf_htab *htab) in htab_free_elems() argument
259 if (!htab_is_percpu(htab)) in htab_free_elems()
262 for (i = 0; i < htab->map.max_entries; i++) { in htab_free_elems()
265 pptr = htab_elem_get_ptr(get_htab_elem(htab, i), in htab_free_elems()
266 htab->map.key_size); in htab_free_elems()
271 bpf_map_area_free(htab->elems); in htab_free_elems()
285 static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key, in prealloc_lru_pop() argument
288 struct bpf_lru_node *node = bpf_lru_pop_free(&htab->lru, hash); in prealloc_lru_pop()
292 bpf_map_inc_elem_count(&htab->map); in prealloc_lru_pop()
294 memcpy(l->key, key, htab->map.key_size); in prealloc_lru_pop()
301 static int prealloc_init(struct bpf_htab *htab) in prealloc_init() argument
303 u32 num_entries = htab->map.max_entries; in prealloc_init()
306 if (htab_has_extra_elems(htab)) in prealloc_init()
309 htab->elems = bpf_map_area_alloc((u64)htab->elem_size * num_entries, in prealloc_init()
310 htab->map.numa_node); in prealloc_init()
311 if (!htab->elems) in prealloc_init()
314 if (!htab_is_percpu(htab)) in prealloc_init()
318 u32 size = round_up(htab->map.value_size, 8); in prealloc_init()
321 pptr = bpf_map_alloc_percpu(&htab->map, size, 8, in prealloc_init()
325 htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size, in prealloc_init()
331 if (htab_is_lru(htab)) in prealloc_init()
332 err = bpf_lru_init(&htab->lru, in prealloc_init()
333 htab->map.map_flags & BPF_F_NO_COMMON_LRU, in prealloc_init()
337 htab); in prealloc_init()
339 err = pcpu_freelist_init(&htab->freelist); in prealloc_init()
344 if (htab_is_lru(htab)) in prealloc_init()
345 bpf_lru_populate(&htab->lru, htab->elems, in prealloc_init()
347 htab->elem_size, num_entries); in prealloc_init()
349 pcpu_freelist_populate(&htab->freelist, in prealloc_init()
350 htab->elems + offsetof(struct htab_elem, fnode), in prealloc_init()
351 htab->elem_size, num_entries); in prealloc_init()
356 htab_free_elems(htab); in prealloc_init()
360 static void prealloc_destroy(struct bpf_htab *htab) in prealloc_destroy() argument
362 htab_free_elems(htab); in prealloc_destroy()
364 if (htab_is_lru(htab)) in prealloc_destroy()
365 bpf_lru_destroy(&htab->lru); in prealloc_destroy()
367 pcpu_freelist_destroy(&htab->freelist); in prealloc_destroy()
370 static int alloc_extra_elems(struct bpf_htab *htab) in alloc_extra_elems() argument
376 pptr = bpf_map_alloc_percpu(&htab->map, sizeof(struct htab_elem *), 8, in alloc_extra_elems()
382 l = pcpu_freelist_pop(&htab->freelist); in alloc_extra_elems()
389 htab->extra_elems = pptr; in alloc_extra_elems()
465 struct bpf_htab *htab; in htab_map_alloc() local
468 htab = bpf_map_area_alloc(sizeof(*htab), NUMA_NO_NODE); in htab_map_alloc()
469 if (!htab) in htab_map_alloc()
472 bpf_map_init_from_attr(&htab->map, attr); in htab_map_alloc()
479 htab->map.max_entries = roundup(attr->max_entries, in htab_map_alloc()
481 if (htab->map.max_entries < attr->max_entries) in htab_map_alloc()
482 htab->map.max_entries = rounddown(attr->max_entries, in htab_map_alloc()
490 if (htab->map.max_entries > 1UL << 31) in htab_map_alloc()
493 htab->n_buckets = roundup_pow_of_two(htab->map.max_entries); in htab_map_alloc()
495 htab->elem_size = sizeof(struct htab_elem) + in htab_map_alloc()
496 round_up(htab->map.key_size, 8); in htab_map_alloc()
498 htab->elem_size += sizeof(void *); in htab_map_alloc()
500 htab->elem_size += round_up(htab->map.value_size, 8); in htab_map_alloc()
503 if (htab->n_buckets > U32_MAX / sizeof(struct bucket)) in htab_map_alloc()
506 err = bpf_map_init_elem_count(&htab->map); in htab_map_alloc()
511 htab->buckets = bpf_map_area_alloc(htab->n_buckets * in htab_map_alloc()
513 htab->map.numa_node); in htab_map_alloc()
514 if (!htab->buckets) in htab_map_alloc()
517 if (htab->map.map_flags & BPF_F_ZERO_SEED) in htab_map_alloc()
518 htab->hashrnd = 0; in htab_map_alloc()
520 htab->hashrnd = get_random_u32(); in htab_map_alloc()
522 htab_init_buckets(htab); in htab_map_alloc()
539 htab->use_percpu_counter = true; in htab_map_alloc()
541 if (htab->use_percpu_counter) { in htab_map_alloc()
542 err = percpu_counter_init(&htab->pcount, 0, GFP_KERNEL); in htab_map_alloc()
548 err = prealloc_init(htab); in htab_map_alloc()
556 err = alloc_extra_elems(htab); in htab_map_alloc()
561 err = bpf_mem_alloc_init(&htab->ma, htab->elem_size, false); in htab_map_alloc()
565 err = bpf_mem_alloc_init(&htab->pcpu_ma, in htab_map_alloc()
566 round_up(htab->map.value_size, 8), true); in htab_map_alloc()
572 return &htab->map; in htab_map_alloc()
575 prealloc_destroy(htab); in htab_map_alloc()
577 if (htab->use_percpu_counter) in htab_map_alloc()
578 percpu_counter_destroy(&htab->pcount); in htab_map_alloc()
579 bpf_map_area_free(htab->buckets); in htab_map_alloc()
580 bpf_mem_alloc_destroy(&htab->pcpu_ma); in htab_map_alloc()
581 bpf_mem_alloc_destroy(&htab->ma); in htab_map_alloc()
583 bpf_map_free_elem_count(&htab->map); in htab_map_alloc()
585 bpf_map_area_free(htab); in htab_map_alloc()
596 static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash) in __select_bucket() argument
598 return &htab->buckets[hash & (htab->n_buckets - 1)]; in __select_bucket()
601 static inline struct hlist_nulls_head *select_bucket(struct bpf_htab *htab, u32 hash) in select_bucket() argument
603 return &__select_bucket(htab, hash)->head; in select_bucket()
649 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_elem() local
659 hash = htab_map_hash(key, key_size, htab->hashrnd); in __htab_map_lookup_elem()
661 head = select_bucket(htab, hash); in __htab_map_lookup_elem()
663 l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); in __htab_map_lookup_elem()
753 static void check_and_free_fields(struct bpf_htab *htab, in check_and_free_fields() argument
756 if (IS_ERR_OR_NULL(htab->map.record)) in check_and_free_fields()
759 if (htab_is_percpu(htab)) { in check_and_free_fields()
760 void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size); in check_and_free_fields()
764 bpf_obj_free_fields(htab->map.record, per_cpu_ptr(pptr, cpu)); in check_and_free_fields()
766 void *map_value = elem->key + round_up(htab->map.key_size, 8); in check_and_free_fields()
768 bpf_obj_free_fields(htab->map.record, map_value); in check_and_free_fields()
777 struct bpf_htab *htab = arg; in htab_lru_map_delete_node() local
786 b = __select_bucket(htab, tgt_l->hash); in htab_lru_map_delete_node()
796 bpf_map_dec_elem_count(&htab->map); in htab_lru_map_delete_node()
803 check_and_free_fields(htab, l); in htab_lru_map_delete_node()
810 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_get_next_key() local
823 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_map_get_next_key()
825 head = select_bucket(htab, hash); in htab_map_get_next_key()
828 l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); in htab_map_get_next_key()
844 i = hash & (htab->n_buckets - 1); in htab_map_get_next_key()
849 for (; i < htab->n_buckets; i++) { in htab_map_get_next_key()
850 head = select_bucket(htab, i); in htab_map_get_next_key()
866 static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l) in htab_elem_free() argument
868 check_and_free_fields(htab, l); in htab_elem_free()
870 if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) in htab_elem_free()
871 bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr); in htab_elem_free()
872 bpf_mem_cache_free(&htab->ma, l); in htab_elem_free()
875 static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l) in htab_put_fd_value() argument
877 struct bpf_map *map = &htab->map; in htab_put_fd_value()
886 static bool is_map_full(struct bpf_htab *htab) in is_map_full() argument
888 if (htab->use_percpu_counter) in is_map_full()
889 return __percpu_counter_compare(&htab->pcount, htab->map.max_entries, in is_map_full()
891 return atomic_read(&htab->count) >= htab->map.max_entries; in is_map_full()
894 static void inc_elem_count(struct bpf_htab *htab) in inc_elem_count() argument
896 bpf_map_inc_elem_count(&htab->map); in inc_elem_count()
898 if (htab->use_percpu_counter) in inc_elem_count()
899 percpu_counter_add_batch(&htab->pcount, 1, PERCPU_COUNTER_BATCH); in inc_elem_count()
901 atomic_inc(&htab->count); in inc_elem_count()
904 static void dec_elem_count(struct bpf_htab *htab) in dec_elem_count() argument
906 bpf_map_dec_elem_count(&htab->map); in dec_elem_count()
908 if (htab->use_percpu_counter) in dec_elem_count()
909 percpu_counter_add_batch(&htab->pcount, -1, PERCPU_COUNTER_BATCH); in dec_elem_count()
911 atomic_dec(&htab->count); in dec_elem_count()
915 static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) in free_htab_elem() argument
917 htab_put_fd_value(htab, l); in free_htab_elem()
919 if (htab_is_prealloc(htab)) { in free_htab_elem()
920 bpf_map_dec_elem_count(&htab->map); in free_htab_elem()
921 check_and_free_fields(htab, l); in free_htab_elem()
922 pcpu_freelist_push(&htab->freelist, &l->fnode); in free_htab_elem()
924 dec_elem_count(htab); in free_htab_elem()
925 htab_elem_free(htab, l); in free_htab_elem()
929 static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, in pcpu_copy_value() argument
934 copy_map_value(&htab->map, this_cpu_ptr(pptr), value); in pcpu_copy_value()
936 u32 size = round_up(htab->map.value_size, 8); in pcpu_copy_value()
940 copy_map_value_long(&htab->map, per_cpu_ptr(pptr, cpu), value + off); in pcpu_copy_value()
946 static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr, in pcpu_init_value() argument
960 copy_map_value_long(&htab->map, per_cpu_ptr(pptr, cpu), value); in pcpu_init_value()
962 zero_map_value(&htab->map, per_cpu_ptr(pptr, cpu)); in pcpu_init_value()
965 pcpu_copy_value(htab, pptr, value, onallcpus); in pcpu_init_value()
969 static bool fd_htab_map_needs_adjust(const struct bpf_htab *htab) in fd_htab_map_needs_adjust() argument
971 return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS && in fd_htab_map_needs_adjust()
975 static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, in alloc_htab_elem() argument
980 u32 size = htab->map.value_size; in alloc_htab_elem()
981 bool prealloc = htab_is_prealloc(htab); in alloc_htab_elem()
990 pl_new = this_cpu_ptr(htab->extra_elems); in alloc_htab_elem()
996 l = __pcpu_freelist_pop(&htab->freelist); in alloc_htab_elem()
1000 bpf_map_inc_elem_count(&htab->map); in alloc_htab_elem()
1003 if (is_map_full(htab)) in alloc_htab_elem()
1011 inc_elem_count(htab); in alloc_htab_elem()
1012 l_new = bpf_mem_cache_alloc(&htab->ma); in alloc_htab_elem()
1025 void *ptr = bpf_mem_cache_alloc(&htab->pcpu_ma); in alloc_htab_elem()
1028 bpf_mem_cache_free(&htab->ma, l_new); in alloc_htab_elem()
1036 pcpu_init_value(htab, pptr, value, onallcpus); in alloc_htab_elem()
1040 } else if (fd_htab_map_needs_adjust(htab)) { in alloc_htab_elem()
1044 copy_map_value(&htab->map, in alloc_htab_elem()
1052 dec_elem_count(htab); in alloc_htab_elem()
1056 static int check_flags(struct bpf_htab *htab, struct htab_elem *l_old, in check_flags() argument
1074 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_update_elem() local
1092 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_map_update_elem()
1094 b = __select_bucket(htab, hash); in htab_map_update_elem()
1102 htab->n_buckets); in htab_map_update_elem()
1103 ret = check_flags(htab, l_old, map_flags); in htab_map_update_elem()
1125 ret = check_flags(htab, l_old, map_flags); in htab_map_update_elem()
1143 l_new = alloc_htab_elem(htab, key, value, key_size, hash, false, false, in htab_map_update_elem()
1164 if (htab_is_prealloc(htab)) { in htab_map_update_elem()
1167 check_and_free_fields(htab, l_old); in htab_map_update_elem()
1174 if (!htab_is_prealloc(htab)) in htab_map_update_elem()
1175 free_htab_elem(htab, l_old); in htab_map_update_elem()
1183 static void htab_lru_push_free(struct bpf_htab *htab, struct htab_elem *elem) in htab_lru_push_free() argument
1185 check_and_free_fields(htab, elem); in htab_lru_push_free()
1186 bpf_map_dec_elem_count(&htab->map); in htab_lru_push_free()
1187 bpf_lru_push_free(&htab->lru, &elem->lru_node); in htab_lru_push_free()
1193 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_lru_map_update_elem() local
1210 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_lru_map_update_elem()
1212 b = __select_bucket(htab, hash); in htab_lru_map_update_elem()
1220 l_new = prealloc_lru_pop(htab, key, hash); in htab_lru_map_update_elem()
1223 copy_map_value(&htab->map, in htab_lru_map_update_elem()
1232 ret = check_flags(htab, l_old, map_flags); in htab_lru_map_update_elem()
1251 htab_lru_push_free(htab, l_new); in htab_lru_map_update_elem()
1253 htab_lru_push_free(htab, l_old); in htab_lru_map_update_elem()
1262 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_percpu_map_update_elem() local
1279 hash = htab_map_hash(key, key_size, htab->hashrnd); in __htab_percpu_map_update_elem()
1281 b = __select_bucket(htab, hash); in __htab_percpu_map_update_elem()
1290 ret = check_flags(htab, l_old, map_flags); in __htab_percpu_map_update_elem()
1296 pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size), in __htab_percpu_map_update_elem()
1299 l_new = alloc_htab_elem(htab, key, value, key_size, in __htab_percpu_map_update_elem()
1317 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_lru_percpu_map_update_elem() local
1334 hash = htab_map_hash(key, key_size, htab->hashrnd); in __htab_lru_percpu_map_update_elem()
1336 b = __select_bucket(htab, hash); in __htab_lru_percpu_map_update_elem()
1345 l_new = prealloc_lru_pop(htab, key, hash); in __htab_lru_percpu_map_update_elem()
1356 ret = check_flags(htab, l_old, map_flags); in __htab_lru_percpu_map_update_elem()
1364 pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size), in __htab_lru_percpu_map_update_elem()
1367 pcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size), in __htab_lru_percpu_map_update_elem()
1377 bpf_map_dec_elem_count(&htab->map); in __htab_lru_percpu_map_update_elem()
1378 bpf_lru_push_free(&htab->lru, &l_new->lru_node); in __htab_lru_percpu_map_update_elem()
1399 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_delete_elem() local
1412 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_map_delete_elem()
1413 b = __select_bucket(htab, hash); in htab_map_delete_elem()
1429 free_htab_elem(htab, l); in htab_map_delete_elem()
1435 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_lru_map_delete_elem() local
1448 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_lru_map_delete_elem()
1449 b = __select_bucket(htab, hash); in htab_lru_map_delete_elem()
1465 htab_lru_push_free(htab, l); in htab_lru_map_delete_elem()
1469 static void delete_all_elements(struct bpf_htab *htab) in delete_all_elements() argument
1476 for (i = 0; i < htab->n_buckets; i++) { in delete_all_elements()
1477 struct hlist_nulls_head *head = select_bucket(htab, i); in delete_all_elements()
1483 htab_elem_free(htab, l); in delete_all_elements()
1489 static void htab_free_malloced_timers_and_wq(struct bpf_htab *htab) in htab_free_malloced_timers_and_wq() argument
1494 for (i = 0; i < htab->n_buckets; i++) { in htab_free_malloced_timers_and_wq()
1495 struct hlist_nulls_head *head = select_bucket(htab, i); in htab_free_malloced_timers_and_wq()
1501 if (btf_record_has_field(htab->map.record, BPF_TIMER)) in htab_free_malloced_timers_and_wq()
1502 bpf_obj_free_timer(htab->map.record, in htab_free_malloced_timers_and_wq()
1503 l->key + round_up(htab->map.key_size, 8)); in htab_free_malloced_timers_and_wq()
1504 if (btf_record_has_field(htab->map.record, BPF_WORKQUEUE)) in htab_free_malloced_timers_and_wq()
1505 bpf_obj_free_workqueue(htab->map.record, in htab_free_malloced_timers_and_wq()
1506 l->key + round_up(htab->map.key_size, 8)); in htab_free_malloced_timers_and_wq()
1515 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_free_timers_and_wq() local
1518 if (btf_record_has_field(htab->map.record, BPF_TIMER | BPF_WORKQUEUE)) { in htab_map_free_timers_and_wq()
1519 if (!htab_is_prealloc(htab)) in htab_map_free_timers_and_wq()
1520 htab_free_malloced_timers_and_wq(htab); in htab_map_free_timers_and_wq()
1522 htab_free_prealloced_timers_and_wq(htab); in htab_map_free_timers_and_wq()
1529 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_free() local
1540 if (!htab_is_prealloc(htab)) { in htab_map_free()
1541 delete_all_elements(htab); in htab_map_free()
1543 htab_free_prealloced_fields(htab); in htab_map_free()
1544 prealloc_destroy(htab); in htab_map_free()
1548 free_percpu(htab->extra_elems); in htab_map_free()
1549 bpf_map_area_free(htab->buckets); in htab_map_free()
1550 bpf_mem_alloc_destroy(&htab->pcpu_ma); in htab_map_free()
1551 bpf_mem_alloc_destroy(&htab->ma); in htab_map_free()
1552 if (htab->use_percpu_counter) in htab_map_free()
1553 percpu_counter_destroy(&htab->pcount); in htab_map_free()
1554 bpf_map_area_free(htab); in htab_map_free()
1582 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_and_delete_elem() local
1592 hash = htab_map_hash(key, key_size, htab->hashrnd); in __htab_map_lookup_and_delete_elem()
1593 b = __select_bucket(htab, hash); in __htab_map_lookup_and_delete_elem()
1613 copy_map_value_long(&htab->map, value + off, per_cpu_ptr(pptr, cpu)); in __htab_map_lookup_and_delete_elem()
1614 check_and_init_map_value(&htab->map, value + off); in __htab_map_lookup_and_delete_elem()
1637 htab_lru_push_free(htab, l); in __htab_map_lookup_and_delete_elem()
1639 free_htab_elem(htab, l); in __htab_map_lookup_and_delete_elem()
1682 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_and_delete_batch() local
1719 if (batch >= htab->n_buckets) in __htab_map_lookup_and_delete_batch()
1722 key_size = htab->map.key_size; in __htab_map_lookup_and_delete_batch()
1723 roundup_key_size = round_up(htab->map.key_size, 8); in __htab_map_lookup_and_delete_batch()
1724 value_size = htab->map.value_size; in __htab_map_lookup_and_delete_batch()
1751 b = &htab->buckets[batch]; in __htab_map_lookup_and_delete_batch()
1810 copy_map_value_long(&htab->map, dst_val + off, per_cpu_ptr(pptr, cpu)); in __htab_map_lookup_and_delete_batch()
1811 check_and_init_map_value(&htab->map, dst_val + off); in __htab_map_lookup_and_delete_batch()
1859 htab_lru_push_free(htab, l); in __htab_map_lookup_and_delete_batch()
1861 free_htab_elem(htab, l); in __htab_map_lookup_and_delete_batch()
1868 if (!bucket_cnt && (batch + 1 < htab->n_buckets)) { in __htab_map_lookup_and_delete_batch()
1885 if (batch >= htab->n_buckets) { in __htab_map_lookup_and_delete_batch()
1978 struct bpf_htab *htab; member
1988 const struct bpf_htab *htab = info->htab; in bpf_hash_map_seq_find_next() local
1997 if (bucket_id >= htab->n_buckets) in bpf_hash_map_seq_find_next()
2011 b = &htab->buckets[bucket_id++]; in bpf_hash_map_seq_find_next()
2016 for (i = bucket_id; i < htab->n_buckets; i++) { in bpf_hash_map_seq_find_next()
2017 b = &htab->buckets[i]; in bpf_hash_map_seq_find_next()
2135 seq_info->htab = container_of(map, struct bpf_htab, map); in bpf_iter_init_hash_map()
2164 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in bpf_for_each_hash_elem() local
2181 is_percpu = htab_is_percpu(htab); in bpf_for_each_hash_elem()
2188 for (i = 0; i < htab->n_buckets; i++) { in bpf_for_each_hash_elem()
2189 b = &htab->buckets[i]; in bpf_for_each_hash_elem()
2218 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_mem_usage() local
2219 u32 value_size = round_up(htab->map.value_size, 8); in htab_map_mem_usage()
2220 bool prealloc = htab_is_prealloc(htab); in htab_map_mem_usage()
2221 bool percpu = htab_is_percpu(htab); in htab_map_mem_usage()
2222 bool lru = htab_is_lru(htab); in htab_map_mem_usage()
2226 usage += sizeof(struct bucket) * htab->n_buckets; in htab_map_mem_usage()
2230 if (htab_has_extra_elems(htab)) in htab_map_mem_usage()
2233 usage += htab->elem_size * num_entries; in htab_map_mem_usage()
2242 num_entries = htab->use_percpu_counter ? in htab_map_mem_usage()
2243 percpu_counter_sum(&htab->pcount) : in htab_map_mem_usage()
2244 atomic_read(&htab->count); in htab_map_mem_usage()
2245 usage += (htab->elem_size + LLIST_NODE_SZ) * num_entries; in htab_map_mem_usage()
2271 BATCH_OPS(htab),
2406 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in bpf_percpu_hash_update() local
2410 if (htab_is_lru(htab)) in bpf_percpu_hash_update()
2500 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in fd_htab_map_free() local
2506 for (i = 0; i < htab->n_buckets; i++) { in fd_htab_map_free()
2507 head = select_bucket(htab, i); in fd_htab_map_free()
2630 BATCH_OPS(htab),