872 static struct bpf_shtab_bucket *sock_hash_select_bucket(struct bpf_shtab *htab,  in sock_hash_select_bucket()  argument
875 	return &htab->buckets[hash & (htab->buckets_num - 1)];  in sock_hash_select_bucket()
895 	struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);  in __sock_hash_lookup_elem()  local
903 	bucket = sock_hash_select_bucket(htab, hash);  in __sock_hash_lookup_elem()
909 static void sock_hash_free_elem(struct bpf_shtab *htab,  in sock_hash_free_elem()  argument
912 	atomic_dec(&htab->count);  in sock_hash_free_elem()
919 	struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);  in sock_hash_delete_from_link()  local
924 	bucket = sock_hash_select_bucket(htab, elem->hash);  in sock_hash_delete_from_link()
936 		sock_hash_free_elem(htab, elem);  in sock_hash_delete_from_link()
943 	struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);  in sock_hash_delete_elem()  local
950 	bucket = sock_hash_select_bucket(htab, hash);  in sock_hash_delete_elem()
957 		sock_hash_free_elem(htab, elem);  in sock_hash_delete_elem()
964 static struct bpf_shtab_elem *sock_hash_alloc_elem(struct bpf_shtab *htab,  in sock_hash_alloc_elem()  argument
971 	if (atomic_inc_return(&htab->count) > htab->map.max_entries) {  in sock_hash_alloc_elem()
973 			atomic_dec(&htab->count);  in sock_hash_alloc_elem()
978 	new = bpf_map_kmalloc_node(&htab->map, htab->elem_size,  in sock_hash_alloc_elem()
980 				   htab->map.numa_node);  in sock_hash_alloc_elem()
982 		atomic_dec(&htab->count);  in sock_hash_alloc_elem()
994 	struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);  in sock_hash_update_common()  local
1018 	bucket = sock_hash_select_bucket(htab, hash);  in sock_hash_update_common()
1030 	elem_new = sock_hash_alloc_elem(htab, key, key_size, hash, sk, elem);  in sock_hash_update_common()
1044 		sock_hash_free_elem(htab, elem);  in sock_hash_update_common()
1059 	struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);  in sock_hash_get_next_key()  local
1068 	head = &sock_hash_select_bucket(htab, hash)->head;  in sock_hash_get_next_key()
1080 	i = hash & (htab->buckets_num - 1);  in sock_hash_get_next_key()
1083 	for (; i < htab->buckets_num; i++) {  in sock_hash_get_next_key()
1084 		head = &sock_hash_select_bucket(htab, i)->head;  in sock_hash_get_next_key()
1098 	struct bpf_shtab *htab;  in sock_hash_alloc()  local
1110 	htab = bpf_map_area_alloc(sizeof(*htab), NUMA_NO_NODE);  in sock_hash_alloc()
1111 	if (!htab)  in sock_hash_alloc()
1114 	bpf_map_init_from_attr(&htab->map, attr);  in sock_hash_alloc()
1116 	htab->buckets_num = roundup_pow_of_two(htab->map.max_entries);  in sock_hash_alloc()
1117 	htab->elem_size = sizeof(struct bpf_shtab_elem) +  in sock_hash_alloc()
1118 			  round_up(htab->map.key_size, 8);  in sock_hash_alloc()
1119 	if (htab->buckets_num == 0 ||  in sock_hash_alloc()
1120 	    htab->buckets_num > U32_MAX / sizeof(struct bpf_shtab_bucket)) {  in sock_hash_alloc()
1125 	htab->buckets = bpf_map_area_alloc(htab->buckets_num *  in sock_hash_alloc()
1127 					   htab->map.numa_node);  in sock_hash_alloc()
1128 	if (!htab->buckets) {  in sock_hash_alloc()
1133 	for (i = 0; i < htab->buckets_num; i++) {  in sock_hash_alloc()
1134 		INIT_HLIST_HEAD(&htab->buckets[i].head);  in sock_hash_alloc()
1135 		spin_lock_init(&htab->buckets[i].lock);  in sock_hash_alloc()
1138 	return &htab->map;  in sock_hash_alloc()
1140 	bpf_map_area_free(htab);  in sock_hash_alloc()
1146 	struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);  in sock_hash_free()  local
1158 	for (i = 0; i < htab->buckets_num; i++) {  in sock_hash_free()
1159 		bucket = sock_hash_select_bucket(htab, i);  in sock_hash_free()
1185 			sock_hash_free_elem(htab, elem);  in sock_hash_free()
1193 	bpf_map_area_free(htab->buckets);  in sock_hash_free()
1194 	bpf_map_area_free(htab);  in sock_hash_free()
1312 	struct bpf_shtab *htab;  member
1319 	const struct bpf_shtab *htab = info->htab;  in sock_hash_seq_find_next()  local
1335 	for (; info->bucket_id < htab->buckets_num; info->bucket_id++) {  in sock_hash_seq_find_next()
1336 		bucket = &htab->buckets[info->bucket_id];  in sock_hash_seq_find_next()
1416 	info->htab = container_of(aux->map, struct bpf_shtab, map);  in sock_hash_init_seq_private()
1429 	struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);  in sock_hash_mem_usage()  local
1430 	u64 usage = sizeof(*htab);  in sock_hash_mem_usage()
1432 	usage += htab->buckets_num * sizeof(struct bpf_shtab_bucket);  in sock_hash_mem_usage()
1433 	usage += atomic_read(&htab->count) * (u64)htab->elem_size;  in sock_hash_mem_usage()