21 select_bucket(struct bpf_local_storage_map *smap,  in select_bucket()  argument
24 	return &smap->buckets[hash_ptr(selem, smap->bucket_log)];  in select_bucket()
27 static int mem_charge(struct bpf_local_storage_map *smap, void *owner, u32 size)  in mem_charge()  argument
29 	struct bpf_map *map = &smap->map;  in mem_charge()
34 	return map->ops->map_local_storage_charge(smap, owner, size);  in mem_charge()
37 static void mem_uncharge(struct bpf_local_storage_map *smap, void *owner,  in mem_uncharge()  argument
40 	struct bpf_map *map = &smap->map;  in mem_uncharge()
43 		map->ops->map_local_storage_uncharge(smap, owner, size);  in mem_uncharge()
47 owner_storage(struct bpf_local_storage_map *smap, void *owner)  in owner_storage()  argument
49 	struct bpf_map *map = &smap->map;  in owner_storage()
75 bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner,  in bpf_selem_alloc()  argument
80 	if (charge_mem && mem_charge(smap, owner, smap->elem_size))  in bpf_selem_alloc()
83 	if (smap->bpf_ma) {  in bpf_selem_alloc()
84 		selem = bpf_mem_cache_alloc_flags(&smap->selem_ma, gfp_flags);  in bpf_selem_alloc()
93 			memset(SDATA(selem)->data, 0, smap->map.value_size);  in bpf_selem_alloc()
95 		selem = bpf_map_kzalloc(&smap->map, smap->elem_size,  in bpf_selem_alloc()
102 			copy_map_value(&smap->map, SDATA(selem)->data, value);  in bpf_selem_alloc()
104 				bpf_obj_swap_uptrs(smap->map.record, SDATA(selem)->data, value);  in bpf_selem_alloc()
110 		mem_uncharge(smap, owner, smap->elem_size);  in bpf_selem_alloc()
158 				   struct bpf_local_storage_map *smap,  in bpf_local_storage_free()  argument
175 	if (smap)  in bpf_local_storage_free()
176 		bpf_mem_cache_free(&smap->storage_ma, local_storage);  in bpf_local_storage_free()
210 	struct bpf_local_storage_map *smap;  in bpf_selem_free_rcu()  local
214 	smap = rcu_dereference_check(SDATA(selem)->smap, 1);  in bpf_selem_free_rcu()
217 	bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);  in bpf_selem_free_rcu()
231 		    struct bpf_local_storage_map *smap,  in bpf_selem_free()  argument
234 	if (!smap->bpf_ma) {  in bpf_selem_free()
240 		bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);  in bpf_selem_free()
252 		bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);  in bpf_selem_free()
257 		bpf_mem_cache_free(&smap->selem_ma, selem);  in bpf_selem_free()
267 	struct bpf_local_storage_map *smap;  in bpf_selem_free_list()  local
276 		smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());  in bpf_selem_free_list()
277 		bpf_selem_free(selem, smap, reuse_now);  in bpf_selem_free_list()
289 	struct bpf_local_storage_map *smap;  in bpf_selem_unlink_storage_nolock()  local
293 	smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());  in bpf_selem_unlink_storage_nolock()
301 		mem_uncharge(smap, owner, smap->elem_size);  in bpf_selem_unlink_storage_nolock()
306 		mem_uncharge(smap, owner, sizeof(struct bpf_local_storage));  in bpf_selem_unlink_storage_nolock()
310 		RCU_INIT_POINTER(*owner_storage(smap, owner), NULL);  in bpf_selem_unlink_storage_nolock()
327 	if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) ==  in bpf_selem_unlink_storage_nolock()
329 		RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL);  in bpf_selem_unlink_storage_nolock()
333 	if (rcu_access_pointer(local_storage->smap) == smap)  in bpf_selem_unlink_storage_nolock()
334 		RCU_INIT_POINTER(local_storage->smap, NULL);  in bpf_selem_unlink_storage_nolock()
369 	selem_smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());  in check_storage_bpf_ma()
389 	storage_smap = rcu_dereference_check(local_storage->smap,  in bpf_selem_unlink_storage()
414 	struct bpf_local_storage_map *smap;  in bpf_selem_unlink_map()  local
422 	smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());  in bpf_selem_unlink_map()
423 	b = select_bucket(smap, selem);  in bpf_selem_unlink_map()
430 void bpf_selem_link_map(struct bpf_local_storage_map *smap,  in bpf_selem_link_map()  argument
433 	struct bpf_local_storage_map_bucket *b = select_bucket(smap, selem);  in bpf_selem_link_map()
437 	RCU_INIT_POINTER(SDATA(selem)->smap, smap);  in bpf_selem_link_map()
453 				      struct bpf_local_storage_map *smap,  in __bpf_local_storage_insert_cache()  argument
465 		rcu_assign_pointer(local_storage->cache[smap->cache_idx], SDATA(selem));  in __bpf_local_storage_insert_cache()
484 			    struct bpf_local_storage_map *smap,  in bpf_local_storage_alloc()  argument
492 	err = mem_charge(smap, owner, sizeof(*storage));  in bpf_local_storage_alloc()
496 	if (smap->bpf_ma)  in bpf_local_storage_alloc()
497 		storage = bpf_mem_cache_alloc_flags(&smap->storage_ma, gfp_flags);  in bpf_local_storage_alloc()
499 		storage = bpf_map_kzalloc(&smap->map, sizeof(*storage),  in bpf_local_storage_alloc()
506 	RCU_INIT_POINTER(storage->smap, smap);  in bpf_local_storage_alloc()
512 	bpf_selem_link_map(smap, first_selem);  in bpf_local_storage_alloc()
515 		(struct bpf_local_storage **)owner_storage(smap, owner);  in bpf_local_storage_alloc()
546 	bpf_local_storage_free(storage, smap, smap->bpf_ma, true);  in bpf_local_storage_alloc()
547 	mem_uncharge(smap, owner, sizeof(*storage));  in bpf_local_storage_alloc()
557 bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,  in bpf_local_storage_update()  argument
571 		     !btf_record_has_field(smap->map.record, BPF_SPIN_LOCK)))  in bpf_local_storage_update()
577 	local_storage = rcu_dereference_check(*owner_storage(smap, owner),  in bpf_local_storage_update()
585 		selem = bpf_selem_alloc(smap, owner, value, true, swap_uptrs, gfp_flags);  in bpf_local_storage_update()
589 		err = bpf_local_storage_alloc(owner, smap, selem, gfp_flags);  in bpf_local_storage_update()
591 			bpf_selem_free(selem, smap, true);  in bpf_local_storage_update()
592 			mem_uncharge(smap, owner, smap->elem_size);  in bpf_local_storage_update()
605 			bpf_local_storage_lookup(local_storage, smap, false);  in bpf_local_storage_update()
610 			copy_map_value_locked(&smap->map, old_sdata->data,  in bpf_local_storage_update()
619 	alloc_selem = selem = bpf_selem_alloc(smap, owner, value, true, swap_uptrs, gfp_flags);  in bpf_local_storage_update()
636 	old_sdata = bpf_local_storage_lookup(local_storage, smap, false);  in bpf_local_storage_update()
642 		copy_map_value_locked(&smap->map, old_sdata->data, value,  in bpf_local_storage_update()
650 	bpf_selem_link_map(smap, selem);  in bpf_local_storage_update()
666 		mem_uncharge(smap, owner, smap->elem_size);  in bpf_local_storage_update()
667 		bpf_selem_free(alloc_selem, smap, true);  in bpf_local_storage_update()
746 	storage_smap = rcu_dereference_check(local_storage->smap, bpf_rcu_lock_held());  in bpf_local_storage_destroy()
783 	struct bpf_local_storage_map *smap = (struct bpf_local_storage_map *)map;  in bpf_local_storage_map_mem_usage()  local
784 	u64 usage = sizeof(*smap);  in bpf_local_storage_map_mem_usage()
787 	usage += sizeof(*smap->buckets) * (1ULL << smap->bucket_log);  in bpf_local_storage_map_mem_usage()
806 	struct bpf_local_storage_map *smap;  in bpf_local_storage_map_alloc()  local
811 	smap = bpf_map_area_alloc(sizeof(*smap), NUMA_NO_NODE);  in bpf_local_storage_map_alloc()
812 	if (!smap)  in bpf_local_storage_map_alloc()
814 	bpf_map_init_from_attr(&smap->map, attr);  in bpf_local_storage_map_alloc()
819 	smap->bucket_log = ilog2(nbuckets);  in bpf_local_storage_map_alloc()
821 	smap->buckets = bpf_map_kvcalloc(&smap->map, nbuckets,  in bpf_local_storage_map_alloc()
822 					 sizeof(*smap->buckets), GFP_USER | __GFP_NOWARN);  in bpf_local_storage_map_alloc()
823 	if (!smap->buckets) {  in bpf_local_storage_map_alloc()
829 		INIT_HLIST_HEAD(&smap->buckets[i].list);  in bpf_local_storage_map_alloc()
830 		raw_spin_lock_init(&smap->buckets[i].lock);  in bpf_local_storage_map_alloc()
833 	smap->elem_size = offsetof(struct bpf_local_storage_elem,  in bpf_local_storage_map_alloc()
840 	smap->bpf_ma = IS_ENABLED(CONFIG_PREEMPT_RT) ? true : bpf_ma;  in bpf_local_storage_map_alloc()
841 	if (smap->bpf_ma) {  in bpf_local_storage_map_alloc()
842 		err = bpf_mem_alloc_init(&smap->selem_ma, smap->elem_size, false);  in bpf_local_storage_map_alloc()
846 		err = bpf_mem_alloc_init(&smap->storage_ma, sizeof(struct bpf_local_storage), false);  in bpf_local_storage_map_alloc()
848 			bpf_mem_alloc_destroy(&smap->selem_ma);  in bpf_local_storage_map_alloc()
853 	smap->cache_idx = bpf_local_storage_cache_idx_get(cache);  in bpf_local_storage_map_alloc()
854 	return &smap->map;  in bpf_local_storage_map_alloc()
857 	kvfree(smap->buckets);  in bpf_local_storage_map_alloc()
858 	bpf_map_area_free(smap);  in bpf_local_storage_map_alloc()
868 	struct bpf_local_storage_map *smap;  in bpf_local_storage_map_free()  local
871 	smap = (struct bpf_local_storage_map *)map;  in bpf_local_storage_map_free()
872 	bpf_local_storage_cache_idx_free(cache, smap->cache_idx);  in bpf_local_storage_map_free()
889 	for (i = 0; i < (1U << smap->bucket_log); i++) {  in bpf_local_storage_map_free()
890 		b = &smap->buckets[i];  in bpf_local_storage_map_free()
921 	if (smap->bpf_ma) {  in bpf_local_storage_map_free()
925 		bpf_mem_alloc_destroy(&smap->selem_ma);  in bpf_local_storage_map_free()
926 		bpf_mem_alloc_destroy(&smap->storage_ma);  in bpf_local_storage_map_free()
928 	kvfree(smap->buckets);  in bpf_local_storage_map_free()
929 	bpf_map_area_free(smap);  in bpf_local_storage_map_free()