485 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_map_sys_lookup_elem()  local
493 	kvalue = &st_map->kvalue;  in bpf_struct_ops_map_sys_lookup_elem()
505 	memcpy(uvalue, st_map->uvalue, map->value_size);  in bpf_struct_ops_map_sys_lookup_elem()
524 static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map)  in bpf_struct_ops_map_put_progs()  argument
528 	for (i = 0; i < st_map->funcs_cnt; i++) {  in bpf_struct_ops_map_put_progs()
529 		if (!st_map->links[i])  in bpf_struct_ops_map_put_progs()
531 		bpf_link_put(st_map->links[i]);  in bpf_struct_ops_map_put_progs()
532 		st_map->links[i] = NULL;  in bpf_struct_ops_map_put_progs()
536 static void bpf_struct_ops_map_free_image(struct bpf_struct_ops_map *st_map)  in bpf_struct_ops_map_free_image()  argument
540 	for (i = 0; i < st_map->image_pages_cnt; i++)  in bpf_struct_ops_map_free_image()
541 		bpf_struct_ops_image_free(st_map->image_pages[i]);  in bpf_struct_ops_map_free_image()
542 	st_map->image_pages_cnt = 0;  in bpf_struct_ops_map_free_image()
642 static void bpf_struct_ops_map_add_ksyms(struct bpf_struct_ops_map *st_map)  in bpf_struct_ops_map_add_ksyms()  argument
646 	for (i = 0; i < st_map->funcs_cnt; i++) {  in bpf_struct_ops_map_add_ksyms()
647 		if (!st_map->ksyms[i])  in bpf_struct_ops_map_add_ksyms()
649 		bpf_image_ksym_add(st_map->ksyms[i]);  in bpf_struct_ops_map_add_ksyms()
653 static void bpf_struct_ops_map_del_ksyms(struct bpf_struct_ops_map *st_map)  in bpf_struct_ops_map_del_ksyms()  argument
657 	for (i = 0; i < st_map->funcs_cnt; i++) {  in bpf_struct_ops_map_del_ksyms()
658 		if (!st_map->ksyms[i])  in bpf_struct_ops_map_del_ksyms()
660 		bpf_image_ksym_del(st_map->ksyms[i]);  in bpf_struct_ops_map_del_ksyms()
664 static void bpf_struct_ops_map_free_ksyms(struct bpf_struct_ops_map *st_map)  in bpf_struct_ops_map_free_ksyms()  argument
668 	for (i = 0; i < st_map->funcs_cnt; i++) {  in bpf_struct_ops_map_free_ksyms()
669 		if (!st_map->ksyms[i])  in bpf_struct_ops_map_free_ksyms()
671 		kfree(st_map->ksyms[i]);  in bpf_struct_ops_map_free_ksyms()
672 		st_map->ksyms[i] = NULL;  in bpf_struct_ops_map_free_ksyms()
679 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_map_update_elem()  local
680 	const struct bpf_struct_ops_desc *st_ops_desc = st_map->st_ops_desc;  in bpf_struct_ops_map_update_elem()
701 	err = check_zero_holes(st_map->btf, st_ops_desc->value_type, value);  in bpf_struct_ops_map_update_elem()
706 	err = check_zero_holes(st_map->btf, t, uvalue->data);  in bpf_struct_ops_map_update_elem()
717 	uvalue = (struct bpf_struct_ops_value *)st_map->uvalue;  in bpf_struct_ops_map_update_elem()
718 	kvalue = (struct bpf_struct_ops_value *)&st_map->kvalue;  in bpf_struct_ops_map_update_elem()
720 	mutex_lock(&st_map->lock);  in bpf_struct_ops_map_update_elem()
732 	plink = st_map->links;  in bpf_struct_ops_map_update_elem()
733 	pksym = st_map->ksyms;  in bpf_struct_ops_map_update_elem()
734 	tname = btf_name_by_offset(st_map->btf, t->name_off);  in bpf_struct_ops_map_update_elem()
744 		mname = btf_name_by_offset(st_map->btf, member->name_off);  in bpf_struct_ops_map_update_elem()
745 		ptype = btf_type_resolve_ptr(st_map->btf, member->type, NULL);  in bpf_struct_ops_map_update_elem()
770 			mtype = btf_type_by_id(st_map->btf, member->type);  in bpf_struct_ops_map_update_elem()
771 			mtype = btf_resolve_size(st_map->btf, mtype, &msize);  in bpf_struct_ops_map_update_elem()
826 						st_map->image_pages_cnt < MAX_TRAMP_IMAGE_PAGES);  in bpf_struct_ops_map_update_elem()
831 			st_map->image_pages[st_map->image_pages_cnt++] = image;  in bpf_struct_ops_map_update_elem()
853 	for (i = 0; i < st_map->image_pages_cnt; i++) {  in bpf_struct_ops_map_update_elem()
854 		err = arch_protect_bpf_trampoline(st_map->image_pages[i],  in bpf_struct_ops_map_update_elem()
860 	if (st_map->map.map_flags & BPF_F_LINK) {  in bpf_struct_ops_map_update_elem()
894 	bpf_struct_ops_map_free_ksyms(st_map);  in bpf_struct_ops_map_update_elem()
895 	bpf_struct_ops_map_free_image(st_map);  in bpf_struct_ops_map_update_elem()
896 	bpf_struct_ops_map_put_progs(st_map);  in bpf_struct_ops_map_update_elem()
901 	mutex_unlock(&st_map->lock);  in bpf_struct_ops_map_update_elem()
903 		bpf_struct_ops_map_add_ksyms(st_map);  in bpf_struct_ops_map_update_elem()
910 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_map_delete_elem()  local
912 	st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_map_delete_elem()
913 	if (st_map->map.map_flags & BPF_F_LINK)  in bpf_struct_ops_map_delete_elem()
916 	prev_state = cmpxchg(&st_map->kvalue.common.state,  in bpf_struct_ops_map_delete_elem()
921 		st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, NULL);  in bpf_struct_ops_map_delete_elem()
938 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_map_seq_show_elem()  local
948 		btf_type_seq_show(st_map->btf,  in bpf_struct_ops_map_seq_show_elem()
959 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in __bpf_struct_ops_map_free()  local
961 	if (st_map->links)  in __bpf_struct_ops_map_free()
962 		bpf_struct_ops_map_put_progs(st_map);  in __bpf_struct_ops_map_free()
963 	if (st_map->ksyms)  in __bpf_struct_ops_map_free()
964 		bpf_struct_ops_map_free_ksyms(st_map);  in __bpf_struct_ops_map_free()
965 	bpf_map_area_free(st_map->links);  in __bpf_struct_ops_map_free()
966 	bpf_map_area_free(st_map->ksyms);  in __bpf_struct_ops_map_free()
967 	bpf_struct_ops_map_free_image(st_map);  in __bpf_struct_ops_map_free()
968 	bpf_map_area_free(st_map->uvalue);  in __bpf_struct_ops_map_free()
969 	bpf_map_area_free(st_map);  in __bpf_struct_ops_map_free()
974 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_map_free()  local
980 	if (btf_is_module(st_map->btf))  in bpf_struct_ops_map_free()
981 		module_put(st_map->st_ops_desc->st_ops->owner);  in bpf_struct_ops_map_free()
983 	bpf_struct_ops_map_del_ksyms(st_map);  in bpf_struct_ops_map_free()
1031 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_map_alloc()  local
1077 	st_map_size = sizeof(*st_map) +  in bpf_struct_ops_map_alloc()
1083 	st_map = bpf_map_area_alloc(st_map_size, NUMA_NO_NODE);  in bpf_struct_ops_map_alloc()
1084 	if (!st_map) {  in bpf_struct_ops_map_alloc()
1089 	st_map->st_ops_desc = st_ops_desc;  in bpf_struct_ops_map_alloc()
1090 	map = &st_map->map;  in bpf_struct_ops_map_alloc()
1092 	st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE);  in bpf_struct_ops_map_alloc()
1093 	st_map->funcs_cnt = count_func_ptrs(btf, t);  in bpf_struct_ops_map_alloc()
1094 	st_map->links =  in bpf_struct_ops_map_alloc()
1095 		bpf_map_area_alloc(st_map->funcs_cnt * sizeof(struct bpf_link *),  in bpf_struct_ops_map_alloc()
1098 	st_map->ksyms =  in bpf_struct_ops_map_alloc()
1099 		bpf_map_area_alloc(st_map->funcs_cnt * sizeof(struct bpf_ksym *),  in bpf_struct_ops_map_alloc()
1101 	if (!st_map->uvalue || !st_map->links || !st_map->ksyms) {  in bpf_struct_ops_map_alloc()
1105 	st_map->btf = btf;  in bpf_struct_ops_map_alloc()
1107 	mutex_init(&st_map->lock);  in bpf_struct_ops_map_alloc()
1122 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_map_mem_usage()  local
1123 	const struct bpf_struct_ops_desc *st_ops_desc = st_map->st_ops_desc;  in bpf_struct_ops_map_mem_usage()
1127 	usage = sizeof(*st_map) +  in bpf_struct_ops_map_mem_usage()
1130 	usage += st_map->funcs_cnt * sizeof(struct bpf_link *);  in bpf_struct_ops_map_mem_usage()
1131 	usage += st_map->funcs_cnt * sizeof(struct bpf_ksym *);  in bpf_struct_ops_map_mem_usage()
1156 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_get()  local
1160 	st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue);  in bpf_struct_ops_get()
1162 	map = __bpf_map_inc_not_zero(&st_map->map, false);  in bpf_struct_ops_get()
1169 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_put()  local
1172 	st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue);  in bpf_struct_ops_put()
1174 	bpf_map_put(&st_map->map);  in bpf_struct_ops_put()
1179 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_valid_to_reg()  local
1184 		smp_load_acquire(&st_map->kvalue.common.state) == BPF_STRUCT_OPS_STATE_READY;  in bpf_struct_ops_valid_to_reg()
1190 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_map_link_dealloc()  local
1193 	st_map = (struct bpf_struct_ops_map *)  in bpf_struct_ops_map_link_dealloc()
1195 	if (st_map) {  in bpf_struct_ops_map_link_dealloc()
1196 		st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, link);  in bpf_struct_ops_map_link_dealloc()
1197 		bpf_map_put(&st_map->map);  in bpf_struct_ops_map_link_dealloc()
1234 	struct bpf_struct_ops_map *st_map, *old_st_map;  in bpf_struct_ops_map_link_update()  local
1240 	st_map = container_of(new_map, struct bpf_struct_ops_map, map);  in bpf_struct_ops_map_link_update()
1245 	if (!st_map->st_ops_desc->st_ops->update)  in bpf_struct_ops_map_link_update()
1262 	if (st_map->st_ops_desc != old_st_map->st_ops_desc) {  in bpf_struct_ops_map_link_update()
1267 	err = st_map->st_ops_desc->st_ops->update(st_map->kvalue.data, old_st_map->kvalue.data, link);  in bpf_struct_ops_map_link_update()
1284 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_map_link_detach()  local
1294 	st_map = container_of(map, struct bpf_struct_ops_map, map);  in bpf_struct_ops_map_link_detach()
1296 	st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, link);  in bpf_struct_ops_map_link_detach()
1302 	bpf_map_put(&st_map->map);  in bpf_struct_ops_map_link_detach()
1334 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_link_create()  local
1342 	st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_link_create()
1366 	err = st_map->st_ops_desc->st_ops->reg(st_map->kvalue.data, &link->link);  in bpf_struct_ops_link_create()
1386 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_map_struct_ops_info_fill()  local
1388 	info->btf_vmlinux_id = btf_obj_id(st_map->btf);  in bpf_map_struct_ops_info_fill()