Lines Matching refs:vn
1058 struct vmap_node *vn; in find_vmap_area_exceed_addr_lock() local
1063 vn = &vmap_nodes[i]; in find_vmap_area_exceed_addr_lock()
1065 spin_lock(&vn->busy.lock); in find_vmap_area_exceed_addr_lock()
1066 *va = __find_vmap_area_exceed_addr(addr, &vn->busy.root); in find_vmap_area_exceed_addr_lock()
1071 spin_unlock(&vn->busy.lock); in find_vmap_area_exceed_addr_lock()
1080 vn = addr_to_node(va_start_lowest); in find_vmap_area_exceed_addr_lock()
1082 spin_lock(&vn->busy.lock); in find_vmap_area_exceed_addr_lock()
1083 *va = __find_vmap_area(va_start_lowest, &vn->busy.root); in find_vmap_area_exceed_addr_lock()
1086 return vn; in find_vmap_area_exceed_addr_lock()
1088 spin_unlock(&vn->busy.lock); in find_vmap_area_exceed_addr_lock()
1799 struct vmap_node *vn = addr_to_node(va->va_start); in free_vmap_area() local
1804 spin_lock(&vn->busy.lock); in free_vmap_area()
1805 unlink_va(va, &vn->busy.root); in free_vmap_area()
1806 spin_unlock(&vn->busy.lock); in free_vmap_area()
1841 size_to_va_pool(struct vmap_node *vn, unsigned long size) in size_to_va_pool() argument
1846 return &vn->pool[idx]; in size_to_va_pool()
1869 node_pool_del_va(struct vmap_node *vn, unsigned long size, in node_pool_del_va() argument
1877 vp = size_to_va_pool(vn, size); in node_pool_del_va()
1881 spin_lock(&vn->pool_lock); in node_pool_del_va()
1905 spin_unlock(&vn->pool_lock); in node_pool_del_va()
1958 struct vmap_node *vn; in alloc_vmap_area() local
2025 vn = addr_to_node(va->va_start); in alloc_vmap_area()
2027 spin_lock(&vn->busy.lock); in alloc_vmap_area()
2028 insert_vmap_area(va, &vn->busy.root, &vn->busy.head); in alloc_vmap_area()
2029 spin_unlock(&vn->busy.lock); in alloc_vmap_area()
2132 decay_va_pool_node(struct vmap_node *vn, bool full_decay) in decay_va_pool_node() argument
2143 if (list_empty(&vn->pool[i].head)) in decay_va_pool_node()
2147 spin_lock(&vn->pool_lock); in decay_va_pool_node()
2148 list_replace_init(&vn->pool[i].head, &tmp_list); in decay_va_pool_node()
2149 spin_unlock(&vn->pool_lock); in decay_va_pool_node()
2152 WRITE_ONCE(vn->pool[i].len, 0); in decay_va_pool_node()
2155 n_decay = vn->pool[i].len >> 2; in decay_va_pool_node()
2162 WRITE_ONCE(vn->pool[i].len, vn->pool[i].len - 1); in decay_va_pool_node()
2176 spin_lock(&vn->pool_lock); in decay_va_pool_node()
2177 list_replace_init(&tmp_list, &vn->pool[i].head); in decay_va_pool_node()
2178 spin_unlock(&vn->pool_lock); in decay_va_pool_node()
2186 kasan_release_vmalloc_node(struct vmap_node *vn) in kasan_release_vmalloc_node() argument
2191 start = list_first_entry(&vn->purge_list, struct vmap_area, list)->va_start; in kasan_release_vmalloc_node()
2192 end = list_last_entry(&vn->purge_list, struct vmap_area, list)->va_end; in kasan_release_vmalloc_node()
2194 list_for_each_entry(va, &vn->purge_list, list) { in kasan_release_vmalloc_node()
2206 struct vmap_node *vn = container_of(work, in purge_vmap_node() local
2213 kasan_release_vmalloc_node(vn); in purge_vmap_node()
2215 vn->nr_purged = 0; in purge_vmap_node()
2217 list_for_each_entry_safe(va, n_va, &vn->purge_list, list) { in purge_vmap_node()
2224 vn->nr_purged++; in purge_vmap_node()
2226 if (is_vn_id_valid(vn_id) && !vn->skip_populate) in purge_vmap_node()
2227 if (node_pool_add_va(vn, va)) in purge_vmap_node()
2248 struct vmap_node *vn; in __purge_vmap_area_lazy() local
2259 vn = &vmap_nodes[i]; in __purge_vmap_area_lazy()
2261 INIT_LIST_HEAD(&vn->purge_list); in __purge_vmap_area_lazy()
2262 vn->skip_populate = full_pool_decay; in __purge_vmap_area_lazy()
2263 decay_va_pool_node(vn, full_pool_decay); in __purge_vmap_area_lazy()
2265 if (RB_EMPTY_ROOT(&vn->lazy.root)) in __purge_vmap_area_lazy()
2268 spin_lock(&vn->lazy.lock); in __purge_vmap_area_lazy()
2269 WRITE_ONCE(vn->lazy.root.rb_node, NULL); in __purge_vmap_area_lazy()
2270 list_replace_init(&vn->lazy.head, &vn->purge_list); in __purge_vmap_area_lazy()
2271 spin_unlock(&vn->lazy.lock); in __purge_vmap_area_lazy()
2273 start = min(start, list_first_entry(&vn->purge_list, in __purge_vmap_area_lazy()
2276 end = max(end, list_last_entry(&vn->purge_list, in __purge_vmap_area_lazy()
2291 vn = &vmap_nodes[i]; in __purge_vmap_area_lazy()
2294 INIT_WORK(&vn->purge_work, purge_vmap_node); in __purge_vmap_area_lazy()
2297 schedule_work_on(i, &vn->purge_work); in __purge_vmap_area_lazy()
2299 schedule_work(&vn->purge_work); in __purge_vmap_area_lazy()
2303 vn->purge_work.func = NULL; in __purge_vmap_area_lazy()
2304 purge_vmap_node(&vn->purge_work); in __purge_vmap_area_lazy()
2305 nr_purged_areas += vn->nr_purged; in __purge_vmap_area_lazy()
2310 vn = &vmap_nodes[i]; in __purge_vmap_area_lazy()
2312 if (vn->purge_work.func) { in __purge_vmap_area_lazy()
2313 flush_work(&vn->purge_work); in __purge_vmap_area_lazy()
2314 nr_purged_areas += vn->nr_purged; in __purge_vmap_area_lazy()
2352 struct vmap_node *vn; in free_vmap_area_noflush() local
2365 vn = is_vn_id_valid(vn_id) ? in free_vmap_area_noflush()
2368 spin_lock(&vn->lazy.lock); in free_vmap_area_noflush()
2369 insert_vmap_area(va, &vn->lazy.root, &vn->lazy.head); in free_vmap_area_noflush()
2370 spin_unlock(&vn->lazy.lock); in free_vmap_area_noflush()
2394 struct vmap_node *vn; in find_vmap_area() local
2416 vn = &vmap_nodes[i]; in find_vmap_area()
2418 spin_lock(&vn->busy.lock); in find_vmap_area()
2419 va = __find_vmap_area(addr, &vn->busy.root); in find_vmap_area()
2420 spin_unlock(&vn->busy.lock); in find_vmap_area()
2431 struct vmap_node *vn; in find_unlink_vmap_area() local
2440 vn = &vmap_nodes[i]; in find_unlink_vmap_area()
2442 spin_lock(&vn->busy.lock); in find_unlink_vmap_area()
2443 va = __find_vmap_area(addr, &vn->busy.root); in find_unlink_vmap_area()
2445 unlink_va(va, &vn->busy.root); in find_unlink_vmap_area()
2446 spin_unlock(&vn->busy.lock); in find_unlink_vmap_area()
2670 struct vmap_node *vn; in free_vmap_block() local
2678 vn = addr_to_node(vb->va->va_start); in free_vmap_block()
2679 spin_lock(&vn->busy.lock); in free_vmap_block()
2680 unlink_va(vb->va, &vn->busy.root); in free_vmap_block()
2681 spin_unlock(&vn->busy.lock); in free_vmap_block()
4357 struct vmap_node *vn; in vread_iter() local
4372 vn = find_vmap_area_exceed_addr_lock((unsigned long) addr, &va); in vread_iter()
4373 if (!vn) in vread_iter()
4439 spin_unlock(&vn->busy.lock); in vread_iter()
4440 } while ((vn = find_vmap_area_exceed_addr_lock(next, &va))); in vread_iter()
4443 if (vn) in vread_iter()
4444 spin_unlock(&vn->busy.lock); in vread_iter()
4450 if (vn) in vread_iter()
4451 spin_unlock(&vn->busy.lock); in vread_iter()
4786 struct vmap_node *vn = addr_to_node(vas[area]->va_start); in pcpu_get_vm_areas() local
4788 spin_lock(&vn->busy.lock); in pcpu_get_vm_areas()
4789 insert_vmap_area(vas[area], &vn->busy.root, &vn->busy.head); in pcpu_get_vm_areas()
4792 spin_unlock(&vn->busy.lock); in pcpu_get_vm_areas()
4907 struct vmap_node *vn; in vmalloc_dump_obj() local
4912 vn = addr_to_node(addr); in vmalloc_dump_obj()
4914 if (!spin_trylock(&vn->busy.lock)) in vmalloc_dump_obj()
4917 va = __find_vmap_area(addr, &vn->busy.root); in vmalloc_dump_obj()
4919 spin_unlock(&vn->busy.lock); in vmalloc_dump_obj()
4927 spin_unlock(&vn->busy.lock); in vmalloc_dump_obj()
4963 struct vmap_node *vn; in show_purge_info() local
4968 vn = &vmap_nodes[i]; in show_purge_info()
4970 spin_lock(&vn->lazy.lock); in show_purge_info()
4971 list_for_each_entry(va, &vn->lazy.head, list) { in show_purge_info()
4976 spin_unlock(&vn->lazy.lock); in show_purge_info()
4982 struct vmap_node *vn; in vmalloc_info_show() local
4988 vn = &vmap_nodes[i]; in vmalloc_info_show()
4990 spin_lock(&vn->busy.lock); in vmalloc_info_show()
4991 list_for_each_entry(va, &vn->busy.head, list) { in vmalloc_info_show()
5039 spin_unlock(&vn->busy.lock); in vmalloc_info_show()
5109 struct vmap_node *vn; in vmap_init_nodes() local
5130 vn = kmalloc_array(n, sizeof(*vn), GFP_NOWAIT | __GFP_NOWARN); in vmap_init_nodes()
5131 if (vn) { in vmap_init_nodes()
5135 vmap_nodes = vn; in vmap_init_nodes()
5143 vn = &vmap_nodes[n]; in vmap_init_nodes()
5144 vn->busy.root = RB_ROOT; in vmap_init_nodes()
5145 INIT_LIST_HEAD(&vn->busy.head); in vmap_init_nodes()
5146 spin_lock_init(&vn->busy.lock); in vmap_init_nodes()
5148 vn->lazy.root = RB_ROOT; in vmap_init_nodes()
5149 INIT_LIST_HEAD(&vn->lazy.head); in vmap_init_nodes()
5150 spin_lock_init(&vn->lazy.lock); in vmap_init_nodes()
5153 INIT_LIST_HEAD(&vn->pool[i].head); in vmap_init_nodes()
5154 WRITE_ONCE(vn->pool[i].len, 0); in vmap_init_nodes()
5157 spin_lock_init(&vn->pool_lock); in vmap_init_nodes()
5165 struct vmap_node *vn; in vmap_node_shrink_count() local
5169 vn = &vmap_nodes[i]; in vmap_node_shrink_count()
5172 count += READ_ONCE(vn->pool[j].len); in vmap_node_shrink_count()
5193 struct vmap_node *vn; in vmalloc_init() local
5230 vn = addr_to_node(va->va_start); in vmalloc_init()
5231 insert_vmap_area(va, &vn->busy.root, &vn->busy.head); in vmalloc_init()