Lines Matching refs:object
220 vm_object_t object);
223 vm_object_t object,
227 vm_object_t object,
427 static void vm_object_reap(vm_object_t object);
428 static void vm_object_reap_async(vm_object_t object);
462 vm_object_t object, in vm_object_set_size() argument
466 object->vo_size = vm_object_round_page(outer_size); in vm_object_set_size()
468 assert(object->vo_size - inner_size <= USHRT_MAX); in vm_object_set_size()
469 object->vo_size_delta = (unsigned short)(object->vo_size - inner_size); in vm_object_set_size()
485 vm_object_t object) in _vm_object_allocate() argument
487 *object = vm_object_template; in _vm_object_allocate()
488 vm_page_queue_init(&object->memq); in _vm_object_allocate()
490 queue_init(&object->uplq); in _vm_object_allocate()
492 vm_object_lock_init(object); in _vm_object_allocate()
493 vm_object_set_size(object, size, size); in _vm_object_allocate()
495 os_ref_init_raw(&object->ref_count, &vm_object_refgrp); in _vm_object_allocate()
499 btlog_record(vm_object_tracking_btlog, object, in _vm_object_allocate()
510 vm_object_t object; in vm_object_allocate() local
512 object = zalloc_flags(vm_object_zone, Z_WAITOK | Z_NOFAIL); in vm_object_allocate()
513 _vm_object_allocate(size, object); in vm_object_allocate()
515 return object; in vm_object_allocate()
620 vm_object_t object) in vm_object_deallocate() argument
627 if (object == VM_OBJECT_NULL) { in vm_object_deallocate()
631 if (is_kernel_object(object) || object == compressor_object || object == retired_pages_object) { in vm_object_deallocate()
632 vm_object_lock_shared(object); in vm_object_deallocate()
634 if (os_ref_get_count_raw(&object->ref_count) == 1) { in vm_object_deallocate()
635 if (is_kernel_object(object)) { in vm_object_deallocate()
637 } else if (object == retired_pages_object) { in vm_object_deallocate()
644 os_ref_release_live_raw(&object->ref_count, &vm_object_refgrp); in vm_object_deallocate()
646 vm_object_unlock(object); in vm_object_deallocate()
650 if (os_ref_get_count_raw(&object->ref_count) == 2 && in vm_object_deallocate()
651 object->named) { in vm_object_deallocate()
657 } else if (os_ref_get_count_raw(&object->ref_count) == 2 && in vm_object_deallocate()
658 object->internal && in vm_object_deallocate()
659 object->shadow != VM_OBJECT_NULL) { in vm_object_deallocate()
666 } else if (os_ref_get_count_raw(&object->ref_count) >= 2) { in vm_object_deallocate()
680 vm_object_lock_shared(object); in vm_object_deallocate()
681 ref_count_p = (volatile UInt32 *) &object->ref_count; in vm_object_deallocate()
682 original_ref_count = os_ref_get_count_raw(&object->ref_count); in vm_object_deallocate()
688 object->named) { in vm_object_deallocate()
692 object->internal && in vm_object_deallocate()
693 object->shadow != VM_OBJECT_NULL) { in vm_object_deallocate()
704 (UInt32 *) &object->ref_count); in vm_object_deallocate()
711 vm_object_unlock(object); in vm_object_deallocate()
729 while (object != VM_OBJECT_NULL) { in vm_object_deallocate()
730 vm_object_lock(object); in vm_object_deallocate()
732 assert(os_ref_get_count_raw(&object->ref_count) > 0); in vm_object_deallocate()
739 if ((os_ref_get_count_raw(&object->ref_count) == 2) && (object->named)) { in vm_object_deallocate()
740 memory_object_t pager = object->pager; in vm_object_deallocate()
746 vm_object_mapping_wait(object, THREAD_UNINT); in vm_object_deallocate()
748 pager = object->pager; in vm_object_deallocate()
749 if (object->ref_count == 2 && in vm_object_deallocate()
750 object->named && in vm_object_deallocate()
752 vm_object_mapping_begin(object); in vm_object_deallocate()
753 vm_object_unlock(object); in vm_object_deallocate()
757 vm_object_lock(object); in vm_object_deallocate()
758 vm_object_mapping_end(object); in vm_object_deallocate()
761 assert(os_ref_get_count_raw(&object->ref_count) > 0); in vm_object_deallocate()
776 if ((os_ref_get_count_raw(&object->ref_count) > 1) || in vm_object_deallocate()
777 object->terminating) { in vm_object_deallocate()
778 vm_object_lock_assert_exclusive(object); in vm_object_deallocate()
779 os_ref_release_live_locked_raw(&object->ref_count, in vm_object_deallocate()
782 if (os_ref_get_count_raw(&object->ref_count) == 1 && in vm_object_deallocate()
783 object->shadow != VM_OBJECT_NULL) { in vm_object_deallocate()
800 vm_object_collapse(object, 0, FALSE); in vm_object_deallocate()
802 vm_object_unlock(object); in vm_object_deallocate()
811 if (object->pager_created && !object->pager_initialized) { in vm_object_deallocate()
812 assert(!object->can_persist); in vm_object_deallocate()
813 vm_object_sleep(object, in vm_object_deallocate()
828 shadow = object->pageout?VM_OBJECT_NULL:object->shadow; in vm_object_deallocate()
830 if (vm_object_terminate(object) != KERN_SUCCESS) { in vm_object_deallocate()
834 object = shadow; in vm_object_deallocate()
845 vm_object_t object) in vm_object_page_grab() argument
851 vm_object_lock_assert_exclusive(object); in vm_object_page_grab()
853 next_p = (vm_page_t)vm_page_queue_first(&object->memq); in vm_object_page_grab()
854 p_limit = MIN(50, object->resident_page_count); in vm_object_page_grab()
856 while (!vm_page_queue_end(&object->memq, (vm_page_queue_entry_t)next_p) && --p_limit > 0) { in vm_object_page_grab()
906 vm_page_queue_remove(&object->memq, p, vmp_listq); in vm_object_page_grab()
907 vm_page_queue_enter(&object->memq, p, vmp_listq); in vm_object_page_grab()
939 vm_object_t object) in vm_object_cache_remove_locked() argument
941 assert(object->purgable == VM_PURGABLE_DENY); in vm_object_cache_remove_locked()
943 queue_remove(&vm_object_cached_list, object, vm_object_t, cached_list); in vm_object_cache_remove_locked()
944 object->cached_list.next = NULL; in vm_object_cache_remove_locked()
945 object->cached_list.prev = NULL; in vm_object_cache_remove_locked()
952 vm_object_t object) in vm_object_cache_remove() argument
956 if (object->cached_list.next && in vm_object_cache_remove()
957 object->cached_list.prev) { in vm_object_cache_remove()
958 vm_object_cache_remove_locked(object); in vm_object_cache_remove()
966 vm_object_t object) in vm_object_cache_add() argument
971 assert(object->purgable == VM_PURGABLE_DENY); in vm_object_cache_add()
973 if (object->resident_page_count == 0) { in vm_object_cache_add()
976 if (object->vo_ledger_tag) { in vm_object_cache_add()
988 if (object->cached_list.next == NULL && in vm_object_cache_add()
989 object->cached_list.prev == NULL) { in vm_object_cache_add()
990 queue_enter(&vm_object_cached_list, object, vm_object_t, cached_list); in vm_object_cache_add()
991 object->vo_cache_ts = sec + EVICT_AGE; in vm_object_cache_add()
992 object->vo_cache_pages_to_scan = object->resident_page_count; in vm_object_cache_add()
1005 vm_object_t object = VM_OBJECT_NULL; in vm_object_cache_evict() local
1053 object = next_obj; in vm_object_cache_evict()
1056 assert(object->purgable == VM_PURGABLE_DENY); in vm_object_cache_evict()
1058 if (sec < object->vo_cache_ts) { // reached the point in the queue beyond the time we started in vm_object_cache_evict()
1059 KDBG_DEBUG(0x130020c, object, object->resident_page_count, object->vo_cache_ts, sec); in vm_object_cache_evict()
1061 vm_object_cache_aging_ts = object->vo_cache_ts; in vm_object_cache_evict()
1062 object = VM_OBJECT_NULL; /* this will cause to break away from the outer loop */ in vm_object_cache_evict()
1065 if (!vm_object_lock_try_scan(object)) { in vm_object_cache_evict()
1071 KDBG_DEBUG(0x13001f8, object, object->resident_page_count); in vm_object_cache_evict()
1073 object = VM_OBJECT_NULL; in vm_object_cache_evict()
1076 if (vm_page_queue_empty(&object->memq) || object->vo_cache_pages_to_scan == 0) { in vm_object_cache_evict()
1082 KDBG_DEBUG(0x13001fc, object, object->resident_page_count, ep_freed, ep_moved); in vm_object_cache_evict()
1084 vm_object_cache_remove_locked(object); in vm_object_cache_evict()
1085 vm_object_unlock(object); in vm_object_cache_evict()
1086 object = VM_OBJECT_NULL; in vm_object_cache_evict()
1097 if (object == VM_OBJECT_NULL) { in vm_object_cache_evict()
1105 next_p = (vm_page_t)vm_page_queue_first(&object->memq); in vm_object_cache_evict()
1120 …while (!vm_page_queue_end(&object->memq, (vm_page_queue_entry_t)next_p) && object->vo_cache_pages_… in vm_object_cache_evict()
1124 object->vo_cache_pages_to_scan--; in vm_object_cache_evict()
1127 vm_page_queue_remove(&object->memq, p, vmp_listq); in vm_object_cache_evict()
1128 vm_page_queue_enter(&object->memq, p, vmp_listq); in vm_object_cache_evict()
1134 vm_page_queue_remove(&object->memq, p, vmp_listq); in vm_object_cache_evict()
1135 vm_page_queue_enter(&object->memq, p, vmp_listq); in vm_object_cache_evict()
1141 KDBG_DEBUG(0x13001f4 | DBG_FUNC_START, object, object->resident_page_count, ep_freed, ep_moved); in vm_object_cache_evict()
1179 KDBG_DEBUG(0x13001f4 | DBG_FUNC_END, object, object->resident_page_count, ep_freed, ep_moved); in vm_object_cache_evict()
1185 if (object->vo_cache_pages_to_scan == 0) { in vm_object_cache_evict()
1186 KDBG_DEBUG(0x1300208, object, object->resident_page_count, ep_freed, ep_moved); in vm_object_cache_evict()
1188 vm_object_cache_remove(object); in vm_object_cache_evict()
1190 KDBG_DEBUG(0x13001fc, object, object->resident_page_count, ep_freed, ep_moved); in vm_object_cache_evict()
1195 vm_object_unlock(object); in vm_object_cache_evict()
1196 object = VM_OBJECT_NULL; in vm_object_cache_evict()
1243 vm_object_t object) in vm_object_terminate() argument
1247 vm_object_lock_assert_exclusive(object); in vm_object_terminate()
1249 if (!object->pageout && (!object->internal && object->can_persist) && in vm_object_terminate()
1250 (object->pager != NULL || object->shadow_severed)) { in vm_object_terminate()
1256 VM_OBJECT_SET_PAGER_TRUSTED(object, FALSE); in vm_object_terminate()
1258 vm_object_reap_pages(object, REAP_TERMINATE); in vm_object_terminate()
1263 if (object->terminating) { in vm_object_terminate()
1264 vm_object_lock_assert_exclusive(object); in vm_object_terminate()
1265 os_ref_release_live_locked_raw(&object->ref_count, &vm_object_refgrp); in vm_object_terminate()
1266 vm_object_unlock(object); in vm_object_terminate()
1274 if (os_ref_get_count_raw(&object->ref_count) != 1) { in vm_object_terminate()
1275 vm_object_lock_assert_exclusive(object); in vm_object_terminate()
1276 os_ref_release_live_locked_raw(&object->ref_count, &vm_object_refgrp); in vm_object_terminate()
1277 vm_object_unlock(object); in vm_object_terminate()
1285 VM_OBJECT_SET_TERMINATING(object, TRUE); in vm_object_terminate()
1286 VM_OBJECT_SET_ALIVE(object, FALSE); in vm_object_terminate()
1288 if (!object->internal && in vm_object_terminate()
1289 object->cached_list.next && in vm_object_terminate()
1290 object->cached_list.prev) { in vm_object_terminate()
1291 vm_object_cache_remove(object); in vm_object_terminate()
1299 if (((shadow_object = object->shadow) != VM_OBJECT_NULL) && in vm_object_terminate()
1300 !(object->pageout)) { in vm_object_terminate()
1302 if (shadow_object->vo_copy == object) { in vm_object_terminate()
1308 if (object->paging_in_progress != 0 || in vm_object_terminate()
1309 object->activity_in_progress != 0) { in vm_object_terminate()
1331 vm_object_reap_async(object); in vm_object_terminate()
1332 vm_object_unlock(object); in vm_object_terminate()
1345 vm_object_reap(object); in vm_object_terminate()
1346 object = VM_OBJECT_NULL; in vm_object_terminate()
1374 vm_object_t object) in vm_object_reap() argument
1379 vm_object_lock_assert_exclusive(object); in vm_object_reap()
1380 assert(object->paging_in_progress == 0); in vm_object_reap()
1381 assert(object->activity_in_progress == 0); in vm_object_reap()
1391 (object->purgable != VM_PURGABLE_DENY || in vm_object_reap()
1392 object->vo_ledger_tag)) { in vm_object_reap()
1397 assert(!object->alive); in vm_object_reap()
1398 assert(object->terminating); in vm_object_reap()
1399 kr = vm_object_ownership_change(object, in vm_object_reap()
1405 assert(object->vo_owner == NULL); in vm_object_reap()
1409 if (object->object_is_shared_cache && in vm_object_reap()
1410 object->pager != NULL && in vm_object_reap()
1411 object->pager->mo_pager_ops == &shared_region_pager_ops) { in vm_object_reap()
1412 OSAddAtomic(-object->resident_page_count, &shared_region_pagers_resident_count); in vm_object_reap()
1416 pager = object->pager; in vm_object_reap()
1417 object->pager = MEMORY_OBJECT_NULL; in vm_object_reap()
1420 memory_object_control_disable(&object->pager_control); in vm_object_reap()
1423 ref_count = os_ref_release_locked_raw(&object->ref_count, in vm_object_reap()
1433 if (object->internal) { in vm_object_reap()
1434 assert(VM_OBJECT_OWNER(object) == TASK_NULL); in vm_object_reap()
1436 VM_OBJECT_UNWIRED(object); in vm_object_reap()
1438 if (object->purgable == VM_PURGABLE_DENY) { in vm_object_reap()
1440 } else if (object->purgable == VM_PURGABLE_VOLATILE) { in vm_object_reap()
1443 queue = vm_purgeable_object_remove(object); in vm_object_reap()
1446 if (object->purgeable_when_ripe) { in vm_object_reap()
1465 assert(object->resident_page_count >= in vm_object_reap()
1466 object->wired_page_count); in vm_object_reap()
1467 delta = (object->resident_page_count - in vm_object_reap()
1468 object->wired_page_count); in vm_object_reap()
1474 if (object->wired_page_count != 0) { in vm_object_reap()
1476 object->wired_page_count); in vm_object_reap()
1477 OSAddAtomic(-object->wired_page_count, in vm_object_reap()
1480 VM_OBJECT_SET_PURGABLE(object, VM_PURGABLE_EMPTY); in vm_object_reap()
1481 } else if (object->purgable == VM_PURGABLE_NONVOLATILE || in vm_object_reap()
1482 object->purgable == VM_PURGABLE_EMPTY) { in vm_object_reap()
1484 vm_purgeable_nonvolatile_dequeue(object); in vm_object_reap()
1487 object, object->purgable); in vm_object_reap()
1489 if (object->transposed && in vm_object_reap()
1490 object->cached_list.next != NULL && in vm_object_reap()
1491 object->cached_list.prev == NULL) { in vm_object_reap()
1497 assert(object->cached_list.next == NULL); in vm_object_reap()
1499 assert(object->cached_list.prev == NULL); in vm_object_reap()
1502 if (object->pageout) { in vm_object_reap()
1508 assert(object->shadow != VM_OBJECT_NULL); in vm_object_reap()
1510 vm_pageout_object_terminate(object); in vm_object_reap()
1511 } else if (object->resident_page_count) { in vm_object_reap()
1516 vm_object_reap_pages(object, REAP_REAP); in vm_object_reap()
1518 assert(vm_page_queue_empty(&object->memq)); in vm_object_reap()
1519 assert(object->paging_in_progress == 0); in vm_object_reap()
1520 assert(object->activity_in_progress == 0); in vm_object_reap()
1521 assert(os_ref_get_count_raw(&object->ref_count) == 0); in vm_object_reap()
1529 vm_object_unlock(object); in vm_object_reap()
1531 vm_object_lock(object); in vm_object_reap()
1535 VM_OBJECT_SET_TERMINATING(object, FALSE); in vm_object_reap()
1536 vm_object_paging_begin(object); in vm_object_reap()
1537 vm_object_paging_end(object); in vm_object_reap()
1538 vm_object_unlock(object); in vm_object_reap()
1540 object->shadow = VM_OBJECT_NULL; in vm_object_reap()
1544 btlog_erase(vm_object_tracking_btlog, object); in vm_object_reap()
1548 vm_object_lock_destroy(object); in vm_object_reap()
1552 zfree(vm_object_zone, object); in vm_object_reap()
1553 object = VM_OBJECT_NULL; in vm_object_reap()
1591 vm_object_t object, in vm_object_reap_pages() argument
1621 if (object->set_cache_attr) { in vm_object_reap_pages()
1634 object->set_cache_attr = FALSE; in vm_object_reap_pages()
1638 if (vm_page_queue_empty(&object->memq)) { in vm_object_reap_pages()
1649 next = (vm_page_t)vm_page_queue_first(&object->memq); in vm_object_reap_pages()
1651 while (!vm_page_queue_end(&object->memq, (vm_page_queue_entry_t)next)) { in vm_object_reap_pages()
1688 object->set_cache_attr = TRUE; in vm_object_reap_pages()
1690 vm_page_sleep(object, p, THREAD_UNINT, LCK_SLEEP_DEFAULT); in vm_object_reap_pages()
1779 if ((p->vmp_dirty || p->vmp_precious) && !VMP_ERROR_GET(p) && object->alive) { in vm_object_reap_pages()
1780 assert(!object->internal); in vm_object_reap_pages()
1800 object->set_cache_attr = TRUE; in vm_object_reap_pages()
1802 vm_object_paging_wait(object, THREAD_UNINT); in vm_object_reap_pages()
1832 object->set_cache_attr = TRUE; in vm_object_reap_pages()
1839 vm_object_t object) in vm_object_reap_async() argument
1841 vm_object_lock_assert_exclusive(object); in vm_object_reap_async()
1848 queue_enter(&vm_object_reaper_queue, object, in vm_object_reap_async()
1861 vm_object_t object, shadow_object; in vm_object_reaper_thread() local
1867 object, in vm_object_reaper_thread()
1872 vm_object_lock(object); in vm_object_reaper_thread()
1874 assert(object->terminating); in vm_object_reaper_thread()
1875 assert(!object->alive); in vm_object_reaper_thread()
1885 vm_object_paging_wait(object, THREAD_UNINT); in vm_object_reaper_thread()
1888 object->pageout ? VM_OBJECT_NULL : object->shadow; in vm_object_reaper_thread()
1890 vm_object_reap(object); in vm_object_reaper_thread()
1892 object = VM_OBJECT_NULL; in vm_object_reaper_thread()
1948 vm_object_t object, in vm_object_destroy() argument
1953 if (object == VM_OBJECT_NULL) { in vm_object_destroy()
1966 vm_object_lock(object); in vm_object_destroy()
1976 if (object->named && in vm_object_destroy()
1977 os_ref_get_count_raw(&object->ref_count) > 2 && in vm_object_destroy()
1978 object->pager != NULL && in vm_object_destroy()
1979 vnode_pager_get_forced_unmount(object->pager, &forced_unmount) == KERN_SUCCESS && in vm_object_destroy()
1982 …ect %p refs %d pager %p (no forced unmount)\n", object, os_ref_get_count_raw(&object->ref_count), … in vm_object_destroy()
1985 vm_object_t, object, in vm_object_destroy()
1986 int, os_ref_get_count_raw(&object->ref_count), in vm_object_destroy()
1987 memory_object_t, object->pager); in vm_object_destroy()
1990 if (object->fbdp_tracked) { in vm_object_destroy()
1991 if (os_ref_get_count_raw(&object->ref_count) > 2 && !system_inshutdown) { in vm_object_destroy()
1993 …829401 object %p refs %d pager %p (tracked)\n", object, os_ref_get_count_raw(&object->ref_count), … in vm_object_destroy()
1996 VM_OBJECT_SET_FBDP_TRACKED(object, false); in vm_object_destroy()
2000 VM_OBJECT_SET_NO_PAGER_REASON(object, reason); in vm_object_destroy()
2002 VM_OBJECT_SET_CAN_PERSIST(object, FALSE); in vm_object_destroy()
2003 VM_OBJECT_SET_NAMED(object, FALSE); in vm_object_destroy()
2005 VM_OBJECT_SET_ALIVE(object, FALSE); in vm_object_destroy()
2009 if (object->object_is_shared_cache && in vm_object_destroy()
2010 object->pager != NULL && in vm_object_destroy()
2011 object->pager->mo_pager_ops == &shared_region_pager_ops) { in vm_object_destroy()
2012 OSAddAtomic(-object->resident_page_count, &shared_region_pagers_resident_count); in vm_object_destroy()
2016 old_pager = object->pager; in vm_object_destroy()
2017 object->pager = MEMORY_OBJECT_NULL; in vm_object_destroy()
2019 memory_object_control_disable(&object->pager_control); in vm_object_destroy()
2027 vm_object_paging_wait(object, THREAD_UNINT); in vm_object_destroy()
2028 vm_object_unlock(object); in vm_object_destroy()
2043 vm_object_deallocate(object); in vm_object_destroy()
2130 vm_object_t object, in page_is_paged_out() argument
2133 if (object->internal && in page_is_paged_out()
2134 object->alive && in page_is_paged_out()
2135 !object->terminating && in page_is_paged_out()
2136 object->pager_ready) { in page_is_paged_out()
2137 if (vm_object_compressor_pager_state_get(object, offset) in page_is_paged_out()
2180 vm_object_t object, argument
2232 if ((m = vm_page_lookup(object, offset)) != VM_PAGE_NULL) {
2251 if ((flags & DEACTIVATE_KILL) && (object->internal)) {
2314 vm_object_compressor_pager_state_clr(object, offset);
2318 assert(!object->all_reusable);
2320 object->reusable_page_count++;
2321 assert(object->resident_page_count >= object->reusable_page_count);
2360 vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, dwp_start, dw_count);
2373 if (page_is_paged_out(object, offset)) {
2381 if ((flags & DEACTIVATE_KILL) && (object->internal)) {
2382 vm_object_compressor_pager_state_clr(object, offset);
2411 vm_page_do_delayed_work(object, VM_KERN_MEMORY_NONE, dwp_start, dw_count);
2443 vm_object_t object; local
2463 object = orig_object;
2471 while (object && CHUNK_NOT_COMPLETE(chunk_state)) {
2472 vm_object_paging_begin(object);
2474 deactivate_pages_in_object(object, offset, length, flags, &chunk_state, pfc, pmap, pmap_offset);
2476 vm_object_paging_end(object);
2484 tmp_object = object->shadow;
2489 offset += object->vo_shadow_offset;
2493 if (object != orig_object) {
2494 vm_object_unlock(object);
2497 object = tmp_object;
2500 if (object && object != orig_object) {
2501 vm_object_unlock(object);
2517 vm_object_t object, argument
2568 object->internal &&
2569 object->vo_size != 0 &&
2570 object->vo_size == size &&
2571 object->reusable_page_count == 0) {
2578 if ((reusable_page || all_reusable) && object->all_reusable) {
2594 if (size > PAGE_SIZE && (!kill_page || !object->shadow)) {
2595 if (kill_page && object->internal) {
2610 length = deactivate_a_chunk(object, offset, size, flags,
2620 if (!object->all_reusable) {
2623 object->all_reusable = TRUE;
2624 assert(object->reusable_page_count == 0);
2626 reusable = object->resident_page_count;
2639 vm_object_t object, argument
2648 #define VM_OBJECT_REUSE_PAGE(object, m, reused) \ argument
2652 assert((object)->reusable_page_count <= \
2653 (object)->resident_page_count); \
2654 assert((object)->reusable_page_count > 0); \
2655 (object)->reusable_page_count--; \
2675 vm_object_lock_assert_exclusive(object);
2677 if (object->all_reusable) {
2679 object);
2680 assert(object->reusable_page_count == 0);
2681 object->all_reusable = FALSE;
2682 if (end_offset - start_offset == object->vo_size ||
2685 reused = object->resident_page_count;
2688 vm_page_queue_iterate(&object->memq, m, vmp_listq) {
2692 object->reusable_page_count++;
2693 assert(object->resident_page_count >= object->reusable_page_count);
2701 } else if (object->resident_page_count >
2707 if (object->reusable_page_count == 0) {
2710 m = vm_page_lookup(object, cur_offset);
2711 VM_OBJECT_REUSE_PAGE(object, m, reused);
2715 vm_page_queue_iterate(&object->memq, m, vmp_listq) {
2716 if (object->reusable_page_count == 0) {
2723 VM_OBJECT_REUSE_PAGE(object, m, reused);
2740 vm_object_t object, argument
2747 if (!object->internal) {
2754 if (object->shadow != NULL ||
2755 object->vo_copy != NULL) {
2762 if (end - start > object->vo_size) {
2766 if (object->terminating || !object->alive) {
2798 vm_object_t object, argument
2804 vm_object_lock_assert_exclusive(object);
2805 ret = vm_object_zero_preflight(object, cur_offset, end_offset);
2811 vm_page_t m = vm_page_lookup(object, cur_offset);
2814 vm_page_sleep(object, m, THREAD_UNINT, LCK_SLEEP_DEFAULT);
2816 ret = vm_object_zero_preflight(object, cur_offset, end_offset);
2829 if (page_is_paged_out(object, cur_offset)) {
2830 vm_object_compressor_pager_state_clr(object, cur_offset);
2871 vm_object_t object, argument
2879 vm_object_pmap_protect_options(object, offset, size, pmap,
2886 vm_object_t object, argument
2900 if (object == VM_OBJECT_NULL) {
2922 vm_object_lock(object);
2924 if (object->phys_contiguous) {
2926 vm_object_unlock(object);
2936 phys_start = object->vo_shadow_offset + offset_in_object;
2939 assert(phys_end <= object->vo_shadow_offset + object->vo_size);
2940 vm_object_unlock(object);
2962 assert(object->internal);
2965 if (ptoa_64(object->resident_page_count) > size_in_object / 2 && pmap != PMAP_NULL) {
2966 vm_object_unlock(object);
2976 …x\n", pmap, (uint64_t)pmap_start, pmap_start + size, prot, offset, size, object, offset_in_object,…
2987 if (ptoa_64(object->resident_page_count / 4) < size_in_object) {
2993 vm_page_queue_iterate(&object->memq, p, vmp_listq) {
3041 p = vm_page_lookup(object, target_off);
3093 next_object = object->shadow;
3095 offset_in_object += object->vo_shadow_offset;
3096 offset += object->vo_shadow_offset;
3098 vm_object_unlock(object);
3099 object = next_object;
3115 vm_object_unlock(object);
3444 vm_object_t object, /* IN */ argument
3452 if (object == VM_OBJECT_NULL) {
3458 vm_object_lock(object);
3460 copy_strategy = object->copy_strategy;
3471 vm_object_reference_locked(object);
3472 VM_OBJECT_SET_SHADOWED(object, TRUE);
3473 vm_object_unlock(object);
3487 vm_object_unlock(object);
3491 vm_object_unlock(object);
3980 vm_object_t *object, /* IN/OUT */ argument
3988 source = *object;
4100 *object = result;
4190 vm_object_t object, argument
4198 if (object != VM_OBJECT_NULL) {
4199 vm_object_lock(object);
4200 assert(object->internal);
4201 assert(object->pager_created);
4202 assert(!object->pager_initialized);
4203 assert(!object->pager_ready);
4204 assert(object->pager_trusted);
4206 object = vm_object_allocate(size);
4207 assert(object != VM_OBJECT_NULL);
4208 vm_object_lock(object);
4209 VM_OBJECT_SET_INTERNAL(object, FALSE);
4210 VM_OBJECT_SET_PAGER_TRUSTED(object, FALSE);
4212 object->copy_strategy = MEMORY_OBJECT_COPY_INVALID;
4219 control = memory_object_control_allocate(object);
4222 assert(!object->pager_ready);
4223 assert(!object->pager_initialized);
4224 assert(object->pager == NULL);
4225 assert(object->pager_control == NULL);
4232 VM_OBJECT_SET_PAGER_CREATED(object, TRUE);
4233 object->pager = pager;
4234 object->pager_control = control;
4235 VM_OBJECT_SET_PAGER_READY(object, FALSE);
4237 vm_object_unlock(object);
4244 object->pager_control,
4247 vm_object_lock(object);
4249 VM_OBJECT_SET_NAMED(object, TRUE);
4251 if (object->internal) {
4252 VM_OBJECT_SET_PAGER_READY(object, TRUE);
4253 vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY);
4256 VM_OBJECT_SET_PAGER_INITIALIZED(object, TRUE);
4257 vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_INIT);
4259 vm_object_unlock(object);
4261 return object;
4280 vm_object_t object) argument
4285 assert(!is_kernel_object(object));
4291 vm_object_paging_begin(object);
4292 if (object->pager_created) {
4297 while (!object->pager_initialized) {
4298 vm_object_sleep(object,
4302 vm_object_paging_end(object);
4306 if ((uint32_t) (object->vo_size / PAGE_SIZE) !=
4307 (object->vo_size / PAGE_SIZE)) {
4311 object,
4312 (uint64_t) object->vo_size,
4315 vm_object_paging_end(object);
4324 VM_OBJECT_SET_PAGER_CREATED(object, TRUE);
4325 VM_OBJECT_SET_PAGER_TRUSTED(object, TRUE);
4326 object->paging_offset = 0;
4328 vm_object_unlock(object);
4339 assert((uint32_t) (object->vo_size / PAGE_SIZE) ==
4340 (object->vo_size / PAGE_SIZE));
4342 (memory_object_size_t) object->vo_size,
4347 object, (uint64_t) object->vo_size);
4358 object,
4359 object->vo_size,
4361 if (pager_object != object) {
4362 …ig_object: %p, orig_object size: 0x%llx)", pager, pager_object, object, (uint64_t) object->vo_size…
4370 vm_object_lock(object);
4375 vm_object_paging_end(object);
4380 vm_object_t object, argument
4384 vm_object_lock_assert_held(object);
4386 if (object->internal &&
4387 object->pager != NULL &&
4388 !object->terminating &&
4389 object->alive) {
4390 return vm_compressor_pager_state_get(object->pager,
4391 offset + object->paging_offset);
4399 vm_object_t object, argument
4403 vm_object_lock_assert_exclusive(object);
4404 if (object->internal &&
4405 object->pager != NULL &&
4406 !object->terminating &&
4407 object->alive) {
4408 num_pages_cleared = vm_compressor_pager_state_clr(object->pager,
4409 offset + object->paging_offset);
4411 vm_compressor_pager_count(object->pager,
4414 object);
4417 (object->purgable != VM_PURGABLE_DENY || object->vo_ledger_tag)) {
4420 vm_object_owner_compressed_update(object, -num_pages_cleared);
4437 void vm_object_do_collapse_compressor(vm_object_t object,
4441 vm_object_t object, argument
4449 vm_object_lock_assert_exclusive(object);
4452 size = object->vo_size;
4459 for (backing_offset = object->vo_shadow_offset;
4460 backing_offset < object->vo_shadow_offset + object->vo_size;
4477 new_offset = backing_offset - object->vo_shadow_offset;
4479 if (new_offset >= object->vo_size) {
4484 if ((vm_page_lookup(object, new_offset) != VM_PAGE_NULL) ||
4485 (vm_compressor_pager_state_get(object->pager,
4487 object->paging_offset)) ==
4506 object->pager,
4507 (new_offset + object->paging_offset),
4527 vm_object_t object, argument
4534 vm_object_lock_assert_exclusive(object);
4537 assert(object->purgable == VM_PURGABLE_DENY);
4540 backing_offset = object->vo_shadow_offset;
4541 size = object->vo_size;
4568 pp = vm_page_lookup(object, new_offset);
4570 if (vm_object_compressor_pager_state_get(object,
4586 vm_page_rename(p, object, new_offset);
4602 object->pager != MEMORY_OBJECT_NULL &&
4605 vm_object_do_collapse_compressor(object, backing_object);
4607 assert((!object->pager_created &&
4608 (object->pager == MEMORY_OBJECT_NULL)) ||
4619 assert(!object->paging_in_progress);
4620 assert(!object->activity_in_progress);
4621 assert(!object->pager_created);
4622 assert(object->pager == NULL);
4623 object->pager = backing_object->pager;
4625 VM_OBJECT_SET_PAGER_CREATED(object, backing_object->pager_created);
4626 object->pager_control = backing_object->pager_control;
4627 VM_OBJECT_SET_PAGER_READY(object, backing_object->pager_ready);
4628 VM_OBJECT_SET_PAGER_INITIALIZED(object, backing_object->pager_initialized);
4629 object->paging_offset =
4631 if (object->pager_control != MEMORY_OBJECT_CONTROL_NULL) {
4632 memory_object_control_collapse(&object->pager_control,
4633 object);
4648 assert(!object->phys_contiguous);
4650 object->shadow = backing_object->shadow;
4651 if (object->shadow) {
4652 assertf(page_aligned(object->vo_shadow_offset),
4654 object, object->vo_shadow_offset);
4658 object->vo_shadow_offset += backing_object->vo_shadow_offset;
4664 object->vo_shadow_offset = 0;
4666 assert((object->shadow == VM_OBJECT_NULL) ||
4667 (object->shadow->vo_copy != backing_object));
4712 vm_object_t object, argument
4720 vm_object_lock_assert_exclusive(object);
4725 assert(!object->phys_contiguous);
4727 object->shadow = backing_object->shadow;
4728 if (object->shadow) {
4729 assertf(page_aligned(object->vo_shadow_offset),
4731 object, object->vo_shadow_offset);
4735 object->vo_shadow_offset += backing_object->vo_shadow_offset;
4738 object->vo_shadow_offset = 0;
4745 if (backing_object->vo_copy == object) {
4796 vm_object_activity_begin(object);
4797 vm_object_unlock(object);
4809 vm_object_lock(object);
4810 vm_object_activity_end(object);
4834 vm_object_t object, argument
4853 if (object == VM_OBJECT_NULL) {
4857 original_object = object;
4869 object = original_object;
4870 vm_object_lock_assert_exclusive(object);
4883 backing_object = object->shadow;
4885 if (object != original_object) {
4886 vm_object_unlock(object);
4900 if (object->paging_in_progress != 0 ||
4901 object->activity_in_progress != 0) {
4903 if (object != original_object) {
4904 vm_object_unlock(object);
4906 object = backing_object;
4924 if (object != original_object) {
4925 vm_object_unlock(object);
4927 object = backing_object;
4940 if (object->purgable != VM_PURGABLE_DENY ||
4944 object, object->purgable,
4947 if (object != original_object) {
4948 vm_object_unlock(object);
4950 object = backing_object;
4968 if (object != original_object) {
4969 vm_object_unlock(object);
4971 object = backing_object;
4991 !object->pager_created
5012 if (object != original_object) {
5013 vm_object_unlock(object);
5026 vm_object_do_collapse(object, backing_object);
5038 if (object != original_object) {
5039 vm_object_unlock(object);
5041 object = backing_object;
5052 object_vcount = object->vo_size >> PAGE_SHIFT;
5053 object_rcount = (vm_object_size_t)object->resident_page_count;
5067 if (object != original_object) {
5068 vm_object_unlock(object);
5070 object = backing_object;
5080 if (object->pager_created) {
5082 if (object != original_object) {
5083 vm_object_unlock(object);
5085 object = backing_object;
5090 backing_offset = object->vo_shadow_offset;
5103 if (object != original_object) {
5104 vm_object_unlock(object);
5106 object = backing_object;
5133 if (object->cow_hint != ~(vm_offset_t)0) {
5134 hint_offset = (vm_object_offset_t)object->cow_hint;
5142 !EXISTS_IN_OBJECT(object, hint_offset, object_rcount)) {
5144 object->cow_hint = (vm_offset_t) hint_offset; /* atomic */
5146 if (object != original_object) {
5147 vm_object_unlock(object);
5149 object = backing_object;
5177 if (offset < object->vo_size &&
5179 !EXISTS_IN_OBJECT(object, offset, rc)) {
5181 object->cow_hint = (vm_offset_t) offset; /* atomic */
5189 if (object != original_object) {
5190 vm_object_unlock(object);
5192 object = backing_object;
5206 (offset + PAGE_SIZE_64 < object->vo_size) ?
5210 !EXISTS_IN_OBJECT(object, offset, object_rcount)) {
5212 object->cow_hint = (vm_offset_t) offset; /* atomic */
5218 if (object != original_object) {
5219 vm_object_unlock(object);
5221 object = backing_object;
5233 if (object != original_object) {
5234 vm_object_unlock(object);
5242 object->cow_hint = (vm_offset_t)0;
5250 vm_object_do_bypass(object, backing_object);
5284 vm_object_t object, argument
5296 if (atop_64(end - start) < (unsigned)object->resident_page_count / 16) {
5300 p = vm_page_lookup(object, start);
5312 p = (vm_page_t) vm_page_queue_first(&object->memq);
5313 while (!vm_page_queue_end(&object->memq, (vm_page_queue_entry_t) p)) {
5439 vm_object_t object, argument
5448 if (!object->private) {
5454 vm_object_lock(object);
5456 if (!object->phys_contiguous) {
5460 vm_object_unlock(object);
5463 base_offset += object->paging_offset;
5466 m = vm_page_lookup(object, base_offset);
5506 vm_page_insert(m, object, base_offset);
5521 object->vo_shadow_offset = (vm_object_offset_t)phys_page << PAGE_SHIFT;
5524 object, (uint64_t)size);
5525 object->vo_size = size;
5527 vm_object_unlock(object);
5539 vm_object_t object; local
5546 object = vm_object_memory_object_associate(pager,
5550 if (object == VM_OBJECT_NULL) {
5555 if (object != VM_OBJECT_NULL) {
5556 vm_object_lock(object);
5557 VM_OBJECT_SET_NAMED(object, TRUE);
5558 while (!object->pager_ready) {
5559 vm_object_sleep(object,
5563 *control = object->pager_control;
5564 vm_object_unlock(object);
5572 vm_object_t object, argument
5586 if (object == VM_OBJECT_NULL) {
5606 vm_object_lock(object);
5607 vm_object_paging_begin(object);
5609 (void)vm_object_update(object,
5612 vm_object_paging_end(object);
5613 vm_object_unlock(object);
5631 vm_object_purge(vm_object_t object, int flags) argument
5637 vm_object_lock_assert_exclusive(object);
5639 if (object->purgable == VM_PURGABLE_DENY) {
5643 assert(object->vo_copy == VM_OBJECT_NULL);
5644 assert(object->copy_strategy == MEMORY_OBJECT_COPY_NONE);
5661 if (object->purgable == VM_PURGABLE_VOLATILE) {
5663 assert(object->resident_page_count >=
5664 object->wired_page_count);
5665 delta = (object->resident_page_count -
5666 object->wired_page_count);
5673 if (object->wired_page_count != 0) {
5675 object->wired_page_count);
5676 OSAddAtomic(-object->wired_page_count,
5679 VM_OBJECT_SET_PURGABLE(object, VM_PURGABLE_EMPTY);
5681 assert(object->purgable == VM_PURGABLE_EMPTY);
5683 object_page_count = object->resident_page_count;
5685 vm_object_reap_pages(object, REAP_PURGEABLE);
5687 if (object->resident_page_count >= object_page_count) {
5690 total_purged_pgcount = object_page_count - object->resident_page_count;
5693 if (object->pager != NULL) {
5696 if (object->activity_in_progress == 0 &&
5697 object->paging_in_progress == 0) {
5707 pgcount = vm_compressor_pager_get_count(object->pager);
5709 pgcount = vm_compressor_pager_reap_pages(object->pager, flags);
5710 vm_compressor_pager_count(object->pager,
5713 object);
5714 vm_object_owner_compressed_update(object,
5718 assert(vm_compressor_pager_get_count(object->pager)
5743 vm_object_lock_assert_exclusive(object);
5748 VM_KERNEL_UNSLIDE_OR_PERM(object), /* purged object */
5834 vm_object_t object, argument
5841 if (object == VM_OBJECT_NULL) {
5848 vm_object_lock_assert_exclusive(object);
5853 old_state = object->purgable;
5859 assert(object->vo_copy == VM_OBJECT_NULL);
5860 assert(object->copy_strategy == MEMORY_OBJECT_COPY_NONE);
5871 object->purgeable_only_by_kernel) {
5881 object->volatile_empty = TRUE;
5884 object->volatile_fault = TRUE;
5893 if (object->volatile_empty) {
5907 VM_OBJECT_SET_PURGABLE(object, new_state);
5912 assert(object->resident_page_count >=
5913 object->wired_page_count);
5914 delta = (object->resident_page_count -
5915 object->wired_page_count);
5923 if (object->wired_page_count != 0) {
5925 object->wired_page_count);
5926 OSAddAtomic(-object->wired_page_count,
5933 assert(object->objq.next != NULL &&
5934 object->objq.prev != NULL);
5941 queue = vm_purgeable_object_remove(object);
5944 if (object->purgeable_when_ripe) {
5957 vm_purgeable_accounting(object, VM_PURGABLE_VOLATILE);
5963 if (object->volatile_fault) {
5967 vm_page_queue_iterate(&object->memq, p, vmp_listq) {
6002 VM_OBJECT_SET_PURGEABLE_WHEN_RIPE(object, FALSE);
6004 VM_OBJECT_SET_PURGEABLE_WHEN_RIPE(object, TRUE);
6007 if (object->purgeable_when_ripe) {
6021 assert(object->resident_page_count >=
6022 object->wired_page_count);
6023 delta = (object->resident_page_count -
6024 object->wired_page_count);
6030 if (object->wired_page_count != 0) {
6031 OSAddAtomic(object->wired_page_count,
6035 VM_OBJECT_SET_PURGABLE(object, new_state);
6038 assert(object->objq.next != NULL);
6039 assert(object->objq.prev != NULL);
6052 assert(object->objq.next != NULL && object->objq.prev != NULL); /* object should be on a queue */
6054 old_queue = vm_purgeable_object_remove(object);
6066 object->purgeable_when_ripe)) {
6071 if (object->purgeable_when_ripe) {
6074 VM_OBJECT_SET_PURGEABLE_WHEN_RIPE(object, purgeable_when_ripe);
6075 if (object->purgeable_when_ripe) {
6083 …vm_purgeable_object_add(object, queue, (*state & VM_VOLATILE_GROUP_MASK) >> VM_VOLATILE_GROUP_SHIF…
6085 vm_purgeable_accounting(object,
6095 if (object->volatile_fault) {
6099 vm_page_queue_iterate(&object->memq, p, vmp_listq) {
6117 assert(object->objq.next != NULL &&
6118 object->objq.prev != NULL);
6120 old_queue = vm_purgeable_object_remove(object);
6122 if (object->purgeable_when_ripe) {
6135 vm_purgeable_accounting(object,
6143 VM_OBJECT_SET_PURGABLE(object, VM_PURGABLE_EMPTY);
6146 (void) vm_object_purge(object, 0);
6147 assert(object->purgable == VM_PURGABLE_EMPTY);
6154 vm_object_lock_assert_exclusive(object);
6161 vm_object_t object, argument
6175 if (object == VM_OBJECT_NULL) {
6184 vm_object_lock_assert_exclusive(object);
6196 if (offset == 0 && (object->vo_size == size)) {
6197 *resident_page_count = object->resident_page_count;
6202 if (object->resident_page_count <= (size >> PAGE_SHIFT)) {
6203 vm_page_queue_iterate(&object->memq, p, vmp_listq) {
6216 p = vm_page_lookup(object, cur_offset);
6253 vm_object_t object) argument
6255 if (object == VM_OBJECT_NULL) {
6259 vm_object_lock(object);
6260 vm_object_reference_locked(object);
6261 vm_object_unlock(object);
6625 vm_object_cluster_size(vm_object_t object, vm_object_offset_t *start, argument
6673 vm_object_lock(object);
6675 if (object->pager == MEMORY_OBJECT_NULL) {
6678 vnode_pager_get_isSSD(object->pager, &isSSD);
6731 if (object->internal) {
6732 object_size = object->vo_size;
6734 vnode_pager_get_object_size(object->pager, &object_size);
6747 if (object->pages_used > object->pages_created) {
6752 object->pages_used = object->pages_created = 0;
6754 if ((sequential_run = object->sequential)) {
6768 if (object->internal && fault_info->user_tag == VM_MEMORY_STACK) {
6783 if (object->pages_created < (20 * (min_ph_size >> PAGE_SHIFT))) {
6795 pre_heat_size = (max_length * (uint64_t)object->pages_used) / object->pages_created;
6833 if (vnode_pager_get_throttle_io_limit(object->pager, &throttle_limit) == KERN_SUCCESS) {
6929 if (vm_object_compressor_pager_state_get(object, offset) == VM_EXTERNAL_STATE_ABSENT) {
6932 if (vm_page_lookup(object, offset) != VM_PAGE_NULL) {
6956 if (vm_object_compressor_pager_state_get(object, offset) == VM_EXTERNAL_STATE_ABSENT) {
6959 if (vm_page_lookup(object, offset) != VM_PAGE_NULL) {
6973 vm_object_unlock(object);
6986 vm_object_t object, argument
6994 vm_object_lock(object);
6997 if (object->phys_contiguous) {
7000 (object->vo_shadow_offset >> PAGE_SHIFT);
7002 vm_object_unlock(object);
7005 vm_object_unlock(object);
7009 if (object->phys_contiguous) {
7010 vm_object_unlock(object);
7015 if ((dst_page = vm_page_lookup(object, offset)) == VM_PAGE_NULL) {
7016 vm_object_unlock(object);
7026 vm_page_sleep(object, dst_page, THREAD_UNINT, LCK_SLEEP_DEFAULT);
7106 vm_page_wakeup(object, dst_page);
7121 vm_object_unlock(object);
7137 vm_object_t object, argument
7146 if (object->resident_page_count == 0) {
7157 vm_object_lock(object);
7159 if (object->phys_contiguous) {
7160 vm_object_unlock(object);
7167 dst_page = vm_page_lookup(object, offset);
7175 vm_page_sleep(object, dst_page, THREAD_UNINT, LCK_SLEEP_DEFAULT);
7203 vm_object_unlock(object);
7226 memory_object_control_t object, argument
7243 pager_object = memory_object_control_to_vm_object(object);
7273 vm_object_lock(vm_object_t object) argument
7275 if (object == vm_pageout_scan_wants_object) {
7280 lck_rw_lock_exclusive(&object->Lock);
7284 vm_object_lock_avoid(vm_object_t object) argument
7286 if (object == vm_pageout_scan_wants_object) {
7294 _vm_object_lock_try(vm_object_t object) argument
7298 retval = lck_rw_try_lock_exclusive(&object->Lock);
7308 vm_object_lock_try(vm_object_t object) argument
7313 if (vm_object_lock_avoid(object) && ml_get_interrupts_enabled() && get_preemption_level() == 0) {
7316 return _vm_object_lock_try(object);
7326 vm_object_lock_check_contended(vm_object_t object) argument
7328 if (object == vm_pageout_scan_wants_object) {
7333 return lck_rw_lock_exclusive_check_contended(&object->Lock);
7337 vm_object_lock_shared(vm_object_t object) argument
7339 if (vm_object_lock_avoid(object)) {
7343 lck_rw_lock_shared(&object->Lock);
7347 vm_object_lock_yield_shared(vm_object_t object) argument
7351 vm_object_lock_assert_shared(object);
7353 force_yield = vm_object_lock_avoid(object);
7355 retval = lck_rw_lock_yield_shared(&object->Lock, force_yield);
7364 vm_object_lock_try_shared(vm_object_t object) argument
7368 if (vm_object_lock_avoid(object)) {
7371 retval = lck_rw_try_lock_shared(&object->Lock);
7379 vm_object_lock_upgrade(vm_object_t object) argument
7383 retval = lck_rw_lock_shared_to_exclusive(&object->Lock);
7393 vm_object_unlock(vm_object_t object) argument
7398 lck_rw_done(&object->Lock);
7408 vm_object_change_wimg_mode(vm_object_t object, unsigned int wimg_mode) argument
7410 vm_object_lock_assert_exclusive(object);
7412 vm_object_paging_only_wait(object, THREAD_UNINT);
7415 .pageq = &object->memq,
7420 object->set_cache_attr = FALSE;
7422 object->set_cache_attr = TRUE;
7425 object->wimg_bits = wimg_mode;
7479 vm_object_t object, uint32_t dirty_budget) argument
7488 assert(object != VM_OBJECT_NULL);
7489 assert(object->internal);
7491 vm_object_lock(object);
7493 if (!object->pager_initialized || object->pager == MEMORY_OBJECT_NULL) {
7494 if (!object->pager_initialized) {
7495 vm_object_collapse(object, (vm_object_offset_t) 0, TRUE);
7497 if (!object->pager_initialized) {
7498 vm_object_compressor_pager_create(object);
7502 if (!object->pager_initialized || object->pager == MEMORY_OBJECT_NULL) {
7503 vm_object_unlock(object);
7518 if (object->paging_in_progress || object->activity_in_progress) {
7519 vm_object_unlock(object);
7531 while (curr_offset < object->vo_size) {
7532 curr_offset = vm_compressor_pager_next_compressed(object->pager, curr_offset);
7538 …retval = vm_compressor_pager_relocate(object->pager, curr_offset, &(freezer_context_global.freezer…
7561 obj_resident_page_count_snapshot = object->resident_page_count;
7563 vm_object_activity_begin(object);
7565 …while ((obj_resident_page_count_snapshot--) && !vm_page_queue_empty(&object->memq) && paged_out_co…
7566 p = (vm_page_t)vm_page_queue_first(&object->memq);
7568 KDBG_DEBUG(0xe0430004 | DBG_FUNC_START, object, local_freed);
7575 KDBG_DEBUG(0xe0430004 | DBG_FUNC_END, object, local_freed, 1);
7577 vm_page_queue_remove(&object->memq, p, vmp_listq);
7578 vm_page_queue_enter(&object->memq, p, vmp_listq);
7606 KDBG_DEBUG(0xe0430004 | DBG_FUNC_END, object, local_freed, 2);
7625 vm_page_queue_remove(&object->memq, p, vmp_listq);
7626 vm_page_queue_enter(&object->memq, p, vmp_listq);
7635 vm_object_activity_begin(object);
7637 vm_object_unlock(object);
7660 KDBG_DEBUG(0xe0430004 | DBG_FUNC_END, object, local_freed);
7667 vm_object_lock(object);
7679 vm_object_activity_end(object);
7681 vm_object_unlock(object);
7695 vm_object_t object) argument
7706 assert(object != VM_OBJECT_NULL );
7708 vm_object_lock(object);
7710 if (!object->internal ||
7711 object->terminating ||
7712 !object->alive) {
7713 vm_object_unlock(object);
7717 if (!object->pager_initialized || object->pager == MEMORY_OBJECT_NULL) {
7718 if (!object->pager_initialized) {
7719 vm_object_collapse(object, (vm_object_offset_t) 0, TRUE);
7721 if (!object->pager_initialized) {
7722 vm_object_compressor_pager_create(object);
7726 if (!object->pager_initialized || object->pager == MEMORY_OBJECT_NULL) {
7727 vm_object_unlock(object);
7733 next = (vm_page_t)vm_page_queue_first(&object->memq);
7735 while (!vm_page_queue_end(&object->memq, (vm_page_queue_entry_t)next)) {
7768 vm_object_unlock(object);
7772 vm_object_lock(object);
7822 vm_object_unlock(object);
7862 vm_object_t object; local
7923 object = io_upl->map_object->shadow;
7925 object = io_upl->map_object;
7929 if (!object || !object->pager ||
7930 vnode_pager_get_object_devvp(object->pager, (uintptr_t *)&devvp) != KERN_SUCCESS) {
8033 vm_object_t object = (vm_object_t)((uintptr_t)wait_event - wait_reason); local
8034 waitinfo->context = VM_KERNEL_ADDRPERM(object);
8045 vm_object_t object, argument
8054 vm_object_lock_assert_exclusive(object);
8083 …KDBG_FILTERED(VMDBG_CODE(DBG_VM_OBJECT_SLEEP) | DBG_FUNC_START, VM_KERNEL_ADDRHIDE(object), reason…
8085 vm_object_set_wanted(object, reason);
8086 wait_event = (event_t)((uintptr_t)object + (uintptr_t)reason);
8087 wr = lck_rw_sleep(&object->Lock, LCK_SLEEP_PROMOTED_PRI | action, wait_event, interruptible);
8089 …KDBG_FILTERED(VMDBG_CODE(DBG_VM_OBJECT_SLEEP) | DBG_FUNC_END, VM_KERNEL_ADDRHIDE(object), reason, …
8095 vm_object_paging_wait(vm_object_t object, wait_interrupt_t interruptible) argument
8098 vm_object_lock_assert_exclusive(object);
8099 while (object->paging_in_progress != 0 ||
8100 object->activity_in_progress != 0) {
8101 wr = vm_object_sleep((object),
8113 vm_object_paging_only_wait(vm_object_t object, wait_interrupt_t interruptible) argument
8116 vm_object_lock_assert_exclusive(object);
8117 while (object->paging_in_progress != 0) {
8118 wr = vm_object_sleep(object,
8130 vm_object_paging_throttle_wait(vm_object_t object, wait_interrupt_t interruptible) argument
8133 vm_object_lock_assert_exclusive(object);
8139 while (object->paging_in_progress >= vm_object_pagein_throttle) {
8140 wr = vm_object_sleep(object,
8152 vm_object_mapping_wait(vm_object_t object, wait_interrupt_t interruptible) argument
8155 vm_object_lock_assert_exclusive(object);
8156 while (object->mapping_in_progress) {
8157 wr = vm_object_sleep(object,
8170 vm_object_t object, argument
8173 vm_object_lock_assert_exclusive(object);
8176 if (vm_object_wanted(object, reason)) {
8177 thread_wakeup((event_t)((uintptr_t)object + (uintptr_t)reason));
8179 object->all_wanted &= ~(1 << reason);
8200 vm_page_sleep(vm_object_t object, vm_page_t m, wait_interrupt_t interruptible, lck_sleep_action_t a… argument
8204 …KDBG_FILTERED((VMDBG_CODE(DBG_VM_PAGE_SLEEP)) | DBG_FUNC_START, VM_KERNEL_ADDRHIDE(object), m->vmp…
8206 …if (object->io_tracking && ((m->vmp_busy == TRUE) || (m->vmp_cleaning == TRUE) || VM_PAGE_WIRED(m)…
8210 vm_page_handle_prio_inversion(object, m);
8216 ret = vm_page_sleep_with_inheritor(&object->Lock, action, (event_t)m, interruptible);
8218 ret = lck_rw_sleep(&object->Lock, LCK_SLEEP_PROMOTED_PRI | action, (event_t)m, interruptible);
8220 …KDBG_FILTERED((VMDBG_CODE(DBG_VM_PAGE_SLEEP)) | DBG_FUNC_END, VM_KERNEL_ADDRHIDE(object), m->vmp_o…
8225 vm_page_wakeup(vm_object_t object, vm_page_t m) argument
8231 if (object != VM_OBJECT_NULL) {
8232 vm_object_lock_assert_exclusive(object);
8237 VM_KERNEL_ADDRHIDE(object), m->vmp_offset,
8245 vm_page_wakeup_done(__assert_only vm_object_t object, vm_page_t m) argument
8247 assert(object);
8249 vm_object_lock_assert_exclusive(object);
8252 VM_KERNEL_ADDRHIDE(object), m->vmp_offset,
8255 vm_page_wakeup(object, m);
8265 vm_page_wakeup_done_with_inheritor(vm_object_t object __unused, vm_page_t m, page_worker_token_t *t…
8268 assert(object);
8270 vm_object_lock_assert_exclusive(object);
8275 VM_KERNEL_ADDRHIDE(object), VM_KERNEL_ADDRHIDE(m),
8288 vm_page_wakeup_done(object, m);
8474 vm_object_t object, argument
8483 vm_object_lock(object);
8484 *access_tracking_p = object->access_tracking;
8486 *access_tracking_reads_p = object->access_tracking_reads;
8489 *access_tracking_writes_p = object->access_tracking_writes;
8491 object->access_tracking = access_tracking;
8492 object->access_tracking_reads = 0;
8493 object->access_tracking_writes = 0;
8494 vm_object_unlock(object);
8497 vm_object_pmap_protect_options(object,
8499 object->vo_size,
8511 vm_object_t object, argument
8520 assert(object->shadow == VM_OBJECT_NULL);
8528 *do_footprint = !object->vo_no_footprint;
8530 if (!object->internal) {
8531 switch (object->vo_ledger_tag) {
8568 __FUNCTION__, object, object->vo_ledger_tag);
8573 assert(object->internal);
8574 switch (object->vo_ledger_tag) {
8581 assert(object->purgable != VM_PURGABLE_DENY);
8665 __FUNCTION__, object, object->vo_ledger_tag);
8671 vm_object_t object, argument
8692 vm_object_lock_assert_exclusive(object);
8697 object->purgable == VM_PURGABLE_DENY) {
8701 if (!object->internal
8702 && !memory_object_is_vnode_pager(object->pager)) {
8709 new_owner = VM_OBJECT_OWNER(object);
8713 new_ledger_tag = object->vo_ledger_tag;
8722 if (object->internal &&
8723 object->vo_ledger_tag == VM_LEDGER_TAG_NONE &&
8724 object->purgable == VM_PURGABLE_DENY) {
8731 if (object->resident_page_count != 0 ||
8732 object->pager != NULL ||
8733 object->pager_created ||
8734 os_ref_get_count_raw(&object->ref_count) != 1 ||
8735 object->vo_owner != TASK_NULL ||
8736 object->copy_strategy != MEMORY_OBJECT_COPY_NONE ||
8749 object->purgable != VM_PURGABLE_DENY &&
8763 assert(object->copy_strategy != MEMORY_OBJECT_COPY_SYMMETRIC);
8764 assert(object->shadow == VM_OBJECT_NULL);
8765 if (object->internal) {
8766 assert(object->copy_strategy == MEMORY_OBJECT_COPY_NONE);
8767 assert(object->vo_copy == VM_OBJECT_NULL);
8770 old_ledger_tag = object->vo_ledger_tag;
8771 old_no_footprint = object->vo_no_footprint;
8772 old_owner = VM_OBJECT_OWNER(object);
8776 vm_object_t, object,
8783 int, VM_OBJECT_ID(object));
8786 resident_count = object->resident_page_count - object->wired_page_count;
8787 wired_count = object->wired_page_count;
8788 if (object->internal) {
8789 compressed_count = vm_compressor_pager_get_count(object->pager);
8806 vm_object_ledger_tag_ledgers(object,
8814 if (object->internal) {
8815 if (object->purgable == VM_PURGABLE_VOLATILE ||
8816 object->purgable == VM_PURGABLE_EMPTY) {
8832 if (object->purgable != VM_PURGABLE_VOLATILE &&
8833 object->purgable != VM_PURGABLE_EMPTY) {
8868 if (object->purgable != VM_PURGABLE_VOLATILE &&
8869 object->purgable != VM_PURGABLE_EMPTY) {
8901 vm_object_t, object,
8907 queue_remove(&old_owner->task_objq, object,
8909 switch (object->purgable) {
8958 object->vo_ledger_tag = new_ledger_tag;
8959 object->vo_owner = new_owner;
8960 object->vo_no_footprint = new_no_footprint;
8988 vm_object_ledger_tag_ledgers(object,
8996 if (object->internal) {
8997 if (object->purgable == VM_PURGABLE_VOLATILE ||
8998 object->purgable == VM_PURGABLE_EMPTY) {
9014 if (object->purgable != VM_PURGABLE_VOLATILE &&
9015 object->purgable != VM_PURGABLE_EMPTY) {
9050 if (object->purgable != VM_PURGABLE_VOLATILE &&
9051 object->purgable != VM_PURGABLE_EMPTY) {
9083 vm_object_t, object,
9087 queue_enter(&new_owner->task_objq, object,
9089 switch (object->purgable) {
9117 vm_object_t object; local
9140 for (object = (vm_object_t) queue_first(&task->task_objq);
9141 !queue_end(&task->task_objq, (queue_entry_t) object);
9142 object = next_object) {
9150 next_object = (vm_object_t) queue_next(&object->task_objq);
9153 assert(object->vo_purgeable_volatilizer == NULL);
9155 assert(object->vo_owner == task);
9156 if (!vm_object_lock_try(object)) {
9162 assert(VM_OBJECT_OWNER(object) != kernel_task);
9164 object,
9165 object->vo_ledger_tag, /* unchanged */
9170 assert(object->vo_owner == VM_OBJECT_OWNER_DISOWNED);
9171 vm_object_unlock(object);
9200 vm_object_t object, argument
9205 vm_object_lock_assert_exclusive(object);
9210 if (object->internal) {
9214 if (!object->vo_ledger_tag) {
9223 owner = VM_OBJECT_OWNER(object);
9234 vm_object_ledger_tag_ledgers(object,