Lines Matching refs:fs
178 static void vm_fault_dontneed(const struct faultstate *fs, vm_offset_t vaddr,
180 static void vm_fault_prefault(const struct faultstate *fs, vm_offset_t addra,
245 vm_fault_unlock_map(struct faultstate *fs) in vm_fault_unlock_map() argument
248 if (fs->lookup_still_valid) { in vm_fault_unlock_map()
249 vm_map_lookup_done(fs->map, fs->entry); in vm_fault_unlock_map()
250 fs->lookup_still_valid = false; in vm_fault_unlock_map()
255 vm_fault_unlock_vp(struct faultstate *fs) in vm_fault_unlock_vp() argument
258 if (fs->vp != NULL) { in vm_fault_unlock_vp()
259 vput(fs->vp); in vm_fault_unlock_vp()
260 fs->vp = NULL; in vm_fault_unlock_vp()
265 vm_fault_deallocate(struct faultstate *fs) in vm_fault_deallocate() argument
268 vm_fault_page_release(&fs->m_cow); in vm_fault_deallocate()
269 vm_fault_page_release(&fs->m); in vm_fault_deallocate()
270 vm_object_pip_wakeup(fs->object); in vm_fault_deallocate()
271 if (fs->object != fs->first_object) { in vm_fault_deallocate()
272 VM_OBJECT_WLOCK(fs->first_object); in vm_fault_deallocate()
273 vm_fault_page_free(&fs->first_m); in vm_fault_deallocate()
274 VM_OBJECT_WUNLOCK(fs->first_object); in vm_fault_deallocate()
275 vm_object_pip_wakeup(fs->first_object); in vm_fault_deallocate()
277 vm_object_deallocate(fs->first_object); in vm_fault_deallocate()
278 vm_fault_unlock_map(fs); in vm_fault_deallocate()
279 vm_fault_unlock_vp(fs); in vm_fault_deallocate()
283 vm_fault_unlock_and_deallocate(struct faultstate *fs) in vm_fault_unlock_and_deallocate() argument
286 VM_OBJECT_UNLOCK(fs->object); in vm_fault_unlock_and_deallocate()
287 vm_fault_deallocate(fs); in vm_fault_unlock_and_deallocate()
291 vm_fault_dirty(struct faultstate *fs, vm_page_t m) in vm_fault_dirty() argument
295 if (((fs->prot & VM_PROT_WRITE) == 0 && in vm_fault_dirty()
296 (fs->fault_flags & VM_FAULT_DIRTY) == 0) || in vm_fault_dirty()
302 need_dirty = ((fs->fault_type & VM_PROT_WRITE) != 0 && in vm_fault_dirty()
303 (fs->fault_flags & VM_FAULT_WIRE) == 0) || in vm_fault_dirty()
304 (fs->fault_flags & VM_FAULT_DIRTY) != 0; in vm_fault_dirty()
325 if ((fs->entry->eflags & MAP_ENTRY_NOSYNC) != 0) in vm_fault_dirty()
337 vm_fault_soft_fast(struct faultstate *fs) in vm_fault_soft_fast() argument
347 MPASS(fs->vp == NULL); in vm_fault_soft_fast()
356 m = vm_page_lookup_unlocked(fs->first_object, fs->first_pindex); in vm_fault_soft_fast()
358 ((fs->prot & VM_PROT_WRITE) != 0 && vm_page_busied(m))) { in vm_fault_soft_fast()
359 VM_OBJECT_WLOCK(fs->first_object); in vm_fault_soft_fast()
363 vaddr = fs->vaddr; in vm_fault_soft_fast()
365 VM_OBJECT_RLOCK(fs->first_object); in vm_fault_soft_fast()
372 if (m->object != fs->first_object || m->pindex != fs->first_pindex) in vm_fault_soft_fast()
375 vm_object_busy(fs->first_object); in vm_fault_soft_fast()
378 ((fs->prot & VM_PROT_WRITE) != 0 && vm_page_busied(m))) in vm_fault_soft_fast()
386 rounddown2(vaddr, pagesizes[m_super->psind]) >= fs->entry->start && in vm_fault_soft_fast()
387 roundup2(vaddr + 1, pagesizes[m_super->psind]) <= fs->entry->end && in vm_fault_soft_fast()
389 (pagesizes[m_super->psind] - 1)) && !fs->wired && in vm_fault_soft_fast()
390 pmap_ps_enabled(fs->map->pmap)) { in vm_fault_soft_fast()
392 if ((fs->prot & VM_PROT_WRITE) != 0) { in vm_fault_soft_fast()
400 if ((fs->first_object->flags & OBJ_UNMANAGED) == 0) in vm_fault_soft_fast()
409 fs->fault_type |= VM_PROT_WRITE; in vm_fault_soft_fast()
413 if (pmap_enter(fs->map->pmap, vaddr, m_map, fs->prot, fs->fault_type | in vm_fault_soft_fast()
414 PMAP_ENTER_NOSLEEP | (fs->wired ? PMAP_ENTER_WIRED : 0), psind) != in vm_fault_soft_fast()
417 if (fs->m_hold != NULL) { in vm_fault_soft_fast()
418 (*fs->m_hold) = m; in vm_fault_soft_fast()
421 if (psind == 0 && !fs->wired) in vm_fault_soft_fast()
422 vm_fault_prefault(fs, vaddr, PFBAK, PFFOR, true); in vm_fault_soft_fast()
423 VM_OBJECT_RUNLOCK(fs->first_object); in vm_fault_soft_fast()
424 vm_fault_dirty(fs, m); in vm_fault_soft_fast()
425 vm_object_unbusy(fs->first_object); in vm_fault_soft_fast()
426 vm_map_lookup_done(fs->map, fs->entry); in vm_fault_soft_fast()
430 vm_object_unbusy(fs->first_object); in vm_fault_soft_fast()
432 if (!VM_OBJECT_TRYUPGRADE(fs->first_object)) { in vm_fault_soft_fast()
433 VM_OBJECT_RUNLOCK(fs->first_object); in vm_fault_soft_fast()
434 VM_OBJECT_WLOCK(fs->first_object); in vm_fault_soft_fast()
440 vm_fault_restore_map_lock(struct faultstate *fs) in vm_fault_restore_map_lock() argument
443 VM_OBJECT_ASSERT_WLOCKED(fs->first_object); in vm_fault_restore_map_lock()
444 MPASS(blockcount_read(&fs->first_object->paging_in_progress) > 0); in vm_fault_restore_map_lock()
446 if (!vm_map_trylock_read(fs->map)) { in vm_fault_restore_map_lock()
447 VM_OBJECT_WUNLOCK(fs->first_object); in vm_fault_restore_map_lock()
448 vm_map_lock_read(fs->map); in vm_fault_restore_map_lock()
449 VM_OBJECT_WLOCK(fs->first_object); in vm_fault_restore_map_lock()
451 fs->lookup_still_valid = true; in vm_fault_restore_map_lock()
486 vm_fault_populate(struct faultstate *fs) in vm_fault_populate() argument
494 MPASS(fs->object == fs->first_object); in vm_fault_populate()
495 VM_OBJECT_ASSERT_WLOCKED(fs->first_object); in vm_fault_populate()
496 MPASS(blockcount_read(&fs->first_object->paging_in_progress) > 0); in vm_fault_populate()
497 MPASS(fs->first_object->backing_object == NULL); in vm_fault_populate()
498 MPASS(fs->lookup_still_valid); in vm_fault_populate()
500 pager_first = OFF_TO_IDX(fs->entry->offset); in vm_fault_populate()
501 pager_last = pager_first + atop(fs->entry->end - fs->entry->start) - 1; in vm_fault_populate()
502 vm_fault_unlock_map(fs); in vm_fault_populate()
503 vm_fault_unlock_vp(fs); in vm_fault_populate()
515 rv = vm_pager_populate(fs->first_object, fs->first_pindex, in vm_fault_populate()
516 fs->fault_type, fs->entry->max_protection, &pager_first, in vm_fault_populate()
519 VM_OBJECT_ASSERT_WLOCKED(fs->first_object); in vm_fault_populate()
525 vm_fault_restore_map_lock(fs); in vm_fault_populate()
526 if (fs->map->timestamp != fs->map_generation) in vm_fault_populate()
535 MPASS(fs->first_pindex <= pager_last); in vm_fault_populate()
536 MPASS(fs->first_pindex >= pager_first); in vm_fault_populate()
537 MPASS(pager_last < fs->first_object->size); in vm_fault_populate()
539 vm_fault_restore_map_lock(fs); in vm_fault_populate()
540 bdry_idx = MAP_ENTRY_SPLIT_BOUNDARY_INDEX(fs->entry); in vm_fault_populate()
541 if (fs->map->timestamp != fs->map_generation) { in vm_fault_populate()
543 vm_fault_populate_cleanup(fs->first_object, pager_first, in vm_fault_populate()
546 m = vm_page_lookup(fs->first_object, pager_first); in vm_fault_populate()
547 if (m != fs->m) in vm_fault_populate()
562 m = vm_page_lookup(fs->first_object, pager_first); in vm_fault_populate()
564 VM_OBJECT_WUNLOCK(fs->first_object); in vm_fault_populate()
565 vaddr = fs->entry->start + IDX_TO_OFF(pager_first) - in vm_fault_populate()
566 fs->entry->offset; in vm_fault_populate()
570 (uintmax_t)fs->entry->start, (uintmax_t)pager_first, in vm_fault_populate()
571 (uintmax_t)fs->entry->offset, (uintmax_t)vaddr)); in vm_fault_populate()
575 rv = pmap_enter(fs->map->pmap, vaddr, m, fs->prot, in vm_fault_populate()
576 fs->fault_type | (fs->wired ? PMAP_ENTER_WIRED : 0) | in vm_fault_populate()
578 VM_OBJECT_WLOCK(fs->first_object); in vm_fault_populate()
584 if ((fs->fault_flags & VM_FAULT_WIRE) != 0) { in vm_fault_populate()
588 if (fs->m_hold != NULL) { in vm_fault_populate()
589 *fs->m_hold = m + (fs->first_pindex - pager_first); in vm_fault_populate()
590 vm_page_wire(*fs->m_hold); in vm_fault_populate()
602 map_first = OFF_TO_IDX(fs->entry->offset); in vm_fault_populate()
604 vm_fault_populate_cleanup(fs->first_object, pager_first, in vm_fault_populate()
608 map_last = map_first + atop(fs->entry->end - fs->entry->start) - 1; in vm_fault_populate()
610 vm_fault_populate_cleanup(fs->first_object, map_last + 1, in vm_fault_populate()
614 for (pidx = pager_first, m = vm_page_lookup(fs->first_object, pidx); in vm_fault_populate()
617 vaddr = fs->entry->start + IDX_TO_OFF(pidx) - fs->entry->offset; in vm_fault_populate()
622 !pmap_ps_enabled(fs->map->pmap) || fs->wired)) in vm_fault_populate()
628 vm_fault_dirty(fs, &m[i]); in vm_fault_populate()
630 VM_OBJECT_WUNLOCK(fs->first_object); in vm_fault_populate()
631 rv = pmap_enter(fs->map->pmap, vaddr, m, fs->prot, fs->fault_type | in vm_fault_populate()
632 (fs->wired ? PMAP_ENTER_WIRED : 0), psind); in vm_fault_populate()
645 MPASS(!fs->wired); in vm_fault_populate()
647 rv = pmap_enter(fs->map->pmap, vaddr + ptoa(i), in vm_fault_populate()
648 &m[i], fs->prot, fs->fault_type, 0); in vm_fault_populate()
653 VM_OBJECT_WLOCK(fs->first_object); in vm_fault_populate()
655 if ((fs->fault_flags & VM_FAULT_WIRE) != 0 && in vm_fault_populate()
656 m[i].pindex == fs->first_pindex) in vm_fault_populate()
660 if (fs->m_hold != NULL && in vm_fault_populate()
661 m[i].pindex == fs->first_pindex) { in vm_fault_populate()
662 (*fs->m_hold) = &m[i]; in vm_fault_populate()
774 vm_fault_object_ensure_wlocked(struct faultstate *fs) in vm_fault_object_ensure_wlocked() argument
776 if (fs->object == fs->first_object) in vm_fault_object_ensure_wlocked()
777 VM_OBJECT_ASSERT_WLOCKED(fs->object); in vm_fault_object_ensure_wlocked()
779 if (!fs->can_read_lock) { in vm_fault_object_ensure_wlocked()
780 VM_OBJECT_ASSERT_WLOCKED(fs->object); in vm_fault_object_ensure_wlocked()
784 if (VM_OBJECT_WOWNED(fs->object)) in vm_fault_object_ensure_wlocked()
787 if (VM_OBJECT_TRYUPGRADE(fs->object)) in vm_fault_object_ensure_wlocked()
794 vm_fault_lock_vnode(struct faultstate *fs, bool objlocked) in vm_fault_lock_vnode() argument
799 if (fs->object->type != OBJT_VNODE) in vm_fault_lock_vnode()
801 vp = fs->object->handle; in vm_fault_lock_vnode()
802 if (vp == fs->vp) { in vm_fault_lock_vnode()
811 vm_fault_unlock_vp(fs); in vm_fault_lock_vnode()
825 fs->vp = vp; in vm_fault_lock_vnode()
831 vm_fault_unlock_and_deallocate(fs); in vm_fault_lock_vnode()
833 vm_fault_deallocate(fs); in vm_fault_lock_vnode()
836 fs->vp = vp; in vm_fault_lock_vnode()
847 vm_fault_readahead(struct faultstate *fs) in vm_fault_readahead() argument
852 KASSERT(fs->lookup_still_valid, ("map unlocked")); in vm_fault_readahead()
853 era = fs->entry->read_ahead; in vm_fault_readahead()
854 behavior = vm_map_entry_behavior(fs->entry); in vm_fault_readahead()
859 if (fs->vaddr == fs->entry->next_read) in vm_fault_readahead()
860 vm_fault_dontneed(fs, fs->vaddr, nera); in vm_fault_readahead()
861 } else if (fs->vaddr == fs->entry->next_read) { in vm_fault_readahead()
876 vm_fault_dontneed(fs, fs->vaddr, nera); in vm_fault_readahead()
888 fs->entry->read_ahead = nera; in vm_fault_readahead()
895 vm_fault_lookup(struct faultstate *fs) in vm_fault_lookup() argument
899 KASSERT(!fs->lookup_still_valid, in vm_fault_lookup()
901 result = vm_map_lookup(&fs->map, fs->vaddr, fs->fault_type | in vm_fault_lookup()
902 VM_PROT_FAULT_LOOKUP, &fs->entry, &fs->first_object, in vm_fault_lookup()
903 &fs->first_pindex, &fs->prot, &fs->wired); in vm_fault_lookup()
905 vm_fault_unlock_vp(fs); in vm_fault_lookup()
909 fs->map_generation = fs->map->timestamp; in vm_fault_lookup()
911 if (fs->entry->eflags & MAP_ENTRY_NOFAULT) { in vm_fault_lookup()
913 __func__, (u_long)fs->vaddr); in vm_fault_lookup()
916 if (fs->entry->eflags & MAP_ENTRY_IN_TRANSITION && in vm_fault_lookup()
917 fs->entry->wiring_thread != curthread) { in vm_fault_lookup()
918 vm_map_unlock_read(fs->map); in vm_fault_lookup()
919 vm_map_lock(fs->map); in vm_fault_lookup()
920 if (vm_map_lookup_entry(fs->map, fs->vaddr, &fs->entry) && in vm_fault_lookup()
921 (fs->entry->eflags & MAP_ENTRY_IN_TRANSITION)) { in vm_fault_lookup()
922 vm_fault_unlock_vp(fs); in vm_fault_lookup()
923 fs->entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP; in vm_fault_lookup()
924 vm_map_unlock_and_wait(fs->map, 0); in vm_fault_lookup()
926 vm_map_unlock(fs->map); in vm_fault_lookup()
930 MPASS((fs->entry->eflags & MAP_ENTRY_GUARD) == 0); in vm_fault_lookup()
932 if (fs->wired) in vm_fault_lookup()
933 fs->fault_type = fs->prot | (fs->fault_type & VM_PROT_COPY); in vm_fault_lookup()
935 KASSERT((fs->fault_flags & VM_FAULT_WIRE) == 0, in vm_fault_lookup()
937 fs->lookup_still_valid = true; in vm_fault_lookup()
943 vm_fault_relookup(struct faultstate *fs) in vm_fault_relookup() argument
950 if (!vm_map_trylock_read(fs->map)) in vm_fault_relookup()
953 fs->lookup_still_valid = true; in vm_fault_relookup()
954 if (fs->map->timestamp == fs->map_generation) in vm_fault_relookup()
957 result = vm_map_lookup_locked(&fs->map, fs->vaddr, fs->fault_type, in vm_fault_relookup()
958 &fs->entry, &retry_object, &retry_pindex, &retry_prot, in vm_fault_relookup()
959 &fs->wired); in vm_fault_relookup()
969 if (retry_object != fs->first_object || in vm_fault_relookup()
970 retry_pindex != fs->first_pindex) in vm_fault_relookup()
981 fs->prot &= retry_prot; in vm_fault_relookup()
982 fs->fault_type &= retry_prot; in vm_fault_relookup()
983 if (fs->prot == 0) in vm_fault_relookup()
987 KASSERT(fs->wired || (fs->fault_flags & VM_FAULT_WIRE) == 0, in vm_fault_relookup()
994 vm_fault_cow(struct faultstate *fs) in vm_fault_cow() argument
998 KASSERT(fs->object != fs->first_object, in vm_fault_cow()
1014 fs->object->shadow_count == 1 && fs->object->ref_count == 1 && in vm_fault_cow()
1018 fs->object->handle == NULL && (fs->object->flags & OBJ_ANON) != 0 && in vm_fault_cow()
1022 (is_first_object_locked = VM_OBJECT_TRYWLOCK(fs->first_object)) && in vm_fault_cow()
1023 fs->object == fs->first_object->backing_object && in vm_fault_cow()
1024 VM_OBJECT_TRYWLOCK(fs->object)) { in vm_fault_cow()
1030 vm_page_remove_xbusy(fs->m); in vm_fault_cow()
1031 vm_page_replace(fs->m, fs->first_object, fs->first_pindex, in vm_fault_cow()
1032 fs->first_m); in vm_fault_cow()
1033 vm_page_dirty(fs->m); in vm_fault_cow()
1038 vm_reserv_rename(fs->m, fs->first_object, fs->object, in vm_fault_cow()
1039 OFF_TO_IDX(fs->first_object->backing_object_offset)); in vm_fault_cow()
1041 VM_OBJECT_WUNLOCK(fs->object); in vm_fault_cow()
1042 VM_OBJECT_WUNLOCK(fs->first_object); in vm_fault_cow()
1043 fs->first_m = fs->m; in vm_fault_cow()
1044 fs->m = NULL; in vm_fault_cow()
1048 VM_OBJECT_WUNLOCK(fs->first_object); in vm_fault_cow()
1052 pmap_copy_page(fs->m, fs->first_m); in vm_fault_cow()
1053 vm_page_valid(fs->first_m); in vm_fault_cow()
1054 if (fs->wired && (fs->fault_flags & VM_FAULT_WIRE) == 0) { in vm_fault_cow()
1055 vm_page_wire(fs->first_m); in vm_fault_cow()
1056 vm_page_unwire(fs->m, PQ_INACTIVE); in vm_fault_cow()
1062 fs->m_cow = fs->m; in vm_fault_cow()
1063 fs->m = NULL; in vm_fault_cow()
1082 vm_page_assert_xbusied(fs->m_cow); in vm_fault_cow()
1083 if ((fs->first_object->flags & OBJ_ONEMAPPING) == 0) in vm_fault_cow()
1084 pmap_remove_all(fs->m_cow); in vm_fault_cow()
1087 vm_object_pip_wakeup(fs->object); in vm_fault_cow()
1092 fs->object = fs->first_object; in vm_fault_cow()
1093 fs->pindex = fs->first_pindex; in vm_fault_cow()
1094 fs->m = fs->first_m; in vm_fault_cow()
1100 vm_fault_next(struct faultstate *fs) in vm_fault_next() argument
1104 if (fs->object == fs->first_object || !fs->can_read_lock) in vm_fault_next()
1105 VM_OBJECT_ASSERT_WLOCKED(fs->object); in vm_fault_next()
1107 VM_OBJECT_ASSERT_LOCKED(fs->object); in vm_fault_next()
1119 if (fs->object == fs->first_object) { in vm_fault_next()
1120 fs->first_m = fs->m; in vm_fault_next()
1121 fs->m = NULL; in vm_fault_next()
1122 } else if (fs->m != NULL) { in vm_fault_next()
1123 if (!vm_fault_object_ensure_wlocked(fs)) { in vm_fault_next()
1124 fs->can_read_lock = false; in vm_fault_next()
1125 vm_fault_unlock_and_deallocate(fs); in vm_fault_next()
1128 vm_fault_page_free(&fs->m); in vm_fault_next()
1135 next_object = fs->object->backing_object; in vm_fault_next()
1138 MPASS(fs->first_m != NULL); in vm_fault_next()
1139 KASSERT(fs->object != next_object, ("object loop %p", next_object)); in vm_fault_next()
1140 if (fs->can_read_lock) in vm_fault_next()
1145 if (fs->object != fs->first_object) in vm_fault_next()
1146 vm_object_pip_wakeup(fs->object); in vm_fault_next()
1147 fs->pindex += OFF_TO_IDX(fs->object->backing_object_offset); in vm_fault_next()
1148 VM_OBJECT_UNLOCK(fs->object); in vm_fault_next()
1149 fs->object = next_object; in vm_fault_next()
1155 vm_fault_zerofill(struct faultstate *fs) in vm_fault_zerofill() argument
1162 if (fs->object != fs->first_object) { in vm_fault_zerofill()
1163 vm_object_pip_wakeup(fs->object); in vm_fault_zerofill()
1164 fs->object = fs->first_object; in vm_fault_zerofill()
1165 fs->pindex = fs->first_pindex; in vm_fault_zerofill()
1167 MPASS(fs->first_m != NULL); in vm_fault_zerofill()
1168 MPASS(fs->m == NULL); in vm_fault_zerofill()
1169 fs->m = fs->first_m; in vm_fault_zerofill()
1170 fs->first_m = NULL; in vm_fault_zerofill()
1175 if ((fs->m->flags & PG_ZERO) == 0) { in vm_fault_zerofill()
1176 pmap_zero_page(fs->m); in vm_fault_zerofill()
1181 vm_page_valid(fs->m); in vm_fault_zerofill()
1189 vm_fault_allocate_oom(struct faultstate *fs) in vm_fault_allocate_oom() argument
1193 vm_fault_unlock_and_deallocate(fs); in vm_fault_allocate_oom()
1196 if (!fs->oom_started) { in vm_fault_allocate_oom()
1197 fs->oom_started = true; in vm_fault_allocate_oom()
1198 getmicrotime(&fs->oom_start_time); in vm_fault_allocate_oom()
1203 timevalsub(&now, &fs->oom_start_time); in vm_fault_allocate_oom()
1212 fs->oom_started = false; in vm_fault_allocate_oom()
1220 vm_fault_allocate(struct faultstate *fs) in vm_fault_allocate() argument
1225 if ((fs->object->flags & OBJ_SIZEVNLOCK) != 0) { in vm_fault_allocate()
1226 res = vm_fault_lock_vnode(fs, true); in vm_fault_allocate()
1232 if (fs->pindex >= fs->object->size) { in vm_fault_allocate()
1233 vm_fault_unlock_and_deallocate(fs); in vm_fault_allocate()
1237 if (fs->object == fs->first_object && in vm_fault_allocate()
1238 (fs->first_object->flags & OBJ_POPULATE) != 0 && in vm_fault_allocate()
1239 fs->first_object->shadow_count == 0) { in vm_fault_allocate()
1240 res = vm_fault_populate(fs); in vm_fault_allocate()
1245 vm_fault_unlock_and_deallocate(fs); in vm_fault_allocate()
1270 dset = fs->object->domain.dr_policy; in vm_fault_allocate()
1275 vm_object_color(fs->object, atop(fs->vaddr) - fs->pindex); in vm_fault_allocate()
1277 if (!vm_pager_can_alloc_page(fs->object, fs->pindex)) { in vm_fault_allocate()
1278 vm_fault_unlock_and_deallocate(fs); in vm_fault_allocate()
1281 fs->m = vm_page_alloc(fs->object, fs->pindex, in vm_fault_allocate()
1284 if (fs->m == NULL) { in vm_fault_allocate()
1285 if (vm_fault_allocate_oom(fs)) in vm_fault_allocate()
1289 fs->oom_started = false; in vm_fault_allocate()
1300 vm_fault_getpages(struct faultstate *fs, int *behindp, int *aheadp) in vm_fault_getpages() argument
1315 e_start = fs->entry->start; in vm_fault_getpages()
1316 e_end = fs->entry->end; in vm_fault_getpages()
1317 behavior = vm_map_entry_behavior(fs->entry); in vm_fault_getpages()
1330 if (fs->nera == -1 && !P_KILLED(curproc)) in vm_fault_getpages()
1331 fs->nera = vm_fault_readahead(fs); in vm_fault_getpages()
1339 vm_fault_unlock_map(fs); in vm_fault_getpages()
1341 status = vm_fault_lock_vnode(fs, false); in vm_fault_getpages()
1345 KASSERT(fs->vp == NULL || !fs->map->system_map, in vm_fault_getpages()
1352 if (fs->nera == -1 || behavior == MAP_ENTRY_BEHAV_RANDOM || in vm_fault_getpages()
1358 if (fs->nera > 0) { in vm_fault_getpages()
1360 ahead = fs->nera; in vm_fault_getpages()
1372 cluster_offset = fs->pindex % VM_FAULT_READ_DEFAULT; in vm_fault_getpages()
1374 atop(fs->vaddr - e_start)); in vm_fault_getpages()
1377 ahead = ulmin(ahead, atop(e_end - fs->vaddr) - 1); in vm_fault_getpages()
1381 rv = vm_pager_get_pages(fs->object, &fs->m, 1, behindp, aheadp); in vm_fault_getpages()
1393 VM_OBJECT_WLOCK(fs->object); in vm_fault_getpages()
1394 vm_fault_page_free(&fs->m); in vm_fault_getpages()
1395 vm_fault_unlock_and_deallocate(fs); in vm_fault_getpages()
1417 vm_fault_busy_sleep(struct faultstate *fs) in vm_fault_busy_sleep() argument
1424 vm_page_aflag_set(fs->m, PGA_REFERENCED); in vm_fault_busy_sleep()
1425 if (fs->object != fs->first_object) { in vm_fault_busy_sleep()
1426 vm_fault_page_release(&fs->first_m); in vm_fault_busy_sleep()
1427 vm_object_pip_wakeup(fs->first_object); in vm_fault_busy_sleep()
1429 vm_object_pip_wakeup(fs->object); in vm_fault_busy_sleep()
1430 vm_fault_unlock_map(fs); in vm_fault_busy_sleep()
1431 if (fs->m != vm_page_lookup(fs->object, fs->pindex) || in vm_fault_busy_sleep()
1432 !vm_page_busy_sleep(fs->m, "vmpfw", 0)) in vm_fault_busy_sleep()
1433 VM_OBJECT_UNLOCK(fs->object); in vm_fault_busy_sleep()
1435 vm_object_deallocate(fs->first_object); in vm_fault_busy_sleep()
1447 vm_fault_object(struct faultstate *fs, int *behindp, int *aheadp) in vm_fault_object() argument
1452 if (fs->object == fs->first_object || !fs->can_read_lock) in vm_fault_object()
1453 VM_OBJECT_ASSERT_WLOCKED(fs->object); in vm_fault_object()
1455 VM_OBJECT_ASSERT_LOCKED(fs->object); in vm_fault_object()
1462 if ((fs->object->flags & OBJ_DEAD) != 0) { in vm_fault_object()
1463 dead = fs->object->type == OBJT_DEAD; in vm_fault_object()
1464 vm_fault_unlock_and_deallocate(fs); in vm_fault_object()
1474 fs->m = vm_page_lookup(fs->object, fs->pindex); in vm_fault_object()
1475 if (fs->m != NULL) { in vm_fault_object()
1476 if (!vm_page_tryxbusy(fs->m)) { in vm_fault_object()
1477 vm_fault_busy_sleep(fs); in vm_fault_object()
1486 if (vm_page_all_valid(fs->m)) { in vm_fault_object()
1487 VM_OBJECT_UNLOCK(fs->object); in vm_fault_object()
1497 if (fs->m == NULL && (vm_fault_object_needs_getpages(fs->object) || in vm_fault_object()
1498 fs->object == fs->first_object)) { in vm_fault_object()
1499 if (!vm_fault_object_ensure_wlocked(fs)) { in vm_fault_object()
1500 fs->can_read_lock = false; in vm_fault_object()
1501 vm_fault_unlock_and_deallocate(fs); in vm_fault_object()
1504 res = vm_fault_allocate(fs); in vm_fault_object()
1514 if (vm_fault_object_needs_getpages(fs->object)) { in vm_fault_object()
1525 VM_OBJECT_UNLOCK(fs->object); in vm_fault_object()
1526 res = vm_fault_getpages(fs, behindp, aheadp); in vm_fault_object()
1528 VM_OBJECT_WLOCK(fs->object); in vm_fault_object()
1539 struct faultstate fs; in vm_fault() local
1550 fs.vp = NULL; in vm_fault()
1551 fs.vaddr = vaddr; in vm_fault()
1552 fs.m_hold = m_hold; in vm_fault()
1553 fs.fault_flags = fault_flags; in vm_fault()
1554 fs.map = map; in vm_fault()
1555 fs.lookup_still_valid = false; in vm_fault()
1556 fs.oom_started = false; in vm_fault()
1557 fs.nera = -1; in vm_fault()
1558 fs.can_read_lock = true; in vm_fault()
1563 fs.fault_type = fault_type; in vm_fault()
1569 rv = vm_fault_lookup(&fs); in vm_fault()
1583 if (fs.vp == NULL /* avoid locked vnode leak */ && in vm_fault()
1584 (fs.entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) == 0 && in vm_fault()
1585 (fs.fault_flags & (VM_FAULT_WIRE | VM_FAULT_DIRTY)) == 0) { in vm_fault()
1586 res = vm_fault_soft_fast(&fs); in vm_fault()
1588 VM_OBJECT_ASSERT_UNLOCKED(fs.first_object); in vm_fault()
1591 VM_OBJECT_ASSERT_WLOCKED(fs.first_object); in vm_fault()
1593 VM_OBJECT_WLOCK(fs.first_object); in vm_fault()
1605 vm_object_reference_locked(fs.first_object); in vm_fault()
1606 vm_object_pip_add(fs.first_object, 1); in vm_fault()
1608 fs.m_cow = fs.m = fs.first_m = NULL; in vm_fault()
1613 fs.object = fs.first_object; in vm_fault()
1614 fs.pindex = fs.first_pindex; in vm_fault()
1616 if ((fs.entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) != 0) { in vm_fault()
1617 res = vm_fault_allocate(&fs); in vm_fault()
1635 KASSERT(fs.m == NULL, in vm_fault()
1636 ("page still set %p at loop start", fs.m)); in vm_fault()
1638 res = vm_fault_object(&fs, &behind, &ahead); in vm_fault()
1667 res_next = vm_fault_next(&fs); in vm_fault()
1673 if ((fs.fault_flags & VM_FAULT_NOFILL) != 0) { in vm_fault()
1674 if (fs.first_object == fs.object) in vm_fault()
1675 vm_fault_page_free(&fs.first_m); in vm_fault()
1676 vm_fault_unlock_and_deallocate(&fs); in vm_fault()
1679 VM_OBJECT_UNLOCK(fs.object); in vm_fault()
1680 vm_fault_zerofill(&fs); in vm_fault()
1691 vm_page_assert_xbusied(fs.m); in vm_fault()
1692 VM_OBJECT_ASSERT_UNLOCKED(fs.object); in vm_fault()
1699 if (fs.object != fs.first_object) { in vm_fault()
1703 if ((fs.fault_type & (VM_PROT_COPY | VM_PROT_WRITE)) != 0) { in vm_fault()
1704 vm_fault_cow(&fs); in vm_fault()
1715 fs.prot &= ~VM_PROT_WRITE; in vm_fault()
1723 if (!fs.lookup_still_valid) { in vm_fault()
1724 rv = vm_fault_relookup(&fs); in vm_fault()
1726 vm_fault_deallocate(&fs); in vm_fault()
1732 VM_OBJECT_ASSERT_UNLOCKED(fs.object); in vm_fault()
1741 fs.entry->next_read = vaddr + ptoa(ahead) + PAGE_SIZE; in vm_fault()
1747 vm_page_assert_xbusied(fs.m); in vm_fault()
1748 KASSERT(vm_page_all_valid(fs.m), in vm_fault()
1749 ("vm_fault: page %p partially invalid", fs.m)); in vm_fault()
1751 vm_fault_dirty(&fs, fs.m); in vm_fault()
1759 pmap_enter(fs.map->pmap, vaddr, fs.m, fs.prot, in vm_fault()
1760 fs.fault_type | (fs.wired ? PMAP_ENTER_WIRED : 0), 0); in vm_fault()
1761 if (faultcount != 1 && (fs.fault_flags & VM_FAULT_WIRE) == 0 && in vm_fault()
1762 fs.wired == 0) in vm_fault()
1763 vm_fault_prefault(&fs, vaddr, in vm_fault()
1771 if ((fs.fault_flags & VM_FAULT_WIRE) != 0) in vm_fault()
1772 vm_page_wire(fs.m); in vm_fault()
1774 vm_page_activate(fs.m); in vm_fault()
1775 if (fs.m_hold != NULL) { in vm_fault()
1776 (*fs.m_hold) = fs.m; in vm_fault()
1777 vm_page_wire(fs.m); in vm_fault()
1779 vm_page_xunbusy(fs.m); in vm_fault()
1780 fs.m = NULL; in vm_fault()
1785 vm_fault_deallocate(&fs); in vm_fault()
1790 if (racct_enable && fs.object->type == OBJT_VNODE) { in vm_fault()
1792 if ((fs.fault_type & (VM_PROT_COPY | VM_PROT_WRITE)) != 0) { in vm_fault()
1823 vm_fault_dontneed(const struct faultstate *fs, vm_offset_t vaddr, int ahead) in vm_fault_dontneed() argument
1832 VM_OBJECT_ASSERT_UNLOCKED(fs->object); in vm_fault_dontneed()
1833 first_object = fs->first_object; in vm_fault_dontneed()
1842 (entry = fs->entry)->start < end) { in vm_fault_dontneed()
1847 pmap_advise(fs->map->pmap, start, end, MADV_DONTNEED); in vm_fault_dontneed()
1887 vm_fault_prefault(const struct faultstate *fs, vm_offset_t addra, in vm_fault_prefault() argument
1898 pmap = fs->map->pmap; in vm_fault_prefault()
1902 entry = fs->entry; in vm_fault_prefault()