91 static int vnode_pager_input_smlfs(vm_object_t object, vm_page_t m);
92 static int vnode_pager_input_old(vm_object_t object, vm_page_t m);
154 	vm_object_t object;  in vnode_create_vobject()  local
161 	object = vp->v_object;  in vnode_create_vobject()
162 	if (object != NULL)  in vnode_create_vobject()
174 	object = vnode_pager_alloc(vp, size, 0, 0, td->td_ucred);  in vnode_create_vobject()
181 	VM_OBJECT_RLOCK(object);  in vnode_create_vobject()
182 	last = refcount_release(&object->ref_count);  in vnode_create_vobject()
183 	VM_OBJECT_RUNLOCK(object);  in vnode_create_vobject()
241 	vm_object_t object;  in vnode_pager_alloc()  local
254 	object = vp->v_object;  in vnode_pager_alloc()
256 	if (object == NULL) {  in vnode_pager_alloc()
260 		object = vm_object_allocate(OBJT_VNODE,  in vnode_pager_alloc()
263 		object->un_pager.vnp.vnp_size = size;  in vnode_pager_alloc()
264 		object->un_pager.vnp.writemappings = 0;  in vnode_pager_alloc()
265 		object->domain.dr_policy = vnode_domainset;  in vnode_pager_alloc()
266 		object->handle = handle;  in vnode_pager_alloc()
268 			VM_OBJECT_WLOCK(object);  in vnode_pager_alloc()
269 			vm_object_set_flag(object, OBJ_SIZEVNLOCK);  in vnode_pager_alloc()
270 			VM_OBJECT_WUNLOCK(object);  in vnode_pager_alloc()
278 			VM_OBJECT_WLOCK(object);  in vnode_pager_alloc()
279 			KASSERT(object->ref_count == 1,  in vnode_pager_alloc()
280 			    ("leaked ref %p %d", object, object->ref_count));  in vnode_pager_alloc()
281 			object->type = OBJT_DEAD;  in vnode_pager_alloc()
282 			refcount_init(&object->ref_count, 0);  in vnode_pager_alloc()
283 			VM_OBJECT_WUNLOCK(object);  in vnode_pager_alloc()
284 			vm_object_destroy(object);  in vnode_pager_alloc()
287 		vp->v_object = object;  in vnode_pager_alloc()
291 		vm_object_reference(object);  in vnode_pager_alloc()
293 		if ((object->flags & OBJ_COLORED) == 0) {  in vnode_pager_alloc()
294 			VM_OBJECT_WLOCK(object);  in vnode_pager_alloc()
295 			vm_object_color(object, 0);  in vnode_pager_alloc()
296 			VM_OBJECT_WUNLOCK(object);  in vnode_pager_alloc()
300 	return (object);  in vnode_pager_alloc()
307 vnode_pager_dealloc(vm_object_t object)  in vnode_pager_dealloc()  argument
312 	vp = object->handle;  in vnode_pager_dealloc()
316 	VM_OBJECT_ASSERT_WLOCKED(object);  in vnode_pager_dealloc()
317 	vm_object_pip_wait(object, "vnpdea");  in vnode_pager_dealloc()
318 	refs = object->ref_count;  in vnode_pager_dealloc()
320 	object->handle = NULL;  in vnode_pager_dealloc()
321 	object->type = OBJT_DEAD;  in vnode_pager_dealloc()
323 	if (object->un_pager.vnp.writemappings > 0) {  in vnode_pager_dealloc()
324 		object->un_pager.vnp.writemappings = 0;  in vnode_pager_dealloc()
342 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_dealloc()
345 	VM_OBJECT_WLOCK(object);  in vnode_pager_dealloc()
349 vnode_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,  in vnode_pager_haspage()  argument
352 	struct vnode *vp = object->handle;  in vnode_pager_haspage()
361 	VM_OBJECT_ASSERT_LOCKED(object);  in vnode_pager_haspage()
372 	if (IDX_TO_OFF(pindex) >= object->un_pager.vnp.vnp_size)  in vnode_pager_haspage()
384 	lockstate = VM_OBJECT_DROP(object);  in vnode_pager_haspage()
386 	VM_OBJECT_PICKUP(object, lockstate);  in vnode_pager_haspage()
405 			    roundup2(object->size, pagesperblock),  in vnode_pager_haspage()
408 			    (uintmax_t )object->size));  in vnode_pager_haspage()
411 			if (pindex + *after >= object->size)  in vnode_pager_haspage()
412 				*after = object->size - 1 - pindex;  in vnode_pager_haspage()
484 	vm_object_t object;  in vnode_pager_setsize()  local
488 	if ((object = vp->v_object) == NULL)  in vnode_pager_setsize()
500 	VM_OBJECT_WLOCK(object);  in vnode_pager_setsize()
501 	if (object->type == OBJT_DEAD) {  in vnode_pager_setsize()
502 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_setsize()
505 	KASSERT(object->type == OBJT_VNODE,  in vnode_pager_setsize()
506 	    ("not vnode-backed object %p", object));  in vnode_pager_setsize()
507 	if (nsize == object->un_pager.vnp.vnp_size) {  in vnode_pager_setsize()
511 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_setsize()
515 	if (nsize < object->un_pager.vnp.vnp_size) {  in vnode_pager_setsize()
519 		if (nobjsize < object->size)  in vnode_pager_setsize()
520 			vm_object_page_remove(object, nobjsize, object->size,  in vnode_pager_setsize()
532 		m = vm_page_grab(object, OFF_TO_IDX(nsize), VM_ALLOC_NOCREAT);  in vnode_pager_setsize()
542 	object->un_pager.vnp.vnp_size = nsize;  in vnode_pager_setsize()
544 	atomic_store_64(&object->un_pager.vnp.vnp_size, nsize);  in vnode_pager_setsize()
546 	object->size = nobjsize;  in vnode_pager_setsize()
547 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_setsize()
560 	struct vm_object *object;  in vnode_pager_purge_range()  local
567 	object = vp->v_object;  in vnode_pager_purge_range()
573 	if ((end != 0 && end <= start) || object == NULL)  in vnode_pager_purge_range()
576 	VM_OBJECT_WLOCK(object);  in vnode_pager_purge_range()
579 		vm_object_page_remove(object, pi, piend, 0);  in vnode_pager_purge_range()
584 		m = vm_page_grab(object, pistart, VM_ALLOC_NOCREAT);  in vnode_pager_purge_range()
596 		m = vm_page_grab(object, piend, VM_ALLOC_NOCREAT);  in vnode_pager_purge_range()
604 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_purge_range()
652 vnode_pager_input_smlfs(vm_object_t object, vm_page_t m)  in vnode_pager_input_smlfs()  argument
664 	vp = object->handle;  in vnode_pager_input_smlfs()
682 		if (address >= object->un_pager.vnp.vnp_size) {  in vnode_pager_input_smlfs()
745 vnode_pager_input_old(vm_object_t object, vm_page_t m)  in vnode_pager_input_old()  argument
754 	VM_OBJECT_ASSERT_WLOCKED(object);  in vnode_pager_input_old()
760 	if (IDX_TO_OFF(m->pindex) >= object->un_pager.vnp.vnp_size) {  in vnode_pager_input_old()
764 		if (IDX_TO_OFF(m->pindex) + size > object->un_pager.vnp.vnp_size)  in vnode_pager_input_old()
765 			size = object->un_pager.vnp.vnp_size - IDX_TO_OFF(m->pindex);  in vnode_pager_input_old()
766 		vp = object->handle;  in vnode_pager_input_old()
767 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_input_old()
797 		VM_OBJECT_WLOCK(object);  in vnode_pager_input_old()
818 vnode_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind,  in vnode_pager_getpages()  argument
825 	vp = object->handle;  in vnode_pager_getpages()
833 vnode_pager_getpages_async(vm_object_t object, vm_page_t *m, int count,  in vnode_pager_getpages_async()  argument
839 	vp = object->handle;  in vnode_pager_getpages_async()
879 	vm_object_t object;  in vnode_pager_generic_getpages()  local
896 	object = vp->v_object;  in vnode_pager_generic_getpages()
901 	KASSERT(foff < object->un_pager.vnp.vnp_size,  in vnode_pager_generic_getpages()
928 		VM_OBJECT_WLOCK(object);  in vnode_pager_generic_getpages()
932 			error = vnode_pager_input_old(object, m[i]);  in vnode_pager_generic_getpages()
936 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_generic_getpages()
952 			error = vnode_pager_input_smlfs(object, m[i]);  in vnode_pager_generic_getpages()
986 	if (m[0]->pindex + after >= object->size)  in vnode_pager_generic_getpages()
987 		after = object->size - 1 - m[0]->pindex;  in vnode_pager_generic_getpages()
998 	rahead = min(rahead, object->size - m[count - 1]->pindex);  in vnode_pager_generic_getpages()
1033 		VM_OBJECT_WLOCK(object);  in vnode_pager_generic_getpages()
1043 			p = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);  in vnode_pager_generic_getpages()
1069 		if (!VM_OBJECT_WOWNED(object))  in vnode_pager_generic_getpages()
1070 			VM_OBJECT_WLOCK(object);  in vnode_pager_generic_getpages()
1075 		if (endpindex > object->size)  in vnode_pager_generic_getpages()
1076 			endpindex = object->size;  in vnode_pager_generic_getpages()
1080 			p = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);  in vnode_pager_generic_getpages()
1091 	if (VM_OBJECT_WOWNED(object))  in vnode_pager_generic_getpages()
1092 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_generic_getpages()
1120 	if ((foff + bytecount) > object->un_pager.vnp.vnp_size)  in vnode_pager_generic_getpages()
1121 		bytecount = object->un_pager.vnp.vnp_size - foff;  in vnode_pager_generic_getpages()
1203 	vm_object_t object;  in vnode_pager_generic_getpages_done()  local
1210 	object = bp->b_vp->v_object;  in vnode_pager_generic_getpages_done()
1234 		VM_OBJECT_WLOCK(object);  in vnode_pager_generic_getpages_done()
1239 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_generic_getpages_done()
1244 	VM_OBJECT_RLOCK(object);  in vnode_pager_generic_getpages_done()
1254 		if (nextoff <= object->un_pager.vnp.vnp_size) {  in vnode_pager_generic_getpages_done()
1272 			    object->un_pager.vnp.vnp_size - tfoff);  in vnode_pager_generic_getpages_done()
1274 			    object->un_pager.vnp.vnp_size - tfoff)) == 0,  in vnode_pager_generic_getpages_done()
1281 	VM_OBJECT_RUNLOCK(object);  in vnode_pager_generic_getpages_done()
1295 vnode_pager_putpages(vm_object_t object, vm_page_t *m, int count,  in vnode_pager_putpages()  argument
1320 	vp = object->handle;  in vnode_pager_putpages()
1321 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_putpages()
1325 	VM_OBJECT_WLOCK(object);  in vnode_pager_putpages()
1359 	vm_object_t object;  in vnode_pager_generic_putpages()  local
1370 	object = vp->v_object;  in vnode_pager_generic_putpages()
1401 	VM_OBJECT_RLOCK(object);  in vnode_pager_generic_putpages()
1402 	if (maxsize + poffset > object->un_pager.vnp.vnp_size) {  in vnode_pager_generic_putpages()
1403 		if (object->un_pager.vnp.vnp_size > poffset) {  in vnode_pager_generic_putpages()
1404 			maxsize = object->un_pager.vnp.vnp_size - poffset;  in vnode_pager_generic_putpages()
1430 	VM_OBJECT_RUNLOCK(object);  in vnode_pager_generic_putpages()
1612 vnode_pager_update_writecount(vm_object_t object, vm_offset_t start,  in vnode_pager_update_writecount()  argument
1618 	VM_OBJECT_WLOCK(object);  in vnode_pager_update_writecount()
1619 	if (object->type != OBJT_VNODE) {  in vnode_pager_update_writecount()
1620 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_update_writecount()
1623 	old_wm = object->un_pager.vnp.writemappings;  in vnode_pager_update_writecount()
1624 	object->un_pager.vnp.writemappings += (vm_ooffset_t)end - start;  in vnode_pager_update_writecount()
1625 	vp = object->handle;  in vnode_pager_update_writecount()
1626 	if (old_wm == 0 && object->un_pager.vnp.writemappings != 0) {  in vnode_pager_update_writecount()
1631 	} else if (old_wm != 0 && object->un_pager.vnp.writemappings == 0) {  in vnode_pager_update_writecount()
1637 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_update_writecount()
1641 vnode_pager_release_writecount(vm_object_t object, vm_offset_t start,  in vnode_pager_release_writecount()  argument
1648 	VM_OBJECT_WLOCK(object);  in vnode_pager_release_writecount()
1654 	if (object->type != OBJT_VNODE) {  in vnode_pager_release_writecount()
1655 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_release_writecount()
1664 	if (object->un_pager.vnp.writemappings != inc) {  in vnode_pager_release_writecount()
1665 		object->un_pager.vnp.writemappings -= inc;  in vnode_pager_release_writecount()
1666 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_release_writecount()
1670 	vp = object->handle;  in vnode_pager_release_writecount()
1672 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_release_writecount()
1683 	vnode_pager_update_writecount(object, end, start);  in vnode_pager_release_writecount()
1691 vnode_pager_getvp(vm_object_t object, struct vnode **vpp, bool *vp_heldp)  in vnode_pager_getvp()  argument
1693 	*vpp = object->handle;  in vnode_pager_getvp()