93 static int vnode_pager_input_smlfs(vm_object_t object, vm_page_t m);
94 static int vnode_pager_input_old(vm_object_t object, vm_page_t m);
156 	vm_object_t object;  in vnode_create_vobject()  local
164 	object = vp->v_object;  in vnode_create_vobject()
165 	if (object != NULL)  in vnode_create_vobject()
178 	object = vnode_pager_alloc(vp, size, 0, 0, td->td_ucred);  in vnode_create_vobject()
185 	VM_OBJECT_RLOCK(object);  in vnode_create_vobject()
186 	last = refcount_release(&object->ref_count);  in vnode_create_vobject()
187 	VM_OBJECT_RUNLOCK(object);  in vnode_create_vobject()
245 	vm_object_t object;  in vnode_pager_alloc()  local
258 	object = vp->v_object;  in vnode_pager_alloc()
260 	if (object == NULL) {  in vnode_pager_alloc()
264 		object = vm_object_allocate(OBJT_VNODE,  in vnode_pager_alloc()
267 		object->un_pager.vnp.vnp_size = size;  in vnode_pager_alloc()
268 		object->un_pager.vnp.writemappings = 0;  in vnode_pager_alloc()
269 		object->domain.dr_policy = vnode_domainset;  in vnode_pager_alloc()
270 		object->handle = handle;  in vnode_pager_alloc()
272 			VM_OBJECT_WLOCK(object);  in vnode_pager_alloc()
273 			vm_object_set_flag(object, OBJ_SIZEVNLOCK);  in vnode_pager_alloc()
274 			VM_OBJECT_WUNLOCK(object);  in vnode_pager_alloc()
282 			VM_OBJECT_WLOCK(object);  in vnode_pager_alloc()
283 			KASSERT(object->ref_count == 1,  in vnode_pager_alloc()
284 			    ("leaked ref %p %d", object, object->ref_count));  in vnode_pager_alloc()
285 			object->type = OBJT_DEAD;  in vnode_pager_alloc()
286 			refcount_init(&object->ref_count, 0);  in vnode_pager_alloc()
287 			VM_OBJECT_WUNLOCK(object);  in vnode_pager_alloc()
288 			vm_object_destroy(object);  in vnode_pager_alloc()
291 		vp->v_object = object;  in vnode_pager_alloc()
295 		vm_object_reference(object);  in vnode_pager_alloc()
297 		if ((object->flags & OBJ_COLORED) == 0) {  in vnode_pager_alloc()
298 			VM_OBJECT_WLOCK(object);  in vnode_pager_alloc()
299 			vm_object_color(object, 0);  in vnode_pager_alloc()
300 			VM_OBJECT_WUNLOCK(object);  in vnode_pager_alloc()
304 	return (object);  in vnode_pager_alloc()
311 vnode_pager_dealloc(vm_object_t object)  in vnode_pager_dealloc()  argument
316 	vp = object->handle;  in vnode_pager_dealloc()
320 	VM_OBJECT_ASSERT_WLOCKED(object);  in vnode_pager_dealloc()
321 	vm_object_pip_wait(object, "vnpdea");  in vnode_pager_dealloc()
322 	refs = object->ref_count;  in vnode_pager_dealloc()
324 	object->handle = NULL;  in vnode_pager_dealloc()
325 	object->type = OBJT_DEAD;  in vnode_pager_dealloc()
327 	if (object->un_pager.vnp.writemappings > 0) {  in vnode_pager_dealloc()
328 		object->un_pager.vnp.writemappings = 0;  in vnode_pager_dealloc()
346 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_dealloc()
349 	VM_OBJECT_WLOCK(object);  in vnode_pager_dealloc()
353 vnode_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,  in vnode_pager_haspage()  argument
356 	struct vnode *vp = object->handle;  in vnode_pager_haspage()
365 	VM_OBJECT_ASSERT_LOCKED(object);  in vnode_pager_haspage()
376 	if (IDX_TO_OFF(pindex) >= object->un_pager.vnp.vnp_size)  in vnode_pager_haspage()
388 	lockstate = VM_OBJECT_DROP(object);  in vnode_pager_haspage()
390 	VM_OBJECT_PICKUP(object, lockstate);  in vnode_pager_haspage()
409 			    roundup2(object->size, pagesperblock),  in vnode_pager_haspage()
412 			    (uintmax_t )object->size));  in vnode_pager_haspage()
415 			if (pindex + *after >= object->size)  in vnode_pager_haspage()
416 				*after = object->size - 1 - pindex;  in vnode_pager_haspage()
441 	vm_object_t object;  in vnode_pager_setsize()  local
445 	if ((object = vp->v_object) == NULL)  in vnode_pager_setsize()
457 	VM_OBJECT_WLOCK(object);  in vnode_pager_setsize()
458 	if (object->type == OBJT_DEAD) {  in vnode_pager_setsize()
459 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_setsize()
462 	KASSERT(object->type == OBJT_VNODE,  in vnode_pager_setsize()
463 	    ("not vnode-backed object %p", object));  in vnode_pager_setsize()
464 	if (nsize == object->un_pager.vnp.vnp_size) {  in vnode_pager_setsize()
468 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_setsize()
472 	if (nsize < object->un_pager.vnp.vnp_size) {  in vnode_pager_setsize()
476 		if (nobjsize < object->size)  in vnode_pager_setsize()
477 			vm_object_page_remove(object, nobjsize, object->size,  in vnode_pager_setsize()
489 		m = vm_page_grab(object, OFF_TO_IDX(nsize), VM_ALLOC_NOCREAT);  in vnode_pager_setsize()
529 	object->un_pager.vnp.vnp_size = nsize;  in vnode_pager_setsize()
531 	atomic_store_64(&object->un_pager.vnp.vnp_size, nsize);  in vnode_pager_setsize()
533 	object->size = nobjsize;  in vnode_pager_setsize()
534 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_setsize()
575 vnode_pager_input_smlfs(vm_object_t object, vm_page_t m)  in vnode_pager_input_smlfs()  argument
587 	vp = object->handle;  in vnode_pager_input_smlfs()
605 		if (address >= object->un_pager.vnp.vnp_size) {  in vnode_pager_input_smlfs()
668 vnode_pager_input_old(vm_object_t object, vm_page_t m)  in vnode_pager_input_old()  argument
677 	VM_OBJECT_ASSERT_WLOCKED(object);  in vnode_pager_input_old()
683 	if (IDX_TO_OFF(m->pindex) >= object->un_pager.vnp.vnp_size) {  in vnode_pager_input_old()
687 		if (IDX_TO_OFF(m->pindex) + size > object->un_pager.vnp.vnp_size)  in vnode_pager_input_old()
688 			size = object->un_pager.vnp.vnp_size - IDX_TO_OFF(m->pindex);  in vnode_pager_input_old()
689 		vp = object->handle;  in vnode_pager_input_old()
690 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_input_old()
720 		VM_OBJECT_WLOCK(object);  in vnode_pager_input_old()
741 vnode_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind,  in vnode_pager_getpages()  argument
748 	vp = object->handle;  in vnode_pager_getpages()
756 vnode_pager_getpages_async(vm_object_t object, vm_page_t *m, int count,  in vnode_pager_getpages_async()  argument
762 	vp = object->handle;  in vnode_pager_getpages_async()
802 	vm_object_t object;  in vnode_pager_generic_getpages()  local
819 	object = vp->v_object;  in vnode_pager_generic_getpages()
824 	KASSERT(foff < object->un_pager.vnp.vnp_size,  in vnode_pager_generic_getpages()
851 		VM_OBJECT_WLOCK(object);  in vnode_pager_generic_getpages()
855 			error = vnode_pager_input_old(object, m[i]);  in vnode_pager_generic_getpages()
859 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_generic_getpages()
875 			error = vnode_pager_input_smlfs(object, m[i]);  in vnode_pager_generic_getpages()
909 	if (m[0]->pindex + after >= object->size)  in vnode_pager_generic_getpages()
910 		after = object->size - 1 - m[0]->pindex;  in vnode_pager_generic_getpages()
921 	rahead = min(rahead, object->size - m[count - 1]->pindex);  in vnode_pager_generic_getpages()
956 		VM_OBJECT_WLOCK(object);  in vnode_pager_generic_getpages()
966 			p = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);  in vnode_pager_generic_getpages()
992 		if (!VM_OBJECT_WOWNED(object))  in vnode_pager_generic_getpages()
993 			VM_OBJECT_WLOCK(object);  in vnode_pager_generic_getpages()
998 		if (endpindex > object->size)  in vnode_pager_generic_getpages()
999 			endpindex = object->size;  in vnode_pager_generic_getpages()
1003 			p = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);  in vnode_pager_generic_getpages()
1014 	if (VM_OBJECT_WOWNED(object))  in vnode_pager_generic_getpages()
1015 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_generic_getpages()
1043 	if ((foff + bytecount) > object->un_pager.vnp.vnp_size)  in vnode_pager_generic_getpages()
1044 		bytecount = object->un_pager.vnp.vnp_size - foff;  in vnode_pager_generic_getpages()
1126 	vm_object_t object;  in vnode_pager_generic_getpages_done()  local
1133 	object = bp->b_vp->v_object;  in vnode_pager_generic_getpages_done()
1155 		VM_OBJECT_WLOCK(object);  in vnode_pager_generic_getpages_done()
1160 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_generic_getpages_done()
1165 	VM_OBJECT_RLOCK(object);  in vnode_pager_generic_getpages_done()
1175 		if (nextoff <= object->un_pager.vnp.vnp_size) {  in vnode_pager_generic_getpages_done()
1193 			    object->un_pager.vnp.vnp_size - tfoff);  in vnode_pager_generic_getpages_done()
1195 			    object->un_pager.vnp.vnp_size - tfoff)) == 0,  in vnode_pager_generic_getpages_done()
1202 	VM_OBJECT_RUNLOCK(object);  in vnode_pager_generic_getpages_done()
1216 vnode_pager_putpages(vm_object_t object, vm_page_t *m, int count,  in vnode_pager_putpages()  argument
1241 	vp = object->handle;  in vnode_pager_putpages()
1242 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_putpages()
1246 	VM_OBJECT_WLOCK(object);  in vnode_pager_putpages()
1280 	vm_object_t object;  in vnode_pager_generic_putpages()  local
1291 	object = vp->v_object;  in vnode_pager_generic_putpages()
1322 	VM_OBJECT_RLOCK(object);  in vnode_pager_generic_putpages()
1323 	if (maxsize + poffset > object->un_pager.vnp.vnp_size) {  in vnode_pager_generic_putpages()
1324 		if (object->un_pager.vnp.vnp_size > poffset) {  in vnode_pager_generic_putpages()
1325 			maxsize = object->un_pager.vnp.vnp_size - poffset;  in vnode_pager_generic_putpages()
1351 	VM_OBJECT_RUNLOCK(object);  in vnode_pager_generic_putpages()
1490 	obj = ma[0]->object;  in vnode_pager_undirty_pages()
1534 vnode_pager_update_writecount(vm_object_t object, vm_offset_t start,  in vnode_pager_update_writecount()  argument
1540 	VM_OBJECT_WLOCK(object);  in vnode_pager_update_writecount()
1541 	if (object->type != OBJT_VNODE) {  in vnode_pager_update_writecount()
1542 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_update_writecount()
1545 	old_wm = object->un_pager.vnp.writemappings;  in vnode_pager_update_writecount()
1546 	object->un_pager.vnp.writemappings += (vm_ooffset_t)end - start;  in vnode_pager_update_writecount()
1547 	vp = object->handle;  in vnode_pager_update_writecount()
1548 	if (old_wm == 0 && object->un_pager.vnp.writemappings != 0) {  in vnode_pager_update_writecount()
1553 	} else if (old_wm != 0 && object->un_pager.vnp.writemappings == 0) {  in vnode_pager_update_writecount()
1559 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_update_writecount()
1563 vnode_pager_release_writecount(vm_object_t object, vm_offset_t start,  in vnode_pager_release_writecount()  argument
1570 	VM_OBJECT_WLOCK(object);  in vnode_pager_release_writecount()
1576 	if (object->type != OBJT_VNODE) {  in vnode_pager_release_writecount()
1577 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_release_writecount()
1586 	if (object->un_pager.vnp.writemappings != inc) {  in vnode_pager_release_writecount()
1587 		object->un_pager.vnp.writemappings -= inc;  in vnode_pager_release_writecount()
1588 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_release_writecount()
1592 	vp = object->handle;  in vnode_pager_release_writecount()
1594 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_release_writecount()
1605 	vnode_pager_update_writecount(object, end, start);  in vnode_pager_release_writecount()
1613 vnode_pager_getvp(vm_object_t object, struct vnode **vpp, bool *vp_heldp)  in vnode_pager_getvp()  argument
1615 	*vpp = object->handle;  in vnode_pager_getvp()