92 static int vnode_pager_input_smlfs(vm_object_t object, vm_page_t m);
93 static int vnode_pager_input_old(vm_object_t object, vm_page_t m);
150 	vm_object_t object;  in vnode_create_vobject()  local
158 	object = vp->v_object;  in vnode_create_vobject()
159 	if (object != NULL)  in vnode_create_vobject()
172 	object = vnode_pager_alloc(vp, size, 0, 0, td->td_ucred);  in vnode_create_vobject()
179 	VM_OBJECT_RLOCK(object);  in vnode_create_vobject()
180 	last = refcount_release(&object->ref_count);  in vnode_create_vobject()
181 	VM_OBJECT_RUNLOCK(object);  in vnode_create_vobject()
239 	vm_object_t object;  in vnode_pager_alloc()  local
252 	object = vp->v_object;  in vnode_pager_alloc()
254 	if (object == NULL) {  in vnode_pager_alloc()
258 		object = vm_object_allocate(OBJT_VNODE,  in vnode_pager_alloc()
261 		object->un_pager.vnp.vnp_size = size;  in vnode_pager_alloc()
262 		object->un_pager.vnp.writemappings = 0;  in vnode_pager_alloc()
263 		object->domain.dr_policy = vnode_domainset;  in vnode_pager_alloc()
264 		object->handle = handle;  in vnode_pager_alloc()
266 			VM_OBJECT_WLOCK(object);  in vnode_pager_alloc()
267 			vm_object_set_flag(object, OBJ_SIZEVNLOCK);  in vnode_pager_alloc()
268 			VM_OBJECT_WUNLOCK(object);  in vnode_pager_alloc()
276 			VM_OBJECT_WLOCK(object);  in vnode_pager_alloc()
277 			KASSERT(object->ref_count == 1,  in vnode_pager_alloc()
278 			    ("leaked ref %p %d", object, object->ref_count));  in vnode_pager_alloc()
279 			object->type = OBJT_DEAD;  in vnode_pager_alloc()
280 			refcount_init(&object->ref_count, 0);  in vnode_pager_alloc()
281 			VM_OBJECT_WUNLOCK(object);  in vnode_pager_alloc()
282 			vm_object_destroy(object);  in vnode_pager_alloc()
285 		vp->v_object = object;  in vnode_pager_alloc()
289 		vm_object_reference(object);  in vnode_pager_alloc()
291 		if ((object->flags & OBJ_COLORED) == 0) {  in vnode_pager_alloc()
292 			VM_OBJECT_WLOCK(object);  in vnode_pager_alloc()
293 			vm_object_color(object, 0);  in vnode_pager_alloc()
294 			VM_OBJECT_WUNLOCK(object);  in vnode_pager_alloc()
298 	return (object);  in vnode_pager_alloc()
305 vnode_pager_dealloc(vm_object_t object)  in vnode_pager_dealloc()  argument
310 	vp = object->handle;  in vnode_pager_dealloc()
314 	VM_OBJECT_ASSERT_WLOCKED(object);  in vnode_pager_dealloc()
315 	vm_object_pip_wait(object, "vnpdea");  in vnode_pager_dealloc()
316 	refs = object->ref_count;  in vnode_pager_dealloc()
318 	object->handle = NULL;  in vnode_pager_dealloc()
319 	object->type = OBJT_DEAD;  in vnode_pager_dealloc()
321 	if (object->un_pager.vnp.writemappings > 0) {  in vnode_pager_dealloc()
322 		object->un_pager.vnp.writemappings = 0;  in vnode_pager_dealloc()
340 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_dealloc()
343 	VM_OBJECT_WLOCK(object);  in vnode_pager_dealloc()
347 vnode_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,  in vnode_pager_haspage()  argument
350 	struct vnode *vp = object->handle;  in vnode_pager_haspage()
359 	VM_OBJECT_ASSERT_LOCKED(object);  in vnode_pager_haspage()
370 	if (IDX_TO_OFF(pindex) >= object->un_pager.vnp.vnp_size)  in vnode_pager_haspage()
382 	lockstate = VM_OBJECT_DROP(object);  in vnode_pager_haspage()
384 	VM_OBJECT_PICKUP(object, lockstate);  in vnode_pager_haspage()
403 			    roundup2(object->size, pagesperblock),  in vnode_pager_haspage()
406 			    (uintmax_t )object->size));  in vnode_pager_haspage()
409 			if (pindex + *after >= object->size)  in vnode_pager_haspage()
410 				*after = object->size - 1 - pindex;  in vnode_pager_haspage()
435 	vm_object_t object;  in vnode_pager_setsize()  local
439 	if ((object = vp->v_object) == NULL)  in vnode_pager_setsize()
451 	VM_OBJECT_WLOCK(object);  in vnode_pager_setsize()
452 	if (object->type == OBJT_DEAD) {  in vnode_pager_setsize()
453 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_setsize()
456 	KASSERT(object->type == OBJT_VNODE,  in vnode_pager_setsize()
457 	    ("not vnode-backed object %p", object));  in vnode_pager_setsize()
458 	if (nsize == object->un_pager.vnp.vnp_size) {  in vnode_pager_setsize()
462 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_setsize()
466 	if (nsize < object->un_pager.vnp.vnp_size) {  in vnode_pager_setsize()
470 		if (nobjsize < object->size)  in vnode_pager_setsize()
471 			vm_object_page_remove(object, nobjsize, object->size,  in vnode_pager_setsize()
483 		m = vm_page_grab(object, OFF_TO_IDX(nsize), VM_ALLOC_NOCREAT);  in vnode_pager_setsize()
523 	object->un_pager.vnp.vnp_size = nsize;  in vnode_pager_setsize()
525 	atomic_store_64(&object->un_pager.vnp.vnp_size, nsize);  in vnode_pager_setsize()
527 	object->size = nobjsize;  in vnode_pager_setsize()
528 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_setsize()
569 vnode_pager_input_smlfs(vm_object_t object, vm_page_t m)  in vnode_pager_input_smlfs()  argument
581 	vp = object->handle;  in vnode_pager_input_smlfs()
599 		if (address >= object->un_pager.vnp.vnp_size) {  in vnode_pager_input_smlfs()
662 vnode_pager_input_old(vm_object_t object, vm_page_t m)  in vnode_pager_input_old()  argument
671 	VM_OBJECT_ASSERT_WLOCKED(object);  in vnode_pager_input_old()
677 	if (IDX_TO_OFF(m->pindex) >= object->un_pager.vnp.vnp_size) {  in vnode_pager_input_old()
681 		if (IDX_TO_OFF(m->pindex) + size > object->un_pager.vnp.vnp_size)  in vnode_pager_input_old()
682 			size = object->un_pager.vnp.vnp_size - IDX_TO_OFF(m->pindex);  in vnode_pager_input_old()
683 		vp = object->handle;  in vnode_pager_input_old()
684 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_input_old()
714 		VM_OBJECT_WLOCK(object);  in vnode_pager_input_old()
735 vnode_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind,  in vnode_pager_getpages()  argument
742 	vp = object->handle;  in vnode_pager_getpages()
750 vnode_pager_getpages_async(vm_object_t object, vm_page_t *m, int count,  in vnode_pager_getpages_async()  argument
756 	vp = object->handle;  in vnode_pager_getpages_async()
796 	vm_object_t object;  in vnode_pager_generic_getpages()  local
813 	object = vp->v_object;  in vnode_pager_generic_getpages()
818 	KASSERT(foff < object->un_pager.vnp.vnp_size,  in vnode_pager_generic_getpages()
845 		VM_OBJECT_WLOCK(object);  in vnode_pager_generic_getpages()
849 			error = vnode_pager_input_old(object, m[i]);  in vnode_pager_generic_getpages()
853 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_generic_getpages()
869 			error = vnode_pager_input_smlfs(object, m[i]);  in vnode_pager_generic_getpages()
903 	if (m[0]->pindex + after >= object->size)  in vnode_pager_generic_getpages()
904 		after = object->size - 1 - m[0]->pindex;  in vnode_pager_generic_getpages()
915 	rahead = min(rahead, object->size - m[count - 1]->pindex);  in vnode_pager_generic_getpages()
950 		VM_OBJECT_WLOCK(object);  in vnode_pager_generic_getpages()
960 			p = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);  in vnode_pager_generic_getpages()
986 		if (!VM_OBJECT_WOWNED(object))  in vnode_pager_generic_getpages()
987 			VM_OBJECT_WLOCK(object);  in vnode_pager_generic_getpages()
992 		if (endpindex > object->size)  in vnode_pager_generic_getpages()
993 			endpindex = object->size;  in vnode_pager_generic_getpages()
997 			p = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);  in vnode_pager_generic_getpages()
1008 	if (VM_OBJECT_WOWNED(object))  in vnode_pager_generic_getpages()
1009 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_generic_getpages()
1037 	if ((foff + bytecount) > object->un_pager.vnp.vnp_size)  in vnode_pager_generic_getpages()
1038 		bytecount = object->un_pager.vnp.vnp_size - foff;  in vnode_pager_generic_getpages()
1120 	vm_object_t object;  in vnode_pager_generic_getpages_done()  local
1127 	object = bp->b_vp->v_object;  in vnode_pager_generic_getpages_done()
1149 		VM_OBJECT_WLOCK(object);  in vnode_pager_generic_getpages_done()
1154 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_generic_getpages_done()
1159 	VM_OBJECT_RLOCK(object);  in vnode_pager_generic_getpages_done()
1169 		if (nextoff <= object->un_pager.vnp.vnp_size) {  in vnode_pager_generic_getpages_done()
1187 			    object->un_pager.vnp.vnp_size - tfoff);  in vnode_pager_generic_getpages_done()
1189 			    object->un_pager.vnp.vnp_size - tfoff)) == 0,  in vnode_pager_generic_getpages_done()
1196 	VM_OBJECT_RUNLOCK(object);  in vnode_pager_generic_getpages_done()
1210 vnode_pager_putpages(vm_object_t object, vm_page_t *m, int count,  in vnode_pager_putpages()  argument
1235 	vp = object->handle;  in vnode_pager_putpages()
1236 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_putpages()
1240 	VM_OBJECT_WLOCK(object);  in vnode_pager_putpages()
1274 	vm_object_t object;  in vnode_pager_generic_putpages()  local
1285 	object = vp->v_object;  in vnode_pager_generic_putpages()
1316 	VM_OBJECT_RLOCK(object);  in vnode_pager_generic_putpages()
1317 	if (maxsize + poffset > object->un_pager.vnp.vnp_size) {  in vnode_pager_generic_putpages()
1318 		if (object->un_pager.vnp.vnp_size > poffset) {  in vnode_pager_generic_putpages()
1319 			maxsize = object->un_pager.vnp.vnp_size - poffset;  in vnode_pager_generic_putpages()
1345 	VM_OBJECT_RUNLOCK(object);  in vnode_pager_generic_putpages()
1484 	obj = ma[0]->object;  in vnode_pager_undirty_pages()
1528 vnode_pager_update_writecount(vm_object_t object, vm_offset_t start,  in vnode_pager_update_writecount()  argument
1534 	VM_OBJECT_WLOCK(object);  in vnode_pager_update_writecount()
1535 	if (object->type != OBJT_VNODE) {  in vnode_pager_update_writecount()
1536 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_update_writecount()
1539 	old_wm = object->un_pager.vnp.writemappings;  in vnode_pager_update_writecount()
1540 	object->un_pager.vnp.writemappings += (vm_ooffset_t)end - start;  in vnode_pager_update_writecount()
1541 	vp = object->handle;  in vnode_pager_update_writecount()
1542 	if (old_wm == 0 && object->un_pager.vnp.writemappings != 0) {  in vnode_pager_update_writecount()
1547 	} else if (old_wm != 0 && object->un_pager.vnp.writemappings == 0) {  in vnode_pager_update_writecount()
1553 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_update_writecount()
1557 vnode_pager_release_writecount(vm_object_t object, vm_offset_t start,  in vnode_pager_release_writecount()  argument
1564 	VM_OBJECT_WLOCK(object);  in vnode_pager_release_writecount()
1570 	if (object->type != OBJT_VNODE) {  in vnode_pager_release_writecount()
1571 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_release_writecount()
1580 	if (object->un_pager.vnp.writemappings != inc) {  in vnode_pager_release_writecount()
1581 		object->un_pager.vnp.writemappings -= inc;  in vnode_pager_release_writecount()
1582 		VM_OBJECT_WUNLOCK(object);  in vnode_pager_release_writecount()
1586 	vp = object->handle;  in vnode_pager_release_writecount()
1588 	VM_OBJECT_WUNLOCK(object);  in vnode_pager_release_writecount()
1599 	vnode_pager_update_writecount(object, end, start);  in vnode_pager_release_writecount()