127 	struct vm_radix_node *rnode;  in vm_radix_node_get()  local
129 	rnode = uma_zalloc_smr(vm_radix_node_zone, M_NOWAIT);  in vm_radix_node_get()
130 	if (rnode == NULL)  in vm_radix_node_get()
138 	if (rnode->rn_last != 0) {  in vm_radix_node_get()
139 		vm_radix_node_store(&rnode->rn_child[rnode->rn_last - 1],  in vm_radix_node_get()
141 		rnode->rn_last = 0;  in vm_radix_node_get()
143 	rnode->rn_owner = owner;  in vm_radix_node_get()
144 	rnode->rn_count = count;  in vm_radix_node_get()
145 	rnode->rn_clev = clevel;  in vm_radix_node_get()
146 	return (rnode);  in vm_radix_node_get()
153 vm_radix_node_put(struct vm_radix_node *rnode, int8_t last)  in vm_radix_node_put()  argument
158 	KASSERT(rnode->rn_count == 0,  in vm_radix_node_put()
159 	    ("vm_radix_node_put: rnode %p has %d children", rnode,  in vm_radix_node_put()
160 	    rnode->rn_count));  in vm_radix_node_put()
164 		KASSERT(smr_unserialized_load(&rnode->rn_child[slot], true) ==  in vm_radix_node_put()
165 		    NULL, ("vm_radix_node_put: rnode %p has a child", rnode));  in vm_radix_node_put()
169 	rnode->rn_last = last + 1;  in vm_radix_node_put()
170 	uma_zfree_smr(vm_radix_node_zone, rnode);  in vm_radix_node_put()
246 vm_radix_root_store(struct vm_radix *rtree, struct vm_radix_node *rnode,  in vm_radix_root_store()  argument
250 	vm_radix_node_store((smrnode_t *)&rtree->rt_root, rnode, access);  in vm_radix_root_store()
257 vm_radix_isleaf(struct vm_radix_node *rnode)  in vm_radix_isleaf()  argument
260 	return (((uintptr_t)rnode & VM_RADIX_ISLEAF) != 0);  in vm_radix_isleaf()
267 vm_radix_topage(struct vm_radix_node *rnode)  in vm_radix_topage()  argument
270 	return ((vm_page_t)((uintptr_t)rnode & ~VM_RADIX_FLAGS));  in vm_radix_topage()
277 vm_radix_addpage(struct vm_radix_node *rnode, vm_pindex_t index, uint16_t clev,  in vm_radix_addpage()  argument
283 	vm_radix_node_store(&rnode->rn_child[slot],  in vm_radix_addpage()
310 vm_radix_keybarr(struct vm_radix_node *rnode, vm_pindex_t idx)  in vm_radix_keybarr()  argument
313 	if (rnode->rn_clev < VM_RADIX_LIMIT) {  in vm_radix_keybarr()
314 		idx = vm_radix_trimkey(idx, rnode->rn_clev + 1);  in vm_radix_keybarr()
315 		return (idx != rnode->rn_owner);  in vm_radix_keybarr()
325 vm_radix_reclaim_allnodes_int(struct vm_radix_node *rnode)  in vm_radix_reclaim_allnodes_int()  argument
330 	KASSERT(rnode->rn_count <= VM_RADIX_COUNT,  in vm_radix_reclaim_allnodes_int()
331 	    ("vm_radix_reclaim_allnodes_int: bad count in rnode %p", rnode));  in vm_radix_reclaim_allnodes_int()
332 	for (slot = 0; rnode->rn_count != 0; slot++) {  in vm_radix_reclaim_allnodes_int()
333 		child = vm_radix_node_load(&rnode->rn_child[slot], UNSERIALIZED);  in vm_radix_reclaim_allnodes_int()
338 		vm_radix_node_store(&rnode->rn_child[slot], NULL, UNSERIALIZED);  in vm_radix_reclaim_allnodes_int()
339 		rnode->rn_count--;  in vm_radix_reclaim_allnodes_int()
341 	vm_radix_node_put(rnode, -1);  in vm_radix_reclaim_allnodes_int()
389 	struct vm_radix_node *rnode, *tmp;  in vm_radix_insert()  local
401 	rnode = vm_radix_root_load(rtree, LOCKED);  in vm_radix_insert()
402 	if (rnode == NULL) {  in vm_radix_insert()
408 		if (vm_radix_isleaf(rnode)) {  in vm_radix_insert()
409 			m = vm_radix_topage(rnode);  in vm_radix_insert()
424 		} else if (vm_radix_keybarr(rnode, index))  in vm_radix_insert()
426 		slot = vm_radix_slot(index, rnode->rn_clev);  in vm_radix_insert()
427 		parentp = &rnode->rn_child[slot];  in vm_radix_insert()
430 			rnode->rn_count++;  in vm_radix_insert()
431 			vm_radix_addpage(rnode, index, rnode->rn_clev, page,  in vm_radix_insert()
435 		rnode = tmp;  in vm_radix_insert()
443 	newind = rnode->rn_owner;  in vm_radix_insert()
451 	vm_radix_node_store(&tmp->rn_child[slot], rnode, UNSERIALIZED);  in vm_radix_insert()
465 	struct vm_radix_node *rnode;  in vm_radix_is_singleton()  local
467 	rnode = vm_radix_root_load(rtree, LOCKED);  in vm_radix_is_singleton()
468 	if (rnode == NULL)  in vm_radix_is_singleton()
470 	return (vm_radix_isleaf(rnode));  in vm_radix_is_singleton()
481 	struct vm_radix_node *rnode;  in _vm_radix_lookup()  local
485 	rnode = vm_radix_root_load(rtree, access);  in _vm_radix_lookup()
486 	while (rnode != NULL) {  in _vm_radix_lookup()
487 		if (vm_radix_isleaf(rnode)) {  in _vm_radix_lookup()
488 			m = vm_radix_topage(rnode);  in _vm_radix_lookup()
493 		if (vm_radix_keybarr(rnode, index))  in _vm_radix_lookup()
495 		slot = vm_radix_slot(index, rnode->rn_clev);  in _vm_radix_lookup()
496 		rnode = vm_radix_node_load(&rnode->rn_child[slot], access);  in _vm_radix_lookup()
539 	struct vm_radix_node *child, *rnode;  in vm_radix_lookup_ge()  local
545 	rnode = vm_radix_root_load(rtree, LOCKED);  in vm_radix_lookup_ge()
546 	if (rnode == NULL)  in vm_radix_lookup_ge()
548 	else if (vm_radix_isleaf(rnode)) {  in vm_radix_lookup_ge()
549 		m = vm_radix_topage(rnode);  in vm_radix_lookup_ge()
564 		if (vm_radix_keybarr(rnode, index)) {  in vm_radix_lookup_ge()
565 			if (index > rnode->rn_owner) {  in vm_radix_lookup_ge()
578 					rnode = stack[--tos];  in vm_radix_lookup_ge()
580 				    rnode->rn_clev) == (VM_RADIX_COUNT - 1));  in vm_radix_lookup_ge()
588 				    rnode->rn_clev);  in vm_radix_lookup_ge()
589 				index += VM_RADIX_UNITLEVEL(rnode->rn_clev);  in vm_radix_lookup_ge()
591 				index = rnode->rn_owner;  in vm_radix_lookup_ge()
592 			KASSERT(!vm_radix_keybarr(rnode, index),  in vm_radix_lookup_ge()
595 		slot = vm_radix_slot(index, rnode->rn_clev);  in vm_radix_lookup_ge()
596 		child = vm_radix_node_load(&rnode->rn_child[slot], LOCKED);  in vm_radix_lookup_ge()
609 			inc = VM_RADIX_UNITLEVEL(rnode->rn_clev);  in vm_radix_lookup_ge()
610 			index = vm_radix_trimkey(index, rnode->rn_clev);  in vm_radix_lookup_ge()
614 				child = vm_radix_node_load(&rnode->rn_child[slot],  in vm_radix_lookup_ge()
633 		KASSERT(rnode->rn_clev > 0,  in vm_radix_lookup_ge()
637 		stack[tos++] = rnode;  in vm_radix_lookup_ge()
638 		rnode = child;  in vm_radix_lookup_ge()
651 	struct vm_radix_node *child, *rnode;  in vm_radix_lookup_le()  local
657 	rnode = vm_radix_root_load(rtree, LOCKED);  in vm_radix_lookup_le()
658 	if (rnode == NULL)  in vm_radix_lookup_le()
660 	else if (vm_radix_isleaf(rnode)) {  in vm_radix_lookup_le()
661 		m = vm_radix_topage(rnode);  in vm_radix_lookup_le()
676 		if (vm_radix_keybarr(rnode, index)) {  in vm_radix_lookup_le()
677 			if (index > rnode->rn_owner) {  in vm_radix_lookup_le()
678 				index = rnode->rn_owner + VM_RADIX_COUNT *  in vm_radix_lookup_le()
679 				    VM_RADIX_UNITLEVEL(rnode->rn_clev);  in vm_radix_lookup_le()
693 					rnode = stack[--tos];  in vm_radix_lookup_le()
695 				    rnode->rn_clev) == 0);  in vm_radix_lookup_le()
703 				    rnode->rn_clev);  in vm_radix_lookup_le()
706 			KASSERT(!vm_radix_keybarr(rnode, index),  in vm_radix_lookup_le()
709 		slot = vm_radix_slot(index, rnode->rn_clev);  in vm_radix_lookup_le()
710 		child = vm_radix_node_load(&rnode->rn_child[slot], LOCKED);  in vm_radix_lookup_le()
723 			inc = VM_RADIX_UNITLEVEL(rnode->rn_clev);  in vm_radix_lookup_le()
728 				child = vm_radix_node_load(&rnode->rn_child[slot],  in vm_radix_lookup_le()
747 		KASSERT(rnode->rn_clev > 0,  in vm_radix_lookup_le()
751 		stack[tos++] = rnode;  in vm_radix_lookup_le()
752 		rnode = child;  in vm_radix_lookup_le()
763 	struct vm_radix_node *rnode, *parent, *tmp;  in vm_radix_remove()  local
767 	rnode = vm_radix_root_load(rtree, LOCKED);  in vm_radix_remove()
768 	if (vm_radix_isleaf(rnode)) {  in vm_radix_remove()
769 		m = vm_radix_topage(rnode);  in vm_radix_remove()
777 		if (rnode == NULL)  in vm_radix_remove()
779 		slot = vm_radix_slot(index, rnode->rn_clev);  in vm_radix_remove()
780 		tmp = vm_radix_node_load(&rnode->rn_child[slot], LOCKED);  in vm_radix_remove()
785 			vm_radix_node_store(&rnode->rn_child[slot], NULL, LOCKED);  in vm_radix_remove()
786 			rnode->rn_count--;  in vm_radix_remove()
787 			if (rnode->rn_count > 1)  in vm_radix_remove()
790 				if (vm_radix_node_load(&rnode->rn_child[i],  in vm_radix_remove()
795 			tmp = vm_radix_node_load(&rnode->rn_child[i], LOCKED);  in vm_radix_remove()
801 				    &parent->rn_child[slot], LOCKED) == rnode,  in vm_radix_remove()
810 			rnode->rn_count--;  in vm_radix_remove()
811 			vm_radix_node_put(rnode, i);  in vm_radix_remove()
814 		parent = rnode;  in vm_radix_remove()
815 		rnode = tmp;  in vm_radix_remove()
844 	struct vm_radix_node *rnode, *tmp;  in vm_radix_replace()  local
850 	rnode = vm_radix_root_load(rtree, LOCKED);  in vm_radix_replace()
851 	if (rnode == NULL)  in vm_radix_replace()
853 	if (vm_radix_isleaf(rnode)) {  in vm_radix_replace()
854 		m = vm_radix_topage(rnode);  in vm_radix_replace()
862 		slot = vm_radix_slot(index, rnode->rn_clev);  in vm_radix_replace()
863 		tmp = vm_radix_node_load(&rnode->rn_child[slot], LOCKED);  in vm_radix_replace()
867 				vm_radix_node_store(&rnode->rn_child[slot],  in vm_radix_replace()
875 		rnode = tmp;  in vm_radix_replace()
892 	struct vm_radix_node *rnode, *tmp;  in DB_SHOW_COMMAND()  local
897 	rnode = (struct vm_radix_node *)addr;  in DB_SHOW_COMMAND()
899 	    (void *)rnode, (uintmax_t)rnode->rn_owner, rnode->rn_count,  in DB_SHOW_COMMAND()
900 	    rnode->rn_clev);  in DB_SHOW_COMMAND()
902 		tmp = vm_radix_node_load(&rnode->rn_child[i], UNSERIALIZED);  in DB_SHOW_COMMAND()
907 			    rnode->rn_clev);  in DB_SHOW_COMMAND()