42 dnode_increase_indirection(dnode_t *dn, dmu_tx_t *tx)  in dnode_increase_indirection()  argument
46 	int nblkptr = dn->dn_phys->dn_nblkptr;  in dnode_increase_indirection()
47 	int old_toplvl = dn->dn_phys->dn_nlevels - 1;  in dnode_increase_indirection()
48 	int new_level = dn->dn_next_nlevels[txgoff];  in dnode_increase_indirection()
51 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);  in dnode_increase_indirection()
54 	ASSERT(dn->dn_phys->dn_type != DMU_OT_NONE);  in dnode_increase_indirection()
55 	ASSERT(new_level > 1 && dn->dn_phys->dn_nlevels > 0);  in dnode_increase_indirection()
57 	db = dbuf_hold_level(dn, dn->dn_phys->dn_nlevels, 0, FTAG);  in dnode_increase_indirection()
60 	dn->dn_phys->dn_nlevels = new_level;  in dnode_increase_indirection()
61 	dprintf("os=%p obj=%llu, increase to %d\n", dn->dn_objset,  in dnode_increase_indirection()
62 	    (u_longlong_t)dn->dn_object, dn->dn_phys->dn_nlevels);  in dnode_increase_indirection()
74 		    dbuf_find(dn->dn_objset, dn->dn_object, old_toplvl, i);  in dnode_increase_indirection()
79 	if (dn->dn_dbuf != NULL)  in dnode_increase_indirection()
80 		rw_enter(&dn->dn_dbuf->db_rwlock, RW_WRITER);  in dnode_increase_indirection()
85 	bcopy(dn->dn_phys->dn_blkptr, db->db.db_data,  in dnode_increase_indirection()
97 		ASSERT3P(DB_DNODE(child), ==, dn);  in dnode_increase_indirection()
100 		if (child->db_parent && child->db_parent != dn->dn_dbuf) {  in dnode_increase_indirection()
103 			    &dn->dn_phys->dn_blkptr[child->db_blkid]);  in dnode_increase_indirection()
108 		    child->db_parent == dn->dn_dbuf);  in dnode_increase_indirection()
122 	bzero(dn->dn_phys->dn_blkptr, sizeof (blkptr_t) * nblkptr);  in dnode_increase_indirection()
125 	if (dn->dn_dbuf != NULL)  in dnode_increase_indirection()
126 		rw_exit(&dn->dn_dbuf->db_rwlock);  in dnode_increase_indirection()
130 	rw_exit(&dn->dn_struct_rwlock);  in dnode_increase_indirection()
134 free_blocks(dnode_t *dn, blkptr_t *bp, int num, dmu_tx_t *tx)  in free_blocks()  argument
136 	dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;  in free_blocks()
139 	dprintf("ds=%p obj=%llx num=%d\n", ds, (u_longlong_t)dn->dn_object,  in free_blocks()
147 		ASSERT3U(bytesfreed, <=, DN_USED_BYTES(dn->dn_phys));  in free_blocks()
163 		if (spa_feature_is_active(dn->dn_objset->os_spa,  in free_blocks()
171 	dnode_diduse_space(dn, -bytesfreed);  in free_blocks()
181 	dnode_t *dn;  in free_verify()  local
184 	dn = DB_DNODE(db);  in free_verify()
185 	epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;  in free_verify()
192 	ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift);  in free_verify()
204 		rw_enter(&dn->dn_struct_rwlock, RW_READER);  in free_verify()
205 		err = dbuf_hold_impl(dn, db->db_level - 1,  in free_verify()
207 		rw_exit(&dn->dn_struct_rwlock);  in free_verify()
269 	dnode_t *dn;  in free_children()  local
308 	dn = DB_DNODE(db);  in free_children()
309 	epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;  in free_children()
329 		free_blocks(dn, bp, end - start + 1, tx);  in free_children()
335 			rw_enter(&dn->dn_struct_rwlock, RW_READER);  in free_children()
336 			VERIFY0(dbuf_hold_impl(dn, db->db_level - 1,  in free_children()
338 			rw_exit(&dn->dn_struct_rwlock);  in free_children()
351 		free_blocks(dn, db->db_blkptr, 1, tx);  in free_children()
364 dnode_sync_free_range_impl(dnode_t *dn, uint64_t blkid, uint64_t nblks,  in dnode_sync_free_range_impl()  argument
367 	blkptr_t *bp = dn->dn_phys->dn_blkptr;  in dnode_sync_free_range_impl()
368 	int dnlevel = dn->dn_phys->dn_nlevels;  in dnode_sync_free_range_impl()
371 	if (blkid > dn->dn_phys->dn_maxblkid)  in dnode_sync_free_range_impl()
374 	ASSERT(dn->dn_phys->dn_maxblkid < UINT64_MAX);  in dnode_sync_free_range_impl()
375 	if (blkid + nblks > dn->dn_phys->dn_maxblkid) {  in dnode_sync_free_range_impl()
376 		nblks = dn->dn_phys->dn_maxblkid - blkid + 1;  in dnode_sync_free_range_impl()
382 		if (blkid >= dn->dn_phys->dn_nblkptr) {  in dnode_sync_free_range_impl()
386 		ASSERT3U(blkid + nblks, <=, dn->dn_phys->dn_nblkptr);  in dnode_sync_free_range_impl()
387 		free_blocks(dn, bp + blkid, nblks, tx);  in dnode_sync_free_range_impl()
390 		    (dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT);  in dnode_sync_free_range_impl()
395 		ASSERT(start < dn->dn_phys->dn_nblkptr);  in dnode_sync_free_range_impl()
400 			rw_enter(&dn->dn_struct_rwlock, RW_READER);  in dnode_sync_free_range_impl()
401 			VERIFY0(dbuf_hold_impl(dn, dnlevel - 1, i,  in dnode_sync_free_range_impl()
403 			rw_exit(&dn->dn_struct_rwlock);  in dnode_sync_free_range_impl()
423 	if (trunc && !dn->dn_objset->os_raw_receive) {  in dnode_sync_free_range_impl()
425 		dn->dn_phys->dn_maxblkid = blkid == 0 ? 0 : blkid - 1;  in dnode_sync_free_range_impl()
427 		off = (dn->dn_phys->dn_maxblkid + 1) *  in dnode_sync_free_range_impl()
428 		    (dn->dn_phys->dn_datablkszsec << SPA_MINBLOCKSHIFT);  in dnode_sync_free_range_impl()
429 		ASSERT(off < dn->dn_phys->dn_maxblkid ||  in dnode_sync_free_range_impl()
430 		    dn->dn_phys->dn_maxblkid == 0 ||  in dnode_sync_free_range_impl()
431 		    dnode_next_offset(dn, 0, &off, 1, 1, 0) != 0);  in dnode_sync_free_range_impl()
445 	dnode_t *dn = dsfra->dsfra_dnode;  in dnode_sync_free_range()  local
447 	mutex_exit(&dn->dn_mtx);  in dnode_sync_free_range()
448 	dnode_sync_free_range_impl(dn, blkid, nblks,  in dnode_sync_free_range()
450 	mutex_enter(&dn->dn_mtx);  in dnode_sync_free_range()
457 dnode_evict_dbufs(dnode_t *dn)  in dnode_evict_dbufs()  argument
464 	mutex_enter(&dn->dn_dbufs_mtx);  in dnode_evict_dbufs()
465 	for (db = avl_first(&dn->dn_dbufs); db != NULL; db = db_next) {  in dnode_evict_dbufs()
469 		ASSERT3P(DB_DNODE(db), ==, dn);  in dnode_evict_dbufs()
479 			avl_insert_here(&dn->dn_dbufs, db_marker, db,  in dnode_evict_dbufs()
497 			db_next = AVL_NEXT(&dn->dn_dbufs, db_marker);  in dnode_evict_dbufs()
498 			avl_remove(&dn->dn_dbufs, db_marker);  in dnode_evict_dbufs()
502 			db_next = AVL_NEXT(&dn->dn_dbufs, db);  in dnode_evict_dbufs()
505 	mutex_exit(&dn->dn_dbufs_mtx);  in dnode_evict_dbufs()
509 	dnode_evict_bonus(dn);  in dnode_evict_dbufs()
513 dnode_evict_bonus(dnode_t *dn)  in dnode_evict_bonus()  argument
515 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);  in dnode_evict_bonus()
516 	if (dn->dn_bonus != NULL) {  in dnode_evict_bonus()
517 		if (zfs_refcount_is_zero(&dn->dn_bonus->db_holds)) {  in dnode_evict_bonus()
518 			mutex_enter(&dn->dn_bonus->db_mtx);  in dnode_evict_bonus()
519 			dbuf_destroy(dn->dn_bonus);  in dnode_evict_bonus()
520 			dn->dn_bonus = NULL;  in dnode_evict_bonus()
522 			dn->dn_bonus->db_pending_evict = TRUE;  in dnode_evict_bonus()
525 	rw_exit(&dn->dn_struct_rwlock);  in dnode_evict_bonus()
561 dnode_sync_free(dnode_t *dn, dmu_tx_t *tx)  in dnode_sync_free()  argument
571 	ASSERT0(DN_USED_BYTES(dn->dn_phys));  in dnode_sync_free()
572 	ASSERT(BP_IS_HOLE(dn->dn_phys->dn_blkptr));  in dnode_sync_free()
574 	dnode_undirty_dbufs(&dn->dn_dirty_records[txgoff]);  in dnode_sync_free()
575 	dnode_evict_dbufs(dn);  in dnode_sync_free()
588 	dn->dn_next_nlevels[txgoff] = 0;  in dnode_sync_free()
589 	dn->dn_next_indblkshift[txgoff] = 0;  in dnode_sync_free()
590 	dn->dn_next_blksz[txgoff] = 0;  in dnode_sync_free()
591 	dn->dn_next_maxblkid[txgoff] = 0;  in dnode_sync_free()
594 	ASSERT(dn->dn_phys->dn_type != DMU_OT_NONE);  in dnode_sync_free()
595 	ASSERT(dn->dn_type != DMU_OT_NONE);  in dnode_sync_free()
597 	ASSERT(dn->dn_free_txg > 0);  in dnode_sync_free()
598 	if (dn->dn_allocated_txg != dn->dn_free_txg)  in dnode_sync_free()
599 		dmu_buf_will_dirty(&dn->dn_dbuf->db, tx);  in dnode_sync_free()
600 	bzero(dn->dn_phys, sizeof (dnode_phys_t) * dn->dn_num_slots);  in dnode_sync_free()
601 	dnode_free_interior_slots(dn);  in dnode_sync_free()
603 	mutex_enter(&dn->dn_mtx);  in dnode_sync_free()
604 	dn->dn_type = DMU_OT_NONE;  in dnode_sync_free()
605 	dn->dn_maxblkid = 0;  in dnode_sync_free()
606 	dn->dn_allocated_txg = 0;  in dnode_sync_free()
607 	dn->dn_free_txg = 0;  in dnode_sync_free()
608 	dn->dn_have_spill = B_FALSE;  in dnode_sync_free()
609 	dn->dn_num_slots = 1;  in dnode_sync_free()
610 	mutex_exit(&dn->dn_mtx);  in dnode_sync_free()
612 	ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT);  in dnode_sync_free()
614 	dnode_rele(dn, (void *)(uintptr_t)tx->tx_txg);  in dnode_sync_free()
625 dnode_sync(dnode_t *dn, dmu_tx_t *tx)  in dnode_sync()  argument
627 	objset_t *os = dn->dn_objset;  in dnode_sync()
628 	dnode_phys_t *dnp = dn->dn_phys;  in dnode_sync()
630 	list_t *list = &dn->dn_dirty_records[txgoff];  in dnode_sync()
635 	ASSERT(dnp->dn_type != DMU_OT_NONE || dn->dn_allocated_txg);  in dnode_sync()
638 	DNODE_VERIFY(dn);  in dnode_sync()
640 	ASSERT(dn->dn_dbuf == NULL || arc_released(dn->dn_dbuf->db_buf));  in dnode_sync()
647 	    !DMU_OBJECT_IS_SPECIAL(dn->dn_object) &&  in dnode_sync()
649 		mutex_enter(&dn->dn_mtx);  in dnode_sync()
650 		dn->dn_oldused = DN_USED_BYTES(dn->dn_phys);  in dnode_sync()
651 		dn->dn_oldflags = dn->dn_phys->dn_flags;  in dnode_sync()
652 		dn->dn_phys->dn_flags |= DNODE_FLAG_USERUSED_ACCOUNTED;  in dnode_sync()
653 		if (dmu_objset_userobjused_enabled(dn->dn_objset))  in dnode_sync()
654 			dn->dn_phys->dn_flags |=  in dnode_sync()
656 		mutex_exit(&dn->dn_mtx);  in dnode_sync()
657 		dmu_objset_userquota_get_ids(dn, B_FALSE, tx);  in dnode_sync()
665 		ASSERT(!(dn->dn_phys->dn_flags &  in dnode_sync()
667 		ASSERT(!(dn->dn_phys->dn_flags &  in dnode_sync()
671 	mutex_enter(&dn->dn_mtx);  in dnode_sync()
672 	if (dn->dn_allocated_txg == tx->tx_txg) {  in dnode_sync()
677 			dnp->dn_nblkptr = dn->dn_nblkptr;  in dnode_sync()
680 		dnp->dn_type = dn->dn_type;  in dnode_sync()
681 		dnp->dn_bonustype = dn->dn_bonustype;  in dnode_sync()
682 		dnp->dn_bonuslen = dn->dn_bonuslen;  in dnode_sync()
685 	dnp->dn_extra_slots = dn->dn_num_slots - 1;  in dnode_sync()
696 	if (dn->dn_next_type[txgoff] != 0) {  in dnode_sync()
697 		dnp->dn_type = dn->dn_type;  in dnode_sync()
698 		dn->dn_next_type[txgoff] = 0;  in dnode_sync()
701 	if (dn->dn_next_blksz[txgoff] != 0) {  in dnode_sync()
702 		ASSERT(P2PHASE(dn->dn_next_blksz[txgoff],  in dnode_sync()
705 		    dn->dn_maxblkid == 0 || list_head(list) != NULL ||  in dnode_sync()
706 		    dn->dn_next_blksz[txgoff] >> SPA_MINBLOCKSHIFT ==  in dnode_sync()
708 		    !range_tree_is_empty(dn->dn_free_ranges[txgoff]));  in dnode_sync()
710 		    dn->dn_next_blksz[txgoff] >> SPA_MINBLOCKSHIFT;  in dnode_sync()
711 		dn->dn_next_blksz[txgoff] = 0;  in dnode_sync()
714 	if (dn->dn_next_bonuslen[txgoff] != 0) {  in dnode_sync()
715 		if (dn->dn_next_bonuslen[txgoff] == DN_ZERO_BONUSLEN)  in dnode_sync()
718 			dnp->dn_bonuslen = dn->dn_next_bonuslen[txgoff];  in dnode_sync()
721 		dn->dn_next_bonuslen[txgoff] = 0;  in dnode_sync()
724 	if (dn->dn_next_bonustype[txgoff] != 0) {  in dnode_sync()
725 		ASSERT(DMU_OT_IS_VALID(dn->dn_next_bonustype[txgoff]));  in dnode_sync()
726 		dnp->dn_bonustype = dn->dn_next_bonustype[txgoff];  in dnode_sync()
727 		dn->dn_next_bonustype[txgoff] = 0;  in dnode_sync()
730 	boolean_t freeing_dnode = dn->dn_free_txg > 0 &&  in dnode_sync()
731 	    dn->dn_free_txg <= tx->tx_txg;  in dnode_sync()
737 	if (dn->dn_rm_spillblk[txgoff] || freeing_dnode) {  in dnode_sync()
740 		dn->dn_rm_spillblk[txgoff] = 0;  in dnode_sync()
743 	if (dn->dn_next_indblkshift[txgoff] != 0) {  in dnode_sync()
745 		dnp->dn_indblkshift = dn->dn_next_indblkshift[txgoff];  in dnode_sync()
746 		dn->dn_next_indblkshift[txgoff] = 0;  in dnode_sync()
754 	dnp->dn_checksum = dn->dn_checksum;  in dnode_sync()
755 	dnp->dn_compress = dn->dn_compress;  in dnode_sync()
757 	mutex_exit(&dn->dn_mtx);  in dnode_sync()
760 		free_blocks(dn, DN_SPILL_BLKPTR(dn->dn_phys), 1, tx);  in dnode_sync()
761 		mutex_enter(&dn->dn_mtx);  in dnode_sync()
763 		mutex_exit(&dn->dn_mtx);  in dnode_sync()
767 	if (dn->dn_free_ranges[txgoff] != NULL) {  in dnode_sync()
769 		dsfra.dsfra_dnode = dn;  in dnode_sync()
772 		mutex_enter(&dn->dn_mtx);  in dnode_sync()
774 			ASSERT(range_tree_contains(dn->dn_free_ranges[txgoff],  in dnode_sync()
775 			    0, dn->dn_maxblkid + 1));  in dnode_sync()
785 		range_tree_walk(dn->dn_free_ranges[txgoff],  in dnode_sync()
787 		range_tree_vacate(dn->dn_free_ranges[txgoff], NULL, NULL);  in dnode_sync()
788 		range_tree_destroy(dn->dn_free_ranges[txgoff]);  in dnode_sync()
789 		dn->dn_free_ranges[txgoff] = NULL;  in dnode_sync()
790 		mutex_exit(&dn->dn_mtx);  in dnode_sync()
794 		dn->dn_objset->os_freed_dnodes++;  in dnode_sync()
795 		dnode_sync_free(dn, tx);  in dnode_sync()
799 	if (dn->dn_num_slots > DNODE_MIN_SLOTS) {  in dnode_sync()
800 		dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;  in dnode_sync()
807 	if (dn->dn_next_nlevels[txgoff]) {  in dnode_sync()
808 		dnode_increase_indirection(dn, tx);  in dnode_sync()
809 		dn->dn_next_nlevels[txgoff] = 0;  in dnode_sync()
817 	if (dn->dn_next_maxblkid[txgoff]) {  in dnode_sync()
818 		mutex_enter(&dn->dn_mtx);  in dnode_sync()
820 		    dn->dn_next_maxblkid[txgoff] & ~DMU_NEXT_MAXBLKID_SET;  in dnode_sync()
821 		dn->dn_next_maxblkid[txgoff] = 0;  in dnode_sync()
822 		mutex_exit(&dn->dn_mtx);  in dnode_sync()
825 	if (dn->dn_next_nblkptr[txgoff]) {  in dnode_sync()
827 		ASSERT(dn->dn_allocated_txg == tx->tx_txg);  in dnode_sync()
828 		if (dn->dn_next_nblkptr[txgoff] > dnp->dn_nblkptr) {  in dnode_sync()
832 			    (dn->dn_next_nblkptr[txgoff] - dnp->dn_nblkptr));  in dnode_sync()
836 			ASSERT(dn->dn_next_nblkptr[txgoff] < dnp->dn_nblkptr);  in dnode_sync()
839 				if (i >= dn->dn_next_nblkptr[txgoff])  in dnode_sync()
844 		mutex_enter(&dn->dn_mtx);  in dnode_sync()
845 		dnp->dn_nblkptr = dn->dn_next_nblkptr[txgoff];  in dnode_sync()
846 		dn->dn_next_nblkptr[txgoff] = 0;  in dnode_sync()
847 		mutex_exit(&dn->dn_mtx);  in dnode_sync()
850 	dbuf_sync_list(list, dn->dn_phys->dn_nlevels - 1, tx);  in dnode_sync()
852 	if (!DMU_OBJECT_IS_SPECIAL(dn->dn_object)) {  in dnode_sync()
854 		dnode_rele(dn, (void *)(uintptr_t)tx->tx_txg);  in dnode_sync()