42 dnode_increase_indirection(dnode_t *dn, dmu_tx_t *tx)  in dnode_increase_indirection()  argument
46 	int nblkptr = dn->dn_phys->dn_nblkptr;  in dnode_increase_indirection()
47 	int old_toplvl = dn->dn_phys->dn_nlevels - 1;  in dnode_increase_indirection()
48 	int new_level = dn->dn_next_nlevels[txgoff];  in dnode_increase_indirection()
51 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);  in dnode_increase_indirection()
54 	ASSERT(dn->dn_phys->dn_type != DMU_OT_NONE);  in dnode_increase_indirection()
55 	ASSERT(new_level > 1 && dn->dn_phys->dn_nlevels > 0);  in dnode_increase_indirection()
57 	db = dbuf_hold_level(dn, dn->dn_phys->dn_nlevels, 0, FTAG);  in dnode_increase_indirection()
60 	dn->dn_phys->dn_nlevels = new_level;  in dnode_increase_indirection()
61 	dprintf("os=%p obj=%llu, increase to %d\n", dn->dn_objset,  in dnode_increase_indirection()
62 	    (u_longlong_t)dn->dn_object, dn->dn_phys->dn_nlevels);  in dnode_increase_indirection()
73 		children[i] = dbuf_find(dn->dn_objset, dn->dn_object,  in dnode_increase_indirection()
79 	if (dn->dn_dbuf != NULL)  in dnode_increase_indirection()
80 		rw_enter(&dn->dn_dbuf->db_rwlock, RW_WRITER);  in dnode_increase_indirection()
85 	memcpy(db->db.db_data, dn->dn_phys->dn_blkptr,  in dnode_increase_indirection()
97 		ASSERT3P(DB_DNODE(child), ==, dn);  in dnode_increase_indirection()
100 		if (child->db_parent && child->db_parent != dn->dn_dbuf) {  in dnode_increase_indirection()
103 			    &dn->dn_phys->dn_blkptr[child->db_blkid]);  in dnode_increase_indirection()
108 		    child->db_parent == dn->dn_dbuf);  in dnode_increase_indirection()
122 	memset(dn->dn_phys->dn_blkptr, 0, sizeof (blkptr_t) * nblkptr);  in dnode_increase_indirection()
125 	if (dn->dn_dbuf != NULL)  in dnode_increase_indirection()
126 		rw_exit(&dn->dn_dbuf->db_rwlock);  in dnode_increase_indirection()
130 	rw_exit(&dn->dn_struct_rwlock);  in dnode_increase_indirection()
134 free_blocks(dnode_t *dn, blkptr_t *bp, int num, dmu_tx_t *tx)  in free_blocks()  argument
136 	dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;  in free_blocks()
139 	dprintf("ds=%p obj=%llx num=%d\n", ds, (u_longlong_t)dn->dn_object,  in free_blocks()
147 		ASSERT3U(bytesfreed, <=, DN_USED_BYTES(dn->dn_phys));  in free_blocks()
163 		if (spa_feature_is_active(dn->dn_objset->os_spa,  in free_blocks()
171 	dnode_diduse_space(dn, -bytesfreed);  in free_blocks()
182 	dnode_t *dn;  in free_verify()  local
185 	dn = DB_DNODE(db);  in free_verify()
186 	epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;  in free_verify()
190 	ASSERT3U(dn->dn_phys->dn_indblkshift, >=, SPA_BLKPTRSHIFT);  in free_verify()
194 	ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift);  in free_verify()
205 		rw_enter(&dn->dn_struct_rwlock, RW_READER);  in free_verify()
206 		err = dbuf_hold_impl(dn, db->db_level - 1,  in free_verify()
208 		rw_exit(&dn->dn_struct_rwlock);  in free_verify()
276 	dnode_t *dn;  in free_children()  local
315 	dn = DB_DNODE(db);  in free_children()
316 	epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;  in free_children()
336 		free_blocks(dn, bp, end - start + 1, tx);  in free_children()
342 			rw_enter(&dn->dn_struct_rwlock, RW_READER);  in free_children()
343 			VERIFY0(dbuf_hold_impl(dn, db->db_level - 1,  in free_children()
345 			rw_exit(&dn->dn_struct_rwlock);  in free_children()
358 		free_blocks(dn, db->db_blkptr, 1, tx);  in free_children()
371 dnode_sync_free_range_impl(dnode_t *dn, uint64_t blkid, uint64_t nblks,  in dnode_sync_free_range_impl()  argument
374 	blkptr_t *bp = dn->dn_phys->dn_blkptr;  in dnode_sync_free_range_impl()
375 	int dnlevel = dn->dn_phys->dn_nlevels;  in dnode_sync_free_range_impl()
378 	if (blkid > dn->dn_phys->dn_maxblkid)  in dnode_sync_free_range_impl()
381 	ASSERT(dn->dn_phys->dn_maxblkid < UINT64_MAX);  in dnode_sync_free_range_impl()
382 	if (blkid + nblks > dn->dn_phys->dn_maxblkid) {  in dnode_sync_free_range_impl()
383 		nblks = dn->dn_phys->dn_maxblkid - blkid + 1;  in dnode_sync_free_range_impl()
389 		if (blkid >= dn->dn_phys->dn_nblkptr) {  in dnode_sync_free_range_impl()
393 		ASSERT3U(blkid + nblks, <=, dn->dn_phys->dn_nblkptr);  in dnode_sync_free_range_impl()
394 		free_blocks(dn, bp + blkid, nblks, tx);  in dnode_sync_free_range_impl()
397 		    (dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT);  in dnode_sync_free_range_impl()
402 		ASSERT(start < dn->dn_phys->dn_nblkptr);  in dnode_sync_free_range_impl()
407 			rw_enter(&dn->dn_struct_rwlock, RW_READER);  in dnode_sync_free_range_impl()
408 			VERIFY0(dbuf_hold_impl(dn, dnlevel - 1, i,  in dnode_sync_free_range_impl()
410 			rw_exit(&dn->dn_struct_rwlock);  in dnode_sync_free_range_impl()
430 	if (trunc && !dn->dn_objset->os_raw_receive) {  in dnode_sync_free_range_impl()
432 		dn->dn_phys->dn_maxblkid = blkid == 0 ? 0 : blkid - 1;  in dnode_sync_free_range_impl()
434 		off = (dn->dn_phys->dn_maxblkid + 1) *  in dnode_sync_free_range_impl()
435 		    (dn->dn_phys->dn_datablkszsec << SPA_MINBLOCKSHIFT);  in dnode_sync_free_range_impl()
436 		ASSERT(off < dn->dn_phys->dn_maxblkid ||  in dnode_sync_free_range_impl()
437 		    dn->dn_phys->dn_maxblkid == 0 ||  in dnode_sync_free_range_impl()
438 		    dnode_next_offset(dn, 0, &off, 1, 1, 0) != 0);  in dnode_sync_free_range_impl()
452 	dnode_t *dn = dsfra->dsfra_dnode;  in dnode_sync_free_range()  local
454 	mutex_exit(&dn->dn_mtx);  in dnode_sync_free_range()
455 	dnode_sync_free_range_impl(dn, blkid, nblks,  in dnode_sync_free_range()
457 	mutex_enter(&dn->dn_mtx);  in dnode_sync_free_range()
464 dnode_evict_dbufs(dnode_t *dn)  in dnode_evict_dbufs()  argument
471 	mutex_enter(&dn->dn_dbufs_mtx);  in dnode_evict_dbufs()
472 	for (db = avl_first(&dn->dn_dbufs); db != NULL; db = db_next) {  in dnode_evict_dbufs()
476 		ASSERT3P(DB_DNODE(db), ==, dn);  in dnode_evict_dbufs()
486 			avl_insert_here(&dn->dn_dbufs, db_marker, db,  in dnode_evict_dbufs()
504 			db_next = AVL_NEXT(&dn->dn_dbufs, db_marker);  in dnode_evict_dbufs()
505 			avl_remove(&dn->dn_dbufs, db_marker);  in dnode_evict_dbufs()
509 			db_next = AVL_NEXT(&dn->dn_dbufs, db);  in dnode_evict_dbufs()
512 	mutex_exit(&dn->dn_dbufs_mtx);  in dnode_evict_dbufs()
516 	dnode_evict_bonus(dn);  in dnode_evict_dbufs()
520 dnode_evict_bonus(dnode_t *dn)  in dnode_evict_bonus()  argument
522 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);  in dnode_evict_bonus()
523 	if (dn->dn_bonus != NULL) {  in dnode_evict_bonus()
524 		if (zfs_refcount_is_zero(&dn->dn_bonus->db_holds)) {  in dnode_evict_bonus()
525 			mutex_enter(&dn->dn_bonus->db_mtx);  in dnode_evict_bonus()
526 			dbuf_destroy(dn->dn_bonus);  in dnode_evict_bonus()
527 			dn->dn_bonus = NULL;  in dnode_evict_bonus()
529 			dn->dn_bonus->db_pending_evict = TRUE;  in dnode_evict_bonus()
532 	rw_exit(&dn->dn_struct_rwlock);  in dnode_evict_bonus()
568 dnode_sync_free(dnode_t *dn, dmu_tx_t *tx)  in dnode_sync_free()  argument
578 	ASSERT0(DN_USED_BYTES(dn->dn_phys));  in dnode_sync_free()
579 	ASSERT(BP_IS_HOLE(dn->dn_phys->dn_blkptr));  in dnode_sync_free()
581 	dnode_undirty_dbufs(&dn->dn_dirty_records[txgoff]);  in dnode_sync_free()
582 	dnode_evict_dbufs(dn);  in dnode_sync_free()
595 	dn->dn_next_nlevels[txgoff] = 0;  in dnode_sync_free()
596 	dn->dn_next_indblkshift[txgoff] = 0;  in dnode_sync_free()
597 	dn->dn_next_blksz[txgoff] = 0;  in dnode_sync_free()
598 	dn->dn_next_maxblkid[txgoff] = 0;  in dnode_sync_free()
601 	ASSERT(dn->dn_phys->dn_type != DMU_OT_NONE);  in dnode_sync_free()
602 	ASSERT(dn->dn_type != DMU_OT_NONE);  in dnode_sync_free()
604 	ASSERT(dn->dn_free_txg > 0);  in dnode_sync_free()
605 	if (dn->dn_allocated_txg != dn->dn_free_txg)  in dnode_sync_free()
606 		dmu_buf_will_dirty(&dn->dn_dbuf->db, tx);  in dnode_sync_free()
607 	memset(dn->dn_phys, 0, sizeof (dnode_phys_t) * dn->dn_num_slots);  in dnode_sync_free()
608 	dnode_free_interior_slots(dn);  in dnode_sync_free()
610 	mutex_enter(&dn->dn_mtx);  in dnode_sync_free()
611 	dn->dn_type = DMU_OT_NONE;  in dnode_sync_free()
612 	dn->dn_maxblkid = 0;  in dnode_sync_free()
613 	dn->dn_allocated_txg = 0;  in dnode_sync_free()
614 	dn->dn_free_txg = 0;  in dnode_sync_free()
615 	dn->dn_have_spill = B_FALSE;  in dnode_sync_free()
616 	dn->dn_num_slots = 1;  in dnode_sync_free()
617 	mutex_exit(&dn->dn_mtx);  in dnode_sync_free()
619 	ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT);  in dnode_sync_free()
621 	dnode_rele(dn, (void *)(uintptr_t)tx->tx_txg);  in dnode_sync_free()
632 dnode_sync(dnode_t *dn, dmu_tx_t *tx)  in dnode_sync()  argument
634 	objset_t *os = dn->dn_objset;  in dnode_sync()
635 	dnode_phys_t *dnp = dn->dn_phys;  in dnode_sync()
637 	list_t *list = &dn->dn_dirty_records[txgoff];  in dnode_sync()
642 	ASSERT(dnp->dn_type != DMU_OT_NONE || dn->dn_allocated_txg);  in dnode_sync()
645 	DNODE_VERIFY(dn);  in dnode_sync()
647 	ASSERT(dn->dn_dbuf == NULL || arc_released(dn->dn_dbuf->db_buf));  in dnode_sync()
654 	    !DMU_OBJECT_IS_SPECIAL(dn->dn_object) &&  in dnode_sync()
656 		mutex_enter(&dn->dn_mtx);  in dnode_sync()
657 		dn->dn_oldused = DN_USED_BYTES(dn->dn_phys);  in dnode_sync()
658 		dn->dn_oldflags = dn->dn_phys->dn_flags;  in dnode_sync()
659 		dn->dn_phys->dn_flags |= DNODE_FLAG_USERUSED_ACCOUNTED;  in dnode_sync()
660 		if (dmu_objset_userobjused_enabled(dn->dn_objset))  in dnode_sync()
661 			dn->dn_phys->dn_flags |=  in dnode_sync()
663 		mutex_exit(&dn->dn_mtx);  in dnode_sync()
664 		dmu_objset_userquota_get_ids(dn, B_FALSE, tx);  in dnode_sync()
672 		ASSERT(!(dn->dn_phys->dn_flags &  in dnode_sync()
674 		ASSERT(!(dn->dn_phys->dn_flags &  in dnode_sync()
678 	mutex_enter(&dn->dn_mtx);  in dnode_sync()
679 	if (dn->dn_allocated_txg == tx->tx_txg) {  in dnode_sync()
684 			dnp->dn_nblkptr = dn->dn_nblkptr;  in dnode_sync()
687 		dnp->dn_type = dn->dn_type;  in dnode_sync()
688 		dnp->dn_bonustype = dn->dn_bonustype;  in dnode_sync()
689 		dnp->dn_bonuslen = dn->dn_bonuslen;  in dnode_sync()
692 	dnp->dn_extra_slots = dn->dn_num_slots - 1;  in dnode_sync()
703 	if (dn->dn_next_type[txgoff] != 0) {  in dnode_sync()
704 		dnp->dn_type = dn->dn_type;  in dnode_sync()
705 		dn->dn_next_type[txgoff] = 0;  in dnode_sync()
708 	if (dn->dn_next_blksz[txgoff] != 0) {  in dnode_sync()
709 		ASSERT(P2PHASE(dn->dn_next_blksz[txgoff],  in dnode_sync()
712 		    dn->dn_maxblkid == 0 || list_head(list) != NULL ||  in dnode_sync()
713 		    dn->dn_next_blksz[txgoff] >> SPA_MINBLOCKSHIFT ==  in dnode_sync()
715 		    !range_tree_is_empty(dn->dn_free_ranges[txgoff]));  in dnode_sync()
717 		    dn->dn_next_blksz[txgoff] >> SPA_MINBLOCKSHIFT;  in dnode_sync()
718 		dn->dn_next_blksz[txgoff] = 0;  in dnode_sync()
721 	if (dn->dn_next_bonuslen[txgoff] != 0) {  in dnode_sync()
722 		if (dn->dn_next_bonuslen[txgoff] == DN_ZERO_BONUSLEN)  in dnode_sync()
725 			dnp->dn_bonuslen = dn->dn_next_bonuslen[txgoff];  in dnode_sync()
728 		dn->dn_next_bonuslen[txgoff] = 0;  in dnode_sync()
731 	if (dn->dn_next_bonustype[txgoff] != 0) {  in dnode_sync()
732 		ASSERT(DMU_OT_IS_VALID(dn->dn_next_bonustype[txgoff]));  in dnode_sync()
733 		dnp->dn_bonustype = dn->dn_next_bonustype[txgoff];  in dnode_sync()
734 		dn->dn_next_bonustype[txgoff] = 0;  in dnode_sync()
737 	boolean_t freeing_dnode = dn->dn_free_txg > 0 &&  in dnode_sync()
738 	    dn->dn_free_txg <= tx->tx_txg;  in dnode_sync()
744 	if (dn->dn_rm_spillblk[txgoff] || freeing_dnode) {  in dnode_sync()
747 		dn->dn_rm_spillblk[txgoff] = 0;  in dnode_sync()
750 	if (dn->dn_next_indblkshift[txgoff] != 0) {  in dnode_sync()
752 		dnp->dn_indblkshift = dn->dn_next_indblkshift[txgoff];  in dnode_sync()
753 		dn->dn_next_indblkshift[txgoff] = 0;  in dnode_sync()
761 	dnp->dn_checksum = dn->dn_checksum;  in dnode_sync()
762 	dnp->dn_compress = dn->dn_compress;  in dnode_sync()
764 	mutex_exit(&dn->dn_mtx);  in dnode_sync()
767 		free_blocks(dn, DN_SPILL_BLKPTR(dn->dn_phys), 1, tx);  in dnode_sync()
768 		mutex_enter(&dn->dn_mtx);  in dnode_sync()
770 		mutex_exit(&dn->dn_mtx);  in dnode_sync()
774 	if (dn->dn_free_ranges[txgoff] != NULL) {  in dnode_sync()
776 		dsfra.dsfra_dnode = dn;  in dnode_sync()
779 		mutex_enter(&dn->dn_mtx);  in dnode_sync()
781 			ASSERT(range_tree_contains(dn->dn_free_ranges[txgoff],  in dnode_sync()
782 			    0, dn->dn_maxblkid + 1));  in dnode_sync()
792 		range_tree_walk(dn->dn_free_ranges[txgoff],  in dnode_sync()
794 		range_tree_vacate(dn->dn_free_ranges[txgoff], NULL, NULL);  in dnode_sync()
795 		range_tree_destroy(dn->dn_free_ranges[txgoff]);  in dnode_sync()
796 		dn->dn_free_ranges[txgoff] = NULL;  in dnode_sync()
797 		mutex_exit(&dn->dn_mtx);  in dnode_sync()
801 		dn->dn_objset->os_freed_dnodes++;  in dnode_sync()
802 		dnode_sync_free(dn, tx);  in dnode_sync()
806 	if (dn->dn_num_slots > DNODE_MIN_SLOTS) {  in dnode_sync()
807 		dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;  in dnode_sync()
814 	if (dn->dn_next_nlevels[txgoff]) {  in dnode_sync()
815 		dnode_increase_indirection(dn, tx);  in dnode_sync()
816 		dn->dn_next_nlevels[txgoff] = 0;  in dnode_sync()
824 	if (dn->dn_next_maxblkid[txgoff]) {  in dnode_sync()
825 		mutex_enter(&dn->dn_mtx);  in dnode_sync()
827 		    dn->dn_next_maxblkid[txgoff] & ~DMU_NEXT_MAXBLKID_SET;  in dnode_sync()
828 		dn->dn_next_maxblkid[txgoff] = 0;  in dnode_sync()
829 		mutex_exit(&dn->dn_mtx);  in dnode_sync()
832 	if (dn->dn_next_nblkptr[txgoff]) {  in dnode_sync()
834 		ASSERT(dn->dn_allocated_txg == tx->tx_txg);  in dnode_sync()
835 		if (dn->dn_next_nblkptr[txgoff] > dnp->dn_nblkptr) {  in dnode_sync()
839 			    (dn->dn_next_nblkptr[txgoff] - dnp->dn_nblkptr));  in dnode_sync()
843 			ASSERT(dn->dn_next_nblkptr[txgoff] < dnp->dn_nblkptr);  in dnode_sync()
846 				if (i >= dn->dn_next_nblkptr[txgoff])  in dnode_sync()
851 		mutex_enter(&dn->dn_mtx);  in dnode_sync()
852 		dnp->dn_nblkptr = dn->dn_next_nblkptr[txgoff];  in dnode_sync()
853 		dn->dn_next_nblkptr[txgoff] = 0;  in dnode_sync()
854 		mutex_exit(&dn->dn_mtx);  in dnode_sync()
857 	dbuf_sync_list(list, dn->dn_phys->dn_nlevels - 1, tx);  in dnode_sync()
859 	if (!DMU_OBJECT_IS_SPECIAL(dn->dn_object)) {  in dnode_sync()
861 		dnode_rele(dn, (void *)(uintptr_t)tx->tx_txg);  in dnode_sync()