Lines Matching refs:dn
117 dnode_t *dn = arg; in dnode_cons() local
119 rw_init(&dn->dn_struct_rwlock, NULL, RW_NOLOCKDEP, NULL); in dnode_cons()
120 mutex_init(&dn->dn_mtx, NULL, MUTEX_DEFAULT, NULL); in dnode_cons()
121 mutex_init(&dn->dn_dbufs_mtx, NULL, MUTEX_DEFAULT, NULL); in dnode_cons()
122 cv_init(&dn->dn_notxholds, NULL, CV_DEFAULT, NULL); in dnode_cons()
123 cv_init(&dn->dn_nodnholds, NULL, CV_DEFAULT, NULL); in dnode_cons()
129 zfs_refcount_create_untracked(&dn->dn_holds); in dnode_cons()
130 zfs_refcount_create(&dn->dn_tx_holds); in dnode_cons()
131 list_link_init(&dn->dn_link); in dnode_cons()
133 memset(dn->dn_next_type, 0, sizeof (dn->dn_next_type)); in dnode_cons()
134 memset(dn->dn_next_nblkptr, 0, sizeof (dn->dn_next_nblkptr)); in dnode_cons()
135 memset(dn->dn_next_nlevels, 0, sizeof (dn->dn_next_nlevels)); in dnode_cons()
136 memset(dn->dn_next_indblkshift, 0, sizeof (dn->dn_next_indblkshift)); in dnode_cons()
137 memset(dn->dn_next_bonustype, 0, sizeof (dn->dn_next_bonustype)); in dnode_cons()
138 memset(dn->dn_rm_spillblk, 0, sizeof (dn->dn_rm_spillblk)); in dnode_cons()
139 memset(dn->dn_next_bonuslen, 0, sizeof (dn->dn_next_bonuslen)); in dnode_cons()
140 memset(dn->dn_next_blksz, 0, sizeof (dn->dn_next_blksz)); in dnode_cons()
141 memset(dn->dn_next_maxblkid, 0, sizeof (dn->dn_next_maxblkid)); in dnode_cons()
144 multilist_link_init(&dn->dn_dirty_link[i]); in dnode_cons()
145 dn->dn_free_ranges[i] = NULL; in dnode_cons()
146 list_create(&dn->dn_dirty_records[i], in dnode_cons()
151 dn->dn_allocated_txg = 0; in dnode_cons()
152 dn->dn_free_txg = 0; in dnode_cons()
153 dn->dn_assigned_txg = 0; in dnode_cons()
154 dn->dn_dirty_txg = 0; in dnode_cons()
155 dn->dn_dirtyctx = 0; in dnode_cons()
156 dn->dn_dirtyctx_firstset = NULL; in dnode_cons()
157 dn->dn_bonus = NULL; in dnode_cons()
158 dn->dn_have_spill = B_FALSE; in dnode_cons()
159 dn->dn_zio = NULL; in dnode_cons()
160 dn->dn_oldused = 0; in dnode_cons()
161 dn->dn_oldflags = 0; in dnode_cons()
162 dn->dn_olduid = 0; in dnode_cons()
163 dn->dn_oldgid = 0; in dnode_cons()
164 dn->dn_oldprojid = ZFS_DEFAULT_PROJID; in dnode_cons()
165 dn->dn_newuid = 0; in dnode_cons()
166 dn->dn_newgid = 0; in dnode_cons()
167 dn->dn_newprojid = ZFS_DEFAULT_PROJID; in dnode_cons()
168 dn->dn_id_flags = 0; in dnode_cons()
170 dn->dn_dbufs_count = 0; in dnode_cons()
171 avl_create(&dn->dn_dbufs, dbuf_compare, sizeof (dmu_buf_impl_t), in dnode_cons()
174 dn->dn_moved = 0; in dnode_cons()
182 dnode_t *dn = arg; in dnode_dest() local
184 rw_destroy(&dn->dn_struct_rwlock); in dnode_dest()
185 mutex_destroy(&dn->dn_mtx); in dnode_dest()
186 mutex_destroy(&dn->dn_dbufs_mtx); in dnode_dest()
187 cv_destroy(&dn->dn_notxholds); in dnode_dest()
188 cv_destroy(&dn->dn_nodnholds); in dnode_dest()
189 zfs_refcount_destroy(&dn->dn_holds); in dnode_dest()
190 zfs_refcount_destroy(&dn->dn_tx_holds); in dnode_dest()
191 ASSERT(!list_link_active(&dn->dn_link)); in dnode_dest()
194 ASSERT(!multilist_link_active(&dn->dn_dirty_link[i])); in dnode_dest()
195 ASSERT3P(dn->dn_free_ranges[i], ==, NULL); in dnode_dest()
196 list_destroy(&dn->dn_dirty_records[i]); in dnode_dest()
197 ASSERT0(dn->dn_next_nblkptr[i]); in dnode_dest()
198 ASSERT0(dn->dn_next_nlevels[i]); in dnode_dest()
199 ASSERT0(dn->dn_next_indblkshift[i]); in dnode_dest()
200 ASSERT0(dn->dn_next_bonustype[i]); in dnode_dest()
201 ASSERT0(dn->dn_rm_spillblk[i]); in dnode_dest()
202 ASSERT0(dn->dn_next_bonuslen[i]); in dnode_dest()
203 ASSERT0(dn->dn_next_blksz[i]); in dnode_dest()
204 ASSERT0(dn->dn_next_maxblkid[i]); in dnode_dest()
207 ASSERT0(dn->dn_allocated_txg); in dnode_dest()
208 ASSERT0(dn->dn_free_txg); in dnode_dest()
209 ASSERT0(dn->dn_assigned_txg); in dnode_dest()
210 ASSERT0(dn->dn_dirty_txg); in dnode_dest()
211 ASSERT0(dn->dn_dirtyctx); in dnode_dest()
212 ASSERT3P(dn->dn_dirtyctx_firstset, ==, NULL); in dnode_dest()
213 ASSERT3P(dn->dn_bonus, ==, NULL); in dnode_dest()
214 ASSERT(!dn->dn_have_spill); in dnode_dest()
215 ASSERT3P(dn->dn_zio, ==, NULL); in dnode_dest()
216 ASSERT0(dn->dn_oldused); in dnode_dest()
217 ASSERT0(dn->dn_oldflags); in dnode_dest()
218 ASSERT0(dn->dn_olduid); in dnode_dest()
219 ASSERT0(dn->dn_oldgid); in dnode_dest()
220 ASSERT0(dn->dn_oldprojid); in dnode_dest()
221 ASSERT0(dn->dn_newuid); in dnode_dest()
222 ASSERT0(dn->dn_newgid); in dnode_dest()
223 ASSERT0(dn->dn_newprojid); in dnode_dest()
224 ASSERT0(dn->dn_id_flags); in dnode_dest()
226 ASSERT0(dn->dn_dbufs_count); in dnode_dest()
227 avl_destroy(&dn->dn_dbufs); in dnode_dest()
387 dnode_verify(dnode_t *dn) in dnode_verify() argument
391 ASSERT(dn->dn_phys); in dnode_verify()
392 ASSERT(dn->dn_objset); in dnode_verify()
393 ASSERT(dn->dn_handle->dnh_dnode == dn); in dnode_verify()
395 ASSERT(DMU_OT_IS_VALID(dn->dn_phys->dn_type)); in dnode_verify()
400 if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { in dnode_verify()
401 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dnode_verify()
404 if (dn->dn_phys->dn_type != DMU_OT_NONE || dn->dn_allocated_txg != 0) { in dnode_verify()
406 int max_bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots); in dnode_verify()
407 ASSERT3U(dn->dn_indblkshift, <=, SPA_MAXBLOCKSHIFT); in dnode_verify()
408 if (dn->dn_datablkshift) { in dnode_verify()
409 ASSERT3U(dn->dn_datablkshift, >=, SPA_MINBLOCKSHIFT); in dnode_verify()
410 ASSERT3U(dn->dn_datablkshift, <=, SPA_MAXBLOCKSHIFT); in dnode_verify()
411 ASSERT3U(1<<dn->dn_datablkshift, ==, dn->dn_datablksz); in dnode_verify()
413 ASSERT3U(dn->dn_nlevels, <=, 30); in dnode_verify()
414 ASSERT(DMU_OT_IS_VALID(dn->dn_type)); in dnode_verify()
415 ASSERT3U(dn->dn_nblkptr, >=, 1); in dnode_verify()
416 ASSERT3U(dn->dn_nblkptr, <=, DN_MAX_NBLKPTR); in dnode_verify()
417 ASSERT3U(dn->dn_bonuslen, <=, max_bonuslen); in dnode_verify()
418 ASSERT3U(dn->dn_datablksz, ==, in dnode_verify()
419 dn->dn_datablkszsec << SPA_MINBLOCKSHIFT); in dnode_verify()
420 ASSERT3U(ISP2(dn->dn_datablksz), ==, dn->dn_datablkshift != 0); in dnode_verify()
421 ASSERT3U((dn->dn_nblkptr - 1) * sizeof (blkptr_t) + in dnode_verify()
422 dn->dn_bonuslen, <=, max_bonuslen); in dnode_verify()
424 ASSERT3U(dn->dn_next_nlevels[i], <=, dn->dn_nlevels); in dnode_verify()
427 if (dn->dn_phys->dn_type != DMU_OT_NONE) in dnode_verify()
428 ASSERT3U(dn->dn_phys->dn_nlevels, <=, dn->dn_nlevels); in dnode_verify()
429 ASSERT(DMU_OBJECT_IS_SPECIAL(dn->dn_object) || dn->dn_dbuf != NULL); in dnode_verify()
430 if (dn->dn_dbuf != NULL) { in dnode_verify()
431 ASSERT3P(dn->dn_phys, ==, in dnode_verify()
432 (dnode_phys_t *)dn->dn_dbuf->db.db_data + in dnode_verify()
433 (dn->dn_object % (dn->dn_dbuf->db.db_size >> DNODE_SHIFT))); in dnode_verify()
436 rw_exit(&dn->dn_struct_rwlock); in dnode_verify()
503 dnode_setbonuslen(dnode_t *dn, int newsize, dmu_tx_t *tx) in dnode_setbonuslen() argument
505 ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1); in dnode_setbonuslen()
507 dnode_setdirty(dn, tx); in dnode_setbonuslen()
508 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_setbonuslen()
509 ASSERT3U(newsize, <=, DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) - in dnode_setbonuslen()
510 (dn->dn_nblkptr-1) * sizeof (blkptr_t)); in dnode_setbonuslen()
512 if (newsize < dn->dn_bonuslen) { in dnode_setbonuslen()
514 size_t diff = dn->dn_bonuslen - newsize; in dnode_setbonuslen()
515 char *data_end = ((char *)dn->dn_bonus->db.db_data) + newsize; in dnode_setbonuslen()
519 dn->dn_bonuslen = newsize; in dnode_setbonuslen()
521 dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = DN_ZERO_BONUSLEN; in dnode_setbonuslen()
523 dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = dn->dn_bonuslen; in dnode_setbonuslen()
524 rw_exit(&dn->dn_struct_rwlock); in dnode_setbonuslen()
528 dnode_setbonus_type(dnode_t *dn, dmu_object_type_t newtype, dmu_tx_t *tx) in dnode_setbonus_type() argument
530 ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1); in dnode_setbonus_type()
531 dnode_setdirty(dn, tx); in dnode_setbonus_type()
532 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_setbonus_type()
533 dn->dn_bonustype = newtype; in dnode_setbonus_type()
534 dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = dn->dn_bonustype; in dnode_setbonus_type()
535 rw_exit(&dn->dn_struct_rwlock); in dnode_setbonus_type()
539 dnode_rm_spill(dnode_t *dn, dmu_tx_t *tx) in dnode_rm_spill() argument
541 ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1); in dnode_rm_spill()
542 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); in dnode_rm_spill()
543 dnode_setdirty(dn, tx); in dnode_rm_spill()
544 dn->dn_rm_spillblk[tx->tx_txg & TXG_MASK] = DN_KILL_SPILLBLK; in dnode_rm_spill()
545 dn->dn_have_spill = B_FALSE; in dnode_rm_spill()
549 dnode_setdblksz(dnode_t *dn, int size) in dnode_setdblksz() argument
555 1<<(sizeof (dn->dn_phys->dn_datablkszsec) * 8)); in dnode_setdblksz()
556 dn->dn_datablksz = size; in dnode_setdblksz()
557 dn->dn_datablkszsec = size >> SPA_MINBLOCKSHIFT; in dnode_setdblksz()
558 dn->dn_datablkshift = ISP2(size) ? highbit64(size - 1) : 0; in dnode_setdblksz()
565 dnode_t *dn; in dnode_create() local
567 dn = kmem_cache_alloc(dnode_cache, KM_SLEEP); in dnode_create()
568 dn->dn_moved = 0; in dnode_create()
574 dn->dn_object = object; in dnode_create()
575 dn->dn_dbuf = db; in dnode_create()
576 dn->dn_handle = dnh; in dnode_create()
577 dn->dn_phys = dnp; in dnode_create()
580 dnode_setdblksz(dn, dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT); in dnode_create()
582 dn->dn_datablksz = 0; in dnode_create()
583 dn->dn_datablkszsec = 0; in dnode_create()
584 dn->dn_datablkshift = 0; in dnode_create()
586 dn->dn_indblkshift = dnp->dn_indblkshift; in dnode_create()
587 dn->dn_nlevels = dnp->dn_nlevels; in dnode_create()
588 dn->dn_type = dnp->dn_type; in dnode_create()
589 dn->dn_nblkptr = dnp->dn_nblkptr; in dnode_create()
590 dn->dn_checksum = dnp->dn_checksum; in dnode_create()
591 dn->dn_compress = dnp->dn_compress; in dnode_create()
592 dn->dn_bonustype = dnp->dn_bonustype; in dnode_create()
593 dn->dn_bonuslen = dnp->dn_bonuslen; in dnode_create()
594 dn->dn_num_slots = dnp->dn_extra_slots + 1; in dnode_create()
595 dn->dn_maxblkid = dnp->dn_maxblkid; in dnode_create()
596 dn->dn_have_spill = ((dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) != 0); in dnode_create()
597 dn->dn_id_flags = 0; in dnode_create()
599 dmu_zfetch_init(&dn->dn_zfetch, dn); in dnode_create()
601 ASSERT(DMU_OT_IS_VALID(dn->dn_phys->dn_type)); in dnode_create()
615 list_insert_head(&os->os_dnodes, dn); in dnode_create()
622 dn->dn_objset = os; in dnode_create()
624 dnh->dnh_dnode = dn; in dnode_create()
629 return (dn); in dnode_create()
636 dnode_destroy(dnode_t *dn) in dnode_destroy() argument
638 objset_t *os = dn->dn_objset; in dnode_destroy()
641 ASSERT((dn->dn_id_flags & DN_ID_NEW_EXIST) == 0); in dnode_destroy()
644 POINTER_INVALIDATE(&dn->dn_objset); in dnode_destroy()
645 if (!DMU_OBJECT_IS_SPECIAL(dn->dn_object)) { in dnode_destroy()
646 list_remove(&os->os_dnodes, dn); in dnode_destroy()
654 if (!zrl_is_locked(&dn->dn_handle->dnh_zrlock)) in dnode_destroy()
655 zrl_remove(&dn->dn_handle->dnh_zrlock); in dnode_destroy()
657 dn->dn_allocated_txg = 0; in dnode_destroy()
658 dn->dn_free_txg = 0; in dnode_destroy()
659 dn->dn_assigned_txg = 0; in dnode_destroy()
660 dn->dn_dirty_txg = 0; in dnode_destroy()
662 dn->dn_dirtyctx = 0; in dnode_destroy()
663 dn->dn_dirtyctx_firstset = NULL; in dnode_destroy()
664 if (dn->dn_bonus != NULL) { in dnode_destroy()
665 mutex_enter(&dn->dn_bonus->db_mtx); in dnode_destroy()
666 dbuf_destroy(dn->dn_bonus); in dnode_destroy()
667 dn->dn_bonus = NULL; in dnode_destroy()
669 dn->dn_zio = NULL; in dnode_destroy()
671 dn->dn_have_spill = B_FALSE; in dnode_destroy()
672 dn->dn_oldused = 0; in dnode_destroy()
673 dn->dn_oldflags = 0; in dnode_destroy()
674 dn->dn_olduid = 0; in dnode_destroy()
675 dn->dn_oldgid = 0; in dnode_destroy()
676 dn->dn_oldprojid = ZFS_DEFAULT_PROJID; in dnode_destroy()
677 dn->dn_newuid = 0; in dnode_destroy()
678 dn->dn_newgid = 0; in dnode_destroy()
679 dn->dn_newprojid = ZFS_DEFAULT_PROJID; in dnode_destroy()
680 dn->dn_id_flags = 0; in dnode_destroy()
682 dmu_zfetch_fini(&dn->dn_zfetch); in dnode_destroy()
683 kmem_cache_free(dnode_cache, dn); in dnode_destroy()
691 dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs, in dnode_allocate() argument
698 spa_maxdnodesize(dmu_objset_spa(dn->dn_objset))); in dnode_allocate()
700 spa_maxblocksize(dmu_objset_spa(dn->dn_objset))); in dnode_allocate()
712 dn->dn_objset, (u_longlong_t)dn->dn_object, in dnode_allocate()
716 ASSERT(dn->dn_type == DMU_OT_NONE); in dnode_allocate()
717 ASSERT0(memcmp(dn->dn_phys, &dnode_phys_zero, sizeof (dnode_phys_t))); in dnode_allocate()
718 ASSERT(dn->dn_phys->dn_type == DMU_OT_NONE); in dnode_allocate()
726 ASSERT(dn->dn_type == DMU_OT_NONE); in dnode_allocate()
727 ASSERT0(dn->dn_maxblkid); in dnode_allocate()
728 ASSERT0(dn->dn_allocated_txg); in dnode_allocate()
729 ASSERT0(dn->dn_assigned_txg); in dnode_allocate()
730 ASSERT(zfs_refcount_is_zero(&dn->dn_tx_holds)); in dnode_allocate()
731 ASSERT3U(zfs_refcount_count(&dn->dn_holds), <=, 1); in dnode_allocate()
732 ASSERT(avl_is_empty(&dn->dn_dbufs)); in dnode_allocate()
735 ASSERT0(dn->dn_next_nblkptr[i]); in dnode_allocate()
736 ASSERT0(dn->dn_next_nlevels[i]); in dnode_allocate()
737 ASSERT0(dn->dn_next_indblkshift[i]); in dnode_allocate()
738 ASSERT0(dn->dn_next_bonuslen[i]); in dnode_allocate()
739 ASSERT0(dn->dn_next_bonustype[i]); in dnode_allocate()
740 ASSERT0(dn->dn_rm_spillblk[i]); in dnode_allocate()
741 ASSERT0(dn->dn_next_blksz[i]); in dnode_allocate()
742 ASSERT0(dn->dn_next_maxblkid[i]); in dnode_allocate()
743 ASSERT(!multilist_link_active(&dn->dn_dirty_link[i])); in dnode_allocate()
744 ASSERT3P(list_head(&dn->dn_dirty_records[i]), ==, NULL); in dnode_allocate()
745 ASSERT3P(dn->dn_free_ranges[i], ==, NULL); in dnode_allocate()
748 dn->dn_type = ot; in dnode_allocate()
749 dnode_setdblksz(dn, blocksize); in dnode_allocate()
750 dn->dn_indblkshift = ibs; in dnode_allocate()
751 dn->dn_nlevels = 1; in dnode_allocate()
752 dn->dn_num_slots = dn_slots; in dnode_allocate()
754 dn->dn_nblkptr = 1; in dnode_allocate()
756 dn->dn_nblkptr = MIN(DN_MAX_NBLKPTR, in dnode_allocate()
761 dn->dn_bonustype = bonustype; in dnode_allocate()
762 dn->dn_bonuslen = bonuslen; in dnode_allocate()
763 dn->dn_checksum = ZIO_CHECKSUM_INHERIT; in dnode_allocate()
764 dn->dn_compress = ZIO_COMPRESS_INHERIT; in dnode_allocate()
765 dn->dn_dirtyctx = 0; in dnode_allocate()
767 dn->dn_free_txg = 0; in dnode_allocate()
768 dn->dn_dirtyctx_firstset = NULL; in dnode_allocate()
769 dn->dn_dirty_txg = 0; in dnode_allocate()
771 dn->dn_allocated_txg = tx->tx_txg; in dnode_allocate()
772 dn->dn_id_flags = 0; in dnode_allocate()
774 dnode_setdirty(dn, tx); in dnode_allocate()
775 dn->dn_next_indblkshift[tx->tx_txg & TXG_MASK] = ibs; in dnode_allocate()
776 dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = dn->dn_bonuslen; in dnode_allocate()
777 dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = dn->dn_bonustype; in dnode_allocate()
778 dn->dn_next_blksz[tx->tx_txg & TXG_MASK] = dn->dn_datablksz; in dnode_allocate()
782 dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, in dnode_reallocate() argument
790 spa_maxblocksize(dmu_objset_spa(dn->dn_objset))); in dnode_reallocate()
792 ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT || dmu_tx_private_ok(tx)); in dnode_reallocate()
799 DN_BONUS_SIZE(spa_maxdnodesize(dmu_objset_spa(dn->dn_objset)))); in dnode_reallocate()
802 dnode_free_interior_slots(dn); in dnode_reallocate()
806 dnode_evict_dbufs(dn); in dnode_reallocate()
808 dn->dn_id_flags = 0; in dnode_reallocate()
810 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_reallocate()
811 dnode_setdirty(dn, tx); in dnode_reallocate()
812 if (dn->dn_datablksz != blocksize) { in dnode_reallocate()
814 ASSERT0(dn->dn_maxblkid); in dnode_reallocate()
815 ASSERT(BP_IS_HOLE(&dn->dn_phys->dn_blkptr[0]) || in dnode_reallocate()
816 dnode_block_freed(dn, 0)); in dnode_reallocate()
818 dnode_setdblksz(dn, blocksize); in dnode_reallocate()
819 dn->dn_next_blksz[tx->tx_txg & TXG_MASK] = blocksize; in dnode_reallocate()
821 if (dn->dn_bonuslen != bonuslen) in dnode_reallocate()
822 dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = bonuslen; in dnode_reallocate()
830 if (dn->dn_bonustype != bonustype) in dnode_reallocate()
831 dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = bonustype; in dnode_reallocate()
832 if (dn->dn_nblkptr != nblkptr) in dnode_reallocate()
833 dn->dn_next_nblkptr[tx->tx_txg & TXG_MASK] = nblkptr; in dnode_reallocate()
834 if (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR && !keep_spill) { in dnode_reallocate()
835 dbuf_rm_spill(dn, tx); in dnode_reallocate()
836 dnode_rm_spill(dn, tx); in dnode_reallocate()
839 rw_exit(&dn->dn_struct_rwlock); in dnode_reallocate()
842 dn->dn_type = ot; in dnode_reallocate()
845 mutex_enter(&dn->dn_mtx); in dnode_reallocate()
846 dn->dn_bonustype = bonustype; in dnode_reallocate()
847 dn->dn_bonuslen = bonuslen; in dnode_reallocate()
848 dn->dn_num_slots = dn_slots; in dnode_reallocate()
849 dn->dn_nblkptr = nblkptr; in dnode_reallocate()
850 dn->dn_checksum = ZIO_CHECKSUM_INHERIT; in dnode_reallocate()
851 dn->dn_compress = ZIO_COMPRESS_INHERIT; in dnode_reallocate()
852 ASSERT3U(dn->dn_nblkptr, <=, DN_MAX_NBLKPTR); in dnode_reallocate()
855 if (dn->dn_bonus) { in dnode_reallocate()
856 dn->dn_bonus->db.db_size = in dnode_reallocate()
857 DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) - in dnode_reallocate()
858 (dn->dn_nblkptr-1) * sizeof (blkptr_t); in dnode_reallocate()
859 ASSERT(dn->dn_bonuslen <= dn->dn_bonus->db.db_size); in dnode_reallocate()
862 dn->dn_allocated_txg = tx->tx_txg; in dnode_reallocate()
863 mutex_exit(&dn->dn_mtx); in dnode_reallocate()
1216 dnode_t *dn = dnh->dnh_dnode; in dnode_check_slots_free() local
1218 if (dn == DN_SLOT_FREE) { in dnode_check_slots_free()
1220 } else if (DN_SLOT_IS_PTR(dn)) { in dnode_check_slots_free()
1221 mutex_enter(&dn->dn_mtx); in dnode_check_slots_free()
1222 boolean_t can_free = (dn->dn_type == DMU_OT_NONE && in dnode_check_slots_free()
1223 zfs_refcount_is_zero(&dn->dn_holds) && in dnode_check_slots_free()
1224 !DNODE_IS_DIRTY(dn)); in dnode_check_slots_free()
1225 mutex_exit(&dn->dn_mtx); in dnode_check_slots_free()
1258 dnode_free_interior_slots(dnode_t *dn) in dnode_free_interior_slots() argument
1260 dnode_children_t *children = dmu_buf_get_user(&dn->dn_dbuf->db); in dnode_free_interior_slots()
1261 int epb = dn->dn_dbuf->db.db_size >> DNODE_SHIFT; in dnode_free_interior_slots()
1262 int idx = (dn->dn_object & (epb - 1)) + 1; in dnode_free_interior_slots()
1263 int slots = dn->dn_num_slots - 1; in dnode_free_interior_slots()
1282 dnode_t *dn = dnh->dnh_dnode; in dnode_special_close() local
1288 mutex_enter(&dn->dn_mtx); in dnode_special_close()
1289 if (zfs_refcount_count(&dn->dn_holds) > 0) in dnode_special_close()
1290 cv_wait(&dn->dn_nodnholds, &dn->dn_mtx); in dnode_special_close()
1291 mutex_exit(&dn->dn_mtx); in dnode_special_close()
1292 ASSERT3U(zfs_refcount_count(&dn->dn_holds), ==, 0); in dnode_special_close()
1294 ASSERT(dn->dn_dbuf == NULL || in dnode_special_close()
1295 dmu_buf_get_user(&dn->dn_dbuf->db) == NULL); in dnode_special_close()
1297 dnode_destroy(dn); /* implicit zrl_remove() */ in dnode_special_close()
1306 dnode_t *dn; in dnode_special_open() local
1311 dn = dnode_create(os, dnp, NULL, object, dnh); in dnode_special_open()
1312 DNODE_VERIFY(dn); in dnode_special_open()
1326 dnode_t *dn; in dnode_buf_evict_async() local
1340 dn = dnh->dnh_dnode; in dnode_buf_evict_async()
1347 ASSERT(zfs_refcount_is_zero(&dn->dn_holds)); in dnode_buf_evict_async()
1348 ASSERT(zfs_refcount_is_zero(&dn->dn_tx_holds)); in dnode_buf_evict_async()
1350 dnode_destroy(dn); /* implicit zrl_remove() for first slot */ in dnode_buf_evict_async()
1395 dnode_t *mdn, *dn; in dnode_hold_impl() local
1420 dn = DMU_USERUSED_DNODE(os); in dnode_hold_impl()
1422 dn = DMU_GROUPUSED_DNODE(os); in dnode_hold_impl()
1424 dn = DMU_PROJECTUSED_DNODE(os); in dnode_hold_impl()
1425 if (dn == NULL) in dnode_hold_impl()
1427 type = dn->dn_type; in dnode_hold_impl()
1432 DNODE_VERIFY(dn); in dnode_hold_impl()
1435 (void) zfs_refcount_add(&dn->dn_holds, tag); in dnode_hold_impl()
1436 *dnp = dn; in dnode_hold_impl()
1538 dn = dnh->dnh_dnode; in dnode_hold_impl()
1563 dn = dnh->dnh_dnode; in dnode_hold_impl()
1565 dn = dnode_create(os, dn_block + idx, db, in dnode_hold_impl()
1570 mutex_enter(&dn->dn_mtx); in dnode_hold_impl()
1571 if (dn->dn_type == DMU_OT_NONE || dn->dn_free_txg != 0) { in dnode_hold_impl()
1573 mutex_exit(&dn->dn_mtx); in dnode_hold_impl()
1581 mutex_exit(&dn->dn_mtx); in dnode_hold_impl()
1629 dn = dnh->dnh_dnode; in dnode_hold_impl()
1631 dn = dnode_create(os, dn_block + idx, db, in dnode_hold_impl()
1635 mutex_enter(&dn->dn_mtx); in dnode_hold_impl()
1636 if (!zfs_refcount_is_zero(&dn->dn_holds) || dn->dn_free_txg) { in dnode_hold_impl()
1638 mutex_exit(&dn->dn_mtx); in dnode_hold_impl()
1646 mutex_exit(&dn->dn_mtx); in dnode_hold_impl()
1659 ASSERT0(dn->dn_free_txg); in dnode_hold_impl()
1661 if (zfs_refcount_add(&dn->dn_holds, tag) == 1) in dnode_hold_impl()
1664 mutex_exit(&dn->dn_mtx); in dnode_hold_impl()
1669 DNODE_VERIFY(dn); in dnode_hold_impl()
1671 ASSERT3P(dn->dn_dbuf, ==, db); in dnode_hold_impl()
1672 ASSERT3U(dn->dn_object, ==, object); in dnode_hold_impl()
1675 *dnp = dn; in dnode_hold_impl()
1695 dnode_add_ref(dnode_t *dn, const void *tag) in dnode_add_ref() argument
1697 mutex_enter(&dn->dn_mtx); in dnode_add_ref()
1698 if (zfs_refcount_is_zero(&dn->dn_holds)) { in dnode_add_ref()
1699 mutex_exit(&dn->dn_mtx); in dnode_add_ref()
1702 VERIFY(1 < zfs_refcount_add(&dn->dn_holds, tag)); in dnode_add_ref()
1703 mutex_exit(&dn->dn_mtx); in dnode_add_ref()
1708 dnode_rele(dnode_t *dn, const void *tag) in dnode_rele() argument
1710 mutex_enter(&dn->dn_mtx); in dnode_rele()
1711 dnode_rele_and_unlock(dn, tag, B_FALSE); in dnode_rele()
1715 dnode_rele_and_unlock(dnode_t *dn, const void *tag, boolean_t evicting) in dnode_rele_and_unlock() argument
1719 dmu_buf_impl_t *db = dn->dn_dbuf; in dnode_rele_and_unlock()
1720 dnode_handle_t *dnh = dn->dn_handle; in dnode_rele_and_unlock()
1722 refs = zfs_refcount_remove(&dn->dn_holds, tag); in dnode_rele_and_unlock()
1724 cv_broadcast(&dn->dn_nodnholds); in dnode_rele_and_unlock()
1725 mutex_exit(&dn->dn_mtx); in dnode_rele_and_unlock()
1777 dnode_is_dirty(dnode_t *dn) in dnode_is_dirty() argument
1779 mutex_enter(&dn->dn_mtx); in dnode_is_dirty()
1782 if (multilist_link_active(&dn->dn_dirty_link[i]) || in dnode_is_dirty()
1783 !list_is_empty(&dn->dn_dirty_records[i])) { in dnode_is_dirty()
1784 mutex_exit(&dn->dn_mtx); in dnode_is_dirty()
1789 mutex_exit(&dn->dn_mtx); in dnode_is_dirty()
1795 dnode_setdirty(dnode_t *dn, dmu_tx_t *tx) in dnode_setdirty() argument
1797 objset_t *os = dn->dn_objset; in dnode_setdirty()
1800 if (DMU_OBJECT_IS_SPECIAL(dn->dn_object)) { in dnode_setdirty()
1805 DNODE_VERIFY(dn); in dnode_setdirty()
1808 mutex_enter(&dn->dn_mtx); in dnode_setdirty()
1809 ASSERT(dn->dn_phys->dn_type || dn->dn_allocated_txg); in dnode_setdirty()
1810 ASSERT(dn->dn_free_txg == 0 || dn->dn_free_txg >= txg); in dnode_setdirty()
1811 mutex_exit(&dn->dn_mtx); in dnode_setdirty()
1817 dmu_objset_userquota_get_ids(dn, B_TRUE, tx); in dnode_setdirty()
1820 multilist_sublist_t *mls = multilist_sublist_lock_obj(dirtylist, dn); in dnode_setdirty()
1825 if (multilist_link_active(&dn->dn_dirty_link[txg & TXG_MASK])) { in dnode_setdirty()
1830 ASSERT(!zfs_refcount_is_zero(&dn->dn_holds) || in dnode_setdirty()
1831 !avl_is_empty(&dn->dn_dbufs)); in dnode_setdirty()
1832 ASSERT(dn->dn_datablksz != 0); in dnode_setdirty()
1833 ASSERT0(dn->dn_next_bonuslen[txg & TXG_MASK]); in dnode_setdirty()
1834 ASSERT0(dn->dn_next_blksz[txg & TXG_MASK]); in dnode_setdirty()
1835 ASSERT0(dn->dn_next_bonustype[txg & TXG_MASK]); in dnode_setdirty()
1838 (u_longlong_t)dn->dn_object, (u_longlong_t)txg); in dnode_setdirty()
1840 multilist_sublist_insert_head(mls, dn); in dnode_setdirty()
1853 VERIFY(dnode_add_ref(dn, (void *)(uintptr_t)tx->tx_txg)); in dnode_setdirty()
1855 (void) dbuf_dirty(dn->dn_dbuf, tx); in dnode_setdirty()
1861 dnode_free(dnode_t *dn, dmu_tx_t *tx) in dnode_free() argument
1863 mutex_enter(&dn->dn_mtx); in dnode_free()
1864 if (dn->dn_type == DMU_OT_NONE || dn->dn_free_txg) { in dnode_free()
1865 mutex_exit(&dn->dn_mtx); in dnode_free()
1868 dn->dn_free_txg = tx->tx_txg; in dnode_free()
1869 mutex_exit(&dn->dn_mtx); in dnode_free()
1871 dnode_setdirty(dn, tx); in dnode_free()
1879 dnode_set_blksz(dnode_t *dn, uint64_t size, int ibs, dmu_tx_t *tx) in dnode_set_blksz() argument
1884 ASSERT3U(size, <=, spa_maxblocksize(dmu_objset_spa(dn->dn_objset))); in dnode_set_blksz()
1890 if (ibs == dn->dn_indblkshift) in dnode_set_blksz()
1893 if (size == dn->dn_datablksz && ibs == 0) in dnode_set_blksz()
1896 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_set_blksz()
1899 if (dn->dn_maxblkid != 0) in dnode_set_blksz()
1902 mutex_enter(&dn->dn_dbufs_mtx); in dnode_set_blksz()
1903 for (db = avl_first(&dn->dn_dbufs); db != NULL; in dnode_set_blksz()
1904 db = AVL_NEXT(&dn->dn_dbufs, db)) { in dnode_set_blksz()
1907 mutex_exit(&dn->dn_dbufs_mtx); in dnode_set_blksz()
1911 mutex_exit(&dn->dn_dbufs_mtx); in dnode_set_blksz()
1913 if (ibs && dn->dn_nlevels != 1) in dnode_set_blksz()
1916 dnode_setdirty(dn, tx); in dnode_set_blksz()
1917 if (size != dn->dn_datablksz) { in dnode_set_blksz()
1919 err = dbuf_hold_impl(dn, 0, 0, TRUE, FALSE, FTAG, &db); in dnode_set_blksz()
1926 dnode_setdblksz(dn, size); in dnode_set_blksz()
1927 dn->dn_next_blksz[tx->tx_txg & TXG_MASK] = size; in dnode_set_blksz()
1932 dn->dn_indblkshift = ibs; in dnode_set_blksz()
1933 dn->dn_next_indblkshift[tx->tx_txg & TXG_MASK] = ibs; in dnode_set_blksz()
1936 rw_exit(&dn->dn_struct_rwlock); in dnode_set_blksz()
1940 rw_exit(&dn->dn_struct_rwlock); in dnode_set_blksz()
1945 dnode_set_nlevels_impl(dnode_t *dn, int new_nlevels, dmu_tx_t *tx) in dnode_set_nlevels_impl() argument
1948 int old_nlevels = dn->dn_nlevels; in dnode_set_nlevels_impl()
1953 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); in dnode_set_nlevels_impl()
1955 ASSERT3U(new_nlevels, >, dn->dn_nlevels); in dnode_set_nlevels_impl()
1956 dn->dn_nlevels = new_nlevels; in dnode_set_nlevels_impl()
1958 ASSERT3U(new_nlevels, >, dn->dn_next_nlevels[txgoff]); in dnode_set_nlevels_impl()
1959 dn->dn_next_nlevels[txgoff] = new_nlevels; in dnode_set_nlevels_impl()
1962 db = dbuf_hold_level(dn, old_nlevels, 0, FTAG); in dnode_set_nlevels_impl()
1968 mutex_enter(&dn->dn_mtx); in dnode_set_nlevels_impl()
1970 list = &dn->dn_dirty_records[txgoff]; in dnode_set_nlevels_impl()
1972 dr_next = list_next(&dn->dn_dirty_records[txgoff], dr); in dnode_set_nlevels_impl()
1979 list_remove(&dn->dn_dirty_records[txgoff], dr); in dnode_set_nlevels_impl()
1985 mutex_exit(&dn->dn_mtx); in dnode_set_nlevels_impl()
1989 dnode_set_nlevels(dnode_t *dn, int nlevels, dmu_tx_t *tx) in dnode_set_nlevels() argument
1993 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_set_nlevels()
1995 if (dn->dn_nlevels == nlevels) { in dnode_set_nlevels()
1998 } else if (nlevels < dn->dn_nlevels) { in dnode_set_nlevels()
2003 dnode_set_nlevels_impl(dn, nlevels, tx); in dnode_set_nlevels()
2006 rw_exit(&dn->dn_struct_rwlock); in dnode_set_nlevels()
2012 dnode_new_blkid(dnode_t *dn, uint64_t blkid, dmu_tx_t *tx, boolean_t have_read, in dnode_new_blkid() argument
2021 RW_READ_HELD(&dn->dn_struct_rwlock) : in dnode_new_blkid()
2022 RW_WRITE_HELD(&dn->dn_struct_rwlock)); in dnode_new_blkid()
2029 if (blkid <= dn->dn_maxblkid) in dnode_new_blkid()
2032 if (!rw_tryupgrade(&dn->dn_struct_rwlock)) { in dnode_new_blkid()
2033 rw_exit(&dn->dn_struct_rwlock); in dnode_new_blkid()
2034 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_new_blkid()
2042 if (!force && blkid <= dn->dn_maxblkid) in dnode_new_blkid()
2050 dn->dn_maxblkid = blkid; in dnode_new_blkid()
2051 dn->dn_next_maxblkid[tx->tx_txg & TXG_MASK] = in dnode_new_blkid()
2059 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; in dnode_new_blkid()
2060 for (sz = dn->dn_nblkptr; in dnode_new_blkid()
2061 sz <= blkid && sz >= dn->dn_nblkptr; sz <<= epbs) in dnode_new_blkid()
2067 if (new_nlevels > dn->dn_nlevels) in dnode_new_blkid()
2068 dnode_set_nlevels_impl(dn, new_nlevels, tx); in dnode_new_blkid()
2070 ASSERT3U(dn->dn_nlevels, >=, new_nlevels); in dnode_new_blkid()
2075 rw_downgrade(&dn->dn_struct_rwlock); in dnode_new_blkid()
2079 dnode_dirty_l1(dnode_t *dn, uint64_t l1blkid, dmu_tx_t *tx) in dnode_dirty_l1() argument
2081 dmu_buf_impl_t *db = dbuf_hold_level(dn, 1, l1blkid, FTAG); in dnode_dirty_l1()
2093 dnode_dirty_l1range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid, in dnode_dirty_l1range() argument
2102 mutex_enter(&dn->dn_dbufs_mtx); in dnode_dirty_l1range()
2109 db = avl_find(&dn->dn_dbufs, db_search, &where); in dnode_dirty_l1range()
2111 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); in dnode_dirty_l1range()
2132 mutex_exit(&dn->dn_dbufs_mtx); in dnode_dirty_l1range()
2133 dnode_dirty_l1(dn, db->db_blkid, tx); in dnode_dirty_l1range()
2134 mutex_enter(&dn->dn_dbufs_mtx); in dnode_dirty_l1range()
2144 db = avl_find(&dn->dn_dbufs, db_search, &where); in dnode_dirty_l1range()
2146 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); in dnode_dirty_l1range()
2147 for (; db != NULL; db = AVL_NEXT(&dn->dn_dbufs, db)) { in dnode_dirty_l1range()
2155 mutex_exit(&dn->dn_dbufs_mtx); in dnode_dirty_l1range()
2159 dnode_set_dirtyctx(dnode_t *dn, dmu_tx_t *tx, const void *tag) in dnode_set_dirtyctx() argument
2165 if (dn->dn_dirtyctx == DN_UNDIRTIED) { in dnode_set_dirtyctx()
2166 dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; in dnode_set_dirtyctx()
2171 if (!BP_IS_HOLE(dn->dn_objset->os_rootbp)) { in dnode_set_dirtyctx()
2173 dn->dn_dirtyctx = DN_DIRTY_SYNC; in dnode_set_dirtyctx()
2175 dn->dn_dirtyctx = DN_DIRTY_OPEN; in dnode_set_dirtyctx()
2176 dn->dn_dirtyctx_firstset = tag; in dnode_set_dirtyctx()
2185 dnode_partial_zero(dnode_t *dn, uint64_t off, uint64_t blkoff, uint64_t len, in dnode_partial_zero() argument
2191 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dnode_partial_zero()
2192 res = dbuf_hold_impl(dn, 0, dbuf_whichblock(dn, 0, off), TRUE, FALSE, in dnode_partial_zero()
2194 rw_exit(&dn->dn_struct_rwlock); in dnode_partial_zero()
2216 dnode_free_range(dnode_t *dn, uint64_t off, uint64_t len, dmu_tx_t *tx) in dnode_free_range() argument
2223 blksz = dn->dn_datablksz; in dnode_free_range()
2224 blkshift = dn->dn_datablkshift; in dnode_free_range()
2225 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; in dnode_free_range()
2238 if ((off >> blkshift) > dn->dn_maxblkid) in dnode_free_range()
2241 ASSERT(dn->dn_maxblkid == 0); in dnode_free_range()
2248 if (dn->dn_nlevels > 1) { in dnode_free_range()
2249 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_free_range()
2250 dnode_dirty_l1(dn, 0, tx); in dnode_free_range()
2251 rw_exit(&dn->dn_struct_rwlock); in dnode_free_range()
2269 dnode_partial_zero(dn, off, blkoff, head, tx); in dnode_free_range()
2279 if ((off >> blkshift) > dn->dn_maxblkid) in dnode_free_range()
2293 dnode_partial_zero(dn, off + len, 0, tail, tx); in dnode_free_range()
2328 if (dn->dn_nlevels > 1) { in dnode_free_range()
2329 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_free_range()
2333 dnode_dirty_l1(dn, first, tx); in dnode_free_range()
2335 last = dn->dn_maxblkid >> epbs; in dnode_free_range()
2339 dnode_dirty_l1(dn, last, tx); in dnode_free_range()
2341 dnode_dirty_l1range(dn, first, last, tx); in dnode_free_range()
2343 int shift = dn->dn_datablkshift + dn->dn_indblkshift - in dnode_free_range()
2353 int err = dnode_next_offset(dn, DNODE_FIND_HAVELOCK, in dnode_free_range()
2372 dnode_dirty_l1(dn, i, tx); in dnode_free_range()
2374 rw_exit(&dn->dn_struct_rwlock); in dnode_free_range()
2382 mutex_enter(&dn->dn_mtx); in dnode_free_range()
2385 if (dn->dn_free_ranges[txgoff] == NULL) { in dnode_free_range()
2386 dn->dn_free_ranges[txgoff] = range_tree_create(NULL, in dnode_free_range()
2389 range_tree_clear(dn->dn_free_ranges[txgoff], blkid, nblks); in dnode_free_range()
2390 range_tree_add(dn->dn_free_ranges[txgoff], blkid, nblks); in dnode_free_range()
2392 dprintf_dnode(dn, "blkid=%llu nblks=%llu txg=%llu\n", in dnode_free_range()
2395 mutex_exit(&dn->dn_mtx); in dnode_free_range()
2397 dbuf_free_range(dn, blkid, blkid + nblks - 1, tx); in dnode_free_range()
2398 dnode_setdirty(dn, tx); in dnode_free_range()
2402 dnode_spill_freed(dnode_t *dn) in dnode_spill_freed() argument
2406 mutex_enter(&dn->dn_mtx); in dnode_spill_freed()
2408 if (dn->dn_rm_spillblk[i] == DN_KILL_SPILLBLK) in dnode_spill_freed()
2411 mutex_exit(&dn->dn_mtx); in dnode_spill_freed()
2417 dnode_block_freed(dnode_t *dn, uint64_t blkid) in dnode_block_freed() argument
2424 if (dn->dn_free_txg) in dnode_block_freed()
2428 return (dnode_spill_freed(dn)); in dnode_block_freed()
2430 mutex_enter(&dn->dn_mtx); in dnode_block_freed()
2432 if (dn->dn_free_ranges[i] != NULL && in dnode_block_freed()
2433 range_tree_contains(dn->dn_free_ranges[i], blkid, 1)) in dnode_block_freed()
2436 mutex_exit(&dn->dn_mtx); in dnode_block_freed()
2442 dnode_diduse_space(dnode_t *dn, int64_t delta) in dnode_diduse_space() argument
2445 dprintf_dnode(dn, "dn=%p dnp=%p used=%llu delta=%lld\n", in dnode_diduse_space()
2446 dn, dn->dn_phys, in dnode_diduse_space()
2447 (u_longlong_t)dn->dn_phys->dn_used, in dnode_diduse_space()
2450 mutex_enter(&dn->dn_mtx); in dnode_diduse_space()
2451 space = DN_USED_BYTES(dn->dn_phys); in dnode_diduse_space()
2458 if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_DNODE_BYTES) { in dnode_diduse_space()
2459 ASSERT((dn->dn_phys->dn_flags & DNODE_FLAG_USED_BYTES) == 0); in dnode_diduse_space()
2461 dn->dn_phys->dn_used = space >> DEV_BSHIFT; in dnode_diduse_space()
2463 dn->dn_phys->dn_used = space; in dnode_diduse_space()
2464 dn->dn_phys->dn_flags |= DNODE_FLAG_USED_BYTES; in dnode_diduse_space()
2466 mutex_exit(&dn->dn_mtx); in dnode_diduse_space()
2488 dnode_next_offset_level(dnode_t *dn, int flags, uint64_t *offset, in dnode_next_offset_level() argument
2493 uint64_t epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; in dnode_next_offset_level()
2499 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); in dnode_next_offset_level()
2505 if (lvl == dn->dn_phys->dn_nlevels) { in dnode_next_offset_level()
2507 epb = dn->dn_phys->dn_nblkptr; in dnode_next_offset_level()
2508 data = dn->dn_phys->dn_blkptr; in dnode_next_offset_level()
2510 uint64_t blkid = dbuf_whichblock(dn, lvl, *offset); in dnode_next_offset_level()
2511 error = dbuf_hold_impl(dn, lvl, blkid, TRUE, FALSE, FTAG, &db); in dnode_next_offset_level()
2548 ASSERT(dn->dn_type == DMU_OT_DNODE); in dnode_next_offset_level()
2565 span = (lvl - 1) * epbs + dn->dn_datablkshift; in dnode_next_offset_level()
2576 ASSERT3U((lvl - 1), ==, dn->dn_phys->dn_nlevels - 1); in dnode_next_offset_level()
2642 dnode_next_offset(dnode_t *dn, int flags, uint64_t *offset, in dnode_next_offset() argument
2650 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dnode_next_offset()
2652 if (dn->dn_phys->dn_nlevels == 0) { in dnode_next_offset()
2657 if (dn->dn_datablkshift == 0) { in dnode_next_offset()
2658 if (*offset < dn->dn_datablksz) { in dnode_next_offset()
2660 *offset = dn->dn_datablksz; in dnode_next_offset()
2667 maxlvl = dn->dn_phys->dn_nlevels; in dnode_next_offset()
2670 error = dnode_next_offset_level(dn, in dnode_next_offset()
2677 error = dnode_next_offset_level(dn, in dnode_next_offset()
2695 rw_exit(&dn->dn_struct_rwlock); in dnode_next_offset()