Lines Matching refs:dn
115 dnode_t *dn = arg; in dnode_cons() local
117 rw_init(&dn->dn_struct_rwlock, NULL, RW_NOLOCKDEP, NULL); in dnode_cons()
118 mutex_init(&dn->dn_mtx, NULL, MUTEX_DEFAULT, NULL); in dnode_cons()
119 mutex_init(&dn->dn_dbufs_mtx, NULL, MUTEX_DEFAULT, NULL); in dnode_cons()
120 cv_init(&dn->dn_notxholds, NULL, CV_DEFAULT, NULL); in dnode_cons()
121 cv_init(&dn->dn_nodnholds, NULL, CV_DEFAULT, NULL); in dnode_cons()
127 zfs_refcount_create_untracked(&dn->dn_holds); in dnode_cons()
128 zfs_refcount_create(&dn->dn_tx_holds); in dnode_cons()
129 list_link_init(&dn->dn_link); in dnode_cons()
131 bzero(&dn->dn_next_type[0], sizeof (dn->dn_next_type)); in dnode_cons()
132 bzero(&dn->dn_next_nblkptr[0], sizeof (dn->dn_next_nblkptr)); in dnode_cons()
133 bzero(&dn->dn_next_nlevels[0], sizeof (dn->dn_next_nlevels)); in dnode_cons()
134 bzero(&dn->dn_next_indblkshift[0], sizeof (dn->dn_next_indblkshift)); in dnode_cons()
135 bzero(&dn->dn_next_bonustype[0], sizeof (dn->dn_next_bonustype)); in dnode_cons()
136 bzero(&dn->dn_rm_spillblk[0], sizeof (dn->dn_rm_spillblk)); in dnode_cons()
137 bzero(&dn->dn_next_bonuslen[0], sizeof (dn->dn_next_bonuslen)); in dnode_cons()
138 bzero(&dn->dn_next_blksz[0], sizeof (dn->dn_next_blksz)); in dnode_cons()
139 bzero(&dn->dn_next_maxblkid[0], sizeof (dn->dn_next_maxblkid)); in dnode_cons()
142 multilist_link_init(&dn->dn_dirty_link[i]); in dnode_cons()
143 dn->dn_free_ranges[i] = NULL; in dnode_cons()
144 list_create(&dn->dn_dirty_records[i], in dnode_cons()
149 dn->dn_allocated_txg = 0; in dnode_cons()
150 dn->dn_free_txg = 0; in dnode_cons()
151 dn->dn_assigned_txg = 0; in dnode_cons()
152 dn->dn_dirty_txg = 0; in dnode_cons()
153 dn->dn_dirtyctx = 0; in dnode_cons()
154 dn->dn_dirtyctx_firstset = NULL; in dnode_cons()
155 dn->dn_bonus = NULL; in dnode_cons()
156 dn->dn_have_spill = B_FALSE; in dnode_cons()
157 dn->dn_zio = NULL; in dnode_cons()
158 dn->dn_oldused = 0; in dnode_cons()
159 dn->dn_oldflags = 0; in dnode_cons()
160 dn->dn_olduid = 0; in dnode_cons()
161 dn->dn_oldgid = 0; in dnode_cons()
162 dn->dn_oldprojid = ZFS_DEFAULT_PROJID; in dnode_cons()
163 dn->dn_newuid = 0; in dnode_cons()
164 dn->dn_newgid = 0; in dnode_cons()
165 dn->dn_newprojid = ZFS_DEFAULT_PROJID; in dnode_cons()
166 dn->dn_id_flags = 0; in dnode_cons()
168 dn->dn_dbufs_count = 0; in dnode_cons()
169 avl_create(&dn->dn_dbufs, dbuf_compare, sizeof (dmu_buf_impl_t), in dnode_cons()
172 dn->dn_moved = 0; in dnode_cons()
180 dnode_t *dn = arg; in dnode_dest() local
182 rw_destroy(&dn->dn_struct_rwlock); in dnode_dest()
183 mutex_destroy(&dn->dn_mtx); in dnode_dest()
184 mutex_destroy(&dn->dn_dbufs_mtx); in dnode_dest()
185 cv_destroy(&dn->dn_notxholds); in dnode_dest()
186 cv_destroy(&dn->dn_nodnholds); in dnode_dest()
187 zfs_refcount_destroy(&dn->dn_holds); in dnode_dest()
188 zfs_refcount_destroy(&dn->dn_tx_holds); in dnode_dest()
189 ASSERT(!list_link_active(&dn->dn_link)); in dnode_dest()
192 ASSERT(!multilist_link_active(&dn->dn_dirty_link[i])); in dnode_dest()
193 ASSERT3P(dn->dn_free_ranges[i], ==, NULL); in dnode_dest()
194 list_destroy(&dn->dn_dirty_records[i]); in dnode_dest()
195 ASSERT0(dn->dn_next_nblkptr[i]); in dnode_dest()
196 ASSERT0(dn->dn_next_nlevels[i]); in dnode_dest()
197 ASSERT0(dn->dn_next_indblkshift[i]); in dnode_dest()
198 ASSERT0(dn->dn_next_bonustype[i]); in dnode_dest()
199 ASSERT0(dn->dn_rm_spillblk[i]); in dnode_dest()
200 ASSERT0(dn->dn_next_bonuslen[i]); in dnode_dest()
201 ASSERT0(dn->dn_next_blksz[i]); in dnode_dest()
202 ASSERT0(dn->dn_next_maxblkid[i]); in dnode_dest()
205 ASSERT0(dn->dn_allocated_txg); in dnode_dest()
206 ASSERT0(dn->dn_free_txg); in dnode_dest()
207 ASSERT0(dn->dn_assigned_txg); in dnode_dest()
208 ASSERT0(dn->dn_dirty_txg); in dnode_dest()
209 ASSERT0(dn->dn_dirtyctx); in dnode_dest()
210 ASSERT3P(dn->dn_dirtyctx_firstset, ==, NULL); in dnode_dest()
211 ASSERT3P(dn->dn_bonus, ==, NULL); in dnode_dest()
212 ASSERT(!dn->dn_have_spill); in dnode_dest()
213 ASSERT3P(dn->dn_zio, ==, NULL); in dnode_dest()
214 ASSERT0(dn->dn_oldused); in dnode_dest()
215 ASSERT0(dn->dn_oldflags); in dnode_dest()
216 ASSERT0(dn->dn_olduid); in dnode_dest()
217 ASSERT0(dn->dn_oldgid); in dnode_dest()
218 ASSERT0(dn->dn_oldprojid); in dnode_dest()
219 ASSERT0(dn->dn_newuid); in dnode_dest()
220 ASSERT0(dn->dn_newgid); in dnode_dest()
221 ASSERT0(dn->dn_newprojid); in dnode_dest()
222 ASSERT0(dn->dn_id_flags); in dnode_dest()
224 ASSERT0(dn->dn_dbufs_count); in dnode_dest()
225 avl_destroy(&dn->dn_dbufs); in dnode_dest()
260 dnode_verify(dnode_t *dn) in dnode_verify() argument
264 ASSERT(dn->dn_phys); in dnode_verify()
265 ASSERT(dn->dn_objset); in dnode_verify()
266 ASSERT(dn->dn_handle->dnh_dnode == dn); in dnode_verify()
268 ASSERT(DMU_OT_IS_VALID(dn->dn_phys->dn_type)); in dnode_verify()
273 if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { in dnode_verify()
274 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dnode_verify()
277 if (dn->dn_phys->dn_type != DMU_OT_NONE || dn->dn_allocated_txg != 0) { in dnode_verify()
279 int max_bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots); in dnode_verify()
280 ASSERT3U(dn->dn_indblkshift, <=, SPA_MAXBLOCKSHIFT); in dnode_verify()
281 if (dn->dn_datablkshift) { in dnode_verify()
282 ASSERT3U(dn->dn_datablkshift, >=, SPA_MINBLOCKSHIFT); in dnode_verify()
283 ASSERT3U(dn->dn_datablkshift, <=, SPA_MAXBLOCKSHIFT); in dnode_verify()
284 ASSERT3U(1<<dn->dn_datablkshift, ==, dn->dn_datablksz); in dnode_verify()
286 ASSERT3U(dn->dn_nlevels, <=, 30); in dnode_verify()
287 ASSERT(DMU_OT_IS_VALID(dn->dn_type)); in dnode_verify()
288 ASSERT3U(dn->dn_nblkptr, >=, 1); in dnode_verify()
289 ASSERT3U(dn->dn_nblkptr, <=, DN_MAX_NBLKPTR); in dnode_verify()
290 ASSERT3U(dn->dn_bonuslen, <=, max_bonuslen); in dnode_verify()
291 ASSERT3U(dn->dn_datablksz, ==, in dnode_verify()
292 dn->dn_datablkszsec << SPA_MINBLOCKSHIFT); in dnode_verify()
293 ASSERT3U(ISP2(dn->dn_datablksz), ==, dn->dn_datablkshift != 0); in dnode_verify()
294 ASSERT3U((dn->dn_nblkptr - 1) * sizeof (blkptr_t) + in dnode_verify()
295 dn->dn_bonuslen, <=, max_bonuslen); in dnode_verify()
297 ASSERT3U(dn->dn_next_nlevels[i], <=, dn->dn_nlevels); in dnode_verify()
300 if (dn->dn_phys->dn_type != DMU_OT_NONE) in dnode_verify()
301 ASSERT3U(dn->dn_phys->dn_nlevels, <=, dn->dn_nlevels); in dnode_verify()
302 ASSERT(DMU_OBJECT_IS_SPECIAL(dn->dn_object) || dn->dn_dbuf != NULL); in dnode_verify()
303 if (dn->dn_dbuf != NULL) { in dnode_verify()
304 ASSERT3P(dn->dn_phys, ==, in dnode_verify()
305 (dnode_phys_t *)dn->dn_dbuf->db.db_data + in dnode_verify()
306 (dn->dn_object % (dn->dn_dbuf->db.db_size >> DNODE_SHIFT))); in dnode_verify()
309 rw_exit(&dn->dn_struct_rwlock); in dnode_verify()
385 dnode_setbonuslen(dnode_t *dn, int newsize, dmu_tx_t *tx) in dnode_setbonuslen() argument
387 ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1); in dnode_setbonuslen()
389 dnode_setdirty(dn, tx); in dnode_setbonuslen()
390 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_setbonuslen()
391 ASSERT3U(newsize, <=, DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) - in dnode_setbonuslen()
392 (dn->dn_nblkptr-1) * sizeof (blkptr_t)); in dnode_setbonuslen()
394 if (newsize < dn->dn_bonuslen) { in dnode_setbonuslen()
396 size_t diff = dn->dn_bonuslen - newsize; in dnode_setbonuslen()
397 char *data_end = ((char *)dn->dn_bonus->db.db_data) + newsize; in dnode_setbonuslen()
401 dn->dn_bonuslen = newsize; in dnode_setbonuslen()
403 dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = DN_ZERO_BONUSLEN; in dnode_setbonuslen()
405 dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = dn->dn_bonuslen; in dnode_setbonuslen()
406 rw_exit(&dn->dn_struct_rwlock); in dnode_setbonuslen()
410 dnode_setbonus_type(dnode_t *dn, dmu_object_type_t newtype, dmu_tx_t *tx) in dnode_setbonus_type() argument
412 ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1); in dnode_setbonus_type()
413 dnode_setdirty(dn, tx); in dnode_setbonus_type()
414 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_setbonus_type()
415 dn->dn_bonustype = newtype; in dnode_setbonus_type()
416 dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = dn->dn_bonustype; in dnode_setbonus_type()
417 rw_exit(&dn->dn_struct_rwlock); in dnode_setbonus_type()
421 dnode_rm_spill(dnode_t *dn, dmu_tx_t *tx) in dnode_rm_spill() argument
423 ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1); in dnode_rm_spill()
424 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); in dnode_rm_spill()
425 dnode_setdirty(dn, tx); in dnode_rm_spill()
426 dn->dn_rm_spillblk[tx->tx_txg & TXG_MASK] = DN_KILL_SPILLBLK; in dnode_rm_spill()
427 dn->dn_have_spill = B_FALSE; in dnode_rm_spill()
431 dnode_setdblksz(dnode_t *dn, int size) in dnode_setdblksz() argument
437 1<<(sizeof (dn->dn_phys->dn_datablkszsec) * 8)); in dnode_setdblksz()
438 dn->dn_datablksz = size; in dnode_setdblksz()
439 dn->dn_datablkszsec = size >> SPA_MINBLOCKSHIFT; in dnode_setdblksz()
440 dn->dn_datablkshift = ISP2(size) ? highbit64(size - 1) : 0; in dnode_setdblksz()
447 dnode_t *dn; in dnode_create() local
449 dn = kmem_cache_alloc(dnode_cache, KM_SLEEP); in dnode_create()
450 dn->dn_moved = 0; in dnode_create()
456 dn->dn_object = object; in dnode_create()
457 dn->dn_dbuf = db; in dnode_create()
458 dn->dn_handle = dnh; in dnode_create()
459 dn->dn_phys = dnp; in dnode_create()
462 dnode_setdblksz(dn, dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT); in dnode_create()
464 dn->dn_datablksz = 0; in dnode_create()
465 dn->dn_datablkszsec = 0; in dnode_create()
466 dn->dn_datablkshift = 0; in dnode_create()
468 dn->dn_indblkshift = dnp->dn_indblkshift; in dnode_create()
469 dn->dn_nlevels = dnp->dn_nlevels; in dnode_create()
470 dn->dn_type = dnp->dn_type; in dnode_create()
471 dn->dn_nblkptr = dnp->dn_nblkptr; in dnode_create()
472 dn->dn_checksum = dnp->dn_checksum; in dnode_create()
473 dn->dn_compress = dnp->dn_compress; in dnode_create()
474 dn->dn_bonustype = dnp->dn_bonustype; in dnode_create()
475 dn->dn_bonuslen = dnp->dn_bonuslen; in dnode_create()
476 dn->dn_num_slots = dnp->dn_extra_slots + 1; in dnode_create()
477 dn->dn_maxblkid = dnp->dn_maxblkid; in dnode_create()
478 dn->dn_have_spill = ((dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) != 0); in dnode_create()
479 dn->dn_id_flags = 0; in dnode_create()
481 dmu_zfetch_init(&dn->dn_zfetch, dn); in dnode_create()
483 ASSERT(DMU_OT_IS_VALID(dn->dn_phys->dn_type)); in dnode_create()
497 list_insert_head(&os->os_dnodes, dn); in dnode_create()
504 dn->dn_objset = os; in dnode_create()
506 dnh->dnh_dnode = dn; in dnode_create()
511 return (dn); in dnode_create()
518 dnode_destroy(dnode_t *dn) in dnode_destroy() argument
520 objset_t *os = dn->dn_objset; in dnode_destroy()
523 ASSERT((dn->dn_id_flags & DN_ID_NEW_EXIST) == 0); in dnode_destroy()
526 POINTER_INVALIDATE(&dn->dn_objset); in dnode_destroy()
527 if (!DMU_OBJECT_IS_SPECIAL(dn->dn_object)) { in dnode_destroy()
528 list_remove(&os->os_dnodes, dn); in dnode_destroy()
536 if (!zrl_is_locked(&dn->dn_handle->dnh_zrlock)) in dnode_destroy()
537 zrl_remove(&dn->dn_handle->dnh_zrlock); in dnode_destroy()
539 dn->dn_allocated_txg = 0; in dnode_destroy()
540 dn->dn_free_txg = 0; in dnode_destroy()
541 dn->dn_assigned_txg = 0; in dnode_destroy()
542 dn->dn_dirty_txg = 0; in dnode_destroy()
544 dn->dn_dirtyctx = 0; in dnode_destroy()
545 dn->dn_dirtyctx_firstset = NULL; in dnode_destroy()
546 if (dn->dn_bonus != NULL) { in dnode_destroy()
547 mutex_enter(&dn->dn_bonus->db_mtx); in dnode_destroy()
548 dbuf_destroy(dn->dn_bonus); in dnode_destroy()
549 dn->dn_bonus = NULL; in dnode_destroy()
551 dn->dn_zio = NULL; in dnode_destroy()
553 dn->dn_have_spill = B_FALSE; in dnode_destroy()
554 dn->dn_oldused = 0; in dnode_destroy()
555 dn->dn_oldflags = 0; in dnode_destroy()
556 dn->dn_olduid = 0; in dnode_destroy()
557 dn->dn_oldgid = 0; in dnode_destroy()
558 dn->dn_oldprojid = ZFS_DEFAULT_PROJID; in dnode_destroy()
559 dn->dn_newuid = 0; in dnode_destroy()
560 dn->dn_newgid = 0; in dnode_destroy()
561 dn->dn_newprojid = ZFS_DEFAULT_PROJID; in dnode_destroy()
562 dn->dn_id_flags = 0; in dnode_destroy()
564 dmu_zfetch_fini(&dn->dn_zfetch); in dnode_destroy()
565 kmem_cache_free(dnode_cache, dn); in dnode_destroy()
573 dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs, in dnode_allocate() argument
580 spa_maxdnodesize(dmu_objset_spa(dn->dn_objset))); in dnode_allocate()
582 spa_maxblocksize(dmu_objset_spa(dn->dn_objset))); in dnode_allocate()
594 dn->dn_objset, (u_longlong_t)dn->dn_object, in dnode_allocate()
598 ASSERT(dn->dn_type == DMU_OT_NONE); in dnode_allocate()
599 ASSERT(bcmp(dn->dn_phys, &dnode_phys_zero, sizeof (dnode_phys_t)) == 0); in dnode_allocate()
600 ASSERT(dn->dn_phys->dn_type == DMU_OT_NONE); in dnode_allocate()
608 ASSERT(dn->dn_type == DMU_OT_NONE); in dnode_allocate()
609 ASSERT0(dn->dn_maxblkid); in dnode_allocate()
610 ASSERT0(dn->dn_allocated_txg); in dnode_allocate()
611 ASSERT0(dn->dn_assigned_txg); in dnode_allocate()
612 ASSERT(zfs_refcount_is_zero(&dn->dn_tx_holds)); in dnode_allocate()
613 ASSERT3U(zfs_refcount_count(&dn->dn_holds), <=, 1); in dnode_allocate()
614 ASSERT(avl_is_empty(&dn->dn_dbufs)); in dnode_allocate()
617 ASSERT0(dn->dn_next_nblkptr[i]); in dnode_allocate()
618 ASSERT0(dn->dn_next_nlevels[i]); in dnode_allocate()
619 ASSERT0(dn->dn_next_indblkshift[i]); in dnode_allocate()
620 ASSERT0(dn->dn_next_bonuslen[i]); in dnode_allocate()
621 ASSERT0(dn->dn_next_bonustype[i]); in dnode_allocate()
622 ASSERT0(dn->dn_rm_spillblk[i]); in dnode_allocate()
623 ASSERT0(dn->dn_next_blksz[i]); in dnode_allocate()
624 ASSERT0(dn->dn_next_maxblkid[i]); in dnode_allocate()
625 ASSERT(!multilist_link_active(&dn->dn_dirty_link[i])); in dnode_allocate()
626 ASSERT3P(list_head(&dn->dn_dirty_records[i]), ==, NULL); in dnode_allocate()
627 ASSERT3P(dn->dn_free_ranges[i], ==, NULL); in dnode_allocate()
630 dn->dn_type = ot; in dnode_allocate()
631 dnode_setdblksz(dn, blocksize); in dnode_allocate()
632 dn->dn_indblkshift = ibs; in dnode_allocate()
633 dn->dn_nlevels = 1; in dnode_allocate()
634 dn->dn_num_slots = dn_slots; in dnode_allocate()
636 dn->dn_nblkptr = 1; in dnode_allocate()
638 dn->dn_nblkptr = MIN(DN_MAX_NBLKPTR, in dnode_allocate()
643 dn->dn_bonustype = bonustype; in dnode_allocate()
644 dn->dn_bonuslen = bonuslen; in dnode_allocate()
645 dn->dn_checksum = ZIO_CHECKSUM_INHERIT; in dnode_allocate()
646 dn->dn_compress = ZIO_COMPRESS_INHERIT; in dnode_allocate()
647 dn->dn_dirtyctx = 0; in dnode_allocate()
649 dn->dn_free_txg = 0; in dnode_allocate()
650 dn->dn_dirtyctx_firstset = NULL; in dnode_allocate()
651 dn->dn_dirty_txg = 0; in dnode_allocate()
653 dn->dn_allocated_txg = tx->tx_txg; in dnode_allocate()
654 dn->dn_id_flags = 0; in dnode_allocate()
656 dnode_setdirty(dn, tx); in dnode_allocate()
657 dn->dn_next_indblkshift[tx->tx_txg & TXG_MASK] = ibs; in dnode_allocate()
658 dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = dn->dn_bonuslen; in dnode_allocate()
659 dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = dn->dn_bonustype; in dnode_allocate()
660 dn->dn_next_blksz[tx->tx_txg & TXG_MASK] = dn->dn_datablksz; in dnode_allocate()
664 dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, in dnode_reallocate() argument
672 spa_maxblocksize(dmu_objset_spa(dn->dn_objset))); in dnode_reallocate()
674 ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT || dmu_tx_private_ok(tx)); in dnode_reallocate()
681 DN_BONUS_SIZE(spa_maxdnodesize(dmu_objset_spa(dn->dn_objset)))); in dnode_reallocate()
684 dnode_free_interior_slots(dn); in dnode_reallocate()
688 dnode_evict_dbufs(dn); in dnode_reallocate()
690 dn->dn_id_flags = 0; in dnode_reallocate()
692 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_reallocate()
693 dnode_setdirty(dn, tx); in dnode_reallocate()
694 if (dn->dn_datablksz != blocksize) { in dnode_reallocate()
696 ASSERT0(dn->dn_maxblkid); in dnode_reallocate()
697 ASSERT(BP_IS_HOLE(&dn->dn_phys->dn_blkptr[0]) || in dnode_reallocate()
698 dnode_block_freed(dn, 0)); in dnode_reallocate()
700 dnode_setdblksz(dn, blocksize); in dnode_reallocate()
701 dn->dn_next_blksz[tx->tx_txg & TXG_MASK] = blocksize; in dnode_reallocate()
703 if (dn->dn_bonuslen != bonuslen) in dnode_reallocate()
704 dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = bonuslen; in dnode_reallocate()
712 if (dn->dn_bonustype != bonustype) in dnode_reallocate()
713 dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = bonustype; in dnode_reallocate()
714 if (dn->dn_nblkptr != nblkptr) in dnode_reallocate()
715 dn->dn_next_nblkptr[tx->tx_txg & TXG_MASK] = nblkptr; in dnode_reallocate()
716 if (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR && !keep_spill) { in dnode_reallocate()
717 dbuf_rm_spill(dn, tx); in dnode_reallocate()
718 dnode_rm_spill(dn, tx); in dnode_reallocate()
721 rw_exit(&dn->dn_struct_rwlock); in dnode_reallocate()
724 dn->dn_type = ot; in dnode_reallocate()
727 mutex_enter(&dn->dn_mtx); in dnode_reallocate()
728 dn->dn_bonustype = bonustype; in dnode_reallocate()
729 dn->dn_bonuslen = bonuslen; in dnode_reallocate()
730 dn->dn_num_slots = dn_slots; in dnode_reallocate()
731 dn->dn_nblkptr = nblkptr; in dnode_reallocate()
732 dn->dn_checksum = ZIO_CHECKSUM_INHERIT; in dnode_reallocate()
733 dn->dn_compress = ZIO_COMPRESS_INHERIT; in dnode_reallocate()
734 ASSERT3U(dn->dn_nblkptr, <=, DN_MAX_NBLKPTR); in dnode_reallocate()
737 if (dn->dn_bonus) { in dnode_reallocate()
738 dn->dn_bonus->db.db_size = in dnode_reallocate()
739 DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) - in dnode_reallocate()
740 (dn->dn_nblkptr-1) * sizeof (blkptr_t); in dnode_reallocate()
741 ASSERT(dn->dn_bonuslen <= dn->dn_bonus->db.db_size); in dnode_reallocate()
744 dn->dn_allocated_txg = tx->tx_txg; in dnode_reallocate()
745 mutex_exit(&dn->dn_mtx); in dnode_reallocate()
1100 dnode_t *dn = dnh->dnh_dnode; in dnode_check_slots_free() local
1102 if (dn == DN_SLOT_FREE) { in dnode_check_slots_free()
1104 } else if (DN_SLOT_IS_PTR(dn)) { in dnode_check_slots_free()
1105 mutex_enter(&dn->dn_mtx); in dnode_check_slots_free()
1106 boolean_t can_free = (dn->dn_type == DMU_OT_NONE && in dnode_check_slots_free()
1107 zfs_refcount_is_zero(&dn->dn_holds) && in dnode_check_slots_free()
1108 !DNODE_IS_DIRTY(dn)); in dnode_check_slots_free()
1109 mutex_exit(&dn->dn_mtx); in dnode_check_slots_free()
1142 dnode_free_interior_slots(dnode_t *dn) in dnode_free_interior_slots() argument
1144 dnode_children_t *children = dmu_buf_get_user(&dn->dn_dbuf->db); in dnode_free_interior_slots()
1145 int epb = dn->dn_dbuf->db.db_size >> DNODE_SHIFT; in dnode_free_interior_slots()
1146 int idx = (dn->dn_object & (epb - 1)) + 1; in dnode_free_interior_slots()
1147 int slots = dn->dn_num_slots - 1; in dnode_free_interior_slots()
1166 dnode_t *dn = dnh->dnh_dnode; in dnode_special_close() local
1172 mutex_enter(&dn->dn_mtx); in dnode_special_close()
1173 if (zfs_refcount_count(&dn->dn_holds) > 0) in dnode_special_close()
1174 cv_wait(&dn->dn_nodnholds, &dn->dn_mtx); in dnode_special_close()
1175 mutex_exit(&dn->dn_mtx); in dnode_special_close()
1176 ASSERT3U(zfs_refcount_count(&dn->dn_holds), ==, 0); in dnode_special_close()
1178 ASSERT(dn->dn_dbuf == NULL || in dnode_special_close()
1179 dmu_buf_get_user(&dn->dn_dbuf->db) == NULL); in dnode_special_close()
1181 dnode_destroy(dn); /* implicit zrl_remove() */ in dnode_special_close()
1190 dnode_t *dn; in dnode_special_open() local
1195 dn = dnode_create(os, dnp, NULL, object, dnh); in dnode_special_open()
1196 DNODE_VERIFY(dn); in dnode_special_open()
1210 dnode_t *dn; in dnode_buf_evict_async() local
1224 dn = dnh->dnh_dnode; in dnode_buf_evict_async()
1231 ASSERT(zfs_refcount_is_zero(&dn->dn_holds)); in dnode_buf_evict_async()
1232 ASSERT(zfs_refcount_is_zero(&dn->dn_tx_holds)); in dnode_buf_evict_async()
1234 dnode_destroy(dn); /* implicit zrl_remove() for first slot */ in dnode_buf_evict_async()
1279 dnode_t *mdn, *dn; in dnode_hold_impl() local
1304 dn = DMU_USERUSED_DNODE(os); in dnode_hold_impl()
1306 dn = DMU_GROUPUSED_DNODE(os); in dnode_hold_impl()
1308 dn = DMU_PROJECTUSED_DNODE(os); in dnode_hold_impl()
1309 if (dn == NULL) in dnode_hold_impl()
1311 type = dn->dn_type; in dnode_hold_impl()
1316 DNODE_VERIFY(dn); in dnode_hold_impl()
1319 (void) zfs_refcount_add(&dn->dn_holds, tag); in dnode_hold_impl()
1320 *dnp = dn; in dnode_hold_impl()
1422 dn = dnh->dnh_dnode; in dnode_hold_impl()
1447 dn = dnh->dnh_dnode; in dnode_hold_impl()
1449 dn = dnode_create(os, dn_block + idx, db, in dnode_hold_impl()
1454 mutex_enter(&dn->dn_mtx); in dnode_hold_impl()
1455 if (dn->dn_type == DMU_OT_NONE || dn->dn_free_txg != 0) { in dnode_hold_impl()
1457 mutex_exit(&dn->dn_mtx); in dnode_hold_impl()
1465 mutex_exit(&dn->dn_mtx); in dnode_hold_impl()
1513 dn = dnh->dnh_dnode; in dnode_hold_impl()
1515 dn = dnode_create(os, dn_block + idx, db, in dnode_hold_impl()
1519 mutex_enter(&dn->dn_mtx); in dnode_hold_impl()
1520 if (!zfs_refcount_is_zero(&dn->dn_holds) || dn->dn_free_txg) { in dnode_hold_impl()
1522 mutex_exit(&dn->dn_mtx); in dnode_hold_impl()
1530 mutex_exit(&dn->dn_mtx); in dnode_hold_impl()
1543 ASSERT0(dn->dn_free_txg); in dnode_hold_impl()
1545 if (zfs_refcount_add(&dn->dn_holds, tag) == 1) in dnode_hold_impl()
1548 mutex_exit(&dn->dn_mtx); in dnode_hold_impl()
1553 DNODE_VERIFY(dn); in dnode_hold_impl()
1555 ASSERT3P(dn->dn_dbuf, ==, db); in dnode_hold_impl()
1556 ASSERT3U(dn->dn_object, ==, object); in dnode_hold_impl()
1559 *dnp = dn; in dnode_hold_impl()
1579 dnode_add_ref(dnode_t *dn, void *tag) in dnode_add_ref() argument
1581 mutex_enter(&dn->dn_mtx); in dnode_add_ref()
1582 if (zfs_refcount_is_zero(&dn->dn_holds)) { in dnode_add_ref()
1583 mutex_exit(&dn->dn_mtx); in dnode_add_ref()
1586 VERIFY(1 < zfs_refcount_add(&dn->dn_holds, tag)); in dnode_add_ref()
1587 mutex_exit(&dn->dn_mtx); in dnode_add_ref()
1592 dnode_rele(dnode_t *dn, void *tag) in dnode_rele() argument
1594 mutex_enter(&dn->dn_mtx); in dnode_rele()
1595 dnode_rele_and_unlock(dn, tag, B_FALSE); in dnode_rele()
1599 dnode_rele_and_unlock(dnode_t *dn, void *tag, boolean_t evicting) in dnode_rele_and_unlock() argument
1603 dmu_buf_impl_t *db = dn->dn_dbuf; in dnode_rele_and_unlock()
1604 dnode_handle_t *dnh = dn->dn_handle; in dnode_rele_and_unlock()
1606 refs = zfs_refcount_remove(&dn->dn_holds, tag); in dnode_rele_and_unlock()
1608 cv_broadcast(&dn->dn_nodnholds); in dnode_rele_and_unlock()
1609 mutex_exit(&dn->dn_mtx); in dnode_rele_and_unlock()
1652 dnode_is_dirty(dnode_t *dn) in dnode_is_dirty() argument
1654 mutex_enter(&dn->dn_mtx); in dnode_is_dirty()
1657 if (multilist_link_active(&dn->dn_dirty_link[i])) { in dnode_is_dirty()
1658 mutex_exit(&dn->dn_mtx); in dnode_is_dirty()
1663 mutex_exit(&dn->dn_mtx); in dnode_is_dirty()
1669 dnode_setdirty(dnode_t *dn, dmu_tx_t *tx) in dnode_setdirty() argument
1671 objset_t *os = dn->dn_objset; in dnode_setdirty()
1674 if (DMU_OBJECT_IS_SPECIAL(dn->dn_object)) { in dnode_setdirty()
1679 DNODE_VERIFY(dn); in dnode_setdirty()
1682 mutex_enter(&dn->dn_mtx); in dnode_setdirty()
1683 ASSERT(dn->dn_phys->dn_type || dn->dn_allocated_txg); in dnode_setdirty()
1684 ASSERT(dn->dn_free_txg == 0 || dn->dn_free_txg >= txg); in dnode_setdirty()
1685 mutex_exit(&dn->dn_mtx); in dnode_setdirty()
1691 dmu_objset_userquota_get_ids(dn, B_TRUE, tx); in dnode_setdirty()
1694 multilist_sublist_t *mls = multilist_sublist_lock_obj(dirtylist, dn); in dnode_setdirty()
1699 if (multilist_link_active(&dn->dn_dirty_link[txg & TXG_MASK])) { in dnode_setdirty()
1704 ASSERT(!zfs_refcount_is_zero(&dn->dn_holds) || in dnode_setdirty()
1705 !avl_is_empty(&dn->dn_dbufs)); in dnode_setdirty()
1706 ASSERT(dn->dn_datablksz != 0); in dnode_setdirty()
1707 ASSERT0(dn->dn_next_bonuslen[txg & TXG_MASK]); in dnode_setdirty()
1708 ASSERT0(dn->dn_next_blksz[txg & TXG_MASK]); in dnode_setdirty()
1709 ASSERT0(dn->dn_next_bonustype[txg & TXG_MASK]); in dnode_setdirty()
1712 (u_longlong_t)dn->dn_object, (u_longlong_t)txg); in dnode_setdirty()
1714 multilist_sublist_insert_head(mls, dn); in dnode_setdirty()
1727 VERIFY(dnode_add_ref(dn, (void *)(uintptr_t)tx->tx_txg)); in dnode_setdirty()
1729 (void) dbuf_dirty(dn->dn_dbuf, tx); in dnode_setdirty()
1735 dnode_free(dnode_t *dn, dmu_tx_t *tx) in dnode_free() argument
1737 mutex_enter(&dn->dn_mtx); in dnode_free()
1738 if (dn->dn_type == DMU_OT_NONE || dn->dn_free_txg) { in dnode_free()
1739 mutex_exit(&dn->dn_mtx); in dnode_free()
1742 dn->dn_free_txg = tx->tx_txg; in dnode_free()
1743 mutex_exit(&dn->dn_mtx); in dnode_free()
1745 dnode_setdirty(dn, tx); in dnode_free()
1753 dnode_set_blksz(dnode_t *dn, uint64_t size, int ibs, dmu_tx_t *tx) in dnode_set_blksz() argument
1758 ASSERT3U(size, <=, spa_maxblocksize(dmu_objset_spa(dn->dn_objset))); in dnode_set_blksz()
1764 if (ibs == dn->dn_indblkshift) in dnode_set_blksz()
1767 if (size >> SPA_MINBLOCKSHIFT == dn->dn_datablkszsec && ibs == 0) in dnode_set_blksz()
1770 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_set_blksz()
1773 if (dn->dn_maxblkid != 0) in dnode_set_blksz()
1776 mutex_enter(&dn->dn_dbufs_mtx); in dnode_set_blksz()
1777 for (db = avl_first(&dn->dn_dbufs); db != NULL; in dnode_set_blksz()
1778 db = AVL_NEXT(&dn->dn_dbufs, db)) { in dnode_set_blksz()
1781 mutex_exit(&dn->dn_dbufs_mtx); in dnode_set_blksz()
1785 mutex_exit(&dn->dn_dbufs_mtx); in dnode_set_blksz()
1787 if (ibs && dn->dn_nlevels != 1) in dnode_set_blksz()
1791 err = dbuf_hold_impl(dn, 0, 0, TRUE, FALSE, FTAG, &db); in dnode_set_blksz()
1798 dnode_setdblksz(dn, size); in dnode_set_blksz()
1799 dnode_setdirty(dn, tx); in dnode_set_blksz()
1800 dn->dn_next_blksz[tx->tx_txg&TXG_MASK] = size; in dnode_set_blksz()
1802 dn->dn_indblkshift = ibs; in dnode_set_blksz()
1803 dn->dn_next_indblkshift[tx->tx_txg&TXG_MASK] = ibs; in dnode_set_blksz()
1809 rw_exit(&dn->dn_struct_rwlock); in dnode_set_blksz()
1813 rw_exit(&dn->dn_struct_rwlock); in dnode_set_blksz()
1818 dnode_set_nlevels_impl(dnode_t *dn, int new_nlevels, dmu_tx_t *tx) in dnode_set_nlevels_impl() argument
1821 int old_nlevels = dn->dn_nlevels; in dnode_set_nlevels_impl()
1826 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); in dnode_set_nlevels_impl()
1828 ASSERT3U(new_nlevels, >, dn->dn_nlevels); in dnode_set_nlevels_impl()
1829 dn->dn_nlevels = new_nlevels; in dnode_set_nlevels_impl()
1831 ASSERT3U(new_nlevels, >, dn->dn_next_nlevels[txgoff]); in dnode_set_nlevels_impl()
1832 dn->dn_next_nlevels[txgoff] = new_nlevels; in dnode_set_nlevels_impl()
1835 db = dbuf_hold_level(dn, old_nlevels, 0, FTAG); in dnode_set_nlevels_impl()
1841 mutex_enter(&dn->dn_mtx); in dnode_set_nlevels_impl()
1843 list = &dn->dn_dirty_records[txgoff]; in dnode_set_nlevels_impl()
1845 dr_next = list_next(&dn->dn_dirty_records[txgoff], dr); in dnode_set_nlevels_impl()
1852 list_remove(&dn->dn_dirty_records[txgoff], dr); in dnode_set_nlevels_impl()
1858 mutex_exit(&dn->dn_mtx); in dnode_set_nlevels_impl()
1862 dnode_set_nlevels(dnode_t *dn, int nlevels, dmu_tx_t *tx) in dnode_set_nlevels() argument
1866 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_set_nlevels()
1868 if (dn->dn_nlevels == nlevels) { in dnode_set_nlevels()
1871 } else if (nlevels < dn->dn_nlevels) { in dnode_set_nlevels()
1876 dnode_set_nlevels_impl(dn, nlevels, tx); in dnode_set_nlevels()
1879 rw_exit(&dn->dn_struct_rwlock); in dnode_set_nlevels()
1885 dnode_new_blkid(dnode_t *dn, uint64_t blkid, dmu_tx_t *tx, boolean_t have_read, in dnode_new_blkid() argument
1894 RW_READ_HELD(&dn->dn_struct_rwlock) : in dnode_new_blkid()
1895 RW_WRITE_HELD(&dn->dn_struct_rwlock)); in dnode_new_blkid()
1902 if (blkid <= dn->dn_maxblkid) in dnode_new_blkid()
1905 if (!rw_tryupgrade(&dn->dn_struct_rwlock)) { in dnode_new_blkid()
1906 rw_exit(&dn->dn_struct_rwlock); in dnode_new_blkid()
1907 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_new_blkid()
1915 if (!force && blkid <= dn->dn_maxblkid) in dnode_new_blkid()
1923 dn->dn_maxblkid = blkid; in dnode_new_blkid()
1924 dn->dn_next_maxblkid[tx->tx_txg & TXG_MASK] = in dnode_new_blkid()
1932 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; in dnode_new_blkid()
1933 for (sz = dn->dn_nblkptr; in dnode_new_blkid()
1934 sz <= blkid && sz >= dn->dn_nblkptr; sz <<= epbs) in dnode_new_blkid()
1940 if (new_nlevels > dn->dn_nlevels) in dnode_new_blkid()
1941 dnode_set_nlevels_impl(dn, new_nlevels, tx); in dnode_new_blkid()
1943 ASSERT3U(dn->dn_nlevels, >=, new_nlevels); in dnode_new_blkid()
1948 rw_downgrade(&dn->dn_struct_rwlock); in dnode_new_blkid()
1952 dnode_dirty_l1(dnode_t *dn, uint64_t l1blkid, dmu_tx_t *tx) in dnode_dirty_l1() argument
1954 dmu_buf_impl_t *db = dbuf_hold_level(dn, 1, l1blkid, FTAG); in dnode_dirty_l1()
1966 dnode_dirty_l1range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid, in dnode_dirty_l1range() argument
1975 mutex_enter(&dn->dn_dbufs_mtx); in dnode_dirty_l1range()
1982 db = avl_find(&dn->dn_dbufs, db_search, &where); in dnode_dirty_l1range()
1984 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); in dnode_dirty_l1range()
2005 mutex_exit(&dn->dn_dbufs_mtx); in dnode_dirty_l1range()
2006 dnode_dirty_l1(dn, db->db_blkid, tx); in dnode_dirty_l1range()
2007 mutex_enter(&dn->dn_dbufs_mtx); in dnode_dirty_l1range()
2017 db = avl_find(&dn->dn_dbufs, db_search, &where); in dnode_dirty_l1range()
2019 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); in dnode_dirty_l1range()
2020 for (; db != NULL; db = AVL_NEXT(&dn->dn_dbufs, db)) { in dnode_dirty_l1range()
2028 mutex_exit(&dn->dn_dbufs_mtx); in dnode_dirty_l1range()
2032 dnode_set_dirtyctx(dnode_t *dn, dmu_tx_t *tx, void *tag) in dnode_set_dirtyctx() argument
2038 if (dn->dn_dirtyctx == DN_UNDIRTIED) { in dnode_set_dirtyctx()
2039 dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; in dnode_set_dirtyctx()
2044 if (!BP_IS_HOLE(dn->dn_objset->os_rootbp)) { in dnode_set_dirtyctx()
2046 dn->dn_dirtyctx = DN_DIRTY_SYNC; in dnode_set_dirtyctx()
2048 dn->dn_dirtyctx = DN_DIRTY_OPEN; in dnode_set_dirtyctx()
2049 dn->dn_dirtyctx_firstset = tag; in dnode_set_dirtyctx()
2058 dnode_free_range(dnode_t *dn, uint64_t off, uint64_t len, dmu_tx_t *tx) in dnode_free_range() argument
2066 blksz = dn->dn_datablksz; in dnode_free_range()
2067 blkshift = dn->dn_datablkshift; in dnode_free_range()
2068 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; in dnode_free_range()
2081 if ((off >> blkshift) > dn->dn_maxblkid) in dnode_free_range()
2084 ASSERT(dn->dn_maxblkid == 0); in dnode_free_range()
2091 if (dn->dn_nlevels > 1) { in dnode_free_range()
2092 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_free_range()
2093 dnode_dirty_l1(dn, 0, tx); in dnode_free_range()
2094 rw_exit(&dn->dn_struct_rwlock); in dnode_free_range()
2113 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dnode_free_range()
2114 res = dbuf_hold_impl(dn, 0, dbuf_whichblock(dn, 0, off), in dnode_free_range()
2116 rw_exit(&dn->dn_struct_rwlock); in dnode_free_range()
2143 if ((off >> blkshift) > dn->dn_maxblkid) in dnode_free_range()
2158 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dnode_free_range()
2159 res = dbuf_hold_impl(dn, 0, dbuf_whichblock(dn, 0, off+len), in dnode_free_range()
2161 rw_exit(&dn->dn_struct_rwlock); in dnode_free_range()
2210 if (dn->dn_nlevels > 1) { in dnode_free_range()
2211 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_free_range()
2215 dnode_dirty_l1(dn, first, tx); in dnode_free_range()
2217 last = dn->dn_maxblkid >> epbs; in dnode_free_range()
2221 dnode_dirty_l1(dn, last, tx); in dnode_free_range()
2223 dnode_dirty_l1range(dn, first, last, tx); in dnode_free_range()
2225 int shift = dn->dn_datablkshift + dn->dn_indblkshift - in dnode_free_range()
2235 int err = dnode_next_offset(dn, DNODE_FIND_HAVELOCK, in dnode_free_range()
2254 dnode_dirty_l1(dn, i, tx); in dnode_free_range()
2256 rw_exit(&dn->dn_struct_rwlock); in dnode_free_range()
2264 mutex_enter(&dn->dn_mtx); in dnode_free_range()
2267 if (dn->dn_free_ranges[txgoff] == NULL) { in dnode_free_range()
2268 dn->dn_free_ranges[txgoff] = range_tree_create(NULL, in dnode_free_range()
2271 range_tree_clear(dn->dn_free_ranges[txgoff], blkid, nblks); in dnode_free_range()
2272 range_tree_add(dn->dn_free_ranges[txgoff], blkid, nblks); in dnode_free_range()
2274 dprintf_dnode(dn, "blkid=%llu nblks=%llu txg=%llu\n", in dnode_free_range()
2277 mutex_exit(&dn->dn_mtx); in dnode_free_range()
2279 dbuf_free_range(dn, blkid, blkid + nblks - 1, tx); in dnode_free_range()
2280 dnode_setdirty(dn, tx); in dnode_free_range()
2284 dnode_spill_freed(dnode_t *dn) in dnode_spill_freed() argument
2288 mutex_enter(&dn->dn_mtx); in dnode_spill_freed()
2290 if (dn->dn_rm_spillblk[i] == DN_KILL_SPILLBLK) in dnode_spill_freed()
2293 mutex_exit(&dn->dn_mtx); in dnode_spill_freed()
2299 dnode_block_freed(dnode_t *dn, uint64_t blkid) in dnode_block_freed() argument
2301 void *dp = spa_get_dsl(dn->dn_objset->os_spa); in dnode_block_freed()
2314 if (dn->dn_free_txg) in dnode_block_freed()
2318 return (dnode_spill_freed(dn)); in dnode_block_freed()
2320 mutex_enter(&dn->dn_mtx); in dnode_block_freed()
2322 if (dn->dn_free_ranges[i] != NULL && in dnode_block_freed()
2323 range_tree_contains(dn->dn_free_ranges[i], blkid, 1)) in dnode_block_freed()
2326 mutex_exit(&dn->dn_mtx); in dnode_block_freed()
2332 dnode_diduse_space(dnode_t *dn, int64_t delta) in dnode_diduse_space() argument
2335 dprintf_dnode(dn, "dn=%p dnp=%p used=%llu delta=%lld\n", in dnode_diduse_space()
2336 dn, dn->dn_phys, in dnode_diduse_space()
2337 (u_longlong_t)dn->dn_phys->dn_used, in dnode_diduse_space()
2340 mutex_enter(&dn->dn_mtx); in dnode_diduse_space()
2341 space = DN_USED_BYTES(dn->dn_phys); in dnode_diduse_space()
2348 if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_DNODE_BYTES) { in dnode_diduse_space()
2349 ASSERT((dn->dn_phys->dn_flags & DNODE_FLAG_USED_BYTES) == 0); in dnode_diduse_space()
2351 dn->dn_phys->dn_used = space >> DEV_BSHIFT; in dnode_diduse_space()
2353 dn->dn_phys->dn_used = space; in dnode_diduse_space()
2354 dn->dn_phys->dn_flags |= DNODE_FLAG_USED_BYTES; in dnode_diduse_space()
2356 mutex_exit(&dn->dn_mtx); in dnode_diduse_space()
2378 dnode_next_offset_level(dnode_t *dn, int flags, uint64_t *offset, in dnode_next_offset_level() argument
2383 uint64_t epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; in dnode_next_offset_level()
2389 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); in dnode_next_offset_level()
2395 if (lvl == dn->dn_phys->dn_nlevels) { in dnode_next_offset_level()
2397 epb = dn->dn_phys->dn_nblkptr; in dnode_next_offset_level()
2398 data = dn->dn_phys->dn_blkptr; in dnode_next_offset_level()
2400 uint64_t blkid = dbuf_whichblock(dn, lvl, *offset); in dnode_next_offset_level()
2401 error = dbuf_hold_impl(dn, lvl, blkid, TRUE, FALSE, FTAG, &db); in dnode_next_offset_level()
2438 ASSERT(dn->dn_type == DMU_OT_DNODE); in dnode_next_offset_level()
2455 span = (lvl - 1) * epbs + dn->dn_datablkshift; in dnode_next_offset_level()
2466 ASSERT3U((lvl - 1), ==, dn->dn_phys->dn_nlevels - 1); in dnode_next_offset_level()
2531 dnode_next_offset(dnode_t *dn, int flags, uint64_t *offset, in dnode_next_offset() argument
2539 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dnode_next_offset()
2541 if (dn->dn_phys->dn_nlevels == 0) { in dnode_next_offset()
2546 if (dn->dn_datablkshift == 0) { in dnode_next_offset()
2547 if (*offset < dn->dn_datablksz) { in dnode_next_offset()
2549 *offset = dn->dn_datablksz; in dnode_next_offset()
2556 maxlvl = dn->dn_phys->dn_nlevels; in dnode_next_offset()
2559 error = dnode_next_offset_level(dn, in dnode_next_offset()
2566 error = dnode_next_offset_level(dn, in dnode_next_offset()
2584 rw_exit(&dn->dn_struct_rwlock); in dnode_next_offset()