1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
25 * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
26 * Copyright 2017 Nexenta Systems, Inc.
27 * Copyright (c) 2021, 2022 by Pawel Jakub Dawidek
28 */
29
30 /* Portions Copyright 2007 Jeremy Teo */
31 /* Portions Copyright 2010 Robert Milkowski */
32
33 #include <sys/types.h>
34 #include <sys/param.h>
35 #include <sys/time.h>
36 #include <sys/sysmacros.h>
37 #include <sys/vfs.h>
38 #include <sys/uio_impl.h>
39 #include <sys/file.h>
40 #include <sys/stat.h>
41 #include <sys/kmem.h>
42 #include <sys/cmn_err.h>
43 #include <sys/errno.h>
44 #include <sys/zfs_dir.h>
45 #include <sys/zfs_acl.h>
46 #include <sys/zfs_ioctl.h>
47 #include <sys/fs/zfs.h>
48 #include <sys/dmu.h>
49 #include <sys/dmu_objset.h>
50 #include <sys/dsl_crypt.h>
51 #include <sys/spa.h>
52 #include <sys/txg.h>
53 #include <sys/dbuf.h>
54 #include <sys/policy.h>
55 #include <sys/zfeature.h>
56 #include <sys/zfs_vnops.h>
57 #include <sys/zfs_quota.h>
58 #include <sys/zfs_vfsops.h>
59 #include <sys/zfs_znode.h>
60
61 /*
62 * Enable the experimental block cloning feature. If this setting is 0, then
63 * even if feature@block_cloning is enabled, attempts to clone blocks will act
64 * as though the feature is disabled.
65 */
66 int zfs_bclone_enabled = 0;
67
68 /*
69 * When set zfs_clone_range() waits for dirty data to be written to disk.
70 * This allows the clone operation to reliably succeed when a file is modified
71 * and then immediately cloned. For small files this may be slower than making
72 * a copy of the file and is therefore not the default. However, in certain
73 * scenarios this behavior may be desirable so a tunable is provided.
74 */
75 static int zfs_bclone_wait_dirty = 0;
76
77 /*
78 * Maximum bytes to read per chunk in zfs_read().
79 */
80 static uint64_t zfs_vnops_read_chunk_size = 1024 * 1024;
81
82 static ulong_t zfs_fsync_sync_cnt = 4;
83
84 int
zfs_fsync(znode_t * zp,int syncflag,cred_t * cr)85 zfs_fsync(znode_t *zp, int syncflag, cred_t *cr)
86 {
87 int error = 0;
88 zfsvfs_t *zfsvfs = ZTOZSB(zp);
89
90 (void) tsd_set(zfs_fsyncer_key, (void *)(uintptr_t)zfs_fsync_sync_cnt);
91
92 if (zfsvfs->z_os->os_sync != ZFS_SYNC_DISABLED) {
93 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
94 goto out;
95 atomic_inc_32(&zp->z_sync_writes_cnt);
96 zil_commit(zfsvfs->z_log, zp->z_id);
97 atomic_dec_32(&zp->z_sync_writes_cnt);
98 zfs_exit(zfsvfs, FTAG);
99 }
100 out:
101 tsd_set(zfs_fsyncer_key, NULL);
102
103 return (error);
104 }
105
106
107 #if defined(SEEK_HOLE) && defined(SEEK_DATA)
108 /*
109 * Lseek support for finding holes (cmd == SEEK_HOLE) and
110 * data (cmd == SEEK_DATA). "off" is an in/out parameter.
111 */
112 static int
zfs_holey_common(znode_t * zp,ulong_t cmd,loff_t * off)113 zfs_holey_common(znode_t *zp, ulong_t cmd, loff_t *off)
114 {
115 zfs_locked_range_t *lr;
116 uint64_t noff = (uint64_t)*off; /* new offset */
117 uint64_t file_sz;
118 int error;
119 boolean_t hole;
120
121 file_sz = zp->z_size;
122 if (noff >= file_sz) {
123 return (SET_ERROR(ENXIO));
124 }
125
126 if (cmd == F_SEEK_HOLE)
127 hole = B_TRUE;
128 else
129 hole = B_FALSE;
130
131 /* Flush any mmap()'d data to disk */
132 if (zn_has_cached_data(zp, 0, file_sz - 1))
133 zn_flush_cached_data(zp, B_TRUE);
134
135 lr = zfs_rangelock_enter(&zp->z_rangelock, 0, UINT64_MAX, RL_READER);
136 error = dmu_offset_next(ZTOZSB(zp)->z_os, zp->z_id, hole, &noff);
137 zfs_rangelock_exit(lr);
138
139 if (error == ESRCH)
140 return (SET_ERROR(ENXIO));
141
142 /* File was dirty, so fall back to using generic logic */
143 if (error == EBUSY) {
144 if (hole)
145 *off = file_sz;
146
147 return (0);
148 }
149
150 /*
151 * We could find a hole that begins after the logical end-of-file,
152 * because dmu_offset_next() only works on whole blocks. If the
153 * EOF falls mid-block, then indicate that the "virtual hole"
154 * at the end of the file begins at the logical EOF, rather than
155 * at the end of the last block.
156 */
157 if (noff > file_sz) {
158 ASSERT(hole);
159 noff = file_sz;
160 }
161
162 if (noff < *off)
163 return (error);
164 *off = noff;
165 return (error);
166 }
167
168 int
zfs_holey(znode_t * zp,ulong_t cmd,loff_t * off)169 zfs_holey(znode_t *zp, ulong_t cmd, loff_t *off)
170 {
171 zfsvfs_t *zfsvfs = ZTOZSB(zp);
172 int error;
173
174 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
175 return (error);
176
177 error = zfs_holey_common(zp, cmd, off);
178
179 zfs_exit(zfsvfs, FTAG);
180 return (error);
181 }
182 #endif /* SEEK_HOLE && SEEK_DATA */
183
184 int
zfs_access(znode_t * zp,int mode,int flag,cred_t * cr)185 zfs_access(znode_t *zp, int mode, int flag, cred_t *cr)
186 {
187 zfsvfs_t *zfsvfs = ZTOZSB(zp);
188 int error;
189
190 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
191 return (error);
192
193 if (flag & V_ACE_MASK)
194 #if defined(__linux__)
195 error = zfs_zaccess(zp, mode, flag, B_FALSE, cr,
196 zfs_init_idmap);
197 #else
198 error = zfs_zaccess(zp, mode, flag, B_FALSE, cr,
199 NULL);
200 #endif
201 else
202 #if defined(__linux__)
203 error = zfs_zaccess_rwx(zp, mode, flag, cr, zfs_init_idmap);
204 #else
205 error = zfs_zaccess_rwx(zp, mode, flag, cr, NULL);
206 #endif
207
208 zfs_exit(zfsvfs, FTAG);
209 return (error);
210 }
211
212 /*
213 * Read bytes from specified file into supplied buffer.
214 *
215 * IN: zp - inode of file to be read from.
216 * uio - structure supplying read location, range info,
217 * and return buffer.
218 * ioflag - O_SYNC flags; used to provide FRSYNC semantics.
219 * O_DIRECT flag; used to bypass page cache.
220 * cr - credentials of caller.
221 *
222 * OUT: uio - updated offset and range, buffer filled.
223 *
224 * RETURN: 0 on success, error code on failure.
225 *
226 * Side Effects:
227 * inode - atime updated if byte count > 0
228 */
229 int
zfs_read(struct znode * zp,zfs_uio_t * uio,int ioflag,cred_t * cr)230 zfs_read(struct znode *zp, zfs_uio_t *uio, int ioflag, cred_t *cr)
231 {
232 (void) cr;
233 int error = 0;
234 boolean_t frsync = B_FALSE;
235
236 zfsvfs_t *zfsvfs = ZTOZSB(zp);
237 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
238 return (error);
239
240 if (zp->z_pflags & ZFS_AV_QUARANTINED) {
241 zfs_exit(zfsvfs, FTAG);
242 return (SET_ERROR(EACCES));
243 }
244
245 /* We don't copy out anything useful for directories. */
246 if (Z_ISDIR(ZTOTYPE(zp))) {
247 zfs_exit(zfsvfs, FTAG);
248 return (SET_ERROR(EISDIR));
249 }
250
251 /*
252 * Validate file offset
253 */
254 if (zfs_uio_offset(uio) < (offset_t)0) {
255 zfs_exit(zfsvfs, FTAG);
256 return (SET_ERROR(EINVAL));
257 }
258
259 /*
260 * Fasttrack empty reads
261 */
262 if (zfs_uio_resid(uio) == 0) {
263 zfs_exit(zfsvfs, FTAG);
264 return (0);
265 }
266
267 #ifdef FRSYNC
268 /*
269 * If we're in FRSYNC mode, sync out this znode before reading it.
270 * Only do this for non-snapshots.
271 *
272 * Some platforms do not support FRSYNC and instead map it
273 * to O_SYNC, which results in unnecessary calls to zil_commit. We
274 * only honor FRSYNC requests on platforms which support it.
275 */
276 frsync = !!(ioflag & FRSYNC);
277 #endif
278 if (zfsvfs->z_log &&
279 (frsync || zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS))
280 zil_commit(zfsvfs->z_log, zp->z_id);
281
282 /*
283 * Lock the range against changes.
284 */
285 zfs_locked_range_t *lr = zfs_rangelock_enter(&zp->z_rangelock,
286 zfs_uio_offset(uio), zfs_uio_resid(uio), RL_READER);
287
288 /*
289 * If we are reading past end-of-file we can skip
290 * to the end; but we might still need to set atime.
291 */
292 if (zfs_uio_offset(uio) >= zp->z_size) {
293 error = 0;
294 goto out;
295 }
296
297 ASSERT(zfs_uio_offset(uio) < zp->z_size);
298 #if defined(__linux__)
299 ssize_t start_offset = zfs_uio_offset(uio);
300 #endif
301 ssize_t n = MIN(zfs_uio_resid(uio), zp->z_size - zfs_uio_offset(uio));
302 ssize_t start_resid = n;
303
304 while (n > 0) {
305 ssize_t nbytes = MIN(n, zfs_vnops_read_chunk_size -
306 P2PHASE(zfs_uio_offset(uio), zfs_vnops_read_chunk_size));
307 #ifdef UIO_NOCOPY
308 if (zfs_uio_segflg(uio) == UIO_NOCOPY)
309 error = mappedread_sf(zp, nbytes, uio);
310 else
311 #endif
312 if (zn_has_cached_data(zp, zfs_uio_offset(uio),
313 zfs_uio_offset(uio) + nbytes - 1) && !(ioflag & O_DIRECT)) {
314 error = mappedread(zp, nbytes, uio);
315 } else {
316 error = dmu_read_uio_dbuf(sa_get_db(zp->z_sa_hdl),
317 uio, nbytes);
318 }
319
320 if (error) {
321 /* convert checksum errors into IO errors */
322 if (error == ECKSUM)
323 error = SET_ERROR(EIO);
324
325 #if defined(__linux__)
326 /*
327 * if we actually read some bytes, bubbling EFAULT
328 * up to become EAGAIN isn't what we want here...
329 *
330 * ...on Linux, at least. On FBSD, doing this breaks.
331 */
332 if (error == EFAULT &&
333 (zfs_uio_offset(uio) - start_offset) != 0)
334 error = 0;
335 #endif
336 break;
337 }
338
339 n -= nbytes;
340 }
341
342 int64_t nread = start_resid - n;
343 dataset_kstats_update_read_kstats(&zfsvfs->z_kstat, nread);
344 task_io_account_read(nread);
345 out:
346 zfs_rangelock_exit(lr);
347
348 ZFS_ACCESSTIME_STAMP(zfsvfs, zp);
349 zfs_exit(zfsvfs, FTAG);
350 return (error);
351 }
352
353 static void
zfs_clear_setid_bits_if_necessary(zfsvfs_t * zfsvfs,znode_t * zp,cred_t * cr,uint64_t * clear_setid_bits_txgp,dmu_tx_t * tx)354 zfs_clear_setid_bits_if_necessary(zfsvfs_t *zfsvfs, znode_t *zp, cred_t *cr,
355 uint64_t *clear_setid_bits_txgp, dmu_tx_t *tx)
356 {
357 zilog_t *zilog = zfsvfs->z_log;
358 const uint64_t uid = KUID_TO_SUID(ZTOUID(zp));
359
360 ASSERT(clear_setid_bits_txgp != NULL);
361 ASSERT(tx != NULL);
362
363 /*
364 * Clear Set-UID/Set-GID bits on successful write if not
365 * privileged and at least one of the execute bits is set.
366 *
367 * It would be nice to do this after all writes have
368 * been done, but that would still expose the ISUID/ISGID
369 * to another app after the partial write is committed.
370 *
371 * Note: we don't call zfs_fuid_map_id() here because
372 * user 0 is not an ephemeral uid.
373 */
374 mutex_enter(&zp->z_acl_lock);
375 if ((zp->z_mode & (S_IXUSR | (S_IXUSR >> 3) | (S_IXUSR >> 6))) != 0 &&
376 (zp->z_mode & (S_ISUID | S_ISGID)) != 0 &&
377 secpolicy_vnode_setid_retain(zp, cr,
378 ((zp->z_mode & S_ISUID) != 0 && uid == 0)) != 0) {
379 uint64_t newmode;
380
381 zp->z_mode &= ~(S_ISUID | S_ISGID);
382 newmode = zp->z_mode;
383 (void) sa_update(zp->z_sa_hdl, SA_ZPL_MODE(zfsvfs),
384 (void *)&newmode, sizeof (uint64_t), tx);
385
386 mutex_exit(&zp->z_acl_lock);
387
388 /*
389 * Make sure SUID/SGID bits will be removed when we replay the
390 * log. If the setid bits are keep coming back, don't log more
391 * than one TX_SETATTR per transaction group.
392 */
393 if (*clear_setid_bits_txgp != dmu_tx_get_txg(tx)) {
394 vattr_t va = {0};
395
396 va.va_mask = ATTR_MODE;
397 va.va_nodeid = zp->z_id;
398 va.va_mode = newmode;
399 zfs_log_setattr(zilog, tx, TX_SETATTR, zp, &va,
400 ATTR_MODE, NULL);
401 *clear_setid_bits_txgp = dmu_tx_get_txg(tx);
402 }
403 } else {
404 mutex_exit(&zp->z_acl_lock);
405 }
406 }
407
408 /*
409 * Write the bytes to a file.
410 *
411 * IN: zp - znode of file to be written to.
412 * uio - structure supplying write location, range info,
413 * and data buffer.
414 * ioflag - O_APPEND flag set if in append mode.
415 * O_DIRECT flag; used to bypass page cache.
416 * cr - credentials of caller.
417 *
418 * OUT: uio - updated offset and range.
419 *
420 * RETURN: 0 if success
421 * error code if failure
422 *
423 * Timestamps:
424 * ip - ctime|mtime updated if byte count > 0
425 */
426 int
zfs_write(znode_t * zp,zfs_uio_t * uio,int ioflag,cred_t * cr)427 zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr)
428 {
429 int error = 0, error1;
430 ssize_t start_resid = zfs_uio_resid(uio);
431 uint64_t clear_setid_bits_txg = 0;
432
433 /*
434 * Fasttrack empty write
435 */
436 ssize_t n = start_resid;
437 if (n == 0)
438 return (0);
439
440 zfsvfs_t *zfsvfs = ZTOZSB(zp);
441 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
442 return (error);
443
444 sa_bulk_attr_t bulk[4];
445 int count = 0;
446 uint64_t mtime[2], ctime[2];
447 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
448 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
449 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL,
450 &zp->z_size, 8);
451 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL,
452 &zp->z_pflags, 8);
453
454 /*
455 * Callers might not be able to detect properly that we are read-only,
456 * so check it explicitly here.
457 */
458 if (zfs_is_readonly(zfsvfs)) {
459 zfs_exit(zfsvfs, FTAG);
460 return (SET_ERROR(EROFS));
461 }
462
463 /*
464 * If immutable or not appending then return EPERM.
465 * Intentionally allow ZFS_READONLY through here.
466 * See zfs_zaccess_common()
467 */
468 if ((zp->z_pflags & ZFS_IMMUTABLE) ||
469 ((zp->z_pflags & ZFS_APPENDONLY) && !(ioflag & O_APPEND) &&
470 (zfs_uio_offset(uio) < zp->z_size))) {
471 zfs_exit(zfsvfs, FTAG);
472 return (SET_ERROR(EPERM));
473 }
474
475 /*
476 * Validate file offset
477 */
478 offset_t woff = ioflag & O_APPEND ? zp->z_size : zfs_uio_offset(uio);
479 if (woff < 0) {
480 zfs_exit(zfsvfs, FTAG);
481 return (SET_ERROR(EINVAL));
482 }
483
484 /*
485 * Pre-fault the pages to ensure slow (eg NFS) pages
486 * don't hold up txg.
487 */
488 ssize_t pfbytes = MIN(n, DMU_MAX_ACCESS >> 1);
489 if (zfs_uio_prefaultpages(pfbytes, uio)) {
490 zfs_exit(zfsvfs, FTAG);
491 return (SET_ERROR(EFAULT));
492 }
493
494 /*
495 * If in append mode, set the io offset pointer to eof.
496 */
497 zfs_locked_range_t *lr;
498 if (ioflag & O_APPEND) {
499 /*
500 * Obtain an appending range lock to guarantee file append
501 * semantics. We reset the write offset once we have the lock.
502 */
503 lr = zfs_rangelock_enter(&zp->z_rangelock, 0, n, RL_APPEND);
504 woff = lr->lr_offset;
505 if (lr->lr_length == UINT64_MAX) {
506 /*
507 * We overlocked the file because this write will cause
508 * the file block size to increase.
509 * Note that zp_size cannot change with this lock held.
510 */
511 woff = zp->z_size;
512 }
513 zfs_uio_setoffset(uio, woff);
514 } else {
515 /*
516 * Note that if the file block size will change as a result of
517 * this write, then this range lock will lock the entire file
518 * so that we can re-write the block safely.
519 */
520 lr = zfs_rangelock_enter(&zp->z_rangelock, woff, n, RL_WRITER);
521 }
522
523 if (zn_rlimit_fsize_uio(zp, uio)) {
524 zfs_rangelock_exit(lr);
525 zfs_exit(zfsvfs, FTAG);
526 return (SET_ERROR(EFBIG));
527 }
528
529 const rlim64_t limit = MAXOFFSET_T;
530
531 if (woff >= limit) {
532 zfs_rangelock_exit(lr);
533 zfs_exit(zfsvfs, FTAG);
534 return (SET_ERROR(EFBIG));
535 }
536
537 if (n > limit - woff)
538 n = limit - woff;
539
540 uint64_t end_size = MAX(zp->z_size, woff + n);
541 zilog_t *zilog = zfsvfs->z_log;
542
543 const uint64_t uid = KUID_TO_SUID(ZTOUID(zp));
544 const uint64_t gid = KGID_TO_SGID(ZTOGID(zp));
545 const uint64_t projid = zp->z_projid;
546
547 /*
548 * Write the file in reasonable size chunks. Each chunk is written
549 * in a separate transaction; this keeps the intent log records small
550 * and allows us to do more fine-grained space accounting.
551 */
552 while (n > 0) {
553 woff = zfs_uio_offset(uio);
554
555 if (zfs_id_overblockquota(zfsvfs, DMU_USERUSED_OBJECT, uid) ||
556 zfs_id_overblockquota(zfsvfs, DMU_GROUPUSED_OBJECT, gid) ||
557 (projid != ZFS_DEFAULT_PROJID &&
558 zfs_id_overblockquota(zfsvfs, DMU_PROJECTUSED_OBJECT,
559 projid))) {
560 error = SET_ERROR(EDQUOT);
561 break;
562 }
563
564 uint64_t blksz;
565 if (lr->lr_length == UINT64_MAX && zp->z_size <= zp->z_blksz) {
566 if (zp->z_blksz > zfsvfs->z_max_blksz &&
567 !ISP2(zp->z_blksz)) {
568 /*
569 * File's blocksize is already larger than the
570 * "recordsize" property. Only let it grow to
571 * the next power of 2.
572 */
573 blksz = 1 << highbit64(zp->z_blksz);
574 } else {
575 blksz = zfsvfs->z_max_blksz;
576 }
577 blksz = MIN(blksz, P2ROUNDUP(end_size,
578 SPA_MINBLOCKSIZE));
579 blksz = MAX(blksz, zp->z_blksz);
580 } else {
581 blksz = zp->z_blksz;
582 }
583
584 arc_buf_t *abuf = NULL;
585 ssize_t nbytes = n;
586 if (n >= blksz && woff >= zp->z_size &&
587 P2PHASE(woff, blksz) == 0 &&
588 (blksz >= SPA_OLD_MAXBLOCKSIZE || n < 4 * blksz)) {
589 /*
590 * This write covers a full block. "Borrow" a buffer
591 * from the dmu so that we can fill it before we enter
592 * a transaction. This avoids the possibility of
593 * holding up the transaction if the data copy hangs
594 * up on a pagefault (e.g., from an NFS server mapping).
595 */
596 abuf = dmu_request_arcbuf(sa_get_db(zp->z_sa_hdl),
597 blksz);
598 ASSERT(abuf != NULL);
599 ASSERT(arc_buf_size(abuf) == blksz);
600 if ((error = zfs_uiocopy(abuf->b_data, blksz,
601 UIO_WRITE, uio, &nbytes))) {
602 dmu_return_arcbuf(abuf);
603 break;
604 }
605 ASSERT3S(nbytes, ==, blksz);
606 } else {
607 nbytes = MIN(n, (DMU_MAX_ACCESS >> 1) -
608 P2PHASE(woff, blksz));
609 if (pfbytes < nbytes) {
610 if (zfs_uio_prefaultpages(nbytes, uio)) {
611 error = SET_ERROR(EFAULT);
612 break;
613 }
614 pfbytes = nbytes;
615 }
616 }
617
618 /*
619 * Start a transaction.
620 */
621 dmu_tx_t *tx = dmu_tx_create(zfsvfs->z_os);
622 dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
623 dmu_buf_impl_t *db = (dmu_buf_impl_t *)sa_get_db(zp->z_sa_hdl);
624 DB_DNODE_ENTER(db);
625 dmu_tx_hold_write_by_dnode(tx, DB_DNODE(db), woff, nbytes);
626 DB_DNODE_EXIT(db);
627 zfs_sa_upgrade_txholds(tx, zp);
628 error = dmu_tx_assign(tx, TXG_WAIT);
629 if (error) {
630 dmu_tx_abort(tx);
631 if (abuf != NULL)
632 dmu_return_arcbuf(abuf);
633 break;
634 }
635
636 /*
637 * NB: We must call zfs_clear_setid_bits_if_necessary before
638 * committing the transaction!
639 */
640
641 /*
642 * If rangelock_enter() over-locked we grow the blocksize
643 * and then reduce the lock range. This will only happen
644 * on the first iteration since rangelock_reduce() will
645 * shrink down lr_length to the appropriate size.
646 */
647 if (lr->lr_length == UINT64_MAX) {
648 zfs_grow_blocksize(zp, blksz, tx);
649 zfs_rangelock_reduce(lr, woff, n);
650 }
651
652 ssize_t tx_bytes;
653 if (abuf == NULL) {
654 tx_bytes = zfs_uio_resid(uio);
655 zfs_uio_fault_disable(uio, B_TRUE);
656 error = dmu_write_uio_dbuf(sa_get_db(zp->z_sa_hdl),
657 uio, nbytes, tx);
658 zfs_uio_fault_disable(uio, B_FALSE);
659 #ifdef __linux__
660 if (error == EFAULT) {
661 zfs_clear_setid_bits_if_necessary(zfsvfs, zp,
662 cr, &clear_setid_bits_txg, tx);
663 dmu_tx_commit(tx);
664 /*
665 * Account for partial writes before
666 * continuing the loop.
667 * Update needs to occur before the next
668 * zfs_uio_prefaultpages, or prefaultpages may
669 * error, and we may break the loop early.
670 */
671 n -= tx_bytes - zfs_uio_resid(uio);
672 pfbytes -= tx_bytes - zfs_uio_resid(uio);
673 continue;
674 }
675 #endif
676 /*
677 * On FreeBSD, EFAULT should be propagated back to the
678 * VFS, which will handle faulting and will retry.
679 */
680 if (error != 0 && error != EFAULT) {
681 zfs_clear_setid_bits_if_necessary(zfsvfs, zp,
682 cr, &clear_setid_bits_txg, tx);
683 dmu_tx_commit(tx);
684 break;
685 }
686 tx_bytes -= zfs_uio_resid(uio);
687 } else {
688 /*
689 * Thus, we're writing a full block at a block-aligned
690 * offset and extending the file past EOF.
691 *
692 * dmu_assign_arcbuf_by_dbuf() will directly assign the
693 * arc buffer to a dbuf.
694 */
695 error = dmu_assign_arcbuf_by_dbuf(
696 sa_get_db(zp->z_sa_hdl), woff, abuf, tx);
697 if (error != 0) {
698 /*
699 * XXX This might not be necessary if
700 * dmu_assign_arcbuf_by_dbuf is guaranteed
701 * to be atomic.
702 */
703 zfs_clear_setid_bits_if_necessary(zfsvfs, zp,
704 cr, &clear_setid_bits_txg, tx);
705 dmu_return_arcbuf(abuf);
706 dmu_tx_commit(tx);
707 break;
708 }
709 ASSERT3S(nbytes, <=, zfs_uio_resid(uio));
710 zfs_uioskip(uio, nbytes);
711 tx_bytes = nbytes;
712 }
713 if (tx_bytes &&
714 zn_has_cached_data(zp, woff, woff + tx_bytes - 1) &&
715 !(ioflag & O_DIRECT)) {
716 update_pages(zp, woff, tx_bytes, zfsvfs->z_os);
717 }
718
719 /*
720 * If we made no progress, we're done. If we made even
721 * partial progress, update the znode and ZIL accordingly.
722 */
723 if (tx_bytes == 0) {
724 (void) sa_update(zp->z_sa_hdl, SA_ZPL_SIZE(zfsvfs),
725 (void *)&zp->z_size, sizeof (uint64_t), tx);
726 dmu_tx_commit(tx);
727 ASSERT(error != 0);
728 break;
729 }
730
731 zfs_clear_setid_bits_if_necessary(zfsvfs, zp, cr,
732 &clear_setid_bits_txg, tx);
733
734 zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime);
735
736 /*
737 * Update the file size (zp_size) if it has changed;
738 * account for possible concurrent updates.
739 */
740 while ((end_size = zp->z_size) < zfs_uio_offset(uio)) {
741 (void) atomic_cas_64(&zp->z_size, end_size,
742 zfs_uio_offset(uio));
743 ASSERT(error == 0 || error == EFAULT);
744 }
745 /*
746 * If we are replaying and eof is non zero then force
747 * the file size to the specified eof. Note, there's no
748 * concurrency during replay.
749 */
750 if (zfsvfs->z_replay && zfsvfs->z_replay_eof != 0)
751 zp->z_size = zfsvfs->z_replay_eof;
752
753 error1 = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
754 if (error1 != 0)
755 /* Avoid clobbering EFAULT. */
756 error = error1;
757
758 /*
759 * NB: During replay, the TX_SETATTR record logged by
760 * zfs_clear_setid_bits_if_necessary must precede any of
761 * the TX_WRITE records logged here.
762 */
763 zfs_log_write(zilog, tx, TX_WRITE, zp, woff, tx_bytes, ioflag,
764 NULL, NULL);
765
766 dmu_tx_commit(tx);
767
768 if (error != 0)
769 break;
770 ASSERT3S(tx_bytes, ==, nbytes);
771 n -= nbytes;
772 pfbytes -= nbytes;
773 }
774
775 zfs_znode_update_vfs(zp);
776 zfs_rangelock_exit(lr);
777
778 /*
779 * If we're in replay mode, or we made no progress, or the
780 * uio data is inaccessible return an error. Otherwise, it's
781 * at least a partial write, so it's successful.
782 */
783 if (zfsvfs->z_replay || zfs_uio_resid(uio) == start_resid ||
784 error == EFAULT) {
785 zfs_exit(zfsvfs, FTAG);
786 return (error);
787 }
788
789 if (ioflag & (O_SYNC | O_DSYNC) ||
790 zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
791 zil_commit(zilog, zp->z_id);
792
793 const int64_t nwritten = start_resid - zfs_uio_resid(uio);
794 dataset_kstats_update_write_kstats(&zfsvfs->z_kstat, nwritten);
795 task_io_account_write(nwritten);
796
797 zfs_exit(zfsvfs, FTAG);
798 return (0);
799 }
800
801 int
zfs_getsecattr(znode_t * zp,vsecattr_t * vsecp,int flag,cred_t * cr)802 zfs_getsecattr(znode_t *zp, vsecattr_t *vsecp, int flag, cred_t *cr)
803 {
804 zfsvfs_t *zfsvfs = ZTOZSB(zp);
805 int error;
806 boolean_t skipaclchk = (flag & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;
807
808 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
809 return (error);
810 error = zfs_getacl(zp, vsecp, skipaclchk, cr);
811 zfs_exit(zfsvfs, FTAG);
812
813 return (error);
814 }
815
816 int
zfs_setsecattr(znode_t * zp,vsecattr_t * vsecp,int flag,cred_t * cr)817 zfs_setsecattr(znode_t *zp, vsecattr_t *vsecp, int flag, cred_t *cr)
818 {
819 zfsvfs_t *zfsvfs = ZTOZSB(zp);
820 int error;
821 boolean_t skipaclchk = (flag & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;
822 zilog_t *zilog;
823
824 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
825 return (error);
826 zilog = zfsvfs->z_log;
827 error = zfs_setacl(zp, vsecp, skipaclchk, cr);
828
829 if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
830 zil_commit(zilog, 0);
831
832 zfs_exit(zfsvfs, FTAG);
833 return (error);
834 }
835
836 #ifdef ZFS_DEBUG
837 static int zil_fault_io = 0;
838 #endif
839
840 static void zfs_get_done(zgd_t *zgd, int error);
841
842 /*
843 * Get data to generate a TX_WRITE intent log record.
844 */
845 int
zfs_get_data(void * arg,uint64_t gen,lr_write_t * lr,char * buf,struct lwb * lwb,zio_t * zio)846 zfs_get_data(void *arg, uint64_t gen, lr_write_t *lr, char *buf,
847 struct lwb *lwb, zio_t *zio)
848 {
849 zfsvfs_t *zfsvfs = arg;
850 objset_t *os = zfsvfs->z_os;
851 znode_t *zp;
852 uint64_t object = lr->lr_foid;
853 uint64_t offset = lr->lr_offset;
854 uint64_t size = lr->lr_length;
855 dmu_buf_t *db;
856 zgd_t *zgd;
857 int error = 0;
858 uint64_t zp_gen;
859
860 ASSERT3P(lwb, !=, NULL);
861 ASSERT3U(size, !=, 0);
862
863 /*
864 * Nothing to do if the file has been removed
865 */
866 if (zfs_zget(zfsvfs, object, &zp) != 0)
867 return (SET_ERROR(ENOENT));
868 if (zp->z_unlinked) {
869 /*
870 * Release the vnode asynchronously as we currently have the
871 * txg stopped from syncing.
872 */
873 zfs_zrele_async(zp);
874 return (SET_ERROR(ENOENT));
875 }
876 /* check if generation number matches */
877 if (sa_lookup(zp->z_sa_hdl, SA_ZPL_GEN(zfsvfs), &zp_gen,
878 sizeof (zp_gen)) != 0) {
879 zfs_zrele_async(zp);
880 return (SET_ERROR(EIO));
881 }
882 if (zp_gen != gen) {
883 zfs_zrele_async(zp);
884 return (SET_ERROR(ENOENT));
885 }
886
887 zgd = kmem_zalloc(sizeof (zgd_t), KM_SLEEP);
888 zgd->zgd_lwb = lwb;
889 zgd->zgd_private = zp;
890
891 /*
892 * Write records come in two flavors: immediate and indirect.
893 * For small writes it's cheaper to store the data with the
894 * log record (immediate); for large writes it's cheaper to
895 * sync the data and get a pointer to it (indirect) so that
896 * we don't have to write the data twice.
897 */
898 if (buf != NULL) { /* immediate write */
899 zgd->zgd_lr = zfs_rangelock_enter(&zp->z_rangelock,
900 offset, size, RL_READER);
901 /* test for truncation needs to be done while range locked */
902 if (offset >= zp->z_size) {
903 error = SET_ERROR(ENOENT);
904 } else {
905 error = dmu_read(os, object, offset, size, buf,
906 DMU_READ_NO_PREFETCH);
907 }
908 ASSERT(error == 0 || error == ENOENT);
909 } else { /* indirect write */
910 ASSERT3P(zio, !=, NULL);
911 /*
912 * Have to lock the whole block to ensure when it's
913 * written out and its checksum is being calculated
914 * that no one can change the data. We need to re-check
915 * blocksize after we get the lock in case it's changed!
916 */
917 for (;;) {
918 uint64_t blkoff;
919 size = zp->z_blksz;
920 blkoff = ISP2(size) ? P2PHASE(offset, size) : offset;
921 offset -= blkoff;
922 zgd->zgd_lr = zfs_rangelock_enter(&zp->z_rangelock,
923 offset, size, RL_READER);
924 if (zp->z_blksz == size)
925 break;
926 offset += blkoff;
927 zfs_rangelock_exit(zgd->zgd_lr);
928 }
929 /* test for truncation needs to be done while range locked */
930 if (lr->lr_offset >= zp->z_size)
931 error = SET_ERROR(ENOENT);
932 #ifdef ZFS_DEBUG
933 if (zil_fault_io) {
934 error = SET_ERROR(EIO);
935 zil_fault_io = 0;
936 }
937 #endif
938 if (error == 0)
939 error = dmu_buf_hold_noread(os, object, offset, zgd,
940 &db);
941
942 if (error == 0) {
943 blkptr_t *bp = &lr->lr_blkptr;
944
945 zgd->zgd_db = db;
946 zgd->zgd_bp = bp;
947
948 ASSERT(db->db_offset == offset);
949 ASSERT(db->db_size == size);
950
951 error = dmu_sync(zio, lr->lr_common.lrc_txg,
952 zfs_get_done, zgd);
953 ASSERT(error || lr->lr_length <= size);
954
955 /*
956 * On success, we need to wait for the write I/O
957 * initiated by dmu_sync() to complete before we can
958 * release this dbuf. We will finish everything up
959 * in the zfs_get_done() callback.
960 */
961 if (error == 0)
962 return (0);
963
964 if (error == EALREADY) {
965 lr->lr_common.lrc_txtype = TX_WRITE2;
966 /*
967 * TX_WRITE2 relies on the data previously
968 * written by the TX_WRITE that caused
969 * EALREADY. We zero out the BP because
970 * it is the old, currently-on-disk BP.
971 */
972 zgd->zgd_bp = NULL;
973 BP_ZERO(bp);
974 error = 0;
975 }
976 }
977 }
978
979 zfs_get_done(zgd, error);
980
981 return (error);
982 }
983
984
985 static void
zfs_get_done(zgd_t * zgd,int error)986 zfs_get_done(zgd_t *zgd, int error)
987 {
988 (void) error;
989 znode_t *zp = zgd->zgd_private;
990
991 if (zgd->zgd_db)
992 dmu_buf_rele(zgd->zgd_db, zgd);
993
994 zfs_rangelock_exit(zgd->zgd_lr);
995
996 /*
997 * Release the vnode asynchronously as we currently have the
998 * txg stopped from syncing.
999 */
1000 zfs_zrele_async(zp);
1001
1002 kmem_free(zgd, sizeof (zgd_t));
1003 }
1004
1005 static int
zfs_enter_two(zfsvfs_t * zfsvfs1,zfsvfs_t * zfsvfs2,const char * tag)1006 zfs_enter_two(zfsvfs_t *zfsvfs1, zfsvfs_t *zfsvfs2, const char *tag)
1007 {
1008 int error;
1009
1010 /* Swap. Not sure if the order of zfs_enter()s is important. */
1011 if (zfsvfs1 > zfsvfs2) {
1012 zfsvfs_t *tmpzfsvfs;
1013
1014 tmpzfsvfs = zfsvfs2;
1015 zfsvfs2 = zfsvfs1;
1016 zfsvfs1 = tmpzfsvfs;
1017 }
1018
1019 error = zfs_enter(zfsvfs1, tag);
1020 if (error != 0)
1021 return (error);
1022 if (zfsvfs1 != zfsvfs2) {
1023 error = zfs_enter(zfsvfs2, tag);
1024 if (error != 0) {
1025 zfs_exit(zfsvfs1, tag);
1026 return (error);
1027 }
1028 }
1029
1030 return (0);
1031 }
1032
1033 static void
zfs_exit_two(zfsvfs_t * zfsvfs1,zfsvfs_t * zfsvfs2,const char * tag)1034 zfs_exit_two(zfsvfs_t *zfsvfs1, zfsvfs_t *zfsvfs2, const char *tag)
1035 {
1036
1037 zfs_exit(zfsvfs1, tag);
1038 if (zfsvfs1 != zfsvfs2)
1039 zfs_exit(zfsvfs2, tag);
1040 }
1041
1042 /*
1043 * We split each clone request in chunks that can fit into a single ZIL
1044 * log entry. Each ZIL log entry can fit 130816 bytes for a block cloning
1045 * operation (see zil_max_log_data() and zfs_log_clone_range()). This gives
1046 * us room for storing 1022 block pointers.
1047 *
1048 * On success, the function return the number of bytes copied in *lenp.
1049 * Note, it doesn't return how much bytes are left to be copied.
1050 * On errors which are caused by any file system limitations or
1051 * brt limitations `EINVAL` is returned. In the most cases a user
1052 * requested bad parameters, it could be possible to clone the file but
1053 * some parameters don't match the requirements.
1054 */
1055 int
zfs_clone_range(znode_t * inzp,uint64_t * inoffp,znode_t * outzp,uint64_t * outoffp,uint64_t * lenp,cred_t * cr)1056 zfs_clone_range(znode_t *inzp, uint64_t *inoffp, znode_t *outzp,
1057 uint64_t *outoffp, uint64_t *lenp, cred_t *cr)
1058 {
1059 zfsvfs_t *inzfsvfs, *outzfsvfs;
1060 objset_t *inos, *outos;
1061 zfs_locked_range_t *inlr, *outlr;
1062 dmu_buf_impl_t *db;
1063 dmu_tx_t *tx;
1064 zilog_t *zilog;
1065 uint64_t inoff, outoff, len, done;
1066 uint64_t outsize, size;
1067 int error;
1068 int count = 0;
1069 sa_bulk_attr_t bulk[3];
1070 uint64_t mtime[2], ctime[2];
1071 uint64_t uid, gid, projid;
1072 blkptr_t *bps;
1073 size_t maxblocks, nbps;
1074 uint_t inblksz;
1075 uint64_t clear_setid_bits_txg = 0;
1076 uint64_t last_synced_txg = 0;
1077
1078 inoff = *inoffp;
1079 outoff = *outoffp;
1080 len = *lenp;
1081 done = 0;
1082
1083 inzfsvfs = ZTOZSB(inzp);
1084 outzfsvfs = ZTOZSB(outzp);
1085
1086 /*
1087 * We need to call zfs_enter() potentially on two different datasets,
1088 * so we need a dedicated function for that.
1089 */
1090 error = zfs_enter_two(inzfsvfs, outzfsvfs, FTAG);
1091 if (error != 0)
1092 return (error);
1093
1094 inos = inzfsvfs->z_os;
1095 outos = outzfsvfs->z_os;
1096
1097 /*
1098 * Both source and destination have to belong to the same storage pool.
1099 */
1100 if (dmu_objset_spa(inos) != dmu_objset_spa(outos)) {
1101 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1102 return (SET_ERROR(EXDEV));
1103 }
1104
1105 /*
1106 * outos and inos belongs to the same storage pool.
1107 * see a few lines above, only one check.
1108 */
1109 if (!spa_feature_is_enabled(dmu_objset_spa(outos),
1110 SPA_FEATURE_BLOCK_CLONING)) {
1111 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1112 return (SET_ERROR(EOPNOTSUPP));
1113 }
1114
1115 ASSERT(!outzfsvfs->z_replay);
1116
1117 /*
1118 * Block cloning from an unencrypted dataset into an encrypted
1119 * dataset and vice versa is not supported.
1120 */
1121 if (inos->os_encrypted != outos->os_encrypted) {
1122 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1123 return (SET_ERROR(EXDEV));
1124 }
1125
1126 /*
1127 * Cloning across encrypted datasets is possible only if they
1128 * share the same master key.
1129 */
1130 if (inos != outos && inos->os_encrypted &&
1131 !dmu_objset_crypto_key_equal(inos, outos)) {
1132 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1133 return (SET_ERROR(EXDEV));
1134 }
1135
1136 error = zfs_verify_zp(inzp);
1137 if (error == 0)
1138 error = zfs_verify_zp(outzp);
1139 if (error != 0) {
1140 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1141 return (error);
1142 }
1143
1144 /*
1145 * We don't copy source file's flags that's why we don't allow to clone
1146 * files that are in quarantine.
1147 */
1148 if (inzp->z_pflags & ZFS_AV_QUARANTINED) {
1149 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1150 return (SET_ERROR(EACCES));
1151 }
1152
1153 if (inoff >= inzp->z_size) {
1154 *lenp = 0;
1155 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1156 return (0);
1157 }
1158 if (len > inzp->z_size - inoff) {
1159 len = inzp->z_size - inoff;
1160 }
1161 if (len == 0) {
1162 *lenp = 0;
1163 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1164 return (0);
1165 }
1166
1167 /*
1168 * Callers might not be able to detect properly that we are read-only,
1169 * so check it explicitly here.
1170 */
1171 if (zfs_is_readonly(outzfsvfs)) {
1172 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1173 return (SET_ERROR(EROFS));
1174 }
1175
1176 /*
1177 * If immutable or not appending then return EPERM.
1178 * Intentionally allow ZFS_READONLY through here.
1179 * See zfs_zaccess_common()
1180 */
1181 if ((outzp->z_pflags & ZFS_IMMUTABLE) != 0) {
1182 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1183 return (SET_ERROR(EPERM));
1184 }
1185
1186 /*
1187 * No overlapping if we are cloning within the same file.
1188 */
1189 if (inzp == outzp) {
1190 if (inoff < outoff + len && outoff < inoff + len) {
1191 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1192 return (SET_ERROR(EINVAL));
1193 }
1194 }
1195
1196 /* Flush any mmap()'d data to disk */
1197 if (zn_has_cached_data(inzp, inoff, inoff + len - 1))
1198 zn_flush_cached_data(inzp, B_TRUE);
1199
1200 /*
1201 * Maintain predictable lock order.
1202 */
1203 if (inzp < outzp || (inzp == outzp && inoff < outoff)) {
1204 inlr = zfs_rangelock_enter(&inzp->z_rangelock, inoff, len,
1205 RL_READER);
1206 outlr = zfs_rangelock_enter(&outzp->z_rangelock, outoff, len,
1207 RL_WRITER);
1208 } else {
1209 outlr = zfs_rangelock_enter(&outzp->z_rangelock, outoff, len,
1210 RL_WRITER);
1211 inlr = zfs_rangelock_enter(&inzp->z_rangelock, inoff, len,
1212 RL_READER);
1213 }
1214
1215 inblksz = inzp->z_blksz;
1216
1217 /*
1218 * We cannot clone into a file with different block size if we can't
1219 * grow it (block size is already bigger, has more than one block, or
1220 * not locked for growth). There are other possible reasons for the
1221 * grow to fail, but we cover what we can before opening transaction
1222 * and the rest detect after we try to do it.
1223 */
1224 if (inblksz < outzp->z_blksz) {
1225 error = SET_ERROR(EINVAL);
1226 goto unlock;
1227 }
1228 if (inblksz != outzp->z_blksz && (outzp->z_size > outzp->z_blksz ||
1229 outlr->lr_length != UINT64_MAX)) {
1230 error = SET_ERROR(EINVAL);
1231 goto unlock;
1232 }
1233
1234 /*
1235 * Block size must be power-of-2 if destination offset != 0.
1236 * There can be no multiple blocks of non-power-of-2 size.
1237 */
1238 if (outoff != 0 && !ISP2(inblksz)) {
1239 error = SET_ERROR(EINVAL);
1240 goto unlock;
1241 }
1242
1243 /*
1244 * Offsets and len must be at block boundries.
1245 */
1246 if ((inoff % inblksz) != 0 || (outoff % inblksz) != 0) {
1247 error = SET_ERROR(EINVAL);
1248 goto unlock;
1249 }
1250 /*
1251 * Length must be multipe of blksz, except for the end of the file.
1252 */
1253 if ((len % inblksz) != 0 &&
1254 (len < inzp->z_size - inoff || len < outzp->z_size - outoff)) {
1255 error = SET_ERROR(EINVAL);
1256 goto unlock;
1257 }
1258
1259 /*
1260 * If we are copying only one block and it is smaller than recordsize
1261 * property, do not allow destination to grow beyond one block if it
1262 * is not there yet. Otherwise the destination will get stuck with
1263 * that block size forever, that can be as small as 512 bytes, no
1264 * matter how big the destination grow later.
1265 */
1266 if (len <= inblksz && inblksz < outzfsvfs->z_max_blksz &&
1267 outzp->z_size <= inblksz && outoff + len > inblksz) {
1268 error = SET_ERROR(EINVAL);
1269 goto unlock;
1270 }
1271
1272 error = zn_rlimit_fsize(outoff + len);
1273 if (error != 0) {
1274 goto unlock;
1275 }
1276
1277 if (inoff >= MAXOFFSET_T || outoff >= MAXOFFSET_T) {
1278 error = SET_ERROR(EFBIG);
1279 goto unlock;
1280 }
1281
1282 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(outzfsvfs), NULL,
1283 &mtime, 16);
1284 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(outzfsvfs), NULL,
1285 &ctime, 16);
1286 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(outzfsvfs), NULL,
1287 &outzp->z_size, 8);
1288
1289 zilog = outzfsvfs->z_log;
1290 maxblocks = zil_max_log_data(zilog, sizeof (lr_clone_range_t)) /
1291 sizeof (bps[0]);
1292
1293 uid = KUID_TO_SUID(ZTOUID(outzp));
1294 gid = KGID_TO_SGID(ZTOGID(outzp));
1295 projid = outzp->z_projid;
1296
1297 bps = vmem_alloc(sizeof (bps[0]) * maxblocks, KM_SLEEP);
1298
1299 /*
1300 * Clone the file in reasonable size chunks. Each chunk is cloned
1301 * in a separate transaction; this keeps the intent log records small
1302 * and allows us to do more fine-grained space accounting.
1303 */
1304 while (len > 0) {
1305 size = MIN(inblksz * maxblocks, len);
1306
1307 if (zfs_id_overblockquota(outzfsvfs, DMU_USERUSED_OBJECT,
1308 uid) ||
1309 zfs_id_overblockquota(outzfsvfs, DMU_GROUPUSED_OBJECT,
1310 gid) ||
1311 (projid != ZFS_DEFAULT_PROJID &&
1312 zfs_id_overblockquota(outzfsvfs, DMU_PROJECTUSED_OBJECT,
1313 projid))) {
1314 error = SET_ERROR(EDQUOT);
1315 break;
1316 }
1317
1318 nbps = maxblocks;
1319 last_synced_txg = spa_last_synced_txg(dmu_objset_spa(inos));
1320 error = dmu_read_l0_bps(inos, inzp->z_id, inoff, size, bps,
1321 &nbps);
1322 if (error != 0) {
1323 /*
1324 * If we are trying to clone a block that was created
1325 * in the current transaction group, the error will be
1326 * EAGAIN here. Based on zfs_bclone_wait_dirty either
1327 * return a shortened range to the caller so it can
1328 * fallback, or wait for the next TXG and check again.
1329 */
1330 if (error == EAGAIN && zfs_bclone_wait_dirty) {
1331 txg_wait_synced(dmu_objset_pool(inos),
1332 last_synced_txg + 1);
1333 continue;
1334 }
1335
1336 break;
1337 }
1338
1339 /*
1340 * Start a transaction.
1341 */
1342 tx = dmu_tx_create(outos);
1343 dmu_tx_hold_sa(tx, outzp->z_sa_hdl, B_FALSE);
1344 db = (dmu_buf_impl_t *)sa_get_db(outzp->z_sa_hdl);
1345 DB_DNODE_ENTER(db);
1346 dmu_tx_hold_clone_by_dnode(tx, DB_DNODE(db), outoff, size);
1347 DB_DNODE_EXIT(db);
1348 zfs_sa_upgrade_txholds(tx, outzp);
1349 error = dmu_tx_assign(tx, TXG_WAIT);
1350 if (error != 0) {
1351 dmu_tx_abort(tx);
1352 break;
1353 }
1354
1355 /*
1356 * Copy source znode's block size. This is done only if the
1357 * whole znode is locked (see zfs_rangelock_cb()) and only
1358 * on the first iteration since zfs_rangelock_reduce() will
1359 * shrink down lr_length to the appropriate size.
1360 */
1361 if (outlr->lr_length == UINT64_MAX) {
1362 zfs_grow_blocksize(outzp, inblksz, tx);
1363
1364 /*
1365 * Block growth may fail for many reasons we can not
1366 * predict here. If it happen the cloning is doomed.
1367 */
1368 if (inblksz != outzp->z_blksz) {
1369 error = SET_ERROR(EINVAL);
1370 dmu_tx_abort(tx);
1371 break;
1372 }
1373
1374 /*
1375 * Round range lock up to the block boundary, so we
1376 * prevent appends until we are done.
1377 */
1378 zfs_rangelock_reduce(outlr, outoff,
1379 ((len - 1) / inblksz + 1) * inblksz);
1380 }
1381
1382 error = dmu_brt_clone(outos, outzp->z_id, outoff, size, tx,
1383 bps, nbps);
1384 if (error != 0) {
1385 dmu_tx_commit(tx);
1386 break;
1387 }
1388
1389 if (zn_has_cached_data(outzp, outoff, outoff + size - 1)) {
1390 update_pages(outzp, outoff, size, outos);
1391 }
1392
1393 zfs_clear_setid_bits_if_necessary(outzfsvfs, outzp, cr,
1394 &clear_setid_bits_txg, tx);
1395
1396 zfs_tstamp_update_setup(outzp, CONTENT_MODIFIED, mtime, ctime);
1397
1398 /*
1399 * Update the file size (zp_size) if it has changed;
1400 * account for possible concurrent updates.
1401 */
1402 while ((outsize = outzp->z_size) < outoff + size) {
1403 (void) atomic_cas_64(&outzp->z_size, outsize,
1404 outoff + size);
1405 }
1406
1407 error = sa_bulk_update(outzp->z_sa_hdl, bulk, count, tx);
1408
1409 zfs_log_clone_range(zilog, tx, TX_CLONE_RANGE, outzp, outoff,
1410 size, inblksz, bps, nbps);
1411
1412 dmu_tx_commit(tx);
1413
1414 if (error != 0)
1415 break;
1416
1417 inoff += size;
1418 outoff += size;
1419 len -= size;
1420 done += size;
1421 }
1422
1423 vmem_free(bps, sizeof (bps[0]) * maxblocks);
1424 zfs_znode_update_vfs(outzp);
1425
1426 unlock:
1427 zfs_rangelock_exit(outlr);
1428 zfs_rangelock_exit(inlr);
1429
1430 if (done > 0) {
1431 /*
1432 * If we have made at least partial progress, reset the error.
1433 */
1434 error = 0;
1435
1436 ZFS_ACCESSTIME_STAMP(inzfsvfs, inzp);
1437
1438 if (outos->os_sync == ZFS_SYNC_ALWAYS) {
1439 zil_commit(zilog, outzp->z_id);
1440 }
1441
1442 *inoffp += done;
1443 *outoffp += done;
1444 *lenp = done;
1445 } else {
1446 /*
1447 * If we made no progress, there must be a good reason.
1448 * EOF is handled explicitly above, before the loop.
1449 */
1450 ASSERT3S(error, !=, 0);
1451 }
1452
1453 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1454
1455 return (error);
1456 }
1457
1458 /*
1459 * Usual pattern would be to call zfs_clone_range() from zfs_replay_clone(),
1460 * but we cannot do that, because when replaying we don't have source znode
1461 * available. This is why we need a dedicated replay function.
1462 */
1463 int
zfs_clone_range_replay(znode_t * zp,uint64_t off,uint64_t len,uint64_t blksz,const blkptr_t * bps,size_t nbps)1464 zfs_clone_range_replay(znode_t *zp, uint64_t off, uint64_t len, uint64_t blksz,
1465 const blkptr_t *bps, size_t nbps)
1466 {
1467 zfsvfs_t *zfsvfs;
1468 dmu_buf_impl_t *db;
1469 dmu_tx_t *tx;
1470 int error;
1471 int count = 0;
1472 sa_bulk_attr_t bulk[3];
1473 uint64_t mtime[2], ctime[2];
1474
1475 ASSERT3U(off, <, MAXOFFSET_T);
1476 ASSERT3U(len, >, 0);
1477 ASSERT3U(nbps, >, 0);
1478
1479 zfsvfs = ZTOZSB(zp);
1480
1481 ASSERT(spa_feature_is_enabled(dmu_objset_spa(zfsvfs->z_os),
1482 SPA_FEATURE_BLOCK_CLONING));
1483
1484 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
1485 return (error);
1486
1487 ASSERT(zfsvfs->z_replay);
1488 ASSERT(!zfs_is_readonly(zfsvfs));
1489
1490 if ((off % blksz) != 0) {
1491 zfs_exit(zfsvfs, FTAG);
1492 return (SET_ERROR(EINVAL));
1493 }
1494
1495 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
1496 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
1497 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL,
1498 &zp->z_size, 8);
1499
1500 /*
1501 * Start a transaction.
1502 */
1503 tx = dmu_tx_create(zfsvfs->z_os);
1504
1505 dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
1506 db = (dmu_buf_impl_t *)sa_get_db(zp->z_sa_hdl);
1507 DB_DNODE_ENTER(db);
1508 dmu_tx_hold_clone_by_dnode(tx, DB_DNODE(db), off, len);
1509 DB_DNODE_EXIT(db);
1510 zfs_sa_upgrade_txholds(tx, zp);
1511 error = dmu_tx_assign(tx, TXG_WAIT);
1512 if (error != 0) {
1513 dmu_tx_abort(tx);
1514 zfs_exit(zfsvfs, FTAG);
1515 return (error);
1516 }
1517
1518 if (zp->z_blksz < blksz)
1519 zfs_grow_blocksize(zp, blksz, tx);
1520
1521 dmu_brt_clone(zfsvfs->z_os, zp->z_id, off, len, tx, bps, nbps);
1522
1523 zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime);
1524
1525 if (zp->z_size < off + len)
1526 zp->z_size = off + len;
1527
1528 error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
1529
1530 /*
1531 * zil_replaying() not only check if we are replaying ZIL, but also
1532 * updates the ZIL header to record replay progress.
1533 */
1534 VERIFY(zil_replaying(zfsvfs->z_log, tx));
1535
1536 dmu_tx_commit(tx);
1537
1538 zfs_znode_update_vfs(zp);
1539
1540 zfs_exit(zfsvfs, FTAG);
1541
1542 return (error);
1543 }
1544
1545 EXPORT_SYMBOL(zfs_access);
1546 EXPORT_SYMBOL(zfs_fsync);
1547 EXPORT_SYMBOL(zfs_holey);
1548 EXPORT_SYMBOL(zfs_read);
1549 EXPORT_SYMBOL(zfs_write);
1550 EXPORT_SYMBOL(zfs_getsecattr);
1551 EXPORT_SYMBOL(zfs_setsecattr);
1552 EXPORT_SYMBOL(zfs_clone_range);
1553 EXPORT_SYMBOL(zfs_clone_range_replay);
1554
1555 ZFS_MODULE_PARAM(zfs_vnops, zfs_vnops_, read_chunk_size, U64, ZMOD_RW,
1556 "Bytes to read per chunk");
1557
1558 ZFS_MODULE_PARAM(zfs, zfs_, bclone_enabled, INT, ZMOD_RW,
1559 "Enable block cloning");
1560
1561 ZFS_MODULE_PARAM(zfs, zfs_, bclone_wait_dirty, INT, ZMOD_RW,
1562 "Wait for dirty blocks when cloning");
1563