1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2012, 2020 by Delphix. All rights reserved.
23 */
24
25 #include <sys/dataset_kstats.h>
26 #include <sys/dbuf.h>
27 #include <sys/dmu_traverse.h>
28 #include <sys/dsl_dataset.h>
29 #include <sys/dsl_prop.h>
30 #include <sys/dsl_dir.h>
31 #include <sys/zap.h>
32 #include <sys/zfeature.h>
33 #include <sys/zil_impl.h>
34 #include <sys/dmu_tx.h>
35 #include <sys/zio.h>
36 #include <sys/zfs_rlock.h>
37 #include <sys/spa_impl.h>
38 #include <sys/zvol.h>
39 #include <sys/zvol_impl.h>
40
41 #include <linux/blkdev_compat.h>
42 #include <linux/task_io_accounting_ops.h>
43
44 unsigned int zvol_major = ZVOL_MAJOR;
45 unsigned int zvol_request_sync = 0;
46 unsigned int zvol_prefetch_bytes = (128 * 1024);
47 unsigned long zvol_max_discard_blocks = 16384;
48 unsigned int zvol_threads = 32;
49
50 struct zvol_state_os {
51 struct gendisk *zvo_disk; /* generic disk */
52 struct request_queue *zvo_queue; /* request queue */
53 dev_t zvo_dev; /* device id */
54 };
55
56 taskq_t *zvol_taskq;
57 static struct ida zvol_ida;
58
59 typedef struct zv_request {
60 zvol_state_t *zv;
61 struct bio *bio;
62 taskq_ent_t ent;
63 } zv_request_t;
64
65 /*
66 * Given a path, return TRUE if path is a ZVOL.
67 */
68 static boolean_t
zvol_is_zvol_impl(const char * path)69 zvol_is_zvol_impl(const char *path)
70 {
71 dev_t dev = 0;
72
73 if (vdev_lookup_bdev(path, &dev) != 0)
74 return (B_FALSE);
75
76 if (MAJOR(dev) == zvol_major)
77 return (B_TRUE);
78
79 return (B_FALSE);
80 }
81
82 static void
zvol_write(void * arg)83 zvol_write(void *arg)
84 {
85 zv_request_t *zvr = arg;
86 struct bio *bio = zvr->bio;
87 int error = 0;
88 uio_t uio;
89
90 uio_bvec_init(&uio, bio);
91
92 zvol_state_t *zv = zvr->zv;
93 ASSERT3P(zv, !=, NULL);
94 ASSERT3U(zv->zv_open_count, >, 0);
95 ASSERT3P(zv->zv_zilog, !=, NULL);
96
97 /* bio marked as FLUSH need to flush before write */
98 if (bio_is_flush(bio))
99 zil_commit(zv->zv_zilog, ZVOL_OBJ);
100
101 /* Some requests are just for flush and nothing else. */
102 if (uio.uio_resid == 0) {
103 rw_exit(&zv->zv_suspend_lock);
104 BIO_END_IO(bio, 0);
105 kmem_free(zvr, sizeof (zv_request_t));
106 return;
107 }
108
109 struct request_queue *q = zv->zv_zso->zvo_queue;
110 struct gendisk *disk = zv->zv_zso->zvo_disk;
111 ssize_t start_resid = uio.uio_resid;
112 unsigned long start_time;
113
114 boolean_t acct = blk_queue_io_stat(q);
115 if (acct)
116 start_time = blk_generic_start_io_acct(q, disk, WRITE, bio);
117
118 boolean_t sync =
119 bio_is_fua(bio) || zv->zv_objset->os_sync == ZFS_SYNC_ALWAYS;
120
121 zfs_locked_range_t *lr = zfs_rangelock_enter(&zv->zv_rangelock,
122 uio.uio_loffset, uio.uio_resid, RL_WRITER);
123
124 uint64_t volsize = zv->zv_volsize;
125 while (uio.uio_resid > 0 && uio.uio_loffset < volsize) {
126 uint64_t bytes = MIN(uio.uio_resid, DMU_MAX_ACCESS >> 1);
127 uint64_t off = uio.uio_loffset;
128 dmu_tx_t *tx = dmu_tx_create(zv->zv_objset);
129
130 if (bytes > volsize - off) /* don't write past the end */
131 bytes = volsize - off;
132
133 dmu_tx_hold_write_by_dnode(tx, zv->zv_dn, off, bytes);
134
135 /* This will only fail for ENOSPC */
136 error = dmu_tx_assign(tx, TXG_WAIT);
137 if (error) {
138 dmu_tx_abort(tx);
139 break;
140 }
141 error = dmu_write_uio_dnode(zv->zv_dn, &uio, bytes, tx);
142 if (error == 0) {
143 zvol_log_write(zv, tx, off, bytes, sync);
144 }
145 dmu_tx_commit(tx);
146
147 if (error)
148 break;
149 }
150 zfs_rangelock_exit(lr);
151
152 int64_t nwritten = start_resid - uio.uio_resid;
153 dataset_kstats_update_write_kstats(&zv->zv_kstat, nwritten);
154 task_io_account_write(nwritten);
155
156 if (sync)
157 zil_commit(zv->zv_zilog, ZVOL_OBJ);
158
159 rw_exit(&zv->zv_suspend_lock);
160
161 if (acct)
162 blk_generic_end_io_acct(q, disk, WRITE, bio, start_time);
163
164 BIO_END_IO(bio, -error);
165 kmem_free(zvr, sizeof (zv_request_t));
166 }
167
168 static void
zvol_discard(void * arg)169 zvol_discard(void *arg)
170 {
171 zv_request_t *zvr = arg;
172 struct bio *bio = zvr->bio;
173 zvol_state_t *zv = zvr->zv;
174 uint64_t start = BIO_BI_SECTOR(bio) << 9;
175 uint64_t size = BIO_BI_SIZE(bio);
176 uint64_t end = start + size;
177 boolean_t sync;
178 int error = 0;
179 dmu_tx_t *tx;
180
181 ASSERT3P(zv, !=, NULL);
182 ASSERT3U(zv->zv_open_count, >, 0);
183 ASSERT3P(zv->zv_zilog, !=, NULL);
184
185 struct request_queue *q = zv->zv_zso->zvo_queue;
186 struct gendisk *disk = zv->zv_zso->zvo_disk;
187 unsigned long start_time;
188
189 boolean_t acct = blk_queue_io_stat(q);
190 if (acct)
191 start_time = blk_generic_start_io_acct(q, disk, WRITE, bio);
192
193 sync = bio_is_fua(bio) || zv->zv_objset->os_sync == ZFS_SYNC_ALWAYS;
194
195 if (end > zv->zv_volsize) {
196 error = SET_ERROR(EIO);
197 goto unlock;
198 }
199
200 /*
201 * Align the request to volume block boundaries when a secure erase is
202 * not required. This will prevent dnode_free_range() from zeroing out
203 * the unaligned parts which is slow (read-modify-write) and useless
204 * since we are not freeing any space by doing so.
205 */
206 if (!bio_is_secure_erase(bio)) {
207 start = P2ROUNDUP(start, zv->zv_volblocksize);
208 end = P2ALIGN(end, zv->zv_volblocksize);
209 size = end - start;
210 }
211
212 if (start >= end)
213 goto unlock;
214
215 zfs_locked_range_t *lr = zfs_rangelock_enter(&zv->zv_rangelock,
216 start, size, RL_WRITER);
217
218 tx = dmu_tx_create(zv->zv_objset);
219 dmu_tx_mark_netfree(tx);
220 error = dmu_tx_assign(tx, TXG_WAIT);
221 if (error != 0) {
222 dmu_tx_abort(tx);
223 } else {
224 zvol_log_truncate(zv, tx, start, size, B_TRUE);
225 dmu_tx_commit(tx);
226 error = dmu_free_long_range(zv->zv_objset,
227 ZVOL_OBJ, start, size);
228 }
229 zfs_rangelock_exit(lr);
230
231 if (error == 0 && sync)
232 zil_commit(zv->zv_zilog, ZVOL_OBJ);
233
234 unlock:
235 rw_exit(&zv->zv_suspend_lock);
236
237 if (acct)
238 blk_generic_end_io_acct(q, disk, WRITE, bio, start_time);
239
240 BIO_END_IO(bio, -error);
241 kmem_free(zvr, sizeof (zv_request_t));
242 }
243
244 static void
zvol_read(void * arg)245 zvol_read(void *arg)
246 {
247 zv_request_t *zvr = arg;
248 struct bio *bio = zvr->bio;
249 int error = 0;
250 uio_t uio;
251
252 uio_bvec_init(&uio, bio);
253
254 zvol_state_t *zv = zvr->zv;
255 ASSERT3P(zv, !=, NULL);
256 ASSERT3U(zv->zv_open_count, >, 0);
257
258 struct request_queue *q = zv->zv_zso->zvo_queue;
259 struct gendisk *disk = zv->zv_zso->zvo_disk;
260 ssize_t start_resid = uio.uio_resid;
261 unsigned long start_time;
262
263 boolean_t acct = blk_queue_io_stat(q);
264 if (acct)
265 start_time = blk_generic_start_io_acct(q, disk, READ, bio);
266
267 zfs_locked_range_t *lr = zfs_rangelock_enter(&zv->zv_rangelock,
268 uio.uio_loffset, uio.uio_resid, RL_READER);
269
270 uint64_t volsize = zv->zv_volsize;
271 while (uio.uio_resid > 0 && uio.uio_loffset < volsize) {
272 uint64_t bytes = MIN(uio.uio_resid, DMU_MAX_ACCESS >> 1);
273
274 /* don't read past the end */
275 if (bytes > volsize - uio.uio_loffset)
276 bytes = volsize - uio.uio_loffset;
277
278 error = dmu_read_uio_dnode(zv->zv_dn, &uio, bytes);
279 if (error) {
280 /* convert checksum errors into IO errors */
281 if (error == ECKSUM)
282 error = SET_ERROR(EIO);
283 break;
284 }
285 }
286 zfs_rangelock_exit(lr);
287
288 int64_t nread = start_resid - uio.uio_resid;
289 dataset_kstats_update_read_kstats(&zv->zv_kstat, nread);
290 task_io_account_read(nread);
291
292 rw_exit(&zv->zv_suspend_lock);
293
294 if (acct)
295 blk_generic_end_io_acct(q, disk, READ, bio, start_time);
296
297 BIO_END_IO(bio, -error);
298 kmem_free(zvr, sizeof (zv_request_t));
299 }
300
301 #ifdef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
302 static blk_qc_t
zvol_submit_bio(struct bio * bio)303 zvol_submit_bio(struct bio *bio)
304 #else
305 static MAKE_REQUEST_FN_RET
306 zvol_request(struct request_queue *q, struct bio *bio)
307 #endif
308 {
309 #ifdef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
310 struct request_queue *q = bio->bi_disk->queue;
311 #endif
312 zvol_state_t *zv = q->queuedata;
313 fstrans_cookie_t cookie = spl_fstrans_mark();
314 uint64_t offset = BIO_BI_SECTOR(bio) << 9;
315 uint64_t size = BIO_BI_SIZE(bio);
316 int rw = bio_data_dir(bio);
317 zv_request_t *zvr;
318
319 if (bio_has_data(bio) && offset + size > zv->zv_volsize) {
320 printk(KERN_INFO
321 "%s: bad access: offset=%llu, size=%lu\n",
322 zv->zv_zso->zvo_disk->disk_name,
323 (long long unsigned)offset,
324 (long unsigned)size);
325
326 BIO_END_IO(bio, -SET_ERROR(EIO));
327 goto out;
328 }
329
330 if (rw == WRITE) {
331 if (unlikely(zv->zv_flags & ZVOL_RDONLY)) {
332 BIO_END_IO(bio, -SET_ERROR(EROFS));
333 goto out;
334 }
335
336 /*
337 * Prevents the zvol from being suspended, or the ZIL being
338 * concurrently opened. Will be released after the i/o
339 * completes.
340 */
341 rw_enter(&zv->zv_suspend_lock, RW_READER);
342
343 /*
344 * Open a ZIL if this is the first time we have written to this
345 * zvol. We protect zv->zv_zilog with zv_suspend_lock rather
346 * than zv_state_lock so that we don't need to acquire an
347 * additional lock in this path.
348 */
349 if (zv->zv_zilog == NULL) {
350 rw_exit(&zv->zv_suspend_lock);
351 rw_enter(&zv->zv_suspend_lock, RW_WRITER);
352 if (zv->zv_zilog == NULL) {
353 zv->zv_zilog = zil_open(zv->zv_objset,
354 zvol_get_data);
355 zv->zv_flags |= ZVOL_WRITTEN_TO;
356 }
357 rw_downgrade(&zv->zv_suspend_lock);
358 }
359
360 zvr = kmem_alloc(sizeof (zv_request_t), KM_SLEEP);
361 zvr->zv = zv;
362 zvr->bio = bio;
363 taskq_init_ent(&zvr->ent);
364
365 /*
366 * We don't want this thread to be blocked waiting for i/o to
367 * complete, so we instead wait from a taskq callback. The
368 * i/o may be a ZIL write (via zil_commit()), or a read of an
369 * indirect block, or a read of a data block (if this is a
370 * partial-block write). We will indicate that the i/o is
371 * complete by calling BIO_END_IO() from the taskq callback.
372 *
373 * This design allows the calling thread to continue and
374 * initiate more concurrent operations by calling
375 * zvol_request() again. There are typically only a small
376 * number of threads available to call zvol_request() (e.g.
377 * one per iSCSI target), so keeping the latency of
378 * zvol_request() low is important for performance.
379 *
380 * The zvol_request_sync module parameter allows this
381 * behavior to be altered, for performance evaluation
382 * purposes. If the callback blocks, setting
383 * zvol_request_sync=1 will result in much worse performance.
384 *
385 * We can have up to zvol_threads concurrent i/o's being
386 * processed for all zvols on the system. This is typically
387 * a vast improvement over the zvol_request_sync=1 behavior
388 * of one i/o at a time per zvol. However, an even better
389 * design would be for zvol_request() to initiate the zio
390 * directly, and then be notified by the zio_done callback,
391 * which would call BIO_END_IO(). Unfortunately, the DMU/ZIL
392 * interfaces lack this functionality (they block waiting for
393 * the i/o to complete).
394 */
395 if (bio_is_discard(bio) || bio_is_secure_erase(bio)) {
396 if (zvol_request_sync) {
397 zvol_discard(zvr);
398 } else {
399 taskq_dispatch_ent(zvol_taskq,
400 zvol_discard, zvr, 0, &zvr->ent);
401 }
402 } else {
403 if (zvol_request_sync) {
404 zvol_write(zvr);
405 } else {
406 taskq_dispatch_ent(zvol_taskq,
407 zvol_write, zvr, 0, &zvr->ent);
408 }
409 }
410 } else {
411 /*
412 * The SCST driver, and possibly others, may issue READ I/Os
413 * with a length of zero bytes. These empty I/Os contain no
414 * data and require no additional handling.
415 */
416 if (size == 0) {
417 BIO_END_IO(bio, 0);
418 goto out;
419 }
420
421 zvr = kmem_alloc(sizeof (zv_request_t), KM_SLEEP);
422 zvr->zv = zv;
423 zvr->bio = bio;
424 taskq_init_ent(&zvr->ent);
425
426 rw_enter(&zv->zv_suspend_lock, RW_READER);
427
428 /* See comment in WRITE case above. */
429 if (zvol_request_sync) {
430 zvol_read(zvr);
431 } else {
432 taskq_dispatch_ent(zvol_taskq,
433 zvol_read, zvr, 0, &zvr->ent);
434 }
435 }
436
437 out:
438 spl_fstrans_unmark(cookie);
439 #if defined(HAVE_MAKE_REQUEST_FN_RET_QC) || \
440 defined(HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS)
441 return (BLK_QC_T_NONE);
442 #endif
443 }
444
445 static int
zvol_open(struct block_device * bdev,fmode_t flag)446 zvol_open(struct block_device *bdev, fmode_t flag)
447 {
448 zvol_state_t *zv;
449 int error = 0;
450 boolean_t drop_suspend = B_TRUE;
451
452 rw_enter(&zvol_state_lock, RW_READER);
453 /*
454 * Obtain a copy of private_data under the zvol_state_lock to make
455 * sure that either the result of zvol free code path setting
456 * bdev->bd_disk->private_data to NULL is observed, or zvol_free()
457 * is not called on this zv because of the positive zv_open_count.
458 */
459 zv = bdev->bd_disk->private_data;
460 if (zv == NULL) {
461 rw_exit(&zvol_state_lock);
462 return (SET_ERROR(-ENXIO));
463 }
464
465 mutex_enter(&zv->zv_state_lock);
466 /*
467 * make sure zvol is not suspended during first open
468 * (hold zv_suspend_lock) and respect proper lock acquisition
469 * ordering - zv_suspend_lock before zv_state_lock
470 */
471 if (zv->zv_open_count == 0) {
472 if (!rw_tryenter(&zv->zv_suspend_lock, RW_READER)) {
473 mutex_exit(&zv->zv_state_lock);
474 rw_enter(&zv->zv_suspend_lock, RW_READER);
475 mutex_enter(&zv->zv_state_lock);
476 /* check to see if zv_suspend_lock is needed */
477 if (zv->zv_open_count != 0) {
478 rw_exit(&zv->zv_suspend_lock);
479 drop_suspend = B_FALSE;
480 }
481 }
482 } else {
483 drop_suspend = B_FALSE;
484 }
485 rw_exit(&zvol_state_lock);
486
487 ASSERT(MUTEX_HELD(&zv->zv_state_lock));
488
489 if (zv->zv_open_count == 0) {
490 ASSERT(RW_READ_HELD(&zv->zv_suspend_lock));
491 error = -zvol_first_open(zv, !(flag & FMODE_WRITE));
492 if (error)
493 goto out_mutex;
494 }
495
496 if ((flag & FMODE_WRITE) && (zv->zv_flags & ZVOL_RDONLY)) {
497 error = -EROFS;
498 goto out_open_count;
499 }
500
501 zv->zv_open_count++;
502
503 mutex_exit(&zv->zv_state_lock);
504 if (drop_suspend)
505 rw_exit(&zv->zv_suspend_lock);
506
507 zfs_check_media_change(bdev);
508
509 return (0);
510
511 out_open_count:
512 if (zv->zv_open_count == 0)
513 zvol_last_close(zv);
514
515 out_mutex:
516 mutex_exit(&zv->zv_state_lock);
517 if (drop_suspend)
518 rw_exit(&zv->zv_suspend_lock);
519 if (error == -EINTR) {
520 error = -ERESTARTSYS;
521 schedule();
522 }
523 return (SET_ERROR(error));
524 }
525
526 static void
zvol_release(struct gendisk * disk,fmode_t mode)527 zvol_release(struct gendisk *disk, fmode_t mode)
528 {
529 zvol_state_t *zv;
530 boolean_t drop_suspend = B_TRUE;
531
532 rw_enter(&zvol_state_lock, RW_READER);
533 zv = disk->private_data;
534
535 mutex_enter(&zv->zv_state_lock);
536 ASSERT3U(zv->zv_open_count, >, 0);
537 /*
538 * make sure zvol is not suspended during last close
539 * (hold zv_suspend_lock) and respect proper lock acquisition
540 * ordering - zv_suspend_lock before zv_state_lock
541 */
542 if (zv->zv_open_count == 1) {
543 if (!rw_tryenter(&zv->zv_suspend_lock, RW_READER)) {
544 mutex_exit(&zv->zv_state_lock);
545 rw_enter(&zv->zv_suspend_lock, RW_READER);
546 mutex_enter(&zv->zv_state_lock);
547 /* check to see if zv_suspend_lock is needed */
548 if (zv->zv_open_count != 1) {
549 rw_exit(&zv->zv_suspend_lock);
550 drop_suspend = B_FALSE;
551 }
552 }
553 } else {
554 drop_suspend = B_FALSE;
555 }
556 rw_exit(&zvol_state_lock);
557
558 ASSERT(MUTEX_HELD(&zv->zv_state_lock));
559
560 zv->zv_open_count--;
561 if (zv->zv_open_count == 0) {
562 ASSERT(RW_READ_HELD(&zv->zv_suspend_lock));
563 zvol_last_close(zv);
564 }
565
566 mutex_exit(&zv->zv_state_lock);
567
568 if (drop_suspend)
569 rw_exit(&zv->zv_suspend_lock);
570 }
571
572 static int
zvol_ioctl(struct block_device * bdev,fmode_t mode,unsigned int cmd,unsigned long arg)573 zvol_ioctl(struct block_device *bdev, fmode_t mode,
574 unsigned int cmd, unsigned long arg)
575 {
576 zvol_state_t *zv = bdev->bd_disk->private_data;
577 int error = 0;
578
579 ASSERT3U(zv->zv_open_count, >, 0);
580
581 switch (cmd) {
582 case BLKFLSBUF:
583 fsync_bdev(bdev);
584 invalidate_bdev(bdev);
585 rw_enter(&zv->zv_suspend_lock, RW_READER);
586
587 if (!(zv->zv_flags & ZVOL_RDONLY))
588 txg_wait_synced(dmu_objset_pool(zv->zv_objset), 0);
589
590 rw_exit(&zv->zv_suspend_lock);
591 break;
592
593 case BLKZNAME:
594 mutex_enter(&zv->zv_state_lock);
595 error = copy_to_user((void *)arg, zv->zv_name, MAXNAMELEN);
596 mutex_exit(&zv->zv_state_lock);
597 break;
598
599 default:
600 error = -ENOTTY;
601 break;
602 }
603
604 return (SET_ERROR(error));
605 }
606
607 #ifdef CONFIG_COMPAT
608 static int
zvol_compat_ioctl(struct block_device * bdev,fmode_t mode,unsigned cmd,unsigned long arg)609 zvol_compat_ioctl(struct block_device *bdev, fmode_t mode,
610 unsigned cmd, unsigned long arg)
611 {
612 return (zvol_ioctl(bdev, mode, cmd, arg));
613 }
614 #else
615 #define zvol_compat_ioctl NULL
616 #endif
617
618 static unsigned int
zvol_check_events(struct gendisk * disk,unsigned int clearing)619 zvol_check_events(struct gendisk *disk, unsigned int clearing)
620 {
621 unsigned int mask = 0;
622
623 rw_enter(&zvol_state_lock, RW_READER);
624
625 zvol_state_t *zv = disk->private_data;
626 if (zv != NULL) {
627 mutex_enter(&zv->zv_state_lock);
628 mask = zv->zv_changed ? DISK_EVENT_MEDIA_CHANGE : 0;
629 zv->zv_changed = 0;
630 mutex_exit(&zv->zv_state_lock);
631 }
632
633 rw_exit(&zvol_state_lock);
634
635 return (mask);
636 }
637
638 static int
zvol_revalidate_disk(struct gendisk * disk)639 zvol_revalidate_disk(struct gendisk *disk)
640 {
641 rw_enter(&zvol_state_lock, RW_READER);
642
643 zvol_state_t *zv = disk->private_data;
644 if (zv != NULL) {
645 mutex_enter(&zv->zv_state_lock);
646 set_capacity(zv->zv_zso->zvo_disk,
647 zv->zv_volsize >> SECTOR_BITS);
648 mutex_exit(&zv->zv_state_lock);
649 }
650
651 rw_exit(&zvol_state_lock);
652
653 return (0);
654 }
655
656 static int
zvol_update_volsize(zvol_state_t * zv,uint64_t volsize)657 zvol_update_volsize(zvol_state_t *zv, uint64_t volsize)
658 {
659 struct gendisk *disk = zv->zv_zso->zvo_disk;
660
661 #if defined(HAVE_REVALIDATE_DISK_SIZE)
662 revalidate_disk_size(disk, zvol_revalidate_disk(disk) == 0);
663 #elif defined(HAVE_REVALIDATE_DISK)
664 revalidate_disk(disk);
665 #else
666 zvol_revalidate_disk(disk);
667 #endif
668 return (0);
669 }
670
671 static void
zvol_clear_private(zvol_state_t * zv)672 zvol_clear_private(zvol_state_t *zv)
673 {
674 /*
675 * Cleared while holding zvol_state_lock as a writer
676 * which will prevent zvol_open() from opening it.
677 */
678 zv->zv_zso->zvo_disk->private_data = NULL;
679 }
680
681 /*
682 * Provide a simple virtual geometry for legacy compatibility. For devices
683 * smaller than 1 MiB a small head and sector count is used to allow very
684 * tiny devices. For devices over 1 Mib a standard head and sector count
685 * is used to keep the cylinders count reasonable.
686 */
687 static int
zvol_getgeo(struct block_device * bdev,struct hd_geometry * geo)688 zvol_getgeo(struct block_device *bdev, struct hd_geometry *geo)
689 {
690 zvol_state_t *zv = bdev->bd_disk->private_data;
691 sector_t sectors;
692
693 ASSERT3U(zv->zv_open_count, >, 0);
694
695 sectors = get_capacity(zv->zv_zso->zvo_disk);
696
697 if (sectors > 2048) {
698 geo->heads = 16;
699 geo->sectors = 63;
700 } else {
701 geo->heads = 2;
702 geo->sectors = 4;
703 }
704
705 geo->start = 0;
706 geo->cylinders = sectors / (geo->heads * geo->sectors);
707
708 return (0);
709 }
710
711 static struct block_device_operations zvol_ops = {
712 .open = zvol_open,
713 .release = zvol_release,
714 .ioctl = zvol_ioctl,
715 .compat_ioctl = zvol_compat_ioctl,
716 .check_events = zvol_check_events,
717 .revalidate_disk = zvol_revalidate_disk,
718 .getgeo = zvol_getgeo,
719 .owner = THIS_MODULE,
720 #ifdef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
721 .submit_bio = zvol_submit_bio,
722 #endif
723 };
724
725 /*
726 * Allocate memory for a new zvol_state_t and setup the required
727 * request queue and generic disk structures for the block device.
728 */
729 static zvol_state_t *
zvol_alloc(dev_t dev,const char * name)730 zvol_alloc(dev_t dev, const char *name)
731 {
732 zvol_state_t *zv;
733 struct zvol_state_os *zso;
734 uint64_t volmode;
735
736 if (dsl_prop_get_integer(name, "volmode", &volmode, NULL) != 0)
737 return (NULL);
738
739 if (volmode == ZFS_VOLMODE_DEFAULT)
740 volmode = zvol_volmode;
741
742 if (volmode == ZFS_VOLMODE_NONE)
743 return (NULL);
744
745 zv = kmem_zalloc(sizeof (zvol_state_t), KM_SLEEP);
746 zso = kmem_zalloc(sizeof (struct zvol_state_os), KM_SLEEP);
747 zv->zv_zso = zso;
748 zv->zv_volmode = volmode;
749
750 list_link_init(&zv->zv_next);
751 mutex_init(&zv->zv_state_lock, NULL, MUTEX_DEFAULT, NULL);
752
753 #ifdef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
754 zso->zvo_queue = blk_alloc_queue(NUMA_NO_NODE);
755 #else
756 zso->zvo_queue = blk_generic_alloc_queue(zvol_request, NUMA_NO_NODE);
757 #endif
758 if (zso->zvo_queue == NULL)
759 goto out_kmem;
760
761 blk_queue_set_write_cache(zso->zvo_queue, B_TRUE, B_TRUE);
762
763 /* Limit read-ahead to a single page to prevent over-prefetching. */
764 blk_queue_set_read_ahead(zso->zvo_queue, 1);
765
766 /* Disable write merging in favor of the ZIO pipeline. */
767 blk_queue_flag_set(QUEUE_FLAG_NOMERGES, zso->zvo_queue);
768
769 zso->zvo_disk = alloc_disk(ZVOL_MINORS);
770 if (zso->zvo_disk == NULL)
771 goto out_queue;
772
773 zso->zvo_queue->queuedata = zv;
774 zso->zvo_dev = dev;
775 zv->zv_open_count = 0;
776 strlcpy(zv->zv_name, name, MAXNAMELEN);
777
778 zfs_rangelock_init(&zv->zv_rangelock, NULL, NULL);
779 rw_init(&zv->zv_suspend_lock, NULL, RW_DEFAULT, NULL);
780
781 zso->zvo_disk->major = zvol_major;
782 zso->zvo_disk->events = DISK_EVENT_MEDIA_CHANGE;
783
784 if (volmode == ZFS_VOLMODE_DEV) {
785 /*
786 * ZFS_VOLMODE_DEV disable partitioning on ZVOL devices: set
787 * gendisk->minors = 1 as noted in include/linux/genhd.h.
788 * Also disable extended partition numbers (GENHD_FL_EXT_DEVT)
789 * and suppresses partition scanning (GENHD_FL_NO_PART_SCAN)
790 * setting gendisk->flags accordingly.
791 */
792 zso->zvo_disk->minors = 1;
793 #if defined(GENHD_FL_EXT_DEVT)
794 zso->zvo_disk->flags &= ~GENHD_FL_EXT_DEVT;
795 #endif
796 #if defined(GENHD_FL_NO_PART_SCAN)
797 zso->zvo_disk->flags |= GENHD_FL_NO_PART_SCAN;
798 #endif
799 }
800 zso->zvo_disk->first_minor = (dev & MINORMASK);
801 zso->zvo_disk->fops = &zvol_ops;
802 zso->zvo_disk->private_data = zv;
803 zso->zvo_disk->queue = zso->zvo_queue;
804 snprintf(zso->zvo_disk->disk_name, DISK_NAME_LEN, "%s%d",
805 ZVOL_DEV_NAME, (dev & MINORMASK));
806
807 return (zv);
808
809 out_queue:
810 blk_cleanup_queue(zso->zvo_queue);
811 out_kmem:
812 kmem_free(zso, sizeof (struct zvol_state_os));
813 kmem_free(zv, sizeof (zvol_state_t));
814 return (NULL);
815 }
816
817 /*
818 * Cleanup then free a zvol_state_t which was created by zvol_alloc().
819 * At this time, the structure is not opened by anyone, is taken off
820 * the zvol_state_list, and has its private data set to NULL.
821 * The zvol_state_lock is dropped.
822 *
823 * This function may take many milliseconds to complete (e.g. we've seen
824 * it take over 256ms), due to the calls to "blk_cleanup_queue" and
825 * "del_gendisk". Thus, consumers need to be careful to account for this
826 * latency when calling this function.
827 */
828 static void
zvol_free(zvol_state_t * zv)829 zvol_free(zvol_state_t *zv)
830 {
831
832 ASSERT(!RW_LOCK_HELD(&zv->zv_suspend_lock));
833 ASSERT(!MUTEX_HELD(&zv->zv_state_lock));
834 ASSERT0(zv->zv_open_count);
835 ASSERT3P(zv->zv_zso->zvo_disk->private_data, ==, NULL);
836
837 rw_destroy(&zv->zv_suspend_lock);
838 zfs_rangelock_fini(&zv->zv_rangelock);
839
840 del_gendisk(zv->zv_zso->zvo_disk);
841 blk_cleanup_queue(zv->zv_zso->zvo_queue);
842 put_disk(zv->zv_zso->zvo_disk);
843
844 ida_simple_remove(&zvol_ida,
845 MINOR(zv->zv_zso->zvo_dev) >> ZVOL_MINOR_BITS);
846
847 mutex_destroy(&zv->zv_state_lock);
848 dataset_kstats_destroy(&zv->zv_kstat);
849
850 kmem_free(zv->zv_zso, sizeof (struct zvol_state_os));
851 kmem_free(zv, sizeof (zvol_state_t));
852 }
853
854 void
zvol_wait_close(zvol_state_t * zv)855 zvol_wait_close(zvol_state_t *zv)
856 {
857 }
858
859 /*
860 * Create a block device minor node and setup the linkage between it
861 * and the specified volume. Once this function returns the block
862 * device is live and ready for use.
863 */
864 static int
zvol_os_create_minor(const char * name)865 zvol_os_create_minor(const char *name)
866 {
867 zvol_state_t *zv;
868 objset_t *os;
869 dmu_object_info_t *doi;
870 uint64_t volsize;
871 uint64_t len;
872 unsigned minor = 0;
873 int error = 0;
874 int idx;
875 uint64_t hash = zvol_name_hash(name);
876
877 if (zvol_inhibit_dev)
878 return (0);
879
880 idx = ida_simple_get(&zvol_ida, 0, 0, kmem_flags_convert(KM_SLEEP));
881 if (idx < 0)
882 return (SET_ERROR(-idx));
883 minor = idx << ZVOL_MINOR_BITS;
884
885 zv = zvol_find_by_name_hash(name, hash, RW_NONE);
886 if (zv) {
887 ASSERT(MUTEX_HELD(&zv->zv_state_lock));
888 mutex_exit(&zv->zv_state_lock);
889 ida_simple_remove(&zvol_ida, idx);
890 return (SET_ERROR(EEXIST));
891 }
892
893 doi = kmem_alloc(sizeof (dmu_object_info_t), KM_SLEEP);
894
895 error = dmu_objset_own(name, DMU_OST_ZVOL, B_TRUE, B_TRUE, FTAG, &os);
896 if (error)
897 goto out_doi;
898
899 error = dmu_object_info(os, ZVOL_OBJ, doi);
900 if (error)
901 goto out_dmu_objset_disown;
902
903 error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize);
904 if (error)
905 goto out_dmu_objset_disown;
906
907 zv = zvol_alloc(MKDEV(zvol_major, minor), name);
908 if (zv == NULL) {
909 error = SET_ERROR(EAGAIN);
910 goto out_dmu_objset_disown;
911 }
912 zv->zv_hash = hash;
913
914 if (dmu_objset_is_snapshot(os))
915 zv->zv_flags |= ZVOL_RDONLY;
916
917 zv->zv_volblocksize = doi->doi_data_block_size;
918 zv->zv_volsize = volsize;
919 zv->zv_objset = os;
920
921 set_capacity(zv->zv_zso->zvo_disk, zv->zv_volsize >> 9);
922
923 blk_queue_max_hw_sectors(zv->zv_zso->zvo_queue,
924 (DMU_MAX_ACCESS / 4) >> 9);
925 blk_queue_max_segments(zv->zv_zso->zvo_queue, UINT16_MAX);
926 blk_queue_max_segment_size(zv->zv_zso->zvo_queue, UINT_MAX);
927 blk_queue_physical_block_size(zv->zv_zso->zvo_queue,
928 zv->zv_volblocksize);
929 blk_queue_io_opt(zv->zv_zso->zvo_queue, zv->zv_volblocksize);
930 blk_queue_max_discard_sectors(zv->zv_zso->zvo_queue,
931 (zvol_max_discard_blocks * zv->zv_volblocksize) >> 9);
932 blk_queue_discard_granularity(zv->zv_zso->zvo_queue,
933 zv->zv_volblocksize);
934 blk_queue_flag_set(QUEUE_FLAG_DISCARD, zv->zv_zso->zvo_queue);
935 #ifdef QUEUE_FLAG_NONROT
936 blk_queue_flag_set(QUEUE_FLAG_NONROT, zv->zv_zso->zvo_queue);
937 #endif
938 #ifdef QUEUE_FLAG_ADD_RANDOM
939 blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, zv->zv_zso->zvo_queue);
940 #endif
941 /* This flag was introduced in kernel version 4.12. */
942 #ifdef QUEUE_FLAG_SCSI_PASSTHROUGH
943 blk_queue_flag_set(QUEUE_FLAG_SCSI_PASSTHROUGH, zv->zv_zso->zvo_queue);
944 #endif
945
946 if (spa_writeable(dmu_objset_spa(os))) {
947 if (zil_replay_disable)
948 zil_destroy(dmu_objset_zil(os), B_FALSE);
949 else
950 zil_replay(os, zv, zvol_replay_vector);
951 }
952 ASSERT3P(zv->zv_kstat.dk_kstats, ==, NULL);
953 dataset_kstats_create(&zv->zv_kstat, zv->zv_objset);
954
955 /*
956 * When udev detects the addition of the device it will immediately
957 * invoke blkid(8) to determine the type of content on the device.
958 * Prefetching the blocks commonly scanned by blkid(8) will speed
959 * up this process.
960 */
961 len = MIN(MAX(zvol_prefetch_bytes, 0), SPA_MAXBLOCKSIZE);
962 if (len > 0) {
963 dmu_prefetch(os, ZVOL_OBJ, 0, 0, len, ZIO_PRIORITY_SYNC_READ);
964 dmu_prefetch(os, ZVOL_OBJ, 0, volsize - len, len,
965 ZIO_PRIORITY_SYNC_READ);
966 }
967
968 zv->zv_objset = NULL;
969 out_dmu_objset_disown:
970 dmu_objset_disown(os, B_TRUE, FTAG);
971 out_doi:
972 kmem_free(doi, sizeof (dmu_object_info_t));
973
974 /*
975 * Keep in mind that once add_disk() is called, the zvol is
976 * announced to the world, and zvol_open()/zvol_release() can
977 * be called at any time. Incidentally, add_disk() itself calls
978 * zvol_open()->zvol_first_open() and zvol_release()->zvol_last_close()
979 * directly as well.
980 */
981 if (error == 0) {
982 rw_enter(&zvol_state_lock, RW_WRITER);
983 zvol_insert(zv);
984 rw_exit(&zvol_state_lock);
985 add_disk(zv->zv_zso->zvo_disk);
986 } else {
987 ida_simple_remove(&zvol_ida, idx);
988 }
989
990 return (error);
991 }
992
993 static void
zvol_rename_minor(zvol_state_t * zv,const char * newname)994 zvol_rename_minor(zvol_state_t *zv, const char *newname)
995 {
996 int readonly = get_disk_ro(zv->zv_zso->zvo_disk);
997
998 ASSERT(RW_LOCK_HELD(&zvol_state_lock));
999 ASSERT(MUTEX_HELD(&zv->zv_state_lock));
1000
1001 strlcpy(zv->zv_name, newname, sizeof (zv->zv_name));
1002
1003 /* move to new hashtable entry */
1004 zv->zv_hash = zvol_name_hash(zv->zv_name);
1005 hlist_del(&zv->zv_hlink);
1006 hlist_add_head(&zv->zv_hlink, ZVOL_HT_HEAD(zv->zv_hash));
1007
1008 /*
1009 * The block device's read-only state is briefly changed causing
1010 * a KOBJ_CHANGE uevent to be issued. This ensures udev detects
1011 * the name change and fixes the symlinks. This does not change
1012 * ZVOL_RDONLY in zv->zv_flags so the actual read-only state never
1013 * changes. This would normally be done using kobject_uevent() but
1014 * that is a GPL-only symbol which is why we need this workaround.
1015 */
1016 set_disk_ro(zv->zv_zso->zvo_disk, !readonly);
1017 set_disk_ro(zv->zv_zso->zvo_disk, readonly);
1018 }
1019
1020 static void
zvol_set_disk_ro_impl(zvol_state_t * zv,int flags)1021 zvol_set_disk_ro_impl(zvol_state_t *zv, int flags)
1022 {
1023
1024 set_disk_ro(zv->zv_zso->zvo_disk, flags);
1025 }
1026
1027 static void
zvol_set_capacity_impl(zvol_state_t * zv,uint64_t capacity)1028 zvol_set_capacity_impl(zvol_state_t *zv, uint64_t capacity)
1029 {
1030
1031 set_capacity(zv->zv_zso->zvo_disk, capacity);
1032 }
1033
1034 const static zvol_platform_ops_t zvol_linux_ops = {
1035 .zv_free = zvol_free,
1036 .zv_rename_minor = zvol_rename_minor,
1037 .zv_create_minor = zvol_os_create_minor,
1038 .zv_update_volsize = zvol_update_volsize,
1039 .zv_clear_private = zvol_clear_private,
1040 .zv_is_zvol = zvol_is_zvol_impl,
1041 .zv_set_disk_ro = zvol_set_disk_ro_impl,
1042 .zv_set_capacity = zvol_set_capacity_impl,
1043 };
1044
1045 int
zvol_init(void)1046 zvol_init(void)
1047 {
1048 int error;
1049 int threads = MIN(MAX(zvol_threads, 1), 1024);
1050
1051 error = register_blkdev(zvol_major, ZVOL_DRIVER);
1052 if (error) {
1053 printk(KERN_INFO "ZFS: register_blkdev() failed %d\n", error);
1054 return (error);
1055 }
1056 zvol_taskq = taskq_create(ZVOL_DRIVER, threads, maxclsyspri,
1057 threads * 2, INT_MAX, TASKQ_PREPOPULATE | TASKQ_DYNAMIC);
1058 if (zvol_taskq == NULL) {
1059 unregister_blkdev(zvol_major, ZVOL_DRIVER);
1060 return (-ENOMEM);
1061 }
1062 zvol_init_impl();
1063 ida_init(&zvol_ida);
1064 zvol_register_ops(&zvol_linux_ops);
1065 return (0);
1066 }
1067
1068 void
zvol_fini(void)1069 zvol_fini(void)
1070 {
1071 zvol_fini_impl();
1072 unregister_blkdev(zvol_major, ZVOL_DRIVER);
1073 taskq_destroy(zvol_taskq);
1074 ida_destroy(&zvol_ida);
1075 }
1076
1077 /* BEGIN CSTYLED */
1078 module_param(zvol_inhibit_dev, uint, 0644);
1079 MODULE_PARM_DESC(zvol_inhibit_dev, "Do not create zvol device nodes");
1080
1081 module_param(zvol_major, uint, 0444);
1082 MODULE_PARM_DESC(zvol_major, "Major number for zvol device");
1083
1084 module_param(zvol_threads, uint, 0444);
1085 MODULE_PARM_DESC(zvol_threads, "Max number of threads to handle I/O requests");
1086
1087 module_param(zvol_request_sync, uint, 0644);
1088 MODULE_PARM_DESC(zvol_request_sync, "Synchronously handle bio requests");
1089
1090 module_param(zvol_max_discard_blocks, ulong, 0444);
1091 MODULE_PARM_DESC(zvol_max_discard_blocks, "Max number of blocks to discard");
1092
1093 module_param(zvol_prefetch_bytes, uint, 0644);
1094 MODULE_PARM_DESC(zvol_prefetch_bytes, "Prefetch N bytes at zvol start+end");
1095
1096 module_param(zvol_volmode, uint, 0644);
1097 MODULE_PARM_DESC(zvol_volmode, "Default volmode property value");
1098 /* END CSTYLED */
1099