1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (C) 2011 Lawrence Livermore National Security, LLC.
24 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
25 * Written by Brian Behlendorf <[email protected]>.
26 * LLNL-CODE-403049.
27 */
28
29 #ifndef _ZFS_BLKDEV_H
30 #define _ZFS_BLKDEV_H
31
32 #include <linux/blkdev.h>
33 #include <linux/backing-dev.h>
34 #include <linux/hdreg.h>
35 #include <linux/major.h>
36 #include <linux/msdos_fs.h> /* for SECTOR_* */
37 #include <linux/bio.h>
38
39 #ifdef HAVE_BLK_MQ
40 #include <linux/blk-mq.h>
41 #endif
42
43 #ifndef HAVE_BLK_QUEUE_FLAG_SET
44 static inline void
blk_queue_flag_set(unsigned int flag,struct request_queue * q)45 blk_queue_flag_set(unsigned int flag, struct request_queue *q)
46 {
47 queue_flag_set(flag, q);
48 }
49 #endif
50
51 #ifndef HAVE_BLK_QUEUE_FLAG_CLEAR
52 static inline void
blk_queue_flag_clear(unsigned int flag,struct request_queue * q)53 blk_queue_flag_clear(unsigned int flag, struct request_queue *q)
54 {
55 queue_flag_clear(flag, q);
56 }
57 #endif
58
59 /*
60 * 6.11 API
61 * Setting the flush flags directly is no longer possible; flush flags are set
62 * on the queue_limits structure and passed to blk_disk_alloc(). In this case
63 * we remove this function entirely.
64 *
65 * 4.7 API,
66 * The blk_queue_write_cache() interface has replaced blk_queue_flush()
67 * interface. However, the new interface is GPL-only thus we implement
68 * our own trivial wrapper when the GPL-only version is detected.
69 *
70 * 2.6.36 - 4.6 API,
71 * The blk_queue_flush() interface has replaced blk_queue_ordered()
72 * interface. However, while the old interface was available to all the
73 * new one is GPL-only. Thus if the GPL-only version is detected we
74 * implement our own trivial helper.
75 */
76 #if !defined(HAVE_BLK_ALLOC_DISK_2ARG) || \
77 !defined(HAVE_BLKDEV_QUEUE_LIMITS_FEATURES)
78 static inline void
blk_queue_set_write_cache(struct request_queue * q,bool on)79 blk_queue_set_write_cache(struct request_queue *q, bool on)
80 {
81 #if defined(HAVE_BLK_QUEUE_WRITE_CACHE_GPL_ONLY)
82 if (on) {
83 blk_queue_flag_set(QUEUE_FLAG_WC, q);
84 blk_queue_flag_set(QUEUE_FLAG_FUA, q);
85 } else {
86 blk_queue_flag_clear(QUEUE_FLAG_WC, q);
87 blk_queue_flag_clear(QUEUE_FLAG_FUA, q);
88 }
89 #elif defined(HAVE_BLK_QUEUE_WRITE_CACHE)
90 blk_queue_write_cache(q, on, on);
91 #elif defined(HAVE_BLK_QUEUE_FLUSH_GPL_ONLY)
92 if (on)
93 q->flush_flags |= REQ_FLUSH | REQ_FUA;
94 else
95 q->flush_flags &= ~(REQ_FLUSH | REQ_FUA);
96 #elif defined(HAVE_BLK_QUEUE_FLUSH)
97 blk_queue_flush(q, on ? (REQ_FLUSH | REQ_FUA) : 0);
98 #else
99 #error "Unsupported kernel"
100 #endif
101 }
102 #endif /* !HAVE_BLK_ALLOC_DISK_2ARG || !HAVE_BLKDEV_QUEUE_LIMITS_FEATURES */
103
104 static inline void
blk_queue_set_read_ahead(struct request_queue * q,unsigned long ra_pages)105 blk_queue_set_read_ahead(struct request_queue *q, unsigned long ra_pages)
106 {
107 #if !defined(HAVE_BLK_QUEUE_UPDATE_READAHEAD) && \
108 !defined(HAVE_DISK_UPDATE_READAHEAD)
109 #if defined(HAVE_BLK_QUEUE_BDI_DYNAMIC)
110 q->backing_dev_info->ra_pages = ra_pages;
111 #elif defined(HAVE_BLK_QUEUE_DISK_BDI)
112 q->disk->bdi->ra_pages = ra_pages;
113 #else
114 q->backing_dev_info.ra_pages = ra_pages;
115 #endif
116 #endif
117 }
118
119 #ifdef HAVE_BIO_BVEC_ITER
120 #define BIO_BI_SECTOR(bio) (bio)->bi_iter.bi_sector
121 #define BIO_BI_SIZE(bio) (bio)->bi_iter.bi_size
122 #define BIO_BI_IDX(bio) (bio)->bi_iter.bi_idx
123 #define BIO_BI_SKIP(bio) (bio)->bi_iter.bi_bvec_done
124 #define bio_for_each_segment4(bv, bvp, b, i) \
125 bio_for_each_segment((bv), (b), (i))
126 typedef struct bvec_iter bvec_iterator_t;
127 #else
128 #define BIO_BI_SECTOR(bio) (bio)->bi_sector
129 #define BIO_BI_SIZE(bio) (bio)->bi_size
130 #define BIO_BI_IDX(bio) (bio)->bi_idx
131 #define BIO_BI_SKIP(bio) (0)
132 #define bio_for_each_segment4(bv, bvp, b, i) \
133 bio_for_each_segment((bvp), (b), (i))
134 typedef int bvec_iterator_t;
135 #endif
136
137 static inline void
bio_set_flags_failfast(struct block_device * bdev,int * flags,bool dev,bool transport,bool driver)138 bio_set_flags_failfast(struct block_device *bdev, int *flags, bool dev,
139 bool transport, bool driver)
140 {
141 #ifdef CONFIG_BUG
142 /*
143 * Disable FAILFAST for loopback devices because of the
144 * following incorrect BUG_ON() in loop_make_request().
145 * This support is also disabled for md devices because the
146 * test suite layers md devices on top of loopback devices.
147 * This may be removed when the loopback driver is fixed.
148 *
149 * BUG_ON(!lo || (rw != READ && rw != WRITE));
150 */
151 if ((MAJOR(bdev->bd_dev) == LOOP_MAJOR) ||
152 (MAJOR(bdev->bd_dev) == MD_MAJOR))
153 return;
154
155 #ifdef BLOCK_EXT_MAJOR
156 if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR)
157 return;
158 #endif /* BLOCK_EXT_MAJOR */
159 #endif /* CONFIG_BUG */
160
161 if (dev)
162 *flags |= REQ_FAILFAST_DEV;
163 if (transport)
164 *flags |= REQ_FAILFAST_TRANSPORT;
165 if (driver)
166 *flags |= REQ_FAILFAST_DRIVER;
167 }
168
169 /*
170 * Maximum disk label length, it may be undefined for some kernels.
171 */
172 #if !defined(DISK_NAME_LEN)
173 #define DISK_NAME_LEN 32
174 #endif /* DISK_NAME_LEN */
175
176 #ifdef HAVE_BIO_BI_STATUS
177 static inline int
bi_status_to_errno(blk_status_t status)178 bi_status_to_errno(blk_status_t status)
179 {
180 switch (status) {
181 case BLK_STS_OK:
182 return (0);
183 case BLK_STS_NOTSUPP:
184 return (EOPNOTSUPP);
185 case BLK_STS_TIMEOUT:
186 return (ETIMEDOUT);
187 case BLK_STS_NOSPC:
188 return (ENOSPC);
189 case BLK_STS_TRANSPORT:
190 return (ENOLINK);
191 case BLK_STS_TARGET:
192 return (EREMOTEIO);
193 #ifdef HAVE_BLK_STS_RESV_CONFLICT
194 case BLK_STS_RESV_CONFLICT:
195 #else
196 case BLK_STS_NEXUS:
197 #endif
198 return (EBADE);
199 case BLK_STS_MEDIUM:
200 return (ENODATA);
201 case BLK_STS_PROTECTION:
202 return (EILSEQ);
203 case BLK_STS_RESOURCE:
204 return (ENOMEM);
205 case BLK_STS_AGAIN:
206 return (EAGAIN);
207 case BLK_STS_IOERR:
208 return (EIO);
209 default:
210 return (EIO);
211 }
212 }
213
214 static inline blk_status_t
errno_to_bi_status(int error)215 errno_to_bi_status(int error)
216 {
217 switch (error) {
218 case 0:
219 return (BLK_STS_OK);
220 case EOPNOTSUPP:
221 return (BLK_STS_NOTSUPP);
222 case ETIMEDOUT:
223 return (BLK_STS_TIMEOUT);
224 case ENOSPC:
225 return (BLK_STS_NOSPC);
226 case ENOLINK:
227 return (BLK_STS_TRANSPORT);
228 case EREMOTEIO:
229 return (BLK_STS_TARGET);
230 case EBADE:
231 #ifdef HAVE_BLK_STS_RESV_CONFLICT
232 return (BLK_STS_RESV_CONFLICT);
233 #else
234 return (BLK_STS_NEXUS);
235 #endif
236 case ENODATA:
237 return (BLK_STS_MEDIUM);
238 case EILSEQ:
239 return (BLK_STS_PROTECTION);
240 case ENOMEM:
241 return (BLK_STS_RESOURCE);
242 case EAGAIN:
243 return (BLK_STS_AGAIN);
244 case EIO:
245 return (BLK_STS_IOERR);
246 default:
247 return (BLK_STS_IOERR);
248 }
249 }
250 #endif /* HAVE_BIO_BI_STATUS */
251
252 /*
253 * 4.3 API change
254 * The bio_endio() prototype changed slightly. These are helper
255 * macro's to ensure the prototype and invocation are handled.
256 */
257 #ifdef HAVE_1ARG_BIO_END_IO_T
258 #ifdef HAVE_BIO_BI_STATUS
259 #define BIO_END_IO_ERROR(bio) bi_status_to_errno(bio->bi_status)
260 #define BIO_END_IO_PROTO(fn, x, z) static void fn(struct bio *x)
261 #define BIO_END_IO(bio, error) bio_set_bi_status(bio, error)
262 static inline void
bio_set_bi_status(struct bio * bio,int error)263 bio_set_bi_status(struct bio *bio, int error)
264 {
265 ASSERT3S(error, <=, 0);
266 bio->bi_status = errno_to_bi_status(-error);
267 bio_endio(bio);
268 }
269 #else
270 #define BIO_END_IO_ERROR(bio) (-(bio->bi_error))
271 #define BIO_END_IO_PROTO(fn, x, z) static void fn(struct bio *x)
272 #define BIO_END_IO(bio, error) bio_set_bi_error(bio, error)
273 static inline void
bio_set_bi_error(struct bio * bio,int error)274 bio_set_bi_error(struct bio *bio, int error)
275 {
276 ASSERT3S(error, <=, 0);
277 bio->bi_error = error;
278 bio_endio(bio);
279 }
280 #endif /* HAVE_BIO_BI_STATUS */
281
282 #else
283 #define BIO_END_IO_PROTO(fn, x, z) static void fn(struct bio *x, int z)
284 #define BIO_END_IO(bio, error) bio_endio(bio, error);
285 #endif /* HAVE_1ARG_BIO_END_IO_T */
286
287 /*
288 * 5.15 MACRO,
289 * GD_DEAD
290 *
291 * 2.6.36 - 5.14 MACRO,
292 * GENHD_FL_UP
293 *
294 * Check the disk status and return B_TRUE if alive
295 * otherwise B_FALSE
296 */
297 static inline boolean_t
zfs_check_disk_status(struct block_device * bdev)298 zfs_check_disk_status(struct block_device *bdev)
299 {
300 #if defined(GENHD_FL_UP)
301 return (!!(bdev->bd_disk->flags & GENHD_FL_UP));
302 #elif defined(GD_DEAD)
303 return (!test_bit(GD_DEAD, &bdev->bd_disk->state));
304 #else
305 /*
306 * This is encountered if neither GENHD_FL_UP nor GD_DEAD is available in
307 * the kernel - likely due to an MACRO change that needs to be chased down.
308 */
309 #error "Unsupported kernel: no usable disk status check"
310 #endif
311 }
312
313 /*
314 * 4.1 API,
315 * 3.10.0 CentOS 7.x API,
316 * blkdev_reread_part()
317 *
318 * For older kernels trigger a re-reading of the partition table by calling
319 * check_disk_change() which calls flush_disk() to invalidate the device.
320 *
321 * For newer kernels (as of 5.10), bdev_check_media_change is used, in favor of
322 * check_disk_change(), with the modification that invalidation is no longer
323 * forced.
324 */
325 #ifdef HAVE_CHECK_DISK_CHANGE
326 #define zfs_check_media_change(bdev) check_disk_change(bdev)
327 #ifdef HAVE_BLKDEV_REREAD_PART
328 #define vdev_bdev_reread_part(bdev) blkdev_reread_part(bdev)
329 #else
330 #define vdev_bdev_reread_part(bdev) check_disk_change(bdev)
331 #endif /* HAVE_BLKDEV_REREAD_PART */
332 #else
333 #ifdef HAVE_BDEV_CHECK_MEDIA_CHANGE
334 static inline int
zfs_check_media_change(struct block_device * bdev)335 zfs_check_media_change(struct block_device *bdev)
336 {
337 #ifdef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
338 struct gendisk *gd = bdev->bd_disk;
339 const struct block_device_operations *bdo = gd->fops;
340 #endif
341
342 if (!bdev_check_media_change(bdev))
343 return (0);
344
345 #ifdef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
346 /*
347 * Force revalidation, to mimic the old behavior of
348 * check_disk_change()
349 */
350 if (bdo->revalidate_disk)
351 bdo->revalidate_disk(gd);
352 #endif
353
354 return (0);
355 }
356 #define vdev_bdev_reread_part(bdev) zfs_check_media_change(bdev)
357 #elif defined(HAVE_DISK_CHECK_MEDIA_CHANGE)
358 #define vdev_bdev_reread_part(bdev) disk_check_media_change(bdev->bd_disk)
359 #define zfs_check_media_change(bdev) disk_check_media_change(bdev->bd_disk)
360 #else
361 /*
362 * This is encountered if check_disk_change() and bdev_check_media_change()
363 * are not available in the kernel - likely due to an API change that needs
364 * to be chased down.
365 */
366 #error "Unsupported kernel: no usable disk change check"
367 #endif /* HAVE_BDEV_CHECK_MEDIA_CHANGE */
368 #endif /* HAVE_CHECK_DISK_CHANGE */
369
370 /*
371 * 2.6.27 API change
372 * The function was exported for use, prior to this it existed but the
373 * symbol was not exported.
374 *
375 * 4.4.0-6.21 API change for Ubuntu
376 * lookup_bdev() gained a second argument, FMODE_*, to check inode permissions.
377 *
378 * 5.11 API change
379 * Changed to take a dev_t argument which is set on success and return a
380 * non-zero error code on failure.
381 */
382 static inline int
vdev_lookup_bdev(const char * path,dev_t * dev)383 vdev_lookup_bdev(const char *path, dev_t *dev)
384 {
385 #if defined(HAVE_DEVT_LOOKUP_BDEV)
386 return (lookup_bdev(path, dev));
387 #elif defined(HAVE_1ARG_LOOKUP_BDEV)
388 struct block_device *bdev = lookup_bdev(path);
389 if (IS_ERR(bdev))
390 return (PTR_ERR(bdev));
391
392 *dev = bdev->bd_dev;
393 bdput(bdev);
394
395 return (0);
396 #elif defined(HAVE_MODE_LOOKUP_BDEV)
397 struct block_device *bdev = lookup_bdev(path, FMODE_READ);
398 if (IS_ERR(bdev))
399 return (PTR_ERR(bdev));
400
401 *dev = bdev->bd_dev;
402 bdput(bdev);
403
404 return (0);
405 #else
406 #error "Unsupported kernel"
407 #endif
408 }
409
410 #if defined(HAVE_BLK_MODE_T)
411 #define blk_mode_is_open_write(flag) ((flag) & BLK_OPEN_WRITE)
412 #else
413 #define blk_mode_is_open_write(flag) ((flag) & FMODE_WRITE)
414 #endif
415
416 /*
417 * Kernels without bio_set_op_attrs use bi_rw for the bio flags.
418 */
419 #if !defined(HAVE_BIO_SET_OP_ATTRS)
420 static inline void
bio_set_op_attrs(struct bio * bio,unsigned rw,unsigned flags)421 bio_set_op_attrs(struct bio *bio, unsigned rw, unsigned flags)
422 {
423 #if defined(HAVE_BIO_BI_OPF)
424 bio->bi_opf = rw | flags;
425 #else
426 bio->bi_rw |= rw | flags;
427 #endif /* HAVE_BIO_BI_OPF */
428 }
429 #endif
430
431 /*
432 * bio_set_flush - Set the appropriate flags in a bio to guarantee
433 * data are on non-volatile media on completion.
434 *
435 * 2.6.37 - 4.8 API,
436 * Introduce WRITE_FLUSH, WRITE_FUA, and WRITE_FLUSH_FUA flags as a
437 * replacement for WRITE_BARRIER to allow expressing richer semantics
438 * to the block layer. It's up to the block layer to implement the
439 * semantics correctly. Use the WRITE_FLUSH_FUA flag combination.
440 *
441 * 4.8 - 4.9 API,
442 * REQ_FLUSH was renamed to REQ_PREFLUSH. For consistency with previous
443 * OpenZFS releases, prefer the WRITE_FLUSH_FUA flag set if it's available.
444 *
445 * 4.10 API,
446 * The read/write flags and their modifiers, including WRITE_FLUSH,
447 * WRITE_FUA and WRITE_FLUSH_FUA were removed from fs.h in
448 * torvalds/linux@70fd7614 and replaced by direct flag modification
449 * of the REQ_ flags in bio->bi_opf. Use REQ_PREFLUSH.
450 */
451 static inline void
bio_set_flush(struct bio * bio)452 bio_set_flush(struct bio *bio)
453 {
454 #if defined(HAVE_REQ_PREFLUSH) /* >= 4.10 */
455 bio_set_op_attrs(bio, 0, REQ_PREFLUSH | REQ_OP_WRITE);
456 #elif defined(WRITE_FLUSH_FUA) /* >= 2.6.37 and <= 4.9 */
457 bio_set_op_attrs(bio, 0, WRITE_FLUSH_FUA);
458 #else
459 #error "Allowing the build will cause bio_set_flush requests to be ignored."
460 #endif
461 }
462
463 /*
464 * 4.8 API,
465 * REQ_OP_FLUSH
466 *
467 * 4.8-rc0 - 4.8-rc1,
468 * REQ_PREFLUSH
469 *
470 * 2.6.36 - 4.7 API,
471 * REQ_FLUSH
472 *
473 * in all cases but may have a performance impact for some kernels. It
474 * has the advantage of minimizing kernel specific changes in the zvol code.
475 *
476 */
477 static inline boolean_t
bio_is_flush(struct bio * bio)478 bio_is_flush(struct bio *bio)
479 {
480 #if defined(HAVE_REQ_OP_FLUSH) && defined(HAVE_BIO_BI_OPF)
481 return ((bio_op(bio) == REQ_OP_FLUSH) || (bio->bi_opf & REQ_PREFLUSH));
482 #elif defined(HAVE_REQ_PREFLUSH) && defined(HAVE_BIO_BI_OPF)
483 return (bio->bi_opf & REQ_PREFLUSH);
484 #elif defined(HAVE_REQ_PREFLUSH) && !defined(HAVE_BIO_BI_OPF)
485 return (bio->bi_rw & REQ_PREFLUSH);
486 #elif defined(HAVE_REQ_FLUSH)
487 return (bio->bi_rw & REQ_FLUSH);
488 #else
489 #error "Unsupported kernel"
490 #endif
491 }
492
493 /*
494 * 4.8 API,
495 * REQ_FUA flag moved to bio->bi_opf
496 *
497 * 2.6.x - 4.7 API,
498 * REQ_FUA
499 */
500 static inline boolean_t
bio_is_fua(struct bio * bio)501 bio_is_fua(struct bio *bio)
502 {
503 #if defined(HAVE_BIO_BI_OPF)
504 return (bio->bi_opf & REQ_FUA);
505 #elif defined(REQ_FUA)
506 return (bio->bi_rw & REQ_FUA);
507 #else
508 #error "Allowing the build will cause fua requests to be ignored."
509 #endif
510 }
511
512 /*
513 * 4.8 API,
514 * REQ_OP_DISCARD
515 *
516 * 2.6.36 - 4.7 API,
517 * REQ_DISCARD
518 *
519 * In all cases the normal I/O path is used for discards. The only
520 * difference is how the kernel tags individual I/Os as discards.
521 */
522 static inline boolean_t
bio_is_discard(struct bio * bio)523 bio_is_discard(struct bio *bio)
524 {
525 #if defined(HAVE_REQ_OP_DISCARD)
526 return (bio_op(bio) == REQ_OP_DISCARD);
527 #elif defined(HAVE_REQ_DISCARD)
528 return (bio->bi_rw & REQ_DISCARD);
529 #else
530 #error "Unsupported kernel"
531 #endif
532 }
533
534 /*
535 * 4.8 API,
536 * REQ_OP_SECURE_ERASE
537 *
538 * 2.6.36 - 4.7 API,
539 * REQ_SECURE
540 */
541 static inline boolean_t
bio_is_secure_erase(struct bio * bio)542 bio_is_secure_erase(struct bio *bio)
543 {
544 #if defined(HAVE_REQ_OP_SECURE_ERASE)
545 return (bio_op(bio) == REQ_OP_SECURE_ERASE);
546 #elif defined(REQ_SECURE)
547 return (bio->bi_rw & REQ_SECURE);
548 #else
549 return (0);
550 #endif
551 }
552
553 /*
554 * 2.6.33 API change
555 * Discard granularity and alignment restrictions may now be set. For
556 * older kernels which do not support this it is safe to skip it.
557 */
558 static inline void
blk_queue_discard_granularity(struct request_queue * q,unsigned int dg)559 blk_queue_discard_granularity(struct request_queue *q, unsigned int dg)
560 {
561 q->limits.discard_granularity = dg;
562 }
563
564 /*
565 * 5.19 API,
566 * bdev_max_discard_sectors()
567 *
568 * 2.6.32 API,
569 * blk_queue_discard()
570 */
571 static inline boolean_t
bdev_discard_supported(struct block_device * bdev)572 bdev_discard_supported(struct block_device *bdev)
573 {
574 #if defined(HAVE_BDEV_MAX_DISCARD_SECTORS)
575 return (bdev_max_discard_sectors(bdev) > 0 &&
576 bdev_discard_granularity(bdev) > 0);
577 #elif defined(HAVE_BLK_QUEUE_DISCARD)
578 return (blk_queue_discard(bdev_get_queue(bdev)) > 0 &&
579 bdev_get_queue(bdev)->limits.discard_granularity > 0);
580 #else
581 #error "Unsupported kernel"
582 #endif
583 }
584
585 /*
586 * 5.19 API,
587 * bdev_max_secure_erase_sectors()
588 *
589 * 4.8 API,
590 * blk_queue_secure_erase()
591 *
592 * 2.6.36 - 4.7 API,
593 * blk_queue_secdiscard()
594 */
595 static inline boolean_t
bdev_secure_discard_supported(struct block_device * bdev)596 bdev_secure_discard_supported(struct block_device *bdev)
597 {
598 #if defined(HAVE_BDEV_MAX_SECURE_ERASE_SECTORS)
599 return (!!bdev_max_secure_erase_sectors(bdev));
600 #elif defined(HAVE_BLK_QUEUE_SECURE_ERASE)
601 return (!!blk_queue_secure_erase(bdev_get_queue(bdev)));
602 #elif defined(HAVE_BLK_QUEUE_SECDISCARD)
603 return (!!blk_queue_secdiscard(bdev_get_queue(bdev)));
604 #else
605 #error "Unsupported kernel"
606 #endif
607 }
608
609 /*
610 * A common holder for vdev_bdev_open() is used to relax the exclusive open
611 * semantics slightly. Internal vdev disk callers may pass VDEV_HOLDER to
612 * allow them to open the device multiple times. Other kernel callers and
613 * user space processes which don't pass this value will get EBUSY. This is
614 * currently required for the correct operation of hot spares.
615 */
616 #define VDEV_HOLDER ((void *)0x2401de7)
617
618 static inline unsigned long
blk_generic_start_io_acct(struct request_queue * q,struct gendisk * disk,int rw,struct bio * bio)619 blk_generic_start_io_acct(struct request_queue *q __attribute__((unused)),
620 struct gendisk *disk __attribute__((unused)),
621 int rw __attribute__((unused)), struct bio *bio)
622 {
623 #if defined(HAVE_BDEV_IO_ACCT_63)
624 return (bdev_start_io_acct(bio->bi_bdev, bio_op(bio),
625 jiffies));
626 #elif defined(HAVE_BDEV_IO_ACCT_OLD)
627 return (bdev_start_io_acct(bio->bi_bdev, bio_sectors(bio),
628 bio_op(bio), jiffies));
629 #elif defined(HAVE_DISK_IO_ACCT)
630 return (disk_start_io_acct(disk, bio_sectors(bio), bio_op(bio)));
631 #elif defined(HAVE_BIO_IO_ACCT)
632 return (bio_start_io_acct(bio));
633 #elif defined(HAVE_GENERIC_IO_ACCT_3ARG)
634 unsigned long start_time = jiffies;
635 generic_start_io_acct(rw, bio_sectors(bio), &disk->part0);
636 return (start_time);
637 #elif defined(HAVE_GENERIC_IO_ACCT_4ARG)
638 unsigned long start_time = jiffies;
639 generic_start_io_acct(q, rw, bio_sectors(bio), &disk->part0);
640 return (start_time);
641 #else
642 /* Unsupported */
643 return (0);
644 #endif
645 }
646
647 static inline void
blk_generic_end_io_acct(struct request_queue * q,struct gendisk * disk,int rw,struct bio * bio,unsigned long start_time)648 blk_generic_end_io_acct(struct request_queue *q __attribute__((unused)),
649 struct gendisk *disk __attribute__((unused)),
650 int rw __attribute__((unused)), struct bio *bio, unsigned long start_time)
651 {
652 #if defined(HAVE_BDEV_IO_ACCT_63)
653 bdev_end_io_acct(bio->bi_bdev, bio_op(bio), bio_sectors(bio),
654 start_time);
655 #elif defined(HAVE_BDEV_IO_ACCT_OLD)
656 bdev_end_io_acct(bio->bi_bdev, bio_op(bio), start_time);
657 #elif defined(HAVE_DISK_IO_ACCT)
658 disk_end_io_acct(disk, bio_op(bio), start_time);
659 #elif defined(HAVE_BIO_IO_ACCT)
660 bio_end_io_acct(bio, start_time);
661 #elif defined(HAVE_GENERIC_IO_ACCT_3ARG)
662 generic_end_io_acct(rw, &disk->part0, start_time);
663 #elif defined(HAVE_GENERIC_IO_ACCT_4ARG)
664 generic_end_io_acct(q, rw, &disk->part0, start_time);
665 #endif
666 }
667
668 #ifndef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
669 static inline struct request_queue *
blk_generic_alloc_queue(make_request_fn make_request,int node_id)670 blk_generic_alloc_queue(make_request_fn make_request, int node_id)
671 {
672 #if defined(HAVE_BLK_ALLOC_QUEUE_REQUEST_FN)
673 return (blk_alloc_queue(make_request, node_id));
674 #elif defined(HAVE_BLK_ALLOC_QUEUE_REQUEST_FN_RH)
675 return (blk_alloc_queue_rh(make_request, node_id));
676 #else
677 struct request_queue *q = blk_alloc_queue(GFP_KERNEL);
678 if (q != NULL)
679 blk_queue_make_request(q, make_request);
680
681 return (q);
682 #endif
683 }
684 #endif /* !HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS */
685
686 /*
687 * All the io_*() helper functions below can operate on a bio, or a rq, but
688 * not both. The older submit_bio() codepath will pass a bio, and the
689 * newer blk-mq codepath will pass a rq.
690 */
691 static inline int
io_data_dir(struct bio * bio,struct request * rq)692 io_data_dir(struct bio *bio, struct request *rq)
693 {
694 #ifdef HAVE_BLK_MQ
695 if (rq != NULL) {
696 if (op_is_write(req_op(rq))) {
697 return (WRITE);
698 } else {
699 return (READ);
700 }
701 }
702 #else
703 ASSERT3P(rq, ==, NULL);
704 #endif
705 return (bio_data_dir(bio));
706 }
707
708 static inline int
io_is_flush(struct bio * bio,struct request * rq)709 io_is_flush(struct bio *bio, struct request *rq)
710 {
711 #ifdef HAVE_BLK_MQ
712 if (rq != NULL)
713 return (req_op(rq) == REQ_OP_FLUSH);
714 #else
715 ASSERT3P(rq, ==, NULL);
716 #endif
717 return (bio_is_flush(bio));
718 }
719
720 static inline int
io_is_discard(struct bio * bio,struct request * rq)721 io_is_discard(struct bio *bio, struct request *rq)
722 {
723 #ifdef HAVE_BLK_MQ
724 if (rq != NULL)
725 return (req_op(rq) == REQ_OP_DISCARD);
726 #else
727 ASSERT3P(rq, ==, NULL);
728 #endif
729 return (bio_is_discard(bio));
730 }
731
732 static inline int
io_is_secure_erase(struct bio * bio,struct request * rq)733 io_is_secure_erase(struct bio *bio, struct request *rq)
734 {
735 #ifdef HAVE_BLK_MQ
736 if (rq != NULL)
737 return (req_op(rq) == REQ_OP_SECURE_ERASE);
738 #else
739 ASSERT3P(rq, ==, NULL);
740 #endif
741 return (bio_is_secure_erase(bio));
742 }
743
744 static inline int
io_is_fua(struct bio * bio,struct request * rq)745 io_is_fua(struct bio *bio, struct request *rq)
746 {
747 #ifdef HAVE_BLK_MQ
748 if (rq != NULL)
749 return (rq->cmd_flags & REQ_FUA);
750 #else
751 ASSERT3P(rq, ==, NULL);
752 #endif
753 return (bio_is_fua(bio));
754 }
755
756
757 static inline uint64_t
io_offset(struct bio * bio,struct request * rq)758 io_offset(struct bio *bio, struct request *rq)
759 {
760 #ifdef HAVE_BLK_MQ
761 if (rq != NULL)
762 return (blk_rq_pos(rq) << 9);
763 #else
764 ASSERT3P(rq, ==, NULL);
765 #endif
766 return (BIO_BI_SECTOR(bio) << 9);
767 }
768
769 static inline uint64_t
io_size(struct bio * bio,struct request * rq)770 io_size(struct bio *bio, struct request *rq)
771 {
772 #ifdef HAVE_BLK_MQ
773 if (rq != NULL)
774 return (blk_rq_bytes(rq));
775 #else
776 ASSERT3P(rq, ==, NULL);
777 #endif
778 return (BIO_BI_SIZE(bio));
779 }
780
781 static inline int
io_has_data(struct bio * bio,struct request * rq)782 io_has_data(struct bio *bio, struct request *rq)
783 {
784 #ifdef HAVE_BLK_MQ
785 if (rq != NULL)
786 return (bio_has_data(rq->bio));
787 #else
788 ASSERT3P(rq, ==, NULL);
789 #endif
790 return (bio_has_data(bio));
791 }
792 #endif /* _ZFS_BLKDEV_H */
793