1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (C) 2011 Lawrence Livermore National Security, LLC.
24 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
25 * Written by Brian Behlendorf <[email protected]>.
26 * LLNL-CODE-403049.
27 */
28
29 #ifndef _ZFS_BLKDEV_H
30 #define _ZFS_BLKDEV_H
31
32 #include <linux/blkdev.h>
33 #include <linux/elevator.h>
34 #include <linux/backing-dev.h>
35 #include <linux/hdreg.h>
36 #include <linux/msdos_fs.h> /* for SECTOR_* */
37
38 #ifndef HAVE_BLK_QUEUE_FLAG_SET
39 static inline void
blk_queue_flag_set(unsigned int flag,struct request_queue * q)40 blk_queue_flag_set(unsigned int flag, struct request_queue *q)
41 {
42 queue_flag_set(flag, q);
43 }
44 #endif
45
46 #ifndef HAVE_BLK_QUEUE_FLAG_CLEAR
47 static inline void
blk_queue_flag_clear(unsigned int flag,struct request_queue * q)48 blk_queue_flag_clear(unsigned int flag, struct request_queue *q)
49 {
50 queue_flag_clear(flag, q);
51 }
52 #endif
53
54 /*
55 * 4.7 - 4.x API,
56 * The blk_queue_write_cache() interface has replaced blk_queue_flush()
57 * interface. However, the new interface is GPL-only thus we implement
58 * our own trivial wrapper when the GPL-only version is detected.
59 *
60 * 2.6.36 - 4.6 API,
61 * The blk_queue_flush() interface has replaced blk_queue_ordered()
62 * interface. However, while the old interface was available to all the
63 * new one is GPL-only. Thus if the GPL-only version is detected we
64 * implement our own trivial helper.
65 */
66 static inline void
blk_queue_set_write_cache(struct request_queue * q,bool wc,bool fua)67 blk_queue_set_write_cache(struct request_queue *q, bool wc, bool fua)
68 {
69 #if defined(HAVE_BLK_QUEUE_WRITE_CACHE_GPL_ONLY)
70 if (wc)
71 blk_queue_flag_set(QUEUE_FLAG_WC, q);
72 else
73 blk_queue_flag_clear(QUEUE_FLAG_WC, q);
74 if (fua)
75 blk_queue_flag_set(QUEUE_FLAG_FUA, q);
76 else
77 blk_queue_flag_clear(QUEUE_FLAG_FUA, q);
78 #elif defined(HAVE_BLK_QUEUE_WRITE_CACHE)
79 blk_queue_write_cache(q, wc, fua);
80 #elif defined(HAVE_BLK_QUEUE_FLUSH_GPL_ONLY)
81 if (wc)
82 q->flush_flags |= REQ_FLUSH;
83 if (fua)
84 q->flush_flags |= REQ_FUA;
85 #elif defined(HAVE_BLK_QUEUE_FLUSH)
86 blk_queue_flush(q, (wc ? REQ_FLUSH : 0) | (fua ? REQ_FUA : 0));
87 #else
88 #error "Unsupported kernel"
89 #endif
90 }
91
92 static inline void
blk_queue_set_read_ahead(struct request_queue * q,unsigned long ra_pages)93 blk_queue_set_read_ahead(struct request_queue *q, unsigned long ra_pages)
94 {
95 #ifdef HAVE_BLK_QUEUE_BDI_DYNAMIC
96 q->backing_dev_info->ra_pages = ra_pages;
97 #else
98 q->backing_dev_info.ra_pages = ra_pages;
99 #endif
100 }
101
102 #ifdef HAVE_BIO_BVEC_ITER
103 #define BIO_BI_SECTOR(bio) (bio)->bi_iter.bi_sector
104 #define BIO_BI_SIZE(bio) (bio)->bi_iter.bi_size
105 #define BIO_BI_IDX(bio) (bio)->bi_iter.bi_idx
106 #define BIO_BI_SKIP(bio) (bio)->bi_iter.bi_bvec_done
107 #define bio_for_each_segment4(bv, bvp, b, i) \
108 bio_for_each_segment((bv), (b), (i))
109 typedef struct bvec_iter bvec_iterator_t;
110 #else
111 #define BIO_BI_SECTOR(bio) (bio)->bi_sector
112 #define BIO_BI_SIZE(bio) (bio)->bi_size
113 #define BIO_BI_IDX(bio) (bio)->bi_idx
114 #define BIO_BI_SKIP(bio) (0)
115 #define bio_for_each_segment4(bv, bvp, b, i) \
116 bio_for_each_segment((bvp), (b), (i))
117 typedef int bvec_iterator_t;
118 #endif
119
120 static inline void
bio_set_flags_failfast(struct block_device * bdev,int * flags)121 bio_set_flags_failfast(struct block_device *bdev, int *flags)
122 {
123 #ifdef CONFIG_BUG
124 /*
125 * Disable FAILFAST for loopback devices because of the
126 * following incorrect BUG_ON() in loop_make_request().
127 * This support is also disabled for md devices because the
128 * test suite layers md devices on top of loopback devices.
129 * This may be removed when the loopback driver is fixed.
130 *
131 * BUG_ON(!lo || (rw != READ && rw != WRITE));
132 */
133 if ((MAJOR(bdev->bd_dev) == LOOP_MAJOR) ||
134 (MAJOR(bdev->bd_dev) == MD_MAJOR))
135 return;
136
137 #ifdef BLOCK_EXT_MAJOR
138 if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR)
139 return;
140 #endif /* BLOCK_EXT_MAJOR */
141 #endif /* CONFIG_BUG */
142
143 *flags |= REQ_FAILFAST_MASK;
144 }
145
146 /*
147 * Maximum disk label length, it may be undefined for some kernels.
148 */
149 #if !defined(DISK_NAME_LEN)
150 #define DISK_NAME_LEN 32
151 #endif /* DISK_NAME_LEN */
152
153 #ifdef HAVE_BIO_BI_STATUS
154 static inline int
bi_status_to_errno(blk_status_t status)155 bi_status_to_errno(blk_status_t status)
156 {
157 switch (status) {
158 case BLK_STS_OK:
159 return (0);
160 case BLK_STS_NOTSUPP:
161 return (EOPNOTSUPP);
162 case BLK_STS_TIMEOUT:
163 return (ETIMEDOUT);
164 case BLK_STS_NOSPC:
165 return (ENOSPC);
166 case BLK_STS_TRANSPORT:
167 return (ENOLINK);
168 case BLK_STS_TARGET:
169 return (EREMOTEIO);
170 case BLK_STS_NEXUS:
171 return (EBADE);
172 case BLK_STS_MEDIUM:
173 return (ENODATA);
174 case BLK_STS_PROTECTION:
175 return (EILSEQ);
176 case BLK_STS_RESOURCE:
177 return (ENOMEM);
178 case BLK_STS_AGAIN:
179 return (EAGAIN);
180 case BLK_STS_IOERR:
181 return (EIO);
182 default:
183 return (EIO);
184 }
185 }
186
187 static inline blk_status_t
errno_to_bi_status(int error)188 errno_to_bi_status(int error)
189 {
190 switch (error) {
191 case 0:
192 return (BLK_STS_OK);
193 case EOPNOTSUPP:
194 return (BLK_STS_NOTSUPP);
195 case ETIMEDOUT:
196 return (BLK_STS_TIMEOUT);
197 case ENOSPC:
198 return (BLK_STS_NOSPC);
199 case ENOLINK:
200 return (BLK_STS_TRANSPORT);
201 case EREMOTEIO:
202 return (BLK_STS_TARGET);
203 case EBADE:
204 return (BLK_STS_NEXUS);
205 case ENODATA:
206 return (BLK_STS_MEDIUM);
207 case EILSEQ:
208 return (BLK_STS_PROTECTION);
209 case ENOMEM:
210 return (BLK_STS_RESOURCE);
211 case EAGAIN:
212 return (BLK_STS_AGAIN);
213 case EIO:
214 return (BLK_STS_IOERR);
215 default:
216 return (BLK_STS_IOERR);
217 }
218 }
219 #endif /* HAVE_BIO_BI_STATUS */
220
221 /*
222 * 4.3 API change
223 * The bio_endio() prototype changed slightly. These are helper
224 * macro's to ensure the prototype and invocation are handled.
225 */
226 #ifdef HAVE_1ARG_BIO_END_IO_T
227 #ifdef HAVE_BIO_BI_STATUS
228 #define BIO_END_IO_ERROR(bio) bi_status_to_errno(bio->bi_status)
229 #define BIO_END_IO_PROTO(fn, x, z) static void fn(struct bio *x)
230 #define BIO_END_IO(bio, error) bio_set_bi_status(bio, error)
231 static inline void
bio_set_bi_status(struct bio * bio,int error)232 bio_set_bi_status(struct bio *bio, int error)
233 {
234 ASSERT3S(error, <=, 0);
235 bio->bi_status = errno_to_bi_status(-error);
236 bio_endio(bio);
237 }
238 #else
239 #define BIO_END_IO_ERROR(bio) (-(bio->bi_error))
240 #define BIO_END_IO_PROTO(fn, x, z) static void fn(struct bio *x)
241 #define BIO_END_IO(bio, error) bio_set_bi_error(bio, error)
242 static inline void
bio_set_bi_error(struct bio * bio,int error)243 bio_set_bi_error(struct bio *bio, int error)
244 {
245 ASSERT3S(error, <=, 0);
246 bio->bi_error = error;
247 bio_endio(bio);
248 }
249 #endif /* HAVE_BIO_BI_STATUS */
250
251 #else
252 #define BIO_END_IO_PROTO(fn, x, z) static void fn(struct bio *x, int z)
253 #define BIO_END_IO(bio, error) bio_endio(bio, error);
254 #endif /* HAVE_1ARG_BIO_END_IO_T */
255
256 /*
257 * 4.1 - x.y.z API,
258 * 3.10.0 CentOS 7.x API,
259 * blkdev_reread_part()
260 *
261 * For older kernels trigger a re-reading of the partition table by calling
262 * check_disk_change() which calls flush_disk() to invalidate the device.
263 *
264 * For newer kernels (as of 5.10), bdev_check_media_chage is used, in favor of
265 * check_disk_change(), with the modification that invalidation is no longer
266 * forced.
267 */
268 #ifdef HAVE_CHECK_DISK_CHANGE
269 #define zfs_check_media_change(bdev) check_disk_change(bdev)
270 #ifdef HAVE_BLKDEV_REREAD_PART
271 #define vdev_bdev_reread_part(bdev) blkdev_reread_part(bdev)
272 #else
273 #define vdev_bdev_reread_part(bdev) check_disk_change(bdev)
274 #endif /* HAVE_BLKDEV_REREAD_PART */
275 #else
276 #ifdef HAVE_BDEV_CHECK_MEDIA_CHANGE
277 static inline int
zfs_check_media_change(struct block_device * bdev)278 zfs_check_media_change(struct block_device *bdev)
279 {
280 struct gendisk *gd = bdev->bd_disk;
281 const struct block_device_operations *bdo = gd->fops;
282
283 if (!bdev_check_media_change(bdev))
284 return (0);
285
286 /*
287 * Force revalidation, to mimic the old behavior of
288 * check_disk_change()
289 */
290 if (bdo->revalidate_disk)
291 bdo->revalidate_disk(gd);
292
293 return (0);
294 }
295 #define vdev_bdev_reread_part(bdev) zfs_check_media_change(bdev)
296 #else
297 /*
298 * This is encountered if check_disk_change() and bdev_check_media_change()
299 * are not available in the kernel - likely due to an API change that needs
300 * to be chased down.
301 */
302 #error "Unsupported kernel: no usable disk change check"
303 #endif /* HAVE_BDEV_CHECK_MEDIA_CHANGE */
304 #endif /* HAVE_CHECK_DISK_CHANGE */
305
306 /*
307 * 2.6.27 API change
308 * The function was exported for use, prior to this it existed but the
309 * symbol was not exported.
310 *
311 * 4.4.0-6.21 API change for Ubuntu
312 * lookup_bdev() gained a second argument, FMODE_*, to check inode permissions.
313 *
314 * 5.11 API change
315 * Changed to take a dev_t argument which is set on success and return a
316 * non-zero error code on failure.
317 */
318 static inline int
vdev_lookup_bdev(const char * path,dev_t * dev)319 vdev_lookup_bdev(const char *path, dev_t *dev)
320 {
321 #if defined(HAVE_DEVT_LOOKUP_BDEV)
322 return (lookup_bdev(path, dev));
323 #elif defined(HAVE_1ARG_LOOKUP_BDEV)
324 struct block_device *bdev = lookup_bdev(path);
325 if (IS_ERR(bdev))
326 return (PTR_ERR(bdev));
327
328 *dev = bdev->bd_dev;
329 bdput(bdev);
330
331 return (0);
332 #elif defined(HAVE_MODE_LOOKUP_BDEV)
333 struct block_device *bdev = lookup_bdev(path, FMODE_READ);
334 if (IS_ERR(bdev))
335 return (PTR_ERR(bdev));
336
337 *dev = bdev->bd_dev;
338 bdput(bdev);
339
340 return (0);
341 #else
342 #error "Unsupported kernel"
343 #endif
344 }
345
346 /*
347 * Kernels without bio_set_op_attrs use bi_rw for the bio flags.
348 */
349 #if !defined(HAVE_BIO_SET_OP_ATTRS)
350 static inline void
bio_set_op_attrs(struct bio * bio,unsigned rw,unsigned flags)351 bio_set_op_attrs(struct bio *bio, unsigned rw, unsigned flags)
352 {
353 bio->bi_rw |= rw | flags;
354 }
355 #endif
356
357 /*
358 * bio_set_flush - Set the appropriate flags in a bio to guarantee
359 * data are on non-volatile media on completion.
360 *
361 * 2.6.37 - 4.8 API,
362 * Introduce WRITE_FLUSH, WRITE_FUA, and WRITE_FLUSH_FUA flags as a
363 * replacement for WRITE_BARRIER to allow expressing richer semantics
364 * to the block layer. It's up to the block layer to implement the
365 * semantics correctly. Use the WRITE_FLUSH_FUA flag combination.
366 *
367 * 4.8 - 4.9 API,
368 * REQ_FLUSH was renamed to REQ_PREFLUSH. For consistency with previous
369 * ZoL releases, prefer the WRITE_FLUSH_FUA flag set if it's available.
370 *
371 * 4.10 API,
372 * The read/write flags and their modifiers, including WRITE_FLUSH,
373 * WRITE_FUA and WRITE_FLUSH_FUA were removed from fs.h in
374 * torvalds/linux@70fd7614 and replaced by direct flag modification
375 * of the REQ_ flags in bio->bi_opf. Use REQ_PREFLUSH.
376 */
377 static inline void
bio_set_flush(struct bio * bio)378 bio_set_flush(struct bio *bio)
379 {
380 #if defined(HAVE_REQ_PREFLUSH) /* >= 4.10 */
381 bio_set_op_attrs(bio, 0, REQ_PREFLUSH);
382 #elif defined(WRITE_FLUSH_FUA) /* >= 2.6.37 and <= 4.9 */
383 bio_set_op_attrs(bio, 0, WRITE_FLUSH_FUA);
384 #else
385 #error "Allowing the build will cause bio_set_flush requests to be ignored."
386 #endif
387 }
388
389 /*
390 * 4.8 - 4.x API,
391 * REQ_OP_FLUSH
392 *
393 * 4.8-rc0 - 4.8-rc1,
394 * REQ_PREFLUSH
395 *
396 * 2.6.36 - 4.7 API,
397 * REQ_FLUSH
398 *
399 * in all cases but may have a performance impact for some kernels. It
400 * has the advantage of minimizing kernel specific changes in the zvol code.
401 *
402 */
403 static inline boolean_t
bio_is_flush(struct bio * bio)404 bio_is_flush(struct bio *bio)
405 {
406 #if defined(HAVE_REQ_OP_FLUSH) && defined(HAVE_BIO_BI_OPF)
407 return ((bio_op(bio) == REQ_OP_FLUSH) || (bio->bi_opf & REQ_PREFLUSH));
408 #elif defined(HAVE_REQ_PREFLUSH) && defined(HAVE_BIO_BI_OPF)
409 return (bio->bi_opf & REQ_PREFLUSH);
410 #elif defined(HAVE_REQ_PREFLUSH) && !defined(HAVE_BIO_BI_OPF)
411 return (bio->bi_rw & REQ_PREFLUSH);
412 #elif defined(HAVE_REQ_FLUSH)
413 return (bio->bi_rw & REQ_FLUSH);
414 #else
415 #error "Unsupported kernel"
416 #endif
417 }
418
419 /*
420 * 4.8 - 4.x API,
421 * REQ_FUA flag moved to bio->bi_opf
422 *
423 * 2.6.x - 4.7 API,
424 * REQ_FUA
425 */
426 static inline boolean_t
bio_is_fua(struct bio * bio)427 bio_is_fua(struct bio *bio)
428 {
429 #if defined(HAVE_BIO_BI_OPF)
430 return (bio->bi_opf & REQ_FUA);
431 #elif defined(REQ_FUA)
432 return (bio->bi_rw & REQ_FUA);
433 #else
434 #error "Allowing the build will cause fua requests to be ignored."
435 #endif
436 }
437
438 /*
439 * 4.8 - 4.x API,
440 * REQ_OP_DISCARD
441 *
442 * 2.6.36 - 4.7 API,
443 * REQ_DISCARD
444 *
445 * In all cases the normal I/O path is used for discards. The only
446 * difference is how the kernel tags individual I/Os as discards.
447 */
448 static inline boolean_t
bio_is_discard(struct bio * bio)449 bio_is_discard(struct bio *bio)
450 {
451 #if defined(HAVE_REQ_OP_DISCARD)
452 return (bio_op(bio) == REQ_OP_DISCARD);
453 #elif defined(HAVE_REQ_DISCARD)
454 return (bio->bi_rw & REQ_DISCARD);
455 #else
456 #error "Unsupported kernel"
457 #endif
458 }
459
460 /*
461 * 4.8 - 4.x API,
462 * REQ_OP_SECURE_ERASE
463 *
464 * 2.6.36 - 4.7 API,
465 * REQ_SECURE
466 */
467 static inline boolean_t
bio_is_secure_erase(struct bio * bio)468 bio_is_secure_erase(struct bio *bio)
469 {
470 #if defined(HAVE_REQ_OP_SECURE_ERASE)
471 return (bio_op(bio) == REQ_OP_SECURE_ERASE);
472 #elif defined(REQ_SECURE)
473 return (bio->bi_rw & REQ_SECURE);
474 #else
475 return (0);
476 #endif
477 }
478
479 /*
480 * 2.6.33 API change
481 * Discard granularity and alignment restrictions may now be set. For
482 * older kernels which do not support this it is safe to skip it.
483 */
484 static inline void
blk_queue_discard_granularity(struct request_queue * q,unsigned int dg)485 blk_queue_discard_granularity(struct request_queue *q, unsigned int dg)
486 {
487 q->limits.discard_granularity = dg;
488 }
489
490 /*
491 * 4.8 - 4.x API,
492 * blk_queue_secure_erase()
493 *
494 * 2.6.36 - 4.7 API,
495 * blk_queue_secdiscard()
496 */
497 static inline int
blk_queue_discard_secure(struct request_queue * q)498 blk_queue_discard_secure(struct request_queue *q)
499 {
500 #if defined(HAVE_BLK_QUEUE_SECURE_ERASE)
501 return (blk_queue_secure_erase(q));
502 #elif defined(HAVE_BLK_QUEUE_SECDISCARD)
503 return (blk_queue_secdiscard(q));
504 #else
505 return (0);
506 #endif
507 }
508
509 /*
510 * A common holder for vdev_bdev_open() is used to relax the exclusive open
511 * semantics slightly. Internal vdev disk callers may pass VDEV_HOLDER to
512 * allow them to open the device multiple times. Other kernel callers and
513 * user space processes which don't pass this value will get EBUSY. This is
514 * currently required for the correct operation of hot spares.
515 */
516 #define VDEV_HOLDER ((void *)0x2401de7)
517
518 static inline unsigned long
blk_generic_start_io_acct(struct request_queue * q,struct gendisk * disk,int rw,struct bio * bio)519 blk_generic_start_io_acct(struct request_queue *q __attribute__((unused)),
520 struct gendisk *disk __attribute__((unused)),
521 int rw __attribute__((unused)), struct bio *bio)
522 {
523 #if defined(HAVE_BIO_IO_ACCT)
524 return (bio_start_io_acct(bio));
525 #elif defined(HAVE_GENERIC_IO_ACCT_3ARG)
526 unsigned long start_time = jiffies;
527 generic_start_io_acct(rw, bio_sectors(bio), &disk->part0);
528 return (start_time);
529 #elif defined(HAVE_GENERIC_IO_ACCT_4ARG)
530 unsigned long start_time = jiffies;
531 generic_start_io_acct(q, rw, bio_sectors(bio), &disk->part0);
532 return (start_time);
533 #else
534 /* Unsupported */
535 return (0);
536 #endif
537 }
538
539 static inline void
blk_generic_end_io_acct(struct request_queue * q,struct gendisk * disk,int rw,struct bio * bio,unsigned long start_time)540 blk_generic_end_io_acct(struct request_queue *q __attribute__((unused)),
541 struct gendisk *disk __attribute__((unused)),
542 int rw __attribute__((unused)), struct bio *bio, unsigned long start_time)
543 {
544 #if defined(HAVE_BIO_IO_ACCT)
545 bio_end_io_acct(bio, start_time);
546 #elif defined(HAVE_GENERIC_IO_ACCT_3ARG)
547 generic_end_io_acct(rw, &disk->part0, start_time);
548 #elif defined(HAVE_GENERIC_IO_ACCT_4ARG)
549 generic_end_io_acct(q, rw, &disk->part0, start_time);
550 #endif
551 }
552
553 #ifndef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
554 static inline struct request_queue *
blk_generic_alloc_queue(make_request_fn make_request,int node_id)555 blk_generic_alloc_queue(make_request_fn make_request, int node_id)
556 {
557 #if defined(HAVE_BLK_ALLOC_QUEUE_REQUEST_FN)
558 return (blk_alloc_queue(make_request, node_id));
559 #elif defined(HAVE_BLK_ALLOC_QUEUE_REQUEST_FN_RH)
560 return (blk_alloc_queue_rh(make_request, node_id));
561 #else
562 struct request_queue *q = blk_alloc_queue(GFP_KERNEL);
563 if (q != NULL)
564 blk_queue_make_request(q, make_request);
565
566 return (q);
567 #endif
568 }
569 #endif /* !HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS */
570
571 #endif /* _ZFS_BLKDEV_H */
572