1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Portions Copyright (C) 1992 Drew Eckhardt 4 */ 5 #ifndef _LINUX_BLKDEV_H 6 #define _LINUX_BLKDEV_H 7 8 #include <linux/types.h> 9 #include <linux/blk_types.h> 10 #include <linux/device.h> 11 #include <linux/list.h> 12 #include <linux/llist.h> 13 #include <linux/minmax.h> 14 #include <linux/timer.h> 15 #include <linux/workqueue.h> 16 #include <linux/wait.h> 17 #include <linux/bio.h> 18 #include <linux/gfp.h> 19 #include <linux/kdev_t.h> 20 #include <linux/rcupdate.h> 21 #include <linux/percpu-refcount.h> 22 #include <linux/blkzoned.h> 23 #include <linux/sched.h> 24 #include <linux/sbitmap.h> 25 #include <linux/uuid.h> 26 #include <linux/xarray.h> 27 #include <linux/file.h> 28 #include <linux/lockdep.h> 29 30 struct module; 31 struct request_queue; 32 struct elevator_queue; 33 struct blk_trace; 34 struct request; 35 struct sg_io_hdr; 36 struct blkcg_gq; 37 struct blk_flush_queue; 38 struct kiocb; 39 struct pr_ops; 40 struct rq_qos; 41 struct blk_queue_stats; 42 struct blk_stat_callback; 43 struct blk_crypto_profile; 44 45 extern const struct device_type disk_type; 46 extern const struct device_type part_type; 47 extern const struct class block_class; 48 49 /* 50 * Maximum number of blkcg policies allowed to be registered concurrently. 51 * Defined here to simplify include dependency. 52 */ 53 #define BLKCG_MAX_POLS 6 54 55 #define DISK_MAX_PARTS 256 56 #define DISK_NAME_LEN 32 57 58 #define PARTITION_META_INFO_VOLNAMELTH 64 59 /* 60 * Enough for the string representation of any kind of UUID plus NULL. 61 * EFI UUID is 36 characters. MSDOS UUID is 11 characters. 62 */ 63 #define PARTITION_META_INFO_UUIDLTH (UUID_STRING_LEN + 1) 64 65 struct partition_meta_info { 66 char uuid[PARTITION_META_INFO_UUIDLTH]; 67 u8 volname[PARTITION_META_INFO_VOLNAMELTH]; 68 }; 69 70 /** 71 * DOC: genhd capability flags 72 * 73 * ``GENHD_FL_REMOVABLE``: indicates that the block device gives access to 74 * removable media. When set, the device remains present even when media is not 75 * inserted. Shall not be set for devices which are removed entirely when the 76 * media is removed. 77 * 78 * ``GENHD_FL_HIDDEN``: the block device is hidden; it doesn't produce events, 79 * doesn't appear in sysfs, and can't be opened from userspace or using 80 * blkdev_get*. Used for the underlying components of multipath devices. 81 * 82 * ``GENHD_FL_NO_PART``: partition support is disabled. The kernel will not 83 * scan for partitions from add_disk, and users can't add partitions manually. 84 * 85 */ 86 enum { 87 GENHD_FL_REMOVABLE = 1 << 0, 88 GENHD_FL_HIDDEN = 1 << 1, 89 GENHD_FL_NO_PART = 1 << 2, 90 }; 91 92 enum { 93 DISK_EVENT_MEDIA_CHANGE = 1 << 0, /* media changed */ 94 DISK_EVENT_EJECT_REQUEST = 1 << 1, /* eject requested */ 95 }; 96 97 enum { 98 /* Poll even if events_poll_msecs is unset */ 99 DISK_EVENT_FLAG_POLL = 1 << 0, 100 /* Forward events to udev */ 101 DISK_EVENT_FLAG_UEVENT = 1 << 1, 102 /* Block event polling when open for exclusive write */ 103 DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE = 1 << 2, 104 }; 105 106 struct disk_events; 107 struct badblocks; 108 109 enum blk_integrity_checksum { 110 BLK_INTEGRITY_CSUM_NONE = 0, 111 BLK_INTEGRITY_CSUM_IP = 1, 112 BLK_INTEGRITY_CSUM_CRC = 2, 113 BLK_INTEGRITY_CSUM_CRC64 = 3, 114 } __packed ; 115 116 struct blk_integrity { 117 unsigned char flags; 118 enum blk_integrity_checksum csum_type; 119 unsigned char tuple_size; 120 unsigned char pi_offset; 121 unsigned char interval_exp; 122 unsigned char tag_size; 123 }; 124 125 typedef unsigned int __bitwise blk_mode_t; 126 127 /* open for reading */ 128 #define BLK_OPEN_READ ((__force blk_mode_t)(1 << 0)) 129 /* open for writing */ 130 #define BLK_OPEN_WRITE ((__force blk_mode_t)(1 << 1)) 131 /* open exclusively (vs other exclusive openers */ 132 #define BLK_OPEN_EXCL ((__force blk_mode_t)(1 << 2)) 133 /* opened with O_NDELAY */ 134 #define BLK_OPEN_NDELAY ((__force blk_mode_t)(1 << 3)) 135 /* open for "writes" only for ioctls (specialy hack for floppy.c) */ 136 #define BLK_OPEN_WRITE_IOCTL ((__force blk_mode_t)(1 << 4)) 137 /* open is exclusive wrt all other BLK_OPEN_WRITE opens to the device */ 138 #define BLK_OPEN_RESTRICT_WRITES ((__force blk_mode_t)(1 << 5)) 139 /* return partition scanning errors */ 140 #define BLK_OPEN_STRICT_SCAN ((__force blk_mode_t)(1 << 6)) 141 142 struct gendisk { 143 /* 144 * major/first_minor/minors should not be set by any new driver, the 145 * block core will take care of allocating them automatically. 146 */ 147 int major; 148 int first_minor; 149 int minors; 150 151 char disk_name[DISK_NAME_LEN]; /* name of major driver */ 152 153 unsigned short events; /* supported events */ 154 unsigned short event_flags; /* flags related to event processing */ 155 156 struct xarray part_tbl; 157 struct block_device *part0; 158 159 const struct block_device_operations *fops; 160 struct request_queue *queue; 161 void *private_data; 162 163 struct bio_set bio_split; 164 165 int flags; 166 unsigned long state; 167 #define GD_NEED_PART_SCAN 0 168 #define GD_READ_ONLY 1 169 #define GD_DEAD 2 170 #define GD_NATIVE_CAPACITY 3 171 #define GD_ADDED 4 172 #define GD_SUPPRESS_PART_SCAN 5 173 #define GD_OWNS_QUEUE 6 174 175 struct mutex open_mutex; /* open/close mutex */ 176 unsigned open_partitions; /* number of open partitions */ 177 178 struct backing_dev_info *bdi; 179 struct kobject queue_kobj; /* the queue/ directory */ 180 struct kobject *slave_dir; 181 #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED 182 struct list_head slave_bdevs; 183 #endif 184 struct timer_rand_state *random; 185 atomic_t sync_io; /* RAID */ 186 struct disk_events *ev; 187 188 #ifdef CONFIG_BLK_DEV_ZONED 189 /* 190 * Zoned block device information. Reads of this information must be 191 * protected with blk_queue_enter() / blk_queue_exit(). Modifying this 192 * information is only allowed while no requests are being processed. 193 * See also blk_mq_freeze_queue() and blk_mq_unfreeze_queue(). 194 */ 195 unsigned int nr_zones; 196 unsigned int zone_capacity; 197 unsigned int last_zone_capacity; 198 unsigned long __rcu *conv_zones_bitmap; 199 unsigned int zone_wplugs_hash_bits; 200 atomic_t nr_zone_wplugs; 201 spinlock_t zone_wplugs_lock; 202 struct mempool_s *zone_wplugs_pool; 203 struct hlist_head *zone_wplugs_hash; 204 struct workqueue_struct *zone_wplugs_wq; 205 #endif /* CONFIG_BLK_DEV_ZONED */ 206 207 #if IS_ENABLED(CONFIG_CDROM) 208 struct cdrom_device_info *cdi; 209 #endif 210 int node_id; 211 struct badblocks *bb; 212 struct lockdep_map lockdep_map; 213 u64 diskseq; 214 blk_mode_t open_mode; 215 216 /* 217 * Independent sector access ranges. This is always NULL for 218 * devices that do not have multiple independent access ranges. 219 */ 220 struct blk_independent_access_ranges *ia_ranges; 221 }; 222 223 /** 224 * disk_openers - returns how many openers are there for a disk 225 * @disk: disk to check 226 * 227 * This returns the number of openers for a disk. Note that this value is only 228 * stable if disk->open_mutex is held. 229 * 230 * Note: Due to a quirk in the block layer open code, each open partition is 231 * only counted once even if there are multiple openers. 232 */ 233 static inline unsigned int disk_openers(struct gendisk *disk) 234 { 235 return atomic_read(&disk->part0->bd_openers); 236 } 237 238 /** 239 * disk_has_partscan - return %true if partition scanning is enabled on a disk 240 * @disk: disk to check 241 * 242 * Returns %true if partitions scanning is enabled for @disk, or %false if 243 * partition scanning is disabled either permanently or temporarily. 244 */ 245 static inline bool disk_has_partscan(struct gendisk *disk) 246 { 247 return !(disk->flags & (GENHD_FL_NO_PART | GENHD_FL_HIDDEN)) && 248 !test_bit(GD_SUPPRESS_PART_SCAN, &disk->state); 249 } 250 251 /* 252 * The gendisk is refcounted by the part0 block_device, and the bd_device 253 * therein is also used for device model presentation in sysfs. 254 */ 255 #define dev_to_disk(device) \ 256 (dev_to_bdev(device)->bd_disk) 257 #define disk_to_dev(disk) \ 258 (&((disk)->part0->bd_device)) 259 260 #if IS_REACHABLE(CONFIG_CDROM) 261 #define disk_to_cdi(disk) ((disk)->cdi) 262 #else 263 #define disk_to_cdi(disk) NULL 264 #endif 265 266 static inline dev_t disk_devt(struct gendisk *disk) 267 { 268 return MKDEV(disk->major, disk->first_minor); 269 } 270 271 /* 272 * We should strive for 1 << (PAGE_SHIFT + MAX_PAGECACHE_ORDER) 273 * however we constrain this to what we can validate and test. 274 */ 275 #define BLK_MAX_BLOCK_SIZE SZ_64K 276 277 /* blk_validate_limits() validates bsize, so drivers don't usually need to */ 278 static inline int blk_validate_block_size(unsigned long bsize) 279 { 280 if (bsize < 512 || bsize > BLK_MAX_BLOCK_SIZE || !is_power_of_2(bsize)) 281 return -EINVAL; 282 283 return 0; 284 } 285 286 static inline bool blk_op_is_passthrough(blk_opf_t op) 287 { 288 op &= REQ_OP_MASK; 289 return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT; 290 } 291 292 /* flags set by the driver in queue_limits.features */ 293 typedef unsigned int __bitwise blk_features_t; 294 295 /* supports a volatile write cache */ 296 #define BLK_FEAT_WRITE_CACHE ((__force blk_features_t)(1u << 0)) 297 298 /* supports passing on the FUA bit */ 299 #define BLK_FEAT_FUA ((__force blk_features_t)(1u << 1)) 300 301 /* rotational device (hard drive or floppy) */ 302 #define BLK_FEAT_ROTATIONAL ((__force blk_features_t)(1u << 2)) 303 304 /* contributes to the random number pool */ 305 #define BLK_FEAT_ADD_RANDOM ((__force blk_features_t)(1u << 3)) 306 307 /* do disk/partitions IO accounting */ 308 #define BLK_FEAT_IO_STAT ((__force blk_features_t)(1u << 4)) 309 310 /* don't modify data until writeback is done */ 311 #define BLK_FEAT_STABLE_WRITES ((__force blk_features_t)(1u << 5)) 312 313 /* always completes in submit context */ 314 #define BLK_FEAT_SYNCHRONOUS ((__force blk_features_t)(1u << 6)) 315 316 /* supports REQ_NOWAIT */ 317 #define BLK_FEAT_NOWAIT ((__force blk_features_t)(1u << 7)) 318 319 /* supports DAX */ 320 #define BLK_FEAT_DAX ((__force blk_features_t)(1u << 8)) 321 322 /* supports I/O polling */ 323 #define BLK_FEAT_POLL ((__force blk_features_t)(1u << 9)) 324 325 /* is a zoned device */ 326 #define BLK_FEAT_ZONED ((__force blk_features_t)(1u << 10)) 327 328 /* supports PCI(e) p2p requests */ 329 #define BLK_FEAT_PCI_P2PDMA ((__force blk_features_t)(1u << 12)) 330 331 /* skip this queue in blk_mq_(un)quiesce_tagset */ 332 #define BLK_FEAT_SKIP_TAGSET_QUIESCE ((__force blk_features_t)(1u << 13)) 333 334 /* bounce all highmem pages */ 335 #define BLK_FEAT_BOUNCE_HIGH ((__force blk_features_t)(1u << 14)) 336 337 /* undocumented magic for bcache */ 338 #define BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE \ 339 ((__force blk_features_t)(1u << 15)) 340 341 /* atomic writes enabled */ 342 #define BLK_FEAT_ATOMIC_WRITES \ 343 ((__force blk_features_t)(1u << 16)) 344 345 /* 346 * Flags automatically inherited when stacking limits. 347 */ 348 #define BLK_FEAT_INHERIT_MASK \ 349 (BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | BLK_FEAT_ROTATIONAL | \ 350 BLK_FEAT_STABLE_WRITES | BLK_FEAT_ZONED | BLK_FEAT_BOUNCE_HIGH | \ 351 BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE) 352 353 /* internal flags in queue_limits.flags */ 354 typedef unsigned int __bitwise blk_flags_t; 355 356 /* do not send FLUSH/FUA commands despite advertising a write cache */ 357 #define BLK_FLAG_WRITE_CACHE_DISABLED ((__force blk_flags_t)(1u << 0)) 358 359 /* I/O topology is misaligned */ 360 #define BLK_FLAG_MISALIGNED ((__force blk_flags_t)(1u << 1)) 361 362 /* passthrough command IO accounting */ 363 #define BLK_FLAG_IOSTATS_PASSTHROUGH ((__force blk_flags_t)(1u << 2)) 364 365 struct queue_limits { 366 blk_features_t features; 367 blk_flags_t flags; 368 unsigned long seg_boundary_mask; 369 unsigned long virt_boundary_mask; 370 371 unsigned int max_hw_sectors; 372 unsigned int max_dev_sectors; 373 unsigned int chunk_sectors; 374 unsigned int max_sectors; 375 unsigned int max_user_sectors; 376 unsigned int max_segment_size; 377 unsigned int min_segment_size; 378 unsigned int physical_block_size; 379 unsigned int logical_block_size; 380 unsigned int alignment_offset; 381 unsigned int io_min; 382 unsigned int io_opt; 383 unsigned int max_discard_sectors; 384 unsigned int max_hw_discard_sectors; 385 unsigned int max_user_discard_sectors; 386 unsigned int max_secure_erase_sectors; 387 unsigned int max_write_zeroes_sectors; 388 unsigned int max_hw_zone_append_sectors; 389 unsigned int max_zone_append_sectors; 390 unsigned int discard_granularity; 391 unsigned int discard_alignment; 392 unsigned int zone_write_granularity; 393 394 /* atomic write limits */ 395 unsigned int atomic_write_hw_max; 396 unsigned int atomic_write_max_sectors; 397 unsigned int atomic_write_hw_boundary; 398 unsigned int atomic_write_boundary_sectors; 399 unsigned int atomic_write_hw_unit_min; 400 unsigned int atomic_write_unit_min; 401 unsigned int atomic_write_hw_unit_max; 402 unsigned int atomic_write_unit_max; 403 404 unsigned short max_segments; 405 unsigned short max_integrity_segments; 406 unsigned short max_discard_segments; 407 408 unsigned int max_open_zones; 409 unsigned int max_active_zones; 410 411 /* 412 * Drivers that set dma_alignment to less than 511 must be prepared to 413 * handle individual bvec's that are not a multiple of a SECTOR_SIZE 414 * due to possible offsets. 415 */ 416 unsigned int dma_alignment; 417 unsigned int dma_pad_mask; 418 419 struct blk_integrity integrity; 420 }; 421 422 typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx, 423 void *data); 424 425 #define BLK_ALL_ZONES ((unsigned int)-1) 426 int blkdev_report_zones(struct block_device *bdev, sector_t sector, 427 unsigned int nr_zones, report_zones_cb cb, void *data); 428 int blkdev_zone_mgmt(struct block_device *bdev, enum req_op op, 429 sector_t sectors, sector_t nr_sectors); 430 int blk_revalidate_disk_zones(struct gendisk *disk); 431 432 /* 433 * Independent access ranges: struct blk_independent_access_range describes 434 * a range of contiguous sectors that can be accessed using device command 435 * execution resources that are independent from the resources used for 436 * other access ranges. This is typically found with single-LUN multi-actuator 437 * HDDs where each access range is served by a different set of heads. 438 * The set of independent ranges supported by the device is defined using 439 * struct blk_independent_access_ranges. The independent ranges must not overlap 440 * and must include all sectors within the disk capacity (no sector holes 441 * allowed). 442 * For a device with multiple ranges, requests targeting sectors in different 443 * ranges can be executed in parallel. A request can straddle an access range 444 * boundary. 445 */ 446 struct blk_independent_access_range { 447 struct kobject kobj; 448 sector_t sector; 449 sector_t nr_sectors; 450 }; 451 452 struct blk_independent_access_ranges { 453 struct kobject kobj; 454 bool sysfs_registered; 455 unsigned int nr_ia_ranges; 456 struct blk_independent_access_range ia_range[]; 457 }; 458 459 struct request_queue { 460 /* 461 * The queue owner gets to use this for whatever they like. 462 * ll_rw_blk doesn't touch it. 463 */ 464 void *queuedata; 465 466 struct elevator_queue *elevator; 467 468 const struct blk_mq_ops *mq_ops; 469 470 /* sw queues */ 471 struct blk_mq_ctx __percpu *queue_ctx; 472 473 /* 474 * various queue flags, see QUEUE_* below 475 */ 476 unsigned long queue_flags; 477 478 unsigned int rq_timeout; 479 480 unsigned int queue_depth; 481 482 refcount_t refs; 483 484 /* hw dispatch queues */ 485 unsigned int nr_hw_queues; 486 struct xarray hctx_table; 487 488 struct percpu_ref q_usage_counter; 489 struct lock_class_key io_lock_cls_key; 490 struct lockdep_map io_lockdep_map; 491 492 struct lock_class_key q_lock_cls_key; 493 struct lockdep_map q_lockdep_map; 494 495 struct request *last_merge; 496 497 spinlock_t queue_lock; 498 499 int quiesce_depth; 500 501 struct gendisk *disk; 502 503 /* 504 * mq queue kobject 505 */ 506 struct kobject *mq_kobj; 507 508 struct queue_limits limits; 509 510 #ifdef CONFIG_PM 511 struct device *dev; 512 enum rpm_status rpm_status; 513 #endif 514 515 /* 516 * Number of contexts that have called blk_set_pm_only(). If this 517 * counter is above zero then only RQF_PM requests are processed. 518 */ 519 atomic_t pm_only; 520 521 struct blk_queue_stats *stats; 522 struct rq_qos *rq_qos; 523 struct mutex rq_qos_mutex; 524 525 /* 526 * ida allocated id for this queue. Used to index queues from 527 * ioctx. 528 */ 529 int id; 530 531 /* 532 * queue settings 533 */ 534 unsigned long nr_requests; /* Max # of requests */ 535 536 #ifdef CONFIG_BLK_INLINE_ENCRYPTION 537 struct blk_crypto_profile *crypto_profile; 538 struct kobject *crypto_kobject; 539 #endif 540 541 struct timer_list timeout; 542 struct work_struct timeout_work; 543 544 atomic_t nr_active_requests_shared_tags; 545 546 struct blk_mq_tags *sched_shared_tags; 547 548 struct list_head icq_list; 549 #ifdef CONFIG_BLK_CGROUP 550 DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS); 551 struct blkcg_gq *root_blkg; 552 struct list_head blkg_list; 553 struct mutex blkcg_mutex; 554 #endif 555 556 int node; 557 558 spinlock_t requeue_lock; 559 struct list_head requeue_list; 560 struct delayed_work requeue_work; 561 562 #ifdef CONFIG_BLK_DEV_IO_TRACE 563 struct blk_trace __rcu *blk_trace; 564 #endif 565 /* 566 * for flush operations 567 */ 568 struct blk_flush_queue *fq; 569 struct list_head flush_list; 570 571 struct mutex sysfs_lock; 572 struct mutex limits_lock; 573 574 /* 575 * for reusing dead hctx instance in case of updating 576 * nr_hw_queues 577 */ 578 struct list_head unused_hctx_list; 579 spinlock_t unused_hctx_lock; 580 581 int mq_freeze_depth; 582 583 #ifdef CONFIG_BLK_DEV_THROTTLING 584 /* Throttle data */ 585 struct throtl_data *td; 586 #endif 587 struct rcu_head rcu_head; 588 #ifdef CONFIG_LOCKDEP 589 struct task_struct *mq_freeze_owner; 590 int mq_freeze_owner_depth; 591 /* 592 * Records disk & queue state in current context, used in unfreeze 593 * queue 594 */ 595 bool mq_freeze_disk_dead; 596 bool mq_freeze_queue_dying; 597 #endif 598 wait_queue_head_t mq_freeze_wq; 599 /* 600 * Protect concurrent access to q_usage_counter by 601 * percpu_ref_kill() and percpu_ref_reinit(). 602 */ 603 struct mutex mq_freeze_lock; 604 605 struct blk_mq_tag_set *tag_set; 606 struct list_head tag_set_list; 607 608 struct dentry *debugfs_dir; 609 struct dentry *sched_debugfs_dir; 610 struct dentry *rqos_debugfs_dir; 611 /* 612 * Serializes all debugfs metadata operations using the above dentries. 613 */ 614 struct mutex debugfs_mutex; 615 }; 616 617 /* Keep blk_queue_flag_name[] in sync with the definitions below */ 618 enum { 619 QUEUE_FLAG_DYING, /* queue being torn down */ 620 QUEUE_FLAG_NOMERGES, /* disable merge attempts */ 621 QUEUE_FLAG_SAME_COMP, /* complete on same CPU-group */ 622 QUEUE_FLAG_FAIL_IO, /* fake timeout */ 623 QUEUE_FLAG_NOXMERGES, /* No extended merges */ 624 QUEUE_FLAG_SAME_FORCE, /* force complete on same CPU */ 625 QUEUE_FLAG_INIT_DONE, /* queue is initialized */ 626 QUEUE_FLAG_STATS, /* track IO start and completion times */ 627 QUEUE_FLAG_REGISTERED, /* queue has been registered to a disk */ 628 QUEUE_FLAG_QUIESCED, /* queue has been quiesced */ 629 QUEUE_FLAG_RQ_ALLOC_TIME, /* record rq->alloc_time_ns */ 630 QUEUE_FLAG_HCTX_ACTIVE, /* at least one blk-mq hctx is active */ 631 QUEUE_FLAG_SQ_SCHED, /* single queue style io dispatch */ 632 QUEUE_FLAG_MAX 633 }; 634 635 #define QUEUE_FLAG_MQ_DEFAULT (1UL << QUEUE_FLAG_SAME_COMP) 636 637 void blk_queue_flag_set(unsigned int flag, struct request_queue *q); 638 void blk_queue_flag_clear(unsigned int flag, struct request_queue *q); 639 640 #define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags) 641 #define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags) 642 #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags) 643 #define blk_queue_noxmerges(q) \ 644 test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags) 645 #define blk_queue_nonrot(q) (!((q)->limits.features & BLK_FEAT_ROTATIONAL)) 646 #define blk_queue_io_stat(q) ((q)->limits.features & BLK_FEAT_IO_STAT) 647 #define blk_queue_passthrough_stat(q) \ 648 ((q)->limits.flags & BLK_FLAG_IOSTATS_PASSTHROUGH) 649 #define blk_queue_dax(q) ((q)->limits.features & BLK_FEAT_DAX) 650 #define blk_queue_pci_p2pdma(q) ((q)->limits.features & BLK_FEAT_PCI_P2PDMA) 651 #ifdef CONFIG_BLK_RQ_ALLOC_TIME 652 #define blk_queue_rq_alloc_time(q) \ 653 test_bit(QUEUE_FLAG_RQ_ALLOC_TIME, &(q)->queue_flags) 654 #else 655 #define blk_queue_rq_alloc_time(q) false 656 #endif 657 658 #define blk_noretry_request(rq) \ 659 ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \ 660 REQ_FAILFAST_DRIVER)) 661 #define blk_queue_quiesced(q) test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags) 662 #define blk_queue_pm_only(q) atomic_read(&(q)->pm_only) 663 #define blk_queue_registered(q) test_bit(QUEUE_FLAG_REGISTERED, &(q)->queue_flags) 664 #define blk_queue_sq_sched(q) test_bit(QUEUE_FLAG_SQ_SCHED, &(q)->queue_flags) 665 #define blk_queue_skip_tagset_quiesce(q) \ 666 ((q)->limits.features & BLK_FEAT_SKIP_TAGSET_QUIESCE) 667 668 extern void blk_set_pm_only(struct request_queue *q); 669 extern void blk_clear_pm_only(struct request_queue *q); 670 671 #define list_entry_rq(ptr) list_entry((ptr), struct request, queuelist) 672 673 #define dma_map_bvec(dev, bv, dir, attrs) \ 674 dma_map_page_attrs(dev, (bv)->bv_page, (bv)->bv_offset, (bv)->bv_len, \ 675 (dir), (attrs)) 676 677 static inline bool queue_is_mq(struct request_queue *q) 678 { 679 return q->mq_ops; 680 } 681 682 #ifdef CONFIG_PM 683 static inline enum rpm_status queue_rpm_status(struct request_queue *q) 684 { 685 return q->rpm_status; 686 } 687 #else 688 static inline enum rpm_status queue_rpm_status(struct request_queue *q) 689 { 690 return RPM_ACTIVE; 691 } 692 #endif 693 694 static inline bool blk_queue_is_zoned(struct request_queue *q) 695 { 696 return IS_ENABLED(CONFIG_BLK_DEV_ZONED) && 697 (q->limits.features & BLK_FEAT_ZONED); 698 } 699 700 #ifdef CONFIG_BLK_DEV_ZONED 701 static inline unsigned int disk_nr_zones(struct gendisk *disk) 702 { 703 return disk->nr_zones; 704 } 705 bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs); 706 #else /* CONFIG_BLK_DEV_ZONED */ 707 static inline unsigned int disk_nr_zones(struct gendisk *disk) 708 { 709 return 0; 710 } 711 static inline bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs) 712 { 713 return false; 714 } 715 #endif /* CONFIG_BLK_DEV_ZONED */ 716 717 static inline unsigned int disk_zone_no(struct gendisk *disk, sector_t sector) 718 { 719 if (!blk_queue_is_zoned(disk->queue)) 720 return 0; 721 return sector >> ilog2(disk->queue->limits.chunk_sectors); 722 } 723 724 static inline unsigned int bdev_nr_zones(struct block_device *bdev) 725 { 726 return disk_nr_zones(bdev->bd_disk); 727 } 728 729 static inline unsigned int bdev_max_open_zones(struct block_device *bdev) 730 { 731 return bdev->bd_disk->queue->limits.max_open_zones; 732 } 733 734 static inline unsigned int bdev_max_active_zones(struct block_device *bdev) 735 { 736 return bdev->bd_disk->queue->limits.max_active_zones; 737 } 738 739 static inline unsigned int blk_queue_depth(struct request_queue *q) 740 { 741 if (q->queue_depth) 742 return q->queue_depth; 743 744 return q->nr_requests; 745 } 746 747 /* 748 * default timeout for SG_IO if none specified 749 */ 750 #define BLK_DEFAULT_SG_TIMEOUT (60 * HZ) 751 #define BLK_MIN_SG_TIMEOUT (7 * HZ) 752 753 /* This should not be used directly - use rq_for_each_segment */ 754 #define for_each_bio(_bio) \ 755 for (; _bio; _bio = _bio->bi_next) 756 757 int __must_check add_disk_fwnode(struct device *parent, struct gendisk *disk, 758 const struct attribute_group **groups, 759 struct fwnode_handle *fwnode); 760 int __must_check device_add_disk(struct device *parent, struct gendisk *disk, 761 const struct attribute_group **groups); 762 static inline int __must_check add_disk(struct gendisk *disk) 763 { 764 return device_add_disk(NULL, disk, NULL); 765 } 766 void del_gendisk(struct gendisk *gp); 767 void invalidate_disk(struct gendisk *disk); 768 void set_disk_ro(struct gendisk *disk, bool read_only); 769 void disk_uevent(struct gendisk *disk, enum kobject_action action); 770 771 static inline u8 bdev_partno(const struct block_device *bdev) 772 { 773 return atomic_read(&bdev->__bd_flags) & BD_PARTNO; 774 } 775 776 static inline bool bdev_test_flag(const struct block_device *bdev, unsigned flag) 777 { 778 return atomic_read(&bdev->__bd_flags) & flag; 779 } 780 781 static inline void bdev_set_flag(struct block_device *bdev, unsigned flag) 782 { 783 atomic_or(flag, &bdev->__bd_flags); 784 } 785 786 static inline void bdev_clear_flag(struct block_device *bdev, unsigned flag) 787 { 788 atomic_andnot(flag, &bdev->__bd_flags); 789 } 790 791 static inline bool get_disk_ro(struct gendisk *disk) 792 { 793 return bdev_test_flag(disk->part0, BD_READ_ONLY) || 794 test_bit(GD_READ_ONLY, &disk->state); 795 } 796 797 static inline bool bdev_read_only(struct block_device *bdev) 798 { 799 return bdev_test_flag(bdev, BD_READ_ONLY) || get_disk_ro(bdev->bd_disk); 800 } 801 802 bool set_capacity_and_notify(struct gendisk *disk, sector_t size); 803 void disk_force_media_change(struct gendisk *disk); 804 void bdev_mark_dead(struct block_device *bdev, bool surprise); 805 806 void add_disk_randomness(struct gendisk *disk) __latent_entropy; 807 void rand_initialize_disk(struct gendisk *disk); 808 809 static inline sector_t get_start_sect(struct block_device *bdev) 810 { 811 return bdev->bd_start_sect; 812 } 813 814 static inline sector_t bdev_nr_sectors(struct block_device *bdev) 815 { 816 return bdev->bd_nr_sectors; 817 } 818 819 static inline loff_t bdev_nr_bytes(struct block_device *bdev) 820 { 821 return (loff_t)bdev_nr_sectors(bdev) << SECTOR_SHIFT; 822 } 823 824 static inline sector_t get_capacity(struct gendisk *disk) 825 { 826 return bdev_nr_sectors(disk->part0); 827 } 828 829 static inline u64 sb_bdev_nr_blocks(struct super_block *sb) 830 { 831 return bdev_nr_sectors(sb->s_bdev) >> 832 (sb->s_blocksize_bits - SECTOR_SHIFT); 833 } 834 835 int bdev_disk_changed(struct gendisk *disk, bool invalidate); 836 837 void put_disk(struct gendisk *disk); 838 struct gendisk *__blk_alloc_disk(struct queue_limits *lim, int node, 839 struct lock_class_key *lkclass); 840 841 /** 842 * blk_alloc_disk - allocate a gendisk structure 843 * @lim: queue limits to be used for this disk. 844 * @node_id: numa node to allocate on 845 * 846 * Allocate and pre-initialize a gendisk structure for use with BIO based 847 * drivers. 848 * 849 * Returns an ERR_PTR on error, else the allocated disk. 850 * 851 * Context: can sleep 852 */ 853 #define blk_alloc_disk(lim, node_id) \ 854 ({ \ 855 static struct lock_class_key __key; \ 856 \ 857 __blk_alloc_disk(lim, node_id, &__key); \ 858 }) 859 860 int __register_blkdev(unsigned int major, const char *name, 861 void (*probe)(dev_t devt)); 862 #define register_blkdev(major, name) \ 863 __register_blkdev(major, name, NULL) 864 void unregister_blkdev(unsigned int major, const char *name); 865 866 bool disk_check_media_change(struct gendisk *disk); 867 void set_capacity(struct gendisk *disk, sector_t size); 868 869 #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED 870 int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk); 871 void bd_unlink_disk_holder(struct block_device *bdev, struct gendisk *disk); 872 #else 873 static inline int bd_link_disk_holder(struct block_device *bdev, 874 struct gendisk *disk) 875 { 876 return 0; 877 } 878 static inline void bd_unlink_disk_holder(struct block_device *bdev, 879 struct gendisk *disk) 880 { 881 } 882 #endif /* CONFIG_BLOCK_HOLDER_DEPRECATED */ 883 884 dev_t part_devt(struct gendisk *disk, u8 partno); 885 void inc_diskseq(struct gendisk *disk); 886 void blk_request_module(dev_t devt); 887 888 extern int blk_register_queue(struct gendisk *disk); 889 extern void blk_unregister_queue(struct gendisk *disk); 890 void submit_bio_noacct(struct bio *bio); 891 struct bio *bio_split_to_limits(struct bio *bio); 892 893 extern int blk_lld_busy(struct request_queue *q); 894 extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags); 895 extern void blk_queue_exit(struct request_queue *q); 896 extern void blk_sync_queue(struct request_queue *q); 897 898 /* Helper to convert REQ_OP_XXX to its string format XXX */ 899 extern const char *blk_op_str(enum req_op op); 900 901 int blk_status_to_errno(blk_status_t status); 902 blk_status_t errno_to_blk_status(int errno); 903 const char *blk_status_to_str(blk_status_t status); 904 905 /* only poll the hardware once, don't continue until a completion was found */ 906 #define BLK_POLL_ONESHOT (1 << 0) 907 int bio_poll(struct bio *bio, struct io_comp_batch *iob, unsigned int flags); 908 int iocb_bio_iopoll(struct kiocb *kiocb, struct io_comp_batch *iob, 909 unsigned int flags); 910 911 static inline struct request_queue *bdev_get_queue(struct block_device *bdev) 912 { 913 return bdev->bd_queue; /* this is never NULL */ 914 } 915 916 /* Helper to convert BLK_ZONE_ZONE_XXX to its string format XXX */ 917 const char *blk_zone_cond_str(enum blk_zone_cond zone_cond); 918 919 static inline unsigned int bio_zone_no(struct bio *bio) 920 { 921 return disk_zone_no(bio->bi_bdev->bd_disk, bio->bi_iter.bi_sector); 922 } 923 924 static inline bool bio_straddles_zones(struct bio *bio) 925 { 926 return bio_sectors(bio) && 927 bio_zone_no(bio) != 928 disk_zone_no(bio->bi_bdev->bd_disk, bio_end_sector(bio) - 1); 929 } 930 931 /* 932 * Return how much within the boundary is left to be used for I/O at a given 933 * offset. 934 */ 935 static inline unsigned int blk_boundary_sectors_left(sector_t offset, 936 unsigned int boundary_sectors) 937 { 938 if (unlikely(!is_power_of_2(boundary_sectors))) 939 return boundary_sectors - sector_div(offset, boundary_sectors); 940 return boundary_sectors - (offset & (boundary_sectors - 1)); 941 } 942 943 /** 944 * queue_limits_start_update - start an atomic update of queue limits 945 * @q: queue to update 946 * 947 * This functions starts an atomic update of the queue limits. It takes a lock 948 * to prevent other updates and returns a snapshot of the current limits that 949 * the caller can modify. The caller must call queue_limits_commit_update() 950 * to finish the update. 951 * 952 * Context: process context. 953 */ 954 static inline struct queue_limits 955 queue_limits_start_update(struct request_queue *q) 956 { 957 mutex_lock(&q->limits_lock); 958 return q->limits; 959 } 960 int queue_limits_commit_update_frozen(struct request_queue *q, 961 struct queue_limits *lim); 962 int queue_limits_commit_update(struct request_queue *q, 963 struct queue_limits *lim); 964 int queue_limits_set(struct request_queue *q, struct queue_limits *lim); 965 int blk_validate_limits(struct queue_limits *lim); 966 967 /** 968 * queue_limits_cancel_update - cancel an atomic update of queue limits 969 * @q: queue to update 970 * 971 * This functions cancels an atomic update of the queue limits started by 972 * queue_limits_start_update() and should be used when an error occurs after 973 * starting update. 974 */ 975 static inline void queue_limits_cancel_update(struct request_queue *q) 976 { 977 mutex_unlock(&q->limits_lock); 978 } 979 980 /* 981 * These helpers are for drivers that have sloppy feature negotiation and might 982 * have to disable DISCARD, WRITE_ZEROES or SECURE_DISCARD from the I/O 983 * completion handler when the device returned an indicator that the respective 984 * feature is not actually supported. They are racy and the driver needs to 985 * cope with that. Try to avoid this scheme if you can. 986 */ 987 static inline void blk_queue_disable_discard(struct request_queue *q) 988 { 989 q->limits.max_discard_sectors = 0; 990 } 991 992 static inline void blk_queue_disable_secure_erase(struct request_queue *q) 993 { 994 q->limits.max_secure_erase_sectors = 0; 995 } 996 997 static inline void blk_queue_disable_write_zeroes(struct request_queue *q) 998 { 999 q->limits.max_write_zeroes_sectors = 0; 1000 } 1001 1002 /* 1003 * Access functions for manipulating queue properties 1004 */ 1005 extern void blk_set_queue_depth(struct request_queue *q, unsigned int depth); 1006 extern void blk_set_stacking_limits(struct queue_limits *lim); 1007 extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, 1008 sector_t offset); 1009 void queue_limits_stack_bdev(struct queue_limits *t, struct block_device *bdev, 1010 sector_t offset, const char *pfx); 1011 extern void blk_queue_rq_timeout(struct request_queue *, unsigned int); 1012 1013 struct blk_independent_access_ranges * 1014 disk_alloc_independent_access_ranges(struct gendisk *disk, int nr_ia_ranges); 1015 void disk_set_independent_access_ranges(struct gendisk *disk, 1016 struct blk_independent_access_ranges *iars); 1017 1018 bool __must_check blk_get_queue(struct request_queue *); 1019 extern void blk_put_queue(struct request_queue *); 1020 1021 void blk_mark_disk_dead(struct gendisk *disk); 1022 1023 struct rq_list { 1024 struct request *head; 1025 struct request *tail; 1026 }; 1027 1028 #ifdef CONFIG_BLOCK 1029 /* 1030 * blk_plug permits building a queue of related requests by holding the I/O 1031 * fragments for a short period. This allows merging of sequential requests 1032 * into single larger request. As the requests are moved from a per-task list to 1033 * the device's request_queue in a batch, this results in improved scalability 1034 * as the lock contention for request_queue lock is reduced. 1035 * 1036 * It is ok not to disable preemption when adding the request to the plug list 1037 * or when attempting a merge. For details, please see schedule() where 1038 * blk_flush_plug() is called. 1039 */ 1040 struct blk_plug { 1041 struct rq_list mq_list; /* blk-mq requests */ 1042 1043 /* if ios_left is > 1, we can batch tag/rq allocations */ 1044 struct rq_list cached_rqs; 1045 u64 cur_ktime; 1046 unsigned short nr_ios; 1047 1048 unsigned short rq_count; 1049 1050 bool multiple_queues; 1051 bool has_elevator; 1052 1053 struct list_head cb_list; /* md requires an unplug callback */ 1054 }; 1055 1056 struct blk_plug_cb; 1057 typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool); 1058 struct blk_plug_cb { 1059 struct list_head list; 1060 blk_plug_cb_fn callback; 1061 void *data; 1062 }; 1063 extern struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, 1064 void *data, int size); 1065 extern void blk_start_plug(struct blk_plug *); 1066 extern void blk_start_plug_nr_ios(struct blk_plug *, unsigned short); 1067 extern void blk_finish_plug(struct blk_plug *); 1068 1069 void __blk_flush_plug(struct blk_plug *plug, bool from_schedule); 1070 static inline void blk_flush_plug(struct blk_plug *plug, bool async) 1071 { 1072 if (plug) 1073 __blk_flush_plug(plug, async); 1074 } 1075 1076 /* 1077 * tsk == current here 1078 */ 1079 static inline void blk_plug_invalidate_ts(struct task_struct *tsk) 1080 { 1081 struct blk_plug *plug = tsk->plug; 1082 1083 if (plug) 1084 plug->cur_ktime = 0; 1085 current->flags &= ~PF_BLOCK_TS; 1086 } 1087 1088 int blkdev_issue_flush(struct block_device *bdev); 1089 long nr_blockdev_pages(void); 1090 #else /* CONFIG_BLOCK */ 1091 struct blk_plug { 1092 }; 1093 1094 static inline void blk_start_plug_nr_ios(struct blk_plug *plug, 1095 unsigned short nr_ios) 1096 { 1097 } 1098 1099 static inline void blk_start_plug(struct blk_plug *plug) 1100 { 1101 } 1102 1103 static inline void blk_finish_plug(struct blk_plug *plug) 1104 { 1105 } 1106 1107 static inline void blk_flush_plug(struct blk_plug *plug, bool async) 1108 { 1109 } 1110 1111 static inline void blk_plug_invalidate_ts(struct task_struct *tsk) 1112 { 1113 } 1114 1115 static inline int blkdev_issue_flush(struct block_device *bdev) 1116 { 1117 return 0; 1118 } 1119 1120 static inline long nr_blockdev_pages(void) 1121 { 1122 return 0; 1123 } 1124 #endif /* CONFIG_BLOCK */ 1125 1126 extern void blk_io_schedule(void); 1127 1128 int blkdev_issue_discard(struct block_device *bdev, sector_t sector, 1129 sector_t nr_sects, gfp_t gfp_mask); 1130 int __blkdev_issue_discard(struct block_device *bdev, sector_t sector, 1131 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop); 1132 int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector, 1133 sector_t nr_sects, gfp_t gfp); 1134 1135 #define BLKDEV_ZERO_NOUNMAP (1 << 0) /* do not free blocks */ 1136 #define BLKDEV_ZERO_NOFALLBACK (1 << 1) /* don't write explicit zeroes */ 1137 #define BLKDEV_ZERO_KILLABLE (1 << 2) /* interruptible by fatal signals */ 1138 1139 extern int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 1140 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop, 1141 unsigned flags); 1142 extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 1143 sector_t nr_sects, gfp_t gfp_mask, unsigned flags); 1144 1145 static inline int sb_issue_discard(struct super_block *sb, sector_t block, 1146 sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags) 1147 { 1148 return blkdev_issue_discard(sb->s_bdev, 1149 block << (sb->s_blocksize_bits - 1150 SECTOR_SHIFT), 1151 nr_blocks << (sb->s_blocksize_bits - 1152 SECTOR_SHIFT), 1153 gfp_mask); 1154 } 1155 static inline int sb_issue_zeroout(struct super_block *sb, sector_t block, 1156 sector_t nr_blocks, gfp_t gfp_mask) 1157 { 1158 return blkdev_issue_zeroout(sb->s_bdev, 1159 block << (sb->s_blocksize_bits - 1160 SECTOR_SHIFT), 1161 nr_blocks << (sb->s_blocksize_bits - 1162 SECTOR_SHIFT), 1163 gfp_mask, 0); 1164 } 1165 1166 static inline bool bdev_is_partition(struct block_device *bdev) 1167 { 1168 return bdev_partno(bdev) != 0; 1169 } 1170 1171 enum blk_default_limits { 1172 BLK_MAX_SEGMENTS = 128, 1173 BLK_SAFE_MAX_SECTORS = 255, 1174 BLK_MAX_SEGMENT_SIZE = 65536, 1175 BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL, 1176 }; 1177 1178 /* 1179 * Default upper limit for the software max_sectors limit used for 1180 * regular file system I/O. This can be increased through sysfs. 1181 * 1182 * Not to be confused with the max_hw_sector limit that is entirely 1183 * controlled by the driver, usually based on hardware limits. 1184 */ 1185 #define BLK_DEF_MAX_SECTORS_CAP 2560u 1186 1187 static inline struct queue_limits *bdev_limits(struct block_device *bdev) 1188 { 1189 return &bdev_get_queue(bdev)->limits; 1190 } 1191 1192 static inline unsigned long queue_segment_boundary(const struct request_queue *q) 1193 { 1194 return q->limits.seg_boundary_mask; 1195 } 1196 1197 static inline unsigned long queue_virt_boundary(const struct request_queue *q) 1198 { 1199 return q->limits.virt_boundary_mask; 1200 } 1201 1202 static inline unsigned int queue_max_sectors(const struct request_queue *q) 1203 { 1204 return q->limits.max_sectors; 1205 } 1206 1207 static inline unsigned int queue_max_bytes(struct request_queue *q) 1208 { 1209 return min_t(unsigned int, queue_max_sectors(q), INT_MAX >> 9) << 9; 1210 } 1211 1212 static inline unsigned int queue_max_hw_sectors(const struct request_queue *q) 1213 { 1214 return q->limits.max_hw_sectors; 1215 } 1216 1217 static inline unsigned short queue_max_segments(const struct request_queue *q) 1218 { 1219 return q->limits.max_segments; 1220 } 1221 1222 static inline unsigned short queue_max_discard_segments(const struct request_queue *q) 1223 { 1224 return q->limits.max_discard_segments; 1225 } 1226 1227 static inline unsigned int queue_max_segment_size(const struct request_queue *q) 1228 { 1229 return q->limits.max_segment_size; 1230 } 1231 1232 static inline bool queue_emulates_zone_append(struct request_queue *q) 1233 { 1234 return blk_queue_is_zoned(q) && !q->limits.max_hw_zone_append_sectors; 1235 } 1236 1237 static inline bool bdev_emulates_zone_append(struct block_device *bdev) 1238 { 1239 return queue_emulates_zone_append(bdev_get_queue(bdev)); 1240 } 1241 1242 static inline unsigned int 1243 bdev_max_zone_append_sectors(struct block_device *bdev) 1244 { 1245 return bdev_limits(bdev)->max_zone_append_sectors; 1246 } 1247 1248 static inline unsigned int bdev_max_segments(struct block_device *bdev) 1249 { 1250 return queue_max_segments(bdev_get_queue(bdev)); 1251 } 1252 1253 static inline unsigned queue_logical_block_size(const struct request_queue *q) 1254 { 1255 return q->limits.logical_block_size; 1256 } 1257 1258 static inline unsigned int bdev_logical_block_size(struct block_device *bdev) 1259 { 1260 return queue_logical_block_size(bdev_get_queue(bdev)); 1261 } 1262 1263 static inline unsigned int queue_physical_block_size(const struct request_queue *q) 1264 { 1265 return q->limits.physical_block_size; 1266 } 1267 1268 static inline unsigned int bdev_physical_block_size(struct block_device *bdev) 1269 { 1270 return queue_physical_block_size(bdev_get_queue(bdev)); 1271 } 1272 1273 static inline unsigned int queue_io_min(const struct request_queue *q) 1274 { 1275 return q->limits.io_min; 1276 } 1277 1278 static inline unsigned int bdev_io_min(struct block_device *bdev) 1279 { 1280 return queue_io_min(bdev_get_queue(bdev)); 1281 } 1282 1283 static inline unsigned int queue_io_opt(const struct request_queue *q) 1284 { 1285 return q->limits.io_opt; 1286 } 1287 1288 static inline unsigned int bdev_io_opt(struct block_device *bdev) 1289 { 1290 return queue_io_opt(bdev_get_queue(bdev)); 1291 } 1292 1293 static inline unsigned int 1294 queue_zone_write_granularity(const struct request_queue *q) 1295 { 1296 return q->limits.zone_write_granularity; 1297 } 1298 1299 static inline unsigned int 1300 bdev_zone_write_granularity(struct block_device *bdev) 1301 { 1302 return queue_zone_write_granularity(bdev_get_queue(bdev)); 1303 } 1304 1305 int bdev_alignment_offset(struct block_device *bdev); 1306 unsigned int bdev_discard_alignment(struct block_device *bdev); 1307 1308 static inline unsigned int bdev_max_discard_sectors(struct block_device *bdev) 1309 { 1310 return bdev_limits(bdev)->max_discard_sectors; 1311 } 1312 1313 static inline unsigned int bdev_discard_granularity(struct block_device *bdev) 1314 { 1315 return bdev_limits(bdev)->discard_granularity; 1316 } 1317 1318 static inline unsigned int 1319 bdev_max_secure_erase_sectors(struct block_device *bdev) 1320 { 1321 return bdev_limits(bdev)->max_secure_erase_sectors; 1322 } 1323 1324 static inline unsigned int bdev_write_zeroes_sectors(struct block_device *bdev) 1325 { 1326 return bdev_limits(bdev)->max_write_zeroes_sectors; 1327 } 1328 1329 static inline bool bdev_nonrot(struct block_device *bdev) 1330 { 1331 return blk_queue_nonrot(bdev_get_queue(bdev)); 1332 } 1333 1334 static inline bool bdev_synchronous(struct block_device *bdev) 1335 { 1336 return bdev->bd_disk->queue->limits.features & BLK_FEAT_SYNCHRONOUS; 1337 } 1338 1339 static inline bool bdev_stable_writes(struct block_device *bdev) 1340 { 1341 struct request_queue *q = bdev_get_queue(bdev); 1342 1343 if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) && 1344 q->limits.integrity.csum_type != BLK_INTEGRITY_CSUM_NONE) 1345 return true; 1346 return q->limits.features & BLK_FEAT_STABLE_WRITES; 1347 } 1348 1349 static inline bool blk_queue_write_cache(struct request_queue *q) 1350 { 1351 return (q->limits.features & BLK_FEAT_WRITE_CACHE) && 1352 !(q->limits.flags & BLK_FLAG_WRITE_CACHE_DISABLED); 1353 } 1354 1355 static inline bool bdev_write_cache(struct block_device *bdev) 1356 { 1357 return blk_queue_write_cache(bdev_get_queue(bdev)); 1358 } 1359 1360 static inline bool bdev_fua(struct block_device *bdev) 1361 { 1362 return bdev_limits(bdev)->features & BLK_FEAT_FUA; 1363 } 1364 1365 static inline bool bdev_nowait(struct block_device *bdev) 1366 { 1367 return bdev->bd_disk->queue->limits.features & BLK_FEAT_NOWAIT; 1368 } 1369 1370 static inline bool bdev_is_zoned(struct block_device *bdev) 1371 { 1372 return blk_queue_is_zoned(bdev_get_queue(bdev)); 1373 } 1374 1375 static inline unsigned int bdev_zone_no(struct block_device *bdev, sector_t sec) 1376 { 1377 return disk_zone_no(bdev->bd_disk, sec); 1378 } 1379 1380 static inline sector_t bdev_zone_sectors(struct block_device *bdev) 1381 { 1382 struct request_queue *q = bdev_get_queue(bdev); 1383 1384 if (!blk_queue_is_zoned(q)) 1385 return 0; 1386 return q->limits.chunk_sectors; 1387 } 1388 1389 static inline sector_t bdev_offset_from_zone_start(struct block_device *bdev, 1390 sector_t sector) 1391 { 1392 return sector & (bdev_zone_sectors(bdev) - 1); 1393 } 1394 1395 static inline sector_t bio_offset_from_zone_start(struct bio *bio) 1396 { 1397 return bdev_offset_from_zone_start(bio->bi_bdev, 1398 bio->bi_iter.bi_sector); 1399 } 1400 1401 static inline bool bdev_is_zone_start(struct block_device *bdev, 1402 sector_t sector) 1403 { 1404 return bdev_offset_from_zone_start(bdev, sector) == 0; 1405 } 1406 1407 /** 1408 * bdev_zone_is_seq - check if a sector belongs to a sequential write zone 1409 * @bdev: block device to check 1410 * @sector: sector number 1411 * 1412 * Check if @sector on @bdev is contained in a sequential write required zone. 1413 */ 1414 static inline bool bdev_zone_is_seq(struct block_device *bdev, sector_t sector) 1415 { 1416 bool is_seq = false; 1417 1418 #if IS_ENABLED(CONFIG_BLK_DEV_ZONED) 1419 if (bdev_is_zoned(bdev)) { 1420 struct gendisk *disk = bdev->bd_disk; 1421 unsigned long *bitmap; 1422 1423 rcu_read_lock(); 1424 bitmap = rcu_dereference(disk->conv_zones_bitmap); 1425 is_seq = !bitmap || 1426 !test_bit(disk_zone_no(disk, sector), bitmap); 1427 rcu_read_unlock(); 1428 } 1429 #endif 1430 1431 return is_seq; 1432 } 1433 1434 int blk_zone_issue_zeroout(struct block_device *bdev, sector_t sector, 1435 sector_t nr_sects, gfp_t gfp_mask); 1436 1437 static inline unsigned int queue_dma_alignment(const struct request_queue *q) 1438 { 1439 return q->limits.dma_alignment; 1440 } 1441 1442 static inline unsigned int 1443 queue_atomic_write_unit_max_bytes(const struct request_queue *q) 1444 { 1445 return q->limits.atomic_write_unit_max; 1446 } 1447 1448 static inline unsigned int 1449 queue_atomic_write_unit_min_bytes(const struct request_queue *q) 1450 { 1451 return q->limits.atomic_write_unit_min; 1452 } 1453 1454 static inline unsigned int 1455 queue_atomic_write_boundary_bytes(const struct request_queue *q) 1456 { 1457 return q->limits.atomic_write_boundary_sectors << SECTOR_SHIFT; 1458 } 1459 1460 static inline unsigned int 1461 queue_atomic_write_max_bytes(const struct request_queue *q) 1462 { 1463 return q->limits.atomic_write_max_sectors << SECTOR_SHIFT; 1464 } 1465 1466 static inline unsigned int bdev_dma_alignment(struct block_device *bdev) 1467 { 1468 return queue_dma_alignment(bdev_get_queue(bdev)); 1469 } 1470 1471 static inline bool bdev_iter_is_aligned(struct block_device *bdev, 1472 struct iov_iter *iter) 1473 { 1474 return iov_iter_is_aligned(iter, bdev_dma_alignment(bdev), 1475 bdev_logical_block_size(bdev) - 1); 1476 } 1477 1478 static inline unsigned int 1479 blk_lim_dma_alignment_and_pad(struct queue_limits *lim) 1480 { 1481 return lim->dma_alignment | lim->dma_pad_mask; 1482 } 1483 1484 static inline bool blk_rq_aligned(struct request_queue *q, unsigned long addr, 1485 unsigned int len) 1486 { 1487 unsigned int alignment = blk_lim_dma_alignment_and_pad(&q->limits); 1488 1489 return !(addr & alignment) && !(len & alignment); 1490 } 1491 1492 /* assumes size > 256 */ 1493 static inline unsigned int blksize_bits(unsigned int size) 1494 { 1495 return order_base_2(size >> SECTOR_SHIFT) + SECTOR_SHIFT; 1496 } 1497 1498 int kblockd_schedule_work(struct work_struct *work); 1499 int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay); 1500 1501 #define MODULE_ALIAS_BLOCKDEV(major,minor) \ 1502 MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor)) 1503 #define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \ 1504 MODULE_ALIAS("block-major-" __stringify(major) "-*") 1505 1506 #ifdef CONFIG_BLK_INLINE_ENCRYPTION 1507 1508 bool blk_crypto_register(struct blk_crypto_profile *profile, 1509 struct request_queue *q); 1510 1511 #else /* CONFIG_BLK_INLINE_ENCRYPTION */ 1512 1513 static inline bool blk_crypto_register(struct blk_crypto_profile *profile, 1514 struct request_queue *q) 1515 { 1516 return true; 1517 } 1518 1519 #endif /* CONFIG_BLK_INLINE_ENCRYPTION */ 1520 1521 enum blk_unique_id { 1522 /* these match the Designator Types specified in SPC */ 1523 BLK_UID_T10 = 1, 1524 BLK_UID_EUI64 = 2, 1525 BLK_UID_NAA = 3, 1526 }; 1527 1528 struct block_device_operations { 1529 void (*submit_bio)(struct bio *bio); 1530 int (*poll_bio)(struct bio *bio, struct io_comp_batch *iob, 1531 unsigned int flags); 1532 int (*open)(struct gendisk *disk, blk_mode_t mode); 1533 void (*release)(struct gendisk *disk); 1534 int (*ioctl)(struct block_device *bdev, blk_mode_t mode, 1535 unsigned cmd, unsigned long arg); 1536 int (*compat_ioctl)(struct block_device *bdev, blk_mode_t mode, 1537 unsigned cmd, unsigned long arg); 1538 unsigned int (*check_events) (struct gendisk *disk, 1539 unsigned int clearing); 1540 void (*unlock_native_capacity) (struct gendisk *); 1541 int (*getgeo)(struct block_device *, struct hd_geometry *); 1542 int (*set_read_only)(struct block_device *bdev, bool ro); 1543 void (*free_disk)(struct gendisk *disk); 1544 /* this callback is with swap_lock and sometimes page table lock held */ 1545 void (*swap_slot_free_notify) (struct block_device *, unsigned long); 1546 int (*report_zones)(struct gendisk *, sector_t sector, 1547 unsigned int nr_zones, report_zones_cb cb, void *data); 1548 char *(*devnode)(struct gendisk *disk, umode_t *mode); 1549 /* returns the length of the identifier or a negative errno: */ 1550 int (*get_unique_id)(struct gendisk *disk, u8 id[16], 1551 enum blk_unique_id id_type); 1552 struct module *owner; 1553 const struct pr_ops *pr_ops; 1554 1555 /* 1556 * Special callback for probing GPT entry at a given sector. 1557 * Needed by Android devices, used by GPT scanner and MMC blk 1558 * driver. 1559 */ 1560 int (*alternative_gpt_sector)(struct gendisk *disk, sector_t *sector); 1561 }; 1562 1563 #ifdef CONFIG_COMPAT 1564 extern int blkdev_compat_ptr_ioctl(struct block_device *, blk_mode_t, 1565 unsigned int, unsigned long); 1566 #else 1567 #define blkdev_compat_ptr_ioctl NULL 1568 #endif 1569 1570 static inline void blk_wake_io_task(struct task_struct *waiter) 1571 { 1572 /* 1573 * If we're polling, the task itself is doing the completions. For 1574 * that case, we don't need to signal a wakeup, it's enough to just 1575 * mark us as RUNNING. 1576 */ 1577 if (waiter == current) 1578 __set_current_state(TASK_RUNNING); 1579 else 1580 wake_up_process(waiter); 1581 } 1582 1583 unsigned long bdev_start_io_acct(struct block_device *bdev, enum req_op op, 1584 unsigned long start_time); 1585 void bdev_end_io_acct(struct block_device *bdev, enum req_op op, 1586 unsigned int sectors, unsigned long start_time); 1587 1588 unsigned long bio_start_io_acct(struct bio *bio); 1589 void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time, 1590 struct block_device *orig_bdev); 1591 1592 /** 1593 * bio_end_io_acct - end I/O accounting for bio based drivers 1594 * @bio: bio to end account for 1595 * @start_time: start time returned by bio_start_io_acct() 1596 */ 1597 static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time) 1598 { 1599 return bio_end_io_acct_remapped(bio, start_time, bio->bi_bdev); 1600 } 1601 1602 int set_blocksize(struct file *file, int size); 1603 1604 int lookup_bdev(const char *pathname, dev_t *dev); 1605 1606 void blkdev_show(struct seq_file *seqf, off_t offset); 1607 1608 #define BDEVNAME_SIZE 32 /* Largest string for a blockdev identifier */ 1609 #define BDEVT_SIZE 10 /* Largest string for MAJ:MIN for blkdev */ 1610 #ifdef CONFIG_BLOCK 1611 #define BLKDEV_MAJOR_MAX 512 1612 #else 1613 #define BLKDEV_MAJOR_MAX 0 1614 #endif 1615 1616 struct blk_holder_ops { 1617 void (*mark_dead)(struct block_device *bdev, bool surprise); 1618 1619 /* 1620 * Sync the file system mounted on the block device. 1621 */ 1622 void (*sync)(struct block_device *bdev); 1623 1624 /* 1625 * Freeze the file system mounted on the block device. 1626 */ 1627 int (*freeze)(struct block_device *bdev); 1628 1629 /* 1630 * Thaw the file system mounted on the block device. 1631 */ 1632 int (*thaw)(struct block_device *bdev); 1633 }; 1634 1635 /* 1636 * For filesystems using @fs_holder_ops, the @holder argument passed to 1637 * helpers used to open and claim block devices via 1638 * bd_prepare_to_claim() must point to a superblock. 1639 */ 1640 extern const struct blk_holder_ops fs_holder_ops; 1641 1642 /* 1643 * Return the correct open flags for blkdev_get_by_* for super block flags 1644 * as stored in sb->s_flags. 1645 */ 1646 #define sb_open_mode(flags) \ 1647 (BLK_OPEN_READ | BLK_OPEN_RESTRICT_WRITES | \ 1648 (((flags) & SB_RDONLY) ? 0 : BLK_OPEN_WRITE)) 1649 1650 struct file *bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder, 1651 const struct blk_holder_ops *hops); 1652 struct file *bdev_file_open_by_path(const char *path, blk_mode_t mode, 1653 void *holder, const struct blk_holder_ops *hops); 1654 int bd_prepare_to_claim(struct block_device *bdev, void *holder, 1655 const struct blk_holder_ops *hops); 1656 void bd_abort_claiming(struct block_device *bdev, void *holder); 1657 1658 /* just for blk-cgroup, don't use elsewhere */ 1659 struct block_device *blkdev_get_no_open(dev_t dev); 1660 void blkdev_put_no_open(struct block_device *bdev); 1661 1662 struct block_device *I_BDEV(struct inode *inode); 1663 struct block_device *file_bdev(struct file *bdev_file); 1664 bool disk_live(struct gendisk *disk); 1665 unsigned int block_size(struct block_device *bdev); 1666 1667 #ifdef CONFIG_BLOCK 1668 void invalidate_bdev(struct block_device *bdev); 1669 int sync_blockdev(struct block_device *bdev); 1670 int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend); 1671 int sync_blockdev_nowait(struct block_device *bdev); 1672 void sync_bdevs(bool wait); 1673 void bdev_statx(struct path *, struct kstat *, u32); 1674 void printk_all_partitions(void); 1675 int __init early_lookup_bdev(const char *pathname, dev_t *dev); 1676 #else 1677 static inline void invalidate_bdev(struct block_device *bdev) 1678 { 1679 } 1680 static inline int sync_blockdev(struct block_device *bdev) 1681 { 1682 return 0; 1683 } 1684 static inline int sync_blockdev_nowait(struct block_device *bdev) 1685 { 1686 return 0; 1687 } 1688 static inline void sync_bdevs(bool wait) 1689 { 1690 } 1691 static inline void bdev_statx(struct path *path, struct kstat *stat, 1692 u32 request_mask) 1693 { 1694 } 1695 static inline void printk_all_partitions(void) 1696 { 1697 } 1698 static inline int early_lookup_bdev(const char *pathname, dev_t *dev) 1699 { 1700 return -EINVAL; 1701 } 1702 #endif /* CONFIG_BLOCK */ 1703 1704 int bdev_freeze(struct block_device *bdev); 1705 int bdev_thaw(struct block_device *bdev); 1706 void bdev_fput(struct file *bdev_file); 1707 1708 struct io_comp_batch { 1709 struct rq_list req_list; 1710 bool need_ts; 1711 void (*complete)(struct io_comp_batch *); 1712 }; 1713 1714 static inline bool blk_atomic_write_start_sect_aligned(sector_t sector, 1715 struct queue_limits *limits) 1716 { 1717 unsigned int alignment = max(limits->atomic_write_hw_unit_min, 1718 limits->atomic_write_hw_boundary); 1719 1720 return IS_ALIGNED(sector, alignment >> SECTOR_SHIFT); 1721 } 1722 1723 static inline bool bdev_can_atomic_write(struct block_device *bdev) 1724 { 1725 struct request_queue *bd_queue = bdev->bd_queue; 1726 struct queue_limits *limits = &bd_queue->limits; 1727 1728 if (!limits->atomic_write_unit_min) 1729 return false; 1730 1731 if (bdev_is_partition(bdev)) 1732 return blk_atomic_write_start_sect_aligned(bdev->bd_start_sect, 1733 limits); 1734 1735 return true; 1736 } 1737 1738 static inline unsigned int 1739 bdev_atomic_write_unit_min_bytes(struct block_device *bdev) 1740 { 1741 if (!bdev_can_atomic_write(bdev)) 1742 return 0; 1743 return queue_atomic_write_unit_min_bytes(bdev_get_queue(bdev)); 1744 } 1745 1746 static inline unsigned int 1747 bdev_atomic_write_unit_max_bytes(struct block_device *bdev) 1748 { 1749 if (!bdev_can_atomic_write(bdev)) 1750 return 0; 1751 return queue_atomic_write_unit_max_bytes(bdev_get_queue(bdev)); 1752 } 1753 1754 #define DEFINE_IO_COMP_BATCH(name) struct io_comp_batch name = { } 1755 1756 #endif /* _LINUX_BLKDEV_H */ 1757