1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Portions Copyright (C) 1992 Drew Eckhardt 4 */ 5 #ifndef _LINUX_BLKDEV_H 6 #define _LINUX_BLKDEV_H 7 8 #include <linux/types.h> 9 #include <linux/blk_types.h> 10 #include <linux/device.h> 11 #include <linux/list.h> 12 #include <linux/llist.h> 13 #include <linux/minmax.h> 14 #include <linux/timer.h> 15 #include <linux/workqueue.h> 16 #include <linux/wait.h> 17 #include <linux/bio.h> 18 #include <linux/gfp.h> 19 #include <linux/kdev_t.h> 20 #include <linux/rcupdate.h> 21 #include <linux/percpu-refcount.h> 22 #include <linux/blkzoned.h> 23 #include <linux/sched.h> 24 #include <linux/sbitmap.h> 25 #include <linux/uuid.h> 26 #include <linux/xarray.h> 27 #include <linux/file.h> 28 #include <linux/lockdep.h> 29 30 struct module; 31 struct request_queue; 32 struct elevator_queue; 33 struct blk_trace; 34 struct request; 35 struct sg_io_hdr; 36 struct blkcg_gq; 37 struct blk_flush_queue; 38 struct kiocb; 39 struct pr_ops; 40 struct rq_qos; 41 struct blk_queue_stats; 42 struct blk_stat_callback; 43 struct blk_crypto_profile; 44 45 extern const struct device_type disk_type; 46 extern const struct device_type part_type; 47 extern const struct class block_class; 48 49 /* 50 * Maximum number of blkcg policies allowed to be registered concurrently. 51 * Defined here to simplify include dependency. 52 */ 53 #define BLKCG_MAX_POLS 6 54 55 #define DISK_MAX_PARTS 256 56 #define DISK_NAME_LEN 32 57 58 #define PARTITION_META_INFO_VOLNAMELTH 64 59 /* 60 * Enough for the string representation of any kind of UUID plus NULL. 61 * EFI UUID is 36 characters. MSDOS UUID is 11 characters. 62 */ 63 #define PARTITION_META_INFO_UUIDLTH (UUID_STRING_LEN + 1) 64 65 struct partition_meta_info { 66 char uuid[PARTITION_META_INFO_UUIDLTH]; 67 u8 volname[PARTITION_META_INFO_VOLNAMELTH]; 68 }; 69 70 /** 71 * DOC: genhd capability flags 72 * 73 * ``GENHD_FL_REMOVABLE``: indicates that the block device gives access to 74 * removable media. When set, the device remains present even when media is not 75 * inserted. Shall not be set for devices which are removed entirely when the 76 * media is removed. 77 * 78 * ``GENHD_FL_HIDDEN``: the block device is hidden; it doesn't produce events, 79 * doesn't appear in sysfs, and can't be opened from userspace or using 80 * blkdev_get*. Used for the underlying components of multipath devices. 81 * 82 * ``GENHD_FL_NO_PART``: partition support is disabled. The kernel will not 83 * scan for partitions from add_disk, and users can't add partitions manually. 84 * 85 */ 86 enum { 87 GENHD_FL_REMOVABLE = 1 << 0, 88 GENHD_FL_HIDDEN = 1 << 1, 89 GENHD_FL_NO_PART = 1 << 2, 90 }; 91 92 enum { 93 DISK_EVENT_MEDIA_CHANGE = 1 << 0, /* media changed */ 94 DISK_EVENT_EJECT_REQUEST = 1 << 1, /* eject requested */ 95 }; 96 97 enum { 98 /* Poll even if events_poll_msecs is unset */ 99 DISK_EVENT_FLAG_POLL = 1 << 0, 100 /* Forward events to udev */ 101 DISK_EVENT_FLAG_UEVENT = 1 << 1, 102 /* Block event polling when open for exclusive write */ 103 DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE = 1 << 2, 104 }; 105 106 struct disk_events; 107 struct badblocks; 108 109 enum blk_integrity_checksum { 110 BLK_INTEGRITY_CSUM_NONE = 0, 111 BLK_INTEGRITY_CSUM_IP = 1, 112 BLK_INTEGRITY_CSUM_CRC = 2, 113 BLK_INTEGRITY_CSUM_CRC64 = 3, 114 } __packed ; 115 116 struct blk_integrity { 117 unsigned char flags; 118 enum blk_integrity_checksum csum_type; 119 unsigned char tuple_size; 120 unsigned char pi_offset; 121 unsigned char interval_exp; 122 unsigned char tag_size; 123 }; 124 125 typedef unsigned int __bitwise blk_mode_t; 126 127 /* open for reading */ 128 #define BLK_OPEN_READ ((__force blk_mode_t)(1 << 0)) 129 /* open for writing */ 130 #define BLK_OPEN_WRITE ((__force blk_mode_t)(1 << 1)) 131 /* open exclusively (vs other exclusive openers */ 132 #define BLK_OPEN_EXCL ((__force blk_mode_t)(1 << 2)) 133 /* opened with O_NDELAY */ 134 #define BLK_OPEN_NDELAY ((__force blk_mode_t)(1 << 3)) 135 /* open for "writes" only for ioctls (specialy hack for floppy.c) */ 136 #define BLK_OPEN_WRITE_IOCTL ((__force blk_mode_t)(1 << 4)) 137 /* open is exclusive wrt all other BLK_OPEN_WRITE opens to the device */ 138 #define BLK_OPEN_RESTRICT_WRITES ((__force blk_mode_t)(1 << 5)) 139 /* return partition scanning errors */ 140 #define BLK_OPEN_STRICT_SCAN ((__force blk_mode_t)(1 << 6)) 141 142 struct gendisk { 143 /* 144 * major/first_minor/minors should not be set by any new driver, the 145 * block core will take care of allocating them automatically. 146 */ 147 int major; 148 int first_minor; 149 int minors; 150 151 char disk_name[DISK_NAME_LEN]; /* name of major driver */ 152 153 unsigned short events; /* supported events */ 154 unsigned short event_flags; /* flags related to event processing */ 155 156 struct xarray part_tbl; 157 struct block_device *part0; 158 159 const struct block_device_operations *fops; 160 struct request_queue *queue; 161 void *private_data; 162 163 struct bio_set bio_split; 164 165 int flags; 166 unsigned long state; 167 #define GD_NEED_PART_SCAN 0 168 #define GD_READ_ONLY 1 169 #define GD_DEAD 2 170 #define GD_NATIVE_CAPACITY 3 171 #define GD_ADDED 4 172 #define GD_SUPPRESS_PART_SCAN 5 173 #define GD_OWNS_QUEUE 6 174 175 struct mutex open_mutex; /* open/close mutex */ 176 unsigned open_partitions; /* number of open partitions */ 177 178 struct backing_dev_info *bdi; 179 struct kobject queue_kobj; /* the queue/ directory */ 180 struct kobject *slave_dir; 181 #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED 182 struct list_head slave_bdevs; 183 #endif 184 struct timer_rand_state *random; 185 atomic_t sync_io; /* RAID */ 186 struct disk_events *ev; 187 188 #ifdef CONFIG_BLK_DEV_ZONED 189 /* 190 * Zoned block device information. Reads of this information must be 191 * protected with blk_queue_enter() / blk_queue_exit(). Modifying this 192 * information is only allowed while no requests are being processed. 193 * See also blk_mq_freeze_queue() and blk_mq_unfreeze_queue(). 194 */ 195 unsigned int nr_zones; 196 unsigned int zone_capacity; 197 unsigned int last_zone_capacity; 198 unsigned long __rcu *conv_zones_bitmap; 199 unsigned int zone_wplugs_hash_bits; 200 atomic_t nr_zone_wplugs; 201 spinlock_t zone_wplugs_lock; 202 struct mempool_s *zone_wplugs_pool; 203 struct hlist_head *zone_wplugs_hash; 204 struct workqueue_struct *zone_wplugs_wq; 205 #endif /* CONFIG_BLK_DEV_ZONED */ 206 207 #if IS_ENABLED(CONFIG_CDROM) 208 struct cdrom_device_info *cdi; 209 #endif 210 int node_id; 211 struct badblocks *bb; 212 struct lockdep_map lockdep_map; 213 u64 diskseq; 214 blk_mode_t open_mode; 215 216 /* 217 * Independent sector access ranges. This is always NULL for 218 * devices that do not have multiple independent access ranges. 219 */ 220 struct blk_independent_access_ranges *ia_ranges; 221 }; 222 223 /** 224 * disk_openers - returns how many openers are there for a disk 225 * @disk: disk to check 226 * 227 * This returns the number of openers for a disk. Note that this value is only 228 * stable if disk->open_mutex is held. 229 * 230 * Note: Due to a quirk in the block layer open code, each open partition is 231 * only counted once even if there are multiple openers. 232 */ 233 static inline unsigned int disk_openers(struct gendisk *disk) 234 { 235 return atomic_read(&disk->part0->bd_openers); 236 } 237 238 /** 239 * disk_has_partscan - return %true if partition scanning is enabled on a disk 240 * @disk: disk to check 241 * 242 * Returns %true if partitions scanning is enabled for @disk, or %false if 243 * partition scanning is disabled either permanently or temporarily. 244 */ 245 static inline bool disk_has_partscan(struct gendisk *disk) 246 { 247 return !(disk->flags & (GENHD_FL_NO_PART | GENHD_FL_HIDDEN)) && 248 !test_bit(GD_SUPPRESS_PART_SCAN, &disk->state); 249 } 250 251 /* 252 * The gendisk is refcounted by the part0 block_device, and the bd_device 253 * therein is also used for device model presentation in sysfs. 254 */ 255 #define dev_to_disk(device) \ 256 (dev_to_bdev(device)->bd_disk) 257 #define disk_to_dev(disk) \ 258 (&((disk)->part0->bd_device)) 259 260 #if IS_REACHABLE(CONFIG_CDROM) 261 #define disk_to_cdi(disk) ((disk)->cdi) 262 #else 263 #define disk_to_cdi(disk) NULL 264 #endif 265 266 static inline dev_t disk_devt(struct gendisk *disk) 267 { 268 return MKDEV(disk->major, disk->first_minor); 269 } 270 271 /* 272 * We should strive for 1 << (PAGE_SHIFT + MAX_PAGECACHE_ORDER) 273 * however we constrain this to what we can validate and test. 274 */ 275 #define BLK_MAX_BLOCK_SIZE SZ_64K 276 277 /* blk_validate_limits() validates bsize, so drivers don't usually need to */ 278 static inline int blk_validate_block_size(unsigned long bsize) 279 { 280 if (bsize < 512 || bsize > BLK_MAX_BLOCK_SIZE || !is_power_of_2(bsize)) 281 return -EINVAL; 282 283 return 0; 284 } 285 286 static inline bool blk_op_is_passthrough(blk_opf_t op) 287 { 288 op &= REQ_OP_MASK; 289 return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT; 290 } 291 292 /* flags set by the driver in queue_limits.features */ 293 typedef unsigned int __bitwise blk_features_t; 294 295 /* supports a volatile write cache */ 296 #define BLK_FEAT_WRITE_CACHE ((__force blk_features_t)(1u << 0)) 297 298 /* supports passing on the FUA bit */ 299 #define BLK_FEAT_FUA ((__force blk_features_t)(1u << 1)) 300 301 /* rotational device (hard drive or floppy) */ 302 #define BLK_FEAT_ROTATIONAL ((__force blk_features_t)(1u << 2)) 303 304 /* contributes to the random number pool */ 305 #define BLK_FEAT_ADD_RANDOM ((__force blk_features_t)(1u << 3)) 306 307 /* do disk/partitions IO accounting */ 308 #define BLK_FEAT_IO_STAT ((__force blk_features_t)(1u << 4)) 309 310 /* don't modify data until writeback is done */ 311 #define BLK_FEAT_STABLE_WRITES ((__force blk_features_t)(1u << 5)) 312 313 /* always completes in submit context */ 314 #define BLK_FEAT_SYNCHRONOUS ((__force blk_features_t)(1u << 6)) 315 316 /* supports REQ_NOWAIT */ 317 #define BLK_FEAT_NOWAIT ((__force blk_features_t)(1u << 7)) 318 319 /* supports DAX */ 320 #define BLK_FEAT_DAX ((__force blk_features_t)(1u << 8)) 321 322 /* supports I/O polling */ 323 #define BLK_FEAT_POLL ((__force blk_features_t)(1u << 9)) 324 325 /* is a zoned device */ 326 #define BLK_FEAT_ZONED ((__force blk_features_t)(1u << 10)) 327 328 /* supports PCI(e) p2p requests */ 329 #define BLK_FEAT_PCI_P2PDMA ((__force blk_features_t)(1u << 12)) 330 331 /* skip this queue in blk_mq_(un)quiesce_tagset */ 332 #define BLK_FEAT_SKIP_TAGSET_QUIESCE ((__force blk_features_t)(1u << 13)) 333 334 /* bounce all highmem pages */ 335 #define BLK_FEAT_BOUNCE_HIGH ((__force blk_features_t)(1u << 14)) 336 337 /* undocumented magic for bcache */ 338 #define BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE \ 339 ((__force blk_features_t)(1u << 15)) 340 341 /* atomic writes enabled */ 342 #define BLK_FEAT_ATOMIC_WRITES \ 343 ((__force blk_features_t)(1u << 16)) 344 345 /* 346 * Flags automatically inherited when stacking limits. 347 */ 348 #define BLK_FEAT_INHERIT_MASK \ 349 (BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | BLK_FEAT_ROTATIONAL | \ 350 BLK_FEAT_STABLE_WRITES | BLK_FEAT_ZONED | BLK_FEAT_BOUNCE_HIGH | \ 351 BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE) 352 353 /* internal flags in queue_limits.flags */ 354 typedef unsigned int __bitwise blk_flags_t; 355 356 /* do not send FLUSH/FUA commands despite advertising a write cache */ 357 #define BLK_FLAG_WRITE_CACHE_DISABLED ((__force blk_flags_t)(1u << 0)) 358 359 /* I/O topology is misaligned */ 360 #define BLK_FLAG_MISALIGNED ((__force blk_flags_t)(1u << 1)) 361 362 /* passthrough command IO accounting */ 363 #define BLK_FLAG_IOSTATS_PASSTHROUGH ((__force blk_flags_t)(1u << 2)) 364 365 struct queue_limits { 366 blk_features_t features; 367 blk_flags_t flags; 368 unsigned long seg_boundary_mask; 369 unsigned long virt_boundary_mask; 370 371 unsigned int max_hw_sectors; 372 unsigned int max_dev_sectors; 373 unsigned int chunk_sectors; 374 unsigned int max_sectors; 375 unsigned int max_user_sectors; 376 unsigned int max_segment_size; 377 unsigned int min_segment_size; 378 unsigned int physical_block_size; 379 unsigned int logical_block_size; 380 unsigned int alignment_offset; 381 unsigned int io_min; 382 unsigned int io_opt; 383 unsigned int max_discard_sectors; 384 unsigned int max_hw_discard_sectors; 385 unsigned int max_user_discard_sectors; 386 unsigned int max_secure_erase_sectors; 387 unsigned int max_write_zeroes_sectors; 388 unsigned int max_hw_zone_append_sectors; 389 unsigned int max_zone_append_sectors; 390 unsigned int discard_granularity; 391 unsigned int discard_alignment; 392 unsigned int zone_write_granularity; 393 394 /* atomic write limits */ 395 unsigned int atomic_write_hw_max; 396 unsigned int atomic_write_max_sectors; 397 unsigned int atomic_write_hw_boundary; 398 unsigned int atomic_write_boundary_sectors; 399 unsigned int atomic_write_hw_unit_min; 400 unsigned int atomic_write_unit_min; 401 unsigned int atomic_write_hw_unit_max; 402 unsigned int atomic_write_unit_max; 403 404 unsigned short max_segments; 405 unsigned short max_integrity_segments; 406 unsigned short max_discard_segments; 407 408 unsigned int max_open_zones; 409 unsigned int max_active_zones; 410 411 /* 412 * Drivers that set dma_alignment to less than 511 must be prepared to 413 * handle individual bvec's that are not a multiple of a SECTOR_SIZE 414 * due to possible offsets. 415 */ 416 unsigned int dma_alignment; 417 unsigned int dma_pad_mask; 418 419 struct blk_integrity integrity; 420 }; 421 422 typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx, 423 void *data); 424 425 #define BLK_ALL_ZONES ((unsigned int)-1) 426 int blkdev_report_zones(struct block_device *bdev, sector_t sector, 427 unsigned int nr_zones, report_zones_cb cb, void *data); 428 int blkdev_zone_mgmt(struct block_device *bdev, enum req_op op, 429 sector_t sectors, sector_t nr_sectors); 430 int blk_revalidate_disk_zones(struct gendisk *disk); 431 432 /* 433 * Independent access ranges: struct blk_independent_access_range describes 434 * a range of contiguous sectors that can be accessed using device command 435 * execution resources that are independent from the resources used for 436 * other access ranges. This is typically found with single-LUN multi-actuator 437 * HDDs where each access range is served by a different set of heads. 438 * The set of independent ranges supported by the device is defined using 439 * struct blk_independent_access_ranges. The independent ranges must not overlap 440 * and must include all sectors within the disk capacity (no sector holes 441 * allowed). 442 * For a device with multiple ranges, requests targeting sectors in different 443 * ranges can be executed in parallel. A request can straddle an access range 444 * boundary. 445 */ 446 struct blk_independent_access_range { 447 struct kobject kobj; 448 sector_t sector; 449 sector_t nr_sectors; 450 }; 451 452 struct blk_independent_access_ranges { 453 struct kobject kobj; 454 bool sysfs_registered; 455 unsigned int nr_ia_ranges; 456 struct blk_independent_access_range ia_range[]; 457 }; 458 459 struct request_queue { 460 /* 461 * The queue owner gets to use this for whatever they like. 462 * ll_rw_blk doesn't touch it. 463 */ 464 void *queuedata; 465 466 struct elevator_queue *elevator; 467 468 const struct blk_mq_ops *mq_ops; 469 470 /* sw queues */ 471 struct blk_mq_ctx __percpu *queue_ctx; 472 473 /* 474 * various queue flags, see QUEUE_* below 475 */ 476 unsigned long queue_flags; 477 478 unsigned int rq_timeout; 479 480 unsigned int queue_depth; 481 482 refcount_t refs; 483 484 /* hw dispatch queues */ 485 unsigned int nr_hw_queues; 486 struct xarray hctx_table; 487 488 struct percpu_ref q_usage_counter; 489 struct lock_class_key io_lock_cls_key; 490 struct lockdep_map io_lockdep_map; 491 492 struct lock_class_key q_lock_cls_key; 493 struct lockdep_map q_lockdep_map; 494 495 struct request *last_merge; 496 497 spinlock_t queue_lock; 498 499 int quiesce_depth; 500 501 struct gendisk *disk; 502 503 /* 504 * mq queue kobject 505 */ 506 struct kobject *mq_kobj; 507 508 struct queue_limits limits; 509 510 #ifdef CONFIG_PM 511 struct device *dev; 512 enum rpm_status rpm_status; 513 #endif 514 515 /* 516 * Number of contexts that have called blk_set_pm_only(). If this 517 * counter is above zero then only RQF_PM requests are processed. 518 */ 519 atomic_t pm_only; 520 521 struct blk_queue_stats *stats; 522 struct rq_qos *rq_qos; 523 struct mutex rq_qos_mutex; 524 525 /* 526 * ida allocated id for this queue. Used to index queues from 527 * ioctx. 528 */ 529 int id; 530 531 /* 532 * queue settings 533 */ 534 unsigned long nr_requests; /* Max # of requests */ 535 536 #ifdef CONFIG_BLK_INLINE_ENCRYPTION 537 struct blk_crypto_profile *crypto_profile; 538 struct kobject *crypto_kobject; 539 #endif 540 541 struct timer_list timeout; 542 struct work_struct timeout_work; 543 544 atomic_t nr_active_requests_shared_tags; 545 546 struct blk_mq_tags *sched_shared_tags; 547 548 struct list_head icq_list; 549 #ifdef CONFIG_BLK_CGROUP 550 DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS); 551 struct blkcg_gq *root_blkg; 552 struct list_head blkg_list; 553 struct mutex blkcg_mutex; 554 #endif 555 556 int node; 557 558 spinlock_t requeue_lock; 559 struct list_head requeue_list; 560 struct delayed_work requeue_work; 561 562 #ifdef CONFIG_BLK_DEV_IO_TRACE 563 struct blk_trace __rcu *blk_trace; 564 #endif 565 /* 566 * for flush operations 567 */ 568 struct blk_flush_queue *fq; 569 struct list_head flush_list; 570 571 /* 572 * Protects against I/O scheduler switching, particularly when updating 573 * q->elevator. Since the elevator update code path may also modify q-> 574 * nr_requests and wbt latency, this lock also protects the sysfs attrs 575 * nr_requests and wbt_lat_usec. Additionally the nr_hw_queues update 576 * may modify hctx tags, reserved-tags and cpumask, so this lock also 577 * helps protect the hctx sysfs/debugfs attrs. To ensure proper locking 578 * order during an elevator or nr_hw_queue update, first freeze the 579 * queue, then acquire ->elevator_lock. 580 */ 581 struct mutex elevator_lock; 582 583 struct mutex sysfs_lock; 584 /* 585 * Protects queue limits and also sysfs attribute read_ahead_kb. 586 */ 587 struct mutex limits_lock; 588 589 /* 590 * for reusing dead hctx instance in case of updating 591 * nr_hw_queues 592 */ 593 struct list_head unused_hctx_list; 594 spinlock_t unused_hctx_lock; 595 596 int mq_freeze_depth; 597 598 #ifdef CONFIG_BLK_DEV_THROTTLING 599 /* Throttle data */ 600 struct throtl_data *td; 601 #endif 602 struct rcu_head rcu_head; 603 #ifdef CONFIG_LOCKDEP 604 struct task_struct *mq_freeze_owner; 605 int mq_freeze_owner_depth; 606 /* 607 * Records disk & queue state in current context, used in unfreeze 608 * queue 609 */ 610 bool mq_freeze_disk_dead; 611 bool mq_freeze_queue_dying; 612 #endif 613 wait_queue_head_t mq_freeze_wq; 614 /* 615 * Protect concurrent access to q_usage_counter by 616 * percpu_ref_kill() and percpu_ref_reinit(). 617 */ 618 struct mutex mq_freeze_lock; 619 620 struct blk_mq_tag_set *tag_set; 621 struct list_head tag_set_list; 622 623 struct dentry *debugfs_dir; 624 struct dentry *sched_debugfs_dir; 625 struct dentry *rqos_debugfs_dir; 626 /* 627 * Serializes all debugfs metadata operations using the above dentries. 628 */ 629 struct mutex debugfs_mutex; 630 }; 631 632 /* Keep blk_queue_flag_name[] in sync with the definitions below */ 633 enum { 634 QUEUE_FLAG_DYING, /* queue being torn down */ 635 QUEUE_FLAG_NOMERGES, /* disable merge attempts */ 636 QUEUE_FLAG_SAME_COMP, /* complete on same CPU-group */ 637 QUEUE_FLAG_FAIL_IO, /* fake timeout */ 638 QUEUE_FLAG_NOXMERGES, /* No extended merges */ 639 QUEUE_FLAG_SAME_FORCE, /* force complete on same CPU */ 640 QUEUE_FLAG_INIT_DONE, /* queue is initialized */ 641 QUEUE_FLAG_STATS, /* track IO start and completion times */ 642 QUEUE_FLAG_REGISTERED, /* queue has been registered to a disk */ 643 QUEUE_FLAG_QUIESCED, /* queue has been quiesced */ 644 QUEUE_FLAG_RQ_ALLOC_TIME, /* record rq->alloc_time_ns */ 645 QUEUE_FLAG_HCTX_ACTIVE, /* at least one blk-mq hctx is active */ 646 QUEUE_FLAG_SQ_SCHED, /* single queue style io dispatch */ 647 QUEUE_FLAG_MAX 648 }; 649 650 #define QUEUE_FLAG_MQ_DEFAULT (1UL << QUEUE_FLAG_SAME_COMP) 651 652 void blk_queue_flag_set(unsigned int flag, struct request_queue *q); 653 void blk_queue_flag_clear(unsigned int flag, struct request_queue *q); 654 655 #define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags) 656 #define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags) 657 #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags) 658 #define blk_queue_noxmerges(q) \ 659 test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags) 660 #define blk_queue_nonrot(q) (!((q)->limits.features & BLK_FEAT_ROTATIONAL)) 661 #define blk_queue_io_stat(q) ((q)->limits.features & BLK_FEAT_IO_STAT) 662 #define blk_queue_passthrough_stat(q) \ 663 ((q)->limits.flags & BLK_FLAG_IOSTATS_PASSTHROUGH) 664 #define blk_queue_dax(q) ((q)->limits.features & BLK_FEAT_DAX) 665 #define blk_queue_pci_p2pdma(q) ((q)->limits.features & BLK_FEAT_PCI_P2PDMA) 666 #ifdef CONFIG_BLK_RQ_ALLOC_TIME 667 #define blk_queue_rq_alloc_time(q) \ 668 test_bit(QUEUE_FLAG_RQ_ALLOC_TIME, &(q)->queue_flags) 669 #else 670 #define blk_queue_rq_alloc_time(q) false 671 #endif 672 673 #define blk_noretry_request(rq) \ 674 ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \ 675 REQ_FAILFAST_DRIVER)) 676 #define blk_queue_quiesced(q) test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags) 677 #define blk_queue_pm_only(q) atomic_read(&(q)->pm_only) 678 #define blk_queue_registered(q) test_bit(QUEUE_FLAG_REGISTERED, &(q)->queue_flags) 679 #define blk_queue_sq_sched(q) test_bit(QUEUE_FLAG_SQ_SCHED, &(q)->queue_flags) 680 #define blk_queue_skip_tagset_quiesce(q) \ 681 ((q)->limits.features & BLK_FEAT_SKIP_TAGSET_QUIESCE) 682 683 extern void blk_set_pm_only(struct request_queue *q); 684 extern void blk_clear_pm_only(struct request_queue *q); 685 686 #define list_entry_rq(ptr) list_entry((ptr), struct request, queuelist) 687 688 #define dma_map_bvec(dev, bv, dir, attrs) \ 689 dma_map_page_attrs(dev, (bv)->bv_page, (bv)->bv_offset, (bv)->bv_len, \ 690 (dir), (attrs)) 691 692 static inline bool queue_is_mq(struct request_queue *q) 693 { 694 return q->mq_ops; 695 } 696 697 #ifdef CONFIG_PM 698 static inline enum rpm_status queue_rpm_status(struct request_queue *q) 699 { 700 return q->rpm_status; 701 } 702 #else 703 static inline enum rpm_status queue_rpm_status(struct request_queue *q) 704 { 705 return RPM_ACTIVE; 706 } 707 #endif 708 709 static inline bool blk_queue_is_zoned(struct request_queue *q) 710 { 711 return IS_ENABLED(CONFIG_BLK_DEV_ZONED) && 712 (q->limits.features & BLK_FEAT_ZONED); 713 } 714 715 #ifdef CONFIG_BLK_DEV_ZONED 716 static inline unsigned int disk_nr_zones(struct gendisk *disk) 717 { 718 return disk->nr_zones; 719 } 720 bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs); 721 #else /* CONFIG_BLK_DEV_ZONED */ 722 static inline unsigned int disk_nr_zones(struct gendisk *disk) 723 { 724 return 0; 725 } 726 static inline bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs) 727 { 728 return false; 729 } 730 #endif /* CONFIG_BLK_DEV_ZONED */ 731 732 static inline unsigned int disk_zone_no(struct gendisk *disk, sector_t sector) 733 { 734 if (!blk_queue_is_zoned(disk->queue)) 735 return 0; 736 return sector >> ilog2(disk->queue->limits.chunk_sectors); 737 } 738 739 static inline unsigned int bdev_nr_zones(struct block_device *bdev) 740 { 741 return disk_nr_zones(bdev->bd_disk); 742 } 743 744 static inline unsigned int bdev_max_open_zones(struct block_device *bdev) 745 { 746 return bdev->bd_disk->queue->limits.max_open_zones; 747 } 748 749 static inline unsigned int bdev_max_active_zones(struct block_device *bdev) 750 { 751 return bdev->bd_disk->queue->limits.max_active_zones; 752 } 753 754 static inline unsigned int blk_queue_depth(struct request_queue *q) 755 { 756 if (q->queue_depth) 757 return q->queue_depth; 758 759 return q->nr_requests; 760 } 761 762 /* 763 * default timeout for SG_IO if none specified 764 */ 765 #define BLK_DEFAULT_SG_TIMEOUT (60 * HZ) 766 #define BLK_MIN_SG_TIMEOUT (7 * HZ) 767 768 /* This should not be used directly - use rq_for_each_segment */ 769 #define for_each_bio(_bio) \ 770 for (; _bio; _bio = _bio->bi_next) 771 772 int __must_check add_disk_fwnode(struct device *parent, struct gendisk *disk, 773 const struct attribute_group **groups, 774 struct fwnode_handle *fwnode); 775 int __must_check device_add_disk(struct device *parent, struct gendisk *disk, 776 const struct attribute_group **groups); 777 static inline int __must_check add_disk(struct gendisk *disk) 778 { 779 return device_add_disk(NULL, disk, NULL); 780 } 781 void del_gendisk(struct gendisk *gp); 782 void invalidate_disk(struct gendisk *disk); 783 void set_disk_ro(struct gendisk *disk, bool read_only); 784 void disk_uevent(struct gendisk *disk, enum kobject_action action); 785 786 static inline u8 bdev_partno(const struct block_device *bdev) 787 { 788 return atomic_read(&bdev->__bd_flags) & BD_PARTNO; 789 } 790 791 static inline bool bdev_test_flag(const struct block_device *bdev, unsigned flag) 792 { 793 return atomic_read(&bdev->__bd_flags) & flag; 794 } 795 796 static inline void bdev_set_flag(struct block_device *bdev, unsigned flag) 797 { 798 atomic_or(flag, &bdev->__bd_flags); 799 } 800 801 static inline void bdev_clear_flag(struct block_device *bdev, unsigned flag) 802 { 803 atomic_andnot(flag, &bdev->__bd_flags); 804 } 805 806 static inline bool get_disk_ro(struct gendisk *disk) 807 { 808 return bdev_test_flag(disk->part0, BD_READ_ONLY) || 809 test_bit(GD_READ_ONLY, &disk->state); 810 } 811 812 static inline bool bdev_read_only(struct block_device *bdev) 813 { 814 return bdev_test_flag(bdev, BD_READ_ONLY) || get_disk_ro(bdev->bd_disk); 815 } 816 817 bool set_capacity_and_notify(struct gendisk *disk, sector_t size); 818 void disk_force_media_change(struct gendisk *disk); 819 void bdev_mark_dead(struct block_device *bdev, bool surprise); 820 821 void add_disk_randomness(struct gendisk *disk) __latent_entropy; 822 void rand_initialize_disk(struct gendisk *disk); 823 824 static inline sector_t get_start_sect(struct block_device *bdev) 825 { 826 return bdev->bd_start_sect; 827 } 828 829 static inline sector_t bdev_nr_sectors(struct block_device *bdev) 830 { 831 return bdev->bd_nr_sectors; 832 } 833 834 static inline loff_t bdev_nr_bytes(struct block_device *bdev) 835 { 836 return (loff_t)bdev_nr_sectors(bdev) << SECTOR_SHIFT; 837 } 838 839 static inline sector_t get_capacity(struct gendisk *disk) 840 { 841 return bdev_nr_sectors(disk->part0); 842 } 843 844 static inline u64 sb_bdev_nr_blocks(struct super_block *sb) 845 { 846 return bdev_nr_sectors(sb->s_bdev) >> 847 (sb->s_blocksize_bits - SECTOR_SHIFT); 848 } 849 850 int bdev_disk_changed(struct gendisk *disk, bool invalidate); 851 852 void put_disk(struct gendisk *disk); 853 struct gendisk *__blk_alloc_disk(struct queue_limits *lim, int node, 854 struct lock_class_key *lkclass); 855 856 /** 857 * blk_alloc_disk - allocate a gendisk structure 858 * @lim: queue limits to be used for this disk. 859 * @node_id: numa node to allocate on 860 * 861 * Allocate and pre-initialize a gendisk structure for use with BIO based 862 * drivers. 863 * 864 * Returns an ERR_PTR on error, else the allocated disk. 865 * 866 * Context: can sleep 867 */ 868 #define blk_alloc_disk(lim, node_id) \ 869 ({ \ 870 static struct lock_class_key __key; \ 871 \ 872 __blk_alloc_disk(lim, node_id, &__key); \ 873 }) 874 875 int __register_blkdev(unsigned int major, const char *name, 876 void (*probe)(dev_t devt)); 877 #define register_blkdev(major, name) \ 878 __register_blkdev(major, name, NULL) 879 void unregister_blkdev(unsigned int major, const char *name); 880 881 bool disk_check_media_change(struct gendisk *disk); 882 void set_capacity(struct gendisk *disk, sector_t size); 883 884 #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED 885 int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk); 886 void bd_unlink_disk_holder(struct block_device *bdev, struct gendisk *disk); 887 #else 888 static inline int bd_link_disk_holder(struct block_device *bdev, 889 struct gendisk *disk) 890 { 891 return 0; 892 } 893 static inline void bd_unlink_disk_holder(struct block_device *bdev, 894 struct gendisk *disk) 895 { 896 } 897 #endif /* CONFIG_BLOCK_HOLDER_DEPRECATED */ 898 899 dev_t part_devt(struct gendisk *disk, u8 partno); 900 void inc_diskseq(struct gendisk *disk); 901 void blk_request_module(dev_t devt); 902 903 extern int blk_register_queue(struct gendisk *disk); 904 extern void blk_unregister_queue(struct gendisk *disk); 905 void submit_bio_noacct(struct bio *bio); 906 struct bio *bio_split_to_limits(struct bio *bio); 907 908 extern int blk_lld_busy(struct request_queue *q); 909 extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags); 910 extern void blk_queue_exit(struct request_queue *q); 911 extern void blk_sync_queue(struct request_queue *q); 912 913 /* Helper to convert REQ_OP_XXX to its string format XXX */ 914 extern const char *blk_op_str(enum req_op op); 915 916 int blk_status_to_errno(blk_status_t status); 917 blk_status_t errno_to_blk_status(int errno); 918 const char *blk_status_to_str(blk_status_t status); 919 920 /* only poll the hardware once, don't continue until a completion was found */ 921 #define BLK_POLL_ONESHOT (1 << 0) 922 int bio_poll(struct bio *bio, struct io_comp_batch *iob, unsigned int flags); 923 int iocb_bio_iopoll(struct kiocb *kiocb, struct io_comp_batch *iob, 924 unsigned int flags); 925 926 static inline struct request_queue *bdev_get_queue(struct block_device *bdev) 927 { 928 return bdev->bd_queue; /* this is never NULL */ 929 } 930 931 /* Helper to convert BLK_ZONE_ZONE_XXX to its string format XXX */ 932 const char *blk_zone_cond_str(enum blk_zone_cond zone_cond); 933 934 static inline unsigned int bio_zone_no(struct bio *bio) 935 { 936 return disk_zone_no(bio->bi_bdev->bd_disk, bio->bi_iter.bi_sector); 937 } 938 939 static inline bool bio_straddles_zones(struct bio *bio) 940 { 941 return bio_sectors(bio) && 942 bio_zone_no(bio) != 943 disk_zone_no(bio->bi_bdev->bd_disk, bio_end_sector(bio) - 1); 944 } 945 946 /* 947 * Return how much within the boundary is left to be used for I/O at a given 948 * offset. 949 */ 950 static inline unsigned int blk_boundary_sectors_left(sector_t offset, 951 unsigned int boundary_sectors) 952 { 953 if (unlikely(!is_power_of_2(boundary_sectors))) 954 return boundary_sectors - sector_div(offset, boundary_sectors); 955 return boundary_sectors - (offset & (boundary_sectors - 1)); 956 } 957 958 /** 959 * queue_limits_start_update - start an atomic update of queue limits 960 * @q: queue to update 961 * 962 * This functions starts an atomic update of the queue limits. It takes a lock 963 * to prevent other updates and returns a snapshot of the current limits that 964 * the caller can modify. The caller must call queue_limits_commit_update() 965 * to finish the update. 966 * 967 * Context: process context. 968 */ 969 static inline struct queue_limits 970 queue_limits_start_update(struct request_queue *q) 971 { 972 mutex_lock(&q->limits_lock); 973 return q->limits; 974 } 975 int queue_limits_commit_update_frozen(struct request_queue *q, 976 struct queue_limits *lim); 977 int queue_limits_commit_update(struct request_queue *q, 978 struct queue_limits *lim); 979 int queue_limits_set(struct request_queue *q, struct queue_limits *lim); 980 int blk_validate_limits(struct queue_limits *lim); 981 982 /** 983 * queue_limits_cancel_update - cancel an atomic update of queue limits 984 * @q: queue to update 985 * 986 * This functions cancels an atomic update of the queue limits started by 987 * queue_limits_start_update() and should be used when an error occurs after 988 * starting update. 989 */ 990 static inline void queue_limits_cancel_update(struct request_queue *q) 991 { 992 mutex_unlock(&q->limits_lock); 993 } 994 995 /* 996 * These helpers are for drivers that have sloppy feature negotiation and might 997 * have to disable DISCARD, WRITE_ZEROES or SECURE_DISCARD from the I/O 998 * completion handler when the device returned an indicator that the respective 999 * feature is not actually supported. They are racy and the driver needs to 1000 * cope with that. Try to avoid this scheme if you can. 1001 */ 1002 static inline void blk_queue_disable_discard(struct request_queue *q) 1003 { 1004 q->limits.max_discard_sectors = 0; 1005 } 1006 1007 static inline void blk_queue_disable_secure_erase(struct request_queue *q) 1008 { 1009 q->limits.max_secure_erase_sectors = 0; 1010 } 1011 1012 static inline void blk_queue_disable_write_zeroes(struct request_queue *q) 1013 { 1014 q->limits.max_write_zeroes_sectors = 0; 1015 } 1016 1017 /* 1018 * Access functions for manipulating queue properties 1019 */ 1020 extern void blk_set_queue_depth(struct request_queue *q, unsigned int depth); 1021 extern void blk_set_stacking_limits(struct queue_limits *lim); 1022 extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, 1023 sector_t offset); 1024 void queue_limits_stack_bdev(struct queue_limits *t, struct block_device *bdev, 1025 sector_t offset, const char *pfx); 1026 extern void blk_queue_rq_timeout(struct request_queue *, unsigned int); 1027 1028 struct blk_independent_access_ranges * 1029 disk_alloc_independent_access_ranges(struct gendisk *disk, int nr_ia_ranges); 1030 void disk_set_independent_access_ranges(struct gendisk *disk, 1031 struct blk_independent_access_ranges *iars); 1032 1033 bool __must_check blk_get_queue(struct request_queue *); 1034 extern void blk_put_queue(struct request_queue *); 1035 1036 void blk_mark_disk_dead(struct gendisk *disk); 1037 1038 struct rq_list { 1039 struct request *head; 1040 struct request *tail; 1041 }; 1042 1043 #ifdef CONFIG_BLOCK 1044 /* 1045 * blk_plug permits building a queue of related requests by holding the I/O 1046 * fragments for a short period. This allows merging of sequential requests 1047 * into single larger request. As the requests are moved from a per-task list to 1048 * the device's request_queue in a batch, this results in improved scalability 1049 * as the lock contention for request_queue lock is reduced. 1050 * 1051 * It is ok not to disable preemption when adding the request to the plug list 1052 * or when attempting a merge. For details, please see schedule() where 1053 * blk_flush_plug() is called. 1054 */ 1055 struct blk_plug { 1056 struct rq_list mq_list; /* blk-mq requests */ 1057 1058 /* if ios_left is > 1, we can batch tag/rq allocations */ 1059 struct rq_list cached_rqs; 1060 u64 cur_ktime; 1061 unsigned short nr_ios; 1062 1063 unsigned short rq_count; 1064 1065 bool multiple_queues; 1066 bool has_elevator; 1067 1068 struct list_head cb_list; /* md requires an unplug callback */ 1069 }; 1070 1071 struct blk_plug_cb; 1072 typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool); 1073 struct blk_plug_cb { 1074 struct list_head list; 1075 blk_plug_cb_fn callback; 1076 void *data; 1077 }; 1078 extern struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, 1079 void *data, int size); 1080 extern void blk_start_plug(struct blk_plug *); 1081 extern void blk_start_plug_nr_ios(struct blk_plug *, unsigned short); 1082 extern void blk_finish_plug(struct blk_plug *); 1083 1084 void __blk_flush_plug(struct blk_plug *plug, bool from_schedule); 1085 static inline void blk_flush_plug(struct blk_plug *plug, bool async) 1086 { 1087 if (plug) 1088 __blk_flush_plug(plug, async); 1089 } 1090 1091 /* 1092 * tsk == current here 1093 */ 1094 static inline void blk_plug_invalidate_ts(struct task_struct *tsk) 1095 { 1096 struct blk_plug *plug = tsk->plug; 1097 1098 if (plug) 1099 plug->cur_ktime = 0; 1100 current->flags &= ~PF_BLOCK_TS; 1101 } 1102 1103 int blkdev_issue_flush(struct block_device *bdev); 1104 long nr_blockdev_pages(void); 1105 #else /* CONFIG_BLOCK */ 1106 struct blk_plug { 1107 }; 1108 1109 static inline void blk_start_plug_nr_ios(struct blk_plug *plug, 1110 unsigned short nr_ios) 1111 { 1112 } 1113 1114 static inline void blk_start_plug(struct blk_plug *plug) 1115 { 1116 } 1117 1118 static inline void blk_finish_plug(struct blk_plug *plug) 1119 { 1120 } 1121 1122 static inline void blk_flush_plug(struct blk_plug *plug, bool async) 1123 { 1124 } 1125 1126 static inline void blk_plug_invalidate_ts(struct task_struct *tsk) 1127 { 1128 } 1129 1130 static inline int blkdev_issue_flush(struct block_device *bdev) 1131 { 1132 return 0; 1133 } 1134 1135 static inline long nr_blockdev_pages(void) 1136 { 1137 return 0; 1138 } 1139 #endif /* CONFIG_BLOCK */ 1140 1141 extern void blk_io_schedule(void); 1142 1143 int blkdev_issue_discard(struct block_device *bdev, sector_t sector, 1144 sector_t nr_sects, gfp_t gfp_mask); 1145 int __blkdev_issue_discard(struct block_device *bdev, sector_t sector, 1146 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop); 1147 int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector, 1148 sector_t nr_sects, gfp_t gfp); 1149 1150 #define BLKDEV_ZERO_NOUNMAP (1 << 0) /* do not free blocks */ 1151 #define BLKDEV_ZERO_NOFALLBACK (1 << 1) /* don't write explicit zeroes */ 1152 #define BLKDEV_ZERO_KILLABLE (1 << 2) /* interruptible by fatal signals */ 1153 1154 extern int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 1155 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop, 1156 unsigned flags); 1157 extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 1158 sector_t nr_sects, gfp_t gfp_mask, unsigned flags); 1159 1160 static inline int sb_issue_discard(struct super_block *sb, sector_t block, 1161 sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags) 1162 { 1163 return blkdev_issue_discard(sb->s_bdev, 1164 block << (sb->s_blocksize_bits - 1165 SECTOR_SHIFT), 1166 nr_blocks << (sb->s_blocksize_bits - 1167 SECTOR_SHIFT), 1168 gfp_mask); 1169 } 1170 static inline int sb_issue_zeroout(struct super_block *sb, sector_t block, 1171 sector_t nr_blocks, gfp_t gfp_mask) 1172 { 1173 return blkdev_issue_zeroout(sb->s_bdev, 1174 block << (sb->s_blocksize_bits - 1175 SECTOR_SHIFT), 1176 nr_blocks << (sb->s_blocksize_bits - 1177 SECTOR_SHIFT), 1178 gfp_mask, 0); 1179 } 1180 1181 static inline bool bdev_is_partition(struct block_device *bdev) 1182 { 1183 return bdev_partno(bdev) != 0; 1184 } 1185 1186 enum blk_default_limits { 1187 BLK_MAX_SEGMENTS = 128, 1188 BLK_SAFE_MAX_SECTORS = 255, 1189 BLK_MAX_SEGMENT_SIZE = 65536, 1190 BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL, 1191 }; 1192 1193 /* 1194 * Default upper limit for the software max_sectors limit used for 1195 * regular file system I/O. This can be increased through sysfs. 1196 * 1197 * Not to be confused with the max_hw_sector limit that is entirely 1198 * controlled by the driver, usually based on hardware limits. 1199 */ 1200 #define BLK_DEF_MAX_SECTORS_CAP 2560u 1201 1202 static inline struct queue_limits *bdev_limits(struct block_device *bdev) 1203 { 1204 return &bdev_get_queue(bdev)->limits; 1205 } 1206 1207 static inline unsigned long queue_segment_boundary(const struct request_queue *q) 1208 { 1209 return q->limits.seg_boundary_mask; 1210 } 1211 1212 static inline unsigned long queue_virt_boundary(const struct request_queue *q) 1213 { 1214 return q->limits.virt_boundary_mask; 1215 } 1216 1217 static inline unsigned int queue_max_sectors(const struct request_queue *q) 1218 { 1219 return q->limits.max_sectors; 1220 } 1221 1222 static inline unsigned int queue_max_bytes(struct request_queue *q) 1223 { 1224 return min_t(unsigned int, queue_max_sectors(q), INT_MAX >> 9) << 9; 1225 } 1226 1227 static inline unsigned int queue_max_hw_sectors(const struct request_queue *q) 1228 { 1229 return q->limits.max_hw_sectors; 1230 } 1231 1232 static inline unsigned short queue_max_segments(const struct request_queue *q) 1233 { 1234 return q->limits.max_segments; 1235 } 1236 1237 static inline unsigned short queue_max_discard_segments(const struct request_queue *q) 1238 { 1239 return q->limits.max_discard_segments; 1240 } 1241 1242 static inline unsigned int queue_max_segment_size(const struct request_queue *q) 1243 { 1244 return q->limits.max_segment_size; 1245 } 1246 1247 static inline bool queue_emulates_zone_append(struct request_queue *q) 1248 { 1249 return blk_queue_is_zoned(q) && !q->limits.max_hw_zone_append_sectors; 1250 } 1251 1252 static inline bool bdev_emulates_zone_append(struct block_device *bdev) 1253 { 1254 return queue_emulates_zone_append(bdev_get_queue(bdev)); 1255 } 1256 1257 static inline unsigned int 1258 bdev_max_zone_append_sectors(struct block_device *bdev) 1259 { 1260 return bdev_limits(bdev)->max_zone_append_sectors; 1261 } 1262 1263 static inline unsigned int bdev_max_segments(struct block_device *bdev) 1264 { 1265 return queue_max_segments(bdev_get_queue(bdev)); 1266 } 1267 1268 static inline unsigned queue_logical_block_size(const struct request_queue *q) 1269 { 1270 return q->limits.logical_block_size; 1271 } 1272 1273 static inline unsigned int bdev_logical_block_size(struct block_device *bdev) 1274 { 1275 return queue_logical_block_size(bdev_get_queue(bdev)); 1276 } 1277 1278 static inline unsigned int queue_physical_block_size(const struct request_queue *q) 1279 { 1280 return q->limits.physical_block_size; 1281 } 1282 1283 static inline unsigned int bdev_physical_block_size(struct block_device *bdev) 1284 { 1285 return queue_physical_block_size(bdev_get_queue(bdev)); 1286 } 1287 1288 static inline unsigned int queue_io_min(const struct request_queue *q) 1289 { 1290 return q->limits.io_min; 1291 } 1292 1293 static inline unsigned int bdev_io_min(struct block_device *bdev) 1294 { 1295 return queue_io_min(bdev_get_queue(bdev)); 1296 } 1297 1298 static inline unsigned int queue_io_opt(const struct request_queue *q) 1299 { 1300 return q->limits.io_opt; 1301 } 1302 1303 static inline unsigned int bdev_io_opt(struct block_device *bdev) 1304 { 1305 return queue_io_opt(bdev_get_queue(bdev)); 1306 } 1307 1308 static inline unsigned int 1309 queue_zone_write_granularity(const struct request_queue *q) 1310 { 1311 return q->limits.zone_write_granularity; 1312 } 1313 1314 static inline unsigned int 1315 bdev_zone_write_granularity(struct block_device *bdev) 1316 { 1317 return queue_zone_write_granularity(bdev_get_queue(bdev)); 1318 } 1319 1320 int bdev_alignment_offset(struct block_device *bdev); 1321 unsigned int bdev_discard_alignment(struct block_device *bdev); 1322 1323 static inline unsigned int bdev_max_discard_sectors(struct block_device *bdev) 1324 { 1325 return bdev_limits(bdev)->max_discard_sectors; 1326 } 1327 1328 static inline unsigned int bdev_discard_granularity(struct block_device *bdev) 1329 { 1330 return bdev_limits(bdev)->discard_granularity; 1331 } 1332 1333 static inline unsigned int 1334 bdev_max_secure_erase_sectors(struct block_device *bdev) 1335 { 1336 return bdev_limits(bdev)->max_secure_erase_sectors; 1337 } 1338 1339 static inline unsigned int bdev_write_zeroes_sectors(struct block_device *bdev) 1340 { 1341 return bdev_limits(bdev)->max_write_zeroes_sectors; 1342 } 1343 1344 static inline bool bdev_nonrot(struct block_device *bdev) 1345 { 1346 return blk_queue_nonrot(bdev_get_queue(bdev)); 1347 } 1348 1349 static inline bool bdev_synchronous(struct block_device *bdev) 1350 { 1351 return bdev->bd_disk->queue->limits.features & BLK_FEAT_SYNCHRONOUS; 1352 } 1353 1354 static inline bool bdev_stable_writes(struct block_device *bdev) 1355 { 1356 struct request_queue *q = bdev_get_queue(bdev); 1357 1358 if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) && 1359 q->limits.integrity.csum_type != BLK_INTEGRITY_CSUM_NONE) 1360 return true; 1361 return q->limits.features & BLK_FEAT_STABLE_WRITES; 1362 } 1363 1364 static inline bool blk_queue_write_cache(struct request_queue *q) 1365 { 1366 return (q->limits.features & BLK_FEAT_WRITE_CACHE) && 1367 !(q->limits.flags & BLK_FLAG_WRITE_CACHE_DISABLED); 1368 } 1369 1370 static inline bool bdev_write_cache(struct block_device *bdev) 1371 { 1372 return blk_queue_write_cache(bdev_get_queue(bdev)); 1373 } 1374 1375 static inline bool bdev_fua(struct block_device *bdev) 1376 { 1377 return bdev_limits(bdev)->features & BLK_FEAT_FUA; 1378 } 1379 1380 static inline bool bdev_nowait(struct block_device *bdev) 1381 { 1382 return bdev->bd_disk->queue->limits.features & BLK_FEAT_NOWAIT; 1383 } 1384 1385 static inline bool bdev_is_zoned(struct block_device *bdev) 1386 { 1387 return blk_queue_is_zoned(bdev_get_queue(bdev)); 1388 } 1389 1390 static inline unsigned int bdev_zone_no(struct block_device *bdev, sector_t sec) 1391 { 1392 return disk_zone_no(bdev->bd_disk, sec); 1393 } 1394 1395 static inline sector_t bdev_zone_sectors(struct block_device *bdev) 1396 { 1397 struct request_queue *q = bdev_get_queue(bdev); 1398 1399 if (!blk_queue_is_zoned(q)) 1400 return 0; 1401 return q->limits.chunk_sectors; 1402 } 1403 1404 static inline sector_t bdev_offset_from_zone_start(struct block_device *bdev, 1405 sector_t sector) 1406 { 1407 return sector & (bdev_zone_sectors(bdev) - 1); 1408 } 1409 1410 static inline sector_t bio_offset_from_zone_start(struct bio *bio) 1411 { 1412 return bdev_offset_from_zone_start(bio->bi_bdev, 1413 bio->bi_iter.bi_sector); 1414 } 1415 1416 static inline bool bdev_is_zone_start(struct block_device *bdev, 1417 sector_t sector) 1418 { 1419 return bdev_offset_from_zone_start(bdev, sector) == 0; 1420 } 1421 1422 /** 1423 * bdev_zone_is_seq - check if a sector belongs to a sequential write zone 1424 * @bdev: block device to check 1425 * @sector: sector number 1426 * 1427 * Check if @sector on @bdev is contained in a sequential write required zone. 1428 */ 1429 static inline bool bdev_zone_is_seq(struct block_device *bdev, sector_t sector) 1430 { 1431 bool is_seq = false; 1432 1433 #if IS_ENABLED(CONFIG_BLK_DEV_ZONED) 1434 if (bdev_is_zoned(bdev)) { 1435 struct gendisk *disk = bdev->bd_disk; 1436 unsigned long *bitmap; 1437 1438 rcu_read_lock(); 1439 bitmap = rcu_dereference(disk->conv_zones_bitmap); 1440 is_seq = !bitmap || 1441 !test_bit(disk_zone_no(disk, sector), bitmap); 1442 rcu_read_unlock(); 1443 } 1444 #endif 1445 1446 return is_seq; 1447 } 1448 1449 int blk_zone_issue_zeroout(struct block_device *bdev, sector_t sector, 1450 sector_t nr_sects, gfp_t gfp_mask); 1451 1452 static inline unsigned int queue_dma_alignment(const struct request_queue *q) 1453 { 1454 return q->limits.dma_alignment; 1455 } 1456 1457 static inline unsigned int 1458 queue_atomic_write_unit_max_bytes(const struct request_queue *q) 1459 { 1460 return q->limits.atomic_write_unit_max; 1461 } 1462 1463 static inline unsigned int 1464 queue_atomic_write_unit_min_bytes(const struct request_queue *q) 1465 { 1466 return q->limits.atomic_write_unit_min; 1467 } 1468 1469 static inline unsigned int 1470 queue_atomic_write_boundary_bytes(const struct request_queue *q) 1471 { 1472 return q->limits.atomic_write_boundary_sectors << SECTOR_SHIFT; 1473 } 1474 1475 static inline unsigned int 1476 queue_atomic_write_max_bytes(const struct request_queue *q) 1477 { 1478 return q->limits.atomic_write_max_sectors << SECTOR_SHIFT; 1479 } 1480 1481 static inline unsigned int bdev_dma_alignment(struct block_device *bdev) 1482 { 1483 return queue_dma_alignment(bdev_get_queue(bdev)); 1484 } 1485 1486 static inline bool bdev_iter_is_aligned(struct block_device *bdev, 1487 struct iov_iter *iter) 1488 { 1489 return iov_iter_is_aligned(iter, bdev_dma_alignment(bdev), 1490 bdev_logical_block_size(bdev) - 1); 1491 } 1492 1493 static inline unsigned int 1494 blk_lim_dma_alignment_and_pad(struct queue_limits *lim) 1495 { 1496 return lim->dma_alignment | lim->dma_pad_mask; 1497 } 1498 1499 static inline bool blk_rq_aligned(struct request_queue *q, unsigned long addr, 1500 unsigned int len) 1501 { 1502 unsigned int alignment = blk_lim_dma_alignment_and_pad(&q->limits); 1503 1504 return !(addr & alignment) && !(len & alignment); 1505 } 1506 1507 /* assumes size > 256 */ 1508 static inline unsigned int blksize_bits(unsigned int size) 1509 { 1510 return order_base_2(size >> SECTOR_SHIFT) + SECTOR_SHIFT; 1511 } 1512 1513 int kblockd_schedule_work(struct work_struct *work); 1514 int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay); 1515 1516 #define MODULE_ALIAS_BLOCKDEV(major,minor) \ 1517 MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor)) 1518 #define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \ 1519 MODULE_ALIAS("block-major-" __stringify(major) "-*") 1520 1521 #ifdef CONFIG_BLK_INLINE_ENCRYPTION 1522 1523 bool blk_crypto_register(struct blk_crypto_profile *profile, 1524 struct request_queue *q); 1525 1526 #else /* CONFIG_BLK_INLINE_ENCRYPTION */ 1527 1528 static inline bool blk_crypto_register(struct blk_crypto_profile *profile, 1529 struct request_queue *q) 1530 { 1531 return true; 1532 } 1533 1534 #endif /* CONFIG_BLK_INLINE_ENCRYPTION */ 1535 1536 enum blk_unique_id { 1537 /* these match the Designator Types specified in SPC */ 1538 BLK_UID_T10 = 1, 1539 BLK_UID_EUI64 = 2, 1540 BLK_UID_NAA = 3, 1541 }; 1542 1543 struct block_device_operations { 1544 void (*submit_bio)(struct bio *bio); 1545 int (*poll_bio)(struct bio *bio, struct io_comp_batch *iob, 1546 unsigned int flags); 1547 int (*open)(struct gendisk *disk, blk_mode_t mode); 1548 void (*release)(struct gendisk *disk); 1549 int (*ioctl)(struct block_device *bdev, blk_mode_t mode, 1550 unsigned cmd, unsigned long arg); 1551 int (*compat_ioctl)(struct block_device *bdev, blk_mode_t mode, 1552 unsigned cmd, unsigned long arg); 1553 unsigned int (*check_events) (struct gendisk *disk, 1554 unsigned int clearing); 1555 void (*unlock_native_capacity) (struct gendisk *); 1556 int (*getgeo)(struct block_device *, struct hd_geometry *); 1557 int (*set_read_only)(struct block_device *bdev, bool ro); 1558 void (*free_disk)(struct gendisk *disk); 1559 /* this callback is with swap_lock and sometimes page table lock held */ 1560 void (*swap_slot_free_notify) (struct block_device *, unsigned long); 1561 int (*report_zones)(struct gendisk *, sector_t sector, 1562 unsigned int nr_zones, report_zones_cb cb, void *data); 1563 char *(*devnode)(struct gendisk *disk, umode_t *mode); 1564 /* returns the length of the identifier or a negative errno: */ 1565 int (*get_unique_id)(struct gendisk *disk, u8 id[16], 1566 enum blk_unique_id id_type); 1567 struct module *owner; 1568 const struct pr_ops *pr_ops; 1569 1570 /* 1571 * Special callback for probing GPT entry at a given sector. 1572 * Needed by Android devices, used by GPT scanner and MMC blk 1573 * driver. 1574 */ 1575 int (*alternative_gpt_sector)(struct gendisk *disk, sector_t *sector); 1576 }; 1577 1578 #ifdef CONFIG_COMPAT 1579 extern int blkdev_compat_ptr_ioctl(struct block_device *, blk_mode_t, 1580 unsigned int, unsigned long); 1581 #else 1582 #define blkdev_compat_ptr_ioctl NULL 1583 #endif 1584 1585 static inline void blk_wake_io_task(struct task_struct *waiter) 1586 { 1587 /* 1588 * If we're polling, the task itself is doing the completions. For 1589 * that case, we don't need to signal a wakeup, it's enough to just 1590 * mark us as RUNNING. 1591 */ 1592 if (waiter == current) 1593 __set_current_state(TASK_RUNNING); 1594 else 1595 wake_up_process(waiter); 1596 } 1597 1598 unsigned long bdev_start_io_acct(struct block_device *bdev, enum req_op op, 1599 unsigned long start_time); 1600 void bdev_end_io_acct(struct block_device *bdev, enum req_op op, 1601 unsigned int sectors, unsigned long start_time); 1602 1603 unsigned long bio_start_io_acct(struct bio *bio); 1604 void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time, 1605 struct block_device *orig_bdev); 1606 1607 /** 1608 * bio_end_io_acct - end I/O accounting for bio based drivers 1609 * @bio: bio to end account for 1610 * @start_time: start time returned by bio_start_io_acct() 1611 */ 1612 static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time) 1613 { 1614 return bio_end_io_acct_remapped(bio, start_time, bio->bi_bdev); 1615 } 1616 1617 int set_blocksize(struct file *file, int size); 1618 1619 int lookup_bdev(const char *pathname, dev_t *dev); 1620 1621 void blkdev_show(struct seq_file *seqf, off_t offset); 1622 1623 #define BDEVNAME_SIZE 32 /* Largest string for a blockdev identifier */ 1624 #define BDEVT_SIZE 10 /* Largest string for MAJ:MIN for blkdev */ 1625 #ifdef CONFIG_BLOCK 1626 #define BLKDEV_MAJOR_MAX 512 1627 #else 1628 #define BLKDEV_MAJOR_MAX 0 1629 #endif 1630 1631 struct blk_holder_ops { 1632 void (*mark_dead)(struct block_device *bdev, bool surprise); 1633 1634 /* 1635 * Sync the file system mounted on the block device. 1636 */ 1637 void (*sync)(struct block_device *bdev); 1638 1639 /* 1640 * Freeze the file system mounted on the block device. 1641 */ 1642 int (*freeze)(struct block_device *bdev); 1643 1644 /* 1645 * Thaw the file system mounted on the block device. 1646 */ 1647 int (*thaw)(struct block_device *bdev); 1648 }; 1649 1650 /* 1651 * For filesystems using @fs_holder_ops, the @holder argument passed to 1652 * helpers used to open and claim block devices via 1653 * bd_prepare_to_claim() must point to a superblock. 1654 */ 1655 extern const struct blk_holder_ops fs_holder_ops; 1656 1657 /* 1658 * Return the correct open flags for blkdev_get_by_* for super block flags 1659 * as stored in sb->s_flags. 1660 */ 1661 #define sb_open_mode(flags) \ 1662 (BLK_OPEN_READ | BLK_OPEN_RESTRICT_WRITES | \ 1663 (((flags) & SB_RDONLY) ? 0 : BLK_OPEN_WRITE)) 1664 1665 struct file *bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder, 1666 const struct blk_holder_ops *hops); 1667 struct file *bdev_file_open_by_path(const char *path, blk_mode_t mode, 1668 void *holder, const struct blk_holder_ops *hops); 1669 int bd_prepare_to_claim(struct block_device *bdev, void *holder, 1670 const struct blk_holder_ops *hops); 1671 void bd_abort_claiming(struct block_device *bdev, void *holder); 1672 1673 /* just for blk-cgroup, don't use elsewhere */ 1674 struct block_device *blkdev_get_no_open(dev_t dev); 1675 void blkdev_put_no_open(struct block_device *bdev); 1676 1677 struct block_device *I_BDEV(struct inode *inode); 1678 struct block_device *file_bdev(struct file *bdev_file); 1679 bool disk_live(struct gendisk *disk); 1680 unsigned int block_size(struct block_device *bdev); 1681 1682 #ifdef CONFIG_BLOCK 1683 void invalidate_bdev(struct block_device *bdev); 1684 int sync_blockdev(struct block_device *bdev); 1685 int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend); 1686 int sync_blockdev_nowait(struct block_device *bdev); 1687 void sync_bdevs(bool wait); 1688 void bdev_statx(struct path *, struct kstat *, u32); 1689 void printk_all_partitions(void); 1690 int __init early_lookup_bdev(const char *pathname, dev_t *dev); 1691 #else 1692 static inline void invalidate_bdev(struct block_device *bdev) 1693 { 1694 } 1695 static inline int sync_blockdev(struct block_device *bdev) 1696 { 1697 return 0; 1698 } 1699 static inline int sync_blockdev_nowait(struct block_device *bdev) 1700 { 1701 return 0; 1702 } 1703 static inline void sync_bdevs(bool wait) 1704 { 1705 } 1706 static inline void bdev_statx(struct path *path, struct kstat *stat, 1707 u32 request_mask) 1708 { 1709 } 1710 static inline void printk_all_partitions(void) 1711 { 1712 } 1713 static inline int early_lookup_bdev(const char *pathname, dev_t *dev) 1714 { 1715 return -EINVAL; 1716 } 1717 #endif /* CONFIG_BLOCK */ 1718 1719 int bdev_freeze(struct block_device *bdev); 1720 int bdev_thaw(struct block_device *bdev); 1721 void bdev_fput(struct file *bdev_file); 1722 1723 struct io_comp_batch { 1724 struct rq_list req_list; 1725 bool need_ts; 1726 void (*complete)(struct io_comp_batch *); 1727 }; 1728 1729 static inline bool blk_atomic_write_start_sect_aligned(sector_t sector, 1730 struct queue_limits *limits) 1731 { 1732 unsigned int alignment = max(limits->atomic_write_hw_unit_min, 1733 limits->atomic_write_hw_boundary); 1734 1735 return IS_ALIGNED(sector, alignment >> SECTOR_SHIFT); 1736 } 1737 1738 static inline bool bdev_can_atomic_write(struct block_device *bdev) 1739 { 1740 struct request_queue *bd_queue = bdev->bd_queue; 1741 struct queue_limits *limits = &bd_queue->limits; 1742 1743 if (!limits->atomic_write_unit_min) 1744 return false; 1745 1746 if (bdev_is_partition(bdev)) 1747 return blk_atomic_write_start_sect_aligned(bdev->bd_start_sect, 1748 limits); 1749 1750 return true; 1751 } 1752 1753 static inline unsigned int 1754 bdev_atomic_write_unit_min_bytes(struct block_device *bdev) 1755 { 1756 if (!bdev_can_atomic_write(bdev)) 1757 return 0; 1758 return queue_atomic_write_unit_min_bytes(bdev_get_queue(bdev)); 1759 } 1760 1761 static inline unsigned int 1762 bdev_atomic_write_unit_max_bytes(struct block_device *bdev) 1763 { 1764 if (!bdev_can_atomic_write(bdev)) 1765 return 0; 1766 return queue_atomic_write_unit_max_bytes(bdev_get_queue(bdev)); 1767 } 1768 1769 #define DEFINE_IO_COMP_BATCH(name) struct io_comp_batch name = { } 1770 1771 #endif /* _LINUX_BLKDEV_H */ 1772