1 #ifndef _LINUX_BLKDEV_H 2 #define _LINUX_BLKDEV_H 3 4 #include <linux/sched.h> 5 6 #ifdef CONFIG_BLOCK 7 8 #include <linux/major.h> 9 #include <linux/genhd.h> 10 #include <linux/list.h> 11 #include <linux/llist.h> 12 #include <linux/timer.h> 13 #include <linux/workqueue.h> 14 #include <linux/pagemap.h> 15 #include <linux/backing-dev-defs.h> 16 #include <linux/wait.h> 17 #include <linux/mempool.h> 18 #include <linux/pfn.h> 19 #include <linux/bio.h> 20 #include <linux/stringify.h> 21 #include <linux/gfp.h> 22 #include <linux/bsg.h> 23 #include <linux/smp.h> 24 #include <linux/rcupdate.h> 25 #include <linux/percpu-refcount.h> 26 #include <linux/scatterlist.h> 27 28 struct module; 29 struct scsi_ioctl_command; 30 31 struct request_queue; 32 struct elevator_queue; 33 struct blk_trace; 34 struct request; 35 struct sg_io_hdr; 36 struct bsg_job; 37 struct blkcg_gq; 38 struct blk_flush_queue; 39 struct pr_ops; 40 41 #define BLKDEV_MIN_RQ 4 42 #define BLKDEV_MAX_RQ 128 /* Default maximum */ 43 44 /* 45 * Maximum number of blkcg policies allowed to be registered concurrently. 46 * Defined here to simplify include dependency. 47 */ 48 #define BLKCG_MAX_POLS 2 49 50 struct request; 51 typedef void (rq_end_io_fn)(struct request *, int); 52 53 #define BLK_RL_SYNCFULL (1U << 0) 54 #define BLK_RL_ASYNCFULL (1U << 1) 55 56 struct request_list { 57 struct request_queue *q; /* the queue this rl belongs to */ 58 #ifdef CONFIG_BLK_CGROUP 59 struct blkcg_gq *blkg; /* blkg this request pool belongs to */ 60 #endif 61 /* 62 * count[], starved[], and wait[] are indexed by 63 * BLK_RW_SYNC/BLK_RW_ASYNC 64 */ 65 int count[2]; 66 int starved[2]; 67 mempool_t *rq_pool; 68 wait_queue_head_t wait[2]; 69 unsigned int flags; 70 }; 71 72 /* 73 * request command types 74 */ 75 enum rq_cmd_type_bits { 76 REQ_TYPE_FS = 1, /* fs request */ 77 REQ_TYPE_BLOCK_PC, /* scsi command */ 78 REQ_TYPE_DRV_PRIV, /* driver defined types from here */ 79 }; 80 81 #define BLK_MAX_CDB 16 82 83 /* 84 * Try to put the fields that are referenced together in the same cacheline. 85 * 86 * If you modify this structure, make sure to update blk_rq_init() and 87 * especially blk_mq_rq_ctx_init() to take care of the added fields. 88 */ 89 struct request { 90 struct list_head queuelist; 91 union { 92 struct call_single_data csd; 93 unsigned long fifo_time; 94 }; 95 96 struct request_queue *q; 97 struct blk_mq_ctx *mq_ctx; 98 99 u64 cmd_flags; 100 unsigned cmd_type; 101 unsigned long atomic_flags; 102 103 int cpu; 104 105 /* the following two fields are internal, NEVER access directly */ 106 unsigned int __data_len; /* total data len */ 107 sector_t __sector; /* sector cursor */ 108 109 struct bio *bio; 110 struct bio *biotail; 111 112 /* 113 * The hash is used inside the scheduler, and killed once the 114 * request reaches the dispatch list. The ipi_list is only used 115 * to queue the request for softirq completion, which is long 116 * after the request has been unhashed (and even removed from 117 * the dispatch list). 118 */ 119 union { 120 struct hlist_node hash; /* merge hash */ 121 struct list_head ipi_list; 122 }; 123 124 /* 125 * The rb_node is only used inside the io scheduler, requests 126 * are pruned when moved to the dispatch queue. So let the 127 * completion_data share space with the rb_node. 128 */ 129 union { 130 struct rb_node rb_node; /* sort/lookup */ 131 void *completion_data; 132 }; 133 134 /* 135 * Three pointers are available for the IO schedulers, if they need 136 * more they have to dynamically allocate it. Flush requests are 137 * never put on the IO scheduler. So let the flush fields share 138 * space with the elevator data. 139 */ 140 union { 141 struct { 142 struct io_cq *icq; 143 void *priv[2]; 144 } elv; 145 146 struct { 147 unsigned int seq; 148 struct list_head list; 149 rq_end_io_fn *saved_end_io; 150 } flush; 151 }; 152 153 struct gendisk *rq_disk; 154 struct hd_struct *part; 155 unsigned long start_time; 156 #ifdef CONFIG_BLK_CGROUP 157 struct request_list *rl; /* rl this rq is alloced from */ 158 unsigned long long start_time_ns; 159 unsigned long long io_start_time_ns; /* when passed to hardware */ 160 #endif 161 /* Number of scatter-gather DMA addr+len pairs after 162 * physical address coalescing is performed. 163 */ 164 unsigned short nr_phys_segments; 165 #if defined(CONFIG_BLK_DEV_INTEGRITY) 166 unsigned short nr_integrity_segments; 167 #endif 168 169 unsigned short ioprio; 170 171 void *special; /* opaque pointer available for LLD use */ 172 173 int tag; 174 int errors; 175 176 /* 177 * when request is used as a packet command carrier 178 */ 179 unsigned char __cmd[BLK_MAX_CDB]; 180 unsigned char *cmd; 181 unsigned short cmd_len; 182 183 unsigned int extra_len; /* length of alignment and padding */ 184 unsigned int sense_len; 185 unsigned int resid_len; /* residual count */ 186 void *sense; 187 188 unsigned long deadline; 189 struct list_head timeout_list; 190 unsigned int timeout; 191 int retries; 192 193 /* 194 * completion callback. 195 */ 196 rq_end_io_fn *end_io; 197 void *end_io_data; 198 199 /* for bidi */ 200 struct request *next_rq; 201 }; 202 203 static inline unsigned short req_get_ioprio(struct request *req) 204 { 205 return req->ioprio; 206 } 207 208 #include <linux/elevator.h> 209 210 struct blk_queue_ctx; 211 212 typedef void (request_fn_proc) (struct request_queue *q); 213 typedef blk_qc_t (make_request_fn) (struct request_queue *q, struct bio *bio); 214 typedef int (prep_rq_fn) (struct request_queue *, struct request *); 215 typedef void (unprep_rq_fn) (struct request_queue *, struct request *); 216 217 struct bio_vec; 218 typedef void (softirq_done_fn)(struct request *); 219 typedef int (dma_drain_needed_fn)(struct request *); 220 typedef int (lld_busy_fn) (struct request_queue *q); 221 typedef int (bsg_job_fn) (struct bsg_job *); 222 223 enum blk_eh_timer_return { 224 BLK_EH_NOT_HANDLED, 225 BLK_EH_HANDLED, 226 BLK_EH_RESET_TIMER, 227 }; 228 229 typedef enum blk_eh_timer_return (rq_timed_out_fn)(struct request *); 230 231 enum blk_queue_state { 232 Queue_down, 233 Queue_up, 234 }; 235 236 struct blk_queue_tag { 237 struct request **tag_index; /* map of busy tags */ 238 unsigned long *tag_map; /* bit map of free/busy tags */ 239 int busy; /* current depth */ 240 int max_depth; /* what we will send to device */ 241 int real_max_depth; /* what the array can hold */ 242 atomic_t refcnt; /* map can be shared */ 243 int alloc_policy; /* tag allocation policy */ 244 int next_tag; /* next tag */ 245 }; 246 #define BLK_TAG_ALLOC_FIFO 0 /* allocate starting from 0 */ 247 #define BLK_TAG_ALLOC_RR 1 /* allocate starting from last allocated tag */ 248 249 #define BLK_SCSI_MAX_CMDS (256) 250 #define BLK_SCSI_CMD_PER_LONG (BLK_SCSI_MAX_CMDS / (sizeof(long) * 8)) 251 252 struct queue_limits { 253 unsigned long bounce_pfn; 254 unsigned long seg_boundary_mask; 255 unsigned long virt_boundary_mask; 256 257 unsigned int max_hw_sectors; 258 unsigned int max_dev_sectors; 259 unsigned int chunk_sectors; 260 unsigned int max_sectors; 261 unsigned int max_segment_size; 262 unsigned int physical_block_size; 263 unsigned int alignment_offset; 264 unsigned int io_min; 265 unsigned int io_opt; 266 unsigned int max_discard_sectors; 267 unsigned int max_hw_discard_sectors; 268 unsigned int max_write_same_sectors; 269 unsigned int discard_granularity; 270 unsigned int discard_alignment; 271 272 unsigned short logical_block_size; 273 unsigned short max_segments; 274 unsigned short max_integrity_segments; 275 276 unsigned char misaligned; 277 unsigned char discard_misaligned; 278 unsigned char cluster; 279 unsigned char discard_zeroes_data; 280 unsigned char raid_partial_stripes_expensive; 281 }; 282 283 struct request_queue { 284 /* 285 * Together with queue_head for cacheline sharing 286 */ 287 struct list_head queue_head; 288 struct request *last_merge; 289 struct elevator_queue *elevator; 290 int nr_rqs[2]; /* # allocated [a]sync rqs */ 291 int nr_rqs_elvpriv; /* # allocated rqs w/ elvpriv */ 292 293 /* 294 * If blkcg is not used, @q->root_rl serves all requests. If blkcg 295 * is used, root blkg allocates from @q->root_rl and all other 296 * blkgs from their own blkg->rl. Which one to use should be 297 * determined using bio_request_list(). 298 */ 299 struct request_list root_rl; 300 301 request_fn_proc *request_fn; 302 make_request_fn *make_request_fn; 303 prep_rq_fn *prep_rq_fn; 304 unprep_rq_fn *unprep_rq_fn; 305 softirq_done_fn *softirq_done_fn; 306 rq_timed_out_fn *rq_timed_out_fn; 307 dma_drain_needed_fn *dma_drain_needed; 308 lld_busy_fn *lld_busy_fn; 309 310 struct blk_mq_ops *mq_ops; 311 312 unsigned int *mq_map; 313 314 /* sw queues */ 315 struct blk_mq_ctx __percpu *queue_ctx; 316 unsigned int nr_queues; 317 318 /* hw dispatch queues */ 319 struct blk_mq_hw_ctx **queue_hw_ctx; 320 unsigned int nr_hw_queues; 321 322 /* 323 * Dispatch queue sorting 324 */ 325 sector_t end_sector; 326 struct request *boundary_rq; 327 328 /* 329 * Delayed queue handling 330 */ 331 struct delayed_work delay_work; 332 333 struct backing_dev_info backing_dev_info; 334 335 /* 336 * The queue owner gets to use this for whatever they like. 337 * ll_rw_blk doesn't touch it. 338 */ 339 void *queuedata; 340 341 /* 342 * various queue flags, see QUEUE_* below 343 */ 344 unsigned long queue_flags; 345 346 /* 347 * ida allocated id for this queue. Used to index queues from 348 * ioctx. 349 */ 350 int id; 351 352 /* 353 * queue needs bounce pages for pages above this limit 354 */ 355 gfp_t bounce_gfp; 356 357 /* 358 * protects queue structures from reentrancy. ->__queue_lock should 359 * _never_ be used directly, it is queue private. always use 360 * ->queue_lock. 361 */ 362 spinlock_t __queue_lock; 363 spinlock_t *queue_lock; 364 365 /* 366 * queue kobject 367 */ 368 struct kobject kobj; 369 370 /* 371 * mq queue kobject 372 */ 373 struct kobject mq_kobj; 374 375 #ifdef CONFIG_BLK_DEV_INTEGRITY 376 struct blk_integrity integrity; 377 #endif /* CONFIG_BLK_DEV_INTEGRITY */ 378 379 #ifdef CONFIG_PM 380 struct device *dev; 381 int rpm_status; 382 unsigned int nr_pending; 383 #endif 384 385 /* 386 * queue settings 387 */ 388 unsigned long nr_requests; /* Max # of requests */ 389 unsigned int nr_congestion_on; 390 unsigned int nr_congestion_off; 391 unsigned int nr_batching; 392 393 unsigned int dma_drain_size; 394 void *dma_drain_buffer; 395 unsigned int dma_pad_mask; 396 unsigned int dma_alignment; 397 398 struct blk_queue_tag *queue_tags; 399 struct list_head tag_busy_list; 400 401 unsigned int nr_sorted; 402 unsigned int in_flight[2]; 403 /* 404 * Number of active block driver functions for which blk_drain_queue() 405 * must wait. Must be incremented around functions that unlock the 406 * queue_lock internally, e.g. scsi_request_fn(). 407 */ 408 unsigned int request_fn_active; 409 410 unsigned int rq_timeout; 411 struct timer_list timeout; 412 struct work_struct timeout_work; 413 struct list_head timeout_list; 414 415 struct list_head icq_list; 416 #ifdef CONFIG_BLK_CGROUP 417 DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS); 418 struct blkcg_gq *root_blkg; 419 struct list_head blkg_list; 420 #endif 421 422 struct queue_limits limits; 423 424 /* 425 * sg stuff 426 */ 427 unsigned int sg_timeout; 428 unsigned int sg_reserved_size; 429 int node; 430 #ifdef CONFIG_BLK_DEV_IO_TRACE 431 struct blk_trace *blk_trace; 432 #endif 433 /* 434 * for flush operations 435 */ 436 unsigned int flush_flags; 437 unsigned int flush_not_queueable:1; 438 struct blk_flush_queue *fq; 439 440 struct list_head requeue_list; 441 spinlock_t requeue_lock; 442 struct work_struct requeue_work; 443 444 struct mutex sysfs_lock; 445 446 int bypass_depth; 447 atomic_t mq_freeze_depth; 448 449 #if defined(CONFIG_BLK_DEV_BSG) 450 bsg_job_fn *bsg_job_fn; 451 int bsg_job_size; 452 struct bsg_class_device bsg_dev; 453 #endif 454 455 #ifdef CONFIG_BLK_DEV_THROTTLING 456 /* Throttle data */ 457 struct throtl_data *td; 458 #endif 459 struct rcu_head rcu_head; 460 wait_queue_head_t mq_freeze_wq; 461 struct percpu_ref q_usage_counter; 462 struct list_head all_q_node; 463 464 struct blk_mq_tag_set *tag_set; 465 struct list_head tag_set_list; 466 struct bio_set *bio_split; 467 468 bool mq_sysfs_init_done; 469 }; 470 471 #define QUEUE_FLAG_QUEUED 1 /* uses generic tag queueing */ 472 #define QUEUE_FLAG_STOPPED 2 /* queue is stopped */ 473 #define QUEUE_FLAG_SYNCFULL 3 /* read queue has been filled */ 474 #define QUEUE_FLAG_ASYNCFULL 4 /* write queue has been filled */ 475 #define QUEUE_FLAG_DYING 5 /* queue being torn down */ 476 #define QUEUE_FLAG_BYPASS 6 /* act as dumb FIFO queue */ 477 #define QUEUE_FLAG_BIDI 7 /* queue supports bidi requests */ 478 #define QUEUE_FLAG_NOMERGES 8 /* disable merge attempts */ 479 #define QUEUE_FLAG_SAME_COMP 9 /* complete on same CPU-group */ 480 #define QUEUE_FLAG_FAIL_IO 10 /* fake timeout */ 481 #define QUEUE_FLAG_STACKABLE 11 /* supports request stacking */ 482 #define QUEUE_FLAG_NONROT 12 /* non-rotational device (SSD) */ 483 #define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT /* paravirt device */ 484 #define QUEUE_FLAG_IO_STAT 13 /* do IO stats */ 485 #define QUEUE_FLAG_DISCARD 14 /* supports DISCARD */ 486 #define QUEUE_FLAG_NOXMERGES 15 /* No extended merges */ 487 #define QUEUE_FLAG_ADD_RANDOM 16 /* Contributes to random pool */ 488 #define QUEUE_FLAG_SECDISCARD 17 /* supports SECDISCARD */ 489 #define QUEUE_FLAG_SAME_FORCE 18 /* force complete on same CPU */ 490 #define QUEUE_FLAG_DEAD 19 /* queue tear-down finished */ 491 #define QUEUE_FLAG_INIT_DONE 20 /* queue is initialized */ 492 #define QUEUE_FLAG_NO_SG_MERGE 21 /* don't attempt to merge SG segments*/ 493 #define QUEUE_FLAG_POLL 22 /* IO polling enabled if set */ 494 495 #define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \ 496 (1 << QUEUE_FLAG_STACKABLE) | \ 497 (1 << QUEUE_FLAG_SAME_COMP) | \ 498 (1 << QUEUE_FLAG_ADD_RANDOM)) 499 500 #define QUEUE_FLAG_MQ_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \ 501 (1 << QUEUE_FLAG_STACKABLE) | \ 502 (1 << QUEUE_FLAG_SAME_COMP) | \ 503 (1 << QUEUE_FLAG_POLL)) 504 505 static inline void queue_lockdep_assert_held(struct request_queue *q) 506 { 507 if (q->queue_lock) 508 lockdep_assert_held(q->queue_lock); 509 } 510 511 static inline void queue_flag_set_unlocked(unsigned int flag, 512 struct request_queue *q) 513 { 514 __set_bit(flag, &q->queue_flags); 515 } 516 517 static inline int queue_flag_test_and_clear(unsigned int flag, 518 struct request_queue *q) 519 { 520 queue_lockdep_assert_held(q); 521 522 if (test_bit(flag, &q->queue_flags)) { 523 __clear_bit(flag, &q->queue_flags); 524 return 1; 525 } 526 527 return 0; 528 } 529 530 static inline int queue_flag_test_and_set(unsigned int flag, 531 struct request_queue *q) 532 { 533 queue_lockdep_assert_held(q); 534 535 if (!test_bit(flag, &q->queue_flags)) { 536 __set_bit(flag, &q->queue_flags); 537 return 0; 538 } 539 540 return 1; 541 } 542 543 static inline void queue_flag_set(unsigned int flag, struct request_queue *q) 544 { 545 queue_lockdep_assert_held(q); 546 __set_bit(flag, &q->queue_flags); 547 } 548 549 static inline void queue_flag_clear_unlocked(unsigned int flag, 550 struct request_queue *q) 551 { 552 __clear_bit(flag, &q->queue_flags); 553 } 554 555 static inline int queue_in_flight(struct request_queue *q) 556 { 557 return q->in_flight[0] + q->in_flight[1]; 558 } 559 560 static inline void queue_flag_clear(unsigned int flag, struct request_queue *q) 561 { 562 queue_lockdep_assert_held(q); 563 __clear_bit(flag, &q->queue_flags); 564 } 565 566 #define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags) 567 #define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags) 568 #define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags) 569 #define blk_queue_dead(q) test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags) 570 #define blk_queue_bypass(q) test_bit(QUEUE_FLAG_BYPASS, &(q)->queue_flags) 571 #define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags) 572 #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags) 573 #define blk_queue_noxmerges(q) \ 574 test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags) 575 #define blk_queue_nonrot(q) test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags) 576 #define blk_queue_io_stat(q) test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags) 577 #define blk_queue_add_random(q) test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags) 578 #define blk_queue_stackable(q) \ 579 test_bit(QUEUE_FLAG_STACKABLE, &(q)->queue_flags) 580 #define blk_queue_discard(q) test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags) 581 #define blk_queue_secdiscard(q) (blk_queue_discard(q) && \ 582 test_bit(QUEUE_FLAG_SECDISCARD, &(q)->queue_flags)) 583 584 #define blk_noretry_request(rq) \ 585 ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \ 586 REQ_FAILFAST_DRIVER)) 587 588 #define blk_account_rq(rq) \ 589 (((rq)->cmd_flags & REQ_STARTED) && \ 590 ((rq)->cmd_type == REQ_TYPE_FS)) 591 592 #define blk_rq_cpu_valid(rq) ((rq)->cpu != -1) 593 #define blk_bidi_rq(rq) ((rq)->next_rq != NULL) 594 /* rq->queuelist of dequeued request must be list_empty() */ 595 #define blk_queued_rq(rq) (!list_empty(&(rq)->queuelist)) 596 597 #define list_entry_rq(ptr) list_entry((ptr), struct request, queuelist) 598 599 #define rq_data_dir(rq) ((int)((rq)->cmd_flags & 1)) 600 601 /* 602 * Driver can handle struct request, if it either has an old style 603 * request_fn defined, or is blk-mq based. 604 */ 605 static inline bool queue_is_rq_based(struct request_queue *q) 606 { 607 return q->request_fn || q->mq_ops; 608 } 609 610 static inline unsigned int blk_queue_cluster(struct request_queue *q) 611 { 612 return q->limits.cluster; 613 } 614 615 /* 616 * We regard a request as sync, if either a read or a sync write 617 */ 618 static inline bool rw_is_sync(unsigned int rw_flags) 619 { 620 return !(rw_flags & REQ_WRITE) || (rw_flags & REQ_SYNC); 621 } 622 623 static inline bool rq_is_sync(struct request *rq) 624 { 625 return rw_is_sync(rq->cmd_flags); 626 } 627 628 static inline bool blk_rl_full(struct request_list *rl, bool sync) 629 { 630 unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL; 631 632 return rl->flags & flag; 633 } 634 635 static inline void blk_set_rl_full(struct request_list *rl, bool sync) 636 { 637 unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL; 638 639 rl->flags |= flag; 640 } 641 642 static inline void blk_clear_rl_full(struct request_list *rl, bool sync) 643 { 644 unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL; 645 646 rl->flags &= ~flag; 647 } 648 649 static inline bool rq_mergeable(struct request *rq) 650 { 651 if (rq->cmd_type != REQ_TYPE_FS) 652 return false; 653 654 if (rq->cmd_flags & REQ_NOMERGE_FLAGS) 655 return false; 656 657 return true; 658 } 659 660 static inline bool blk_check_merge_flags(unsigned int flags1, 661 unsigned int flags2) 662 { 663 if ((flags1 & REQ_DISCARD) != (flags2 & REQ_DISCARD)) 664 return false; 665 666 if ((flags1 & REQ_SECURE) != (flags2 & REQ_SECURE)) 667 return false; 668 669 if ((flags1 & REQ_WRITE_SAME) != (flags2 & REQ_WRITE_SAME)) 670 return false; 671 672 return true; 673 } 674 675 static inline bool blk_write_same_mergeable(struct bio *a, struct bio *b) 676 { 677 if (bio_data(a) == bio_data(b)) 678 return true; 679 680 return false; 681 } 682 683 /* 684 * q->prep_rq_fn return values 685 */ 686 enum { 687 BLKPREP_OK, /* serve it */ 688 BLKPREP_KILL, /* fatal error, kill, return -EIO */ 689 BLKPREP_DEFER, /* leave on queue */ 690 BLKPREP_INVALID, /* invalid command, kill, return -EREMOTEIO */ 691 }; 692 693 extern unsigned long blk_max_low_pfn, blk_max_pfn; 694 695 /* 696 * standard bounce addresses: 697 * 698 * BLK_BOUNCE_HIGH : bounce all highmem pages 699 * BLK_BOUNCE_ANY : don't bounce anything 700 * BLK_BOUNCE_ISA : bounce pages above ISA DMA boundary 701 */ 702 703 #if BITS_PER_LONG == 32 704 #define BLK_BOUNCE_HIGH ((u64)blk_max_low_pfn << PAGE_SHIFT) 705 #else 706 #define BLK_BOUNCE_HIGH -1ULL 707 #endif 708 #define BLK_BOUNCE_ANY (-1ULL) 709 #define BLK_BOUNCE_ISA (DMA_BIT_MASK(24)) 710 711 /* 712 * default timeout for SG_IO if none specified 713 */ 714 #define BLK_DEFAULT_SG_TIMEOUT (60 * HZ) 715 #define BLK_MIN_SG_TIMEOUT (7 * HZ) 716 717 #ifdef CONFIG_BOUNCE 718 extern int init_emergency_isa_pool(void); 719 extern void blk_queue_bounce(struct request_queue *q, struct bio **bio); 720 #else 721 static inline int init_emergency_isa_pool(void) 722 { 723 return 0; 724 } 725 static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio) 726 { 727 } 728 #endif /* CONFIG_MMU */ 729 730 struct rq_map_data { 731 struct page **pages; 732 int page_order; 733 int nr_entries; 734 unsigned long offset; 735 int null_mapped; 736 int from_user; 737 }; 738 739 struct req_iterator { 740 struct bvec_iter iter; 741 struct bio *bio; 742 }; 743 744 /* This should not be used directly - use rq_for_each_segment */ 745 #define for_each_bio(_bio) \ 746 for (; _bio; _bio = _bio->bi_next) 747 #define __rq_for_each_bio(_bio, rq) \ 748 if ((rq->bio)) \ 749 for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next) 750 751 #define rq_for_each_segment(bvl, _rq, _iter) \ 752 __rq_for_each_bio(_iter.bio, _rq) \ 753 bio_for_each_segment(bvl, _iter.bio, _iter.iter) 754 755 #define rq_iter_last(bvec, _iter) \ 756 (_iter.bio->bi_next == NULL && \ 757 bio_iter_last(bvec, _iter.iter)) 758 759 #ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 760 # error "You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform" 761 #endif 762 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 763 extern void rq_flush_dcache_pages(struct request *rq); 764 #else 765 static inline void rq_flush_dcache_pages(struct request *rq) 766 { 767 } 768 #endif 769 770 extern int blk_register_queue(struct gendisk *disk); 771 extern void blk_unregister_queue(struct gendisk *disk); 772 extern blk_qc_t generic_make_request(struct bio *bio); 773 extern void blk_rq_init(struct request_queue *q, struct request *rq); 774 extern void blk_put_request(struct request *); 775 extern void __blk_put_request(struct request_queue *, struct request *); 776 extern struct request *blk_get_request(struct request_queue *, int, gfp_t); 777 extern struct request *blk_make_request(struct request_queue *, struct bio *, 778 gfp_t); 779 extern void blk_rq_set_block_pc(struct request *); 780 extern void blk_requeue_request(struct request_queue *, struct request *); 781 extern void blk_add_request_payload(struct request *rq, struct page *page, 782 unsigned int len); 783 extern int blk_lld_busy(struct request_queue *q); 784 extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src, 785 struct bio_set *bs, gfp_t gfp_mask, 786 int (*bio_ctr)(struct bio *, struct bio *, void *), 787 void *data); 788 extern void blk_rq_unprep_clone(struct request *rq); 789 extern int blk_insert_cloned_request(struct request_queue *q, 790 struct request *rq); 791 extern void blk_delay_queue(struct request_queue *, unsigned long); 792 extern void blk_queue_split(struct request_queue *, struct bio **, 793 struct bio_set *); 794 extern void blk_recount_segments(struct request_queue *, struct bio *); 795 extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int); 796 extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t, 797 unsigned int, void __user *); 798 extern int scsi_cmd_ioctl(struct request_queue *, struct gendisk *, fmode_t, 799 unsigned int, void __user *); 800 extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t, 801 struct scsi_ioctl_command __user *); 802 803 extern int blk_queue_enter(struct request_queue *q, bool nowait); 804 extern void blk_queue_exit(struct request_queue *q); 805 extern void blk_start_queue(struct request_queue *q); 806 extern void blk_start_queue_async(struct request_queue *q); 807 extern void blk_stop_queue(struct request_queue *q); 808 extern void blk_sync_queue(struct request_queue *q); 809 extern void __blk_stop_queue(struct request_queue *q); 810 extern void __blk_run_queue(struct request_queue *q); 811 extern void __blk_run_queue_uncond(struct request_queue *q); 812 extern void blk_run_queue(struct request_queue *); 813 extern void blk_run_queue_async(struct request_queue *q); 814 extern int blk_rq_map_user(struct request_queue *, struct request *, 815 struct rq_map_data *, void __user *, unsigned long, 816 gfp_t); 817 extern int blk_rq_unmap_user(struct bio *); 818 extern int blk_rq_map_kern(struct request_queue *, struct request *, void *, unsigned int, gfp_t); 819 extern int blk_rq_map_user_iov(struct request_queue *, struct request *, 820 struct rq_map_data *, const struct iov_iter *, 821 gfp_t); 822 extern int blk_execute_rq(struct request_queue *, struct gendisk *, 823 struct request *, int); 824 extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *, 825 struct request *, int, rq_end_io_fn *); 826 827 bool blk_poll(struct request_queue *q, blk_qc_t cookie); 828 829 static inline struct request_queue *bdev_get_queue(struct block_device *bdev) 830 { 831 return bdev->bd_disk->queue; /* this is never NULL */ 832 } 833 834 /* 835 * blk_rq_pos() : the current sector 836 * blk_rq_bytes() : bytes left in the entire request 837 * blk_rq_cur_bytes() : bytes left in the current segment 838 * blk_rq_err_bytes() : bytes left till the next error boundary 839 * blk_rq_sectors() : sectors left in the entire request 840 * blk_rq_cur_sectors() : sectors left in the current segment 841 */ 842 static inline sector_t blk_rq_pos(const struct request *rq) 843 { 844 return rq->__sector; 845 } 846 847 static inline unsigned int blk_rq_bytes(const struct request *rq) 848 { 849 return rq->__data_len; 850 } 851 852 static inline int blk_rq_cur_bytes(const struct request *rq) 853 { 854 return rq->bio ? bio_cur_bytes(rq->bio) : 0; 855 } 856 857 extern unsigned int blk_rq_err_bytes(const struct request *rq); 858 859 static inline unsigned int blk_rq_sectors(const struct request *rq) 860 { 861 return blk_rq_bytes(rq) >> 9; 862 } 863 864 static inline unsigned int blk_rq_cur_sectors(const struct request *rq) 865 { 866 return blk_rq_cur_bytes(rq) >> 9; 867 } 868 869 static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q, 870 unsigned int cmd_flags) 871 { 872 if (unlikely(cmd_flags & REQ_DISCARD)) 873 return min(q->limits.max_discard_sectors, UINT_MAX >> 9); 874 875 if (unlikely(cmd_flags & REQ_WRITE_SAME)) 876 return q->limits.max_write_same_sectors; 877 878 return q->limits.max_sectors; 879 } 880 881 /* 882 * Return maximum size of a request at given offset. Only valid for 883 * file system requests. 884 */ 885 static inline unsigned int blk_max_size_offset(struct request_queue *q, 886 sector_t offset) 887 { 888 if (!q->limits.chunk_sectors) 889 return q->limits.max_sectors; 890 891 return q->limits.chunk_sectors - 892 (offset & (q->limits.chunk_sectors - 1)); 893 } 894 895 static inline unsigned int blk_rq_get_max_sectors(struct request *rq) 896 { 897 struct request_queue *q = rq->q; 898 899 if (unlikely(rq->cmd_type != REQ_TYPE_FS)) 900 return q->limits.max_hw_sectors; 901 902 if (!q->limits.chunk_sectors || (rq->cmd_flags & REQ_DISCARD)) 903 return blk_queue_get_max_sectors(q, rq->cmd_flags); 904 905 return min(blk_max_size_offset(q, blk_rq_pos(rq)), 906 blk_queue_get_max_sectors(q, rq->cmd_flags)); 907 } 908 909 static inline unsigned int blk_rq_count_bios(struct request *rq) 910 { 911 unsigned int nr_bios = 0; 912 struct bio *bio; 913 914 __rq_for_each_bio(bio, rq) 915 nr_bios++; 916 917 return nr_bios; 918 } 919 920 /* 921 * Request issue related functions. 922 */ 923 extern struct request *blk_peek_request(struct request_queue *q); 924 extern void blk_start_request(struct request *rq); 925 extern struct request *blk_fetch_request(struct request_queue *q); 926 927 /* 928 * Request completion related functions. 929 * 930 * blk_update_request() completes given number of bytes and updates 931 * the request without completing it. 932 * 933 * blk_end_request() and friends. __blk_end_request() must be called 934 * with the request queue spinlock acquired. 935 * 936 * Several drivers define their own end_request and call 937 * blk_end_request() for parts of the original function. 938 * This prevents code duplication in drivers. 939 */ 940 extern bool blk_update_request(struct request *rq, int error, 941 unsigned int nr_bytes); 942 extern void blk_finish_request(struct request *rq, int error); 943 extern bool blk_end_request(struct request *rq, int error, 944 unsigned int nr_bytes); 945 extern void blk_end_request_all(struct request *rq, int error); 946 extern bool blk_end_request_cur(struct request *rq, int error); 947 extern bool blk_end_request_err(struct request *rq, int error); 948 extern bool __blk_end_request(struct request *rq, int error, 949 unsigned int nr_bytes); 950 extern void __blk_end_request_all(struct request *rq, int error); 951 extern bool __blk_end_request_cur(struct request *rq, int error); 952 extern bool __blk_end_request_err(struct request *rq, int error); 953 954 extern void blk_complete_request(struct request *); 955 extern void __blk_complete_request(struct request *); 956 extern void blk_abort_request(struct request *); 957 extern void blk_unprep_request(struct request *); 958 959 /* 960 * Access functions for manipulating queue properties 961 */ 962 extern struct request_queue *blk_init_queue_node(request_fn_proc *rfn, 963 spinlock_t *lock, int node_id); 964 extern struct request_queue *blk_init_queue(request_fn_proc *, spinlock_t *); 965 extern struct request_queue *blk_init_allocated_queue(struct request_queue *, 966 request_fn_proc *, spinlock_t *); 967 extern void blk_cleanup_queue(struct request_queue *); 968 extern void blk_queue_make_request(struct request_queue *, make_request_fn *); 969 extern void blk_queue_bounce_limit(struct request_queue *, u64); 970 extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int); 971 extern void blk_queue_chunk_sectors(struct request_queue *, unsigned int); 972 extern void blk_queue_max_segments(struct request_queue *, unsigned short); 973 extern void blk_queue_max_segment_size(struct request_queue *, unsigned int); 974 extern void blk_queue_max_discard_sectors(struct request_queue *q, 975 unsigned int max_discard_sectors); 976 extern void blk_queue_max_write_same_sectors(struct request_queue *q, 977 unsigned int max_write_same_sectors); 978 extern void blk_queue_logical_block_size(struct request_queue *, unsigned short); 979 extern void blk_queue_physical_block_size(struct request_queue *, unsigned int); 980 extern void blk_queue_alignment_offset(struct request_queue *q, 981 unsigned int alignment); 982 extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min); 983 extern void blk_queue_io_min(struct request_queue *q, unsigned int min); 984 extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt); 985 extern void blk_queue_io_opt(struct request_queue *q, unsigned int opt); 986 extern void blk_set_default_limits(struct queue_limits *lim); 987 extern void blk_set_stacking_limits(struct queue_limits *lim); 988 extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, 989 sector_t offset); 990 extern int bdev_stack_limits(struct queue_limits *t, struct block_device *bdev, 991 sector_t offset); 992 extern void disk_stack_limits(struct gendisk *disk, struct block_device *bdev, 993 sector_t offset); 994 extern void blk_queue_stack_limits(struct request_queue *t, struct request_queue *b); 995 extern void blk_queue_dma_pad(struct request_queue *, unsigned int); 996 extern void blk_queue_update_dma_pad(struct request_queue *, unsigned int); 997 extern int blk_queue_dma_drain(struct request_queue *q, 998 dma_drain_needed_fn *dma_drain_needed, 999 void *buf, unsigned int size); 1000 extern void blk_queue_lld_busy(struct request_queue *q, lld_busy_fn *fn); 1001 extern void blk_queue_segment_boundary(struct request_queue *, unsigned long); 1002 extern void blk_queue_virt_boundary(struct request_queue *, unsigned long); 1003 extern void blk_queue_prep_rq(struct request_queue *, prep_rq_fn *pfn); 1004 extern void blk_queue_unprep_rq(struct request_queue *, unprep_rq_fn *ufn); 1005 extern void blk_queue_dma_alignment(struct request_queue *, int); 1006 extern void blk_queue_update_dma_alignment(struct request_queue *, int); 1007 extern void blk_queue_softirq_done(struct request_queue *, softirq_done_fn *); 1008 extern void blk_queue_rq_timed_out(struct request_queue *, rq_timed_out_fn *); 1009 extern void blk_queue_rq_timeout(struct request_queue *, unsigned int); 1010 extern void blk_queue_flush(struct request_queue *q, unsigned int flush); 1011 extern void blk_queue_flush_queueable(struct request_queue *q, bool queueable); 1012 extern struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev); 1013 1014 extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *); 1015 extern void blk_dump_rq_flags(struct request *, char *); 1016 extern long nr_blockdev_pages(void); 1017 1018 bool __must_check blk_get_queue(struct request_queue *); 1019 struct request_queue *blk_alloc_queue(gfp_t); 1020 struct request_queue *blk_alloc_queue_node(gfp_t, int); 1021 extern void blk_put_queue(struct request_queue *); 1022 extern void blk_set_queue_dying(struct request_queue *); 1023 1024 /* 1025 * block layer runtime pm functions 1026 */ 1027 #ifdef CONFIG_PM 1028 extern void blk_pm_runtime_init(struct request_queue *q, struct device *dev); 1029 extern int blk_pre_runtime_suspend(struct request_queue *q); 1030 extern void blk_post_runtime_suspend(struct request_queue *q, int err); 1031 extern void blk_pre_runtime_resume(struct request_queue *q); 1032 extern void blk_post_runtime_resume(struct request_queue *q, int err); 1033 extern void blk_set_runtime_active(struct request_queue *q); 1034 #else 1035 static inline void blk_pm_runtime_init(struct request_queue *q, 1036 struct device *dev) {} 1037 static inline int blk_pre_runtime_suspend(struct request_queue *q) 1038 { 1039 return -ENOSYS; 1040 } 1041 static inline void blk_post_runtime_suspend(struct request_queue *q, int err) {} 1042 static inline void blk_pre_runtime_resume(struct request_queue *q) {} 1043 static inline void blk_post_runtime_resume(struct request_queue *q, int err) {} 1044 extern inline void blk_set_runtime_active(struct request_queue *q) {} 1045 #endif 1046 1047 /* 1048 * blk_plug permits building a queue of related requests by holding the I/O 1049 * fragments for a short period. This allows merging of sequential requests 1050 * into single larger request. As the requests are moved from a per-task list to 1051 * the device's request_queue in a batch, this results in improved scalability 1052 * as the lock contention for request_queue lock is reduced. 1053 * 1054 * It is ok not to disable preemption when adding the request to the plug list 1055 * or when attempting a merge, because blk_schedule_flush_list() will only flush 1056 * the plug list when the task sleeps by itself. For details, please see 1057 * schedule() where blk_schedule_flush_plug() is called. 1058 */ 1059 struct blk_plug { 1060 struct list_head list; /* requests */ 1061 struct list_head mq_list; /* blk-mq requests */ 1062 struct list_head cb_list; /* md requires an unplug callback */ 1063 }; 1064 #define BLK_MAX_REQUEST_COUNT 16 1065 1066 struct blk_plug_cb; 1067 typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool); 1068 struct blk_plug_cb { 1069 struct list_head list; 1070 blk_plug_cb_fn callback; 1071 void *data; 1072 }; 1073 extern struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, 1074 void *data, int size); 1075 extern void blk_start_plug(struct blk_plug *); 1076 extern void blk_finish_plug(struct blk_plug *); 1077 extern void blk_flush_plug_list(struct blk_plug *, bool); 1078 1079 static inline void blk_flush_plug(struct task_struct *tsk) 1080 { 1081 struct blk_plug *plug = tsk->plug; 1082 1083 if (plug) 1084 blk_flush_plug_list(plug, false); 1085 } 1086 1087 static inline void blk_schedule_flush_plug(struct task_struct *tsk) 1088 { 1089 struct blk_plug *plug = tsk->plug; 1090 1091 if (plug) 1092 blk_flush_plug_list(plug, true); 1093 } 1094 1095 static inline bool blk_needs_flush_plug(struct task_struct *tsk) 1096 { 1097 struct blk_plug *plug = tsk->plug; 1098 1099 return plug && 1100 (!list_empty(&plug->list) || 1101 !list_empty(&plug->mq_list) || 1102 !list_empty(&plug->cb_list)); 1103 } 1104 1105 /* 1106 * tag stuff 1107 */ 1108 extern int blk_queue_start_tag(struct request_queue *, struct request *); 1109 extern struct request *blk_queue_find_tag(struct request_queue *, int); 1110 extern void blk_queue_end_tag(struct request_queue *, struct request *); 1111 extern int blk_queue_init_tags(struct request_queue *, int, struct blk_queue_tag *, int); 1112 extern void blk_queue_free_tags(struct request_queue *); 1113 extern int blk_queue_resize_tags(struct request_queue *, int); 1114 extern void blk_queue_invalidate_tags(struct request_queue *); 1115 extern struct blk_queue_tag *blk_init_tags(int, int); 1116 extern void blk_free_tags(struct blk_queue_tag *); 1117 1118 static inline struct request *blk_map_queue_find_tag(struct blk_queue_tag *bqt, 1119 int tag) 1120 { 1121 if (unlikely(bqt == NULL || tag >= bqt->real_max_depth)) 1122 return NULL; 1123 return bqt->tag_index[tag]; 1124 } 1125 1126 #define BLKDEV_DISCARD_SECURE 0x01 /* secure discard */ 1127 1128 extern int blkdev_issue_flush(struct block_device *, gfp_t, sector_t *); 1129 extern int blkdev_issue_discard(struct block_device *bdev, sector_t sector, 1130 sector_t nr_sects, gfp_t gfp_mask, unsigned long flags); 1131 extern int blkdev_issue_write_same(struct block_device *bdev, sector_t sector, 1132 sector_t nr_sects, gfp_t gfp_mask, struct page *page); 1133 extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 1134 sector_t nr_sects, gfp_t gfp_mask, bool discard); 1135 static inline int sb_issue_discard(struct super_block *sb, sector_t block, 1136 sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags) 1137 { 1138 return blkdev_issue_discard(sb->s_bdev, block << (sb->s_blocksize_bits - 9), 1139 nr_blocks << (sb->s_blocksize_bits - 9), 1140 gfp_mask, flags); 1141 } 1142 static inline int sb_issue_zeroout(struct super_block *sb, sector_t block, 1143 sector_t nr_blocks, gfp_t gfp_mask) 1144 { 1145 return blkdev_issue_zeroout(sb->s_bdev, 1146 block << (sb->s_blocksize_bits - 9), 1147 nr_blocks << (sb->s_blocksize_bits - 9), 1148 gfp_mask, true); 1149 } 1150 1151 extern int blk_verify_command(unsigned char *cmd, fmode_t has_write_perm); 1152 1153 enum blk_default_limits { 1154 BLK_MAX_SEGMENTS = 128, 1155 BLK_SAFE_MAX_SECTORS = 255, 1156 BLK_DEF_MAX_SECTORS = 2560, 1157 BLK_MAX_SEGMENT_SIZE = 65536, 1158 BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL, 1159 }; 1160 1161 #define blkdev_entry_to_request(entry) list_entry((entry), struct request, queuelist) 1162 1163 static inline unsigned long queue_bounce_pfn(struct request_queue *q) 1164 { 1165 return q->limits.bounce_pfn; 1166 } 1167 1168 static inline unsigned long queue_segment_boundary(struct request_queue *q) 1169 { 1170 return q->limits.seg_boundary_mask; 1171 } 1172 1173 static inline unsigned long queue_virt_boundary(struct request_queue *q) 1174 { 1175 return q->limits.virt_boundary_mask; 1176 } 1177 1178 static inline unsigned int queue_max_sectors(struct request_queue *q) 1179 { 1180 return q->limits.max_sectors; 1181 } 1182 1183 static inline unsigned int queue_max_hw_sectors(struct request_queue *q) 1184 { 1185 return q->limits.max_hw_sectors; 1186 } 1187 1188 static inline unsigned short queue_max_segments(struct request_queue *q) 1189 { 1190 return q->limits.max_segments; 1191 } 1192 1193 static inline unsigned int queue_max_segment_size(struct request_queue *q) 1194 { 1195 return q->limits.max_segment_size; 1196 } 1197 1198 static inline unsigned short queue_logical_block_size(struct request_queue *q) 1199 { 1200 int retval = 512; 1201 1202 if (q && q->limits.logical_block_size) 1203 retval = q->limits.logical_block_size; 1204 1205 return retval; 1206 } 1207 1208 static inline unsigned short bdev_logical_block_size(struct block_device *bdev) 1209 { 1210 return queue_logical_block_size(bdev_get_queue(bdev)); 1211 } 1212 1213 static inline unsigned int queue_physical_block_size(struct request_queue *q) 1214 { 1215 return q->limits.physical_block_size; 1216 } 1217 1218 static inline unsigned int bdev_physical_block_size(struct block_device *bdev) 1219 { 1220 return queue_physical_block_size(bdev_get_queue(bdev)); 1221 } 1222 1223 static inline unsigned int queue_io_min(struct request_queue *q) 1224 { 1225 return q->limits.io_min; 1226 } 1227 1228 static inline int bdev_io_min(struct block_device *bdev) 1229 { 1230 return queue_io_min(bdev_get_queue(bdev)); 1231 } 1232 1233 static inline unsigned int queue_io_opt(struct request_queue *q) 1234 { 1235 return q->limits.io_opt; 1236 } 1237 1238 static inline int bdev_io_opt(struct block_device *bdev) 1239 { 1240 return queue_io_opt(bdev_get_queue(bdev)); 1241 } 1242 1243 static inline int queue_alignment_offset(struct request_queue *q) 1244 { 1245 if (q->limits.misaligned) 1246 return -1; 1247 1248 return q->limits.alignment_offset; 1249 } 1250 1251 static inline int queue_limit_alignment_offset(struct queue_limits *lim, sector_t sector) 1252 { 1253 unsigned int granularity = max(lim->physical_block_size, lim->io_min); 1254 unsigned int alignment = sector_div(sector, granularity >> 9) << 9; 1255 1256 return (granularity + lim->alignment_offset - alignment) % granularity; 1257 } 1258 1259 static inline int bdev_alignment_offset(struct block_device *bdev) 1260 { 1261 struct request_queue *q = bdev_get_queue(bdev); 1262 1263 if (q->limits.misaligned) 1264 return -1; 1265 1266 if (bdev != bdev->bd_contains) 1267 return bdev->bd_part->alignment_offset; 1268 1269 return q->limits.alignment_offset; 1270 } 1271 1272 static inline int queue_discard_alignment(struct request_queue *q) 1273 { 1274 if (q->limits.discard_misaligned) 1275 return -1; 1276 1277 return q->limits.discard_alignment; 1278 } 1279 1280 static inline int queue_limit_discard_alignment(struct queue_limits *lim, sector_t sector) 1281 { 1282 unsigned int alignment, granularity, offset; 1283 1284 if (!lim->max_discard_sectors) 1285 return 0; 1286 1287 /* Why are these in bytes, not sectors? */ 1288 alignment = lim->discard_alignment >> 9; 1289 granularity = lim->discard_granularity >> 9; 1290 if (!granularity) 1291 return 0; 1292 1293 /* Offset of the partition start in 'granularity' sectors */ 1294 offset = sector_div(sector, granularity); 1295 1296 /* And why do we do this modulus *again* in blkdev_issue_discard()? */ 1297 offset = (granularity + alignment - offset) % granularity; 1298 1299 /* Turn it back into bytes, gaah */ 1300 return offset << 9; 1301 } 1302 1303 static inline int bdev_discard_alignment(struct block_device *bdev) 1304 { 1305 struct request_queue *q = bdev_get_queue(bdev); 1306 1307 if (bdev != bdev->bd_contains) 1308 return bdev->bd_part->discard_alignment; 1309 1310 return q->limits.discard_alignment; 1311 } 1312 1313 static inline unsigned int queue_discard_zeroes_data(struct request_queue *q) 1314 { 1315 if (q->limits.max_discard_sectors && q->limits.discard_zeroes_data == 1) 1316 return 1; 1317 1318 return 0; 1319 } 1320 1321 static inline unsigned int bdev_discard_zeroes_data(struct block_device *bdev) 1322 { 1323 return queue_discard_zeroes_data(bdev_get_queue(bdev)); 1324 } 1325 1326 static inline unsigned int bdev_write_same(struct block_device *bdev) 1327 { 1328 struct request_queue *q = bdev_get_queue(bdev); 1329 1330 if (q) 1331 return q->limits.max_write_same_sectors; 1332 1333 return 0; 1334 } 1335 1336 static inline int queue_dma_alignment(struct request_queue *q) 1337 { 1338 return q ? q->dma_alignment : 511; 1339 } 1340 1341 static inline int blk_rq_aligned(struct request_queue *q, unsigned long addr, 1342 unsigned int len) 1343 { 1344 unsigned int alignment = queue_dma_alignment(q) | q->dma_pad_mask; 1345 return !(addr & alignment) && !(len & alignment); 1346 } 1347 1348 /* assumes size > 256 */ 1349 static inline unsigned int blksize_bits(unsigned int size) 1350 { 1351 unsigned int bits = 8; 1352 do { 1353 bits++; 1354 size >>= 1; 1355 } while (size > 256); 1356 return bits; 1357 } 1358 1359 static inline unsigned int block_size(struct block_device *bdev) 1360 { 1361 return bdev->bd_block_size; 1362 } 1363 1364 static inline bool queue_flush_queueable(struct request_queue *q) 1365 { 1366 return !q->flush_not_queueable; 1367 } 1368 1369 typedef struct {struct page *v;} Sector; 1370 1371 unsigned char *read_dev_sector(struct block_device *, sector_t, Sector *); 1372 1373 static inline void put_dev_sector(Sector p) 1374 { 1375 put_page(p.v); 1376 } 1377 1378 static inline bool __bvec_gap_to_prev(struct request_queue *q, 1379 struct bio_vec *bprv, unsigned int offset) 1380 { 1381 return offset || 1382 ((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q)); 1383 } 1384 1385 /* 1386 * Check if adding a bio_vec after bprv with offset would create a gap in 1387 * the SG list. Most drivers don't care about this, but some do. 1388 */ 1389 static inline bool bvec_gap_to_prev(struct request_queue *q, 1390 struct bio_vec *bprv, unsigned int offset) 1391 { 1392 if (!queue_virt_boundary(q)) 1393 return false; 1394 return __bvec_gap_to_prev(q, bprv, offset); 1395 } 1396 1397 static inline bool bio_will_gap(struct request_queue *q, struct bio *prev, 1398 struct bio *next) 1399 { 1400 if (bio_has_data(prev) && queue_virt_boundary(q)) { 1401 struct bio_vec pb, nb; 1402 1403 bio_get_last_bvec(prev, &pb); 1404 bio_get_first_bvec(next, &nb); 1405 1406 return __bvec_gap_to_prev(q, &pb, nb.bv_offset); 1407 } 1408 1409 return false; 1410 } 1411 1412 static inline bool req_gap_back_merge(struct request *req, struct bio *bio) 1413 { 1414 return bio_will_gap(req->q, req->biotail, bio); 1415 } 1416 1417 static inline bool req_gap_front_merge(struct request *req, struct bio *bio) 1418 { 1419 return bio_will_gap(req->q, bio, req->bio); 1420 } 1421 1422 struct work_struct; 1423 int kblockd_schedule_work(struct work_struct *work); 1424 int kblockd_schedule_delayed_work(struct delayed_work *dwork, unsigned long delay); 1425 int kblockd_schedule_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay); 1426 1427 #ifdef CONFIG_BLK_CGROUP 1428 /* 1429 * This should not be using sched_clock(). A real patch is in progress 1430 * to fix this up, until that is in place we need to disable preemption 1431 * around sched_clock() in this function and set_io_start_time_ns(). 1432 */ 1433 static inline void set_start_time_ns(struct request *req) 1434 { 1435 preempt_disable(); 1436 req->start_time_ns = sched_clock(); 1437 preempt_enable(); 1438 } 1439 1440 static inline void set_io_start_time_ns(struct request *req) 1441 { 1442 preempt_disable(); 1443 req->io_start_time_ns = sched_clock(); 1444 preempt_enable(); 1445 } 1446 1447 static inline uint64_t rq_start_time_ns(struct request *req) 1448 { 1449 return req->start_time_ns; 1450 } 1451 1452 static inline uint64_t rq_io_start_time_ns(struct request *req) 1453 { 1454 return req->io_start_time_ns; 1455 } 1456 #else 1457 static inline void set_start_time_ns(struct request *req) {} 1458 static inline void set_io_start_time_ns(struct request *req) {} 1459 static inline uint64_t rq_start_time_ns(struct request *req) 1460 { 1461 return 0; 1462 } 1463 static inline uint64_t rq_io_start_time_ns(struct request *req) 1464 { 1465 return 0; 1466 } 1467 #endif 1468 1469 #define MODULE_ALIAS_BLOCKDEV(major,minor) \ 1470 MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor)) 1471 #define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \ 1472 MODULE_ALIAS("block-major-" __stringify(major) "-*") 1473 1474 #if defined(CONFIG_BLK_DEV_INTEGRITY) 1475 1476 enum blk_integrity_flags { 1477 BLK_INTEGRITY_VERIFY = 1 << 0, 1478 BLK_INTEGRITY_GENERATE = 1 << 1, 1479 BLK_INTEGRITY_DEVICE_CAPABLE = 1 << 2, 1480 BLK_INTEGRITY_IP_CHECKSUM = 1 << 3, 1481 }; 1482 1483 struct blk_integrity_iter { 1484 void *prot_buf; 1485 void *data_buf; 1486 sector_t seed; 1487 unsigned int data_size; 1488 unsigned short interval; 1489 const char *disk_name; 1490 }; 1491 1492 typedef int (integrity_processing_fn) (struct blk_integrity_iter *); 1493 1494 struct blk_integrity_profile { 1495 integrity_processing_fn *generate_fn; 1496 integrity_processing_fn *verify_fn; 1497 const char *name; 1498 }; 1499 1500 extern void blk_integrity_register(struct gendisk *, struct blk_integrity *); 1501 extern void blk_integrity_unregister(struct gendisk *); 1502 extern int blk_integrity_compare(struct gendisk *, struct gendisk *); 1503 extern int blk_rq_map_integrity_sg(struct request_queue *, struct bio *, 1504 struct scatterlist *); 1505 extern int blk_rq_count_integrity_sg(struct request_queue *, struct bio *); 1506 extern bool blk_integrity_merge_rq(struct request_queue *, struct request *, 1507 struct request *); 1508 extern bool blk_integrity_merge_bio(struct request_queue *, struct request *, 1509 struct bio *); 1510 1511 static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk) 1512 { 1513 struct blk_integrity *bi = &disk->queue->integrity; 1514 1515 if (!bi->profile) 1516 return NULL; 1517 1518 return bi; 1519 } 1520 1521 static inline 1522 struct blk_integrity *bdev_get_integrity(struct block_device *bdev) 1523 { 1524 return blk_get_integrity(bdev->bd_disk); 1525 } 1526 1527 static inline bool blk_integrity_rq(struct request *rq) 1528 { 1529 return rq->cmd_flags & REQ_INTEGRITY; 1530 } 1531 1532 static inline void blk_queue_max_integrity_segments(struct request_queue *q, 1533 unsigned int segs) 1534 { 1535 q->limits.max_integrity_segments = segs; 1536 } 1537 1538 static inline unsigned short 1539 queue_max_integrity_segments(struct request_queue *q) 1540 { 1541 return q->limits.max_integrity_segments; 1542 } 1543 1544 static inline bool integrity_req_gap_back_merge(struct request *req, 1545 struct bio *next) 1546 { 1547 struct bio_integrity_payload *bip = bio_integrity(req->bio); 1548 struct bio_integrity_payload *bip_next = bio_integrity(next); 1549 1550 return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1], 1551 bip_next->bip_vec[0].bv_offset); 1552 } 1553 1554 static inline bool integrity_req_gap_front_merge(struct request *req, 1555 struct bio *bio) 1556 { 1557 struct bio_integrity_payload *bip = bio_integrity(bio); 1558 struct bio_integrity_payload *bip_next = bio_integrity(req->bio); 1559 1560 return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1], 1561 bip_next->bip_vec[0].bv_offset); 1562 } 1563 1564 #else /* CONFIG_BLK_DEV_INTEGRITY */ 1565 1566 struct bio; 1567 struct block_device; 1568 struct gendisk; 1569 struct blk_integrity; 1570 1571 static inline int blk_integrity_rq(struct request *rq) 1572 { 1573 return 0; 1574 } 1575 static inline int blk_rq_count_integrity_sg(struct request_queue *q, 1576 struct bio *b) 1577 { 1578 return 0; 1579 } 1580 static inline int blk_rq_map_integrity_sg(struct request_queue *q, 1581 struct bio *b, 1582 struct scatterlist *s) 1583 { 1584 return 0; 1585 } 1586 static inline struct blk_integrity *bdev_get_integrity(struct block_device *b) 1587 { 1588 return NULL; 1589 } 1590 static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk) 1591 { 1592 return NULL; 1593 } 1594 static inline int blk_integrity_compare(struct gendisk *a, struct gendisk *b) 1595 { 1596 return 0; 1597 } 1598 static inline void blk_integrity_register(struct gendisk *d, 1599 struct blk_integrity *b) 1600 { 1601 } 1602 static inline void blk_integrity_unregister(struct gendisk *d) 1603 { 1604 } 1605 static inline void blk_queue_max_integrity_segments(struct request_queue *q, 1606 unsigned int segs) 1607 { 1608 } 1609 static inline unsigned short queue_max_integrity_segments(struct request_queue *q) 1610 { 1611 return 0; 1612 } 1613 static inline bool blk_integrity_merge_rq(struct request_queue *rq, 1614 struct request *r1, 1615 struct request *r2) 1616 { 1617 return true; 1618 } 1619 static inline bool blk_integrity_merge_bio(struct request_queue *rq, 1620 struct request *r, 1621 struct bio *b) 1622 { 1623 return true; 1624 } 1625 1626 static inline bool integrity_req_gap_back_merge(struct request *req, 1627 struct bio *next) 1628 { 1629 return false; 1630 } 1631 static inline bool integrity_req_gap_front_merge(struct request *req, 1632 struct bio *bio) 1633 { 1634 return false; 1635 } 1636 1637 #endif /* CONFIG_BLK_DEV_INTEGRITY */ 1638 1639 /** 1640 * struct blk_dax_ctl - control and output parameters for ->direct_access 1641 * @sector: (input) offset relative to a block_device 1642 * @addr: (output) kernel virtual address for @sector populated by driver 1643 * @pfn: (output) page frame number for @addr populated by driver 1644 * @size: (input) number of bytes requested 1645 */ 1646 struct blk_dax_ctl { 1647 sector_t sector; 1648 void __pmem *addr; 1649 long size; 1650 pfn_t pfn; 1651 }; 1652 1653 struct block_device_operations { 1654 int (*open) (struct block_device *, fmode_t); 1655 void (*release) (struct gendisk *, fmode_t); 1656 int (*rw_page)(struct block_device *, sector_t, struct page *, int rw); 1657 int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); 1658 int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); 1659 long (*direct_access)(struct block_device *, sector_t, void __pmem **, 1660 pfn_t *); 1661 unsigned int (*check_events) (struct gendisk *disk, 1662 unsigned int clearing); 1663 /* ->media_changed() is DEPRECATED, use ->check_events() instead */ 1664 int (*media_changed) (struct gendisk *); 1665 void (*unlock_native_capacity) (struct gendisk *); 1666 int (*revalidate_disk) (struct gendisk *); 1667 int (*getgeo)(struct block_device *, struct hd_geometry *); 1668 /* this callback is with swap_lock and sometimes page table lock held */ 1669 void (*swap_slot_free_notify) (struct block_device *, unsigned long); 1670 struct module *owner; 1671 const struct pr_ops *pr_ops; 1672 }; 1673 1674 extern int __blkdev_driver_ioctl(struct block_device *, fmode_t, unsigned int, 1675 unsigned long); 1676 extern int bdev_read_page(struct block_device *, sector_t, struct page *); 1677 extern int bdev_write_page(struct block_device *, sector_t, struct page *, 1678 struct writeback_control *); 1679 extern long bdev_direct_access(struct block_device *, struct blk_dax_ctl *); 1680 #else /* CONFIG_BLOCK */ 1681 1682 struct block_device; 1683 1684 /* 1685 * stubs for when the block layer is configured out 1686 */ 1687 #define buffer_heads_over_limit 0 1688 1689 static inline long nr_blockdev_pages(void) 1690 { 1691 return 0; 1692 } 1693 1694 struct blk_plug { 1695 }; 1696 1697 static inline void blk_start_plug(struct blk_plug *plug) 1698 { 1699 } 1700 1701 static inline void blk_finish_plug(struct blk_plug *plug) 1702 { 1703 } 1704 1705 static inline void blk_flush_plug(struct task_struct *task) 1706 { 1707 } 1708 1709 static inline void blk_schedule_flush_plug(struct task_struct *task) 1710 { 1711 } 1712 1713 1714 static inline bool blk_needs_flush_plug(struct task_struct *tsk) 1715 { 1716 return false; 1717 } 1718 1719 static inline int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, 1720 sector_t *error_sector) 1721 { 1722 return 0; 1723 } 1724 1725 #endif /* CONFIG_BLOCK */ 1726 1727 #endif 1728