1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 1991, 1992 Linus Torvalds 4 * Copyright (C) 2001 Andrea Arcangeli <[email protected]> SuSE 5 * Copyright (C) 2016 - 2020 Christoph Hellwig 6 */ 7 8 #include <linux/init.h> 9 #include <linux/mm.h> 10 #include <linux/slab.h> 11 #include <linux/kmod.h> 12 #include <linux/major.h> 13 #include <linux/device_cgroup.h> 14 #include <linux/blkdev.h> 15 #include <linux/blk-integrity.h> 16 #include <linux/backing-dev.h> 17 #include <linux/module.h> 18 #include <linux/blkpg.h> 19 #include <linux/magic.h> 20 #include <linux/buffer_head.h> 21 #include <linux/swap.h> 22 #include <linux/writeback.h> 23 #include <linux/mount.h> 24 #include <linux/pseudo_fs.h> 25 #include <linux/uio.h> 26 #include <linux/namei.h> 27 #include <linux/part_stat.h> 28 #include <linux/uaccess.h> 29 #include <linux/stat.h> 30 #include "../fs/internal.h" 31 #include "blk.h" 32 33 /* Should we allow writing to mounted block devices? */ 34 static bool bdev_allow_write_mounted = IS_ENABLED(CONFIG_BLK_DEV_WRITE_MOUNTED); 35 36 struct bdev_inode { 37 struct block_device bdev; 38 struct inode vfs_inode; 39 }; 40 41 static inline struct bdev_inode *BDEV_I(struct inode *inode) 42 { 43 return container_of(inode, struct bdev_inode, vfs_inode); 44 } 45 46 struct block_device *I_BDEV(struct inode *inode) 47 { 48 return &BDEV_I(inode)->bdev; 49 } 50 EXPORT_SYMBOL(I_BDEV); 51 52 struct block_device *file_bdev(struct file *bdev_file) 53 { 54 return I_BDEV(bdev_file->f_mapping->host); 55 } 56 EXPORT_SYMBOL(file_bdev); 57 58 static void bdev_write_inode(struct block_device *bdev) 59 { 60 struct inode *inode = bdev->bd_inode; 61 int ret; 62 63 spin_lock(&inode->i_lock); 64 while (inode->i_state & I_DIRTY) { 65 spin_unlock(&inode->i_lock); 66 ret = write_inode_now(inode, true); 67 if (ret) 68 pr_warn_ratelimited( 69 "VFS: Dirty inode writeback failed for block device %pg (err=%d).\n", 70 bdev, ret); 71 spin_lock(&inode->i_lock); 72 } 73 spin_unlock(&inode->i_lock); 74 } 75 76 /* Kill _all_ buffers and pagecache , dirty or not.. */ 77 static void kill_bdev(struct block_device *bdev) 78 { 79 struct address_space *mapping = bdev->bd_inode->i_mapping; 80 81 if (mapping_empty(mapping)) 82 return; 83 84 invalidate_bh_lrus(); 85 truncate_inode_pages(mapping, 0); 86 } 87 88 /* Invalidate clean unused buffers and pagecache. */ 89 void invalidate_bdev(struct block_device *bdev) 90 { 91 struct address_space *mapping = bdev->bd_inode->i_mapping; 92 93 if (mapping->nrpages) { 94 invalidate_bh_lrus(); 95 lru_add_drain_all(); /* make sure all lru add caches are flushed */ 96 invalidate_mapping_pages(mapping, 0, -1); 97 } 98 } 99 EXPORT_SYMBOL(invalidate_bdev); 100 101 /* 102 * Drop all buffers & page cache for given bdev range. This function bails 103 * with error if bdev has other exclusive owner (such as filesystem). 104 */ 105 int truncate_bdev_range(struct block_device *bdev, blk_mode_t mode, 106 loff_t lstart, loff_t lend) 107 { 108 /* 109 * If we don't hold exclusive handle for the device, upgrade to it 110 * while we discard the buffer cache to avoid discarding buffers 111 * under live filesystem. 112 */ 113 if (!(mode & BLK_OPEN_EXCL)) { 114 int err = bd_prepare_to_claim(bdev, truncate_bdev_range, NULL); 115 if (err) 116 goto invalidate; 117 } 118 119 truncate_inode_pages_range(bdev->bd_inode->i_mapping, lstart, lend); 120 if (!(mode & BLK_OPEN_EXCL)) 121 bd_abort_claiming(bdev, truncate_bdev_range); 122 return 0; 123 124 invalidate: 125 /* 126 * Someone else has handle exclusively open. Try invalidating instead. 127 * The 'end' argument is inclusive so the rounding is safe. 128 */ 129 return invalidate_inode_pages2_range(bdev->bd_inode->i_mapping, 130 lstart >> PAGE_SHIFT, 131 lend >> PAGE_SHIFT); 132 } 133 134 static void set_init_blocksize(struct block_device *bdev) 135 { 136 unsigned int bsize = bdev_logical_block_size(bdev); 137 loff_t size = i_size_read(bdev->bd_inode); 138 139 while (bsize < PAGE_SIZE) { 140 if (size & bsize) 141 break; 142 bsize <<= 1; 143 } 144 bdev->bd_inode->i_blkbits = blksize_bits(bsize); 145 } 146 147 int set_blocksize(struct file *file, int size) 148 { 149 struct inode *inode = file->f_mapping->host; 150 struct block_device *bdev = I_BDEV(inode); 151 152 /* Size must be a power of two, and between 512 and PAGE_SIZE */ 153 if (size > PAGE_SIZE || size < 512 || !is_power_of_2(size)) 154 return -EINVAL; 155 156 /* Size cannot be smaller than the size supported by the device */ 157 if (size < bdev_logical_block_size(bdev)) 158 return -EINVAL; 159 160 if (!file->private_data) 161 return -EINVAL; 162 163 /* Don't change the size if it is same as current */ 164 if (inode->i_blkbits != blksize_bits(size)) { 165 sync_blockdev(bdev); 166 inode->i_blkbits = blksize_bits(size); 167 kill_bdev(bdev); 168 } 169 return 0; 170 } 171 172 EXPORT_SYMBOL(set_blocksize); 173 174 int sb_set_blocksize(struct super_block *sb, int size) 175 { 176 if (set_blocksize(sb->s_bdev_file, size)) 177 return 0; 178 /* If we get here, we know size is power of two 179 * and it's value is between 512 and PAGE_SIZE */ 180 sb->s_blocksize = size; 181 sb->s_blocksize_bits = blksize_bits(size); 182 return sb->s_blocksize; 183 } 184 185 EXPORT_SYMBOL(sb_set_blocksize); 186 187 int sb_min_blocksize(struct super_block *sb, int size) 188 { 189 int minsize = bdev_logical_block_size(sb->s_bdev); 190 if (size < minsize) 191 size = minsize; 192 return sb_set_blocksize(sb, size); 193 } 194 195 EXPORT_SYMBOL(sb_min_blocksize); 196 197 int sync_blockdev_nowait(struct block_device *bdev) 198 { 199 if (!bdev) 200 return 0; 201 return filemap_flush(bdev->bd_inode->i_mapping); 202 } 203 EXPORT_SYMBOL_GPL(sync_blockdev_nowait); 204 205 /* 206 * Write out and wait upon all the dirty data associated with a block 207 * device via its mapping. Does not take the superblock lock. 208 */ 209 int sync_blockdev(struct block_device *bdev) 210 { 211 if (!bdev) 212 return 0; 213 return filemap_write_and_wait(bdev->bd_inode->i_mapping); 214 } 215 EXPORT_SYMBOL(sync_blockdev); 216 217 int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend) 218 { 219 return filemap_write_and_wait_range(bdev->bd_inode->i_mapping, 220 lstart, lend); 221 } 222 EXPORT_SYMBOL(sync_blockdev_range); 223 224 /** 225 * bdev_freeze - lock a filesystem and force it into a consistent state 226 * @bdev: blockdevice to lock 227 * 228 * If a superblock is found on this device, we take the s_umount semaphore 229 * on it to make sure nobody unmounts until the snapshot creation is done. 230 * The reference counter (bd_fsfreeze_count) guarantees that only the last 231 * unfreeze process can unfreeze the frozen filesystem actually when multiple 232 * freeze requests arrive simultaneously. It counts up in bdev_freeze() and 233 * count down in bdev_thaw(). When it becomes 0, thaw_bdev() will unfreeze 234 * actually. 235 * 236 * Return: On success zero is returned, negative error code on failure. 237 */ 238 int bdev_freeze(struct block_device *bdev) 239 { 240 int error = 0; 241 242 mutex_lock(&bdev->bd_fsfreeze_mutex); 243 244 if (atomic_inc_return(&bdev->bd_fsfreeze_count) > 1) { 245 mutex_unlock(&bdev->bd_fsfreeze_mutex); 246 return 0; 247 } 248 249 mutex_lock(&bdev->bd_holder_lock); 250 if (bdev->bd_holder_ops && bdev->bd_holder_ops->freeze) { 251 error = bdev->bd_holder_ops->freeze(bdev); 252 lockdep_assert_not_held(&bdev->bd_holder_lock); 253 } else { 254 mutex_unlock(&bdev->bd_holder_lock); 255 error = sync_blockdev(bdev); 256 } 257 258 if (error) 259 atomic_dec(&bdev->bd_fsfreeze_count); 260 261 mutex_unlock(&bdev->bd_fsfreeze_mutex); 262 return error; 263 } 264 EXPORT_SYMBOL(bdev_freeze); 265 266 /** 267 * bdev_thaw - unlock filesystem 268 * @bdev: blockdevice to unlock 269 * 270 * Unlocks the filesystem and marks it writeable again after bdev_freeze(). 271 * 272 * Return: On success zero is returned, negative error code on failure. 273 */ 274 int bdev_thaw(struct block_device *bdev) 275 { 276 int error = -EINVAL, nr_freeze; 277 278 mutex_lock(&bdev->bd_fsfreeze_mutex); 279 280 /* 281 * If this returns < 0 it means that @bd_fsfreeze_count was 282 * already 0 and no decrement was performed. 283 */ 284 nr_freeze = atomic_dec_if_positive(&bdev->bd_fsfreeze_count); 285 if (nr_freeze < 0) 286 goto out; 287 288 error = 0; 289 if (nr_freeze > 0) 290 goto out; 291 292 mutex_lock(&bdev->bd_holder_lock); 293 if (bdev->bd_holder_ops && bdev->bd_holder_ops->thaw) { 294 error = bdev->bd_holder_ops->thaw(bdev); 295 lockdep_assert_not_held(&bdev->bd_holder_lock); 296 } else { 297 mutex_unlock(&bdev->bd_holder_lock); 298 } 299 300 if (error) 301 atomic_inc(&bdev->bd_fsfreeze_count); 302 out: 303 mutex_unlock(&bdev->bd_fsfreeze_mutex); 304 return error; 305 } 306 EXPORT_SYMBOL(bdev_thaw); 307 308 /* 309 * pseudo-fs 310 */ 311 312 static __cacheline_aligned_in_smp DEFINE_MUTEX(bdev_lock); 313 static struct kmem_cache *bdev_cachep __ro_after_init; 314 315 static struct inode *bdev_alloc_inode(struct super_block *sb) 316 { 317 struct bdev_inode *ei = alloc_inode_sb(sb, bdev_cachep, GFP_KERNEL); 318 319 if (!ei) 320 return NULL; 321 memset(&ei->bdev, 0, sizeof(ei->bdev)); 322 return &ei->vfs_inode; 323 } 324 325 static void bdev_free_inode(struct inode *inode) 326 { 327 struct block_device *bdev = I_BDEV(inode); 328 329 free_percpu(bdev->bd_stats); 330 kfree(bdev->bd_meta_info); 331 332 if (!bdev_is_partition(bdev)) { 333 if (bdev->bd_disk && bdev->bd_disk->bdi) 334 bdi_put(bdev->bd_disk->bdi); 335 kfree(bdev->bd_disk); 336 } 337 338 if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR) 339 blk_free_ext_minor(MINOR(bdev->bd_dev)); 340 341 kmem_cache_free(bdev_cachep, BDEV_I(inode)); 342 } 343 344 static void init_once(void *data) 345 { 346 struct bdev_inode *ei = data; 347 348 inode_init_once(&ei->vfs_inode); 349 } 350 351 static void bdev_evict_inode(struct inode *inode) 352 { 353 truncate_inode_pages_final(&inode->i_data); 354 invalidate_inode_buffers(inode); /* is it needed here? */ 355 clear_inode(inode); 356 } 357 358 static const struct super_operations bdev_sops = { 359 .statfs = simple_statfs, 360 .alloc_inode = bdev_alloc_inode, 361 .free_inode = bdev_free_inode, 362 .drop_inode = generic_delete_inode, 363 .evict_inode = bdev_evict_inode, 364 }; 365 366 static int bd_init_fs_context(struct fs_context *fc) 367 { 368 struct pseudo_fs_context *ctx = init_pseudo(fc, BDEVFS_MAGIC); 369 if (!ctx) 370 return -ENOMEM; 371 fc->s_iflags |= SB_I_CGROUPWB; 372 ctx->ops = &bdev_sops; 373 return 0; 374 } 375 376 static struct file_system_type bd_type = { 377 .name = "bdev", 378 .init_fs_context = bd_init_fs_context, 379 .kill_sb = kill_anon_super, 380 }; 381 382 struct super_block *blockdev_superblock __ro_after_init; 383 struct vfsmount *blockdev_mnt __ro_after_init; 384 EXPORT_SYMBOL_GPL(blockdev_superblock); 385 386 void __init bdev_cache_init(void) 387 { 388 int err; 389 390 bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode), 391 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| 392 SLAB_ACCOUNT|SLAB_PANIC), 393 init_once); 394 err = register_filesystem(&bd_type); 395 if (err) 396 panic("Cannot register bdev pseudo-fs"); 397 blockdev_mnt = kern_mount(&bd_type); 398 if (IS_ERR(blockdev_mnt)) 399 panic("Cannot create bdev pseudo-fs"); 400 blockdev_superblock = blockdev_mnt->mnt_sb; /* For writeback */ 401 } 402 403 struct block_device *bdev_alloc(struct gendisk *disk, u8 partno) 404 { 405 struct block_device *bdev; 406 struct inode *inode; 407 408 inode = new_inode(blockdev_superblock); 409 if (!inode) 410 return NULL; 411 inode->i_mode = S_IFBLK; 412 inode->i_rdev = 0; 413 inode->i_data.a_ops = &def_blk_aops; 414 mapping_set_gfp_mask(&inode->i_data, GFP_USER); 415 416 bdev = I_BDEV(inode); 417 mutex_init(&bdev->bd_fsfreeze_mutex); 418 spin_lock_init(&bdev->bd_size_lock); 419 mutex_init(&bdev->bd_holder_lock); 420 bdev->bd_partno = partno; 421 bdev->bd_inode = inode; 422 bdev->bd_mapping = &inode->i_data; 423 bdev->bd_queue = disk->queue; 424 if (partno) 425 bdev->bd_has_submit_bio = disk->part0->bd_has_submit_bio; 426 else 427 bdev->bd_has_submit_bio = false; 428 bdev->bd_stats = alloc_percpu(struct disk_stats); 429 if (!bdev->bd_stats) { 430 iput(inode); 431 return NULL; 432 } 433 bdev->bd_disk = disk; 434 return bdev; 435 } 436 437 void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors) 438 { 439 spin_lock(&bdev->bd_size_lock); 440 i_size_write(bdev->bd_inode, (loff_t)sectors << SECTOR_SHIFT); 441 bdev->bd_nr_sectors = sectors; 442 spin_unlock(&bdev->bd_size_lock); 443 } 444 445 void bdev_add(struct block_device *bdev, dev_t dev) 446 { 447 if (bdev_stable_writes(bdev)) 448 mapping_set_stable_writes(bdev->bd_inode->i_mapping); 449 bdev->bd_dev = dev; 450 bdev->bd_inode->i_rdev = dev; 451 bdev->bd_inode->i_ino = dev; 452 insert_inode_hash(bdev->bd_inode); 453 } 454 455 void bdev_unhash(struct block_device *bdev) 456 { 457 remove_inode_hash(bdev->bd_inode); 458 } 459 460 void bdev_drop(struct block_device *bdev) 461 { 462 iput(bdev->bd_inode); 463 } 464 465 long nr_blockdev_pages(void) 466 { 467 struct inode *inode; 468 long ret = 0; 469 470 spin_lock(&blockdev_superblock->s_inode_list_lock); 471 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) 472 ret += inode->i_mapping->nrpages; 473 spin_unlock(&blockdev_superblock->s_inode_list_lock); 474 475 return ret; 476 } 477 478 /** 479 * bd_may_claim - test whether a block device can be claimed 480 * @bdev: block device of interest 481 * @holder: holder trying to claim @bdev 482 * @hops: holder ops 483 * 484 * Test whether @bdev can be claimed by @holder. 485 * 486 * RETURNS: 487 * %true if @bdev can be claimed, %false otherwise. 488 */ 489 static bool bd_may_claim(struct block_device *bdev, void *holder, 490 const struct blk_holder_ops *hops) 491 { 492 struct block_device *whole = bdev_whole(bdev); 493 494 lockdep_assert_held(&bdev_lock); 495 496 if (bdev->bd_holder) { 497 /* 498 * The same holder can always re-claim. 499 */ 500 if (bdev->bd_holder == holder) { 501 if (WARN_ON_ONCE(bdev->bd_holder_ops != hops)) 502 return false; 503 return true; 504 } 505 return false; 506 } 507 508 /* 509 * If the whole devices holder is set to bd_may_claim, a partition on 510 * the device is claimed, but not the whole device. 511 */ 512 if (whole != bdev && 513 whole->bd_holder && whole->bd_holder != bd_may_claim) 514 return false; 515 return true; 516 } 517 518 /** 519 * bd_prepare_to_claim - claim a block device 520 * @bdev: block device of interest 521 * @holder: holder trying to claim @bdev 522 * @hops: holder ops. 523 * 524 * Claim @bdev. This function fails if @bdev is already claimed by another 525 * holder and waits if another claiming is in progress. return, the caller 526 * has ownership of bd_claiming and bd_holder[s]. 527 * 528 * RETURNS: 529 * 0 if @bdev can be claimed, -EBUSY otherwise. 530 */ 531 int bd_prepare_to_claim(struct block_device *bdev, void *holder, 532 const struct blk_holder_ops *hops) 533 { 534 struct block_device *whole = bdev_whole(bdev); 535 536 if (WARN_ON_ONCE(!holder)) 537 return -EINVAL; 538 retry: 539 mutex_lock(&bdev_lock); 540 /* if someone else claimed, fail */ 541 if (!bd_may_claim(bdev, holder, hops)) { 542 mutex_unlock(&bdev_lock); 543 return -EBUSY; 544 } 545 546 /* if claiming is already in progress, wait for it to finish */ 547 if (whole->bd_claiming) { 548 wait_queue_head_t *wq = bit_waitqueue(&whole->bd_claiming, 0); 549 DEFINE_WAIT(wait); 550 551 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE); 552 mutex_unlock(&bdev_lock); 553 schedule(); 554 finish_wait(wq, &wait); 555 goto retry; 556 } 557 558 /* yay, all mine */ 559 whole->bd_claiming = holder; 560 mutex_unlock(&bdev_lock); 561 return 0; 562 } 563 EXPORT_SYMBOL_GPL(bd_prepare_to_claim); /* only for the loop driver */ 564 565 static void bd_clear_claiming(struct block_device *whole, void *holder) 566 { 567 lockdep_assert_held(&bdev_lock); 568 /* tell others that we're done */ 569 BUG_ON(whole->bd_claiming != holder); 570 whole->bd_claiming = NULL; 571 wake_up_bit(&whole->bd_claiming, 0); 572 } 573 574 /** 575 * bd_finish_claiming - finish claiming of a block device 576 * @bdev: block device of interest 577 * @holder: holder that has claimed @bdev 578 * @hops: block device holder operations 579 * 580 * Finish exclusive open of a block device. Mark the device as exlusively 581 * open by the holder and wake up all waiters for exclusive open to finish. 582 */ 583 static void bd_finish_claiming(struct block_device *bdev, void *holder, 584 const struct blk_holder_ops *hops) 585 { 586 struct block_device *whole = bdev_whole(bdev); 587 588 mutex_lock(&bdev_lock); 589 BUG_ON(!bd_may_claim(bdev, holder, hops)); 590 /* 591 * Note that for a whole device bd_holders will be incremented twice, 592 * and bd_holder will be set to bd_may_claim before being set to holder 593 */ 594 whole->bd_holders++; 595 whole->bd_holder = bd_may_claim; 596 bdev->bd_holders++; 597 mutex_lock(&bdev->bd_holder_lock); 598 bdev->bd_holder = holder; 599 bdev->bd_holder_ops = hops; 600 mutex_unlock(&bdev->bd_holder_lock); 601 bd_clear_claiming(whole, holder); 602 mutex_unlock(&bdev_lock); 603 } 604 605 /** 606 * bd_abort_claiming - abort claiming of a block device 607 * @bdev: block device of interest 608 * @holder: holder that has claimed @bdev 609 * 610 * Abort claiming of a block device when the exclusive open failed. This can be 611 * also used when exclusive open is not actually desired and we just needed 612 * to block other exclusive openers for a while. 613 */ 614 void bd_abort_claiming(struct block_device *bdev, void *holder) 615 { 616 mutex_lock(&bdev_lock); 617 bd_clear_claiming(bdev_whole(bdev), holder); 618 mutex_unlock(&bdev_lock); 619 } 620 EXPORT_SYMBOL(bd_abort_claiming); 621 622 static void bd_end_claim(struct block_device *bdev, void *holder) 623 { 624 struct block_device *whole = bdev_whole(bdev); 625 bool unblock = false; 626 627 /* 628 * Release a claim on the device. The holder fields are protected with 629 * bdev_lock. open_mutex is used to synchronize disk_holder unlinking. 630 */ 631 mutex_lock(&bdev_lock); 632 WARN_ON_ONCE(bdev->bd_holder != holder); 633 WARN_ON_ONCE(--bdev->bd_holders < 0); 634 WARN_ON_ONCE(--whole->bd_holders < 0); 635 if (!bdev->bd_holders) { 636 mutex_lock(&bdev->bd_holder_lock); 637 bdev->bd_holder = NULL; 638 bdev->bd_holder_ops = NULL; 639 mutex_unlock(&bdev->bd_holder_lock); 640 if (bdev->bd_write_holder) 641 unblock = true; 642 } 643 if (!whole->bd_holders) 644 whole->bd_holder = NULL; 645 mutex_unlock(&bdev_lock); 646 647 /* 648 * If this was the last claim, remove holder link and unblock evpoll if 649 * it was a write holder. 650 */ 651 if (unblock) { 652 disk_unblock_events(bdev->bd_disk); 653 bdev->bd_write_holder = false; 654 } 655 } 656 657 static void blkdev_flush_mapping(struct block_device *bdev) 658 { 659 WARN_ON_ONCE(bdev->bd_holders); 660 sync_blockdev(bdev); 661 kill_bdev(bdev); 662 bdev_write_inode(bdev); 663 } 664 665 static int blkdev_get_whole(struct block_device *bdev, blk_mode_t mode) 666 { 667 struct gendisk *disk = bdev->bd_disk; 668 int ret; 669 670 if (disk->fops->open) { 671 ret = disk->fops->open(disk, mode); 672 if (ret) { 673 /* avoid ghost partitions on a removed medium */ 674 if (ret == -ENOMEDIUM && 675 test_bit(GD_NEED_PART_SCAN, &disk->state)) 676 bdev_disk_changed(disk, true); 677 return ret; 678 } 679 } 680 681 if (!atomic_read(&bdev->bd_openers)) 682 set_init_blocksize(bdev); 683 if (test_bit(GD_NEED_PART_SCAN, &disk->state)) 684 bdev_disk_changed(disk, false); 685 atomic_inc(&bdev->bd_openers); 686 return 0; 687 } 688 689 static void blkdev_put_whole(struct block_device *bdev) 690 { 691 if (atomic_dec_and_test(&bdev->bd_openers)) 692 blkdev_flush_mapping(bdev); 693 if (bdev->bd_disk->fops->release) 694 bdev->bd_disk->fops->release(bdev->bd_disk); 695 } 696 697 static int blkdev_get_part(struct block_device *part, blk_mode_t mode) 698 { 699 struct gendisk *disk = part->bd_disk; 700 int ret; 701 702 ret = blkdev_get_whole(bdev_whole(part), mode); 703 if (ret) 704 return ret; 705 706 ret = -ENXIO; 707 if (!bdev_nr_sectors(part)) 708 goto out_blkdev_put; 709 710 if (!atomic_read(&part->bd_openers)) { 711 disk->open_partitions++; 712 set_init_blocksize(part); 713 } 714 atomic_inc(&part->bd_openers); 715 return 0; 716 717 out_blkdev_put: 718 blkdev_put_whole(bdev_whole(part)); 719 return ret; 720 } 721 722 int bdev_permission(dev_t dev, blk_mode_t mode, void *holder) 723 { 724 int ret; 725 726 ret = devcgroup_check_permission(DEVCG_DEV_BLOCK, 727 MAJOR(dev), MINOR(dev), 728 ((mode & BLK_OPEN_READ) ? DEVCG_ACC_READ : 0) | 729 ((mode & BLK_OPEN_WRITE) ? DEVCG_ACC_WRITE : 0)); 730 if (ret) 731 return ret; 732 733 /* Blocking writes requires exclusive opener */ 734 if (mode & BLK_OPEN_RESTRICT_WRITES && !holder) 735 return -EINVAL; 736 737 /* 738 * We're using error pointers to indicate to ->release() when we 739 * failed to open that block device. Also this doesn't make sense. 740 */ 741 if (WARN_ON_ONCE(IS_ERR(holder))) 742 return -EINVAL; 743 744 return 0; 745 } 746 747 static void blkdev_put_part(struct block_device *part) 748 { 749 struct block_device *whole = bdev_whole(part); 750 751 if (atomic_dec_and_test(&part->bd_openers)) { 752 blkdev_flush_mapping(part); 753 whole->bd_disk->open_partitions--; 754 } 755 blkdev_put_whole(whole); 756 } 757 758 struct block_device *blkdev_get_no_open(dev_t dev) 759 { 760 struct block_device *bdev; 761 struct inode *inode; 762 763 inode = ilookup(blockdev_superblock, dev); 764 if (!inode && IS_ENABLED(CONFIG_BLOCK_LEGACY_AUTOLOAD)) { 765 blk_request_module(dev); 766 inode = ilookup(blockdev_superblock, dev); 767 if (inode) 768 pr_warn_ratelimited( 769 "block device autoloading is deprecated and will be removed.\n"); 770 } 771 if (!inode) 772 return NULL; 773 774 /* switch from the inode reference to a device mode one: */ 775 bdev = &BDEV_I(inode)->bdev; 776 if (!kobject_get_unless_zero(&bdev->bd_device.kobj)) 777 bdev = NULL; 778 iput(inode); 779 return bdev; 780 } 781 782 void blkdev_put_no_open(struct block_device *bdev) 783 { 784 put_device(&bdev->bd_device); 785 } 786 787 static bool bdev_writes_blocked(struct block_device *bdev) 788 { 789 return bdev->bd_writers < 0; 790 } 791 792 static void bdev_block_writes(struct block_device *bdev) 793 { 794 bdev->bd_writers--; 795 } 796 797 static void bdev_unblock_writes(struct block_device *bdev) 798 { 799 bdev->bd_writers++; 800 } 801 802 static bool bdev_may_open(struct block_device *bdev, blk_mode_t mode) 803 { 804 if (bdev_allow_write_mounted) 805 return true; 806 /* Writes blocked? */ 807 if (mode & BLK_OPEN_WRITE && bdev_writes_blocked(bdev)) 808 return false; 809 if (mode & BLK_OPEN_RESTRICT_WRITES && bdev->bd_writers > 0) 810 return false; 811 return true; 812 } 813 814 static void bdev_claim_write_access(struct block_device *bdev, blk_mode_t mode) 815 { 816 if (bdev_allow_write_mounted) 817 return; 818 819 /* Claim exclusive or shared write access. */ 820 if (mode & BLK_OPEN_RESTRICT_WRITES) 821 bdev_block_writes(bdev); 822 else if (mode & BLK_OPEN_WRITE) 823 bdev->bd_writers++; 824 } 825 826 static inline bool bdev_unclaimed(const struct file *bdev_file) 827 { 828 return bdev_file->private_data == BDEV_I(bdev_file->f_mapping->host); 829 } 830 831 static void bdev_yield_write_access(struct file *bdev_file) 832 { 833 struct block_device *bdev; 834 835 if (bdev_allow_write_mounted) 836 return; 837 838 if (bdev_unclaimed(bdev_file)) 839 return; 840 841 bdev = file_bdev(bdev_file); 842 843 if (bdev_file->f_mode & FMODE_WRITE_RESTRICTED) 844 bdev_unblock_writes(bdev); 845 else if (bdev_file->f_mode & FMODE_WRITE) 846 bdev->bd_writers--; 847 } 848 849 /** 850 * bdev_open - open a block device 851 * @bdev: block device to open 852 * @mode: open mode (BLK_OPEN_*) 853 * @holder: exclusive holder identifier 854 * @hops: holder operations 855 * @bdev_file: file for the block device 856 * 857 * Open the block device. If @holder is not %NULL, the block device is opened 858 * with exclusive access. Exclusive opens may nest for the same @holder. 859 * 860 * CONTEXT: 861 * Might sleep. 862 * 863 * RETURNS: 864 * zero on success, -errno on failure. 865 */ 866 int bdev_open(struct block_device *bdev, blk_mode_t mode, void *holder, 867 const struct blk_holder_ops *hops, struct file *bdev_file) 868 { 869 bool unblock_events = true; 870 struct gendisk *disk = bdev->bd_disk; 871 int ret; 872 873 if (holder) { 874 mode |= BLK_OPEN_EXCL; 875 ret = bd_prepare_to_claim(bdev, holder, hops); 876 if (ret) 877 return ret; 878 } else { 879 if (WARN_ON_ONCE(mode & BLK_OPEN_EXCL)) 880 return -EIO; 881 } 882 883 disk_block_events(disk); 884 885 mutex_lock(&disk->open_mutex); 886 ret = -ENXIO; 887 if (!disk_live(disk)) 888 goto abort_claiming; 889 if (!try_module_get(disk->fops->owner)) 890 goto abort_claiming; 891 ret = -EBUSY; 892 if (!bdev_may_open(bdev, mode)) 893 goto abort_claiming; 894 if (bdev_is_partition(bdev)) 895 ret = blkdev_get_part(bdev, mode); 896 else 897 ret = blkdev_get_whole(bdev, mode); 898 if (ret) 899 goto put_module; 900 bdev_claim_write_access(bdev, mode); 901 if (holder) { 902 bd_finish_claiming(bdev, holder, hops); 903 904 /* 905 * Block event polling for write claims if requested. Any write 906 * holder makes the write_holder state stick until all are 907 * released. This is good enough and tracking individual 908 * writeable reference is too fragile given the way @mode is 909 * used in blkdev_get/put(). 910 */ 911 if ((mode & BLK_OPEN_WRITE) && !bdev->bd_write_holder && 912 (disk->event_flags & DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE)) { 913 bdev->bd_write_holder = true; 914 unblock_events = false; 915 } 916 } 917 mutex_unlock(&disk->open_mutex); 918 919 if (unblock_events) 920 disk_unblock_events(disk); 921 922 bdev_file->f_flags |= O_LARGEFILE; 923 bdev_file->f_mode |= FMODE_BUF_RASYNC | FMODE_CAN_ODIRECT; 924 if (bdev_nowait(bdev)) 925 bdev_file->f_mode |= FMODE_NOWAIT; 926 if (mode & BLK_OPEN_RESTRICT_WRITES) 927 bdev_file->f_mode |= FMODE_WRITE_RESTRICTED; 928 bdev_file->f_mapping = bdev->bd_inode->i_mapping; 929 bdev_file->f_wb_err = filemap_sample_wb_err(bdev_file->f_mapping); 930 bdev_file->private_data = holder; 931 932 return 0; 933 put_module: 934 module_put(disk->fops->owner); 935 abort_claiming: 936 if (holder) 937 bd_abort_claiming(bdev, holder); 938 mutex_unlock(&disk->open_mutex); 939 disk_unblock_events(disk); 940 return ret; 941 } 942 943 /* 944 * If BLK_OPEN_WRITE_IOCTL is set then this is a historical quirk 945 * associated with the floppy driver where it has allowed ioctls if the 946 * file was opened for writing, but does not allow reads or writes. 947 * Make sure that this quirk is reflected in @f_flags. 948 * 949 * It can also happen if a block device is opened as O_RDWR | O_WRONLY. 950 */ 951 static unsigned blk_to_file_flags(blk_mode_t mode) 952 { 953 unsigned int flags = 0; 954 955 if ((mode & (BLK_OPEN_READ | BLK_OPEN_WRITE)) == 956 (BLK_OPEN_READ | BLK_OPEN_WRITE)) 957 flags |= O_RDWR; 958 else if (mode & BLK_OPEN_WRITE_IOCTL) 959 flags |= O_RDWR | O_WRONLY; 960 else if (mode & BLK_OPEN_WRITE) 961 flags |= O_WRONLY; 962 else if (mode & BLK_OPEN_READ) 963 flags |= O_RDONLY; /* homeopathic, because O_RDONLY is 0 */ 964 else 965 WARN_ON_ONCE(true); 966 967 if (mode & BLK_OPEN_NDELAY) 968 flags |= O_NDELAY; 969 970 return flags; 971 } 972 973 struct file *bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder, 974 const struct blk_holder_ops *hops) 975 { 976 struct file *bdev_file; 977 struct block_device *bdev; 978 unsigned int flags; 979 int ret; 980 981 ret = bdev_permission(dev, mode, holder); 982 if (ret) 983 return ERR_PTR(ret); 984 985 bdev = blkdev_get_no_open(dev); 986 if (!bdev) 987 return ERR_PTR(-ENXIO); 988 989 flags = blk_to_file_flags(mode); 990 bdev_file = alloc_file_pseudo_noaccount(bdev->bd_inode, 991 blockdev_mnt, "", flags | O_LARGEFILE, &def_blk_fops); 992 if (IS_ERR(bdev_file)) { 993 blkdev_put_no_open(bdev); 994 return bdev_file; 995 } 996 ihold(bdev->bd_inode); 997 998 ret = bdev_open(bdev, mode, holder, hops, bdev_file); 999 if (ret) { 1000 /* We failed to open the block device. Let ->release() know. */ 1001 bdev_file->private_data = ERR_PTR(ret); 1002 fput(bdev_file); 1003 return ERR_PTR(ret); 1004 } 1005 return bdev_file; 1006 } 1007 EXPORT_SYMBOL(bdev_file_open_by_dev); 1008 1009 struct file *bdev_file_open_by_path(const char *path, blk_mode_t mode, 1010 void *holder, 1011 const struct blk_holder_ops *hops) 1012 { 1013 struct file *file; 1014 dev_t dev; 1015 int error; 1016 1017 error = lookup_bdev(path, &dev); 1018 if (error) 1019 return ERR_PTR(error); 1020 1021 file = bdev_file_open_by_dev(dev, mode, holder, hops); 1022 if (!IS_ERR(file) && (mode & BLK_OPEN_WRITE)) { 1023 if (bdev_read_only(file_bdev(file))) { 1024 fput(file); 1025 file = ERR_PTR(-EACCES); 1026 } 1027 } 1028 1029 return file; 1030 } 1031 EXPORT_SYMBOL(bdev_file_open_by_path); 1032 1033 static inline void bd_yield_claim(struct file *bdev_file) 1034 { 1035 struct block_device *bdev = file_bdev(bdev_file); 1036 void *holder = bdev_file->private_data; 1037 1038 lockdep_assert_held(&bdev->bd_disk->open_mutex); 1039 1040 if (WARN_ON_ONCE(IS_ERR_OR_NULL(holder))) 1041 return; 1042 1043 if (!bdev_unclaimed(bdev_file)) 1044 bd_end_claim(bdev, holder); 1045 } 1046 1047 void bdev_release(struct file *bdev_file) 1048 { 1049 struct block_device *bdev = file_bdev(bdev_file); 1050 void *holder = bdev_file->private_data; 1051 struct gendisk *disk = bdev->bd_disk; 1052 1053 /* We failed to open that block device. */ 1054 if (IS_ERR(holder)) 1055 goto put_no_open; 1056 1057 /* 1058 * Sync early if it looks like we're the last one. If someone else 1059 * opens the block device between now and the decrement of bd_openers 1060 * then we did a sync that we didn't need to, but that's not the end 1061 * of the world and we want to avoid long (could be several minute) 1062 * syncs while holding the mutex. 1063 */ 1064 if (atomic_read(&bdev->bd_openers) == 1) 1065 sync_blockdev(bdev); 1066 1067 mutex_lock(&disk->open_mutex); 1068 bdev_yield_write_access(bdev_file); 1069 1070 if (holder) 1071 bd_yield_claim(bdev_file); 1072 1073 /* 1074 * Trigger event checking and tell drivers to flush MEDIA_CHANGE 1075 * event. This is to ensure detection of media removal commanded 1076 * from userland - e.g. eject(1). 1077 */ 1078 disk_flush_events(disk, DISK_EVENT_MEDIA_CHANGE); 1079 1080 if (bdev_is_partition(bdev)) 1081 blkdev_put_part(bdev); 1082 else 1083 blkdev_put_whole(bdev); 1084 mutex_unlock(&disk->open_mutex); 1085 1086 module_put(disk->fops->owner); 1087 put_no_open: 1088 blkdev_put_no_open(bdev); 1089 } 1090 1091 /** 1092 * bdev_fput - yield claim to the block device and put the file 1093 * @bdev_file: open block device 1094 * 1095 * Yield claim on the block device and put the file. Ensure that the 1096 * block device can be reclaimed before the file is closed which is a 1097 * deferred operation. 1098 */ 1099 void bdev_fput(struct file *bdev_file) 1100 { 1101 if (WARN_ON_ONCE(bdev_file->f_op != &def_blk_fops)) 1102 return; 1103 1104 if (bdev_file->private_data) { 1105 struct block_device *bdev = file_bdev(bdev_file); 1106 struct gendisk *disk = bdev->bd_disk; 1107 1108 mutex_lock(&disk->open_mutex); 1109 bdev_yield_write_access(bdev_file); 1110 bd_yield_claim(bdev_file); 1111 /* 1112 * Tell release we already gave up our hold on the 1113 * device and if write restrictions are available that 1114 * we already gave up write access to the device. 1115 */ 1116 bdev_file->private_data = BDEV_I(bdev_file->f_mapping->host); 1117 mutex_unlock(&disk->open_mutex); 1118 } 1119 1120 fput(bdev_file); 1121 } 1122 EXPORT_SYMBOL(bdev_fput); 1123 1124 /** 1125 * lookup_bdev() - Look up a struct block_device by name. 1126 * @pathname: Name of the block device in the filesystem. 1127 * @dev: Pointer to the block device's dev_t, if found. 1128 * 1129 * Lookup the block device's dev_t at @pathname in the current 1130 * namespace if possible and return it in @dev. 1131 * 1132 * Context: May sleep. 1133 * Return: 0 if succeeded, negative errno otherwise. 1134 */ 1135 int lookup_bdev(const char *pathname, dev_t *dev) 1136 { 1137 struct inode *inode; 1138 struct path path; 1139 int error; 1140 1141 if (!pathname || !*pathname) 1142 return -EINVAL; 1143 1144 error = kern_path(pathname, LOOKUP_FOLLOW, &path); 1145 if (error) 1146 return error; 1147 1148 inode = d_backing_inode(path.dentry); 1149 error = -ENOTBLK; 1150 if (!S_ISBLK(inode->i_mode)) 1151 goto out_path_put; 1152 error = -EACCES; 1153 if (!may_open_dev(&path)) 1154 goto out_path_put; 1155 1156 *dev = inode->i_rdev; 1157 error = 0; 1158 out_path_put: 1159 path_put(&path); 1160 return error; 1161 } 1162 EXPORT_SYMBOL(lookup_bdev); 1163 1164 /** 1165 * bdev_mark_dead - mark a block device as dead 1166 * @bdev: block device to operate on 1167 * @surprise: indicate a surprise removal 1168 * 1169 * Tell the file system that this devices or media is dead. If @surprise is set 1170 * to %true the device or media is already gone, if not we are preparing for an 1171 * orderly removal. 1172 * 1173 * This calls into the file system, which then typicall syncs out all dirty data 1174 * and writes back inodes and then invalidates any cached data in the inodes on 1175 * the file system. In addition we also invalidate the block device mapping. 1176 */ 1177 void bdev_mark_dead(struct block_device *bdev, bool surprise) 1178 { 1179 mutex_lock(&bdev->bd_holder_lock); 1180 if (bdev->bd_holder_ops && bdev->bd_holder_ops->mark_dead) 1181 bdev->bd_holder_ops->mark_dead(bdev, surprise); 1182 else { 1183 mutex_unlock(&bdev->bd_holder_lock); 1184 sync_blockdev(bdev); 1185 } 1186 1187 invalidate_bdev(bdev); 1188 } 1189 /* 1190 * New drivers should not use this directly. There are some drivers however 1191 * that needs this for historical reasons. For example, the DASD driver has 1192 * historically had a shutdown to offline mode that doesn't actually remove the 1193 * gendisk that otherwise looks a lot like a safe device removal. 1194 */ 1195 EXPORT_SYMBOL_GPL(bdev_mark_dead); 1196 1197 void sync_bdevs(bool wait) 1198 { 1199 struct inode *inode, *old_inode = NULL; 1200 1201 spin_lock(&blockdev_superblock->s_inode_list_lock); 1202 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) { 1203 struct address_space *mapping = inode->i_mapping; 1204 struct block_device *bdev; 1205 1206 spin_lock(&inode->i_lock); 1207 if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) || 1208 mapping->nrpages == 0) { 1209 spin_unlock(&inode->i_lock); 1210 continue; 1211 } 1212 __iget(inode); 1213 spin_unlock(&inode->i_lock); 1214 spin_unlock(&blockdev_superblock->s_inode_list_lock); 1215 /* 1216 * We hold a reference to 'inode' so it couldn't have been 1217 * removed from s_inodes list while we dropped the 1218 * s_inode_list_lock We cannot iput the inode now as we can 1219 * be holding the last reference and we cannot iput it under 1220 * s_inode_list_lock. So we keep the reference and iput it 1221 * later. 1222 */ 1223 iput(old_inode); 1224 old_inode = inode; 1225 bdev = I_BDEV(inode); 1226 1227 mutex_lock(&bdev->bd_disk->open_mutex); 1228 if (!atomic_read(&bdev->bd_openers)) { 1229 ; /* skip */ 1230 } else if (wait) { 1231 /* 1232 * We keep the error status of individual mapping so 1233 * that applications can catch the writeback error using 1234 * fsync(2). See filemap_fdatawait_keep_errors() for 1235 * details. 1236 */ 1237 filemap_fdatawait_keep_errors(inode->i_mapping); 1238 } else { 1239 filemap_fdatawrite(inode->i_mapping); 1240 } 1241 mutex_unlock(&bdev->bd_disk->open_mutex); 1242 1243 spin_lock(&blockdev_superblock->s_inode_list_lock); 1244 } 1245 spin_unlock(&blockdev_superblock->s_inode_list_lock); 1246 iput(old_inode); 1247 } 1248 1249 /* 1250 * Handle STATX_DIOALIGN for block devices. 1251 * 1252 * Note that the inode passed to this is the inode of a block device node file, 1253 * not the block device's internal inode. Therefore it is *not* valid to use 1254 * I_BDEV() here; the block device has to be looked up by i_rdev instead. 1255 */ 1256 void bdev_statx_dioalign(struct inode *inode, struct kstat *stat) 1257 { 1258 struct block_device *bdev; 1259 1260 bdev = blkdev_get_no_open(inode->i_rdev); 1261 if (!bdev) 1262 return; 1263 1264 stat->dio_mem_align = bdev_dma_alignment(bdev) + 1; 1265 stat->dio_offset_align = bdev_logical_block_size(bdev); 1266 stat->result_mask |= STATX_DIOALIGN; 1267 1268 blkdev_put_no_open(bdev); 1269 } 1270 1271 bool disk_live(struct gendisk *disk) 1272 { 1273 return !inode_unhashed(disk->part0->bd_inode); 1274 } 1275 EXPORT_SYMBOL_GPL(disk_live); 1276 1277 unsigned int block_size(struct block_device *bdev) 1278 { 1279 return 1 << bdev->bd_inode->i_blkbits; 1280 } 1281 EXPORT_SYMBOL_GPL(block_size); 1282 1283 static int __init setup_bdev_allow_write_mounted(char *str) 1284 { 1285 if (kstrtobool(str, &bdev_allow_write_mounted)) 1286 pr_warn("Invalid option string for bdev_allow_write_mounted:" 1287 " '%s'\n", str); 1288 return 1; 1289 } 1290 __setup("bdev_allow_write_mounted=", setup_bdev_allow_write_mounted); 1291