59 void set_capacity(struct gendisk *disk, sector_t sectors)  in set_capacity()  argument
63 				disk->disk_name, sectors,  in set_capacity()
68 	bdev_set_nr_sectors(disk->part0, sectors);  in set_capacity()
76 bool set_capacity_and_notify(struct gendisk *disk, sector_t size)  in set_capacity_and_notify()  argument
78 	sector_t capacity = get_capacity(disk);  in set_capacity_and_notify()
81 	set_capacity(disk, size);  in set_capacity_and_notify()
89 	    !disk_live(disk) ||  in set_capacity_and_notify()
90 	    (disk->flags & GENHD_FL_HIDDEN))  in set_capacity_and_notify()
94 		disk->disk_name, capacity, size);  in set_capacity_and_notify()
102 	kobject_uevent_env(&disk_to_dev(disk)->kobj, KOBJ_CHANGE, envp);  in set_capacity_and_notify()
329 void disk_uevent(struct gendisk *disk, enum kobject_action action)  in disk_uevent()  argument
335 	xa_for_each(&disk->part_tbl, idx, part) {  in disk_uevent()
350 int disk_scan_partitions(struct gendisk *disk, blk_mode_t mode)  in disk_scan_partitions()  argument
355 	if (!disk_has_partscan(disk))  in disk_scan_partitions()
357 	if (disk->open_partitions)  in disk_scan_partitions()
367 		ret = bd_prepare_to_claim(disk->part0, disk_scan_partitions,  in disk_scan_partitions()
373 	set_bit(GD_NEED_PART_SCAN, &disk->state);  in disk_scan_partitions()
374 	file = bdev_file_open_by_dev(disk_devt(disk), mode & ~BLK_OPEN_EXCL,  in disk_scan_partitions()
386 	clear_bit(GD_NEED_PART_SCAN, &disk->state);  in disk_scan_partitions()
388 		bd_abort_claiming(disk->part0, disk_scan_partitions);  in disk_scan_partitions()
402 int __must_check add_disk_fwnode(struct device *parent, struct gendisk *disk,  in add_disk_fwnode()  argument
407 	struct device *ddev = disk_to_dev(disk);  in add_disk_fwnode()
410 	if (WARN_ON_ONCE(bdev_nr_sectors(disk->part0) > BLK_DEV_MAX_SECTORS))  in add_disk_fwnode()
413 	if (queue_is_mq(disk->queue)) {  in add_disk_fwnode()
417 		if (disk->fops->submit_bio || disk->fops->poll_bio)  in add_disk_fwnode()
424 		elevator_init_mq(disk->queue);  in add_disk_fwnode()
426 		if (!disk->fops->submit_bio)  in add_disk_fwnode()
428 		bdev_set_flag(disk->part0, BD_HAS_SUBMIT_BIO);  in add_disk_fwnode()
439 	if (disk->major) {  in add_disk_fwnode()
440 		if (WARN_ON(!disk->minors))  in add_disk_fwnode()
443 		if (disk->minors > DISK_MAX_PARTS) {  in add_disk_fwnode()
446 			disk->minors = DISK_MAX_PARTS;  in add_disk_fwnode()
448 		if (disk->first_minor > MINORMASK ||  in add_disk_fwnode()
449 		    disk->minors > MINORMASK + 1 ||  in add_disk_fwnode()
450 		    disk->first_minor + disk->minors > MINORMASK + 1)  in add_disk_fwnode()
453 		if (WARN_ON(disk->minors))  in add_disk_fwnode()
459 		disk->major = BLOCK_EXT_MAJOR;  in add_disk_fwnode()
460 		disk->first_minor = ret;  in add_disk_fwnode()
468 	dev_set_name(ddev, "%s", disk->disk_name);  in add_disk_fwnode()
471 	if (!(disk->flags & GENHD_FL_HIDDEN))  in add_disk_fwnode()
472 		ddev->devt = MKDEV(disk->major, disk->first_minor);  in add_disk_fwnode()
477 	ret = disk_alloc_events(disk);  in add_disk_fwnode()
493 	disk->part0->bd_holder_dir =  in add_disk_fwnode()
495 	if (!disk->part0->bd_holder_dir) {  in add_disk_fwnode()
499 	disk->slave_dir = kobject_create_and_add("slaves", &ddev->kobj);  in add_disk_fwnode()
500 	if (!disk->slave_dir) {  in add_disk_fwnode()
505 	ret = blk_register_queue(disk);  in add_disk_fwnode()
509 	if (!(disk->flags & GENHD_FL_HIDDEN)) {  in add_disk_fwnode()
510 		ret = bdi_register(disk->bdi, "%u:%u",  in add_disk_fwnode()
511 				   disk->major, disk->first_minor);  in add_disk_fwnode()
514 		bdi_set_owner(disk->bdi, ddev);  in add_disk_fwnode()
516 					&disk->bdi->dev->kobj, "bdi");  in add_disk_fwnode()
521 		if (get_capacity(disk) && disk_has_partscan(disk))  in add_disk_fwnode()
522 			set_bit(GD_NEED_PART_SCAN, &disk->state);  in add_disk_fwnode()
524 		bdev_add(disk->part0, ddev->devt);  in add_disk_fwnode()
525 		if (get_capacity(disk))  in add_disk_fwnode()
526 			disk_scan_partitions(disk, BLK_OPEN_READ);  in add_disk_fwnode()
533 		disk_uevent(disk, KOBJ_ADD);  in add_disk_fwnode()
540 		disk->part0->bd_dev = MKDEV(disk->major, disk->first_minor);  in add_disk_fwnode()
543 	blk_apply_bdi_limits(disk->bdi, &disk->queue->limits);  in add_disk_fwnode()
544 	disk_add_events(disk);  in add_disk_fwnode()
545 	set_bit(GD_ADDED, &disk->state);  in add_disk_fwnode()
549 	if (!(disk->flags & GENHD_FL_HIDDEN))  in add_disk_fwnode()
550 		bdi_unregister(disk->bdi);  in add_disk_fwnode()
552 	blk_unregister_queue(disk);  in add_disk_fwnode()
553 	rq_qos_exit(disk->queue);  in add_disk_fwnode()
555 	kobject_put(disk->slave_dir);  in add_disk_fwnode()
556 	disk->slave_dir = NULL;  in add_disk_fwnode()
558 	kobject_put(disk->part0->bd_holder_dir);  in add_disk_fwnode()
565 	if (disk->major == BLOCK_EXT_MAJOR)  in add_disk_fwnode()
566 		blk_free_ext_minor(disk->first_minor);  in add_disk_fwnode()
568 	if (disk->queue->elevator) {  in add_disk_fwnode()
569 		mutex_lock(&disk->queue->elevator_lock);  in add_disk_fwnode()
570 		elevator_exit(disk->queue);  in add_disk_fwnode()
571 		mutex_unlock(&disk->queue->elevator_lock);  in add_disk_fwnode()
586 int __must_check device_add_disk(struct device *parent, struct gendisk *disk,  in device_add_disk()  argument
589 	return add_disk_fwnode(parent, disk, groups, NULL);  in device_add_disk()
593 static void blk_report_disk_dead(struct gendisk *disk, bool surprise)  in blk_report_disk_dead()  argument
603 	lockdep_assert_not_held(&disk->open_mutex);  in blk_report_disk_dead()
606 	xa_for_each(&disk->part_tbl, idx, bdev) {  in blk_report_disk_dead()
619 static bool __blk_mark_disk_dead(struct gendisk *disk)  in __blk_mark_disk_dead()  argument
624 	if (test_and_set_bit(GD_DEAD, &disk->state))  in __blk_mark_disk_dead()
627 	if (test_bit(GD_OWNS_QUEUE, &disk->state))  in __blk_mark_disk_dead()
628 		blk_queue_flag_set(QUEUE_FLAG_DYING, disk->queue);  in __blk_mark_disk_dead()
633 	set_capacity(disk, 0);  in __blk_mark_disk_dead()
638 	return blk_queue_start_drain(disk->queue);  in __blk_mark_disk_dead()
648 void blk_mark_disk_dead(struct gendisk *disk)  in blk_mark_disk_dead()  argument
650 	__blk_mark_disk_dead(disk);  in blk_mark_disk_dead()
651 	blk_report_disk_dead(disk, true);  in blk_mark_disk_dead()
674 void del_gendisk(struct gendisk *disk)  in del_gendisk()  argument
676 	struct request_queue *q = disk->queue;  in del_gendisk()
683 	if (WARN_ON_ONCE(!disk_live(disk) && !(disk->flags & GENHD_FL_HIDDEN)))  in del_gendisk()
686 	disk_del_events(disk);  in del_gendisk()
691 	mutex_lock(&disk->open_mutex);  in del_gendisk()
692 	xa_for_each(&disk->part_tbl, idx, part)  in del_gendisk()
694 	mutex_unlock(&disk->open_mutex);  in del_gendisk()
700 	if (!test_bit(GD_DEAD, &disk->state))  in del_gendisk()
701 		blk_report_disk_dead(disk, false);  in del_gendisk()
706 	mutex_lock(&disk->open_mutex);  in del_gendisk()
707 	start_drain = __blk_mark_disk_dead(disk);  in del_gendisk()
710 	xa_for_each_start(&disk->part_tbl, idx, part, 1)  in del_gendisk()
712 	mutex_unlock(&disk->open_mutex);  in del_gendisk()
714 	if (!(disk->flags & GENHD_FL_HIDDEN)) {  in del_gendisk()
715 		sysfs_remove_link(&disk_to_dev(disk)->kobj, "bdi");  in del_gendisk()
721 		bdi_unregister(disk->bdi);  in del_gendisk()
724 	blk_unregister_queue(disk);  in del_gendisk()
726 	kobject_put(disk->part0->bd_holder_dir);  in del_gendisk()
727 	kobject_put(disk->slave_dir);  in del_gendisk()
728 	disk->slave_dir = NULL;  in del_gendisk()
730 	part_stat_set_all(disk->part0, 0);  in del_gendisk()
731 	disk->part0->bd_stamp = 0;  in del_gendisk()
732 	sysfs_remove_link(block_depr, dev_name(disk_to_dev(disk)));  in del_gendisk()
733 	pm_runtime_set_memalloc_noio(disk_to_dev(disk), false);  in del_gendisk()
734 	device_del(disk_to_dev(disk));  in del_gendisk()
738 	blk_throtl_cancel_bios(disk);  in del_gendisk()
759 	if (!test_bit(GD_OWNS_QUEUE, &disk->state))  in del_gendisk()
779 void invalidate_disk(struct gendisk *disk)  in invalidate_disk()  argument
781 	struct block_device *bdev = disk->part0;  in invalidate_disk()
785 	set_capacity(disk, 0);  in invalidate_disk()
794 	struct gendisk *disk = dev_to_disk(dev);  in disk_badblocks_show()  local
796 	if (!disk->bb)  in disk_badblocks_show()
799 	return badblocks_show(disk->bb, page, 0);  in disk_badblocks_show()
806 	struct gendisk *disk = dev_to_disk(dev);  in disk_badblocks_store()  local
808 	if (!disk->bb)  in disk_badblocks_store()
811 	return badblocks_store(disk->bb, page, len, 0);  in disk_badblocks_store()
955 	struct gendisk *disk = dev_to_disk(dev);  in disk_range_show()  local
957 	return sysfs_emit(buf, "%d\n", disk->minors);  in disk_range_show()
963 	struct gendisk *disk = dev_to_disk(dev);  in disk_ext_range_show()  local
966 		(disk->flags & GENHD_FL_NO_PART) ? 1 : DISK_MAX_PARTS);  in disk_ext_range_show()
972 	struct gendisk *disk = dev_to_disk(dev);  in disk_removable_show()  local
975 		       (disk->flags & GENHD_FL_REMOVABLE ? 1 : 0));  in disk_removable_show()
981 	struct gendisk *disk = dev_to_disk(dev);  in disk_hidden_show()  local
984 		       (disk->flags & GENHD_FL_HIDDEN ? 1 : 0));  in disk_hidden_show()
990 	struct gendisk *disk = dev_to_disk(dev);  in disk_ro_show()  local
992 	return sysfs_emit(buf, "%d\n", get_disk_ro(disk) ? 1 : 0);  in disk_ro_show()
1071 	struct gendisk *disk = dev_to_disk(dev);  in disk_alignment_offset_show()  local
1073 	return sysfs_emit(buf, "%d\n", bdev_alignment_offset(disk->part0));  in disk_alignment_offset_show()
1080 	struct gendisk *disk = dev_to_disk(dev);  in disk_discard_alignment_show()  local
1082 	return sysfs_emit(buf, "%d\n", bdev_alignment_offset(disk->part0));  in disk_discard_alignment_show()
1088 	struct gendisk *disk = dev_to_disk(dev);  in diskseq_show()  local
1090 	return sysfs_emit(buf, "%llu\n", disk->diskseq);  in diskseq_show()
1176 	struct gendisk *disk = dev_to_disk(dev);  in disk_visible()  local
1178 	if (a == &dev_attr_badblocks.attr && !disk->bb)  in disk_visible()
1215 	struct gendisk *disk = dev_to_disk(dev);  in disk_release()  local
1218 	WARN_ON_ONCE(disk_live(disk));  in disk_release()
1220 	blk_trace_remove(disk->queue);  in disk_release()
1229 	if (queue_is_mq(disk->queue) &&  in disk_release()
1230 	    test_bit(GD_OWNS_QUEUE, &disk->state) &&  in disk_release()
1231 	    !test_bit(GD_ADDED, &disk->state))  in disk_release()
1232 		blk_mq_exit_queue(disk->queue);  in disk_release()
1234 	blkcg_exit_disk(disk);  in disk_release()
1236 	bioset_exit(&disk->bio_split);  in disk_release()
1238 	disk_release_events(disk);  in disk_release()
1239 	kfree(disk->random);  in disk_release()
1240 	disk_free_zone_resources(disk);  in disk_release()
1241 	xa_destroy(&disk->part_tbl);  in disk_release()
1243 	disk->queue->disk = NULL;  in disk_release()
1244 	blk_put_queue(disk->queue);  in disk_release()
1246 	if (test_bit(GD_ADDED, &disk->state) && disk->fops->free_disk)  in disk_release()
1247 		disk->fops->free_disk(disk);  in disk_release()
1249 	bdev_drop(disk->part0);	/* frees the disk */  in disk_release()
1254 	const struct gendisk *disk = dev_to_disk(dev);  in block_uevent()  local
1256 	return add_uevent_var(env, "DISKSEQ=%llu", disk->diskseq);  in block_uevent()
1267 	struct gendisk *disk = dev_to_disk(dev);  in block_devnode()  local
1269 	if (disk->fops->devnode)  in block_devnode()
1270 		return disk->fops->devnode(disk, mode);  in block_devnode()
1368 dev_t part_devt(struct gendisk *disk, u8 partno)  in part_devt()  argument
1374 	part = xa_load(&disk->part_tbl, partno);  in part_devt()
1385 	struct gendisk *disk;  in __alloc_disk_node()  local
1387 	disk = kzalloc_node(sizeof(struct gendisk), GFP_KERNEL, node_id);  in __alloc_disk_node()
1388 	if (!disk)  in __alloc_disk_node()
1391 	if (bioset_init(&disk->bio_split, BIO_POOL_SIZE, 0, 0))  in __alloc_disk_node()
1394 	disk->bdi = bdi_alloc(node_id);  in __alloc_disk_node()
1395 	if (!disk->bdi)  in __alloc_disk_node()
1399 	disk->queue = q;  in __alloc_disk_node()
1401 	disk->part0 = bdev_alloc(disk, 0);  in __alloc_disk_node()
1402 	if (!disk->part0)  in __alloc_disk_node()
1405 	disk->node_id = node_id;  in __alloc_disk_node()
1406 	mutex_init(&disk->open_mutex);  in __alloc_disk_node()
1407 	xa_init(&disk->part_tbl);  in __alloc_disk_node()
1408 	if (xa_insert(&disk->part_tbl, 0, disk->part0, GFP_KERNEL))  in __alloc_disk_node()
1411 	if (blkcg_init_disk(disk))  in __alloc_disk_node()
1414 	disk_init_zone_resources(disk);  in __alloc_disk_node()
1415 	rand_initialize_disk(disk);  in __alloc_disk_node()
1416 	disk_to_dev(disk)->class = &block_class;  in __alloc_disk_node()
1417 	disk_to_dev(disk)->type = &disk_type;  in __alloc_disk_node()
1418 	device_initialize(disk_to_dev(disk));  in __alloc_disk_node()
1419 	inc_diskseq(disk);  in __alloc_disk_node()
1420 	q->disk = disk;  in __alloc_disk_node()
1421 	lockdep_init_map(&disk->lockdep_map, "(bio completion)", lkclass, 0);  in __alloc_disk_node()
1423 	INIT_LIST_HEAD(&disk->slave_bdevs);  in __alloc_disk_node()
1425 	return disk;  in __alloc_disk_node()
1428 	xa_erase(&disk->part_tbl, 0);  in __alloc_disk_node()
1430 	xa_destroy(&disk->part_tbl);  in __alloc_disk_node()
1431 	disk->part0->bd_disk = NULL;  in __alloc_disk_node()
1432 	bdev_drop(disk->part0);  in __alloc_disk_node()
1434 	bdi_put(disk->bdi);  in __alloc_disk_node()
1436 	bioset_exit(&disk->bio_split);  in __alloc_disk_node()
1438 	kfree(disk);  in __alloc_disk_node()
1447 	struct gendisk *disk;  in __blk_alloc_disk()  local
1453 	disk = __alloc_disk_node(q, node, lkclass);  in __blk_alloc_disk()
1454 	if (!disk) {  in __blk_alloc_disk()
1458 	set_bit(GD_OWNS_QUEUE, &disk->state);  in __blk_alloc_disk()
1459 	return disk;  in __blk_alloc_disk()
1476 void put_disk(struct gendisk *disk)  in put_disk()  argument
1478 	if (disk)  in put_disk()
1479 		put_device(disk_to_dev(disk));  in put_disk()
1502 void set_disk_ro(struct gendisk *disk, bool read_only)  in set_disk_ro()  argument
1505 		if (test_and_set_bit(GD_READ_ONLY, &disk->state))  in set_disk_ro()
1508 		if (!test_and_clear_bit(GD_READ_ONLY, &disk->state))  in set_disk_ro()
1511 	set_disk_ro_uevent(disk, read_only);  in set_disk_ro()
1515 void inc_diskseq(struct gendisk *disk)  in inc_diskseq()  argument
1517 	disk->diskseq = atomic64_inc_return(&diskseq);  in inc_diskseq()