66 static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
70 int btrfs_init_dev_replace(struct btrfs_fs_info *fs_info)  in btrfs_init_dev_replace()  argument
74 	struct btrfs_root *dev_root = fs_info->dev_root;  in btrfs_init_dev_replace()
75 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_init_dev_replace()
101 		if (btrfs_find_device(fs_info->fs_devices, &args)) {  in btrfs_init_dev_replace()
102 			btrfs_err(fs_info,  in btrfs_init_dev_replace()
130 		btrfs_warn(fs_info,  in btrfs_init_dev_replace()
161 		if (btrfs_find_device(fs_info->fs_devices, &args)) {  in btrfs_init_dev_replace()
162 			btrfs_err(fs_info,  in btrfs_init_dev_replace()
172 		dev_replace->tgtdev = btrfs_find_device(fs_info->fs_devices, &args);  in btrfs_init_dev_replace()
174 		dev_replace->srcdev = btrfs_find_device(fs_info->fs_devices, &args);  in btrfs_init_dev_replace()
181 		    !btrfs_test_opt(fs_info, DEGRADED)) {  in btrfs_init_dev_replace()
183 			btrfs_warn(fs_info,  in btrfs_init_dev_replace()
185 			btrfs_warn(fs_info,  in btrfs_init_dev_replace()
190 		    !btrfs_test_opt(fs_info, DEGRADED)) {  in btrfs_init_dev_replace()
192 			btrfs_warn(fs_info,  in btrfs_init_dev_replace()
194 			btrfs_warn(fs_info,  in btrfs_init_dev_replace()
214 			WARN_ON(fs_info->fs_devices->rw_devices == 0);  in btrfs_init_dev_replace()
215 			dev_replace->tgtdev->io_width = fs_info->sectorsize;  in btrfs_init_dev_replace()
216 			dev_replace->tgtdev->io_align = fs_info->sectorsize;  in btrfs_init_dev_replace()
217 			dev_replace->tgtdev->sector_size = fs_info->sectorsize;  in btrfs_init_dev_replace()
218 			dev_replace->tgtdev->fs_info = fs_info;  in btrfs_init_dev_replace()
234 static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,  in btrfs_init_dev_replace_tgtdev()  argument
239 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;  in btrfs_init_dev_replace_tgtdev()
248 		btrfs_err(fs_info, "the filesystem is a seed filesystem!");  in btrfs_init_dev_replace_tgtdev()
253 					fs_info->bdev_holder, NULL);  in btrfs_init_dev_replace_tgtdev()
255 		btrfs_err(fs_info, "target device %s is invalid!", device_path);  in btrfs_init_dev_replace_tgtdev()
260 	if (!btrfs_check_device_zone_type(fs_info, bdev)) {  in btrfs_init_dev_replace_tgtdev()
261 		btrfs_err(fs_info,  in btrfs_init_dev_replace_tgtdev()
271 			btrfs_err(fs_info,  in btrfs_init_dev_replace_tgtdev()
280 		btrfs_err(fs_info,  in btrfs_init_dev_replace_tgtdev()
299 	device->io_width = fs_info->sectorsize;  in btrfs_init_dev_replace_tgtdev()
300 	device->io_align = fs_info->sectorsize;  in btrfs_init_dev_replace_tgtdev()
301 	device->sector_size = fs_info->sectorsize;  in btrfs_init_dev_replace_tgtdev()
307 	device->fs_info = fs_info;  in btrfs_init_dev_replace_tgtdev()
340 	struct btrfs_fs_info *fs_info = trans->fs_info;  in btrfs_run_dev_replace()  local
342 	struct btrfs_root *dev_root = fs_info->dev_root;  in btrfs_run_dev_replace()
347 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_run_dev_replace()
367 		btrfs_warn(fs_info,  in btrfs_run_dev_replace()
388 			btrfs_warn(fs_info,  in btrfs_run_dev_replace()
402 			btrfs_warn(fs_info,  in btrfs_run_dev_replace()
440 static int mark_block_group_to_copy(struct btrfs_fs_info *fs_info,  in mark_block_group_to_copy()  argument
446 	struct btrfs_root *root = fs_info->dev_root;  in mark_block_group_to_copy()
455 	if (!btrfs_is_zoned(fs_info))  in mark_block_group_to_copy()
458 	mutex_lock(&fs_info->chunk_mutex);  in mark_block_group_to_copy()
461 	spin_lock(&fs_info->trans_lock);  in mark_block_group_to_copy()
462 	while (fs_info->running_transaction &&  in mark_block_group_to_copy()
463 	       !list_empty(&fs_info->running_transaction->dev_update_list)) {  in mark_block_group_to_copy()
464 		spin_unlock(&fs_info->trans_lock);  in mark_block_group_to_copy()
465 		mutex_unlock(&fs_info->chunk_mutex);  in mark_block_group_to_copy()
469 			mutex_lock(&fs_info->chunk_mutex);  in mark_block_group_to_copy()
471 				spin_lock(&fs_info->trans_lock);  in mark_block_group_to_copy()
479 		mutex_lock(&fs_info->chunk_mutex);  in mark_block_group_to_copy()
483 		spin_lock(&fs_info->trans_lock);  in mark_block_group_to_copy()
485 	spin_unlock(&fs_info->trans_lock);  in mark_block_group_to_copy()
517 		cache = btrfs_lookup_block_group(fs_info, chunk_offset);  in mark_block_group_to_copy()
529 	mutex_unlock(&fs_info->chunk_mutex);  in mark_block_group_to_copy()
538 	struct btrfs_fs_info *fs_info = cache->fs_info;  in btrfs_finish_block_group_to_copy()  local
545 	if (!btrfs_is_zoned(fs_info))  in btrfs_finish_block_group_to_copy()
555 	map = btrfs_get_chunk_map(fs_info, chunk_offset, 1);  in btrfs_finish_block_group_to_copy()
586 static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_start()  argument
590 	struct btrfs_root *root = fs_info->dev_root;  in btrfs_dev_replace_start()
592 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_start()
597 	src_device = btrfs_find_device_by_devspec(fs_info, srcdevid,  in btrfs_dev_replace_start()
602 	if (btrfs_pinned_by_swapfile(fs_info, src_device)) {  in btrfs_dev_replace_start()
603 		btrfs_warn_in_rcu(fs_info,  in btrfs_dev_replace_start()
622 	ret = btrfs_init_dev_replace_tgtdev(fs_info, tgtdev_name,  in btrfs_dev_replace_start()
627 	ret = mark_block_group_to_copy(fs_info, src_device);  in btrfs_dev_replace_start()
650 	btrfs_info_in_rcu(fs_info,  in btrfs_dev_replace_start()
674 		btrfs_err(fs_info, "kobj add dev failed %d", ret);  in btrfs_dev_replace_start()
676 	btrfs_wait_ordered_roots(fs_info, U64_MAX, NULL);  in btrfs_dev_replace_start()
700 	ret = btrfs_scrub_dev(fs_info, src_device->devid, 0,  in btrfs_dev_replace_start()
704 	ret = btrfs_dev_replace_finishing(fs_info, ret);  in btrfs_dev_replace_start()
732 int btrfs_dev_replace_by_ioctl(struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_by_ioctl()  argument
748 	ret = btrfs_dev_replace_start(fs_info, args->start.tgtdev_name,  in btrfs_dev_replace_by_ioctl()
764 static void btrfs_rm_dev_replace_blocked(struct btrfs_fs_info *fs_info)  in btrfs_rm_dev_replace_blocked()  argument
766 	set_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);  in btrfs_rm_dev_replace_blocked()
767 	wait_event(fs_info->dev_replace.replace_wait, !percpu_counter_sum(  in btrfs_rm_dev_replace_blocked()
768 		   &fs_info->dev_replace.bio_counter));  in btrfs_rm_dev_replace_blocked()
774 static void btrfs_rm_dev_replace_unblocked(struct btrfs_fs_info *fs_info)  in btrfs_rm_dev_replace_unblocked()  argument
776 	clear_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);  in btrfs_rm_dev_replace_unblocked()
777 	wake_up(&fs_info->dev_replace.replace_wait);  in btrfs_rm_dev_replace_unblocked()
795 	lockdep_assert_held(&srcdev->fs_info->chunk_mutex);  in btrfs_set_target_alloc_state()
812 						struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_update_device_in_mapping_tree()  argument
824 	lockdep_assert_held(&fs_info->chunk_mutex);  in btrfs_dev_replace_update_device_in_mapping_tree()
826 	write_lock(&fs_info->mapping_tree_lock);  in btrfs_dev_replace_update_device_in_mapping_tree()
827 	node = rb_first_cached(&fs_info->mapping_tree);  in btrfs_dev_replace_update_device_in_mapping_tree()
840 		if (cond_resched_rwlock_write(&fs_info->mapping_tree_lock)) {  in btrfs_dev_replace_update_device_in_mapping_tree()
841 			map = btrfs_find_chunk_map_nolock(fs_info, next_start, U64_MAX);  in btrfs_dev_replace_update_device_in_mapping_tree()
855 	write_unlock(&fs_info->mapping_tree_lock);  in btrfs_dev_replace_update_device_in_mapping_tree()
858 static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_finishing()  argument
861 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_finishing()
862 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;  in btrfs_dev_replace_finishing()
865 	struct btrfs_root *root = fs_info->tree_root;  in btrfs_dev_replace_finishing()
890 	ret = btrfs_start_delalloc_roots(fs_info, LONG_MAX, false);  in btrfs_dev_replace_finishing()
895 	btrfs_wait_ordered_roots(fs_info, U64_MAX, NULL);  in btrfs_dev_replace_finishing()
914 		mutex_lock(&fs_info->chunk_mutex);  in btrfs_dev_replace_finishing()
918 			mutex_unlock(&fs_info->chunk_mutex);  in btrfs_dev_replace_finishing()
941 		btrfs_dev_replace_update_device_in_mapping_tree(fs_info,  in btrfs_dev_replace_finishing()
946 			btrfs_err_in_rcu(fs_info,  in btrfs_dev_replace_finishing()
953 		mutex_unlock(&fs_info->chunk_mutex);  in btrfs_dev_replace_finishing()
955 		btrfs_rm_dev_replace_blocked(fs_info);  in btrfs_dev_replace_finishing()
958 		btrfs_rm_dev_replace_unblocked(fs_info);  in btrfs_dev_replace_finishing()
964 	btrfs_info_in_rcu(fs_info,  in btrfs_dev_replace_finishing()
988 	btrfs_rm_dev_replace_blocked(fs_info);  in btrfs_dev_replace_finishing()
992 	btrfs_rm_dev_replace_unblocked(fs_info);  in btrfs_dev_replace_finishing()
1007 	mutex_unlock(&fs_info->chunk_mutex);  in btrfs_dev_replace_finishing()
1014 		btrfs_scratch_superblocks(fs_info, src_device);  in btrfs_dev_replace_finishing()
1033 static u64 btrfs_dev_replace_progress(struct btrfs_fs_info *fs_info)  in btrfs_dev_replace_progress()  argument
1035 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_progress()
1057 void btrfs_dev_replace_status(struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_status()  argument
1060 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_status()
1073 	args->status.progress_1000 = btrfs_dev_replace_progress(fs_info);  in btrfs_dev_replace_status()
1077 int btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info)  in btrfs_dev_replace_cancel()  argument
1079 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_cancel()
1083 	struct btrfs_root *root = fs_info->tree_root;  in btrfs_dev_replace_cancel()
1087 	if (sb_rdonly(fs_info->sb))  in btrfs_dev_replace_cancel()
1103 		ret = btrfs_scrub_cancel(fs_info);  in btrfs_dev_replace_cancel()
1112 			btrfs_info_in_rcu(fs_info,  in btrfs_dev_replace_cancel()
1136 		btrfs_scrub_cancel(fs_info);  in btrfs_dev_replace_cancel()
1146 		btrfs_info_in_rcu(fs_info,  in btrfs_dev_replace_cancel()
1163 void btrfs_dev_replace_suspend_for_unmount(struct btrfs_fs_info *fs_info)  in btrfs_dev_replace_suspend_for_unmount()  argument
1165 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_suspend_for_unmount()
1181 		btrfs_info(fs_info, "suspending dev_replace for unmount");  in btrfs_dev_replace_suspend_for_unmount()
1190 int btrfs_resume_dev_replace_async(struct btrfs_fs_info *fs_info)  in btrfs_resume_dev_replace_async()  argument
1193 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_resume_dev_replace_async()
1211 		btrfs_info(fs_info,  in btrfs_resume_dev_replace_async()
1213 		btrfs_info(fs_info,  in btrfs_resume_dev_replace_async()
1227 	if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_DEV_REPLACE)) {  in btrfs_resume_dev_replace_async()
1232 		btrfs_info(fs_info,  in btrfs_resume_dev_replace_async()
1237 	task = kthread_run(btrfs_dev_replace_kthread, fs_info, "btrfs-devrepl");  in btrfs_resume_dev_replace_async()
1243 	struct btrfs_fs_info *fs_info = data;  in btrfs_dev_replace_kthread()  local
1244 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_kthread()
1248 	progress = btrfs_dev_replace_progress(fs_info);  in btrfs_dev_replace_kthread()
1250 	btrfs_info_in_rcu(fs_info,  in btrfs_dev_replace_kthread()
1257 	ret = btrfs_scrub_dev(fs_info, dev_replace->srcdev->devid,  in btrfs_dev_replace_kthread()
1261 	ret = btrfs_dev_replace_finishing(fs_info, ret);  in btrfs_dev_replace_kthread()
1264 	btrfs_exclop_finish(fs_info);  in btrfs_dev_replace_kthread()
1295 void btrfs_bio_counter_sub(struct btrfs_fs_info *fs_info, s64 amount)  in btrfs_bio_counter_sub()  argument
1297 	percpu_counter_sub(&fs_info->dev_replace.bio_counter, amount);  in btrfs_bio_counter_sub()
1298 	cond_wake_up_nomb(&fs_info->dev_replace.replace_wait);  in btrfs_bio_counter_sub()
1301 void btrfs_bio_counter_inc_blocked(struct btrfs_fs_info *fs_info)  in btrfs_bio_counter_inc_blocked()  argument
1304 		percpu_counter_inc(&fs_info->dev_replace.bio_counter);  in btrfs_bio_counter_inc_blocked()
1306 				     &fs_info->fs_state)))  in btrfs_bio_counter_inc_blocked()
1309 		btrfs_bio_counter_dec(fs_info);  in btrfs_bio_counter_inc_blocked()
1310 		wait_event(fs_info->dev_replace.replace_wait,  in btrfs_bio_counter_inc_blocked()
1312 				     &fs_info->fs_state));  in btrfs_bio_counter_inc_blocked()