75 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_init_dev_replace()  local
106 		dev_replace->replace_state =  in btrfs_init_dev_replace()
108 		dev_replace->cont_reading_from_srcdev_mode =  in btrfs_init_dev_replace()
110 		dev_replace->time_started = 0;  in btrfs_init_dev_replace()
111 		dev_replace->time_stopped = 0;  in btrfs_init_dev_replace()
112 		atomic64_set(&dev_replace->num_write_errors, 0);  in btrfs_init_dev_replace()
113 		atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0);  in btrfs_init_dev_replace()
114 		dev_replace->cursor_left = 0;  in btrfs_init_dev_replace()
115 		dev_replace->committed_cursor_left = 0;  in btrfs_init_dev_replace()
116 		dev_replace->cursor_left_last_write_of_item = 0;  in btrfs_init_dev_replace()
117 		dev_replace->cursor_right = 0;  in btrfs_init_dev_replace()
118 		dev_replace->srcdev = NULL;  in btrfs_init_dev_replace()
119 		dev_replace->tgtdev = NULL;  in btrfs_init_dev_replace()
120 		dev_replace->is_valid = 0;  in btrfs_init_dev_replace()
121 		dev_replace->item_needs_writeback = 0;  in btrfs_init_dev_replace()
136 	dev_replace->cont_reading_from_srcdev_mode =  in btrfs_init_dev_replace()
138 	dev_replace->replace_state = btrfs_dev_replace_replace_state(eb, ptr);  in btrfs_init_dev_replace()
139 	dev_replace->time_started = btrfs_dev_replace_time_started(eb, ptr);  in btrfs_init_dev_replace()
140 	dev_replace->time_stopped =  in btrfs_init_dev_replace()
142 	atomic64_set(&dev_replace->num_write_errors,  in btrfs_init_dev_replace()
144 	atomic64_set(&dev_replace->num_uncorrectable_read_errors,  in btrfs_init_dev_replace()
146 	dev_replace->cursor_left = btrfs_dev_replace_cursor_left(eb, ptr);  in btrfs_init_dev_replace()
147 	dev_replace->committed_cursor_left = dev_replace->cursor_left;  in btrfs_init_dev_replace()
148 	dev_replace->cursor_left_last_write_of_item = dev_replace->cursor_left;  in btrfs_init_dev_replace()
149 	dev_replace->cursor_right = btrfs_dev_replace_cursor_right(eb, ptr);  in btrfs_init_dev_replace()
150 	dev_replace->is_valid = 1;  in btrfs_init_dev_replace()
152 	dev_replace->item_needs_writeback = 0;  in btrfs_init_dev_replace()
153 	switch (dev_replace->replace_state) {  in btrfs_init_dev_replace()
166 			dev_replace->srcdev = NULL;  in btrfs_init_dev_replace()
167 			dev_replace->tgtdev = NULL;  in btrfs_init_dev_replace()
172 		dev_replace->tgtdev = btrfs_find_device(fs_info->fs_devices, &args);  in btrfs_init_dev_replace()
174 		dev_replace->srcdev = btrfs_find_device(fs_info->fs_devices, &args);  in btrfs_init_dev_replace()
180 		if (!dev_replace->srcdev &&  in btrfs_init_dev_replace()
189 		if (!dev_replace->tgtdev &&  in btrfs_init_dev_replace()
198 		if (dev_replace->tgtdev) {  in btrfs_init_dev_replace()
199 			if (dev_replace->srcdev) {  in btrfs_init_dev_replace()
200 				dev_replace->tgtdev->total_bytes =  in btrfs_init_dev_replace()
201 					dev_replace->srcdev->total_bytes;  in btrfs_init_dev_replace()
202 				dev_replace->tgtdev->disk_total_bytes =  in btrfs_init_dev_replace()
203 					dev_replace->srcdev->disk_total_bytes;  in btrfs_init_dev_replace()
204 				dev_replace->tgtdev->commit_total_bytes =  in btrfs_init_dev_replace()
205 					dev_replace->srcdev->commit_total_bytes;  in btrfs_init_dev_replace()
206 				dev_replace->tgtdev->bytes_used =  in btrfs_init_dev_replace()
207 					dev_replace->srcdev->bytes_used;  in btrfs_init_dev_replace()
208 				dev_replace->tgtdev->commit_bytes_used =  in btrfs_init_dev_replace()
209 					dev_replace->srcdev->commit_bytes_used;  in btrfs_init_dev_replace()
212 				&dev_replace->tgtdev->dev_state);  in btrfs_init_dev_replace()
215 			dev_replace->tgtdev->io_width = fs_info->sectorsize;  in btrfs_init_dev_replace()
216 			dev_replace->tgtdev->io_align = fs_info->sectorsize;  in btrfs_init_dev_replace()
217 			dev_replace->tgtdev->sector_size = fs_info->sectorsize;  in btrfs_init_dev_replace()
218 			dev_replace->tgtdev->fs_info = fs_info;  in btrfs_init_dev_replace()
220 				&dev_replace->tgtdev->dev_state);  in btrfs_init_dev_replace()
347 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_run_dev_replace()  local
349 	down_read(&dev_replace->rwsem);  in btrfs_run_dev_replace()
350 	if (!dev_replace->is_valid ||  in btrfs_run_dev_replace()
351 	    !dev_replace->item_needs_writeback) {  in btrfs_run_dev_replace()
352 		up_read(&dev_replace->rwsem);  in btrfs_run_dev_replace()
355 	up_read(&dev_replace->rwsem);  in btrfs_run_dev_replace()
412 	down_write(&dev_replace->rwsem);  in btrfs_run_dev_replace()
413 	if (dev_replace->srcdev)  in btrfs_run_dev_replace()
415 			dev_replace->srcdev->devid);  in btrfs_run_dev_replace()
419 		dev_replace->cont_reading_from_srcdev_mode);  in btrfs_run_dev_replace()
421 		dev_replace->replace_state);  in btrfs_run_dev_replace()
422 	btrfs_set_dev_replace_time_started(eb, ptr, dev_replace->time_started);  in btrfs_run_dev_replace()
423 	btrfs_set_dev_replace_time_stopped(eb, ptr, dev_replace->time_stopped);  in btrfs_run_dev_replace()
425 		atomic64_read(&dev_replace->num_write_errors));  in btrfs_run_dev_replace()
427 		atomic64_read(&dev_replace->num_uncorrectable_read_errors));  in btrfs_run_dev_replace()
428 	dev_replace->cursor_left_last_write_of_item =  in btrfs_run_dev_replace()
429 		dev_replace->cursor_left;  in btrfs_run_dev_replace()
431 		dev_replace->cursor_left_last_write_of_item);  in btrfs_run_dev_replace()
433 		dev_replace->cursor_right);  in btrfs_run_dev_replace()
434 	dev_replace->item_needs_writeback = 0;  in btrfs_run_dev_replace()
435 	up_write(&dev_replace->rwsem);  in btrfs_run_dev_replace()
592 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_start()  local
631 	down_write(&dev_replace->rwsem);  in btrfs_dev_replace_start()
632 	dev_replace->replace_task = current;  in btrfs_dev_replace_start()
633 	switch (dev_replace->replace_state) {  in btrfs_dev_replace_start()
642 		up_write(&dev_replace->rwsem);  in btrfs_dev_replace_start()
646 	dev_replace->cont_reading_from_srcdev_mode = read_src;  in btrfs_dev_replace_start()
647 	dev_replace->srcdev = src_device;  in btrfs_dev_replace_start()
648 	dev_replace->tgtdev = tgt_device;  in btrfs_dev_replace_start()
660 	dev_replace->replace_state = BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED;  in btrfs_dev_replace_start()
661 	dev_replace->time_started = ktime_get_real_seconds();  in btrfs_dev_replace_start()
662 	dev_replace->cursor_left = 0;  in btrfs_dev_replace_start()
663 	dev_replace->committed_cursor_left = 0;  in btrfs_dev_replace_start()
664 	dev_replace->cursor_left_last_write_of_item = 0;  in btrfs_dev_replace_start()
665 	dev_replace->cursor_right = 0;  in btrfs_dev_replace_start()
666 	dev_replace->is_valid = 1;  in btrfs_dev_replace_start()
667 	dev_replace->item_needs_writeback = 1;  in btrfs_dev_replace_start()
668 	atomic64_set(&dev_replace->num_write_errors, 0);  in btrfs_dev_replace_start()
669 	atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0);  in btrfs_dev_replace_start()
670 	up_write(&dev_replace->rwsem);  in btrfs_dev_replace_start()
687 		down_write(&dev_replace->rwsem);  in btrfs_dev_replace_start()
688 		dev_replace->replace_state =  in btrfs_dev_replace_start()
690 		dev_replace->srcdev = NULL;  in btrfs_dev_replace_start()
691 		dev_replace->tgtdev = NULL;  in btrfs_dev_replace_start()
692 		up_write(&dev_replace->rwsem);  in btrfs_dev_replace_start()
702 			      &dev_replace->scrub_progress, 0, 1);  in btrfs_dev_replace_start()
767 	wait_event(fs_info->dev_replace.replace_wait, !percpu_counter_sum(  in btrfs_rm_dev_replace_blocked()
768 		   &fs_info->dev_replace.bio_counter));  in btrfs_rm_dev_replace_blocked()
777 	wake_up(&fs_info->dev_replace.replace_wait);  in btrfs_rm_dev_replace_unblocked()
861 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_finishing()  local
871 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
873 	down_read(&dev_replace->rwsem);  in btrfs_dev_replace_finishing()
875 	if (dev_replace->replace_state !=  in btrfs_dev_replace_finishing()
877 		up_read(&dev_replace->rwsem);  in btrfs_dev_replace_finishing()
878 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
882 	tgt_device = dev_replace->tgtdev;  in btrfs_dev_replace_finishing()
883 	src_device = dev_replace->srcdev;  in btrfs_dev_replace_finishing()
884 	up_read(&dev_replace->rwsem);  in btrfs_dev_replace_finishing()
892 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
905 			mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
924 	down_write(&dev_replace->rwsem);  in btrfs_dev_replace_finishing()
925 	dev_replace->replace_state =  in btrfs_dev_replace_finishing()
928 	dev_replace->tgtdev = NULL;  in btrfs_dev_replace_finishing()
929 	dev_replace->srcdev = NULL;  in btrfs_dev_replace_finishing()
930 	dev_replace->time_stopped = ktime_get_real_seconds();  in btrfs_dev_replace_finishing()
931 	dev_replace->item_needs_writeback = 1;  in btrfs_dev_replace_finishing()
952 		up_write(&dev_replace->rwsem);  in btrfs_dev_replace_finishing()
959 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
986 	dev_replace->replace_task = NULL;  in btrfs_dev_replace_finishing()
987 	up_write(&dev_replace->rwsem);  in btrfs_dev_replace_finishing()
1021 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
1035 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_progress()  local
1038 	switch (dev_replace->replace_state) {  in btrfs_dev_replace_progress()
1048 		ret = div64_u64(dev_replace->cursor_left,  in btrfs_dev_replace_progress()
1050 						dev_replace->srcdev), 1000));  in btrfs_dev_replace_progress()
1060 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_status()  local
1062 	down_read(&dev_replace->rwsem);  in btrfs_dev_replace_status()
1066 	args->status.replace_state = dev_replace->replace_state;  in btrfs_dev_replace_status()
1067 	args->status.time_started = dev_replace->time_started;  in btrfs_dev_replace_status()
1068 	args->status.time_stopped = dev_replace->time_stopped;  in btrfs_dev_replace_status()
1070 		atomic64_read(&dev_replace->num_write_errors);  in btrfs_dev_replace_status()
1072 		atomic64_read(&dev_replace->num_uncorrectable_read_errors);  in btrfs_dev_replace_status()
1074 	up_read(&dev_replace->rwsem);  in btrfs_dev_replace_status()
1079 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_cancel()  local
1090 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_cancel()
1091 	down_write(&dev_replace->rwsem);  in btrfs_dev_replace_cancel()
1092 	switch (dev_replace->replace_state) {  in btrfs_dev_replace_cancel()
1097 		up_write(&dev_replace->rwsem);  in btrfs_dev_replace_cancel()
1100 		tgt_device = dev_replace->tgtdev;  in btrfs_dev_replace_cancel()
1101 		src_device = dev_replace->srcdev;  in btrfs_dev_replace_cancel()
1102 		up_write(&dev_replace->rwsem);  in btrfs_dev_replace_cancel()
1124 		tgt_device = dev_replace->tgtdev;  in btrfs_dev_replace_cancel()
1125 		src_device = dev_replace->srcdev;  in btrfs_dev_replace_cancel()
1126 		dev_replace->tgtdev = NULL;  in btrfs_dev_replace_cancel()
1127 		dev_replace->srcdev = NULL;  in btrfs_dev_replace_cancel()
1128 		dev_replace->replace_state =  in btrfs_dev_replace_cancel()
1130 		dev_replace->time_stopped = ktime_get_real_seconds();  in btrfs_dev_replace_cancel()
1131 		dev_replace->item_needs_writeback = 1;  in btrfs_dev_replace_cancel()
1133 		up_write(&dev_replace->rwsem);  in btrfs_dev_replace_cancel()
1140 			mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_cancel()
1155 		up_write(&dev_replace->rwsem);  in btrfs_dev_replace_cancel()
1159 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_cancel()
1165 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_suspend_for_unmount()  local
1167 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_suspend_for_unmount()
1168 	down_write(&dev_replace->rwsem);  in btrfs_dev_replace_suspend_for_unmount()
1170 	switch (dev_replace->replace_state) {  in btrfs_dev_replace_suspend_for_unmount()
1177 		dev_replace->replace_state =  in btrfs_dev_replace_suspend_for_unmount()
1179 		dev_replace->time_stopped = ktime_get_real_seconds();  in btrfs_dev_replace_suspend_for_unmount()
1180 		dev_replace->item_needs_writeback = 1;  in btrfs_dev_replace_suspend_for_unmount()
1185 	up_write(&dev_replace->rwsem);  in btrfs_dev_replace_suspend_for_unmount()
1186 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_suspend_for_unmount()
1193 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_resume_dev_replace_async()  local
1195 	down_write(&dev_replace->rwsem);  in btrfs_resume_dev_replace_async()
1197 	switch (dev_replace->replace_state) {  in btrfs_resume_dev_replace_async()
1201 		up_write(&dev_replace->rwsem);  in btrfs_resume_dev_replace_async()
1206 		dev_replace->replace_state =  in btrfs_resume_dev_replace_async()
1210 	if (!dev_replace->tgtdev || !dev_replace->tgtdev->bdev) {  in btrfs_resume_dev_replace_async()
1215 		dev_replace->replace_state =  in btrfs_resume_dev_replace_async()
1217 		up_write(&dev_replace->rwsem);  in btrfs_resume_dev_replace_async()
1220 	up_write(&dev_replace->rwsem);  in btrfs_resume_dev_replace_async()
1228 		down_write(&dev_replace->rwsem);  in btrfs_resume_dev_replace_async()
1229 		dev_replace->replace_state =  in btrfs_resume_dev_replace_async()
1231 		up_write(&dev_replace->rwsem);  in btrfs_resume_dev_replace_async()
1244 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_kthread()  local
1252 		btrfs_dev_name(dev_replace->srcdev),  in btrfs_dev_replace_kthread()
1253 		dev_replace->srcdev->devid,  in btrfs_dev_replace_kthread()
1254 		btrfs_dev_name(dev_replace->tgtdev),  in btrfs_dev_replace_kthread()
1257 	ret = btrfs_scrub_dev(fs_info, dev_replace->srcdev->devid,  in btrfs_dev_replace_kthread()
1258 			      dev_replace->committed_cursor_left,  in btrfs_dev_replace_kthread()
1259 			      btrfs_device_get_total_bytes(dev_replace->srcdev),  in btrfs_dev_replace_kthread()
1260 			      &dev_replace->scrub_progress, 0, 1);  in btrfs_dev_replace_kthread()
1268 int __pure btrfs_dev_replace_is_ongoing(struct btrfs_dev_replace *dev_replace)  in btrfs_dev_replace_is_ongoing()  argument
1270 	if (!dev_replace->is_valid)  in btrfs_dev_replace_is_ongoing()
1273 	switch (dev_replace->replace_state) {  in btrfs_dev_replace_is_ongoing()
1297 	percpu_counter_sub(&fs_info->dev_replace.bio_counter, amount);  in btrfs_bio_counter_sub()
1298 	cond_wake_up_nomb(&fs_info->dev_replace.replace_wait);  in btrfs_bio_counter_sub()
1304 		percpu_counter_inc(&fs_info->dev_replace.bio_counter);  in btrfs_bio_counter_inc_blocked()
1310 		wait_event(fs_info->dev_replace.replace_wait,  in btrfs_bio_counter_inc_blocked()