Lines Matching refs:r10_bio
76 static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio);
134 struct r10bio *r10_bio; in r10buf_pool_alloc() local
140 r10_bio = r10bio_pool_alloc(gfp_flags, conf); in r10buf_pool_alloc()
141 if (!r10_bio) in r10buf_pool_alloc()
167 r10_bio->devs[j].bio = bio; in r10buf_pool_alloc()
174 r10_bio->devs[j].repl_bio = bio; in r10buf_pool_alloc()
181 struct bio *rbio = r10_bio->devs[j].repl_bio; in r10buf_pool_alloc()
188 bio = r10_bio->devs[j].bio; in r10buf_pool_alloc()
199 rp->raid_bio = r10_bio; in r10buf_pool_alloc()
207 return r10_bio; in r10buf_pool_alloc()
216 if (r10_bio->devs[j].bio) in r10buf_pool_alloc()
217 bio_uninit(r10_bio->devs[j].bio); in r10buf_pool_alloc()
218 kfree(r10_bio->devs[j].bio); in r10buf_pool_alloc()
219 if (r10_bio->devs[j].repl_bio) in r10buf_pool_alloc()
220 bio_uninit(r10_bio->devs[j].repl_bio); in r10buf_pool_alloc()
221 kfree(r10_bio->devs[j].repl_bio); in r10buf_pool_alloc()
225 rbio_pool_free(r10_bio, conf); in r10buf_pool_alloc()
259 static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio) in put_all_bios() argument
264 struct bio **bio = & r10_bio->devs[i].bio; in put_all_bios()
268 bio = &r10_bio->devs[i].repl_bio; in put_all_bios()
269 if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio)) in put_all_bios()
275 static void free_r10bio(struct r10bio *r10_bio) in free_r10bio() argument
277 struct r10conf *conf = r10_bio->mddev->private; in free_r10bio()
279 put_all_bios(conf, r10_bio); in free_r10bio()
280 mempool_free(r10_bio, &conf->r10bio_pool); in free_r10bio()
283 static void put_buf(struct r10bio *r10_bio) in put_buf() argument
285 struct r10conf *conf = r10_bio->mddev->private; in put_buf()
287 mempool_free(r10_bio, &conf->r10buf_pool); in put_buf()
298 static void reschedule_retry(struct r10bio *r10_bio) in reschedule_retry() argument
301 struct mddev *mddev = r10_bio->mddev; in reschedule_retry()
305 list_add(&r10_bio->retry_list, &conf->retry_list); in reschedule_retry()
320 static void raid_end_bio_io(struct r10bio *r10_bio) in raid_end_bio_io() argument
322 struct bio *bio = r10_bio->master_bio; in raid_end_bio_io()
323 struct r10conf *conf = r10_bio->mddev->private; in raid_end_bio_io()
325 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) in raid_end_bio_io()
335 free_r10bio(r10_bio); in raid_end_bio_io()
341 static inline void update_head_pos(int slot, struct r10bio *r10_bio) in update_head_pos() argument
343 struct r10conf *conf = r10_bio->mddev->private; in update_head_pos()
345 conf->mirrors[r10_bio->devs[slot].devnum].head_position = in update_head_pos()
346 r10_bio->devs[slot].addr + (r10_bio->sectors); in update_head_pos()
352 static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio, in find_bio_disk() argument
359 if (r10_bio->devs[slot].bio == bio) in find_bio_disk()
361 if (r10_bio->devs[slot].repl_bio == bio) { in find_bio_disk()
367 update_head_pos(slot, r10_bio); in find_bio_disk()
373 return r10_bio->devs[slot].devnum; in find_bio_disk()
379 struct r10bio *r10_bio = bio->bi_private; in raid10_end_read_request() local
382 struct r10conf *conf = r10_bio->mddev->private; in raid10_end_read_request()
384 slot = r10_bio->read_slot; in raid10_end_read_request()
385 rdev = r10_bio->devs[slot].rdev; in raid10_end_read_request()
389 update_head_pos(slot, r10_bio); in raid10_end_read_request()
401 set_bit(R10BIO_Uptodate, &r10_bio->state); in raid10_end_read_request()
408 if (!_enough(conf, test_bit(R10BIO_Previous, &r10_bio->state), in raid10_end_read_request()
413 raid_end_bio_io(r10_bio); in raid10_end_read_request()
422 (unsigned long long)r10_bio->sector); in raid10_end_read_request()
423 set_bit(R10BIO_ReadError, &r10_bio->state); in raid10_end_read_request()
424 reschedule_retry(r10_bio); in raid10_end_read_request()
428 static void close_write(struct r10bio *r10_bio) in close_write() argument
430 struct mddev *mddev = r10_bio->mddev; in close_write()
435 static void one_write_done(struct r10bio *r10_bio) in one_write_done() argument
437 if (atomic_dec_and_test(&r10_bio->remaining)) { in one_write_done()
438 if (test_bit(R10BIO_WriteError, &r10_bio->state)) in one_write_done()
439 reschedule_retry(r10_bio); in one_write_done()
441 close_write(r10_bio); in one_write_done()
442 if (test_bit(R10BIO_MadeGood, &r10_bio->state)) in one_write_done()
443 reschedule_retry(r10_bio); in one_write_done()
445 raid_end_bio_io(r10_bio); in one_write_done()
452 struct r10bio *r10_bio = bio->bi_private; in raid10_end_write_request() local
455 struct r10conf *conf = r10_bio->mddev->private; in raid10_end_write_request()
463 dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl); in raid10_end_write_request()
498 set_bit(R10BIO_WriteError, &r10_bio->state); in raid10_end_write_request()
501 r10_bio->devs[slot].bio = NULL; in raid10_end_write_request()
525 set_bit(R10BIO_Uptodate, &r10_bio->state); in raid10_end_write_request()
528 if (rdev_has_badblock(rdev, r10_bio->devs[slot].addr, in raid10_end_write_request()
529 r10_bio->sectors) && in raid10_end_write_request()
533 r10_bio->devs[slot].repl_bio = IO_MADE_GOOD; in raid10_end_write_request()
535 r10_bio->devs[slot].bio = IO_MADE_GOOD; in raid10_end_write_request()
537 set_bit(R10BIO_MadeGood, &r10_bio->state); in raid10_end_write_request()
546 one_write_done(r10_bio); in raid10_end_write_request()
721 struct r10bio *r10_bio, in read_balance() argument
724 const sector_t this_sector = r10_bio->sector; in read_balance()
726 int sectors = r10_bio->sectors; in read_balance()
736 raid10_find_phys(conf, r10_bio); in read_balance()
744 clear_bit(R10BIO_FailFast, &r10_bio->state); in read_balance()
756 if (r10_bio->devs[slot].bio == IO_BLOCKED) in read_balance()
758 disk = r10_bio->devs[slot].devnum; in read_balance()
761 r10_bio->devs[slot].addr + sectors > in read_balance()
768 r10_bio->devs[slot].addr + sectors > rdev->recovery_offset) in read_balance()
771 dev_sector = r10_bio->devs[slot].addr; in read_balance()
817 set_bit(R10BIO_FailFast, &r10_bio->state); in read_balance()
827 new_distance = r10_bio->devs[slot].addr; in read_balance()
829 new_distance = abs(r10_bio->devs[slot].addr - in read_balance()
850 r10_bio->read_slot = slot; in read_balance()
1071 static sector_t choose_data_offset(struct r10bio *r10_bio, in choose_data_offset() argument
1075 test_bit(R10BIO_Previous, &r10_bio->state)) in choose_data_offset()
1146 struct r10bio *r10_bio, bool io_accounting) in raid10_read_request() argument
1153 int slot = r10_bio->read_slot; in raid10_read_request()
1158 if (slot >= 0 && r10_bio->devs[slot].rdev) { in raid10_read_request()
1173 disk = r10_bio->devs[slot].devnum; in raid10_read_request()
1180 err_rdev = r10_bio->devs[slot].rdev; in raid10_read_request()
1184 if (!regular_request_wait(mddev, conf, bio, r10_bio->sectors)) in raid10_read_request()
1186 rdev = read_balance(conf, r10_bio, &max_sectors); in raid10_read_request()
1191 (unsigned long long)r10_bio->sector); in raid10_read_request()
1193 raid_end_bio_io(r10_bio); in raid10_read_request()
1200 (unsigned long long)r10_bio->sector); in raid10_read_request()
1213 r10_bio->master_bio = bio; in raid10_read_request()
1214 r10_bio->sectors = max_sectors; in raid10_read_request()
1216 slot = r10_bio->read_slot; in raid10_read_request()
1220 r10_bio->master_bio = bio; in raid10_read_request()
1224 r10_bio->devs[slot].bio = read_bio; in raid10_read_request()
1225 r10_bio->devs[slot].rdev = rdev; in raid10_read_request()
1227 read_bio->bi_iter.bi_sector = r10_bio->devs[slot].addr + in raid10_read_request()
1228 choose_data_offset(r10_bio, rdev); in raid10_read_request()
1231 test_bit(R10BIO_FailFast, &r10_bio->state)) in raid10_read_request()
1233 read_bio->bi_private = r10_bio; in raid10_read_request()
1234 mddev_trace_remap(mddev, read_bio, r10_bio->sector); in raid10_read_request()
1240 set_bit(R10BIO_Uptodate, &r10_bio->state); in raid10_read_request()
1241 raid_end_bio_io(r10_bio); in raid10_read_request()
1244 static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio, in raid10_write_one_disk() argument
1251 int devnum = r10_bio->devs[n_copy].devnum; in raid10_write_one_disk()
1259 r10_bio->devs[n_copy].repl_bio = mbio; in raid10_write_one_disk()
1261 r10_bio->devs[n_copy].bio = mbio; in raid10_write_one_disk()
1263 mbio->bi_iter.bi_sector = (r10_bio->devs[n_copy].addr + in raid10_write_one_disk()
1264 choose_data_offset(r10_bio, rdev)); in raid10_write_one_disk()
1270 mbio->bi_private = r10_bio; in raid10_write_one_disk()
1271 mddev_trace_remap(mddev, mbio, r10_bio->sector); in raid10_write_one_disk()
1275 atomic_inc(&r10_bio->remaining); in raid10_write_one_disk()
1285 static void wait_blocked_dev(struct mddev *mddev, struct r10bio *r10_bio) in wait_blocked_dev() argument
1298 sector_t dev_sector = r10_bio->devs[i].addr; in wait_blocked_dev()
1305 r10_bio->sectors && in wait_blocked_dev()
1307 r10_bio->sectors) < 0) in wait_blocked_dev()
1342 struct r10bio *r10_bio) in raid10_write_request() argument
1371 sectors = r10_bio->sectors; in raid10_write_request()
1408 r10_bio->read_slot = -1; /* make sure repl_bio gets freed */ in raid10_write_request()
1409 raid10_find_phys(conf, r10_bio); in raid10_write_request()
1411 wait_blocked_dev(mddev, r10_bio); in raid10_write_request()
1413 max_sectors = r10_bio->sectors; in raid10_write_request()
1416 int d = r10_bio->devs[i].devnum; in raid10_write_request()
1426 r10_bio->devs[i].bio = NULL; in raid10_write_request()
1427 r10_bio->devs[i].repl_bio = NULL; in raid10_write_request()
1433 sector_t dev_sector = r10_bio->devs[i].addr; in raid10_write_request()
1470 r10_bio->devs[i].bio = bio; in raid10_write_request()
1474 r10_bio->devs[i].repl_bio = bio; in raid10_write_request()
1479 if (max_sectors < r10_bio->sectors) in raid10_write_request()
1480 r10_bio->sectors = max_sectors; in raid10_write_request()
1482 if (r10_bio->sectors < bio_sectors(bio)) { in raid10_write_request()
1483 struct bio *split = bio_split(bio, r10_bio->sectors, in raid10_write_request()
1494 r10_bio->master_bio = bio; in raid10_write_request()
1498 r10_bio->master_bio = bio; in raid10_write_request()
1499 atomic_set(&r10_bio->remaining, 1); in raid10_write_request()
1502 if (r10_bio->devs[i].bio) in raid10_write_request()
1503 raid10_write_one_disk(mddev, r10_bio, bio, false, i); in raid10_write_request()
1504 if (r10_bio->devs[i].repl_bio) in raid10_write_request()
1505 raid10_write_one_disk(mddev, r10_bio, bio, true, i); in raid10_write_request()
1507 one_write_done(r10_bio); in raid10_write_request()
1511 int d = r10_bio->devs[k].devnum; in raid10_write_request()
1515 if (r10_bio->devs[k].bio) { in raid10_write_request()
1517 r10_bio->devs[k].bio = NULL; in raid10_write_request()
1519 if (r10_bio->devs[k].repl_bio) { in raid10_write_request()
1521 r10_bio->devs[k].repl_bio = NULL; in raid10_write_request()
1526 set_bit(R10BIO_Uptodate, &r10_bio->state); in raid10_write_request()
1527 raid_end_bio_io(r10_bio); in raid10_write_request()
1533 struct r10bio *r10_bio; in __make_request() local
1535 r10_bio = mempool_alloc(&conf->r10bio_pool, GFP_NOIO); in __make_request()
1537 r10_bio->master_bio = bio; in __make_request()
1538 r10_bio->sectors = sectors; in __make_request()
1540 r10_bio->mddev = mddev; in __make_request()
1541 r10_bio->sector = bio->bi_iter.bi_sector; in __make_request()
1542 r10_bio->state = 0; in __make_request()
1543 r10_bio->read_slot = -1; in __make_request()
1544 memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) * in __make_request()
1548 raid10_read_request(mddev, bio, r10_bio, true); in __make_request()
1550 raid10_write_request(mddev, bio, r10_bio); in __make_request()
1577 struct r10bio *r10_bio = bio->bi_private; in raid10_end_discard_request() local
1578 struct r10conf *conf = r10_bio->mddev->private; in raid10_end_discard_request()
1586 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) in raid10_end_discard_request()
1587 set_bit(R10BIO_Uptodate, &r10_bio->state); in raid10_end_discard_request()
1589 dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl); in raid10_end_discard_request()
1593 raid_end_discard_bio(r10_bio); in raid10_end_discard_request()
1609 struct r10bio *r10_bio, *first_r10bio; in raid10_handle_discard() local
1723 r10_bio = mempool_alloc(&conf->r10bio_pool, GFP_NOIO); in raid10_handle_discard()
1724 r10_bio->mddev = mddev; in raid10_handle_discard()
1725 r10_bio->state = 0; in raid10_handle_discard()
1726 r10_bio->sectors = 0; in raid10_handle_discard()
1727 memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) * geo->raid_disks); in raid10_handle_discard()
1728 wait_blocked_dev(mddev, r10_bio); in raid10_handle_discard()
1739 r10_bio->master_bio = bio; in raid10_handle_discard()
1740 set_bit(R10BIO_Discard, &r10_bio->state); in raid10_handle_discard()
1742 first_r10bio = r10_bio; in raid10_handle_discard()
1744 r10_bio->master_bio = (struct bio *)first_r10bio; in raid10_handle_discard()
1756 r10_bio->devs[disk].bio = NULL; in raid10_handle_discard()
1757 r10_bio->devs[disk].repl_bio = NULL; in raid10_handle_discard()
1767 r10_bio->devs[disk].bio = bio; in raid10_handle_discard()
1771 r10_bio->devs[disk].repl_bio = bio; in raid10_handle_discard()
1776 atomic_set(&r10_bio->remaining, 1); in raid10_handle_discard()
1813 if (r10_bio->devs[disk].bio) { in raid10_handle_discard()
1818 mbio->bi_private = r10_bio; in raid10_handle_discard()
1819 r10_bio->devs[disk].bio = mbio; in raid10_handle_discard()
1820 r10_bio->devs[disk].devnum = disk; in raid10_handle_discard()
1821 atomic_inc(&r10_bio->remaining); in raid10_handle_discard()
1823 dev_start + choose_data_offset(r10_bio, rdev), in raid10_handle_discard()
1827 if (r10_bio->devs[disk].repl_bio) { in raid10_handle_discard()
1832 rbio->bi_private = r10_bio; in raid10_handle_discard()
1833 r10_bio->devs[disk].repl_bio = rbio; in raid10_handle_discard()
1834 r10_bio->devs[disk].devnum = disk; in raid10_handle_discard()
1835 atomic_inc(&r10_bio->remaining); in raid10_handle_discard()
1837 dev_start + choose_data_offset(r10_bio, rrdev), in raid10_handle_discard()
1849 raid_end_discard_bio(r10_bio); in raid10_handle_discard()
1854 raid_end_discard_bio(r10_bio); in raid10_handle_discard()
2220 static void __end_sync_read(struct r10bio *r10_bio, struct bio *bio, int d) in __end_sync_read() argument
2222 struct r10conf *conf = r10_bio->mddev->private; in __end_sync_read()
2225 set_bit(R10BIO_Uptodate, &r10_bio->state); in __end_sync_read()
2230 atomic_add(r10_bio->sectors, in __end_sync_read()
2237 if (test_bit(R10BIO_IsRecover, &r10_bio->state) || in __end_sync_read()
2238 atomic_dec_and_test(&r10_bio->remaining)) { in __end_sync_read()
2242 reschedule_retry(r10_bio); in __end_sync_read()
2248 struct r10bio *r10_bio = get_resync_r10bio(bio); in end_sync_read() local
2249 struct r10conf *conf = r10_bio->mddev->private; in end_sync_read()
2250 int d = find_bio_disk(conf, r10_bio, bio, NULL, NULL); in end_sync_read()
2252 __end_sync_read(r10_bio, bio, d); in end_sync_read()
2258 struct r10bio *r10_bio = bio->bi_private; in end_reshape_read() local
2260 __end_sync_read(r10_bio, bio, r10_bio->read_slot); in end_reshape_read()
2263 static void end_sync_request(struct r10bio *r10_bio) in end_sync_request() argument
2265 struct mddev *mddev = r10_bio->mddev; in end_sync_request()
2267 while (atomic_dec_and_test(&r10_bio->remaining)) { in end_sync_request()
2268 if (r10_bio->master_bio == NULL) { in end_sync_request()
2270 sector_t s = r10_bio->sectors; in end_sync_request()
2271 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || in end_sync_request()
2272 test_bit(R10BIO_WriteError, &r10_bio->state)) in end_sync_request()
2273 reschedule_retry(r10_bio); in end_sync_request()
2275 put_buf(r10_bio); in end_sync_request()
2279 struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio; in end_sync_request()
2280 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || in end_sync_request()
2281 test_bit(R10BIO_WriteError, &r10_bio->state)) in end_sync_request()
2282 reschedule_retry(r10_bio); in end_sync_request()
2284 put_buf(r10_bio); in end_sync_request()
2285 r10_bio = r10_bio2; in end_sync_request()
2292 struct r10bio *r10_bio = get_resync_r10bio(bio); in end_sync_write() local
2293 struct mddev *mddev = r10_bio->mddev; in end_sync_write()
2300 d = find_bio_disk(conf, r10_bio, bio, &slot, &repl); in end_sync_write()
2314 set_bit(R10BIO_WriteError, &r10_bio->state); in end_sync_write()
2316 } else if (rdev_has_badblock(rdev, r10_bio->devs[slot].addr, in end_sync_write()
2317 r10_bio->sectors)) { in end_sync_write()
2318 set_bit(R10BIO_MadeGood, &r10_bio->state); in end_sync_write()
2323 end_sync_request(r10_bio); in end_sync_write()
2342 static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio) in sync_request_write() argument
2350 atomic_set(&r10_bio->remaining, 1); in sync_request_write()
2354 if (!r10_bio->devs[i].bio->bi_status) in sync_request_write()
2361 fbio = r10_bio->devs[i].bio; in sync_request_write()
2362 fbio->bi_iter.bi_size = r10_bio->sectors << 9; in sync_request_write()
2366 vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9); in sync_request_write()
2373 tbio = r10_bio->devs[i].bio; in sync_request_write()
2381 d = r10_bio->devs[i].devnum; in sync_request_write()
2383 if (!r10_bio->devs[i].bio->bi_status) { in sync_request_write()
2388 int sectors = r10_bio->sectors; in sync_request_write()
2401 atomic64_add(r10_bio->sectors, &mddev->resync_mismatches); in sync_request_write()
2420 rp->raid_bio = r10_bio; in sync_request_write()
2422 tbio->bi_iter.bi_sector = r10_bio->devs[i].addr; in sync_request_write()
2428 atomic_inc(&r10_bio->remaining); in sync_request_write()
2443 tbio = r10_bio->devs[i].repl_bio; in sync_request_write()
2446 if (r10_bio->devs[i].bio->bi_end_io != end_sync_write in sync_request_write()
2447 && r10_bio->devs[i].bio != fbio) in sync_request_write()
2449 d = r10_bio->devs[i].devnum; in sync_request_write()
2450 atomic_inc(&r10_bio->remaining); in sync_request_write()
2457 if (atomic_dec_and_test(&r10_bio->remaining)) { in sync_request_write()
2458 md_done_sync(mddev, r10_bio->sectors, 1); in sync_request_write()
2459 put_buf(r10_bio); in sync_request_write()
2473 static void fix_recovery_read_error(struct r10bio *r10_bio) in fix_recovery_read_error() argument
2482 struct mddev *mddev = r10_bio->mddev; in fix_recovery_read_error()
2484 struct bio *bio = r10_bio->devs[0].bio; in fix_recovery_read_error()
2486 int sectors = r10_bio->sectors; in fix_recovery_read_error()
2488 int dr = r10_bio->devs[0].devnum; in fix_recovery_read_error()
2489 int dw = r10_bio->devs[1].devnum; in fix_recovery_read_error()
2502 addr = r10_bio->devs[0].addr + sect; in fix_recovery_read_error()
2510 addr = r10_bio->devs[1].addr + sect; in fix_recovery_read_error()
2534 addr = r10_bio->devs[1].addr + sect; in fix_recovery_read_error()
2556 static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio) in recovery_request_write() argument
2560 struct bio *wbio = r10_bio->devs[1].bio; in recovery_request_write()
2561 struct bio *wbio2 = r10_bio->devs[1].repl_bio; in recovery_request_write()
2570 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) { in recovery_request_write()
2571 fix_recovery_read_error(r10_bio); in recovery_request_write()
2573 end_sync_request(r10_bio); in recovery_request_write()
2575 end_sync_request(r10_bio); in recovery_request_write()
2583 d = r10_bio->devs[1].devnum; in recovery_request_write()
2626 static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio) in fix_read_error() argument
2629 int sectors = r10_bio->sectors, slot = r10_bio->read_slot; in fix_read_error()
2631 int d = r10_bio->devs[slot].devnum; in fix_read_error()
2644 r10_bio->devs[slot].bio = IO_BLOCKED; in fix_read_error()
2658 d = r10_bio->devs[sl].devnum; in fix_read_error()
2664 r10_bio->devs[sl].addr + sect, in fix_read_error()
2668 r10_bio->devs[sl].addr + in fix_read_error()
2687 int dn = r10_bio->devs[slot].devnum; in fix_read_error()
2692 r10_bio->devs[slot].addr in fix_read_error()
2696 r10_bio->devs[slot].bio in fix_read_error()
2708 d = r10_bio->devs[sl].devnum; in fix_read_error()
2717 r10_bio->devs[sl].addr + in fix_read_error()
2726 choose_data_offset(r10_bio, in fix_read_error()
2740 d = r10_bio->devs[sl].devnum; in fix_read_error()
2749 r10_bio->devs[sl].addr + in fix_read_error()
2758 choose_data_offset(r10_bio, rdev)), in fix_read_error()
2769 choose_data_offset(r10_bio, rdev)), in fix_read_error()
2782 static bool narrow_write_error(struct r10bio *r10_bio, int i) in narrow_write_error() argument
2784 struct bio *bio = r10_bio->master_bio; in narrow_write_error()
2785 struct mddev *mddev = r10_bio->mddev; in narrow_write_error()
2787 struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev; in narrow_write_error()
2802 int sect_to_write = r10_bio->sectors; in narrow_write_error()
2810 sector = r10_bio->sector; in narrow_write_error()
2811 sectors = ((r10_bio->sector + block_sectors) in narrow_write_error()
2824 wsector = r10_bio->devs[i].addr + (sector - r10_bio->sector); in narrow_write_error()
2826 choose_data_offset(r10_bio, rdev); in narrow_write_error()
2843 static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio) in handle_read_error() argument
2845 int slot = r10_bio->read_slot; in handle_read_error()
2848 struct md_rdev *rdev = r10_bio->devs[slot].rdev; in handle_read_error()
2858 bio = r10_bio->devs[slot].bio; in handle_read_error()
2860 r10_bio->devs[slot].bio = NULL; in handle_read_error()
2863 r10_bio->devs[slot].bio = IO_BLOCKED; in handle_read_error()
2866 fix_read_error(conf, mddev, r10_bio); in handle_read_error()
2872 r10_bio->state = 0; in handle_read_error()
2873 raid10_read_request(mddev, r10_bio->master_bio, r10_bio, false); in handle_read_error()
2881 static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio) in handle_write_completed() argument
2892 if (test_bit(R10BIO_IsSync, &r10_bio->state) || in handle_write_completed()
2893 test_bit(R10BIO_IsRecover, &r10_bio->state)) { in handle_write_completed()
2895 int dev = r10_bio->devs[m].devnum; in handle_write_completed()
2897 if (r10_bio->devs[m].bio == NULL || in handle_write_completed()
2898 r10_bio->devs[m].bio->bi_end_io == NULL) in handle_write_completed()
2900 if (!r10_bio->devs[m].bio->bi_status) { in handle_write_completed()
2903 r10_bio->devs[m].addr, in handle_write_completed()
2904 r10_bio->sectors, 0); in handle_write_completed()
2908 r10_bio->devs[m].addr, in handle_write_completed()
2909 r10_bio->sectors, 0)) in handle_write_completed()
2913 if (r10_bio->devs[m].repl_bio == NULL || in handle_write_completed()
2914 r10_bio->devs[m].repl_bio->bi_end_io == NULL) in handle_write_completed()
2917 if (!r10_bio->devs[m].repl_bio->bi_status) { in handle_write_completed()
2920 r10_bio->devs[m].addr, in handle_write_completed()
2921 r10_bio->sectors, 0); in handle_write_completed()
2925 r10_bio->devs[m].addr, in handle_write_completed()
2926 r10_bio->sectors, 0)) in handle_write_completed()
2930 put_buf(r10_bio); in handle_write_completed()
2934 int dev = r10_bio->devs[m].devnum; in handle_write_completed()
2935 struct bio *bio = r10_bio->devs[m].bio; in handle_write_completed()
2940 r10_bio->devs[m].addr, in handle_write_completed()
2941 r10_bio->sectors, 0); in handle_write_completed()
2945 if (!narrow_write_error(r10_bio, m)) in handle_write_completed()
2949 bio = r10_bio->devs[m].repl_bio; in handle_write_completed()
2954 r10_bio->devs[m].addr, in handle_write_completed()
2955 r10_bio->sectors, 0); in handle_write_completed()
2961 list_add(&r10_bio->retry_list, &conf->bio_end_io_list); in handle_write_completed()
2972 &r10_bio->state)) in handle_write_completed()
2973 close_write(r10_bio); in handle_write_completed()
2974 raid_end_bio_io(r10_bio); in handle_write_completed()
2982 struct r10bio *r10_bio; in raid10d() local
3002 r10_bio = list_first_entry(&tmp, struct r10bio, in raid10d()
3004 list_del(&r10_bio->retry_list); in raid10d()
3007 &r10_bio->state)) in raid10d()
3008 close_write(r10_bio); in raid10d()
3009 raid_end_bio_io(r10_bio); in raid10d()
3023 r10_bio = list_entry(head->prev, struct r10bio, retry_list); in raid10d()
3028 mddev = r10_bio->mddev; in raid10d()
3030 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || in raid10d()
3031 test_bit(R10BIO_WriteError, &r10_bio->state)) in raid10d()
3032 handle_write_completed(conf, r10_bio); in raid10d()
3033 else if (test_bit(R10BIO_IsReshape, &r10_bio->state)) in raid10d()
3034 reshape_request_write(mddev, r10_bio); in raid10d()
3035 else if (test_bit(R10BIO_IsSync, &r10_bio->state)) in raid10d()
3036 sync_request_write(mddev, r10_bio); in raid10d()
3037 else if (test_bit(R10BIO_IsRecover, &r10_bio->state)) in raid10d()
3038 recovery_request_write(mddev, r10_bio); in raid10d()
3039 else if (test_bit(R10BIO_ReadError, &r10_bio->state)) in raid10d()
3040 handle_read_error(mddev, r10_bio); in raid10d()
3171 struct r10bio *r10_bio; in raid10_sync_request() local
3318 r10_bio = NULL; in raid10_sync_request()
3343 rb2 = r10_bio; in raid10_sync_request()
3373 r10_bio = raid10_alloc_init_r10buf(conf); in raid10_sync_request()
3374 r10_bio->state = 0; in raid10_sync_request()
3376 atomic_set(&r10_bio->remaining, 0); in raid10_sync_request()
3378 r10_bio->master_bio = (struct bio*)rb2; in raid10_sync_request()
3381 r10_bio->mddev = mddev; in raid10_sync_request()
3382 set_bit(R10BIO_IsRecover, &r10_bio->state); in raid10_sync_request()
3383 r10_bio->sector = sect; in raid10_sync_request()
3385 raid10_find_phys(conf, r10_bio); in raid10_sync_request()
3405 int d = r10_bio->devs[j].devnum; in raid10_sync_request()
3415 sector = r10_bio->devs[j].addr; in raid10_sync_request()
3429 bio = r10_bio->devs[0].bio; in raid10_sync_request()
3436 from_addr = r10_bio->devs[j].addr; in raid10_sync_request()
3444 if (r10_bio->devs[k].devnum == i) in raid10_sync_request()
3447 to_addr = r10_bio->devs[k].addr; in raid10_sync_request()
3448 r10_bio->devs[0].devnum = d; in raid10_sync_request()
3449 r10_bio->devs[0].addr = from_addr; in raid10_sync_request()
3450 r10_bio->devs[1].devnum = i; in raid10_sync_request()
3451 r10_bio->devs[1].addr = to_addr; in raid10_sync_request()
3454 bio = r10_bio->devs[1].bio; in raid10_sync_request()
3462 atomic_inc(&r10_bio->remaining); in raid10_sync_request()
3464 r10_bio->devs[1].bio->bi_end_io = NULL; in raid10_sync_request()
3467 bio = r10_bio->devs[1].repl_bio; in raid10_sync_request()
3483 atomic_inc(&r10_bio->remaining); in raid10_sync_request()
3495 if (r10_bio->devs[k].devnum == i) in raid10_sync_request()
3501 r10_bio->devs[k].addr, in raid10_sync_request()
3507 r10_bio->devs[k].addr, in raid10_sync_request()
3521 put_buf(r10_bio); in raid10_sync_request()
3524 r10_bio = rb2; in raid10_sync_request()
3535 if (r10_bio->devs[0].bio->bi_opf & MD_FAILFAST) { in raid10_sync_request()
3542 int d = r10_bio->devs[j].devnum; in raid10_sync_request()
3549 r10_bio->devs[0].bio->bi_opf in raid10_sync_request()
3554 while (r10_bio) { in raid10_sync_request()
3555 struct r10bio *rb2 = r10_bio; in raid10_sync_request()
3556 r10_bio = (struct r10bio*) rb2->master_bio; in raid10_sync_request()
3588 r10_bio = raid10_alloc_init_r10buf(conf); in raid10_sync_request()
3589 r10_bio->state = 0; in raid10_sync_request()
3591 r10_bio->mddev = mddev; in raid10_sync_request()
3592 atomic_set(&r10_bio->remaining, 0); in raid10_sync_request()
3596 r10_bio->master_bio = NULL; in raid10_sync_request()
3597 r10_bio->sector = sector_nr; in raid10_sync_request()
3598 set_bit(R10BIO_IsSync, &r10_bio->state); in raid10_sync_request()
3599 raid10_find_phys(conf, r10_bio); in raid10_sync_request()
3600 r10_bio->sectors = (sector_nr | chunk_mask) - sector_nr + 1; in raid10_sync_request()
3603 int d = r10_bio->devs[i].devnum; in raid10_sync_request()
3608 if (r10_bio->devs[i].repl_bio) in raid10_sync_request()
3609 r10_bio->devs[i].repl_bio->bi_end_io = NULL; in raid10_sync_request()
3611 bio = r10_bio->devs[i].bio; in raid10_sync_request()
3617 sector = r10_bio->devs[i].addr; in raid10_sync_request()
3630 atomic_inc(&r10_bio->remaining); in raid10_sync_request()
3648 bio = r10_bio->devs[i].repl_bio; in raid10_sync_request()
3651 sector = r10_bio->devs[i].addr; in raid10_sync_request()
3665 int d = r10_bio->devs[i].devnum; in raid10_sync_request()
3666 if (r10_bio->devs[i].bio->bi_end_io) in raid10_sync_request()
3669 if (r10_bio->devs[i].repl_bio && in raid10_sync_request()
3670 r10_bio->devs[i].repl_bio->bi_end_io) in raid10_sync_request()
3675 put_buf(r10_bio); in raid10_sync_request()
3703 r10_bio->sectors = nr_sectors; in raid10_sync_request()
3756 r10_bio = get_resync_r10bio(bio); in raid10_sync_request()
3757 r10_bio->sectors = nr_sectors; in raid10_sync_request()
4671 struct r10bio *r10_bio; in reshape_request() local
4776 r10_bio = raid10_alloc_init_r10buf(conf); in reshape_request()
4777 r10_bio->state = 0; in reshape_request()
4779 atomic_set(&r10_bio->remaining, 0); in reshape_request()
4780 r10_bio->mddev = mddev; in reshape_request()
4781 r10_bio->sector = sector_nr; in reshape_request()
4782 set_bit(R10BIO_IsReshape, &r10_bio->state); in reshape_request()
4783 r10_bio->sectors = last - sector_nr + 1; in reshape_request()
4784 rdev = read_balance(conf, r10_bio, &max_sectors); in reshape_request()
4785 BUG_ON(!test_bit(R10BIO_Previous, &r10_bio->state)); in reshape_request()
4792 mempool_free(r10_bio, &conf->r10buf_pool); in reshape_request()
4799 read_bio->bi_iter.bi_sector = (r10_bio->devs[r10_bio->read_slot].addr in reshape_request()
4801 read_bio->bi_private = r10_bio; in reshape_request()
4803 r10_bio->master_bio = read_bio; in reshape_request()
4804 r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum; in reshape_request()
4833 __raid10_find_phys(&conf->geo, r10_bio); in reshape_request()
4840 int d = r10_bio->devs[s/2].devnum; in reshape_request()
4844 b = r10_bio->devs[s/2].repl_bio; in reshape_request()
4847 b = r10_bio->devs[s/2].bio; in reshape_request()
4853 b->bi_iter.bi_sector = r10_bio->devs[s/2].addr + in reshape_request()
4864 pages = get_resync_pages(r10_bio->devs[0].bio)->pages; in reshape_request()
4880 r10_bio->sectors = nr_sectors; in reshape_request()
4883 md_sync_acct_bio(read_bio, r10_bio->sectors); in reshape_request()
4884 atomic_inc(&r10_bio->remaining); in reshape_request()
4904 static void end_reshape_request(struct r10bio *r10_bio);
4906 struct r10bio *r10_bio);
4907 static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio) in reshape_request_write() argument
4917 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) in reshape_request_write()
4918 if (handle_reshape_read_error(mddev, r10_bio) < 0) { in reshape_request_write()
4920 md_done_sync(mddev, r10_bio->sectors, 0); in reshape_request_write()
4927 atomic_set(&r10_bio->remaining, 1); in reshape_request_write()
4930 int d = r10_bio->devs[s/2].devnum; in reshape_request_write()
4934 b = r10_bio->devs[s/2].repl_bio; in reshape_request_write()
4937 b = r10_bio->devs[s/2].bio; in reshape_request_write()
4943 md_sync_acct_bio(b, r10_bio->sectors); in reshape_request_write()
4944 atomic_inc(&r10_bio->remaining); in reshape_request_write()
4948 end_reshape_request(r10_bio); in reshape_request_write()
4982 struct r10bio *r10_bio) in handle_reshape_read_error() argument
4985 int sectors = r10_bio->sectors; in handle_reshape_read_error()
4999 pages = get_resync_pages(r10_bio->devs[0].bio)->pages; in handle_reshape_read_error()
5001 r10b->sector = r10_bio->sector; in handle_reshape_read_error()
5054 struct r10bio *r10_bio = get_resync_r10bio(bio); in end_reshape_write() local
5055 struct mddev *mddev = r10_bio->mddev; in end_reshape_write()
5062 d = find_bio_disk(conf, r10_bio, bio, &slot, &repl); in end_reshape_write()
5072 end_reshape_request(r10_bio); in end_reshape_write()
5075 static void end_reshape_request(struct r10bio *r10_bio) in end_reshape_request() argument
5077 if (!atomic_dec_and_test(&r10_bio->remaining)) in end_reshape_request()
5079 md_done_sync(r10_bio->mddev, r10_bio->sectors, 1); in end_reshape_request()
5080 bio_put(r10_bio->master_bio); in end_reshape_request()
5081 put_buf(r10_bio); in end_reshape_request()