Lines Matching refs:bh
57 static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh,
62 inline void touch_buffer(struct buffer_head *bh) in touch_buffer() argument
64 trace_block_touch_buffer(bh); in touch_buffer()
65 folio_mark_accessed(bh->b_folio); in touch_buffer()
69 void __lock_buffer(struct buffer_head *bh) in __lock_buffer() argument
71 wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE); in __lock_buffer()
75 void unlock_buffer(struct buffer_head *bh) in unlock_buffer() argument
77 clear_bit_unlock(BH_Lock, &bh->b_state); in unlock_buffer()
79 wake_up_bit(&bh->b_state, BH_Lock); in unlock_buffer()
91 struct buffer_head *head, *bh; in buffer_check_dirty_writeback() local
104 bh = head; in buffer_check_dirty_writeback()
106 if (buffer_locked(bh)) in buffer_check_dirty_writeback()
109 if (buffer_dirty(bh)) in buffer_check_dirty_writeback()
112 bh = bh->b_this_page; in buffer_check_dirty_writeback()
113 } while (bh != head); in buffer_check_dirty_writeback()
121 void __wait_on_buffer(struct buffer_head * bh) in __wait_on_buffer() argument
123 wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE); in __wait_on_buffer()
127 static void buffer_io_error(struct buffer_head *bh, char *msg) in buffer_io_error() argument
129 if (!test_bit(BH_Quiet, &bh->b_state)) in buffer_io_error()
132 bh->b_bdev, (unsigned long long)bh->b_blocknr, msg); in buffer_io_error()
143 static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate) in __end_buffer_read_notouch() argument
146 set_buffer_uptodate(bh); in __end_buffer_read_notouch()
149 clear_buffer_uptodate(bh); in __end_buffer_read_notouch()
151 unlock_buffer(bh); in __end_buffer_read_notouch()
158 void end_buffer_read_sync(struct buffer_head *bh, int uptodate) in end_buffer_read_sync() argument
160 __end_buffer_read_notouch(bh, uptodate); in end_buffer_read_sync()
161 put_bh(bh); in end_buffer_read_sync()
165 void end_buffer_write_sync(struct buffer_head *bh, int uptodate) in end_buffer_write_sync() argument
168 set_buffer_uptodate(bh); in end_buffer_write_sync()
170 buffer_io_error(bh, ", lost sync page write"); in end_buffer_write_sync()
171 mark_buffer_write_io_error(bh); in end_buffer_write_sync()
172 clear_buffer_uptodate(bh); in end_buffer_write_sync()
174 unlock_buffer(bh); in end_buffer_write_sync()
175 put_bh(bh); in end_buffer_write_sync()
186 struct buffer_head *bh; in __find_get_block_slow() local
219 bh = head; in __find_get_block_slow()
221 if (!buffer_mapped(bh)) in __find_get_block_slow()
223 else if (bh->b_blocknr == block) { in __find_get_block_slow()
224 ret = bh; in __find_get_block_slow()
225 get_bh(bh); in __find_get_block_slow()
228 bh = bh->b_this_page; in __find_get_block_slow()
229 } while (bh != head); in __find_get_block_slow()
242 (unsigned long long)bh->b_blocknr, in __find_get_block_slow()
243 bh->b_state, bh->b_size, bdev, in __find_get_block_slow()
256 static void end_buffer_async_read(struct buffer_head *bh, int uptodate) in end_buffer_async_read() argument
264 BUG_ON(!buffer_async_read(bh)); in end_buffer_async_read()
266 folio = bh->b_folio; in end_buffer_async_read()
268 set_buffer_uptodate(bh); in end_buffer_async_read()
270 clear_buffer_uptodate(bh); in end_buffer_async_read()
271 buffer_io_error(bh, ", async page read"); in end_buffer_async_read()
281 clear_buffer_async_read(bh); in end_buffer_async_read()
282 unlock_buffer(bh); in end_buffer_async_read()
283 tmp = bh; in end_buffer_async_read()
292 } while (tmp != bh); in end_buffer_async_read()
304 struct buffer_head *bh; member
311 struct buffer_head *bh = ctx->bh; in verify_bh() local
314 valid = fsverity_verify_blocks(bh->b_folio, bh->b_size, bh_offset(bh)); in verify_bh()
315 end_buffer_async_read(bh, valid); in verify_bh()
319 static bool need_fsverity(struct buffer_head *bh) in need_fsverity() argument
321 struct folio *folio = bh->b_folio; in need_fsverity()
333 struct buffer_head *bh = ctx->bh; in decrypt_bh() local
336 err = fscrypt_decrypt_pagecache_blocks(bh->b_folio, bh->b_size, in decrypt_bh()
337 bh_offset(bh)); in decrypt_bh()
338 if (err == 0 && need_fsverity(bh)) { in decrypt_bh()
348 end_buffer_async_read(bh, err == 0); in decrypt_bh()
356 static void end_buffer_async_read_io(struct buffer_head *bh, int uptodate) in end_buffer_async_read_io() argument
358 struct inode *inode = bh->b_folio->mapping->host; in end_buffer_async_read_io()
360 bool verify = need_fsverity(bh); in end_buffer_async_read_io()
368 ctx->bh = bh; in end_buffer_async_read_io()
380 end_buffer_async_read(bh, uptodate); in end_buffer_async_read_io()
387 static void end_buffer_async_write(struct buffer_head *bh, int uptodate) in end_buffer_async_write() argument
394 BUG_ON(!buffer_async_write(bh)); in end_buffer_async_write()
396 folio = bh->b_folio; in end_buffer_async_write()
398 set_buffer_uptodate(bh); in end_buffer_async_write()
400 buffer_io_error(bh, ", lost async page write"); in end_buffer_async_write()
401 mark_buffer_write_io_error(bh); in end_buffer_async_write()
402 clear_buffer_uptodate(bh); in end_buffer_async_write()
408 clear_buffer_async_write(bh); in end_buffer_async_write()
409 unlock_buffer(bh); in end_buffer_async_write()
410 tmp = bh->b_this_page; in end_buffer_async_write()
411 while (tmp != bh) { in end_buffer_async_write()
447 static void mark_buffer_async_read(struct buffer_head *bh) in mark_buffer_async_read() argument
449 bh->b_end_io = end_buffer_async_read_io; in mark_buffer_async_read()
450 set_buffer_async_read(bh); in mark_buffer_async_read()
453 static void mark_buffer_async_write_endio(struct buffer_head *bh, in mark_buffer_async_write_endio() argument
456 bh->b_end_io = handler; in mark_buffer_async_write_endio()
457 set_buffer_async_write(bh); in mark_buffer_async_write_endio()
460 void mark_buffer_async_write(struct buffer_head *bh) in mark_buffer_async_write() argument
462 mark_buffer_async_write_endio(bh, end_buffer_async_write); in mark_buffer_async_write()
519 static void __remove_assoc_queue(struct buffer_head *bh) in __remove_assoc_queue() argument
521 list_del_init(&bh->b_assoc_buffers); in __remove_assoc_queue()
522 WARN_ON(!bh->b_assoc_map); in __remove_assoc_queue()
523 bh->b_assoc_map = NULL; in __remove_assoc_queue()
543 struct buffer_head *bh; in osync_buffers_list() local
550 bh = BH_ENTRY(p); in osync_buffers_list()
551 if (buffer_locked(bh)) { in osync_buffers_list()
552 get_bh(bh); in osync_buffers_list()
554 wait_on_buffer(bh); in osync_buffers_list()
555 if (!buffer_uptodate(bh)) in osync_buffers_list()
557 brelse(bh); in osync_buffers_list()
668 struct buffer_head *bh; in write_boundary_block() local
670 bh = __find_get_block_nonatomic(bdev, bblock + 1, blocksize); in write_boundary_block()
671 if (bh) { in write_boundary_block()
672 if (buffer_dirty(bh)) in write_boundary_block()
673 write_dirty_buffer(bh, 0); in write_boundary_block()
674 put_bh(bh); in write_boundary_block()
678 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode) in mark_buffer_dirty_inode() argument
681 struct address_space *buffer_mapping = bh->b_folio->mapping; in mark_buffer_dirty_inode()
683 mark_buffer_dirty(bh); in mark_buffer_dirty_inode()
689 if (!bh->b_assoc_map) { in mark_buffer_dirty_inode()
691 list_move_tail(&bh->b_assoc_buffers, in mark_buffer_dirty_inode()
693 bh->b_assoc_map = mapping; in mark_buffer_dirty_inode()
739 struct buffer_head *bh = head; in block_dirty_folio() local
742 set_buffer_dirty(bh); in block_dirty_folio()
743 bh = bh->b_this_page; in block_dirty_folio()
744 } while (bh != head); in block_dirty_folio()
784 struct buffer_head *bh; in fsync_buffers_list() local
794 bh = BH_ENTRY(list->next); in fsync_buffers_list()
795 mapping = bh->b_assoc_map; in fsync_buffers_list()
796 __remove_assoc_queue(bh); in fsync_buffers_list()
800 if (buffer_dirty(bh) || buffer_locked(bh)) { in fsync_buffers_list()
801 list_add(&bh->b_assoc_buffers, &tmp); in fsync_buffers_list()
802 bh->b_assoc_map = mapping; in fsync_buffers_list()
803 if (buffer_dirty(bh)) { in fsync_buffers_list()
804 get_bh(bh); in fsync_buffers_list()
813 write_dirty_buffer(bh, REQ_SYNC); in fsync_buffers_list()
821 brelse(bh); in fsync_buffers_list()
832 bh = BH_ENTRY(tmp.prev); in fsync_buffers_list()
833 get_bh(bh); in fsync_buffers_list()
834 mapping = bh->b_assoc_map; in fsync_buffers_list()
835 __remove_assoc_queue(bh); in fsync_buffers_list()
839 if (buffer_dirty(bh)) { in fsync_buffers_list()
840 list_add(&bh->b_assoc_buffers, in fsync_buffers_list()
842 bh->b_assoc_map = mapping; in fsync_buffers_list()
845 wait_on_buffer(bh); in fsync_buffers_list()
846 if (!buffer_uptodate(bh)) in fsync_buffers_list()
848 brelse(bh); in fsync_buffers_list()
900 struct buffer_head *bh = BH_ENTRY(list->next); in remove_inode_buffers() local
901 if (buffer_dirty(bh)) { in remove_inode_buffers()
905 __remove_assoc_queue(bh); in remove_inode_buffers()
924 struct buffer_head *bh, *head; in folio_alloc_buffers() local
935 bh = alloc_buffer_head(gfp); in folio_alloc_buffers()
936 if (!bh) in folio_alloc_buffers()
939 bh->b_this_page = head; in folio_alloc_buffers()
940 bh->b_blocknr = -1; in folio_alloc_buffers()
941 head = bh; in folio_alloc_buffers()
943 bh->b_size = size; in folio_alloc_buffers()
946 folio_set_bh(bh, folio, offset); in folio_alloc_buffers()
957 bh = head; in folio_alloc_buffers()
959 free_buffer_head(bh); in folio_alloc_buffers()
978 struct buffer_head *bh, *tail; in link_dev_buffers() local
980 bh = head; in link_dev_buffers()
982 tail = bh; in link_dev_buffers()
983 bh = bh->b_this_page; in link_dev_buffers()
984 } while (bh); in link_dev_buffers()
1008 struct buffer_head *bh = head; in folio_init_buffers() local
1014 if (!buffer_mapped(bh)) { in folio_init_buffers()
1015 bh->b_end_io = NULL; in folio_init_buffers()
1016 bh->b_private = NULL; in folio_init_buffers()
1017 bh->b_bdev = bdev; in folio_init_buffers()
1018 bh->b_blocknr = block; in folio_init_buffers()
1020 set_buffer_uptodate(bh); in folio_init_buffers()
1022 set_buffer_mapped(bh); in folio_init_buffers()
1025 bh = bh->b_this_page; in folio_init_buffers()
1026 } while (bh != head); in folio_init_buffers()
1047 struct buffer_head *bh; in grow_dev_folio() local
1055 bh = folio_buffers(folio); in grow_dev_folio()
1056 if (bh) { in grow_dev_folio()
1057 if (bh->b_size == size) { in grow_dev_folio()
1075 bh = folio_alloc_buffers(folio, size, gfp | __GFP_ACCOUNT); in grow_dev_folio()
1076 if (!bh) in grow_dev_folio()
1085 link_dev_buffers(folio, bh); in grow_dev_folio()
1138 struct buffer_head *bh; in __getblk_slow() local
1144 bh = __find_get_block_nonatomic(bdev, block, size); in __getblk_slow()
1146 bh = __find_get_block(bdev, block, size); in __getblk_slow()
1147 if (bh) in __getblk_slow()
1148 return bh; in __getblk_slow()
1187 void mark_buffer_dirty(struct buffer_head *bh) in mark_buffer_dirty() argument
1189 WARN_ON_ONCE(!buffer_uptodate(bh)); in mark_buffer_dirty()
1191 trace_block_dirty_buffer(bh); in mark_buffer_dirty()
1199 if (buffer_dirty(bh)) { in mark_buffer_dirty()
1201 if (buffer_dirty(bh)) in mark_buffer_dirty()
1205 if (!test_set_buffer_dirty(bh)) { in mark_buffer_dirty()
1206 struct folio *folio = bh->b_folio; in mark_buffer_dirty()
1220 void mark_buffer_write_io_error(struct buffer_head *bh) in mark_buffer_write_io_error() argument
1222 set_buffer_write_io_error(bh); in mark_buffer_write_io_error()
1224 if (bh->b_folio && bh->b_folio->mapping) in mark_buffer_write_io_error()
1225 mapping_set_error(bh->b_folio->mapping, -EIO); in mark_buffer_write_io_error()
1226 if (bh->b_assoc_map) in mark_buffer_write_io_error()
1227 mapping_set_error(bh->b_assoc_map, -EIO); in mark_buffer_write_io_error()
1237 void __brelse(struct buffer_head *bh) in __brelse() argument
1239 if (atomic_read(&bh->b_count)) { in __brelse()
1240 put_bh(bh); in __brelse()
1254 void __bforget(struct buffer_head *bh) in __bforget() argument
1256 clear_buffer_dirty(bh); in __bforget()
1257 if (bh->b_assoc_map) { in __bforget()
1258 struct address_space *buffer_mapping = bh->b_folio->mapping; in __bforget()
1261 list_del_init(&bh->b_assoc_buffers); in __bforget()
1262 bh->b_assoc_map = NULL; in __bforget()
1265 __brelse(bh); in __bforget()
1269 static struct buffer_head *__bread_slow(struct buffer_head *bh) in __bread_slow() argument
1271 lock_buffer(bh); in __bread_slow()
1272 if (buffer_uptodate(bh)) { in __bread_slow()
1273 unlock_buffer(bh); in __bread_slow()
1274 return bh; in __bread_slow()
1276 get_bh(bh); in __bread_slow()
1277 bh->b_end_io = end_buffer_read_sync; in __bread_slow()
1278 submit_bh(REQ_OP_READ, bh); in __bread_slow()
1279 wait_on_buffer(bh); in __bread_slow()
1280 if (buffer_uptodate(bh)) in __bread_slow()
1281 return bh; in __bread_slow()
1283 brelse(bh); in __bread_slow()
1329 static void bh_lru_install(struct buffer_head *bh) in bh_lru_install() argument
1331 struct buffer_head *evictee = bh; in bh_lru_install()
1352 if (evictee == bh) { in bh_lru_install()
1358 get_bh(bh); in bh_lru_install()
1379 struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]); in lookup_bh_lru() local
1381 if (bh && bh->b_blocknr == block && bh->b_bdev == bdev && in lookup_bh_lru()
1382 bh->b_size == size) { in lookup_bh_lru()
1389 __this_cpu_write(bh_lrus.bhs[0], bh); in lookup_bh_lru()
1391 get_bh(bh); in lookup_bh_lru()
1392 ret = bh; in lookup_bh_lru()
1410 struct buffer_head *bh = lookup_bh_lru(bdev, block, size); in find_get_block_common() local
1412 if (bh == NULL) { in find_get_block_common()
1414 bh = __find_get_block_slow(bdev, block, atomic); in find_get_block_common()
1415 if (bh) in find_get_block_common()
1416 bh_lru_install(bh); in find_get_block_common()
1418 touch_buffer(bh); in find_get_block_common()
1420 return bh; in find_get_block_common()
1456 struct buffer_head *bh; in bdev_getblk() local
1459 bh = __find_get_block_nonatomic(bdev, block, size); in bdev_getblk()
1461 bh = __find_get_block(bdev, block, size); in bdev_getblk()
1464 if (bh) in bdev_getblk()
1465 return bh; in bdev_getblk()
1476 struct buffer_head *bh = bdev_getblk(bdev, block, size, in __breadahead() local
1479 if (likely(bh)) { in __breadahead()
1480 bh_readahead(bh, REQ_RAHEAD); in __breadahead()
1481 brelse(bh); in __breadahead()
1511 struct buffer_head *bh; in __bread_gfp() local
1521 bh = bdev_getblk(bdev, block, size, gfp); in __bread_gfp()
1523 if (likely(bh) && !buffer_uptodate(bh)) in __bread_gfp()
1524 bh = __bread_slow(bh); in __bread_gfp()
1525 return bh; in __bread_gfp()
1584 void folio_set_bh(struct buffer_head *bh, struct folio *folio, in folio_set_bh() argument
1587 bh->b_folio = folio; in folio_set_bh()
1593 bh->b_data = (char *)(0 + offset); in folio_set_bh()
1595 bh->b_data = folio_address(folio) + offset; in folio_set_bh()
1608 static void discard_buffer(struct buffer_head * bh) in discard_buffer() argument
1612 lock_buffer(bh); in discard_buffer()
1613 clear_buffer_dirty(bh); in discard_buffer()
1614 bh->b_bdev = NULL; in discard_buffer()
1615 b_state = READ_ONCE(bh->b_state); in discard_buffer()
1617 } while (!try_cmpxchg_relaxed(&bh->b_state, &b_state, in discard_buffer()
1619 unlock_buffer(bh); in discard_buffer()
1639 struct buffer_head *head, *bh, *next; in block_invalidate_folio() local
1654 bh = head; in block_invalidate_folio()
1656 size_t next_off = curr_off + bh->b_size; in block_invalidate_folio()
1657 next = bh->b_this_page; in block_invalidate_folio()
1669 discard_buffer(bh); in block_invalidate_folio()
1671 bh = next; in block_invalidate_folio()
1672 } while (bh != head); in block_invalidate_folio()
1694 struct buffer_head *bh, *head, *tail; in create_empty_buffers() local
1698 bh = head; in create_empty_buffers()
1700 bh->b_state |= b_state; in create_empty_buffers()
1701 tail = bh; in create_empty_buffers()
1702 bh = bh->b_this_page; in create_empty_buffers()
1703 } while (bh); in create_empty_buffers()
1708 bh = head; in create_empty_buffers()
1711 set_buffer_dirty(bh); in create_empty_buffers()
1713 set_buffer_uptodate(bh); in create_empty_buffers()
1714 bh = bh->b_this_page; in create_empty_buffers()
1715 } while (bh != head); in create_empty_buffers()
1752 struct buffer_head *bh; in clean_bdev_aliases() local
1774 bh = head; in clean_bdev_aliases()
1776 if (!buffer_mapped(bh) || (bh->b_blocknr < block)) in clean_bdev_aliases()
1778 if (bh->b_blocknr >= block + len) in clean_bdev_aliases()
1780 clear_buffer_dirty(bh); in clean_bdev_aliases()
1781 wait_on_buffer(bh); in clean_bdev_aliases()
1782 clear_buffer_req(bh); in clean_bdev_aliases()
1784 bh = bh->b_this_page; in clean_bdev_aliases()
1785 } while (bh != head); in clean_bdev_aliases()
1802 struct buffer_head *bh; in folio_create_buffers() local
1806 bh = folio_buffers(folio); in folio_create_buffers()
1807 if (!bh) in folio_create_buffers()
1808 bh = create_empty_buffers(folio, in folio_create_buffers()
1810 return bh; in folio_create_buffers()
1848 struct buffer_head *bh, *head; in __block_write_full_folio() local
1866 bh = head; in __block_write_full_folio()
1867 blocksize = bh->b_size; in __block_write_full_folio()
1886 clear_buffer_dirty(bh); in __block_write_full_folio()
1887 set_buffer_uptodate(bh); in __block_write_full_folio()
1888 } else if ((!buffer_mapped(bh) || buffer_delay(bh)) && in __block_write_full_folio()
1889 buffer_dirty(bh)) { in __block_write_full_folio()
1890 WARN_ON(bh->b_size != blocksize); in __block_write_full_folio()
1891 err = get_block(inode, block, bh, 1); in __block_write_full_folio()
1894 clear_buffer_delay(bh); in __block_write_full_folio()
1895 if (buffer_new(bh)) { in __block_write_full_folio()
1897 clear_buffer_new(bh); in __block_write_full_folio()
1898 clean_bdev_bh_alias(bh); in __block_write_full_folio()
1901 bh = bh->b_this_page; in __block_write_full_folio()
1903 } while (bh != head); in __block_write_full_folio()
1906 if (!buffer_mapped(bh)) in __block_write_full_folio()
1916 lock_buffer(bh); in __block_write_full_folio()
1917 } else if (!trylock_buffer(bh)) { in __block_write_full_folio()
1921 if (test_clear_buffer_dirty(bh)) { in __block_write_full_folio()
1922 mark_buffer_async_write_endio(bh, in __block_write_full_folio()
1925 unlock_buffer(bh); in __block_write_full_folio()
1927 } while ((bh = bh->b_this_page) != head); in __block_write_full_folio()
1937 struct buffer_head *next = bh->b_this_page; in __block_write_full_folio()
1938 if (buffer_async_write(bh)) { in __block_write_full_folio()
1939 submit_bh_wbc(REQ_OP_WRITE | write_flags, bh, in __block_write_full_folio()
1943 bh = next; in __block_write_full_folio()
1944 } while (bh != head); in __block_write_full_folio()
1971 bh = head; in __block_write_full_folio()
1974 if (buffer_mapped(bh) && buffer_dirty(bh) && in __block_write_full_folio()
1975 !buffer_delay(bh)) { in __block_write_full_folio()
1976 lock_buffer(bh); in __block_write_full_folio()
1977 mark_buffer_async_write_endio(bh, in __block_write_full_folio()
1984 clear_buffer_dirty(bh); in __block_write_full_folio()
1986 } while ((bh = bh->b_this_page) != head); in __block_write_full_folio()
1991 struct buffer_head *next = bh->b_this_page; in __block_write_full_folio()
1992 if (buffer_async_write(bh)) { in __block_write_full_folio()
1993 clear_buffer_dirty(bh); in __block_write_full_folio()
1994 submit_bh_wbc(REQ_OP_WRITE | write_flags, bh, in __block_write_full_folio()
1998 bh = next; in __block_write_full_folio()
1999 } while (bh != head); in __block_write_full_folio()
2013 struct buffer_head *head, *bh; in folio_zero_new_buffers() local
2020 bh = head; in folio_zero_new_buffers()
2023 block_end = block_start + bh->b_size; in folio_zero_new_buffers()
2025 if (buffer_new(bh)) { in folio_zero_new_buffers()
2034 set_buffer_uptodate(bh); in folio_zero_new_buffers()
2037 clear_buffer_new(bh); in folio_zero_new_buffers()
2038 mark_buffer_dirty(bh); in folio_zero_new_buffers()
2043 bh = bh->b_this_page; in folio_zero_new_buffers()
2044 } while (bh != head); in folio_zero_new_buffers()
2049 iomap_to_bh(struct inode *inode, sector_t block, struct buffer_head *bh, in iomap_to_bh() argument
2054 bh->b_bdev = iomap->bdev; in iomap_to_bh()
2072 if (!buffer_uptodate(bh) || in iomap_to_bh()
2074 set_buffer_new(bh); in iomap_to_bh()
2077 if (!buffer_uptodate(bh) || in iomap_to_bh()
2079 set_buffer_new(bh); in iomap_to_bh()
2080 set_buffer_uptodate(bh); in iomap_to_bh()
2081 set_buffer_mapped(bh); in iomap_to_bh()
2082 set_buffer_delay(bh); in iomap_to_bh()
2090 set_buffer_new(bh); in iomap_to_bh()
2091 set_buffer_unwritten(bh); in iomap_to_bh()
2104 set_buffer_new(bh); in iomap_to_bh()
2106 bh->b_blocknr = (iomap->addr + offset - iomap->offset) >> in iomap_to_bh()
2108 set_buffer_mapped(bh); in iomap_to_bh()
2126 struct buffer_head *bh, *head, *wait[2], **wait_bh=wait; in __block_write_begin_int() local
2136 for (bh = head, block_start = 0; bh != head || !block_start; in __block_write_begin_int()
2137 block++, block_start=block_end, bh = bh->b_this_page) { in __block_write_begin_int()
2141 if (!buffer_uptodate(bh)) in __block_write_begin_int()
2142 set_buffer_uptodate(bh); in __block_write_begin_int()
2146 if (buffer_new(bh)) in __block_write_begin_int()
2147 clear_buffer_new(bh); in __block_write_begin_int()
2148 if (!buffer_mapped(bh)) { in __block_write_begin_int()
2149 WARN_ON(bh->b_size != blocksize); in __block_write_begin_int()
2151 err = get_block(inode, block, bh, 1); in __block_write_begin_int()
2153 err = iomap_to_bh(inode, block, bh, iomap); in __block_write_begin_int()
2157 if (buffer_new(bh)) { in __block_write_begin_int()
2158 clean_bdev_bh_alias(bh); in __block_write_begin_int()
2160 clear_buffer_new(bh); in __block_write_begin_int()
2161 set_buffer_uptodate(bh); in __block_write_begin_int()
2162 mark_buffer_dirty(bh); in __block_write_begin_int()
2173 if (!buffer_uptodate(bh)) in __block_write_begin_int()
2174 set_buffer_uptodate(bh); in __block_write_begin_int()
2177 if (!buffer_uptodate(bh) && !buffer_delay(bh) && in __block_write_begin_int()
2178 !buffer_unwritten(bh) && in __block_write_begin_int()
2180 bh_read_nowait(bh, 0); in __block_write_begin_int()
2181 *wait_bh++=bh; in __block_write_begin_int()
2209 struct buffer_head *bh, *head; in block_commit_write() local
2211 bh = head = folio_buffers(folio); in block_commit_write()
2212 if (!bh) in block_commit_write()
2214 blocksize = bh->b_size; in block_commit_write()
2220 if (!buffer_uptodate(bh)) in block_commit_write()
2223 set_buffer_uptodate(bh); in block_commit_write()
2224 mark_buffer_dirty(bh); in block_commit_write()
2226 if (buffer_new(bh)) in block_commit_write()
2227 clear_buffer_new(bh); in block_commit_write()
2230 bh = bh->b_this_page; in block_commit_write()
2231 } while (bh != head); in block_commit_write()
2357 struct buffer_head *bh, *head; in block_is_partially_uptodate() local
2369 bh = head; in block_is_partially_uptodate()
2374 if (!buffer_uptodate(bh)) { in block_is_partially_uptodate()
2382 bh = bh->b_this_page; in block_is_partially_uptodate()
2383 } while (bh != head); in block_is_partially_uptodate()
2400 struct buffer_head *bh, *head, *prev = NULL; in block_read_full_folio() local
2415 bh = head; in block_read_full_folio()
2418 if (buffer_uptodate(bh)) in block_read_full_folio()
2421 if (!buffer_mapped(bh)) { in block_read_full_folio()
2426 WARN_ON(bh->b_size != blocksize); in block_read_full_folio()
2427 err = get_block(inode, iblock, bh, 0); in block_read_full_folio()
2431 if (!buffer_mapped(bh)) { in block_read_full_folio()
2432 folio_zero_range(folio, bh_offset(bh), in block_read_full_folio()
2435 set_buffer_uptodate(bh); in block_read_full_folio()
2442 if (buffer_uptodate(bh)) in block_read_full_folio()
2446 lock_buffer(bh); in block_read_full_folio()
2447 if (buffer_uptodate(bh)) { in block_read_full_folio()
2448 unlock_buffer(bh); in block_read_full_folio()
2452 mark_buffer_async_read(bh); in block_read_full_folio()
2455 prev = bh; in block_read_full_folio()
2456 } while (iblock++, (bh = bh->b_this_page) != head); in block_read_full_folio()
2670 struct buffer_head *bh; in block_truncate_page() local
2687 bh = folio_buffers(folio); in block_truncate_page()
2688 if (!bh) in block_truncate_page()
2689 bh = create_empty_buffers(folio, blocksize, 0); in block_truncate_page()
2695 bh = bh->b_this_page; in block_truncate_page()
2700 if (!buffer_mapped(bh)) { in block_truncate_page()
2701 WARN_ON(bh->b_size != blocksize); in block_truncate_page()
2702 err = get_block(inode, iblock, bh, 0); in block_truncate_page()
2706 if (!buffer_mapped(bh)) in block_truncate_page()
2712 set_buffer_uptodate(bh); in block_truncate_page()
2714 if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) { in block_truncate_page()
2715 err = bh_read(bh, 0); in block_truncate_page()
2722 mark_buffer_dirty(bh); in block_truncate_page()
2778 struct buffer_head *bh = bio->bi_private; in end_bio_bh_io_sync() local
2781 set_bit(BH_Quiet, &bh->b_state); in end_bio_bh_io_sync()
2783 bh->b_end_io(bh, !bio->bi_status); in end_bio_bh_io_sync()
2787 static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, in submit_bh_wbc() argument
2794 BUG_ON(!buffer_locked(bh)); in submit_bh_wbc()
2795 BUG_ON(!buffer_mapped(bh)); in submit_bh_wbc()
2796 BUG_ON(!bh->b_end_io); in submit_bh_wbc()
2797 BUG_ON(buffer_delay(bh)); in submit_bh_wbc()
2798 BUG_ON(buffer_unwritten(bh)); in submit_bh_wbc()
2803 if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE)) in submit_bh_wbc()
2804 clear_buffer_write_io_error(bh); in submit_bh_wbc()
2806 if (buffer_meta(bh)) in submit_bh_wbc()
2808 if (buffer_prio(bh)) in submit_bh_wbc()
2811 bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO); in submit_bh_wbc()
2813 fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO); in submit_bh_wbc()
2815 bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); in submit_bh_wbc()
2818 bio_add_folio_nofail(bio, bh->b_folio, bh->b_size, bh_offset(bh)); in submit_bh_wbc()
2821 bio->bi_private = bh; in submit_bh_wbc()
2828 wbc_account_cgroup_owner(wbc, bh->b_folio, bh->b_size); in submit_bh_wbc()
2834 void submit_bh(blk_opf_t opf, struct buffer_head *bh) in submit_bh() argument
2836 submit_bh_wbc(opf, bh, WRITE_LIFE_NOT_SET, NULL); in submit_bh()
2840 void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) in write_dirty_buffer() argument
2842 lock_buffer(bh); in write_dirty_buffer()
2843 if (!test_clear_buffer_dirty(bh)) { in write_dirty_buffer()
2844 unlock_buffer(bh); in write_dirty_buffer()
2847 bh->b_end_io = end_buffer_write_sync; in write_dirty_buffer()
2848 get_bh(bh); in write_dirty_buffer()
2849 submit_bh(REQ_OP_WRITE | op_flags, bh); in write_dirty_buffer()
2858 int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) in __sync_dirty_buffer() argument
2860 WARN_ON(atomic_read(&bh->b_count) < 1); in __sync_dirty_buffer()
2861 lock_buffer(bh); in __sync_dirty_buffer()
2862 if (test_clear_buffer_dirty(bh)) { in __sync_dirty_buffer()
2867 if (!buffer_mapped(bh)) { in __sync_dirty_buffer()
2868 unlock_buffer(bh); in __sync_dirty_buffer()
2872 get_bh(bh); in __sync_dirty_buffer()
2873 bh->b_end_io = end_buffer_write_sync; in __sync_dirty_buffer()
2874 submit_bh(REQ_OP_WRITE | op_flags, bh); in __sync_dirty_buffer()
2875 wait_on_buffer(bh); in __sync_dirty_buffer()
2876 if (!buffer_uptodate(bh)) in __sync_dirty_buffer()
2879 unlock_buffer(bh); in __sync_dirty_buffer()
2885 int sync_dirty_buffer(struct buffer_head *bh) in sync_dirty_buffer() argument
2887 return __sync_dirty_buffer(bh, REQ_SYNC); in sync_dirty_buffer()
2891 static inline int buffer_busy(struct buffer_head *bh) in buffer_busy() argument
2893 return atomic_read(&bh->b_count) | in buffer_busy()
2894 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock))); in buffer_busy()
2901 struct buffer_head *bh; in drop_buffers() local
2903 bh = head; in drop_buffers()
2905 if (buffer_busy(bh)) in drop_buffers()
2907 bh = bh->b_this_page; in drop_buffers()
2908 } while (bh != head); in drop_buffers()
2911 struct buffer_head *next = bh->b_this_page; in drop_buffers()
2913 if (bh->b_assoc_map) in drop_buffers()
2914 __remove_assoc_queue(bh); in drop_buffers()
2915 bh = next; in drop_buffers()
2916 } while (bh != head); in drop_buffers()
2985 struct buffer_head *bh = buffers_to_free; in try_to_free_buffers() local
2988 struct buffer_head *next = bh->b_this_page; in try_to_free_buffers()
2989 free_buffer_head(bh); in try_to_free_buffers()
2990 bh = next; in try_to_free_buffers()
2991 } while (bh != buffers_to_free); in try_to_free_buffers()
3045 void free_buffer_head(struct buffer_head *bh) in free_buffer_head() argument
3047 BUG_ON(!list_empty(&bh->b_assoc_buffers)); in free_buffer_head()
3048 kmem_cache_free(bh_cachep, bh); in free_buffer_head()
3077 int bh_uptodate_or_lock(struct buffer_head *bh) in bh_uptodate_or_lock() argument
3079 if (!buffer_uptodate(bh)) { in bh_uptodate_or_lock()
3080 lock_buffer(bh); in bh_uptodate_or_lock()
3081 if (!buffer_uptodate(bh)) in bh_uptodate_or_lock()
3083 unlock_buffer(bh); in bh_uptodate_or_lock()
3097 int __bh_read(struct buffer_head *bh, blk_opf_t op_flags, bool wait) in __bh_read() argument
3101 BUG_ON(!buffer_locked(bh)); in __bh_read()
3103 get_bh(bh); in __bh_read()
3104 bh->b_end_io = end_buffer_read_sync; in __bh_read()
3105 submit_bh(REQ_OP_READ | op_flags, bh); in __bh_read()
3107 wait_on_buffer(bh); in __bh_read()
3108 if (!buffer_uptodate(bh)) in __bh_read()
3131 struct buffer_head *bh = bhs[i]; in __bh_read_batch() local
3133 if (buffer_uptodate(bh)) in __bh_read_batch()
3137 lock_buffer(bh); in __bh_read_batch()
3139 if (!trylock_buffer(bh)) in __bh_read_batch()
3142 if (buffer_uptodate(bh)) { in __bh_read_batch()
3143 unlock_buffer(bh); in __bh_read_batch()
3147 bh->b_end_io = end_buffer_read_sync; in __bh_read_batch()
3148 get_bh(bh); in __bh_read_batch()
3149 submit_bh(REQ_OP_READ | op_flags, bh); in __bh_read_batch()