719 	struct ceph_writeback_ctl ceph_wbc;  in write_folio_nounlock()  local
738 	oldest = get_oldest_context(inode, &ceph_wbc, snapc);  in write_folio_nounlock()
751 	if (page_off >= ceph_wbc.i_size) {  in write_folio_nounlock()
753 		      ceph_vinop(inode), folio->index, ceph_wbc.i_size);  in write_folio_nounlock()
758 	if (ceph_wbc.i_size < page_off + len)  in write_folio_nounlock()
759 		len = ceph_wbc.i_size - page_off;  in write_folio_nounlock()
773 				    ceph_wbc.truncate_seq,  in write_folio_nounlock()
774 				    ceph_wbc.truncate_size, true);  in write_folio_nounlock()
1001 void ceph_folio_batch_init(struct ceph_writeback_ctl *ceph_wbc)  in ceph_folio_batch_init()  argument
1003 	folio_batch_init(&ceph_wbc->fbatch);  in ceph_folio_batch_init()
1004 	ceph_wbc->processed_in_fbatch = 0;  in ceph_folio_batch_init()
1008 void ceph_folio_batch_reinit(struct ceph_writeback_ctl *ceph_wbc)  in ceph_folio_batch_reinit()  argument
1010 	folio_batch_release(&ceph_wbc->fbatch);  in ceph_folio_batch_reinit()
1011 	ceph_folio_batch_init(ceph_wbc);  in ceph_folio_batch_reinit()
1017 			     struct ceph_writeback_ctl *ceph_wbc)  in ceph_init_writeback_ctl()  argument
1019 	ceph_wbc->snapc = NULL;  in ceph_init_writeback_ctl()
1020 	ceph_wbc->last_snapc = NULL;  in ceph_init_writeback_ctl()
1022 	ceph_wbc->strip_unit_end = 0;  in ceph_init_writeback_ctl()
1023 	ceph_wbc->wsize = ceph_define_write_size(mapping);  in ceph_init_writeback_ctl()
1025 	ceph_wbc->nr_folios = 0;  in ceph_init_writeback_ctl()
1026 	ceph_wbc->max_pages = 0;  in ceph_init_writeback_ctl()
1027 	ceph_wbc->locked_pages = 0;  in ceph_init_writeback_ctl()
1029 	ceph_wbc->done = false;  in ceph_init_writeback_ctl()
1030 	ceph_wbc->should_loop = false;  in ceph_init_writeback_ctl()
1031 	ceph_wbc->range_whole = false;  in ceph_init_writeback_ctl()
1033 	ceph_wbc->start_index = wbc->range_cyclic ? mapping->writeback_index : 0;  in ceph_init_writeback_ctl()
1034 	ceph_wbc->index = ceph_wbc->start_index;  in ceph_init_writeback_ctl()
1035 	ceph_wbc->end = -1;  in ceph_init_writeback_ctl()
1038 		ceph_wbc->tag = PAGECACHE_TAG_TOWRITE;  in ceph_init_writeback_ctl()
1040 		ceph_wbc->tag = PAGECACHE_TAG_DIRTY;  in ceph_init_writeback_ctl()
1043 	ceph_wbc->op_idx = -1;  in ceph_init_writeback_ctl()
1044 	ceph_wbc->num_ops = 0;  in ceph_init_writeback_ctl()
1045 	ceph_wbc->offset = 0;  in ceph_init_writeback_ctl()
1046 	ceph_wbc->len = 0;  in ceph_init_writeback_ctl()
1047 	ceph_wbc->from_pool = false;  in ceph_init_writeback_ctl()
1049 	ceph_folio_batch_init(ceph_wbc);  in ceph_init_writeback_ctl()
1051 	ceph_wbc->pages = NULL;  in ceph_init_writeback_ctl()
1052 	ceph_wbc->data_pages = NULL;  in ceph_init_writeback_ctl()
1058 				struct ceph_writeback_ctl *ceph_wbc)  in ceph_define_writeback_range()  argument
1065 	ceph_wbc->snapc = get_oldest_context(inode, ceph_wbc, NULL);  in ceph_define_writeback_range()
1066 	if (!ceph_wbc->snapc) {  in ceph_define_writeback_range()
1074 	      ceph_wbc->snapc, ceph_wbc->snapc->seq,  in ceph_define_writeback_range()
1075 	      ceph_wbc->snapc->num_snaps);  in ceph_define_writeback_range()
1077 	ceph_wbc->should_loop = false;  in ceph_define_writeback_range()
1079 	if (ceph_wbc->head_snapc && ceph_wbc->snapc != ceph_wbc->last_snapc) {  in ceph_define_writeback_range()
1082 			ceph_wbc->index = ceph_wbc->start_index;  in ceph_define_writeback_range()
1083 			ceph_wbc->end = -1;  in ceph_define_writeback_range()
1084 			if (ceph_wbc->index > 0)  in ceph_define_writeback_range()
1085 				ceph_wbc->should_loop = true;  in ceph_define_writeback_range()
1086 			doutc(cl, " cyclic, start at %lu\n", ceph_wbc->index);  in ceph_define_writeback_range()
1088 			ceph_wbc->index = wbc->range_start >> PAGE_SHIFT;  in ceph_define_writeback_range()
1089 			ceph_wbc->end = wbc->range_end >> PAGE_SHIFT;  in ceph_define_writeback_range()
1091 				ceph_wbc->range_whole = true;  in ceph_define_writeback_range()
1093 				ceph_wbc->index, ceph_wbc->end);  in ceph_define_writeback_range()
1095 	} else if (!ceph_wbc->head_snapc) {  in ceph_define_writeback_range()
1100 		if (ceph_wbc->index > 0)  in ceph_define_writeback_range()
1101 			ceph_wbc->should_loop = true;  in ceph_define_writeback_range()
1105 	ceph_put_snap_context(ceph_wbc->last_snapc);  in ceph_define_writeback_range()
1106 	ceph_wbc->last_snapc = ceph_wbc->snapc;  in ceph_define_writeback_range()
1112 bool has_writeback_done(struct ceph_writeback_ctl *ceph_wbc)  in has_writeback_done()  argument
1114 	return ceph_wbc->done && ceph_wbc->index > ceph_wbc->end;  in has_writeback_done()
1118 bool can_next_page_be_processed(struct ceph_writeback_ctl *ceph_wbc,  in can_next_page_be_processed()  argument
1121 	return index < ceph_wbc->nr_folios &&  in can_next_page_be_processed()
1122 		ceph_wbc->locked_pages < ceph_wbc->max_pages;  in can_next_page_be_processed()
1128 				 struct ceph_writeback_ctl *ceph_wbc,  in ceph_check_page_before_write()  argument
1144 	if (pgsnapc != ceph_wbc->snapc) {  in ceph_check_page_before_write()
1147 		      ceph_wbc->snapc, ceph_wbc->snapc->seq);  in ceph_check_page_before_write()
1149 		if (!ceph_wbc->should_loop && !ceph_wbc->head_snapc &&  in ceph_check_page_before_write()
1151 			ceph_wbc->should_loop = true;  in ceph_check_page_before_write()
1156 	if (folio_pos(folio) >= ceph_wbc->i_size) {  in ceph_check_page_before_write()
1158 		      folio->index, ceph_wbc->i_size);  in ceph_check_page_before_write()
1160 		if ((ceph_wbc->size_stable ||  in ceph_check_page_before_write()
1168 	if (ceph_wbc->strip_unit_end &&  in ceph_check_page_before_write()
1169 	    (folio->index > ceph_wbc->strip_unit_end)) {  in ceph_check_page_before_write()
1178 void __ceph_allocate_page_array(struct ceph_writeback_ctl *ceph_wbc,  in __ceph_allocate_page_array()  argument
1181 	ceph_wbc->pages = kmalloc_array(max_pages,  in __ceph_allocate_page_array()
1182 					sizeof(*ceph_wbc->pages),  in __ceph_allocate_page_array()
1184 	if (!ceph_wbc->pages) {  in __ceph_allocate_page_array()
1185 		ceph_wbc->from_pool = true;  in __ceph_allocate_page_array()
1186 		ceph_wbc->pages = mempool_alloc(ceph_wb_pagevec_pool, GFP_NOFS);  in __ceph_allocate_page_array()
1187 		BUG_ON(!ceph_wbc->pages);  in __ceph_allocate_page_array()
1193 			      struct ceph_writeback_ctl *ceph_wbc,  in ceph_allocate_page_array()  argument
1203 	ceph_wbc->offset = (u64)folio_pos(folio);  in ceph_allocate_page_array()
1205 					ceph_wbc->offset, ceph_wbc->wsize,  in ceph_allocate_page_array()
1208 	ceph_wbc->num_ops = 1;  in ceph_allocate_page_array()
1209 	ceph_wbc->strip_unit_end = folio->index + ((xlen - 1) >> PAGE_SHIFT);  in ceph_allocate_page_array()
1211 	BUG_ON(ceph_wbc->pages);  in ceph_allocate_page_array()
1212 	ceph_wbc->max_pages = calc_pages_for(0, (u64)xlen);  in ceph_allocate_page_array()
1213 	__ceph_allocate_page_array(ceph_wbc, ceph_wbc->max_pages);  in ceph_allocate_page_array()
1215 	ceph_wbc->len = 0;  in ceph_allocate_page_array()
1219 bool is_folio_index_contiguous(const struct ceph_writeback_ctl *ceph_wbc,  in is_folio_index_contiguous()  argument
1222 	return folio->index == (ceph_wbc->offset + ceph_wbc->len) >> PAGE_SHIFT;  in is_folio_index_contiguous()
1226 bool is_num_ops_too_big(struct ceph_writeback_ctl *ceph_wbc)  in is_num_ops_too_big()  argument
1228 	return ceph_wbc->num_ops >=  in is_num_ops_too_big()
1229 		(ceph_wbc->from_pool ?  CEPH_OSD_SLAB_OPS : CEPH_OSD_MAX_OPS);  in is_num_ops_too_big()
1241 		struct ceph_writeback_ctl *ceph_wbc, struct folio *folio)  in move_dirty_folio_in_page_array()  argument
1246 	struct page **pages = ceph_wbc->pages;  in move_dirty_folio_in_page_array()
1247 	unsigned int index = ceph_wbc->locked_pages;  in move_dirty_folio_in_page_array()
1248 	gfp_t gfp_flags = ceph_wbc->locked_pages ? GFP_NOWAIT : GFP_NOFS;  in move_dirty_folio_in_page_array()
1262 			BUG_ON(ceph_wbc->locked_pages == 0);  in move_dirty_folio_in_page_array()
1271 	ceph_wbc->locked_pages++;  in move_dirty_folio_in_page_array()
1279 			     struct ceph_writeback_ctl *ceph_wbc)  in ceph_process_folio_batch()  argument
1288 	for (i = 0; can_next_page_be_processed(ceph_wbc, i); i++) {  in ceph_process_folio_batch()
1289 		folio = ceph_wbc->fbatch.folios[i];  in ceph_process_folio_batch()
1308 		if (ceph_wbc->locked_pages == 0)  in ceph_process_folio_batch()
1314 						  ceph_wbc, folio);  in ceph_process_folio_batch()
1318 			ceph_wbc->fbatch.folios[i] = NULL;  in ceph_process_folio_batch()
1323 			ceph_wbc->fbatch.folios[i] = NULL;  in ceph_process_folio_batch()
1330 			ceph_wbc->fbatch.folios[i] = NULL;  in ceph_process_folio_batch()
1340 		if (ceph_wbc->locked_pages == 0) {  in ceph_process_folio_batch()
1341 			ceph_allocate_page_array(mapping, ceph_wbc, folio);  in ceph_process_folio_batch()
1342 		} else if (!is_folio_index_contiguous(ceph_wbc, folio)) {  in ceph_process_folio_batch()
1343 			if (is_num_ops_too_big(ceph_wbc)) {  in ceph_process_folio_batch()
1349 			ceph_wbc->num_ops++;  in ceph_process_folio_batch()
1350 			ceph_wbc->offset = (u64)folio_pos(folio);  in ceph_process_folio_batch()
1351 			ceph_wbc->len = 0;  in ceph_process_folio_batch()
1360 		rc = move_dirty_folio_in_page_array(mapping, wbc, ceph_wbc,  in ceph_process_folio_batch()
1368 		ceph_wbc->fbatch.folios[i] = NULL;  in ceph_process_folio_batch()
1369 		ceph_wbc->len += folio_size(folio);  in ceph_process_folio_batch()
1372 	ceph_wbc->processed_in_fbatch = i;  in ceph_process_folio_batch()
1400 			struct ceph_writeback_ctl *ceph_wbc)  in ceph_submit_write()  argument
1415 	offset = ceph_fscrypt_page_offset(ceph_wbc->pages[0]);  in ceph_submit_write()
1416 	len = ceph_wbc->wsize;  in ceph_submit_write()
1420 				    offset, &len, 0, ceph_wbc->num_ops,  in ceph_submit_write()
1422 				    ceph_wbc->snapc, ceph_wbc->truncate_seq,  in ceph_submit_write()
1423 				    ceph_wbc->truncate_size, false);  in ceph_submit_write()
1428 					    min(ceph_wbc->num_ops,  in ceph_submit_write()
1432 					    ceph_wbc->snapc,  in ceph_submit_write()
1433 					    ceph_wbc->truncate_seq,  in ceph_submit_write()
1434 					    ceph_wbc->truncate_size,  in ceph_submit_write()
1439 	page = ceph_wbc->pages[ceph_wbc->locked_pages - 1];  in ceph_submit_write()
1443 		for (i = 0; i < folio_batch_count(&ceph_wbc->fbatch); i++) {  in ceph_submit_write()
1444 			struct folio *folio = ceph_wbc->fbatch.folios[i];  in ceph_submit_write()
1454 		for (i = 0; i < ceph_wbc->locked_pages; i++) {  in ceph_submit_write()
1455 			page = ceph_fscrypt_pagecache_page(ceph_wbc->pages[i]);  in ceph_submit_write()
1473 	ceph_wbc->data_pages = ceph_wbc->pages;  in ceph_submit_write()
1474 	ceph_wbc->op_idx = 0;  in ceph_submit_write()
1475 	for (i = 0; i < ceph_wbc->locked_pages; i++) {  in ceph_submit_write()
1478 		page = ceph_fscrypt_pagecache_page(ceph_wbc->pages[i]);  in ceph_submit_write()
1487 			if (ceph_wbc->op_idx + 1 == req->r_num_ops)  in ceph_submit_write()
1494 			osd_req_op_extent_dup_last(req, ceph_wbc->op_idx,  in ceph_submit_write()
1499 			osd_req_op_extent_osd_data_pages(req, ceph_wbc->op_idx,  in ceph_submit_write()
1500 							 ceph_wbc->data_pages,  in ceph_submit_write()
1502 							 ceph_wbc->from_pool,  in ceph_submit_write()
1504 			osd_req_op_extent_update(req, ceph_wbc->op_idx, len);  in ceph_submit_write()
1508 			ceph_wbc->data_pages = ceph_wbc->pages + i;  in ceph_submit_write()
1509 			ceph_wbc->op_idx++;  in ceph_submit_write()
1522 	if (ceph_wbc->size_stable) {  in ceph_submit_write()
1523 		len = min(len, ceph_wbc->i_size - offset);  in ceph_submit_write()
1524 	} else if (i == ceph_wbc->locked_pages) {  in ceph_submit_write()
1530 						 ceph_wbc->pages[i - 1],  in ceph_submit_write()
1547 	osd_req_op_extent_osd_data_pages(req, ceph_wbc->op_idx,  in ceph_submit_write()
1548 					 ceph_wbc->data_pages, len,  in ceph_submit_write()
1549 					 0, ceph_wbc->from_pool, false);  in ceph_submit_write()
1550 	osd_req_op_extent_update(req, ceph_wbc->op_idx, len);  in ceph_submit_write()
1552 	BUG_ON(ceph_wbc->op_idx + 1 != req->r_num_ops);  in ceph_submit_write()
1554 	ceph_wbc->from_pool = false;  in ceph_submit_write()
1555 	if (i < ceph_wbc->locked_pages) {  in ceph_submit_write()
1556 		BUG_ON(ceph_wbc->num_ops <= req->r_num_ops);  in ceph_submit_write()
1557 		ceph_wbc->num_ops -= req->r_num_ops;  in ceph_submit_write()
1558 		ceph_wbc->locked_pages -= i;  in ceph_submit_write()
1561 		ceph_wbc->data_pages = ceph_wbc->pages;  in ceph_submit_write()
1562 		__ceph_allocate_page_array(ceph_wbc, ceph_wbc->locked_pages);  in ceph_submit_write()
1563 		memcpy(ceph_wbc->pages, ceph_wbc->data_pages + i,  in ceph_submit_write()
1564 			ceph_wbc->locked_pages * sizeof(*ceph_wbc->pages));  in ceph_submit_write()
1565 		memset(ceph_wbc->data_pages + i, 0,  in ceph_submit_write()
1566 			ceph_wbc->locked_pages * sizeof(*ceph_wbc->pages));  in ceph_submit_write()
1568 		BUG_ON(ceph_wbc->num_ops != req->r_num_ops);  in ceph_submit_write()
1570 		ceph_wbc->pages = NULL;  in ceph_submit_write()
1578 	if (ceph_wbc->pages)  in ceph_submit_write()
1587 					     struct ceph_writeback_ctl *ceph_wbc)  in ceph_wait_until_current_writes_complete()  argument
1593 	    ceph_wbc->start_index == 0 && /* all dirty pages were checked */  in ceph_wait_until_current_writes_complete()
1594 	    !ceph_wbc->head_snapc) {  in ceph_wait_until_current_writes_complete()
1595 		ceph_wbc->index = 0;  in ceph_wait_until_current_writes_complete()
1597 		while ((ceph_wbc->index <= ceph_wbc->end) &&  in ceph_wait_until_current_writes_complete()
1599 						     &ceph_wbc->index,  in ceph_wait_until_current_writes_complete()
1602 						     &ceph_wbc->fbatch))) {  in ceph_wait_until_current_writes_complete()
1604 				page = &ceph_wbc->fbatch.folios[i]->page;  in ceph_wait_until_current_writes_complete()
1605 				if (page_snap_context(page) != ceph_wbc->snapc)  in ceph_wait_until_current_writes_complete()
1610 			folio_batch_release(&ceph_wbc->fbatch);  in ceph_wait_until_current_writes_complete()
1625 	struct ceph_writeback_ctl ceph_wbc;  in ceph_writepages_start()  local
1640 	ceph_init_writeback_ctl(mapping, wbc, &ceph_wbc);  in ceph_writepages_start()
1648 	rc = ceph_define_writeback_range(mapping, wbc, &ceph_wbc);  in ceph_writepages_start()
1657 		tag_pages_for_writeback(mapping, ceph_wbc.index, ceph_wbc.end);  in ceph_writepages_start()
1659 	while (!has_writeback_done(&ceph_wbc)) {  in ceph_writepages_start()
1660 		ceph_wbc.locked_pages = 0;  in ceph_writepages_start()
1661 		ceph_wbc.max_pages = ceph_wbc.wsize >> PAGE_SHIFT;  in ceph_writepages_start()
1664 		ceph_folio_batch_reinit(&ceph_wbc);  in ceph_writepages_start()
1666 		ceph_wbc.nr_folios = filemap_get_folios_tag(mapping,  in ceph_writepages_start()
1667 							    &ceph_wbc.index,  in ceph_writepages_start()
1668 							    ceph_wbc.end,  in ceph_writepages_start()
1669 							    ceph_wbc.tag,  in ceph_writepages_start()
1670 							    &ceph_wbc.fbatch);  in ceph_writepages_start()
1672 			ceph_wbc.tag, ceph_wbc.nr_folios);  in ceph_writepages_start()
1674 		if (!ceph_wbc.nr_folios && !ceph_wbc.locked_pages)  in ceph_writepages_start()
1678 		rc = ceph_process_folio_batch(mapping, wbc, &ceph_wbc);  in ceph_writepages_start()
1683 		if (!ceph_wbc.locked_pages)  in ceph_writepages_start()
1686 		if (ceph_wbc.processed_in_fbatch) {  in ceph_writepages_start()
1687 			ceph_shift_unused_folios_left(&ceph_wbc.fbatch);  in ceph_writepages_start()
1689 			if (folio_batch_count(&ceph_wbc.fbatch) == 0 &&  in ceph_writepages_start()
1690 			    ceph_wbc.locked_pages < ceph_wbc.max_pages) {  in ceph_writepages_start()
1696 		rc = ceph_submit_write(mapping, wbc, &ceph_wbc);  in ceph_writepages_start()
1700 		ceph_wbc.locked_pages = 0;  in ceph_writepages_start()
1701 		ceph_wbc.strip_unit_end = 0;  in ceph_writepages_start()
1703 		if (folio_batch_count(&ceph_wbc.fbatch) > 0) {  in ceph_writepages_start()
1704 			ceph_wbc.nr_folios =  in ceph_writepages_start()
1705 				folio_batch_count(&ceph_wbc.fbatch);  in ceph_writepages_start()
1716 			ceph_wbc.done = true;  in ceph_writepages_start()
1720 		      (int)ceph_wbc.fbatch.nr,  in ceph_writepages_start()
1721 		      ceph_wbc.fbatch.nr ? ceph_wbc.fbatch.folios[0] : NULL);  in ceph_writepages_start()
1722 		folio_batch_release(&ceph_wbc.fbatch);  in ceph_writepages_start()
1725 	if (ceph_wbc.should_loop && !ceph_wbc.done) {  in ceph_writepages_start()
1729 		ceph_wbc.end = ceph_wbc.start_index - 1;  in ceph_writepages_start()
1733 		ceph_wait_until_current_writes_complete(mapping, wbc, &ceph_wbc);  in ceph_writepages_start()
1735 		ceph_wbc.start_index = 0;  in ceph_writepages_start()
1736 		ceph_wbc.index = 0;  in ceph_writepages_start()
1740 	if (wbc->range_cyclic || (ceph_wbc.range_whole && wbc->nr_to_write > 0))  in ceph_writepages_start()
1741 		mapping->writeback_index = ceph_wbc.index;  in ceph_writepages_start()
1747 	ceph_put_snap_context(ceph_wbc.last_snapc);  in ceph_writepages_start()