132 #define GDTC_INIT(__wb)		.wb = (__wb),				\
138 #define MDTC_INIT(__wb, __gdtc)	.wb = (__wb),				\
158 static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb)  in wb_memcg_completions()  argument
160 	return &wb->memcg_completions;  in wb_memcg_completions()
163 static void wb_min_max_ratio(struct bdi_writeback *wb,  in wb_min_max_ratio()  argument
166 	unsigned long this_bw = READ_ONCE(wb->avg_write_bandwidth);  in wb_min_max_ratio()
167 	unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth);  in wb_min_max_ratio()
168 	unsigned long long min = wb->bdi->min_ratio;  in wb_min_max_ratio()
169 	unsigned long long max = wb->bdi->max_ratio;  in wb_min_max_ratio()
192 #define GDTC_INIT(__wb)		.wb = (__wb),                           \
212 static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb)  in wb_memcg_completions()  argument
217 static void wb_min_max_ratio(struct bdi_writeback *wb,  in wb_min_max_ratio()  argument
220 	*minp = wb->bdi->min_ratio;  in wb_min_max_ratio()
221 	*maxp = wb->bdi->max_ratio;  in wb_min_max_ratio()
580 static inline void __wb_writeout_add(struct bdi_writeback *wb, long nr)  in __wb_writeout_add()  argument
584 	wb_stat_mod(wb, WB_WRITTEN, nr);  in __wb_writeout_add()
585 	wb_domain_writeout_add(&global_wb_domain, &wb->completions,  in __wb_writeout_add()
586 			       wb->bdi->max_prop_frac, nr);  in __wb_writeout_add()
588 	cgdom = mem_cgroup_wb_domain(wb);  in __wb_writeout_add()
590 		wb_domain_writeout_add(cgdom, wb_memcg_completions(wb),  in __wb_writeout_add()
591 				       wb->bdi->max_prop_frac, nr);  in __wb_writeout_add()
594 void wb_writeout_inc(struct bdi_writeback *wb)  in wb_writeout_inc()  argument
599 	__wb_writeout_add(wb, 1);  in wb_writeout_inc()
867 		mem_cgroup_wb_stats(dtc->wb, &filepages, &headroom, &dtc->dirty,  in domain_dirty_avail()
903 	struct bdi_writeback *wb = dtc->wb;  in __wb_calc_thresh()  local
919 	wb_min_max_ratio(wb, &wb_min_ratio, &wb_max_ratio);  in __wb_calc_thresh()
931 		if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT))  in __wb_calc_thresh()
944 unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh)  in wb_calc_thresh()  argument
946 	struct dirty_throttle_control gdtc = { GDTC_INIT(wb) };  in wb_calc_thresh()
952 unsigned long cgwb_calc_thresh(struct bdi_writeback *wb)  in cgwb_calc_thresh()  argument
955 	struct dirty_throttle_control mdtc = { MDTC_INIT(wb, &gdtc) };  in cgwb_calc_thresh()
1072 	struct bdi_writeback *wb = dtc->wb;  in wb_position_ratio()  local
1073 	unsigned long write_bw = READ_ONCE(wb->avg_write_bandwidth);  in wb_position_ratio()
1122 	if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) {  in wb_position_ratio()
1235 static void wb_update_write_bandwidth(struct bdi_writeback *wb,  in wb_update_write_bandwidth()  argument
1240 	unsigned long avg = wb->avg_write_bandwidth;  in wb_update_write_bandwidth()
1241 	unsigned long old = wb->write_bandwidth;  in wb_update_write_bandwidth()
1254 	bw = written - min(written, wb->written_stamp);  in wb_update_write_bandwidth()
1261 	bw += (u64)wb->write_bandwidth * (period - elapsed);  in wb_update_write_bandwidth()
1276 	if (wb_has_dirty_io(wb)) {  in wb_update_write_bandwidth()
1277 		long delta = avg - wb->avg_write_bandwidth;  in wb_update_write_bandwidth()
1279 					&wb->bdi->tot_write_bandwidth) <= 0);  in wb_update_write_bandwidth()
1281 	wb->write_bandwidth = bw;  in wb_update_write_bandwidth()
1282 	WRITE_ONCE(wb->avg_write_bandwidth, avg);  in wb_update_write_bandwidth()
1343 	struct bdi_writeback *wb = dtc->wb;  in wb_update_dirty_ratelimit()  local
1348 	unsigned long write_bw = wb->avg_write_bandwidth;  in wb_update_dirty_ratelimit()
1349 	unsigned long dirty_ratelimit = wb->dirty_ratelimit;  in wb_update_dirty_ratelimit()
1361 	dirty_rate = (dirtied - wb->dirtied_stamp) * HZ / elapsed;  in wb_update_dirty_ratelimit()
1451 	if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) {  in wb_update_dirty_ratelimit()
1457 		x = min3(wb->balanced_dirty_ratelimit,  in wb_update_dirty_ratelimit()
1462 		x = max3(wb->balanced_dirty_ratelimit,  in wb_update_dirty_ratelimit()
1484 	WRITE_ONCE(wb->dirty_ratelimit, max(dirty_ratelimit, 1UL));  in wb_update_dirty_ratelimit()
1485 	wb->balanced_dirty_ratelimit = balanced_dirty_ratelimit;  in wb_update_dirty_ratelimit()
1487 	trace_bdi_dirty_ratelimit(wb, dirty_rate, task_ratelimit);  in wb_update_dirty_ratelimit()
1494 	struct bdi_writeback *wb = gdtc->wb;  in __wb_update_bandwidth()  local
1500 	spin_lock(&wb->list_lock);  in __wb_update_bandwidth()
1508 	elapsed = max(now - wb->bw_time_stamp, 1UL);  in __wb_update_bandwidth()
1509 	dirtied = percpu_counter_read(&wb->stat[WB_DIRTIED]);  in __wb_update_bandwidth()
1510 	written = percpu_counter_read(&wb->stat[WB_WRITTEN]);  in __wb_update_bandwidth()
1525 	wb_update_write_bandwidth(wb, elapsed, written);  in __wb_update_bandwidth()
1527 	wb->dirtied_stamp = dirtied;  in __wb_update_bandwidth()
1528 	wb->written_stamp = written;  in __wb_update_bandwidth()
1529 	WRITE_ONCE(wb->bw_time_stamp, now);  in __wb_update_bandwidth()
1530 	spin_unlock(&wb->list_lock);  in __wb_update_bandwidth()
1533 void wb_update_bandwidth(struct bdi_writeback *wb)  in wb_update_bandwidth()  argument
1535 	struct dirty_throttle_control gdtc = { GDTC_INIT(wb) };  in wb_update_bandwidth()
1543 static void wb_bandwidth_estimate_start(struct bdi_writeback *wb)  in wb_bandwidth_estimate_start()  argument
1546 	unsigned long elapsed = now - READ_ONCE(wb->bw_time_stamp);  in wb_bandwidth_estimate_start()
1549 	    !atomic_read(&wb->writeback_inodes)) {  in wb_bandwidth_estimate_start()
1550 		spin_lock(&wb->list_lock);  in wb_bandwidth_estimate_start()
1551 		wb->dirtied_stamp = wb_stat(wb, WB_DIRTIED);  in wb_bandwidth_estimate_start()
1552 		wb->written_stamp = wb_stat(wb, WB_WRITTEN);  in wb_bandwidth_estimate_start()
1553 		WRITE_ONCE(wb->bw_time_stamp, now);  in wb_bandwidth_estimate_start()
1554 		spin_unlock(&wb->list_lock);  in wb_bandwidth_estimate_start()
1575 static unsigned long wb_max_pause(struct bdi_writeback *wb,  in wb_max_pause()  argument
1578 	unsigned long bw = READ_ONCE(wb->avg_write_bandwidth);  in wb_max_pause()
1594 static long wb_min_pause(struct bdi_writeback *wb,  in wb_min_pause()  argument
1600 	long hi = ilog2(READ_ONCE(wb->avg_write_bandwidth));  in wb_min_pause()
1601 	long lo = ilog2(READ_ONCE(wb->dirty_ratelimit));  in wb_min_pause()
1671 	struct bdi_writeback *wb = dtc->wb;  in wb_dirty_limits()  local
1702 		wb_reclaimable = wb_stat_sum(wb, WB_RECLAIMABLE);  in wb_dirty_limits()
1703 		dtc->wb_dirty = wb_reclaimable + wb_stat_sum(wb, WB_WRITEBACK);  in wb_dirty_limits()
1705 		wb_reclaimable = wb_stat(wb, WB_RECLAIMABLE);  in wb_dirty_limits()
1706 		dtc->wb_dirty = wb_reclaimable + wb_stat(wb, WB_WRITEBACK);  in wb_dirty_limits()
1811 static int balance_dirty_pages(struct bdi_writeback *wb,  in balance_dirty_pages()  argument
1814 	struct dirty_throttle_control gdtc_stor = { GDTC_INIT(wb) };  in balance_dirty_pages()
1815 	struct dirty_throttle_control mdtc_stor = { MDTC_INIT(wb, &gdtc_stor) };  in balance_dirty_pages()
1828 	struct backing_dev_info *bdi = wb->bdi;  in balance_dirty_pages()
1857 		    !writeback_in_progress(wb))  in balance_dirty_pages()
1858 			wb_start_background_writeback(wb);  in balance_dirty_pages()
1881 		if (unlikely(!writeback_in_progress(wb)))  in balance_dirty_pages()
1882 			wb_start_background_writeback(wb);  in balance_dirty_pages()
1884 		mem_cgroup_flush_foreign(wb);  in balance_dirty_pages()
1909 		wb->dirty_exceeded = gdtc->dirty_exceeded ||  in balance_dirty_pages()
1911 		if (time_is_before_jiffies(READ_ONCE(wb->bw_time_stamp) +  in balance_dirty_pages()
1916 		dirty_ratelimit = READ_ONCE(wb->dirty_ratelimit);  in balance_dirty_pages()
1919 		max_pause = wb_max_pause(wb, sdtc->wb_dirty);  in balance_dirty_pages()
1920 		min_pause = wb_min_pause(wb, max_pause,  in balance_dirty_pages()
1941 			trace_balance_dirty_pages(wb,  in balance_dirty_pages()
1966 		trace_balance_dirty_pages(wb,  in balance_dirty_pages()
2052 	struct bdi_writeback *wb = NULL;  in balance_dirty_pages_ratelimited_flags()  local
2061 		wb = wb_get_create_current(bdi, GFP_KERNEL);  in balance_dirty_pages_ratelimited_flags()
2062 	if (!wb)  in balance_dirty_pages_ratelimited_flags()
2063 		wb = &bdi->wb;  in balance_dirty_pages_ratelimited_flags()
2066 	if (wb->dirty_exceeded)  in balance_dirty_pages_ratelimited_flags()
2098 		ret = balance_dirty_pages(wb, current->nr_dirtied, flags);  in balance_dirty_pages_ratelimited_flags()
2100 	wb_put(wb);  in balance_dirty_pages_ratelimited_flags()
2129 	struct bdi_writeback *wb = dtc->wb;  in wb_bg_dirty_limits()  local
2133 		dtc->wb_dirty = wb_stat_sum(wb, WB_RECLAIMABLE);  in wb_bg_dirty_limits()
2135 		dtc->wb_dirty = wb_stat(wb, WB_RECLAIMABLE);  in wb_bg_dirty_limits()
2161 bool wb_over_bg_thresh(struct bdi_writeback *wb)  in wb_over_bg_thresh()  argument
2163 	struct dirty_throttle_control gdtc = { GDTC_INIT(wb) };  in wb_over_bg_thresh()
2164 	struct dirty_throttle_control mdtc = { MDTC_INIT(wb, &gdtc) };  in wb_over_bg_thresh()
2648 	struct bdi_writeback *wb;  in do_writepages()  local
2652 	wb = inode_to_wb_wbc(mapping->host, wbc);  in do_writepages()
2653 	wb_bandwidth_estimate_start(wb);  in do_writepages()
2680 	if (time_is_before_jiffies(READ_ONCE(wb->bw_time_stamp) +  in do_writepages()
2682 		wb_update_bandwidth(wb);  in do_writepages()
2710 		struct bdi_writeback *wb;  in folio_account_dirtied()  local
2714 		wb = inode_to_wb(inode);  in folio_account_dirtied()
2719 		wb_stat_mod(wb, WB_RECLAIMABLE, nr);  in folio_account_dirtied()
2720 		wb_stat_mod(wb, WB_DIRTIED, nr);  in folio_account_dirtied()
2725 		mem_cgroup_track_foreign_dirty(folio, wb);  in folio_account_dirtied()
2733 void folio_account_cleaned(struct folio *folio, struct bdi_writeback *wb)  in folio_account_cleaned()  argument
2739 	wb_stat_mod(wb, WB_RECLAIMABLE, -nr);  in folio_account_cleaned()
2830 		struct bdi_writeback *wb;  in folio_redirty_for_writepage()  local
2833 		wb = unlocked_inode_to_wb_begin(inode, &cookie);  in folio_redirty_for_writepage()
2836 		wb_stat_mod(wb, WB_DIRTIED, -nr);  in folio_redirty_for_writepage()
2921 		struct bdi_writeback *wb;  in __folio_cancel_dirty()  local
2924 		wb = unlocked_inode_to_wb_begin(inode, &cookie);  in __folio_cancel_dirty()
2927 			folio_account_cleaned(folio, wb);  in __folio_cancel_dirty()
2959 		struct bdi_writeback *wb;  in folio_clear_dirty_for_io()  local
2997 		wb = unlocked_inode_to_wb_begin(inode, &cookie);  in folio_clear_dirty_for_io()
3002 			wb_stat_mod(wb, WB_RECLAIMABLE, -nr);  in folio_clear_dirty_for_io()
3012 static void wb_inode_writeback_start(struct bdi_writeback *wb)  in wb_inode_writeback_start()  argument
3014 	atomic_inc(&wb->writeback_inodes);  in wb_inode_writeback_start()
3017 static void wb_inode_writeback_end(struct bdi_writeback *wb)  in wb_inode_writeback_end()  argument
3020 	atomic_dec(&wb->writeback_inodes);  in wb_inode_writeback_end()
3028 	spin_lock_irqsave(&wb->work_lock, flags);  in wb_inode_writeback_end()
3029 	if (test_bit(WB_registered, &wb->state))  in wb_inode_writeback_end()
3030 		queue_delayed_work(bdi_wq, &wb->bw_dwork, BANDWIDTH_INTERVAL);  in wb_inode_writeback_end()
3031 	spin_unlock_irqrestore(&wb->work_lock, flags);  in wb_inode_writeback_end()
3050 			struct bdi_writeback *wb = inode_to_wb(inode);  in __folio_end_writeback()  local
3052 			wb_stat_mod(wb, WB_WRITEBACK, -nr);  in __folio_end_writeback()
3053 			__wb_writeout_add(wb, nr);  in __folio_end_writeback()
3055 				wb_inode_writeback_end(wb);  in __folio_end_writeback()
3098 			struct bdi_writeback *wb = inode_to_wb(inode);  in __folio_start_writeback()  local
3100 			wb_stat_mod(wb, WB_WRITEBACK, nr);  in __folio_start_writeback()
3102 				wb_inode_writeback_start(wb);  in __folio_start_writeback()