28 static inline int folio_is_file_lru(struct folio *folio)  in folio_is_file_lru()  argument
30 	return !folio_test_swapbacked(folio);  in folio_is_file_lru()
66 static __always_inline void __folio_clear_lru_flags(struct folio *folio)  in __folio_clear_lru_flags()  argument
68 	VM_BUG_ON_FOLIO(!folio_test_lru(folio), folio);  in __folio_clear_lru_flags()
70 	__folio_clear_lru(folio);  in __folio_clear_lru_flags()
73 	if (folio_test_active(folio) && folio_test_unevictable(folio))  in __folio_clear_lru_flags()
76 	__folio_clear_active(folio);  in __folio_clear_lru_flags()
77 	__folio_clear_unevictable(folio);  in __folio_clear_lru_flags()
87 static __always_inline enum lru_list folio_lru_list(struct folio *folio)  in folio_lru_list()  argument
91 	VM_BUG_ON_FOLIO(folio_test_active(folio) && folio_test_unevictable(folio), folio);  in folio_lru_list()
93 	if (folio_test_unevictable(folio))  in folio_lru_list()
96 	lru = folio_is_file_lru(folio) ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON;  in folio_lru_list()
97 	if (folio_test_active(folio))  in folio_lru_list()
144 static inline int folio_lru_refs(struct folio *folio)  in folio_lru_refs()  argument
146 	unsigned long flags = READ_ONCE(folio->flags);  in folio_lru_refs()
157 static inline int folio_lru_gen(struct folio *folio)  in folio_lru_gen()  argument
159 	unsigned long flags = READ_ONCE(folio->flags);  in folio_lru_gen()
174 static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio,  in lru_gen_update_size()  argument
177 	int type = folio_is_file_lru(folio);  in lru_gen_update_size()
178 	int zone = folio_zonenum(folio);  in lru_gen_update_size()
179 	int delta = folio_nr_pages(folio);  in lru_gen_update_size()
220 static inline unsigned long lru_gen_folio_seq(struct lruvec *lruvec, struct folio *folio,  in lru_gen_folio_seq()  argument
224 	int type = folio_is_file_lru(folio);  in lru_gen_folio_seq()
239 	if (folio_test_active(folio))  in lru_gen_folio_seq()
240 		gen = MIN_NR_GENS - folio_test_workingset(folio);  in lru_gen_folio_seq()
243 	else if ((!folio_is_file_lru(folio) && !folio_test_swapcache(folio)) ||  in lru_gen_folio_seq()
244 		 (folio_test_reclaim(folio) &&  in lru_gen_folio_seq()
245 		  (folio_test_dirty(folio) || folio_test_writeback(folio))))  in lru_gen_folio_seq()
248 		gen = MAX_NR_GENS - folio_test_workingset(folio);  in lru_gen_folio_seq()
253 static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)  in lru_gen_add_folio()  argument
257 	int gen = folio_lru_gen(folio);  in lru_gen_add_folio()
258 	int type = folio_is_file_lru(folio);  in lru_gen_add_folio()
259 	int zone = folio_zonenum(folio);  in lru_gen_add_folio()
262 	VM_WARN_ON_ONCE_FOLIO(gen != -1, folio);  in lru_gen_add_folio()
264 	if (folio_test_unevictable(folio) || !lrugen->enabled)  in lru_gen_add_folio()
267 	seq = lru_gen_folio_seq(lruvec, folio, reclaiming);  in lru_gen_add_folio()
271 	set_mask_bits(&folio->flags, LRU_GEN_MASK | BIT(PG_active), flags);  in lru_gen_add_folio()
273 	lru_gen_update_size(lruvec, folio, -1, gen);  in lru_gen_add_folio()
276 		list_add_tail(&folio->lru, &lrugen->folios[gen][type][zone]);  in lru_gen_add_folio()
278 		list_add(&folio->lru, &lrugen->folios[gen][type][zone]);  in lru_gen_add_folio()
283 static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)  in lru_gen_del_folio()  argument
286 	int gen = folio_lru_gen(folio);  in lru_gen_del_folio()
291 	VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);  in lru_gen_del_folio()
292 	VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);  in lru_gen_del_folio()
296 	flags = set_mask_bits(&folio->flags, LRU_GEN_MASK, flags);  in lru_gen_del_folio()
299 	lru_gen_update_size(lruvec, folio, gen, -1);  in lru_gen_del_folio()
300 	list_del(&folio->lru);  in lru_gen_del_folio()
305 static inline void folio_migrate_refs(struct folio *new, struct folio *old)  in folio_migrate_refs()
323 static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)  in lru_gen_add_folio()  argument
328 static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)  in lru_gen_del_folio()  argument
333 static inline void folio_migrate_refs(struct folio *new, struct folio *old)  in folio_migrate_refs()
340 void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio)  in lruvec_add_folio()  argument
342 	enum lru_list lru = folio_lru_list(folio);  in lruvec_add_folio()
344 	if (lru_gen_add_folio(lruvec, folio, false))  in lruvec_add_folio()
347 	update_lru_size(lruvec, lru, folio_zonenum(folio),  in lruvec_add_folio()
348 			folio_nr_pages(folio));  in lruvec_add_folio()
350 		list_add(&folio->lru, &lruvec->lists[lru]);  in lruvec_add_folio()
354 void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio)  in lruvec_add_folio_tail()  argument
356 	enum lru_list lru = folio_lru_list(folio);  in lruvec_add_folio_tail()
358 	if (lru_gen_add_folio(lruvec, folio, true))  in lruvec_add_folio_tail()
361 	update_lru_size(lruvec, lru, folio_zonenum(folio),  in lruvec_add_folio_tail()
362 			folio_nr_pages(folio));  in lruvec_add_folio_tail()
364 	list_add_tail(&folio->lru, &lruvec->lists[lru]);  in lruvec_add_folio_tail()
368 void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio)  in lruvec_del_folio()  argument
370 	enum lru_list lru = folio_lru_list(folio);  in lruvec_del_folio()
372 	if (lru_gen_del_folio(lruvec, folio, false))  in lruvec_del_folio()
376 		list_del(&folio->lru);  in lruvec_del_folio()
377 	update_lru_size(lruvec, lru, folio_zonenum(folio),  in lruvec_del_folio()
378 			-folio_nr_pages(folio));  in lruvec_del_folio()