38 static __always_inline void __update_lru_size(struct lruvec *lruvec,  in __update_lru_size()  argument
42 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in __update_lru_size()
44 	lockdep_assert_held(&lruvec->lru_lock);  in __update_lru_size()
47 	__mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages);  in __update_lru_size()
52 static __always_inline void update_lru_size(struct lruvec *lruvec,  in update_lru_size()  argument
56 	__update_lru_size(lruvec, lru, zid, nr_pages);  in update_lru_size()
58 	mem_cgroup_update_lru_size(lruvec, lru, zid, nr_pages);  in update_lru_size()
164 static inline bool lru_gen_is_active(struct lruvec *lruvec, int gen)  in lru_gen_is_active()  argument
166 	unsigned long max_seq = lruvec->lrugen.max_seq;  in lru_gen_is_active()
174 static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio,  in lru_gen_update_size()  argument
181 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in lru_gen_update_size()
196 		if (lru_gen_is_active(lruvec, new_gen))  in lru_gen_update_size()
198 		__update_lru_size(lruvec, lru, zone, delta);  in lru_gen_update_size()
204 		if (lru_gen_is_active(lruvec, old_gen))  in lru_gen_update_size()
206 		__update_lru_size(lruvec, lru, zone, -delta);  in lru_gen_update_size()
211 	if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) {  in lru_gen_update_size()
212 		__update_lru_size(lruvec, lru, zone, -delta);  in lru_gen_update_size()
213 		__update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta);  in lru_gen_update_size()
217 	VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen));  in lru_gen_update_size()
220 static inline unsigned long lru_gen_folio_seq(struct lruvec *lruvec, struct folio *folio,  in lru_gen_folio_seq()  argument
225 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in lru_gen_folio_seq()
253 static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)  in lru_gen_add_folio()  argument
260 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in lru_gen_add_folio()
267 	seq = lru_gen_folio_seq(lruvec, folio, reclaiming);  in lru_gen_add_folio()
273 	lru_gen_update_size(lruvec, folio, -1, gen);  in lru_gen_add_folio()
283 static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)  in lru_gen_del_folio()  argument
295 	flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0;  in lru_gen_del_folio()
299 	lru_gen_update_size(lruvec, folio, gen, -1);  in lru_gen_del_folio()
323 static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)  in lru_gen_add_folio()  argument
328 static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)  in lru_gen_del_folio()  argument
340 void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio)  in lruvec_add_folio()  argument
344 	if (lru_gen_add_folio(lruvec, folio, false))  in lruvec_add_folio()
347 	update_lru_size(lruvec, lru, folio_zonenum(folio),  in lruvec_add_folio()
350 		list_add(&folio->lru, &lruvec->lists[lru]);  in lruvec_add_folio()
354 void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio)  in lruvec_add_folio_tail()  argument
358 	if (lru_gen_add_folio(lruvec, folio, true))  in lruvec_add_folio_tail()
361 	update_lru_size(lruvec, lru, folio_zonenum(folio),  in lruvec_add_folio_tail()
364 	list_add_tail(&folio->lru, &lruvec->lists[lru]);  in lruvec_add_folio_tail()
368 void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio)  in lruvec_del_folio()  argument
372 	if (lru_gen_del_folio(lruvec, folio, false))  in lruvec_del_folio()
377 	update_lru_size(lruvec, lru, folio_zonenum(folio),  in lruvec_del_folio()