204 static __always_inline const struct page *page_fixed_fake_head(const struct page *page)  in page_fixed_fake_head()  argument
207 		return page;  in page_fixed_fake_head()
215 	if (IS_ALIGNED((unsigned long)page, PAGE_SIZE) &&  in page_fixed_fake_head()
216 	    test_bit(PG_head, &page->flags)) {  in page_fixed_fake_head()
222 		unsigned long head = READ_ONCE(page[1].compound_head);  in page_fixed_fake_head()
225 			return (const struct page *)(head - 1);  in page_fixed_fake_head()
227 	return page;  in page_fixed_fake_head()
230 static __always_inline bool page_count_writable(const struct page *page, int u)  in page_count_writable()  argument
256 	if (atomic_read_acquire(&page->_refcount) == u)  in page_count_writable()
259 	return page_fixed_fake_head(page) == page;  in page_count_writable()
262 static inline const struct page *page_fixed_fake_head(const struct page *page)  in page_fixed_fake_head()  argument
264 	return page;  in page_fixed_fake_head()
267 static inline bool page_count_writable(const struct page *page, int u)  in page_count_writable()  argument
273 static __always_inline int page_is_fake_head(const struct page *page)  in page_is_fake_head()  argument
275 	return page_fixed_fake_head(page) != page;  in page_is_fake_head()
278 static __always_inline unsigned long _compound_head(const struct page *page)  in _compound_head()  argument
280 	unsigned long head = READ_ONCE(page->compound_head);  in _compound_head()
284 	return (unsigned long)page_fixed_fake_head(page);  in _compound_head()
287 #define compound_head(page)	((typeof(page))_compound_head(page))  argument
303 	const struct page *:	(const struct folio *)_compound_head(p), \
304 	struct page *:		(struct folio *)_compound_head(p)))
315 #define folio_page(folio, n)	nth_page(&(folio)->page, n)
317 static __always_inline int PageTail(const struct page *page)  in PageTail()  argument
319 	return READ_ONCE(page->compound_head) & 1 || page_is_fake_head(page);  in PageTail()
322 static __always_inline int PageCompound(const struct page *page)  in PageCompound()  argument
324 	return test_bit(PG_head, &page->flags) ||  in PageCompound()
325 	       READ_ONCE(page->compound_head) & 1;  in PageCompound()
329 static inline int PagePoisoned(const struct page *page)  in PagePoisoned()  argument
331 	return READ_ONCE(page->flags) == PAGE_POISON_PATTERN;  in PagePoisoned()
335 void page_init_poison(struct page *page, size_t size);
337 static inline void page_init_poison(struct page *page, size_t size)  in page_init_poison()  argument
345 	const struct page *page = &folio->page;  in const_folio_flags()  local
347 	VM_BUG_ON_PGFLAGS(page->compound_head & 1, page);  in const_folio_flags()
348 	VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page);  in const_folio_flags()
349 	return &page[n].flags;  in const_folio_flags()
354 	struct page *page = &folio->page;  in folio_flags()  local
356 	VM_BUG_ON_PGFLAGS(page->compound_head & 1, page);  in folio_flags()
357 	VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page);  in folio_flags()
358 	return &page[n].flags;  in folio_flags()
384 #define PF_POISONED_CHECK(page) ({					\  argument
385 		VM_BUG_ON_PGFLAGS(PagePoisoned(page), page);		\
386 		page; })
387 #define PF_ANY(page, enforce)	PF_POISONED_CHECK(page)  argument
388 #define PF_HEAD(page, enforce)	PF_POISONED_CHECK(compound_head(page))  argument
389 #define PF_NO_TAIL(page, enforce) ({					\  argument
390 		VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page);	\
391 		PF_POISONED_CHECK(compound_head(page)); })
392 #define PF_NO_COMPOUND(page, enforce) ({				\  argument
393 		VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page);	\
394 		PF_POISONED_CHECK(page); })
395 #define PF_SECOND(page, enforce) ({					\  argument
396 		VM_BUG_ON_PGFLAGS(!PageHead(page), page);		\
397 		PF_POISONED_CHECK(&page[1]); })
412 #define FOLIO_TEST_FLAG(name, page)					\  argument
414 { return test_bit(PG_##name, const_folio_flags(folio, page)); }
416 #define FOLIO_SET_FLAG(name, page)					\  argument
418 { set_bit(PG_##name, folio_flags(folio, page)); }
420 #define FOLIO_CLEAR_FLAG(name, page)					\  argument
422 { clear_bit(PG_##name, folio_flags(folio, page)); }
424 #define __FOLIO_SET_FLAG(name, page)					\  argument
426 { __set_bit(PG_##name, folio_flags(folio, page)); }
428 #define __FOLIO_CLEAR_FLAG(name, page)					\  argument
430 { __clear_bit(PG_##name, folio_flags(folio, page)); }
432 #define FOLIO_TEST_SET_FLAG(name, page)					\  argument
434 { return test_and_set_bit(PG_##name, folio_flags(folio, page)); }
436 #define FOLIO_TEST_CLEAR_FLAG(name, page)				\  argument
438 { return test_and_clear_bit(PG_##name, folio_flags(folio, page)); }
440 #define FOLIO_FLAG(name, page)						\  argument
441 FOLIO_TEST_FLAG(name, page)						\
442 FOLIO_SET_FLAG(name, page)						\
443 FOLIO_CLEAR_FLAG(name, page)
447 static __always_inline int Page##uname(const struct page *page)		\
448 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
452 static __always_inline void SetPage##uname(struct page *page)		\
453 { set_bit(PG_##lname, &policy(page, 1)->flags); }
457 static __always_inline void ClearPage##uname(struct page *page)		\
458 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
462 static __always_inline void __SetPage##uname(struct page *page)		\
463 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
467 static __always_inline void __ClearPage##uname(struct page *page)	\
468 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
472 static __always_inline int TestSetPage##uname(struct page *page)	\
473 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
477 static __always_inline int TestClearPage##uname(struct page *page)	\
478 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
519 static inline int Page##uname(const struct page *page) { return 0; }
523 static inline void SetPage##uname(struct page *page) {  }
527 static inline void ClearPage##uname(struct page *page) {  }
531 static inline void __ClearPage##uname(struct page *page) {  }
535 static inline int TestSetPage##uname(struct page *page) { return 0; }
539 static inline int TestClearPage##uname(struct page *page) { return 0; }
725 static __always_inline bool PageMappingFlags(const struct page *page)  in PageMappingFlags()  argument
727 	return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0;  in PageMappingFlags()
735 static __always_inline bool PageAnonNotKsm(const struct page *page)  in PageAnonNotKsm()  argument
737 	unsigned long flags = (unsigned long)page_folio(page)->mapping;  in PageAnonNotKsm()
742 static __always_inline bool PageAnon(const struct page *page)  in PageAnon()  argument
744 	return folio_test_anon(page_folio(page));  in PageAnon()
753 static __always_inline bool __PageMovable(const struct page *page)  in __PageMovable()  argument
755 	return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==  in __PageMovable()
775 u64 stable_page_flags(const struct page *page);
823 static inline bool PageUptodate(const struct page *page)  in PageUptodate()  argument
825 	return folio_test_uptodate(page_folio(page));  in PageUptodate()
845 static __always_inline void __SetPageUptodate(struct page *page)  in __SetPageUptodate()  argument
847 	__folio_mark_uptodate((struct folio *)page);  in __SetPageUptodate()
850 static __always_inline void SetPageUptodate(struct page *page)  in SetPageUptodate()  argument
852 	folio_mark_uptodate((struct folio *)page);  in SetPageUptodate()
858 void set_page_writeback(struct page *page);
870 static __always_inline int PageHead(const struct page *page)  in PageHead()  argument
872 	PF_POISONED_CHECK(page);  in PageHead()
873 	return test_bit(PG_head, &page->flags) && !page_is_fake_head(page);  in PageHead()
891 static __always_inline void set_compound_head(struct page *page, struct page *head)  in set_compound_head()  argument
893 	WRITE_ONCE(page->compound_head, (unsigned long)head + 1);  in set_compound_head()
896 static __always_inline void clear_compound_head(struct page *page)  in clear_compound_head()  argument
898 	WRITE_ONCE(page->compound_head, 0);  in clear_compound_head()
902 static inline void ClearPageCompound(struct page *page)  in ClearPageCompound()  argument
904 	BUG_ON(!PageHead(page));  in ClearPageCompound()
905 	ClearPageHead(page);  in ClearPageCompound()
925 static inline int PageTransHuge(const struct page *page)  in FOLIO_FLAG()
927 	VM_BUG_ON_PAGE(PageTail(page), page);  in FOLIO_FLAG()
928 	return PageHead(page);  in FOLIO_FLAG()
936 static inline int PageTransCompound(const struct page *page)  in PageTransCompound()  argument
938 	return PageCompound(page);  in PageTransCompound()
990 static inline bool page_has_type(const struct page *page)  in page_has_type()  argument
992 	return page_mapcount_is_type(data_race(page->page_type));  in page_has_type()
998 	return data_race(folio->page.page_type >> 24) == PGTY_##lname;	\
1004 	VM_BUG_ON_FOLIO(data_race(folio->page.page_type) != UINT_MAX,	\
1006 	folio->page.page_type = (unsigned int)PGTY_##lname << 24;	\
1010 	if (folio->page.page_type == UINT_MAX)				\
1013 	folio->page.page_type = UINT_MAX;				\
1018 static __always_inline int Page##uname(const struct page *page)		\
1020 	return data_race(page->page_type >> 24) == PGTY_##lname;	\
1022 static __always_inline void __SetPage##uname(struct page *page)		\
1024 	if (Page##uname(page))						\
1026 	VM_BUG_ON_PAGE(data_race(page->page_type) != UINT_MAX, page);	\
1027 	page->page_type = (unsigned int)PGTY_##lname << 24;		\
1029 static __always_inline void __ClearPage##uname(struct page *page)	\
1031 	if (page->page_type == UINT_MAX)				\
1033 	VM_BUG_ON_PAGE(!Page##uname(page), page);			\
1034 	page->page_type = UINT_MAX;					\
1098 static inline bool PageSlab(const struct page *page)  in PAGE_TYPE_OPS()
1100 	return folio_test_slab(page_folio(page));  in PAGE_TYPE_OPS()
1127 static inline bool PageHuge(const struct page *page)  in FOLIO_TYPE_OPS()
1129 	return folio_test_hugetlb(page_folio(page));  in FOLIO_TYPE_OPS()
1137 static inline bool is_page_hwpoison(const struct page *page)  in is_page_hwpoison()  argument
1141 	if (PageHWPoison(page))  in is_page_hwpoison()
1143 	folio = page_folio(page);  in is_page_hwpoison()
1144 	return folio_test_hugetlb(folio) && PageHWPoison(&folio->page);  in is_page_hwpoison()
1153 bool is_free_buddy_page(const struct page *page);
1157 static __always_inline int PageAnonExclusive(const struct page *page)  in PageAnonExclusive()  argument
1159 	VM_BUG_ON_PGFLAGS(!PageAnon(page), page);  in PageAnonExclusive()
1164 	if (PageHuge(page))  in PageAnonExclusive()
1165 		page = compound_head(page);  in PageAnonExclusive()
1166 	return test_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in PageAnonExclusive()
1169 static __always_inline void SetPageAnonExclusive(struct page *page)  in SetPageAnonExclusive()  argument
1171 	VM_BUG_ON_PGFLAGS(!PageAnonNotKsm(page), page);  in SetPageAnonExclusive()
1172 	VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);  in SetPageAnonExclusive()
1173 	set_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in SetPageAnonExclusive()
1176 static __always_inline void ClearPageAnonExclusive(struct page *page)  in ClearPageAnonExclusive()  argument
1178 	VM_BUG_ON_PGFLAGS(!PageAnonNotKsm(page), page);  in ClearPageAnonExclusive()
1179 	VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);  in ClearPageAnonExclusive()
1180 	clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in ClearPageAnonExclusive()
1183 static __always_inline void __ClearPageAnonExclusive(struct page *page)  in __ClearPageAnonExclusive()  argument
1185 	VM_BUG_ON_PGFLAGS(!PageAnon(page), page);  in __ClearPageAnonExclusive()
1186 	VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);  in __ClearPageAnonExclusive()
1187 	__clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in __ClearPageAnonExclusive()