Lines Matching refs:page
217 static void __free_pages_ok(struct page *page, unsigned int order,
292 static bool page_contains_unaccepted(struct page *page, unsigned int order);
295 static bool __free_unaccepted(struct page *page);
336 static inline unsigned long *get_pageblock_bitmap(const struct page *page, in get_pageblock_bitmap() argument
342 return page_zone(page)->pageblock_flags; in get_pageblock_bitmap()
346 static inline int pfn_to_bitidx(const struct page *page, unsigned long pfn) in pfn_to_bitidx() argument
351 pfn = pfn - pageblock_start_pfn(page_zone(page)->zone_start_pfn); in pfn_to_bitidx()
364 unsigned long get_pfnblock_flags_mask(const struct page *page, in get_pfnblock_flags_mask() argument
371 bitmap = get_pageblock_bitmap(page, pfn); in get_pfnblock_flags_mask()
372 bitidx = pfn_to_bitidx(page, pfn); in get_pfnblock_flags_mask()
384 static __always_inline int get_pfnblock_migratetype(const struct page *page, in get_pfnblock_migratetype() argument
387 return get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK); in get_pfnblock_migratetype()
397 void set_pfnblock_flags_mask(struct page *page, unsigned long flags, in set_pfnblock_flags_mask() argument
408 bitmap = get_pageblock_bitmap(page, pfn); in set_pfnblock_flags_mask()
409 bitidx = pfn_to_bitidx(page, pfn); in set_pfnblock_flags_mask()
413 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); in set_pfnblock_flags_mask()
423 void set_pageblock_migratetype(struct page *page, int migratetype) in set_pageblock_migratetype() argument
429 set_pfnblock_flags_mask(page, (unsigned long)migratetype, in set_pageblock_migratetype()
430 page_to_pfn(page), MIGRATETYPE_MASK); in set_pageblock_migratetype()
434 static int page_outside_zone_boundaries(struct zone *zone, struct page *page) in page_outside_zone_boundaries() argument
438 unsigned long pfn = page_to_pfn(page); in page_outside_zone_boundaries()
459 static bool __maybe_unused bad_range(struct zone *zone, struct page *page) in bad_range() argument
461 if (page_outside_zone_boundaries(zone, page)) in bad_range()
463 if (zone != page_zone(page)) in bad_range()
469 static inline bool __maybe_unused bad_range(struct zone *zone, struct page *page) in bad_range() argument
475 static void bad_page(struct page *page, const char *reason) in bad_page() argument
502 current->comm, page_to_pfn(page)); in bad_page()
503 dump_page(page, reason); in bad_page()
509 if (PageBuddy(page)) in bad_page()
510 __ClearPageBuddy(page); in bad_page()
570 void prep_compound_page(struct page *page, unsigned int order) in prep_compound_page() argument
575 __SetPageHead(page); in prep_compound_page()
577 prep_compound_tail(page, i); in prep_compound_page()
579 prep_compound_head(page, order); in prep_compound_page()
582 static inline void set_buddy_order(struct page *page, unsigned int order) in set_buddy_order() argument
584 set_page_private(page, order); in set_buddy_order()
585 __SetPageBuddy(page); in set_buddy_order()
595 !capc->page && in task_capc()
600 compaction_capture(struct capture_control *capc, struct page *page, in compaction_capture() argument
623 trace_mm_page_alloc_extfrag(page, capc->cc->order, order, in compaction_capture()
626 capc->page = page; in compaction_capture()
637 compaction_capture(struct capture_control *capc, struct page *page, in compaction_capture() argument
662 static inline void __add_to_free_list(struct page *page, struct zone *zone, in __add_to_free_list() argument
669 VM_WARN_ONCE(get_pageblock_migratetype(page) != migratetype, in __add_to_free_list()
671 get_pageblock_migratetype(page), migratetype, nr_pages); in __add_to_free_list()
674 list_add_tail(&page->buddy_list, &area->free_list[migratetype]); in __add_to_free_list()
676 list_add(&page->buddy_list, &area->free_list[migratetype]); in __add_to_free_list()
688 static inline void move_to_free_list(struct page *page, struct zone *zone, in move_to_free_list() argument
695 VM_WARN_ONCE(get_pageblock_migratetype(page) != old_mt, in move_to_free_list()
697 get_pageblock_migratetype(page), old_mt, nr_pages); in move_to_free_list()
699 list_move_tail(&page->buddy_list, &area->free_list[new_mt]); in move_to_free_list()
712 static inline void __del_page_from_free_list(struct page *page, struct zone *zone, in __del_page_from_free_list() argument
717 VM_WARN_ONCE(get_pageblock_migratetype(page) != migratetype, in __del_page_from_free_list()
719 get_pageblock_migratetype(page), migratetype, nr_pages); in __del_page_from_free_list()
722 if (page_reported(page)) in __del_page_from_free_list()
723 __ClearPageReported(page); in __del_page_from_free_list()
725 list_del(&page->buddy_list); in __del_page_from_free_list()
726 __ClearPageBuddy(page); in __del_page_from_free_list()
727 set_page_private(page, 0); in __del_page_from_free_list()
734 static inline void del_page_from_free_list(struct page *page, struct zone *zone, in del_page_from_free_list() argument
737 __del_page_from_free_list(page, zone, order, migratetype); in del_page_from_free_list()
741 static inline struct page *get_page_from_free_area(struct free_area *area, in get_page_from_free_area()
745 struct page, buddy_list); in get_page_from_free_area()
758 struct page *page, unsigned int order) in buddy_merge_likely() argument
761 struct page *higher_page; in buddy_merge_likely()
767 higher_page = page + (higher_page_pfn - pfn); in buddy_merge_likely()
797 static inline void __free_one_page(struct page *page, in __free_one_page() argument
805 struct page *buddy; in __free_one_page()
809 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); in __free_one_page()
812 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page); in __free_one_page()
813 VM_BUG_ON_PAGE(bad_range(zone, page), page); in __free_one_page()
820 if (compaction_capture(capc, page, order, migratetype)) { in __free_one_page()
825 buddy = find_buddy_page_pfn(page, pfn, order, &buddy_pfn); in __free_one_page()
863 page = page + (combined_pfn - pfn); in __free_one_page()
869 set_buddy_order(page, order); in __free_one_page()
876 to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order); in __free_one_page()
878 __add_to_free_list(page, zone, order, migratetype, to_tail); in __free_one_page()
890 static inline bool page_expected_state(struct page *page, in page_expected_state() argument
893 if (unlikely(atomic_read(&page->_mapcount) != -1)) in page_expected_state()
896 if (unlikely((unsigned long)page->mapping | in page_expected_state()
897 page_ref_count(page) | in page_expected_state()
899 page->memcg_data | in page_expected_state()
902 ((page->pp_magic & ~0x3UL) == PP_SIGNATURE) | in page_expected_state()
904 (page->flags & check_flags))) in page_expected_state()
910 static const char *page_bad_reason(struct page *page, unsigned long flags) in page_bad_reason() argument
914 if (unlikely(atomic_read(&page->_mapcount) != -1)) in page_bad_reason()
916 if (unlikely(page->mapping != NULL)) in page_bad_reason()
918 if (unlikely(page_ref_count(page) != 0)) in page_bad_reason()
920 if (unlikely(page->flags & flags)) { in page_bad_reason()
927 if (unlikely(page->memcg_data)) in page_bad_reason()
931 if (unlikely((page->pp_magic & ~0x3UL) == PP_SIGNATURE)) in page_bad_reason()
937 static void free_page_is_bad_report(struct page *page) in free_page_is_bad_report() argument
939 bad_page(page, in free_page_is_bad_report()
940 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE)); in free_page_is_bad_report()
943 static inline bool free_page_is_bad(struct page *page) in free_page_is_bad() argument
945 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE))) in free_page_is_bad()
949 free_page_is_bad_report(page); in free_page_is_bad()
958 static int free_tail_page_prepare(struct page *head_page, struct page *page) in free_tail_page_prepare() argument
973 switch (page - head_page) { in free_tail_page_prepare()
977 bad_page(page, "nonzero large_mapcount"); in free_tail_page_prepare()
982 bad_page(page, "nonzero nr_pages_mapped"); in free_tail_page_prepare()
987 bad_page(page, "nonzero mm mapcount 0"); in free_tail_page_prepare()
991 bad_page(page, "nonzero mm mapcount 1"); in free_tail_page_prepare()
997 bad_page(page, "nonzero entire_mapcount"); in free_tail_page_prepare()
1001 bad_page(page, "nonzero pincount"); in free_tail_page_prepare()
1009 bad_page(page, "on deferred list"); in free_tail_page_prepare()
1014 bad_page(page, "nonzero entire_mapcount"); in free_tail_page_prepare()
1018 bad_page(page, "nonzero pincount"); in free_tail_page_prepare()
1029 if (page->mapping != TAIL_MAPPING) { in free_tail_page_prepare()
1030 bad_page(page, "corrupted mapping in tail page"); in free_tail_page_prepare()
1035 if (unlikely(!PageTail(page))) { in free_tail_page_prepare()
1036 bad_page(page, "PageTail not set"); in free_tail_page_prepare()
1039 if (unlikely(compound_head(page) != head_page)) { in free_tail_page_prepare()
1040 bad_page(page, "compound_head not consistent"); in free_tail_page_prepare()
1045 page->mapping = NULL; in free_tail_page_prepare()
1046 clear_compound_head(page); in free_tail_page_prepare()
1079 static inline bool should_skip_kasan_poison(struct page *page) in should_skip_kasan_poison() argument
1084 return page_kasan_tag(page) == KASAN_TAG_KERNEL; in should_skip_kasan_poison()
1087 static void kernel_init_pages(struct page *page, int numpages) in kernel_init_pages() argument
1094 clear_highpage_kasan_tagged(page + i); in kernel_init_pages()
1101 void __clear_page_tag_ref(struct page *page) in __clear_page_tag_ref() argument
1106 if (get_page_tag_ref(page, &ref, &handle)) { in __clear_page_tag_ref()
1115 void __pgalloc_tag_add(struct page *page, struct task_struct *task, in __pgalloc_tag_add() argument
1121 if (get_page_tag_ref(page, &ref, &handle)) { in __pgalloc_tag_add()
1128 static inline void pgalloc_tag_add(struct page *page, struct task_struct *task, in pgalloc_tag_add() argument
1132 __pgalloc_tag_add(page, task, nr); in pgalloc_tag_add()
1137 void __pgalloc_tag_sub(struct page *page, unsigned int nr) in __pgalloc_tag_sub() argument
1142 if (get_page_tag_ref(page, &ref, &handle)) { in __pgalloc_tag_sub()
1149 static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) in pgalloc_tag_sub() argument
1152 __pgalloc_tag_sub(page, nr); in pgalloc_tag_sub()
1164 static inline void pgalloc_tag_add(struct page *page, struct task_struct *task, in pgalloc_tag_add() argument
1166 static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {} in pgalloc_tag_sub() argument
1171 __always_inline bool free_pages_prepare(struct page *page, in free_pages_prepare() argument
1175 bool skip_kasan_poison = should_skip_kasan_poison(page); in free_pages_prepare()
1177 bool compound = PageCompound(page); in free_pages_prepare()
1178 struct folio *folio = page_folio(page); in free_pages_prepare()
1180 VM_BUG_ON_PAGE(PageTail(page), page); in free_pages_prepare()
1182 trace_mm_page_free(page, order); in free_pages_prepare()
1183 kmsan_free_page(page, order); in free_pages_prepare()
1185 if (memcg_kmem_online() && PageMemcgKmem(page)) in free_pages_prepare()
1186 __memcg_kmem_uncharge_page(page, order); in free_pages_prepare()
1202 if (unlikely(PageHWPoison(page)) && !order) { in free_pages_prepare()
1204 reset_page_owner(page, order); in free_pages_prepare()
1205 page_table_check_free(page, order); in free_pages_prepare()
1206 pgalloc_tag_sub(page, 1 << order); in free_pages_prepare()
1213 clear_page_tag_ref(page); in free_pages_prepare()
1217 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); in free_pages_prepare()
1227 page[1].flags &= ~PAGE_FLAGS_SECOND; in free_pages_prepare()
1234 bad += free_tail_page_prepare(page, page + i); in free_pages_prepare()
1236 if (free_page_is_bad(page + i)) { in free_pages_prepare()
1241 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; in free_pages_prepare()
1244 if (PageMappingFlags(page)) { in free_pages_prepare()
1245 if (PageAnon(page)) in free_pages_prepare()
1247 page->mapping = NULL; in free_pages_prepare()
1250 if (free_page_is_bad(page)) in free_pages_prepare()
1256 page_cpupid_reset_last(page); in free_pages_prepare()
1257 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; in free_pages_prepare()
1258 reset_page_owner(page, order); in free_pages_prepare()
1259 page_table_check_free(page, order); in free_pages_prepare()
1260 pgalloc_tag_sub(page, 1 << order); in free_pages_prepare()
1262 if (!PageHighMem(page)) { in free_pages_prepare()
1263 debug_check_no_locks_freed(page_address(page), in free_pages_prepare()
1265 debug_check_no_obj_freed(page_address(page), in free_pages_prepare()
1269 kernel_poison_pages(page, 1 << order); in free_pages_prepare()
1280 kasan_poison_pages(page, order, init); in free_pages_prepare()
1287 kernel_init_pages(page, 1 << order); in free_pages_prepare()
1294 arch_free_page(page, order); in free_pages_prepare()
1296 debug_pagealloc_unmap_pages(page, 1 << order); in free_pages_prepare()
1312 struct page *page; in free_pcppages_bulk() local
1342 page = list_last_entry(list, struct page, pcp_list); in free_pcppages_bulk()
1343 pfn = page_to_pfn(page); in free_pcppages_bulk()
1344 mt = get_pfnblock_migratetype(page, pfn); in free_pcppages_bulk()
1347 list_del(&page->pcp_list); in free_pcppages_bulk()
1351 __free_one_page(page, pfn, zone, order, mt, FPI_NONE); in free_pcppages_bulk()
1352 trace_mm_page_pcpu_drain(page, order, mt); in free_pcppages_bulk()
1360 static void split_large_buddy(struct zone *zone, struct page *page, in split_large_buddy() argument
1367 VM_WARN_ON_ONCE(PageBuddy(page)); in split_large_buddy()
1373 int mt = get_pfnblock_migratetype(page, pfn); in split_large_buddy()
1375 __free_one_page(page, pfn, zone, order, mt, fpi); in split_large_buddy()
1379 page = pfn_to_page(pfn); in split_large_buddy()
1383 static void add_page_to_zone_llist(struct zone *zone, struct page *page, in add_page_to_zone_llist() argument
1387 page->order = order; in add_page_to_zone_llist()
1389 llist_add(&page->pcp_llist, &zone->trylock_free_pages); in add_page_to_zone_llist()
1392 static void free_one_page(struct zone *zone, struct page *page, in free_one_page() argument
1401 add_page_to_zone_llist(zone, page, order); in free_one_page()
1412 struct page *p, *tmp; in free_one_page()
1422 split_large_buddy(zone, page, pfn, order, fpi_flags); in free_one_page()
1428 static void __free_pages_ok(struct page *page, unsigned int order, in __free_pages_ok() argument
1431 unsigned long pfn = page_to_pfn(page); in __free_pages_ok()
1432 struct zone *zone = page_zone(page); in __free_pages_ok()
1434 if (free_pages_prepare(page, order)) in __free_pages_ok()
1435 free_one_page(zone, page, pfn, order, fpi_flags); in __free_pages_ok()
1438 void __meminit __free_pages_core(struct page *page, unsigned int order, in __free_pages_core() argument
1442 struct page *p = page; in __free_pages_core()
1461 adjust_managed_page_count(page, nr_pages); in __free_pages_core()
1469 atomic_long_add(nr_pages, &page_zone(page)->managed_pages); in __free_pages_core()
1472 if (page_contains_unaccepted(page, order)) { in __free_pages_core()
1473 if (order == MAX_PAGE_ORDER && __free_unaccepted(page)) in __free_pages_core()
1476 accept_memory(page_to_phys(page), PAGE_SIZE << order); in __free_pages_core()
1483 __free_pages_ok(page, order, FPI_TO_TAIL); in __free_pages_core()
1510 struct page *__pageblock_pfn_to_page(unsigned long start_pfn, in __pageblock_pfn_to_page()
1513 struct page *start_page; in __pageblock_pfn_to_page()
1514 struct page *end_page; in __pageblock_pfn_to_page()
1552 static inline unsigned int expand(struct zone *zone, struct page *page, int low, in expand() argument
1561 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); in expand()
1569 if (set_page_guard(zone, &page[size], high)) in expand()
1572 __add_to_free_list(&page[size], zone, high, migratetype, false); in expand()
1573 set_buddy_order(&page[size], high); in expand()
1581 struct page *page, int low, in page_del_and_expand() argument
1586 __del_page_from_free_list(page, zone, high, migratetype); in page_del_and_expand()
1587 nr_pages -= expand(zone, page, low, high, migratetype); in page_del_and_expand()
1591 static void check_new_page_bad(struct page *page) in check_new_page_bad() argument
1593 if (unlikely(PageHWPoison(page))) { in check_new_page_bad()
1595 if (PageBuddy(page)) in check_new_page_bad()
1596 __ClearPageBuddy(page); in check_new_page_bad()
1600 bad_page(page, in check_new_page_bad()
1601 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP)); in check_new_page_bad()
1607 static bool check_new_page(struct page *page) in check_new_page() argument
1609 if (likely(page_expected_state(page, in check_new_page()
1613 check_new_page_bad(page); in check_new_page()
1617 static inline bool check_new_pages(struct page *page, unsigned int order) in check_new_pages() argument
1621 struct page *p = page + i; in check_new_pages()
1659 inline void post_alloc_hook(struct page *page, unsigned int order, in post_alloc_hook() argument
1667 set_page_private(page, 0); in post_alloc_hook()
1669 arch_alloc_page(page, order); in post_alloc_hook()
1670 debug_pagealloc_map_pages(page, 1 << order); in post_alloc_hook()
1677 kernel_unpoison_pages(page, 1 << order); in post_alloc_hook()
1692 tag_clear_highpage(page + i); in post_alloc_hook()
1698 kasan_unpoison_pages(page, order, init)) { in post_alloc_hook()
1708 page_kasan_tag_reset(page + i); in post_alloc_hook()
1712 kernel_init_pages(page, 1 << order); in post_alloc_hook()
1714 set_page_owner(page, order, gfp_flags); in post_alloc_hook()
1715 page_table_check_alloc(page, order); in post_alloc_hook()
1716 pgalloc_tag_add(page, current, 1 << order); in post_alloc_hook()
1719 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, in prep_new_page() argument
1722 post_alloc_hook(page, order, gfp_flags); in prep_new_page()
1725 prep_compound_page(page, order); in prep_new_page()
1734 set_page_pfmemalloc(page); in prep_new_page()
1736 clear_page_pfmemalloc(page); in prep_new_page()
1744 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, in __rmqueue_smallest()
1749 struct page *page; in __rmqueue_smallest() local
1754 page = get_page_from_free_area(area, migratetype); in __rmqueue_smallest()
1755 if (!page) in __rmqueue_smallest()
1758 page_del_and_expand(zone, page, order, current_order, in __rmqueue_smallest()
1760 trace_mm_page_alloc_zone_locked(page, order, migratetype, in __rmqueue_smallest()
1763 return page; in __rmqueue_smallest()
1783 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone, in __rmqueue_cma_fallback()
1789 static inline struct page *__rmqueue_cma_fallback(struct zone *zone, in __rmqueue_cma_fallback()
1800 struct page *page; in __move_freepages_block() local
1809 page = pfn_to_page(pfn); in __move_freepages_block()
1810 if (!PageBuddy(page)) { in __move_freepages_block()
1816 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); in __move_freepages_block()
1817 VM_BUG_ON_PAGE(page_zone(page) != zone, page); in __move_freepages_block()
1819 order = buddy_order(page); in __move_freepages_block()
1821 move_to_free_list(page, zone, order, old_mt, new_mt); in __move_freepages_block()
1832 static bool prep_move_freepages_block(struct zone *zone, struct page *page, in prep_move_freepages_block() argument
1838 pfn = page_to_pfn(page); in prep_move_freepages_block()
1860 page = pfn_to_page(pfn); in prep_move_freepages_block()
1861 if (PageBuddy(page)) { in prep_move_freepages_block()
1862 int nr = 1 << buddy_order(page); in prep_move_freepages_block()
1873 if (PageLRU(page) || __PageMovable(page)) in prep_move_freepages_block()
1882 static int move_freepages_block(struct zone *zone, struct page *page, in move_freepages_block() argument
1887 if (!prep_move_freepages_block(zone, page, &start_pfn, NULL, NULL)) in move_freepages_block()
1898 struct page *page; in find_large_buddy() local
1901 while (!PageBuddy(page = pfn_to_page(pfn))) { in find_large_buddy()
1911 if (pfn + (1 << buddy_order(page)) > start_pfn) in find_large_buddy()
1937 bool move_freepages_block_isolate(struct zone *zone, struct page *page, in move_freepages_block_isolate() argument
1942 if (!prep_move_freepages_block(zone, page, &start_pfn, NULL, NULL)) in move_freepages_block_isolate()
1952 struct page *buddy = pfn_to_page(pfn); in move_freepages_block_isolate()
1957 set_pageblock_migratetype(page, migratetype); in move_freepages_block_isolate()
1963 if (PageBuddy(page) && buddy_order(page) > pageblock_order) { in move_freepages_block_isolate()
1964 int order = buddy_order(page); in move_freepages_block_isolate()
1966 del_page_from_free_list(page, zone, order, in move_freepages_block_isolate()
1967 get_pfnblock_migratetype(page, pfn)); in move_freepages_block_isolate()
1968 set_pageblock_migratetype(page, migratetype); in move_freepages_block_isolate()
1969 split_large_buddy(zone, page, pfn, order, FPI_NONE); in move_freepages_block_isolate()
1974 get_pfnblock_migratetype(page, start_pfn), in move_freepages_block_isolate()
1980 static void change_pageblock_range(struct page *pageblock_page, in change_pageblock_range()
2115 static struct page *
2116 try_to_claim_block(struct zone *zone, struct page *page, in try_to_claim_block() argument
2127 del_page_from_free_list(page, zone, current_order, block_type); in try_to_claim_block()
2128 change_pageblock_range(page, current_order, start_type); in try_to_claim_block()
2129 nr_added = expand(zone, page, order, current_order, start_type); in try_to_claim_block()
2131 return page; in try_to_claim_block()
2143 if (!prep_move_freepages_block(zone, page, &start_pfn, &free_pages, in try_to_claim_block()
2189 static __always_inline struct page *
2196 struct page *page; in __rmqueue_claim() local
2224 page = get_page_from_free_area(area, fallback_mt); in __rmqueue_claim()
2225 page = try_to_claim_block(zone, page, current_order, order, in __rmqueue_claim()
2228 if (page) { in __rmqueue_claim()
2229 trace_mm_page_alloc_extfrag(page, order, current_order, in __rmqueue_claim()
2231 return page; in __rmqueue_claim()
2242 static __always_inline struct page *
2247 struct page *page; in __rmqueue_steal() local
2258 page = get_page_from_free_area(area, fallback_mt); in __rmqueue_steal()
2259 page_del_and_expand(zone, page, order, current_order, fallback_mt); in __rmqueue_steal()
2260 trace_mm_page_alloc_extfrag(page, order, current_order, in __rmqueue_steal()
2262 return page; in __rmqueue_steal()
2279 static __always_inline struct page *
2283 struct page *page; in __rmqueue() local
2294 page = __rmqueue_cma_fallback(zone, order); in __rmqueue()
2295 if (page) in __rmqueue()
2296 return page; in __rmqueue()
2311 page = __rmqueue_smallest(zone, order, migratetype); in __rmqueue()
2312 if (page) in __rmqueue()
2313 return page; in __rmqueue()
2317 page = __rmqueue_cma_fallback(zone, order); in __rmqueue()
2318 if (page) { in __rmqueue()
2320 return page; in __rmqueue()
2325 page = __rmqueue_claim(zone, order, migratetype, alloc_flags); in __rmqueue()
2326 if (page) { in __rmqueue()
2329 return page; in __rmqueue()
2334 page = __rmqueue_steal(zone, order, migratetype); in __rmqueue()
2335 if (page) { in __rmqueue()
2337 return page; in __rmqueue()
2364 struct page *page = __rmqueue(zone, order, migratetype, in rmqueue_bulk() local
2366 if (unlikely(page == NULL)) in rmqueue_bulk()
2379 list_add_tail(&page->pcp_list, list); in rmqueue_bulk()
2649 struct per_cpu_pages *pcp, struct page *page, int migratetype, in free_frozen_page_commit() argument
2664 list_add(&page->pcp_list, &pcp->lists[pindex]); in free_frozen_page_commit()
2707 static void __free_frozen_pages(struct page *page, unsigned int order, in __free_frozen_pages() argument
2713 unsigned long pfn = page_to_pfn(page); in __free_frozen_pages()
2717 __free_pages_ok(page, order, fpi_flags); in __free_frozen_pages()
2721 if (!free_pages_prepare(page, order)) in __free_frozen_pages()
2731 zone = page_zone(page); in __free_frozen_pages()
2732 migratetype = get_pfnblock_migratetype(page, pfn); in __free_frozen_pages()
2735 free_one_page(zone, page, pfn, order, fpi_flags); in __free_frozen_pages()
2743 add_page_to_zone_llist(zone, page, order); in __free_frozen_pages()
2749 free_frozen_page_commit(zone, pcp, page, migratetype, order, fpi_flags); in __free_frozen_pages()
2752 free_one_page(zone, page, pfn, order, fpi_flags); in __free_frozen_pages()
2757 void free_frozen_pages(struct page *page, unsigned int order) in free_frozen_pages() argument
2759 __free_frozen_pages(page, order, FPI_NONE); in free_frozen_pages()
2778 if (!free_pages_prepare(&folio->page, order)) in free_unref_folios()
2785 free_one_page(folio_zone(folio), &folio->page, in free_unref_folios()
2804 migratetype = get_pfnblock_migratetype(&folio->page, pfn); in free_unref_folios()
2821 free_one_page(zone, &folio->page, pfn, in free_unref_folios()
2834 free_one_page(zone, &folio->page, pfn, in free_unref_folios()
2848 trace_mm_page_free_batched(&folio->page); in free_unref_folios()
2849 free_frozen_page_commit(zone, pcp, &folio->page, migratetype, in free_unref_folios()
2868 void split_page(struct page *page, unsigned int order) in split_page() argument
2872 VM_BUG_ON_PAGE(PageCompound(page), page); in split_page()
2873 VM_BUG_ON_PAGE(!page_count(page), page); in split_page()
2876 set_page_refcounted(page + i); in split_page()
2877 split_page_owner(page, order, 0); in split_page()
2878 pgalloc_tag_split(page_folio(page), order, 0); in split_page()
2879 split_page_memcg(page, order); in split_page()
2883 int __isolate_free_page(struct page *page, unsigned int order) in __isolate_free_page() argument
2885 struct zone *zone = page_zone(page); in __isolate_free_page()
2886 int mt = get_pageblock_migratetype(page); in __isolate_free_page()
2901 del_page_from_free_list(page, zone, order, mt); in __isolate_free_page()
2908 struct page *endpage = page + (1 << order) - 1; in __isolate_free_page()
2909 for (; page < endpage; page += pageblock_nr_pages) { in __isolate_free_page()
2910 int mt = get_pageblock_migratetype(page); in __isolate_free_page()
2916 move_freepages_block(zone, page, mt, in __isolate_free_page()
2933 void __putback_isolated_page(struct page *page, unsigned int order, int mt) in __putback_isolated_page() argument
2935 struct zone *zone = page_zone(page); in __putback_isolated_page()
2941 __free_one_page(page, page_to_pfn(page), zone, order, mt, in __putback_isolated_page()
2972 struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone, in rmqueue_buddy()
2976 struct page *page; in rmqueue_buddy() local
2980 page = NULL; in rmqueue_buddy()
2988 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); in rmqueue_buddy()
2989 if (!page) { in rmqueue_buddy()
2992 page = __rmqueue(zone, order, migratetype, alloc_flags, &rmqm); in rmqueue_buddy()
3000 if (!page && (alloc_flags & (ALLOC_OOM|ALLOC_NON_BLOCK))) in rmqueue_buddy()
3001 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); in rmqueue_buddy()
3003 if (!page) { in rmqueue_buddy()
3009 } while (check_new_pages(page, order)); in rmqueue_buddy()
3011 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); in rmqueue_buddy()
3014 return page; in rmqueue_buddy()
3069 struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order, in __rmqueue_pcplist()
3075 struct page *page; in __rmqueue_pcplist() local
3091 page = list_first_entry(list, struct page, pcp_list); in __rmqueue_pcplist()
3092 list_del(&page->pcp_list); in __rmqueue_pcplist()
3094 } while (check_new_pages(page, order)); in __rmqueue_pcplist()
3096 return page; in __rmqueue_pcplist()
3100 static struct page *rmqueue_pcplist(struct zone *preferred_zone, in rmqueue_pcplist()
3106 struct page *page; in rmqueue_pcplist() local
3124 page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp, list); in rmqueue_pcplist()
3127 if (page) { in rmqueue_pcplist()
3128 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); in rmqueue_pcplist()
3131 return page; in rmqueue_pcplist()
3147 struct page *rmqueue(struct zone *preferred_zone, in rmqueue()
3152 struct page *page; in rmqueue() local
3155 page = rmqueue_pcplist(preferred_zone, zone, order, in rmqueue()
3157 if (likely(page)) in rmqueue()
3161 page = rmqueue_buddy(preferred_zone, zone, order, alloc_flags, in rmqueue()
3172 VM_BUG_ON_PAGE(page && bad_range(zone, page), page); in rmqueue()
3173 return page; in rmqueue()
3181 static void reserve_highatomic_pageblock(struct page *page, int order, in reserve_highatomic_pageblock() argument
3206 mt = get_pageblock_migratetype(page); in reserve_highatomic_pageblock()
3212 if (move_freepages_block(zone, page, mt, MIGRATE_HIGHATOMIC) == -1) in reserve_highatomic_pageblock()
3216 change_pageblock_range(page, order, MIGRATE_HIGHATOMIC); in reserve_highatomic_pageblock()
3240 struct page *page; in unreserve_highatomic_pageblock() local
3259 page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC); in unreserve_highatomic_pageblock()
3260 if (!page) in unreserve_highatomic_pageblock()
3285 ret = move_freepages_block(zone, page, in unreserve_highatomic_pageblock()
3289 move_to_free_list(page, zone, order, in unreserve_highatomic_pageblock()
3292 change_pageblock_range(page, order, in unreserve_highatomic_pageblock()
3544 static struct page *
3563 struct page *page; in get_page_from_freelist() local
3681 page = rmqueue(zonelist_zone(ac->preferred_zoneref), zone, order, in get_page_from_freelist()
3683 if (page) { in get_page_from_freelist()
3684 prep_new_page(page, order, gfp_mask, alloc_flags); in get_page_from_freelist()
3691 reserve_highatomic_pageblock(page, order, zone); in get_page_from_freelist()
3693 return page; in get_page_from_freelist()
3762 static inline struct page *
3767 struct page *page; in __alloc_pages_cpuset_fallback() local
3769 page = get_page_from_freelist(gfp_mask, order, in __alloc_pages_cpuset_fallback()
3775 if (!page) in __alloc_pages_cpuset_fallback()
3776 page = get_page_from_freelist(gfp_mask, order, in __alloc_pages_cpuset_fallback()
3778 return page; in __alloc_pages_cpuset_fallback()
3781 static inline struct page *
3792 struct page *page; in __alloc_pages_may_oom() local
3813 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & in __alloc_pages_may_oom()
3816 if (page) in __alloc_pages_may_oom()
3860 page = __alloc_pages_cpuset_fallback(gfp_mask, order, in __alloc_pages_may_oom()
3865 return page; in __alloc_pages_may_oom()
3876 static struct page *
3881 struct page *page = NULL; in __alloc_pages_direct_compact() local
3893 prio, &page); in __alloc_pages_direct_compact()
3908 if (page) in __alloc_pages_direct_compact()
3909 prep_new_page(page, order, gfp_mask, alloc_flags); in __alloc_pages_direct_compact()
3912 if (!page) in __alloc_pages_direct_compact()
3913 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); in __alloc_pages_direct_compact()
3915 if (page) { in __alloc_pages_direct_compact()
3916 struct zone *zone = page_zone(page); in __alloc_pages_direct_compact()
3921 return page; in __alloc_pages_direct_compact()
4001 static inline struct page *
4148 static inline struct page *
4153 struct page *page = NULL; in __alloc_pages_direct_reclaim() local
4163 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); in __alloc_pages_direct_reclaim()
4170 if (!page && !drained) { in __alloc_pages_direct_reclaim()
4179 return page; in __alloc_pages_direct_reclaim()
4421 static inline struct page *
4429 struct page *page = NULL; in __alloc_pages_slowpath() local
4505 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); in __alloc_pages_slowpath()
4506 if (page) in __alloc_pages_slowpath()
4522 page = __alloc_pages_direct_compact(gfp_mask, order, in __alloc_pages_slowpath()
4526 if (page) in __alloc_pages_slowpath()
4594 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); in __alloc_pages_slowpath()
4595 if (page) in __alloc_pages_slowpath()
4607 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, in __alloc_pages_slowpath()
4609 if (page) in __alloc_pages_slowpath()
4613 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, in __alloc_pages_slowpath()
4615 if (page) in __alloc_pages_slowpath()
4661 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); in __alloc_pages_slowpath()
4662 if (page) in __alloc_pages_slowpath()
4706 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_MIN_RESERVE, ac); in __alloc_pages_slowpath()
4707 if (page) in __alloc_pages_slowpath()
4717 return page; in __alloc_pages_slowpath()
4786 struct page **page_array) in alloc_pages_bulk_noprof()
4788 struct page *page; in alloc_pages_bulk_noprof() local
4898 page = __rmqueue_pcplist(zone, 0, ac.migratetype, alloc_flags, in alloc_pages_bulk_noprof()
4900 if (unlikely(!page)) { in alloc_pages_bulk_noprof()
4910 prep_new_page(page, 0, gfp, 0); in alloc_pages_bulk_noprof()
4911 set_page_refcounted(page); in alloc_pages_bulk_noprof()
4912 page_array[nr_populated++] = page; in alloc_pages_bulk_noprof()
4928 page = __alloc_pages_noprof(gfp, 0, preferred_nid, nodemask); in alloc_pages_bulk_noprof()
4929 if (page) in alloc_pages_bulk_noprof()
4930 page_array[nr_populated++] = page; in alloc_pages_bulk_noprof()
4938 struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, in __alloc_frozen_pages_noprof()
4941 struct page *page; in __alloc_frozen_pages_noprof() local
4974 page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac); in __alloc_frozen_pages_noprof()
4975 if (likely(page)) in __alloc_frozen_pages_noprof()
4987 page = __alloc_pages_slowpath(alloc_gfp, order, &ac); in __alloc_frozen_pages_noprof()
4990 if (memcg_kmem_online() && (gfp & __GFP_ACCOUNT) && page && in __alloc_frozen_pages_noprof()
4991 unlikely(__memcg_kmem_charge_page(page, gfp, order) != 0)) { in __alloc_frozen_pages_noprof()
4992 free_frozen_pages(page, order); in __alloc_frozen_pages_noprof()
4993 page = NULL; in __alloc_frozen_pages_noprof()
4996 trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype); in __alloc_frozen_pages_noprof()
4997 kmsan_alloc_page(page, order, alloc_gfp); in __alloc_frozen_pages_noprof()
4999 return page; in __alloc_frozen_pages_noprof()
5003 struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, in __alloc_pages_noprof()
5006 struct page *page; in __alloc_pages_noprof() local
5008 page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, nodemask); in __alloc_pages_noprof()
5009 if (page) in __alloc_pages_noprof()
5010 set_page_refcounted(page); in __alloc_pages_noprof()
5011 return page; in __alloc_pages_noprof()
5018 struct page *page = __alloc_pages_noprof(gfp | __GFP_COMP, order, in __folio_alloc_noprof() local
5020 return page_rmappable_folio(page); in __folio_alloc_noprof()
5031 struct page *page; in get_free_pages_noprof() local
5033 page = alloc_pages_noprof(gfp_mask & ~__GFP_HIGHMEM, order); in get_free_pages_noprof()
5034 if (!page) in get_free_pages_noprof()
5036 return (unsigned long) page_address(page); in get_free_pages_noprof()
5067 static void ___free_pages(struct page *page, unsigned int order, in ___free_pages() argument
5071 int head = PageHead(page); in ___free_pages()
5073 struct alloc_tag *tag = pgalloc_tag_get(page); in ___free_pages()
5075 if (put_page_testzero(page)) in ___free_pages()
5076 __free_frozen_pages(page, order, fpi_flags); in ___free_pages()
5080 __free_frozen_pages(page + (1 << order), order, in ___free_pages()
5084 void __free_pages(struct page *page, unsigned int order) in __free_pages() argument
5086 ___free_pages(page, order, FPI_NONE); in __free_pages()
5094 void free_pages_nolock(struct page *page, unsigned int order) in free_pages_nolock() argument
5096 ___free_pages(page, order, FPI_TRYLOCK); in free_pages_nolock()
5114 struct page *page = virt_to_page((void *)addr); in make_alloc_exact() local
5115 struct page *last = page + nr; in make_alloc_exact()
5117 split_page_owner(page, order, 0); in make_alloc_exact()
5118 pgalloc_tag_split(page_folio(page), order, 0); in make_alloc_exact()
5119 split_page_memcg(page, order); in make_alloc_exact()
5120 while (page < --last) in make_alloc_exact()
5123 last = page + (1UL << order); in make_alloc_exact()
5124 for (page += nr; page < last; page++) in make_alloc_exact()
5125 __free_pages_ok(page, 0, FPI_TO_TAIL); in make_alloc_exact()
5173 struct page *p; in alloc_pages_exact_nid_noprof()
5940 void adjust_managed_page_count(struct page *page, long count) in adjust_managed_page_count() argument
5942 atomic_long_add(count, &page_zone(page)->managed_pages); in adjust_managed_page_count()
5956 struct page *page = virt_to_page(pos); in free_reserved_area() local
5966 direct_map_addr = page_address(page); in free_reserved_area()
5975 free_reserved_page(page); in free_reserved_area()
5984 void free_reserved_page(struct page *page) in free_reserved_page() argument
5986 clear_page_tag_ref(page); in free_reserved_page()
5987 ClearPageReserved(page); in free_reserved_page()
5988 init_page_count(page); in free_reserved_page()
5989 __free_page(page); in free_reserved_page()
5990 adjust_managed_page_count(page, 1); in free_reserved_page()
6515 struct page *page; in alloc_contig_dump_pages() local
6518 list_for_each_entry(page, page_list, lru) in alloc_contig_dump_pages()
6519 dump_page(page, "migration failure"); in alloc_contig_dump_pages()
6541 struct page *page; in __alloc_contig_migrate_range() local
6572 list_for_each_entry(page, &cc->migratepages, lru) { in __alloc_contig_migrate_range()
6573 struct folio *folio = page_folio(page); in __alloc_contig_migrate_range()
6613 struct page *page, *next; in split_free_pages() local
6616 list_for_each_entry_safe(page, next, &list[order], lru) { in split_free_pages()
6619 post_alloc_hook(page, order, gfp_mask); in split_free_pages()
6620 set_page_refcounted(page); in split_free_pages()
6624 split_page(page, order); in split_free_pages()
6627 list_del(&page->lru); in split_free_pages()
6629 list_add_tail(&page[i].lru, &list[0]); in split_free_pages()
6804 struct page *head = pfn_to_page(start); in alloc_contig_range_noprof()
6834 struct page *page; in pfn_range_valid_contig() local
6837 page = pfn_to_online_page(i); in pfn_range_valid_contig()
6838 if (!page) in pfn_range_valid_contig()
6841 if (page_zone(page) != z) in pfn_range_valid_contig()
6844 if (PageReserved(page)) in pfn_range_valid_contig()
6847 if (PageHuge(page)) in pfn_range_valid_contig()
6884 struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask, in alloc_contig_pages_noprof()
6939 struct page *page = pfn_to_page(pfn); in free_contig_range() local
6941 count += page_count(page) != 1; in free_contig_range()
6942 __free_page(page); in free_contig_range()
7003 struct page *page; in __offline_isolated_pages() local
7011 page = pfn_to_page(pfn); in __offline_isolated_pages()
7016 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { in __offline_isolated_pages()
7024 if (PageOffline(page)) { in __offline_isolated_pages()
7025 BUG_ON(page_count(page)); in __offline_isolated_pages()
7026 BUG_ON(PageBuddy(page)); in __offline_isolated_pages()
7032 BUG_ON(page_count(page)); in __offline_isolated_pages()
7033 BUG_ON(!PageBuddy(page)); in __offline_isolated_pages()
7034 VM_WARN_ON(get_pageblock_migratetype(page) != MIGRATE_ISOLATE); in __offline_isolated_pages()
7035 order = buddy_order(page); in __offline_isolated_pages()
7036 del_page_from_free_list(page, zone, order, MIGRATE_ISOLATE); in __offline_isolated_pages()
7048 bool is_free_buddy_page(const struct page *page) in is_free_buddy_page() argument
7050 unsigned long pfn = page_to_pfn(page); in is_free_buddy_page()
7054 const struct page *head = page - (pfn & ((1 << order) - 1)); in is_free_buddy_page()
7066 static inline void add_to_free_list(struct page *page, struct zone *zone, in add_to_free_list() argument
7070 __add_to_free_list(page, zone, order, migratetype, tail); in add_to_free_list()
7078 static void break_down_buddy_pages(struct zone *zone, struct page *page, in break_down_buddy_pages() argument
7079 struct page *target, int low, int high, in break_down_buddy_pages()
7083 struct page *current_buddy; in break_down_buddy_pages()
7089 if (target >= &page[size]) { in break_down_buddy_pages()
7090 current_buddy = page; in break_down_buddy_pages()
7091 page = page + size; in break_down_buddy_pages()
7093 current_buddy = page + size; in break_down_buddy_pages()
7107 bool take_page_off_buddy(struct page *page) in take_page_off_buddy() argument
7109 struct zone *zone = page_zone(page); in take_page_off_buddy()
7110 unsigned long pfn = page_to_pfn(page); in take_page_off_buddy()
7117 struct page *page_head = page - (pfn & ((1 << order) - 1)); in take_page_off_buddy()
7127 break_down_buddy_pages(zone, page_head, page, 0, in take_page_off_buddy()
7129 SetPageHWPoisonTakenOff(page); in take_page_off_buddy()
7143 bool put_page_back_buddy(struct page *page) in put_page_back_buddy() argument
7145 struct zone *zone = page_zone(page); in put_page_back_buddy()
7150 if (put_page_testzero(page)) { in put_page_back_buddy()
7151 unsigned long pfn = page_to_pfn(page); in put_page_back_buddy()
7152 int migratetype = get_pfnblock_migratetype(page, pfn); in put_page_back_buddy()
7154 ClearPageHWPoisonTakenOff(page); in put_page_back_buddy()
7155 __free_one_page(page, pfn, zone, 0, migratetype, FPI_NONE); in put_page_back_buddy()
7156 if (TestClearPageHWPoison(page)) { in put_page_back_buddy()
7199 static bool page_contains_unaccepted(struct page *page, unsigned int order) in page_contains_unaccepted() argument
7201 phys_addr_t start = page_to_phys(page); in page_contains_unaccepted()
7207 struct page *page) in __accept_page() argument
7209 list_del(&page->lru); in __accept_page()
7212 __ClearPageUnaccepted(page); in __accept_page()
7215 accept_memory(page_to_phys(page), PAGE_SIZE << MAX_PAGE_ORDER); in __accept_page()
7217 __free_pages_ok(page, MAX_PAGE_ORDER, FPI_TO_TAIL); in __accept_page()
7220 void accept_page(struct page *page) in accept_page() argument
7222 struct zone *zone = page_zone(page); in accept_page()
7226 if (!PageUnaccepted(page)) { in accept_page()
7232 __accept_page(zone, &flags, page); in accept_page()
7238 struct page *page; in try_to_accept_memory_one() local
7241 page = list_first_entry_or_null(&zone->unaccepted_pages, in try_to_accept_memory_one()
7242 struct page, lru); in try_to_accept_memory_one()
7243 if (!page) { in try_to_accept_memory_one()
7249 __accept_page(zone, &flags, page); in try_to_accept_memory_one()
7293 static bool __free_unaccepted(struct page *page) in __free_unaccepted() argument
7295 struct zone *zone = page_zone(page); in __free_unaccepted()
7302 list_add_tail(&page->lru, &zone->unaccepted_pages); in __free_unaccepted()
7305 __SetPageUnaccepted(page); in __free_unaccepted()
7313 static bool page_contains_unaccepted(struct page *page, unsigned int order) in page_contains_unaccepted() argument
7324 static bool __free_unaccepted(struct page *page) in __free_unaccepted() argument
7346 struct page *try_alloc_pages_noprof(int nid, unsigned int order) in try_alloc_pages_noprof()
7372 struct page *page; in try_alloc_pages_noprof() local
7404 page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac); in try_alloc_pages_noprof()
7408 if (page) in try_alloc_pages_noprof()
7409 set_page_refcounted(page); in try_alloc_pages_noprof()
7411 if (memcg_kmem_online() && page && in try_alloc_pages_noprof()
7412 unlikely(__memcg_kmem_charge_page(page, alloc_gfp, order) != 0)) { in try_alloc_pages_noprof()
7413 free_pages_nolock(page, order); in try_alloc_pages_noprof()
7414 page = NULL; in try_alloc_pages_noprof()
7416 trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype); in try_alloc_pages_noprof()
7417 kmsan_alloc_page(page, order, alloc_gfp); in try_alloc_pages_noprof()
7418 return page; in try_alloc_pages_noprof()