Lines Matching refs:gfp_mask

3426 				unsigned int alloc_flags, gfp_t gfp_mask)  in zone_watermark_fast()  argument
3493 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) in alloc_flags_nofragment() argument
3501 alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM); in alloc_flags_nofragment()
3530 static inline unsigned int gfp_to_alloc_flags_cma(gfp_t gfp_mask, in gfp_to_alloc_flags_cma() argument
3534 if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) in gfp_to_alloc_flags_cma()
3545 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, in get_page_from_freelist() argument
3568 !__cpuset_zone_allowed(zone, gfp_mask)) in get_page_from_freelist()
3630 gfp_mask)) in get_page_from_freelist()
3639 gfp_mask)) { in get_page_from_freelist()
3662 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); in get_page_from_freelist()
3682 gfp_mask, alloc_flags, ac->migratetype); in get_page_from_freelist()
3684 prep_new_page(page, order, gfp_mask, alloc_flags); in get_page_from_freelist()
3718 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) in warn_alloc_show_mem() argument
3727 if (!(gfp_mask & __GFP_NOMEMALLOC)) in warn_alloc_show_mem()
3731 if (!in_task() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) in warn_alloc_show_mem()
3734 __show_mem(filter, nodemask, gfp_zone(gfp_mask)); in warn_alloc_show_mem()
3737 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) in warn_alloc() argument
3743 if ((gfp_mask & __GFP_NOWARN) || in warn_alloc()
3745 ((gfp_mask & __GFP_DMA) && !has_managed_dma())) in warn_alloc()
3752 current->comm, &vaf, gfp_mask, &gfp_mask, in warn_alloc()
3759 warn_alloc_show_mem(gfp_mask, nodemask); in warn_alloc()
3763 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order, in __alloc_pages_cpuset_fallback() argument
3769 page = get_page_from_freelist(gfp_mask, order, in __alloc_pages_cpuset_fallback()
3776 page = get_page_from_freelist(gfp_mask, order, in __alloc_pages_cpuset_fallback()
3782 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, in __alloc_pages_may_oom() argument
3789 .gfp_mask = gfp_mask, in __alloc_pages_may_oom()
3813 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & in __alloc_pages_may_oom()
3833 if (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_THISNODE)) in __alloc_pages_may_oom()
3852 WARN_ON_ONCE_GFP(gfp_mask & __GFP_NOFAIL, gfp_mask)) { in __alloc_pages_may_oom()
3859 if (gfp_mask & __GFP_NOFAIL) in __alloc_pages_may_oom()
3860 page = __alloc_pages_cpuset_fallback(gfp_mask, order, in __alloc_pages_may_oom()
3877 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, in __alloc_pages_direct_compact() argument
3892 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, in __alloc_pages_direct_compact()
3909 prep_new_page(page, order, gfp_mask, alloc_flags); in __alloc_pages_direct_compact()
3913 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); in __alloc_pages_direct_compact()
4002 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, in __alloc_pages_direct_compact() argument
4042 static bool __need_reclaim(gfp_t gfp_mask) in __need_reclaim() argument
4045 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) in __need_reclaim()
4052 if (gfp_mask & __GFP_NOLOCKDEP) in __need_reclaim()
4068 void fs_reclaim_acquire(gfp_t gfp_mask) in fs_reclaim_acquire() argument
4070 gfp_mask = current_gfp_context(gfp_mask); in fs_reclaim_acquire()
4072 if (__need_reclaim(gfp_mask)) { in fs_reclaim_acquire()
4073 if (gfp_mask & __GFP_FS) in fs_reclaim_acquire()
4085 void fs_reclaim_release(gfp_t gfp_mask) in fs_reclaim_release() argument
4087 gfp_mask = current_gfp_context(gfp_mask); in fs_reclaim_release()
4089 if (__need_reclaim(gfp_mask)) { in fs_reclaim_release()
4090 if (gfp_mask & __GFP_FS) in fs_reclaim_release()
4123 __perform_reclaim(gfp_t gfp_mask, unsigned int order, in __perform_reclaim() argument
4133 fs_reclaim_acquire(gfp_mask); in __perform_reclaim()
4136 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, in __perform_reclaim()
4140 fs_reclaim_release(gfp_mask); in __perform_reclaim()
4149 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, in __alloc_pages_direct_reclaim() argument
4158 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); in __alloc_pages_direct_reclaim()
4163 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); in __alloc_pages_direct_reclaim()
4182 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, in wake_all_kswapds() argument
4202 wakeup_kswapd(zone, gfp_mask, reclaim_order, highest_zoneidx); in wake_all_kswapds()
4208 gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) in gfp_to_alloc_flags() argument
4227 (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM)); in gfp_to_alloc_flags()
4229 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) { in gfp_to_alloc_flags()
4234 if (!(gfp_mask & __GFP_NOMEMALLOC)) { in gfp_to_alloc_flags()
4251 alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, alloc_flags); in gfp_to_alloc_flags()
4278 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask) in __gfp_pfmemalloc_flags() argument
4280 if (unlikely(gfp_mask & __GFP_NOMEMALLOC)) in __gfp_pfmemalloc_flags()
4282 if (gfp_mask & __GFP_MEMALLOC) in __gfp_pfmemalloc_flags()
4296 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) in gfp_pfmemalloc_allowed() argument
4298 return !!__gfp_pfmemalloc_flags(gfp_mask); in gfp_pfmemalloc_allowed()
4312 should_reclaim_retry(gfp_t gfp_mask, unsigned order, in should_reclaim_retry() argument
4349 !__cpuset_zone_allowed(zone, gfp_mask)) in should_reclaim_retry()
4422 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, in __alloc_pages_slowpath() argument
4425 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; in __alloc_pages_slowpath()
4426 bool can_compact = gfp_compaction_allowed(gfp_mask); in __alloc_pages_slowpath()
4427 bool nofail = gfp_mask & __GFP_NOFAIL; in __alloc_pages_slowpath()
4472 alloc_flags = gfp_to_alloc_flags(gfp_mask, order); in __alloc_pages_slowpath()
4490 if (cpusets_insane_config() && (gfp_mask & __GFP_HARDWALL)) { in __alloc_pages_slowpath()
4499 wake_all_kswapds(order, gfp_mask, ac); in __alloc_pages_slowpath()
4505 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); in __alloc_pages_slowpath()
4521 && !gfp_pfmemalloc_allowed(gfp_mask)) { in __alloc_pages_slowpath()
4522 page = __alloc_pages_direct_compact(gfp_mask, order, in __alloc_pages_slowpath()
4533 if (costly_order && (gfp_mask & __GFP_NORETRY)) { in __alloc_pages_slowpath()
4575 wake_all_kswapds(order, gfp_mask, ac); in __alloc_pages_slowpath()
4577 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); in __alloc_pages_slowpath()
4579 alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, reserve_flags) | in __alloc_pages_slowpath()
4594 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); in __alloc_pages_slowpath()
4607 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, in __alloc_pages_slowpath()
4613 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, in __alloc_pages_slowpath()
4619 if (gfp_mask & __GFP_NORETRY) in __alloc_pages_slowpath()
4627 !(gfp_mask & __GFP_RETRY_MAYFAIL))) in __alloc_pages_slowpath()
4630 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, in __alloc_pages_slowpath()
4661 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); in __alloc_pages_slowpath()
4668 (gfp_mask & __GFP_NOMEMALLOC))) in __alloc_pages_slowpath()
4706 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_MIN_RESERVE, ac); in __alloc_pages_slowpath()
4714 warn_alloc(gfp_mask, ac->nodemask, in __alloc_pages_slowpath()
4720 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, in prepare_alloc_pages() argument
4725 ac->highest_zoneidx = gfp_zone(gfp_mask); in prepare_alloc_pages()
4726 ac->zonelist = node_zonelist(preferred_nid, gfp_mask); in prepare_alloc_pages()
4728 ac->migratetype = gfp_migratetype(gfp_mask); in prepare_alloc_pages()
4742 might_alloc(gfp_mask); in prepare_alloc_pages()
4749 should_fail_alloc_page(gfp_mask, order)) in prepare_alloc_pages()
4752 *alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, *alloc_flags); in prepare_alloc_pages()
4755 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); in prepare_alloc_pages()
5029 unsigned long get_free_pages_noprof(gfp_t gfp_mask, unsigned int order) in get_free_pages_noprof() argument
5033 page = alloc_pages_noprof(gfp_mask & ~__GFP_HIGHMEM, order); in get_free_pages_noprof()
5040 unsigned long get_zeroed_page_noprof(gfp_t gfp_mask) in get_zeroed_page_noprof() argument
5042 return get_free_pages_noprof(gfp_mask | __GFP_ZERO, 0); in get_zeroed_page_noprof()
5145 void *alloc_pages_exact_noprof(size_t size, gfp_t gfp_mask) in alloc_pages_exact_noprof() argument
5150 if (WARN_ON_ONCE(gfp_mask & (__GFP_COMP | __GFP_HIGHMEM))) in alloc_pages_exact_noprof()
5151 gfp_mask &= ~(__GFP_COMP | __GFP_HIGHMEM); in alloc_pages_exact_noprof()
5153 addr = get_free_pages_noprof(gfp_mask, order); in alloc_pages_exact_noprof()
5170 void * __meminit alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask) in alloc_pages_exact_nid_noprof() argument
5175 if (WARN_ON_ONCE(gfp_mask & (__GFP_COMP | __GFP_HIGHMEM))) in alloc_pages_exact_nid_noprof()
5176 gfp_mask &= ~(__GFP_COMP | __GFP_HIGHMEM); in alloc_pages_exact_nid_noprof()
5178 p = alloc_pages_node_noprof(nid, gfp_mask, order); in alloc_pages_exact_nid_noprof()
6538 .gfp_mask = cc->gfp_mask, in __alloc_contig_migrate_range()
6596 if (!(cc->gfp_mask & __GFP_NOWARN) && ret == -EBUSY) in __alloc_contig_migrate_range()
6608 static void split_free_pages(struct list_head *list, gfp_t gfp_mask) in split_free_pages() argument
6619 post_alloc_hook(page, order, gfp_mask); in split_free_pages()
6634 static int __alloc_contig_verify_gfp_mask(gfp_t gfp_mask, gfp_t *gfp_cc_mask) in __alloc_contig_verify_gfp_mask() argument
6645 gfp_mask &= ~(GFP_ZONEMASK | __GFP_RECLAIMABLE | __GFP_WRITE | in __alloc_contig_verify_gfp_mask()
6652 if (gfp_mask & ~(reclaim_mask | action_mask)) in __alloc_contig_verify_gfp_mask()
6663 *gfp_cc_mask = (gfp_mask & (reclaim_mask | cc_action_mask)) | in __alloc_contig_verify_gfp_mask()
6692 unsigned migratetype, gfp_t gfp_mask) in alloc_contig_range_noprof() argument
6708 gfp_mask = current_gfp_context(gfp_mask); in alloc_contig_range_noprof()
6709 if (__alloc_contig_verify_gfp_mask(gfp_mask, (gfp_t *)&cc.gfp_mask)) in alloc_contig_range_noprof()
6795 if (!(gfp_mask & __GFP_COMP)) { in alloc_contig_range_noprof()
6796 split_free_pages(cc.freepages, gfp_mask); in alloc_contig_range_noprof()
6808 prep_new_page(head, order, gfp_mask, 0); in alloc_contig_range_noprof()
6822 unsigned long nr_pages, gfp_t gfp_mask) in __alloc_contig_pages() argument
6827 gfp_mask); in __alloc_contig_pages()
6884 struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask, in alloc_contig_pages_noprof() argument
6892 zonelist = node_zonelist(nid, gfp_mask); in alloc_contig_pages_noprof()
6894 gfp_zone(gfp_mask), nodemask) { in alloc_contig_pages_noprof()
6909 gfp_mask); in alloc_contig_pages_noprof()