1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Page fragment allocator 3 * 4 * Page Fragment: 5 * An arbitrary-length arbitrary-offset area of memory which resides within a 6 * 0 or higher order page. Multiple fragments within that page are 7 * individually refcounted, in the page's reference counter. 8 * 9 * The page_frag functions provide a simple allocation framework for page 10 * fragments. This is used by the network stack and network device drivers to 11 * provide a backing region of memory for use as either an sk_buff->head, or to 12 * be used in the "frags" portion of skb_shared_info. 13 */ 14 15 #include <linux/export.h> 16 #include <linux/gfp_types.h> 17 #include <linux/init.h> 18 #include <linux/mm.h> 19 #include <linux/page_frag_cache.h> 20 #include "internal.h" 21 22 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, 23 gfp_t gfp_mask) 24 { 25 struct page *page = NULL; 26 gfp_t gfp = gfp_mask; 27 28 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 29 gfp_mask = (gfp_mask & ~__GFP_DIRECT_RECLAIM) | __GFP_COMP | 30 __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC; 31 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, 32 PAGE_FRAG_CACHE_MAX_ORDER); 33 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE; 34 #endif 35 if (unlikely(!page)) 36 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 37 38 nc->va = page ? page_address(page) : NULL; 39 40 return page; 41 } 42 43 void page_frag_cache_drain(struct page_frag_cache *nc) 44 { 45 if (!nc->va) 46 return; 47 48 __page_frag_cache_drain(virt_to_head_page(nc->va), nc->pagecnt_bias); 49 nc->va = NULL; 50 } 51 EXPORT_SYMBOL(page_frag_cache_drain); 52 53 void __page_frag_cache_drain(struct page *page, unsigned int count) 54 { 55 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 56 57 if (page_ref_sub_and_test(page, count)) 58 free_unref_page(page, compound_order(page)); 59 } 60 EXPORT_SYMBOL(__page_frag_cache_drain); 61 62 void *__page_frag_alloc_align(struct page_frag_cache *nc, 63 unsigned int fragsz, gfp_t gfp_mask, 64 unsigned int align_mask) 65 { 66 unsigned int size = PAGE_SIZE; 67 struct page *page; 68 int offset; 69 70 if (unlikely(!nc->va)) { 71 refill: 72 page = __page_frag_cache_refill(nc, gfp_mask); 73 if (!page) 74 return NULL; 75 76 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 77 /* if size can vary use size else just use PAGE_SIZE */ 78 size = nc->size; 79 #endif 80 /* Even if we own the page, we do not use atomic_set(). 81 * This would break get_page_unless_zero() users. 82 */ 83 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE); 84 85 /* reset page count bias and offset to start of new frag */ 86 nc->pfmemalloc = page_is_pfmemalloc(page); 87 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 88 nc->offset = size; 89 } 90 91 offset = nc->offset - fragsz; 92 if (unlikely(offset < 0)) { 93 page = virt_to_page(nc->va); 94 95 if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) 96 goto refill; 97 98 if (unlikely(nc->pfmemalloc)) { 99 free_unref_page(page, compound_order(page)); 100 goto refill; 101 } 102 103 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 104 /* if size can vary use size else just use PAGE_SIZE */ 105 size = nc->size; 106 #endif 107 /* OK, page count is 0, we can safely set it */ 108 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1); 109 110 /* reset page count bias and offset to start of new frag */ 111 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 112 offset = size - fragsz; 113 if (unlikely(offset < 0)) { 114 /* 115 * The caller is trying to allocate a fragment 116 * with fragsz > PAGE_SIZE but the cache isn't big 117 * enough to satisfy the request, this may 118 * happen in low memory conditions. 119 * We don't release the cache page because 120 * it could make memory pressure worse 121 * so we simply return NULL here. 122 */ 123 return NULL; 124 } 125 } 126 127 nc->pagecnt_bias--; 128 offset &= align_mask; 129 nc->offset = offset; 130 131 return nc->va + offset; 132 } 133 EXPORT_SYMBOL(__page_frag_alloc_align); 134 135 /* 136 * Frees a page fragment allocated out of either a compound or order 0 page. 137 */ 138 void page_frag_free(void *addr) 139 { 140 struct page *page = virt_to_head_page(addr); 141 142 if (unlikely(put_page_testzero(page))) 143 free_unref_page(page, compound_order(page)); 144 } 145 EXPORT_SYMBOL(page_frag_free); 146