1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Page fragment allocator 3 * 4 * Page Fragment: 5 * An arbitrary-length arbitrary-offset area of memory which resides within a 6 * 0 or higher order page. Multiple fragments within that page are 7 * individually refcounted, in the page's reference counter. 8 * 9 * The page_frag functions provide a simple allocation framework for page 10 * fragments. This is used by the network stack and network device drivers to 11 * provide a backing region of memory for use as either an sk_buff->head, or to 12 * be used in the "frags" portion of skb_shared_info. 13 */ 14 15 #include <linux/export.h> 16 #include <linux/gfp_types.h> 17 #include <linux/init.h> 18 #include <linux/mm.h> 19 #include <linux/page_frag_cache.h> 20 #include "internal.h" 21 22 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, 23 gfp_t gfp_mask) 24 { 25 struct page *page = NULL; 26 gfp_t gfp = gfp_mask; 27 28 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 29 gfp_mask = (gfp_mask & ~__GFP_DIRECT_RECLAIM) | __GFP_COMP | 30 __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC; 31 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, 32 PAGE_FRAG_CACHE_MAX_ORDER); 33 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE; 34 #endif 35 if (unlikely(!page)) 36 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); 37 38 nc->va = page ? page_address(page) : NULL; 39 40 return page; 41 } 42 43 void page_frag_cache_drain(struct page_frag_cache *nc) 44 { 45 if (!nc->va) 46 return; 47 48 __page_frag_cache_drain(virt_to_head_page(nc->va), nc->pagecnt_bias); 49 nc->va = NULL; 50 } 51 EXPORT_SYMBOL(page_frag_cache_drain); 52 53 void __page_frag_cache_drain(struct page *page, unsigned int count) 54 { 55 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); 56 57 if (page_ref_sub_and_test(page, count)) 58 free_unref_page(page, compound_order(page)); 59 } 60 EXPORT_SYMBOL(__page_frag_cache_drain); 61 62 void *__page_frag_alloc_align(struct page_frag_cache *nc, 63 unsigned int fragsz, gfp_t gfp_mask, 64 unsigned int align_mask) 65 { 66 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 67 unsigned int size = nc->size; 68 #else 69 unsigned int size = PAGE_SIZE; 70 #endif 71 unsigned int offset; 72 struct page *page; 73 74 if (unlikely(!nc->va)) { 75 refill: 76 page = __page_frag_cache_refill(nc, gfp_mask); 77 if (!page) 78 return NULL; 79 80 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) 81 /* if size can vary use size else just use PAGE_SIZE */ 82 size = nc->size; 83 #endif 84 /* Even if we own the page, we do not use atomic_set(). 85 * This would break get_page_unless_zero() users. 86 */ 87 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE); 88 89 /* reset page count bias and offset to start of new frag */ 90 nc->pfmemalloc = page_is_pfmemalloc(page); 91 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 92 nc->offset = 0; 93 } 94 95 offset = __ALIGN_KERNEL_MASK(nc->offset, ~align_mask); 96 if (unlikely(offset + fragsz > size)) { 97 if (unlikely(fragsz > PAGE_SIZE)) { 98 /* 99 * The caller is trying to allocate a fragment 100 * with fragsz > PAGE_SIZE but the cache isn't big 101 * enough to satisfy the request, this may 102 * happen in low memory conditions. 103 * We don't release the cache page because 104 * it could make memory pressure worse 105 * so we simply return NULL here. 106 */ 107 return NULL; 108 } 109 110 page = virt_to_page(nc->va); 111 112 if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) 113 goto refill; 114 115 if (unlikely(nc->pfmemalloc)) { 116 free_unref_page(page, compound_order(page)); 117 goto refill; 118 } 119 120 /* OK, page count is 0, we can safely set it */ 121 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1); 122 123 /* reset page count bias and offset to start of new frag */ 124 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; 125 offset = 0; 126 } 127 128 nc->pagecnt_bias--; 129 nc->offset = offset + fragsz; 130 131 return nc->va + offset; 132 } 133 EXPORT_SYMBOL(__page_frag_alloc_align); 134 135 /* 136 * Frees a page fragment allocated out of either a compound or order 0 page. 137 */ 138 void page_frag_free(void *addr) 139 { 140 struct page *page = virt_to_head_page(addr); 141 142 if (unlikely(put_page_testzero(page))) 143 free_unref_page(page, compound_order(page)); 144 } 145 EXPORT_SYMBOL(page_frag_free); 146