1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * page allocation tagging 4 */ 5 #ifndef _LINUX_PGALLOC_TAG_H 6 #define _LINUX_PGALLOC_TAG_H 7 8 #include <linux/alloc_tag.h> 9 10 #ifdef CONFIG_MEM_ALLOC_PROFILING 11 12 #include <linux/page_ext.h> 13 14 union pgtag_ref_handle { 15 union codetag_ref *ref; /* reference in page extension */ 16 }; 17 18 extern struct page_ext_operations page_alloc_tagging_ops; 19 20 /* Should be called only if mem_alloc_profiling_enabled() */ 21 static inline bool get_page_tag_ref(struct page *page, union codetag_ref *ref, 22 union pgtag_ref_handle *handle) 23 { 24 struct page_ext *page_ext; 25 union codetag_ref *tmp; 26 27 if (!page) 28 return false; 29 30 page_ext = page_ext_get(page); 31 if (!page_ext) 32 return false; 33 34 tmp = (union codetag_ref *)page_ext_data(page_ext, &page_alloc_tagging_ops); 35 ref->ct = tmp->ct; 36 handle->ref = tmp; 37 return true; 38 } 39 40 static inline void put_page_tag_ref(union pgtag_ref_handle handle) 41 { 42 if (WARN_ON(!handle.ref)) 43 return; 44 45 page_ext_put((void *)handle.ref - page_alloc_tagging_ops.offset); 46 } 47 48 static inline void update_page_tag_ref(union pgtag_ref_handle handle, 49 union codetag_ref *ref) 50 { 51 if (WARN_ON(!handle.ref || !ref)) 52 return; 53 54 handle.ref->ct = ref->ct; 55 } 56 57 static inline void clear_page_tag_ref(struct page *page) 58 { 59 if (mem_alloc_profiling_enabled()) { 60 union pgtag_ref_handle handle; 61 union codetag_ref ref; 62 63 if (get_page_tag_ref(page, &ref, &handle)) { 64 set_codetag_empty(&ref); 65 update_page_tag_ref(handle, &ref); 66 put_page_tag_ref(handle); 67 } 68 } 69 } 70 71 static inline void pgalloc_tag_add(struct page *page, struct task_struct *task, 72 unsigned int nr) 73 { 74 if (mem_alloc_profiling_enabled()) { 75 union pgtag_ref_handle handle; 76 union codetag_ref ref; 77 78 if (get_page_tag_ref(page, &ref, &handle)) { 79 alloc_tag_add(&ref, task->alloc_tag, PAGE_SIZE * nr); 80 update_page_tag_ref(handle, &ref); 81 put_page_tag_ref(handle); 82 } 83 } 84 } 85 86 static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) 87 { 88 if (mem_alloc_profiling_enabled()) { 89 union pgtag_ref_handle handle; 90 union codetag_ref ref; 91 92 if (get_page_tag_ref(page, &ref, &handle)) { 93 alloc_tag_sub(&ref, PAGE_SIZE * nr); 94 update_page_tag_ref(handle, &ref); 95 put_page_tag_ref(handle); 96 } 97 } 98 } 99 100 static inline struct alloc_tag *pgalloc_tag_get(struct page *page) 101 { 102 struct alloc_tag *tag = NULL; 103 104 if (mem_alloc_profiling_enabled()) { 105 union pgtag_ref_handle handle; 106 union codetag_ref ref; 107 108 if (get_page_tag_ref(page, &ref, &handle)) { 109 alloc_tag_sub_check(&ref); 110 if (ref.ct) 111 tag = ct_to_alloc_tag(ref.ct); 112 put_page_tag_ref(handle); 113 } 114 } 115 116 return tag; 117 } 118 119 static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) 120 { 121 if (mem_alloc_profiling_enabled() && tag) 122 this_cpu_sub(tag->counters->bytes, PAGE_SIZE * nr); 123 } 124 125 #else /* CONFIG_MEM_ALLOC_PROFILING */ 126 127 static inline void clear_page_tag_ref(struct page *page) {} 128 static inline void pgalloc_tag_add(struct page *page, struct task_struct *task, 129 unsigned int nr) {} 130 static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {} 131 static inline struct alloc_tag *pgalloc_tag_get(struct page *page) { return NULL; } 132 static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) {} 133 134 #endif /* CONFIG_MEM_ALLOC_PROFILING */ 135 136 #endif /* _LINUX_PGALLOC_TAG_H */ 137