1 /* 2 * Copyright IBM Corporation, 2012 3 * Author Aneesh Kumar K.V <[email protected]> 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms of version 2.1 of the GNU Lesser General Public License 7 * as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it would be useful, but 10 * WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 * 13 */ 14 15 #ifndef _LINUX_HUGETLB_CGROUP_H 16 #define _LINUX_HUGETLB_CGROUP_H 17 18 #include <linux/mmdebug.h> 19 20 struct hugetlb_cgroup; 21 struct resv_map; 22 struct file_region; 23 24 #ifdef CONFIG_CGROUP_HUGETLB 25 /* 26 * Minimum page order trackable by hugetlb cgroup. 27 * At least 4 pages are necessary for all the tracking information. 28 * The second tail page (hpage[SUBPAGE_INDEX_CGROUP]) is the fault 29 * usage cgroup. The third tail page (hpage[SUBPAGE_INDEX_CGROUP_RSVD]) 30 * is the reservation usage cgroup. 31 */ 32 #define HUGETLB_CGROUP_MIN_ORDER order_base_2(__MAX_CGROUP_SUBPAGE_INDEX + 1) 33 34 enum hugetlb_memory_event { 35 HUGETLB_MAX, 36 HUGETLB_NR_MEMORY_EVENTS, 37 }; 38 39 struct hugetlb_cgroup_per_node { 40 /* hugetlb usage in pages over all hstates. */ 41 unsigned long usage[HUGE_MAX_HSTATE]; 42 }; 43 44 struct hugetlb_cgroup { 45 struct cgroup_subsys_state css; 46 47 /* 48 * the counter to account for hugepages from hugetlb. 49 */ 50 struct page_counter hugepage[HUGE_MAX_HSTATE]; 51 52 /* 53 * the counter to account for hugepage reservations from hugetlb. 54 */ 55 struct page_counter rsvd_hugepage[HUGE_MAX_HSTATE]; 56 57 atomic_long_t events[HUGE_MAX_HSTATE][HUGETLB_NR_MEMORY_EVENTS]; 58 atomic_long_t events_local[HUGE_MAX_HSTATE][HUGETLB_NR_MEMORY_EVENTS]; 59 60 /* Handle for "hugetlb.events" */ 61 struct cgroup_file events_file[HUGE_MAX_HSTATE]; 62 63 /* Handle for "hugetlb.events.local" */ 64 struct cgroup_file events_local_file[HUGE_MAX_HSTATE]; 65 66 struct hugetlb_cgroup_per_node *nodeinfo[]; 67 }; 68 69 static inline struct hugetlb_cgroup * 70 __hugetlb_cgroup_from_folio(struct folio *folio, bool rsvd) 71 { 72 struct page *tail; 73 74 VM_BUG_ON_FOLIO(!folio_test_hugetlb(folio), folio); 75 if (folio_order(folio) < HUGETLB_CGROUP_MIN_ORDER) 76 return NULL; 77 78 if (rsvd) { 79 tail = folio_page(folio, SUBPAGE_INDEX_CGROUP_RSVD); 80 return (void *)page_private(tail); 81 } 82 83 else { 84 tail = folio_page(folio, SUBPAGE_INDEX_CGROUP); 85 return (void *)page_private(tail); 86 } 87 } 88 89 static inline struct hugetlb_cgroup *hugetlb_cgroup_from_folio(struct folio *folio) 90 { 91 return __hugetlb_cgroup_from_folio(folio, false); 92 } 93 94 static inline struct hugetlb_cgroup * 95 hugetlb_cgroup_from_folio_rsvd(struct folio *folio) 96 { 97 return __hugetlb_cgroup_from_folio(folio, true); 98 } 99 100 static inline void __set_hugetlb_cgroup(struct folio *folio, 101 struct hugetlb_cgroup *h_cg, bool rsvd) 102 { 103 VM_BUG_ON_FOLIO(!folio_test_hugetlb(folio), folio); 104 105 if (folio_order(folio) < HUGETLB_CGROUP_MIN_ORDER) 106 return; 107 if (rsvd) 108 set_page_private(folio_page(folio, SUBPAGE_INDEX_CGROUP_RSVD), 109 (unsigned long)h_cg); 110 else 111 set_page_private(folio_page(folio, SUBPAGE_INDEX_CGROUP), 112 (unsigned long)h_cg); 113 } 114 115 static inline void set_hugetlb_cgroup(struct folio *folio, 116 struct hugetlb_cgroup *h_cg) 117 { 118 __set_hugetlb_cgroup(folio, h_cg, false); 119 } 120 121 static inline void set_hugetlb_cgroup_rsvd(struct folio *folio, 122 struct hugetlb_cgroup *h_cg) 123 { 124 __set_hugetlb_cgroup(folio, h_cg, true); 125 } 126 127 static inline bool hugetlb_cgroup_disabled(void) 128 { 129 return !cgroup_subsys_enabled(hugetlb_cgrp_subsys); 130 } 131 132 static inline void hugetlb_cgroup_put_rsvd_cgroup(struct hugetlb_cgroup *h_cg) 133 { 134 css_put(&h_cg->css); 135 } 136 137 static inline void resv_map_dup_hugetlb_cgroup_uncharge_info( 138 struct resv_map *resv_map) 139 { 140 if (resv_map->css) 141 css_get(resv_map->css); 142 } 143 144 static inline void resv_map_put_hugetlb_cgroup_uncharge_info( 145 struct resv_map *resv_map) 146 { 147 if (resv_map->css) 148 css_put(resv_map->css); 149 } 150 151 extern int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, 152 struct hugetlb_cgroup **ptr); 153 extern int hugetlb_cgroup_charge_cgroup_rsvd(int idx, unsigned long nr_pages, 154 struct hugetlb_cgroup **ptr); 155 extern void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages, 156 struct hugetlb_cgroup *h_cg, 157 struct page *page); 158 extern void hugetlb_cgroup_commit_charge_rsvd(int idx, unsigned long nr_pages, 159 struct hugetlb_cgroup *h_cg, 160 struct page *page); 161 extern void hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages, 162 struct page *page); 163 extern void hugetlb_cgroup_uncharge_page_rsvd(int idx, unsigned long nr_pages, 164 struct page *page); 165 166 extern void hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages, 167 struct hugetlb_cgroup *h_cg); 168 extern void hugetlb_cgroup_uncharge_cgroup_rsvd(int idx, unsigned long nr_pages, 169 struct hugetlb_cgroup *h_cg); 170 extern void hugetlb_cgroup_uncharge_counter(struct resv_map *resv, 171 unsigned long start, 172 unsigned long end); 173 174 extern void hugetlb_cgroup_uncharge_file_region(struct resv_map *resv, 175 struct file_region *rg, 176 unsigned long nr_pages, 177 bool region_del); 178 179 extern void hugetlb_cgroup_file_init(void) __init; 180 extern void hugetlb_cgroup_migrate(struct folio *old_folio, 181 struct folio *new_folio); 182 183 #else 184 static inline void hugetlb_cgroup_uncharge_file_region(struct resv_map *resv, 185 struct file_region *rg, 186 unsigned long nr_pages, 187 bool region_del) 188 { 189 } 190 191 static inline struct hugetlb_cgroup *hugetlb_cgroup_from_folio(struct folio *folio) 192 { 193 return NULL; 194 } 195 196 static inline struct hugetlb_cgroup * 197 hugetlb_cgroup_from_folio_rsvd(struct folio *folio) 198 { 199 return NULL; 200 } 201 202 static inline void set_hugetlb_cgroup(struct folio *folio, 203 struct hugetlb_cgroup *h_cg) 204 { 205 } 206 207 static inline void set_hugetlb_cgroup_rsvd(struct folio *folio, 208 struct hugetlb_cgroup *h_cg) 209 { 210 } 211 212 static inline bool hugetlb_cgroup_disabled(void) 213 { 214 return true; 215 } 216 217 static inline void hugetlb_cgroup_put_rsvd_cgroup(struct hugetlb_cgroup *h_cg) 218 { 219 } 220 221 static inline void resv_map_dup_hugetlb_cgroup_uncharge_info( 222 struct resv_map *resv_map) 223 { 224 } 225 226 static inline void resv_map_put_hugetlb_cgroup_uncharge_info( 227 struct resv_map *resv_map) 228 { 229 } 230 231 static inline int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, 232 struct hugetlb_cgroup **ptr) 233 { 234 return 0; 235 } 236 237 static inline int hugetlb_cgroup_charge_cgroup_rsvd(int idx, 238 unsigned long nr_pages, 239 struct hugetlb_cgroup **ptr) 240 { 241 return 0; 242 } 243 244 static inline void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages, 245 struct hugetlb_cgroup *h_cg, 246 struct page *page) 247 { 248 } 249 250 static inline void 251 hugetlb_cgroup_commit_charge_rsvd(int idx, unsigned long nr_pages, 252 struct hugetlb_cgroup *h_cg, 253 struct page *page) 254 { 255 } 256 257 static inline void hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages, 258 struct page *page) 259 { 260 } 261 262 static inline void hugetlb_cgroup_uncharge_page_rsvd(int idx, 263 unsigned long nr_pages, 264 struct page *page) 265 { 266 } 267 static inline void hugetlb_cgroup_uncharge_cgroup(int idx, 268 unsigned long nr_pages, 269 struct hugetlb_cgroup *h_cg) 270 { 271 } 272 273 static inline void 274 hugetlb_cgroup_uncharge_cgroup_rsvd(int idx, unsigned long nr_pages, 275 struct hugetlb_cgroup *h_cg) 276 { 277 } 278 279 static inline void hugetlb_cgroup_uncharge_counter(struct resv_map *resv, 280 unsigned long start, 281 unsigned long end) 282 { 283 } 284 285 static inline void hugetlb_cgroup_file_init(void) 286 { 287 } 288 289 static inline void hugetlb_cgroup_migrate(struct folio *old_folio, 290 struct folio *new_folio) 291 { 292 } 293 294 #endif /* CONFIG_MEM_RES_CTLR_HUGETLB */ 295 #endif 296