xref: /linux-6.15/lib/alloc_tag.c (revision 0db6f8d7) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/alloc_tag.h>
3 #include <linux/execmem.h>
4 #include <linux/fs.h>
5 #include <linux/gfp.h>
6 #include <linux/module.h>
7 #include <linux/page_ext.h>
8 #include <linux/proc_fs.h>
9 #include <linux/seq_buf.h>
10 #include <linux/seq_file.h>
11 
12 #define ALLOCINFO_FILE_NAME		"allocinfo"
13 #define MODULE_ALLOC_TAG_VMAP_SIZE	(100000UL * sizeof(struct alloc_tag))
14 
15 #ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
16 static bool mem_profiling_support __meminitdata = true;
17 #else
18 static bool mem_profiling_support __meminitdata;
19 #endif
20 
21 static struct codetag_type *alloc_tag_cttype;
22 
23 DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
24 EXPORT_SYMBOL(_shared_alloc_tag);
25 
26 DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
27 			mem_alloc_profiling_key);
28 
29 struct allocinfo_private {
30 	struct codetag_iterator iter;
31 	bool print_header;
32 };
33 
34 static void *allocinfo_start(struct seq_file *m, loff_t *pos)
35 {
36 	struct allocinfo_private *priv;
37 	struct codetag *ct;
38 	loff_t node = *pos;
39 
40 	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
41 	m->private = priv;
42 	if (!priv)
43 		return NULL;
44 
45 	priv->print_header = (node == 0);
46 	codetag_lock_module_list(alloc_tag_cttype, true);
47 	priv->iter = codetag_get_ct_iter(alloc_tag_cttype);
48 	while ((ct = codetag_next_ct(&priv->iter)) != NULL && node)
49 		node--;
50 
51 	return ct ? priv : NULL;
52 }
53 
54 static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos)
55 {
56 	struct allocinfo_private *priv = (struct allocinfo_private *)arg;
57 	struct codetag *ct = codetag_next_ct(&priv->iter);
58 
59 	(*pos)++;
60 	if (!ct)
61 		return NULL;
62 
63 	return priv;
64 }
65 
66 static void allocinfo_stop(struct seq_file *m, void *arg)
67 {
68 	struct allocinfo_private *priv = (struct allocinfo_private *)m->private;
69 
70 	if (priv) {
71 		codetag_lock_module_list(alloc_tag_cttype, false);
72 		kfree(priv);
73 	}
74 }
75 
76 static void print_allocinfo_header(struct seq_buf *buf)
77 {
78 	/* Output format version, so we can change it. */
79 	seq_buf_printf(buf, "allocinfo - version: 1.0\n");
80 	seq_buf_printf(buf, "#     <size>  <calls> <tag info>\n");
81 }
82 
83 static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct)
84 {
85 	struct alloc_tag *tag = ct_to_alloc_tag(ct);
86 	struct alloc_tag_counters counter = alloc_tag_read(tag);
87 	s64 bytes = counter.bytes;
88 
89 	seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls);
90 	codetag_to_text(out, ct);
91 	seq_buf_putc(out, ' ');
92 	seq_buf_putc(out, '\n');
93 }
94 
95 static int allocinfo_show(struct seq_file *m, void *arg)
96 {
97 	struct allocinfo_private *priv = (struct allocinfo_private *)arg;
98 	char *bufp;
99 	size_t n = seq_get_buf(m, &bufp);
100 	struct seq_buf buf;
101 
102 	seq_buf_init(&buf, bufp, n);
103 	if (priv->print_header) {
104 		print_allocinfo_header(&buf);
105 		priv->print_header = false;
106 	}
107 	alloc_tag_to_text(&buf, priv->iter.ct);
108 	seq_commit(m, seq_buf_used(&buf));
109 	return 0;
110 }
111 
112 static const struct seq_operations allocinfo_seq_op = {
113 	.start	= allocinfo_start,
114 	.next	= allocinfo_next,
115 	.stop	= allocinfo_stop,
116 	.show	= allocinfo_show,
117 };
118 
119 size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep)
120 {
121 	struct codetag_iterator iter;
122 	struct codetag *ct;
123 	struct codetag_bytes n;
124 	unsigned int i, nr = 0;
125 
126 	if (can_sleep)
127 		codetag_lock_module_list(alloc_tag_cttype, true);
128 	else if (!codetag_trylock_module_list(alloc_tag_cttype))
129 		return 0;
130 
131 	iter = codetag_get_ct_iter(alloc_tag_cttype);
132 	while ((ct = codetag_next_ct(&iter))) {
133 		struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct));
134 
135 		n.ct	= ct;
136 		n.bytes = counter.bytes;
137 
138 		for (i = 0; i < nr; i++)
139 			if (n.bytes > tags[i].bytes)
140 				break;
141 
142 		if (i < count) {
143 			nr -= nr == count;
144 			memmove(&tags[i + 1],
145 				&tags[i],
146 				sizeof(tags[0]) * (nr - i));
147 			nr++;
148 			tags[i] = n;
149 		}
150 	}
151 
152 	codetag_lock_module_list(alloc_tag_cttype, false);
153 
154 	return nr;
155 }
156 
157 static void __init shutdown_mem_profiling(void)
158 {
159 	if (mem_alloc_profiling_enabled())
160 		static_branch_disable(&mem_alloc_profiling_key);
161 
162 	if (!mem_profiling_support)
163 		return;
164 
165 	mem_profiling_support = false;
166 }
167 
168 static void __init procfs_init(void)
169 {
170 	if (!mem_profiling_support)
171 		return;
172 
173 	if (!proc_create_seq(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_seq_op)) {
174 		pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME);
175 		shutdown_mem_profiling();
176 	}
177 }
178 
179 #ifdef CONFIG_MODULES
180 
181 static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE);
182 /* A dummy object used to indicate an unloaded module */
183 static struct module unloaded_mod;
184 /* A dummy object used to indicate a module prepended area */
185 static struct module prepend_mod;
186 
187 static struct alloc_tag_module_section module_tags;
188 
189 static bool needs_section_mem(struct module *mod, unsigned long size)
190 {
191 	return size >= sizeof(struct alloc_tag);
192 }
193 
194 static struct alloc_tag *find_used_tag(struct alloc_tag *from, struct alloc_tag *to)
195 {
196 	while (from <= to) {
197 		struct alloc_tag_counters counter;
198 
199 		counter = alloc_tag_read(from);
200 		if (counter.bytes)
201 			return from;
202 		from++;
203 	}
204 
205 	return NULL;
206 }
207 
208 /* Called with mod_area_mt locked */
209 static void clean_unused_module_areas_locked(void)
210 {
211 	MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
212 	struct module *val;
213 
214 	mas_for_each(&mas, val, module_tags.size) {
215 		if (val != &unloaded_mod)
216 			continue;
217 
218 		/* Release area if all tags are unused */
219 		if (!find_used_tag((struct alloc_tag *)(module_tags.start_addr + mas.index),
220 				   (struct alloc_tag *)(module_tags.start_addr + mas.last)))
221 			mas_erase(&mas);
222 	}
223 }
224 
225 /* Called with mod_area_mt locked */
226 static bool find_aligned_area(struct ma_state *mas, unsigned long section_size,
227 			      unsigned long size, unsigned int prepend, unsigned long align)
228 {
229 	bool cleanup_done = false;
230 
231 repeat:
232 	/* Try finding exact size and hope the start is aligned */
233 	if (!mas_empty_area(mas, 0, section_size - 1, prepend + size)) {
234 		if (IS_ALIGNED(mas->index + prepend, align))
235 			return true;
236 
237 		/* Try finding larger area to align later */
238 		mas_reset(mas);
239 		if (!mas_empty_area(mas, 0, section_size - 1,
240 				    size + prepend + align - 1))
241 			return true;
242 	}
243 
244 	/* No free area, try cleanup stale data and repeat the search once */
245 	if (!cleanup_done) {
246 		clean_unused_module_areas_locked();
247 		cleanup_done = true;
248 		mas_reset(mas);
249 		goto repeat;
250 	}
251 
252 	return false;
253 }
254 
255 static void *reserve_module_tags(struct module *mod, unsigned long size,
256 				 unsigned int prepend, unsigned long align)
257 {
258 	unsigned long section_size = module_tags.end_addr - module_tags.start_addr;
259 	MA_STATE(mas, &mod_area_mt, 0, section_size - 1);
260 	unsigned long offset;
261 	void *ret = NULL;
262 
263 	/* If no tags return error */
264 	if (size < sizeof(struct alloc_tag))
265 		return ERR_PTR(-EINVAL);
266 
267 	/*
268 	 * align is always power of 2, so we can use IS_ALIGNED and ALIGN.
269 	 * align 0 or 1 means no alignment, to simplify set to 1.
270 	 */
271 	if (!align)
272 		align = 1;
273 
274 	mas_lock(&mas);
275 	if (!find_aligned_area(&mas, section_size, size, prepend, align)) {
276 		ret = ERR_PTR(-ENOMEM);
277 		goto unlock;
278 	}
279 
280 	/* Mark found area as reserved */
281 	offset = mas.index;
282 	offset += prepend;
283 	offset = ALIGN(offset, align);
284 	if (offset != mas.index) {
285 		unsigned long pad_start = mas.index;
286 
287 		mas.last = offset - 1;
288 		mas_store(&mas, &prepend_mod);
289 		if (mas_is_err(&mas)) {
290 			ret = ERR_PTR(xa_err(mas.node));
291 			goto unlock;
292 		}
293 		mas.index = offset;
294 		mas.last = offset + size - 1;
295 		mas_store(&mas, mod);
296 		if (mas_is_err(&mas)) {
297 			mas.index = pad_start;
298 			mas_erase(&mas);
299 			ret = ERR_PTR(xa_err(mas.node));
300 		}
301 	} else {
302 		mas.last = offset + size - 1;
303 		mas_store(&mas, mod);
304 		if (mas_is_err(&mas))
305 			ret = ERR_PTR(xa_err(mas.node));
306 	}
307 unlock:
308 	mas_unlock(&mas);
309 
310 	if (IS_ERR(ret))
311 		return ret;
312 
313 	if (module_tags.size < offset + size)
314 		module_tags.size = offset + size;
315 
316 	return (struct alloc_tag *)(module_tags.start_addr + offset);
317 }
318 
319 static void release_module_tags(struct module *mod, bool used)
320 {
321 	MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size);
322 	struct alloc_tag *tag;
323 	struct module *val;
324 
325 	mas_lock(&mas);
326 	mas_for_each_rev(&mas, val, 0)
327 		if (val == mod)
328 			break;
329 
330 	if (!val) /* module not found */
331 		goto out;
332 
333 	if (!used)
334 		goto release_area;
335 
336 	/* Find out if the area is used */
337 	tag = find_used_tag((struct alloc_tag *)(module_tags.start_addr + mas.index),
338 			    (struct alloc_tag *)(module_tags.start_addr + mas.last));
339 	if (tag) {
340 		struct alloc_tag_counters counter = alloc_tag_read(tag);
341 
342 		pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n",
343 			tag->ct.filename, tag->ct.lineno, tag->ct.modname,
344 			tag->ct.function, counter.bytes);
345 	} else {
346 		used = false;
347 	}
348 release_area:
349 	mas_store(&mas, used ? &unloaded_mod : NULL);
350 	val = mas_prev_range(&mas, 0);
351 	if (val == &prepend_mod)
352 		mas_store(&mas, NULL);
353 out:
354 	mas_unlock(&mas);
355 }
356 
357 static void replace_module(struct module *mod, struct module *new_mod)
358 {
359 	MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
360 	struct module *val;
361 
362 	mas_lock(&mas);
363 	mas_for_each(&mas, val, module_tags.size) {
364 		if (val != mod)
365 			continue;
366 
367 		mas_store_gfp(&mas, new_mod, GFP_KERNEL);
368 		break;
369 	}
370 	mas_unlock(&mas);
371 }
372 
373 static int __init alloc_mod_tags_mem(void)
374 {
375 	/* Allocate space to copy allocation tags */
376 	module_tags.start_addr = (unsigned long)execmem_alloc(EXECMEM_MODULE_DATA,
377 							      MODULE_ALLOC_TAG_VMAP_SIZE);
378 	if (!module_tags.start_addr)
379 		return -ENOMEM;
380 
381 	module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE;
382 
383 	return 0;
384 }
385 
386 static void __init free_mod_tags_mem(void)
387 {
388 	execmem_free((void *)module_tags.start_addr);
389 	module_tags.start_addr = 0;
390 }
391 
392 #else /* CONFIG_MODULES */
393 
394 static inline int alloc_mod_tags_mem(void) { return 0; }
395 static inline void free_mod_tags_mem(void) {}
396 
397 #endif /* CONFIG_MODULES */
398 
399 static int __init setup_early_mem_profiling(char *str)
400 {
401 	bool enable;
402 
403 	if (!str || !str[0])
404 		return -EINVAL;
405 
406 	if (!strncmp(str, "never", 5)) {
407 		enable = false;
408 		mem_profiling_support = false;
409 	} else {
410 		int res;
411 
412 		res = kstrtobool(str, &enable);
413 		if (res)
414 			return res;
415 
416 		mem_profiling_support = true;
417 	}
418 
419 	if (enable != static_key_enabled(&mem_alloc_profiling_key)) {
420 		if (enable)
421 			static_branch_enable(&mem_alloc_profiling_key);
422 		else
423 			static_branch_disable(&mem_alloc_profiling_key);
424 	}
425 
426 	return 0;
427 }
428 early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling);
429 
430 static __init bool need_page_alloc_tagging(void)
431 {
432 	return mem_profiling_support;
433 }
434 
435 static __init void init_page_alloc_tagging(void)
436 {
437 }
438 
439 struct page_ext_operations page_alloc_tagging_ops = {
440 	.size = sizeof(union codetag_ref),
441 	.need = need_page_alloc_tagging,
442 	.init = init_page_alloc_tagging,
443 };
444 EXPORT_SYMBOL(page_alloc_tagging_ops);
445 
446 #ifdef CONFIG_SYSCTL
447 static struct ctl_table memory_allocation_profiling_sysctls[] = {
448 	{
449 		.procname	= "mem_profiling",
450 		.data		= &mem_alloc_profiling_key,
451 #ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
452 		.mode		= 0444,
453 #else
454 		.mode		= 0644,
455 #endif
456 		.proc_handler	= proc_do_static_key,
457 	},
458 };
459 
460 static void __init sysctl_init(void)
461 {
462 	if (!mem_profiling_support)
463 		memory_allocation_profiling_sysctls[0].mode = 0444;
464 
465 	register_sysctl_init("vm", memory_allocation_profiling_sysctls);
466 }
467 #else /* CONFIG_SYSCTL */
468 static inline void sysctl_init(void) {}
469 #endif /* CONFIG_SYSCTL */
470 
471 static int __init alloc_tag_init(void)
472 {
473 	const struct codetag_type_desc desc = {
474 		.section		= ALLOC_TAG_SECTION_NAME,
475 		.tag_size		= sizeof(struct alloc_tag),
476 #ifdef CONFIG_MODULES
477 		.needs_section_mem	= needs_section_mem,
478 		.alloc_section_mem	= reserve_module_tags,
479 		.free_section_mem	= release_module_tags,
480 		.module_replaced	= replace_module,
481 #endif
482 	};
483 	int res;
484 
485 	res = alloc_mod_tags_mem();
486 	if (res)
487 		return res;
488 
489 	alloc_tag_cttype = codetag_register_type(&desc);
490 	if (IS_ERR(alloc_tag_cttype)) {
491 		free_mod_tags_mem();
492 		return PTR_ERR(alloc_tag_cttype);
493 	}
494 
495 	sysctl_init();
496 	procfs_init();
497 
498 	return 0;
499 }
500 module_init(alloc_tag_init);
501