1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2005 Robert N. M. Watson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 * $FreeBSD$
29 */
30
31 #include <sys/cdefs.h>
32 #include <sys/param.h>
33 #include <sys/malloc.h>
34 #include <sys/sysctl.h>
35
36 #include <err.h>
37 #include <errno.h>
38 #include <kvm.h>
39 #include <nlist.h>
40 #include <stdio.h>
41 #include <stdlib.h>
42 #include <string.h>
43
44 #include "memstat.h"
45 #include "memstat_internal.h"
46
47 static int memstat_malloc_zone_count;
48 static int memstat_malloc_zone_sizes[32];
49
50 static int memstat_malloc_zone_init(void);
51 static int memstat_malloc_zone_init_kvm(kvm_t *kvm);
52
53 static struct nlist namelist[] = {
54 #define X_KMEMSTATISTICS 0
55 { .n_name = "_kmemstatistics" },
56 #define X_KMEMZONES 1
57 { .n_name = "_kmemzones" },
58 #define X_NUMZONES 2
59 { .n_name = "_numzones" },
60 #define X_VM_MALLOC_ZONE_COUNT 3
61 { .n_name = "_vm_malloc_zone_count" },
62 #define X_MP_MAXCPUS 4
63 { .n_name = "_mp_maxcpus" },
64 { .n_name = "" },
65 };
66
67 /*
68 * Extract malloc(9) statistics from the running kernel, and store all memory
69 * type information in the passed list. For each type, check the list for an
70 * existing entry with the right name/allocator -- if present, update that
71 * entry. Otherwise, add a new entry. On error, the entire list will be
72 * cleared, as entries will be in an inconsistent state.
73 *
74 * To reduce the level of work for a list that starts empty, we keep around a
75 * hint as to whether it was empty when we began, so we can avoid searching
76 * the list for entries to update. Updates are O(n^2) due to searching for
77 * each entry before adding it.
78 */
79 int
memstat_sysctl_malloc(struct memory_type_list * list,int flags)80 memstat_sysctl_malloc(struct memory_type_list *list, int flags)
81 {
82 struct malloc_type_stream_header *mtshp;
83 struct malloc_type_header *mthp;
84 struct malloc_type_stats *mtsp;
85 struct memory_type *mtp;
86 int count, hint_dontsearch, i, j, maxcpus;
87 char *buffer, *p;
88 size_t size;
89
90 hint_dontsearch = LIST_EMPTY(&list->mtl_list);
91
92 /*
93 * Query the number of CPUs, number of malloc types so that we can
94 * guess an initial buffer size. We loop until we succeed or really
95 * fail. Note that the value of maxcpus we query using sysctl is not
96 * the version we use when processing the real data -- that is read
97 * from the header.
98 */
99 retry:
100 size = sizeof(maxcpus);
101 if (sysctlbyname("kern.smp.maxcpus", &maxcpus, &size, NULL, 0) < 0) {
102 if (errno == EACCES || errno == EPERM)
103 list->mtl_error = MEMSTAT_ERROR_PERMISSION;
104 else
105 list->mtl_error = MEMSTAT_ERROR_DATAERROR;
106 return (-1);
107 }
108 if (size != sizeof(maxcpus)) {
109 list->mtl_error = MEMSTAT_ERROR_DATAERROR;
110 return (-1);
111 }
112
113 size = sizeof(count);
114 if (sysctlbyname("kern.malloc_count", &count, &size, NULL, 0) < 0) {
115 if (errno == EACCES || errno == EPERM)
116 list->mtl_error = MEMSTAT_ERROR_PERMISSION;
117 else
118 list->mtl_error = MEMSTAT_ERROR_VERSION;
119 return (-1);
120 }
121 if (size != sizeof(count)) {
122 list->mtl_error = MEMSTAT_ERROR_DATAERROR;
123 return (-1);
124 }
125
126 if (memstat_malloc_zone_init() == -1) {
127 list->mtl_error = MEMSTAT_ERROR_VERSION;
128 return (-1);
129 }
130
131 size = sizeof(*mthp) + count * (sizeof(*mthp) + sizeof(*mtsp) *
132 maxcpus);
133
134 buffer = malloc(size);
135 if (buffer == NULL) {
136 list->mtl_error = MEMSTAT_ERROR_NOMEMORY;
137 return (-1);
138 }
139
140 if (sysctlbyname("kern.malloc_stats", buffer, &size, NULL, 0) < 0) {
141 /*
142 * XXXRW: ENOMEM is an ambiguous return, we should bound the
143 * number of loops, perhaps.
144 */
145 if (errno == ENOMEM) {
146 free(buffer);
147 goto retry;
148 }
149 if (errno == EACCES || errno == EPERM)
150 list->mtl_error = MEMSTAT_ERROR_PERMISSION;
151 else
152 list->mtl_error = MEMSTAT_ERROR_VERSION;
153 free(buffer);
154 return (-1);
155 }
156
157 if (size == 0) {
158 free(buffer);
159 return (0);
160 }
161
162 if (size < sizeof(*mtshp)) {
163 list->mtl_error = MEMSTAT_ERROR_VERSION;
164 free(buffer);
165 return (-1);
166 }
167 p = buffer;
168 mtshp = (struct malloc_type_stream_header *)p;
169 p += sizeof(*mtshp);
170
171 if (mtshp->mtsh_version != MALLOC_TYPE_STREAM_VERSION) {
172 list->mtl_error = MEMSTAT_ERROR_VERSION;
173 free(buffer);
174 return (-1);
175 }
176
177 /*
178 * For the remainder of this function, we are quite trusting about
179 * the layout of structures and sizes, since we've determined we have
180 * a matching version and acceptable CPU count.
181 */
182 maxcpus = mtshp->mtsh_maxcpus;
183 count = mtshp->mtsh_count;
184 for (i = 0; i < count; i++) {
185 mthp = (struct malloc_type_header *)p;
186 p += sizeof(*mthp);
187
188 if (hint_dontsearch == 0) {
189 mtp = memstat_mtl_find(list, ALLOCATOR_MALLOC,
190 mthp->mth_name);
191 } else
192 mtp = NULL;
193 if (mtp == NULL)
194 mtp = _memstat_mt_allocate(list, ALLOCATOR_MALLOC,
195 mthp->mth_name, maxcpus);
196 if (mtp == NULL) {
197 _memstat_mtl_empty(list);
198 free(buffer);
199 list->mtl_error = MEMSTAT_ERROR_NOMEMORY;
200 return (-1);
201 }
202
203 /*
204 * Reset the statistics on a current node.
205 */
206 _memstat_mt_reset_stats(mtp, maxcpus);
207
208 for (j = 0; j < maxcpus; j++) {
209 mtsp = (struct malloc_type_stats *)p;
210 p += sizeof(*mtsp);
211
212 /*
213 * Sumarize raw statistics across CPUs into coalesced
214 * statistics.
215 */
216 mtp->mt_memalloced += mtsp->mts_memalloced;
217 mtp->mt_memfreed += mtsp->mts_memfreed;
218 mtp->mt_numallocs += mtsp->mts_numallocs;
219 mtp->mt_numfrees += mtsp->mts_numfrees;
220 mtp->mt_sizemask |= mtsp->mts_size;
221
222 /*
223 * Copies of per-CPU statistics.
224 */
225 mtp->mt_percpu_alloc[j].mtp_memalloced =
226 mtsp->mts_memalloced;
227 mtp->mt_percpu_alloc[j].mtp_memfreed =
228 mtsp->mts_memfreed;
229 mtp->mt_percpu_alloc[j].mtp_numallocs =
230 mtsp->mts_numallocs;
231 mtp->mt_percpu_alloc[j].mtp_numfrees =
232 mtsp->mts_numfrees;
233 mtp->mt_percpu_alloc[j].mtp_sizemask =
234 mtsp->mts_size;
235 }
236
237 /*
238 * Derived cross-CPU statistics.
239 */
240 mtp->mt_bytes = mtp->mt_memalloced - mtp->mt_memfreed;
241 mtp->mt_count = mtp->mt_numallocs - mtp->mt_numfrees;
242 }
243
244 free(buffer);
245
246 return (0);
247 }
248
249 static int
kread(kvm_t * kvm,void * kvm_pointer,void * address,size_t size,size_t offset)250 kread(kvm_t *kvm, void *kvm_pointer, void *address, size_t size,
251 size_t offset)
252 {
253 ssize_t ret;
254
255 ret = kvm_read(kvm, (unsigned long)kvm_pointer + offset, address,
256 size);
257 if (ret < 0)
258 return (MEMSTAT_ERROR_KVM);
259 if ((size_t)ret != size)
260 return (MEMSTAT_ERROR_KVM_SHORTREAD);
261 return (0);
262 }
263
264 static int
kread_string(kvm_t * kvm,const void * kvm_pointer,char * buffer,int buflen)265 kread_string(kvm_t *kvm, const void *kvm_pointer, char *buffer, int buflen)
266 {
267 ssize_t ret;
268 int i;
269
270 for (i = 0; i < buflen; i++) {
271 ret = kvm_read(kvm, __DECONST(unsigned long, kvm_pointer) +
272 i, &(buffer[i]), sizeof(char));
273 if (ret < 0)
274 return (MEMSTAT_ERROR_KVM);
275 if ((size_t)ret != sizeof(char))
276 return (MEMSTAT_ERROR_KVM_SHORTREAD);
277 if (buffer[i] == '\0')
278 return (0);
279 }
280 /* Truncate. */
281 buffer[i-1] = '\0';
282 return (0);
283 }
284
285 static int
kread_symbol(kvm_t * kvm,int index,void * address,size_t size,size_t offset)286 kread_symbol(kvm_t *kvm, int index, void *address, size_t size,
287 size_t offset)
288 {
289 ssize_t ret;
290
291 ret = kvm_read(kvm, namelist[index].n_value + offset, address, size);
292 if (ret < 0)
293 return (MEMSTAT_ERROR_KVM);
294 if ((size_t)ret != size)
295 return (MEMSTAT_ERROR_KVM_SHORTREAD);
296 return (0);
297 }
298
299 static int
kread_zpcpu(kvm_t * kvm,u_long base,void * buf,size_t size,int cpu)300 kread_zpcpu(kvm_t *kvm, u_long base, void *buf, size_t size, int cpu)
301 {
302 ssize_t ret;
303
304 ret = kvm_read_zpcpu(kvm, base, buf, size, cpu);
305 if (ret < 0)
306 return (MEMSTAT_ERROR_KVM);
307 if ((size_t)ret != size)
308 return (MEMSTAT_ERROR_KVM_SHORTREAD);
309 return (0);
310 }
311
312 int
memstat_kvm_malloc(struct memory_type_list * list,void * kvm_handle)313 memstat_kvm_malloc(struct memory_type_list *list, void *kvm_handle)
314 {
315 struct memory_type *mtp;
316 void *kmemstatistics;
317 int hint_dontsearch, j, mp_maxcpus, mp_ncpus, ret;
318 char name[MEMTYPE_MAXNAME];
319 struct malloc_type_stats mts;
320 struct malloc_type_internal *mtip;
321 struct malloc_type type, *typep;
322 kvm_t *kvm;
323
324 kvm = (kvm_t *)kvm_handle;
325
326 hint_dontsearch = LIST_EMPTY(&list->mtl_list);
327
328 if (kvm_nlist(kvm, namelist) != 0) {
329 list->mtl_error = MEMSTAT_ERROR_KVM;
330 return (-1);
331 }
332
333 if (namelist[X_KMEMSTATISTICS].n_type == 0 ||
334 namelist[X_KMEMSTATISTICS].n_value == 0) {
335 list->mtl_error = MEMSTAT_ERROR_KVM_NOSYMBOL;
336 return (-1);
337 }
338
339 ret = kread_symbol(kvm, X_MP_MAXCPUS, &mp_maxcpus,
340 sizeof(mp_maxcpus), 0);
341 if (ret != 0) {
342 list->mtl_error = ret;
343 return (-1);
344 }
345
346 ret = kread_symbol(kvm, X_KMEMSTATISTICS, &kmemstatistics,
347 sizeof(kmemstatistics), 0);
348 if (ret != 0) {
349 list->mtl_error = ret;
350 return (-1);
351 }
352
353 ret = memstat_malloc_zone_init_kvm(kvm);
354 if (ret != 0) {
355 list->mtl_error = ret;
356 return (-1);
357 }
358
359 mp_ncpus = kvm_getncpus(kvm);
360
361 for (typep = kmemstatistics; typep != NULL; typep = type.ks_next) {
362 ret = kread(kvm, typep, &type, sizeof(type), 0);
363 if (ret != 0) {
364 _memstat_mtl_empty(list);
365 list->mtl_error = ret;
366 return (-1);
367 }
368 ret = kread_string(kvm, (void *)type.ks_shortdesc, name,
369 MEMTYPE_MAXNAME);
370 if (ret != 0) {
371 _memstat_mtl_empty(list);
372 list->mtl_error = ret;
373 return (-1);
374 }
375 if (type.ks_version != M_VERSION) {
376 warnx("type %s with unsupported version %lu; skipped",
377 name, type.ks_version);
378 continue;
379 }
380
381 /*
382 * Since our compile-time value for MAXCPU may differ from the
383 * kernel's, we populate our own array.
384 */
385 mtip = &type.ks_mti;
386
387 if (hint_dontsearch == 0) {
388 mtp = memstat_mtl_find(list, ALLOCATOR_MALLOC, name);
389 } else
390 mtp = NULL;
391 if (mtp == NULL)
392 mtp = _memstat_mt_allocate(list, ALLOCATOR_MALLOC,
393 name, mp_maxcpus);
394 if (mtp == NULL) {
395 _memstat_mtl_empty(list);
396 list->mtl_error = MEMSTAT_ERROR_NOMEMORY;
397 return (-1);
398 }
399
400 /*
401 * This logic is replicated from kern_malloc.c, and should
402 * be kept in sync.
403 */
404 _memstat_mt_reset_stats(mtp, mp_maxcpus);
405 for (j = 0; j < mp_ncpus; j++) {
406 ret = kread_zpcpu(kvm, (u_long)mtip->mti_stats, &mts,
407 sizeof(mts), j);
408 if (ret != 0) {
409 _memstat_mtl_empty(list);
410 list->mtl_error = ret;
411 return (-1);
412 }
413 mtp->mt_memalloced += mts.mts_memalloced;
414 mtp->mt_memfreed += mts.mts_memfreed;
415 mtp->mt_numallocs += mts.mts_numallocs;
416 mtp->mt_numfrees += mts.mts_numfrees;
417 mtp->mt_sizemask |= mts.mts_size;
418
419 mtp->mt_percpu_alloc[j].mtp_memalloced =
420 mts.mts_memalloced;
421 mtp->mt_percpu_alloc[j].mtp_memfreed =
422 mts.mts_memfreed;
423 mtp->mt_percpu_alloc[j].mtp_numallocs =
424 mts.mts_numallocs;
425 mtp->mt_percpu_alloc[j].mtp_numfrees =
426 mts.mts_numfrees;
427 mtp->mt_percpu_alloc[j].mtp_sizemask =
428 mts.mts_size;
429 }
430 for (; j < mp_maxcpus; j++) {
431 bzero(&mtp->mt_percpu_alloc[j],
432 sizeof(mtp->mt_percpu_alloc[0]));
433 }
434
435 mtp->mt_bytes = mtp->mt_memalloced - mtp->mt_memfreed;
436 mtp->mt_count = mtp->mt_numallocs - mtp->mt_numfrees;
437 }
438
439 return (0);
440 }
441
442 static int
memstat_malloc_zone_init(void)443 memstat_malloc_zone_init(void)
444 {
445 size_t size;
446
447 size = sizeof(memstat_malloc_zone_count);
448 if (sysctlbyname("vm.malloc.zone_count", &memstat_malloc_zone_count,
449 &size, NULL, 0) < 0) {
450 return (-1);
451 }
452
453 if (memstat_malloc_zone_count > (int)nitems(memstat_malloc_zone_sizes)) {
454 return (-1);
455 }
456
457 size = sizeof(memstat_malloc_zone_sizes);
458 if (sysctlbyname("vm.malloc.zone_sizes", &memstat_malloc_zone_sizes,
459 &size, NULL, 0) < 0) {
460 return (-1);
461 }
462
463 return (0);
464 }
465
466 /*
467 * Copied from kern_malloc.c
468 *
469 * kz_zone is an array sized at compilation time, the size is exported in
470 * "numzones". Below we need to iterate kz_size.
471 */
472 struct memstat_kmemzone {
473 int kz_size;
474 const char *kz_name;
475 void *kz_zone[1];
476 };
477
478 static int
memstat_malloc_zone_init_kvm(kvm_t * kvm)479 memstat_malloc_zone_init_kvm(kvm_t *kvm)
480 {
481 struct memstat_kmemzone *kmemzones, *kz;
482 int numzones, objsize, allocsize, ret;
483 int i;
484
485 ret = kread_symbol(kvm, X_VM_MALLOC_ZONE_COUNT,
486 &memstat_malloc_zone_count, sizeof(memstat_malloc_zone_count), 0);
487 if (ret != 0) {
488 return (ret);
489 }
490
491 ret = kread_symbol(kvm, X_NUMZONES, &numzones, sizeof(numzones), 0);
492 if (ret != 0) {
493 return (ret);
494 }
495
496 objsize = __offsetof(struct memstat_kmemzone, kz_zone) +
497 sizeof(void *) * numzones;
498
499 allocsize = objsize * memstat_malloc_zone_count;
500 kmemzones = malloc(allocsize);
501 if (kmemzones == NULL) {
502 return (MEMSTAT_ERROR_NOMEMORY);
503 }
504 ret = kread_symbol(kvm, X_KMEMZONES, kmemzones, allocsize, 0);
505 if (ret != 0) {
506 free(kmemzones);
507 return (ret);
508 }
509
510 kz = kmemzones;
511 for (i = 0; i < (int)nitems(memstat_malloc_zone_sizes); i++) {
512 memstat_malloc_zone_sizes[i] = kz->kz_size;
513 kz = (struct memstat_kmemzone *)((char *)kz + objsize);
514 }
515
516 free(kmemzones);
517 return (0);
518 }
519
520 size_t
memstat_malloc_zone_get_count(void)521 memstat_malloc_zone_get_count(void)
522 {
523
524 return (memstat_malloc_zone_count);
525 }
526
527 size_t
memstat_malloc_zone_get_size(size_t n)528 memstat_malloc_zone_get_size(size_t n)
529 {
530
531 if (n >= nitems(memstat_malloc_zone_sizes)) {
532 return (-1);
533 }
534
535 return (memstat_malloc_zone_sizes[n]);
536 }
537
538 int
memstat_malloc_zone_used(const struct memory_type * mtp,size_t n)539 memstat_malloc_zone_used(const struct memory_type *mtp, size_t n)
540 {
541
542 if (memstat_get_sizemask(mtp) & (1 << n))
543 return (1);
544
545 return (0);
546 }
547