1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2005 Robert N. M. Watson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 #include <sys/param.h>
31 #include <sys/malloc.h>
32 #include <sys/sysctl.h>
33
34 #include <err.h>
35 #include <errno.h>
36 #include <kvm.h>
37 #include <nlist.h>
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <string.h>
41
42 #include "memstat.h"
43 #include "memstat_internal.h"
44
45 static int memstat_malloc_zone_count;
46 static int memstat_malloc_zone_sizes[32];
47
48 static int memstat_malloc_zone_init(void);
49 static int memstat_malloc_zone_init_kvm(kvm_t *kvm);
50
51 static struct nlist namelist[] = {
52 #define X_KMEMSTATISTICS 0
53 { .n_name = "_kmemstatistics" },
54 #define X_KMEMZONES 1
55 { .n_name = "_kmemzones" },
56 #define X_NUMZONES 2
57 { .n_name = "_numzones" },
58 #define X_VM_MALLOC_ZONE_COUNT 3
59 { .n_name = "_vm_malloc_zone_count" },
60 #define X_MP_MAXCPUS 4
61 { .n_name = "_mp_maxcpus" },
62 { .n_name = "" },
63 };
64
65 /*
66 * Extract malloc(9) statistics from the running kernel, and store all memory
67 * type information in the passed list. For each type, check the list for an
68 * existing entry with the right name/allocator -- if present, update that
69 * entry. Otherwise, add a new entry. On error, the entire list will be
70 * cleared, as entries will be in an inconsistent state.
71 *
72 * To reduce the level of work for a list that starts empty, we keep around a
73 * hint as to whether it was empty when we began, so we can avoid searching
74 * the list for entries to update. Updates are O(n^2) due to searching for
75 * each entry before adding it.
76 */
77 int
memstat_sysctl_malloc(struct memory_type_list * list,int flags)78 memstat_sysctl_malloc(struct memory_type_list *list, int flags)
79 {
80 struct malloc_type_stream_header *mtshp;
81 struct malloc_type_header *mthp;
82 struct malloc_type_stats *mtsp;
83 struct memory_type *mtp;
84 int count, hint_dontsearch, i, j, maxcpus;
85 char *buffer, *p;
86 size_t size;
87
88 hint_dontsearch = LIST_EMPTY(&list->mtl_list);
89
90 /*
91 * Query the number of CPUs, number of malloc types so that we can
92 * guess an initial buffer size. We loop until we succeed or really
93 * fail. Note that the value of maxcpus we query using sysctl is not
94 * the version we use when processing the real data -- that is read
95 * from the header.
96 */
97 retry:
98 size = sizeof(maxcpus);
99 if (sysctlbyname("kern.smp.maxcpus", &maxcpus, &size, NULL, 0) < 0) {
100 if (errno == EACCES || errno == EPERM)
101 list->mtl_error = MEMSTAT_ERROR_PERMISSION;
102 else
103 list->mtl_error = MEMSTAT_ERROR_DATAERROR;
104 return (-1);
105 }
106 if (size != sizeof(maxcpus)) {
107 list->mtl_error = MEMSTAT_ERROR_DATAERROR;
108 return (-1);
109 }
110
111 size = sizeof(count);
112 if (sysctlbyname("kern.malloc_count", &count, &size, NULL, 0) < 0) {
113 if (errno == EACCES || errno == EPERM)
114 list->mtl_error = MEMSTAT_ERROR_PERMISSION;
115 else
116 list->mtl_error = MEMSTAT_ERROR_VERSION;
117 return (-1);
118 }
119 if (size != sizeof(count)) {
120 list->mtl_error = MEMSTAT_ERROR_DATAERROR;
121 return (-1);
122 }
123
124 if (memstat_malloc_zone_init() == -1) {
125 list->mtl_error = MEMSTAT_ERROR_VERSION;
126 return (-1);
127 }
128
129 size = sizeof(*mthp) + count * (sizeof(*mthp) + sizeof(*mtsp) *
130 maxcpus);
131
132 buffer = malloc(size);
133 if (buffer == NULL) {
134 list->mtl_error = MEMSTAT_ERROR_NOMEMORY;
135 return (-1);
136 }
137
138 if (sysctlbyname("kern.malloc_stats", buffer, &size, NULL, 0) < 0) {
139 /*
140 * XXXRW: ENOMEM is an ambiguous return, we should bound the
141 * number of loops, perhaps.
142 */
143 if (errno == ENOMEM) {
144 free(buffer);
145 goto retry;
146 }
147 if (errno == EACCES || errno == EPERM)
148 list->mtl_error = MEMSTAT_ERROR_PERMISSION;
149 else
150 list->mtl_error = MEMSTAT_ERROR_VERSION;
151 free(buffer);
152 return (-1);
153 }
154
155 if (size == 0) {
156 free(buffer);
157 return (0);
158 }
159
160 if (size < sizeof(*mtshp)) {
161 list->mtl_error = MEMSTAT_ERROR_VERSION;
162 free(buffer);
163 return (-1);
164 }
165 p = buffer;
166 mtshp = (struct malloc_type_stream_header *)p;
167 p += sizeof(*mtshp);
168
169 if (mtshp->mtsh_version != MALLOC_TYPE_STREAM_VERSION) {
170 list->mtl_error = MEMSTAT_ERROR_VERSION;
171 free(buffer);
172 return (-1);
173 }
174
175 /*
176 * For the remainder of this function, we are quite trusting about
177 * the layout of structures and sizes, since we've determined we have
178 * a matching version and acceptable CPU count.
179 */
180 maxcpus = mtshp->mtsh_maxcpus;
181 count = mtshp->mtsh_count;
182 for (i = 0; i < count; i++) {
183 mthp = (struct malloc_type_header *)p;
184 p += sizeof(*mthp);
185
186 if (hint_dontsearch == 0) {
187 mtp = memstat_mtl_find(list, ALLOCATOR_MALLOC,
188 mthp->mth_name);
189 } else
190 mtp = NULL;
191 if (mtp == NULL)
192 mtp = _memstat_mt_allocate(list, ALLOCATOR_MALLOC,
193 mthp->mth_name, maxcpus);
194 if (mtp == NULL) {
195 _memstat_mtl_empty(list);
196 free(buffer);
197 list->mtl_error = MEMSTAT_ERROR_NOMEMORY;
198 return (-1);
199 }
200
201 /*
202 * Reset the statistics on a current node.
203 */
204 _memstat_mt_reset_stats(mtp, maxcpus);
205
206 for (j = 0; j < maxcpus; j++) {
207 mtsp = (struct malloc_type_stats *)p;
208 p += sizeof(*mtsp);
209
210 /*
211 * Sumarize raw statistics across CPUs into coalesced
212 * statistics.
213 */
214 mtp->mt_memalloced += mtsp->mts_memalloced;
215 mtp->mt_memfreed += mtsp->mts_memfreed;
216 mtp->mt_numallocs += mtsp->mts_numallocs;
217 mtp->mt_numfrees += mtsp->mts_numfrees;
218 mtp->mt_sizemask |= mtsp->mts_size;
219
220 /*
221 * Copies of per-CPU statistics.
222 */
223 mtp->mt_percpu_alloc[j].mtp_memalloced =
224 mtsp->mts_memalloced;
225 mtp->mt_percpu_alloc[j].mtp_memfreed =
226 mtsp->mts_memfreed;
227 mtp->mt_percpu_alloc[j].mtp_numallocs =
228 mtsp->mts_numallocs;
229 mtp->mt_percpu_alloc[j].mtp_numfrees =
230 mtsp->mts_numfrees;
231 mtp->mt_percpu_alloc[j].mtp_sizemask =
232 mtsp->mts_size;
233 }
234
235 /*
236 * Derived cross-CPU statistics.
237 */
238 mtp->mt_bytes = mtp->mt_memalloced - mtp->mt_memfreed;
239 mtp->mt_count = mtp->mt_numallocs - mtp->mt_numfrees;
240 }
241
242 free(buffer);
243
244 return (0);
245 }
246
247 static int
kread(kvm_t * kvm,void * kvm_pointer,void * address,size_t size,size_t offset)248 kread(kvm_t *kvm, void *kvm_pointer, void *address, size_t size,
249 size_t offset)
250 {
251 ssize_t ret;
252
253 ret = kvm_read(kvm, (unsigned long)kvm_pointer + offset, address,
254 size);
255 if (ret < 0)
256 return (MEMSTAT_ERROR_KVM);
257 if ((size_t)ret != size)
258 return (MEMSTAT_ERROR_KVM_SHORTREAD);
259 return (0);
260 }
261
262 static int
kread_string(kvm_t * kvm,const void * kvm_pointer,char * buffer,int buflen)263 kread_string(kvm_t *kvm, const void *kvm_pointer, char *buffer, int buflen)
264 {
265 ssize_t ret;
266 int i;
267
268 for (i = 0; i < buflen; i++) {
269 ret = kvm_read(kvm, __DECONST(unsigned long, kvm_pointer) +
270 i, &(buffer[i]), sizeof(char));
271 if (ret < 0)
272 return (MEMSTAT_ERROR_KVM);
273 if ((size_t)ret != sizeof(char))
274 return (MEMSTAT_ERROR_KVM_SHORTREAD);
275 if (buffer[i] == '\0')
276 return (0);
277 }
278 /* Truncate. */
279 buffer[i-1] = '\0';
280 return (0);
281 }
282
283 static int
kread_symbol(kvm_t * kvm,int index,void * address,size_t size,size_t offset)284 kread_symbol(kvm_t *kvm, int index, void *address, size_t size,
285 size_t offset)
286 {
287 ssize_t ret;
288
289 ret = kvm_read(kvm, namelist[index].n_value + offset, address, size);
290 if (ret < 0)
291 return (MEMSTAT_ERROR_KVM);
292 if ((size_t)ret != size)
293 return (MEMSTAT_ERROR_KVM_SHORTREAD);
294 return (0);
295 }
296
297 static int
kread_zpcpu(kvm_t * kvm,u_long base,void * buf,size_t size,int cpu)298 kread_zpcpu(kvm_t *kvm, u_long base, void *buf, size_t size, int cpu)
299 {
300 ssize_t ret;
301
302 ret = kvm_read_zpcpu(kvm, base, buf, size, cpu);
303 if (ret < 0)
304 return (MEMSTAT_ERROR_KVM);
305 if ((size_t)ret != size)
306 return (MEMSTAT_ERROR_KVM_SHORTREAD);
307 return (0);
308 }
309
310 int
memstat_kvm_malloc(struct memory_type_list * list,void * kvm_handle)311 memstat_kvm_malloc(struct memory_type_list *list, void *kvm_handle)
312 {
313 struct memory_type *mtp;
314 void *kmemstatistics;
315 int hint_dontsearch, j, mp_maxcpus, mp_ncpus, ret;
316 char name[MEMTYPE_MAXNAME];
317 struct malloc_type_stats mts;
318 struct malloc_type_internal *mtip;
319 struct malloc_type type, *typep;
320 kvm_t *kvm;
321
322 kvm = (kvm_t *)kvm_handle;
323
324 hint_dontsearch = LIST_EMPTY(&list->mtl_list);
325
326 if (kvm_nlist(kvm, namelist) != 0) {
327 list->mtl_error = MEMSTAT_ERROR_KVM;
328 return (-1);
329 }
330
331 if (namelist[X_KMEMSTATISTICS].n_type == 0 ||
332 namelist[X_KMEMSTATISTICS].n_value == 0) {
333 list->mtl_error = MEMSTAT_ERROR_KVM_NOSYMBOL;
334 return (-1);
335 }
336
337 ret = kread_symbol(kvm, X_MP_MAXCPUS, &mp_maxcpus,
338 sizeof(mp_maxcpus), 0);
339 if (ret != 0) {
340 list->mtl_error = ret;
341 return (-1);
342 }
343
344 ret = kread_symbol(kvm, X_KMEMSTATISTICS, &kmemstatistics,
345 sizeof(kmemstatistics), 0);
346 if (ret != 0) {
347 list->mtl_error = ret;
348 return (-1);
349 }
350
351 ret = memstat_malloc_zone_init_kvm(kvm);
352 if (ret != 0) {
353 list->mtl_error = ret;
354 return (-1);
355 }
356
357 mp_ncpus = kvm_getncpus(kvm);
358
359 for (typep = kmemstatistics; typep != NULL; typep = type.ks_next) {
360 ret = kread(kvm, typep, &type, sizeof(type), 0);
361 if (ret != 0) {
362 _memstat_mtl_empty(list);
363 list->mtl_error = ret;
364 return (-1);
365 }
366 ret = kread_string(kvm, (void *)type.ks_shortdesc, name,
367 MEMTYPE_MAXNAME);
368 if (ret != 0) {
369 _memstat_mtl_empty(list);
370 list->mtl_error = ret;
371 return (-1);
372 }
373 if (type.ks_version != M_VERSION) {
374 warnx("type %s with unsupported version %lu; skipped",
375 name, type.ks_version);
376 continue;
377 }
378
379 /*
380 * Since our compile-time value for MAXCPU may differ from the
381 * kernel's, we populate our own array.
382 */
383 mtip = &type.ks_mti;
384
385 if (hint_dontsearch == 0) {
386 mtp = memstat_mtl_find(list, ALLOCATOR_MALLOC, name);
387 } else
388 mtp = NULL;
389 if (mtp == NULL)
390 mtp = _memstat_mt_allocate(list, ALLOCATOR_MALLOC,
391 name, mp_maxcpus);
392 if (mtp == NULL) {
393 _memstat_mtl_empty(list);
394 list->mtl_error = MEMSTAT_ERROR_NOMEMORY;
395 return (-1);
396 }
397
398 /*
399 * This logic is replicated from kern_malloc.c, and should
400 * be kept in sync.
401 */
402 _memstat_mt_reset_stats(mtp, mp_maxcpus);
403 for (j = 0; j < mp_ncpus; j++) {
404 ret = kread_zpcpu(kvm, (u_long)mtip->mti_stats, &mts,
405 sizeof(mts), j);
406 if (ret != 0) {
407 _memstat_mtl_empty(list);
408 list->mtl_error = ret;
409 return (-1);
410 }
411 mtp->mt_memalloced += mts.mts_memalloced;
412 mtp->mt_memfreed += mts.mts_memfreed;
413 mtp->mt_numallocs += mts.mts_numallocs;
414 mtp->mt_numfrees += mts.mts_numfrees;
415 mtp->mt_sizemask |= mts.mts_size;
416
417 mtp->mt_percpu_alloc[j].mtp_memalloced =
418 mts.mts_memalloced;
419 mtp->mt_percpu_alloc[j].mtp_memfreed =
420 mts.mts_memfreed;
421 mtp->mt_percpu_alloc[j].mtp_numallocs =
422 mts.mts_numallocs;
423 mtp->mt_percpu_alloc[j].mtp_numfrees =
424 mts.mts_numfrees;
425 mtp->mt_percpu_alloc[j].mtp_sizemask =
426 mts.mts_size;
427 }
428 for (; j < mp_maxcpus; j++) {
429 bzero(&mtp->mt_percpu_alloc[j],
430 sizeof(mtp->mt_percpu_alloc[0]));
431 }
432
433 mtp->mt_bytes = mtp->mt_memalloced - mtp->mt_memfreed;
434 mtp->mt_count = mtp->mt_numallocs - mtp->mt_numfrees;
435 }
436
437 return (0);
438 }
439
440 static int
memstat_malloc_zone_init(void)441 memstat_malloc_zone_init(void)
442 {
443 size_t size;
444
445 size = sizeof(memstat_malloc_zone_count);
446 if (sysctlbyname("vm.malloc.zone_count", &memstat_malloc_zone_count,
447 &size, NULL, 0) < 0) {
448 return (-1);
449 }
450
451 if (memstat_malloc_zone_count > (int)nitems(memstat_malloc_zone_sizes)) {
452 return (-1);
453 }
454
455 size = sizeof(memstat_malloc_zone_sizes);
456 if (sysctlbyname("vm.malloc.zone_sizes", &memstat_malloc_zone_sizes,
457 &size, NULL, 0) < 0) {
458 return (-1);
459 }
460
461 return (0);
462 }
463
464 /*
465 * Copied from kern_malloc.c
466 *
467 * kz_zone is an array sized at compilation time, the size is exported in
468 * "numzones". Below we need to iterate kz_size.
469 */
470 struct memstat_kmemzone {
471 int kz_size;
472 const char *kz_name;
473 void *kz_zone[1];
474 };
475
476 static int
memstat_malloc_zone_init_kvm(kvm_t * kvm)477 memstat_malloc_zone_init_kvm(kvm_t *kvm)
478 {
479 struct memstat_kmemzone *kmemzones, *kz;
480 int numzones, objsize, allocsize, ret;
481 int i;
482
483 ret = kread_symbol(kvm, X_VM_MALLOC_ZONE_COUNT,
484 &memstat_malloc_zone_count, sizeof(memstat_malloc_zone_count), 0);
485 if (ret != 0) {
486 return (ret);
487 }
488
489 ret = kread_symbol(kvm, X_NUMZONES, &numzones, sizeof(numzones), 0);
490 if (ret != 0) {
491 return (ret);
492 }
493
494 objsize = __offsetof(struct memstat_kmemzone, kz_zone) +
495 sizeof(void *) * numzones;
496
497 allocsize = objsize * memstat_malloc_zone_count;
498 kmemzones = malloc(allocsize);
499 if (kmemzones == NULL) {
500 return (MEMSTAT_ERROR_NOMEMORY);
501 }
502 ret = kread_symbol(kvm, X_KMEMZONES, kmemzones, allocsize, 0);
503 if (ret != 0) {
504 free(kmemzones);
505 return (ret);
506 }
507
508 kz = kmemzones;
509 for (i = 0; i < (int)nitems(memstat_malloc_zone_sizes); i++) {
510 memstat_malloc_zone_sizes[i] = kz->kz_size;
511 kz = (struct memstat_kmemzone *)((char *)kz + objsize);
512 }
513
514 free(kmemzones);
515 return (0);
516 }
517
518 size_t
memstat_malloc_zone_get_count(void)519 memstat_malloc_zone_get_count(void)
520 {
521
522 return (memstat_malloc_zone_count);
523 }
524
525 size_t
memstat_malloc_zone_get_size(size_t n)526 memstat_malloc_zone_get_size(size_t n)
527 {
528
529 if (n >= nitems(memstat_malloc_zone_sizes)) {
530 return (-1);
531 }
532
533 return (memstat_malloc_zone_sizes[n]);
534 }
535
536 int
memstat_malloc_zone_used(const struct memory_type * mtp,size_t n)537 memstat_malloc_zone_used(const struct memory_type *mtp, size_t n)
538 {
539
540 if (memstat_get_sizemask(mtp) & (1 << n))
541 return (1);
542
543 return (0);
544 }
545