1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 2013 Gleb Smirnoff <[email protected]>
5 * Copyright (c) 2010 Juniper Networks, Inc.
6 * Copyright (c) 2009 Robert N. M. Watson
7 * Copyright (c) 2009 Bjoern A. Zeeb <[email protected]>
8 * Copyright (c) 2008 Yahoo!, Inc.
9 * All rights reserved.
10 *
11 * Written by: John Baldwin <[email protected]>
12 *
13 * This software was developed by Robert N. M. Watson under contract
14 * to Juniper Networks, Inc.
15 *
16 * Redistribution and use in source and binary forms, with or without
17 * modification, are permitted provided that the following conditions
18 * are met:
19 * 1. Redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer.
21 * 2. Redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution.
24 * 3. Neither the name of the author nor the names of any co-contributors
25 * may be used to endorse or promote products derived from this software
26 * without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38 * SUCH DAMAGE.
39 */
40
41 #include <sys/cdefs.h>
42 __FBSDID("$FreeBSD$");
43
44 #include <sys/param.h>
45 #include <sys/pcpu.h>
46 #include <sys/sysctl.h>
47 #include <kvm.h>
48 #include <limits.h>
49 #include <stdlib.h>
50
51 #include "kvm_private.h"
52
53 static struct nlist kvm_pcpu_nl[] = {
54 { .n_name = "_cpuid_to_pcpu" },
55 { .n_name = "_mp_maxcpus" },
56 { .n_name = "_mp_ncpus" },
57 { .n_name = NULL },
58 };
59 #define NL_CPUID_TO_PCPU 0
60 #define NL_MP_MAXCPUS 1
61 #define NL_MP_NCPUS 2
62
63 /*
64 * Kernel per-CPU data state. We cache this stuff on the first
65 * access.
66 *
67 * XXXRW: Possibly, this (and kvmpcpu_nl) should be per-kvm_t, in case the
68 * consumer has multiple handles in flight to differently configured
69 * kernels/crashdumps.
70 */
71 static void **pcpu_data;
72 static int maxcpu;
73 static int mp_ncpus;
74
75 static int
_kvm_pcpu_init(kvm_t * kd)76 _kvm_pcpu_init(kvm_t *kd)
77 {
78 size_t len;
79 int max;
80 void *data;
81
82 if (kvm_nlist(kd, kvm_pcpu_nl) < 0)
83 return (-1);
84 if (kvm_pcpu_nl[NL_CPUID_TO_PCPU].n_value == 0) {
85 _kvm_err(kd, kd->program, "unable to find cpuid_to_pcpu");
86 return (-1);
87 }
88 if (kvm_pcpu_nl[NL_MP_MAXCPUS].n_value == 0) {
89 _kvm_err(kd, kd->program, "unable to find mp_maxcpus");
90 return (-1);
91 }
92 if (kvm_read(kd, kvm_pcpu_nl[NL_MP_MAXCPUS].n_value, &max,
93 sizeof(max)) != sizeof(max)) {
94 _kvm_err(kd, kd->program, "cannot read mp_maxcpus");
95 return (-1);
96 }
97 if (kvm_pcpu_nl[NL_MP_NCPUS].n_value == 0) {
98 _kvm_err(kd, kd->program, "unable to find mp_ncpus");
99 return (-1);
100 }
101 if (kvm_read(kd, kvm_pcpu_nl[NL_MP_NCPUS].n_value, &mp_ncpus,
102 sizeof(mp_ncpus)) != sizeof(mp_ncpus)) {
103 _kvm_err(kd, kd->program, "cannot read mp_ncpus");
104 return (-1);
105 }
106 len = max * sizeof(void *);
107 data = malloc(len);
108 if (data == NULL) {
109 _kvm_err(kd, kd->program, "out of memory");
110 return (-1);
111 }
112 if (kvm_read(kd, kvm_pcpu_nl[NL_CPUID_TO_PCPU].n_value, data, len) !=
113 (ssize_t)len) {
114 _kvm_err(kd, kd->program, "cannot read cpuid_to_pcpu array");
115 free(data);
116 return (-1);
117 }
118 pcpu_data = data;
119 maxcpu = max;
120 return (0);
121 }
122
123 static void
_kvm_pcpu_clear(void)124 _kvm_pcpu_clear(void)
125 {
126
127 maxcpu = 0;
128 free(pcpu_data);
129 pcpu_data = NULL;
130 }
131
132 void *
kvm_getpcpu(kvm_t * kd,int cpu)133 kvm_getpcpu(kvm_t *kd, int cpu)
134 {
135 char *buf;
136
137 if (kd == NULL) {
138 _kvm_pcpu_clear();
139 return (NULL);
140 }
141
142 if (maxcpu == 0)
143 if (_kvm_pcpu_init(kd) < 0)
144 return ((void *)-1);
145
146 if (cpu >= maxcpu || pcpu_data[cpu] == NULL)
147 return (NULL);
148
149 buf = malloc(sizeof(struct pcpu));
150 if (buf == NULL) {
151 _kvm_err(kd, kd->program, "out of memory");
152 return ((void *)-1);
153 }
154 if (kvm_read(kd, (uintptr_t)pcpu_data[cpu], buf,
155 sizeof(struct pcpu)) != sizeof(struct pcpu)) {
156 _kvm_err(kd, kd->program, "unable to read per-CPU data");
157 free(buf);
158 return ((void *)-1);
159 }
160 return (buf);
161 }
162
163 int
kvm_getmaxcpu(kvm_t * kd)164 kvm_getmaxcpu(kvm_t *kd)
165 {
166
167 if (kd == NULL) {
168 _kvm_pcpu_clear();
169 return (0);
170 }
171
172 if (maxcpu == 0)
173 if (_kvm_pcpu_init(kd) < 0)
174 return (-1);
175 return (maxcpu);
176 }
177
178 int
kvm_getncpus(kvm_t * kd)179 kvm_getncpus(kvm_t *kd)
180 {
181
182 if (mp_ncpus == 0)
183 if (_kvm_pcpu_init(kd) < 0)
184 return (-1);
185 return (mp_ncpus);
186 }
187
188 static int
_kvm_dpcpu_setcpu(kvm_t * kd,u_int cpu,int report_error)189 _kvm_dpcpu_setcpu(kvm_t *kd, u_int cpu, int report_error)
190 {
191
192 if (!kd->dpcpu_initialized) {
193 if (report_error)
194 _kvm_err(kd, kd->program, "%s: not initialized",
195 __func__);
196 return (-1);
197 }
198 if (cpu >= kd->dpcpu_maxcpus) {
199 if (report_error)
200 _kvm_err(kd, kd->program, "%s: CPU %u too big",
201 __func__, cpu);
202 return (-1);
203 }
204 if (kd->dpcpu_off[cpu] == 0) {
205 if (report_error)
206 _kvm_err(kd, kd->program, "%s: CPU %u not found",
207 __func__, cpu);
208 return (-1);
209 }
210 kd->dpcpu_curcpu = cpu;
211 kd->dpcpu_curoff = kd->dpcpu_off[cpu];
212 return (0);
213 }
214
215 /*
216 * Set up libkvm to handle dynamic per-CPU memory.
217 */
218 static int
_kvm_dpcpu_init(kvm_t * kd)219 _kvm_dpcpu_init(kvm_t *kd)
220 {
221 struct kvm_nlist nl[] = {
222 #define NLIST_START_SET_PCPU 0
223 { .n_name = "___start_" DPCPU_SETNAME },
224 #define NLIST_STOP_SET_PCPU 1
225 { .n_name = "___stop_" DPCPU_SETNAME },
226 #define NLIST_DPCPU_OFF 2
227 { .n_name = "_dpcpu_off" },
228 #define NLIST_MP_MAXCPUS 3
229 { .n_name = "_mp_maxcpus" },
230 { .n_name = NULL },
231 };
232 uintptr_t *dpcpu_off_buf;
233 size_t len;
234 u_int dpcpu_maxcpus;
235
236 /*
237 * XXX: This only works for native kernels for now.
238 */
239 if (!kvm_native(kd))
240 return (-1);
241
242 /*
243 * Locate and cache locations of important symbols using the internal
244 * version of _kvm_nlist, turning off initialization to avoid
245 * recursion in case of unresolveable symbols.
246 */
247 if (_kvm_nlist(kd, nl, 0) != 0)
248 return (-1);
249 if (kvm_read(kd, nl[NLIST_MP_MAXCPUS].n_value, &dpcpu_maxcpus,
250 sizeof(dpcpu_maxcpus)) != sizeof(dpcpu_maxcpus))
251 return (-1);
252 len = dpcpu_maxcpus * sizeof(*dpcpu_off_buf);
253 dpcpu_off_buf = malloc(len);
254 if (dpcpu_off_buf == NULL)
255 return (-1);
256 if (kvm_read(kd, nl[NLIST_DPCPU_OFF].n_value, dpcpu_off_buf, len) !=
257 (ssize_t)len) {
258 free(dpcpu_off_buf);
259 return (-1);
260 }
261 kd->dpcpu_start = nl[NLIST_START_SET_PCPU].n_value;
262 kd->dpcpu_stop = nl[NLIST_STOP_SET_PCPU].n_value;
263 kd->dpcpu_maxcpus = dpcpu_maxcpus;
264 kd->dpcpu_off = dpcpu_off_buf;
265 kd->dpcpu_initialized = 1;
266 (void)_kvm_dpcpu_setcpu(kd, 0, 0);
267 return (0);
268 }
269
270 /*
271 * Check whether the dpcpu module has been initialized successfully or not,
272 * initialize it if permitted.
273 */
274 int
_kvm_dpcpu_initialized(kvm_t * kd,int intialize)275 _kvm_dpcpu_initialized(kvm_t *kd, int intialize)
276 {
277
278 if (kd->dpcpu_initialized || !intialize)
279 return (kd->dpcpu_initialized);
280
281 (void)_kvm_dpcpu_init(kd);
282
283 return (kd->dpcpu_initialized);
284 }
285
286 /*
287 * Check whether the value is within the dpcpu symbol range and only if so
288 * adjust the offset relative to the current offset.
289 */
290 kvaddr_t
_kvm_dpcpu_validaddr(kvm_t * kd,kvaddr_t value)291 _kvm_dpcpu_validaddr(kvm_t *kd, kvaddr_t value)
292 {
293
294 if (value == 0)
295 return (value);
296
297 if (!kd->dpcpu_initialized)
298 return (value);
299
300 if (value < kd->dpcpu_start || value >= kd->dpcpu_stop)
301 return (value);
302
303 return (kd->dpcpu_curoff + value);
304 }
305
306 int
kvm_dpcpu_setcpu(kvm_t * kd,u_int cpu)307 kvm_dpcpu_setcpu(kvm_t *kd, u_int cpu)
308 {
309 int ret;
310
311 if (!kd->dpcpu_initialized) {
312 ret = _kvm_dpcpu_init(kd);
313 if (ret != 0) {
314 _kvm_err(kd, kd->program, "%s: init failed",
315 __func__);
316 return (ret);
317 }
318 }
319
320 return (_kvm_dpcpu_setcpu(kd, cpu, 1));
321 }
322
323 /*
324 * Obtain a per-CPU copy for given cpu from UMA_ZONE_PCPU allocation.
325 */
326 ssize_t
kvm_read_zpcpu(kvm_t * kd,u_long base,void * buf,size_t size,int cpu)327 kvm_read_zpcpu(kvm_t *kd, u_long base, void *buf, size_t size, int cpu)
328 {
329
330 if (!kvm_native(kd))
331 return (-1);
332 return (kvm_read(kd, (uintptr_t)(base + sizeof(struct pcpu) * cpu),
333 buf, size));
334 }
335
336 /*
337 * Fetch value of a counter(9).
338 */
339 uint64_t
kvm_counter_u64_fetch(kvm_t * kd,u_long base)340 kvm_counter_u64_fetch(kvm_t *kd, u_long base)
341 {
342 uint64_t r, c;
343
344 if (mp_ncpus == 0)
345 if (_kvm_pcpu_init(kd) < 0)
346 return (0);
347
348 r = 0;
349 for (int i = 0; i < mp_ncpus; i++) {
350 if (kvm_read_zpcpu(kd, base, &c, sizeof(c), i) != sizeof(c))
351 return (0);
352 r += c;
353 }
354
355 return (r);
356 }
357