1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2003-2008 Joseph Koshy
5 * Copyright (c) 2007 The FreeBSD Foundation
6 * All rights reserved.
7 *
8 * Portions of this software were developed by A. Joseph Koshy under
9 * sponsorship from the FreeBSD Foundation and Google, Inc.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
35
36 /* Support for the AMD K7 and later processors */
37
38 #include <sys/param.h>
39 #include <sys/lock.h>
40 #include <sys/malloc.h>
41 #include <sys/mutex.h>
42 #include <sys/pcpu.h>
43 #include <sys/pmc.h>
44 #include <sys/pmckern.h>
45 #include <sys/smp.h>
46 #include <sys/systm.h>
47
48 #include <machine/cpu.h>
49 #include <machine/cpufunc.h>
50 #include <machine/md_var.h>
51 #include <machine/specialreg.h>
52
53 #ifdef HWPMC_DEBUG
54 enum pmc_class amd_pmc_class;
55 #endif
56
57 #define OVERFLOW_WAIT_COUNT 50
58
59 DPCPU_DEFINE_STATIC(uint32_t, nmi_counter);
60
61 /* AMD K7 & K8 PMCs */
62 struct amd_descr {
63 struct pmc_descr pm_descr; /* "base class" */
64 uint32_t pm_evsel; /* address of EVSEL register */
65 uint32_t pm_perfctr; /* address of PERFCTR register */
66 };
67
68 static struct amd_descr amd_pmcdesc[AMD_NPMCS] =
69 {
70 {
71 .pm_descr =
72 {
73 .pd_name = "",
74 .pd_class = -1,
75 .pd_caps = AMD_PMC_CAPS,
76 .pd_width = 48
77 },
78 .pm_evsel = AMD_PMC_EVSEL_0,
79 .pm_perfctr = AMD_PMC_PERFCTR_0
80 },
81 {
82 .pm_descr =
83 {
84 .pd_name = "",
85 .pd_class = -1,
86 .pd_caps = AMD_PMC_CAPS,
87 .pd_width = 48
88 },
89 .pm_evsel = AMD_PMC_EVSEL_1,
90 .pm_perfctr = AMD_PMC_PERFCTR_1
91 },
92 {
93 .pm_descr =
94 {
95 .pd_name = "",
96 .pd_class = -1,
97 .pd_caps = AMD_PMC_CAPS,
98 .pd_width = 48
99 },
100 .pm_evsel = AMD_PMC_EVSEL_2,
101 .pm_perfctr = AMD_PMC_PERFCTR_2
102 },
103 {
104 .pm_descr =
105 {
106 .pd_name = "",
107 .pd_class = -1,
108 .pd_caps = AMD_PMC_CAPS,
109 .pd_width = 48
110 },
111 .pm_evsel = AMD_PMC_EVSEL_3,
112 .pm_perfctr = AMD_PMC_PERFCTR_3
113 },
114 {
115 .pm_descr =
116 {
117 .pd_name = "",
118 .pd_class = -1,
119 .pd_caps = AMD_PMC_CAPS,
120 .pd_width = 48
121 },
122 .pm_evsel = AMD_PMC_EVSEL_4,
123 .pm_perfctr = AMD_PMC_PERFCTR_4
124 },
125 {
126 .pm_descr =
127 {
128 .pd_name = "",
129 .pd_class = -1,
130 .pd_caps = AMD_PMC_CAPS,
131 .pd_width = 48
132 },
133 .pm_evsel = AMD_PMC_EVSEL_5,
134 .pm_perfctr = AMD_PMC_PERFCTR_5
135 },
136 {
137 .pm_descr =
138 {
139 .pd_name = "",
140 .pd_class = -1,
141 .pd_caps = AMD_PMC_CAPS,
142 .pd_width = 48
143 },
144 .pm_evsel = AMD_PMC_EVSEL_EP_L3_0,
145 .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_0
146 },
147 {
148 .pm_descr =
149 {
150 .pd_name = "",
151 .pd_class = -1,
152 .pd_caps = AMD_PMC_CAPS,
153 .pd_width = 48
154 },
155 .pm_evsel = AMD_PMC_EVSEL_EP_L3_1,
156 .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_1
157 },
158 {
159 .pm_descr =
160 {
161 .pd_name = "",
162 .pd_class = -1,
163 .pd_caps = AMD_PMC_CAPS,
164 .pd_width = 48
165 },
166 .pm_evsel = AMD_PMC_EVSEL_EP_L3_2,
167 .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_2
168 },
169 {
170 .pm_descr =
171 {
172 .pd_name = "",
173 .pd_class = -1,
174 .pd_caps = AMD_PMC_CAPS,
175 .pd_width = 48
176 },
177 .pm_evsel = AMD_PMC_EVSEL_EP_L3_3,
178 .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_3
179 },
180 {
181 .pm_descr =
182 {
183 .pd_name = "",
184 .pd_class = -1,
185 .pd_caps = AMD_PMC_CAPS,
186 .pd_width = 48
187 },
188 .pm_evsel = AMD_PMC_EVSEL_EP_L3_4,
189 .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_4
190 },
191 {
192 .pm_descr =
193 {
194 .pd_name = "",
195 .pd_class = -1,
196 .pd_caps = AMD_PMC_CAPS,
197 .pd_width = 48
198 },
199 .pm_evsel = AMD_PMC_EVSEL_EP_L3_5,
200 .pm_perfctr = AMD_PMC_PERFCTR_EP_L3_5
201 },
202 {
203 .pm_descr =
204 {
205 .pd_name = "",
206 .pd_class = -1,
207 .pd_caps = AMD_PMC_CAPS,
208 .pd_width = 48
209 },
210 .pm_evsel = AMD_PMC_EVSEL_EP_DF_0,
211 .pm_perfctr = AMD_PMC_PERFCTR_EP_DF_0
212 },
213 {
214 .pm_descr =
215 {
216 .pd_name = "",
217 .pd_class = -1,
218 .pd_caps = AMD_PMC_CAPS,
219 .pd_width = 48
220 },
221 .pm_evsel = AMD_PMC_EVSEL_EP_DF_1,
222 .pm_perfctr = AMD_PMC_PERFCTR_EP_DF_1
223 },
224 {
225 .pm_descr =
226 {
227 .pd_name = "",
228 .pd_class = -1,
229 .pd_caps = AMD_PMC_CAPS,
230 .pd_width = 48
231 },
232 .pm_evsel = AMD_PMC_EVSEL_EP_DF_2,
233 .pm_perfctr = AMD_PMC_PERFCTR_EP_DF_2
234 },
235 {
236 .pm_descr =
237 {
238 .pd_name = "",
239 .pd_class = -1,
240 .pd_caps = AMD_PMC_CAPS,
241 .pd_width = 48
242 },
243 .pm_evsel = AMD_PMC_EVSEL_EP_DF_3,
244 .pm_perfctr = AMD_PMC_PERFCTR_EP_DF_3
245 }
246 };
247
248 struct amd_event_code_map {
249 enum pmc_event pe_ev; /* enum value */
250 uint16_t pe_code; /* encoded event mask */
251 uint8_t pe_mask; /* bits allowed in unit mask */
252 };
253
254 const struct amd_event_code_map amd_event_codes[] = {
255 #if defined(__i386__) /* 32 bit Athlon (K7) only */
256 { PMC_EV_K7_DC_ACCESSES, 0x40, 0 },
257 { PMC_EV_K7_DC_MISSES, 0x41, 0 },
258 { PMC_EV_K7_DC_REFILLS_FROM_L2, 0x42, AMD_PMC_UNITMASK_MOESI },
259 { PMC_EV_K7_DC_REFILLS_FROM_SYSTEM, 0x43, AMD_PMC_UNITMASK_MOESI },
260 { PMC_EV_K7_DC_WRITEBACKS, 0x44, AMD_PMC_UNITMASK_MOESI },
261 { PMC_EV_K7_L1_DTLB_MISS_AND_L2_DTLB_HITS, 0x45, 0 },
262 { PMC_EV_K7_L1_AND_L2_DTLB_MISSES, 0x46, 0 },
263 { PMC_EV_K7_MISALIGNED_REFERENCES, 0x47, 0 },
264
265 { PMC_EV_K7_IC_FETCHES, 0x80, 0 },
266 { PMC_EV_K7_IC_MISSES, 0x81, 0 },
267
268 { PMC_EV_K7_L1_ITLB_MISSES, 0x84, 0 },
269 { PMC_EV_K7_L1_L2_ITLB_MISSES, 0x85, 0 },
270
271 { PMC_EV_K7_RETIRED_INSTRUCTIONS, 0xC0, 0 },
272 { PMC_EV_K7_RETIRED_OPS, 0xC1, 0 },
273 { PMC_EV_K7_RETIRED_BRANCHES, 0xC2, 0 },
274 { PMC_EV_K7_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0 },
275 { PMC_EV_K7_RETIRED_TAKEN_BRANCHES, 0xC4, 0 },
276 { PMC_EV_K7_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0 },
277 { PMC_EV_K7_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0 },
278 { PMC_EV_K7_RETIRED_RESYNC_BRANCHES, 0xC7, 0 },
279 { PMC_EV_K7_INTERRUPTS_MASKED_CYCLES, 0xCD, 0 },
280 { PMC_EV_K7_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0 },
281 { PMC_EV_K7_HARDWARE_INTERRUPTS, 0xCF, 0 },
282 #endif
283
284 { PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F },
285 { PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 },
286 { PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 },
287
288 { PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F },
289 { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE,
290 0x21, 0x00 },
291 { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 },
292 { PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 },
293 { PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 },
294 { PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 },
295 { PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 },
296 { PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 },
297
298 { PMC_EV_K8_DC_ACCESS, 0x40, 0x00 },
299 { PMC_EV_K8_DC_MISS, 0x41, 0x00 },
300 { PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F },
301 { PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F },
302 { PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F },
303 { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 },
304 { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 },
305 { PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 },
306 { PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 },
307 { PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 },
308 { PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 },
309 { PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 },
310 { PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 },
311
312 { PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 },
313 { PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F },
314 { PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 },
315 { PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 },
316
317 { PMC_EV_K8_IC_FETCH, 0x80, 0x00 },
318 { PMC_EV_K8_IC_MISS, 0x81, 0x00 },
319 { PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 },
320 { PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 },
321 { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 },
322 { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 },
323 { PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 },
324 { PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 },
325 { PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 },
326 { PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 },
327
328 { PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 },
329 { PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 },
330 { PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 },
331 { PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 },
332 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 },
333 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 },
334 { PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 },
335 { PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 },
336 { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 },
337 { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 },
338 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE,
339 0xCA, 0x00 },
340 { PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F },
341 { PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS,
342 0xCC, 0x07 },
343 { PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 },
344 { PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 },
345 { PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 },
346
347 { PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 },
348 { PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 },
349 { PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE,
350 0xD2, 0x00 },
351 { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 },
352 { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 },
353 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL,
354 0xD5, 0x00 },
355 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL,
356 0xD6, 0x00 },
357 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 },
358 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 },
359 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET,
360 0xD9, 0x00 },
361 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING,
362 0xDA, 0x00 },
363 { PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F },
364 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 },
365 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 },
366 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 },
367 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 },
368
369 { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 },
370 { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 },
371 { PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED,
372 0xE2, 0x00 },
373 { PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 },
374 { PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F },
375 { PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F },
376 { PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F },
377 { PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F },
378 { PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F },
379 { PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F }
380
381 };
382
383 const int amd_event_codes_size = nitems(amd_event_codes);
384
385 /*
386 * Per-processor information
387 */
388
389 struct amd_cpu {
390 struct pmc_hw pc_amdpmcs[AMD_NPMCS];
391 };
392
393 static struct amd_cpu **amd_pcpu;
394
395 /*
396 * read a pmc register
397 */
398
399 static int
amd_read_pmc(int cpu,int ri,pmc_value_t * v)400 amd_read_pmc(int cpu, int ri, pmc_value_t *v)
401 {
402 enum pmc_mode mode;
403 const struct amd_descr *pd;
404 struct pmc *pm;
405 pmc_value_t tmp;
406
407 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
408 ("[amd,%d] illegal CPU value %d", __LINE__, cpu));
409 KASSERT(ri >= 0 && ri < AMD_NPMCS,
410 ("[amd,%d] illegal row-index %d", __LINE__, ri));
411 KASSERT(amd_pcpu[cpu],
412 ("[amd,%d] null per-cpu, cpu %d", __LINE__, cpu));
413
414 pm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc;
415 pd = &amd_pmcdesc[ri];
416
417 KASSERT(pm != NULL,
418 ("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__,
419 cpu, ri));
420
421 mode = PMC_TO_MODE(pm);
422
423 PMCDBG2(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class);
424
425 #ifdef HWPMC_DEBUG
426 KASSERT(pd->pm_descr.pd_class == amd_pmc_class,
427 ("[amd,%d] unknown PMC class (%d)", __LINE__,
428 pd->pm_descr.pd_class));
429 #endif
430
431 tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */
432 PMCDBG2(MDP,REA,2,"amd-read (pre-munge) id=%d -> %jd", ri, tmp);
433 if (PMC_IS_SAMPLING_MODE(mode)) {
434 /*
435 * Clamp value to 0 if the counter just overflowed,
436 * otherwise the returned reload count would wrap to a
437 * huge value.
438 */
439 if ((tmp & (1ULL << 47)) == 0)
440 tmp = 0;
441 else {
442 /* Sign extend 48 bit value to 64 bits. */
443 tmp = (pmc_value_t) ((int64_t)(tmp << 16) >> 16);
444 tmp = AMD_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
445 }
446 }
447 *v = tmp;
448
449 PMCDBG2(MDP,REA,2,"amd-read (post-munge) id=%d -> %jd", ri, *v);
450
451 return 0;
452 }
453
454 /*
455 * Write a PMC MSR.
456 */
457
458 static int
amd_write_pmc(int cpu,int ri,pmc_value_t v)459 amd_write_pmc(int cpu, int ri, pmc_value_t v)
460 {
461 const struct amd_descr *pd;
462 enum pmc_mode mode;
463 struct pmc *pm;
464
465 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
466 ("[amd,%d] illegal CPU value %d", __LINE__, cpu));
467 KASSERT(ri >= 0 && ri < AMD_NPMCS,
468 ("[amd,%d] illegal row-index %d", __LINE__, ri));
469
470 pm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc;
471 pd = &amd_pmcdesc[ri];
472
473 KASSERT(pm != NULL,
474 ("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__,
475 cpu, ri));
476
477 mode = PMC_TO_MODE(pm);
478
479 #ifdef HWPMC_DEBUG
480 KASSERT(pd->pm_descr.pd_class == amd_pmc_class,
481 ("[amd,%d] unknown PMC class (%d)", __LINE__,
482 pd->pm_descr.pd_class));
483 #endif
484
485 /* use 2's complement of the count for sampling mode PMCs */
486 if (PMC_IS_SAMPLING_MODE(mode))
487 v = AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
488
489 PMCDBG3(MDP,WRI,1,"amd-write cpu=%d ri=%d v=%jx", cpu, ri, v);
490
491 /* write the PMC value */
492 wrmsr(pd->pm_perfctr, v);
493 return 0;
494 }
495
496 /*
497 * configure hardware pmc according to the configuration recorded in
498 * pmc 'pm'.
499 */
500
501 static int
amd_config_pmc(int cpu,int ri,struct pmc * pm)502 amd_config_pmc(int cpu, int ri, struct pmc *pm)
503 {
504 struct pmc_hw *phw;
505
506 PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
507
508 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
509 ("[amd,%d] illegal CPU value %d", __LINE__, cpu));
510 KASSERT(ri >= 0 && ri < AMD_NPMCS,
511 ("[amd,%d] illegal row-index %d", __LINE__, ri));
512
513 phw = &amd_pcpu[cpu]->pc_amdpmcs[ri];
514
515 KASSERT(pm == NULL || phw->phw_pmc == NULL,
516 ("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured",
517 __LINE__, pm, phw->phw_pmc));
518
519 phw->phw_pmc = pm;
520 return 0;
521 }
522
523 /*
524 * Retrieve a configured PMC pointer from hardware state.
525 */
526
527 static int
amd_get_config(int cpu,int ri,struct pmc ** ppm)528 amd_get_config(int cpu, int ri, struct pmc **ppm)
529 {
530 *ppm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc;
531
532 return 0;
533 }
534
535 /*
536 * Machine dependent actions taken during the context switch in of a
537 * thread.
538 */
539
540 static int
amd_switch_in(struct pmc_cpu * pc,struct pmc_process * pp)541 amd_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
542 {
543 (void) pc;
544
545 PMCDBG3(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp,
546 (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0);
547
548 /* enable the RDPMC instruction if needed */
549 if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS)
550 load_cr4(rcr4() | CR4_PCE);
551
552 return 0;
553 }
554
555 /*
556 * Machine dependent actions taken during the context switch out of a
557 * thread.
558 */
559
560 static int
amd_switch_out(struct pmc_cpu * pc,struct pmc_process * pp)561 amd_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
562 {
563 (void) pc;
564 (void) pp; /* can be NULL */
565
566 PMCDBG3(MDP,SWO,1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ?
567 (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) == 1 : 0);
568
569 /* always turn off the RDPMC instruction */
570 load_cr4(rcr4() & ~CR4_PCE);
571
572 return 0;
573 }
574
575 /*
576 * Check if a given allocation is feasible.
577 */
578
579 static int
amd_allocate_pmc(int cpu,int ri,struct pmc * pm,const struct pmc_op_pmcallocate * a)580 amd_allocate_pmc(int cpu, int ri, struct pmc *pm,
581 const struct pmc_op_pmcallocate *a)
582 {
583 int i;
584 uint64_t allowed_unitmask, caps, config, unitmask;
585 enum pmc_event pe;
586 const struct pmc_descr *pd;
587
588 (void) cpu;
589
590 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
591 ("[amd,%d] illegal CPU value %d", __LINE__, cpu));
592 KASSERT(ri >= 0 && ri < AMD_NPMCS,
593 ("[amd,%d] illegal row index %d", __LINE__, ri));
594
595 pd = &amd_pmcdesc[ri].pm_descr;
596
597 /* check class match */
598 if (pd->pd_class != a->pm_class)
599 return EINVAL;
600
601 caps = pm->pm_caps;
602
603 PMCDBG2(MDP,ALL,1,"amd-allocate ri=%d caps=0x%x", ri, caps);
604
605 if((ri >= 0 && ri < 6) && !(a->pm_md.pm_amd.pm_amd_sub_class == PMC_AMD_SUB_CLASS_CORE))
606 return EINVAL;
607 if((ri >= 6 && ri < 12) && !(a->pm_md.pm_amd.pm_amd_sub_class == PMC_AMD_SUB_CLASS_L3_CACHE))
608 return EINVAL;
609 if((ri >= 12 && ri < 16) && !(a->pm_md.pm_amd.pm_amd_sub_class == PMC_AMD_SUB_CLASS_DATA_FABRIC))
610 return EINVAL;
611
612 if ((pd->pd_caps & caps) != caps)
613 return EPERM;
614 if (strlen(pmc_cpuid) != 0) {
615 pm->pm_md.pm_amd.pm_amd_evsel =
616 a->pm_md.pm_amd.pm_amd_config;
617 PMCDBG2(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, a->pm_md.pm_amd.pm_amd_config);
618 return (0);
619 }
620
621 pe = a->pm_ev;
622
623 /* map ev to the correct event mask code */
624 config = allowed_unitmask = 0;
625 for (i = 0; i < amd_event_codes_size; i++)
626 if (amd_event_codes[i].pe_ev == pe) {
627 config =
628 AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code);
629 allowed_unitmask =
630 AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask);
631 break;
632 }
633 if (i == amd_event_codes_size)
634 return EINVAL;
635
636 unitmask = a->pm_md.pm_amd.pm_amd_config & AMD_PMC_UNITMASK;
637 if (unitmask & ~allowed_unitmask) /* disallow reserved bits */
638 return EINVAL;
639
640 if (unitmask && (caps & PMC_CAP_QUALIFIER))
641 config |= unitmask;
642
643 if (caps & PMC_CAP_THRESHOLD)
644 config |= a->pm_md.pm_amd.pm_amd_config & AMD_PMC_COUNTERMASK;
645
646 /* set at least one of the 'usr' or 'os' caps */
647 if (caps & PMC_CAP_USER)
648 config |= AMD_PMC_USR;
649 if (caps & PMC_CAP_SYSTEM)
650 config |= AMD_PMC_OS;
651 if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0)
652 config |= (AMD_PMC_USR|AMD_PMC_OS);
653
654 if (caps & PMC_CAP_EDGE)
655 config |= AMD_PMC_EDGE;
656 if (caps & PMC_CAP_INVERT)
657 config |= AMD_PMC_INVERT;
658 if (caps & PMC_CAP_INTERRUPT)
659 config |= AMD_PMC_INT;
660
661 pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */
662
663 PMCDBG2(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, config);
664
665 return 0;
666 }
667
668 /*
669 * Release machine dependent state associated with a PMC. This is a
670 * no-op on this architecture.
671 *
672 */
673
674 /* ARGSUSED0 */
675 static int
amd_release_pmc(int cpu,int ri,struct pmc * pmc)676 amd_release_pmc(int cpu, int ri, struct pmc *pmc)
677 {
678 #ifdef HWPMC_DEBUG
679 const struct amd_descr *pd;
680 #endif
681 struct pmc_hw *phw __diagused;
682
683 (void) pmc;
684
685 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
686 ("[amd,%d] illegal CPU value %d", __LINE__, cpu));
687 KASSERT(ri >= 0 && ri < AMD_NPMCS,
688 ("[amd,%d] illegal row-index %d", __LINE__, ri));
689
690 phw = &amd_pcpu[cpu]->pc_amdpmcs[ri];
691
692 KASSERT(phw->phw_pmc == NULL,
693 ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
694
695 #ifdef HWPMC_DEBUG
696 pd = &amd_pmcdesc[ri];
697 if (pd->pm_descr.pd_class == amd_pmc_class)
698 KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel),
699 ("[amd,%d] PMC %d released while active", __LINE__, ri));
700 #endif
701
702 return 0;
703 }
704
705 /*
706 * start a PMC.
707 */
708
709 static int
amd_start_pmc(int cpu,int ri)710 amd_start_pmc(int cpu, int ri)
711 {
712 uint64_t config;
713 struct pmc *pm;
714 struct pmc_hw *phw;
715 const struct amd_descr *pd;
716
717 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
718 ("[amd,%d] illegal CPU value %d", __LINE__, cpu));
719 KASSERT(ri >= 0 && ri < AMD_NPMCS,
720 ("[amd,%d] illegal row-index %d", __LINE__, ri));
721
722 phw = &amd_pcpu[cpu]->pc_amdpmcs[ri];
723 pm = phw->phw_pmc;
724 pd = &amd_pmcdesc[ri];
725
726 KASSERT(pm != NULL,
727 ("[amd,%d] starting cpu%d,pmc%d with null pmc record", __LINE__,
728 cpu, ri));
729
730 PMCDBG2(MDP,STA,1,"amd-start cpu=%d ri=%d", cpu, ri);
731
732 KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel),
733 ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__,
734 ri, cpu, pd->pm_descr.pd_name));
735
736 /* turn on the PMC ENABLE bit */
737 config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE;
738
739 PMCDBG1(MDP,STA,2,"amd-start config=0x%x", config);
740
741 wrmsr(pd->pm_evsel, config);
742 return 0;
743 }
744
745 /*
746 * Stop a PMC.
747 */
748
749 static int
amd_stop_pmc(int cpu,int ri)750 amd_stop_pmc(int cpu, int ri)
751 {
752 struct pmc *pm;
753 struct pmc_hw *phw;
754 const struct amd_descr *pd;
755 uint64_t config;
756 int i;
757
758 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
759 ("[amd,%d] illegal CPU value %d", __LINE__, cpu));
760 KASSERT(ri >= 0 && ri < AMD_NPMCS,
761 ("[amd,%d] illegal row-index %d", __LINE__, ri));
762
763 phw = &amd_pcpu[cpu]->pc_amdpmcs[ri];
764 pm = phw->phw_pmc;
765 pd = &amd_pmcdesc[ri];
766
767 KASSERT(pm != NULL,
768 ("[amd,%d] cpu%d,pmc%d no PMC to stop", __LINE__,
769 cpu, ri));
770 KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel),
771 ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped",
772 __LINE__, ri, cpu, pd->pm_descr.pd_name));
773
774 PMCDBG1(MDP,STO,1,"amd-stop ri=%d", ri);
775
776 /* turn off the PMC ENABLE bit */
777 config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE;
778 wrmsr(pd->pm_evsel, config);
779
780 /*
781 * Due to NMI latency on newer AMD processors
782 * NMI interrupts are ignored, which leads to
783 * panic or messages based on kernel configuraiton
784 */
785
786 /* Wait for the count to be reset */
787 for (i = 0; i < OVERFLOW_WAIT_COUNT; i++) {
788 if (rdmsr(pd->pm_perfctr) & (1 << (pd->pm_descr.pd_width - 1)))
789 break;
790
791 DELAY(1);
792 }
793
794 return 0;
795 }
796
797 /*
798 * Interrupt handler. This function needs to return '1' if the
799 * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed
800 * to sleep or do anything a 'fast' interrupt handler is not allowed
801 * to do.
802 */
803
804 static int
amd_intr(struct trapframe * tf)805 amd_intr(struct trapframe *tf)
806 {
807 int i, error, retval, cpu;
808 uint64_t config, evsel, perfctr;
809 struct pmc *pm;
810 struct amd_cpu *pac;
811 pmc_value_t v;
812 uint32_t active = 0, count = 0;
813
814 cpu = curcpu;
815 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
816 ("[amd,%d] out of range CPU %d", __LINE__, cpu));
817
818 PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf,
819 TRAPF_USERMODE(tf));
820
821 retval = 0;
822
823 pac = amd_pcpu[cpu];
824
825 /*
826 * look for all PMCs that have interrupted:
827 * - look for a running, sampling PMC which has overflowed
828 * and which has a valid 'struct pmc' association
829 *
830 * If found, we call a helper to process the interrupt.
831 *
832 * PMCs interrupting at the same time are collapsed into
833 * a single interrupt. Check all the valid pmcs for
834 * overflow.
835 */
836
837 for (i = 0; i < AMD_CORE_NPMCS; i++) {
838
839 if ((pm = pac->pc_amdpmcs[i].phw_pmc) == NULL ||
840 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
841 continue;
842 }
843
844 /* Consider pmc with valid handle as active */
845 active++;
846
847 if (!AMD_PMC_HAS_OVERFLOWED(i))
848 continue;
849
850 retval = 1; /* Found an interrupting PMC. */
851
852 if (pm->pm_state != PMC_STATE_RUNNING)
853 continue;
854
855 /* Stop the PMC, reload count. */
856 evsel = amd_pmcdesc[i].pm_evsel;
857 perfctr = amd_pmcdesc[i].pm_perfctr;
858 v = pm->pm_sc.pm_reloadcount;
859 config = rdmsr(evsel);
860
861 KASSERT((config & ~AMD_PMC_ENABLE) ==
862 (pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE),
863 ("[amd,%d] config mismatch reg=0x%jx pm=0x%jx", __LINE__,
864 (uintmax_t)config, (uintmax_t)pm->pm_md.pm_amd.pm_amd_evsel));
865
866 wrmsr(evsel, config & ~AMD_PMC_ENABLE);
867 wrmsr(perfctr, AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v));
868
869 /* Restart the counter if logging succeeded. */
870 error = pmc_process_interrupt(PMC_HR, pm, tf);
871 if (error == 0)
872 wrmsr(evsel, config);
873 }
874
875 /*
876 * Due to NMI latency, there can be a scenario in which
877 * multiple pmcs gets serviced in an earlier NMI and we
878 * do not find an overflow in the subsequent NMI.
879 *
880 * For such cases we keep a per-cpu count of active NMIs
881 * and compare it with min(active pmcs, 2) to determine
882 * if this NMI was for a pmc overflow which was serviced
883 * in an earlier request or should be ignored.
884 */
885
886 if (retval) {
887 DPCPU_SET(nmi_counter, min(2, active));
888 } else {
889 if ((count = DPCPU_GET(nmi_counter))) {
890 retval = 1;
891 DPCPU_SET(nmi_counter, --count);
892 }
893 }
894
895 if (retval)
896 counter_u64_add(pmc_stats.pm_intr_processed, 1);
897 else
898 counter_u64_add(pmc_stats.pm_intr_ignored, 1);
899
900 PMCDBG1(MDP,INT,2, "retval=%d", retval);
901 return (retval);
902 }
903
904 /*
905 * describe a PMC
906 */
907 static int
amd_describe(int cpu,int ri,struct pmc_info * pi,struct pmc ** ppmc)908 amd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
909 {
910 int error;
911 size_t copied;
912 const struct amd_descr *pd;
913 struct pmc_hw *phw;
914
915 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
916 ("[amd,%d] illegal CPU %d", __LINE__, cpu));
917 KASSERT(ri >= 0 && ri < AMD_NPMCS,
918 ("[amd,%d] row-index %d out of range", __LINE__, ri));
919
920 phw = &amd_pcpu[cpu]->pc_amdpmcs[ri];
921 pd = &amd_pmcdesc[ri];
922
923 if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name,
924 PMC_NAME_MAX, &copied)) != 0)
925 return error;
926
927 pi->pm_class = pd->pm_descr.pd_class;
928
929 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
930 pi->pm_enabled = TRUE;
931 *ppmc = phw->phw_pmc;
932 } else {
933 pi->pm_enabled = FALSE;
934 *ppmc = NULL;
935 }
936
937 return 0;
938 }
939
940 /*
941 * i386 specific entry points
942 */
943
944 /*
945 * return the MSR address of the given PMC.
946 */
947
948 static int
amd_get_msr(int ri,uint32_t * msr)949 amd_get_msr(int ri, uint32_t *msr)
950 {
951 KASSERT(ri >= 0 && ri < AMD_NPMCS,
952 ("[amd,%d] ri %d out of range", __LINE__, ri));
953
954 *msr = amd_pmcdesc[ri].pm_perfctr - AMD_PMC_PERFCTR_0;
955
956 return (0);
957 }
958
959 /*
960 * processor dependent initialization.
961 */
962
963 static int
amd_pcpu_init(struct pmc_mdep * md,int cpu)964 amd_pcpu_init(struct pmc_mdep *md, int cpu)
965 {
966 int classindex, first_ri, n;
967 struct pmc_cpu *pc;
968 struct amd_cpu *pac;
969 struct pmc_hw *phw;
970
971 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
972 ("[amd,%d] insane cpu number %d", __LINE__, cpu));
973
974 PMCDBG1(MDP,INI,1,"amd-init cpu=%d", cpu);
975
976 amd_pcpu[cpu] = pac = malloc(sizeof(struct amd_cpu), M_PMC,
977 M_WAITOK|M_ZERO);
978
979 /*
980 * Set the content of the hardware descriptors to a known
981 * state and initialize pointers in the MI per-cpu descriptor.
982 */
983 pc = pmc_pcpu[cpu];
984 #if defined(__amd64__)
985 classindex = PMC_MDEP_CLASS_INDEX_K8;
986 #elif defined(__i386__)
987 classindex = md->pmd_cputype == PMC_CPU_AMD_K8 ?
988 PMC_MDEP_CLASS_INDEX_K8 : PMC_MDEP_CLASS_INDEX_K7;
989 #endif
990 first_ri = md->pmd_classdep[classindex].pcd_ri;
991
992 KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu pointer", __LINE__));
993
994 for (n = 0, phw = pac->pc_amdpmcs; n < AMD_NPMCS; n++, phw++) {
995 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
996 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
997 phw->phw_pmc = NULL;
998 pc->pc_hwpmcs[n + first_ri] = phw;
999 }
1000
1001 return (0);
1002 }
1003
1004
1005 /*
1006 * processor dependent cleanup prior to the KLD
1007 * being unloaded
1008 */
1009
1010 static int
amd_pcpu_fini(struct pmc_mdep * md,int cpu)1011 amd_pcpu_fini(struct pmc_mdep *md, int cpu)
1012 {
1013 int classindex, first_ri, i;
1014 uint32_t evsel;
1015 struct pmc_cpu *pc;
1016 struct amd_cpu *pac;
1017
1018 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
1019 ("[amd,%d] insane cpu number (%d)", __LINE__, cpu));
1020
1021 PMCDBG1(MDP,INI,1,"amd-cleanup cpu=%d", cpu);
1022
1023 /*
1024 * First, turn off all PMCs on this CPU.
1025 */
1026 for (i = 0; i < 4; i++) { /* XXX this loop is now not needed */
1027 evsel = rdmsr(AMD_PMC_EVSEL_0 + i);
1028 evsel &= ~AMD_PMC_ENABLE;
1029 wrmsr(AMD_PMC_EVSEL_0 + i, evsel);
1030 }
1031
1032 /*
1033 * Next, free up allocated space.
1034 */
1035 if ((pac = amd_pcpu[cpu]) == NULL)
1036 return (0);
1037
1038 amd_pcpu[cpu] = NULL;
1039
1040 #ifdef HWPMC_DEBUG
1041 for (i = 0; i < AMD_NPMCS; i++) {
1042 KASSERT(pac->pc_amdpmcs[i].phw_pmc == NULL,
1043 ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i));
1044 KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + i),
1045 ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i));
1046 }
1047 #endif
1048
1049 pc = pmc_pcpu[cpu];
1050 KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu state", __LINE__));
1051
1052 #if defined(__amd64__)
1053 classindex = PMC_MDEP_CLASS_INDEX_K8;
1054 #elif defined(__i386__)
1055 classindex = md->pmd_cputype == PMC_CPU_AMD_K8 ? PMC_MDEP_CLASS_INDEX_K8 :
1056 PMC_MDEP_CLASS_INDEX_K7;
1057 #endif
1058 first_ri = md->pmd_classdep[classindex].pcd_ri;
1059
1060 /*
1061 * Reset pointers in the MI 'per-cpu' state.
1062 */
1063 for (i = 0; i < AMD_NPMCS; i++) {
1064 pc->pc_hwpmcs[i + first_ri] = NULL;
1065 }
1066
1067
1068 free(pac, M_PMC);
1069
1070 return (0);
1071 }
1072
1073 /*
1074 * Initialize ourselves.
1075 */
1076
1077 struct pmc_mdep *
pmc_amd_initialize(void)1078 pmc_amd_initialize(void)
1079 {
1080 int classindex, error, i, ncpus;
1081 struct pmc_classdep *pcd;
1082 enum pmc_cputype cputype;
1083 struct pmc_mdep *pmc_mdep;
1084 enum pmc_class class;
1085 int family, model, stepping;
1086 char *name;
1087
1088 /*
1089 * The presence of hardware performance counters on the AMD
1090 * Athlon, Duron or later processors, is _not_ indicated by
1091 * any of the processor feature flags set by the 'CPUID'
1092 * instruction, so we only check the 'instruction family'
1093 * field returned by CPUID for instruction family >= 6.
1094 */
1095
1096 name = NULL;
1097 family = CPUID_TO_FAMILY(cpu_id);
1098 model = CPUID_TO_MODEL(cpu_id);
1099 stepping = CPUID_TO_STEPPING(cpu_id);
1100
1101 if (family == 0x18)
1102 snprintf(pmc_cpuid, sizeof(pmc_cpuid), "HygonGenuine-%d-%02X-%X",
1103 family, model, stepping);
1104 else
1105 snprintf(pmc_cpuid, sizeof(pmc_cpuid), "AuthenticAMD-%d-%02X-%X",
1106 family, model, stepping);
1107
1108 switch (cpu_id & 0xF00) {
1109 #if defined(__i386__)
1110 case 0x600: /* Athlon(tm) processor */
1111 classindex = PMC_MDEP_CLASS_INDEX_K7;
1112 cputype = PMC_CPU_AMD_K7;
1113 class = PMC_CLASS_K7;
1114 name = "K7";
1115 break;
1116 #endif
1117 case 0xF00: /* Athlon64/Opteron processor */
1118 classindex = PMC_MDEP_CLASS_INDEX_K8;
1119 cputype = PMC_CPU_AMD_K8;
1120 class = PMC_CLASS_K8;
1121 name = "K8";
1122 break;
1123
1124 default:
1125 (void) printf("pmc: Unknown AMD CPU %x %d-%d.\n", cpu_id, (cpu_id & 0xF00) >> 8, model);
1126 return NULL;
1127 }
1128
1129 #ifdef HWPMC_DEBUG
1130 amd_pmc_class = class;
1131 #endif
1132
1133 /*
1134 * Allocate space for pointers to PMC HW descriptors and for
1135 * the MDEP structure used by MI code.
1136 */
1137 amd_pcpu = malloc(sizeof(struct amd_cpu *) * pmc_cpu_max(), M_PMC,
1138 M_WAITOK|M_ZERO);
1139
1140 /*
1141 * These processors have two classes of PMCs: the TSC and
1142 * programmable PMCs.
1143 */
1144 pmc_mdep = pmc_mdep_alloc(2);
1145
1146 pmc_mdep->pmd_cputype = cputype;
1147
1148 ncpus = pmc_cpu_max();
1149
1150 /* Initialize the TSC. */
1151 error = pmc_tsc_initialize(pmc_mdep, ncpus);
1152 if (error)
1153 goto error;
1154
1155 /* Initialize AMD K7 and K8 PMC handling. */
1156 pcd = &pmc_mdep->pmd_classdep[classindex];
1157
1158 pcd->pcd_caps = AMD_PMC_CAPS;
1159 pcd->pcd_class = class;
1160 pcd->pcd_num = AMD_NPMCS;
1161 pcd->pcd_ri = pmc_mdep->pmd_npmc;
1162 pcd->pcd_width = 48;
1163
1164 /* fill in the correct pmc name and class */
1165 for (i = 0; i < AMD_NPMCS; i++) {
1166 (void) snprintf(amd_pmcdesc[i].pm_descr.pd_name,
1167 sizeof(amd_pmcdesc[i].pm_descr.pd_name), "%s-%d",
1168 name, i);
1169 amd_pmcdesc[i].pm_descr.pd_class = class;
1170 }
1171
1172 pcd->pcd_allocate_pmc = amd_allocate_pmc;
1173 pcd->pcd_config_pmc = amd_config_pmc;
1174 pcd->pcd_describe = amd_describe;
1175 pcd->pcd_get_config = amd_get_config;
1176 pcd->pcd_get_msr = amd_get_msr;
1177 pcd->pcd_pcpu_fini = amd_pcpu_fini;
1178 pcd->pcd_pcpu_init = amd_pcpu_init;
1179 pcd->pcd_read_pmc = amd_read_pmc;
1180 pcd->pcd_release_pmc = amd_release_pmc;
1181 pcd->pcd_start_pmc = amd_start_pmc;
1182 pcd->pcd_stop_pmc = amd_stop_pmc;
1183 pcd->pcd_write_pmc = amd_write_pmc;
1184
1185 pmc_mdep->pmd_pcpu_init = NULL;
1186 pmc_mdep->pmd_pcpu_fini = NULL;
1187 pmc_mdep->pmd_intr = amd_intr;
1188 pmc_mdep->pmd_switch_in = amd_switch_in;
1189 pmc_mdep->pmd_switch_out = amd_switch_out;
1190
1191 pmc_mdep->pmd_npmc += AMD_NPMCS;
1192
1193 PMCDBG0(MDP,INI,0,"amd-initialize");
1194
1195 return (pmc_mdep);
1196
1197 error:
1198 if (error) {
1199 free(pmc_mdep, M_PMC);
1200 pmc_mdep = NULL;
1201 }
1202
1203 return (NULL);
1204 }
1205
1206 /*
1207 * Finalization code for AMD CPUs.
1208 */
1209
1210 void
pmc_amd_finalize(struct pmc_mdep * md)1211 pmc_amd_finalize(struct pmc_mdep *md)
1212 {
1213 #if defined(INVARIANTS)
1214 int classindex, i, ncpus, pmcclass;
1215 #endif
1216
1217 pmc_tsc_finalize(md);
1218
1219 KASSERT(amd_pcpu != NULL, ("[amd,%d] NULL per-cpu array pointer",
1220 __LINE__));
1221
1222 #if defined(INVARIANTS)
1223 switch (md->pmd_cputype) {
1224 #if defined(__i386__)
1225 case PMC_CPU_AMD_K7:
1226 classindex = PMC_MDEP_CLASS_INDEX_K7;
1227 pmcclass = PMC_CLASS_K7;
1228 break;
1229 #endif
1230 default:
1231 classindex = PMC_MDEP_CLASS_INDEX_K8;
1232 pmcclass = PMC_CLASS_K8;
1233 }
1234
1235 KASSERT(md->pmd_classdep[classindex].pcd_class == pmcclass,
1236 ("[amd,%d] pmc class mismatch", __LINE__));
1237
1238 ncpus = pmc_cpu_max();
1239
1240 for (i = 0; i < ncpus; i++)
1241 KASSERT(amd_pcpu[i] == NULL, ("[amd,%d] non-null pcpu",
1242 __LINE__));
1243 #endif
1244
1245 free(amd_pcpu, M_PMC);
1246 amd_pcpu = NULL;
1247 }
1248