1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2008 Joseph Koshy
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 /*
30 * Intel Core PMCs.
31 */
32
33 #include <sys/cdefs.h>
34 #include <sys/param.h>
35 #include <sys/bus.h>
36 #include <sys/pmc.h>
37 #include <sys/pmckern.h>
38 #include <sys/smp.h>
39 #include <sys/systm.h>
40
41 #include <machine/intr_machdep.h>
42 #include <x86/apicvar.h>
43 #include <machine/cpu.h>
44 #include <machine/cpufunc.h>
45 #include <machine/md_var.h>
46 #include <machine/specialreg.h>
47
48 #define CORE_CPUID_REQUEST 0xA
49 #define CORE_CPUID_REQUEST_SIZE 0x4
50 #define CORE_CPUID_EAX 0x0
51 #define CORE_CPUID_EBX 0x1
52 #define CORE_CPUID_ECX 0x2
53 #define CORE_CPUID_EDX 0x3
54
55 #define IAF_PMC_CAPS \
56 (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \
57 PMC_CAP_USER | PMC_CAP_SYSTEM)
58 #define IAF_RI_TO_MSR(RI) ((RI) + (1 << 30))
59
60 #define IAP_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \
61 PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \
62 PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE)
63
64 #define EV_IS_NOTARCH 0
65 #define EV_IS_ARCH_SUPP 1
66 #define EV_IS_ARCH_NOTSUPP -1
67
68 /*
69 * "Architectural" events defined by Intel. The values of these
70 * symbols correspond to positions in the bitmask returned by
71 * the CPUID.0AH instruction.
72 */
73 enum core_arch_events {
74 CORE_AE_BRANCH_INSTRUCTION_RETIRED = 5,
75 CORE_AE_BRANCH_MISSES_RETIRED = 6,
76 CORE_AE_INSTRUCTION_RETIRED = 1,
77 CORE_AE_LLC_MISSES = 4,
78 CORE_AE_LLC_REFERENCE = 3,
79 CORE_AE_UNHALTED_REFERENCE_CYCLES = 2,
80 CORE_AE_UNHALTED_CORE_CYCLES = 0
81 };
82
83 static enum pmc_cputype core_cputype;
84 static int core_version;
85
86 struct core_cpu {
87 volatile uint32_t pc_iafctrl; /* Fixed function control. */
88 volatile uint64_t pc_globalctrl; /* Global control register. */
89 struct pmc_hw pc_corepmcs[];
90 };
91
92 static struct core_cpu **core_pcpu;
93
94 static uint32_t core_architectural_events;
95 static uint64_t core_pmcmask;
96
97 static int core_iaf_ri; /* relative index of fixed counters */
98 static int core_iaf_width;
99 static int core_iaf_npmc;
100
101 static int core_iap_width;
102 static int core_iap_npmc;
103 static int core_iap_wroffset;
104
105 static u_int pmc_alloc_refs;
106 static bool pmc_tsx_force_abort_set;
107
108 static int
core_pcpu_noop(struct pmc_mdep * md,int cpu)109 core_pcpu_noop(struct pmc_mdep *md, int cpu)
110 {
111 (void) md;
112 (void) cpu;
113 return (0);
114 }
115
116 static int
core_pcpu_init(struct pmc_mdep * md,int cpu)117 core_pcpu_init(struct pmc_mdep *md, int cpu)
118 {
119 struct pmc_cpu *pc;
120 struct core_cpu *cc;
121 struct pmc_hw *phw;
122 int core_ri, n, npmc;
123
124 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
125 ("[iaf,%d] insane cpu number %d", __LINE__, cpu));
126
127 PMCDBG1(MDP,INI,1,"core-init cpu=%d", cpu);
128
129 core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri;
130 npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num;
131
132 if (core_version >= 2)
133 npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num;
134
135 cc = malloc(sizeof(struct core_cpu) + npmc * sizeof(struct pmc_hw),
136 M_PMC, M_WAITOK | M_ZERO);
137
138 core_pcpu[cpu] = cc;
139 pc = pmc_pcpu[cpu];
140
141 KASSERT(pc != NULL && cc != NULL,
142 ("[core,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu));
143
144 for (n = 0, phw = cc->pc_corepmcs; n < npmc; n++, phw++) {
145 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
146 PMC_PHW_CPU_TO_STATE(cpu) |
147 PMC_PHW_INDEX_TO_STATE(n + core_ri);
148 phw->phw_pmc = NULL;
149 pc->pc_hwpmcs[n + core_ri] = phw;
150 }
151
152 if (core_version >= 2 && vm_guest == VM_GUEST_NO) {
153 /* Enable Freezing PMCs on PMI. */
154 wrmsr(MSR_DEBUGCTLMSR, rdmsr(MSR_DEBUGCTLMSR) | 0x1000);
155 }
156
157 return (0);
158 }
159
160 static int
core_pcpu_fini(struct pmc_mdep * md,int cpu)161 core_pcpu_fini(struct pmc_mdep *md, int cpu)
162 {
163 int core_ri, n, npmc;
164 struct pmc_cpu *pc;
165 struct core_cpu *cc;
166
167 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
168 ("[core,%d] insane cpu number (%d)", __LINE__, cpu));
169
170 PMCDBG1(MDP,INI,1,"core-pcpu-fini cpu=%d", cpu);
171
172 if ((cc = core_pcpu[cpu]) == NULL)
173 return (0);
174
175 core_pcpu[cpu] = NULL;
176
177 pc = pmc_pcpu[cpu];
178
179 KASSERT(pc != NULL, ("[core,%d] NULL per-cpu %d state", __LINE__,
180 cpu));
181
182 npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num;
183 core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri;
184
185 for (n = 0; n < npmc; n++)
186 wrmsr(IAP_EVSEL0 + n, 0);
187
188 if (core_version >= 2) {
189 wrmsr(IAF_CTRL, 0);
190 npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num;
191 }
192
193 for (n = 0; n < npmc; n++)
194 pc->pc_hwpmcs[n + core_ri] = NULL;
195
196 free(cc, M_PMC);
197
198 return (0);
199 }
200
201 /*
202 * Fixed function counters.
203 */
204
205 static pmc_value_t
iaf_perfctr_value_to_reload_count(pmc_value_t v)206 iaf_perfctr_value_to_reload_count(pmc_value_t v)
207 {
208
209 /* If the PMC has overflowed, return a reload count of zero. */
210 if ((v & (1ULL << (core_iaf_width - 1))) == 0)
211 return (0);
212 v &= (1ULL << core_iaf_width) - 1;
213 return (1ULL << core_iaf_width) - v;
214 }
215
216 static pmc_value_t
iaf_reload_count_to_perfctr_value(pmc_value_t rlc)217 iaf_reload_count_to_perfctr_value(pmc_value_t rlc)
218 {
219 return (1ULL << core_iaf_width) - rlc;
220 }
221
222 static int
iaf_allocate_pmc(int cpu,int ri,struct pmc * pm,const struct pmc_op_pmcallocate * a)223 iaf_allocate_pmc(int cpu, int ri, struct pmc *pm,
224 const struct pmc_op_pmcallocate *a)
225 {
226 uint8_t ev, umask;
227 uint32_t caps;
228 uint64_t config, flags;
229 const struct pmc_md_iap_op_pmcallocate *iap;
230
231 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
232 ("[core,%d] illegal CPU %d", __LINE__, cpu));
233
234 PMCDBG2(MDP,ALL,1, "iaf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps);
235
236 if (ri < 0 || ri > core_iaf_npmc)
237 return (EINVAL);
238
239 if (a->pm_class != PMC_CLASS_IAF)
240 return (EINVAL);
241
242 if ((a->pm_flags & PMC_F_EV_PMU) == 0)
243 return (EINVAL);
244
245 iap = &a->pm_md.pm_iap;
246 config = iap->pm_iap_config;
247 ev = IAP_EVSEL_GET(config);
248 umask = IAP_UMASK_GET(config);
249
250 if (ev == 0x0) {
251 if (umask != ri + 1)
252 return (EINVAL);
253 } else {
254 switch (ri) {
255 case 0: /* INST_RETIRED.ANY */
256 if (ev != 0xC0 || umask != 0x00)
257 return (EINVAL);
258 break;
259 case 1: /* CPU_CLK_UNHALTED.THREAD */
260 if (ev != 0x3C || umask != 0x00)
261 return (EINVAL);
262 break;
263 case 2: /* CPU_CLK_UNHALTED.REF */
264 if (ev != 0x3C || umask != 0x01)
265 return (EINVAL);
266 break;
267 case 3: /* TOPDOWN.SLOTS */
268 if (ev != 0xA4 || umask != 0x01)
269 return (EINVAL);
270 break;
271 default:
272 return (EINVAL);
273 }
274 }
275
276 pmc_alloc_refs++;
277 if ((cpu_stdext_feature3 & CPUID_STDEXT3_TSXFA) != 0 &&
278 !pmc_tsx_force_abort_set) {
279 pmc_tsx_force_abort_set = true;
280 x86_msr_op(MSR_TSX_FORCE_ABORT, MSR_OP_RENDEZVOUS_ALL |
281 MSR_OP_WRITE, 1, NULL);
282 }
283
284 flags = 0;
285 if (config & IAP_OS)
286 flags |= IAF_OS;
287 if (config & IAP_USR)
288 flags |= IAF_USR;
289 if (config & IAP_ANY)
290 flags |= IAF_ANY;
291 if (config & IAP_INT)
292 flags |= IAF_PMI;
293
294 caps = a->pm_caps;
295 if (caps & PMC_CAP_INTERRUPT)
296 flags |= IAF_PMI;
297 if (caps & PMC_CAP_SYSTEM)
298 flags |= IAF_OS;
299 if (caps & PMC_CAP_USER)
300 flags |= IAF_USR;
301 if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0)
302 flags |= (IAF_OS | IAF_USR);
303
304 pm->pm_md.pm_iaf.pm_iaf_ctrl = (flags << (ri * 4));
305
306 PMCDBG1(MDP,ALL,2, "iaf-allocate config=0x%jx",
307 (uintmax_t) pm->pm_md.pm_iaf.pm_iaf_ctrl);
308
309 return (0);
310 }
311
312 static int
iaf_config_pmc(int cpu,int ri,struct pmc * pm)313 iaf_config_pmc(int cpu, int ri, struct pmc *pm)
314 {
315 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
316 ("[core,%d] illegal CPU %d", __LINE__, cpu));
317
318 KASSERT(ri >= 0 && ri < core_iaf_npmc,
319 ("[core,%d] illegal row-index %d", __LINE__, ri));
320
321 PMCDBG3(MDP,CFG,1, "iaf-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
322
323 KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__,
324 cpu));
325
326 core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc = pm;
327
328 return (0);
329 }
330
331 static int
iaf_describe(int cpu,int ri,struct pmc_info * pi,struct pmc ** ppmc)332 iaf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
333 {
334 struct pmc_hw *phw;
335
336 phw = &core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri];
337
338 snprintf(pi->pm_name, sizeof(pi->pm_name), "IAF-%d", ri);
339 pi->pm_class = PMC_CLASS_IAF;
340
341 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
342 pi->pm_enabled = TRUE;
343 *ppmc = phw->phw_pmc;
344 } else {
345 pi->pm_enabled = FALSE;
346 *ppmc = NULL;
347 }
348
349 return (0);
350 }
351
352 static int
iaf_get_config(int cpu,int ri,struct pmc ** ppm)353 iaf_get_config(int cpu, int ri, struct pmc **ppm)
354 {
355 *ppm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
356
357 return (0);
358 }
359
360 static int
iaf_get_msr(int ri,uint32_t * msr)361 iaf_get_msr(int ri, uint32_t *msr)
362 {
363 KASSERT(ri >= 0 && ri < core_iaf_npmc,
364 ("[iaf,%d] ri %d out of range", __LINE__, ri));
365
366 *msr = IAF_RI_TO_MSR(ri);
367
368 return (0);
369 }
370
371 static int
iaf_read_pmc(int cpu,int ri,struct pmc * pm,pmc_value_t * v)372 iaf_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v)
373 {
374 pmc_value_t tmp;
375
376 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
377 ("[core,%d] illegal cpu value %d", __LINE__, cpu));
378 KASSERT(ri >= 0 && ri < core_iaf_npmc,
379 ("[core,%d] illegal row-index %d", __LINE__, ri));
380
381 tmp = rdpmc(IAF_RI_TO_MSR(ri));
382
383 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
384 *v = iaf_perfctr_value_to_reload_count(tmp);
385 else
386 *v = tmp & ((1ULL << core_iaf_width) - 1);
387
388 PMCDBG4(MDP,REA,1, "iaf-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
389 IAF_RI_TO_MSR(ri), *v);
390
391 return (0);
392 }
393
394 static int
iaf_release_pmc(int cpu,int ri,struct pmc * pmc)395 iaf_release_pmc(int cpu, int ri, struct pmc *pmc)
396 {
397 PMCDBG3(MDP,REL,1, "iaf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc);
398
399 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
400 ("[core,%d] illegal CPU value %d", __LINE__, cpu));
401 KASSERT(ri >= 0 && ri < core_iaf_npmc,
402 ("[core,%d] illegal row-index %d", __LINE__, ri));
403
404 KASSERT(core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc == NULL,
405 ("[core,%d] PHW pmc non-NULL", __LINE__));
406
407 MPASS(pmc_alloc_refs > 0);
408 if (pmc_alloc_refs-- == 1 && pmc_tsx_force_abort_set) {
409 pmc_tsx_force_abort_set = false;
410 x86_msr_op(MSR_TSX_FORCE_ABORT, MSR_OP_RENDEZVOUS_ALL |
411 MSR_OP_WRITE, 0, NULL);
412 }
413
414 return (0);
415 }
416
417 static int
iaf_start_pmc(int cpu,int ri,struct pmc * pm)418 iaf_start_pmc(int cpu, int ri, struct pmc *pm)
419 {
420 struct core_cpu *cc;
421
422 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
423 ("[core,%d] illegal CPU value %d", __LINE__, cpu));
424 KASSERT(ri >= 0 && ri < core_iaf_npmc,
425 ("[core,%d] illegal row-index %d", __LINE__, ri));
426
427 PMCDBG2(MDP,STA,1,"iaf-start cpu=%d ri=%d", cpu, ri);
428
429 cc = core_pcpu[cpu];
430 cc->pc_iafctrl |= pm->pm_md.pm_iaf.pm_iaf_ctrl;
431 wrmsr(IAF_CTRL, cc->pc_iafctrl);
432
433 cc->pc_globalctrl |= (1ULL << (ri + IAF_OFFSET));
434 wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
435
436 PMCDBG4(MDP,STA,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)",
437 cc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL),
438 cc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL));
439
440 return (0);
441 }
442
443 static int
iaf_stop_pmc(int cpu,int ri,struct pmc * pm)444 iaf_stop_pmc(int cpu, int ri, struct pmc *pm)
445 {
446 struct core_cpu *cc;
447
448 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
449 ("[core,%d] illegal CPU value %d", __LINE__, cpu));
450 KASSERT(ri >= 0 && ri < core_iaf_npmc,
451 ("[core,%d] illegal row-index %d", __LINE__, ri));
452
453 PMCDBG2(MDP,STA,1,"iaf-stop cpu=%d ri=%d", cpu, ri);
454
455 cc = core_pcpu[cpu];
456
457 cc->pc_iafctrl &= ~(IAF_MASK << (ri * 4));
458 wrmsr(IAF_CTRL, cc->pc_iafctrl);
459
460 /* Don't need to write IA_GLOBAL_CTRL, one disable is enough. */
461
462 PMCDBG4(MDP,STO,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)",
463 cc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL),
464 cc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL));
465
466 return (0);
467 }
468
469 static int
iaf_write_pmc(int cpu,int ri,struct pmc * pm,pmc_value_t v)470 iaf_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v)
471 {
472 struct core_cpu *cc;
473
474 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
475 ("[core,%d] illegal cpu value %d", __LINE__, cpu));
476 KASSERT(ri >= 0 && ri < core_iaf_npmc,
477 ("[core,%d] illegal row-index %d", __LINE__, ri));
478
479 cc = core_pcpu[cpu];
480
481 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
482 v = iaf_reload_count_to_perfctr_value(v);
483
484 /* Turn off the fixed counter */
485 wrmsr(IAF_CTRL, cc->pc_iafctrl & ~(IAF_MASK << (ri * 4)));
486
487 wrmsr(IAF_CTR0 + ri, v & ((1ULL << core_iaf_width) - 1));
488
489 /* Turn on fixed counters */
490 wrmsr(IAF_CTRL, cc->pc_iafctrl);
491
492 PMCDBG6(MDP,WRI,1, "iaf-write cpu=%d ri=%d msr=0x%x v=%jx iafctrl=%jx "
493 "pmc=%jx", cpu, ri, IAF_RI_TO_MSR(ri), v,
494 (uintmax_t) rdmsr(IAF_CTRL),
495 (uintmax_t) rdpmc(IAF_RI_TO_MSR(ri)));
496
497 return (0);
498 }
499
500
501 static void
iaf_initialize(struct pmc_mdep * md,int maxcpu,int npmc,int pmcwidth)502 iaf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth)
503 {
504 struct pmc_classdep *pcd;
505
506 KASSERT(md != NULL, ("[iaf,%d] md is NULL", __LINE__));
507
508 PMCDBG0(MDP,INI,1, "iaf-initialize");
509
510 pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF];
511
512 pcd->pcd_caps = IAF_PMC_CAPS;
513 pcd->pcd_class = PMC_CLASS_IAF;
514 pcd->pcd_num = npmc;
515 pcd->pcd_ri = md->pmd_npmc;
516 pcd->pcd_width = pmcwidth;
517
518 pcd->pcd_allocate_pmc = iaf_allocate_pmc;
519 pcd->pcd_config_pmc = iaf_config_pmc;
520 pcd->pcd_describe = iaf_describe;
521 pcd->pcd_get_config = iaf_get_config;
522 pcd->pcd_get_msr = iaf_get_msr;
523 pcd->pcd_pcpu_fini = core_pcpu_noop;
524 pcd->pcd_pcpu_init = core_pcpu_noop;
525 pcd->pcd_read_pmc = iaf_read_pmc;
526 pcd->pcd_release_pmc = iaf_release_pmc;
527 pcd->pcd_start_pmc = iaf_start_pmc;
528 pcd->pcd_stop_pmc = iaf_stop_pmc;
529 pcd->pcd_write_pmc = iaf_write_pmc;
530
531 md->pmd_npmc += npmc;
532 }
533
534 /*
535 * Intel programmable PMCs.
536 */
537
538 /* Sub fields of UMASK that this event supports. */
539 #define IAP_M_CORE (1 << 0) /* Core specificity */
540 #define IAP_M_AGENT (1 << 1) /* Agent specificity */
541 #define IAP_M_PREFETCH (1 << 2) /* Prefetch */
542 #define IAP_M_MESI (1 << 3) /* MESI */
543 #define IAP_M_SNOOPRESPONSE (1 << 4) /* Snoop response */
544 #define IAP_M_SNOOPTYPE (1 << 5) /* Snoop type */
545 #define IAP_M_TRANSITION (1 << 6) /* Transition */
546
547 #define IAP_F_CORE (0x3 << 14) /* Core specificity */
548 #define IAP_F_AGENT (0x1 << 13) /* Agent specificity */
549 #define IAP_F_PREFETCH (0x3 << 12) /* Prefetch */
550 #define IAP_F_MESI (0xF << 8) /* MESI */
551 #define IAP_F_SNOOPRESPONSE (0xB << 8) /* Snoop response */
552 #define IAP_F_SNOOPTYPE (0x3 << 8) /* Snoop type */
553 #define IAP_F_TRANSITION (0x1 << 12) /* Transition */
554
555 #define IAP_PREFETCH_RESERVED (0x2 << 12)
556 #define IAP_CORE_THIS (0x1 << 14)
557 #define IAP_CORE_ALL (0x3 << 14)
558 #define IAP_F_CMASK 0xFF000000
559
560 static pmc_value_t
iap_perfctr_value_to_reload_count(pmc_value_t v)561 iap_perfctr_value_to_reload_count(pmc_value_t v)
562 {
563
564 /* If the PMC has overflowed, return a reload count of zero. */
565 if ((v & (1ULL << (core_iap_width - 1))) == 0)
566 return (0);
567 v &= (1ULL << core_iap_width) - 1;
568 return (1ULL << core_iap_width) - v;
569 }
570
571 static pmc_value_t
iap_reload_count_to_perfctr_value(pmc_value_t rlc)572 iap_reload_count_to_perfctr_value(pmc_value_t rlc)
573 {
574 return (1ULL << core_iap_width) - rlc;
575 }
576
577 static int
iap_pmc_has_overflowed(int ri)578 iap_pmc_has_overflowed(int ri)
579 {
580 uint64_t v;
581
582 /*
583 * We treat a Core (i.e., Intel architecture v1) PMC as has
584 * having overflowed if its MSB is zero.
585 */
586 v = rdpmc(ri);
587 return ((v & (1ULL << (core_iap_width - 1))) == 0);
588 }
589
590 static int
iap_event_corei7_ok_on_counter(uint8_t evsel,int ri)591 iap_event_corei7_ok_on_counter(uint8_t evsel, int ri)
592 {
593 uint32_t mask;
594
595 switch (evsel) {
596 /* Events valid only on counter 0, 1. */
597 case 0x40:
598 case 0x41:
599 case 0x42:
600 case 0x43:
601 case 0x4C:
602 case 0x4E:
603 case 0x51:
604 case 0x52:
605 case 0x53:
606 case 0x63:
607 mask = 0x3;
608 break;
609 /* Any row index is ok. */
610 default:
611 mask = ~0;
612 }
613
614 return (mask & (1 << ri));
615 }
616
617 static int
iap_event_westmere_ok_on_counter(uint8_t evsel,int ri)618 iap_event_westmere_ok_on_counter(uint8_t evsel, int ri)
619 {
620 uint32_t mask;
621
622 switch (evsel) {
623 /* Events valid only on counter 0. */
624 case 0x60:
625 case 0xB3:
626 mask = 0x1;
627 break;
628
629 /* Events valid only on counter 0, 1. */
630 case 0x4C:
631 case 0x4E:
632 case 0x51:
633 case 0x52:
634 case 0x63:
635 mask = 0x3;
636 break;
637 /* Any row index is ok. */
638 default:
639 mask = ~0;
640 }
641
642 return (mask & (1 << ri));
643 }
644
645 static int
iap_event_sb_sbx_ib_ibx_ok_on_counter(uint8_t evsel,int ri)646 iap_event_sb_sbx_ib_ibx_ok_on_counter(uint8_t evsel, int ri)
647 {
648 uint32_t mask;
649
650 switch (evsel) {
651 /* Events valid only on counter 0. */
652 case 0xB7:
653 mask = 0x1;
654 break;
655 /* Events valid only on counter 1. */
656 case 0xC0:
657 mask = 0x2;
658 break;
659 /* Events valid only on counter 2. */
660 case 0x48:
661 case 0xA2:
662 case 0xA3:
663 mask = 0x4;
664 break;
665 /* Events valid only on counter 3. */
666 case 0xBB:
667 case 0xCD:
668 mask = 0x8;
669 break;
670 /* Any row index is ok. */
671 default:
672 mask = ~0;
673 }
674
675 return (mask & (1 << ri));
676 }
677
678 static int
iap_event_core_ok_on_counter(uint8_t evsel,int ri)679 iap_event_core_ok_on_counter(uint8_t evsel, int ri)
680 {
681 uint32_t mask;
682
683 switch (evsel) {
684 /*
685 * Events valid only on counter 0.
686 */
687 case 0x10:
688 case 0x14:
689 case 0x18:
690 case 0xB3:
691 case 0xC1:
692 case 0xCB:
693 mask = (1 << 0);
694 break;
695
696 /*
697 * Events valid only on counter 1.
698 */
699 case 0x11:
700 case 0x12:
701 case 0x13:
702 mask = (1 << 1);
703 break;
704
705 default:
706 mask = ~0; /* Any row index is ok. */
707 }
708
709 return (mask & (1 << ri));
710 }
711
712 static int
iap_allocate_pmc(int cpu,int ri,struct pmc * pm,const struct pmc_op_pmcallocate * a)713 iap_allocate_pmc(int cpu, int ri, struct pmc *pm,
714 const struct pmc_op_pmcallocate *a)
715 {
716 uint8_t ev;
717 const struct pmc_md_iap_op_pmcallocate *iap;
718
719 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
720 ("[core,%d] illegal CPU %d", __LINE__, cpu));
721 KASSERT(ri >= 0 && ri < core_iap_npmc,
722 ("[core,%d] illegal row-index value %d", __LINE__, ri));
723
724 if (a->pm_class != PMC_CLASS_IAP)
725 return (EINVAL);
726
727 if ((a->pm_flags & PMC_F_EV_PMU) == 0)
728 return (EINVAL);
729
730 iap = &a->pm_md.pm_iap;
731 ev = IAP_EVSEL_GET(iap->pm_iap_config);
732
733 switch (core_cputype) {
734 case PMC_CPU_INTEL_CORE:
735 case PMC_CPU_INTEL_CORE2:
736 case PMC_CPU_INTEL_CORE2EXTREME:
737 if (iap_event_core_ok_on_counter(ev, ri) == 0)
738 return (EINVAL);
739 case PMC_CPU_INTEL_COREI7:
740 case PMC_CPU_INTEL_NEHALEM_EX:
741 if (iap_event_corei7_ok_on_counter(ev, ri) == 0)
742 return (EINVAL);
743 break;
744 case PMC_CPU_INTEL_WESTMERE:
745 case PMC_CPU_INTEL_WESTMERE_EX:
746 if (iap_event_westmere_ok_on_counter(ev, ri) == 0)
747 return (EINVAL);
748 break;
749 case PMC_CPU_INTEL_SANDYBRIDGE:
750 case PMC_CPU_INTEL_SANDYBRIDGE_XEON:
751 case PMC_CPU_INTEL_IVYBRIDGE:
752 case PMC_CPU_INTEL_IVYBRIDGE_XEON:
753 case PMC_CPU_INTEL_HASWELL:
754 case PMC_CPU_INTEL_HASWELL_XEON:
755 case PMC_CPU_INTEL_BROADWELL:
756 case PMC_CPU_INTEL_BROADWELL_XEON:
757 if (iap_event_sb_sbx_ib_ibx_ok_on_counter(ev, ri) == 0)
758 return (EINVAL);
759 break;
760 case PMC_CPU_INTEL_ATOM:
761 case PMC_CPU_INTEL_ATOM_SILVERMONT:
762 case PMC_CPU_INTEL_ATOM_GOLDMONT:
763 case PMC_CPU_INTEL_ATOM_GOLDMONT_P:
764 case PMC_CPU_INTEL_ATOM_TREMONT:
765 case PMC_CPU_INTEL_SKYLAKE:
766 case PMC_CPU_INTEL_SKYLAKE_XEON:
767 case PMC_CPU_INTEL_ICELAKE:
768 case PMC_CPU_INTEL_ICELAKE_XEON:
769 case PMC_CPU_INTEL_ALDERLAKE:
770 default:
771 break;
772 }
773
774 pm->pm_md.pm_iap.pm_iap_evsel = iap->pm_iap_config;
775 return (0);
776 }
777
778 static int
iap_config_pmc(int cpu,int ri,struct pmc * pm)779 iap_config_pmc(int cpu, int ri, struct pmc *pm)
780 {
781 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
782 ("[core,%d] illegal CPU %d", __LINE__, cpu));
783
784 KASSERT(ri >= 0 && ri < core_iap_npmc,
785 ("[core,%d] illegal row-index %d", __LINE__, ri));
786
787 PMCDBG3(MDP,CFG,1, "iap-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
788
789 KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__,
790 cpu));
791
792 core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc = pm;
793
794 return (0);
795 }
796
797 static int
iap_describe(int cpu,int ri,struct pmc_info * pi,struct pmc ** ppmc)798 iap_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
799 {
800 struct pmc_hw *phw;
801
802 phw = &core_pcpu[cpu]->pc_corepmcs[ri];
803
804 snprintf(pi->pm_name, sizeof(pi->pm_name), "IAP-%d", ri);
805 pi->pm_class = PMC_CLASS_IAP;
806
807 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
808 pi->pm_enabled = TRUE;
809 *ppmc = phw->phw_pmc;
810 } else {
811 pi->pm_enabled = FALSE;
812 *ppmc = NULL;
813 }
814
815 return (0);
816 }
817
818 static int
iap_get_config(int cpu,int ri,struct pmc ** ppm)819 iap_get_config(int cpu, int ri, struct pmc **ppm)
820 {
821 *ppm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc;
822
823 return (0);
824 }
825
826 static int
iap_get_msr(int ri,uint32_t * msr)827 iap_get_msr(int ri, uint32_t *msr)
828 {
829 KASSERT(ri >= 0 && ri < core_iap_npmc,
830 ("[iap,%d] ri %d out of range", __LINE__, ri));
831
832 *msr = ri;
833
834 return (0);
835 }
836
837 static int
iap_read_pmc(int cpu,int ri,struct pmc * pm,pmc_value_t * v)838 iap_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v)
839 {
840 pmc_value_t tmp;
841
842 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
843 ("[core,%d] illegal cpu value %d", __LINE__, cpu));
844 KASSERT(ri >= 0 && ri < core_iap_npmc,
845 ("[core,%d] illegal row-index %d", __LINE__, ri));
846
847 tmp = rdpmc(ri);
848 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
849 *v = iap_perfctr_value_to_reload_count(tmp);
850 else
851 *v = tmp & ((1ULL << core_iap_width) - 1);
852
853 PMCDBG4(MDP,REA,1, "iap-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
854 IAP_PMC0 + ri, *v);
855
856 return (0);
857 }
858
859 static int
iap_release_pmc(int cpu,int ri,struct pmc * pm)860 iap_release_pmc(int cpu, int ri, struct pmc *pm)
861 {
862 (void) pm;
863
864 PMCDBG3(MDP,REL,1, "iap-release cpu=%d ri=%d pm=%p", cpu, ri,
865 pm);
866
867 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
868 ("[core,%d] illegal CPU value %d", __LINE__, cpu));
869 KASSERT(ri >= 0 && ri < core_iap_npmc,
870 ("[core,%d] illegal row-index %d", __LINE__, ri));
871
872 KASSERT(core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc
873 == NULL, ("[core,%d] PHW pmc non-NULL", __LINE__));
874
875 return (0);
876 }
877
878 static int
iap_start_pmc(int cpu,int ri,struct pmc * pm)879 iap_start_pmc(int cpu, int ri, struct pmc *pm)
880 {
881 uint64_t evsel;
882 struct core_cpu *cc;
883
884 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
885 ("[core,%d] illegal CPU value %d", __LINE__, cpu));
886 KASSERT(ri >= 0 && ri < core_iap_npmc,
887 ("[core,%d] illegal row-index %d", __LINE__, ri));
888
889 cc = core_pcpu[cpu];
890
891 PMCDBG2(MDP,STA,1, "iap-start cpu=%d ri=%d", cpu, ri);
892
893 evsel = pm->pm_md.pm_iap.pm_iap_evsel;
894
895 PMCDBG4(MDP,STA,2, "iap-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x",
896 cpu, ri, IAP_EVSEL0 + ri, evsel);
897
898 /* Event specific configuration. */
899
900 switch (IAP_EVSEL_GET(evsel)) {
901 case 0xB7:
902 wrmsr(IA_OFFCORE_RSP0, pm->pm_md.pm_iap.pm_iap_rsp);
903 break;
904 case 0xBB:
905 wrmsr(IA_OFFCORE_RSP1, pm->pm_md.pm_iap.pm_iap_rsp);
906 break;
907 default:
908 break;
909 }
910
911 wrmsr(IAP_EVSEL0 + ri, evsel | IAP_EN);
912
913 if (core_version >= 2) {
914 cc->pc_globalctrl |= (1ULL << ri);
915 wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
916 }
917
918 return (0);
919 }
920
921 static int
iap_stop_pmc(int cpu,int ri,struct pmc * pm __unused)922 iap_stop_pmc(int cpu, int ri, struct pmc *pm __unused)
923 {
924
925 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
926 ("[core,%d] illegal cpu value %d", __LINE__, cpu));
927 KASSERT(ri >= 0 && ri < core_iap_npmc,
928 ("[core,%d] illegal row index %d", __LINE__, ri));
929
930 PMCDBG2(MDP,STO,1, "iap-stop cpu=%d ri=%d", cpu, ri);
931
932 wrmsr(IAP_EVSEL0 + ri, 0);
933
934 /* Don't need to write IA_GLOBAL_CTRL, one disable is enough. */
935
936 return (0);
937 }
938
939 static int
iap_write_pmc(int cpu,int ri,struct pmc * pm,pmc_value_t v)940 iap_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v)
941 {
942
943 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
944 ("[core,%d] illegal cpu value %d", __LINE__, cpu));
945 KASSERT(ri >= 0 && ri < core_iap_npmc,
946 ("[core,%d] illegal row index %d", __LINE__, ri));
947
948 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
949 v = iap_reload_count_to_perfctr_value(v);
950
951 v &= (1ULL << core_iap_width) - 1;
952
953 PMCDBG4(MDP,WRI,1, "iap-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
954 IAP_PMC0 + ri, v);
955
956 /*
957 * Write the new value to the counter (or it's alias). The
958 * counter will be in a stopped state when the pcd_write()
959 * entry point is called.
960 */
961 wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v);
962 return (0);
963 }
964
965
966 static void
iap_initialize(struct pmc_mdep * md,int maxcpu,int npmc,int pmcwidth,int flags)967 iap_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth,
968 int flags)
969 {
970 struct pmc_classdep *pcd;
971
972 KASSERT(md != NULL, ("[iap,%d] md is NULL", __LINE__));
973
974 PMCDBG0(MDP,INI,1, "iap-initialize");
975
976 /* Remember the set of architectural events supported. */
977 core_architectural_events = ~flags;
978
979 pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP];
980
981 pcd->pcd_caps = IAP_PMC_CAPS;
982 pcd->pcd_class = PMC_CLASS_IAP;
983 pcd->pcd_num = npmc;
984 pcd->pcd_ri = md->pmd_npmc;
985 pcd->pcd_width = pmcwidth;
986
987 pcd->pcd_allocate_pmc = iap_allocate_pmc;
988 pcd->pcd_config_pmc = iap_config_pmc;
989 pcd->pcd_describe = iap_describe;
990 pcd->pcd_get_config = iap_get_config;
991 pcd->pcd_get_msr = iap_get_msr;
992 pcd->pcd_pcpu_fini = core_pcpu_fini;
993 pcd->pcd_pcpu_init = core_pcpu_init;
994 pcd->pcd_read_pmc = iap_read_pmc;
995 pcd->pcd_release_pmc = iap_release_pmc;
996 pcd->pcd_start_pmc = iap_start_pmc;
997 pcd->pcd_stop_pmc = iap_stop_pmc;
998 pcd->pcd_write_pmc = iap_write_pmc;
999
1000 md->pmd_npmc += npmc;
1001 }
1002
1003 static int
core_intr(struct trapframe * tf)1004 core_intr(struct trapframe *tf)
1005 {
1006 pmc_value_t v;
1007 struct pmc *pm;
1008 struct core_cpu *cc;
1009 int error, found_interrupt, ri;
1010
1011 PMCDBG3(MDP,INT, 1, "cpu=%d tf=%p um=%d", curcpu, (void *) tf,
1012 TRAPF_USERMODE(tf));
1013
1014 found_interrupt = 0;
1015 cc = core_pcpu[curcpu];
1016
1017 for (ri = 0; ri < core_iap_npmc; ri++) {
1018
1019 if ((pm = cc->pc_corepmcs[ri].phw_pmc) == NULL ||
1020 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1021 continue;
1022
1023 if (!iap_pmc_has_overflowed(ri))
1024 continue;
1025
1026 found_interrupt = 1;
1027
1028 if (pm->pm_state != PMC_STATE_RUNNING)
1029 continue;
1030
1031 error = pmc_process_interrupt(PMC_HR, pm, tf);
1032
1033 v = pm->pm_sc.pm_reloadcount;
1034 v = iap_reload_count_to_perfctr_value(v);
1035
1036 /*
1037 * Stop the counter, reload it but only restart it if
1038 * the PMC is not stalled.
1039 */
1040 wrmsr(IAP_EVSEL0 + ri, pm->pm_md.pm_iap.pm_iap_evsel);
1041 wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v);
1042
1043 if (__predict_false(error))
1044 continue;
1045
1046 wrmsr(IAP_EVSEL0 + ri, pm->pm_md.pm_iap.pm_iap_evsel | IAP_EN);
1047 }
1048
1049 if (found_interrupt)
1050 counter_u64_add(pmc_stats.pm_intr_processed, 1);
1051 else
1052 counter_u64_add(pmc_stats.pm_intr_ignored, 1);
1053
1054 if (found_interrupt)
1055 lapic_reenable_pmc();
1056
1057 return (found_interrupt);
1058 }
1059
1060 static int
core2_intr(struct trapframe * tf)1061 core2_intr(struct trapframe *tf)
1062 {
1063 int error, found_interrupt = 0, n, cpu;
1064 uint64_t flag, intrstatus, intrdisable = 0;
1065 struct pmc *pm;
1066 struct core_cpu *cc;
1067 pmc_value_t v;
1068
1069 cpu = curcpu;
1070 PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
1071 TRAPF_USERMODE(tf));
1072
1073 /*
1074 * The IA_GLOBAL_STATUS (MSR 0x38E) register indicates which
1075 * PMCs have a pending PMI interrupt. We take a 'snapshot' of
1076 * the current set of interrupting PMCs and process these
1077 * after stopping them.
1078 */
1079 intrstatus = rdmsr(IA_GLOBAL_STATUS);
1080 PMCDBG2(MDP,INT, 1, "cpu=%d intrstatus=%jx", cpu,
1081 (uintmax_t) intrstatus);
1082
1083 /*
1084 * Stop PMCs unless hardware already done it.
1085 */
1086 if ((intrstatus & IA_GLOBAL_STATUS_FLAG_CTR_FRZ) == 0)
1087 wrmsr(IA_GLOBAL_CTRL, 0);
1088
1089 cc = core_pcpu[cpu];
1090 KASSERT(cc != NULL, ("[core,%d] null pcpu", __LINE__));
1091
1092 /*
1093 * Look for interrupts from fixed function PMCs.
1094 */
1095 for (n = 0, flag = (1ULL << IAF_OFFSET); n < core_iaf_npmc;
1096 n++, flag <<= 1) {
1097
1098 if ((intrstatus & flag) == 0)
1099 continue;
1100
1101 found_interrupt = 1;
1102
1103 pm = cc->pc_corepmcs[n + core_iaf_ri].phw_pmc;
1104 if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING ||
1105 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1106 continue;
1107
1108 error = pmc_process_interrupt(PMC_HR, pm, tf);
1109 if (__predict_false(error))
1110 intrdisable |= flag;
1111
1112 v = iaf_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount);
1113
1114 /* Reload sampling count. */
1115 wrmsr(IAF_CTR0 + n, v);
1116
1117 PMCDBG4(MDP,INT, 1, "iaf-intr cpu=%d error=%d v=%jx(%jx)", curcpu,
1118 error, (uintmax_t) v, (uintmax_t) rdpmc(IAF_RI_TO_MSR(n)));
1119 }
1120
1121 /*
1122 * Process interrupts from the programmable counters.
1123 */
1124 for (n = 0, flag = 1; n < core_iap_npmc; n++, flag <<= 1) {
1125 if ((intrstatus & flag) == 0)
1126 continue;
1127
1128 found_interrupt = 1;
1129
1130 pm = cc->pc_corepmcs[n].phw_pmc;
1131 if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING ||
1132 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1133 continue;
1134
1135 error = pmc_process_interrupt(PMC_HR, pm, tf);
1136 if (__predict_false(error))
1137 intrdisable |= flag;
1138
1139 v = iap_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount);
1140
1141 PMCDBG3(MDP,INT, 1, "iap-intr cpu=%d error=%d v=%jx", cpu, error,
1142 (uintmax_t) v);
1143
1144 /* Reload sampling count. */
1145 wrmsr(core_iap_wroffset + IAP_PMC0 + n, v);
1146 }
1147
1148 if (found_interrupt)
1149 counter_u64_add(pmc_stats.pm_intr_processed, 1);
1150 else
1151 counter_u64_add(pmc_stats.pm_intr_ignored, 1);
1152
1153 if (found_interrupt)
1154 lapic_reenable_pmc();
1155
1156 /*
1157 * Reenable all non-stalled PMCs.
1158 */
1159 if ((intrstatus & IA_GLOBAL_STATUS_FLAG_CTR_FRZ) == 0) {
1160 wrmsr(IA_GLOBAL_OVF_CTRL, intrstatus);
1161 cc->pc_globalctrl &= ~intrdisable;
1162 wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
1163 } else {
1164 if (__predict_false(intrdisable)) {
1165 cc->pc_globalctrl &= ~intrdisable;
1166 wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
1167 }
1168 wrmsr(IA_GLOBAL_OVF_CTRL, intrstatus);
1169 }
1170
1171 PMCDBG4(MDP, INT, 1, "cpu=%d fixedctrl=%jx globalctrl=%jx status=%jx",
1172 cpu, (uintmax_t) rdmsr(IAF_CTRL),
1173 (uintmax_t) rdmsr(IA_GLOBAL_CTRL),
1174 (uintmax_t) rdmsr(IA_GLOBAL_STATUS));
1175
1176 return (found_interrupt);
1177 }
1178
1179 int
pmc_core_initialize(struct pmc_mdep * md,int maxcpu,int version_override)1180 pmc_core_initialize(struct pmc_mdep *md, int maxcpu, int version_override)
1181 {
1182 int cpuid[CORE_CPUID_REQUEST_SIZE];
1183 int flags, nflags;
1184
1185 do_cpuid(CORE_CPUID_REQUEST, cpuid);
1186
1187 core_cputype = md->pmd_cputype;
1188 core_version = (version_override > 0) ? version_override :
1189 cpuid[CORE_CPUID_EAX] & 0xFF;
1190
1191 PMCDBG3(MDP,INI,1,"core-init cputype=%d ncpu=%d version=%d",
1192 core_cputype, maxcpu, core_version);
1193
1194 if (core_version < 1 || core_version > 5 ||
1195 (core_cputype != PMC_CPU_INTEL_CORE && core_version == 1)) {
1196 /* Unknown PMC architecture. */
1197 printf("hwpmc_core: unknown PMC architecture: %d\n",
1198 core_version);
1199 return (EPROGMISMATCH);
1200 }
1201
1202 core_iap_wroffset = 0;
1203 if (cpu_feature2 & CPUID2_PDCM) {
1204 if (rdmsr(IA32_PERF_CAPABILITIES) & PERFCAP_FW_WRITE) {
1205 PMCDBG0(MDP, INI, 1,
1206 "core-init full-width write supported");
1207 core_iap_wroffset = IAP_A_PMC0 - IAP_PMC0;
1208 } else
1209 PMCDBG0(MDP, INI, 1,
1210 "core-init full-width write NOT supported");
1211 } else
1212 PMCDBG0(MDP, INI, 1, "core-init pdcm not supported");
1213
1214 core_pmcmask = 0;
1215
1216 /*
1217 * Initialize programmable counters.
1218 */
1219 core_iap_npmc = (cpuid[CORE_CPUID_EAX] >> 8) & 0xFF;
1220 core_iap_width = (cpuid[CORE_CPUID_EAX] >> 16) & 0xFF;
1221
1222 core_pmcmask |= ((1ULL << core_iap_npmc) - 1);
1223
1224 nflags = (cpuid[CORE_CPUID_EAX] >> 24) & 0xFF;
1225 flags = cpuid[CORE_CPUID_EBX] & ((1 << nflags) - 1);
1226
1227 iap_initialize(md, maxcpu, core_iap_npmc, core_iap_width, flags);
1228
1229 /*
1230 * Initialize fixed function counters, if present.
1231 */
1232 if (core_version >= 2) {
1233 core_iaf_ri = core_iap_npmc;
1234 core_iaf_npmc = cpuid[CORE_CPUID_EDX] & 0x1F;
1235 core_iaf_width = (cpuid[CORE_CPUID_EDX] >> 5) & 0xFF;
1236
1237 iaf_initialize(md, maxcpu, core_iaf_npmc, core_iaf_width);
1238 core_pmcmask |= ((1ULL << core_iaf_npmc) - 1) << IAF_OFFSET;
1239 }
1240
1241 PMCDBG2(MDP,INI,1,"core-init pmcmask=0x%jx iafri=%d", core_pmcmask,
1242 core_iaf_ri);
1243
1244 core_pcpu = malloc(sizeof(*core_pcpu) * maxcpu, M_PMC,
1245 M_ZERO | M_WAITOK);
1246
1247 /*
1248 * Choose the appropriate interrupt handler.
1249 */
1250 if (core_version >= 2)
1251 md->pmd_intr = core2_intr;
1252 else
1253 md->pmd_intr = core_intr;
1254
1255 return (0);
1256 }
1257
1258 void
pmc_core_finalize(struct pmc_mdep * md)1259 pmc_core_finalize(struct pmc_mdep *md)
1260 {
1261 PMCDBG0(MDP,INI,1, "core-finalize");
1262
1263 free(core_pcpu, M_PMC);
1264 core_pcpu = NULL;
1265 }
1266