Lines Matching refs:pm
203 static int pmc_add_sample(ring_type_t ring, struct pmc *pm,
207 static int pmc_attach_process(struct proc *p, struct pmc *pm);
210 static int pmc_attach_one_process(struct proc *p, struct pmc *pm);
214 static int pmc_can_attach(struct pmc *pm, struct proc *p);
218 static int pmc_detach_process(struct proc *p, struct pmc *pm);
219 static int pmc_detach_one_process(struct proc *p, struct pmc *pm,
222 static void pmc_destroy_pmc_descriptor(struct pmc *pm);
225 static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm);
233 static void pmc_link_target_process(struct pmc *pm,
236 static void pmc_log_kernel_mappings(struct pmc *pm);
240 static void pmc_process_allproc(struct pmc *pm);
258 static int pmc_start(struct pmc *pm);
259 static int pmc_stop(struct pmc *pm);
845 struct pmc *pm, *tmp; in pmc_remove_owner() local
855 LIST_FOREACH_SAFE(pm, &po->po_pmcs, pm_next, tmp) { in pmc_remove_owner()
856 PMCDBG1(OWN,ORM,2, "pmc=%p", pm); in pmc_remove_owner()
857 KASSERT(pm->pm_owner == po, in pmc_remove_owner()
858 ("[pmc,%d] owner %p != po %p", __LINE__, pm->pm_owner, po)); in pmc_remove_owner()
860 pmc_release_pmc_descriptor(pm); /* will unlink from the list */ in pmc_remove_owner()
861 pmc_destroy_pmc_descriptor(pm); in pmc_remove_owner()
899 pmc_link_target_process(struct pmc *pm, struct pmc_process *pp) in pmc_link_target_process() argument
906 KASSERT(pm != NULL && pp != NULL, in pmc_link_target_process()
907 ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp)); in pmc_link_target_process()
908 KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)), in pmc_link_target_process()
910 __LINE__, pm, pp->pp_proc->p_pid)); in pmc_link_target_process()
915 ri = PMC_TO_ROWINDEX(pm); in pmc_link_target_process()
918 pm, ri, pp); in pmc_link_target_process()
921 LIST_FOREACH(pt, &pm->pm_targets, pt_next) { in pmc_link_target_process()
924 __LINE__, pp, pm)); in pmc_link_target_process()
930 LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next); in pmc_link_target_process()
933 (uintptr_t)pm); in pmc_link_target_process()
935 if (pm->pm_owner->po_owner == pp->pp_proc) in pmc_link_target_process()
936 pm->pm_flags |= PMC_F_ATTACHED_TO_OWNER; in pmc_link_target_process()
941 pp->pp_pmcs[ri].pp_pmcval = PMC_TO_MODE(pm) == PMC_MODE_TS ? in pmc_link_target_process()
942 pm->pm_sc.pm_reloadcount : 0; in pmc_link_target_process()
947 if (PMC_TO_MODE(pm) == PMC_MODE_TS) { in pmc_link_target_process()
963 pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp) in pmc_unlink_target_process() argument
972 KASSERT(pm != NULL && pp != NULL, in pmc_unlink_target_process()
973 ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp)); in pmc_unlink_target_process()
979 ri = PMC_TO_ROWINDEX(pm); in pmc_unlink_target_process()
982 pm, ri, pp); in pmc_unlink_target_process()
984 KASSERT(pp->pp_pmcs[ri].pp_pmc == pm, in pmc_unlink_target_process()
986 ri, pm, pp->pp_pmcs[ri].pp_pmc)); in pmc_unlink_target_process()
992 if (PMC_TO_MODE(pm) == PMC_MODE_TS) { in pmc_unlink_target_process()
1000 if (pm->pm_owner->po_owner == pp->pp_proc) { in pmc_unlink_target_process()
1002 pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER; in pmc_unlink_target_process()
1008 LIST_FOREACH(ptgt, &pm->pm_targets, pt_next) in pmc_unlink_target_process()
1013 "in pmc %p", __LINE__, pp->pp_proc, pp, pm)); in pmc_unlink_target_process()
1019 if (LIST_EMPTY(&pm->pm_targets)) { in pmc_unlink_target_process()
1020 p = pm->pm_owner->po_owner; in pmc_unlink_target_process()
1034 pmc_can_attach(struct pmc *pm, struct proc *t) in pmc_can_attach() argument
1044 if ((o = pm->pm_owner->po_owner) == t) in pmc_can_attach()
1088 pmc_attach_one_process(struct proc *p, struct pmc *pm) in pmc_attach_one_process() argument
1096 PMCDBG5(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm, in pmc_attach_one_process()
1097 PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); in pmc_attach_one_process()
1109 ri = PMC_TO_ROWINDEX(pm); in pmc_attach_one_process()
1121 if (pp->pp_pmcs[ri].pp_pmc == pm) {/* already present at slot [ri] */ in pmc_attach_one_process()
1131 pmc_link_target_process(pm, pp); in pmc_attach_one_process()
1133 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) && in pmc_attach_one_process()
1134 (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) == 0) in pmc_attach_one_process()
1135 pm->pm_flags |= PMC_F_NEEDS_LOGFILE; in pmc_attach_one_process()
1137 pm->pm_flags |= PMC_F_ATTACH_DONE; /* mark as attached */ in pmc_attach_one_process()
1140 if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) { in pmc_attach_one_process()
1146 pmclog_process_pmcattach(pm, p->p_pid, fullpath); in pmc_attach_one_process()
1149 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) in pmc_attach_one_process()
1150 pmc_log_process_mappings(pm->pm_owner, p); in pmc_attach_one_process()
1165 pmc_attach_process(struct proc *p, struct pmc *pm) in pmc_attach_process() argument
1172 PMCDBG5(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm, in pmc_attach_process()
1173 PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); in pmc_attach_process()
1179 if ((pm->pm_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0) in pmc_attach_process()
1182 if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0) in pmc_attach_process()
1183 return (pmc_attach_one_process(p, pm)); in pmc_attach_process()
1193 if ((error = pmc_attach_one_process(p, pm)) != 0) in pmc_attach_process()
1209 (void)pmc_detach_process(top, pm); in pmc_attach_process()
1222 pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags) in pmc_detach_one_process() argument
1229 KASSERT(pm != NULL, in pmc_detach_one_process()
1232 ri = PMC_TO_ROWINDEX(pm); in pmc_detach_one_process()
1235 pm, ri, p, p->p_pid, p->p_comm, flags); in pmc_detach_one_process()
1240 if (pp->pp_pmcs[ri].pp_pmc != pm) in pmc_detach_one_process()
1243 pmc_unlink_target_process(pm, pp); in pmc_detach_one_process()
1246 if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) in pmc_detach_one_process()
1247 pmclog_process_pmcdetach(pm, p->p_pid); in pmc_detach_one_process()
1277 pmc_detach_process(struct proc *p, struct pmc *pm) in pmc_detach_process() argument
1283 PMCDBG5(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm, in pmc_detach_process()
1284 PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); in pmc_detach_process()
1286 if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0) in pmc_detach_process()
1287 return (pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE)); in pmc_detach_process()
1298 (void)pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE); in pmc_detach_process()
1316 if (LIST_EMPTY(&pm->pm_targets)) in pmc_detach_process()
1317 pm->pm_flags &= ~PMC_F_ATTACH_DONE; in pmc_detach_process()
1331 struct pmc *pm; in pmc_process_exec() local
1389 if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL) in pmc_process_exec()
1392 po = pm->pm_owner; in pmc_process_exec()
1395 pmclog_process_procexec(po, pm->pm_id, p->p_pid, in pmc_process_exec()
1415 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) { in pmc_process_exec()
1416 if (pmc_can_attach(pm, td->td_proc) != 0) { in pmc_process_exec()
1417 pmc_detach_one_process(td->td_proc, pm, in pmc_process_exec()
1444 struct pmc *pm; in pmc_process_csw_in() local
1475 if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL) in pmc_process_csw_in()
1478 KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)), in pmc_process_csw_in()
1480 __LINE__, PMC_TO_MODE(pm))); in pmc_process_csw_in()
1481 KASSERT(PMC_TO_ROWINDEX(pm) == ri, in pmc_process_csw_in()
1483 __LINE__, PMC_TO_ROWINDEX(pm), ri)); in pmc_process_csw_in()
1489 if (pm->pm_state != PMC_STATE_RUNNING) in pmc_process_csw_in()
1492 KASSERT(counter_u64_fetch(pm->pm_runcount) >= 0, in pmc_process_csw_in()
1493 ("[pmc,%d] pm=%p runcount %ju", __LINE__, pm, in pmc_process_csw_in()
1494 (uintmax_t)counter_u64_fetch(pm->pm_runcount))); in pmc_process_csw_in()
1497 counter_u64_add(pm->pm_runcount, 1); in pmc_process_csw_in()
1501 (void)pcd->pcd_config_pmc(cpu, adjri, pm); in pmc_process_csw_in()
1508 KASSERT(phw->phw_pmc == pm, in pmc_process_csw_in()
1510 phw->phw_pmc, pm)); in pmc_process_csw_in()
1519 if (PMC_TO_MODE(pm) == PMC_MODE_TS) { in pmc_process_csw_in()
1528 mtx_pool_lock_spin(pmc_mtxpool, pm); in pmc_process_csw_in()
1543 newvalue = pm->pm_sc.pm_reloadcount; in pmc_process_csw_in()
1554 pm->pm_sc.pm_reloadcount; in pmc_process_csw_in()
1556 mtx_pool_unlock_spin(pmc_mtxpool, pm); in pmc_process_csw_in()
1558 pm->pm_sc.pm_reloadcount, in pmc_process_csw_in()
1561 cpu, ri, newvalue, pm->pm_sc.pm_reloadcount)); in pmc_process_csw_in()
1563 KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC, in pmc_process_csw_in()
1565 PMC_TO_MODE(pm))); in pmc_process_csw_in()
1566 mtx_pool_lock_spin(pmc_mtxpool, pm); in pmc_process_csw_in()
1568 pm->pm_gv.pm_savedvalue; in pmc_process_csw_in()
1569 mtx_pool_unlock_spin(pmc_mtxpool, pm); in pmc_process_csw_in()
1574 (void)pcd->pcd_write_pmc(cpu, adjri, pm, newvalue); in pmc_process_csw_in()
1577 if (PMC_TO_MODE(pm) == PMC_MODE_TS) in pmc_process_csw_in()
1578 pm->pm_pcpu_state[cpu].pps_stalled = 0; in pmc_process_csw_in()
1581 pm->pm_pcpu_state[cpu].pps_cpustate = 1; in pmc_process_csw_in()
1584 (void)pcd->pcd_start_pmc(cpu, adjri, pm); in pmc_process_csw_in()
1602 struct pmc *pm; in pmc_process_csw_out() local
1654 pm = NULL; in pmc_process_csw_out()
1655 (void)(*pcd->pcd_get_config)(cpu, adjri, &pm); in pmc_process_csw_out()
1657 if (pm == NULL) /* nothing at this row index */ in pmc_process_csw_out()
1660 mode = PMC_TO_MODE(pm); in pmc_process_csw_out()
1664 KASSERT(PMC_TO_ROWINDEX(pm) == ri, in pmc_process_csw_out()
1666 __LINE__, PMC_TO_ROWINDEX(pm), ri)); in pmc_process_csw_out()
1674 pm->pm_pcpu_state[cpu].pps_cpustate = 0; in pmc_process_csw_out()
1675 if (pm->pm_pcpu_state[cpu].pps_stalled == 0) in pmc_process_csw_out()
1676 (void)pcd->pcd_stop_pmc(cpu, adjri, pm); in pmc_process_csw_out()
1678 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0, in pmc_process_csw_out()
1679 ("[pmc,%d] pm=%p runcount %ju", __LINE__, pm, in pmc_process_csw_out()
1680 (uintmax_t)counter_u64_fetch(pm->pm_runcount))); in pmc_process_csw_out()
1683 counter_u64_add(pm->pm_runcount, -1); in pmc_process_csw_out()
1689 if (pm->pm_state != PMC_STATE_DELETED && pp != NULL && in pmc_process_csw_out()
1691 KASSERT(pm == pp->pp_pmcs[ri].pp_pmc, in pmc_process_csw_out()
1693 pm, ri, pp->pp_pmcs[ri].pp_pmc)); in pmc_process_csw_out()
1698 (void)pcd->pcd_read_pmc(cpu, adjri, pm, &newvalue); in pmc_process_csw_out()
1712 mtx_pool_lock_spin(pmc_mtxpool, pm); in pmc_process_csw_out()
1740 pm->pm_sc.pm_reloadcount) { in pmc_process_csw_out()
1742 pm->pm_sc.pm_reloadcount; in pmc_process_csw_out()
1745 mtx_pool_unlock_spin(pmc_mtxpool, pm); in pmc_process_csw_out()
1764 mtx_pool_lock_spin(pmc_mtxpool, pm); in pmc_process_csw_out()
1765 pm->pm_gv.pm_savedvalue += tmp; in pmc_process_csw_out()
1767 mtx_pool_unlock_spin(pmc_mtxpool, pm); in pmc_process_csw_out()
1769 if (pm->pm_flags & PMC_F_LOG_PROCCSW) in pmc_process_csw_out()
1770 pmclog_process_proccsw(pm, pp, tmp, td); in pmc_process_csw_out()
1831 const struct pmc *pm; in pmc_process_mmap() local
1860 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL && in pmc_process_mmap()
1861 PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { in pmc_process_mmap()
1862 pmclog_process_map_in(pm->pm_owner, in pmc_process_mmap()
1879 const struct pmc *pm; in pmc_process_munmap() local
1899 pm = pp->pp_pmcs[ri].pp_pmc; in pmc_process_munmap()
1900 if (pm != NULL && PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { in pmc_process_munmap()
1901 pmclog_process_map_out(pm->pm_owner, pid, in pmc_process_munmap()
1911 pmc_log_kernel_mappings(struct pmc *pm) in pmc_log_kernel_mappings() argument
1917 KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)), in pmc_log_kernel_mappings()
1919 __LINE__, (void *) pm)); in pmc_log_kernel_mappings()
1921 po = pm->pm_owner; in pmc_log_kernel_mappings()
1925 if (PMC_TO_MODE(pm) == PMC_MODE_SS) in pmc_log_kernel_mappings()
1926 pmc_process_allproc(pm); in pmc_log_kernel_mappings()
2655 pmc_destroy_pmc_descriptor(struct pmc *pm) in pmc_destroy_pmc_descriptor() argument
2658 KASSERT(pm->pm_state == PMC_STATE_DELETED || in pmc_destroy_pmc_descriptor()
2659 pm->pm_state == PMC_STATE_FREE, in pmc_destroy_pmc_descriptor()
2661 KASSERT(LIST_EMPTY(&pm->pm_targets), in pmc_destroy_pmc_descriptor()
2663 KASSERT(pm->pm_owner == NULL, in pmc_destroy_pmc_descriptor()
2665 KASSERT(counter_u64_fetch(pm->pm_runcount) == 0, in pmc_destroy_pmc_descriptor()
2667 (uintmax_t)counter_u64_fetch(pm->pm_runcount))); in pmc_destroy_pmc_descriptor()
2669 counter_u64_free(pm->pm_runcount); in pmc_destroy_pmc_descriptor()
2670 free(pm->pm_pcpu_state, M_PMC); in pmc_destroy_pmc_descriptor()
2671 free(pm, M_PMC); in pmc_destroy_pmc_descriptor()
2675 pmc_wait_for_pmc_idle(struct pmc *pm) in pmc_wait_for_pmc_idle() argument
2686 pmclog_flush(pm->pm_owner, 1); in pmc_wait_for_pmc_idle()
2687 while (counter_u64_fetch(pm->pm_runcount) > 0) { in pmc_wait_for_pmc_idle()
2688 pmclog_flush(pm->pm_owner, 1); in pmc_wait_for_pmc_idle()
2693 "pmc to be free", __LINE__, PMC_TO_ROWINDEX(pm), in pmc_wait_for_pmc_idle()
2694 (uintmax_t)counter_u64_fetch(pm->pm_runcount))); in pmc_wait_for_pmc_idle()
2712 pmc_release_pmc_descriptor(struct pmc *pm) in pmc_release_pmc_descriptor() argument
2724 KASSERT(pm, ("[pmc,%d] null pmc", __LINE__)); in pmc_release_pmc_descriptor()
2726 ri = PMC_TO_ROWINDEX(pm); in pmc_release_pmc_descriptor()
2728 mode = PMC_TO_MODE(pm); in pmc_release_pmc_descriptor()
2730 PMCDBG3(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri, in pmc_release_pmc_descriptor()
2743 cpu = PMC_TO_CPU(pm); in pmc_release_pmc_descriptor()
2747 pm->pm_pcpu_state[cpu].pps_cpustate = 0; in pmc_release_pmc_descriptor()
2748 if (pm->pm_state == PMC_STATE_RUNNING && in pmc_release_pmc_descriptor()
2749 pm->pm_pcpu_state[cpu].pps_stalled == 0) { in pmc_release_pmc_descriptor()
2753 KASSERT(phw->phw_pmc == pm, in pmc_release_pmc_descriptor()
2755 __LINE__, ri, phw->phw_pmc, pm)); in pmc_release_pmc_descriptor()
2759 (void)pcd->pcd_stop_pmc(cpu, adjri, pm); in pmc_release_pmc_descriptor()
2770 if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) { in pmc_release_pmc_descriptor()
2771 po = pm->pm_owner; in pmc_release_pmc_descriptor()
2779 pm->pm_state = PMC_STATE_DELETED; in pmc_release_pmc_descriptor()
2787 pmc_wait_for_pmc_idle(pm); in pmc_release_pmc_descriptor()
2799 pm->pm_state = PMC_STATE_DELETED; in pmc_release_pmc_descriptor()
2802 pmc_wait_for_pmc_idle(pm); in pmc_release_pmc_descriptor()
2811 LIST_FOREACH_SAFE(ptgt, &pm->pm_targets, pt_next, tmp) { in pmc_release_pmc_descriptor()
2813 pmc_unlink_target_process(pm, pp); /* frees 'ptgt' */ in pmc_release_pmc_descriptor()
2833 (void)pcd->pcd_release_pmc(cpu, adjri, pm); in pmc_release_pmc_descriptor()
2838 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) in pmc_release_pmc_descriptor()
2844 if (pm->pm_owner != NULL) { in pmc_release_pmc_descriptor()
2845 LIST_REMOVE(pm, pm_next); in pmc_release_pmc_descriptor()
2846 pm->pm_owner = NULL; in pmc_release_pmc_descriptor()
2909 struct pmc *pm; in pmc_can_allocate_rowindex() local
2925 LIST_FOREACH(pm, &po->po_pmcs, pm_next) { in pmc_can_allocate_rowindex()
2926 if (PMC_TO_ROWINDEX(pm) == ri) { in pmc_can_allocate_rowindex()
2927 mode = PMC_TO_MODE(pm); in pmc_can_allocate_rowindex()
2931 PMC_TO_CPU(pm) == cpu) in pmc_can_allocate_rowindex()
2996 struct pmc *pm; in pmc_find_pmc_descriptor_in_process() local
3002 LIST_FOREACH(pm, &po->po_pmcs, pm_next) { in pmc_find_pmc_descriptor_in_process()
3003 if (pm->pm_id == pmcid) in pmc_find_pmc_descriptor_in_process()
3004 return (pm); in pmc_find_pmc_descriptor_in_process()
3013 struct pmc *pm, *opm; in pmc_find_pmc() local
3042 if ((pm = pmc_find_pmc_descriptor_in_process(po, pmcid)) == NULL) in pmc_find_pmc()
3045 PMCDBG2(PMC,FND,2, "find-pmc id=%d -> pmc=%p", pmcid, pm); in pmc_find_pmc()
3047 *pmc = pm; in pmc_find_pmc()
3055 pmc_start(struct pmc *pm) in pmc_start() argument
3064 KASSERT(pm != NULL, in pmc_start()
3067 mode = PMC_TO_MODE(pm); in pmc_start()
3068 ri = PMC_TO_ROWINDEX(pm); in pmc_start()
3072 po = pm->pm_owner; in pmc_start()
3074 PMCDBG3(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri); in pmc_start()
3076 po = pm->pm_owner; in pmc_start()
3082 if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) != 0 && in pmc_start()
3090 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) in pmc_start()
3091 pmc_log_kernel_mappings(pm); in pmc_start()
3098 if (LIST_EMPTY(&pm->pm_targets)) { in pmc_start()
3099 error = (pm->pm_flags & PMC_F_ATTACH_DONE) != 0 ? in pmc_start()
3100 ESRCH : pmc_attach_process(po->po_owner, pm); in pmc_start()
3108 pm->pm_state = PMC_STATE_RUNNING; in pmc_start()
3109 if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) != 0) in pmc_start()
3145 cpu = PMC_TO_CPU(pm); in pmc_start()
3154 pm->pm_state = PMC_STATE_RUNNING; in pmc_start()
3157 v = PMC_IS_SAMPLING_MODE(mode) ? pm->pm_sc.pm_reloadcount : in pmc_start()
3158 pm->pm_sc.pm_initial; in pmc_start()
3159 if ((error = pcd->pcd_write_pmc(cpu, adjri, pm, v)) == 0) { in pmc_start()
3162 pm->pm_pcpu_state[cpu].pps_stalled = 0; in pmc_start()
3165 pm->pm_pcpu_state[cpu].pps_cpustate = 1; in pmc_start()
3166 error = pcd->pcd_start_pmc(cpu, adjri, pm); in pmc_start()
3178 pmc_stop(struct pmc *pm) in pmc_stop() argument
3185 KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__)); in pmc_stop()
3187 PMCDBG3(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm, PMC_TO_MODE(pm), in pmc_stop()
3188 PMC_TO_ROWINDEX(pm)); in pmc_stop()
3190 pm->pm_state = PMC_STATE_STOPPED; in pmc_stop()
3199 if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) in pmc_stop()
3210 cpu = PMC_TO_CPU(pm); in pmc_stop()
3218 ri = PMC_TO_ROWINDEX(pm); in pmc_stop()
3221 pm->pm_pcpu_state[cpu].pps_cpustate = 0; in pmc_stop()
3223 if ((error = pcd->pcd_stop_pmc(cpu, adjri, pm)) == 0) { in pmc_stop()
3224 error = pcd->pcd_read_pmc(cpu, adjri, pm, in pmc_stop()
3225 &pm->pm_sc.pm_initial); in pmc_stop()
3232 po = pm->pm_owner; in pmc_stop()
3233 if (PMC_TO_MODE(pm) == PMC_MODE_SS) { in pmc_stop()
3534 struct pmc *pm; in pmc_do_op_pmcattach() local
3546 error = pmc_find_pmc(a.pm_pmc, &pm); in pmc_do_op_pmcattach()
3550 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) in pmc_do_op_pmcattach()
3554 if (pm->pm_state == PMC_STATE_RUNNING) { in pmc_do_op_pmcattach()
3556 } else if (pm->pm_state != PMC_STATE_ALLOCATED && in pmc_do_op_pmcattach()
3557 pm->pm_state != PMC_STATE_STOPPED) { in pmc_do_op_pmcattach()
3581 error = pmc_attach_process(p, pm); in pmc_do_op_pmcattach()
3592 struct pmc *pm; in pmc_do_op_pmcdetach() local
3601 error = pmc_find_pmc(a.pm_pmc, &pm); in pmc_do_op_pmcdetach()
3620 error = pmc_detach_process(p, pm); in pmc_do_op_pmcdetach()
3632 struct pmc *pm; in pmc_do_op_pmcrelease() local
3647 error = pmc_find_pmc(pmcid, &pm); in pmc_do_op_pmcrelease()
3651 po = pm->pm_owner; in pmc_do_op_pmcrelease()
3652 pmc_release_pmc_descriptor(pm); in pmc_do_op_pmcrelease()
3654 pmc_destroy_pmc_descriptor(pm); in pmc_do_op_pmcrelease()
3667 struct pmc *pm; in pmc_do_op_pmcrw() local
3678 error = pmc_find_pmc(prw->pm_pmcid, &pm); in pmc_do_op_pmcrw()
3683 if (pm->pm_state != PMC_STATE_ALLOCATED && in pmc_do_op_pmcrw()
3684 pm->pm_state != PMC_STATE_STOPPED && in pmc_do_op_pmcrw()
3685 pm->pm_state != PMC_STATE_RUNNING) in pmc_do_op_pmcrw()
3689 if (pm->pm_state == PMC_STATE_RUNNING && in pmc_do_op_pmcrw()
3693 if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) { in pmc_do_op_pmcrw()
3705 ri = PMC_TO_ROWINDEX(pm); in pmc_do_op_pmcrw()
3708 mtx_pool_lock_spin(pmc_mtxpool, pm); in pmc_do_op_pmcrw()
3712 if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) && in pmc_do_op_pmcrw()
3713 (pm->pm_state == PMC_STATE_RUNNING)) { in pmc_do_op_pmcrw()
3714 error = (*pcd->pcd_read_pmc)(cpu, adjri, pm, in pmc_do_op_pmcrw()
3717 *valp = pm->pm_gv.pm_savedvalue; in pmc_do_op_pmcrw()
3722 pm->pm_gv.pm_savedvalue = prw->pm_value; in pmc_do_op_pmcrw()
3724 mtx_pool_unlock_spin(pmc_mtxpool, pm); in pmc_do_op_pmcrw()
3726 cpu = PMC_TO_CPU(pm); in pmc_do_op_pmcrw()
3727 ri = PMC_TO_ROWINDEX(pm); in pmc_do_op_pmcrw()
3740 error = (*pcd->pcd_read_pmc)(cpu, adjri, pm, valp); in pmc_do_op_pmcrw()
3744 error = (*pcd->pcd_write_pmc)(cpu, adjri, pm, in pmc_do_op_pmcrw()
3814 struct pmc *pm; in pmc_syscall_handler() local
3853 LIST_FOREACH(pm, &po->po_pmcs, pm_next) in pmc_syscall_handler()
3854 if (pm->pm_flags & PMC_F_NEEDS_LOGFILE && in pmc_syscall_handler()
3855 pm->pm_state == PMC_STATE_RUNNING) in pmc_syscall_handler()
3856 pmc_stop(pm); in pmc_syscall_handler()
4032 struct pmc *pm; in pmc_syscall_handler() local
4076 if ((error = pcd->pcd_describe(cpu, ari, p, &pm)) != 0) in pmc_syscall_handler()
4088 if (pm == NULL) /* no PMC associated */ in pmc_syscall_handler()
4091 po = pm->pm_owner; in pmc_syscall_handler()
4098 p->pm_mode = PMC_TO_MODE(pm); in pmc_syscall_handler()
4099 p->pm_event = pm->pm_event; in pmc_syscall_handler()
4100 p->pm_flags = pm->pm_flags; in pmc_syscall_handler()
4102 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) in pmc_syscall_handler()
4104 pm->pm_sc.pm_reloadcount; in pmc_syscall_handler()
4275 struct pmc *pm; in pmc_syscall_handler() local
4285 if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0) in pmc_syscall_handler()
4299 if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) || in pmc_syscall_handler()
4300 (pm->pm_flags & PMC_F_DESCENDANTS)) { in pmc_syscall_handler()
4313 if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL || in pmc_syscall_handler()
4315 pt->pt_process->pp_proc != pm->pm_owner->po_owner) { in pmc_syscall_handler()
4320 ri = PMC_TO_ROWINDEX(pm); in pmc_syscall_handler()
4397 struct pmc *pm; in pmc_syscall_handler() local
4405 if ((error = pmc_find_pmc(sc.pm_pmcid, &pm)) != 0) in pmc_syscall_handler()
4408 if (pm->pm_state == PMC_STATE_RUNNING) { in pmc_syscall_handler()
4413 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { in pmc_syscall_handler()
4423 pm->pm_sc.pm_reloadcount = MAX(MAX(1, pmc_mincount), in pmc_syscall_handler()
4426 pm->pm_sc.pm_initial = sc.pm_count; in pmc_syscall_handler()
4438 struct pmc *pm; in pmc_syscall_handler() local
4448 if ((error = pmc_find_pmc(pmcid, &pm)) != 0) in pmc_syscall_handler()
4451 KASSERT(pmcid == pm->pm_id, in pmc_syscall_handler()
4453 pm->pm_id, pmcid)); in pmc_syscall_handler()
4455 if (pm->pm_state == PMC_STATE_RUNNING) /* already running */ in pmc_syscall_handler()
4457 else if (pm->pm_state != PMC_STATE_STOPPED && in pmc_syscall_handler()
4458 pm->pm_state != PMC_STATE_ALLOCATED) { in pmc_syscall_handler()
4463 error = pmc_start(pm); in pmc_syscall_handler()
4475 struct pmc *pm; in pmc_syscall_handler() local
4490 if ((error = pmc_find_pmc(pmcid, &pm)) != 0) in pmc_syscall_handler()
4493 KASSERT(pmcid == pm->pm_id, in pmc_syscall_handler()
4495 pm->pm_id, pmcid)); in pmc_syscall_handler()
4497 if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */ in pmc_syscall_handler()
4499 else if (pm->pm_state != PMC_STATE_RUNNING) { in pmc_syscall_handler()
4504 error = pmc_stop(pm); in pmc_syscall_handler()
4602 pmc_add_sample(ring_type_t ring, struct pmc *pm, struct trapframe *tf) in pmc_add_sample() argument
4621 pm->pm_pcpu_state[cpu].pps_stalled = 1; in pmc_add_sample()
4624 cpu, pm, tf, inuserspace, in pmc_add_sample()
4633 PMCDBG6(SAM,INT,1,"cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", cpu, pm, tf, in pmc_add_sample()
4638 ps->ps_pmc = pm; in pmc_add_sample()
4647 callchaindepth = (pm->pm_flags & PMC_F_CALLCHAIN) ? in pmc_add_sample()
4673 KASSERT(counter_u64_fetch(pm->pm_runcount) >= 0, in pmc_add_sample()
4674 ("[pmc,%d] pm=%p runcount %ju", __LINE__, pm, in pmc_add_sample()
4675 (uintmax_t)counter_u64_fetch(pm->pm_runcount))); in pmc_add_sample()
4677 counter_u64_add(pm->pm_runcount, 1); /* hold onto PMC */ in pmc_add_sample()
4695 pmc_process_interrupt(int ring, struct pmc *pm, struct trapframe *tf) in pmc_process_interrupt() argument
4700 if ((pm->pm_flags & PMC_F_USERCALLCHAIN) && in pmc_process_interrupt()
4703 return (pmc_add_sample(PMC_UR, pm, tf)); in pmc_process_interrupt()
4705 return (pmc_add_sample(ring, pm, tf)); in pmc_process_interrupt()
4716 struct pmc *pm; in pmc_capture_user_callchain() local
4759 pm = ps->ps_pmc; in pmc_capture_user_callchain()
4760 KASSERT(pm->pm_flags & PMC_F_CALLCHAIN, in pmc_capture_user_callchain()
4763 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0, in pmc_capture_user_callchain()
4765 (uintmax_t)counter_u64_fetch(pm->pm_runcount))); in pmc_capture_user_callchain()
4798 counter_u64_add(pm->pm_runcount, -1); in pmc_capture_user_callchain()
4827 struct pmc *pm; in pmc_process_samples() local
4851 pm = ps->ps_pmc; in pmc_process_samples()
4852 if (pm->pm_state != PMC_STATE_RUNNING) in pmc_process_samples()
4855 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0, in pmc_process_samples()
4856 ("[pmc,%d] pm=%p runcount %ju", __LINE__, pm, in pmc_process_samples()
4857 (uintmax_t)counter_u64_fetch(pm->pm_runcount))); in pmc_process_samples()
4858 KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)), in pmc_process_samples()
4860 pm, PMC_TO_MODE(pm))); in pmc_process_samples()
4862 po = pm->pm_owner; in pmc_process_samples()
4886 pm, ps->ps_nsamples, ps->ps_flags, in pmc_process_samples()
4901 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) { in pmc_process_samples()
4907 pmclog_process_callchain(pm, ps); in pmc_process_samples()
4911 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0, in pmc_process_samples()
4912 ("[pmc,%d] pm=%p runcount %ju", __LINE__, pm, in pmc_process_samples()
4913 (uintmax_t)counter_u64_fetch(pm->pm_runcount))); in pmc_process_samples()
4915 counter_u64_add(pm->pm_runcount, -1); in pmc_process_samples()
4935 (void)(*pcd->pcd_get_config)(cpu, adjri, &pm); in pmc_process_samples()
4937 if (pm == NULL || /* !cfg'ed */ in pmc_process_samples()
4938 pm->pm_state != PMC_STATE_RUNNING || /* !active */ in pmc_process_samples()
4939 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || /* !sampling */ in pmc_process_samples()
4940 !pm->pm_pcpu_state[cpu].pps_cpustate || /* !desired */ in pmc_process_samples()
4941 !pm->pm_pcpu_state[cpu].pps_stalled) /* !stalled */ in pmc_process_samples()
4944 pm->pm_pcpu_state[cpu].pps_stalled = 0; in pmc_process_samples()
4945 (void)(*pcd->pcd_start_pmc)(cpu, adjri, pm); in pmc_process_samples()
4971 struct pmc *pm; in pmc_process_exit() local
5037 pm = NULL; in pmc_process_exit()
5040 (void)(*pcd->pcd_get_config)(cpu, adjri, &pm); in pmc_process_exit()
5042 PMCDBG2(PRC,EXT,2, "ri=%d pm=%p", ri, pm); in pmc_process_exit()
5044 if (pm == NULL || !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) in pmc_process_exit()
5048 pp->pp_pmcs[ri].pp_pmc, pm, pm->pm_state); in pmc_process_exit()
5050 KASSERT(PMC_TO_ROWINDEX(pm) == ri, in pmc_process_exit()
5052 PMC_TO_ROWINDEX(pm), ri)); in pmc_process_exit()
5053 KASSERT(pm == pp->pp_pmcs[ri].pp_pmc, in pmc_process_exit()
5054 ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__, pm, ri, in pmc_process_exit()
5056 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0, in pmc_process_exit()
5058 (uintmax_t)counter_u64_fetch(pm->pm_runcount))); in pmc_process_exit()
5066 if (pm->pm_pcpu_state[cpu].pps_cpustate) { in pmc_process_exit()
5067 pm->pm_pcpu_state[cpu].pps_cpustate = 0; in pmc_process_exit()
5068 if (!pm->pm_pcpu_state[cpu].pps_stalled) { in pmc_process_exit()
5069 (void)pcd->pcd_stop_pmc(cpu, adjri, pm); in pmc_process_exit()
5071 if (PMC_TO_MODE(pm) == PMC_MODE_TC) { in pmc_process_exit()
5072 pcd->pcd_read_pmc(cpu, adjri, pm, in pmc_process_exit()
5076 mtx_pool_lock_spin(pmc_mtxpool, pm); in pmc_process_exit()
5077 pm->pm_gv.pm_savedvalue += tmp; in pmc_process_exit()
5079 mtx_pool_unlock_spin(pmc_mtxpool, pm); in pmc_process_exit()
5084 KASSERT(counter_u64_fetch(pm->pm_runcount) > 0, in pmc_process_exit()
5087 counter_u64_add(pm->pm_runcount, -1); in pmc_process_exit()
5105 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) { in pmc_process_exit()
5106 if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) != 0 && in pmc_process_exit()
5107 PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm))) { in pmc_process_exit()
5108 pmclog_process_procexit(pm, pp); in pmc_process_exit()
5110 pmc_unlink_target_process(pm, pp); in pmc_process_exit()
5139 struct pmc *pm; in pmc_process_fork() local
5181 if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL && in pmc_process_fork()
5182 (pm->pm_flags & PMC_F_DESCENDANTS) != 0) { in pmc_process_fork()
5211 if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL && in pmc_process_fork()
5212 (pm->pm_flags & PMC_F_DESCENDANTS) != 0) { in pmc_process_fork()
5213 pmc_link_target_process(pm, ppnew); in pmc_process_fork()
5214 po = pm->pm_owner; in pmc_process_fork()
5267 pmc_process_allproc(struct pmc *pm) in pmc_process_allproc() argument
5273 po = pm->pm_owner; in pmc_process_allproc()