211 static bool	pmc_can_allocate_row(int ri, enum pmc_mode mode);
212 static bool	pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,
610 pmc_ri_to_classdep(struct pmc_mdep *md __unused, int ri, int *adjri)  in pmc_ri_to_classdep()  argument
614 	KASSERT(ri >= 0 && ri < md->pmd_npmc,  in pmc_ri_to_classdep()
615 	    ("[pmc,%d] illegal row-index %d", __LINE__, ri));  in pmc_ri_to_classdep()
617 	pcd = pmc_rowindex_to_classdep[ri];  in pmc_ri_to_classdep()
619 	    ("[pmc,%d] ri %d null pcd", __LINE__, ri));  in pmc_ri_to_classdep()
621 	*adjri = ri - pcd->pcd_ri;  in pmc_ri_to_classdep()
903 	int ri;  in pmc_link_target_process()  local
915 	ri = PMC_TO_ROWINDEX(pm);  in pmc_link_target_process()
918 	    pm, ri, pp);  in pmc_link_target_process()
932 	atomic_store_rel_ptr((uintptr_t *)&pp->pp_pmcs[ri].pp_pmc,  in pmc_link_target_process()
941 	pp->pp_pmcs[ri].pp_pmcval = PMC_TO_MODE(pm) == PMC_MODE_TS ?  in pmc_link_target_process()
950 			KASSERT(pt_td->pt_pmcs[ri].pt_pmcval == (pmc_value_t) 0,  in pmc_link_target_process()
952 			    "ri=%d", __LINE__, pp->pp_proc->p_pid, ri));  in pmc_link_target_process()
965 	int ri;  in pmc_unlink_target_process()  local
979 	ri = PMC_TO_ROWINDEX(pm);  in pmc_unlink_target_process()
982 	    pm, ri, pp);  in pmc_unlink_target_process()
984 	KASSERT(pp->pp_pmcs[ri].pp_pmc == pm,  in pmc_unlink_target_process()
986 		ri, pm, pp->pp_pmcs[ri].pp_pmc));  in pmc_unlink_target_process()
988 	pp->pp_pmcs[ri].pp_pmc = NULL;  in pmc_unlink_target_process()
989 	pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t)0;  in pmc_unlink_target_process()
995 			pt->pt_pmcs[ri].pt_pmcval = (pmc_value_t)0;  in pmc_unlink_target_process()
1090 	int ri, error;  in pmc_attach_one_process()  local
1109 	ri = PMC_TO_ROWINDEX(pm);  in pmc_attach_one_process()
1121 	if (pp->pp_pmcs[ri].pp_pmc == pm) {/* already present at slot [ri] */  in pmc_attach_one_process()
1126 	if (pp->pp_pmcs[ri].pp_pmc != NULL) {  in pmc_attach_one_process()
1224 	int ri;  in pmc_detach_one_process()  local
1232 	ri = PMC_TO_ROWINDEX(pm);  in pmc_detach_one_process()
1235 	    pm, ri, p, p->p_pid, p->p_comm, flags);  in pmc_detach_one_process()
1240 	if (pp->pp_pmcs[ri].pp_pmc != pm)  in pmc_detach_one_process()
1336 	u_int ri;  in pmc_process_exec()  local
1388 	for (ri = 0; ri < md->pmd_npmc; ri++) {  in pmc_process_exec()
1389 		if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL)  in pmc_process_exec()
1414 	for (ri = 0; ri < md->pmd_npmc; ri++) {  in pmc_process_exec()
1415 		if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {  in pmc_process_exec()
1453 	u_int adjri, ri;  in pmc_process_csw_in()  local
1474 	for (ri = 0; ri < md->pmd_npmc; ri++) {  in pmc_process_csw_in()
1475 		if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL)  in pmc_process_csw_in()
1481 		KASSERT(PMC_TO_ROWINDEX(pm) == ri,  in pmc_process_csw_in()
1483 		    __LINE__, PMC_TO_ROWINDEX(pm), ri));  in pmc_process_csw_in()
1500 		pcd = pmc_ri_to_classdep(md, ri, &adjri);  in pmc_process_csw_in()
1503 		phw = pc->pc_hwpmcs[ri];  in pmc_process_csw_in()
1540 				if (pt->pt_pmcs[ri].pt_pmcval > 0)  in pmc_process_csw_in()
1541 					newvalue = pt->pt_pmcs[ri].pt_pmcval;  in pmc_process_csw_in()
1552 				newvalue = pp->pp_pmcs[ri].pp_pmcval;  in pmc_process_csw_in()
1553 				pp->pp_pmcs[ri].pp_pmcval =  in pmc_process_csw_in()
1561 			    cpu, ri, newvalue, pm->pm_sc.pm_reloadcount));  in pmc_process_csw_in()
1567 			newvalue = PMC_PCPU_SAVED(cpu, ri) =  in pmc_process_csw_in()
1572 		PMCDBG3(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue);  in pmc_process_csw_in()
1612 	u_int adjri, ri;  in pmc_process_csw_out()  local
1652 	for (ri = 0; ri < md->pmd_npmc; ri++) {  in pmc_process_csw_out()
1653 		pcd = pmc_ri_to_classdep(md, ri, &adjri);  in pmc_process_csw_out()
1664 		KASSERT(PMC_TO_ROWINDEX(pm) == ri,  in pmc_process_csw_out()
1666 			__LINE__, PMC_TO_ROWINDEX(pm), ri));  in pmc_process_csw_out()
1690 		    pp->pp_pmcs[ri].pp_pmc != NULL) {  in pmc_process_csw_out()
1691 			KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,  in pmc_process_csw_out()
1693 				pm, ri, pp->pp_pmcs[ri].pp_pmc));  in pmc_process_csw_out()
1702 				    cpu, ri, newvalue);  in pmc_process_csw_out()
1724 					pt->pt_pmcs[ri].pt_pmcval = newvalue;  in pmc_process_csw_out()
1738 					pp->pp_pmcs[ri].pp_pmcval += newvalue;  in pmc_process_csw_out()
1739 					if (pp->pp_pmcs[ri].pp_pmcval >  in pmc_process_csw_out()
1741 						pp->pp_pmcs[ri].pp_pmcval -=  in pmc_process_csw_out()
1747 				tmp = newvalue - PMC_PCPU_SAVED(cpu, ri);  in pmc_process_csw_out()
1750 				    cpu, ri, tmp);  in pmc_process_csw_out()
1761 				     "incr=%jx", __LINE__, cpu, ri,  in pmc_process_csw_out()
1762 				     newvalue, PMC_PCPU_SAVED(cpu, ri), tmp));  in pmc_process_csw_out()
1766 				pp->pp_pmcs[ri].pp_pmcval += tmp;  in pmc_process_csw_out()
1836 	int ri;  in pmc_process_mmap()  local
1859 	for (ri = 0; ri < md->pmd_npmc; ri++) {  in pmc_process_mmap()
1860 		if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&  in pmc_process_mmap()
1883 	int ri;  in pmc_process_munmap()  local
1898 	for (ri = 0; ri < md->pmd_npmc; ri++) {  in pmc_process_munmap()
1899 		pm = pp->pp_pmcs[ri].pp_pmc;  in pmc_process_munmap()
2721 	u_int adjri, ri, cpu;  in pmc_release_pmc_descriptor()  local
2726 	ri   = PMC_TO_ROWINDEX(pm);  in pmc_release_pmc_descriptor()
2727 	pcd  = pmc_ri_to_classdep(md, ri, &adjri);  in pmc_release_pmc_descriptor()
2730 	PMCDBG3(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri,  in pmc_release_pmc_descriptor()
2751 			phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];  in pmc_release_pmc_descriptor()
2755 				__LINE__, ri, phw->phw_pmc, pm));  in pmc_release_pmc_descriptor()
2756 			PMCDBG2(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri);  in pmc_release_pmc_descriptor()
2763 		PMCDBG2(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri);  in pmc_release_pmc_descriptor()
2839 		PMC_UNMARK_ROW_STANDALONE(ri);  in pmc_release_pmc_descriptor()
2841 		PMC_UNMARK_ROW_THREAD(ri);  in pmc_release_pmc_descriptor()
2891 pmc_getrowdisp(int ri)  in pmc_getrowdisp()  argument
2893 	return (pmc_pmcdisp[ri]);  in pmc_getrowdisp()
2907 pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu)  in pmc_can_allocate_rowindex()  argument
2915 	    "cpu=%d", p, p->p_pid, p->p_comm, ri, cpu);  in pmc_can_allocate_rowindex()
2926 			if (PMC_TO_ROWINDEX(pm) == ri) {  in pmc_can_allocate_rowindex()
2942 		if (pp->pp_pmcs[ri].pp_pmc != NULL)  in pmc_can_allocate_rowindex()
2946 	    p, p->p_pid, p->p_comm, ri);  in pmc_can_allocate_rowindex()
2955 pmc_can_allocate_row(int ri, enum pmc_mode mode)  in pmc_can_allocate_row()  argument
2961 	PMCDBG2(PMC,ALR,1, "can-allocate-row ri=%d mode=%d", ri, mode);  in pmc_can_allocate_row()
2978 	if (!PMC_ROW_DISP_IS_FREE(ri) &&  in pmc_can_allocate_row()
2979 	    !(disp == PMC_DISP_THREAD && PMC_ROW_DISP_IS_THREAD(ri)) &&  in pmc_can_allocate_row()
2980 	    !(disp == PMC_DISP_STANDALONE && PMC_ROW_DISP_IS_STANDALONE(ri)))  in pmc_can_allocate_row()
2986 	PMCDBG2(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode);  in pmc_can_allocate_row()
3062 	int adjri, error, cpu, ri;  in pmc_start()  local
3068 	ri   = PMC_TO_ROWINDEX(pm);  in pmc_start()
3069 	pcd  = pmc_ri_to_classdep(md, ri, &adjri);  in pmc_start()
3074 	PMCDBG3(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri);  in pmc_start()
3183 	int adjri, cpu, error, ri;  in pmc_stop()  local
3218 	ri = PMC_TO_ROWINDEX(pm);  in pmc_stop()
3219 	pcd = pmc_ri_to_classdep(md, ri, &adjri);  in pmc_stop()
3668 	u_int cpu, ri, adjri;  in pmc_do_op_pmcrw()  local
3705 		ri = PMC_TO_ROWINDEX(pm);  in pmc_do_op_pmcrw()
3706 		pcd = pmc_ri_to_classdep(md, ri, &adjri);  in pmc_do_op_pmcrw()
3727 		ri  = PMC_TO_ROWINDEX(pm);  in pmc_do_op_pmcrw()
3728 		pcd = pmc_ri_to_classdep(md, ri, &adjri);  in pmc_do_op_pmcrw()
3756 		    ri, prw->pm_value, *valp);  in pmc_do_op_pmcrw()
3758 		PMCDBG2(PMC,OPS,2, "rw id=%d -> old %jx", ri, *valp);  in pmc_do_op_pmcrw()
4125 		int cpu, ri;  in pmc_syscall_handler()  local
4164 		ri = pma.pm_pmc; /* pmc id == row index */  in pmc_syscall_handler()
4165 		if (ri < 0 || ri >= (int) md->pmd_npmc) {  in pmc_syscall_handler()
4175 		if (PMC_ROW_DISP_IS_THREAD(ri) &&  in pmc_syscall_handler()
4190 		phw = pc->pc_hwpmcs[ri];  in pmc_syscall_handler()
4200 				PMC_MARK_ROW_STANDALONE(ri);  in pmc_syscall_handler()
4204 				PMC_UNMARK_ROW_STANDALONE(ri);  in pmc_syscall_handler()
4274 		int adjri, ri;  in pmc_syscall_handler()  local
4320 		ri = PMC_TO_ROWINDEX(pm);  in pmc_syscall_handler()
4321 		pcd = pmc_ri_to_classdep(md, ri, &adjri);  in pmc_syscall_handler()
4976 	int ri, adjri, cpu;  in pmc_process_exit()  local
5032 	for (ri = 0; ri < md->pmd_npmc; ri++) {  in pmc_process_exit()
5038 		pcd = pmc_ri_to_classdep(md, ri, &adjri);  in pmc_process_exit()
5042 		PMCDBG2(PRC,EXT,2, "ri=%d pm=%p", ri, pm);  in pmc_process_exit()
5047 		PMCDBG4(PRC,EXT,2, "ppmcs[%d]=%p pm=%p state=%d", ri,  in pmc_process_exit()
5048 		    pp->pp_pmcs[ri].pp_pmc, pm, pm->pm_state);  in pmc_process_exit()
5050 		KASSERT(PMC_TO_ROWINDEX(pm) == ri,  in pmc_process_exit()
5052 		    PMC_TO_ROWINDEX(pm), ri));  in pmc_process_exit()
5053 		KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,  in pmc_process_exit()
5054 		    ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__, pm, ri,  in pmc_process_exit()
5055 		    pp->pp_pmcs[ri].pp_pmc));  in pmc_process_exit()
5057 		    ("[pmc,%d] bad runcount ri %d rc %ju", __LINE__, ri,  in pmc_process_exit()
5074 					tmp = newvalue - PMC_PCPU_SAVED(cpu, ri);  in pmc_process_exit()
5078 					pp->pp_pmcs[ri].pp_pmcval += tmp;  in pmc_process_exit()
5085 		    ("[pmc,%d] runcount is %d", __LINE__, ri));  in pmc_process_exit()
5104 	for (ri = 0; ri < md->pmd_npmc; ri++) {  in pmc_process_exit()
5105 		if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {  in pmc_process_exit()
5142 	unsigned int ri;  in pmc_process_fork()  local
5180 	for (ri = 0; ri < md->pmd_npmc; ri++) {  in pmc_process_fork()
5181 		if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL &&  in pmc_process_fork()
5210 	for (ri = 0; ri < md->pmd_npmc; ri++) {  in pmc_process_fork()
5211 		if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL &&  in pmc_process_fork()
5419 	int c, cpu, error, n, ri;  in pmc_initialize()  local
5495 	for (ri = c = 0; c < md->pmd_nclass; c++) {  in pmc_initialize()
5497 		pcd->pcd_ri = ri;  in pmc_initialize()
5498 		ri += pcd->pcd_num;  in pmc_initialize()
5511 	for (ri = c = 0; c < md->pmd_nclass; c++) {  in pmc_initialize()
5513 		for (n = 0; n < pcd->pcd_num; n++, ri++)  in pmc_initialize()
5514 			pmc_rowindex_to_classdep[ri] = pcd;  in pmc_initialize()
5517 	KASSERT(ri == md->pmd_npmc,  in pmc_initialize()
5519 	    ri, md->pmd_npmc));  in pmc_initialize()