6586 dtrace_action_breakpoint(dtrace_ecb_t *ecb)  in dtrace_action_breakpoint()  argument
6588 	dtrace_probe_t *probe = ecb->dte_probe;  in dtrace_action_breakpoint()
6594 	uintptr_t val = (uintptr_t)ecb;  in dtrace_action_breakpoint()
6649 dtrace_action_panic(dtrace_ecb_t *ecb)  in dtrace_action_panic()  argument
6651 	dtrace_probe_t *probe = ecb->dte_probe;  in dtrace_action_panic()
6674 	    probe->dtpr_func, probe->dtpr_name, (void *)ecb);  in dtrace_action_panic()
7055 	dtrace_ecb_t *ecb;  in dtrace_probe()  local
7166 	for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {  in dtrace_probe()
7167 		dtrace_predicate_t *pred = ecb->dte_predicate;  in dtrace_probe()
7168 		dtrace_state_t *state = ecb->dte_state;  in dtrace_probe()
7231 		if (ecb->dte_cond) {  in dtrace_probe()
7240 			if ((ecb->dte_cond & DTRACE_COND_USERMODE) &&  in dtrace_probe()
7258 			if (ecb->dte_cond & DTRACE_COND_OWNER) {  in dtrace_probe()
7261 				    ecb->dte_state->dts_cred.dcr_cred;  in dtrace_probe()
7287 			if (ecb->dte_cond & DTRACE_COND_ZONEOWNER) {  in dtrace_probe()
7290 				    ecb->dte_state->dts_cred.dcr_cred;  in dtrace_probe()
7330 		if ((offs = dtrace_buffer_reserve(buf, ecb->dte_needed,  in dtrace_probe()
7331 		    ecb->dte_alignment, state, &mstate)) < 0)  in dtrace_probe()
7340 		if (ecb->dte_size != 0) {  in dtrace_probe()
7348 			ASSERT(ecb->dte_size >= sizeof(dtrace_rechdr_t));  in dtrace_probe()
7350 			dtrh.dtrh_epid = ecb->dte_epid;  in dtrace_probe()
7355 		mstate.dtms_epid = ecb->dte_epid;  in dtrace_probe()
7385 		for (act = ecb->dte_action; !(*flags & CPU_DTRACE_ERROR) &&  in dtrace_probe()
7428 					dtrace_action_breakpoint(ecb);  in dtrace_probe()
7433 					dtrace_action_panic(ecb);  in dtrace_probe()
7514 				    ecb->dte_needed, ecb->dte_alignment,  in dtrace_probe()
7525 				if (ecb->dte_size == 0)  in dtrace_probe()
7528 				ASSERT(ecb->dte_size >= sizeof(dtrace_rechdr_t));  in dtrace_probe()
7530 				dtrh->dtrh_epid = ecb->dte_epid;  in dtrace_probe()
7565 				if (ecb->dte_size)  in dtrace_probe()
7566 					buf->dtb_offset = offs + ecb->dte_size;  in dtrace_probe()
7743 			for (err = ecb->dte_action, ndx = 0;  in dtrace_probe()
7747 			dtrace_probe_error(state, ecb->dte_epid, ndx,  in dtrace_probe()
7756 			buf->dtb_offset = offs + ecb->dte_size;  in dtrace_probe()
11014 	dtrace_ecb_t *ecb;  in dtrace_ecb_add()  local
11019 	ecb = kmem_zalloc(sizeof (dtrace_ecb_t), KM_SLEEP);  in dtrace_ecb_add()
11020 	ecb->dte_predicate = NULL;  in dtrace_ecb_add()
11021 	ecb->dte_probe = probe;  in dtrace_ecb_add()
11027 	ecb->dte_size = ecb->dte_needed = sizeof (dtrace_rechdr_t);  in dtrace_ecb_add()
11028 	ecb->dte_alignment = sizeof (dtrace_epid_t);  in dtrace_ecb_add()
11069 	ecb->dte_state = state;  in dtrace_ecb_add()
11073 	state->dts_ecbs[(ecb->dte_epid = epid) - 1] = ecb;  in dtrace_ecb_add()
11075 	return (ecb);  in dtrace_ecb_add()
11079 dtrace_ecb_enable(dtrace_ecb_t *ecb)  in dtrace_ecb_enable()  argument
11081 	dtrace_probe_t *probe = ecb->dte_probe;  in dtrace_ecb_enable()
11085 	ASSERT(ecb->dte_next == NULL);  in dtrace_ecb_enable()
11101 		probe->dtpr_ecb = probe->dtpr_ecb_last = ecb;  in dtrace_ecb_enable()
11103 		if (ecb->dte_predicate != NULL)  in dtrace_ecb_enable()
11104 			probe->dtpr_predcache = ecb->dte_predicate->dtp_cacheid;  in dtrace_ecb_enable()
11115 		probe->dtpr_ecb_last->dte_next = ecb;  in dtrace_ecb_enable()
11116 		probe->dtpr_ecb_last = ecb;  in dtrace_ecb_enable()
11125 dtrace_ecb_resize(dtrace_ecb_t *ecb)  in dtrace_ecb_resize()  argument
11135 	ecb->dte_size = sizeof (dtrace_rechdr_t);  in dtrace_ecb_resize()
11136 	ecb->dte_alignment = sizeof (dtrace_epid_t);  in dtrace_ecb_resize()
11138 	for (act = ecb->dte_action; act != NULL; act = act->dta_next) {  in dtrace_ecb_resize()
11142 		ecb->dte_alignment = MAX(ecb->dte_alignment, rec->dtrd_alignment);  in dtrace_ecb_resize()
11159 			ecb->dte_needed = MAX(ecb->dte_needed, curneeded);  in dtrace_ecb_resize()
11173 				curneeded = P2PHASEUP(ecb->dte_size,  in dtrace_ecb_resize()
11189 			ecb->dte_size = P2ROUNDUP(ecb->dte_size, rec->dtrd_alignment);  in dtrace_ecb_resize()
11190 			rec->dtrd_offset = ecb->dte_size;  in dtrace_ecb_resize()
11191 			if (ecb->dte_size + rec->dtrd_size < ecb->dte_size)  in dtrace_ecb_resize()
11193 			ecb->dte_size += rec->dtrd_size;  in dtrace_ecb_resize()
11194 			ecb->dte_needed = MAX(ecb->dte_needed, ecb->dte_size);  in dtrace_ecb_resize()
11198 	if ((act = ecb->dte_action) != NULL &&  in dtrace_ecb_resize()
11200 	    ecb->dte_size == sizeof (dtrace_rechdr_t)) {  in dtrace_ecb_resize()
11205 		ecb->dte_size = 0;  in dtrace_ecb_resize()
11208 	ecb->dte_size = P2ROUNDUP(ecb->dte_size, sizeof (dtrace_epid_t));  in dtrace_ecb_resize()
11209 	ecb->dte_needed = P2ROUNDUP(ecb->dte_needed, (sizeof (dtrace_epid_t)));  in dtrace_ecb_resize()
11210 	ecb->dte_state->dts_needed = MAX(ecb->dte_state->dts_needed, ecb->dte_needed);  in dtrace_ecb_resize()
11215 dtrace_ecb_aggregation_create(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)  in dtrace_ecb_aggregation_create()  argument
11223 	dtrace_state_t *state = ecb->dte_state;  in dtrace_ecb_aggregation_create()
11226 	agg->dtag_ecb = ecb;  in dtrace_ecb_aggregation_create()
11319 	for (act = ecb->dte_action_last; act != NULL; act = act->dta_prev) {  in dtrace_ecb_aggregation_create()
11345 	ASSERT(ecb->dte_action_last != NULL);  in dtrace_ecb_aggregation_create()
11346 	act = ecb->dte_action_last;  in dtrace_ecb_aggregation_create()
11401 dtrace_ecb_aggregation_destroy(dtrace_ecb_t *ecb, dtrace_action_t *act)  in dtrace_ecb_aggregation_destroy()  argument
11404 	dtrace_state_t *state = ecb->dte_state;  in dtrace_ecb_aggregation_destroy()
11417 dtrace_ecb_action_add(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)  in dtrace_ecb_action_add()  argument
11424 	dtrace_state_t *state = ecb->dte_state;  in dtrace_ecb_action_add()
11430 	ASSERT(ecb->dte_action == NULL || ecb->dte_action->dta_refcnt == 1);  in dtrace_ecb_action_add()
11439 		for (act = ecb->dte_action; act != NULL; act = act->dta_next) {  in dtrace_ecb_action_add()
11447 		action = dtrace_ecb_aggregation_create(ecb, desc);  in dtrace_ecb_action_add()
11583 			if (ecb->dte_size > sizeof (dtrace_rechdr_t))  in dtrace_ecb_action_add()
11593 			dtrace_action_t *act = ecb->dte_action;  in dtrace_ecb_action_add()
11615 			dtrace_action_t *act = ecb->dte_action;  in dtrace_ecb_action_add()
11649 	if ((last = ecb->dte_action_last) != NULL) {  in dtrace_ecb_action_add()
11650 		ASSERT(ecb->dte_action != NULL);  in dtrace_ecb_action_add()
11654 		ASSERT(ecb->dte_action == NULL);  in dtrace_ecb_action_add()
11655 		ecb->dte_action = action;  in dtrace_ecb_action_add()
11658 	ecb->dte_action_last = action;  in dtrace_ecb_action_add()
11664 dtrace_ecb_action_remove(dtrace_ecb_t *ecb)  in dtrace_ecb_action_remove()  argument
11666 	dtrace_action_t *act = ecb->dte_action, *next;  in dtrace_ecb_action_remove()
11667 	dtrace_vstate_t *vstate = &ecb->dte_state->dts_vstate;  in dtrace_ecb_action_remove()
11677 			ASSERT(next != NULL || act == ecb->dte_action_last);  in dtrace_ecb_action_remove()
11681 				dtrace_format_remove(ecb->dte_state, format);  in dtrace_ecb_action_remove()
11687 				dtrace_ecb_aggregation_destroy(ecb, act);  in dtrace_ecb_action_remove()
11694 	ecb->dte_action = NULL;  in dtrace_ecb_action_remove()
11695 	ecb->dte_action_last = NULL;  in dtrace_ecb_action_remove()
11696 	ecb->dte_size = 0;  in dtrace_ecb_action_remove()
11700 dtrace_ecb_disable(dtrace_ecb_t *ecb)  in dtrace_ecb_disable()  argument
11706 	dtrace_probe_t *probe = ecb->dte_probe;  in dtrace_ecb_disable()
11718 		if (pecb == ecb)  in dtrace_ecb_disable()
11726 		probe->dtpr_ecb = ecb->dte_next;  in dtrace_ecb_disable()
11728 		prev->dte_next = ecb->dte_next;  in dtrace_ecb_disable()
11731 	if (ecb == probe->dtpr_ecb_last) {  in dtrace_ecb_disable()
11732 		ASSERT(ecb->dte_next == NULL);  in dtrace_ecb_disable()
11751 		ASSERT(ecb->dte_next == NULL);  in dtrace_ecb_disable()
11775 		ecb->dte_next = NULL;  in dtrace_ecb_disable()
11780 dtrace_ecb_destroy(dtrace_ecb_t *ecb)  in dtrace_ecb_destroy()  argument
11782 	dtrace_state_t *state = ecb->dte_state;  in dtrace_ecb_destroy()
11785 	dtrace_epid_t epid = ecb->dte_epid;  in dtrace_ecb_destroy()
11788 	ASSERT(ecb->dte_next == NULL);  in dtrace_ecb_destroy()
11789 	ASSERT(ecb->dte_probe == NULL || ecb->dte_probe->dtpr_ecb != ecb);  in dtrace_ecb_destroy()
11791 	if ((pred = ecb->dte_predicate) != NULL)  in dtrace_ecb_destroy()
11794 	dtrace_ecb_action_remove(ecb);  in dtrace_ecb_destroy()
11796 	ASSERT(state->dts_ecbs[epid - 1] == ecb);  in dtrace_ecb_destroy()
11799 	kmem_free(ecb, sizeof (dtrace_ecb_t));  in dtrace_ecb_destroy()
11806 	dtrace_ecb_t *ecb;  in dtrace_ecb_create()  local
11815 	ecb = dtrace_ecb_add(state, probe);  in dtrace_ecb_create()
11816 	ecb->dte_uarg = desc->dted_uarg;  in dtrace_ecb_create()
11820 		ecb->dte_predicate = pred;  in dtrace_ecb_create()
11838 			ecb->dte_cond |= DTRACE_COND_OWNER;  in dtrace_ecb_create()
11842 			ecb->dte_cond |= DTRACE_COND_ZONEOWNER;  in dtrace_ecb_create()
11851 			ecb->dte_cond |= DTRACE_COND_USERMODE;  in dtrace_ecb_create()
11865 			ecb->dte_action = act_if;  in dtrace_ecb_create()
11866 			ecb->dte_action_last = cached->dte_action_last;  in dtrace_ecb_create()
11867 			ecb->dte_needed = cached->dte_needed;  in dtrace_ecb_create()
11868 			ecb->dte_size = cached->dte_size;  in dtrace_ecb_create()
11869 			ecb->dte_alignment = cached->dte_alignment;  in dtrace_ecb_create()
11872 		return (ecb);  in dtrace_ecb_create()
11876 		if ((enab->dten_error = dtrace_ecb_action_add(ecb, act)) != 0) {  in dtrace_ecb_create()
11877 			dtrace_ecb_destroy(ecb);  in dtrace_ecb_create()
11882 	if ((enab->dten_error = dtrace_ecb_resize(ecb)) != 0) {  in dtrace_ecb_create()
11883 		dtrace_ecb_destroy(ecb);  in dtrace_ecb_create()
11887 	return (dtrace_ecb_create_cache = ecb);  in dtrace_ecb_create()
11893 	dtrace_ecb_t *ecb;  in dtrace_ecb_create_enable()  local
11909 	if ((ecb = dtrace_ecb_create(state, probe, enab)) == NULL)  in dtrace_ecb_create_enable()
11912 	if (dtrace_ecb_enable(ecb) < 0)  in dtrace_ecb_create_enable()
11921 	dtrace_ecb_t *ecb;  in dtrace_epid2ecb()  local
11922 #pragma unused(ecb) /* __APPLE__ */  in dtrace_epid2ecb()
11930 	ASSERT((ecb = state->dts_ecbs[id - 1]) == NULL || ecb->dte_epid == id);  in dtrace_epid2ecb()
12496 dtrace_enabling_add(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb)  in dtrace_enabling_add()  argument
12509 	if (ecb == NULL) return;  in dtrace_enabling_add()
12512 		enab->dten_desc[enab->dten_ndesc++] = ecb;  in dtrace_enabling_add()
12532 	enab->dten_desc[enab->dten_ndesc++] = ecb;  in dtrace_enabling_add()
12536 dtrace_enabling_addlike(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb,  in dtrace_enabling_addlike()  argument
12549 	if ((pred = ecb->dted_pred.dtpdd_predicate) != NULL)  in dtrace_enabling_addlike()
12552 	for (act = ecb->dted_action; act != NULL; act = act->dtad_next)  in dtrace_enabling_addlike()
12555 	new->dted_action = ecb->dted_action;  in dtrace_enabling_addlike()
12556 	new->dted_pred = ecb->dted_pred;  in dtrace_enabling_addlike()
12558 	new->dted_uarg = ecb->dted_uarg;  in dtrace_enabling_addlike()
13624 	dof_ecbdesc_t *ecb;  in dtrace_dof_ecbdesc()  local
13643 	ecb = (dof_ecbdesc_t *)((uintptr_t)dof + (uintptr_t)sec->dofs_offset);  in dtrace_dof_ecbdesc()
13644 	sec = dtrace_dof_sect(dof, DOF_SECT_PROBEDESC, ecb->dofe_probes);  in dtrace_dof_ecbdesc()
13650 	ep->dted_uarg = ecb->dofe_uarg;  in dtrace_dof_ecbdesc()
13656 	if (ecb->dofe_pred != DOF_SECIDX_NONE) {  in dtrace_dof_ecbdesc()
13658 		    DOF_SECT_DIFOHDR, ecb->dofe_pred)) == NULL)  in dtrace_dof_ecbdesc()
13667 	if (ecb->dofe_actions != DOF_SECIDX_NONE) {  in dtrace_dof_ecbdesc()
13669 		    DOF_SECT_ACTDESC, ecb->dofe_actions)) == NULL)  in dtrace_dof_ecbdesc()
14477 	dtrace_ecb_t *ecb;  in dtrace_state_prereserve()  local
14492 	for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {  in dtrace_state_prereserve()
14493 		if (ecb->dte_state != state)  in dtrace_state_prereserve()
14496 		state->dts_reserve += ecb->dte_needed + ecb->dte_alignment;  in dtrace_state_prereserve()
14895 	dtrace_ecb_t *ecb;  in dtrace_state_destroy()  local
14942 			if ((ecb = state->dts_ecbs[i]) == NULL)  in dtrace_state_destroy()
14945 			if (match && ecb->dte_probe != NULL) {  in dtrace_state_destroy()
14946 				dtrace_probe_t *probe = ecb->dte_probe;  in dtrace_state_destroy()
14953 			dtrace_ecb_disable(ecb);  in dtrace_state_destroy()
14954 			dtrace_ecb_destroy(ecb);  in dtrace_state_destroy()
17893 		dtrace_ecb_t *ecb;  in dtrace_ioctl()  local
17905 		if ((ecb = dtrace_epid2ecb(state, epdesc.dtepd_epid)) == NULL) {  in dtrace_ioctl()
17910 		if (ecb->dte_probe == NULL) {  in dtrace_ioctl()
17915 		epdesc.dtepd_probeid = ecb->dte_probe->dtpr_id;  in dtrace_ioctl()
17916 		epdesc.dtepd_uarg = ecb->dte_uarg;  in dtrace_ioctl()
17917 		epdesc.dtepd_size = ecb->dte_size;  in dtrace_ioctl()
17921 		for (act = ecb->dte_action; act != NULL; act = act->dta_next) {  in dtrace_ioctl()
17943 		for (act = ecb->dte_action; act != NULL; act = act->dta_next) {  in dtrace_ioctl()