49 callstack_fixup_user(struct kp_ucallstack *cs, thread_t thread)  in callstack_fixup_user()  argument
52 	assert(cs->kpuc_nframes < MAX_UCALLSTACK_FRAMES);  in callstack_fixup_user()
88 		cs->kpuc_frames[0] |= 1ULL;  in callstack_fixup_user()
99 	cs->kpuc_frames[cs->kpuc_nframes++] = fixup_val;  in callstack_fixup_user()
111 	uint64_t cs;  in interrupted_kernel_sp_value()  local
122 		cs = saved_state64(state)->isf.cs;  in interrupted_kernel_sp_value()
124 		cs = saved_state32(state)->cs;  in interrupted_kernel_sp_value()
127 	if ((cs & SEL_PL) == SEL_PL_U) {  in interrupted_kernel_sp_value()
172 callstack_fixup_interrupted(struct kp_kcallstack *cs)  in callstack_fixup_interrupted()  argument
175 	assert(cs->kpkc_nframes < MAX_KCALLSTACK_FRAMES);  in callstack_fixup_interrupted()
188 	assert(cs->kpkc_flags & CALLSTACK_KERNEL);  in callstack_fixup_interrupted()
189 	cs->kpkc_frames[cs->kpkc_nframes++] = fixup_val;  in callstack_fixup_interrupted()
193 kperf_continuation_sample(struct kp_kcallstack *cs, struct kperf_context *context)  in kperf_continuation_sample()  argument
197 	assert(cs != NULL);  in kperf_continuation_sample()
204 	cs->kpkc_flags = CALLSTACK_CONTINUATION | CALLSTACK_VALID | CALLSTACK_KERNEL;  in kperf_continuation_sample()
206 	cs->kpkc_flags |= CALLSTACK_64BIT;  in kperf_continuation_sample()
209 	cs->kpkc_nframes = 1;  in kperf_continuation_sample()
210 	cs->kpkc_frames[0] = VM_KERNEL_UNSLIDE(thread->continuation);  in kperf_continuation_sample()
214 kperf_backtrace_sample(struct kp_kcallstack *cs, struct kperf_context *context)  in kperf_backtrace_sample()  argument
216 	assert(cs != NULL);  in kperf_backtrace_sample()
220 	cs->kpkc_flags = CALLSTACK_KERNEL | CALLSTACK_KERNEL_WORDS;  in kperf_backtrace_sample()
222 	cs->kpkc_flags |= CALLSTACK_64BIT;  in kperf_backtrace_sample()
231 	cs->kpkc_nframes = backtrace(cs->kpkc_word_frames, cs->kpkc_nframes - 1,  in kperf_backtrace_sample()
233 	if (cs->kpkc_nframes > 0) {  in kperf_backtrace_sample()
234 		cs->kpkc_flags |= CALLSTACK_VALID;  in kperf_backtrace_sample()
236 		cs->kpkc_exclaves_offset = 0;  in kperf_backtrace_sample()
239 			cs->kpkc_exclaves_offset = exclaves_stack_offset(cs->kpkc_word_frames, cs->kpkc_nframes, true);  in kperf_backtrace_sample()
247 		cs->kpkc_word_frames[cs->kpkc_nframes + 1] = 0;  in kperf_backtrace_sample()
248 		cs->kpkc_nframes += 1;  in kperf_backtrace_sample()
251 		cs->kpkc_flags |= CALLSTACK_TRUNCATED;  in kperf_backtrace_sample()
254 	BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, cs->kpkc_nframes);  in kperf_backtrace_sample()
262 kperf_kcallstack_sample(struct kp_kcallstack *cs, struct kperf_context *context)  in kperf_kcallstack_sample()  argument
266 	assert(cs != NULL);  in kperf_kcallstack_sample()
268 	assert(cs->kpkc_nframes <= MAX_KCALLSTACK_FRAMES);  in kperf_kcallstack_sample()
274 	    cs->kpkc_nframes);  in kperf_kcallstack_sample()
276 	cs->kpkc_flags = CALLSTACK_KERNEL;  in kperf_kcallstack_sample()
278 	cs->kpkc_flags |= CALLSTACK_64BIT;  in kperf_kcallstack_sample()
283 		cs->kpkc_flags |= CALLSTACK_KERNEL_WORDS;  in kperf_kcallstack_sample()
286 		cs->kpkc_nframes = backtrace(cs->kpkc_word_frames, cs->kpkc_nframes - 1,  in kperf_kcallstack_sample()
288 		if (cs->kpkc_nframes != 0) {  in kperf_kcallstack_sample()
289 			callstack_fixup_interrupted(cs);  in kperf_kcallstack_sample()
292 			cs->kpkc_flags |= CALLSTACK_TRUNCATED;  in kperf_kcallstack_sample()
295 		cs->kpkc_exclaves_offset = 0;  in kperf_kcallstack_sample()
298 			cs->kpkc_exclaves_offset = exclaves_stack_offset(cs->kpkc_word_frames, cs->kpkc_nframes, true);  in kperf_kcallstack_sample()
308 		    cs->kpkc_frames, &cs->kpkc_nframes, FALSE);  in kperf_kcallstack_sample()
310 			cs->kpkc_flags |= CALLSTACK_VALID;  in kperf_kcallstack_sample()
312 			cs->kpkc_flags |= CALLSTACK_VALID;  in kperf_kcallstack_sample()
313 			cs->kpkc_flags |= CALLSTACK_TRUNCATED;  in kperf_kcallstack_sample()
315 			cs->kpkc_nframes = 0;  in kperf_kcallstack_sample()
319 	if (!(cs->kpkc_flags & CALLSTACK_VALID)) {  in kperf_kcallstack_sample()
324 	    cs->kpkc_flags, cs->kpkc_nframes);  in kperf_kcallstack_sample()
328 kperf_ucallstack_sample(struct kp_ucallstack *cs, struct kperf_context *context)  in kperf_ucallstack_sample()  argument
336 	    (uintptr_t)thread_tid(thread), cs->kpuc_nframes);  in kperf_ucallstack_sample()
342 	unsigned int maxnframes = cs->kpuc_nframes - 1;  in kperf_ucallstack_sample()
344 	unsigned int nframes = backtrace_user(cs->kpuc_frames, maxnframes, &ctl,  in kperf_ucallstack_sample()
346 	cs->kpuc_nframes = MIN(maxnframes, nframes);  in kperf_ucallstack_sample()
348 	cs->kpuc_flags |= CALLSTACK_KERNEL_WORDS |  in kperf_ucallstack_sample()
356 		callstack_fixup_user(cs, thread);  in kperf_ucallstack_sample()
357 		cs->kpuc_flags |= CALLSTACK_VALID;  in kperf_ucallstack_sample()
359 		if (cs->kpuc_nframes < maxnframes &&  in kperf_ucallstack_sample()
361 			cs->kpuc_async_index = btinfo.btui_async_start_index;  in kperf_ucallstack_sample()
364 			maxnframes -= cs->kpuc_nframes;  in kperf_ucallstack_sample()
367 			    &cs->kpuc_frames[cs->kpuc_nframes], maxnframes, &ctl, &btinfo);  in kperf_ucallstack_sample()
369 				cs->kpuc_flags |= CALLSTACK_TRUNCATED;  in kperf_ucallstack_sample()
372 				cs->kpuc_flags |= CALLSTACK_HAS_ASYNC;  in kperf_ucallstack_sample()
373 				cs->kpuc_async_nframes = nasync_frames;  in kperf_ucallstack_sample()
377 		cs->kpuc_nframes = 0;  in kperf_ucallstack_sample()
382 	    cs->kpuc_flags, cs->kpuc_nframes);  in kperf_ucallstack_sample()
451 kperf_kcallstack_log(struct kp_kcallstack *cs)  in kperf_kcallstack_log()  argument
453 	callstack_log(PERF_CS_KHDR, PERF_CS_KDATA, cs->kpkc_frames,  in kperf_kcallstack_log()
454 	    cs->kpkc_nframes, cs->kpkc_flags, 0, 0);  in kperf_kcallstack_log()
456 	if (cs->kpkc_exclaves_offset != 0) {  in kperf_kcallstack_log()
457 		BUF_DATA(PERF_CS_KEXOFFSET, cs->kpkc_exclaves_offset);  in kperf_kcallstack_log()
462 kperf_ucallstack_log(struct kp_ucallstack *cs)  in kperf_ucallstack_log()  argument
464 	callstack_log(PERF_CS_UHDR, PERF_CS_UDATA, cs->kpuc_frames,  in kperf_ucallstack_log()
465 	    cs->kpuc_nframes + cs->kpuc_async_nframes, cs->kpuc_flags,  in kperf_ucallstack_log()
466 	    cs->kpuc_async_index, cs->kpuc_async_nframes);  in kperf_ucallstack_log()
1240 				supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U);
1244 				supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U);
1275 			supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U);
1279 			supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U);