1 /*
2 * Copyright (c) 2011-2022 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /* Collect kernel callstacks */
30
31 #include <mach/mach_types.h>
32 #include <kern/thread.h>
33 #include <kern/backtrace.h>
34 #include <kern/cambria_layout.h>
35 #include <vm/vm_map_xnu.h>
36 #include <kperf/buffer.h>
37 #include <kperf/context.h>
38 #include <kperf/callstack.h>
39 #include <kperf/ast.h>
40 #include <sys/errno.h>
41 #include <mach/exclaves.h>
42
43 #if defined(__arm64__)
44 #include <arm/cpu_data.h>
45 #include <arm/cpu_data_internal.h>
46 #endif
47
48 static void
callstack_fixup_user(struct kp_ucallstack * cs,thread_t thread)49 callstack_fixup_user(struct kp_ucallstack *cs, thread_t thread)
50 {
51 uint64_t fixup_val = 0;
52 assert(cs->kpuc_nframes < MAX_UCALLSTACK_FRAMES);
53
54 #if defined(__x86_64__)
55 user_addr_t sp_user;
56 bool user_64;
57 x86_saved_state_t *state;
58
59 state = get_user_regs(thread);
60 if (!state) {
61 goto out;
62 }
63
64 user_64 = is_saved_state64(state);
65 if (user_64) {
66 sp_user = saved_state64(state)->isf.rsp;
67 } else {
68 sp_user = saved_state32(state)->uesp;
69 }
70
71 if (thread == current_thread()) {
72 (void)copyin(sp_user, (char *)&fixup_val,
73 user_64 ? sizeof(uint64_t) : sizeof(uint32_t));
74 } else {
75 (void)vm_map_read_user(get_task_map(get_threadtask(thread)), sp_user,
76 &fixup_val, user_64 ? sizeof(uint64_t) : sizeof(uint32_t));
77 }
78
79 #elif defined(__arm64__)
80
81 struct arm_saved_state *state = get_user_regs(thread);
82 if (!state) {
83 goto out;
84 }
85
86 /* encode thumb mode into low bit of PC */
87 if (is_saved_state32(state) && (get_saved_state_cpsr(state) & PSR_TF)) {
88 cs->kpuc_frames[0] |= 1ULL;
89 }
90
91
92 fixup_val = get_saved_state_lr(state);
93
94 #else
95 #error "callstack_fixup_user: unsupported architecture"
96 #endif
97
98 out:
99 cs->kpuc_frames[cs->kpuc_nframes++] = fixup_val;
100 }
101
102 #if defined(__x86_64__)
103
104 __attribute__((used))
105 static kern_return_t
interrupted_kernel_sp_value(uintptr_t * sp_val)106 interrupted_kernel_sp_value(uintptr_t *sp_val)
107 {
108 x86_saved_state_t *state;
109 uintptr_t sp;
110 bool state_64;
111 uint64_t cs;
112 uintptr_t top, bottom;
113
114 state = current_cpu_datap()->cpu_int_state;
115 if (!state) {
116 return KERN_FAILURE;
117 }
118
119 state_64 = is_saved_state64(state);
120
121 if (state_64) {
122 cs = saved_state64(state)->isf.cs;
123 } else {
124 cs = saved_state32(state)->cs;
125 }
126 /* return early if interrupted a thread in user space */
127 if ((cs & SEL_PL) == SEL_PL_U) {
128 return KERN_FAILURE;
129 }
130
131 if (state_64) {
132 sp = saved_state64(state)->isf.rsp;
133 } else {
134 sp = saved_state32(state)->uesp;
135 }
136
137 /* make sure the stack pointer is pointing somewhere in this stack */
138 bottom = current_thread()->kernel_stack;
139 top = bottom + kernel_stack_size;
140 if (sp >= bottom && sp < top) {
141 return KERN_FAILURE;
142 }
143
144 *sp_val = *(uintptr_t *)sp;
145 return KERN_SUCCESS;
146 }
147
148 #elif defined(__arm64__)
149
150 __attribute__((used))
151 static kern_return_t
interrupted_kernel_lr(uintptr_t * lr)152 interrupted_kernel_lr(uintptr_t *lr)
153 {
154 struct arm_saved_state *state;
155
156 state = getCpuDatap()->cpu_int_state;
157
158 /* return early if interrupted a thread in user space */
159 if (PSR64_IS_USER(get_saved_state_cpsr(state))) {
160 return KERN_FAILURE;
161 }
162
163 *lr = get_saved_state_lr(state);
164 return KERN_SUCCESS;
165 }
166 #else /* defined(__arm64__) */
167 #error "interrupted_kernel_{sp,lr}: unsupported architecture"
168 #endif /* !defined(__arm64__) */
169
170
171 static void
callstack_fixup_interrupted(struct kp_kcallstack * cs)172 callstack_fixup_interrupted(struct kp_kcallstack *cs)
173 {
174 uintptr_t fixup_val = 0;
175 assert(cs->kpkc_nframes < MAX_KCALLSTACK_FRAMES);
176
177 /*
178 * Only provide arbitrary data on development or debug kernels.
179 */
180 #if DEVELOPMENT || DEBUG
181 #if defined(__x86_64__)
182 (void)interrupted_kernel_sp_value(&fixup_val);
183 #elif defined(__arm64__)
184 (void)interrupted_kernel_lr(&fixup_val);
185 #endif /* defined(__x86_64__) */
186 #endif /* DEVELOPMENT || DEBUG */
187
188 assert(cs->kpkc_flags & CALLSTACK_KERNEL);
189 cs->kpkc_frames[cs->kpkc_nframes++] = fixup_val;
190 }
191
192 void
kperf_continuation_sample(struct kp_kcallstack * cs,struct kperf_context * context)193 kperf_continuation_sample(struct kp_kcallstack *cs, struct kperf_context *context)
194 {
195 thread_t thread;
196
197 assert(cs != NULL);
198 assert(context != NULL);
199
200 thread = context->cur_thread;
201 assert(thread != NULL);
202 assert(thread->continuation != NULL);
203
204 cs->kpkc_flags = CALLSTACK_CONTINUATION | CALLSTACK_VALID | CALLSTACK_KERNEL;
205 #ifdef __LP64__
206 cs->kpkc_flags |= CALLSTACK_64BIT;
207 #endif
208
209 cs->kpkc_nframes = 1;
210 cs->kpkc_frames[0] = VM_KERNEL_UNSLIDE(thread->continuation);
211 }
212
213 void
kperf_backtrace_sample(struct kp_kcallstack * cs,struct kperf_context * context)214 kperf_backtrace_sample(struct kp_kcallstack *cs, struct kperf_context *context)
215 {
216 assert(cs != NULL);
217 assert(context != NULL);
218 assert(context->cur_thread == current_thread());
219
220 cs->kpkc_flags = CALLSTACK_KERNEL | CALLSTACK_KERNEL_WORDS;
221 #ifdef __LP64__
222 cs->kpkc_flags |= CALLSTACK_64BIT;
223 #endif
224
225 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, 1);
226
227 backtrace_info_t btinfo = BTI_NONE;
228 struct backtrace_control ctl = {
229 .btc_frame_addr = (uintptr_t)context->starting_fp,
230 };
231 cs->kpkc_nframes = backtrace(cs->kpkc_word_frames, cs->kpkc_nframes - 1,
232 &ctl, &btinfo);
233 if (cs->kpkc_nframes > 0) {
234 cs->kpkc_flags |= CALLSTACK_VALID;
235
236 cs->kpkc_exclaves_offset = 0;
237 #if CONFIG_EXCLAVES
238 if ((context->cur_thread->th_exclaves_state & TH_EXCLAVES_RPC) != 0) {
239 cs->kpkc_exclaves_offset = exclaves_stack_offset(cs->kpkc_word_frames, cs->kpkc_nframes, true);
240 }
241 #endif /* CONFIG_EXCLAVES */
242
243 /*
244 * Fake the value pointed to by the stack pointer or the link
245 * register for symbolicators.
246 */
247 cs->kpkc_word_frames[cs->kpkc_nframes + 1] = 0;
248 cs->kpkc_nframes += 1;
249 }
250 if ((btinfo & BTI_TRUNCATED)) {
251 cs->kpkc_flags |= CALLSTACK_TRUNCATED;
252 }
253
254 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, cs->kpkc_nframes);
255 }
256
257 kern_return_t chudxnu_thread_get_callstack64_kperf(thread_t thread,
258 uint64_t *callStack, mach_msg_type_number_t *count,
259 boolean_t user_only);
260
261 void
kperf_kcallstack_sample(struct kp_kcallstack * cs,struct kperf_context * context)262 kperf_kcallstack_sample(struct kp_kcallstack *cs, struct kperf_context *context)
263 {
264 thread_t thread;
265
266 assert(cs != NULL);
267 assert(context != NULL);
268 assert(cs->kpkc_nframes <= MAX_KCALLSTACK_FRAMES);
269
270 thread = context->cur_thread;
271 assert(thread != NULL);
272
273 BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread),
274 cs->kpkc_nframes);
275
276 cs->kpkc_flags = CALLSTACK_KERNEL;
277 #ifdef __LP64__
278 cs->kpkc_flags |= CALLSTACK_64BIT;
279 #endif
280
281 if (ml_at_interrupt_context()) {
282 assert(thread == current_thread());
283 cs->kpkc_flags |= CALLSTACK_KERNEL_WORDS;
284 backtrace_info_t btinfo = BTI_NONE;
285 struct backtrace_control ctl = { .btc_flags = BTF_KERN_INTERRUPTED, };
286 cs->kpkc_nframes = backtrace(cs->kpkc_word_frames, cs->kpkc_nframes - 1,
287 &ctl, &btinfo);
288 if (cs->kpkc_nframes != 0) {
289 callstack_fixup_interrupted(cs);
290 }
291 if ((btinfo & BTI_TRUNCATED)) {
292 cs->kpkc_flags |= CALLSTACK_TRUNCATED;
293 }
294
295 cs->kpkc_exclaves_offset = 0;
296 #if CONFIG_EXCLAVES
297 if ((thread->th_exclaves_state & TH_EXCLAVES_RPC) != 0) {
298 cs->kpkc_exclaves_offset = exclaves_stack_offset(cs->kpkc_word_frames, cs->kpkc_nframes, true);
299 }
300 #endif /* CONFIG_EXCLAVES */
301 } else {
302 /*
303 * Rely on legacy CHUD backtracer to backtrace kernel stacks on
304 * other threads.
305 */
306 kern_return_t kr;
307 kr = chudxnu_thread_get_callstack64_kperf(thread,
308 cs->kpkc_frames, &cs->kpkc_nframes, FALSE);
309 if (kr == KERN_SUCCESS) {
310 cs->kpkc_flags |= CALLSTACK_VALID;
311 } else if (kr == KERN_RESOURCE_SHORTAGE) {
312 cs->kpkc_flags |= CALLSTACK_VALID;
313 cs->kpkc_flags |= CALLSTACK_TRUNCATED;
314 } else {
315 cs->kpkc_nframes = 0;
316 }
317 }
318
319 if (!(cs->kpkc_flags & CALLSTACK_VALID)) {
320 BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK);
321 }
322
323 BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread),
324 cs->kpkc_flags, cs->kpkc_nframes);
325 }
326
327 void
kperf_ucallstack_sample(struct kp_ucallstack * cs,struct kperf_context * context)328 kperf_ucallstack_sample(struct kp_ucallstack *cs, struct kperf_context *context)
329 {
330 assert(ml_get_interrupts_enabled() == TRUE);
331
332 thread_t thread = context->cur_thread;
333 assert(thread != NULL);
334
335 BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_START,
336 (uintptr_t)thread_tid(thread), cs->kpuc_nframes);
337
338 struct backtrace_user_info btinfo = BTUINFO_INIT;
339 /*
340 * Leave space for the fixup information.
341 */
342 unsigned int maxnframes = cs->kpuc_nframes - 1;
343 struct backtrace_control ctl = { .btc_user_thread = thread, };
344 unsigned int nframes = backtrace_user(cs->kpuc_frames, maxnframes, &ctl,
345 &btinfo);
346 cs->kpuc_nframes = MIN(maxnframes, nframes);
347
348 cs->kpuc_flags |= CALLSTACK_KERNEL_WORDS |
349 ((btinfo.btui_info & BTI_TRUNCATED) ? CALLSTACK_TRUNCATED : 0) |
350 ((btinfo.btui_info & BTI_64_BIT) ? CALLSTACK_64BIT : 0);
351
352 /*
353 * Ignore EFAULT to get as much of the stack as possible.
354 */
355 if (btinfo.btui_error == 0 || btinfo.btui_error == EFAULT) {
356 callstack_fixup_user(cs, thread);
357 cs->kpuc_flags |= CALLSTACK_VALID;
358
359 if (cs->kpuc_nframes < maxnframes &&
360 btinfo.btui_async_frame_addr != 0) {
361 cs->kpuc_async_index = btinfo.btui_async_start_index;
362 ctl.btc_frame_addr = btinfo.btui_async_frame_addr;
363 ctl.btc_addr_offset = BTCTL_ASYNC_ADDR_OFFSET;
364 maxnframes -= cs->kpuc_nframes;
365 btinfo = BTUINFO_INIT;
366 unsigned int nasync_frames = backtrace_user(
367 &cs->kpuc_frames[cs->kpuc_nframes], maxnframes, &ctl, &btinfo);
368 if (btinfo.btui_info & BTI_TRUNCATED) {
369 cs->kpuc_flags |= CALLSTACK_TRUNCATED;
370 }
371 if (btinfo.btui_error == 0 || btinfo.btui_error == EFAULT) {
372 cs->kpuc_flags |= CALLSTACK_HAS_ASYNC;
373 cs->kpuc_async_nframes = nasync_frames;
374 }
375 }
376 } else {
377 cs->kpuc_nframes = 0;
378 BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK, btinfo.btui_error);
379 }
380
381 BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread),
382 cs->kpuc_flags, cs->kpuc_nframes);
383 }
384
385 static inline uintptr_t
scrub_word(uintptr_t * bt,int n_frames,int frame,bool kern)386 scrub_word(uintptr_t *bt, int n_frames, int frame, bool kern)
387 {
388 if (frame < n_frames) {
389 if (kern) {
390 return VM_KERNEL_UNSLIDE(bt[frame]);
391 } else {
392 return bt[frame];
393 }
394 } else {
395 return 0;
396 }
397 }
398
399 static inline uintptr_t
scrub_frame(uint64_t * bt,int n_frames,int frame)400 scrub_frame(uint64_t *bt, int n_frames, int frame)
401 {
402 if (frame < n_frames) {
403 return (uintptr_t)(bt[frame]);
404 } else {
405 return 0;
406 }
407 }
408
409 static void
callstack_log(uint32_t hdrid,uint32_t dataid,void * vframes,unsigned int nframes,unsigned int flags,unsigned int async_index,unsigned int async_nframes)410 callstack_log(uint32_t hdrid, uint32_t dataid, void *vframes,
411 unsigned int nframes, unsigned int flags, unsigned int async_index,
412 unsigned int async_nframes)
413 {
414 BUF_VERB(PERF_CS_LOG | DBG_FUNC_START, flags, nframes);
415 BUF_DATA(hdrid, flags, nframes - async_nframes, async_index, async_nframes);
416
417 unsigned int nevts = nframes / 4;
418 unsigned int ovf = nframes % 4;
419 if (ovf != 0) {
420 nevts++;
421 }
422
423 bool kern = flags & CALLSTACK_KERNEL;
424
425 if (flags & CALLSTACK_KERNEL_WORDS) {
426 uintptr_t *frames = vframes;
427 for (unsigned int i = 0; i < nevts; i++) {
428 unsigned int j = i * 4;
429 BUF_DATA(dataid,
430 scrub_word(frames, nframes, j + 0, kern),
431 scrub_word(frames, nframes, j + 1, kern),
432 scrub_word(frames, nframes, j + 2, kern),
433 scrub_word(frames, nframes, j + 3, kern));
434 }
435 } else {
436 for (unsigned int i = 0; i < nevts; i++) {
437 uint64_t *frames = vframes;
438 unsigned int j = i * 4;
439 BUF_DATA(dataid,
440 scrub_frame(frames, nframes, j + 0),
441 scrub_frame(frames, nframes, j + 1),
442 scrub_frame(frames, nframes, j + 2),
443 scrub_frame(frames, nframes, j + 3));
444 }
445 }
446
447 BUF_VERB(PERF_CS_LOG | DBG_FUNC_END, flags, nframes);
448 }
449
450 void
kperf_kcallstack_log(struct kp_kcallstack * cs)451 kperf_kcallstack_log(struct kp_kcallstack *cs)
452 {
453 callstack_log(PERF_CS_KHDR, PERF_CS_KDATA, cs->kpkc_frames,
454 cs->kpkc_nframes, cs->kpkc_flags, 0, 0);
455
456 if (cs->kpkc_exclaves_offset != 0) {
457 BUF_DATA(PERF_CS_KEXOFFSET, cs->kpkc_exclaves_offset);
458 }
459 }
460
461 void
kperf_ucallstack_log(struct kp_ucallstack * cs)462 kperf_ucallstack_log(struct kp_ucallstack *cs)
463 {
464 callstack_log(PERF_CS_UHDR, PERF_CS_UDATA, cs->kpuc_frames,
465 cs->kpuc_nframes + cs->kpuc_async_nframes, cs->kpuc_flags,
466 cs->kpuc_async_index, cs->kpuc_async_nframes);
467 }
468
469 #if CONFIG_EXCLAVES
470 void
kperf_excallstack_log(const stackshottypes_ipcstackentry_s * ipcstack)471 kperf_excallstack_log(const stackshottypes_ipcstackentry_s *ipcstack)
472 {
473 __block unsigned int nframes = 0;
474 __block unsigned int flags = CALLSTACK_VALID;
475 uint64_t frames[MAX_EXCALLSTACK_FRAMES] = {};
476 uint64_t *frames_block = frames;
477
478 BUF_DATA(PERF_CS_EXSTACK, ipcstack->asid);
479
480 if (ipcstack->stacktrace.has_value) {
481 address__v_visit(&ipcstack->stacktrace.value, ^(size_t i, const stackshottypes_address_s item) {
482 if (i >= MAX_EXCALLSTACK_FRAMES) {
483 flags |= CALLSTACK_TRUNCATED;
484 return;
485 }
486 frames_block[i] = item;
487 nframes += 1;
488 });
489 callstack_log(PERF_CS_EXHDR, PERF_CS_EXDATA, frames, nframes, flags, 0, 0);
490 }
491 }
492
493 bool
kperf_exclave_callstack_pend(struct kperf_context * context,unsigned int actionid)494 kperf_exclave_callstack_pend(struct kperf_context *context, unsigned int actionid)
495 {
496 if ((context->cur_thread->th_exclaves_state & TH_EXCLAVES_RPC)
497 && (os_atomic_load(&context->cur_thread->th_exclaves_inspection_state, relaxed) & TH_EXCLAVES_INSPECTION_NOINSPECT) == 0) {
498 os_atomic_or(&context->cur_thread->th_exclaves_inspection_state, TH_EXCLAVES_INSPECTION_KPERF, relaxed);
499 context->cur_thread->kperf_exclaves_ast |= T_KPERF_SET_ACTIONID(actionid);
500 return true;
501 }
502 return false;
503 }
504 #endif /* CONFIG_EXCLAVES */
505
506 int
kperf_ucallstack_pend(struct kperf_context * context,uint32_t depth,unsigned int actionid)507 kperf_ucallstack_pend(struct kperf_context * context, uint32_t depth,
508 unsigned int actionid)
509 {
510 if (depth < 2) {
511 panic("HUH");
512 }
513 kperf_ast_set_callstack_depth(context->cur_thread, depth);
514 return kperf_ast_pend(context->cur_thread, T_KPERF_AST_CALLSTACK,
515 actionid);
516 }
517
518 static kern_return_t
chudxnu_kern_read(void * dstaddr,vm_offset_t srcaddr,vm_size_t size)519 chudxnu_kern_read(void *dstaddr, vm_offset_t srcaddr, vm_size_t size)
520 {
521 return (ml_nofault_copy(srcaddr, (vm_offset_t)dstaddr, size) == size) ?
522 KERN_SUCCESS : KERN_FAILURE;
523 }
524
525 static kern_return_t
chudxnu_task_read(task_t task,void * kernaddr,uint64_t usraddr,vm_size_t size)526 chudxnu_task_read(
527 task_t task,
528 void *kernaddr,
529 uint64_t usraddr,
530 vm_size_t size)
531 {
532 //ppc version ported to arm
533 kern_return_t ret = KERN_SUCCESS;
534
535 if (ml_at_interrupt_context()) {
536 return KERN_FAILURE; // can't look at tasks on interrupt stack
537 }
538
539 if (current_task() == task) {
540 if (copyin(usraddr, kernaddr, size)) {
541 ret = KERN_FAILURE;
542 }
543 } else {
544 vm_map_t map = get_task_map(task);
545 ret = vm_map_read_user(map, usraddr, kernaddr, size);
546 }
547
548 return ret;
549 }
550
551 static inline uint64_t
chudxnu_vm_unslide(uint64_t ptr,int kaddr)552 chudxnu_vm_unslide( uint64_t ptr, int kaddr )
553 {
554 if (!kaddr) {
555 return ptr;
556 }
557
558 return VM_KERNEL_UNSLIDE(ptr);
559 }
560
561 #if __arm64__
562
563 #if defined(HAS_APPLE_PAC)
564 #include <ptrauth.h>
565 #endif
566
567 // chudxnu_thread_get_callstack gathers a raw callstack along with any information needed to
568 // fix it up later (in case we stopped program as it was saving values into prev stack frame, etc.)
569 // after sampling has finished.
570 //
571 // For an N-entry callstack:
572 //
573 // [0] current pc
574 // [1..N-3] stack frames (including current one)
575 // [N-2] current LR (return value if we're in a leaf function)
576 // [N-1] current r0 (in case we've saved LR in r0) (optional)
577 //
578 //
579 #define CS_FLAG_EXTRASP 1 // capture extra sp register
580
581 static kern_return_t
chudxnu_thread_get_callstack64_internal(thread_t thread,uint64_t * callStack,mach_msg_type_number_t * count,boolean_t user_only,int flags)582 chudxnu_thread_get_callstack64_internal(
583 thread_t thread,
584 uint64_t *callStack,
585 mach_msg_type_number_t *count,
586 boolean_t user_only,
587 int flags)
588 {
589 kern_return_t kr = KERN_SUCCESS;
590 task_t task;
591 uint64_t currPC = 0ULL, currLR = 0ULL, currSP = 0ULL;
592 uint64_t prevPC = 0ULL;
593 uint64_t kernStackMin = thread->kernel_stack;
594 uint64_t kernStackMax = kernStackMin + kernel_stack_size;
595 uint64_t *buffer = callStack;
596 int bufferIndex = 0;
597 int bufferMaxIndex = 0;
598 boolean_t kernel = FALSE;
599 struct arm_saved_state *sstate = NULL;
600 uint64_t pc = 0ULL;
601
602 task = get_threadtask(thread);
603 bufferMaxIndex = *count;
604 //get thread state
605 if (user_only) {
606 sstate = find_user_regs(thread);
607 } else {
608 sstate = find_kern_regs(thread);
609 }
610
611 if (!sstate) {
612 *count = 0;
613 return KERN_FAILURE;
614 }
615
616 if (is_saved_state64(sstate)) {
617 struct arm_saved_state64 *state = NULL;
618 uint64_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
619 uint64_t frame[2];
620
621 state = saved_state64(sstate);
622
623 /* make sure it is safe to dereference before you do it */
624 kernel = PSR64_IS_KERNEL(state->cpsr);
625
626 /* can't take a kernel callstack if we've got a user frame */
627 if (!user_only && !kernel) {
628 return KERN_FAILURE;
629 }
630
631 /*
632 * Reserve space for saving LR (and sometimes SP) at the end of the
633 * backtrace.
634 */
635 if (flags & CS_FLAG_EXTRASP) {
636 bufferMaxIndex -= 2;
637 } else {
638 bufferMaxIndex -= 1;
639 }
640
641 if (bufferMaxIndex < 2) {
642 *count = 0;
643 return KERN_RESOURCE_SHORTAGE;
644 }
645
646 currPC = state->pc;
647 currLR = state->lr;
648 currSP = state->sp;
649
650 fp = (uint64_t *)state->fp; /* frame pointer */
651 #if defined(HAS_APPLE_PAC)
652 /* frame pointers on stack will be signed by arm64e ABI */
653 fp = ptrauth_strip(fp, ptrauth_key_frame_pointer);
654 #endif
655 topfp = fp;
656
657 bufferIndex = 0; // start with a stack of size zero
658 buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, kernel); // save PC in position 0.
659
660 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, kernel, 0);
661
662 // Now, fill buffer with stack backtraces.
663 while (bufferIndex < bufferMaxIndex) {
664 pc = 0ULL;
665 /*
666 * Below the frame pointer, the following values are saved:
667 * -> FP
668 */
669
670 /*
671 * Note that we read the pc even for the first stack frame
672 * (which, in theory, is always empty because the callee fills
673 * it in just before it lowers the stack. However, if we
674 * catch the program in between filling in the return address
675 * and lowering the stack, we want to still have a valid
676 * backtrace. FixupStack correctly disregards this value if
677 * necessary.
678 */
679
680 if ((uint64_t)fp == 0 || ((uint64_t)fp & 0x3) != 0) {
681 /* frame pointer is invalid - stop backtracing */
682 pc = 0ULL;
683 break;
684 }
685
686 if (kernel) {
687 if (((uint64_t)fp > kernStackMax) ||
688 ((uint64_t)fp < kernStackMin)) {
689 kr = KERN_FAILURE;
690 } else {
691 kr = chudxnu_kern_read(&frame,
692 (vm_offset_t)fp,
693 (vm_size_t)sizeof(frame));
694 if (kr == KERN_SUCCESS) {
695 #if defined(HAS_APPLE_PAC)
696 /* return addresses on stack will be signed by arm64e ABI */
697 pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address);
698 #else
699 pc = frame[1];
700 #endif
701 nextFramePointer = (uint64_t *)frame[0];
702 #if defined(HAS_APPLE_PAC)
703 /* frame pointers on stack will be signed by arm64e ABI */
704 nextFramePointer = ptrauth_strip(nextFramePointer, ptrauth_key_frame_pointer);
705 #endif
706 } else {
707 pc = 0ULL;
708 nextFramePointer = 0ULL;
709 kr = KERN_FAILURE;
710 }
711 }
712 } else {
713 kr = chudxnu_task_read(task,
714 &frame,
715 (vm_offset_t)fp,
716 (vm_size_t)sizeof(frame));
717 if (kr == KERN_SUCCESS) {
718 #if defined(HAS_APPLE_PAC)
719 /* return addresses on stack will be signed by arm64e ABI */
720 pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address);
721 #else
722 pc = frame[1];
723 #endif
724 nextFramePointer = (uint64_t *)(frame[0]);
725 #if defined(HAS_APPLE_PAC)
726 /* frame pointers on stack will be signed by arm64e ABI */
727 nextFramePointer = ptrauth_strip(nextFramePointer, ptrauth_key_frame_pointer);
728 #endif
729 } else {
730 pc = 0ULL;
731 nextFramePointer = 0ULL;
732 kr = KERN_FAILURE;
733 }
734 }
735
736 if (kr != KERN_SUCCESS) {
737 pc = 0ULL;
738 break;
739 }
740
741 if (nextFramePointer) {
742 buffer[bufferIndex++] = chudxnu_vm_unslide(pc, kernel);
743 prevPC = pc;
744 }
745
746 if (nextFramePointer < fp) {
747 break;
748 } else {
749 fp = nextFramePointer;
750 }
751 }
752
753 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, bufferIndex);
754
755 if (bufferIndex >= bufferMaxIndex) {
756 bufferIndex = bufferMaxIndex;
757 kr = KERN_RESOURCE_SHORTAGE;
758 } else {
759 kr = KERN_SUCCESS;
760 }
761
762 // Save link register and SP at bottom of stack (used for later fixup).
763 buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, kernel);
764 if (flags & CS_FLAG_EXTRASP) {
765 buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, kernel);
766 }
767 } else {
768 struct arm_saved_state32 *state = NULL;
769 uint32_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL;
770
771 /* 64-bit kernel stacks, 32-bit user stacks */
772 uint64_t frame[2];
773 uint32_t frame32[2];
774
775 state = saved_state32(sstate);
776
777 /* make sure it is safe to dereference before you do it */
778 kernel = PSR_IS_KERNEL(state->cpsr);
779
780 /* can't take a kernel callstack if we've got a user frame */
781 if (!user_only && !kernel) {
782 return KERN_FAILURE;
783 }
784
785 /*
786 * Reserve space for saving LR (and sometimes SP) at the end of the
787 * backtrace.
788 */
789 if (flags & CS_FLAG_EXTRASP) {
790 bufferMaxIndex -= 2;
791 } else {
792 bufferMaxIndex -= 1;
793 }
794
795 if (bufferMaxIndex < 2) {
796 *count = 0;
797 return KERN_RESOURCE_SHORTAGE;
798 }
799
800 currPC = (uint64_t)state->pc; /* r15 */
801 if (state->cpsr & PSR_TF) {
802 currPC |= 1ULL; /* encode thumb mode into low bit of PC */
803 }
804 currLR = (uint64_t)state->lr; /* r14 */
805 currSP = (uint64_t)state->sp; /* r13 */
806
807 fp = (uint32_t *)(uintptr_t)state->r[7]; /* frame pointer */
808 topfp = fp;
809
810 bufferIndex = 0; // start with a stack of size zero
811 buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, kernel); // save PC in position 0.
812
813 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, kernel, 1);
814
815 // Now, fill buffer with stack backtraces.
816 while (bufferIndex < bufferMaxIndex) {
817 pc = 0ULL;
818 /*
819 * Below the frame pointer, the following values are saved:
820 * -> FP
821 */
822
823 /*
824 * Note that we read the pc even for the first stack frame
825 * (which, in theory, is always empty because the callee fills
826 * it in just before it lowers the stack. However, if we
827 * catch the program in between filling in the return address
828 * and lowering the stack, we want to still have a valid
829 * backtrace. FixupStack correctly disregards this value if
830 * necessary.
831 */
832
833 if ((uint32_t)fp == 0 || ((uint32_t)fp & 0x3) != 0) {
834 /* frame pointer is invalid - stop backtracing */
835 pc = 0ULL;
836 break;
837 }
838
839 if (kernel) {
840 if (((uint32_t)fp > kernStackMax) ||
841 ((uint32_t)fp < kernStackMin)) {
842 kr = KERN_FAILURE;
843 } else {
844 kr = chudxnu_kern_read(&frame,
845 (vm_offset_t)fp,
846 (vm_size_t)sizeof(frame));
847 if (kr == KERN_SUCCESS) {
848 pc = (uint64_t)frame[1];
849 nextFramePointer = (uint32_t *) (frame[0]);
850 } else {
851 pc = 0ULL;
852 nextFramePointer = 0ULL;
853 kr = KERN_FAILURE;
854 }
855 }
856 } else {
857 kr = chudxnu_task_read(task,
858 &frame32,
859 (((uint64_t)(uint32_t)fp) & 0x00000000FFFFFFFFULL),
860 sizeof(frame32));
861 if (kr == KERN_SUCCESS) {
862 pc = (uint64_t)frame32[1];
863 nextFramePointer = (uint32_t *)(uintptr_t)(frame32[0]);
864 } else {
865 pc = 0ULL;
866 nextFramePointer = 0ULL;
867 kr = KERN_FAILURE;
868 }
869 }
870
871 if (kr != KERN_SUCCESS) {
872 pc = 0ULL;
873 break;
874 }
875
876 if (nextFramePointer) {
877 buffer[bufferIndex++] = chudxnu_vm_unslide(pc, kernel);
878 prevPC = pc;
879 }
880
881 if (nextFramePointer < fp) {
882 break;
883 } else {
884 fp = nextFramePointer;
885 }
886 }
887
888 BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, bufferIndex);
889
890 /* clamp callstack size to max */
891 if (bufferIndex >= bufferMaxIndex) {
892 bufferIndex = bufferMaxIndex;
893 kr = KERN_RESOURCE_SHORTAGE;
894 } else {
895 /* ignore all other failures */
896 kr = KERN_SUCCESS;
897 }
898
899 // Save link register and R13 (sp) at bottom of stack (used for later fixup).
900 buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, kernel);
901 if (flags & CS_FLAG_EXTRASP) {
902 buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, kernel);
903 }
904 }
905
906 *count = bufferIndex;
907 return kr;
908 }
909
910 kern_return_t
chudxnu_thread_get_callstack64_kperf(thread_t thread,uint64_t * callStack,mach_msg_type_number_t * count,boolean_t user_only)911 chudxnu_thread_get_callstack64_kperf(
912 thread_t thread,
913 uint64_t *callStack,
914 mach_msg_type_number_t *count,
915 boolean_t user_only)
916 {
917 return chudxnu_thread_get_callstack64_internal( thread, callStack, count, user_only, 0 );
918 }
919 #elif __x86_64__
920
921 #define VALID_STACK_ADDRESS(supervisor, addr, minKernAddr, maxKernAddr) (supervisor ? (addr>=minKernAddr && addr<=maxKernAddr) : TRUE)
922 // don't try to read in the hole
923 #define VALID_STACK_ADDRESS64(supervisor, addr, minKernAddr, maxKernAddr) \
924 (supervisor ? ((uint64_t)addr >= minKernAddr && (uint64_t)addr <= maxKernAddr) : \
925 ((uint64_t)addr != 0ULL && ((uint64_t)addr <= 0x00007FFFFFFFFFFFULL || (uint64_t)addr >= 0xFFFF800000000000ULL)))
926
927 typedef struct _cframe64_t {
928 uint64_t prevFP; // can't use a real pointer here until we're a 64 bit kernel
929 uint64_t caller;
930 uint64_t args[0];
931 }cframe64_t;
932
933
934 typedef struct _cframe_t {
935 uint32_t prev; // this is really a user32-space pointer to the previous frame
936 uint32_t caller;
937 uint32_t args[0];
938 } cframe_t;
939
940 extern void * find_user_regs(thread_t);
941 extern x86_saved_state32_t *find_kern_regs(thread_t);
942
943 static kern_return_t
do_kernel_backtrace(thread_t thread,struct x86_kernel_state * regs,uint64_t * frames,mach_msg_type_number_t * start_idx,mach_msg_type_number_t max_idx)944 do_kernel_backtrace(
945 thread_t thread,
946 struct x86_kernel_state *regs,
947 uint64_t *frames,
948 mach_msg_type_number_t *start_idx,
949 mach_msg_type_number_t max_idx)
950 {
951 uint64_t kernStackMin = (uint64_t)thread->kernel_stack;
952 uint64_t kernStackMax = (uint64_t)kernStackMin + kernel_stack_size;
953 mach_msg_type_number_t ct = *start_idx;
954 kern_return_t kr = KERN_FAILURE;
955
956 #if __LP64__
957 uint64_t currPC = 0ULL;
958 uint64_t currFP = 0ULL;
959 uint64_t prevPC = 0ULL;
960 uint64_t prevFP = 0ULL;
961 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(regs->k_rip), sizeof(uint64_t))) {
962 return KERN_FAILURE;
963 }
964 if (KERN_SUCCESS != chudxnu_kern_read(&currFP, (vm_offset_t)&(regs->k_rbp), sizeof(uint64_t))) {
965 return KERN_FAILURE;
966 }
967 #else
968 uint32_t currPC = 0U;
969 uint32_t currFP = 0U;
970 uint32_t prevPC = 0U;
971 uint32_t prevFP = 0U;
972 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(regs->k_eip), sizeof(uint32_t))) {
973 return KERN_FAILURE;
974 }
975 if (KERN_SUCCESS != chudxnu_kern_read(&currFP, (vm_offset_t)&(regs->k_ebp), sizeof(uint32_t))) {
976 return KERN_FAILURE;
977 }
978 #endif
979
980 if (*start_idx >= max_idx) {
981 return KERN_RESOURCE_SHORTAGE; // no frames traced
982 }
983 if (!currPC) {
984 return KERN_FAILURE;
985 }
986
987 frames[ct++] = chudxnu_vm_unslide((uint64_t)currPC, 1);
988
989 // build a backtrace of this kernel state
990 #if __LP64__
991 while (VALID_STACK_ADDRESS64(TRUE, currFP, kernStackMin, kernStackMax)) {
992 // this is the address where caller lives in the user thread
993 uint64_t caller = currFP + sizeof(uint64_t);
994 #else
995 while (VALID_STACK_ADDRESS(TRUE, currFP, kernStackMin, kernStackMax)) {
996 uint32_t caller = (uint32_t)currFP + sizeof(uint32_t);
997 #endif
998
999 if (!currFP || !currPC) {
1000 currPC = 0;
1001 break;
1002 }
1003
1004 if (ct >= max_idx) {
1005 *start_idx = ct;
1006 return KERN_RESOURCE_SHORTAGE;
1007 }
1008
1009 /* read our caller */
1010 kr = chudxnu_kern_read(&currPC, (vm_offset_t)caller, sizeof(currPC));
1011
1012 if (kr != KERN_SUCCESS || !currPC) {
1013 currPC = 0UL;
1014 break;
1015 }
1016
1017 /*
1018 * retrive contents of the frame pointer and advance to the next stack
1019 * frame if it's valid
1020 */
1021 prevFP = 0;
1022 kr = chudxnu_kern_read(&prevFP, (vm_offset_t)currFP, sizeof(currPC));
1023
1024 #if __LP64__
1025 if (VALID_STACK_ADDRESS64(TRUE, prevFP, kernStackMin, kernStackMax)) {
1026 #else
1027 if (VALID_STACK_ADDRESS(TRUE, prevFP, kernStackMin, kernStackMax)) {
1028 #endif
1029 frames[ct++] = chudxnu_vm_unslide((uint64_t)currPC, 1);
1030 prevPC = currPC;
1031 }
1032 if (prevFP <= currFP) {
1033 break;
1034 } else {
1035 currFP = prevFP;
1036 }
1037 }
1038
1039 *start_idx = ct;
1040 return KERN_SUCCESS;
1041 }
1042
1043
1044
1045 static kern_return_t
1046 do_backtrace32(
1047 task_t task,
1048 thread_t thread,
1049 x86_saved_state32_t *regs,
1050 uint64_t *frames,
1051 mach_msg_type_number_t *start_idx,
1052 mach_msg_type_number_t max_idx,
1053 boolean_t supervisor)
1054 {
1055 uint32_t tmpWord = 0UL;
1056 uint64_t currPC = (uint64_t) regs->eip;
1057 uint64_t currFP = (uint64_t) regs->ebp;
1058 uint64_t prevPC = 0ULL;
1059 uint64_t prevFP = 0ULL;
1060 uint64_t kernStackMin = thread->kernel_stack;
1061 uint64_t kernStackMax = kernStackMin + kernel_stack_size;
1062 mach_msg_type_number_t ct = *start_idx;
1063 kern_return_t kr = KERN_FAILURE;
1064
1065 if (ct >= max_idx) {
1066 return KERN_RESOURCE_SHORTAGE; // no frames traced
1067 }
1068 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1069
1070 // build a backtrace of this 32 bit state.
1071 while (VALID_STACK_ADDRESS(supervisor, currFP, kernStackMin, kernStackMax)) {
1072 cframe_t *fp = (cframe_t *) (uintptr_t) currFP;
1073
1074 if (!currFP) {
1075 currPC = 0;
1076 break;
1077 }
1078
1079 if (ct >= max_idx) {
1080 *start_idx = ct;
1081 return KERN_RESOURCE_SHORTAGE;
1082 }
1083
1084 /* read our caller */
1085 if (supervisor) {
1086 kr = chudxnu_kern_read(&tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t));
1087 } else {
1088 kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t));
1089 }
1090
1091 if (kr != KERN_SUCCESS) {
1092 currPC = 0ULL;
1093 break;
1094 }
1095
1096 currPC = (uint64_t) tmpWord; // promote 32 bit address
1097
1098 /*
1099 * retrive contents of the frame pointer and advance to the next stack
1100 * frame if it's valid
1101 */
1102 prevFP = 0;
1103 if (supervisor) {
1104 kr = chudxnu_kern_read(&tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t));
1105 } else {
1106 kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t));
1107 }
1108 prevFP = (uint64_t) tmpWord; // promote 32 bit address
1109
1110 if (prevFP) {
1111 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1112 prevPC = currPC;
1113 }
1114 if (prevFP < currFP) {
1115 break;
1116 } else {
1117 currFP = prevFP;
1118 }
1119 }
1120
1121 *start_idx = ct;
1122 return KERN_SUCCESS;
1123 }
1124
1125 static kern_return_t
1126 do_backtrace64(
1127 task_t task,
1128 thread_t thread,
1129 x86_saved_state64_t *regs,
1130 uint64_t *frames,
1131 mach_msg_type_number_t *start_idx,
1132 mach_msg_type_number_t max_idx,
1133 boolean_t supervisor)
1134 {
1135 uint64_t currPC = regs->isf.rip;
1136 uint64_t currFP = regs->rbp;
1137 uint64_t prevPC = 0ULL;
1138 uint64_t prevFP = 0ULL;
1139 uint64_t kernStackMin = (uint64_t)thread->kernel_stack;
1140 uint64_t kernStackMax = (uint64_t)kernStackMin + kernel_stack_size;
1141 mach_msg_type_number_t ct = *start_idx;
1142 kern_return_t kr = KERN_FAILURE;
1143
1144 if (*start_idx >= max_idx) {
1145 return KERN_RESOURCE_SHORTAGE; // no frames traced
1146 }
1147 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1148
1149 // build a backtrace of this 32 bit state.
1150 while (VALID_STACK_ADDRESS64(supervisor, currFP, kernStackMin, kernStackMax)) {
1151 // this is the address where caller lives in the user thread
1152 uint64_t caller = currFP + sizeof(uint64_t);
1153
1154 if (!currFP) {
1155 currPC = 0;
1156 break;
1157 }
1158
1159 if (ct >= max_idx) {
1160 *start_idx = ct;
1161 return KERN_RESOURCE_SHORTAGE;
1162 }
1163
1164 /* read our caller */
1165 if (supervisor) {
1166 kr = chudxnu_kern_read(&currPC, (vm_offset_t)caller, sizeof(uint64_t));
1167 } else {
1168 kr = chudxnu_task_read(task, &currPC, caller, sizeof(uint64_t));
1169 }
1170
1171 if (kr != KERN_SUCCESS) {
1172 currPC = 0ULL;
1173 break;
1174 }
1175
1176 /*
1177 * retrive contents of the frame pointer and advance to the next stack
1178 * frame if it's valid
1179 */
1180 prevFP = 0;
1181 if (supervisor) {
1182 kr = chudxnu_kern_read(&prevFP, (vm_offset_t)currFP, sizeof(uint64_t));
1183 } else {
1184 kr = chudxnu_task_read(task, &prevFP, currFP, sizeof(uint64_t));
1185 }
1186
1187 if (VALID_STACK_ADDRESS64(supervisor, prevFP, kernStackMin, kernStackMax)) {
1188 frames[ct++] = chudxnu_vm_unslide(currPC, supervisor);
1189 prevPC = currPC;
1190 }
1191 if (prevFP < currFP) {
1192 break;
1193 } else {
1194 currFP = prevFP;
1195 }
1196 }
1197
1198 *start_idx = ct;
1199 return KERN_SUCCESS;
1200 }
1201
1202 static kern_return_t
1203 chudxnu_thread_get_callstack64_internal(
1204 thread_t thread,
1205 uint64_t *callstack,
1206 mach_msg_type_number_t *count,
1207 boolean_t user_only,
1208 boolean_t kern_only)
1209 {
1210 kern_return_t kr = KERN_FAILURE;
1211 task_t task = get_threadtask(thread);
1212 uint64_t currPC = 0ULL;
1213 boolean_t supervisor = FALSE;
1214 mach_msg_type_number_t bufferIndex = 0;
1215 mach_msg_type_number_t bufferMaxIndex = *count;
1216 x86_saved_state_t *tagged_regs = NULL; // kernel register state
1217 x86_saved_state64_t *regs64 = NULL;
1218 x86_saved_state32_t *regs32 = NULL;
1219 x86_saved_state32_t *u_regs32 = NULL;
1220 x86_saved_state64_t *u_regs64 = NULL;
1221 struct x86_kernel_state *kregs = NULL;
1222
1223 if (ml_at_interrupt_context()) {
1224 if (user_only) {
1225 /* can't backtrace user state on interrupt stack. */
1226 return KERN_FAILURE;
1227 }
1228
1229 /* backtracing at interrupt context? */
1230 if (thread == current_thread() && current_cpu_datap()->cpu_int_state) {
1231 /*
1232 * Locate the registers for the interrupted thread, assuming it is
1233 * current_thread().
1234 */
1235 tagged_regs = current_cpu_datap()->cpu_int_state;
1236
1237 if (is_saved_state64(tagged_regs)) {
1238 /* 64 bit registers */
1239 regs64 = saved_state64(tagged_regs);
1240 supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U);
1241 } else {
1242 /* 32 bit registers */
1243 regs32 = saved_state32(tagged_regs);
1244 supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U);
1245 }
1246 }
1247 }
1248
1249 if (!ml_at_interrupt_context() && kernel_task == task) {
1250 if (!thread->kernel_stack) {
1251 return KERN_FAILURE;
1252 }
1253
1254 // Kernel thread not at interrupt context
1255 kregs = (struct x86_kernel_state *)NULL;
1256
1257 // nofault read of the thread->kernel_stack pointer
1258 if (KERN_SUCCESS != chudxnu_kern_read(&kregs, (vm_offset_t)&(thread->kernel_stack), sizeof(void *))) {
1259 return KERN_FAILURE;
1260 }
1261
1262 // Adjust to find the saved kernel state
1263 kregs = STACK_IKS((vm_offset_t)(uintptr_t)kregs);
1264
1265 supervisor = TRUE;
1266 } else if (!tagged_regs) {
1267 /*
1268 * not at interrupt context, or tracing a different thread than
1269 * current_thread() at interrupt context
1270 */
1271 tagged_regs = USER_STATE(thread);
1272 if (is_saved_state64(tagged_regs)) {
1273 /* 64 bit registers */
1274 regs64 = saved_state64(tagged_regs);
1275 supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U);
1276 } else {
1277 /* 32 bit registers */
1278 regs32 = saved_state32(tagged_regs);
1279 supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U);
1280 }
1281 }
1282
1283 *count = 0;
1284
1285 if (supervisor) {
1286 // the caller only wants a user callstack.
1287 if (user_only) {
1288 // bail - we've only got kernel state
1289 return KERN_FAILURE;
1290 }
1291 } else {
1292 // regs32(64) is not in supervisor mode.
1293 u_regs32 = regs32;
1294 u_regs64 = regs64;
1295 regs32 = NULL;
1296 regs64 = NULL;
1297 }
1298
1299 if (user_only) {
1300 /* we only want to backtrace the user mode */
1301 if (!(u_regs32 || u_regs64)) {
1302 /* no user state to look at */
1303 return KERN_FAILURE;
1304 }
1305 }
1306
1307 /*
1308 * Order of preference for top of stack:
1309 * 64 bit kernel state (not likely)
1310 * 32 bit kernel state
1311 * 64 bit user land state
1312 * 32 bit user land state
1313 */
1314
1315 if (kregs) {
1316 /*
1317 * nofault read of the registers from the kernel stack (as they can
1318 * disappear on the fly).
1319 */
1320
1321 if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(kregs->k_rip), sizeof(uint64_t))) {
1322 return KERN_FAILURE;
1323 }
1324 } else if (regs64) {
1325 currPC = regs64->isf.rip;
1326 } else if (regs32) {
1327 currPC = (uint64_t) regs32->eip;
1328 } else if (u_regs64) {
1329 currPC = u_regs64->isf.rip;
1330 } else if (u_regs32) {
1331 currPC = (uint64_t) u_regs32->eip;
1332 }
1333
1334 if (!currPC) {
1335 /* no top of the stack, bail out */
1336 return KERN_FAILURE;
1337 }
1338
1339 bufferIndex = 0;
1340
1341 if (bufferMaxIndex < 1) {
1342 *count = 0;
1343 return KERN_RESOURCE_SHORTAGE;
1344 }
1345
1346 /* backtrace kernel */
1347 if (kregs) {
1348 addr64_t address = 0ULL;
1349 size_t size = 0UL;
1350
1351 // do the backtrace
1352 kr = do_kernel_backtrace(thread, kregs, callstack, &bufferIndex, bufferMaxIndex);
1353
1354 // and do a nofault read of (r|e)sp
1355 uint64_t rsp = 0ULL;
1356 size = sizeof(uint64_t);
1357
1358 if (KERN_SUCCESS != chudxnu_kern_read(&address, (vm_offset_t)&(kregs->k_rsp), size)) {
1359 address = 0ULL;
1360 }
1361
1362 if (address && KERN_SUCCESS == chudxnu_kern_read(&rsp, (vm_offset_t)address, size) && bufferIndex < bufferMaxIndex) {
1363 callstack[bufferIndex++] = (uint64_t)rsp;
1364 }
1365 } else if (regs64) {
1366 uint64_t rsp = 0ULL;
1367
1368 // backtrace the 64bit side.
1369 kr = do_backtrace64(task, thread, regs64, callstack, &bufferIndex,
1370 bufferMaxIndex - 1, TRUE);
1371
1372 if (KERN_SUCCESS == chudxnu_kern_read(&rsp, (vm_offset_t) regs64->isf.rsp, sizeof(uint64_t)) &&
1373 bufferIndex < bufferMaxIndex) {
1374 callstack[bufferIndex++] = rsp;
1375 }
1376 } else if (regs32) {
1377 uint32_t esp = 0UL;
1378
1379 // backtrace the 32bit side.
1380 kr = do_backtrace32(task, thread, regs32, callstack, &bufferIndex,
1381 bufferMaxIndex - 1, TRUE);
1382
1383 if (KERN_SUCCESS == chudxnu_kern_read(&esp, (vm_offset_t) regs32->uesp, sizeof(uint32_t)) &&
1384 bufferIndex < bufferMaxIndex) {
1385 callstack[bufferIndex++] = (uint64_t) esp;
1386 }
1387 } else if (u_regs64 && !kern_only) {
1388 /* backtrace user land */
1389 uint64_t rsp = 0ULL;
1390
1391 kr = do_backtrace64(task, thread, u_regs64, callstack, &bufferIndex,
1392 bufferMaxIndex - 1, FALSE);
1393
1394 if (KERN_SUCCESS == chudxnu_task_read(task, &rsp, (addr64_t) u_regs64->isf.rsp, sizeof(uint64_t)) &&
1395 bufferIndex < bufferMaxIndex) {
1396 callstack[bufferIndex++] = rsp;
1397 }
1398 } else if (u_regs32 && !kern_only) {
1399 uint32_t esp = 0UL;
1400
1401 kr = do_backtrace32(task, thread, u_regs32, callstack, &bufferIndex,
1402 bufferMaxIndex - 1, FALSE);
1403
1404 if (KERN_SUCCESS == chudxnu_task_read(task, &esp, (addr64_t) u_regs32->uesp, sizeof(uint32_t)) &&
1405 bufferIndex < bufferMaxIndex) {
1406 callstack[bufferIndex++] = (uint64_t) esp;
1407 }
1408 }
1409
1410 *count = bufferIndex;
1411 return kr;
1412 }
1413
1414 __private_extern__
1415 kern_return_t
1416 chudxnu_thread_get_callstack64_kperf(
1417 thread_t thread,
1418 uint64_t *callstack,
1419 mach_msg_type_number_t *count,
1420 boolean_t is_user)
1421 {
1422 return chudxnu_thread_get_callstack64_internal(thread, callstack, count, is_user, !is_user);
1423 }
1424 #else /* !__arm64__ && !__x86_64__ */
1425 #error kperf: unsupported architecture
1426 #endif /* !__arm64__ && !__x86_64__ */
1427