/* * Copyright (c) 2011-2022 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* Collect kernel callstacks */ #include #include #include #include #include #include #include #include #include #include #include #if defined(__arm64__) #include #include #endif static void callstack_fixup_user(struct kp_ucallstack *cs, thread_t thread) { uint64_t fixup_val = 0; assert(cs->kpuc_nframes < MAX_UCALLSTACK_FRAMES); #if defined(__x86_64__) user_addr_t sp_user; bool user_64; x86_saved_state_t *state; state = get_user_regs(thread); if (!state) { goto out; } user_64 = is_saved_state64(state); if (user_64) { sp_user = saved_state64(state)->isf.rsp; } else { sp_user = saved_state32(state)->uesp; } if (thread == current_thread()) { (void)copyin(sp_user, (char *)&fixup_val, user_64 ? sizeof(uint64_t) : sizeof(uint32_t)); } else { (void)vm_map_read_user(get_task_map(get_threadtask(thread)), sp_user, &fixup_val, user_64 ? sizeof(uint64_t) : sizeof(uint32_t)); } #elif defined(__arm64__) struct arm_saved_state *state = get_user_regs(thread); if (!state) { goto out; } /* encode thumb mode into low bit of PC */ if (is_saved_state32(state) && (get_saved_state_cpsr(state) & PSR_TF)) { cs->kpuc_frames[0] |= 1ULL; } fixup_val = get_saved_state_lr(state); #else #error "callstack_fixup_user: unsupported architecture" #endif out: cs->kpuc_frames[cs->kpuc_nframes++] = fixup_val; } #if defined(__x86_64__) __attribute__((used)) static kern_return_t interrupted_kernel_sp_value(uintptr_t *sp_val) { x86_saved_state_t *state; uintptr_t sp; bool state_64; uint64_t cs; uintptr_t top, bottom; state = current_cpu_datap()->cpu_int_state; if (!state) { return KERN_FAILURE; } state_64 = is_saved_state64(state); if (state_64) { cs = saved_state64(state)->isf.cs; } else { cs = saved_state32(state)->cs; } /* return early if interrupted a thread in user space */ if ((cs & SEL_PL) == SEL_PL_U) { return KERN_FAILURE; } if (state_64) { sp = saved_state64(state)->isf.rsp; } else { sp = saved_state32(state)->uesp; } /* make sure the stack pointer is pointing somewhere in this stack */ bottom = current_thread()->kernel_stack; top = bottom + kernel_stack_size; if (sp >= bottom && sp < top) { return KERN_FAILURE; } *sp_val = *(uintptr_t *)sp; return KERN_SUCCESS; } #elif defined(__arm64__) __attribute__((used)) static kern_return_t interrupted_kernel_lr(uintptr_t *lr) { struct arm_saved_state *state; state = getCpuDatap()->cpu_int_state; /* return early if interrupted a thread in user space */ if (PSR64_IS_USER(get_saved_state_cpsr(state))) { return KERN_FAILURE; } *lr = get_saved_state_lr(state); return KERN_SUCCESS; } #else /* defined(__arm64__) */ #error "interrupted_kernel_{sp,lr}: unsupported architecture" #endif /* !defined(__arm64__) */ static void callstack_fixup_interrupted(struct kp_kcallstack *cs) { uintptr_t fixup_val = 0; assert(cs->kpkc_nframes < MAX_KCALLSTACK_FRAMES); /* * Only provide arbitrary data on development or debug kernels. */ #if DEVELOPMENT || DEBUG #if defined(__x86_64__) (void)interrupted_kernel_sp_value(&fixup_val); #elif defined(__arm64__) (void)interrupted_kernel_lr(&fixup_val); #endif /* defined(__x86_64__) */ #endif /* DEVELOPMENT || DEBUG */ assert(cs->kpkc_flags & CALLSTACK_KERNEL); cs->kpkc_frames[cs->kpkc_nframes++] = fixup_val; } void kperf_continuation_sample(struct kp_kcallstack *cs, struct kperf_context *context) { thread_t thread; assert(cs != NULL); assert(context != NULL); thread = context->cur_thread; assert(thread != NULL); assert(thread->continuation != NULL); cs->kpkc_flags = CALLSTACK_CONTINUATION | CALLSTACK_VALID | CALLSTACK_KERNEL; #ifdef __LP64__ cs->kpkc_flags |= CALLSTACK_64BIT; #endif cs->kpkc_nframes = 1; cs->kpkc_frames[0] = VM_KERNEL_UNSLIDE(thread->continuation); } void kperf_backtrace_sample(struct kp_kcallstack *cs, struct kperf_context *context) { assert(cs != NULL); assert(context != NULL); assert(context->cur_thread == current_thread()); cs->kpkc_flags = CALLSTACK_KERNEL | CALLSTACK_KERNEL_WORDS; #ifdef __LP64__ cs->kpkc_flags |= CALLSTACK_64BIT; #endif BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, 1); backtrace_info_t btinfo = BTI_NONE; struct backtrace_control ctl = { .btc_frame_addr = (uintptr_t)context->starting_fp, }; cs->kpkc_nframes = backtrace(cs->kpkc_word_frames, cs->kpkc_nframes - 1, &ctl, &btinfo); if (cs->kpkc_nframes > 0) { cs->kpkc_flags |= CALLSTACK_VALID; cs->kpkc_exclaves_offset = 0; #if CONFIG_EXCLAVES if ((context->cur_thread->th_exclaves_state & TH_EXCLAVES_RPC) != 0) { cs->kpkc_exclaves_offset = exclaves_stack_offset(cs->kpkc_word_frames, cs->kpkc_nframes, true); } #endif /* CONFIG_EXCLAVES */ /* * Fake the value pointed to by the stack pointer or the link * register for symbolicators. */ cs->kpkc_word_frames[cs->kpkc_nframes + 1] = 0; cs->kpkc_nframes += 1; } if ((btinfo & BTI_TRUNCATED)) { cs->kpkc_flags |= CALLSTACK_TRUNCATED; } BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, cs->kpkc_nframes); } kern_return_t chudxnu_thread_get_callstack64_kperf(thread_t thread, uint64_t *callStack, mach_msg_type_number_t *count, boolean_t user_only); void kperf_kcallstack_sample(struct kp_kcallstack *cs, struct kperf_context *context) { thread_t thread; assert(cs != NULL); assert(context != NULL); assert(cs->kpkc_nframes <= MAX_KCALLSTACK_FRAMES); thread = context->cur_thread; assert(thread != NULL); BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread), cs->kpkc_nframes); cs->kpkc_flags = CALLSTACK_KERNEL; #ifdef __LP64__ cs->kpkc_flags |= CALLSTACK_64BIT; #endif if (ml_at_interrupt_context()) { assert(thread == current_thread()); cs->kpkc_flags |= CALLSTACK_KERNEL_WORDS; backtrace_info_t btinfo = BTI_NONE; struct backtrace_control ctl = { .btc_flags = BTF_KERN_INTERRUPTED, }; cs->kpkc_nframes = backtrace(cs->kpkc_word_frames, cs->kpkc_nframes - 1, &ctl, &btinfo); if (cs->kpkc_nframes != 0) { callstack_fixup_interrupted(cs); } if ((btinfo & BTI_TRUNCATED)) { cs->kpkc_flags |= CALLSTACK_TRUNCATED; } cs->kpkc_exclaves_offset = 0; #if CONFIG_EXCLAVES if ((thread->th_exclaves_state & TH_EXCLAVES_RPC) != 0) { cs->kpkc_exclaves_offset = exclaves_stack_offset(cs->kpkc_word_frames, cs->kpkc_nframes, true); } #endif /* CONFIG_EXCLAVES */ } else { /* * Rely on legacy CHUD backtracer to backtrace kernel stacks on * other threads. */ kern_return_t kr; kr = chudxnu_thread_get_callstack64_kperf(thread, cs->kpkc_frames, &cs->kpkc_nframes, FALSE); if (kr == KERN_SUCCESS) { cs->kpkc_flags |= CALLSTACK_VALID; } else if (kr == KERN_RESOURCE_SHORTAGE) { cs->kpkc_flags |= CALLSTACK_VALID; cs->kpkc_flags |= CALLSTACK_TRUNCATED; } else { cs->kpkc_nframes = 0; } } if (!(cs->kpkc_flags & CALLSTACK_VALID)) { BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK); } BUF_INFO(PERF_CS_KSAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread), cs->kpkc_flags, cs->kpkc_nframes); } void kperf_ucallstack_sample(struct kp_ucallstack *cs, struct kperf_context *context) { assert(ml_get_interrupts_enabled() == TRUE); thread_t thread = context->cur_thread; assert(thread != NULL); BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_START, (uintptr_t)thread_tid(thread), cs->kpuc_nframes); struct backtrace_user_info btinfo = BTUINFO_INIT; /* * Leave space for the fixup information. */ unsigned int maxnframes = cs->kpuc_nframes - 1; struct backtrace_control ctl = { .btc_user_thread = thread, }; unsigned int nframes = backtrace_user(cs->kpuc_frames, maxnframes, &ctl, &btinfo); cs->kpuc_nframes = MIN(maxnframes, nframes); cs->kpuc_flags |= CALLSTACK_KERNEL_WORDS | ((btinfo.btui_info & BTI_TRUNCATED) ? CALLSTACK_TRUNCATED : 0) | ((btinfo.btui_info & BTI_64_BIT) ? CALLSTACK_64BIT : 0); /* * Ignore EFAULT to get as much of the stack as possible. */ if (btinfo.btui_error == 0 || btinfo.btui_error == EFAULT) { callstack_fixup_user(cs, thread); cs->kpuc_flags |= CALLSTACK_VALID; if (cs->kpuc_nframes < maxnframes && btinfo.btui_async_frame_addr != 0) { cs->kpuc_async_index = btinfo.btui_async_start_index; ctl.btc_frame_addr = btinfo.btui_async_frame_addr; ctl.btc_addr_offset = BTCTL_ASYNC_ADDR_OFFSET; maxnframes -= cs->kpuc_nframes; btinfo = BTUINFO_INIT; unsigned int nasync_frames = backtrace_user( &cs->kpuc_frames[cs->kpuc_nframes], maxnframes, &ctl, &btinfo); if (btinfo.btui_info & BTI_TRUNCATED) { cs->kpuc_flags |= CALLSTACK_TRUNCATED; } if (btinfo.btui_error == 0 || btinfo.btui_error == EFAULT) { cs->kpuc_flags |= CALLSTACK_HAS_ASYNC; cs->kpuc_async_nframes = nasync_frames; } } } else { cs->kpuc_nframes = 0; BUF_INFO(PERF_CS_ERROR, ERR_GETSTACK, btinfo.btui_error); } BUF_INFO(PERF_CS_USAMPLE | DBG_FUNC_END, (uintptr_t)thread_tid(thread), cs->kpuc_flags, cs->kpuc_nframes); } static inline uintptr_t scrub_word(uintptr_t *bt, int n_frames, int frame, bool kern) { if (frame < n_frames) { if (kern) { return VM_KERNEL_UNSLIDE(bt[frame]); } else { return bt[frame]; } } else { return 0; } } static inline uintptr_t scrub_frame(uint64_t *bt, int n_frames, int frame) { if (frame < n_frames) { return (uintptr_t)(bt[frame]); } else { return 0; } } static void callstack_log(uint32_t hdrid, uint32_t dataid, void *vframes, unsigned int nframes, unsigned int flags, unsigned int async_index, unsigned int async_nframes) { BUF_VERB(PERF_CS_LOG | DBG_FUNC_START, flags, nframes); BUF_DATA(hdrid, flags, nframes - async_nframes, async_index, async_nframes); unsigned int nevts = nframes / 4; unsigned int ovf = nframes % 4; if (ovf != 0) { nevts++; } bool kern = flags & CALLSTACK_KERNEL; if (flags & CALLSTACK_KERNEL_WORDS) { uintptr_t *frames = vframes; for (unsigned int i = 0; i < nevts; i++) { unsigned int j = i * 4; BUF_DATA(dataid, scrub_word(frames, nframes, j + 0, kern), scrub_word(frames, nframes, j + 1, kern), scrub_word(frames, nframes, j + 2, kern), scrub_word(frames, nframes, j + 3, kern)); } } else { for (unsigned int i = 0; i < nevts; i++) { uint64_t *frames = vframes; unsigned int j = i * 4; BUF_DATA(dataid, scrub_frame(frames, nframes, j + 0), scrub_frame(frames, nframes, j + 1), scrub_frame(frames, nframes, j + 2), scrub_frame(frames, nframes, j + 3)); } } BUF_VERB(PERF_CS_LOG | DBG_FUNC_END, flags, nframes); } void kperf_kcallstack_log(struct kp_kcallstack *cs) { callstack_log(PERF_CS_KHDR, PERF_CS_KDATA, cs->kpkc_frames, cs->kpkc_nframes, cs->kpkc_flags, 0, 0); if (cs->kpkc_exclaves_offset != 0) { BUF_DATA(PERF_CS_KEXOFFSET, cs->kpkc_exclaves_offset); } } void kperf_ucallstack_log(struct kp_ucallstack *cs) { callstack_log(PERF_CS_UHDR, PERF_CS_UDATA, cs->kpuc_frames, cs->kpuc_nframes + cs->kpuc_async_nframes, cs->kpuc_flags, cs->kpuc_async_index, cs->kpuc_async_nframes); } #if CONFIG_EXCLAVES void kperf_excallstack_log(const stackshottypes_ipcstackentry_s *ipcstack) { __block unsigned int nframes = 0; __block unsigned int flags = CALLSTACK_VALID; uint64_t frames[MAX_EXCALLSTACK_FRAMES] = {}; uint64_t *frames_block = frames; BUF_DATA(PERF_CS_EXSTACK, ipcstack->asid); if (ipcstack->stacktrace.has_value) { address__v_visit(&ipcstack->stacktrace.value, ^(size_t i, const stackshottypes_address_s item) { if (i >= MAX_EXCALLSTACK_FRAMES) { flags |= CALLSTACK_TRUNCATED; return; } frames_block[i] = item; nframes += 1; }); callstack_log(PERF_CS_EXHDR, PERF_CS_EXDATA, frames, nframes, flags, 0, 0); } } bool kperf_exclave_callstack_pend(struct kperf_context *context, unsigned int actionid) { if ((context->cur_thread->th_exclaves_state & TH_EXCLAVES_RPC) && (os_atomic_load(&context->cur_thread->th_exclaves_inspection_state, relaxed) & TH_EXCLAVES_INSPECTION_NOINSPECT) == 0) { os_atomic_or(&context->cur_thread->th_exclaves_inspection_state, TH_EXCLAVES_INSPECTION_KPERF, relaxed); context->cur_thread->kperf_exclaves_ast |= T_KPERF_SET_ACTIONID(actionid); return true; } return false; } #endif /* CONFIG_EXCLAVES */ int kperf_ucallstack_pend(struct kperf_context * context, uint32_t depth, unsigned int actionid) { if (depth < 2) { panic("HUH"); } kperf_ast_set_callstack_depth(context->cur_thread, depth); return kperf_ast_pend(context->cur_thread, T_KPERF_AST_CALLSTACK, actionid); } static kern_return_t chudxnu_kern_read(void *dstaddr, vm_offset_t srcaddr, vm_size_t size) { return (ml_nofault_copy(srcaddr, (vm_offset_t)dstaddr, size) == size) ? KERN_SUCCESS : KERN_FAILURE; } static kern_return_t chudxnu_task_read( task_t task, void *kernaddr, uint64_t usraddr, vm_size_t size) { //ppc version ported to arm kern_return_t ret = KERN_SUCCESS; if (ml_at_interrupt_context()) { return KERN_FAILURE; // can't look at tasks on interrupt stack } if (current_task() == task) { if (copyin(usraddr, kernaddr, size)) { ret = KERN_FAILURE; } } else { vm_map_t map = get_task_map(task); ret = vm_map_read_user(map, usraddr, kernaddr, size); } return ret; } static inline uint64_t chudxnu_vm_unslide( uint64_t ptr, int kaddr ) { if (!kaddr) { return ptr; } return VM_KERNEL_UNSLIDE(ptr); } #if __arm64__ #if defined(HAS_APPLE_PAC) #include #endif // chudxnu_thread_get_callstack gathers a raw callstack along with any information needed to // fix it up later (in case we stopped program as it was saving values into prev stack frame, etc.) // after sampling has finished. // // For an N-entry callstack: // // [0] current pc // [1..N-3] stack frames (including current one) // [N-2] current LR (return value if we're in a leaf function) // [N-1] current r0 (in case we've saved LR in r0) (optional) // // #define CS_FLAG_EXTRASP 1 // capture extra sp register static kern_return_t chudxnu_thread_get_callstack64_internal( thread_t thread, uint64_t *callStack, mach_msg_type_number_t *count, boolean_t user_only, int flags) { kern_return_t kr = KERN_SUCCESS; task_t task; uint64_t currPC = 0ULL, currLR = 0ULL, currSP = 0ULL; uint64_t prevPC = 0ULL; uint64_t kernStackMin = thread->kernel_stack; uint64_t kernStackMax = kernStackMin + kernel_stack_size; uint64_t *buffer = callStack; int bufferIndex = 0; int bufferMaxIndex = 0; boolean_t kernel = FALSE; struct arm_saved_state *sstate = NULL; uint64_t pc = 0ULL; task = get_threadtask(thread); bufferMaxIndex = *count; //get thread state if (user_only) { sstate = find_user_regs(thread); } else { sstate = find_kern_regs(thread); } if (!sstate) { *count = 0; return KERN_FAILURE; } if (is_saved_state64(sstate)) { struct arm_saved_state64 *state = NULL; uint64_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL; uint64_t frame[2]; state = saved_state64(sstate); /* make sure it is safe to dereference before you do it */ kernel = PSR64_IS_KERNEL(state->cpsr); /* can't take a kernel callstack if we've got a user frame */ if (!user_only && !kernel) { return KERN_FAILURE; } /* * Reserve space for saving LR (and sometimes SP) at the end of the * backtrace. */ if (flags & CS_FLAG_EXTRASP) { bufferMaxIndex -= 2; } else { bufferMaxIndex -= 1; } if (bufferMaxIndex < 2) { *count = 0; return KERN_RESOURCE_SHORTAGE; } currPC = state->pc; currLR = state->lr; currSP = state->sp; fp = (uint64_t *)state->fp; /* frame pointer */ #if defined(HAS_APPLE_PAC) /* frame pointers on stack will be signed by arm64e ABI */ fp = ptrauth_strip(fp, ptrauth_key_frame_pointer); #endif topfp = fp; bufferIndex = 0; // start with a stack of size zero buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, kernel); // save PC in position 0. BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, kernel, 0); // Now, fill buffer with stack backtraces. while (bufferIndex < bufferMaxIndex) { pc = 0ULL; /* * Below the frame pointer, the following values are saved: * -> FP */ /* * Note that we read the pc even for the first stack frame * (which, in theory, is always empty because the callee fills * it in just before it lowers the stack. However, if we * catch the program in between filling in the return address * and lowering the stack, we want to still have a valid * backtrace. FixupStack correctly disregards this value if * necessary. */ if ((uint64_t)fp == 0 || ((uint64_t)fp & 0x3) != 0) { /* frame pointer is invalid - stop backtracing */ pc = 0ULL; break; } if (kernel) { if (((uint64_t)fp > kernStackMax) || ((uint64_t)fp < kernStackMin)) { kr = KERN_FAILURE; } else { kr = chudxnu_kern_read(&frame, (vm_offset_t)fp, (vm_size_t)sizeof(frame)); if (kr == KERN_SUCCESS) { #if defined(HAS_APPLE_PAC) /* return addresses on stack will be signed by arm64e ABI */ pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address); #else pc = frame[1]; #endif nextFramePointer = (uint64_t *)frame[0]; #if defined(HAS_APPLE_PAC) /* frame pointers on stack will be signed by arm64e ABI */ nextFramePointer = ptrauth_strip(nextFramePointer, ptrauth_key_frame_pointer); #endif } else { pc = 0ULL; nextFramePointer = 0ULL; kr = KERN_FAILURE; } } } else { kr = chudxnu_task_read(task, &frame, (vm_offset_t)fp, (vm_size_t)sizeof(frame)); if (kr == KERN_SUCCESS) { #if defined(HAS_APPLE_PAC) /* return addresses on stack will be signed by arm64e ABI */ pc = (uint64_t)ptrauth_strip((void *)frame[1], ptrauth_key_return_address); #else pc = frame[1]; #endif nextFramePointer = (uint64_t *)(frame[0]); #if defined(HAS_APPLE_PAC) /* frame pointers on stack will be signed by arm64e ABI */ nextFramePointer = ptrauth_strip(nextFramePointer, ptrauth_key_frame_pointer); #endif } else { pc = 0ULL; nextFramePointer = 0ULL; kr = KERN_FAILURE; } } if (kr != KERN_SUCCESS) { pc = 0ULL; break; } if (nextFramePointer) { buffer[bufferIndex++] = chudxnu_vm_unslide(pc, kernel); prevPC = pc; } if (nextFramePointer < fp) { break; } else { fp = nextFramePointer; } } BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, bufferIndex); if (bufferIndex >= bufferMaxIndex) { bufferIndex = bufferMaxIndex; kr = KERN_RESOURCE_SHORTAGE; } else { kr = KERN_SUCCESS; } // Save link register and SP at bottom of stack (used for later fixup). buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, kernel); if (flags & CS_FLAG_EXTRASP) { buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, kernel); } } else { struct arm_saved_state32 *state = NULL; uint32_t *fp = NULL, *nextFramePointer = NULL, *topfp = NULL; /* 64-bit kernel stacks, 32-bit user stacks */ uint64_t frame[2]; uint32_t frame32[2]; state = saved_state32(sstate); /* make sure it is safe to dereference before you do it */ kernel = PSR_IS_KERNEL(state->cpsr); /* can't take a kernel callstack if we've got a user frame */ if (!user_only && !kernel) { return KERN_FAILURE; } /* * Reserve space for saving LR (and sometimes SP) at the end of the * backtrace. */ if (flags & CS_FLAG_EXTRASP) { bufferMaxIndex -= 2; } else { bufferMaxIndex -= 1; } if (bufferMaxIndex < 2) { *count = 0; return KERN_RESOURCE_SHORTAGE; } currPC = (uint64_t)state->pc; /* r15 */ if (state->cpsr & PSR_TF) { currPC |= 1ULL; /* encode thumb mode into low bit of PC */ } currLR = (uint64_t)state->lr; /* r14 */ currSP = (uint64_t)state->sp; /* r13 */ fp = (uint32_t *)(uintptr_t)state->r[7]; /* frame pointer */ topfp = fp; bufferIndex = 0; // start with a stack of size zero buffer[bufferIndex++] = chudxnu_vm_unslide(currPC, kernel); // save PC in position 0. BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_START, kernel, 1); // Now, fill buffer with stack backtraces. while (bufferIndex < bufferMaxIndex) { pc = 0ULL; /* * Below the frame pointer, the following values are saved: * -> FP */ /* * Note that we read the pc even for the first stack frame * (which, in theory, is always empty because the callee fills * it in just before it lowers the stack. However, if we * catch the program in between filling in the return address * and lowering the stack, we want to still have a valid * backtrace. FixupStack correctly disregards this value if * necessary. */ if ((uint32_t)fp == 0 || ((uint32_t)fp & 0x3) != 0) { /* frame pointer is invalid - stop backtracing */ pc = 0ULL; break; } if (kernel) { if (((uint32_t)fp > kernStackMax) || ((uint32_t)fp < kernStackMin)) { kr = KERN_FAILURE; } else { kr = chudxnu_kern_read(&frame, (vm_offset_t)fp, (vm_size_t)sizeof(frame)); if (kr == KERN_SUCCESS) { pc = (uint64_t)frame[1]; nextFramePointer = (uint32_t *) (frame[0]); } else { pc = 0ULL; nextFramePointer = 0ULL; kr = KERN_FAILURE; } } } else { kr = chudxnu_task_read(task, &frame32, (((uint64_t)(uint32_t)fp) & 0x00000000FFFFFFFFULL), sizeof(frame32)); if (kr == KERN_SUCCESS) { pc = (uint64_t)frame32[1]; nextFramePointer = (uint32_t *)(uintptr_t)(frame32[0]); } else { pc = 0ULL; nextFramePointer = 0ULL; kr = KERN_FAILURE; } } if (kr != KERN_SUCCESS) { pc = 0ULL; break; } if (nextFramePointer) { buffer[bufferIndex++] = chudxnu_vm_unslide(pc, kernel); prevPC = pc; } if (nextFramePointer < fp) { break; } else { fp = nextFramePointer; } } BUF_VERB(PERF_CS_BACKTRACE | DBG_FUNC_END, bufferIndex); /* clamp callstack size to max */ if (bufferIndex >= bufferMaxIndex) { bufferIndex = bufferMaxIndex; kr = KERN_RESOURCE_SHORTAGE; } else { /* ignore all other failures */ kr = KERN_SUCCESS; } // Save link register and R13 (sp) at bottom of stack (used for later fixup). buffer[bufferIndex++] = chudxnu_vm_unslide(currLR, kernel); if (flags & CS_FLAG_EXTRASP) { buffer[bufferIndex++] = chudxnu_vm_unslide(currSP, kernel); } } *count = bufferIndex; return kr; } kern_return_t chudxnu_thread_get_callstack64_kperf( thread_t thread, uint64_t *callStack, mach_msg_type_number_t *count, boolean_t user_only) { return chudxnu_thread_get_callstack64_internal( thread, callStack, count, user_only, 0 ); } #elif __x86_64__ #define VALID_STACK_ADDRESS(supervisor, addr, minKernAddr, maxKernAddr) (supervisor ? (addr>=minKernAddr && addr<=maxKernAddr) : TRUE) // don't try to read in the hole #define VALID_STACK_ADDRESS64(supervisor, addr, minKernAddr, maxKernAddr) \ (supervisor ? ((uint64_t)addr >= minKernAddr && (uint64_t)addr <= maxKernAddr) : \ ((uint64_t)addr != 0ULL && ((uint64_t)addr <= 0x00007FFFFFFFFFFFULL || (uint64_t)addr >= 0xFFFF800000000000ULL))) typedef struct _cframe64_t { uint64_t prevFP; // can't use a real pointer here until we're a 64 bit kernel uint64_t caller; uint64_t args[0]; }cframe64_t; typedef struct _cframe_t { uint32_t prev; // this is really a user32-space pointer to the previous frame uint32_t caller; uint32_t args[0]; } cframe_t; extern void * find_user_regs(thread_t); extern x86_saved_state32_t *find_kern_regs(thread_t); static kern_return_t do_kernel_backtrace( thread_t thread, struct x86_kernel_state *regs, uint64_t *frames, mach_msg_type_number_t *start_idx, mach_msg_type_number_t max_idx) { uint64_t kernStackMin = (uint64_t)thread->kernel_stack; uint64_t kernStackMax = (uint64_t)kernStackMin + kernel_stack_size; mach_msg_type_number_t ct = *start_idx; kern_return_t kr = KERN_FAILURE; #if __LP64__ uint64_t currPC = 0ULL; uint64_t currFP = 0ULL; uint64_t prevPC = 0ULL; uint64_t prevFP = 0ULL; if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(regs->k_rip), sizeof(uint64_t))) { return KERN_FAILURE; } if (KERN_SUCCESS != chudxnu_kern_read(&currFP, (vm_offset_t)&(regs->k_rbp), sizeof(uint64_t))) { return KERN_FAILURE; } #else uint32_t currPC = 0U; uint32_t currFP = 0U; uint32_t prevPC = 0U; uint32_t prevFP = 0U; if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(regs->k_eip), sizeof(uint32_t))) { return KERN_FAILURE; } if (KERN_SUCCESS != chudxnu_kern_read(&currFP, (vm_offset_t)&(regs->k_ebp), sizeof(uint32_t))) { return KERN_FAILURE; } #endif if (*start_idx >= max_idx) { return KERN_RESOURCE_SHORTAGE; // no frames traced } if (!currPC) { return KERN_FAILURE; } frames[ct++] = chudxnu_vm_unslide((uint64_t)currPC, 1); // build a backtrace of this kernel state #if __LP64__ while (VALID_STACK_ADDRESS64(TRUE, currFP, kernStackMin, kernStackMax)) { // this is the address where caller lives in the user thread uint64_t caller = currFP + sizeof(uint64_t); #else while (VALID_STACK_ADDRESS(TRUE, currFP, kernStackMin, kernStackMax)) { uint32_t caller = (uint32_t)currFP + sizeof(uint32_t); #endif if (!currFP || !currPC) { currPC = 0; break; } if (ct >= max_idx) { *start_idx = ct; return KERN_RESOURCE_SHORTAGE; } /* read our caller */ kr = chudxnu_kern_read(&currPC, (vm_offset_t)caller, sizeof(currPC)); if (kr != KERN_SUCCESS || !currPC) { currPC = 0UL; break; } /* * retrive contents of the frame pointer and advance to the next stack * frame if it's valid */ prevFP = 0; kr = chudxnu_kern_read(&prevFP, (vm_offset_t)currFP, sizeof(currPC)); #if __LP64__ if (VALID_STACK_ADDRESS64(TRUE, prevFP, kernStackMin, kernStackMax)) { #else if (VALID_STACK_ADDRESS(TRUE, prevFP, kernStackMin, kernStackMax)) { #endif frames[ct++] = chudxnu_vm_unslide((uint64_t)currPC, 1); prevPC = currPC; } if (prevFP <= currFP) { break; } else { currFP = prevFP; } } *start_idx = ct; return KERN_SUCCESS; } static kern_return_t do_backtrace32( task_t task, thread_t thread, x86_saved_state32_t *regs, uint64_t *frames, mach_msg_type_number_t *start_idx, mach_msg_type_number_t max_idx, boolean_t supervisor) { uint32_t tmpWord = 0UL; uint64_t currPC = (uint64_t) regs->eip; uint64_t currFP = (uint64_t) regs->ebp; uint64_t prevPC = 0ULL; uint64_t prevFP = 0ULL; uint64_t kernStackMin = thread->kernel_stack; uint64_t kernStackMax = kernStackMin + kernel_stack_size; mach_msg_type_number_t ct = *start_idx; kern_return_t kr = KERN_FAILURE; if (ct >= max_idx) { return KERN_RESOURCE_SHORTAGE; // no frames traced } frames[ct++] = chudxnu_vm_unslide(currPC, supervisor); // build a backtrace of this 32 bit state. while (VALID_STACK_ADDRESS(supervisor, currFP, kernStackMin, kernStackMax)) { cframe_t *fp = (cframe_t *) (uintptr_t) currFP; if (!currFP) { currPC = 0; break; } if (ct >= max_idx) { *start_idx = ct; return KERN_RESOURCE_SHORTAGE; } /* read our caller */ if (supervisor) { kr = chudxnu_kern_read(&tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t)); } else { kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t)); } if (kr != KERN_SUCCESS) { currPC = 0ULL; break; } currPC = (uint64_t) tmpWord; // promote 32 bit address /* * retrive contents of the frame pointer and advance to the next stack * frame if it's valid */ prevFP = 0; if (supervisor) { kr = chudxnu_kern_read(&tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t)); } else { kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t)); } prevFP = (uint64_t) tmpWord; // promote 32 bit address if (prevFP) { frames[ct++] = chudxnu_vm_unslide(currPC, supervisor); prevPC = currPC; } if (prevFP < currFP) { break; } else { currFP = prevFP; } } *start_idx = ct; return KERN_SUCCESS; } static kern_return_t do_backtrace64( task_t task, thread_t thread, x86_saved_state64_t *regs, uint64_t *frames, mach_msg_type_number_t *start_idx, mach_msg_type_number_t max_idx, boolean_t supervisor) { uint64_t currPC = regs->isf.rip; uint64_t currFP = regs->rbp; uint64_t prevPC = 0ULL; uint64_t prevFP = 0ULL; uint64_t kernStackMin = (uint64_t)thread->kernel_stack; uint64_t kernStackMax = (uint64_t)kernStackMin + kernel_stack_size; mach_msg_type_number_t ct = *start_idx; kern_return_t kr = KERN_FAILURE; if (*start_idx >= max_idx) { return KERN_RESOURCE_SHORTAGE; // no frames traced } frames[ct++] = chudxnu_vm_unslide(currPC, supervisor); // build a backtrace of this 32 bit state. while (VALID_STACK_ADDRESS64(supervisor, currFP, kernStackMin, kernStackMax)) { // this is the address where caller lives in the user thread uint64_t caller = currFP + sizeof(uint64_t); if (!currFP) { currPC = 0; break; } if (ct >= max_idx) { *start_idx = ct; return KERN_RESOURCE_SHORTAGE; } /* read our caller */ if (supervisor) { kr = chudxnu_kern_read(&currPC, (vm_offset_t)caller, sizeof(uint64_t)); } else { kr = chudxnu_task_read(task, &currPC, caller, sizeof(uint64_t)); } if (kr != KERN_SUCCESS) { currPC = 0ULL; break; } /* * retrive contents of the frame pointer and advance to the next stack * frame if it's valid */ prevFP = 0; if (supervisor) { kr = chudxnu_kern_read(&prevFP, (vm_offset_t)currFP, sizeof(uint64_t)); } else { kr = chudxnu_task_read(task, &prevFP, currFP, sizeof(uint64_t)); } if (VALID_STACK_ADDRESS64(supervisor, prevFP, kernStackMin, kernStackMax)) { frames[ct++] = chudxnu_vm_unslide(currPC, supervisor); prevPC = currPC; } if (prevFP < currFP) { break; } else { currFP = prevFP; } } *start_idx = ct; return KERN_SUCCESS; } static kern_return_t chudxnu_thread_get_callstack64_internal( thread_t thread, uint64_t *callstack, mach_msg_type_number_t *count, boolean_t user_only, boolean_t kern_only) { kern_return_t kr = KERN_FAILURE; task_t task = get_threadtask(thread); uint64_t currPC = 0ULL; boolean_t supervisor = FALSE; mach_msg_type_number_t bufferIndex = 0; mach_msg_type_number_t bufferMaxIndex = *count; x86_saved_state_t *tagged_regs = NULL; // kernel register state x86_saved_state64_t *regs64 = NULL; x86_saved_state32_t *regs32 = NULL; x86_saved_state32_t *u_regs32 = NULL; x86_saved_state64_t *u_regs64 = NULL; struct x86_kernel_state *kregs = NULL; if (ml_at_interrupt_context()) { if (user_only) { /* can't backtrace user state on interrupt stack. */ return KERN_FAILURE; } /* backtracing at interrupt context? */ if (thread == current_thread() && current_cpu_datap()->cpu_int_state) { /* * Locate the registers for the interrupted thread, assuming it is * current_thread(). */ tagged_regs = current_cpu_datap()->cpu_int_state; if (is_saved_state64(tagged_regs)) { /* 64 bit registers */ regs64 = saved_state64(tagged_regs); supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U); } else { /* 32 bit registers */ regs32 = saved_state32(tagged_regs); supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U); } } } if (!ml_at_interrupt_context() && kernel_task == task) { if (!thread->kernel_stack) { return KERN_FAILURE; } // Kernel thread not at interrupt context kregs = (struct x86_kernel_state *)NULL; // nofault read of the thread->kernel_stack pointer if (KERN_SUCCESS != chudxnu_kern_read(&kregs, (vm_offset_t)&(thread->kernel_stack), sizeof(void *))) { return KERN_FAILURE; } // Adjust to find the saved kernel state kregs = STACK_IKS((vm_offset_t)(uintptr_t)kregs); supervisor = TRUE; } else if (!tagged_regs) { /* * not at interrupt context, or tracing a different thread than * current_thread() at interrupt context */ tagged_regs = USER_STATE(thread); if (is_saved_state64(tagged_regs)) { /* 64 bit registers */ regs64 = saved_state64(tagged_regs); supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U); } else { /* 32 bit registers */ regs32 = saved_state32(tagged_regs); supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U); } } *count = 0; if (supervisor) { // the caller only wants a user callstack. if (user_only) { // bail - we've only got kernel state return KERN_FAILURE; } } else { // regs32(64) is not in supervisor mode. u_regs32 = regs32; u_regs64 = regs64; regs32 = NULL; regs64 = NULL; } if (user_only) { /* we only want to backtrace the user mode */ if (!(u_regs32 || u_regs64)) { /* no user state to look at */ return KERN_FAILURE; } } /* * Order of preference for top of stack: * 64 bit kernel state (not likely) * 32 bit kernel state * 64 bit user land state * 32 bit user land state */ if (kregs) { /* * nofault read of the registers from the kernel stack (as they can * disappear on the fly). */ if (KERN_SUCCESS != chudxnu_kern_read(&currPC, (vm_offset_t)&(kregs->k_rip), sizeof(uint64_t))) { return KERN_FAILURE; } } else if (regs64) { currPC = regs64->isf.rip; } else if (regs32) { currPC = (uint64_t) regs32->eip; } else if (u_regs64) { currPC = u_regs64->isf.rip; } else if (u_regs32) { currPC = (uint64_t) u_regs32->eip; } if (!currPC) { /* no top of the stack, bail out */ return KERN_FAILURE; } bufferIndex = 0; if (bufferMaxIndex < 1) { *count = 0; return KERN_RESOURCE_SHORTAGE; } /* backtrace kernel */ if (kregs) { addr64_t address = 0ULL; size_t size = 0UL; // do the backtrace kr = do_kernel_backtrace(thread, kregs, callstack, &bufferIndex, bufferMaxIndex); // and do a nofault read of (r|e)sp uint64_t rsp = 0ULL; size = sizeof(uint64_t); if (KERN_SUCCESS != chudxnu_kern_read(&address, (vm_offset_t)&(kregs->k_rsp), size)) { address = 0ULL; } if (address && KERN_SUCCESS == chudxnu_kern_read(&rsp, (vm_offset_t)address, size) && bufferIndex < bufferMaxIndex) { callstack[bufferIndex++] = (uint64_t)rsp; } } else if (regs64) { uint64_t rsp = 0ULL; // backtrace the 64bit side. kr = do_backtrace64(task, thread, regs64, callstack, &bufferIndex, bufferMaxIndex - 1, TRUE); if (KERN_SUCCESS == chudxnu_kern_read(&rsp, (vm_offset_t) regs64->isf.rsp, sizeof(uint64_t)) && bufferIndex < bufferMaxIndex) { callstack[bufferIndex++] = rsp; } } else if (regs32) { uint32_t esp = 0UL; // backtrace the 32bit side. kr = do_backtrace32(task, thread, regs32, callstack, &bufferIndex, bufferMaxIndex - 1, TRUE); if (KERN_SUCCESS == chudxnu_kern_read(&esp, (vm_offset_t) regs32->uesp, sizeof(uint32_t)) && bufferIndex < bufferMaxIndex) { callstack[bufferIndex++] = (uint64_t) esp; } } else if (u_regs64 && !kern_only) { /* backtrace user land */ uint64_t rsp = 0ULL; kr = do_backtrace64(task, thread, u_regs64, callstack, &bufferIndex, bufferMaxIndex - 1, FALSE); if (KERN_SUCCESS == chudxnu_task_read(task, &rsp, (addr64_t) u_regs64->isf.rsp, sizeof(uint64_t)) && bufferIndex < bufferMaxIndex) { callstack[bufferIndex++] = rsp; } } else if (u_regs32 && !kern_only) { uint32_t esp = 0UL; kr = do_backtrace32(task, thread, u_regs32, callstack, &bufferIndex, bufferMaxIndex - 1, FALSE); if (KERN_SUCCESS == chudxnu_task_read(task, &esp, (addr64_t) u_regs32->uesp, sizeof(uint32_t)) && bufferIndex < bufferMaxIndex) { callstack[bufferIndex++] = (uint64_t) esp; } } *count = bufferIndex; return kr; } __private_extern__ kern_return_t chudxnu_thread_get_callstack64_kperf( thread_t thread, uint64_t *callstack, mach_msg_type_number_t *count, boolean_t is_user) { return chudxnu_thread_get_callstack64_internal(thread, callstack, count, is_user, !is_user); } #else /* !__arm64__ && !__x86_64__ */ #error kperf: unsupported architecture #endif /* !__arm64__ && !__x86_64__ */