1 /* 2 * Copyright (c) 2000,2008-2009 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* 29 * Copyright (c) 1997 Apple Inc. 30 * 31 */ 32 #include <libkern/c++/OSMetaClass.h> 33 #include <libkern/c++/OSKext.h> 34 #include <libkern/c++/OSLib.h> 35 #include <libkern/c++/OSSymbol.h> 36 #include <IOKit/IOKitDebug.h> 37 38 #include <sys/cdefs.h> 39 #if defined(HAS_APPLE_PAC) 40 #include <ptrauth.h> 41 #define PTRAUTH_STRIP_STRUCTOR(x) ((uintptr_t) ptrauth_strip(ptrauth_nop_cast(void *, (x)), ptrauth_key_function_pointer)) 42 #else /* defined(HAS_APPLE_PAC) */ 43 #define PTRAUTH_STRIP_STRUCTOR(x) ((uintptr_t) (x)) 44 #endif /* !defined(HAS_APPLE_PAC) */ 45 46 __BEGIN_DECLS 47 48 #include <string.h> 49 #include <mach/mach_types.h> 50 #include <libkern/kernel_mach_header.h> 51 #include <libkern/prelink.h> 52 #include <stdarg.h> 53 54 #if KASAN 55 #include <san/kasan.h> 56 #endif 57 58 #if CONFIG_SPTM 59 #include <arm64/sptm/sptm.h> 60 #endif 61 62 #if PRAGMA_MARK 63 #pragma mark Constants &c. 64 #endif /* PRAGMA_MARK */ 65 OSKextLogSpec kOSRuntimeLogSpec = 66 kOSKextLogErrorLevel | 67 kOSKextLogLoadFlag | 68 kOSKextLogKextBookkeepingFlag; 69 70 #if PRAGMA_MARK 71 #pragma mark Logging Bootstrap 72 #endif /* PRAGMA_MARK */ 73 /********************************************************************* 74 * kern_os Logging Bootstrap 75 * 76 * We can't call in to OSKext until the kernel's C++ environment is up 77 * and running, so let's mask those references with a check variable. 78 * We print unconditionally if C++ isn't up, but if that's the case 79 * we've generally hit a serious error in kernel init! 80 *********************************************************************/ 81 static bool gKernelCPPInitialized = false; 82 83 #define OSRuntimeLog(kext, flags, format, args ...) \ 84 do { \ 85 if (gKernelCPPInitialized) { \ 86 OSKextLog((kext), (flags), (format), ## args); \ 87 } else { \ 88 printf((format), ## args); \ 89 } \ 90 } while (0) 91 92 #if PRAGMA_MARK 93 #pragma mark Libkern Init 94 #endif /* PRAGMA_MARK */ 95 /********************************************************************* 96 * Libkern Init 97 *********************************************************************/ 98 99 #if __GNUC__ >= 3 100 void __dead2 101 __cxa_pure_virtual( void ) 102 { 103 panic("%s", __FUNCTION__); 104 } 105 #else 106 void __dead2 107 __pure_virtual( void ) 108 { 109 panic("%s", __FUNCTION__); 110 } 111 #endif 112 113 extern lck_grp_t * IOLockGroup; 114 extern kmod_info_t g_kernel_kmod_info; 115 #if CONFIG_SPTM 116 extern kmod_info_t g_sptm_kmod_info, g_txm_kmod_info; 117 #endif /* CONFIG_SPTM */ 118 119 enum { 120 kOSSectionNamesDefault = 0, 121 kOSSectionNamesBuiltinKext = 1, 122 kOSSectionNamesCount = 2, 123 }; 124 enum { 125 kOSSectionNameInitializer = 0, 126 kOSSectionNameFinalizer = 1, 127 kOSSectionNameCount = 2 128 }; 129 130 static const char * 131 gOSStructorSectionNames[kOSSectionNamesCount][kOSSectionNameCount] = { 132 { SECT_MODINITFUNC, SECT_MODTERMFUNC }, 133 { kBuiltinInitSection, kBuiltinTermSection } 134 }; 135 136 void 137 OSlibkernInit(void) 138 { 139 // This must be called before calling OSRuntimeInitializeCPP. 140 OSMetaClassBase::initialize(); 141 142 g_kernel_kmod_info.address = (vm_address_t) &_mh_execute_header; 143 #if CONFIG_SPTM 144 g_sptm_kmod_info.address = (vm_offset_t)SPTMArgs->debug_header->image[DEBUG_HEADER_ENTRY_SPTM]; 145 g_txm_kmod_info.address = (vm_offset_t)SPTMArgs->debug_header->image[DEBUG_HEADER_ENTRY_TXM]; 146 #endif /* CONFIG_SPTM */ 147 148 if (kOSReturnSuccess != OSRuntimeInitializeCPP(NULL)) { 149 // &g_kernel_kmod_info, gOSSectionNamesStandard, 0, 0)) { 150 panic("OSRuntime: C++ runtime failed to initialize."); 151 } 152 153 gKernelCPPInitialized = true; 154 155 return; 156 } 157 158 __END_DECLS 159 160 #if PRAGMA_MARK 161 #pragma mark C++ Runtime Load/Unload 162 #endif /* PRAGMA_MARK */ 163 /********************************************************************* 164 * kern_os C++ Runtime Load/Unload 165 *********************************************************************/ 166 167 typedef void (*structor_t)(void); 168 169 static bool 170 OSRuntimeCallStructorsInSection( 171 OSKext * theKext, 172 kmod_info_t * kmodInfo, 173 void * metaHandle, 174 kernel_segment_command_t * segment, 175 const char * sectionName, 176 uintptr_t textStart, 177 uintptr_t textEnd) 178 { 179 kernel_section_t * section; 180 bool result = TRUE; 181 182 for (section = firstsect(segment); 183 section != NULL; 184 section = nextsect(segment, section)) { 185 if (strncmp(section->sectname, sectionName, sizeof(section->sectname) - 1)) { 186 continue; 187 } 188 if (section->size == 0) { 189 continue; 190 } 191 192 structor_t * structors = (structor_t *)section->addr; 193 if (!structors) { 194 continue; 195 } 196 197 structor_t structor; 198 uintptr_t value; 199 unsigned long num_structors = section->size / sizeof(structor_t); 200 unsigned int hit_null_structor = 0; 201 unsigned long firstIndex = 0; 202 203 if (textStart) { 204 // bsearch for any in range 205 unsigned long baseIdx; 206 unsigned long lim; 207 firstIndex = num_structors; 208 for (lim = num_structors, baseIdx = 0; lim; lim >>= 1) { 209 structor = structors[baseIdx + (lim >> 1)]; 210 if (!structor) { 211 panic("%s: null structor", kmodInfo->name); 212 } 213 value = PTRAUTH_STRIP_STRUCTOR(structor); 214 if ((value >= textStart) && (value < textEnd)) { 215 firstIndex = (baseIdx + (lim >> 1)); 216 // scan back for the first in range 217 for (; firstIndex; firstIndex--) { 218 structor = structors[firstIndex - 1]; 219 value = PTRAUTH_STRIP_STRUCTOR(structor); 220 if ((value < textStart) || (value >= textEnd)) { 221 break; 222 } 223 } 224 break; 225 } 226 if (textStart > value) { 227 // move right 228 baseIdx += (lim >> 1) + 1; 229 lim--; 230 } 231 // else move left 232 } 233 baseIdx = (baseIdx + (lim >> 1)); 234 } 235 for (; 236 (firstIndex < num_structors) 237 && (!metaHandle || OSMetaClass::checkModLoad(metaHandle)); 238 firstIndex++) { 239 if ((structor = structors[firstIndex])) { 240 value = PTRAUTH_STRIP_STRUCTOR(structor); 241 if ((textStart && (value < textStart)) 242 || (textEnd && (value >= textEnd))) { 243 break; 244 } 245 (*structor)(); 246 } else if (!hit_null_structor) { 247 hit_null_structor = 1; 248 OSRuntimeLog(theKext, kOSRuntimeLogSpec, 249 "Null structor in kext %s segment %s!", 250 kmodInfo->name, section->segname); 251 } 252 } 253 if (metaHandle) { 254 result = OSMetaClass::checkModLoad(metaHandle); 255 } 256 break; 257 } /* for (section...) */ 258 return result; 259 } 260 261 /********************************************************************* 262 *********************************************************************/ 263 kern_return_t 264 OSRuntimeFinalizeCPP( 265 OSKext * theKext) 266 { 267 kern_return_t result = KMOD_RETURN_FAILURE; 268 void * metaHandle = NULL;// do not free 269 kernel_mach_header_t * header; 270 kernel_segment_command_t * segment; 271 kmod_info_t * kmodInfo; 272 const char ** sectionNames; 273 uintptr_t textStart; 274 uintptr_t textEnd; 275 276 textStart = 0; 277 textEnd = 0; 278 sectionNames = gOSStructorSectionNames[kOSSectionNamesDefault]; 279 if (theKext) { 280 if (!theKext->isCPPInitialized()) { 281 result = KMOD_RETURN_SUCCESS; 282 goto finish; 283 } 284 kmodInfo = theKext->kmod_info; 285 if (!kmodInfo || !kmodInfo->address) { 286 result = kOSKextReturnInvalidArgument; 287 goto finish; 288 } 289 header = (kernel_mach_header_t *)kmodInfo->address; 290 if (theKext->flags.builtin) { 291 header = (kernel_mach_header_t *)g_kernel_kmod_info.address; 292 textStart = kmodInfo->address; 293 textEnd = textStart + kmodInfo->size; 294 sectionNames = gOSStructorSectionNames[kOSSectionNamesBuiltinKext]; 295 } 296 } else { 297 kmodInfo = &g_kernel_kmod_info; 298 header = (kernel_mach_header_t *)kmodInfo->address; 299 } 300 301 /* OSKext checks for this condition now, but somebody might call 302 * this function directly (the symbol is exported....). 303 */ 304 if (OSMetaClass::modHasInstance(kmodInfo->name)) { 305 // xxx - Don't log under errors? this is more of an info thing 306 OSRuntimeLog(theKext, kOSRuntimeLogSpec, 307 "Can't tear down kext %s C++; classes have instances:", 308 kmodInfo->name); 309 OSKext::reportOSMetaClassInstances(kmodInfo->name, kOSRuntimeLogSpec); 310 result = kOSMetaClassHasInstances; 311 goto finish; 312 } 313 314 /* Tell the meta class system that we are starting to unload. 315 * metaHandle isn't actually needed on the finalize path, 316 * so we don't check it here, even though OSMetaClass::postModLoad() will 317 * return a failure (it only does actual work on the init path anyhow). 318 */ 319 metaHandle = OSMetaClass::preModLoad(kmodInfo->name); 320 321 OSSymbol::checkForPageUnload((void *)kmodInfo->address, 322 (void *)(kmodInfo->address + kmodInfo->size)); 323 324 header = (kernel_mach_header_t *)kmodInfo->address; 325 segment = firstsegfromheader(header); 326 327 for (segment = firstsegfromheader(header); 328 segment != NULL; 329 segment = nextsegfromheader(header, segment)) { 330 OSRuntimeCallStructorsInSection(theKext, kmodInfo, NULL, segment, 331 sectionNames[kOSSectionNameFinalizer], textStart, textEnd); 332 } 333 334 (void)OSMetaClass::postModLoad(metaHandle); 335 336 if (theKext) { 337 theKext->setCPPInitialized(false); 338 } 339 result = KMOD_RETURN_SUCCESS; 340 finish: 341 return result; 342 } 343 344 #if defined(HAS_APPLE_PAC) 345 #if !KASAN 346 /* 347 * Place this function in __KLD,__text on non-kasan builds so it gets unmapped 348 * after CTRR lockdown. 349 */ 350 __attribute__((noinline, section("__KLD,__text"))) 351 #endif 352 static void 353 OSRuntimeSignStructorsInSegment(kernel_segment_command_t *segment) 354 { 355 kernel_section_t * section; 356 structor_t * structors; 357 volatile structor_t structor; 358 size_t idx, num_structors; 359 360 for (section = firstsect(segment); 361 section != NULL; 362 section = nextsect(segment, section)) { 363 if ((S_MOD_INIT_FUNC_POINTERS != (SECTION_TYPE & section->flags)) 364 && (S_MOD_TERM_FUNC_POINTERS != (SECTION_TYPE & section->flags))) { 365 continue; 366 } 367 structors = (structor_t *)section->addr; 368 if (!structors) { 369 continue; 370 } 371 num_structors = section->size / sizeof(structor_t); 372 for (idx = 0; idx < num_structors; idx++) { 373 structor = structors[idx]; 374 if (NULL == structor) { 375 continue; 376 } 377 structor = ptrauth_strip(structor, ptrauth_key_function_pointer); 378 structor = ptrauth_sign_unauthenticated(structor, ptrauth_key_function_pointer, ptrauth_function_pointer_type_discriminator(void (*)(void))); 379 structors[idx] = structor; 380 } 381 } /* for (section...) */ 382 } 383 #endif 384 385 /********************************************************************* 386 *********************************************************************/ 387 void 388 OSRuntimeSignStructors( 389 kernel_mach_header_t * header __unused) 390 { 391 #if defined(HAS_APPLE_PAC) 392 393 kernel_segment_command_t * segment; 394 395 for (segment = firstsegfromheader(header); 396 segment != NULL; 397 segment = nextsegfromheader(header, segment)) { 398 OSRuntimeSignStructorsInSegment(segment); 399 } /* for (segment...) */ 400 #endif /* !defined(XXX) && defined(HAS_APPLE_PAC) */ 401 } 402 403 /********************************************************************* 404 *********************************************************************/ 405 void 406 OSRuntimeSignStructorsInFileset( 407 kernel_mach_header_t * fileset_header __unused) 408 { 409 #if defined(HAS_APPLE_PAC) 410 struct load_command *lc; 411 412 lc = (struct load_command *)((uintptr_t)fileset_header + sizeof(*fileset_header)); 413 for (uint32_t i = 0; i < fileset_header->ncmds; i++, 414 lc = (struct load_command *)((uintptr_t)lc + lc->cmdsize)) { 415 if (lc->cmd == LC_FILESET_ENTRY) { 416 struct fileset_entry_command *fse; 417 kernel_mach_header_t *mh; 418 419 fse = (struct fileset_entry_command *)(uintptr_t)lc; 420 mh = (kernel_mach_header_t *)((uintptr_t)fse->vmaddr); 421 OSRuntimeSignStructors(mh); 422 } else if (lc->cmd == LC_SEGMENT_64) { 423 /* 424 * Slide/adjust all LC_SEGMENT_64 commands in the fileset 425 * (and any sections in those segments) 426 */ 427 kernel_segment_command_t *seg; 428 seg = (kernel_segment_command_t *)(uintptr_t)lc; 429 OSRuntimeSignStructorsInSegment(seg); 430 } 431 } 432 433 #endif /* defined(HAS_APPLE_PAC) */ 434 } 435 436 /********************************************************************* 437 *********************************************************************/ 438 kern_return_t 439 OSRuntimeInitializeCPP( 440 OSKext * theKext) 441 { 442 kern_return_t result = KMOD_RETURN_FAILURE; 443 kernel_mach_header_t * header = NULL; 444 void * metaHandle = NULL;// do not free 445 bool load_success = true; 446 kernel_segment_command_t * segment = NULL;// do not free 447 kernel_segment_command_t * failure_segment = NULL; // do not free 448 kmod_info_t * kmodInfo; 449 const char ** sectionNames; 450 uintptr_t textStart; 451 uintptr_t textEnd; 452 453 textStart = 0; 454 textEnd = 0; 455 sectionNames = gOSStructorSectionNames[kOSSectionNamesDefault]; 456 if (theKext) { 457 if (theKext->isCPPInitialized()) { 458 result = KMOD_RETURN_SUCCESS; 459 goto finish; 460 } 461 462 kmodInfo = theKext->kmod_info; 463 if (!kmodInfo || !kmodInfo->address) { 464 result = kOSKextReturnInvalidArgument; 465 goto finish; 466 } 467 header = (kernel_mach_header_t *)kmodInfo->address; 468 469 if (theKext->flags.builtin) { 470 header = (kernel_mach_header_t *)g_kernel_kmod_info.address; 471 textStart = kmodInfo->address; 472 textEnd = textStart + kmodInfo->size; 473 sectionNames = gOSStructorSectionNames[kOSSectionNamesBuiltinKext]; 474 } 475 } else { 476 kmodInfo = &g_kernel_kmod_info; 477 header = (kernel_mach_header_t *)kmodInfo->address; 478 } 479 480 /* Tell the meta class system that we are starting the load 481 */ 482 metaHandle = OSMetaClass::preModLoad(kmodInfo->name); 483 assert(metaHandle); 484 if (!metaHandle) { 485 goto finish; 486 } 487 488 /* NO GOTO PAST HERE. */ 489 490 /* Scan the header for all constructor sections, in any 491 * segment, and invoke the constructors within those sections. 492 */ 493 for (segment = firstsegfromheader(header); 494 segment != NULL && load_success; 495 segment = nextsegfromheader(header, segment)) { 496 /* Record the current segment in the event of a failure. 497 */ 498 failure_segment = segment; 499 load_success = OSRuntimeCallStructorsInSection( 500 theKext, kmodInfo, metaHandle, segment, 501 sectionNames[kOSSectionNameInitializer], 502 textStart, textEnd); 503 } /* for (segment...) */ 504 505 /* We failed so call all of the destructors. We must do this before 506 * calling OSMetaClass::postModLoad() as the OSMetaClass destructors 507 * will alter state (in the metaHandle) used by that function. 508 */ 509 if (!load_success) { 510 /* Scan the header for all destructor sections, in any 511 * segment, and invoke the constructors within those sections. 512 */ 513 for (segment = firstsegfromheader(header); 514 segment != failure_segment && segment != NULL; 515 segment = nextsegfromheader(header, segment)) { 516 OSRuntimeCallStructorsInSection(theKext, kmodInfo, NULL, segment, 517 sectionNames[kOSSectionNameFinalizer], textStart, textEnd); 518 } /* for (segment...) */ 519 } 520 521 /* Now, regardless of success so far, do the post-init registration 522 * and cleanup. If we had to call the unloadCPP function, static 523 * destructors have removed classes from the stalled list so no 524 * metaclasses will actually be registered. 525 */ 526 result = OSMetaClass::postModLoad(metaHandle); 527 528 /* If we've otherwise been fine up to now, but OSMetaClass::postModLoad() 529 * fails (typically due to a duplicate class), tear down all the C++ 530 * stuff from the kext. This isn't necessary for libkern/OSMetaClass stuff, 531 * but may be necessary for other C++ code. We ignore the return value 532 * because it's only a fail when there are existing instances of libkern 533 * classes, and there had better not be any created on the C++ init path. 534 */ 535 if (load_success && result != KMOD_RETURN_SUCCESS) { 536 (void)OSRuntimeFinalizeCPP(theKext); //kmodInfo, sectionNames, textStart, textEnd); 537 } 538 539 if (theKext && load_success && result == KMOD_RETURN_SUCCESS) { 540 theKext->setCPPInitialized(true); 541 } 542 finish: 543 return result; 544 } 545 546 /********************************************************************* 547 * Unload a kernel segment. 548 *********************************************************************/ 549 550 void 551 OSRuntimeUnloadCPPForSegment( 552 kernel_segment_command_t * segment) 553 { 554 OSRuntimeCallStructorsInSection(NULL, &g_kernel_kmod_info, NULL, segment, 555 gOSStructorSectionNames[kOSSectionNamesDefault][kOSSectionNameFinalizer], 0, 0); 556 } 557 558 #if PRAGMA_MARK 559 #pragma mark C++ Allocators & Deallocators 560 #endif /* PRAGMA_MARK */ 561 /********************************************************************* 562 * C++ Allocators & Deallocators 563 *********************************************************************/ 564 __typed_allocators_ignore_push 565 566 void * 567 operator new(size_t size) 568 { 569 assert(size); 570 return kheap_alloc(KERN_OS_MALLOC, size, 571 Z_VM_TAG_BT(Z_WAITOK_ZERO, VM_KERN_MEMORY_LIBKERN)); 572 } 573 574 void 575 operator delete(void * addr) 576 #if __cplusplus >= 201103L 577 noexcept 578 #endif 579 { 580 kheap_free_addr(KERN_OS_MALLOC, addr); 581 return; 582 } 583 584 void * 585 operator new[](unsigned long size) 586 { 587 return kheap_alloc(KERN_OS_MALLOC, size, 588 Z_VM_TAG_BT(Z_WAITOK_ZERO, VM_KERN_MEMORY_LIBKERN)); 589 } 590 591 void 592 operator delete[](void * ptr) 593 #if __cplusplus >= 201103L 594 noexcept 595 #endif 596 { 597 if (ptr) { 598 #if KASAN 599 /* 600 * Unpoison the C++ array cookie inserted (but not removed) by the 601 * compiler on new[]. 602 */ 603 kasan_unpoison_cxx_array_cookie(ptr); 604 #endif 605 kheap_free_addr(KERN_OS_MALLOC, ptr); 606 } 607 return; 608 } 609 610 #if __cplusplus >= 201103L 611 612 void 613 operator delete(void * addr, size_t sz) noexcept 614 { 615 kheap_free(KERN_OS_MALLOC, addr, sz); 616 } 617 618 void 619 operator delete[](void * addr, size_t sz) noexcept 620 { 621 if (addr) { 622 kheap_free(KERN_OS_MALLOC, addr, sz); 623 } 624 } 625 626 __typed_allocators_ignore_pop 627 628 #endif /* __cplusplus >= 201103L */ 629 630 /* PR-6481964 - The compiler is going to check for size overflows in calls to 631 * new[], and if there is an overflow, it will call __throw_length_error. 632 * This is an unrecoverable error by the C++ standard, so we must panic here. 633 * 634 * We have to put the function inside the std namespace because of how the 635 * compiler expects the name to be mangled. 636 */ 637 namespace std { 638 void __dead2 639 __throw_length_error(const char *msg __unused) 640 { 641 panic("Size of array created by new[] has overflowed"); 642 } 643 }; 644