1 /* 2 * Copyright (c) 2000,2008-2009 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* 29 * Copyright (c) 1997 Apple Inc. 30 * 31 */ 32 #include <libkern/c++/OSMetaClass.h> 33 #include <libkern/c++/OSKext.h> 34 #include <libkern/c++/OSLib.h> 35 #include <libkern/c++/OSSymbol.h> 36 #include <IOKit/IOKitDebug.h> 37 38 #include <sys/cdefs.h> 39 #if defined(HAS_APPLE_PAC) 40 #include <ptrauth.h> 41 #define PTRAUTH_STRIP_STRUCTOR(x) ((uintptr_t) ptrauth_strip(ptrauth_nop_cast(void *, (x)), ptrauth_key_function_pointer)) 42 #else /* defined(HAS_APPLE_PAC) */ 43 #define PTRAUTH_STRIP_STRUCTOR(x) ((uintptr_t) (x)) 44 #endif /* !defined(HAS_APPLE_PAC) */ 45 46 __BEGIN_DECLS 47 48 #include <string.h> 49 #include <mach/mach_types.h> 50 #include <libkern/kernel_mach_header.h> 51 #include <libkern/prelink.h> 52 #include <stdarg.h> 53 54 #if KASAN 55 #include <san/kasan.h> 56 #endif 57 58 #if PRAGMA_MARK 59 #pragma mark Constants &c. 60 #endif /* PRAGMA_MARK */ 61 OSKextLogSpec kOSRuntimeLogSpec = 62 kOSKextLogErrorLevel | 63 kOSKextLogLoadFlag | 64 kOSKextLogKextBookkeepingFlag; 65 66 #if PRAGMA_MARK 67 #pragma mark Logging Bootstrap 68 #endif /* PRAGMA_MARK */ 69 /********************************************************************* 70 * kern_os Logging Bootstrap 71 * 72 * We can't call in to OSKext until the kernel's C++ environment is up 73 * and running, so let's mask those references with a check variable. 74 * We print unconditionally if C++ isn't up, but if that's the case 75 * we've generally hit a serious error in kernel init! 76 *********************************************************************/ 77 static bool gKernelCPPInitialized = false; 78 79 #define OSRuntimeLog(kext, flags, format, args ...) \ 80 do { \ 81 if (gKernelCPPInitialized) { \ 82 OSKextLog((kext), (flags), (format), ## args); \ 83 } else { \ 84 printf((format), ## args); \ 85 } \ 86 } while (0) 87 88 #if PRAGMA_MARK 89 #pragma mark Libkern Init 90 #endif /* PRAGMA_MARK */ 91 /********************************************************************* 92 * Libkern Init 93 *********************************************************************/ 94 95 #if __GNUC__ >= 3 96 void __dead2 97 __cxa_pure_virtual( void ) 98 { 99 panic("%s", __FUNCTION__); 100 } 101 #else 102 void __dead2 103 __pure_virtual( void ) 104 { 105 panic("%s", __FUNCTION__); 106 } 107 #endif 108 109 extern lck_grp_t * IOLockGroup; 110 extern kmod_info_t g_kernel_kmod_info; 111 112 enum { 113 kOSSectionNamesDefault = 0, 114 kOSSectionNamesBuiltinKext = 1, 115 kOSSectionNamesCount = 2, 116 }; 117 enum { 118 kOSSectionNameInitializer = 0, 119 kOSSectionNameFinalizer = 1, 120 kOSSectionNameCount = 2 121 }; 122 123 static const char * 124 gOSStructorSectionNames[kOSSectionNamesCount][kOSSectionNameCount] = { 125 { SECT_MODINITFUNC, SECT_MODTERMFUNC }, 126 { kBuiltinInitSection, kBuiltinTermSection } 127 }; 128 129 void 130 OSlibkernInit(void) 131 { 132 // This must be called before calling OSRuntimeInitializeCPP. 133 OSMetaClassBase::initialize(); 134 135 g_kernel_kmod_info.address = (vm_address_t) &_mh_execute_header; 136 if (kOSReturnSuccess != OSRuntimeInitializeCPP(NULL)) { 137 // &g_kernel_kmod_info, gOSSectionNamesStandard, 0, 0)) { 138 panic("OSRuntime: C++ runtime failed to initialize."); 139 } 140 141 gKernelCPPInitialized = true; 142 143 return; 144 } 145 146 __END_DECLS 147 148 #if PRAGMA_MARK 149 #pragma mark C++ Runtime Load/Unload 150 #endif /* PRAGMA_MARK */ 151 /********************************************************************* 152 * kern_os C++ Runtime Load/Unload 153 *********************************************************************/ 154 155 typedef void (*structor_t)(void); 156 157 static bool 158 OSRuntimeCallStructorsInSection( 159 OSKext * theKext, 160 kmod_info_t * kmodInfo, 161 void * metaHandle, 162 kernel_segment_command_t * segment, 163 const char * sectionName, 164 uintptr_t textStart, 165 uintptr_t textEnd) 166 { 167 kernel_section_t * section; 168 bool result = TRUE; 169 170 for (section = firstsect(segment); 171 section != NULL; 172 section = nextsect(segment, section)) { 173 if (strncmp(section->sectname, sectionName, sizeof(section->sectname) - 1)) { 174 continue; 175 } 176 if (section->size == 0) { 177 continue; 178 } 179 180 structor_t * structors = (structor_t *)section->addr; 181 if (!structors) { 182 continue; 183 } 184 185 structor_t structor; 186 uintptr_t value; 187 unsigned long num_structors = section->size / sizeof(structor_t); 188 unsigned int hit_null_structor = 0; 189 unsigned long firstIndex = 0; 190 191 if (textStart) { 192 // bsearch for any in range 193 unsigned long baseIdx; 194 unsigned long lim; 195 firstIndex = num_structors; 196 for (lim = num_structors, baseIdx = 0; lim; lim >>= 1) { 197 structor = structors[baseIdx + (lim >> 1)]; 198 if (!structor) { 199 panic("%s: null structor", kmodInfo->name); 200 } 201 value = PTRAUTH_STRIP_STRUCTOR(structor); 202 if ((value >= textStart) && (value < textEnd)) { 203 firstIndex = (baseIdx + (lim >> 1)); 204 // scan back for the first in range 205 for (; firstIndex; firstIndex--) { 206 structor = structors[firstIndex - 1]; 207 value = PTRAUTH_STRIP_STRUCTOR(structor); 208 if ((value < textStart) || (value >= textEnd)) { 209 break; 210 } 211 } 212 break; 213 } 214 if (textStart > value) { 215 // move right 216 baseIdx += (lim >> 1) + 1; 217 lim--; 218 } 219 // else move left 220 } 221 baseIdx = (baseIdx + (lim >> 1)); 222 } 223 for (; 224 (firstIndex < num_structors) 225 && (!metaHandle || OSMetaClass::checkModLoad(metaHandle)); 226 firstIndex++) { 227 if ((structor = structors[firstIndex])) { 228 value = PTRAUTH_STRIP_STRUCTOR(structor); 229 if ((textStart && (value < textStart)) 230 || (textEnd && (value >= textEnd))) { 231 break; 232 } 233 (*structor)(); 234 } else if (!hit_null_structor) { 235 hit_null_structor = 1; 236 OSRuntimeLog(theKext, kOSRuntimeLogSpec, 237 "Null structor in kext %s segment %s!", 238 kmodInfo->name, section->segname); 239 } 240 } 241 if (metaHandle) { 242 result = OSMetaClass::checkModLoad(metaHandle); 243 } 244 break; 245 } /* for (section...) */ 246 return result; 247 } 248 249 /********************************************************************* 250 *********************************************************************/ 251 kern_return_t 252 OSRuntimeFinalizeCPP( 253 OSKext * theKext) 254 { 255 kern_return_t result = KMOD_RETURN_FAILURE; 256 void * metaHandle = NULL;// do not free 257 kernel_mach_header_t * header; 258 kernel_segment_command_t * segment; 259 kmod_info_t * kmodInfo; 260 const char ** sectionNames; 261 uintptr_t textStart; 262 uintptr_t textEnd; 263 264 textStart = 0; 265 textEnd = 0; 266 sectionNames = gOSStructorSectionNames[kOSSectionNamesDefault]; 267 if (theKext) { 268 if (!theKext->isCPPInitialized()) { 269 result = KMOD_RETURN_SUCCESS; 270 goto finish; 271 } 272 kmodInfo = theKext->kmod_info; 273 if (!kmodInfo || !kmodInfo->address) { 274 result = kOSKextReturnInvalidArgument; 275 goto finish; 276 } 277 header = (kernel_mach_header_t *)kmodInfo->address; 278 if (theKext->flags.builtin) { 279 header = (kernel_mach_header_t *)g_kernel_kmod_info.address; 280 textStart = kmodInfo->address; 281 textEnd = textStart + kmodInfo->size; 282 sectionNames = gOSStructorSectionNames[kOSSectionNamesBuiltinKext]; 283 } 284 } else { 285 kmodInfo = &g_kernel_kmod_info; 286 header = (kernel_mach_header_t *)kmodInfo->address; 287 } 288 289 /* OSKext checks for this condition now, but somebody might call 290 * this function directly (the symbol is exported....). 291 */ 292 if (OSMetaClass::modHasInstance(kmodInfo->name)) { 293 // xxx - Don't log under errors? this is more of an info thing 294 OSRuntimeLog(theKext, kOSRuntimeLogSpec, 295 "Can't tear down kext %s C++; classes have instances:", 296 kmodInfo->name); 297 OSKext::reportOSMetaClassInstances(kmodInfo->name, kOSRuntimeLogSpec); 298 result = kOSMetaClassHasInstances; 299 goto finish; 300 } 301 302 /* Tell the meta class system that we are starting to unload. 303 * metaHandle isn't actually needed on the finalize path, 304 * so we don't check it here, even though OSMetaClass::postModLoad() will 305 * return a failure (it only does actual work on the init path anyhow). 306 */ 307 metaHandle = OSMetaClass::preModLoad(kmodInfo->name); 308 309 OSSymbol::checkForPageUnload((void *)kmodInfo->address, 310 (void *)(kmodInfo->address + kmodInfo->size)); 311 312 header = (kernel_mach_header_t *)kmodInfo->address; 313 segment = firstsegfromheader(header); 314 315 for (segment = firstsegfromheader(header); 316 segment != NULL; 317 segment = nextsegfromheader(header, segment)) { 318 OSRuntimeCallStructorsInSection(theKext, kmodInfo, NULL, segment, 319 sectionNames[kOSSectionNameFinalizer], textStart, textEnd); 320 } 321 322 (void)OSMetaClass::postModLoad(metaHandle); 323 324 if (theKext) { 325 theKext->setCPPInitialized(false); 326 } 327 result = KMOD_RETURN_SUCCESS; 328 finish: 329 return result; 330 } 331 332 #if defined(HAS_APPLE_PAC) 333 #if !KASAN 334 /* 335 * Place this function in __KLD,__text on non-kasan builds so it gets unmapped 336 * after CTRR lockdown. 337 */ 338 __attribute__((noinline, section("__KLD,__text"))) 339 #endif 340 static void 341 OSRuntimeSignStructorsInSegment(kernel_segment_command_t *segment) 342 { 343 kernel_section_t * section; 344 structor_t * structors; 345 volatile structor_t structor; 346 size_t idx, num_structors; 347 348 for (section = firstsect(segment); 349 section != NULL; 350 section = nextsect(segment, section)) { 351 if ((S_MOD_INIT_FUNC_POINTERS != (SECTION_TYPE & section->flags)) 352 && (S_MOD_TERM_FUNC_POINTERS != (SECTION_TYPE & section->flags))) { 353 continue; 354 } 355 structors = (structor_t *)section->addr; 356 if (!structors) { 357 continue; 358 } 359 num_structors = section->size / sizeof(structor_t); 360 for (idx = 0; idx < num_structors; idx++) { 361 structor = structors[idx]; 362 if (NULL == structor) { 363 continue; 364 } 365 structor = ptrauth_strip(structor, ptrauth_key_function_pointer); 366 structor = ptrauth_sign_unauthenticated(structor, ptrauth_key_function_pointer, ptrauth_function_pointer_type_discriminator(void (*)(void))); 367 structors[idx] = structor; 368 } 369 } /* for (section...) */ 370 } 371 #endif 372 373 /********************************************************************* 374 *********************************************************************/ 375 void 376 OSRuntimeSignStructors( 377 kernel_mach_header_t * header __unused) 378 { 379 #if defined(HAS_APPLE_PAC) 380 381 kernel_segment_command_t * segment; 382 383 for (segment = firstsegfromheader(header); 384 segment != NULL; 385 segment = nextsegfromheader(header, segment)) { 386 OSRuntimeSignStructorsInSegment(segment); 387 } /* for (segment...) */ 388 #endif /* !defined(XXX) && defined(HAS_APPLE_PAC) */ 389 } 390 391 /********************************************************************* 392 *********************************************************************/ 393 void 394 OSRuntimeSignStructorsInFileset( 395 kernel_mach_header_t * fileset_header __unused) 396 { 397 #if defined(HAS_APPLE_PAC) 398 struct load_command *lc; 399 400 lc = (struct load_command *)((uintptr_t)fileset_header + sizeof(*fileset_header)); 401 for (uint32_t i = 0; i < fileset_header->ncmds; i++, 402 lc = (struct load_command *)((uintptr_t)lc + lc->cmdsize)) { 403 if (lc->cmd == LC_FILESET_ENTRY) { 404 struct fileset_entry_command *fse; 405 kernel_mach_header_t *mh; 406 407 fse = (struct fileset_entry_command *)(uintptr_t)lc; 408 mh = (kernel_mach_header_t *)((uintptr_t)fse->vmaddr); 409 OSRuntimeSignStructors(mh); 410 } else if (lc->cmd == LC_SEGMENT_64) { 411 /* 412 * Slide/adjust all LC_SEGMENT_64 commands in the fileset 413 * (and any sections in those segments) 414 */ 415 kernel_segment_command_t *seg; 416 seg = (kernel_segment_command_t *)(uintptr_t)lc; 417 OSRuntimeSignStructorsInSegment(seg); 418 } 419 } 420 421 #endif /* defined(HAS_APPLE_PAC) */ 422 } 423 424 /********************************************************************* 425 *********************************************************************/ 426 kern_return_t 427 OSRuntimeInitializeCPP( 428 OSKext * theKext) 429 { 430 kern_return_t result = KMOD_RETURN_FAILURE; 431 kernel_mach_header_t * header = NULL; 432 void * metaHandle = NULL;// do not free 433 bool load_success = true; 434 kernel_segment_command_t * segment = NULL;// do not free 435 kernel_segment_command_t * failure_segment = NULL; // do not free 436 kmod_info_t * kmodInfo; 437 const char ** sectionNames; 438 uintptr_t textStart; 439 uintptr_t textEnd; 440 441 textStart = 0; 442 textEnd = 0; 443 sectionNames = gOSStructorSectionNames[kOSSectionNamesDefault]; 444 if (theKext) { 445 if (theKext->isCPPInitialized()) { 446 result = KMOD_RETURN_SUCCESS; 447 goto finish; 448 } 449 450 kmodInfo = theKext->kmod_info; 451 if (!kmodInfo || !kmodInfo->address) { 452 result = kOSKextReturnInvalidArgument; 453 goto finish; 454 } 455 header = (kernel_mach_header_t *)kmodInfo->address; 456 457 if (theKext->flags.builtin) { 458 header = (kernel_mach_header_t *)g_kernel_kmod_info.address; 459 textStart = kmodInfo->address; 460 textEnd = textStart + kmodInfo->size; 461 sectionNames = gOSStructorSectionNames[kOSSectionNamesBuiltinKext]; 462 } 463 } else { 464 kmodInfo = &g_kernel_kmod_info; 465 header = (kernel_mach_header_t *)kmodInfo->address; 466 } 467 468 /* Tell the meta class system that we are starting the load 469 */ 470 metaHandle = OSMetaClass::preModLoad(kmodInfo->name); 471 assert(metaHandle); 472 if (!metaHandle) { 473 goto finish; 474 } 475 476 /* NO GOTO PAST HERE. */ 477 478 /* Scan the header for all constructor sections, in any 479 * segment, and invoke the constructors within those sections. 480 */ 481 for (segment = firstsegfromheader(header); 482 segment != NULL && load_success; 483 segment = nextsegfromheader(header, segment)) { 484 /* Record the current segment in the event of a failure. 485 */ 486 failure_segment = segment; 487 load_success = OSRuntimeCallStructorsInSection( 488 theKext, kmodInfo, metaHandle, segment, 489 sectionNames[kOSSectionNameInitializer], 490 textStart, textEnd); 491 } /* for (segment...) */ 492 493 /* We failed so call all of the destructors. We must do this before 494 * calling OSMetaClass::postModLoad() as the OSMetaClass destructors 495 * will alter state (in the metaHandle) used by that function. 496 */ 497 if (!load_success) { 498 /* Scan the header for all destructor sections, in any 499 * segment, and invoke the constructors within those sections. 500 */ 501 for (segment = firstsegfromheader(header); 502 segment != failure_segment && segment != NULL; 503 segment = nextsegfromheader(header, segment)) { 504 OSRuntimeCallStructorsInSection(theKext, kmodInfo, NULL, segment, 505 sectionNames[kOSSectionNameFinalizer], textStart, textEnd); 506 } /* for (segment...) */ 507 } 508 509 /* Now, regardless of success so far, do the post-init registration 510 * and cleanup. If we had to call the unloadCPP function, static 511 * destructors have removed classes from the stalled list so no 512 * metaclasses will actually be registered. 513 */ 514 result = OSMetaClass::postModLoad(metaHandle); 515 516 /* If we've otherwise been fine up to now, but OSMetaClass::postModLoad() 517 * fails (typically due to a duplicate class), tear down all the C++ 518 * stuff from the kext. This isn't necessary for libkern/OSMetaClass stuff, 519 * but may be necessary for other C++ code. We ignore the return value 520 * because it's only a fail when there are existing instances of libkern 521 * classes, and there had better not be any created on the C++ init path. 522 */ 523 if (load_success && result != KMOD_RETURN_SUCCESS) { 524 (void)OSRuntimeFinalizeCPP(theKext); //kmodInfo, sectionNames, textStart, textEnd); 525 } 526 527 if (theKext && load_success && result == KMOD_RETURN_SUCCESS) { 528 theKext->setCPPInitialized(true); 529 } 530 finish: 531 return result; 532 } 533 534 /********************************************************************* 535 * Unload a kernel segment. 536 *********************************************************************/ 537 538 void 539 OSRuntimeUnloadCPPForSegment( 540 kernel_segment_command_t * segment) 541 { 542 OSRuntimeCallStructorsInSection(NULL, &g_kernel_kmod_info, NULL, segment, 543 gOSStructorSectionNames[kOSSectionNamesDefault][kOSSectionNameFinalizer], 0, 0); 544 } 545 546 #if PRAGMA_MARK 547 #pragma mark C++ Allocators & Deallocators 548 #endif /* PRAGMA_MARK */ 549 /********************************************************************* 550 * C++ Allocators & Deallocators 551 *********************************************************************/ 552 void * 553 operator new(size_t size) 554 { 555 assert(size); 556 return kheap_alloc(KERN_OS_MALLOC, size, 557 Z_VM_TAG_BT(Z_WAITOK_ZERO, VM_KERN_MEMORY_LIBKERN)); 558 } 559 560 void 561 operator delete(void * addr) 562 #if __cplusplus >= 201103L 563 noexcept 564 #endif 565 { 566 kheap_free_addr(KERN_OS_MALLOC, addr); 567 return; 568 } 569 570 void * 571 operator new[](unsigned long size) 572 { 573 return kheap_alloc(KERN_OS_MALLOC, size, 574 Z_VM_TAG_BT(Z_WAITOK_ZERO, VM_KERN_MEMORY_LIBKERN)); 575 } 576 577 void 578 operator delete[](void * ptr) 579 #if __cplusplus >= 201103L 580 noexcept 581 #endif 582 { 583 if (ptr) { 584 #if KASAN 585 /* 586 * Unpoison the C++ array cookie inserted (but not removed) by the 587 * compiler on new[]. 588 */ 589 kasan_unpoison_cxx_array_cookie(ptr); 590 #endif 591 kheap_free_addr(KERN_OS_MALLOC, ptr); 592 } 593 return; 594 } 595 596 #if __cplusplus >= 201103L 597 598 void 599 operator delete(void * addr, size_t sz) noexcept 600 { 601 kheap_free(KERN_OS_MALLOC, addr, sz); 602 } 603 604 void 605 operator delete[](void * addr, size_t sz) noexcept 606 { 607 if (addr) { 608 kheap_free(KERN_OS_MALLOC, addr, sz); 609 } 610 } 611 612 #endif /* __cplusplus >= 201103L */ 613 614 /* PR-6481964 - The compiler is going to check for size overflows in calls to 615 * new[], and if there is an overflow, it will call __throw_length_error. 616 * This is an unrecoverable error by the C++ standard, so we must panic here. 617 * 618 * We have to put the function inside the std namespace because of how the 619 * compiler expects the name to be mangled. 620 */ 621 namespace std { 622 void __dead2 623 __throw_length_error(const char *msg __unused) 624 { 625 panic("Size of array created by new[] has overflowed"); 626 } 627 }; 628