1 /* 2 * Copyright (c) 2000,2008-2009 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* 29 * Copyright (c) 1997 Apple Inc. 30 * 31 */ 32 #include <libkern/c++/OSMetaClass.h> 33 #include <libkern/c++/OSKext.h> 34 #include <libkern/c++/OSLib.h> 35 #include <libkern/c++/OSSymbol.h> 36 #include <IOKit/IOKitDebug.h> 37 38 #include <sys/cdefs.h> 39 #if defined(HAS_APPLE_PAC) 40 #include <ptrauth.h> 41 #define PTRAUTH_STRIP_STRUCTOR(x) ((uintptr_t) ptrauth_strip(ptrauth_nop_cast(void *, (x)), ptrauth_key_function_pointer)) 42 #else /* defined(HAS_APPLE_PAC) */ 43 #define PTRAUTH_STRIP_STRUCTOR(x) ((uintptr_t) (x)) 44 #endif /* !defined(HAS_APPLE_PAC) */ 45 46 __BEGIN_DECLS 47 48 #include <string.h> 49 #include <mach/mach_types.h> 50 #include <libkern/kernel_mach_header.h> 51 #include <libkern/prelink.h> 52 #include <stdarg.h> 53 54 #if KASAN 55 #include <san/kasan.h> 56 #endif 57 58 #if PRAGMA_MARK 59 #pragma mark Constants &c. 60 #endif /* PRAGMA_MARK */ 61 OSKextLogSpec kOSRuntimeLogSpec = 62 kOSKextLogErrorLevel | 63 kOSKextLogLoadFlag | 64 kOSKextLogKextBookkeepingFlag; 65 66 #if PRAGMA_MARK 67 #pragma mark Logging Bootstrap 68 #endif /* PRAGMA_MARK */ 69 /********************************************************************* 70 * kern_os Logging Bootstrap 71 * 72 * We can't call in to OSKext until the kernel's C++ environment is up 73 * and running, so let's mask those references with a check variable. 74 * We print unconditionally if C++ isn't up, but if that's the case 75 * we've generally hit a serious error in kernel init! 76 *********************************************************************/ 77 static bool gKernelCPPInitialized = false; 78 79 #define OSRuntimeLog(kext, flags, format, args ...) \ 80 do { \ 81 if (gKernelCPPInitialized) { \ 82 OSKextLog((kext), (flags), (format), ## args); \ 83 } else { \ 84 printf((format), ## args); \ 85 } \ 86 } while (0) 87 88 #if PRAGMA_MARK 89 #pragma mark Libkern Init 90 #endif /* PRAGMA_MARK */ 91 /********************************************************************* 92 * Libkern Init 93 *********************************************************************/ 94 95 #if __GNUC__ >= 3 96 void __dead2 97 __cxa_pure_virtual( void ) 98 { 99 panic("%s", __FUNCTION__); 100 } 101 #else 102 void __dead2 103 __pure_virtual( void ) 104 { 105 panic("%s", __FUNCTION__); 106 } 107 #endif 108 109 extern lck_grp_t * IOLockGroup; 110 extern kmod_info_t g_kernel_kmod_info; 111 112 enum { 113 kOSSectionNamesDefault = 0, 114 kOSSectionNamesBuiltinKext = 1, 115 kOSSectionNamesCount = 2, 116 }; 117 enum { 118 kOSSectionNameInitializer = 0, 119 kOSSectionNameFinalizer = 1, 120 kOSSectionNameCount = 2 121 }; 122 123 static const char * 124 gOSStructorSectionNames[kOSSectionNamesCount][kOSSectionNameCount] = { 125 { SECT_MODINITFUNC, SECT_MODTERMFUNC }, 126 { kBuiltinInitSection, kBuiltinTermSection } 127 }; 128 129 void 130 OSlibkernInit(void) 131 { 132 // This must be called before calling OSRuntimeInitializeCPP. 133 OSMetaClassBase::initialize(); 134 135 g_kernel_kmod_info.address = (vm_address_t) &_mh_execute_header; 136 if (kOSReturnSuccess != OSRuntimeInitializeCPP(NULL)) { 137 // &g_kernel_kmod_info, gOSSectionNamesStandard, 0, 0)) { 138 panic("OSRuntime: C++ runtime failed to initialize."); 139 } 140 141 gKernelCPPInitialized = true; 142 143 return; 144 } 145 146 __END_DECLS 147 148 #if PRAGMA_MARK 149 #pragma mark C++ Runtime Load/Unload 150 #endif /* PRAGMA_MARK */ 151 /********************************************************************* 152 * kern_os C++ Runtime Load/Unload 153 *********************************************************************/ 154 155 typedef void (*structor_t)(void); 156 157 static bool 158 OSRuntimeCallStructorsInSection( 159 OSKext * theKext, 160 kmod_info_t * kmodInfo, 161 void * metaHandle, 162 kernel_segment_command_t * segment, 163 const char * sectionName, 164 uintptr_t textStart, 165 uintptr_t textEnd) 166 { 167 kernel_section_t * section; 168 bool result = TRUE; 169 170 for (section = firstsect(segment); 171 section != NULL; 172 section = nextsect(segment, section)) { 173 if (strncmp(section->sectname, sectionName, sizeof(section->sectname) - 1)) { 174 continue; 175 } 176 if (section->size == 0) { 177 continue; 178 } 179 180 structor_t * structors = (structor_t *)section->addr; 181 if (!structors) { 182 continue; 183 } 184 185 structor_t structor; 186 uintptr_t value; 187 unsigned long num_structors = section->size / sizeof(structor_t); 188 unsigned int hit_null_structor = 0; 189 unsigned long firstIndex = 0; 190 191 if (textStart) { 192 // bsearch for any in range 193 unsigned long baseIdx; 194 unsigned long lim; 195 firstIndex = num_structors; 196 for (lim = num_structors, baseIdx = 0; lim; lim >>= 1) { 197 structor = structors[baseIdx + (lim >> 1)]; 198 if (!structor) { 199 panic("%s: null structor", kmodInfo->name); 200 } 201 value = PTRAUTH_STRIP_STRUCTOR(structor); 202 if ((value >= textStart) && (value < textEnd)) { 203 firstIndex = (baseIdx + (lim >> 1)); 204 // scan back for the first in range 205 for (; firstIndex; firstIndex--) { 206 structor = structors[firstIndex - 1]; 207 value = PTRAUTH_STRIP_STRUCTOR(structor); 208 if ((value < textStart) || (value >= textEnd)) { 209 break; 210 } 211 } 212 break; 213 } 214 if (textStart > value) { 215 // move right 216 baseIdx += (lim >> 1) + 1; 217 lim--; 218 } 219 // else move left 220 } 221 baseIdx = (baseIdx + (lim >> 1)); 222 } 223 for (; 224 (firstIndex < num_structors) 225 && (!metaHandle || OSMetaClass::checkModLoad(metaHandle)); 226 firstIndex++) { 227 if ((structor = structors[firstIndex])) { 228 value = PTRAUTH_STRIP_STRUCTOR(structor); 229 if ((textStart && (value < textStart)) 230 || (textEnd && (value >= textEnd))) { 231 break; 232 } 233 (*structor)(); 234 } else if (!hit_null_structor) { 235 hit_null_structor = 1; 236 OSRuntimeLog(theKext, kOSRuntimeLogSpec, 237 "Null structor in kext %s segment %s!", 238 kmodInfo->name, section->segname); 239 } 240 } 241 if (metaHandle) { 242 result = OSMetaClass::checkModLoad(metaHandle); 243 } 244 break; 245 } /* for (section...) */ 246 return result; 247 } 248 249 /********************************************************************* 250 *********************************************************************/ 251 kern_return_t 252 OSRuntimeFinalizeCPP( 253 OSKext * theKext) 254 { 255 kern_return_t result = KMOD_RETURN_FAILURE; 256 void * metaHandle = NULL;// do not free 257 kernel_mach_header_t * header; 258 kernel_segment_command_t * segment; 259 kmod_info_t * kmodInfo; 260 const char ** sectionNames; 261 uintptr_t textStart; 262 uintptr_t textEnd; 263 264 textStart = 0; 265 textEnd = 0; 266 sectionNames = gOSStructorSectionNames[kOSSectionNamesDefault]; 267 if (theKext) { 268 if (!theKext->isCPPInitialized()) { 269 result = KMOD_RETURN_SUCCESS; 270 goto finish; 271 } 272 kmodInfo = theKext->kmod_info; 273 if (!kmodInfo || !kmodInfo->address) { 274 result = kOSKextReturnInvalidArgument; 275 goto finish; 276 } 277 header = (kernel_mach_header_t *)kmodInfo->address; 278 if (theKext->flags.builtin) { 279 header = (kernel_mach_header_t *)g_kernel_kmod_info.address; 280 textStart = kmodInfo->address; 281 textEnd = textStart + kmodInfo->size; 282 sectionNames = gOSStructorSectionNames[kOSSectionNamesBuiltinKext]; 283 } 284 } else { 285 kmodInfo = &g_kernel_kmod_info; 286 header = (kernel_mach_header_t *)kmodInfo->address; 287 } 288 289 /* OSKext checks for this condition now, but somebody might call 290 * this function directly (the symbol is exported....). 291 */ 292 if (OSMetaClass::modHasInstance(kmodInfo->name)) { 293 // xxx - Don't log under errors? this is more of an info thing 294 OSRuntimeLog(theKext, kOSRuntimeLogSpec, 295 "Can't tear down kext %s C++; classes have instances:", 296 kmodInfo->name); 297 OSKext::reportOSMetaClassInstances(kmodInfo->name, kOSRuntimeLogSpec); 298 result = kOSMetaClassHasInstances; 299 goto finish; 300 } 301 302 /* Tell the meta class system that we are starting to unload. 303 * metaHandle isn't actually needed on the finalize path, 304 * so we don't check it here, even though OSMetaClass::postModLoad() will 305 * return a failure (it only does actual work on the init path anyhow). 306 */ 307 metaHandle = OSMetaClass::preModLoad(kmodInfo->name); 308 309 OSSymbol::checkForPageUnload((void *)kmodInfo->address, 310 (void *)(kmodInfo->address + kmodInfo->size)); 311 312 header = (kernel_mach_header_t *)kmodInfo->address; 313 segment = firstsegfromheader(header); 314 315 for (segment = firstsegfromheader(header); 316 segment != NULL; 317 segment = nextsegfromheader(header, segment)) { 318 OSRuntimeCallStructorsInSection(theKext, kmodInfo, NULL, segment, 319 sectionNames[kOSSectionNameFinalizer], textStart, textEnd); 320 } 321 322 (void)OSMetaClass::postModLoad(metaHandle); 323 324 if (theKext) { 325 theKext->setCPPInitialized(false); 326 } 327 result = KMOD_RETURN_SUCCESS; 328 finish: 329 return result; 330 } 331 332 #if defined(HAS_APPLE_PAC) 333 static inline void 334 OSRuntimeSignStructorsInSegment(kernel_segment_command_t *segment) 335 { 336 kernel_section_t * section; 337 structor_t * structors; 338 volatile structor_t structor; 339 size_t idx, num_structors; 340 341 for (section = firstsect(segment); 342 section != NULL; 343 section = nextsect(segment, section)) { 344 if ((S_MOD_INIT_FUNC_POINTERS != (SECTION_TYPE & section->flags)) 345 && (S_MOD_TERM_FUNC_POINTERS != (SECTION_TYPE & section->flags))) { 346 continue; 347 } 348 structors = (structor_t *)section->addr; 349 if (!structors) { 350 continue; 351 } 352 num_structors = section->size / sizeof(structor_t); 353 for (idx = 0; idx < num_structors; idx++) { 354 structor = structors[idx]; 355 if (NULL == structor) { 356 continue; 357 } 358 structor = ptrauth_strip(structor, ptrauth_key_function_pointer); 359 structor = ptrauth_sign_unauthenticated(structor, ptrauth_key_function_pointer, ptrauth_function_pointer_type_discriminator(void (*)(void))); 360 structors[idx] = structor; 361 } 362 } /* for (section...) */ 363 } 364 #endif 365 366 /********************************************************************* 367 *********************************************************************/ 368 void 369 OSRuntimeSignStructors( 370 kernel_mach_header_t * header __unused) 371 { 372 #if defined(HAS_APPLE_PAC) 373 374 kernel_segment_command_t * segment; 375 376 for (segment = firstsegfromheader(header); 377 segment != NULL; 378 segment = nextsegfromheader(header, segment)) { 379 OSRuntimeSignStructorsInSegment(segment); 380 } /* for (segment...) */ 381 #endif /* !defined(XXX) && defined(HAS_APPLE_PAC) */ 382 } 383 384 /********************************************************************* 385 *********************************************************************/ 386 void 387 OSRuntimeSignStructorsInFileset( 388 kernel_mach_header_t * fileset_header __unused) 389 { 390 #if defined(HAS_APPLE_PAC) 391 struct load_command *lc; 392 393 lc = (struct load_command *)((uintptr_t)fileset_header + sizeof(*fileset_header)); 394 for (uint32_t i = 0; i < fileset_header->ncmds; i++, 395 lc = (struct load_command *)((uintptr_t)lc + lc->cmdsize)) { 396 if (lc->cmd == LC_FILESET_ENTRY) { 397 struct fileset_entry_command *fse; 398 kernel_mach_header_t *mh; 399 400 fse = (struct fileset_entry_command *)(uintptr_t)lc; 401 mh = (kernel_mach_header_t *)((uintptr_t)fse->vmaddr); 402 OSRuntimeSignStructors(mh); 403 } else if (lc->cmd == LC_SEGMENT_64) { 404 /* 405 * Slide/adjust all LC_SEGMENT_64 commands in the fileset 406 * (and any sections in those segments) 407 */ 408 kernel_segment_command_t *seg; 409 seg = (kernel_segment_command_t *)(uintptr_t)lc; 410 OSRuntimeSignStructorsInSegment(seg); 411 } 412 } 413 414 #endif /* defined(HAS_APPLE_PAC) */ 415 } 416 417 /********************************************************************* 418 *********************************************************************/ 419 kern_return_t 420 OSRuntimeInitializeCPP( 421 OSKext * theKext) 422 { 423 kern_return_t result = KMOD_RETURN_FAILURE; 424 kernel_mach_header_t * header = NULL; 425 void * metaHandle = NULL;// do not free 426 bool load_success = true; 427 kernel_segment_command_t * segment = NULL;// do not free 428 kernel_segment_command_t * failure_segment = NULL; // do not free 429 kmod_info_t * kmodInfo; 430 const char ** sectionNames; 431 uintptr_t textStart; 432 uintptr_t textEnd; 433 434 textStart = 0; 435 textEnd = 0; 436 sectionNames = gOSStructorSectionNames[kOSSectionNamesDefault]; 437 if (theKext) { 438 if (theKext->isCPPInitialized()) { 439 result = KMOD_RETURN_SUCCESS; 440 goto finish; 441 } 442 443 kmodInfo = theKext->kmod_info; 444 if (!kmodInfo || !kmodInfo->address) { 445 result = kOSKextReturnInvalidArgument; 446 goto finish; 447 } 448 header = (kernel_mach_header_t *)kmodInfo->address; 449 450 if (theKext->flags.builtin) { 451 header = (kernel_mach_header_t *)g_kernel_kmod_info.address; 452 textStart = kmodInfo->address; 453 textEnd = textStart + kmodInfo->size; 454 sectionNames = gOSStructorSectionNames[kOSSectionNamesBuiltinKext]; 455 } 456 } else { 457 kmodInfo = &g_kernel_kmod_info; 458 header = (kernel_mach_header_t *)kmodInfo->address; 459 } 460 461 /* Tell the meta class system that we are starting the load 462 */ 463 metaHandle = OSMetaClass::preModLoad(kmodInfo->name); 464 assert(metaHandle); 465 if (!metaHandle) { 466 goto finish; 467 } 468 469 /* NO GOTO PAST HERE. */ 470 471 /* Scan the header for all constructor sections, in any 472 * segment, and invoke the constructors within those sections. 473 */ 474 for (segment = firstsegfromheader(header); 475 segment != NULL && load_success; 476 segment = nextsegfromheader(header, segment)) { 477 /* Record the current segment in the event of a failure. 478 */ 479 failure_segment = segment; 480 load_success = OSRuntimeCallStructorsInSection( 481 theKext, kmodInfo, metaHandle, segment, 482 sectionNames[kOSSectionNameInitializer], 483 textStart, textEnd); 484 } /* for (segment...) */ 485 486 /* We failed so call all of the destructors. We must do this before 487 * calling OSMetaClass::postModLoad() as the OSMetaClass destructors 488 * will alter state (in the metaHandle) used by that function. 489 */ 490 if (!load_success) { 491 /* Scan the header for all destructor sections, in any 492 * segment, and invoke the constructors within those sections. 493 */ 494 for (segment = firstsegfromheader(header); 495 segment != failure_segment && segment != NULL; 496 segment = nextsegfromheader(header, segment)) { 497 OSRuntimeCallStructorsInSection(theKext, kmodInfo, NULL, segment, 498 sectionNames[kOSSectionNameFinalizer], textStart, textEnd); 499 } /* for (segment...) */ 500 } 501 502 /* Now, regardless of success so far, do the post-init registration 503 * and cleanup. If we had to call the unloadCPP function, static 504 * destructors have removed classes from the stalled list so no 505 * metaclasses will actually be registered. 506 */ 507 result = OSMetaClass::postModLoad(metaHandle); 508 509 /* If we've otherwise been fine up to now, but OSMetaClass::postModLoad() 510 * fails (typically due to a duplicate class), tear down all the C++ 511 * stuff from the kext. This isn't necessary for libkern/OSMetaClass stuff, 512 * but may be necessary for other C++ code. We ignore the return value 513 * because it's only a fail when there are existing instances of libkern 514 * classes, and there had better not be any created on the C++ init path. 515 */ 516 if (load_success && result != KMOD_RETURN_SUCCESS) { 517 (void)OSRuntimeFinalizeCPP(theKext); //kmodInfo, sectionNames, textStart, textEnd); 518 } 519 520 if (theKext && load_success && result == KMOD_RETURN_SUCCESS) { 521 theKext->setCPPInitialized(true); 522 } 523 finish: 524 return result; 525 } 526 527 /********************************************************************* 528 * Unload a kernel segment. 529 *********************************************************************/ 530 531 void 532 OSRuntimeUnloadCPPForSegment( 533 kernel_segment_command_t * segment) 534 { 535 OSRuntimeCallStructorsInSection(NULL, &g_kernel_kmod_info, NULL, segment, 536 gOSStructorSectionNames[kOSSectionNamesDefault][kOSSectionNameFinalizer], 0, 0); 537 } 538 539 #if PRAGMA_MARK 540 #pragma mark C++ Allocators & Deallocators 541 #endif /* PRAGMA_MARK */ 542 /********************************************************************* 543 * C++ Allocators & Deallocators 544 *********************************************************************/ 545 void * 546 operator new(size_t size) 547 { 548 assert(size); 549 return kheap_alloc_tag_bt(KERN_OS_MALLOC, size, 550 (zalloc_flags_t) (Z_WAITOK | Z_ZERO), VM_KERN_MEMORY_LIBKERN); 551 } 552 553 void 554 operator delete(void * addr) 555 #if __cplusplus >= 201103L 556 noexcept 557 #endif 558 { 559 kheap_free_addr(KERN_OS_MALLOC, addr); 560 return; 561 } 562 563 void * 564 operator new[](unsigned long sz) 565 { 566 return kheap_alloc_tag_bt(KERN_OS_MALLOC, sz, 567 (zalloc_flags_t) (Z_WAITOK | Z_ZERO), VM_KERN_MEMORY_LIBKERN); 568 } 569 570 void 571 operator delete[](void * ptr) 572 #if __cplusplus >= 201103L 573 noexcept 574 #endif 575 { 576 if (ptr) { 577 #if KASAN 578 /* 579 * Unpoison the C++ array cookie inserted (but not removed) by the 580 * compiler on new[]. 581 */ 582 kasan_unpoison_cxx_array_cookie(ptr); 583 #endif 584 kheap_free_addr(KERN_OS_MALLOC, ptr); 585 } 586 return; 587 } 588 589 #if __cplusplus >= 201103L 590 591 void 592 operator delete(void * addr, size_t sz) noexcept 593 { 594 kheap_free(KERN_OS_MALLOC, addr, sz); 595 } 596 597 void 598 operator delete[](void * addr, size_t sz) noexcept 599 { 600 if (addr) { 601 kheap_free(KERN_OS_MALLOC, addr, sz); 602 } 603 } 604 605 #endif /* __cplusplus >= 201103L */ 606 607 /* PR-6481964 - The compiler is going to check for size overflows in calls to 608 * new[], and if there is an overflow, it will call __throw_length_error. 609 * This is an unrecoverable error by the C++ standard, so we must panic here. 610 * 611 * We have to put the function inside the std namespace because of how the 612 * compiler expects the name to be mangled. 613 */ 614 namespace std { 615 void __dead2 616 __throw_length_error(const char *msg __unused) 617 { 618 panic("Size of array created by new[] has overflowed"); 619 } 620 }; 621