1 /* 2 * Copyright (c) 1998-2016 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #include <sys/sysctl.h> 30 extern "C" { 31 #include <vm/vm_kern.h> 32 #include <kern/task.h> 33 #include <kern/debug.h> 34 } 35 36 #include <libkern/c++/OSContainers.h> 37 #include <libkern/OSDebug.h> 38 #include <libkern/c++/OSCPPDebug.h> 39 #include <kern/backtrace.h> 40 #include <kern/btlog.h> 41 42 #include <IOKit/IOKitDebug.h> 43 #include <IOKit/IOLib.h> 44 #include <IOKit/assert.h> 45 #include <IOKit/IODeviceTreeSupport.h> 46 #include <IOKit/IOService.h> 47 48 #include "IOKitKernelInternal.h" 49 50 TUNABLE_WRITEABLE(SInt64, gIOKitDebug, "io", DEBUG_INIT_VALUE); 51 TUNABLE_DEV_WRITEABLE(SInt64, gIOKitTrace, "iotrace", 0); 52 53 #if DEVELOPMENT || DEBUG 54 #define IODEBUG_CTLFLAGS CTLFLAG_RW 55 #else 56 #define IODEBUG_CTLFLAGS CTLFLAG_RD 57 #endif 58 59 SYSCTL_QUAD(_debug, OID_AUTO, iotrace, IODEBUG_CTLFLAGS | CTLFLAG_LOCKED, &gIOKitTrace, "trace io"); 60 61 static int 62 sysctl_debug_iokit 63 (__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req) 64 { 65 SInt64 newValue; 66 int changed, error = sysctl_io_number(req, gIOKitDebug, sizeof(gIOKitDebug), &newValue, &changed); 67 if (changed) { 68 gIOKitDebug = ((gIOKitDebug & ~kIOKitDebugUserOptions) | (newValue & kIOKitDebugUserOptions)); 69 } 70 return error; 71 } 72 73 SYSCTL_PROC(_debug, OID_AUTO, iokit, 74 CTLTYPE_QUAD | IODEBUG_CTLFLAGS | CTLFLAG_KERN | CTLFLAG_LOCKED, 75 &gIOKitDebug, 0, sysctl_debug_iokit, "Q", "boot_arg io"); 76 77 void (*gIOTrackingLeakScanCallback)(uint32_t notification) = NULL; 78 79 size_t debug_malloc_size; 80 size_t debug_iomalloc_size; 81 82 vm_size_t debug_iomallocpageable_size; 83 size_t debug_container_malloc_size; 84 // int debug_ivars_size; // in OSObject.cpp 85 86 extern "C" { 87 #if 0 88 #define DEBG(fmt, args...) { kprintf(fmt, ## args); } 89 #else 90 #define DEBG(fmt, args...) { IOLog(fmt, ## args); } 91 #endif 92 93 void 94 IOPrintPlane( const IORegistryPlane * plane ) 95 { 96 IORegistryEntry * next; 97 IORegistryIterator * iter; 98 OSOrderedSet * all; 99 IOService * service; 100 101 iter = IORegistryIterator::iterateOver( plane ); 102 assert( iter ); 103 all = iter->iterateAll(); 104 if (all) { 105 DEBG("Count %d\n", all->getCount()); 106 all->release(); 107 } else { 108 DEBG("Empty\n"); 109 } 110 111 iter->reset(); 112 while ((next = iter->getNextObjectRecursive())) { 113 DEBG( "%*s\033[33m%s", 2 * next->getDepth( plane ), "", next->getName( plane )); 114 if ((next->getLocation( plane ))) { 115 DEBG("@%s", next->getLocation( plane )); 116 } 117 DEBG("\033[0m <class %s", next->getMetaClass()->getClassName()); 118 if ((service = OSDynamicCast(IOService, next))) { 119 DEBG(", busy %ld", (long) service->getBusyState()); 120 } 121 DEBG( ">\n"); 122 // IOSleep(250); 123 } 124 iter->release(); 125 126 #undef IOPrintPlaneFormat 127 } 128 129 void 130 db_piokjunk(void) 131 { 132 } 133 134 void 135 db_dumpiojunk( const IORegistryPlane * plane __unused ) 136 { 137 } 138 139 void 140 IOPrintMemory( void ) 141 { 142 // OSMetaClass::printInstanceCounts(); 143 144 IOLog("\n" 145 "ivar kalloc() 0x%08lx\n" 146 "malloc() 0x%08lx\n" 147 "containers kalloc() 0x%08lx\n" 148 "IOMalloc() 0x%08lx\n" 149 "----------------------------------------\n", 150 debug_ivars_size, 151 debug_malloc_size, 152 debug_container_malloc_size, 153 debug_iomalloc_size 154 ); 155 } 156 } /* extern "C" */ 157 158 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 159 160 #define super OSObject 161 OSDefineMetaClassAndStructors(IOKitDiagnostics, OSObject) 162 163 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 164 165 OSObject * IOKitDiagnostics::diagnostics( void ) 166 { 167 IOKitDiagnostics * diags; 168 169 diags = new IOKitDiagnostics; 170 if (diags && !diags->init()) { 171 diags->release(); 172 diags = NULL; 173 } 174 175 return diags; 176 } 177 178 void 179 IOKitDiagnostics::updateOffset( OSDictionary * dict, 180 UInt64 value, const char * name ) 181 { 182 OSNumber * off; 183 184 off = OSNumber::withNumber( value, 64 ); 185 if (!off) { 186 return; 187 } 188 189 dict->setObject( name, off ); 190 off->release(); 191 } 192 193 bool 194 IOKitDiagnostics::serialize(OSSerialize *s) const 195 { 196 OSDictionary * dict; 197 bool ok; 198 199 dict = OSDictionary::withCapacity( 5 ); 200 if (!dict) { 201 return false; 202 } 203 204 updateOffset( dict, debug_ivars_size, "Instance allocation" ); 205 updateOffset( dict, debug_container_malloc_size, "Container allocation" ); 206 updateOffset( dict, debug_iomalloc_size, "IOMalloc allocation" ); 207 updateOffset( dict, debug_iomallocpageable_size, "Pageable allocation" ); 208 209 OSMetaClass::serializeClassDictionary(dict); 210 211 ok = dict->serialize( s ); 212 213 dict->release(); 214 215 return ok; 216 } 217 218 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 219 220 #if IOTRACKING 221 222 #include <libkern/c++/OSCPPDebug.h> 223 #include <libkern/c++/OSKext.h> 224 #include <kern/zalloc.h> 225 226 __private_extern__ "C" void qsort( 227 void * array, 228 size_t nmembers, 229 size_t member_size, 230 int (*)(const void *, const void *)); 231 232 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); 233 extern "C" ppnum_t pmap_valid_page(ppnum_t pn); 234 235 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 236 237 struct IOTRecursiveLock { 238 lck_mtx_t * mutex; 239 thread_t thread; 240 UInt32 count; 241 }; 242 243 struct IOTrackingQueue { 244 queue_chain_t link; 245 IOTRecursiveLock lock; 246 const char * name; 247 uintptr_t btEntry; 248 size_t allocSize; 249 size_t minCaptureSize; 250 uint32_t siteCount; 251 uint32_t type; 252 uint32_t numSiteQs; 253 uint8_t captureOn; 254 queue_head_t sites[]; 255 }; 256 257 258 struct IOTrackingCallSiteUser { 259 pid_t pid; 260 uint8_t user32; 261 uint8_t userCount; 262 uintptr_t bt[kIOTrackingCallSiteBTs]; 263 }; 264 265 struct IOTrackingCallSite { 266 queue_chain_t link; 267 queue_head_t instances; 268 IOTrackingQueue * queue; 269 IOTracking ** addresses; 270 size_t size[2]; 271 uint32_t crc; 272 uint32_t count; 273 274 vm_tag_t tag; 275 uint8_t user32; 276 uint8_t userCount; 277 pid_t btPID; 278 279 uintptr_t bt[kIOTrackingCallSiteBTs]; 280 IOTrackingCallSiteUser user[0]; 281 }; 282 283 struct IOTrackingCallSiteWithUser { 284 struct IOTrackingCallSite site; 285 struct IOTrackingCallSiteUser user; 286 }; 287 288 static void IOTrackingFreeCallSite(uint32_t type, IOTrackingCallSite ** site); 289 290 struct IOTrackingLeaksRef { 291 uintptr_t * instances; 292 uint32_t zoneSize; 293 uint32_t count; 294 uint32_t found; 295 uint32_t foundzlen; 296 size_t bytes; 297 }; 298 299 lck_mtx_t * gIOTrackingLock; 300 queue_head_t gIOTrackingQ; 301 302 enum{ 303 kTrackingAddressFlagAllocated = 0x00000001 304 }; 305 306 #if defined(__LP64__) 307 #define IOTrackingAddressFlags(ptr) (ptr->flags) 308 #else 309 #define IOTrackingAddressFlags(ptr) (ptr->tracking.flags) 310 #endif 311 312 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 313 314 static void 315 IOTRecursiveLockLock(IOTRecursiveLock * lock) 316 { 317 if (lock->thread == current_thread()) { 318 lock->count++; 319 } else { 320 lck_mtx_lock(lock->mutex); 321 assert(lock->thread == NULL); 322 assert(lock->count == 0); 323 lock->thread = current_thread(); 324 lock->count = 1; 325 } 326 } 327 328 static void 329 IOTRecursiveLockUnlock(IOTRecursiveLock * lock) 330 { 331 assert(lock->thread == current_thread()); 332 if (0 == (--lock->count)) { 333 lock->thread = NULL; 334 lck_mtx_unlock(lock->mutex); 335 } 336 } 337 338 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 339 340 void 341 IOTrackingInit(void) 342 { 343 queue_init(&gIOTrackingQ); 344 gIOTrackingLock = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL); 345 } 346 347 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 348 349 IOTrackingQueue * 350 IOTrackingQueueAlloc(const char * name, uintptr_t btEntry, 351 size_t allocSize, size_t minCaptureSize, 352 uint32_t type, uint32_t numSiteQs) 353 { 354 IOTrackingQueue * queue; 355 uint32_t idx; 356 357 if (!numSiteQs) { 358 numSiteQs = 1; 359 } 360 queue = kalloc_type(IOTrackingQueue, queue_head_t, numSiteQs, Z_WAITOK_ZERO); 361 queue->name = name; 362 queue->btEntry = btEntry; 363 queue->allocSize = allocSize; 364 queue->minCaptureSize = minCaptureSize; 365 queue->lock.mutex = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL); 366 queue->numSiteQs = numSiteQs; 367 queue->type = type; 368 enum { kFlags = (kIOTracking | kIOTrackingBoot) }; 369 queue->captureOn = (kFlags == (kFlags & gIOKitDebug)) 370 || (kIOTrackingQueueTypeDefaultOn & type); 371 372 for (idx = 0; idx < numSiteQs; idx++) { 373 queue_init(&queue->sites[idx]); 374 } 375 376 lck_mtx_lock(gIOTrackingLock); 377 queue_enter(&gIOTrackingQ, queue, IOTrackingQueue *, link); 378 lck_mtx_unlock(gIOTrackingLock); 379 380 return queue; 381 }; 382 383 void 384 IOTrackingQueueCollectUser(IOTrackingQueue * queue) 385 { 386 assert(0 == queue->siteCount); 387 queue->type |= kIOTrackingQueueTypeUser; 388 } 389 390 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 391 392 void 393 IOTrackingQueueFree(IOTrackingQueue * queue) 394 { 395 lck_mtx_lock(gIOTrackingLock); 396 IOTrackingReset(queue); 397 remque(&queue->link); 398 lck_mtx_unlock(gIOTrackingLock); 399 400 lck_mtx_free(queue->lock.mutex, IOLockGroup); 401 402 kfree_type(IOTrackingQueue, queue_head_t, queue->numSiteQs, queue); 403 }; 404 405 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 406 407 /* fasthash 408 * The MIT License 409 * 410 * Copyright (C) 2012 Zilong Tan ([email protected]) 411 * 412 * Permission is hereby granted, free of charge, to any person 413 * obtaining a copy of this software and associated documentation 414 * files (the "Software"), to deal in the Software without 415 * restriction, including without limitation the rights to use, copy, 416 * modify, merge, publish, distribute, sublicense, and/or sell copies 417 * of the Software, and to permit persons to whom the Software is 418 * furnished to do so, subject to the following conditions: 419 * 420 * The above copyright notice and this permission notice shall be 421 * included in all copies or substantial portions of the Software. 422 * 423 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 424 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 425 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 426 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 427 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 428 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 429 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 430 * SOFTWARE. 431 */ 432 433 434 // Compression function for Merkle-Damgard construction. 435 // This function is generated using the framework provided. 436 #define mix(h) ({ \ 437 (h) ^= (h) >> 23; \ 438 (h) *= 0x2127599bf4325c37ULL; \ 439 (h) ^= (h) >> 47; }) 440 441 static uint64_t 442 fasthash64(const void *buf, size_t len, uint64_t seed) 443 { 444 const uint64_t m = 0x880355f21e6d1965ULL; 445 const uint64_t *pos = (const uint64_t *)buf; 446 const uint64_t *end = pos + (len / 8); 447 const unsigned char *pos2; 448 uint64_t h = seed ^ (len * m); 449 uint64_t v; 450 451 while (pos != end) { 452 v = *pos++; 453 h ^= mix(v); 454 h *= m; 455 } 456 457 pos2 = (const unsigned char*)pos; 458 v = 0; 459 460 switch (len & 7) { 461 case 7: v ^= (uint64_t)pos2[6] << 48; 462 [[clang::fallthrough]]; 463 case 6: v ^= (uint64_t)pos2[5] << 40; 464 [[clang::fallthrough]]; 465 case 5: v ^= (uint64_t)pos2[4] << 32; 466 [[clang::fallthrough]]; 467 case 4: v ^= (uint64_t)pos2[3] << 24; 468 [[clang::fallthrough]]; 469 case 3: v ^= (uint64_t)pos2[2] << 16; 470 [[clang::fallthrough]]; 471 case 2: v ^= (uint64_t)pos2[1] << 8; 472 [[clang::fallthrough]]; 473 case 1: v ^= (uint64_t)pos2[0]; 474 h ^= mix(v); 475 h *= m; 476 } 477 478 return mix(h); 479 } 480 481 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 482 483 static uint32_t 484 fasthash32(const void *buf, size_t len, uint32_t seed) 485 { 486 // the following trick converts the 64-bit hashcode to Fermat 487 // residue, which shall retain information from both the higher 488 // and lower parts of hashcode. 489 uint64_t h = fasthash64(buf, len, seed); 490 return (uint32_t) (h - (h >> 32)); 491 } 492 493 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 494 495 void 496 IOTrackingAddUser(IOTrackingQueue * queue, IOTrackingUser * mem, vm_size_t size) 497 { 498 uint32_t num; 499 int pid; 500 501 if (!queue->captureOn) { 502 return; 503 } 504 if (size < queue->minCaptureSize) { 505 return; 506 } 507 508 assert(!mem->link.next); 509 510 num = backtrace(&mem->bt[0], kIOTrackingCallSiteBTs, NULL, NULL); 511 num = 0; 512 if ((kernel_task != current_task()) && (pid = proc_selfpid())) { 513 struct backtrace_user_info btinfo = BTUINFO_INIT; 514 mem->btPID = pid; 515 num = backtrace_user(&mem->btUser[0], kIOTrackingCallSiteBTs - 1, 516 NULL, &btinfo); 517 mem->user32 = !(btinfo.btui_info & BTI_64_BIT); 518 } 519 assert(num <= kIOTrackingCallSiteBTs); 520 static_assert(kIOTrackingCallSiteBTs <= UINT8_MAX); 521 mem->userCount = ((uint8_t) num); 522 523 IOTRecursiveLockLock(&queue->lock); 524 queue_enter/*last*/ (&queue->sites[0], mem, IOTrackingUser *, link); 525 queue->siteCount++; 526 IOTRecursiveLockUnlock(&queue->lock); 527 } 528 529 void 530 IOTrackingRemoveUser(IOTrackingQueue * queue, IOTrackingUser * mem) 531 { 532 if (!mem->link.next) { 533 return; 534 } 535 536 IOTRecursiveLockLock(&queue->lock); 537 if (mem->link.next) { 538 remque(&mem->link); 539 assert(queue->siteCount); 540 queue->siteCount--; 541 } 542 IOTRecursiveLockUnlock(&queue->lock); 543 } 544 545 uint64_t gIOTrackingAddTime; 546 547 void 548 IOTrackingAdd(IOTrackingQueue * queue, IOTracking * mem, size_t size, bool address, vm_tag_t tag) 549 { 550 IOTrackingCallSite * site; 551 uint32_t crc, num; 552 uintptr_t bt[kIOTrackingCallSiteBTs + 1]; 553 uintptr_t btUser[kIOTrackingCallSiteBTs]; 554 queue_head_t * que; 555 bool user; 556 int pid; 557 int userCount; 558 559 if (mem->site) { 560 return; 561 } 562 if (!queue->captureOn) { 563 return; 564 } 565 if (size < queue->minCaptureSize) { 566 return; 567 } 568 569 user = (0 != (kIOTrackingQueueTypeUser & queue->type)); 570 571 assert(!mem->link.next); 572 573 num = backtrace(&bt[0], kIOTrackingCallSiteBTs + 1, NULL, NULL); 574 if (!num) { 575 return; 576 } 577 num--; 578 crc = fasthash32(&bt[1], num * sizeof(bt[0]), 0x04C11DB7); 579 580 userCount = 0; 581 pid = 0; 582 backtrace_info_t btinfo = BTI_NONE; 583 if (user) { 584 if ((kernel_task != current_task()) && (pid = proc_selfpid())) { 585 struct backtrace_user_info btuinfo = BTUINFO_INIT; 586 userCount = backtrace_user(&btUser[0], kIOTrackingCallSiteBTs, 587 NULL, &btuinfo); 588 assert(userCount <= kIOTrackingCallSiteBTs); 589 btinfo = btuinfo.btui_info; 590 crc = fasthash32(&btUser[0], userCount * sizeof(bt[0]), crc); 591 } 592 } 593 594 IOTRecursiveLockLock(&queue->lock); 595 que = &queue->sites[crc % queue->numSiteQs]; 596 queue_iterate(que, site, IOTrackingCallSite *, link) 597 { 598 if (tag != site->tag) { 599 continue; 600 } 601 if (user && (pid != site->user[0].pid)) { 602 continue; 603 } 604 if (crc == site->crc) { 605 break; 606 } 607 } 608 609 if (queue_end(que, (queue_entry_t) site)) { 610 if (user) { 611 site = &kalloc_type(IOTrackingCallSiteWithUser, 612 Z_WAITOK_ZERO_NOFAIL)->site; 613 } else { 614 site = kalloc_type(IOTrackingCallSite, 615 Z_WAITOK_ZERO_NOFAIL); 616 } 617 618 queue_init(&site->instances); 619 site->addresses = NULL; 620 site->queue = queue; 621 site->crc = crc; 622 site->count = 0; 623 site->tag = tag; 624 memset(&site->size[0], 0, sizeof(site->size)); 625 bcopy(&bt[1], &site->bt[0], num * sizeof(site->bt[0])); 626 assert(num <= kIOTrackingCallSiteBTs); 627 bzero(&site->bt[num], (kIOTrackingCallSiteBTs - num) * sizeof(site->bt[0])); 628 if (user) { 629 bcopy(&btUser[0], &site->user[0].bt[0], userCount * sizeof(site->user[0].bt[0])); 630 assert(userCount <= kIOTrackingCallSiteBTs); 631 bzero(&site->user[0].bt[userCount], (kIOTrackingCallSiteBTs - userCount) * sizeof(site->user[0].bt[0])); 632 site->user[0].pid = pid; 633 site->user[0].user32 = !(btinfo & BTI_64_BIT); 634 static_assert(kIOTrackingCallSiteBTs <= UINT8_MAX); 635 site->user[0].userCount = ((uint8_t) userCount); 636 } 637 queue_enter_first(que, site, IOTrackingCallSite *, link); 638 queue->siteCount++; 639 } 640 641 if (address) { 642 IOTrackingAddress * memAddr = (typeof(memAddr))mem; 643 uint32_t hashIdx; 644 645 if (NULL == site->addresses) { 646 site->addresses = kalloc_type(IOTracking *, queue->numSiteQs, Z_WAITOK_ZERO_NOFAIL); 647 for (hashIdx = 0; hashIdx < queue->numSiteQs; hashIdx++) { 648 site->addresses[hashIdx] = (IOTracking *) &site->instances; 649 } 650 } 651 hashIdx = atop(memAddr->address) % queue->numSiteQs; 652 if (queue_end(&site->instances, (queue_entry_t)site->addresses[hashIdx])) { 653 queue_enter/*last*/ (&site->instances, mem, IOTracking *, link); 654 } else { 655 queue_insert_before(&site->instances, mem, site->addresses[hashIdx], IOTracking *, link); 656 } 657 site->addresses[hashIdx] = mem; 658 } else { 659 queue_enter_first(&site->instances, mem, IOTracking *, link); 660 } 661 662 mem->site = site; 663 site->size[0] += size; 664 site->count++; 665 666 IOTRecursiveLockUnlock(&queue->lock); 667 } 668 669 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 670 671 static void 672 IOTrackingRemoveInternal(IOTrackingQueue * queue, IOTracking * mem, size_t size, uint32_t addressIdx) 673 { 674 IOTrackingCallSite * site; 675 IOTrackingAddress * nextAddress; 676 677 if (!mem->link.next) { 678 return; 679 } 680 681 IOTRecursiveLockLock(&queue->lock); 682 if (mem->link.next) { 683 assert(mem->site); 684 site = mem->site; 685 686 if ((-1U != addressIdx) && (mem == site->addresses[addressIdx])) { 687 nextAddress = (IOTrackingAddress *) queue_next(&mem->link); 688 if (!queue_end(&site->instances, &nextAddress->tracking.link) 689 && (addressIdx != (atop(nextAddress->address) % queue->numSiteQs))) { 690 nextAddress = (IOTrackingAddress *) &site->instances; 691 } 692 site->addresses[addressIdx] = &nextAddress->tracking; 693 } 694 695 remque(&mem->link); 696 assert(site->count); 697 site->count--; 698 assert(site->size[0] >= size); 699 site->size[0] -= size; 700 if (!site->count) { 701 assert(queue_empty(&site->instances)); 702 assert(!site->size[0]); 703 assert(!site->size[1]); 704 705 remque(&site->link); 706 assert(queue->siteCount); 707 queue->siteCount--; 708 IOTrackingFreeCallSite(queue->type, &site); 709 } 710 mem->site = NULL; 711 } 712 IOTRecursiveLockUnlock(&queue->lock); 713 } 714 715 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 716 717 void 718 IOTrackingRemove(IOTrackingQueue * queue, IOTracking * mem, size_t size) 719 { 720 return IOTrackingRemoveInternal(queue, mem, size, -1U); 721 } 722 723 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 724 725 void 726 IOTrackingRemoveAddress(IOTrackingQueue * queue, IOTrackingAddress * mem, size_t size) 727 { 728 uint32_t addressIdx; 729 uint64_t address; 730 731 address = mem->address; 732 addressIdx = atop(address) % queue->numSiteQs; 733 734 return IOTrackingRemoveInternal(queue, &mem->tracking, size, addressIdx); 735 } 736 737 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 738 739 void 740 IOTrackingAlloc(IOTrackingQueue * queue, uintptr_t address, size_t size) 741 { 742 IOTrackingAddress * tracking; 743 744 if (!queue->captureOn) { 745 return; 746 } 747 if (size < queue->minCaptureSize) { 748 return; 749 } 750 751 address = ~address; 752 tracking = kalloc_type(IOTrackingAddress, (zalloc_flags_t)(Z_WAITOK | Z_ZERO)); 753 IOTrackingAddressFlags(tracking) |= kTrackingAddressFlagAllocated; 754 tracking->address = address; 755 tracking->size = size; 756 757 IOTrackingAdd(queue, &tracking->tracking, size, true, VM_KERN_MEMORY_NONE); 758 } 759 760 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 761 762 void 763 IOTrackingFree(IOTrackingQueue * queue, uintptr_t address, size_t size) 764 { 765 IOTrackingCallSite * site; 766 IOTrackingAddress * tracking; 767 IOTrackingAddress * nextAddress; 768 uint32_t idx, hashIdx; 769 bool done; 770 771 address = ~address; 772 IOTRecursiveLockLock(&queue->lock); 773 774 hashIdx = atop(address) % queue->numSiteQs; 775 776 done = false; 777 for (idx = 0; idx < queue->numSiteQs; idx++) { 778 queue_iterate(&queue->sites[idx], site, IOTrackingCallSite *, link) 779 { 780 if (!site->addresses) { 781 continue; 782 } 783 tracking = (IOTrackingAddress *) site->addresses[hashIdx]; 784 while (!queue_end(&site->instances, &tracking->tracking.link)) { 785 nextAddress = (IOTrackingAddress *) queue_next(&tracking->tracking.link); 786 if (!queue_end(&site->instances, &nextAddress->tracking.link) 787 && (hashIdx != (atop(nextAddress->address) % queue->numSiteQs))) { 788 nextAddress = (IOTrackingAddress *) &site->instances; 789 } 790 if ((done = (address == tracking->address))) { 791 if (tracking == (IOTrackingAddress *) site->addresses[hashIdx]) { 792 site->addresses[hashIdx] = &nextAddress->tracking; 793 } 794 IOTrackingRemoveInternal(queue, &tracking->tracking, size, -1U); 795 kfree_type(IOTrackingAddress, tracking); 796 break; 797 } 798 tracking = nextAddress; 799 } 800 if (done) { 801 break; 802 } 803 } 804 if (done) { 805 break; 806 } 807 } 808 IOTRecursiveLockUnlock(&queue->lock); 809 } 810 811 static void 812 IOTrackingFreeCallSite(uint32_t type, IOTrackingCallSite ** pSite) 813 { 814 IOTrackingCallSite * site; 815 void ** ptr; 816 817 site = *pSite; 818 kfree_type(IOTracking *, site->queue->numSiteQs, site->addresses); 819 820 ptr = reinterpret_cast<void **>(pSite); 821 if (kIOTrackingQueueTypeUser & type) { 822 kfree_type(IOTrackingCallSiteWithUser, *ptr); 823 } else { 824 kfree_type(IOTrackingCallSite, *ptr); 825 } 826 } 827 828 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 829 830 void 831 IOTrackingAccumSize(IOTrackingQueue * queue, IOTracking * mem, size_t size) 832 { 833 IOTRecursiveLockLock(&queue->lock); 834 if (mem->link.next) { 835 assert(mem->site); 836 assert((size > 0) || (mem->site->size[1] >= -size)); 837 mem->site->size[1] += size; 838 } 839 ; 840 IOTRecursiveLockUnlock(&queue->lock); 841 } 842 843 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 844 845 void 846 IOTrackingReset(IOTrackingQueue * queue) 847 { 848 IOTrackingCallSite * site; 849 IOTrackingUser * user; 850 IOTracking * tracking; 851 IOTrackingAddress * trackingAddress; 852 uint32_t idx, hashIdx; 853 bool addresses; 854 855 IOTRecursiveLockLock(&queue->lock); 856 for (idx = 0; idx < queue->numSiteQs; idx++) { 857 while (!queue_empty(&queue->sites[idx])) { 858 if (kIOTrackingQueueTypeMap & queue->type) { 859 queue_remove_first(&queue->sites[idx], user, IOTrackingUser *, link); 860 user->link.next = user->link.prev = NULL; 861 } else { 862 queue_remove_first(&queue->sites[idx], site, IOTrackingCallSite *, link); 863 addresses = false; 864 while (!queue_empty(&site->instances)) { 865 queue_remove_first(&site->instances, tracking, IOTracking *, link); 866 if (site->addresses) { 867 for (hashIdx = 0; !addresses && (hashIdx < queue->numSiteQs); hashIdx++) { 868 if (tracking == site->addresses[hashIdx]) { 869 addresses = true; 870 } 871 } 872 } 873 if (addresses) { 874 trackingAddress = (typeof(trackingAddress))tracking; 875 if (kTrackingAddressFlagAllocated & IOTrackingAddressFlags(trackingAddress)) { 876 kfree_type(IOTrackingAddress, trackingAddress); 877 } 878 } 879 } 880 IOTrackingFreeCallSite(queue->type, &site); 881 } 882 } 883 } 884 queue->siteCount = 0; 885 IOTRecursiveLockUnlock(&queue->lock); 886 } 887 888 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 889 890 static int 891 IOTrackingCallSiteInfoCompare(const void * left, const void * right) 892 { 893 IOTrackingCallSiteInfo * l = (typeof(l))left; 894 IOTrackingCallSiteInfo * r = (typeof(r))right; 895 size_t lsize, rsize; 896 897 rsize = r->size[0] + r->size[1]; 898 lsize = l->size[0] + l->size[1]; 899 900 return (rsize > lsize) ? 1 : ((rsize == lsize) ? 0 : -1); 901 } 902 903 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 904 905 static int 906 IOTrackingAddressCompare(const void * left, const void * right) 907 { 908 IOTracking * instance; 909 uintptr_t inst, laddr, raddr; 910 911 inst = ((typeof(inst) *)left)[0]; 912 instance = (typeof(instance))INSTANCE_GET(inst); 913 if (kInstanceFlagAddress & inst) { 914 laddr = ~((IOTrackingAddress *)instance)->address; 915 } else { 916 laddr = (uintptr_t) (instance + 1); 917 } 918 919 inst = ((typeof(inst) *)right)[0]; 920 instance = (typeof(instance))(inst & ~kInstanceFlags); 921 if (kInstanceFlagAddress & inst) { 922 raddr = ~((IOTrackingAddress *)instance)->address; 923 } else { 924 raddr = (uintptr_t) (instance + 1); 925 } 926 927 return (laddr > raddr) ? 1 : ((laddr == raddr) ? 0 : -1); 928 } 929 930 931 static int 932 IOTrackingZoneElementCompare(const void * left, const void * right) 933 { 934 uintptr_t inst, laddr, raddr; 935 936 inst = ((typeof(inst) *)left)[0]; 937 laddr = INSTANCE_PUT(inst); 938 inst = ((typeof(inst) *)right)[0]; 939 raddr = INSTANCE_PUT(inst); 940 941 return (laddr > raddr) ? 1 : ((laddr == raddr) ? 0 : -1); 942 } 943 944 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 945 946 static void 947 CopyOutBacktraces(IOTrackingCallSite * site, IOTrackingCallSiteInfo * siteInfo) 948 { 949 uint32_t j; 950 mach_vm_address_t bt, btEntry; 951 952 btEntry = site->queue->btEntry; 953 for (j = 0; j < kIOTrackingCallSiteBTs; j++) { 954 bt = site->bt[j]; 955 if (btEntry 956 && (!bt || (j == (kIOTrackingCallSiteBTs - 1)))) { 957 bt = btEntry; 958 btEntry = 0; 959 } 960 siteInfo->bt[0][j] = VM_KERNEL_UNSLIDE(bt); 961 } 962 963 siteInfo->btPID = 0; 964 if (kIOTrackingQueueTypeUser & site->queue->type) { 965 siteInfo->btPID = site->user[0].pid; 966 uint32_t * bt32 = (typeof(bt32))((void *) &site->user[0].bt[0]); 967 uint64_t * bt64 = (typeof(bt64))((void *) &site->user[0].bt[0]); 968 for (uint32_t j = 0; j < kIOTrackingCallSiteBTs; j++) { 969 if (j >= site->user[0].userCount) { 970 siteInfo->bt[1][j] = 0; 971 } else if (site->user[0].user32) { 972 siteInfo->bt[1][j] = bt32[j]; 973 } else { 974 siteInfo->bt[1][j] = bt64[j]; 975 } 976 } 977 } 978 } 979 980 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 981 982 static void 983 IOTrackingLeakScan(void * refcon) 984 { 985 IOTrackingLeaksRef * ref = (typeof(ref))refcon; 986 uintptr_t * instances; 987 IOTracking * instance; 988 uint64_t vaddr, vincr; 989 ppnum_t ppn; 990 uintptr_t ptr, addr, vphysaddr, inst; 991 size_t size, origsize; 992 uint32_t baseIdx, lim, ptrIdx, count; 993 boolean_t is; 994 AbsoluteTime deadline; 995 996 instances = ref->instances; 997 count = ref->count; 998 size = origsize = ref->zoneSize; 999 1000 if (gIOTrackingLeakScanCallback) { 1001 gIOTrackingLeakScanCallback(kIOTrackingLeakScanStart); 1002 } 1003 1004 for (deadline = 0, vaddr = VM_MIN_KERNEL_AND_KEXT_ADDRESS; 1005 ; 1006 vaddr += vincr) { 1007 if ((mach_absolute_time() > deadline) || (vaddr >= VM_MAX_KERNEL_ADDRESS)) { 1008 if (deadline) { 1009 #if SCHED_HYGIENE_DEBUG 1010 if (is) { 1011 // Reset the interrupt timeout to avoid panics 1012 ml_spin_debug_clear_self(); 1013 } 1014 #endif /* SCHED_HYGIENE_DEBUG */ 1015 ml_set_interrupts_enabled(is); 1016 IODelay(10); 1017 } 1018 if (vaddr >= VM_MAX_KERNEL_ADDRESS) { 1019 break; 1020 } 1021 is = ml_set_interrupts_enabled(false); 1022 clock_interval_to_deadline(10, kMillisecondScale, &deadline); 1023 } 1024 1025 ppn = kernel_pmap_present_mapping(vaddr, &vincr, &vphysaddr); 1026 // check noencrypt to avoid VM structs (map entries) with pointers 1027 if (ppn && (!pmap_valid_page(ppn) || (!ref->zoneSize && pmap_is_noencrypt(ppn)))) { 1028 ppn = 0; 1029 } 1030 if (!ppn) { 1031 continue; 1032 } 1033 1034 for (ptrIdx = 0; ptrIdx < (page_size / sizeof(uintptr_t)); ptrIdx++) { 1035 ptr = ((uintptr_t *)vphysaddr)[ptrIdx]; 1036 #if defined(HAS_APPLE_PAC) 1037 // strip possible ptrauth signature from candidate data pointer 1038 ptr = (uintptr_t)ptrauth_strip((void*)ptr, ptrauth_key_process_independent_data); 1039 #endif /* defined(HAS_APPLE_PAC) */ 1040 1041 for (lim = count, baseIdx = 0; lim; lim >>= 1) { 1042 inst = instances[baseIdx + (lim >> 1)]; 1043 instance = (typeof(instance))INSTANCE_GET(inst); 1044 1045 if (ref->zoneSize) { 1046 addr = INSTANCE_PUT(inst) & ~kInstanceFlags; 1047 } else if (kInstanceFlagAddress & inst) { 1048 addr = ~((IOTrackingAddress *)instance)->address; 1049 origsize = size = ((IOTrackingAddress *)instance)->size; 1050 if (!size) { 1051 size = 1; 1052 } 1053 } else { 1054 addr = (uintptr_t) (instance + 1); 1055 origsize = size = instance->site->queue->allocSize; 1056 } 1057 if ((ptr >= addr) && (ptr < (addr + size)) 1058 1059 && (((vaddr + ptrIdx * sizeof(uintptr_t)) < addr) 1060 || ((vaddr + ptrIdx * sizeof(uintptr_t)) >= (addr + size)))) { 1061 if (!(kInstanceFlagReferenced & inst)) { 1062 inst |= kInstanceFlagReferenced; 1063 instances[baseIdx + (lim >> 1)] = inst; 1064 ref->found++; 1065 if (!origsize) { 1066 ref->foundzlen++; 1067 } 1068 } 1069 break; 1070 } 1071 if (ptr > addr) { 1072 // move right 1073 baseIdx += (lim >> 1) + 1; 1074 lim--; 1075 } 1076 // else move left 1077 } 1078 } 1079 ref->bytes += page_size; 1080 } 1081 1082 if (gIOTrackingLeakScanCallback) { 1083 gIOTrackingLeakScanCallback(kIOTrackingLeakScanEnd); 1084 } 1085 } 1086 1087 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1088 1089 extern "C" void 1090 zone_leaks_scan(uintptr_t * instances, uint32_t count, uint32_t zoneSize, uint32_t * found) 1091 { 1092 IOTrackingLeaksRef ref; 1093 IOTrackingCallSiteInfo siteInfo; 1094 uint32_t idx; 1095 1096 qsort(instances, count, sizeof(*instances), &IOTrackingZoneElementCompare); 1097 1098 bzero(&siteInfo, sizeof(siteInfo)); 1099 bzero(&ref, sizeof(ref)); 1100 ref.instances = instances; 1101 ref.count = count; 1102 ref.zoneSize = zoneSize; 1103 1104 for (idx = 0; idx < 2; idx++) { 1105 ref.bytes = 0; 1106 IOTrackingLeakScan(&ref); 1107 IOLog("leaks(%d) scanned %ld MB, instance count %d, found %d\n", idx, ref.bytes / 1024 / 1024, count, ref.found); 1108 if (count <= ref.found) { 1109 break; 1110 } 1111 } 1112 1113 *found = ref.found; 1114 } 1115 1116 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1117 1118 static OSData * 1119 IOTrackingLeaks(LIBKERN_CONSUMED OSData * data) 1120 { 1121 IOTrackingLeaksRef ref; 1122 IOTrackingCallSiteInfo siteInfo; 1123 IOTrackingCallSite * site; 1124 OSData * leakData; 1125 uintptr_t * instances; 1126 IOTracking * instance; 1127 uintptr_t inst; 1128 uint32_t count, idx, numSites, dups, siteCount; 1129 1130 instances = (typeof(instances))data->getBytesNoCopy(); 1131 count = (data->getLength() / sizeof(*instances)); 1132 qsort(instances, count, sizeof(*instances), &IOTrackingAddressCompare); 1133 1134 bzero(&siteInfo, sizeof(siteInfo)); 1135 bzero(&ref, sizeof(ref)); 1136 ref.instances = instances; 1137 ref.count = count; 1138 for (idx = 0; idx < 2; idx++) { 1139 ref.bytes = 0; 1140 IOTrackingLeakScan(&ref); 1141 IOLog("leaks(%d) scanned %ld MB, instance count %d, found %d (zlen %d)\n", idx, ref.bytes / 1024 / 1024, count, ref.found, ref.foundzlen); 1142 if (count <= ref.found) { 1143 break; 1144 } 1145 } 1146 1147 leakData = OSData::withCapacity(128 * sizeof(IOTrackingCallSiteInfo)); 1148 1149 for (numSites = 0, idx = 0; idx < count; idx++) { 1150 inst = instances[idx]; 1151 if (kInstanceFlagReferenced & inst) { 1152 continue; 1153 } 1154 instance = (typeof(instance))INSTANCE_GET(inst); 1155 site = instance->site; 1156 instances[numSites] = (uintptr_t) site; 1157 numSites++; 1158 } 1159 1160 for (idx = 0; idx < numSites; idx++) { 1161 inst = instances[idx]; 1162 if (!inst) { 1163 continue; 1164 } 1165 site = (typeof(site))inst; 1166 for (siteCount = 1, dups = (idx + 1); dups < numSites; dups++) { 1167 if (instances[dups] == (uintptr_t) site) { 1168 siteCount++; 1169 instances[dups] = 0; 1170 } 1171 } 1172 siteInfo.count = siteCount; 1173 siteInfo.size[0] = (site->size[0] * site->count) / siteCount; 1174 siteInfo.size[1] = (site->size[1] * site->count) / siteCount; 1175 CopyOutBacktraces(site, &siteInfo); 1176 leakData->appendBytes(&siteInfo, sizeof(siteInfo)); 1177 } 1178 data->release(); 1179 1180 return leakData; 1181 } 1182 1183 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1184 1185 static bool 1186 SkipName(uint32_t options, const char * name, size_t namesLen, const char * names) 1187 { 1188 const char * scan; 1189 const char * next; 1190 bool exclude, found; 1191 size_t qLen, sLen; 1192 1193 if (!namesLen || !names) { 1194 return false; 1195 } 1196 // <len><name>...<len><name><0> 1197 exclude = (0 != (kIOTrackingExcludeNames & options)); 1198 qLen = strlen(name); 1199 scan = names; 1200 found = false; 1201 do{ 1202 sLen = scan[0]; 1203 scan++; 1204 next = scan + sLen; 1205 if (next >= (names + namesLen)) { 1206 break; 1207 } 1208 found = ((sLen == qLen) && !strncmp(scan, name, sLen)); 1209 scan = next; 1210 }while (!found && (scan < (names + namesLen))); 1211 1212 return !(exclude ^ found); 1213 } 1214 1215 #endif /* IOTRACKING */ 1216 1217 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1218 1219 static kern_return_t 1220 IOTrackingDebug(uint32_t selector, uint32_t options, uint64_t value, 1221 uint32_t intag, uint32_t inzsize, 1222 const char * names, size_t namesLen, 1223 size_t size, OSObject ** result) 1224 { 1225 kern_return_t ret; 1226 OSData * data; 1227 1228 if (result) { 1229 *result = NULL; 1230 } 1231 data = NULL; 1232 ret = kIOReturnNotReady; 1233 1234 #if IOTRACKING 1235 1236 kern_return_t kr; 1237 IOTrackingQueue * queue; 1238 IOTracking * instance; 1239 IOTrackingCallSite * site; 1240 IOTrackingCallSiteInfo siteInfo; 1241 IOTrackingUser * user; 1242 task_t mapTask; 1243 mach_vm_address_t mapAddress; 1244 mach_vm_size_t mapSize; 1245 uint32_t num, idx, qIdx; 1246 uintptr_t instFlags; 1247 proc_t proc; 1248 bool addresses; 1249 1250 ret = kIOReturnNotFound; 1251 proc = NULL; 1252 if (kIOTrackingGetMappings == selector) { 1253 if (value != -1ULL) { 1254 proc = proc_find((pid_t) value); 1255 if (!proc) { 1256 return kIOReturnNotFound; 1257 } 1258 } 1259 } 1260 1261 bzero(&siteInfo, sizeof(siteInfo)); 1262 lck_mtx_lock(gIOTrackingLock); 1263 queue_iterate(&gIOTrackingQ, queue, IOTrackingQueue *, link) 1264 { 1265 if (SkipName(options, queue->name, namesLen, names)) { 1266 continue; 1267 } 1268 1269 if (!(kIOTracking & gIOKitDebug) && (kIOTrackingQueueTypeAlloc & queue->type)) { 1270 continue; 1271 } 1272 1273 switch (selector) { 1274 case kIOTrackingResetTracking: 1275 { 1276 IOTrackingReset(queue); 1277 ret = kIOReturnSuccess; 1278 break; 1279 } 1280 1281 case kIOTrackingStartCapture: 1282 case kIOTrackingStopCapture: 1283 { 1284 queue->captureOn = (kIOTrackingStartCapture == selector); 1285 ret = kIOReturnSuccess; 1286 break; 1287 } 1288 1289 case kIOTrackingSetMinCaptureSize: 1290 { 1291 queue->minCaptureSize = size; 1292 ret = kIOReturnSuccess; 1293 break; 1294 } 1295 1296 case kIOTrackingLeaks: 1297 { 1298 if (!(kIOTrackingQueueTypeAlloc & queue->type)) { 1299 break; 1300 } 1301 1302 if (!data) { 1303 data = OSData::withCapacity(1024 * sizeof(uintptr_t)); 1304 } 1305 1306 IOTRecursiveLockLock(&queue->lock); 1307 for (idx = 0; idx < queue->numSiteQs; idx++) { 1308 queue_iterate(&queue->sites[idx], site, IOTrackingCallSite *, link) 1309 { 1310 addresses = false; 1311 queue_iterate(&site->instances, instance, IOTracking *, link) 1312 { 1313 if (site->addresses) { 1314 for (uint32_t hashIdx = 0; !addresses && (hashIdx < queue->numSiteQs); hashIdx++) { 1315 if (instance == site->addresses[hashIdx]) { 1316 addresses = true; 1317 } 1318 } 1319 } 1320 instFlags = (typeof(instFlags))instance; 1321 if (addresses) { 1322 instFlags |= kInstanceFlagAddress; 1323 } 1324 data->appendValue(instFlags); 1325 } 1326 } 1327 } 1328 // queue is locked 1329 ret = kIOReturnSuccess; 1330 break; 1331 } 1332 1333 1334 case kIOTrackingGetTracking: 1335 { 1336 if (kIOTrackingQueueTypeMap & queue->type) { 1337 break; 1338 } 1339 1340 if (!data) { 1341 data = OSData::withCapacity(128 * sizeof(IOTrackingCallSiteInfo)); 1342 } 1343 1344 IOTRecursiveLockLock(&queue->lock); 1345 num = queue->siteCount; 1346 idx = 0; 1347 for (qIdx = 0; qIdx < queue->numSiteQs; qIdx++) { 1348 queue_iterate(&queue->sites[qIdx], site, IOTrackingCallSite *, link) 1349 { 1350 assert(idx < num); 1351 idx++; 1352 1353 size_t tsize[2]; 1354 uint32_t count = site->count; 1355 tsize[0] = site->size[0]; 1356 tsize[1] = site->size[1]; 1357 1358 if (intag || inzsize) { 1359 uintptr_t addr; 1360 vm_size_t size, zoneSize; 1361 vm_tag_t tag; 1362 1363 if (kIOTrackingQueueTypeAlloc & queue->type) { 1364 addresses = false; 1365 count = 0; 1366 tsize[0] = tsize[1] = 0; 1367 queue_iterate(&site->instances, instance, IOTracking *, link) 1368 { 1369 if (site->addresses) { 1370 for (uint32_t hashIdx = 0; !addresses && (hashIdx < queue->numSiteQs); hashIdx++) { 1371 if (instance == site->addresses[hashIdx]) { 1372 addresses = true; 1373 } 1374 } 1375 } 1376 1377 if (addresses) { 1378 addr = ~((IOTrackingAddress *)instance)->address; 1379 } else { 1380 addr = (uintptr_t) (instance + 1); 1381 } 1382 1383 kr = vm_kern_allocation_info(addr, &size, &tag, &zoneSize); 1384 if (KERN_SUCCESS != kr) { 1385 continue; 1386 } 1387 1388 if ((VM_KERN_MEMORY_NONE != intag) && (intag != tag)) { 1389 continue; 1390 } 1391 if (inzsize && (inzsize != zoneSize)) { 1392 continue; 1393 } 1394 1395 count++; 1396 tsize[0] += size; 1397 } 1398 } else { 1399 if (!intag || inzsize || (intag != site->tag)) { 1400 continue; 1401 } 1402 } 1403 } 1404 1405 if (!count) { 1406 continue; 1407 } 1408 if (size && ((tsize[0] + tsize[1]) < size)) { 1409 continue; 1410 } 1411 siteInfo.count = count; 1412 siteInfo.size[0] = tsize[0]; 1413 siteInfo.size[1] = tsize[1]; 1414 CopyOutBacktraces(site, &siteInfo); 1415 data->appendBytes(&siteInfo, sizeof(siteInfo)); 1416 } 1417 } 1418 assert(idx == num); 1419 IOTRecursiveLockUnlock(&queue->lock); 1420 ret = kIOReturnSuccess; 1421 break; 1422 } 1423 1424 case kIOTrackingGetMappings: 1425 { 1426 if (!(kIOTrackingQueueTypeMap & queue->type)) { 1427 break; 1428 } 1429 if (!data) { 1430 data = OSData::withCapacity((unsigned int) page_size); 1431 } 1432 1433 IOTRecursiveLockLock(&queue->lock); 1434 num = queue->siteCount; 1435 idx = 0; 1436 for (qIdx = 0; qIdx < queue->numSiteQs; qIdx++) { 1437 queue_iterate(&queue->sites[qIdx], user, IOTrackingUser *, link) 1438 { 1439 assert(idx < num); 1440 idx++; 1441 1442 kr = IOMemoryMapTracking(user, &mapTask, &mapAddress, &mapSize); 1443 if (kIOReturnSuccess != kr) { 1444 continue; 1445 } 1446 if (proc && (mapTask != proc_task(proc))) { 1447 continue; 1448 } 1449 if (size && (mapSize < size)) { 1450 continue; 1451 } 1452 1453 siteInfo.count = 1; 1454 siteInfo.size[0] = mapSize; 1455 siteInfo.address = mapAddress; 1456 siteInfo.addressPID = task_pid(mapTask); 1457 siteInfo.btPID = user->btPID; 1458 1459 for (uint32_t j = 0; j < kIOTrackingCallSiteBTs; j++) { 1460 siteInfo.bt[0][j] = VM_KERNEL_UNSLIDE(user->bt[j]); 1461 } 1462 uint32_t * bt32 = (typeof(bt32)) & user->btUser[0]; 1463 uint64_t * bt64 = (typeof(bt64))((void *) &user->btUser[0]); 1464 for (uint32_t j = 0; j < kIOTrackingCallSiteBTs; j++) { 1465 if (j >= user->userCount) { 1466 siteInfo.bt[1][j] = 0; 1467 } else if (user->user32) { 1468 siteInfo.bt[1][j] = bt32[j]; 1469 } else { 1470 siteInfo.bt[1][j] = bt64[j]; 1471 } 1472 } 1473 data->appendBytes(&siteInfo, sizeof(siteInfo)); 1474 } 1475 } 1476 assert(idx == num); 1477 IOTRecursiveLockUnlock(&queue->lock); 1478 ret = kIOReturnSuccess; 1479 break; 1480 } 1481 1482 default: 1483 ret = kIOReturnUnsupported; 1484 break; 1485 } 1486 } 1487 1488 if ((kIOTrackingLeaks == selector) && data) { 1489 data = IOTrackingLeaks(data); 1490 queue_iterate(&gIOTrackingQ, queue, IOTrackingQueue *, link) 1491 { 1492 if (SkipName(options, queue->name, namesLen, names)) { 1493 continue; 1494 } 1495 if (!(kIOTrackingQueueTypeAlloc & queue->type)) { 1496 continue; 1497 } 1498 IOTRecursiveLockUnlock(&queue->lock); 1499 } 1500 } 1501 1502 lck_mtx_unlock(gIOTrackingLock); 1503 1504 if ((kIOTrackingLeaks == selector) && namesLen && names) { 1505 const char * scan; 1506 const char * next; 1507 uint8_t sLen; 1508 1509 if (!data) { 1510 data = OSData::withCapacity(4096 * sizeof(uintptr_t)); 1511 } 1512 1513 // <len><name>...<len><name><0> 1514 scan = names; 1515 do{ 1516 sLen = ((uint8_t) scan[0]); 1517 scan++; 1518 next = scan + sLen; 1519 if (next >= (names + namesLen)) { 1520 break; 1521 } 1522 kr = zone_leaks(scan, sLen, ^(uint32_t count, uint32_t eSize, btref_t ref) { 1523 IOTrackingCallSiteInfo siteInfo = { 1524 .count = count, 1525 .size[0] = eSize * count, 1526 }; 1527 1528 btref_decode_unslide(ref, siteInfo.bt[0]); 1529 1530 data->appendBytes(&siteInfo, sizeof(siteInfo)); 1531 }); 1532 if (KERN_SUCCESS == kr) { 1533 ret = kIOReturnSuccess; 1534 } else if (KERN_INVALID_NAME != kr) { 1535 ret = kIOReturnVMError; 1536 } 1537 scan = next; 1538 }while (scan < (names + namesLen)); 1539 } 1540 1541 if (data) { 1542 switch (selector) { 1543 case kIOTrackingLeaks: 1544 case kIOTrackingGetTracking: 1545 case kIOTrackingGetMappings: 1546 { 1547 IOTrackingCallSiteInfo * siteInfos; 1548 siteInfos = (typeof(siteInfos))data->getBytesNoCopy(); 1549 num = (data->getLength() / sizeof(*siteInfos)); 1550 qsort(siteInfos, num, sizeof(*siteInfos), &IOTrackingCallSiteInfoCompare); 1551 break; 1552 } 1553 default: assert(false); break; 1554 } 1555 } 1556 1557 *result = data; 1558 if (proc) { 1559 proc_rele(proc); 1560 } 1561 1562 #endif /* IOTRACKING */ 1563 1564 return ret; 1565 } 1566 1567 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1568 1569 #include <IOKit/IOKitDiagnosticsUserClient.h> 1570 1571 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1572 1573 #undef super 1574 #define super IOUserClient2022 1575 1576 OSDefineMetaClassAndStructors(IOKitDiagnosticsClient, IOUserClient2022) 1577 1578 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1579 1580 IOUserClient * IOKitDiagnosticsClient::withTask(task_t owningTask) 1581 { 1582 #if IOTRACKING 1583 IOKitDiagnosticsClient * inst; 1584 1585 inst = new IOKitDiagnosticsClient; 1586 if (inst && !inst->init()) { 1587 inst->release(); 1588 inst = NULL; 1589 } 1590 1591 inst->setProperty(kIOUserClientDefaultLockingKey, kOSBooleanTrue); 1592 inst->setProperty(kIOUserClientDefaultLockingSetPropertiesKey, kOSBooleanTrue); 1593 inst->setProperty(kIOUserClientDefaultLockingSingleThreadExternalMethodKey, kOSBooleanTrue); 1594 1595 inst->setProperty(kIOUserClientEntitlementsKey, kOSBooleanFalse); 1596 1597 return inst; 1598 #else 1599 return NULL; 1600 #endif 1601 } 1602 1603 IOReturn 1604 IOKitDiagnosticsClient::clientClose(void) 1605 { 1606 terminate(); 1607 return kIOReturnSuccess; 1608 } 1609 1610 IOReturn 1611 IOKitDiagnosticsClient::setProperties(OSObject * properties) 1612 { 1613 IOReturn kr = kIOReturnUnsupported; 1614 return kr; 1615 } 1616 1617 1618 IOReturn 1619 IOTrackingMethodDispatched(OSObject * target, void * reference, 1620 IOExternalMethodArguments * args) 1621 { 1622 IOReturn ret = kIOReturnBadArgument; 1623 const IOKitDiagnosticsParameters * params; 1624 const char * names; 1625 size_t namesLen; 1626 OSObject * result; 1627 1628 if (args->structureInputSize < sizeof(IOKitDiagnosticsParameters)) { 1629 return kIOReturnBadArgument; 1630 } 1631 params = (typeof(params))args->structureInput; 1632 if (!params) { 1633 return kIOReturnBadArgument; 1634 } 1635 1636 names = NULL; 1637 namesLen = args->structureInputSize - sizeof(IOKitDiagnosticsParameters); 1638 if (namesLen) { 1639 names = (typeof(names))(params + 1); 1640 } 1641 1642 ret = IOTrackingDebug(args->selector, params->options, params->value, params->tag, params->zsize, names, namesLen, params->size, &result); 1643 if ((kIOReturnSuccess == ret) && args->structureVariableOutputData) { 1644 *args->structureVariableOutputData = result; 1645 } else if (result) { 1646 result->release(); 1647 } 1648 return ret; 1649 } 1650 1651 IOReturn 1652 IOKitDiagnosticsClient::externalMethod(uint32_t selector, IOExternalMethodArgumentsOpaque * args) 1653 { 1654 static const IOExternalMethodDispatch2022 dispatchArray[] = { 1655 [kIOTrackingGetTracking] = { 1656 .function = &IOTrackingMethodDispatched, 1657 .checkScalarInputCount = 0, 1658 .checkStructureInputSize = kIOUCVariableStructureSize, 1659 .checkScalarOutputCount = 0, 1660 .checkStructureOutputSize = 0, 1661 .allowAsync = false, 1662 .checkEntitlement = NULL, 1663 }, 1664 [kIOTrackingGetMappings] = { 1665 .function = &IOTrackingMethodDispatched, 1666 .checkScalarInputCount = 0, 1667 .checkStructureInputSize = kIOUCVariableStructureSize, 1668 .checkScalarOutputCount = 0, 1669 .checkStructureOutputSize = 0, 1670 .allowAsync = false, 1671 .checkEntitlement = NULL, 1672 }, 1673 [kIOTrackingResetTracking] = { 1674 .function = &IOTrackingMethodDispatched, 1675 .checkScalarInputCount = 0, 1676 .checkStructureInputSize = kIOUCVariableStructureSize, 1677 .checkScalarOutputCount = 0, 1678 .checkStructureOutputSize = 0, 1679 .allowAsync = false, 1680 .checkEntitlement = NULL, 1681 }, 1682 [kIOTrackingStartCapture] = { 1683 .function = &IOTrackingMethodDispatched, 1684 .checkScalarInputCount = 0, 1685 .checkStructureInputSize = kIOUCVariableStructureSize, 1686 .checkScalarOutputCount = 0, 1687 .checkStructureOutputSize = 0, 1688 .allowAsync = false, 1689 .checkEntitlement = NULL, 1690 }, 1691 [kIOTrackingStopCapture] = { 1692 .function = &IOTrackingMethodDispatched, 1693 .checkScalarInputCount = 0, 1694 .checkStructureInputSize = kIOUCVariableStructureSize, 1695 .checkScalarOutputCount = 0, 1696 .checkStructureOutputSize = 0, 1697 .allowAsync = false, 1698 .checkEntitlement = NULL, 1699 }, 1700 [kIOTrackingSetMinCaptureSize] = { 1701 .function = &IOTrackingMethodDispatched, 1702 .checkScalarInputCount = 0, 1703 .checkStructureInputSize = kIOUCVariableStructureSize, 1704 .checkScalarOutputCount = 0, 1705 .checkStructureOutputSize = 0, 1706 .allowAsync = false, 1707 .checkEntitlement = NULL, 1708 }, 1709 [kIOTrackingLeaks] = { 1710 .function = &IOTrackingMethodDispatched, 1711 .checkScalarInputCount = 0, 1712 .checkStructureInputSize = kIOUCVariableStructureSize, 1713 .checkScalarOutputCount = 0, 1714 .checkStructureOutputSize = 0, 1715 .allowAsync = false, 1716 .checkEntitlement = NULL, 1717 }, 1718 }; 1719 1720 return dispatchExternalMethod(selector, args, dispatchArray, sizeof(dispatchArray) / sizeof(dispatchArray[0]), this, NULL); 1721 } 1722 1723 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1724