1 /* 2 * Copyright (c) 1998-2016 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 30 #include <sys/sysctl.h> 31 extern "C" { 32 #include <vm/vm_kern.h> 33 #include <kern/task.h> 34 #include <kern/debug.h> 35 } 36 37 #include <libkern/c++/OSContainers.h> 38 #include <libkern/OSDebug.h> 39 #include <libkern/c++/OSCPPDebug.h> 40 #include <kern/backtrace.h> 41 #include <kern/btlog.h> 42 43 #include <IOKit/IOKitDebug.h> 44 #include <IOKit/IOLib.h> 45 #include <IOKit/assert.h> 46 #include <IOKit/IODeviceTreeSupport.h> 47 #include <IOKit/IOService.h> 48 49 #include "IOKitKernelInternal.h" 50 51 TUNABLE_WRITEABLE(SInt64, gIOKitDebug, "io", DEBUG_INIT_VALUE); 52 TUNABLE(SInt64, gIOKitTrace, "iotrace", 0); 53 54 #if DEVELOPMENT || DEBUG 55 #define IODEBUG_CTLFLAGS CTLFLAG_RW 56 #else 57 #define IODEBUG_CTLFLAGS CTLFLAG_RD 58 #endif 59 60 SYSCTL_QUAD(_debug, OID_AUTO, iotrace, CTLFLAG_RW | CTLFLAG_LOCKED, &gIOKitTrace, "trace io"); 61 62 static int 63 sysctl_debug_iokit 64 (__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req) 65 { 66 SInt64 newValue; 67 int changed, error = sysctl_io_number(req, gIOKitDebug, sizeof(gIOKitDebug), &newValue, &changed); 68 if (changed) { 69 gIOKitDebug = ((gIOKitDebug & ~kIOKitDebugUserOptions) | (newValue & kIOKitDebugUserOptions)); 70 } 71 return error; 72 } 73 74 SYSCTL_PROC(_debug, OID_AUTO, iokit, 75 CTLTYPE_QUAD | IODEBUG_CTLFLAGS | CTLFLAG_KERN | CTLFLAG_LOCKED, 76 &gIOKitDebug, 0, sysctl_debug_iokit, "Q", "boot_arg io"); 77 78 size_t debug_malloc_size; 79 size_t debug_iomalloc_size; 80 81 vm_size_t debug_iomallocpageable_size; 82 size_t debug_container_malloc_size; 83 // int debug_ivars_size; // in OSObject.cpp 84 85 extern "C" { 86 #if 0 87 #define DEBG(fmt, args...) { kprintf(fmt, ## args); } 88 #else 89 #define DEBG(fmt, args...) { IOLog(fmt, ## args); } 90 #endif 91 92 void 93 IOPrintPlane( const IORegistryPlane * plane ) 94 { 95 IORegistryEntry * next; 96 IORegistryIterator * iter; 97 OSOrderedSet * all; 98 IOService * service; 99 100 iter = IORegistryIterator::iterateOver( plane ); 101 assert( iter ); 102 all = iter->iterateAll(); 103 if (all) { 104 DEBG("Count %d\n", all->getCount()); 105 all->release(); 106 } else { 107 DEBG("Empty\n"); 108 } 109 110 iter->reset(); 111 while ((next = iter->getNextObjectRecursive())) { 112 DEBG( "%*s\033[33m%s", 2 * next->getDepth( plane ), "", next->getName( plane )); 113 if ((next->getLocation( plane ))) { 114 DEBG("@%s", next->getLocation( plane )); 115 } 116 DEBG("\033[0m <class %s", next->getMetaClass()->getClassName()); 117 if ((service = OSDynamicCast(IOService, next))) { 118 DEBG(", busy %ld", (long) service->getBusyState()); 119 } 120 DEBG( ">\n"); 121 // IOSleep(250); 122 } 123 iter->release(); 124 125 #undef IOPrintPlaneFormat 126 } 127 128 void 129 db_piokjunk(void) 130 { 131 } 132 133 void 134 db_dumpiojunk( const IORegistryPlane * plane __unused ) 135 { 136 } 137 138 void 139 IOPrintMemory( void ) 140 { 141 // OSMetaClass::printInstanceCounts(); 142 143 IOLog("\n" 144 "ivar kalloc() 0x%08lx\n" 145 "malloc() 0x%08lx\n" 146 "containers kalloc() 0x%08lx\n" 147 "IOMalloc() 0x%08lx\n" 148 "----------------------------------------\n", 149 debug_ivars_size, 150 debug_malloc_size, 151 debug_container_malloc_size, 152 debug_iomalloc_size 153 ); 154 } 155 } /* extern "C" */ 156 157 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 158 159 #define super OSObject 160 OSDefineMetaClassAndStructors(IOKitDiagnostics, OSObject) 161 162 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 163 164 OSObject * IOKitDiagnostics::diagnostics( void ) 165 { 166 IOKitDiagnostics * diags; 167 168 diags = new IOKitDiagnostics; 169 if (diags && !diags->init()) { 170 diags->release(); 171 diags = NULL; 172 } 173 174 return diags; 175 } 176 177 void 178 IOKitDiagnostics::updateOffset( OSDictionary * dict, 179 UInt64 value, const char * name ) 180 { 181 OSNumber * off; 182 183 off = OSNumber::withNumber( value, 64 ); 184 if (!off) { 185 return; 186 } 187 188 dict->setObject( name, off ); 189 off->release(); 190 } 191 192 bool 193 IOKitDiagnostics::serialize(OSSerialize *s) const 194 { 195 OSDictionary * dict; 196 bool ok; 197 198 dict = OSDictionary::withCapacity( 5 ); 199 if (!dict) { 200 return false; 201 } 202 203 updateOffset( dict, debug_ivars_size, "Instance allocation" ); 204 updateOffset( dict, debug_container_malloc_size, "Container allocation" ); 205 updateOffset( dict, debug_iomalloc_size, "IOMalloc allocation" ); 206 updateOffset( dict, debug_iomallocpageable_size, "Pageable allocation" ); 207 208 OSMetaClass::serializeClassDictionary(dict); 209 210 ok = dict->serialize( s ); 211 212 dict->release(); 213 214 return ok; 215 } 216 217 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 218 219 #if IOTRACKING 220 221 #include <libkern/c++/OSCPPDebug.h> 222 #include <libkern/c++/OSKext.h> 223 #include <kern/zalloc.h> 224 225 __private_extern__ "C" void qsort( 226 void * array, 227 size_t nmembers, 228 size_t member_size, 229 int (*)(const void *, const void *)); 230 231 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); 232 extern "C" ppnum_t pmap_valid_page(ppnum_t pn); 233 234 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 235 236 struct IOTRecursiveLock { 237 lck_mtx_t * mutex; 238 thread_t thread; 239 UInt32 count; 240 }; 241 242 struct IOTrackingQueue { 243 queue_chain_t link; 244 IOTRecursiveLock lock; 245 const char * name; 246 uintptr_t btEntry; 247 size_t allocSize; 248 size_t minCaptureSize; 249 uint32_t siteCount; 250 uint32_t type; 251 uint32_t numSiteQs; 252 uint8_t captureOn; 253 queue_head_t sites[]; 254 }; 255 256 257 struct IOTrackingCallSiteUser { 258 pid_t pid; 259 uint8_t user32; 260 uint8_t userCount; 261 uintptr_t bt[kIOTrackingCallSiteBTs]; 262 }; 263 264 struct IOTrackingCallSite { 265 queue_chain_t link; 266 queue_head_t instances; 267 IOTrackingQueue * queue; 268 IOTracking * addresses; 269 size_t size[2]; 270 uint32_t crc; 271 uint32_t count; 272 273 vm_tag_t tag; 274 uint8_t user32; 275 uint8_t userCount; 276 pid_t btPID; 277 278 uintptr_t bt[kIOTrackingCallSiteBTs]; 279 IOTrackingCallSiteUser user[0]; 280 }; 281 282 struct IOTrackingCallSiteWithUser { 283 struct IOTrackingCallSite site; 284 struct IOTrackingCallSiteUser user; 285 }; 286 287 static void IOTrackingFreeCallSite(uint32_t type, IOTrackingCallSite ** site); 288 289 struct IOTrackingLeaksRef { 290 uintptr_t * instances; 291 uint32_t zoneSize; 292 uint32_t count; 293 uint32_t found; 294 uint32_t foundzlen; 295 size_t bytes; 296 }; 297 298 lck_mtx_t * gIOTrackingLock; 299 queue_head_t gIOTrackingQ; 300 301 enum{ 302 kTrackingAddressFlagAllocated = 0x00000001 303 }; 304 305 #if defined(__LP64__) 306 #define IOTrackingAddressFlags(ptr) (ptr->flags) 307 #else 308 #define IOTrackingAddressFlags(ptr) (ptr->tracking.flags) 309 #endif 310 311 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 312 313 static void 314 IOTRecursiveLockLock(IOTRecursiveLock * lock) 315 { 316 if (lock->thread == current_thread()) { 317 lock->count++; 318 } else { 319 lck_mtx_lock(lock->mutex); 320 assert(lock->thread == NULL); 321 assert(lock->count == 0); 322 lock->thread = current_thread(); 323 lock->count = 1; 324 } 325 } 326 327 static void 328 IOTRecursiveLockUnlock(IOTRecursiveLock * lock) 329 { 330 assert(lock->thread == current_thread()); 331 if (0 == (--lock->count)) { 332 lock->thread = NULL; 333 lck_mtx_unlock(lock->mutex); 334 } 335 } 336 337 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 338 339 void 340 IOTrackingInit(void) 341 { 342 queue_init(&gIOTrackingQ); 343 gIOTrackingLock = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL); 344 } 345 346 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 347 348 IOTrackingQueue * 349 IOTrackingQueueAlloc(const char * name, uintptr_t btEntry, 350 size_t allocSize, size_t minCaptureSize, 351 uint32_t type, uint32_t numSiteQs) 352 { 353 IOTrackingQueue * queue; 354 uint32_t idx; 355 356 if (!numSiteQs) { 357 numSiteQs = 1; 358 } 359 queue = kalloc_type(IOTrackingQueue, queue_head_t, numSiteQs, Z_WAITOK_ZERO); 360 queue->name = name; 361 queue->btEntry = btEntry; 362 queue->allocSize = allocSize; 363 queue->minCaptureSize = minCaptureSize; 364 queue->lock.mutex = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL); 365 queue->numSiteQs = numSiteQs; 366 queue->type = type; 367 enum { kFlags = (kIOTracking | kIOTrackingBoot) }; 368 queue->captureOn = (kFlags == (kFlags & gIOKitDebug)) 369 || (kIOTrackingQueueTypeDefaultOn & type); 370 371 for (idx = 0; idx < numSiteQs; idx++) { 372 queue_init(&queue->sites[idx]); 373 } 374 375 lck_mtx_lock(gIOTrackingLock); 376 queue_enter(&gIOTrackingQ, queue, IOTrackingQueue *, link); 377 lck_mtx_unlock(gIOTrackingLock); 378 379 return queue; 380 }; 381 382 void 383 IOTrackingQueueCollectUser(IOTrackingQueue * queue) 384 { 385 assert(0 == queue->siteCount); 386 queue->type |= kIOTrackingQueueTypeUser; 387 } 388 389 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 390 391 void 392 IOTrackingQueueFree(IOTrackingQueue * queue) 393 { 394 lck_mtx_lock(gIOTrackingLock); 395 IOTrackingReset(queue); 396 remque(&queue->link); 397 lck_mtx_unlock(gIOTrackingLock); 398 399 lck_mtx_free(queue->lock.mutex, IOLockGroup); 400 401 kfree_type(IOTrackingQueue, queue_head_t, queue->numSiteQs, queue); 402 }; 403 404 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 405 406 /* fasthash 407 * The MIT License 408 * 409 * Copyright (C) 2012 Zilong Tan ([email protected]) 410 * 411 * Permission is hereby granted, free of charge, to any person 412 * obtaining a copy of this software and associated documentation 413 * files (the "Software"), to deal in the Software without 414 * restriction, including without limitation the rights to use, copy, 415 * modify, merge, publish, distribute, sublicense, and/or sell copies 416 * of the Software, and to permit persons to whom the Software is 417 * furnished to do so, subject to the following conditions: 418 * 419 * The above copyright notice and this permission notice shall be 420 * included in all copies or substantial portions of the Software. 421 * 422 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 423 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 424 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 425 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 426 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 427 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 428 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 429 * SOFTWARE. 430 */ 431 432 433 // Compression function for Merkle-Damgard construction. 434 // This function is generated using the framework provided. 435 #define mix(h) ({ \ 436 (h) ^= (h) >> 23; \ 437 (h) *= 0x2127599bf4325c37ULL; \ 438 (h) ^= (h) >> 47; }) 439 440 static uint64_t 441 fasthash64(const void *buf, size_t len, uint64_t seed) 442 { 443 const uint64_t m = 0x880355f21e6d1965ULL; 444 const uint64_t *pos = (const uint64_t *)buf; 445 const uint64_t *end = pos + (len / 8); 446 const unsigned char *pos2; 447 uint64_t h = seed ^ (len * m); 448 uint64_t v; 449 450 while (pos != end) { 451 v = *pos++; 452 h ^= mix(v); 453 h *= m; 454 } 455 456 pos2 = (const unsigned char*)pos; 457 v = 0; 458 459 switch (len & 7) { 460 case 7: v ^= (uint64_t)pos2[6] << 48; 461 [[clang::fallthrough]]; 462 case 6: v ^= (uint64_t)pos2[5] << 40; 463 [[clang::fallthrough]]; 464 case 5: v ^= (uint64_t)pos2[4] << 32; 465 [[clang::fallthrough]]; 466 case 4: v ^= (uint64_t)pos2[3] << 24; 467 [[clang::fallthrough]]; 468 case 3: v ^= (uint64_t)pos2[2] << 16; 469 [[clang::fallthrough]]; 470 case 2: v ^= (uint64_t)pos2[1] << 8; 471 [[clang::fallthrough]]; 472 case 1: v ^= (uint64_t)pos2[0]; 473 h ^= mix(v); 474 h *= m; 475 } 476 477 return mix(h); 478 } 479 480 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 481 482 static uint32_t 483 fasthash32(const void *buf, size_t len, uint32_t seed) 484 { 485 // the following trick converts the 64-bit hashcode to Fermat 486 // residue, which shall retain information from both the higher 487 // and lower parts of hashcode. 488 uint64_t h = fasthash64(buf, len, seed); 489 return (uint32_t) (h - (h >> 32)); 490 } 491 492 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 493 494 void 495 IOTrackingAddUser(IOTrackingQueue * queue, IOTrackingUser * mem, vm_size_t size) 496 { 497 uint32_t num; 498 int pid; 499 500 if (!queue->captureOn) { 501 return; 502 } 503 if (size < queue->minCaptureSize) { 504 return; 505 } 506 507 assert(!mem->link.next); 508 509 num = backtrace(&mem->bt[0], kIOTrackingCallSiteBTs, NULL, NULL); 510 num = 0; 511 if ((kernel_task != current_task()) && (pid = proc_selfpid())) { 512 struct backtrace_user_info btinfo = BTUINFO_INIT; 513 mem->btPID = pid; 514 num = backtrace_user(&mem->btUser[0], kIOTrackingCallSiteBTs - 1, 515 NULL, &btinfo); 516 mem->user32 = !(btinfo.btui_info & BTI_64_BIT); 517 } 518 assert(num <= kIOTrackingCallSiteBTs); 519 static_assert(kIOTrackingCallSiteBTs <= UINT8_MAX); 520 mem->userCount = ((uint8_t) num); 521 522 IOTRecursiveLockLock(&queue->lock); 523 queue_enter/*last*/ (&queue->sites[0], mem, IOTrackingUser *, link); 524 queue->siteCount++; 525 IOTRecursiveLockUnlock(&queue->lock); 526 } 527 528 void 529 IOTrackingRemoveUser(IOTrackingQueue * queue, IOTrackingUser * mem) 530 { 531 if (!mem->link.next) { 532 return; 533 } 534 535 IOTRecursiveLockLock(&queue->lock); 536 if (mem->link.next) { 537 remque(&mem->link); 538 assert(queue->siteCount); 539 queue->siteCount--; 540 } 541 IOTRecursiveLockUnlock(&queue->lock); 542 } 543 544 uint64_t gIOTrackingAddTime; 545 546 void 547 IOTrackingAdd(IOTrackingQueue * queue, IOTracking * mem, size_t size, bool address, vm_tag_t tag) 548 { 549 IOTrackingCallSite * site; 550 uint32_t crc, num; 551 uintptr_t bt[kIOTrackingCallSiteBTs + 1]; 552 uintptr_t btUser[kIOTrackingCallSiteBTs]; 553 queue_head_t * que; 554 bool user; 555 int pid; 556 int userCount; 557 558 if (mem->site) { 559 return; 560 } 561 if (!queue->captureOn) { 562 return; 563 } 564 if (size < queue->minCaptureSize) { 565 return; 566 } 567 568 user = (0 != (kIOTrackingQueueTypeUser & queue->type)); 569 570 assert(!mem->link.next); 571 572 num = backtrace(&bt[0], kIOTrackingCallSiteBTs + 1, NULL, NULL); 573 if (!num) { 574 return; 575 } 576 num--; 577 crc = fasthash32(&bt[1], num * sizeof(bt[0]), 0x04C11DB7); 578 579 userCount = 0; 580 pid = 0; 581 backtrace_info_t btinfo = BTI_NONE; 582 if (user) { 583 if ((kernel_task != current_task()) && (pid = proc_selfpid())) { 584 struct backtrace_user_info btuinfo = BTUINFO_INIT; 585 userCount = backtrace_user(&btUser[0], kIOTrackingCallSiteBTs, 586 NULL, &btuinfo); 587 assert(userCount <= kIOTrackingCallSiteBTs); 588 btinfo = btuinfo.btui_info; 589 crc = fasthash32(&btUser[0], userCount * sizeof(bt[0]), crc); 590 } 591 } 592 593 IOTRecursiveLockLock(&queue->lock); 594 que = &queue->sites[crc % queue->numSiteQs]; 595 queue_iterate(que, site, IOTrackingCallSite *, link) 596 { 597 if (tag != site->tag) { 598 continue; 599 } 600 if (user && (pid != site->user[0].pid)) { 601 continue; 602 } 603 if (crc == site->crc) { 604 break; 605 } 606 } 607 608 if (queue_end(que, (queue_entry_t) site)) { 609 if (user) { 610 site = &kalloc_type(IOTrackingCallSiteWithUser, 611 Z_WAITOK_ZERO_NOFAIL)->site; 612 } else { 613 site = kalloc_type(IOTrackingCallSite, 614 Z_WAITOK_ZERO_NOFAIL); 615 } 616 617 queue_init(&site->instances); 618 site->addresses = (IOTracking *) &site->instances; 619 site->queue = queue; 620 site->crc = crc; 621 site->count = 0; 622 site->tag = tag; 623 memset(&site->size[0], 0, sizeof(site->size)); 624 bcopy(&bt[1], &site->bt[0], num * sizeof(site->bt[0])); 625 assert(num <= kIOTrackingCallSiteBTs); 626 bzero(&site->bt[num], (kIOTrackingCallSiteBTs - num) * sizeof(site->bt[0])); 627 if (user) { 628 bcopy(&btUser[0], &site->user[0].bt[0], userCount * sizeof(site->user[0].bt[0])); 629 assert(userCount <= kIOTrackingCallSiteBTs); 630 bzero(&site->user[0].bt[userCount], (kIOTrackingCallSiteBTs - userCount) * sizeof(site->user[0].bt[0])); 631 site->user[0].pid = pid; 632 site->user[0].user32 = !(btinfo & BTI_64_BIT); 633 static_assert(kIOTrackingCallSiteBTs <= UINT8_MAX); 634 site->user[0].userCount = ((uint8_t) userCount); 635 } 636 queue_enter_first(que, site, IOTrackingCallSite *, link); 637 queue->siteCount++; 638 } 639 640 if (address) { 641 queue_enter/*last*/ (&site->instances, mem, IOTracking *, link); 642 if (queue_end(&site->instances, (queue_entry_t)site->addresses)) { 643 site->addresses = mem; 644 } 645 } else { 646 queue_enter_first(&site->instances, mem, IOTracking *, link); 647 } 648 649 mem->site = site; 650 site->size[0] += size; 651 site->count++; 652 653 IOTRecursiveLockUnlock(&queue->lock); 654 } 655 656 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 657 658 void 659 IOTrackingRemove(IOTrackingQueue * queue, IOTracking * mem, size_t size) 660 { 661 if (!mem->link.next) { 662 return; 663 } 664 665 IOTRecursiveLockLock(&queue->lock); 666 if (mem->link.next) { 667 assert(mem->site); 668 669 if (mem == mem->site->addresses) { 670 mem->site->addresses = (IOTracking *) queue_next(&mem->link); 671 } 672 remque(&mem->link); 673 674 assert(mem->site->count); 675 mem->site->count--; 676 assert(mem->site->size[0] >= size); 677 mem->site->size[0] -= size; 678 if (!mem->site->count) { 679 assert(queue_empty(&mem->site->instances)); 680 assert(!mem->site->size[0]); 681 assert(!mem->site->size[1]); 682 683 remque(&mem->site->link); 684 assert(queue->siteCount); 685 queue->siteCount--; 686 IOTrackingFreeCallSite(queue->type, &mem->site); 687 } 688 mem->site = NULL; 689 } 690 IOTRecursiveLockUnlock(&queue->lock); 691 } 692 693 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 694 695 void 696 IOTrackingAlloc(IOTrackingQueue * queue, uintptr_t address, size_t size) 697 { 698 IOTrackingAddress * tracking; 699 700 if (!queue->captureOn) { 701 return; 702 } 703 if (size < queue->minCaptureSize) { 704 return; 705 } 706 707 address = ~address; 708 tracking = kalloc_type(IOTrackingAddress, (zalloc_flags_t)(Z_WAITOK | Z_ZERO)); 709 IOTrackingAddressFlags(tracking) |= kTrackingAddressFlagAllocated; 710 tracking->address = address; 711 tracking->size = size; 712 713 IOTrackingAdd(queue, &tracking->tracking, size, true, VM_KERN_MEMORY_NONE); 714 } 715 716 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 717 718 void 719 IOTrackingFree(IOTrackingQueue * queue, uintptr_t address, size_t size) 720 { 721 IOTrackingCallSite * site; 722 IOTrackingAddress * tracking; 723 uint32_t idx; 724 bool done; 725 726 address = ~address; 727 IOTRecursiveLockLock(&queue->lock); 728 done = false; 729 for (idx = 0; idx < queue->numSiteQs; idx++) { 730 queue_iterate(&queue->sites[idx], site, IOTrackingCallSite *, link) 731 { 732 tracking = (IOTrackingAddress *) site->addresses; 733 while (!queue_end(&site->instances, &tracking->tracking.link)) { 734 if ((done = (address == tracking->address))) { 735 IOTrackingRemove(queue, &tracking->tracking, size); 736 kfree_type(IOTrackingAddress, tracking); 737 break; 738 } else { 739 tracking = (IOTrackingAddress *) queue_next(&tracking->tracking.link); 740 } 741 } 742 if (done) { 743 break; 744 } 745 } 746 if (done) { 747 break; 748 } 749 } 750 IOTRecursiveLockUnlock(&queue->lock); 751 } 752 753 static void 754 IOTrackingFreeCallSite(uint32_t type, IOTrackingCallSite ** site) 755 { 756 void ** ptr; 757 758 ptr = reinterpret_cast<void **>(site); 759 if (kIOTrackingQueueTypeUser & type) { 760 kfree_type(IOTrackingCallSiteWithUser, *ptr); 761 } else { 762 kfree_type(IOTrackingCallSite, *ptr); 763 } 764 } 765 766 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 767 768 void 769 IOTrackingAccumSize(IOTrackingQueue * queue, IOTracking * mem, size_t size) 770 { 771 IOTRecursiveLockLock(&queue->lock); 772 if (mem->link.next) { 773 assert(mem->site); 774 assert((size > 0) || (mem->site->size[1] >= -size)); 775 mem->site->size[1] += size; 776 } 777 ; 778 IOTRecursiveLockUnlock(&queue->lock); 779 } 780 781 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 782 783 void 784 IOTrackingReset(IOTrackingQueue * queue) 785 { 786 IOTrackingCallSite * site; 787 IOTrackingUser * user; 788 IOTracking * tracking; 789 IOTrackingAddress * trackingAddress; 790 uint32_t idx; 791 bool addresses; 792 793 IOTRecursiveLockLock(&queue->lock); 794 for (idx = 0; idx < queue->numSiteQs; idx++) { 795 while (!queue_empty(&queue->sites[idx])) { 796 if (kIOTrackingQueueTypeMap & queue->type) { 797 queue_remove_first(&queue->sites[idx], user, IOTrackingUser *, link); 798 user->link.next = user->link.prev = NULL; 799 } else { 800 queue_remove_first(&queue->sites[idx], site, IOTrackingCallSite *, link); 801 addresses = false; 802 while (!queue_empty(&site->instances)) { 803 queue_remove_first(&site->instances, tracking, IOTracking *, link); 804 if (tracking == site->addresses) { 805 addresses = true; 806 } 807 if (addresses) { 808 trackingAddress = (typeof(trackingAddress))tracking; 809 if (kTrackingAddressFlagAllocated & IOTrackingAddressFlags(trackingAddress)) { 810 kfree_type(IOTrackingAddress, trackingAddress); 811 } 812 } 813 } 814 IOTrackingFreeCallSite(queue->type, &site); 815 } 816 } 817 } 818 queue->siteCount = 0; 819 IOTRecursiveLockUnlock(&queue->lock); 820 } 821 822 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 823 824 static int 825 IOTrackingCallSiteInfoCompare(const void * left, const void * right) 826 { 827 IOTrackingCallSiteInfo * l = (typeof(l))left; 828 IOTrackingCallSiteInfo * r = (typeof(r))right; 829 size_t lsize, rsize; 830 831 rsize = r->size[0] + r->size[1]; 832 lsize = l->size[0] + l->size[1]; 833 834 return (rsize > lsize) ? 1 : ((rsize == lsize) ? 0 : -1); 835 } 836 837 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 838 839 static int 840 IOTrackingAddressCompare(const void * left, const void * right) 841 { 842 IOTracking * instance; 843 uintptr_t inst, laddr, raddr; 844 845 inst = ((typeof(inst) *)left)[0]; 846 instance = (typeof(instance))INSTANCE_GET(inst); 847 if (kInstanceFlagAddress & inst) { 848 laddr = ~((IOTrackingAddress *)instance)->address; 849 } else { 850 laddr = (uintptr_t) (instance + 1); 851 } 852 853 inst = ((typeof(inst) *)right)[0]; 854 instance = (typeof(instance))(inst & ~kInstanceFlags); 855 if (kInstanceFlagAddress & inst) { 856 raddr = ~((IOTrackingAddress *)instance)->address; 857 } else { 858 raddr = (uintptr_t) (instance + 1); 859 } 860 861 return (laddr > raddr) ? 1 : ((laddr == raddr) ? 0 : -1); 862 } 863 864 865 static int 866 IOTrackingZoneElementCompare(const void * left, const void * right) 867 { 868 uintptr_t inst, laddr, raddr; 869 870 inst = ((typeof(inst) *)left)[0]; 871 laddr = INSTANCE_PUT(inst); 872 inst = ((typeof(inst) *)right)[0]; 873 raddr = INSTANCE_PUT(inst); 874 875 return (laddr > raddr) ? 1 : ((laddr == raddr) ? 0 : -1); 876 } 877 878 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 879 880 static void 881 CopyOutBacktraces(IOTrackingCallSite * site, IOTrackingCallSiteInfo * siteInfo) 882 { 883 uint32_t j; 884 mach_vm_address_t bt, btEntry; 885 886 btEntry = site->queue->btEntry; 887 for (j = 0; j < kIOTrackingCallSiteBTs; j++) { 888 bt = site->bt[j]; 889 if (btEntry 890 && (!bt || (j == (kIOTrackingCallSiteBTs - 1)))) { 891 bt = btEntry; 892 btEntry = 0; 893 } 894 siteInfo->bt[0][j] = VM_KERNEL_UNSLIDE(bt); 895 } 896 897 siteInfo->btPID = 0; 898 if (kIOTrackingQueueTypeUser & site->queue->type) { 899 siteInfo->btPID = site->user[0].pid; 900 uint32_t * bt32 = (typeof(bt32))((void *) &site->user[0].bt[0]); 901 uint64_t * bt64 = (typeof(bt64))((void *) &site->user[0].bt[0]); 902 for (uint32_t j = 0; j < kIOTrackingCallSiteBTs; j++) { 903 if (j >= site->user[0].userCount) { 904 siteInfo->bt[1][j] = 0; 905 } else if (site->user[0].user32) { 906 siteInfo->bt[1][j] = bt32[j]; 907 } else { 908 siteInfo->bt[1][j] = bt64[j]; 909 } 910 } 911 } 912 } 913 914 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 915 916 static void 917 IOTrackingLeakScan(void * refcon) 918 { 919 IOTrackingLeaksRef * ref = (typeof(ref))refcon; 920 uintptr_t * instances; 921 IOTracking * instance; 922 uint64_t vaddr, vincr; 923 ppnum_t ppn; 924 uintptr_t ptr, addr, vphysaddr, inst; 925 size_t size, origsize; 926 uint32_t baseIdx, lim, ptrIdx, count; 927 boolean_t is; 928 AbsoluteTime deadline; 929 930 instances = ref->instances; 931 count = ref->count; 932 size = origsize = ref->zoneSize; 933 934 for (deadline = 0, vaddr = VM_MIN_KERNEL_AND_KEXT_ADDRESS; 935 ; 936 vaddr += vincr) { 937 if ((mach_absolute_time() > deadline) || (vaddr >= VM_MAX_KERNEL_ADDRESS)) { 938 if (deadline) { 939 ml_set_interrupts_enabled(is); 940 IODelay(10); 941 } 942 if (vaddr >= VM_MAX_KERNEL_ADDRESS) { 943 break; 944 } 945 is = ml_set_interrupts_enabled(false); 946 clock_interval_to_deadline(10, kMillisecondScale, &deadline); 947 } 948 949 ppn = kernel_pmap_present_mapping(vaddr, &vincr, &vphysaddr); 950 // check noencrypt to avoid VM structs (map entries) with pointers 951 if (ppn && (!pmap_valid_page(ppn) || (!ref->zoneSize && pmap_is_noencrypt(ppn)))) { 952 ppn = 0; 953 } 954 if (!ppn) { 955 continue; 956 } 957 958 for (ptrIdx = 0; ptrIdx < (page_size / sizeof(uintptr_t)); ptrIdx++) { 959 ptr = ((uintptr_t *)vphysaddr)[ptrIdx]; 960 #if defined(HAS_APPLE_PAC) 961 // strip possible ptrauth signature from candidate data pointer 962 ptr = (uintptr_t)ptrauth_strip((void*)ptr, ptrauth_key_process_independent_data); 963 #endif /* defined(HAS_APPLE_PAC) */ 964 965 for (lim = count, baseIdx = 0; lim; lim >>= 1) { 966 inst = instances[baseIdx + (lim >> 1)]; 967 instance = (typeof(instance))INSTANCE_GET(inst); 968 969 if (ref->zoneSize) { 970 addr = INSTANCE_PUT(inst) & ~kInstanceFlags; 971 } else if (kInstanceFlagAddress & inst) { 972 addr = ~((IOTrackingAddress *)instance)->address; 973 origsize = size = ((IOTrackingAddress *)instance)->size; 974 if (!size) { 975 size = 1; 976 } 977 } else { 978 addr = (uintptr_t) (instance + 1); 979 origsize = size = instance->site->queue->allocSize; 980 } 981 if ((ptr >= addr) && (ptr < (addr + size)) 982 983 && (((vaddr + ptrIdx * sizeof(uintptr_t)) < addr) 984 || ((vaddr + ptrIdx * sizeof(uintptr_t)) >= (addr + size)))) { 985 if (!(kInstanceFlagReferenced & inst)) { 986 inst |= kInstanceFlagReferenced; 987 instances[baseIdx + (lim >> 1)] = inst; 988 ref->found++; 989 if (!origsize) { 990 ref->foundzlen++; 991 } 992 } 993 break; 994 } 995 if (ptr > addr) { 996 // move right 997 baseIdx += (lim >> 1) + 1; 998 lim--; 999 } 1000 // else move left 1001 } 1002 } 1003 ref->bytes += page_size; 1004 } 1005 } 1006 1007 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1008 1009 extern "C" void 1010 zone_leaks_scan(uintptr_t * instances, uint32_t count, uint32_t zoneSize, uint32_t * found) 1011 { 1012 IOTrackingLeaksRef ref; 1013 IOTrackingCallSiteInfo siteInfo; 1014 uint32_t idx; 1015 1016 qsort(instances, count, sizeof(*instances), &IOTrackingZoneElementCompare); 1017 1018 bzero(&siteInfo, sizeof(siteInfo)); 1019 bzero(&ref, sizeof(ref)); 1020 ref.instances = instances; 1021 ref.count = count; 1022 ref.zoneSize = zoneSize; 1023 1024 for (idx = 0; idx < 2; idx++) { 1025 ref.bytes = 0; 1026 IOTrackingLeakScan(&ref); 1027 IOLog("leaks(%d) scanned %ld MB, instance count %d, found %d\n", idx, ref.bytes / 1024 / 1024, count, ref.found); 1028 if (count <= ref.found) { 1029 break; 1030 } 1031 } 1032 1033 *found = ref.found; 1034 } 1035 1036 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1037 1038 static OSData * 1039 IOTrackingLeaks(LIBKERN_CONSUMED OSData * data) 1040 { 1041 IOTrackingLeaksRef ref; 1042 IOTrackingCallSiteInfo siteInfo; 1043 IOTrackingCallSite * site; 1044 OSData * leakData; 1045 uintptr_t * instances; 1046 IOTracking * instance; 1047 uintptr_t inst; 1048 uint32_t count, idx, numSites, dups, siteCount; 1049 1050 instances = (typeof(instances))data->getBytesNoCopy(); 1051 count = (data->getLength() / sizeof(*instances)); 1052 qsort(instances, count, sizeof(*instances), &IOTrackingAddressCompare); 1053 1054 bzero(&siteInfo, sizeof(siteInfo)); 1055 bzero(&ref, sizeof(ref)); 1056 ref.instances = instances; 1057 ref.count = count; 1058 for (idx = 0; idx < 2; idx++) { 1059 ref.bytes = 0; 1060 IOTrackingLeakScan(&ref); 1061 IOLog("leaks(%d) scanned %ld MB, instance count %d, found %d (zlen %d)\n", idx, ref.bytes / 1024 / 1024, count, ref.found, ref.foundzlen); 1062 if (count <= ref.found) { 1063 break; 1064 } 1065 } 1066 1067 leakData = OSData::withCapacity(128 * sizeof(IOTrackingCallSiteInfo)); 1068 1069 for (numSites = 0, idx = 0; idx < count; idx++) { 1070 inst = instances[idx]; 1071 if (kInstanceFlagReferenced & inst) { 1072 continue; 1073 } 1074 instance = (typeof(instance))INSTANCE_GET(inst); 1075 site = instance->site; 1076 instances[numSites] = (uintptr_t) site; 1077 numSites++; 1078 } 1079 1080 for (idx = 0; idx < numSites; idx++) { 1081 inst = instances[idx]; 1082 if (!inst) { 1083 continue; 1084 } 1085 site = (typeof(site))inst; 1086 for (siteCount = 1, dups = (idx + 1); dups < numSites; dups++) { 1087 if (instances[dups] == (uintptr_t) site) { 1088 siteCount++; 1089 instances[dups] = 0; 1090 } 1091 } 1092 siteInfo.count = siteCount; 1093 siteInfo.size[0] = (site->size[0] * site->count) / siteCount; 1094 siteInfo.size[1] = (site->size[1] * site->count) / siteCount; 1095 CopyOutBacktraces(site, &siteInfo); 1096 leakData->appendBytes(&siteInfo, sizeof(siteInfo)); 1097 } 1098 data->release(); 1099 1100 return leakData; 1101 } 1102 1103 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1104 1105 static bool 1106 SkipName(uint32_t options, const char * name, size_t namesLen, const char * names) 1107 { 1108 const char * scan; 1109 const char * next; 1110 bool exclude, found; 1111 size_t qLen, sLen; 1112 1113 if (!namesLen || !names) { 1114 return false; 1115 } 1116 // <len><name>...<len><name><0> 1117 exclude = (0 != (kIOTrackingExcludeNames & options)); 1118 qLen = strlen(name); 1119 scan = names; 1120 found = false; 1121 do{ 1122 sLen = scan[0]; 1123 scan++; 1124 next = scan + sLen; 1125 if (next >= (names + namesLen)) { 1126 break; 1127 } 1128 found = ((sLen == qLen) && !strncmp(scan, name, sLen)); 1129 scan = next; 1130 }while (!found && (scan < (names + namesLen))); 1131 1132 return !(exclude ^ found); 1133 } 1134 1135 #endif /* IOTRACKING */ 1136 1137 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1138 1139 static kern_return_t 1140 IOTrackingDebug(uint32_t selector, uint32_t options, uint64_t value, 1141 uint32_t intag, uint32_t inzsize, 1142 const char * names, size_t namesLen, 1143 size_t size, OSObject ** result) 1144 { 1145 kern_return_t ret; 1146 OSData * data; 1147 1148 if (result) { 1149 *result = NULL; 1150 } 1151 data = NULL; 1152 ret = kIOReturnNotReady; 1153 1154 #if IOTRACKING 1155 1156 kern_return_t kr; 1157 IOTrackingQueue * queue; 1158 IOTracking * instance; 1159 IOTrackingCallSite * site; 1160 IOTrackingCallSiteInfo siteInfo; 1161 IOTrackingUser * user; 1162 task_t mapTask; 1163 mach_vm_address_t mapAddress; 1164 mach_vm_size_t mapSize; 1165 uint32_t num, idx, qIdx; 1166 uintptr_t instFlags; 1167 proc_t proc; 1168 bool addresses; 1169 1170 ret = kIOReturnNotFound; 1171 proc = NULL; 1172 if (kIOTrackingGetMappings == selector) { 1173 if (value != -1ULL) { 1174 proc = proc_find((pid_t) value); 1175 if (!proc) { 1176 return kIOReturnNotFound; 1177 } 1178 } 1179 } 1180 1181 bzero(&siteInfo, sizeof(siteInfo)); 1182 lck_mtx_lock(gIOTrackingLock); 1183 queue_iterate(&gIOTrackingQ, queue, IOTrackingQueue *, link) 1184 { 1185 if (SkipName(options, queue->name, namesLen, names)) { 1186 continue; 1187 } 1188 1189 if (!(kIOTracking & gIOKitDebug) && (kIOTrackingQueueTypeAlloc & queue->type)) { 1190 continue; 1191 } 1192 1193 switch (selector) { 1194 case kIOTrackingResetTracking: 1195 { 1196 IOTrackingReset(queue); 1197 ret = kIOReturnSuccess; 1198 break; 1199 } 1200 1201 case kIOTrackingStartCapture: 1202 case kIOTrackingStopCapture: 1203 { 1204 queue->captureOn = (kIOTrackingStartCapture == selector); 1205 ret = kIOReturnSuccess; 1206 break; 1207 } 1208 1209 case kIOTrackingSetMinCaptureSize: 1210 { 1211 queue->minCaptureSize = size; 1212 ret = kIOReturnSuccess; 1213 break; 1214 } 1215 1216 case kIOTrackingLeaks: 1217 { 1218 if (!(kIOTrackingQueueTypeAlloc & queue->type)) { 1219 break; 1220 } 1221 1222 if (!data) { 1223 data = OSData::withCapacity(1024 * sizeof(uintptr_t)); 1224 } 1225 1226 IOTRecursiveLockLock(&queue->lock); 1227 for (idx = 0; idx < queue->numSiteQs; idx++) { 1228 queue_iterate(&queue->sites[idx], site, IOTrackingCallSite *, link) 1229 { 1230 addresses = false; 1231 queue_iterate(&site->instances, instance, IOTracking *, link) 1232 { 1233 if (instance == site->addresses) { 1234 addresses = true; 1235 } 1236 instFlags = (typeof(instFlags))instance; 1237 if (addresses) { 1238 instFlags |= kInstanceFlagAddress; 1239 } 1240 data->appendValue(instFlags); 1241 } 1242 } 1243 } 1244 // queue is locked 1245 ret = kIOReturnSuccess; 1246 break; 1247 } 1248 1249 1250 case kIOTrackingGetTracking: 1251 { 1252 if (kIOTrackingQueueTypeMap & queue->type) { 1253 break; 1254 } 1255 1256 if (!data) { 1257 data = OSData::withCapacity(128 * sizeof(IOTrackingCallSiteInfo)); 1258 } 1259 1260 IOTRecursiveLockLock(&queue->lock); 1261 num = queue->siteCount; 1262 idx = 0; 1263 for (qIdx = 0; qIdx < queue->numSiteQs; qIdx++) { 1264 queue_iterate(&queue->sites[qIdx], site, IOTrackingCallSite *, link) 1265 { 1266 assert(idx < num); 1267 idx++; 1268 1269 size_t tsize[2]; 1270 uint32_t count = site->count; 1271 tsize[0] = site->size[0]; 1272 tsize[1] = site->size[1]; 1273 1274 if (intag || inzsize) { 1275 uintptr_t addr; 1276 vm_size_t size, zoneSize; 1277 vm_tag_t tag; 1278 1279 if (kIOTrackingQueueTypeAlloc & queue->type) { 1280 addresses = false; 1281 count = 0; 1282 tsize[0] = tsize[1] = 0; 1283 queue_iterate(&site->instances, instance, IOTracking *, link) 1284 { 1285 if (instance == site->addresses) { 1286 addresses = true; 1287 } 1288 1289 if (addresses) { 1290 addr = ~((IOTrackingAddress *)instance)->address; 1291 } else { 1292 addr = (uintptr_t) (instance + 1); 1293 } 1294 1295 kr = vm_kern_allocation_info(addr, &size, &tag, &zoneSize); 1296 if (KERN_SUCCESS != kr) { 1297 continue; 1298 } 1299 1300 if ((VM_KERN_MEMORY_NONE != intag) && (intag != tag)) { 1301 continue; 1302 } 1303 if (inzsize && (inzsize != zoneSize)) { 1304 continue; 1305 } 1306 1307 count++; 1308 tsize[0] += size; 1309 } 1310 } else { 1311 if (!intag || inzsize || (intag != site->tag)) { 1312 continue; 1313 } 1314 } 1315 } 1316 1317 if (!count) { 1318 continue; 1319 } 1320 if (size && ((tsize[0] + tsize[1]) < size)) { 1321 continue; 1322 } 1323 siteInfo.count = count; 1324 siteInfo.size[0] = tsize[0]; 1325 siteInfo.size[1] = tsize[1]; 1326 CopyOutBacktraces(site, &siteInfo); 1327 data->appendBytes(&siteInfo, sizeof(siteInfo)); 1328 } 1329 } 1330 assert(idx == num); 1331 IOTRecursiveLockUnlock(&queue->lock); 1332 ret = kIOReturnSuccess; 1333 break; 1334 } 1335 1336 case kIOTrackingGetMappings: 1337 { 1338 if (!(kIOTrackingQueueTypeMap & queue->type)) { 1339 break; 1340 } 1341 if (!data) { 1342 data = OSData::withCapacity((unsigned int) page_size); 1343 } 1344 1345 IOTRecursiveLockLock(&queue->lock); 1346 num = queue->siteCount; 1347 idx = 0; 1348 for (qIdx = 0; qIdx < queue->numSiteQs; qIdx++) { 1349 queue_iterate(&queue->sites[qIdx], user, IOTrackingUser *, link) 1350 { 1351 assert(idx < num); 1352 idx++; 1353 1354 kr = IOMemoryMapTracking(user, &mapTask, &mapAddress, &mapSize); 1355 if (kIOReturnSuccess != kr) { 1356 continue; 1357 } 1358 if (proc && (mapTask != proc_task(proc))) { 1359 continue; 1360 } 1361 if (size && (mapSize < size)) { 1362 continue; 1363 } 1364 1365 siteInfo.count = 1; 1366 siteInfo.size[0] = mapSize; 1367 siteInfo.address = mapAddress; 1368 siteInfo.addressPID = task_pid(mapTask); 1369 siteInfo.btPID = user->btPID; 1370 1371 for (uint32_t j = 0; j < kIOTrackingCallSiteBTs; j++) { 1372 siteInfo.bt[0][j] = VM_KERNEL_UNSLIDE(user->bt[j]); 1373 } 1374 uint32_t * bt32 = (typeof(bt32)) & user->btUser[0]; 1375 uint64_t * bt64 = (typeof(bt64))((void *) &user->btUser[0]); 1376 for (uint32_t j = 0; j < kIOTrackingCallSiteBTs; j++) { 1377 if (j >= user->userCount) { 1378 siteInfo.bt[1][j] = 0; 1379 } else if (user->user32) { 1380 siteInfo.bt[1][j] = bt32[j]; 1381 } else { 1382 siteInfo.bt[1][j] = bt64[j]; 1383 } 1384 } 1385 data->appendBytes(&siteInfo, sizeof(siteInfo)); 1386 } 1387 } 1388 assert(idx == num); 1389 IOTRecursiveLockUnlock(&queue->lock); 1390 ret = kIOReturnSuccess; 1391 break; 1392 } 1393 1394 default: 1395 ret = kIOReturnUnsupported; 1396 break; 1397 } 1398 } 1399 1400 if ((kIOTrackingLeaks == selector) && data) { 1401 data = IOTrackingLeaks(data); 1402 queue_iterate(&gIOTrackingQ, queue, IOTrackingQueue *, link) 1403 { 1404 if (SkipName(options, queue->name, namesLen, names)) { 1405 continue; 1406 } 1407 if (!(kIOTrackingQueueTypeAlloc & queue->type)) { 1408 continue; 1409 } 1410 IOTRecursiveLockUnlock(&queue->lock); 1411 } 1412 } 1413 1414 lck_mtx_unlock(gIOTrackingLock); 1415 1416 if ((kIOTrackingLeaks == selector) && namesLen && names) { 1417 const char * scan; 1418 const char * next; 1419 uint8_t sLen; 1420 1421 if (!data) { 1422 data = OSData::withCapacity(4096 * sizeof(uintptr_t)); 1423 } 1424 1425 // <len><name>...<len><name><0> 1426 scan = names; 1427 do{ 1428 sLen = ((uint8_t) scan[0]); 1429 scan++; 1430 next = scan + sLen; 1431 if (next >= (names + namesLen)) { 1432 break; 1433 } 1434 kr = zone_leaks(scan, sLen, ^(uint32_t count, uint32_t eSize, btref_t ref) { 1435 IOTrackingCallSiteInfo siteInfo = { 1436 .count = count, 1437 .size[0] = eSize * count, 1438 }; 1439 1440 btref_decode_unslide(ref, siteInfo.bt[0]); 1441 1442 data->appendBytes(&siteInfo, sizeof(siteInfo)); 1443 }); 1444 if (KERN_SUCCESS == kr) { 1445 ret = kIOReturnSuccess; 1446 } else if (KERN_INVALID_NAME != kr) { 1447 ret = kIOReturnVMError; 1448 } 1449 scan = next; 1450 }while (scan < (names + namesLen)); 1451 } 1452 1453 if (data) { 1454 switch (selector) { 1455 case kIOTrackingLeaks: 1456 case kIOTrackingGetTracking: 1457 case kIOTrackingGetMappings: 1458 { 1459 IOTrackingCallSiteInfo * siteInfos; 1460 siteInfos = (typeof(siteInfos))data->getBytesNoCopy(); 1461 num = (data->getLength() / sizeof(*siteInfos)); 1462 qsort(siteInfos, num, sizeof(*siteInfos), &IOTrackingCallSiteInfoCompare); 1463 break; 1464 } 1465 default: assert(false); break; 1466 } 1467 } 1468 1469 *result = data; 1470 if (proc) { 1471 proc_rele(proc); 1472 } 1473 1474 #endif /* IOTRACKING */ 1475 1476 return ret; 1477 } 1478 1479 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1480 1481 #include <IOKit/IOKitDiagnosticsUserClient.h> 1482 1483 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1484 1485 #undef super 1486 #define super IOUserClient 1487 1488 OSDefineMetaClassAndStructors(IOKitDiagnosticsClient, IOUserClient) 1489 1490 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1491 1492 IOUserClient * IOKitDiagnosticsClient::withTask(task_t owningTask) 1493 { 1494 IOKitDiagnosticsClient * inst; 1495 1496 inst = new IOKitDiagnosticsClient; 1497 if (inst && !inst->init()) { 1498 inst->release(); 1499 inst = NULL; 1500 } 1501 1502 return inst; 1503 } 1504 1505 IOReturn 1506 IOKitDiagnosticsClient::clientClose(void) 1507 { 1508 terminate(); 1509 return kIOReturnSuccess; 1510 } 1511 1512 IOReturn 1513 IOKitDiagnosticsClient::setProperties(OSObject * properties) 1514 { 1515 IOReturn kr = kIOReturnUnsupported; 1516 return kr; 1517 } 1518 1519 IOReturn 1520 IOKitDiagnosticsClient::externalMethod(uint32_t selector, IOExternalMethodArguments * args, 1521 IOExternalMethodDispatch * dispatch, OSObject * target, void * reference) 1522 { 1523 IOReturn ret = kIOReturnBadArgument; 1524 const IOKitDiagnosticsParameters * params; 1525 const char * names; 1526 size_t namesLen; 1527 OSObject * result; 1528 1529 if (args->structureInputSize < sizeof(IOKitDiagnosticsParameters)) { 1530 return kIOReturnBadArgument; 1531 } 1532 params = (typeof(params))args->structureInput; 1533 if (!params) { 1534 return kIOReturnBadArgument; 1535 } 1536 1537 names = NULL; 1538 namesLen = args->structureInputSize - sizeof(IOKitDiagnosticsParameters); 1539 if (namesLen) { 1540 names = (typeof(names))(params + 1); 1541 } 1542 1543 ret = IOTrackingDebug(selector, params->options, params->value, params->tag, params->zsize, names, namesLen, params->size, &result); 1544 1545 if ((kIOReturnSuccess == ret) && args->structureVariableOutputData) { 1546 *args->structureVariableOutputData = result; 1547 } else if (result) { 1548 result->release(); 1549 } 1550 1551 return ret; 1552 } 1553 1554 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1555