1 /* 2 * Copyright (c) 1998-2016 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 30 #include <sys/sysctl.h> 31 extern "C" { 32 #include <vm/vm_kern.h> 33 #include <kern/task.h> 34 #include <kern/debug.h> 35 } 36 37 #include <libkern/c++/OSContainers.h> 38 #include <libkern/OSDebug.h> 39 #include <libkern/c++/OSCPPDebug.h> 40 #include <kern/backtrace.h> 41 42 #include <IOKit/IOKitDebug.h> 43 #include <IOKit/IOLib.h> 44 #include <IOKit/assert.h> 45 #include <IOKit/IODeviceTreeSupport.h> 46 #include <IOKit/IOService.h> 47 48 #include "IOKitKernelInternal.h" 49 50 TUNABLE_WRITEABLE(SInt64, gIOKitDebug, "io", DEBUG_INIT_VALUE); 51 TUNABLE(SInt64, gIOKitTrace, "iotrace", 0); 52 53 #if DEVELOPMENT || DEBUG 54 #define IODEBUG_CTLFLAGS CTLFLAG_RW 55 #else 56 #define IODEBUG_CTLFLAGS CTLFLAG_RD 57 #endif 58 59 SYSCTL_QUAD(_debug, OID_AUTO, iotrace, CTLFLAG_RW | CTLFLAG_LOCKED, &gIOKitTrace, "trace io"); 60 61 static int 62 sysctl_debug_iokit 63 (__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req) 64 { 65 SInt64 newValue; 66 int changed, error = sysctl_io_number(req, gIOKitDebug, sizeof(gIOKitDebug), &newValue, &changed); 67 if (changed) { 68 gIOKitDebug = ((gIOKitDebug & ~kIOKitDebugUserOptions) | (newValue & kIOKitDebugUserOptions)); 69 } 70 return error; 71 } 72 73 SYSCTL_PROC(_debug, OID_AUTO, iokit, 74 CTLTYPE_QUAD | IODEBUG_CTLFLAGS | CTLFLAG_KERN | CTLFLAG_LOCKED, 75 &gIOKitDebug, 0, sysctl_debug_iokit, "Q", "boot_arg io"); 76 77 size_t debug_malloc_size; 78 size_t debug_iomalloc_size; 79 80 vm_size_t debug_iomallocpageable_size; 81 size_t debug_container_malloc_size; 82 // int debug_ivars_size; // in OSObject.cpp 83 84 extern "C" { 85 #if 0 86 #define DEBG(fmt, args...) { kprintf(fmt, ## args); } 87 #else 88 #define DEBG(fmt, args...) { IOLog(fmt, ## args); } 89 #endif 90 91 void 92 IOPrintPlane( const IORegistryPlane * plane ) 93 { 94 IORegistryEntry * next; 95 IORegistryIterator * iter; 96 OSOrderedSet * all; 97 char format[] = "%xxxs"; 98 IOService * service; 99 100 iter = IORegistryIterator::iterateOver( plane ); 101 assert( iter ); 102 all = iter->iterateAll(); 103 if (all) { 104 DEBG("Count %d\n", all->getCount()); 105 all->release(); 106 } else { 107 DEBG("Empty\n"); 108 } 109 110 iter->reset(); 111 while ((next = iter->getNextObjectRecursive())) { 112 snprintf(format + 1, sizeof(format) - 1, "%ds", 2 * next->getDepth( plane )); 113 DEBG( format, ""); 114 DEBG( "\033[33m%s", next->getName( plane )); 115 if ((next->getLocation( plane ))) { 116 DEBG("@%s", next->getLocation( plane )); 117 } 118 DEBG("\033[0m <class %s", next->getMetaClass()->getClassName()); 119 if ((service = OSDynamicCast(IOService, next))) { 120 DEBG(", busy %ld", (long) service->getBusyState()); 121 } 122 DEBG( ">\n"); 123 // IOSleep(250); 124 } 125 iter->release(); 126 } 127 128 void 129 db_piokjunk(void) 130 { 131 } 132 133 void 134 db_dumpiojunk( const IORegistryPlane * plane __unused ) 135 { 136 } 137 138 void 139 IOPrintMemory( void ) 140 { 141 // OSMetaClass::printInstanceCounts(); 142 143 IOLog("\n" 144 "ivar kalloc() 0x%08lx\n" 145 "malloc() 0x%08lx\n" 146 "containers kalloc() 0x%08lx\n" 147 "IOMalloc() 0x%08lx\n" 148 "----------------------------------------\n", 149 debug_ivars_size, 150 debug_malloc_size, 151 debug_container_malloc_size, 152 debug_iomalloc_size 153 ); 154 } 155 } /* extern "C" */ 156 157 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 158 159 #define super OSObject 160 OSDefineMetaClassAndStructors(IOKitDiagnostics, OSObject) 161 162 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 163 164 OSObject * IOKitDiagnostics::diagnostics( void ) 165 { 166 IOKitDiagnostics * diags; 167 168 diags = new IOKitDiagnostics; 169 if (diags && !diags->init()) { 170 diags->release(); 171 diags = NULL; 172 } 173 174 return diags; 175 } 176 177 void 178 IOKitDiagnostics::updateOffset( OSDictionary * dict, 179 UInt64 value, const char * name ) 180 { 181 OSNumber * off; 182 183 off = OSNumber::withNumber( value, 64 ); 184 if (!off) { 185 return; 186 } 187 188 dict->setObject( name, off ); 189 off->release(); 190 } 191 192 bool 193 IOKitDiagnostics::serialize(OSSerialize *s) const 194 { 195 OSDictionary * dict; 196 bool ok; 197 198 dict = OSDictionary::withCapacity( 5 ); 199 if (!dict) { 200 return false; 201 } 202 203 updateOffset( dict, debug_ivars_size, "Instance allocation" ); 204 updateOffset( dict, debug_container_malloc_size, "Container allocation" ); 205 updateOffset( dict, debug_iomalloc_size, "IOMalloc allocation" ); 206 updateOffset( dict, debug_iomallocpageable_size, "Pageable allocation" ); 207 208 OSMetaClass::serializeClassDictionary(dict); 209 210 ok = dict->serialize( s ); 211 212 dict->release(); 213 214 return ok; 215 } 216 217 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 218 219 #if IOTRACKING 220 221 #include <libkern/c++/OSCPPDebug.h> 222 #include <libkern/c++/OSKext.h> 223 #include <kern/zalloc.h> 224 225 __private_extern__ "C" void qsort( 226 void * array, 227 size_t nmembers, 228 size_t member_size, 229 int (*)(const void *, const void *)); 230 231 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); 232 extern "C" ppnum_t pmap_valid_page(ppnum_t pn); 233 234 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 235 236 struct IOTRecursiveLock { 237 lck_mtx_t * mutex; 238 thread_t thread; 239 UInt32 count; 240 }; 241 242 struct IOTrackingQueue { 243 queue_chain_t link; 244 IOTRecursiveLock lock; 245 const char * name; 246 uintptr_t btEntry; 247 size_t allocSize; 248 size_t minCaptureSize; 249 uint32_t siteCount; 250 uint32_t type; 251 uint32_t numSiteQs; 252 uint8_t captureOn; 253 queue_head_t sites[]; 254 }; 255 256 257 struct IOTrackingCallSiteUser { 258 pid_t pid; 259 uint8_t user32; 260 uint8_t userCount; 261 uintptr_t bt[kIOTrackingCallSiteBTs]; 262 }; 263 264 struct IOTrackingCallSite { 265 queue_chain_t link; 266 queue_head_t instances; 267 IOTrackingQueue * queue; 268 IOTracking * addresses; 269 size_t size[2]; 270 uint32_t crc; 271 uint32_t count; 272 273 vm_tag_t tag; 274 uint8_t user32; 275 uint8_t userCount; 276 pid_t btPID; 277 278 uintptr_t bt[kIOTrackingCallSiteBTs]; 279 IOTrackingCallSiteUser user[0]; 280 }; 281 282 struct IOTrackingCallSiteWithUser { 283 struct IOTrackingCallSite site; 284 struct IOTrackingCallSiteUser user; 285 }; 286 287 struct IOTrackingLeaksRef { 288 uintptr_t * instances; 289 uint32_t zoneSize; 290 uint32_t count; 291 uint32_t found; 292 uint32_t foundzlen; 293 size_t bytes; 294 }; 295 296 lck_mtx_t * gIOTrackingLock; 297 queue_head_t gIOTrackingQ; 298 299 enum{ 300 kTrackingAddressFlagAllocated = 0x00000001 301 }; 302 303 #if defined(__LP64__) 304 #define IOTrackingAddressFlags(ptr) (ptr->flags) 305 #else 306 #define IOTrackingAddressFlags(ptr) (ptr->tracking.flags) 307 #endif 308 309 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 310 311 static void 312 IOTRecursiveLockLock(IOTRecursiveLock * lock) 313 { 314 if (lock->thread == current_thread()) { 315 lock->count++; 316 } else { 317 lck_mtx_lock(lock->mutex); 318 assert(lock->thread == NULL); 319 assert(lock->count == 0); 320 lock->thread = current_thread(); 321 lock->count = 1; 322 } 323 } 324 325 static void 326 IOTRecursiveLockUnlock(IOTRecursiveLock * lock) 327 { 328 assert(lock->thread == current_thread()); 329 if (0 == (--lock->count)) { 330 lock->thread = NULL; 331 lck_mtx_unlock(lock->mutex); 332 } 333 } 334 335 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 336 337 void 338 IOTrackingInit(void) 339 { 340 queue_init(&gIOTrackingQ); 341 gIOTrackingLock = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL); 342 } 343 344 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 345 346 IOTrackingQueue * 347 IOTrackingQueueAlloc(const char * name, uintptr_t btEntry, 348 size_t allocSize, size_t minCaptureSize, 349 uint32_t type, uint32_t numSiteQs) 350 { 351 IOTrackingQueue * queue; 352 uint32_t idx; 353 354 if (!numSiteQs) { 355 numSiteQs = 1; 356 } 357 queue = kalloc_type(IOTrackingQueue, queue_head_t, numSiteQs, Z_WAITOK_ZERO); 358 queue->name = name; 359 queue->btEntry = btEntry; 360 queue->allocSize = allocSize; 361 queue->minCaptureSize = minCaptureSize; 362 queue->lock.mutex = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL); 363 queue->numSiteQs = numSiteQs; 364 queue->type = type; 365 enum { kFlags = (kIOTracking | kIOTrackingBoot) }; 366 queue->captureOn = (kFlags == (kFlags & gIOKitDebug)) 367 || (kIOTrackingQueueTypeDefaultOn & type); 368 369 for (idx = 0; idx < numSiteQs; idx++) { 370 queue_init(&queue->sites[idx]); 371 } 372 373 lck_mtx_lock(gIOTrackingLock); 374 queue_enter(&gIOTrackingQ, queue, IOTrackingQueue *, link); 375 lck_mtx_unlock(gIOTrackingLock); 376 377 return queue; 378 }; 379 380 void 381 IOTrackingQueueCollectUser(IOTrackingQueue * queue) 382 { 383 assert(0 == queue->siteCount); 384 queue->type |= kIOTrackingQueueTypeUser; 385 } 386 387 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 388 389 void 390 IOTrackingQueueFree(IOTrackingQueue * queue) 391 { 392 lck_mtx_lock(gIOTrackingLock); 393 IOTrackingReset(queue); 394 remque(&queue->link); 395 lck_mtx_unlock(gIOTrackingLock); 396 397 lck_mtx_free(queue->lock.mutex, IOLockGroup); 398 399 kfree_type(IOTrackingQueue, queue_head_t, queue->numSiteQs, queue); 400 }; 401 402 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 403 404 /* fasthash 405 * The MIT License 406 * 407 * Copyright (C) 2012 Zilong Tan ([email protected]) 408 * 409 * Permission is hereby granted, free of charge, to any person 410 * obtaining a copy of this software and associated documentation 411 * files (the "Software"), to deal in the Software without 412 * restriction, including without limitation the rights to use, copy, 413 * modify, merge, publish, distribute, sublicense, and/or sell copies 414 * of the Software, and to permit persons to whom the Software is 415 * furnished to do so, subject to the following conditions: 416 * 417 * The above copyright notice and this permission notice shall be 418 * included in all copies or substantial portions of the Software. 419 * 420 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 421 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 422 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 423 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 424 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 425 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 426 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 427 * SOFTWARE. 428 */ 429 430 431 // Compression function for Merkle-Damgard construction. 432 // This function is generated using the framework provided. 433 #define mix(h) ({ \ 434 (h) ^= (h) >> 23; \ 435 (h) *= 0x2127599bf4325c37ULL; \ 436 (h) ^= (h) >> 47; }) 437 438 static uint64_t 439 fasthash64(const void *buf, size_t len, uint64_t seed) 440 { 441 const uint64_t m = 0x880355f21e6d1965ULL; 442 const uint64_t *pos = (const uint64_t *)buf; 443 const uint64_t *end = pos + (len / 8); 444 const unsigned char *pos2; 445 uint64_t h = seed ^ (len * m); 446 uint64_t v; 447 448 while (pos != end) { 449 v = *pos++; 450 h ^= mix(v); 451 h *= m; 452 } 453 454 pos2 = (const unsigned char*)pos; 455 v = 0; 456 457 switch (len & 7) { 458 case 7: v ^= (uint64_t)pos2[6] << 48; 459 [[clang::fallthrough]]; 460 case 6: v ^= (uint64_t)pos2[5] << 40; 461 [[clang::fallthrough]]; 462 case 5: v ^= (uint64_t)pos2[4] << 32; 463 [[clang::fallthrough]]; 464 case 4: v ^= (uint64_t)pos2[3] << 24; 465 [[clang::fallthrough]]; 466 case 3: v ^= (uint64_t)pos2[2] << 16; 467 [[clang::fallthrough]]; 468 case 2: v ^= (uint64_t)pos2[1] << 8; 469 [[clang::fallthrough]]; 470 case 1: v ^= (uint64_t)pos2[0]; 471 h ^= mix(v); 472 h *= m; 473 } 474 475 return mix(h); 476 } 477 478 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 479 480 static uint32_t 481 fasthash32(const void *buf, size_t len, uint32_t seed) 482 { 483 // the following trick converts the 64-bit hashcode to Fermat 484 // residue, which shall retain information from both the higher 485 // and lower parts of hashcode. 486 uint64_t h = fasthash64(buf, len, seed); 487 return (uint32_t) (h - (h >> 32)); 488 } 489 490 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 491 492 void 493 IOTrackingAddUser(IOTrackingQueue * queue, IOTrackingUser * mem, vm_size_t size) 494 { 495 uint32_t num; 496 int pid; 497 498 if (!queue->captureOn) { 499 return; 500 } 501 if (size < queue->minCaptureSize) { 502 return; 503 } 504 505 assert(!mem->link.next); 506 507 num = backtrace(&mem->bt[0], kIOTrackingCallSiteBTs, NULL, NULL); 508 num = 0; 509 if ((kernel_task != current_task()) && (pid = proc_selfpid())) { 510 struct backtrace_user_info btinfo = BTUINFO_INIT; 511 mem->btPID = pid; 512 num = backtrace_user(&mem->btUser[0], kIOTrackingCallSiteBTs - 1, 513 NULL, &btinfo); 514 mem->user32 = !(btinfo.btui_info & BTI_64_BIT); 515 } 516 assert(num <= kIOTrackingCallSiteBTs); 517 static_assert(kIOTrackingCallSiteBTs <= UINT8_MAX); 518 mem->userCount = ((uint8_t) num); 519 520 IOTRecursiveLockLock(&queue->lock); 521 queue_enter/*last*/ (&queue->sites[0], mem, IOTrackingUser *, link); 522 queue->siteCount++; 523 IOTRecursiveLockUnlock(&queue->lock); 524 } 525 526 void 527 IOTrackingRemoveUser(IOTrackingQueue * queue, IOTrackingUser * mem) 528 { 529 if (!mem->link.next) { 530 return; 531 } 532 533 IOTRecursiveLockLock(&queue->lock); 534 if (mem->link.next) { 535 remque(&mem->link); 536 assert(queue->siteCount); 537 queue->siteCount--; 538 } 539 IOTRecursiveLockUnlock(&queue->lock); 540 } 541 542 uint64_t gIOTrackingAddTime; 543 544 void 545 IOTrackingAdd(IOTrackingQueue * queue, IOTracking * mem, size_t size, bool address, vm_tag_t tag) 546 { 547 IOTrackingCallSite * site; 548 uint32_t crc, num; 549 uintptr_t bt[kIOTrackingCallSiteBTs + 1]; 550 uintptr_t btUser[kIOTrackingCallSiteBTs]; 551 queue_head_t * que; 552 bool user; 553 int pid; 554 int userCount; 555 556 if (mem->site) { 557 return; 558 } 559 if (!queue->captureOn) { 560 return; 561 } 562 if (size < queue->minCaptureSize) { 563 return; 564 } 565 566 user = (0 != (kIOTrackingQueueTypeUser & queue->type)); 567 568 assert(!mem->link.next); 569 570 num = backtrace(&bt[0], kIOTrackingCallSiteBTs + 1, NULL, NULL); 571 if (!num) { 572 return; 573 } 574 num--; 575 crc = fasthash32(&bt[1], num * sizeof(bt[0]), 0x04C11DB7); 576 577 userCount = 0; 578 pid = 0; 579 backtrace_info_t btinfo = BTI_NONE; 580 if (user) { 581 if ((kernel_task != current_task()) && (pid = proc_selfpid())) { 582 struct backtrace_user_info btuinfo = BTUINFO_INIT; 583 userCount = backtrace_user(&btUser[0], kIOTrackingCallSiteBTs, 584 NULL, &btuinfo); 585 assert(userCount <= kIOTrackingCallSiteBTs); 586 btinfo = btuinfo.btui_info; 587 crc = fasthash32(&btUser[0], userCount * sizeof(bt[0]), crc); 588 } 589 } 590 591 IOTRecursiveLockLock(&queue->lock); 592 que = &queue->sites[crc % queue->numSiteQs]; 593 queue_iterate(que, site, IOTrackingCallSite *, link) 594 { 595 if (tag != site->tag) { 596 continue; 597 } 598 if (user && (pid != site->user[0].pid)) { 599 continue; 600 } 601 if (crc == site->crc) { 602 break; 603 } 604 } 605 606 if (queue_end(que, (queue_entry_t) site)) { 607 if (user) { 608 site = &kalloc_type(IOTrackingCallSiteWithUser, 609 Z_WAITOK_ZERO_NOFAIL)->site; 610 } else { 611 site = kalloc_type(IOTrackingCallSite, 612 Z_WAITOK_ZERO_NOFAIL); 613 } 614 615 queue_init(&site->instances); 616 site->addresses = (IOTracking *) &site->instances; 617 site->queue = queue; 618 site->crc = crc; 619 site->count = 0; 620 site->tag = tag; 621 memset(&site->size[0], 0, sizeof(site->size)); 622 bcopy(&bt[1], &site->bt[0], num * sizeof(site->bt[0])); 623 assert(num <= kIOTrackingCallSiteBTs); 624 bzero(&site->bt[num], (kIOTrackingCallSiteBTs - num) * sizeof(site->bt[0])); 625 if (user) { 626 bcopy(&btUser[0], &site->user[0].bt[0], userCount * sizeof(site->user[0].bt[0])); 627 assert(userCount <= kIOTrackingCallSiteBTs); 628 bzero(&site->user[0].bt[userCount], (kIOTrackingCallSiteBTs - userCount) * sizeof(site->user[0].bt[0])); 629 site->user[0].pid = pid; 630 site->user[0].user32 = !(btinfo & BTI_64_BIT); 631 static_assert(kIOTrackingCallSiteBTs <= UINT8_MAX); 632 site->user[0].userCount = ((uint8_t) userCount); 633 } 634 queue_enter_first(que, site, IOTrackingCallSite *, link); 635 queue->siteCount++; 636 } 637 638 if (address) { 639 queue_enter/*last*/ (&site->instances, mem, IOTracking *, link); 640 if (queue_end(&site->instances, (queue_entry_t)site->addresses)) { 641 site->addresses = mem; 642 } 643 } else { 644 queue_enter_first(&site->instances, mem, IOTracking *, link); 645 } 646 647 mem->site = site; 648 site->size[0] += size; 649 site->count++; 650 651 IOTRecursiveLockUnlock(&queue->lock); 652 } 653 654 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 655 656 void 657 IOTrackingRemove(IOTrackingQueue * queue, IOTracking * mem, size_t size) 658 { 659 if (!mem->link.next) { 660 return; 661 } 662 663 IOTRecursiveLockLock(&queue->lock); 664 if (mem->link.next) { 665 assert(mem->site); 666 667 if (mem == mem->site->addresses) { 668 mem->site->addresses = (IOTracking *) queue_next(&mem->link); 669 } 670 remque(&mem->link); 671 672 assert(mem->site->count); 673 mem->site->count--; 674 assert(mem->site->size[0] >= size); 675 mem->site->size[0] -= size; 676 if (!mem->site->count) { 677 assert(queue_empty(&mem->site->instances)); 678 assert(!mem->site->size[0]); 679 assert(!mem->site->size[1]); 680 681 remque(&mem->site->link); 682 assert(queue->siteCount); 683 queue->siteCount--; 684 if (kIOTrackingQueueTypeUser & queue->type) { 685 kfree_type(IOTrackingCallSiteWithUser, mem->site); 686 } else { 687 kfree_type(IOTrackingCallSite, mem->site); 688 } 689 } 690 mem->site = NULL; 691 } 692 IOTRecursiveLockUnlock(&queue->lock); 693 } 694 695 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 696 697 void 698 IOTrackingAlloc(IOTrackingQueue * queue, uintptr_t address, size_t size) 699 { 700 IOTrackingAddress * tracking; 701 702 if (!queue->captureOn) { 703 return; 704 } 705 if (size < queue->minCaptureSize) { 706 return; 707 } 708 709 address = ~address; 710 tracking = kalloc_type(IOTrackingAddress, (zalloc_flags_t)(Z_WAITOK | Z_ZERO)); 711 IOTrackingAddressFlags(tracking) |= kTrackingAddressFlagAllocated; 712 tracking->address = address; 713 tracking->size = size; 714 715 IOTrackingAdd(queue, &tracking->tracking, size, true, VM_KERN_MEMORY_NONE); 716 } 717 718 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 719 720 void 721 IOTrackingFree(IOTrackingQueue * queue, uintptr_t address, size_t size) 722 { 723 IOTrackingCallSite * site; 724 IOTrackingAddress * tracking; 725 uint32_t idx; 726 bool done; 727 728 address = ~address; 729 IOTRecursiveLockLock(&queue->lock); 730 done = false; 731 for (idx = 0; idx < queue->numSiteQs; idx++) { 732 queue_iterate(&queue->sites[idx], site, IOTrackingCallSite *, link) 733 { 734 tracking = (IOTrackingAddress *) site->addresses; 735 while (!queue_end(&site->instances, &tracking->tracking.link)) { 736 if ((done = (address == tracking->address))) { 737 IOTrackingRemove(queue, &tracking->tracking, size); 738 kfree_type(IOTrackingAddress, tracking); 739 break; 740 } else { 741 tracking = (IOTrackingAddress *) queue_next(&tracking->tracking.link); 742 } 743 } 744 if (done) { 745 break; 746 } 747 } 748 if (done) { 749 break; 750 } 751 } 752 IOTRecursiveLockUnlock(&queue->lock); 753 } 754 755 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 756 757 void 758 IOTrackingAccumSize(IOTrackingQueue * queue, IOTracking * mem, size_t size) 759 { 760 IOTRecursiveLockLock(&queue->lock); 761 if (mem->link.next) { 762 assert(mem->site); 763 assert((size > 0) || (mem->site->size[1] >= -size)); 764 mem->site->size[1] += size; 765 } 766 ; 767 IOTRecursiveLockUnlock(&queue->lock); 768 } 769 770 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 771 772 void 773 IOTrackingReset(IOTrackingQueue * queue) 774 { 775 IOTrackingCallSite * site; 776 IOTrackingUser * user; 777 IOTracking * tracking; 778 IOTrackingAddress * trackingAddress; 779 uint32_t idx; 780 bool addresses; 781 782 IOTRecursiveLockLock(&queue->lock); 783 for (idx = 0; idx < queue->numSiteQs; idx++) { 784 while (!queue_empty(&queue->sites[idx])) { 785 if (kIOTrackingQueueTypeMap & queue->type) { 786 queue_remove_first(&queue->sites[idx], user, IOTrackingUser *, link); 787 user->link.next = user->link.prev = NULL; 788 } else { 789 queue_remove_first(&queue->sites[idx], site, IOTrackingCallSite *, link); 790 addresses = false; 791 while (!queue_empty(&site->instances)) { 792 queue_remove_first(&site->instances, tracking, IOTracking *, link); 793 if (tracking == site->addresses) { 794 addresses = true; 795 } 796 if (addresses) { 797 trackingAddress = (typeof(trackingAddress))tracking; 798 if (kTrackingAddressFlagAllocated & IOTrackingAddressFlags(trackingAddress)) { 799 kfree_type(IOTrackingAddress, tracking); 800 } 801 } 802 } 803 if (kIOTrackingQueueTypeUser & queue->type) { 804 kfree_type(IOTrackingCallSiteWithUser, site); 805 } else { 806 kfree_type(IOTrackingCallSite, site); 807 } 808 } 809 } 810 } 811 queue->siteCount = 0; 812 IOTRecursiveLockUnlock(&queue->lock); 813 } 814 815 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 816 817 static int 818 IOTrackingCallSiteInfoCompare(const void * left, const void * right) 819 { 820 IOTrackingCallSiteInfo * l = (typeof(l))left; 821 IOTrackingCallSiteInfo * r = (typeof(r))right; 822 size_t lsize, rsize; 823 824 rsize = r->size[0] + r->size[1]; 825 lsize = l->size[0] + l->size[1]; 826 827 return (rsize > lsize) ? 1 : ((rsize == lsize) ? 0 : -1); 828 } 829 830 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 831 832 static int 833 IOTrackingAddressCompare(const void * left, const void * right) 834 { 835 IOTracking * instance; 836 uintptr_t inst, laddr, raddr; 837 838 inst = ((typeof(inst) *)left)[0]; 839 instance = (typeof(instance))INSTANCE_GET(inst); 840 if (kInstanceFlagAddress & inst) { 841 laddr = ~((IOTrackingAddress *)instance)->address; 842 } else { 843 laddr = (uintptr_t) (instance + 1); 844 } 845 846 inst = ((typeof(inst) *)right)[0]; 847 instance = (typeof(instance))(inst & ~kInstanceFlags); 848 if (kInstanceFlagAddress & inst) { 849 raddr = ~((IOTrackingAddress *)instance)->address; 850 } else { 851 raddr = (uintptr_t) (instance + 1); 852 } 853 854 return (laddr > raddr) ? 1 : ((laddr == raddr) ? 0 : -1); 855 } 856 857 858 static int 859 IOTrackingZoneElementCompare(const void * left, const void * right) 860 { 861 uintptr_t inst, laddr, raddr; 862 863 inst = ((typeof(inst) *)left)[0]; 864 laddr = INSTANCE_PUT(inst); 865 inst = ((typeof(inst) *)right)[0]; 866 raddr = INSTANCE_PUT(inst); 867 868 return (laddr > raddr) ? 1 : ((laddr == raddr) ? 0 : -1); 869 } 870 871 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 872 873 static void 874 CopyOutBacktraces(IOTrackingCallSite * site, IOTrackingCallSiteInfo * siteInfo) 875 { 876 uint32_t j; 877 mach_vm_address_t bt, btEntry; 878 879 btEntry = site->queue->btEntry; 880 for (j = 0; j < kIOTrackingCallSiteBTs; j++) { 881 bt = site->bt[j]; 882 if (btEntry 883 && (!bt || (j == (kIOTrackingCallSiteBTs - 1)))) { 884 bt = btEntry; 885 btEntry = 0; 886 } 887 siteInfo->bt[0][j] = VM_KERNEL_UNSLIDE(bt); 888 } 889 890 siteInfo->btPID = 0; 891 if (kIOTrackingQueueTypeUser & site->queue->type) { 892 siteInfo->btPID = site->user[0].pid; 893 uint32_t * bt32 = (typeof(bt32))((void *) &site->user[0].bt[0]); 894 uint64_t * bt64 = (typeof(bt64))((void *) &site->user[0].bt[0]); 895 for (uint32_t j = 0; j < kIOTrackingCallSiteBTs; j++) { 896 if (j >= site->user[0].userCount) { 897 siteInfo->bt[1][j] = 0; 898 } else if (site->user[0].user32) { 899 siteInfo->bt[1][j] = bt32[j]; 900 } else { 901 siteInfo->bt[1][j] = bt64[j]; 902 } 903 } 904 } 905 } 906 907 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 908 909 static void 910 IOTrackingLeakScan(void * refcon) 911 { 912 IOTrackingLeaksRef * ref = (typeof(ref))refcon; 913 uintptr_t * instances; 914 IOTracking * instance; 915 uint64_t vaddr, vincr; 916 ppnum_t ppn; 917 uintptr_t ptr, addr, vphysaddr, inst; 918 size_t size, origsize; 919 uint32_t baseIdx, lim, ptrIdx, count; 920 boolean_t is; 921 AbsoluteTime deadline; 922 923 instances = ref->instances; 924 count = ref->count; 925 size = origsize = ref->zoneSize; 926 927 for (deadline = 0, vaddr = VM_MIN_KERNEL_AND_KEXT_ADDRESS; 928 ; 929 vaddr += vincr) { 930 if ((mach_absolute_time() > deadline) || (vaddr >= VM_MAX_KERNEL_ADDRESS)) { 931 if (deadline) { 932 ml_set_interrupts_enabled(is); 933 IODelay(10); 934 } 935 if (vaddr >= VM_MAX_KERNEL_ADDRESS) { 936 break; 937 } 938 is = ml_set_interrupts_enabled(false); 939 clock_interval_to_deadline(10, kMillisecondScale, &deadline); 940 } 941 942 ppn = kernel_pmap_present_mapping(vaddr, &vincr, &vphysaddr); 943 // check noencrypt to avoid VM structs (map entries) with pointers 944 if (ppn && (!pmap_valid_page(ppn) || (!ref->zoneSize && pmap_is_noencrypt(ppn)))) { 945 ppn = 0; 946 } 947 if (!ppn) { 948 continue; 949 } 950 951 for (ptrIdx = 0; ptrIdx < (page_size / sizeof(uintptr_t)); ptrIdx++) { 952 ptr = ((uintptr_t *)vphysaddr)[ptrIdx]; 953 #if defined(HAS_APPLE_PAC) 954 // strip possible ptrauth signature from candidate data pointer 955 ptr = (uintptr_t)ptrauth_strip((void*)ptr, ptrauth_key_process_independent_data); 956 #endif /* defined(HAS_APPLE_PAC) */ 957 958 for (lim = count, baseIdx = 0; lim; lim >>= 1) { 959 inst = instances[baseIdx + (lim >> 1)]; 960 instance = (typeof(instance))INSTANCE_GET(inst); 961 962 if (ref->zoneSize) { 963 addr = INSTANCE_PUT(inst) & ~kInstanceFlags; 964 } else if (kInstanceFlagAddress & inst) { 965 addr = ~((IOTrackingAddress *)instance)->address; 966 origsize = size = ((IOTrackingAddress *)instance)->size; 967 if (!size) { 968 size = 1; 969 } 970 } else { 971 addr = (uintptr_t) (instance + 1); 972 origsize = size = instance->site->queue->allocSize; 973 } 974 if ((ptr >= addr) && (ptr < (addr + size)) 975 976 && (((vaddr + ptrIdx * sizeof(uintptr_t)) < addr) 977 || ((vaddr + ptrIdx * sizeof(uintptr_t)) >= (addr + size)))) { 978 if (!(kInstanceFlagReferenced & inst)) { 979 inst |= kInstanceFlagReferenced; 980 instances[baseIdx + (lim >> 1)] = inst; 981 ref->found++; 982 if (!origsize) { 983 ref->foundzlen++; 984 } 985 } 986 break; 987 } 988 if (ptr > addr) { 989 // move right 990 baseIdx += (lim >> 1) + 1; 991 lim--; 992 } 993 // else move left 994 } 995 } 996 ref->bytes += page_size; 997 } 998 } 999 1000 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1001 1002 extern "C" void 1003 zone_leaks_scan(uintptr_t * instances, uint32_t count, uint32_t zoneSize, uint32_t * found) 1004 { 1005 IOTrackingLeaksRef ref; 1006 IOTrackingCallSiteInfo siteInfo; 1007 uint32_t idx; 1008 1009 qsort(instances, count, sizeof(*instances), &IOTrackingZoneElementCompare); 1010 1011 bzero(&siteInfo, sizeof(siteInfo)); 1012 bzero(&ref, sizeof(ref)); 1013 ref.instances = instances; 1014 ref.count = count; 1015 ref.zoneSize = zoneSize; 1016 1017 for (idx = 0; idx < 2; idx++) { 1018 ref.bytes = 0; 1019 IOTrackingLeakScan(&ref); 1020 IOLog("leaks(%d) scanned %ld MB, instance count %d, found %d\n", idx, ref.bytes / 1024 / 1024, count, ref.found); 1021 if (count <= ref.found) { 1022 break; 1023 } 1024 } 1025 1026 *found = ref.found; 1027 } 1028 1029 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1030 1031 static void 1032 ZoneSiteProc(void * refCon, uint32_t siteCount, uint32_t zoneSize, 1033 uintptr_t * btrace, uint32_t btCount) 1034 { 1035 IOTrackingCallSiteInfo siteInfo; 1036 OSData * leakData; 1037 uint32_t idx; 1038 1039 leakData = (typeof(leakData))refCon; 1040 1041 bzero(&siteInfo, sizeof(siteInfo)); 1042 siteInfo.count = siteCount; 1043 siteInfo.size[0] = zoneSize * siteCount; 1044 1045 for (idx = 0; (idx < btCount) && (idx < kIOTrackingCallSiteBTs); idx++) { 1046 siteInfo.bt[0][idx] = VM_KERNEL_UNSLIDE(btrace[idx]); 1047 } 1048 1049 leakData->appendBytes(&siteInfo, sizeof(siteInfo)); 1050 } 1051 1052 1053 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1054 1055 static OSData * 1056 IOTrackingLeaks(LIBKERN_CONSUMED OSData * data) 1057 { 1058 IOTrackingLeaksRef ref; 1059 IOTrackingCallSiteInfo siteInfo; 1060 IOTrackingCallSite * site; 1061 OSData * leakData; 1062 uintptr_t * instances; 1063 IOTracking * instance; 1064 uintptr_t inst; 1065 uint32_t count, idx, numSites, dups, siteCount; 1066 1067 instances = (typeof(instances))data->getBytesNoCopy(); 1068 count = (data->getLength() / sizeof(*instances)); 1069 qsort(instances, count, sizeof(*instances), &IOTrackingAddressCompare); 1070 1071 bzero(&siteInfo, sizeof(siteInfo)); 1072 bzero(&ref, sizeof(ref)); 1073 ref.instances = instances; 1074 ref.count = count; 1075 for (idx = 0; idx < 2; idx++) { 1076 ref.bytes = 0; 1077 IOTrackingLeakScan(&ref); 1078 IOLog("leaks(%d) scanned %ld MB, instance count %d, found %d (zlen %d)\n", idx, ref.bytes / 1024 / 1024, count, ref.found, ref.foundzlen); 1079 if (count <= ref.found) { 1080 break; 1081 } 1082 } 1083 1084 leakData = OSData::withCapacity(128 * sizeof(IOTrackingCallSiteInfo)); 1085 1086 for (numSites = 0, idx = 0; idx < count; idx++) { 1087 inst = instances[idx]; 1088 if (kInstanceFlagReferenced & inst) { 1089 continue; 1090 } 1091 instance = (typeof(instance))INSTANCE_GET(inst); 1092 site = instance->site; 1093 instances[numSites] = (uintptr_t) site; 1094 numSites++; 1095 } 1096 1097 for (idx = 0; idx < numSites; idx++) { 1098 inst = instances[idx]; 1099 if (!inst) { 1100 continue; 1101 } 1102 site = (typeof(site))inst; 1103 for (siteCount = 1, dups = (idx + 1); dups < numSites; dups++) { 1104 if (instances[dups] == (uintptr_t) site) { 1105 siteCount++; 1106 instances[dups] = 0; 1107 } 1108 } 1109 siteInfo.count = siteCount; 1110 siteInfo.size[0] = (site->size[0] * site->count) / siteCount; 1111 siteInfo.size[1] = (site->size[1] * site->count) / siteCount; 1112 CopyOutBacktraces(site, &siteInfo); 1113 leakData->appendBytes(&siteInfo, sizeof(siteInfo)); 1114 } 1115 data->release(); 1116 1117 return leakData; 1118 } 1119 1120 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1121 1122 static bool 1123 SkipName(uint32_t options, const char * name, size_t namesLen, const char * names) 1124 { 1125 const char * scan; 1126 const char * next; 1127 bool exclude, found; 1128 size_t qLen, sLen; 1129 1130 if (!namesLen || !names) { 1131 return false; 1132 } 1133 // <len><name>...<len><name><0> 1134 exclude = (0 != (kIOTrackingExcludeNames & options)); 1135 qLen = strlen(name); 1136 scan = names; 1137 found = false; 1138 do{ 1139 sLen = scan[0]; 1140 scan++; 1141 next = scan + sLen; 1142 if (next >= (names + namesLen)) { 1143 break; 1144 } 1145 found = ((sLen == qLen) && !strncmp(scan, name, sLen)); 1146 scan = next; 1147 }while (!found && (scan < (names + namesLen))); 1148 1149 return !(exclude ^ found); 1150 } 1151 1152 #endif /* IOTRACKING */ 1153 1154 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1155 1156 static kern_return_t 1157 IOTrackingDebug(uint32_t selector, uint32_t options, uint64_t value, 1158 uint32_t intag, uint32_t inzsize, 1159 const char * names, size_t namesLen, 1160 size_t size, OSObject ** result) 1161 { 1162 kern_return_t ret; 1163 OSData * data; 1164 1165 if (result) { 1166 *result = NULL; 1167 } 1168 data = NULL; 1169 ret = kIOReturnNotReady; 1170 1171 #if IOTRACKING 1172 1173 kern_return_t kr; 1174 IOTrackingQueue * queue; 1175 IOTracking * instance; 1176 IOTrackingCallSite * site; 1177 IOTrackingCallSiteInfo siteInfo; 1178 IOTrackingUser * user; 1179 task_t mapTask; 1180 mach_vm_address_t mapAddress; 1181 mach_vm_size_t mapSize; 1182 uint32_t num, idx, qIdx; 1183 uintptr_t instFlags; 1184 proc_t proc; 1185 bool addresses; 1186 1187 ret = kIOReturnNotFound; 1188 proc = NULL; 1189 if (kIOTrackingGetMappings == selector) { 1190 if (value != -1ULL) { 1191 proc = proc_find((pid_t) value); 1192 if (!proc) { 1193 return kIOReturnNotFound; 1194 } 1195 } 1196 } 1197 1198 bzero(&siteInfo, sizeof(siteInfo)); 1199 lck_mtx_lock(gIOTrackingLock); 1200 queue_iterate(&gIOTrackingQ, queue, IOTrackingQueue *, link) 1201 { 1202 if (SkipName(options, queue->name, namesLen, names)) { 1203 continue; 1204 } 1205 1206 if (!(kIOTracking & gIOKitDebug) && (kIOTrackingQueueTypeAlloc & queue->type)) { 1207 continue; 1208 } 1209 1210 switch (selector) { 1211 case kIOTrackingResetTracking: 1212 { 1213 IOTrackingReset(queue); 1214 ret = kIOReturnSuccess; 1215 break; 1216 } 1217 1218 case kIOTrackingStartCapture: 1219 case kIOTrackingStopCapture: 1220 { 1221 queue->captureOn = (kIOTrackingStartCapture == selector); 1222 ret = kIOReturnSuccess; 1223 break; 1224 } 1225 1226 case kIOTrackingSetMinCaptureSize: 1227 { 1228 queue->minCaptureSize = size; 1229 ret = kIOReturnSuccess; 1230 break; 1231 } 1232 1233 case kIOTrackingLeaks: 1234 { 1235 if (!(kIOTrackingQueueTypeAlloc & queue->type)) { 1236 break; 1237 } 1238 1239 if (!data) { 1240 data = OSData::withCapacity(1024 * sizeof(uintptr_t)); 1241 } 1242 1243 IOTRecursiveLockLock(&queue->lock); 1244 for (idx = 0; idx < queue->numSiteQs; idx++) { 1245 queue_iterate(&queue->sites[idx], site, IOTrackingCallSite *, link) 1246 { 1247 addresses = false; 1248 queue_iterate(&site->instances, instance, IOTracking *, link) 1249 { 1250 if (instance == site->addresses) { 1251 addresses = true; 1252 } 1253 instFlags = (typeof(instFlags))instance; 1254 if (addresses) { 1255 instFlags |= kInstanceFlagAddress; 1256 } 1257 data->appendBytes(&instFlags, sizeof(instFlags)); 1258 } 1259 } 1260 } 1261 // queue is locked 1262 ret = kIOReturnSuccess; 1263 break; 1264 } 1265 1266 1267 case kIOTrackingGetTracking: 1268 { 1269 if (kIOTrackingQueueTypeMap & queue->type) { 1270 break; 1271 } 1272 1273 if (!data) { 1274 data = OSData::withCapacity(128 * sizeof(IOTrackingCallSiteInfo)); 1275 } 1276 1277 IOTRecursiveLockLock(&queue->lock); 1278 num = queue->siteCount; 1279 idx = 0; 1280 for (qIdx = 0; qIdx < queue->numSiteQs; qIdx++) { 1281 queue_iterate(&queue->sites[qIdx], site, IOTrackingCallSite *, link) 1282 { 1283 assert(idx < num); 1284 idx++; 1285 1286 size_t tsize[2]; 1287 uint32_t count = site->count; 1288 tsize[0] = site->size[0]; 1289 tsize[1] = site->size[1]; 1290 1291 if (intag || inzsize) { 1292 uintptr_t addr; 1293 vm_size_t size, zoneSize; 1294 vm_tag_t tag; 1295 1296 if (kIOTrackingQueueTypeAlloc & queue->type) { 1297 addresses = false; 1298 count = 0; 1299 tsize[0] = tsize[1] = 0; 1300 queue_iterate(&site->instances, instance, IOTracking *, link) 1301 { 1302 if (instance == site->addresses) { 1303 addresses = true; 1304 } 1305 1306 if (addresses) { 1307 addr = ~((IOTrackingAddress *)instance)->address; 1308 } else { 1309 addr = (uintptr_t) (instance + 1); 1310 } 1311 1312 kr = vm_kern_allocation_info(addr, &size, &tag, &zoneSize); 1313 if (KERN_SUCCESS != kr) { 1314 continue; 1315 } 1316 1317 if ((VM_KERN_MEMORY_NONE != intag) && (intag != tag)) { 1318 continue; 1319 } 1320 if (inzsize && (inzsize != zoneSize)) { 1321 continue; 1322 } 1323 1324 count++; 1325 tsize[0] += size; 1326 } 1327 } else { 1328 if (!intag || inzsize || (intag != site->tag)) { 1329 continue; 1330 } 1331 } 1332 } 1333 1334 if (!count) { 1335 continue; 1336 } 1337 if (size && ((tsize[0] + tsize[1]) < size)) { 1338 continue; 1339 } 1340 siteInfo.count = count; 1341 siteInfo.size[0] = tsize[0]; 1342 siteInfo.size[1] = tsize[1]; 1343 CopyOutBacktraces(site, &siteInfo); 1344 data->appendBytes(&siteInfo, sizeof(siteInfo)); 1345 } 1346 } 1347 assert(idx == num); 1348 IOTRecursiveLockUnlock(&queue->lock); 1349 ret = kIOReturnSuccess; 1350 break; 1351 } 1352 1353 case kIOTrackingGetMappings: 1354 { 1355 if (!(kIOTrackingQueueTypeMap & queue->type)) { 1356 break; 1357 } 1358 if (!data) { 1359 data = OSData::withCapacity((unsigned int) page_size); 1360 } 1361 1362 IOTRecursiveLockLock(&queue->lock); 1363 num = queue->siteCount; 1364 idx = 0; 1365 for (qIdx = 0; qIdx < queue->numSiteQs; qIdx++) { 1366 queue_iterate(&queue->sites[qIdx], user, IOTrackingUser *, link) 1367 { 1368 assert(idx < num); 1369 idx++; 1370 1371 kr = IOMemoryMapTracking(user, &mapTask, &mapAddress, &mapSize); 1372 if (kIOReturnSuccess != kr) { 1373 continue; 1374 } 1375 if (proc && (mapTask != proc_task(proc))) { 1376 continue; 1377 } 1378 if (size && (mapSize < size)) { 1379 continue; 1380 } 1381 1382 siteInfo.count = 1; 1383 siteInfo.size[0] = mapSize; 1384 siteInfo.address = mapAddress; 1385 siteInfo.addressPID = task_pid(mapTask); 1386 siteInfo.btPID = user->btPID; 1387 1388 for (uint32_t j = 0; j < kIOTrackingCallSiteBTs; j++) { 1389 siteInfo.bt[0][j] = VM_KERNEL_UNSLIDE(user->bt[j]); 1390 } 1391 uint32_t * bt32 = (typeof(bt32)) & user->btUser[0]; 1392 uint64_t * bt64 = (typeof(bt64))((void *) &user->btUser[0]); 1393 for (uint32_t j = 0; j < kIOTrackingCallSiteBTs; j++) { 1394 if (j >= user->userCount) { 1395 siteInfo.bt[1][j] = 0; 1396 } else if (user->user32) { 1397 siteInfo.bt[1][j] = bt32[j]; 1398 } else { 1399 siteInfo.bt[1][j] = bt64[j]; 1400 } 1401 } 1402 data->appendBytes(&siteInfo, sizeof(siteInfo)); 1403 } 1404 } 1405 assert(idx == num); 1406 IOTRecursiveLockUnlock(&queue->lock); 1407 ret = kIOReturnSuccess; 1408 break; 1409 } 1410 1411 default: 1412 ret = kIOReturnUnsupported; 1413 break; 1414 } 1415 } 1416 1417 if ((kIOTrackingLeaks == selector) && data) { 1418 data = IOTrackingLeaks(data); 1419 queue_iterate(&gIOTrackingQ, queue, IOTrackingQueue *, link) 1420 { 1421 if (SkipName(options, queue->name, namesLen, names)) { 1422 continue; 1423 } 1424 if (!(kIOTrackingQueueTypeAlloc & queue->type)) { 1425 continue; 1426 } 1427 IOTRecursiveLockUnlock(&queue->lock); 1428 } 1429 } 1430 1431 lck_mtx_unlock(gIOTrackingLock); 1432 1433 if ((kIOTrackingLeaks == selector) && namesLen && names) { 1434 const char * scan; 1435 const char * next; 1436 uint8_t sLen; 1437 1438 if (!data) { 1439 data = OSData::withCapacity(4096 * sizeof(uintptr_t)); 1440 } 1441 1442 // <len><name>...<len><name><0> 1443 scan = names; 1444 do{ 1445 sLen = ((uint8_t) scan[0]); 1446 scan++; 1447 next = scan + sLen; 1448 if (next >= (names + namesLen)) { 1449 break; 1450 } 1451 kr = zone_leaks(scan, sLen, &ZoneSiteProc, data); 1452 if (KERN_SUCCESS == kr) { 1453 ret = kIOReturnSuccess; 1454 } else if (KERN_INVALID_NAME != kr) { 1455 ret = kIOReturnVMError; 1456 } 1457 scan = next; 1458 }while (scan < (names + namesLen)); 1459 } 1460 1461 if (data) { 1462 switch (selector) { 1463 case kIOTrackingLeaks: 1464 case kIOTrackingGetTracking: 1465 case kIOTrackingGetMappings: 1466 { 1467 IOTrackingCallSiteInfo * siteInfos; 1468 siteInfos = (typeof(siteInfos))data->getBytesNoCopy(); 1469 num = (data->getLength() / sizeof(*siteInfos)); 1470 qsort(siteInfos, num, sizeof(*siteInfos), &IOTrackingCallSiteInfoCompare); 1471 break; 1472 } 1473 default: assert(false); break; 1474 } 1475 } 1476 1477 *result = data; 1478 if (proc) { 1479 proc_rele(proc); 1480 } 1481 1482 #endif /* IOTRACKING */ 1483 1484 return ret; 1485 } 1486 1487 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1488 1489 #include <IOKit/IOKitDiagnosticsUserClient.h> 1490 1491 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1492 1493 #undef super 1494 #define super IOUserClient 1495 1496 OSDefineMetaClassAndStructors(IOKitDiagnosticsClient, IOUserClient) 1497 1498 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1499 1500 IOUserClient * IOKitDiagnosticsClient::withTask(task_t owningTask) 1501 { 1502 IOKitDiagnosticsClient * inst; 1503 1504 inst = new IOKitDiagnosticsClient; 1505 if (inst && !inst->init()) { 1506 inst->release(); 1507 inst = NULL; 1508 } 1509 1510 return inst; 1511 } 1512 1513 IOReturn 1514 IOKitDiagnosticsClient::clientClose(void) 1515 { 1516 terminate(); 1517 return kIOReturnSuccess; 1518 } 1519 1520 IOReturn 1521 IOKitDiagnosticsClient::setProperties(OSObject * properties) 1522 { 1523 IOReturn kr = kIOReturnUnsupported; 1524 return kr; 1525 } 1526 1527 IOReturn 1528 IOKitDiagnosticsClient::externalMethod(uint32_t selector, IOExternalMethodArguments * args, 1529 IOExternalMethodDispatch * dispatch, OSObject * target, void * reference) 1530 { 1531 IOReturn ret = kIOReturnBadArgument; 1532 const IOKitDiagnosticsParameters * params; 1533 const char * names; 1534 size_t namesLen; 1535 OSObject * result; 1536 1537 if (args->structureInputSize < sizeof(IOKitDiagnosticsParameters)) { 1538 return kIOReturnBadArgument; 1539 } 1540 params = (typeof(params))args->structureInput; 1541 if (!params) { 1542 return kIOReturnBadArgument; 1543 } 1544 1545 names = NULL; 1546 namesLen = args->structureInputSize - sizeof(IOKitDiagnosticsParameters); 1547 if (namesLen) { 1548 names = (typeof(names))(params + 1); 1549 } 1550 1551 ret = IOTrackingDebug(selector, params->options, params->value, params->tag, params->zsize, names, namesLen, params->size, &result); 1552 1553 if ((kIOReturnSuccess == ret) && args->structureVariableOutputData) { 1554 *args->structureVariableOutputData = result; 1555 } else if (result) { 1556 result->release(); 1557 } 1558 1559 return ret; 1560 } 1561 1562 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1563