1 /* 2 * Copyright (c) 1998-2010 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 30 #include <sys/sysctl.h> 31 extern "C" { 32 #include <vm/vm_kern.h> 33 } 34 35 #include <libkern/c++/OSContainers.h> 36 #include <libkern/OSDebug.h> 37 #include <libkern/c++/OSCPPDebug.h> 38 39 #include <IOKit/IOKitDebug.h> 40 #include <IOKit/IOLib.h> 41 #include <IOKit/assert.h> 42 #include <IOKit/IODeviceTreeSupport.h> 43 #include <IOKit/IOService.h> 44 45 #ifdef IOKITDEBUG 46 #define DEBUG_INIT_VALUE IOKITDEBUG 47 #else 48 #define DEBUG_INIT_VALUE 0 49 #endif 50 51 SInt64 gIOKitDebug = DEBUG_INIT_VALUE; 52 SInt64 gIOKitTrace = 0; 53 54 #if DEVELOPMENT || DEBUG 55 #define IODEBUG_CTLFLAGS CTLFLAG_RW 56 #else 57 #define IODEBUG_CTLFLAGS CTLFLAG_RD 58 #endif 59 60 SYSCTL_QUAD(_debug, OID_AUTO, iokit, IODEBUG_CTLFLAGS | CTLFLAG_LOCKED, &gIOKitDebug, "boot_arg io"); 61 SYSCTL_QUAD(_debug, OID_AUTO, iotrace, CTLFLAG_RW | CTLFLAG_LOCKED, &gIOKitTrace, "trace io"); 62 63 64 int debug_malloc_size; 65 int debug_iomalloc_size; 66 67 vm_size_t debug_iomallocpageable_size; 68 int debug_container_malloc_size; 69 // int debug_ivars_size; // in OSObject.cpp 70 71 extern "C" { 72 73 #if 0 74 #define DEBG(fmt, args...) { kprintf(fmt, ## args); } 75 #else 76 #define DEBG(fmt, args...) { IOLog(fmt, ## args); } 77 #endif 78 79 void IOPrintPlane( const IORegistryPlane * plane ) 80 { 81 IORegistryEntry * next; 82 IORegistryIterator * iter; 83 OSOrderedSet * all; 84 char format[] = "%xxxs"; 85 IOService * service; 86 87 iter = IORegistryIterator::iterateOver( plane ); 88 assert( iter ); 89 all = iter->iterateAll(); 90 if( all) { 91 DEBG("Count %d\n", all->getCount() ); 92 all->release(); 93 } else 94 DEBG("Empty\n"); 95 96 iter->reset(); 97 while( (next = iter->getNextObjectRecursive())) { 98 snprintf(format + 1, sizeof(format) - 1, "%ds", 2 * next->getDepth( plane )); 99 DEBG( format, ""); 100 DEBG( "\033[33m%s", next->getName( plane )); 101 if( (next->getLocation( plane ))) 102 DEBG("@%s", next->getLocation( plane )); 103 DEBG("\033[0m <class %s", next->getMetaClass()->getClassName()); 104 if( (service = OSDynamicCast(IOService, next))) 105 DEBG(", busy %ld", (long) service->getBusyState()); 106 DEBG( ">\n"); 107 // IOSleep(250); 108 } 109 iter->release(); 110 } 111 112 void db_piokjunk(void) 113 { 114 } 115 116 void db_dumpiojunk( const IORegistryPlane * plane __unused ) 117 { 118 } 119 120 void IOPrintMemory( void ) 121 { 122 123 // OSMetaClass::printInstanceCounts(); 124 125 IOLog("\n" 126 "ivar kalloc() 0x%08x\n" 127 "malloc() 0x%08x\n" 128 "containers kalloc() 0x%08x\n" 129 "IOMalloc() 0x%08x\n" 130 "----------------------------------------\n", 131 debug_ivars_size, 132 debug_malloc_size, 133 debug_container_malloc_size, 134 debug_iomalloc_size 135 ); 136 } 137 138 } /* extern "C" */ 139 140 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 141 142 #define super OSObject 143 OSDefineMetaClassAndStructors(IOKitDiagnostics, OSObject) 144 145 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 146 147 OSObject * IOKitDiagnostics::diagnostics( void ) 148 { 149 IOKitDiagnostics * diags; 150 151 diags = new IOKitDiagnostics; 152 if( diags && !diags->init()) { 153 diags->release(); 154 diags = 0; 155 } 156 157 return( diags ); 158 } 159 160 void IOKitDiagnostics::updateOffset( OSDictionary * dict, 161 UInt64 value, const char * name ) 162 { 163 OSNumber * off; 164 165 off = OSNumber::withNumber( value, 64 ); 166 if( !off) 167 return; 168 169 dict->setObject( name, off ); 170 off->release(); 171 } 172 173 bool IOKitDiagnostics::serialize(OSSerialize *s) const 174 { 175 OSDictionary * dict; 176 bool ok; 177 178 dict = OSDictionary::withCapacity( 5 ); 179 if( !dict) 180 return( false ); 181 182 updateOffset( dict, debug_ivars_size, "Instance allocation" ); 183 updateOffset( dict, debug_container_malloc_size, "Container allocation" ); 184 updateOffset( dict, debug_iomalloc_size, "IOMalloc allocation" ); 185 updateOffset( dict, debug_iomallocpageable_size, "Pageable allocation" ); 186 187 OSMetaClass::serializeClassDictionary(dict); 188 189 ok = dict->serialize( s ); 190 191 dict->release(); 192 193 return( ok ); 194 } 195 196 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 197 198 #if IOTRACKING 199 200 #include <libkern/c++/OSCPPDebug.h> 201 #include <libkern/c++/OSKext.h> 202 #include <kern/zalloc.h> 203 204 __private_extern__ "C" void qsort( 205 void * array, 206 size_t nmembers, 207 size_t member_size, 208 int (*)(const void *, const void *)); 209 210 extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); 211 extern "C" ppnum_t pmap_valid_page(ppnum_t pn); 212 213 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 214 215 struct IOTRecursiveLock 216 { 217 lck_mtx_t * mutex; 218 thread_t thread; 219 UInt32 count; 220 }; 221 222 struct IOTrackingQueue 223 { 224 queue_chain_t link; 225 IOTRecursiveLock lock; 226 queue_head_t sites; 227 const char * name; 228 size_t allocSize; 229 size_t minCaptureSize; 230 uint32_t siteCount; 231 uint8_t captureOn; 232 uint8_t isAlloc; 233 }; 234 235 struct IOTrackingCallSite 236 { 237 queue_chain_t link; 238 IOTrackingQueue * queue; 239 uint32_t crc; 240 IOTrackingCallSiteInfo info; 241 queue_chain_t instances; 242 IOTracking * addresses; 243 }; 244 245 struct IOTrackingLeaksRef 246 { 247 uintptr_t * instances; 248 uint32_t count; 249 uint32_t found; 250 size_t bytes; 251 }; 252 253 enum 254 { 255 kInstanceFlagAddress = 0x01UL, 256 kInstanceFlagReferenced = 0x02UL, 257 kInstanceFlags = 0x03UL 258 }; 259 260 lck_mtx_t * gIOTrackingLock; 261 queue_head_t gIOTrackingQ; 262 263 enum 264 { 265 kTrackingAddressFlagAllocated = 0x00000001 266 }; 267 268 #if defined(__LP64__) 269 #define IOTrackingAddressFlags(ptr) (ptr->flags) 270 #else 271 #define IOTrackingAddressFlags(ptr) (ptr->tracking.flags) 272 #endif 273 274 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 275 276 static void 277 IOTRecursiveLockLock(IOTRecursiveLock * lock) 278 { 279 if (lock->thread == current_thread()) lock->count++; 280 else 281 { 282 lck_mtx_lock(lock->mutex); 283 assert(lock->thread == 0); 284 assert(lock->count == 0); 285 lock->thread = current_thread(); 286 lock->count = 1; 287 } 288 } 289 290 static void 291 IOTRecursiveLockUnlock(IOTRecursiveLock * lock) 292 { 293 assert(lock->thread == current_thread()); 294 if (0 == (--lock->count)) 295 { 296 lock->thread = 0; 297 lck_mtx_unlock(lock->mutex); 298 } 299 } 300 301 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 302 303 void 304 IOTrackingInit(void) 305 { 306 queue_init(&gIOTrackingQ); 307 gIOTrackingLock = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL); 308 } 309 310 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 311 312 IOTrackingQueue * 313 IOTrackingQueueAlloc(const char * name, size_t allocSize, size_t minCaptureSize, bool isAlloc) 314 { 315 IOTrackingQueue * queue; 316 queue = (typeof(queue)) kalloc(sizeof(IOTrackingQueue)); 317 bzero(queue, sizeof(IOTrackingQueue)); 318 319 queue->name = name; 320 queue->allocSize = allocSize; 321 queue->minCaptureSize = minCaptureSize; 322 queue->lock.mutex = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL); 323 queue_init(&queue->sites); 324 325 queue->captureOn = (0 != (kIOTrackingBoot & gIOKitDebug)); 326 queue->isAlloc = isAlloc; 327 328 lck_mtx_lock(gIOTrackingLock); 329 queue_enter(&gIOTrackingQ, queue, IOTrackingQueue *, link); 330 lck_mtx_unlock(gIOTrackingLock); 331 332 return (queue); 333 }; 334 335 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 336 337 void 338 IOTrackingQueueFree(IOTrackingQueue * queue) 339 { 340 lck_mtx_lock(gIOTrackingLock); 341 IOTrackingReset(queue); 342 remque(&queue->link); 343 lck_mtx_unlock(gIOTrackingLock); 344 345 lck_mtx_free(queue->lock.mutex, IOLockGroup); 346 347 kfree(queue, sizeof(IOTrackingQueue)); 348 }; 349 350 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 351 352 /* fasthash 353 The MIT License 354 355 Copyright (C) 2012 Zilong Tan ([email protected]) 356 357 Permission is hereby granted, free of charge, to any person 358 obtaining a copy of this software and associated documentation 359 files (the "Software"), to deal in the Software without 360 restriction, including without limitation the rights to use, copy, 361 modify, merge, publish, distribute, sublicense, and/or sell copies 362 of the Software, and to permit persons to whom the Software is 363 furnished to do so, subject to the following conditions: 364 365 The above copyright notice and this permission notice shall be 366 included in all copies or substantial portions of the Software. 367 368 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 369 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 370 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 371 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 372 BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 373 ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 374 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 375 SOFTWARE. 376 */ 377 378 379 // Compression function for Merkle-Damgard construction. 380 // This function is generated using the framework provided. 381 #define mix(h) ({ \ 382 (h) ^= (h) >> 23; \ 383 (h) *= 0x2127599bf4325c37ULL; \ 384 (h) ^= (h) >> 47; }) 385 386 static uint64_t 387 fasthash64(const void *buf, size_t len, uint64_t seed) 388 { 389 const uint64_t m = 0x880355f21e6d1965ULL; 390 const uint64_t *pos = (const uint64_t *)buf; 391 const uint64_t *end = pos + (len / 8); 392 const unsigned char *pos2; 393 uint64_t h = seed ^ (len * m); 394 uint64_t v; 395 396 while (pos != end) { 397 v = *pos++; 398 h ^= mix(v); 399 h *= m; 400 } 401 402 pos2 = (const unsigned char*)pos; 403 v = 0; 404 405 switch (len & 7) { 406 case 7: v ^= (uint64_t)pos2[6] << 48; 407 case 6: v ^= (uint64_t)pos2[5] << 40; 408 case 5: v ^= (uint64_t)pos2[4] << 32; 409 case 4: v ^= (uint64_t)pos2[3] << 24; 410 case 3: v ^= (uint64_t)pos2[2] << 16; 411 case 2: v ^= (uint64_t)pos2[1] << 8; 412 case 1: v ^= (uint64_t)pos2[0]; 413 h ^= mix(v); 414 h *= m; 415 } 416 417 return mix(h); 418 } 419 420 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 421 422 static uint32_t 423 fasthash32(const void *buf, size_t len, uint32_t seed) 424 { 425 // the following trick converts the 64-bit hashcode to Fermat 426 // residue, which shall retain information from both the higher 427 // and lower parts of hashcode. 428 uint64_t h = fasthash64(buf, len, seed); 429 return h - (h >> 32); 430 } 431 432 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 433 434 void 435 IOTrackingAdd(IOTrackingQueue * queue, IOTracking * mem, size_t size, bool address) 436 { 437 IOTrackingCallSite * site; 438 uint32_t crc, num; 439 uintptr_t bt[kIOTrackingCallSiteBTs + 1]; 440 441 if (mem->site) return; 442 if (!queue->captureOn) return; 443 if (size < queue->minCaptureSize) return; 444 445 assert(!mem->link.next); 446 447 num = fastbacktrace(&bt[0], kIOTrackingCallSiteBTs + 1); 448 num--; 449 crc = fasthash32(&bt[1], num * sizeof(bt[0]), 0x04C11DB7); 450 451 IOTRecursiveLockLock(&queue->lock); 452 queue_iterate(&queue->sites, site, IOTrackingCallSite *, link) 453 { 454 if (crc == site->crc) break; 455 } 456 457 if (queue_end(&queue->sites, (queue_entry_t) site)) 458 { 459 site = (typeof(site)) kalloc(sizeof(IOTrackingCallSite)); 460 461 queue_init(&site->instances); 462 site->addresses = (IOTracking *) &site->instances; 463 site->queue = queue; 464 site->crc = crc; 465 site->info.count = 0; 466 memset(&site->info.size[0], 0, sizeof(site->info.size)); 467 bcopy(&bt[1], &site->info.bt[0], num * sizeof(site->info.bt[0])); 468 assert(num <= kIOTrackingCallSiteBTs); 469 bzero(&site->info.bt[num], (kIOTrackingCallSiteBTs - num) * sizeof(site->info.bt[0])); 470 471 queue_enter_first(&queue->sites, site, IOTrackingCallSite *, link); 472 queue->siteCount++; 473 } 474 475 if (address) 476 { 477 queue_enter/*last*/(&site->instances, mem, IOTrackingCallSite *, link); 478 if (queue_end(&site->instances, (queue_entry_t)site->addresses)) site->addresses = mem; 479 } 480 else queue_enter_first(&site->instances, mem, IOTrackingCallSite *, link); 481 482 mem->site = site; 483 site->info.size[0] += size; 484 site->info.count++; 485 486 IOTRecursiveLockUnlock(&queue->lock); 487 } 488 489 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 490 491 void 492 IOTrackingRemove(IOTrackingQueue * queue, IOTracking * mem, size_t size) 493 { 494 if (!mem->link.next) return; 495 496 IOTRecursiveLockLock(&queue->lock); 497 498 assert(mem->site); 499 500 if (mem == mem->site->addresses) mem->site->addresses = (IOTracking *) queue_next(&mem->link); 501 remque(&mem->link); 502 503 assert(mem->site->info.count); 504 mem->site->info.count--; 505 assert(mem->site->info.size[0] >= size); 506 mem->site->info.size[0] -= size; 507 if (!mem->site->info.count) 508 { 509 assert(queue_empty(&mem->site->instances)); 510 assert(!mem->site->info.size[0]); 511 assert(!mem->site->info.size[1]); 512 513 remque(&mem->site->link); 514 assert(queue->siteCount); 515 queue->siteCount--; 516 kfree(mem->site, sizeof(IOTrackingCallSite)); 517 } 518 IOTRecursiveLockUnlock(&queue->lock); 519 } 520 521 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 522 523 void 524 IOTrackingAlloc(IOTrackingQueue * queue, uintptr_t address, size_t size) 525 { 526 IOTrackingAddress * tracking; 527 528 if (!queue->captureOn) return; 529 if (size < queue->minCaptureSize) return; 530 531 address = ~address; 532 tracking = (typeof(tracking)) kalloc(sizeof(IOTrackingAddress)); 533 bzero(tracking, sizeof(IOTrackingAddress)); 534 IOTrackingAddressFlags(tracking) |= kTrackingAddressFlagAllocated; 535 tracking->address = address; 536 tracking->size = size; 537 538 IOTrackingAdd(queue, &tracking->tracking, size, true); 539 } 540 541 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 542 543 void 544 IOTrackingFree(IOTrackingQueue * queue, uintptr_t address, size_t size) 545 { 546 IOTrackingCallSite * site; 547 IOTrackingAddress * tracking; 548 bool done; 549 550 address = ~address; 551 IOTRecursiveLockLock(&queue->lock); 552 done = false; 553 queue_iterate(&queue->sites, site, IOTrackingCallSite *, link) 554 { 555 for (tracking = (IOTrackingAddress *) site->addresses; 556 !done && !queue_end(&site->instances, (queue_entry_t) tracking); 557 tracking = (IOTrackingAddress *) queue_next(&tracking->tracking.link)) 558 { 559 if ((done = (address == tracking->address))) 560 { 561 IOTrackingRemove(queue, &tracking->tracking, size); 562 kfree(tracking, sizeof(IOTrackingAddress)); 563 } 564 } 565 if (done) break; 566 } 567 568 IOTRecursiveLockUnlock(&queue->lock); 569 } 570 571 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 572 573 void 574 IOTrackingAccumSize(IOTrackingQueue * queue, IOTracking * mem, size_t size) 575 { 576 IOTRecursiveLockLock(&queue->lock); 577 if (mem->link.next) 578 { 579 assert(mem->site); 580 assert((size > 0) || (mem->site->info.size[1] >= -size)); 581 mem->site->info.size[1] += size; 582 }; 583 IOTRecursiveLockUnlock(&queue->lock); 584 } 585 586 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 587 588 void 589 IOTrackingReset(IOTrackingQueue * queue) 590 { 591 IOTrackingCallSite * site; 592 IOTracking * tracking; 593 IOTrackingAddress * trackingAddress; 594 bool addresses; 595 596 IOTRecursiveLockLock(&queue->lock); 597 while (!queue_empty(&queue->sites)) 598 { 599 queue_remove_first(&queue->sites, site, IOTrackingCallSite *, link); 600 addresses = false; 601 while (!queue_empty(&site->instances)) 602 { 603 queue_remove_first(&site->instances, tracking, IOTracking *, link); 604 tracking->link.next = 0; 605 if (tracking == site->addresses) addresses = true; 606 if (addresses) 607 { 608 trackingAddress = (typeof(trackingAddress)) tracking; 609 if (kTrackingAddressFlagAllocated & IOTrackingAddressFlags(trackingAddress)) 610 { 611 kfree(tracking, sizeof(IOTrackingAddress)); 612 } 613 } 614 } 615 kfree(site, sizeof(IOTrackingCallSite)); 616 } 617 queue->siteCount = 0; 618 IOTRecursiveLockUnlock(&queue->lock); 619 } 620 621 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 622 623 static int 624 IOTrackingCallSiteInfoCompare(const void * left, const void * right) 625 { 626 IOTrackingCallSiteInfo * l = (typeof(l)) left; 627 IOTrackingCallSiteInfo * r = (typeof(r)) right; 628 size_t lsize, rsize; 629 630 rsize = r->size[0] + r->size[1]; 631 lsize = l->size[0] + l->size[1]; 632 633 return ((rsize > lsize) ? 1 : ((rsize == lsize) ? 0 : -1)); 634 } 635 636 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 637 638 static int 639 IOTrackingAddressCompare(const void * left, const void * right) 640 { 641 IOTracking * instance; 642 uintptr_t inst, laddr, raddr; 643 644 inst = ((typeof(inst) *) left)[0]; 645 instance = (typeof(instance)) (inst & ~kInstanceFlags); 646 if (kInstanceFlagAddress & inst) laddr = ~((IOTrackingAddress *)instance)->address; 647 else laddr = (uintptr_t) (instance + 1); 648 649 inst = ((typeof(inst) *) right)[0]; 650 instance = (typeof(instance)) (inst & ~kInstanceFlags); 651 if (kInstanceFlagAddress & inst) raddr = ~((IOTrackingAddress *)instance)->address; 652 else raddr = (uintptr_t) (instance + 1); 653 654 return ((laddr > raddr) ? 1 : ((laddr == raddr) ? 0 : -1)); 655 } 656 657 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 658 659 static void 660 IOTrackingLeakScan(void * refcon) 661 { 662 IOTrackingLeaksRef * ref = (typeof(ref)) refcon; 663 uintptr_t * instances; 664 IOTracking * instance; 665 uint64_t vaddr, vincr; 666 ppnum_t ppn; 667 uintptr_t ptr, addr, inst; 668 size_t size; 669 uint32_t baseIdx, lim, ptrIdx, count; 670 boolean_t is; 671 672 // if (cpu_number()) return; 673 674 instances = ref->instances; 675 count = ref->count; 676 677 for (vaddr = VM_MIN_KERNEL_AND_KEXT_ADDRESS; 678 vaddr < VM_MAX_KERNEL_ADDRESS; 679 ml_set_interrupts_enabled(is), vaddr += vincr) 680 { 681 #if !defined(__LP64__) 682 thread_block(NULL); 683 #endif 684 is = ml_set_interrupts_enabled(false); 685 686 ppn = kernel_pmap_present_mapping(vaddr, &vincr); 687 // check noencrypt to avoid VM structs (map entries) with pointers 688 if (ppn && (!pmap_valid_page(ppn) || pmap_is_noencrypt(ppn))) ppn = 0; 689 if (!ppn) continue; 690 691 for (ptrIdx = 0; ptrIdx < (page_size / sizeof(uintptr_t)); ptrIdx++) 692 { 693 ptr = ((uintptr_t *)vaddr)[ptrIdx]; 694 695 for (lim = count, baseIdx = 0; lim; lim >>= 1) 696 { 697 inst = instances[baseIdx + (lim >> 1)]; 698 instance = (typeof(instance)) (inst & ~kInstanceFlags); 699 if (kInstanceFlagAddress & inst) 700 { 701 addr = ~((IOTrackingAddress *)instance)->address; 702 size = ((IOTrackingAddress *)instance)->size; 703 } 704 else 705 { 706 addr = (uintptr_t) (instance + 1); 707 size = instance->site->queue->allocSize; 708 } 709 if ((ptr >= addr) && (ptr < (addr + size))) 710 { 711 if (!(kInstanceFlagReferenced & inst)) 712 { 713 inst |= kInstanceFlagReferenced; 714 instances[baseIdx + (lim >> 1)] = inst; 715 ref->found++; 716 } 717 break; 718 } 719 if (ptr > addr) 720 { 721 // move right 722 baseIdx += (lim >> 1) + 1; 723 lim--; 724 } 725 // else move left 726 } 727 } 728 ref->bytes += page_size; 729 } 730 } 731 732 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 733 734 static OSData * 735 IOTrackingLeaks(OSData * data) 736 { 737 IOTrackingLeaksRef ref; 738 IOTrackingCallSiteInfo unslideInfo; 739 IOTrackingCallSite * site; 740 OSData * leakData; 741 uintptr_t * instances; 742 IOTracking * instance; 743 uintptr_t inst; 744 uint32_t count, idx, numSites, dups, siteCount; 745 746 instances = (typeof(instances)) data->getBytesNoCopy(); 747 count = (data->getLength() / sizeof(*instances)); 748 qsort(instances, count, sizeof(*instances), &IOTrackingAddressCompare); 749 750 bzero(&ref, sizeof(ref)); 751 ref.instances = instances; 752 ref.count = count; 753 754 IOTrackingLeakScan(&ref); 755 756 IOLog("leaks scanned %ld MB, instance count %d, found %d\n", ref.bytes / 1024 / 1024, count, ref.found); 757 758 leakData = OSData::withCapacity(128 * sizeof(IOTrackingCallSiteInfo)); 759 760 for (numSites = 0, idx = 0; idx < count; idx++) 761 { 762 inst = instances[idx]; 763 if (kInstanceFlagReferenced & inst) continue; 764 instance = (typeof(instance)) (inst & ~kInstanceFlags); 765 site = instance->site; 766 instances[numSites] = (uintptr_t) site; 767 numSites++; 768 } 769 770 for (idx = 0; idx < numSites; idx++) 771 { 772 inst = instances[idx]; 773 if (!inst) continue; 774 site = (typeof(site)) inst; 775 for (siteCount = 1, dups = (idx + 1); dups < numSites; dups++) 776 { 777 if (instances[dups] == (uintptr_t) site) 778 { 779 siteCount++; 780 instances[dups] = 0; 781 } 782 } 783 unslideInfo.count = siteCount; 784 unslideInfo.size[0] = (site->info.size[0] * site->info.count) / siteCount; 785 unslideInfo.size[1] = (site->info.size[1] * site->info.count) / siteCount;; 786 for (uint32_t j = 0; j < kIOTrackingCallSiteBTs; j++) 787 { 788 unslideInfo.bt[j] = VM_KERNEL_UNSLIDE(site->info.bt[j]); 789 } 790 leakData->appendBytes(&unslideInfo, sizeof(unslideInfo)); 791 } 792 data->release(); 793 794 return (leakData); 795 } 796 797 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 798 799 static bool 800 SkipName(uint32_t options, const char * name, size_t namesLen, const char * names) 801 { 802 const char * scan; 803 const char * next; 804 bool exclude, found; 805 size_t qLen, sLen; 806 807 if (!namesLen || !names) return (false); 808 // <len><name>...<len><name><0> 809 exclude = (0 != (kIOTrackingExcludeNames & options)); 810 qLen = strlen(name); 811 scan = names; 812 found = false; 813 do 814 { 815 sLen = scan[0]; 816 scan++; 817 next = scan + sLen; 818 if (next >= (names + namesLen)) break; 819 found = ((sLen == qLen) && !strncmp(scan, name, sLen)); 820 scan = next; 821 } 822 while (!found && (scan < (names + namesLen))); 823 824 return (!(exclude ^ found)); 825 } 826 827 #endif /* IOTRACKING */ 828 829 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 830 831 kern_return_t 832 IOTrackingDebug(uint32_t selector, uint32_t options, 833 const char * names, size_t namesLen, 834 size_t size, OSObject ** result) 835 { 836 kern_return_t ret; 837 OSData * data; 838 839 if (result) *result = 0; 840 data = 0; 841 ret = kIOReturnNotReady; 842 843 #if IOTRACKING 844 845 IOTrackingQueue * queue; 846 IOTracking * instance; 847 IOTrackingCallSite * site; 848 IOTrackingCallSiteInfo * siteInfos; 849 IOTrackingCallSiteInfo * siteInfo; 850 bool addresses; 851 uint32_t num, idx; 852 uintptr_t instFlags; 853 854 if (!(kIOTracking & gIOKitDebug)) return (kIOReturnNotReady); 855 ret = kIOReturnNotFound; 856 857 lck_mtx_lock(gIOTrackingLock); 858 queue_iterate(&gIOTrackingQ, queue, IOTrackingQueue *, link) 859 { 860 if (SkipName(options, queue->name, namesLen, names)) continue; 861 862 switch (selector) 863 { 864 case kIOTrackingResetTracking: 865 { 866 IOTrackingReset(queue); 867 ret = kIOReturnSuccess; 868 break; 869 } 870 871 case kIOTrackingStartCapture: 872 case kIOTrackingStopCapture: 873 { 874 queue->captureOn = (kIOTrackingStartCapture == selector); 875 ret = kIOReturnSuccess; 876 break; 877 } 878 879 case kIOTrackingSetMinCaptureSize: 880 { 881 queue->minCaptureSize = size; 882 ret = kIOReturnSuccess; 883 break; 884 } 885 886 case kIOTrackingLeaks: 887 { 888 if (!queue->isAlloc) break; 889 890 if (!data) data = OSData::withCapacity(1024 * sizeof(uintptr_t)); 891 892 IOTRecursiveLockLock(&queue->lock); 893 queue_iterate(&queue->sites, site, IOTrackingCallSite *, link) 894 { 895 addresses = false; 896 queue_iterate(&site->instances, instance, IOTracking *, link) 897 { 898 if (instance == site->addresses) addresses = true; 899 instFlags = (typeof(instFlags)) instance; 900 if (addresses) instFlags |= kInstanceFlagAddress; 901 data->appendBytes(&instFlags, sizeof(instFlags)); 902 } 903 } 904 // queue is locked 905 ret = kIOReturnSuccess; 906 break; 907 } 908 909 case kIOTrackingGetTracking: 910 case kIOTrackingPrintTracking: 911 { 912 if (!data) data = OSData::withCapacity(128 * sizeof(IOTrackingCallSiteInfo)); 913 914 IOTRecursiveLockLock(&queue->lock); 915 num = queue->siteCount; 916 idx = 0; 917 queue_iterate(&queue->sites, site, IOTrackingCallSite *, link) 918 { 919 assert(idx < num); 920 idx++; 921 922 if (size && ((site->info.size[0] + site->info.size[1]) < size)) continue; 923 924 IOTrackingCallSiteInfo unslideInfo; 925 unslideInfo.count = site->info.count; 926 memcpy(&unslideInfo.size[0], &site->info.size[0], sizeof(unslideInfo.size)); 927 928 for (uint32_t j = 0; j < kIOTrackingCallSiteBTs; j++) 929 { 930 unslideInfo.bt[j] = VM_KERNEL_UNSLIDE(site->info.bt[j]); 931 } 932 data->appendBytes(&unslideInfo, sizeof(unslideInfo)); 933 } 934 assert(idx == num); 935 IOTRecursiveLockUnlock(&queue->lock); 936 ret = kIOReturnSuccess; 937 break; 938 } 939 default: 940 ret = kIOReturnUnsupported; 941 break; 942 } 943 } 944 945 if ((kIOTrackingLeaks == selector) && data) 946 { 947 data = IOTrackingLeaks(data); 948 queue_iterate(&gIOTrackingQ, queue, IOTrackingQueue *, link) 949 { 950 if (SkipName(options, queue->name, namesLen, names)) continue; 951 if (!queue->isAlloc) continue; 952 IOTRecursiveLockUnlock(&queue->lock); 953 } 954 } 955 956 lck_mtx_unlock(gIOTrackingLock); 957 958 if (data) 959 { 960 siteInfos = (typeof(siteInfos)) data->getBytesNoCopy(); 961 num = (data->getLength() / sizeof(IOTrackingCallSiteInfo)); 962 qsort(siteInfos, num, sizeof(*siteInfos), &IOTrackingCallSiteInfoCompare); 963 964 if (kIOTrackingPrintTracking == selector) 965 { 966 for (idx = 0; idx < num; idx++) 967 { 968 siteInfo = &siteInfos[idx]; 969 printf("\n0x%lx bytes (0x%lx + 0x%lx), %d call%s, [%d]\n", 970 siteInfo->size[0] + siteInfo->size[1], 971 siteInfo->size[0], siteInfo->size[1], 972 siteInfo->count, (siteInfo->count != 1) ? "s" : "", idx); 973 uintptr_t * bt = &siteInfo->bt[0]; 974 printf(" Backtrace 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx\n", 975 bt[0], bt[1], bt[2], bt[3], bt[4], bt[5], bt[6], bt[7], 976 bt[8], bt[9], bt[10], bt[11], bt[12], bt[13], bt[14], bt[15]); 977 kmod_dump_log((vm_offset_t *) &bt[0], kIOTrackingCallSiteBTs, FALSE); 978 } 979 data->release(); 980 data = 0; 981 } 982 } 983 984 *result = data; 985 986 #endif /* IOTRACKING */ 987 988 return (ret); 989 } 990 991 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 992 993 #include <IOKit/IOKitDiagnosticsUserClient.h> 994 995 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 996 997 #undef super 998 #define super IOUserClient 999 1000 OSDefineMetaClassAndStructors(IOKitDiagnosticsClient, IOUserClient) 1001 1002 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1003 1004 IOUserClient * IOKitDiagnosticsClient::withTask(task_t owningTask) 1005 { 1006 IOKitDiagnosticsClient * inst; 1007 1008 inst = new IOKitDiagnosticsClient; 1009 if (inst && !inst->init()) 1010 { 1011 inst->release(); 1012 inst = 0; 1013 } 1014 1015 return (inst); 1016 } 1017 1018 IOReturn IOKitDiagnosticsClient::clientClose(void) 1019 { 1020 terminate(); 1021 return (kIOReturnSuccess); 1022 } 1023 1024 IOReturn IOKitDiagnosticsClient::setProperties(OSObject * properties) 1025 { 1026 IOReturn kr = kIOReturnUnsupported; 1027 return (kr); 1028 } 1029 1030 IOReturn IOKitDiagnosticsClient::externalMethod(uint32_t selector, IOExternalMethodArguments * args, 1031 IOExternalMethodDispatch * dispatch, OSObject * target, void * reference) 1032 { 1033 IOReturn ret = kIOReturnBadArgument; 1034 const IOKitDiagnosticsParameters * params; 1035 const char * names; 1036 size_t namesLen; 1037 OSObject * result; 1038 1039 if (args->structureInputSize < sizeof(IOKitDiagnosticsParameters)) return (kIOReturnBadArgument); 1040 params = (typeof(params)) args->structureInput; 1041 if (!params) return (kIOReturnBadArgument); 1042 1043 names = 0; 1044 namesLen = args->structureInputSize - sizeof(IOKitDiagnosticsParameters); 1045 if (namesLen) names = (typeof(names))(params + 1); 1046 1047 ret = IOTrackingDebug(selector, params->options, names, namesLen, params->size, &result); 1048 1049 if ((kIOReturnSuccess == ret) && args->structureVariableOutputData) *args->structureVariableOutputData = result; 1050 else if (result) result->release(); 1051 1052 return (ret); 1053 } 1054 1055 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 1056