1 /* 2 * Copyright (c) 1998-2014 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #define IOKIT_ENABLE_SHARED_PTR 30 31 #include <ptrauth.h> 32 #include <IOKit/IOInterruptEventSource.h> 33 #include <IOKit/IOKitDebug.h> 34 #include <IOKit/IOLib.h> 35 #include <IOKit/IOService.h> 36 #include <IOKit/IOInterrupts.h> 37 #include <IOKit/IOTimeStamp.h> 38 #include <IOKit/IOWorkLoop.h> 39 #include <IOKit/IOInterruptAccountingPrivate.h> 40 #include <libkern/Block_private.h> 41 42 #if IOKITSTATS 43 44 #define IOStatisticsInitializeCounter() \ 45 do { \ 46 IOStatistics::setCounterType(IOEventSource::reserved->counter, kIOStatisticsInterruptEventSourceCounter); \ 47 } while (0) 48 49 #define IOStatisticsCheckForWork() \ 50 do { \ 51 IOStatistics::countInterruptCheckForWork(IOEventSource::reserved->counter); \ 52 } while (0) 53 54 #define IOStatisticsInterrupt() \ 55 do { \ 56 IOStatistics::countInterrupt(IOEventSource::reserved->counter); \ 57 } while (0) 58 59 #else 60 61 #define IOStatisticsInitializeCounter() 62 #define IOStatisticsCheckForWork() 63 #define IOStatisticsInterrupt() 64 65 #endif // IOKITSTATS 66 67 #define super IOEventSource 68 69 OSDefineMetaClassAndStructors(IOInterruptEventSource, IOEventSource) 70 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 0); 71 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 1); 72 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 2); 73 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 3); 74 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 4); 75 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 5); 76 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 6); 77 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 7); 78 79 bool 80 IOInterruptEventSource::init(OSObject *inOwner, 81 Action inAction, 82 IOService *inProvider, 83 int inIntIndex) 84 { 85 bool res = true; 86 87 if (inIntIndex < 0) { 88 return false; 89 } 90 91 if (!super::init(inOwner, (IOEventSourceAction) inAction)) { 92 return false; 93 } 94 95 reserved = IOMallocType(ExpansionData); 96 97 provider = inProvider; 98 producerCount = consumerCount = 0; 99 autoDisable = explicitDisable = false; 100 intIndex = ~inIntIndex; 101 102 // Assumes inOwner holds a reference(retain) on the provider 103 if (inProvider) { 104 if (IA_ANY_STATISTICS_ENABLED) { 105 /* 106 * We only treat this as an "interrupt" if it has a provider; if it does, 107 * set up the objects necessary to track interrupt statistics. Interrupt 108 * event sources without providers are most likely being used as simple 109 * event source in order to poke at workloops and kick off work. 110 * 111 * We also avoid try to avoid interrupt accounting overhead if none of 112 * the statistics are enabled. 113 */ 114 reserved->statistics = IOMallocType(IOInterruptAccountingData); 115 116 reserved->statistics->owner = this; 117 } 118 119 res = (kIOReturnSuccess == registerInterruptHandler(inProvider, inIntIndex)); 120 121 if (res) { 122 intIndex = inIntIndex; 123 } 124 } 125 126 IOStatisticsInitializeCounter(); 127 128 return res; 129 } 130 131 IOReturn 132 IOInterruptEventSource::registerInterruptHandler(IOService *inProvider, 133 int inIntIndex) 134 { 135 IOReturn ret; 136 int intType; 137 IOInterruptAction intHandler; 138 139 ret = inProvider->getInterruptType(inIntIndex, &intType); 140 if (kIOReturnSuccess != ret) { 141 return ret; 142 } 143 144 autoDisable = (intType == kIOInterruptTypeLevel); 145 if (autoDisable) { 146 intHandler = OSMemberFunctionCast(IOInterruptAction, 147 this, &IOInterruptEventSource::disableInterruptOccurred); 148 } else { 149 intHandler = OSMemberFunctionCast(IOInterruptAction, 150 this, &IOInterruptEventSource::normalInterruptOccurred); 151 } 152 153 ret = provider->registerInterrupt(inIntIndex, this, intHandler); 154 155 /* 156 * Add statistics to the provider. The setWorkLoop convention should ensure 157 * that we always go down the unregister path before we register (outside of 158 * init()), so we don't have to worry that we will invoke addInterruptStatistics 159 * erroneously. 160 */ 161 if ((ret == kIOReturnSuccess) && (reserved->statistics)) { 162 /* 163 * Stash the normal index value, for the sake of debugging. 164 */ 165 reserved->statistics->interruptIndex = inIntIndex; 166 167 /* 168 * We need to hook the interrupt information up to the provider so that it 169 * can find the statistics for this interrupt when desired. The provider is 170 * responsible for maintaining the reporter for a particular interrupt, and 171 * needs a handle on the statistics so that it can request that the reporter 172 * be updated as needed. Errors are considered "soft" for the moment (it 173 * will either panic, or fail in a way such that we can still service the 174 * interrupt). 175 */ 176 provider->addInterruptStatistics(reserved->statistics, inIntIndex); 177 178 /* 179 * Add the statistics object to the global list of statistics objects; this 180 * is an aid to debugging (we can trivially find statistics for all eligible 181 * interrupts, and dump them; potentially helpful if the system is wedged 182 * due to interrupt activity). 183 */ 184 interruptAccountingDataAddToList(reserved->statistics); 185 } 186 187 return ret; 188 } 189 190 void 191 IOInterruptEventSource::unregisterInterruptHandler(IOService *inProvider, 192 int inIntIndex) 193 { 194 if (reserved->statistics) { 195 interruptAccountingDataRemoveFromList(reserved->statistics); 196 provider->removeInterruptStatistics(reserved->statistics->interruptIndex); 197 } 198 199 provider->unregisterInterrupt(inIntIndex); 200 } 201 202 203 OSSharedPtr<IOInterruptEventSource> 204 IOInterruptEventSource::interruptEventSource(OSObject *inOwner, 205 Action inAction, 206 IOService *inProvider, 207 int inIntIndex) 208 { 209 OSSharedPtr<IOInterruptEventSource> me = OSMakeShared<IOInterruptEventSource>(); 210 211 if (me && !me->init(inOwner, inAction, inProvider, inIntIndex)) { 212 return nullptr; 213 } 214 215 return me; 216 } 217 218 OSSharedPtr<IOInterruptEventSource> 219 IOInterruptEventSource::interruptEventSource(OSObject *inOwner, 220 IOService *inProvider, 221 int inIntIndex, 222 ActionBlock inAction) 223 { 224 OSSharedPtr<IOInterruptEventSource> ies; 225 ies = IOInterruptEventSource::interruptEventSource(inOwner, (Action) NULL, inProvider, inIntIndex); 226 if (ies) { 227 ies->setActionBlock((IOEventSource::ActionBlock) inAction); 228 } 229 230 return ies; 231 } 232 233 void 234 IOInterruptEventSource::free() 235 { 236 if (provider && intIndex >= 0) { 237 unregisterInterruptHandler(provider, intIndex); 238 } 239 240 if (reserved) { 241 if (reserved->statistics) { 242 IOFreeType(reserved->statistics, IOInterruptAccountingData); 243 } 244 245 IOFreeType(reserved, ExpansionData); 246 } 247 248 super::free(); 249 } 250 251 void 252 IOInterruptEventSource::enable() 253 { 254 if (provider && intIndex >= 0) { 255 provider->enableInterrupt(intIndex); 256 explicitDisable = false; 257 enabled = true; 258 } 259 } 260 261 void 262 IOInterruptEventSource::disable() 263 { 264 if (provider && intIndex >= 0) { 265 provider->disableInterrupt(intIndex); 266 explicitDisable = true; 267 enabled = false; 268 } 269 } 270 271 void 272 IOInterruptEventSource::setWorkLoop(IOWorkLoop *inWorkLoop) 273 { 274 if (inWorkLoop) { 275 super::setWorkLoop(inWorkLoop); 276 } 277 278 if (provider) { 279 if (!inWorkLoop) { 280 if (intIndex >= 0) { 281 /* 282 * It isn't necessarily safe to wait until free() to unregister the interrupt; 283 * our provider may disappear. 284 */ 285 unregisterInterruptHandler(provider, intIndex); 286 intIndex = ~intIndex; 287 } 288 } else if ((intIndex < 0) && (kIOReturnSuccess == registerInterruptHandler(provider, ~intIndex))) { 289 intIndex = ~intIndex; 290 } 291 } 292 293 if (!inWorkLoop) { 294 super::setWorkLoop(inWorkLoop); 295 } 296 } 297 298 const IOService * 299 IOInterruptEventSource::getProvider() const 300 { 301 return provider; 302 } 303 304 int 305 IOInterruptEventSource::getIntIndex() const 306 { 307 return intIndex; 308 } 309 310 bool 311 IOInterruptEventSource::getAutoDisable() const 312 { 313 return autoDisable; 314 } 315 316 bool 317 IOInterruptEventSource::checkForWork() 318 { 319 uint64_t startSystemTime = 0; 320 uint64_t endSystemTime = 0; 321 uint64_t startCPUTime = 0; 322 uint64_t endCPUTime = 0; 323 unsigned int cacheProdCount = producerCount; 324 int numInts = cacheProdCount - consumerCount; 325 IOEventSource::Action intAction = action; 326 ActionBlock intActionBlock = (ActionBlock) actionBlock; 327 void *address; 328 bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false; 329 330 if (kActionBlock & flags) { 331 address = ptrauth_nop_cast(void *, _Block_get_invoke_fn((struct Block_layout *)intActionBlock)); 332 } else { 333 address = ptrauth_nop_cast(void *, intAction); 334 } 335 336 IOStatisticsCheckForWork(); 337 338 if (numInts > 0) { 339 if (trace) { 340 IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION), 341 VM_KERNEL_ADDRHIDE(address), 342 VM_KERNEL_ADDRHIDE(owner), 343 VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); 344 } 345 346 if (reserved->statistics) { 347 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) { 348 startSystemTime = mach_absolute_time(); 349 } 350 351 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) { 352 startCPUTime = thread_get_runtime_self(); 353 } 354 } 355 356 // Call the handler 357 if (kActionBlock & flags) { 358 (intActionBlock)(this, numInts); 359 } else { 360 ((IOInterruptEventAction)intAction)(owner, this, numInts); 361 } 362 363 if (reserved->statistics) { 364 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCountIndex)) { 365 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCountIndex], 1); 366 } 367 368 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) { 369 endCPUTime = thread_get_runtime_self(); 370 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCPUTimeIndex], endCPUTime - startCPUTime); 371 } 372 373 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) { 374 endSystemTime = mach_absolute_time(); 375 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelSystemTimeIndex], endSystemTime - startSystemTime); 376 } 377 } 378 379 if (trace) { 380 IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION), 381 VM_KERNEL_ADDRHIDE(address), 382 VM_KERNEL_ADDRHIDE(owner), 383 VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); 384 } 385 386 consumerCount = cacheProdCount; 387 if (autoDisable && !explicitDisable) { 388 enable(); 389 } 390 } else if (numInts < 0) { 391 if (trace) { 392 IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION), 393 VM_KERNEL_ADDRHIDE(address), 394 VM_KERNEL_ADDRHIDE(owner), 395 VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); 396 } 397 398 if (reserved->statistics) { 399 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) { 400 startSystemTime = mach_absolute_time(); 401 } 402 403 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) { 404 startCPUTime = thread_get_runtime_self(); 405 } 406 } 407 408 // Call the handler 409 if (kActionBlock & flags) { 410 (intActionBlock)(this, numInts); 411 } else { 412 ((IOInterruptEventAction)intAction)(owner, this, numInts); 413 } 414 415 if (reserved->statistics) { 416 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) { 417 endSystemTime = mach_absolute_time(); 418 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelSystemTimeIndex], endSystemTime - startSystemTime); 419 } 420 421 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) { 422 endCPUTime = thread_get_runtime_self(); 423 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCPUTimeIndex], endCPUTime - startCPUTime); 424 } 425 426 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCountIndex)) { 427 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCountIndex], 1); 428 } 429 } 430 431 if (trace) { 432 IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION), 433 VM_KERNEL_ADDRHIDE(address), 434 VM_KERNEL_ADDRHIDE(owner), 435 VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); 436 } 437 438 consumerCount = cacheProdCount; 439 if (autoDisable && !explicitDisable) { 440 enable(); 441 } 442 } 443 444 return false; 445 } 446 447 void 448 IOInterruptEventSource::normalInterruptOccurred 449 (void */*refcon*/, IOService */*prov*/, int /*source*/) 450 { 451 bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false; 452 453 IOStatisticsInterrupt(); 454 producerCount++; 455 456 if (trace) { 457 IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); 458 } 459 460 if (reserved->statistics) { 461 if (reserved->statistics->enablePrimaryTimestamp) { 462 reserved->statistics->primaryTimestamp = mach_absolute_time(); 463 } 464 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) { 465 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1); 466 } 467 } 468 469 signalWorkAvailable(); 470 471 if (trace) { 472 IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); 473 } 474 } 475 476 void 477 IOInterruptEventSource::disableInterruptOccurred 478 (void */*refcon*/, IOService *prov, int source) 479 { 480 bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false; 481 482 prov->disableInterrupt(source); /* disable the interrupt */ 483 484 IOStatisticsInterrupt(); 485 producerCount++; 486 487 if (trace) { 488 IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); 489 } 490 491 if (reserved->statistics) { 492 if (reserved->statistics->enablePrimaryTimestamp) { 493 reserved->statistics->primaryTimestamp = mach_absolute_time(); 494 } 495 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) { 496 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1); 497 } 498 } 499 500 signalWorkAvailable(); 501 502 if (trace) { 503 IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); 504 } 505 } 506 507 void 508 IOInterruptEventSource::interruptOccurred 509 (void *_refcon, IOService *prov, int source) 510 { 511 if (autoDisable && prov) { 512 disableInterruptOccurred(_refcon, prov, source); 513 } else { 514 normalInterruptOccurred(_refcon, prov, source); 515 } 516 } 517 518 IOReturn 519 IOInterruptEventSource::warmCPU 520 (uint64_t abstime) 521 { 522 return ml_interrupt_prewarm(abstime); 523 } 524 525 void 526 IOInterruptEventSource::enablePrimaryInterruptTimestamp(bool enable) 527 { 528 if (reserved->statistics) { 529 reserved->statistics->enablePrimaryTimestamp = enable; 530 } 531 } 532 533 uint64_t 534 IOInterruptEventSource::getPrimaryInterruptTimestamp() 535 { 536 if (reserved->statistics && reserved->statistics->enablePrimaryTimestamp) { 537 return reserved->statistics->primaryTimestamp; 538 } 539 return -1ULL; 540 } 541