1 /* 2 * Copyright (c) 1998-2014 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #define IOKIT_ENABLE_SHARED_PTR 30 31 #include <ptrauth.h> 32 #include <IOKit/IOInterruptEventSource.h> 33 #include <IOKit/IOKitDebug.h> 34 #include <IOKit/IOLib.h> 35 #include <IOKit/IOService.h> 36 #include <IOKit/IOInterrupts.h> 37 #include <IOKit/IOTimeStamp.h> 38 #include <IOKit/IOWorkLoop.h> 39 #include <IOKit/IOInterruptAccountingPrivate.h> 40 #include <libkern/Block_private.h> 41 42 #if IOKITSTATS 43 44 #define IOStatisticsInitializeCounter() \ 45 do { \ 46 IOStatistics::setCounterType(IOEventSource::reserved->counter, kIOStatisticsInterruptEventSourceCounter); \ 47 } while (0) 48 49 #define IOStatisticsCheckForWork() \ 50 do { \ 51 IOStatistics::countInterruptCheckForWork(IOEventSource::reserved->counter); \ 52 } while (0) 53 54 #define IOStatisticsInterrupt() \ 55 do { \ 56 IOStatistics::countInterrupt(IOEventSource::reserved->counter); \ 57 } while (0) 58 59 #else 60 61 #define IOStatisticsInitializeCounter() 62 #define IOStatisticsCheckForWork() 63 #define IOStatisticsInterrupt() 64 65 #endif // IOKITSTATS 66 67 #define super IOEventSource 68 69 OSDefineMetaClassAndStructors(IOInterruptEventSource, IOEventSource) 70 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 0); 71 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 1); 72 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 2); 73 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 3); 74 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 4); 75 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 5); 76 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 6); 77 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 7); 78 79 bool 80 IOInterruptEventSource::init(OSObject *inOwner, 81 Action inAction, 82 IOService *inProvider, 83 int inIntIndex) 84 { 85 bool res = true; 86 87 if (!super::init(inOwner, (IOEventSourceAction) inAction)) { 88 return false; 89 } 90 91 reserved = IOMallocType(ExpansionData); 92 93 provider = inProvider; 94 producerCount = consumerCount = 0; 95 autoDisable = explicitDisable = false; 96 intIndex = ~inIntIndex; 97 98 // Assumes inOwner holds a reference(retain) on the provider 99 if (inProvider) { 100 if (IA_ANY_STATISTICS_ENABLED) { 101 /* 102 * We only treat this as an "interrupt" if it has a provider; if it does, 103 * set up the objects necessary to track interrupt statistics. Interrupt 104 * event sources without providers are most likely being used as simple 105 * event source in order to poke at workloops and kick off work. 106 * 107 * We also avoid try to avoid interrupt accounting overhead if none of 108 * the statistics are enabled. 109 */ 110 reserved->statistics = IOMallocType(IOInterruptAccountingData); 111 112 reserved->statistics->owner = this; 113 } 114 115 res = (kIOReturnSuccess == registerInterruptHandler(inProvider, inIntIndex)); 116 117 if (res) { 118 intIndex = inIntIndex; 119 } 120 } 121 122 IOStatisticsInitializeCounter(); 123 124 return res; 125 } 126 127 IOReturn 128 IOInterruptEventSource::registerInterruptHandler(IOService *inProvider, 129 int inIntIndex) 130 { 131 IOReturn ret; 132 int intType; 133 IOInterruptAction intHandler; 134 135 ret = inProvider->getInterruptType(inIntIndex, &intType); 136 if (kIOReturnSuccess != ret) { 137 return ret; 138 } 139 140 autoDisable = (intType == kIOInterruptTypeLevel); 141 if (autoDisable) { 142 intHandler = OSMemberFunctionCast(IOInterruptAction, 143 this, &IOInterruptEventSource::disableInterruptOccurred); 144 } else { 145 intHandler = OSMemberFunctionCast(IOInterruptAction, 146 this, &IOInterruptEventSource::normalInterruptOccurred); 147 } 148 149 ret = provider->registerInterrupt(inIntIndex, this, intHandler); 150 151 /* 152 * Add statistics to the provider. The setWorkLoop convention should ensure 153 * that we always go down the unregister path before we register (outside of 154 * init()), so we don't have to worry that we will invoke addInterruptStatistics 155 * erroneously. 156 */ 157 if ((ret == kIOReturnSuccess) && (reserved->statistics)) { 158 /* 159 * Stash the normal index value, for the sake of debugging. 160 */ 161 reserved->statistics->interruptIndex = inIntIndex; 162 163 /* 164 * We need to hook the interrupt information up to the provider so that it 165 * can find the statistics for this interrupt when desired. The provider is 166 * responsible for maintaining the reporter for a particular interrupt, and 167 * needs a handle on the statistics so that it can request that the reporter 168 * be updated as needed. Errors are considered "soft" for the moment (it 169 * will either panic, or fail in a way such that we can still service the 170 * interrupt). 171 */ 172 provider->addInterruptStatistics(reserved->statistics, inIntIndex); 173 174 /* 175 * Add the statistics object to the global list of statistics objects; this 176 * is an aid to debugging (we can trivially find statistics for all eligible 177 * interrupts, and dump them; potentially helpful if the system is wedged 178 * due to interrupt activity). 179 */ 180 interruptAccountingDataAddToList(reserved->statistics); 181 } 182 183 return ret; 184 } 185 186 void 187 IOInterruptEventSource::unregisterInterruptHandler(IOService *inProvider, 188 int inIntIndex) 189 { 190 if (reserved->statistics) { 191 interruptAccountingDataRemoveFromList(reserved->statistics); 192 provider->removeInterruptStatistics(reserved->statistics->interruptIndex); 193 } 194 195 provider->unregisterInterrupt(inIntIndex); 196 } 197 198 199 OSSharedPtr<IOInterruptEventSource> 200 IOInterruptEventSource::interruptEventSource(OSObject *inOwner, 201 Action inAction, 202 IOService *inProvider, 203 int inIntIndex) 204 { 205 OSSharedPtr<IOInterruptEventSource> me = OSMakeShared<IOInterruptEventSource>(); 206 207 if (me && !me->init(inOwner, inAction, inProvider, inIntIndex)) { 208 return nullptr; 209 } 210 211 return me; 212 } 213 214 OSSharedPtr<IOInterruptEventSource> 215 IOInterruptEventSource::interruptEventSource(OSObject *inOwner, 216 IOService *inProvider, 217 int inIntIndex, 218 ActionBlock inAction) 219 { 220 OSSharedPtr<IOInterruptEventSource> ies; 221 ies = IOInterruptEventSource::interruptEventSource(inOwner, (Action) NULL, inProvider, inIntIndex); 222 if (ies) { 223 ies->setActionBlock((IOEventSource::ActionBlock) inAction); 224 } 225 226 return ies; 227 } 228 229 void 230 IOInterruptEventSource::free() 231 { 232 if (provider && intIndex >= 0) { 233 unregisterInterruptHandler(provider, intIndex); 234 } 235 236 if (reserved) { 237 if (reserved->statistics) { 238 IOFreeType(reserved->statistics, IOInterruptAccountingData); 239 } 240 241 IOFreeType(reserved, ExpansionData); 242 } 243 244 super::free(); 245 } 246 247 void 248 IOInterruptEventSource::enable() 249 { 250 if (provider && intIndex >= 0) { 251 provider->enableInterrupt(intIndex); 252 explicitDisable = false; 253 enabled = true; 254 } 255 } 256 257 void 258 IOInterruptEventSource::disable() 259 { 260 if (provider && intIndex >= 0) { 261 provider->disableInterrupt(intIndex); 262 explicitDisable = true; 263 enabled = false; 264 } 265 } 266 267 void 268 IOInterruptEventSource::setWorkLoop(IOWorkLoop *inWorkLoop) 269 { 270 if (inWorkLoop) { 271 super::setWorkLoop(inWorkLoop); 272 } 273 274 if (provider) { 275 if (!inWorkLoop) { 276 if (intIndex >= 0) { 277 /* 278 * It isn't necessarily safe to wait until free() to unregister the interrupt; 279 * our provider may disappear. 280 */ 281 unregisterInterruptHandler(provider, intIndex); 282 intIndex = ~intIndex; 283 } 284 } else if ((intIndex < 0) && (kIOReturnSuccess == registerInterruptHandler(provider, ~intIndex))) { 285 intIndex = ~intIndex; 286 } 287 } 288 289 if (!inWorkLoop) { 290 super::setWorkLoop(inWorkLoop); 291 } 292 } 293 294 const IOService * 295 IOInterruptEventSource::getProvider() const 296 { 297 return provider; 298 } 299 300 int 301 IOInterruptEventSource::getIntIndex() const 302 { 303 return intIndex; 304 } 305 306 bool 307 IOInterruptEventSource::getAutoDisable() const 308 { 309 return autoDisable; 310 } 311 312 bool 313 IOInterruptEventSource::checkForWork() 314 { 315 uint64_t startSystemTime = 0; 316 uint64_t endSystemTime = 0; 317 uint64_t startCPUTime = 0; 318 uint64_t endCPUTime = 0; 319 unsigned int cacheProdCount = producerCount; 320 int numInts = cacheProdCount - consumerCount; 321 IOEventSource::Action intAction = action; 322 ActionBlock intActionBlock = (ActionBlock) actionBlock; 323 void *address; 324 bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false; 325 326 if (kActionBlock & flags) { 327 address = ptrauth_nop_cast(void *, _Block_get_invoke_fn((struct Block_layout *)intActionBlock)); 328 } else { 329 address = ptrauth_nop_cast(void *, intAction); 330 } 331 332 IOStatisticsCheckForWork(); 333 334 if (numInts > 0) { 335 if (trace) { 336 IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION), 337 VM_KERNEL_ADDRHIDE(address), 338 VM_KERNEL_ADDRHIDE(owner), 339 VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); 340 } 341 342 if (reserved->statistics) { 343 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) { 344 startSystemTime = mach_absolute_time(); 345 } 346 347 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) { 348 startCPUTime = thread_get_runtime_self(); 349 } 350 } 351 352 // Call the handler 353 if (kActionBlock & flags) { 354 (intActionBlock)(this, numInts); 355 } else { 356 ((IOInterruptEventAction)intAction)(owner, this, numInts); 357 } 358 359 if (reserved->statistics) { 360 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCountIndex)) { 361 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCountIndex], 1); 362 } 363 364 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) { 365 endCPUTime = thread_get_runtime_self(); 366 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCPUTimeIndex], endCPUTime - startCPUTime); 367 } 368 369 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) { 370 endSystemTime = mach_absolute_time(); 371 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelSystemTimeIndex], endSystemTime - startSystemTime); 372 } 373 } 374 375 if (trace) { 376 IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION), 377 VM_KERNEL_ADDRHIDE(address), 378 VM_KERNEL_ADDRHIDE(owner), 379 VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); 380 } 381 382 consumerCount = cacheProdCount; 383 if (autoDisable && !explicitDisable) { 384 enable(); 385 } 386 } else if (numInts < 0) { 387 if (trace) { 388 IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION), 389 VM_KERNEL_ADDRHIDE(address), 390 VM_KERNEL_ADDRHIDE(owner), 391 VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); 392 } 393 394 if (reserved->statistics) { 395 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) { 396 startSystemTime = mach_absolute_time(); 397 } 398 399 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) { 400 startCPUTime = thread_get_runtime_self(); 401 } 402 } 403 404 // Call the handler 405 if (kActionBlock & flags) { 406 (intActionBlock)(this, numInts); 407 } else { 408 ((IOInterruptEventAction)intAction)(owner, this, numInts); 409 } 410 411 if (reserved->statistics) { 412 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCountIndex)) { 413 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCountIndex], 1); 414 } 415 416 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) { 417 endCPUTime = thread_get_runtime_self(); 418 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCPUTimeIndex], endCPUTime - startCPUTime); 419 } 420 421 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) { 422 endSystemTime = mach_absolute_time(); 423 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelSystemTimeIndex], endSystemTime - startSystemTime); 424 } 425 } 426 427 if (trace) { 428 IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION), 429 VM_KERNEL_ADDRHIDE(address), 430 VM_KERNEL_ADDRHIDE(owner), 431 VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); 432 } 433 434 consumerCount = cacheProdCount; 435 if (autoDisable && !explicitDisable) { 436 enable(); 437 } 438 } 439 440 return false; 441 } 442 443 void 444 IOInterruptEventSource::normalInterruptOccurred 445 (void */*refcon*/, IOService */*prov*/, int /*source*/) 446 { 447 bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false; 448 449 IOStatisticsInterrupt(); 450 producerCount++; 451 452 if (trace) { 453 IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); 454 } 455 456 if (reserved->statistics) { 457 if (reserved->statistics->enablePrimaryTimestamp) { 458 reserved->statistics->primaryTimestamp = mach_absolute_time(); 459 } 460 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) { 461 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1); 462 } 463 } 464 465 signalWorkAvailable(); 466 467 if (trace) { 468 IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); 469 } 470 } 471 472 void 473 IOInterruptEventSource::disableInterruptOccurred 474 (void */*refcon*/, IOService *prov, int source) 475 { 476 bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false; 477 478 prov->disableInterrupt(source); /* disable the interrupt */ 479 480 IOStatisticsInterrupt(); 481 producerCount++; 482 483 if (trace) { 484 IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); 485 } 486 487 if (reserved->statistics) { 488 if (reserved->statistics->enablePrimaryTimestamp) { 489 reserved->statistics->primaryTimestamp = mach_absolute_time(); 490 } 491 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) { 492 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1); 493 } 494 } 495 496 signalWorkAvailable(); 497 498 if (trace) { 499 IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); 500 } 501 } 502 503 void 504 IOInterruptEventSource::interruptOccurred 505 (void *_refcon, IOService *prov, int source) 506 { 507 if (autoDisable && prov) { 508 disableInterruptOccurred(_refcon, prov, source); 509 } else { 510 normalInterruptOccurred(_refcon, prov, source); 511 } 512 } 513 514 IOReturn 515 IOInterruptEventSource::warmCPU 516 (uint64_t abstime) 517 { 518 return ml_interrupt_prewarm(abstime); 519 } 520 521 void 522 IOInterruptEventSource::enablePrimaryInterruptTimestamp(bool enable) 523 { 524 if (reserved->statistics) { 525 reserved->statistics->enablePrimaryTimestamp = enable; 526 } 527 } 528 529 uint64_t 530 IOInterruptEventSource::getPrimaryInterruptTimestamp() 531 { 532 if (reserved->statistics && reserved->statistics->enablePrimaryTimestamp) { 533 return reserved->statistics->primaryTimestamp; 534 } 535 return -1ULL; 536 } 537