1 /* 2 * Copyright (c) 1998-2010 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #include <pexpert/pexpert.h> 30 #include <IOKit/IOWorkLoop.h> 31 #include <IOKit/IOEventSource.h> 32 #include <IOKit/IOInterruptEventSource.h> 33 #include <IOKit/IOCommandGate.h> 34 #include <IOKit/IOTimeStamp.h> 35 #include <IOKit/IOKitDebug.h> 36 #include <libkern/OSDebug.h> 37 #include <kern/thread.h> 38 39 #define super OSObject 40 41 OSDefineMetaClassAndStructors(IOWorkLoop, OSObject); 42 43 // Block of unused functions intended for future use 44 #if __LP64__ 45 OSMetaClassDefineReservedUnused(IOWorkLoop, 0); 46 OSMetaClassDefineReservedUnused(IOWorkLoop, 1); 47 OSMetaClassDefineReservedUnused(IOWorkLoop, 2); 48 #else 49 OSMetaClassDefineReservedUsed(IOWorkLoop, 0); 50 OSMetaClassDefineReservedUsed(IOWorkLoop, 1); 51 OSMetaClassDefineReservedUsed(IOWorkLoop, 2); 52 #endif 53 OSMetaClassDefineReservedUnused(IOWorkLoop, 3); 54 OSMetaClassDefineReservedUnused(IOWorkLoop, 4); 55 OSMetaClassDefineReservedUnused(IOWorkLoop, 5); 56 OSMetaClassDefineReservedUnused(IOWorkLoop, 6); 57 OSMetaClassDefineReservedUnused(IOWorkLoop, 7); 58 59 enum IOWorkLoopState { kLoopRestart = 0x1, kLoopTerminate = 0x2 }; 60 static inline void SETP(void *addr, unsigned int flag) 61 { unsigned char *num = (unsigned char *) addr; *num |= flag; } 62 static inline void CLRP(void *addr, unsigned int flag) 63 { unsigned char *num = (unsigned char *) addr; *num &= ~flag; } 64 static inline bool ISSETP(void *addr, unsigned int flag) 65 { unsigned char *num = (unsigned char *) addr; return (*num & flag) != 0; } 66 67 #define fFlags loopRestart 68 69 #define passiveEventChain reserved->passiveEventChain 70 71 #if IOKITSTATS 72 73 #define IOStatisticsRegisterCounter() \ 74 do { \ 75 reserved->counter = IOStatistics::registerWorkLoop(this); \ 76 } while(0) 77 78 #define IOStatisticsUnregisterCounter() \ 79 do { \ 80 if (reserved) \ 81 IOStatistics::unregisterWorkLoop(reserved->counter); \ 82 } while(0) 83 84 #define IOStatisticsOpenGate() \ 85 do { \ 86 IOStatistics::countWorkLoopOpenGate(reserved->counter); \ 87 } while(0) 88 89 #define IOStatisticsCloseGate() \ 90 do { \ 91 IOStatistics::countWorkLoopCloseGate(reserved->counter); \ 92 } while(0) 93 94 #define IOStatisticsAttachEventSource() \ 95 do { \ 96 IOStatistics::attachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \ 97 } while(0) 98 99 #define IOStatisticsDetachEventSource() \ 100 do { \ 101 IOStatistics::detachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \ 102 } while(0) 103 104 #else 105 106 #define IOStatisticsRegisterCounter() 107 #define IOStatisticsUnregisterCounter() 108 #define IOStatisticsOpenGate() 109 #define IOStatisticsCloseGate() 110 #define IOStatisticsAttachEventSource() 111 #define IOStatisticsDetachEventSource() 112 113 #endif /* IOKITSTATS */ 114 115 bool IOWorkLoop::init() 116 { 117 // The super init and gateLock allocation MUST be done first. 118 if ( !super::init() ) 119 return false; 120 121 // Allocate our ExpansionData if it hasn't been allocated already. 122 if ( !reserved ) 123 { 124 reserved = IONew(ExpansionData,1); 125 if ( !reserved ) 126 return false; 127 128 bzero(reserved,sizeof(ExpansionData)); 129 } 130 131 #if DEBUG 132 OSBacktrace ( reserved->allocationBacktrace, sizeof ( reserved->allocationBacktrace ) / sizeof ( reserved->allocationBacktrace[0] ) ); 133 #endif 134 135 if ( gateLock == NULL ) { 136 if ( !( gateLock = IORecursiveLockAlloc()) ) 137 return false; 138 } 139 140 if ( workToDoLock == NULL ) { 141 if ( !(workToDoLock = IOSimpleLockAlloc()) ) 142 return false; 143 IOSimpleLockInit(workToDoLock); 144 workToDo = false; 145 } 146 147 IOStatisticsRegisterCounter(); 148 149 if ( controlG == NULL ) { 150 controlG = IOCommandGate::commandGate( 151 this, 152 OSMemberFunctionCast( 153 IOCommandGate::Action, 154 this, 155 &IOWorkLoop::_maintRequest)); 156 157 if ( !controlG ) 158 return false; 159 // Point the controlGate at the workLoop. Usually addEventSource 160 // does this automatically. The problem is in this case addEventSource 161 // uses the control gate and it has to be bootstrapped. 162 controlG->setWorkLoop(this); 163 if (addEventSource(controlG) != kIOReturnSuccess) 164 return false; 165 } 166 167 if ( workThread == NULL ) { 168 thread_continue_t cptr = OSMemberFunctionCast( 169 thread_continue_t, 170 this, 171 &IOWorkLoop::threadMain); 172 if (KERN_SUCCESS != kernel_thread_start(cptr, this, &workThread)) 173 return false; 174 } 175 176 (void) thread_set_tag(workThread, THREAD_TAG_IOWORKLOOP); 177 return true; 178 } 179 180 IOWorkLoop * 181 IOWorkLoop::workLoop() 182 { 183 return IOWorkLoop::workLoopWithOptions(0); 184 } 185 186 IOWorkLoop * 187 IOWorkLoop::workLoopWithOptions(IOOptionBits options) 188 { 189 IOWorkLoop *me = new IOWorkLoop; 190 191 if (me && options) { 192 me->reserved = IONew(ExpansionData,1); 193 if (!me->reserved) { 194 me->release(); 195 return 0; 196 } 197 bzero(me->reserved,sizeof(ExpansionData)); 198 me->reserved->options = options; 199 } 200 201 if (me && !me->init()) { 202 me->release(); 203 return 0; 204 } 205 206 return me; 207 } 208 209 // Free is called twice: 210 // First when the atomic retainCount transitions from 1 -> 0 211 // Secondly when the work loop itself is commiting hari kari 212 // Hence the each leg of the free must be single threaded. 213 void IOWorkLoop::free() 214 { 215 if (workThread) { 216 IOInterruptState is; 217 218 // If we are here then we must be trying to shut down this work loop 219 // in this case disable all of the event source, mark the loop 220 // as terminating and wakeup the work thread itself and return 221 // Note: we hold the gate across the entire operation mainly for the 222 // benefit of our event sources so we can disable them cleanly. 223 closeGate(); 224 225 disableAllEventSources(); 226 227 is = IOSimpleLockLockDisableInterrupt(workToDoLock); 228 SETP(&fFlags, kLoopTerminate); 229 thread_wakeup_one((void *) &workToDo); 230 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); 231 232 openGate(); 233 } 234 else /* !workThread */ { 235 IOEventSource *event, *next; 236 237 for (event = eventChain; event; event = next) { 238 next = event->getNext(); 239 event->setWorkLoop(0); 240 event->setNext(0); 241 event->release(); 242 } 243 eventChain = 0; 244 245 for (event = passiveEventChain; event; event = next) { 246 next = event->getNext(); 247 event->setWorkLoop(0); 248 event->setNext(0); 249 event->release(); 250 } 251 passiveEventChain = 0; 252 253 // Either we have a partial initialization to clean up 254 // or the workThread itself is performing hari-kari. 255 // Either way clean up all of our resources and return. 256 257 if (controlG) { 258 controlG->workLoop = 0; 259 controlG->release(); 260 controlG = 0; 261 } 262 263 if (workToDoLock) { 264 IOSimpleLockFree(workToDoLock); 265 workToDoLock = 0; 266 } 267 268 if (gateLock) { 269 IORecursiveLockFree(gateLock); 270 gateLock = 0; 271 } 272 273 IOStatisticsUnregisterCounter(); 274 275 if (reserved) { 276 IODelete(reserved, ExpansionData, 1); 277 reserved = 0; 278 } 279 280 super::free(); 281 } 282 } 283 284 IOReturn IOWorkLoop::addEventSource(IOEventSource *newEvent) 285 { 286 return controlG->runCommand((void *) mAddEvent, (void *) newEvent); 287 } 288 289 IOReturn IOWorkLoop::removeEventSource(IOEventSource *toRemove) 290 { 291 return controlG->runCommand((void *) mRemoveEvent, (void *) toRemove); 292 } 293 294 void IOWorkLoop::enableAllEventSources() const 295 { 296 IOEventSource *event; 297 298 for (event = eventChain; event; event = event->getNext()) 299 event->enable(); 300 301 for (event = passiveEventChain; event; event = event->getNext()) 302 event->enable(); 303 } 304 305 void IOWorkLoop::disableAllEventSources() const 306 { 307 IOEventSource *event; 308 309 for (event = eventChain; event; event = event->getNext()) 310 event->disable(); 311 312 /* NOTE: controlG is in passiveEventChain since it's an IOCommandGate */ 313 for (event = passiveEventChain; event; event = event->getNext()) 314 if (event != controlG) // Don't disable the control gate 315 event->disable(); 316 } 317 318 void IOWorkLoop::enableAllInterrupts() const 319 { 320 IOEventSource *event; 321 322 for (event = eventChain; event; event = event->getNext()) 323 if (OSDynamicCast(IOInterruptEventSource, event)) 324 event->enable(); 325 } 326 327 void IOWorkLoop::disableAllInterrupts() const 328 { 329 IOEventSource *event; 330 331 for (event = eventChain; event; event = event->getNext()) 332 if (OSDynamicCast(IOInterruptEventSource, event)) 333 event->disable(); 334 } 335 336 337 /* virtual */ bool IOWorkLoop::runEventSources() 338 { 339 bool res = false; 340 bool traceWL = (gIOKitTrace & kIOTraceWorkLoops) ? true : false; 341 bool traceES = (gIOKitTrace & kIOTraceEventSources) ? true : false; 342 343 closeGate(); 344 if (ISSETP(&fFlags, kLoopTerminate)) 345 goto abort; 346 347 if (traceWL) 348 IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this); 349 350 bool more; 351 do { 352 CLRP(&fFlags, kLoopRestart); 353 more = false; 354 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock); 355 workToDo = false; 356 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); 357 /* NOTE: only loop over event sources in eventChain. Bypass "passive" event sources for performance */ 358 for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) { 359 360 if (traceES) 361 IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt); 362 363 more |= evnt->checkForWork(); 364 365 if (traceES) 366 IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt); 367 368 if (ISSETP(&fFlags, kLoopTerminate)) 369 goto abort; 370 else if (fFlags & kLoopRestart) { 371 more = true; 372 break; 373 } 374 } 375 } while (more); 376 377 res = true; 378 379 if (traceWL) 380 IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this); 381 382 abort: 383 openGate(); 384 return res; 385 } 386 387 /* virtual */ void IOWorkLoop::threadMain() 388 { 389 restartThread: 390 do { 391 if ( !runEventSources() ) 392 goto exitThread; 393 394 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock); 395 if ( !ISSETP(&fFlags, kLoopTerminate) && !workToDo) { 396 assert_wait((void *) &workToDo, false); 397 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); 398 thread_continue_t cptr = NULL; 399 if (!reserved || !(kPreciousStack & reserved->options)) 400 cptr = OSMemberFunctionCast( 401 thread_continue_t, this, &IOWorkLoop::threadMain); 402 thread_block_parameter(cptr, this); 403 goto restartThread; 404 /* NOTREACHED */ 405 } 406 407 // At this point we either have work to do or we need 408 // to commit suicide. But no matter 409 // Clear the simple lock and retore the interrupt state 410 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); 411 } while(workToDo); 412 413 exitThread: 414 thread_t thread = workThread; 415 workThread = 0; // Say we don't have a loop and free ourselves 416 free(); 417 418 thread_deallocate(thread); 419 (void) thread_terminate(thread); 420 } 421 422 IOThread IOWorkLoop::getThread() const 423 { 424 return workThread; 425 } 426 427 bool IOWorkLoop::onThread() const 428 { 429 return (IOThreadSelf() == workThread); 430 } 431 432 bool IOWorkLoop::inGate() const 433 { 434 return IORecursiveLockHaveLock(gateLock); 435 } 436 437 // Internal APIs used by event sources to control the thread 438 void IOWorkLoop::signalWorkAvailable() 439 { 440 if (workToDoLock) { 441 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock); 442 workToDo = true; 443 thread_wakeup_one((void *) &workToDo); 444 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); 445 } 446 } 447 448 void IOWorkLoop::openGate() 449 { 450 IOStatisticsOpenGate(); 451 IORecursiveLockUnlock(gateLock); 452 } 453 454 void IOWorkLoop::closeGate() 455 { 456 IORecursiveLockLock(gateLock); 457 IOStatisticsCloseGate(); 458 } 459 460 bool IOWorkLoop::tryCloseGate() 461 { 462 bool res = (IORecursiveLockTryLock(gateLock) != 0); 463 if (res) { 464 IOStatisticsCloseGate(); 465 } 466 return res; 467 } 468 469 int IOWorkLoop::sleepGate(void *event, UInt32 interuptibleType) 470 { 471 int res; 472 IOStatisticsOpenGate(); 473 res = IORecursiveLockSleep(gateLock, event, interuptibleType); 474 IOStatisticsCloseGate(); 475 return res; 476 } 477 478 int IOWorkLoop::sleepGate(void *event, AbsoluteTime deadline, UInt32 interuptibleType) 479 { 480 int res; 481 IOStatisticsOpenGate(); 482 res = IORecursiveLockSleepDeadline(gateLock, event, deadline, interuptibleType); 483 IOStatisticsCloseGate(); 484 return res; 485 } 486 487 void IOWorkLoop::wakeupGate(void *event, bool oneThread) 488 { 489 IORecursiveLockWakeup(gateLock, event, oneThread); 490 } 491 492 IOReturn IOWorkLoop::runAction(Action inAction, OSObject *target, 493 void *arg0, void *arg1, 494 void *arg2, void *arg3) 495 { 496 IOReturn res; 497 498 // closeGate is recursive so don't worry if we already hold the lock. 499 closeGate(); 500 res = (*inAction)(target, arg0, arg1, arg2, arg3); 501 openGate(); 502 503 return res; 504 } 505 506 IOReturn IOWorkLoop::_maintRequest(void *inC, void *inD, void *, void *) 507 { 508 maintCommandEnum command = (maintCommandEnum) (uintptr_t) inC; 509 IOEventSource *inEvent = (IOEventSource *) inD; 510 IOReturn res = kIOReturnSuccess; 511 512 switch (command) 513 { 514 case mAddEvent: 515 if (!inEvent->getWorkLoop()) { 516 SETP(&fFlags, kLoopRestart); 517 518 inEvent->retain(); 519 inEvent->setWorkLoop(this); 520 inEvent->setNext(0); 521 522 /* Check if this is a passive or active event source being added */ 523 if (eventSourcePerformsWork(inEvent)) { 524 525 if (!eventChain) 526 eventChain = inEvent; 527 else { 528 IOEventSource *event, *next; 529 530 for (event = eventChain; (next = event->getNext()); event = next) 531 ; 532 event->setNext(inEvent); 533 534 } 535 536 } 537 else { 538 539 if (!passiveEventChain) 540 passiveEventChain = inEvent; 541 else { 542 IOEventSource *event, *next; 543 544 for (event = passiveEventChain; (next = event->getNext()); event = next) 545 ; 546 event->setNext(inEvent); 547 548 } 549 550 } 551 IOStatisticsAttachEventSource(); 552 } 553 break; 554 555 case mRemoveEvent: 556 if (inEvent->getWorkLoop()) { 557 IOStatisticsDetachEventSource(); 558 559 if (eventSourcePerformsWork(inEvent)) { 560 if (eventChain == inEvent) 561 eventChain = inEvent->getNext(); 562 else { 563 IOEventSource *event, *next = 0; 564 565 event = eventChain; 566 if (event) while ((next = event->getNext()) && (next != inEvent)) 567 event = next; 568 569 if (!next) { 570 res = kIOReturnBadArgument; 571 break; 572 } 573 event->setNext(inEvent->getNext()); 574 } 575 } 576 else { 577 if (passiveEventChain == inEvent) 578 passiveEventChain = inEvent->getNext(); 579 else { 580 IOEventSource *event, *next = 0; 581 582 event = passiveEventChain; 583 if (event) while ((next = event->getNext()) && (next != inEvent)) 584 event = next; 585 586 if (!next) { 587 res = kIOReturnBadArgument; 588 break; 589 } 590 event->setNext(inEvent->getNext()); 591 } 592 } 593 594 inEvent->setWorkLoop(0); 595 inEvent->setNext(0); 596 inEvent->release(); 597 SETP(&fFlags, kLoopRestart); 598 } 599 break; 600 601 default: 602 return kIOReturnUnsupported; 603 } 604 605 return res; 606 } 607 608 bool 609 IOWorkLoop::eventSourcePerformsWork(IOEventSource *inEventSource) 610 { 611 bool result = true; 612 613 /* 614 * The idea here is to see if the subclass of IOEventSource has overridden checkForWork(). 615 * The assumption is that if you override checkForWork(), you need to be 616 * active and not passive. 617 * 618 * We picked a known quantity controlG that does not override 619 * IOEventSource::checkForWork(), namely the IOCommandGate associated with 620 * the workloop to which this event source is getting attached. 621 * 622 * We do a pointer comparison on the offset in the vtable for inNewEvent against 623 * the offset in the vtable for inReferenceEvent. This works because 624 * IOCommandGate's slot for checkForWork() has the address of 625 * IOEventSource::checkForWork() in it. 626 * 627 * Think of OSMemberFunctionCast yielding the value at the vtable offset for 628 * checkForWork() here. We're just testing to see if it's the same or not. 629 * 630 */ 631 if (controlG) { 632 void * ptr1; 633 void * ptr2; 634 635 ptr1 = OSMemberFunctionCast(void*, inEventSource, &IOEventSource::checkForWork); 636 ptr2 = OSMemberFunctionCast(void*, controlG, &IOEventSource::checkForWork); 637 638 if (ptr1 == ptr2) 639 result = false; 640 } 641 642 return result; 643 } 644