1 /* 2 * Copyright (c) 1998-2010 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #include <pexpert/pexpert.h> 30 #include <IOKit/IOWorkLoop.h> 31 #include <IOKit/IOEventSource.h> 32 #include <IOKit/IOInterruptEventSource.h> 33 #include <IOKit/IOCommandGate.h> 34 #include <IOKit/IOTimeStamp.h> 35 #include <IOKit/IOKitDebug.h> 36 #include <libkern/OSDebug.h> 37 #include <kern/thread.h> 38 39 #define super OSObject 40 41 OSDefineMetaClassAndStructors(IOWorkLoop, OSObject); 42 43 // Block of unused functions intended for future use 44 #if __LP64__ 45 OSMetaClassDefineReservedUnused(IOWorkLoop, 0); 46 OSMetaClassDefineReservedUnused(IOWorkLoop, 1); 47 OSMetaClassDefineReservedUnused(IOWorkLoop, 2); 48 #else 49 OSMetaClassDefineReservedUsed(IOWorkLoop, 0); 50 OSMetaClassDefineReservedUsed(IOWorkLoop, 1); 51 OSMetaClassDefineReservedUsed(IOWorkLoop, 2); 52 #endif 53 OSMetaClassDefineReservedUnused(IOWorkLoop, 3); 54 OSMetaClassDefineReservedUnused(IOWorkLoop, 4); 55 OSMetaClassDefineReservedUnused(IOWorkLoop, 5); 56 OSMetaClassDefineReservedUnused(IOWorkLoop, 6); 57 OSMetaClassDefineReservedUnused(IOWorkLoop, 7); 58 59 enum IOWorkLoopState { kLoopRestart = 0x1, kLoopTerminate = 0x2 }; 60 static inline void SETP(void *addr, unsigned int flag) 61 { unsigned char *num = (unsigned char *) addr; *num |= flag; } 62 static inline void CLRP(void *addr, unsigned int flag) 63 { unsigned char *num = (unsigned char *) addr; *num &= ~flag; } 64 static inline bool ISSETP(void *addr, unsigned int flag) 65 { unsigned char *num = (unsigned char *) addr; return (*num & flag) != 0; } 66 67 #define fFlags loopRestart 68 69 #define passiveEventChain reserved->passiveEventChain 70 71 #if IOKITSTATS 72 73 #define IOStatisticsRegisterCounter() \ 74 do { \ 75 reserved->counter = IOStatistics::registerWorkLoop(this); \ 76 } while(0) 77 78 #define IOStatisticsUnregisterCounter() \ 79 do { \ 80 if (reserved) \ 81 IOStatistics::unregisterWorkLoop(reserved->counter); \ 82 } while(0) 83 84 #define IOStatisticsOpenGate() \ 85 do { \ 86 IOStatistics::countWorkLoopOpenGate(reserved->counter); \ 87 } while(0) 88 89 #define IOStatisticsCloseGate() \ 90 do { \ 91 IOStatistics::countWorkLoopCloseGate(reserved->counter); \ 92 } while(0) 93 94 #define IOStatisticsAttachEventSource() \ 95 do { \ 96 IOStatistics::attachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \ 97 } while(0) 98 99 #define IOStatisticsDetachEventSource() \ 100 do { \ 101 IOStatistics::detachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \ 102 } while(0) 103 104 #else 105 106 #define IOStatisticsRegisterCounter() 107 #define IOStatisticsUnregisterCounter() 108 #define IOStatisticsOpenGate() 109 #define IOStatisticsCloseGate() 110 #define IOStatisticsAttachEventSource() 111 #define IOStatisticsDetachEventSource() 112 113 #endif /* IOKITSTATS */ 114 115 bool IOWorkLoop::init() 116 { 117 // The super init and gateLock allocation MUST be done first. 118 if ( !super::init() ) 119 return false; 120 121 // Allocate our ExpansionData if it hasn't been allocated already. 122 if ( !reserved ) 123 { 124 reserved = IONew(ExpansionData,1); 125 if ( !reserved ) 126 return false; 127 128 bzero(reserved,sizeof(ExpansionData)); 129 } 130 131 #if DEBUG 132 OSBacktrace ( reserved->allocationBacktrace, sizeof ( reserved->allocationBacktrace ) / sizeof ( reserved->allocationBacktrace[0] ) ); 133 #endif 134 135 if ( gateLock == NULL ) { 136 if ( !( gateLock = IORecursiveLockAlloc()) ) 137 return false; 138 } 139 140 if ( workToDoLock == NULL ) { 141 if ( !(workToDoLock = IOSimpleLockAlloc()) ) 142 return false; 143 IOSimpleLockInit(workToDoLock); 144 workToDo = false; 145 } 146 147 IOStatisticsRegisterCounter(); 148 149 if ( controlG == NULL ) { 150 controlG = IOCommandGate::commandGate( 151 this, 152 OSMemberFunctionCast( 153 IOCommandGate::Action, 154 this, 155 &IOWorkLoop::_maintRequest)); 156 157 if ( !controlG ) 158 return false; 159 // Point the controlGate at the workLoop. Usually addEventSource 160 // does this automatically. The problem is in this case addEventSource 161 // uses the control gate and it has to be bootstrapped. 162 controlG->setWorkLoop(this); 163 if (addEventSource(controlG) != kIOReturnSuccess) 164 return false; 165 } 166 167 if ( workThread == NULL ) { 168 thread_continue_t cptr = OSMemberFunctionCast( 169 thread_continue_t, 170 this, 171 &IOWorkLoop::threadMain); 172 if (KERN_SUCCESS != kernel_thread_start(cptr, this, &workThread)) 173 return false; 174 } 175 176 (void) thread_set_tag(workThread, THREAD_TAG_IOWORKLOOP); 177 return true; 178 } 179 180 IOWorkLoop * 181 IOWorkLoop::workLoop() 182 { 183 return IOWorkLoop::workLoopWithOptions(0); 184 } 185 186 IOWorkLoop * 187 IOWorkLoop::workLoopWithOptions(IOOptionBits options) 188 { 189 IOWorkLoop *me = new IOWorkLoop; 190 191 if (me && options) { 192 me->reserved = IONew(ExpansionData,1); 193 if (!me->reserved) { 194 me->release(); 195 return 0; 196 } 197 bzero(me->reserved,sizeof(ExpansionData)); 198 me->reserved->options = options; 199 } 200 201 if (me && !me->init()) { 202 me->release(); 203 return 0; 204 } 205 206 return me; 207 } 208 209 // Free is called twice: 210 // First when the atomic retainCount transitions from 1 -> 0 211 // Secondly when the work loop itself is commiting hari kari 212 // Hence the each leg of the free must be single threaded. 213 void IOWorkLoop::free() 214 { 215 if (workThread) { 216 IOInterruptState is; 217 218 // If we are here then we must be trying to shut down this work loop 219 // in this case disable all of the event source, mark the loop 220 // as terminating and wakeup the work thread itself and return 221 // Note: we hold the gate across the entire operation mainly for the 222 // benefit of our event sources so we can disable them cleanly. 223 closeGate(); 224 225 disableAllEventSources(); 226 227 is = IOSimpleLockLockDisableInterrupt(workToDoLock); 228 SETP(&fFlags, kLoopTerminate); 229 thread_wakeup_one((void *) &workToDo); 230 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); 231 232 openGate(); 233 } 234 else /* !workThread */ { 235 IOEventSource *event, *next; 236 237 for (event = eventChain; event; event = next) { 238 next = event->getNext(); 239 event->setWorkLoop(0); 240 event->setNext(0); 241 event->release(); 242 } 243 eventChain = 0; 244 245 for (event = passiveEventChain; event; event = next) { 246 next = event->getNext(); 247 event->setWorkLoop(0); 248 event->setNext(0); 249 event->release(); 250 } 251 passiveEventChain = 0; 252 253 // Either we have a partial initialization to clean up 254 // or the workThread itself is performing hari-kari. 255 // Either way clean up all of our resources and return. 256 257 if (controlG) { 258 controlG->release(); 259 controlG = 0; 260 } 261 262 if (workToDoLock) { 263 IOSimpleLockFree(workToDoLock); 264 workToDoLock = 0; 265 } 266 267 if (gateLock) { 268 IORecursiveLockFree(gateLock); 269 gateLock = 0; 270 } 271 272 IOStatisticsUnregisterCounter(); 273 274 if (reserved) { 275 IODelete(reserved, ExpansionData, 1); 276 reserved = 0; 277 } 278 279 super::free(); 280 } 281 } 282 283 IOReturn IOWorkLoop::addEventSource(IOEventSource *newEvent) 284 { 285 return controlG->runCommand((void *) mAddEvent, (void *) newEvent); 286 } 287 288 IOReturn IOWorkLoop::removeEventSource(IOEventSource *toRemove) 289 { 290 return controlG->runCommand((void *) mRemoveEvent, (void *) toRemove); 291 } 292 293 void IOWorkLoop::enableAllEventSources() const 294 { 295 IOEventSource *event; 296 297 for (event = eventChain; event; event = event->getNext()) 298 event->enable(); 299 300 for (event = passiveEventChain; event; event = event->getNext()) 301 event->enable(); 302 } 303 304 void IOWorkLoop::disableAllEventSources() const 305 { 306 IOEventSource *event; 307 308 for (event = eventChain; event; event = event->getNext()) 309 event->disable(); 310 311 /* NOTE: controlG is in passiveEventChain since it's an IOCommandGate */ 312 for (event = passiveEventChain; event; event = event->getNext()) 313 if (event != controlG) // Don't disable the control gate 314 event->disable(); 315 } 316 317 void IOWorkLoop::enableAllInterrupts() const 318 { 319 IOEventSource *event; 320 321 for (event = eventChain; event; event = event->getNext()) 322 if (OSDynamicCast(IOInterruptEventSource, event)) 323 event->enable(); 324 } 325 326 void IOWorkLoop::disableAllInterrupts() const 327 { 328 IOEventSource *event; 329 330 for (event = eventChain; event; event = event->getNext()) 331 if (OSDynamicCast(IOInterruptEventSource, event)) 332 event->disable(); 333 } 334 335 336 /* virtual */ bool IOWorkLoop::runEventSources() 337 { 338 bool res = false; 339 bool traceWL = (gIOKitTrace & kIOTraceWorkLoops) ? true : false; 340 bool traceES = (gIOKitTrace & kIOTraceEventSources) ? true : false; 341 342 closeGate(); 343 if (ISSETP(&fFlags, kLoopTerminate)) 344 goto abort; 345 346 if (traceWL) 347 IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this); 348 349 bool more; 350 do { 351 CLRP(&fFlags, kLoopRestart); 352 more = false; 353 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock); 354 workToDo = false; 355 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); 356 /* NOTE: only loop over event sources in eventChain. Bypass "passive" event sources for performance */ 357 for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) { 358 359 if (traceES) 360 IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt); 361 362 more |= evnt->checkForWork(); 363 364 if (traceES) 365 IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt); 366 367 if (ISSETP(&fFlags, kLoopTerminate)) 368 goto abort; 369 else if (fFlags & kLoopRestart) { 370 more = true; 371 break; 372 } 373 } 374 } while (more); 375 376 res = true; 377 378 if (traceWL) 379 IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this); 380 381 abort: 382 openGate(); 383 return res; 384 } 385 386 /* virtual */ void IOWorkLoop::threadMain() 387 { 388 restartThread: 389 do { 390 if ( !runEventSources() ) 391 goto exitThread; 392 393 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock); 394 if ( !ISSETP(&fFlags, kLoopTerminate) && !workToDo) { 395 assert_wait((void *) &workToDo, false); 396 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); 397 thread_continue_t cptr = NULL; 398 if (!reserved || !(kPreciousStack & reserved->options)) 399 cptr = OSMemberFunctionCast( 400 thread_continue_t, this, &IOWorkLoop::threadMain); 401 thread_block_parameter(cptr, this); 402 goto restartThread; 403 /* NOTREACHED */ 404 } 405 406 // At this point we either have work to do or we need 407 // to commit suicide. But no matter 408 // Clear the simple lock and retore the interrupt state 409 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); 410 } while(workToDo); 411 412 exitThread: 413 thread_t thread = workThread; 414 workThread = 0; // Say we don't have a loop and free ourselves 415 free(); 416 417 thread_deallocate(thread); 418 (void) thread_terminate(thread); 419 } 420 421 IOThread IOWorkLoop::getThread() const 422 { 423 return workThread; 424 } 425 426 bool IOWorkLoop::onThread() const 427 { 428 return (IOThreadSelf() == workThread); 429 } 430 431 bool IOWorkLoop::inGate() const 432 { 433 return IORecursiveLockHaveLock(gateLock); 434 } 435 436 // Internal APIs used by event sources to control the thread 437 void IOWorkLoop::signalWorkAvailable() 438 { 439 if (workToDoLock) { 440 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock); 441 workToDo = true; 442 thread_wakeup_one((void *) &workToDo); 443 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); 444 } 445 } 446 447 void IOWorkLoop::openGate() 448 { 449 IOStatisticsOpenGate(); 450 IORecursiveLockUnlock(gateLock); 451 } 452 453 void IOWorkLoop::closeGate() 454 { 455 IORecursiveLockLock(gateLock); 456 IOStatisticsCloseGate(); 457 } 458 459 bool IOWorkLoop::tryCloseGate() 460 { 461 bool res = (IORecursiveLockTryLock(gateLock) != 0); 462 if (res) { 463 IOStatisticsCloseGate(); 464 } 465 return res; 466 } 467 468 int IOWorkLoop::sleepGate(void *event, UInt32 interuptibleType) 469 { 470 int res; 471 IOStatisticsOpenGate(); 472 res = IORecursiveLockSleep(gateLock, event, interuptibleType); 473 IOStatisticsCloseGate(); 474 return res; 475 } 476 477 int IOWorkLoop::sleepGate(void *event, AbsoluteTime deadline, UInt32 interuptibleType) 478 { 479 int res; 480 IOStatisticsOpenGate(); 481 res = IORecursiveLockSleepDeadline(gateLock, event, deadline, interuptibleType); 482 IOStatisticsCloseGate(); 483 return res; 484 } 485 486 void IOWorkLoop::wakeupGate(void *event, bool oneThread) 487 { 488 IORecursiveLockWakeup(gateLock, event, oneThread); 489 } 490 491 IOReturn IOWorkLoop::runAction(Action inAction, OSObject *target, 492 void *arg0, void *arg1, 493 void *arg2, void *arg3) 494 { 495 IOReturn res; 496 497 // closeGate is recursive so don't worry if we already hold the lock. 498 closeGate(); 499 res = (*inAction)(target, arg0, arg1, arg2, arg3); 500 openGate(); 501 502 return res; 503 } 504 505 IOReturn IOWorkLoop::_maintRequest(void *inC, void *inD, void *, void *) 506 { 507 maintCommandEnum command = (maintCommandEnum) (uintptr_t) inC; 508 IOEventSource *inEvent = (IOEventSource *) inD; 509 IOReturn res = kIOReturnSuccess; 510 511 switch (command) 512 { 513 case mAddEvent: 514 if (!inEvent->getWorkLoop()) { 515 SETP(&fFlags, kLoopRestart); 516 517 inEvent->retain(); 518 inEvent->setWorkLoop(this); 519 inEvent->setNext(0); 520 521 /* Check if this is a passive or active event source being added */ 522 if (eventSourcePerformsWork(inEvent)) { 523 524 if (!eventChain) 525 eventChain = inEvent; 526 else { 527 IOEventSource *event, *next; 528 529 for (event = eventChain; (next = event->getNext()); event = next) 530 ; 531 event->setNext(inEvent); 532 533 } 534 535 } 536 else { 537 538 if (!passiveEventChain) 539 passiveEventChain = inEvent; 540 else { 541 IOEventSource *event, *next; 542 543 for (event = passiveEventChain; (next = event->getNext()); event = next) 544 ; 545 event->setNext(inEvent); 546 547 } 548 549 } 550 IOStatisticsAttachEventSource(); 551 } 552 break; 553 554 case mRemoveEvent: 555 if (inEvent->getWorkLoop()) { 556 IOStatisticsDetachEventSource(); 557 558 if (eventSourcePerformsWork(inEvent)) { 559 if (eventChain == inEvent) 560 eventChain = inEvent->getNext(); 561 else { 562 IOEventSource *event, *next; 563 564 event = eventChain; 565 while ((next = event->getNext()) && next != inEvent) 566 event = next; 567 568 if (!next) { 569 res = kIOReturnBadArgument; 570 break; 571 } 572 event->setNext(inEvent->getNext()); 573 } 574 } 575 else { 576 if (passiveEventChain == inEvent) 577 passiveEventChain = inEvent->getNext(); 578 else { 579 IOEventSource *event, *next; 580 581 event = passiveEventChain; 582 while ((next = event->getNext()) && next != inEvent) 583 event = next; 584 585 if (!next) { 586 res = kIOReturnBadArgument; 587 break; 588 } 589 event->setNext(inEvent->getNext()); 590 } 591 } 592 593 inEvent->setWorkLoop(0); 594 inEvent->setNext(0); 595 inEvent->release(); 596 SETP(&fFlags, kLoopRestart); 597 } 598 break; 599 600 default: 601 return kIOReturnUnsupported; 602 } 603 604 return res; 605 } 606 607 bool 608 IOWorkLoop::eventSourcePerformsWork(IOEventSource *inEventSource) 609 { 610 bool result = true; 611 612 /* 613 * The idea here is to see if the subclass of IOEventSource has overridden checkForWork(). 614 * The assumption is that if you override checkForWork(), you need to be 615 * active and not passive. 616 * 617 * We picked a known quantity controlG that does not override 618 * IOEventSource::checkForWork(), namely the IOCommandGate associated with 619 * the workloop to which this event source is getting attached. 620 * 621 * We do a pointer comparison on the offset in the vtable for inNewEvent against 622 * the offset in the vtable for inReferenceEvent. This works because 623 * IOCommandGate's slot for checkForWork() has the address of 624 * IOEventSource::checkForWork() in it. 625 * 626 * Think of OSMemberFunctionCast yielding the value at the vtable offset for 627 * checkForWork() here. We're just testing to see if it's the same or not. 628 * 629 */ 630 if (controlG) { 631 void * ptr1; 632 void * ptr2; 633 634 ptr1 = OSMemberFunctionCast(void*, inEventSource, &IOEventSource::checkForWork); 635 ptr2 = OSMemberFunctionCast(void*, controlG, &IOEventSource::checkForWork); 636 637 if (ptr1 == ptr2) 638 result = false; 639 } 640 641 return result; 642 } 643