xref: /xnu-11215/iokit/Kernel/IOWorkLoop.cpp (revision 855239e5) 
1 /*
2  * Copyright (c) 1998-2010 Apple Inc. All rights reserved.
3  *
4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5  *
6  * This file contains Original Code and/or Modifications of Original Code
7  * as defined in and that are subject to the Apple Public Source License
8  * Version 2.0 (the 'License'). You may not use this file except in
9  * compliance with the License. The rights granted to you under the License
10  * may not be used to create, or enable the creation or redistribution of,
11  * unlawful or unlicensed copies of an Apple operating system, or to
12  * circumvent, violate, or enable the circumvention or violation of, any
13  * terms of an Apple operating system software license agreement.
14  *
15  * Please obtain a copy of the License at
16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
17  *
18  * The Original Code and all software distributed under the License are
19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23  * Please see the License for the specific language governing rights and
24  * limitations under the License.
25  *
26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27  */
28 
29 #include <pexpert/pexpert.h>
30 #include <IOKit/IOWorkLoop.h>
31 #include <IOKit/IOEventSource.h>
32 #include <IOKit/IOInterruptEventSource.h>
33 #include <IOKit/IOCommandGate.h>
34 #include <IOKit/IOTimeStamp.h>
35 #include <IOKit/IOKitDebug.h>
36 #include <libkern/OSDebug.h>
37 
38 #define super OSObject
39 
40 OSDefineMetaClassAndStructors(IOWorkLoop, OSObject);
41 
42 // Block of unused functions intended for future use
43 #if __LP64__
44 OSMetaClassDefineReservedUnused(IOWorkLoop, 0);
45 OSMetaClassDefineReservedUnused(IOWorkLoop, 1);
46 OSMetaClassDefineReservedUnused(IOWorkLoop, 2);
47 #else
48 OSMetaClassDefineReservedUsed(IOWorkLoop, 0);
49 OSMetaClassDefineReservedUsed(IOWorkLoop, 1);
50 OSMetaClassDefineReservedUsed(IOWorkLoop, 2);
51 #endif
52 OSMetaClassDefineReservedUnused(IOWorkLoop, 3);
53 OSMetaClassDefineReservedUnused(IOWorkLoop, 4);
54 OSMetaClassDefineReservedUnused(IOWorkLoop, 5);
55 OSMetaClassDefineReservedUnused(IOWorkLoop, 6);
56 OSMetaClassDefineReservedUnused(IOWorkLoop, 7);
57 
58 enum IOWorkLoopState { kLoopRestart = 0x1, kLoopTerminate = 0x2 };
59 static inline void SETP(void *addr, unsigned int flag)
60     { unsigned char *num = (unsigned char *) addr; *num |= flag; }
61 static inline void CLRP(void *addr, unsigned int flag)
62     { unsigned char *num = (unsigned char *) addr; *num &= ~flag; }
63 static inline bool ISSETP(void *addr, unsigned int flag)
64     { unsigned char *num = (unsigned char *) addr; return (*num & flag) != 0; }
65 
66 #define fFlags loopRestart
67 
68 #define passiveEventChain	reserved->passiveEventChain
69 
70 #if IOKITSTATS
71 
72 #define IOStatisticsRegisterCounter() \
73 do { \
74 	reserved->counter = IOStatistics::registerWorkLoop(this); \
75 } while(0)
76 
77 #define IOStatisticsUnregisterCounter() \
78 do { \
79 	if (reserved) \
80 		IOStatistics::unregisterWorkLoop(reserved->counter); \
81 } while(0)
82 
83 #define IOStatisticsOpenGate() \
84 do { \
85 	IOStatistics::countWorkLoopOpenGate(reserved->counter); \
86 } while(0)
87 
88 #define IOStatisticsCloseGate() \
89 do { \
90 	IOStatistics::countWorkLoopCloseGate(reserved->counter); \
91 } while(0)
92 
93 #define IOStatisticsAttachEventSource() \
94 do { \
95 	IOStatistics::attachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \
96 } while(0)
97 
98 #define IOStatisticsDetachEventSource() \
99 do { \
100 	IOStatistics::detachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \
101 } while(0)
102 
103 #else
104 
105 #define IOStatisticsRegisterCounter()
106 #define IOStatisticsUnregisterCounter()
107 #define IOStatisticsOpenGate()
108 #define IOStatisticsCloseGate()
109 #define IOStatisticsAttachEventSource()
110 #define IOStatisticsDetachEventSource()
111 
112 #endif /* IOKITSTATS */
113 
114 bool IOWorkLoop::init()
115 {
116     // The super init and gateLock allocation MUST be done first.
117     if ( !super::init() )
118         return false;
119 
120 	// Allocate our ExpansionData if it hasn't been allocated already.
121 	if ( !reserved )
122 	{
123 		reserved = IONew(ExpansionData,1);
124 		if ( !reserved )
125 			return false;
126 
127 		bzero(reserved,sizeof(ExpansionData));
128 	}
129 
130 #if DEBUG
131 	OSBacktrace ( reserved->allocationBacktrace, sizeof ( reserved->allocationBacktrace ) / sizeof ( reserved->allocationBacktrace[0] ) );
132 #endif
133 
134     if ( gateLock == NULL ) {
135         if ( !( gateLock = IORecursiveLockAlloc()) )
136             return false;
137     }
138 
139     if ( workToDoLock == NULL ) {
140         if ( !(workToDoLock = IOSimpleLockAlloc()) )
141             return false;
142         IOSimpleLockInit(workToDoLock);
143         workToDo = false;
144     }
145 
146     if (!reserved) {
147         reserved = IONew(ExpansionData, 1);
148         reserved->options = 0;
149     }
150 
151     IOStatisticsRegisterCounter();
152 
153     if ( controlG == NULL ) {
154         controlG = IOCommandGate::commandGate(
155             this,
156             OSMemberFunctionCast(
157                 IOCommandGate::Action,
158                 this,
159                 &IOWorkLoop::_maintRequest));
160 
161         if ( !controlG )
162             return false;
163         // Point the controlGate at the workLoop.  Usually addEventSource
164         // does this automatically.  The problem is in this case addEventSource
165         // uses the control gate and it has to be bootstrapped.
166         controlG->setWorkLoop(this);
167         if (addEventSource(controlG) != kIOReturnSuccess)
168             return false;
169     }
170 
171     if ( workThread == NULL ) {
172         thread_continue_t cptr = OSMemberFunctionCast(
173             thread_continue_t,
174             this,
175             &IOWorkLoop::threadMain);
176         if (KERN_SUCCESS != kernel_thread_start(cptr, this, &workThread))
177             return false;
178     }
179 
180     return true;
181 }
182 
183 IOWorkLoop *
184 IOWorkLoop::workLoop()
185 {
186     return IOWorkLoop::workLoopWithOptions(0);
187 }
188 
189 IOWorkLoop *
190 IOWorkLoop::workLoopWithOptions(IOOptionBits options)
191 {
192 	IOWorkLoop *me = new IOWorkLoop;
193 
194 	if (me && options) {
195 		me->reserved = IONew(ExpansionData,1);
196 		if (!me->reserved) {
197 			me->release();
198 			return 0;
199 		}
200 		bzero(me->reserved,sizeof(ExpansionData));
201 		me->reserved->options = options;
202 	}
203 
204 	if (me && !me->init()) {
205 		me->release();
206 		return 0;
207 	}
208 
209 	return me;
210 }
211 
212 // Free is called twice:
213 // First when the atomic retainCount transitions from 1 -> 0
214 // Secondly when the work loop itself is commiting hari kari
215 // Hence the each leg of the free must be single threaded.
216 void IOWorkLoop::free()
217 {
218     if (workThread) {
219 	IOInterruptState is;
220 
221 	// If we are here then we must be trying to shut down this work loop
222 	// in this case disable all of the event source, mark the loop
223 	// as terminating and wakeup the work thread itself and return
224 	// Note: we hold the gate across the entire operation mainly for the
225 	// benefit of our event sources so we can disable them cleanly.
226 	closeGate();
227 
228 	disableAllEventSources();
229 
230         is = IOSimpleLockLockDisableInterrupt(workToDoLock);
231 	SETP(&fFlags, kLoopTerminate);
232         thread_wakeup_one((void *) &workToDo);
233         IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
234 
235 	openGate();
236     }
237     else /* !workThread */ {
238         IOEventSource *event, *next;
239 
240         for (event = eventChain; event; event = next) {
241             next = event->getNext();
242             event->setWorkLoop(0);
243             event->setNext(0);
244             event->release();
245         }
246         eventChain = 0;
247 
248         for (event = passiveEventChain; event; event = next) {
249             next = event->getNext();
250             event->setWorkLoop(0);
251             event->setNext(0);
252             event->release();
253         }
254         passiveEventChain = 0;
255 
256 	// Either we have a partial initialization to clean up
257 	// or the workThread itself is performing hari-kari.
258 	// Either way clean up all of our resources and return.
259 
260 	if (controlG) {
261 	    controlG->release();
262 	    controlG = 0;
263 	}
264 
265 	if (workToDoLock) {
266 	    IOSimpleLockFree(workToDoLock);
267 	    workToDoLock = 0;
268 	}
269 
270 	if (gateLock) {
271 	    IORecursiveLockFree(gateLock);
272 	    gateLock = 0;
273 	}
274 
275 	IOStatisticsUnregisterCounter();
276 
277 	if (reserved) {
278 	    IODelete(reserved, ExpansionData, 1);
279 	    reserved = 0;
280 	}
281 
282 	super::free();
283     }
284 }
285 
286 IOReturn IOWorkLoop::addEventSource(IOEventSource *newEvent)
287 {
288     return controlG->runCommand((void *) mAddEvent, (void *) newEvent);
289 }
290 
291 IOReturn IOWorkLoop::removeEventSource(IOEventSource *toRemove)
292 {
293     return controlG->runCommand((void *) mRemoveEvent, (void *) toRemove);
294 }
295 
296 void IOWorkLoop::enableAllEventSources() const
297 {
298     IOEventSource *event;
299 
300     for (event = eventChain; event; event = event->getNext())
301         event->enable();
302 
303     for (event = passiveEventChain; event; event = event->getNext())
304         event->enable();
305 }
306 
307 void IOWorkLoop::disableAllEventSources() const
308 {
309     IOEventSource *event;
310 
311     for (event = eventChain; event; event = event->getNext())
312 		event->disable();
313 
314 	/* NOTE: controlG is in passiveEventChain since it's an IOCommandGate */
315     for (event = passiveEventChain; event; event = event->getNext())
316         if (event != controlG)	// Don't disable the control gate
317             event->disable();
318 }
319 
320 void IOWorkLoop::enableAllInterrupts() const
321 {
322     IOEventSource *event;
323 
324     for (event = eventChain; event; event = event->getNext())
325         if (OSDynamicCast(IOInterruptEventSource, event))
326             event->enable();
327 }
328 
329 void IOWorkLoop::disableAllInterrupts() const
330 {
331     IOEventSource *event;
332 
333     for (event = eventChain; event; event = event->getNext())
334         if (OSDynamicCast(IOInterruptEventSource, event))
335             event->disable();
336 }
337 
338 
339 /* virtual */ bool IOWorkLoop::runEventSources()
340 {
341     bool res = false;
342     bool traceWL = (gIOKitTrace & kIOTraceWorkLoops) ? true : false;
343     bool traceES = (gIOKitTrace & kIOTraceEventSources) ? true : false;
344 
345     closeGate();
346     if (ISSETP(&fFlags, kLoopTerminate))
347 		goto abort;
348 
349     if (traceWL)
350     	IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this);
351 
352     bool more;
353     do {
354 		CLRP(&fFlags, kLoopRestart);
355 		more = false;
356 		IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
357 		workToDo = false;
358 		IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
359 		/* NOTE: only loop over event sources in eventChain. Bypass "passive" event sources for performance */
360 		for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) {
361 
362 			if (traceES)
363 				IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt);
364 
365 			more |= evnt->checkForWork();
366 
367 			if (traceES)
368 				IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt);
369 
370 			if (ISSETP(&fFlags, kLoopTerminate))
371 				goto abort;
372 			else if (fFlags & kLoopRestart) {
373 				more = true;
374 				break;
375 			}
376 		}
377     } while (more);
378 
379     res = true;
380 
381     if (traceWL)
382     	IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this);
383 
384 abort:
385     openGate();
386     return res;
387 }
388 
389 /* virtual */ void IOWorkLoop::threadMain()
390 {
391 restartThread:
392     do {
393 	if ( !runEventSources() )
394 	    goto exitThread;
395 
396 	IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
397         if ( !ISSETP(&fFlags, kLoopTerminate) && !workToDo) {
398 	    assert_wait((void *) &workToDo, false);
399 	    IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
400 	    thread_continue_t cptr = NULL;
401 	    if (!reserved || !(kPreciousStack & reserved->options))
402 		cptr = OSMemberFunctionCast(
403 			thread_continue_t, this, &IOWorkLoop::threadMain);
404 	    thread_block_parameter(cptr, this);
405 	    goto restartThread;
406 	    /* NOTREACHED */
407 	}
408 
409 	// At this point we either have work to do or we need
410 	// to commit suicide.  But no matter
411 	// Clear the simple lock and retore the interrupt state
412 	IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
413     } while(workToDo);
414 
415 exitThread:
416 	thread_t thread = workThread;
417     workThread = 0;	// Say we don't have a loop and free ourselves
418     free();
419 
420 	thread_deallocate(thread);
421     (void) thread_terminate(thread);
422 }
423 
424 IOThread IOWorkLoop::getThread() const
425 {
426     return workThread;
427 }
428 
429 bool IOWorkLoop::onThread() const
430 {
431     return (IOThreadSelf() == workThread);
432 }
433 
434 bool IOWorkLoop::inGate() const
435 {
436     return IORecursiveLockHaveLock(gateLock);
437 }
438 
439 // Internal APIs used by event sources to control the thread
440 void IOWorkLoop::signalWorkAvailable()
441 {
442     if (workToDoLock) {
443         IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
444         workToDo = true;
445         thread_wakeup_one((void *) &workToDo);
446         IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
447     }
448 }
449 
450 void IOWorkLoop::openGate()
451 {
452     IOStatisticsOpenGate();
453     IORecursiveLockUnlock(gateLock);
454 }
455 
456 void IOWorkLoop::closeGate()
457 {
458     IORecursiveLockLock(gateLock);
459     IOStatisticsCloseGate();
460 }
461 
462 bool IOWorkLoop::tryCloseGate()
463 {
464     bool res = (IORecursiveLockTryLock(gateLock) != 0);
465     if (res) {
466         IOStatisticsCloseGate();
467     }
468     return res;
469 }
470 
471 int IOWorkLoop::sleepGate(void *event, UInt32 interuptibleType)
472 {
473     int res;
474     IOStatisticsOpenGate();
475     res = IORecursiveLockSleep(gateLock, event, interuptibleType);
476     IOStatisticsCloseGate();
477     return res;
478 }
479 
480 int IOWorkLoop::sleepGate(void *event, AbsoluteTime deadline, UInt32 interuptibleType)
481 {
482     int res;
483     IOStatisticsOpenGate();
484     res = IORecursiveLockSleepDeadline(gateLock, event, deadline, interuptibleType);
485     IOStatisticsCloseGate();
486     return res;
487 }
488 
489 void IOWorkLoop::wakeupGate(void *event, bool oneThread)
490 {
491     IORecursiveLockWakeup(gateLock, event, oneThread);
492 }
493 
494 IOReturn IOWorkLoop::runAction(Action inAction, OSObject *target,
495                                   void *arg0, void *arg1,
496                                   void *arg2, void *arg3)
497 {
498     IOReturn res;
499 
500     // closeGate is recursive so don't worry if we already hold the lock.
501     closeGate();
502     res = (*inAction)(target, arg0, arg1, arg2, arg3);
503     openGate();
504 
505     return res;
506 }
507 
508 IOReturn IOWorkLoop::_maintRequest(void *inC, void *inD, void *, void *)
509 {
510     maintCommandEnum command = (maintCommandEnum) (uintptr_t) inC;
511     IOEventSource *inEvent = (IOEventSource *) inD;
512     IOReturn res = kIOReturnSuccess;
513 
514     switch (command)
515     {
516     case mAddEvent:
517         if (!inEvent->getWorkLoop()) {
518             SETP(&fFlags, kLoopRestart);
519 
520             inEvent->retain();
521             inEvent->setWorkLoop(this);
522             inEvent->setNext(0);
523 
524     		/* Check if this is a passive or active event source being added */
525     		if (eventSourcePerformsWork(inEvent)) {
526 
527 	            if (!eventChain)
528     	            eventChain = inEvent;
529         	    else {
530             	    IOEventSource *event, *next;
531 
532                 	for (event = eventChain; (next = event->getNext()); event = next)
533                     	;
534                 	event->setNext(inEvent);
535 
536             	}
537 
538             }
539             else {
540 
541 	            if (!passiveEventChain)
542     	            passiveEventChain = inEvent;
543         	    else {
544             	    IOEventSource *event, *next;
545 
546                 	for (event = passiveEventChain; (next = event->getNext()); event = next)
547                     	;
548                 	event->setNext(inEvent);
549 
550             	}
551 
552             }
553             IOStatisticsAttachEventSource();
554         }
555         break;
556 
557     case mRemoveEvent:
558         if (inEvent->getWorkLoop()) {
559         	if (eventSourcePerformsWork(inEvent)) {
560 				if (eventChain == inEvent)
561 					eventChain = inEvent->getNext();
562 				else {
563 					IOEventSource *event, *next;
564 
565 					event = eventChain;
566 					while ((next = event->getNext()) && next != inEvent)
567 						event = next;
568 
569 					if (!next) {
570 						res = kIOReturnBadArgument;
571 						break;
572 					}
573 					event->setNext(inEvent->getNext());
574 				}
575     		}
576     		else {
577 				if (passiveEventChain == inEvent)
578 					passiveEventChain = inEvent->getNext();
579 				else {
580 					IOEventSource *event, *next;
581 
582 					event = passiveEventChain;
583 					while ((next = event->getNext()) && next != inEvent)
584 						event = next;
585 
586 					if (!next) {
587 						res = kIOReturnBadArgument;
588 						break;
589 					}
590 					event->setNext(inEvent->getNext());
591 				}
592     		}
593 
594             inEvent->setWorkLoop(0);
595             inEvent->setNext(0);
596             inEvent->release();
597             SETP(&fFlags, kLoopRestart);
598             IOStatisticsDetachEventSource();
599         }
600         break;
601 
602     default:
603         return kIOReturnUnsupported;
604     }
605 
606     return res;
607 }
608 
609 bool
610 IOWorkLoop::eventSourcePerformsWork(IOEventSource *inEventSource)
611 {
612 	bool	result = true;
613 
614 	/*
615 	 * The idea here is to see if the subclass of IOEventSource has overridden checkForWork().
616 	 * The assumption is that if you override checkForWork(), you need to be
617 	 * active and not passive.
618 	 *
619 	 * We picked a known quantity controlG that does not override
620 	 * IOEventSource::checkForWork(), namely the IOCommandGate associated with
621 	 * the workloop to which this event source is getting attached.
622 	 *
623 	 * We do a pointer comparison on the offset in the vtable for inNewEvent against
624 	 * the offset in the vtable for inReferenceEvent. This works because
625 	 * IOCommandGate's slot for checkForWork() has the address of
626 	 * IOEventSource::checkForWork() in it.
627 	 *
628 	 * Think of OSMemberFunctionCast yielding the value at the vtable offset for
629 	 * checkForWork() here. We're just testing to see if it's the same or not.
630 	 *
631 	 */
632 	if (controlG) {
633 		void *	ptr1;
634 		void *	ptr2;
635 
636 		ptr1 = OSMemberFunctionCast(void*, inEventSource, &IOEventSource::checkForWork);
637 		ptr2 = OSMemberFunctionCast(void*, controlG, &IOEventSource::checkForWork);
638 
639 		if (ptr1 == ptr2)
640 			result = false;
641 	}
642 
643     return result;
644 }
645