1 /*
2  * Copyright (c) 1998-2014 Apple Inc. All rights reserved.
3  *
4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5  *
6  * This file contains Original Code and/or Modifications of Original Code
7  * as defined in and that are subject to the Apple Public Source License
8  * Version 2.0 (the 'License'). You may not use this file except in
9  * compliance with the License. The rights granted to you under the License
10  * may not be used to create, or enable the creation or redistribution of,
11  * unlawful or unlicensed copies of an Apple operating system, or to
12  * circumvent, violate, or enable the circumvention or violation of, any
13  * terms of an Apple operating system software license agreement.
14  *
15  * Please obtain a copy of the License at
16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
17  *
18  * The Original Code and all software distributed under the License are
19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23  * Please see the License for the specific language governing rights and
24  * limitations under the License.
25  *
26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27  */
28 
29 #include <IOKit/IOInterruptEventSource.h>
30 #include <IOKit/IOKitDebug.h>
31 #include <IOKit/IOLib.h>
32 #include <IOKit/IOService.h>
33 #include <IOKit/IOInterrupts.h>
34 #include <IOKit/IOTimeStamp.h>
35 #include <IOKit/IOWorkLoop.h>
36 #include <IOKit/IOInterruptAccountingPrivate.h>
37 
38 #if IOKITSTATS
39 
40 #define IOStatisticsInitializeCounter() \
41 do { \
42 	IOStatistics::setCounterType(IOEventSource::reserved->counter, kIOStatisticsInterruptEventSourceCounter); \
43 } while (0)
44 
45 #define IOStatisticsCheckForWork() \
46 do { \
47 	IOStatistics::countInterruptCheckForWork(IOEventSource::reserved->counter); \
48 } while (0)
49 
50 #define IOStatisticsInterrupt() \
51 do { \
52 	IOStatistics::countInterrupt(IOEventSource::reserved->counter); \
53 } while (0)
54 
55 #else
56 
57 #define IOStatisticsInitializeCounter()
58 #define IOStatisticsCheckForWork()
59 #define IOStatisticsInterrupt()
60 
61 #endif // IOKITSTATS
62 
63 #define super IOEventSource
64 
65 OSDefineMetaClassAndStructors(IOInterruptEventSource, IOEventSource)
66 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 0);
67 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 1);
68 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 2);
69 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 3);
70 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 4);
71 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 5);
72 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 6);
73 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 7);
74 
75 bool
76 IOInterruptEventSource::init(OSObject *inOwner,
77     Action inAction,
78     IOService *inProvider,
79     int inIntIndex)
80 {
81 	bool res = true;
82 
83 	if (!super::init(inOwner, (IOEventSourceAction) inAction)) {
84 		return false;
85 	}
86 
87 	reserved = IONew(ExpansionData, 1);
88 
89 	if (!reserved) {
90 		return false;
91 	}
92 
93 	bzero(reserved, sizeof(ExpansionData));
94 
95 	provider = inProvider;
96 	producerCount = consumerCount = 0;
97 	autoDisable = explicitDisable = false;
98 	intIndex = ~inIntIndex;
99 
100 	// Assumes inOwner holds a reference(retain) on the provider
101 	if (inProvider) {
102 		if (IA_ANY_STATISTICS_ENABLED) {
103 			/*
104 			 * We only treat this as an "interrupt" if it has a provider; if it does,
105 			 * set up the objects necessary to track interrupt statistics.  Interrupt
106 			 * event sources without providers are most likely being used as simple
107 			 * event source in order to poke at workloops and kick off work.
108 			 *
109 			 * We also avoid try to avoid interrupt accounting overhead if none of
110 			 * the statistics are enabled.
111 			 */
112 			reserved->statistics = IONew(IOInterruptAccountingData, 1);
113 
114 			if (!reserved->statistics) {
115 				/*
116 				 * We rely on the free() routine to clean up after us if init fails
117 				 * midway.
118 				 */
119 				return false;
120 			}
121 
122 			bzero(reserved->statistics, sizeof(IOInterruptAccountingData));
123 
124 			reserved->statistics->owner = this;
125 		}
126 
127 		res = (kIOReturnSuccess == registerInterruptHandler(inProvider, inIntIndex));
128 
129 		if (res) {
130 			intIndex = inIntIndex;
131 		}
132 	}
133 
134 	IOStatisticsInitializeCounter();
135 
136 	return res;
137 }
138 
139 IOReturn
140 IOInterruptEventSource::registerInterruptHandler(IOService *inProvider,
141     int inIntIndex)
142 {
143 	IOReturn ret;
144 	int intType;
145 	IOInterruptAction intHandler;
146 
147 	ret = inProvider->getInterruptType(inIntIndex, &intType);
148 	if (kIOReturnSuccess != ret) {
149 		return ret;
150 	}
151 
152 	autoDisable = (intType == kIOInterruptTypeLevel);
153 	if (autoDisable) {
154 		intHandler = OSMemberFunctionCast(IOInterruptAction,
155 		    this, &IOInterruptEventSource::disableInterruptOccurred);
156 	} else {
157 		intHandler = OSMemberFunctionCast(IOInterruptAction,
158 		    this, &IOInterruptEventSource::normalInterruptOccurred);
159 	}
160 
161 	ret = provider->registerInterrupt(inIntIndex, this, intHandler);
162 
163 	/*
164 	 * Add statistics to the provider.  The setWorkLoop convention should ensure
165 	 * that we always go down the unregister path before we register (outside of
166 	 * init()), so we don't have to worry that we will invoke addInterruptStatistics
167 	 * erroneously.
168 	 */
169 	if ((ret == kIOReturnSuccess) && (reserved->statistics)) {
170 		/*
171 		 * Stash the normal index value, for the sake of debugging.
172 		 */
173 		reserved->statistics->interruptIndex = inIntIndex;
174 
175 		/*
176 		 * We need to hook the interrupt information up to the provider so that it
177 		 * can find the statistics for this interrupt when desired.  The provider is
178 		 * responsible for maintaining the reporter for a particular interrupt, and
179 		 * needs a handle on the statistics so that it can request that the reporter
180 		 * be updated as needed.  Errors are considered "soft" for the moment (it
181 		 * will either panic, or fail in a way such that we can still service the
182 		 * interrupt).
183 		 */
184 		provider->addInterruptStatistics(reserved->statistics, inIntIndex);
185 
186 		/*
187 		 * Add the statistics object to the global list of statistics objects; this
188 		 * is an aid to debugging (we can trivially find statistics for all eligible
189 		 * interrupts, and dump them; potentially helpful if the system is wedged
190 		 * due to interrupt activity).
191 		 */
192 		interruptAccountingDataAddToList(reserved->statistics);
193 	}
194 
195 	return ret;
196 }
197 
198 void
199 IOInterruptEventSource::unregisterInterruptHandler(IOService *inProvider,
200     int inIntIndex)
201 {
202 	if (reserved->statistics) {
203 		interruptAccountingDataRemoveFromList(reserved->statistics);
204 		provider->removeInterruptStatistics(reserved->statistics->interruptIndex);
205 	}
206 
207 	provider->unregisterInterrupt(inIntIndex);
208 }
209 
210 
211 IOInterruptEventSource *
212 IOInterruptEventSource::interruptEventSource(OSObject *inOwner,
213     Action inAction,
214     IOService *inProvider,
215     int inIntIndex)
216 {
217 	IOInterruptEventSource *me = new IOInterruptEventSource;
218 
219 	if (me && !me->init(inOwner, inAction, inProvider, inIntIndex)) {
220 		me->release();
221 		return NULL;
222 	}
223 
224 	return me;
225 }
226 
227 IOInterruptEventSource *
228 IOInterruptEventSource::interruptEventSource(OSObject *inOwner,
229     IOService *inProvider,
230     int inIntIndex,
231     ActionBlock inAction)
232 {
233 	IOInterruptEventSource * ies;
234 	ies = IOInterruptEventSource::interruptEventSource(inOwner, (Action) NULL, inProvider, inIntIndex);
235 	if (ies) {
236 		ies->setActionBlock((IOEventSource::ActionBlock) inAction);
237 	}
238 
239 	return ies;
240 }
241 
242 void
243 IOInterruptEventSource::free()
244 {
245 	if (provider && intIndex >= 0) {
246 		unregisterInterruptHandler(provider, intIndex);
247 	}
248 
249 	if (reserved) {
250 		if (reserved->statistics) {
251 			IODelete(reserved->statistics, IOInterruptAccountingData, 1);
252 		}
253 
254 		IODelete(reserved, ExpansionData, 1);
255 	}
256 
257 	super::free();
258 }
259 
260 void
261 IOInterruptEventSource::enable()
262 {
263 	if (provider && intIndex >= 0) {
264 		provider->enableInterrupt(intIndex);
265 		explicitDisable = false;
266 		enabled = true;
267 	}
268 }
269 
270 void
271 IOInterruptEventSource::disable()
272 {
273 	if (provider && intIndex >= 0) {
274 		provider->disableInterrupt(intIndex);
275 		explicitDisable = true;
276 		enabled = false;
277 	}
278 }
279 
280 void
281 IOInterruptEventSource::setWorkLoop(IOWorkLoop *inWorkLoop)
282 {
283 	if (inWorkLoop) {
284 		super::setWorkLoop(inWorkLoop);
285 	}
286 
287 	if (provider) {
288 		if (!inWorkLoop) {
289 			if (intIndex >= 0) {
290 				/*
291 				 * It isn't necessarily safe to wait until free() to unregister the interrupt;
292 				 * our provider may disappear.
293 				 */
294 				unregisterInterruptHandler(provider, intIndex);
295 				intIndex = ~intIndex;
296 			}
297 		} else if ((intIndex < 0) && (kIOReturnSuccess == registerInterruptHandler(provider, ~intIndex))) {
298 			intIndex = ~intIndex;
299 		}
300 	}
301 
302 	if (!inWorkLoop) {
303 		super::setWorkLoop(inWorkLoop);
304 	}
305 }
306 
307 const IOService *
308 IOInterruptEventSource::getProvider() const
309 {
310 	return provider;
311 }
312 
313 int
314 IOInterruptEventSource::getIntIndex() const
315 {
316 	return intIndex;
317 }
318 
319 bool
320 IOInterruptEventSource::getAutoDisable() const
321 {
322 	return autoDisable;
323 }
324 
325 bool
326 IOInterruptEventSource::checkForWork()
327 {
328 	uint64_t startSystemTime = 0;
329 	uint64_t endSystemTime = 0;
330 	uint64_t startCPUTime = 0;
331 	uint64_t endCPUTime = 0;
332 	unsigned int cacheProdCount = producerCount;
333 	int numInts = cacheProdCount - consumerCount;
334 	IOInterruptEventAction intAction = (IOInterruptEventAction) action;
335 	ActionBlock intActionBlock = (ActionBlock) actionBlock;
336 	bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
337 
338 	IOStatisticsCheckForWork();
339 
340 	if (numInts > 0) {
341 		if (trace) {
342 			IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION),
343 			    VM_KERNEL_ADDRHIDE(intAction), VM_KERNEL_ADDRHIDE(owner),
344 			    VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
345 		}
346 
347 		if (reserved->statistics) {
348 			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
349 				startSystemTime = mach_absolute_time();
350 			}
351 
352 			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
353 				startCPUTime = thread_get_runtime_self();
354 			}
355 		}
356 
357 		// Call the handler
358 		if (kActionBlock & flags) {
359 			(intActionBlock)(this, numInts);
360 		} else {
361 			(*intAction)(owner, this, numInts);
362 		}
363 
364 		if (reserved->statistics) {
365 			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCountIndex)) {
366 				IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCountIndex], 1);
367 			}
368 
369 			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
370 				endCPUTime = thread_get_runtime_self();
371 				IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCPUTimeIndex], endCPUTime - startCPUTime);
372 			}
373 
374 			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
375 				endSystemTime = mach_absolute_time();
376 				IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelSystemTimeIndex], endSystemTime - startSystemTime);
377 			}
378 		}
379 
380 		if (trace) {
381 			IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION),
382 			    VM_KERNEL_ADDRHIDE(intAction), VM_KERNEL_ADDRHIDE(owner),
383 			    VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
384 		}
385 
386 		consumerCount = cacheProdCount;
387 		if (autoDisable && !explicitDisable) {
388 			enable();
389 		}
390 	} else if (numInts < 0) {
391 		if (trace) {
392 			IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION),
393 			    VM_KERNEL_ADDRHIDE(intAction), VM_KERNEL_ADDRHIDE(owner),
394 			    VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
395 		}
396 
397 		if (reserved->statistics) {
398 			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
399 				startSystemTime = mach_absolute_time();
400 			}
401 
402 			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
403 				startCPUTime = thread_get_runtime_self();
404 			}
405 		}
406 
407 		// Call the handler
408 		if (kActionBlock & flags) {
409 			(intActionBlock)(this, numInts);
410 		} else {
411 			(*intAction)(owner, this, numInts);
412 		}
413 
414 		if (reserved->statistics) {
415 			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCountIndex)) {
416 				IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCountIndex], 1);
417 			}
418 
419 			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
420 				endCPUTime = thread_get_runtime_self();
421 				IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCPUTimeIndex], endCPUTime - startCPUTime);
422 			}
423 
424 			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
425 				endSystemTime = mach_absolute_time();
426 				IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelSystemTimeIndex], endSystemTime - startSystemTime);
427 			}
428 		}
429 
430 		if (trace) {
431 			IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION),
432 			    VM_KERNEL_ADDRHIDE(intAction), VM_KERNEL_ADDRHIDE(owner),
433 			    VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
434 		}
435 
436 		consumerCount = cacheProdCount;
437 		if (autoDisable && !explicitDisable) {
438 			enable();
439 		}
440 	}
441 
442 	return false;
443 }
444 
445 void
446 IOInterruptEventSource::normalInterruptOccurred
447 (void */*refcon*/, IOService */*prov*/, int /*source*/)
448 {
449 	bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
450 
451 	IOStatisticsInterrupt();
452 	producerCount++;
453 
454 	if (trace) {
455 		IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
456 	}
457 
458 	if (reserved->statistics) {
459 		if (reserved->statistics->enablePrimaryTimestamp) {
460 			reserved->statistics->primaryTimestamp = mach_absolute_time();
461 		}
462 		if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) {
463 			IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1);
464 		}
465 	}
466 
467 	signalWorkAvailable();
468 
469 	if (trace) {
470 		IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
471 	}
472 }
473 
474 void
475 IOInterruptEventSource::disableInterruptOccurred
476 (void */*refcon*/, IOService *prov, int source)
477 {
478 	bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
479 
480 	prov->disableInterrupt(source); /* disable the interrupt */
481 
482 	IOStatisticsInterrupt();
483 	producerCount++;
484 
485 	if (trace) {
486 		IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
487 	}
488 
489 	if (reserved->statistics) {
490 		if (reserved->statistics->enablePrimaryTimestamp) {
491 			reserved->statistics->primaryTimestamp = mach_absolute_time();
492 		}
493 		if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) {
494 			IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1);
495 		}
496 	}
497 
498 	signalWorkAvailable();
499 
500 	if (trace) {
501 		IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
502 	}
503 }
504 
505 void
506 IOInterruptEventSource::interruptOccurred
507 (void *_refcon, IOService *prov, int source)
508 {
509 	if (autoDisable && prov) {
510 		disableInterruptOccurred(_refcon, prov, source);
511 	} else {
512 		normalInterruptOccurred(_refcon, prov, source);
513 	}
514 }
515 
516 IOReturn
517 IOInterruptEventSource::warmCPU
518 (uint64_t abstime)
519 {
520 	return ml_interrupt_prewarm(abstime);
521 }
522 
523 void
524 IOInterruptEventSource::enablePrimaryInterruptTimestamp(bool enable)
525 {
526 	if (reserved->statistics) {
527 		reserved->statistics->enablePrimaryTimestamp = enable;
528 	}
529 }
530 
531 uint64_t
532 IOInterruptEventSource::getPimaryInterruptTimestamp()
533 {
534 	if (reserved->statistics && reserved->statistics->enablePrimaryTimestamp) {
535 		return reserved->statistics->primaryTimestamp;
536 	}
537 	return -1ULL;
538 }
539