1 /*
2  * Copyright (c) 1998-2014 Apple Inc. All rights reserved.
3  *
4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5  *
6  * This file contains Original Code and/or Modifications of Original Code
7  * as defined in and that are subject to the Apple Public Source License
8  * Version 2.0 (the 'License'). You may not use this file except in
9  * compliance with the License. The rights granted to you under the License
10  * may not be used to create, or enable the creation or redistribution of,
11  * unlawful or unlicensed copies of an Apple operating system, or to
12  * circumvent, violate, or enable the circumvention or violation of, any
13  * terms of an Apple operating system software license agreement.
14  *
15  * Please obtain a copy of the License at
16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
17  *
18  * The Original Code and all software distributed under the License are
19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23  * Please see the License for the specific language governing rights and
24  * limitations under the License.
25  *
26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27  */
28 
29 #define IOKIT_ENABLE_SHARED_PTR
30 
31 #include <ptrauth.h>
32 #include <IOKit/IOInterruptEventSource.h>
33 #include <IOKit/IOKitDebug.h>
34 #include <IOKit/IOLib.h>
35 #include <IOKit/IOService.h>
36 #include <IOKit/IOInterrupts.h>
37 #include <IOKit/IOTimeStamp.h>
38 #include <IOKit/IOWorkLoop.h>
39 #include <IOKit/IOInterruptAccountingPrivate.h>
40 #include <libkern/Block_private.h>
41 
42 #if IOKITSTATS
43 
44 #define IOStatisticsInitializeCounter() \
45 do { \
46 	IOStatistics::setCounterType(IOEventSource::reserved->counter, kIOStatisticsInterruptEventSourceCounter); \
47 } while (0)
48 
49 #define IOStatisticsCheckForWork() \
50 do { \
51 	IOStatistics::countInterruptCheckForWork(IOEventSource::reserved->counter); \
52 } while (0)
53 
54 #define IOStatisticsInterrupt() \
55 do { \
56 	IOStatistics::countInterrupt(IOEventSource::reserved->counter); \
57 } while (0)
58 
59 #else
60 
61 #define IOStatisticsInitializeCounter()
62 #define IOStatisticsCheckForWork()
63 #define IOStatisticsInterrupt()
64 
65 #endif // IOKITSTATS
66 
67 #define super IOEventSource
68 
69 OSDefineMetaClassAndStructors(IOInterruptEventSource, IOEventSource)
70 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 0);
71 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 1);
72 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 2);
73 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 3);
74 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 4);
75 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 5);
76 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 6);
77 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 7);
78 
79 bool
80 IOInterruptEventSource::init(OSObject *inOwner,
81     Action inAction,
82     IOService *inProvider,
83     int inIntIndex)
84 {
85 	bool res = true;
86 
87 	if (!super::init(inOwner, (IOEventSourceAction) inAction)) {
88 		return false;
89 	}
90 
91 	reserved = IONew(ExpansionData, 1);
92 
93 	if (!reserved) {
94 		return false;
95 	}
96 
97 	bzero(reserved, sizeof(ExpansionData));
98 
99 	provider = inProvider;
100 	producerCount = consumerCount = 0;
101 	autoDisable = explicitDisable = false;
102 	intIndex = ~inIntIndex;
103 
104 	// Assumes inOwner holds a reference(retain) on the provider
105 	if (inProvider) {
106 		if (IA_ANY_STATISTICS_ENABLED) {
107 			/*
108 			 * We only treat this as an "interrupt" if it has a provider; if it does,
109 			 * set up the objects necessary to track interrupt statistics.  Interrupt
110 			 * event sources without providers are most likely being used as simple
111 			 * event source in order to poke at workloops and kick off work.
112 			 *
113 			 * We also avoid try to avoid interrupt accounting overhead if none of
114 			 * the statistics are enabled.
115 			 */
116 			reserved->statistics = IONew(IOInterruptAccountingData, 1);
117 
118 			if (!reserved->statistics) {
119 				/*
120 				 * We rely on the free() routine to clean up after us if init fails
121 				 * midway.
122 				 */
123 				return false;
124 			}
125 
126 			bzero(reserved->statistics, sizeof(IOInterruptAccountingData));
127 
128 			reserved->statistics->owner = this;
129 		}
130 
131 		res = (kIOReturnSuccess == registerInterruptHandler(inProvider, inIntIndex));
132 
133 		if (res) {
134 			intIndex = inIntIndex;
135 		}
136 	}
137 
138 	IOStatisticsInitializeCounter();
139 
140 	return res;
141 }
142 
143 IOReturn
144 IOInterruptEventSource::registerInterruptHandler(IOService *inProvider,
145     int inIntIndex)
146 {
147 	IOReturn ret;
148 	int intType;
149 	IOInterruptAction intHandler;
150 
151 	ret = inProvider->getInterruptType(inIntIndex, &intType);
152 	if (kIOReturnSuccess != ret) {
153 		return ret;
154 	}
155 
156 	autoDisable = (intType == kIOInterruptTypeLevel);
157 	if (autoDisable) {
158 		intHandler = OSMemberFunctionCast(IOInterruptAction,
159 		    this, &IOInterruptEventSource::disableInterruptOccurred);
160 	} else {
161 		intHandler = OSMemberFunctionCast(IOInterruptAction,
162 		    this, &IOInterruptEventSource::normalInterruptOccurred);
163 	}
164 
165 	ret = provider->registerInterrupt(inIntIndex, this, intHandler);
166 
167 	/*
168 	 * Add statistics to the provider.  The setWorkLoop convention should ensure
169 	 * that we always go down the unregister path before we register (outside of
170 	 * init()), so we don't have to worry that we will invoke addInterruptStatistics
171 	 * erroneously.
172 	 */
173 	if ((ret == kIOReturnSuccess) && (reserved->statistics)) {
174 		/*
175 		 * Stash the normal index value, for the sake of debugging.
176 		 */
177 		reserved->statistics->interruptIndex = inIntIndex;
178 
179 		/*
180 		 * We need to hook the interrupt information up to the provider so that it
181 		 * can find the statistics for this interrupt when desired.  The provider is
182 		 * responsible for maintaining the reporter for a particular interrupt, and
183 		 * needs a handle on the statistics so that it can request that the reporter
184 		 * be updated as needed.  Errors are considered "soft" for the moment (it
185 		 * will either panic, or fail in a way such that we can still service the
186 		 * interrupt).
187 		 */
188 		provider->addInterruptStatistics(reserved->statistics, inIntIndex);
189 
190 		/*
191 		 * Add the statistics object to the global list of statistics objects; this
192 		 * is an aid to debugging (we can trivially find statistics for all eligible
193 		 * interrupts, and dump them; potentially helpful if the system is wedged
194 		 * due to interrupt activity).
195 		 */
196 		interruptAccountingDataAddToList(reserved->statistics);
197 	}
198 
199 	return ret;
200 }
201 
202 void
203 IOInterruptEventSource::unregisterInterruptHandler(IOService *inProvider,
204     int inIntIndex)
205 {
206 	if (reserved->statistics) {
207 		interruptAccountingDataRemoveFromList(reserved->statistics);
208 		provider->removeInterruptStatistics(reserved->statistics->interruptIndex);
209 	}
210 
211 	provider->unregisterInterrupt(inIntIndex);
212 }
213 
214 
215 OSSharedPtr<IOInterruptEventSource>
216 IOInterruptEventSource::interruptEventSource(OSObject *inOwner,
217     Action inAction,
218     IOService *inProvider,
219     int inIntIndex)
220 {
221 	OSSharedPtr<IOInterruptEventSource> me = OSMakeShared<IOInterruptEventSource>();
222 
223 	if (me && !me->init(inOwner, inAction, inProvider, inIntIndex)) {
224 		return nullptr;
225 	}
226 
227 	return me;
228 }
229 
230 OSSharedPtr<IOInterruptEventSource>
231 IOInterruptEventSource::interruptEventSource(OSObject *inOwner,
232     IOService *inProvider,
233     int inIntIndex,
234     ActionBlock inAction)
235 {
236 	OSSharedPtr<IOInterruptEventSource> ies;
237 	ies = IOInterruptEventSource::interruptEventSource(inOwner, (Action) NULL, inProvider, inIntIndex);
238 	if (ies) {
239 		ies->setActionBlock((IOEventSource::ActionBlock) inAction);
240 	}
241 
242 	return ies;
243 }
244 
245 void
246 IOInterruptEventSource::free()
247 {
248 	if (provider && intIndex >= 0) {
249 		unregisterInterruptHandler(provider, intIndex);
250 	}
251 
252 	if (reserved) {
253 		if (reserved->statistics) {
254 			IODelete(reserved->statistics, IOInterruptAccountingData, 1);
255 		}
256 
257 		IODelete(reserved, ExpansionData, 1);
258 	}
259 
260 	super::free();
261 }
262 
263 void
264 IOInterruptEventSource::enable()
265 {
266 	if (provider && intIndex >= 0) {
267 		provider->enableInterrupt(intIndex);
268 		explicitDisable = false;
269 		enabled = true;
270 	}
271 }
272 
273 void
274 IOInterruptEventSource::disable()
275 {
276 	if (provider && intIndex >= 0) {
277 		provider->disableInterrupt(intIndex);
278 		explicitDisable = true;
279 		enabled = false;
280 	}
281 }
282 
283 void
284 IOInterruptEventSource::setWorkLoop(IOWorkLoop *inWorkLoop)
285 {
286 	if (inWorkLoop) {
287 		super::setWorkLoop(inWorkLoop);
288 	}
289 
290 	if (provider) {
291 		if (!inWorkLoop) {
292 			if (intIndex >= 0) {
293 				/*
294 				 * It isn't necessarily safe to wait until free() to unregister the interrupt;
295 				 * our provider may disappear.
296 				 */
297 				unregisterInterruptHandler(provider, intIndex);
298 				intIndex = ~intIndex;
299 			}
300 		} else if ((intIndex < 0) && (kIOReturnSuccess == registerInterruptHandler(provider, ~intIndex))) {
301 			intIndex = ~intIndex;
302 		}
303 	}
304 
305 	if (!inWorkLoop) {
306 		super::setWorkLoop(inWorkLoop);
307 	}
308 }
309 
310 const IOService *
311 IOInterruptEventSource::getProvider() const
312 {
313 	return provider;
314 }
315 
316 int
317 IOInterruptEventSource::getIntIndex() const
318 {
319 	return intIndex;
320 }
321 
322 bool
323 IOInterruptEventSource::getAutoDisable() const
324 {
325 	return autoDisable;
326 }
327 
328 bool
329 IOInterruptEventSource::checkForWork()
330 {
331 	uint64_t startSystemTime = 0;
332 	uint64_t endSystemTime = 0;
333 	uint64_t startCPUTime = 0;
334 	uint64_t endCPUTime = 0;
335 	unsigned int cacheProdCount = producerCount;
336 	int numInts = cacheProdCount - consumerCount;
337 	IOEventSource::Action intAction = action;
338 	ActionBlock intActionBlock = (ActionBlock) actionBlock;
339 	void *address;
340 	bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
341 
342 	if (kActionBlock & flags) {
343 		address = ptrauth_nop_cast(void *, _Block_get_invoke_fn((struct Block_layout *)intActionBlock));
344 	} else {
345 		address = ptrauth_nop_cast(void *, intAction);
346 	}
347 
348 	IOStatisticsCheckForWork();
349 
350 	if (numInts > 0) {
351 		if (trace) {
352 			IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION),
353 			    VM_KERNEL_ADDRHIDE(address),
354 			    VM_KERNEL_ADDRHIDE(owner),
355 			    VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
356 		}
357 
358 		if (reserved->statistics) {
359 			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
360 				startSystemTime = mach_absolute_time();
361 			}
362 
363 			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
364 				startCPUTime = thread_get_runtime_self();
365 			}
366 		}
367 
368 		// Call the handler
369 		if (kActionBlock & flags) {
370 			(intActionBlock)(this, numInts);
371 		} else {
372 			((IOInterruptEventAction)intAction)(owner, this, numInts);
373 		}
374 
375 		if (reserved->statistics) {
376 			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCountIndex)) {
377 				IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCountIndex], 1);
378 			}
379 
380 			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
381 				endCPUTime = thread_get_runtime_self();
382 				IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCPUTimeIndex], endCPUTime - startCPUTime);
383 			}
384 
385 			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
386 				endSystemTime = mach_absolute_time();
387 				IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelSystemTimeIndex], endSystemTime - startSystemTime);
388 			}
389 		}
390 
391 		if (trace) {
392 			IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION),
393 			    VM_KERNEL_ADDRHIDE(address),
394 			    VM_KERNEL_ADDRHIDE(owner),
395 			    VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
396 		}
397 
398 		consumerCount = cacheProdCount;
399 		if (autoDisable && !explicitDisable) {
400 			enable();
401 		}
402 	} else if (numInts < 0) {
403 		if (trace) {
404 			IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION),
405 			    VM_KERNEL_ADDRHIDE(address),
406 			    VM_KERNEL_ADDRHIDE(owner),
407 			    VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
408 		}
409 
410 		if (reserved->statistics) {
411 			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
412 				startSystemTime = mach_absolute_time();
413 			}
414 
415 			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
416 				startCPUTime = thread_get_runtime_self();
417 			}
418 		}
419 
420 		// Call the handler
421 		if (kActionBlock & flags) {
422 			(intActionBlock)(this, numInts);
423 		} else {
424 			((IOInterruptEventAction)intAction)(owner, this, numInts);
425 		}
426 
427 		if (reserved->statistics) {
428 			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCountIndex)) {
429 				IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCountIndex], 1);
430 			}
431 
432 			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
433 				endCPUTime = thread_get_runtime_self();
434 				IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCPUTimeIndex], endCPUTime - startCPUTime);
435 			}
436 
437 			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
438 				endSystemTime = mach_absolute_time();
439 				IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelSystemTimeIndex], endSystemTime - startSystemTime);
440 			}
441 		}
442 
443 		if (trace) {
444 			IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION),
445 			    VM_KERNEL_ADDRHIDE(address),
446 			    VM_KERNEL_ADDRHIDE(owner),
447 			    VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
448 		}
449 
450 		consumerCount = cacheProdCount;
451 		if (autoDisable && !explicitDisable) {
452 			enable();
453 		}
454 	}
455 
456 	return false;
457 }
458 
459 void
460 IOInterruptEventSource::normalInterruptOccurred
461 (void */*refcon*/, IOService */*prov*/, int /*source*/)
462 {
463 	bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
464 
465 	IOStatisticsInterrupt();
466 	producerCount++;
467 
468 	if (trace) {
469 		IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
470 	}
471 
472 	if (reserved->statistics) {
473 		if (reserved->statistics->enablePrimaryTimestamp) {
474 			reserved->statistics->primaryTimestamp = mach_absolute_time();
475 		}
476 		if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) {
477 			IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1);
478 		}
479 	}
480 
481 	signalWorkAvailable();
482 
483 	if (trace) {
484 		IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
485 	}
486 }
487 
488 void
489 IOInterruptEventSource::disableInterruptOccurred
490 (void */*refcon*/, IOService *prov, int source)
491 {
492 	bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
493 
494 	prov->disableInterrupt(source); /* disable the interrupt */
495 
496 	IOStatisticsInterrupt();
497 	producerCount++;
498 
499 	if (trace) {
500 		IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
501 	}
502 
503 	if (reserved->statistics) {
504 		if (reserved->statistics->enablePrimaryTimestamp) {
505 			reserved->statistics->primaryTimestamp = mach_absolute_time();
506 		}
507 		if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) {
508 			IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1);
509 		}
510 	}
511 
512 	signalWorkAvailable();
513 
514 	if (trace) {
515 		IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
516 	}
517 }
518 
519 void
520 IOInterruptEventSource::interruptOccurred
521 (void *_refcon, IOService *prov, int source)
522 {
523 	if (autoDisable && prov) {
524 		disableInterruptOccurred(_refcon, prov, source);
525 	} else {
526 		normalInterruptOccurred(_refcon, prov, source);
527 	}
528 }
529 
530 IOReturn
531 IOInterruptEventSource::warmCPU
532 (uint64_t abstime)
533 {
534 	return ml_interrupt_prewarm(abstime);
535 }
536 
537 void
538 IOInterruptEventSource::enablePrimaryInterruptTimestamp(bool enable)
539 {
540 	if (reserved->statistics) {
541 		reserved->statistics->enablePrimaryTimestamp = enable;
542 	}
543 }
544 
545 uint64_t
546 IOInterruptEventSource::getPrimaryInterruptTimestamp()
547 {
548 	if (reserved->statistics && reserved->statistics->enablePrimaryTimestamp) {
549 		return reserved->statistics->primaryTimestamp;
550 	}
551 	return -1ULL;
552 }
553