1 /*
2 * Copyright (c) 1998-2014 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #define IOKIT_ENABLE_SHARED_PTR
30
31 #include <ptrauth.h>
32 #include <IOKit/IOInterruptEventSource.h>
33 #include <IOKit/IOKitDebug.h>
34 #include <IOKit/IOLib.h>
35 #include <IOKit/IOService.h>
36 #include <IOKit/IOInterrupts.h>
37 #include <IOKit/IOTimeStamp.h>
38 #include <IOKit/IOWorkLoop.h>
39 #include <IOKit/IOInterruptAccountingPrivate.h>
40 #include <libkern/Block_private.h>
41
42 #if IOKITSTATS
43
44 #define IOStatisticsInitializeCounter() \
45 do { \
46 IOStatistics::setCounterType(IOEventSource::reserved->counter, kIOStatisticsInterruptEventSourceCounter); \
47 } while (0)
48
49 #define IOStatisticsCheckForWork() \
50 do { \
51 IOStatistics::countInterruptCheckForWork(IOEventSource::reserved->counter); \
52 } while (0)
53
54 #define IOStatisticsInterrupt() \
55 do { \
56 IOStatistics::countInterrupt(IOEventSource::reserved->counter); \
57 } while (0)
58
59 #else
60
61 #define IOStatisticsInitializeCounter()
62 #define IOStatisticsCheckForWork()
63 #define IOStatisticsInterrupt()
64
65 #endif // IOKITSTATS
66
67 #define super IOEventSource
68
69 OSDefineMetaClassAndStructors(IOInterruptEventSource, IOEventSource)
70 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 0);
71 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 1);
72 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 2);
73 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 3);
74 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 4);
75 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 5);
76 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 6);
77 OSMetaClassDefineReservedUnused(IOInterruptEventSource, 7);
78
79 bool
init(OSObject * inOwner,Action inAction,IOService * inProvider,int inIntIndex)80 IOInterruptEventSource::init(OSObject *inOwner,
81 Action inAction,
82 IOService *inProvider,
83 int inIntIndex)
84 {
85 bool res = true;
86
87 if (inIntIndex < 0) {
88 return false;
89 }
90
91 if (!super::init(inOwner, (IOEventSourceAction) inAction)) {
92 return false;
93 }
94
95 reserved = IOMallocType(ExpansionData);
96
97 provider = inProvider;
98 producerCount = consumerCount = 0;
99 autoDisable = explicitDisable = false;
100 intIndex = ~inIntIndex;
101
102 // Assumes inOwner holds a reference(retain) on the provider
103 if (inProvider) {
104 if (IA_ANY_STATISTICS_ENABLED) {
105 /*
106 * We only treat this as an "interrupt" if it has a provider; if it does,
107 * set up the objects necessary to track interrupt statistics. Interrupt
108 * event sources without providers are most likely being used as simple
109 * event source in order to poke at workloops and kick off work.
110 *
111 * We also avoid try to avoid interrupt accounting overhead if none of
112 * the statistics are enabled.
113 */
114 reserved->statistics = IOMallocType(IOInterruptAccountingData);
115
116 reserved->statistics->owner = this;
117 }
118
119 res = (kIOReturnSuccess == registerInterruptHandler(inProvider, inIntIndex));
120
121 if (res) {
122 intIndex = inIntIndex;
123 }
124 }
125
126 IOStatisticsInitializeCounter();
127
128 return res;
129 }
130
131 IOReturn
registerInterruptHandler(IOService * inProvider,int inIntIndex)132 IOInterruptEventSource::registerInterruptHandler(IOService *inProvider,
133 int inIntIndex)
134 {
135 IOReturn ret;
136 int intType;
137 IOInterruptAction intHandler;
138
139 ret = inProvider->getInterruptType(inIntIndex, &intType);
140 if (kIOReturnSuccess != ret) {
141 return ret;
142 }
143
144 autoDisable = (intType == kIOInterruptTypeLevel);
145 if (autoDisable) {
146 intHandler = OSMemberFunctionCast(IOInterruptAction,
147 this, &IOInterruptEventSource::disableInterruptOccurred);
148 } else {
149 intHandler = OSMemberFunctionCast(IOInterruptAction,
150 this, &IOInterruptEventSource::normalInterruptOccurred);
151 }
152
153 ret = provider->registerInterrupt(inIntIndex, this, intHandler);
154
155 /*
156 * Add statistics to the provider. The setWorkLoop convention should ensure
157 * that we always go down the unregister path before we register (outside of
158 * init()), so we don't have to worry that we will invoke addInterruptStatistics
159 * erroneously.
160 */
161 if ((ret == kIOReturnSuccess) && (reserved->statistics)) {
162 /*
163 * Stash the normal index value, for the sake of debugging.
164 */
165 reserved->statistics->interruptIndex = inIntIndex;
166
167 /*
168 * We need to hook the interrupt information up to the provider so that it
169 * can find the statistics for this interrupt when desired. The provider is
170 * responsible for maintaining the reporter for a particular interrupt, and
171 * needs a handle on the statistics so that it can request that the reporter
172 * be updated as needed. Errors are considered "soft" for the moment (it
173 * will either panic, or fail in a way such that we can still service the
174 * interrupt).
175 */
176 provider->addInterruptStatistics(reserved->statistics, inIntIndex);
177
178 /*
179 * Add the statistics object to the global list of statistics objects; this
180 * is an aid to debugging (we can trivially find statistics for all eligible
181 * interrupts, and dump them; potentially helpful if the system is wedged
182 * due to interrupt activity).
183 */
184 interruptAccountingDataAddToList(reserved->statistics);
185 }
186
187 return ret;
188 }
189
190 void
unregisterInterruptHandler(IOService * inProvider,int inIntIndex)191 IOInterruptEventSource::unregisterInterruptHandler(IOService *inProvider,
192 int inIntIndex)
193 {
194 if (reserved->statistics) {
195 interruptAccountingDataRemoveFromList(reserved->statistics);
196 provider->removeInterruptStatistics(reserved->statistics->interruptIndex);
197 }
198
199 provider->unregisterInterrupt(inIntIndex);
200 }
201
202
203 OSSharedPtr<IOInterruptEventSource>
interruptEventSource(OSObject * inOwner,Action inAction,IOService * inProvider,int inIntIndex)204 IOInterruptEventSource::interruptEventSource(OSObject *inOwner,
205 Action inAction,
206 IOService *inProvider,
207 int inIntIndex)
208 {
209 OSSharedPtr<IOInterruptEventSource> me = OSMakeShared<IOInterruptEventSource>();
210
211 if (me && !me->init(inOwner, inAction, inProvider, inIntIndex)) {
212 return nullptr;
213 }
214
215 return me;
216 }
217
218 OSSharedPtr<IOInterruptEventSource>
interruptEventSource(OSObject * inOwner,IOService * inProvider,int inIntIndex,ActionBlock inAction)219 IOInterruptEventSource::interruptEventSource(OSObject *inOwner,
220 IOService *inProvider,
221 int inIntIndex,
222 ActionBlock inAction)
223 {
224 OSSharedPtr<IOInterruptEventSource> ies;
225 ies = IOInterruptEventSource::interruptEventSource(inOwner, (Action) NULL, inProvider, inIntIndex);
226 if (ies) {
227 ies->setActionBlock((IOEventSource::ActionBlock) inAction);
228 }
229
230 return ies;
231 }
232
233 void
free()234 IOInterruptEventSource::free()
235 {
236 if (provider && intIndex >= 0) {
237 unregisterInterruptHandler(provider, intIndex);
238 }
239
240 if (reserved) {
241 if (reserved->statistics) {
242 IOFreeType(reserved->statistics, IOInterruptAccountingData);
243 }
244
245 IOFreeType(reserved, ExpansionData);
246 }
247
248 super::free();
249 }
250
251 void
enable()252 IOInterruptEventSource::enable()
253 {
254 if (provider && intIndex >= 0) {
255 provider->enableInterrupt(intIndex);
256 explicitDisable = false;
257 enabled = true;
258 }
259 }
260
261 void
disable()262 IOInterruptEventSource::disable()
263 {
264 if (provider && intIndex >= 0) {
265 provider->disableInterrupt(intIndex);
266 explicitDisable = true;
267 enabled = false;
268 }
269 }
270
271 void
setWorkLoop(IOWorkLoop * inWorkLoop)272 IOInterruptEventSource::setWorkLoop(IOWorkLoop *inWorkLoop)
273 {
274 if (inWorkLoop) {
275 super::setWorkLoop(inWorkLoop);
276 }
277
278 if (provider) {
279 if (!inWorkLoop) {
280 if (intIndex >= 0) {
281 /*
282 * It isn't necessarily safe to wait until free() to unregister the interrupt;
283 * our provider may disappear.
284 */
285 unregisterInterruptHandler(provider, intIndex);
286 intIndex = ~intIndex;
287 }
288 } else if ((intIndex < 0) && (kIOReturnSuccess == registerInterruptHandler(provider, ~intIndex))) {
289 intIndex = ~intIndex;
290 }
291 }
292
293 if (!inWorkLoop) {
294 super::setWorkLoop(inWorkLoop);
295 }
296 }
297
298 const IOService *
getProvider() const299 IOInterruptEventSource::getProvider() const
300 {
301 return provider;
302 }
303
304 int
getIntIndex() const305 IOInterruptEventSource::getIntIndex() const
306 {
307 return intIndex;
308 }
309
310 bool
getAutoDisable() const311 IOInterruptEventSource::getAutoDisable() const
312 {
313 return autoDisable;
314 }
315
316 bool
checkForWork()317 IOInterruptEventSource::checkForWork()
318 {
319 uint64_t startSystemTime = 0;
320 uint64_t endSystemTime = 0;
321 uint64_t startCPUTime = 0;
322 uint64_t endCPUTime = 0;
323 unsigned int cacheProdCount = producerCount;
324 int numInts = cacheProdCount - consumerCount;
325 IOEventSource::Action intAction = action;
326 ActionBlock intActionBlock = (ActionBlock) actionBlock;
327 void *address;
328 bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
329
330 if (kActionBlock & flags) {
331 address = ptrauth_nop_cast(void *, _Block_get_invoke_fn((struct Block_layout *)intActionBlock));
332 } else {
333 address = ptrauth_nop_cast(void *, intAction);
334 }
335
336 IOStatisticsCheckForWork();
337
338 if (numInts > 0) {
339 if (trace) {
340 IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION),
341 VM_KERNEL_ADDRHIDE(address),
342 VM_KERNEL_ADDRHIDE(owner),
343 VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
344 }
345
346 if (reserved->statistics) {
347 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
348 startSystemTime = mach_absolute_time();
349 }
350
351 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
352 startCPUTime = thread_get_runtime_self();
353 }
354 }
355
356 // Call the handler
357 if (kActionBlock & flags) {
358 (intActionBlock)(this, numInts);
359 } else {
360 ((IOInterruptEventAction)intAction)(owner, this, numInts);
361 }
362
363 if (reserved->statistics) {
364 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCountIndex)) {
365 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCountIndex], 1);
366 }
367
368 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
369 endCPUTime = thread_get_runtime_self();
370 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCPUTimeIndex], endCPUTime - startCPUTime);
371 }
372
373 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
374 endSystemTime = mach_absolute_time();
375 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelSystemTimeIndex], endSystemTime - startSystemTime);
376 }
377 }
378
379 if (trace) {
380 IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION),
381 VM_KERNEL_ADDRHIDE(address),
382 VM_KERNEL_ADDRHIDE(owner),
383 VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
384 }
385
386 consumerCount = cacheProdCount;
387 if (autoDisable && !explicitDisable) {
388 enable();
389 }
390 } else if (numInts < 0) {
391 if (trace) {
392 IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION),
393 VM_KERNEL_ADDRHIDE(address),
394 VM_KERNEL_ADDRHIDE(owner),
395 VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
396 }
397
398 if (reserved->statistics) {
399 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
400 startSystemTime = mach_absolute_time();
401 }
402
403 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
404 startCPUTime = thread_get_runtime_self();
405 }
406 }
407
408 // Call the handler
409 if (kActionBlock & flags) {
410 (intActionBlock)(this, numInts);
411 } else {
412 ((IOInterruptEventAction)intAction)(owner, this, numInts);
413 }
414
415 if (reserved->statistics) {
416 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
417 endSystemTime = mach_absolute_time();
418 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelSystemTimeIndex], endSystemTime - startSystemTime);
419 }
420
421 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
422 endCPUTime = thread_get_runtime_self();
423 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCPUTimeIndex], endCPUTime - startCPUTime);
424 }
425
426 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCountIndex)) {
427 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCountIndex], 1);
428 }
429 }
430
431 if (trace) {
432 IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION),
433 VM_KERNEL_ADDRHIDE(address),
434 VM_KERNEL_ADDRHIDE(owner),
435 VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop));
436 }
437
438 consumerCount = cacheProdCount;
439 if (autoDisable && !explicitDisable) {
440 enable();
441 }
442 }
443
444 return false;
445 }
446
447 void
normalInterruptOccurred(void *,IOService *,int)448 IOInterruptEventSource::normalInterruptOccurred
449 (void */*refcon*/, IOService */*prov*/, int /*source*/)
450 {
451 bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
452
453 IOStatisticsInterrupt();
454 producerCount++;
455
456 if (trace) {
457 IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
458 }
459
460 if (reserved->statistics) {
461 if (reserved->statistics->enablePrimaryTimestamp) {
462 reserved->statistics->primaryTimestamp = mach_absolute_time();
463 }
464 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) {
465 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1);
466 }
467 }
468
469 signalWorkAvailable();
470
471 if (trace) {
472 IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
473 }
474 }
475
476 void
disableInterruptOccurred(void *,IOService * prov,int source)477 IOInterruptEventSource::disableInterruptOccurred
478 (void */*refcon*/, IOService *prov, int source)
479 {
480 bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
481
482 prov->disableInterrupt(source); /* disable the interrupt */
483
484 IOStatisticsInterrupt();
485 producerCount++;
486
487 if (trace) {
488 IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
489 }
490
491 if (reserved->statistics) {
492 if (reserved->statistics->enablePrimaryTimestamp) {
493 reserved->statistics->primaryTimestamp = mach_absolute_time();
494 }
495 if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) {
496 IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1);
497 }
498 }
499
500 signalWorkAvailable();
501
502 if (trace) {
503 IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner));
504 }
505 }
506
507 void
interruptOccurred(void * _refcon,IOService * prov,int source)508 IOInterruptEventSource::interruptOccurred
509 (void *_refcon, IOService *prov, int source)
510 {
511 if (autoDisable && prov) {
512 disableInterruptOccurred(_refcon, prov, source);
513 } else {
514 normalInterruptOccurred(_refcon, prov, source);
515 }
516 }
517
518 IOReturn
warmCPU(uint64_t abstime)519 IOInterruptEventSource::warmCPU
520 (uint64_t abstime)
521 {
522 return ml_interrupt_prewarm(abstime);
523 }
524
525 void
enablePrimaryInterruptTimestamp(bool enable)526 IOInterruptEventSource::enablePrimaryInterruptTimestamp(bool enable)
527 {
528 if (reserved->statistics) {
529 reserved->statistics->enablePrimaryTimestamp = enable;
530 }
531 }
532
533 uint64_t
getPrimaryInterruptTimestamp()534 IOInterruptEventSource::getPrimaryInterruptTimestamp()
535 {
536 if (reserved->statistics && reserved->statistics->enablePrimaryTimestamp) {
537 return reserved->statistics->primaryTimestamp;
538 }
539 return -1ULL;
540 }
541