1 /*
2 * Copyright (c) 1998-2000, 2009-2010 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <ptrauth.h>
30 #include <sys/cdefs.h>
31
32 __BEGIN_DECLS
33 #include <kern/thread_call.h>
34 __END_DECLS
35
36 #include <IOKit/assert.h>
37 #include <IOKit/system.h>
38
39 #include <IOKit/IOLib.h>
40 #include <IOKit/IOTimerEventSource.h>
41 #include <IOKit/IOWorkLoop.h>
42
43 #include <IOKit/IOTimeStamp.h>
44 #include <IOKit/IOKitDebug.h>
45 #if CONFIG_DTRACE
46 #include <mach/sdt.h>
47 #endif
48
49 #include <libkern/Block.h>
50 #include <libkern/Block_private.h>
51
52
53 #define super IOEventSource
54 OSDefineMetaClassAndStructors(IOTimerEventSource, IOEventSource)
55 OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 0);
56 OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 1);
57 OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 2);
58 OSMetaClassDefineReservedUnused(IOTimerEventSource, 3);
59 OSMetaClassDefineReservedUnused(IOTimerEventSource, 4);
60 OSMetaClassDefineReservedUnused(IOTimerEventSource, 5);
61 OSMetaClassDefineReservedUnused(IOTimerEventSource, 6);
62 OSMetaClassDefineReservedUnused(IOTimerEventSource, 7);
63
64 #if IOKITSTATS
65
66 #define IOStatisticsInitializeCounter() \
67 do { \
68 IOStatistics::setCounterType(IOEventSource::reserved->counter, kIOStatisticsTimerEventSourceCounter); \
69 } while (0)
70
71 #define IOStatisticsOpenGate() \
72 do { \
73 IOStatistics::countOpenGate(me->IOEventSource::reserved->counter); \
74 } while (0)
75
76 #define IOStatisticsCloseGate() \
77 do { \
78 IOStatistics::countCloseGate(me->IOEventSource::reserved->counter); \
79 } while (0)
80
81 #define IOStatisticsTimeout() \
82 do { \
83 IOStatistics::countTimerTimeout(me->IOEventSource::reserved->counter); \
84 } while (0)
85
86 #else
87
88 #define IOStatisticsInitializeCounter()
89 #define IOStatisticsOpenGate()
90 #define IOStatisticsCloseGate()
91 #define IOStatisticsTimeout()
92
93 #endif /* IOKITSTATS */
94
95 //
96 // reserved != 0 means IOTimerEventSource::timeoutAndRelease is being used,
97 // not a subclassed implementation.
98 //
99
100 // Timeout handler function. This function is called by the kernel when
101 // the timeout interval expires.
102 //
103
104 __inline__ void
invokeAction(IOEventSource::Action _action,IOTimerEventSource * ts,OSObject * _owner,IOWorkLoop * _workLoop)105 IOTimerEventSource::invokeAction(IOEventSource::Action _action, IOTimerEventSource * ts,
106 OSObject * _owner, IOWorkLoop * _workLoop)
107 {
108 bool trace = (gIOKitTrace & kIOTraceTimers) ? true : false;
109 void * address;
110
111 if (kActionBlock & flags) {
112 address = ptrauth_nop_cast(void *, _Block_get_invoke_fn((struct Block_layout *) actionBlock));
113 } else {
114 address = ptrauth_nop_cast(void *, _action);
115 }
116
117 if (trace) {
118 IOTimeStampStartConstant(IODBG_TIMES(IOTIMES_ACTION),
119 VM_KERNEL_ADDRHIDE(address),
120 VM_KERNEL_ADDRHIDE(_owner));
121 }
122
123 if (kActionBlock & flags) {
124 ((IOTimerEventSource::ActionBlock) actionBlock)(ts);
125 } else {
126 ((IOTimerEventSource::Action)_action)(_owner, ts);
127 }
128
129 #if CONFIG_DTRACE
130 DTRACE_TMR3(iotescallout__expire, Action, address, OSObject, _owner, void, _workLoop);
131 #endif
132
133 if (trace) {
134 IOTimeStampEndConstant(IODBG_TIMES(IOTIMES_ACTION),
135 VM_KERNEL_UNSLIDE(address),
136 VM_KERNEL_ADDRHIDE(_owner));
137 }
138 }
139
140 void
timeout(void * self)141 IOTimerEventSource::timeout(void *self)
142 {
143 IOTimerEventSource *me = (IOTimerEventSource *) self;
144
145 IOStatisticsTimeout();
146
147 if (me->enabled && me->action) {
148 IOWorkLoop *
149 wl = me->workLoop;
150 if (wl) {
151 IOEventSource::Action doit;
152 wl->closeGate();
153 IOStatisticsCloseGate();
154 doit = me->action;
155 if (doit && me->enabled && AbsoluteTime_to_scalar(&me->abstime)) {
156 me->invokeAction(doit, me, me->owner, me->workLoop);
157 }
158 IOStatisticsOpenGate();
159 wl->openGate();
160 }
161 }
162 }
163
164 void
timeoutAndRelease(void * self,void * c)165 IOTimerEventSource::timeoutAndRelease(void * self, void * c)
166 {
167 IOTimerEventSource *me = (IOTimerEventSource *) self;
168 /* The second parameter (a pointer) gets abused to carry an SInt32, so on LP64, "count"
169 * must be cast to "long" before, in order to tell GCC we're not truncating a pointer. */
170 SInt32 count = (SInt32) (long) c;
171
172 IOStatisticsTimeout();
173
174 if (me->enabled && me->action) {
175 IOWorkLoop *
176 wl = me->reserved->workLoop;
177 if (wl) {
178 IOEventSource::Action doit;
179 wl->closeGate();
180 IOStatisticsCloseGate();
181 doit = me->action;
182 if (doit && (me->reserved->calloutGeneration == count)) {
183 thread_call_start_iotes_invocation((thread_call_t)me->calloutEntry);
184 me->invokeAction(doit, me, me->owner, me->workLoop);
185 }
186 IOStatisticsOpenGate();
187 wl->openGate();
188 }
189 }
190 if (me->reserved->workLoop) {
191 me->reserved->workLoop->release();
192 }
193 me->release();
194 }
195
196 // -- work loop delivery
197
198 bool
checkForWork()199 IOTimerEventSource::checkForWork()
200 {
201 IOEventSource::Action doit;
202
203 if (reserved
204 && (reserved->calloutGenerationSignaled == reserved->calloutGeneration)
205 && enabled && (doit = action)) {
206 reserved->calloutGenerationSignaled = ~reserved->calloutGeneration;
207 invokeAction(doit, this, owner, workLoop);
208 }
209
210 return false;
211 }
212
213 void
timeoutSignaled(void * self,void * c)214 IOTimerEventSource::timeoutSignaled(void * self, void * c)
215 {
216 IOTimerEventSource *me = (IOTimerEventSource *) self;
217
218 me->reserved->calloutGenerationSignaled = (SInt32)(long) c;
219 if (me->enabled) {
220 me->signalWorkAvailable();
221 }
222 }
223
224 // --
225
226 void
setTimeoutFunc()227 IOTimerEventSource::setTimeoutFunc()
228 {
229 thread_call_priority_t pri;
230 uint32_t options;
231
232 if (reserved) {
233 panic("setTimeoutFunc already %p, %p", this, reserved);
234 }
235
236 // reserved != 0 means IOTimerEventSource::timeoutAndRelease is being used,
237 // not a subclassed implementation
238 reserved = IOMallocType(ExpansionData);
239
240 reserved->calloutGenerationSignaled = ~reserved->calloutGeneration;
241 // make use of an existing ivar for parameter passing
242 options = (uint32_t) abstime;
243 abstime = 0;
244
245 thread_call_options_t tcoptions = 0;
246 thread_call_func_t func = NULL;
247
248 switch (kIOTimerEventSourceOptionsPriorityMask & options) {
249 case kIOTimerEventSourceOptionsPriorityHigh:
250 pri = THREAD_CALL_PRIORITY_HIGH;
251 func = &IOTimerEventSource::timeoutAndRelease;
252 break;
253
254 case kIOTimerEventSourceOptionsPriorityKernel:
255 pri = THREAD_CALL_PRIORITY_KERNEL;
256 func = &IOTimerEventSource::timeoutAndRelease;
257 break;
258
259 case kIOTimerEventSourceOptionsPriorityKernelHigh:
260 pri = THREAD_CALL_PRIORITY_KERNEL_HIGH;
261 func = &IOTimerEventSource::timeoutAndRelease;
262 break;
263
264 case kIOTimerEventSourceOptionsPriorityUser:
265 pri = THREAD_CALL_PRIORITY_USER;
266 func = &IOTimerEventSource::timeoutAndRelease;
267 break;
268
269 case kIOTimerEventSourceOptionsPriorityLow:
270 pri = THREAD_CALL_PRIORITY_LOW;
271 func = &IOTimerEventSource::timeoutAndRelease;
272 break;
273
274 case kIOTimerEventSourceOptionsPriorityWorkLoop:
275 pri = THREAD_CALL_PRIORITY_KERNEL;
276 tcoptions |= THREAD_CALL_OPTIONS_SIGNAL;
277 if (kIOTimerEventSourceOptionsAllowReenter & options) {
278 break;
279 }
280 func = &IOTimerEventSource::timeoutSignaled;
281 break;
282
283 default:
284 break;
285 }
286
287 assertf(func, "IOTimerEventSource options 0x%x", options);
288 if (!func) {
289 return; // init will fail
290 }
291 if (THREAD_CALL_OPTIONS_SIGNAL & tcoptions) {
292 flags |= kActive;
293 } else {
294 flags |= kPassive;
295 }
296
297 if (!(kIOTimerEventSourceOptionsAllowReenter & options)) {
298 tcoptions |= THREAD_CALL_OPTIONS_ONCE;
299 }
300
301 calloutEntry = (void *) thread_call_allocate_with_options(func,
302 (thread_call_param_t) this, pri, tcoptions);
303 assert(calloutEntry);
304 }
305
306 bool
init(OSObject * inOwner,Action inAction)307 IOTimerEventSource::init(OSObject *inOwner, Action inAction)
308 {
309 if (!super::init(inOwner, (IOEventSource::Action) inAction)) {
310 return false;
311 }
312
313 setTimeoutFunc();
314 if (!calloutEntry) {
315 return false;
316 }
317
318 IOStatisticsInitializeCounter();
319
320 return true;
321 }
322
323 bool
init(uint32_t options,OSObject * inOwner,Action inAction)324 IOTimerEventSource::init(uint32_t options, OSObject *inOwner, Action inAction)
325 {
326 // make use of an existing ivar for parameter passing
327 abstime = options;
328 return init(inOwner, inAction);
329 }
330
331 IOTimerEventSource *
timerEventSource(uint32_t inOptions,OSObject * inOwner,Action inAction)332 IOTimerEventSource::timerEventSource(uint32_t inOptions, OSObject *inOwner, Action inAction)
333 {
334 IOTimerEventSource *me = new IOTimerEventSource;
335
336 if (me && !me->init(inOptions, inOwner, inAction)) {
337 me->release();
338 return NULL;
339 }
340
341 return me;
342 }
343
344 IOTimerEventSource *
timerEventSource(uint32_t options,OSObject * inOwner,ActionBlock _action)345 IOTimerEventSource::timerEventSource(uint32_t options, OSObject *inOwner, ActionBlock _action)
346 {
347 IOTimerEventSource * tes;
348 tes = IOTimerEventSource::timerEventSource(options, inOwner, (Action) NULL);
349 if (tes) {
350 tes->setActionBlock((IOEventSource::ActionBlock) _action);
351 }
352
353 return tes;
354 }
355
356 #define _thread_call_cancel(tc) ((kActive & flags) ? thread_call_cancel_wait((tc)) : thread_call_cancel((tc)))
357
358 IOTimerEventSource *
timerEventSource(OSObject * inOwner,Action inAction)359 IOTimerEventSource::timerEventSource(OSObject *inOwner, Action inAction)
360 {
361 return IOTimerEventSource::timerEventSource(
362 kIOTimerEventSourceOptionsPriorityKernelHigh,
363 inOwner, inAction);
364 }
365
366 void
free()367 IOTimerEventSource::free()
368 {
369 if (calloutEntry) {
370 __assert_only bool freed;
371
372 cancelTimeout();
373
374 freed = thread_call_free((thread_call_t) calloutEntry);
375 assert(freed);
376 }
377
378 if (reserved) {
379 IOFreeType(reserved, ExpansionData);
380 }
381
382 super::free();
383 }
384
385 void
cancelTimeout()386 IOTimerEventSource::cancelTimeout()
387 {
388 if (reserved) {
389 reserved->calloutGeneration++;
390 }
391 bool active = _thread_call_cancel((thread_call_t) calloutEntry);
392 AbsoluteTime_to_scalar(&abstime) = 0;
393 if (active && reserved && (kPassive & flags)) {
394 release();
395 workLoop->release();
396 }
397 }
398
399 void
enable()400 IOTimerEventSource::enable()
401 {
402 super::enable();
403 if (kIOReturnSuccess != wakeAtTime(abstime)) {
404 super::disable(); // Problem re-scheduling timeout ignore enable
405 }
406 }
407
408 void
disable()409 IOTimerEventSource::disable()
410 {
411 if (reserved) {
412 reserved->calloutGeneration++;
413 }
414 bool active = _thread_call_cancel((thread_call_t) calloutEntry);
415 super::disable();
416 if (active && reserved && (kPassive & flags)) {
417 release();
418 workLoop->release();
419 }
420 }
421
422 IOReturn
setTimeoutTicks(UInt32 ticks)423 IOTimerEventSource::setTimeoutTicks(UInt32 ticks)
424 {
425 return setTimeout(ticks, kTickScale);
426 }
427
428 IOReturn
setTimeoutMS(UInt32 ms)429 IOTimerEventSource::setTimeoutMS(UInt32 ms)
430 {
431 return setTimeout(ms, kMillisecondScale);
432 }
433
434 IOReturn
setTimeoutUS(UInt32 us)435 IOTimerEventSource::setTimeoutUS(UInt32 us)
436 {
437 return setTimeout(us, kMicrosecondScale);
438 }
439
440 IOReturn
setTimeout(UInt32 interval,UInt32 scale_factor)441 IOTimerEventSource::setTimeout(UInt32 interval, UInt32 scale_factor)
442 {
443 AbsoluteTime end;
444
445 clock_interval_to_deadline(interval, scale_factor, &end);
446 return wakeAtTime(end);
447 }
448
449 #if !defined(__LP64__)
450 IOReturn
setTimeout(mach_timespec_t interval)451 IOTimerEventSource::setTimeout(mach_timespec_t interval)
452 {
453 AbsoluteTime end, nsecs;
454
455 clock_interval_to_absolutetime_interval
456 (interval.tv_nsec, kNanosecondScale, &nsecs);
457 clock_interval_to_deadline
458 (interval.tv_sec, NSEC_PER_SEC, &end);
459 ADD_ABSOLUTETIME(&end, &nsecs);
460
461 return wakeAtTime(end);
462 }
463 #endif
464
465 IOReturn
setTimeout(AbsoluteTime interval)466 IOTimerEventSource::setTimeout(AbsoluteTime interval)
467 {
468 AbsoluteTime end;
469 clock_absolutetime_interval_to_deadline(interval, &end);
470 return wakeAtTime(end);
471 }
472
473 IOReturn
setTimeout(uint32_t options,AbsoluteTime abstime,AbsoluteTime leeway)474 IOTimerEventSource::setTimeout(uint32_t options,
475 AbsoluteTime abstime, AbsoluteTime leeway)
476 {
477 AbsoluteTime end;
478 if (options & kIOTimeOptionsContinuous) {
479 clock_continuoustime_interval_to_deadline(abstime, &end);
480 } else {
481 clock_absolutetime_interval_to_deadline(abstime, &end);
482 }
483
484 return wakeAtTime(options, end, leeway);
485 }
486
487 IOReturn
wakeAtTimeTicks(UInt32 ticks)488 IOTimerEventSource::wakeAtTimeTicks(UInt32 ticks)
489 {
490 return wakeAtTime(ticks, kTickScale);
491 }
492
493 IOReturn
wakeAtTimeMS(UInt32 ms)494 IOTimerEventSource::wakeAtTimeMS(UInt32 ms)
495 {
496 return wakeAtTime(ms, kMillisecondScale);
497 }
498
499 IOReturn
wakeAtTimeUS(UInt32 us)500 IOTimerEventSource::wakeAtTimeUS(UInt32 us)
501 {
502 return wakeAtTime(us, kMicrosecondScale);
503 }
504
505 IOReturn
wakeAtTime(UInt32 inAbstime,UInt32 scale_factor)506 IOTimerEventSource::wakeAtTime(UInt32 inAbstime, UInt32 scale_factor)
507 {
508 AbsoluteTime end;
509 clock_interval_to_absolutetime_interval(inAbstime, scale_factor, &end);
510
511 return wakeAtTime(end);
512 }
513
514 #if !defined(__LP64__)
515 IOReturn
wakeAtTime(mach_timespec_t inAbstime)516 IOTimerEventSource::wakeAtTime(mach_timespec_t inAbstime)
517 {
518 AbsoluteTime end, nsecs;
519
520 clock_interval_to_absolutetime_interval
521 (inAbstime.tv_nsec, kNanosecondScale, &nsecs);
522 clock_interval_to_absolutetime_interval
523 (inAbstime.tv_sec, kSecondScale, &end);
524 ADD_ABSOLUTETIME(&end, &nsecs);
525
526 return wakeAtTime(end);
527 }
528 #endif
529
530 void
setWorkLoop(IOWorkLoop * inWorkLoop)531 IOTimerEventSource::setWorkLoop(IOWorkLoop *inWorkLoop)
532 {
533 super::setWorkLoop(inWorkLoop);
534 if (enabled && AbsoluteTime_to_scalar(&abstime) && workLoop) {
535 wakeAtTime(abstime);
536 }
537 }
538
539 IOReturn
wakeAtTime(AbsoluteTime inAbstime)540 IOTimerEventSource::wakeAtTime(AbsoluteTime inAbstime)
541 {
542 return wakeAtTime(0, inAbstime, 0);
543 }
544
545 IOReturn
wakeAtTime(uint32_t options,AbsoluteTime inAbstime,AbsoluteTime leeway)546 IOTimerEventSource::wakeAtTime(uint32_t options, AbsoluteTime inAbstime, AbsoluteTime leeway)
547 {
548 if (!action) {
549 return kIOReturnNoResources;
550 }
551
552 abstime = inAbstime;
553 if (enabled && AbsoluteTime_to_scalar(&inAbstime) && AbsoluteTime_to_scalar(&abstime) && workLoop) {
554 uint32_t tcoptions = 0;
555
556 if (kIOTimeOptionsWithLeeway & options) {
557 tcoptions |= THREAD_CALL_DELAY_LEEWAY;
558 }
559 if (kIOTimeOptionsContinuous & options) {
560 tcoptions |= THREAD_CALL_CONTINUOUS;
561 }
562
563 if (reserved) {
564 if (kPassive & flags) {
565 retain();
566 workLoop->retain();
567 }
568 reserved->workLoop = workLoop;
569 reserved->calloutGeneration++;
570 if (thread_call_enter_delayed_with_leeway((thread_call_t) calloutEntry,
571 (void *)(uintptr_t) reserved->calloutGeneration, inAbstime, leeway, tcoptions)
572 && (kPassive & flags)) {
573 release();
574 workLoop->release();
575 }
576 } else {
577 thread_call_enter_delayed_with_leeway((thread_call_t) calloutEntry,
578 NULL, inAbstime, leeway, tcoptions);
579 }
580 }
581
582 return kIOReturnSuccess;
583 }
584