1 /*
2  * Copyright (c) 1998-2000, 2009-2010 Apple Inc. All rights reserved.
3  *
4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5  *
6  * This file contains Original Code and/or Modifications of Original Code
7  * as defined in and that are subject to the Apple Public Source License
8  * Version 2.0 (the 'License'). You may not use this file except in
9  * compliance with the License. The rights granted to you under the License
10  * may not be used to create, or enable the creation or redistribution of,
11  * unlawful or unlicensed copies of an Apple operating system, or to
12  * circumvent, violate, or enable the circumvention or violation of, any
13  * terms of an Apple operating system software license agreement.
14  *
15  * Please obtain a copy of the License at
16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
17  *
18  * The Original Code and all software distributed under the License are
19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23  * Please see the License for the specific language governing rights and
24  * limitations under the License.
25  *
26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27  */
28 
29 #include <ptrauth.h>
30 #include <sys/cdefs.h>
31 
32 __BEGIN_DECLS
33 #include <kern/thread_call.h>
34 __END_DECLS
35 
36 #include <IOKit/assert.h>
37 #include <IOKit/system.h>
38 
39 #include <IOKit/IOLib.h>
40 #include <IOKit/IOTimerEventSource.h>
41 #include <IOKit/IOWorkLoop.h>
42 
43 #include <IOKit/IOTimeStamp.h>
44 #include <IOKit/IOKitDebug.h>
45 #if CONFIG_DTRACE
46 #include <mach/sdt.h>
47 #endif
48 
49 #include <libkern/Block.h>
50 #include <libkern/Block_private.h>
51 
52 
53 #define super IOEventSource
54 OSDefineMetaClassAndStructors(IOTimerEventSource, IOEventSource)
55 OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 0);
56 OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 1);
57 OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 2);
58 OSMetaClassDefineReservedUnused(IOTimerEventSource, 3);
59 OSMetaClassDefineReservedUnused(IOTimerEventSource, 4);
60 OSMetaClassDefineReservedUnused(IOTimerEventSource, 5);
61 OSMetaClassDefineReservedUnused(IOTimerEventSource, 6);
62 OSMetaClassDefineReservedUnused(IOTimerEventSource, 7);
63 
64 #if IOKITSTATS
65 
66 #define IOStatisticsInitializeCounter() \
67 do { \
68 	IOStatistics::setCounterType(IOEventSource::reserved->counter, kIOStatisticsTimerEventSourceCounter); \
69 } while (0)
70 
71 #define IOStatisticsOpenGate() \
72 do { \
73 	IOStatistics::countOpenGate(me->IOEventSource::reserved->counter); \
74 } while (0)
75 
76 #define IOStatisticsCloseGate() \
77 do { \
78 	IOStatistics::countCloseGate(me->IOEventSource::reserved->counter); \
79 } while (0)
80 
81 #define IOStatisticsTimeout() \
82 do { \
83 	IOStatistics::countTimerTimeout(me->IOEventSource::reserved->counter); \
84 } while (0)
85 
86 #else
87 
88 #define IOStatisticsInitializeCounter()
89 #define IOStatisticsOpenGate()
90 #define IOStatisticsCloseGate()
91 #define IOStatisticsTimeout()
92 
93 #endif /* IOKITSTATS */
94 
95 //
96 // reserved != 0 means IOTimerEventSource::timeoutAndRelease is being used,
97 // not a subclassed implementation.
98 //
99 
100 // Timeout handler function. This function is called by the kernel when
101 // the timeout interval expires.
102 //
103 
104 __inline__ void
105 IOTimerEventSource::invokeAction(IOEventSource::Action _action, IOTimerEventSource * ts,
106     OSObject * _owner, IOWorkLoop * _workLoop)
107 {
108 	bool    trace = (gIOKitTrace & kIOTraceTimers) ? true : false;
109 	void  * address;
110 
111 	if (kActionBlock & flags) {
112 		address = ptrauth_nop_cast(void *, _Block_get_invoke_fn((struct Block_layout *) actionBlock));
113 	} else {
114 		address = ptrauth_nop_cast(void *, _action);
115 	}
116 
117 	if (trace) {
118 		IOTimeStampStartConstant(IODBG_TIMES(IOTIMES_ACTION),
119 		    VM_KERNEL_ADDRHIDE(address),
120 		    VM_KERNEL_ADDRHIDE(_owner));
121 	}
122 
123 	if (kActionBlock & flags) {
124 		((IOTimerEventSource::ActionBlock) actionBlock)(ts);
125 	} else {
126 		((IOTimerEventSource::Action)_action)(_owner, ts);
127 	}
128 
129 #if CONFIG_DTRACE
130 	DTRACE_TMR3(iotescallout__expire, Action, address, OSObject, _owner, void, _workLoop);
131 #endif
132 
133 	if (trace) {
134 		IOTimeStampEndConstant(IODBG_TIMES(IOTIMES_ACTION),
135 		    VM_KERNEL_UNSLIDE(address),
136 		    VM_KERNEL_ADDRHIDE(_owner));
137 	}
138 }
139 
140 void
141 IOTimerEventSource::timeout(void *self)
142 {
143 	IOTimerEventSource *me = (IOTimerEventSource *) self;
144 
145 	IOStatisticsTimeout();
146 
147 	if (me->enabled && me->action) {
148 		IOWorkLoop *
149 		    wl = me->workLoop;
150 		if (wl) {
151 			IOEventSource::Action doit;
152 			wl->closeGate();
153 			IOStatisticsCloseGate();
154 			doit = me->action;
155 			if (doit && me->enabled && AbsoluteTime_to_scalar(&me->abstime)) {
156 				me->invokeAction(doit, me, me->owner, me->workLoop);
157 			}
158 			IOStatisticsOpenGate();
159 			wl->openGate();
160 		}
161 	}
162 }
163 
164 void
165 IOTimerEventSource::timeoutAndRelease(void * self, void * c)
166 {
167 	IOTimerEventSource *me = (IOTimerEventSource *) self;
168 	/* The second parameter (a pointer) gets abused to carry an SInt32, so on LP64, "count"
169 	 *  must be cast to "long" before, in order to tell GCC we're not truncating a pointer. */
170 	SInt32 count = (SInt32) (long) c;
171 
172 	IOStatisticsTimeout();
173 
174 	if (me->enabled && me->action) {
175 		IOWorkLoop *
176 		    wl = me->reserved->workLoop;
177 		if (wl) {
178 			IOEventSource::Action doit;
179 			wl->closeGate();
180 			IOStatisticsCloseGate();
181 			doit = me->action;
182 			if (doit && (me->reserved->calloutGeneration == count)) {
183 				thread_call_start_iotes_invocation((thread_call_t)me->calloutEntry);
184 				me->invokeAction(doit, me, me->owner, me->workLoop);
185 			}
186 			IOStatisticsOpenGate();
187 			wl->openGate();
188 		}
189 	}
190 
191 	me->reserved->workLoop->release();
192 	me->release();
193 }
194 
195 // -- work loop delivery
196 
197 bool
198 IOTimerEventSource::checkForWork()
199 {
200 	IOEventSource::Action doit;
201 
202 	if (reserved
203 	    && (reserved->calloutGenerationSignaled == reserved->calloutGeneration)
204 	    && enabled && (doit = action)) {
205 		reserved->calloutGenerationSignaled = ~reserved->calloutGeneration;
206 		invokeAction(doit, this, owner, workLoop);
207 	}
208 
209 	return false;
210 }
211 
212 void
213 IOTimerEventSource::timeoutSignaled(void * self, void * c)
214 {
215 	IOTimerEventSource *me = (IOTimerEventSource *) self;
216 
217 	me->reserved->calloutGenerationSignaled = (SInt32)(long) c;
218 	if (me->enabled) {
219 		me->signalWorkAvailable();
220 	}
221 }
222 
223 // --
224 
225 void
226 IOTimerEventSource::setTimeoutFunc()
227 {
228 	thread_call_priority_t pri;
229 	uint32_t options;
230 
231 	if (reserved) {
232 		panic("setTimeoutFunc already %p, %p", this, reserved);
233 	}
234 
235 	// reserved != 0 means IOTimerEventSource::timeoutAndRelease is being used,
236 	// not a subclassed implementation
237 	reserved = IONewZero(ExpansionData, 1);
238 
239 	reserved->calloutGenerationSignaled = ~reserved->calloutGeneration;
240 	// make use of an existing ivar for parameter passing
241 	options = (uint32_t) abstime;
242 	abstime = 0;
243 
244 	thread_call_options_t tcoptions = 0;
245 	thread_call_func_t    func      = NULL;
246 
247 	switch (kIOTimerEventSourceOptionsPriorityMask & options) {
248 	case kIOTimerEventSourceOptionsPriorityHigh:
249 		pri = THREAD_CALL_PRIORITY_HIGH;
250 		func = &IOTimerEventSource::timeoutAndRelease;
251 		break;
252 
253 	case kIOTimerEventSourceOptionsPriorityKernel:
254 		pri = THREAD_CALL_PRIORITY_KERNEL;
255 		func = &IOTimerEventSource::timeoutAndRelease;
256 		break;
257 
258 	case kIOTimerEventSourceOptionsPriorityKernelHigh:
259 		pri = THREAD_CALL_PRIORITY_KERNEL_HIGH;
260 		func = &IOTimerEventSource::timeoutAndRelease;
261 		break;
262 
263 	case kIOTimerEventSourceOptionsPriorityUser:
264 		pri = THREAD_CALL_PRIORITY_USER;
265 		func = &IOTimerEventSource::timeoutAndRelease;
266 		break;
267 
268 	case kIOTimerEventSourceOptionsPriorityLow:
269 		pri = THREAD_CALL_PRIORITY_LOW;
270 		func = &IOTimerEventSource::timeoutAndRelease;
271 		break;
272 
273 	case kIOTimerEventSourceOptionsPriorityWorkLoop:
274 		pri = THREAD_CALL_PRIORITY_KERNEL;
275 		tcoptions |= THREAD_CALL_OPTIONS_SIGNAL;
276 		if (kIOTimerEventSourceOptionsAllowReenter & options) {
277 			break;
278 		}
279 		func = &IOTimerEventSource::timeoutSignaled;
280 		break;
281 
282 	default:
283 		break;
284 	}
285 
286 	assertf(func, "IOTimerEventSource options 0x%x", options);
287 	if (!func) {
288 		return;                                              // init will fail
289 	}
290 	if (THREAD_CALL_OPTIONS_SIGNAL & tcoptions) {
291 		flags |= kActive;
292 	} else {
293 		flags |= kPassive;
294 	}
295 
296 	if (!(kIOTimerEventSourceOptionsAllowReenter & options)) {
297 		tcoptions |= THREAD_CALL_OPTIONS_ONCE;
298 	}
299 
300 	calloutEntry = (void *) thread_call_allocate_with_options(func,
301 	    (thread_call_param_t) this, pri, tcoptions);
302 	assert(calloutEntry);
303 }
304 
305 bool
306 IOTimerEventSource::init(OSObject *inOwner, Action inAction)
307 {
308 	if (!super::init(inOwner, (IOEventSource::Action) inAction)) {
309 		return false;
310 	}
311 
312 	setTimeoutFunc();
313 	if (!calloutEntry) {
314 		return false;
315 	}
316 
317 	IOStatisticsInitializeCounter();
318 
319 	return true;
320 }
321 
322 bool
323 IOTimerEventSource::init(uint32_t options, OSObject *inOwner, Action inAction)
324 {
325 	// make use of an existing ivar for parameter passing
326 	abstime = options;
327 	return init(inOwner, inAction);
328 }
329 
330 IOTimerEventSource *
331 IOTimerEventSource::timerEventSource(uint32_t inOptions, OSObject *inOwner, Action inAction)
332 {
333 	IOTimerEventSource *me = new IOTimerEventSource;
334 
335 	if (me && !me->init(inOptions, inOwner, inAction)) {
336 		me->release();
337 		return NULL;
338 	}
339 
340 	return me;
341 }
342 
343 IOTimerEventSource *
344 IOTimerEventSource::timerEventSource(uint32_t options, OSObject *inOwner, ActionBlock _action)
345 {
346 	IOTimerEventSource * tes;
347 	tes = IOTimerEventSource::timerEventSource(options, inOwner, (Action) NULL);
348 	if (tes) {
349 		tes->setActionBlock((IOEventSource::ActionBlock) _action);
350 	}
351 
352 	return tes;
353 }
354 
355 #define _thread_call_cancel(tc)   ((kActive & flags) ? thread_call_cancel_wait((tc)) : thread_call_cancel((tc)))
356 
357 IOTimerEventSource *
358 IOTimerEventSource::timerEventSource(OSObject *inOwner, Action inAction)
359 {
360 	return IOTimerEventSource::timerEventSource(
361 		kIOTimerEventSourceOptionsPriorityKernelHigh,
362 		inOwner, inAction);
363 }
364 
365 void
366 IOTimerEventSource::free()
367 {
368 	if (calloutEntry) {
369 		__assert_only bool freed;
370 
371 		cancelTimeout();
372 
373 		freed = thread_call_free((thread_call_t) calloutEntry);
374 		assert(freed);
375 	}
376 
377 	if (reserved) {
378 		IODelete(reserved, ExpansionData, 1);
379 	}
380 
381 	super::free();
382 }
383 
384 void
385 IOTimerEventSource::cancelTimeout()
386 {
387 	if (reserved) {
388 		reserved->calloutGeneration++;
389 	}
390 	bool active = _thread_call_cancel((thread_call_t) calloutEntry);
391 	AbsoluteTime_to_scalar(&abstime) = 0;
392 	if (active && reserved && (kPassive & flags)) {
393 		release();
394 		workLoop->release();
395 	}
396 }
397 
398 void
399 IOTimerEventSource::enable()
400 {
401 	super::enable();
402 	if (kIOReturnSuccess != wakeAtTime(abstime)) {
403 		super::disable(); // Problem re-scheduling timeout ignore enable
404 	}
405 }
406 
407 void
408 IOTimerEventSource::disable()
409 {
410 	if (reserved) {
411 		reserved->calloutGeneration++;
412 	}
413 	bool active = _thread_call_cancel((thread_call_t) calloutEntry);
414 	super::disable();
415 	if (active && reserved && (kPassive & flags)) {
416 		release();
417 		workLoop->release();
418 	}
419 }
420 
421 IOReturn
422 IOTimerEventSource::setTimeoutTicks(UInt32 ticks)
423 {
424 	return setTimeout(ticks, kTickScale);
425 }
426 
427 IOReturn
428 IOTimerEventSource::setTimeoutMS(UInt32 ms)
429 {
430 	return setTimeout(ms, kMillisecondScale);
431 }
432 
433 IOReturn
434 IOTimerEventSource::setTimeoutUS(UInt32 us)
435 {
436 	return setTimeout(us, kMicrosecondScale);
437 }
438 
439 IOReturn
440 IOTimerEventSource::setTimeout(UInt32 interval, UInt32 scale_factor)
441 {
442 	AbsoluteTime end;
443 
444 	clock_interval_to_deadline(interval, scale_factor, &end);
445 	return wakeAtTime(end);
446 }
447 
448 #if !defined(__LP64__)
449 IOReturn
450 IOTimerEventSource::setTimeout(mach_timespec_t interval)
451 {
452 	AbsoluteTime end, nsecs;
453 
454 	clock_interval_to_absolutetime_interval
455 	(interval.tv_nsec, kNanosecondScale, &nsecs);
456 	clock_interval_to_deadline
457 	(interval.tv_sec, NSEC_PER_SEC, &end);
458 	ADD_ABSOLUTETIME(&end, &nsecs);
459 
460 	return wakeAtTime(end);
461 }
462 #endif
463 
464 IOReturn
465 IOTimerEventSource::setTimeout(AbsoluteTime interval)
466 {
467 	AbsoluteTime end;
468 	clock_absolutetime_interval_to_deadline(interval, &end);
469 	return wakeAtTime(end);
470 }
471 
472 IOReturn
473 IOTimerEventSource::setTimeout(uint32_t options,
474     AbsoluteTime abstime, AbsoluteTime leeway)
475 {
476 	AbsoluteTime end;
477 	if (options & kIOTimeOptionsContinuous) {
478 		clock_continuoustime_interval_to_deadline(abstime, &end);
479 	} else {
480 		clock_absolutetime_interval_to_deadline(abstime, &end);
481 	}
482 
483 	return wakeAtTime(options, end, leeway);
484 }
485 
486 IOReturn
487 IOTimerEventSource::wakeAtTimeTicks(UInt32 ticks)
488 {
489 	return wakeAtTime(ticks, kTickScale);
490 }
491 
492 IOReturn
493 IOTimerEventSource::wakeAtTimeMS(UInt32 ms)
494 {
495 	return wakeAtTime(ms, kMillisecondScale);
496 }
497 
498 IOReturn
499 IOTimerEventSource::wakeAtTimeUS(UInt32 us)
500 {
501 	return wakeAtTime(us, kMicrosecondScale);
502 }
503 
504 IOReturn
505 IOTimerEventSource::wakeAtTime(UInt32 inAbstime, UInt32 scale_factor)
506 {
507 	AbsoluteTime end;
508 	clock_interval_to_absolutetime_interval(inAbstime, scale_factor, &end);
509 
510 	return wakeAtTime(end);
511 }
512 
513 #if !defined(__LP64__)
514 IOReturn
515 IOTimerEventSource::wakeAtTime(mach_timespec_t inAbstime)
516 {
517 	AbsoluteTime end, nsecs;
518 
519 	clock_interval_to_absolutetime_interval
520 	(inAbstime.tv_nsec, kNanosecondScale, &nsecs);
521 	clock_interval_to_absolutetime_interval
522 	(inAbstime.tv_sec, kSecondScale, &end);
523 	ADD_ABSOLUTETIME(&end, &nsecs);
524 
525 	return wakeAtTime(end);
526 }
527 #endif
528 
529 void
530 IOTimerEventSource::setWorkLoop(IOWorkLoop *inWorkLoop)
531 {
532 	super::setWorkLoop(inWorkLoop);
533 	if (enabled && AbsoluteTime_to_scalar(&abstime) && workLoop) {
534 		wakeAtTime(abstime);
535 	}
536 }
537 
538 IOReturn
539 IOTimerEventSource::wakeAtTime(AbsoluteTime inAbstime)
540 {
541 	return wakeAtTime(0, inAbstime, 0);
542 }
543 
544 IOReturn
545 IOTimerEventSource::wakeAtTime(uint32_t options, AbsoluteTime inAbstime, AbsoluteTime leeway)
546 {
547 	if (!action) {
548 		return kIOReturnNoResources;
549 	}
550 
551 	abstime = inAbstime;
552 	if (enabled && AbsoluteTime_to_scalar(&inAbstime) && AbsoluteTime_to_scalar(&abstime) && workLoop) {
553 		uint32_t tcoptions = 0;
554 
555 		if (kIOTimeOptionsWithLeeway & options) {
556 			tcoptions |= THREAD_CALL_DELAY_LEEWAY;
557 		}
558 		if (kIOTimeOptionsContinuous & options) {
559 			tcoptions |= THREAD_CALL_CONTINUOUS;
560 		}
561 
562 		if (reserved) {
563 			if (kPassive & flags) {
564 				retain();
565 				workLoop->retain();
566 			}
567 			reserved->workLoop = workLoop;
568 			reserved->calloutGeneration++;
569 			if (thread_call_enter_delayed_with_leeway((thread_call_t) calloutEntry,
570 			    (void *)(uintptr_t) reserved->calloutGeneration, inAbstime, leeway, tcoptions)
571 			    && (kPassive & flags)) {
572 				release();
573 				workLoop->release();
574 			}
575 		} else {
576 			thread_call_enter_delayed_with_leeway((thread_call_t) calloutEntry,
577 			    NULL, inAbstime, leeway, tcoptions);
578 		}
579 	}
580 
581 	return kIOReturnSuccess;
582 }
583