1 /*
2  * Copyright (c) 1998-2000, 2009-2010 Apple Inc. All rights reserved.
3  *
4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5  *
6  * This file contains Original Code and/or Modifications of Original Code
7  * as defined in and that are subject to the Apple Public Source License
8  * Version 2.0 (the 'License'). You may not use this file except in
9  * compliance with the License. The rights granted to you under the License
10  * may not be used to create, or enable the creation or redistribution of,
11  * unlawful or unlicensed copies of an Apple operating system, or to
12  * circumvent, violate, or enable the circumvention or violation of, any
13  * terms of an Apple operating system software license agreement.
14  *
15  * Please obtain a copy of the License at
16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
17  *
18  * The Original Code and all software distributed under the License are
19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23  * Please see the License for the specific language governing rights and
24  * limitations under the License.
25  *
26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27  */
28 
29 #include <ptrauth.h>
30 #include <sys/cdefs.h>
31 
32 __BEGIN_DECLS
33 #include <kern/thread_call.h>
34 __END_DECLS
35 
36 #include <IOKit/assert.h>
37 #include <IOKit/system.h>
38 
39 #include <IOKit/IOLib.h>
40 #include <IOKit/IOTimerEventSource.h>
41 #include <IOKit/IOWorkLoop.h>
42 
43 #include <IOKit/IOTimeStamp.h>
44 #include <IOKit/IOKitDebug.h>
45 #if CONFIG_DTRACE
46 #include <mach/sdt.h>
47 #endif
48 
49 #include <libkern/Block.h>
50 #include <libkern/Block_private.h>
51 
52 
53 #define super IOEventSource
54 OSDefineMetaClassAndStructors(IOTimerEventSource, IOEventSource)
55 OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 0);
56 OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 1);
57 OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 2);
58 OSMetaClassDefineReservedUnused(IOTimerEventSource, 3);
59 OSMetaClassDefineReservedUnused(IOTimerEventSource, 4);
60 OSMetaClassDefineReservedUnused(IOTimerEventSource, 5);
61 OSMetaClassDefineReservedUnused(IOTimerEventSource, 6);
62 OSMetaClassDefineReservedUnused(IOTimerEventSource, 7);
63 
64 #if IOKITSTATS
65 
66 #define IOStatisticsInitializeCounter() \
67 do { \
68 	IOStatistics::setCounterType(IOEventSource::reserved->counter, kIOStatisticsTimerEventSourceCounter); \
69 } while (0)
70 
71 #define IOStatisticsOpenGate() \
72 do { \
73 	IOStatistics::countOpenGate(me->IOEventSource::reserved->counter); \
74 } while (0)
75 
76 #define IOStatisticsCloseGate() \
77 do { \
78 	IOStatistics::countCloseGate(me->IOEventSource::reserved->counter); \
79 } while (0)
80 
81 #define IOStatisticsTimeout() \
82 do { \
83 	IOStatistics::countTimerTimeout(me->IOEventSource::reserved->counter); \
84 } while (0)
85 
86 #else
87 
88 #define IOStatisticsInitializeCounter()
89 #define IOStatisticsOpenGate()
90 #define IOStatisticsCloseGate()
91 #define IOStatisticsTimeout()
92 
93 #endif /* IOKITSTATS */
94 
95 //
96 // reserved != 0 means IOTimerEventSource::timeoutAndRelease is being used,
97 // not a subclassed implementation.
98 //
99 
100 // Timeout handler function. This function is called by the kernel when
101 // the timeout interval expires.
102 //
103 
104 __inline__ void
105 IOTimerEventSource::invokeAction(IOEventSource::Action _action, IOTimerEventSource * ts,
106     OSObject * _owner, IOWorkLoop * _workLoop)
107 {
108 	bool    trace = (gIOKitTrace & kIOTraceTimers) ? true : false;
109 	void  * address;
110 
111 	if (kActionBlock & flags) {
112 		address = ptrauth_nop_cast(void *, _Block_get_invoke_fn((struct Block_layout *) actionBlock));
113 	} else {
114 		address = ptrauth_nop_cast(void *, _action);
115 	}
116 
117 	if (trace) {
118 		IOTimeStampStartConstant(IODBG_TIMES(IOTIMES_ACTION),
119 		    VM_KERNEL_ADDRHIDE(address),
120 		    VM_KERNEL_ADDRHIDE(_owner));
121 	}
122 
123 	if (kActionBlock & flags) {
124 		((IOTimerEventSource::ActionBlock) actionBlock)(ts);
125 	} else {
126 		((IOTimerEventSource::Action)_action)(_owner, ts);
127 	}
128 
129 #if CONFIG_DTRACE
130 	DTRACE_TMR3(iotescallout__expire, Action, address, OSObject, _owner, void, _workLoop);
131 #endif
132 
133 	if (trace) {
134 		IOTimeStampEndConstant(IODBG_TIMES(IOTIMES_ACTION),
135 		    VM_KERNEL_UNSLIDE(address),
136 		    VM_KERNEL_ADDRHIDE(_owner));
137 	}
138 }
139 
140 void
141 IOTimerEventSource::timeout(void *self)
142 {
143 	IOTimerEventSource *me = (IOTimerEventSource *) self;
144 
145 	IOStatisticsTimeout();
146 
147 	if (me->enabled && me->action) {
148 		IOWorkLoop *
149 		    wl = me->workLoop;
150 		if (wl) {
151 			IOEventSource::Action doit;
152 			wl->closeGate();
153 			IOStatisticsCloseGate();
154 			doit = me->action;
155 			if (doit && me->enabled && AbsoluteTime_to_scalar(&me->abstime)) {
156 				me->invokeAction(doit, me, me->owner, me->workLoop);
157 			}
158 			IOStatisticsOpenGate();
159 			wl->openGate();
160 		}
161 	}
162 }
163 
164 void
165 IOTimerEventSource::timeoutAndRelease(void * self, void * c)
166 {
167 	IOTimerEventSource *me = (IOTimerEventSource *) self;
168 	/* The second parameter (a pointer) gets abused to carry an SInt32, so on LP64, "count"
169 	 *  must be cast to "long" before, in order to tell GCC we're not truncating a pointer. */
170 	SInt32 count = (SInt32) (long) c;
171 
172 	IOStatisticsTimeout();
173 
174 	if (me->enabled && me->action) {
175 		IOWorkLoop *
176 		    wl = me->reserved->workLoop;
177 		if (wl) {
178 			IOEventSource::Action doit;
179 			wl->closeGate();
180 			IOStatisticsCloseGate();
181 			doit = me->action;
182 			if (doit && (me->reserved->calloutGeneration == count)) {
183 				thread_call_start_iotes_invocation((thread_call_t)me->calloutEntry);
184 				me->invokeAction(doit, me, me->owner, me->workLoop);
185 			}
186 			IOStatisticsOpenGate();
187 			wl->openGate();
188 		}
189 	}
190 	if (me->reserved->workLoop) {
191 		me->reserved->workLoop->release();
192 	}
193 	me->release();
194 }
195 
196 // -- work loop delivery
197 
198 bool
199 IOTimerEventSource::checkForWork()
200 {
201 	IOEventSource::Action doit;
202 
203 	if (reserved
204 	    && (reserved->calloutGenerationSignaled == reserved->calloutGeneration)
205 	    && enabled && (doit = action)) {
206 		reserved->calloutGenerationSignaled = ~reserved->calloutGeneration;
207 		invokeAction(doit, this, owner, workLoop);
208 	}
209 
210 	return false;
211 }
212 
213 void
214 IOTimerEventSource::timeoutSignaled(void * self, void * c)
215 {
216 	IOTimerEventSource *me = (IOTimerEventSource *) self;
217 
218 	me->reserved->calloutGenerationSignaled = (SInt32)(long) c;
219 	if (me->enabled) {
220 		me->signalWorkAvailable();
221 	}
222 }
223 
224 // --
225 
226 void
227 IOTimerEventSource::setTimeoutFunc()
228 {
229 	thread_call_priority_t pri;
230 	uint32_t options;
231 
232 	if (reserved) {
233 		panic("setTimeoutFunc already %p, %p", this, reserved);
234 	}
235 
236 	// reserved != 0 means IOTimerEventSource::timeoutAndRelease is being used,
237 	// not a subclassed implementation
238 	reserved = IOMallocType(ExpansionData);
239 
240 	reserved->calloutGenerationSignaled = ~reserved->calloutGeneration;
241 	// make use of an existing ivar for parameter passing
242 	options = (uint32_t) abstime;
243 	abstime = 0;
244 
245 	thread_call_options_t tcoptions = 0;
246 	thread_call_func_t    func      = NULL;
247 
248 	switch (kIOTimerEventSourceOptionsPriorityMask & options) {
249 	case kIOTimerEventSourceOptionsPriorityHigh:
250 		pri = THREAD_CALL_PRIORITY_HIGH;
251 		func = &IOTimerEventSource::timeoutAndRelease;
252 		break;
253 
254 	case kIOTimerEventSourceOptionsPriorityKernel:
255 		pri = THREAD_CALL_PRIORITY_KERNEL;
256 		func = &IOTimerEventSource::timeoutAndRelease;
257 		break;
258 
259 	case kIOTimerEventSourceOptionsPriorityKernelHigh:
260 		pri = THREAD_CALL_PRIORITY_KERNEL_HIGH;
261 		func = &IOTimerEventSource::timeoutAndRelease;
262 		break;
263 
264 	case kIOTimerEventSourceOptionsPriorityUser:
265 		pri = THREAD_CALL_PRIORITY_USER;
266 		func = &IOTimerEventSource::timeoutAndRelease;
267 		break;
268 
269 	case kIOTimerEventSourceOptionsPriorityLow:
270 		pri = THREAD_CALL_PRIORITY_LOW;
271 		func = &IOTimerEventSource::timeoutAndRelease;
272 		break;
273 
274 	case kIOTimerEventSourceOptionsPriorityWorkLoop:
275 		pri = THREAD_CALL_PRIORITY_KERNEL;
276 		tcoptions |= THREAD_CALL_OPTIONS_SIGNAL;
277 		if (kIOTimerEventSourceOptionsAllowReenter & options) {
278 			break;
279 		}
280 		func = &IOTimerEventSource::timeoutSignaled;
281 		break;
282 
283 	default:
284 		break;
285 	}
286 
287 	assertf(func, "IOTimerEventSource options 0x%x", options);
288 	if (!func) {
289 		return;                                              // init will fail
290 	}
291 	if (THREAD_CALL_OPTIONS_SIGNAL & tcoptions) {
292 		flags |= kActive;
293 	} else {
294 		flags |= kPassive;
295 	}
296 
297 	if (!(kIOTimerEventSourceOptionsAllowReenter & options)) {
298 		tcoptions |= THREAD_CALL_OPTIONS_ONCE;
299 	}
300 
301 	calloutEntry = (void *) thread_call_allocate_with_options(func,
302 	    (thread_call_param_t) this, pri, tcoptions);
303 	assert(calloutEntry);
304 }
305 
306 bool
307 IOTimerEventSource::init(OSObject *inOwner, Action inAction)
308 {
309 	if (!super::init(inOwner, (IOEventSource::Action) inAction)) {
310 		return false;
311 	}
312 
313 	setTimeoutFunc();
314 	if (!calloutEntry) {
315 		return false;
316 	}
317 
318 	IOStatisticsInitializeCounter();
319 
320 	return true;
321 }
322 
323 bool
324 IOTimerEventSource::init(uint32_t options, OSObject *inOwner, Action inAction)
325 {
326 	// make use of an existing ivar for parameter passing
327 	abstime = options;
328 	return init(inOwner, inAction);
329 }
330 
331 IOTimerEventSource *
332 IOTimerEventSource::timerEventSource(uint32_t inOptions, OSObject *inOwner, Action inAction)
333 {
334 	IOTimerEventSource *me = new IOTimerEventSource;
335 
336 	if (me && !me->init(inOptions, inOwner, inAction)) {
337 		me->release();
338 		return NULL;
339 	}
340 
341 	return me;
342 }
343 
344 IOTimerEventSource *
345 IOTimerEventSource::timerEventSource(uint32_t options, OSObject *inOwner, ActionBlock _action)
346 {
347 	IOTimerEventSource * tes;
348 	tes = IOTimerEventSource::timerEventSource(options, inOwner, (Action) NULL);
349 	if (tes) {
350 		tes->setActionBlock((IOEventSource::ActionBlock) _action);
351 	}
352 
353 	return tes;
354 }
355 
356 #define _thread_call_cancel(tc)   ((kActive & flags) ? thread_call_cancel_wait((tc)) : thread_call_cancel((tc)))
357 
358 IOTimerEventSource *
359 IOTimerEventSource::timerEventSource(OSObject *inOwner, Action inAction)
360 {
361 	return IOTimerEventSource::timerEventSource(
362 		kIOTimerEventSourceOptionsPriorityKernelHigh,
363 		inOwner, inAction);
364 }
365 
366 void
367 IOTimerEventSource::free()
368 {
369 	if (calloutEntry) {
370 		__assert_only bool freed;
371 
372 		cancelTimeout();
373 
374 		freed = thread_call_free((thread_call_t) calloutEntry);
375 		assert(freed);
376 	}
377 
378 	if (reserved) {
379 		IOFreeType(reserved, ExpansionData);
380 	}
381 
382 	super::free();
383 }
384 
385 void
386 IOTimerEventSource::cancelTimeout()
387 {
388 	if (reserved) {
389 		reserved->calloutGeneration++;
390 	}
391 	bool active = _thread_call_cancel((thread_call_t) calloutEntry);
392 	AbsoluteTime_to_scalar(&abstime) = 0;
393 	if (active && reserved && (kPassive & flags)) {
394 		release();
395 		workLoop->release();
396 	}
397 }
398 
399 void
400 IOTimerEventSource::enable()
401 {
402 	super::enable();
403 	if (kIOReturnSuccess != wakeAtTime(abstime)) {
404 		super::disable(); // Problem re-scheduling timeout ignore enable
405 	}
406 }
407 
408 void
409 IOTimerEventSource::disable()
410 {
411 	if (reserved) {
412 		reserved->calloutGeneration++;
413 	}
414 	bool active = _thread_call_cancel((thread_call_t) calloutEntry);
415 	super::disable();
416 	if (active && reserved && (kPassive & flags)) {
417 		release();
418 		workLoop->release();
419 	}
420 }
421 
422 IOReturn
423 IOTimerEventSource::setTimeoutTicks(UInt32 ticks)
424 {
425 	return setTimeout(ticks, kTickScale);
426 }
427 
428 IOReturn
429 IOTimerEventSource::setTimeoutMS(UInt32 ms)
430 {
431 	return setTimeout(ms, kMillisecondScale);
432 }
433 
434 IOReturn
435 IOTimerEventSource::setTimeoutUS(UInt32 us)
436 {
437 	return setTimeout(us, kMicrosecondScale);
438 }
439 
440 IOReturn
441 IOTimerEventSource::setTimeout(UInt32 interval, UInt32 scale_factor)
442 {
443 	AbsoluteTime end;
444 
445 	clock_interval_to_deadline(interval, scale_factor, &end);
446 	return wakeAtTime(end);
447 }
448 
449 #if !defined(__LP64__)
450 IOReturn
451 IOTimerEventSource::setTimeout(mach_timespec_t interval)
452 {
453 	AbsoluteTime end, nsecs;
454 
455 	clock_interval_to_absolutetime_interval
456 	(interval.tv_nsec, kNanosecondScale, &nsecs);
457 	clock_interval_to_deadline
458 	(interval.tv_sec, NSEC_PER_SEC, &end);
459 	ADD_ABSOLUTETIME(&end, &nsecs);
460 
461 	return wakeAtTime(end);
462 }
463 #endif
464 
465 IOReturn
466 IOTimerEventSource::setTimeout(AbsoluteTime interval)
467 {
468 	AbsoluteTime end;
469 	clock_absolutetime_interval_to_deadline(interval, &end);
470 	return wakeAtTime(end);
471 }
472 
473 IOReturn
474 IOTimerEventSource::setTimeout(uint32_t options,
475     AbsoluteTime abstime, AbsoluteTime leeway)
476 {
477 	AbsoluteTime end;
478 	if (options & kIOTimeOptionsContinuous) {
479 		clock_continuoustime_interval_to_deadline(abstime, &end);
480 	} else {
481 		clock_absolutetime_interval_to_deadline(abstime, &end);
482 	}
483 
484 	return wakeAtTime(options, end, leeway);
485 }
486 
487 IOReturn
488 IOTimerEventSource::wakeAtTimeTicks(UInt32 ticks)
489 {
490 	return wakeAtTime(ticks, kTickScale);
491 }
492 
493 IOReturn
494 IOTimerEventSource::wakeAtTimeMS(UInt32 ms)
495 {
496 	return wakeAtTime(ms, kMillisecondScale);
497 }
498 
499 IOReturn
500 IOTimerEventSource::wakeAtTimeUS(UInt32 us)
501 {
502 	return wakeAtTime(us, kMicrosecondScale);
503 }
504 
505 IOReturn
506 IOTimerEventSource::wakeAtTime(UInt32 inAbstime, UInt32 scale_factor)
507 {
508 	AbsoluteTime end;
509 	clock_interval_to_absolutetime_interval(inAbstime, scale_factor, &end);
510 
511 	return wakeAtTime(end);
512 }
513 
514 #if !defined(__LP64__)
515 IOReturn
516 IOTimerEventSource::wakeAtTime(mach_timespec_t inAbstime)
517 {
518 	AbsoluteTime end, nsecs;
519 
520 	clock_interval_to_absolutetime_interval
521 	(inAbstime.tv_nsec, kNanosecondScale, &nsecs);
522 	clock_interval_to_absolutetime_interval
523 	(inAbstime.tv_sec, kSecondScale, &end);
524 	ADD_ABSOLUTETIME(&end, &nsecs);
525 
526 	return wakeAtTime(end);
527 }
528 #endif
529 
530 void
531 IOTimerEventSource::setWorkLoop(IOWorkLoop *inWorkLoop)
532 {
533 	super::setWorkLoop(inWorkLoop);
534 	if (enabled && AbsoluteTime_to_scalar(&abstime) && workLoop) {
535 		wakeAtTime(abstime);
536 	}
537 }
538 
539 IOReturn
540 IOTimerEventSource::wakeAtTime(AbsoluteTime inAbstime)
541 {
542 	return wakeAtTime(0, inAbstime, 0);
543 }
544 
545 IOReturn
546 IOTimerEventSource::wakeAtTime(uint32_t options, AbsoluteTime inAbstime, AbsoluteTime leeway)
547 {
548 	if (!action) {
549 		return kIOReturnNoResources;
550 	}
551 
552 	abstime = inAbstime;
553 	if (enabled && AbsoluteTime_to_scalar(&inAbstime) && AbsoluteTime_to_scalar(&abstime) && workLoop) {
554 		uint32_t tcoptions = 0;
555 
556 		if (kIOTimeOptionsWithLeeway & options) {
557 			tcoptions |= THREAD_CALL_DELAY_LEEWAY;
558 		}
559 		if (kIOTimeOptionsContinuous & options) {
560 			tcoptions |= THREAD_CALL_CONTINUOUS;
561 		}
562 
563 		if (reserved) {
564 			if (kPassive & flags) {
565 				retain();
566 				workLoop->retain();
567 			}
568 			reserved->workLoop = workLoop;
569 			reserved->calloutGeneration++;
570 			if (thread_call_enter_delayed_with_leeway((thread_call_t) calloutEntry,
571 			    (void *)(uintptr_t) reserved->calloutGeneration, inAbstime, leeway, tcoptions)
572 			    && (kPassive & flags)) {
573 				release();
574 				workLoop->release();
575 			}
576 		} else {
577 			thread_call_enter_delayed_with_leeway((thread_call_t) calloutEntry,
578 			    NULL, inAbstime, leeway, tcoptions);
579 		}
580 	}
581 
582 	return kIOReturnSuccess;
583 }
584