1 /*
2 * Copyright (c) 2000-2008 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * @APPLE_FREE_COPYRIGHT@
33 */
34 /*
35 * File: timer.c
36 * Purpose: Routines for handling the machine independent timer.
37 */
38
39 #include <mach/mach_types.h>
40
41 #include <kern/timer_queue.h>
42 #include <kern/timer_call.h>
43 #include <kern/clock.h>
44 #include <kern/thread.h>
45 #include <kern/processor.h>
46 #include <kern/macro_help.h>
47 #include <kern/spl.h>
48 #include <kern/timer_queue.h>
49 #include <kern/pms.h>
50
51 #include <machine/commpage.h>
52 #include <machine/machine_routines.h>
53
54 #include <sys/kdebug.h>
55 #include <i386/cpu_data.h>
56 #include <i386/cpu_topology.h>
57 #include <i386/cpu_threads.h>
58
59 uint32_t spurious_timers;
60
61 /*
62 * Event timer interrupt.
63 *
64 * XXX a drawback of this implementation is that events serviced earlier must not set deadlines
65 * that occur before the entire chain completes.
66 *
67 * XXX a better implementation would use a set of generic callouts and iterate over them
68 */
69 void
timer_intr(int user_mode,uint64_t rip)70 timer_intr(int user_mode, uint64_t rip)
71 {
72 uint64_t orig_abstime, abstime;
73 rtclock_timer_t *mytimer;
74 cpu_data_t *pp;
75 uint64_t pmdeadline;
76 uint64_t min_deadline = EndOfAllTime;
77 uint64_t run_deadline = EndOfAllTime;
78 bool timer_processed = false;
79
80 pp = current_cpu_datap();
81
82 SCHED_STATS_INC(timer_pop_count);
83
84 orig_abstime = abstime = mach_absolute_time();
85
86 /*
87 * Has a pending clock timer expired?
88 */
89 mytimer = &pp->rtclock_timer;
90 timer_processed = (mytimer->deadline <= abstime ||
91 abstime >= mytimer->queue.earliest_soft_deadline);
92 if (timer_processed) {
93 uint64_t rtclock_deadline = MAX(mytimer->deadline, mytimer->when_set);
94 /*
95 * When opportunistically processing coalesced timers, don't factor
96 * their latency into the trace event.
97 */
98 if (abstime > rtclock_deadline) {
99 TCOAL_DEBUG(0xEEEE0000, abstime,
100 mytimer->queue.earliest_soft_deadline,
101 abstime - mytimer->queue.earliest_soft_deadline, 0, 0);
102 } else {
103 min_deadline = rtclock_deadline;
104 }
105
106 mytimer->has_expired = TRUE;
107 mytimer->deadline = timer_queue_expire(&mytimer->queue, abstime);
108 mytimer->has_expired = FALSE;
109
110 /*
111 * Get a more up-to-date current time after expiring the timer queue.
112 */
113 abstime = mach_absolute_time();
114 mytimer->when_set = abstime;
115 }
116
117 /*
118 * Has a per-CPU running timer expired?
119 */
120 run_deadline = running_timers_expire(pp->cpu_processor, abstime);
121 if (run_deadline != EndOfAllTime) {
122 if (run_deadline < min_deadline) {
123 min_deadline = run_deadline;
124 }
125 timer_processed = true;
126 abstime = mach_absolute_time();
127 }
128
129 /*
130 * Log the timer latency *before* the power management events.
131 */
132 if (__probable(timer_processed)) {
133 /*
134 * Log the maximum interrupt service latency experienced by a timer.
135 */
136 int64_t latency = min_deadline == EndOfAllTime ? 0 :
137 (int64_t)(abstime - min_deadline);
138 /*
139 * Log interrupt service latency (-ve value expected by tool)
140 * a non-PM event is expected next.
141 * The requested deadline may be earlier than when it was set
142 * - use MAX to avoid reporting bogus latencies.
143 */
144 KDBG_RELEASE(DECR_TRAP_LATENCY, -latency,
145 user_mode != 0 ? rip : VM_KERNEL_UNSLIDE(rip), user_mode);
146 }
147
148 /*
149 * Is it time for power management state change?
150 */
151 if ((pmdeadline = pmCPUGetDeadline(pp)) && (pmdeadline <= abstime)) {
152 KDBG_RELEASE(DECR_PM_DEADLINE | DBG_FUNC_START);
153 pmCPUDeadline(pp);
154 KDBG_RELEASE(DECR_PM_DEADLINE | DBG_FUNC_END);
155 timer_processed = true;
156 /*
157 * XXX Nothing below needs an updated abstime, so omit the update.
158 */
159 }
160
161 /*
162 * Schedule the next deadline.
163 */
164 x86_lcpu()->rtcDeadline = EndOfAllTime;
165 timer_resync_deadlines();
166
167 if (__improbable(!timer_processed)) {
168 spurious_timers++;
169 }
170 }
171
172 /*
173 * Set the clock deadline.
174 */
175 void
timer_set_deadline(uint64_t deadline)176 timer_set_deadline(uint64_t deadline)
177 {
178 rtclock_timer_t *mytimer;
179 spl_t s;
180 cpu_data_t *pp;
181
182 s = splclock(); /* no interruptions */
183 pp = current_cpu_datap();
184
185 mytimer = &pp->rtclock_timer; /* Point to the timer itself */
186 mytimer->deadline = deadline; /* Set new expiration time */
187 mytimer->when_set = mach_absolute_time();
188
189 timer_resync_deadlines();
190
191 splx(s);
192 }
193
194 /*
195 * Re-evaluate the outstanding deadlines and select the most proximate.
196 *
197 * Should be called at splclock.
198 */
199 void
timer_resync_deadlines(void)200 timer_resync_deadlines(void)
201 {
202 uint64_t deadline = EndOfAllTime;
203 uint64_t pmdeadline;
204 rtclock_timer_t *mytimer;
205 spl_t s = splclock();
206 cpu_data_t *pp;
207 uint32_t decr;
208
209 pp = current_cpu_datap();
210 if (!pp->cpu_running) {
211 /* There's really nothing to do if this processor is down */
212 return;
213 }
214
215 /*
216 * If we have a clock timer set, pick that.
217 */
218 mytimer = &pp->rtclock_timer;
219 if (!mytimer->has_expired &&
220 0 < mytimer->deadline && mytimer->deadline < EndOfAllTime) {
221 deadline = mytimer->deadline;
222 }
223
224 /*
225 * If we have a power management deadline, see if that's earlier.
226 */
227 pmdeadline = pmCPUGetDeadline(pp);
228 if (0 < pmdeadline && pmdeadline < deadline) {
229 deadline = pmdeadline;
230 }
231
232 uint64_t run_deadline = running_timers_deadline(pp->cpu_processor);
233 if (run_deadline < deadline) {
234 deadline = run_deadline;
235 }
236
237 /*
238 * Go and set the "pop" event.
239 */
240 decr = (uint32_t) setPop(deadline);
241
242 /* Record non-PM deadline for latency tool */
243 if (decr != 0 && deadline != pmdeadline) {
244 uint64_t queue_count = 0;
245 if (deadline != run_deadline) {
246 /*
247 * For non-quantum timer put the queue count
248 * in the tracepoint.
249 */
250 queue_count = mytimer->queue.count;
251 }
252 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
253 DECR_SET_DEADLINE | DBG_FUNC_NONE,
254 decr, 2,
255 deadline,
256 queue_count, 0);
257 }
258 splx(s);
259 }
260
261 void
timer_queue_expire_local(__unused void * arg)262 timer_queue_expire_local(
263 __unused void *arg)
264 {
265 rtclock_timer_t *mytimer;
266 uint64_t abstime;
267 cpu_data_t *pp;
268
269 pp = current_cpu_datap();
270
271 mytimer = &pp->rtclock_timer;
272
273 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
274 DECR_TIMER_EXPIRE_LOCAL | DBG_FUNC_START,
275 mytimer->deadline, 0, 0, 0, 0);
276
277 abstime = mach_absolute_time();
278
279 mytimer->has_expired = TRUE;
280 mytimer->deadline = timer_queue_expire(&mytimer->queue, abstime);
281 mytimer->has_expired = FALSE;
282 mytimer->when_set = mach_absolute_time();
283
284 timer_resync_deadlines();
285
286 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
287 DECR_TIMER_EXPIRE_LOCAL | DBG_FUNC_END,
288 mytimer->deadline, 0, 0, 0, 0);
289 }
290
291 void
timer_queue_expire_rescan(__unused void * arg)292 timer_queue_expire_rescan(
293 __unused void *arg)
294 {
295 rtclock_timer_t *mytimer;
296 uint64_t abstime;
297 cpu_data_t *pp;
298
299 assert(ml_get_interrupts_enabled() == FALSE);
300 pp = current_cpu_datap();
301
302 mytimer = &pp->rtclock_timer;
303 abstime = mach_absolute_time();
304
305 mytimer->has_expired = TRUE;
306 mytimer->deadline = timer_queue_expire_with_options(&mytimer->queue, abstime, TRUE);
307 mytimer->has_expired = FALSE;
308 mytimer->when_set = mach_absolute_time();
309
310 timer_resync_deadlines();
311 }
312
313 #define TIMER_RESORT_THRESHOLD_ABSTIME (50 * NSEC_PER_MSEC)
314
315 #if TCOAL_PRIO_STATS
316 int32_t nc_tcl, rt_tcl, bg_tcl, kt_tcl, fp_tcl, ts_tcl, qos_tcl;
317 #define TCOAL_PRIO_STAT(x) (x++)
318 #else
319 #define TCOAL_PRIO_STAT(x)
320 #endif
321
322 boolean_t
timer_resort_threshold(uint64_t skew)323 timer_resort_threshold(uint64_t skew)
324 {
325 if (skew >= TIMER_RESORT_THRESHOLD_ABSTIME) {
326 return TRUE;
327 } else {
328 return FALSE;
329 }
330 }
331
332 /*
333 * Return the local timer queue for a running processor
334 * else return the boot processor's timer queue.
335 */
336 mpqueue_head_t *
timer_queue_assign(uint64_t deadline)337 timer_queue_assign(
338 uint64_t deadline)
339 {
340 cpu_data_t *cdp = current_cpu_datap();
341 mpqueue_head_t *queue;
342
343 if (cdp->cpu_running) {
344 queue = &cdp->rtclock_timer.queue;
345
346 if (deadline < cdp->rtclock_timer.deadline) {
347 timer_set_deadline(deadline);
348 }
349 } else {
350 queue = &cpu_datap(master_cpu)->rtclock_timer.queue;
351 }
352
353 return queue;
354 }
355
356 void
timer_queue_cancel(mpqueue_head_t * queue,uint64_t deadline,uint64_t new_deadline)357 timer_queue_cancel(
358 mpqueue_head_t *queue,
359 uint64_t deadline,
360 uint64_t new_deadline)
361 {
362 if (queue == ¤t_cpu_datap()->rtclock_timer.queue) {
363 if (deadline < new_deadline) {
364 timer_set_deadline(new_deadline);
365 }
366 }
367 }
368
369 /*
370 * timer_queue_migrate_cpu() is called from the Power-Management kext
371 * when a logical processor goes idle (in a deep C-state) with a distant
372 * deadline so that it's timer queue can be moved to another processor.
373 * This target processor should be the least idle (most busy) --
374 * currently this is the primary processor for the calling thread's package.
375 * Locking restrictions demand that the target cpu must be the boot cpu.
376 */
377 uint32_t
timer_queue_migrate_cpu(int target_cpu)378 timer_queue_migrate_cpu(int target_cpu)
379 {
380 cpu_data_t *target_cdp = cpu_datap(target_cpu);
381 cpu_data_t *cdp = current_cpu_datap();
382 int ntimers_moved;
383
384 assert(!ml_get_interrupts_enabled());
385 assert(target_cpu != cdp->cpu_number);
386 assert(target_cpu == master_cpu);
387
388 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
389 DECR_TIMER_MIGRATE | DBG_FUNC_START,
390 target_cpu,
391 cdp->rtclock_timer.deadline, (cdp->rtclock_timer.deadline >> 32),
392 0, 0);
393
394 /*
395 * Move timer requests from the local queue to the target processor's.
396 * The return value is the number of requests moved. If this is 0,
397 * it indicates that the first (i.e. earliest) timer is earlier than
398 * the earliest for the target processor. Since this would force a
399 * resync, the move of this and all later requests is aborted.
400 */
401 ntimers_moved = timer_queue_migrate(&cdp->rtclock_timer.queue,
402 &target_cdp->rtclock_timer.queue);
403
404 /*
405 * Assuming we moved stuff, clear local deadline.
406 */
407 if (ntimers_moved > 0) {
408 cdp->rtclock_timer.deadline = EndOfAllTime;
409 setPop(EndOfAllTime);
410 }
411
412 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
413 DECR_TIMER_MIGRATE | DBG_FUNC_END,
414 target_cpu, ntimers_moved, 0, 0, 0);
415
416 return ntimers_moved;
417 }
418
419 mpqueue_head_t *
timer_queue_cpu(int cpu)420 timer_queue_cpu(int cpu)
421 {
422 return &cpu_datap(cpu)->rtclock_timer.queue;
423 }
424
425 void
timer_call_cpu(int cpu,void (* fn)(void *),void * arg)426 timer_call_cpu(int cpu, void (*fn)(void *), void *arg)
427 {
428 mp_cpus_call(cpu_to_cpumask(cpu), SYNC, fn, arg);
429 }
430
431 void
timer_call_nosync_cpu(int cpu,void (* fn)(void *),void * arg)432 timer_call_nosync_cpu(int cpu, void (*fn)(void *), void *arg)
433 {
434 /* XXX Needs error checking and retry */
435 mp_cpus_call(cpu_to_cpumask(cpu), NOSYNC, fn, arg);
436 }
437
438
439 static timer_coalescing_priority_params_ns_t tcoal_prio_params_init =
440 {
441 .idle_entry_timer_processing_hdeadline_threshold_ns = 5000ULL * NSEC_PER_USEC,
442 .interrupt_timer_coalescing_ilat_threshold_ns = 30ULL * NSEC_PER_USEC,
443 .timer_resort_threshold_ns = 50 * NSEC_PER_MSEC,
444 .timer_coalesce_rt_shift = 0,
445 .timer_coalesce_bg_shift = -5,
446 .timer_coalesce_kt_shift = 3,
447 .timer_coalesce_fp_shift = 3,
448 .timer_coalesce_ts_shift = 3,
449 .timer_coalesce_rt_ns_max = 0ULL,
450 .timer_coalesce_bg_ns_max = 100 * NSEC_PER_MSEC,
451 .timer_coalesce_kt_ns_max = 1 * NSEC_PER_MSEC,
452 .timer_coalesce_fp_ns_max = 1 * NSEC_PER_MSEC,
453 .timer_coalesce_ts_ns_max = 1 * NSEC_PER_MSEC,
454 .latency_qos_scale = {3, 2, 1, -2, -15, -15},
455 .latency_qos_ns_max = {1 * NSEC_PER_MSEC, 5 * NSEC_PER_MSEC, 20 * NSEC_PER_MSEC,
456 75 * NSEC_PER_MSEC, 10000 * NSEC_PER_MSEC, 10000 * NSEC_PER_MSEC},
457 .latency_tier_rate_limited = {FALSE, FALSE, FALSE, FALSE, TRUE, TRUE},
458 };
459
460 timer_coalescing_priority_params_ns_t *
timer_call_get_priority_params(void)461 timer_call_get_priority_params(void)
462 {
463 return &tcoal_prio_params_init;
464 }
465