1 /*
2 * Copyright (c) 2007-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * @APPLE_FREE_COPYRIGHT@
33 */
34 /*
35 * File: arm/rtclock.c
36 * Purpose: Routines for handling the machine dependent
37 * real-time clock.
38 */
39
40 #include <mach/mach_types.h>
41
42 #include <kern/clock.h>
43 #include <kern/thread.h>
44 #include <kern/macro_help.h>
45 #include <kern/spl.h>
46 #include <kern/timer_queue.h>
47
48 #include <kern/host_notify.h>
49
50 #include <machine/commpage.h>
51 #include <machine/machine_routines.h>
52 #include <machine/config.h>
53 #include <arm/exception.h>
54 #include <arm/cpu_data_internal.h>
55 #if __arm64__
56 #include <arm64/proc_reg.h>
57 #else
58 #error Unsupported arch
59 #endif
60 #include <arm/rtclock.h>
61
62 #include <IOKit/IOPlatformExpert.h>
63 #include <libkern/OSAtomic.h>
64
65 #include <sys/kdebug.h>
66
67 #define MAX_TIMEBASE_TRIES 10
68
69 int rtclock_init(void);
70
71 static int
72 deadline_to_decrementer(uint64_t deadline,
73 uint64_t now);
74 static void
75 timebase_callback(struct timebase_freq_t * freq);
76
77 #if DEVELOPMENT || DEBUG
78 uint32_t timebase_validation = 0;
79 #endif
80
81 /*
82 * Configure the real-time clock device at boot
83 */
84 void
rtclock_early_init(void)85 rtclock_early_init(void)
86 {
87 PE_register_timebase_callback(timebase_callback);
88 #if DEVELOPMENT || DEBUG
89 uint32_t tmp_mv = 1;
90
91 #if defined(APPLE_ARM64_ARCH_FAMILY)
92 /* Enable MAT validation on A0 hardware by default. */
93 timebase_validation = ml_get_topology_info()->chip_revision == CPU_VERSION_A0;
94 #endif
95
96 if (kern_feature_override(KF_MATV_OVRD)) {
97 timebase_validation = 0;
98 }
99 if (PE_parse_boot_argn("timebase_validation", &tmp_mv, sizeof(tmp_mv))) {
100 timebase_validation = tmp_mv;
101 }
102 #endif
103 }
104
105 static void
timebase_callback(struct timebase_freq_t * freq)106 timebase_callback(struct timebase_freq_t * freq)
107 {
108 unsigned long numer, denom;
109 uint64_t t64_1, t64_2;
110 uint32_t divisor;
111
112 if (freq->timebase_den < 1 || freq->timebase_den > 4 ||
113 freq->timebase_num < freq->timebase_den) {
114 panic("rtclock timebase_callback: invalid constant %ld / %ld",
115 freq->timebase_num, freq->timebase_den);
116 }
117
118 denom = freq->timebase_num;
119 numer = freq->timebase_den * NSEC_PER_SEC;
120 // reduce by the greatest common denominator to minimize overflow
121 if (numer > denom) {
122 t64_1 = numer;
123 t64_2 = denom;
124 } else {
125 t64_1 = denom;
126 t64_2 = numer;
127 }
128 while (t64_2 != 0) {
129 uint64_t temp = t64_2;
130 t64_2 = t64_1 % t64_2;
131 t64_1 = temp;
132 }
133 numer /= t64_1;
134 denom /= t64_1;
135
136 rtclock_timebase_const.numer = (uint32_t)numer;
137 rtclock_timebase_const.denom = (uint32_t)denom;
138 divisor = (uint32_t)(freq->timebase_num / freq->timebase_den);
139
140 rtclock_sec_divisor = divisor;
141 rtclock_usec_divisor = divisor / USEC_PER_SEC;
142 }
143
144 /*
145 * Initialize the system clock device for the current cpu
146 */
147 int
rtclock_init(void)148 rtclock_init(void)
149 {
150 uint64_t abstime;
151 cpu_data_t * cdp;
152
153 clock_timebase_init();
154
155 cdp = getCpuDatap();
156
157 abstime = mach_absolute_time();
158 cdp->rtcPop = EndOfAllTime; /* Init Pop time */
159 timer_resync_deadlines(); /* Start the timers going */
160
161 return 1;
162 }
163
164
165 uint64_t
mach_absolute_time(void)166 mach_absolute_time(void)
167 {
168 #if DEVELOPMENT || DEBUG
169 if (__improbable(timebase_validation)) {
170 #if __ARM_ARCH_8_6__ || HAS_ACNTVCT
171 static _Atomic uint64_t s_last_absolute_time = 1;
172
173 uint64_t old_absolute_time = os_atomic_load(&s_last_absolute_time, relaxed);
174
175 /*
176 * Because this timebase read is nonspeculative, it cannot begin reading
177 * the timebase value until after the load of the old value completes.
178 */
179
180 if (old_absolute_time == 0) {
181 timebase_validation = 0; // we know it's bad, now prevent nested panics
182 panic("old_absolute_time was 0");
183 }
184
185 uint64_t new_absolute_time = ml_get_timebase();
186
187 if (old_absolute_time > new_absolute_time) {
188 timebase_validation = 0; // prevent nested panics
189 panic("mach_absolute_time returning non-monotonically increasing value 0x%llx (old value 0x%llx)",
190 new_absolute_time, old_absolute_time);
191 }
192
193 if (old_absolute_time < new_absolute_time) {
194 /* read again, to pretest the atomic max */
195 uint64_t pretest_absolute_time = os_atomic_load(&s_last_absolute_time, relaxed);
196 if (pretest_absolute_time < new_absolute_time) {
197 uint64_t fresh_last_absolute_time = os_atomic_max_orig(&s_last_absolute_time, new_absolute_time, relaxed);
198
199 if (fresh_last_absolute_time != pretest_absolute_time) {
200 /*
201 * Someone else published a newer time after we loaded s_last_absolute_time.
202 * Enforce that our timebase is not behind this new one.
203 * We can't compare it with our previous timebase read, as it is too old.
204 */
205
206 uint64_t newest_absolute_time = ml_get_timebase();
207
208 if (fresh_last_absolute_time > newest_absolute_time) {
209 timebase_validation = 0; // prevent nested panics
210 panic("mach_absolute_time returning non-monotonically increasing value 0x%llx (old values 0x%llx, 0x%llx, 0x%llx, 0x%llx)\n",
211 newest_absolute_time, fresh_last_absolute_time, pretest_absolute_time, old_absolute_time, new_absolute_time);
212 }
213 }
214 }
215 }
216
217 return new_absolute_time;
218
219 #else /* !(__ARM_ARCH_8_6__ || HAS_ACNTVCT) */
220 static volatile uint64_t s_last_absolute_time = 0;
221 uint64_t new_absolute_time, old_absolute_time;
222 int attempts = 0;
223
224 /* ARM 64: We need a dsb here to ensure that the load of s_last_absolute_time
225 * completes before the timebase read. Were the load to complete after the
226 * timebase read, there would be a window for another CPU to update
227 * s_last_absolute_time and leave us in an inconsistent state. Consider the
228 * following interleaving:
229 *
230 * Let s_last_absolute_time = t0
231 * CPU0: Read timebase at t1
232 * CPU1: Read timebase at t2
233 * CPU1: Update s_last_absolute_time to t2
234 * CPU0: Load completes
235 * CPU0: Update s_last_absolute_time to t1
236 *
237 * This would cause the assertion to fail even though time did not go
238 * backwards. Thus, we use a dsb to guarantee completion of the load before
239 * the timebase read.
240 */
241 do {
242 attempts++;
243 old_absolute_time = s_last_absolute_time;
244
245 __builtin_arm_dsb(DSB_ISHLD);
246
247 new_absolute_time = ml_get_timebase();
248 } while (attempts < MAX_TIMEBASE_TRIES && !OSCompareAndSwap64(old_absolute_time, new_absolute_time, &s_last_absolute_time));
249
250 if (attempts < MAX_TIMEBASE_TRIES && old_absolute_time > new_absolute_time) {
251 timebase_validation = 0; // we know it's bad, now prevent nested panics
252 panic("mach_absolute_time returning non-monotonically increasing value 0x%llx (old value 0x%llx\n)",
253 new_absolute_time, old_absolute_time);
254 }
255 return new_absolute_time;
256 #endif /* __ARM_ARCH_8_6__ || HAS_ACNTVCT */
257 }
258 #endif /* DEVELOPMENT || DEBUG */
259
260 return ml_get_timebase();
261 }
262
263 uint64_t
mach_approximate_time(void)264 mach_approximate_time(void)
265 {
266 #if __ARM_TIME__ || __ARM_TIME_TIMEBASE_ONLY__ || __arm64__
267 /* Hardware supports a fast timestamp, so grab it without asserting monotonicity */
268 return ml_get_timebase();
269 #else
270 processor_t processor;
271 uint64_t approx_time;
272
273 disable_preemption();
274 processor = current_processor();
275 approx_time = processor->last_dispatch;
276 enable_preemption();
277
278 return approx_time;
279 #endif
280 }
281
282 void
clock_get_system_microtime(clock_sec_t * secs,clock_usec_t * microsecs)283 clock_get_system_microtime(clock_sec_t * secs,
284 clock_usec_t * microsecs)
285 {
286 absolutetime_to_microtime(mach_absolute_time(), secs, microsecs);
287 }
288
289 void
clock_get_system_nanotime(clock_sec_t * secs,clock_nsec_t * nanosecs)290 clock_get_system_nanotime(clock_sec_t * secs,
291 clock_nsec_t * nanosecs)
292 {
293 uint64_t abstime;
294 uint64_t t64;
295
296 abstime = mach_absolute_time();
297 *secs = (t64 = abstime / rtclock_sec_divisor);
298 abstime -= (t64 * rtclock_sec_divisor);
299
300 *nanosecs = (clock_nsec_t)((abstime * NSEC_PER_SEC) / rtclock_sec_divisor);
301 }
302
303 void
clock_gettimeofday_set_commpage(uint64_t abstime,uint64_t sec,uint64_t frac,uint64_t scale,uint64_t tick_per_sec)304 clock_gettimeofday_set_commpage(uint64_t abstime,
305 uint64_t sec,
306 uint64_t frac,
307 uint64_t scale,
308 uint64_t tick_per_sec)
309 {
310 commpage_set_timestamp(abstime, sec, frac, scale, tick_per_sec);
311 }
312
313 void
clock_timebase_info(mach_timebase_info_t info)314 clock_timebase_info(mach_timebase_info_t info)
315 {
316 *info = rtclock_timebase_const;
317 }
318
319 /*
320 * Real-time clock device interrupt.
321 */
322 void
rtclock_intr(__unused unsigned int is_user_context)323 rtclock_intr(__unused unsigned int is_user_context)
324 {
325 uint64_t abstime;
326 cpu_data_t * cdp;
327 struct arm_saved_state * regs;
328 unsigned int user_mode;
329 uintptr_t pc;
330
331 cdp = getCpuDatap();
332
333 cdp->cpu_stat.timer_cnt++;
334 SCHED_STATS_INC(timer_pop_count);
335
336 assert(!ml_get_interrupts_enabled());
337
338 abstime = mach_absolute_time();
339
340 if (cdp->cpu_idle_pop != 0x0ULL) {
341 if ((cdp->rtcPop - abstime) < cdp->cpu_idle_latency) {
342 cdp->cpu_idle_pop = 0x0ULL;
343 while (abstime < cdp->rtcPop) {
344 abstime = mach_absolute_time();
345 }
346 } else {
347 ClearIdlePop(FALSE);
348 }
349 }
350
351 if ((regs = cdp->cpu_int_state)) {
352 pc = get_saved_state_pc(regs);
353
354 #if __arm64__
355 user_mode = PSR64_IS_USER(get_saved_state_cpsr(regs));
356 #endif
357 } else {
358 pc = 0;
359 user_mode = 0;
360 }
361 if (abstime >= cdp->rtcPop) {
362 /* Log the interrupt service latency (-ve value expected by tool) */
363 KDBG_RELEASE(DECR_TRAP_LATENCY | DBG_FUNC_NONE,
364 -(abstime - cdp->rtcPop),
365 user_mode ? pc : VM_KERNEL_UNSLIDE(pc), user_mode);
366 }
367
368 /* call the generic etimer */
369 timer_intr(user_mode, pc);
370 }
371
372 static int
deadline_to_decrementer(uint64_t deadline,uint64_t now)373 deadline_to_decrementer(uint64_t deadline,
374 uint64_t now)
375 {
376 uint64_t delt;
377
378 if (deadline <= now) {
379 return DECREMENTER_MIN;
380 } else {
381 delt = deadline - now;
382
383 return (delt >= (DECREMENTER_MAX + 1)) ? DECREMENTER_MAX : ((delt >= (DECREMENTER_MIN + 1)) ? (int)delt : DECREMENTER_MIN);
384 }
385 }
386
387 /*
388 * Request a decrementer pop
389 */
390 int
setPop(uint64_t time)391 setPop(uint64_t time)
392 {
393 int delay_time;
394 uint64_t current_time;
395 cpu_data_t * cdp;
396
397 cdp = getCpuDatap();
398 current_time = mach_absolute_time();
399
400 delay_time = deadline_to_decrementer(time, current_time);
401 cdp->rtcPop = delay_time + current_time;
402
403 ml_set_decrementer((uint32_t) delay_time);
404
405 return delay_time;
406 }
407
408 /*
409 * Request decrementer Idle Pop. Return true if set
410 */
411 boolean_t
SetIdlePop(void)412 SetIdlePop(void)
413 {
414 int delay_time;
415 uint64_t time;
416 uint64_t current_time;
417 cpu_data_t * cdp;
418
419 cdp = getCpuDatap();
420 current_time = mach_absolute_time();
421
422 if (((cdp->rtcPop < current_time) ||
423 (cdp->rtcPop - current_time) < cdp->cpu_idle_latency)) {
424 return FALSE;
425 }
426
427 time = cdp->rtcPop - cdp->cpu_idle_latency;
428
429 delay_time = deadline_to_decrementer(time, current_time);
430 cdp->cpu_idle_pop = delay_time + current_time;
431 ml_set_decrementer((uint32_t) delay_time);
432
433 return TRUE;
434 }
435
436 /*
437 * Clear decrementer Idle Pop
438 */
439 void
ClearIdlePop(boolean_t wfi)440 ClearIdlePop(
441 boolean_t wfi)
442 {
443 cpu_data_t * cdp;
444
445 cdp = getCpuDatap();
446 cdp->cpu_idle_pop = 0x0ULL;
447
448 /*
449 * Don't update the HW timer if there's a pending
450 * interrupt (we can lose interrupt assertion);
451 * we want to take the interrupt right now and update
452 * the deadline from the handler).
453 *
454 * ARM64_TODO: consider this more carefully.
455 */
456 if (!(wfi && ml_get_timer_pending())) {
457 setPop(cdp->rtcPop);
458 }
459 }
460
461 void
absolutetime_to_microtime(uint64_t abstime,clock_sec_t * secs,clock_usec_t * microsecs)462 absolutetime_to_microtime(uint64_t abstime,
463 clock_sec_t * secs,
464 clock_usec_t * microsecs)
465 {
466 uint64_t t64;
467
468 *secs = t64 = abstime / rtclock_sec_divisor;
469 abstime -= (t64 * rtclock_sec_divisor);
470
471 *microsecs = (uint32_t)(abstime / rtclock_usec_divisor);
472 }
473
474 void
absolutetime_to_nanoseconds(uint64_t abstime,uint64_t * result)475 absolutetime_to_nanoseconds(uint64_t abstime,
476 uint64_t * result)
477 {
478 uint64_t t64;
479
480 *result = (t64 = abstime / rtclock_sec_divisor) * NSEC_PER_SEC;
481 abstime -= (t64 * rtclock_sec_divisor);
482 *result += (abstime * NSEC_PER_SEC) / rtclock_sec_divisor;
483 }
484
485 void
nanoseconds_to_absolutetime(uint64_t nanosecs,uint64_t * result)486 nanoseconds_to_absolutetime(uint64_t nanosecs,
487 uint64_t * result)
488 {
489 uint64_t t64;
490
491 *result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor;
492 nanosecs -= (t64 * NSEC_PER_SEC);
493 *result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
494 }
495
496 void
nanotime_to_absolutetime(clock_sec_t secs,clock_nsec_t nanosecs,uint64_t * result)497 nanotime_to_absolutetime(clock_sec_t secs,
498 clock_nsec_t nanosecs,
499 uint64_t * result)
500 {
501 *result = ((uint64_t) secs * rtclock_sec_divisor) +
502 ((uint64_t) nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
503 }
504
505 void
clock_interval_to_absolutetime_interval(uint32_t interval,uint32_t scale_factor,uint64_t * result)506 clock_interval_to_absolutetime_interval(uint32_t interval,
507 uint32_t scale_factor,
508 uint64_t * result)
509 {
510 uint64_t nanosecs = (uint64_t) interval * scale_factor;
511 uint64_t t64;
512
513 *result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor;
514 nanosecs -= (t64 * NSEC_PER_SEC);
515 *result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
516 }
517
518 void
machine_delay_until(uint64_t interval,uint64_t deadline)519 machine_delay_until(uint64_t interval,
520 uint64_t deadline)
521 {
522 #pragma unused(interval)
523 uint64_t now;
524
525 do {
526 __builtin_arm_wfe();
527 now = mach_absolute_time();
528 } while (now < deadline);
529 }
530