1 /*
2 * Copyright (c) 2007-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * File: arm64/cpu.c
30 *
31 * cpu specific routines
32 */
33
34 #include <pexpert/arm64/board_config.h>
35 #include <kern/kalloc.h>
36 #include <kern/machine.h>
37 #include <kern/cpu_number.h>
38 #include <kern/percpu.h>
39 #include <kern/thread.h>
40 #include <kern/timer_queue.h>
41 #include <arm/cpu_data.h>
42 #include <arm/cpuid.h>
43 #include <arm/caches_internal.h>
44 #include <arm/cpu_data_internal.h>
45 #include <arm/cpu_internal.h>
46 #include <arm/misc_protos.h>
47 #include <arm/machine_cpu.h>
48 #include <arm/rtclock.h>
49 #include <arm64/proc_reg.h>
50 #include <mach/processor_info.h>
51 #include <vm/pmap.h>
52 #include <vm/vm_kern_xnu.h>
53 #include <vm/vm_map.h>
54 #include <pexpert/arm/protos.h>
55 #include <pexpert/device_tree.h>
56 #include <sys/kdebug.h>
57 #include <arm/machine_routines.h>
58
59 #include <machine/atomic.h>
60
61 #include <san/kasan.h>
62
63 #include <kern/kpc.h>
64 #if CONFIG_CPU_COUNTERS
65 #include <kern/monotonic.h>
66 #endif /* CONFIG_CPU_COUNTERS */
67
68 #if KPERF
69 #include <kperf/kptimer.h>
70 #endif /* KPERF */
71
72 #if HIBERNATION
73 #include <IOKit/IOPlatformExpert.h>
74 #include <IOKit/IOHibernatePrivate.h>
75 #endif /* HIBERNATION */
76
77
78 #include <libkern/section_keywords.h>
79
80 extern boolean_t idle_enable;
81 extern uint64_t wake_abstime;
82
83 #if WITH_CLASSIC_S2R
84 void sleep_token_buffer_init(void);
85 #endif
86
87 #if !CONFIG_SPTM
88 extern uintptr_t resume_idle_cpu;
89 extern uintptr_t start_cpu;
90 vm_address_t start_cpu_paddr;
91 #endif
92
93 #if __ARM_KERNEL_PROTECT__
94 extern void exc_vectors_table;
95 #endif /* __ARM_KERNEL_PROTECT__ */
96
97 #if APPLEVIRTUALPLATFORM
98 extern vm_offset_t reset_vector_vaddr;
99 #endif /* APPLEVIRTUALPLATFORM */
100
101 #if APPLEVIRTUALPLATFORM
102 extern void __attribute__((noreturn)) arm64_prepare_for_sleep(boolean_t deep_sleep, unsigned int cpu, uint64_t entry_pa);
103 #else
104 extern void __attribute__((noreturn)) arm64_prepare_for_sleep(boolean_t deep_sleep);
105 #endif
106 extern void arm64_force_wfi_clock_gate(void);
107 #if defined(APPLETYPHOON)
108 // <rdar://problem/15827409>
109 extern void typhoon_prepare_for_wfi(void);
110 extern void typhoon_return_from_wfi(void);
111 #endif
112
113 #if HAS_RETENTION_STATE
114 extern void arm64_retention_wfi(void);
115 #endif
116
117 sysreg_restore_t sysreg_restore __attribute__((section("__DATA, __const"))) = {
118 .tcr_el1 = TCR_EL1_BOOT,
119 };
120
121 // wfi - wfi mode
122 // 0 : disabled
123 // 1 : normal
124 // 2 : overhead simulation (delay & flags)
125 TUNABLE(unsigned int, wfi, "wfi", 1);
126 #if DEVELOPMENT || DEBUG
127
128 // wfi_flags
129 // 1 << 0 : flush L1s
130 // 1 << 1 : flush TLBs
131 static int wfi_flags = 0;
132
133 // wfi_delay - delay ticks after wfi exit
134 static uint64_t wfi_delay = 0;
135
136 #endif /* DEVELOPMENT || DEBUG */
137
138 #define CPUPM_IDLE_WFE 0x5310300
139 #define CPUPM_IDLE_TIMER_WFE 0x5310304
140
141 #define DEFAULT_EXPECTING_IPI_WFE_TIMEOUT_USEC (60ULL)
142 TUNABLE(uint32_t, expecting_ipi_wfe_timeout_usec,
143 "expecting_ipi_wfe_timeout_usec", DEFAULT_EXPECTING_IPI_WFE_TIMEOUT_USEC);
144 uint64_t expecting_ipi_wfe_timeout_mt = 0x0ULL; /* initialized to a non-zero value in sched_init */
145
146 /* When recommended, issue WFE with [FI]IRQ unmasked in the idle
147 * loop. The default.
148 */
149 uint32_t idle_proximate_io_wfe_unmasked = 1;
150 #if DEVELOPMENT || DEBUG
151 uint32_t idle_proximate_timer_wfe = 1;
152 uint32_t idle_proximate_io_wfe_masked = 0;
153 #else
154 /* Issue WFE in lieu of WFI when awaiting a proximate timer. */
155 static uint32_t idle_proximate_timer_wfe = 1;
156 /* When recommended, issue WFE with [FI]IRQ masked in the idle loop.
157 * Non-default, retained for experimentation.
158 */
159 static uint32_t idle_proximate_io_wfe_masked = 0;
160 #endif
161
162 #if __ARM_GLOBAL_SLEEP_BIT__
163 volatile boolean_t arm64_stall_sleep = TRUE;
164 #endif
165
166 #if WITH_CLASSIC_S2R
167 /*
168 * These must be aligned to avoid issues with calling bcopy_phys on them before
169 * we are done with pmap initialization.
170 */
171 static const uint8_t __attribute__ ((aligned(8))) suspend_signature[] = {'X', 'S', 'O', 'M', 'P', 'S', 'U', 'S'};
172 static const uint8_t __attribute__ ((aligned(8))) running_signature[] = {'X', 'S', 'O', 'M', 'N', 'N', 'U', 'R'};
173 #endif
174
175 #if WITH_CLASSIC_S2R
176 static vm_offset_t sleepTokenBuffer = (vm_offset_t)NULL;
177 #endif
178 static boolean_t coresight_debug_enabled = FALSE;
179
180 #if defined(CONFIG_XNUPOST)
181 void arm64_ipi_test_callback(void *);
182 void arm64_immediate_ipi_test_callback(void *);
183
184 void
arm64_ipi_test_callback(void * parm)185 arm64_ipi_test_callback(void *parm)
186 {
187 volatile uint64_t *ipi_test_data = parm;
188 cpu_data_t *cpu_data;
189
190 cpu_data = getCpuDatap();
191
192 *ipi_test_data = cpu_data->cpu_number;
193 }
194
195 void
arm64_immediate_ipi_test_callback(void * parm)196 arm64_immediate_ipi_test_callback(void *parm)
197 {
198 volatile uint64_t *ipi_test_data = parm;
199 cpu_data_t *cpu_data;
200
201 cpu_data = getCpuDatap();
202
203 *ipi_test_data = cpu_data->cpu_number + MAX_CPUS;
204 }
205
206 uint64_t arm64_ipi_test_data[MAX_CPUS * 2];
207
208 MACHINE_TIMEOUT(arm64_ipi_test_timeout, "arm64-ipi-test", 100, MACHINE_TIMEOUT_UNIT_MSEC, NULL);
209
210 void
arm64_ipi_test()211 arm64_ipi_test()
212 {
213 volatile uint64_t *ipi_test_data, *immediate_ipi_test_data;
214 uint64_t timeout_ms = os_atomic_load(&arm64_ipi_test_timeout, relaxed);
215 uint64_t then, now, delta;
216 int current_cpu_number = getCpuDatap()->cpu_number;
217
218 /*
219 * probably the only way to have this on most systems is with the
220 * cpus=1 boot-arg, but nonetheless, if we only have 1 CPU active,
221 * IPI is not available
222 */
223 if (real_ncpus == 1) {
224 return;
225 }
226
227 const unsigned int max_cpu_id = ml_get_max_cpu_number();
228 for (unsigned int i = 0; i <= max_cpu_id; ++i) {
229 ipi_test_data = &arm64_ipi_test_data[i];
230 immediate_ipi_test_data = &arm64_ipi_test_data[i + MAX_CPUS];
231 *ipi_test_data = ~i;
232 kern_return_t error = cpu_xcall((int)i, (void *)arm64_ipi_test_callback, (void *)(uintptr_t)ipi_test_data);
233 if (error != KERN_SUCCESS) {
234 panic("CPU %d was unable to IPI CPU %u: error %d", current_cpu_number, i, error);
235 }
236
237 while ((error = cpu_immediate_xcall((int)i, (void *)arm64_immediate_ipi_test_callback,
238 (void *)(uintptr_t)immediate_ipi_test_data)) == KERN_ALREADY_WAITING) {
239 now = mach_absolute_time();
240 absolutetime_to_nanoseconds(now - then, &delta);
241 if ((delta / NSEC_PER_MSEC) > timeout_ms) {
242 panic("CPU %d was unable to immediate-IPI CPU %u within %lldms", current_cpu_number, i, timeout_ms);
243 }
244 }
245
246 if (error != KERN_SUCCESS) {
247 panic("CPU %d was unable to immediate-IPI CPU %u: error %d", current_cpu_number, i, error);
248 }
249
250 then = mach_absolute_time();
251
252 while ((*ipi_test_data != i) || (*immediate_ipi_test_data != (i + MAX_CPUS))) {
253 now = mach_absolute_time();
254 absolutetime_to_nanoseconds(now - then, &delta);
255 if ((delta / NSEC_PER_MSEC) > timeout_ms) {
256 panic("CPU %d tried to IPI CPU %d but didn't get correct responses within %lldms, responses: %llx, %llx",
257 current_cpu_number, i, timeout_ms, *ipi_test_data, *immediate_ipi_test_data);
258 }
259 }
260 }
261 }
262 #endif /* defined(CONFIG_XNUPOST) */
263
264 static void
configure_coresight_registers(cpu_data_t * cdp)265 configure_coresight_registers(cpu_data_t *cdp)
266 {
267 int i;
268
269 assert(cdp);
270 vm_offset_t coresight_regs = ml_get_topology_info()->cpus[cdp->cpu_number].coresight_regs;
271
272 /*
273 * ARMv8 coresight registers are optional. If the device tree did not
274 * provide either cpu_regmap_paddr (from the legacy "reg-private" EDT property)
275 * or coresight_regs (from the new "coresight-reg" property), assume that
276 * coresight registers are not supported.
277 */
278 if (cdp->cpu_regmap_paddr || coresight_regs) {
279 for (i = 0; i < CORESIGHT_REGIONS; ++i) {
280 if (i == CORESIGHT_CTI || i == CORESIGHT_PMU) {
281 continue;
282 }
283 /* Skip debug-only registers on production chips */
284 if (((i == CORESIGHT_ED) || (i == CORESIGHT_UTT)) && !coresight_debug_enabled) {
285 continue;
286 }
287
288 if (!cdp->coresight_base[i]) {
289 if (coresight_regs) {
290 cdp->coresight_base[i] = coresight_regs + CORESIGHT_OFFSET(i);
291 } else {
292 uint64_t addr = cdp->cpu_regmap_paddr + CORESIGHT_OFFSET(i);
293 cdp->coresight_base[i] = (vm_offset_t)ml_io_map(addr, CORESIGHT_SIZE);
294 }
295 }
296 /* Unlock EDLAR (CTI and PMU are skipped above). */
297 if (i != CORESIGHT_UTT) {
298 *(volatile uint32_t *)(cdp->coresight_base[i] + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY;
299 }
300 }
301 }
302 }
303
304
305 /*
306 * Routine: cpu_bootstrap
307 * Function:
308 */
309 void
cpu_bootstrap(void)310 cpu_bootstrap(void)
311 {
312 }
313
314 /*
315 * Routine: cpu_sleep
316 * Function:
317 */
318 void
cpu_sleep(void)319 cpu_sleep(void)
320 {
321 cpu_data_t *cpu_data_ptr = getCpuDatap();
322
323 cpu_data_ptr->cpu_active_thread = current_thread();
324 #if CONFIG_SPTM
325 cpu_data_ptr->cpu_reset_handler = (uintptr_t) VM_KERNEL_STRIP_PTR(arm_init_cpu);
326 #else
327 cpu_data_ptr->cpu_reset_handler = (uintptr_t) start_cpu_paddr;
328 #endif
329 os_atomic_or(&cpu_data_ptr->cpu_flags, SleepState, relaxed);
330
331 if (cpu_data_ptr->cpu_user_debug != NULL) {
332 arm_debug_set(NULL);
333 }
334
335 #if CONFIG_CPU_COUNTERS
336 kpc_idle();
337 mt_cpu_down(cpu_data_ptr);
338 #endif /* CONFIG_CPU_COUNTERS */
339 #if KPERF
340 kptimer_stop_curcpu();
341 #endif /* KPERF */
342
343 CleanPoC_Dcache();
344
345 #if USE_APPLEARMSMP
346 if (ml_is_quiescing()) {
347 PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id);
348 } else {
349 bool deep_sleep = PE_cpu_down(cpu_data_ptr->cpu_id);
350 cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;
351 // hang CPU on spurious wakeup
352 cpu_data_ptr->cpu_reset_handler = (uintptr_t)0;
353 __builtin_arm_dsb(DSB_ISH);
354 CleanPoU_Dcache();
355 #if APPLEVIRTUALPLATFORM
356 arm64_prepare_for_sleep(deep_sleep, cpu_data_ptr->cpu_number, ml_vtophys(reset_vector_vaddr));
357 #else /* APPLEVIRTUALPLATFORM */
358 arm64_prepare_for_sleep(deep_sleep);
359 #endif /* APPLEVIRTUALPLATFORM */
360 }
361 #else
362 PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id);
363 #endif
364 /*NOTREACHED*/
365 }
366
367 /*
368 * Routine: cpu_interrupt_is_pending
369 * Function: Returns a bool signifying a non-zero ISR_EL1,
370 * indicating a pending IRQ, FIQ or external abort.
371 */
372
373 bool
cpu_interrupt_is_pending(void)374 cpu_interrupt_is_pending(void)
375 {
376 uint64_t isr_value;
377 isr_value = __builtin_arm_rsr64("ISR_EL1");
378 return isr_value != 0;
379 }
380
381 static bool
cpu_proximate_timer(void)382 cpu_proximate_timer(void)
383 {
384 return !SetIdlePop();
385 }
386
387 #ifdef ARM64_BOARD_CONFIG_T6000
388 int wfe_allowed = 0;
389 #else
390 int wfe_allowed = 1;
391 #endif /* ARM64_BOARD_CONFIG_T6000 */
392
393 #if DEVELOPMENT || DEBUG
394 #define WFE_STAT(x) \
395 do { \
396 (x); \
397 } while(0)
398 #else
399 #define WFE_STAT(x) do {} while(0)
400 #endif /* DEVELOPMENT || DEBUG */
401
402 bool
wfe_to_deadline_or_interrupt(uint32_t cid,uint64_t wfe_deadline,cpu_data_t * cdp,bool unmask,bool check_cluster_recommendation)403 wfe_to_deadline_or_interrupt(uint32_t cid, uint64_t wfe_deadline, cpu_data_t *cdp, bool unmask, bool check_cluster_recommendation)
404 {
405 bool ipending = false;
406 uint64_t irqc = 0, nirqc = 0;
407
408 /* The ARMv8 architecture permits a processor dwelling in WFE
409 * with F/IRQ masked to ignore a pending interrupt, i.e.
410 * not classify it as an 'event'. This is potentially
411 * problematic with AICv2's IRQ distribution model, as
412 * a transient interrupt masked interval can cause an SIQ
413 * query rejection, possibly routing the interrupt to
414 * another core/cluster in a powergated state.
415 * Hence, optionally unmask IRQs+FIQs across WFE.
416 */
417 if (unmask) {
418 /* Latch SW IRQ+FIQ counter prior to unmasking
419 * interrupts.
420 */
421 irqc = nirqc = os_atomic_load(&cdp->cpu_stat.irq_ex_cnt_wake, relaxed);
422 __builtin_arm_wsr("DAIFClr", DAIFSC_STANDARD_DISABLE);
423 }
424
425 while ((ipending = (cpu_interrupt_is_pending())) == false) {
426 if (unmask) {
427 /* If WFE was issued with IRQs unmasked, an
428 * interrupt may have been processed.
429 * Consult the SW IRQ counter to determine
430 * whether the 'idle loop' must be
431 * re-evaluated.
432 */
433 nirqc = os_atomic_load(&cdp->cpu_stat.irq_ex_cnt_wake, relaxed);
434 if (nirqc != irqc) {
435 break;
436 }
437 }
438
439 if (__probable(wfe_allowed)) {
440 /*
441 * If IRQs are unmasked, there's a small window
442 * where an 'extra' WFE may be issued after
443 * the consultation of the SW interrupt counter
444 * and new interrupt arrival. Hence this WFE
445 * relies on the [FI]RQ interrupt handler
446 * epilogue issuing a 'SEVL', to post an
447 * event which causes the next WFE on the same
448 * PE to retire immediately.
449 */
450
451 __builtin_arm_wfe();
452 }
453
454 WFE_STAT(cdp->wfe_count++);
455 if (wfe_deadline != ~0ULL) {
456 WFE_STAT(cdp->wfe_deadline_checks++);
457 /* Check if the WFE recommendation has expired.
458 * We do not recompute the deadline here.
459 */
460 if ((check_cluster_recommendation && ml_cluster_wfe_timeout(cid) == 0) ||
461 mach_absolute_time() >= wfe_deadline) {
462 WFE_STAT(cdp->wfe_terminations++);
463 break;
464 }
465 }
466 }
467
468 if (unmask) {
469 __builtin_arm_wsr64("DAIFSet", DAIFSC_STANDARD_DISABLE);
470 /* Refetch SW interrupt counter with IRQs masked
471 * It is important that this routine accurately flags
472 * any observed interrupts via its return value,
473 * inaccuracy may lead to an erroneous WFI fallback.
474 */
475 nirqc = os_atomic_load(&cdp->cpu_stat.irq_ex_cnt_wake, relaxed);
476 }
477
478 return ipending || (nirqc != irqc);
479 }
480
481 /*
482 * Routine: cpu_idle
483 * Function:
484 */
485 void __attribute__((noreturn))
cpu_idle(void)486 cpu_idle(void)
487 {
488 cpu_data_t *cpu_data_ptr = getCpuDatap();
489 processor_t processor = current_processor();
490 uint64_t new_idle_timeout_ticks = 0x0ULL, lastPop;
491 bool idle_disallowed = false;
492 /* Read and reset the next_idle_short flag */
493 bool next_idle_short = processor->next_idle_short;
494 processor->next_idle_short = false;
495
496 if (__improbable((!idle_enable))) {
497 idle_disallowed = true;
498 } else if (__improbable(cpu_data_ptr->cpu_signal & SIGPdisabled)) {
499 idle_disallowed = true;
500 }
501
502 if (__improbable(idle_disallowed)) {
503 Idle_load_context();
504 }
505
506 bool ipending = false;
507 uint32_t cid = cpu_data_ptr->cpu_cluster_id;
508 bool check_cluster_recommendation = true;
509 uint64_t wfe_timeout = 0;
510
511 if (idle_proximate_io_wfe_masked == 1) {
512 /* Check for an active perf. controller generated
513 * WFE recommendation for this cluster.
514 */
515 wfe_timeout = ml_cluster_wfe_timeout(cid);
516 }
517
518 if (next_idle_short && expecting_ipi_wfe_timeout_mt > wfe_timeout) {
519 /* In this case we should WFE because a response IPI
520 * is expected soon.
521 */
522 wfe_timeout = expecting_ipi_wfe_timeout_mt;
523 check_cluster_recommendation = false;
524 }
525
526 if (wfe_timeout != 0) {
527 uint64_t wfe_deadline = mach_absolute_time() + wfe_timeout;
528 /* Poll issuing event-bounded WFEs until an interrupt
529 * arrives or the WFE recommendation expires
530 */
531 KDBG(CPUPM_IDLE_WFE | DBG_FUNC_START, ipending, cpu_data_ptr->wfe_count, wfe_timeout, !check_cluster_recommendation);
532 ipending = wfe_to_deadline_or_interrupt(cid, wfe_deadline, cpu_data_ptr, false, check_cluster_recommendation);
533 KDBG(CPUPM_IDLE_WFE | DBG_FUNC_END, ipending, cpu_data_ptr->wfe_count, wfe_deadline);
534 if (ipending == true) {
535 /* Back to machine_idle() */
536 Idle_load_context();
537 }
538 }
539
540 if (__improbable(cpu_proximate_timer())) {
541 if (idle_proximate_timer_wfe == 1) {
542 /* Poll issuing WFEs until the expected
543 * timer FIQ arrives.
544 */
545 KDBG(CPUPM_IDLE_TIMER_WFE | DBG_FUNC_START, ipending, cpu_data_ptr->wfe_count, ~0ULL);
546 ipending = wfe_to_deadline_or_interrupt(cid, ~0ULL, cpu_data_ptr, false, false);
547 KDBG(CPUPM_IDLE_TIMER_WFE | DBG_FUNC_END, ipending, cpu_data_ptr->wfe_count, ~0ULL);
548 assert(ipending == true);
549 }
550 Idle_load_context();
551 }
552
553 lastPop = cpu_data_ptr->rtcPop;
554
555 cpu_data_ptr->cpu_active_thread = current_thread();
556
557 if (wfi && (cpu_data_ptr->cpu_idle_notify != NULL)) {
558 cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, TRUE, &new_idle_timeout_ticks);
559 }
560
561 if (cpu_data_ptr->idle_timer_notify != NULL) {
562 if (new_idle_timeout_ticks == 0x0ULL) {
563 /* turn off the idle timer */
564 cpu_data_ptr->idle_timer_deadline = 0x0ULL;
565 } else {
566 /* set the new idle timeout */
567 clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline);
568 }
569 timer_resync_deadlines();
570 if (cpu_data_ptr->rtcPop != lastPop) {
571 SetIdlePop();
572 }
573 }
574
575 #if CONFIG_CPU_COUNTERS
576 kpc_idle();
577 mt_cpu_idle(cpu_data_ptr);
578 #endif /* CONFIG_CPU_COUNTERS */
579
580 if (wfi) {
581 #if !defined(APPLE_ARM64_ARCH_FAMILY)
582 platform_cache_idle_enter();
583 #endif
584
585 #if DEVELOPMENT || DEBUG
586 // When simulating wfi overhead,
587 // force wfi to clock gating only
588 if (wfi == 2) {
589 arm64_force_wfi_clock_gate();
590 }
591 #endif /* DEVELOPMENT || DEBUG */
592
593 #if defined(APPLETYPHOON)
594 // <rdar://problem/15827409> CPU1 Stuck in WFIWT Because of MMU Prefetch
595 typhoon_prepare_for_wfi();
596 #endif
597 __builtin_arm_dsb(DSB_SY);
598 #if HAS_RETENTION_STATE
599 arm64_retention_wfi();
600 #else
601 __builtin_arm_wfi();
602 #endif
603
604 #if defined(APPLETYPHOON)
605 // <rdar://problem/15827409> CPU1 Stuck in WFIWT Because of MMU Prefetch
606 typhoon_return_from_wfi();
607 #endif
608
609 #if DEVELOPMENT || DEBUG
610 // Handle wfi overhead simulation
611 if (wfi == 2) {
612 uint64_t deadline;
613
614 // Calculate wfi delay deadline
615 clock_absolutetime_interval_to_deadline(wfi_delay, &deadline);
616
617 // Flush L1 caches
618 if ((wfi_flags & 1) != 0) {
619 InvalidatePoU_Icache();
620 FlushPoC_Dcache();
621 }
622
623 // Flush TLBs
624 if ((wfi_flags & 2) != 0) {
625 flush_core_tlb();
626 }
627
628 // Wait for the ballance of the wfi delay
629 clock_delay_until(deadline);
630 }
631 #endif /* DEVELOPMENT || DEBUG */
632 }
633
634 ClearIdlePop(TRUE);
635
636 cpu_idle_exit(FALSE);
637 }
638
639 /*
640 * Routine: cpu_idle_exit
641 * Function:
642 */
643 void
cpu_idle_exit(boolean_t from_reset)644 cpu_idle_exit(boolean_t from_reset)
645 {
646 uint64_t new_idle_timeout_ticks = 0x0ULL;
647 cpu_data_t *cpu_data_ptr = getCpuDatap();
648
649 assert(exception_stack_pointer() != 0);
650
651 /* Back from WFI, unlock OSLAR and EDLAR. */
652 if (from_reset) {
653 configure_coresight_registers(cpu_data_ptr);
654 }
655
656 #if CONFIG_CPU_COUNTERS
657 kpc_idle_exit();
658 mt_cpu_run(cpu_data_ptr);
659 #endif /* CONFIG_CPU_COUNTERS */
660
661 if (wfi && (cpu_data_ptr->cpu_idle_notify != NULL)) {
662 cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, FALSE, &new_idle_timeout_ticks);
663 }
664
665 if (cpu_data_ptr->idle_timer_notify != NULL) {
666 if (new_idle_timeout_ticks == 0x0ULL) {
667 /* turn off the idle timer */
668 cpu_data_ptr->idle_timer_deadline = 0x0ULL;
669 } else {
670 /* set the new idle timeout */
671 clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline);
672 }
673 timer_resync_deadlines();
674 }
675
676 #if KASAN_TBI
677 kasan_unpoison_curstack(false);
678 #endif /* KASAN_TBI */
679
680 Idle_load_context();
681 }
682
683 void
cpu_init(void)684 cpu_init(void)
685 {
686 cpu_data_t *cdp = getCpuDatap();
687 arm_cpu_info_t *cpu_info_p;
688
689 assert(exception_stack_pointer() != 0);
690
691 if (cdp->cpu_type != CPU_TYPE_ARM64) {
692 cdp->cpu_type = CPU_TYPE_ARM64;
693
694 timer_call_queue_init(&cdp->rtclock_timer.queue);
695 cdp->rtclock_timer.deadline = EndOfAllTime;
696
697 if (cdp == &BootCpuData) {
698 do_cpuid();
699 do_mvfpid();
700 } else {
701 /*
702 * We initialize non-boot CPUs here; the boot CPU is
703 * dealt with as part of pmap_bootstrap.
704 */
705 pmap_cpu_data_init();
706 }
707
708 do_cacheid();
709
710 /* ARM_SMP: Assuming identical cpu */
711 do_debugid();
712
713 cpu_info_p = cpuid_info();
714
715 /* switch based on CPU's reported architecture */
716 switch (cpu_info_p->arm_info.arm_arch) {
717 case CPU_ARCH_ARMv8:
718 cdp->cpu_subtype = CPU_SUBTYPE_ARM64_V8;
719 break;
720 case CPU_ARCH_ARMv8E:
721 cdp->cpu_subtype = CPU_SUBTYPE_ARM64E;
722 break;
723 default:
724 //cdp->cpu_subtype = CPU_SUBTYPE_ARM64_ALL;
725 /* this panic doesn't work this early in startup */
726 panic("Unknown CPU subtype...");
727 break;
728 }
729
730 cdp->cpu_threadtype = CPU_THREADTYPE_NONE;
731 }
732 cdp->cpu_stat.irq_ex_cnt_wake = 0;
733 cdp->cpu_stat.ipi_cnt_wake = 0;
734 #if CONFIG_CPU_COUNTERS
735 cdp->cpu_stat.pmi_cnt_wake = 0;
736 #endif /* CONFIG_CPU_COUNTERS */
737 cdp->cpu_running = TRUE;
738 cdp->cpu_sleep_token_last = cdp->cpu_sleep_token;
739 cdp->cpu_sleep_token = 0x0UL;
740 #if CONFIG_CPU_COUNTERS
741 kpc_idle_exit();
742 mt_cpu_up(cdp);
743 #endif /* CONFIG_CPU_COUNTERS */
744 }
745
746 void
cpu_stack_alloc(cpu_data_t * cpu_data_ptr)747 cpu_stack_alloc(cpu_data_t *cpu_data_ptr)
748 {
749 vm_offset_t irq_stack = 0;
750 vm_offset_t exc_stack = 0;
751
752 kmem_alloc(kernel_map, &irq_stack,
753 INTSTACK_SIZE + ptoa(2), KMA_NOFAIL | KMA_PERMANENT | KMA_ZERO |
754 KMA_GUARD_FIRST | KMA_GUARD_LAST | KMA_KSTACK | KMA_KOBJECT,
755 VM_KERN_MEMORY_STACK);
756
757 cpu_data_ptr->intstack_top = irq_stack + PAGE_SIZE + INTSTACK_SIZE;
758 cpu_data_ptr->istackptr = (void *)cpu_data_ptr->intstack_top;
759
760 kmem_alloc(kernel_map, &exc_stack,
761 EXCEPSTACK_SIZE + ptoa(2), KMA_NOFAIL | KMA_PERMANENT | KMA_ZERO |
762 KMA_GUARD_FIRST | KMA_GUARD_LAST | KMA_KSTACK | KMA_KOBJECT,
763 VM_KERN_MEMORY_STACK);
764
765 cpu_data_ptr->excepstack_top = exc_stack + PAGE_SIZE + EXCEPSTACK_SIZE;
766 cpu_data_ptr->excepstackptr = (void *)cpu_data_ptr->excepstack_top;
767 }
768
769 void
cpu_data_init(cpu_data_t * cpu_data_ptr)770 cpu_data_init(cpu_data_t *cpu_data_ptr)
771 {
772 uint32_t i;
773
774 os_atomic_store(&cpu_data_ptr->cpu_flags, 0, relaxed);
775 cpu_data_ptr->cpu_int_state = 0;
776 cpu_data_ptr->cpu_pending_ast = AST_NONE;
777 cpu_data_ptr->cpu_cache_dispatch = NULL;
778 cpu_data_ptr->rtcPop = EndOfAllTime;
779 cpu_data_ptr->rtclock_datap = &RTClockData;
780 cpu_data_ptr->cpu_user_debug = NULL;
781
782
783 cpu_data_ptr->cpu_base_timebase = 0;
784 cpu_data_ptr->cpu_idle_notify = NULL;
785 cpu_data_ptr->cpu_idle_latency = 0x0ULL;
786 cpu_data_ptr->cpu_idle_pop = 0x0ULL;
787 cpu_data_ptr->cpu_reset_type = 0x0UL;
788 cpu_data_ptr->cpu_reset_handler = 0x0UL;
789 cpu_data_ptr->cpu_reset_assist = 0x0UL;
790 cpu_data_ptr->cpu_regmap_paddr = 0x0ULL;
791 cpu_data_ptr->cpu_phys_id = 0x0UL;
792 cpu_data_ptr->cpu_l2_access_penalty = 0;
793 cpu_data_ptr->cpu_cluster_type = CLUSTER_TYPE_SMP;
794 cpu_data_ptr->cpu_cluster_id = 0;
795 cpu_data_ptr->cpu_l2_id = 0;
796 cpu_data_ptr->cpu_l2_size = 0;
797 cpu_data_ptr->cpu_l3_id = 0;
798 cpu_data_ptr->cpu_l3_size = 0;
799
800 cpu_data_ptr->cpu_signal = SIGPdisabled;
801
802 cpu_data_ptr->cpu_get_fiq_handler = NULL;
803 cpu_data_ptr->cpu_tbd_hardware_addr = NULL;
804 cpu_data_ptr->cpu_tbd_hardware_val = NULL;
805 cpu_data_ptr->cpu_get_decrementer_func = NULL;
806 cpu_data_ptr->cpu_set_decrementer_func = NULL;
807 cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;
808 cpu_data_ptr->cpu_sleep_token_last = 0x00000000UL;
809 cpu_data_ptr->cpu_xcall_p0 = NULL;
810 cpu_data_ptr->cpu_xcall_p1 = NULL;
811 cpu_data_ptr->cpu_imm_xcall_p0 = NULL;
812 cpu_data_ptr->cpu_imm_xcall_p1 = NULL;
813
814 for (i = 0; i < CORESIGHT_REGIONS; ++i) {
815 cpu_data_ptr->coresight_base[i] = 0;
816 }
817
818 #if !XNU_MONITOR
819 pmap_cpu_data_t * pmap_cpu_data_ptr = &cpu_data_ptr->cpu_pmap_cpu_data;
820
821 pmap_cpu_data_ptr->cpu_number = PMAP_INVALID_CPU_NUM;
822 pmap_cpu_data_ptr->pv_free.list = NULL;
823 pmap_cpu_data_ptr->pv_free.count = 0;
824 pmap_cpu_data_ptr->pv_free_spill_marker = NULL;
825 #if !CONFIG_SPTM
826 pmap_cpu_data_ptr->cpu_nested_pmap = (struct pmap *) NULL;
827 bzero(&(pmap_cpu_data_ptr->cpu_sw_asids[0]), sizeof(pmap_cpu_data_ptr->cpu_sw_asids));
828 #endif
829 #endif /* !XNU_MONITOR */
830 cpu_data_ptr->halt_status = CPU_NOT_HALTED;
831 #if __ARM_KERNEL_PROTECT__
832 cpu_data_ptr->cpu_exc_vectors = (vm_offset_t)&exc_vectors_table;
833 #endif /* __ARM_KERNEL_PROTECT__ */
834
835 #if defined(HAS_APPLE_PAC)
836 cpu_data_ptr->rop_key = 0;
837 cpu_data_ptr->jop_key = ml_default_jop_pid();
838 #endif
839 }
840
841 void
cpu_data_register(cpu_data_t * cpu_data_ptr)842 cpu_data_register(cpu_data_t *cpu_data_ptr)
843 {
844 int cpu = cpu_data_ptr->cpu_number;
845
846 #if KASAN
847 for (int i = 0; i < CPUWINDOWS_MAX; i++) {
848 kasan_notify_address_nopoison(pmap_cpu_windows_copy_addr(cpu, i), PAGE_SIZE);
849 }
850 #endif
851
852 __builtin_arm_dmb(DMB_ISH); // Ensure prior stores to cpu data are visible
853 CpuDataEntries[cpu].cpu_data_vaddr = cpu_data_ptr;
854 CpuDataEntries[cpu].cpu_data_paddr = (void *)ml_vtophys((vm_offset_t)cpu_data_ptr);
855 }
856
857 #if defined(KERNEL_INTEGRITY_CTRR)
858 /* Hibernation needs to reset this state, so data and text are in the hib segment;
859 * this allows them be accessed and executed early.
860 */
861 LCK_GRP_DECLARE(ctrr_cpu_start_lock_grp, "ctrr_cpu_start_lock");
862 LCK_SPIN_DECLARE(ctrr_cpu_start_lck, &ctrr_cpu_start_lock_grp);
863 enum ctrr_cluster_states ctrr_cluster_locked[MAX_CPU_CLUSTERS] MARK_AS_HIBERNATE_DATA;
864
865 MARK_AS_HIBERNATE_TEXT
866 void
init_ctrr_cluster_states(void)867 init_ctrr_cluster_states(void)
868 {
869 for (int i = 0; i < MAX_CPU_CLUSTERS; i++) {
870 ctrr_cluster_locked[i] = CTRR_UNLOCKED;
871 }
872 }
873 #endif
874
875 void
cpu_start(int cpu)876 cpu_start(int cpu)
877 {
878 cpu_data_t *cpu_data_ptr = CpuDataEntries[cpu].cpu_data_vaddr;
879 processor_t processor = PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data_ptr);
880
881 if (processor_should_kprintf(processor, true)) {
882 kprintf("cpu_start() cpu: %d\n", cpu);
883 }
884
885 if (cpu == cpu_number()) {
886 /* Current CPU is already running, just needs initialization */
887 cpu_machine_init();
888 configure_coresight_registers(cpu_data_ptr);
889 } else {
890 thread_t first_thread;
891 #if CONFIG_SPTM
892 cpu_data_ptr->cpu_reset_handler = (vm_offset_t) VM_KERNEL_STRIP_PTR(arm_init_cpu);
893 #else
894 cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr;
895 #if !XNU_MONITOR
896 cpu_data_ptr->cpu_pmap_cpu_data.cpu_nested_pmap = NULL;
897 #endif
898 #endif /* !CONFIG_SPTM */
899
900 if (processor->startup_thread != THREAD_NULL) {
901 first_thread = processor->startup_thread;
902 } else {
903 first_thread = processor->idle_thread;
904 }
905 cpu_data_ptr->cpu_active_thread = first_thread;
906 first_thread->machine.CpuDatap = cpu_data_ptr;
907 first_thread->machine.pcpu_data_base_and_cpu_number =
908 ml_make_pcpu_base_and_cpu_number((char *)cpu_data_ptr - __PERCPU_ADDR(cpu_data),
909 cpu_data_ptr->cpu_number);
910
911 configure_coresight_registers(cpu_data_ptr);
912
913 flush_dcache((vm_offset_t)&CpuDataEntries[cpu], sizeof(cpu_data_entry_t), FALSE);
914 flush_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE);
915 #if CONFIG_SPTM
916 /**
917 * On SPTM devices, CTRR is configured entirely by the SPTM. Due to this, this logic
918 * is no longer required in XNU.
919 */
920 #else
921 #if defined(KERNEL_INTEGRITY_CTRR)
922
923 /* First CPU being started within a cluster goes ahead to lock CTRR for cluster;
924 * other CPUs block until cluster is locked. */
925 lck_spin_lock(&ctrr_cpu_start_lck);
926 switch (ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id]) {
927 case CTRR_UNLOCKED:
928 ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] = CTRR_LOCKING;
929 lck_spin_unlock(&ctrr_cpu_start_lck);
930 break;
931 case CTRR_LOCKING:
932
933 lck_spin_sleep(&ctrr_cpu_start_lck, LCK_SLEEP_UNLOCK,
934 &ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id],
935 THREAD_UNINT | THREAD_WAIT_NOREPORT);
936
937 assert(ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] != CTRR_LOCKING);
938 break;
939 default: // CTRR_LOCKED
940 lck_spin_unlock(&ctrr_cpu_start_lck);
941 break;
942 }
943 #endif
944 #endif /* CONFIG_SPTM */
945
946 PE_cpu_start_internal(cpu_data_ptr->cpu_id, (vm_offset_t)NULL, (vm_offset_t)NULL);
947
948 }
949 }
950
951
952 void
cpu_timebase_init(boolean_t from_boot)953 cpu_timebase_init(boolean_t from_boot)
954 {
955 cpu_data_t *cdp = getCpuDatap();
956 uint64_t timebase_offset = 0;
957
958 if (cdp->cpu_get_fiq_handler == NULL) {
959 cdp->cpu_get_fiq_handler = rtclock_timebase_func.tbd_fiq_handler;
960 cdp->cpu_get_decrementer_func = rtclock_timebase_func.tbd_get_decrementer;
961 cdp->cpu_set_decrementer_func = rtclock_timebase_func.tbd_set_decrementer;
962 cdp->cpu_tbd_hardware_addr = (void *)rtclock_timebase_addr;
963 cdp->cpu_tbd_hardware_val = (void *)rtclock_timebase_val;
964 }
965
966 if (!from_boot && (cdp == &BootCpuData) && ml_is_quiescing()) {
967 /*
968 * When we wake from sleep, we have no guarantee about the state
969 * of the hardware timebase. It may have kept ticking across sleep, or
970 * it may have reset.
971 *
972 * To deal with this, we calculate an offset to the clock that will
973 * produce a timebase value wake_abstime at the point the boot
974 * CPU calls cpu_timebase_init on wake.
975 *
976 * This ensures that mach_absolute_time() stops ticking across sleep.
977 */
978 rtclock_base_abstime = wake_abstime - ml_get_hwclock();
979 } else if (from_boot) {
980 #if DEBUG || DEVELOPMENT
981 if (PE_parse_boot_argn("timebase_offset", &timebase_offset, sizeof(timebase_offset))) {
982 rtclock_base_abstime += timebase_offset;
983 }
984 #endif
985 /* On initial boot, initialize time_since_reset to CNTPCT_EL0. */
986 ml_set_reset_time(ml_get_hwclock());
987 }
988
989 cdp->cpu_decrementer = 0x7FFFFFFFUL;
990 cdp->cpu_timebase = timebase_offset;
991 cdp->cpu_base_timebase = rtclock_base_abstime;
992 }
993
994 int
cpu_cluster_id(void)995 cpu_cluster_id(void)
996 {
997 return getCpuDatap()->cpu_cluster_id;
998 }
999
1000 __attribute__((noreturn))
1001 void
ml_arm_sleep(void)1002 ml_arm_sleep(void)
1003 {
1004 cpu_data_t *cpu_data_ptr = getCpuDatap();
1005
1006 if (cpu_data_ptr == &BootCpuData) {
1007 cpu_data_t *target_cdp;
1008 int cpu;
1009 int max_cpu;
1010
1011 max_cpu = ml_get_max_cpu_number();
1012 for (cpu = 0; cpu <= max_cpu; cpu++) {
1013 target_cdp = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr;
1014
1015 if ((target_cdp == NULL) || (target_cdp == cpu_data_ptr)) {
1016 continue;
1017 }
1018
1019 while (target_cdp->cpu_sleep_token != ARM_CPU_ON_SLEEP_PATH) {
1020 ;
1021 }
1022 }
1023
1024 /*
1025 * Now that the other cores have entered the sleep path, set
1026 * the abstime value we'll use when we resume.
1027 */
1028 wake_abstime = ml_get_timebase();
1029 ml_set_reset_time(UINT64_MAX);
1030 } else {
1031 CleanPoU_Dcache();
1032 }
1033
1034 cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH;
1035
1036 if (cpu_data_ptr == &BootCpuData) {
1037 #if WITH_CLASSIC_S2R
1038 // Classic suspend to RAM writes the suspend signature into the
1039 // sleep token buffer so that iBoot knows that it's on the warm
1040 // boot (wake) path (as opposed to the cold boot path). Newer SoC
1041 // do not go through SecureROM/iBoot on the warm boot path. The
1042 // reconfig engine script brings the CPU out of reset at the kernel's
1043 // reset vector which points to the warm boot initialization code.
1044 if (sleepTokenBuffer != (vm_offset_t) NULL) {
1045 platform_cache_shutdown();
1046 bcopy((const void *)suspend_signature, (void *)sleepTokenBuffer, sizeof(SleepToken));
1047 } else {
1048 panic("No sleep token buffer");
1049 }
1050 #endif
1051
1052 #if __ARM_GLOBAL_SLEEP_BIT__
1053 /* Allow other CPUs to go to sleep. */
1054 arm64_stall_sleep = FALSE;
1055 __builtin_arm_dmb(DMB_ISH);
1056 #endif
1057
1058 /* Architectural debug state: <rdar://problem/12390433>:
1059 * Grab debug lock EDLAR and clear bit 0 in EDPRCR,
1060 * tell debugger to not prevent power gating .
1061 */
1062 if (cpu_data_ptr->coresight_base[CORESIGHT_ED]) {
1063 *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY;
1064 *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGPRCR) = 0;
1065 }
1066
1067 #if HIBERNATION
1068 uint32_t mode = hibernate_write_image();
1069 if (mode == kIOHibernatePostWriteHalt) {
1070 HIBLOG("powering off after writing hibernation image\n");
1071 int halt_result = -1;
1072 if (PE_halt_restart) {
1073 /**
1074 * Drain serial FIFOs now as the normal call further down won't
1075 * be hit when the CPU halts here for hibernation. Here, it'll
1076 * make sure the preceding HIBLOG is flushed as well.
1077 */
1078 serial_go_to_sleep();
1079 halt_result = (*PE_halt_restart)(kPEHaltCPU);
1080 }
1081 panic("can't shutdown: PE_halt_restart returned %d", halt_result);
1082 }
1083 #endif /* HIBERNATION */
1084
1085 serial_go_to_sleep();
1086
1087 #if CONFIG_CPU_COUNTERS
1088 mt_sleep();
1089 #endif /* CONFIG_CPU_COUNTERS */
1090 /* ARM64-specific preparation */
1091 #if APPLEVIRTUALPLATFORM
1092 extern bool test_sleep_in_vm;
1093 if (test_sleep_in_vm) {
1094 /*
1095 * Until sleep is supported on APPLEVIRTUALPLATFORM, use this
1096 * trick for testing sleep - just jump straight to the CPU resume point.
1097 */
1098 arm_init_cpu(cpu_data_ptr, 0);
1099 }
1100 arm64_prepare_for_sleep(true, cpu_data_ptr->cpu_number, ml_vtophys(reset_vector_vaddr));
1101 #else /* APPLEVIRTUALPLATFORM */
1102 arm64_prepare_for_sleep(true);
1103 #endif /* APPLEVIRTUALPLATFORM */
1104 } else {
1105 #if __ARM_GLOBAL_SLEEP_BIT__
1106 /*
1107 * With the exception of the CPU revisions listed above, our ARM64 CPUs have a
1108 * global register to manage entering deep sleep, as opposed to a per-CPU
1109 * register. We cannot update this register until all CPUs are ready to enter
1110 * deep sleep, because if a CPU executes WFI outside of the deep sleep context
1111 * (by idling), it will hang (due to the side effects of enabling deep sleep),
1112 * which can hang the sleep process or cause memory corruption on wake.
1113 *
1114 * To avoid these issues, we'll stall on this global value, which CPU0 will
1115 * manage.
1116 */
1117 while (arm64_stall_sleep) {
1118 __builtin_arm_wfe();
1119 }
1120 #endif
1121 CleanPoU_DcacheRegion((vm_offset_t) cpu_data_ptr, sizeof(cpu_data_t));
1122
1123 /* Architectural debug state: <rdar://problem/12390433>:
1124 * Grab debug lock EDLAR and clear bit 0 in EDPRCR,
1125 * tell debugger to not prevent power gating .
1126 */
1127 if (cpu_data_ptr->coresight_base[CORESIGHT_ED]) {
1128 *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY;
1129 *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGPRCR) = 0;
1130 }
1131
1132 /* ARM64-specific preparation */
1133 #if APPLEVIRTUALPLATFORM
1134 arm64_prepare_for_sleep(true, cpu_data_ptr->cpu_number, ml_vtophys(reset_vector_vaddr));
1135 #else /* APPLEVIRTUALPLATFORM */
1136 arm64_prepare_for_sleep(true);
1137 #endif /* APPLEVIRTUALPLATFORM */
1138 }
1139 }
1140
1141 void
cpu_machine_idle_init(boolean_t from_boot)1142 cpu_machine_idle_init(boolean_t from_boot)
1143 {
1144 #if !CONFIG_SPTM
1145 static vm_address_t resume_idle_cpu_paddr = (vm_address_t)NULL;
1146 #endif
1147 cpu_data_t *cpu_data_ptr = getCpuDatap();
1148
1149 if (from_boot) {
1150 uint32_t production = 1;
1151 DTEntry entry;
1152
1153 unsigned long jtag = 0;
1154
1155 if (PE_parse_boot_argn("jtag", &jtag, sizeof(jtag))) {
1156 if (jtag != 0) {
1157 idle_enable = FALSE;
1158 } else {
1159 idle_enable = TRUE;
1160 }
1161 } else {
1162 idle_enable = TRUE;
1163 }
1164
1165 #if DEVELOPMENT || DEBUG
1166 uint32_t wfe_mode = 0;
1167 if (PE_parse_boot_argn("wfe_mode", &wfe_mode, sizeof(wfe_mode))) {
1168 idle_proximate_timer_wfe = ((wfe_mode & 1) == 1);
1169 idle_proximate_io_wfe_masked = ((wfe_mode & 2) == 2);
1170 extern uint32_t idle_proximate_io_wfe_unmasked;
1171 idle_proximate_io_wfe_unmasked = ((wfe_mode & 4) == 4);
1172 }
1173 #endif
1174
1175 // bits 7..0 give the wfi type
1176 switch (wfi & 0xff) {
1177 case 0:
1178 // disable wfi
1179 wfi = 0;
1180 break;
1181
1182 #if DEVELOPMENT || DEBUG
1183 case 2:
1184 // wfi overhead simulation
1185 // 31..16 - wfi delay is us
1186 // 15..8 - flags
1187 // 7..0 - 2
1188 wfi = 2;
1189 wfi_flags = (wfi >> 8) & 0xFF;
1190 nanoseconds_to_absolutetime(((wfi >> 16) & 0xFFFF) * NSEC_PER_MSEC, &wfi_delay);
1191 break;
1192 #endif /* DEVELOPMENT || DEBUG */
1193
1194 case 1:
1195 default:
1196 // do nothing
1197 break;
1198 }
1199 #if !CONFIG_SPTM
1200 ResetHandlerData.assist_reset_handler = 0;
1201 ResetHandlerData.cpu_data_entries = ml_static_vtop((vm_offset_t)CpuDataEntries);
1202 #endif
1203
1204 #ifdef MONITOR
1205 monitor_call(MONITOR_SET_ENTRY, (uintptr_t)ml_static_vtop((vm_offset_t)&LowResetVectorBase), 0, 0);
1206 #elif !defined(NO_MONITOR)
1207 #error MONITOR undefined, WFI power gating may not operate correctly
1208 #endif /* MONITOR */
1209
1210 // Determine if we are on production or debug chip
1211 if (kSuccess == SecureDTLookupEntry(NULL, "/chosen", &entry)) {
1212 unsigned int size;
1213 void const *prop;
1214
1215 if (kSuccess == SecureDTGetProperty(entry, "effective-production-status-ap", &prop, &size)) {
1216 if (size == 4) {
1217 bcopy(prop, &production, size);
1218 }
1219 }
1220 }
1221 if (!production) {
1222 #if defined(APPLE_ARM64_ARCH_FAMILY)
1223 // Enable coresight debug registers on debug-fused chips
1224 coresight_debug_enabled = TRUE;
1225 #endif
1226 }
1227 #if !CONFIG_SPTM
1228 start_cpu_paddr = ml_static_vtop((vm_offset_t)&start_cpu);
1229 resume_idle_cpu_paddr = ml_static_vtop((vm_offset_t)&resume_idle_cpu);
1230 #endif
1231 }
1232
1233 #if WITH_CLASSIC_S2R
1234 if (cpu_data_ptr == &BootCpuData) {
1235 static addr64_t SleepToken_low_paddr = (addr64_t)NULL;
1236 if (sleepTokenBuffer != (vm_offset_t) NULL) {
1237 SleepToken_low_paddr = ml_vtophys(sleepTokenBuffer);
1238 } else {
1239 panic("No sleep token buffer");
1240 }
1241
1242 bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)running_signature),
1243 SleepToken_low_paddr, sizeof(SleepToken));
1244 flush_dcache((vm_offset_t)SleepToken, sizeof(SleepToken), TRUE);
1245 }
1246 ;
1247 #endif
1248 #if CONFIG_SPTM
1249 cpu_data_ptr->cpu_reset_handler = (uintptr_t) VM_KERNEL_STRIP_PTR(arm_init_idle_cpu);
1250 #else
1251 cpu_data_ptr->cpu_reset_handler = resume_idle_cpu_paddr;
1252 #endif
1253 clean_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE);
1254 }
1255
1256 _Atomic uint32_t cpu_idle_count = 0;
1257
1258 void
machine_track_platform_idle(boolean_t entry)1259 machine_track_platform_idle(boolean_t entry)
1260 {
1261 if (entry) {
1262 os_atomic_inc(&cpu_idle_count, relaxed);
1263 } else {
1264 os_atomic_dec(&cpu_idle_count, relaxed);
1265 }
1266 }
1267
1268 #if WITH_CLASSIC_S2R
1269 void
sleep_token_buffer_init(void)1270 sleep_token_buffer_init(void)
1271 {
1272 cpu_data_t *cpu_data_ptr = getCpuDatap();
1273 DTEntry entry;
1274 size_t size;
1275 void const * const *prop;
1276
1277 if ((cpu_data_ptr == &BootCpuData) && (sleepTokenBuffer == (vm_offset_t) NULL)) {
1278 /* Find the stpage node in the device tree */
1279 if (kSuccess != SecureDTLookupEntry(0, "stram", &entry)) {
1280 return;
1281 }
1282
1283 if (kSuccess != SecureDTGetProperty(entry, "reg", (const void **)&prop, (unsigned int *)&size)) {
1284 return;
1285 }
1286
1287 /* Map the page into the kernel space */
1288 sleepTokenBuffer = ml_io_map(((vm_offset_t const *)prop)[0], ((vm_size_t const *)prop)[1]);
1289 }
1290 }
1291 #endif
1292