1 /*
2 * Copyright (c) 2000-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58
59 /*
60 * processor.c: processor and processor_set manipulation routines.
61 */
62
63 #include <mach/boolean.h>
64 #include <mach/policy.h>
65 #include <mach/processor.h>
66 #include <mach/processor_info.h>
67 #include <mach/vm_param.h>
68 #include <kern/cpu_number.h>
69 #include <kern/host.h>
70 #include <kern/ipc_host.h>
71 #include <kern/ipc_tt.h>
72 #include <kern/kalloc.h>
73 #include <kern/machine.h>
74 #include <kern/misc_protos.h>
75 #include <kern/processor.h>
76 #include <kern/sched.h>
77 #include <kern/task.h>
78 #include <kern/thread.h>
79 #include <kern/timer.h>
80 #if KPERF
81 #include <kperf/kperf.h>
82 #endif /* KPERF */
83 #include <ipc/ipc_port.h>
84 #include <machine/commpage.h>
85
86 #include <security/mac_mach_internal.h>
87
88 #if defined(CONFIG_XNUPOST)
89
90 #include <tests/xnupost.h>
91
92 #endif /* CONFIG_XNUPOST */
93
94 /*
95 * Exported interface
96 */
97 #include <mach/mach_host_server.h>
98 #include <mach/processor_set_server.h>
99 #include <san/kcov.h>
100
101 /* The boot pset and pset node */
102 struct processor_set pset0;
103 struct pset_node pset_node0;
104
105 #if __AMP__
106 /* Additional AMP node */
107 static struct pset_node pset_node1;
108 /*
109 * For AMP platforms, all clusters of the same type are part of
110 * the same pset_node. This allows for easier CPU selection logic.
111 */
112 pset_node_t ecore_node;
113 pset_node_t pcore_node;
114 #endif /* __AMP__ */
115
116 LCK_SPIN_DECLARE(pset_node_lock, LCK_GRP_NULL);
117
118 LCK_GRP_DECLARE(pset_lck_grp, "pset");
119
120 queue_head_t tasks;
121 queue_head_t terminated_tasks; /* To be used ONLY for stackshot. */
122 queue_head_t corpse_tasks;
123 int tasks_count;
124 int terminated_tasks_count;
125 queue_head_t threads;
126 queue_head_t terminated_threads;
127 int threads_count;
128 int terminated_threads_count;
129 LCK_GRP_DECLARE(task_lck_grp, "task");
130 LCK_ATTR_DECLARE(task_lck_attr, 0, 0);
131 LCK_MTX_DECLARE_ATTR(tasks_threads_lock, &task_lck_grp, &task_lck_attr);
132 LCK_MTX_DECLARE_ATTR(tasks_corpse_lock, &task_lck_grp, &task_lck_attr);
133
134 processor_t processor_list;
135 unsigned int processor_count;
136 static processor_t processor_list_tail;
137 SIMPLE_LOCK_DECLARE(processor_list_lock, 0);
138 SIMPLE_LOCK_DECLARE(processor_start_state_lock, 0);
139
140 uint32_t processor_avail_count;
141 uint32_t processor_avail_count_user;
142 uint32_t primary_processor_avail_count;
143 uint32_t primary_processor_avail_count_user;
144
145 #if XNU_SUPPORT_BOOTCPU_SHUTDOWN
146 TUNABLE(bool, support_bootcpu_shutdown, "support_bootcpu_shutdown", true);
147 #else
148 TUNABLE(bool, support_bootcpu_shutdown, "support_bootcpu_shutdown", false);
149 #endif
150
151 #if __x86_64__ || XNU_ENABLE_PROCESSOR_EXIT
152 TUNABLE(bool, enable_processor_exit, "processor_exit", true);
153 #else
154 TUNABLE(bool, enable_processor_exit, "processor_exit", false);
155 #endif
156
157 SECURITY_READ_ONLY_LATE(int) master_cpu = 0;
158
159 struct processor PERCPU_DATA(processor);
160 processor_t processor_array[MAX_SCHED_CPUS] = { 0 };
161 processor_set_t pset_array[MAX_PSETS] = { 0 };
162
163 static timer_call_func_t running_timer_funcs[] = {
164 [RUNNING_TIMER_QUANTUM] = thread_quantum_expire,
165 [RUNNING_TIMER_PREEMPT] = thread_preempt_expire,
166 [RUNNING_TIMER_KPERF] = kperf_timer_expire,
167 };
168 static_assert(sizeof(running_timer_funcs) / sizeof(running_timer_funcs[0])
169 == RUNNING_TIMER_MAX, "missing running timer function");
170
171 #if defined(CONFIG_XNUPOST)
172 kern_return_t ipi_test(void);
173 extern void arm64_ipi_test(void);
174
175 kern_return_t
ipi_test()176 ipi_test()
177 {
178 #if __arm64__
179 processor_t p;
180
181 for (p = processor_list; p != NULL; p = p->processor_list) {
182 thread_bind(p);
183 thread_block(THREAD_CONTINUE_NULL);
184 kprintf("Running IPI test on cpu %d\n", p->cpu_id);
185 arm64_ipi_test();
186 }
187
188 /* unbind thread from specific cpu */
189 thread_bind(PROCESSOR_NULL);
190 thread_block(THREAD_CONTINUE_NULL);
191
192 T_PASS("Done running IPI tests");
193 #else
194 T_PASS("Unsupported platform. Not running IPI tests");
195
196 #endif /* __arm64__ */
197
198 return KERN_SUCCESS;
199 }
200 #endif /* defined(CONFIG_XNUPOST) */
201
202 int sched_enable_smt = 1;
203
204 cpumap_t processor_offline_state_map[PROCESSOR_OFFLINE_MAX];
205
206 void
processor_update_offline_state_locked(processor_t processor,processor_offline_state_t new_state)207 processor_update_offline_state_locked(processor_t processor,
208 processor_offline_state_t new_state)
209 {
210 simple_lock_assert(&sched_available_cores_lock, LCK_ASSERT_OWNED);
211
212 processor_offline_state_t old_state = processor->processor_offline_state;
213
214 uint cpuid = (uint)processor->cpu_id;
215
216 assert(old_state < PROCESSOR_OFFLINE_MAX);
217 assert(new_state < PROCESSOR_OFFLINE_MAX);
218
219 processor->processor_offline_state = new_state;
220
221 bit_clear(processor_offline_state_map[old_state], cpuid);
222 bit_set(processor_offline_state_map[new_state], cpuid);
223 }
224
225 void
processor_update_offline_state(processor_t processor,processor_offline_state_t new_state)226 processor_update_offline_state(processor_t processor,
227 processor_offline_state_t new_state)
228 {
229 spl_t s = splsched();
230 simple_lock(&sched_available_cores_lock, LCK_GRP_NULL);
231 processor_update_offline_state_locked(processor, new_state);
232 simple_unlock(&sched_available_cores_lock);
233 splx(s);
234 }
235
236 void
processor_bootstrap(void)237 processor_bootstrap(void)
238 {
239 simple_lock_init(&sched_available_cores_lock, 0);
240 simple_lock_init(&processor_start_state_lock, 0);
241
242 /* Initialize boot pset node */
243 pset_node0.psets = &pset0;
244 pset_node0.pset_cluster_type = PSET_SMP;
245
246 #if __AMP__
247 const ml_topology_info_t *topology_info = ml_get_topology_info();
248
249 /*
250 * Continue initializing boot pset and node.
251 * Since this is an AMP system, fill up cluster type and ID information; this should do the
252 * same kind of initialization done via ml_processor_register()
253 */
254 ml_topology_cluster_t *boot_cluster = topology_info->boot_cluster;
255 pset0.pset_id = boot_cluster->cluster_id;
256 pset0.pset_cluster_id = boot_cluster->cluster_id;
257 pset_cluster_type_t boot_type = cluster_type_to_pset_cluster_type(boot_cluster->cluster_type);
258 pset0.pset_cluster_type = boot_type;
259 pset_node0.pset_cluster_type = boot_type;
260
261 /* Initialize pset node pointers according to their type */
262 switch (boot_type) {
263 case PSET_AMP_P:
264 pcore_node = &pset_node0;
265 ecore_node = &pset_node1;
266 break;
267 case PSET_AMP_E:
268 ecore_node = &pset_node0;
269 pcore_node = &pset_node1;
270 break;
271 default:
272 panic("Unexpected boot pset cluster type %d", boot_type);
273 }
274 ecore_node->pset_cluster_type = PSET_AMP_E;
275 pcore_node->pset_cluster_type = PSET_AMP_P;
276
277 /* Link pset_node1 to pset_node0 */
278 pset_node0.node_list = &pset_node1;
279 #endif /* __AMP__ */
280
281 pset_init(&pset0, &pset_node0);
282 queue_init(&tasks);
283 queue_init(&terminated_tasks);
284 queue_init(&threads);
285 queue_init(&terminated_threads);
286 queue_init(&corpse_tasks);
287
288 processor_init(master_processor, master_cpu, &pset0);
289 }
290
291 /*
292 * Initialize the given processor for the cpu
293 * indicated by cpu_id, and assign to the
294 * specified processor set.
295 */
296 void
processor_init(processor_t processor,int cpu_id,processor_set_t pset)297 processor_init(
298 processor_t processor,
299 int cpu_id,
300 processor_set_t pset)
301 {
302 spl_t s;
303
304 assert(cpu_id < MAX_SCHED_CPUS);
305 processor->cpu_id = cpu_id;
306
307 if (processor != master_processor) {
308 /* Scheduler state for master_processor initialized in sched_init() */
309 SCHED(processor_init)(processor);
310 smr_cpu_init(processor);
311 }
312
313 processor->state = PROCESSOR_OFF_LINE;
314 processor->active_thread = processor->startup_thread = processor->idle_thread = THREAD_NULL;
315 processor->processor_set = pset;
316 processor_state_update_idle(processor);
317 processor->starting_pri = MINPRI;
318 processor->quantum_end = UINT64_MAX;
319 processor->deadline = UINT64_MAX;
320 processor->first_timeslice = FALSE;
321 processor->processor_online = false;
322 processor->processor_primary = processor; /* no SMT relationship known at this point */
323 processor->processor_secondary = NULL;
324 processor->is_SMT = false;
325 processor->processor_self = IP_NULL;
326 processor->processor_list = NULL;
327 processor->must_idle = false;
328 processor->next_idle_short = false;
329 processor->last_startup_reason = REASON_SYSTEM;
330 processor->last_shutdown_reason = REASON_NONE;
331 processor->shutdown_temporary = false;
332 processor->processor_inshutdown = false;
333 processor->processor_instartup = false;
334 processor->last_derecommend_reason = REASON_NONE;
335 processor->running_timers_active = false;
336 for (int i = 0; i < RUNNING_TIMER_MAX; i++) {
337 timer_call_setup(&processor->running_timers[i],
338 running_timer_funcs[i], processor);
339 running_timer_clear(processor, i);
340 }
341 recount_processor_init(processor);
342
343 s = splsched();
344 simple_lock(&sched_available_cores_lock, LCK_GRP_NULL);
345
346 pset_lock(pset);
347 bit_set(pset->cpu_bitmask, cpu_id);
348 bit_set(pset->recommended_bitmask, cpu_id);
349 atomic_bit_set(&pset->node->pset_recommended_map, pset->pset_id, memory_order_relaxed);
350 bit_set(pset->primary_map, cpu_id);
351 bit_set(pset->cpu_state_map[PROCESSOR_OFF_LINE], cpu_id);
352 if (pset->cpu_set_count++ == 0) {
353 pset->cpu_set_low = pset->cpu_set_hi = cpu_id;
354 } else {
355 pset->cpu_set_low = (cpu_id < pset->cpu_set_low)? cpu_id: pset->cpu_set_low;
356 pset->cpu_set_hi = (cpu_id > pset->cpu_set_hi)? cpu_id: pset->cpu_set_hi;
357 }
358
359 processor->last_recommend_reason = REASON_SYSTEM;
360 sched_processor_change_mode_locked(processor, PCM_RECOMMENDED, true);
361 pset_unlock(pset);
362
363 processor->processor_offline_state = PROCESSOR_OFFLINE_NOT_BOOTED;
364 bit_set(processor_offline_state_map[processor->processor_offline_state], cpu_id);
365
366 if (processor == master_processor) {
367 processor_update_offline_state_locked(processor, PROCESSOR_OFFLINE_STARTING);
368 }
369
370 simple_unlock(&sched_available_cores_lock);
371 splx(s);
372
373 simple_lock(&processor_list_lock, LCK_GRP_NULL);
374 if (processor_list == NULL) {
375 processor_list = processor;
376 } else {
377 processor_list_tail->processor_list = processor;
378 }
379 processor_list_tail = processor;
380 processor_count++;
381 simple_unlock(&processor_list_lock);
382 processor_array[cpu_id] = processor;
383 }
384
385 bool system_is_SMT = false;
386
387 void
processor_set_primary(processor_t processor,processor_t primary)388 processor_set_primary(
389 processor_t processor,
390 processor_t primary)
391 {
392 assert(processor->processor_primary == primary || processor->processor_primary == processor);
393 /* Re-adjust primary point for this (possibly) secondary processor */
394 processor->processor_primary = primary;
395
396 assert(primary->processor_secondary == NULL || primary->processor_secondary == processor);
397 if (primary != processor) {
398 /* Link primary to secondary, assumes a 2-way SMT model
399 * We'll need to move to a queue if any future architecture
400 * requires otherwise.
401 */
402 assert(processor->processor_secondary == NULL);
403 primary->processor_secondary = processor;
404 /* Mark both processors as SMT siblings */
405 primary->is_SMT = TRUE;
406 processor->is_SMT = TRUE;
407
408 if (!system_is_SMT) {
409 system_is_SMT = true;
410 sched_rt_n_backup_processors = SCHED_DEFAULT_BACKUP_PROCESSORS_SMT;
411 }
412
413 processor_set_t pset = processor->processor_set;
414 spl_t s = splsched();
415 pset_lock(pset);
416 if (!pset->is_SMT) {
417 pset->is_SMT = true;
418 }
419 bit_clear(pset->primary_map, processor->cpu_id);
420 pset_unlock(pset);
421 splx(s);
422 }
423 }
424
425 processor_set_t
processor_pset(processor_t processor)426 processor_pset(
427 processor_t processor)
428 {
429 return processor->processor_set;
430 }
431
432 #if CONFIG_SCHED_EDGE
433
434 /* Returns the scheduling type for the pset */
435 cluster_type_t
pset_type_for_id(uint32_t cluster_id)436 pset_type_for_id(uint32_t cluster_id)
437 {
438 return pset_array[cluster_id]->pset_type;
439 }
440
441 /*
442 * Processor foreign threads
443 *
444 * With the Edge scheduler, each pset maintains a bitmap of processors running threads
445 * which are foreign to the pset/cluster. A thread is defined as foreign for a cluster
446 * if its of a different type than its preferred cluster type (E/P). The bitmap should
447 * be updated every time a new thread is assigned to run on a processor. Cluster shared
448 * resource intensive threads are also not counted as foreign threads since these
449 * threads should not be rebalanced when running on non-preferred clusters.
450 *
451 * This bitmap allows the Edge scheduler to quickly find CPUs running foreign threads
452 * for rebalancing.
453 */
454 static void
processor_state_update_running_foreign(processor_t processor,thread_t thread)455 processor_state_update_running_foreign(processor_t processor, thread_t thread)
456 {
457 cluster_type_t current_processor_type = pset_type_for_id(processor->processor_set->pset_cluster_id);
458 cluster_type_t thread_type = pset_type_for_id(sched_edge_thread_preferred_cluster(thread));
459
460 boolean_t non_rt_thr = (processor->current_pri < BASEPRI_RTQUEUES);
461 boolean_t non_bound_thr = (thread->bound_processor == PROCESSOR_NULL);
462 if (non_rt_thr && non_bound_thr && (current_processor_type != thread_type)) {
463 bit_set(processor->processor_set->cpu_running_foreign, processor->cpu_id);
464 } else {
465 bit_clear(processor->processor_set->cpu_running_foreign, processor->cpu_id);
466 }
467 }
468
469 /*
470 * Cluster shared resource intensive threads
471 *
472 * With the Edge scheduler, each pset maintains a bitmap of processors running
473 * threads that are shared resource intensive. This per-thread property is set
474 * by the performance controller or explicitly via dispatch SPIs. The bitmap
475 * allows the Edge scheduler to calculate the cluster shared resource load on
476 * any given cluster and load balance intensive threads accordingly.
477 */
478 static void
processor_state_update_running_cluster_shared_rsrc(processor_t processor,thread_t thread)479 processor_state_update_running_cluster_shared_rsrc(processor_t processor, thread_t thread)
480 {
481 if (thread_shared_rsrc_policy_get(thread, CLUSTER_SHARED_RSRC_TYPE_RR)) {
482 bit_set(processor->processor_set->cpu_running_cluster_shared_rsrc_thread[CLUSTER_SHARED_RSRC_TYPE_RR], processor->cpu_id);
483 } else {
484 bit_clear(processor->processor_set->cpu_running_cluster_shared_rsrc_thread[CLUSTER_SHARED_RSRC_TYPE_RR], processor->cpu_id);
485 }
486 if (thread_shared_rsrc_policy_get(thread, CLUSTER_SHARED_RSRC_TYPE_NATIVE_FIRST)) {
487 bit_set(processor->processor_set->cpu_running_cluster_shared_rsrc_thread[CLUSTER_SHARED_RSRC_TYPE_NATIVE_FIRST], processor->cpu_id);
488 } else {
489 bit_clear(processor->processor_set->cpu_running_cluster_shared_rsrc_thread[CLUSTER_SHARED_RSRC_TYPE_NATIVE_FIRST], processor->cpu_id);
490 }
491 }
492
493 #endif /* CONFIG_SCHED_EDGE */
494
495 void
processor_state_update_idle(processor_t processor)496 processor_state_update_idle(processor_t processor)
497 {
498 processor->current_pri = IDLEPRI;
499 processor->current_sfi_class = SFI_CLASS_KERNEL;
500 processor->current_recommended_pset_type = PSET_SMP;
501 #if CONFIG_THREAD_GROUPS
502 processor->current_thread_group = NULL;
503 #endif
504 processor->current_perfctl_class = PERFCONTROL_CLASS_IDLE;
505 processor->current_urgency = THREAD_URGENCY_NONE;
506 processor->current_is_NO_SMT = false;
507 processor->current_is_bound = false;
508 processor->current_is_eagerpreempt = false;
509 #if CONFIG_SCHED_EDGE
510 os_atomic_store(&processor->processor_set->cpu_running_buckets[processor->cpu_id], TH_BUCKET_SCHED_MAX, relaxed);
511 bit_clear(processor->processor_set->cpu_running_cluster_shared_rsrc_thread[CLUSTER_SHARED_RSRC_TYPE_RR], processor->cpu_id);
512 bit_clear(processor->processor_set->cpu_running_cluster_shared_rsrc_thread[CLUSTER_SHARED_RSRC_TYPE_NATIVE_FIRST], processor->cpu_id);
513 #endif /* CONFIG_SCHED_EDGE */
514 sched_update_pset_load_average(processor->processor_set, 0);
515 }
516
517 void
processor_state_update_from_thread(processor_t processor,thread_t thread,boolean_t pset_lock_held)518 processor_state_update_from_thread(processor_t processor, thread_t thread, boolean_t pset_lock_held)
519 {
520 processor->current_pri = thread->sched_pri;
521 processor->current_sfi_class = thread->sfi_class;
522 processor->current_recommended_pset_type = recommended_pset_type(thread);
523 #if CONFIG_SCHED_EDGE
524 processor_state_update_running_foreign(processor, thread);
525 processor_state_update_running_cluster_shared_rsrc(processor, thread);
526 /* Since idle and bound threads are not tracked by the edge scheduler, ignore when those threads go on-core */
527 sched_bucket_t bucket = ((thread->state & TH_IDLE) || (thread->bound_processor != PROCESSOR_NULL)) ? TH_BUCKET_SCHED_MAX : thread->th_sched_bucket;
528 os_atomic_store(&processor->processor_set->cpu_running_buckets[processor->cpu_id], bucket, relaxed);
529 #endif /* CONFIG_SCHED_EDGE */
530
531 #if CONFIG_THREAD_GROUPS
532 processor->current_thread_group = thread_group_get(thread);
533 #endif
534 processor->current_perfctl_class = thread_get_perfcontrol_class(thread);
535 processor->current_urgency = thread_get_urgency(thread, NULL, NULL);
536 processor->current_is_NO_SMT = thread_no_smt(thread);
537 processor->current_is_bound = thread->bound_processor != PROCESSOR_NULL;
538 processor->current_is_eagerpreempt = thread_is_eager_preempt(thread);
539 if (pset_lock_held) {
540 /* Only update the pset load average when the pset lock is held */
541 sched_update_pset_load_average(processor->processor_set, 0);
542 }
543 }
544
545 void
processor_state_update_explicit(processor_t processor,int pri,sfi_class_id_t sfi_class,pset_cluster_type_t pset_type,perfcontrol_class_t perfctl_class,thread_urgency_t urgency,__unused sched_bucket_t bucket)546 processor_state_update_explicit(processor_t processor, int pri, sfi_class_id_t sfi_class,
547 pset_cluster_type_t pset_type, perfcontrol_class_t perfctl_class, thread_urgency_t urgency, __unused sched_bucket_t bucket)
548 {
549 processor->current_pri = pri;
550 processor->current_sfi_class = sfi_class;
551 processor->current_recommended_pset_type = pset_type;
552 processor->current_perfctl_class = perfctl_class;
553 processor->current_urgency = urgency;
554 #if CONFIG_SCHED_EDGE
555 os_atomic_store(&processor->processor_set->cpu_running_buckets[processor->cpu_id], bucket, relaxed);
556 bit_clear(processor->processor_set->cpu_running_cluster_shared_rsrc_thread[CLUSTER_SHARED_RSRC_TYPE_RR], processor->cpu_id);
557 bit_clear(processor->processor_set->cpu_running_cluster_shared_rsrc_thread[CLUSTER_SHARED_RSRC_TYPE_NATIVE_FIRST], processor->cpu_id);
558 #endif /* CONFIG_SCHED_EDGE */
559 }
560
561 pset_node_t
pset_node_root(void)562 pset_node_root(void)
563 {
564 return &pset_node0;
565 }
566
567 LCK_GRP_DECLARE(pset_create_grp, "pset_create");
568 LCK_MTX_DECLARE(pset_create_lock, &pset_create_grp);
569
570 processor_set_t
pset_create(pset_node_t node,pset_cluster_type_t pset_type,uint32_t pset_cluster_id,int pset_id)571 pset_create(
572 pset_node_t node,
573 pset_cluster_type_t pset_type,
574 uint32_t pset_cluster_id,
575 int pset_id)
576 {
577 /* some schedulers do not support multiple psets */
578 if (SCHED(multiple_psets_enabled) == FALSE) {
579 return processor_pset(master_processor);
580 }
581
582 processor_set_t *prev, pset = zalloc_permanent_type(struct processor_set);
583
584 if (pset != PROCESSOR_SET_NULL) {
585 pset->pset_cluster_type = pset_type;
586 pset->pset_cluster_id = pset_cluster_id;
587 pset->pset_id = pset_id;
588 pset_init(pset, node);
589
590 lck_spin_lock(&pset_node_lock);
591
592 prev = &node->psets;
593 while (*prev != PROCESSOR_SET_NULL) {
594 prev = &(*prev)->pset_list;
595 }
596
597 *prev = pset;
598
599 lck_spin_unlock(&pset_node_lock);
600 }
601
602 return pset;
603 }
604
605 /*
606 * Find processor set with specified cluster_id.
607 * Returns default_pset if not found.
608 */
609 processor_set_t
pset_find(uint32_t cluster_id,processor_set_t default_pset)610 pset_find(
611 uint32_t cluster_id,
612 processor_set_t default_pset)
613 {
614 lck_spin_lock(&pset_node_lock);
615 pset_node_t node = &pset_node0;
616 processor_set_t pset = NULL;
617
618 do {
619 pset = node->psets;
620 while (pset != NULL) {
621 if (pset->pset_cluster_id == cluster_id) {
622 break;
623 }
624 pset = pset->pset_list;
625 }
626 } while (pset == NULL && (node = node->node_list) != NULL);
627 lck_spin_unlock(&pset_node_lock);
628 if (pset == NULL) {
629 return default_pset;
630 }
631 return pset;
632 }
633
634 /*
635 * Initialize the given processor_set structure.
636 */
637 void
pset_init(processor_set_t pset,pset_node_t node)638 pset_init(
639 processor_set_t pset,
640 pset_node_t node)
641 {
642 pset->online_processor_count = 0;
643 pset->load_average = 0;
644 bzero(&pset->pset_load_average, sizeof(pset->pset_load_average));
645 #if CONFIG_SCHED_EDGE
646 bzero(&pset->pset_runnable_depth, sizeof(pset->pset_runnable_depth));
647 #endif /* CONFIG_SCHED_EDGE */
648 pset->cpu_set_low = pset->cpu_set_hi = 0;
649 pset->cpu_set_count = 0;
650 pset->last_chosen = -1;
651 pset->cpu_bitmask = 0;
652 pset->recommended_bitmask = 0;
653 pset->primary_map = 0;
654 pset->realtime_map = 0;
655 pset->cpu_available_map = 0;
656
657 for (uint i = 0; i < PROCESSOR_STATE_LEN; i++) {
658 pset->cpu_state_map[i] = 0;
659 }
660 pset->pending_AST_URGENT_cpu_mask = 0;
661 pset->pending_AST_PREEMPT_cpu_mask = 0;
662 #if defined(CONFIG_SCHED_DEFERRED_AST)
663 pset->pending_deferred_AST_cpu_mask = 0;
664 #endif
665 pset->pending_spill_cpu_mask = 0;
666 pset->rt_pending_spill_cpu_mask = 0;
667 pset_lock_init(pset);
668 pset->pset_self = IP_NULL;
669 pset->pset_name_self = IP_NULL;
670 pset->pset_list = PROCESSOR_SET_NULL;
671 pset->is_SMT = false;
672 #if CONFIG_SCHED_EDGE
673 bzero(&pset->pset_execution_time, sizeof(pset->pset_execution_time));
674 pset->cpu_running_foreign = 0;
675 for (cluster_shared_rsrc_type_t shared_rsrc_type = CLUSTER_SHARED_RSRC_TYPE_MIN; shared_rsrc_type < CLUSTER_SHARED_RSRC_TYPE_COUNT; shared_rsrc_type++) {
676 pset->cpu_running_cluster_shared_rsrc_thread[shared_rsrc_type] = 0;
677 pset->pset_cluster_shared_rsrc_load[shared_rsrc_type] = 0;
678 }
679 #endif /* CONFIG_SCHED_EDGE */
680
681 /*
682 * No initial preferences or forced migrations, so use the least numbered
683 * available idle core when picking amongst idle cores in a cluster.
684 */
685 pset->perfcontrol_cpu_preferred_bitmask = 0;
686 pset->perfcontrol_cpu_migration_bitmask = 0;
687 pset->cpu_preferred_last_chosen = -1;
688
689 pset->stealable_rt_threads_earliest_deadline = UINT64_MAX;
690
691 if (pset != &pset0) {
692 /*
693 * Scheduler runqueue initialization for non-boot psets.
694 * This initialization for pset0 happens in sched_init().
695 */
696 SCHED(pset_init)(pset);
697 SCHED(rt_init)(pset);
698 }
699
700 /*
701 * Because the pset_node_lock is not taken by every client of the pset_map,
702 * we need to make sure that the initialized pset contents are visible to any
703 * client that loads a non-NULL value from pset_array.
704 */
705 os_atomic_store(&pset_array[pset->pset_id], pset, release);
706
707 lck_spin_lock(&pset_node_lock);
708 bit_set(node->pset_map, pset->pset_id);
709 pset->node = node;
710 lck_spin_unlock(&pset_node_lock);
711 }
712
713 kern_return_t
processor_info_count(processor_flavor_t flavor,mach_msg_type_number_t * count)714 processor_info_count(
715 processor_flavor_t flavor,
716 mach_msg_type_number_t *count)
717 {
718 switch (flavor) {
719 case PROCESSOR_BASIC_INFO:
720 *count = PROCESSOR_BASIC_INFO_COUNT;
721 break;
722
723 case PROCESSOR_CPU_LOAD_INFO:
724 *count = PROCESSOR_CPU_LOAD_INFO_COUNT;
725 break;
726
727 default:
728 return cpu_info_count(flavor, count);
729 }
730
731 return KERN_SUCCESS;
732 }
733
734 void
processor_cpu_load_info(processor_t processor,natural_t ticks[static CPU_STATE_MAX])735 processor_cpu_load_info(processor_t processor,
736 natural_t ticks[static CPU_STATE_MAX])
737 {
738 struct recount_usage usage = { 0 };
739 uint64_t idle_time = 0;
740 recount_processor_usage(&processor->pr_recount, &usage, &idle_time);
741
742 ticks[CPU_STATE_USER] += (uint32_t)(usage.ru_metrics[RCT_LVL_USER].rm_time_mach /
743 hz_tick_interval);
744 ticks[CPU_STATE_SYSTEM] += (uint32_t)(
745 recount_usage_system_time_mach(&usage) / hz_tick_interval);
746 ticks[CPU_STATE_IDLE] += (uint32_t)(idle_time / hz_tick_interval);
747 }
748
749 kern_return_t
processor_info(processor_t processor,processor_flavor_t flavor,host_t * host,processor_info_t info,mach_msg_type_number_t * count)750 processor_info(
751 processor_t processor,
752 processor_flavor_t flavor,
753 host_t *host,
754 processor_info_t info,
755 mach_msg_type_number_t *count)
756 {
757 int cpu_id, state;
758 kern_return_t result;
759
760 if (processor == PROCESSOR_NULL) {
761 return KERN_INVALID_ARGUMENT;
762 }
763
764 cpu_id = processor->cpu_id;
765
766 switch (flavor) {
767 case PROCESSOR_BASIC_INFO:
768 {
769 processor_basic_info_t basic_info;
770
771 if (*count < PROCESSOR_BASIC_INFO_COUNT) {
772 return KERN_FAILURE;
773 }
774
775 basic_info = (processor_basic_info_t) info;
776 basic_info->cpu_type = slot_type(cpu_id);
777 basic_info->cpu_subtype = slot_subtype(cpu_id);
778 state = processor->state;
779 if (((state == PROCESSOR_OFF_LINE || state == PROCESSOR_PENDING_OFFLINE) && !processor->shutdown_temporary)
780 #if defined(__x86_64__)
781 || !processor->is_recommended
782 #endif
783 ) {
784 basic_info->running = FALSE;
785 } else {
786 basic_info->running = TRUE;
787 }
788 basic_info->slot_num = cpu_id;
789 if (processor == master_processor) {
790 basic_info->is_master = TRUE;
791 } else {
792 basic_info->is_master = FALSE;
793 }
794
795 *count = PROCESSOR_BASIC_INFO_COUNT;
796 *host = &realhost;
797
798 return KERN_SUCCESS;
799 }
800
801 case PROCESSOR_CPU_LOAD_INFO:
802 {
803 processor_cpu_load_info_t cpu_load_info;
804
805 if (*count < PROCESSOR_CPU_LOAD_INFO_COUNT) {
806 return KERN_FAILURE;
807 }
808
809 cpu_load_info = (processor_cpu_load_info_t) info;
810
811 cpu_load_info->cpu_ticks[CPU_STATE_SYSTEM] = 0;
812 cpu_load_info->cpu_ticks[CPU_STATE_USER] = 0;
813 cpu_load_info->cpu_ticks[CPU_STATE_IDLE] = 0;
814 processor_cpu_load_info(processor, cpu_load_info->cpu_ticks);
815 cpu_load_info->cpu_ticks[CPU_STATE_NICE] = 0;
816
817 *count = PROCESSOR_CPU_LOAD_INFO_COUNT;
818 *host = &realhost;
819
820 return KERN_SUCCESS;
821 }
822
823 default:
824 result = cpu_info(flavor, cpu_id, info, count);
825 if (result == KERN_SUCCESS) {
826 *host = &realhost;
827 }
828
829 return result;
830 }
831 }
832
833 /*
834 * Now that we're enforcing all CPUs actually boot, we may need a way to
835 * relax the timeout.
836 */
837 TUNABLE(uint32_t, cpu_boot_timeout_secs, "cpu_boot_timeout_secs", 1); /* seconds, default to 1 second */
838
839 static const char *
840 processor_start_panic_strings[] = {
841 [PROCESSOR_FIRST_BOOT] = "boot for the first time",
842 [PROCESSOR_BEFORE_ENTERING_SLEEP] = "come online while entering system sleep",
843 [PROCESSOR_WAKE_FROM_SLEEP] = "come online after returning from system sleep",
844 [PROCESSOR_CLUSTER_POWERDOWN_SUSPEND] = "come online while disabling cluster powerdown",
845 [PROCESSOR_CLUSTER_POWERDOWN_RESUME] = "come online before enabling cluster powerdown",
846 [PROCESSOR_POWERED_CORES_CHANGE] = "come online during dynamic cluster power state change",
847 };
848
849 void
processor_wait_for_start(processor_t processor,processor_start_kind_t start_kind)850 processor_wait_for_start(processor_t processor, processor_start_kind_t start_kind)
851 {
852 if (!processor->processor_booted) {
853 panic("processor_boot() missing for cpu %d", processor->cpu_id);
854 }
855
856 uint32_t boot_timeout_extended = cpu_boot_timeout_secs *
857 debug_cpu_performance_degradation_factor;
858
859 spl_t s = splsched();
860 simple_lock(&processor_start_state_lock, LCK_GRP_NULL);
861 while (processor->processor_instartup) {
862 assert_wait_timeout((event_t)&processor->processor_instartup,
863 THREAD_UNINT, boot_timeout_extended, NSEC_PER_SEC);
864 simple_unlock(&processor_start_state_lock);
865 splx(s);
866
867 wait_result_t wait_result = thread_block(THREAD_CONTINUE_NULL);
868 if (wait_result == THREAD_TIMED_OUT) {
869 panic("cpu %d failed to %s, waited %d seconds\n",
870 processor->cpu_id,
871 processor_start_panic_strings[start_kind],
872 boot_timeout_extended);
873 }
874
875 s = splsched();
876 simple_lock(&processor_start_state_lock, LCK_GRP_NULL);
877 }
878
879 if (processor->processor_inshutdown) {
880 panic("%s>cpu %d still in shutdown",
881 __func__, processor->cpu_id);
882 }
883
884 simple_unlock(&processor_start_state_lock);
885
886 simple_lock(&sched_available_cores_lock, LCK_GRP_NULL);
887
888 if (!processor->processor_online) {
889 panic("%s>cpu %d not online",
890 __func__, processor->cpu_id);
891 }
892
893 if (processor->processor_offline_state == PROCESSOR_OFFLINE_STARTED_NOT_WAITED) {
894 processor_update_offline_state_locked(processor, PROCESSOR_OFFLINE_RUNNING);
895 } else {
896 assert(processor->processor_offline_state == PROCESSOR_OFFLINE_RUNNING);
897 }
898
899 simple_unlock(&sched_available_cores_lock);
900 splx(s);
901 }
902
903 LCK_GRP_DECLARE(processor_updown_grp, "processor_updown");
904 LCK_MTX_DECLARE(processor_updown_lock, &processor_updown_grp);
905
906 static void
processor_dostartup(processor_t processor,bool first_boot)907 processor_dostartup(
908 processor_t processor,
909 bool first_boot)
910 {
911 if (!processor->processor_booted && !first_boot) {
912 panic("processor %d not booted", processor->cpu_id);
913 }
914
915 lck_mtx_assert(&cluster_powerdown_lock, LCK_MTX_ASSERT_OWNED);
916 lck_mtx_assert(&processor_updown_lock, LCK_MTX_ASSERT_OWNED);
917
918 processor_set_t pset = processor->processor_set;
919
920 assert(processor->processor_self);
921
922 spl_t s = splsched();
923
924 simple_lock(&processor_start_state_lock, LCK_GRP_NULL);
925 assert(processor->processor_inshutdown || first_boot);
926 processor->processor_inshutdown = false;
927 assert(processor->processor_instartup == false);
928 processor->processor_instartup = true;
929 simple_unlock(&processor_start_state_lock);
930
931 simple_lock(&sched_available_cores_lock, LCK_GRP_NULL);
932
933 pset_lock(pset);
934
935 if (first_boot) {
936 assert(processor->processor_offline_state == PROCESSOR_OFFLINE_NOT_BOOTED);
937 } else {
938 assert(processor->processor_offline_state == PROCESSOR_OFFLINE_FULLY_OFFLINE);
939 }
940
941 processor_update_offline_state_locked(processor, PROCESSOR_OFFLINE_STARTING);
942
943 assert(processor->state == PROCESSOR_OFF_LINE);
944
945 pset_update_processor_state(pset, processor, PROCESSOR_START);
946 pset_unlock(pset);
947
948 simple_unlock(&sched_available_cores_lock);
949
950 splx(s);
951
952 ml_cpu_power_enable(processor->cpu_id);
953 ml_cpu_begin_state_transition(processor->cpu_id);
954 ml_broadcast_cpu_event(CPU_BOOT_REQUESTED, processor->cpu_id);
955
956 cpu_start(processor->cpu_id);
957
958 s = splsched();
959 simple_lock(&sched_available_cores_lock, LCK_GRP_NULL);
960
961 if (processor->processor_offline_state == PROCESSOR_OFFLINE_STARTING) {
962 processor_update_offline_state_locked(processor, PROCESSOR_OFFLINE_STARTED_NOT_RUNNING);
963 } else {
964 assert(processor->processor_offline_state == PROCESSOR_OFFLINE_STARTED_NOT_WAITED);
965 }
966
967 simple_unlock(&sched_available_cores_lock);
968 splx(s);
969
970 ml_cpu_end_state_transition(processor->cpu_id);
971 /*
972 * Note: Because the actual wait-for-start happens sometime later,
973 * this races with processor_up calling CPU_BOOTED.
974 * To fix that, this should happen after the first wait for start
975 * confirms the CPU has booted.
976 */
977 ml_broadcast_cpu_event(CPU_ACTIVE, processor->cpu_id);
978 }
979
980 void
processor_exit_reason(processor_t processor,processor_reason_t reason,bool is_system_sleep)981 processor_exit_reason(processor_t processor, processor_reason_t reason, bool is_system_sleep)
982 {
983 assert(processor);
984 assert(processor->processor_set);
985
986 lck_mtx_lock(&processor_updown_lock);
987
988 if (sched_is_in_sleep()) {
989 assert(reason == REASON_SYSTEM);
990 }
991
992 assert((processor != master_processor) || (reason == REASON_SYSTEM) || support_bootcpu_shutdown);
993
994 processor->last_shutdown_reason = reason;
995
996 bool is_final_system_sleep = is_system_sleep && (processor == master_processor);
997
998 processor_doshutdown(processor, is_final_system_sleep);
999
1000 lck_mtx_unlock(&processor_updown_lock);
1001 }
1002
1003 /*
1004 * Called `processor_exit` in Unsupported KPI.
1005 * AppleARMCPU and AppleACPIPlatform call this in response to haltCPU().
1006 *
1007 * Behavior change: on both platforms, now xnu does the processor_sleep,
1008 * and ignores processor_exit calls from kexts.
1009 */
1010 kern_return_t
processor_exit_from_kext(__unused processor_t processor)1011 processor_exit_from_kext(
1012 __unused processor_t processor)
1013 {
1014 /* This is a no-op now. */
1015 return KERN_FAILURE;
1016 }
1017
1018 void
processor_sleep(processor_t processor)1019 processor_sleep(
1020 processor_t processor)
1021 {
1022 lck_mtx_assert(&cluster_powerdown_lock, LCK_MTX_ASSERT_OWNED);
1023
1024 processor_exit_reason(processor, REASON_SYSTEM, true);
1025 }
1026
1027 kern_return_t
processor_exit_from_user(processor_t processor)1028 processor_exit_from_user(
1029 processor_t processor)
1030 {
1031 if (processor == PROCESSOR_NULL) {
1032 return KERN_INVALID_ARGUMENT;
1033 }
1034
1035 kern_return_t result;
1036
1037 lck_mtx_lock(&cluster_powerdown_lock);
1038
1039 result = sched_processor_exit_user(processor);
1040
1041 lck_mtx_unlock(&cluster_powerdown_lock);
1042
1043 return result;
1044 }
1045
1046 void
processor_start_reason(processor_t processor,processor_reason_t reason)1047 processor_start_reason(processor_t processor, processor_reason_t reason)
1048 {
1049 lck_mtx_lock(&processor_updown_lock);
1050
1051 assert(processor);
1052 assert(processor->processor_set);
1053 assert(processor->processor_booted);
1054
1055 if (sched_is_in_sleep()) {
1056 assert(reason == REASON_SYSTEM);
1057 }
1058
1059 processor->last_startup_reason = reason;
1060
1061 processor_dostartup(processor, false);
1062
1063 lck_mtx_unlock(&processor_updown_lock);
1064 }
1065
1066 /*
1067 * Called `processor_start` in Unsupported KPI.
1068 * AppleARMCPU calls this to boot processors.
1069 * AppleACPIPlatform expects ml_processor_register to call processor_boot.
1070 *
1071 * Behavior change: now ml_processor_register also boots CPUs on ARM, and xnu
1072 * ignores processor_start calls from kexts.
1073 */
1074 kern_return_t
processor_start_from_kext(__unused processor_t processor)1075 processor_start_from_kext(
1076 __unused processor_t processor)
1077 {
1078 /* This is a no-op now. */
1079 return KERN_FAILURE;
1080 }
1081
1082 kern_return_t
processor_start_from_user(processor_t processor)1083 processor_start_from_user(
1084 processor_t processor)
1085 {
1086 if (processor == PROCESSOR_NULL) {
1087 return KERN_INVALID_ARGUMENT;
1088 }
1089
1090 kern_return_t result;
1091
1092 lck_mtx_lock(&cluster_powerdown_lock);
1093
1094 result = sched_processor_start_user(processor);
1095
1096 lck_mtx_unlock(&cluster_powerdown_lock);
1097
1098 return result;
1099 }
1100
1101 /*
1102 * Boot up a processor for the first time.
1103 *
1104 * This will also be called against the main processor during system boot,
1105 * even though it's already running.
1106 */
1107 void
processor_boot(processor_t processor)1108 processor_boot(
1109 processor_t processor)
1110 {
1111 lck_mtx_lock(&cluster_powerdown_lock);
1112 lck_mtx_lock(&processor_updown_lock);
1113
1114 assert(!sched_is_in_sleep());
1115 assert(!sched_is_cpu_init_completed());
1116
1117 if (processor->processor_booted) {
1118 panic("processor %d already booted", processor->cpu_id);
1119 }
1120
1121 if (processor == master_processor) {
1122 assert(processor->processor_offline_state == PROCESSOR_OFFLINE_STARTED_NOT_WAITED);
1123 } else {
1124 assert(processor->processor_offline_state == PROCESSOR_OFFLINE_NOT_BOOTED);
1125 }
1126
1127 /*
1128 * Create the idle processor thread.
1129 */
1130 if (processor->idle_thread == THREAD_NULL) {
1131 idle_thread_create(processor, processor_start_thread);
1132 }
1133
1134 if (processor->processor_self == IP_NULL) {
1135 ipc_processor_init(processor);
1136 }
1137
1138 if (processor == master_processor) {
1139 processor->last_startup_reason = REASON_SYSTEM;
1140
1141 ml_cpu_power_enable(processor->cpu_id);
1142
1143 processor_t prev = thread_bind(processor);
1144 thread_block(THREAD_CONTINUE_NULL);
1145
1146 cpu_start(processor->cpu_id);
1147
1148 assert(processor->state == PROCESSOR_RUNNING);
1149 processor_update_offline_state(processor, PROCESSOR_OFFLINE_RUNNING);
1150
1151 thread_bind(prev);
1152 } else {
1153 processor->last_startup_reason = REASON_SYSTEM;
1154
1155 /*
1156 * We don't wait for startup to finish, so all CPUs can start
1157 * in parallel.
1158 */
1159 processor_dostartup(processor, true);
1160 }
1161
1162 processor->processor_booted = true;
1163
1164 lck_mtx_unlock(&processor_updown_lock);
1165 lck_mtx_unlock(&cluster_powerdown_lock);
1166 }
1167
1168 /*
1169 * Wake a previously booted processor from a temporarily powered off state.
1170 */
1171 void
processor_wake(processor_t processor)1172 processor_wake(
1173 processor_t processor)
1174 {
1175 lck_mtx_assert(&cluster_powerdown_lock, LCK_MTX_ASSERT_OWNED);
1176
1177 assert(processor->processor_booted);
1178 processor_start_reason(processor, REASON_SYSTEM);
1179 }
1180
1181 kern_return_t
enable_smt_processors(bool enable)1182 enable_smt_processors(bool enable)
1183 {
1184 if (machine_info.logical_cpu_max == machine_info.physical_cpu_max) {
1185 /* Not an SMT system */
1186 return KERN_INVALID_ARGUMENT;
1187 }
1188
1189 int ncpus = machine_info.logical_cpu_max;
1190
1191 for (int i = 1; i < ncpus; i++) {
1192 processor_t processor = processor_array[i];
1193
1194 if (processor->processor_primary != processor) {
1195 if (enable) {
1196 processor_start_from_user(processor);
1197 } else { /* Disable */
1198 processor_exit_from_user(processor);
1199 }
1200 }
1201 }
1202
1203 #define BSD_HOST 1
1204 host_basic_info_data_t hinfo;
1205 mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
1206 kern_return_t kret = host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
1207 if (kret != KERN_SUCCESS) {
1208 return kret;
1209 }
1210
1211 if (enable && (hinfo.logical_cpu != hinfo.logical_cpu_max)) {
1212 return KERN_FAILURE;
1213 }
1214
1215 if (!enable && (hinfo.logical_cpu != hinfo.physical_cpu)) {
1216 return KERN_FAILURE;
1217 }
1218
1219 return KERN_SUCCESS;
1220 }
1221
1222 bool
processor_should_kprintf(processor_t processor,bool starting)1223 processor_should_kprintf(processor_t processor, bool starting)
1224 {
1225 processor_reason_t reason = starting ? processor->last_startup_reason : processor->last_shutdown_reason;
1226
1227 return reason != REASON_CLPC_SYSTEM;
1228 }
1229
1230 kern_return_t
processor_control(processor_t processor,processor_info_t info,mach_msg_type_number_t count)1231 processor_control(
1232 processor_t processor,
1233 processor_info_t info,
1234 mach_msg_type_number_t count)
1235 {
1236 if (processor == PROCESSOR_NULL) {
1237 return KERN_INVALID_ARGUMENT;
1238 }
1239
1240 return cpu_control(processor->cpu_id, info, count);
1241 }
1242
1243 kern_return_t
processor_get_assignment(processor_t processor,processor_set_t * pset)1244 processor_get_assignment(
1245 processor_t processor,
1246 processor_set_t *pset)
1247 {
1248 int state;
1249
1250 if (processor == PROCESSOR_NULL) {
1251 return KERN_INVALID_ARGUMENT;
1252 }
1253
1254 state = processor->state;
1255 if (state == PROCESSOR_OFF_LINE || state == PROCESSOR_PENDING_OFFLINE) {
1256 return KERN_FAILURE;
1257 }
1258
1259 *pset = &pset0;
1260
1261 return KERN_SUCCESS;
1262 }
1263
1264 kern_return_t
processor_set_info(processor_set_t pset,int flavor,host_t * host,processor_set_info_t info,mach_msg_type_number_t * count)1265 processor_set_info(
1266 processor_set_t pset,
1267 int flavor,
1268 host_t *host,
1269 processor_set_info_t info,
1270 mach_msg_type_number_t *count)
1271 {
1272 if (pset == PROCESSOR_SET_NULL) {
1273 return KERN_INVALID_ARGUMENT;
1274 }
1275
1276 if (flavor == PROCESSOR_SET_BASIC_INFO) {
1277 processor_set_basic_info_t basic_info;
1278
1279 if (*count < PROCESSOR_SET_BASIC_INFO_COUNT) {
1280 return KERN_FAILURE;
1281 }
1282
1283 basic_info = (processor_set_basic_info_t) info;
1284 #if defined(__x86_64__)
1285 basic_info->processor_count = processor_avail_count_user;
1286 #else
1287 basic_info->processor_count = processor_avail_count;
1288 #endif
1289 basic_info->default_policy = POLICY_TIMESHARE;
1290
1291 *count = PROCESSOR_SET_BASIC_INFO_COUNT;
1292 *host = &realhost;
1293 return KERN_SUCCESS;
1294 } else if (flavor == PROCESSOR_SET_TIMESHARE_DEFAULT) {
1295 policy_timeshare_base_t ts_base;
1296
1297 if (*count < POLICY_TIMESHARE_BASE_COUNT) {
1298 return KERN_FAILURE;
1299 }
1300
1301 ts_base = (policy_timeshare_base_t) info;
1302 ts_base->base_priority = BASEPRI_DEFAULT;
1303
1304 *count = POLICY_TIMESHARE_BASE_COUNT;
1305 *host = &realhost;
1306 return KERN_SUCCESS;
1307 } else if (flavor == PROCESSOR_SET_FIFO_DEFAULT) {
1308 policy_fifo_base_t fifo_base;
1309
1310 if (*count < POLICY_FIFO_BASE_COUNT) {
1311 return KERN_FAILURE;
1312 }
1313
1314 fifo_base = (policy_fifo_base_t) info;
1315 fifo_base->base_priority = BASEPRI_DEFAULT;
1316
1317 *count = POLICY_FIFO_BASE_COUNT;
1318 *host = &realhost;
1319 return KERN_SUCCESS;
1320 } else if (flavor == PROCESSOR_SET_RR_DEFAULT) {
1321 policy_rr_base_t rr_base;
1322
1323 if (*count < POLICY_RR_BASE_COUNT) {
1324 return KERN_FAILURE;
1325 }
1326
1327 rr_base = (policy_rr_base_t) info;
1328 rr_base->base_priority = BASEPRI_DEFAULT;
1329 rr_base->quantum = 1;
1330
1331 *count = POLICY_RR_BASE_COUNT;
1332 *host = &realhost;
1333 return KERN_SUCCESS;
1334 } else if (flavor == PROCESSOR_SET_TIMESHARE_LIMITS) {
1335 policy_timeshare_limit_t ts_limit;
1336
1337 if (*count < POLICY_TIMESHARE_LIMIT_COUNT) {
1338 return KERN_FAILURE;
1339 }
1340
1341 ts_limit = (policy_timeshare_limit_t) info;
1342 ts_limit->max_priority = MAXPRI_KERNEL;
1343
1344 *count = POLICY_TIMESHARE_LIMIT_COUNT;
1345 *host = &realhost;
1346 return KERN_SUCCESS;
1347 } else if (flavor == PROCESSOR_SET_FIFO_LIMITS) {
1348 policy_fifo_limit_t fifo_limit;
1349
1350 if (*count < POLICY_FIFO_LIMIT_COUNT) {
1351 return KERN_FAILURE;
1352 }
1353
1354 fifo_limit = (policy_fifo_limit_t) info;
1355 fifo_limit->max_priority = MAXPRI_KERNEL;
1356
1357 *count = POLICY_FIFO_LIMIT_COUNT;
1358 *host = &realhost;
1359 return KERN_SUCCESS;
1360 } else if (flavor == PROCESSOR_SET_RR_LIMITS) {
1361 policy_rr_limit_t rr_limit;
1362
1363 if (*count < POLICY_RR_LIMIT_COUNT) {
1364 return KERN_FAILURE;
1365 }
1366
1367 rr_limit = (policy_rr_limit_t) info;
1368 rr_limit->max_priority = MAXPRI_KERNEL;
1369
1370 *count = POLICY_RR_LIMIT_COUNT;
1371 *host = &realhost;
1372 return KERN_SUCCESS;
1373 } else if (flavor == PROCESSOR_SET_ENABLED_POLICIES) {
1374 int *enabled;
1375
1376 if (*count < (sizeof(*enabled) / sizeof(int))) {
1377 return KERN_FAILURE;
1378 }
1379
1380 enabled = (int *) info;
1381 *enabled = POLICY_TIMESHARE | POLICY_RR | POLICY_FIFO;
1382
1383 *count = sizeof(*enabled) / sizeof(int);
1384 *host = &realhost;
1385 return KERN_SUCCESS;
1386 }
1387
1388
1389 *host = HOST_NULL;
1390 return KERN_INVALID_ARGUMENT;
1391 }
1392
1393 /*
1394 * processor_set_statistics
1395 *
1396 * Returns scheduling statistics for a processor set.
1397 */
1398 kern_return_t
processor_set_statistics(processor_set_t pset,int flavor,processor_set_info_t info,mach_msg_type_number_t * count)1399 processor_set_statistics(
1400 processor_set_t pset,
1401 int flavor,
1402 processor_set_info_t info,
1403 mach_msg_type_number_t *count)
1404 {
1405 if (pset == PROCESSOR_SET_NULL || pset != &pset0) {
1406 return KERN_INVALID_PROCESSOR_SET;
1407 }
1408
1409 if (flavor == PROCESSOR_SET_LOAD_INFO) {
1410 processor_set_load_info_t load_info;
1411
1412 if (*count < PROCESSOR_SET_LOAD_INFO_COUNT) {
1413 return KERN_FAILURE;
1414 }
1415
1416 load_info = (processor_set_load_info_t) info;
1417
1418 load_info->mach_factor = sched_mach_factor;
1419 load_info->load_average = sched_load_average;
1420
1421 load_info->task_count = tasks_count;
1422 load_info->thread_count = threads_count;
1423
1424 *count = PROCESSOR_SET_LOAD_INFO_COUNT;
1425 return KERN_SUCCESS;
1426 }
1427
1428 return KERN_INVALID_ARGUMENT;
1429 }
1430
1431 /*
1432 * processor_set_things:
1433 *
1434 * Common internals for processor_set_{threads,tasks}
1435 */
1436 static kern_return_t
processor_set_things(processor_set_t pset,mach_port_array_t * thing_list,mach_msg_type_number_t * countp,int type,mach_task_flavor_t flavor)1437 processor_set_things(
1438 processor_set_t pset,
1439 mach_port_array_t *thing_list,
1440 mach_msg_type_number_t *countp,
1441 int type,
1442 mach_task_flavor_t flavor)
1443 {
1444 unsigned int i;
1445 task_t task;
1446 thread_t thread;
1447
1448 mach_port_array_t task_addr;
1449 task_t *task_list;
1450 vm_size_t actual_tasks, task_count_cur, task_count_needed;
1451
1452 mach_port_array_t thread_addr;
1453 thread_t *thread_list;
1454 vm_size_t actual_threads, thread_count_cur, thread_count_needed;
1455
1456 mach_port_array_t addr, newaddr;
1457 vm_size_t count, count_needed;
1458
1459 if (pset == PROCESSOR_SET_NULL || pset != &pset0) {
1460 return KERN_INVALID_ARGUMENT;
1461 }
1462
1463 task_count_cur = 0;
1464 task_count_needed = 0;
1465 task_list = NULL;
1466 task_addr = NULL;
1467 actual_tasks = 0;
1468
1469 thread_count_cur = 0;
1470 thread_count_needed = 0;
1471 thread_list = NULL;
1472 thread_addr = NULL;
1473 actual_threads = 0;
1474
1475 for (;;) {
1476 lck_mtx_lock(&tasks_threads_lock);
1477
1478 /* do we have the memory we need? */
1479 if (type == PSET_THING_THREAD) {
1480 thread_count_needed = threads_count;
1481 }
1482 #if !CONFIG_MACF
1483 else
1484 #endif
1485 task_count_needed = tasks_count;
1486
1487 if (task_count_needed <= task_count_cur &&
1488 thread_count_needed <= thread_count_cur) {
1489 break;
1490 }
1491
1492 /* unlock and allocate more memory */
1493 lck_mtx_unlock(&tasks_threads_lock);
1494
1495 /* grow task array */
1496 if (task_count_needed > task_count_cur) {
1497 mach_port_array_free(task_addr, task_count_cur);
1498 assert(task_count_needed > 0);
1499 task_count_cur = task_count_needed;
1500
1501 task_addr = mach_port_array_alloc(task_count_cur,
1502 Z_WAITOK | Z_ZERO);
1503 if (task_addr == NULL) {
1504 mach_port_array_free(thread_addr, thread_count_cur);
1505 return KERN_RESOURCE_SHORTAGE;
1506 }
1507 task_list = (task_t *)task_addr;
1508 }
1509
1510 /* grow thread array */
1511 if (thread_count_needed > thread_count_cur) {
1512 mach_port_array_free(thread_addr, thread_count_cur);
1513 assert(thread_count_needed > 0);
1514 thread_count_cur = thread_count_needed;
1515
1516 thread_addr = mach_port_array_alloc(thread_count_cur,
1517 Z_WAITOK | Z_ZERO);
1518 if (thread_addr == NULL) {
1519 mach_port_array_free(task_addr, task_count_cur);
1520 return KERN_RESOURCE_SHORTAGE;
1521 }
1522 thread_list = (thread_t *)thread_addr;
1523 }
1524 }
1525
1526 /* OK, have memory and the list locked */
1527
1528 /* If we need it, get the thread list */
1529 if (type == PSET_THING_THREAD) {
1530 queue_iterate(&threads, thread, thread_t, threads) {
1531 task = get_threadtask(thread);
1532 #if defined(SECURE_KERNEL)
1533 if (task == kernel_task) {
1534 /* skip threads belonging to kernel_task */
1535 continue;
1536 }
1537 #endif
1538 if (!task->ipc_active || task_is_exec_copy(task)) {
1539 /* skip threads in inactive tasks (in the middle of exec/fork/spawn) */
1540 continue;
1541 }
1542
1543 thread_reference(thread);
1544 thread_list[actual_threads++] = thread;
1545 }
1546 }
1547 #if !CONFIG_MACF
1548 else
1549 #endif
1550 {
1551 /* get a list of the tasks */
1552 queue_iterate(&tasks, task, task_t, tasks) {
1553 #if defined(SECURE_KERNEL)
1554 if (task == kernel_task) {
1555 /* skip kernel_task */
1556 continue;
1557 }
1558 #endif
1559 if (!task->ipc_active || task_is_exec_copy(task)) {
1560 /* skip inactive tasks (in the middle of exec/fork/spawn) */
1561 continue;
1562 }
1563
1564 task_reference(task);
1565 task_list[actual_tasks++] = task;
1566 }
1567 }
1568
1569 lck_mtx_unlock(&tasks_threads_lock);
1570
1571 #if CONFIG_MACF
1572 unsigned int j, used;
1573
1574 /* for each task, make sure we are allowed to examine it */
1575 for (i = used = 0; i < actual_tasks; i++) {
1576 if (mac_task_check_expose_task(task_list[i], flavor)) {
1577 task_deallocate(task_list[i]);
1578 continue;
1579 }
1580 task_list[used++] = task_list[i];
1581 }
1582 actual_tasks = used;
1583 task_count_needed = actual_tasks;
1584
1585 if (type == PSET_THING_THREAD) {
1586 /* for each thread (if any), make sure it's task is in the allowed list */
1587 for (i = used = 0; i < actual_threads; i++) {
1588 boolean_t found_task = FALSE;
1589
1590 task = get_threadtask(thread_list[i]);
1591 for (j = 0; j < actual_tasks; j++) {
1592 if (task_list[j] == task) {
1593 found_task = TRUE;
1594 break;
1595 }
1596 }
1597 if (found_task) {
1598 thread_list[used++] = thread_list[i];
1599 } else {
1600 thread_deallocate(thread_list[i]);
1601 }
1602 }
1603 actual_threads = used;
1604 thread_count_needed = actual_threads;
1605
1606 /* done with the task list */
1607 for (i = 0; i < actual_tasks; i++) {
1608 task_deallocate(task_list[i]);
1609 }
1610 mach_port_array_free(task_addr, task_count_cur);
1611 task_list = NULL;
1612 task_count_cur = 0;
1613 actual_tasks = 0;
1614 }
1615 #endif
1616
1617 if (type == PSET_THING_THREAD) {
1618 if (actual_threads == 0) {
1619 /* no threads available to return */
1620 assert(task_count_cur == 0);
1621 mach_port_array_free(thread_addr, thread_count_cur);
1622 thread_list = NULL;
1623 *thing_list = NULL;
1624 *countp = 0;
1625 return KERN_SUCCESS;
1626 }
1627 count_needed = actual_threads;
1628 count = thread_count_cur;
1629 addr = thread_addr;
1630 } else {
1631 if (actual_tasks == 0) {
1632 /* no tasks available to return */
1633 assert(thread_count_cur == 0);
1634 mach_port_array_free(task_addr, task_count_cur);
1635 *thing_list = NULL;
1636 *countp = 0;
1637 return KERN_SUCCESS;
1638 }
1639 count_needed = actual_tasks;
1640 count = task_count_cur;
1641 addr = task_addr;
1642 }
1643
1644 /* if we allocated too much, must copy */
1645 if (count_needed < count) {
1646 newaddr = mach_port_array_alloc(count_needed, Z_WAITOK | Z_ZERO);
1647 if (newaddr == NULL) {
1648 for (i = 0; i < actual_tasks; i++) {
1649 if (type == PSET_THING_THREAD) {
1650 thread_deallocate(thread_list[i]);
1651 } else {
1652 task_deallocate(task_list[i]);
1653 }
1654 }
1655 mach_port_array_free(addr, count);
1656 return KERN_RESOURCE_SHORTAGE;
1657 }
1658
1659 bcopy(addr, newaddr, count_needed * sizeof(void *));
1660 mach_port_array_free(addr, count);
1661
1662 addr = newaddr;
1663 count = count_needed;
1664 }
1665
1666 *thing_list = addr;
1667 *countp = (mach_msg_type_number_t)count;
1668
1669 return KERN_SUCCESS;
1670 }
1671
1672 /*
1673 * processor_set_tasks:
1674 *
1675 * List all tasks in the processor set.
1676 */
1677 static kern_return_t
processor_set_tasks_internal(processor_set_t pset,task_array_t * task_list,mach_msg_type_number_t * count,mach_task_flavor_t flavor)1678 processor_set_tasks_internal(
1679 processor_set_t pset,
1680 task_array_t *task_list,
1681 mach_msg_type_number_t *count,
1682 mach_task_flavor_t flavor)
1683 {
1684 kern_return_t ret;
1685
1686 ret = processor_set_things(pset, task_list, count, PSET_THING_TASK, flavor);
1687 if (ret != KERN_SUCCESS) {
1688 return ret;
1689 }
1690
1691 /* do the conversion that Mig should handle */
1692 convert_task_array_to_ports(*task_list, *count, flavor);
1693 return KERN_SUCCESS;
1694 }
1695
1696 kern_return_t
processor_set_tasks(processor_set_t pset,task_array_t * task_list,mach_msg_type_number_t * count)1697 processor_set_tasks(
1698 processor_set_t pset,
1699 task_array_t *task_list,
1700 mach_msg_type_number_t *count)
1701 {
1702 return processor_set_tasks_internal(pset, task_list, count, TASK_FLAVOR_CONTROL);
1703 }
1704
1705 /*
1706 * processor_set_tasks_with_flavor:
1707 *
1708 * Based on flavor, return task/inspect/read port to all tasks in the processor set.
1709 */
1710 kern_return_t
processor_set_tasks_with_flavor(processor_set_t pset,mach_task_flavor_t flavor,task_array_t * task_list,mach_msg_type_number_t * count)1711 processor_set_tasks_with_flavor(
1712 processor_set_t pset,
1713 mach_task_flavor_t flavor,
1714 task_array_t *task_list,
1715 mach_msg_type_number_t *count)
1716 {
1717 switch (flavor) {
1718 case TASK_FLAVOR_CONTROL:
1719 case TASK_FLAVOR_READ:
1720 case TASK_FLAVOR_INSPECT:
1721 case TASK_FLAVOR_NAME:
1722 return processor_set_tasks_internal(pset, task_list, count, flavor);
1723 default:
1724 return KERN_INVALID_ARGUMENT;
1725 }
1726 }
1727
1728 /*
1729 * processor_set_threads:
1730 *
1731 * List all threads in the processor set.
1732 */
1733 #if defined(SECURE_KERNEL)
1734 kern_return_t
processor_set_threads(__unused processor_set_t pset,__unused thread_act_array_t * thread_list,__unused mach_msg_type_number_t * count)1735 processor_set_threads(
1736 __unused processor_set_t pset,
1737 __unused thread_act_array_t *thread_list,
1738 __unused mach_msg_type_number_t *count)
1739 {
1740 return KERN_FAILURE;
1741 }
1742 #elif !defined(XNU_TARGET_OS_OSX)
1743 kern_return_t
processor_set_threads(__unused processor_set_t pset,__unused thread_act_array_t * thread_list,__unused mach_msg_type_number_t * count)1744 processor_set_threads(
1745 __unused processor_set_t pset,
1746 __unused thread_act_array_t *thread_list,
1747 __unused mach_msg_type_number_t *count)
1748 {
1749 return KERN_NOT_SUPPORTED;
1750 }
1751 #else
1752 kern_return_t
processor_set_threads(processor_set_t pset,thread_act_array_t * thread_list,mach_msg_type_number_t * count)1753 processor_set_threads(
1754 processor_set_t pset,
1755 thread_act_array_t *thread_list,
1756 mach_msg_type_number_t *count)
1757 {
1758 kern_return_t ret;
1759
1760 ret = processor_set_things(pset, thread_list, count,
1761 PSET_THING_THREAD, TASK_FLAVOR_CONTROL);
1762 if (ret != KERN_SUCCESS) {
1763 return ret;
1764 }
1765
1766 /* do the conversion that Mig should handle */
1767 convert_thread_array_to_ports(*thread_list, *count, TASK_FLAVOR_CONTROL);
1768 return KERN_SUCCESS;
1769 }
1770 #endif
1771
1772 pset_cluster_type_t
recommended_pset_type(thread_t thread)1773 recommended_pset_type(thread_t thread)
1774 {
1775 /* Only used by the AMP scheduler policy */
1776 #if CONFIG_THREAD_GROUPS && __AMP__ && !CONFIG_SCHED_EDGE
1777 if (thread == THREAD_NULL) {
1778 return PSET_AMP_E;
1779 }
1780
1781 #if DEVELOPMENT || DEBUG
1782 extern bool system_ecore_only;
1783 extern int enable_task_set_cluster_type;
1784 task_t task = get_threadtask(thread);
1785 if (enable_task_set_cluster_type && (task->t_flags & TF_USE_PSET_HINT_CLUSTER_TYPE)) {
1786 processor_set_t pset_hint = task->pset_hint;
1787 if (pset_hint) {
1788 return pset_hint->pset_cluster_type;
1789 }
1790 }
1791
1792 if (system_ecore_only) {
1793 return PSET_AMP_E;
1794 }
1795 #endif
1796
1797 if (thread->th_bound_cluster_id != THREAD_BOUND_CLUSTER_NONE) {
1798 return pset_array[thread->th_bound_cluster_id]->pset_cluster_type;
1799 }
1800
1801 if (thread->base_pri <= MAXPRI_THROTTLE) {
1802 if (os_atomic_load(&sched_perfctl_policy_bg, relaxed) != SCHED_PERFCTL_POLICY_FOLLOW_GROUP) {
1803 return PSET_AMP_E;
1804 }
1805 } else if (thread->base_pri <= BASEPRI_UTILITY) {
1806 if (os_atomic_load(&sched_perfctl_policy_util, relaxed) != SCHED_PERFCTL_POLICY_FOLLOW_GROUP) {
1807 return PSET_AMP_E;
1808 }
1809 }
1810
1811 struct thread_group *tg = thread_group_get(thread);
1812 cluster_type_t recommendation = thread_group_recommendation(tg);
1813 switch (recommendation) {
1814 case CLUSTER_TYPE_SMP:
1815 default:
1816 if (get_threadtask(thread) == kernel_task) {
1817 return PSET_AMP_E;
1818 }
1819 return PSET_AMP_P;
1820 case CLUSTER_TYPE_E:
1821 return PSET_AMP_E;
1822 case CLUSTER_TYPE_P:
1823 return PSET_AMP_P;
1824 }
1825 #else /* !CONFIG_THREAD_GROUPS || !__AMP__ || CONFIG_SCHED_EDGE */
1826 (void)thread;
1827 return PSET_SMP;
1828 #endif /* !CONFIG_THREAD_GROUPS || !__AMP__ || CONFIG_SCHED_EDGE */
1829 }
1830
1831 #if __arm64__
1832
1833 pset_cluster_type_t
cluster_type_to_pset_cluster_type(cluster_type_t cluster_type)1834 cluster_type_to_pset_cluster_type(cluster_type_t cluster_type)
1835 {
1836 switch (cluster_type) {
1837 #if __AMP__
1838 case CLUSTER_TYPE_E:
1839 return PSET_AMP_E;
1840 case CLUSTER_TYPE_P:
1841 return PSET_AMP_P;
1842 #endif /* __AMP__ */
1843 case CLUSTER_TYPE_SMP:
1844 return PSET_SMP;
1845 default:
1846 panic("Unexpected cluster type %d", cluster_type);
1847 }
1848 }
1849
1850 pset_node_t
cluster_type_to_pset_node(cluster_type_t cluster_type)1851 cluster_type_to_pset_node(cluster_type_t cluster_type)
1852 {
1853 switch (cluster_type) {
1854 #if __AMP__
1855 case CLUSTER_TYPE_E:
1856 return ecore_node;
1857 case CLUSTER_TYPE_P:
1858 return pcore_node;
1859 #endif /* __AMP__ */
1860 case CLUSTER_TYPE_SMP:
1861 return &pset_node0;
1862 default:
1863 panic("Unexpected cluster type %d", cluster_type);
1864 }
1865 }
1866
1867 #endif /* __arm64__ */
1868
1869 #if CONFIG_THREAD_GROUPS && __AMP__ && !CONFIG_SCHED_EDGE
1870
1871 void
sched_perfcontrol_inherit_recommendation_from_tg(perfcontrol_class_t perfctl_class,boolean_t inherit)1872 sched_perfcontrol_inherit_recommendation_from_tg(perfcontrol_class_t perfctl_class, boolean_t inherit)
1873 {
1874 sched_perfctl_class_policy_t sched_policy = inherit ? SCHED_PERFCTL_POLICY_FOLLOW_GROUP : SCHED_PERFCTL_POLICY_RESTRICT_E;
1875
1876 KDBG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_AMP_PERFCTL_POLICY_CHANGE) | DBG_FUNC_NONE, perfctl_class, sched_policy, 0, 0);
1877
1878 switch (perfctl_class) {
1879 case PERFCONTROL_CLASS_UTILITY:
1880 os_atomic_store(&sched_perfctl_policy_util, sched_policy, relaxed);
1881 break;
1882 case PERFCONTROL_CLASS_BACKGROUND:
1883 os_atomic_store(&sched_perfctl_policy_bg, sched_policy, relaxed);
1884 break;
1885 default:
1886 panic("perfctl_class invalid");
1887 break;
1888 }
1889 }
1890
1891 #elif defined(__arm64__)
1892
1893 /* Define a stub routine since this symbol is exported on all arm64 platforms */
1894 void
sched_perfcontrol_inherit_recommendation_from_tg(__unused perfcontrol_class_t perfctl_class,__unused boolean_t inherit)1895 sched_perfcontrol_inherit_recommendation_from_tg(__unused perfcontrol_class_t perfctl_class, __unused boolean_t inherit)
1896 {
1897 }
1898
1899 #endif /* defined(__arm64__) */
1900