159 struct processor        PERCPU_DATA(processor);
207 processor_update_offline_state_locked(processor_t processor,  in processor_update_offline_state_locked()  argument
212 	processor_offline_state_t old_state = processor->processor_offline_state;  in processor_update_offline_state_locked()
214 	uint cpuid = (uint)processor->cpu_id;  in processor_update_offline_state_locked()
219 	processor->processor_offline_state = new_state;  in processor_update_offline_state_locked()
226 processor_update_offline_state(processor_t processor,  in processor_update_offline_state()  argument
231 	processor_update_offline_state_locked(processor, new_state);  in processor_update_offline_state()
298 	processor_t            processor,  in processor_init()  argument
305 	processor->cpu_id = cpu_id;  in processor_init()
307 	if (processor != master_processor) {  in processor_init()
309 		SCHED(processor_init)(processor);  in processor_init()
310 		smr_cpu_init(processor);  in processor_init()
313 	processor->state = PROCESSOR_OFF_LINE;  in processor_init()
314 	processor->active_thread = processor->startup_thread = processor->idle_thread = THREAD_NULL;  in processor_init()
315 	processor->processor_set = pset;  in processor_init()
316 	processor_state_update_idle(processor);  in processor_init()
317 	processor->starting_pri = MINPRI;  in processor_init()
318 	processor->quantum_end = UINT64_MAX;  in processor_init()
319 	processor->deadline = UINT64_MAX;  in processor_init()
320 	processor->first_timeslice = FALSE;  in processor_init()
321 	processor->processor_online = false;  in processor_init()
322 	processor->processor_primary = processor; /* no SMT relationship known at this point */  in processor_init()
323 	processor->processor_secondary = NULL;  in processor_init()
324 	processor->is_SMT = false;  in processor_init()
325 	processor->processor_self = IP_NULL;  in processor_init()
326 	processor->processor_list = NULL;  in processor_init()
327 	processor->must_idle = false;  in processor_init()
328 	processor->next_idle_short = false;  in processor_init()
329 	processor->last_startup_reason = REASON_SYSTEM;  in processor_init()
330 	processor->last_shutdown_reason = REASON_NONE;  in processor_init()
331 	processor->shutdown_temporary = false;  in processor_init()
332 	processor->processor_inshutdown = false;  in processor_init()
333 	processor->processor_instartup = false;  in processor_init()
334 	processor->last_derecommend_reason = REASON_NONE;  in processor_init()
335 	processor->running_timers_active = false;  in processor_init()
337 		timer_call_setup(&processor->running_timers[i],  in processor_init()
338 		    running_timer_funcs[i], processor);  in processor_init()
339 		running_timer_clear(processor, i);  in processor_init()
341 	recount_processor_init(processor);  in processor_init()
359 	processor->last_recommend_reason = REASON_SYSTEM;  in processor_init()
360 	sched_processor_change_mode_locked(processor, PCM_RECOMMENDED, true);  in processor_init()
363 	processor->processor_offline_state = PROCESSOR_OFFLINE_NOT_BOOTED;  in processor_init()
364 	bit_set(processor_offline_state_map[processor->processor_offline_state], cpu_id);  in processor_init()
366 	if (processor == master_processor) {  in processor_init()
367 		processor_update_offline_state_locked(processor, PROCESSOR_OFFLINE_STARTING);  in processor_init()
375 		processor_list = processor;  in processor_init()
377 		processor_list_tail->processor_list = processor;  in processor_init()
379 	processor_list_tail = processor;  in processor_init()
382 	processor_array[cpu_id] = processor;  in processor_init()
389 	processor_t             processor,  in processor_set_primary()  argument
392 	assert(processor->processor_primary == primary || processor->processor_primary == processor);  in processor_set_primary()
394 	processor->processor_primary = primary;  in processor_set_primary()
396 	assert(primary->processor_secondary == NULL || primary->processor_secondary == processor);  in processor_set_primary()
397 	if (primary != processor) {  in processor_set_primary()
402 		assert(processor->processor_secondary == NULL);  in processor_set_primary()
403 		primary->processor_secondary = processor;  in processor_set_primary()
406 		processor->is_SMT = TRUE;  in processor_set_primary()
413 		processor_set_t pset = processor->processor_set;  in processor_set_primary()
419 		bit_clear(pset->primary_map, processor->cpu_id);  in processor_set_primary()
427 	processor_t     processor)  in processor_pset()  argument
429 	return processor->processor_set;  in processor_pset()
455 processor_state_update_running_foreign(processor_t processor, thread_t thread)  in processor_state_update_running_foreign()  argument
457 …cluster_type_t current_processor_type = pset_type_for_id(processor->processor_set->pset_cluster_id…  in processor_state_update_running_foreign()
460 	boolean_t non_rt_thr = (processor->current_pri < BASEPRI_RTQUEUES);  in processor_state_update_running_foreign()
463 		bit_set(processor->processor_set->cpu_running_foreign, processor->cpu_id);  in processor_state_update_running_foreign()
465 		bit_clear(processor->processor_set->cpu_running_foreign, processor->cpu_id);  in processor_state_update_running_foreign()
479 processor_state_update_running_cluster_shared_rsrc(processor_t processor, thread_t thread)  in processor_state_update_running_cluster_shared_rsrc()  argument
482 …bit_set(processor->processor_set->cpu_running_cluster_shared_rsrc_thread[CLUSTER_SHARED_RSRC_TYPE_…  in processor_state_update_running_cluster_shared_rsrc()
484 …bit_clear(processor->processor_set->cpu_running_cluster_shared_rsrc_thread[CLUSTER_SHARED_RSRC_TYP…  in processor_state_update_running_cluster_shared_rsrc()
487 …bit_set(processor->processor_set->cpu_running_cluster_shared_rsrc_thread[CLUSTER_SHARED_RSRC_TYPE_…  in processor_state_update_running_cluster_shared_rsrc()
489 …bit_clear(processor->processor_set->cpu_running_cluster_shared_rsrc_thread[CLUSTER_SHARED_RSRC_TYP…  in processor_state_update_running_cluster_shared_rsrc()
496 processor_state_update_idle(processor_t processor)  in processor_state_update_idle()  argument
498 	processor->current_pri = IDLEPRI;  in processor_state_update_idle()
499 	processor->current_sfi_class = SFI_CLASS_KERNEL;  in processor_state_update_idle()
500 	processor->current_recommended_pset_type = PSET_SMP;  in processor_state_update_idle()
502 	processor->current_thread_group = NULL;  in processor_state_update_idle()
504 	processor->current_perfctl_class = PERFCONTROL_CLASS_IDLE;  in processor_state_update_idle()
505 	processor->current_urgency = THREAD_URGENCY_NONE;  in processor_state_update_idle()
506 	processor->current_is_NO_SMT = false;  in processor_state_update_idle()
507 	processor->current_is_bound = false;  in processor_state_update_idle()
508 	processor->current_is_eagerpreempt = false;  in processor_state_update_idle()
510 …os_atomic_store(&processor->processor_set->cpu_running_buckets[processor->cpu_id], TH_BUCKET_SCHED…  in processor_state_update_idle()
511 …bit_clear(processor->processor_set->cpu_running_cluster_shared_rsrc_thread[CLUSTER_SHARED_RSRC_TYP…  in processor_state_update_idle()
512 …bit_clear(processor->processor_set->cpu_running_cluster_shared_rsrc_thread[CLUSTER_SHARED_RSRC_TYP…  in processor_state_update_idle()
514 	sched_update_pset_load_average(processor->processor_set, 0);  in processor_state_update_idle()
518 processor_state_update_from_thread(processor_t processor, thread_t thread, boolean_t pset_lock_held)  in processor_state_update_from_thread()  argument
520 	processor->current_pri = thread->sched_pri;  in processor_state_update_from_thread()
521 	processor->current_sfi_class = thread->sfi_class;  in processor_state_update_from_thread()
522 	processor->current_recommended_pset_type = recommended_pset_type(thread);  in processor_state_update_from_thread()
524 	processor_state_update_running_foreign(processor, thread);  in processor_state_update_from_thread()
525 	processor_state_update_running_cluster_shared_rsrc(processor, thread);  in processor_state_update_from_thread()
528 …os_atomic_store(&processor->processor_set->cpu_running_buckets[processor->cpu_id], bucket, relaxed…  in processor_state_update_from_thread()
532 	processor->current_thread_group = thread_group_get(thread);  in processor_state_update_from_thread()
534 	processor->current_perfctl_class = thread_get_perfcontrol_class(thread);  in processor_state_update_from_thread()
535 	processor->current_urgency = thread_get_urgency(thread, NULL, NULL);  in processor_state_update_from_thread()
536 	processor->current_is_NO_SMT = thread_no_smt(thread);  in processor_state_update_from_thread()
537 	processor->current_is_bound = thread->bound_processor != PROCESSOR_NULL;  in processor_state_update_from_thread()
538 	processor->current_is_eagerpreempt = thread_is_eager_preempt(thread);  in processor_state_update_from_thread()
541 		sched_update_pset_load_average(processor->processor_set, 0);  in processor_state_update_from_thread()
546 processor_state_update_explicit(processor_t processor, int pri, sfi_class_id_t sfi_class,  in processor_state_update_explicit()  argument
549 	processor->current_pri = pri;  in processor_state_update_explicit()
550 	processor->current_sfi_class = sfi_class;  in processor_state_update_explicit()
551 	processor->current_recommended_pset_type = pset_type;  in processor_state_update_explicit()
552 	processor->current_perfctl_class = perfctl_class;  in processor_state_update_explicit()
553 	processor->current_urgency = urgency;  in processor_state_update_explicit()
555 …os_atomic_store(&processor->processor_set->cpu_running_buckets[processor->cpu_id], bucket, relaxed…  in processor_state_update_explicit()
556 …bit_clear(processor->processor_set->cpu_running_cluster_shared_rsrc_thread[CLUSTER_SHARED_RSRC_TYP…  in processor_state_update_explicit()
557 …bit_clear(processor->processor_set->cpu_running_cluster_shared_rsrc_thread[CLUSTER_SHARED_RSRC_TYP…  in processor_state_update_explicit()
735 processor_cpu_load_info(processor_t processor,  in processor_cpu_load_info()  argument
740 	recount_processor_usage(&processor->pr_recount, &usage, &idle_time);  in processor_cpu_load_info()
751 	processor_t     processor,  in processor_info()  argument
760 	if (processor == PROCESSOR_NULL) {  in processor_info()
764 	cpu_id = processor->cpu_id;  in processor_info()
778 		state = processor->state;  in processor_info()
779 …if (((state == PROCESSOR_OFF_LINE || state == PROCESSOR_PENDING_OFFLINE) && !processor->shutdown_t…  in processor_info()
781 		    || !processor->is_recommended  in processor_info()
789 		if (processor == master_processor) {  in processor_info()
814 		processor_cpu_load_info(processor, cpu_load_info->cpu_ticks);  in processor_info()
850 processor_wait_for_start(processor_t processor, processor_start_kind_t start_kind)  in processor_wait_for_start()  argument
852 	if (!processor->processor_booted) {  in processor_wait_for_start()
853 		panic("processor_boot() missing for cpu %d", processor->cpu_id);  in processor_wait_for_start()
861 	while (processor->processor_instartup) {  in processor_wait_for_start()
862 		assert_wait_timeout((event_t)&processor->processor_instartup,  in processor_wait_for_start()
870 			    processor->cpu_id,  in processor_wait_for_start()
879 	if (processor->processor_inshutdown) {  in processor_wait_for_start()
881 		    __func__, processor->cpu_id);  in processor_wait_for_start()
888 	if (!processor->processor_online) {  in processor_wait_for_start()
890 		    __func__, processor->cpu_id);  in processor_wait_for_start()
893 	if (processor->processor_offline_state == PROCESSOR_OFFLINE_STARTED_NOT_WAITED) {  in processor_wait_for_start()
894 		processor_update_offline_state_locked(processor, PROCESSOR_OFFLINE_RUNNING);  in processor_wait_for_start()
896 		assert(processor->processor_offline_state == PROCESSOR_OFFLINE_RUNNING);  in processor_wait_for_start()
908 	processor_t     processor,  in processor_dostartup()  argument
911 	if (!processor->processor_booted && !first_boot) {  in processor_dostartup()
912 		panic("processor %d not booted", processor->cpu_id);  in processor_dostartup()
918 	processor_set_t pset = processor->processor_set;  in processor_dostartup()
920 	assert(processor->processor_self);  in processor_dostartup()
925 	assert(processor->processor_inshutdown || first_boot);  in processor_dostartup()
926 	processor->processor_inshutdown = false;  in processor_dostartup()
927 	assert(processor->processor_instartup == false);  in processor_dostartup()
928 	processor->processor_instartup = true;  in processor_dostartup()
936 		assert(processor->processor_offline_state == PROCESSOR_OFFLINE_NOT_BOOTED);  in processor_dostartup()
938 		assert(processor->processor_offline_state == PROCESSOR_OFFLINE_FULLY_OFFLINE);  in processor_dostartup()
941 	processor_update_offline_state_locked(processor, PROCESSOR_OFFLINE_STARTING);  in processor_dostartup()
943 	assert(processor->state == PROCESSOR_OFF_LINE);  in processor_dostartup()
945 	pset_update_processor_state(pset, processor, PROCESSOR_START);  in processor_dostartup()
952 	ml_cpu_power_enable(processor->cpu_id);  in processor_dostartup()
953 	ml_cpu_begin_state_transition(processor->cpu_id);  in processor_dostartup()
954 	ml_broadcast_cpu_event(CPU_BOOT_REQUESTED, processor->cpu_id);  in processor_dostartup()
956 	cpu_start(processor->cpu_id);  in processor_dostartup()
961 	if (processor->processor_offline_state == PROCESSOR_OFFLINE_STARTING) {  in processor_dostartup()
962 		processor_update_offline_state_locked(processor, PROCESSOR_OFFLINE_STARTED_NOT_RUNNING);  in processor_dostartup()
964 		assert(processor->processor_offline_state == PROCESSOR_OFFLINE_STARTED_NOT_WAITED);  in processor_dostartup()
970 	ml_cpu_end_state_transition(processor->cpu_id);  in processor_dostartup()
977 	ml_broadcast_cpu_event(CPU_ACTIVE, processor->cpu_id);  in processor_dostartup()
981 processor_exit_reason(processor_t processor, processor_reason_t reason, bool is_system_sleep)  in processor_exit_reason()  argument
983 	assert(processor);  in processor_exit_reason()
984 	assert(processor->processor_set);  in processor_exit_reason()
992 	assert((processor != master_processor) || (reason == REASON_SYSTEM) || support_bootcpu_shutdown);  in processor_exit_reason()
994 	processor->last_shutdown_reason = reason;  in processor_exit_reason()
996 	bool is_final_system_sleep = is_system_sleep && (processor == master_processor);  in processor_exit_reason()
998 	processor_doshutdown(processor, is_final_system_sleep);  in processor_exit_reason()
1012 	__unused processor_t processor)  in processor_exit_from_kext()  argument
1020 	processor_t     processor)  in processor_sleep()  argument
1024 	processor_exit_reason(processor, REASON_SYSTEM, true);  in processor_sleep()
1029 	processor_t     processor)  in processor_exit_from_user()  argument
1031 	if (processor == PROCESSOR_NULL) {  in processor_exit_from_user()
1039 	result = sched_processor_exit_user(processor);  in processor_exit_from_user()
1047 processor_start_reason(processor_t processor, processor_reason_t reason)  in processor_start_reason()  argument
1051 	assert(processor);  in processor_start_reason()
1052 	assert(processor->processor_set);  in processor_start_reason()
1053 	assert(processor->processor_booted);  in processor_start_reason()
1059 	processor->last_startup_reason = reason;  in processor_start_reason()
1061 	processor_dostartup(processor, false);  in processor_start_reason()
1076 	__unused processor_t processor)  in processor_start_from_kext()  argument
1084 	processor_t                     processor)  in processor_start_from_user()  argument
1086 	if (processor == PROCESSOR_NULL) {  in processor_start_from_user()
1094 	result = sched_processor_start_user(processor);  in processor_start_from_user()
1109 	processor_t                     processor)  in processor_boot()  argument
1117 	if (processor->processor_booted) {  in processor_boot()
1118 		panic("processor %d already booted", processor->cpu_id);  in processor_boot()
1121 	if (processor == master_processor) {  in processor_boot()
1122 		assert(processor->processor_offline_state == PROCESSOR_OFFLINE_STARTED_NOT_WAITED);  in processor_boot()
1124 		assert(processor->processor_offline_state == PROCESSOR_OFFLINE_NOT_BOOTED);  in processor_boot()
1130 	if (processor->idle_thread == THREAD_NULL) {  in processor_boot()
1131 		idle_thread_create(processor, processor_start_thread);  in processor_boot()
1134 	if (processor->processor_self == IP_NULL) {  in processor_boot()
1135 		ipc_processor_init(processor);  in processor_boot()
1138 	if (processor == master_processor) {  in processor_boot()
1139 		processor->last_startup_reason = REASON_SYSTEM;  in processor_boot()
1141 		ml_cpu_power_enable(processor->cpu_id);  in processor_boot()
1143 		processor_t prev = thread_bind(processor);  in processor_boot()
1146 		cpu_start(processor->cpu_id);  in processor_boot()
1148 		assert(processor->state == PROCESSOR_RUNNING);  in processor_boot()
1149 		processor_update_offline_state(processor, PROCESSOR_OFFLINE_RUNNING);  in processor_boot()
1153 		processor->last_startup_reason = REASON_SYSTEM;  in processor_boot()
1159 		processor_dostartup(processor, true);  in processor_boot()
1162 	processor->processor_booted = true;  in processor_boot()
1173 	processor_t                     processor)  in processor_wake()  argument
1177 	assert(processor->processor_booted);  in processor_wake()
1178 	processor_start_reason(processor, REASON_SYSTEM);  in processor_wake()
1192 		processor_t processor = processor_array[i];  in enable_smt_processors()  local
1194 		if (processor->processor_primary != processor) {  in enable_smt_processors()
1196 				processor_start_from_user(processor);  in enable_smt_processors()
1198 				processor_exit_from_user(processor);  in enable_smt_processors()
1223 processor_should_kprintf(processor_t processor, bool starting)  in processor_should_kprintf()  argument
1225 …processor_reason_t reason = starting ? processor->last_startup_reason : processor->last_shutdown_r…  in processor_should_kprintf()
1232 	processor_t             processor,  in processor_control()  argument
1236 	if (processor == PROCESSOR_NULL) {  in processor_control()
1240 	return cpu_control(processor->cpu_id, info, count);  in processor_control()
1245 	processor_t     processor,  in processor_get_assignment()  argument
1250 	if (processor == PROCESSOR_NULL) {  in processor_get_assignment()
1254 	state = processor->state;  in processor_get_assignment()