101 	thread_t                        thread = p1;  in thread_quantum_expire()  local
106 	assert(thread == current_thread());  in thread_quantum_expire()
125 	ledger_credit(thread->t_ledger, task_ledgers.cpu_time, thread->quantum_remaining);  in thread_quantum_expire()
126 	ledger_credit(thread->t_threadledger, thread_ledgers.cpu_time, thread->quantum_remaining);  in thread_quantum_expire()
127 	if (thread->t_bankledger) {  in thread_quantum_expire()
128 		ledger_credit(thread->t_bankledger, bank_ledgers.cpu_time,  in thread_quantum_expire()
129 		    (thread->quantum_remaining - thread->t_deduct_bank_ledger_time));  in thread_quantum_expire()
131 	thread->t_deduct_bank_ledger_time = 0;  in thread_quantum_expire()
141 …pdate_pset_avg_execution_time(processor->processor_set, thread->quantum_remaining, ctime, thread->…  in thread_quantum_expire()
143 	recount_switch_thread(&snap, thread, get_threadtask(thread));  in thread_quantum_expire()
146 	thread_lock(thread);  in thread_quantum_expire()
153 	thread->last_run_time = ctime;  in thread_quantum_expire()
158 	if ((thread->sched_mode == TH_MODE_REALTIME || thread->sched_mode == TH_MODE_FIXED) &&  in thread_quantum_expire()
159 	    !(thread->sched_flags & TH_SFLAG_PROMOTED) &&  in thread_quantum_expire()
160 	    !(thread->kern_promotion_schedpri != 0) &&  in thread_quantum_expire()
161 	    !(thread->sched_flags & TH_SFLAG_PROMOTE_REASON_MASK) &&  in thread_quantum_expire()
162 	    !(thread->options & TH_OPT_SYSTEM_CRITICAL)) {  in thread_quantum_expire()
165 		new_computation = ctime - thread->computation_epoch;  in thread_quantum_expire()
166 		new_computation += thread->computation_metered;  in thread_quantum_expire()
171 …new_computation -= recount_current_thread_interrupt_time_mach() - thread->computation_interrupt_ep…  in thread_quantum_expire()
174 		switch (thread->sched_mode) {  in thread_quantum_expire()
177 				thread->safe_release = ctime + sched_safe_rt_duration;  in thread_quantum_expire()
183 				thread->safe_release = ctime + sched_safe_fixed_duration;  in thread_quantum_expire()
188 			panic("unexpected mode: %d", thread->sched_mode);  in thread_quantum_expire()
193 			    (uintptr_t)thread->sched_pri, (uintptr_t)thread->sched_mode, 0, 0, 0);  in thread_quantum_expire()
194 			sched_thread_mode_demote(thread, TH_SFLAG_FAILSAFE);  in thread_quantum_expire()
201 	if (SCHED(can_update_priority)(thread)) {  in thread_quantum_expire()
202 		SCHED(update_priority)(thread);  in thread_quantum_expire()
204 		SCHED(lightweight_update_priority)(thread);  in thread_quantum_expire()
207 	if (thread->sched_mode != TH_MODE_REALTIME) {  in thread_quantum_expire()
208 		SCHED(quantum_expire)(thread);  in thread_quantum_expire()
216 	thread_quantum_init(thread, ctime);  in thread_quantum_expire()
218 	timer_update(&thread->runnable_timer, ctime);  in thread_quantum_expire()
220 	processor->quantum_end = ctime + thread->quantum_remaining;  in thread_quantum_expire()
230 	if (get_threadtask(thread) == kernel_task) {  in thread_quantum_expire()
234 	if ((preempt = csw_check(thread, processor, check_reason)) != AST_NONE) {  in thread_quantum_expire()
243 	ast_propagate(thread);  in thread_quantum_expire()
245 	thread_unlock(thread);  in thread_quantum_expire()
249 	if (thread_get_tag(thread) & THREAD_TAG_WORKQUEUE) {  in thread_quantum_expire()
250 		thread_evaluate_workqueue_quantum_expiry(thread);  in thread_quantum_expire()
253 	running_timer_enter(processor, RUNNING_TIMER_QUANTUM, thread,  in thread_quantum_expire()
257 	thread_urgency_t urgency = thread_get_urgency(thread, NULL, NULL);  in thread_quantum_expire()
258 	machine_thread_going_on_core(thread, urgency, 0, 0, ctime);  in thread_quantum_expire()
260 	    0, thread);  in thread_quantum_expire()
267 	if (thread->sched_mode == TH_MODE_REALTIME) {  in thread_quantum_expire()
268 		sched_consider_recommended_cores(ctime, thread);  in thread_quantum_expire()
286 sched_set_thread_base_priority(thread_t thread, int priority)  in sched_set_thread_base_priority()  argument
291 	if (thread->sched_mode == TH_MODE_REALTIME) {  in sched_set_thread_base_priority()
297 	int old_base_pri = thread->base_pri;  in sched_set_thread_base_priority()
298 	thread->req_base_pri = (int16_t)priority;  in sched_set_thread_base_priority()
299 	if (thread->sched_flags & TH_SFLAG_BASE_PRI_FROZEN) {  in sched_set_thread_base_priority()
302 	thread->base_pri = (int16_t)priority;  in sched_set_thread_base_priority()
304 	if ((thread->state & TH_RUN) == TH_RUN) {  in sched_set_thread_base_priority()
305 		assert(thread->last_made_runnable_time != THREAD_NOT_RUNNABLE);  in sched_set_thread_base_priority()
307 		thread->last_basepri_change_time = ctime;  in sched_set_thread_base_priority()
309 		assert(thread->last_basepri_change_time == THREAD_NOT_RUNNABLE);  in sched_set_thread_base_priority()
310 		assert(thread->last_made_runnable_time == THREAD_NOT_RUNNABLE);  in sched_set_thread_base_priority()
318 	if (thread == current_thread() && old_base_pri != priority) {  in sched_set_thread_base_priority()
323 		    ctime, PERFCONTROL_CALLOUT_WAKE_UNSAFE, thread);  in sched_set_thread_base_priority()
327 	SCHED(update_thread_bucket)(thread);  in sched_set_thread_base_priority()
330 	thread_recompute_sched_pri(thread, SETPRI_DEFAULT);  in sched_set_thread_base_priority()
342 sched_set_kernel_thread_priority(thread_t thread, int new_priority)  in sched_set_kernel_thread_priority()  argument
346 	thread_lock(thread);  in sched_set_kernel_thread_priority()
348 	assert(thread->sched_mode != TH_MODE_REALTIME);  in sched_set_kernel_thread_priority()
349 	assert(thread->effective_policy.thep_qos == THREAD_QOS_UNSPECIFIED);  in sched_set_kernel_thread_priority()
351 	if (new_priority > thread->max_priority) {  in sched_set_kernel_thread_priority()
352 		new_priority = thread->max_priority;  in sched_set_kernel_thread_priority()
360 	thread->importance = new_priority - thread->task_priority;  in sched_set_kernel_thread_priority()
362 	sched_set_thread_base_priority(thread, new_priority);  in sched_set_kernel_thread_priority()
364 	thread_unlock(thread);  in sched_set_kernel_thread_priority()
384 thread_recompute_sched_pri(thread_t thread, set_sched_pri_options_t options)  in thread_recompute_sched_pri()  argument
386 	uint32_t     sched_flags = thread->sched_flags;  in thread_recompute_sched_pri()
387 	sched_mode_t sched_mode  = thread->sched_mode;  in thread_recompute_sched_pri()
389 	int16_t priority = thread->base_pri;  in thread_recompute_sched_pri()
392 		priority = (int16_t)SCHED(compute_timeshare_priority)(thread);  in thread_recompute_sched_pri()
404 		if (thread->kern_promotion_schedpri > 0) {  in thread_recompute_sched_pri()
405 			priority = MAX(priority, thread->kern_promotion_schedpri);  in thread_recompute_sched_pri()
413 			priority = MAX(priority, thread->promotion_priority);  in thread_recompute_sched_pri()
439 	set_sched_pri(thread, priority, options);  in thread_recompute_sched_pri()
443 sched_default_quantum_expire(thread_t thread __unused)  in sched_default_quantum_expire()
467 lightweight_update_priority(thread_t thread)  in lightweight_update_priority()  argument
469 	thread_assert_runq_null(thread);  in lightweight_update_priority()
470 	assert(thread == current_thread());  in lightweight_update_priority()
472 	if (thread->sched_mode == TH_MODE_TIMESHARE) {  in lightweight_update_priority()
476 		sched_tick_delta(thread, delta);  in lightweight_update_priority()
483 		if (thread->pri_shift < INT8_MAX) {  in lightweight_update_priority()
484 			if (thread_no_smt(thread) && smt_timeshare_enabled) {  in lightweight_update_priority()
485 				thread->sched_usage += (delta + ((delta * smt_sched_bonus_16ths) >> 4));  in lightweight_update_priority()
487 				thread->sched_usage += delta;  in lightweight_update_priority()
491 		thread->cpu_delta += delta;  in lightweight_update_priority()
499 		sched_clutch_cpu_usage_update(thread, delta);  in lightweight_update_priority()
502 		priority = sched_compute_timeshare_priority(thread);  in lightweight_update_priority()
504 		if (priority != thread->sched_pri) {  in lightweight_update_priority()
505 			thread_recompute_sched_pri(thread, SETPRI_LAZY);  in lightweight_update_priority()
565 sched_compute_timeshare_priority(thread_t thread)  in sched_compute_timeshare_priority()  argument
570 	if (thread->base_pri > BASEPRI_FOREGROUND) {  in sched_compute_timeshare_priority()
571 		decay_limit += (thread->base_pri - BASEPRI_FOREGROUND);  in sched_compute_timeshare_priority()
574 	if (thread->pri_shift == INT8_MAX) {  in sched_compute_timeshare_priority()
577 		decay_amount = (thread->sched_usage >> thread->pri_shift);  in sched_compute_timeshare_priority()
585 	int priority = thread->base_pri - decay_amount;  in sched_compute_timeshare_priority()
588 		if (get_threadtask(thread)->max_priority > MAXPRI_THROTTLE) {  in sched_compute_timeshare_priority()
603 sched_compute_timeshare_priority(thread_t thread)  in sched_compute_timeshare_priority()  argument
606 	int priority = thread->base_pri;  in sched_compute_timeshare_priority()
608 	if (thread->pri_shift != INT8_MAX) {  in sched_compute_timeshare_priority()
609 		priority -= (thread->sched_usage >> thread->pri_shift);  in sched_compute_timeshare_priority()
632 	thread_t        thread)  in can_update_priority()  argument
634 	if (sched_tick == thread->sched_stamp) {  in can_update_priority()
650 	thread_t        thread)  in update_priority()  argument
654 	ticks = sched_tick - thread->sched_stamp;  in update_priority()
657 	thread->sched_stamp += ticks;  in update_priority()
667 	sched_tick_delta(thread, delta);  in update_priority()
674 		if (thread->pri_shift < INT8_MAX) {  in update_priority()
675 			if (thread_no_smt(thread) && smt_timeshare_enabled) {  in update_priority()
676 				thread->sched_usage += (delta + ((delta * smt_sched_bonus_16ths) >> 4));  in update_priority()
678 				thread->sched_usage += delta;  in update_priority()
682 		thread->cpu_usage += delta + thread->cpu_delta;  in update_priority()
683 		thread->cpu_delta = 0;  in update_priority()
691 		sched_clutch_cpu_usage_update(thread, delta);  in update_priority()
697 			thread->cpu_usage =   (thread->cpu_usage >> shiftp->shift1) +  in update_priority()
698 			    (thread->cpu_usage >> shiftp->shift2);  in update_priority()
699 			thread->sched_usage = (thread->sched_usage >> shiftp->shift1) +  in update_priority()
700 			    (thread->sched_usage >> shiftp->shift2);  in update_priority()
702 			thread->cpu_usage =   (thread->cpu_usage >>   shiftp->shift1) -  in update_priority()
703 			    (thread->cpu_usage >> -(shiftp->shift2));  in update_priority()
704 			thread->sched_usage = (thread->sched_usage >>   shiftp->shift1) -  in update_priority()
705 			    (thread->sched_usage >> -(shiftp->shift2));  in update_priority()
708 		thread->cpu_usage = thread->cpu_delta = 0;  in update_priority()
709 		thread->sched_usage = 0;  in update_priority()
715 	if ((thread->sched_flags & TH_SFLAG_FAILSAFE) &&  in update_priority()
716 	    mach_absolute_time() >= thread->safe_release) {  in update_priority()
717 		sched_thread_mode_undemote(thread, TH_SFLAG_FAILSAFE);  in update_priority()
728 	thread->pri_shift = sched_clutch_thread_pri_shift(thread, thread->th_sched_bucket);  in update_priority()
730 	thread->pri_shift = sched_pri_shifts[thread->th_sched_bucket];  in update_priority()
734 	if (thread->sched_mode == TH_MODE_TIMESHARE) {  in update_priority()
735 		thread_recompute_sched_pri(thread, SETPRI_LAZY);  in update_priority()
794 sched_run_incr(thread_t thread)  in sched_run_incr()  argument
796 	assert((thread->state & (TH_RUN | TH_IDLE)) == TH_RUN);  in sched_run_incr()
800 	sched_incr_bucket(thread->th_sched_bucket);  in sched_run_incr()
806 sched_run_decr(thread_t thread)  in sched_run_decr()  argument
808 	assert((thread->state & (TH_RUN | TH_IDLE)) != TH_RUN);  in sched_run_decr()
810 	sched_decr_bucket(thread->th_sched_bucket);  in sched_run_decr()
818 sched_smt_run_incr(thread_t thread)  in sched_smt_run_incr()  argument
820 	assert((thread->state & (TH_RUN | TH_IDLE)) == TH_RUN);  in sched_smt_run_incr()
822 	uint8_t run_weight = (thread_no_smt(thread) && smt_timeshare_enabled) ? 2 : 1;  in sched_smt_run_incr()
823 	thread->sched_saved_run_weight = run_weight;  in sched_smt_run_incr()
827 	sched_add_bucket(thread->th_sched_bucket, run_weight);  in sched_smt_run_incr()
833 sched_smt_run_decr(thread_t thread)  in sched_smt_run_decr()  argument
835 	assert((thread->state & (TH_RUN | TH_IDLE)) != TH_RUN);  in sched_smt_run_decr()
837 	uint8_t run_weight = thread->sched_saved_run_weight;  in sched_smt_run_decr()
839 	sched_sub_bucket(thread->th_sched_bucket, run_weight);  in sched_smt_run_decr()
847 sched_update_thread_bucket(thread_t thread)  in sched_update_thread_bucket()  argument
849 	sched_bucket_t old_bucket = thread->th_sched_bucket;  in sched_update_thread_bucket()
852 	switch (thread->sched_mode) {  in sched_update_thread_bucket()
859 		if (thread->base_pri > BASEPRI_DEFAULT) {  in sched_update_thread_bucket()
861 		} else if (thread->base_pri > BASEPRI_UTILITY) {  in sched_update_thread_bucket()
863 		} else if (thread->base_pri > MAXPRI_THROTTLE) {  in sched_update_thread_bucket()
871 		panic("unexpected mode: %d", thread->sched_mode);  in sched_update_thread_bucket()
876 		thread->th_sched_bucket = new_bucket;  in sched_update_thread_bucket()
877 		thread->pri_shift = sched_pri_shifts[new_bucket];  in sched_update_thread_bucket()
879 		if ((thread->state & (TH_RUN | TH_IDLE)) == TH_RUN) {  in sched_update_thread_bucket()
887 sched_smt_update_thread_bucket(thread_t thread)  in sched_smt_update_thread_bucket()  argument
889 	sched_bucket_t old_bucket = thread->th_sched_bucket;  in sched_smt_update_thread_bucket()
892 	switch (thread->sched_mode) {  in sched_smt_update_thread_bucket()
899 		if (thread->base_pri > BASEPRI_DEFAULT) {  in sched_smt_update_thread_bucket()
901 		} else if (thread->base_pri > BASEPRI_UTILITY) {  in sched_smt_update_thread_bucket()
903 		} else if (thread->base_pri > MAXPRI_THROTTLE) {  in sched_smt_update_thread_bucket()
911 		panic("unexpected mode: %d", thread->sched_mode);  in sched_smt_update_thread_bucket()
916 		thread->th_sched_bucket = new_bucket;  in sched_smt_update_thread_bucket()
917 		thread->pri_shift = sched_pri_shifts[new_bucket];  in sched_smt_update_thread_bucket()
919 		if ((thread->state & (TH_RUN | TH_IDLE)) == TH_RUN) {  in sched_smt_update_thread_bucket()
920 			sched_sub_bucket(old_bucket, thread->sched_saved_run_weight);  in sched_smt_update_thread_bucket()
921 			sched_add_bucket(new_bucket, thread->sched_saved_run_weight);  in sched_smt_update_thread_bucket()
949 sched_set_thread_mode(thread_t thread, sched_mode_t new_mode)  in sched_set_thread_mode()  argument
951 	thread_assert_runq_null(thread);  in sched_set_thread_mode()
961 	if ((thread->sched_flags & TH_SFLAG_THREAD_GROUP_AUTO_JOIN) && (new_mode != TH_MODE_REALTIME)) {  in sched_set_thread_mode()
962 …assert((thread->sched_mode == TH_MODE_REALTIME) || (thread->th_work_interval_flags & TH_WORK_INTER…  in sched_set_thread_mode()
963 		work_interval_auto_join_demote(thread);  in sched_set_thread_mode()
967 	thread->sched_mode = new_mode;  in sched_set_thread_mode()
969 	SCHED(update_thread_bucket)(thread);  in sched_set_thread_mode()
978 sched_set_thread_mode_user(thread_t thread, sched_mode_t new_mode)  in sched_set_thread_mode_user()  argument
980 	thread_assert_runq_null(thread);  in sched_set_thread_mode_user()
985 	if (thread->sched_flags & TH_SFLAG_DEMOTED_MASK) {  in sched_set_thread_mode_user()
986 		thread->saved_mode = new_mode;  in sched_set_thread_mode_user()
988 		sched_set_thread_mode(thread, new_mode);  in sched_set_thread_mode_user()
993 sched_get_thread_mode_user(thread_t thread)  in sched_get_thread_mode_user()  argument
995 	if (thread->sched_flags & TH_SFLAG_DEMOTED_MASK) {  in sched_get_thread_mode_user()
996 		return thread->saved_mode;  in sched_get_thread_mode_user()
998 		return thread->sched_mode;  in sched_get_thread_mode_user()
1006 sched_thread_mode_demote(thread_t thread, uint32_t reason)  in sched_thread_mode_demote()  argument
1009 	assert((thread->sched_flags & reason) != reason);  in sched_thread_mode_demote()
1011 	if (thread->policy_reset) {  in sched_thread_mode_demote()
1018 		    thread_tid(thread), thread->sched_flags);  in sched_thread_mode_demote()
1022 		    thread_tid(thread), thread->sched_flags);  in sched_thread_mode_demote()
1026 		    thread_tid(thread), thread->sched_flags);  in sched_thread_mode_demote()
1030 	if (thread->sched_flags & TH_SFLAG_DEMOTED_MASK) {  in sched_thread_mode_demote()
1032 		thread->sched_flags |= reason;  in sched_thread_mode_demote()
1036 	assert(thread->saved_mode == TH_MODE_NONE);  in sched_thread_mode_demote()
1038 	boolean_t removed = thread_run_queue_remove(thread);  in sched_thread_mode_demote()
1040 	thread->sched_flags |= reason;  in sched_thread_mode_demote()
1042 	thread->saved_mode = thread->sched_mode;  in sched_thread_mode_demote()
1044 	sched_set_thread_mode(thread, TH_MODE_TIMESHARE);  in sched_thread_mode_demote()
1046 	thread_recompute_priority(thread);  in sched_thread_mode_demote()
1049 		thread_run_queue_reinsert(thread, SCHED_TAILQ);  in sched_thread_mode_demote()
1057 sched_thread_mode_has_demotion(thread_t thread, uint32_t reason)  in sched_thread_mode_has_demotion()  argument
1060 	return (thread->sched_flags & reason) != 0;  in sched_thread_mode_has_demotion()
1067 sched_thread_mode_undemote(thread_t thread, uint32_t reason)  in sched_thread_mode_undemote()  argument
1070 	assert((thread->sched_flags & reason) == reason);  in sched_thread_mode_undemote()
1071 	assert(thread->saved_mode != TH_MODE_NONE);  in sched_thread_mode_undemote()
1072 	assert(thread->sched_mode == TH_MODE_TIMESHARE);  in sched_thread_mode_undemote()
1073 	assert(thread->policy_reset == 0);  in sched_thread_mode_undemote()
1078 		    thread_tid(thread), thread->sched_flags);  in sched_thread_mode_undemote()
1082 		    thread_tid(thread), thread->sched_flags);  in sched_thread_mode_undemote()
1086 		    thread_tid(thread), thread->sched_flags);  in sched_thread_mode_undemote()
1090 	thread->sched_flags &= ~reason;  in sched_thread_mode_undemote()
1092 	if (thread->sched_flags & TH_SFLAG_DEMOTED_MASK) {  in sched_thread_mode_undemote()
1097 	boolean_t removed = thread_run_queue_remove(thread);  in sched_thread_mode_undemote()
1099 	sched_set_thread_mode(thread, thread->saved_mode);  in sched_thread_mode_undemote()
1101 	thread->saved_mode = TH_MODE_NONE;  in sched_thread_mode_undemote()
1103 	thread_recompute_priority(thread);  in sched_thread_mode_undemote()
1106 		thread_run_queue_reinsert(thread, SCHED_TAILQ);  in sched_thread_mode_undemote()
1120 sched_thread_promote_reason(thread_t    thread,  in sched_thread_promote_reason()  argument
1125 	assert((thread->sched_flags & reason) != reason);  in sched_thread_promote_reason()
1130 		    thread_tid(thread), thread->sched_pri,  in sched_thread_promote_reason()
1131 		    thread->base_pri, trace_obj);  in sched_thread_promote_reason()
1135 		    thread_tid(thread), thread->sched_pri,  in sched_thread_promote_reason()
1136 		    thread->base_pri, trace_obj);  in sched_thread_promote_reason()
1140 		    thread_tid(thread), thread->sched_pri,  in sched_thread_promote_reason()
1141 		    thread->base_pri, trace_obj);  in sched_thread_promote_reason()
1145 		    thread_tid(thread), thread->sched_pri,  in sched_thread_promote_reason()
1146 		    thread->base_pri, trace_obj);  in sched_thread_promote_reason()
1150 	thread->sched_flags |= reason;  in sched_thread_promote_reason()
1151 	thread_recompute_sched_pri(thread, SETPRI_DEFAULT);  in sched_thread_promote_reason()
1161 sched_thread_unpromote_reason(thread_t  thread,  in sched_thread_unpromote_reason()  argument
1166 	assert((thread->sched_flags & reason) == reason);  in sched_thread_unpromote_reason()
1171 		    thread_tid(thread), thread->sched_pri,  in sched_thread_unpromote_reason()
1172 		    thread->base_pri, trace_obj);  in sched_thread_unpromote_reason()
1176 		    thread_tid(thread), thread->sched_pri,  in sched_thread_unpromote_reason()
1177 		    thread->base_pri, trace_obj);  in sched_thread_unpromote_reason()
1181 		    thread_tid(thread), thread->sched_pri,  in sched_thread_unpromote_reason()
1182 		    thread->base_pri, trace_obj);  in sched_thread_unpromote_reason()
1186 		    thread_tid(thread), thread->sched_pri,  in sched_thread_unpromote_reason()
1187 		    thread->base_pri, trace_obj);  in sched_thread_unpromote_reason()
1191 	thread->sched_flags &= ~reason;  in sched_thread_unpromote_reason()
1193 	thread_recompute_sched_pri(thread, SETPRI_DEFAULT);  in sched_thread_unpromote_reason()