85 static void workq_unpark_continue(void *uth, wait_result_t wr) __dead2;
87 static void workq_bound_thread_unpark_continue(void *uth, wait_result_t wr) __dead2;
89 static void workq_bound_thread_initialize_and_unpark_continue(void *uth, wait_result_t wr) __dead2;
91 static void workq_bound_thread_setup_and_run(struct uthread *uth, int setup_flags) __dead2;
96 static bool workq_threadreq_admissible(struct workqueue *wq, struct uthread *uth,
100     thread_qos_t at_qos, struct uthread *uth,
294 workq_parked_wait_event(struct uthread *uth)  in workq_parked_wait_event()  argument
296 	return (event_t)&uth->uu_workq_stackaddr;  in workq_parked_wait_event()
300 workq_thread_wakeup(struct uthread *uth)  in workq_thread_wakeup()  argument
302 	thread_wakeup_thread(workq_parked_wait_event(uth), get_machthread(uth));  in workq_thread_wakeup()
548 workq_thread_needs_params_change(workq_threadreq_t req, struct uthread *uth)  in workq_thread_needs_params_change()  argument
552 	cur_trp.trp_value = uth->uu_save.uus_workq_park_data.workloop_params;  in workq_thread_needs_params_change()
584 workq_thread_needs_priority_change(workq_threadreq_t req, struct uthread *uth)  in workq_thread_needs_priority_change()  argument
586 	if (workq_thread_needs_params_change(req, uth)) {  in workq_thread_needs_priority_change()
590 	if (req->tr_qos != workq_pri_override(uth->uu_workq_pri)) {  in workq_thread_needs_priority_change()
618 workq_thread_update_bucket(proc_t p, struct workqueue *wq, struct uthread *uth,  in workq_thread_update_bucket()  argument
622 	assert(uth == current_uthread());  in workq_thread_update_bucket()
628 	    !workq_thread_is_permanently_bound(uth)) {  in workq_thread_update_bucket()
633 	uth->uu_workq_pri = new_pri;  in workq_thread_update_bucket()
636 		thread_set_workq_override(get_machthread(uth), new_pri.qos_override);  in workq_thread_update_bucket()
640 	    !workq_thread_is_permanently_bound(uth) &&  in workq_thread_update_bucket()
664 workq_thread_reset_cpupercent(workq_threadreq_t req, struct uthread *uth)  in workq_thread_reset_cpupercent()  argument
666 	assert(uth == current_uthread());  in workq_thread_reset_cpupercent()
673 	if (uth->uu_workq_flags & UT_WORKQ_CPUPERCENT) {  in workq_thread_reset_cpupercent()
680 		uth->uu_workq_flags &= ~UT_WORKQ_CPUPERCENT;  in workq_thread_reset_cpupercent()
686 		uth->uu_workq_flags |= UT_WORKQ_CPUPERCENT;  in workq_thread_reset_cpupercent()
696 workq_thread_reset_pri(struct workqueue *wq, struct uthread *uth,  in workq_thread_reset_pri()  argument
699 	thread_t th = get_machthread(uth);  in workq_thread_reset_pri()
709 	uth->uu_workq_pri = WORKQ_POLICY_INIT(qos);  in workq_thread_reset_pri()
710 	uth->uu_workq_flags &= ~UT_WORKQ_OUTSIDE_QOS;  in workq_thread_reset_pri()
713 		uth->uu_save.uus_workq_park_data.workloop_params = trp.trp_value;  in workq_thread_reset_pri()
715 		uth->uu_save.uus_workq_park_data.qos = qos;  in workq_thread_reset_pri()
734 			uth->uu_workq_flags |= UT_WORKQ_OUTSIDE_QOS;  in workq_thread_reset_pri()
832 	struct uthread *uth = current_uthread();  in workq_thread_set_max_qos()  local
836 	if (uth->uu_workq_pri.qos_max == qos) {  in workq_thread_set_max_qos()
841 	old_pri = new_pri = uth->uu_workq_pri;  in workq_thread_set_max_qos()
843 	workq_thread_update_bucket(p, wq, uth, old_pri, new_pri, false);  in workq_thread_set_max_qos()
852 	struct uthread *uth = TAILQ_LAST(&wq->wq_thidlelist, workq_uthread_head);  in workq_oldest_killable_idle_thread()  local
854 	if (uth && !uth->uu_save.uus_workq_park_data.has_stack) {  in workq_oldest_killable_idle_thread()
855 		uth = TAILQ_PREV(uth, workq_uthread_head, uu_workq_entry);  in workq_oldest_killable_idle_thread()
856 		if (uth) {  in workq_oldest_killable_idle_thread()
857 			assert(uth->uu_save.uus_workq_park_data.has_stack);  in workq_oldest_killable_idle_thread()
860 	return uth;  in workq_oldest_killable_idle_thread()
886 workq_should_kill_idle_thread(struct workqueue *wq, struct uthread *uth,  in workq_should_kill_idle_thread()  argument
890 	return now - uth->uu_save.uus_workq_park_data.idle_stamp > delay;  in workq_should_kill_idle_thread()
924 	struct uthread *uth;  in workq_death_policy_evaluate()  local
935 	if ((uth = workq_oldest_killable_idle_thread(wq)) == NULL) {  in workq_death_policy_evaluate()
942 	if (now - uth->uu_save.uus_workq_park_data.idle_stamp > delay) {  in workq_death_policy_evaluate()
946 		uth->uu_workq_flags |= UT_WORKQ_DYING;  in workq_death_policy_evaluate()
947 		if ((uth->uu_workq_flags & UT_WORKQ_IDLE_CLEANUP) == 0) {  in workq_death_policy_evaluate()
948 			workq_thread_wakeup(uth);  in workq_death_policy_evaluate()
954 	    uth->uu_save.uus_workq_park_data.idle_stamp + delay);  in workq_death_policy_evaluate()
958 workq_thread_terminate(struct proc *p, struct uthread *uth)  in workq_thread_terminate()  argument
963 	if (!workq_thread_is_permanently_bound(uth)) {  in workq_thread_terminate()
964 		TAILQ_REMOVE(&wq->wq_thrunlist, uth, uu_workq_entry);  in workq_thread_terminate()
965 		if (uth->uu_workq_flags & UT_WORKQ_DYING) {  in workq_thread_terminate()
982 	thread_deallocate(get_machthread(uth));  in workq_thread_terminate()
1002 	struct uthread *uth;  in workq_pop_idle_thread()  local
1004 	if ((uth = TAILQ_FIRST(&wq->wq_thidlelist))) {  in workq_pop_idle_thread()
1005 		TAILQ_REMOVE(&wq->wq_thidlelist, uth, uu_workq_entry);  in workq_pop_idle_thread()
1007 		uth = TAILQ_FIRST(&wq->wq_thnewlist);  in workq_pop_idle_thread()
1008 		TAILQ_REMOVE(&wq->wq_thnewlist, uth, uu_workq_entry);  in workq_pop_idle_thread()
1010 	TAILQ_INSERT_TAIL(&wq->wq_thrunlist, uth, uu_workq_entry);  in workq_pop_idle_thread()
1012 	assert((uth->uu_workq_flags & UT_WORKQ_RUNNING) == 0);  in workq_pop_idle_thread()
1013 	uth->uu_workq_flags |= UT_WORKQ_RUNNING | uu_flags;  in workq_pop_idle_thread()
1024 	if (__improbable(uth->uu_workq_flags & UT_WORKQ_DYING)) {  in workq_pop_idle_thread()
1025 		uth->uu_workq_flags ^= UT_WORKQ_DYING;  in workq_pop_idle_thread()
1028 	} else if (uth->uu_workq_flags & UT_WORKQ_IDLE_CLEANUP) {  in workq_pop_idle_thread()
1033 	return uth;  in workq_pop_idle_thread()
1043 	struct uthread *uth = get_bsdthread_info(th);  in workq_thread_init_and_wq_lock()  local
1045 	uth->uu_workq_flags = UT_WORKQ_NEW;  in workq_thread_init_and_wq_lock()
1046 	uth->uu_workq_pri = WORKQ_POLICY_INIT(THREAD_QOS_LEGACY);  in workq_thread_init_and_wq_lock()
1047 	uth->uu_workq_thport = MACH_PORT_NULL;  in workq_thread_init_and_wq_lock()
1048 	uth->uu_workq_stackaddr = 0;  in workq_thread_init_and_wq_lock()
1049 	uth->uu_workq_pthread_kill_allowed = 0;  in workq_thread_init_and_wq_lock()
1055 	return workq_parked_wait_event(uth);  in workq_thread_init_and_wq_lock()
1104 	struct uthread *uth = get_bsdthread_info(th);  in workq_add_new_idle_thread()  local
1105 	uth->uu_workq_stackaddr = (user_addr_t)th_stackaddr;  in workq_add_new_idle_thread()
1110 		TAILQ_INSERT_TAIL(&wq->wq_thnewlist, uth, uu_workq_entry);  in workq_add_new_idle_thread()
1132 workq_thread_is_overcommit(struct uthread *uth)  in workq_thread_is_overcommit()  argument
1134 	return (uth->uu_workq_flags & UT_WORKQ_OVERCOMMIT) != 0;  in workq_thread_is_overcommit()
1138 workq_thread_is_nonovercommit(struct uthread *uth)  in workq_thread_is_nonovercommit()  argument
1140 	return (uth->uu_workq_flags & (UT_WORKQ_OVERCOMMIT |  in workq_thread_is_nonovercommit()
1145 workq_thread_is_cooperative(struct uthread *uth)  in workq_thread_is_cooperative()  argument
1147 	return (uth->uu_workq_flags & UT_WORKQ_COOPERATIVE) != 0;  in workq_thread_is_cooperative()
1151 workq_thread_is_permanently_bound(struct uthread *uth)  in workq_thread_is_permanently_bound()  argument
1153 	return (uth->uu_workq_flags & UT_WORKQ_PERMANENT_BIND) != 0;  in workq_thread_is_permanently_bound()
1157 workq_thread_set_type(struct uthread *uth, uint16_t flags)  in workq_thread_set_type()  argument
1159 	uth->uu_workq_flags &= ~(UT_WORKQ_OVERCOMMIT | UT_WORKQ_COOPERATIVE);  in workq_thread_set_type()
1160 	uth->uu_workq_flags |= flags;  in workq_thread_set_type()
1169     struct uthread *uth, uint32_t death_flags, uint32_t setup_flags)  in workq_unpark_for_death_and_unlock()  argument
1171 	thread_qos_t qos = workq_pri_override(uth->uu_workq_pri);  in workq_unpark_for_death_and_unlock()
1172 	bool first_use = uth->uu_workq_flags & UT_WORKQ_NEW;  in workq_unpark_for_death_and_unlock()
1175 		workq_thread_reset_pri(wq, uth, NULL, /*unpark*/ true);  in workq_unpark_for_death_and_unlock()
1179 	workq_thread_reset_cpupercent(NULL, uth);  in workq_unpark_for_death_and_unlock()
1184 			TAILQ_REMOVE(&wq->wq_thnewlist, uth, uu_workq_entry);  in workq_unpark_for_death_and_unlock()
1186 			TAILQ_REMOVE(&wq->wq_thidlelist, uth, uu_workq_entry);  in workq_unpark_for_death_and_unlock()
1189 	TAILQ_INSERT_TAIL(&wq->wq_thrunlist, uth, uu_workq_entry);  in workq_unpark_for_death_and_unlock()
1200 	thread_t th = get_machthread(uth);  in workq_unpark_for_death_and_unlock()
1207 	pthread_functions->workq_setup_thread(p, th, vmap, uth->uu_workq_stackaddr,  in workq_unpark_for_death_and_unlock()
1208 	    uth->uu_workq_thport, 0, WQ_SETUP_EXIT_THREAD, flags);  in workq_unpark_for_death_and_unlock()
1247 workq_push_idle_thread(proc_t p, struct workqueue *wq, struct uthread *uth,  in workq_push_idle_thread()  argument
1251 	bool is_creator = (uth == wq->wq_creator);  in workq_push_idle_thread()
1253 	if (workq_thread_is_cooperative(uth)) {  in workq_push_idle_thread()
1256 		thread_qos_t thread_qos = uth->uu_workq_pri.qos_req;  in workq_push_idle_thread()
1266 	} else if (workq_thread_is_nonovercommit(uth)) {  in workq_push_idle_thread()
1272 	uth->uu_workq_flags &= ~(UT_WORKQ_RUNNING | UT_WORKQ_OVERCOMMIT | UT_WORKQ_COOPERATIVE);  in workq_push_idle_thread()
1273 	TAILQ_REMOVE(&wq->wq_thrunlist, uth, uu_workq_entry);  in workq_push_idle_thread()
1279 		    uth->uu_save.uus_workq_park_data.yields);  in workq_push_idle_thread()
1282 	if (wq->wq_inheritor == get_machthread(uth)) {  in workq_push_idle_thread()
1291 	if (uth->uu_workq_flags & UT_WORKQ_NEW) {  in workq_push_idle_thread()
1293 		TAILQ_INSERT_TAIL(&wq->wq_thnewlist, uth, uu_workq_entry);  in workq_push_idle_thread()
1299 		_wq_thactive_dec(wq, uth->uu_workq_pri.qos_bucket);  in workq_push_idle_thread()
1300 		wq->wq_thscheduled_count[_wq_bucket(uth->uu_workq_pri.qos_bucket)]--;  in workq_push_idle_thread()
1301 		uth->uu_workq_flags |= UT_WORKQ_IDLE_CLEANUP;  in workq_push_idle_thread()
1304 	uth->uu_save.uus_workq_park_data.idle_stamp = now;  in workq_push_idle_thread()
1329 		uth->uu_workq_flags |= UT_WORKQ_DYING;  in workq_push_idle_thread()
1330 		uth->uu_workq_flags &= ~UT_WORKQ_IDLE_CLEANUP;  in workq_push_idle_thread()
1331 		workq_unpark_for_death_and_unlock(p, wq, uth, 0, setup_flags);  in workq_push_idle_thread()
1342 		uth->uu_save.uus_workq_park_data.has_stack = false;  in workq_push_idle_thread()
1343 		TAILQ_INSERT_TAIL(&wq->wq_thidlelist, uth, uu_workq_entry);  in workq_push_idle_thread()
1345 		uth->uu_save.uus_workq_park_data.has_stack = true;  in workq_push_idle_thread()
1346 		TAILQ_INSERT_HEAD(&wq->wq_thidlelist, uth, uu_workq_entry);  in workq_push_idle_thread()
1563 workq_cooperative_allowance(struct workqueue *wq, thread_qos_t qos, struct uthread *uth,  in workq_cooperative_allowance()  argument
1572 	if (uth && workq_thread_is_cooperative(uth)) {  in workq_cooperative_allowance()
1574 		_wq_cooperative_queue_scheduled_count_dec(wq, uth->uu_workq_pri.qos_req);  in workq_cooperative_allowance()
1619 		_wq_cooperative_queue_scheduled_count_inc(wq, uth->uu_workq_pri.qos_req);  in workq_cooperative_allowance()
1944 	struct uthread *uth = get_bsdthread_info(thread);  in workq_sched_callback()  local
1946 	thread_qos_t req_qos, qos = uth->uu_workq_pri.qos_bucket;  in workq_sched_callback()
2265 	struct uthread *uth, *tmp;  in workq_exit()  local
2303 		TAILQ_FOREACH_SAFE(uth, &wq->wq_thrunlist, uu_workq_entry, tmp) {  in workq_exit()
2304 			thread_t mth = get_machthread(uth);  in workq_exit()
2324 bsdthread_part_of_cooperative_workqueue(struct uthread *uth)  in bsdthread_part_of_cooperative_workqueue()  argument
2326 	return (workq_thread_is_cooperative(uth) || workq_thread_is_nonovercommit(uth)) &&  in bsdthread_part_of_cooperative_workqueue()
2327 	       (uth->uu_workq_pri.qos_bucket != WORKQ_THREAD_QOS_MANAGER) &&  in bsdthread_part_of_cooperative_workqueue()
2328 	       (!workq_thread_is_permanently_bound(uth));  in bsdthread_part_of_cooperative_workqueue()
2348 	struct uthread *uth = get_bsdthread_info(th);  in bsdthread_set_self()  local
2362 		if (uth->uu_workq_pri.qos_bucket == WORKQ_THREAD_QOS_MANAGER) {  in bsdthread_set_self()
2367 		workq_threadreq_t kqr = uth->uu_kqr_bound;  in bsdthread_set_self()
2424 		} else if (uth->uu_workq_pri.qos_bucket == WORKQ_THREAD_QOS_MANAGER ||  in bsdthread_set_self()
2425 		    uth->uu_workq_pri.qos_bucket == WORKQ_THREAD_QOS_ABOVEUI) {  in bsdthread_set_self()
2446 			if (workq_thread_is_overcommit(uth) && _pthread_priority_is_cooperative(priority)) {  in bsdthread_set_self()
2449 			} else if (workq_thread_is_cooperative(uth) && !_pthread_priority_is_cooperative(priority)) {  in bsdthread_set_self()
2452 			} else if (workq_thread_is_nonovercommit(uth) && _pthread_priority_is_cooperative(priority)) {  in bsdthread_set_self()
2473 			old_pri = new_pri = uth->uu_workq_pri;  in bsdthread_set_self()
2495 			if (workq_thread_is_overcommit(uth) && _pthread_priority_is_nonovercommit(priority)) {  in bsdthread_set_self()
2496 				workq_thread_set_type(uth, 0);  in bsdthread_set_self()
2500 			} else if (workq_thread_is_nonovercommit(uth) && _pthread_priority_is_overcommit(priority)) {  in bsdthread_set_self()
2501 				workq_thread_set_type(uth, UT_WORKQ_OVERCOMMIT);  in bsdthread_set_self()
2505 			} else if (workq_thread_is_cooperative(uth)) {  in bsdthread_set_self()
2518 			workq_thread_update_bucket(p, wq, uth, old_pri, new_pri, force_run);  in bsdthread_set_self()
2525 			if (workq_thread_is_overcommit(uth)) {  in bsdthread_set_self()
2669 	struct uthread *uth = get_bsdthread_info(thread);  in workq_thread_add_dispatch_override()  local
2697 	old_pri = uth->uu_workq_pri;  in workq_thread_add_dispatch_override()
2703 		workq_thread_update_bucket(p, wq, uth, old_pri, new_pri, false);  in workq_thread_add_dispatch_override()
2705 		uth->uu_workq_pri.qos_override = qos_override;  in workq_thread_add_dispatch_override()
2724 	struct uthread *uth = get_bsdthread_info(thread);  in workq_thread_reset_dispatch_override()  local
2740 	old_pri = new_pri = uth->uu_workq_pri;  in workq_thread_reset_dispatch_override()
2742 	workq_thread_update_bucket(p, wq, uth, old_pri, new_pri, false);  in workq_thread_reset_dispatch_override()
2761 	struct uthread *uth = get_bsdthread_info(thread);  in workq_thread_allow_kill()  local
2762 	uth->uu_workq_pthread_kill_allowed = enable;  in workq_thread_allow_kill()
2891     struct uthread *uth, uint32_t setup_flags);
2895     struct uthread *uth, uint32_t setup_flags);
2897 static void workq_setup_and_run(proc_t p, struct uthread *uth, int flags) __dead2;
3006 			struct uthread *uth;  in workq_reqthreads()  local
3014 			uth = workq_pop_idle_thread(wq, uu_flags, &needs_wakeup);  in workq_reqthreads()
3018 			workq_thread_reset_pri(wq, uth, req, /*unpark*/ true);  in workq_reqthreads()
3021 			uth->uu_save.uus_workq_park_data.upcall_flags = upcall_flags;  in workq_reqthreads()
3022 			uth->uu_save.uus_workq_park_data.thread_request = req;  in workq_reqthreads()
3024 				workq_thread_wakeup(uth);  in workq_reqthreads()
3059 	struct uthread *uth = NULL;  in workq_kern_threadreq_initiate()  local
3091 		uth = current_uthread();  in workq_kern_threadreq_initiate()
3092 		assert(uth->uu_kqr_bound == NULL);  in workq_kern_threadreq_initiate()
3102 	if (uth && workq_threadreq_admissible(wq, uth, req)) {  in workq_kern_threadreq_initiate()
3106 		assert(uth != wq->wq_creator);  in workq_kern_threadreq_initiate()
3108 		if (uth->uu_workq_pri.qos_bucket != req->tr_qos) {  in workq_kern_threadreq_initiate()
3109 			_wq_thactive_move(wq, uth->uu_workq_pri.qos_bucket, req->tr_qos);  in workq_kern_threadreq_initiate()
3110 			workq_thread_reset_pri(wq, uth, req, /*unpark*/ false);  in workq_kern_threadreq_initiate()
3120 		kqueue_threadreq_bind(p, req, get_machthread(uth), 0);  in workq_kern_threadreq_initiate()
3254 workq_kern_bound_thread_reset_pri(workq_threadreq_t req, struct uthread *uth)  in workq_kern_bound_thread_reset_pri()  argument
3256 	assert(workq_thread_is_permanently_bound(uth));  in workq_kern_bound_thread_reset_pri()
3270 		workq_thread_reset_pri(wq, uth, req, /*unpark*/ true);  in workq_kern_bound_thread_reset_pri()
3272 		thread_qos_t qos = workq_pri_override(uth->uu_workq_pri);  in workq_kern_bound_thread_reset_pri()
3274 			workq_thread_reset_pri(wq, uth, NULL, /*unpark*/ true);  in workq_kern_bound_thread_reset_pri()
3276 			uth->uu_save.uus_workq_park_data.qos = qos;  in workq_kern_bound_thread_reset_pri()
3378 			struct uthread *uth = get_bsdthread_info(new_thread);  in workq_kern_threadreq_permanent_bind()  local
3380 				workq_thread_reset_pri(wq, uth, kqr, /*unpark*/ true);  in workq_kern_threadreq_permanent_bind()
3387 			kqueue_threadreq_bind_prepost(p, kqr, uth);  in workq_kern_threadreq_permanent_bind()
3388 			uth->uu_workq_flags |= UT_WORKQ_PERMANENT_BIND;  in workq_kern_threadreq_permanent_bind()
3407 	struct uthread *uth = get_bsdthread_info(kqr->tr_thread);  in workq_kern_bound_thread_wakeup()  local
3414 	assert((uth->uu_workq_flags & (UT_WORKQ_RUNNING | UT_WORKQ_DYING)) == 0);  in workq_kern_bound_thread_wakeup()
3417 		uth->uu_workq_flags |= UT_WORKQ_DYING;  in workq_kern_bound_thread_wakeup()
3419 		uth->uu_workq_flags |= UT_WORKQ_RUNNING;  in workq_kern_bound_thread_wakeup()
3422 	workq_thread_wakeup(uth);  in workq_kern_bound_thread_wakeup()
3434 	struct uthread *uth = get_bsdthread_info(kqr->tr_thread);  in workq_kern_bound_thread_park()  local
3435 	assert(uth == current_uthread());  in workq_kern_bound_thread_park()
3441 	uth->uu_workq_flags &= ~(UT_WORKQ_RUNNING);  in workq_kern_bound_thread_park()
3443 	thread_disarm_workqueue_quantum(get_machthread(uth));  in workq_kern_bound_thread_park()
3458 	    workq_parked_wait_event(uth), workq_bound_thread_unpark_continue);  in workq_kern_bound_thread_park()
3473 	struct uthread *uth = get_bsdthread_info(kqr->tr_thread);  in workq_kern_bound_thread_terminate()  local
3476 	assert(uth == current_uthread());  in workq_kern_bound_thread_terminate()
3486 		kr = kern_work_interval_join(get_machthread(uth), MACH_PORT_NULL);  in workq_kern_bound_thread_terminate()
3499 	upcall_flags |= uth->uu_save.uus_workq_park_data.qos |  in workq_kern_bound_thread_terminate()
3502 	thread_t th = get_machthread(uth);  in workq_kern_bound_thread_terminate()
3509 	pthread_functions->workq_setup_thread(p, th, vmap, uth->uu_workq_stackaddr,  in workq_kern_bound_thread_terminate()
3510 	    uth->uu_workq_thport, 0, WQ_SETUP_EXIT_THREAD, upcall_flags);  in workq_kern_bound_thread_terminate()
3536 	struct uthread *uth = get_bsdthread_info(thread);  in workq_kern_quantum_expiry_reevaluate()  local
3538 	if (uth->uu_workq_flags & UT_WORKQ_DYING) {  in workq_kern_quantum_expiry_reevaluate()
3546 	thread_qos_t qos = uth->uu_workq_pri.qos_bucket;  in workq_kern_quantum_expiry_reevaluate()
3566 	if (workq_thread_is_cooperative(uth)) {  in workq_kern_quantum_expiry_reevaluate()
3567 		if (!workq_cooperative_allowance(wq, qos, uth, false)) {  in workq_kern_quantum_expiry_reevaluate()
3576 	} else if (workq_thread_is_nonovercommit(uth)) {  in workq_kern_quantum_expiry_reevaluate()
3577 		if (!workq_constrained_allowance(wq, qos, uth, false, false)) {  in workq_kern_quantum_expiry_reevaluate()
3589 	thread_arm_workqueue_quantum(get_machthread(uth));  in workq_kern_quantum_expiry_reevaluate()
3607 	struct uthread *uth = get_bsdthread_info(th);  in workq_thread_return()  local
3608 	workq_threadreq_t kqr = uth->uu_kqr_bound;  in workq_thread_return()
3614 	    (uth->uu_workq_flags & UT_WORKQ_DYING)) {  in workq_thread_return()
3626 	sigset_t resettable = uth->uu_sigmask & ~p->p_workq_allow_sigmask;  in workq_thread_return()
3629 		uth->uu_sigmask |= ~workq_threadmask & ~p->p_workq_allow_sigmask;  in workq_thread_return()
3663 		if (uth->uu_workq_pri.qos_bucket == WORKQ_THREAD_QOS_MANAGER) {  in workq_thread_return()
3666 			if (workq_thread_is_overcommit(uth)) {  in workq_thread_return()
3669 			if (uth->uu_workq_flags & UT_WORKQ_OUTSIDE_QOS) {  in workq_thread_return()
3672 				upcall_flags |= uth->uu_workq_pri.qos_req |  in workq_thread_return()
3677 		    get_task_map(proc_task(p)), uth->uu_workq_stackaddr,  in workq_thread_return()
3678 		    uth->uu_workq_thport, eventlist, nevents, upcall_flags);  in workq_thread_return()
3680 			assert(uth->uu_kqr_bound == kqr);  in workq_thread_return()
3689 		assert(uth->uu_kqr_bound == NULL);  in workq_thread_return()
3697 	UU_KEVENT_HISTORY_WRITE_ENTRY(uth, { .uu_error = -1, });  in workq_thread_return()
3702 	uth->uu_save.uus_workq_park_data.workloop_params = trp.trp_value;  in workq_thread_return()
3703 	workq_select_threadreq_or_park_and_unlock(p, wq, uth,  in workq_thread_return()
3811 		struct uthread *uth = get_bsdthread_info(th);  in workq_kernreturn()  local
3813 		    (uth->uu_workq_flags & (UT_WORKQ_DYING | UT_WORKQ_OVERCOMMIT))) {  in workq_kernreturn()
3824 		bool should_narrow = !workq_constrained_allowance(wq, qos, uth, false, false);  in workq_kernreturn()
3874 workq_park_and_unlock(proc_t p, struct workqueue *wq, struct uthread *uth,  in workq_park_and_unlock()  argument
3877 	assert(uth == current_uthread());  in workq_park_and_unlock()
3878 	assert(uth->uu_kqr_bound == NULL);  in workq_park_and_unlock()
3879 	workq_push_idle_thread(p, wq, uth, setup_flags); // may not return  in workq_park_and_unlock()
3881 	workq_thread_reset_cpupercent(NULL, uth);  in workq_park_and_unlock()
3902 	thread_set_preadopt_thread_group(get_machthread(uth), NULL);  in workq_park_and_unlock()
3905 	if ((uth->uu_workq_flags & UT_WORKQ_IDLE_CLEANUP) &&  in workq_park_and_unlock()
3906 	    !(uth->uu_workq_flags & UT_WORKQ_DYING)) {  in workq_park_and_unlock()
3913 		if (!uth->uu_save.uus_workq_park_data.has_stack) {  in workq_park_and_unlock()
3915 			    get_machthread(uth), get_task_map(proc_task(p)),  in workq_park_and_unlock()
3916 			    uth->uu_workq_stackaddr);  in workq_park_and_unlock()
3935 		uth->uu_workq_flags &= ~UT_WORKQ_IDLE_CLEANUP;  in workq_park_and_unlock()
3941 	if (uth->uu_workq_flags & UT_WORKQ_RUNNING) {  in workq_park_and_unlock()
3948 		workq_unpark_select_threadreq_or_park_and_unlock(p, wq, uth, setup_flags);  in workq_park_and_unlock()
3952 	if (uth->uu_workq_flags & UT_WORKQ_DYING) {  in workq_park_and_unlock()
3953 		workq_unpark_for_death_and_unlock(p, wq, uth,  in workq_park_and_unlock()
3959 	thread_disarm_workqueue_quantum(get_machthread(uth));  in workq_park_and_unlock()
3961 	thread_set_pending_block_hint(get_machthread(uth), kThreadWaitParkedWorkQueue);  in workq_park_and_unlock()
3962 	assert_wait(workq_parked_wait_event(uth), THREAD_INTERRUPTIBLE);  in workq_park_and_unlock()
3969 workq_may_start_event_mgr_thread(struct workqueue *wq, struct uthread *uth)  in workq_may_start_event_mgr_thread()  argument
3977 	       (uth && uth->uu_workq_pri.qos_bucket == WORKQ_THREAD_QOS_MANAGER);  in workq_may_start_event_mgr_thread()
3983     struct uthread *uth, bool may_start_timer, bool record_failed_allowance)  in workq_constrained_allowance()  argument
3990 	if (uth && workq_thread_is_nonovercommit(uth)) {  in workq_constrained_allowance()
4027 	if (uth && uth->uu_workq_pri.qos_bucket != WORKQ_THREAD_QOS_MANAGER &&  in workq_constrained_allowance()
4028 	    at_qos <= uth->uu_workq_pri.qos_bucket) {  in workq_constrained_allowance()
4067 workq_threadreq_admissible(struct workqueue *wq, struct uthread *uth,  in workq_threadreq_admissible()  argument
4071 		return workq_may_start_event_mgr_thread(wq, uth);  in workq_threadreq_admissible()
4074 		return workq_cooperative_allowance(wq, req->tr_qos, uth, true);  in workq_threadreq_admissible()
4077 		return workq_constrained_allowance(wq, req->tr_qos, uth, true, true);  in workq_threadreq_admissible()
4088 workq_cooperative_queue_best_req(struct workqueue *wq, struct uthread *uth)  in workq_cooperative_queue_best_req()  argument
4098 	if (uth && workq_thread_is_cooperative(uth)) {  in workq_cooperative_queue_best_req()
4099 		_wq_cooperative_queue_scheduled_count_dec(wq, uth->uu_workq_pri.qos_req);  in workq_cooperative_queue_best_req()
4103 		_wq_cooperative_queue_scheduled_count_inc(wq, uth->uu_workq_pri.qos_req);  in workq_cooperative_queue_best_req()
4252     struct uthread *uth, thread_qos_t old_thread_qos, workq_tr_flags_t tr_flags)  in workq_adjust_cooperative_constrained_schedule_counts()  argument
4270 	thread_qos_t new_thread_qos = uth->uu_workq_pri.qos_req;  in workq_adjust_cooperative_constrained_schedule_counts()
4279 	if (workq_thread_is_overcommit(uth)) {  in workq_adjust_cooperative_constrained_schedule_counts()
4288 	} else if (workq_thread_is_cooperative(uth)) {  in workq_adjust_cooperative_constrained_schedule_counts()
4319 workq_threadreq_select(struct workqueue *wq, struct uthread *uth)  in workq_threadreq_select()  argument
4326 	if (uth == wq->wq_creator) {  in workq_threadreq_select()
4327 		uth = NULL;  in workq_threadreq_select()
4362 	if (req_mgr && workq_may_start_event_mgr_thread(wq, uth)) {  in workq_threadreq_select()
4385 	req_tmp = workq_cooperative_queue_best_req(wq, uth);  in workq_threadreq_select()
4394 		if (workq_cooperative_allowance(wq, req_tmp->tr_qos, uth, true)) {  in workq_threadreq_select()
4416 		if (workq_constrained_allowance(wq, req_tmp->tr_qos, uth, true, true)) {  in workq_threadreq_select()
4458 	struct uthread *uth;  in workq_schedule_creator()  local
4465 	uth = wq->wq_creator;  in workq_schedule_creator()
4476 		if (uth == NULL) {  in workq_schedule_creator()
4493 		if (uth == NULL) {  in workq_schedule_creator()
4500 	if (uth) {  in workq_schedule_creator()
4504 		if (workq_thread_needs_priority_change(req, uth)) {  in workq_schedule_creator()
4506 			    wq, 1, uthread_tid(uth), req->tr_qos);  in workq_schedule_creator()
4507 			workq_thread_reset_pri(wq, uth, req, /*unpark*/ true);  in workq_schedule_creator()
4509 		assert(wq->wq_inheritor == get_machthread(uth));  in workq_schedule_creator()
4514 		wq->wq_creator = uth = workq_pop_idle_thread(wq, UT_WORKQ_OVERCOMMIT,  in workq_schedule_creator()
4517 		workq_thread_reset_pri(wq, uth, req, /*unpark*/ true);  in workq_schedule_creator()
4518 		workq_turnstile_update_inheritor(wq, get_machthread(uth),  in workq_schedule_creator()
4521 		    wq, 2, uthread_tid(uth), req->tr_qos);  in workq_schedule_creator()
4522 		uth->uu_save.uus_workq_park_data.fulfilled_snapshot = wq->wq_fulfilled;  in workq_schedule_creator()
4523 		uth->uu_save.uus_workq_park_data.yields = 0;  in workq_schedule_creator()
4525 			workq_thread_wakeup(uth);  in workq_schedule_creator()
4572     struct uthread *uth, uint32_t setup_flags)  in workq_select_threadreq_or_park_and_unlock()  argument
4575 	bool is_creator = (wq->wq_creator == uth);  in workq_select_threadreq_or_park_and_unlock()
4588 	req = workq_threadreq_select(wq, uth);  in workq_select_threadreq_or_park_and_unlock()
4594 	struct uu_workq_policy old_pri = uth->uu_workq_pri;  in workq_select_threadreq_or_park_and_unlock()
4611 			turnstile_update_inheritor(req_ts, get_machthread(uth),  in workq_select_threadreq_or_park_and_unlock()
4625 		    uth->uu_save.uus_workq_park_data.yields);  in workq_select_threadreq_or_park_and_unlock()
4632 	workq_thread_reset_pri(wq, uth, req, /*unpark*/ true);  in workq_select_threadreq_or_park_and_unlock()
4641 	    workq_adjust_cooperative_constrained_schedule_counts(wq, uth,  in workq_select_threadreq_or_park_and_unlock()
4645 		workq_thread_set_type(uth, UT_WORKQ_OVERCOMMIT);  in workq_select_threadreq_or_park_and_unlock()
4647 		workq_thread_set_type(uth, UT_WORKQ_COOPERATIVE);  in workq_select_threadreq_or_park_and_unlock()
4649 		workq_thread_set_type(uth, 0);  in workq_select_threadreq_or_park_and_unlock()
4652 	if (__improbable(thread_unfreeze_base_pri(get_machthread(uth)) && !is_creator)) {  in workq_select_threadreq_or_park_and_unlock()
4690 	workq_thread_reset_cpupercent(req, uth);  in workq_select_threadreq_or_park_and_unlock()
4693 		kqueue_threadreq_bind_prepost(p, req, uth);  in workq_select_threadreq_or_park_and_unlock()
4699 	if (uth->uu_workq_flags & UT_WORKQ_NEW) {  in workq_select_threadreq_or_park_and_unlock()
4700 		uth->uu_workq_flags ^= UT_WORKQ_NEW;  in workq_select_threadreq_or_park_and_unlock()
4727 	if (uth->uu_workq_pri.qos_bucket == WORKQ_THREAD_QOS_MANAGER) {  in workq_select_threadreq_or_park_and_unlock()
4742 	uth->uu_save.uus_workq_park_data.upcall_flags = upcall_flags;  in workq_select_threadreq_or_park_and_unlock()
4745 		kqueue_threadreq_bind_commit(p, get_machthread(uth));  in workq_select_threadreq_or_park_and_unlock()
4754 		thread_set_preadopt_thread_group(get_machthread(uth), NULL);  in workq_select_threadreq_or_park_and_unlock()
4758 	workq_setup_and_run(p, uth, setup_flags);  in workq_select_threadreq_or_park_and_unlock()
4762 	thread_unfreeze_base_pri(get_machthread(uth));  in workq_select_threadreq_or_park_and_unlock()
4764 	workq_park_and_unlock(p, wq, uth, setup_flags);  in workq_select_threadreq_or_park_and_unlock()
4786     struct uthread *uth, uint32_t setup_flags)  in workq_unpark_select_threadreq_or_park_and_unlock()  argument
4788 	if (uth->uu_workq_flags & UT_WORKQ_EARLY_BOUND) {  in workq_unpark_select_threadreq_or_park_and_unlock()
4789 		if (uth->uu_workq_flags & UT_WORKQ_NEW) {  in workq_unpark_select_threadreq_or_park_and_unlock()
4792 		uth->uu_workq_flags &= ~(UT_WORKQ_NEW | UT_WORKQ_EARLY_BOUND);  in workq_unpark_select_threadreq_or_park_and_unlock()
4796 		workq_threadreq_t req = uth->uu_save.uus_workq_park_data.thread_request;  in workq_unpark_select_threadreq_or_park_and_unlock()
4802 		workq_setup_and_run(p, uth, setup_flags);  in workq_unpark_select_threadreq_or_park_and_unlock()
4806 	thread_freeze_base_pri(get_machthread(uth));  in workq_unpark_select_threadreq_or_park_and_unlock()
4807 	workq_select_threadreq_or_park_and_unlock(p, wq, uth, setup_flags);  in workq_unpark_select_threadreq_or_park_and_unlock()
4811 workq_creator_should_yield(struct workqueue *wq, struct uthread *uth)  in workq_creator_should_yield()  argument
4813 	thread_qos_t qos = workq_pri_override(uth->uu_workq_pri);  in workq_creator_should_yield()
4819 	uint32_t snapshot = uth->uu_save.uus_workq_park_data.fulfilled_snapshot;  in workq_creator_should_yield()
4853 	struct uthread *uth = get_bsdthread_info(th);  in workq_unpark_continue()  local
4859 	if (wq->wq_creator == uth && workq_creator_should_yield(wq, uth)) {  in workq_unpark_continue()
4865 		uth->uu_save.uus_workq_park_data.fulfilled_snapshot = wq->wq_fulfilled;  in workq_unpark_continue()
4866 		uth->uu_save.uus_workq_park_data.yields++;  in workq_unpark_continue()
4872 	if (__probable(uth->uu_workq_flags & UT_WORKQ_RUNNING)) {  in workq_unpark_continue()
4873 		workq_unpark_select_threadreq_or_park_and_unlock(p, wq, uth, WQ_SETUP_NONE);  in workq_unpark_continue()
4881 		assert(uth->uu_workq_flags & UT_WORKQ_DYING);  in workq_unpark_continue()
4882 		assert((uth->uu_workq_flags & UT_WORKQ_NEW) == 0);  in workq_unpark_continue()
4893 		uth->uu_workq_flags |= UT_WORKQ_DYING;  in workq_unpark_continue()
4896 	workq_unpark_for_death_and_unlock(p, wq, uth,  in workq_unpark_continue()
4903 workq_setup_and_run(proc_t p, struct uthread *uth, int setup_flags)  in workq_setup_and_run()  argument
4905 	thread_t th = get_machthread(uth);  in workq_setup_and_run()
4927 	uint32_t upcall_flags = uth->uu_save.uus_workq_park_data.upcall_flags;  in workq_setup_and_run()
4932 	if (uth->uu_workq_flags & UT_WORKQ_OUTSIDE_QOS) {  in workq_setup_and_run()
4945 		upcall_flags |= uth->uu_save.uus_workq_park_data.qos |  in workq_setup_and_run()
4949 	if (uth->uu_workq_thport == MACH_PORT_NULL) {  in workq_setup_and_run()
4955 		uth->uu_workq_thport = ipc_port_copyout_send_pinned(port, get_task_ipcspace(proc_task(p)));  in workq_setup_and_run()
4973 	if (workq_thread_is_cooperative(uth) || workq_thread_is_permanently_bound(uth)) {  in workq_setup_and_run()
4979 	pthread_functions->workq_setup_thread(p, th, vmap, uth->uu_workq_stackaddr,  in workq_setup_and_run()
4980 	    uth->uu_workq_thport, 0, setup_flags, upcall_flags);  in workq_setup_and_run()
4990 workq_bound_thread_setup_and_run(struct uthread *uth, int setup_flags)  in workq_bound_thread_setup_and_run()  argument
4992 	struct workq_threadreq_s * kqr = uth->uu_kqr_bound;  in workq_bound_thread_setup_and_run()
4997 		workq_thread_set_type(uth, UT_WORKQ_OVERCOMMIT);  in workq_bound_thread_setup_and_run()
5000 	uth->uu_save.uus_workq_park_data.upcall_flags = upcall_flags;  in workq_bound_thread_setup_and_run()
5001 	workq_setup_and_run(current_proc(), uth, setup_flags);  in workq_bound_thread_setup_and_run()
5032 	struct uthread *uth = get_bsdthread_info(current_thread());  in workq_bound_thread_initialize_and_unpark_continue()  local
5036 		assert((uth->uu_workq_flags & (UT_WORKQ_RUNNING | UT_WORKQ_DYING))  in workq_bound_thread_initialize_and_unpark_continue()
5039 		assert(workq_thread_is_permanently_bound(uth));  in workq_bound_thread_initialize_and_unpark_continue()
5041 		if (uth->uu_workq_flags & UT_WORKQ_RUNNING) {  in workq_bound_thread_initialize_and_unpark_continue()
5042 			assert(uth->uu_workq_flags & UT_WORKQ_NEW);  in workq_bound_thread_initialize_and_unpark_continue()
5043 			uth->uu_workq_flags &= ~UT_WORKQ_NEW;  in workq_bound_thread_initialize_and_unpark_continue()
5045 			struct workq_threadreq_s * kqr = uth->uu_kqr_bound;  in workq_bound_thread_initialize_and_unpark_continue()
5048 				kr = kern_work_interval_explicit_join(get_machthread(uth),  in workq_bound_thread_initialize_and_unpark_continue()
5060 					uth->uu_workq_flags |= UT_WORKQ_WORK_INTERVAL_JOINED;  in workq_bound_thread_initialize_and_unpark_continue()
5063 					uth->uu_workq_flags |= UT_WORKQ_WORK_INTERVAL_FAILED;  in workq_bound_thread_initialize_and_unpark_continue()
5066 			workq_thread_reset_cpupercent(kqr, uth);  in workq_bound_thread_initialize_and_unpark_continue()
5067 			workq_bound_thread_setup_and_run(uth, WQ_SETUP_FIRST_USE);  in workq_bound_thread_initialize_and_unpark_continue()
5078 			assert(uth->uu_workq_flags & UT_WORKQ_DYING);  in workq_bound_thread_initialize_and_unpark_continue()
5096 		assert(workq_thread_is_permanently_bound(uth));  in workq_bound_thread_initialize_and_unpark_continue()
5104 		kqueue_threadreq_bind_commit(p, get_machthread(uth));  in workq_bound_thread_initialize_and_unpark_continue()
5106 	workq_kern_bound_thread_terminate(uth->uu_kqr_bound);  in workq_bound_thread_initialize_and_unpark_continue()
5117 	struct uthread *uth = get_bsdthread_info(current_thread());  in workq_bound_thread_unpark_continue()  local
5118 	assert(workq_thread_is_permanently_bound(uth));  in workq_bound_thread_unpark_continue()
5122 		assert((uth->uu_workq_flags & (UT_WORKQ_RUNNING | UT_WORKQ_DYING))  in workq_bound_thread_unpark_continue()
5124 		if (uth->uu_workq_flags & UT_WORKQ_RUNNING) {  in workq_bound_thread_unpark_continue()
5125 			workq_bound_thread_setup_and_run(uth, WQ_SETUP_NONE);  in workq_bound_thread_unpark_continue()
5127 			assert(uth->uu_workq_flags & UT_WORKQ_DYING);  in workq_bound_thread_unpark_continue()
5132 	workq_kern_bound_thread_terminate(uth->uu_kqr_bound);  in workq_bound_thread_unpark_continue()