93 static void workq_schedule_creator(proc_t p, struct workqueue *wq,
96 static bool workq_threadreq_admissible(struct workqueue *wq, struct uthread *uth,
99 static uint32_t workq_constrained_allowance(struct workqueue *wq,
103 static bool _wq_cooperative_queue_refresh_best_req_qos(struct workqueue *wq);
111 workq_schedule_delayed_thread_creation(struct workqueue *wq, int flags);
114 workq_lock_spin(struct workqueue *wq);
117 workq_unlock(struct workqueue *wq);
136     sizeof(struct workqueue), ZC_NONE);
160 wq_cooperative_queue_max_size(struct workqueue *wq)  in wq_cooperative_queue_max_size()
215 	struct workqueue *wq = proc_get_wqptr(p);
243 #define WQPTR_IS_INITING_VALUE ((struct workqueue *)~(uintptr_t)0)
245 static struct workqueue *
251 struct workqueue *
254 	struct workqueue *wq = proc_get_wqptr_fast(p);  in proc_get_wqptr()
259 proc_set_wqptr(struct proc *p, struct workqueue *wq)  in proc_set_wqptr()
272 	struct workqueue *wq;  in proc_init_wqptr_or_wait()
329 _wq_thactive(struct workqueue *wq)  in _wq_thactive()
351 _wq_thactive_best_constrained_req_qos(struct workqueue *wq)  in _wq_thactive_best_constrained_req_qos()
360 _wq_thactive_refresh_best_constrained_req_qos(struct workqueue *wq)  in _wq_thactive_refresh_best_constrained_req_qos()
395 _wq_thactive_inc(struct workqueue *wq, thread_qos_t qos)  in _wq_thactive_inc()
402 _wq_thactive_dec(struct workqueue *wq, thread_qos_t qos)  in _wq_thactive_dec()
409 _wq_thactive_move(struct workqueue *wq,  in _wq_thactive_move()
420 _wq_thactive_aggregate_downto_qos(struct workqueue *wq, wq_thactive_t v,  in _wq_thactive_aggregate_downto_qos()
461 _wq_cooperative_queue_scheduled_count_dec(struct workqueue *wq, thread_qos_t qos)  in _wq_cooperative_queue_scheduled_count_dec()
470 _wq_cooperative_queue_scheduled_count_inc(struct workqueue *wq, thread_qos_t qos)  in _wq_cooperative_queue_scheduled_count_inc()
479 _wq_flags(struct workqueue *wq)  in _wq_flags()
485 _wq_exiting(struct workqueue *wq)  in _wq_exiting()
493 	struct workqueue *wq = proc_get_wqptr(p);  in workq_is_exiting()
498 #pragma mark workqueue lock
501 workq_lock_is_acquired_kdp(struct workqueue *wq)  in workq_lock_is_acquired_kdp()
507 workq_lock_spin(struct workqueue *wq)  in workq_lock_spin()
513 workq_lock_held(struct workqueue *wq)  in workq_lock_held()
519 workq_lock_try(struct workqueue *wq)  in workq_lock_try()
525 workq_unlock(struct workqueue *wq)  in workq_unlock()
618 workq_thread_update_bucket(proc_t p, struct workqueue *wq, struct uthread *uth,  in workq_thread_update_bucket()
696 workq_thread_reset_pri(struct workqueue *wq, struct uthread *uth,  in workq_thread_reset_pri()
833 	struct workqueue *wq = proc_get_wqptr_fast(p);  in workq_thread_set_max_qos()
850 workq_oldest_killable_idle_thread(struct workqueue *wq)  in workq_oldest_killable_idle_thread()
864 workq_kill_delay_for_idle_thread(struct workqueue *wq)  in workq_kill_delay_for_idle_thread()
886 workq_should_kill_idle_thread(struct workqueue *wq, struct uthread *uth,  in workq_should_kill_idle_thread()
894 workq_death_call_schedule(struct workqueue *wq, uint64_t deadline)  in workq_death_call_schedule()
922 workq_death_policy_evaluate(struct workqueue *wq, uint16_t decrement)  in workq_death_policy_evaluate()
960 	struct workqueue *wq = proc_get_wqptr_fast(p);  in workq_thread_terminate()
988 	struct workqueue *wq = param0;  in workq_kill_old_threads_call()
999 workq_pop_idle_thread(struct workqueue *wq, uint16_t uu_flags,  in workq_pop_idle_thread()
1068 	struct workqueue  *wq,  in workq_add_new_idle_thread()
1168 workq_unpark_for_death_and_unlock(proc_t p, struct workqueue *wq,  in workq_unpark_for_death_and_unlock()
1213 workq_is_current_thread_updating_turnstile(struct workqueue *wq)  in workq_is_current_thread_updating_turnstile()
1220 workq_perform_turnstile_operation_locked(struct workqueue *wq,
1230 workq_turnstile_update_inheritor(struct workqueue *wq,  in workq_turnstile_update_inheritor()
1247 workq_push_idle_thread(proc_t p, struct workqueue *wq, struct uthread *uth,  in workq_push_idle_thread()
1395 workq_priority_queue_for_req(struct workqueue *wq, workq_threadreq_t req)  in workq_priority_queue_for_req()
1410 workq_num_cooperative_threads_scheduled_to_qos_internal(struct workqueue *wq, thread_qos_t qos)  in workq_num_cooperative_threads_scheduled_to_qos_internal()
1424 workq_num_cooperative_threads_scheduled_to_qos_locked(struct workqueue *wq, thread_qos_t qos)  in workq_num_cooperative_threads_scheduled_to_qos_locked()
1431 workq_num_cooperative_threads_scheduled_total(struct workqueue *wq)  in workq_num_cooperative_threads_scheduled_total()
1437 workq_has_cooperative_thread_requests(struct workqueue *wq)  in workq_has_cooperative_thread_requests()
1471 _wq_cooperative_queue_refresh_best_req_qos(struct workqueue *wq)  in _wq_cooperative_queue_refresh_best_req_qos()
1563 workq_cooperative_allowance(struct workqueue *wq, thread_qos_t qos, struct uthread *uth,  in workq_cooperative_allowance()
1629 workq_threadreq_enqueue(struct workqueue *wq, workq_threadreq_t req)  in workq_threadreq_enqueue()
1685 workq_threadreq_dequeue(struct workqueue *wq, workq_threadreq_t req,  in workq_threadreq_dequeue()
1767 #pragma mark workqueue thread creation thread calls
1770 workq_thread_call_prepost(struct workqueue *wq, uint32_t sched, uint32_t pend,  in workq_thread_call_prepost()
1792 workq_schedule_delayed_thread_creation(struct workqueue *wq, int flags)  in workq_schedule_delayed_thread_creation()
1831 workq_schedule_immediate_thread_creation(struct workqueue *wq)  in workq_schedule_immediate_thread_creation()
1850 	struct workqueue *wq = proc_get_wqptr(p);  in workq_proc_suspended()
1860 	struct workqueue *wq = proc_get_wqptr(p);  in workq_proc_resumed()
1907 	struct workqueue *wq = proc_get_wqptr(p);  in workq_add_new_threads_call()
1945 	struct workqueue *wq = proc_get_wqptr(tro->tro_proc);  in workq_sched_callback()
2036 workq_reference(struct workqueue *wq)  in workq_reference()
2045 	struct workqueue *wq;  in workq_deallocate_queue_invoke()
2048 	wq = mpsc_queue_element(e, struct workqueue, wq_destroy_link);  in workq_deallocate_queue_invoke()
2061 workq_deallocate(struct workqueue *wq)  in workq_deallocate()
2070 workq_deallocate_safe(struct workqueue *wq)  in workq_deallocate_safe()
2085 	struct workqueue *wq;  in workq_open()
2190 	struct workqueue *wq = proc_get_wqptr(p);  in workq_mark_exiting()
2264 	struct workqueue *wq;  in workq_exit()
2349 	struct workqueue *wq = proc_get_wqptr(p);  in bsdthread_set_self()
2656 	struct workqueue *wq = proc_get_wqptr(p);  in workq_thread_add_dispatch_override()
2723 	struct workqueue *wq = proc_get_wqptr(p);  in workq_thread_reset_dispatch_override()
2887 #pragma mark workqueue thread manipulation
2890 workq_unpark_select_threadreq_or_park_and_unlock(proc_t p, struct workqueue *wq,
2894 workq_select_threadreq_or_park_and_unlock(proc_t p, struct workqueue *wq,
2920 	struct workqueue *wq = proc_get_wqptr(p);  in workq_reqthreads()
3058 	struct workqueue *wq = proc_get_wqptr_fast(p);  in workq_kern_threadreq_initiate()
3151 	struct workqueue *wq = proc_get_wqptr_fast(p);  in workq_kern_threadreq_modify()
3267 	struct workqueue *wq = proc_get_wqptr_fast(current_proc());  in workq_kern_bound_thread_reset_pri()
3298 	struct workqueue *wq = proc_get_wqptr_fast(p);  in workq_kern_threadreq_update_inheritor()
3351 	struct workqueue *wq = proc_get_wqptr_fast(p);  in workq_kern_threadreq_permanent_bind()
3517 	struct workqueue *wq = proc_get_wqptr_fast(p);  in workq_kern_threadreq_redrive()
3553 	struct workqueue *wq = proc_get_wqptr(proc);  in workq_kern_quantum_expiry_reevaluate()
3593 workq_schedule_creator_turnstile_redrive(struct workqueue *wq, bool locked)  in workq_schedule_creator_turnstile_redrive()
3604     struct workqueue *wq)  in workq_thread_return()
3717 	struct workqueue *wq = proc_get_wqptr(p);  in workq_kernreturn()
3874 workq_park_and_unlock(proc_t p, struct workqueue *wq, struct uthread *uth,  in workq_park_and_unlock()
3969 workq_may_start_event_mgr_thread(struct workqueue *wq, struct uthread *uth)  in workq_may_start_event_mgr_thread()
3982 workq_constrained_allowance(struct workqueue *wq, thread_qos_t at_qos,  in workq_constrained_allowance()
4067 workq_threadreq_admissible(struct workqueue *wq, struct uthread *uth,  in workq_threadreq_admissible()
4088 workq_cooperative_queue_best_req(struct workqueue *wq, struct uthread *uth)  in workq_cooperative_queue_best_req()
4129 workq_threadreq_select_for_creator(struct workqueue *wq)  in workq_threadreq_select_for_creator()
4251 workq_adjust_cooperative_constrained_schedule_counts(struct workqueue *wq,  in workq_adjust_cooperative_constrained_schedule_counts()
4319 workq_threadreq_select(struct workqueue *wq, struct uthread *uth)  in workq_threadreq_select()
4454 workq_schedule_creator(proc_t p, struct workqueue *wq,  in workq_schedule_creator()
4571 workq_select_threadreq_or_park_and_unlock(proc_t p, struct workqueue *wq,  in workq_select_threadreq_or_park_and_unlock()
4785 workq_unpark_select_threadreq_or_park_and_unlock(proc_t p, struct workqueue *wq,  in workq_unpark_select_threadreq_or_park_and_unlock()
4811 workq_creator_should_yield(struct workqueue *wq, struct uthread *uth)  in workq_creator_should_yield()
4855 	struct workqueue *wq = proc_get_wqptr_fast(p);  in workq_unpark_continue()
5094 		struct workqueue *wq = proc_get_wqptr_fast(p);  in workq_bound_thread_initialize_and_unpark_continue()
5141 	struct workqueue *wq = proc_get_wqptr(p);  in fill_procworkqueue()
5243 	struct workqueue *wq = proc_get_wqptr(p);  in workqueue_get_task_ss_flags_from_pwq_state_kdp()