97     workq_threadreq_t req);
172 	int error = sysctl_handle_int(oidp, &v->usecs, 0, req);
173 	if (error || !req->newptr) {
195 	error = sysctl_io_number(req, 0, sizeof(int), &input_pool_size, &changed);
214 	proc_t p = req->p;
363 	workq_threadreq_t req;  in _wq_thactive_refresh_best_constrained_req_qos()  local
365 	req = priority_queue_max(&wq->wq_constrained_queue,  in _wq_thactive_refresh_best_constrained_req_qos()
367 	new_qos = req ? req->tr_qos : THREAD_QOS_UNSPECIFIED;  in _wq_thactive_refresh_best_constrained_req_qos()
536 workq_pri_bucket(struct uu_workq_policy req)  in workq_pri_bucket()  argument
538 	return MAX(MAX(req.qos_req, req.qos_max), req.qos_override);  in workq_pri_bucket()
542 workq_pri_override(struct uu_workq_policy req)  in workq_pri_override()  argument
544 	return MAX(workq_pri_bucket(req), req.qos_bucket);  in workq_pri_override()
548 workq_thread_needs_params_change(workq_threadreq_t req, struct uthread *uth)  in workq_thread_needs_params_change()  argument
553 	if (req->tr_flags & WORKQ_TR_FLAG_WL_PARAMS) {  in workq_thread_needs_params_change()
554 		req_trp = kqueue_threadreq_workloop_param(req);  in workq_thread_needs_params_change()
584 workq_thread_needs_priority_change(workq_threadreq_t req, struct uthread *uth)  in workq_thread_needs_priority_change()  argument
586 	if (workq_thread_needs_params_change(req, uth)) {  in workq_thread_needs_priority_change()
590 	if (req->tr_qos != workq_pri_override(uth->uu_workq_pri)) {  in workq_thread_needs_priority_change()
595 	thread_group_qos_t tg = kqr_preadopt_thread_group(req);  in workq_thread_needs_priority_change()
664 workq_thread_reset_cpupercent(workq_threadreq_t req, struct uthread *uth)  in workq_thread_reset_cpupercent()  argument
669 	if (req && (req->tr_flags & WORKQ_TR_FLAG_WL_PARAMS)) {  in workq_thread_reset_cpupercent()
670 		trp = kqueue_threadreq_workloop_param(req);  in workq_thread_reset_cpupercent()
697     workq_threadreq_t req, bool unpark)  in workq_thread_reset_pri()  argument
700 	thread_qos_t qos = req ? req->tr_qos : WORKQ_THREAD_QOS_CLEANUP;  in workq_thread_reset_pri()
705 	if (req && (req->tr_flags & WORKQ_TR_FLAG_WL_PARAMS)) {  in workq_thread_reset_pri()
706 		trp = kqueue_threadreq_workloop_param(req);  in workq_thread_reset_pri()
743 	if (req && (req->tr_flags & WORKQ_TR_FLAG_WORKLOOP)) {  in workq_thread_reset_pri()
766 		_Atomic(struct thread_group *) * tg_loc = kqr_preadopt_thread_group_addr(req);  in workq_thread_reset_pri()
1377 #define workq_threadreq_is_overcommit(req) workq_tr_is_overcommit((req)->tr_flags)  argument
1378 #define workq_threadreq_is_nonovercommit(req) workq_tr_is_nonovercommit((req)->tr_flags)  argument
1379 #define workq_threadreq_is_cooperative(req) workq_tr_is_cooperative((req)->tr_flags)  argument
1382 workq_priority_for_req(workq_threadreq_t req)  in workq_priority_for_req()  argument
1384 	thread_qos_t qos = req->tr_qos;  in workq_priority_for_req()
1386 	if (req->tr_flags & WORKQ_TR_FLAG_WL_OUTSIDE_QOS) {  in workq_priority_for_req()
1387 		workq_threadreq_param_t trp = kqueue_threadreq_workloop_param(req);  in workq_priority_for_req()
1395 workq_priority_queue_for_req(struct workqueue *wq, workq_threadreq_t req)  in workq_priority_queue_for_req()  argument
1397 	assert(!workq_tr_is_cooperative(req->tr_flags));  in workq_priority_queue_for_req()
1399 	if (req->tr_flags & WORKQ_TR_FLAG_WL_OUTSIDE_QOS) {  in workq_priority_queue_for_req()
1401 	} else if (workq_tr_is_overcommit(req->tr_flags)) {  in workq_priority_queue_for_req()
1629 workq_threadreq_enqueue(struct workqueue *wq, workq_threadreq_t req)  in workq_threadreq_enqueue()  argument
1631 	assert(req->tr_state == WORKQ_TR_STATE_NEW);  in workq_threadreq_enqueue()
1633 	req->tr_state = WORKQ_TR_STATE_QUEUED;  in workq_threadreq_enqueue()
1634 	wq->wq_reqcount += req->tr_count;  in workq_threadreq_enqueue()
1636 	if (req->tr_qos == WORKQ_THREAD_QOS_MANAGER) {  in workq_threadreq_enqueue()
1638 		assert(req->tr_flags & WORKQ_TR_FLAG_KEVENT);  in workq_threadreq_enqueue()
1639 		assert(req->tr_count == 1);  in workq_threadreq_enqueue()
1640 		wq->wq_event_manager_threadreq = req;  in workq_threadreq_enqueue()
1644 	if (workq_threadreq_is_cooperative(req)) {  in workq_threadreq_enqueue()
1645 		assert(req->tr_qos != WORKQ_THREAD_QOS_MANAGER);  in workq_threadreq_enqueue()
1646 		assert(req->tr_qos != WORKQ_THREAD_QOS_ABOVEUI);  in workq_threadreq_enqueue()
1648 		struct workq_threadreq_tailq *bucket = &wq->wq_cooperative_queue[_wq_bucket(req->tr_qos)];  in workq_threadreq_enqueue()
1649 		STAILQ_INSERT_TAIL(bucket, req, tr_link);  in workq_threadreq_enqueue()
1654 	struct priority_queue_sched_max *q = workq_priority_queue_for_req(wq, req);  in workq_threadreq_enqueue()
1656 	priority_queue_entry_set_sched_pri(q, &req->tr_entry,  in workq_threadreq_enqueue()
1657 	    workq_priority_for_req(req), false);  in workq_threadreq_enqueue()
1659 	if (priority_queue_insert(q, &req->tr_entry)) {  in workq_threadreq_enqueue()
1660 		if (workq_threadreq_is_nonovercommit(req)) {  in workq_threadreq_enqueue()
1685 workq_threadreq_dequeue(struct workqueue *wq, workq_threadreq_t req,  in workq_threadreq_dequeue()  argument
1692 	if (--req->tr_count == 0) {  in workq_threadreq_dequeue()
1693 		if (req->tr_qos == WORKQ_THREAD_QOS_MANAGER) {  in workq_threadreq_dequeue()
1694 			assert(wq->wq_event_manager_threadreq == req);  in workq_threadreq_dequeue()
1695 			assert(req->tr_count == 0);  in workq_threadreq_dequeue()
1708 		if (workq_threadreq_is_cooperative(req)) {  in workq_threadreq_dequeue()
1709 			assert(req->tr_qos != WORKQ_THREAD_QOS_MANAGER);  in workq_threadreq_dequeue()
1710 			assert(req->tr_qos != WORKQ_THREAD_QOS_ABOVEUI);  in workq_threadreq_dequeue()
1714 			assert(_wq_bucket(req->tr_qos) == _wq_bucket(wq->wq_cooperative_queue_best_req_qos));  in workq_threadreq_dequeue()
1716 			struct workq_threadreq_tailq *bucket = &wq->wq_cooperative_queue[_wq_bucket(req->tr_qos)];  in workq_threadreq_dequeue()
1719 			assert(head == req);  in workq_threadreq_dequeue()
1738 		if (!workq_threadreq_is_cooperative(req)) {  in workq_threadreq_dequeue()
1743 			if (priority_queue_remove(workq_priority_queue_for_req(wq, req),  in workq_threadreq_dequeue()
1744 			    &req->tr_entry)) {  in workq_threadreq_dequeue()
1746 				if (workq_threadreq_is_nonovercommit(req)) {  in workq_threadreq_dequeue()
1757 workq_threadreq_destroy(proc_t p, workq_threadreq_t req)  in workq_threadreq_destroy()  argument
1759 	req->tr_state = WORKQ_TR_STATE_CANCELED;  in workq_threadreq_destroy()
1760 	if (req->tr_flags & (WORKQ_TR_FLAG_WORKLOOP | WORKQ_TR_FLAG_KEVENT)) {  in workq_threadreq_destroy()
1761 		kqueue_threadreq_cancel(p, req);  in workq_threadreq_destroy()
1763 		zfree(workq_zone_threadreq, req);  in workq_threadreq_destroy()
2901 workq_trace_req_id(workq_threadreq_t req)  in workq_trace_req_id()  argument
2904 	if (req->tr_flags & WORKQ_TR_FLAG_WORKLOOP) {  in workq_trace_req_id()
2905 		kqwl = __container_of(req, struct kqworkloop, kqwl_request);  in workq_trace_req_id()
2909 	return VM_KERNEL_ADDRHIDE(req);  in workq_trace_req_id()
2933 	workq_threadreq_t req = zalloc(workq_zone_threadreq);  in workq_reqthreads()  local
2934 	priority_queue_entry_init(&req->tr_entry);  in workq_reqthreads()
2935 	req->tr_state = WORKQ_TR_STATE_NEW;  in workq_reqthreads()
2936 	req->tr_qos   = qos;  in workq_reqthreads()
2960 	req->tr_flags = tr_flags;  in workq_reqthreads()
2963 	    wq, workq_trace_req_id(req), req->tr_qos, reqcount);  in workq_reqthreads()
2986 			assert(!workq_threadreq_is_cooperative(req));  in workq_reqthreads()
2988 			if (workq_threadreq_is_nonovercommit(req)) {  in workq_reqthreads()
3000 		assert(!(req->tr_flags & WORKQ_TR_FLAG_WL_PARAMS));  in workq_reqthreads()
3010 			if (workq_tr_is_overcommit(req->tr_flags)) {  in workq_reqthreads()
3018 			workq_thread_reset_pri(wq, uth, req, /*unpark*/ true);  in workq_reqthreads()
3022 			uth->uu_save.uus_workq_park_data.thread_request = req;  in workq_reqthreads()
3037 	req->tr_count = (uint16_t)reqcount;  in workq_reqthreads()
3038 	if (workq_threadreq_enqueue(wq, req)) {  in workq_reqthreads()
3048 	zfree(workq_zone_threadreq, req);  in workq_reqthreads()
3054 workq_kern_threadreq_initiate(struct proc *p, workq_threadreq_t req,  in workq_kern_threadreq_initiate()  argument
3061 	assert(req->tr_flags & (WORKQ_TR_FLAG_WORKLOOP | WORKQ_TR_FLAG_KEVENT));  in workq_kern_threadreq_initiate()
3068 	if (req->tr_flags & WORKQ_TR_FLAG_WL_OUTSIDE_QOS) {  in workq_kern_threadreq_initiate()
3069 		workq_threadreq_param_t trp = kqueue_threadreq_workloop_param(req);  in workq_kern_threadreq_initiate()
3076 	assert(req->tr_state == WORKQ_TR_STATE_IDLE);  in workq_kern_threadreq_initiate()
3077 	priority_queue_entry_init(&req->tr_entry);  in workq_kern_threadreq_initiate()
3078 	req->tr_count = 1;  in workq_kern_threadreq_initiate()
3079 	req->tr_state = WORKQ_TR_STATE_NEW;  in workq_kern_threadreq_initiate()
3080 	req->tr_qos   = qos;  in workq_kern_threadreq_initiate()
3083 	    workq_trace_req_id(req), qos, 1);  in workq_kern_threadreq_initiate()
3097 		req->tr_state = WORKQ_TR_STATE_IDLE;  in workq_kern_threadreq_initiate()
3102 	if (uth && workq_threadreq_admissible(wq, uth, req)) {  in workq_kern_threadreq_initiate()
3108 		if (uth->uu_workq_pri.qos_bucket != req->tr_qos) {  in workq_kern_threadreq_initiate()
3109 			_wq_thactive_move(wq, uth->uu_workq_pri.qos_bucket, req->tr_qos);  in workq_kern_threadreq_initiate()
3110 			workq_thread_reset_pri(wq, uth, req, /*unpark*/ false);  in workq_kern_threadreq_initiate()
3117 		    workq_trace_req_id(req), req->tr_flags, 0);  in workq_kern_threadreq_initiate()
3120 		kqueue_threadreq_bind(p, req, get_machthread(uth), 0);  in workq_kern_threadreq_initiate()
3137 		if (workq_threadreq_enqueue(wq, req) || reevaluate_creator_thread_group) {  in workq_kern_threadreq_initiate()
3148 workq_kern_threadreq_modify(struct proc *p, workq_threadreq_t req,  in workq_kern_threadreq_modify()  argument
3154 	if (req->tr_flags & WORKQ_TR_FLAG_WL_OUTSIDE_QOS) {  in workq_kern_threadreq_modify()
3161 	assert(req->tr_qos != WORKQ_THREAD_QOS_MANAGER);  in workq_kern_threadreq_modify()
3162 	assert(req->tr_flags & (WORKQ_TR_FLAG_KEVENT | WORKQ_TR_FLAG_WORKLOOP));  in workq_kern_threadreq_modify()
3164 	if (req->tr_state == WORKQ_TR_STATE_BINDING) {  in workq_kern_threadreq_modify()
3165 		kqueue_threadreq_bind(p, req, req->tr_thread, 0);  in workq_kern_threadreq_modify()
3174 		assert(!workq_threadreq_is_cooperative(req));  in workq_kern_threadreq_modify()
3176 		make_overcommit = workq_threadreq_is_nonovercommit(req);  in workq_kern_threadreq_modify()
3179 	if (_wq_exiting(wq) || (req->tr_qos == qos && !make_overcommit)) {  in workq_kern_threadreq_modify()
3184 	assert(req->tr_count == 1);  in workq_kern_threadreq_modify()
3185 	if (req->tr_state != WORKQ_TR_STATE_QUEUED) {  in workq_kern_threadreq_modify()
3186 		panic("Invalid thread request (%p) state %d", req, req->tr_state);  in workq_kern_threadreq_modify()
3190 	    workq_trace_req_id(req), qos, 0);  in workq_kern_threadreq_modify()
3192 	struct priority_queue_sched_max *pq = workq_priority_queue_for_req(wq, req);  in workq_kern_threadreq_modify()
3201 	if (priority_queue_remove(pq, &req->tr_entry)) {  in workq_kern_threadreq_modify()
3202 		if (workq_threadreq_is_nonovercommit(req)) {  in workq_kern_threadreq_modify()
3214 		req->tr_flags ^= WORKQ_TR_FLAG_OVERCOMMIT;  in workq_kern_threadreq_modify()
3215 		pq = workq_priority_queue_for_req(wq, req);  in workq_kern_threadreq_modify()
3217 	req->tr_qos = qos;  in workq_kern_threadreq_modify()
3221 		priority_queue_entry_set_sched_pri(pq, &req->tr_entry,  in workq_kern_threadreq_modify()
3222 		    workq_priority_for_req(req), false);  in workq_kern_threadreq_modify()
3223 		priority_queue_insert(pq, &req->tr_entry);  in workq_kern_threadreq_modify()
3237 	req->tr_state = WORKQ_TR_STATE_NEW;  in workq_kern_threadreq_modify()
3247 	if (workq_threadreq_enqueue(wq, req) || reevaluate_creator_tg) {  in workq_kern_threadreq_modify()
3254 workq_kern_bound_thread_reset_pri(workq_threadreq_t req, struct uthread *uth)  in workq_kern_bound_thread_reset_pri()  argument
3258 	if (req && (req->tr_flags & WORKQ_TR_FLAG_WL_OUTSIDE_QOS)) {  in workq_kern_bound_thread_reset_pri()
3268 	if (req) {  in workq_kern_bound_thread_reset_pri()
3269 		assert(req->tr_qos != WORKQ_THREAD_QOS_MANAGER);  in workq_kern_bound_thread_reset_pri()
3270 		workq_thread_reset_pri(wq, uth, req, /*unpark*/ true);  in workq_kern_bound_thread_reset_pri()
3294 workq_kern_threadreq_update_inheritor(struct proc *p, workq_threadreq_t req,  in workq_kern_threadreq_update_inheritor()  argument
3301 	assert(req->tr_qos != WORKQ_THREAD_QOS_MANAGER);  in workq_kern_threadreq_update_inheritor()
3302 	assert(req->tr_flags & WORKQ_TR_FLAG_WORKLOOP);  in workq_kern_threadreq_update_inheritor()
3305 	if (req->tr_state == WORKQ_TR_STATE_BINDING) {  in workq_kern_threadreq_update_inheritor()
3306 		kqueue_threadreq_bind(p, req, req->tr_thread,  in workq_kern_threadreq_update_inheritor()
3314 		if (req->tr_state != WORKQ_TR_STATE_QUEUED) {  in workq_kern_threadreq_update_inheritor()
3315 			panic("Invalid thread request (%p) state %d", req, req->tr_state);  in workq_kern_threadreq_update_inheritor()
4068     workq_threadreq_t req)  in workq_threadreq_admissible()  argument
4070 	if (req->tr_qos == WORKQ_THREAD_QOS_MANAGER) {  in workq_threadreq_admissible()
4073 	if (workq_threadreq_is_cooperative(req)) {  in workq_threadreq_admissible()
4074 		return workq_cooperative_allowance(wq, req->tr_qos, uth, true);  in workq_threadreq_admissible()
4076 	if (workq_threadreq_is_nonovercommit(req)) {  in workq_threadreq_admissible()
4077 		return workq_constrained_allowance(wq, req->tr_qos, uth, true, true);  in workq_threadreq_admissible()
4457 	workq_threadreq_t req;  in workq_schedule_creator()  local
4482 	req = workq_threadreq_select_for_creator(wq);  in workq_schedule_creator()
4483 	if (req == NULL) {  in workq_schedule_creator()
4504 		if (workq_thread_needs_priority_change(req, uth)) {  in workq_schedule_creator()
4506 			    wq, 1, uthread_tid(uth), req->tr_qos);  in workq_schedule_creator()
4507 			workq_thread_reset_pri(wq, uth, req, /*unpark*/ true);  in workq_schedule_creator()
4517 		workq_thread_reset_pri(wq, uth, req, /*unpark*/ true);  in workq_schedule_creator()
4521 		    wq, 2, uthread_tid(uth), req->tr_qos);  in workq_schedule_creator()
4574 	workq_threadreq_t req = NULL;  in workq_select_threadreq_or_park_and_unlock()  local
4588 	req = workq_threadreq_select(wq, uth);  in workq_select_threadreq_or_park_and_unlock()
4589 	if (__improbable(req == NULL)) {  in workq_select_threadreq_or_park_and_unlock()
4595 	uint8_t tr_flags = req->tr_flags;  in workq_select_threadreq_or_park_and_unlock()
4596 	struct turnstile *req_ts = kqueue_threadreq_get_turnstile(req);  in workq_select_threadreq_or_park_and_unlock()
4627 		_wq_thactive_inc(wq, req->tr_qos);  in workq_select_threadreq_or_park_and_unlock()
4628 		wq->wq_thscheduled_count[_wq_bucket(req->tr_qos)]++;  in workq_select_threadreq_or_park_and_unlock()
4629 	} else if (old_pri.qos_bucket != req->tr_qos) {  in workq_select_threadreq_or_park_and_unlock()
4630 		_wq_thactive_move(wq, old_pri.qos_bucket, req->tr_qos);  in workq_select_threadreq_or_park_and_unlock()
4632 	workq_thread_reset_pri(wq, uth, req, /*unpark*/ true);  in workq_select_threadreq_or_park_and_unlock()
4673 			if (req->tr_qos == WORKQ_THREAD_QOS_MANAGER) {  in workq_select_threadreq_or_park_and_unlock()
4685 	    workq_trace_req_id(req), tr_flags, 0);  in workq_select_threadreq_or_park_and_unlock()
4687 	schedule_creator = workq_threadreq_dequeue(wq, req,  in workq_select_threadreq_or_park_and_unlock()
4690 	workq_thread_reset_cpupercent(req, uth);  in workq_select_threadreq_or_park_and_unlock()
4693 		kqueue_threadreq_bind_prepost(p, req, uth);  in workq_select_threadreq_or_park_and_unlock()
4694 		req = NULL;  in workq_select_threadreq_or_park_and_unlock()
4695 	} else if (req->tr_count > 0) {  in workq_select_threadreq_or_park_and_unlock()
4696 		req = NULL;  in workq_select_threadreq_or_park_and_unlock()
4719 	if (req) {  in workq_select_threadreq_or_park_and_unlock()
4720 		zfree(workq_zone_threadreq, req);  in workq_select_threadreq_or_park_and_unlock()
4796 		workq_threadreq_t req = uth->uu_save.uus_workq_park_data.thread_request;  in workq_unpark_select_threadreq_or_park_and_unlock()  local
4799 		    VM_KERNEL_ADDRHIDE(req), 0, 0);  in workq_unpark_select_threadreq_or_park_and_unlock()
4800 		(void)req;  in workq_unpark_select_threadreq_or_park_and_unlock()