1 /* 2 * Copyright (c) 2014 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifndef _WORKQUEUE_INTERNAL_H_ 30 #define _WORKQUEUE_INTERNAL_H_ 31 32 // Sometimes something gets passed a bucket number and we need a way to express 33 // that it's actually the event manager. Use the (0)th bucket for that. 34 #define WORKQ_THREAD_QOS_MIN (THREAD_QOS_MAINTENANCE) 35 #define WORKQ_THREAD_QOS_MAX (THREAD_QOS_LAST) 36 #define WORKQ_THREAD_QOS_CLEANUP (THREAD_QOS_LEGACY) 37 #define WORKQ_THREAD_QOS_ABOVEUI (THREAD_QOS_LAST) 38 #define WORKQ_THREAD_QOS_MANAGER (THREAD_QOS_LAST + 1) // outside of MIN/MAX 39 40 #define WORKQ_NUM_QOS_BUCKETS (WORKQ_THREAD_QOS_MAX - 1) // MT/BG shared 41 #define WORKQ_NUM_BUCKETS (WORKQ_NUM_QOS_BUCKETS + 1) // + mgr 42 43 /* These definitions are only available to the kext, to avoid bleeding 44 * constants and types across the boundary to the userspace library. 45 */ 46 #ifdef KERNEL 47 #pragma mark wq structs 48 49 /* These defines come from kern/thread.h but are XNU_KERNEL_PRIVATE so do not get 50 * exported to kernel extensions. 51 */ 52 #define SCHED_CALL_BLOCK 0x1 53 #define SCHED_CALL_UNBLOCK 0x2 54 55 /* old workq priority scheme */ 56 57 #define WORKQUEUE_HIGH_PRIOQUEUE 0 /* high priority queue */ 58 #define WORKQUEUE_DEFAULT_PRIOQUEUE 1 /* default priority queue */ 59 #define WORKQUEUE_LOW_PRIOQUEUE 2 /* low priority queue */ 60 #define WORKQUEUE_BG_PRIOQUEUE 3 /* background priority queue */ 61 62 /* wq_max_constrained_threads = max(64, N_CPU * WORKQUEUE_CONSTRAINED_FACTOR) 63 * This used to be WORKQ_NUM_BUCKETS + 1 when NUM_BUCKETS was 4, yielding 64 * N_CPU * 5. When NUM_BUCKETS changed, we decided that the limit should 65 * not change. So the factor is now always 5. 66 */ 67 #define WORKQUEUE_CONSTRAINED_FACTOR 5 68 69 #if BSD_KERNEL_PRIVATE 70 #include <kern/mpsc_queue.h> 71 #include <kern/priority_queue.h> 72 #include <kern/thread_call.h> 73 #include <kern/turnstile.h> 74 #include <mach/kern_return.h> 75 #include <sys/queue.h> 76 #include <sys/kernel_types.h> 77 78 /* struct uthread::uu_workq_flags */ 79 #define UT_WORKQ_NEW 0x01 /* First return to userspace */ 80 #define UT_WORKQ_RUNNING 0x02 /* On thrunlist, not parked. */ 81 #define UT_WORKQ_DYING 0x04 /* Thread is being killed */ 82 #define UT_WORKQ_OVERCOMMIT 0x08 /* Overcommit thread. */ 83 #define UT_WORKQ_OUTSIDE_QOS 0x10 /* Thread should avoid send QoS changes to kernel */ 84 #define UT_WORKQ_IDLE_CLEANUP 0x20 /* Thread is removing its voucher or stack */ 85 #define UT_WORKQ_EARLY_BOUND 0x40 /* Thread has been bound early */ 86 #define UT_WORKQ_CPUPERCENT 0x80 /* Thread has CPU percent policy active */ 87 #define UT_WORKQ_COOPERATIVE 0x100 /* Thread is part of cooperative pool */ 88 /* Thread is permanently bound to a thread request. This is a sticky flag. */ 89 #define UT_WORKQ_PERMANENT_BIND 0x200 90 #define UT_WORKQ_WORK_INTERVAL_JOINED 0x400 /* Thread has joined a work interval */ 91 #define UT_WORKQ_WORK_INTERVAL_FAILED 0x800 /* Thread has failed to join a work interval */ 92 93 typedef union workq_threadreq_param_s { 94 struct { 95 uint16_t trp_flags; 96 uint8_t trp_pri; 97 uint8_t trp_pol; 98 uint32_t trp_cpupercent: 8, 99 trp_refillms: 24; 100 }; 101 uint64_t trp_value; 102 } workq_threadreq_param_t; 103 104 #define TRP_PRIORITY 0x1 105 #define TRP_POLICY 0x2 106 #define TRP_CPUPERCENT 0x4 107 #define TRP_BOUND_THREAD 0x8 108 #define TRP_RELEASED 0x8000 109 110 struct workq_threadreq_extended_param_s { 111 struct work_interval *trp_work_interval; 112 #if CONFIG_PREADOPT_TG 113 struct thread_group *trp_permanent_preadopt_tg; 114 #endif 115 }; 116 117 /*! 118 * @enum workq_tr_state_t 119 * 120 * @brief 121 * This enum represents the state of a workq thread request. 122 * 123 * @discussion 124 * The states are used and set by both kevent and the workq subsystem under very 125 * precise locking domains. 126 * 127 * When for kevent requests, this structure is embedded on the kqueue itself, 128 * for non kevent related thread requests, it is allocated. 129 * 130 * Only the BINDING state isn't set under the kqlock, but then only QUEUED could 131 * be read by kqueue in its stead. 132 * 133 * @const WORKQ_TR_STATE_IDLE 134 * This thread request is idle. 135 * The state is only transient for non kevent thread requests. 136 * Set under the kqlock (kevent) or after allocation (workq). 137 * 138 * tr_entry/tr_thread are unused. 139 * 140 * @const WORKQ_TR_STATE_NEW 141 * This thread request is being initialized. This state is transient. 142 * Set workq lock for all kinds, set under the kqlock to for kevent requests. 143 * 144 * tr_entry is initialized, tr_thread is unused. 145 * 146 * @const WORKQ_TR_STATE_QUEUED 147 * This thread request has been pended, waiting for a thread to be bound. 148 * Set workq lock for all kinds, set under the kqlock to for kevent requests. 149 * 150 * tr_entry is used as linkage in a workq priority queue, tr_thread is unused. 151 * 152 * @const WORKQ_TR_STATE_CANCELED 153 * When the process exits, Queued thread requests are marked canceled. 154 * This happens under the workqueue lock. 155 * 156 * @const WORKQ_TR_STATE_BINDING (kevent only) 157 * A thread was found to bind to the thread request. 158 * The bind is preposted this way under the workq lock and will be 159 * acknowledged by the kevent subsystem. 160 * 161 * tr_entry is unused, tr_thread is the thread we're binding to. 162 * 163 * @const WORKQ_TR_STATE_BOUND (kevent only) 164 * A thread bind has been acknowledged by the kevent subsystem. 165 * This is always set under the kqlock, sometimes also under the workq lock. 166 * 167 * tr_entry is unused, tr_thread is the thread we're bound to. 168 * 169 */ 170 __enum_decl(workq_tr_state_t, uint8_t, { 171 WORKQ_TR_STATE_IDLE = 0, /* request isn't in flight */ 172 WORKQ_TR_STATE_NEW = 1, /* request is being initiated */ 173 WORKQ_TR_STATE_QUEUED = 2, /* request is being queued */ 174 WORKQ_TR_STATE_CANCELED = 3, /* request is canceled */ 175 WORKQ_TR_STATE_BINDING = 4, /* request is preposted for bind */ 176 WORKQ_TR_STATE_BOUND = 5, /* request is bound to a thread */ 177 }); 178 179 __options_decl(workq_tr_flags_t, uint8_t, { 180 WORKQ_TR_FLAG_KEVENT = 0x01, 181 WORKQ_TR_FLAG_WORKLOOP = 0x02, 182 WORKQ_TR_FLAG_OVERCOMMIT = 0x04, 183 WORKQ_TR_FLAG_WL_PARAMS = 0x08, 184 WORKQ_TR_FLAG_WL_OUTSIDE_QOS = 0x10, 185 WORKQ_TR_FLAG_COOPERATIVE = 0x20, 186 /* 187 * A workqueue thread request with this flag will be permanently 188 * bound to a newly created workqueue thread since the creation of 189 * the associated kqworkloop until its teardown. 190 */ 191 WORKQ_TR_FLAG_PERMANENT_BIND = 0x40, 192 }); 193 194 typedef struct workq_threadreq_s { 195 union { 196 struct priority_queue_entry_sched tr_entry; 197 STAILQ_ENTRY(workq_threadreq_s) tr_link; 198 struct { 199 thread_t tr_thread; 200 /* 201 * tr_work_interval is only used when TRP_BOUND_THREAD is also set. 202 */ 203 struct work_interval *tr_work_interval; 204 }; 205 }; 206 uint16_t tr_count; 207 workq_tr_flags_t tr_flags; 208 workq_tr_state_t tr_state; 209 thread_qos_t tr_qos; /* qos for the thread request */ 210 211 /* kqueue states, modified under the kqlock */ 212 kq_index_t tr_kq_override_index; /* highest wakeup override index */ 213 kq_index_t tr_kq_qos_index; /* QoS for the servicer */ 214 } workq_threadreq_s, *workq_threadreq_t; 215 216 STAILQ_HEAD(workq_threadreq_tailq, workq_threadreq_s); 217 218 #if defined(__LP64__) 219 typedef unsigned __int128 wq_thactive_t; 220 #else 221 typedef uint64_t wq_thactive_t; 222 #endif 223 224 __options_decl(workq_state_flags_t, uint32_t, { 225 WQ_EXITING = 0x0001, 226 WQ_PROC_SUSPENDED = 0x0002, 227 WQ_DEATH_CALL_SCHEDULED = 0x0004, 228 229 WQ_DELAYED_CALL_SCHEDULED = 0x0010, 230 WQ_DELAYED_CALL_PENDED = 0x0020, 231 WQ_IMMEDIATE_CALL_SCHEDULED = 0x0040, 232 WQ_IMMEDIATE_CALL_PENDED = 0x0080, 233 }); 234 235 TAILQ_HEAD(workq_uthread_head, uthread); 236 237 struct workqueue { 238 thread_call_t wq_delayed_call; 239 thread_call_t wq_immediate_call; 240 thread_call_t wq_death_call; 241 242 union { 243 struct turnstile *wq_turnstile; 244 struct mpsc_queue_chain wq_destroy_link; 245 }; 246 247 lck_ticket_t wq_lock; 248 249 uint64_t wq_thread_call_last_run; 250 struct os_refcnt wq_refcnt; 251 workq_state_flags_t _Atomic wq_flags; 252 uint32_t wq_fulfilled; 253 uint32_t wq_creations; 254 uint32_t wq_timer_interval; 255 uint32_t wq_event_manager_priority; 256 uint32_t wq_reqcount; /* number of elements on the wq_*_reqlists */ 257 uint16_t wq_thdying_count; 258 uint16_t wq_threads_scheduled; 259 uint16_t wq_constrained_threads_scheduled; 260 uint16_t wq_nthreads; 261 uint16_t wq_thidlecount; 262 uint16_t wq_thscheduled_count[WORKQ_NUM_BUCKETS]; // incl. manager 263 264 workq_threadreq_t wq_event_manager_threadreq; 265 266 _Atomic wq_thactive_t wq_thactive; 267 _Atomic uint64_t wq_lastblocked_ts[WORKQ_NUM_QOS_BUCKETS]; 268 269 struct proc *wq_proc; 270 struct uthread *wq_creator; 271 turnstile_inheritor_t wq_inheritor; 272 thread_t wq_turnstile_updater; // thread doing a turnstile_update_ineritor 273 struct workq_uthread_head wq_thrunlist; 274 struct workq_uthread_head wq_thnewlist; 275 struct workq_uthread_head wq_thidlelist; 276 277 struct priority_queue_sched_max wq_overcommit_queue; 278 struct priority_queue_sched_max wq_constrained_queue; 279 struct priority_queue_sched_max wq_special_queue; 280 281 // BG/MT, UT, DEF, IN, UI, AUI. No manager bucket for cooperative pool 282 uint8_t wq_cooperative_queue_scheduled_count[WORKQ_NUM_QOS_BUCKETS]; 283 uint16_t wq_cooperative_queue_best_req_qos: 3, /* UN means no request, returns BG for BG/MT bucket */ 284 wq_cooperative_queue_has_limited_max_size:1, /* if set, max size of cooperative pool per QoS is 1 */ 285 wq_exceeded_active_constrained_thread_limit:1, 286 unused:11; 287 struct workq_threadreq_tailq wq_cooperative_queue[WORKQ_NUM_QOS_BUCKETS]; 288 }; 289 290 #define WORKQUEUE_MAXTHREADS 512 291 #define WQ_STALLED_WINDOW_USECS 200 292 #define WQ_REDUCE_POOL_WINDOW_USECS 5000000 293 #define WQ_MAX_TIMER_INTERVAL_USECS 50000 294 295 #pragma mark definitions 296 297 struct workq_threadreq_s; 298 uint32_t _get_pwq_state_kdp(proc_t p); 299 300 void workq_exit(struct proc *p); 301 void workq_mark_exiting(struct proc *p); 302 303 bool workq_is_exiting(struct proc *p); 304 305 void workq_thread_set_max_qos(struct proc *p, struct workq_threadreq_s *kqr); 306 307 void workq_thread_terminate(struct proc *p, struct uthread *uth); 308 309 __options_decl(workq_kern_threadreq_flags_t, uint32_t, { 310 WORKQ_THREADREQ_NONE = 0x00, 311 WORKQ_THREADREQ_SET_AST_ON_FAILURE = 0x01, 312 WORKQ_THREADREQ_ATTEMPT_REBIND = 0x02, 313 WORKQ_THREADREQ_CAN_CREATE_THREADS = 0x04, 314 WORKQ_THREADREQ_MAKE_OVERCOMMIT = 0x08, 315 #if CONFIG_PREADOPT_TG 316 WORKQ_THREADREQ_REEVALUATE_PREADOPT_TG = 0x10, 317 #endif 318 }); 319 320 // called with the kq req lock held 321 bool workq_kern_threadreq_initiate(struct proc *p, struct workq_threadreq_s *kqr, 322 struct turnstile *ts, thread_qos_t qos, workq_kern_threadreq_flags_t flags); 323 324 // called with the kq req lock held 325 void workq_kern_threadreq_modify(struct proc *p, struct workq_threadreq_s *kqr, 326 thread_qos_t qos, workq_kern_threadreq_flags_t flags); 327 328 // called with the kq req lock held 329 void workq_kern_threadreq_update_inheritor(struct proc *p, struct workq_threadreq_s *kqr, 330 thread_t owner, struct turnstile *ts, turnstile_update_flags_t flags); 331 332 kern_return_t workq_kern_threadreq_permanent_bind(struct proc *p, struct workq_threadreq_s *kqr); 333 334 void workq_kern_bound_thread_wakeup(struct workq_threadreq_s *kqr); 335 336 void workq_kern_bound_thread_park(struct workq_threadreq_s *kqr); 337 338 void workq_kern_bound_thread_terminate(struct workq_threadreq_s *kqr); 339 340 bool workq_thread_is_permanently_bound(struct uthread *uth); 341 342 // called with the kq req lock held 343 void workq_kern_bound_thread_reset_pri(struct workq_threadreq_s *kqr, 344 struct uthread *uth); 345 346 void workq_kern_threadreq_lock(struct proc *p); 347 void workq_kern_threadreq_unlock(struct proc *p); 348 349 void workq_kern_threadreq_redrive(struct proc *p, workq_kern_threadreq_flags_t flags); 350 351 void workq_kern_quantum_expiry_reevaluate(struct proc *p, thread_t thread); 352 bool bsdthread_part_of_cooperative_workqueue(struct uthread *uth); 353 354 // This enum matches _pthread_set_flags in libpthread's qos_private.h 355 enum workq_set_self_flags { 356 WORKQ_SET_SELF_QOS_FLAG = 0x01, 357 WORKQ_SET_SELF_VOUCHER_FLAG = 0x02, 358 WORKQ_SET_SELF_FIXEDPRIORITY_FLAG = 0x04, 359 WORKQ_SET_SELF_TIMESHARE_FLAG = 0x08, 360 WORKQ_SET_SELF_WQ_KEVENT_UNBIND = 0x10, 361 WORKQ_SET_SELF_QOS_OVERRIDE_FLAG = 0x40, 362 }; 363 364 void workq_proc_suspended(struct proc *p); 365 void workq_proc_resumed(struct proc *p); 366 struct workqueue *proc_get_wqptr(struct proc *p); 367 368 #endif // BSD_KERNEL_PRIVATE 369 370 void workq_init(void); 371 372 #endif // KERNEL 373 374 #endif // _WORKQUEUE_INTERNAL_H_ 375