Lines Matching refs:sqd
30 void io_sq_thread_unpark(struct io_sq_data *sqd) in io_sq_thread_unpark() argument
31 __releases(&sqd->lock) in io_sq_thread_unpark()
33 WARN_ON_ONCE(sqd->thread == current); in io_sq_thread_unpark()
39 clear_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); in io_sq_thread_unpark()
40 if (atomic_dec_return(&sqd->park_pending)) in io_sq_thread_unpark()
41 set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); in io_sq_thread_unpark()
42 mutex_unlock(&sqd->lock); in io_sq_thread_unpark()
43 wake_up(&sqd->wait); in io_sq_thread_unpark()
46 void io_sq_thread_park(struct io_sq_data *sqd) in io_sq_thread_park() argument
47 __acquires(&sqd->lock) in io_sq_thread_park()
49 WARN_ON_ONCE(data_race(sqd->thread) == current); in io_sq_thread_park()
51 atomic_inc(&sqd->park_pending); in io_sq_thread_park()
52 set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); in io_sq_thread_park()
53 mutex_lock(&sqd->lock); in io_sq_thread_park()
54 if (sqd->thread) in io_sq_thread_park()
55 wake_up_process(sqd->thread); in io_sq_thread_park()
58 void io_sq_thread_stop(struct io_sq_data *sqd) in io_sq_thread_stop() argument
60 WARN_ON_ONCE(sqd->thread == current); in io_sq_thread_stop()
61 WARN_ON_ONCE(test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state)); in io_sq_thread_stop()
63 set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); in io_sq_thread_stop()
64 mutex_lock(&sqd->lock); in io_sq_thread_stop()
65 if (sqd->thread) in io_sq_thread_stop()
66 wake_up_process(sqd->thread); in io_sq_thread_stop()
67 mutex_unlock(&sqd->lock); in io_sq_thread_stop()
68 wait_for_completion(&sqd->exited); in io_sq_thread_stop()
71 void io_put_sq_data(struct io_sq_data *sqd) in io_put_sq_data() argument
73 if (refcount_dec_and_test(&sqd->refs)) { in io_put_sq_data()
74 WARN_ON_ONCE(atomic_read(&sqd->park_pending)); in io_put_sq_data()
76 io_sq_thread_stop(sqd); in io_put_sq_data()
77 kfree(sqd); in io_put_sq_data()
81 static __cold void io_sqd_update_thread_idle(struct io_sq_data *sqd) in io_sqd_update_thread_idle() argument
86 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) in io_sqd_update_thread_idle()
88 sqd->sq_thread_idle = sq_thread_idle; in io_sqd_update_thread_idle()
93 struct io_sq_data *sqd = ctx->sq_data; in io_sq_thread_finish() local
95 if (sqd) { in io_sq_thread_finish()
96 io_sq_thread_park(sqd); in io_sq_thread_finish()
98 io_sqd_update_thread_idle(sqd); in io_sq_thread_finish()
99 io_sq_thread_unpark(sqd); in io_sq_thread_finish()
101 io_put_sq_data(sqd); in io_sq_thread_finish()
109 struct io_sq_data *sqd; in io_attach_sq_data() local
118 sqd = ctx_attach->sq_data; in io_attach_sq_data()
119 if (!sqd) in io_attach_sq_data()
121 if (sqd->task_tgid != current->tgid) in io_attach_sq_data()
124 refcount_inc(&sqd->refs); in io_attach_sq_data()
125 return sqd; in io_attach_sq_data()
131 struct io_sq_data *sqd; in io_get_sq_data() local
135 sqd = io_attach_sq_data(p); in io_get_sq_data()
136 if (!IS_ERR(sqd)) { in io_get_sq_data()
138 return sqd; in io_get_sq_data()
141 if (PTR_ERR(sqd) != -EPERM) in io_get_sq_data()
142 return sqd; in io_get_sq_data()
145 sqd = kzalloc(sizeof(*sqd), GFP_KERNEL); in io_get_sq_data()
146 if (!sqd) in io_get_sq_data()
149 atomic_set(&sqd->park_pending, 0); in io_get_sq_data()
150 refcount_set(&sqd->refs, 1); in io_get_sq_data()
151 INIT_LIST_HEAD(&sqd->ctx_list); in io_get_sq_data()
152 mutex_init(&sqd->lock); in io_get_sq_data()
153 init_waitqueue_head(&sqd->wait); in io_get_sq_data()
154 init_completion(&sqd->exited); in io_get_sq_data()
155 return sqd; in io_get_sq_data()
158 static inline bool io_sqd_events_pending(struct io_sq_data *sqd) in io_sqd_events_pending() argument
160 return READ_ONCE(sqd->state); in io_sqd_events_pending()
201 static bool io_sqd_handle_event(struct io_sq_data *sqd) in io_sqd_handle_event() argument
206 if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) || in io_sqd_handle_event()
208 mutex_unlock(&sqd->lock); in io_sqd_handle_event()
211 wait_event(sqd->wait, !atomic_read(&sqd->park_pending)); in io_sqd_handle_event()
212 mutex_lock(&sqd->lock); in io_sqd_handle_event()
213 sqd->sq_cpu = raw_smp_processor_id(); in io_sqd_handle_event()
215 return did_sig || test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); in io_sqd_handle_event()
249 static void io_sq_update_worktime(struct io_sq_data *sqd, struct rusage *start) in io_sq_update_worktime() argument
257 sqd->work_time += end.ru_stime.tv_usec + end.ru_stime.tv_sec * 1000000; in io_sq_update_worktime()
263 struct io_sq_data *sqd = data; in io_sq_thread() local
272 mutex_lock(&sqd->lock); in io_sq_thread()
273 sqd->thread = NULL; in io_sq_thread()
274 mutex_unlock(&sqd->lock); in io_sq_thread()
278 snprintf(buf, sizeof(buf), "iou-sqp-%d", sqd->task_pid); in io_sq_thread()
282 sqd->task_pid = current->pid; in io_sq_thread()
284 if (sqd->sq_cpu != -1) { in io_sq_thread()
285 set_cpus_allowed_ptr(current, cpumask_of(sqd->sq_cpu)); in io_sq_thread()
288 sqd->sq_cpu = raw_smp_processor_id(); in io_sq_thread()
299 mutex_lock(&sqd->lock); in io_sq_thread()
303 if (io_sqd_events_pending(sqd) || signal_pending(current)) { in io_sq_thread()
304 if (io_sqd_handle_event(sqd)) in io_sq_thread()
306 timeout = jiffies + sqd->sq_thread_idle; in io_sq_thread()
309 cap_entries = !list_is_singular(&sqd->ctx_list); in io_sq_thread()
311 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { in io_sq_thread()
320 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) in io_sq_thread()
326 io_sq_update_worktime(sqd, &start); in io_sq_thread()
327 timeout = jiffies + sqd->sq_thread_idle; in io_sq_thread()
330 mutex_unlock(&sqd->lock); in io_sq_thread()
332 mutex_lock(&sqd->lock); in io_sq_thread()
333 sqd->sq_cpu = raw_smp_processor_id(); in io_sq_thread()
338 prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE); in io_sq_thread()
339 if (!io_sqd_events_pending(sqd) && !io_sq_tw_pending(retry_list)) { in io_sq_thread()
342 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { in io_sq_thread()
364 mutex_unlock(&sqd->lock); in io_sq_thread()
366 mutex_lock(&sqd->lock); in io_sq_thread()
367 sqd->sq_cpu = raw_smp_processor_id(); in io_sq_thread()
369 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) in io_sq_thread()
374 finish_wait(&sqd->wait, &wait); in io_sq_thread()
375 timeout = jiffies + sqd->sq_thread_idle; in io_sq_thread()
381 io_uring_cancel_generic(true, sqd); in io_sq_thread()
382 sqd->thread = NULL; in io_sq_thread()
383 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) in io_sq_thread()
386 mutex_unlock(&sqd->lock); in io_sq_thread()
388 complete(&sqd->exited); in io_sq_thread()
426 struct io_sq_data *sqd; in io_sq_offload_create() local
433 sqd = io_get_sq_data(p, &attached); in io_sq_offload_create()
434 if (IS_ERR(sqd)) { in io_sq_offload_create()
435 ret = PTR_ERR(sqd); in io_sq_offload_create()
440 ctx->sq_data = sqd; in io_sq_offload_create()
445 io_sq_thread_park(sqd); in io_sq_offload_create()
446 list_add(&ctx->sqd_list, &sqd->ctx_list); in io_sq_offload_create()
447 io_sqd_update_thread_idle(sqd); in io_sq_offload_create()
449 ret = (attached && !sqd->thread) ? -ENXIO : 0; in io_sq_offload_create()
450 io_sq_thread_unpark(sqd); in io_sq_offload_create()
474 sqd->sq_cpu = cpu; in io_sq_offload_create()
476 sqd->sq_cpu = -1; in io_sq_offload_create()
479 sqd->task_pid = current->pid; in io_sq_offload_create()
480 sqd->task_tgid = current->tgid; in io_sq_offload_create()
481 tsk = create_io_thread(io_sq_thread, sqd, NUMA_NO_NODE); in io_sq_offload_create()
487 sqd->thread = tsk; in io_sq_offload_create()
514 struct io_sq_data *sqd = ctx->sq_data; in io_sqpoll_wq_cpu_affinity() local
517 if (sqd) { in io_sqpoll_wq_cpu_affinity()
518 io_sq_thread_park(sqd); in io_sqpoll_wq_cpu_affinity()
520 if (sqd->thread) in io_sqpoll_wq_cpu_affinity()
521 ret = io_wq_cpu_affinity(sqd->thread->io_uring, mask); in io_sqpoll_wq_cpu_affinity()
522 io_sq_thread_unpark(sqd); in io_sqpoll_wq_cpu_affinity()