147 					 struct io_uring_task *tctx,
207 bool io_match_task_safe(struct io_kiocb *head, struct io_uring_task *tctx,  in io_match_task_safe()  argument
212 	if (tctx && head->tctx != tctx)  in io_match_task_safe()
419 		atomic_dec(&req->tctx->inflight_tracked);  in io_clean_op()
433 		atomic_inc(&req->tctx->inflight_tracked);  in io_req_track_inflight()
503 	struct io_uring_task *tctx = req->tctx;  in io_queue_iowq()  local
505 	BUG_ON(!tctx);  in io_queue_iowq()
507 	if ((current->flags & PF_KTHREAD) || !tctx->io_wq) {  in io_queue_iowq()
522 	if (WARN_ON_ONCE(!same_thread_group(tctx->task, current)))  in io_queue_iowq()
526 	io_wq_enqueue(tctx->io_wq, &req->work);  in io_queue_iowq()
671 	struct io_uring_task *tctx = req->tctx;  in io_put_task()  local
673 	if (likely(tctx->task == current)) {  in io_put_task()
674 		tctx->cached_refs++;  in io_put_task()
676 		percpu_counter_sub(&tctx->inflight, 1);  in io_put_task()
677 		if (unlikely(atomic_read(&tctx->in_cancel)))  in io_put_task()
678 			wake_up(&tctx->wait);  in io_put_task()
679 		put_task_struct(tctx->task);  in io_put_task()
683 void io_task_refs_refill(struct io_uring_task *tctx)  in io_task_refs_refill()  argument
685 	unsigned int refill = -tctx->cached_refs + IO_TCTX_REFS_CACHE_NR;  in io_task_refs_refill()
687 	percpu_counter_add(&tctx->inflight, refill);  in io_task_refs_refill()
689 	tctx->cached_refs += refill;  in io_task_refs_refill()
694 	struct io_uring_task *tctx = task->io_uring;  in io_uring_drop_tctx_refs()  local
695 	unsigned int refs = tctx->cached_refs;  in io_uring_drop_tctx_refs()
698 		tctx->cached_refs = 0;  in io_uring_drop_tctx_refs()
699 		percpu_counter_sub(&tctx->inflight, refs);  in io_uring_drop_tctx_refs()
1094 static void io_fallback_tw(struct io_uring_task *tctx, bool sync)  in io_fallback_tw()  argument
1096 	struct llist_node *node = llist_del_all(&tctx->task_list);  in io_fallback_tw()
1101 struct llist_node *tctx_task_work_run(struct io_uring_task *tctx,  in tctx_task_work_run()  argument
1108 		io_fallback_tw(tctx, true);  in tctx_task_work_run()
1112 	node = llist_del_all(&tctx->task_list);  in tctx_task_work_run()
1119 	if (unlikely(atomic_read(&tctx->in_cancel)))  in tctx_task_work_run()
1122 	trace_io_uring_task_work_run(tctx, *count);  in tctx_task_work_run()
1128 	struct io_uring_task *tctx;  in tctx_task_work()  local
1132 	tctx = container_of(cb, struct io_uring_task, task_work);  in tctx_task_work()
1133 	ret = tctx_task_work_run(tctx, UINT_MAX, &count);  in tctx_task_work()
1210 	struct io_uring_task *tctx = req->tctx;  in io_req_normal_work_add()  local
1214 	if (!llist_add(&req->io_task_work.node, &tctx->task_list))  in io_req_normal_work_add()
1222 		__set_notify_signal(tctx->task);  in io_req_normal_work_add()
1226 	if (likely(!task_work_add(tctx->task, &tctx->task_work, ctx->notify_method)))  in io_req_normal_work_add()
1229 	io_fallback_tw(tctx, false);  in io_req_normal_work_add()
2057 	req->tctx = current->io_uring;  in io_init_req()
2388 	struct io_uring_task *tctx = current->io_uring;  in current_pending_io()  local
2390 	if (!tctx)  in current_pending_io()
2392 	return percpu_counter_read_positive(&tctx->inflight);  in current_pending_io()
2843 	struct io_uring_task *tctx = current->io_uring;  in io_tctx_exit_cb()  local
2853 	if (tctx && !atomic_read(&tctx->in_cancel))  in io_tctx_exit_cb()
2999 	struct io_uring_task *tctx;  member
3008 	return io_match_task_safe(req, cancel->tctx, cancel->all);  in io_cancel_task_cb()
3012 					 struct io_uring_task *tctx,  in io_cancel_defer_files()  argument
3020 		if (io_match_task_safe(de->req, tctx, cancel_all)) {  in io_cancel_defer_files()
3046 		struct io_uring_task *tctx = node->task->io_uring;  in io_uring_try_cancel_iowq()  local
3052 		if (!tctx || !tctx->io_wq)  in io_uring_try_cancel_iowq()
3054 		cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_ctx_cb, ctx, true);  in io_uring_try_cancel_iowq()
3063 						struct io_uring_task *tctx,  in io_uring_try_cancel_requests()  argument
3067 	struct io_task_cancel cancel = { .tctx = tctx, .all = cancel_all, };  in io_uring_try_cancel_requests()
3081 	if (!tctx) {  in io_uring_try_cancel_requests()
3083 	} else if (tctx->io_wq) {  in io_uring_try_cancel_requests()
3088 		cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb,  in io_uring_try_cancel_requests()
3106 	ret |= io_cancel_defer_files(ctx, tctx, cancel_all);  in io_uring_try_cancel_requests()
3108 	ret |= io_poll_remove_all(ctx, tctx, cancel_all);  in io_uring_try_cancel_requests()
3109 	ret |= io_waitid_remove_all(ctx, tctx, cancel_all);  in io_uring_try_cancel_requests()
3110 	ret |= io_futex_remove_all(ctx, tctx, cancel_all);  in io_uring_try_cancel_requests()
3111 	ret |= io_uring_try_cancel_uring_cmd(ctx, tctx, cancel_all);  in io_uring_try_cancel_requests()
3113 	ret |= io_kill_timeouts(ctx, tctx, cancel_all);  in io_uring_try_cancel_requests()
3114 	if (tctx)  in io_uring_try_cancel_requests()
3121 static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked)  in tctx_inflight()  argument
3124 		return atomic_read(&tctx->inflight_tracked);  in tctx_inflight()
3125 	return percpu_counter_sum(&tctx->inflight);  in tctx_inflight()
3134 	struct io_uring_task *tctx = current->io_uring;  in io_uring_cancel_generic()  local
3145 	if (tctx->io_wq)  in io_uring_cancel_generic()
3146 		io_wq_exit_start(tctx->io_wq);  in io_uring_cancel_generic()
3148 	atomic_inc(&tctx->in_cancel);  in io_uring_cancel_generic()
3153 		if (!tctx_inflight(tctx, !cancel_all))  in io_uring_cancel_generic()
3157 		inflight = tctx_inflight(tctx, false);  in io_uring_cancel_generic()
3162 			xa_for_each(&tctx->xa, index, node) {  in io_uring_cancel_generic()
3184 		prepare_to_wait(&tctx->wait, &wait, TASK_INTERRUPTIBLE);  in io_uring_cancel_generic()
3187 		xa_for_each(&tctx->xa, index, node) {  in io_uring_cancel_generic()
3199 		if (inflight == tctx_inflight(tctx, !cancel_all))  in io_uring_cancel_generic()
3202 		finish_wait(&tctx->wait, &wait);  in io_uring_cancel_generic()
3205 	io_uring_clean_tctx(tctx);  in io_uring_cancel_generic()
3211 		atomic_dec(&tctx->in_cancel);  in io_uring_cancel_generic()
3352 		struct io_uring_task *tctx = current->io_uring;  in SYSCALL_DEFINE6()  local
3354 		if (unlikely(!tctx || fd >= IO_RINGFD_REG_MAX))  in SYSCALL_DEFINE6()
3357 		file = tctx->registered_rings[fd];  in SYSCALL_DEFINE6()
3664 	struct io_uring_task *tctx;  in io_uring_create()  local
3772 	tctx = current->io_uring;  in io_uring_create()
3779 		ret = io_ring_add_registered_file(tctx, file, 0, IO_RINGFD_REG_MAX);  in io_uring_create()