6266 static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,  in scx_dsq_move()  argument
6269 	struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq;  in scx_dsq_move()
6313 		     u32_before(kit->cursor.priv, p->scx.dsq_seq) ||  in scx_dsq_move()
6328 	if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME)  in scx_dsq_move()
6329 		p->scx.dsq_vtime = kit->vtime;  in scx_dsq_move()
6330 	if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE)  in scx_dsq_move()
6331 		p->scx.slice = kit->slice;  in scx_dsq_move()
6346 	kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE |  in scx_dsq_move()
6448 	struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter;  in scx_bpf_dsq_move_set_slice()  local
6450 	kit->slice = slice;  in scx_bpf_dsq_move_set_slice()
6451 	kit->cursor.flags |= __SCX_DSQ_ITER_HAS_SLICE;  in scx_bpf_dsq_move_set_slice()
6475 	struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter;  in scx_bpf_dsq_move_set_vtime()  local
6477 	kit->vtime = vtime;  in scx_bpf_dsq_move_set_vtime()
6478 	kit->cursor.flags |= __SCX_DSQ_ITER_HAS_VTIME;  in scx_bpf_dsq_move_set_vtime()
6823 	struct bpf_iter_scx_dsq_kern *kit = (void *)it;  in bpf_iter_scx_dsq_new()  local
6834 	kit->dsq = NULL;  in bpf_iter_scx_dsq_new()
6839 	kit->dsq = find_user_dsq(dsq_id);  in bpf_iter_scx_dsq_new()
6840 	if (!kit->dsq)  in bpf_iter_scx_dsq_new()
6843 	INIT_LIST_HEAD(&kit->cursor.node);  in bpf_iter_scx_dsq_new()
6844 	kit->cursor.flags = SCX_DSQ_LNODE_ITER_CURSOR | flags;  in bpf_iter_scx_dsq_new()
6845 	kit->cursor.priv = READ_ONCE(kit->dsq->seq);  in bpf_iter_scx_dsq_new()
6858 	struct bpf_iter_scx_dsq_kern *kit = (void *)it;  in bpf_iter_scx_dsq_next()  local
6859 	bool rev = kit->cursor.flags & SCX_DSQ_ITER_REV;  in bpf_iter_scx_dsq_next()
6863 	if (!kit->dsq)  in bpf_iter_scx_dsq_next()
6866 	raw_spin_lock_irqsave(&kit->dsq->lock, flags);  in bpf_iter_scx_dsq_next()
6868 	if (list_empty(&kit->cursor.node))  in bpf_iter_scx_dsq_next()
6871 		p = container_of(&kit->cursor, struct task_struct, scx.dsq_list);  in bpf_iter_scx_dsq_next()
6879 		p = nldsq_next_task(kit->dsq, p, rev);  in bpf_iter_scx_dsq_next()
6880 	} while (p && unlikely(u32_before(kit->cursor.priv, p->scx.dsq_seq)));  in bpf_iter_scx_dsq_next()
6884 			list_move_tail(&kit->cursor.node, &p->scx.dsq_list.node);  in bpf_iter_scx_dsq_next()
6886 			list_move(&kit->cursor.node, &p->scx.dsq_list.node);  in bpf_iter_scx_dsq_next()
6888 		list_del_init(&kit->cursor.node);  in bpf_iter_scx_dsq_next()
6891 	raw_spin_unlock_irqrestore(&kit->dsq->lock, flags);  in bpf_iter_scx_dsq_next()
6904 	struct bpf_iter_scx_dsq_kern *kit = (void *)it;  in bpf_iter_scx_dsq_destroy()  local
6906 	if (!kit->dsq)  in bpf_iter_scx_dsq_destroy()
6909 	if (!list_empty(&kit->cursor.node)) {  in bpf_iter_scx_dsq_destroy()
6912 		raw_spin_lock_irqsave(&kit->dsq->lock, flags);  in bpf_iter_scx_dsq_destroy()
6913 		list_del_init(&kit->cursor.node);  in bpf_iter_scx_dsq_destroy()
6914 		raw_spin_unlock_irqrestore(&kit->dsq->lock, flags);  in bpf_iter_scx_dsq_destroy()
6916 	kit->dsq = NULL;  in bpf_iter_scx_dsq_destroy()