Lines Matching refs:dsq
1280 static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq, in nldsq_next_task() argument
1286 lockdep_assert_held(&dsq->lock); in nldsq_next_task()
1291 list_node = &dsq->list; in nldsq_next_task()
1300 if (list_node == &dsq->list) in nldsq_next_task()
1310 #define nldsq_for_each_task(p, dsq) \ argument
1311 for ((p) = nldsq_next_task((dsq), NULL, false); (p); \
1312 (p) = nldsq_next_task((dsq), (p), false))
1336 struct scx_dispatch_q *dsq; member
1846 static void dsq_mod_nr(struct scx_dispatch_q *dsq, s32 delta) in dsq_mod_nr() argument
1849 WRITE_ONCE(dsq->nr, dsq->nr + delta); in dsq_mod_nr()
1852 static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p, in dispatch_enqueue() argument
1855 bool is_local = dsq->id == SCX_DSQ_LOCAL; in dispatch_enqueue()
1857 WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); in dispatch_enqueue()
1862 raw_spin_lock(&dsq->lock); in dispatch_enqueue()
1863 if (unlikely(dsq->id == SCX_DSQ_INVALID)) { in dispatch_enqueue()
1866 raw_spin_unlock(&dsq->lock); in dispatch_enqueue()
1867 dsq = find_global_dsq(p); in dispatch_enqueue()
1868 raw_spin_lock(&dsq->lock); in dispatch_enqueue()
1872 if (unlikely((dsq->id & SCX_DSQ_FLAG_BUILTIN) && in dispatch_enqueue()
1893 if (unlikely(RB_EMPTY_ROOT(&dsq->priq) && in dispatch_enqueue()
1894 nldsq_next_task(dsq, NULL, false))) in dispatch_enqueue()
1896 dsq->id); in dispatch_enqueue()
1899 rb_add(&p->scx.dsq_priq, &dsq->priq, scx_dsq_priq_less); in dispatch_enqueue()
1912 list_add(&p->scx.dsq_list.node, &dsq->list); in dispatch_enqueue()
1916 if (unlikely(!RB_EMPTY_ROOT(&dsq->priq))) in dispatch_enqueue()
1918 dsq->id); in dispatch_enqueue()
1921 list_add(&p->scx.dsq_list.node, &dsq->list); in dispatch_enqueue()
1923 list_add_tail(&p->scx.dsq_list.node, &dsq->list); in dispatch_enqueue()
1927 dsq->seq++; in dispatch_enqueue()
1928 p->scx.dsq_seq = dsq->seq; in dispatch_enqueue()
1930 dsq_mod_nr(dsq, 1); in dispatch_enqueue()
1931 p->scx.dsq = dsq; in dispatch_enqueue()
1950 struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); in dispatch_enqueue()
1963 raw_spin_unlock(&dsq->lock); in dispatch_enqueue()
1968 struct scx_dispatch_q *dsq) in task_unlink_from_dsq() argument
1973 rb_erase(&p->scx.dsq_priq, &dsq->priq); in task_unlink_from_dsq()
1979 dsq_mod_nr(dsq, -1); in task_unlink_from_dsq()
1984 struct scx_dispatch_q *dsq = p->scx.dsq; in dispatch_dequeue() local
1985 bool is_local = dsq == &rq->scx.local_dsq; in dispatch_dequeue()
1987 if (!dsq) { in dispatch_dequeue()
2008 raw_spin_lock(&dsq->lock); in dispatch_dequeue()
2016 task_unlink_from_dsq(p, dsq); in dispatch_dequeue()
2027 p->scx.dsq = NULL; in dispatch_dequeue()
2030 raw_spin_unlock(&dsq->lock); in dispatch_dequeue()
2036 struct scx_dispatch_q *dsq; in find_dsq_for_dispatch() local
2051 dsq = find_global_dsq(p); in find_dsq_for_dispatch()
2053 dsq = find_user_dsq(dsq_id); in find_dsq_for_dispatch()
2055 if (unlikely(!dsq)) { in find_dsq_for_dispatch()
2061 return dsq; in find_dsq_for_dispatch()
2097 struct scx_dispatch_q *dsq = in direct_dispatch() local
2110 if (dsq->id == SCX_DSQ_LOCAL && dsq != &rq->scx.local_dsq) { in direct_dispatch()
2132 WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); in direct_dispatch()
2139 dispatch_enqueue(dsq, p, p->scx.ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); in direct_dispatch()
2447 p->scx.dsq = dst_dsq; in move_local_task_to_local_dsq()
2582 struct scx_dispatch_q *dsq, in unlink_dsq_and_lock_src_rq() argument
2587 lockdep_assert_held(&dsq->lock); in unlink_dsq_and_lock_src_rq()
2590 task_unlink_from_dsq(p, dsq); in unlink_dsq_and_lock_src_rq()
2593 raw_spin_unlock(&dsq->lock); in unlink_dsq_and_lock_src_rq()
2602 struct scx_dispatch_q *dsq, struct rq *src_rq) in consume_remote_task() argument
2606 if (unlink_dsq_and_lock_src_rq(p, dsq, src_rq)) { in consume_remote_task()
2618 …ote_task(struct rq *this_rq, struct task_struct *p, struct scx_dispatch_q *dsq, struct rq *task_rq… in consume_remote_task() argument
2680 p->scx.dsq = NULL; in move_task_between_dsqs()
2718 static bool consume_dispatch_q(struct rq *rq, struct scx_dispatch_q *dsq) in consume_dispatch_q() argument
2735 if (list_empty(&dsq->list)) in consume_dispatch_q()
2738 raw_spin_lock(&dsq->lock); in consume_dispatch_q()
2740 nldsq_for_each_task(p, dsq) { in consume_dispatch_q()
2744 task_unlink_from_dsq(p, dsq); in consume_dispatch_q()
2745 move_local_task_to_local_dsq(p, 0, dsq, rq); in consume_dispatch_q()
2746 raw_spin_unlock(&dsq->lock); in consume_dispatch_q()
2751 if (likely(consume_remote_task(rq, p, dsq, task_rq))) in consume_dispatch_q()
2757 raw_spin_unlock(&dsq->lock); in consume_dispatch_q()
2889 struct scx_dispatch_q *dsq; in finish_dispatch() local
2938 dsq = find_dsq_for_dispatch(this_rq(), dsq_id, p); in finish_dispatch()
2940 if (dsq->id == SCX_DSQ_LOCAL) in finish_dispatch()
2941 dispatch_to_local_dsq(rq, dsq, p, enq_flags); in finish_dispatch()
2943 dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); in finish_dispatch()
3124 struct scx_dispatch_q *dsq; in process_ddsp_deferred_locals() local
3128 dsq = find_dsq_for_dispatch(rq, p->scx.ddsp_dsq_id, p); in process_ddsp_deferred_locals()
3129 if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) in process_ddsp_deferred_locals()
3130 dispatch_to_local_dsq(rq, dsq, p, p->scx.ddsp_enq_flags); in process_ddsp_deferred_locals()
4154 static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) in init_dsq() argument
4156 memset(dsq, 0, sizeof(*dsq)); in init_dsq()
4158 raw_spin_lock_init(&dsq->lock); in init_dsq()
4159 INIT_LIST_HEAD(&dsq->list); in init_dsq()
4160 dsq->id = dsq_id; in init_dsq()
4165 struct scx_dispatch_q *dsq; in create_dsq() local
4171 dsq = kmalloc_node(sizeof(*dsq), GFP_KERNEL, node); in create_dsq()
4172 if (!dsq) in create_dsq()
4175 init_dsq(dsq, dsq_id); in create_dsq()
4177 ret = rhashtable_lookup_insert_fast(&dsq_hash, &dsq->hash_node, in create_dsq()
4180 kfree(dsq); in create_dsq()
4183 return dsq; in create_dsq()
4189 struct scx_dispatch_q *dsq, *tmp_dsq; in free_dsq_irq_workfn() local
4191 llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) in free_dsq_irq_workfn()
4192 kfree_rcu(dsq, rcu); in free_dsq_irq_workfn()
4199 struct scx_dispatch_q *dsq; in destroy_dsq() local
4204 dsq = find_user_dsq(dsq_id); in destroy_dsq()
4205 if (!dsq) in destroy_dsq()
4208 raw_spin_lock_irqsave(&dsq->lock, flags); in destroy_dsq()
4210 if (dsq->nr) { in destroy_dsq()
4212 dsq->id, dsq->nr); in destroy_dsq()
4216 if (rhashtable_remove_fast(&dsq_hash, &dsq->hash_node, dsq_hash_params)) in destroy_dsq()
4225 dsq->id = SCX_DSQ_INVALID; in destroy_dsq()
4226 llist_add(&dsq->free_node, &dsqs_to_free); in destroy_dsq()
4230 raw_spin_unlock_irqrestore(&dsq->lock, flags); in destroy_dsq()
4677 struct scx_dispatch_q *dsq; in scx_ops_disable_workfn() local
4817 while ((dsq = rhashtable_walk_next(&rht_iter)) && !IS_ERR(dsq)) in scx_ops_disable_workfn()
4818 destroy_dsq(dsq->id); in scx_ops_disable_workfn()
4821 } while (dsq == ERR_PTR(-EAGAIN)); in scx_ops_disable_workfn()
4984 if (p->scx.dsq) in scx_dump_task()
4986 (unsigned long long)p->scx.dsq->id); in scx_dump_task()
5301 struct scx_dispatch_q *dsq; in scx_ops_enable() local
5303 dsq = kzalloc_node(sizeof(*dsq), GFP_KERNEL, node); in scx_ops_enable()
5304 if (!dsq) { in scx_ops_enable()
5312 init_dsq(dsq, SCX_DSQ_GLOBAL); in scx_ops_enable()
5313 dsqs[node] = dsq; in scx_ops_enable()
6269 struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq; in scx_dsq_move()
6312 if (unlikely(p->scx.dsq != src_dsq || in scx_dsq_move()
6402 struct scx_dispatch_q *dsq; in scx_bpf_dsq_move_to_local() local
6409 dsq = find_user_dsq(dsq_id); in scx_bpf_dsq_move_to_local()
6410 if (unlikely(!dsq)) { in scx_bpf_dsq_move_to_local()
6415 if (consume_dispatch_q(dspc->rq, dsq)) { in scx_bpf_dsq_move_to_local()
6768 struct scx_dispatch_q *dsq; in scx_bpf_dsq_nr_queued() local
6784 dsq = find_user_dsq(dsq_id); in scx_bpf_dsq_nr_queued()
6785 if (dsq) { in scx_bpf_dsq_nr_queued()
6786 ret = READ_ONCE(dsq->nr); in scx_bpf_dsq_nr_queued()
6834 kit->dsq = NULL; in bpf_iter_scx_dsq_new()
6839 kit->dsq = find_user_dsq(dsq_id); in bpf_iter_scx_dsq_new()
6840 if (!kit->dsq) in bpf_iter_scx_dsq_new()
6845 kit->cursor.priv = READ_ONCE(kit->dsq->seq); in bpf_iter_scx_dsq_new()
6863 if (!kit->dsq) in bpf_iter_scx_dsq_next()
6866 raw_spin_lock_irqsave(&kit->dsq->lock, flags); in bpf_iter_scx_dsq_next()
6879 p = nldsq_next_task(kit->dsq, p, rev); in bpf_iter_scx_dsq_next()
6891 raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); in bpf_iter_scx_dsq_next()
6906 if (!kit->dsq) in bpf_iter_scx_dsq_destroy()
6912 raw_spin_lock_irqsave(&kit->dsq->lock, flags); in bpf_iter_scx_dsq_destroy()
6914 raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); in bpf_iter_scx_dsq_destroy()
6916 kit->dsq = NULL; in bpf_iter_scx_dsq_destroy()