Lines Matching refs:scx
1108 WARN_ONCE((mask | higher_bits(mask)) & current->scx.kf_mask, in scx_kf_allow()
1110 current->scx.kf_mask, mask); in scx_kf_allow()
1111 current->scx.kf_mask |= mask; in scx_kf_allow()
1118 current->scx.kf_mask &= ~mask; in scx_kf_disallow()
1193 current->scx.kf_tasks[0] = task; \
1195 current->scx.kf_tasks[0] = NULL; \
1202 current->scx.kf_tasks[0] = task; \
1204 current->scx.kf_tasks[0] = NULL; \
1212 current->scx.kf_tasks[0] = task0; \
1213 current->scx.kf_tasks[1] = task1; \
1215 current->scx.kf_tasks[0] = NULL; \
1216 current->scx.kf_tasks[1] = NULL; \
1223 if (unlikely(!(current->scx.kf_mask & mask))) { in scx_kf_allowed()
1225 mask, current->scx.kf_mask); in scx_kf_allowed()
1237 (current->scx.kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { in scx_kf_allowed()
1243 (current->scx.kf_mask & higher_bits(SCX_KF_DISPATCH)))) { in scx_kf_allowed()
1258 if (unlikely((p != current->scx.kf_tasks[0] && in scx_kf_allowed_on_arg_tasks()
1259 p != current->scx.kf_tasks[1]))) { in scx_kf_allowed_on_arg_tasks()
1269 return !current->scx.kf_mask; in scx_kf_allowed_if_unlocked()
1289 list_node = &cur->scx.dsq_list.node; in nldsq_next_task()
1307 return container_of(dsq_lnode, struct task_struct, scx.dsq_list); in nldsq_next_task()
1458 return container_of(pos, struct task_struct, scx); in scx_task_iter_next()
1650 return unlikely(rq->scx.flags & SCX_RQ_BYPASSING); in scx_rq_bypassing()
1667 } while (atomic_long_read_acquire(&p->scx.ops_state) == opss); in wait_ops_state()
1725 struct rq *rq = container_of(irq_work, struct rq, scx.deferred_irq_work); in deferred_irq_workfn()
1750 if (rq->scx.flags & SCX_RQ_IN_WAKEUP) in schedule_deferred()
1757 if (rq->scx.flags & SCX_RQ_IN_BALANCE) { in schedule_deferred()
1758 queue_balance_callback(rq, &rq->scx.deferred_bal_cb, in schedule_deferred()
1769 irq_work_queue(&rq->scx.deferred_irq_work); in schedule_deferred()
1795 p->scx.core_sched_at = sched_clock_cpu(cpu_of(rq)); in touch_core_sched()
1828 if (curr->scx.slice != SCX_SLICE_INF) { in update_curr_scx()
1829 curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec); in update_curr_scx()
1830 if (!curr->scx.slice) in update_curr_scx()
1839 container_of(node_a, struct task_struct, scx.dsq_priq); in scx_dsq_priq_less()
1841 container_of(node_b, struct task_struct, scx.dsq_priq); in scx_dsq_priq_less()
1843 return time_before64(a->scx.dsq_vtime, b->scx.dsq_vtime); in scx_dsq_priq_less()
1857 WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); in dispatch_enqueue()
1858 WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || in dispatch_enqueue()
1859 !RB_EMPTY_NODE(&p->scx.dsq_priq)); in dispatch_enqueue()
1898 p->scx.dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; in dispatch_enqueue()
1899 rb_add(&p->scx.dsq_priq, &dsq->priq, scx_dsq_priq_less); in dispatch_enqueue()
1905 rbp = rb_prev(&p->scx.dsq_priq); in dispatch_enqueue()
1909 scx.dsq_priq); in dispatch_enqueue()
1910 list_add(&p->scx.dsq_list.node, &prev->scx.dsq_list.node); in dispatch_enqueue()
1912 list_add(&p->scx.dsq_list.node, &dsq->list); in dispatch_enqueue()
1921 list_add(&p->scx.dsq_list.node, &dsq->list); in dispatch_enqueue()
1923 list_add_tail(&p->scx.dsq_list.node, &dsq->list); in dispatch_enqueue()
1928 p->scx.dsq_seq = dsq->seq; in dispatch_enqueue()
1931 p->scx.dsq = dsq; in dispatch_enqueue()
1939 p->scx.ddsp_dsq_id = SCX_DSQ_INVALID; in dispatch_enqueue()
1940 p->scx.ddsp_enq_flags = 0; in dispatch_enqueue()
1947 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); in dispatch_enqueue()
1950 struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); in dispatch_enqueue()
1955 rq->curr->scx.slice = 0; in dispatch_enqueue()
1970 WARN_ON_ONCE(list_empty(&p->scx.dsq_list.node)); in task_unlink_from_dsq()
1972 if (p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { in task_unlink_from_dsq()
1973 rb_erase(&p->scx.dsq_priq, &dsq->priq); in task_unlink_from_dsq()
1974 RB_CLEAR_NODE(&p->scx.dsq_priq); in task_unlink_from_dsq()
1975 p->scx.dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; in task_unlink_from_dsq()
1978 list_del_init(&p->scx.dsq_list.node); in task_unlink_from_dsq()
1984 struct scx_dispatch_q *dsq = p->scx.dsq; in dispatch_dequeue()
1985 bool is_local = dsq == &rq->scx.local_dsq; in dispatch_dequeue()
1992 if (unlikely(!list_empty(&p->scx.dsq_list.node))) in dispatch_dequeue()
1993 list_del_init(&p->scx.dsq_list.node); in dispatch_dequeue()
2001 if (p->scx.holding_cpu >= 0) in dispatch_dequeue()
2002 p->scx.holding_cpu = -1; in dispatch_dequeue()
2014 if (p->scx.holding_cpu < 0) { in dispatch_dequeue()
2024 WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node)); in dispatch_dequeue()
2025 p->scx.holding_cpu = -1; in dispatch_dequeue()
2027 p->scx.dsq = NULL; in dispatch_dequeue()
2039 return &rq->scx.local_dsq; in find_dsq_for_dispatch()
2047 return &cpu_rq(cpu)->scx.local_dsq; in find_dsq_for_dispatch()
2087 WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID); in mark_direct_dispatch()
2088 WARN_ON_ONCE(p->scx.ddsp_enq_flags); in mark_direct_dispatch()
2090 p->scx.ddsp_dsq_id = dsq_id; in mark_direct_dispatch()
2091 p->scx.ddsp_enq_flags = enq_flags; in mark_direct_dispatch()
2098 find_dsq_for_dispatch(rq, p->scx.ddsp_dsq_id, p); in direct_dispatch()
2102 p->scx.ddsp_enq_flags |= enq_flags; in direct_dispatch()
2110 if (dsq->id == SCX_DSQ_LOCAL && dsq != &rq->scx.local_dsq) { in direct_dispatch()
2113 opss = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_STATE_MASK; in direct_dispatch()
2123 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); in direct_dispatch()
2128 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); in direct_dispatch()
2132 WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); in direct_dispatch()
2133 list_add_tail(&p->scx.dsq_list.node, in direct_dispatch()
2134 &rq->scx.ddsp_deferred_locals); in direct_dispatch()
2139 dispatch_enqueue(dsq, p, p->scx.ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); in direct_dispatch()
2151 return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq))); in scx_rq_online()
2160 WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_QUEUED)); in do_enqueue_task()
2179 if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) in do_enqueue_task()
2200 qseq = rq->scx.ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; in do_enqueue_task()
2202 WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); in do_enqueue_task()
2203 atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq); in do_enqueue_task()
2212 if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) in do_enqueue_task()
2219 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq); in do_enqueue_task()
2233 p->scx.slice = SCX_SLICE_DFL; in do_enqueue_task()
2236 dispatch_enqueue(&rq->scx.local_dsq, p, enq_flags); in do_enqueue_task()
2241 p->scx.slice = SCX_SLICE_DFL; in do_enqueue_task()
2248 return !list_empty(&p->scx.runnable_node); in task_runnable()
2255 if (p->scx.flags & SCX_TASK_RESET_RUNNABLE_AT) { in set_task_runnable()
2256 p->scx.runnable_at = jiffies; in set_task_runnable()
2257 p->scx.flags &= ~SCX_TASK_RESET_RUNNABLE_AT; in set_task_runnable()
2264 list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list); in set_task_runnable()
2269 list_del_init(&p->scx.runnable_node); in clr_task_runnable()
2271 p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; in clr_task_runnable()
2276 int sticky_cpu = p->scx.sticky_cpu; in enqueue_task_scx()
2279 rq->scx.flags |= SCX_RQ_IN_WAKEUP; in enqueue_task_scx()
2281 enq_flags |= rq->scx.extra_enq_flags; in enqueue_task_scx()
2284 p->scx.sticky_cpu = -1; in enqueue_task_scx()
2295 if (p->scx.flags & SCX_TASK_QUEUED) { in enqueue_task_scx()
2301 p->scx.flags |= SCX_TASK_QUEUED; in enqueue_task_scx()
2302 rq->scx.nr_running++; in enqueue_task_scx()
2313 rq->scx.flags &= ~SCX_RQ_IN_WAKEUP; in enqueue_task_scx()
2316 unlikely(cpu_of(rq) != p->scx.selected_cpu)) in enqueue_task_scx()
2328 opss = atomic_long_read_acquire(&p->scx.ops_state); in ops_dequeue()
2343 if (atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, in ops_dequeue()
2362 BUG_ON(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); in ops_dequeue()
2369 if (!(p->scx.flags & SCX_TASK_QUEUED)) { in dequeue_task_scx()
2397 p->scx.flags |= SCX_TASK_DEQD_FOR_SLEEP; in dequeue_task_scx()
2399 p->scx.flags &= ~SCX_TASK_DEQD_FOR_SLEEP; in dequeue_task_scx()
2401 p->scx.flags &= ~SCX_TASK_QUEUED; in dequeue_task_scx()
2402 rq->scx.nr_running--; in dequeue_task_scx()
2416 p->scx.slice = 0; in yield_task_scx()
2433 struct scx_dispatch_q *dst_dsq = &dst_rq->scx.local_dsq; in move_local_task_to_local_dsq()
2439 WARN_ON_ONCE(p->scx.holding_cpu >= 0); in move_local_task_to_local_dsq()
2442 list_add(&p->scx.dsq_list.node, &dst_dsq->list); in move_local_task_to_local_dsq()
2444 list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list); in move_local_task_to_local_dsq()
2447 p->scx.dsq = dst_dsq; in move_local_task_to_local_dsq()
2468 p->scx.sticky_cpu = cpu_of(dst_rq); in move_remote_task_to_local_dsq()
2479 WARN_ON_ONCE(dst_rq->scx.extra_enq_flags); in move_remote_task_to_local_dsq()
2480 dst_rq->scx.extra_enq_flags = enq_flags; in move_remote_task_to_local_dsq()
2482 dst_rq->scx.extra_enq_flags = 0; in move_remote_task_to_local_dsq()
2589 WARN_ON_ONCE(p->scx.holding_cpu >= 0); in unlink_dsq_and_lock_src_rq()
2591 p->scx.holding_cpu = cpu; in unlink_dsq_and_lock_src_rq()
2597 return likely(p->scx.holding_cpu == cpu) && in unlink_dsq_and_lock_src_rq()
2647 dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); in move_task_between_dsqs()
2680 p->scx.dsq = NULL; in move_task_between_dsqs()
2786 struct rq *dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); in dispatch_to_local_dsq()
2821 p->scx.holding_cpu = raw_smp_processor_id(); in dispatch_to_local_dsq()
2824 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); in dispatch_to_local_dsq()
2834 if (likely(p->scx.holding_cpu == raw_smp_processor_id()) && in dispatch_to_local_dsq()
2842 p->scx.holding_cpu = -1; in dispatch_to_local_dsq()
2843 dispatch_enqueue(&dst_rq->scx.local_dsq, p, enq_flags); in dispatch_to_local_dsq()
2898 opss = atomic_long_read(&p->scx.ops_state); in finish_dispatch()
2921 if (likely(atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, in finish_dispatch()
2936 BUG_ON(!(p->scx.flags & SCX_TASK_QUEUED)); in finish_dispatch()
2966 bool prev_on_rq = prev->scx.flags & SCX_TASK_QUEUED; in balance_one()
2970 rq->scx.flags |= SCX_RQ_IN_BALANCE; in balance_one()
2971 rq->scx.flags &= ~(SCX_RQ_BAL_PENDING | SCX_RQ_BAL_KEEP); in balance_one()
2974 unlikely(rq->scx.cpu_released)) { in balance_one()
2983 rq->scx.cpu_released = false; in balance_one()
2999 if (prev_on_rq && prev->scx.slice && !scx_rq_bypassing(rq)) { in balance_one()
3000 rq->scx.flags |= SCX_RQ_BAL_KEEP; in balance_one()
3006 if (rq->scx.local_dsq.nr) in balance_one()
3032 if (prev_on_rq && prev->scx.slice) { in balance_one()
3033 rq->scx.flags |= SCX_RQ_BAL_KEEP; in balance_one()
3036 if (rq->scx.local_dsq.nr) in balance_one()
3063 rq->scx.flags |= SCX_RQ_BAL_KEEP; in balance_one()
3067 rq->scx.flags &= ~SCX_RQ_IN_BALANCE; in balance_one()
3071 rq->scx.flags &= ~SCX_RQ_IN_BALANCE; in balance_one()
3122 while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals, in process_ddsp_deferred_locals()
3123 struct task_struct, scx.dsq_list.node))) { in process_ddsp_deferred_locals()
3126 list_del_init(&p->scx.dsq_list.node); in process_ddsp_deferred_locals()
3128 dsq = find_dsq_for_dispatch(rq, p->scx.ddsp_dsq_id, p); in process_ddsp_deferred_locals()
3130 dispatch_to_local_dsq(rq, dsq, p, p->scx.ddsp_enq_flags); in process_ddsp_deferred_locals()
3136 if (p->scx.flags & SCX_TASK_QUEUED) { in set_next_task_scx()
3148 if (SCX_HAS_OP(running) && (p->scx.flags & SCX_TASK_QUEUED)) in set_next_task_scx()
3157 if ((p->scx.slice == SCX_SLICE_INF) != in set_next_task_scx()
3158 (bool)(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) { in set_next_task_scx()
3159 if (p->scx.slice == SCX_SLICE_INF) in set_next_task_scx()
3160 rq->scx.flags |= SCX_RQ_CAN_STOP_TICK; in set_next_task_scx()
3162 rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK; in set_next_task_scx()
3200 smp_store_release(&rq->scx.pnt_seq, rq->scx.pnt_seq + 1); in switch_class()
3223 if (!rq->scx.cpu_released) { in switch_class()
3232 rq->scx.cpu_released = true; in switch_class()
3242 if (SCX_HAS_OP(stopping) && (p->scx.flags & SCX_TASK_QUEUED)) in put_prev_task_scx()
3245 if (p->scx.flags & SCX_TASK_QUEUED) { in put_prev_task_scx()
3254 if (p->scx.slice && !scx_rq_bypassing(rq)) { in put_prev_task_scx()
3255 dispatch_enqueue(&rq->scx.local_dsq, p, SCX_ENQ_HEAD); in put_prev_task_scx()
3280 return list_first_entry_or_null(&rq->scx.local_dsq.list, in first_local_task()
3281 struct task_struct, scx.dsq_list.node); in first_local_task()
3288 bool keep_prev = rq->scx.flags & SCX_RQ_BAL_KEEP; in pick_task_scx()
3306 if (unlikely(rq->scx.flags & SCX_RQ_BAL_PENDING)) { in pick_task_scx()
3307 if (prev->scx.flags & SCX_TASK_QUEUED) { in pick_task_scx()
3330 if (!p->scx.slice) { in pick_task_scx()
3331 p->scx.slice = SCX_SLICE_DFL; in pick_task_scx()
3342 if (unlikely(!p->scx.slice)) { in pick_task_scx()
3348 p->scx.slice = SCX_SLICE_DFL; in pick_task_scx()
3388 return time_after64(a->scx.core_sched_at, b->scx.core_sched_at); in scx_prio_less()
3422 p->scx.selected_cpu = cpu; in select_task_rq_scx()
3433 p->scx.slice = SCX_SLICE_DFL; in select_task_rq_scx()
3434 p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL; in select_task_rq_scx()
3439 p->scx.selected_cpu = cpu; in select_task_rq_scx()
3501 rq->scx.flags |= SCX_RQ_ONLINE; in rq_online_scx()
3506 rq->scx.flags &= ~SCX_RQ_ONLINE; in rq_offline_scx()
3518 list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) { in check_rq_for_timeouts()
3519 unsigned long last_runnable = p->scx.runnable_at; in check_rq_for_timeouts()
3583 curr->scx.slice = 0; in task_tick_scx()
3589 if (!curr->scx.slice) in task_tick_scx()
3617 return (p->scx.flags & SCX_TASK_STATE_MASK) >> SCX_TASK_STATE_SHIFT; in scx_get_task_state()
3647 p->scx.flags &= ~SCX_TASK_STATE_MASK; in scx_set_task_state()
3648 p->scx.flags |= state << SCX_TASK_STATE_SHIFT; in scx_set_task_state()
3655 p->scx.disallow = false; in scx_ops_init_task()
3672 if (p->scx.disallow) { in scx_ops_init_task()
3698 p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; in scx_ops_init_task()
3718 p->scx.weight = sched_weight_to_cgroup(weight); in scx_ops_enable_task()
3725 SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, rq, p, p->scx.weight); in scx_ops_enable_task()
3769 void init_scx_entity(struct sched_ext_entity *scx) in init_scx_entity() argument
3771 memset(scx, 0, sizeof(*scx)); in init_scx_entity()
3772 INIT_LIST_HEAD(&scx->dsq_list.node); in init_scx_entity()
3773 RB_CLEAR_NODE(&scx->dsq_priq); in init_scx_entity()
3774 scx->sticky_cpu = -1; in init_scx_entity()
3775 scx->holding_cpu = -1; in init_scx_entity()
3776 INIT_LIST_HEAD(&scx->runnable_node); in init_scx_entity()
3777 scx->runnable_at = jiffies; in init_scx_entity()
3778 scx->ddsp_dsq_id = SCX_DSQ_INVALID; in init_scx_entity()
3779 scx->slice = SCX_SLICE_DFL; in init_scx_entity()
3824 list_add_tail(&p->scx.tasks_node, &scx_tasks); in scx_post_fork()
3850 list_del_init(&p->scx.tasks_node); in sched_ext_free()
3872 p->scx.weight = sched_weight_to_cgroup(scale_load_down(lw->weight)); in reweight_task_scx()
3874 SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, rq, p, p->scx.weight); in reweight_task_scx()
3907 if (scx_enabled() && READ_ONCE(p->scx.disallow) && in scx_check_setscheduler()
3930 return rq->scx.flags & SCX_RQ_CAN_STOP_TICK; in scx_can_stop_tick()
3996 WARN_ON_ONCE(p->scx.cgrp_moving_from); in scx_cgroup_can_attach()
4013 p->scx.cgrp_moving_from = from; in scx_cgroup_can_attach()
4020 if (SCX_HAS_OP(cgroup_cancel_move) && p->scx.cgrp_moving_from) in scx_cgroup_can_attach()
4022 p, p->scx.cgrp_moving_from, css->cgroup); in scx_cgroup_can_attach()
4023 p->scx.cgrp_moving_from = NULL; in scx_cgroup_can_attach()
4039 if (SCX_HAS_OP(cgroup_move) && !WARN_ON_ONCE(!p->scx.cgrp_moving_from)) in scx_cgroup_move_task()
4041 p, p->scx.cgrp_moving_from, tg_cgrp(task_group(p))); in scx_cgroup_move_task()
4042 p->scx.cgrp_moving_from = NULL; in scx_cgroup_move_task()
4059 if (SCX_HAS_OP(cgroup_cancel_move) && p->scx.cgrp_moving_from) in scx_cgroup_cancel_attach()
4061 p, p->scx.cgrp_moving_from, css->cgroup); in scx_cgroup_cancel_attach()
4062 p->scx.cgrp_moving_from = NULL; in scx_cgroup_cancel_attach()
4575 WARN_ON_ONCE(rq->scx.flags & SCX_RQ_BYPASSING); in scx_ops_bypass()
4576 rq->scx.flags |= SCX_RQ_BYPASSING; in scx_ops_bypass()
4578 WARN_ON_ONCE(!(rq->scx.flags & SCX_RQ_BYPASSING)); in scx_ops_bypass()
4579 rq->scx.flags &= ~SCX_RQ_BYPASSING; in scx_ops_bypass()
4599 list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, in scx_ops_bypass()
4600 scx.runnable_node) { in scx_ops_bypass()
4981 unsigned long ops_state = atomic_long_read(&p->scx.ops_state); in scx_dump_task()
4984 if (p->scx.dsq) in scx_dump_task()
4986 (unsigned long long)p->scx.dsq->id); in scx_dump_task()
4991 jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies)); in scx_dump_task()
4993 scx_get_task_state(p), p->scx.flags & ~SCX_TASK_STATE_MASK, in scx_dump_task()
4994 p->scx.dsq_flags, ops_state & SCX_OPSS_STATE_MASK, in scx_dump_task()
4997 p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf); in scx_dump_task()
4999 p->scx.dsq_vtime, p->scx.slice, p->scx.weight); in scx_dump_task()
5069 idle = list_empty(&rq->scx.runnable_list) && in scx_dump_state()
5086 cpu, rq->scx.nr_running, rq->scx.flags, in scx_dump_state()
5087 rq->scx.cpu_released, rq->scx.ops_qseq, in scx_dump_state()
5088 rq->scx.pnt_seq); in scx_dump_state()
5092 if (!cpumask_empty(rq->scx.cpus_to_kick)) in scx_dump_state()
5094 cpumask_pr_args(rq->scx.cpus_to_kick)); in scx_dump_state()
5095 if (!cpumask_empty(rq->scx.cpus_to_kick_if_idle)) in scx_dump_state()
5097 cpumask_pr_args(rq->scx.cpus_to_kick_if_idle)); in scx_dump_state()
5098 if (!cpumask_empty(rq->scx.cpus_to_preempt)) in scx_dump_state()
5100 cpumask_pr_args(rq->scx.cpus_to_preempt)); in scx_dump_state()
5101 if (!cpumask_empty(rq->scx.cpus_to_wait)) in scx_dump_state()
5103 cpumask_pr_args(rq->scx.cpus_to_wait)); in scx_dump_state()
5132 list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) in scx_dump_state()
5356 cpu_rq(cpu)->scx.cpuperf_target = SCX_CPUPERF_ONE; in scx_ops_enable()
5515 p->scx.slice = SCX_SLICE_DFL; in scx_ops_enable()
5611 if (off >= offsetof(struct task_struct, scx.slice) && in bpf_scx_btf_struct_access()
5612 off + size <= offsetofend(struct task_struct, scx.slice)) in bpf_scx_btf_struct_access()
5614 if (off >= offsetof(struct task_struct, scx.dsq_vtime) && in bpf_scx_btf_struct_access()
5615 off + size <= offsetofend(struct task_struct, scx.dsq_vtime)) in bpf_scx_btf_struct_access()
5617 if (off >= offsetof(struct task_struct, scx.disallow) && in bpf_scx_btf_struct_access()
5618 off + size <= offsetofend(struct task_struct, scx.disallow)) in bpf_scx_btf_struct_access()
5888 return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_BALANCE); in can_skip_idle_kick()
5894 struct scx_rq *this_scx = &this_rq->scx; in kick_one_cpu()
5907 rq->curr->scx.slice = 0; in kick_one_cpu()
5912 pseqs[cpu] = rq->scx.pnt_seq; in kick_one_cpu()
5944 struct scx_rq *this_scx = &this_rq->scx; in kick_cpus_irq_workfn()
5964 unsigned long *wait_pnt_seq = &cpu_rq(cpu)->scx.pnt_seq; in kick_cpus_irq_workfn()
6017 if (!copy_from_kernel_nofault(&runnable_at, &p->scx.runnable_at, in print_scx_info()
6080 init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL); in init_sched_ext_class()
6081 INIT_LIST_HEAD(&rq->scx.runnable_list); in init_sched_ext_class()
6082 INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals); in init_sched_ext_class()
6084 BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick, GFP_KERNEL, n)); in init_sched_ext_class()
6085 BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick_if_idle, GFP_KERNEL, n)); in init_sched_ext_class()
6086 BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_preempt, GFP_KERNEL, n)); in init_sched_ext_class()
6087 BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_wait, GFP_KERNEL, n)); in init_sched_ext_class()
6088 init_irq_work(&rq->scx.deferred_irq_work, deferred_irq_workfn); in init_sched_ext_class()
6089 init_irq_work(&rq->scx.kick_cpus_irq_work, kick_cpus_irq_workfn); in init_sched_ext_class()
6092 cpu_rq(cpu)->scx.flags |= SCX_RQ_ONLINE; in init_sched_ext_class()
6143 .qseq = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_QSEQ_MASK, in scx_dsq_insert_commit()
6192 p->scx.slice = slice; in scx_bpf_dsq_insert()
6194 p->scx.slice = p->scx.slice ?: 1; in scx_bpf_dsq_insert()
6235 p->scx.slice = slice; in scx_bpf_dsq_insert_vtime()
6237 p->scx.slice = p->scx.slice ?: 1; in scx_bpf_dsq_insert_vtime()
6239 p->scx.dsq_vtime = vtime; in scx_bpf_dsq_insert_vtime()
6287 in_balance = this_rq->scx.flags & SCX_RQ_IN_BALANCE; in scx_dsq_move()
6312 if (unlikely(p->scx.dsq != src_dsq || in scx_dsq_move()
6313 u32_before(kit->cursor.priv, p->scx.dsq_seq) || in scx_dsq_move()
6314 p->scx.holding_cpu >= 0) || in scx_dsq_move()
6329 p->scx.dsq_vtime = kit->vtime; in scx_dsq_move()
6331 p->scx.slice = kit->slice; in scx_dsq_move()
6613 list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list, in scx_bpf_reenqueue_local()
6614 scx.dsq_list.node) { in scx_bpf_reenqueue_local()
6634 list_add_tail(&p->scx.dsq_list.node, &tasks); in scx_bpf_reenqueue_local()
6637 list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { in scx_bpf_reenqueue_local()
6638 list_del_init(&p->scx.dsq_list.node); in scx_bpf_reenqueue_local()
6744 cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick_if_idle); in scx_bpf_kick_cpu()
6746 cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick); in scx_bpf_kick_cpu()
6749 cpumask_set_cpu(cpu, this_rq->scx.cpus_to_preempt); in scx_bpf_kick_cpu()
6751 cpumask_set_cpu(cpu, this_rq->scx.cpus_to_wait); in scx_bpf_kick_cpu()
6754 irq_work_queue(&this_rq->scx.kick_cpus_irq_work); in scx_bpf_kick_cpu()
6774 ret = READ_ONCE(this_rq()->scx.local_dsq.nr); in scx_bpf_dsq_nr_queued()
6780 ret = READ_ONCE(cpu_rq(cpu)->scx.local_dsq.nr); in scx_bpf_dsq_nr_queued()
6871 p = container_of(&kit->cursor, struct task_struct, scx.dsq_list); in bpf_iter_scx_dsq_next()
6880 } while (p && unlikely(u32_before(kit->cursor.priv, p->scx.dsq_seq))); in bpf_iter_scx_dsq_next()
6884 list_move_tail(&kit->cursor.node, &p->scx.dsq_list.node); in bpf_iter_scx_dsq_next()
6886 list_move(&kit->cursor.node, &p->scx.dsq_list.node); in bpf_iter_scx_dsq_next()
7143 rq->scx.cpuperf_target = perf; in scx_bpf_cpuperf_set()
7295 if (smp_load_acquire(&rq->scx.flags) & SCX_RQ_CLK_VALID) { in scx_bpf_now()
7304 clock = READ_ONCE(rq->scx.clock); in scx_bpf_now()