Lines Matching refs:rq
119 DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
265 void sched_core_enqueue(struct rq *rq, struct task_struct *p) in sched_core_enqueue() argument
270 rq->core->core_task_seq++; in sched_core_enqueue()
275 rb_add(&p->core_node, &rq->core_tree, rb_sched_core_less); in sched_core_enqueue()
278 void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) in sched_core_dequeue() argument
283 rq->core->core_task_seq++; in sched_core_dequeue()
286 rb_erase(&p->core_node, &rq->core_tree); in sched_core_dequeue()
295 if (!(flags & DEQUEUE_SAVE) && rq->nr_running == 1 && in sched_core_dequeue()
296 rq->core->core_forceidle_count && rq->curr == rq->idle) in sched_core_dequeue()
297 resched_curr(rq); in sched_core_dequeue()
331 static struct task_struct *sched_core_find(struct rq *rq, unsigned long cookie) in sched_core_find() argument
336 node = rb_find_first((void *)cookie, &rq->core_tree, rb_sched_core_cmp); in sched_core_find()
341 if (!sched_task_is_throttled(p, rq->cpu)) in sched_core_find()
485 static inline void sched_core_enqueue(struct rq *rq, struct task_struct *p) { } in sched_core_enqueue() argument
487 sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { } in sched_core_dequeue() argument
598 void raw_spin_rq_lock_nested(struct rq *rq, int subclass) in raw_spin_rq_lock_nested() argument
605 raw_spin_lock_nested(&rq->__lock, subclass); in raw_spin_rq_lock_nested()
612 lock = __rq_lockp(rq); in raw_spin_rq_lock_nested()
614 if (likely(lock == __rq_lockp(rq))) { in raw_spin_rq_lock_nested()
623 bool raw_spin_rq_trylock(struct rq *rq) in raw_spin_rq_trylock() argument
631 ret = raw_spin_trylock(&rq->__lock); in raw_spin_rq_trylock()
637 lock = __rq_lockp(rq); in raw_spin_rq_trylock()
639 if (!ret || (likely(lock == __rq_lockp(rq)))) { in raw_spin_rq_trylock()
647 void raw_spin_rq_unlock(struct rq *rq) in raw_spin_rq_unlock() argument
649 raw_spin_unlock(rq_lockp(rq)); in raw_spin_rq_unlock()
656 void double_rq_lock(struct rq *rq1, struct rq *rq2) in double_rq_lock()
674 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf) in __task_rq_lock()
675 __acquires(rq->lock) in __task_rq_lock()
677 struct rq *rq; in __task_rq_lock() local
682 rq = task_rq(p); in __task_rq_lock()
683 raw_spin_rq_lock(rq); in __task_rq_lock()
684 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { in __task_rq_lock()
685 rq_pin_lock(rq, rf); in __task_rq_lock()
686 return rq; in __task_rq_lock()
688 raw_spin_rq_unlock(rq); in __task_rq_lock()
698 struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf) in task_rq_lock()
700 __acquires(rq->lock) in task_rq_lock()
702 struct rq *rq; in task_rq_lock() local
706 rq = task_rq(p); in task_rq_lock()
707 raw_spin_rq_lock(rq); in task_rq_lock()
725 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { in task_rq_lock()
726 rq_pin_lock(rq, rf); in task_rq_lock()
727 return rq; in task_rq_lock()
729 raw_spin_rq_unlock(rq); in task_rq_lock()
741 static void update_rq_clock_task(struct rq *rq, s64 delta) in update_rq_clock_task() argument
751 irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; in update_rq_clock_task()
771 rq->prev_irq_time += irq_delta; in update_rq_clock_task()
773 delayacct_irq(rq->curr, irq_delta); in update_rq_clock_task()
780 steal = prev_steal = paravirt_steal_clock(cpu_of(rq)); in update_rq_clock_task()
781 steal -= rq->prev_steal_time_rq; in update_rq_clock_task()
786 rq->prev_steal_time_rq = prev_steal; in update_rq_clock_task()
791 rq->clock_task += delta; in update_rq_clock_task()
795 update_irq_load_avg(rq, irq_delta + steal); in update_rq_clock_task()
797 update_rq_clock_pelt(rq, delta); in update_rq_clock_task()
800 void update_rq_clock(struct rq *rq) in update_rq_clock() argument
805 lockdep_assert_rq_held(rq); in update_rq_clock()
807 if (rq->clock_update_flags & RQCF_ACT_SKIP) in update_rq_clock()
811 WARN_ON_ONCE(rq->clock_update_flags & RQCF_UPDATED); in update_rq_clock()
812 rq->clock_update_flags |= RQCF_UPDATED; in update_rq_clock()
814 clock = sched_clock_cpu(cpu_of(rq)); in update_rq_clock()
815 scx_rq_clock_update(rq, clock); in update_rq_clock()
817 delta = clock - rq->clock; in update_rq_clock()
820 rq->clock += delta; in update_rq_clock()
822 update_rq_clock_task(rq, delta); in update_rq_clock()
830 static void hrtick_clear(struct rq *rq) in hrtick_clear() argument
832 if (hrtimer_active(&rq->hrtick_timer)) in hrtick_clear()
833 hrtimer_cancel(&rq->hrtick_timer); in hrtick_clear()
842 struct rq *rq = container_of(timer, struct rq, hrtick_timer); in hrtick() local
845 WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); in hrtick()
847 rq_lock(rq, &rf); in hrtick()
848 update_rq_clock(rq); in hrtick()
849 rq->donor->sched_class->task_tick(rq, rq->curr, 1); in hrtick()
850 rq_unlock(rq, &rf); in hrtick()
857 static void __hrtick_restart(struct rq *rq) in __hrtick_restart() argument
859 struct hrtimer *timer = &rq->hrtick_timer; in __hrtick_restart()
860 ktime_t time = rq->hrtick_time; in __hrtick_restart()
870 struct rq *rq = arg; in __hrtick_start() local
873 rq_lock(rq, &rf); in __hrtick_start()
874 __hrtick_restart(rq); in __hrtick_start()
875 rq_unlock(rq, &rf); in __hrtick_start()
883 void hrtick_start(struct rq *rq, u64 delay) in hrtick_start() argument
885 struct hrtimer *timer = &rq->hrtick_timer; in hrtick_start()
893 rq->hrtick_time = ktime_add_ns(timer->base->get_time(), delta); in hrtick_start()
895 if (rq == this_rq()) in hrtick_start()
896 __hrtick_restart(rq); in hrtick_start()
898 smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd); in hrtick_start()
907 void hrtick_start(struct rq *rq, u64 delay) in hrtick_start() argument
914 hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), in hrtick_start()
920 static void hrtick_rq_init(struct rq *rq) in hrtick_rq_init() argument
923 INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq); in hrtick_rq_init()
925 hrtimer_setup(&rq->hrtick_timer, hrtick, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); in hrtick_rq_init()
928 static inline void hrtick_clear(struct rq *rq) in hrtick_clear() argument
932 static inline void hrtick_rq_init(struct rq *rq) in hrtick_rq_init() argument
1092 static void __resched_curr(struct rq *rq, int tif) in __resched_curr() argument
1094 struct task_struct *curr = rq->curr; in __resched_curr()
1098 lockdep_assert_rq_held(rq); in __resched_curr()
1110 cpu = cpu_of(rq); in __resched_curr()
1127 void resched_curr(struct rq *rq) in resched_curr() argument
1129 __resched_curr(rq, TIF_NEED_RESCHED); in resched_curr()
1153 void resched_curr_lazy(struct rq *rq) in resched_curr_lazy() argument
1155 __resched_curr(rq, get_lazy_tif_bit()); in resched_curr_lazy()
1160 struct rq *rq = cpu_rq(cpu); in resched_cpu() local
1163 raw_spin_rq_lock_irqsave(rq, flags); in resched_cpu()
1165 resched_curr(rq); in resched_cpu()
1166 raw_spin_rq_unlock_irqrestore(rq, flags); in resched_cpu()
1223 struct rq *rq = cpu_rq(cpu); in wake_up_idle_cpu() local
1250 if (set_nr_and_not_polling(task_thread_info(rq->idle), TIF_NEED_RESCHED)) in wake_up_idle_cpu()
1289 struct rq *rq = info; in nohz_csd_func() local
1290 int cpu = cpu_of(rq); in nohz_csd_func()
1299 rq->idle_balance = idle_cpu(cpu); in nohz_csd_func()
1300 if (rq->idle_balance) { in nohz_csd_func()
1301 rq->nohz_idle_balance = flags; in nohz_csd_func()
1309 static inline bool __need_bw_check(struct rq *rq, struct task_struct *p) in __need_bw_check() argument
1311 if (rq->nr_running != 1) in __need_bw_check()
1323 bool sched_can_stop_tick(struct rq *rq) in sched_can_stop_tick() argument
1328 if (rq->dl.dl_nr_running) in sched_can_stop_tick()
1335 if (rq->rt.rr_nr_running) { in sched_can_stop_tick()
1336 if (rq->rt.rr_nr_running == 1) in sched_can_stop_tick()
1346 fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running; in sched_can_stop_tick()
1355 if (scx_enabled() && !scx_can_stop_tick(rq)) in sched_can_stop_tick()
1358 if (rq->cfs.h_nr_queued > 1) in sched_can_stop_tick()
1368 if (__need_bw_check(rq, rq->curr)) { in sched_can_stop_tick()
1369 if (cfs_task_bw_constrained(rq->curr)) in sched_can_stop_tick()
1506 uclamp_idle_value(struct rq *rq, enum uclamp_id clamp_id, in uclamp_idle_value() argument
1515 rq->uclamp_flags |= UCLAMP_FLAG_IDLE; in uclamp_idle_value()
1522 static inline void uclamp_idle_reset(struct rq *rq, enum uclamp_id clamp_id, in uclamp_idle_reset() argument
1526 if (!(rq->uclamp_flags & UCLAMP_FLAG_IDLE)) in uclamp_idle_reset()
1529 uclamp_rq_set(rq, clamp_id, clamp_value); in uclamp_idle_reset()
1533 unsigned int uclamp_rq_max_value(struct rq *rq, enum uclamp_id clamp_id, in uclamp_rq_max_value() argument
1536 struct uclamp_bucket *bucket = rq->uclamp[clamp_id].bucket; in uclamp_rq_max_value()
1550 return uclamp_idle_value(rq, clamp_id, clamp_value); in uclamp_rq_max_value()
1651 static inline void uclamp_rq_inc_id(struct rq *rq, struct task_struct *p, in uclamp_rq_inc_id() argument
1654 struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; in uclamp_rq_inc_id()
1658 lockdep_assert_rq_held(rq); in uclamp_rq_inc_id()
1667 uclamp_idle_reset(rq, clamp_id, uc_se->value); in uclamp_rq_inc_id()
1676 if (uc_se->value > uclamp_rq_get(rq, clamp_id)) in uclamp_rq_inc_id()
1677 uclamp_rq_set(rq, clamp_id, uc_se->value); in uclamp_rq_inc_id()
1689 static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p, in uclamp_rq_dec_id() argument
1692 struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; in uclamp_rq_dec_id()
1698 lockdep_assert_rq_held(rq); in uclamp_rq_dec_id()
1743 rq_clamp = uclamp_rq_get(rq, clamp_id); in uclamp_rq_dec_id()
1750 bkt_clamp = uclamp_rq_max_value(rq, clamp_id, uc_se->value); in uclamp_rq_dec_id()
1751 uclamp_rq_set(rq, clamp_id, bkt_clamp); in uclamp_rq_dec_id()
1755 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) in uclamp_rq_inc() argument
1775 uclamp_rq_inc_id(rq, p, clamp_id); in uclamp_rq_inc()
1778 if (rq->uclamp_flags & UCLAMP_FLAG_IDLE) in uclamp_rq_inc()
1779 rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; in uclamp_rq_inc()
1782 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) in uclamp_rq_dec() argument
1802 uclamp_rq_dec_id(rq, p, clamp_id); in uclamp_rq_dec()
1805 static inline void uclamp_rq_reinc_id(struct rq *rq, struct task_struct *p, in uclamp_rq_reinc_id() argument
1811 uclamp_rq_dec_id(rq, p, clamp_id); in uclamp_rq_reinc_id()
1812 uclamp_rq_inc_id(rq, p, clamp_id); in uclamp_rq_reinc_id()
1818 if (clamp_id == UCLAMP_MAX && (rq->uclamp_flags & UCLAMP_FLAG_IDLE)) in uclamp_rq_reinc_id()
1819 rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; in uclamp_rq_reinc_id()
1827 struct rq *rq; in uclamp_update_active() local
1837 rq = task_rq_lock(p, &rf); in uclamp_update_active()
1846 uclamp_rq_reinc_id(rq, p, clamp_id); in uclamp_update_active()
1848 task_rq_unlock(rq, p, &rf); in uclamp_update_active()
1999 static void __init init_uclamp_rq(struct rq *rq) in init_uclamp_rq() argument
2002 struct uclamp_rq *uc_rq = rq->uclamp; in init_uclamp_rq()
2010 rq->uclamp_flags = UCLAMP_FLAG_IDLE; in init_uclamp_rq()
2039 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) { } in uclamp_rq_inc() argument
2040 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) { } in uclamp_rq_dec() argument
2070 void enqueue_task(struct rq *rq, struct task_struct *p, int flags) in enqueue_task() argument
2073 update_rq_clock(rq); in enqueue_task()
2075 p->sched_class->enqueue_task(rq, p, flags); in enqueue_task()
2080 uclamp_rq_inc(rq, p); in enqueue_task()
2085 sched_info_enqueue(rq, p); in enqueue_task()
2087 if (sched_core_enabled(rq)) in enqueue_task()
2088 sched_core_enqueue(rq, p); in enqueue_task()
2094 inline bool dequeue_task(struct rq *rq, struct task_struct *p, int flags) in dequeue_task() argument
2096 if (sched_core_enabled(rq)) in dequeue_task()
2097 sched_core_dequeue(rq, p, flags); in dequeue_task()
2100 update_rq_clock(rq); in dequeue_task()
2103 sched_info_dequeue(rq, p); in dequeue_task()
2111 uclamp_rq_dec(rq, p); in dequeue_task()
2112 return p->sched_class->dequeue_task(rq, p, flags); in dequeue_task()
2115 void activate_task(struct rq *rq, struct task_struct *p, int flags) in activate_task() argument
2120 sched_mm_cid_migrate_to(rq, p); in activate_task()
2122 enqueue_task(rq, p, flags); in activate_task()
2128 void deactivate_task(struct rq *rq, struct task_struct *p, int flags) in deactivate_task() argument
2140 dequeue_task(rq, p, flags); in deactivate_task()
2143 static void block_task(struct rq *rq, struct task_struct *p, int flags) in block_task() argument
2145 if (dequeue_task(rq, p, DEQUEUE_SLEEP | flags)) in block_task()
2146 __block_task(rq, p); in block_task()
2164 void check_class_changing(struct rq *rq, struct task_struct *p, in check_class_changing() argument
2168 p->sched_class->switching_to(rq, p); in check_class_changing()
2178 void check_class_changed(struct rq *rq, struct task_struct *p, in check_class_changed() argument
2184 prev_class->switched_from(rq, p); in check_class_changed()
2186 p->sched_class->switched_to(rq, p); in check_class_changed()
2188 p->sched_class->prio_changed(rq, p, oldprio); in check_class_changed()
2191 void wakeup_preempt(struct rq *rq, struct task_struct *p, int flags) in wakeup_preempt() argument
2193 struct task_struct *donor = rq->donor; in wakeup_preempt()
2196 donor->sched_class->wakeup_preempt(rq, p, flags); in wakeup_preempt()
2198 resched_curr(rq); in wakeup_preempt()
2204 if (task_on_rq_queued(donor) && test_tsk_need_resched(rq->curr)) in wakeup_preempt()
2205 rq_clock_skip_update(rq); in wakeup_preempt()
2252 struct rq *rq; in wait_task_inactive() local
2261 rq = task_rq(p); in wait_task_inactive()
2274 while (task_on_cpu(rq, p)) { in wait_task_inactive()
2285 rq = task_rq_lock(p, &rf); in wait_task_inactive()
2287 running = task_on_cpu(rq, p); in wait_task_inactive()
2299 task_rq_unlock(rq, p, &rf); in wait_task_inactive()
2351 static void migrate_disable_switch(struct rq *rq, struct task_struct *p) in migrate_disable_switch() argument
2354 .new_mask = cpumask_of(rq->cpu), in migrate_disable_switch()
2431 static inline bool rq_has_pinned_tasks(struct rq *rq) in rq_has_pinned_tasks() argument
2433 return rq->nr_pinned; in rq_has_pinned_tasks()
2485 static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, in move_queued_task() argument
2488 lockdep_assert_rq_held(rq); in move_queued_task()
2490 deactivate_task(rq, p, DEQUEUE_NOCLOCK); in move_queued_task()
2492 rq_unlock(rq, rf); in move_queued_task()
2494 rq = cpu_rq(new_cpu); in move_queued_task()
2496 rq_lock(rq, rf); in move_queued_task()
2498 activate_task(rq, p, 0); in move_queued_task()
2499 wakeup_preempt(rq, p, 0); in move_queued_task()
2501 return rq; in move_queued_task()
2531 static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, in __migrate_task() argument
2536 return rq; in __migrate_task()
2538 rq = move_queued_task(rq, rf, p, dest_cpu); in __migrate_task()
2540 return rq; in __migrate_task()
2553 struct rq *rq = this_rq(); in migration_cpu_stop() local
2570 rq_lock(rq, &rf); in migration_cpu_stop()
2583 if (task_rq(p) == rq) { in migration_cpu_stop()
2596 update_rq_clock(rq); in migration_cpu_stop()
2597 rq = __migrate_task(rq, &rf, p, arg->dest_cpu); in migration_cpu_stop()
2637 task_rq_unlock(rq, p, &rf); in migration_cpu_stop()
2646 task_rq_unlock(rq, p, &rf); in migration_cpu_stop()
2656 struct rq *lowest_rq = NULL, *rq = this_rq(); in push_cpu_stop() local
2660 raw_spin_rq_lock(rq); in push_cpu_stop()
2662 if (task_rq(p) != rq) in push_cpu_stop()
2673 lowest_rq = p->sched_class->find_lock_rq(p, rq); in push_cpu_stop()
2679 if (task_rq(p) == rq) { in push_cpu_stop()
2680 move_queued_task_locked(rq, lowest_rq, p); in push_cpu_stop()
2684 double_unlock_balance(rq, lowest_rq); in push_cpu_stop()
2687 rq->push_busy = false; in push_cpu_stop()
2688 raw_spin_rq_unlock(rq); in push_cpu_stop()
2719 struct rq *rq = task_rq(p); in __do_set_cpus_allowed() local
2740 running = task_current_donor(rq, p); in __do_set_cpus_allowed()
2747 lockdep_assert_rq_held(rq); in __do_set_cpus_allowed()
2748 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); in __do_set_cpus_allowed()
2751 put_prev_task(rq, p); in __do_set_cpus_allowed()
2757 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); in __do_set_cpus_allowed()
2759 set_next_task(rq, p); in __do_set_cpus_allowed()
2921 static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flags *rf, in affine_move_task() argument
2923 __releases(rq->lock) in affine_move_task()
2934 (p->migration_flags & MDF_PUSH) && !rq->push_busy) { in affine_move_task()
2935 rq->push_busy = true; in affine_move_task()
2950 task_rq_unlock(rq, p, rf); in affine_move_task()
2952 stop_one_cpu_nowait(rq->cpu, push_cpu_stop, in affine_move_task()
2953 p, &rq->push_work); in affine_move_task()
3004 task_rq_unlock(rq, p, rf); in affine_move_task()
3008 if (task_on_cpu(rq, p) || READ_ONCE(p->__state) == TASK_WAKING) { in affine_move_task()
3022 task_rq_unlock(rq, p, rf); in affine_move_task()
3024 stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop, in affine_move_task()
3035 rq = move_queued_task(rq, rf, p, dest_cpu); in affine_move_task()
3042 task_rq_unlock(rq, p, rf); in affine_move_task()
3070 struct rq *rq, in __set_cpus_allowed_ptr_locked() argument
3072 __releases(rq->lock) in __set_cpus_allowed_ptr_locked()
3081 update_rq_clock(rq); in __set_cpus_allowed_ptr_locked()
3139 return affine_move_task(rq, p, rf, dest_cpu, ctx->flags); in __set_cpus_allowed_ptr_locked()
3142 task_rq_unlock(rq, p, rf); in __set_cpus_allowed_ptr_locked()
3159 struct rq *rq; in __set_cpus_allowed_ptr() local
3161 rq = task_rq_lock(p, &rf); in __set_cpus_allowed_ptr()
3168 cpumask_and(rq->scratch_mask, ctx->new_mask, p->user_cpus_ptr)) in __set_cpus_allowed_ptr()
3169 ctx->new_mask = rq->scratch_mask; in __set_cpus_allowed_ptr()
3171 return __set_cpus_allowed_ptr_locked(p, ctx, rq, &rf); in __set_cpus_allowed_ptr()
3203 struct rq *rq; in restrict_cpus_allowed_ptr() local
3206 rq = task_rq_lock(p, &rf); in restrict_cpus_allowed_ptr()
3223 return __set_cpus_allowed_ptr_locked(p, &ac, rq, &rf); in restrict_cpus_allowed_ptr()
3226 task_rq_unlock(rq, p, &rf); in restrict_cpus_allowed_ptr()
3356 struct rq *src_rq, *dst_rq; in __migrate_swap_task()
3389 struct rq *src_rq, *dst_rq; in migrate_swap_stop()
3652 static inline void migrate_disable_switch(struct rq *rq, struct task_struct *p) { } in migrate_disable_switch() argument
3654 static inline bool rq_has_pinned_tasks(struct rq *rq) in rq_has_pinned_tasks() argument
3664 struct rq *rq; in ttwu_stat() local
3669 rq = this_rq(); in ttwu_stat()
3672 if (cpu == rq->cpu) { in ttwu_stat()
3673 __schedstat_inc(rq->ttwu_local); in ttwu_stat()
3681 for_each_domain(rq->cpu, sd) { in ttwu_stat()
3693 __schedstat_inc(rq->ttwu_count); in ttwu_stat()
3710 ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, in ttwu_do_activate() argument
3715 lockdep_assert_rq_held(rq); in ttwu_do_activate()
3718 rq->nr_uninterruptible--; in ttwu_do_activate()
3732 activate_task(rq, p, en_flags); in ttwu_do_activate()
3733 wakeup_preempt(rq, p, wake_flags); in ttwu_do_activate()
3743 rq_unpin_lock(rq, rf); in ttwu_do_activate()
3744 p->sched_class->task_woken(rq, p); in ttwu_do_activate()
3745 rq_repin_lock(rq, rf); in ttwu_do_activate()
3748 if (rq->idle_stamp) { in ttwu_do_activate()
3749 u64 delta = rq_clock(rq) - rq->idle_stamp; in ttwu_do_activate()
3750 u64 max = 2*rq->max_idle_balance_cost; in ttwu_do_activate()
3752 update_avg(&rq->avg_idle, delta); in ttwu_do_activate()
3754 if (rq->avg_idle > max) in ttwu_do_activate()
3755 rq->avg_idle = max; in ttwu_do_activate()
3757 rq->idle_stamp = 0; in ttwu_do_activate()
3790 struct rq *rq; in ttwu_runnable() local
3793 rq = __task_rq_lock(p, &rf); in ttwu_runnable()
3795 update_rq_clock(rq); in ttwu_runnable()
3797 enqueue_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_DELAYED); in ttwu_runnable()
3798 if (!task_on_cpu(rq, p)) { in ttwu_runnable()
3803 wakeup_preempt(rq, p, wake_flags); in ttwu_runnable()
3808 __task_rq_unlock(rq, &rf); in ttwu_runnable()
3817 struct rq *rq = this_rq(); in sched_ttwu_pending() local
3824 rq_lock_irqsave(rq, &rf); in sched_ttwu_pending()
3825 update_rq_clock(rq); in sched_ttwu_pending()
3831 if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq))) in sched_ttwu_pending()
3832 set_task_cpu(p, cpu_of(rq)); in sched_ttwu_pending()
3834 ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf); in sched_ttwu_pending()
3847 WRITE_ONCE(rq->ttwu_pending, 0); in sched_ttwu_pending()
3848 rq_unlock_irqrestore(rq, &rf); in sched_ttwu_pending()
3875 struct rq *rq = cpu_rq(cpu); in __ttwu_queue_wakelist() local
3879 WRITE_ONCE(rq->ttwu_pending, 1); in __ttwu_queue_wakelist()
3885 struct rq *rq = cpu_rq(cpu); in wake_up_if_idle() local
3888 if (is_idle_task(rcu_dereference(rq->curr))) { in wake_up_if_idle()
3889 guard(rq_lock_irqsave)(rq); in wake_up_if_idle()
3890 if (is_idle_task(rq->curr)) in wake_up_if_idle()
3891 resched_curr(rq); in wake_up_if_idle()
3992 struct rq *rq = cpu_rq(cpu); in ttwu_queue() local
3998 rq_lock(rq, &rf); in ttwu_queue()
3999 update_rq_clock(rq); in ttwu_queue()
4000 ttwu_do_activate(rq, p, wake_flags, &rf); in ttwu_queue()
4001 rq_unlock(rq, &rf); in ttwu_queue()
4388 struct rq *rq = NULL; in task_call_func() local
4395 rq = __task_rq_lock(p, &rf); in task_call_func()
4409 if (rq) in task_call_func()
4410 rq_unlock(rq, &rf); in task_call_func()
4435 struct rq *rq = cpu_rq(cpu); in cpu_curr_snapshot() local
4439 rq_lock_irqsave(rq, &rf); in cpu_curr_snapshot()
4442 rq_unlock_irqrestore(rq, &rf); in cpu_curr_snapshot()
4855 struct rq *rq; in wake_up_new_task() local
4873 rq = __task_rq_lock(p, &rf); in wake_up_new_task()
4874 update_rq_clock(rq); in wake_up_new_task()
4877 activate_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_INITIAL); in wake_up_new_task()
4879 wakeup_preempt(rq, p, wake_flags); in wake_up_new_task()
4886 rq_unpin_lock(rq, &rf); in wake_up_new_task()
4887 p->sched_class->task_woken(rq, p); in wake_up_new_task()
4888 rq_repin_lock(rq, &rf); in wake_up_new_task()
4891 task_rq_unlock(rq, p, &rf); in wake_up_new_task()
5015 static void do_balance_callbacks(struct rq *rq, struct balance_callback *head) in do_balance_callbacks() argument
5017 void (*func)(struct rq *rq); in do_balance_callbacks()
5020 lockdep_assert_rq_held(rq); in do_balance_callbacks()
5023 func = (void (*)(struct rq *))head->func; in do_balance_callbacks()
5028 func(rq); in do_balance_callbacks()
5032 static void balance_push(struct rq *rq);
5051 __splice_balance_callbacks(struct rq *rq, bool split) in __splice_balance_callbacks() argument
5053 struct balance_callback *head = rq->balance_callback; in __splice_balance_callbacks()
5058 lockdep_assert_rq_held(rq); in __splice_balance_callbacks()
5070 rq->balance_callback = NULL; in __splice_balance_callbacks()
5075 struct balance_callback *splice_balance_callbacks(struct rq *rq) in splice_balance_callbacks() argument
5077 return __splice_balance_callbacks(rq, true); in splice_balance_callbacks()
5080 static void __balance_callbacks(struct rq *rq) in __balance_callbacks() argument
5082 do_balance_callbacks(rq, __splice_balance_callbacks(rq, false)); in __balance_callbacks()
5085 void balance_callbacks(struct rq *rq, struct balance_callback *head) in balance_callbacks() argument
5090 raw_spin_rq_lock_irqsave(rq, flags); in balance_callbacks()
5091 do_balance_callbacks(rq, head); in balance_callbacks()
5092 raw_spin_rq_unlock_irqrestore(rq, flags); in balance_callbacks()
5098 static inline void __balance_callbacks(struct rq *rq) in __balance_callbacks() argument
5105 prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf) in prepare_lock_switch() argument
5113 rq_unpin_lock(rq, rf); in prepare_lock_switch()
5114 spin_release(&__rq_lockp(rq)->dep_map, _THIS_IP_); in prepare_lock_switch()
5117 rq_lockp(rq)->owner = next; in prepare_lock_switch()
5121 static inline void finish_lock_switch(struct rq *rq) in finish_lock_switch() argument
5128 spin_acquire(&__rq_lockp(rq)->dep_map, 0, 0, _THIS_IP_); in finish_lock_switch()
5129 __balance_callbacks(rq); in finish_lock_switch()
5130 raw_spin_rq_unlock_irq(rq); in finish_lock_switch()
5175 prepare_task_switch(struct rq *rq, struct task_struct *prev, in prepare_task_switch() argument
5179 sched_info_switch(rq, prev, next); in prepare_task_switch()
5207 static struct rq *finish_task_switch(struct task_struct *prev) in finish_task_switch()
5208 __releases(rq->lock) in finish_task_switch()
5210 struct rq *rq = this_rq(); in finish_task_switch() local
5211 struct mm_struct *mm = rq->prev_mm; in finish_task_switch()
5230 rq->prev_mm = NULL; in finish_task_switch()
5248 finish_lock_switch(rq); in finish_task_switch()
5288 return rq; in finish_task_switch()
5296 __releases(rq->lock) in schedule_tail()
5325 static __always_inline struct rq *
5326 context_switch(struct rq *rq, struct task_struct *prev, in context_switch() argument
5329 prepare_task_switch(rq, prev, next); in context_switch()
5357 membarrier_switch_mm(rq, prev->active_mm, next->mm); in context_switch()
5371 rq->prev_mm = prev->active_mm; in context_switch()
5377 switch_mm_cid(rq, prev, next); in context_switch()
5379 prepare_lock_switch(rq, next, rf); in context_switch()
5549 struct rq *rq; in task_sched_runtime() local
5568 rq = task_rq_lock(p, &rf); in task_sched_runtime()
5574 if (task_current_donor(rq, p) && task_on_rq_queued(p)) { in task_sched_runtime()
5576 update_rq_clock(rq); in task_sched_runtime()
5577 p->sched_class->update_curr(rq); in task_sched_runtime()
5580 task_rq_unlock(rq, p, &rf); in task_sched_runtime()
5585 static u64 cpu_resched_latency(struct rq *rq) in cpu_resched_latency() argument
5588 u64 resched_latency, now = rq_clock(rq); in cpu_resched_latency()
5600 if (!rq->last_seen_need_resched_ns) { in cpu_resched_latency()
5601 rq->last_seen_need_resched_ns = now; in cpu_resched_latency()
5602 rq->ticks_without_resched = 0; in cpu_resched_latency()
5606 rq->ticks_without_resched++; in cpu_resched_latency()
5607 resched_latency = now - rq->last_seen_need_resched_ns; in cpu_resched_latency()
5637 struct rq *rq = cpu_rq(cpu); in sched_tick() local
5649 rq_lock(rq, &rf); in sched_tick()
5650 donor = rq->donor; in sched_tick()
5652 psi_account_irqtime(rq, donor, NULL); in sched_tick()
5654 update_rq_clock(rq); in sched_tick()
5655 hw_pressure = arch_scale_hw_pressure(cpu_of(rq)); in sched_tick()
5656 update_hw_load_avg(rq_clock_task(rq), rq, hw_pressure); in sched_tick()
5659 resched_curr(rq); in sched_tick()
5661 donor->sched_class->task_tick(rq, donor, 0); in sched_tick()
5663 resched_latency = cpu_resched_latency(rq); in sched_tick()
5664 calc_global_load_tick(rq); in sched_tick()
5665 sched_core_tick(rq); in sched_tick()
5666 task_tick_mm_cid(rq, donor); in sched_tick()
5667 scx_tick(rq); in sched_tick()
5669 rq_unlock(rq, &rf); in sched_tick()
5681 rq->idle_balance = idle_cpu(cpu); in sched_tick()
5682 sched_balance_trigger(rq); in sched_tick()
5729 struct rq *rq = cpu_rq(cpu); in sched_tick_remote() local
5740 guard(rq_lock_irq)(rq); in sched_tick_remote()
5741 struct task_struct *curr = rq->curr; in sched_tick_remote()
5749 WARN_ON_ONCE(rq->curr != rq->donor); in sched_tick_remote()
5750 update_rq_clock(rq); in sched_tick_remote()
5757 u64 delta = rq_clock_task(rq) - curr->se.exec_start; in sched_tick_remote()
5760 curr->sched_class->task_tick(rq, curr, 0); in sched_tick_remote()
5762 calc_load_nohz_remote(rq); in sched_tick_remote()
5976 static void prev_balance(struct rq *rq, struct task_struct *prev, in prev_balance() argument
5989 rq->scx.flags |= SCX_RQ_BAL_PENDING; in prev_balance()
6004 if (class->balance && class->balance(rq, prev, rf)) in prev_balance()
6013 __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) in __pick_next_task() argument
6018 rq->dl_server = NULL; in __pick_next_task()
6030 rq->nr_running == rq->cfs.h_nr_queued)) { in __pick_next_task()
6032 p = pick_next_task_fair(rq, prev, rf); in __pick_next_task()
6038 p = pick_task_idle(rq); in __pick_next_task()
6039 put_prev_set_next_task(rq, prev, p); in __pick_next_task()
6046 prev_balance(rq, prev, rf); in __pick_next_task()
6050 p = class->pick_next_task(rq, prev); in __pick_next_task()
6054 p = class->pick_task(rq); in __pick_next_task()
6056 put_prev_set_next_task(rq, prev, p); in __pick_next_task()
6084 static inline struct task_struct *pick_task(struct rq *rq) in pick_task() argument
6089 rq->dl_server = NULL; in pick_task()
6092 p = class->pick_task(rq); in pick_task()
6100 extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi);
6102 static void queue_core_balance(struct rq *rq);
6105 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) in pick_next_task() argument
6110 bool core_clock_updated = (rq == rq->core); in pick_next_task()
6113 struct rq *rq_i; in pick_next_task()
6116 if (!sched_core_enabled(rq)) in pick_next_task()
6117 return __pick_next_task(rq, prev, rf); in pick_next_task()
6119 cpu = cpu_of(rq); in pick_next_task()
6128 rq->core_pick = NULL; in pick_next_task()
6129 rq->core_dl_server = NULL; in pick_next_task()
6130 return __pick_next_task(rq, prev, rf); in pick_next_task()
6142 if (rq->core->core_pick_seq == rq->core->core_task_seq && in pick_next_task()
6143 rq->core->core_pick_seq != rq->core_sched_seq && in pick_next_task()
6144 rq->core_pick) { in pick_next_task()
6145 WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq); in pick_next_task()
6147 next = rq->core_pick; in pick_next_task()
6148 rq->dl_server = rq->core_dl_server; in pick_next_task()
6149 rq->core_pick = NULL; in pick_next_task()
6150 rq->core_dl_server = NULL; in pick_next_task()
6154 prev_balance(rq, prev, rf); in pick_next_task()
6157 need_sync = !!rq->core->core_cookie; in pick_next_task()
6160 rq->core->core_cookie = 0UL; in pick_next_task()
6161 if (rq->core->core_forceidle_count) { in pick_next_task()
6163 update_rq_clock(rq->core); in pick_next_task()
6166 sched_core_account_forceidle(rq); in pick_next_task()
6168 rq->core->core_forceidle_start = 0; in pick_next_task()
6169 rq->core->core_forceidle_count = 0; in pick_next_task()
6170 rq->core->core_forceidle_occupation = 0; in pick_next_task()
6185 rq->core->core_task_seq++; in pick_next_task()
6192 next = pick_task(rq); in pick_next_task()
6194 rq->core_pick = NULL; in pick_next_task()
6195 rq->core_dl_server = NULL; in pick_next_task()
6201 task_vruntime_update(rq, next, false); in pick_next_task()
6220 if (i != cpu && (rq_i != rq->core || !core_clock_updated)) in pick_next_task()
6230 cookie = rq->core->core_cookie = max->core_cookie; in pick_next_task()
6253 rq->core->core_forceidle_count++; in pick_next_task()
6255 rq->core->core_forceidle_seq++; in pick_next_task()
6262 if (schedstat_enabled() && rq->core->core_forceidle_count) { in pick_next_task()
6263 rq->core->core_forceidle_start = rq_clock(rq->core); in pick_next_task()
6264 rq->core->core_forceidle_occupation = occ; in pick_next_task()
6267 rq->core->core_pick_seq = rq->core->core_task_seq; in pick_next_task()
6268 next = rq->core_pick; in pick_next_task()
6269 rq->core_sched_seq = rq->core->core_pick_seq; in pick_next_task()
6303 if (!(fi_before && rq->core->core_forceidle_count)) in pick_next_task()
6304 task_vruntime_update(rq_i, rq_i->core_pick, !!rq->core->core_forceidle_count); in pick_next_task()
6327 put_prev_set_next_task(rq, prev, next); in pick_next_task()
6328 if (rq->core->core_forceidle_count && next == rq->idle) in pick_next_task()
6329 queue_core_balance(rq); in pick_next_task()
6336 struct rq *dst = cpu_rq(this), *src = cpu_rq(that); in try_steal_cookie()
6404 static void sched_core_balance(struct rq *rq) in sched_core_balance() argument
6407 int cpu = cpu_of(rq); in sched_core_balance()
6412 raw_spin_rq_unlock_irq(rq); in sched_core_balance()
6420 raw_spin_rq_lock_irq(rq); in sched_core_balance()
6425 static void queue_core_balance(struct rq *rq) in queue_core_balance() argument
6427 if (!sched_core_enabled(rq)) in queue_core_balance()
6430 if (!rq->core->core_cookie) in queue_core_balance()
6433 if (!rq->nr_running) /* not forced idle */ in queue_core_balance()
6436 queue_balance_callback(rq, &per_cpu(core_balance_head, rq->cpu), sched_core_balance); in queue_core_balance()
6447 struct rq *rq = cpu_rq(cpu), *core_rq = NULL; in sched_core_cpu_starting() local
6452 WARN_ON_ONCE(rq->core != rq); in sched_core_cpu_starting()
6462 rq = cpu_rq(t); in sched_core_cpu_starting()
6463 if (rq->core == rq) { in sched_core_cpu_starting()
6464 core_rq = rq; in sched_core_cpu_starting()
6474 rq = cpu_rq(t); in sched_core_cpu_starting()
6477 rq->core = core_rq; in sched_core_cpu_starting()
6479 WARN_ON_ONCE(rq->core != core_rq); in sched_core_cpu_starting()
6486 struct rq *rq = cpu_rq(cpu), *core_rq = NULL; in sched_core_cpu_deactivate() local
6493 WARN_ON_ONCE(rq->core != rq); in sched_core_cpu_deactivate()
6498 if (rq->core != rq) in sched_core_cpu_deactivate()
6513 core_rq->core_task_seq = rq->core_task_seq; in sched_core_cpu_deactivate()
6514 core_rq->core_pick_seq = rq->core_pick_seq; in sched_core_cpu_deactivate()
6515 core_rq->core_cookie = rq->core_cookie; in sched_core_cpu_deactivate()
6516 core_rq->core_forceidle_count = rq->core_forceidle_count; in sched_core_cpu_deactivate()
6517 core_rq->core_forceidle_seq = rq->core_forceidle_seq; in sched_core_cpu_deactivate()
6518 core_rq->core_forceidle_occupation = rq->core_forceidle_occupation; in sched_core_cpu_deactivate()
6529 rq = cpu_rq(t); in sched_core_cpu_deactivate()
6530 rq->core = core_rq; in sched_core_cpu_deactivate()
6536 struct rq *rq = cpu_rq(cpu); in sched_core_cpu_dying() local
6538 if (rq->core != rq) in sched_core_cpu_dying()
6539 rq->core = rq; in sched_core_cpu_dying()
6549 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) in pick_next_task() argument
6551 return __pick_next_task(rq, prev, rf); in pick_next_task()
6573 static bool try_to_block_task(struct rq *rq, struct task_struct *p, in try_to_block_task() argument
6602 block_task(rq, p, flags); in try_to_block_task()
6657 struct rq *rq; in __schedule() local
6663 rq = cpu_rq(cpu); in __schedule()
6664 prev = rq->curr; in __schedule()
6669 hrtick_clear(rq); in __schedule()
6691 rq_lock(rq, &rf); in __schedule()
6695 rq->clock_update_flags <<= 1; in __schedule()
6696 update_rq_clock(rq); in __schedule()
6697 rq->clock_update_flags = RQCF_UPDATED; in __schedule()
6711 if (!rq->nr_running && !scx_enabled()) { in __schedule()
6716 try_to_block_task(rq, prev, prev_state); in __schedule()
6720 next = pick_next_task(rq, prev, &rf); in __schedule()
6721 rq_set_donor(rq, next); in __schedule()
6725 rq->last_seen_need_resched_ns = 0; in __schedule()
6729 rq->nr_switches++; in __schedule()
6734 RCU_INIT_POINTER(rq->curr, next); in __schedule()
6759 migrate_disable_switch(rq, prev); in __schedule()
6760 psi_account_irqtime(rq, prev, next); in __schedule()
6767 rq = context_switch(rq, prev, next, &rf); in __schedule()
6769 rq_unpin_lock(rq, &rf); in __schedule()
6770 __balance_callbacks(rq); in __schedule()
6771 raw_spin_rq_unlock_irq(rq); in __schedule()
7167 struct rq *rq; in rt_mutex_setprio() local
7178 rq = __task_rq_lock(p, &rf); in rt_mutex_setprio()
7179 update_rq_clock(rq); in rt_mutex_setprio()
7210 if (unlikely(p == rq->idle)) { in rt_mutex_setprio()
7211 WARN_ON(p != rq->curr); in rt_mutex_setprio()
7226 dequeue_task(rq, p, DEQUEUE_SLEEP | DEQUEUE_DELAYED | DEQUEUE_NOCLOCK); in rt_mutex_setprio()
7229 running = task_current_donor(rq, p); in rt_mutex_setprio()
7231 dequeue_task(rq, p, queue_flag); in rt_mutex_setprio()
7233 put_prev_task(rq, p); in rt_mutex_setprio()
7268 check_class_changing(rq, p, prev_class); in rt_mutex_setprio()
7271 enqueue_task(rq, p, queue_flag); in rt_mutex_setprio()
7273 set_next_task(rq, p); in rt_mutex_setprio()
7275 check_class_changed(rq, p, prev_class, oldprio); in rt_mutex_setprio()
7280 rq_unpin_lock(rq, &rf); in rt_mutex_setprio()
7281 __balance_callbacks(rq); in rt_mutex_setprio()
7282 raw_spin_rq_unlock(rq); in rt_mutex_setprio()
7847 struct rq *rq = cpu_rq(cpu); in init_idle() local
7851 raw_spin_rq_lock(rq); in init_idle()
7883 rq->idle = idle; in init_idle()
7884 rq_set_donor(rq, idle); in init_idle()
7885 rcu_assign_pointer(rq->curr, idle); in init_idle()
7890 raw_spin_rq_unlock(rq); in init_idle()
7970 struct rq *rq; in sched_setnuma() local
7972 rq = task_rq_lock(p, &rf); in sched_setnuma()
7974 running = task_current_donor(rq, p); in sched_setnuma()
7977 dequeue_task(rq, p, DEQUEUE_SAVE); in sched_setnuma()
7979 put_prev_task(rq, p); in sched_setnuma()
7984 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); in sched_setnuma()
7986 set_next_task(rq, p); in sched_setnuma()
7987 task_rq_unlock(rq, p, &rf); in sched_setnuma()
8021 struct rq *rq = this_rq(); in __balance_push_cpu_stop() local
8026 rq_lock(rq, &rf); in __balance_push_cpu_stop()
8028 update_rq_clock(rq); in __balance_push_cpu_stop()
8030 if (task_rq(p) == rq && task_on_rq_queued(p)) { in __balance_push_cpu_stop()
8031 cpu = select_fallback_rq(rq->cpu, p); in __balance_push_cpu_stop()
8032 rq = __migrate_task(rq, &rf, p, cpu); in __balance_push_cpu_stop()
8035 rq_unlock(rq, &rf); in __balance_push_cpu_stop()
8051 static void balance_push(struct rq *rq) in balance_push() argument
8053 struct task_struct *push_task = rq->curr; in balance_push()
8055 lockdep_assert_rq_held(rq); in balance_push()
8060 rq->balance_callback = &balance_push_callback; in balance_push()
8066 if (!cpu_dying(rq->cpu) || rq != this_rq()) in balance_push()
8087 if (!rq->nr_running && !rq_has_pinned_tasks(rq) && in balance_push()
8088 rcuwait_active(&rq->hotplug_wait)) { in balance_push()
8089 raw_spin_rq_unlock(rq); in balance_push()
8090 rcuwait_wake_up(&rq->hotplug_wait); in balance_push()
8091 raw_spin_rq_lock(rq); in balance_push()
8102 raw_spin_rq_unlock(rq); in balance_push()
8103 stop_one_cpu_nowait(rq->cpu, __balance_push_cpu_stop, push_task, in balance_push()
8111 raw_spin_rq_lock(rq); in balance_push()
8116 struct rq *rq = cpu_rq(cpu); in balance_push_set() local
8119 rq_lock_irqsave(rq, &rf); in balance_push_set()
8121 WARN_ON_ONCE(rq->balance_callback); in balance_push_set()
8122 rq->balance_callback = &balance_push_callback; in balance_push_set()
8123 } else if (rq->balance_callback == &balance_push_callback) { in balance_push_set()
8124 rq->balance_callback = NULL; in balance_push_set()
8126 rq_unlock_irqrestore(rq, &rf); in balance_push_set()
8137 struct rq *rq = this_rq(); in balance_hotplug_wait() local
8139 rcuwait_wait_event(&rq->hotplug_wait, in balance_hotplug_wait()
8140 rq->nr_running == 1 && !rq_has_pinned_tasks(rq), in balance_hotplug_wait()
8146 static inline void balance_push(struct rq *rq) in balance_push() argument
8160 void set_rq_online(struct rq *rq) in set_rq_online() argument
8162 if (!rq->online) { in set_rq_online()
8165 cpumask_set_cpu(rq->cpu, rq->rd->online); in set_rq_online()
8166 rq->online = 1; in set_rq_online()
8170 class->rq_online(rq); in set_rq_online()
8175 void set_rq_offline(struct rq *rq) in set_rq_offline() argument
8177 if (rq->online) { in set_rq_offline()
8180 update_rq_clock(rq); in set_rq_offline()
8183 class->rq_offline(rq); in set_rq_offline()
8186 cpumask_clear_cpu(rq->cpu, rq->rd->online); in set_rq_offline()
8187 rq->online = 0; in set_rq_offline()
8191 static inline void sched_set_rq_online(struct rq *rq, int cpu) in sched_set_rq_online() argument
8195 rq_lock_irqsave(rq, &rf); in sched_set_rq_online()
8196 if (rq->rd) { in sched_set_rq_online()
8197 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); in sched_set_rq_online()
8198 set_rq_online(rq); in sched_set_rq_online()
8200 rq_unlock_irqrestore(rq, &rf); in sched_set_rq_online()
8203 static inline void sched_set_rq_offline(struct rq *rq, int cpu) in sched_set_rq_offline() argument
8207 rq_lock_irqsave(rq, &rf); in sched_set_rq_offline()
8208 if (rq->rd) { in sched_set_rq_offline()
8209 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); in sched_set_rq_offline()
8210 set_rq_offline(rq); in sched_set_rq_offline()
8212 rq_unlock_irqrestore(rq, &rf); in sched_set_rq_offline()
8278 struct rq *rq = cpu_rq(cpu); in sched_cpu_activate() local
8298 scx_rq_activate(rq); in sched_cpu_activate()
8309 sched_set_rq_online(rq, cpu); in sched_cpu_activate()
8316 struct rq *rq = cpu_rq(cpu); in sched_cpu_deactivate() local
8328 nohz_balance_exit_idle(rq); in sched_cpu_deactivate()
8352 sched_set_rq_offline(rq, cpu); in sched_cpu_deactivate()
8354 scx_rq_deactivate(rq); in sched_cpu_deactivate()
8376 struct rq *rq = cpu_rq(cpu); in sched_rq_cpu_starting() local
8378 rq->calc_load_update = calc_load_update; in sched_rq_cpu_starting()
8419 static void calc_load_migrate(struct rq *rq) in calc_load_migrate() argument
8421 long delta = calc_load_fold_active(rq, 1); in calc_load_migrate()
8427 static void dump_rq_tasks(struct rq *rq, const char *loglvl) in dump_rq_tasks() argument
8430 int cpu = cpu_of(rq); in dump_rq_tasks()
8432 lockdep_assert_rq_held(rq); in dump_rq_tasks()
8434 printk("%sCPU%d enqueued tasks (%u total):\n", loglvl, cpu, rq->nr_running); in dump_rq_tasks()
8448 struct rq *rq = cpu_rq(cpu); in sched_cpu_dying() local
8454 rq_lock_irqsave(rq, &rf); in sched_cpu_dying()
8455 if (rq->nr_running != 1 || rq_has_pinned_tasks(rq)) { in sched_cpu_dying()
8457 dump_rq_tasks(rq, KERN_WARNING); in sched_cpu_dying()
8459 rq_unlock_irqrestore(rq, &rf); in sched_cpu_dying()
8461 calc_load_migrate(rq); in sched_cpu_dying()
8463 hrtick_clear(rq); in sched_cpu_dying()
8597 struct rq *rq; in sched_init() local
8599 rq = cpu_rq(i); in sched_init()
8600 raw_spin_lock_init(&rq->__lock); in sched_init()
8601 rq->nr_running = 0; in sched_init()
8602 rq->calc_load_active = 0; in sched_init()
8603 rq->calc_load_update = jiffies + LOAD_FREQ; in sched_init()
8604 init_cfs_rq(&rq->cfs); in sched_init()
8605 init_rt_rq(&rq->rt); in sched_init()
8606 init_dl_rq(&rq->dl); in sched_init()
8608 INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); in sched_init()
8609 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; in sched_init()
8629 init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL); in sched_init()
8638 rq->rt.rt_runtime = global_rt_runtime(); in sched_init()
8639 init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL); in sched_init()
8642 rq->sd = NULL; in sched_init()
8643 rq->rd = NULL; in sched_init()
8644 rq->cpu_capacity = SCHED_CAPACITY_SCALE; in sched_init()
8645 rq->balance_callback = &balance_push_callback; in sched_init()
8646 rq->active_balance = 0; in sched_init()
8647 rq->next_balance = jiffies; in sched_init()
8648 rq->push_cpu = 0; in sched_init()
8649 rq->cpu = i; in sched_init()
8650 rq->online = 0; in sched_init()
8651 rq->idle_stamp = 0; in sched_init()
8652 rq->avg_idle = 2*sysctl_sched_migration_cost; in sched_init()
8653 rq->max_idle_balance_cost = sysctl_sched_migration_cost; in sched_init()
8655 INIT_LIST_HEAD(&rq->cfs_tasks); in sched_init()
8657 rq_attach_root(rq, &def_root_domain); in sched_init()
8659 rq->last_blocked_load_update_tick = jiffies; in sched_init()
8660 atomic_set(&rq->nohz_flags, 0); in sched_init()
8662 INIT_CSD(&rq->nohz_csd, nohz_csd_func, rq); in sched_init()
8665 rcuwait_init(&rq->hotplug_wait); in sched_init()
8668 hrtick_rq_init(rq); in sched_init()
8669 atomic_set(&rq->nr_iowait, 0); in sched_init()
8670 fair_server_init(rq); in sched_init()
8673 rq->core = rq; in sched_init()
8674 rq->core_pick = NULL; in sched_init()
8675 rq->core_dl_server = NULL; in sched_init()
8676 rq->core_enabled = 0; in sched_init()
8677 rq->core_tree = RB_ROOT; in sched_init()
8678 rq->core_forceidle_count = 0; in sched_init()
8679 rq->core_forceidle_occupation = 0; in sched_init()
8680 rq->core_forceidle_start = 0; in sched_init()
8682 rq->core_cookie = 0UL; in sched_init()
8684 zalloc_cpumask_var_node(&rq->scratch_mask, GFP_KERNEL, cpu_to_node(i)); in sched_init()
9103 struct rq *rq; in sched_move_task() local
9106 rq = rq_guard.rq; in sched_move_task()
9108 update_rq_clock(rq); in sched_move_task()
9110 running = task_current_donor(rq, tsk); in sched_move_task()
9114 dequeue_task(rq, tsk, queue_flags); in sched_move_task()
9116 put_prev_task(rq, tsk); in sched_move_task()
9123 enqueue_task(rq, tsk, queue_flags); in sched_move_task()
9125 set_next_task(rq, tsk); in sched_move_task()
9131 resched_curr(rq); in sched_move_task()
9518 struct rq *rq = cfs_rq->rq; in tg_set_cfs_bandwidth() local
9520 guard(rq_lock_irq)(rq); in tg_set_cfs_bandwidth()
10189 void call_trace_sched_update_nr_running(struct rq *rq, int count) in call_trace_sched_update_nr_running() argument
10191 trace_sched_update_nr_running_tp(rq, count); in call_trace_sched_update_nr_running()
10310 int __sched_mm_cid_migrate_from_fetch_cid(struct rq *src_rq, in __sched_mm_cid_migrate_from_fetch_cid()
10349 int __sched_mm_cid_migrate_from_try_steal_cid(struct rq *src_rq, in __sched_mm_cid_migrate_from_try_steal_cid()
10413 void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t) in sched_mm_cid_migrate_to()
10419 struct rq *src_rq; in sched_mm_cid_migrate_to()
10471 struct rq *rq = cpu_rq(cpu); in sched_mm_cid_remote_clear() local
10508 t = rcu_dereference(rq->curr); in sched_mm_cid_remote_clear()
10526 struct rq *rq = cpu_rq(cpu); in sched_mm_cid_remote_clear_old() local
10535 rq_clock = READ_ONCE(rq->clock); in sched_mm_cid_remote_clear_old()
10544 curr = rcu_dereference(rq->curr); in sched_mm_cid_remote_clear_old()
10627 void task_tick_mm_cid(struct rq *rq, struct task_struct *curr) in task_tick_mm_cid() argument
10645 struct rq *rq; in sched_mm_cid_exit_signals() local
10651 rq = this_rq(); in sched_mm_cid_exit_signals()
10652 guard(rq_lock_irqsave)(rq); in sched_mm_cid_exit_signals()
10667 struct rq *rq; in sched_mm_cid_before_execve() local
10673 rq = this_rq(); in sched_mm_cid_before_execve()
10674 guard(rq_lock_irqsave)(rq); in sched_mm_cid_before_execve()
10689 struct rq *rq; in sched_mm_cid_after_execve() local
10695 rq = this_rq(); in sched_mm_cid_after_execve()
10696 scoped_guard (rq_lock_irqsave, rq) { in sched_mm_cid_after_execve()
10704 t->last_mm_cid = t->mm_cid = mm_cid_get(rq, t, mm); in sched_mm_cid_after_execve()
10719 struct rq *rq = task_rq(p); in sched_deq_and_put_task() local
10721 lockdep_assert_rq_held(rq); in sched_deq_and_put_task()
10727 .running = task_current(rq, p), in sched_deq_and_put_task()
10730 update_rq_clock(rq); in sched_deq_and_put_task()
10732 dequeue_task(rq, p, queue_flags | DEQUEUE_NOCLOCK); in sched_deq_and_put_task()
10734 put_prev_task(rq, p); in sched_deq_and_put_task()
10739 struct rq *rq = task_rq(ctx->p); in sched_enq_and_set_task() local
10741 lockdep_assert_rq_held(rq); in sched_enq_and_set_task()
10744 enqueue_task(rq, ctx->p, ctx->queue_flags | ENQUEUE_NOCLOCK); in sched_enq_and_set_task()
10746 set_next_task(rq, ctx->p); in sched_enq_and_set_task()