Lines Matching refs:rt_se
167 #define rt_entity_is_task(rt_se) (!(rt_se)->my_q) argument
169 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se) in rt_task_of() argument
171 WARN_ON_ONCE(!rt_entity_is_task(rt_se)); in rt_task_of()
173 return container_of(rt_se, struct task_struct, rt); in rt_task_of()
181 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) in rt_rq_of_se() argument
183 return rt_se->rt_rq; in rt_rq_of_se()
186 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) in rq_of_rt_se() argument
188 struct rt_rq *rt_rq = rt_se->rt_rq; in rq_of_rt_se()
195 if (tg->rt_se) in unregister_rt_sched_group()
206 if (tg->rt_se) in free_rt_sched_group()
207 kfree(tg->rt_se[i]); in free_rt_sched_group()
211 kfree(tg->rt_se); in free_rt_sched_group()
215 struct sched_rt_entity *rt_se, int cpu, in init_tg_rt_entry() argument
226 tg->rt_se[cpu] = rt_se; in init_tg_rt_entry()
228 if (!rt_se) in init_tg_rt_entry()
232 rt_se->rt_rq = &rq->rt; in init_tg_rt_entry()
234 rt_se->rt_rq = parent->my_q; in init_tg_rt_entry()
236 rt_se->my_q = rt_rq; in init_tg_rt_entry()
237 rt_se->parent = parent; in init_tg_rt_entry()
238 INIT_LIST_HEAD(&rt_se->run_list); in init_tg_rt_entry()
244 struct sched_rt_entity *rt_se; in alloc_rt_sched_group() local
250 tg->rt_se = kcalloc(nr_cpu_ids, sizeof(rt_se), GFP_KERNEL); in alloc_rt_sched_group()
251 if (!tg->rt_se) in alloc_rt_sched_group()
262 rt_se = kzalloc_node(sizeof(struct sched_rt_entity), in alloc_rt_sched_group()
264 if (!rt_se) in alloc_rt_sched_group()
269 init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]); in alloc_rt_sched_group()
282 #define rt_entity_is_task(rt_se) (1) argument
284 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se) in rt_task_of() argument
286 return container_of(rt_se, struct task_struct, rt); in rt_task_of()
294 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) in rq_of_rt_se() argument
296 struct task_struct *p = rt_task_of(rt_se); in rq_of_rt_se()
301 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) in rt_rq_of_se() argument
303 struct rq *rq = rq_of_rt_se(rt_se); in rt_rq_of_se()
437 static inline int on_rt_rq(struct sched_rt_entity *rt_se) in on_rt_rq() argument
439 return rt_se->on_rq; in on_rt_rq()
516 #define for_each_sched_rt_entity(rt_se) \ argument
517 for (; rt_se; rt_se = rt_se->parent)
519 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se) in group_rt_rq() argument
521 return rt_se->my_q; in group_rt_rq()
524 static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
525 static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
531 struct sched_rt_entity *rt_se; in sched_rt_rq_enqueue() local
535 rt_se = rt_rq->tg->rt_se[cpu]; in sched_rt_rq_enqueue()
538 if (!rt_se) in sched_rt_rq_enqueue()
540 else if (!on_rt_rq(rt_se)) in sched_rt_rq_enqueue()
541 enqueue_rt_entity(rt_se, 0); in sched_rt_rq_enqueue()
550 struct sched_rt_entity *rt_se; in sched_rt_rq_dequeue() local
553 rt_se = rt_rq->tg->rt_se[cpu]; in sched_rt_rq_dequeue()
555 if (!rt_se) { in sched_rt_rq_dequeue()
560 else if (on_rt_rq(rt_se)) in sched_rt_rq_dequeue()
561 dequeue_rt_entity(rt_se, 0); in sched_rt_rq_dequeue()
569 static int rt_se_boosted(struct sched_rt_entity *rt_se) in rt_se_boosted() argument
571 struct rt_rq *rt_rq = group_rt_rq(rt_se); in rt_se_boosted()
577 p = rt_task_of(rt_se); in rt_se_boosted()
924 #define for_each_sched_rt_entity(rt_se) \ argument
925 for (; rt_se; rt_se = NULL)
927 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se) in group_rt_rq() argument
971 static inline int rt_se_prio(struct sched_rt_entity *rt_se) in rt_se_prio() argument
974 struct rt_rq *rt_rq = group_rt_rq(rt_se); in rt_se_prio()
980 return rt_task_of(rt_se)->prio; in rt_se_prio()
1000 struct sched_rt_entity *rt_se = &donor->rt; in update_curr_rt() local
1005 for_each_sched_rt_entity(rt_se) { in update_curr_rt()
1006 struct rt_rq *rt_rq = rt_rq_of_se(rt_se); in update_curr_rt()
1154 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in inc_rt_group() argument
1156 if (rt_se_boosted(rt_se)) in inc_rt_group()
1164 dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in dec_rt_group() argument
1166 if (rt_se_boosted(rt_se)) in dec_rt_group()
1175 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in inc_rt_group() argument
1180 void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {} in dec_rt_group() argument
1185 unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se) in rt_se_nr_running() argument
1187 struct rt_rq *group_rq = group_rt_rq(rt_se); in rt_se_nr_running()
1196 unsigned int rt_se_rr_nr_running(struct sched_rt_entity *rt_se) in rt_se_rr_nr_running() argument
1198 struct rt_rq *group_rq = group_rt_rq(rt_se); in rt_se_rr_nr_running()
1204 tsk = rt_task_of(rt_se); in rt_se_rr_nr_running()
1210 void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in inc_rt_tasks() argument
1212 int prio = rt_se_prio(rt_se); in inc_rt_tasks()
1215 rt_rq->rt_nr_running += rt_se_nr_running(rt_se); in inc_rt_tasks()
1216 rt_rq->rr_nr_running += rt_se_rr_nr_running(rt_se); in inc_rt_tasks()
1219 inc_rt_group(rt_se, rt_rq); in inc_rt_tasks()
1223 void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) in dec_rt_tasks() argument
1225 WARN_ON(!rt_prio(rt_se_prio(rt_se))); in dec_rt_tasks()
1227 rt_rq->rt_nr_running -= rt_se_nr_running(rt_se); in dec_rt_tasks()
1228 rt_rq->rr_nr_running -= rt_se_rr_nr_running(rt_se); in dec_rt_tasks()
1230 dec_rt_prio(rt_rq, rt_se_prio(rt_se)); in dec_rt_tasks()
1231 dec_rt_group(rt_se, rt_rq); in dec_rt_tasks()
1247 static void __delist_rt_entity(struct sched_rt_entity *rt_se, struct rt_prio_array *array) in __delist_rt_entity() argument
1249 list_del_init(&rt_se->run_list); in __delist_rt_entity()
1251 if (list_empty(array->queue + rt_se_prio(rt_se))) in __delist_rt_entity()
1252 __clear_bit(rt_se_prio(rt_se), array->bitmap); in __delist_rt_entity()
1254 rt_se->on_list = 0; in __delist_rt_entity()
1258 __schedstats_from_rt_se(struct sched_rt_entity *rt_se) in __schedstats_from_rt_se() argument
1262 if (!rt_entity_is_task(rt_se)) in __schedstats_from_rt_se()
1266 return &rt_task_of(rt_se)->stats; in __schedstats_from_rt_se()
1270 update_stats_wait_start_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se) in update_stats_wait_start_rt() argument
1278 if (rt_entity_is_task(rt_se)) in update_stats_wait_start_rt()
1279 p = rt_task_of(rt_se); in update_stats_wait_start_rt()
1281 stats = __schedstats_from_rt_se(rt_se); in update_stats_wait_start_rt()
1289 update_stats_enqueue_sleeper_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se) in update_stats_enqueue_sleeper_rt() argument
1297 if (rt_entity_is_task(rt_se)) in update_stats_enqueue_sleeper_rt()
1298 p = rt_task_of(rt_se); in update_stats_enqueue_sleeper_rt()
1300 stats = __schedstats_from_rt_se(rt_se); in update_stats_enqueue_sleeper_rt()
1308 update_stats_enqueue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, in update_stats_enqueue_rt() argument
1315 update_stats_enqueue_sleeper_rt(rt_rq, rt_se); in update_stats_enqueue_rt()
1319 update_stats_wait_end_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se) in update_stats_wait_end_rt() argument
1327 if (rt_entity_is_task(rt_se)) in update_stats_wait_end_rt()
1328 p = rt_task_of(rt_se); in update_stats_wait_end_rt()
1330 stats = __schedstats_from_rt_se(rt_se); in update_stats_wait_end_rt()
1338 update_stats_dequeue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, in update_stats_dequeue_rt() argument
1346 if (rt_entity_is_task(rt_se)) in update_stats_dequeue_rt()
1347 p = rt_task_of(rt_se); in update_stats_dequeue_rt()
1363 static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) in __enqueue_rt_entity() argument
1365 struct rt_rq *rt_rq = rt_rq_of_se(rt_se); in __enqueue_rt_entity()
1367 struct rt_rq *group_rq = group_rt_rq(rt_se); in __enqueue_rt_entity()
1368 struct list_head *queue = array->queue + rt_se_prio(rt_se); in __enqueue_rt_entity()
1377 if (rt_se->on_list) in __enqueue_rt_entity()
1378 __delist_rt_entity(rt_se, array); in __enqueue_rt_entity()
1383 WARN_ON_ONCE(rt_se->on_list); in __enqueue_rt_entity()
1385 list_add(&rt_se->run_list, queue); in __enqueue_rt_entity()
1387 list_add_tail(&rt_se->run_list, queue); in __enqueue_rt_entity()
1389 __set_bit(rt_se_prio(rt_se), array->bitmap); in __enqueue_rt_entity()
1390 rt_se->on_list = 1; in __enqueue_rt_entity()
1392 rt_se->on_rq = 1; in __enqueue_rt_entity()
1394 inc_rt_tasks(rt_se, rt_rq); in __enqueue_rt_entity()
1397 static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) in __dequeue_rt_entity() argument
1399 struct rt_rq *rt_rq = rt_rq_of_se(rt_se); in __dequeue_rt_entity()
1403 WARN_ON_ONCE(!rt_se->on_list); in __dequeue_rt_entity()
1404 __delist_rt_entity(rt_se, array); in __dequeue_rt_entity()
1406 rt_se->on_rq = 0; in __dequeue_rt_entity()
1408 dec_rt_tasks(rt_se, rt_rq); in __dequeue_rt_entity()
1415 static void dequeue_rt_stack(struct sched_rt_entity *rt_se, unsigned int flags) in dequeue_rt_stack() argument
1420 for_each_sched_rt_entity(rt_se) { in dequeue_rt_stack()
1421 rt_se->back = back; in dequeue_rt_stack()
1422 back = rt_se; in dequeue_rt_stack()
1427 for (rt_se = back; rt_se; rt_se = rt_se->back) { in dequeue_rt_stack()
1428 if (on_rt_rq(rt_se)) in dequeue_rt_stack()
1429 __dequeue_rt_entity(rt_se, flags); in dequeue_rt_stack()
1435 static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) in enqueue_rt_entity() argument
1437 struct rq *rq = rq_of_rt_se(rt_se); in enqueue_rt_entity()
1439 update_stats_enqueue_rt(rt_rq_of_se(rt_se), rt_se, flags); in enqueue_rt_entity()
1441 dequeue_rt_stack(rt_se, flags); in enqueue_rt_entity()
1442 for_each_sched_rt_entity(rt_se) in enqueue_rt_entity()
1443 __enqueue_rt_entity(rt_se, flags); in enqueue_rt_entity()
1447 static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) in dequeue_rt_entity() argument
1449 struct rq *rq = rq_of_rt_se(rt_se); in dequeue_rt_entity()
1451 update_stats_dequeue_rt(rt_rq_of_se(rt_se), rt_se, flags); in dequeue_rt_entity()
1453 dequeue_rt_stack(rt_se, flags); in dequeue_rt_entity()
1455 for_each_sched_rt_entity(rt_se) { in dequeue_rt_entity()
1456 struct rt_rq *rt_rq = group_rt_rq(rt_se); in dequeue_rt_entity()
1459 __enqueue_rt_entity(rt_se, flags); in dequeue_rt_entity()
1470 struct sched_rt_entity *rt_se = &p->rt; in enqueue_task_rt() local
1473 rt_se->timeout = 0; in enqueue_task_rt()
1476 update_stats_wait_start_rt(rt_rq_of_se(rt_se), rt_se); in enqueue_task_rt()
1478 enqueue_rt_entity(rt_se, flags); in enqueue_task_rt()
1486 struct sched_rt_entity *rt_se = &p->rt; in dequeue_task_rt() local
1489 dequeue_rt_entity(rt_se, flags); in dequeue_task_rt()
1501 requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head) in requeue_rt_entity() argument
1503 if (on_rt_rq(rt_se)) { in requeue_rt_entity()
1505 struct list_head *queue = array->queue + rt_se_prio(rt_se); in requeue_rt_entity()
1508 list_move(&rt_se->run_list, queue); in requeue_rt_entity()
1510 list_move_tail(&rt_se->run_list, queue); in requeue_rt_entity()
1516 struct sched_rt_entity *rt_se = &p->rt; in requeue_task_rt() local
1519 for_each_sched_rt_entity(rt_se) { in requeue_task_rt()
1520 rt_rq = rt_rq_of_se(rt_se); in requeue_task_rt()
1521 requeue_rt_entity(rt_rq, rt_se, head); in requeue_task_rt()
1679 struct sched_rt_entity *rt_se = &p->rt; in set_next_task_rt() local
1684 update_stats_wait_end_rt(rt_rq, rt_se); in set_next_task_rt()
1723 struct sched_rt_entity *rt_se; in _pick_next_task_rt() local
1727 rt_se = pick_next_rt_entity(rt_rq); in _pick_next_task_rt()
1728 if (unlikely(!rt_se)) in _pick_next_task_rt()
1730 rt_rq = group_rt_rq(rt_se); in _pick_next_task_rt()
1733 return rt_task_of(rt_se); in _pick_next_task_rt()
1750 struct sched_rt_entity *rt_se = &p->rt; in put_prev_task_rt() local
1754 update_stats_wait_start_rt(rt_rq, rt_se); in put_prev_task_rt()
2557 struct sched_rt_entity *rt_se = &p->rt; in task_tick_rt() local
2580 for_each_sched_rt_entity(rt_se) { in task_tick_rt()
2581 if (rt_se->run_list.prev != rt_se->run_list.next) { in task_tick_rt()