Lines Matching refs:group
437 static bool tmigr_check_migrator(struct tmigr_group *group, u8 childmask) in tmigr_check_migrator() argument
441 s.state = atomic_read(&group->migr_state); in tmigr_check_migrator()
449 static bool tmigr_check_migrator_and_lonely(struct tmigr_group *group, u8 childmask) in tmigr_check_migrator_and_lonely() argument
455 s.state = atomic_read(&group->migr_state); in tmigr_check_migrator_and_lonely()
466 static bool tmigr_check_lonely(struct tmigr_group *group) in tmigr_check_lonely() argument
471 s.state = atomic_read(&group->migr_state); in tmigr_check_lonely()
528 struct tmigr_group *child = NULL, *group = tmc->tmgroup; in __walk_groups() local
531 WARN_ON_ONCE(group->level >= tmigr_hierarchy_levels); in __walk_groups()
533 if (up(group, child, data)) in __walk_groups()
536 child = group; in __walk_groups()
541 group = READ_ONCE(group->parent); in __walk_groups()
544 } while (group); in __walk_groups()
560 static struct tmigr_event *tmigr_next_groupevt(struct tmigr_group *group) in tmigr_next_groupevt() argument
565 lockdep_assert_held(&group->lock); in tmigr_next_groupevt()
567 WRITE_ONCE(group->next_expiry, KTIME_MAX); in tmigr_next_groupevt()
569 while ((node = timerqueue_getnext(&group->events))) { in tmigr_next_groupevt()
573 WRITE_ONCE(group->next_expiry, evt->nextevt.expires); in tmigr_next_groupevt()
581 if (!timerqueue_del(&group->events, node)) in tmigr_next_groupevt()
593 static struct tmigr_event *tmigr_next_expired_groupevt(struct tmigr_group *group, in tmigr_next_expired_groupevt() argument
596 struct tmigr_event *evt = tmigr_next_groupevt(group); in tmigr_next_expired_groupevt()
604 timerqueue_del(&group->events, &evt->nextevt); in tmigr_next_expired_groupevt()
605 tmigr_next_groupevt(group); in tmigr_next_expired_groupevt()
610 static u64 tmigr_next_groupevt_expires(struct tmigr_group *group) in tmigr_next_groupevt_expires() argument
614 evt = tmigr_next_groupevt(group); in tmigr_next_groupevt_expires()
622 static bool tmigr_active_up(struct tmigr_group *group, in tmigr_active_up() argument
636 curstate.state = atomic_read(&group->migr_state); in tmigr_active_up()
652 } while (!atomic_try_cmpxchg(&group->migr_state, &curstate.state, newstate.state)); in tmigr_active_up()
654 trace_tmigr_group_set_cpu_active(group, newstate, childmask); in tmigr_active_up()
668 WRITE_ONCE(group->groupevt.ignore, true); in tmigr_active_up()
722 bool tmigr_update_events(struct tmigr_group *group, struct tmigr_group *child, in tmigr_update_events() argument
734 raw_spin_lock_nested(&group->lock, SINGLE_DEPTH_NESTING); in tmigr_update_events()
737 groupstate.state = atomic_read(&group->migr_state); in tmigr_update_events()
786 if (ignore && !remote && group->parent) in tmigr_update_events()
789 raw_spin_lock(&group->lock); in tmigr_update_events()
792 groupstate.state = atomic_read(&group->migr_state); in tmigr_update_events()
806 if (!timerqueue_del(&group->events, &evt->nextevt)) in tmigr_update_events()
807 WRITE_ONCE(group->next_expiry, KTIME_MAX); in tmigr_update_events()
830 if (timerqueue_add(&group->events, &evt->nextevt)) in tmigr_update_events()
831 WRITE_ONCE(group->next_expiry, nextexp); in tmigr_update_events()
835 if (!group->parent && (groupstate.migrator == TMIGR_NONE)) { in tmigr_update_events()
854 data->firstexp = tmigr_next_groupevt_expires(group); in tmigr_update_events()
857 trace_tmigr_update_events(child, group, childstate, groupstate, in tmigr_update_events()
861 raw_spin_unlock(&group->lock); in tmigr_update_events()
869 static bool tmigr_new_timer_up(struct tmigr_group *group, in tmigr_new_timer_up() argument
873 return tmigr_update_events(group, child, data); in tmigr_new_timer_up()
1002 static bool tmigr_handle_remote_up(struct tmigr_group *group, in tmigr_handle_remote_up() argument
1016 trace_tmigr_handle_remote(group); in tmigr_handle_remote_up()
1023 if (!tmigr_check_migrator(group, childmask)) in tmigr_handle_remote_up()
1026 raw_spin_lock_irq(&group->lock); in tmigr_handle_remote_up()
1028 evt = tmigr_next_expired_groupevt(group, now); in tmigr_handle_remote_up()
1033 raw_spin_unlock_irq(&group->lock); in tmigr_handle_remote_up()
1046 data->firstexp = group->next_expiry; in tmigr_handle_remote_up()
1048 raw_spin_unlock_irq(&group->lock); in tmigr_handle_remote_up()
1101 static bool tmigr_requires_handle_remote_up(struct tmigr_group *group, in tmigr_requires_handle_remote_up() argument
1114 if (!tmigr_check_migrator(group, childmask)) in tmigr_requires_handle_remote_up()
1122 if (group->parent && !data->tmc_active) in tmigr_requires_handle_remote_up()
1132 data->firstexp = READ_ONCE(group->next_expiry); in tmigr_requires_handle_remote_up()
1138 raw_spin_lock(&group->lock); in tmigr_requires_handle_remote_up()
1139 data->firstexp = group->next_expiry; in tmigr_requires_handle_remote_up()
1140 if (data->now >= group->next_expiry) { in tmigr_requires_handle_remote_up()
1142 raw_spin_unlock(&group->lock); in tmigr_requires_handle_remote_up()
1145 raw_spin_unlock(&group->lock); in tmigr_requires_handle_remote_up()
1244 static bool tmigr_inactive_up(struct tmigr_group *group, in tmigr_inactive_up() argument
1261 curstate.state = atomic_read_acquire(&group->migr_state); in tmigr_inactive_up()
1299 if (atomic_try_cmpxchg(&group->migr_state, &curstate.state, newstate.state)) { in tmigr_inactive_up()
1300 trace_tmigr_group_set_cpu_inactive(group, newstate, childmask); in tmigr_inactive_up()
1316 tmigr_update_events(group, child, data); in tmigr_inactive_up()
1396 struct tmigr_group *group = tmc->tmgroup; in tmigr_quick_check() local
1408 if (!tmigr_check_lonely(group)) { in tmigr_quick_check()
1417 nextevt = min_t(u64, nextevt, READ_ONCE(group->next_expiry)); in tmigr_quick_check()
1418 if (!group->parent) in tmigr_quick_check()
1421 group = group->parent; in tmigr_quick_check()
1422 } while (group); in tmigr_quick_check()
1487 static void tmigr_init_group(struct tmigr_group *group, unsigned int lvl, in tmigr_init_group() argument
1492 raw_spin_lock_init(&group->lock); in tmigr_init_group()
1494 group->level = lvl; in tmigr_init_group()
1495 group->numa_node = lvl < tmigr_crossnode_level ? node : NUMA_NO_NODE; in tmigr_init_group()
1497 group->num_children = 0; in tmigr_init_group()
1502 atomic_set(&group->migr_state, s.state); in tmigr_init_group()
1510 group->groupmask = BIT(0); in tmigr_init_group()
1516 group->num_children = 1; in tmigr_init_group()
1519 timerqueue_init_head(&group->events); in tmigr_init_group()
1520 timerqueue_init(&group->groupevt.nextevt); in tmigr_init_group()
1521 group->groupevt.nextevt.expires = KTIME_MAX; in tmigr_init_group()
1522 WRITE_ONCE(group->next_expiry, KTIME_MAX); in tmigr_init_group()
1523 group->groupevt.ignore = true; in tmigr_init_group()
1529 struct tmigr_group *tmp, *group = NULL; in tmigr_get_group() local
1553 group = tmp; in tmigr_get_group()
1557 if (group) in tmigr_get_group()
1558 return group; in tmigr_get_group()
1561 group = kzalloc_node(sizeof(*group), GFP_KERNEL, node); in tmigr_get_group()
1562 if (!group) in tmigr_get_group()
1565 tmigr_init_group(group, lvl, node); in tmigr_get_group()
1568 list_add(&group->list, &tmigr_level_list[lvl]); in tmigr_get_group()
1569 trace_tmigr_group_set(group); in tmigr_get_group()
1570 return group; in tmigr_get_group()
1647 struct tmigr_group *group, *child, **stack; in tmigr_setup_groups() local
1656 group = tmigr_get_group(cpu, node, i); in tmigr_setup_groups()
1657 if (IS_ERR(group)) { in tmigr_setup_groups()
1658 err = PTR_ERR(group); in tmigr_setup_groups()
1663 stack[i++] = group; in tmigr_setup_groups()
1673 if (group->parent || list_is_singular(&tmigr_level_list[i - 1])) in tmigr_setup_groups()
1679 WARN_ON_ONCE(!err && !group->parent && !list_is_singular(&tmigr_level_list[top])); in tmigr_setup_groups()
1682 group = stack[--i]; in tmigr_setup_groups()
1685 list_del(&group->list); in tmigr_setup_groups()
1686 kfree(group); in tmigr_setup_groups()
1690 WARN_ON_ONCE(i != group->level); in tmigr_setup_groups()
1698 raw_spin_lock_irq(&group->lock); in tmigr_setup_groups()
1700 tmc->tmgroup = group; in tmigr_setup_groups()
1701 tmc->groupmask = BIT(group->num_children++); in tmigr_setup_groups()
1703 raw_spin_unlock_irq(&group->lock); in tmigr_setup_groups()
1712 tmigr_connect_child_parent(child, group, false); in tmigr_setup_groups()
1727 if (group->num_children == 2 && list_is_singular(lvllist)) { in tmigr_setup_groups()
1742 tmigr_connect_child_parent(child, group, true); in tmigr_setup_groups()