Lines Matching refs:th
220 int gtid = other_threads[thr]->th.th_info.ds.ds_gtid; in __kmp_dist_barrier_wakeup()
237 team = this_thr->th.th_team; in __kmp_dist_barrier_gather()
238 nproc = this_thr->th.th_team_nproc; in __kmp_dist_barrier_gather()
252 this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time = in __kmp_dist_barrier_gather()
274 kmp_task_team_t *task_team = this_thr->th.th_task_team; in __kmp_dist_barrier_gather()
283 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP; in __kmp_dist_barrier_gather()
286 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP; in __kmp_dist_barrier_gather()
294 this_thr->th.th_reap_state == KMP_SAFE_TO_REAP) { in __kmp_dist_barrier_gather()
295 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP; in __kmp_dist_barrier_gather()
304 (*reduce)(this_thr->th.th_local.reduce_data, in __kmp_dist_barrier_gather()
305 other_threads[thr]->th.th_local.reduce_data); in __kmp_dist_barrier_gather()
323 kmp_task_team_t *task_team = this_thr->th.th_task_team; in __kmp_dist_barrier_gather()
332 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP; in __kmp_dist_barrier_gather()
335 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP; in __kmp_dist_barrier_gather()
343 this_thr->th.th_reap_state == KMP_SAFE_TO_REAP) { in __kmp_dist_barrier_gather()
344 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP; in __kmp_dist_barrier_gather()
354 (*reduce)(this_thr->th.th_local.reduce_data, in __kmp_dist_barrier_gather()
355 other_threads[thr]->th.th_local.reduce_data); in __kmp_dist_barrier_gather()
389 thr_bar = &this_thr->th.th_bar[bt].bb; in __kmp_dist_barrier_release()
394 if (this_thr->th.th_used_in_team.load() != 1 && in __kmp_dist_barrier_release()
395 this_thr->th.th_used_in_team.load() != 3) { in __kmp_dist_barrier_release()
400 kmp_flag_32<false, false> my_flag(&(this_thr->th.th_used_in_team), 3); in __kmp_dist_barrier_release()
401 if (KMP_COMPARE_AND_STORE_ACQ32(&(this_thr->th.th_used_in_team), 2, in __kmp_dist_barrier_release()
403 this_thr->th.th_used_in_team.load() == 0) { in __kmp_dist_barrier_release()
426 if (this_thr->th.th_used_in_team.load() != 1 && in __kmp_dist_barrier_release()
427 this_thr->th.th_used_in_team.load() != 3) // spurious wake-up? in __kmp_dist_barrier_release()
439 team = this_thr->th.th_team; in __kmp_dist_barrier_release()
446 if (this_thr->th.th_used_in_team.load() == 3) { in __kmp_dist_barrier_release()
447 KMP_COMPARE_AND_STORE_ACQ32(&(this_thr->th.th_used_in_team), 3, 1); in __kmp_dist_barrier_release()
467 if (this_thr->th.th_used_in_team.load() == 1) in __kmp_dist_barrier_release()
498 size_t nproc = this_thr->th.th_team_nproc; in __kmp_dist_barrier_release()
505 team = this_thr->th.th_team; in __kmp_dist_barrier_release()
523 size_t nproc = this_thr->th.th_team_nproc; in __kmp_dist_barrier_release()
538 size_t nproc = this_thr->th.th_team_nproc; in __kmp_dist_barrier_release()
560 kmp_team_t *team = this_thr->th.th_team; in __kmp_linear_barrier_gather_template()
561 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; in __kmp_linear_barrier_gather_template()
568 KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]); in __kmp_linear_barrier_gather_template()
573 this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time = in __kmp_linear_barrier_gather_template()
594 int nproc = this_thr->th.th_team_nproc; in __kmp_linear_barrier_gather_template()
604 KMP_CACHE_PREFETCH(&other_threads[i + 1]->th.th_bar[bt].bb.b_arrived); in __kmp_linear_barrier_gather_template()
610 &other_threads[i]->th.th_bar[bt].bb.b_arrived, new_state)); in __kmp_linear_barrier_gather_template()
615 &other_threads[i]->th.th_bar[bt].bb.b_arrived, new_state); in __kmp_linear_barrier_gather_template()
619 kmp_flag_64<> flag(&other_threads[i]->th.th_bar[bt].bb.b_arrived, in __kmp_linear_barrier_gather_template()
627 this_thr->th.th_bar_min_time = KMP_MIN( in __kmp_linear_barrier_gather_template()
628 this_thr->th.th_bar_min_time, other_threads[i]->th.th_bar_min_time); in __kmp_linear_barrier_gather_template()
638 (*reduce)(this_thr->th.th_local.reduce_data, in __kmp_linear_barrier_gather_template()
639 other_threads[i]->th.th_local.reduce_data); in __kmp_linear_barrier_gather_template()
662 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; in __kmp_linear_barrier_release_template()
667 kmp_uint32 nproc = this_thr->th.th_team_nproc; in __kmp_linear_barrier_release_template()
670 team = __kmp_threads[gtid]->th.th_team; in __kmp_linear_barrier_release_template()
700 KMP_CACHE_PREFETCH(&other_threads[i + 1]->th.th_bar[bt].bb.b_go); in __kmp_linear_barrier_release_template()
706 gtid, team->t.t_id, tid, other_threads[i]->th.th_info.ds.ds_gtid, in __kmp_linear_barrier_release_template()
707 team->t.t_id, i, &other_threads[i]->th.th_bar[bt].bb.b_go, in __kmp_linear_barrier_release_template()
708 other_threads[i]->th.th_bar[bt].bb.b_go, in __kmp_linear_barrier_release_template()
709 other_threads[i]->th.th_bar[bt].bb.b_go + KMP_BARRIER_STATE_BUMP)); in __kmp_linear_barrier_release_template()
710 kmp_flag_64<> flag(&other_threads[i]->th.th_bar[bt].bb.b_go, in __kmp_linear_barrier_release_template()
749 team = __kmp_threads[gtid]->th.th_team; in __kmp_linear_barrier_release_template()
798 kmp_team_t *team = this_thr->th.th_team; in __kmp_tree_barrier_gather()
799 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; in __kmp_tree_barrier_gather()
801 kmp_uint32 nproc = this_thr->th.th_team_nproc; in __kmp_tree_barrier_gather()
811 KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]); in __kmp_tree_barrier_gather()
816 this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time = in __kmp_tree_barrier_gather()
829 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb; in __kmp_tree_barrier_gather()
834 &other_threads[child_tid + 1]->th.th_bar[bt].bb.b_arrived); in __kmp_tree_barrier_gather()
848 this_thr->th.th_bar_min_time = KMP_MIN(this_thr->th.th_bar_min_time, in __kmp_tree_barrier_gather()
849 child_thr->th.th_bar_min_time); in __kmp_tree_barrier_gather()
859 (*reduce)(this_thr->th.th_local.reduce_data, in __kmp_tree_barrier_gather()
860 child_thr->th.th_local.reduce_data); in __kmp_tree_barrier_gather()
905 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; in __kmp_tree_barrier_release()
942 team = __kmp_threads[gtid]->th.th_team; in __kmp_tree_barrier_release()
952 team = __kmp_threads[gtid]->th.th_team; in __kmp_tree_barrier_release()
958 nproc = this_thr->th.th_team_nproc; in __kmp_tree_barrier_release()
967 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb; in __kmp_tree_barrier_release()
972 &other_threads[child_tid + 1]->th.th_bar[bt].bb.b_go); in __kmp_tree_barrier_release()
1010 kmp_team_t *team = this_thr->th.th_team; in __kmp_hyper_barrier_gather()
1011 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; in __kmp_hyper_barrier_gather()
1014 kmp_uint32 num_threads = this_thr->th.th_team_nproc; in __kmp_hyper_barrier_gather()
1024 KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]); in __kmp_hyper_barrier_gather()
1029 this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time = in __kmp_hyper_barrier_gather()
1068 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb; in __kmp_hyper_barrier_gather()
1074 &other_threads[next_child_tid]->th.th_bar[bt].bb.b_arrived); in __kmp_hyper_barrier_gather()
1089 this_thr->th.th_bar_min_time = KMP_MIN(this_thr->th.th_bar_min_time, in __kmp_hyper_barrier_gather()
1090 child_thr->th.th_bar_min_time); in __kmp_hyper_barrier_gather()
1100 (*reduce)(this_thr->th.th_local.reduce_data, in __kmp_hyper_barrier_gather()
1101 child_thr->th.th_local.reduce_data); in __kmp_hyper_barrier_gather()
1130 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; in __kmp_hyper_barrier_release()
1145 team = __kmp_threads[gtid]->th.th_team; in __kmp_hyper_barrier_release()
1183 team = __kmp_threads[gtid]->th.th_team; in __kmp_hyper_barrier_release()
1193 num_threads = this_thr->th.th_team_nproc; in __kmp_hyper_barrier_release()
1234 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb; in __kmp_hyper_barrier_release()
1244 &other_threads[next_child_tid]->th.th_bar[bt].bb.b_go); in __kmp_hyper_barrier_release()
1335 &team->t.t_threads[thr_bar->parent_tid]->th.th_bar[bt].bb; in __kmp_init_hierarchical_barrier_thread()
1340 &team->t.t_threads[thr_bar->parent_tid]->th.th_bar[bt].bb; in __kmp_init_hierarchical_barrier_thread()
1361 kmp_team_t *team = this_thr->th.th_team; in __kmp_hierarchical_barrier_gather()
1362 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; in __kmp_hierarchical_barrier_gather()
1363 kmp_uint32 nproc = this_thr->th.th_team_nproc; in __kmp_hierarchical_barrier_gather()
1369 ->th.th_teams_microtask) // are we inside the teams construct? in __kmp_hierarchical_barrier_gather()
1370 if (this_thr->th.th_teams_size.nteams > 1) in __kmp_hierarchical_barrier_gather()
1380 KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]); in __kmp_hierarchical_barrier_gather()
1385 this_thr->th.th_bar_arrive_time = __itt_get_timestamp(); in __kmp_hierarchical_barrier_gather()
1419 (*reduce)(this_thr->th.th_local.reduce_data, in __kmp_hierarchical_barrier_gather()
1420 other_threads[child_tid]->th.th_local.reduce_data); in __kmp_hierarchical_barrier_gather()
1436 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb; in __kmp_hierarchical_barrier_gather()
1451 (*reduce)(this_thr->th.th_local.reduce_data, in __kmp_hierarchical_barrier_gather()
1452 child_thr->th.th_local.reduce_data); in __kmp_hierarchical_barrier_gather()
1465 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb; in __kmp_hierarchical_barrier_gather()
1480 (*reduce)(this_thr->th.th_local.reduce_data, in __kmp_hierarchical_barrier_gather()
1481 child_thr->th.th_local.reduce_data); in __kmp_hierarchical_barrier_gather()
1531 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; in __kmp_hierarchical_barrier_release()
1536 team = __kmp_threads[gtid]->th.th_team; in __kmp_hierarchical_barrier_release()
1574 team = __kmp_threads[gtid]->th.th_team; in __kmp_hierarchical_barrier_release()
1585 nproc = this_thr->th.th_team_nproc; in __kmp_hierarchical_barrier_release()
1588 ->th.th_teams_microtask) { // are we inside the teams construct? in __kmp_hierarchical_barrier_release()
1590 this_thr->th.th_teams_level == level) in __kmp_hierarchical_barrier_release()
1592 if (this_thr->th.th_teams_size.nteams > 1) in __kmp_hierarchical_barrier_release()
1655 &team->t.t_threads[child_tid]->th.th_bar[bt].bb; in __kmp_hierarchical_barrier_release()
1686 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb; in __kmp_hierarchical_barrier_release()
1712 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb; in __kmp_hierarchical_barrier_release()
1774 kmp_team_t *team = this_thr->th.th_team; in __kmp_barrier_template()
1808 this_thr->th.ompt_thread_info.state = ompt_state_wait_barrier; in __kmp_barrier_template()
1834 this_thr->th.th_team_bt_intervals = in __kmp_barrier_template()
1836 this_thr->th.th_team_bt_set = in __kmp_barrier_template()
1839 this_thr->th.th_team_bt_intervals = KMP_BLOCKTIME_INTERVAL(team, tid); in __kmp_barrier_template()
1852 this_thr->th.th_bar[bt].bb.b_worker_arrived += 1; in __kmp_barrier_template()
1857 this_thr->th.th_local.reduce_data = reduce_data; in __kmp_barrier_template()
1933 (this_thr->th.th_teams_microtask == NULL || // either not in teams in __kmp_barrier_template()
1934 this_thr->th.th_teams_size.nteams == 1) && // or inside single team in __kmp_barrier_template()
1936 ident_t *loc = __kmp_threads[gtid]->th.th_ident; in __kmp_barrier_template()
1939 int nproc = this_thr->th.th_team_nproc; in __kmp_barrier_template()
1943 __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0, in __kmp_barrier_template()
1945 this_thr->th.th_frame_time = cur_time; in __kmp_barrier_template()
1949 __kmp_itt_frame_submit(gtid, this_thr->th.th_bar_min_time, cur_time, in __kmp_barrier_template()
1955 kmp_uint64 delta = cur_time - this_thr->th.th_bar_arrive_time; in __kmp_barrier_template()
1958 this_thr->th.th_bar_arrive_time = 0; in __kmp_barrier_template()
1960 delta += (cur_time - other_threads[i]->th.th_bar_arrive_time); in __kmp_barrier_template()
1961 other_threads[i]->th.th_bar_arrive_time = 0; in __kmp_barrier_template()
1963 __kmp_itt_metadata_imbalance(gtid, this_thr->th.th_frame_time, in __kmp_barrier_template()
1967 __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0, in __kmp_barrier_template()
1969 this_thr->th.th_frame_time = cur_time; in __kmp_barrier_template()
2031 if (this_thr->th.th_task_team != NULL) { in __kmp_barrier_template()
2041 this_thr->th.th_task_team->tt.tt_found_proxy_tasks == TRUE || in __kmp_barrier_template()
2042 this_thr->th.th_task_team->tt.tt_hidden_helper_task_encountered == in __kmp_barrier_template()
2072 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_barrier_template()
2102 this_thr->th.th_bar[bs_plain_barrier].bb.b_arrived -= in __kmp_barrier_gomp_cancel()
2119 kmp_team_t *team = this_thr->th.th_team; in __kmp_end_split_barrier()
2179 int nproc = this_thr->th.th_team_nproc; in __kmp_join_barrier()
2184 team = this_thr->th.th_team; in __kmp_join_barrier()
2189 kmp_info_t *master_thread = this_thr->th.th_team_master; in __kmp_join_barrier()
2198 KMP_DEBUG_ASSERT(TCR_PTR(this_thr->th.th_team)); in __kmp_join_barrier()
2199 KMP_DEBUG_ASSERT(TCR_PTR(this_thr->th.th_root)); in __kmp_join_barrier()
2210 int ds_tid = this_thr->th.th_info.ds.ds_tid; in __kmp_join_barrier()
2228 this_thr->th.ompt_thread_info.task_data = *OMPT_CUR_TASK_DATA(this_thr); in __kmp_join_barrier()
2230 this_thr->th.ompt_thread_info.state = ompt_state_wait_barrier_implicit; in __kmp_join_barrier()
2244 team->t.t_task_team[this_thr->th.th_task_state], in __kmp_join_barrier()
2245 this_thr->th.th_task_team)); in __kmp_join_barrier()
2246 if (this_thr->th.th_task_team) in __kmp_join_barrier()
2247 KMP_DEBUG_ASSERT(this_thr->th.th_task_team == in __kmp_join_barrier()
2248 team->t.t_task_team[this_thr->th.th_task_state]); in __kmp_join_barrier()
2259 this_thr->th.th_team_bt_intervals = in __kmp_join_barrier()
2261 this_thr->th.th_team_bt_set = in __kmp_join_barrier()
2264 this_thr->th.th_team_bt_intervals = KMP_BLOCKTIME_INTERVAL(team, tid); in __kmp_join_barrier()
2321 team_thread->th.th_stats->setIdleFlag(); in __kmp_join_barrier()
2323 team_thread->th.th_sleep_loc != NULL) in __kmp_join_barrier()
2336 (this_thr->th.th_teams_microtask == NULL || // either not in teams in __kmp_join_barrier()
2337 this_thr->th.th_teams_size.nteams == 1) && // or inside single team in __kmp_join_barrier()
2344 __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0, in __kmp_join_barrier()
2348 __kmp_itt_frame_submit(gtid, this_thr->th.th_bar_min_time, cur_time, 1, in __kmp_join_barrier()
2354 kmp_uint64 delta = cur_time - this_thr->th.th_bar_arrive_time; in __kmp_join_barrier()
2357 this_thr->th.th_bar_arrive_time = 0; in __kmp_join_barrier()
2359 delta += (cur_time - other_threads[i]->th.th_bar_arrive_time); in __kmp_join_barrier()
2360 other_threads[i]->th.th_bar_arrive_time = 0; in __kmp_join_barrier()
2362 __kmp_itt_metadata_imbalance(gtid, this_thr->th.th_frame_time, in __kmp_join_barrier()
2365 __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0, in __kmp_join_barrier()
2367 this_thr->th.th_frame_time = cur_time; in __kmp_join_barrier()
2402 kmp_team_t *team = (tid == 0) ? this_thr->th.th_team : NULL; in __kmp_fork_barrier()
2433 gtid, team->t.t_id, other_threads[i]->th.th_info.ds.ds_gtid, in __kmp_fork_barrier()
2434 team->t.t_id, other_threads[i]->th.th_info.ds.ds_tid, in __kmp_fork_barrier()
2435 other_threads[i]->th.th_bar[bs_forkjoin_barrier].bb.b_go)); in __kmp_fork_barrier()
2437 (TCR_4(other_threads[i]->th.th_bar[bs_forkjoin_barrier].bb.b_go) & in __kmp_fork_barrier()
2439 KMP_DEBUG_ASSERT(other_threads[i]->th.th_team == team); in __kmp_fork_barrier()
2456 this_thr->th.th_team_bt_intervals = in __kmp_fork_barrier()
2458 this_thr->th.th_team_bt_set = in __kmp_fork_barrier()
2461 this_thr->th.th_team_bt_intervals = KMP_BLOCKTIME_INTERVAL(team, tid); in __kmp_fork_barrier()
2497 this_thr->th.ompt_thread_info.state == ompt_state_wait_barrier_implicit) { in __kmp_fork_barrier()
2498 int ds_tid = this_thr->th.th_info.ds.ds_tid; in __kmp_fork_barrier()
2501 : &(this_thr->th.ompt_thread_info.task_data); in __kmp_fork_barrier()
2502 this_thr->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_barrier()
2530 this_thr->th.th_task_team = NULL; in __kmp_fork_barrier()
2549 team = (kmp_team_t *)TCR_PTR(this_thr->th.th_team); in __kmp_fork_barrier()
2571 ->th.th_bar[bs_forkjoin_barrier] in __kmp_fork_barrier()
2589 if (this_thr->th.th_new_place == this_thr->th.th_current_place) { in __kmp_fork_barrier()
2592 this_thr->th.th_current_place)); in __kmp_fork_barrier()
2607 this_thr->th.th_prev_num_threads = team->t.t_nproc; in __kmp_fork_barrier()
2608 this_thr->th.th_prev_level = team->t.t_level; in __kmp_fork_barrier()
2612 KMP_CHECK_UPDATE(this_thr->th.th_def_allocator, team->t.t_def_allocator); in __kmp_fork_barrier()
2644 &team->t.t_threads[0]->th.th_bar[bs_forkjoin_barrier].bb.th_fixed_icvs, in __kmp_setup_icv_copy()