Lines Matching refs:th
170 stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize); in __kmp_get_global_thread_id()
171 stack_base = (char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase); in __kmp_get_global_thread_id()
201 if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)) { in __kmp_get_global_thread_id()
205 stack_base = (char *)other_threads[i]->th.th_info.ds.ds_stackbase; in __kmp_get_global_thread_id()
207 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr); in __kmp_get_global_thread_id()
208 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize, in __kmp_get_global_thread_id()
209 other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr - in __kmp_get_global_thread_id()
212 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize, in __kmp_get_global_thread_id()
218 char *stack_end = (char *)other_threads[i]->th.th_info.ds.ds_stackbase; in __kmp_get_global_thread_id()
219 char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize; in __kmp_get_global_thread_id()
221 other_threads[i]->th.th_info.ds.ds_stacksize, in __kmp_get_global_thread_id()
270 void __kmp_check_stack_overlap(kmp_info_t *th) { in __kmp_check_stack_overlap() argument
278 stack_end = (char *)th->th.th_info.ds.ds_stackbase; in __kmp_check_stack_overlap()
279 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize; in __kmp_check_stack_overlap()
281 gtid = __kmp_gtid_from_thread(th); in __kmp_check_stack_overlap()
285 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize, in __kmp_check_stack_overlap()
287 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual"); in __kmp_check_stack_overlap()
290 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize, in __kmp_check_stack_overlap()
292 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual"); in __kmp_check_stack_overlap()
298 gtid = __kmp_gtid_from_thread(th); in __kmp_check_stack_overlap()
303 stack_end = (char *)th->th.th_info.ds.ds_stackbase; in __kmp_check_stack_overlap()
304 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize; in __kmp_check_stack_overlap()
310 if (f_th && f_th != th) { in __kmp_check_stack_overlap()
312 (char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase); in __kmp_check_stack_overlap()
314 other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize); in __kmp_check_stack_overlap()
322 (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize), in __kmp_check_stack_overlap()
475 __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team, in __kmp_print_thread_storage_map()
478 __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head, in __kmp_print_thread_storage_map()
482 gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier], in __kmp_print_thread_storage_map()
485 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier], in __kmp_print_thread_storage_map()
486 &thr->th.th_bar[bs_plain_barrier + 1], in __kmp_print_thread_storage_map()
490 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier], in __kmp_print_thread_storage_map()
491 &thr->th.th_bar[bs_forkjoin_barrier + 1], in __kmp_print_thread_storage_map()
496 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier], in __kmp_print_thread_storage_map()
497 &thr->th.th_bar[bs_reduction_barrier + 1], in __kmp_print_thread_storage_map()
616 if (__kmp_threads[gtid]->th.th_root->r.r_active) in __kmp_parallel_deo()
642 if (__kmp_threads[gtid]->th.th_root->r.r_active) in __kmp_parallel_dxo()
663 kmp_info_t *th; in __kmp_enter_single() local
670 th = __kmp_threads[gtid]; in __kmp_enter_single()
671 team = th->th.th_team; in __kmp_enter_single()
674 th->th.th_ident = id_ref; in __kmp_enter_single()
679 kmp_int32 old_this = th->th.th_local.this_construct; in __kmp_enter_single()
681 ++th->th.th_local.this_construct; in __kmp_enter_single()
687 th->th.th_local.this_construct); in __kmp_enter_single()
691 KMP_MASTER_GTID(gtid) && th->th.th_teams_microtask == NULL && in __kmp_enter_single()
825 int cg_nthreads = this_thr->th.th_cg_roots->cg_nthreads; in __kmp_reserve_threads()
826 int max_cg_threads = this_thr->th.th_cg_roots->cg_thread_limit; in __kmp_reserve_threads()
927 master_th->th.th_info.ds.ds_tid = 0; in __kmp_fork_team_threads()
928 master_th->th.th_team = team; in __kmp_fork_team_threads()
929 master_th->th.th_team_nproc = team->t.t_nproc; in __kmp_fork_team_threads()
930 master_th->th.th_team_master = master_th; in __kmp_fork_team_threads()
931 master_th->th.th_team_serialized = FALSE; in __kmp_fork_team_threads()
932 master_th->th.th_dispatch = &team->t.t_dispatch[0]; in __kmp_fork_team_threads()
937 kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams; in __kmp_fork_team_threads()
941 if (master_th->th.th_teams_microtask) { // are we inside the teams? in __kmp_fork_team_threads()
942 if (master_th->th.th_teams_size.nteams > 1) { in __kmp_fork_team_threads()
947 master_th->th.th_teams_level == team->t.t_level) { in __kmp_fork_team_threads()
982 KMP_DEBUG_ASSERT(thr->th.th_team == team); in __kmp_fork_team_threads()
990 thr->th.th_teams_microtask = master_th->th.th_teams_microtask; in __kmp_fork_team_threads()
991 thr->th.th_teams_level = master_th->th.th_teams_level; in __kmp_fork_team_threads()
992 thr->th.th_teams_size = master_th->th.th_teams_size; in __kmp_fork_team_threads()
995 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar; in __kmp_fork_team_threads()
1019 if (thr->th.th_prev_num_threads != team->t.t_nproc || in __kmp_fork_team_threads()
1020 thr->th.th_prev_level != team->t.t_level) { in __kmp_fork_team_threads()
1113 serial_team = this_thr->th.th_serial_team; in __kmp_serialized_parallel()
1121 this_thr->th.th_task_team == in __kmp_serialized_parallel()
1122 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]); in __kmp_serialized_parallel()
1123 KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] == in __kmp_serialized_parallel()
1127 global_tid, this_thr->th.th_task_team, this_thr->th.th_team)); in __kmp_serialized_parallel()
1128 this_thr->th.th_task_team = NULL; in __kmp_serialized_parallel()
1131 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind; in __kmp_serialized_parallel()
1132 if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) { in __kmp_serialized_parallel()
1137 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind; in __kmp_serialized_parallel()
1140 this_thr->th.th_set_proc_bind = proc_bind_default; in __kmp_serialized_parallel()
1146 this_thr->th.ompt_thread_info.state != ompt_state_overhead) { in __kmp_serialized_parallel()
1163 if (this_thr->th.th_team != serial_team) { in __kmp_serialized_parallel()
1165 int level = this_thr->th.th_team->t.t_level; in __kmp_serialized_parallel()
1175 __kmp_allocate_team(this_thr->th.th_root, 1, 1, in __kmp_serialized_parallel()
1179 proc_bind, &this_thr->th.th_current_task->td_icvs, in __kmp_serialized_parallel()
1186 new_team->t.t_parent = this_thr->th.th_team; in __kmp_serialized_parallel()
1188 this_thr->th.th_serial_team = serial_team; in __kmp_serialized_parallel()
1208 KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team); in __kmp_serialized_parallel()
1212 serial_team->t.t_parent = this_thr->th.th_team; in __kmp_serialized_parallel()
1213 serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched; in __kmp_serialized_parallel()
1214 this_thr->th.th_team = serial_team; in __kmp_serialized_parallel()
1215 serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid; in __kmp_serialized_parallel()
1218 this_thr->th.th_current_task)); in __kmp_serialized_parallel()
1219 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1); in __kmp_serialized_parallel()
1220 this_thr->th.th_current_task->td_flags.executing = 0; in __kmp_serialized_parallel()
1227 copy_icvs(&this_thr->th.th_current_task->td_icvs, in __kmp_serialized_parallel()
1228 &this_thr->th.th_current_task->td_parent->td_icvs); in __kmp_serialized_parallel()
1233 this_thr->th.th_current_task->td_icvs.nproc = in __kmp_serialized_parallel()
1239 this_thr->th.th_current_task->td_icvs.proc_bind = in __kmp_serialized_parallel()
1246 this_thr->th.th_info.ds.ds_tid = 0; in __kmp_serialized_parallel()
1249 this_thr->th.th_team_nproc = 1; in __kmp_serialized_parallel()
1250 this_thr->th.th_team_master = this_thr; in __kmp_serialized_parallel()
1251 this_thr->th.th_team_serialized = 1; in __kmp_serialized_parallel()
1255 serial_team->t.t_def_allocator = this_thr->th.th_def_allocator; // save in __kmp_serialized_parallel()
1266 this_thr->th.th_dispatch = serial_team->t.t_dispatch; in __kmp_serialized_parallel()
1273 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team); in __kmp_serialized_parallel()
1277 this_thr->th.th_team_serialized = serial_team->t.t_serialized; in __kmp_serialized_parallel()
1280 int level = this_thr->th.th_team->t.t_level; in __kmp_serialized_parallel()
1284 this_thr->th.th_current_task->td_icvs.nproc = in __kmp_serialized_parallel()
1301 this_thr->th.th_dispatch = serial_team->t.t_dispatch; in __kmp_serialized_parallel()
1310 if (this_thr->th.th_prev_level != serial_team->t.t_level || in __kmp_serialized_parallel()
1311 this_thr->th.th_prev_num_threads != 1) { in __kmp_serialized_parallel()
1314 this_thr->th.th_prev_level = serial_team->t.t_level; in __kmp_serialized_parallel()
1315 this_thr->th.th_prev_num_threads = 1; in __kmp_serialized_parallel()
1324 this_thr->th.ompt_thread_info.state != ompt_state_overhead) { in __kmp_serialized_parallel()
1346 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_serialized_parallel()
1400 parent_team = master_th->th.th_team; in __kmp_fork_call()
1401 master_tid = master_th->th.th_info.ds.ds_tid; in __kmp_fork_call()
1402 master_this_cons = master_th->th.th_local.this_construct; in __kmp_fork_call()
1403 root = master_th->th.th_root; in __kmp_fork_call()
1405 master_set_numthreads = master_th->th.th_set_nproc; in __kmp_fork_call()
1429 teams_level = master_th->th.th_teams_level; in __kmp_fork_call()
1431 p_hot_teams = &master_th->th.th_hot_teams; in __kmp_fork_call()
1455 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_call()
1459 master_th->th.th_ident = loc; in __kmp_fork_call()
1461 if (master_th->th.th_teams_microtask && ap && in __kmp_fork_call()
1473 if (parent_team == master_th->th.th_serial_team) { in __kmp_fork_call()
1515 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_fork_call()
1552 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_call()
1563 parent_team->t.t_def_allocator = master_th->th.th_def_allocator; // save in __kmp_fork_call()
1576 if (master_set_numthreads <= master_th->th.th_teams_size.nth) { in __kmp_fork_call()
1581 int old_proc = master_th->th.th_teams_size.nth; in __kmp_fork_call()
1590 other_threads[i]->th.th_team_nproc = master_set_numthreads; in __kmp_fork_call()
1594 master_th->th.th_set_nproc = 0; in __kmp_fork_call()
1607 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind; in __kmp_fork_call()
1610 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) { in __kmp_fork_call()
1615 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind; in __kmp_fork_call()
1624 master_th->th.th_current_task->td_icvs.proc_bind)) { in __kmp_fork_call()
1631 master_th->th.th_current_task->td_icvs.proc_bind != proc_bind_icv) { in __kmp_fork_call()
1633 for (i = 0; i < master_th->th.th_team_nproc; ++i) { in __kmp_fork_call()
1634 other_threads[i]->th.th_current_task->td_icvs.proc_bind = in __kmp_fork_call()
1639 master_th->th.th_set_proc_bind = proc_bind_default; in __kmp_fork_call()
1646 && master_th->th.th_teams_size.nteams == 1) { in __kmp_fork_call()
1648 master_th->th.th_frame_time = tmp_time; in __kmp_fork_call()
1690 KMP_DEBUG_ASSERT(master_th->th.th_task_team == in __kmp_fork_call()
1691 parent_team->t.t_task_team[master_th->th.th_task_state]); in __kmp_fork_call()
1700 master_th->th.th_current_task->td_icvs.max_active_levels) { in __kmp_fork_call()
1742 master_th->th.th_set_nproc = 0; in __kmp_fork_call()
1761 master_th->th.th_serial_team->t.t_pkfn = microtask; in __kmp_fork_call()
1766 master_th->th.th_serial_team->t.t_ident = loc; in __kmp_fork_call()
1769 master_th->th.th_serial_team->t.t_level--; in __kmp_fork_call()
1799 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_fork_call()
1834 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_call()
1838 KMP_DEBUG_ASSERT(master_th->th.th_team == in __kmp_fork_call()
1839 master_th->th.th_serial_team); in __kmp_fork_call()
1840 team = master_th->th.th_team; in __kmp_fork_call()
1873 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_call()
1909 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_fork_call()
1944 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_call()
1977 master_th->th.th_current_task, in __kmp_fork_call()
1978 master_th->th.th_current_task->td_icvs.max_active_levels)); in __kmp_fork_call()
1982 master_th->th.th_current_task->td_flags.executing = 0; in __kmp_fork_call()
1984 if (!master_th->th.th_teams_microtask || level > teams_level) { in __kmp_fork_call()
1990 int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc; in __kmp_fork_call()
1999 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind; in __kmp_fork_call()
2002 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) { in __kmp_fork_call()
2008 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind; in __kmp_fork_call()
2011 if (master_th->th.th_teams_microtask && in __kmp_fork_call()
2021 master_th->th.th_current_task->td_icvs.proc_bind)) { in __kmp_fork_call()
2024 if (!master_th->th.th_teams_microtask || in __kmp_fork_call()
2031 master_th->th.th_set_proc_bind = proc_bind_default; in __kmp_fork_call()
2035 copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs); in __kmp_fork_call()
2062 &master_th->th.th_current_task->td_icvs, in __kmp_fork_call()
2066 &master_th->th.th_current_task->td_icvs); in __kmp_fork_call()
2083 if (!master_th->th.th_teams_microtask || level > teams_level) { in __kmp_fork_call()
2100 KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator); in __kmp_fork_call()
2112 KMP_DEBUG_ASSERT(master_th->th.th_task_team == in __kmp_fork_call()
2113 parent_team->t.t_task_team[master_th->th.th_task_state]); in __kmp_fork_call()
2117 master_th->th.th_task_team, parent_team, in __kmp_fork_call()
2118 team->t.t_task_team[master_th->th.th_task_state], team)); in __kmp_fork_call()
2120 if (active_level || master_th->th.th_task_team) { in __kmp_fork_call()
2122 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack); in __kmp_fork_call()
2123 if (master_th->th.th_task_state_top >= in __kmp_fork_call()
2124 master_th->th.th_task_state_stack_sz) { // increase size in __kmp_fork_call()
2125 kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz; in __kmp_fork_call()
2129 for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) { in __kmp_fork_call()
2130 new_stack[i] = master_th->th.th_task_state_memo_stack[i]; in __kmp_fork_call()
2132 for (i = master_th->th.th_task_state_stack_sz; i < new_size; in __kmp_fork_call()
2136 old_stack = master_th->th.th_task_state_memo_stack; in __kmp_fork_call()
2137 master_th->th.th_task_state_memo_stack = new_stack; in __kmp_fork_call()
2138 master_th->th.th_task_state_stack_sz = new_size; in __kmp_fork_call()
2142 master_th->th in __kmp_fork_call()
2143 .th_task_state_memo_stack[master_th->th.th_task_state_top] = in __kmp_fork_call()
2144 master_th->th.th_task_state; in __kmp_fork_call()
2145 master_th->th.th_task_state_top++; in __kmp_fork_call()
2147 if (master_th->th.th_hot_teams && in __kmp_fork_call()
2149 team == master_th->th.th_hot_teams[active_level].hot_team) { in __kmp_fork_call()
2151 master_th->th.th_task_state = in __kmp_fork_call()
2152 master_th->th in __kmp_fork_call()
2153 .th_task_state_memo_stack[master_th->th.th_task_state_top]; in __kmp_fork_call()
2156 master_th->th.th_task_state = 0; in __kmp_fork_call()
2162 KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) || in __kmp_fork_call()
2200 &master_th->th.th_current_task->td_icvs, loc); in __kmp_fork_call()
2203 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_fork_call()
2210 && !master_th->th.th_teams_microtask) { // not in teams construct in __kmp_fork_call()
2219 master_th->th.th_frame_time = tmp_time; in __kmp_fork_call()
2234 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team); in __kmp_fork_call()
2302 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_call()
2313 thread->th.ompt_thread_info.state = in __kmp_join_restore_state()
2350 root = master_th->th.th_root; in __kmp_join_call()
2351 team = master_th->th.th_team; in __kmp_join_call()
2354 master_th->th.th_ident = loc; in __kmp_join_call()
2363 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_join_call()
2372 team->t.t_task_team[master_th->th.th_task_state], in __kmp_join_call()
2373 master_th->th.th_task_team)); in __kmp_join_call()
2374 KMP_DEBUG_ASSERT(master_th->th.th_task_team == in __kmp_join_call()
2375 team->t.t_task_team[master_th->th.th_task_state]); in __kmp_join_call()
2380 if (master_th->th.th_teams_microtask) { in __kmp_join_call()
2383 int tlevel = master_th->th.th_teams_level; in __kmp_join_call()
2421 master_th->th.th_task_state = in __kmp_join_call()
2451 (!master_th->th.th_teams_microtask || /* not in teams construct */ in __kmp_join_call()
2452 master_th->th.th_teams_size.nteams == 1)) { in __kmp_join_call()
2453 master_th->th.th_ident = loc; in __kmp_join_call()
2459 master_th->th.th_frame_time, 0, loc, in __kmp_join_call()
2460 master_th->th.th_team_nproc, 1); in __kmp_join_call()
2470 master_th->th.th_first_place = team->t.t_first_place; in __kmp_join_call()
2471 master_th->th.th_last_place = team->t.t_last_place; in __kmp_join_call()
2475 if (master_th->th.th_teams_microtask && !exit_teams && in __kmp_join_call()
2477 team->t.t_level == master_th->th.th_teams_level + 1) { in __kmp_join_call()
2506 if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) { in __kmp_join_call()
2507 int old_num = master_th->th.th_team_nproc; in __kmp_join_call()
2508 int new_num = master_th->th.th_teams_size.nth; in __kmp_join_call()
2512 other_threads[i]->th.th_team_nproc = new_num; in __kmp_join_call()
2518 kmp_balign_t *balign = other_threads[i]->th.th_bar; in __kmp_join_call()
2528 other_threads[i]->th.th_task_state = master_th->th.th_task_state; in __kmp_join_call()
2544 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid; in __kmp_join_call()
2545 master_th->th.th_local.this_construct = team->t.t_master_this_cons; in __kmp_join_call()
2547 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid]; in __kmp_join_call()
2554 if (!master_th->th.th_teams_microtask || in __kmp_join_call()
2555 team->t.t_level > master_th->th.th_teams_level) { in __kmp_join_call()
2582 master_th->th.th_def_allocator = team->t.t_def_allocator; in __kmp_join_call()
2601 master_th->th.th_team = parent_team; in __kmp_join_call()
2602 master_th->th.th_team_nproc = parent_team->t.t_nproc; in __kmp_join_call()
2603 master_th->th.th_team_master = parent_team->t.t_threads[0]; in __kmp_join_call()
2604 master_th->th.th_team_serialized = parent_team->t.t_serialized; in __kmp_join_call()
2608 parent_team != master_th->th.th_serial_team && in __kmp_join_call()
2611 master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL)); in __kmp_join_call()
2612 master_th->th.th_serial_team = parent_team; in __kmp_join_call()
2616 if (master_th->th.th_task_state_top > in __kmp_join_call()
2618 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack); in __kmp_join_call()
2620 master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] = in __kmp_join_call()
2621 master_th->th.th_task_state; in __kmp_join_call()
2622 --master_th->th.th_task_state_top; // pop in __kmp_join_call()
2624 master_th->th.th_task_state = in __kmp_join_call()
2625 master_th->th in __kmp_join_call()
2626 .th_task_state_memo_stack[master_th->th.th_task_state_top]; in __kmp_join_call()
2629 master_th->th.th_task_team = in __kmp_join_call()
2630 parent_team->t.t_task_team[master_th->th.th_task_state]; in __kmp_join_call()
2633 __kmp_gtid_from_thread(master_th), master_th->th.th_task_team, in __kmp_join_call()
2640 master_th->th.th_current_task->td_flags.executing = 1; in __kmp_join_call()
2645 if (master_th->th.th_team->t.t_level == 0 && __kmp_affin_reset) { in __kmp_join_call()
2668 if (thread->th.th_team != thread->th.th_serial_team) { in __kmp_save_internal_controls()
2671 if (thread->th.th_team->t.t_serialized > 1) { in __kmp_save_internal_controls()
2674 if (thread->th.th_team->t.t_control_stack_top == NULL) { in __kmp_save_internal_controls()
2677 if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level != in __kmp_save_internal_controls()
2678 thread->th.th_team->t.t_serialized) { in __kmp_save_internal_controls()
2687 copy_icvs(control, &thread->th.th_current_task->td_icvs); in __kmp_save_internal_controls()
2689 control->serial_nesting_level = thread->th.th_team->t.t_serialized; in __kmp_save_internal_controls()
2691 control->next = thread->th.th_team->t.t_control_stack_top; in __kmp_save_internal_controls()
2692 thread->th.th_team->t.t_control_stack_top = control; in __kmp_save_internal_controls()
2712 if (thread->th.th_current_task->td_icvs.nproc == new_nth) in __kmp_set_num_threads()
2722 root = thread->th.th_root; in __kmp_set_num_threads()
2743 hot_team->t.t_threads[f]->th.th_task_team = NULL; in __kmp_set_num_threads()
2750 if (thread->th.th_hot_teams) { in __kmp_set_num_threads()
2751 KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team); in __kmp_set_num_threads()
2752 thread->th.th_hot_teams[0].hot_team_nth = new_nth; in __kmp_set_num_threads()
2766 hot_team->t.t_threads[f]->th.th_team_nproc = new_nth; in __kmp_set_num_threads()
2826 KMP_DEBUG_ASSERT(thread->th.th_current_task); in __kmp_get_max_active_levels()
2829 gtid, thread->th.th_current_task, in __kmp_get_max_active_levels()
2830 thread->th.th_current_task->td_icvs.max_active_levels)); in __kmp_get_max_active_levels()
2831 return thread->th.th_current_task->td_icvs.max_active_levels; in __kmp_get_max_active_levels()
2885 thread->th.th_current_task->td_icvs.sched.r_sched_type = kmp_sch_static; in __kmp_set_schedule()
2887 thread->th.th_current_task->td_icvs.sched.r_sched_type = in __kmp_set_schedule()
2893 thread->th.th_current_task->td_icvs.sched.r_sched_type = in __kmp_set_schedule()
2898 orig_kind, &(thread->th.th_current_task->td_icvs.sched.r_sched_type)); in __kmp_set_schedule()
2901 thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK; in __kmp_set_schedule()
2903 thread->th.th_current_task->td_icvs.sched.chunk = chunk; in __kmp_set_schedule()
2917 th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type; in __kmp_get_schedule()
2953 *chunk = thread->th.th_current_task->td_icvs.sched.chunk; in __kmp_get_schedule()
2971 team = thr->th.th_team; in __kmp_get_ancestor_thread_num()
2976 if (thr->th.th_teams_microtask) { in __kmp_get_ancestor_thread_num()
2978 int tlevel = thr->th.th_teams_level; // the level of the teams construct in __kmp_get_ancestor_thread_num()
3029 team = thr->th.th_team; in __kmp_get_team_size()
3034 if (thr->th.th_teams_microtask) { in __kmp_get_team_size()
3036 int tlevel = thr->th.th_teams_level; // the level of the teams construct in __kmp_get_team_size()
3243 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs); in __kmp_get_x_global_icvs()
3409 __kmp_printf("%2d %p\n", thread->th.th_info.ds.ds_gtid, thread); in __kmp_print_structure_thread()
3450 __kmp_printf(" Our Root: %p\n", thread->th.th_root); in __kmp_print_structure()
3451 __kmp_print_structure_team(" Our Team: ", thread->th.th_team); in __kmp_print_structure()
3453 thread->th.th_serial_team); in __kmp_print_structure()
3454 __kmp_printf(" Threads: %2d\n", thread->th.th_team_nproc); in __kmp_print_structure()
3456 thread->th.th_team_master); in __kmp_print_structure()
3457 __kmp_printf(" Serialized?: %2d\n", thread->th.th_team_serialized); in __kmp_print_structure()
3458 __kmp_printf(" Set NProc: %2d\n", thread->th.th_set_nproc); in __kmp_print_structure()
3459 __kmp_printf(" Set Proc Bind: %2d\n", thread->th.th_set_proc_bind); in __kmp_print_structure()
3461 thread->th.th_next_pool); in __kmp_print_structure()
3463 __kmp_print_structure_team_accum(list, thread->th.th_team); in __kmp_print_structure()
3464 __kmp_print_structure_team_accum(list, thread->th.th_serial_team); in __kmp_print_structure()
3554 unsigned x = thread->th.th_x; in __kmp_get_random()
3557 thread->th.th_x = x * thread->th.th_a + 1; in __kmp_get_random()
3560 thread->th.th_info.ds.ds_tid, r)); in __kmp_get_random()
3567 unsigned seed = thread->th.th_info.ds.ds_tid; in __kmp_init_random()
3569 thread->th.th_a = in __kmp_init_random()
3571 thread->th.th_x = (seed + 1) * thread->th.th_a + 1; in __kmp_init_random()
3573 ("__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a)); in __kmp_init_random()
3836 root_thread->th.th_info.ds.ds_gtid = gtid; in __kmp_register_root()
3838 root_thread->th.ompt_thread_info.thread_data = ompt_data_none; in __kmp_register_root()
3840 root_thread->th.th_root = root; in __kmp_register_root()
3842 root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid); in __kmp_register_root()
3849 KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL); in __kmp_register_root()
3856 if (!root_thread->th.th_serial_team) { in __kmp_register_root()
3859 root_thread->th.th_serial_team = __kmp_allocate_team( in __kmp_register_root()
3866 KMP_ASSERT(root_thread->th.th_serial_team); in __kmp_register_root()
3868 root_thread->th.th_serial_team)); in __kmp_register_root()
3875 root_thread->th.th_serial_team->t.t_threads[0] = root_thread; in __kmp_register_root()
3877 root_thread->th.th_serial_team->t.t_serialized = 0; in __kmp_register_root()
3905 root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE; in __kmp_register_root()
3907 root_thread->th.th_bar[b].bb.b_worker_arrived = 0; in __kmp_register_root()
3915 root_thread->th.th_current_place = KMP_PLACE_UNDEFINED; in __kmp_register_root()
3916 root_thread->th.th_new_place = KMP_PLACE_UNDEFINED; in __kmp_register_root()
3917 root_thread->th.th_first_place = KMP_PLACE_UNDEFINED; in __kmp_register_root()
3918 root_thread->th.th_last_place = KMP_PLACE_UNDEFINED; in __kmp_register_root()
3920 root_thread->th.th_def_allocator = __kmp_def_allocator; in __kmp_register_root()
3921 root_thread->th.th_prev_level = 0; in __kmp_register_root()
3922 root_thread->th.th_prev_num_threads = 1; in __kmp_register_root()
3932 root_thread->th.th_cg_roots = tmp; in __kmp_register_root()
3974 kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams; in __kmp_free_hot_teams()
3984 kmp_info_t *th = team->t.t_threads[i]; in __kmp_free_hot_teams() local
3985 n += __kmp_free_hot_teams(root, th, level + 1, max_level); in __kmp_free_hot_teams()
3986 if (i > 0 && th->th.th_hot_teams) { in __kmp_free_hot_teams()
3987 __kmp_free(th->th.th_hot_teams); in __kmp_free_hot_teams()
3988 th->th.th_hot_teams = NULL; in __kmp_free_hot_teams()
4016 kmp_info_t *th = hot_team->t.t_threads[i]; in __kmp_reset_root() local
4018 n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level); in __kmp_reset_root()
4020 if (th->th.th_hot_teams) { in __kmp_reset_root()
4021 __kmp_free(th->th.th_hot_teams); in __kmp_reset_root()
4022 th->th.th_hot_teams = NULL; in __kmp_reset_root()
4041 (LPVOID) & (root->r.r_uber_thread->th), in __kmp_reset_root()
4042 root->r.r_uber_thread->th.th_info.ds.ds_thread)); in __kmp_reset_root()
4043 __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread); in __kmp_reset_root()
4062 &(root->r.r_uber_thread->th.ompt_thread_info.thread_data)); in __kmp_reset_root()
4068 i = root->r.r_uber_thread->th.th_cg_roots->cg_nthreads--; in __kmp_reset_root()
4071 root->r.r_uber_thread, root->r.r_uber_thread->th.th_cg_roots, in __kmp_reset_root()
4072 root->r.r_uber_thread->th.th_cg_roots->cg_nthreads)); in __kmp_reset_root()
4076 root->r.r_uber_thread->th.th_cg_roots->cg_root); in __kmp_reset_root()
4077 KMP_DEBUG_ASSERT(root->r.r_uber_thread->th.th_cg_roots->up == NULL); in __kmp_reset_root()
4078 __kmp_free(root->r.r_uber_thread->th.th_cg_roots); in __kmp_reset_root()
4079 root->r.r_uber_thread->th.th_cg_roots = NULL; in __kmp_reset_root()
4109 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root); in __kmp_unregister_root_current_thread()
4115 kmp_team_t *team = thread->th.th_team; in __kmp_unregister_root_current_thread()
4116 kmp_task_team_t *task_team = thread->th.th_task_team; in __kmp_unregister_root_current_thread()
4123 thread->th.ompt_thread_info.state = ompt_state_undefined; in __kmp_unregister_root_current_thread()
4148 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root); in __kmp_unregister_root_other_thread()
4164 kmp_team_t *steam = this_thr->th.th_serial_team; in __kmp_task_info()
4165 kmp_team_t *team = this_thr->th.th_team; in __kmp_task_info()
4170 gtid, tid, this_thr, team, steam, this_thr->th.th_current_task, in __kmp_task_info()
4184 KMP_DEBUG_ASSERT(this_thr->th.th_serial_team); in __kmp_initialize_info()
4190 KMP_DEBUG_ASSERT(master->th.th_root); in __kmp_initialize_info()
4194 TCW_SYNC_PTR(this_thr->th.th_team, team); in __kmp_initialize_info()
4196 this_thr->th.th_info.ds.ds_tid = tid; in __kmp_initialize_info()
4197 this_thr->th.th_set_nproc = 0; in __kmp_initialize_info()
4201 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP; in __kmp_initialize_info()
4203 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP; in __kmp_initialize_info()
4204 this_thr->th.th_set_proc_bind = proc_bind_default; in __kmp_initialize_info()
4206 this_thr->th.th_new_place = this_thr->th.th_current_place; in __kmp_initialize_info()
4208 this_thr->th.th_root = master->th.th_root; in __kmp_initialize_info()
4211 this_thr->th.th_team_nproc = team->t.t_nproc; in __kmp_initialize_info()
4212 this_thr->th.th_team_master = master; in __kmp_initialize_info()
4213 this_thr->th.th_team_serialized = team->t.t_serialized; in __kmp_initialize_info()
4218 tid, gtid, this_thr, this_thr->th.th_current_task)); in __kmp_initialize_info()
4220 __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr, in __kmp_initialize_info()
4224 tid, gtid, this_thr, this_thr->th.th_current_task)); in __kmp_initialize_info()
4229 this_thr->th.th_dispatch = &team->t.t_dispatch[tid]; in __kmp_initialize_info()
4231 this_thr->th.th_local.this_construct = 0; in __kmp_initialize_info()
4233 if (!this_thr->th.th_pri_common) { in __kmp_initialize_info()
4234 this_thr->th.th_pri_common = in __kmp_initialize_info()
4238 gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1, in __kmp_initialize_info()
4241 this_thr->th.th_pri_head = NULL; in __kmp_initialize_info()
4245 this_thr->th.th_cg_roots != master->th.th_cg_roots) { // CG root not set in __kmp_initialize_info()
4247 KMP_DEBUG_ASSERT(master->th.th_cg_roots); in __kmp_initialize_info()
4248 kmp_cg_root_t *tmp = this_thr->th.th_cg_roots; in __kmp_initialize_info()
4259 this_thr->th.th_cg_roots = master->th.th_cg_roots; in __kmp_initialize_info()
4261 this_thr->th.th_cg_roots->cg_nthreads++; in __kmp_initialize_info()
4264 this_thr, this_thr->th.th_cg_roots, in __kmp_initialize_info()
4265 this_thr->th.th_cg_roots->cg_root, in __kmp_initialize_info()
4266 this_thr->th.th_cg_roots->cg_nthreads)); in __kmp_initialize_info()
4267 this_thr->th.th_current_task->td_icvs.thread_limit = in __kmp_initialize_info()
4268 this_thr->th.th_cg_roots->cg_thread_limit; in __kmp_initialize_info()
4273 volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch; in __kmp_initialize_info()
4312 this_thr->th.th_next_pool = NULL; in __kmp_initialize_info()
4314 if (!this_thr->th.th_task_state_memo_stack) { in __kmp_initialize_info()
4316 this_thr->th.th_task_state_memo_stack = in __kmp_initialize_info()
4318 this_thr->th.th_task_state_top = 0; in __kmp_initialize_info()
4319 this_thr->th.th_task_state_stack_sz = 4; in __kmp_initialize_info()
4320 for (i = 0; i < this_thr->th.th_task_state_stack_sz; in __kmp_initialize_info()
4322 this_thr->th.th_task_state_memo_stack[i] = 0; in __kmp_initialize_info()
4325 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here); in __kmp_initialize_info()
4326 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0); in __kmp_initialize_info()
4352 __kmp_thread_pool = (volatile kmp_info_t *)new_thr->th.th_next_pool; in __kmp_allocate_thread()
4356 TCW_4(new_thr->th.th_in_pool, FALSE); in __kmp_allocate_thread()
4359 if (new_thr->th.th_active_in_pool == TRUE) { in __kmp_allocate_thread()
4360 KMP_DEBUG_ASSERT(new_thr->th.th_active == TRUE); in __kmp_allocate_thread()
4362 new_thr->th.th_active_in_pool = FALSE; in __kmp_allocate_thread()
4367 __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid)); in __kmp_allocate_thread()
4368 KMP_ASSERT(!new_thr->th.th_team); in __kmp_allocate_thread()
4373 new_thr->th.th_info.ds.ds_gtid); in __kmp_allocate_thread()
4374 KMP_DEBUG_ASSERT(new_thr->th.th_serial_team); in __kmp_allocate_thread()
4378 new_thr->th.th_task_state = 0; in __kmp_allocate_thread()
4379 new_thr->th.th_task_state_top = 0; in __kmp_allocate_thread()
4380 new_thr->th.th_task_state_stack_sz = 4; in __kmp_allocate_thread()
4384 KMP_DEBUG_ASSERT(new_thr->th.th_used_in_team.load() == 0); in __kmp_allocate_thread()
4402 kmp_balign_t *balign = new_thr->th.th_bar; in __kmp_allocate_thread()
4408 __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid)); in __kmp_allocate_thread()
4475 &new_thr->th.th_sleep_loc, sizeof(new_thr->th.th_sleep_loc)); in __kmp_allocate_thread()
4478 &new_thr->th.th_reap_state, sizeof(new_thr->th.th_reap_state)); in __kmp_allocate_thread()
4482 &new_thr->th.th_suspend_init, sizeof(new_thr->th.th_suspend_init)); in __kmp_allocate_thread()
4486 &new_thr->th.th_suspend_init_count, in __kmp_allocate_thread()
4487 sizeof(new_thr->th.th_suspend_init_count)); in __kmp_allocate_thread()
4492 CCAST(kmp_uint64 *, &new_thr->th.th_bar[0].bb.b_go), in __kmp_allocate_thread()
4493 sizeof(new_thr->th.th_bar[0].bb.b_go)); in __kmp_allocate_thread()
4496 CCAST(kmp_uint64 *, &new_thr->th.th_bar[1].bb.b_go), in __kmp_allocate_thread()
4497 sizeof(new_thr->th.th_bar[1].bb.b_go)); in __kmp_allocate_thread()
4500 CCAST(kmp_uint64 *, &new_thr->th.th_bar[2].bb.b_go), in __kmp_allocate_thread()
4501 sizeof(new_thr->th.th_bar[2].bb.b_go)); in __kmp_allocate_thread()
4511 new_thr->th.th_serial_team = serial_team = in __kmp_allocate_thread()
4535 KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL); in __kmp_allocate_thread()
4547 kmp_balign_t *balign = new_thr->th.th_bar; in __kmp_allocate_thread()
4555 TCW_PTR(new_thr->th.th_sleep_loc, NULL); in __kmp_allocate_thread()
4556 new_thr->th.th_sleep_loc_type = flag_unset; in __kmp_allocate_thread()
4558 new_thr->th.th_spin_here = FALSE; in __kmp_allocate_thread()
4559 new_thr->th.th_next_waiting = 0; in __kmp_allocate_thread()
4561 new_thr->th.th_blocking = false; in __kmp_allocate_thread()
4565 new_thr->th.th_current_place = KMP_PLACE_UNDEFINED; in __kmp_allocate_thread()
4566 new_thr->th.th_new_place = KMP_PLACE_UNDEFINED; in __kmp_allocate_thread()
4567 new_thr->th.th_first_place = KMP_PLACE_UNDEFINED; in __kmp_allocate_thread()
4568 new_thr->th.th_last_place = KMP_PLACE_UNDEFINED; in __kmp_allocate_thread()
4570 new_thr->th.th_def_allocator = __kmp_def_allocator; in __kmp_allocate_thread()
4571 new_thr->th.th_prev_level = 0; in __kmp_allocate_thread()
4572 new_thr->th.th_prev_num_threads = 1; in __kmp_allocate_thread()
4574 TCW_4(new_thr->th.th_in_pool, FALSE); in __kmp_allocate_thread()
4575 new_thr->th.th_active_in_pool = FALSE; in __kmp_allocate_thread()
4576 TCW_4(new_thr->th.th_active, TRUE); in __kmp_allocate_thread()
4731 int first_place = master_th->th.th_first_place; in __kmp_partition_places()
4732 int last_place = master_th->th.th_last_place; in __kmp_partition_places()
4733 int masters_place = master_th->th.th_current_place; in __kmp_partition_places()
4754 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places() local
4755 KMP_DEBUG_ASSERT(th != NULL); in __kmp_partition_places()
4756 th->th.th_first_place = first_place; in __kmp_partition_places()
4757 th->th.th_last_place = last_place; in __kmp_partition_places()
4758 th->th.th_new_place = masters_place; in __kmp_partition_places()
4759 if (__kmp_display_affinity && masters_place != th->th.th_current_place && in __kmp_partition_places()
4783 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places() local
4784 KMP_DEBUG_ASSERT(th != NULL); in __kmp_partition_places()
4793 th->th.th_first_place = first_place; in __kmp_partition_places()
4794 th->th.th_last_place = last_place; in __kmp_partition_places()
4795 th->th.th_new_place = place; in __kmp_partition_places()
4796 if (__kmp_display_affinity && place != th->th.th_current_place && in __kmp_partition_places()
4815 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places() local
4816 KMP_DEBUG_ASSERT(th != NULL); in __kmp_partition_places()
4818 th->th.th_first_place = first_place; in __kmp_partition_places()
4819 th->th.th_last_place = last_place; in __kmp_partition_places()
4820 th->th.th_new_place = place; in __kmp_partition_places()
4821 if (__kmp_display_affinity && place != th->th.th_current_place && in __kmp_partition_places()
4857 th->th.th_new_place, first_place, last_place)); in __kmp_partition_places()
4888 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places() local
4889 KMP_DEBUG_ASSERT(th != NULL); in __kmp_partition_places()
4891 th->th.th_first_place = place; in __kmp_partition_places()
4892 th->th.th_new_place = place; in __kmp_partition_places()
4893 if (__kmp_display_affinity && place != th->th.th_current_place && in __kmp_partition_places()
4919 th->th.th_last_place = place; in __kmp_partition_places()
4934 f, th->th.th_new_place, th->th.th_first_place, in __kmp_partition_places()
4935 th->th.th_last_place, __kmp_affinity_num_masks)); in __kmp_partition_places()
4945 kmp_info_t *th; in __kmp_partition_places() local
4983 th = team->t.t_threads[f]; in __kmp_partition_places()
4984 KMP_DEBUG_ASSERT(th); in __kmp_partition_places()
4985 th->th.th_first_place = first; in __kmp_partition_places()
4986 th->th.th_new_place = place; in __kmp_partition_places()
4987 th->th.th_last_place = last; in __kmp_partition_places()
4988 if (__kmp_display_affinity && place != th->th.th_current_place && in __kmp_partition_places()
4996 team->t.t_id, f, th->th.th_new_place, in __kmp_partition_places()
4997 th->th.th_first_place, th->th.th_last_place, spacing)); in __kmp_partition_places()
5014 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places() local
5015 KMP_DEBUG_ASSERT(th != NULL); in __kmp_partition_places()
5017 th->th.th_first_place = place; in __kmp_partition_places()
5018 th->th.th_last_place = place; in __kmp_partition_places()
5019 th->th.th_new_place = place; in __kmp_partition_places()
5020 if (__kmp_display_affinity && place != th->th.th_current_place && in __kmp_partition_places()
5055 team->t.t_id, f, th->th.th_new_place, in __kmp_partition_places()
5056 th->th.th_first_place, th->th.th_last_place)); in __kmp_partition_places()
5096 team = master->th.th_team; in __kmp_allocate_team()
5098 if (master->th.th_teams_microtask) { // in teams construct? in __kmp_allocate_team()
5099 if (master->th.th_teams_size.nteams > 1 && in __kmp_allocate_team()
5103 master->th.th_teams_level < in __kmp_allocate_team()
5110 if ((master->th.th_teams_size.nteams == 1 && in __kmp_allocate_team()
5111 master->th.th_teams_level >= team->t.t_level) || in __kmp_allocate_team()
5115 hot_teams = master->th.th_hot_teams; in __kmp_allocate_team()
5174 root->r.r_uber_thread->th.th_ident); in __kmp_allocate_team()
5226 team->t.t_threads[f]->th.th_task_team = NULL; in __kmp_allocate_team()
5238 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar; in __kmp_allocate_team()
5252 root->r.r_uber_thread->th.th_ident); in __kmp_allocate_team()
5256 team->t.t_threads[f]->th.th_team_nproc = new_nproc; in __kmp_allocate_team()
5269 team->t.t_threads[f]->th.th_team_nproc == in __kmp_allocate_team()
5303 kmp_balign_t *balign = other_threads[f]->th.th_bar; in __kmp_allocate_team()
5353 kmp_balign_t *balign = new_worker->th.th_bar; in __kmp_allocate_team()
5383 root->r.r_uber_thread->th.th_ident); in __kmp_allocate_team()
5398 team->t.t_threads[f]->th.th_task_state = in __kmp_allocate_team()
5399 team->t.t_threads[0]->th.th_task_state_memo_stack[level]; in __kmp_allocate_team()
5402 kmp_uint8 old_state = team->t.t_threads[0]->th.th_task_state; in __kmp_allocate_team()
5404 team->t.t_threads[f]->th.th_task_state = old_state; in __kmp_allocate_team()
5410 team->t.t_threads[f]->th.th_team_nproc == in __kmp_allocate_team()
5424 if (master->th.th_teams_microtask) { in __kmp_allocate_team()
5428 thr->th.th_teams_microtask = master->th.th_teams_microtask; in __kmp_allocate_team()
5429 thr->th.th_teams_level = master->th.th_teams_level; in __kmp_allocate_team()
5430 thr->th.th_teams_size = master->th.th_teams_size; in __kmp_allocate_team()
5440 kmp_balign_t *balign = thr->th.th_bar; in __kmp_allocate_team()
5628 if (master->th.th_teams_microtask) { // in teams construct? in __kmp_free_team()
5629 if (master->th.th_teams_size.nteams > 1) { in __kmp_free_team()
5634 master->th.th_teams_level == team->t.t_level) { in __kmp_free_team()
5640 kmp_hot_team_ptr_t *hot_teams = master->th.th_hot_teams; in __kmp_free_team()
5663 kmp_info_t *th = team->t.t_threads[f]; in __kmp_free_team() local
5664 volatile kmp_uint32 *state = &th->th.th_reap_state; in __kmp_free_team()
5669 if (!__kmp_is_thread_alive(th, &ecode)) { in __kmp_free_team()
5675 kmp_flag_64<> fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th); in __kmp_free_team()
5677 fl.resume(__kmp_gtid_from_thread(th)); in __kmp_free_team()
5689 team->t.t_threads[f]->th.th_task_team = NULL; in __kmp_free_team()
5712 KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team), in __kmp_free_team()
5726 team->t.t_threads[f]->th.th_info.ds.ds_gtid, in __kmp_free_team()
5733 while (team->t.t_threads[f]->th.th_used_in_team.load() != 0) in __kmp_free_team()
5755 team->t.t_threads[1]->th.th_cg_roots); in __kmp_free_team()
5756 if (team->t.t_threads[1]->th.th_cg_roots->cg_root == team->t.t_threads[1]) { in __kmp_free_team()
5760 KMP_DEBUG_ASSERT(thr && thr->th.th_cg_roots && in __kmp_free_team()
5761 thr->th.th_cg_roots->cg_root == thr); in __kmp_free_team()
5763 kmp_cg_root_t *tmp = thr->th.th_cg_roots; in __kmp_free_team()
5764 thr->th.th_cg_roots = tmp->up; in __kmp_free_team()
5767 thr, tmp, thr->th.th_cg_roots, tmp->cg_nthreads)); in __kmp_free_team()
5773 if (thr->th.th_cg_roots) in __kmp_free_team()
5774 thr->th.th_current_task->td_icvs.thread_limit = in __kmp_free_team()
5775 thr->th.th_cg_roots->cg_thread_limit; in __kmp_free_team()
5835 __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid)); in __kmp_free_thread()
5842 kmp_balign_t *balign = this_th->th.th_bar; in __kmp_free_thread()
5849 this_th->th.th_task_state = 0; in __kmp_free_thread()
5850 this_th->th.th_reap_state = KMP_SAFE_TO_REAP; in __kmp_free_thread()
5853 TCW_PTR(this_th->th.th_team, NULL); in __kmp_free_thread()
5854 TCW_PTR(this_th->th.th_root, NULL); in __kmp_free_thread()
5855 TCW_PTR(this_th->th.th_dispatch, NULL); /* NOT NEEDED */ in __kmp_free_thread()
5857 while (this_th->th.th_cg_roots) { in __kmp_free_thread()
5858 this_th->th.th_cg_roots->cg_nthreads--; in __kmp_free_thread()
5861 this_th, this_th->th.th_cg_roots, in __kmp_free_thread()
5862 this_th->th.th_cg_roots->cg_root, in __kmp_free_thread()
5863 this_th->th.th_cg_roots->cg_nthreads)); in __kmp_free_thread()
5864 kmp_cg_root_t *tmp = this_th->th.th_cg_roots; in __kmp_free_thread()
5869 this_th->th.th_cg_roots = tmp->up; in __kmp_free_thread()
5875 this_th->th.th_cg_roots = NULL; in __kmp_free_thread()
5886 this_th->th.th_current_task = NULL; in __kmp_free_thread()
5890 gtid = this_th->th.th_info.ds.ds_gtid; in __kmp_free_thread()
5893 if (__kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid) { in __kmp_free_thread()
5904 scan = &(__kmp_thread_pool_insert_pt->th.th_next_pool); in __kmp_free_thread()
5908 for (; (*scan != NULL) && ((*scan)->th.th_info.ds.ds_gtid < gtid); in __kmp_free_thread()
5909 scan = &((*scan)->th.th_next_pool)) in __kmp_free_thread()
5914 TCW_PTR(this_th->th.th_next_pool, *scan); in __kmp_free_thread()
5916 KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) || in __kmp_free_thread()
5917 (this_th->th.th_info.ds.ds_gtid < in __kmp_free_thread()
5918 this_th->th.th_next_pool->th.th_info.ds.ds_gtid)); in __kmp_free_thread()
5919 TCW_4(this_th->th.th_in_pool, TRUE); in __kmp_free_thread()
5922 if (this_th->th.th_active == TRUE) { in __kmp_free_thread()
5924 this_th->th.th_active_in_pool = TRUE; in __kmp_free_thread()
5928 KMP_DEBUG_ASSERT(this_th->th.th_active_in_pool == FALSE); in __kmp_free_thread()
5959 int gtid = this_thr->th.th_info.ds.ds_gtid; in __kmp_launch_thread()
5967 this_thr->th.th_cons = __kmp_allocate_cons_stack(gtid); // ATT: Memory leak? in __kmp_launch_thread()
5978 thread_data = &(this_thr->th.ompt_thread_info.thread_data); in __kmp_launch_thread()
5981 this_thr->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_launch_thread()
5982 this_thr->th.ompt_thread_info.wait_id = 0; in __kmp_launch_thread()
5983 this_thr->th.ompt_thread_info.idle_frame = OMPT_GET_FRAME_ADDRESS(0); in __kmp_launch_thread()
5984 this_thr->th.ompt_thread_info.parallel_flags = 0; in __kmp_launch_thread()
5989 this_thr->th.ompt_thread_info.state = ompt_state_idle; in __kmp_launch_thread()
6006 this_thr->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_launch_thread()
6010 pteam = &this_thr->th.th_team; in __kmp_launch_thread()
6026 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_launch_thread()
6043 this_thr->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_launch_thread()
6063 this_thr->th.th_task_team = NULL; in __kmp_launch_thread()
6138 gtid = thread->th.th_info.ds.ds_gtid; in __kmp_reap_thread()
6148 !KMP_COMPARE_AND_STORE_ACQ32(&(thread->th.th_used_in_team), 0, 3)) in __kmp_reap_thread()
6154 kmp_flag_64<> flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, in __kmp_reap_thread()
6173 if (thread->th.th_active_in_pool) { in __kmp_reap_thread()
6174 thread->th.th_active_in_pool = FALSE; in __kmp_reap_thread()
6208 if (thread->th.th_cons) { in __kmp_reap_thread()
6209 __kmp_free_cons_stack(thread->th.th_cons); in __kmp_reap_thread()
6210 thread->th.th_cons = NULL; in __kmp_reap_thread()
6214 if (thread->th.th_pri_common != NULL) { in __kmp_reap_thread()
6215 __kmp_free(thread->th.th_pri_common); in __kmp_reap_thread()
6216 thread->th.th_pri_common = NULL; in __kmp_reap_thread()
6219 if (thread->th.th_task_state_memo_stack != NULL) { in __kmp_reap_thread()
6220 __kmp_free(thread->th.th_task_state_memo_stack); in __kmp_reap_thread()
6221 thread->th.th_task_state_memo_stack = NULL; in __kmp_reap_thread()
6225 if (thread->th.th_local.bget_data != NULL) { in __kmp_reap_thread()
6231 if (thread->th.th_affin_mask != NULL) { in __kmp_reap_thread()
6232 KMP_CPU_FREE(thread->th.th_affin_mask); in __kmp_reap_thread()
6233 thread->th.th_affin_mask = NULL; in __kmp_reap_thread()
6238 if (thread->th.th_hier_bar_data != NULL) { in __kmp_reap_thread()
6239 __kmp_free(thread->th.th_hier_bar_data); in __kmp_reap_thread()
6240 thread->th.th_hier_bar_data = NULL; in __kmp_reap_thread()
6244 __kmp_reap_team(thread->th.th_serial_team); in __kmp_reap_thread()
6245 thread->th.th_serial_team = NULL; in __kmp_reap_thread()
6252 static void __kmp_itthash_clean(kmp_info_t *th) { in __kmp_itthash_clean() argument
6259 __kmp_thread_free(th, bucket); in __kmp_itthash_clean()
6269 __kmp_thread_free(th, bucket); in __kmp_itthash_clean()
6339 __kmp_thread_pool = thread->th.th_next_pool; in __kmp_internal_end()
6341 KMP_DEBUG_ASSERT(thread->th.th_reap_state == KMP_SAFE_TO_REAP); in __kmp_internal_end()
6342 thread->th.th_next_pool = NULL; in __kmp_internal_end()
6343 thread->th.th_in_pool = FALSE; in __kmp_internal_end()
6367 while (thr && KMP_ATOMIC_LD_ACQ(&thr->th.th_blocking)) in __kmp_internal_end()
6596 __kmp_threads[gtid]->th.th_task_team = NULL; in __kmp_internal_end_thread()
7335 if (thread->th.th_current_task->td_icvs.nproc != 0) in __kmp_do_middle_initialize()
7492 this_thr->th.th_local.this_construct = 0; in __kmp_run_before_invoked_task()
7494 KMP_CACHE_PREFETCH(&this_thr->th.th_bar[bs_forkjoin_barrier].bb.b_arrived); in __kmp_run_before_invoked_task()
7496 dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch); in __kmp_run_before_invoked_task()
7522 kmp_team_t *team = this_thr->th.th_team; in __kmp_invoke_task_func()
7586 this_thr->th.ompt_thread_info.parallel_flags |= ompt_parallel_team; in __kmp_invoke_task_func()
7616 kmp_team_t *team = thr->th.th_team; in __kmp_teams_master()
7618 thr->th.th_set_nproc = thr->th.th_teams_size.nth; in __kmp_teams_master()
7619 KMP_DEBUG_ASSERT(thr->th.th_teams_microtask); in __kmp_teams_master()
7620 KMP_DEBUG_ASSERT(thr->th.th_set_nproc); in __kmp_teams_master()
7622 __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask)); in __kmp_teams_master()
7628 tmp->cg_thread_limit = thr->th.th_current_task->td_icvs.thread_limit; in __kmp_teams_master()
7633 tmp->up = thr->th.th_cg_roots; in __kmp_teams_master()
7634 thr->th.th_cg_roots = tmp; in __kmp_teams_master()
7642 (microtask_t)thr->th.th_teams_microtask, // "wrapped" task in __kmp_teams_master()
7648 if (thr->th.th_team_nproc < thr->th.th_teams_size.nth) in __kmp_teams_master()
7649 thr->th.th_teams_size.nth = thr->th.th_team_nproc; in __kmp_teams_master()
7663 kmp_team_t *team = this_thr->th.th_team; in __kmp_invoke_teams_master()
7665 if (!__kmp_threads[gtid]->th.th_team->t.t_serialized) in __kmp_invoke_teams_master()
7666 KMP_DEBUG_ASSERT((void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn == in __kmp_invoke_teams_master()
7684 this_thr->th.ompt_thread_info.parallel_flags |= ompt_parallel_league; in __kmp_invoke_teams_master()
7699 thr->th.th_set_nproc = num_threads; in __kmp_push_num_threads()
7724 if (num_threads > thr->th.th_current_task->td_icvs.thread_limit) { in __kmp_push_thread_limit()
7725 num_threads = thr->th.th_current_task->td_icvs.thread_limit; in __kmp_push_thread_limit()
7741 thr->th.th_current_task->td_icvs.thread_limit = num_threads; in __kmp_push_thread_limit()
7762 thr->th.th_teams_size.nth = num_threads; in __kmp_push_thread_limit()
7795 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams; in __kmp_push_num_teams()
7852 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams; in __kmp_push_num_teams_51()
7860 thr->th.th_set_proc_bind = proc_bind; in __kmp_push_proc_bind()
7873 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); in __kmp_internal_fork()
7895 KMP_ASSERT(this_thr->th.th_team == team); in __kmp_internal_fork()
7900 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc); in __kmp_internal_fork()
7912 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); in __kmp_internal_join()
7920 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) { in __kmp_internal_join()
7925 gtid, __kmp_threads[gtid]->th.th_team_nproc, team, in __kmp_internal_join()
7930 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc); in __kmp_internal_join()
7936 this_thr->th.ompt_thread_info.state == ompt_state_wait_barrier_implicit) { in __kmp_internal_join()
7937 int ds_tid = this_thr->th.th_info.ds.ds_tid; in __kmp_internal_join()
7939 this_thr->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_internal_join()
7967 KMP_ASSERT(this_thr->th.th_team == team); in __kmp_internal_join()
7992 if (hot_team->t.t_threads[i]->th.th_active) { in __kmp_active_hot_team_nproc()
8012 ->th.th_current_task->td_icvs.dynamic == TRUE); in __kmp_load_balance_nproc()
8211 root = __kmp_threads[gtid]->th.th_root; in __kmp_internal_begin()
8239 root = thread->th.th_root; in __kmp_user_set_library()
8251 thread->th.th_set_nproc = 0; in __kmp_user_set_library()
8255 thread->th.th_set_nproc = 0; in __kmp_user_set_library()
8260 thread->th.th_set_nproc = 0; in __kmp_user_set_library()
8328 if (thr->th.th_teams_microtask) { in __kmp_aux_get_team_info()
8329 kmp_team_t *team = thr->th.th_team; in __kmp_aux_get_team_info()
8330 int tlevel = thr->th.th_teams_level; // the level of the teams construct in __kmp_aux_get_team_info()
8434 static int __kmp_aux_capture_affinity_field(int gtid, const kmp_info_t *th, in __kmp_aux_capture_affinity_field() argument
8445 KMP_DEBUG_ASSERT(th); in __kmp_aux_capture_affinity_field()
8545 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_level); in __kmp_aux_capture_affinity_field()
8563 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_nproc); in __kmp_aux_capture_affinity_field()
8567 __kmp_get_ancestor_thread_num(gtid, th->th.th_team->t.t_level - 1); in __kmp_aux_capture_affinity_field()
8574 __kmp_affinity_str_buf_mask(&buf, th->th.th_affin_mask); in __kmp_aux_capture_affinity_field()
8607 const kmp_info_t *th; in __kmp_aux_capture_affinity() local
8616 th = __kmp_threads[gtid]; in __kmp_aux_capture_affinity()
8631 int rc = __kmp_aux_capture_affinity_field(gtid, th, &parse_ptr, &field); in __kmp_aux_capture_affinity()
8671 set__blocktime_team(thread->th.th_team, tid, blocktime); in __kmp_aux_set_blocktime()
8672 set__blocktime_team(thread->th.th_serial_team, 0, blocktime); in __kmp_aux_set_blocktime()
8678 set__bt_intervals_team(thread->th.th_team, tid, bt_intervals); in __kmp_aux_set_blocktime()
8679 set__bt_intervals_team(thread->th.th_serial_team, 0, bt_intervals); in __kmp_aux_set_blocktime()
8685 set__bt_set_team(thread->th.th_team, tid, bt_set); in __kmp_aux_set_blocktime()
8686 set__bt_set_team(thread->th.th_serial_team, 0, bt_set); in __kmp_aux_set_blocktime()
8690 __kmp_gtid_from_tid(tid, thread->th.th_team), in __kmp_aux_set_blocktime()
8691 thread->th.th_team->t.t_id, tid, blocktime, bt_intervals, in __kmp_aux_set_blocktime()
8695 __kmp_gtid_from_tid(tid, thread->th.th_team), in __kmp_aux_set_blocktime()
8696 thread->th.th_team->t.t_id, tid, blocktime)); in __kmp_aux_set_blocktime()
8868 return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8); in __kmp_get_reduce_method()
8890 kmp_flag_64<> fl(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, in __kmp_resume_if_soft_paused()
8967 if (team->t.t_threads[f]->th.th_used_in_team.load() == 0) { in __kmp_resize_dist_barrier()
8973 if (team->t.t_threads[f]->th.th_used_in_team.load() == 3) { in __kmp_resize_dist_barrier()
8974 while (team->t.t_threads[f]->th.th_used_in_team.load() == 3) in __kmp_resize_dist_barrier()
8978 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 1); in __kmp_resize_dist_barrier()
8980 team->t.t_threads[f]->th.th_used_in_team.store(2); in __kmp_resize_dist_barrier()
8981 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 2); in __kmp_resize_dist_barrier()
8994 if (other_threads[f]->th.th_used_in_team.load() != 0) { in __kmp_resize_dist_barrier()
8997 void *, other_threads[f]->th.th_sleep_loc); in __kmp_resize_dist_barrier()
8998 __kmp_atomic_resume_64(other_threads[f]->th.th_info.ds.ds_gtid, flag); in __kmp_resize_dist_barrier()
9001 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 0); in __kmp_resize_dist_barrier()
9021 KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team), 0, in __kmp_add_threads_to_team()
9024 __kmp_resume_32(team->t.t_threads[f]->th.th_info.ds.ds_gtid, in __kmp_add_threads_to_team()
9035 if (team->t.t_threads[f]->th.th_used_in_team.load() == 1) { in __kmp_add_threads_to_team()
9086 __kmp_hidden_helper_main_thread->th.th_set_nproc = in __kmp_hidden_helper_threads_initz_routine()