Lines Matching refs:th
166 stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize); in __kmp_get_global_thread_id()
167 stack_base = (char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase); in __kmp_get_global_thread_id()
197 if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)) { in __kmp_get_global_thread_id()
201 stack_base = (char *)other_threads[i]->th.th_info.ds.ds_stackbase; in __kmp_get_global_thread_id()
203 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr); in __kmp_get_global_thread_id()
204 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize, in __kmp_get_global_thread_id()
205 other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr - in __kmp_get_global_thread_id()
208 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize, in __kmp_get_global_thread_id()
214 char *stack_end = (char *)other_threads[i]->th.th_info.ds.ds_stackbase; in __kmp_get_global_thread_id()
215 char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize; in __kmp_get_global_thread_id()
217 other_threads[i]->th.th_info.ds.ds_stacksize, in __kmp_get_global_thread_id()
266 void __kmp_check_stack_overlap(kmp_info_t *th) { in __kmp_check_stack_overlap() argument
274 stack_end = (char *)th->th.th_info.ds.ds_stackbase; in __kmp_check_stack_overlap()
275 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize; in __kmp_check_stack_overlap()
277 gtid = __kmp_gtid_from_thread(th); in __kmp_check_stack_overlap()
281 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize, in __kmp_check_stack_overlap()
283 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual"); in __kmp_check_stack_overlap()
286 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize, in __kmp_check_stack_overlap()
288 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual"); in __kmp_check_stack_overlap()
294 gtid = __kmp_gtid_from_thread(th); in __kmp_check_stack_overlap()
299 stack_end = (char *)th->th.th_info.ds.ds_stackbase; in __kmp_check_stack_overlap()
300 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize; in __kmp_check_stack_overlap()
306 if (f_th && f_th != th) { in __kmp_check_stack_overlap()
308 (char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase); in __kmp_check_stack_overlap()
310 other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize); in __kmp_check_stack_overlap()
318 (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize), in __kmp_check_stack_overlap()
471 __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team, in __kmp_print_thread_storage_map()
474 __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head, in __kmp_print_thread_storage_map()
478 gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier], in __kmp_print_thread_storage_map()
481 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier], in __kmp_print_thread_storage_map()
482 &thr->th.th_bar[bs_plain_barrier + 1], in __kmp_print_thread_storage_map()
486 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier], in __kmp_print_thread_storage_map()
487 &thr->th.th_bar[bs_forkjoin_barrier + 1], in __kmp_print_thread_storage_map()
492 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier], in __kmp_print_thread_storage_map()
493 &thr->th.th_bar[bs_reduction_barrier + 1], in __kmp_print_thread_storage_map()
612 if (__kmp_threads[gtid]->th.th_root->r.r_active) in __kmp_parallel_deo()
638 if (__kmp_threads[gtid]->th.th_root->r.r_active) in __kmp_parallel_dxo()
659 kmp_info_t *th; in __kmp_enter_single() local
666 th = __kmp_threads[gtid]; in __kmp_enter_single()
667 team = th->th.th_team; in __kmp_enter_single()
670 th->th.th_ident = id_ref; in __kmp_enter_single()
675 kmp_int32 old_this = th->th.th_local.this_construct; in __kmp_enter_single()
677 ++th->th.th_local.this_construct; in __kmp_enter_single()
683 th->th.th_local.this_construct); in __kmp_enter_single()
687 KMP_MASTER_GTID(gtid) && th->th.th_teams_microtask == NULL && in __kmp_enter_single()
821 int cg_nthreads = this_thr->th.th_cg_roots->cg_nthreads; in __kmp_reserve_threads()
822 int max_cg_threads = this_thr->th.th_cg_roots->cg_thread_limit; in __kmp_reserve_threads()
922 master_th->th.th_info.ds.ds_tid = 0; in __kmp_fork_team_threads()
923 master_th->th.th_team = team; in __kmp_fork_team_threads()
924 master_th->th.th_team_nproc = team->t.t_nproc; in __kmp_fork_team_threads()
925 master_th->th.th_team_master = master_th; in __kmp_fork_team_threads()
926 master_th->th.th_team_serialized = FALSE; in __kmp_fork_team_threads()
927 master_th->th.th_dispatch = &team->t.t_dispatch[0]; in __kmp_fork_team_threads()
932 kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams; in __kmp_fork_team_threads()
936 if (master_th->th.th_teams_microtask) { // are we inside the teams? in __kmp_fork_team_threads()
937 if (master_th->th.th_teams_size.nteams > 1) { in __kmp_fork_team_threads()
942 master_th->th.th_teams_level == team->t.t_level) { in __kmp_fork_team_threads()
977 KMP_DEBUG_ASSERT(thr->th.th_team == team); in __kmp_fork_team_threads()
985 thr->th.th_teams_microtask = master_th->th.th_teams_microtask; in __kmp_fork_team_threads()
986 thr->th.th_teams_level = master_th->th.th_teams_level; in __kmp_fork_team_threads()
987 thr->th.th_teams_size = master_th->th.th_teams_size; in __kmp_fork_team_threads()
990 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar; in __kmp_fork_team_threads()
1009 if (thr->th.th_prev_num_threads != team->t.t_nproc || in __kmp_fork_team_threads()
1010 thr->th.th_prev_level != team->t.t_level) { in __kmp_fork_team_threads()
1103 serial_team = this_thr->th.th_serial_team; in __kmp_serialized_parallel()
1111 this_thr->th.th_task_team == in __kmp_serialized_parallel()
1112 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]); in __kmp_serialized_parallel()
1113 KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] == in __kmp_serialized_parallel()
1117 global_tid, this_thr->th.th_task_team, this_thr->th.th_team)); in __kmp_serialized_parallel()
1118 this_thr->th.th_task_team = NULL; in __kmp_serialized_parallel()
1121 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind; in __kmp_serialized_parallel()
1122 if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) { in __kmp_serialized_parallel()
1127 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind; in __kmp_serialized_parallel()
1130 this_thr->th.th_set_proc_bind = proc_bind_default; in __kmp_serialized_parallel()
1136 this_thr->th.ompt_thread_info.state != ompt_state_overhead) { in __kmp_serialized_parallel()
1153 if (this_thr->th.th_team != serial_team) { in __kmp_serialized_parallel()
1155 int level = this_thr->th.th_team->t.t_level; in __kmp_serialized_parallel()
1165 __kmp_allocate_team(this_thr->th.th_root, 1, 1, in __kmp_serialized_parallel()
1169 proc_bind, &this_thr->th.th_current_task->td_icvs, in __kmp_serialized_parallel()
1176 new_team->t.t_parent = this_thr->th.th_team; in __kmp_serialized_parallel()
1178 this_thr->th.th_serial_team = serial_team; in __kmp_serialized_parallel()
1198 KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team); in __kmp_serialized_parallel()
1202 serial_team->t.t_parent = this_thr->th.th_team; in __kmp_serialized_parallel()
1203 serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched; in __kmp_serialized_parallel()
1204 this_thr->th.th_team = serial_team; in __kmp_serialized_parallel()
1205 serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid; in __kmp_serialized_parallel()
1208 this_thr->th.th_current_task)); in __kmp_serialized_parallel()
1209 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1); in __kmp_serialized_parallel()
1210 this_thr->th.th_current_task->td_flags.executing = 0; in __kmp_serialized_parallel()
1217 copy_icvs(&this_thr->th.th_current_task->td_icvs, in __kmp_serialized_parallel()
1218 &this_thr->th.th_current_task->td_parent->td_icvs); in __kmp_serialized_parallel()
1223 this_thr->th.th_current_task->td_icvs.nproc = in __kmp_serialized_parallel()
1229 this_thr->th.th_current_task->td_icvs.proc_bind = in __kmp_serialized_parallel()
1236 this_thr->th.th_info.ds.ds_tid = 0; in __kmp_serialized_parallel()
1239 this_thr->th.th_team_nproc = 1; in __kmp_serialized_parallel()
1240 this_thr->th.th_team_master = this_thr; in __kmp_serialized_parallel()
1241 this_thr->th.th_team_serialized = 1; in __kmp_serialized_parallel()
1245 serial_team->t.t_def_allocator = this_thr->th.th_def_allocator; // save in __kmp_serialized_parallel()
1256 this_thr->th.th_dispatch = serial_team->t.t_dispatch; in __kmp_serialized_parallel()
1263 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team); in __kmp_serialized_parallel()
1267 this_thr->th.th_team_serialized = serial_team->t.t_serialized; in __kmp_serialized_parallel()
1270 int level = this_thr->th.th_team->t.t_level; in __kmp_serialized_parallel()
1274 this_thr->th.th_current_task->td_icvs.nproc = in __kmp_serialized_parallel()
1291 this_thr->th.th_dispatch = serial_team->t.t_dispatch; in __kmp_serialized_parallel()
1300 if (this_thr->th.th_prev_level != serial_team->t.t_level || in __kmp_serialized_parallel()
1301 this_thr->th.th_prev_num_threads != 1) { in __kmp_serialized_parallel()
1304 this_thr->th.th_prev_level = serial_team->t.t_level; in __kmp_serialized_parallel()
1305 this_thr->th.th_prev_num_threads = 1; in __kmp_serialized_parallel()
1314 this_thr->th.ompt_thread_info.state != ompt_state_overhead) { in __kmp_serialized_parallel()
1336 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_serialized_parallel()
1390 parent_team = master_th->th.th_team; in __kmp_fork_call()
1391 master_tid = master_th->th.th_info.ds.ds_tid; in __kmp_fork_call()
1392 master_this_cons = master_th->th.th_local.this_construct; in __kmp_fork_call()
1393 root = master_th->th.th_root; in __kmp_fork_call()
1395 master_set_numthreads = master_th->th.th_set_nproc; in __kmp_fork_call()
1419 teams_level = master_th->th.th_teams_level; in __kmp_fork_call()
1421 p_hot_teams = &master_th->th.th_hot_teams; in __kmp_fork_call()
1445 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_call()
1449 master_th->th.th_ident = loc; in __kmp_fork_call()
1451 if (master_th->th.th_teams_microtask && ap && in __kmp_fork_call()
1463 if (parent_team == master_th->th.th_serial_team) { in __kmp_fork_call()
1505 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_fork_call()
1542 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_call()
1553 parent_team->t.t_def_allocator = master_th->th.th_def_allocator; // save in __kmp_fork_call()
1566 if (master_set_numthreads < master_th->th.th_teams_size.nth) { in __kmp_fork_call()
1571 other_threads[i]->th.th_team_nproc = master_set_numthreads; in __kmp_fork_call()
1575 master_th->th.th_set_nproc = 0; in __kmp_fork_call()
1592 && master_th->th.th_teams_size.nteams == 1) { in __kmp_fork_call()
1594 master_th->th.th_frame_time = tmp_time; in __kmp_fork_call()
1633 KMP_DEBUG_ASSERT(master_th->th.th_task_team == in __kmp_fork_call()
1634 parent_team->t.t_task_team[master_th->th.th_task_state]); in __kmp_fork_call()
1640 master_th->th.th_current_task->td_icvs.max_active_levels) { in __kmp_fork_call()
1685 master_th->th.th_set_nproc = 0; in __kmp_fork_call()
1704 master_th->th.th_serial_team->t.t_pkfn = microtask; in __kmp_fork_call()
1709 master_th->th.th_serial_team->t.t_ident = loc; in __kmp_fork_call()
1712 master_th->th.th_serial_team->t.t_level--; in __kmp_fork_call()
1742 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_fork_call()
1777 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_call()
1781 KMP_DEBUG_ASSERT(master_th->th.th_team == in __kmp_fork_call()
1782 master_th->th.th_serial_team); in __kmp_fork_call()
1783 team = master_th->th.th_team; in __kmp_fork_call()
1816 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_call()
1852 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_fork_call()
1887 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_call()
1920 master_th->th.th_current_task, in __kmp_fork_call()
1921 master_th->th.th_current_task->td_icvs.max_active_levels)); in __kmp_fork_call()
1925 master_th->th.th_current_task->td_flags.executing = 0; in __kmp_fork_call()
1927 if (!master_th->th.th_teams_microtask || level > teams_level) { in __kmp_fork_call()
1933 int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc; in __kmp_fork_call()
1942 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind; in __kmp_fork_call()
1945 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) { in __kmp_fork_call()
1951 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind; in __kmp_fork_call()
1959 master_th->th.th_current_task->td_icvs.proc_bind)) { in __kmp_fork_call()
1965 master_th->th.th_set_proc_bind = proc_bind_default; in __kmp_fork_call()
1969 copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs); in __kmp_fork_call()
1994 &master_th->th.th_current_task->td_icvs, in __kmp_fork_call()
2012 if (!master_th->th.th_teams_microtask || level > teams_level) { in __kmp_fork_call()
2029 KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator); in __kmp_fork_call()
2041 KMP_DEBUG_ASSERT(master_th->th.th_task_team == in __kmp_fork_call()
2042 parent_team->t.t_task_team[master_th->th.th_task_state]); in __kmp_fork_call()
2046 master_th->th.th_task_team, parent_team, in __kmp_fork_call()
2047 team->t.t_task_team[master_th->th.th_task_state], team)); in __kmp_fork_call()
2049 if (active_level || master_th->th.th_task_team) { in __kmp_fork_call()
2051 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack); in __kmp_fork_call()
2052 if (master_th->th.th_task_state_top >= in __kmp_fork_call()
2053 master_th->th.th_task_state_stack_sz) { // increase size in __kmp_fork_call()
2054 kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz; in __kmp_fork_call()
2058 for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) { in __kmp_fork_call()
2059 new_stack[i] = master_th->th.th_task_state_memo_stack[i]; in __kmp_fork_call()
2061 for (i = master_th->th.th_task_state_stack_sz; i < new_size; in __kmp_fork_call()
2065 old_stack = master_th->th.th_task_state_memo_stack; in __kmp_fork_call()
2066 master_th->th.th_task_state_memo_stack = new_stack; in __kmp_fork_call()
2067 master_th->th.th_task_state_stack_sz = new_size; in __kmp_fork_call()
2071 master_th->th in __kmp_fork_call()
2072 .th_task_state_memo_stack[master_th->th.th_task_state_top] = in __kmp_fork_call()
2073 master_th->th.th_task_state; in __kmp_fork_call()
2074 master_th->th.th_task_state_top++; in __kmp_fork_call()
2076 if (master_th->th.th_hot_teams && in __kmp_fork_call()
2078 team == master_th->th.th_hot_teams[active_level].hot_team) { in __kmp_fork_call()
2080 master_th->th.th_task_state = in __kmp_fork_call()
2081 master_th->th in __kmp_fork_call()
2082 .th_task_state_memo_stack[master_th->th.th_task_state_top]; in __kmp_fork_call()
2085 master_th->th.th_task_state = 0; in __kmp_fork_call()
2091 KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) || in __kmp_fork_call()
2129 &master_th->th.th_current_task->td_icvs, loc); in __kmp_fork_call()
2132 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_fork_call()
2139 && !master_th->th.th_teams_microtask) { // not in teams construct in __kmp_fork_call()
2148 master_th->th.th_frame_time = tmp_time; in __kmp_fork_call()
2163 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team); in __kmp_fork_call()
2231 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_call()
2242 thread->th.ompt_thread_info.state = in __kmp_join_restore_state()
2279 root = master_th->th.th_root; in __kmp_join_call()
2280 team = master_th->th.th_team; in __kmp_join_call()
2283 master_th->th.th_ident = loc; in __kmp_join_call()
2292 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_join_call()
2301 team->t.t_task_team[master_th->th.th_task_state], in __kmp_join_call()
2302 master_th->th.th_task_team)); in __kmp_join_call()
2303 KMP_DEBUG_ASSERT(master_th->th.th_task_team == in __kmp_join_call()
2304 team->t.t_task_team[master_th->th.th_task_state]); in __kmp_join_call()
2309 if (master_th->th.th_teams_microtask) { in __kmp_join_call()
2312 int tlevel = master_th->th.th_teams_level; in __kmp_join_call()
2350 master_th->th.th_task_state = in __kmp_join_call()
2374 (!master_th->th.th_teams_microtask || /* not in teams construct */ in __kmp_join_call()
2375 master_th->th.th_teams_size.nteams == 1)) { in __kmp_join_call()
2376 master_th->th.th_ident = loc; in __kmp_join_call()
2382 master_th->th.th_frame_time, 0, loc, in __kmp_join_call()
2383 master_th->th.th_team_nproc, 1); in __kmp_join_call()
2390 if (master_th->th.th_teams_microtask && !exit_teams && in __kmp_join_call()
2392 team->t.t_level == master_th->th.th_teams_level + 1) { in __kmp_join_call()
2421 if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) { in __kmp_join_call()
2422 int old_num = master_th->th.th_team_nproc; in __kmp_join_call()
2423 int new_num = master_th->th.th_teams_size.nth; in __kmp_join_call()
2427 other_threads[i]->th.th_team_nproc = new_num; in __kmp_join_call()
2433 kmp_balign_t *balign = other_threads[i]->th.th_bar; in __kmp_join_call()
2443 other_threads[i]->th.th_task_state = master_th->th.th_task_state; in __kmp_join_call()
2459 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid; in __kmp_join_call()
2460 master_th->th.th_local.this_construct = team->t.t_master_this_cons; in __kmp_join_call()
2462 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid]; in __kmp_join_call()
2469 if (!master_th->th.th_teams_microtask || in __kmp_join_call()
2470 team->t.t_level > master_th->th.th_teams_level) { in __kmp_join_call()
2499 master_th->th.th_first_place = team->t.t_first_place; in __kmp_join_call()
2500 master_th->th.th_last_place = team->t.t_last_place; in __kmp_join_call()
2502 master_th->th.th_def_allocator = team->t.t_def_allocator; in __kmp_join_call()
2521 master_th->th.th_team = parent_team; in __kmp_join_call()
2522 master_th->th.th_team_nproc = parent_team->t.t_nproc; in __kmp_join_call()
2523 master_th->th.th_team_master = parent_team->t.t_threads[0]; in __kmp_join_call()
2524 master_th->th.th_team_serialized = parent_team->t.t_serialized; in __kmp_join_call()
2528 parent_team != master_th->th.th_serial_team && in __kmp_join_call()
2531 master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL)); in __kmp_join_call()
2532 master_th->th.th_serial_team = parent_team; in __kmp_join_call()
2536 if (master_th->th.th_task_state_top > in __kmp_join_call()
2538 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack); in __kmp_join_call()
2540 master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] = in __kmp_join_call()
2541 master_th->th.th_task_state; in __kmp_join_call()
2542 --master_th->th.th_task_state_top; // pop in __kmp_join_call()
2544 master_th->th.th_task_state = in __kmp_join_call()
2545 master_th->th in __kmp_join_call()
2546 .th_task_state_memo_stack[master_th->th.th_task_state_top]; in __kmp_join_call()
2549 master_th->th.th_task_team = in __kmp_join_call()
2550 parent_team->t.t_task_team[master_th->th.th_task_state]; in __kmp_join_call()
2553 __kmp_gtid_from_thread(master_th), master_th->th.th_task_team, in __kmp_join_call()
2560 master_th->th.th_current_task->td_flags.executing = 1; in __kmp_join_call()
2583 if (thread->th.th_team != thread->th.th_serial_team) { in __kmp_save_internal_controls()
2586 if (thread->th.th_team->t.t_serialized > 1) { in __kmp_save_internal_controls()
2589 if (thread->th.th_team->t.t_control_stack_top == NULL) { in __kmp_save_internal_controls()
2592 if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level != in __kmp_save_internal_controls()
2593 thread->th.th_team->t.t_serialized) { in __kmp_save_internal_controls()
2602 copy_icvs(control, &thread->th.th_current_task->td_icvs); in __kmp_save_internal_controls()
2604 control->serial_nesting_level = thread->th.th_team->t.t_serialized; in __kmp_save_internal_controls()
2606 control->next = thread->th.th_team->t.t_control_stack_top; in __kmp_save_internal_controls()
2607 thread->th.th_team->t.t_control_stack_top = control; in __kmp_save_internal_controls()
2627 if (thread->th.th_current_task->td_icvs.nproc == new_nth) in __kmp_set_num_threads()
2637 root = thread->th.th_root; in __kmp_set_num_threads()
2655 hot_team->t.t_threads[f]->th.th_task_team = NULL; in __kmp_set_num_threads()
2662 if (thread->th.th_hot_teams) { in __kmp_set_num_threads()
2663 KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team); in __kmp_set_num_threads()
2664 thread->th.th_hot_teams[0].hot_team_nth = new_nth; in __kmp_set_num_threads()
2673 hot_team->t.t_threads[f]->th.th_team_nproc = new_nth; in __kmp_set_num_threads()
2733 KMP_DEBUG_ASSERT(thread->th.th_current_task); in __kmp_get_max_active_levels()
2736 gtid, thread->th.th_current_task, in __kmp_get_max_active_levels()
2737 thread->th.th_current_task->td_icvs.max_active_levels)); in __kmp_get_max_active_levels()
2738 return thread->th.th_current_task->td_icvs.max_active_levels; in __kmp_get_max_active_levels()
2792 thread->th.th_current_task->td_icvs.sched.r_sched_type = kmp_sch_static; in __kmp_set_schedule()
2794 thread->th.th_current_task->td_icvs.sched.r_sched_type = in __kmp_set_schedule()
2800 thread->th.th_current_task->td_icvs.sched.r_sched_type = in __kmp_set_schedule()
2805 orig_kind, &(thread->th.th_current_task->td_icvs.sched.r_sched_type)); in __kmp_set_schedule()
2808 thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK; in __kmp_set_schedule()
2810 thread->th.th_current_task->td_icvs.sched.chunk = chunk; in __kmp_set_schedule()
2824 th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type; in __kmp_get_schedule()
2860 *chunk = thread->th.th_current_task->td_icvs.sched.chunk; in __kmp_get_schedule()
2878 team = thr->th.th_team; in __kmp_get_ancestor_thread_num()
2883 if (thr->th.th_teams_microtask) { in __kmp_get_ancestor_thread_num()
2885 int tlevel = thr->th.th_teams_level; // the level of the teams construct in __kmp_get_ancestor_thread_num()
2936 team = thr->th.th_team; in __kmp_get_team_size()
2941 if (thr->th.th_teams_microtask) { in __kmp_get_team_size()
2943 int tlevel = thr->th.th_teams_level; // the level of the teams construct in __kmp_get_team_size()
3150 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs); in __kmp_get_x_global_icvs()
3316 __kmp_printf("%2d %p\n", thread->th.th_info.ds.ds_gtid, thread); in __kmp_print_structure_thread()
3357 __kmp_printf(" Our Root: %p\n", thread->th.th_root); in __kmp_print_structure()
3358 __kmp_print_structure_team(" Our Team: ", thread->th.th_team); in __kmp_print_structure()
3360 thread->th.th_serial_team); in __kmp_print_structure()
3361 __kmp_printf(" Threads: %2d\n", thread->th.th_team_nproc); in __kmp_print_structure()
3363 thread->th.th_team_master); in __kmp_print_structure()
3364 __kmp_printf(" Serialized?: %2d\n", thread->th.th_team_serialized); in __kmp_print_structure()
3365 __kmp_printf(" Set NProc: %2d\n", thread->th.th_set_nproc); in __kmp_print_structure()
3366 __kmp_printf(" Set Proc Bind: %2d\n", thread->th.th_set_proc_bind); in __kmp_print_structure()
3368 thread->th.th_next_pool); in __kmp_print_structure()
3370 __kmp_print_structure_team_accum(list, thread->th.th_team); in __kmp_print_structure()
3371 __kmp_print_structure_team_accum(list, thread->th.th_serial_team); in __kmp_print_structure()
3461 unsigned x = thread->th.th_x; in __kmp_get_random()
3464 thread->th.th_x = x * thread->th.th_a + 1; in __kmp_get_random()
3467 thread->th.th_info.ds.ds_tid, r)); in __kmp_get_random()
3474 unsigned seed = thread->th.th_info.ds.ds_tid; in __kmp_init_random()
3476 thread->th.th_a = in __kmp_init_random()
3478 thread->th.th_x = (seed + 1) * thread->th.th_a + 1; in __kmp_init_random()
3480 ("__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a)); in __kmp_init_random()
3738 root_thread->th.th_info.ds.ds_gtid = gtid; in __kmp_register_root()
3740 root_thread->th.ompt_thread_info.thread_data = ompt_data_none; in __kmp_register_root()
3742 root_thread->th.th_root = root; in __kmp_register_root()
3744 root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid); in __kmp_register_root()
3751 KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL); in __kmp_register_root()
3758 if (!root_thread->th.th_serial_team) { in __kmp_register_root()
3761 root_thread->th.th_serial_team = __kmp_allocate_team( in __kmp_register_root()
3768 KMP_ASSERT(root_thread->th.th_serial_team); in __kmp_register_root()
3770 root_thread->th.th_serial_team)); in __kmp_register_root()
3777 root_thread->th.th_serial_team->t.t_threads[0] = root_thread; in __kmp_register_root()
3779 root_thread->th.th_serial_team->t.t_serialized = 0; in __kmp_register_root()
3807 root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE; in __kmp_register_root()
3809 root_thread->th.th_bar[b].bb.b_worker_arrived = 0; in __kmp_register_root()
3817 root_thread->th.th_current_place = KMP_PLACE_UNDEFINED; in __kmp_register_root()
3818 root_thread->th.th_new_place = KMP_PLACE_UNDEFINED; in __kmp_register_root()
3819 root_thread->th.th_first_place = KMP_PLACE_UNDEFINED; in __kmp_register_root()
3820 root_thread->th.th_last_place = KMP_PLACE_UNDEFINED; in __kmp_register_root()
3822 root_thread->th.th_def_allocator = __kmp_def_allocator; in __kmp_register_root()
3823 root_thread->th.th_prev_level = 0; in __kmp_register_root()
3824 root_thread->th.th_prev_num_threads = 1; in __kmp_register_root()
3834 root_thread->th.th_cg_roots = tmp; in __kmp_register_root()
3876 kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams; in __kmp_free_hot_teams()
3886 kmp_info_t *th = team->t.t_threads[i]; in __kmp_free_hot_teams() local
3887 n += __kmp_free_hot_teams(root, th, level + 1, max_level); in __kmp_free_hot_teams()
3888 if (i > 0 && th->th.th_hot_teams) { in __kmp_free_hot_teams()
3889 __kmp_free(th->th.th_hot_teams); in __kmp_free_hot_teams()
3890 th->th.th_hot_teams = NULL; in __kmp_free_hot_teams()
3918 kmp_info_t *th = hot_team->t.t_threads[i]; in __kmp_reset_root() local
3920 n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level); in __kmp_reset_root()
3922 if (th->th.th_hot_teams) { in __kmp_reset_root()
3923 __kmp_free(th->th.th_hot_teams); in __kmp_reset_root()
3924 th->th.th_hot_teams = NULL; in __kmp_reset_root()
3943 (LPVOID) & (root->r.r_uber_thread->th), in __kmp_reset_root()
3944 root->r.r_uber_thread->th.th_info.ds.ds_thread)); in __kmp_reset_root()
3945 __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread); in __kmp_reset_root()
3964 &(root->r.r_uber_thread->th.ompt_thread_info.thread_data)); in __kmp_reset_root()
3970 i = root->r.r_uber_thread->th.th_cg_roots->cg_nthreads--; in __kmp_reset_root()
3973 root->r.r_uber_thread, root->r.r_uber_thread->th.th_cg_roots, in __kmp_reset_root()
3974 root->r.r_uber_thread->th.th_cg_roots->cg_nthreads)); in __kmp_reset_root()
3978 root->r.r_uber_thread->th.th_cg_roots->cg_root); in __kmp_reset_root()
3979 KMP_DEBUG_ASSERT(root->r.r_uber_thread->th.th_cg_roots->up == NULL); in __kmp_reset_root()
3980 __kmp_free(root->r.r_uber_thread->th.th_cg_roots); in __kmp_reset_root()
3981 root->r.r_uber_thread->th.th_cg_roots = NULL; in __kmp_reset_root()
4011 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root); in __kmp_unregister_root_current_thread()
4017 kmp_team_t *team = thread->th.th_team; in __kmp_unregister_root_current_thread()
4018 kmp_task_team_t *task_team = thread->th.th_task_team; in __kmp_unregister_root_current_thread()
4024 thread->th.ompt_thread_info.state = ompt_state_undefined; in __kmp_unregister_root_current_thread()
4049 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root); in __kmp_unregister_root_other_thread()
4065 kmp_team_t *steam = this_thr->th.th_serial_team; in __kmp_task_info()
4066 kmp_team_t *team = this_thr->th.th_team; in __kmp_task_info()
4071 gtid, tid, this_thr, team, steam, this_thr->th.th_current_task, in __kmp_task_info()
4085 KMP_DEBUG_ASSERT(this_thr->th.th_serial_team); in __kmp_initialize_info()
4091 KMP_DEBUG_ASSERT(master->th.th_root); in __kmp_initialize_info()
4095 TCW_SYNC_PTR(this_thr->th.th_team, team); in __kmp_initialize_info()
4097 this_thr->th.th_info.ds.ds_tid = tid; in __kmp_initialize_info()
4098 this_thr->th.th_set_nproc = 0; in __kmp_initialize_info()
4102 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP; in __kmp_initialize_info()
4104 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP; in __kmp_initialize_info()
4105 this_thr->th.th_set_proc_bind = proc_bind_default; in __kmp_initialize_info()
4107 this_thr->th.th_new_place = this_thr->th.th_current_place; in __kmp_initialize_info()
4109 this_thr->th.th_root = master->th.th_root; in __kmp_initialize_info()
4112 this_thr->th.th_team_nproc = team->t.t_nproc; in __kmp_initialize_info()
4113 this_thr->th.th_team_master = master; in __kmp_initialize_info()
4114 this_thr->th.th_team_serialized = team->t.t_serialized; in __kmp_initialize_info()
4115 TCW_PTR(this_thr->th.th_sleep_loc, NULL); in __kmp_initialize_info()
4120 tid, gtid, this_thr, this_thr->th.th_current_task)); in __kmp_initialize_info()
4122 __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr, in __kmp_initialize_info()
4126 tid, gtid, this_thr, this_thr->th.th_current_task)); in __kmp_initialize_info()
4131 this_thr->th.th_dispatch = &team->t.t_dispatch[tid]; in __kmp_initialize_info()
4133 this_thr->th.th_local.this_construct = 0; in __kmp_initialize_info()
4135 if (!this_thr->th.th_pri_common) { in __kmp_initialize_info()
4136 this_thr->th.th_pri_common = in __kmp_initialize_info()
4140 gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1, in __kmp_initialize_info()
4143 this_thr->th.th_pri_head = NULL; in __kmp_initialize_info()
4147 this_thr->th.th_cg_roots != master->th.th_cg_roots) { // CG root not set in __kmp_initialize_info()
4149 KMP_DEBUG_ASSERT(master->th.th_cg_roots); in __kmp_initialize_info()
4150 kmp_cg_root_t *tmp = this_thr->th.th_cg_roots; in __kmp_initialize_info()
4161 this_thr->th.th_cg_roots = master->th.th_cg_roots; in __kmp_initialize_info()
4163 this_thr->th.th_cg_roots->cg_nthreads++; in __kmp_initialize_info()
4166 this_thr, this_thr->th.th_cg_roots, in __kmp_initialize_info()
4167 this_thr->th.th_cg_roots->cg_root, in __kmp_initialize_info()
4168 this_thr->th.th_cg_roots->cg_nthreads)); in __kmp_initialize_info()
4169 this_thr->th.th_current_task->td_icvs.thread_limit = in __kmp_initialize_info()
4170 this_thr->th.th_cg_roots->cg_thread_limit; in __kmp_initialize_info()
4175 volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch; in __kmp_initialize_info()
4214 this_thr->th.th_next_pool = NULL; in __kmp_initialize_info()
4216 if (!this_thr->th.th_task_state_memo_stack) { in __kmp_initialize_info()
4218 this_thr->th.th_task_state_memo_stack = in __kmp_initialize_info()
4220 this_thr->th.th_task_state_top = 0; in __kmp_initialize_info()
4221 this_thr->th.th_task_state_stack_sz = 4; in __kmp_initialize_info()
4222 for (i = 0; i < this_thr->th.th_task_state_stack_sz; in __kmp_initialize_info()
4224 this_thr->th.th_task_state_memo_stack[i] = 0; in __kmp_initialize_info()
4227 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here); in __kmp_initialize_info()
4228 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0); in __kmp_initialize_info()
4254 __kmp_thread_pool = (volatile kmp_info_t *)new_thr->th.th_next_pool; in __kmp_allocate_thread()
4258 TCW_4(new_thr->th.th_in_pool, FALSE); in __kmp_allocate_thread()
4261 if (new_thr->th.th_active_in_pool == TRUE) { in __kmp_allocate_thread()
4262 KMP_DEBUG_ASSERT(new_thr->th.th_active == TRUE); in __kmp_allocate_thread()
4264 new_thr->th.th_active_in_pool = FALSE; in __kmp_allocate_thread()
4269 __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid)); in __kmp_allocate_thread()
4270 KMP_ASSERT(!new_thr->th.th_team); in __kmp_allocate_thread()
4275 new_thr->th.th_info.ds.ds_gtid); in __kmp_allocate_thread()
4276 KMP_DEBUG_ASSERT(new_thr->th.th_serial_team); in __kmp_allocate_thread()
4280 new_thr->th.th_task_state = 0; in __kmp_allocate_thread()
4281 new_thr->th.th_task_state_top = 0; in __kmp_allocate_thread()
4282 new_thr->th.th_task_state_stack_sz = 4; in __kmp_allocate_thread()
4298 kmp_balign_t *balign = new_thr->th.th_bar; in __kmp_allocate_thread()
4304 __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid)); in __kmp_allocate_thread()
4371 &new_thr->th.th_sleep_loc, sizeof(new_thr->th.th_sleep_loc)); in __kmp_allocate_thread()
4374 &new_thr->th.th_reap_state, sizeof(new_thr->th.th_reap_state)); in __kmp_allocate_thread()
4378 &new_thr->th.th_suspend_init, sizeof(new_thr->th.th_suspend_init)); in __kmp_allocate_thread()
4382 &new_thr->th.th_suspend_init_count, in __kmp_allocate_thread()
4383 sizeof(new_thr->th.th_suspend_init_count)); in __kmp_allocate_thread()
4388 CCAST(kmp_uint64 *, &new_thr->th.th_bar[0].bb.b_go), in __kmp_allocate_thread()
4389 sizeof(new_thr->th.th_bar[0].bb.b_go)); in __kmp_allocate_thread()
4392 CCAST(kmp_uint64 *, &new_thr->th.th_bar[1].bb.b_go), in __kmp_allocate_thread()
4393 sizeof(new_thr->th.th_bar[1].bb.b_go)); in __kmp_allocate_thread()
4396 CCAST(kmp_uint64 *, &new_thr->th.th_bar[2].bb.b_go), in __kmp_allocate_thread()
4397 sizeof(new_thr->th.th_bar[2].bb.b_go)); in __kmp_allocate_thread()
4407 new_thr->th.th_serial_team = serial_team = in __kmp_allocate_thread()
4431 KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL); in __kmp_allocate_thread()
4443 kmp_balign_t *balign = new_thr->th.th_bar; in __kmp_allocate_thread()
4451 new_thr->th.th_spin_here = FALSE; in __kmp_allocate_thread()
4452 new_thr->th.th_next_waiting = 0; in __kmp_allocate_thread()
4454 new_thr->th.th_blocking = false; in __kmp_allocate_thread()
4458 new_thr->th.th_current_place = KMP_PLACE_UNDEFINED; in __kmp_allocate_thread()
4459 new_thr->th.th_new_place = KMP_PLACE_UNDEFINED; in __kmp_allocate_thread()
4460 new_thr->th.th_first_place = KMP_PLACE_UNDEFINED; in __kmp_allocate_thread()
4461 new_thr->th.th_last_place = KMP_PLACE_UNDEFINED; in __kmp_allocate_thread()
4463 new_thr->th.th_def_allocator = __kmp_def_allocator; in __kmp_allocate_thread()
4464 new_thr->th.th_prev_level = 0; in __kmp_allocate_thread()
4465 new_thr->th.th_prev_num_threads = 1; in __kmp_allocate_thread()
4467 TCW_4(new_thr->th.th_in_pool, FALSE); in __kmp_allocate_thread()
4468 new_thr->th.th_active_in_pool = FALSE; in __kmp_allocate_thread()
4469 TCW_4(new_thr->th.th_active, TRUE); in __kmp_allocate_thread()
4624 int first_place = master_th->th.th_first_place; in __kmp_partition_places()
4625 int last_place = master_th->th.th_last_place; in __kmp_partition_places()
4626 int masters_place = master_th->th.th_current_place; in __kmp_partition_places()
4647 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places() local
4648 KMP_DEBUG_ASSERT(th != NULL); in __kmp_partition_places()
4649 th->th.th_first_place = first_place; in __kmp_partition_places()
4650 th->th.th_last_place = last_place; in __kmp_partition_places()
4651 th->th.th_new_place = masters_place; in __kmp_partition_places()
4652 if (__kmp_display_affinity && masters_place != th->th.th_current_place && in __kmp_partition_places()
4676 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places() local
4677 KMP_DEBUG_ASSERT(th != NULL); in __kmp_partition_places()
4686 th->th.th_first_place = first_place; in __kmp_partition_places()
4687 th->th.th_last_place = last_place; in __kmp_partition_places()
4688 th->th.th_new_place = place; in __kmp_partition_places()
4689 if (__kmp_display_affinity && place != th->th.th_current_place && in __kmp_partition_places()
4708 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places() local
4709 KMP_DEBUG_ASSERT(th != NULL); in __kmp_partition_places()
4711 th->th.th_first_place = first_place; in __kmp_partition_places()
4712 th->th.th_last_place = last_place; in __kmp_partition_places()
4713 th->th.th_new_place = place; in __kmp_partition_places()
4714 if (__kmp_display_affinity && place != th->th.th_current_place && in __kmp_partition_places()
4750 th->th.th_new_place, first_place, last_place)); in __kmp_partition_places()
4781 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places() local
4782 KMP_DEBUG_ASSERT(th != NULL); in __kmp_partition_places()
4784 th->th.th_first_place = place; in __kmp_partition_places()
4785 th->th.th_new_place = place; in __kmp_partition_places()
4786 if (__kmp_display_affinity && place != th->th.th_current_place && in __kmp_partition_places()
4812 th->th.th_last_place = place; in __kmp_partition_places()
4827 f, th->th.th_new_place, th->th.th_first_place, in __kmp_partition_places()
4828 th->th.th_last_place, __kmp_affinity_num_masks)); in __kmp_partition_places()
4838 kmp_info_t *th; in __kmp_partition_places() local
4876 th = team->t.t_threads[f]; in __kmp_partition_places()
4877 KMP_DEBUG_ASSERT(th); in __kmp_partition_places()
4878 th->th.th_first_place = first; in __kmp_partition_places()
4879 th->th.th_new_place = place; in __kmp_partition_places()
4880 th->th.th_last_place = last; in __kmp_partition_places()
4881 if (__kmp_display_affinity && place != th->th.th_current_place && in __kmp_partition_places()
4889 team->t.t_id, f, th->th.th_new_place, in __kmp_partition_places()
4890 th->th.th_first_place, th->th.th_last_place, spacing)); in __kmp_partition_places()
4907 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places() local
4908 KMP_DEBUG_ASSERT(th != NULL); in __kmp_partition_places()
4910 th->th.th_first_place = place; in __kmp_partition_places()
4911 th->th.th_last_place = place; in __kmp_partition_places()
4912 th->th.th_new_place = place; in __kmp_partition_places()
4913 if (__kmp_display_affinity && place != th->th.th_current_place && in __kmp_partition_places()
4948 team->t.t_id, f, th->th.th_new_place, in __kmp_partition_places()
4949 th->th.th_first_place, th->th.th_last_place)); in __kmp_partition_places()
4988 team = master->th.th_team; in __kmp_allocate_team()
4990 if (master->th.th_teams_microtask) { // in teams construct? in __kmp_allocate_team()
4991 if (master->th.th_teams_size.nteams > 1 && in __kmp_allocate_team()
4995 master->th.th_teams_level < in __kmp_allocate_team()
5001 hot_teams = master->th.th_hot_teams; in __kmp_allocate_team()
5049 root->r.r_uber_thread->th.th_ident); in __kmp_allocate_team()
5092 team->t.t_threads[f]->th.th_task_team = NULL; in __kmp_allocate_team()
5104 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar; in __kmp_allocate_team()
5118 root->r.r_uber_thread->th.th_ident); in __kmp_allocate_team()
5122 team->t.t_threads[f]->th.th_team_nproc = new_nproc; in __kmp_allocate_team()
5135 team->t.t_threads[f]->th.th_team_nproc == in __kmp_allocate_team()
5167 kmp_balign_t *balign = other_threads[f]->th.th_bar; in __kmp_allocate_team()
5218 kmp_balign_t *balign = new_worker->th.th_bar; in __kmp_allocate_team()
5244 root->r.r_uber_thread->th.th_ident); in __kmp_allocate_team()
5259 team->t.t_threads[f]->th.th_task_state = in __kmp_allocate_team()
5260 team->t.t_threads[0]->th.th_task_state_memo_stack[level]; in __kmp_allocate_team()
5263 kmp_uint8 old_state = team->t.t_threads[0]->th.th_task_state; in __kmp_allocate_team()
5265 team->t.t_threads[f]->th.th_task_state = old_state; in __kmp_allocate_team()
5271 team->t.t_threads[f]->th.th_team_nproc == in __kmp_allocate_team()
5283 if (master->th.th_teams_microtask) { in __kmp_allocate_team()
5287 thr->th.th_teams_microtask = master->th.th_teams_microtask; in __kmp_allocate_team()
5288 thr->th.th_teams_level = master->th.th_teams_level; in __kmp_allocate_team()
5289 thr->th.th_teams_size = master->th.th_teams_size; in __kmp_allocate_team()
5299 kmp_balign_t *balign = thr->th.th_bar; in __kmp_allocate_team()
5475 if (master->th.th_teams_microtask) { // in teams construct? in __kmp_free_team()
5476 if (master->th.th_teams_size.nteams > 1) { in __kmp_free_team()
5481 master->th.th_teams_level == team->t.t_level) { in __kmp_free_team()
5486 hot_teams = master->th.th_hot_teams; in __kmp_free_team()
5508 kmp_info_t *th = team->t.t_threads[f]; in __kmp_free_team() local
5509 volatile kmp_uint32 *state = &th->th.th_reap_state; in __kmp_free_team()
5514 if (!__kmp_is_thread_alive(th, &ecode)) { in __kmp_free_team()
5520 kmp_flag_64<> fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th); in __kmp_free_team()
5522 fl.resume(__kmp_gtid_from_thread(th)); in __kmp_free_team()
5534 team->t.t_threads[f]->th.th_task_team = NULL; in __kmp_free_team()
5567 team->t.t_threads[1]->th.th_cg_roots); in __kmp_free_team()
5568 if (team->t.t_threads[1]->th.th_cg_roots->cg_root == team->t.t_threads[1]) { in __kmp_free_team()
5572 KMP_DEBUG_ASSERT(thr && thr->th.th_cg_roots && in __kmp_free_team()
5573 thr->th.th_cg_roots->cg_root == thr); in __kmp_free_team()
5575 kmp_cg_root_t *tmp = thr->th.th_cg_roots; in __kmp_free_team()
5576 thr->th.th_cg_roots = tmp->up; in __kmp_free_team()
5579 thr, tmp, thr->th.th_cg_roots, tmp->cg_nthreads)); in __kmp_free_team()
5585 if (thr->th.th_cg_roots) in __kmp_free_team()
5586 thr->th.th_current_task->td_icvs.thread_limit = in __kmp_free_team()
5587 thr->th.th_cg_roots->cg_thread_limit; in __kmp_free_team()
5647 __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid)); in __kmp_free_thread()
5654 kmp_balign_t *balign = this_th->th.th_bar; in __kmp_free_thread()
5661 this_th->th.th_task_state = 0; in __kmp_free_thread()
5662 this_th->th.th_reap_state = KMP_SAFE_TO_REAP; in __kmp_free_thread()
5665 TCW_PTR(this_th->th.th_team, NULL); in __kmp_free_thread()
5666 TCW_PTR(this_th->th.th_root, NULL); in __kmp_free_thread()
5667 TCW_PTR(this_th->th.th_dispatch, NULL); /* NOT NEEDED */ in __kmp_free_thread()
5669 while (this_th->th.th_cg_roots) { in __kmp_free_thread()
5670 this_th->th.th_cg_roots->cg_nthreads--; in __kmp_free_thread()
5673 this_th, this_th->th.th_cg_roots, in __kmp_free_thread()
5674 this_th->th.th_cg_roots->cg_root, in __kmp_free_thread()
5675 this_th->th.th_cg_roots->cg_nthreads)); in __kmp_free_thread()
5676 kmp_cg_root_t *tmp = this_th->th.th_cg_roots; in __kmp_free_thread()
5681 this_th->th.th_cg_roots = tmp->up; in __kmp_free_thread()
5687 this_th->th.th_cg_roots = NULL; in __kmp_free_thread()
5698 this_th->th.th_current_task = NULL; in __kmp_free_thread()
5702 gtid = this_th->th.th_info.ds.ds_gtid; in __kmp_free_thread()
5705 if (__kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid) { in __kmp_free_thread()
5716 scan = &(__kmp_thread_pool_insert_pt->th.th_next_pool); in __kmp_free_thread()
5720 for (; (*scan != NULL) && ((*scan)->th.th_info.ds.ds_gtid < gtid); in __kmp_free_thread()
5721 scan = &((*scan)->th.th_next_pool)) in __kmp_free_thread()
5726 TCW_PTR(this_th->th.th_next_pool, *scan); in __kmp_free_thread()
5728 KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) || in __kmp_free_thread()
5729 (this_th->th.th_info.ds.ds_gtid < in __kmp_free_thread()
5730 this_th->th.th_next_pool->th.th_info.ds.ds_gtid)); in __kmp_free_thread()
5731 TCW_4(this_th->th.th_in_pool, TRUE); in __kmp_free_thread()
5734 if (this_th->th.th_active == TRUE) { in __kmp_free_thread()
5736 this_th->th.th_active_in_pool = TRUE; in __kmp_free_thread()
5740 KMP_DEBUG_ASSERT(this_th->th.th_active_in_pool == FALSE); in __kmp_free_thread()
5771 int gtid = this_thr->th.th_info.ds.ds_gtid; in __kmp_launch_thread()
5779 this_thr->th.th_cons = __kmp_allocate_cons_stack(gtid); // ATT: Memory leak? in __kmp_launch_thread()
5790 thread_data = &(this_thr->th.ompt_thread_info.thread_data); in __kmp_launch_thread()
5793 this_thr->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_launch_thread()
5794 this_thr->th.ompt_thread_info.wait_id = 0; in __kmp_launch_thread()
5795 this_thr->th.ompt_thread_info.idle_frame = OMPT_GET_FRAME_ADDRESS(0); in __kmp_launch_thread()
5796 this_thr->th.ompt_thread_info.parallel_flags = 0; in __kmp_launch_thread()
5801 this_thr->th.ompt_thread_info.state = ompt_state_idle; in __kmp_launch_thread()
5818 this_thr->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_launch_thread()
5822 pteam = &this_thr->th.th_team; in __kmp_launch_thread()
5838 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_launch_thread()
5855 this_thr->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_launch_thread()
5875 this_thr->th.th_task_team = NULL; in __kmp_launch_thread()
5950 gtid = thread->th.th_info.ds.ds_gtid; in __kmp_reap_thread()
5960 kmp_flag_64<> flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, in __kmp_reap_thread()
5978 if (thread->th.th_active_in_pool) { in __kmp_reap_thread()
5979 thread->th.th_active_in_pool = FALSE; in __kmp_reap_thread()
6013 if (thread->th.th_cons) { in __kmp_reap_thread()
6014 __kmp_free_cons_stack(thread->th.th_cons); in __kmp_reap_thread()
6015 thread->th.th_cons = NULL; in __kmp_reap_thread()
6019 if (thread->th.th_pri_common != NULL) { in __kmp_reap_thread()
6020 __kmp_free(thread->th.th_pri_common); in __kmp_reap_thread()
6021 thread->th.th_pri_common = NULL; in __kmp_reap_thread()
6024 if (thread->th.th_task_state_memo_stack != NULL) { in __kmp_reap_thread()
6025 __kmp_free(thread->th.th_task_state_memo_stack); in __kmp_reap_thread()
6026 thread->th.th_task_state_memo_stack = NULL; in __kmp_reap_thread()
6030 if (thread->th.th_local.bget_data != NULL) { in __kmp_reap_thread()
6036 if (thread->th.th_affin_mask != NULL) { in __kmp_reap_thread()
6037 KMP_CPU_FREE(thread->th.th_affin_mask); in __kmp_reap_thread()
6038 thread->th.th_affin_mask = NULL; in __kmp_reap_thread()
6043 if (thread->th.th_hier_bar_data != NULL) { in __kmp_reap_thread()
6044 __kmp_free(thread->th.th_hier_bar_data); in __kmp_reap_thread()
6045 thread->th.th_hier_bar_data = NULL; in __kmp_reap_thread()
6049 __kmp_reap_team(thread->th.th_serial_team); in __kmp_reap_thread()
6050 thread->th.th_serial_team = NULL; in __kmp_reap_thread()
6119 __kmp_thread_pool = thread->th.th_next_pool; in __kmp_internal_end()
6121 KMP_DEBUG_ASSERT(thread->th.th_reap_state == KMP_SAFE_TO_REAP); in __kmp_internal_end()
6122 thread->th.th_next_pool = NULL; in __kmp_internal_end()
6123 thread->th.th_in_pool = FALSE; in __kmp_internal_end()
6147 while (thr && KMP_ATOMIC_LD_ACQ(&thr->th.th_blocking)) in __kmp_internal_end()
6375 __kmp_threads[gtid]->th.th_task_team = NULL; in __kmp_internal_end_thread()
7104 if (thread->th.th_current_task->td_icvs.nproc != 0) in __kmp_do_middle_initialize()
7261 this_thr->th.th_local.this_construct = 0; in __kmp_run_before_invoked_task()
7263 KMP_CACHE_PREFETCH(&this_thr->th.th_bar[bs_forkjoin_barrier].bb.b_arrived); in __kmp_run_before_invoked_task()
7265 dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch); in __kmp_run_before_invoked_task()
7291 kmp_team_t *team = this_thr->th.th_team; in __kmp_invoke_task_func()
7355 this_thr->th.ompt_thread_info.parallel_flags |= ompt_parallel_team; in __kmp_invoke_task_func()
7385 kmp_team_t *team = thr->th.th_team; in __kmp_teams_master()
7387 thr->th.th_set_nproc = thr->th.th_teams_size.nth; in __kmp_teams_master()
7388 KMP_DEBUG_ASSERT(thr->th.th_teams_microtask); in __kmp_teams_master()
7389 KMP_DEBUG_ASSERT(thr->th.th_set_nproc); in __kmp_teams_master()
7391 __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask)); in __kmp_teams_master()
7397 tmp->cg_thread_limit = thr->th.th_current_task->td_icvs.thread_limit; in __kmp_teams_master()
7402 tmp->up = thr->th.th_cg_roots; in __kmp_teams_master()
7403 thr->th.th_cg_roots = tmp; in __kmp_teams_master()
7411 (microtask_t)thr->th.th_teams_microtask, // "wrapped" task in __kmp_teams_master()
7417 if (thr->th.th_team_nproc < thr->th.th_teams_size.nth) in __kmp_teams_master()
7418 thr->th.th_teams_size.nth = thr->th.th_team_nproc; in __kmp_teams_master()
7432 kmp_team_t *team = this_thr->th.th_team; in __kmp_invoke_teams_master()
7434 if (!__kmp_threads[gtid]->th.th_team->t.t_serialized) in __kmp_invoke_teams_master()
7435 KMP_DEBUG_ASSERT((void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn == in __kmp_invoke_teams_master()
7453 this_thr->th.ompt_thread_info.parallel_flags |= ompt_parallel_league; in __kmp_invoke_teams_master()
7468 thr->th.th_set_nproc = num_threads; in __kmp_push_num_threads()
7493 if (num_threads > thr->th.th_current_task->td_icvs.thread_limit) { in __kmp_push_thread_limit()
7494 num_threads = thr->th.th_current_task->td_icvs.thread_limit; in __kmp_push_thread_limit()
7505 thr->th.th_current_task->td_icvs.thread_limit = num_threads; in __kmp_push_thread_limit()
7526 thr->th.th_teams_size.nth = num_threads; in __kmp_push_thread_limit()
7555 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams; in __kmp_push_num_teams()
7612 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams; in __kmp_push_num_teams_51()
7620 thr->th.th_set_proc_bind = proc_bind; in __kmp_push_proc_bind()
7633 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); in __kmp_internal_fork()
7655 KMP_ASSERT(this_thr->th.th_team == team); in __kmp_internal_fork()
7660 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc); in __kmp_internal_fork()
7672 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); in __kmp_internal_join()
7680 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) { in __kmp_internal_join()
7685 gtid, __kmp_threads[gtid]->th.th_team_nproc, team, in __kmp_internal_join()
7690 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc); in __kmp_internal_join()
7696 this_thr->th.ompt_thread_info.state == ompt_state_wait_barrier_implicit) { in __kmp_internal_join()
7697 int ds_tid = this_thr->th.th_info.ds.ds_tid; in __kmp_internal_join()
7699 this_thr->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_internal_join()
7727 KMP_ASSERT(this_thr->th.th_team == team); in __kmp_internal_join()
7752 if (hot_team->t.t_threads[i]->th.th_active) { in __kmp_active_hot_team_nproc()
7772 ->th.th_current_task->td_icvs.dynamic == TRUE); in __kmp_load_balance_nproc()
7962 root = __kmp_threads[gtid]->th.th_root; in __kmp_internal_begin()
7990 root = thread->th.th_root; in __kmp_user_set_library()
8002 thread->th.th_set_nproc = 0; in __kmp_user_set_library()
8006 thread->th.th_set_nproc = 0; in __kmp_user_set_library()
8011 thread->th.th_set_nproc = 0; in __kmp_user_set_library()
8079 if (thr->th.th_teams_microtask) { in __kmp_aux_get_team_info()
8080 kmp_team_t *team = thr->th.th_team; in __kmp_aux_get_team_info()
8081 int tlevel = thr->th.th_teams_level; // the level of the teams construct in __kmp_aux_get_team_info()
8185 static int __kmp_aux_capture_affinity_field(int gtid, const kmp_info_t *th, in __kmp_aux_capture_affinity_field() argument
8196 KMP_DEBUG_ASSERT(th); in __kmp_aux_capture_affinity_field()
8296 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_level); in __kmp_aux_capture_affinity_field()
8314 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_nproc); in __kmp_aux_capture_affinity_field()
8318 __kmp_get_ancestor_thread_num(gtid, th->th.th_team->t.t_level - 1); in __kmp_aux_capture_affinity_field()
8325 __kmp_affinity_str_buf_mask(&buf, th->th.th_affin_mask); in __kmp_aux_capture_affinity_field()
8358 const kmp_info_t *th; in __kmp_aux_capture_affinity() local
8367 th = __kmp_threads[gtid]; in __kmp_aux_capture_affinity()
8382 int rc = __kmp_aux_capture_affinity_field(gtid, th, &parse_ptr, &field); in __kmp_aux_capture_affinity()
8422 set__blocktime_team(thread->th.th_team, tid, blocktime); in __kmp_aux_set_blocktime()
8423 set__blocktime_team(thread->th.th_serial_team, 0, blocktime); in __kmp_aux_set_blocktime()
8429 set__bt_intervals_team(thread->th.th_team, tid, bt_intervals); in __kmp_aux_set_blocktime()
8430 set__bt_intervals_team(thread->th.th_serial_team, 0, bt_intervals); in __kmp_aux_set_blocktime()
8436 set__bt_set_team(thread->th.th_team, tid, bt_set); in __kmp_aux_set_blocktime()
8437 set__bt_set_team(thread->th.th_serial_team, 0, bt_set); in __kmp_aux_set_blocktime()
8441 __kmp_gtid_from_tid(tid, thread->th.th_team), in __kmp_aux_set_blocktime()
8442 thread->th.th_team->t.t_id, tid, blocktime, bt_intervals, in __kmp_aux_set_blocktime()
8446 __kmp_gtid_from_tid(tid, thread->th.th_team), in __kmp_aux_set_blocktime()
8447 thread->th.th_team->t.t_id, tid, blocktime)); in __kmp_aux_set_blocktime()
8618 return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8); in __kmp_get_reduce_method()
8640 kmp_flag_64<> fl(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, in __kmp_resume_if_soft_paused()
8749 __kmp_hidden_helper_main_thread->th.th_set_nproc = in __kmp_hidden_helper_threads_initz_routine()