Lines Matching refs:th
172 stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize); in __kmp_get_global_thread_id()
173 stack_base = (char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase); in __kmp_get_global_thread_id()
214 if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)) { in __kmp_get_global_thread_id()
218 stack_base = (char *)other_threads[i]->th.th_info.ds.ds_stackbase; in __kmp_get_global_thread_id()
220 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr); in __kmp_get_global_thread_id()
221 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize, in __kmp_get_global_thread_id()
222 other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr - in __kmp_get_global_thread_id()
225 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize, in __kmp_get_global_thread_id()
231 char *stack_end = (char *)other_threads[i]->th.th_info.ds.ds_stackbase; in __kmp_get_global_thread_id()
232 char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize; in __kmp_get_global_thread_id()
234 other_threads[i]->th.th_info.ds.ds_stacksize, in __kmp_get_global_thread_id()
283 void __kmp_check_stack_overlap(kmp_info_t *th) { in __kmp_check_stack_overlap() argument
291 stack_end = (char *)th->th.th_info.ds.ds_stackbase; in __kmp_check_stack_overlap()
292 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize; in __kmp_check_stack_overlap()
294 gtid = __kmp_gtid_from_thread(th); in __kmp_check_stack_overlap()
298 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize, in __kmp_check_stack_overlap()
300 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual"); in __kmp_check_stack_overlap()
303 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize, in __kmp_check_stack_overlap()
305 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual"); in __kmp_check_stack_overlap()
311 gtid = __kmp_gtid_from_thread(th); in __kmp_check_stack_overlap()
316 stack_end = (char *)th->th.th_info.ds.ds_stackbase; in __kmp_check_stack_overlap()
317 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize; in __kmp_check_stack_overlap()
323 if (f_th && f_th != th) { in __kmp_check_stack_overlap()
325 (char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase); in __kmp_check_stack_overlap()
327 other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize); in __kmp_check_stack_overlap()
335 (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize), in __kmp_check_stack_overlap()
490 __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team, in __kmp_print_thread_storage_map()
493 __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head, in __kmp_print_thread_storage_map()
497 gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier], in __kmp_print_thread_storage_map()
500 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier], in __kmp_print_thread_storage_map()
501 &thr->th.th_bar[bs_plain_barrier + 1], in __kmp_print_thread_storage_map()
505 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier], in __kmp_print_thread_storage_map()
506 &thr->th.th_bar[bs_forkjoin_barrier + 1], in __kmp_print_thread_storage_map()
511 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier], in __kmp_print_thread_storage_map()
512 &thr->th.th_bar[bs_reduction_barrier + 1], in __kmp_print_thread_storage_map()
639 if (__kmp_threads[gtid]->th.th_root->r.r_active) in __kmp_parallel_deo()
665 if (__kmp_threads[gtid]->th.th_root->r.r_active) in __kmp_parallel_dxo()
686 kmp_info_t *th; in __kmp_enter_single() local
693 th = __kmp_threads[gtid]; in __kmp_enter_single()
694 team = th->th.th_team; in __kmp_enter_single()
697 th->th.th_ident = id_ref; in __kmp_enter_single()
702 kmp_int32 old_this = th->th.th_local.this_construct; in __kmp_enter_single()
704 ++th->th.th_local.this_construct; in __kmp_enter_single()
710 th->th.th_local.this_construct); in __kmp_enter_single()
714 KMP_MASTER_GTID(gtid) && th->th.th_teams_microtask == NULL && in __kmp_enter_single()
848 int cg_nthreads = this_thr->th.th_cg_roots->cg_nthreads; in __kmp_reserve_threads()
849 int max_cg_threads = this_thr->th.th_cg_roots->cg_thread_limit; in __kmp_reserve_threads()
950 master_th->th.th_info.ds.ds_tid = 0; in __kmp_fork_team_threads()
951 master_th->th.th_team = team; in __kmp_fork_team_threads()
952 master_th->th.th_team_nproc = team->t.t_nproc; in __kmp_fork_team_threads()
953 master_th->th.th_team_master = master_th; in __kmp_fork_team_threads()
954 master_th->th.th_team_serialized = FALSE; in __kmp_fork_team_threads()
955 master_th->th.th_dispatch = &team->t.t_dispatch[0]; in __kmp_fork_team_threads()
960 kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams; in __kmp_fork_team_threads()
964 if (master_th->th.th_teams_microtask) { // are we inside the teams? in __kmp_fork_team_threads()
965 if (master_th->th.th_teams_size.nteams > 1) { in __kmp_fork_team_threads()
970 master_th->th.th_teams_level == team->t.t_level) { in __kmp_fork_team_threads()
1005 KMP_DEBUG_ASSERT(thr->th.th_team == team); in __kmp_fork_team_threads()
1013 thr->th.th_teams_microtask = master_th->th.th_teams_microtask; in __kmp_fork_team_threads()
1014 thr->th.th_teams_level = master_th->th.th_teams_level; in __kmp_fork_team_threads()
1015 thr->th.th_teams_size = master_th->th.th_teams_size; in __kmp_fork_team_threads()
1018 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar; in __kmp_fork_team_threads()
1048 if (thr->th.th_prev_num_threads != team->t.t_nproc || in __kmp_fork_team_threads()
1049 thr->th.th_prev_level != team->t.t_level) { in __kmp_fork_team_threads()
1142 serial_team = this_thr->th.th_serial_team; in __kmp_serialized_parallel()
1150 this_thr->th.th_task_team == in __kmp_serialized_parallel()
1151 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]); in __kmp_serialized_parallel()
1152 KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] == in __kmp_serialized_parallel()
1156 global_tid, this_thr->th.th_task_team, this_thr->th.th_team)); in __kmp_serialized_parallel()
1157 this_thr->th.th_task_team = NULL; in __kmp_serialized_parallel()
1160 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind; in __kmp_serialized_parallel()
1161 if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) { in __kmp_serialized_parallel()
1166 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind; in __kmp_serialized_parallel()
1169 this_thr->th.th_set_proc_bind = proc_bind_default; in __kmp_serialized_parallel()
1172 this_thr->th.th_set_nproc = 0; in __kmp_serialized_parallel()
1178 this_thr->th.ompt_thread_info.state != ompt_state_overhead) { in __kmp_serialized_parallel()
1195 if (this_thr->th.th_team != serial_team) { in __kmp_serialized_parallel()
1197 int level = this_thr->th.th_team->t.t_level; in __kmp_serialized_parallel()
1207 __kmp_allocate_team(this_thr->th.th_root, 1, 1, in __kmp_serialized_parallel()
1211 proc_bind, &this_thr->th.th_current_task->td_icvs, in __kmp_serialized_parallel()
1218 new_team->t.t_parent = this_thr->th.th_team; in __kmp_serialized_parallel()
1220 this_thr->th.th_serial_team = serial_team; in __kmp_serialized_parallel()
1240 KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team); in __kmp_serialized_parallel()
1244 serial_team->t.t_parent = this_thr->th.th_team; in __kmp_serialized_parallel()
1245 serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched; in __kmp_serialized_parallel()
1246 this_thr->th.th_team = serial_team; in __kmp_serialized_parallel()
1247 serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid; in __kmp_serialized_parallel()
1250 this_thr->th.th_current_task)); in __kmp_serialized_parallel()
1251 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1); in __kmp_serialized_parallel()
1252 this_thr->th.th_current_task->td_flags.executing = 0; in __kmp_serialized_parallel()
1259 copy_icvs(&this_thr->th.th_current_task->td_icvs, in __kmp_serialized_parallel()
1260 &this_thr->th.th_current_task->td_parent->td_icvs); in __kmp_serialized_parallel()
1265 this_thr->th.th_current_task->td_icvs.nproc = in __kmp_serialized_parallel()
1271 this_thr->th.th_current_task->td_icvs.proc_bind = in __kmp_serialized_parallel()
1278 this_thr->th.th_info.ds.ds_tid = 0; in __kmp_serialized_parallel()
1281 this_thr->th.th_team_nproc = 1; in __kmp_serialized_parallel()
1282 this_thr->th.th_team_master = this_thr; in __kmp_serialized_parallel()
1283 this_thr->th.th_team_serialized = 1; in __kmp_serialized_parallel()
1287 serial_team->t.t_def_allocator = this_thr->th.th_def_allocator; // save in __kmp_serialized_parallel()
1298 this_thr->th.th_dispatch = serial_team->t.t_dispatch; in __kmp_serialized_parallel()
1305 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team); in __kmp_serialized_parallel()
1309 this_thr->th.th_team_serialized = serial_team->t.t_serialized; in __kmp_serialized_parallel()
1312 int level = this_thr->th.th_team->t.t_level; in __kmp_serialized_parallel()
1316 this_thr->th.th_current_task->td_icvs.nproc = in __kmp_serialized_parallel()
1333 this_thr->th.th_dispatch = serial_team->t.t_dispatch; in __kmp_serialized_parallel()
1342 if (this_thr->th.th_prev_level != serial_team->t.t_level || in __kmp_serialized_parallel()
1343 this_thr->th.th_prev_num_threads != 1) { in __kmp_serialized_parallel()
1346 this_thr->th.th_prev_level = serial_team->t.t_level; in __kmp_serialized_parallel()
1347 this_thr->th.th_prev_num_threads = 1; in __kmp_serialized_parallel()
1356 this_thr->th.ompt_thread_info.state != ompt_state_overhead) { in __kmp_serialized_parallel()
1378 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_serialized_parallel()
1389 return (master_th->th.th_teams_microtask && ap && in __kmp_is_fork_in_teams()
1424 if (parent_team == master_th->th.th_serial_team) { in __kmp_fork_in_teams()
1464 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_fork_in_teams()
1501 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_in_teams()
1512 parent_team->t.t_def_allocator = master_th->th.th_def_allocator; // save in __kmp_fork_in_teams()
1519 master_th->th.th_teams_size.nth = parent_team->t.t_nproc; in __kmp_fork_in_teams()
1532 if (master_set_numthreads <= master_th->th.th_teams_size.nth) { in __kmp_fork_in_teams()
1537 int old_proc = master_th->th.th_teams_size.nth; in __kmp_fork_in_teams()
1544 other_threads[i]->th.th_team_nproc = master_set_numthreads; in __kmp_fork_in_teams()
1548 master_th->th.th_set_nproc = 0; in __kmp_fork_in_teams()
1561 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind; in __kmp_fork_in_teams()
1564 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) { in __kmp_fork_in_teams()
1569 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind; in __kmp_fork_in_teams()
1577 master_th->th.th_current_task->td_icvs.proc_bind)) { in __kmp_fork_in_teams()
1584 master_th->th.th_current_task->td_icvs.proc_bind != proc_bind_icv) { in __kmp_fork_in_teams()
1586 for (i = 0; i < master_th->th.th_team_nproc; ++i) { in __kmp_fork_in_teams()
1587 other_threads[i]->th.th_current_task->td_icvs.proc_bind = proc_bind_icv; in __kmp_fork_in_teams()
1591 master_th->th.th_set_proc_bind = proc_bind_default; in __kmp_fork_in_teams()
1598 && master_th->th.th_teams_size.nteams == 1) { in __kmp_fork_in_teams()
1600 master_th->th.th_frame_time = tmp_time; in __kmp_fork_in_teams()
1669 master_th->th.th_serial_team->t.t_pkfn = microtask; in __kmp_serial_fork_call()
1674 master_th->th.th_serial_team->t.t_ident = loc; in __kmp_serial_fork_call()
1677 master_th->th.th_serial_team->t.t_level--; in __kmp_serial_fork_call()
1703 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_serial_fork_call()
1735 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_serial_fork_call()
1739 KMP_DEBUG_ASSERT(master_th->th.th_team == master_th->th.th_serial_team); in __kmp_serial_fork_call()
1740 team = master_th->th.th_team; in __kmp_serial_fork_call()
1773 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_serial_fork_call()
1808 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_serial_fork_call()
1841 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_serial_fork_call()
1921 parent_team = master_th->th.th_team; in __kmp_fork_call()
1922 master_tid = master_th->th.th_info.ds.ds_tid; in __kmp_fork_call()
1923 master_this_cons = master_th->th.th_local.this_construct; in __kmp_fork_call()
1924 root = master_th->th.th_root; in __kmp_fork_call()
1926 master_set_numthreads = master_th->th.th_set_nproc; in __kmp_fork_call()
1928 master_th->th.th_current_task->td_icvs.task_thread_limit; in __kmp_fork_call()
1951 teams_level = master_th->th.th_teams_level; in __kmp_fork_call()
1953 p_hot_teams = &master_th->th.th_hot_teams; in __kmp_fork_call()
1977 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_call()
1981 master_th->th.th_ident = loc; in __kmp_fork_call()
1996 KMP_DEBUG_ASSERT(master_th->th.th_task_team == in __kmp_fork_call()
1997 parent_team->t.t_task_team[master_th->th.th_task_state]); in __kmp_fork_call()
2010 master_th->th.th_current_task->td_icvs.max_active_levels)) || in __kmp_fork_call()
2046 master_th->th.th_set_nproc = 0; in __kmp_fork_call()
2063 master_th->th.th_current_task, in __kmp_fork_call()
2064 master_th->th.th_current_task->td_icvs.max_active_levels)); in __kmp_fork_call()
2068 master_th->th.th_current_task->td_flags.executing = 0; in __kmp_fork_call()
2070 if (!master_th->th.th_teams_microtask || level > teams_level) { in __kmp_fork_call()
2076 int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc; in __kmp_fork_call()
2085 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind; in __kmp_fork_call()
2088 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) { in __kmp_fork_call()
2094 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind; in __kmp_fork_call()
2097 if (master_th->th.th_teams_microtask && in __kmp_fork_call()
2107 master_th->th.th_current_task->td_icvs.proc_bind)) { in __kmp_fork_call()
2110 if (!master_th->th.th_teams_microtask || in __kmp_fork_call()
2117 master_th->th.th_set_proc_bind = proc_bind_default; in __kmp_fork_call()
2121 copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs); in __kmp_fork_call()
2148 &master_th->th.th_current_task->td_icvs, in __kmp_fork_call()
2152 &master_th->th.th_current_task->td_icvs); in __kmp_fork_call()
2169 if (!master_th->th.th_teams_microtask || level > teams_level) { in __kmp_fork_call()
2186 KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator); in __kmp_fork_call()
2198 KMP_DEBUG_ASSERT(master_th->th.th_task_team == in __kmp_fork_call()
2199 parent_team->t.t_task_team[master_th->th.th_task_state]); in __kmp_fork_call()
2203 master_th->th.th_task_team, parent_team, in __kmp_fork_call()
2204 team->t.t_task_team[master_th->th.th_task_state], team)); in __kmp_fork_call()
2206 if (active_level || master_th->th.th_task_team) { in __kmp_fork_call()
2208 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack); in __kmp_fork_call()
2209 if (master_th->th.th_task_state_top >= in __kmp_fork_call()
2210 master_th->th.th_task_state_stack_sz) { // increase size in __kmp_fork_call()
2211 kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz; in __kmp_fork_call()
2215 for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) { in __kmp_fork_call()
2216 new_stack[i] = master_th->th.th_task_state_memo_stack[i]; in __kmp_fork_call()
2218 for (i = master_th->th.th_task_state_stack_sz; i < new_size; in __kmp_fork_call()
2222 old_stack = master_th->th.th_task_state_memo_stack; in __kmp_fork_call()
2223 master_th->th.th_task_state_memo_stack = new_stack; in __kmp_fork_call()
2224 master_th->th.th_task_state_stack_sz = new_size; in __kmp_fork_call()
2228 master_th->th in __kmp_fork_call()
2229 .th_task_state_memo_stack[master_th->th.th_task_state_top] = in __kmp_fork_call()
2230 master_th->th.th_task_state; in __kmp_fork_call()
2231 master_th->th.th_task_state_top++; in __kmp_fork_call()
2233 if (master_th->th.th_hot_teams && in __kmp_fork_call()
2235 team == master_th->th.th_hot_teams[active_level].hot_team) { in __kmp_fork_call()
2237 master_th->th.th_task_state = in __kmp_fork_call()
2238 master_th->th in __kmp_fork_call()
2239 .th_task_state_memo_stack[master_th->th.th_task_state_top]; in __kmp_fork_call()
2242 master_th->th.th_task_state = 0; in __kmp_fork_call()
2248 KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) || in __kmp_fork_call()
2286 &master_th->th.th_current_task->td_icvs, loc); in __kmp_fork_call()
2289 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_fork_call()
2296 && !master_th->th.th_teams_microtask) { // not in teams construct in __kmp_fork_call()
2305 master_th->th.th_frame_time = tmp_time; in __kmp_fork_call()
2320 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team); in __kmp_fork_call()
2388 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_call()
2399 thread->th.ompt_thread_info.state = in __kmp_join_restore_state()
2436 root = master_th->th.th_root; in __kmp_join_call()
2437 team = master_th->th.th_team; in __kmp_join_call()
2440 master_th->th.th_ident = loc; in __kmp_join_call()
2449 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_join_call()
2458 team->t.t_task_team[master_th->th.th_task_state], in __kmp_join_call()
2459 master_th->th.th_task_team)); in __kmp_join_call()
2460 KMP_DEBUG_ASSERT(master_th->th.th_task_team == in __kmp_join_call()
2461 team->t.t_task_team[master_th->th.th_task_state]); in __kmp_join_call()
2466 if (master_th->th.th_teams_microtask) { in __kmp_join_call()
2469 int tlevel = master_th->th.th_teams_level; in __kmp_join_call()
2510 master_th->th.th_task_state = in __kmp_join_call()
2534 (!master_th->th.th_teams_microtask || /* not in teams construct */ in __kmp_join_call()
2535 master_th->th.th_teams_size.nteams == 1)) { in __kmp_join_call()
2536 master_th->th.th_ident = loc; in __kmp_join_call()
2542 master_th->th.th_frame_time, 0, loc, in __kmp_join_call()
2543 master_th->th.th_team_nproc, 1); in __kmp_join_call()
2553 master_th->th.th_first_place = team->t.t_first_place; in __kmp_join_call()
2554 master_th->th.th_last_place = team->t.t_last_place; in __kmp_join_call()
2558 if (master_th->th.th_teams_microtask && !exit_teams && in __kmp_join_call()
2560 team->t.t_level == master_th->th.th_teams_level + 1) { in __kmp_join_call()
2589 if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) { in __kmp_join_call()
2590 int old_num = master_th->th.th_team_nproc; in __kmp_join_call()
2591 int new_num = master_th->th.th_teams_size.nth; in __kmp_join_call()
2595 other_threads[i]->th.th_team_nproc = new_num; in __kmp_join_call()
2601 kmp_balign_t *balign = other_threads[i]->th.th_bar; in __kmp_join_call()
2611 other_threads[i]->th.th_task_state = master_th->th.th_task_state; in __kmp_join_call()
2627 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid; in __kmp_join_call()
2628 master_th->th.th_local.this_construct = team->t.t_master_this_cons; in __kmp_join_call()
2630 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid]; in __kmp_join_call()
2637 if (!master_th->th.th_teams_microtask || in __kmp_join_call()
2638 team->t.t_level > master_th->th.th_teams_level) { in __kmp_join_call()
2665 master_th->th.th_def_allocator = team->t.t_def_allocator; in __kmp_join_call()
2684 master_th->th.th_team = parent_team; in __kmp_join_call()
2685 master_th->th.th_team_nproc = parent_team->t.t_nproc; in __kmp_join_call()
2686 master_th->th.th_team_master = parent_team->t.t_threads[0]; in __kmp_join_call()
2687 master_th->th.th_team_serialized = parent_team->t.t_serialized; in __kmp_join_call()
2691 parent_team != master_th->th.th_serial_team && in __kmp_join_call()
2694 master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL)); in __kmp_join_call()
2695 master_th->th.th_serial_team = parent_team; in __kmp_join_call()
2699 if (master_th->th.th_task_state_top > in __kmp_join_call()
2701 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack); in __kmp_join_call()
2703 master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] = in __kmp_join_call()
2704 master_th->th.th_task_state; in __kmp_join_call()
2705 --master_th->th.th_task_state_top; // pop in __kmp_join_call()
2707 master_th->th.th_task_state = in __kmp_join_call()
2708 master_th->th in __kmp_join_call()
2709 .th_task_state_memo_stack[master_th->th.th_task_state_top]; in __kmp_join_call()
2715 master_th->th.th_task_state = 0; in __kmp_join_call()
2718 master_th->th.th_task_team = in __kmp_join_call()
2719 parent_team->t.t_task_team[master_th->th.th_task_state]; in __kmp_join_call()
2722 __kmp_gtid_from_thread(master_th), master_th->th.th_task_team, in __kmp_join_call()
2729 master_th->th.th_current_task->td_flags.executing = 1; in __kmp_join_call()
2734 if (master_th->th.th_team->t.t_level == 0 && __kmp_affinity.flags.reset) { in __kmp_join_call()
2757 if (thread->th.th_team != thread->th.th_serial_team) { in __kmp_save_internal_controls()
2760 if (thread->th.th_team->t.t_serialized > 1) { in __kmp_save_internal_controls()
2763 if (thread->th.th_team->t.t_control_stack_top == NULL) { in __kmp_save_internal_controls()
2766 if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level != in __kmp_save_internal_controls()
2767 thread->th.th_team->t.t_serialized) { in __kmp_save_internal_controls()
2776 copy_icvs(control, &thread->th.th_current_task->td_icvs); in __kmp_save_internal_controls()
2778 control->serial_nesting_level = thread->th.th_team->t.t_serialized; in __kmp_save_internal_controls()
2780 control->next = thread->th.th_team->t.t_control_stack_top; in __kmp_save_internal_controls()
2781 thread->th.th_team->t.t_control_stack_top = control; in __kmp_save_internal_controls()
2801 if (thread->th.th_current_task->td_icvs.nproc == new_nth) in __kmp_set_num_threads()
2811 root = thread->th.th_root; in __kmp_set_num_threads()
2832 hot_team->t.t_threads[f]->th.th_task_team = NULL; in __kmp_set_num_threads()
2839 if (thread->th.th_hot_teams) { in __kmp_set_num_threads()
2840 KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team); in __kmp_set_num_threads()
2841 thread->th.th_hot_teams[0].hot_team_nth = new_nth; in __kmp_set_num_threads()
2855 hot_team->t.t_threads[f]->th.th_team_nproc = new_nth; in __kmp_set_num_threads()
2915 KMP_DEBUG_ASSERT(thread->th.th_current_task); in __kmp_get_max_active_levels()
2918 gtid, thread->th.th_current_task, in __kmp_get_max_active_levels()
2919 thread->th.th_current_task->td_icvs.max_active_levels)); in __kmp_get_max_active_levels()
2920 return thread->th.th_current_task->td_icvs.max_active_levels; in __kmp_get_max_active_levels()
2974 thread->th.th_current_task->td_icvs.sched.r_sched_type = kmp_sch_static; in __kmp_set_schedule()
2976 thread->th.th_current_task->td_icvs.sched.r_sched_type = in __kmp_set_schedule()
2982 thread->th.th_current_task->td_icvs.sched.r_sched_type = in __kmp_set_schedule()
2987 orig_kind, &(thread->th.th_current_task->td_icvs.sched.r_sched_type)); in __kmp_set_schedule()
2990 thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK; in __kmp_set_schedule()
2992 thread->th.th_current_task->td_icvs.sched.chunk = chunk; in __kmp_set_schedule()
3006 th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type; in __kmp_get_schedule()
3042 *chunk = thread->th.th_current_task->td_icvs.sched.chunk; in __kmp_get_schedule()
3060 team = thr->th.th_team; in __kmp_get_ancestor_thread_num()
3065 if (thr->th.th_teams_microtask) { in __kmp_get_ancestor_thread_num()
3067 int tlevel = thr->th.th_teams_level; // the level of the teams construct in __kmp_get_ancestor_thread_num()
3118 team = thr->th.th_team; in __kmp_get_team_size()
3123 if (thr->th.th_teams_microtask) { in __kmp_get_team_size()
3125 int tlevel = thr->th.th_teams_level; // the level of the teams construct in __kmp_get_team_size()
3334 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs); in __kmp_get_x_global_icvs()
3500 __kmp_printf("%2d %p\n", thread->th.th_info.ds.ds_gtid, thread); in __kmp_print_structure_thread()
3541 __kmp_printf(" Our Root: %p\n", thread->th.th_root); in __kmp_print_structure()
3542 __kmp_print_structure_team(" Our Team: ", thread->th.th_team); in __kmp_print_structure()
3544 thread->th.th_serial_team); in __kmp_print_structure()
3545 __kmp_printf(" Threads: %2d\n", thread->th.th_team_nproc); in __kmp_print_structure()
3547 thread->th.th_team_master); in __kmp_print_structure()
3548 __kmp_printf(" Serialized?: %2d\n", thread->th.th_team_serialized); in __kmp_print_structure()
3549 __kmp_printf(" Set NProc: %2d\n", thread->th.th_set_nproc); in __kmp_print_structure()
3550 __kmp_printf(" Set Proc Bind: %2d\n", thread->th.th_set_proc_bind); in __kmp_print_structure()
3552 thread->th.th_next_pool); in __kmp_print_structure()
3554 __kmp_print_structure_team_accum(list, thread->th.th_team); in __kmp_print_structure()
3555 __kmp_print_structure_team_accum(list, thread->th.th_serial_team); in __kmp_print_structure()
3645 unsigned x = thread->th.th_x; in __kmp_get_random()
3648 thread->th.th_x = x * thread->th.th_a + 1; in __kmp_get_random()
3651 thread->th.th_info.ds.ds_tid, r)); in __kmp_get_random()
3658 unsigned seed = thread->th.th_info.ds.ds_tid; in __kmp_init_random()
3660 thread->th.th_a = in __kmp_init_random()
3662 thread->th.th_x = (seed + 1) * thread->th.th_a + 1; in __kmp_init_random()
3664 ("__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a)); in __kmp_init_random()
3927 root_thread->th.th_info.ds.ds_gtid = gtid; in __kmp_register_root()
3929 root_thread->th.ompt_thread_info.thread_data = ompt_data_none; in __kmp_register_root()
3931 root_thread->th.th_root = root; in __kmp_register_root()
3933 root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid); in __kmp_register_root()
3940 KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL); in __kmp_register_root()
3947 if (!root_thread->th.th_serial_team) { in __kmp_register_root()
3950 root_thread->th.th_serial_team = __kmp_allocate_team( in __kmp_register_root()
3957 KMP_ASSERT(root_thread->th.th_serial_team); in __kmp_register_root()
3959 root_thread->th.th_serial_team)); in __kmp_register_root()
3966 root_thread->th.th_serial_team->t.t_threads[0] = root_thread; in __kmp_register_root()
3968 root_thread->th.th_serial_team->t.t_serialized = 0; in __kmp_register_root()
3996 root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE; in __kmp_register_root()
3998 root_thread->th.th_bar[b].bb.b_worker_arrived = 0; in __kmp_register_root()
4006 root_thread->th.th_current_place = KMP_PLACE_UNDEFINED; in __kmp_register_root()
4007 root_thread->th.th_new_place = KMP_PLACE_UNDEFINED; in __kmp_register_root()
4008 root_thread->th.th_first_place = KMP_PLACE_UNDEFINED; in __kmp_register_root()
4009 root_thread->th.th_last_place = KMP_PLACE_UNDEFINED; in __kmp_register_root()
4011 root_thread->th.th_def_allocator = __kmp_def_allocator; in __kmp_register_root()
4012 root_thread->th.th_prev_level = 0; in __kmp_register_root()
4013 root_thread->th.th_prev_num_threads = 1; in __kmp_register_root()
4023 root_thread->th.th_cg_roots = tmp; in __kmp_register_root()
4065 kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams; in __kmp_free_hot_teams()
4075 kmp_info_t *th = team->t.t_threads[i]; in __kmp_free_hot_teams() local
4076 n += __kmp_free_hot_teams(root, th, level + 1, max_level); in __kmp_free_hot_teams()
4077 if (i > 0 && th->th.th_hot_teams) { in __kmp_free_hot_teams()
4078 __kmp_free(th->th.th_hot_teams); in __kmp_free_hot_teams()
4079 th->th.th_hot_teams = NULL; in __kmp_free_hot_teams()
4107 kmp_info_t *th = hot_team->t.t_threads[i]; in __kmp_reset_root() local
4109 n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level); in __kmp_reset_root()
4111 if (th->th.th_hot_teams) { in __kmp_reset_root()
4112 __kmp_free(th->th.th_hot_teams); in __kmp_reset_root()
4113 th->th.th_hot_teams = NULL; in __kmp_reset_root()
4132 (LPVOID) & (root->r.r_uber_thread->th), in __kmp_reset_root()
4133 root->r.r_uber_thread->th.th_info.ds.ds_thread)); in __kmp_reset_root()
4134 __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread); in __kmp_reset_root()
4153 &(root->r.r_uber_thread->th.ompt_thread_info.thread_data)); in __kmp_reset_root()
4159 i = root->r.r_uber_thread->th.th_cg_roots->cg_nthreads--; in __kmp_reset_root()
4162 root->r.r_uber_thread, root->r.r_uber_thread->th.th_cg_roots, in __kmp_reset_root()
4163 root->r.r_uber_thread->th.th_cg_roots->cg_nthreads)); in __kmp_reset_root()
4167 root->r.r_uber_thread->th.th_cg_roots->cg_root); in __kmp_reset_root()
4168 KMP_DEBUG_ASSERT(root->r.r_uber_thread->th.th_cg_roots->up == NULL); in __kmp_reset_root()
4169 __kmp_free(root->r.r_uber_thread->th.th_cg_roots); in __kmp_reset_root()
4170 root->r.r_uber_thread->th.th_cg_roots = NULL; in __kmp_reset_root()
4200 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root); in __kmp_unregister_root_current_thread()
4206 kmp_team_t *team = thread->th.th_team; in __kmp_unregister_root_current_thread()
4207 kmp_task_team_t *task_team = thread->th.th_task_team; in __kmp_unregister_root_current_thread()
4214 thread->th.ompt_thread_info.state = ompt_state_undefined; in __kmp_unregister_root_current_thread()
4239 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root); in __kmp_unregister_root_other_thread()
4255 kmp_team_t *steam = this_thr->th.th_serial_team; in __kmp_task_info()
4256 kmp_team_t *team = this_thr->th.th_team; in __kmp_task_info()
4261 gtid, tid, this_thr, team, steam, this_thr->th.th_current_task, in __kmp_task_info()
4275 KMP_DEBUG_ASSERT(this_thr->th.th_serial_team); in __kmp_initialize_info()
4281 KMP_DEBUG_ASSERT(master->th.th_root); in __kmp_initialize_info()
4285 TCW_SYNC_PTR(this_thr->th.th_team, team); in __kmp_initialize_info()
4287 this_thr->th.th_info.ds.ds_tid = tid; in __kmp_initialize_info()
4288 this_thr->th.th_set_nproc = 0; in __kmp_initialize_info()
4292 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP; in __kmp_initialize_info()
4294 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP; in __kmp_initialize_info()
4295 this_thr->th.th_set_proc_bind = proc_bind_default; in __kmp_initialize_info()
4297 this_thr->th.th_new_place = this_thr->th.th_current_place; in __kmp_initialize_info()
4299 this_thr->th.th_root = master->th.th_root; in __kmp_initialize_info()
4302 this_thr->th.th_team_nproc = team->t.t_nproc; in __kmp_initialize_info()
4303 this_thr->th.th_team_master = master; in __kmp_initialize_info()
4304 this_thr->th.th_team_serialized = team->t.t_serialized; in __kmp_initialize_info()
4309 tid, gtid, this_thr, this_thr->th.th_current_task)); in __kmp_initialize_info()
4311 __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr, in __kmp_initialize_info()
4315 tid, gtid, this_thr, this_thr->th.th_current_task)); in __kmp_initialize_info()
4320 this_thr->th.th_dispatch = &team->t.t_dispatch[tid]; in __kmp_initialize_info()
4322 this_thr->th.th_local.this_construct = 0; in __kmp_initialize_info()
4324 if (!this_thr->th.th_pri_common) { in __kmp_initialize_info()
4325 this_thr->th.th_pri_common = in __kmp_initialize_info()
4329 gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1, in __kmp_initialize_info()
4332 this_thr->th.th_pri_head = NULL; in __kmp_initialize_info()
4336 this_thr->th.th_cg_roots != master->th.th_cg_roots) { // CG root not set in __kmp_initialize_info()
4338 KMP_DEBUG_ASSERT(master->th.th_cg_roots); in __kmp_initialize_info()
4339 kmp_cg_root_t *tmp = this_thr->th.th_cg_roots; in __kmp_initialize_info()
4350 this_thr->th.th_cg_roots = master->th.th_cg_roots; in __kmp_initialize_info()
4352 this_thr->th.th_cg_roots->cg_nthreads++; in __kmp_initialize_info()
4355 this_thr, this_thr->th.th_cg_roots, in __kmp_initialize_info()
4356 this_thr->th.th_cg_roots->cg_root, in __kmp_initialize_info()
4357 this_thr->th.th_cg_roots->cg_nthreads)); in __kmp_initialize_info()
4358 this_thr->th.th_current_task->td_icvs.thread_limit = in __kmp_initialize_info()
4359 this_thr->th.th_cg_roots->cg_thread_limit; in __kmp_initialize_info()
4364 volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch; in __kmp_initialize_info()
4403 this_thr->th.th_next_pool = NULL; in __kmp_initialize_info()
4405 if (!this_thr->th.th_task_state_memo_stack) { in __kmp_initialize_info()
4407 this_thr->th.th_task_state_memo_stack = in __kmp_initialize_info()
4409 this_thr->th.th_task_state_top = 0; in __kmp_initialize_info()
4410 this_thr->th.th_task_state_stack_sz = 4; in __kmp_initialize_info()
4411 for (i = 0; i < this_thr->th.th_task_state_stack_sz; in __kmp_initialize_info()
4413 this_thr->th.th_task_state_memo_stack[i] = 0; in __kmp_initialize_info()
4416 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here); in __kmp_initialize_info()
4417 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0); in __kmp_initialize_info()
4443 __kmp_thread_pool = (volatile kmp_info_t *)new_thr->th.th_next_pool; in __kmp_allocate_thread()
4447 TCW_4(new_thr->th.th_in_pool, FALSE); in __kmp_allocate_thread()
4450 if (new_thr->th.th_active_in_pool == TRUE) { in __kmp_allocate_thread()
4451 KMP_DEBUG_ASSERT(new_thr->th.th_active == TRUE); in __kmp_allocate_thread()
4453 new_thr->th.th_active_in_pool = FALSE; in __kmp_allocate_thread()
4458 __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid)); in __kmp_allocate_thread()
4459 KMP_ASSERT(!new_thr->th.th_team); in __kmp_allocate_thread()
4464 new_thr->th.th_info.ds.ds_gtid); in __kmp_allocate_thread()
4465 KMP_DEBUG_ASSERT(new_thr->th.th_serial_team); in __kmp_allocate_thread()
4469 new_thr->th.th_task_state = 0; in __kmp_allocate_thread()
4470 new_thr->th.th_task_state_top = 0; in __kmp_allocate_thread()
4471 new_thr->th.th_task_state_stack_sz = 4; in __kmp_allocate_thread()
4475 KMP_DEBUG_ASSERT(new_thr->th.th_used_in_team.load() == 0); in __kmp_allocate_thread()
4493 kmp_balign_t *balign = new_thr->th.th_bar; in __kmp_allocate_thread()
4499 __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid)); in __kmp_allocate_thread()
4566 &new_thr->th.th_sleep_loc, sizeof(new_thr->th.th_sleep_loc)); in __kmp_allocate_thread()
4569 &new_thr->th.th_reap_state, sizeof(new_thr->th.th_reap_state)); in __kmp_allocate_thread()
4573 &new_thr->th.th_suspend_init, sizeof(new_thr->th.th_suspend_init)); in __kmp_allocate_thread()
4577 &new_thr->th.th_suspend_init_count, in __kmp_allocate_thread()
4578 sizeof(new_thr->th.th_suspend_init_count)); in __kmp_allocate_thread()
4583 CCAST(kmp_uint64 *, &new_thr->th.th_bar[0].bb.b_go), in __kmp_allocate_thread()
4584 sizeof(new_thr->th.th_bar[0].bb.b_go)); in __kmp_allocate_thread()
4587 CCAST(kmp_uint64 *, &new_thr->th.th_bar[1].bb.b_go), in __kmp_allocate_thread()
4588 sizeof(new_thr->th.th_bar[1].bb.b_go)); in __kmp_allocate_thread()
4591 CCAST(kmp_uint64 *, &new_thr->th.th_bar[2].bb.b_go), in __kmp_allocate_thread()
4592 sizeof(new_thr->th.th_bar[2].bb.b_go)); in __kmp_allocate_thread()
4602 new_thr->th.th_serial_team = serial_team = in __kmp_allocate_thread()
4626 KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL); in __kmp_allocate_thread()
4638 kmp_balign_t *balign = new_thr->th.th_bar; in __kmp_allocate_thread()
4646 TCW_PTR(new_thr->th.th_sleep_loc, NULL); in __kmp_allocate_thread()
4647 new_thr->th.th_sleep_loc_type = flag_unset; in __kmp_allocate_thread()
4649 new_thr->th.th_spin_here = FALSE; in __kmp_allocate_thread()
4650 new_thr->th.th_next_waiting = 0; in __kmp_allocate_thread()
4652 new_thr->th.th_blocking = false; in __kmp_allocate_thread()
4656 new_thr->th.th_current_place = KMP_PLACE_UNDEFINED; in __kmp_allocate_thread()
4657 new_thr->th.th_new_place = KMP_PLACE_UNDEFINED; in __kmp_allocate_thread()
4658 new_thr->th.th_first_place = KMP_PLACE_UNDEFINED; in __kmp_allocate_thread()
4659 new_thr->th.th_last_place = KMP_PLACE_UNDEFINED; in __kmp_allocate_thread()
4661 new_thr->th.th_def_allocator = __kmp_def_allocator; in __kmp_allocate_thread()
4662 new_thr->th.th_prev_level = 0; in __kmp_allocate_thread()
4663 new_thr->th.th_prev_num_threads = 1; in __kmp_allocate_thread()
4665 TCW_4(new_thr->th.th_in_pool, FALSE); in __kmp_allocate_thread()
4666 new_thr->th.th_active_in_pool = FALSE; in __kmp_allocate_thread()
4667 TCW_4(new_thr->th.th_active, TRUE); in __kmp_allocate_thread()
4795 static inline void __kmp_set_thread_place(kmp_team_t *team, kmp_info_t *th, in __kmp_set_thread_place() argument
4797 th->th.th_first_place = first; in __kmp_set_thread_place()
4798 th->th.th_last_place = last; in __kmp_set_thread_place()
4799 th->th.th_new_place = newp; in __kmp_set_thread_place()
4800 if (newp != th->th.th_current_place) { in __kmp_set_thread_place()
4804 th->th.th_topology_ids = __kmp_affinity.ids[th->th.th_new_place]; in __kmp_set_thread_place()
4805 th->th.th_topology_attrs = __kmp_affinity.attrs[th->th.th_new_place]; in __kmp_set_thread_place()
4821 int first_place = master_th->th.th_first_place; in __kmp_partition_places()
4822 int last_place = master_th->th.th_last_place; in __kmp_partition_places()
4823 int masters_place = master_th->th.th_current_place; in __kmp_partition_places()
4845 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places() local
4846 KMP_DEBUG_ASSERT(th != NULL); in __kmp_partition_places()
4847 __kmp_set_thread_place(team, th, first_place, last_place, masters_place); in __kmp_partition_places()
4868 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places() local
4869 KMP_DEBUG_ASSERT(th != NULL); in __kmp_partition_places()
4878 __kmp_set_thread_place(team, th, first_place, last_place, place); in __kmp_partition_places()
4894 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places() local
4895 KMP_DEBUG_ASSERT(th != NULL); in __kmp_partition_places()
4897 __kmp_set_thread_place(team, th, first_place, last_place, place); in __kmp_partition_places()
4930 th->th.th_new_place, first_place, last_place)); in __kmp_partition_places()
4961 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places() local
4962 KMP_DEBUG_ASSERT(th != NULL); in __kmp_partition_places()
4987 __kmp_set_thread_place(team, th, fplace, place, nplace); in __kmp_partition_places()
5002 f, th->th.th_new_place, th->th.th_first_place, in __kmp_partition_places()
5003 th->th.th_last_place, num_masks)); in __kmp_partition_places()
5013 kmp_info_t *th; in __kmp_partition_places() local
5051 th = team->t.t_threads[f]; in __kmp_partition_places()
5052 KMP_DEBUG_ASSERT(th); in __kmp_partition_places()
5053 __kmp_set_thread_place(team, th, first, last, place); in __kmp_partition_places()
5058 team->t.t_id, f, th->th.th_new_place, in __kmp_partition_places()
5059 th->th.th_first_place, th->th.th_last_place, spacing)); in __kmp_partition_places()
5076 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places() local
5077 KMP_DEBUG_ASSERT(th != NULL); in __kmp_partition_places()
5079 __kmp_set_thread_place(team, th, place, place, place); in __kmp_partition_places()
5111 team->t.t_id, f, th->th.th_new_place, in __kmp_partition_places()
5112 th->th.th_first_place, th->th.th_last_place)); in __kmp_partition_places()
5152 team = master->th.th_team; in __kmp_allocate_team()
5154 if (master->th.th_teams_microtask) { // in teams construct? in __kmp_allocate_team()
5155 if (master->th.th_teams_size.nteams > 1 && in __kmp_allocate_team()
5159 master->th.th_teams_level < in __kmp_allocate_team()
5166 if ((master->th.th_teams_size.nteams == 1 && in __kmp_allocate_team()
5167 master->th.th_teams_level >= team->t.t_level) || in __kmp_allocate_team()
5171 hot_teams = master->th.th_hot_teams; in __kmp_allocate_team()
5230 root->r.r_uber_thread->th.th_ident); in __kmp_allocate_team()
5282 team->t.t_threads[f]->th.th_task_team = NULL; in __kmp_allocate_team()
5294 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar; in __kmp_allocate_team()
5308 root->r.r_uber_thread->th.th_ident); in __kmp_allocate_team()
5312 team->t.t_threads[f]->th.th_team_nproc = new_nproc; in __kmp_allocate_team()
5325 team->t.t_threads[f]->th.th_team_nproc == in __kmp_allocate_team()
5353 kmp_balign_t *balign = other_threads[f]->th.th_bar; in __kmp_allocate_team()
5403 kmp_balign_t *balign = new_worker->th.th_bar; in __kmp_allocate_team()
5430 root->r.r_uber_thread->th.th_ident); in __kmp_allocate_team()
5439 kmp_uint8 old_state = team->t.t_threads[old_nproc - 1]->th.th_task_state; in __kmp_allocate_team()
5441 team->t.t_threads[f]->th.th_task_state = old_state; in __kmp_allocate_team()
5446 team->t.t_threads[f]->th.th_team_nproc == in __kmp_allocate_team()
5460 if (master->th.th_teams_microtask) { in __kmp_allocate_team()
5464 thr->th.th_teams_microtask = master->th.th_teams_microtask; in __kmp_allocate_team()
5465 thr->th.th_teams_level = master->th.th_teams_level; in __kmp_allocate_team()
5466 thr->th.th_teams_size = master->th.th_teams_size; in __kmp_allocate_team()
5476 kmp_balign_t *balign = thr->th.th_bar; in __kmp_allocate_team()
5664 if (master->th.th_teams_microtask) { // in teams construct? in __kmp_free_team()
5665 if (master->th.th_teams_size.nteams > 1) { in __kmp_free_team()
5670 master->th.th_teams_level == team->t.t_level) { in __kmp_free_team()
5676 kmp_hot_team_ptr_t *hot_teams = master->th.th_hot_teams; in __kmp_free_team()
5699 kmp_info_t *th = team->t.t_threads[f]; in __kmp_free_team() local
5700 volatile kmp_uint32 *state = &th->th.th_reap_state; in __kmp_free_team()
5705 if (!__kmp_is_thread_alive(th, &ecode)) { in __kmp_free_team()
5711 kmp_flag_64<> fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th); in __kmp_free_team()
5713 fl.resume(__kmp_gtid_from_thread(th)); in __kmp_free_team()
5725 team->t.t_threads[f]->th.th_task_team = NULL; in __kmp_free_team()
5748 KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team), in __kmp_free_team()
5762 team->t.t_threads[f]->th.th_info.ds.ds_gtid, in __kmp_free_team()
5769 while (team->t.t_threads[f]->th.th_used_in_team.load() != 0) in __kmp_free_team()
5791 team->t.t_threads[1]->th.th_cg_roots); in __kmp_free_team()
5792 if (team->t.t_threads[1]->th.th_cg_roots->cg_root == team->t.t_threads[1]) { in __kmp_free_team()
5796 KMP_DEBUG_ASSERT(thr && thr->th.th_cg_roots && in __kmp_free_team()
5797 thr->th.th_cg_roots->cg_root == thr); in __kmp_free_team()
5799 kmp_cg_root_t *tmp = thr->th.th_cg_roots; in __kmp_free_team()
5800 thr->th.th_cg_roots = tmp->up; in __kmp_free_team()
5803 thr, tmp, thr->th.th_cg_roots, tmp->cg_nthreads)); in __kmp_free_team()
5809 if (thr->th.th_cg_roots) in __kmp_free_team()
5810 thr->th.th_current_task->td_icvs.thread_limit = in __kmp_free_team()
5811 thr->th.th_cg_roots->cg_thread_limit; in __kmp_free_team()
5871 __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid)); in __kmp_free_thread()
5878 kmp_balign_t *balign = this_th->th.th_bar; in __kmp_free_thread()
5885 this_th->th.th_task_state = 0; in __kmp_free_thread()
5886 this_th->th.th_reap_state = KMP_SAFE_TO_REAP; in __kmp_free_thread()
5889 TCW_PTR(this_th->th.th_team, NULL); in __kmp_free_thread()
5890 TCW_PTR(this_th->th.th_root, NULL); in __kmp_free_thread()
5891 TCW_PTR(this_th->th.th_dispatch, NULL); /* NOT NEEDED */ in __kmp_free_thread()
5893 while (this_th->th.th_cg_roots) { in __kmp_free_thread()
5894 this_th->th.th_cg_roots->cg_nthreads--; in __kmp_free_thread()
5897 this_th, this_th->th.th_cg_roots, in __kmp_free_thread()
5898 this_th->th.th_cg_roots->cg_root, in __kmp_free_thread()
5899 this_th->th.th_cg_roots->cg_nthreads)); in __kmp_free_thread()
5900 kmp_cg_root_t *tmp = this_th->th.th_cg_roots; in __kmp_free_thread()
5905 this_th->th.th_cg_roots = tmp->up; in __kmp_free_thread()
5911 this_th->th.th_cg_roots = NULL; in __kmp_free_thread()
5922 this_th->th.th_current_task = NULL; in __kmp_free_thread()
5926 gtid = this_th->th.th_info.ds.ds_gtid; in __kmp_free_thread()
5929 if (__kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid) { in __kmp_free_thread()
5940 scan = &(__kmp_thread_pool_insert_pt->th.th_next_pool); in __kmp_free_thread()
5944 for (; (*scan != NULL) && ((*scan)->th.th_info.ds.ds_gtid < gtid); in __kmp_free_thread()
5945 scan = &((*scan)->th.th_next_pool)) in __kmp_free_thread()
5950 TCW_PTR(this_th->th.th_next_pool, *scan); in __kmp_free_thread()
5952 KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) || in __kmp_free_thread()
5953 (this_th->th.th_info.ds.ds_gtid < in __kmp_free_thread()
5954 this_th->th.th_next_pool->th.th_info.ds.ds_gtid)); in __kmp_free_thread()
5955 TCW_4(this_th->th.th_in_pool, TRUE); in __kmp_free_thread()
5958 if (this_th->th.th_active == TRUE) { in __kmp_free_thread()
5960 this_th->th.th_active_in_pool = TRUE; in __kmp_free_thread()
5964 KMP_DEBUG_ASSERT(this_th->th.th_active_in_pool == FALSE); in __kmp_free_thread()
5995 int gtid = this_thr->th.th_info.ds.ds_gtid; in __kmp_launch_thread()
6003 this_thr->th.th_cons = __kmp_allocate_cons_stack(gtid); // ATT: Memory leak? in __kmp_launch_thread()
6014 thread_data = &(this_thr->th.ompt_thread_info.thread_data); in __kmp_launch_thread()
6017 this_thr->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_launch_thread()
6018 this_thr->th.ompt_thread_info.wait_id = 0; in __kmp_launch_thread()
6019 this_thr->th.ompt_thread_info.idle_frame = OMPT_GET_FRAME_ADDRESS(0); in __kmp_launch_thread()
6020 this_thr->th.ompt_thread_info.parallel_flags = 0; in __kmp_launch_thread()
6025 this_thr->th.ompt_thread_info.state = ompt_state_idle; in __kmp_launch_thread()
6042 this_thr->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_launch_thread()
6046 pteam = &this_thr->th.th_team; in __kmp_launch_thread()
6062 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_launch_thread()
6079 this_thr->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_launch_thread()
6098 this_thr->th.th_task_team = NULL; in __kmp_launch_thread()
6173 gtid = thread->th.th_info.ds.ds_gtid; in __kmp_reap_thread()
6183 !KMP_COMPARE_AND_STORE_ACQ32(&(thread->th.th_used_in_team), 0, 3)) in __kmp_reap_thread()
6189 kmp_flag_64<> flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, in __kmp_reap_thread()
6208 if (thread->th.th_active_in_pool) { in __kmp_reap_thread()
6209 thread->th.th_active_in_pool = FALSE; in __kmp_reap_thread()
6243 if (thread->th.th_cons) { in __kmp_reap_thread()
6244 __kmp_free_cons_stack(thread->th.th_cons); in __kmp_reap_thread()
6245 thread->th.th_cons = NULL; in __kmp_reap_thread()
6249 if (thread->th.th_pri_common != NULL) { in __kmp_reap_thread()
6250 __kmp_free(thread->th.th_pri_common); in __kmp_reap_thread()
6251 thread->th.th_pri_common = NULL; in __kmp_reap_thread()
6254 if (thread->th.th_task_state_memo_stack != NULL) { in __kmp_reap_thread()
6255 __kmp_free(thread->th.th_task_state_memo_stack); in __kmp_reap_thread()
6256 thread->th.th_task_state_memo_stack = NULL; in __kmp_reap_thread()
6260 if (thread->th.th_local.bget_data != NULL) { in __kmp_reap_thread()
6266 if (thread->th.th_affin_mask != NULL) { in __kmp_reap_thread()
6267 KMP_CPU_FREE(thread->th.th_affin_mask); in __kmp_reap_thread()
6268 thread->th.th_affin_mask = NULL; in __kmp_reap_thread()
6273 if (thread->th.th_hier_bar_data != NULL) { in __kmp_reap_thread()
6274 __kmp_free(thread->th.th_hier_bar_data); in __kmp_reap_thread()
6275 thread->th.th_hier_bar_data = NULL; in __kmp_reap_thread()
6279 __kmp_reap_team(thread->th.th_serial_team); in __kmp_reap_thread()
6280 thread->th.th_serial_team = NULL; in __kmp_reap_thread()
6287 static void __kmp_itthash_clean(kmp_info_t *th) { in __kmp_itthash_clean() argument
6294 __kmp_thread_free(th, bucket); in __kmp_itthash_clean()
6304 __kmp_thread_free(th, bucket); in __kmp_itthash_clean()
6374 __kmp_thread_pool = thread->th.th_next_pool; in __kmp_internal_end()
6376 KMP_DEBUG_ASSERT(thread->th.th_reap_state == KMP_SAFE_TO_REAP); in __kmp_internal_end()
6377 thread->th.th_next_pool = NULL; in __kmp_internal_end()
6378 thread->th.th_in_pool = FALSE; in __kmp_internal_end()
6402 while (thr && KMP_ATOMIC_LD_ACQ(&thr->th.th_blocking)) in __kmp_internal_end()
6631 __kmp_threads[gtid]->th.th_task_team = NULL; in __kmp_internal_end_thread()
7474 if (thread->th.th_current_task->td_icvs.nproc != 0) in __kmp_do_middle_initialize()
7639 this_thr->th.th_local.this_construct = 0; in __kmp_run_before_invoked_task()
7641 KMP_CACHE_PREFETCH(&this_thr->th.th_bar[bs_forkjoin_barrier].bb.b_arrived); in __kmp_run_before_invoked_task()
7643 dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch); in __kmp_run_before_invoked_task()
7669 kmp_team_t *team = this_thr->th.th_team; in __kmp_invoke_task_func()
7733 this_thr->th.ompt_thread_info.parallel_flags |= ompt_parallel_team; in __kmp_invoke_task_func()
7763 kmp_team_t *team = thr->th.th_team; in __kmp_teams_master()
7765 thr->th.th_set_nproc = thr->th.th_teams_size.nth; in __kmp_teams_master()
7766 KMP_DEBUG_ASSERT(thr->th.th_teams_microtask); in __kmp_teams_master()
7767 KMP_DEBUG_ASSERT(thr->th.th_set_nproc); in __kmp_teams_master()
7769 __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask)); in __kmp_teams_master()
7775 tmp->cg_thread_limit = thr->th.th_current_task->td_icvs.thread_limit; in __kmp_teams_master()
7780 tmp->up = thr->th.th_cg_roots; in __kmp_teams_master()
7781 thr->th.th_cg_roots = tmp; in __kmp_teams_master()
7789 (microtask_t)thr->th.th_teams_microtask, // "wrapped" task in __kmp_teams_master()
7795 if (thr->th.th_team_nproc < thr->th.th_teams_size.nth) in __kmp_teams_master()
7796 thr->th.th_teams_size.nth = thr->th.th_team_nproc; in __kmp_teams_master()
7810 kmp_team_t *team = this_thr->th.th_team; in __kmp_invoke_teams_master()
7812 if (!__kmp_threads[gtid]->th.th_team->t.t_serialized) in __kmp_invoke_teams_master()
7813 KMP_DEBUG_ASSERT((void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn == in __kmp_invoke_teams_master()
7831 this_thr->th.ompt_thread_info.parallel_flags |= ompt_parallel_league; in __kmp_invoke_teams_master()
7846 thr->th.th_set_nproc = num_threads; in __kmp_push_num_threads()
7871 if (num_threads > thr->th.th_current_task->td_icvs.thread_limit) { in __kmp_push_thread_limit()
7872 num_threads = thr->th.th_current_task->td_icvs.thread_limit; in __kmp_push_thread_limit()
7888 thr->th.th_current_task->td_icvs.thread_limit = num_threads; in __kmp_push_thread_limit()
7909 thr->th.th_teams_size.nth = num_threads; in __kmp_push_thread_limit()
7942 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams; in __kmp_push_num_teams()
7999 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams; in __kmp_push_num_teams_51()
8007 thr->th.th_set_proc_bind = proc_bind; in __kmp_push_proc_bind()
8020 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); in __kmp_internal_fork()
8042 KMP_ASSERT(this_thr->th.th_team == team); in __kmp_internal_fork()
8047 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc); in __kmp_internal_fork()
8059 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); in __kmp_internal_join()
8067 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) { in __kmp_internal_join()
8072 gtid, __kmp_threads[gtid]->th.th_team_nproc, team, in __kmp_internal_join()
8077 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc); in __kmp_internal_join()
8083 this_thr->th.ompt_thread_info.state == ompt_state_wait_barrier_implicit) { in __kmp_internal_join()
8084 int ds_tid = this_thr->th.th_info.ds.ds_tid; in __kmp_internal_join()
8086 this_thr->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_internal_join()
8114 KMP_ASSERT(this_thr->th.th_team == team); in __kmp_internal_join()
8139 if (hot_team->t.t_threads[i]->th.th_active) { in __kmp_active_hot_team_nproc()
8159 ->th.th_current_task->td_icvs.dynamic == TRUE); in __kmp_load_balance_nproc()
8358 root = __kmp_threads[gtid]->th.th_root; in __kmp_internal_begin()
8386 root = thread->th.th_root; in __kmp_user_set_library()
8398 thread->th.th_set_nproc = 0; in __kmp_user_set_library()
8402 thread->th.th_set_nproc = 0; in __kmp_user_set_library()
8407 thread->th.th_set_nproc = 0; in __kmp_user_set_library()
8475 if (thr->th.th_teams_microtask) { in __kmp_aux_get_team_info()
8476 kmp_team_t *team = thr->th.th_team; in __kmp_aux_get_team_info()
8477 int tlevel = thr->th.th_teams_level; // the level of the teams construct in __kmp_aux_get_team_info()
8581 static int __kmp_aux_capture_affinity_field(int gtid, const kmp_info_t *th, in __kmp_aux_capture_affinity_field() argument
8592 KMP_DEBUG_ASSERT(th); in __kmp_aux_capture_affinity_field()
8692 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_level); in __kmp_aux_capture_affinity_field()
8710 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_nproc); in __kmp_aux_capture_affinity_field()
8714 __kmp_get_ancestor_thread_num(gtid, th->th.th_team->t.t_level - 1); in __kmp_aux_capture_affinity_field()
8721 __kmp_affinity_str_buf_mask(&buf, th->th.th_affin_mask); in __kmp_aux_capture_affinity_field()
8754 const kmp_info_t *th; in __kmp_aux_capture_affinity() local
8763 th = __kmp_threads[gtid]; in __kmp_aux_capture_affinity()
8778 int rc = __kmp_aux_capture_affinity_field(gtid, th, &parse_ptr, &field); in __kmp_aux_capture_affinity()
8817 set__blocktime_team(thread->th.th_team, tid, blocktime); in __kmp_aux_set_blocktime()
8818 set__blocktime_team(thread->th.th_serial_team, 0, blocktime); in __kmp_aux_set_blocktime()
8824 set__bt_intervals_team(thread->th.th_team, tid, bt_intervals); in __kmp_aux_set_blocktime()
8825 set__bt_intervals_team(thread->th.th_serial_team, 0, bt_intervals); in __kmp_aux_set_blocktime()
8831 set__bt_set_team(thread->th.th_team, tid, bt_set); in __kmp_aux_set_blocktime()
8832 set__bt_set_team(thread->th.th_serial_team, 0, bt_set); in __kmp_aux_set_blocktime()
8836 __kmp_gtid_from_tid(tid, thread->th.th_team), in __kmp_aux_set_blocktime()
8837 thread->th.th_team->t.t_id, tid, blocktime, bt_intervals, in __kmp_aux_set_blocktime()
8841 __kmp_gtid_from_tid(tid, thread->th.th_team), in __kmp_aux_set_blocktime()
8842 thread->th.th_team->t.t_id, tid, blocktime)); in __kmp_aux_set_blocktime()
9019 return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8); in __kmp_get_reduce_method()
9041 kmp_flag_64<> fl(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, in __kmp_resume_if_soft_paused()
9118 if (team->t.t_threads[f]->th.th_used_in_team.load() == 0) { in __kmp_resize_dist_barrier()
9124 if (team->t.t_threads[f]->th.th_used_in_team.load() == 3) { in __kmp_resize_dist_barrier()
9125 while (team->t.t_threads[f]->th.th_used_in_team.load() == 3) in __kmp_resize_dist_barrier()
9129 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 1); in __kmp_resize_dist_barrier()
9131 team->t.t_threads[f]->th.th_used_in_team.store(2); in __kmp_resize_dist_barrier()
9132 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 2); in __kmp_resize_dist_barrier()
9145 if (other_threads[f]->th.th_used_in_team.load() != 0) { in __kmp_resize_dist_barrier()
9148 void *, other_threads[f]->th.th_sleep_loc); in __kmp_resize_dist_barrier()
9149 __kmp_atomic_resume_64(other_threads[f]->th.th_info.ds.ds_gtid, flag); in __kmp_resize_dist_barrier()
9152 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 0); in __kmp_resize_dist_barrier()
9172 KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team), 0, in __kmp_add_threads_to_team()
9175 __kmp_resume_32(team->t.t_threads[f]->th.th_info.ds.ds_gtid, in __kmp_add_threads_to_team()
9186 if (team->t.t_threads[f]->th.th_used_in_team.load() == 1) { in __kmp_add_threads_to_team()
9237 __kmp_hidden_helper_main_thread->th.th_set_nproc = in __kmp_hidden_helper_threads_initz_routine()