Lines Matching refs:team
88 static void __kmp_initialize_team(kmp_team_t *team, int new_nproc,
92 static void __kmp_partition_places(kmp_team_t *team,
98 void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc,
112 void __kmp_resize_dist_barrier(kmp_team_t *team, int old_nthreads,
114 void __kmp_add_threads_to_team(kmp_team_t *team, int new_nthreads);
521 static void __kmp_print_team_storage_map(const char *header, kmp_team_t *team, in __kmp_print_team_storage_map() argument
523 int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2; in __kmp_print_team_storage_map()
524 __kmp_print_storage_map_gtid(-1, team, team + 1, sizeof(kmp_team_t), "%s_%d", in __kmp_print_team_storage_map()
527 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0], in __kmp_print_team_storage_map()
528 &team->t.t_bar[bs_last_barrier], in __kmp_print_team_storage_map()
532 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier], in __kmp_print_team_storage_map()
533 &team->t.t_bar[bs_plain_barrier + 1], in __kmp_print_team_storage_map()
537 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier], in __kmp_print_team_storage_map()
538 &team->t.t_bar[bs_forkjoin_barrier + 1], in __kmp_print_team_storage_map()
543 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier], in __kmp_print_team_storage_map()
544 &team->t.t_bar[bs_reduction_barrier + 1], in __kmp_print_team_storage_map()
550 -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr], in __kmp_print_team_storage_map()
554 -1, &team->t.t_threads[0], &team->t.t_threads[num_thr], in __kmp_print_team_storage_map()
557 __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0], in __kmp_print_team_storage_map()
558 &team->t.t_disp_buffer[num_disp_buff], in __kmp_print_team_storage_map()
635 kmp_team_t *team = __kmp_team_from_gtid(gtid); in __kmp_parallel_deo() local
647 if (!team->t.t_serialized) { in __kmp_parallel_deo()
649 KMP_WAIT(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid), KMP_EQ, in __kmp_parallel_deo()
661 kmp_team_t *team = __kmp_team_from_gtid(gtid); in __kmp_parallel_dxo() local
669 if (!team->t.t_serialized) { in __kmp_parallel_dxo()
674 team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc); in __kmp_parallel_dxo()
687 kmp_team_t *team; in __kmp_enter_single() local
694 team = th->th.th_team; in __kmp_enter_single()
699 if (team->t.t_serialized) { in __kmp_enter_single()
708 if (team->t.t_construct == old_this) { in __kmp_enter_single()
709 status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this, in __kmp_enter_single()
715 team->t.t_active_level == 1) { in __kmp_enter_single()
939 static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team, in __kmp_fork_team_threads() argument
945 KA_TRACE(10, ("__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc)); in __kmp_fork_team_threads()
951 master_th->th.th_team = team; in __kmp_fork_team_threads()
952 master_th->th.th_team_nproc = team->t.t_nproc; in __kmp_fork_team_threads()
955 master_th->th.th_dispatch = &team->t.t_dispatch[0]; in __kmp_fork_team_threads()
963 int level = team->t.t_active_level - 1; // index in array of hot teams in __kmp_fork_team_threads()
969 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master && in __kmp_fork_team_threads()
970 master_th->th.th_teams_level == team->t.t_level) { in __kmp_fork_team_threads()
978 KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team); in __kmp_fork_team_threads()
982 hot_teams[level].hot_team = team; // remember new hot team in __kmp_fork_team_threads()
983 hot_teams[level].hot_team_nth = team->t.t_nproc; in __kmp_fork_team_threads()
990 use_hot_team = team == root->r.r_hot_team; in __kmp_fork_team_threads()
995 team->t.t_threads[0] = master_th; in __kmp_fork_team_threads()
996 __kmp_initialize_info(master_th, team, 0, master_gtid); in __kmp_fork_team_threads()
999 for (i = 1; i < team->t.t_nproc; i++) { in __kmp_fork_team_threads()
1002 kmp_info_t *thr = __kmp_allocate_thread(root, team, i); in __kmp_fork_team_threads()
1003 team->t.t_threads[i] = thr; in __kmp_fork_team_threads()
1005 KMP_DEBUG_ASSERT(thr->th.th_team == team); in __kmp_fork_team_threads()
1009 __kmp_gtid_from_tid(0, team), team->t.t_id, 0, in __kmp_fork_team_threads()
1010 __kmp_gtid_from_tid(i, team), team->t.t_id, i, in __kmp_fork_team_threads()
1011 team->t.t_bar[bs_forkjoin_barrier].b_arrived, in __kmp_fork_team_threads()
1012 team->t.t_bar[bs_plain_barrier].b_arrived)); in __kmp_fork_team_threads()
1018 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar; in __kmp_fork_team_threads()
1020 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; in __kmp_fork_team_threads()
1023 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; in __kmp_fork_team_threads()
1034 __kmp_partition_places(team); in __kmp_fork_team_threads()
1038 if (team->t.t_nproc > 1 && in __kmp_fork_team_threads()
1040 team->t.b->update_num_threads(team->t.t_nproc); in __kmp_fork_team_threads()
1041 __kmp_add_threads_to_team(team, team->t.t_nproc); in __kmp_fork_team_threads()
1045 if (__kmp_display_affinity && team->t.t_display_affinity != 1) { in __kmp_fork_team_threads()
1046 for (i = 0; i < team->t.t_nproc; i++) { in __kmp_fork_team_threads()
1047 kmp_info_t *thr = team->t.t_threads[i]; in __kmp_fork_team_threads()
1048 if (thr->th.th_prev_num_threads != team->t.t_nproc || in __kmp_fork_team_threads()
1049 thr->th.th_prev_level != team->t.t_level) { in __kmp_fork_team_threads()
1050 team->t.t_display_affinity = 1; in __kmp_fork_team_threads()
1063 inline static void propagateFPControl(kmp_team_t *team) { in propagateFPControl() argument
1082 KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word); in propagateFPControl()
1083 KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr); in propagateFPControl()
1086 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE); in propagateFPControl()
1090 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE); in propagateFPControl()
1096 inline static void updateHWFPControl(kmp_team_t *team) { in updateHWFPControl() argument
1097 if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) { in updateHWFPControl()
1106 if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) { in updateHWFPControl()
1108 __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word); in updateHWFPControl()
1111 if (team->t.t_mxcsr != mxcsr) { in updateHWFPControl()
1112 __kmp_load_mxcsr(&team->t.t_mxcsr); in updateHWFPControl()
1121 static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team,
1650 kmp_team_t *team; in __kmp_serial_fork_call() local
1740 team = master_th->th.th_team; in __kmp_serial_fork_call()
1742 team->t.t_invoke = invoker; in __kmp_serial_fork_call()
1743 __kmp_alloc_argv_entries(argc, team, TRUE); in __kmp_serial_fork_call()
1744 team->t.t_argc = argc; in __kmp_serial_fork_call()
1745 argv = (void **)team->t.t_argv; in __kmp_serial_fork_call()
1756 team->t.t_level--; in __kmp_serial_fork_call()
1881 kmp_team_t *team; in __kmp_fork_call() local
2132 team = __kmp_allocate_team(root, nthreads, nthreads, in __kmp_fork_call()
2139 copy_icvs((kmp_internal_control_t *)team->t.b->team_icvs, &new_icvs); in __kmp_fork_call()
2143 team = __kmp_allocate_team(root, nthreads, nthreads, in __kmp_fork_call()
2151 copy_icvs((kmp_internal_control_t *)team->t.b->team_icvs, in __kmp_fork_call()
2155 10, ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team)); in __kmp_fork_call()
2158 KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid); in __kmp_fork_call()
2159 KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons); in __kmp_fork_call()
2160 KMP_CHECK_UPDATE(team->t.t_ident, loc); in __kmp_fork_call()
2161 KMP_CHECK_UPDATE(team->t.t_parent, parent_team); in __kmp_fork_call()
2162 KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask); in __kmp_fork_call()
2164 KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.master_return_address, in __kmp_fork_call()
2167 KMP_CHECK_UPDATE(team->t.t_invoke, invoker); // TODO move to root, maybe in __kmp_fork_call()
2171 KMP_CHECK_UPDATE(team->t.t_level, new_level); in __kmp_fork_call()
2173 KMP_CHECK_UPDATE(team->t.t_active_level, new_level); in __kmp_fork_call()
2177 KMP_CHECK_UPDATE(team->t.t_level, new_level); in __kmp_fork_call()
2179 KMP_CHECK_UPDATE(team->t.t_active_level, new_level); in __kmp_fork_call()
2183 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched); in __kmp_fork_call()
2185 KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq); in __kmp_fork_call()
2186 KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator); in __kmp_fork_call()
2189 propagateFPControl(team); in __kmp_fork_call()
2204 team->t.t_task_team[master_th->th.th_task_state], team)); in __kmp_fork_call()
2235 team == master_th->th.th_hot_teams[active_level].hot_team) { in __kmp_fork_call()
2249 (team == root->r.r_hot_team)); in __kmp_fork_call()
2256 gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id, in __kmp_fork_call()
2257 team->t.t_nproc)); in __kmp_fork_call()
2258 KMP_DEBUG_ASSERT(team != root->r.r_hot_team || in __kmp_fork_call()
2259 (team->t.t_master_tid == 0 && in __kmp_fork_call()
2260 (team->t.t_parent == root->r.r_root_team || in __kmp_fork_call()
2261 team->t.t_parent->t.t_serialized))); in __kmp_fork_call()
2265 argv = (void **)team->t.t_argv; in __kmp_fork_call()
2275 KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i]); in __kmp_fork_call()
2280 KMP_CHECK_UPDATE(team->t.t_master_active, master_active); in __kmp_fork_call()
2284 __kmp_fork_team_threads(root, team, master_th, gtid, !ap); in __kmp_fork_call()
2285 __kmp_setup_icv_copy(team, nthreads, in __kmp_fork_call()
2295 if (team->t.t_active_level == 1 // only report frames at level 1 in __kmp_fork_call()
2307 team->t.t_region_time = tmp_time; in __kmp_fork_call()
2314 __kmp_itt_region_forking(gtid, team->t.t_nproc, 0); in __kmp_fork_call()
2320 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team); in __kmp_fork_call()
2324 root, team, master_th, gtid)); in __kmp_fork_call()
2330 KMP_DEBUG_ASSERT(team->t.t_stack_id == NULL); in __kmp_fork_call()
2331 team->t.t_stack_id = __kmp_itt_stack_caller_create(); in __kmp_fork_call()
2346 __kmp_internal_fork(loc, gtid, team); in __kmp_fork_call()
2349 root, team, master_th, gtid)); in __kmp_fork_call()
2359 team->t.t_id, team->t.t_pkfn)); in __kmp_fork_call()
2370 if (!team->t.t_invoke(gtid)) { in __kmp_fork_call()
2382 team->t.t_id, team->t.t_pkfn)); in __kmp_fork_call()
2397 kmp_team_t *team) { in __kmp_join_restore_state() argument
2400 ((team->t.t_serialized) ? ompt_state_work_serial in __kmp_join_restore_state()
2405 kmp_team_t *team, ompt_data_t *parallel_data, in __kmp_join_ompt() argument
2414 __kmp_join_restore_state(thread, team); in __kmp_join_ompt()
2426 kmp_team_t *team; in __kmp_join_call() local
2437 team = master_th->th.th_team; in __kmp_join_call()
2438 parent_team = team->t.t_parent; in __kmp_join_call()
2443 void *team_microtask = (void *)team->t.t_pkfn; in __kmp_join_call()
2448 !(team->t.t_serialized && fork_context == fork_context_gnu)) { in __kmp_join_call()
2457 __kmp_gtid_from_thread(master_th), team, in __kmp_join_call()
2458 team->t.t_task_team[master_th->th.th_task_state], in __kmp_join_call()
2461 team->t.t_task_team[master_th->th.th_task_state]); in __kmp_join_call()
2465 if (team->t.t_serialized) { in __kmp_join_call()
2468 int level = team->t.t_level; in __kmp_join_call()
2473 team->t.t_level++; in __kmp_join_call()
2478 team->t.t_serialized++; in __kmp_join_call()
2495 master_active = team->t.t_master_active; in __kmp_join_call()
2500 __kmp_internal_join(loc, gtid, team); in __kmp_join_call()
2503 KMP_DEBUG_ASSERT(team->t.t_stack_id != NULL); in __kmp_join_call()
2505 __kmp_itt_stack_caller_destroy((__itt_caller)team->t.t_stack_id); in __kmp_join_call()
2506 team->t.t_stack_id = NULL; in __kmp_join_call()
2527 ompt_data_t *parallel_data = &(team->t.ompt_team_info.parallel_data); in __kmp_join_call()
2528 void *codeptr = team->t.ompt_team_info.master_return_address; in __kmp_join_call()
2533 if (team->t.t_active_level == 1 && in __kmp_join_call()
2541 __kmp_itt_frame_submit(gtid, team->t.t_region_time, in __kmp_join_call()
2553 master_th->th.th_first_place = team->t.t_first_place; in __kmp_join_call()
2554 master_th->th.th_last_place = team->t.t_last_place; in __kmp_join_call()
2559 team->t.t_pkfn != (microtask_t)__kmp_teams_master && in __kmp_join_call()
2560 team->t.t_level == master_th->th.th_teams_level + 1) { in __kmp_join_call()
2569 int ompt_team_size = team->t.t_nproc; in __kmp_join_call()
2581 team->t.t_level--; in __kmp_join_call()
2582 team->t.t_active_level--; in __kmp_join_call()
2592 kmp_info_t **other_threads = team->t.t_threads; in __kmp_join_call()
2593 team->t.t_nproc = new_num; in __kmp_join_call()
2603 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; in __kmp_join_call()
2606 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; in __kmp_join_call()
2627 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid; in __kmp_join_call()
2628 master_th->th.th_local.this_construct = team->t.t_master_this_cons; in __kmp_join_call()
2630 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid]; in __kmp_join_call()
2638 team->t.t_level > master_th->th.th_teams_level) { in __kmp_join_call()
2651 int ompt_team_size = (flags == ompt_task_initial) ? 0 : team->t.t_nproc; in __kmp_join_call()
2662 master_th, team)); in __kmp_join_call()
2665 master_th->th.th_def_allocator = team->t.t_def_allocator; in __kmp_join_call()
2671 updateHWFPControl(team); in __kmp_join_call()
2676 __kmp_free_team(root, team USE_NESTED_HOT_ARG( in __kmp_join_call()
2710 } else if (team != root->r.r_hot_team) { in __kmp_join_call()
3048 kmp_team_t *team; in __kmp_get_ancestor_thread_num() local
3060 team = thr->th.th_team; in __kmp_get_ancestor_thread_num()
3061 ii = team->t.t_level; in __kmp_get_ancestor_thread_num()
3084 dd = team->t.t_serialized; in __kmp_get_ancestor_thread_num()
3087 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) { in __kmp_get_ancestor_thread_num()
3089 if ((team->t.t_serialized) && (!dd)) { in __kmp_get_ancestor_thread_num()
3090 team = team->t.t_parent; in __kmp_get_ancestor_thread_num()
3094 team = team->t.t_parent; in __kmp_get_ancestor_thread_num()
3095 dd = team->t.t_serialized; in __kmp_get_ancestor_thread_num()
3100 return (dd > 1) ? (0) : (team->t.t_master_tid); in __kmp_get_ancestor_thread_num()
3106 kmp_team_t *team; in __kmp_get_team_size() local
3118 team = thr->th.th_team; in __kmp_get_team_size()
3119 ii = team->t.t_level; in __kmp_get_team_size()
3140 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) { in __kmp_get_team_size()
3142 if (team->t.t_serialized && (!dd)) { in __kmp_get_team_size()
3143 team = team->t.t_parent; in __kmp_get_team_size()
3147 team = team->t.t_parent; in __kmp_get_team_size()
3152 return team->t.t_nproc; in __kmp_get_team_size()
3191 static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team, int realloc) { in __kmp_alloc_argv_entries() argument
3193 KMP_DEBUG_ASSERT(team); in __kmp_alloc_argv_entries()
3194 if (!realloc || argc > team->t.t_max_argc) { in __kmp_alloc_argv_entries()
3198 team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0)); in __kmp_alloc_argv_entries()
3200 if (realloc && team->t.t_argv != &team->t.t_inline_argv[0]) in __kmp_alloc_argv_entries()
3201 __kmp_free((void *)team->t.t_argv); in __kmp_alloc_argv_entries()
3205 team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES; in __kmp_alloc_argv_entries()
3208 team->t.t_id, team->t.t_max_argc)); in __kmp_alloc_argv_entries()
3209 team->t.t_argv = &team->t.t_inline_argv[0]; in __kmp_alloc_argv_entries()
3212 -1, &team->t.t_inline_argv[0], in __kmp_alloc_argv_entries()
3213 &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES], in __kmp_alloc_argv_entries()
3215 team->t.t_id); in __kmp_alloc_argv_entries()
3219 team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES >> 1)) in __kmp_alloc_argv_entries()
3224 team->t.t_id, team->t.t_max_argc)); in __kmp_alloc_argv_entries()
3225 team->t.t_argv = in __kmp_alloc_argv_entries()
3226 (void **)__kmp_page_allocate(sizeof(void *) * team->t.t_max_argc); in __kmp_alloc_argv_entries()
3228 __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0], in __kmp_alloc_argv_entries()
3229 &team->t.t_argv[team->t.t_max_argc], in __kmp_alloc_argv_entries()
3230 sizeof(void *) * team->t.t_max_argc, in __kmp_alloc_argv_entries()
3231 "team_%d.t_argv", team->t.t_id); in __kmp_alloc_argv_entries()
3237 static void __kmp_allocate_team_arrays(kmp_team_t *team, int max_nth) { in __kmp_allocate_team_arrays() argument
3240 team->t.t_threads = in __kmp_allocate_team_arrays()
3242 team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate( in __kmp_allocate_team_arrays()
3244 team->t.t_dispatch = in __kmp_allocate_team_arrays()
3246 team->t.t_implicit_task_taskdata = in __kmp_allocate_team_arrays()
3248 team->t.t_max_nproc = max_nth; in __kmp_allocate_team_arrays()
3252 team->t.t_disp_buffer[i].buffer_index = i; in __kmp_allocate_team_arrays()
3253 team->t.t_disp_buffer[i].doacross_buf_idx = i; in __kmp_allocate_team_arrays()
3257 static void __kmp_free_team_arrays(kmp_team_t *team) { in __kmp_free_team_arrays() argument
3260 for (i = 0; i < team->t.t_max_nproc; ++i) { in __kmp_free_team_arrays()
3261 if (team->t.t_dispatch[i].th_disp_buffer != NULL) { in __kmp_free_team_arrays()
3262 __kmp_free(team->t.t_dispatch[i].th_disp_buffer); in __kmp_free_team_arrays()
3263 team->t.t_dispatch[i].th_disp_buffer = NULL; in __kmp_free_team_arrays()
3267 __kmp_dispatch_free_hierarchies(team); in __kmp_free_team_arrays()
3269 __kmp_free(team->t.t_threads); in __kmp_free_team_arrays()
3270 __kmp_free(team->t.t_disp_buffer); in __kmp_free_team_arrays()
3271 __kmp_free(team->t.t_dispatch); in __kmp_free_team_arrays()
3272 __kmp_free(team->t.t_implicit_task_taskdata); in __kmp_free_team_arrays()
3273 team->t.t_threads = NULL; in __kmp_free_team_arrays()
3274 team->t.t_disp_buffer = NULL; in __kmp_free_team_arrays()
3275 team->t.t_dispatch = NULL; in __kmp_free_team_arrays()
3276 team->t.t_implicit_task_taskdata = 0; in __kmp_free_team_arrays()
3279 static void __kmp_reallocate_team_arrays(kmp_team_t *team, int max_nth) { in __kmp_reallocate_team_arrays() argument
3280 kmp_info_t **oldThreads = team->t.t_threads; in __kmp_reallocate_team_arrays()
3282 __kmp_free(team->t.t_disp_buffer); in __kmp_reallocate_team_arrays()
3283 __kmp_free(team->t.t_dispatch); in __kmp_reallocate_team_arrays()
3284 __kmp_free(team->t.t_implicit_task_taskdata); in __kmp_reallocate_team_arrays()
3285 __kmp_allocate_team_arrays(team, max_nth); in __kmp_reallocate_team_arrays()
3287 KMP_MEMCPY(team->t.t_threads, oldThreads, in __kmp_reallocate_team_arrays()
3288 team->t.t_nproc * sizeof(kmp_info_t *)); in __kmp_reallocate_team_arrays()
3329 static kmp_internal_control_t __kmp_get_x_global_icvs(const kmp_team_t *team) { in __kmp_get_x_global_icvs() argument
3334 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs); in __kmp_get_x_global_icvs()
3442 kmp_team_p const *team // Team to add. in __kmp_print_structure_team_accum() argument
3453 if (team == NULL) { in __kmp_print_structure_team_accum()
3457 __kmp_print_structure_team_accum(list, team->t.t_parent); in __kmp_print_structure_team_accum()
3458 __kmp_print_structure_team_accum(list, team->t.t_next_pool); in __kmp_print_structure_team_accum()
3462 while (l->next != NULL && l->entry != team) { in __kmp_print_structure_team_accum()
3471 while (l->next != NULL && l->entry->t.t_id <= team->t.t_id) { in __kmp_print_structure_team_accum()
3480 l->entry = team; in __kmp_print_structure_team_accum()
3485 static void __kmp_print_structure_team(char const *title, kmp_team_p const *team in __kmp_print_structure_team() argument
3489 if (team != NULL) { in __kmp_print_structure_team()
3490 __kmp_printf("%2x %p\n", team->t.t_id, team); in __kmp_print_structure_team()
3590 kmp_team_p const *team = list->entry; in __kmp_print_structure() local
3592 __kmp_printf("Team %2x %p:\n", team->t.t_id, team); in __kmp_print_structure()
3593 __kmp_print_structure_team(" Parent Team: ", team->t.t_parent); in __kmp_print_structure()
3594 __kmp_printf(" Primary TID: %2d\n", team->t.t_master_tid); in __kmp_print_structure()
3595 __kmp_printf(" Max threads: %2d\n", team->t.t_max_nproc); in __kmp_print_structure()
3596 __kmp_printf(" Levels of serial: %2d\n", team->t.t_serialized); in __kmp_print_structure()
3597 __kmp_printf(" Number threads: %2d\n", team->t.t_nproc); in __kmp_print_structure()
3598 for (i = 0; i < team->t.t_nproc; ++i) { in __kmp_print_structure()
3600 __kmp_print_structure_thread("", team->t.t_threads[i]); in __kmp_print_structure()
3602 __kmp_print_structure_team(" Next in pool: ", team->t.t_next_pool); in __kmp_print_structure()
4070 kmp_team_t *team = hot_teams[level].hot_team; in __kmp_free_hot_teams() local
4075 kmp_info_t *th = team->t.t_threads[i]; in __kmp_free_hot_teams()
4083 __kmp_free_team(root, team, NULL); in __kmp_free_hot_teams()
4206 kmp_team_t *team = thread->th.th_team; in __kmp_unregister_root_current_thread() local
4216 __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL)); in __kmp_unregister_root_current_thread()
4256 kmp_team_t *team = this_thr->th.th_team; in __kmp_task_info() local
4261 gtid, tid, this_thr, team, steam, this_thr->th.th_current_task, in __kmp_task_info()
4262 team->t.t_implicit_task_taskdata[tid].td_parent); in __kmp_task_info()
4269 static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team, in __kmp_initialize_info() argument
4276 KMP_DEBUG_ASSERT(team); in __kmp_initialize_info()
4277 KMP_DEBUG_ASSERT(team->t.t_threads); in __kmp_initialize_info()
4278 KMP_DEBUG_ASSERT(team->t.t_dispatch); in __kmp_initialize_info()
4279 kmp_info_t *master = team->t.t_threads[0]; in __kmp_initialize_info()
4285 TCW_SYNC_PTR(this_thr->th.th_team, team); in __kmp_initialize_info()
4302 this_thr->th.th_team_nproc = team->t.t_nproc; in __kmp_initialize_info()
4304 this_thr->th.th_team_serialized = team->t.t_serialized; in __kmp_initialize_info()
4306 KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata); in __kmp_initialize_info()
4312 team, tid, TRUE); in __kmp_initialize_info()
4320 this_thr->th.th_dispatch = &team->t.t_dispatch[tid]; in __kmp_initialize_info()
4368 (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers); in __kmp_initialize_info()
4370 team->t.t_max_nproc)); in __kmp_initialize_info()
4372 KMP_DEBUG_ASSERT(team->t.t_dispatch); in __kmp_initialize_info()
4373 KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid]); in __kmp_initialize_info()
4384 &dispatch->th_disp_buffer[team->t.t_max_nproc == 1 in __kmp_initialize_info()
4390 gtid, team->t.t_id, gtid); in __kmp_initialize_info()
4427 kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team, in __kmp_allocate_thread() argument
4434 KMP_DEBUG_ASSERT(root && team); in __kmp_allocate_thread()
4463 __kmp_initialize_info(new_thr, team, new_tid, in __kmp_allocate_thread()
4600 kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team); in __kmp_allocate_thread()
4619 __kmp_initialize_info(new_thr, team, new_tid, new_gtid); in __kmp_allocate_thread()
4641 balign[b].bb.team = NULL; in __kmp_allocate_thread()
4720 static void __kmp_reinitialize_team(kmp_team_t *team, in __kmp_reinitialize_team() argument
4724 team->t.t_threads[0], team)); in __kmp_reinitialize_team()
4725 KMP_DEBUG_ASSERT(team && new_icvs); in __kmp_reinitialize_team()
4727 KMP_CHECK_UPDATE(team->t.t_ident, loc); in __kmp_reinitialize_team()
4729 KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID()); in __kmp_reinitialize_team()
4731 __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE); in __kmp_reinitialize_team()
4732 copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs); in __kmp_reinitialize_team()
4735 team->t.t_threads[0], team)); in __kmp_reinitialize_team()
4741 static void __kmp_initialize_team(kmp_team_t *team, int new_nproc, in __kmp_initialize_team() argument
4744 KF_TRACE(10, ("__kmp_initialize_team: enter: team=%p\n", team)); in __kmp_initialize_team()
4747 KMP_DEBUG_ASSERT(team); in __kmp_initialize_team()
4748 KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc); in __kmp_initialize_team()
4749 KMP_DEBUG_ASSERT(team->t.t_threads); in __kmp_initialize_team()
4752 team->t.t_master_tid = 0; /* not needed */ in __kmp_initialize_team()
4754 team->t.t_serialized = new_nproc > 1 ? 0 : 1; in __kmp_initialize_team()
4755 team->t.t_nproc = new_nproc; in __kmp_initialize_team()
4758 team->t.t_next_pool = NULL; in __kmp_initialize_team()
4762 TCW_SYNC_PTR(team->t.t_pkfn, NULL); /* not needed */ in __kmp_initialize_team()
4763 team->t.t_invoke = NULL; /* not needed */ in __kmp_initialize_team()
4766 team->t.t_sched.sched = new_icvs->sched.sched; in __kmp_initialize_team()
4769 team->t.t_fp_control_saved = FALSE; /* not needed */ in __kmp_initialize_team()
4770 team->t.t_x87_fpu_control_word = 0; /* not needed */ in __kmp_initialize_team()
4771 team->t.t_mxcsr = 0; /* not needed */ in __kmp_initialize_team()
4774 team->t.t_construct = 0; in __kmp_initialize_team()
4776 team->t.t_ordered.dt.t_value = 0; in __kmp_initialize_team()
4777 team->t.t_master_active = FALSE; in __kmp_initialize_team()
4780 team->t.t_copypriv_data = NULL; /* not necessary, but nice for debugging */ in __kmp_initialize_team()
4783 team->t.t_copyin_counter = 0; /* for barrier-free copyin implementation */ in __kmp_initialize_team()
4786 team->t.t_control_stack_top = NULL; in __kmp_initialize_team()
4788 __kmp_reinitialize_team(team, new_icvs, loc); in __kmp_initialize_team()
4791 KF_TRACE(10, ("__kmp_initialize_team: exit: team=%p\n", team)); in __kmp_initialize_team()
4795 static inline void __kmp_set_thread_place(kmp_team_t *team, kmp_info_t *th, in __kmp_set_thread_place() argument
4801 if (__kmp_display_affinity && team->t.t_display_affinity != 1) in __kmp_set_thread_place()
4802 team->t.t_display_affinity = 1; in __kmp_set_thread_place()
4813 static void __kmp_partition_places(kmp_team_t *team, int update_master_only) { in __kmp_partition_places() argument
4815 if (KMP_HIDDEN_HELPER_TEAM(team)) in __kmp_partition_places()
4818 kmp_info_t *master_th = team->t.t_threads[0]; in __kmp_partition_places()
4820 kmp_proc_bind_t proc_bind = team->t.t_proc_bind; in __kmp_partition_places()
4825 team->t.t_first_place = first_place; in __kmp_partition_places()
4826 team->t.t_last_place = last_place; in __kmp_partition_places()
4830 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]), in __kmp_partition_places()
4831 team->t.t_id, masters_place, first_place, last_place)); in __kmp_partition_places()
4838 KMP_DEBUG_ASSERT(team->t.t_nproc == 1); in __kmp_partition_places()
4843 int n_th = team->t.t_nproc; in __kmp_partition_places()
4845 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places()
4847 __kmp_set_thread_place(team, th, first_place, last_place, masters_place); in __kmp_partition_places()
4851 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, in __kmp_partition_places()
4858 int n_th = team->t.t_nproc; in __kmp_partition_places()
4868 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places()
4878 __kmp_set_thread_place(team, th, first_place, last_place, place); in __kmp_partition_places()
4882 __kmp_gtid_from_thread(team->t.t_threads[f]), in __kmp_partition_places()
4883 team->t.t_id, f, place, first_place, last_place)); in __kmp_partition_places()
4894 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places()
4897 __kmp_set_thread_place(team, th, first_place, last_place, place); in __kmp_partition_places()
4929 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f, in __kmp_partition_places()
4938 int n_th = team->t.t_nproc; in __kmp_partition_places()
4961 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places()
4987 __kmp_set_thread_place(team, th, fplace, place, nplace); in __kmp_partition_places()
5001 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, in __kmp_partition_places()
5051 th = team->t.t_threads[f]; in __kmp_partition_places()
5053 __kmp_set_thread_place(team, th, first, last, place); in __kmp_partition_places()
5057 __kmp_gtid_from_thread(team->t.t_threads[f]), in __kmp_partition_places()
5058 team->t.t_id, f, th->th.th_new_place, in __kmp_partition_places()
5076 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places()
5079 __kmp_set_thread_place(team, th, place, place, place); in __kmp_partition_places()
5110 __kmp_gtid_from_thread(team->t.t_threads[f]), in __kmp_partition_places()
5111 team->t.t_id, f, th->th.th_new_place, in __kmp_partition_places()
5122 KA_TRACE(20, ("__kmp_partition_places: exit T#%d\n", team->t.t_id)); in __kmp_partition_places()
5139 kmp_team_t *team; in __kmp_allocate_team() local
5152 team = master->th.th_team; in __kmp_allocate_team()
5153 level = team->t.t_active_level; in __kmp_allocate_team()
5157 team->t.t_pkfn == in __kmp_allocate_team()
5160 team->t.t_level)) { // or nested parallel inside the teams in __kmp_allocate_team()
5167 master->th.th_teams_level >= team->t.t_level) || in __kmp_allocate_team()
5168 (team->t.t_pkfn == (microtask_t)__kmp_teams_master)) in __kmp_allocate_team()
5188 team = hot_teams[level].hot_team; in __kmp_allocate_team()
5190 team = root->r.r_hot_team; in __kmp_allocate_team()
5196 team->t.t_task_team[0], team->t.t_task_team[1])); in __kmp_allocate_team()
5200 if (team->t.t_nproc != new_nproc && in __kmp_allocate_team()
5203 int old_nthr = team->t.t_nproc; in __kmp_allocate_team()
5204 __kmp_resize_dist_barrier(team, old_nthr, new_nproc); in __kmp_allocate_team()
5210 team->t.t_proc_bind = proc_bind_default; in __kmp_allocate_team()
5214 if (team->t.t_nproc == new_nproc) { // Check changes in number of threads in __kmp_allocate_team()
5218 if (team->t.t_size_changed == -1) { in __kmp_allocate_team()
5219 team->t.t_size_changed = 1; in __kmp_allocate_team()
5221 KMP_CHECK_UPDATE(team->t.t_size_changed, 0); in __kmp_allocate_team()
5227 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched); in __kmp_allocate_team()
5229 __kmp_reinitialize_team(team, new_icvs, in __kmp_allocate_team()
5233 team->t.t_threads[0], team)); in __kmp_allocate_team()
5234 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0); in __kmp_allocate_team()
5237 if ((team->t.t_size_changed == 0) && in __kmp_allocate_team()
5238 (team->t.t_proc_bind == new_proc_bind)) { in __kmp_allocate_team()
5242 __kmp_partition_places(team, 1); in __kmp_allocate_team()
5247 team->t.t_id, new_proc_bind, team->t.t_first_place, in __kmp_allocate_team()
5248 team->t.t_last_place)); in __kmp_allocate_team()
5251 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); in __kmp_allocate_team()
5252 __kmp_partition_places(team); in __kmp_allocate_team()
5256 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); in __kmp_allocate_team()
5258 } else if (team->t.t_nproc > new_nproc) { in __kmp_allocate_team()
5263 team->t.t_size_changed = 1; in __kmp_allocate_team()
5267 __kmp_add_threads_to_team(team, new_nproc); in __kmp_allocate_team()
5273 KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc); in __kmp_allocate_team()
5277 for (f = new_nproc; f < team->t.t_nproc; f++) { in __kmp_allocate_team()
5278 KMP_DEBUG_ASSERT(team->t.t_threads[f]); in __kmp_allocate_team()
5282 team->t.t_threads[f]->th.th_task_team = NULL; in __kmp_allocate_team()
5284 __kmp_free_thread(team->t.t_threads[f]); in __kmp_allocate_team()
5285 team->t.t_threads[f] = NULL; in __kmp_allocate_team()
5292 for (f = new_nproc; f < team->t.t_nproc; ++f) { in __kmp_allocate_team()
5293 KMP_DEBUG_ASSERT(team->t.t_threads[f]); in __kmp_allocate_team()
5294 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar; in __kmp_allocate_team()
5304 team->t.t_nproc = new_nproc; in __kmp_allocate_team()
5306 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_icvs->sched.sched); in __kmp_allocate_team()
5307 __kmp_reinitialize_team(team, new_icvs, in __kmp_allocate_team()
5312 team->t.t_threads[f]->th.th_team_nproc = new_nproc; in __kmp_allocate_team()
5318 team->t.t_threads[0], team)); in __kmp_allocate_team()
5320 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0); in __kmp_allocate_team()
5323 for (f = 0; f < team->t.t_nproc; f++) { in __kmp_allocate_team()
5324 KMP_DEBUG_ASSERT(team->t.t_threads[f] && in __kmp_allocate_team()
5325 team->t.t_threads[f]->th.th_team_nproc == in __kmp_allocate_team()
5326 team->t.t_nproc); in __kmp_allocate_team()
5331 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); in __kmp_allocate_team()
5333 __kmp_partition_places(team); in __kmp_allocate_team()
5341 int old_nproc = team->t.t_nproc; // save old value and use to update only in __kmp_allocate_team()
5342 team->t.t_size_changed = 1; in __kmp_allocate_team()
5348 kmp_info_t **other_threads = team->t.t_threads; in __kmp_allocate_team()
5349 for (f = team->t.t_nproc; f < avail_threads; ++f) { in __kmp_allocate_team()
5355 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; in __kmp_allocate_team()
5358 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; in __kmp_allocate_team()
5366 team->t.t_nproc = new_nproc; // just get reserved threads involved in __kmp_allocate_team()
5370 team->t.t_nproc = hot_teams[level].hot_team_nth; in __kmp_allocate_team()
5373 if (team->t.t_max_nproc < new_nproc) { in __kmp_allocate_team()
5375 __kmp_reallocate_team_arrays(team, new_nproc); in __kmp_allocate_team()
5376 __kmp_reinitialize_team(team, new_icvs, NULL); in __kmp_allocate_team()
5389 for (f = team->t.t_nproc; f < new_nproc; f++) { in __kmp_allocate_team()
5390 kmp_info_t *new_worker = __kmp_allocate_thread(root, team, f); in __kmp_allocate_team()
5392 team->t.t_threads[f] = new_worker; in __kmp_allocate_team()
5397 team->t.t_id, __kmp_gtid_from_tid(f, team), team->t.t_id, f, in __kmp_allocate_team()
5398 team->t.t_bar[bs_forkjoin_barrier].b_arrived, in __kmp_allocate_team()
5399 team->t.t_bar[bs_plain_barrier].b_arrived)); in __kmp_allocate_team()
5405 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; in __kmp_allocate_team()
5409 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; in __kmp_allocate_team()
5425 __kmp_add_threads_to_team(team, new_nproc); in __kmp_allocate_team()
5429 __kmp_initialize_team(team, new_nproc, new_icvs, in __kmp_allocate_team()
5433 KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc); in __kmp_allocate_team()
5434 for (f = 0; f < team->t.t_nproc; ++f) in __kmp_allocate_team()
5435 __kmp_initialize_info(team->t.t_threads[f], team, f, in __kmp_allocate_team()
5436 __kmp_gtid_from_tid(f, team)); in __kmp_allocate_team()
5439 kmp_uint8 old_state = team->t.t_threads[old_nproc - 1]->th.th_task_state; in __kmp_allocate_team()
5440 for (f = old_nproc; f < team->t.t_nproc; ++f) in __kmp_allocate_team()
5441 team->t.t_threads[f]->th.th_task_state = old_state; in __kmp_allocate_team()
5444 for (f = 0; f < team->t.t_nproc; ++f) { in __kmp_allocate_team()
5445 KMP_DEBUG_ASSERT(team->t.t_threads[f] && in __kmp_allocate_team()
5446 team->t.t_threads[f]->th.th_team_nproc == in __kmp_allocate_team()
5447 team->t.t_nproc); in __kmp_allocate_team()
5452 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); in __kmp_allocate_team()
5454 __kmp_partition_places(team); in __kmp_allocate_team()
5459 kmp_info_t *master = team->t.t_threads[0]; in __kmp_allocate_team()
5463 kmp_info_t *thr = team->t.t_threads[f]; in __kmp_allocate_team()
5474 kmp_info_t *thr = team->t.t_threads[f]; in __kmp_allocate_team()
5478 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; in __kmp_allocate_team()
5481 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; in __kmp_allocate_team()
5489 __kmp_alloc_argv_entries(argc, team, TRUE); in __kmp_allocate_team()
5490 KMP_CHECK_UPDATE(team->t.t_argc, argc); in __kmp_allocate_team()
5494 KF_TRACE(10, (" hot_team = %p\n", team)); in __kmp_allocate_team()
5500 team->t.t_task_team[0], team->t.t_task_team[1])); in __kmp_allocate_team()
5505 __ompt_team_assign_id(team, ompt_parallel_data); in __kmp_allocate_team()
5510 return team; in __kmp_allocate_team()
5515 for (team = CCAST(kmp_team_t *, __kmp_team_pool); (team);) { in __kmp_allocate_team()
5518 if (team->t.t_max_nproc >= max_nproc) { in __kmp_allocate_team()
5520 __kmp_team_pool = team->t.t_next_pool; in __kmp_allocate_team()
5524 if (!team->t.b) { // Allocate barrier structure in __kmp_allocate_team()
5525 team->t.b = distributedBarrier::allocate(__kmp_dflt_team_nth_ub); in __kmp_allocate_team()
5530 __kmp_initialize_team(team, new_nproc, new_icvs, NULL); in __kmp_allocate_team()
5534 &team->t.t_task_team[0], &team->t.t_task_team[1])); in __kmp_allocate_team()
5535 team->t.t_task_team[0] = NULL; in __kmp_allocate_team()
5536 team->t.t_task_team[1] = NULL; in __kmp_allocate_team()
5539 __kmp_alloc_argv_entries(argc, team, TRUE); in __kmp_allocate_team()
5540 KMP_CHECK_UPDATE(team->t.t_argc, argc); in __kmp_allocate_team()
5544 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); in __kmp_allocate_team()
5548 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE; in __kmp_allocate_team()
5550 team->t.t_bar[b].b_master_arrived = 0; in __kmp_allocate_team()
5551 team->t.t_bar[b].b_team_arrived = 0; in __kmp_allocate_team()
5556 team->t.t_proc_bind = new_proc_bind; in __kmp_allocate_team()
5559 team->t.t_id)); in __kmp_allocate_team()
5562 __ompt_team_assign_id(team, ompt_parallel_data); in __kmp_allocate_team()
5567 return team; in __kmp_allocate_team()
5574 team = __kmp_reap_team(team); in __kmp_allocate_team()
5575 __kmp_team_pool = team; in __kmp_allocate_team()
5580 team = (kmp_team_t *)__kmp_allocate(sizeof(kmp_team_t)); in __kmp_allocate_team()
5583 team->t.t_max_nproc = max_nproc; in __kmp_allocate_team()
5587 team->t.b = distributedBarrier::allocate(__kmp_dflt_team_nth_ub); in __kmp_allocate_team()
5592 __kmp_allocate_team_arrays(team, max_nproc); in __kmp_allocate_team()
5595 __kmp_initialize_team(team, new_nproc, new_icvs, NULL); in __kmp_allocate_team()
5599 &team->t.t_task_team[0], &team->t.t_task_team[1])); in __kmp_allocate_team()
5600 team->t.t_task_team[0] = NULL; // to be removed, as __kmp_allocate zeroes in __kmp_allocate_team()
5602 team->t.t_task_team[1] = NULL; // to be removed, as __kmp_allocate zeroes in __kmp_allocate_team()
5606 __kmp_print_team_storage_map("team", team, team->t.t_id, new_nproc); in __kmp_allocate_team()
5610 __kmp_alloc_argv_entries(argc, team, FALSE); in __kmp_allocate_team()
5611 team->t.t_argc = argc; in __kmp_allocate_team()
5615 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); in __kmp_allocate_team()
5619 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE; in __kmp_allocate_team()
5621 team->t.t_bar[b].b_master_arrived = 0; in __kmp_allocate_team()
5622 team->t.t_bar[b].b_team_arrived = 0; in __kmp_allocate_team()
5627 team->t.t_proc_bind = new_proc_bind; in __kmp_allocate_team()
5630 __ompt_team_assign_id(team, ompt_parallel_data); in __kmp_allocate_team()
5631 team->t.ompt_serialized_team_info = NULL; in __kmp_allocate_team()
5637 team->t.t_id)); in __kmp_allocate_team()
5639 return team; in __kmp_allocate_team()
5648 kmp_team_t *team USE_NESTED_HOT_ARG(kmp_info_t *master)) { in __kmp_free_team()
5651 team->t.t_id)); in __kmp_free_team()
5655 KMP_DEBUG_ASSERT(team); in __kmp_free_team()
5656 KMP_DEBUG_ASSERT(team->t.t_nproc <= team->t.t_max_nproc); in __kmp_free_team()
5657 KMP_DEBUG_ASSERT(team->t.t_threads); in __kmp_free_team()
5659 int use_hot_team = team == root->r.r_hot_team; in __kmp_free_team()
5663 level = team->t.t_active_level - 1; in __kmp_free_team()
5669 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master && in __kmp_free_team()
5670 master->th.th_teams_level == team->t.t_level) { in __kmp_free_team()
5679 KMP_DEBUG_ASSERT(team == hot_teams[level].hot_team); in __kmp_free_team()
5686 TCW_SYNC_PTR(team->t.t_pkfn, in __kmp_free_team()
5689 team->t.t_copyin_counter = 0; // init counter for possible reuse in __kmp_free_team()
5697 for (f = 1; f < team->t.t_nproc; ++f) { in __kmp_free_team()
5698 KMP_DEBUG_ASSERT(team->t.t_threads[f]); in __kmp_free_team()
5699 kmp_info_t *th = team->t.t_threads[f]; in __kmp_free_team()
5721 kmp_task_team_t *task_team = team->t.t_task_team[tt_idx]; in __kmp_free_team()
5723 for (f = 0; f < team->t.t_nproc; ++f) { // threads unref task teams in __kmp_free_team()
5724 KMP_DEBUG_ASSERT(team->t.t_threads[f]); in __kmp_free_team()
5725 team->t.t_threads[f]->th.th_task_team = NULL; in __kmp_free_team()
5730 __kmp_get_gtid(), task_team, team->t.t_id)); in __kmp_free_team()
5734 team->t.t_task_team[tt_idx] = NULL; in __kmp_free_team()
5740 team->t.t_parent = NULL; in __kmp_free_team()
5741 team->t.t_level = 0; in __kmp_free_team()
5742 team->t.t_active_level = 0; in __kmp_free_team()
5745 for (f = 1; f < team->t.t_nproc; ++f) { in __kmp_free_team()
5746 KMP_DEBUG_ASSERT(team->t.t_threads[f]); in __kmp_free_team()
5748 KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team), in __kmp_free_team()
5751 __kmp_free_thread(team->t.t_threads[f]); in __kmp_free_team()
5755 if (team->t.b) { in __kmp_free_team()
5757 team->t.b->go_release(); in __kmp_free_team()
5759 for (f = 1; f < team->t.t_nproc; ++f) { in __kmp_free_team()
5760 if (team->t.b->sleep[f].sleep) { in __kmp_free_team()
5762 team->t.t_threads[f]->th.th_info.ds.ds_gtid, in __kmp_free_team()
5768 for (int f = 1; f < team->t.t_nproc; ++f) { in __kmp_free_team()
5769 while (team->t.t_threads[f]->th.th_used_in_team.load() != 0) in __kmp_free_team()
5775 for (f = 1; f < team->t.t_nproc; ++f) { in __kmp_free_team()
5776 team->t.t_threads[f] = NULL; in __kmp_free_team()
5779 if (team->t.t_max_nproc > 1 && in __kmp_free_team()
5781 distributedBarrier::deallocate(team->t.b); in __kmp_free_team()
5782 team->t.b = NULL; in __kmp_free_team()
5786 team->t.t_next_pool = CCAST(kmp_team_t *, __kmp_team_pool); in __kmp_free_team()
5787 __kmp_team_pool = (volatile kmp_team_t *)team; in __kmp_free_team()
5790 KMP_DEBUG_ASSERT(team->t.t_threads[1] && in __kmp_free_team()
5791 team->t.t_threads[1]->th.th_cg_roots); in __kmp_free_team()
5792 if (team->t.t_threads[1]->th.th_cg_roots->cg_root == team->t.t_threads[1]) { in __kmp_free_team()
5794 for (f = 1; f < team->t.t_nproc; ++f) { in __kmp_free_team()
5795 kmp_info_t *thr = team->t.t_threads[f]; in __kmp_free_team()
5820 kmp_team_t *__kmp_reap_team(kmp_team_t *team) { in __kmp_reap_team() argument
5821 kmp_team_t *next_pool = team->t.t_next_pool; in __kmp_reap_team()
5823 KMP_DEBUG_ASSERT(team); in __kmp_reap_team()
5824 KMP_DEBUG_ASSERT(team->t.t_dispatch); in __kmp_reap_team()
5825 KMP_DEBUG_ASSERT(team->t.t_disp_buffer); in __kmp_reap_team()
5826 KMP_DEBUG_ASSERT(team->t.t_threads); in __kmp_reap_team()
5827 KMP_DEBUG_ASSERT(team->t.t_argv); in __kmp_reap_team()
5832 __kmp_free_team_arrays(team); in __kmp_reap_team()
5833 if (team->t.t_argv != &team->t.t_inline_argv[0]) in __kmp_reap_team()
5834 __kmp_free((void *)team->t.t_argv); in __kmp_reap_team()
5835 __kmp_free(team); in __kmp_reap_team()
5882 balign[b].bb.team = NULL; in __kmp_free_thread()
6386 kmp_team_t *team = CCAST(kmp_team_t *, __kmp_team_pool); in __kmp_internal_end() local
6387 __kmp_team_pool = team->t.t_next_pool; in __kmp_internal_end()
6389 team->t.t_next_pool = NULL; in __kmp_internal_end()
6390 __kmp_reap_team(team); in __kmp_internal_end()
7633 kmp_team_t *team) { in __kmp_run_before_invoked_task() argument
7645 KMP_DEBUG_ASSERT(team->t.t_dispatch); in __kmp_run_before_invoked_task()
7652 __kmp_push_parallel(gtid, team->t.t_ident); in __kmp_run_before_invoked_task()
7658 kmp_team_t *team) { in __kmp_run_after_invoked_task() argument
7660 __kmp_pop_parallel(gtid, team->t.t_ident); in __kmp_run_after_invoked_task()
7669 kmp_team_t *team = this_thr->th.th_team; in __kmp_invoke_task_func() local
7671 __kmp_run_before_invoked_task(gtid, tid, this_thr, team); in __kmp_invoke_task_func()
7675 if (team->t.t_stack_id != NULL) { in __kmp_invoke_task_func()
7676 __kmp_itt_stack_callee_enter((__itt_caller)team->t.t_stack_id); in __kmp_invoke_task_func()
7678 KMP_DEBUG_ASSERT(team->t.t_parent->t.t_stack_id != NULL); in __kmp_invoke_task_func()
7680 (__itt_caller)team->t.t_parent->t.t_stack_id); in __kmp_invoke_task_func()
7696 exit_frame_p = &(team->t.t_implicit_task_taskdata[tid] in __kmp_invoke_task_func()
7703 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data); in __kmp_invoke_task_func()
7704 my_parallel_data = &(team->t.ompt_team_info.parallel_data); in __kmp_invoke_task_func()
7706 ompt_team_size = team->t.t_nproc; in __kmp_invoke_task_func()
7724 rc = __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn), gtid, in __kmp_invoke_task_func()
7725 tid, (int)team->t.t_argc, (void **)team->t.t_argv in __kmp_invoke_task_func()
7746 if (team->t.t_stack_id != NULL) { in __kmp_invoke_task_func()
7747 __kmp_itt_stack_callee_leave((__itt_caller)team->t.t_stack_id); in __kmp_invoke_task_func()
7749 KMP_DEBUG_ASSERT(team->t.t_parent->t.t_stack_id != NULL); in __kmp_invoke_task_func()
7751 (__itt_caller)team->t.t_parent->t.t_stack_id); in __kmp_invoke_task_func()
7755 __kmp_run_after_invoked_task(gtid, tid, this_thr, team); in __kmp_invoke_task_func()
7763 kmp_team_t *team = thr->th.th_team; in __kmp_teams_master() local
7764 ident_t *loc = team->t.t_ident; in __kmp_teams_master()
7788 __kmp_fork_call(loc, gtid, fork_context_intel, team->t.t_argc, in __kmp_teams_master()
7810 kmp_team_t *team = this_thr->th.th_team; in __kmp_invoke_teams_master() local
7816 __kmp_run_before_invoked_task(gtid, 0, this_thr, team); in __kmp_invoke_teams_master()
7820 &team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data; in __kmp_invoke_teams_master()
7821 ompt_data_t *parallel_data = &team->t.ompt_team_info.parallel_data; in __kmp_invoke_teams_master()
7824 ompt_scope_begin, parallel_data, task_data, team->t.t_nproc, tid, in __kmp_invoke_teams_master()
7833 __kmp_run_after_invoked_task(gtid, 0, this_thr, team); in __kmp_invoke_teams_master()
8012 void __kmp_internal_fork(ident_t *id, int gtid, kmp_team_t *team) { in __kmp_internal_fork() argument
8019 KMP_DEBUG_ASSERT(team); in __kmp_internal_fork()
8020 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); in __kmp_internal_fork()
8024 team->t.t_construct = 0; /* no single directives seen yet */ in __kmp_internal_fork()
8025 team->t.t_ordered.dt.t_value = in __kmp_internal_fork()
8029 KMP_DEBUG_ASSERT(team->t.t_disp_buffer); in __kmp_internal_fork()
8030 if (team->t.t_max_nproc > 1) { in __kmp_internal_fork()
8033 team->t.t_disp_buffer[i].buffer_index = i; in __kmp_internal_fork()
8034 team->t.t_disp_buffer[i].doacross_buf_idx = i; in __kmp_internal_fork()
8037 team->t.t_disp_buffer[0].buffer_index = 0; in __kmp_internal_fork()
8038 team->t.t_disp_buffer[0].doacross_buf_idx = 0; in __kmp_internal_fork()
8042 KMP_ASSERT(this_thr->th.th_team == team); in __kmp_internal_fork()
8045 for (f = 0; f < team->t.t_nproc; f++) { in __kmp_internal_fork()
8046 KMP_DEBUG_ASSERT(team->t.t_threads[f] && in __kmp_internal_fork()
8047 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc); in __kmp_internal_fork()
8055 void __kmp_internal_join(ident_t *id, int gtid, kmp_team_t *team) { in __kmp_internal_join() argument
8058 KMP_DEBUG_ASSERT(team); in __kmp_internal_join()
8059 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); in __kmp_internal_join()
8067 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) { in __kmp_internal_join()
8072 gtid, __kmp_threads[gtid]->th.th_team_nproc, team, in __kmp_internal_join()
8073 team->t.t_nproc); in __kmp_internal_join()
8077 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc); in __kmp_internal_join()
8114 KMP_ASSERT(this_thr->th.th_team == team); in __kmp_internal_join()
8476 kmp_team_t *team = thr->th.th_team; in __kmp_aux_get_team_info() local
8478 int ii = team->t.t_level; in __kmp_aux_get_team_info()
8479 teams_serialized = team->t.t_serialized; in __kmp_aux_get_team_info()
8483 for (teams_serialized = team->t.t_serialized; in __kmp_aux_get_team_info()
8486 if (team->t.t_serialized && (!teams_serialized)) { in __kmp_aux_get_team_info()
8487 team = team->t.t_parent; in __kmp_aux_get_team_info()
8491 team = team->t.t_parent; in __kmp_aux_get_team_info()
8495 return team; in __kmp_aux_get_team_info()
8502 kmp_team_t *team = __kmp_aux_get_team_info(serialized); in __kmp_aux_get_team_num() local
8503 if (team) { in __kmp_aux_get_team_num()
8507 return team->t.t_master_tid; in __kmp_aux_get_team_num()
8515 kmp_team_t *team = __kmp_aux_get_team_info(serialized); in __kmp_aux_get_num_teams() local
8516 if (team) { in __kmp_aux_get_num_teams()
8520 return team->t.t_parent->t.t_nproc; in __kmp_aux_get_num_teams()
9107 void __kmp_resize_dist_barrier(kmp_team_t *team, int old_nthreads, in __kmp_resize_dist_barrier() argument
9111 kmp_info_t **other_threads = team->t.t_threads; in __kmp_resize_dist_barrier()
9118 if (team->t.t_threads[f]->th.th_used_in_team.load() == 0) { in __kmp_resize_dist_barrier()
9124 if (team->t.t_threads[f]->th.th_used_in_team.load() == 3) { in __kmp_resize_dist_barrier()
9125 while (team->t.t_threads[f]->th.th_used_in_team.load() == 3) in __kmp_resize_dist_barrier()
9129 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 1); in __kmp_resize_dist_barrier()
9131 team->t.t_threads[f]->th.th_used_in_team.store(2); in __kmp_resize_dist_barrier()
9132 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 2); in __kmp_resize_dist_barrier()
9135 team->t.b->go_release(); in __kmp_resize_dist_barrier()
9152 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 0); in __kmp_resize_dist_barrier()
9158 team->t.b->update_num_threads(new_nthreads); in __kmp_resize_dist_barrier()
9159 team->t.b->go_reset(); in __kmp_resize_dist_barrier()
9162 void __kmp_add_threads_to_team(kmp_team_t *team, int new_nthreads) { in __kmp_add_threads_to_team() argument
9164 KMP_DEBUG_ASSERT(team); in __kmp_add_threads_to_team()
9171 KMP_DEBUG_ASSERT(team->t.t_threads[f]); in __kmp_add_threads_to_team()
9172 KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team), 0, in __kmp_add_threads_to_team()
9175 __kmp_resume_32(team->t.t_threads[f]->th.th_info.ds.ds_gtid, in __kmp_add_threads_to_team()
9186 if (team->t.t_threads[f]->th.th_used_in_team.load() == 1) { in __kmp_add_threads_to_team()