Lines Matching refs:team
86 static void __kmp_initialize_team(kmp_team_t *team, int new_nproc,
90 static void __kmp_partition_places(kmp_team_t *team,
96 void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc,
110 void __kmp_resize_dist_barrier(kmp_team_t *team, int old_nthreads,
112 void __kmp_add_threads_to_team(kmp_team_t *team, int new_nthreads);
506 static void __kmp_print_team_storage_map(const char *header, kmp_team_t *team, in __kmp_print_team_storage_map() argument
508 int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2; in __kmp_print_team_storage_map()
509 __kmp_print_storage_map_gtid(-1, team, team + 1, sizeof(kmp_team_t), "%s_%d", in __kmp_print_team_storage_map()
512 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0], in __kmp_print_team_storage_map()
513 &team->t.t_bar[bs_last_barrier], in __kmp_print_team_storage_map()
517 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier], in __kmp_print_team_storage_map()
518 &team->t.t_bar[bs_plain_barrier + 1], in __kmp_print_team_storage_map()
522 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier], in __kmp_print_team_storage_map()
523 &team->t.t_bar[bs_forkjoin_barrier + 1], in __kmp_print_team_storage_map()
528 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier], in __kmp_print_team_storage_map()
529 &team->t.t_bar[bs_reduction_barrier + 1], in __kmp_print_team_storage_map()
535 -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr], in __kmp_print_team_storage_map()
539 -1, &team->t.t_threads[0], &team->t.t_threads[num_thr], in __kmp_print_team_storage_map()
542 __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0], in __kmp_print_team_storage_map()
543 &team->t.t_disp_buffer[num_disp_buff], in __kmp_print_team_storage_map()
612 kmp_team_t *team = __kmp_team_from_gtid(gtid); in __kmp_parallel_deo() local
624 if (!team->t.t_serialized) { in __kmp_parallel_deo()
626 KMP_WAIT(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid), KMP_EQ, in __kmp_parallel_deo()
638 kmp_team_t *team = __kmp_team_from_gtid(gtid); in __kmp_parallel_dxo() local
646 if (!team->t.t_serialized) { in __kmp_parallel_dxo()
651 team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc); in __kmp_parallel_dxo()
664 kmp_team_t *team; in __kmp_enter_single() local
671 team = th->th.th_team; in __kmp_enter_single()
676 if (team->t.t_serialized) { in __kmp_enter_single()
685 if (team->t.t_construct == old_this) { in __kmp_enter_single()
686 status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this, in __kmp_enter_single()
692 team->t.t_active_level == 1) { in __kmp_enter_single()
916 static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team, in __kmp_fork_team_threads() argument
922 KA_TRACE(10, ("__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc)); in __kmp_fork_team_threads()
928 master_th->th.th_team = team; in __kmp_fork_team_threads()
929 master_th->th.th_team_nproc = team->t.t_nproc; in __kmp_fork_team_threads()
932 master_th->th.th_dispatch = &team->t.t_dispatch[0]; in __kmp_fork_team_threads()
940 int level = team->t.t_active_level - 1; // index in array of hot teams in __kmp_fork_team_threads()
946 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master && in __kmp_fork_team_threads()
947 master_th->th.th_teams_level == team->t.t_level) { in __kmp_fork_team_threads()
955 KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team); in __kmp_fork_team_threads()
959 hot_teams[level].hot_team = team; // remember new hot team in __kmp_fork_team_threads()
960 hot_teams[level].hot_team_nth = team->t.t_nproc; in __kmp_fork_team_threads()
967 use_hot_team = team == root->r.r_hot_team; in __kmp_fork_team_threads()
972 team->t.t_threads[0] = master_th; in __kmp_fork_team_threads()
973 __kmp_initialize_info(master_th, team, 0, master_gtid); in __kmp_fork_team_threads()
976 for (i = 1; i < team->t.t_nproc; i++) { in __kmp_fork_team_threads()
979 kmp_info_t *thr = __kmp_allocate_thread(root, team, i); in __kmp_fork_team_threads()
980 team->t.t_threads[i] = thr; in __kmp_fork_team_threads()
982 KMP_DEBUG_ASSERT(thr->th.th_team == team); in __kmp_fork_team_threads()
986 __kmp_gtid_from_tid(0, team), team->t.t_id, 0, in __kmp_fork_team_threads()
987 __kmp_gtid_from_tid(i, team), team->t.t_id, i, in __kmp_fork_team_threads()
988 team->t.t_bar[bs_forkjoin_barrier].b_arrived, in __kmp_fork_team_threads()
989 team->t.t_bar[bs_plain_barrier].b_arrived)); in __kmp_fork_team_threads()
995 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar; in __kmp_fork_team_threads()
997 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; in __kmp_fork_team_threads()
1000 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; in __kmp_fork_team_threads()
1011 __kmp_partition_places(team); in __kmp_fork_team_threads()
1016 if (__kmp_display_affinity && team->t.t_display_affinity != 1) { in __kmp_fork_team_threads()
1017 for (i = 0; i < team->t.t_nproc; i++) { in __kmp_fork_team_threads()
1018 kmp_info_t *thr = team->t.t_threads[i]; in __kmp_fork_team_threads()
1019 if (thr->th.th_prev_num_threads != team->t.t_nproc || in __kmp_fork_team_threads()
1020 thr->th.th_prev_level != team->t.t_level) { in __kmp_fork_team_threads()
1021 team->t.t_display_affinity = 1; in __kmp_fork_team_threads()
1034 inline static void propagateFPControl(kmp_team_t *team) { in propagateFPControl() argument
1053 KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word); in propagateFPControl()
1054 KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr); in propagateFPControl()
1057 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE); in propagateFPControl()
1061 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE); in propagateFPControl()
1067 inline static void updateHWFPControl(kmp_team_t *team) { in updateHWFPControl() argument
1068 if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) { in updateHWFPControl()
1077 if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) { in updateHWFPControl()
1079 __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word); in updateHWFPControl()
1082 if (team->t.t_mxcsr != mxcsr) { in updateHWFPControl()
1083 __kmp_load_mxcsr(&team->t.t_mxcsr); in updateHWFPControl()
1092 static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team,
1363 kmp_team_t *team; in __kmp_fork_call() local
1840 team = master_th->th.th_team; in __kmp_fork_call()
1842 team->t.t_invoke = invoker; in __kmp_fork_call()
1843 __kmp_alloc_argv_entries(argc, team, TRUE); in __kmp_fork_call()
1844 team->t.t_argc = argc; in __kmp_fork_call()
1845 argv = (void **)team->t.t_argv; in __kmp_fork_call()
1856 team->t.t_level--; in __kmp_fork_call()
2046 team = __kmp_allocate_team(root, nthreads, nthreads, in __kmp_fork_call()
2053 copy_icvs((kmp_internal_control_t *)team->t.b->team_icvs, &new_icvs); in __kmp_fork_call()
2057 team = __kmp_allocate_team(root, nthreads, nthreads, in __kmp_fork_call()
2065 copy_icvs((kmp_internal_control_t *)team->t.b->team_icvs, in __kmp_fork_call()
2069 10, ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team)); in __kmp_fork_call()
2072 KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid); in __kmp_fork_call()
2073 KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons); in __kmp_fork_call()
2074 KMP_CHECK_UPDATE(team->t.t_ident, loc); in __kmp_fork_call()
2075 KMP_CHECK_UPDATE(team->t.t_parent, parent_team); in __kmp_fork_call()
2076 KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask); in __kmp_fork_call()
2078 KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.master_return_address, in __kmp_fork_call()
2081 KMP_CHECK_UPDATE(team->t.t_invoke, invoker); // TODO move to root, maybe in __kmp_fork_call()
2085 KMP_CHECK_UPDATE(team->t.t_level, new_level); in __kmp_fork_call()
2087 KMP_CHECK_UPDATE(team->t.t_active_level, new_level); in __kmp_fork_call()
2091 KMP_CHECK_UPDATE(team->t.t_level, new_level); in __kmp_fork_call()
2093 KMP_CHECK_UPDATE(team->t.t_active_level, new_level); in __kmp_fork_call()
2097 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched); in __kmp_fork_call()
2099 KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq); in __kmp_fork_call()
2100 KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator); in __kmp_fork_call()
2103 propagateFPControl(team); in __kmp_fork_call()
2118 team->t.t_task_team[master_th->th.th_task_state], team)); in __kmp_fork_call()
2149 team == master_th->th.th_hot_teams[active_level].hot_team) { in __kmp_fork_call()
2163 (team == root->r.r_hot_team)); in __kmp_fork_call()
2170 gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id, in __kmp_fork_call()
2171 team->t.t_nproc)); in __kmp_fork_call()
2172 KMP_DEBUG_ASSERT(team != root->r.r_hot_team || in __kmp_fork_call()
2173 (team->t.t_master_tid == 0 && in __kmp_fork_call()
2174 (team->t.t_parent == root->r.r_root_team || in __kmp_fork_call()
2175 team->t.t_parent->t.t_serialized))); in __kmp_fork_call()
2179 argv = (void **)team->t.t_argv; in __kmp_fork_call()
2189 KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i]); in __kmp_fork_call()
2194 KMP_CHECK_UPDATE(team->t.t_master_active, master_active); in __kmp_fork_call()
2198 __kmp_fork_team_threads(root, team, master_th, gtid, !ap); in __kmp_fork_call()
2199 __kmp_setup_icv_copy(team, nthreads, in __kmp_fork_call()
2209 if (team->t.t_active_level == 1 // only report frames at level 1 in __kmp_fork_call()
2221 team->t.t_region_time = tmp_time; in __kmp_fork_call()
2228 __kmp_itt_region_forking(gtid, team->t.t_nproc, 0); in __kmp_fork_call()
2234 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team); in __kmp_fork_call()
2238 root, team, master_th, gtid)); in __kmp_fork_call()
2244 KMP_DEBUG_ASSERT(team->t.t_stack_id == NULL); in __kmp_fork_call()
2245 team->t.t_stack_id = __kmp_itt_stack_caller_create(); in __kmp_fork_call()
2260 __kmp_internal_fork(loc, gtid, team); in __kmp_fork_call()
2263 root, team, master_th, gtid)); in __kmp_fork_call()
2273 team->t.t_id, team->t.t_pkfn)); in __kmp_fork_call()
2284 if (!team->t.t_invoke(gtid)) { in __kmp_fork_call()
2296 team->t.t_id, team->t.t_pkfn)); in __kmp_fork_call()
2311 kmp_team_t *team) { in __kmp_join_restore_state() argument
2314 ((team->t.t_serialized) ? ompt_state_work_serial in __kmp_join_restore_state()
2319 kmp_team_t *team, ompt_data_t *parallel_data, in __kmp_join_ompt() argument
2328 __kmp_join_restore_state(thread, team); in __kmp_join_ompt()
2340 kmp_team_t *team; in __kmp_join_call() local
2351 team = master_th->th.th_team; in __kmp_join_call()
2352 parent_team = team->t.t_parent; in __kmp_join_call()
2357 void *team_microtask = (void *)team->t.t_pkfn; in __kmp_join_call()
2362 !(team->t.t_serialized && fork_context == fork_context_gnu)) { in __kmp_join_call()
2371 __kmp_gtid_from_thread(master_th), team, in __kmp_join_call()
2372 team->t.t_task_team[master_th->th.th_task_state], in __kmp_join_call()
2375 team->t.t_task_team[master_th->th.th_task_state]); in __kmp_join_call()
2379 if (team->t.t_serialized) { in __kmp_join_call()
2382 int level = team->t.t_level; in __kmp_join_call()
2387 team->t.t_level++; in __kmp_join_call()
2392 team->t.t_serialized++; in __kmp_join_call()
2406 master_active = team->t.t_master_active; in __kmp_join_call()
2411 __kmp_internal_join(loc, gtid, team); in __kmp_join_call()
2414 KMP_DEBUG_ASSERT(team->t.t_stack_id != NULL); in __kmp_join_call()
2416 __kmp_itt_stack_caller_destroy((__itt_caller)team->t.t_stack_id); in __kmp_join_call()
2417 team->t.t_stack_id = NULL; in __kmp_join_call()
2434 if (team->t.t_nproc > 1 && in __kmp_join_call()
2436 team->t.b->update_num_threads(team->t.t_nproc); in __kmp_join_call()
2437 __kmp_add_threads_to_team(team, team->t.t_nproc); in __kmp_join_call()
2444 ompt_data_t *parallel_data = &(team->t.ompt_team_info.parallel_data); in __kmp_join_call()
2445 void *codeptr = team->t.ompt_team_info.master_return_address; in __kmp_join_call()
2450 if (team->t.t_active_level == 1 && in __kmp_join_call()
2458 __kmp_itt_frame_submit(gtid, team->t.t_region_time, in __kmp_join_call()
2470 master_th->th.th_first_place = team->t.t_first_place; in __kmp_join_call()
2471 master_th->th.th_last_place = team->t.t_last_place; in __kmp_join_call()
2476 team->t.t_pkfn != (microtask_t)__kmp_teams_master && in __kmp_join_call()
2477 team->t.t_level == master_th->th.th_teams_level + 1) { in __kmp_join_call()
2486 int ompt_team_size = team->t.t_nproc; in __kmp_join_call()
2498 team->t.t_level--; in __kmp_join_call()
2499 team->t.t_active_level--; in __kmp_join_call()
2509 kmp_info_t **other_threads = team->t.t_threads; in __kmp_join_call()
2510 team->t.t_nproc = new_num; in __kmp_join_call()
2520 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; in __kmp_join_call()
2523 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; in __kmp_join_call()
2544 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid; in __kmp_join_call()
2545 master_th->th.th_local.this_construct = team->t.t_master_this_cons; in __kmp_join_call()
2547 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid]; in __kmp_join_call()
2555 team->t.t_level > master_th->th.th_teams_level) { in __kmp_join_call()
2568 int ompt_team_size = (flags == ompt_task_initial) ? 0 : team->t.t_nproc; in __kmp_join_call()
2579 master_th, team)); in __kmp_join_call()
2582 master_th->th.th_def_allocator = team->t.t_def_allocator; in __kmp_join_call()
2588 updateHWFPControl(team); in __kmp_join_call()
2593 __kmp_free_team(root, team USE_NESTED_HOT_ARG( in __kmp_join_call()
2959 kmp_team_t *team; in __kmp_get_ancestor_thread_num() local
2971 team = thr->th.th_team; in __kmp_get_ancestor_thread_num()
2972 ii = team->t.t_level; in __kmp_get_ancestor_thread_num()
2995 dd = team->t.t_serialized; in __kmp_get_ancestor_thread_num()
2998 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) { in __kmp_get_ancestor_thread_num()
3000 if ((team->t.t_serialized) && (!dd)) { in __kmp_get_ancestor_thread_num()
3001 team = team->t.t_parent; in __kmp_get_ancestor_thread_num()
3005 team = team->t.t_parent; in __kmp_get_ancestor_thread_num()
3006 dd = team->t.t_serialized; in __kmp_get_ancestor_thread_num()
3011 return (dd > 1) ? (0) : (team->t.t_master_tid); in __kmp_get_ancestor_thread_num()
3017 kmp_team_t *team; in __kmp_get_team_size() local
3029 team = thr->th.th_team; in __kmp_get_team_size()
3030 ii = team->t.t_level; in __kmp_get_team_size()
3051 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) { in __kmp_get_team_size()
3053 if (team->t.t_serialized && (!dd)) { in __kmp_get_team_size()
3054 team = team->t.t_parent; in __kmp_get_team_size()
3058 team = team->t.t_parent; in __kmp_get_team_size()
3063 return team->t.t_nproc; in __kmp_get_team_size()
3102 static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team, int realloc) { in __kmp_alloc_argv_entries() argument
3104 KMP_DEBUG_ASSERT(team); in __kmp_alloc_argv_entries()
3105 if (!realloc || argc > team->t.t_max_argc) { in __kmp_alloc_argv_entries()
3109 team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0)); in __kmp_alloc_argv_entries()
3111 if (realloc && team->t.t_argv != &team->t.t_inline_argv[0]) in __kmp_alloc_argv_entries()
3112 __kmp_free((void *)team->t.t_argv); in __kmp_alloc_argv_entries()
3116 team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES; in __kmp_alloc_argv_entries()
3119 team->t.t_id, team->t.t_max_argc)); in __kmp_alloc_argv_entries()
3120 team->t.t_argv = &team->t.t_inline_argv[0]; in __kmp_alloc_argv_entries()
3123 -1, &team->t.t_inline_argv[0], in __kmp_alloc_argv_entries()
3124 &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES], in __kmp_alloc_argv_entries()
3126 team->t.t_id); in __kmp_alloc_argv_entries()
3130 team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES >> 1)) in __kmp_alloc_argv_entries()
3135 team->t.t_id, team->t.t_max_argc)); in __kmp_alloc_argv_entries()
3136 team->t.t_argv = in __kmp_alloc_argv_entries()
3137 (void **)__kmp_page_allocate(sizeof(void *) * team->t.t_max_argc); in __kmp_alloc_argv_entries()
3139 __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0], in __kmp_alloc_argv_entries()
3140 &team->t.t_argv[team->t.t_max_argc], in __kmp_alloc_argv_entries()
3141 sizeof(void *) * team->t.t_max_argc, in __kmp_alloc_argv_entries()
3142 "team_%d.t_argv", team->t.t_id); in __kmp_alloc_argv_entries()
3148 static void __kmp_allocate_team_arrays(kmp_team_t *team, int max_nth) { in __kmp_allocate_team_arrays() argument
3151 team->t.t_threads = in __kmp_allocate_team_arrays()
3153 team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate( in __kmp_allocate_team_arrays()
3155 team->t.t_dispatch = in __kmp_allocate_team_arrays()
3157 team->t.t_implicit_task_taskdata = in __kmp_allocate_team_arrays()
3159 team->t.t_max_nproc = max_nth; in __kmp_allocate_team_arrays()
3163 team->t.t_disp_buffer[i].buffer_index = i; in __kmp_allocate_team_arrays()
3164 team->t.t_disp_buffer[i].doacross_buf_idx = i; in __kmp_allocate_team_arrays()
3168 static void __kmp_free_team_arrays(kmp_team_t *team) { in __kmp_free_team_arrays() argument
3171 for (i = 0; i < team->t.t_max_nproc; ++i) { in __kmp_free_team_arrays()
3172 if (team->t.t_dispatch[i].th_disp_buffer != NULL) { in __kmp_free_team_arrays()
3173 __kmp_free(team->t.t_dispatch[i].th_disp_buffer); in __kmp_free_team_arrays()
3174 team->t.t_dispatch[i].th_disp_buffer = NULL; in __kmp_free_team_arrays()
3178 __kmp_dispatch_free_hierarchies(team); in __kmp_free_team_arrays()
3180 __kmp_free(team->t.t_threads); in __kmp_free_team_arrays()
3181 __kmp_free(team->t.t_disp_buffer); in __kmp_free_team_arrays()
3182 __kmp_free(team->t.t_dispatch); in __kmp_free_team_arrays()
3183 __kmp_free(team->t.t_implicit_task_taskdata); in __kmp_free_team_arrays()
3184 team->t.t_threads = NULL; in __kmp_free_team_arrays()
3185 team->t.t_disp_buffer = NULL; in __kmp_free_team_arrays()
3186 team->t.t_dispatch = NULL; in __kmp_free_team_arrays()
3187 team->t.t_implicit_task_taskdata = 0; in __kmp_free_team_arrays()
3190 static void __kmp_reallocate_team_arrays(kmp_team_t *team, int max_nth) { in __kmp_reallocate_team_arrays() argument
3191 kmp_info_t **oldThreads = team->t.t_threads; in __kmp_reallocate_team_arrays()
3193 __kmp_free(team->t.t_disp_buffer); in __kmp_reallocate_team_arrays()
3194 __kmp_free(team->t.t_dispatch); in __kmp_reallocate_team_arrays()
3195 __kmp_free(team->t.t_implicit_task_taskdata); in __kmp_reallocate_team_arrays()
3196 __kmp_allocate_team_arrays(team, max_nth); in __kmp_reallocate_team_arrays()
3198 KMP_MEMCPY(team->t.t_threads, oldThreads, in __kmp_reallocate_team_arrays()
3199 team->t.t_nproc * sizeof(kmp_info_t *)); in __kmp_reallocate_team_arrays()
3238 static kmp_internal_control_t __kmp_get_x_global_icvs(const kmp_team_t *team) { in __kmp_get_x_global_icvs() argument
3243 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs); in __kmp_get_x_global_icvs()
3351 kmp_team_p const *team // Team to add. in __kmp_print_structure_team_accum() argument
3362 if (team == NULL) { in __kmp_print_structure_team_accum()
3366 __kmp_print_structure_team_accum(list, team->t.t_parent); in __kmp_print_structure_team_accum()
3367 __kmp_print_structure_team_accum(list, team->t.t_next_pool); in __kmp_print_structure_team_accum()
3371 while (l->next != NULL && l->entry != team) { in __kmp_print_structure_team_accum()
3380 while (l->next != NULL && l->entry->t.t_id <= team->t.t_id) { in __kmp_print_structure_team_accum()
3389 l->entry = team; in __kmp_print_structure_team_accum()
3394 static void __kmp_print_structure_team(char const *title, kmp_team_p const *team in __kmp_print_structure_team() argument
3398 if (team != NULL) { in __kmp_print_structure_team()
3399 __kmp_printf("%2x %p\n", team->t.t_id, team); in __kmp_print_structure_team()
3499 kmp_team_p const *team = list->entry; in __kmp_print_structure() local
3501 __kmp_printf("Team %2x %p:\n", team->t.t_id, team); in __kmp_print_structure()
3502 __kmp_print_structure_team(" Parent Team: ", team->t.t_parent); in __kmp_print_structure()
3503 __kmp_printf(" Primary TID: %2d\n", team->t.t_master_tid); in __kmp_print_structure()
3504 __kmp_printf(" Max threads: %2d\n", team->t.t_max_nproc); in __kmp_print_structure()
3505 __kmp_printf(" Levels of serial: %2d\n", team->t.t_serialized); in __kmp_print_structure()
3506 __kmp_printf(" Number threads: %2d\n", team->t.t_nproc); in __kmp_print_structure()
3507 for (i = 0; i < team->t.t_nproc; ++i) { in __kmp_print_structure()
3509 __kmp_print_structure_thread("", team->t.t_threads[i]); in __kmp_print_structure()
3511 __kmp_print_structure_team(" Next in pool: ", team->t.t_next_pool); in __kmp_print_structure()
3979 kmp_team_t *team = hot_teams[level].hot_team; in __kmp_free_hot_teams() local
3984 kmp_info_t *th = team->t.t_threads[i]; in __kmp_free_hot_teams()
3992 __kmp_free_team(root, team, NULL); in __kmp_free_hot_teams()
4115 kmp_team_t *team = thread->th.th_team; in __kmp_unregister_root_current_thread() local
4125 __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL)); in __kmp_unregister_root_current_thread()
4165 kmp_team_t *team = this_thr->th.th_team; in __kmp_task_info() local
4170 gtid, tid, this_thr, team, steam, this_thr->th.th_current_task, in __kmp_task_info()
4171 team->t.t_implicit_task_taskdata[tid].td_parent); in __kmp_task_info()
4178 static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team, in __kmp_initialize_info() argument
4185 KMP_DEBUG_ASSERT(team); in __kmp_initialize_info()
4186 KMP_DEBUG_ASSERT(team->t.t_threads); in __kmp_initialize_info()
4187 KMP_DEBUG_ASSERT(team->t.t_dispatch); in __kmp_initialize_info()
4188 kmp_info_t *master = team->t.t_threads[0]; in __kmp_initialize_info()
4194 TCW_SYNC_PTR(this_thr->th.th_team, team); in __kmp_initialize_info()
4211 this_thr->th.th_team_nproc = team->t.t_nproc; in __kmp_initialize_info()
4213 this_thr->th.th_team_serialized = team->t.t_serialized; in __kmp_initialize_info()
4215 KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata); in __kmp_initialize_info()
4221 team, tid, TRUE); in __kmp_initialize_info()
4229 this_thr->th.th_dispatch = &team->t.t_dispatch[tid]; in __kmp_initialize_info()
4277 (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers); in __kmp_initialize_info()
4279 team->t.t_max_nproc)); in __kmp_initialize_info()
4281 KMP_DEBUG_ASSERT(team->t.t_dispatch); in __kmp_initialize_info()
4282 KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid]); in __kmp_initialize_info()
4293 &dispatch->th_disp_buffer[team->t.t_max_nproc == 1 in __kmp_initialize_info()
4299 gtid, team->t.t_id, gtid); in __kmp_initialize_info()
4336 kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team, in __kmp_allocate_thread() argument
4343 KMP_DEBUG_ASSERT(root && team); in __kmp_allocate_thread()
4372 __kmp_initialize_info(new_thr, team, new_tid, in __kmp_allocate_thread()
4509 kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team); in __kmp_allocate_thread()
4528 __kmp_initialize_info(new_thr, team, new_tid, new_gtid); in __kmp_allocate_thread()
4550 balign[b].bb.team = NULL; in __kmp_allocate_thread()
4624 static void __kmp_reinitialize_team(kmp_team_t *team, in __kmp_reinitialize_team() argument
4628 team->t.t_threads[0], team)); in __kmp_reinitialize_team()
4629 KMP_DEBUG_ASSERT(team && new_icvs); in __kmp_reinitialize_team()
4631 KMP_CHECK_UPDATE(team->t.t_ident, loc); in __kmp_reinitialize_team()
4633 KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID()); in __kmp_reinitialize_team()
4635 __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE); in __kmp_reinitialize_team()
4636 copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs); in __kmp_reinitialize_team()
4639 team->t.t_threads[0], team)); in __kmp_reinitialize_team()
4645 static void __kmp_initialize_team(kmp_team_t *team, int new_nproc, in __kmp_initialize_team() argument
4648 KF_TRACE(10, ("__kmp_initialize_team: enter: team=%p\n", team)); in __kmp_initialize_team()
4651 KMP_DEBUG_ASSERT(team); in __kmp_initialize_team()
4652 KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc); in __kmp_initialize_team()
4653 KMP_DEBUG_ASSERT(team->t.t_threads); in __kmp_initialize_team()
4656 team->t.t_master_tid = 0; /* not needed */ in __kmp_initialize_team()
4658 team->t.t_serialized = new_nproc > 1 ? 0 : 1; in __kmp_initialize_team()
4659 team->t.t_nproc = new_nproc; in __kmp_initialize_team()
4662 team->t.t_next_pool = NULL; in __kmp_initialize_team()
4666 TCW_SYNC_PTR(team->t.t_pkfn, NULL); /* not needed */ in __kmp_initialize_team()
4667 team->t.t_invoke = NULL; /* not needed */ in __kmp_initialize_team()
4670 team->t.t_sched.sched = new_icvs->sched.sched; in __kmp_initialize_team()
4673 team->t.t_fp_control_saved = FALSE; /* not needed */ in __kmp_initialize_team()
4674 team->t.t_x87_fpu_control_word = 0; /* not needed */ in __kmp_initialize_team()
4675 team->t.t_mxcsr = 0; /* not needed */ in __kmp_initialize_team()
4678 team->t.t_construct = 0; in __kmp_initialize_team()
4680 team->t.t_ordered.dt.t_value = 0; in __kmp_initialize_team()
4681 team->t.t_master_active = FALSE; in __kmp_initialize_team()
4684 team->t.t_copypriv_data = NULL; /* not necessary, but nice for debugging */ in __kmp_initialize_team()
4687 team->t.t_copyin_counter = 0; /* for barrier-free copyin implementation */ in __kmp_initialize_team()
4690 team->t.t_control_stack_top = NULL; in __kmp_initialize_team()
4692 __kmp_reinitialize_team(team, new_icvs, loc); in __kmp_initialize_team()
4695 KF_TRACE(10, ("__kmp_initialize_team: exit: team=%p\n", team)); in __kmp_initialize_team()
4723 static void __kmp_partition_places(kmp_team_t *team, int update_master_only) { in __kmp_partition_places() argument
4725 if (KMP_HIDDEN_HELPER_TEAM(team)) in __kmp_partition_places()
4728 kmp_info_t *master_th = team->t.t_threads[0]; in __kmp_partition_places()
4730 kmp_proc_bind_t proc_bind = team->t.t_proc_bind; in __kmp_partition_places()
4734 team->t.t_first_place = first_place; in __kmp_partition_places()
4735 team->t.t_last_place = last_place; in __kmp_partition_places()
4739 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]), in __kmp_partition_places()
4740 team->t.t_id, masters_place, first_place, last_place)); in __kmp_partition_places()
4747 KMP_DEBUG_ASSERT(team->t.t_nproc == 1); in __kmp_partition_places()
4752 int n_th = team->t.t_nproc; in __kmp_partition_places()
4754 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places()
4760 team->t.t_display_affinity != 1) { in __kmp_partition_places()
4761 team->t.t_display_affinity = 1; in __kmp_partition_places()
4766 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, in __kmp_partition_places()
4773 int n_th = team->t.t_nproc; in __kmp_partition_places()
4783 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places()
4797 team->t.t_display_affinity != 1) { in __kmp_partition_places()
4798 team->t.t_display_affinity = 1; in __kmp_partition_places()
4803 __kmp_gtid_from_thread(team->t.t_threads[f]), in __kmp_partition_places()
4804 team->t.t_id, f, place, first_place, last_place)); in __kmp_partition_places()
4815 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places()
4822 team->t.t_display_affinity != 1) { in __kmp_partition_places()
4823 team->t.t_display_affinity = 1; in __kmp_partition_places()
4856 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f, in __kmp_partition_places()
4865 int n_th = team->t.t_nproc; in __kmp_partition_places()
4888 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places()
4894 team->t.t_display_affinity != 1) { in __kmp_partition_places()
4895 team->t.t_display_affinity = 1; in __kmp_partition_places()
4933 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, in __kmp_partition_places()
4983 th = team->t.t_threads[f]; in __kmp_partition_places()
4989 team->t.t_display_affinity != 1) { in __kmp_partition_places()
4990 team->t.t_display_affinity = 1; in __kmp_partition_places()
4995 __kmp_gtid_from_thread(team->t.t_threads[f]), in __kmp_partition_places()
4996 team->t.t_id, f, th->th.th_new_place, in __kmp_partition_places()
5014 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places()
5021 team->t.t_display_affinity != 1) { in __kmp_partition_places()
5022 team->t.t_display_affinity = 1; in __kmp_partition_places()
5054 __kmp_gtid_from_thread(team->t.t_threads[f]), in __kmp_partition_places()
5055 team->t.t_id, f, th->th.th_new_place, in __kmp_partition_places()
5066 KA_TRACE(20, ("__kmp_partition_places: exit T#%d\n", team->t.t_id)); in __kmp_partition_places()
5083 kmp_team_t *team; in __kmp_allocate_team() local
5096 team = master->th.th_team; in __kmp_allocate_team()
5097 level = team->t.t_active_level; in __kmp_allocate_team()
5101 team->t.t_pkfn == in __kmp_allocate_team()
5104 team->t.t_level)) { // or nested parallel inside the teams in __kmp_allocate_team()
5111 master->th.th_teams_level >= team->t.t_level) || in __kmp_allocate_team()
5112 (team->t.t_pkfn == (microtask_t)__kmp_teams_master)) in __kmp_allocate_team()
5132 team = hot_teams[level].hot_team; in __kmp_allocate_team()
5134 team = root->r.r_hot_team; in __kmp_allocate_team()
5140 team->t.t_task_team[0], team->t.t_task_team[1])); in __kmp_allocate_team()
5144 if (team->t.t_nproc != new_nproc && in __kmp_allocate_team()
5147 int old_nthr = team->t.t_nproc; in __kmp_allocate_team()
5148 __kmp_resize_dist_barrier(team, old_nthr, new_nproc); in __kmp_allocate_team()
5154 team->t.t_proc_bind = proc_bind_default; in __kmp_allocate_team()
5158 if (team->t.t_nproc == new_nproc) { // Check changes in number of threads in __kmp_allocate_team()
5162 if (team->t.t_size_changed == -1) { in __kmp_allocate_team()
5163 team->t.t_size_changed = 1; in __kmp_allocate_team()
5165 KMP_CHECK_UPDATE(team->t.t_size_changed, 0); in __kmp_allocate_team()
5171 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched); in __kmp_allocate_team()
5173 __kmp_reinitialize_team(team, new_icvs, in __kmp_allocate_team()
5177 team->t.t_threads[0], team)); in __kmp_allocate_team()
5178 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0); in __kmp_allocate_team()
5181 if ((team->t.t_size_changed == 0) && in __kmp_allocate_team()
5182 (team->t.t_proc_bind == new_proc_bind)) { in __kmp_allocate_team()
5186 __kmp_partition_places(team, 1); in __kmp_allocate_team()
5191 team->t.t_id, new_proc_bind, team->t.t_first_place, in __kmp_allocate_team()
5192 team->t.t_last_place)); in __kmp_allocate_team()
5195 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); in __kmp_allocate_team()
5196 __kmp_partition_places(team); in __kmp_allocate_team()
5200 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); in __kmp_allocate_team()
5202 } else if (team->t.t_nproc > new_nproc) { in __kmp_allocate_team()
5207 team->t.t_size_changed = 1; in __kmp_allocate_team()
5211 __kmp_add_threads_to_team(team, new_nproc); in __kmp_allocate_team()
5217 KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc); in __kmp_allocate_team()
5221 for (f = new_nproc; f < team->t.t_nproc; f++) { in __kmp_allocate_team()
5222 KMP_DEBUG_ASSERT(team->t.t_threads[f]); in __kmp_allocate_team()
5226 team->t.t_threads[f]->th.th_task_team = NULL; in __kmp_allocate_team()
5228 __kmp_free_thread(team->t.t_threads[f]); in __kmp_allocate_team()
5229 team->t.t_threads[f] = NULL; in __kmp_allocate_team()
5236 for (f = new_nproc; f < team->t.t_nproc; ++f) { in __kmp_allocate_team()
5237 KMP_DEBUG_ASSERT(team->t.t_threads[f]); in __kmp_allocate_team()
5238 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar; in __kmp_allocate_team()
5248 team->t.t_nproc = new_nproc; in __kmp_allocate_team()
5250 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_icvs->sched.sched); in __kmp_allocate_team()
5251 __kmp_reinitialize_team(team, new_icvs, in __kmp_allocate_team()
5256 team->t.t_threads[f]->th.th_team_nproc = new_nproc; in __kmp_allocate_team()
5262 team->t.t_threads[0], team)); in __kmp_allocate_team()
5264 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0); in __kmp_allocate_team()
5267 for (f = 0; f < team->t.t_nproc; f++) { in __kmp_allocate_team()
5268 KMP_DEBUG_ASSERT(team->t.t_threads[f] && in __kmp_allocate_team()
5269 team->t.t_threads[f]->th.th_team_nproc == in __kmp_allocate_team()
5270 team->t.t_nproc); in __kmp_allocate_team()
5275 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); in __kmp_allocate_team()
5277 __kmp_partition_places(team); in __kmp_allocate_team()
5291 int old_nproc = team->t.t_nproc; // save old value and use to update only in __kmp_allocate_team()
5292 team->t.t_size_changed = 1; in __kmp_allocate_team()
5298 kmp_info_t **other_threads = team->t.t_threads; in __kmp_allocate_team()
5299 for (f = team->t.t_nproc; f < avail_threads; ++f) { in __kmp_allocate_team()
5305 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; in __kmp_allocate_team()
5308 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; in __kmp_allocate_team()
5316 team->t.t_nproc = new_nproc; // just get reserved threads involved in __kmp_allocate_team()
5320 team->t.t_nproc = hot_teams[level].hot_team_nth; in __kmp_allocate_team()
5323 if (team->t.t_max_nproc < new_nproc) { in __kmp_allocate_team()
5325 __kmp_reallocate_team_arrays(team, new_nproc); in __kmp_allocate_team()
5326 __kmp_reinitialize_team(team, new_icvs, NULL); in __kmp_allocate_team()
5339 for (f = team->t.t_nproc; f < new_nproc; f++) { in __kmp_allocate_team()
5340 kmp_info_t *new_worker = __kmp_allocate_thread(root, team, f); in __kmp_allocate_team()
5342 team->t.t_threads[f] = new_worker; in __kmp_allocate_team()
5347 team->t.t_id, __kmp_gtid_from_tid(f, team), team->t.t_id, f, in __kmp_allocate_team()
5348 team->t.t_bar[bs_forkjoin_barrier].b_arrived, in __kmp_allocate_team()
5349 team->t.t_bar[bs_plain_barrier].b_arrived)); in __kmp_allocate_team()
5355 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; in __kmp_allocate_team()
5359 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; in __kmp_allocate_team()
5378 __kmp_add_threads_to_team(team, new_nproc); in __kmp_allocate_team()
5382 __kmp_initialize_team(team, new_nproc, new_icvs, in __kmp_allocate_team()
5386 KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc); in __kmp_allocate_team()
5387 for (f = 0; f < team->t.t_nproc; ++f) in __kmp_allocate_team()
5388 __kmp_initialize_info(team->t.t_threads[f], team, f, in __kmp_allocate_team()
5389 __kmp_gtid_from_tid(f, team)); in __kmp_allocate_team()
5397 for (f = old_nproc; f < team->t.t_nproc; ++f) in __kmp_allocate_team()
5398 team->t.t_threads[f]->th.th_task_state = in __kmp_allocate_team()
5399 team->t.t_threads[0]->th.th_task_state_memo_stack[level]; in __kmp_allocate_team()
5402 kmp_uint8 old_state = team->t.t_threads[0]->th.th_task_state; in __kmp_allocate_team()
5403 for (f = old_nproc; f < team->t.t_nproc; ++f) in __kmp_allocate_team()
5404 team->t.t_threads[f]->th.th_task_state = old_state; in __kmp_allocate_team()
5408 for (f = 0; f < team->t.t_nproc; ++f) { in __kmp_allocate_team()
5409 KMP_DEBUG_ASSERT(team->t.t_threads[f] && in __kmp_allocate_team()
5410 team->t.t_threads[f]->th.th_team_nproc == in __kmp_allocate_team()
5411 team->t.t_nproc); in __kmp_allocate_team()
5416 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); in __kmp_allocate_team()
5418 __kmp_partition_places(team); in __kmp_allocate_team()
5423 kmp_info_t *master = team->t.t_threads[0]; in __kmp_allocate_team()
5427 kmp_info_t *thr = team->t.t_threads[f]; in __kmp_allocate_team()
5438 kmp_info_t *thr = team->t.t_threads[f]; in __kmp_allocate_team()
5442 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; in __kmp_allocate_team()
5445 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; in __kmp_allocate_team()
5453 __kmp_alloc_argv_entries(argc, team, TRUE); in __kmp_allocate_team()
5454 KMP_CHECK_UPDATE(team->t.t_argc, argc); in __kmp_allocate_team()
5458 KF_TRACE(10, (" hot_team = %p\n", team)); in __kmp_allocate_team()
5464 team->t.t_task_team[0], team->t.t_task_team[1])); in __kmp_allocate_team()
5469 __ompt_team_assign_id(team, ompt_parallel_data); in __kmp_allocate_team()
5474 return team; in __kmp_allocate_team()
5479 for (team = CCAST(kmp_team_t *, __kmp_team_pool); (team);) { in __kmp_allocate_team()
5482 if (team->t.t_max_nproc >= max_nproc) { in __kmp_allocate_team()
5484 __kmp_team_pool = team->t.t_next_pool; in __kmp_allocate_team()
5488 if (!team->t.b) { // Allocate barrier structure in __kmp_allocate_team()
5489 team->t.b = distributedBarrier::allocate(__kmp_dflt_team_nth_ub); in __kmp_allocate_team()
5494 __kmp_initialize_team(team, new_nproc, new_icvs, NULL); in __kmp_allocate_team()
5498 &team->t.t_task_team[0], &team->t.t_task_team[1])); in __kmp_allocate_team()
5499 team->t.t_task_team[0] = NULL; in __kmp_allocate_team()
5500 team->t.t_task_team[1] = NULL; in __kmp_allocate_team()
5503 __kmp_alloc_argv_entries(argc, team, TRUE); in __kmp_allocate_team()
5504 KMP_CHECK_UPDATE(team->t.t_argc, argc); in __kmp_allocate_team()
5508 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); in __kmp_allocate_team()
5512 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE; in __kmp_allocate_team()
5514 team->t.t_bar[b].b_master_arrived = 0; in __kmp_allocate_team()
5515 team->t.t_bar[b].b_team_arrived = 0; in __kmp_allocate_team()
5520 team->t.t_proc_bind = new_proc_bind; in __kmp_allocate_team()
5523 team->t.t_id)); in __kmp_allocate_team()
5526 __ompt_team_assign_id(team, ompt_parallel_data); in __kmp_allocate_team()
5531 return team; in __kmp_allocate_team()
5538 team = __kmp_reap_team(team); in __kmp_allocate_team()
5539 __kmp_team_pool = team; in __kmp_allocate_team()
5544 team = (kmp_team_t *)__kmp_allocate(sizeof(kmp_team_t)); in __kmp_allocate_team()
5547 team->t.t_max_nproc = max_nproc; in __kmp_allocate_team()
5551 team->t.b = distributedBarrier::allocate(__kmp_dflt_team_nth_ub); in __kmp_allocate_team()
5556 __kmp_allocate_team_arrays(team, max_nproc); in __kmp_allocate_team()
5559 __kmp_initialize_team(team, new_nproc, new_icvs, NULL); in __kmp_allocate_team()
5563 &team->t.t_task_team[0], &team->t.t_task_team[1])); in __kmp_allocate_team()
5564 team->t.t_task_team[0] = NULL; // to be removed, as __kmp_allocate zeroes in __kmp_allocate_team()
5566 team->t.t_task_team[1] = NULL; // to be removed, as __kmp_allocate zeroes in __kmp_allocate_team()
5570 __kmp_print_team_storage_map("team", team, team->t.t_id, new_nproc); in __kmp_allocate_team()
5574 __kmp_alloc_argv_entries(argc, team, FALSE); in __kmp_allocate_team()
5575 team->t.t_argc = argc; in __kmp_allocate_team()
5579 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); in __kmp_allocate_team()
5583 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE; in __kmp_allocate_team()
5585 team->t.t_bar[b].b_master_arrived = 0; in __kmp_allocate_team()
5586 team->t.t_bar[b].b_team_arrived = 0; in __kmp_allocate_team()
5591 team->t.t_proc_bind = new_proc_bind; in __kmp_allocate_team()
5594 __ompt_team_assign_id(team, ompt_parallel_data); in __kmp_allocate_team()
5595 team->t.ompt_serialized_team_info = NULL; in __kmp_allocate_team()
5601 team->t.t_id)); in __kmp_allocate_team()
5603 return team; in __kmp_allocate_team()
5612 kmp_team_t *team USE_NESTED_HOT_ARG(kmp_info_t *master)) { in __kmp_free_team()
5615 team->t.t_id)); in __kmp_free_team()
5619 KMP_DEBUG_ASSERT(team); in __kmp_free_team()
5620 KMP_DEBUG_ASSERT(team->t.t_nproc <= team->t.t_max_nproc); in __kmp_free_team()
5621 KMP_DEBUG_ASSERT(team->t.t_threads); in __kmp_free_team()
5623 int use_hot_team = team == root->r.r_hot_team; in __kmp_free_team()
5627 level = team->t.t_active_level - 1; in __kmp_free_team()
5633 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master && in __kmp_free_team()
5634 master->th.th_teams_level == team->t.t_level) { in __kmp_free_team()
5643 KMP_DEBUG_ASSERT(team == hot_teams[level].hot_team); in __kmp_free_team()
5650 TCW_SYNC_PTR(team->t.t_pkfn, in __kmp_free_team()
5653 team->t.t_copyin_counter = 0; // init counter for possible reuse in __kmp_free_team()
5661 for (f = 1; f < team->t.t_nproc; ++f) { in __kmp_free_team()
5662 KMP_DEBUG_ASSERT(team->t.t_threads[f]); in __kmp_free_team()
5663 kmp_info_t *th = team->t.t_threads[f]; in __kmp_free_team()
5685 kmp_task_team_t *task_team = team->t.t_task_team[tt_idx]; in __kmp_free_team()
5687 for (f = 0; f < team->t.t_nproc; ++f) { // threads unref task teams in __kmp_free_team()
5688 KMP_DEBUG_ASSERT(team->t.t_threads[f]); in __kmp_free_team()
5689 team->t.t_threads[f]->th.th_task_team = NULL; in __kmp_free_team()
5694 __kmp_get_gtid(), task_team, team->t.t_id)); in __kmp_free_team()
5698 team->t.t_task_team[tt_idx] = NULL; in __kmp_free_team()
5704 team->t.t_parent = NULL; in __kmp_free_team()
5705 team->t.t_level = 0; in __kmp_free_team()
5706 team->t.t_active_level = 0; in __kmp_free_team()
5709 for (f = 1; f < team->t.t_nproc; ++f) { in __kmp_free_team()
5710 KMP_DEBUG_ASSERT(team->t.t_threads[f]); in __kmp_free_team()
5712 KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team), in __kmp_free_team()
5715 __kmp_free_thread(team->t.t_threads[f]); in __kmp_free_team()
5719 if (team->t.b) { in __kmp_free_team()
5721 team->t.b->go_release(); in __kmp_free_team()
5723 for (f = 1; f < team->t.t_nproc; ++f) { in __kmp_free_team()
5724 if (team->t.b->sleep[f].sleep) { in __kmp_free_team()
5726 team->t.t_threads[f]->th.th_info.ds.ds_gtid, in __kmp_free_team()
5732 for (int f = 1; f < team->t.t_nproc; ++f) { in __kmp_free_team()
5733 while (team->t.t_threads[f]->th.th_used_in_team.load() != 0) in __kmp_free_team()
5739 for (f = 1; f < team->t.t_nproc; ++f) { in __kmp_free_team()
5740 team->t.t_threads[f] = NULL; in __kmp_free_team()
5743 if (team->t.t_max_nproc > 1 && in __kmp_free_team()
5745 distributedBarrier::deallocate(team->t.b); in __kmp_free_team()
5746 team->t.b = NULL; in __kmp_free_team()
5750 team->t.t_next_pool = CCAST(kmp_team_t *, __kmp_team_pool); in __kmp_free_team()
5751 __kmp_team_pool = (volatile kmp_team_t *)team; in __kmp_free_team()
5754 KMP_DEBUG_ASSERT(team->t.t_threads[1] && in __kmp_free_team()
5755 team->t.t_threads[1]->th.th_cg_roots); in __kmp_free_team()
5756 if (team->t.t_threads[1]->th.th_cg_roots->cg_root == team->t.t_threads[1]) { in __kmp_free_team()
5758 for (f = 1; f < team->t.t_nproc; ++f) { in __kmp_free_team()
5759 kmp_info_t *thr = team->t.t_threads[f]; in __kmp_free_team()
5784 kmp_team_t *__kmp_reap_team(kmp_team_t *team) { in __kmp_reap_team() argument
5785 kmp_team_t *next_pool = team->t.t_next_pool; in __kmp_reap_team()
5787 KMP_DEBUG_ASSERT(team); in __kmp_reap_team()
5788 KMP_DEBUG_ASSERT(team->t.t_dispatch); in __kmp_reap_team()
5789 KMP_DEBUG_ASSERT(team->t.t_disp_buffer); in __kmp_reap_team()
5790 KMP_DEBUG_ASSERT(team->t.t_threads); in __kmp_reap_team()
5791 KMP_DEBUG_ASSERT(team->t.t_argv); in __kmp_reap_team()
5796 __kmp_free_team_arrays(team); in __kmp_reap_team()
5797 if (team->t.t_argv != &team->t.t_inline_argv[0]) in __kmp_reap_team()
5798 __kmp_free((void *)team->t.t_argv); in __kmp_reap_team()
5799 __kmp_free(team); in __kmp_reap_team()
5846 balign[b].bb.team = NULL; in __kmp_free_thread()
6351 kmp_team_t *team = CCAST(kmp_team_t *, __kmp_team_pool); in __kmp_internal_end() local
6352 __kmp_team_pool = team->t.t_next_pool; in __kmp_internal_end()
6354 team->t.t_next_pool = NULL; in __kmp_internal_end()
6355 __kmp_reap_team(team); in __kmp_internal_end()
7486 kmp_team_t *team) { in __kmp_run_before_invoked_task() argument
7498 KMP_DEBUG_ASSERT(team->t.t_dispatch); in __kmp_run_before_invoked_task()
7505 __kmp_push_parallel(gtid, team->t.t_ident); in __kmp_run_before_invoked_task()
7511 kmp_team_t *team) { in __kmp_run_after_invoked_task() argument
7513 __kmp_pop_parallel(gtid, team->t.t_ident); in __kmp_run_after_invoked_task()
7522 kmp_team_t *team = this_thr->th.th_team; in __kmp_invoke_task_func() local
7524 __kmp_run_before_invoked_task(gtid, tid, this_thr, team); in __kmp_invoke_task_func()
7528 if (team->t.t_stack_id != NULL) { in __kmp_invoke_task_func()
7529 __kmp_itt_stack_callee_enter((__itt_caller)team->t.t_stack_id); in __kmp_invoke_task_func()
7531 KMP_DEBUG_ASSERT(team->t.t_parent->t.t_stack_id != NULL); in __kmp_invoke_task_func()
7533 (__itt_caller)team->t.t_parent->t.t_stack_id); in __kmp_invoke_task_func()
7549 exit_frame_p = &(team->t.t_implicit_task_taskdata[tid] in __kmp_invoke_task_func()
7556 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data); in __kmp_invoke_task_func()
7557 my_parallel_data = &(team->t.ompt_team_info.parallel_data); in __kmp_invoke_task_func()
7559 ompt_team_size = team->t.t_nproc; in __kmp_invoke_task_func()
7577 rc = __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn), gtid, in __kmp_invoke_task_func()
7578 tid, (int)team->t.t_argc, (void **)team->t.t_argv in __kmp_invoke_task_func()
7599 if (team->t.t_stack_id != NULL) { in __kmp_invoke_task_func()
7600 __kmp_itt_stack_callee_leave((__itt_caller)team->t.t_stack_id); in __kmp_invoke_task_func()
7602 KMP_DEBUG_ASSERT(team->t.t_parent->t.t_stack_id != NULL); in __kmp_invoke_task_func()
7604 (__itt_caller)team->t.t_parent->t.t_stack_id); in __kmp_invoke_task_func()
7608 __kmp_run_after_invoked_task(gtid, tid, this_thr, team); in __kmp_invoke_task_func()
7616 kmp_team_t *team = thr->th.th_team; in __kmp_teams_master() local
7617 ident_t *loc = team->t.t_ident; in __kmp_teams_master()
7641 __kmp_fork_call(loc, gtid, fork_context_intel, team->t.t_argc, in __kmp_teams_master()
7663 kmp_team_t *team = this_thr->th.th_team; in __kmp_invoke_teams_master() local
7669 __kmp_run_before_invoked_task(gtid, 0, this_thr, team); in __kmp_invoke_teams_master()
7673 &team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data; in __kmp_invoke_teams_master()
7674 ompt_data_t *parallel_data = &team->t.ompt_team_info.parallel_data; in __kmp_invoke_teams_master()
7677 ompt_scope_begin, parallel_data, task_data, team->t.t_nproc, tid, in __kmp_invoke_teams_master()
7686 __kmp_run_after_invoked_task(gtid, 0, this_thr, team); in __kmp_invoke_teams_master()
7865 void __kmp_internal_fork(ident_t *id, int gtid, kmp_team_t *team) { in __kmp_internal_fork() argument
7872 KMP_DEBUG_ASSERT(team); in __kmp_internal_fork()
7873 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); in __kmp_internal_fork()
7877 team->t.t_construct = 0; /* no single directives seen yet */ in __kmp_internal_fork()
7878 team->t.t_ordered.dt.t_value = in __kmp_internal_fork()
7882 KMP_DEBUG_ASSERT(team->t.t_disp_buffer); in __kmp_internal_fork()
7883 if (team->t.t_max_nproc > 1) { in __kmp_internal_fork()
7886 team->t.t_disp_buffer[i].buffer_index = i; in __kmp_internal_fork()
7887 team->t.t_disp_buffer[i].doacross_buf_idx = i; in __kmp_internal_fork()
7890 team->t.t_disp_buffer[0].buffer_index = 0; in __kmp_internal_fork()
7891 team->t.t_disp_buffer[0].doacross_buf_idx = 0; in __kmp_internal_fork()
7895 KMP_ASSERT(this_thr->th.th_team == team); in __kmp_internal_fork()
7898 for (f = 0; f < team->t.t_nproc; f++) { in __kmp_internal_fork()
7899 KMP_DEBUG_ASSERT(team->t.t_threads[f] && in __kmp_internal_fork()
7900 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc); in __kmp_internal_fork()
7908 void __kmp_internal_join(ident_t *id, int gtid, kmp_team_t *team) { in __kmp_internal_join() argument
7911 KMP_DEBUG_ASSERT(team); in __kmp_internal_join()
7912 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); in __kmp_internal_join()
7920 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) { in __kmp_internal_join()
7925 gtid, __kmp_threads[gtid]->th.th_team_nproc, team, in __kmp_internal_join()
7926 team->t.t_nproc); in __kmp_internal_join()
7930 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc); in __kmp_internal_join()
7967 KMP_ASSERT(this_thr->th.th_team == team); in __kmp_internal_join()
8329 kmp_team_t *team = thr->th.th_team; in __kmp_aux_get_team_info() local
8331 int ii = team->t.t_level; in __kmp_aux_get_team_info()
8332 teams_serialized = team->t.t_serialized; in __kmp_aux_get_team_info()
8336 for (teams_serialized = team->t.t_serialized; in __kmp_aux_get_team_info()
8339 if (team->t.t_serialized && (!teams_serialized)) { in __kmp_aux_get_team_info()
8340 team = team->t.t_parent; in __kmp_aux_get_team_info()
8344 team = team->t.t_parent; in __kmp_aux_get_team_info()
8348 return team; in __kmp_aux_get_team_info()
8355 kmp_team_t *team = __kmp_aux_get_team_info(serialized); in __kmp_aux_get_team_num() local
8356 if (team) { in __kmp_aux_get_team_num()
8360 return team->t.t_master_tid; in __kmp_aux_get_team_num()
8368 kmp_team_t *team = __kmp_aux_get_team_info(serialized); in __kmp_aux_get_num_teams() local
8369 if (team) { in __kmp_aux_get_num_teams()
8373 return team->t.t_parent->t.t_nproc; in __kmp_aux_get_num_teams()
8956 void __kmp_resize_dist_barrier(kmp_team_t *team, int old_nthreads, in __kmp_resize_dist_barrier() argument
8960 kmp_info_t **other_threads = team->t.t_threads; in __kmp_resize_dist_barrier()
8967 if (team->t.t_threads[f]->th.th_used_in_team.load() == 0) { in __kmp_resize_dist_barrier()
8973 if (team->t.t_threads[f]->th.th_used_in_team.load() == 3) { in __kmp_resize_dist_barrier()
8974 while (team->t.t_threads[f]->th.th_used_in_team.load() == 3) in __kmp_resize_dist_barrier()
8978 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 1); in __kmp_resize_dist_barrier()
8980 team->t.t_threads[f]->th.th_used_in_team.store(2); in __kmp_resize_dist_barrier()
8981 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 2); in __kmp_resize_dist_barrier()
8984 team->t.b->go_release(); in __kmp_resize_dist_barrier()
9001 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 0); in __kmp_resize_dist_barrier()
9007 team->t.b->update_num_threads(new_nthreads); in __kmp_resize_dist_barrier()
9008 team->t.b->go_reset(); in __kmp_resize_dist_barrier()
9011 void __kmp_add_threads_to_team(kmp_team_t *team, int new_nthreads) { in __kmp_add_threads_to_team() argument
9013 KMP_DEBUG_ASSERT(team); in __kmp_add_threads_to_team()
9020 KMP_DEBUG_ASSERT(team->t.t_threads[f]); in __kmp_add_threads_to_team()
9021 KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team), 0, in __kmp_add_threads_to_team()
9024 __kmp_resume_32(team->t.t_threads[f]->th.th_info.ds.ds_gtid, in __kmp_add_threads_to_team()
9035 if (team->t.t_threads[f]->th.th_used_in_team.load() == 1) { in __kmp_add_threads_to_team()