Lines Matching refs:task_team
25 static void __kmp_enable_tasking(kmp_task_team_t *task_team,
30 kmp_task_team_t *task_team);
342 __kmp_get_priority_deque_data(kmp_task_team_t *task_team, kmp_int32 pri) { in __kmp_get_priority_deque_data() argument
344 kmp_task_pri_t *lst = task_team->tt.tt_task_pri_list; in __kmp_get_priority_deque_data()
355 task_team->tt.tt_task_pri_list = list; in __kmp_get_priority_deque_data()
388 kmp_task_team_t *task_team, in __kmp_push_priority_task() argument
396 kmp_task_pri_t *lst = task_team->tt.tt_task_pri_list; in __kmp_push_priority_task()
398 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_task_pri_lock); in __kmp_push_priority_task()
399 if (task_team->tt.tt_task_pri_list == NULL) { in __kmp_push_priority_task()
405 task_team->tt.tt_task_pri_list = list; in __kmp_push_priority_task()
408 thread_data = __kmp_get_priority_deque_data(task_team, pri); in __kmp_push_priority_task()
410 __kmp_release_bootstrap_lock(&task_team->tt.tt_task_pri_lock); in __kmp_push_priority_task()
416 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_task_pri_lock); in __kmp_push_priority_task()
417 thread_data = __kmp_get_priority_deque_data(task_team, pri); in __kmp_push_priority_task()
418 __kmp_release_bootstrap_lock(&task_team->tt.tt_task_pri_lock); in __kmp_push_priority_task()
456 task_team->tt.tt_num_task_pri++; // atomic inc in __kmp_push_priority_task()
477 kmp_task_team_t *task_team = thread->th.th_task_team; in __kmp_push_task() local
506 if (UNLIKELY(!KMP_TASKING_ENABLED(task_team))) { in __kmp_push_task()
507 __kmp_enable_tasking(task_team, thread); in __kmp_push_task()
509 KMP_DEBUG_ASSERT(TCR_4(task_team->tt.tt_found_tasks) == TRUE); in __kmp_push_task()
510 KMP_DEBUG_ASSERT(TCR_PTR(task_team->tt.tt_threads_data) != NULL); in __kmp_push_task()
515 return __kmp_push_priority_task(gtid, thread, taskdata, task_team, pri); in __kmp_push_task()
519 thread_data = &task_team->tt.tt_threads_data[tid]; in __kmp_push_task()
992 kmp_task_team_t *task_team = in __kmp_task_finish() local
1116 } else if (task_team && (task_team->tt.tt_found_proxy_tasks || in __kmp_task_finish()
1117 task_team->tt.tt_hidden_helper_task_encountered)) { in __kmp_task_finish()
1416 kmp_task_team_t *task_team = thread->th.th_task_team; in __kmp_task_alloc() local
1419 if (!KMP_TASKING_ENABLED(task_team)) { in __kmp_task_alloc()
1423 __kmp_enable_tasking(task_team, thread); in __kmp_task_alloc()
1425 kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid]; in __kmp_task_alloc()
1433 task_team->tt.tt_found_proxy_tasks == FALSE) in __kmp_task_alloc()
1434 TCW_4(task_team->tt.tt_found_proxy_tasks, TRUE); in __kmp_task_alloc()
1436 task_team->tt.tt_hidden_helper_task_encountered == FALSE) in __kmp_task_alloc()
1437 TCW_4(task_team->tt.tt_hidden_helper_task_encountered, TRUE); in __kmp_task_alloc()
2217 kmp_task_team_t *task_team = thread->th.th_task_team; in __kmpc_omp_taskyield() local
2218 if (task_team != NULL) { in __kmpc_omp_taskyield()
2219 if (KMP_TASKING_ENABLED(task_team)) { in __kmpc_omp_taskyield()
2846 kmp_task_team_t *task_team, in __kmp_get_priority_task() argument
2852 int ntasks = task_team->tt.tt_num_task_pri; in __kmp_get_priority_task()
2860 if (__kmp_atomic_compare_store(&task_team->tt.tt_num_task_pri, ntasks, in __kmp_get_priority_task()
2871 kmp_task_pri_t *list = task_team->tt.tt_task_pri_list; in __kmp_get_priority_task()
2893 if (!task_team->tt.tt_untied_task_encountered) { in __kmp_get_priority_task()
2898 gtid, thread_data, task_team, deque_ntasks, target, in __kmp_get_priority_task()
2900 task_team->tt.tt_num_task_pri++; // atomic inc, restore value in __kmp_get_priority_task()
2921 gtid, thread_data, task_team, deque_ntasks, in __kmp_get_priority_task()
2923 task_team->tt.tt_num_task_pri++; // atomic inc, restore value in __kmp_get_priority_task()
2946 kmp_task_team_t *task_team, in __kmp_remove_my_task() argument
2954 KMP_DEBUG_ASSERT(task_team->tt.tt_threads_data != in __kmp_remove_my_task()
2957 thread_data = &task_team->tt.tt_threads_data[__kmp_tid_from_gtid(gtid)]; in __kmp_remove_my_task()
3018 kmp_task_team_t *task_team, in __kmp_steal_task() argument
3031 threads_data = task_team->tt.tt_threads_data; in __kmp_steal_task()
3039 gtid, __kmp_gtid_from_thread(victim_thr), task_team, in __kmp_steal_task()
3046 gtid, __kmp_gtid_from_thread(victim_thr), task_team, in __kmp_steal_task()
3060 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks, in __kmp_steal_task()
3073 if (!task_team->tt.tt_untied_task_encountered) { in __kmp_steal_task()
3078 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks, in __kmp_steal_task()
3100 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks, in __kmp_steal_task()
3127 gtid, count + 1, task_team)); in __kmp_steal_task()
3138 gtid, taskdata, __kmp_gtid_from_thread(victim_thr), task_team, in __kmp_steal_task()
3159 kmp_task_team_t *task_team = thread->th.th_task_team; in __kmp_execute_tasks_template() local
3171 if (task_team == NULL || current_task == NULL) in __kmp_execute_tasks_template()
3179 threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data); in __kmp_execute_tasks_template()
3183 nthreads = task_team->tt.tt_nproc; in __kmp_execute_tasks_template()
3184 unfinished_threads = &(task_team->tt.tt_unfinished_threads); in __kmp_execute_tasks_template()
3185 KMP_DEBUG_ASSERT(nthreads > 1 || task_team->tt.tt_found_proxy_tasks || in __kmp_execute_tasks_template()
3186 task_team->tt.tt_hidden_helper_task_encountered); in __kmp_execute_tasks_template()
3193 if (task_team->tt.tt_num_task_pri) { // get priority task first in __kmp_execute_tasks_template()
3194 task = __kmp_get_priority_task(gtid, task_team, is_constrained); in __kmp_execute_tasks_template()
3197 task = __kmp_remove_my_task(thread, gtid, task_team, is_constrained); in __kmp_execute_tasks_template()
3250 task = __kmp_steal_task(other_thread, gtid, task_team, in __kmp_execute_tasks_template()
3328 gtid, count, task_team)); in __kmp_execute_tasks_template()
3446 static void __kmp_enable_tasking(kmp_task_team_t *task_team, in __kmp_enable_tasking() argument
3454 KMP_DEBUG_ASSERT(task_team != NULL); in __kmp_enable_tasking()
3457 nthreads = task_team->tt.tt_nproc; in __kmp_enable_tasking()
3462 is_init_thread = __kmp_realloc_task_threads_data(this_thr, task_team); in __kmp_enable_tasking()
3472 threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data); in __kmp_enable_tasking()
3604 kmp_task_team_t *task_team) { in __kmp_realloc_task_threads_data() argument
3609 if (TCR_4(task_team->tt.tt_found_tasks)) { in __kmp_realloc_task_threads_data()
3614 threads_data_p = &task_team->tt.tt_threads_data; in __kmp_realloc_task_threads_data()
3615 nthreads = task_team->tt.tt_nproc; in __kmp_realloc_task_threads_data()
3616 maxthreads = task_team->tt.tt_max_threads; in __kmp_realloc_task_threads_data()
3621 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_threads_lock); in __kmp_realloc_task_threads_data()
3623 if (!TCR_4(task_team->tt.tt_found_tasks)) { in __kmp_realloc_task_threads_data()
3639 __kmp_gtid_from_thread(thread), task_team, nthreads, maxthreads)); in __kmp_realloc_task_threads_data()
3663 __kmp_gtid_from_thread(thread), task_team, nthreads)); in __kmp_realloc_task_threads_data()
3677 task_team->tt.tt_max_threads = nthreads; in __kmp_realloc_task_threads_data()
3697 TCW_SYNC_4(task_team->tt.tt_found_tasks, TRUE); in __kmp_realloc_task_threads_data()
3700 __kmp_release_bootstrap_lock(&task_team->tt.tt_threads_lock); in __kmp_realloc_task_threads_data()
3707 static void __kmp_free_task_threads_data(kmp_task_team_t *task_team) { in __kmp_free_task_threads_data() argument
3708 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_threads_lock); in __kmp_free_task_threads_data()
3709 if (task_team->tt.tt_threads_data != NULL) { in __kmp_free_task_threads_data()
3711 for (i = 0; i < task_team->tt.tt_max_threads; i++) { in __kmp_free_task_threads_data()
3712 __kmp_free_task_deque(&task_team->tt.tt_threads_data[i]); in __kmp_free_task_threads_data()
3714 __kmp_free(task_team->tt.tt_threads_data); in __kmp_free_task_threads_data()
3715 task_team->tt.tt_threads_data = NULL; in __kmp_free_task_threads_data()
3717 __kmp_release_bootstrap_lock(&task_team->tt.tt_threads_lock); in __kmp_free_task_threads_data()
3723 static void __kmp_free_task_pri_list(kmp_task_team_t *task_team) { in __kmp_free_task_pri_list() argument
3724 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_task_pri_lock); in __kmp_free_task_pri_list()
3725 if (task_team->tt.tt_task_pri_list != NULL) { in __kmp_free_task_pri_list()
3726 kmp_task_pri_t *list = task_team->tt.tt_task_pri_list; in __kmp_free_task_pri_list()
3733 task_team->tt.tt_task_pri_list = NULL; in __kmp_free_task_pri_list()
3735 __kmp_release_bootstrap_lock(&task_team->tt.tt_task_pri_lock); in __kmp_free_task_pri_list()
3744 kmp_task_team_t *task_team = NULL; in __kmp_allocate_task_team() local
3754 task_team = __kmp_free_task_teams; in __kmp_allocate_task_team()
3755 TCW_PTR(__kmp_free_task_teams, task_team->tt.tt_next); in __kmp_allocate_task_team()
3756 task_team->tt.tt_next = NULL; in __kmp_allocate_task_team()
3761 if (task_team == NULL) { in __kmp_allocate_task_team()
3767 task_team = (kmp_task_team_t *)__kmp_allocate(sizeof(kmp_task_team_t)); in __kmp_allocate_task_team()
3768 __kmp_init_bootstrap_lock(&task_team->tt.tt_threads_lock); in __kmp_allocate_task_team()
3769 __kmp_init_bootstrap_lock(&task_team->tt.tt_task_pri_lock); in __kmp_allocate_task_team()
3775 &task_team->tt.tt_found_tasks, sizeof(task_team->tt.tt_found_tasks)); in __kmp_allocate_task_team()
3778 CCAST(kmp_uint32 *, &task_team->tt.tt_active), in __kmp_allocate_task_team()
3779 sizeof(task_team->tt.tt_active)); in __kmp_allocate_task_team()
3787 TCW_4(task_team->tt.tt_found_tasks, FALSE); in __kmp_allocate_task_team()
3788 TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE); in __kmp_allocate_task_team()
3789 TCW_4(task_team->tt.tt_hidden_helper_task_encountered, FALSE); in __kmp_allocate_task_team()
3790 task_team->tt.tt_nproc = nthreads = team->t.t_nproc; in __kmp_allocate_task_team()
3792 KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads, nthreads); in __kmp_allocate_task_team()
3793 TCW_4(task_team->tt.tt_hidden_helper_task_encountered, FALSE); in __kmp_allocate_task_team()
3794 TCW_4(task_team->tt.tt_active, TRUE); in __kmp_allocate_task_team()
3798 (thread ? __kmp_gtid_from_thread(thread) : -1), task_team, in __kmp_allocate_task_team()
3799 KMP_ATOMIC_LD_RLX(&task_team->tt.tt_unfinished_threads))); in __kmp_allocate_task_team()
3800 return task_team; in __kmp_allocate_task_team()
3806 void __kmp_free_task_team(kmp_info_t *thread, kmp_task_team_t *task_team) { in __kmp_free_task_team() argument
3808 thread ? __kmp_gtid_from_thread(thread) : -1, task_team)); in __kmp_free_task_team()
3813 KMP_DEBUG_ASSERT(task_team->tt.tt_next == NULL); in __kmp_free_task_team()
3814 task_team->tt.tt_next = __kmp_free_task_teams; in __kmp_free_task_team()
3815 TCW_PTR(__kmp_free_task_teams, task_team); in __kmp_free_task_team()
3826 kmp_task_team_t *task_team; in __kmp_reap_task_teams() local
3831 while ((task_team = __kmp_free_task_teams) != NULL) { in __kmp_reap_task_teams()
3832 __kmp_free_task_teams = task_team->tt.tt_next; in __kmp_reap_task_teams()
3833 task_team->tt.tt_next = NULL; in __kmp_reap_task_teams()
3836 if (task_team->tt.tt_threads_data != NULL) { in __kmp_reap_task_teams()
3837 __kmp_free_task_threads_data(task_team); in __kmp_reap_task_teams()
3839 if (task_team->tt.tt_task_pri_list != NULL) { in __kmp_reap_task_teams()
3840 __kmp_free_task_pri_list(task_team); in __kmp_reap_task_teams()
3842 __kmp_free(task_team); in __kmp_reap_task_teams()
3951 kmp_task_team_t *task_team = team->t.t_task_team[other_team]; in __kmp_task_team_setup() local
3952 if (!task_team->tt.tt_active || in __kmp_task_team_setup()
3953 team->t.t_nproc != task_team->tt.tt_nproc) { in __kmp_task_team_setup()
3954 TCW_4(task_team->tt.tt_nproc, team->t.t_nproc); in __kmp_task_team_setup()
3955 TCW_4(task_team->tt.tt_found_tasks, FALSE); in __kmp_task_team_setup()
3956 TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE); in __kmp_task_team_setup()
3957 TCW_4(task_team->tt.tt_hidden_helper_task_encountered, FALSE); in __kmp_task_team_setup()
3958 KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads, in __kmp_task_team_setup()
3960 TCW_4(task_team->tt.tt_active, TRUE); in __kmp_task_team_setup()
3977 kmp_task_team_t *task_team = team->t.t_task_team[i]; in __kmp_task_team_setup() local
3978 if (KMP_TASKING_ENABLED(task_team)) { in __kmp_task_team_setup()
3981 __kmp_enable_tasking(task_team, this_thr); in __kmp_task_team_setup()
3982 for (int j = 0; j < task_team->tt.tt_nproc; ++j) { in __kmp_task_team_setup()
3983 kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[j]; in __kmp_task_team_setup()
4023 kmp_task_team_t *task_team = team->t.t_task_team[this_thr->th.th_task_state]; in __kmp_task_team_wait() local
4026 KMP_DEBUG_ASSERT(task_team == this_thr->th.th_task_team); in __kmp_task_team_wait()
4028 if ((task_team != NULL) && KMP_TASKING_ENABLED(task_team)) { in __kmp_task_team_wait()
4032 __kmp_gtid_from_thread(this_thr), task_team)); in __kmp_task_team_wait()
4038 &task_team->tt.tt_unfinished_threads), in __kmp_task_team_wait()
4048 __kmp_gtid_from_thread(this_thr), task_team)); in __kmp_task_team_wait()
4049 KMP_DEBUG_ASSERT(task_team->tt.tt_nproc > 1 || in __kmp_task_team_wait()
4050 task_team->tt.tt_found_proxy_tasks == TRUE || in __kmp_task_team_wait()
4051 task_team->tt.tt_hidden_helper_task_encountered == TRUE); in __kmp_task_team_wait()
4052 TCW_SYNC_4(task_team->tt.tt_found_proxy_tasks, FALSE); in __kmp_task_team_wait()
4053 TCW_SYNC_4(task_team->tt.tt_hidden_helper_task_encountered, FALSE); in __kmp_task_team_wait()
4054 KMP_CHECK_UPDATE(task_team->tt.tt_untied_task_encountered, 0); in __kmp_task_team_wait()
4055 TCW_SYNC_4(task_team->tt.tt_active, FALSE); in __kmp_task_team_wait()
4105 kmp_task_team_t *task_team = taskdata->td_task_team; in __kmp_give_task() local
4111 KMP_DEBUG_ASSERT(task_team != NULL); in __kmp_give_task()
4114 kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid]; in __kmp_give_task()