Lines Matching refs:taskdata
387 kmp_taskdata_t *taskdata, in __kmp_push_priority_task() argument
393 gtid, taskdata, pri)); in __kmp_push_priority_task()
428 __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata, in __kmp_push_priority_task()
433 gtid, taskdata)); in __kmp_push_priority_task()
443 thread_data->td.td_deque[thread_data->td.td_deque_tail] = taskdata; in __kmp_push_priority_task()
450 KMP_FSYNC_RELEASING(taskdata); // releasing child in __kmp_push_priority_task()
453 gtid, taskdata, thread_data->td.td_deque_ntasks, in __kmp_push_priority_task()
463 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_push_task() local
468 if (UNLIKELY(taskdata->td_flags.hidden_helper && in __kmp_push_task()
482 ("__kmp_push_task: T#%d trying to push task %p.\n", gtid, taskdata)); in __kmp_push_task()
484 if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) { in __kmp_push_task()
487 kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count); in __kmp_push_task()
492 gtid, counter, taskdata)); in __kmp_push_task()
496 if (UNLIKELY(taskdata->td_flags.task_serial)) { in __kmp_push_task()
499 gtid, taskdata)); in __kmp_push_task()
512 if (taskdata->td_flags.priority_specified && task->data2.priority > 0 && in __kmp_push_task()
515 return __kmp_push_priority_task(gtid, thread, taskdata, task_team, pri); in __kmp_push_task()
533 __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata, in __kmp_push_task()
537 gtid, taskdata)); in __kmp_push_task()
556 __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata, in __kmp_push_task()
561 gtid, taskdata)); in __kmp_push_task()
574 taskdata; // Push taskdata in __kmp_push_task()
581 KMP_FSYNC_RELEASING(taskdata); // releasing child in __kmp_push_task()
584 gtid, taskdata, thread_data->td.td_deque_ntasks, in __kmp_push_task()
656 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_task_start() local
661 gtid, taskdata, current_task)); in __kmp_task_start()
663 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); in __kmp_task_start()
672 if (taskdata->td_flags.tiedness == TASK_TIED) { in __kmp_task_start()
673 __kmp_push_task_stack(gtid, thread, taskdata); in __kmp_task_start()
678 thread->th.th_current_task = taskdata; in __kmp_task_start()
680 KMP_DEBUG_ASSERT(taskdata->td_flags.started == 0 || in __kmp_task_start()
681 taskdata->td_flags.tiedness == TASK_UNTIED); in __kmp_task_start()
682 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0 || in __kmp_task_start()
683 taskdata->td_flags.tiedness == TASK_UNTIED); in __kmp_task_start()
684 taskdata->td_flags.started = 1; in __kmp_task_start()
685 taskdata->td_flags.executing = 1; in __kmp_task_start()
686 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); in __kmp_task_start()
687 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); in __kmp_task_start()
694 KA_TRACE(10, ("__kmp_task_start(exit): T#%d task=%p\n", gtid, taskdata)); in __kmp_task_start()
723 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __ompt_task_start() local
733 &(taskdata->ompt_task_info.task_data)); in __ompt_task_start()
735 taskdata->ompt_task_info.scheduling_parent = current_task; in __ompt_task_start()
744 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __ompt_task_finish() local
745 if (__kmp_omp_cancellation && taskdata->td_taskgroup && in __ompt_task_finish()
746 taskdata->td_taskgroup->cancel_request == cancel_taskgroup) { in __ompt_task_finish()
752 &(taskdata->ompt_task_info.task_data), status, in __ompt_task_finish()
763 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmpc_omp_task_begin_if0_template() local
768 gtid, loc_ref, taskdata, current_task)); in __kmpc_omp_task_begin_if0_template()
770 if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) { in __kmpc_omp_task_begin_if0_template()
773 kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count); in __kmpc_omp_task_begin_if0_template()
777 gtid, counter, taskdata)); in __kmpc_omp_task_begin_if0_template()
780 taskdata->td_flags.task_serial = in __kmpc_omp_task_begin_if0_template()
788 taskdata->ompt_task_info.frame.exit_frame.ptr = frame_address; in __kmpc_omp_task_begin_if0_template()
790 taskdata->ompt_task_info.frame.exit_frame_flags = in __kmpc_omp_task_begin_if0_template()
797 &(taskdata->ompt_task_info.task_data), in __kmpc_omp_task_begin_if0_template()
798 ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(taskdata), 0, in __kmpc_omp_task_begin_if0_template()
806 loc_ref, taskdata)); in __kmpc_omp_task_begin_if0_template()
864 static void __kmp_free_task(kmp_int32 gtid, kmp_taskdata_t *taskdata, in __kmp_free_task() argument
867 taskdata)); in __kmp_free_task()
870 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); in __kmp_free_task()
871 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0); in __kmp_free_task()
872 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 1); in __kmp_free_task()
873 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); in __kmp_free_task()
874 KMP_DEBUG_ASSERT(taskdata->td_allocated_child_tasks == 0 || in __kmp_free_task()
875 taskdata->td_flags.task_serial == 1); in __kmp_free_task()
876 KMP_DEBUG_ASSERT(taskdata->td_incomplete_child_tasks == 0); in __kmp_free_task()
877 kmp_task_t *task = KMP_TASKDATA_TO_TASK(taskdata); in __kmp_free_task()
882 taskdata->td_flags.freed = 1; in __kmp_free_task()
885 __kmp_fast_free(thread, taskdata); in __kmp_free_task()
887 __kmp_thread_free(thread, taskdata); in __kmp_free_task()
889 KA_TRACE(20, ("__kmp_free_task: T#%d freed task %p\n", gtid, taskdata)); in __kmp_free_task()
899 kmp_taskdata_t *taskdata, in __kmp_free_task_and_ancestors() argument
904 (taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser) && in __kmp_free_task_and_ancestors()
905 !taskdata->td_flags.proxy; in __kmp_free_task_and_ancestors()
906 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); in __kmp_free_task_and_ancestors()
908 kmp_int32 children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks) - 1; in __kmp_free_task_and_ancestors()
913 kmp_taskdata_t *parent_taskdata = taskdata->td_parent; in __kmp_free_task_and_ancestors()
917 gtid, taskdata)); in __kmp_free_task_and_ancestors()
920 __kmp_free_task(gtid, taskdata, thread); in __kmp_free_task_and_ancestors()
922 taskdata = parent_taskdata; in __kmp_free_task_and_ancestors()
928 if (taskdata->td_flags.tasktype == TASK_IMPLICIT) { in __kmp_free_task_and_ancestors()
929 if (taskdata->td_dephash) { // do we need to cleanup dephash? in __kmp_free_task_and_ancestors()
930 int children = KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks); in __kmp_free_task_and_ancestors()
931 kmp_tasking_flags_t flags_old = taskdata->td_flags; in __kmp_free_task_and_ancestors()
936 RCAST(kmp_int32 *, &taskdata->td_flags), in __kmp_free_task_and_ancestors()
941 gtid, taskdata)); in __kmp_free_task_and_ancestors()
943 __kmp_dephash_free_entries(thread, taskdata->td_dephash); in __kmp_free_task_and_ancestors()
950 children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks) - 1; in __kmp_free_task_and_ancestors()
957 gtid, taskdata, children)); in __kmp_free_task_and_ancestors()
968 static bool __kmp_track_children_task(kmp_taskdata_t *taskdata) { in __kmp_track_children_task() argument
969 kmp_tasking_flags_t flags = taskdata->td_flags; in __kmp_track_children_task()
974 KMP_ATOMIC_LD_ACQ(&taskdata->td_parent->td_incomplete_child_tasks) > 0; in __kmp_track_children_task()
990 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_task_finish() local
999 gtid, taskdata, resumed_task)); in __kmp_task_finish()
1001 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); in __kmp_task_finish()
1005 if (taskdata->td_flags.tiedness == TASK_TIED) { in __kmp_task_finish()
1006 __kmp_pop_task_stack(gtid, thread, taskdata); in __kmp_task_finish()
1010 if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) { in __kmp_task_finish()
1013 kmp_int32 counter = KMP_ATOMIC_DEC(&taskdata->td_untied_count) - 1; in __kmp_task_finish()
1017 gtid, counter, taskdata)); in __kmp_task_finish()
1022 KMP_DEBUG_ASSERT(taskdata->td_flags.task_serial); in __kmp_task_finish()
1023 resumed_task = taskdata->td_parent; // In a serialized task, the resumed in __kmp_task_finish()
1030 gtid, taskdata, resumed_task)); in __kmp_task_finish()
1038 (taskdata->td_flags.tasking_ser || taskdata->td_flags.task_serial) == in __kmp_task_finish()
1039 taskdata->td_flags.task_serial); in __kmp_task_finish()
1040 if (taskdata->td_flags.task_serial) { in __kmp_task_finish()
1042 resumed_task = taskdata->td_parent; // In a serialized task, the resumed in __kmp_task_finish()
1056 if (UNLIKELY(taskdata->td_flags.destructors_thunk)) { in __kmp_task_finish()
1062 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); in __kmp_task_finish()
1063 KMP_DEBUG_ASSERT(taskdata->td_flags.started == 1); in __kmp_task_finish()
1064 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); in __kmp_task_finish()
1067 if (UNLIKELY(taskdata->td_flags.detachable == TASK_DETACHABLE)) { in __kmp_task_finish()
1068 if (taskdata->td_allow_completion_event.type == in __kmp_task_finish()
1071 __kmp_acquire_tas_lock(&taskdata->td_allow_completion_event.lock, gtid); in __kmp_task_finish()
1072 if (taskdata->td_allow_completion_event.type == in __kmp_task_finish()
1075 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1); in __kmp_task_finish()
1076 taskdata->td_flags.executing = 0; // suspend the finishing task in __kmp_task_finish()
1089 taskdata->td_flags.proxy = TASK_PROXY; // proxify! in __kmp_task_finish()
1092 __kmp_release_tas_lock(&taskdata->td_allow_completion_event.lock, gtid); in __kmp_task_finish()
1097 taskdata->td_flags.complete = 1; // mark the task as completed in __kmp_task_finish()
1106 if (__kmp_track_children_task(taskdata)) { in __kmp_task_finish()
1107 __kmp_release_deps(gtid, taskdata); in __kmp_task_finish()
1112 KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks); in __kmp_task_finish()
1114 if (taskdata->td_taskgroup) in __kmp_task_finish()
1115 KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count); in __kmp_task_finish()
1120 __kmp_release_deps(gtid, taskdata); in __kmp_task_finish()
1126 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1); in __kmp_task_finish()
1127 taskdata->td_flags.executing = 0; // suspend the finishing task in __kmp_task_finish()
1132 gtid, taskdata, children)); in __kmp_task_finish()
1140 __kmp_free_task_and_ancestors(gtid, taskdata, thread); in __kmp_task_finish()
1148 gtid, taskdata, resumed_task)); in __kmp_task_finish()
1356 kmp_taskdata_t *taskdata; in __kmp_task_alloc() local
1453 taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, shareds_offset + in __kmp_task_alloc()
1456 taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, shareds_offset + in __kmp_task_alloc()
1460 task = KMP_TASKDATA_TO_TASK(taskdata); in __kmp_task_alloc()
1464 KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(double) - 1)) == 0); in __kmp_task_alloc()
1467 KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(_Quad) - 1)) == 0); in __kmp_task_alloc()
1472 task->shareds = &((char *)taskdata)[shareds_offset]; in __kmp_task_alloc()
1482 taskdata->td_task_id = KMP_GEN_TASK_ID(); in __kmp_task_alloc()
1483 taskdata->td_team = thread->th.th_team; in __kmp_task_alloc()
1484 taskdata->td_alloc_thread = thread; in __kmp_task_alloc()
1485 taskdata->td_parent = parent_task; in __kmp_task_alloc()
1486 taskdata->td_level = parent_task->td_level + 1; // increment nesting level in __kmp_task_alloc()
1487 KMP_ATOMIC_ST_RLX(&taskdata->td_untied_count, 0); in __kmp_task_alloc()
1488 taskdata->td_ident = loc_ref; in __kmp_task_alloc()
1489 taskdata->td_taskwait_ident = NULL; in __kmp_task_alloc()
1490 taskdata->td_taskwait_counter = 0; in __kmp_task_alloc()
1491 taskdata->td_taskwait_thread = 0; in __kmp_task_alloc()
1492 KMP_DEBUG_ASSERT(taskdata->td_parent != NULL); in __kmp_task_alloc()
1495 copy_icvs(&taskdata->td_icvs, &taskdata->td_parent->td_icvs); in __kmp_task_alloc()
1497 taskdata->td_flags = *flags; in __kmp_task_alloc()
1498 taskdata->td_task_team = thread->th.th_task_team; in __kmp_task_alloc()
1499 taskdata->td_size_alloc = shareds_offset + sizeof_shareds; in __kmp_task_alloc()
1500 taskdata->td_flags.tasktype = TASK_EXPLICIT; in __kmp_task_alloc()
1505 taskdata->td_team = shadow_thread->th.th_team; in __kmp_task_alloc()
1506 taskdata->td_task_team = shadow_thread->th.th_task_team; in __kmp_task_alloc()
1510 taskdata->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec); in __kmp_task_alloc()
1513 taskdata->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0; in __kmp_task_alloc()
1519 taskdata->td_flags.task_serial = in __kmp_task_alloc()
1520 (parent_task->td_flags.final || taskdata->td_flags.team_serial || in __kmp_task_alloc()
1521 taskdata->td_flags.tasking_ser || flags->merged_if0); in __kmp_task_alloc()
1523 taskdata->td_flags.started = 0; in __kmp_task_alloc()
1524 taskdata->td_flags.executing = 0; in __kmp_task_alloc()
1525 taskdata->td_flags.complete = 0; in __kmp_task_alloc()
1526 taskdata->td_flags.freed = 0; in __kmp_task_alloc()
1528 KMP_ATOMIC_ST_RLX(&taskdata->td_incomplete_child_tasks, 0); in __kmp_task_alloc()
1530 KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 1); in __kmp_task_alloc()
1531 taskdata->td_taskgroup = in __kmp_task_alloc()
1533 taskdata->td_dephash = NULL; in __kmp_task_alloc()
1534 taskdata->td_depnode = NULL; in __kmp_task_alloc()
1536 taskdata->td_last_tied = NULL; // will be set when the task is scheduled in __kmp_task_alloc()
1538 taskdata->td_last_tied = taskdata; in __kmp_task_alloc()
1539 taskdata->td_allow_completion_event.type = KMP_EVENT_UNINITIALIZED; in __kmp_task_alloc()
1542 __ompt_task_init(taskdata, gtid); in __kmp_task_alloc()
1546 if (__kmp_track_children_task(taskdata)) { in __kmp_task_alloc()
1552 if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT) { in __kmp_task_alloc()
1553 KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks); in __kmp_task_alloc()
1556 taskdata->td_flags.task_serial = FALSE; in __kmp_task_alloc()
1563 gtid, taskdata, taskdata->td_parent)); in __kmp_task_alloc()
1636 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_invoke_task() local
1641 gtid, taskdata, current_task)); in __kmp_invoke_task()
1643 if (UNLIKELY(taskdata->td_flags.proxy == TASK_PROXY && in __kmp_invoke_task()
1644 taskdata->td_flags.complete == 1)) { in __kmp_invoke_task()
1650 gtid, taskdata)); in __kmp_invoke_task()
1656 gtid, taskdata, current_task)); in __kmp_invoke_task()
1673 taskdata->ompt_task_info.frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); in __kmp_invoke_task()
1678 if (taskdata->td_flags.hidden_helper) { in __kmp_invoke_task()
1685 if (taskdata->td_flags.proxy != TASK_PROXY) { in __kmp_invoke_task()
1695 kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup; in __kmp_invoke_task()
1719 if (taskdata->td_flags.tiedness == TASK_UNTIED) { in __kmp_invoke_task()
1720 taskdata->td_last_tied = current_task->td_last_tied; in __kmp_invoke_task()
1721 KMP_DEBUG_ASSERT(taskdata->td_last_tied); in __kmp_invoke_task()
1754 taskdata->ompt_task_info.dispatch_chunk.iterations > 0)) { in __kmp_invoke_task()
1756 instance.ptr = &(taskdata->ompt_task_info.dispatch_chunk); in __kmp_invoke_task()
1759 &(team_info->parallel_data), &(taskdata->ompt_task_info.task_data), in __kmp_invoke_task()
1761 taskdata->ompt_task_info.dispatch_chunk = {0, 0}; in __kmp_invoke_task()
1773 __kmp_forkjoin_frames_mode == 3 && !taskdata->td_flags.task_serial && in __kmp_invoke_task()
1783 KMP_FSYNC_ACQUIRED(taskdata); // acquired self (new task) in __kmp_invoke_task()
1788 if (taskdata->td_flags.native) { in __kmp_invoke_task()
1803 KMP_FSYNC_CANCEL(taskdata); // destroy self (just executed) in __kmp_invoke_task()
1804 KMP_FSYNC_RELEASING(taskdata->td_parent); // releasing parent in __kmp_invoke_task()
1814 if (taskdata->td_flags.proxy != TASK_PROXY) { in __kmp_invoke_task()
1818 if (taskdata->td_flags.tiedness == TASK_TIED) { in __kmp_invoke_task()
1819 taskdata->ompt_task_info.frame.exit_frame = ompt_data_none; in __kmp_invoke_task()
1830 gtid, taskdata, current_task)); in __kmp_invoke_task()
2053 kmp_taskdata_t *taskdata = nullptr; in __kmpc_omp_taskwait_template() local
2063 taskdata = thread->th.th_current_task; in __kmpc_omp_taskwait_template()
2070 my_task_data = &(taskdata->ompt_task_info.task_data); in __kmpc_omp_taskwait_template()
2073 taskdata->ompt_task_info.frame.enter_frame.ptr = frame_address; in __kmpc_omp_taskwait_template()
2094 taskdata->td_taskwait_counter += 1; in __kmpc_omp_taskwait_template()
2095 taskdata->td_taskwait_ident = loc_ref; in __kmpc_omp_taskwait_template()
2096 taskdata->td_taskwait_thread = gtid + 1; in __kmpc_omp_taskwait_template()
2106 !taskdata->td_flags.team_serial && !taskdata->td_flags.final; in __kmpc_omp_taskwait_template()
2119 &(taskdata->td_incomplete_child_tasks)), in __kmpc_omp_taskwait_template()
2121 while (KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) != 0) { in __kmpc_omp_taskwait_template()
2129 KMP_FSYNC_ACQUIRED(taskdata); // acquire self - sync with children in __kmpc_omp_taskwait_template()
2134 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; in __kmpc_omp_taskwait_template()
2148 taskdata->ompt_task_info.frame.enter_frame = ompt_data_none; in __kmpc_omp_taskwait_template()
2156 gtid, taskdata)); in __kmpc_omp_taskwait_template()
2186 kmp_taskdata_t *taskdata = NULL; in __kmpc_omp_taskyield() local
2199 taskdata = thread->th.th_current_task; in __kmpc_omp_taskyield()
2206 taskdata->td_taskwait_counter += 1; in __kmpc_omp_taskyield()
2207 taskdata->td_taskwait_ident = loc_ref; in __kmpc_omp_taskyield()
2208 taskdata->td_taskwait_thread = gtid + 1; in __kmpc_omp_taskyield()
2216 if (!taskdata->td_flags.team_serial) { in __kmpc_omp_taskyield()
2241 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; in __kmpc_omp_taskyield()
2246 gtid, taskdata)); in __kmpc_omp_taskyield()
2671 kmp_taskdata_t *taskdata = thread->th.th_current_task; in __kmpc_taskgroup() local
2677 tg_new->parent = taskdata->td_taskgroup; in __kmpc_taskgroup()
2681 taskdata->td_taskgroup = tg_new; in __kmpc_taskgroup()
2689 ompt_data_t my_task_data = taskdata->ompt_task_info.task_data; in __kmpc_taskgroup()
2705 kmp_taskdata_t *taskdata = thread->th.th_current_task; in __kmpc_end_taskgroup() local
2706 kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup; in __kmpc_end_taskgroup()
2716 my_task_data = taskdata->ompt_task_info.task_data; in __kmpc_end_taskgroup()
2731 taskdata->td_taskwait_counter += 1; in __kmpc_end_taskgroup()
2732 taskdata->td_taskwait_ident = loc; in __kmpc_end_taskgroup()
2733 taskdata->td_taskwait_thread = gtid + 1; in __kmpc_end_taskgroup()
2751 if (!taskdata->td_flags.team_serial || in __kmpc_end_taskgroup()
2763 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; // end waiting in __kmpc_end_taskgroup()
2775 KMP_FSYNC_ACQUIRED(taskdata); // acquire self - sync with descendants in __kmpc_end_taskgroup()
2830 taskdata->td_taskgroup = taskgroup->parent; in __kmpc_end_taskgroup()
2834 gtid, taskdata)); in __kmpc_end_taskgroup()
2849 kmp_taskdata_t *taskdata; in __kmp_get_priority_task() local
2887 taskdata = thread_data->td.td_deque[target]; in __kmp_get_priority_task()
2888 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) { in __kmp_get_priority_task()
2905 taskdata = NULL; in __kmp_get_priority_task()
2908 taskdata = thread_data->td.td_deque[target]; in __kmp_get_priority_task()
2909 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) { in __kmp_get_priority_task()
2912 taskdata = NULL; in __kmp_get_priority_task()
2915 if (taskdata == NULL) { in __kmp_get_priority_task()
2940 task = KMP_TASKDATA_TO_TASK(taskdata); in __kmp_get_priority_task()
2949 kmp_taskdata_t *taskdata; in __kmp_remove_my_task() local
2986 taskdata = thread_data->td.td_deque[tail]; in __kmp_remove_my_task()
2988 if (!__kmp_task_is_allowed(gtid, is_constrained, taskdata, in __kmp_remove_my_task()
3007 gtid, taskdata, thread_data->td.td_deque_ntasks, in __kmp_remove_my_task()
3010 task = KMP_TASKDATA_TO_TASK(taskdata); in __kmp_remove_my_task()
3023 kmp_taskdata_t *taskdata; in __kmp_steal_task() local
3067 taskdata = victim_td->td.td_deque[victim_td->td.td_deque_head]; in __kmp_steal_task()
3068 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) { in __kmp_steal_task()
3085 taskdata = NULL; in __kmp_steal_task()
3088 taskdata = victim_td->td.td_deque[target]; in __kmp_steal_task()
3089 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) { in __kmp_steal_task()
3092 taskdata = NULL; in __kmp_steal_task()
3095 if (taskdata == NULL) { in __kmp_steal_task()
3138 gtid, taskdata, __kmp_gtid_from_thread(victim_thr), task_team, in __kmp_steal_task()
3141 task = KMP_TASKDATA_TO_TASK(taskdata); in __kmp_steal_task()
4104 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_give_task() local
4105 kmp_task_team_t *task_team = taskdata->td_task_team; in __kmp_give_task()
4108 taskdata, tid)); in __kmp_give_task()
4121 tid, taskdata)); in __kmp_give_task()
4130 taskdata, tid)); in __kmp_give_task()
4152 taskdata, tid)); in __kmp_give_task()
4165 thread_data->td.td_deque[thread_data->td.td_deque_tail] = taskdata; in __kmp_give_task()
4174 taskdata, tid)); in __kmp_give_task()
4199 static void __kmp_first_top_half_finish_proxy(kmp_taskdata_t *taskdata) { in __kmp_first_top_half_finish_proxy() argument
4200 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); in __kmp_first_top_half_finish_proxy()
4201 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); in __kmp_first_top_half_finish_proxy()
4202 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); in __kmp_first_top_half_finish_proxy()
4203 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); in __kmp_first_top_half_finish_proxy()
4205 taskdata->td_flags.complete = 1; // mark the task as completed in __kmp_first_top_half_finish_proxy()
4207 if (taskdata->td_taskgroup) in __kmp_first_top_half_finish_proxy()
4208 KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count); in __kmp_first_top_half_finish_proxy()
4212 KMP_ATOMIC_OR(&taskdata->td_incomplete_child_tasks, PROXY_TASK_FLAG); in __kmp_first_top_half_finish_proxy()
4215 static void __kmp_second_top_half_finish_proxy(kmp_taskdata_t *taskdata) { in __kmp_second_top_half_finish_proxy() argument
4221 KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks); in __kmp_second_top_half_finish_proxy()
4225 KMP_ATOMIC_AND(&taskdata->td_incomplete_child_tasks, ~PROXY_TASK_FLAG); in __kmp_second_top_half_finish_proxy()
4229 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); in __kmp_bottom_half_finish_proxy() local
4232 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); in __kmp_bottom_half_finish_proxy()
4233 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == in __kmp_bottom_half_finish_proxy()
4238 while ((KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) & in __kmp_bottom_half_finish_proxy()
4242 __kmp_release_deps(gtid, taskdata); in __kmp_bottom_half_finish_proxy()
4243 __kmp_free_task_and_ancestors(gtid, taskdata, thread); in __kmp_bottom_half_finish_proxy()
4256 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); in __kmpc_proxy_task_completed() local
4259 gtid, taskdata)); in __kmpc_proxy_task_completed()
4261 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); in __kmpc_proxy_task_completed()
4263 __kmp_first_top_half_finish_proxy(taskdata); in __kmpc_proxy_task_completed()
4264 __kmp_second_top_half_finish_proxy(taskdata); in __kmpc_proxy_task_completed()
4269 gtid, taskdata)); in __kmpc_proxy_task_completed()
4274 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); in __kmpc_give_task() local
4278 kmp_team_t *team = taskdata->td_team; in __kmpc_give_task()
4320 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); in __kmpc_proxy_task_completed_ooo() local
4325 taskdata)); in __kmpc_proxy_task_completed_ooo()
4327 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); in __kmpc_proxy_task_completed_ooo()
4329 __kmp_first_top_half_finish_proxy(taskdata); in __kmpc_proxy_task_completed_ooo()
4333 __kmp_second_top_half_finish_proxy(taskdata); in __kmpc_proxy_task_completed_ooo()
4338 taskdata)); in __kmpc_proxy_task_completed_ooo()
4355 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); in __kmp_fulfill_event() local
4363 if (taskdata->td_flags.proxy == TASK_PROXY) { in __kmp_fulfill_event()
4385 kmp_team_t *team = taskdata->td_team; in __kmp_fulfill_event()
4407 kmp_taskdata_t *taskdata; in __kmp_task_dup_alloc() local
4424 taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, task_size); in __kmp_task_dup_alloc()
4426 taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, task_size); in __kmp_task_dup_alloc()
4428 KMP_MEMCPY(taskdata, taskdata_src, task_size); in __kmp_task_dup_alloc()
4430 task = KMP_TASKDATA_TO_TASK(taskdata); in __kmp_task_dup_alloc()
4433 taskdata->td_task_id = KMP_GEN_TASK_ID(); in __kmp_task_dup_alloc()
4436 task->shareds = &((char *)taskdata)[shareds_offset]; in __kmp_task_dup_alloc()
4440 taskdata->td_alloc_thread = thread; in __kmp_task_dup_alloc()
4441 taskdata->td_parent = parent_task; in __kmp_task_dup_alloc()
4443 taskdata->td_taskgroup = parent_task->td_taskgroup; in __kmp_task_dup_alloc()
4446 if (taskdata->td_flags.tiedness == TASK_TIED) in __kmp_task_dup_alloc()
4447 taskdata->td_last_tied = taskdata; in __kmp_task_dup_alloc()
4451 if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) { in __kmp_task_dup_alloc()
4457 if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT) in __kmp_task_dup_alloc()
4458 KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks); in __kmp_task_dup_alloc()
4463 thread, taskdata, taskdata->td_parent)); in __kmp_task_dup_alloc()
4466 __ompt_task_init(taskdata, thread->th.th_info.ds.ds_gtid); in __kmp_task_dup_alloc()
4484 const kmp_taskdata_t *taskdata; member in kmp_taskloop_bounds_t
4490 : task(_task), taskdata(KMP_TASK_TO_TASKDATA(task)), in kmp_taskloop_bounds_t()
4497 : task(_task), taskdata(KMP_TASK_TO_TASKDATA(_task)), in kmp_taskloop_bounds_t()
4505 if (!taskdata->td_flags.native) { in get_lb()
4509 if (taskdata->td_size_loop_bounds == 4) { in get_lb()
4518 (void)taskdata; in get_lb()
4527 if (!taskdata->td_flags.native) { in get_ub()
4531 if (taskdata->td_size_loop_bounds == 4) { in get_ub()
4547 if (!taskdata->td_flags.native) { in set_lb()
4551 if (taskdata->td_size_loop_bounds == 4) { in set_lb()
4566 if (!taskdata->td_flags.native) { in set_ub()
4570 if (taskdata->td_size_loop_bounds == 4) { in set_ub()
4751 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_taskloop_task() local
4756 gtid, taskdata, num_tasks, grainsize, extras, last_chunk, *lb, *ub, in __kmp_taskloop_task()
4807 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_taskloop_recur() local
4813 gtid, taskdata, num_tasks, grainsize, extras, last_chunk, *lb, *ub, in __kmp_taskloop_recur()
4867 thread->th.th_current_task = taskdata->td_parent; in __kmp_taskloop_recur()
4920 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_taskloop() local
4946 gtid, taskdata, lower, upper, st, grainsize, sched, modifier, in __kmp_taskloop()
5029 taskdata->td_flags.task_serial = 1; in __kmp_taskloop()
5030 taskdata->td_flags.tiedness = TASK_TIED; // AC: serial task cannot be untied in __kmp_taskloop()
5040 } else if (num_tasks > num_tasks_min && !taskdata->td_flags.native) { in __kmp_taskloop()