Lines Matching refs:taskdata
325 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_push_task() local
328 if (taskdata->td_flags.hidden_helper && !KMP_HIDDEN_HELPER_THREAD(gtid)) { in __kmp_push_task()
338 ("__kmp_push_task: T#%d trying to push task %p.\n", gtid, taskdata)); in __kmp_push_task()
340 if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) { in __kmp_push_task()
343 kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count); in __kmp_push_task()
348 gtid, counter, taskdata)); in __kmp_push_task()
352 if (UNLIKELY(taskdata->td_flags.task_serial)) { in __kmp_push_task()
355 gtid, taskdata)); in __kmp_push_task()
383 __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata, in __kmp_push_task()
387 gtid, taskdata)); in __kmp_push_task()
406 __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata, in __kmp_push_task()
411 gtid, taskdata)); in __kmp_push_task()
424 taskdata; // Push taskdata in __kmp_push_task()
431 KMP_FSYNC_RELEASING(taskdata); // releasing child in __kmp_push_task()
434 gtid, taskdata, thread_data->td.td_deque_ntasks, in __kmp_push_task()
437 auto hidden_helper = taskdata->td_flags.hidden_helper; in __kmp_push_task()
514 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_task_start() local
519 gtid, taskdata, current_task)); in __kmp_task_start()
521 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); in __kmp_task_start()
530 if (taskdata->td_flags.tiedness == TASK_TIED) { in __kmp_task_start()
531 __kmp_push_task_stack(gtid, thread, taskdata); in __kmp_task_start()
536 thread->th.th_current_task = taskdata; in __kmp_task_start()
538 KMP_DEBUG_ASSERT(taskdata->td_flags.started == 0 || in __kmp_task_start()
539 taskdata->td_flags.tiedness == TASK_UNTIED); in __kmp_task_start()
540 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0 || in __kmp_task_start()
541 taskdata->td_flags.tiedness == TASK_UNTIED); in __kmp_task_start()
542 taskdata->td_flags.started = 1; in __kmp_task_start()
543 taskdata->td_flags.executing = 1; in __kmp_task_start()
544 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); in __kmp_task_start()
545 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); in __kmp_task_start()
552 KA_TRACE(10, ("__kmp_task_start(exit): T#%d task=%p\n", gtid, taskdata)); in __kmp_task_start()
579 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __ompt_task_start() local
589 &(taskdata->ompt_task_info.task_data)); in __ompt_task_start()
591 taskdata->ompt_task_info.scheduling_parent = current_task; in __ompt_task_start()
600 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __ompt_task_finish() local
601 if (__kmp_omp_cancellation && taskdata->td_taskgroup && in __ompt_task_finish()
602 taskdata->td_taskgroup->cancel_request == cancel_taskgroup) { in __ompt_task_finish()
608 &(taskdata->ompt_task_info.task_data), status, in __ompt_task_finish()
619 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmpc_omp_task_begin_if0_template() local
624 gtid, loc_ref, taskdata, current_task)); in __kmpc_omp_task_begin_if0_template()
626 if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) { in __kmpc_omp_task_begin_if0_template()
629 kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count); in __kmpc_omp_task_begin_if0_template()
633 gtid, counter, taskdata)); in __kmpc_omp_task_begin_if0_template()
636 taskdata->td_flags.task_serial = in __kmpc_omp_task_begin_if0_template()
644 taskdata->ompt_task_info.frame.exit_frame.ptr = frame_address; in __kmpc_omp_task_begin_if0_template()
646 taskdata->ompt_task_info.frame.exit_frame_flags = in __kmpc_omp_task_begin_if0_template()
653 &(taskdata->ompt_task_info.task_data), in __kmpc_omp_task_begin_if0_template()
654 ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(taskdata), 0, in __kmpc_omp_task_begin_if0_template()
662 loc_ref, taskdata)); in __kmpc_omp_task_begin_if0_template()
720 static void __kmp_free_task(kmp_int32 gtid, kmp_taskdata_t *taskdata, in __kmp_free_task() argument
723 taskdata)); in __kmp_free_task()
726 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); in __kmp_free_task()
727 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0); in __kmp_free_task()
728 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 1); in __kmp_free_task()
729 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); in __kmp_free_task()
730 KMP_DEBUG_ASSERT(taskdata->td_allocated_child_tasks == 0 || in __kmp_free_task()
731 taskdata->td_flags.task_serial == 1); in __kmp_free_task()
732 KMP_DEBUG_ASSERT(taskdata->td_incomplete_child_tasks == 0); in __kmp_free_task()
734 taskdata->td_flags.freed = 1; in __kmp_free_task()
737 __kmp_fast_free(thread, taskdata); in __kmp_free_task()
739 __kmp_thread_free(thread, taskdata); in __kmp_free_task()
741 KA_TRACE(20, ("__kmp_free_task: T#%d freed task %p\n", gtid, taskdata)); in __kmp_free_task()
751 kmp_taskdata_t *taskdata, in __kmp_free_task_and_ancestors() argument
756 (taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser) && in __kmp_free_task_and_ancestors()
757 !taskdata->td_flags.proxy; in __kmp_free_task_and_ancestors()
758 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); in __kmp_free_task_and_ancestors()
760 kmp_int32 children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks) - 1; in __kmp_free_task_and_ancestors()
765 kmp_taskdata_t *parent_taskdata = taskdata->td_parent; in __kmp_free_task_and_ancestors()
769 gtid, taskdata)); in __kmp_free_task_and_ancestors()
772 __kmp_free_task(gtid, taskdata, thread); in __kmp_free_task_and_ancestors()
774 taskdata = parent_taskdata; in __kmp_free_task_and_ancestors()
780 if (taskdata->td_flags.tasktype == TASK_IMPLICIT) { in __kmp_free_task_and_ancestors()
781 if (taskdata->td_dephash) { // do we need to cleanup dephash? in __kmp_free_task_and_ancestors()
782 int children = KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks); in __kmp_free_task_and_ancestors()
783 kmp_tasking_flags_t flags_old = taskdata->td_flags; in __kmp_free_task_and_ancestors()
788 RCAST(kmp_int32 *, &taskdata->td_flags), in __kmp_free_task_and_ancestors()
793 gtid, taskdata)); in __kmp_free_task_and_ancestors()
795 __kmp_dephash_free_entries(thread, taskdata->td_dephash); in __kmp_free_task_and_ancestors()
802 children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks) - 1; in __kmp_free_task_and_ancestors()
809 gtid, taskdata, children)); in __kmp_free_task_and_ancestors()
824 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_task_finish() local
832 gtid, taskdata, resumed_task)); in __kmp_task_finish()
834 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); in __kmp_task_finish()
838 if (taskdata->td_flags.tiedness == TASK_TIED) { in __kmp_task_finish()
839 __kmp_pop_task_stack(gtid, thread, taskdata); in __kmp_task_finish()
843 if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) { in __kmp_task_finish()
846 kmp_int32 counter = KMP_ATOMIC_DEC(&taskdata->td_untied_count) - 1; in __kmp_task_finish()
850 gtid, counter, taskdata)); in __kmp_task_finish()
855 KMP_DEBUG_ASSERT(taskdata->td_flags.task_serial); in __kmp_task_finish()
856 resumed_task = taskdata->td_parent; // In a serialized task, the resumed in __kmp_task_finish()
863 gtid, taskdata, resumed_task)); in __kmp_task_finish()
871 (taskdata->td_flags.tasking_ser || taskdata->td_flags.task_serial) == in __kmp_task_finish()
872 taskdata->td_flags.task_serial); in __kmp_task_finish()
873 if (taskdata->td_flags.task_serial) { in __kmp_task_finish()
875 resumed_task = taskdata->td_parent; // In a serialized task, the resumed in __kmp_task_finish()
889 if (UNLIKELY(taskdata->td_flags.destructors_thunk)) { in __kmp_task_finish()
895 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); in __kmp_task_finish()
896 KMP_DEBUG_ASSERT(taskdata->td_flags.started == 1); in __kmp_task_finish()
897 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); in __kmp_task_finish()
900 if (UNLIKELY(taskdata->td_flags.detachable == TASK_DETACHABLE)) { in __kmp_task_finish()
901 if (taskdata->td_allow_completion_event.type == in __kmp_task_finish()
904 __kmp_acquire_tas_lock(&taskdata->td_allow_completion_event.lock, gtid); in __kmp_task_finish()
905 if (taskdata->td_allow_completion_event.type == in __kmp_task_finish()
908 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1); in __kmp_task_finish()
909 taskdata->td_flags.executing = 0; // suspend the finishing task in __kmp_task_finish()
922 taskdata->td_flags.proxy = TASK_PROXY; // proxify! in __kmp_task_finish()
925 __kmp_release_tas_lock(&taskdata->td_allow_completion_event.lock, gtid); in __kmp_task_finish()
930 taskdata->td_flags.complete = 1; // mark the task as completed in __kmp_task_finish()
940 if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser) || in __kmp_task_finish()
941 taskdata->td_flags.detachable == TASK_DETACHABLE || in __kmp_task_finish()
942 taskdata->td_flags.hidden_helper) { in __kmp_task_finish()
943 __kmp_release_deps(gtid, taskdata); in __kmp_task_finish()
946 KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks) - 1; in __kmp_task_finish()
948 if (taskdata->td_taskgroup) in __kmp_task_finish()
949 KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count); in __kmp_task_finish()
954 __kmp_release_deps(gtid, taskdata); in __kmp_task_finish()
960 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1); in __kmp_task_finish()
961 taskdata->td_flags.executing = 0; // suspend the finishing task in __kmp_task_finish()
966 gtid, taskdata, children)); in __kmp_task_finish()
974 __kmp_free_task_and_ancestors(gtid, taskdata, thread); in __kmp_task_finish()
982 gtid, taskdata, resumed_task)); in __kmp_task_finish()
1190 kmp_taskdata_t *taskdata; in __kmp_task_alloc() local
1300 taskdata = (kmp_taskdata_t *)__kmp_fast_allocate( in __kmp_task_alloc()
1303 taskdata = (kmp_taskdata_t *)__kmp_thread_malloc( in __kmp_task_alloc()
1307 task = KMP_TASKDATA_TO_TASK(taskdata); in __kmp_task_alloc()
1311 KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(double) - 1)) == 0); in __kmp_task_alloc()
1314 KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(_Quad) - 1)) == 0); in __kmp_task_alloc()
1319 task->shareds = &((char *)taskdata)[shareds_offset]; in __kmp_task_alloc()
1329 taskdata->td_task_id = KMP_GEN_TASK_ID(); in __kmp_task_alloc()
1330 taskdata->td_team = thread->th.th_team; in __kmp_task_alloc()
1331 taskdata->td_alloc_thread = encountering_thread; in __kmp_task_alloc()
1332 taskdata->td_parent = parent_task; in __kmp_task_alloc()
1333 taskdata->td_level = parent_task->td_level + 1; // increment nesting level in __kmp_task_alloc()
1334 KMP_ATOMIC_ST_RLX(&taskdata->td_untied_count, 0); in __kmp_task_alloc()
1335 taskdata->td_ident = loc_ref; in __kmp_task_alloc()
1336 taskdata->td_taskwait_ident = NULL; in __kmp_task_alloc()
1337 taskdata->td_taskwait_counter = 0; in __kmp_task_alloc()
1338 taskdata->td_taskwait_thread = 0; in __kmp_task_alloc()
1339 KMP_DEBUG_ASSERT(taskdata->td_parent != NULL); in __kmp_task_alloc()
1342 copy_icvs(&taskdata->td_icvs, &taskdata->td_parent->td_icvs); in __kmp_task_alloc()
1344 taskdata->td_flags = *flags; in __kmp_task_alloc()
1345 taskdata->encountering_gtid = gtid; in __kmp_task_alloc()
1346 taskdata->td_task_team = thread->th.th_task_team; in __kmp_task_alloc()
1347 taskdata->td_size_alloc = shareds_offset + sizeof_shareds; in __kmp_task_alloc()
1348 taskdata->td_flags.tasktype = TASK_EXPLICIT; in __kmp_task_alloc()
1351 taskdata->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec); in __kmp_task_alloc()
1354 taskdata->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0; in __kmp_task_alloc()
1360 taskdata->td_flags.task_serial = in __kmp_task_alloc()
1361 (parent_task->td_flags.final || taskdata->td_flags.team_serial || in __kmp_task_alloc()
1362 taskdata->td_flags.tasking_ser || flags->merged_if0); in __kmp_task_alloc()
1364 taskdata->td_flags.started = 0; in __kmp_task_alloc()
1365 taskdata->td_flags.executing = 0; in __kmp_task_alloc()
1366 taskdata->td_flags.complete = 0; in __kmp_task_alloc()
1367 taskdata->td_flags.freed = 0; in __kmp_task_alloc()
1369 KMP_ATOMIC_ST_RLX(&taskdata->td_incomplete_child_tasks, 0); in __kmp_task_alloc()
1371 KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 1); in __kmp_task_alloc()
1372 taskdata->td_taskgroup = in __kmp_task_alloc()
1374 taskdata->td_dephash = NULL; in __kmp_task_alloc()
1375 taskdata->td_depnode = NULL; in __kmp_task_alloc()
1377 taskdata->td_last_tied = NULL; // will be set when the task is scheduled in __kmp_task_alloc()
1379 taskdata->td_last_tied = taskdata; in __kmp_task_alloc()
1380 taskdata->td_allow_completion_event.type = KMP_EVENT_UNINITIALIZED; in __kmp_task_alloc()
1383 __ompt_task_init(taskdata, gtid); in __kmp_task_alloc()
1389 !(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) { in __kmp_task_alloc()
1395 if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT) { in __kmp_task_alloc()
1396 KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks); in __kmp_task_alloc()
1399 taskdata->td_flags.task_serial = FALSE; in __kmp_task_alloc()
1406 gtid, taskdata, taskdata->td_parent)); in __kmp_task_alloc()
1478 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_invoke_task() local
1483 gtid, taskdata, current_task)); in __kmp_invoke_task()
1485 if (UNLIKELY(taskdata->td_flags.proxy == TASK_PROXY && in __kmp_invoke_task()
1486 taskdata->td_flags.complete == 1)) { in __kmp_invoke_task()
1492 gtid, taskdata)); in __kmp_invoke_task()
1498 gtid, taskdata, current_task)); in __kmp_invoke_task()
1515 taskdata->ompt_task_info.frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); in __kmp_invoke_task()
1520 if (taskdata->td_flags.hidden_helper) { in __kmp_invoke_task()
1527 if (taskdata->td_flags.proxy != TASK_PROXY) { in __kmp_invoke_task()
1537 kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup; in __kmp_invoke_task()
1561 if (taskdata->td_flags.tiedness == TASK_UNTIED) { in __kmp_invoke_task()
1562 taskdata->td_last_tied = current_task->td_last_tied; in __kmp_invoke_task()
1563 KMP_DEBUG_ASSERT(taskdata->td_last_tied); in __kmp_invoke_task()
1603 __kmp_forkjoin_frames_mode == 3 && !taskdata->td_flags.task_serial && in __kmp_invoke_task()
1613 KMP_FSYNC_ACQUIRED(taskdata); // acquired self (new task) in __kmp_invoke_task()
1617 if (taskdata->td_flags.native) { in __kmp_invoke_task()
1631 KMP_FSYNC_CANCEL(taskdata); // destroy self (just executed) in __kmp_invoke_task()
1632 KMP_FSYNC_RELEASING(taskdata->td_parent); // releasing parent in __kmp_invoke_task()
1642 if (taskdata->td_flags.proxy != TASK_PROXY) { in __kmp_invoke_task()
1646 if (taskdata->td_flags.tiedness == TASK_TIED) { in __kmp_invoke_task()
1647 taskdata->ompt_task_info.frame.exit_frame = ompt_data_none; in __kmp_invoke_task()
1658 gtid, taskdata, current_task)); in __kmp_invoke_task()
1868 kmp_taskdata_t *taskdata = nullptr; in __kmpc_omp_taskwait_template() local
1878 taskdata = thread->th.th_current_task; in __kmpc_omp_taskwait_template()
1885 my_task_data = &(taskdata->ompt_task_info.task_data); in __kmpc_omp_taskwait_template()
1888 taskdata->ompt_task_info.frame.enter_frame.ptr = frame_address; in __kmpc_omp_taskwait_template()
1909 taskdata->td_taskwait_counter += 1; in __kmpc_omp_taskwait_template()
1910 taskdata->td_taskwait_ident = loc_ref; in __kmpc_omp_taskwait_template()
1911 taskdata->td_taskwait_thread = gtid + 1; in __kmpc_omp_taskwait_template()
1921 !taskdata->td_flags.team_serial && !taskdata->td_flags.final; in __kmpc_omp_taskwait_template()
1934 &(taskdata->td_incomplete_child_tasks)), in __kmpc_omp_taskwait_template()
1936 while (KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) != 0) { in __kmpc_omp_taskwait_template()
1944 KMP_FSYNC_ACQUIRED(taskdata); // acquire self - sync with children in __kmpc_omp_taskwait_template()
1949 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; in __kmpc_omp_taskwait_template()
1963 taskdata->ompt_task_info.frame.enter_frame = ompt_data_none; in __kmpc_omp_taskwait_template()
1971 gtid, taskdata)); in __kmpc_omp_taskwait_template()
2001 kmp_taskdata_t *taskdata = NULL; in __kmpc_omp_taskyield() local
2014 taskdata = thread->th.th_current_task; in __kmpc_omp_taskyield()
2021 taskdata->td_taskwait_counter += 1; in __kmpc_omp_taskyield()
2022 taskdata->td_taskwait_ident = loc_ref; in __kmpc_omp_taskyield()
2023 taskdata->td_taskwait_thread = gtid + 1; in __kmpc_omp_taskyield()
2031 if (!taskdata->td_flags.team_serial) { in __kmpc_omp_taskyield()
2056 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; in __kmpc_omp_taskyield()
2061 gtid, taskdata)); in __kmpc_omp_taskyield()
2486 kmp_taskdata_t *taskdata = thread->th.th_current_task; in __kmpc_taskgroup() local
2492 tg_new->parent = taskdata->td_taskgroup; in __kmpc_taskgroup()
2496 taskdata->td_taskgroup = tg_new; in __kmpc_taskgroup()
2504 ompt_data_t my_task_data = taskdata->ompt_task_info.task_data; in __kmpc_taskgroup()
2520 kmp_taskdata_t *taskdata = thread->th.th_current_task; in __kmpc_end_taskgroup() local
2521 kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup; in __kmpc_end_taskgroup()
2531 my_task_data = taskdata->ompt_task_info.task_data; in __kmpc_end_taskgroup()
2546 taskdata->td_taskwait_counter += 1; in __kmpc_end_taskgroup()
2547 taskdata->td_taskwait_ident = loc; in __kmpc_end_taskgroup()
2548 taskdata->td_taskwait_thread = gtid + 1; in __kmpc_end_taskgroup()
2566 if (!taskdata->td_flags.team_serial || in __kmpc_end_taskgroup()
2578 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; // end waiting in __kmpc_end_taskgroup()
2590 KMP_FSYNC_ACQUIRED(taskdata); // acquire self - sync with descendants in __kmpc_end_taskgroup()
2645 taskdata->td_taskgroup = taskgroup->parent; in __kmpc_end_taskgroup()
2649 gtid, taskdata)); in __kmpc_end_taskgroup()
2665 kmp_taskdata_t *taskdata; in __kmp_remove_my_task() local
2702 taskdata = thread_data->td.td_deque[tail]; in __kmp_remove_my_task()
2704 if (!__kmp_task_is_allowed(gtid, is_constrained, taskdata, in __kmp_remove_my_task()
2723 gtid, taskdata, thread_data->td.td_deque_ntasks, in __kmp_remove_my_task()
2726 task = KMP_TASKDATA_TO_TASK(taskdata); in __kmp_remove_my_task()
2739 kmp_taskdata_t *taskdata; in __kmp_steal_task() local
2783 taskdata = victim_td->td.td_deque[victim_td->td.td_deque_head]; in __kmp_steal_task()
2784 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) { in __kmp_steal_task()
2801 taskdata = NULL; in __kmp_steal_task()
2804 taskdata = victim_td->td.td_deque[target]; in __kmp_steal_task()
2805 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) { in __kmp_steal_task()
2808 taskdata = NULL; in __kmp_steal_task()
2811 if (taskdata == NULL) { in __kmp_steal_task()
2855 gtid, taskdata, __kmp_gtid_from_thread(victim_thr), task_team, in __kmp_steal_task()
2858 task = KMP_TASKDATA_TO_TASK(taskdata); in __kmp_steal_task()
3760 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_give_task() local
3761 kmp_task_team_t *task_team = taskdata->td_task_team; in __kmp_give_task()
3764 taskdata, tid)); in __kmp_give_task()
3777 tid, taskdata)); in __kmp_give_task()
3786 taskdata, tid)); in __kmp_give_task()
3808 taskdata, tid)); in __kmp_give_task()
3821 thread_data->td.td_deque[thread_data->td.td_deque_tail] = taskdata; in __kmp_give_task()
3830 taskdata, tid)); in __kmp_give_task()
3855 static void __kmp_first_top_half_finish_proxy(kmp_taskdata_t *taskdata) { in __kmp_first_top_half_finish_proxy() argument
3856 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); in __kmp_first_top_half_finish_proxy()
3857 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); in __kmp_first_top_half_finish_proxy()
3858 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); in __kmp_first_top_half_finish_proxy()
3859 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); in __kmp_first_top_half_finish_proxy()
3861 taskdata->td_flags.complete = 1; // mark the task as completed in __kmp_first_top_half_finish_proxy()
3863 if (taskdata->td_taskgroup) in __kmp_first_top_half_finish_proxy()
3864 KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count); in __kmp_first_top_half_finish_proxy()
3868 KMP_ATOMIC_OR(&taskdata->td_incomplete_child_tasks, PROXY_TASK_FLAG); in __kmp_first_top_half_finish_proxy()
3871 static void __kmp_second_top_half_finish_proxy(kmp_taskdata_t *taskdata) { in __kmp_second_top_half_finish_proxy() argument
3876 KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks) - 1; in __kmp_second_top_half_finish_proxy()
3880 KMP_ATOMIC_AND(&taskdata->td_incomplete_child_tasks, ~PROXY_TASK_FLAG); in __kmp_second_top_half_finish_proxy()
3884 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); in __kmp_bottom_half_finish_proxy() local
3887 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); in __kmp_bottom_half_finish_proxy()
3888 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == in __kmp_bottom_half_finish_proxy()
3893 while ((KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) & in __kmp_bottom_half_finish_proxy()
3897 __kmp_release_deps(gtid, taskdata); in __kmp_bottom_half_finish_proxy()
3898 __kmp_free_task_and_ancestors(gtid, taskdata, thread); in __kmp_bottom_half_finish_proxy()
3911 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); in __kmpc_proxy_task_completed() local
3914 gtid, taskdata)); in __kmpc_proxy_task_completed()
3916 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); in __kmpc_proxy_task_completed()
3918 __kmp_first_top_half_finish_proxy(taskdata); in __kmpc_proxy_task_completed()
3919 __kmp_second_top_half_finish_proxy(taskdata); in __kmpc_proxy_task_completed()
3924 gtid, taskdata)); in __kmpc_proxy_task_completed()
3929 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); in __kmpc_give_task() local
3933 kmp_team_t *team = taskdata->td_team; in __kmpc_give_task()
3964 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); in __kmpc_proxy_task_completed_ooo() local
3969 taskdata)); in __kmpc_proxy_task_completed_ooo()
3971 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); in __kmpc_proxy_task_completed_ooo()
3973 __kmp_first_top_half_finish_proxy(taskdata); in __kmpc_proxy_task_completed_ooo()
3977 __kmp_second_top_half_finish_proxy(taskdata); in __kmpc_proxy_task_completed_ooo()
3982 taskdata)); in __kmpc_proxy_task_completed_ooo()
3999 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); in __kmp_fulfill_event() local
4007 if (taskdata->td_flags.proxy == TASK_PROXY) { in __kmp_fulfill_event()
4029 kmp_team_t *team = taskdata->td_team; in __kmp_fulfill_event()
4051 kmp_taskdata_t *taskdata; in __kmp_task_dup_alloc() local
4068 taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, task_size); in __kmp_task_dup_alloc()
4070 taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, task_size); in __kmp_task_dup_alloc()
4072 KMP_MEMCPY(taskdata, taskdata_src, task_size); in __kmp_task_dup_alloc()
4074 task = KMP_TASKDATA_TO_TASK(taskdata); in __kmp_task_dup_alloc()
4077 taskdata->td_task_id = KMP_GEN_TASK_ID(); in __kmp_task_dup_alloc()
4080 task->shareds = &((char *)taskdata)[shareds_offset]; in __kmp_task_dup_alloc()
4084 taskdata->td_alloc_thread = thread; in __kmp_task_dup_alloc()
4085 taskdata->td_parent = parent_task; in __kmp_task_dup_alloc()
4087 taskdata->td_taskgroup = parent_task->td_taskgroup; in __kmp_task_dup_alloc()
4090 if (taskdata->td_flags.tiedness == TASK_TIED) in __kmp_task_dup_alloc()
4091 taskdata->td_last_tied = taskdata; in __kmp_task_dup_alloc()
4095 if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) { in __kmp_task_dup_alloc()
4101 if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT) in __kmp_task_dup_alloc()
4102 KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks); in __kmp_task_dup_alloc()
4107 thread, taskdata, taskdata->td_parent)); in __kmp_task_dup_alloc()
4110 __ompt_task_init(taskdata, thread->th.th_info.ds.ds_gtid); in __kmp_task_dup_alloc()
4128 const kmp_taskdata_t *taskdata; member in kmp_taskloop_bounds_t
4134 : task(_task), taskdata(KMP_TASK_TO_TASKDATA(task)), in kmp_taskloop_bounds_t()
4141 : task(_task), taskdata(KMP_TASK_TO_TASKDATA(_task)), in kmp_taskloop_bounds_t()
4149 if (!taskdata->td_flags.native) { in get_lb()
4153 if (taskdata->td_size_loop_bounds == 4) { in get_lb()
4162 (void)taskdata; in get_lb()
4171 if (!taskdata->td_flags.native) { in get_ub()
4175 if (taskdata->td_size_loop_bounds == 4) { in get_ub()
4191 if (!taskdata->td_flags.native) { in set_lb()
4195 if (taskdata->td_size_loop_bounds == 4) { in set_lb()
4210 if (!taskdata->td_flags.native) { in set_ub()
4214 if (taskdata->td_size_loop_bounds == 4) { in set_ub()
4389 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_taskloop_task() local
4394 gtid, taskdata, num_tasks, grainsize, extras, last_chunk, *lb, *ub, in __kmp_taskloop_task()
4445 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_taskloop_recur() local
4451 gtid, taskdata, num_tasks, grainsize, extras, last_chunk, *lb, *ub, in __kmp_taskloop_recur()
4505 thread->th.th_current_task = taskdata->td_parent; in __kmp_taskloop_recur()
4558 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_taskloop() local
4584 gtid, taskdata, lower, upper, st, grainsize, sched, modifier, in __kmp_taskloop()
4667 taskdata->td_flags.task_serial = 1; in __kmp_taskloop()
4668 taskdata->td_flags.tiedness = TASK_TIED; // AC: serial task cannot be untied in __kmp_taskloop()
4678 } else if (num_tasks > num_tasks_min && !taskdata->td_flags.native) { in __kmp_taskloop()