Lines Matching refs:taskdata
330 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_push_task() local
336 ("__kmp_push_task: T#%d trying to push task %p.\n", gtid, taskdata)); in __kmp_push_task()
338 if (taskdata->td_flags.tiedness == TASK_UNTIED) { in __kmp_push_task()
341 kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count); in __kmp_push_task()
346 gtid, counter, taskdata)); in __kmp_push_task()
350 if (taskdata->td_flags.task_serial) { in __kmp_push_task()
353 gtid, taskdata)); in __kmp_push_task()
378 if (__kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata, in __kmp_push_task()
382 gtid, taskdata)); in __kmp_push_task()
398 if (__kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata, in __kmp_push_task()
403 gtid, taskdata)); in __kmp_push_task()
417 taskdata; // Push taskdata in __kmp_push_task()
426 gtid, taskdata, thread_data->td.td_deque_ntasks, in __kmp_push_task()
498 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_task_start() local
503 gtid, taskdata, current_task)); in __kmp_task_start()
505 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); in __kmp_task_start()
514 if (taskdata->td_flags.tiedness == TASK_TIED) { in __kmp_task_start()
515 __kmp_push_task_stack(gtid, thread, taskdata); in __kmp_task_start()
520 thread->th.th_current_task = taskdata; in __kmp_task_start()
522 KMP_DEBUG_ASSERT(taskdata->td_flags.started == 0 || in __kmp_task_start()
523 taskdata->td_flags.tiedness == TASK_UNTIED); in __kmp_task_start()
524 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0 || in __kmp_task_start()
525 taskdata->td_flags.tiedness == TASK_UNTIED); in __kmp_task_start()
526 taskdata->td_flags.started = 1; in __kmp_task_start()
527 taskdata->td_flags.executing = 1; in __kmp_task_start()
528 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); in __kmp_task_start()
529 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); in __kmp_task_start()
536 KA_TRACE(10, ("__kmp_task_start(exit): T#%d task=%p\n", gtid, taskdata)); in __kmp_task_start()
565 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __ompt_task_start() local
575 &(taskdata->ompt_task_info.task_data)); in __ompt_task_start()
577 taskdata->ompt_task_info.scheduling_parent = current_task; in __ompt_task_start()
585 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __ompt_task_finish() local
586 if (__kmp_omp_cancellation && taskdata->td_taskgroup && in __ompt_task_finish()
587 taskdata->td_taskgroup->cancel_request == cancel_taskgroup) { in __ompt_task_finish()
594 &(taskdata->ompt_task_info.task_data), status, in __ompt_task_finish()
596 : (taskdata->ompt_task_info.scheduling_parent in __ompt_task_finish()
597 ? taskdata->ompt_task_info.scheduling_parent in __ompt_task_finish()
598 : taskdata->td_parent)) in __ompt_task_finish()
609 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmpc_omp_task_begin_if0_template() local
614 gtid, loc_ref, taskdata, current_task)); in __kmpc_omp_task_begin_if0_template()
616 if (taskdata->td_flags.tiedness == TASK_UNTIED) { in __kmpc_omp_task_begin_if0_template()
619 kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count); in __kmpc_omp_task_begin_if0_template()
623 gtid, counter, taskdata)); in __kmpc_omp_task_begin_if0_template()
626 taskdata->td_flags.task_serial = in __kmpc_omp_task_begin_if0_template()
634 taskdata->ompt_task_info.frame.exit_frame.ptr = frame_address; in __kmpc_omp_task_begin_if0_template()
636 …taskdata->ompt_task_info.frame.exit_frame_flags = ompt_frame_application | ompt_frame_framepointer; in __kmpc_omp_task_begin_if0_template()
642 &(taskdata->ompt_task_info.task_data), in __kmpc_omp_task_begin_if0_template()
643 ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(taskdata), 0, in __kmpc_omp_task_begin_if0_template()
651 loc_ref, taskdata)); in __kmpc_omp_task_begin_if0_template()
709 static void __kmp_free_task(kmp_int32 gtid, kmp_taskdata_t *taskdata, in __kmp_free_task() argument
712 taskdata)); in __kmp_free_task()
715 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); in __kmp_free_task()
716 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0); in __kmp_free_task()
717 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 1); in __kmp_free_task()
718 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); in __kmp_free_task()
719 KMP_DEBUG_ASSERT(taskdata->td_allocated_child_tasks == 0 || in __kmp_free_task()
720 taskdata->td_flags.task_serial == 1); in __kmp_free_task()
721 KMP_DEBUG_ASSERT(taskdata->td_incomplete_child_tasks == 0); in __kmp_free_task()
723 taskdata->td_flags.freed = 1; in __kmp_free_task()
724 ANNOTATE_HAPPENS_BEFORE(taskdata); in __kmp_free_task()
727 __kmp_fast_free(thread, taskdata); in __kmp_free_task()
729 __kmp_thread_free(thread, taskdata); in __kmp_free_task()
732 KA_TRACE(20, ("__kmp_free_task: T#%d freed task %p\n", gtid, taskdata)); in __kmp_free_task()
742 kmp_taskdata_t *taskdata, in __kmp_free_task_and_ancestors() argument
748 (taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser) && in __kmp_free_task_and_ancestors()
749 !taskdata->td_flags.proxy; in __kmp_free_task_and_ancestors()
752 taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser; in __kmp_free_task_and_ancestors()
754 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); in __kmp_free_task_and_ancestors()
756 kmp_int32 children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks) - 1; in __kmp_free_task_and_ancestors()
761 kmp_taskdata_t *parent_taskdata = taskdata->td_parent; in __kmp_free_task_and_ancestors()
765 gtid, taskdata)); in __kmp_free_task_and_ancestors()
768 __kmp_free_task(gtid, taskdata, thread); in __kmp_free_task_and_ancestors()
770 taskdata = parent_taskdata; in __kmp_free_task_and_ancestors()
776 if (taskdata->td_flags.tasktype == TASK_IMPLICIT) { in __kmp_free_task_and_ancestors()
777 if (taskdata->td_dephash) { // do we need to cleanup dephash? in __kmp_free_task_and_ancestors()
778 int children = KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks); in __kmp_free_task_and_ancestors()
779 kmp_tasking_flags_t flags_old = taskdata->td_flags; in __kmp_free_task_and_ancestors()
784 RCAST(kmp_int32 *, &taskdata->td_flags), in __kmp_free_task_and_ancestors()
789 gtid, taskdata)); in __kmp_free_task_and_ancestors()
791 __kmp_dephash_free_entries(thread, taskdata->td_dephash); in __kmp_free_task_and_ancestors()
798 children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks) - 1; in __kmp_free_task_and_ancestors()
805 gtid, taskdata, children)); in __kmp_free_task_and_ancestors()
816 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_task_finish() local
826 gtid, taskdata, resumed_task)); in __kmp_task_finish()
828 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); in __kmp_task_finish()
832 if (taskdata->td_flags.tiedness == TASK_TIED) { in __kmp_task_finish()
833 __kmp_pop_task_stack(gtid, thread, taskdata); in __kmp_task_finish()
837 if (taskdata->td_flags.tiedness == TASK_UNTIED) { in __kmp_task_finish()
840 kmp_int32 counter = KMP_ATOMIC_DEC(&taskdata->td_untied_count) - 1; in __kmp_task_finish()
844 gtid, counter, taskdata)); in __kmp_task_finish()
849 KMP_DEBUG_ASSERT(taskdata->td_flags.task_serial); in __kmp_task_finish()
850 resumed_task = taskdata->td_parent; // In a serialized task, the resumed in __kmp_task_finish()
857 gtid, taskdata, resumed_task)); in __kmp_task_finish()
867 kmp_depnode_t *node = taskdata->td_depnode; in __kmp_task_finish()
877 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); in __kmp_task_finish()
878 taskdata->td_flags.complete = 1; // mark the task as completed in __kmp_task_finish()
879 KMP_DEBUG_ASSERT(taskdata->td_flags.started == 1); in __kmp_task_finish()
880 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); in __kmp_task_finish()
884 if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) { in __kmp_task_finish()
887 KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks) - 1; in __kmp_task_finish()
890 if (taskdata->td_taskgroup) in __kmp_task_finish()
891 KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count); in __kmp_task_finish()
892 __kmp_release_deps(gtid, taskdata); in __kmp_task_finish()
897 __kmp_release_deps(gtid, taskdata); in __kmp_task_finish()
905 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1); in __kmp_task_finish()
906 taskdata->td_flags.executing = 0; // suspend the finishing task in __kmp_task_finish()
910 gtid, taskdata, children)); in __kmp_task_finish()
919 if (taskdata->td_flags.destructors_thunk) { in __kmp_task_finish()
929 (taskdata->td_flags.tasking_ser || taskdata->td_flags.task_serial) == in __kmp_task_finish()
930 taskdata->td_flags.task_serial); in __kmp_task_finish()
931 if (taskdata->td_flags.task_serial) { in __kmp_task_finish()
933 resumed_task = taskdata->td_parent; // In a serialized task, the resumed in __kmp_task_finish()
946 __kmp_free_task_and_ancestors(gtid, taskdata, thread); in __kmp_task_finish()
954 gtid, taskdata, resumed_task)); in __kmp_task_finish()
1165 kmp_taskdata_t *taskdata; in __kmp_task_alloc() local
1244 taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, shareds_offset + in __kmp_task_alloc()
1247 taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, shareds_offset + in __kmp_task_alloc()
1250 ANNOTATE_HAPPENS_AFTER(taskdata); in __kmp_task_alloc()
1252 task = KMP_TASKDATA_TO_TASK(taskdata); in __kmp_task_alloc()
1256 KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(double) - 1)) == 0); in __kmp_task_alloc()
1259 KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(_Quad) - 1)) == 0); in __kmp_task_alloc()
1264 task->shareds = &((char *)taskdata)[shareds_offset]; in __kmp_task_alloc()
1274 taskdata->td_task_id = KMP_GEN_TASK_ID(); in __kmp_task_alloc()
1275 taskdata->td_team = team; in __kmp_task_alloc()
1276 taskdata->td_alloc_thread = thread; in __kmp_task_alloc()
1277 taskdata->td_parent = parent_task; in __kmp_task_alloc()
1278 taskdata->td_level = parent_task->td_level + 1; // increment nesting level in __kmp_task_alloc()
1279 KMP_ATOMIC_ST_RLX(&taskdata->td_untied_count, 0); in __kmp_task_alloc()
1280 taskdata->td_ident = loc_ref; in __kmp_task_alloc()
1281 taskdata->td_taskwait_ident = NULL; in __kmp_task_alloc()
1282 taskdata->td_taskwait_counter = 0; in __kmp_task_alloc()
1283 taskdata->td_taskwait_thread = 0; in __kmp_task_alloc()
1284 KMP_DEBUG_ASSERT(taskdata->td_parent != NULL); in __kmp_task_alloc()
1289 copy_icvs(&taskdata->td_icvs, &taskdata->td_parent->td_icvs); in __kmp_task_alloc()
1291 taskdata->td_flags.tiedness = flags->tiedness; in __kmp_task_alloc()
1292 taskdata->td_flags.final = flags->final; in __kmp_task_alloc()
1293 taskdata->td_flags.merged_if0 = flags->merged_if0; in __kmp_task_alloc()
1295 taskdata->td_flags.destructors_thunk = flags->destructors_thunk; in __kmp_task_alloc()
1298 taskdata->td_flags.proxy = flags->proxy; in __kmp_task_alloc()
1299 taskdata->td_task_team = thread->th.th_task_team; in __kmp_task_alloc()
1300 taskdata->td_size_alloc = shareds_offset + sizeof_shareds; in __kmp_task_alloc()
1302 taskdata->td_flags.tasktype = TASK_EXPLICIT; in __kmp_task_alloc()
1305 taskdata->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec); in __kmp_task_alloc()
1308 taskdata->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0; in __kmp_task_alloc()
1314 taskdata->td_flags.task_serial = in __kmp_task_alloc()
1315 (parent_task->td_flags.final || taskdata->td_flags.team_serial || in __kmp_task_alloc()
1316 taskdata->td_flags.tasking_ser); in __kmp_task_alloc()
1318 taskdata->td_flags.started = 0; in __kmp_task_alloc()
1319 taskdata->td_flags.executing = 0; in __kmp_task_alloc()
1320 taskdata->td_flags.complete = 0; in __kmp_task_alloc()
1321 taskdata->td_flags.freed = 0; in __kmp_task_alloc()
1323 taskdata->td_flags.native = flags->native; in __kmp_task_alloc()
1325 KMP_ATOMIC_ST_RLX(&taskdata->td_incomplete_child_tasks, 0); in __kmp_task_alloc()
1327 KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 1); in __kmp_task_alloc()
1329 taskdata->td_taskgroup = in __kmp_task_alloc()
1331 taskdata->td_dephash = NULL; in __kmp_task_alloc()
1332 taskdata->td_depnode = NULL; in __kmp_task_alloc()
1335 taskdata->td_last_tied = NULL; // will be set when the task is scheduled in __kmp_task_alloc()
1337 taskdata->td_last_tied = taskdata; in __kmp_task_alloc()
1341 __ompt_task_init(taskdata, gtid); in __kmp_task_alloc()
1347 !(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) in __kmp_task_alloc()
1349 if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) in __kmp_task_alloc()
1359 if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT) { in __kmp_task_alloc()
1360 KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks); in __kmp_task_alloc()
1365 gtid, taskdata, taskdata->td_parent)); in __kmp_task_alloc()
1431 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_invoke_task() local
1438 gtid, taskdata, current_task)); in __kmp_invoke_task()
1441 if (taskdata->td_flags.proxy == TASK_PROXY && in __kmp_invoke_task()
1442 taskdata->td_flags.complete == 1) { in __kmp_invoke_task()
1448 gtid, taskdata)); in __kmp_invoke_task()
1454 gtid, taskdata, current_task)); in __kmp_invoke_task()
1472 taskdata->ompt_task_info.frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); in __kmp_invoke_task()
1478 if (taskdata->td_flags.proxy != TASK_PROXY) { in __kmp_invoke_task()
1493 kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup; in __kmp_invoke_task()
1517 if (taskdata->td_flags.tiedness == TASK_UNTIED) { in __kmp_invoke_task()
1518 taskdata->td_last_tied = current_task->td_last_tied; in __kmp_invoke_task()
1519 KMP_DEBUG_ASSERT(taskdata->td_last_tied); in __kmp_invoke_task()
1555 __kmp_forkjoin_frames_mode == 3 && !taskdata->td_flags.task_serial && in __kmp_invoke_task()
1568 if (taskdata->td_flags.native) { in __kmp_invoke_task()
1591 if (taskdata->td_flags.proxy != TASK_PROXY) { in __kmp_invoke_task()
1593 ANNOTATE_HAPPENS_BEFORE(taskdata->td_parent); in __kmp_invoke_task()
1597 if (taskdata->td_flags.tiedness == TASK_TIED) { in __kmp_invoke_task()
1598 taskdata->ompt_task_info.frame.exit_frame = ompt_data_none; in __kmp_invoke_task()
1611 gtid, taskdata, current_task)); in __kmp_invoke_task()
1830 kmp_taskdata_t *taskdata; in __kmpc_omp_taskwait_template() local
1839 taskdata = thread->th.th_current_task; in __kmpc_omp_taskwait_template()
1846 my_task_data = &(taskdata->ompt_task_info.task_data); in __kmpc_omp_taskwait_template()
1849 taskdata->ompt_task_info.frame.enter_frame.ptr = frame_address; in __kmpc_omp_taskwait_template()
1870 taskdata->td_taskwait_counter += 1; in __kmpc_omp_taskwait_template()
1871 taskdata->td_taskwait_ident = loc_ref; in __kmpc_omp_taskwait_template()
1872 taskdata->td_taskwait_thread = gtid + 1; in __kmpc_omp_taskwait_template()
1881 !taskdata->td_flags.team_serial && !taskdata->td_flags.final; in __kmpc_omp_taskwait_template()
1889 &(taskdata->td_incomplete_child_tasks)), in __kmpc_omp_taskwait_template()
1891 while (KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) != 0) { in __kmpc_omp_taskwait_template()
1904 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; in __kmpc_omp_taskwait_template()
1918 taskdata->ompt_task_info.frame.enter_frame = ompt_data_none; in __kmpc_omp_taskwait_template()
1922 ANNOTATE_HAPPENS_AFTER(taskdata); in __kmpc_omp_taskwait_template()
1927 gtid, taskdata)); in __kmpc_omp_taskwait_template()
1957 kmp_taskdata_t *taskdata; in __kmpc_omp_taskyield() local
1969 taskdata = thread->th.th_current_task; in __kmpc_omp_taskyield()
1976 taskdata->td_taskwait_counter += 1; in __kmpc_omp_taskyield()
1977 taskdata->td_taskwait_ident = loc_ref; in __kmpc_omp_taskyield()
1978 taskdata->td_taskwait_thread = gtid + 1; in __kmpc_omp_taskyield()
1985 if (!taskdata->td_flags.team_serial) { in __kmpc_omp_taskyield()
2011 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; in __kmpc_omp_taskyield()
2016 gtid, taskdata)); in __kmpc_omp_taskyield()
2217 kmp_taskdata_t *taskdata = thread->th.th_current_task; in __kmpc_taskgroup() local
2223 tg_new->parent = taskdata->td_taskgroup; in __kmpc_taskgroup()
2228 taskdata->td_taskgroup = tg_new; in __kmpc_taskgroup()
2236 ompt_data_t my_task_data = taskdata->ompt_task_info.task_data; in __kmpc_taskgroup()
2251 kmp_taskdata_t *taskdata = thread->th.th_current_task; in __kmpc_end_taskgroup() local
2252 kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup; in __kmpc_end_taskgroup()
2262 my_task_data = taskdata->ompt_task_info.task_data; in __kmpc_end_taskgroup()
2277 taskdata->td_taskwait_counter += 1; in __kmpc_end_taskgroup()
2278 taskdata->td_taskwait_ident = loc; in __kmpc_end_taskgroup()
2279 taskdata->td_taskwait_thread = gtid + 1; in __kmpc_end_taskgroup()
2297 if (!taskdata->td_flags.team_serial || in __kmpc_end_taskgroup()
2301 if (!taskdata->td_flags.team_serial) in __kmpc_end_taskgroup()
2312 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; // end waiting in __kmpc_end_taskgroup()
2334 taskdata->td_taskgroup = taskgroup->parent; in __kmpc_end_taskgroup()
2338 gtid, taskdata)); in __kmpc_end_taskgroup()
2339 ANNOTATE_HAPPENS_AFTER(taskdata); in __kmpc_end_taskgroup()
2356 kmp_taskdata_t *taskdata; in __kmp_remove_my_task() local
2393 taskdata = thread_data->td.td_deque[tail]; in __kmp_remove_my_task()
2395 if (!__kmp_task_is_allowed(gtid, is_constrained, taskdata, in __kmp_remove_my_task()
2414 gtid, taskdata, thread_data->td.td_deque_ntasks, in __kmp_remove_my_task()
2417 task = KMP_TASKDATA_TO_TASK(taskdata); in __kmp_remove_my_task()
2430 kmp_taskdata_t *taskdata; in __kmp_steal_task() local
2474 taskdata = victim_td->td.td_deque[victim_td->td.td_deque_head]; in __kmp_steal_task()
2475 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) { in __kmp_steal_task()
2492 taskdata = NULL; in __kmp_steal_task()
2495 taskdata = victim_td->td.td_deque[target]; in __kmp_steal_task()
2496 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) { in __kmp_steal_task()
2499 taskdata = NULL; in __kmp_steal_task()
2502 if (taskdata == NULL) { in __kmp_steal_task()
2546 gtid, taskdata, __kmp_gtid_from_thread(victim_thr), task_team, in __kmp_steal_task()
2549 task = KMP_TASKDATA_TO_TASK(taskdata); in __kmp_steal_task()
3430 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_give_task() local
3431 kmp_task_team_t *task_team = taskdata->td_task_team; in __kmp_give_task()
3434 taskdata, tid)); in __kmp_give_task()
3447 tid, taskdata)); in __kmp_give_task()
3456 taskdata, tid)); in __kmp_give_task()
3474 taskdata, tid)); in __kmp_give_task()
3487 thread_data->td.td_deque[thread_data->td.td_deque_tail] = taskdata; in __kmp_give_task()
3496 taskdata, tid)); in __kmp_give_task()
3520 static void __kmp_first_top_half_finish_proxy(kmp_taskdata_t *taskdata) { in __kmp_first_top_half_finish_proxy() argument
3521 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); in __kmp_first_top_half_finish_proxy()
3522 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); in __kmp_first_top_half_finish_proxy()
3523 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); in __kmp_first_top_half_finish_proxy()
3524 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); in __kmp_first_top_half_finish_proxy()
3526 taskdata->td_flags.complete = 1; // mark the task as completed in __kmp_first_top_half_finish_proxy()
3528 if (taskdata->td_taskgroup) in __kmp_first_top_half_finish_proxy()
3529 KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count); in __kmp_first_top_half_finish_proxy()
3533 KMP_ATOMIC_INC(&taskdata->td_incomplete_child_tasks); in __kmp_first_top_half_finish_proxy()
3536 static void __kmp_second_top_half_finish_proxy(kmp_taskdata_t *taskdata) { in __kmp_second_top_half_finish_proxy() argument
3541 KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks) - 1; in __kmp_second_top_half_finish_proxy()
3545 KMP_ATOMIC_DEC(&taskdata->td_incomplete_child_tasks); in __kmp_second_top_half_finish_proxy()
3549 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); in __kmp_bottom_half_finish_proxy() local
3552 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); in __kmp_bottom_half_finish_proxy()
3553 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == in __kmp_bottom_half_finish_proxy()
3558 while (KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) > 0) in __kmp_bottom_half_finish_proxy()
3561 __kmp_release_deps(gtid, taskdata); in __kmp_bottom_half_finish_proxy()
3562 __kmp_free_task_and_ancestors(gtid, taskdata, thread); in __kmp_bottom_half_finish_proxy()
3575 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); in __kmpc_proxy_task_completed() local
3578 gtid, taskdata)); in __kmpc_proxy_task_completed()
3580 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); in __kmpc_proxy_task_completed()
3582 __kmp_first_top_half_finish_proxy(taskdata); in __kmpc_proxy_task_completed()
3583 __kmp_second_top_half_finish_proxy(taskdata); in __kmpc_proxy_task_completed()
3588 gtid, taskdata)); in __kmpc_proxy_task_completed()
3600 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); in __kmpc_proxy_task_completed_ooo() local
3605 taskdata)); in __kmpc_proxy_task_completed_ooo()
3607 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); in __kmpc_proxy_task_completed_ooo()
3609 __kmp_first_top_half_finish_proxy(taskdata); in __kmpc_proxy_task_completed_ooo()
3613 kmp_team_t *team = taskdata->td_team; in __kmpc_proxy_task_completed_ooo()
3634 __kmp_second_top_half_finish_proxy(taskdata); in __kmpc_proxy_task_completed_ooo()
3639 taskdata)); in __kmpc_proxy_task_completed_ooo()
3650 kmp_taskdata_t *taskdata; in __kmp_task_dup_alloc() local
3668 taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, task_size); in __kmp_task_dup_alloc()
3670 taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, task_size); in __kmp_task_dup_alloc()
3672 KMP_MEMCPY(taskdata, taskdata_src, task_size); in __kmp_task_dup_alloc()
3674 task = KMP_TASKDATA_TO_TASK(taskdata); in __kmp_task_dup_alloc()
3677 taskdata->td_task_id = KMP_GEN_TASK_ID(); in __kmp_task_dup_alloc()
3680 task->shareds = &((char *)taskdata)[shareds_offset]; in __kmp_task_dup_alloc()
3684 taskdata->td_alloc_thread = thread; in __kmp_task_dup_alloc()
3685 taskdata->td_parent = parent_task; in __kmp_task_dup_alloc()
3686 taskdata->td_taskgroup = in __kmp_task_dup_alloc()
3692 if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) { in __kmp_task_dup_alloc()
3698 if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT) in __kmp_task_dup_alloc()
3699 KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks); in __kmp_task_dup_alloc()
3704 thread, taskdata, taskdata->td_parent)); in __kmp_task_dup_alloc()
3707 __ompt_task_init(taskdata, thread->th.th_info.ds.ds_gtid); in __kmp_task_dup_alloc()
3725 const kmp_taskdata_t *taskdata; member in kmp_taskloop_bounds_t
3731 : task(_task), taskdata(KMP_TASK_TO_TASKDATA(task)), in kmp_taskloop_bounds_t()
3738 : task(_task), taskdata(KMP_TASK_TO_TASKDATA(_task)), in kmp_taskloop_bounds_t()
3746 if (!taskdata->td_flags.native) { in get_lb()
3750 if (taskdata->td_size_loop_bounds == 4) { in get_lb()
3767 if (!taskdata->td_flags.native) { in get_ub()
3771 if (taskdata->td_size_loop_bounds == 4) { in get_ub()
3787 if (!taskdata->td_flags.native) { in set_lb()
3791 if (taskdata->td_size_loop_bounds == 4) { in set_lb()
3806 if (!taskdata->td_flags.native) { in set_ub()
3810 if (taskdata->td_size_loop_bounds == 4) { in set_ub()
3976 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_taskloop_task() local
3980 gtid, taskdata, num_tasks, grainsize, extras, *lb, *ub, st, in __kmp_taskloop_task()
4030 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmp_taskloop_recur() local
4035 gtid, taskdata, num_tasks, grainsize, extras, *lb, *ub, st, in __kmp_taskloop_recur()
4145 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); in __kmpc_taskloop() local
4170 gtid, taskdata, lower, upper, st, grainsize, sched, task_dup)); in __kmpc_taskloop()
4243 taskdata->td_flags.task_serial = 1; in __kmpc_taskloop()
4244 taskdata->td_flags.tiedness = TASK_TIED; // AC: serial task cannot be untied in __kmpc_taskloop()
4254 } else if (num_tasks > num_tasks_min && !taskdata->td_flags.native) { in __kmpc_taskloop()