Lines Matching refs:queue
177 static void __kmp_dump_task_queue(kmp_taskq_t *tq, kmpc_task_queue_t *queue, in __kmp_dump_task_queue() argument
183 __kmp_printf("Task Queue at %p on (%d):\n", queue, global_tid); in __kmp_dump_task_queue()
185 if (queue != NULL) { in __kmp_dump_task_queue()
186 int in_parallel = queue->tq_flags & TQF_PARALLEL_CONTEXT; in __kmp_dump_task_queue()
202 __kmp_printf(" tq_parent : %p\n", queue->tq.tq_parent); in __kmp_dump_task_queue()
203 __kmp_printf(" tq_first_child : %p\n", queue->tq_first_child); in __kmp_dump_task_queue()
204 __kmp_printf(" tq_next_child : %p\n", queue->tq_next_child); in __kmp_dump_task_queue()
205 __kmp_printf(" tq_prev_child : %p\n", queue->tq_prev_child); in __kmp_dump_task_queue()
206 __kmp_printf(" tq_ref_count : %d\n", queue->tq_ref_count); in __kmp_dump_task_queue()
222 for (i = 0; i < ((queue == tq->tq_root) ? queue->tq_nproc : 1); i++) in __kmp_dump_task_queue()
223 __kmp_printf("%p ", queue->tq_shareds[i].ai_data); in __kmp_dump_task_queue()
227 __kmp_printf(" tq_tasknum_queuing : %u\n", queue->tq_tasknum_queuing); in __kmp_dump_task_queue()
228 __kmp_printf(" tq_tasknum_serving : %u\n", queue->tq_tasknum_serving); in __kmp_dump_task_queue()
231 __kmp_printf(" tq_queue : %p\n", queue->tq_queue); in __kmp_dump_task_queue()
232 __kmp_printf(" tq_thunk_space : %p\n", queue->tq_thunk_space); in __kmp_dump_task_queue()
233 __kmp_printf(" tq_taskq_slot : %p\n", queue->tq_taskq_slot); in __kmp_dump_task_queue()
236 for (thunk = queue->tq_free_thunks; thunk != NULL; in __kmp_dump_task_queue()
241 __kmp_printf(" tq_nslots : %d\n", queue->tq_nslots); in __kmp_dump_task_queue()
242 __kmp_printf(" tq_head : %d\n", queue->tq_head); in __kmp_dump_task_queue()
243 __kmp_printf(" tq_tail : %d\n", queue->tq_tail); in __kmp_dump_task_queue()
244 __kmp_printf(" tq_nfull : %d\n", queue->tq_nfull); in __kmp_dump_task_queue()
245 __kmp_printf(" tq_hiwat : %d\n", queue->tq_hiwat); in __kmp_dump_task_queue()
247 __kmp_dump_TQF(queue->tq_flags); in __kmp_dump_task_queue()
252 for (i = 0; i < queue->tq_nproc; i++) { in __kmp_dump_task_queue()
253 __kmp_printf("%d ", queue->tq_th_thunks[i].ai_data); in __kmp_dump_task_queue()
261 qs = queue->tq_tail; in __kmp_dump_task_queue()
262 for (count = 0; count < queue->tq_nfull; ++count) { in __kmp_dump_task_queue()
264 __kmp_dump_thunk(tq, queue->tq_queue[qs].qs_thunk, global_tid); in __kmp_dump_task_queue()
265 qs = (qs + 1) % queue->tq_nslots; in __kmp_dump_task_queue()
271 if (queue->tq_taskq_slot != NULL) { in __kmp_dump_task_queue()
273 __kmp_dump_thunk(tq, CCAST(kmpc_thunk_t *, queue->tq_taskq_slot), in __kmp_dump_task_queue()
304 kmpc_task_queue_t *queue = curr_queue; in __kmp_aux_dump_task_queue_tree() local
350 for (queue = CCAST(kmpc_task_queue_t *, curr_queue->tq_first_child); in __kmp_aux_dump_task_queue_tree()
351 queue != NULL; queue = queue->tq_next_child) { in __kmp_aux_dump_task_queue_tree()
352 __kmp_aux_dump_task_queue_tree(tq, queue, level + 1, global_tid); in __kmp_aux_dump_task_queue_tree()
603 static kmpc_thunk_t *__kmp_alloc_thunk(kmpc_task_queue_t *queue, in __kmp_alloc_thunk() argument
608 __kmp_acquire_lock(&queue->tq_free_thunks_lck, global_tid); in __kmp_alloc_thunk()
614 fl = queue->tq_free_thunks; in __kmp_alloc_thunk()
618 queue->tq_free_thunks = fl->th.th_next_free; in __kmp_alloc_thunk()
622 __kmp_release_lock(&queue->tq_free_thunks_lck, global_tid); in __kmp_alloc_thunk()
627 static void __kmp_free_thunk(kmpc_task_queue_t *queue, kmpc_thunk_t *p, in __kmp_free_thunk() argument
638 __kmp_acquire_lock(&queue->tq_free_thunks_lck, global_tid); in __kmp_free_thunk()
644 p->th.th_next_free = queue->tq_free_thunks; in __kmp_free_thunk()
645 queue->tq_free_thunks = p; in __kmp_free_thunk()
652 __kmp_release_lock(&queue->tq_free_thunks_lck, global_tid); in __kmp_free_thunk()
657 kmpc_task_queue_t *queue, in __kmp_enqueue_task() argument
664 __kmp_acquire_lock(&queue->tq_queue_lck, global_tid); in __kmp_enqueue_task()
670 KMP_DEBUG_ASSERT(queue->tq_nfull < queue->tq_nslots); // check queue not full in __kmp_enqueue_task()
672 queue->tq_queue[(queue->tq_head)++].qs_thunk = thunk; in __kmp_enqueue_task()
674 if (queue->tq_head >= queue->tq_nslots) in __kmp_enqueue_task()
675 queue->tq_head = 0; in __kmp_enqueue_task()
677 (queue->tq_nfull)++; in __kmp_enqueue_task()
682 ret = (in_parallel) ? (queue->tq_nfull == queue->tq_nslots) : FALSE; in __kmp_enqueue_task()
686 __kmp_release_lock(&queue->tq_queue_lck, global_tid); in __kmp_enqueue_task()
706 kmpc_task_queue_t *queue, in __kmp_dequeue_task() argument
711 KMP_DEBUG_ASSERT(queue->tq_nfull > 0); /* check queue not empty */ in __kmp_dequeue_task()
713 if (queue->tq.tq_parent != NULL && in_parallel) { in __kmp_dequeue_task()
715 __kmp_acquire_lock(&queue->tq.tq_parent->tq_link_lck, global_tid); in __kmp_dequeue_task()
716 ct = ++(queue->tq_ref_count); in __kmp_dequeue_task()
717 __kmp_release_lock(&queue->tq.tq_parent->tq_link_lck, global_tid); in __kmp_dequeue_task()
719 ("line %d gtid %d: Q %p inc %d\n", __LINE__, global_tid, queue, ct)); in __kmp_dequeue_task()
722 pt = queue->tq_queue[(queue->tq_tail)++].qs_thunk; in __kmp_dequeue_task()
724 if (queue->tq_tail >= queue->tq_nslots) in __kmp_dequeue_task()
725 queue->tq_tail = 0; in __kmp_dequeue_task()
728 queue->tq_th_thunks[tid].ai_data++; in __kmp_dequeue_task()
735 global_tid, tid, queue->tq_th_thunks[tid].ai_data, queue)); in __kmp_dequeue_task()
738 (queue->tq_nfull)--; in __kmp_dequeue_task()
746 KMP_DEBUG_ASSERT(queue->tq_nfull >= 0); in __kmp_dequeue_task()
749 KMP_DEBUG_ASSERT(queue->tq_th_thunks[tid].ai_data <= in __kmp_dequeue_task()
781 kmpc_task_queue_t *queue) { in __kmp_find_task_in_queue() argument
786 if (!(queue->tq_flags & TQF_DEALLOCATED)) { in __kmp_find_task_in_queue()
788 __kmp_acquire_lock(&queue->tq_queue_lck, global_tid); in __kmp_find_task_in_queue()
791 if (!(queue->tq_flags & TQF_DEALLOCATED)) { in __kmp_find_task_in_queue()
796 if ((queue->tq_taskq_slot != NULL) && in __kmp_find_task_in_queue()
797 (queue->tq_nfull <= queue->tq_hiwat)) { in __kmp_find_task_in_queue()
800 pt = CCAST(kmpc_thunk_t *, queue->tq_taskq_slot); in __kmp_find_task_in_queue()
801 queue->tq_taskq_slot = NULL; in __kmp_find_task_in_queue()
802 } else if (queue->tq_nfull == 0 || in __kmp_find_task_in_queue()
803 queue->tq_th_thunks[tid].ai_data >= in __kmp_find_task_in_queue()
808 } else if (queue->tq_nfull > 1) { in __kmp_find_task_in_queue()
811 pt = __kmp_dequeue_task(global_tid, queue, TRUE); in __kmp_find_task_in_queue()
812 } else if (!(queue->tq_flags & TQF_IS_LASTPRIVATE)) { in __kmp_find_task_in_queue()
814 pt = __kmp_dequeue_task(global_tid, queue, TRUE); in __kmp_find_task_in_queue()
815 } else if (queue->tq_flags & TQF_IS_LAST_TASK) { in __kmp_find_task_in_queue()
819 pt = __kmp_dequeue_task(global_tid, queue, TRUE); in __kmp_find_task_in_queue()
826 __kmp_release_lock(&queue->tq_queue_lck, global_tid); in __kmp_find_task_in_queue()
839 kmpc_task_queue_t *queue = curr_queue; in __kmp_find_task_in_descendant_queue() local
847 queue = CCAST(kmpc_task_queue_t *, curr_queue->tq_first_child); in __kmp_find_task_in_descendant_queue()
848 if (queue == NULL) { in __kmp_find_task_in_descendant_queue()
853 while (queue != NULL) { in __kmp_find_task_in_descendant_queue()
857 ct = ++(queue->tq_ref_count); in __kmp_find_task_in_descendant_queue()
860 ("line %d gtid %d: Q %p inc %d\n", __LINE__, global_tid, queue, ct)); in __kmp_find_task_in_descendant_queue()
862 pt = __kmp_find_task_in_queue(global_tid, queue); in __kmp_find_task_in_descendant_queue()
872 ct = --(queue->tq_ref_count); in __kmp_find_task_in_descendant_queue()
874 global_tid, queue, ct)); in __kmp_find_task_in_descendant_queue()
875 KMP_DEBUG_ASSERT(queue->tq_ref_count >= 0); in __kmp_find_task_in_descendant_queue()
886 pt = __kmp_find_task_in_descendant_queue(global_tid, queue); in __kmp_find_task_in_descendant_queue()
896 ct = --(queue->tq_ref_count); in __kmp_find_task_in_descendant_queue()
898 global_tid, queue, ct)); in __kmp_find_task_in_descendant_queue()
911 next = queue->tq_next_child; in __kmp_find_task_in_descendant_queue()
913 ct = --(queue->tq_ref_count); in __kmp_find_task_in_descendant_queue()
915 ("line %d gtid %d: Q %p dec %d\n", __LINE__, global_tid, queue, ct)); in __kmp_find_task_in_descendant_queue()
918 queue = next; in __kmp_find_task_in_descendant_queue()
933 kmpc_task_queue_t *queue; in __kmp_find_task_in_ancestor_queue() local
939 queue = curr_queue->tq.tq_parent; in __kmp_find_task_in_ancestor_queue()
941 while (queue != NULL) { in __kmp_find_task_in_ancestor_queue()
942 if (queue->tq.tq_parent != NULL) { in __kmp_find_task_in_ancestor_queue()
944 __kmp_acquire_lock(&queue->tq.tq_parent->tq_link_lck, global_tid); in __kmp_find_task_in_ancestor_queue()
949 ct = ++(queue->tq_ref_count); in __kmp_find_task_in_ancestor_queue()
950 __kmp_release_lock(&queue->tq.tq_parent->tq_link_lck, global_tid); in __kmp_find_task_in_ancestor_queue()
952 global_tid, queue, ct)); in __kmp_find_task_in_ancestor_queue()
955 pt = __kmp_find_task_in_queue(global_tid, queue); in __kmp_find_task_in_ancestor_queue()
957 if (queue->tq.tq_parent != NULL) { in __kmp_find_task_in_ancestor_queue()
959 __kmp_acquire_lock(&queue->tq.tq_parent->tq_link_lck, global_tid); in __kmp_find_task_in_ancestor_queue()
964 ct = --(queue->tq_ref_count); in __kmp_find_task_in_ancestor_queue()
966 global_tid, queue, ct)); in __kmp_find_task_in_ancestor_queue()
969 __kmp_release_lock(&queue->tq.tq_parent->tq_link_lck, global_tid); in __kmp_find_task_in_ancestor_queue()
975 if (queue->tq.tq_parent != NULL) { in __kmp_find_task_in_ancestor_queue()
977 __kmp_acquire_lock(&queue->tq.tq_parent->tq_link_lck, global_tid); in __kmp_find_task_in_ancestor_queue()
982 ct = --(queue->tq_ref_count); in __kmp_find_task_in_ancestor_queue()
984 global_tid, queue, ct)); in __kmp_find_task_in_ancestor_queue()
987 queue = queue->tq.tq_parent; in __kmp_find_task_in_ancestor_queue()
989 if (queue != NULL) in __kmp_find_task_in_ancestor_queue()
990 __kmp_release_lock(&queue->tq_link_lck, global_tid); in __kmp_find_task_in_ancestor_queue()
999 static int __kmp_taskq_tasks_finished(kmpc_task_queue_t *queue) { in __kmp_taskq_tasks_finished() argument
1004 for (i = 0; i < queue->tq_nproc; i++) { in __kmp_taskq_tasks_finished()
1005 if (queue->tq_th_thunks[i].ai_data != 0) in __kmp_taskq_tasks_finished()
1012 static int __kmp_taskq_has_any_children(kmpc_task_queue_t *queue) { in __kmp_taskq_has_any_children() argument
1013 return (queue->tq_first_child != NULL); in __kmp_taskq_has_any_children()
1017 kmpc_task_queue_t *queue, in __kmp_remove_queue_from_tree() argument
1025 ("Before Deletion of TaskQ at %p on (%d):\n", queue, global_tid)); in __kmp_remove_queue_from_tree()
1026 KF_DUMP(50, __kmp_dump_task_queue(tq, queue, global_tid)); in __kmp_remove_queue_from_tree()
1029 KMP_DEBUG_ASSERT(queue->tq.tq_parent != NULL); in __kmp_remove_queue_from_tree()
1032 __kmp_acquire_lock(&queue->tq.tq_parent->tq_link_lck, global_tid); in __kmp_remove_queue_from_tree()
1038 KMP_DEBUG_ASSERT(queue->tq_first_child == NULL); in __kmp_remove_queue_from_tree()
1041 if (queue->tq_prev_child != NULL) in __kmp_remove_queue_from_tree()
1042 queue->tq_prev_child->tq_next_child = queue->tq_next_child; in __kmp_remove_queue_from_tree()
1043 if (queue->tq_next_child != NULL) in __kmp_remove_queue_from_tree()
1044 queue->tq_next_child->tq_prev_child = queue->tq_prev_child; in __kmp_remove_queue_from_tree()
1045 if (queue->tq.tq_parent->tq_first_child == queue) in __kmp_remove_queue_from_tree()
1046 queue->tq.tq_parent->tq_first_child = queue->tq_next_child; in __kmp_remove_queue_from_tree()
1048 queue->tq_prev_child = NULL; in __kmp_remove_queue_from_tree()
1049 queue->tq_next_child = NULL; in __kmp_remove_queue_from_tree()
1054 __LINE__, global_tid, queue, queue->tq_ref_count)); in __kmp_remove_queue_from_tree()
1057 while (queue->tq_ref_count > 1) { in __kmp_remove_queue_from_tree()
1058 __kmp_release_lock(&queue->tq.tq_parent->tq_link_lck, global_tid); in __kmp_remove_queue_from_tree()
1060 KMP_WAIT_YIELD((volatile kmp_uint32 *)&queue->tq_ref_count, 1, KMP_LE, in __kmp_remove_queue_from_tree()
1063 __kmp_acquire_lock(&queue->tq.tq_parent->tq_link_lck, global_tid); in __kmp_remove_queue_from_tree()
1069 __kmp_release_lock(&queue->tq.tq_parent->tq_link_lck, global_tid); in __kmp_remove_queue_from_tree()
1073 ("line %d gtid %d: Q %p freeing queue\n", __LINE__, global_tid, queue)); in __kmp_remove_queue_from_tree()
1076 KMP_DEBUG_ASSERT(queue->tq_flags & TQF_ALL_TASKS_QUEUED); in __kmp_remove_queue_from_tree()
1077 KMP_DEBUG_ASSERT(queue->tq_nfull == 0); in __kmp_remove_queue_from_tree()
1079 for (i = 0; i < queue->tq_nproc; i++) { in __kmp_remove_queue_from_tree()
1080 KMP_DEBUG_ASSERT(queue->tq_th_thunks[i].ai_data == 0); in __kmp_remove_queue_from_tree()
1084 for (thunk = queue->tq_free_thunks; thunk != NULL; in __kmp_remove_queue_from_tree()
1089 queue->tq_nslots + (queue->tq_nproc * __KMP_TASKQ_THUNKS_PER_TH)); in __kmp_remove_queue_from_tree()
1093 __kmp_free_taskq(tq, queue, TRUE, global_tid); in __kmp_remove_queue_from_tree()
1095 KF_TRACE(50, ("After Deletion of TaskQ at %p on (%d):\n", queue, global_tid)); in __kmp_remove_queue_from_tree()
1105 kmpc_task_queue_t *queue = curr_queue; in __kmp_find_and_remove_finished_child_taskq() local
1113 queue = CCAST(kmpc_task_queue_t *, curr_queue->tq_first_child); in __kmp_find_and_remove_finished_child_taskq()
1114 if (queue != NULL) { in __kmp_find_and_remove_finished_child_taskq()
1119 while (queue != NULL) { in __kmp_find_and_remove_finished_child_taskq()
1121 int ct = ++(queue->tq_ref_count); in __kmp_find_and_remove_finished_child_taskq()
1123 ("line %d gtid %d: Q %p inc %d\n", __LINE__, global_tid, queue, ct)); in __kmp_find_and_remove_finished_child_taskq()
1129 if (queue->tq_flags & TQF_IS_NOWAIT) { in __kmp_find_and_remove_finished_child_taskq()
1130 __kmp_find_and_remove_finished_child_taskq(tq, global_tid, queue); in __kmp_find_and_remove_finished_child_taskq()
1132 if ((queue->tq_flags & TQF_ALL_TASKS_QUEUED) && in __kmp_find_and_remove_finished_child_taskq()
1133 (queue->tq_nfull == 0) && __kmp_taskq_tasks_finished(queue) && in __kmp_find_and_remove_finished_child_taskq()
1134 !__kmp_taskq_has_any_children(queue)) { in __kmp_find_and_remove_finished_child_taskq()
1139 if (__kmp_test_lock(&queue->tq_queue_lck, global_tid)) { in __kmp_find_and_remove_finished_child_taskq()
1140 if (!(queue->tq_flags & TQF_DEALLOCATED)) { in __kmp_find_and_remove_finished_child_taskq()
1141 queue->tq_flags |= TQF_DEALLOCATED; in __kmp_find_and_remove_finished_child_taskq()
1142 __kmp_release_lock(&queue->tq_queue_lck, global_tid); in __kmp_find_and_remove_finished_child_taskq()
1144 __kmp_remove_queue_from_tree(tq, global_tid, queue, TRUE); in __kmp_find_and_remove_finished_child_taskq()
1150 __kmp_release_lock(&queue->tq_queue_lck, global_tid); in __kmp_find_and_remove_finished_child_taskq()
1162 next = queue->tq_next_child; in __kmp_find_and_remove_finished_child_taskq()
1164 ct = --(queue->tq_ref_count); in __kmp_find_and_remove_finished_child_taskq()
1166 ("line %d gtid %d: Q %p dec %d\n", __LINE__, global_tid, queue, ct)); in __kmp_find_and_remove_finished_child_taskq()
1169 queue = next; in __kmp_find_and_remove_finished_child_taskq()
1180 kmpc_task_queue_t *queue) { in __kmp_remove_all_child_taskq() argument
1183 queue = CCAST(kmpc_task_queue_t *, queue->tq_first_child); in __kmp_remove_all_child_taskq()
1185 while (queue != NULL) { in __kmp_remove_all_child_taskq()
1186 __kmp_remove_all_child_taskq(tq, global_tid, queue); in __kmp_remove_all_child_taskq()
1188 next_child = queue->tq_next_child; in __kmp_remove_all_child_taskq()
1189 queue->tq_flags |= TQF_DEALLOCATED; in __kmp_remove_all_child_taskq()
1190 __kmp_remove_queue_from_tree(tq, global_tid, queue, FALSE); in __kmp_remove_all_child_taskq()
1191 queue = next_child; in __kmp_remove_all_child_taskq()
1199 kmpc_task_queue_t *queue = thunk->th.th_shareds->sv_queue; in __kmp_execute_task_from_queue() local
1204 KF_TRACE(100, ("Task Queue: %p looks like this (%d):\n", queue, global_tid)); in __kmp_execute_task_from_queue()
1205 KF_DUMP(100, __kmp_dump_task_queue(tq, queue, global_tid)); in __kmp_execute_task_from_queue()
1222 kmp_int32 index = (queue == tq->tq_root) ? tid : 0; in __kmp_execute_task_from_queue()
1224 CCAST(kmpc_shared_vars_t *, queue->tq_shareds[index].ai_data); in __kmp_execute_task_from_queue()
1228 (queue->tq_flags & TQF_IS_ORDERED) ? ct_task_ordered in __kmp_execute_task_from_queue()
1230 queue->tq_loc); in __kmp_execute_task_from_queue()
1234 __kmp_push_workshare(global_tid, ct_taskq, queue->tq_loc); in __kmp_execute_task_from_queue()
1245 queue, global_tid)); in __kmp_execute_task_from_queue()
1247 KF_TRACE(50, ("End Executing Thunk %p from queue %p on (%d)\n", thunk, queue, in __kmp_execute_task_from_queue()
1253 (queue->tq_flags & TQF_IS_ORDERED) ? ct_task_ordered in __kmp_execute_task_from_queue()
1255 queue->tq_loc); in __kmp_execute_task_from_queue()
1267 __kmp_free_thunk(queue, thunk, in_parallel, global_tid); in __kmp_execute_task_from_queue()
1271 KF_DUMP(100, __kmp_dump_task_queue(tq, queue, global_tid)); in __kmp_execute_task_from_queue()
1277 KMP_DEBUG_ASSERT(queue->tq_th_thunks[tid].ai_data >= 1); in __kmp_execute_task_from_queue()
1282 global_tid, queue->tq_th_thunks[tid].ai_data - 1, queue)); in __kmp_execute_task_from_queue()
1284 queue->tq_th_thunks[tid].ai_data--; in __kmp_execute_task_from_queue()
1289 if (queue->tq.tq_parent != NULL && in_parallel) { in __kmp_execute_task_from_queue()
1291 __kmp_acquire_lock(&queue->tq.tq_parent->tq_link_lck, global_tid); in __kmp_execute_task_from_queue()
1292 ct = --(queue->tq_ref_count); in __kmp_execute_task_from_queue()
1293 __kmp_release_lock(&queue->tq.tq_parent->tq_link_lck, global_tid); in __kmp_execute_task_from_queue()
1295 ("line %d gtid %d: Q %p dec %d\n", __LINE__, global_tid, queue, ct)); in __kmp_execute_task_from_queue()
1523 kmpc_task_queue_t *queue; in __kmpc_end_taskq() local
1534 queue = (taskq_thunk == NULL) ? tq->tq_root in __kmpc_end_taskq()
1537 KE_TRACE(50, ("__kmpc_end_taskq queue=%p (%d) \n", queue, global_tid)); in __kmpc_end_taskq()
1538 is_outermost = (queue == tq->tq_root); in __kmpc_end_taskq()
1539 in_parallel = (queue->tq_flags & TQF_PARALLEL_CONTEXT); in __kmpc_end_taskq()
1562 while ((queue->tq_nfull == 0) && (queue->tq_taskq_slot == NULL) && in __kmpc_end_taskq()
1563 (!__kmp_taskq_has_any_children(queue)) && in __kmpc_end_taskq()
1564 (!(queue->tq_flags & TQF_ALL_TASKS_QUEUED))) { in __kmpc_end_taskq()
1569 while (((queue->tq_nfull != 0) || (queue->tq_taskq_slot != NULL)) && in __kmpc_end_taskq()
1570 (thunk = __kmp_find_task_in_queue(global_tid, queue)) != NULL) { in __kmpc_end_taskq()
1572 queue, global_tid)); in __kmpc_end_taskq()
1577 if ((__kmp_taskq_has_any_children(queue)) && in __kmpc_end_taskq()
1578 (thunk = __kmp_find_task_in_descendant_queue(global_tid, queue)) != in __kmpc_end_taskq()
1584 thunk, thunk->th.th_shareds->sv_queue, queue, global_tid)); in __kmpc_end_taskq()
1589 } while ((!(queue->tq_flags & TQF_ALL_TASKS_QUEUED)) || in __kmpc_end_taskq()
1590 (queue->tq_nfull != 0)); in __kmpc_end_taskq()
1592 KF_TRACE(50, ("All tasks queued and dequeued in queue: %p (%d)\n", queue, in __kmpc_end_taskq()
1598 while ((!__kmp_taskq_tasks_finished(queue)) && in __kmpc_end_taskq()
1599 (thunk = __kmp_find_task_in_descendant_queue(global_tid, queue)) != in __kmpc_end_taskq()
1604 thunk, thunk->th.th_shareds->sv_queue, queue, global_tid)); in __kmpc_end_taskq()
1611 queue, global_tid)); in __kmpc_end_taskq()
1616 if (queue->tq_flags & TQF_IS_NOWAIT) { in __kmpc_end_taskq()
1617 __kmp_acquire_lock(&queue->tq.tq_parent->tq_link_lck, global_tid); in __kmpc_end_taskq()
1618 queue->tq_ref_count--; in __kmpc_end_taskq()
1619 KMP_DEBUG_ASSERT(queue->tq_ref_count >= 0); in __kmpc_end_taskq()
1620 __kmp_release_lock(&queue->tq.tq_parent->tq_link_lck, global_tid); in __kmpc_end_taskq()
1628 __kmp_find_and_remove_finished_child_taskq(tq, global_tid, queue); in __kmpc_end_taskq()
1634 while (!__kmp_taskq_tasks_finished(queue) || in __kmpc_end_taskq()
1635 __kmp_taskq_has_any_children(queue)) { in __kmpc_end_taskq()
1636 thunk = __kmp_find_task_in_ancestor_queue(tq, global_tid, queue); in __kmpc_end_taskq()
1642 thunk, thunk->th.th_shareds->sv_queue, queue, global_tid)); in __kmpc_end_taskq()
1649 __kmp_find_and_remove_finished_child_taskq(tq, global_tid, queue); in __kmpc_end_taskq()
1652 __kmp_acquire_lock(&queue->tq_queue_lck, global_tid); in __kmpc_end_taskq()
1653 if (!(queue->tq_flags & TQF_DEALLOCATED)) { in __kmpc_end_taskq()
1654 queue->tq_flags |= TQF_DEALLOCATED; in __kmpc_end_taskq()
1656 __kmp_release_lock(&queue->tq_queue_lck, global_tid); in __kmpc_end_taskq()
1660 __kmp_remove_queue_from_tree(tq, global_tid, queue, TRUE); in __kmpc_end_taskq()
1675 while (!__kmp_taskq_tasks_finished(queue)) { in __kmpc_end_taskq()
1676 thunk = __kmp_find_task_in_descendant_queue(global_tid, queue); in __kmpc_end_taskq()
1682 thunk, thunk->th.th_shareds->sv_queue, queue, global_tid)); in __kmpc_end_taskq()
1700 __kmp_remove_all_child_taskq(tq, global_tid, queue); in __kmpc_end_taskq()
1704 global_tid, queue)); in __kmpc_end_taskq()
1705 KF_DUMP(100, __kmp_dump_task_queue(tq, queue, global_tid)); in __kmpc_end_taskq()
1709 KMP_DEBUG_ASSERT((queue->tq.tq_parent == NULL) && in __kmpc_end_taskq()
1710 (queue->tq_next_child == NULL)); in __kmpc_end_taskq()
1713 KMP_DEBUG_ASSERT(queue->tq_first_child == NULL); in __kmpc_end_taskq()
1716 KMP_DEBUG_ASSERT(queue->tq_th_thunks[i].ai_data == 0); in __kmpc_end_taskq()
1719 for (i = 0, thunk = queue->tq_free_thunks; thunk != NULL; in __kmpc_end_taskq()
1724 queue->tq_nslots + (nproc * __KMP_TASKQ_THUNKS_PER_TH)); in __kmpc_end_taskq()
1734 KF_TRACE(50, ("After Deletion of top-level TaskQ at %p on (%d):\n", queue, in __kmpc_end_taskq()
1737 queue->tq_flags |= TQF_DEALLOCATED; in __kmpc_end_taskq()
1738 __kmp_free_taskq(tq, queue, in_parallel, global_tid); in __kmpc_end_taskq()
1757 if (queue->tq_nfull > 0) { in __kmpc_end_taskq()
1758 KMP_DEBUG_ASSERT(queue->tq_nfull == 1); in __kmpc_end_taskq()
1760 thunk = __kmp_dequeue_task(global_tid, queue, in_parallel); in __kmpc_end_taskq()
1762 if (queue->tq_flags & TQF_IS_LAST_TASK) { in __kmpc_end_taskq()
1772 thunk, queue)); in __kmpc_end_taskq()
1779 queue, global_tid)); in __kmpc_end_taskq()
1780 KF_DUMP(100, __kmp_dump_task_queue(tq, queue, global_tid)); in __kmpc_end_taskq()
1784 for (thunk = queue->tq_free_thunks; thunk != NULL; in __kmpc_end_taskq()
1787 KMP_DEBUG_ASSERT(i == queue->tq_nslots + 1); in __kmpc_end_taskq()
1791 ("Serialized TaskQ at %p deleted on (%d).\n", queue, global_tid)); in __kmpc_end_taskq()
1793 queue->tq_flags |= TQF_DEALLOCATED; in __kmpc_end_taskq()
1794 __kmp_free_taskq(tq, queue, in_parallel, global_tid); in __kmpc_end_taskq()
1805 kmpc_task_queue_t *queue; in __kmpc_task() local
1815 queue = thunk->th.th_shareds->sv_queue; in __kmpc_task()
1816 in_parallel = (queue->tq_flags & TQF_PARALLEL_CONTEXT); in __kmpc_task()
1819 thunk->th_tasknum = ++queue->tq_tasknum_queuing; in __kmpc_task()
1825 if (!in_parallel && queue->tq_nfull > 0) { in __kmpc_task()
1828 KMP_DEBUG_ASSERT(queue->tq_nfull == 1); in __kmpc_task()
1830 prev_thunk = __kmp_dequeue_task(global_tid, queue, in_parallel); in __kmpc_task()
1833 prev_thunk, queue)); in __kmpc_task()
1847 ret = __kmp_enqueue_task(tq, global_tid, queue, thunk, in_parallel); in __kmpc_task()
1850 KF_DUMP(100, __kmp_dump_task_queue(tq, queue, global_tid)); in __kmpc_task()
1862 kmpc_task_queue_t *queue; in __kmpc_taskq_task() local
1870 queue = thunk->th.th_shareds->sv_queue; in __kmpc_taskq_task()
1879 KMP_DEBUG_ASSERT(queue->tq_taskq_slot == NULL); in __kmpc_taskq_task()
1896 queue->tq_taskq_slot = thunk; in __kmpc_taskq_task()
1906 kmpc_task_queue_t *queue; in __kmpc_end_taskq_task() local
1913 queue = thunk->th.th_shareds->sv_queue; in __kmpc_end_taskq_task()
1914 in_parallel = (queue->tq_flags & TQF_PARALLEL_CONTEXT); in __kmpc_end_taskq_task()
1922 KMP_TEST_THEN_OR32(RCAST(volatile kmp_uint32 *, &queue->tq_flags), in __kmpc_end_taskq_task()
1926 __kmp_acquire_lock(&queue->tq_queue_lck, global_tid); in __kmpc_end_taskq_task()
1932 queue->tq_flags |= TQF_ALL_TASKS_QUEUED; in __kmpc_end_taskq_task()
1933 __kmp_release_lock(&queue->tq_queue_lck, global_tid); in __kmpc_end_taskq_task()
1950 queue->tq_flags |= TQF_IS_LAST_TASK; in __kmpc_end_taskq_task()
1953 KMP_TEST_THEN_OR32(RCAST(volatile kmp_uint32 *, &queue->tq_flags), in __kmpc_end_taskq_task()
1957 __kmp_acquire_lock(&queue->tq_queue_lck, global_tid); in __kmpc_end_taskq_task()
1963 queue->tq_flags |= TQF_IS_LAST_TASK; in __kmpc_end_taskq_task()
1964 __kmp_release_lock(&queue->tq_queue_lck, global_tid); in __kmpc_end_taskq_task()
1990 kmpc_task_queue_t *queue; in __kmpc_task_buffer() local
2001 queue = taskq_thunk->th.th_shareds->sv_queue; in __kmpc_task_buffer()
2002 in_parallel = (queue->tq_flags & TQF_PARALLEL_CONTEXT); in __kmpc_task_buffer()
2010 new_thunk = __kmp_alloc_thunk(queue, in_parallel, global_tid); in __kmpc_task_buffer()
2012 CCAST(kmpc_shared_vars_t *, queue->tq_shareds[0].ai_data); in __kmpc_task_buffer()
2017 new_thunk->th_flags = queue->tq_flags & TQF_INTERFACE_FLAGS; in __kmpc_task_buffer()