1 /* 2 * kmp_tasking.cpp -- OpenMP 3.0 tasking support. 3 */ 4 5 //===----------------------------------------------------------------------===// 6 // 7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 8 // See https://llvm.org/LICENSE.txt for license information. 9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "kmp.h" 14 #include "kmp_i18n.h" 15 #include "kmp_itt.h" 16 #include "kmp_stats.h" 17 #include "kmp_wait_release.h" 18 #include "kmp_taskdeps.h" 19 20 #if OMPT_SUPPORT 21 #include "ompt-specific.h" 22 #endif 23 24 #include "tsan_annotations.h" 25 26 /* forward declaration */ 27 static void __kmp_enable_tasking(kmp_task_team_t *task_team, 28 kmp_info_t *this_thr); 29 static void __kmp_alloc_task_deque(kmp_info_t *thread, 30 kmp_thread_data_t *thread_data); 31 static int __kmp_realloc_task_threads_data(kmp_info_t *thread, 32 kmp_task_team_t *task_team); 33 static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask); 34 35 #ifdef BUILD_TIED_TASK_STACK 36 37 // __kmp_trace_task_stack: print the tied tasks from the task stack in order 38 // from top do bottom 39 // 40 // gtid: global thread identifier for thread containing stack 41 // thread_data: thread data for task team thread containing stack 42 // threshold: value above which the trace statement triggers 43 // location: string identifying call site of this function (for trace) 44 static void __kmp_trace_task_stack(kmp_int32 gtid, 45 kmp_thread_data_t *thread_data, 46 int threshold, char *location) { 47 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; 48 kmp_taskdata_t **stack_top = task_stack->ts_top; 49 kmp_int32 entries = task_stack->ts_entries; 50 kmp_taskdata_t *tied_task; 51 52 KA_TRACE( 53 threshold, 54 ("__kmp_trace_task_stack(start): location = %s, gtid = %d, entries = %d, " 55 "first_block = %p, stack_top = %p \n", 56 location, gtid, entries, task_stack->ts_first_block, stack_top)); 57 58 KMP_DEBUG_ASSERT(stack_top != NULL); 59 KMP_DEBUG_ASSERT(entries > 0); 60 61 while (entries != 0) { 62 KMP_DEBUG_ASSERT(stack_top != &task_stack->ts_first_block.sb_block[0]); 63 // fix up ts_top if we need to pop from previous block 64 if (entries & TASK_STACK_INDEX_MASK == 0) { 65 kmp_stack_block_t *stack_block = (kmp_stack_block_t *)(stack_top); 66 67 stack_block = stack_block->sb_prev; 68 stack_top = &stack_block->sb_block[TASK_STACK_BLOCK_SIZE]; 69 } 70 71 // finish bookkeeping 72 stack_top--; 73 entries--; 74 75 tied_task = *stack_top; 76 77 KMP_DEBUG_ASSERT(tied_task != NULL); 78 KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED); 79 80 KA_TRACE(threshold, 81 ("__kmp_trace_task_stack(%s): gtid=%d, entry=%d, " 82 "stack_top=%p, tied_task=%p\n", 83 location, gtid, entries, stack_top, tied_task)); 84 } 85 KMP_DEBUG_ASSERT(stack_top == &task_stack->ts_first_block.sb_block[0]); 86 87 KA_TRACE(threshold, 88 ("__kmp_trace_task_stack(exit): location = %s, gtid = %d\n", 89 location, gtid)); 90 } 91 92 // __kmp_init_task_stack: initialize the task stack for the first time 93 // after a thread_data structure is created. 94 // It should not be necessary to do this again (assuming the stack works). 95 // 96 // gtid: global thread identifier of calling thread 97 // thread_data: thread data for task team thread containing stack 98 static void __kmp_init_task_stack(kmp_int32 gtid, 99 kmp_thread_data_t *thread_data) { 100 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; 101 kmp_stack_block_t *first_block; 102 103 // set up the first block of the stack 104 first_block = &task_stack->ts_first_block; 105 task_stack->ts_top = (kmp_taskdata_t **)first_block; 106 memset((void *)first_block, '\0', 107 TASK_STACK_BLOCK_SIZE * sizeof(kmp_taskdata_t *)); 108 109 // initialize the stack to be empty 110 task_stack->ts_entries = TASK_STACK_EMPTY; 111 first_block->sb_next = NULL; 112 first_block->sb_prev = NULL; 113 } 114 115 // __kmp_free_task_stack: free the task stack when thread_data is destroyed. 116 // 117 // gtid: global thread identifier for calling thread 118 // thread_data: thread info for thread containing stack 119 static void __kmp_free_task_stack(kmp_int32 gtid, 120 kmp_thread_data_t *thread_data) { 121 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; 122 kmp_stack_block_t *stack_block = &task_stack->ts_first_block; 123 124 KMP_DEBUG_ASSERT(task_stack->ts_entries == TASK_STACK_EMPTY); 125 // free from the second block of the stack 126 while (stack_block != NULL) { 127 kmp_stack_block_t *next_block = (stack_block) ? stack_block->sb_next : NULL; 128 129 stack_block->sb_next = NULL; 130 stack_block->sb_prev = NULL; 131 if (stack_block != &task_stack->ts_first_block) { 132 __kmp_thread_free(thread, 133 stack_block); // free the block, if not the first 134 } 135 stack_block = next_block; 136 } 137 // initialize the stack to be empty 138 task_stack->ts_entries = 0; 139 task_stack->ts_top = NULL; 140 } 141 142 // __kmp_push_task_stack: Push the tied task onto the task stack. 143 // Grow the stack if necessary by allocating another block. 144 // 145 // gtid: global thread identifier for calling thread 146 // thread: thread info for thread containing stack 147 // tied_task: the task to push on the stack 148 static void __kmp_push_task_stack(kmp_int32 gtid, kmp_info_t *thread, 149 kmp_taskdata_t *tied_task) { 150 // GEH - need to consider what to do if tt_threads_data not allocated yet 151 kmp_thread_data_t *thread_data = 152 &thread->th.th_task_team->tt.tt_threads_data[__kmp_tid_from_gtid(gtid)]; 153 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; 154 155 if (tied_task->td_flags.team_serial || tied_task->td_flags.tasking_ser) { 156 return; // Don't push anything on stack if team or team tasks are serialized 157 } 158 159 KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED); 160 KMP_DEBUG_ASSERT(task_stack->ts_top != NULL); 161 162 KA_TRACE(20, 163 ("__kmp_push_task_stack(enter): GTID: %d; THREAD: %p; TASK: %p\n", 164 gtid, thread, tied_task)); 165 // Store entry 166 *(task_stack->ts_top) = tied_task; 167 168 // Do bookkeeping for next push 169 task_stack->ts_top++; 170 task_stack->ts_entries++; 171 172 if (task_stack->ts_entries & TASK_STACK_INDEX_MASK == 0) { 173 // Find beginning of this task block 174 kmp_stack_block_t *stack_block = 175 (kmp_stack_block_t *)(task_stack->ts_top - TASK_STACK_BLOCK_SIZE); 176 177 // Check if we already have a block 178 if (stack_block->sb_next != 179 NULL) { // reset ts_top to beginning of next block 180 task_stack->ts_top = &stack_block->sb_next->sb_block[0]; 181 } else { // Alloc new block and link it up 182 kmp_stack_block_t *new_block = (kmp_stack_block_t *)__kmp_thread_calloc( 183 thread, sizeof(kmp_stack_block_t)); 184 185 task_stack->ts_top = &new_block->sb_block[0]; 186 stack_block->sb_next = new_block; 187 new_block->sb_prev = stack_block; 188 new_block->sb_next = NULL; 189 190 KA_TRACE( 191 30, 192 ("__kmp_push_task_stack(): GTID: %d; TASK: %p; Alloc new block: %p\n", 193 gtid, tied_task, new_block)); 194 } 195 } 196 KA_TRACE(20, ("__kmp_push_task_stack(exit): GTID: %d; TASK: %p\n", gtid, 197 tied_task)); 198 } 199 200 // __kmp_pop_task_stack: Pop the tied task from the task stack. Don't return 201 // the task, just check to make sure it matches the ending task passed in. 202 // 203 // gtid: global thread identifier for the calling thread 204 // thread: thread info structure containing stack 205 // tied_task: the task popped off the stack 206 // ending_task: the task that is ending (should match popped task) 207 static void __kmp_pop_task_stack(kmp_int32 gtid, kmp_info_t *thread, 208 kmp_taskdata_t *ending_task) { 209 // GEH - need to consider what to do if tt_threads_data not allocated yet 210 kmp_thread_data_t *thread_data = 211 &thread->th.th_task_team->tt_threads_data[__kmp_tid_from_gtid(gtid)]; 212 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; 213 kmp_taskdata_t *tied_task; 214 215 if (ending_task->td_flags.team_serial || ending_task->td_flags.tasking_ser) { 216 // Don't pop anything from stack if team or team tasks are serialized 217 return; 218 } 219 220 KMP_DEBUG_ASSERT(task_stack->ts_top != NULL); 221 KMP_DEBUG_ASSERT(task_stack->ts_entries > 0); 222 223 KA_TRACE(20, ("__kmp_pop_task_stack(enter): GTID: %d; THREAD: %p\n", gtid, 224 thread)); 225 226 // fix up ts_top if we need to pop from previous block 227 if (task_stack->ts_entries & TASK_STACK_INDEX_MASK == 0) { 228 kmp_stack_block_t *stack_block = (kmp_stack_block_t *)(task_stack->ts_top); 229 230 stack_block = stack_block->sb_prev; 231 task_stack->ts_top = &stack_block->sb_block[TASK_STACK_BLOCK_SIZE]; 232 } 233 234 // finish bookkeeping 235 task_stack->ts_top--; 236 task_stack->ts_entries--; 237 238 tied_task = *(task_stack->ts_top); 239 240 KMP_DEBUG_ASSERT(tied_task != NULL); 241 KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED); 242 KMP_DEBUG_ASSERT(tied_task == ending_task); // If we built the stack correctly 243 244 KA_TRACE(20, ("__kmp_pop_task_stack(exit): GTID: %d; TASK: %p\n", gtid, 245 tied_task)); 246 return; 247 } 248 #endif /* BUILD_TIED_TASK_STACK */ 249 250 // returns 1 if new task is allowed to execute, 0 otherwise 251 // checks Task Scheduling constraint (if requested) and 252 // mutexinoutset dependencies if any 253 static bool __kmp_task_is_allowed(int gtid, const kmp_int32 is_constrained, 254 const kmp_taskdata_t *tasknew, 255 const kmp_taskdata_t *taskcurr) { 256 if (is_constrained && (tasknew->td_flags.tiedness == TASK_TIED)) { 257 // Check if the candidate obeys the Task Scheduling Constraints (TSC) 258 // only descendant of all deferred tied tasks can be scheduled, checking 259 // the last one is enough, as it in turn is the descendant of all others 260 kmp_taskdata_t *current = taskcurr->td_last_tied; 261 KMP_DEBUG_ASSERT(current != NULL); 262 // check if the task is not suspended on barrier 263 if (current->td_flags.tasktype == TASK_EXPLICIT || 264 current->td_taskwait_thread > 0) { // <= 0 on barrier 265 kmp_int32 level = current->td_level; 266 kmp_taskdata_t *parent = tasknew->td_parent; 267 while (parent != current && parent->td_level > level) { 268 // check generation up to the level of the current task 269 parent = parent->td_parent; 270 KMP_DEBUG_ASSERT(parent != NULL); 271 } 272 if (parent != current) 273 return false; 274 } 275 } 276 // Check mutexinoutset dependencies, acquire locks 277 kmp_depnode_t *node = tasknew->td_depnode; 278 if (node && (node->dn.mtx_num_locks > 0)) { 279 for (int i = 0; i < node->dn.mtx_num_locks; ++i) { 280 KMP_DEBUG_ASSERT(node->dn.mtx_locks[i] != NULL); 281 if (__kmp_test_lock(node->dn.mtx_locks[i], gtid)) 282 continue; 283 // could not get the lock, release previous locks 284 for (int j = i - 1; j >= 0; --j) 285 __kmp_release_lock(node->dn.mtx_locks[j], gtid); 286 return false; 287 } 288 // negative num_locks means all locks acquired successfully 289 node->dn.mtx_num_locks = -node->dn.mtx_num_locks; 290 } 291 return true; 292 } 293 294 // __kmp_realloc_task_deque: 295 // Re-allocates a task deque for a particular thread, copies the content from 296 // the old deque and adjusts the necessary data structures relating to the 297 // deque. This operation must be done with the deque_lock being held 298 static void __kmp_realloc_task_deque(kmp_info_t *thread, 299 kmp_thread_data_t *thread_data) { 300 kmp_int32 size = TASK_DEQUE_SIZE(thread_data->td); 301 kmp_int32 new_size = 2 * size; 302 303 KE_TRACE(10, ("__kmp_realloc_task_deque: T#%d reallocating deque[from %d to " 304 "%d] for thread_data %p\n", 305 __kmp_gtid_from_thread(thread), size, new_size, thread_data)); 306 307 kmp_taskdata_t **new_deque = 308 (kmp_taskdata_t **)__kmp_allocate(new_size * sizeof(kmp_taskdata_t *)); 309 310 int i, j; 311 for (i = thread_data->td.td_deque_head, j = 0; j < size; 312 i = (i + 1) & TASK_DEQUE_MASK(thread_data->td), j++) 313 new_deque[j] = thread_data->td.td_deque[i]; 314 315 __kmp_free(thread_data->td.td_deque); 316 317 thread_data->td.td_deque_head = 0; 318 thread_data->td.td_deque_tail = size; 319 thread_data->td.td_deque = new_deque; 320 thread_data->td.td_deque_size = new_size; 321 } 322 323 // __kmp_push_task: Add a task to the thread's deque 324 static kmp_int32 __kmp_push_task(kmp_int32 gtid, kmp_task_t *task) { 325 kmp_info_t *thread = __kmp_threads[gtid]; 326 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 327 kmp_task_team_t *task_team = thread->th.th_task_team; 328 kmp_int32 tid = __kmp_tid_from_gtid(gtid); 329 kmp_thread_data_t *thread_data; 330 331 KA_TRACE(20, 332 ("__kmp_push_task: T#%d trying to push task %p.\n", gtid, taskdata)); 333 334 if (taskdata->td_flags.tiedness == TASK_UNTIED) { 335 // untied task needs to increment counter so that the task structure is not 336 // freed prematurely 337 kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count); 338 KMP_DEBUG_USE_VAR(counter); 339 KA_TRACE( 340 20, 341 ("__kmp_push_task: T#%d untied_count (%d) incremented for task %p\n", 342 gtid, counter, taskdata)); 343 } 344 345 // The first check avoids building task_team thread data if serialized 346 if (taskdata->td_flags.task_serial) { 347 KA_TRACE(20, ("__kmp_push_task: T#%d team serialized; returning " 348 "TASK_NOT_PUSHED for task %p\n", 349 gtid, taskdata)); 350 return TASK_NOT_PUSHED; 351 } 352 353 // Now that serialized tasks have returned, we can assume that we are not in 354 // immediate exec mode 355 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 356 if (!KMP_TASKING_ENABLED(task_team)) { 357 __kmp_enable_tasking(task_team, thread); 358 } 359 KMP_DEBUG_ASSERT(TCR_4(task_team->tt.tt_found_tasks) == TRUE); 360 KMP_DEBUG_ASSERT(TCR_PTR(task_team->tt.tt_threads_data) != NULL); 361 362 // Find tasking deque specific to encountering thread 363 thread_data = &task_team->tt.tt_threads_data[tid]; 364 365 // No lock needed since only owner can allocate 366 if (thread_data->td.td_deque == NULL) { 367 __kmp_alloc_task_deque(thread, thread_data); 368 } 369 370 int locked = 0; 371 // Check if deque is full 372 if (TCR_4(thread_data->td.td_deque_ntasks) >= 373 TASK_DEQUE_SIZE(thread_data->td)) { 374 if (__kmp_enable_task_throttling && 375 __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata, 376 thread->th.th_current_task)) { 377 KA_TRACE(20, ("__kmp_push_task: T#%d deque is full; returning " 378 "TASK_NOT_PUSHED for task %p\n", 379 gtid, taskdata)); 380 return TASK_NOT_PUSHED; 381 } else { 382 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 383 locked = 1; 384 // expand deque to push the task which is not allowed to execute 385 __kmp_realloc_task_deque(thread, thread_data); 386 } 387 } 388 // Lock the deque for the task push operation 389 if (!locked) { 390 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 391 // Need to recheck as we can get a proxy task from thread outside of OpenMP 392 if (TCR_4(thread_data->td.td_deque_ntasks) >= 393 TASK_DEQUE_SIZE(thread_data->td)) { 394 if (__kmp_enable_task_throttling && 395 __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata, 396 thread->th.th_current_task)) { 397 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 398 KA_TRACE(20, ("__kmp_push_task: T#%d deque is full on 2nd check; " 399 "returning TASK_NOT_PUSHED for task %p\n", 400 gtid, taskdata)); 401 return TASK_NOT_PUSHED; 402 } else { 403 // expand deque to push the task which is not allowed to execute 404 __kmp_realloc_task_deque(thread, thread_data); 405 } 406 } 407 } 408 // Must have room since no thread can add tasks but calling thread 409 KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) < 410 TASK_DEQUE_SIZE(thread_data->td)); 411 412 thread_data->td.td_deque[thread_data->td.td_deque_tail] = 413 taskdata; // Push taskdata 414 // Wrap index. 415 thread_data->td.td_deque_tail = 416 (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td); 417 TCW_4(thread_data->td.td_deque_ntasks, 418 TCR_4(thread_data->td.td_deque_ntasks) + 1); // Adjust task count 419 420 KA_TRACE(20, ("__kmp_push_task: T#%d returning TASK_SUCCESSFULLY_PUSHED: " 421 "task=%p ntasks=%d head=%u tail=%u\n", 422 gtid, taskdata, thread_data->td.td_deque_ntasks, 423 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 424 425 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 426 427 return TASK_SUCCESSFULLY_PUSHED; 428 } 429 430 // __kmp_pop_current_task_from_thread: set up current task from called thread 431 // when team ends 432 // 433 // this_thr: thread structure to set current_task in. 434 void __kmp_pop_current_task_from_thread(kmp_info_t *this_thr) { 435 KF_TRACE(10, ("__kmp_pop_current_task_from_thread(enter): T#%d " 436 "this_thread=%p, curtask=%p, " 437 "curtask_parent=%p\n", 438 0, this_thr, this_thr->th.th_current_task, 439 this_thr->th.th_current_task->td_parent)); 440 441 this_thr->th.th_current_task = this_thr->th.th_current_task->td_parent; 442 443 KF_TRACE(10, ("__kmp_pop_current_task_from_thread(exit): T#%d " 444 "this_thread=%p, curtask=%p, " 445 "curtask_parent=%p\n", 446 0, this_thr, this_thr->th.th_current_task, 447 this_thr->th.th_current_task->td_parent)); 448 } 449 450 // __kmp_push_current_task_to_thread: set up current task in called thread for a 451 // new team 452 // 453 // this_thr: thread structure to set up 454 // team: team for implicit task data 455 // tid: thread within team to set up 456 void __kmp_push_current_task_to_thread(kmp_info_t *this_thr, kmp_team_t *team, 457 int tid) { 458 // current task of the thread is a parent of the new just created implicit 459 // tasks of new team 460 KF_TRACE(10, ("__kmp_push_current_task_to_thread(enter): T#%d this_thread=%p " 461 "curtask=%p " 462 "parent_task=%p\n", 463 tid, this_thr, this_thr->th.th_current_task, 464 team->t.t_implicit_task_taskdata[tid].td_parent)); 465 466 KMP_DEBUG_ASSERT(this_thr != NULL); 467 468 if (tid == 0) { 469 if (this_thr->th.th_current_task != &team->t.t_implicit_task_taskdata[0]) { 470 team->t.t_implicit_task_taskdata[0].td_parent = 471 this_thr->th.th_current_task; 472 this_thr->th.th_current_task = &team->t.t_implicit_task_taskdata[0]; 473 } 474 } else { 475 team->t.t_implicit_task_taskdata[tid].td_parent = 476 team->t.t_implicit_task_taskdata[0].td_parent; 477 this_thr->th.th_current_task = &team->t.t_implicit_task_taskdata[tid]; 478 } 479 480 KF_TRACE(10, ("__kmp_push_current_task_to_thread(exit): T#%d this_thread=%p " 481 "curtask=%p " 482 "parent_task=%p\n", 483 tid, this_thr, this_thr->th.th_current_task, 484 team->t.t_implicit_task_taskdata[tid].td_parent)); 485 } 486 487 // __kmp_task_start: bookkeeping for a task starting execution 488 // 489 // GTID: global thread id of calling thread 490 // task: task starting execution 491 // current_task: task suspending 492 static void __kmp_task_start(kmp_int32 gtid, kmp_task_t *task, 493 kmp_taskdata_t *current_task) { 494 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 495 kmp_info_t *thread = __kmp_threads[gtid]; 496 497 KA_TRACE(10, 498 ("__kmp_task_start(enter): T#%d starting task %p: current_task=%p\n", 499 gtid, taskdata, current_task)); 500 501 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); 502 503 // mark currently executing task as suspended 504 // TODO: GEH - make sure root team implicit task is initialized properly. 505 // KMP_DEBUG_ASSERT( current_task -> td_flags.executing == 1 ); 506 current_task->td_flags.executing = 0; 507 508 // Add task to stack if tied 509 #ifdef BUILD_TIED_TASK_STACK 510 if (taskdata->td_flags.tiedness == TASK_TIED) { 511 __kmp_push_task_stack(gtid, thread, taskdata); 512 } 513 #endif /* BUILD_TIED_TASK_STACK */ 514 515 // mark starting task as executing and as current task 516 thread->th.th_current_task = taskdata; 517 518 KMP_DEBUG_ASSERT(taskdata->td_flags.started == 0 || 519 taskdata->td_flags.tiedness == TASK_UNTIED); 520 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0 || 521 taskdata->td_flags.tiedness == TASK_UNTIED); 522 taskdata->td_flags.started = 1; 523 taskdata->td_flags.executing = 1; 524 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); 525 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); 526 527 // GEH TODO: shouldn't we pass some sort of location identifier here? 528 // APT: yes, we will pass location here. 529 // need to store current thread state (in a thread or taskdata structure) 530 // before setting work_state, otherwise wrong state is set after end of task 531 532 KA_TRACE(10, ("__kmp_task_start(exit): T#%d task=%p\n", gtid, taskdata)); 533 534 return; 535 } 536 537 #if OMPT_SUPPORT 538 //------------------------------------------------------------------------------ 539 // __ompt_task_init: 540 // Initialize OMPT fields maintained by a task. This will only be called after 541 // ompt_start_tool, so we already know whether ompt is enabled or not. 542 543 static inline void __ompt_task_init(kmp_taskdata_t *task, int tid) { 544 // The calls to __ompt_task_init already have the ompt_enabled condition. 545 task->ompt_task_info.task_data.value = 0; 546 task->ompt_task_info.frame.exit_frame = ompt_data_none; 547 task->ompt_task_info.frame.enter_frame = ompt_data_none; 548 task->ompt_task_info.frame.exit_frame_flags = ompt_frame_runtime | ompt_frame_framepointer; 549 task->ompt_task_info.frame.enter_frame_flags = ompt_frame_runtime | ompt_frame_framepointer; 550 task->ompt_task_info.ndeps = 0; 551 task->ompt_task_info.deps = NULL; 552 } 553 554 // __ompt_task_start: 555 // Build and trigger task-begin event 556 static inline void __ompt_task_start(kmp_task_t *task, 557 kmp_taskdata_t *current_task, 558 kmp_int32 gtid) { 559 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 560 ompt_task_status_t status = ompt_task_switch; 561 if (__kmp_threads[gtid]->th.ompt_thread_info.ompt_task_yielded) { 562 status = ompt_task_yield; 563 __kmp_threads[gtid]->th.ompt_thread_info.ompt_task_yielded = 0; 564 } 565 /* let OMPT know that we're about to run this task */ 566 if (ompt_enabled.ompt_callback_task_schedule) { 567 ompt_callbacks.ompt_callback(ompt_callback_task_schedule)( 568 &(current_task->ompt_task_info.task_data), status, 569 &(taskdata->ompt_task_info.task_data)); 570 } 571 taskdata->ompt_task_info.scheduling_parent = current_task; 572 } 573 574 // __ompt_task_finish: 575 // Build and trigger final task-schedule event 576 static inline void 577 __ompt_task_finish(kmp_task_t *task, kmp_taskdata_t *resumed_task, 578 ompt_task_status_t status = ompt_task_complete) { 579 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 580 if (__kmp_omp_cancellation && taskdata->td_taskgroup && 581 taskdata->td_taskgroup->cancel_request == cancel_taskgroup) { 582 status = ompt_task_cancel; 583 } 584 585 /* let OMPT know that we're returning to the callee task */ 586 if (ompt_enabled.ompt_callback_task_schedule) { 587 ompt_callbacks.ompt_callback(ompt_callback_task_schedule)( 588 &(taskdata->ompt_task_info.task_data), status, 589 &((resumed_task ? resumed_task 590 : (taskdata->ompt_task_info.scheduling_parent 591 ? taskdata->ompt_task_info.scheduling_parent 592 : taskdata->td_parent)) 593 ->ompt_task_info.task_data)); 594 } 595 } 596 #endif 597 598 template <bool ompt> 599 static void __kmpc_omp_task_begin_if0_template(ident_t *loc_ref, kmp_int32 gtid, 600 kmp_task_t *task, 601 void *frame_address, 602 void *return_address) { 603 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 604 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task; 605 606 KA_TRACE(10, ("__kmpc_omp_task_begin_if0(enter): T#%d loc=%p task=%p " 607 "current_task=%p\n", 608 gtid, loc_ref, taskdata, current_task)); 609 610 if (taskdata->td_flags.tiedness == TASK_UNTIED) { 611 // untied task needs to increment counter so that the task structure is not 612 // freed prematurely 613 kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count); 614 KMP_DEBUG_USE_VAR(counter); 615 KA_TRACE(20, ("__kmpc_omp_task_begin_if0: T#%d untied_count (%d) " 616 "incremented for task %p\n", 617 gtid, counter, taskdata)); 618 } 619 620 taskdata->td_flags.task_serial = 621 1; // Execute this task immediately, not deferred. 622 __kmp_task_start(gtid, task, current_task); 623 624 #if OMPT_SUPPORT 625 if (ompt) { 626 if (current_task->ompt_task_info.frame.enter_frame.ptr == NULL) { 627 current_task->ompt_task_info.frame.enter_frame.ptr = 628 taskdata->ompt_task_info.frame.exit_frame.ptr = frame_address; 629 current_task->ompt_task_info.frame.enter_frame_flags = 630 taskdata->ompt_task_info.frame.exit_frame_flags = ompt_frame_application | ompt_frame_framepointer; 631 } 632 if (ompt_enabled.ompt_callback_task_create) { 633 ompt_task_info_t *parent_info = &(current_task->ompt_task_info); 634 ompt_callbacks.ompt_callback(ompt_callback_task_create)( 635 &(parent_info->task_data), &(parent_info->frame), 636 &(taskdata->ompt_task_info.task_data), 637 ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(taskdata), 0, 638 return_address); 639 } 640 __ompt_task_start(task, current_task, gtid); 641 } 642 #endif // OMPT_SUPPORT 643 644 KA_TRACE(10, ("__kmpc_omp_task_begin_if0(exit): T#%d loc=%p task=%p,\n", gtid, 645 loc_ref, taskdata)); 646 } 647 648 #if OMPT_SUPPORT 649 OMPT_NOINLINE 650 static void __kmpc_omp_task_begin_if0_ompt(ident_t *loc_ref, kmp_int32 gtid, 651 kmp_task_t *task, 652 void *frame_address, 653 void *return_address) { 654 __kmpc_omp_task_begin_if0_template<true>(loc_ref, gtid, task, frame_address, 655 return_address); 656 } 657 #endif // OMPT_SUPPORT 658 659 // __kmpc_omp_task_begin_if0: report that a given serialized task has started 660 // execution 661 // 662 // loc_ref: source location information; points to beginning of task block. 663 // gtid: global thread number. 664 // task: task thunk for the started task. 665 void __kmpc_omp_task_begin_if0(ident_t *loc_ref, kmp_int32 gtid, 666 kmp_task_t *task) { 667 #if OMPT_SUPPORT 668 if (UNLIKELY(ompt_enabled.enabled)) { 669 OMPT_STORE_RETURN_ADDRESS(gtid); 670 __kmpc_omp_task_begin_if0_ompt(loc_ref, gtid, task, 671 OMPT_GET_FRAME_ADDRESS(1), 672 OMPT_LOAD_RETURN_ADDRESS(gtid)); 673 return; 674 } 675 #endif 676 __kmpc_omp_task_begin_if0_template<false>(loc_ref, gtid, task, NULL, NULL); 677 } 678 679 #ifdef TASK_UNUSED 680 // __kmpc_omp_task_begin: report that a given task has started execution 681 // NEVER GENERATED BY COMPILER, DEPRECATED!!! 682 void __kmpc_omp_task_begin(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task) { 683 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task; 684 685 KA_TRACE( 686 10, 687 ("__kmpc_omp_task_begin(enter): T#%d loc=%p task=%p current_task=%p\n", 688 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task), current_task)); 689 690 __kmp_task_start(gtid, task, current_task); 691 692 KA_TRACE(10, ("__kmpc_omp_task_begin(exit): T#%d loc=%p task=%p,\n", gtid, 693 loc_ref, KMP_TASK_TO_TASKDATA(task))); 694 return; 695 } 696 #endif // TASK_UNUSED 697 698 // __kmp_free_task: free the current task space and the space for shareds 699 // 700 // gtid: Global thread ID of calling thread 701 // taskdata: task to free 702 // thread: thread data structure of caller 703 static void __kmp_free_task(kmp_int32 gtid, kmp_taskdata_t *taskdata, 704 kmp_info_t *thread) { 705 KA_TRACE(30, ("__kmp_free_task: T#%d freeing data from task %p\n", gtid, 706 taskdata)); 707 708 // Check to make sure all flags and counters have the correct values 709 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); 710 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0); 711 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 1); 712 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); 713 KMP_DEBUG_ASSERT(taskdata->td_allocated_child_tasks == 0 || 714 taskdata->td_flags.task_serial == 1); 715 KMP_DEBUG_ASSERT(taskdata->td_incomplete_child_tasks == 0); 716 717 taskdata->td_flags.freed = 1; 718 ANNOTATE_HAPPENS_BEFORE(taskdata); 719 // deallocate the taskdata and shared variable blocks associated with this task 720 #if USE_FAST_MEMORY 721 __kmp_fast_free(thread, taskdata); 722 #else /* ! USE_FAST_MEMORY */ 723 __kmp_thread_free(thread, taskdata); 724 #endif 725 726 KA_TRACE(20, ("__kmp_free_task: T#%d freed task %p\n", gtid, taskdata)); 727 } 728 729 // __kmp_free_task_and_ancestors: free the current task and ancestors without 730 // children 731 // 732 // gtid: Global thread ID of calling thread 733 // taskdata: task to free 734 // thread: thread data structure of caller 735 static void __kmp_free_task_and_ancestors(kmp_int32 gtid, 736 kmp_taskdata_t *taskdata, 737 kmp_info_t *thread) { 738 // Proxy tasks must always be allowed to free their parents 739 // because they can be run in background even in serial mode. 740 kmp_int32 team_serial = 741 (taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser) && 742 !taskdata->td_flags.proxy; 743 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); 744 745 kmp_int32 children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks) - 1; 746 KMP_DEBUG_ASSERT(children >= 0); 747 748 // Now, go up the ancestor tree to see if any ancestors can now be freed. 749 while (children == 0) { 750 kmp_taskdata_t *parent_taskdata = taskdata->td_parent; 751 752 KA_TRACE(20, ("__kmp_free_task_and_ancestors(enter): T#%d task %p complete " 753 "and freeing itself\n", 754 gtid, taskdata)); 755 756 // --- Deallocate my ancestor task --- 757 __kmp_free_task(gtid, taskdata, thread); 758 759 taskdata = parent_taskdata; 760 761 if (team_serial) 762 return; 763 // Stop checking ancestors at implicit task instead of walking up ancestor 764 // tree to avoid premature deallocation of ancestors. 765 if (taskdata->td_flags.tasktype == TASK_IMPLICIT) { 766 if (taskdata->td_dephash) { // do we need to cleanup dephash? 767 int children = KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks); 768 kmp_tasking_flags_t flags_old = taskdata->td_flags; 769 if (children == 0 && flags_old.complete == 1) { 770 kmp_tasking_flags_t flags_new = flags_old; 771 flags_new.complete = 0; 772 if (KMP_COMPARE_AND_STORE_ACQ32( 773 RCAST(kmp_int32 *, &taskdata->td_flags), 774 *RCAST(kmp_int32 *, &flags_old), 775 *RCAST(kmp_int32 *, &flags_new))) { 776 KA_TRACE(100, ("__kmp_free_task_and_ancestors: T#%d cleans " 777 "dephash of implicit task %p\n", 778 gtid, taskdata)); 779 // cleanup dephash of finished implicit task 780 __kmp_dephash_free_entries(thread, taskdata->td_dephash); 781 } 782 } 783 } 784 return; 785 } 786 // Predecrement simulated by "- 1" calculation 787 children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks) - 1; 788 KMP_DEBUG_ASSERT(children >= 0); 789 } 790 791 KA_TRACE( 792 20, ("__kmp_free_task_and_ancestors(exit): T#%d task %p has %d children; " 793 "not freeing it yet\n", 794 gtid, taskdata, children)); 795 } 796 797 // __kmp_task_finish: bookkeeping to do when a task finishes execution 798 // 799 // gtid: global thread ID for calling thread 800 // task: task to be finished 801 // resumed_task: task to be resumed. (may be NULL if task is serialized) 802 template <bool ompt> 803 static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task, 804 kmp_taskdata_t *resumed_task) { 805 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 806 kmp_info_t *thread = __kmp_threads[gtid]; 807 kmp_task_team_t *task_team = 808 thread->th.th_task_team; // might be NULL for serial teams... 809 kmp_int32 children = 0; 810 811 KA_TRACE(10, ("__kmp_task_finish(enter): T#%d finishing task %p and resuming " 812 "task %p\n", 813 gtid, taskdata, resumed_task)); 814 815 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); 816 817 // Pop task from stack if tied 818 #ifdef BUILD_TIED_TASK_STACK 819 if (taskdata->td_flags.tiedness == TASK_TIED) { 820 __kmp_pop_task_stack(gtid, thread, taskdata); 821 } 822 #endif /* BUILD_TIED_TASK_STACK */ 823 824 if (taskdata->td_flags.tiedness == TASK_UNTIED) { 825 // untied task needs to check the counter so that the task structure is not 826 // freed prematurely 827 kmp_int32 counter = KMP_ATOMIC_DEC(&taskdata->td_untied_count) - 1; 828 KA_TRACE( 829 20, 830 ("__kmp_task_finish: T#%d untied_count (%d) decremented for task %p\n", 831 gtid, counter, taskdata)); 832 if (counter > 0) { 833 // untied task is not done, to be continued possibly by other thread, do 834 // not free it now 835 if (resumed_task == NULL) { 836 KMP_DEBUG_ASSERT(taskdata->td_flags.task_serial); 837 resumed_task = taskdata->td_parent; // In a serialized task, the resumed 838 // task is the parent 839 } 840 thread->th.th_current_task = resumed_task; // restore current_task 841 resumed_task->td_flags.executing = 1; // resume previous task 842 KA_TRACE(10, ("__kmp_task_finish(exit): T#%d partially done task %p, " 843 "resuming task %p\n", 844 gtid, taskdata, resumed_task)); 845 return; 846 } 847 } 848 #if OMPT_SUPPORT 849 if (ompt) 850 __ompt_task_finish(task, resumed_task); 851 #endif 852 853 // Check mutexinoutset dependencies, release locks 854 kmp_depnode_t *node = taskdata->td_depnode; 855 if (node && (node->dn.mtx_num_locks < 0)) { 856 // negative num_locks means all locks were acquired 857 node->dn.mtx_num_locks = -node->dn.mtx_num_locks; 858 for (int i = node->dn.mtx_num_locks - 1; i >= 0; --i) { 859 KMP_DEBUG_ASSERT(node->dn.mtx_locks[i] != NULL); 860 __kmp_release_lock(node->dn.mtx_locks[i], gtid); 861 } 862 } 863 864 // bookkeeping for resuming task: 865 // GEH - note tasking_ser => task_serial 866 KMP_DEBUG_ASSERT( 867 (taskdata->td_flags.tasking_ser || taskdata->td_flags.task_serial) == 868 taskdata->td_flags.task_serial); 869 if (taskdata->td_flags.task_serial) { 870 if (resumed_task == NULL) { 871 resumed_task = taskdata->td_parent; // In a serialized task, the resumed 872 // task is the parent 873 } 874 } else { 875 KMP_DEBUG_ASSERT(resumed_task != 876 NULL); // verify that resumed task is passed as argument 877 } 878 879 /* If the tasks' destructor thunk flag has been set, we need to invoke the 880 destructor thunk that has been generated by the compiler. The code is 881 placed here, since at this point other tasks might have been released 882 hence overlapping the destructor invocations with some other work in the 883 released tasks. The OpenMP spec is not specific on when the destructors 884 are invoked, so we should be free to choose. */ 885 if (taskdata->td_flags.destructors_thunk) { 886 kmp_routine_entry_t destr_thunk = task->data1.destructors; 887 KMP_ASSERT(destr_thunk); 888 destr_thunk(gtid, task); 889 } 890 891 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); 892 KMP_DEBUG_ASSERT(taskdata->td_flags.started == 1); 893 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); 894 895 bool detach = false; 896 if (taskdata->td_flags.detachable == TASK_DETACHABLE) { 897 if (taskdata->td_allow_completion_event.type == 898 KMP_EVENT_ALLOW_COMPLETION) { 899 // event hasn't been fulfilled yet. Try to detach task. 900 __kmp_acquire_tas_lock(&taskdata->td_allow_completion_event.lock, gtid); 901 if (taskdata->td_allow_completion_event.type == 902 KMP_EVENT_ALLOW_COMPLETION) { 903 // task finished execution 904 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1); 905 taskdata->td_flags.executing = 0; // suspend the finishing task 906 // no access to taskdata after this point! 907 // __kmp_fulfill_event might free taskdata at any time from now 908 taskdata->td_flags.proxy = TASK_PROXY; // proxify! 909 detach = true; 910 } 911 __kmp_release_tas_lock(&taskdata->td_allow_completion_event.lock, gtid); 912 } 913 } 914 915 if (!detach) { 916 taskdata->td_flags.complete = 1; // mark the task as completed 917 918 // Only need to keep track of count if team parallel and tasking not 919 // serialized 920 if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) { 921 // Predecrement simulated by "- 1" calculation 922 children = 923 KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks) - 1; 924 KMP_DEBUG_ASSERT(children >= 0); 925 if (taskdata->td_taskgroup) 926 KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count); 927 __kmp_release_deps(gtid, taskdata); 928 } else if (task_team && task_team->tt.tt_found_proxy_tasks) { 929 // if we found proxy tasks there could exist a dependency chain 930 // with the proxy task as origin 931 __kmp_release_deps(gtid, taskdata); 932 } 933 // td_flags.executing must be marked as 0 after __kmp_release_deps has been 934 // called. Othertwise, if a task is executed immediately from the 935 // release_deps code, the flag will be reset to 1 again by this same 936 // function 937 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1); 938 taskdata->td_flags.executing = 0; // suspend the finishing task 939 } 940 941 942 KA_TRACE( 943 20, ("__kmp_task_finish: T#%d finished task %p, %d incomplete children\n", 944 gtid, taskdata, children)); 945 946 // Free this task and then ancestor tasks if they have no children. 947 // Restore th_current_task first as suggested by John: 948 // johnmc: if an asynchronous inquiry peers into the runtime system 949 // it doesn't see the freed task as the current task. 950 thread->th.th_current_task = resumed_task; 951 if (!detach) 952 __kmp_free_task_and_ancestors(gtid, taskdata, thread); 953 954 // TODO: GEH - make sure root team implicit task is initialized properly. 955 // KMP_DEBUG_ASSERT( resumed_task->td_flags.executing == 0 ); 956 resumed_task->td_flags.executing = 1; // resume previous task 957 958 KA_TRACE( 959 10, ("__kmp_task_finish(exit): T#%d finished task %p, resuming task %p\n", 960 gtid, taskdata, resumed_task)); 961 962 return; 963 } 964 965 template <bool ompt> 966 static void __kmpc_omp_task_complete_if0_template(ident_t *loc_ref, 967 kmp_int32 gtid, 968 kmp_task_t *task) { 969 KA_TRACE(10, ("__kmpc_omp_task_complete_if0(enter): T#%d loc=%p task=%p\n", 970 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task))); 971 // this routine will provide task to resume 972 __kmp_task_finish<ompt>(gtid, task, NULL); 973 974 KA_TRACE(10, ("__kmpc_omp_task_complete_if0(exit): T#%d loc=%p task=%p\n", 975 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task))); 976 977 #if OMPT_SUPPORT 978 if (ompt) { 979 ompt_frame_t *ompt_frame; 980 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 981 ompt_frame->enter_frame = ompt_data_none; 982 ompt_frame->enter_frame_flags = ompt_frame_runtime | ompt_frame_framepointer; 983 } 984 #endif 985 986 return; 987 } 988 989 #if OMPT_SUPPORT 990 OMPT_NOINLINE 991 void __kmpc_omp_task_complete_if0_ompt(ident_t *loc_ref, kmp_int32 gtid, 992 kmp_task_t *task) { 993 __kmpc_omp_task_complete_if0_template<true>(loc_ref, gtid, task); 994 } 995 #endif // OMPT_SUPPORT 996 997 // __kmpc_omp_task_complete_if0: report that a task has completed execution 998 // 999 // loc_ref: source location information; points to end of task block. 1000 // gtid: global thread number. 1001 // task: task thunk for the completed task. 1002 void __kmpc_omp_task_complete_if0(ident_t *loc_ref, kmp_int32 gtid, 1003 kmp_task_t *task) { 1004 #if OMPT_SUPPORT 1005 if (UNLIKELY(ompt_enabled.enabled)) { 1006 __kmpc_omp_task_complete_if0_ompt(loc_ref, gtid, task); 1007 return; 1008 } 1009 #endif 1010 __kmpc_omp_task_complete_if0_template<false>(loc_ref, gtid, task); 1011 } 1012 1013 #ifdef TASK_UNUSED 1014 // __kmpc_omp_task_complete: report that a task has completed execution 1015 // NEVER GENERATED BY COMPILER, DEPRECATED!!! 1016 void __kmpc_omp_task_complete(ident_t *loc_ref, kmp_int32 gtid, 1017 kmp_task_t *task) { 1018 KA_TRACE(10, ("__kmpc_omp_task_complete(enter): T#%d loc=%p task=%p\n", gtid, 1019 loc_ref, KMP_TASK_TO_TASKDATA(task))); 1020 1021 __kmp_task_finish<false>(gtid, task, 1022 NULL); // Not sure how to find task to resume 1023 1024 KA_TRACE(10, ("__kmpc_omp_task_complete(exit): T#%d loc=%p task=%p\n", gtid, 1025 loc_ref, KMP_TASK_TO_TASKDATA(task))); 1026 return; 1027 } 1028 #endif // TASK_UNUSED 1029 1030 // __kmp_init_implicit_task: Initialize the appropriate fields in the implicit 1031 // task for a given thread 1032 // 1033 // loc_ref: reference to source location of parallel region 1034 // this_thr: thread data structure corresponding to implicit task 1035 // team: team for this_thr 1036 // tid: thread id of given thread within team 1037 // set_curr_task: TRUE if need to push current task to thread 1038 // NOTE: Routine does not set up the implicit task ICVS. This is assumed to 1039 // have already been done elsewhere. 1040 // TODO: Get better loc_ref. Value passed in may be NULL 1041 void __kmp_init_implicit_task(ident_t *loc_ref, kmp_info_t *this_thr, 1042 kmp_team_t *team, int tid, int set_curr_task) { 1043 kmp_taskdata_t *task = &team->t.t_implicit_task_taskdata[tid]; 1044 1045 KF_TRACE( 1046 10, 1047 ("__kmp_init_implicit_task(enter): T#:%d team=%p task=%p, reinit=%s\n", 1048 tid, team, task, set_curr_task ? "TRUE" : "FALSE")); 1049 1050 task->td_task_id = KMP_GEN_TASK_ID(); 1051 task->td_team = team; 1052 // task->td_parent = NULL; // fix for CQ230101 (broken parent task info 1053 // in debugger) 1054 task->td_ident = loc_ref; 1055 task->td_taskwait_ident = NULL; 1056 task->td_taskwait_counter = 0; 1057 task->td_taskwait_thread = 0; 1058 1059 task->td_flags.tiedness = TASK_TIED; 1060 task->td_flags.tasktype = TASK_IMPLICIT; 1061 task->td_flags.proxy = TASK_FULL; 1062 1063 // All implicit tasks are executed immediately, not deferred 1064 task->td_flags.task_serial = 1; 1065 task->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec); 1066 task->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0; 1067 1068 task->td_flags.started = 1; 1069 task->td_flags.executing = 1; 1070 task->td_flags.complete = 0; 1071 task->td_flags.freed = 0; 1072 1073 task->td_depnode = NULL; 1074 task->td_last_tied = task; 1075 task->td_allow_completion_event.type = KMP_EVENT_UNINITIALIZED; 1076 1077 if (set_curr_task) { // only do this init first time thread is created 1078 KMP_ATOMIC_ST_REL(&task->td_incomplete_child_tasks, 0); 1079 // Not used: don't need to deallocate implicit task 1080 KMP_ATOMIC_ST_REL(&task->td_allocated_child_tasks, 0); 1081 task->td_taskgroup = NULL; // An implicit task does not have taskgroup 1082 task->td_dephash = NULL; 1083 __kmp_push_current_task_to_thread(this_thr, team, tid); 1084 } else { 1085 KMP_DEBUG_ASSERT(task->td_incomplete_child_tasks == 0); 1086 KMP_DEBUG_ASSERT(task->td_allocated_child_tasks == 0); 1087 } 1088 1089 #if OMPT_SUPPORT 1090 if (UNLIKELY(ompt_enabled.enabled)) 1091 __ompt_task_init(task, tid); 1092 #endif 1093 1094 KF_TRACE(10, ("__kmp_init_implicit_task(exit): T#:%d team=%p task=%p\n", tid, 1095 team, task)); 1096 } 1097 1098 // __kmp_finish_implicit_task: Release resources associated to implicit tasks 1099 // at the end of parallel regions. Some resources are kept for reuse in the next 1100 // parallel region. 1101 // 1102 // thread: thread data structure corresponding to implicit task 1103 void __kmp_finish_implicit_task(kmp_info_t *thread) { 1104 kmp_taskdata_t *task = thread->th.th_current_task; 1105 if (task->td_dephash) { 1106 int children; 1107 task->td_flags.complete = 1; 1108 children = KMP_ATOMIC_LD_ACQ(&task->td_incomplete_child_tasks); 1109 kmp_tasking_flags_t flags_old = task->td_flags; 1110 if (children == 0 && flags_old.complete == 1) { 1111 kmp_tasking_flags_t flags_new = flags_old; 1112 flags_new.complete = 0; 1113 if (KMP_COMPARE_AND_STORE_ACQ32(RCAST(kmp_int32 *, &task->td_flags), 1114 *RCAST(kmp_int32 *, &flags_old), 1115 *RCAST(kmp_int32 *, &flags_new))) { 1116 KA_TRACE(100, ("__kmp_finish_implicit_task: T#%d cleans " 1117 "dephash of implicit task %p\n", 1118 thread->th.th_info.ds.ds_gtid, task)); 1119 __kmp_dephash_free_entries(thread, task->td_dephash); 1120 } 1121 } 1122 } 1123 } 1124 1125 // __kmp_free_implicit_task: Release resources associated to implicit tasks 1126 // when these are destroyed regions 1127 // 1128 // thread: thread data structure corresponding to implicit task 1129 void __kmp_free_implicit_task(kmp_info_t *thread) { 1130 kmp_taskdata_t *task = thread->th.th_current_task; 1131 if (task && task->td_dephash) { 1132 __kmp_dephash_free(thread, task->td_dephash); 1133 task->td_dephash = NULL; 1134 } 1135 } 1136 1137 // Round up a size to a power of two specified by val: Used to insert padding 1138 // between structures co-allocated using a single malloc() call 1139 static size_t __kmp_round_up_to_val(size_t size, size_t val) { 1140 if (size & (val - 1)) { 1141 size &= ~(val - 1); 1142 if (size <= KMP_SIZE_T_MAX - val) { 1143 size += val; // Round up if there is no overflow. 1144 } 1145 } 1146 return size; 1147 } // __kmp_round_up_to_va 1148 1149 // __kmp_task_alloc: Allocate the taskdata and task data structures for a task 1150 // 1151 // loc_ref: source location information 1152 // gtid: global thread number. 1153 // flags: include tiedness & task type (explicit vs. implicit) of the ''new'' 1154 // task encountered. Converted from kmp_int32 to kmp_tasking_flags_t in routine. 1155 // sizeof_kmp_task_t: Size in bytes of kmp_task_t data structure including 1156 // private vars accessed in task. 1157 // sizeof_shareds: Size in bytes of array of pointers to shared vars accessed 1158 // in task. 1159 // task_entry: Pointer to task code entry point generated by compiler. 1160 // returns: a pointer to the allocated kmp_task_t structure (task). 1161 kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid, 1162 kmp_tasking_flags_t *flags, 1163 size_t sizeof_kmp_task_t, size_t sizeof_shareds, 1164 kmp_routine_entry_t task_entry) { 1165 kmp_task_t *task; 1166 kmp_taskdata_t *taskdata; 1167 kmp_info_t *thread = __kmp_threads[gtid]; 1168 kmp_team_t *team = thread->th.th_team; 1169 kmp_taskdata_t *parent_task = thread->th.th_current_task; 1170 size_t shareds_offset; 1171 1172 if (!TCR_4(__kmp_init_middle)) 1173 __kmp_middle_initialize(); 1174 1175 KA_TRACE(10, ("__kmp_task_alloc(enter): T#%d loc=%p, flags=(0x%x) " 1176 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n", 1177 gtid, loc_ref, *((kmp_int32 *)flags), sizeof_kmp_task_t, 1178 sizeof_shareds, task_entry)); 1179 1180 if (parent_task->td_flags.final) { 1181 if (flags->merged_if0) { 1182 } 1183 flags->final = 1; 1184 } 1185 if (flags->tiedness == TASK_UNTIED && !team->t.t_serialized) { 1186 // Untied task encountered causes the TSC algorithm to check entire deque of 1187 // the victim thread. If no untied task encountered, then checking the head 1188 // of the deque should be enough. 1189 KMP_CHECK_UPDATE(thread->th.th_task_team->tt.tt_untied_task_encountered, 1); 1190 } 1191 1192 // Detachable tasks are not proxy tasks yet but could be in the future. Doing 1193 // the tasking setup 1194 // when that happens is too late. 1195 if (flags->proxy == TASK_PROXY || flags->detachable == TASK_DETACHABLE) { 1196 if (flags->proxy == TASK_PROXY) { 1197 flags->tiedness = TASK_UNTIED; 1198 flags->merged_if0 = 1; 1199 } 1200 /* are we running in a sequential parallel or tskm_immediate_exec... we need 1201 tasking support enabled */ 1202 if ((thread->th.th_task_team) == NULL) { 1203 /* This should only happen if the team is serialized 1204 setup a task team and propagate it to the thread */ 1205 KMP_DEBUG_ASSERT(team->t.t_serialized); 1206 KA_TRACE(30, 1207 ("T#%d creating task team in __kmp_task_alloc for proxy task\n", 1208 gtid)); 1209 __kmp_task_team_setup( 1210 thread, team, 1211 1); // 1 indicates setup the current team regardless of nthreads 1212 thread->th.th_task_team = team->t.t_task_team[thread->th.th_task_state]; 1213 } 1214 kmp_task_team_t *task_team = thread->th.th_task_team; 1215 1216 /* tasking must be enabled now as the task might not be pushed */ 1217 if (!KMP_TASKING_ENABLED(task_team)) { 1218 KA_TRACE( 1219 30, 1220 ("T#%d enabling tasking in __kmp_task_alloc for proxy task\n", gtid)); 1221 __kmp_enable_tasking(task_team, thread); 1222 kmp_int32 tid = thread->th.th_info.ds.ds_tid; 1223 kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid]; 1224 // No lock needed since only owner can allocate 1225 if (thread_data->td.td_deque == NULL) { 1226 __kmp_alloc_task_deque(thread, thread_data); 1227 } 1228 } 1229 1230 if (task_team->tt.tt_found_proxy_tasks == FALSE) 1231 TCW_4(task_team->tt.tt_found_proxy_tasks, TRUE); 1232 } 1233 1234 // Calculate shared structure offset including padding after kmp_task_t struct 1235 // to align pointers in shared struct 1236 shareds_offset = sizeof(kmp_taskdata_t) + sizeof_kmp_task_t; 1237 shareds_offset = __kmp_round_up_to_val(shareds_offset, sizeof(void *)); 1238 1239 // Allocate a kmp_taskdata_t block and a kmp_task_t block. 1240 KA_TRACE(30, ("__kmp_task_alloc: T#%d First malloc size: %ld\n", gtid, 1241 shareds_offset)); 1242 KA_TRACE(30, ("__kmp_task_alloc: T#%d Second malloc size: %ld\n", gtid, 1243 sizeof_shareds)); 1244 1245 // Avoid double allocation here by combining shareds with taskdata 1246 #if USE_FAST_MEMORY 1247 taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, shareds_offset + 1248 sizeof_shareds); 1249 #else /* ! USE_FAST_MEMORY */ 1250 taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, shareds_offset + 1251 sizeof_shareds); 1252 #endif /* USE_FAST_MEMORY */ 1253 ANNOTATE_HAPPENS_AFTER(taskdata); 1254 1255 task = KMP_TASKDATA_TO_TASK(taskdata); 1256 1257 // Make sure task & taskdata are aligned appropriately 1258 #if KMP_ARCH_X86 || KMP_ARCH_PPC64 || !KMP_HAVE_QUAD 1259 KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(double) - 1)) == 0); 1260 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (sizeof(double) - 1)) == 0); 1261 #else 1262 KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(_Quad) - 1)) == 0); 1263 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (sizeof(_Quad) - 1)) == 0); 1264 #endif 1265 if (sizeof_shareds > 0) { 1266 // Avoid double allocation here by combining shareds with taskdata 1267 task->shareds = &((char *)taskdata)[shareds_offset]; 1268 // Make sure shareds struct is aligned to pointer size 1269 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) == 1270 0); 1271 } else { 1272 task->shareds = NULL; 1273 } 1274 task->routine = task_entry; 1275 task->part_id = 0; // AC: Always start with 0 part id 1276 1277 taskdata->td_task_id = KMP_GEN_TASK_ID(); 1278 taskdata->td_team = team; 1279 taskdata->td_alloc_thread = thread; 1280 taskdata->td_parent = parent_task; 1281 taskdata->td_level = parent_task->td_level + 1; // increment nesting level 1282 KMP_ATOMIC_ST_RLX(&taskdata->td_untied_count, 0); 1283 taskdata->td_ident = loc_ref; 1284 taskdata->td_taskwait_ident = NULL; 1285 taskdata->td_taskwait_counter = 0; 1286 taskdata->td_taskwait_thread = 0; 1287 KMP_DEBUG_ASSERT(taskdata->td_parent != NULL); 1288 // avoid copying icvs for proxy tasks 1289 if (flags->proxy == TASK_FULL) 1290 copy_icvs(&taskdata->td_icvs, &taskdata->td_parent->td_icvs); 1291 1292 taskdata->td_flags.tiedness = flags->tiedness; 1293 taskdata->td_flags.final = flags->final; 1294 taskdata->td_flags.merged_if0 = flags->merged_if0; 1295 taskdata->td_flags.destructors_thunk = flags->destructors_thunk; 1296 taskdata->td_flags.proxy = flags->proxy; 1297 taskdata->td_flags.detachable = flags->detachable; 1298 taskdata->td_task_team = thread->th.th_task_team; 1299 taskdata->td_size_alloc = shareds_offset + sizeof_shareds; 1300 taskdata->td_flags.tasktype = TASK_EXPLICIT; 1301 1302 // GEH - TODO: fix this to copy parent task's value of tasking_ser flag 1303 taskdata->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec); 1304 1305 // GEH - TODO: fix this to copy parent task's value of team_serial flag 1306 taskdata->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0; 1307 1308 // GEH - Note we serialize the task if the team is serialized to make sure 1309 // implicit parallel region tasks are not left until program termination to 1310 // execute. Also, it helps locality to execute immediately. 1311 1312 taskdata->td_flags.task_serial = 1313 (parent_task->td_flags.final || taskdata->td_flags.team_serial || 1314 taskdata->td_flags.tasking_ser); 1315 1316 taskdata->td_flags.started = 0; 1317 taskdata->td_flags.executing = 0; 1318 taskdata->td_flags.complete = 0; 1319 taskdata->td_flags.freed = 0; 1320 1321 taskdata->td_flags.native = flags->native; 1322 1323 KMP_ATOMIC_ST_RLX(&taskdata->td_incomplete_child_tasks, 0); 1324 // start at one because counts current task and children 1325 KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 1); 1326 taskdata->td_taskgroup = 1327 parent_task->td_taskgroup; // task inherits taskgroup from the parent task 1328 taskdata->td_dephash = NULL; 1329 taskdata->td_depnode = NULL; 1330 if (flags->tiedness == TASK_UNTIED) 1331 taskdata->td_last_tied = NULL; // will be set when the task is scheduled 1332 else 1333 taskdata->td_last_tied = taskdata; 1334 taskdata->td_allow_completion_event.type = KMP_EVENT_UNINITIALIZED; 1335 #if OMPT_SUPPORT 1336 if (UNLIKELY(ompt_enabled.enabled)) 1337 __ompt_task_init(taskdata, gtid); 1338 #endif 1339 // Only need to keep track of child task counts if team parallel and tasking not 1340 // serialized or if it is a proxy or detachable task 1341 if (flags->proxy == TASK_PROXY || 1342 flags->detachable == TASK_DETACHABLE || 1343 !(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) 1344 { 1345 KMP_ATOMIC_INC(&parent_task->td_incomplete_child_tasks); 1346 if (parent_task->td_taskgroup) 1347 KMP_ATOMIC_INC(&parent_task->td_taskgroup->count); 1348 // Only need to keep track of allocated child tasks for explicit tasks since 1349 // implicit not deallocated 1350 if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT) { 1351 KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks); 1352 } 1353 } 1354 1355 KA_TRACE(20, ("__kmp_task_alloc(exit): T#%d created task %p parent=%p\n", 1356 gtid, taskdata, taskdata->td_parent)); 1357 ANNOTATE_HAPPENS_BEFORE(task); 1358 1359 return task; 1360 } 1361 1362 kmp_task_t *__kmpc_omp_task_alloc(ident_t *loc_ref, kmp_int32 gtid, 1363 kmp_int32 flags, size_t sizeof_kmp_task_t, 1364 size_t sizeof_shareds, 1365 kmp_routine_entry_t task_entry) { 1366 kmp_task_t *retval; 1367 kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *)&flags; 1368 1369 input_flags->native = FALSE; 1370 // __kmp_task_alloc() sets up all other runtime flags 1371 1372 KA_TRACE(10, ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s %s %s) " 1373 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n", 1374 gtid, loc_ref, input_flags->tiedness ? "tied " : "untied", 1375 input_flags->proxy ? "proxy" : "", 1376 input_flags->detachable ? "detachable" : "", sizeof_kmp_task_t, 1377 sizeof_shareds, task_entry)); 1378 1379 retval = __kmp_task_alloc(loc_ref, gtid, input_flags, sizeof_kmp_task_t, 1380 sizeof_shareds, task_entry); 1381 1382 KA_TRACE(20, ("__kmpc_omp_task_alloc(exit): T#%d retval %p\n", gtid, retval)); 1383 1384 return retval; 1385 } 1386 1387 kmp_task_t *__kmpc_omp_target_task_alloc(ident_t *loc_ref, kmp_int32 gtid, 1388 kmp_int32 flags, 1389 size_t sizeof_kmp_task_t, 1390 size_t sizeof_shareds, 1391 kmp_routine_entry_t task_entry, 1392 kmp_int64 device_id) { 1393 return __kmpc_omp_task_alloc(loc_ref, gtid, flags, sizeof_kmp_task_t, 1394 sizeof_shareds, task_entry); 1395 } 1396 1397 /*! 1398 @ingroup TASKING 1399 @param loc_ref location of the original task directive 1400 @param gtid Global Thread ID of encountering thread 1401 @param new_task task thunk allocated by __kmpc_omp_task_alloc() for the ''new 1402 task'' 1403 @param naffins Number of affinity items 1404 @param affin_list List of affinity items 1405 @return Returns non-zero if registering affinity information was not successful. 1406 Returns 0 if registration was successful 1407 This entry registers the affinity information attached to a task with the task 1408 thunk structure kmp_taskdata_t. 1409 */ 1410 kmp_int32 1411 __kmpc_omp_reg_task_with_affinity(ident_t *loc_ref, kmp_int32 gtid, 1412 kmp_task_t *new_task, kmp_int32 naffins, 1413 kmp_task_affinity_info_t *affin_list) { 1414 return 0; 1415 } 1416 1417 // __kmp_invoke_task: invoke the specified task 1418 // 1419 // gtid: global thread ID of caller 1420 // task: the task to invoke 1421 // current_task: the task to resume after task invocation 1422 static void __kmp_invoke_task(kmp_int32 gtid, kmp_task_t *task, 1423 kmp_taskdata_t *current_task) { 1424 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 1425 kmp_info_t *thread; 1426 int discard = 0 /* false */; 1427 KA_TRACE( 1428 30, ("__kmp_invoke_task(enter): T#%d invoking task %p, current_task=%p\n", 1429 gtid, taskdata, current_task)); 1430 KMP_DEBUG_ASSERT(task); 1431 if (taskdata->td_flags.proxy == TASK_PROXY && 1432 taskdata->td_flags.complete == 1) { 1433 // This is a proxy task that was already completed but it needs to run 1434 // its bottom-half finish 1435 KA_TRACE( 1436 30, 1437 ("__kmp_invoke_task: T#%d running bottom finish for proxy task %p\n", 1438 gtid, taskdata)); 1439 1440 __kmp_bottom_half_finish_proxy(gtid, task); 1441 1442 KA_TRACE(30, ("__kmp_invoke_task(exit): T#%d completed bottom finish for " 1443 "proxy task %p, resuming task %p\n", 1444 gtid, taskdata, current_task)); 1445 1446 return; 1447 } 1448 1449 #if OMPT_SUPPORT 1450 // For untied tasks, the first task executed only calls __kmpc_omp_task and 1451 // does not execute code. 1452 ompt_thread_info_t oldInfo; 1453 if (UNLIKELY(ompt_enabled.enabled)) { 1454 // Store the threads states and restore them after the task 1455 thread = __kmp_threads[gtid]; 1456 oldInfo = thread->th.ompt_thread_info; 1457 thread->th.ompt_thread_info.wait_id = 0; 1458 thread->th.ompt_thread_info.state = (thread->th.th_team_serialized) 1459 ? ompt_state_work_serial 1460 : ompt_state_work_parallel; 1461 taskdata->ompt_task_info.frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1462 } 1463 #endif 1464 1465 // Proxy tasks are not handled by the runtime 1466 if (taskdata->td_flags.proxy != TASK_PROXY) { 1467 ANNOTATE_HAPPENS_AFTER(task); 1468 __kmp_task_start(gtid, task, current_task); // OMPT only if not discarded 1469 } 1470 1471 // TODO: cancel tasks if the parallel region has also been cancelled 1472 // TODO: check if this sequence can be hoisted above __kmp_task_start 1473 // if cancellation has been enabled for this run ... 1474 if (__kmp_omp_cancellation) { 1475 thread = __kmp_threads[gtid]; 1476 kmp_team_t *this_team = thread->th.th_team; 1477 kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup; 1478 if ((taskgroup && taskgroup->cancel_request) || 1479 (this_team->t.t_cancel_request == cancel_parallel)) { 1480 #if OMPT_SUPPORT && OMPT_OPTIONAL 1481 ompt_data_t *task_data; 1482 if (UNLIKELY(ompt_enabled.ompt_callback_cancel)) { 1483 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL, NULL); 1484 ompt_callbacks.ompt_callback(ompt_callback_cancel)( 1485 task_data, 1486 ((taskgroup && taskgroup->cancel_request) ? ompt_cancel_taskgroup 1487 : ompt_cancel_parallel) | 1488 ompt_cancel_discarded_task, 1489 NULL); 1490 } 1491 #endif 1492 KMP_COUNT_BLOCK(TASK_cancelled); 1493 // this task belongs to a task group and we need to cancel it 1494 discard = 1 /* true */; 1495 } 1496 } 1497 1498 // Invoke the task routine and pass in relevant data. 1499 // Thunks generated by gcc take a different argument list. 1500 if (!discard) { 1501 if (taskdata->td_flags.tiedness == TASK_UNTIED) { 1502 taskdata->td_last_tied = current_task->td_last_tied; 1503 KMP_DEBUG_ASSERT(taskdata->td_last_tied); 1504 } 1505 #if KMP_STATS_ENABLED 1506 KMP_COUNT_BLOCK(TASK_executed); 1507 switch (KMP_GET_THREAD_STATE()) { 1508 case FORK_JOIN_BARRIER: 1509 KMP_PUSH_PARTITIONED_TIMER(OMP_task_join_bar); 1510 break; 1511 case PLAIN_BARRIER: 1512 KMP_PUSH_PARTITIONED_TIMER(OMP_task_plain_bar); 1513 break; 1514 case TASKYIELD: 1515 KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskyield); 1516 break; 1517 case TASKWAIT: 1518 KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskwait); 1519 break; 1520 case TASKGROUP: 1521 KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskgroup); 1522 break; 1523 default: 1524 KMP_PUSH_PARTITIONED_TIMER(OMP_task_immediate); 1525 break; 1526 } 1527 #endif // KMP_STATS_ENABLED 1528 1529 // OMPT task begin 1530 #if OMPT_SUPPORT 1531 if (UNLIKELY(ompt_enabled.enabled)) 1532 __ompt_task_start(task, current_task, gtid); 1533 #endif 1534 1535 #if USE_ITT_BUILD && USE_ITT_NOTIFY 1536 kmp_uint64 cur_time; 1537 kmp_int32 kmp_itt_count_task = 1538 __kmp_forkjoin_frames_mode == 3 && !taskdata->td_flags.task_serial && 1539 current_task->td_flags.tasktype == TASK_IMPLICIT; 1540 if (kmp_itt_count_task) { 1541 thread = __kmp_threads[gtid]; 1542 // Time outer level explicit task on barrier for adjusting imbalance time 1543 if (thread->th.th_bar_arrive_time) 1544 cur_time = __itt_get_timestamp(); 1545 else 1546 kmp_itt_count_task = 0; // thread is not on a barrier - skip timing 1547 } 1548 #endif 1549 1550 #ifdef KMP_GOMP_COMPAT 1551 if (taskdata->td_flags.native) { 1552 ((void (*)(void *))(*(task->routine)))(task->shareds); 1553 } else 1554 #endif /* KMP_GOMP_COMPAT */ 1555 { 1556 (*(task->routine))(gtid, task); 1557 } 1558 KMP_POP_PARTITIONED_TIMER(); 1559 1560 #if USE_ITT_BUILD && USE_ITT_NOTIFY 1561 if (kmp_itt_count_task) { 1562 // Barrier imbalance - adjust arrive time with the task duration 1563 thread->th.th_bar_arrive_time += (__itt_get_timestamp() - cur_time); 1564 } 1565 #endif 1566 1567 } 1568 1569 1570 // Proxy tasks are not handled by the runtime 1571 if (taskdata->td_flags.proxy != TASK_PROXY) { 1572 ANNOTATE_HAPPENS_BEFORE(taskdata->td_parent); 1573 #if OMPT_SUPPORT 1574 if (UNLIKELY(ompt_enabled.enabled)) { 1575 thread->th.ompt_thread_info = oldInfo; 1576 if (taskdata->td_flags.tiedness == TASK_TIED) { 1577 taskdata->ompt_task_info.frame.exit_frame = ompt_data_none; 1578 } 1579 __kmp_task_finish<true>(gtid, task, current_task); 1580 } else 1581 #endif 1582 __kmp_task_finish<false>(gtid, task, current_task); 1583 } 1584 1585 KA_TRACE( 1586 30, 1587 ("__kmp_invoke_task(exit): T#%d completed task %p, resuming task %p\n", 1588 gtid, taskdata, current_task)); 1589 return; 1590 } 1591 1592 // __kmpc_omp_task_parts: Schedule a thread-switchable task for execution 1593 // 1594 // loc_ref: location of original task pragma (ignored) 1595 // gtid: Global Thread ID of encountering thread 1596 // new_task: task thunk allocated by __kmp_omp_task_alloc() for the ''new task'' 1597 // Returns: 1598 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to 1599 // be resumed later. 1600 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be 1601 // resumed later. 1602 kmp_int32 __kmpc_omp_task_parts(ident_t *loc_ref, kmp_int32 gtid, 1603 kmp_task_t *new_task) { 1604 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task); 1605 1606 KA_TRACE(10, ("__kmpc_omp_task_parts(enter): T#%d loc=%p task=%p\n", gtid, 1607 loc_ref, new_taskdata)); 1608 1609 #if OMPT_SUPPORT 1610 kmp_taskdata_t *parent; 1611 if (UNLIKELY(ompt_enabled.enabled)) { 1612 parent = new_taskdata->td_parent; 1613 if (ompt_enabled.ompt_callback_task_create) { 1614 ompt_data_t task_data = ompt_data_none; 1615 ompt_callbacks.ompt_callback(ompt_callback_task_create)( 1616 parent ? &(parent->ompt_task_info.task_data) : &task_data, 1617 parent ? &(parent->ompt_task_info.frame) : NULL, 1618 &(new_taskdata->ompt_task_info.task_data), ompt_task_explicit, 0, 1619 OMPT_GET_RETURN_ADDRESS(0)); 1620 } 1621 } 1622 #endif 1623 1624 /* Should we execute the new task or queue it? For now, let's just always try 1625 to queue it. If the queue fills up, then we'll execute it. */ 1626 1627 if (__kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED) // if cannot defer 1628 { // Execute this task immediately 1629 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task; 1630 new_taskdata->td_flags.task_serial = 1; 1631 __kmp_invoke_task(gtid, new_task, current_task); 1632 } 1633 1634 KA_TRACE( 1635 10, 1636 ("__kmpc_omp_task_parts(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: " 1637 "loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n", 1638 gtid, loc_ref, new_taskdata)); 1639 1640 ANNOTATE_HAPPENS_BEFORE(new_task); 1641 #if OMPT_SUPPORT 1642 if (UNLIKELY(ompt_enabled.enabled)) { 1643 parent->ompt_task_info.frame.enter_frame = ompt_data_none; 1644 } 1645 #endif 1646 return TASK_CURRENT_NOT_QUEUED; 1647 } 1648 1649 // __kmp_omp_task: Schedule a non-thread-switchable task for execution 1650 // 1651 // gtid: Global Thread ID of encountering thread 1652 // new_task:non-thread-switchable task thunk allocated by __kmp_omp_task_alloc() 1653 // serialize_immediate: if TRUE then if the task is executed immediately its 1654 // execution will be serialized 1655 // Returns: 1656 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to 1657 // be resumed later. 1658 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be 1659 // resumed later. 1660 kmp_int32 __kmp_omp_task(kmp_int32 gtid, kmp_task_t *new_task, 1661 bool serialize_immediate) { 1662 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task); 1663 1664 /* Should we execute the new task or queue it? For now, let's just always try 1665 to queue it. If the queue fills up, then we'll execute it. */ 1666 if (new_taskdata->td_flags.proxy == TASK_PROXY || 1667 __kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED) // if cannot defer 1668 { // Execute this task immediately 1669 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task; 1670 if (serialize_immediate) 1671 new_taskdata->td_flags.task_serial = 1; 1672 __kmp_invoke_task(gtid, new_task, current_task); 1673 } 1674 1675 ANNOTATE_HAPPENS_BEFORE(new_task); 1676 return TASK_CURRENT_NOT_QUEUED; 1677 } 1678 1679 // __kmpc_omp_task: Wrapper around __kmp_omp_task to schedule a 1680 // non-thread-switchable task from the parent thread only! 1681 // 1682 // loc_ref: location of original task pragma (ignored) 1683 // gtid: Global Thread ID of encountering thread 1684 // new_task: non-thread-switchable task thunk allocated by 1685 // __kmp_omp_task_alloc() 1686 // Returns: 1687 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to 1688 // be resumed later. 1689 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be 1690 // resumed later. 1691 kmp_int32 __kmpc_omp_task(ident_t *loc_ref, kmp_int32 gtid, 1692 kmp_task_t *new_task) { 1693 kmp_int32 res; 1694 KMP_SET_THREAD_STATE_BLOCK(EXPLICIT_TASK); 1695 1696 #if KMP_DEBUG || OMPT_SUPPORT 1697 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task); 1698 #endif 1699 KA_TRACE(10, ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref, 1700 new_taskdata)); 1701 1702 #if OMPT_SUPPORT 1703 kmp_taskdata_t *parent = NULL; 1704 if (UNLIKELY(ompt_enabled.enabled)) { 1705 if (!new_taskdata->td_flags.started) { 1706 OMPT_STORE_RETURN_ADDRESS(gtid); 1707 parent = new_taskdata->td_parent; 1708 if (!parent->ompt_task_info.frame.enter_frame.ptr) { 1709 parent->ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1710 } 1711 if (ompt_enabled.ompt_callback_task_create) { 1712 ompt_data_t task_data = ompt_data_none; 1713 ompt_callbacks.ompt_callback(ompt_callback_task_create)( 1714 parent ? &(parent->ompt_task_info.task_data) : &task_data, 1715 parent ? &(parent->ompt_task_info.frame) : NULL, 1716 &(new_taskdata->ompt_task_info.task_data), 1717 ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata), 0, 1718 OMPT_LOAD_RETURN_ADDRESS(gtid)); 1719 } 1720 } else { 1721 // We are scheduling the continuation of an UNTIED task. 1722 // Scheduling back to the parent task. 1723 __ompt_task_finish(new_task, 1724 new_taskdata->ompt_task_info.scheduling_parent, 1725 ompt_task_switch); 1726 new_taskdata->ompt_task_info.frame.exit_frame = ompt_data_none; 1727 } 1728 } 1729 #endif 1730 1731 res = __kmp_omp_task(gtid, new_task, true); 1732 1733 KA_TRACE(10, ("__kmpc_omp_task(exit): T#%d returning " 1734 "TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n", 1735 gtid, loc_ref, new_taskdata)); 1736 #if OMPT_SUPPORT 1737 if (UNLIKELY(ompt_enabled.enabled && parent != NULL)) { 1738 parent->ompt_task_info.frame.enter_frame = ompt_data_none; 1739 } 1740 #endif 1741 return res; 1742 } 1743 1744 // __kmp_omp_taskloop_task: Wrapper around __kmp_omp_task to schedule 1745 // a taskloop task with the correct OMPT return address 1746 // 1747 // loc_ref: location of original task pragma (ignored) 1748 // gtid: Global Thread ID of encountering thread 1749 // new_task: non-thread-switchable task thunk allocated by 1750 // __kmp_omp_task_alloc() 1751 // codeptr_ra: return address for OMPT callback 1752 // Returns: 1753 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to 1754 // be resumed later. 1755 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be 1756 // resumed later. 1757 kmp_int32 __kmp_omp_taskloop_task(ident_t *loc_ref, kmp_int32 gtid, 1758 kmp_task_t *new_task, void *codeptr_ra) { 1759 kmp_int32 res; 1760 KMP_SET_THREAD_STATE_BLOCK(EXPLICIT_TASK); 1761 1762 #if KMP_DEBUG || OMPT_SUPPORT 1763 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task); 1764 #endif 1765 KA_TRACE(10, ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref, 1766 new_taskdata)); 1767 1768 #if OMPT_SUPPORT 1769 kmp_taskdata_t *parent = NULL; 1770 if (UNLIKELY(ompt_enabled.enabled && !new_taskdata->td_flags.started)) { 1771 parent = new_taskdata->td_parent; 1772 if (!parent->ompt_task_info.frame.enter_frame.ptr) 1773 parent->ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1774 if (ompt_enabled.ompt_callback_task_create) { 1775 ompt_data_t task_data = ompt_data_none; 1776 ompt_callbacks.ompt_callback(ompt_callback_task_create)( 1777 parent ? &(parent->ompt_task_info.task_data) : &task_data, 1778 parent ? &(parent->ompt_task_info.frame) : NULL, 1779 &(new_taskdata->ompt_task_info.task_data), 1780 ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata), 0, 1781 codeptr_ra); 1782 } 1783 } 1784 #endif 1785 1786 res = __kmp_omp_task(gtid, new_task, true); 1787 1788 KA_TRACE(10, ("__kmpc_omp_task(exit): T#%d returning " 1789 "TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n", 1790 gtid, loc_ref, new_taskdata)); 1791 #if OMPT_SUPPORT 1792 if (UNLIKELY(ompt_enabled.enabled && parent != NULL)) { 1793 parent->ompt_task_info.frame.enter_frame = ompt_data_none; 1794 } 1795 #endif 1796 return res; 1797 } 1798 1799 template <bool ompt> 1800 static kmp_int32 __kmpc_omp_taskwait_template(ident_t *loc_ref, kmp_int32 gtid, 1801 void *frame_address, 1802 void *return_address) { 1803 kmp_taskdata_t *taskdata; 1804 kmp_info_t *thread; 1805 int thread_finished = FALSE; 1806 KMP_SET_THREAD_STATE_BLOCK(TASKWAIT); 1807 1808 KA_TRACE(10, ("__kmpc_omp_taskwait(enter): T#%d loc=%p\n", gtid, loc_ref)); 1809 1810 if (__kmp_tasking_mode != tskm_immediate_exec) { 1811 thread = __kmp_threads[gtid]; 1812 taskdata = thread->th.th_current_task; 1813 1814 #if OMPT_SUPPORT && OMPT_OPTIONAL 1815 ompt_data_t *my_task_data; 1816 ompt_data_t *my_parallel_data; 1817 1818 if (ompt) { 1819 my_task_data = &(taskdata->ompt_task_info.task_data); 1820 my_parallel_data = OMPT_CUR_TEAM_DATA(thread); 1821 1822 taskdata->ompt_task_info.frame.enter_frame.ptr = frame_address; 1823 1824 if (ompt_enabled.ompt_callback_sync_region) { 1825 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 1826 ompt_sync_region_taskwait, ompt_scope_begin, my_parallel_data, 1827 my_task_data, return_address); 1828 } 1829 1830 if (ompt_enabled.ompt_callback_sync_region_wait) { 1831 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 1832 ompt_sync_region_taskwait, ompt_scope_begin, my_parallel_data, 1833 my_task_data, return_address); 1834 } 1835 } 1836 #endif // OMPT_SUPPORT && OMPT_OPTIONAL 1837 1838 // Debugger: The taskwait is active. Store location and thread encountered the 1839 // taskwait. 1840 #if USE_ITT_BUILD 1841 // Note: These values are used by ITT events as well. 1842 #endif /* USE_ITT_BUILD */ 1843 taskdata->td_taskwait_counter += 1; 1844 taskdata->td_taskwait_ident = loc_ref; 1845 taskdata->td_taskwait_thread = gtid + 1; 1846 1847 #if USE_ITT_BUILD 1848 void *itt_sync_obj = __kmp_itt_taskwait_object(gtid); 1849 if (itt_sync_obj != NULL) 1850 __kmp_itt_taskwait_starting(gtid, itt_sync_obj); 1851 #endif /* USE_ITT_BUILD */ 1852 1853 bool must_wait = 1854 !taskdata->td_flags.team_serial && !taskdata->td_flags.final; 1855 1856 must_wait = must_wait || (thread->th.th_task_team != NULL && 1857 thread->th.th_task_team->tt.tt_found_proxy_tasks); 1858 if (must_wait) { 1859 kmp_flag_32 flag(RCAST(std::atomic<kmp_uint32> *, 1860 &(taskdata->td_incomplete_child_tasks)), 1861 0U); 1862 while (KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) != 0) { 1863 flag.execute_tasks(thread, gtid, FALSE, 1864 &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), 1865 __kmp_task_stealing_constraint); 1866 } 1867 } 1868 #if USE_ITT_BUILD 1869 if (itt_sync_obj != NULL) 1870 __kmp_itt_taskwait_finished(gtid, itt_sync_obj); 1871 #endif /* USE_ITT_BUILD */ 1872 1873 // Debugger: The taskwait is completed. Location remains, but thread is 1874 // negated. 1875 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; 1876 1877 #if OMPT_SUPPORT && OMPT_OPTIONAL 1878 if (ompt) { 1879 if (ompt_enabled.ompt_callback_sync_region_wait) { 1880 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 1881 ompt_sync_region_taskwait, ompt_scope_end, my_parallel_data, 1882 my_task_data, return_address); 1883 } 1884 if (ompt_enabled.ompt_callback_sync_region) { 1885 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 1886 ompt_sync_region_taskwait, ompt_scope_end, my_parallel_data, 1887 my_task_data, return_address); 1888 } 1889 taskdata->ompt_task_info.frame.enter_frame = ompt_data_none; 1890 } 1891 #endif // OMPT_SUPPORT && OMPT_OPTIONAL 1892 1893 ANNOTATE_HAPPENS_AFTER(taskdata); 1894 } 1895 1896 KA_TRACE(10, ("__kmpc_omp_taskwait(exit): T#%d task %p finished waiting, " 1897 "returning TASK_CURRENT_NOT_QUEUED\n", 1898 gtid, taskdata)); 1899 1900 return TASK_CURRENT_NOT_QUEUED; 1901 } 1902 1903 #if OMPT_SUPPORT && OMPT_OPTIONAL 1904 OMPT_NOINLINE 1905 static kmp_int32 __kmpc_omp_taskwait_ompt(ident_t *loc_ref, kmp_int32 gtid, 1906 void *frame_address, 1907 void *return_address) { 1908 return __kmpc_omp_taskwait_template<true>(loc_ref, gtid, frame_address, 1909 return_address); 1910 } 1911 #endif // OMPT_SUPPORT && OMPT_OPTIONAL 1912 1913 // __kmpc_omp_taskwait: Wait until all tasks generated by the current task are 1914 // complete 1915 kmp_int32 __kmpc_omp_taskwait(ident_t *loc_ref, kmp_int32 gtid) { 1916 #if OMPT_SUPPORT && OMPT_OPTIONAL 1917 if (UNLIKELY(ompt_enabled.enabled)) { 1918 OMPT_STORE_RETURN_ADDRESS(gtid); 1919 return __kmpc_omp_taskwait_ompt(loc_ref, gtid, OMPT_GET_FRAME_ADDRESS(0), 1920 OMPT_LOAD_RETURN_ADDRESS(gtid)); 1921 } 1922 #endif 1923 return __kmpc_omp_taskwait_template<false>(loc_ref, gtid, NULL, NULL); 1924 } 1925 1926 // __kmpc_omp_taskyield: switch to a different task 1927 kmp_int32 __kmpc_omp_taskyield(ident_t *loc_ref, kmp_int32 gtid, int end_part) { 1928 kmp_taskdata_t *taskdata; 1929 kmp_info_t *thread; 1930 int thread_finished = FALSE; 1931 1932 KMP_COUNT_BLOCK(OMP_TASKYIELD); 1933 KMP_SET_THREAD_STATE_BLOCK(TASKYIELD); 1934 1935 KA_TRACE(10, ("__kmpc_omp_taskyield(enter): T#%d loc=%p end_part = %d\n", 1936 gtid, loc_ref, end_part)); 1937 1938 if (__kmp_tasking_mode != tskm_immediate_exec && __kmp_init_parallel) { 1939 thread = __kmp_threads[gtid]; 1940 taskdata = thread->th.th_current_task; 1941 // Should we model this as a task wait or not? 1942 // Debugger: The taskwait is active. Store location and thread encountered the 1943 // taskwait. 1944 #if USE_ITT_BUILD 1945 // Note: These values are used by ITT events as well. 1946 #endif /* USE_ITT_BUILD */ 1947 taskdata->td_taskwait_counter += 1; 1948 taskdata->td_taskwait_ident = loc_ref; 1949 taskdata->td_taskwait_thread = gtid + 1; 1950 1951 #if USE_ITT_BUILD 1952 void *itt_sync_obj = __kmp_itt_taskwait_object(gtid); 1953 if (itt_sync_obj != NULL) 1954 __kmp_itt_taskwait_starting(gtid, itt_sync_obj); 1955 #endif /* USE_ITT_BUILD */ 1956 if (!taskdata->td_flags.team_serial) { 1957 kmp_task_team_t *task_team = thread->th.th_task_team; 1958 if (task_team != NULL) { 1959 if (KMP_TASKING_ENABLED(task_team)) { 1960 #if OMPT_SUPPORT 1961 if (UNLIKELY(ompt_enabled.enabled)) 1962 thread->th.ompt_thread_info.ompt_task_yielded = 1; 1963 #endif 1964 __kmp_execute_tasks_32( 1965 thread, gtid, NULL, FALSE, 1966 &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), 1967 __kmp_task_stealing_constraint); 1968 #if OMPT_SUPPORT 1969 if (UNLIKELY(ompt_enabled.enabled)) 1970 thread->th.ompt_thread_info.ompt_task_yielded = 0; 1971 #endif 1972 } 1973 } 1974 } 1975 #if USE_ITT_BUILD 1976 if (itt_sync_obj != NULL) 1977 __kmp_itt_taskwait_finished(gtid, itt_sync_obj); 1978 #endif /* USE_ITT_BUILD */ 1979 1980 // Debugger: The taskwait is completed. Location remains, but thread is 1981 // negated. 1982 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; 1983 } 1984 1985 KA_TRACE(10, ("__kmpc_omp_taskyield(exit): T#%d task %p resuming, " 1986 "returning TASK_CURRENT_NOT_QUEUED\n", 1987 gtid, taskdata)); 1988 1989 return TASK_CURRENT_NOT_QUEUED; 1990 } 1991 1992 // Task Reduction implementation 1993 // 1994 // Note: initial implementation didn't take into account the possibility 1995 // to specify omp_orig for initializer of the UDR (user defined reduction). 1996 // Corrected implementation takes into account the omp_orig object. 1997 // Compiler is free to use old implementation if omp_orig is not specified. 1998 1999 /*! 2000 @ingroup BASIC_TYPES 2001 @{ 2002 */ 2003 2004 /*! 2005 Flags for special info per task reduction item. 2006 */ 2007 typedef struct kmp_taskred_flags { 2008 /*! 1 - use lazy alloc/init (e.g. big objects, #tasks < #threads) */ 2009 unsigned lazy_priv : 1; 2010 unsigned reserved31 : 31; 2011 } kmp_taskred_flags_t; 2012 2013 /*! 2014 Internal struct for reduction data item related info set up by compiler. 2015 */ 2016 typedef struct kmp_task_red_input { 2017 void *reduce_shar; /**< shared between tasks item to reduce into */ 2018 size_t reduce_size; /**< size of data item in bytes */ 2019 // three compiler-generated routines (init, fini are optional): 2020 void *reduce_init; /**< data initialization routine (single parameter) */ 2021 void *reduce_fini; /**< data finalization routine */ 2022 void *reduce_comb; /**< data combiner routine */ 2023 kmp_taskred_flags_t flags; /**< flags for additional info from compiler */ 2024 } kmp_task_red_input_t; 2025 2026 /*! 2027 Internal struct for reduction data item related info saved by the library. 2028 */ 2029 typedef struct kmp_taskred_data { 2030 void *reduce_shar; /**< shared between tasks item to reduce into */ 2031 size_t reduce_size; /**< size of data item */ 2032 kmp_taskred_flags_t flags; /**< flags for additional info from compiler */ 2033 void *reduce_priv; /**< array of thread specific items */ 2034 void *reduce_pend; /**< end of private data for faster comparison op */ 2035 // three compiler-generated routines (init, fini are optional): 2036 void *reduce_comb; /**< data combiner routine */ 2037 void *reduce_init; /**< data initialization routine (two parameters) */ 2038 void *reduce_fini; /**< data finalization routine */ 2039 void *reduce_orig; /**< original item (can be used in UDR initializer) */ 2040 } kmp_taskred_data_t; 2041 2042 /*! 2043 Internal struct for reduction data item related info set up by compiler. 2044 2045 New interface: added reduce_orig field to provide omp_orig for UDR initializer. 2046 */ 2047 typedef struct kmp_taskred_input { 2048 void *reduce_shar; /**< shared between tasks item to reduce into */ 2049 void *reduce_orig; /**< original reduction item used for initialization */ 2050 size_t reduce_size; /**< size of data item */ 2051 // three compiler-generated routines (init, fini are optional): 2052 void *reduce_init; /**< data initialization routine (two parameters) */ 2053 void *reduce_fini; /**< data finalization routine */ 2054 void *reduce_comb; /**< data combiner routine */ 2055 kmp_taskred_flags_t flags; /**< flags for additional info from compiler */ 2056 } kmp_taskred_input_t; 2057 /*! 2058 @} 2059 */ 2060 2061 template <typename T> void __kmp_assign_orig(kmp_taskred_data_t &item, T &src); 2062 template <> 2063 void __kmp_assign_orig<kmp_task_red_input_t>(kmp_taskred_data_t &item, 2064 kmp_task_red_input_t &src) { 2065 item.reduce_orig = NULL; 2066 } 2067 template <> 2068 void __kmp_assign_orig<kmp_taskred_input_t>(kmp_taskred_data_t &item, 2069 kmp_taskred_input_t &src) { 2070 if (src.reduce_orig != NULL) { 2071 item.reduce_orig = src.reduce_orig; 2072 } else { 2073 item.reduce_orig = src.reduce_shar; 2074 } // non-NULL reduce_orig means new interface used 2075 } 2076 2077 template <typename T> void __kmp_call_init(kmp_taskred_data_t &item, int j); 2078 template <> 2079 void __kmp_call_init<kmp_task_red_input_t>(kmp_taskred_data_t &item, 2080 int offset) { 2081 ((void (*)(void *))item.reduce_init)((char *)(item.reduce_priv) + offset); 2082 } 2083 template <> 2084 void __kmp_call_init<kmp_taskred_input_t>(kmp_taskred_data_t &item, 2085 int offset) { 2086 ((void (*)(void *, void *))item.reduce_init)( 2087 (char *)(item.reduce_priv) + offset, item.reduce_orig); 2088 } 2089 2090 template <typename T> 2091 void *__kmp_task_reduction_init(int gtid, int num, T *data) { 2092 kmp_info_t *thread = __kmp_threads[gtid]; 2093 kmp_taskgroup_t *tg = thread->th.th_current_task->td_taskgroup; 2094 kmp_int32 nth = thread->th.th_team_nproc; 2095 kmp_taskred_data_t *arr; 2096 2097 // check input data just in case 2098 KMP_ASSERT(tg != NULL); 2099 KMP_ASSERT(data != NULL); 2100 KMP_ASSERT(num > 0); 2101 if (nth == 1) { 2102 KA_TRACE(10, ("__kmpc_task_reduction_init: T#%d, tg %p, exiting nth=1\n", 2103 gtid, tg)); 2104 return (void *)tg; 2105 } 2106 KA_TRACE(10, ("__kmpc_task_reduction_init: T#%d, taskgroup %p, #items %d\n", 2107 gtid, tg, num)); 2108 arr = (kmp_taskred_data_t *)__kmp_thread_malloc( 2109 thread, num * sizeof(kmp_taskred_data_t)); 2110 for (int i = 0; i < num; ++i) { 2111 size_t size = data[i].reduce_size - 1; 2112 // round the size up to cache line per thread-specific item 2113 size += CACHE_LINE - size % CACHE_LINE; 2114 KMP_ASSERT(data[i].reduce_comb != NULL); // combiner is mandatory 2115 arr[i].reduce_shar = data[i].reduce_shar; 2116 arr[i].reduce_size = size; 2117 arr[i].flags = data[i].flags; 2118 arr[i].reduce_comb = data[i].reduce_comb; 2119 arr[i].reduce_init = data[i].reduce_init; 2120 arr[i].reduce_fini = data[i].reduce_fini; 2121 __kmp_assign_orig<T>(arr[i], data[i]); 2122 if (!arr[i].flags.lazy_priv) { 2123 // allocate cache-line aligned block and fill it with zeros 2124 arr[i].reduce_priv = __kmp_allocate(nth * size); 2125 arr[i].reduce_pend = (char *)(arr[i].reduce_priv) + nth * size; 2126 if (arr[i].reduce_init != NULL) { 2127 // initialize all thread-specific items 2128 for (int j = 0; j < nth; ++j) { 2129 __kmp_call_init<T>(arr[i], j * size); 2130 } 2131 } 2132 } else { 2133 // only allocate space for pointers now, 2134 // objects will be lazily allocated/initialized if/when requested 2135 // note that __kmp_allocate zeroes the allocated memory 2136 arr[i].reduce_priv = __kmp_allocate(nth * sizeof(void *)); 2137 } 2138 } 2139 tg->reduce_data = (void *)arr; 2140 tg->reduce_num_data = num; 2141 return (void *)tg; 2142 } 2143 2144 /*! 2145 @ingroup TASKING 2146 @param gtid Global thread ID 2147 @param num Number of data items to reduce 2148 @param data Array of data for reduction 2149 @return The taskgroup identifier 2150 2151 Initialize task reduction for the taskgroup. 2152 2153 Note: this entry supposes the optional compiler-generated initializer routine 2154 has single parameter - pointer to object to be initialized. That means 2155 the reduction either does not use omp_orig object, or the omp_orig is accessible 2156 without help of the runtime library. 2157 */ 2158 void *__kmpc_task_reduction_init(int gtid, int num, void *data) { 2159 return __kmp_task_reduction_init(gtid, num, (kmp_task_red_input_t *)data); 2160 } 2161 2162 /*! 2163 @ingroup TASKING 2164 @param gtid Global thread ID 2165 @param num Number of data items to reduce 2166 @param data Array of data for reduction 2167 @return The taskgroup identifier 2168 2169 Initialize task reduction for the taskgroup. 2170 2171 Note: this entry supposes the optional compiler-generated initializer routine 2172 has two parameters, pointer to object to be initialized and pointer to omp_orig 2173 */ 2174 void *__kmpc_taskred_init(int gtid, int num, void *data) { 2175 return __kmp_task_reduction_init(gtid, num, (kmp_taskred_input_t *)data); 2176 } 2177 2178 // Copy task reduction data (except for shared pointers). 2179 template <typename T> 2180 void __kmp_task_reduction_init_copy(kmp_info_t *thr, int num, T *data, 2181 kmp_taskgroup_t *tg, void *reduce_data) { 2182 kmp_taskred_data_t *arr; 2183 KA_TRACE(20, ("__kmp_task_reduction_init_copy: Th %p, init taskgroup %p," 2184 " from data %p\n", 2185 thr, tg, reduce_data)); 2186 arr = (kmp_taskred_data_t *)__kmp_thread_malloc( 2187 thr, num * sizeof(kmp_taskred_data_t)); 2188 // threads will share private copies, thunk routines, sizes, flags, etc.: 2189 KMP_MEMCPY(arr, reduce_data, num * sizeof(kmp_taskred_data_t)); 2190 for (int i = 0; i < num; ++i) { 2191 arr[i].reduce_shar = data[i].reduce_shar; // init unique shared pointers 2192 } 2193 tg->reduce_data = (void *)arr; 2194 tg->reduce_num_data = num; 2195 } 2196 2197 /*! 2198 @ingroup TASKING 2199 @param gtid Global thread ID 2200 @param tskgrp The taskgroup ID (optional) 2201 @param data Shared location of the item 2202 @return The pointer to per-thread data 2203 2204 Get thread-specific location of data item 2205 */ 2206 void *__kmpc_task_reduction_get_th_data(int gtid, void *tskgrp, void *data) { 2207 kmp_info_t *thread = __kmp_threads[gtid]; 2208 kmp_int32 nth = thread->th.th_team_nproc; 2209 if (nth == 1) 2210 return data; // nothing to do 2211 2212 kmp_taskgroup_t *tg = (kmp_taskgroup_t *)tskgrp; 2213 if (tg == NULL) 2214 tg = thread->th.th_current_task->td_taskgroup; 2215 KMP_ASSERT(tg != NULL); 2216 kmp_taskred_data_t *arr = (kmp_taskred_data_t *)(tg->reduce_data); 2217 kmp_int32 num = tg->reduce_num_data; 2218 kmp_int32 tid = thread->th.th_info.ds.ds_tid; 2219 2220 KMP_ASSERT(data != NULL); 2221 while (tg != NULL) { 2222 for (int i = 0; i < num; ++i) { 2223 if (!arr[i].flags.lazy_priv) { 2224 if (data == arr[i].reduce_shar || 2225 (data >= arr[i].reduce_priv && data < arr[i].reduce_pend)) 2226 return (char *)(arr[i].reduce_priv) + tid * arr[i].reduce_size; 2227 } else { 2228 // check shared location first 2229 void **p_priv = (void **)(arr[i].reduce_priv); 2230 if (data == arr[i].reduce_shar) 2231 goto found; 2232 // check if we get some thread specific location as parameter 2233 for (int j = 0; j < nth; ++j) 2234 if (data == p_priv[j]) 2235 goto found; 2236 continue; // not found, continue search 2237 found: 2238 if (p_priv[tid] == NULL) { 2239 // allocate thread specific object lazily 2240 p_priv[tid] = __kmp_allocate(arr[i].reduce_size); 2241 if (arr[i].reduce_init != NULL) { 2242 if (arr[i].reduce_orig != NULL) { // new interface 2243 ((void (*)(void *, void *))arr[i].reduce_init)( 2244 p_priv[tid], arr[i].reduce_orig); 2245 } else { // old interface (single parameter) 2246 ((void (*)(void *))arr[i].reduce_init)(p_priv[tid]); 2247 } 2248 } 2249 } 2250 return p_priv[tid]; 2251 } 2252 } 2253 tg = tg->parent; 2254 arr = (kmp_taskred_data_t *)(tg->reduce_data); 2255 num = tg->reduce_num_data; 2256 } 2257 KMP_ASSERT2(0, "Unknown task reduction item"); 2258 return NULL; // ERROR, this line never executed 2259 } 2260 2261 // Finalize task reduction. 2262 // Called from __kmpc_end_taskgroup() 2263 static void __kmp_task_reduction_fini(kmp_info_t *th, kmp_taskgroup_t *tg) { 2264 kmp_int32 nth = th->th.th_team_nproc; 2265 KMP_DEBUG_ASSERT(nth > 1); // should not be called if nth == 1 2266 kmp_taskred_data_t *arr = (kmp_taskred_data_t *)tg->reduce_data; 2267 kmp_int32 num = tg->reduce_num_data; 2268 for (int i = 0; i < num; ++i) { 2269 void *sh_data = arr[i].reduce_shar; 2270 void (*f_fini)(void *) = (void (*)(void *))(arr[i].reduce_fini); 2271 void (*f_comb)(void *, void *) = 2272 (void (*)(void *, void *))(arr[i].reduce_comb); 2273 if (!arr[i].flags.lazy_priv) { 2274 void *pr_data = arr[i].reduce_priv; 2275 size_t size = arr[i].reduce_size; 2276 for (int j = 0; j < nth; ++j) { 2277 void *priv_data = (char *)pr_data + j * size; 2278 f_comb(sh_data, priv_data); // combine results 2279 if (f_fini) 2280 f_fini(priv_data); // finalize if needed 2281 } 2282 } else { 2283 void **pr_data = (void **)(arr[i].reduce_priv); 2284 for (int j = 0; j < nth; ++j) { 2285 if (pr_data[j] != NULL) { 2286 f_comb(sh_data, pr_data[j]); // combine results 2287 if (f_fini) 2288 f_fini(pr_data[j]); // finalize if needed 2289 __kmp_free(pr_data[j]); 2290 } 2291 } 2292 } 2293 __kmp_free(arr[i].reduce_priv); 2294 } 2295 __kmp_thread_free(th, arr); 2296 tg->reduce_data = NULL; 2297 tg->reduce_num_data = 0; 2298 } 2299 2300 // Cleanup task reduction data for parallel or worksharing, 2301 // do not touch task private data other threads still working with. 2302 // Called from __kmpc_end_taskgroup() 2303 static void __kmp_task_reduction_clean(kmp_info_t *th, kmp_taskgroup_t *tg) { 2304 __kmp_thread_free(th, tg->reduce_data); 2305 tg->reduce_data = NULL; 2306 tg->reduce_num_data = 0; 2307 } 2308 2309 template <typename T> 2310 void *__kmp_task_reduction_modifier_init(ident_t *loc, int gtid, int is_ws, 2311 int num, T *data) { 2312 kmp_info_t *thr = __kmp_threads[gtid]; 2313 kmp_int32 nth = thr->th.th_team_nproc; 2314 __kmpc_taskgroup(loc, gtid); // form new taskgroup first 2315 if (nth == 1) { 2316 KA_TRACE(10, 2317 ("__kmpc_reduction_modifier_init: T#%d, tg %p, exiting nth=1\n", 2318 gtid, thr->th.th_current_task->td_taskgroup)); 2319 return (void *)thr->th.th_current_task->td_taskgroup; 2320 } 2321 kmp_team_t *team = thr->th.th_team; 2322 void *reduce_data; 2323 kmp_taskgroup_t *tg; 2324 reduce_data = KMP_ATOMIC_LD_RLX(&team->t.t_tg_reduce_data[is_ws]); 2325 if (reduce_data == NULL && 2326 __kmp_atomic_compare_store(&team->t.t_tg_reduce_data[is_ws], reduce_data, 2327 (void *)1)) { 2328 // single thread enters this block to initialize common reduction data 2329 KMP_DEBUG_ASSERT(reduce_data == NULL); 2330 // first initialize own data, then make a copy other threads can use 2331 tg = (kmp_taskgroup_t *)__kmp_task_reduction_init<T>(gtid, num, data); 2332 reduce_data = __kmp_thread_malloc(thr, num * sizeof(kmp_taskred_data_t)); 2333 KMP_MEMCPY(reduce_data, tg->reduce_data, num * sizeof(kmp_taskred_data_t)); 2334 // fini counters should be 0 at this point 2335 KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&team->t.t_tg_fini_counter[0]) == 0); 2336 KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&team->t.t_tg_fini_counter[1]) == 0); 2337 KMP_ATOMIC_ST_REL(&team->t.t_tg_reduce_data[is_ws], reduce_data); 2338 } else { 2339 while ( 2340 (reduce_data = KMP_ATOMIC_LD_ACQ(&team->t.t_tg_reduce_data[is_ws])) == 2341 (void *)1) { // wait for task reduction initialization 2342 KMP_CPU_PAUSE(); 2343 } 2344 KMP_DEBUG_ASSERT(reduce_data > (void *)1); // should be valid pointer here 2345 tg = thr->th.th_current_task->td_taskgroup; 2346 __kmp_task_reduction_init_copy<T>(thr, num, data, tg, reduce_data); 2347 } 2348 return tg; 2349 } 2350 2351 /*! 2352 @ingroup TASKING 2353 @param loc Source location info 2354 @param gtid Global thread ID 2355 @param is_ws Is 1 if the reduction is for worksharing, 0 otherwise 2356 @param num Number of data items to reduce 2357 @param data Array of data for reduction 2358 @return The taskgroup identifier 2359 2360 Initialize task reduction for a parallel or worksharing. 2361 2362 Note: this entry supposes the optional compiler-generated initializer routine 2363 has single parameter - pointer to object to be initialized. That means 2364 the reduction either does not use omp_orig object, or the omp_orig is accessible 2365 without help of the runtime library. 2366 */ 2367 void *__kmpc_task_reduction_modifier_init(ident_t *loc, int gtid, int is_ws, 2368 int num, void *data) { 2369 return __kmp_task_reduction_modifier_init(loc, gtid, is_ws, num, 2370 (kmp_task_red_input_t *)data); 2371 } 2372 2373 /*! 2374 @ingroup TASKING 2375 @param loc Source location info 2376 @param gtid Global thread ID 2377 @param is_ws Is 1 if the reduction is for worksharing, 0 otherwise 2378 @param num Number of data items to reduce 2379 @param data Array of data for reduction 2380 @return The taskgroup identifier 2381 2382 Initialize task reduction for a parallel or worksharing. 2383 2384 Note: this entry supposes the optional compiler-generated initializer routine 2385 has two parameters, pointer to object to be initialized and pointer to omp_orig 2386 */ 2387 void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int is_ws, int num, 2388 void *data) { 2389 return __kmp_task_reduction_modifier_init(loc, gtid, is_ws, num, 2390 (kmp_taskred_input_t *)data); 2391 } 2392 2393 /*! 2394 @ingroup TASKING 2395 @param loc Source location info 2396 @param gtid Global thread ID 2397 @param is_ws Is 1 if the reduction is for worksharing, 0 otherwise 2398 2399 Finalize task reduction for a parallel or worksharing. 2400 */ 2401 void __kmpc_task_reduction_modifier_fini(ident_t *loc, int gtid, int is_ws) { 2402 __kmpc_end_taskgroup(loc, gtid); 2403 } 2404 2405 // __kmpc_taskgroup: Start a new taskgroup 2406 void __kmpc_taskgroup(ident_t *loc, int gtid) { 2407 kmp_info_t *thread = __kmp_threads[gtid]; 2408 kmp_taskdata_t *taskdata = thread->th.th_current_task; 2409 kmp_taskgroup_t *tg_new = 2410 (kmp_taskgroup_t *)__kmp_thread_malloc(thread, sizeof(kmp_taskgroup_t)); 2411 KA_TRACE(10, ("__kmpc_taskgroup: T#%d loc=%p group=%p\n", gtid, loc, tg_new)); 2412 KMP_ATOMIC_ST_RLX(&tg_new->count, 0); 2413 KMP_ATOMIC_ST_RLX(&tg_new->cancel_request, cancel_noreq); 2414 tg_new->parent = taskdata->td_taskgroup; 2415 tg_new->reduce_data = NULL; 2416 tg_new->reduce_num_data = 0; 2417 taskdata->td_taskgroup = tg_new; 2418 2419 #if OMPT_SUPPORT && OMPT_OPTIONAL 2420 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)) { 2421 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); 2422 if (!codeptr) 2423 codeptr = OMPT_GET_RETURN_ADDRESS(0); 2424 kmp_team_t *team = thread->th.th_team; 2425 ompt_data_t my_task_data = taskdata->ompt_task_info.task_data; 2426 // FIXME: I think this is wrong for lwt! 2427 ompt_data_t my_parallel_data = team->t.ompt_team_info.parallel_data; 2428 2429 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 2430 ompt_sync_region_taskgroup, ompt_scope_begin, &(my_parallel_data), 2431 &(my_task_data), codeptr); 2432 } 2433 #endif 2434 } 2435 2436 // __kmpc_end_taskgroup: Wait until all tasks generated by the current task 2437 // and its descendants are complete 2438 void __kmpc_end_taskgroup(ident_t *loc, int gtid) { 2439 kmp_info_t *thread = __kmp_threads[gtid]; 2440 kmp_taskdata_t *taskdata = thread->th.th_current_task; 2441 kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup; 2442 int thread_finished = FALSE; 2443 2444 #if OMPT_SUPPORT && OMPT_OPTIONAL 2445 kmp_team_t *team; 2446 ompt_data_t my_task_data; 2447 ompt_data_t my_parallel_data; 2448 void *codeptr; 2449 if (UNLIKELY(ompt_enabled.enabled)) { 2450 team = thread->th.th_team; 2451 my_task_data = taskdata->ompt_task_info.task_data; 2452 // FIXME: I think this is wrong for lwt! 2453 my_parallel_data = team->t.ompt_team_info.parallel_data; 2454 codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); 2455 if (!codeptr) 2456 codeptr = OMPT_GET_RETURN_ADDRESS(0); 2457 } 2458 #endif 2459 2460 KA_TRACE(10, ("__kmpc_end_taskgroup(enter): T#%d loc=%p\n", gtid, loc)); 2461 KMP_DEBUG_ASSERT(taskgroup != NULL); 2462 KMP_SET_THREAD_STATE_BLOCK(TASKGROUP); 2463 2464 if (__kmp_tasking_mode != tskm_immediate_exec) { 2465 // mark task as waiting not on a barrier 2466 taskdata->td_taskwait_counter += 1; 2467 taskdata->td_taskwait_ident = loc; 2468 taskdata->td_taskwait_thread = gtid + 1; 2469 #if USE_ITT_BUILD 2470 // For ITT the taskgroup wait is similar to taskwait until we need to 2471 // distinguish them 2472 void *itt_sync_obj = __kmp_itt_taskwait_object(gtid); 2473 if (itt_sync_obj != NULL) 2474 __kmp_itt_taskwait_starting(gtid, itt_sync_obj); 2475 #endif /* USE_ITT_BUILD */ 2476 2477 #if OMPT_SUPPORT && OMPT_OPTIONAL 2478 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region_wait)) { 2479 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 2480 ompt_sync_region_taskgroup, ompt_scope_begin, &(my_parallel_data), 2481 &(my_task_data), codeptr); 2482 } 2483 #endif 2484 2485 if (!taskdata->td_flags.team_serial || 2486 (thread->th.th_task_team != NULL && 2487 thread->th.th_task_team->tt.tt_found_proxy_tasks)) { 2488 kmp_flag_32 flag(RCAST(std::atomic<kmp_uint32> *, &(taskgroup->count)), 2489 0U); 2490 while (KMP_ATOMIC_LD_ACQ(&taskgroup->count) != 0) { 2491 flag.execute_tasks(thread, gtid, FALSE, 2492 &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), 2493 __kmp_task_stealing_constraint); 2494 } 2495 } 2496 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; // end waiting 2497 2498 #if OMPT_SUPPORT && OMPT_OPTIONAL 2499 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region_wait)) { 2500 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 2501 ompt_sync_region_taskgroup, ompt_scope_end, &(my_parallel_data), 2502 &(my_task_data), codeptr); 2503 } 2504 #endif 2505 2506 #if USE_ITT_BUILD 2507 if (itt_sync_obj != NULL) 2508 __kmp_itt_taskwait_finished(gtid, itt_sync_obj); 2509 #endif /* USE_ITT_BUILD */ 2510 } 2511 KMP_DEBUG_ASSERT(taskgroup->count == 0); 2512 2513 if (taskgroup->reduce_data != NULL) { // need to reduce? 2514 int cnt; 2515 void *reduce_data; 2516 kmp_team_t *t = thread->th.th_team; 2517 kmp_taskred_data_t *arr = (kmp_taskred_data_t *)taskgroup->reduce_data; 2518 // check if <priv> data of the first reduction variable shared for the team 2519 void *priv0 = arr[0].reduce_priv; 2520 if ((reduce_data = KMP_ATOMIC_LD_ACQ(&t->t.t_tg_reduce_data[0])) != NULL && 2521 ((kmp_taskred_data_t *)reduce_data)[0].reduce_priv == priv0) { 2522 // finishing task reduction on parallel 2523 cnt = KMP_ATOMIC_INC(&t->t.t_tg_fini_counter[0]); 2524 if (cnt == thread->th.th_team_nproc - 1) { 2525 // we are the last thread passing __kmpc_reduction_modifier_fini() 2526 // finalize task reduction: 2527 __kmp_task_reduction_fini(thread, taskgroup); 2528 // cleanup fields in the team structure: 2529 // TODO: is relaxed store enough here (whole barrier should follow)? 2530 __kmp_thread_free(thread, reduce_data); 2531 KMP_ATOMIC_ST_REL(&t->t.t_tg_reduce_data[0], NULL); 2532 KMP_ATOMIC_ST_REL(&t->t.t_tg_fini_counter[0], 0); 2533 } else { 2534 // we are not the last thread passing __kmpc_reduction_modifier_fini(), 2535 // so do not finalize reduction, just clean own copy of the data 2536 __kmp_task_reduction_clean(thread, taskgroup); 2537 } 2538 } else if ((reduce_data = KMP_ATOMIC_LD_ACQ(&t->t.t_tg_reduce_data[1])) != 2539 NULL && 2540 ((kmp_taskred_data_t *)reduce_data)[0].reduce_priv == priv0) { 2541 // finishing task reduction on worksharing 2542 cnt = KMP_ATOMIC_INC(&t->t.t_tg_fini_counter[1]); 2543 if (cnt == thread->th.th_team_nproc - 1) { 2544 // we are the last thread passing __kmpc_reduction_modifier_fini() 2545 __kmp_task_reduction_fini(thread, taskgroup); 2546 // cleanup fields in team structure: 2547 // TODO: is relaxed store enough here (whole barrier should follow)? 2548 __kmp_thread_free(thread, reduce_data); 2549 KMP_ATOMIC_ST_REL(&t->t.t_tg_reduce_data[1], NULL); 2550 KMP_ATOMIC_ST_REL(&t->t.t_tg_fini_counter[1], 0); 2551 } else { 2552 // we are not the last thread passing __kmpc_reduction_modifier_fini(), 2553 // so do not finalize reduction, just clean own copy of the data 2554 __kmp_task_reduction_clean(thread, taskgroup); 2555 } 2556 } else { 2557 // finishing task reduction on taskgroup 2558 __kmp_task_reduction_fini(thread, taskgroup); 2559 } 2560 } 2561 // Restore parent taskgroup for the current task 2562 taskdata->td_taskgroup = taskgroup->parent; 2563 __kmp_thread_free(thread, taskgroup); 2564 2565 KA_TRACE(10, ("__kmpc_end_taskgroup(exit): T#%d task %p finished waiting\n", 2566 gtid, taskdata)); 2567 ANNOTATE_HAPPENS_AFTER(taskdata); 2568 2569 #if OMPT_SUPPORT && OMPT_OPTIONAL 2570 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)) { 2571 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 2572 ompt_sync_region_taskgroup, ompt_scope_end, &(my_parallel_data), 2573 &(my_task_data), codeptr); 2574 } 2575 #endif 2576 } 2577 2578 // __kmp_remove_my_task: remove a task from my own deque 2579 static kmp_task_t *__kmp_remove_my_task(kmp_info_t *thread, kmp_int32 gtid, 2580 kmp_task_team_t *task_team, 2581 kmp_int32 is_constrained) { 2582 kmp_task_t *task; 2583 kmp_taskdata_t *taskdata; 2584 kmp_thread_data_t *thread_data; 2585 kmp_uint32 tail; 2586 2587 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 2588 KMP_DEBUG_ASSERT(task_team->tt.tt_threads_data != 2589 NULL); // Caller should check this condition 2590 2591 thread_data = &task_team->tt.tt_threads_data[__kmp_tid_from_gtid(gtid)]; 2592 2593 KA_TRACE(10, ("__kmp_remove_my_task(enter): T#%d ntasks=%d head=%u tail=%u\n", 2594 gtid, thread_data->td.td_deque_ntasks, 2595 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 2596 2597 if (TCR_4(thread_data->td.td_deque_ntasks) == 0) { 2598 KA_TRACE(10, 2599 ("__kmp_remove_my_task(exit #1): T#%d No tasks to remove: " 2600 "ntasks=%d head=%u tail=%u\n", 2601 gtid, thread_data->td.td_deque_ntasks, 2602 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 2603 return NULL; 2604 } 2605 2606 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 2607 2608 if (TCR_4(thread_data->td.td_deque_ntasks) == 0) { 2609 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 2610 KA_TRACE(10, 2611 ("__kmp_remove_my_task(exit #2): T#%d No tasks to remove: " 2612 "ntasks=%d head=%u tail=%u\n", 2613 gtid, thread_data->td.td_deque_ntasks, 2614 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 2615 return NULL; 2616 } 2617 2618 tail = (thread_data->td.td_deque_tail - 1) & 2619 TASK_DEQUE_MASK(thread_data->td); // Wrap index. 2620 taskdata = thread_data->td.td_deque[tail]; 2621 2622 if (!__kmp_task_is_allowed(gtid, is_constrained, taskdata, 2623 thread->th.th_current_task)) { 2624 // The TSC does not allow to steal victim task 2625 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 2626 KA_TRACE(10, 2627 ("__kmp_remove_my_task(exit #3): T#%d TSC blocks tail task: " 2628 "ntasks=%d head=%u tail=%u\n", 2629 gtid, thread_data->td.td_deque_ntasks, 2630 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 2631 return NULL; 2632 } 2633 2634 thread_data->td.td_deque_tail = tail; 2635 TCW_4(thread_data->td.td_deque_ntasks, thread_data->td.td_deque_ntasks - 1); 2636 2637 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 2638 2639 KA_TRACE(10, ("__kmp_remove_my_task(exit #4): T#%d task %p removed: " 2640 "ntasks=%d head=%u tail=%u\n", 2641 gtid, taskdata, thread_data->td.td_deque_ntasks, 2642 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 2643 2644 task = KMP_TASKDATA_TO_TASK(taskdata); 2645 return task; 2646 } 2647 2648 // __kmp_steal_task: remove a task from another thread's deque 2649 // Assume that calling thread has already checked existence of 2650 // task_team thread_data before calling this routine. 2651 static kmp_task_t *__kmp_steal_task(kmp_info_t *victim_thr, kmp_int32 gtid, 2652 kmp_task_team_t *task_team, 2653 std::atomic<kmp_int32> *unfinished_threads, 2654 int *thread_finished, 2655 kmp_int32 is_constrained) { 2656 kmp_task_t *task; 2657 kmp_taskdata_t *taskdata; 2658 kmp_taskdata_t *current; 2659 kmp_thread_data_t *victim_td, *threads_data; 2660 kmp_int32 target; 2661 kmp_int32 victim_tid; 2662 2663 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 2664 2665 threads_data = task_team->tt.tt_threads_data; 2666 KMP_DEBUG_ASSERT(threads_data != NULL); // Caller should check this condition 2667 2668 victim_tid = victim_thr->th.th_info.ds.ds_tid; 2669 victim_td = &threads_data[victim_tid]; 2670 2671 KA_TRACE(10, ("__kmp_steal_task(enter): T#%d try to steal from T#%d: " 2672 "task_team=%p ntasks=%d head=%u tail=%u\n", 2673 gtid, __kmp_gtid_from_thread(victim_thr), task_team, 2674 victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head, 2675 victim_td->td.td_deque_tail)); 2676 2677 if (TCR_4(victim_td->td.td_deque_ntasks) == 0) { 2678 KA_TRACE(10, ("__kmp_steal_task(exit #1): T#%d could not steal from T#%d: " 2679 "task_team=%p ntasks=%d head=%u tail=%u\n", 2680 gtid, __kmp_gtid_from_thread(victim_thr), task_team, 2681 victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head, 2682 victim_td->td.td_deque_tail)); 2683 return NULL; 2684 } 2685 2686 __kmp_acquire_bootstrap_lock(&victim_td->td.td_deque_lock); 2687 2688 int ntasks = TCR_4(victim_td->td.td_deque_ntasks); 2689 // Check again after we acquire the lock 2690 if (ntasks == 0) { 2691 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock); 2692 KA_TRACE(10, ("__kmp_steal_task(exit #2): T#%d could not steal from T#%d: " 2693 "task_team=%p ntasks=%d head=%u tail=%u\n", 2694 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks, 2695 victim_td->td.td_deque_head, victim_td->td.td_deque_tail)); 2696 return NULL; 2697 } 2698 2699 KMP_DEBUG_ASSERT(victim_td->td.td_deque != NULL); 2700 current = __kmp_threads[gtid]->th.th_current_task; 2701 taskdata = victim_td->td.td_deque[victim_td->td.td_deque_head]; 2702 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) { 2703 // Bump head pointer and Wrap. 2704 victim_td->td.td_deque_head = 2705 (victim_td->td.td_deque_head + 1) & TASK_DEQUE_MASK(victim_td->td); 2706 } else { 2707 if (!task_team->tt.tt_untied_task_encountered) { 2708 // The TSC does not allow to steal victim task 2709 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock); 2710 KA_TRACE(10, ("__kmp_steal_task(exit #3): T#%d could not steal from " 2711 "T#%d: task_team=%p ntasks=%d head=%u tail=%u\n", 2712 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks, 2713 victim_td->td.td_deque_head, victim_td->td.td_deque_tail)); 2714 return NULL; 2715 } 2716 int i; 2717 // walk through victim's deque trying to steal any task 2718 target = victim_td->td.td_deque_head; 2719 taskdata = NULL; 2720 for (i = 1; i < ntasks; ++i) { 2721 target = (target + 1) & TASK_DEQUE_MASK(victim_td->td); 2722 taskdata = victim_td->td.td_deque[target]; 2723 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) { 2724 break; // found victim task 2725 } else { 2726 taskdata = NULL; 2727 } 2728 } 2729 if (taskdata == NULL) { 2730 // No appropriate candidate to steal found 2731 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock); 2732 KA_TRACE(10, ("__kmp_steal_task(exit #4): T#%d could not steal from " 2733 "T#%d: task_team=%p ntasks=%d head=%u tail=%u\n", 2734 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks, 2735 victim_td->td.td_deque_head, victim_td->td.td_deque_tail)); 2736 return NULL; 2737 } 2738 int prev = target; 2739 for (i = i + 1; i < ntasks; ++i) { 2740 // shift remaining tasks in the deque left by 1 2741 target = (target + 1) & TASK_DEQUE_MASK(victim_td->td); 2742 victim_td->td.td_deque[prev] = victim_td->td.td_deque[target]; 2743 prev = target; 2744 } 2745 KMP_DEBUG_ASSERT( 2746 victim_td->td.td_deque_tail == 2747 (kmp_uint32)((target + 1) & TASK_DEQUE_MASK(victim_td->td))); 2748 victim_td->td.td_deque_tail = target; // tail -= 1 (wrapped)) 2749 } 2750 if (*thread_finished) { 2751 // We need to un-mark this victim as a finished victim. This must be done 2752 // before releasing the lock, or else other threads (starting with the 2753 // master victim) might be prematurely released from the barrier!!! 2754 kmp_int32 count; 2755 2756 count = KMP_ATOMIC_INC(unfinished_threads); 2757 2758 KA_TRACE( 2759 20, 2760 ("__kmp_steal_task: T#%d inc unfinished_threads to %d: task_team=%p\n", 2761 gtid, count + 1, task_team)); 2762 2763 *thread_finished = FALSE; 2764 } 2765 TCW_4(victim_td->td.td_deque_ntasks, ntasks - 1); 2766 2767 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock); 2768 2769 KMP_COUNT_BLOCK(TASK_stolen); 2770 KA_TRACE(10, 2771 ("__kmp_steal_task(exit #5): T#%d stole task %p from T#%d: " 2772 "task_team=%p ntasks=%d head=%u tail=%u\n", 2773 gtid, taskdata, __kmp_gtid_from_thread(victim_thr), task_team, 2774 ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail)); 2775 2776 task = KMP_TASKDATA_TO_TASK(taskdata); 2777 return task; 2778 } 2779 2780 // __kmp_execute_tasks_template: Choose and execute tasks until either the 2781 // condition is statisfied (return true) or there are none left (return false). 2782 // 2783 // final_spin is TRUE if this is the spin at the release barrier. 2784 // thread_finished indicates whether the thread is finished executing all 2785 // the tasks it has on its deque, and is at the release barrier. 2786 // spinner is the location on which to spin. 2787 // spinner == NULL means only execute a single task and return. 2788 // checker is the value to check to terminate the spin. 2789 template <class C> 2790 static inline int __kmp_execute_tasks_template( 2791 kmp_info_t *thread, kmp_int32 gtid, C *flag, int final_spin, 2792 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 2793 kmp_int32 is_constrained) { 2794 kmp_task_team_t *task_team = thread->th.th_task_team; 2795 kmp_thread_data_t *threads_data; 2796 kmp_task_t *task; 2797 kmp_info_t *other_thread; 2798 kmp_taskdata_t *current_task = thread->th.th_current_task; 2799 std::atomic<kmp_int32> *unfinished_threads; 2800 kmp_int32 nthreads, victim_tid = -2, use_own_tasks = 1, new_victim = 0, 2801 tid = thread->th.th_info.ds.ds_tid; 2802 2803 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 2804 KMP_DEBUG_ASSERT(thread == __kmp_threads[gtid]); 2805 2806 if (task_team == NULL || current_task == NULL) 2807 return FALSE; 2808 2809 KA_TRACE(15, ("__kmp_execute_tasks_template(enter): T#%d final_spin=%d " 2810 "*thread_finished=%d\n", 2811 gtid, final_spin, *thread_finished)); 2812 2813 thread->th.th_reap_state = KMP_NOT_SAFE_TO_REAP; 2814 threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data); 2815 KMP_DEBUG_ASSERT(threads_data != NULL); 2816 2817 nthreads = task_team->tt.tt_nproc; 2818 unfinished_threads = &(task_team->tt.tt_unfinished_threads); 2819 KMP_DEBUG_ASSERT(nthreads > 1 || task_team->tt.tt_found_proxy_tasks); 2820 KMP_DEBUG_ASSERT(*unfinished_threads >= 0); 2821 2822 while (1) { // Outer loop keeps trying to find tasks in case of single thread 2823 // getting tasks from target constructs 2824 while (1) { // Inner loop to find a task and execute it 2825 task = NULL; 2826 if (use_own_tasks) { // check on own queue first 2827 task = __kmp_remove_my_task(thread, gtid, task_team, is_constrained); 2828 } 2829 if ((task == NULL) && (nthreads > 1)) { // Steal a task 2830 int asleep = 1; 2831 use_own_tasks = 0; 2832 // Try to steal from the last place I stole from successfully. 2833 if (victim_tid == -2) { // haven't stolen anything yet 2834 victim_tid = threads_data[tid].td.td_deque_last_stolen; 2835 if (victim_tid != 2836 -1) // if we have a last stolen from victim, get the thread 2837 other_thread = threads_data[victim_tid].td.td_thr; 2838 } 2839 if (victim_tid != -1) { // found last victim 2840 asleep = 0; 2841 } else if (!new_victim) { // no recent steals and we haven't already 2842 // used a new victim; select a random thread 2843 do { // Find a different thread to steal work from. 2844 // Pick a random thread. Initial plan was to cycle through all the 2845 // threads, and only return if we tried to steal from every thread, 2846 // and failed. Arch says that's not such a great idea. 2847 victim_tid = __kmp_get_random(thread) % (nthreads - 1); 2848 if (victim_tid >= tid) { 2849 ++victim_tid; // Adjusts random distribution to exclude self 2850 } 2851 // Found a potential victim 2852 other_thread = threads_data[victim_tid].td.td_thr; 2853 // There is a slight chance that __kmp_enable_tasking() did not wake 2854 // up all threads waiting at the barrier. If victim is sleeping, 2855 // then wake it up. Since we were going to pay the cache miss 2856 // penalty for referencing another thread's kmp_info_t struct 2857 // anyway, 2858 // the check shouldn't cost too much performance at this point. In 2859 // extra barrier mode, tasks do not sleep at the separate tasking 2860 // barrier, so this isn't a problem. 2861 asleep = 0; 2862 if ((__kmp_tasking_mode == tskm_task_teams) && 2863 (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) && 2864 (TCR_PTR(CCAST(void *, other_thread->th.th_sleep_loc)) != 2865 NULL)) { 2866 asleep = 1; 2867 __kmp_null_resume_wrapper(__kmp_gtid_from_thread(other_thread), 2868 other_thread->th.th_sleep_loc); 2869 // A sleeping thread should not have any tasks on it's queue. 2870 // There is a slight possibility that it resumes, steals a task 2871 // from another thread, which spawns more tasks, all in the time 2872 // that it takes this thread to check => don't write an assertion 2873 // that the victim's queue is empty. Try stealing from a 2874 // different thread. 2875 } 2876 } while (asleep); 2877 } 2878 2879 if (!asleep) { 2880 // We have a victim to try to steal from 2881 task = __kmp_steal_task(other_thread, gtid, task_team, 2882 unfinished_threads, thread_finished, 2883 is_constrained); 2884 } 2885 if (task != NULL) { // set last stolen to victim 2886 if (threads_data[tid].td.td_deque_last_stolen != victim_tid) { 2887 threads_data[tid].td.td_deque_last_stolen = victim_tid; 2888 // The pre-refactored code did not try more than 1 successful new 2889 // vicitm, unless the last one generated more local tasks; 2890 // new_victim keeps track of this 2891 new_victim = 1; 2892 } 2893 } else { // No tasks found; unset last_stolen 2894 KMP_CHECK_UPDATE(threads_data[tid].td.td_deque_last_stolen, -1); 2895 victim_tid = -2; // no successful victim found 2896 } 2897 } 2898 2899 if (task == NULL) // break out of tasking loop 2900 break; 2901 2902 // Found a task; execute it 2903 #if USE_ITT_BUILD && USE_ITT_NOTIFY 2904 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) { 2905 if (itt_sync_obj == NULL) { // we are at fork barrier where we could not 2906 // get the object reliably 2907 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier); 2908 } 2909 __kmp_itt_task_starting(itt_sync_obj); 2910 } 2911 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */ 2912 __kmp_invoke_task(gtid, task, current_task); 2913 #if USE_ITT_BUILD 2914 if (itt_sync_obj != NULL) 2915 __kmp_itt_task_finished(itt_sync_obj); 2916 #endif /* USE_ITT_BUILD */ 2917 // If this thread is only partway through the barrier and the condition is 2918 // met, then return now, so that the barrier gather/release pattern can 2919 // proceed. If this thread is in the last spin loop in the barrier, 2920 // waiting to be released, we know that the termination condition will not 2921 // be satisfied, so don't waste any cycles checking it. 2922 if (flag == NULL || (!final_spin && flag->done_check())) { 2923 KA_TRACE( 2924 15, 2925 ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n", 2926 gtid)); 2927 return TRUE; 2928 } 2929 if (thread->th.th_task_team == NULL) { 2930 break; 2931 } 2932 KMP_YIELD(__kmp_library == library_throughput); // Yield before next task 2933 // If execution of a stolen task results in more tasks being placed on our 2934 // run queue, reset use_own_tasks 2935 if (!use_own_tasks && TCR_4(threads_data[tid].td.td_deque_ntasks) != 0) { 2936 KA_TRACE(20, ("__kmp_execute_tasks_template: T#%d stolen task spawned " 2937 "other tasks, restart\n", 2938 gtid)); 2939 use_own_tasks = 1; 2940 new_victim = 0; 2941 } 2942 } 2943 2944 // The task source has been exhausted. If in final spin loop of barrier, 2945 // check if termination condition is satisfied. The work queue may be empty 2946 // but there might be proxy tasks still executing. 2947 if (final_spin && 2948 KMP_ATOMIC_LD_ACQ(¤t_task->td_incomplete_child_tasks) == 0) { 2949 // First, decrement the #unfinished threads, if that has not already been 2950 // done. This decrement might be to the spin location, and result in the 2951 // termination condition being satisfied. 2952 if (!*thread_finished) { 2953 kmp_int32 count; 2954 2955 count = KMP_ATOMIC_DEC(unfinished_threads) - 1; 2956 KA_TRACE(20, ("__kmp_execute_tasks_template: T#%d dec " 2957 "unfinished_threads to %d task_team=%p\n", 2958 gtid, count, task_team)); 2959 *thread_finished = TRUE; 2960 } 2961 2962 // It is now unsafe to reference thread->th.th_team !!! 2963 // Decrementing task_team->tt.tt_unfinished_threads can allow the master 2964 // thread to pass through the barrier, where it might reset each thread's 2965 // th.th_team field for the next parallel region. If we can steal more 2966 // work, we know that this has not happened yet. 2967 if (flag != NULL && flag->done_check()) { 2968 KA_TRACE( 2969 15, 2970 ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n", 2971 gtid)); 2972 return TRUE; 2973 } 2974 } 2975 2976 // If this thread's task team is NULL, master has recognized that there are 2977 // no more tasks; bail out 2978 if (thread->th.th_task_team == NULL) { 2979 KA_TRACE(15, 2980 ("__kmp_execute_tasks_template: T#%d no more tasks\n", gtid)); 2981 return FALSE; 2982 } 2983 2984 // We could be getting tasks from target constructs; if this is the only 2985 // thread, keep trying to execute tasks from own queue 2986 if (nthreads == 1) 2987 use_own_tasks = 1; 2988 else { 2989 KA_TRACE(15, 2990 ("__kmp_execute_tasks_template: T#%d can't find work\n", gtid)); 2991 return FALSE; 2992 } 2993 } 2994 } 2995 2996 int __kmp_execute_tasks_32( 2997 kmp_info_t *thread, kmp_int32 gtid, kmp_flag_32 *flag, int final_spin, 2998 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 2999 kmp_int32 is_constrained) { 3000 return __kmp_execute_tasks_template( 3001 thread, gtid, flag, final_spin, 3002 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained); 3003 } 3004 3005 int __kmp_execute_tasks_64( 3006 kmp_info_t *thread, kmp_int32 gtid, kmp_flag_64 *flag, int final_spin, 3007 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 3008 kmp_int32 is_constrained) { 3009 return __kmp_execute_tasks_template( 3010 thread, gtid, flag, final_spin, 3011 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained); 3012 } 3013 3014 int __kmp_execute_tasks_oncore( 3015 kmp_info_t *thread, kmp_int32 gtid, kmp_flag_oncore *flag, int final_spin, 3016 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 3017 kmp_int32 is_constrained) { 3018 return __kmp_execute_tasks_template( 3019 thread, gtid, flag, final_spin, 3020 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained); 3021 } 3022 3023 // __kmp_enable_tasking: Allocate task team and resume threads sleeping at the 3024 // next barrier so they can assist in executing enqueued tasks. 3025 // First thread in allocates the task team atomically. 3026 static void __kmp_enable_tasking(kmp_task_team_t *task_team, 3027 kmp_info_t *this_thr) { 3028 kmp_thread_data_t *threads_data; 3029 int nthreads, i, is_init_thread; 3030 3031 KA_TRACE(10, ("__kmp_enable_tasking(enter): T#%d\n", 3032 __kmp_gtid_from_thread(this_thr))); 3033 3034 KMP_DEBUG_ASSERT(task_team != NULL); 3035 KMP_DEBUG_ASSERT(this_thr->th.th_team != NULL); 3036 3037 nthreads = task_team->tt.tt_nproc; 3038 KMP_DEBUG_ASSERT(nthreads > 0); 3039 KMP_DEBUG_ASSERT(nthreads == this_thr->th.th_team->t.t_nproc); 3040 3041 // Allocate or increase the size of threads_data if necessary 3042 is_init_thread = __kmp_realloc_task_threads_data(this_thr, task_team); 3043 3044 if (!is_init_thread) { 3045 // Some other thread already set up the array. 3046 KA_TRACE( 3047 20, 3048 ("__kmp_enable_tasking(exit): T#%d: threads array already set up.\n", 3049 __kmp_gtid_from_thread(this_thr))); 3050 return; 3051 } 3052 threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data); 3053 KMP_DEBUG_ASSERT(threads_data != NULL); 3054 3055 if (__kmp_tasking_mode == tskm_task_teams && 3056 (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME)) { 3057 // Release any threads sleeping at the barrier, so that they can steal 3058 // tasks and execute them. In extra barrier mode, tasks do not sleep 3059 // at the separate tasking barrier, so this isn't a problem. 3060 for (i = 0; i < nthreads; i++) { 3061 volatile void *sleep_loc; 3062 kmp_info_t *thread = threads_data[i].td.td_thr; 3063 3064 if (i == this_thr->th.th_info.ds.ds_tid) { 3065 continue; 3066 } 3067 // Since we haven't locked the thread's suspend mutex lock at this 3068 // point, there is a small window where a thread might be putting 3069 // itself to sleep, but hasn't set the th_sleep_loc field yet. 3070 // To work around this, __kmp_execute_tasks_template() periodically checks 3071 // see if other threads are sleeping (using the same random mechanism that 3072 // is used for task stealing) and awakens them if they are. 3073 if ((sleep_loc = TCR_PTR(CCAST(void *, thread->th.th_sleep_loc))) != 3074 NULL) { 3075 KF_TRACE(50, ("__kmp_enable_tasking: T#%d waking up thread T#%d\n", 3076 __kmp_gtid_from_thread(this_thr), 3077 __kmp_gtid_from_thread(thread))); 3078 __kmp_null_resume_wrapper(__kmp_gtid_from_thread(thread), sleep_loc); 3079 } else { 3080 KF_TRACE(50, ("__kmp_enable_tasking: T#%d don't wake up thread T#%d\n", 3081 __kmp_gtid_from_thread(this_thr), 3082 __kmp_gtid_from_thread(thread))); 3083 } 3084 } 3085 } 3086 3087 KA_TRACE(10, ("__kmp_enable_tasking(exit): T#%d\n", 3088 __kmp_gtid_from_thread(this_thr))); 3089 } 3090 3091 /* // TODO: Check the comment consistency 3092 * Utility routines for "task teams". A task team (kmp_task_t) is kind of 3093 * like a shadow of the kmp_team_t data struct, with a different lifetime. 3094 * After a child * thread checks into a barrier and calls __kmp_release() from 3095 * the particular variant of __kmp_<barrier_kind>_barrier_gather(), it can no 3096 * longer assume that the kmp_team_t structure is intact (at any moment, the 3097 * master thread may exit the barrier code and free the team data structure, 3098 * and return the threads to the thread pool). 3099 * 3100 * This does not work with the tasking code, as the thread is still 3101 * expected to participate in the execution of any tasks that may have been 3102 * spawned my a member of the team, and the thread still needs access to all 3103 * to each thread in the team, so that it can steal work from it. 3104 * 3105 * Enter the existence of the kmp_task_team_t struct. It employs a reference 3106 * counting mechanism, and is allocated by the master thread before calling 3107 * __kmp_<barrier_kind>_release, and then is release by the last thread to 3108 * exit __kmp_<barrier_kind>_release at the next barrier. I.e. the lifetimes 3109 * of the kmp_task_team_t structs for consecutive barriers can overlap 3110 * (and will, unless the master thread is the last thread to exit the barrier 3111 * release phase, which is not typical). The existence of such a struct is 3112 * useful outside the context of tasking. 3113 * 3114 * We currently use the existence of the threads array as an indicator that 3115 * tasks were spawned since the last barrier. If the structure is to be 3116 * useful outside the context of tasking, then this will have to change, but 3117 * not setting the field minimizes the performance impact of tasking on 3118 * barriers, when no explicit tasks were spawned (pushed, actually). 3119 */ 3120 3121 static kmp_task_team_t *__kmp_free_task_teams = 3122 NULL; // Free list for task_team data structures 3123 // Lock for task team data structures 3124 kmp_bootstrap_lock_t __kmp_task_team_lock = 3125 KMP_BOOTSTRAP_LOCK_INITIALIZER(__kmp_task_team_lock); 3126 3127 // __kmp_alloc_task_deque: 3128 // Allocates a task deque for a particular thread, and initialize the necessary 3129 // data structures relating to the deque. This only happens once per thread 3130 // per task team since task teams are recycled. No lock is needed during 3131 // allocation since each thread allocates its own deque. 3132 static void __kmp_alloc_task_deque(kmp_info_t *thread, 3133 kmp_thread_data_t *thread_data) { 3134 __kmp_init_bootstrap_lock(&thread_data->td.td_deque_lock); 3135 KMP_DEBUG_ASSERT(thread_data->td.td_deque == NULL); 3136 3137 // Initialize last stolen task field to "none" 3138 thread_data->td.td_deque_last_stolen = -1; 3139 3140 KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) == 0); 3141 KMP_DEBUG_ASSERT(thread_data->td.td_deque_head == 0); 3142 KMP_DEBUG_ASSERT(thread_data->td.td_deque_tail == 0); 3143 3144 KE_TRACE( 3145 10, 3146 ("__kmp_alloc_task_deque: T#%d allocating deque[%d] for thread_data %p\n", 3147 __kmp_gtid_from_thread(thread), INITIAL_TASK_DEQUE_SIZE, thread_data)); 3148 // Allocate space for task deque, and zero the deque 3149 // Cannot use __kmp_thread_calloc() because threads not around for 3150 // kmp_reap_task_team( ). 3151 thread_data->td.td_deque = (kmp_taskdata_t **)__kmp_allocate( 3152 INITIAL_TASK_DEQUE_SIZE * sizeof(kmp_taskdata_t *)); 3153 thread_data->td.td_deque_size = INITIAL_TASK_DEQUE_SIZE; 3154 } 3155 3156 // __kmp_free_task_deque: 3157 // Deallocates a task deque for a particular thread. Happens at library 3158 // deallocation so don't need to reset all thread data fields. 3159 static void __kmp_free_task_deque(kmp_thread_data_t *thread_data) { 3160 if (thread_data->td.td_deque != NULL) { 3161 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 3162 TCW_4(thread_data->td.td_deque_ntasks, 0); 3163 __kmp_free(thread_data->td.td_deque); 3164 thread_data->td.td_deque = NULL; 3165 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 3166 } 3167 3168 #ifdef BUILD_TIED_TASK_STACK 3169 // GEH: Figure out what to do here for td_susp_tied_tasks 3170 if (thread_data->td.td_susp_tied_tasks.ts_entries != TASK_STACK_EMPTY) { 3171 __kmp_free_task_stack(__kmp_thread_from_gtid(gtid), thread_data); 3172 } 3173 #endif // BUILD_TIED_TASK_STACK 3174 } 3175 3176 // __kmp_realloc_task_threads_data: 3177 // Allocates a threads_data array for a task team, either by allocating an 3178 // initial array or enlarging an existing array. Only the first thread to get 3179 // the lock allocs or enlarges the array and re-initializes the array elements. 3180 // That thread returns "TRUE", the rest return "FALSE". 3181 // Assumes that the new array size is given by task_team -> tt.tt_nproc. 3182 // The current size is given by task_team -> tt.tt_max_threads. 3183 static int __kmp_realloc_task_threads_data(kmp_info_t *thread, 3184 kmp_task_team_t *task_team) { 3185 kmp_thread_data_t **threads_data_p; 3186 kmp_int32 nthreads, maxthreads; 3187 int is_init_thread = FALSE; 3188 3189 if (TCR_4(task_team->tt.tt_found_tasks)) { 3190 // Already reallocated and initialized. 3191 return FALSE; 3192 } 3193 3194 threads_data_p = &task_team->tt.tt_threads_data; 3195 nthreads = task_team->tt.tt_nproc; 3196 maxthreads = task_team->tt.tt_max_threads; 3197 3198 // All threads must lock when they encounter the first task of the implicit 3199 // task region to make sure threads_data fields are (re)initialized before 3200 // used. 3201 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_threads_lock); 3202 3203 if (!TCR_4(task_team->tt.tt_found_tasks)) { 3204 // first thread to enable tasking 3205 kmp_team_t *team = thread->th.th_team; 3206 int i; 3207 3208 is_init_thread = TRUE; 3209 if (maxthreads < nthreads) { 3210 3211 if (*threads_data_p != NULL) { 3212 kmp_thread_data_t *old_data = *threads_data_p; 3213 kmp_thread_data_t *new_data = NULL; 3214 3215 KE_TRACE( 3216 10, 3217 ("__kmp_realloc_task_threads_data: T#%d reallocating " 3218 "threads data for task_team %p, new_size = %d, old_size = %d\n", 3219 __kmp_gtid_from_thread(thread), task_team, nthreads, maxthreads)); 3220 // Reallocate threads_data to have more elements than current array 3221 // Cannot use __kmp_thread_realloc() because threads not around for 3222 // kmp_reap_task_team( ). Note all new array entries are initialized 3223 // to zero by __kmp_allocate(). 3224 new_data = (kmp_thread_data_t *)__kmp_allocate( 3225 nthreads * sizeof(kmp_thread_data_t)); 3226 // copy old data to new data 3227 KMP_MEMCPY_S((void *)new_data, nthreads * sizeof(kmp_thread_data_t), 3228 (void *)old_data, maxthreads * sizeof(kmp_thread_data_t)); 3229 3230 #ifdef BUILD_TIED_TASK_STACK 3231 // GEH: Figure out if this is the right thing to do 3232 for (i = maxthreads; i < nthreads; i++) { 3233 kmp_thread_data_t *thread_data = &(*threads_data_p)[i]; 3234 __kmp_init_task_stack(__kmp_gtid_from_thread(thread), thread_data); 3235 } 3236 #endif // BUILD_TIED_TASK_STACK 3237 // Install the new data and free the old data 3238 (*threads_data_p) = new_data; 3239 __kmp_free(old_data); 3240 } else { 3241 KE_TRACE(10, ("__kmp_realloc_task_threads_data: T#%d allocating " 3242 "threads data for task_team %p, size = %d\n", 3243 __kmp_gtid_from_thread(thread), task_team, nthreads)); 3244 // Make the initial allocate for threads_data array, and zero entries 3245 // Cannot use __kmp_thread_calloc() because threads not around for 3246 // kmp_reap_task_team( ). 3247 ANNOTATE_IGNORE_WRITES_BEGIN(); 3248 *threads_data_p = (kmp_thread_data_t *)__kmp_allocate( 3249 nthreads * sizeof(kmp_thread_data_t)); 3250 ANNOTATE_IGNORE_WRITES_END(); 3251 #ifdef BUILD_TIED_TASK_STACK 3252 // GEH: Figure out if this is the right thing to do 3253 for (i = 0; i < nthreads; i++) { 3254 kmp_thread_data_t *thread_data = &(*threads_data_p)[i]; 3255 __kmp_init_task_stack(__kmp_gtid_from_thread(thread), thread_data); 3256 } 3257 #endif // BUILD_TIED_TASK_STACK 3258 } 3259 task_team->tt.tt_max_threads = nthreads; 3260 } else { 3261 // If array has (more than) enough elements, go ahead and use it 3262 KMP_DEBUG_ASSERT(*threads_data_p != NULL); 3263 } 3264 3265 // initialize threads_data pointers back to thread_info structures 3266 for (i = 0; i < nthreads; i++) { 3267 kmp_thread_data_t *thread_data = &(*threads_data_p)[i]; 3268 thread_data->td.td_thr = team->t.t_threads[i]; 3269 3270 if (thread_data->td.td_deque_last_stolen >= nthreads) { 3271 // The last stolen field survives across teams / barrier, and the number 3272 // of threads may have changed. It's possible (likely?) that a new 3273 // parallel region will exhibit the same behavior as previous region. 3274 thread_data->td.td_deque_last_stolen = -1; 3275 } 3276 } 3277 3278 KMP_MB(); 3279 TCW_SYNC_4(task_team->tt.tt_found_tasks, TRUE); 3280 } 3281 3282 __kmp_release_bootstrap_lock(&task_team->tt.tt_threads_lock); 3283 return is_init_thread; 3284 } 3285 3286 // __kmp_free_task_threads_data: 3287 // Deallocates a threads_data array for a task team, including any attached 3288 // tasking deques. Only occurs at library shutdown. 3289 static void __kmp_free_task_threads_data(kmp_task_team_t *task_team) { 3290 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_threads_lock); 3291 if (task_team->tt.tt_threads_data != NULL) { 3292 int i; 3293 for (i = 0; i < task_team->tt.tt_max_threads; i++) { 3294 __kmp_free_task_deque(&task_team->tt.tt_threads_data[i]); 3295 } 3296 __kmp_free(task_team->tt.tt_threads_data); 3297 task_team->tt.tt_threads_data = NULL; 3298 } 3299 __kmp_release_bootstrap_lock(&task_team->tt.tt_threads_lock); 3300 } 3301 3302 // __kmp_allocate_task_team: 3303 // Allocates a task team associated with a specific team, taking it from 3304 // the global task team free list if possible. Also initializes data 3305 // structures. 3306 static kmp_task_team_t *__kmp_allocate_task_team(kmp_info_t *thread, 3307 kmp_team_t *team) { 3308 kmp_task_team_t *task_team = NULL; 3309 int nthreads; 3310 3311 KA_TRACE(20, ("__kmp_allocate_task_team: T#%d entering; team = %p\n", 3312 (thread ? __kmp_gtid_from_thread(thread) : -1), team)); 3313 3314 if (TCR_PTR(__kmp_free_task_teams) != NULL) { 3315 // Take a task team from the task team pool 3316 __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock); 3317 if (__kmp_free_task_teams != NULL) { 3318 task_team = __kmp_free_task_teams; 3319 TCW_PTR(__kmp_free_task_teams, task_team->tt.tt_next); 3320 task_team->tt.tt_next = NULL; 3321 } 3322 __kmp_release_bootstrap_lock(&__kmp_task_team_lock); 3323 } 3324 3325 if (task_team == NULL) { 3326 KE_TRACE(10, ("__kmp_allocate_task_team: T#%d allocating " 3327 "task team for team %p\n", 3328 __kmp_gtid_from_thread(thread), team)); 3329 // Allocate a new task team if one is not available. 3330 // Cannot use __kmp_thread_malloc() because threads not around for 3331 // kmp_reap_task_team( ). 3332 task_team = (kmp_task_team_t *)__kmp_allocate(sizeof(kmp_task_team_t)); 3333 __kmp_init_bootstrap_lock(&task_team->tt.tt_threads_lock); 3334 // AC: __kmp_allocate zeroes returned memory 3335 // task_team -> tt.tt_threads_data = NULL; 3336 // task_team -> tt.tt_max_threads = 0; 3337 // task_team -> tt.tt_next = NULL; 3338 } 3339 3340 TCW_4(task_team->tt.tt_found_tasks, FALSE); 3341 TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE); 3342 task_team->tt.tt_nproc = nthreads = team->t.t_nproc; 3343 3344 KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads, nthreads); 3345 TCW_4(task_team->tt.tt_active, TRUE); 3346 3347 KA_TRACE(20, ("__kmp_allocate_task_team: T#%d exiting; task_team = %p " 3348 "unfinished_threads init'd to %d\n", 3349 (thread ? __kmp_gtid_from_thread(thread) : -1), task_team, 3350 KMP_ATOMIC_LD_RLX(&task_team->tt.tt_unfinished_threads))); 3351 return task_team; 3352 } 3353 3354 // __kmp_free_task_team: 3355 // Frees the task team associated with a specific thread, and adds it 3356 // to the global task team free list. 3357 void __kmp_free_task_team(kmp_info_t *thread, kmp_task_team_t *task_team) { 3358 KA_TRACE(20, ("__kmp_free_task_team: T#%d task_team = %p\n", 3359 thread ? __kmp_gtid_from_thread(thread) : -1, task_team)); 3360 3361 // Put task team back on free list 3362 __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock); 3363 3364 KMP_DEBUG_ASSERT(task_team->tt.tt_next == NULL); 3365 task_team->tt.tt_next = __kmp_free_task_teams; 3366 TCW_PTR(__kmp_free_task_teams, task_team); 3367 3368 __kmp_release_bootstrap_lock(&__kmp_task_team_lock); 3369 } 3370 3371 // __kmp_reap_task_teams: 3372 // Free all the task teams on the task team free list. 3373 // Should only be done during library shutdown. 3374 // Cannot do anything that needs a thread structure or gtid since they are 3375 // already gone. 3376 void __kmp_reap_task_teams(void) { 3377 kmp_task_team_t *task_team; 3378 3379 if (TCR_PTR(__kmp_free_task_teams) != NULL) { 3380 // Free all task_teams on the free list 3381 __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock); 3382 while ((task_team = __kmp_free_task_teams) != NULL) { 3383 __kmp_free_task_teams = task_team->tt.tt_next; 3384 task_team->tt.tt_next = NULL; 3385 3386 // Free threads_data if necessary 3387 if (task_team->tt.tt_threads_data != NULL) { 3388 __kmp_free_task_threads_data(task_team); 3389 } 3390 __kmp_free(task_team); 3391 } 3392 __kmp_release_bootstrap_lock(&__kmp_task_team_lock); 3393 } 3394 } 3395 3396 // __kmp_wait_to_unref_task_teams: 3397 // Some threads could still be in the fork barrier release code, possibly 3398 // trying to steal tasks. Wait for each thread to unreference its task team. 3399 void __kmp_wait_to_unref_task_teams(void) { 3400 kmp_info_t *thread; 3401 kmp_uint32 spins; 3402 int done; 3403 3404 KMP_INIT_YIELD(spins); 3405 3406 for (;;) { 3407 done = TRUE; 3408 3409 // TODO: GEH - this may be is wrong because some sync would be necessary 3410 // in case threads are added to the pool during the traversal. Need to 3411 // verify that lock for thread pool is held when calling this routine. 3412 for (thread = CCAST(kmp_info_t *, __kmp_thread_pool); thread != NULL; 3413 thread = thread->th.th_next_pool) { 3414 #if KMP_OS_WINDOWS 3415 DWORD exit_val; 3416 #endif 3417 if (TCR_PTR(thread->th.th_task_team) == NULL) { 3418 KA_TRACE(10, ("__kmp_wait_to_unref_task_team: T#%d task_team == NULL\n", 3419 __kmp_gtid_from_thread(thread))); 3420 continue; 3421 } 3422 #if KMP_OS_WINDOWS 3423 // TODO: GEH - add this check for Linux* OS / OS X* as well? 3424 if (!__kmp_is_thread_alive(thread, &exit_val)) { 3425 thread->th.th_task_team = NULL; 3426 continue; 3427 } 3428 #endif 3429 3430 done = FALSE; // Because th_task_team pointer is not NULL for this thread 3431 3432 KA_TRACE(10, ("__kmp_wait_to_unref_task_team: Waiting for T#%d to " 3433 "unreference task_team\n", 3434 __kmp_gtid_from_thread(thread))); 3435 3436 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) { 3437 volatile void *sleep_loc; 3438 // If the thread is sleeping, awaken it. 3439 if ((sleep_loc = TCR_PTR(CCAST(void *, thread->th.th_sleep_loc))) != 3440 NULL) { 3441 KA_TRACE( 3442 10, 3443 ("__kmp_wait_to_unref_task_team: T#%d waking up thread T#%d\n", 3444 __kmp_gtid_from_thread(thread), __kmp_gtid_from_thread(thread))); 3445 __kmp_null_resume_wrapper(__kmp_gtid_from_thread(thread), sleep_loc); 3446 } 3447 } 3448 } 3449 if (done) { 3450 break; 3451 } 3452 3453 // If oversubscribed or have waited a bit, yield. 3454 KMP_YIELD_OVERSUB_ELSE_SPIN(spins); 3455 } 3456 } 3457 3458 // __kmp_task_team_setup: Create a task_team for the current team, but use 3459 // an already created, unused one if it already exists. 3460 void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team, int always) { 3461 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 3462 3463 // If this task_team hasn't been created yet, allocate it. It will be used in 3464 // the region after the next. 3465 // If it exists, it is the current task team and shouldn't be touched yet as 3466 // it may still be in use. 3467 if (team->t.t_task_team[this_thr->th.th_task_state] == NULL && 3468 (always || team->t.t_nproc > 1)) { 3469 team->t.t_task_team[this_thr->th.th_task_state] = 3470 __kmp_allocate_task_team(this_thr, team); 3471 KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d created new task_team %p " 3472 "for team %d at parity=%d\n", 3473 __kmp_gtid_from_thread(this_thr), 3474 team->t.t_task_team[this_thr->th.th_task_state], 3475 ((team != NULL) ? team->t.t_id : -1), 3476 this_thr->th.th_task_state)); 3477 } 3478 3479 // After threads exit the release, they will call sync, and then point to this 3480 // other task_team; make sure it is allocated and properly initialized. As 3481 // threads spin in the barrier release phase, they will continue to use the 3482 // previous task_team struct(above), until they receive the signal to stop 3483 // checking for tasks (they can't safely reference the kmp_team_t struct, 3484 // which could be reallocated by the master thread). No task teams are formed 3485 // for serialized teams. 3486 if (team->t.t_nproc > 1) { 3487 int other_team = 1 - this_thr->th.th_task_state; 3488 if (team->t.t_task_team[other_team] == NULL) { // setup other team as well 3489 team->t.t_task_team[other_team] = 3490 __kmp_allocate_task_team(this_thr, team); 3491 KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d created second new " 3492 "task_team %p for team %d at parity=%d\n", 3493 __kmp_gtid_from_thread(this_thr), 3494 team->t.t_task_team[other_team], 3495 ((team != NULL) ? team->t.t_id : -1), other_team)); 3496 } else { // Leave the old task team struct in place for the upcoming region; 3497 // adjust as needed 3498 kmp_task_team_t *task_team = team->t.t_task_team[other_team]; 3499 if (!task_team->tt.tt_active || 3500 team->t.t_nproc != task_team->tt.tt_nproc) { 3501 TCW_4(task_team->tt.tt_nproc, team->t.t_nproc); 3502 TCW_4(task_team->tt.tt_found_tasks, FALSE); 3503 TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE); 3504 KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads, 3505 team->t.t_nproc); 3506 TCW_4(task_team->tt.tt_active, TRUE); 3507 } 3508 // if team size has changed, the first thread to enable tasking will 3509 // realloc threads_data if necessary 3510 KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d reset next task_team " 3511 "%p for team %d at parity=%d\n", 3512 __kmp_gtid_from_thread(this_thr), 3513 team->t.t_task_team[other_team], 3514 ((team != NULL) ? team->t.t_id : -1), other_team)); 3515 } 3516 } 3517 } 3518 3519 // __kmp_task_team_sync: Propagation of task team data from team to threads 3520 // which happens just after the release phase of a team barrier. This may be 3521 // called by any thread, but only for teams with # threads > 1. 3522 void __kmp_task_team_sync(kmp_info_t *this_thr, kmp_team_t *team) { 3523 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 3524 3525 // Toggle the th_task_state field, to switch which task_team this thread 3526 // refers to 3527 this_thr->th.th_task_state = 1 - this_thr->th.th_task_state; 3528 // It is now safe to propagate the task team pointer from the team struct to 3529 // the current thread. 3530 TCW_PTR(this_thr->th.th_task_team, 3531 team->t.t_task_team[this_thr->th.th_task_state]); 3532 KA_TRACE(20, 3533 ("__kmp_task_team_sync: Thread T#%d task team switched to task_team " 3534 "%p from Team #%d (parity=%d)\n", 3535 __kmp_gtid_from_thread(this_thr), this_thr->th.th_task_team, 3536 ((team != NULL) ? team->t.t_id : -1), this_thr->th.th_task_state)); 3537 } 3538 3539 // __kmp_task_team_wait: Master thread waits for outstanding tasks after the 3540 // barrier gather phase. Only called by master thread if #threads in team > 1 or 3541 // if proxy tasks were created. 3542 // 3543 // wait is a flag that defaults to 1 (see kmp.h), but waiting can be turned off 3544 // by passing in 0 optionally as the last argument. When wait is zero, master 3545 // thread does not wait for unfinished_threads to reach 0. 3546 void __kmp_task_team_wait( 3547 kmp_info_t *this_thr, 3548 kmp_team_t *team USE_ITT_BUILD_ARG(void *itt_sync_obj), int wait) { 3549 kmp_task_team_t *task_team = team->t.t_task_team[this_thr->th.th_task_state]; 3550 3551 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 3552 KMP_DEBUG_ASSERT(task_team == this_thr->th.th_task_team); 3553 3554 if ((task_team != NULL) && KMP_TASKING_ENABLED(task_team)) { 3555 if (wait) { 3556 KA_TRACE(20, ("__kmp_task_team_wait: Master T#%d waiting for all tasks " 3557 "(for unfinished_threads to reach 0) on task_team = %p\n", 3558 __kmp_gtid_from_thread(this_thr), task_team)); 3559 // Worker threads may have dropped through to release phase, but could 3560 // still be executing tasks. Wait here for tasks to complete. To avoid 3561 // memory contention, only master thread checks termination condition. 3562 kmp_flag_32 flag(RCAST(std::atomic<kmp_uint32> *, 3563 &task_team->tt.tt_unfinished_threads), 3564 0U); 3565 flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj)); 3566 } 3567 // Deactivate the old task team, so that the worker threads will stop 3568 // referencing it while spinning. 3569 KA_TRACE( 3570 20, 3571 ("__kmp_task_team_wait: Master T#%d deactivating task_team %p: " 3572 "setting active to false, setting local and team's pointer to NULL\n", 3573 __kmp_gtid_from_thread(this_thr), task_team)); 3574 KMP_DEBUG_ASSERT(task_team->tt.tt_nproc > 1 || 3575 task_team->tt.tt_found_proxy_tasks == TRUE); 3576 TCW_SYNC_4(task_team->tt.tt_found_proxy_tasks, FALSE); 3577 KMP_CHECK_UPDATE(task_team->tt.tt_untied_task_encountered, 0); 3578 TCW_SYNC_4(task_team->tt.tt_active, FALSE); 3579 KMP_MB(); 3580 3581 TCW_PTR(this_thr->th.th_task_team, NULL); 3582 } 3583 } 3584 3585 // __kmp_tasking_barrier: 3586 // This routine may only called when __kmp_tasking_mode == tskm_extra_barrier. 3587 // Internal function to execute all tasks prior to a regular barrier or a join 3588 // barrier. It is a full barrier itself, which unfortunately turns regular 3589 // barriers into double barriers and join barriers into 1 1/2 barriers. 3590 void __kmp_tasking_barrier(kmp_team_t *team, kmp_info_t *thread, int gtid) { 3591 std::atomic<kmp_uint32> *spin = RCAST( 3592 std::atomic<kmp_uint32> *, 3593 &team->t.t_task_team[thread->th.th_task_state]->tt.tt_unfinished_threads); 3594 int flag = FALSE; 3595 KMP_DEBUG_ASSERT(__kmp_tasking_mode == tskm_extra_barrier); 3596 3597 #if USE_ITT_BUILD 3598 KMP_FSYNC_SPIN_INIT(spin, NULL); 3599 #endif /* USE_ITT_BUILD */ 3600 kmp_flag_32 spin_flag(spin, 0U); 3601 while (!spin_flag.execute_tasks(thread, gtid, TRUE, 3602 &flag USE_ITT_BUILD_ARG(NULL), 0)) { 3603 #if USE_ITT_BUILD 3604 // TODO: What about itt_sync_obj?? 3605 KMP_FSYNC_SPIN_PREPARE(RCAST(void *, spin)); 3606 #endif /* USE_ITT_BUILD */ 3607 3608 if (TCR_4(__kmp_global.g.g_done)) { 3609 if (__kmp_global.g.g_abort) 3610 __kmp_abort_thread(); 3611 break; 3612 } 3613 KMP_YIELD(TRUE); 3614 } 3615 #if USE_ITT_BUILD 3616 KMP_FSYNC_SPIN_ACQUIRED(RCAST(void *, spin)); 3617 #endif /* USE_ITT_BUILD */ 3618 } 3619 3620 // __kmp_give_task puts a task into a given thread queue if: 3621 // - the queue for that thread was created 3622 // - there's space in that queue 3623 // Because of this, __kmp_push_task needs to check if there's space after 3624 // getting the lock 3625 static bool __kmp_give_task(kmp_info_t *thread, kmp_int32 tid, kmp_task_t *task, 3626 kmp_int32 pass) { 3627 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 3628 kmp_task_team_t *task_team = taskdata->td_task_team; 3629 3630 KA_TRACE(20, ("__kmp_give_task: trying to give task %p to thread %d.\n", 3631 taskdata, tid)); 3632 3633 // If task_team is NULL something went really bad... 3634 KMP_DEBUG_ASSERT(task_team != NULL); 3635 3636 bool result = false; 3637 kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid]; 3638 3639 if (thread_data->td.td_deque == NULL) { 3640 // There's no queue in this thread, go find another one 3641 // We're guaranteed that at least one thread has a queue 3642 KA_TRACE(30, 3643 ("__kmp_give_task: thread %d has no queue while giving task %p.\n", 3644 tid, taskdata)); 3645 return result; 3646 } 3647 3648 if (TCR_4(thread_data->td.td_deque_ntasks) >= 3649 TASK_DEQUE_SIZE(thread_data->td)) { 3650 KA_TRACE( 3651 30, 3652 ("__kmp_give_task: queue is full while giving task %p to thread %d.\n", 3653 taskdata, tid)); 3654 3655 // if this deque is bigger than the pass ratio give a chance to another 3656 // thread 3657 if (TASK_DEQUE_SIZE(thread_data->td) / INITIAL_TASK_DEQUE_SIZE >= pass) 3658 return result; 3659 3660 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 3661 __kmp_realloc_task_deque(thread, thread_data); 3662 3663 } else { 3664 3665 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 3666 3667 if (TCR_4(thread_data->td.td_deque_ntasks) >= 3668 TASK_DEQUE_SIZE(thread_data->td)) { 3669 KA_TRACE(30, ("__kmp_give_task: queue is full while giving task %p to " 3670 "thread %d.\n", 3671 taskdata, tid)); 3672 3673 // if this deque is bigger than the pass ratio give a chance to another 3674 // thread 3675 if (TASK_DEQUE_SIZE(thread_data->td) / INITIAL_TASK_DEQUE_SIZE >= pass) 3676 goto release_and_exit; 3677 3678 __kmp_realloc_task_deque(thread, thread_data); 3679 } 3680 } 3681 3682 // lock is held here, and there is space in the deque 3683 3684 thread_data->td.td_deque[thread_data->td.td_deque_tail] = taskdata; 3685 // Wrap index. 3686 thread_data->td.td_deque_tail = 3687 (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td); 3688 TCW_4(thread_data->td.td_deque_ntasks, 3689 TCR_4(thread_data->td.td_deque_ntasks) + 1); 3690 3691 result = true; 3692 KA_TRACE(30, ("__kmp_give_task: successfully gave task %p to thread %d.\n", 3693 taskdata, tid)); 3694 3695 release_and_exit: 3696 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 3697 3698 return result; 3699 } 3700 3701 /* The finish of the proxy tasks is divided in two pieces: 3702 - the top half is the one that can be done from a thread outside the team 3703 - the bottom half must be run from a thread within the team 3704 3705 In order to run the bottom half the task gets queued back into one of the 3706 threads of the team. Once the td_incomplete_child_task counter of the parent 3707 is decremented the threads can leave the barriers. So, the bottom half needs 3708 to be queued before the counter is decremented. The top half is therefore 3709 divided in two parts: 3710 - things that can be run before queuing the bottom half 3711 - things that must be run after queuing the bottom half 3712 3713 This creates a second race as the bottom half can free the task before the 3714 second top half is executed. To avoid this we use the 3715 td_incomplete_child_task of the proxy task to synchronize the top and bottom 3716 half. */ 3717 static void __kmp_first_top_half_finish_proxy(kmp_taskdata_t *taskdata) { 3718 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); 3719 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); 3720 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); 3721 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); 3722 3723 taskdata->td_flags.complete = 1; // mark the task as completed 3724 3725 if (taskdata->td_taskgroup) 3726 KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count); 3727 3728 // Create an imaginary children for this task so the bottom half cannot 3729 // release the task before we have completed the second top half 3730 KMP_ATOMIC_INC(&taskdata->td_incomplete_child_tasks); 3731 } 3732 3733 static void __kmp_second_top_half_finish_proxy(kmp_taskdata_t *taskdata) { 3734 kmp_int32 children = 0; 3735 3736 // Predecrement simulated by "- 1" calculation 3737 children = 3738 KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks) - 1; 3739 KMP_DEBUG_ASSERT(children >= 0); 3740 3741 // Remove the imaginary children 3742 KMP_ATOMIC_DEC(&taskdata->td_incomplete_child_tasks); 3743 } 3744 3745 static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask) { 3746 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); 3747 kmp_info_t *thread = __kmp_threads[gtid]; 3748 3749 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); 3750 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 3751 1); // top half must run before bottom half 3752 3753 // We need to wait to make sure the top half is finished 3754 // Spinning here should be ok as this should happen quickly 3755 while (KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) > 0) 3756 ; 3757 3758 __kmp_release_deps(gtid, taskdata); 3759 __kmp_free_task_and_ancestors(gtid, taskdata, thread); 3760 } 3761 3762 /*! 3763 @ingroup TASKING 3764 @param gtid Global Thread ID of encountering thread 3765 @param ptask Task which execution is completed 3766 3767 Execute the completion of a proxy task from a thread of that is part of the 3768 team. Run first and bottom halves directly. 3769 */ 3770 void __kmpc_proxy_task_completed(kmp_int32 gtid, kmp_task_t *ptask) { 3771 KMP_DEBUG_ASSERT(ptask != NULL); 3772 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); 3773 KA_TRACE( 3774 10, ("__kmp_proxy_task_completed(enter): T#%d proxy task %p completing\n", 3775 gtid, taskdata)); 3776 3777 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); 3778 3779 __kmp_first_top_half_finish_proxy(taskdata); 3780 __kmp_second_top_half_finish_proxy(taskdata); 3781 __kmp_bottom_half_finish_proxy(gtid, ptask); 3782 3783 KA_TRACE(10, 3784 ("__kmp_proxy_task_completed(exit): T#%d proxy task %p completing\n", 3785 gtid, taskdata)); 3786 } 3787 3788 /*! 3789 @ingroup TASKING 3790 @param ptask Task which execution is completed 3791 3792 Execute the completion of a proxy task from a thread that could not belong to 3793 the team. 3794 */ 3795 void __kmpc_proxy_task_completed_ooo(kmp_task_t *ptask) { 3796 KMP_DEBUG_ASSERT(ptask != NULL); 3797 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); 3798 3799 KA_TRACE( 3800 10, 3801 ("__kmp_proxy_task_completed_ooo(enter): proxy task completing ooo %p\n", 3802 taskdata)); 3803 3804 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); 3805 3806 __kmp_first_top_half_finish_proxy(taskdata); 3807 3808 // Enqueue task to complete bottom half completion from a thread within the 3809 // corresponding team 3810 kmp_team_t *team = taskdata->td_team; 3811 kmp_int32 nthreads = team->t.t_nproc; 3812 kmp_info_t *thread; 3813 3814 // This should be similar to start_k = __kmp_get_random( thread ) % nthreads 3815 // but we cannot use __kmp_get_random here 3816 kmp_int32 start_k = 0; 3817 kmp_int32 pass = 1; 3818 kmp_int32 k = start_k; 3819 3820 do { 3821 // For now we're just linearly trying to find a thread 3822 thread = team->t.t_threads[k]; 3823 k = (k + 1) % nthreads; 3824 3825 // we did a full pass through all the threads 3826 if (k == start_k) 3827 pass = pass << 1; 3828 3829 } while (!__kmp_give_task(thread, k, ptask, pass)); 3830 3831 __kmp_second_top_half_finish_proxy(taskdata); 3832 3833 KA_TRACE( 3834 10, 3835 ("__kmp_proxy_task_completed_ooo(exit): proxy task completing ooo %p\n", 3836 taskdata)); 3837 } 3838 3839 kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref, int gtid, 3840 kmp_task_t *task) { 3841 kmp_taskdata_t *td = KMP_TASK_TO_TASKDATA(task); 3842 if (td->td_allow_completion_event.type == KMP_EVENT_UNINITIALIZED) { 3843 td->td_allow_completion_event.type = KMP_EVENT_ALLOW_COMPLETION; 3844 td->td_allow_completion_event.ed.task = task; 3845 __kmp_init_tas_lock(&td->td_allow_completion_event.lock); 3846 } 3847 return &td->td_allow_completion_event; 3848 } 3849 3850 void __kmp_fulfill_event(kmp_event_t *event) { 3851 if (event->type == KMP_EVENT_ALLOW_COMPLETION) { 3852 kmp_task_t *ptask = event->ed.task; 3853 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); 3854 bool detached = false; 3855 int gtid = __kmp_get_gtid(); 3856 3857 // The associated task might have completed or could be completing at this 3858 // point. 3859 // We need to take the lock to avoid races 3860 __kmp_acquire_tas_lock(&event->lock, gtid); 3861 if (taskdata->td_flags.proxy == TASK_PROXY) 3862 detached = true; 3863 event->type = KMP_EVENT_UNINITIALIZED; 3864 __kmp_release_tas_lock(&event->lock, gtid); 3865 3866 if (detached) { 3867 // If the task detached complete the proxy task 3868 if (gtid >= 0) { 3869 kmp_team_t *team = taskdata->td_team; 3870 kmp_info_t *thread = __kmp_get_thread(); 3871 if (thread->th.th_team == team) { 3872 __kmpc_proxy_task_completed(gtid, ptask); 3873 return; 3874 } 3875 } 3876 3877 // fallback 3878 __kmpc_proxy_task_completed_ooo(ptask); 3879 } 3880 } 3881 } 3882 3883 // __kmp_task_dup_alloc: Allocate the taskdata and make a copy of source task 3884 // for taskloop 3885 // 3886 // thread: allocating thread 3887 // task_src: pointer to source task to be duplicated 3888 // returns: a pointer to the allocated kmp_task_t structure (task). 3889 kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src) { 3890 kmp_task_t *task; 3891 kmp_taskdata_t *taskdata; 3892 kmp_taskdata_t *taskdata_src; 3893 kmp_taskdata_t *parent_task = thread->th.th_current_task; 3894 size_t shareds_offset; 3895 size_t task_size; 3896 3897 KA_TRACE(10, ("__kmp_task_dup_alloc(enter): Th %p, source task %p\n", thread, 3898 task_src)); 3899 taskdata_src = KMP_TASK_TO_TASKDATA(task_src); 3900 KMP_DEBUG_ASSERT(taskdata_src->td_flags.proxy == 3901 TASK_FULL); // it should not be proxy task 3902 KMP_DEBUG_ASSERT(taskdata_src->td_flags.tasktype == TASK_EXPLICIT); 3903 task_size = taskdata_src->td_size_alloc; 3904 3905 // Allocate a kmp_taskdata_t block and a kmp_task_t block. 3906 KA_TRACE(30, ("__kmp_task_dup_alloc: Th %p, malloc size %ld\n", thread, 3907 task_size)); 3908 #if USE_FAST_MEMORY 3909 taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, task_size); 3910 #else 3911 taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, task_size); 3912 #endif /* USE_FAST_MEMORY */ 3913 KMP_MEMCPY(taskdata, taskdata_src, task_size); 3914 3915 task = KMP_TASKDATA_TO_TASK(taskdata); 3916 3917 // Initialize new task (only specific fields not affected by memcpy) 3918 taskdata->td_task_id = KMP_GEN_TASK_ID(); 3919 if (task->shareds != NULL) { // need setup shareds pointer 3920 shareds_offset = (char *)task_src->shareds - (char *)taskdata_src; 3921 task->shareds = &((char *)taskdata)[shareds_offset]; 3922 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) == 3923 0); 3924 } 3925 taskdata->td_alloc_thread = thread; 3926 taskdata->td_parent = parent_task; 3927 // task inherits the taskgroup from the parent task 3928 taskdata->td_taskgroup = parent_task->td_taskgroup; 3929 // tied task needs to initialize the td_last_tied at creation, 3930 // untied one does this when it is scheduled for execution 3931 if (taskdata->td_flags.tiedness == TASK_TIED) 3932 taskdata->td_last_tied = taskdata; 3933 3934 // Only need to keep track of child task counts if team parallel and tasking 3935 // not serialized 3936 if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) { 3937 KMP_ATOMIC_INC(&parent_task->td_incomplete_child_tasks); 3938 if (parent_task->td_taskgroup) 3939 KMP_ATOMIC_INC(&parent_task->td_taskgroup->count); 3940 // Only need to keep track of allocated child tasks for explicit tasks since 3941 // implicit not deallocated 3942 if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT) 3943 KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks); 3944 } 3945 3946 KA_TRACE(20, 3947 ("__kmp_task_dup_alloc(exit): Th %p, created task %p, parent=%p\n", 3948 thread, taskdata, taskdata->td_parent)); 3949 #if OMPT_SUPPORT 3950 if (UNLIKELY(ompt_enabled.enabled)) 3951 __ompt_task_init(taskdata, thread->th.th_info.ds.ds_gtid); 3952 #endif 3953 return task; 3954 } 3955 3956 // Routine optionally generated by the compiler for setting the lastprivate flag 3957 // and calling needed constructors for private/firstprivate objects 3958 // (used to form taskloop tasks from pattern task) 3959 // Parameters: dest task, src task, lastprivate flag. 3960 typedef void (*p_task_dup_t)(kmp_task_t *, kmp_task_t *, kmp_int32); 3961 3962 KMP_BUILD_ASSERT(sizeof(long) == 4 || sizeof(long) == 8); 3963 3964 // class to encapsulate manipulating loop bounds in a taskloop task. 3965 // this abstracts away the Intel vs GOMP taskloop interface for setting/getting 3966 // the loop bound variables. 3967 class kmp_taskloop_bounds_t { 3968 kmp_task_t *task; 3969 const kmp_taskdata_t *taskdata; 3970 size_t lower_offset; 3971 size_t upper_offset; 3972 3973 public: 3974 kmp_taskloop_bounds_t(kmp_task_t *_task, kmp_uint64 *lb, kmp_uint64 *ub) 3975 : task(_task), taskdata(KMP_TASK_TO_TASKDATA(task)), 3976 lower_offset((char *)lb - (char *)task), 3977 upper_offset((char *)ub - (char *)task) { 3978 KMP_DEBUG_ASSERT((char *)lb > (char *)_task); 3979 KMP_DEBUG_ASSERT((char *)ub > (char *)_task); 3980 } 3981 kmp_taskloop_bounds_t(kmp_task_t *_task, const kmp_taskloop_bounds_t &bounds) 3982 : task(_task), taskdata(KMP_TASK_TO_TASKDATA(_task)), 3983 lower_offset(bounds.lower_offset), upper_offset(bounds.upper_offset) {} 3984 size_t get_lower_offset() const { return lower_offset; } 3985 size_t get_upper_offset() const { return upper_offset; } 3986 kmp_uint64 get_lb() const { 3987 kmp_int64 retval; 3988 #if defined(KMP_GOMP_COMPAT) 3989 // Intel task just returns the lower bound normally 3990 if (!taskdata->td_flags.native) { 3991 retval = *(kmp_int64 *)((char *)task + lower_offset); 3992 } else { 3993 // GOMP task has to take into account the sizeof(long) 3994 if (taskdata->td_size_loop_bounds == 4) { 3995 kmp_int32 *lb = RCAST(kmp_int32 *, task->shareds); 3996 retval = (kmp_int64)*lb; 3997 } else { 3998 kmp_int64 *lb = RCAST(kmp_int64 *, task->shareds); 3999 retval = (kmp_int64)*lb; 4000 } 4001 } 4002 #else 4003 retval = *(kmp_int64 *)((char *)task + lower_offset); 4004 #endif // defined(KMP_GOMP_COMPAT) 4005 return retval; 4006 } 4007 kmp_uint64 get_ub() const { 4008 kmp_int64 retval; 4009 #if defined(KMP_GOMP_COMPAT) 4010 // Intel task just returns the upper bound normally 4011 if (!taskdata->td_flags.native) { 4012 retval = *(kmp_int64 *)((char *)task + upper_offset); 4013 } else { 4014 // GOMP task has to take into account the sizeof(long) 4015 if (taskdata->td_size_loop_bounds == 4) { 4016 kmp_int32 *ub = RCAST(kmp_int32 *, task->shareds) + 1; 4017 retval = (kmp_int64)*ub; 4018 } else { 4019 kmp_int64 *ub = RCAST(kmp_int64 *, task->shareds) + 1; 4020 retval = (kmp_int64)*ub; 4021 } 4022 } 4023 #else 4024 retval = *(kmp_int64 *)((char *)task + upper_offset); 4025 #endif // defined(KMP_GOMP_COMPAT) 4026 return retval; 4027 } 4028 void set_lb(kmp_uint64 lb) { 4029 #if defined(KMP_GOMP_COMPAT) 4030 // Intel task just sets the lower bound normally 4031 if (!taskdata->td_flags.native) { 4032 *(kmp_uint64 *)((char *)task + lower_offset) = lb; 4033 } else { 4034 // GOMP task has to take into account the sizeof(long) 4035 if (taskdata->td_size_loop_bounds == 4) { 4036 kmp_uint32 *lower = RCAST(kmp_uint32 *, task->shareds); 4037 *lower = (kmp_uint32)lb; 4038 } else { 4039 kmp_uint64 *lower = RCAST(kmp_uint64 *, task->shareds); 4040 *lower = (kmp_uint64)lb; 4041 } 4042 } 4043 #else 4044 *(kmp_uint64 *)((char *)task + lower_offset) = lb; 4045 #endif // defined(KMP_GOMP_COMPAT) 4046 } 4047 void set_ub(kmp_uint64 ub) { 4048 #if defined(KMP_GOMP_COMPAT) 4049 // Intel task just sets the upper bound normally 4050 if (!taskdata->td_flags.native) { 4051 *(kmp_uint64 *)((char *)task + upper_offset) = ub; 4052 } else { 4053 // GOMP task has to take into account the sizeof(long) 4054 if (taskdata->td_size_loop_bounds == 4) { 4055 kmp_uint32 *upper = RCAST(kmp_uint32 *, task->shareds) + 1; 4056 *upper = (kmp_uint32)ub; 4057 } else { 4058 kmp_uint64 *upper = RCAST(kmp_uint64 *, task->shareds) + 1; 4059 *upper = (kmp_uint64)ub; 4060 } 4061 } 4062 #else 4063 *(kmp_uint64 *)((char *)task + upper_offset) = ub; 4064 #endif // defined(KMP_GOMP_COMPAT) 4065 } 4066 }; 4067 4068 // __kmp_taskloop_linear: Start tasks of the taskloop linearly 4069 // 4070 // loc Source location information 4071 // gtid Global thread ID 4072 // task Pattern task, exposes the loop iteration range 4073 // lb Pointer to loop lower bound in task structure 4074 // ub Pointer to loop upper bound in task structure 4075 // st Loop stride 4076 // ub_glob Global upper bound (used for lastprivate check) 4077 // num_tasks Number of tasks to execute 4078 // grainsize Number of loop iterations per task 4079 // extras Number of chunks with grainsize+1 iterations 4080 // tc Iterations count 4081 // task_dup Tasks duplication routine 4082 // codeptr_ra Return address for OMPT events 4083 void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task, 4084 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, 4085 kmp_uint64 ub_glob, kmp_uint64 num_tasks, 4086 kmp_uint64 grainsize, kmp_uint64 extras, 4087 kmp_uint64 tc, 4088 #if OMPT_SUPPORT 4089 void *codeptr_ra, 4090 #endif 4091 void *task_dup) { 4092 KMP_COUNT_BLOCK(OMP_TASKLOOP); 4093 KMP_TIME_PARTITIONED_BLOCK(OMP_taskloop_scheduling); 4094 p_task_dup_t ptask_dup = (p_task_dup_t)task_dup; 4095 // compiler provides global bounds here 4096 kmp_taskloop_bounds_t task_bounds(task, lb, ub); 4097 kmp_uint64 lower = task_bounds.get_lb(); 4098 kmp_uint64 upper = task_bounds.get_ub(); 4099 kmp_uint64 i; 4100 kmp_info_t *thread = __kmp_threads[gtid]; 4101 kmp_taskdata_t *current_task = thread->th.th_current_task; 4102 kmp_task_t *next_task; 4103 kmp_int32 lastpriv = 0; 4104 4105 KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + extras); 4106 KMP_DEBUG_ASSERT(num_tasks > extras); 4107 KMP_DEBUG_ASSERT(num_tasks > 0); 4108 KA_TRACE(20, ("__kmp_taskloop_linear: T#%d: %lld tasks, grainsize %lld, " 4109 "extras %lld, i=%lld,%lld(%d)%lld, dup %p\n", 4110 gtid, num_tasks, grainsize, extras, lower, upper, ub_glob, st, 4111 task_dup)); 4112 4113 // Launch num_tasks tasks, assign grainsize iterations each task 4114 for (i = 0; i < num_tasks; ++i) { 4115 kmp_uint64 chunk_minus_1; 4116 if (extras == 0) { 4117 chunk_minus_1 = grainsize - 1; 4118 } else { 4119 chunk_minus_1 = grainsize; 4120 --extras; // first extras iterations get bigger chunk (grainsize+1) 4121 } 4122 upper = lower + st * chunk_minus_1; 4123 if (i == num_tasks - 1) { 4124 // schedule the last task, set lastprivate flag if needed 4125 if (st == 1) { // most common case 4126 KMP_DEBUG_ASSERT(upper == *ub); 4127 if (upper == ub_glob) 4128 lastpriv = 1; 4129 } else if (st > 0) { // positive loop stride 4130 KMP_DEBUG_ASSERT((kmp_uint64)st > *ub - upper); 4131 if ((kmp_uint64)st > ub_glob - upper) 4132 lastpriv = 1; 4133 } else { // negative loop stride 4134 KMP_DEBUG_ASSERT(upper + st < *ub); 4135 if (upper - ub_glob < (kmp_uint64)(-st)) 4136 lastpriv = 1; 4137 } 4138 } 4139 next_task = __kmp_task_dup_alloc(thread, task); // allocate new task 4140 kmp_taskdata_t *next_taskdata = KMP_TASK_TO_TASKDATA(next_task); 4141 kmp_taskloop_bounds_t next_task_bounds = 4142 kmp_taskloop_bounds_t(next_task, task_bounds); 4143 4144 // adjust task-specific bounds 4145 next_task_bounds.set_lb(lower); 4146 if (next_taskdata->td_flags.native) { 4147 next_task_bounds.set_ub(upper + (st > 0 ? 1 : -1)); 4148 } else { 4149 next_task_bounds.set_ub(upper); 4150 } 4151 if (ptask_dup != NULL) // set lastprivate flag, construct firstprivates, 4152 // etc. 4153 ptask_dup(next_task, task, lastpriv); 4154 KA_TRACE(40, 4155 ("__kmp_taskloop_linear: T#%d; task #%llu: task %p: lower %lld, " 4156 "upper %lld stride %lld, (offsets %p %p)\n", 4157 gtid, i, next_task, lower, upper, st, 4158 next_task_bounds.get_lower_offset(), 4159 next_task_bounds.get_upper_offset())); 4160 #if OMPT_SUPPORT 4161 __kmp_omp_taskloop_task(NULL, gtid, next_task, 4162 codeptr_ra); // schedule new task 4163 #else 4164 __kmp_omp_task(gtid, next_task, true); // schedule new task 4165 #endif 4166 lower = upper + st; // adjust lower bound for the next iteration 4167 } 4168 // free the pattern task and exit 4169 __kmp_task_start(gtid, task, current_task); // make internal bookkeeping 4170 // do not execute the pattern task, just do internal bookkeeping 4171 __kmp_task_finish<false>(gtid, task, current_task); 4172 } 4173 4174 // Structure to keep taskloop parameters for auxiliary task 4175 // kept in the shareds of the task structure. 4176 typedef struct __taskloop_params { 4177 kmp_task_t *task; 4178 kmp_uint64 *lb; 4179 kmp_uint64 *ub; 4180 void *task_dup; 4181 kmp_int64 st; 4182 kmp_uint64 ub_glob; 4183 kmp_uint64 num_tasks; 4184 kmp_uint64 grainsize; 4185 kmp_uint64 extras; 4186 kmp_uint64 tc; 4187 kmp_uint64 num_t_min; 4188 #if OMPT_SUPPORT 4189 void *codeptr_ra; 4190 #endif 4191 } __taskloop_params_t; 4192 4193 void __kmp_taskloop_recur(ident_t *, int, kmp_task_t *, kmp_uint64 *, 4194 kmp_uint64 *, kmp_int64, kmp_uint64, kmp_uint64, 4195 kmp_uint64, kmp_uint64, kmp_uint64, kmp_uint64, 4196 #if OMPT_SUPPORT 4197 void *, 4198 #endif 4199 void *); 4200 4201 // Execute part of the taskloop submitted as a task. 4202 int __kmp_taskloop_task(int gtid, void *ptask) { 4203 __taskloop_params_t *p = 4204 (__taskloop_params_t *)((kmp_task_t *)ptask)->shareds; 4205 kmp_task_t *task = p->task; 4206 kmp_uint64 *lb = p->lb; 4207 kmp_uint64 *ub = p->ub; 4208 void *task_dup = p->task_dup; 4209 // p_task_dup_t ptask_dup = (p_task_dup_t)task_dup; 4210 kmp_int64 st = p->st; 4211 kmp_uint64 ub_glob = p->ub_glob; 4212 kmp_uint64 num_tasks = p->num_tasks; 4213 kmp_uint64 grainsize = p->grainsize; 4214 kmp_uint64 extras = p->extras; 4215 kmp_uint64 tc = p->tc; 4216 kmp_uint64 num_t_min = p->num_t_min; 4217 #if OMPT_SUPPORT 4218 void *codeptr_ra = p->codeptr_ra; 4219 #endif 4220 #if KMP_DEBUG 4221 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 4222 KMP_DEBUG_ASSERT(task != NULL); 4223 KA_TRACE(20, ("__kmp_taskloop_task: T#%d, task %p: %lld tasks, grainsize" 4224 " %lld, extras %lld, i=%lld,%lld(%d), dup %p\n", 4225 gtid, taskdata, num_tasks, grainsize, extras, *lb, *ub, st, 4226 task_dup)); 4227 #endif 4228 KMP_DEBUG_ASSERT(num_tasks * 2 + 1 > num_t_min); 4229 if (num_tasks > num_t_min) 4230 __kmp_taskloop_recur(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks, 4231 grainsize, extras, tc, num_t_min, 4232 #if OMPT_SUPPORT 4233 codeptr_ra, 4234 #endif 4235 task_dup); 4236 else 4237 __kmp_taskloop_linear(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks, 4238 grainsize, extras, tc, 4239 #if OMPT_SUPPORT 4240 codeptr_ra, 4241 #endif 4242 task_dup); 4243 4244 KA_TRACE(40, ("__kmp_taskloop_task(exit): T#%d\n", gtid)); 4245 return 0; 4246 } 4247 4248 // Schedule part of the taskloop as a task, 4249 // execute the rest of the taskloop. 4250 // 4251 // loc Source location information 4252 // gtid Global thread ID 4253 // task Pattern task, exposes the loop iteration range 4254 // lb Pointer to loop lower bound in task structure 4255 // ub Pointer to loop upper bound in task structure 4256 // st Loop stride 4257 // ub_glob Global upper bound (used for lastprivate check) 4258 // num_tasks Number of tasks to execute 4259 // grainsize Number of loop iterations per task 4260 // extras Number of chunks with grainsize+1 iterations 4261 // tc Iterations count 4262 // num_t_min Threshold to launch tasks recursively 4263 // task_dup Tasks duplication routine 4264 // codeptr_ra Return address for OMPT events 4265 void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task, 4266 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, 4267 kmp_uint64 ub_glob, kmp_uint64 num_tasks, 4268 kmp_uint64 grainsize, kmp_uint64 extras, 4269 kmp_uint64 tc, kmp_uint64 num_t_min, 4270 #if OMPT_SUPPORT 4271 void *codeptr_ra, 4272 #endif 4273 void *task_dup) { 4274 #if KMP_DEBUG 4275 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 4276 KMP_DEBUG_ASSERT(task != NULL); 4277 KMP_DEBUG_ASSERT(num_tasks > num_t_min); 4278 KA_TRACE(20, ("__kmp_taskloop_recur: T#%d, task %p: %lld tasks, grainsize" 4279 " %lld, extras %lld, i=%lld,%lld(%d), dup %p\n", 4280 gtid, taskdata, num_tasks, grainsize, extras, *lb, *ub, st, 4281 task_dup)); 4282 #endif 4283 p_task_dup_t ptask_dup = (p_task_dup_t)task_dup; 4284 kmp_uint64 lower = *lb; 4285 kmp_info_t *thread = __kmp_threads[gtid]; 4286 // kmp_taskdata_t *current_task = thread->th.th_current_task; 4287 kmp_task_t *next_task; 4288 size_t lower_offset = 4289 (char *)lb - (char *)task; // remember offset of lb in the task structure 4290 size_t upper_offset = 4291 (char *)ub - (char *)task; // remember offset of ub in the task structure 4292 4293 KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + extras); 4294 KMP_DEBUG_ASSERT(num_tasks > extras); 4295 KMP_DEBUG_ASSERT(num_tasks > 0); 4296 4297 // split the loop in two halves 4298 kmp_uint64 lb1, ub0, tc0, tc1, ext0, ext1; 4299 kmp_uint64 gr_size0 = grainsize; 4300 kmp_uint64 n_tsk0 = num_tasks >> 1; // num_tasks/2 to execute 4301 kmp_uint64 n_tsk1 = num_tasks - n_tsk0; // to schedule as a task 4302 if (n_tsk0 <= extras) { 4303 gr_size0++; // integrate extras into grainsize 4304 ext0 = 0; // no extra iters in 1st half 4305 ext1 = extras - n_tsk0; // remaining extras 4306 tc0 = gr_size0 * n_tsk0; 4307 tc1 = tc - tc0; 4308 } else { // n_tsk0 > extras 4309 ext1 = 0; // no extra iters in 2nd half 4310 ext0 = extras; 4311 tc1 = grainsize * n_tsk1; 4312 tc0 = tc - tc1; 4313 } 4314 ub0 = lower + st * (tc0 - 1); 4315 lb1 = ub0 + st; 4316 4317 // create pattern task for 2nd half of the loop 4318 next_task = __kmp_task_dup_alloc(thread, task); // duplicate the task 4319 // adjust lower bound (upper bound is not changed) for the 2nd half 4320 *(kmp_uint64 *)((char *)next_task + lower_offset) = lb1; 4321 if (ptask_dup != NULL) // construct firstprivates, etc. 4322 ptask_dup(next_task, task, 0); 4323 *ub = ub0; // adjust upper bound for the 1st half 4324 4325 // create auxiliary task for 2nd half of the loop 4326 kmp_task_t *new_task = 4327 __kmpc_omp_task_alloc(loc, gtid, 1, 3 * sizeof(void *), 4328 sizeof(__taskloop_params_t), &__kmp_taskloop_task); 4329 __taskloop_params_t *p = (__taskloop_params_t *)new_task->shareds; 4330 p->task = next_task; 4331 p->lb = (kmp_uint64 *)((char *)next_task + lower_offset); 4332 p->ub = (kmp_uint64 *)((char *)next_task + upper_offset); 4333 p->task_dup = task_dup; 4334 p->st = st; 4335 p->ub_glob = ub_glob; 4336 p->num_tasks = n_tsk1; 4337 p->grainsize = grainsize; 4338 p->extras = ext1; 4339 p->tc = tc1; 4340 p->num_t_min = num_t_min; 4341 #if OMPT_SUPPORT 4342 p->codeptr_ra = codeptr_ra; 4343 #endif 4344 4345 #if OMPT_SUPPORT 4346 // schedule new task with correct return address for OMPT events 4347 __kmp_omp_taskloop_task(NULL, gtid, new_task, codeptr_ra); 4348 #else 4349 __kmp_omp_task(gtid, new_task, true); // schedule new task 4350 #endif 4351 4352 // execute the 1st half of current subrange 4353 if (n_tsk0 > num_t_min) 4354 __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0, gr_size0, 4355 ext0, tc0, num_t_min, 4356 #if OMPT_SUPPORT 4357 codeptr_ra, 4358 #endif 4359 task_dup); 4360 else 4361 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0, 4362 gr_size0, ext0, tc0, 4363 #if OMPT_SUPPORT 4364 codeptr_ra, 4365 #endif 4366 task_dup); 4367 4368 KA_TRACE(40, ("__kmpc_taskloop_recur(exit): T#%d\n", gtid)); 4369 } 4370 4371 /*! 4372 @ingroup TASKING 4373 @param loc Source location information 4374 @param gtid Global thread ID 4375 @param task Task structure 4376 @param if_val Value of the if clause 4377 @param lb Pointer to loop lower bound in task structure 4378 @param ub Pointer to loop upper bound in task structure 4379 @param st Loop stride 4380 @param nogroup Flag, 1 if no taskgroup needs to be added, 0 otherwise 4381 @param sched Schedule specified 0/1/2 for none/grainsize/num_tasks 4382 @param grainsize Schedule value if specified 4383 @param task_dup Tasks duplication routine 4384 4385 Execute the taskloop construct. 4386 */ 4387 void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val, 4388 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, 4389 int sched, kmp_uint64 grainsize, void *task_dup) { 4390 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 4391 KMP_DEBUG_ASSERT(task != NULL); 4392 4393 if (nogroup == 0) { 4394 #if OMPT_SUPPORT && OMPT_OPTIONAL 4395 OMPT_STORE_RETURN_ADDRESS(gtid); 4396 #endif 4397 __kmpc_taskgroup(loc, gtid); 4398 } 4399 4400 // ========================================================================= 4401 // calculate loop parameters 4402 kmp_taskloop_bounds_t task_bounds(task, lb, ub); 4403 kmp_uint64 tc; 4404 // compiler provides global bounds here 4405 kmp_uint64 lower = task_bounds.get_lb(); 4406 kmp_uint64 upper = task_bounds.get_ub(); 4407 kmp_uint64 ub_glob = upper; // global upper used to calc lastprivate flag 4408 kmp_uint64 num_tasks = 0, extras = 0; 4409 kmp_uint64 num_tasks_min = __kmp_taskloop_min_tasks; 4410 kmp_info_t *thread = __kmp_threads[gtid]; 4411 kmp_taskdata_t *current_task = thread->th.th_current_task; 4412 4413 KA_TRACE(20, ("__kmpc_taskloop: T#%d, task %p, lb %lld, ub %lld, st %lld, " 4414 "grain %llu(%d), dup %p\n", 4415 gtid, taskdata, lower, upper, st, grainsize, sched, task_dup)); 4416 4417 // compute trip count 4418 if (st == 1) { // most common case 4419 tc = upper - lower + 1; 4420 } else if (st < 0) { 4421 tc = (lower - upper) / (-st) + 1; 4422 } else { // st > 0 4423 tc = (upper - lower) / st + 1; 4424 } 4425 if (tc == 0) { 4426 KA_TRACE(20, ("__kmpc_taskloop(exit): T#%d zero-trip loop\n", gtid)); 4427 // free the pattern task and exit 4428 __kmp_task_start(gtid, task, current_task); 4429 // do not execute anything for zero-trip loop 4430 __kmp_task_finish<false>(gtid, task, current_task); 4431 return; 4432 } 4433 4434 #if OMPT_SUPPORT && OMPT_OPTIONAL 4435 ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL); 4436 ompt_task_info_t *task_info = __ompt_get_task_info_object(0); 4437 if (ompt_enabled.ompt_callback_work) { 4438 ompt_callbacks.ompt_callback(ompt_callback_work)( 4439 ompt_work_taskloop, ompt_scope_begin, &(team_info->parallel_data), 4440 &(task_info->task_data), tc, OMPT_GET_RETURN_ADDRESS(0)); 4441 } 4442 #endif 4443 4444 if (num_tasks_min == 0) 4445 // TODO: can we choose better default heuristic? 4446 num_tasks_min = 4447 KMP_MIN(thread->th.th_team_nproc * 10, INITIAL_TASK_DEQUE_SIZE); 4448 4449 // compute num_tasks/grainsize based on the input provided 4450 switch (sched) { 4451 case 0: // no schedule clause specified, we can choose the default 4452 // let's try to schedule (team_size*10) tasks 4453 grainsize = thread->th.th_team_nproc * 10; 4454 KMP_FALLTHROUGH(); 4455 case 2: // num_tasks provided 4456 if (grainsize > tc) { 4457 num_tasks = tc; // too big num_tasks requested, adjust values 4458 grainsize = 1; 4459 extras = 0; 4460 } else { 4461 num_tasks = grainsize; 4462 grainsize = tc / num_tasks; 4463 extras = tc % num_tasks; 4464 } 4465 break; 4466 case 1: // grainsize provided 4467 if (grainsize > tc) { 4468 num_tasks = 1; // too big grainsize requested, adjust values 4469 grainsize = tc; 4470 extras = 0; 4471 } else { 4472 num_tasks = tc / grainsize; 4473 // adjust grainsize for balanced distribution of iterations 4474 grainsize = tc / num_tasks; 4475 extras = tc % num_tasks; 4476 } 4477 break; 4478 default: 4479 KMP_ASSERT2(0, "unknown scheduling of taskloop"); 4480 } 4481 KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + extras); 4482 KMP_DEBUG_ASSERT(num_tasks > extras); 4483 KMP_DEBUG_ASSERT(num_tasks > 0); 4484 // ========================================================================= 4485 4486 // check if clause value first 4487 // Also require GOMP_taskloop to reduce to linear (taskdata->td_flags.native) 4488 if (if_val == 0) { // if(0) specified, mark task as serial 4489 taskdata->td_flags.task_serial = 1; 4490 taskdata->td_flags.tiedness = TASK_TIED; // AC: serial task cannot be untied 4491 // always start serial tasks linearly 4492 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks, 4493 grainsize, extras, tc, 4494 #if OMPT_SUPPORT 4495 OMPT_GET_RETURN_ADDRESS(0), 4496 #endif 4497 task_dup); 4498 // !taskdata->td_flags.native => currently force linear spawning of tasks 4499 // for GOMP_taskloop 4500 } else if (num_tasks > num_tasks_min && !taskdata->td_flags.native) { 4501 KA_TRACE(20, ("__kmpc_taskloop: T#%d, go recursive: tc %llu, #tasks %llu" 4502 "(%lld), grain %llu, extras %llu\n", 4503 gtid, tc, num_tasks, num_tasks_min, grainsize, extras)); 4504 __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, num_tasks, 4505 grainsize, extras, tc, num_tasks_min, 4506 #if OMPT_SUPPORT 4507 OMPT_GET_RETURN_ADDRESS(0), 4508 #endif 4509 task_dup); 4510 } else { 4511 KA_TRACE(20, ("__kmpc_taskloop: T#%d, go linear: tc %llu, #tasks %llu" 4512 "(%lld), grain %llu, extras %llu\n", 4513 gtid, tc, num_tasks, num_tasks_min, grainsize, extras)); 4514 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks, 4515 grainsize, extras, tc, 4516 #if OMPT_SUPPORT 4517 OMPT_GET_RETURN_ADDRESS(0), 4518 #endif 4519 task_dup); 4520 } 4521 4522 #if OMPT_SUPPORT && OMPT_OPTIONAL 4523 if (ompt_enabled.ompt_callback_work) { 4524 ompt_callbacks.ompt_callback(ompt_callback_work)( 4525 ompt_work_taskloop, ompt_scope_end, &(team_info->parallel_data), 4526 &(task_info->task_data), tc, OMPT_GET_RETURN_ADDRESS(0)); 4527 } 4528 #endif 4529 4530 if (nogroup == 0) { 4531 #if OMPT_SUPPORT && OMPT_OPTIONAL 4532 OMPT_STORE_RETURN_ADDRESS(gtid); 4533 #endif 4534 __kmpc_end_taskgroup(loc, gtid); 4535 } 4536 KA_TRACE(20, ("__kmpc_taskloop(exit): T#%d\n", gtid)); 4537 } 4538