1 /* 2 * kmp_tasking.cpp -- OpenMP 3.0 tasking support. 3 */ 4 5 //===----------------------------------------------------------------------===// 6 // 7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 8 // See https://llvm.org/LICENSE.txt for license information. 9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "kmp.h" 14 #include "kmp_i18n.h" 15 #include "kmp_itt.h" 16 #include "kmp_stats.h" 17 #include "kmp_wait_release.h" 18 #include "kmp_taskdeps.h" 19 20 #if OMPT_SUPPORT 21 #include "ompt-specific.h" 22 #endif 23 24 #include "tsan_annotations.h" 25 26 /* forward declaration */ 27 static void __kmp_enable_tasking(kmp_task_team_t *task_team, 28 kmp_info_t *this_thr); 29 static void __kmp_alloc_task_deque(kmp_info_t *thread, 30 kmp_thread_data_t *thread_data); 31 static int __kmp_realloc_task_threads_data(kmp_info_t *thread, 32 kmp_task_team_t *task_team); 33 static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask); 34 35 #ifdef BUILD_TIED_TASK_STACK 36 37 // __kmp_trace_task_stack: print the tied tasks from the task stack in order 38 // from top do bottom 39 // 40 // gtid: global thread identifier for thread containing stack 41 // thread_data: thread data for task team thread containing stack 42 // threshold: value above which the trace statement triggers 43 // location: string identifying call site of this function (for trace) 44 static void __kmp_trace_task_stack(kmp_int32 gtid, 45 kmp_thread_data_t *thread_data, 46 int threshold, char *location) { 47 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; 48 kmp_taskdata_t **stack_top = task_stack->ts_top; 49 kmp_int32 entries = task_stack->ts_entries; 50 kmp_taskdata_t *tied_task; 51 52 KA_TRACE( 53 threshold, 54 ("__kmp_trace_task_stack(start): location = %s, gtid = %d, entries = %d, " 55 "first_block = %p, stack_top = %p \n", 56 location, gtid, entries, task_stack->ts_first_block, stack_top)); 57 58 KMP_DEBUG_ASSERT(stack_top != NULL); 59 KMP_DEBUG_ASSERT(entries > 0); 60 61 while (entries != 0) { 62 KMP_DEBUG_ASSERT(stack_top != &task_stack->ts_first_block.sb_block[0]); 63 // fix up ts_top if we need to pop from previous block 64 if (entries & TASK_STACK_INDEX_MASK == 0) { 65 kmp_stack_block_t *stack_block = (kmp_stack_block_t *)(stack_top); 66 67 stack_block = stack_block->sb_prev; 68 stack_top = &stack_block->sb_block[TASK_STACK_BLOCK_SIZE]; 69 } 70 71 // finish bookkeeping 72 stack_top--; 73 entries--; 74 75 tied_task = *stack_top; 76 77 KMP_DEBUG_ASSERT(tied_task != NULL); 78 KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED); 79 80 KA_TRACE(threshold, 81 ("__kmp_trace_task_stack(%s): gtid=%d, entry=%d, " 82 "stack_top=%p, tied_task=%p\n", 83 location, gtid, entries, stack_top, tied_task)); 84 } 85 KMP_DEBUG_ASSERT(stack_top == &task_stack->ts_first_block.sb_block[0]); 86 87 KA_TRACE(threshold, 88 ("__kmp_trace_task_stack(exit): location = %s, gtid = %d\n", 89 location, gtid)); 90 } 91 92 // __kmp_init_task_stack: initialize the task stack for the first time 93 // after a thread_data structure is created. 94 // It should not be necessary to do this again (assuming the stack works). 95 // 96 // gtid: global thread identifier of calling thread 97 // thread_data: thread data for task team thread containing stack 98 static void __kmp_init_task_stack(kmp_int32 gtid, 99 kmp_thread_data_t *thread_data) { 100 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; 101 kmp_stack_block_t *first_block; 102 103 // set up the first block of the stack 104 first_block = &task_stack->ts_first_block; 105 task_stack->ts_top = (kmp_taskdata_t **)first_block; 106 memset((void *)first_block, '\0', 107 TASK_STACK_BLOCK_SIZE * sizeof(kmp_taskdata_t *)); 108 109 // initialize the stack to be empty 110 task_stack->ts_entries = TASK_STACK_EMPTY; 111 first_block->sb_next = NULL; 112 first_block->sb_prev = NULL; 113 } 114 115 // __kmp_free_task_stack: free the task stack when thread_data is destroyed. 116 // 117 // gtid: global thread identifier for calling thread 118 // thread_data: thread info for thread containing stack 119 static void __kmp_free_task_stack(kmp_int32 gtid, 120 kmp_thread_data_t *thread_data) { 121 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; 122 kmp_stack_block_t *stack_block = &task_stack->ts_first_block; 123 124 KMP_DEBUG_ASSERT(task_stack->ts_entries == TASK_STACK_EMPTY); 125 // free from the second block of the stack 126 while (stack_block != NULL) { 127 kmp_stack_block_t *next_block = (stack_block) ? stack_block->sb_next : NULL; 128 129 stack_block->sb_next = NULL; 130 stack_block->sb_prev = NULL; 131 if (stack_block != &task_stack->ts_first_block) { 132 __kmp_thread_free(thread, 133 stack_block); // free the block, if not the first 134 } 135 stack_block = next_block; 136 } 137 // initialize the stack to be empty 138 task_stack->ts_entries = 0; 139 task_stack->ts_top = NULL; 140 } 141 142 // __kmp_push_task_stack: Push the tied task onto the task stack. 143 // Grow the stack if necessary by allocating another block. 144 // 145 // gtid: global thread identifier for calling thread 146 // thread: thread info for thread containing stack 147 // tied_task: the task to push on the stack 148 static void __kmp_push_task_stack(kmp_int32 gtid, kmp_info_t *thread, 149 kmp_taskdata_t *tied_task) { 150 // GEH - need to consider what to do if tt_threads_data not allocated yet 151 kmp_thread_data_t *thread_data = 152 &thread->th.th_task_team->tt.tt_threads_data[__kmp_tid_from_gtid(gtid)]; 153 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; 154 155 if (tied_task->td_flags.team_serial || tied_task->td_flags.tasking_ser) { 156 return; // Don't push anything on stack if team or team tasks are serialized 157 } 158 159 KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED); 160 KMP_DEBUG_ASSERT(task_stack->ts_top != NULL); 161 162 KA_TRACE(20, 163 ("__kmp_push_task_stack(enter): GTID: %d; THREAD: %p; TASK: %p\n", 164 gtid, thread, tied_task)); 165 // Store entry 166 *(task_stack->ts_top) = tied_task; 167 168 // Do bookkeeping for next push 169 task_stack->ts_top++; 170 task_stack->ts_entries++; 171 172 if (task_stack->ts_entries & TASK_STACK_INDEX_MASK == 0) { 173 // Find beginning of this task block 174 kmp_stack_block_t *stack_block = 175 (kmp_stack_block_t *)(task_stack->ts_top - TASK_STACK_BLOCK_SIZE); 176 177 // Check if we already have a block 178 if (stack_block->sb_next != 179 NULL) { // reset ts_top to beginning of next block 180 task_stack->ts_top = &stack_block->sb_next->sb_block[0]; 181 } else { // Alloc new block and link it up 182 kmp_stack_block_t *new_block = (kmp_stack_block_t *)__kmp_thread_calloc( 183 thread, sizeof(kmp_stack_block_t)); 184 185 task_stack->ts_top = &new_block->sb_block[0]; 186 stack_block->sb_next = new_block; 187 new_block->sb_prev = stack_block; 188 new_block->sb_next = NULL; 189 190 KA_TRACE( 191 30, 192 ("__kmp_push_task_stack(): GTID: %d; TASK: %p; Alloc new block: %p\n", 193 gtid, tied_task, new_block)); 194 } 195 } 196 KA_TRACE(20, ("__kmp_push_task_stack(exit): GTID: %d; TASK: %p\n", gtid, 197 tied_task)); 198 } 199 200 // __kmp_pop_task_stack: Pop the tied task from the task stack. Don't return 201 // the task, just check to make sure it matches the ending task passed in. 202 // 203 // gtid: global thread identifier for the calling thread 204 // thread: thread info structure containing stack 205 // tied_task: the task popped off the stack 206 // ending_task: the task that is ending (should match popped task) 207 static void __kmp_pop_task_stack(kmp_int32 gtid, kmp_info_t *thread, 208 kmp_taskdata_t *ending_task) { 209 // GEH - need to consider what to do if tt_threads_data not allocated yet 210 kmp_thread_data_t *thread_data = 211 &thread->th.th_task_team->tt_threads_data[__kmp_tid_from_gtid(gtid)]; 212 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; 213 kmp_taskdata_t *tied_task; 214 215 if (ending_task->td_flags.team_serial || ending_task->td_flags.tasking_ser) { 216 // Don't pop anything from stack if team or team tasks are serialized 217 return; 218 } 219 220 KMP_DEBUG_ASSERT(task_stack->ts_top != NULL); 221 KMP_DEBUG_ASSERT(task_stack->ts_entries > 0); 222 223 KA_TRACE(20, ("__kmp_pop_task_stack(enter): GTID: %d; THREAD: %p\n", gtid, 224 thread)); 225 226 // fix up ts_top if we need to pop from previous block 227 if (task_stack->ts_entries & TASK_STACK_INDEX_MASK == 0) { 228 kmp_stack_block_t *stack_block = (kmp_stack_block_t *)(task_stack->ts_top); 229 230 stack_block = stack_block->sb_prev; 231 task_stack->ts_top = &stack_block->sb_block[TASK_STACK_BLOCK_SIZE]; 232 } 233 234 // finish bookkeeping 235 task_stack->ts_top--; 236 task_stack->ts_entries--; 237 238 tied_task = *(task_stack->ts_top); 239 240 KMP_DEBUG_ASSERT(tied_task != NULL); 241 KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED); 242 KMP_DEBUG_ASSERT(tied_task == ending_task); // If we built the stack correctly 243 244 KA_TRACE(20, ("__kmp_pop_task_stack(exit): GTID: %d; TASK: %p\n", gtid, 245 tied_task)); 246 return; 247 } 248 #endif /* BUILD_TIED_TASK_STACK */ 249 250 // returns 1 if new task is allowed to execute, 0 otherwise 251 // checks Task Scheduling constraint (if requested) and 252 // mutexinoutset dependencies if any 253 static bool __kmp_task_is_allowed(int gtid, const kmp_int32 is_constrained, 254 const kmp_taskdata_t *tasknew, 255 const kmp_taskdata_t *taskcurr) { 256 if (is_constrained && (tasknew->td_flags.tiedness == TASK_TIED)) { 257 // Check if the candidate obeys the Task Scheduling Constraints (TSC) 258 // only descendant of all deferred tied tasks can be scheduled, checking 259 // the last one is enough, as it in turn is the descendant of all others 260 kmp_taskdata_t *current = taskcurr->td_last_tied; 261 KMP_DEBUG_ASSERT(current != NULL); 262 // check if the task is not suspended on barrier 263 if (current->td_flags.tasktype == TASK_EXPLICIT || 264 current->td_taskwait_thread > 0) { // <= 0 on barrier 265 kmp_int32 level = current->td_level; 266 kmp_taskdata_t *parent = tasknew->td_parent; 267 while (parent != current && parent->td_level > level) { 268 // check generation up to the level of the current task 269 parent = parent->td_parent; 270 KMP_DEBUG_ASSERT(parent != NULL); 271 } 272 if (parent != current) 273 return false; 274 } 275 } 276 // Check mutexinoutset dependencies, acquire locks 277 kmp_depnode_t *node = tasknew->td_depnode; 278 if (node && (node->dn.mtx_num_locks > 0)) { 279 for (int i = 0; i < node->dn.mtx_num_locks; ++i) { 280 KMP_DEBUG_ASSERT(node->dn.mtx_locks[i] != NULL); 281 if (__kmp_test_lock(node->dn.mtx_locks[i], gtid)) 282 continue; 283 // could not get the lock, release previous locks 284 for (int j = i - 1; j >= 0; --j) 285 __kmp_release_lock(node->dn.mtx_locks[j], gtid); 286 return false; 287 } 288 // negative num_locks means all locks acquired successfully 289 node->dn.mtx_num_locks = -node->dn.mtx_num_locks; 290 } 291 return true; 292 } 293 294 // __kmp_realloc_task_deque: 295 // Re-allocates a task deque for a particular thread, copies the content from 296 // the old deque and adjusts the necessary data structures relating to the 297 // deque. This operation must be done with the deque_lock being held 298 static void __kmp_realloc_task_deque(kmp_info_t *thread, 299 kmp_thread_data_t *thread_data) { 300 kmp_int32 size = TASK_DEQUE_SIZE(thread_data->td); 301 kmp_int32 new_size = 2 * size; 302 303 KE_TRACE(10, ("__kmp_realloc_task_deque: T#%d reallocating deque[from %d to " 304 "%d] for thread_data %p\n", 305 __kmp_gtid_from_thread(thread), size, new_size, thread_data)); 306 307 kmp_taskdata_t **new_deque = 308 (kmp_taskdata_t **)__kmp_allocate(new_size * sizeof(kmp_taskdata_t *)); 309 310 int i, j; 311 for (i = thread_data->td.td_deque_head, j = 0; j < size; 312 i = (i + 1) & TASK_DEQUE_MASK(thread_data->td), j++) 313 new_deque[j] = thread_data->td.td_deque[i]; 314 315 __kmp_free(thread_data->td.td_deque); 316 317 thread_data->td.td_deque_head = 0; 318 thread_data->td.td_deque_tail = size; 319 thread_data->td.td_deque = new_deque; 320 thread_data->td.td_deque_size = new_size; 321 } 322 323 // __kmp_push_task: Add a task to the thread's deque 324 static kmp_int32 __kmp_push_task(kmp_int32 gtid, kmp_task_t *task) { 325 kmp_info_t *thread = __kmp_threads[gtid]; 326 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 327 kmp_task_team_t *task_team = thread->th.th_task_team; 328 kmp_int32 tid = __kmp_tid_from_gtid(gtid); 329 kmp_thread_data_t *thread_data; 330 331 KA_TRACE(20, 332 ("__kmp_push_task: T#%d trying to push task %p.\n", gtid, taskdata)); 333 334 if (taskdata->td_flags.tiedness == TASK_UNTIED) { 335 // untied task needs to increment counter so that the task structure is not 336 // freed prematurely 337 kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count); 338 KMP_DEBUG_USE_VAR(counter); 339 KA_TRACE( 340 20, 341 ("__kmp_push_task: T#%d untied_count (%d) incremented for task %p\n", 342 gtid, counter, taskdata)); 343 } 344 345 // The first check avoids building task_team thread data if serialized 346 if (taskdata->td_flags.task_serial) { 347 KA_TRACE(20, ("__kmp_push_task: T#%d team serialized; returning " 348 "TASK_NOT_PUSHED for task %p\n", 349 gtid, taskdata)); 350 return TASK_NOT_PUSHED; 351 } 352 353 // Now that serialized tasks have returned, we can assume that we are not in 354 // immediate exec mode 355 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 356 if (!KMP_TASKING_ENABLED(task_team)) { 357 __kmp_enable_tasking(task_team, thread); 358 } 359 KMP_DEBUG_ASSERT(TCR_4(task_team->tt.tt_found_tasks) == TRUE); 360 KMP_DEBUG_ASSERT(TCR_PTR(task_team->tt.tt_threads_data) != NULL); 361 362 // Find tasking deque specific to encountering thread 363 thread_data = &task_team->tt.tt_threads_data[tid]; 364 365 // No lock needed since only owner can allocate 366 if (thread_data->td.td_deque == NULL) { 367 __kmp_alloc_task_deque(thread, thread_data); 368 } 369 370 int locked = 0; 371 // Check if deque is full 372 if (TCR_4(thread_data->td.td_deque_ntasks) >= 373 TASK_DEQUE_SIZE(thread_data->td)) { 374 if (__kmp_enable_task_throttling && 375 __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata, 376 thread->th.th_current_task)) { 377 KA_TRACE(20, ("__kmp_push_task: T#%d deque is full; returning " 378 "TASK_NOT_PUSHED for task %p\n", 379 gtid, taskdata)); 380 return TASK_NOT_PUSHED; 381 } else { 382 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 383 locked = 1; 384 // expand deque to push the task which is not allowed to execute 385 __kmp_realloc_task_deque(thread, thread_data); 386 } 387 } 388 // Lock the deque for the task push operation 389 if (!locked) { 390 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 391 // Need to recheck as we can get a proxy task from thread outside of OpenMP 392 if (TCR_4(thread_data->td.td_deque_ntasks) >= 393 TASK_DEQUE_SIZE(thread_data->td)) { 394 if (__kmp_enable_task_throttling && 395 __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata, 396 thread->th.th_current_task)) { 397 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 398 KA_TRACE(20, ("__kmp_push_task: T#%d deque is full on 2nd check; " 399 "returning TASK_NOT_PUSHED for task %p\n", 400 gtid, taskdata)); 401 return TASK_NOT_PUSHED; 402 } else { 403 // expand deque to push the task which is not allowed to execute 404 __kmp_realloc_task_deque(thread, thread_data); 405 } 406 } 407 } 408 // Must have room since no thread can add tasks but calling thread 409 KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) < 410 TASK_DEQUE_SIZE(thread_data->td)); 411 412 thread_data->td.td_deque[thread_data->td.td_deque_tail] = 413 taskdata; // Push taskdata 414 // Wrap index. 415 thread_data->td.td_deque_tail = 416 (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td); 417 TCW_4(thread_data->td.td_deque_ntasks, 418 TCR_4(thread_data->td.td_deque_ntasks) + 1); // Adjust task count 419 420 KA_TRACE(20, ("__kmp_push_task: T#%d returning TASK_SUCCESSFULLY_PUSHED: " 421 "task=%p ntasks=%d head=%u tail=%u\n", 422 gtid, taskdata, thread_data->td.td_deque_ntasks, 423 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 424 425 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 426 427 return TASK_SUCCESSFULLY_PUSHED; 428 } 429 430 // __kmp_pop_current_task_from_thread: set up current task from called thread 431 // when team ends 432 // 433 // this_thr: thread structure to set current_task in. 434 void __kmp_pop_current_task_from_thread(kmp_info_t *this_thr) { 435 KF_TRACE(10, ("__kmp_pop_current_task_from_thread(enter): T#%d " 436 "this_thread=%p, curtask=%p, " 437 "curtask_parent=%p\n", 438 0, this_thr, this_thr->th.th_current_task, 439 this_thr->th.th_current_task->td_parent)); 440 441 this_thr->th.th_current_task = this_thr->th.th_current_task->td_parent; 442 443 KF_TRACE(10, ("__kmp_pop_current_task_from_thread(exit): T#%d " 444 "this_thread=%p, curtask=%p, " 445 "curtask_parent=%p\n", 446 0, this_thr, this_thr->th.th_current_task, 447 this_thr->th.th_current_task->td_parent)); 448 } 449 450 // __kmp_push_current_task_to_thread: set up current task in called thread for a 451 // new team 452 // 453 // this_thr: thread structure to set up 454 // team: team for implicit task data 455 // tid: thread within team to set up 456 void __kmp_push_current_task_to_thread(kmp_info_t *this_thr, kmp_team_t *team, 457 int tid) { 458 // current task of the thread is a parent of the new just created implicit 459 // tasks of new team 460 KF_TRACE(10, ("__kmp_push_current_task_to_thread(enter): T#%d this_thread=%p " 461 "curtask=%p " 462 "parent_task=%p\n", 463 tid, this_thr, this_thr->th.th_current_task, 464 team->t.t_implicit_task_taskdata[tid].td_parent)); 465 466 KMP_DEBUG_ASSERT(this_thr != NULL); 467 468 if (tid == 0) { 469 if (this_thr->th.th_current_task != &team->t.t_implicit_task_taskdata[0]) { 470 team->t.t_implicit_task_taskdata[0].td_parent = 471 this_thr->th.th_current_task; 472 this_thr->th.th_current_task = &team->t.t_implicit_task_taskdata[0]; 473 } 474 } else { 475 team->t.t_implicit_task_taskdata[tid].td_parent = 476 team->t.t_implicit_task_taskdata[0].td_parent; 477 this_thr->th.th_current_task = &team->t.t_implicit_task_taskdata[tid]; 478 } 479 480 KF_TRACE(10, ("__kmp_push_current_task_to_thread(exit): T#%d this_thread=%p " 481 "curtask=%p " 482 "parent_task=%p\n", 483 tid, this_thr, this_thr->th.th_current_task, 484 team->t.t_implicit_task_taskdata[tid].td_parent)); 485 } 486 487 // __kmp_task_start: bookkeeping for a task starting execution 488 // 489 // GTID: global thread id of calling thread 490 // task: task starting execution 491 // current_task: task suspending 492 static void __kmp_task_start(kmp_int32 gtid, kmp_task_t *task, 493 kmp_taskdata_t *current_task) { 494 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 495 kmp_info_t *thread = __kmp_threads[gtid]; 496 497 KA_TRACE(10, 498 ("__kmp_task_start(enter): T#%d starting task %p: current_task=%p\n", 499 gtid, taskdata, current_task)); 500 501 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); 502 503 // mark currently executing task as suspended 504 // TODO: GEH - make sure root team implicit task is initialized properly. 505 // KMP_DEBUG_ASSERT( current_task -> td_flags.executing == 1 ); 506 current_task->td_flags.executing = 0; 507 508 // Add task to stack if tied 509 #ifdef BUILD_TIED_TASK_STACK 510 if (taskdata->td_flags.tiedness == TASK_TIED) { 511 __kmp_push_task_stack(gtid, thread, taskdata); 512 } 513 #endif /* BUILD_TIED_TASK_STACK */ 514 515 // mark starting task as executing and as current task 516 thread->th.th_current_task = taskdata; 517 518 KMP_DEBUG_ASSERT(taskdata->td_flags.started == 0 || 519 taskdata->td_flags.tiedness == TASK_UNTIED); 520 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0 || 521 taskdata->td_flags.tiedness == TASK_UNTIED); 522 taskdata->td_flags.started = 1; 523 taskdata->td_flags.executing = 1; 524 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); 525 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); 526 527 // GEH TODO: shouldn't we pass some sort of location identifier here? 528 // APT: yes, we will pass location here. 529 // need to store current thread state (in a thread or taskdata structure) 530 // before setting work_state, otherwise wrong state is set after end of task 531 532 KA_TRACE(10, ("__kmp_task_start(exit): T#%d task=%p\n", gtid, taskdata)); 533 534 return; 535 } 536 537 #if OMPT_SUPPORT 538 //------------------------------------------------------------------------------ 539 // __ompt_task_init: 540 // Initialize OMPT fields maintained by a task. This will only be called after 541 // ompt_start_tool, so we already know whether ompt is enabled or not. 542 543 static inline void __ompt_task_init(kmp_taskdata_t *task, int tid) { 544 // The calls to __ompt_task_init already have the ompt_enabled condition. 545 task->ompt_task_info.task_data.value = 0; 546 task->ompt_task_info.frame.exit_frame = ompt_data_none; 547 task->ompt_task_info.frame.enter_frame = ompt_data_none; 548 task->ompt_task_info.frame.exit_frame_flags = ompt_frame_runtime | ompt_frame_framepointer; 549 task->ompt_task_info.frame.enter_frame_flags = ompt_frame_runtime | ompt_frame_framepointer; 550 task->ompt_task_info.ndeps = 0; 551 task->ompt_task_info.deps = NULL; 552 } 553 554 // __ompt_task_start: 555 // Build and trigger task-begin event 556 static inline void __ompt_task_start(kmp_task_t *task, 557 kmp_taskdata_t *current_task, 558 kmp_int32 gtid) { 559 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 560 ompt_task_status_t status = ompt_task_switch; 561 if (__kmp_threads[gtid]->th.ompt_thread_info.ompt_task_yielded) { 562 status = ompt_task_yield; 563 __kmp_threads[gtid]->th.ompt_thread_info.ompt_task_yielded = 0; 564 } 565 /* let OMPT know that we're about to run this task */ 566 if (ompt_enabled.ompt_callback_task_schedule) { 567 ompt_callbacks.ompt_callback(ompt_callback_task_schedule)( 568 &(current_task->ompt_task_info.task_data), status, 569 &(taskdata->ompt_task_info.task_data)); 570 } 571 taskdata->ompt_task_info.scheduling_parent = current_task; 572 } 573 574 // __ompt_task_finish: 575 // Build and trigger final task-schedule event 576 static inline void 577 __ompt_task_finish(kmp_task_t *task, kmp_taskdata_t *resumed_task, 578 ompt_task_status_t status = ompt_task_complete) { 579 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 580 if (__kmp_omp_cancellation && taskdata->td_taskgroup && 581 taskdata->td_taskgroup->cancel_request == cancel_taskgroup) { 582 status = ompt_task_cancel; 583 } 584 585 /* let OMPT know that we're returning to the callee task */ 586 if (ompt_enabled.ompt_callback_task_schedule) { 587 ompt_callbacks.ompt_callback(ompt_callback_task_schedule)( 588 &(taskdata->ompt_task_info.task_data), status, 589 &((resumed_task ? resumed_task 590 : (taskdata->ompt_task_info.scheduling_parent 591 ? taskdata->ompt_task_info.scheduling_parent 592 : taskdata->td_parent)) 593 ->ompt_task_info.task_data)); 594 } 595 } 596 #endif 597 598 template <bool ompt> 599 static void __kmpc_omp_task_begin_if0_template(ident_t *loc_ref, kmp_int32 gtid, 600 kmp_task_t *task, 601 void *frame_address, 602 void *return_address) { 603 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 604 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task; 605 606 KA_TRACE(10, ("__kmpc_omp_task_begin_if0(enter): T#%d loc=%p task=%p " 607 "current_task=%p\n", 608 gtid, loc_ref, taskdata, current_task)); 609 610 if (taskdata->td_flags.tiedness == TASK_UNTIED) { 611 // untied task needs to increment counter so that the task structure is not 612 // freed prematurely 613 kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count); 614 KMP_DEBUG_USE_VAR(counter); 615 KA_TRACE(20, ("__kmpc_omp_task_begin_if0: T#%d untied_count (%d) " 616 "incremented for task %p\n", 617 gtid, counter, taskdata)); 618 } 619 620 taskdata->td_flags.task_serial = 621 1; // Execute this task immediately, not deferred. 622 __kmp_task_start(gtid, task, current_task); 623 624 #if OMPT_SUPPORT 625 if (ompt) { 626 if (current_task->ompt_task_info.frame.enter_frame.ptr == NULL) { 627 current_task->ompt_task_info.frame.enter_frame.ptr = 628 taskdata->ompt_task_info.frame.exit_frame.ptr = frame_address; 629 current_task->ompt_task_info.frame.enter_frame_flags = 630 taskdata->ompt_task_info.frame.exit_frame_flags = ompt_frame_application | ompt_frame_framepointer; 631 } 632 if (ompt_enabled.ompt_callback_task_create) { 633 ompt_task_info_t *parent_info = &(current_task->ompt_task_info); 634 ompt_callbacks.ompt_callback(ompt_callback_task_create)( 635 &(parent_info->task_data), &(parent_info->frame), 636 &(taskdata->ompt_task_info.task_data), 637 ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(taskdata), 0, 638 return_address); 639 } 640 __ompt_task_start(task, current_task, gtid); 641 } 642 #endif // OMPT_SUPPORT 643 644 KA_TRACE(10, ("__kmpc_omp_task_begin_if0(exit): T#%d loc=%p task=%p,\n", gtid, 645 loc_ref, taskdata)); 646 } 647 648 #if OMPT_SUPPORT 649 OMPT_NOINLINE 650 static void __kmpc_omp_task_begin_if0_ompt(ident_t *loc_ref, kmp_int32 gtid, 651 kmp_task_t *task, 652 void *frame_address, 653 void *return_address) { 654 __kmpc_omp_task_begin_if0_template<true>(loc_ref, gtid, task, frame_address, 655 return_address); 656 } 657 #endif // OMPT_SUPPORT 658 659 // __kmpc_omp_task_begin_if0: report that a given serialized task has started 660 // execution 661 // 662 // loc_ref: source location information; points to beginning of task block. 663 // gtid: global thread number. 664 // task: task thunk for the started task. 665 void __kmpc_omp_task_begin_if0(ident_t *loc_ref, kmp_int32 gtid, 666 kmp_task_t *task) { 667 #if OMPT_SUPPORT 668 if (UNLIKELY(ompt_enabled.enabled)) { 669 OMPT_STORE_RETURN_ADDRESS(gtid); 670 __kmpc_omp_task_begin_if0_ompt(loc_ref, gtid, task, 671 OMPT_GET_FRAME_ADDRESS(1), 672 OMPT_LOAD_RETURN_ADDRESS(gtid)); 673 return; 674 } 675 #endif 676 __kmpc_omp_task_begin_if0_template<false>(loc_ref, gtid, task, NULL, NULL); 677 } 678 679 #ifdef TASK_UNUSED 680 // __kmpc_omp_task_begin: report that a given task has started execution 681 // NEVER GENERATED BY COMPILER, DEPRECATED!!! 682 void __kmpc_omp_task_begin(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task) { 683 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task; 684 685 KA_TRACE( 686 10, 687 ("__kmpc_omp_task_begin(enter): T#%d loc=%p task=%p current_task=%p\n", 688 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task), current_task)); 689 690 __kmp_task_start(gtid, task, current_task); 691 692 KA_TRACE(10, ("__kmpc_omp_task_begin(exit): T#%d loc=%p task=%p,\n", gtid, 693 loc_ref, KMP_TASK_TO_TASKDATA(task))); 694 return; 695 } 696 #endif // TASK_UNUSED 697 698 // __kmp_free_task: free the current task space and the space for shareds 699 // 700 // gtid: Global thread ID of calling thread 701 // taskdata: task to free 702 // thread: thread data structure of caller 703 static void __kmp_free_task(kmp_int32 gtid, kmp_taskdata_t *taskdata, 704 kmp_info_t *thread) { 705 KA_TRACE(30, ("__kmp_free_task: T#%d freeing data from task %p\n", gtid, 706 taskdata)); 707 708 // Check to make sure all flags and counters have the correct values 709 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); 710 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0); 711 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 1); 712 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); 713 KMP_DEBUG_ASSERT(taskdata->td_allocated_child_tasks == 0 || 714 taskdata->td_flags.task_serial == 1); 715 KMP_DEBUG_ASSERT(taskdata->td_incomplete_child_tasks == 0); 716 717 taskdata->td_flags.freed = 1; 718 ANNOTATE_HAPPENS_BEFORE(taskdata); 719 // deallocate the taskdata and shared variable blocks associated with this task 720 #if USE_FAST_MEMORY 721 __kmp_fast_free(thread, taskdata); 722 #else /* ! USE_FAST_MEMORY */ 723 __kmp_thread_free(thread, taskdata); 724 #endif 725 726 KA_TRACE(20, ("__kmp_free_task: T#%d freed task %p\n", gtid, taskdata)); 727 } 728 729 // __kmp_free_task_and_ancestors: free the current task and ancestors without 730 // children 731 // 732 // gtid: Global thread ID of calling thread 733 // taskdata: task to free 734 // thread: thread data structure of caller 735 static void __kmp_free_task_and_ancestors(kmp_int32 gtid, 736 kmp_taskdata_t *taskdata, 737 kmp_info_t *thread) { 738 // Proxy tasks must always be allowed to free their parents 739 // because they can be run in background even in serial mode. 740 kmp_int32 team_serial = 741 (taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser) && 742 !taskdata->td_flags.proxy; 743 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); 744 745 kmp_int32 children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks) - 1; 746 KMP_DEBUG_ASSERT(children >= 0); 747 748 // Now, go up the ancestor tree to see if any ancestors can now be freed. 749 while (children == 0) { 750 kmp_taskdata_t *parent_taskdata = taskdata->td_parent; 751 752 KA_TRACE(20, ("__kmp_free_task_and_ancestors(enter): T#%d task %p complete " 753 "and freeing itself\n", 754 gtid, taskdata)); 755 756 // --- Deallocate my ancestor task --- 757 __kmp_free_task(gtid, taskdata, thread); 758 759 taskdata = parent_taskdata; 760 761 if (team_serial) 762 return; 763 // Stop checking ancestors at implicit task instead of walking up ancestor 764 // tree to avoid premature deallocation of ancestors. 765 if (taskdata->td_flags.tasktype == TASK_IMPLICIT) { 766 if (taskdata->td_dephash) { // do we need to cleanup dephash? 767 int children = KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks); 768 kmp_tasking_flags_t flags_old = taskdata->td_flags; 769 if (children == 0 && flags_old.complete == 1) { 770 kmp_tasking_flags_t flags_new = flags_old; 771 flags_new.complete = 0; 772 if (KMP_COMPARE_AND_STORE_ACQ32( 773 RCAST(kmp_int32 *, &taskdata->td_flags), 774 *RCAST(kmp_int32 *, &flags_old), 775 *RCAST(kmp_int32 *, &flags_new))) { 776 KA_TRACE(100, ("__kmp_free_task_and_ancestors: T#%d cleans " 777 "dephash of implicit task %p\n", 778 gtid, taskdata)); 779 // cleanup dephash of finished implicit task 780 __kmp_dephash_free_entries(thread, taskdata->td_dephash); 781 } 782 } 783 } 784 return; 785 } 786 // Predecrement simulated by "- 1" calculation 787 children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks) - 1; 788 KMP_DEBUG_ASSERT(children >= 0); 789 } 790 791 KA_TRACE( 792 20, ("__kmp_free_task_and_ancestors(exit): T#%d task %p has %d children; " 793 "not freeing it yet\n", 794 gtid, taskdata, children)); 795 } 796 797 // __kmp_task_finish: bookkeeping to do when a task finishes execution 798 // 799 // gtid: global thread ID for calling thread 800 // task: task to be finished 801 // resumed_task: task to be resumed. (may be NULL if task is serialized) 802 template <bool ompt> 803 static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task, 804 kmp_taskdata_t *resumed_task) { 805 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 806 kmp_info_t *thread = __kmp_threads[gtid]; 807 kmp_task_team_t *task_team = 808 thread->th.th_task_team; // might be NULL for serial teams... 809 kmp_int32 children = 0; 810 811 KA_TRACE(10, ("__kmp_task_finish(enter): T#%d finishing task %p and resuming " 812 "task %p\n", 813 gtid, taskdata, resumed_task)); 814 815 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); 816 817 // Pop task from stack if tied 818 #ifdef BUILD_TIED_TASK_STACK 819 if (taskdata->td_flags.tiedness == TASK_TIED) { 820 __kmp_pop_task_stack(gtid, thread, taskdata); 821 } 822 #endif /* BUILD_TIED_TASK_STACK */ 823 824 if (taskdata->td_flags.tiedness == TASK_UNTIED) { 825 // untied task needs to check the counter so that the task structure is not 826 // freed prematurely 827 kmp_int32 counter = KMP_ATOMIC_DEC(&taskdata->td_untied_count) - 1; 828 KA_TRACE( 829 20, 830 ("__kmp_task_finish: T#%d untied_count (%d) decremented for task %p\n", 831 gtid, counter, taskdata)); 832 if (counter > 0) { 833 // untied task is not done, to be continued possibly by other thread, do 834 // not free it now 835 if (resumed_task == NULL) { 836 KMP_DEBUG_ASSERT(taskdata->td_flags.task_serial); 837 resumed_task = taskdata->td_parent; // In a serialized task, the resumed 838 // task is the parent 839 } 840 thread->th.th_current_task = resumed_task; // restore current_task 841 resumed_task->td_flags.executing = 1; // resume previous task 842 KA_TRACE(10, ("__kmp_task_finish(exit): T#%d partially done task %p, " 843 "resuming task %p\n", 844 gtid, taskdata, resumed_task)); 845 return; 846 } 847 } 848 #if OMPT_SUPPORT 849 if (ompt) 850 __ompt_task_finish(task, resumed_task); 851 #endif 852 853 // Check mutexinoutset dependencies, release locks 854 kmp_depnode_t *node = taskdata->td_depnode; 855 if (node && (node->dn.mtx_num_locks < 0)) { 856 // negative num_locks means all locks were acquired 857 node->dn.mtx_num_locks = -node->dn.mtx_num_locks; 858 for (int i = node->dn.mtx_num_locks - 1; i >= 0; --i) { 859 KMP_DEBUG_ASSERT(node->dn.mtx_locks[i] != NULL); 860 __kmp_release_lock(node->dn.mtx_locks[i], gtid); 861 } 862 } 863 864 // bookkeeping for resuming task: 865 // GEH - note tasking_ser => task_serial 866 KMP_DEBUG_ASSERT( 867 (taskdata->td_flags.tasking_ser || taskdata->td_flags.task_serial) == 868 taskdata->td_flags.task_serial); 869 if (taskdata->td_flags.task_serial) { 870 if (resumed_task == NULL) { 871 resumed_task = taskdata->td_parent; // In a serialized task, the resumed 872 // task is the parent 873 } 874 } else { 875 KMP_DEBUG_ASSERT(resumed_task != 876 NULL); // verify that resumed task is passed as argument 877 } 878 879 /* If the tasks' destructor thunk flag has been set, we need to invoke the 880 destructor thunk that has been generated by the compiler. The code is 881 placed here, since at this point other tasks might have been released 882 hence overlapping the destructor invocations with some other work in the 883 released tasks. The OpenMP spec is not specific on when the destructors 884 are invoked, so we should be free to choose. */ 885 if (taskdata->td_flags.destructors_thunk) { 886 kmp_routine_entry_t destr_thunk = task->data1.destructors; 887 KMP_ASSERT(destr_thunk); 888 destr_thunk(gtid, task); 889 } 890 891 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); 892 KMP_DEBUG_ASSERT(taskdata->td_flags.started == 1); 893 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); 894 895 bool detach = false; 896 if (taskdata->td_flags.detachable == TASK_DETACHABLE) { 897 if (taskdata->td_allow_completion_event.type == 898 KMP_EVENT_ALLOW_COMPLETION) { 899 // event hasn't been fulfilled yet. Try to detach task. 900 __kmp_acquire_tas_lock(&taskdata->td_allow_completion_event.lock, gtid); 901 if (taskdata->td_allow_completion_event.type == 902 KMP_EVENT_ALLOW_COMPLETION) { 903 // task finished execution 904 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1); 905 taskdata->td_flags.executing = 0; // suspend the finishing task 906 // no access to taskdata after this point! 907 // __kmp_fulfill_event might free taskdata at any time from now 908 taskdata->td_flags.proxy = TASK_PROXY; // proxify! 909 detach = true; 910 } 911 __kmp_release_tas_lock(&taskdata->td_allow_completion_event.lock, gtid); 912 } 913 } 914 915 if (!detach) { 916 taskdata->td_flags.complete = 1; // mark the task as completed 917 918 // Only need to keep track of count if team parallel and tasking not 919 // serialized, or task is detachable and event has already been fulfilled 920 if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser) || 921 taskdata->td_flags.detachable == TASK_DETACHABLE) { 922 // Predecrement simulated by "- 1" calculation 923 children = 924 KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks) - 1; 925 KMP_DEBUG_ASSERT(children >= 0); 926 if (taskdata->td_taskgroup) 927 KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count); 928 __kmp_release_deps(gtid, taskdata); 929 } else if (task_team && task_team->tt.tt_found_proxy_tasks) { 930 // if we found proxy tasks there could exist a dependency chain 931 // with the proxy task as origin 932 __kmp_release_deps(gtid, taskdata); 933 } 934 // td_flags.executing must be marked as 0 after __kmp_release_deps has been 935 // called. Othertwise, if a task is executed immediately from the 936 // release_deps code, the flag will be reset to 1 again by this same 937 // function 938 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1); 939 taskdata->td_flags.executing = 0; // suspend the finishing task 940 } 941 942 943 KA_TRACE( 944 20, ("__kmp_task_finish: T#%d finished task %p, %d incomplete children\n", 945 gtid, taskdata, children)); 946 947 // Free this task and then ancestor tasks if they have no children. 948 // Restore th_current_task first as suggested by John: 949 // johnmc: if an asynchronous inquiry peers into the runtime system 950 // it doesn't see the freed task as the current task. 951 thread->th.th_current_task = resumed_task; 952 if (!detach) 953 __kmp_free_task_and_ancestors(gtid, taskdata, thread); 954 955 // TODO: GEH - make sure root team implicit task is initialized properly. 956 // KMP_DEBUG_ASSERT( resumed_task->td_flags.executing == 0 ); 957 resumed_task->td_flags.executing = 1; // resume previous task 958 959 KA_TRACE( 960 10, ("__kmp_task_finish(exit): T#%d finished task %p, resuming task %p\n", 961 gtid, taskdata, resumed_task)); 962 963 return; 964 } 965 966 template <bool ompt> 967 static void __kmpc_omp_task_complete_if0_template(ident_t *loc_ref, 968 kmp_int32 gtid, 969 kmp_task_t *task) { 970 KA_TRACE(10, ("__kmpc_omp_task_complete_if0(enter): T#%d loc=%p task=%p\n", 971 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task))); 972 // this routine will provide task to resume 973 __kmp_task_finish<ompt>(gtid, task, NULL); 974 975 KA_TRACE(10, ("__kmpc_omp_task_complete_if0(exit): T#%d loc=%p task=%p\n", 976 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task))); 977 978 #if OMPT_SUPPORT 979 if (ompt) { 980 ompt_frame_t *ompt_frame; 981 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 982 ompt_frame->enter_frame = ompt_data_none; 983 ompt_frame->enter_frame_flags = ompt_frame_runtime | ompt_frame_framepointer; 984 } 985 #endif 986 987 return; 988 } 989 990 #if OMPT_SUPPORT 991 OMPT_NOINLINE 992 void __kmpc_omp_task_complete_if0_ompt(ident_t *loc_ref, kmp_int32 gtid, 993 kmp_task_t *task) { 994 __kmpc_omp_task_complete_if0_template<true>(loc_ref, gtid, task); 995 } 996 #endif // OMPT_SUPPORT 997 998 // __kmpc_omp_task_complete_if0: report that a task has completed execution 999 // 1000 // loc_ref: source location information; points to end of task block. 1001 // gtid: global thread number. 1002 // task: task thunk for the completed task. 1003 void __kmpc_omp_task_complete_if0(ident_t *loc_ref, kmp_int32 gtid, 1004 kmp_task_t *task) { 1005 #if OMPT_SUPPORT 1006 if (UNLIKELY(ompt_enabled.enabled)) { 1007 __kmpc_omp_task_complete_if0_ompt(loc_ref, gtid, task); 1008 return; 1009 } 1010 #endif 1011 __kmpc_omp_task_complete_if0_template<false>(loc_ref, gtid, task); 1012 } 1013 1014 #ifdef TASK_UNUSED 1015 // __kmpc_omp_task_complete: report that a task has completed execution 1016 // NEVER GENERATED BY COMPILER, DEPRECATED!!! 1017 void __kmpc_omp_task_complete(ident_t *loc_ref, kmp_int32 gtid, 1018 kmp_task_t *task) { 1019 KA_TRACE(10, ("__kmpc_omp_task_complete(enter): T#%d loc=%p task=%p\n", gtid, 1020 loc_ref, KMP_TASK_TO_TASKDATA(task))); 1021 1022 __kmp_task_finish<false>(gtid, task, 1023 NULL); // Not sure how to find task to resume 1024 1025 KA_TRACE(10, ("__kmpc_omp_task_complete(exit): T#%d loc=%p task=%p\n", gtid, 1026 loc_ref, KMP_TASK_TO_TASKDATA(task))); 1027 return; 1028 } 1029 #endif // TASK_UNUSED 1030 1031 // __kmp_init_implicit_task: Initialize the appropriate fields in the implicit 1032 // task for a given thread 1033 // 1034 // loc_ref: reference to source location of parallel region 1035 // this_thr: thread data structure corresponding to implicit task 1036 // team: team for this_thr 1037 // tid: thread id of given thread within team 1038 // set_curr_task: TRUE if need to push current task to thread 1039 // NOTE: Routine does not set up the implicit task ICVS. This is assumed to 1040 // have already been done elsewhere. 1041 // TODO: Get better loc_ref. Value passed in may be NULL 1042 void __kmp_init_implicit_task(ident_t *loc_ref, kmp_info_t *this_thr, 1043 kmp_team_t *team, int tid, int set_curr_task) { 1044 kmp_taskdata_t *task = &team->t.t_implicit_task_taskdata[tid]; 1045 1046 KF_TRACE( 1047 10, 1048 ("__kmp_init_implicit_task(enter): T#:%d team=%p task=%p, reinit=%s\n", 1049 tid, team, task, set_curr_task ? "TRUE" : "FALSE")); 1050 1051 task->td_task_id = KMP_GEN_TASK_ID(); 1052 task->td_team = team; 1053 // task->td_parent = NULL; // fix for CQ230101 (broken parent task info 1054 // in debugger) 1055 task->td_ident = loc_ref; 1056 task->td_taskwait_ident = NULL; 1057 task->td_taskwait_counter = 0; 1058 task->td_taskwait_thread = 0; 1059 1060 task->td_flags.tiedness = TASK_TIED; 1061 task->td_flags.tasktype = TASK_IMPLICIT; 1062 task->td_flags.proxy = TASK_FULL; 1063 1064 // All implicit tasks are executed immediately, not deferred 1065 task->td_flags.task_serial = 1; 1066 task->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec); 1067 task->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0; 1068 1069 task->td_flags.started = 1; 1070 task->td_flags.executing = 1; 1071 task->td_flags.complete = 0; 1072 task->td_flags.freed = 0; 1073 1074 task->td_depnode = NULL; 1075 task->td_last_tied = task; 1076 task->td_allow_completion_event.type = KMP_EVENT_UNINITIALIZED; 1077 1078 if (set_curr_task) { // only do this init first time thread is created 1079 KMP_ATOMIC_ST_REL(&task->td_incomplete_child_tasks, 0); 1080 // Not used: don't need to deallocate implicit task 1081 KMP_ATOMIC_ST_REL(&task->td_allocated_child_tasks, 0); 1082 task->td_taskgroup = NULL; // An implicit task does not have taskgroup 1083 task->td_dephash = NULL; 1084 __kmp_push_current_task_to_thread(this_thr, team, tid); 1085 } else { 1086 KMP_DEBUG_ASSERT(task->td_incomplete_child_tasks == 0); 1087 KMP_DEBUG_ASSERT(task->td_allocated_child_tasks == 0); 1088 } 1089 1090 #if OMPT_SUPPORT 1091 if (UNLIKELY(ompt_enabled.enabled)) 1092 __ompt_task_init(task, tid); 1093 #endif 1094 1095 KF_TRACE(10, ("__kmp_init_implicit_task(exit): T#:%d team=%p task=%p\n", tid, 1096 team, task)); 1097 } 1098 1099 // __kmp_finish_implicit_task: Release resources associated to implicit tasks 1100 // at the end of parallel regions. Some resources are kept for reuse in the next 1101 // parallel region. 1102 // 1103 // thread: thread data structure corresponding to implicit task 1104 void __kmp_finish_implicit_task(kmp_info_t *thread) { 1105 kmp_taskdata_t *task = thread->th.th_current_task; 1106 if (task->td_dephash) { 1107 int children; 1108 task->td_flags.complete = 1; 1109 children = KMP_ATOMIC_LD_ACQ(&task->td_incomplete_child_tasks); 1110 kmp_tasking_flags_t flags_old = task->td_flags; 1111 if (children == 0 && flags_old.complete == 1) { 1112 kmp_tasking_flags_t flags_new = flags_old; 1113 flags_new.complete = 0; 1114 if (KMP_COMPARE_AND_STORE_ACQ32(RCAST(kmp_int32 *, &task->td_flags), 1115 *RCAST(kmp_int32 *, &flags_old), 1116 *RCAST(kmp_int32 *, &flags_new))) { 1117 KA_TRACE(100, ("__kmp_finish_implicit_task: T#%d cleans " 1118 "dephash of implicit task %p\n", 1119 thread->th.th_info.ds.ds_gtid, task)); 1120 __kmp_dephash_free_entries(thread, task->td_dephash); 1121 } 1122 } 1123 } 1124 } 1125 1126 // __kmp_free_implicit_task: Release resources associated to implicit tasks 1127 // when these are destroyed regions 1128 // 1129 // thread: thread data structure corresponding to implicit task 1130 void __kmp_free_implicit_task(kmp_info_t *thread) { 1131 kmp_taskdata_t *task = thread->th.th_current_task; 1132 if (task && task->td_dephash) { 1133 __kmp_dephash_free(thread, task->td_dephash); 1134 task->td_dephash = NULL; 1135 } 1136 } 1137 1138 // Round up a size to a power of two specified by val: Used to insert padding 1139 // between structures co-allocated using a single malloc() call 1140 static size_t __kmp_round_up_to_val(size_t size, size_t val) { 1141 if (size & (val - 1)) { 1142 size &= ~(val - 1); 1143 if (size <= KMP_SIZE_T_MAX - val) { 1144 size += val; // Round up if there is no overflow. 1145 } 1146 } 1147 return size; 1148 } // __kmp_round_up_to_va 1149 1150 // __kmp_task_alloc: Allocate the taskdata and task data structures for a task 1151 // 1152 // loc_ref: source location information 1153 // gtid: global thread number. 1154 // flags: include tiedness & task type (explicit vs. implicit) of the ''new'' 1155 // task encountered. Converted from kmp_int32 to kmp_tasking_flags_t in routine. 1156 // sizeof_kmp_task_t: Size in bytes of kmp_task_t data structure including 1157 // private vars accessed in task. 1158 // sizeof_shareds: Size in bytes of array of pointers to shared vars accessed 1159 // in task. 1160 // task_entry: Pointer to task code entry point generated by compiler. 1161 // returns: a pointer to the allocated kmp_task_t structure (task). 1162 kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid, 1163 kmp_tasking_flags_t *flags, 1164 size_t sizeof_kmp_task_t, size_t sizeof_shareds, 1165 kmp_routine_entry_t task_entry) { 1166 kmp_task_t *task; 1167 kmp_taskdata_t *taskdata; 1168 kmp_info_t *thread = __kmp_threads[gtid]; 1169 kmp_team_t *team = thread->th.th_team; 1170 kmp_taskdata_t *parent_task = thread->th.th_current_task; 1171 size_t shareds_offset; 1172 1173 if (!TCR_4(__kmp_init_middle)) 1174 __kmp_middle_initialize(); 1175 1176 KA_TRACE(10, ("__kmp_task_alloc(enter): T#%d loc=%p, flags=(0x%x) " 1177 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n", 1178 gtid, loc_ref, *((kmp_int32 *)flags), sizeof_kmp_task_t, 1179 sizeof_shareds, task_entry)); 1180 1181 if (parent_task->td_flags.final) { 1182 if (flags->merged_if0) { 1183 } 1184 flags->final = 1; 1185 } 1186 if (flags->tiedness == TASK_UNTIED && !team->t.t_serialized) { 1187 // Untied task encountered causes the TSC algorithm to check entire deque of 1188 // the victim thread. If no untied task encountered, then checking the head 1189 // of the deque should be enough. 1190 KMP_CHECK_UPDATE(thread->th.th_task_team->tt.tt_untied_task_encountered, 1); 1191 } 1192 1193 // Detachable tasks are not proxy tasks yet but could be in the future. Doing 1194 // the tasking setup 1195 // when that happens is too late. 1196 if (flags->proxy == TASK_PROXY || flags->detachable == TASK_DETACHABLE) { 1197 if (flags->proxy == TASK_PROXY) { 1198 flags->tiedness = TASK_UNTIED; 1199 flags->merged_if0 = 1; 1200 } 1201 /* are we running in a sequential parallel or tskm_immediate_exec... we need 1202 tasking support enabled */ 1203 if ((thread->th.th_task_team) == NULL) { 1204 /* This should only happen if the team is serialized 1205 setup a task team and propagate it to the thread */ 1206 KMP_DEBUG_ASSERT(team->t.t_serialized); 1207 KA_TRACE(30, 1208 ("T#%d creating task team in __kmp_task_alloc for proxy task\n", 1209 gtid)); 1210 __kmp_task_team_setup( 1211 thread, team, 1212 1); // 1 indicates setup the current team regardless of nthreads 1213 thread->th.th_task_team = team->t.t_task_team[thread->th.th_task_state]; 1214 } 1215 kmp_task_team_t *task_team = thread->th.th_task_team; 1216 1217 /* tasking must be enabled now as the task might not be pushed */ 1218 if (!KMP_TASKING_ENABLED(task_team)) { 1219 KA_TRACE( 1220 30, 1221 ("T#%d enabling tasking in __kmp_task_alloc for proxy task\n", gtid)); 1222 __kmp_enable_tasking(task_team, thread); 1223 kmp_int32 tid = thread->th.th_info.ds.ds_tid; 1224 kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid]; 1225 // No lock needed since only owner can allocate 1226 if (thread_data->td.td_deque == NULL) { 1227 __kmp_alloc_task_deque(thread, thread_data); 1228 } 1229 } 1230 1231 if (task_team->tt.tt_found_proxy_tasks == FALSE) 1232 TCW_4(task_team->tt.tt_found_proxy_tasks, TRUE); 1233 } 1234 1235 // Calculate shared structure offset including padding after kmp_task_t struct 1236 // to align pointers in shared struct 1237 shareds_offset = sizeof(kmp_taskdata_t) + sizeof_kmp_task_t; 1238 shareds_offset = __kmp_round_up_to_val(shareds_offset, sizeof(void *)); 1239 1240 // Allocate a kmp_taskdata_t block and a kmp_task_t block. 1241 KA_TRACE(30, ("__kmp_task_alloc: T#%d First malloc size: %ld\n", gtid, 1242 shareds_offset)); 1243 KA_TRACE(30, ("__kmp_task_alloc: T#%d Second malloc size: %ld\n", gtid, 1244 sizeof_shareds)); 1245 1246 // Avoid double allocation here by combining shareds with taskdata 1247 #if USE_FAST_MEMORY 1248 taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, shareds_offset + 1249 sizeof_shareds); 1250 #else /* ! USE_FAST_MEMORY */ 1251 taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, shareds_offset + 1252 sizeof_shareds); 1253 #endif /* USE_FAST_MEMORY */ 1254 ANNOTATE_HAPPENS_AFTER(taskdata); 1255 1256 task = KMP_TASKDATA_TO_TASK(taskdata); 1257 1258 // Make sure task & taskdata are aligned appropriately 1259 #if KMP_ARCH_X86 || KMP_ARCH_PPC64 || !KMP_HAVE_QUAD 1260 KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(double) - 1)) == 0); 1261 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (sizeof(double) - 1)) == 0); 1262 #else 1263 KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(_Quad) - 1)) == 0); 1264 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (sizeof(_Quad) - 1)) == 0); 1265 #endif 1266 if (sizeof_shareds > 0) { 1267 // Avoid double allocation here by combining shareds with taskdata 1268 task->shareds = &((char *)taskdata)[shareds_offset]; 1269 // Make sure shareds struct is aligned to pointer size 1270 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) == 1271 0); 1272 } else { 1273 task->shareds = NULL; 1274 } 1275 task->routine = task_entry; 1276 task->part_id = 0; // AC: Always start with 0 part id 1277 1278 taskdata->td_task_id = KMP_GEN_TASK_ID(); 1279 taskdata->td_team = team; 1280 taskdata->td_alloc_thread = thread; 1281 taskdata->td_parent = parent_task; 1282 taskdata->td_level = parent_task->td_level + 1; // increment nesting level 1283 KMP_ATOMIC_ST_RLX(&taskdata->td_untied_count, 0); 1284 taskdata->td_ident = loc_ref; 1285 taskdata->td_taskwait_ident = NULL; 1286 taskdata->td_taskwait_counter = 0; 1287 taskdata->td_taskwait_thread = 0; 1288 KMP_DEBUG_ASSERT(taskdata->td_parent != NULL); 1289 // avoid copying icvs for proxy tasks 1290 if (flags->proxy == TASK_FULL) 1291 copy_icvs(&taskdata->td_icvs, &taskdata->td_parent->td_icvs); 1292 1293 taskdata->td_flags.tiedness = flags->tiedness; 1294 taskdata->td_flags.final = flags->final; 1295 taskdata->td_flags.merged_if0 = flags->merged_if0; 1296 taskdata->td_flags.destructors_thunk = flags->destructors_thunk; 1297 taskdata->td_flags.proxy = flags->proxy; 1298 taskdata->td_flags.detachable = flags->detachable; 1299 taskdata->td_task_team = thread->th.th_task_team; 1300 taskdata->td_size_alloc = shareds_offset + sizeof_shareds; 1301 taskdata->td_flags.tasktype = TASK_EXPLICIT; 1302 1303 // GEH - TODO: fix this to copy parent task's value of tasking_ser flag 1304 taskdata->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec); 1305 1306 // GEH - TODO: fix this to copy parent task's value of team_serial flag 1307 taskdata->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0; 1308 1309 // GEH - Note we serialize the task if the team is serialized to make sure 1310 // implicit parallel region tasks are not left until program termination to 1311 // execute. Also, it helps locality to execute immediately. 1312 1313 taskdata->td_flags.task_serial = 1314 (parent_task->td_flags.final || taskdata->td_flags.team_serial || 1315 taskdata->td_flags.tasking_ser); 1316 1317 taskdata->td_flags.started = 0; 1318 taskdata->td_flags.executing = 0; 1319 taskdata->td_flags.complete = 0; 1320 taskdata->td_flags.freed = 0; 1321 1322 taskdata->td_flags.native = flags->native; 1323 1324 KMP_ATOMIC_ST_RLX(&taskdata->td_incomplete_child_tasks, 0); 1325 // start at one because counts current task and children 1326 KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 1); 1327 taskdata->td_taskgroup = 1328 parent_task->td_taskgroup; // task inherits taskgroup from the parent task 1329 taskdata->td_dephash = NULL; 1330 taskdata->td_depnode = NULL; 1331 if (flags->tiedness == TASK_UNTIED) 1332 taskdata->td_last_tied = NULL; // will be set when the task is scheduled 1333 else 1334 taskdata->td_last_tied = taskdata; 1335 taskdata->td_allow_completion_event.type = KMP_EVENT_UNINITIALIZED; 1336 #if OMPT_SUPPORT 1337 if (UNLIKELY(ompt_enabled.enabled)) 1338 __ompt_task_init(taskdata, gtid); 1339 #endif 1340 // Only need to keep track of child task counts if team parallel and tasking not 1341 // serialized or if it is a proxy or detachable task 1342 if (flags->proxy == TASK_PROXY || 1343 flags->detachable == TASK_DETACHABLE || 1344 !(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) 1345 { 1346 KMP_ATOMIC_INC(&parent_task->td_incomplete_child_tasks); 1347 if (parent_task->td_taskgroup) 1348 KMP_ATOMIC_INC(&parent_task->td_taskgroup->count); 1349 // Only need to keep track of allocated child tasks for explicit tasks since 1350 // implicit not deallocated 1351 if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT) { 1352 KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks); 1353 } 1354 } 1355 1356 KA_TRACE(20, ("__kmp_task_alloc(exit): T#%d created task %p parent=%p\n", 1357 gtid, taskdata, taskdata->td_parent)); 1358 ANNOTATE_HAPPENS_BEFORE(task); 1359 1360 return task; 1361 } 1362 1363 kmp_task_t *__kmpc_omp_task_alloc(ident_t *loc_ref, kmp_int32 gtid, 1364 kmp_int32 flags, size_t sizeof_kmp_task_t, 1365 size_t sizeof_shareds, 1366 kmp_routine_entry_t task_entry) { 1367 kmp_task_t *retval; 1368 kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *)&flags; 1369 1370 input_flags->native = FALSE; 1371 // __kmp_task_alloc() sets up all other runtime flags 1372 1373 KA_TRACE(10, ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s %s %s) " 1374 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n", 1375 gtid, loc_ref, input_flags->tiedness ? "tied " : "untied", 1376 input_flags->proxy ? "proxy" : "", 1377 input_flags->detachable ? "detachable" : "", sizeof_kmp_task_t, 1378 sizeof_shareds, task_entry)); 1379 1380 retval = __kmp_task_alloc(loc_ref, gtid, input_flags, sizeof_kmp_task_t, 1381 sizeof_shareds, task_entry); 1382 1383 KA_TRACE(20, ("__kmpc_omp_task_alloc(exit): T#%d retval %p\n", gtid, retval)); 1384 1385 return retval; 1386 } 1387 1388 kmp_task_t *__kmpc_omp_target_task_alloc(ident_t *loc_ref, kmp_int32 gtid, 1389 kmp_int32 flags, 1390 size_t sizeof_kmp_task_t, 1391 size_t sizeof_shareds, 1392 kmp_routine_entry_t task_entry, 1393 kmp_int64 device_id) { 1394 return __kmpc_omp_task_alloc(loc_ref, gtid, flags, sizeof_kmp_task_t, 1395 sizeof_shareds, task_entry); 1396 } 1397 1398 /*! 1399 @ingroup TASKING 1400 @param loc_ref location of the original task directive 1401 @param gtid Global Thread ID of encountering thread 1402 @param new_task task thunk allocated by __kmpc_omp_task_alloc() for the ''new 1403 task'' 1404 @param naffins Number of affinity items 1405 @param affin_list List of affinity items 1406 @return Returns non-zero if registering affinity information was not successful. 1407 Returns 0 if registration was successful 1408 This entry registers the affinity information attached to a task with the task 1409 thunk structure kmp_taskdata_t. 1410 */ 1411 kmp_int32 1412 __kmpc_omp_reg_task_with_affinity(ident_t *loc_ref, kmp_int32 gtid, 1413 kmp_task_t *new_task, kmp_int32 naffins, 1414 kmp_task_affinity_info_t *affin_list) { 1415 return 0; 1416 } 1417 1418 // __kmp_invoke_task: invoke the specified task 1419 // 1420 // gtid: global thread ID of caller 1421 // task: the task to invoke 1422 // current_task: the task to resume after task invocation 1423 static void __kmp_invoke_task(kmp_int32 gtid, kmp_task_t *task, 1424 kmp_taskdata_t *current_task) { 1425 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 1426 kmp_info_t *thread; 1427 int discard = 0 /* false */; 1428 KA_TRACE( 1429 30, ("__kmp_invoke_task(enter): T#%d invoking task %p, current_task=%p\n", 1430 gtid, taskdata, current_task)); 1431 KMP_DEBUG_ASSERT(task); 1432 if (taskdata->td_flags.proxy == TASK_PROXY && 1433 taskdata->td_flags.complete == 1) { 1434 // This is a proxy task that was already completed but it needs to run 1435 // its bottom-half finish 1436 KA_TRACE( 1437 30, 1438 ("__kmp_invoke_task: T#%d running bottom finish for proxy task %p\n", 1439 gtid, taskdata)); 1440 1441 __kmp_bottom_half_finish_proxy(gtid, task); 1442 1443 KA_TRACE(30, ("__kmp_invoke_task(exit): T#%d completed bottom finish for " 1444 "proxy task %p, resuming task %p\n", 1445 gtid, taskdata, current_task)); 1446 1447 return; 1448 } 1449 1450 #if OMPT_SUPPORT 1451 // For untied tasks, the first task executed only calls __kmpc_omp_task and 1452 // does not execute code. 1453 ompt_thread_info_t oldInfo; 1454 if (UNLIKELY(ompt_enabled.enabled)) { 1455 // Store the threads states and restore them after the task 1456 thread = __kmp_threads[gtid]; 1457 oldInfo = thread->th.ompt_thread_info; 1458 thread->th.ompt_thread_info.wait_id = 0; 1459 thread->th.ompt_thread_info.state = (thread->th.th_team_serialized) 1460 ? ompt_state_work_serial 1461 : ompt_state_work_parallel; 1462 taskdata->ompt_task_info.frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1463 } 1464 #endif 1465 1466 // Proxy tasks are not handled by the runtime 1467 if (taskdata->td_flags.proxy != TASK_PROXY) { 1468 ANNOTATE_HAPPENS_AFTER(task); 1469 __kmp_task_start(gtid, task, current_task); // OMPT only if not discarded 1470 } 1471 1472 // TODO: cancel tasks if the parallel region has also been cancelled 1473 // TODO: check if this sequence can be hoisted above __kmp_task_start 1474 // if cancellation has been enabled for this run ... 1475 if (__kmp_omp_cancellation) { 1476 thread = __kmp_threads[gtid]; 1477 kmp_team_t *this_team = thread->th.th_team; 1478 kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup; 1479 if ((taskgroup && taskgroup->cancel_request) || 1480 (this_team->t.t_cancel_request == cancel_parallel)) { 1481 #if OMPT_SUPPORT && OMPT_OPTIONAL 1482 ompt_data_t *task_data; 1483 if (UNLIKELY(ompt_enabled.ompt_callback_cancel)) { 1484 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL, NULL); 1485 ompt_callbacks.ompt_callback(ompt_callback_cancel)( 1486 task_data, 1487 ((taskgroup && taskgroup->cancel_request) ? ompt_cancel_taskgroup 1488 : ompt_cancel_parallel) | 1489 ompt_cancel_discarded_task, 1490 NULL); 1491 } 1492 #endif 1493 KMP_COUNT_BLOCK(TASK_cancelled); 1494 // this task belongs to a task group and we need to cancel it 1495 discard = 1 /* true */; 1496 } 1497 } 1498 1499 // Invoke the task routine and pass in relevant data. 1500 // Thunks generated by gcc take a different argument list. 1501 if (!discard) { 1502 if (taskdata->td_flags.tiedness == TASK_UNTIED) { 1503 taskdata->td_last_tied = current_task->td_last_tied; 1504 KMP_DEBUG_ASSERT(taskdata->td_last_tied); 1505 } 1506 #if KMP_STATS_ENABLED 1507 KMP_COUNT_BLOCK(TASK_executed); 1508 switch (KMP_GET_THREAD_STATE()) { 1509 case FORK_JOIN_BARRIER: 1510 KMP_PUSH_PARTITIONED_TIMER(OMP_task_join_bar); 1511 break; 1512 case PLAIN_BARRIER: 1513 KMP_PUSH_PARTITIONED_TIMER(OMP_task_plain_bar); 1514 break; 1515 case TASKYIELD: 1516 KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskyield); 1517 break; 1518 case TASKWAIT: 1519 KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskwait); 1520 break; 1521 case TASKGROUP: 1522 KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskgroup); 1523 break; 1524 default: 1525 KMP_PUSH_PARTITIONED_TIMER(OMP_task_immediate); 1526 break; 1527 } 1528 #endif // KMP_STATS_ENABLED 1529 1530 // OMPT task begin 1531 #if OMPT_SUPPORT 1532 if (UNLIKELY(ompt_enabled.enabled)) 1533 __ompt_task_start(task, current_task, gtid); 1534 #endif 1535 1536 #if USE_ITT_BUILD && USE_ITT_NOTIFY 1537 kmp_uint64 cur_time; 1538 kmp_int32 kmp_itt_count_task = 1539 __kmp_forkjoin_frames_mode == 3 && !taskdata->td_flags.task_serial && 1540 current_task->td_flags.tasktype == TASK_IMPLICIT; 1541 if (kmp_itt_count_task) { 1542 thread = __kmp_threads[gtid]; 1543 // Time outer level explicit task on barrier for adjusting imbalance time 1544 if (thread->th.th_bar_arrive_time) 1545 cur_time = __itt_get_timestamp(); 1546 else 1547 kmp_itt_count_task = 0; // thread is not on a barrier - skip timing 1548 } 1549 #endif 1550 1551 #ifdef KMP_GOMP_COMPAT 1552 if (taskdata->td_flags.native) { 1553 ((void (*)(void *))(*(task->routine)))(task->shareds); 1554 } else 1555 #endif /* KMP_GOMP_COMPAT */ 1556 { 1557 (*(task->routine))(gtid, task); 1558 } 1559 KMP_POP_PARTITIONED_TIMER(); 1560 1561 #if USE_ITT_BUILD && USE_ITT_NOTIFY 1562 if (kmp_itt_count_task) { 1563 // Barrier imbalance - adjust arrive time with the task duration 1564 thread->th.th_bar_arrive_time += (__itt_get_timestamp() - cur_time); 1565 } 1566 #endif 1567 1568 } 1569 1570 1571 // Proxy tasks are not handled by the runtime 1572 if (taskdata->td_flags.proxy != TASK_PROXY) { 1573 ANNOTATE_HAPPENS_BEFORE(taskdata->td_parent); 1574 #if OMPT_SUPPORT 1575 if (UNLIKELY(ompt_enabled.enabled)) { 1576 thread->th.ompt_thread_info = oldInfo; 1577 if (taskdata->td_flags.tiedness == TASK_TIED) { 1578 taskdata->ompt_task_info.frame.exit_frame = ompt_data_none; 1579 } 1580 __kmp_task_finish<true>(gtid, task, current_task); 1581 } else 1582 #endif 1583 __kmp_task_finish<false>(gtid, task, current_task); 1584 } 1585 1586 KA_TRACE( 1587 30, 1588 ("__kmp_invoke_task(exit): T#%d completed task %p, resuming task %p\n", 1589 gtid, taskdata, current_task)); 1590 return; 1591 } 1592 1593 // __kmpc_omp_task_parts: Schedule a thread-switchable task for execution 1594 // 1595 // loc_ref: location of original task pragma (ignored) 1596 // gtid: Global Thread ID of encountering thread 1597 // new_task: task thunk allocated by __kmp_omp_task_alloc() for the ''new task'' 1598 // Returns: 1599 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to 1600 // be resumed later. 1601 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be 1602 // resumed later. 1603 kmp_int32 __kmpc_omp_task_parts(ident_t *loc_ref, kmp_int32 gtid, 1604 kmp_task_t *new_task) { 1605 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task); 1606 1607 KA_TRACE(10, ("__kmpc_omp_task_parts(enter): T#%d loc=%p task=%p\n", gtid, 1608 loc_ref, new_taskdata)); 1609 1610 #if OMPT_SUPPORT 1611 kmp_taskdata_t *parent; 1612 if (UNLIKELY(ompt_enabled.enabled)) { 1613 parent = new_taskdata->td_parent; 1614 if (ompt_enabled.ompt_callback_task_create) { 1615 ompt_data_t task_data = ompt_data_none; 1616 ompt_callbacks.ompt_callback(ompt_callback_task_create)( 1617 parent ? &(parent->ompt_task_info.task_data) : &task_data, 1618 parent ? &(parent->ompt_task_info.frame) : NULL, 1619 &(new_taskdata->ompt_task_info.task_data), ompt_task_explicit, 0, 1620 OMPT_GET_RETURN_ADDRESS(0)); 1621 } 1622 } 1623 #endif 1624 1625 /* Should we execute the new task or queue it? For now, let's just always try 1626 to queue it. If the queue fills up, then we'll execute it. */ 1627 1628 if (__kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED) // if cannot defer 1629 { // Execute this task immediately 1630 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task; 1631 new_taskdata->td_flags.task_serial = 1; 1632 __kmp_invoke_task(gtid, new_task, current_task); 1633 } 1634 1635 KA_TRACE( 1636 10, 1637 ("__kmpc_omp_task_parts(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: " 1638 "loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n", 1639 gtid, loc_ref, new_taskdata)); 1640 1641 ANNOTATE_HAPPENS_BEFORE(new_task); 1642 #if OMPT_SUPPORT 1643 if (UNLIKELY(ompt_enabled.enabled)) { 1644 parent->ompt_task_info.frame.enter_frame = ompt_data_none; 1645 } 1646 #endif 1647 return TASK_CURRENT_NOT_QUEUED; 1648 } 1649 1650 // __kmp_omp_task: Schedule a non-thread-switchable task for execution 1651 // 1652 // gtid: Global Thread ID of encountering thread 1653 // new_task:non-thread-switchable task thunk allocated by __kmp_omp_task_alloc() 1654 // serialize_immediate: if TRUE then if the task is executed immediately its 1655 // execution will be serialized 1656 // Returns: 1657 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to 1658 // be resumed later. 1659 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be 1660 // resumed later. 1661 kmp_int32 __kmp_omp_task(kmp_int32 gtid, kmp_task_t *new_task, 1662 bool serialize_immediate) { 1663 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task); 1664 1665 /* Should we execute the new task or queue it? For now, let's just always try 1666 to queue it. If the queue fills up, then we'll execute it. */ 1667 if (new_taskdata->td_flags.proxy == TASK_PROXY || 1668 __kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED) // if cannot defer 1669 { // Execute this task immediately 1670 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task; 1671 if (serialize_immediate) 1672 new_taskdata->td_flags.task_serial = 1; 1673 __kmp_invoke_task(gtid, new_task, current_task); 1674 } 1675 1676 ANNOTATE_HAPPENS_BEFORE(new_task); 1677 return TASK_CURRENT_NOT_QUEUED; 1678 } 1679 1680 // __kmpc_omp_task: Wrapper around __kmp_omp_task to schedule a 1681 // non-thread-switchable task from the parent thread only! 1682 // 1683 // loc_ref: location of original task pragma (ignored) 1684 // gtid: Global Thread ID of encountering thread 1685 // new_task: non-thread-switchable task thunk allocated by 1686 // __kmp_omp_task_alloc() 1687 // Returns: 1688 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to 1689 // be resumed later. 1690 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be 1691 // resumed later. 1692 kmp_int32 __kmpc_omp_task(ident_t *loc_ref, kmp_int32 gtid, 1693 kmp_task_t *new_task) { 1694 kmp_int32 res; 1695 KMP_SET_THREAD_STATE_BLOCK(EXPLICIT_TASK); 1696 1697 #if KMP_DEBUG || OMPT_SUPPORT 1698 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task); 1699 #endif 1700 KA_TRACE(10, ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref, 1701 new_taskdata)); 1702 1703 #if OMPT_SUPPORT 1704 kmp_taskdata_t *parent = NULL; 1705 if (UNLIKELY(ompt_enabled.enabled)) { 1706 if (!new_taskdata->td_flags.started) { 1707 OMPT_STORE_RETURN_ADDRESS(gtid); 1708 parent = new_taskdata->td_parent; 1709 if (!parent->ompt_task_info.frame.enter_frame.ptr) { 1710 parent->ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1711 } 1712 if (ompt_enabled.ompt_callback_task_create) { 1713 ompt_data_t task_data = ompt_data_none; 1714 ompt_callbacks.ompt_callback(ompt_callback_task_create)( 1715 parent ? &(parent->ompt_task_info.task_data) : &task_data, 1716 parent ? &(parent->ompt_task_info.frame) : NULL, 1717 &(new_taskdata->ompt_task_info.task_data), 1718 ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata), 0, 1719 OMPT_LOAD_RETURN_ADDRESS(gtid)); 1720 } 1721 } else { 1722 // We are scheduling the continuation of an UNTIED task. 1723 // Scheduling back to the parent task. 1724 __ompt_task_finish(new_task, 1725 new_taskdata->ompt_task_info.scheduling_parent, 1726 ompt_task_switch); 1727 new_taskdata->ompt_task_info.frame.exit_frame = ompt_data_none; 1728 } 1729 } 1730 #endif 1731 1732 res = __kmp_omp_task(gtid, new_task, true); 1733 1734 KA_TRACE(10, ("__kmpc_omp_task(exit): T#%d returning " 1735 "TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n", 1736 gtid, loc_ref, new_taskdata)); 1737 #if OMPT_SUPPORT 1738 if (UNLIKELY(ompt_enabled.enabled && parent != NULL)) { 1739 parent->ompt_task_info.frame.enter_frame = ompt_data_none; 1740 } 1741 #endif 1742 return res; 1743 } 1744 1745 // __kmp_omp_taskloop_task: Wrapper around __kmp_omp_task to schedule 1746 // a taskloop task with the correct OMPT return address 1747 // 1748 // loc_ref: location of original task pragma (ignored) 1749 // gtid: Global Thread ID of encountering thread 1750 // new_task: non-thread-switchable task thunk allocated by 1751 // __kmp_omp_task_alloc() 1752 // codeptr_ra: return address for OMPT callback 1753 // Returns: 1754 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to 1755 // be resumed later. 1756 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be 1757 // resumed later. 1758 kmp_int32 __kmp_omp_taskloop_task(ident_t *loc_ref, kmp_int32 gtid, 1759 kmp_task_t *new_task, void *codeptr_ra) { 1760 kmp_int32 res; 1761 KMP_SET_THREAD_STATE_BLOCK(EXPLICIT_TASK); 1762 1763 #if KMP_DEBUG || OMPT_SUPPORT 1764 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task); 1765 #endif 1766 KA_TRACE(10, ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref, 1767 new_taskdata)); 1768 1769 #if OMPT_SUPPORT 1770 kmp_taskdata_t *parent = NULL; 1771 if (UNLIKELY(ompt_enabled.enabled && !new_taskdata->td_flags.started)) { 1772 parent = new_taskdata->td_parent; 1773 if (!parent->ompt_task_info.frame.enter_frame.ptr) 1774 parent->ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1775 if (ompt_enabled.ompt_callback_task_create) { 1776 ompt_data_t task_data = ompt_data_none; 1777 ompt_callbacks.ompt_callback(ompt_callback_task_create)( 1778 parent ? &(parent->ompt_task_info.task_data) : &task_data, 1779 parent ? &(parent->ompt_task_info.frame) : NULL, 1780 &(new_taskdata->ompt_task_info.task_data), 1781 ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata), 0, 1782 codeptr_ra); 1783 } 1784 } 1785 #endif 1786 1787 res = __kmp_omp_task(gtid, new_task, true); 1788 1789 KA_TRACE(10, ("__kmpc_omp_task(exit): T#%d returning " 1790 "TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n", 1791 gtid, loc_ref, new_taskdata)); 1792 #if OMPT_SUPPORT 1793 if (UNLIKELY(ompt_enabled.enabled && parent != NULL)) { 1794 parent->ompt_task_info.frame.enter_frame = ompt_data_none; 1795 } 1796 #endif 1797 return res; 1798 } 1799 1800 template <bool ompt> 1801 static kmp_int32 __kmpc_omp_taskwait_template(ident_t *loc_ref, kmp_int32 gtid, 1802 void *frame_address, 1803 void *return_address) { 1804 kmp_taskdata_t *taskdata; 1805 kmp_info_t *thread; 1806 int thread_finished = FALSE; 1807 KMP_SET_THREAD_STATE_BLOCK(TASKWAIT); 1808 1809 KA_TRACE(10, ("__kmpc_omp_taskwait(enter): T#%d loc=%p\n", gtid, loc_ref)); 1810 1811 if (__kmp_tasking_mode != tskm_immediate_exec) { 1812 thread = __kmp_threads[gtid]; 1813 taskdata = thread->th.th_current_task; 1814 1815 #if OMPT_SUPPORT && OMPT_OPTIONAL 1816 ompt_data_t *my_task_data; 1817 ompt_data_t *my_parallel_data; 1818 1819 if (ompt) { 1820 my_task_data = &(taskdata->ompt_task_info.task_data); 1821 my_parallel_data = OMPT_CUR_TEAM_DATA(thread); 1822 1823 taskdata->ompt_task_info.frame.enter_frame.ptr = frame_address; 1824 1825 if (ompt_enabled.ompt_callback_sync_region) { 1826 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 1827 ompt_sync_region_taskwait, ompt_scope_begin, my_parallel_data, 1828 my_task_data, return_address); 1829 } 1830 1831 if (ompt_enabled.ompt_callback_sync_region_wait) { 1832 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 1833 ompt_sync_region_taskwait, ompt_scope_begin, my_parallel_data, 1834 my_task_data, return_address); 1835 } 1836 } 1837 #endif // OMPT_SUPPORT && OMPT_OPTIONAL 1838 1839 // Debugger: The taskwait is active. Store location and thread encountered the 1840 // taskwait. 1841 #if USE_ITT_BUILD 1842 // Note: These values are used by ITT events as well. 1843 #endif /* USE_ITT_BUILD */ 1844 taskdata->td_taskwait_counter += 1; 1845 taskdata->td_taskwait_ident = loc_ref; 1846 taskdata->td_taskwait_thread = gtid + 1; 1847 1848 #if USE_ITT_BUILD 1849 void *itt_sync_obj = __kmp_itt_taskwait_object(gtid); 1850 if (itt_sync_obj != NULL) 1851 __kmp_itt_taskwait_starting(gtid, itt_sync_obj); 1852 #endif /* USE_ITT_BUILD */ 1853 1854 bool must_wait = 1855 !taskdata->td_flags.team_serial && !taskdata->td_flags.final; 1856 1857 must_wait = must_wait || (thread->th.th_task_team != NULL && 1858 thread->th.th_task_team->tt.tt_found_proxy_tasks); 1859 if (must_wait) { 1860 kmp_flag_32 flag(RCAST(std::atomic<kmp_uint32> *, 1861 &(taskdata->td_incomplete_child_tasks)), 1862 0U); 1863 while (KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) != 0) { 1864 flag.execute_tasks(thread, gtid, FALSE, 1865 &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), 1866 __kmp_task_stealing_constraint); 1867 } 1868 } 1869 #if USE_ITT_BUILD 1870 if (itt_sync_obj != NULL) 1871 __kmp_itt_taskwait_finished(gtid, itt_sync_obj); 1872 #endif /* USE_ITT_BUILD */ 1873 1874 // Debugger: The taskwait is completed. Location remains, but thread is 1875 // negated. 1876 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; 1877 1878 #if OMPT_SUPPORT && OMPT_OPTIONAL 1879 if (ompt) { 1880 if (ompt_enabled.ompt_callback_sync_region_wait) { 1881 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 1882 ompt_sync_region_taskwait, ompt_scope_end, my_parallel_data, 1883 my_task_data, return_address); 1884 } 1885 if (ompt_enabled.ompt_callback_sync_region) { 1886 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 1887 ompt_sync_region_taskwait, ompt_scope_end, my_parallel_data, 1888 my_task_data, return_address); 1889 } 1890 taskdata->ompt_task_info.frame.enter_frame = ompt_data_none; 1891 } 1892 #endif // OMPT_SUPPORT && OMPT_OPTIONAL 1893 1894 ANNOTATE_HAPPENS_AFTER(taskdata); 1895 } 1896 1897 KA_TRACE(10, ("__kmpc_omp_taskwait(exit): T#%d task %p finished waiting, " 1898 "returning TASK_CURRENT_NOT_QUEUED\n", 1899 gtid, taskdata)); 1900 1901 return TASK_CURRENT_NOT_QUEUED; 1902 } 1903 1904 #if OMPT_SUPPORT && OMPT_OPTIONAL 1905 OMPT_NOINLINE 1906 static kmp_int32 __kmpc_omp_taskwait_ompt(ident_t *loc_ref, kmp_int32 gtid, 1907 void *frame_address, 1908 void *return_address) { 1909 return __kmpc_omp_taskwait_template<true>(loc_ref, gtid, frame_address, 1910 return_address); 1911 } 1912 #endif // OMPT_SUPPORT && OMPT_OPTIONAL 1913 1914 // __kmpc_omp_taskwait: Wait until all tasks generated by the current task are 1915 // complete 1916 kmp_int32 __kmpc_omp_taskwait(ident_t *loc_ref, kmp_int32 gtid) { 1917 #if OMPT_SUPPORT && OMPT_OPTIONAL 1918 if (UNLIKELY(ompt_enabled.enabled)) { 1919 OMPT_STORE_RETURN_ADDRESS(gtid); 1920 return __kmpc_omp_taskwait_ompt(loc_ref, gtid, OMPT_GET_FRAME_ADDRESS(0), 1921 OMPT_LOAD_RETURN_ADDRESS(gtid)); 1922 } 1923 #endif 1924 return __kmpc_omp_taskwait_template<false>(loc_ref, gtid, NULL, NULL); 1925 } 1926 1927 // __kmpc_omp_taskyield: switch to a different task 1928 kmp_int32 __kmpc_omp_taskyield(ident_t *loc_ref, kmp_int32 gtid, int end_part) { 1929 kmp_taskdata_t *taskdata; 1930 kmp_info_t *thread; 1931 int thread_finished = FALSE; 1932 1933 KMP_COUNT_BLOCK(OMP_TASKYIELD); 1934 KMP_SET_THREAD_STATE_BLOCK(TASKYIELD); 1935 1936 KA_TRACE(10, ("__kmpc_omp_taskyield(enter): T#%d loc=%p end_part = %d\n", 1937 gtid, loc_ref, end_part)); 1938 1939 if (__kmp_tasking_mode != tskm_immediate_exec && __kmp_init_parallel) { 1940 thread = __kmp_threads[gtid]; 1941 taskdata = thread->th.th_current_task; 1942 // Should we model this as a task wait or not? 1943 // Debugger: The taskwait is active. Store location and thread encountered the 1944 // taskwait. 1945 #if USE_ITT_BUILD 1946 // Note: These values are used by ITT events as well. 1947 #endif /* USE_ITT_BUILD */ 1948 taskdata->td_taskwait_counter += 1; 1949 taskdata->td_taskwait_ident = loc_ref; 1950 taskdata->td_taskwait_thread = gtid + 1; 1951 1952 #if USE_ITT_BUILD 1953 void *itt_sync_obj = __kmp_itt_taskwait_object(gtid); 1954 if (itt_sync_obj != NULL) 1955 __kmp_itt_taskwait_starting(gtid, itt_sync_obj); 1956 #endif /* USE_ITT_BUILD */ 1957 if (!taskdata->td_flags.team_serial) { 1958 kmp_task_team_t *task_team = thread->th.th_task_team; 1959 if (task_team != NULL) { 1960 if (KMP_TASKING_ENABLED(task_team)) { 1961 #if OMPT_SUPPORT 1962 if (UNLIKELY(ompt_enabled.enabled)) 1963 thread->th.ompt_thread_info.ompt_task_yielded = 1; 1964 #endif 1965 __kmp_execute_tasks_32( 1966 thread, gtid, NULL, FALSE, 1967 &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), 1968 __kmp_task_stealing_constraint); 1969 #if OMPT_SUPPORT 1970 if (UNLIKELY(ompt_enabled.enabled)) 1971 thread->th.ompt_thread_info.ompt_task_yielded = 0; 1972 #endif 1973 } 1974 } 1975 } 1976 #if USE_ITT_BUILD 1977 if (itt_sync_obj != NULL) 1978 __kmp_itt_taskwait_finished(gtid, itt_sync_obj); 1979 #endif /* USE_ITT_BUILD */ 1980 1981 // Debugger: The taskwait is completed. Location remains, but thread is 1982 // negated. 1983 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; 1984 } 1985 1986 KA_TRACE(10, ("__kmpc_omp_taskyield(exit): T#%d task %p resuming, " 1987 "returning TASK_CURRENT_NOT_QUEUED\n", 1988 gtid, taskdata)); 1989 1990 return TASK_CURRENT_NOT_QUEUED; 1991 } 1992 1993 // Task Reduction implementation 1994 // 1995 // Note: initial implementation didn't take into account the possibility 1996 // to specify omp_orig for initializer of the UDR (user defined reduction). 1997 // Corrected implementation takes into account the omp_orig object. 1998 // Compiler is free to use old implementation if omp_orig is not specified. 1999 2000 /*! 2001 @ingroup BASIC_TYPES 2002 @{ 2003 */ 2004 2005 /*! 2006 Flags for special info per task reduction item. 2007 */ 2008 typedef struct kmp_taskred_flags { 2009 /*! 1 - use lazy alloc/init (e.g. big objects, #tasks < #threads) */ 2010 unsigned lazy_priv : 1; 2011 unsigned reserved31 : 31; 2012 } kmp_taskred_flags_t; 2013 2014 /*! 2015 Internal struct for reduction data item related info set up by compiler. 2016 */ 2017 typedef struct kmp_task_red_input { 2018 void *reduce_shar; /**< shared between tasks item to reduce into */ 2019 size_t reduce_size; /**< size of data item in bytes */ 2020 // three compiler-generated routines (init, fini are optional): 2021 void *reduce_init; /**< data initialization routine (single parameter) */ 2022 void *reduce_fini; /**< data finalization routine */ 2023 void *reduce_comb; /**< data combiner routine */ 2024 kmp_taskred_flags_t flags; /**< flags for additional info from compiler */ 2025 } kmp_task_red_input_t; 2026 2027 /*! 2028 Internal struct for reduction data item related info saved by the library. 2029 */ 2030 typedef struct kmp_taskred_data { 2031 void *reduce_shar; /**< shared between tasks item to reduce into */ 2032 size_t reduce_size; /**< size of data item */ 2033 kmp_taskred_flags_t flags; /**< flags for additional info from compiler */ 2034 void *reduce_priv; /**< array of thread specific items */ 2035 void *reduce_pend; /**< end of private data for faster comparison op */ 2036 // three compiler-generated routines (init, fini are optional): 2037 void *reduce_comb; /**< data combiner routine */ 2038 void *reduce_init; /**< data initialization routine (two parameters) */ 2039 void *reduce_fini; /**< data finalization routine */ 2040 void *reduce_orig; /**< original item (can be used in UDR initializer) */ 2041 } kmp_taskred_data_t; 2042 2043 /*! 2044 Internal struct for reduction data item related info set up by compiler. 2045 2046 New interface: added reduce_orig field to provide omp_orig for UDR initializer. 2047 */ 2048 typedef struct kmp_taskred_input { 2049 void *reduce_shar; /**< shared between tasks item to reduce into */ 2050 void *reduce_orig; /**< original reduction item used for initialization */ 2051 size_t reduce_size; /**< size of data item */ 2052 // three compiler-generated routines (init, fini are optional): 2053 void *reduce_init; /**< data initialization routine (two parameters) */ 2054 void *reduce_fini; /**< data finalization routine */ 2055 void *reduce_comb; /**< data combiner routine */ 2056 kmp_taskred_flags_t flags; /**< flags for additional info from compiler */ 2057 } kmp_taskred_input_t; 2058 /*! 2059 @} 2060 */ 2061 2062 template <typename T> void __kmp_assign_orig(kmp_taskred_data_t &item, T &src); 2063 template <> 2064 void __kmp_assign_orig<kmp_task_red_input_t>(kmp_taskred_data_t &item, 2065 kmp_task_red_input_t &src) { 2066 item.reduce_orig = NULL; 2067 } 2068 template <> 2069 void __kmp_assign_orig<kmp_taskred_input_t>(kmp_taskred_data_t &item, 2070 kmp_taskred_input_t &src) { 2071 if (src.reduce_orig != NULL) { 2072 item.reduce_orig = src.reduce_orig; 2073 } else { 2074 item.reduce_orig = src.reduce_shar; 2075 } // non-NULL reduce_orig means new interface used 2076 } 2077 2078 template <typename T> void __kmp_call_init(kmp_taskred_data_t &item, int j); 2079 template <> 2080 void __kmp_call_init<kmp_task_red_input_t>(kmp_taskred_data_t &item, 2081 int offset) { 2082 ((void (*)(void *))item.reduce_init)((char *)(item.reduce_priv) + offset); 2083 } 2084 template <> 2085 void __kmp_call_init<kmp_taskred_input_t>(kmp_taskred_data_t &item, 2086 int offset) { 2087 ((void (*)(void *, void *))item.reduce_init)( 2088 (char *)(item.reduce_priv) + offset, item.reduce_orig); 2089 } 2090 2091 template <typename T> 2092 void *__kmp_task_reduction_init(int gtid, int num, T *data) { 2093 kmp_info_t *thread = __kmp_threads[gtid]; 2094 kmp_taskgroup_t *tg = thread->th.th_current_task->td_taskgroup; 2095 kmp_int32 nth = thread->th.th_team_nproc; 2096 kmp_taskred_data_t *arr; 2097 2098 // check input data just in case 2099 KMP_ASSERT(tg != NULL); 2100 KMP_ASSERT(data != NULL); 2101 KMP_ASSERT(num > 0); 2102 if (nth == 1) { 2103 KA_TRACE(10, ("__kmpc_task_reduction_init: T#%d, tg %p, exiting nth=1\n", 2104 gtid, tg)); 2105 return (void *)tg; 2106 } 2107 KA_TRACE(10, ("__kmpc_task_reduction_init: T#%d, taskgroup %p, #items %d\n", 2108 gtid, tg, num)); 2109 arr = (kmp_taskred_data_t *)__kmp_thread_malloc( 2110 thread, num * sizeof(kmp_taskred_data_t)); 2111 for (int i = 0; i < num; ++i) { 2112 size_t size = data[i].reduce_size - 1; 2113 // round the size up to cache line per thread-specific item 2114 size += CACHE_LINE - size % CACHE_LINE; 2115 KMP_ASSERT(data[i].reduce_comb != NULL); // combiner is mandatory 2116 arr[i].reduce_shar = data[i].reduce_shar; 2117 arr[i].reduce_size = size; 2118 arr[i].flags = data[i].flags; 2119 arr[i].reduce_comb = data[i].reduce_comb; 2120 arr[i].reduce_init = data[i].reduce_init; 2121 arr[i].reduce_fini = data[i].reduce_fini; 2122 __kmp_assign_orig<T>(arr[i], data[i]); 2123 if (!arr[i].flags.lazy_priv) { 2124 // allocate cache-line aligned block and fill it with zeros 2125 arr[i].reduce_priv = __kmp_allocate(nth * size); 2126 arr[i].reduce_pend = (char *)(arr[i].reduce_priv) + nth * size; 2127 if (arr[i].reduce_init != NULL) { 2128 // initialize all thread-specific items 2129 for (int j = 0; j < nth; ++j) { 2130 __kmp_call_init<T>(arr[i], j * size); 2131 } 2132 } 2133 } else { 2134 // only allocate space for pointers now, 2135 // objects will be lazily allocated/initialized if/when requested 2136 // note that __kmp_allocate zeroes the allocated memory 2137 arr[i].reduce_priv = __kmp_allocate(nth * sizeof(void *)); 2138 } 2139 } 2140 tg->reduce_data = (void *)arr; 2141 tg->reduce_num_data = num; 2142 return (void *)tg; 2143 } 2144 2145 /*! 2146 @ingroup TASKING 2147 @param gtid Global thread ID 2148 @param num Number of data items to reduce 2149 @param data Array of data for reduction 2150 @return The taskgroup identifier 2151 2152 Initialize task reduction for the taskgroup. 2153 2154 Note: this entry supposes the optional compiler-generated initializer routine 2155 has single parameter - pointer to object to be initialized. That means 2156 the reduction either does not use omp_orig object, or the omp_orig is accessible 2157 without help of the runtime library. 2158 */ 2159 void *__kmpc_task_reduction_init(int gtid, int num, void *data) { 2160 return __kmp_task_reduction_init(gtid, num, (kmp_task_red_input_t *)data); 2161 } 2162 2163 /*! 2164 @ingroup TASKING 2165 @param gtid Global thread ID 2166 @param num Number of data items to reduce 2167 @param data Array of data for reduction 2168 @return The taskgroup identifier 2169 2170 Initialize task reduction for the taskgroup. 2171 2172 Note: this entry supposes the optional compiler-generated initializer routine 2173 has two parameters, pointer to object to be initialized and pointer to omp_orig 2174 */ 2175 void *__kmpc_taskred_init(int gtid, int num, void *data) { 2176 return __kmp_task_reduction_init(gtid, num, (kmp_taskred_input_t *)data); 2177 } 2178 2179 // Copy task reduction data (except for shared pointers). 2180 template <typename T> 2181 void __kmp_task_reduction_init_copy(kmp_info_t *thr, int num, T *data, 2182 kmp_taskgroup_t *tg, void *reduce_data) { 2183 kmp_taskred_data_t *arr; 2184 KA_TRACE(20, ("__kmp_task_reduction_init_copy: Th %p, init taskgroup %p," 2185 " from data %p\n", 2186 thr, tg, reduce_data)); 2187 arr = (kmp_taskred_data_t *)__kmp_thread_malloc( 2188 thr, num * sizeof(kmp_taskred_data_t)); 2189 // threads will share private copies, thunk routines, sizes, flags, etc.: 2190 KMP_MEMCPY(arr, reduce_data, num * sizeof(kmp_taskred_data_t)); 2191 for (int i = 0; i < num; ++i) { 2192 arr[i].reduce_shar = data[i].reduce_shar; // init unique shared pointers 2193 } 2194 tg->reduce_data = (void *)arr; 2195 tg->reduce_num_data = num; 2196 } 2197 2198 /*! 2199 @ingroup TASKING 2200 @param gtid Global thread ID 2201 @param tskgrp The taskgroup ID (optional) 2202 @param data Shared location of the item 2203 @return The pointer to per-thread data 2204 2205 Get thread-specific location of data item 2206 */ 2207 void *__kmpc_task_reduction_get_th_data(int gtid, void *tskgrp, void *data) { 2208 kmp_info_t *thread = __kmp_threads[gtid]; 2209 kmp_int32 nth = thread->th.th_team_nproc; 2210 if (nth == 1) 2211 return data; // nothing to do 2212 2213 kmp_taskgroup_t *tg = (kmp_taskgroup_t *)tskgrp; 2214 if (tg == NULL) 2215 tg = thread->th.th_current_task->td_taskgroup; 2216 KMP_ASSERT(tg != NULL); 2217 kmp_taskred_data_t *arr = (kmp_taskred_data_t *)(tg->reduce_data); 2218 kmp_int32 num = tg->reduce_num_data; 2219 kmp_int32 tid = thread->th.th_info.ds.ds_tid; 2220 2221 KMP_ASSERT(data != NULL); 2222 while (tg != NULL) { 2223 for (int i = 0; i < num; ++i) { 2224 if (!arr[i].flags.lazy_priv) { 2225 if (data == arr[i].reduce_shar || 2226 (data >= arr[i].reduce_priv && data < arr[i].reduce_pend)) 2227 return (char *)(arr[i].reduce_priv) + tid * arr[i].reduce_size; 2228 } else { 2229 // check shared location first 2230 void **p_priv = (void **)(arr[i].reduce_priv); 2231 if (data == arr[i].reduce_shar) 2232 goto found; 2233 // check if we get some thread specific location as parameter 2234 for (int j = 0; j < nth; ++j) 2235 if (data == p_priv[j]) 2236 goto found; 2237 continue; // not found, continue search 2238 found: 2239 if (p_priv[tid] == NULL) { 2240 // allocate thread specific object lazily 2241 p_priv[tid] = __kmp_allocate(arr[i].reduce_size); 2242 if (arr[i].reduce_init != NULL) { 2243 if (arr[i].reduce_orig != NULL) { // new interface 2244 ((void (*)(void *, void *))arr[i].reduce_init)( 2245 p_priv[tid], arr[i].reduce_orig); 2246 } else { // old interface (single parameter) 2247 ((void (*)(void *))arr[i].reduce_init)(p_priv[tid]); 2248 } 2249 } 2250 } 2251 return p_priv[tid]; 2252 } 2253 } 2254 tg = tg->parent; 2255 arr = (kmp_taskred_data_t *)(tg->reduce_data); 2256 num = tg->reduce_num_data; 2257 } 2258 KMP_ASSERT2(0, "Unknown task reduction item"); 2259 return NULL; // ERROR, this line never executed 2260 } 2261 2262 // Finalize task reduction. 2263 // Called from __kmpc_end_taskgroup() 2264 static void __kmp_task_reduction_fini(kmp_info_t *th, kmp_taskgroup_t *tg) { 2265 kmp_int32 nth = th->th.th_team_nproc; 2266 KMP_DEBUG_ASSERT(nth > 1); // should not be called if nth == 1 2267 kmp_taskred_data_t *arr = (kmp_taskred_data_t *)tg->reduce_data; 2268 kmp_int32 num = tg->reduce_num_data; 2269 for (int i = 0; i < num; ++i) { 2270 void *sh_data = arr[i].reduce_shar; 2271 void (*f_fini)(void *) = (void (*)(void *))(arr[i].reduce_fini); 2272 void (*f_comb)(void *, void *) = 2273 (void (*)(void *, void *))(arr[i].reduce_comb); 2274 if (!arr[i].flags.lazy_priv) { 2275 void *pr_data = arr[i].reduce_priv; 2276 size_t size = arr[i].reduce_size; 2277 for (int j = 0; j < nth; ++j) { 2278 void *priv_data = (char *)pr_data + j * size; 2279 f_comb(sh_data, priv_data); // combine results 2280 if (f_fini) 2281 f_fini(priv_data); // finalize if needed 2282 } 2283 } else { 2284 void **pr_data = (void **)(arr[i].reduce_priv); 2285 for (int j = 0; j < nth; ++j) { 2286 if (pr_data[j] != NULL) { 2287 f_comb(sh_data, pr_data[j]); // combine results 2288 if (f_fini) 2289 f_fini(pr_data[j]); // finalize if needed 2290 __kmp_free(pr_data[j]); 2291 } 2292 } 2293 } 2294 __kmp_free(arr[i].reduce_priv); 2295 } 2296 __kmp_thread_free(th, arr); 2297 tg->reduce_data = NULL; 2298 tg->reduce_num_data = 0; 2299 } 2300 2301 // Cleanup task reduction data for parallel or worksharing, 2302 // do not touch task private data other threads still working with. 2303 // Called from __kmpc_end_taskgroup() 2304 static void __kmp_task_reduction_clean(kmp_info_t *th, kmp_taskgroup_t *tg) { 2305 __kmp_thread_free(th, tg->reduce_data); 2306 tg->reduce_data = NULL; 2307 tg->reduce_num_data = 0; 2308 } 2309 2310 template <typename T> 2311 void *__kmp_task_reduction_modifier_init(ident_t *loc, int gtid, int is_ws, 2312 int num, T *data) { 2313 kmp_info_t *thr = __kmp_threads[gtid]; 2314 kmp_int32 nth = thr->th.th_team_nproc; 2315 __kmpc_taskgroup(loc, gtid); // form new taskgroup first 2316 if (nth == 1) { 2317 KA_TRACE(10, 2318 ("__kmpc_reduction_modifier_init: T#%d, tg %p, exiting nth=1\n", 2319 gtid, thr->th.th_current_task->td_taskgroup)); 2320 return (void *)thr->th.th_current_task->td_taskgroup; 2321 } 2322 kmp_team_t *team = thr->th.th_team; 2323 void *reduce_data; 2324 kmp_taskgroup_t *tg; 2325 reduce_data = KMP_ATOMIC_LD_RLX(&team->t.t_tg_reduce_data[is_ws]); 2326 if (reduce_data == NULL && 2327 __kmp_atomic_compare_store(&team->t.t_tg_reduce_data[is_ws], reduce_data, 2328 (void *)1)) { 2329 // single thread enters this block to initialize common reduction data 2330 KMP_DEBUG_ASSERT(reduce_data == NULL); 2331 // first initialize own data, then make a copy other threads can use 2332 tg = (kmp_taskgroup_t *)__kmp_task_reduction_init<T>(gtid, num, data); 2333 reduce_data = __kmp_thread_malloc(thr, num * sizeof(kmp_taskred_data_t)); 2334 KMP_MEMCPY(reduce_data, tg->reduce_data, num * sizeof(kmp_taskred_data_t)); 2335 // fini counters should be 0 at this point 2336 KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&team->t.t_tg_fini_counter[0]) == 0); 2337 KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&team->t.t_tg_fini_counter[1]) == 0); 2338 KMP_ATOMIC_ST_REL(&team->t.t_tg_reduce_data[is_ws], reduce_data); 2339 } else { 2340 while ( 2341 (reduce_data = KMP_ATOMIC_LD_ACQ(&team->t.t_tg_reduce_data[is_ws])) == 2342 (void *)1) { // wait for task reduction initialization 2343 KMP_CPU_PAUSE(); 2344 } 2345 KMP_DEBUG_ASSERT(reduce_data > (void *)1); // should be valid pointer here 2346 tg = thr->th.th_current_task->td_taskgroup; 2347 __kmp_task_reduction_init_copy<T>(thr, num, data, tg, reduce_data); 2348 } 2349 return tg; 2350 } 2351 2352 /*! 2353 @ingroup TASKING 2354 @param loc Source location info 2355 @param gtid Global thread ID 2356 @param is_ws Is 1 if the reduction is for worksharing, 0 otherwise 2357 @param num Number of data items to reduce 2358 @param data Array of data for reduction 2359 @return The taskgroup identifier 2360 2361 Initialize task reduction for a parallel or worksharing. 2362 2363 Note: this entry supposes the optional compiler-generated initializer routine 2364 has single parameter - pointer to object to be initialized. That means 2365 the reduction either does not use omp_orig object, or the omp_orig is accessible 2366 without help of the runtime library. 2367 */ 2368 void *__kmpc_task_reduction_modifier_init(ident_t *loc, int gtid, int is_ws, 2369 int num, void *data) { 2370 return __kmp_task_reduction_modifier_init(loc, gtid, is_ws, num, 2371 (kmp_task_red_input_t *)data); 2372 } 2373 2374 /*! 2375 @ingroup TASKING 2376 @param loc Source location info 2377 @param gtid Global thread ID 2378 @param is_ws Is 1 if the reduction is for worksharing, 0 otherwise 2379 @param num Number of data items to reduce 2380 @param data Array of data for reduction 2381 @return The taskgroup identifier 2382 2383 Initialize task reduction for a parallel or worksharing. 2384 2385 Note: this entry supposes the optional compiler-generated initializer routine 2386 has two parameters, pointer to object to be initialized and pointer to omp_orig 2387 */ 2388 void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int is_ws, int num, 2389 void *data) { 2390 return __kmp_task_reduction_modifier_init(loc, gtid, is_ws, num, 2391 (kmp_taskred_input_t *)data); 2392 } 2393 2394 /*! 2395 @ingroup TASKING 2396 @param loc Source location info 2397 @param gtid Global thread ID 2398 @param is_ws Is 1 if the reduction is for worksharing, 0 otherwise 2399 2400 Finalize task reduction for a parallel or worksharing. 2401 */ 2402 void __kmpc_task_reduction_modifier_fini(ident_t *loc, int gtid, int is_ws) { 2403 __kmpc_end_taskgroup(loc, gtid); 2404 } 2405 2406 // __kmpc_taskgroup: Start a new taskgroup 2407 void __kmpc_taskgroup(ident_t *loc, int gtid) { 2408 kmp_info_t *thread = __kmp_threads[gtid]; 2409 kmp_taskdata_t *taskdata = thread->th.th_current_task; 2410 kmp_taskgroup_t *tg_new = 2411 (kmp_taskgroup_t *)__kmp_thread_malloc(thread, sizeof(kmp_taskgroup_t)); 2412 KA_TRACE(10, ("__kmpc_taskgroup: T#%d loc=%p group=%p\n", gtid, loc, tg_new)); 2413 KMP_ATOMIC_ST_RLX(&tg_new->count, 0); 2414 KMP_ATOMIC_ST_RLX(&tg_new->cancel_request, cancel_noreq); 2415 tg_new->parent = taskdata->td_taskgroup; 2416 tg_new->reduce_data = NULL; 2417 tg_new->reduce_num_data = 0; 2418 taskdata->td_taskgroup = tg_new; 2419 2420 #if OMPT_SUPPORT && OMPT_OPTIONAL 2421 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)) { 2422 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); 2423 if (!codeptr) 2424 codeptr = OMPT_GET_RETURN_ADDRESS(0); 2425 kmp_team_t *team = thread->th.th_team; 2426 ompt_data_t my_task_data = taskdata->ompt_task_info.task_data; 2427 // FIXME: I think this is wrong for lwt! 2428 ompt_data_t my_parallel_data = team->t.ompt_team_info.parallel_data; 2429 2430 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 2431 ompt_sync_region_taskgroup, ompt_scope_begin, &(my_parallel_data), 2432 &(my_task_data), codeptr); 2433 } 2434 #endif 2435 } 2436 2437 // __kmpc_end_taskgroup: Wait until all tasks generated by the current task 2438 // and its descendants are complete 2439 void __kmpc_end_taskgroup(ident_t *loc, int gtid) { 2440 kmp_info_t *thread = __kmp_threads[gtid]; 2441 kmp_taskdata_t *taskdata = thread->th.th_current_task; 2442 kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup; 2443 int thread_finished = FALSE; 2444 2445 #if OMPT_SUPPORT && OMPT_OPTIONAL 2446 kmp_team_t *team; 2447 ompt_data_t my_task_data; 2448 ompt_data_t my_parallel_data; 2449 void *codeptr; 2450 if (UNLIKELY(ompt_enabled.enabled)) { 2451 team = thread->th.th_team; 2452 my_task_data = taskdata->ompt_task_info.task_data; 2453 // FIXME: I think this is wrong for lwt! 2454 my_parallel_data = team->t.ompt_team_info.parallel_data; 2455 codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); 2456 if (!codeptr) 2457 codeptr = OMPT_GET_RETURN_ADDRESS(0); 2458 } 2459 #endif 2460 2461 KA_TRACE(10, ("__kmpc_end_taskgroup(enter): T#%d loc=%p\n", gtid, loc)); 2462 KMP_DEBUG_ASSERT(taskgroup != NULL); 2463 KMP_SET_THREAD_STATE_BLOCK(TASKGROUP); 2464 2465 if (__kmp_tasking_mode != tskm_immediate_exec) { 2466 // mark task as waiting not on a barrier 2467 taskdata->td_taskwait_counter += 1; 2468 taskdata->td_taskwait_ident = loc; 2469 taskdata->td_taskwait_thread = gtid + 1; 2470 #if USE_ITT_BUILD 2471 // For ITT the taskgroup wait is similar to taskwait until we need to 2472 // distinguish them 2473 void *itt_sync_obj = __kmp_itt_taskwait_object(gtid); 2474 if (itt_sync_obj != NULL) 2475 __kmp_itt_taskwait_starting(gtid, itt_sync_obj); 2476 #endif /* USE_ITT_BUILD */ 2477 2478 #if OMPT_SUPPORT && OMPT_OPTIONAL 2479 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region_wait)) { 2480 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 2481 ompt_sync_region_taskgroup, ompt_scope_begin, &(my_parallel_data), 2482 &(my_task_data), codeptr); 2483 } 2484 #endif 2485 2486 if (!taskdata->td_flags.team_serial || 2487 (thread->th.th_task_team != NULL && 2488 thread->th.th_task_team->tt.tt_found_proxy_tasks)) { 2489 kmp_flag_32 flag(RCAST(std::atomic<kmp_uint32> *, &(taskgroup->count)), 2490 0U); 2491 while (KMP_ATOMIC_LD_ACQ(&taskgroup->count) != 0) { 2492 flag.execute_tasks(thread, gtid, FALSE, 2493 &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), 2494 __kmp_task_stealing_constraint); 2495 } 2496 } 2497 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; // end waiting 2498 2499 #if OMPT_SUPPORT && OMPT_OPTIONAL 2500 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region_wait)) { 2501 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 2502 ompt_sync_region_taskgroup, ompt_scope_end, &(my_parallel_data), 2503 &(my_task_data), codeptr); 2504 } 2505 #endif 2506 2507 #if USE_ITT_BUILD 2508 if (itt_sync_obj != NULL) 2509 __kmp_itt_taskwait_finished(gtid, itt_sync_obj); 2510 #endif /* USE_ITT_BUILD */ 2511 } 2512 KMP_DEBUG_ASSERT(taskgroup->count == 0); 2513 2514 if (taskgroup->reduce_data != NULL) { // need to reduce? 2515 int cnt; 2516 void *reduce_data; 2517 kmp_team_t *t = thread->th.th_team; 2518 kmp_taskred_data_t *arr = (kmp_taskred_data_t *)taskgroup->reduce_data; 2519 // check if <priv> data of the first reduction variable shared for the team 2520 void *priv0 = arr[0].reduce_priv; 2521 if ((reduce_data = KMP_ATOMIC_LD_ACQ(&t->t.t_tg_reduce_data[0])) != NULL && 2522 ((kmp_taskred_data_t *)reduce_data)[0].reduce_priv == priv0) { 2523 // finishing task reduction on parallel 2524 cnt = KMP_ATOMIC_INC(&t->t.t_tg_fini_counter[0]); 2525 if (cnt == thread->th.th_team_nproc - 1) { 2526 // we are the last thread passing __kmpc_reduction_modifier_fini() 2527 // finalize task reduction: 2528 __kmp_task_reduction_fini(thread, taskgroup); 2529 // cleanup fields in the team structure: 2530 // TODO: is relaxed store enough here (whole barrier should follow)? 2531 __kmp_thread_free(thread, reduce_data); 2532 KMP_ATOMIC_ST_REL(&t->t.t_tg_reduce_data[0], NULL); 2533 KMP_ATOMIC_ST_REL(&t->t.t_tg_fini_counter[0], 0); 2534 } else { 2535 // we are not the last thread passing __kmpc_reduction_modifier_fini(), 2536 // so do not finalize reduction, just clean own copy of the data 2537 __kmp_task_reduction_clean(thread, taskgroup); 2538 } 2539 } else if ((reduce_data = KMP_ATOMIC_LD_ACQ(&t->t.t_tg_reduce_data[1])) != 2540 NULL && 2541 ((kmp_taskred_data_t *)reduce_data)[0].reduce_priv == priv0) { 2542 // finishing task reduction on worksharing 2543 cnt = KMP_ATOMIC_INC(&t->t.t_tg_fini_counter[1]); 2544 if (cnt == thread->th.th_team_nproc - 1) { 2545 // we are the last thread passing __kmpc_reduction_modifier_fini() 2546 __kmp_task_reduction_fini(thread, taskgroup); 2547 // cleanup fields in team structure: 2548 // TODO: is relaxed store enough here (whole barrier should follow)? 2549 __kmp_thread_free(thread, reduce_data); 2550 KMP_ATOMIC_ST_REL(&t->t.t_tg_reduce_data[1], NULL); 2551 KMP_ATOMIC_ST_REL(&t->t.t_tg_fini_counter[1], 0); 2552 } else { 2553 // we are not the last thread passing __kmpc_reduction_modifier_fini(), 2554 // so do not finalize reduction, just clean own copy of the data 2555 __kmp_task_reduction_clean(thread, taskgroup); 2556 } 2557 } else { 2558 // finishing task reduction on taskgroup 2559 __kmp_task_reduction_fini(thread, taskgroup); 2560 } 2561 } 2562 // Restore parent taskgroup for the current task 2563 taskdata->td_taskgroup = taskgroup->parent; 2564 __kmp_thread_free(thread, taskgroup); 2565 2566 KA_TRACE(10, ("__kmpc_end_taskgroup(exit): T#%d task %p finished waiting\n", 2567 gtid, taskdata)); 2568 ANNOTATE_HAPPENS_AFTER(taskdata); 2569 2570 #if OMPT_SUPPORT && OMPT_OPTIONAL 2571 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)) { 2572 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 2573 ompt_sync_region_taskgroup, ompt_scope_end, &(my_parallel_data), 2574 &(my_task_data), codeptr); 2575 } 2576 #endif 2577 } 2578 2579 // __kmp_remove_my_task: remove a task from my own deque 2580 static kmp_task_t *__kmp_remove_my_task(kmp_info_t *thread, kmp_int32 gtid, 2581 kmp_task_team_t *task_team, 2582 kmp_int32 is_constrained) { 2583 kmp_task_t *task; 2584 kmp_taskdata_t *taskdata; 2585 kmp_thread_data_t *thread_data; 2586 kmp_uint32 tail; 2587 2588 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 2589 KMP_DEBUG_ASSERT(task_team->tt.tt_threads_data != 2590 NULL); // Caller should check this condition 2591 2592 thread_data = &task_team->tt.tt_threads_data[__kmp_tid_from_gtid(gtid)]; 2593 2594 KA_TRACE(10, ("__kmp_remove_my_task(enter): T#%d ntasks=%d head=%u tail=%u\n", 2595 gtid, thread_data->td.td_deque_ntasks, 2596 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 2597 2598 if (TCR_4(thread_data->td.td_deque_ntasks) == 0) { 2599 KA_TRACE(10, 2600 ("__kmp_remove_my_task(exit #1): T#%d No tasks to remove: " 2601 "ntasks=%d head=%u tail=%u\n", 2602 gtid, thread_data->td.td_deque_ntasks, 2603 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 2604 return NULL; 2605 } 2606 2607 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 2608 2609 if (TCR_4(thread_data->td.td_deque_ntasks) == 0) { 2610 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 2611 KA_TRACE(10, 2612 ("__kmp_remove_my_task(exit #2): T#%d No tasks to remove: " 2613 "ntasks=%d head=%u tail=%u\n", 2614 gtid, thread_data->td.td_deque_ntasks, 2615 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 2616 return NULL; 2617 } 2618 2619 tail = (thread_data->td.td_deque_tail - 1) & 2620 TASK_DEQUE_MASK(thread_data->td); // Wrap index. 2621 taskdata = thread_data->td.td_deque[tail]; 2622 2623 if (!__kmp_task_is_allowed(gtid, is_constrained, taskdata, 2624 thread->th.th_current_task)) { 2625 // The TSC does not allow to steal victim task 2626 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 2627 KA_TRACE(10, 2628 ("__kmp_remove_my_task(exit #3): T#%d TSC blocks tail task: " 2629 "ntasks=%d head=%u tail=%u\n", 2630 gtid, thread_data->td.td_deque_ntasks, 2631 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 2632 return NULL; 2633 } 2634 2635 thread_data->td.td_deque_tail = tail; 2636 TCW_4(thread_data->td.td_deque_ntasks, thread_data->td.td_deque_ntasks - 1); 2637 2638 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 2639 2640 KA_TRACE(10, ("__kmp_remove_my_task(exit #4): T#%d task %p removed: " 2641 "ntasks=%d head=%u tail=%u\n", 2642 gtid, taskdata, thread_data->td.td_deque_ntasks, 2643 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 2644 2645 task = KMP_TASKDATA_TO_TASK(taskdata); 2646 return task; 2647 } 2648 2649 // __kmp_steal_task: remove a task from another thread's deque 2650 // Assume that calling thread has already checked existence of 2651 // task_team thread_data before calling this routine. 2652 static kmp_task_t *__kmp_steal_task(kmp_info_t *victim_thr, kmp_int32 gtid, 2653 kmp_task_team_t *task_team, 2654 std::atomic<kmp_int32> *unfinished_threads, 2655 int *thread_finished, 2656 kmp_int32 is_constrained) { 2657 kmp_task_t *task; 2658 kmp_taskdata_t *taskdata; 2659 kmp_taskdata_t *current; 2660 kmp_thread_data_t *victim_td, *threads_data; 2661 kmp_int32 target; 2662 kmp_int32 victim_tid; 2663 2664 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 2665 2666 threads_data = task_team->tt.tt_threads_data; 2667 KMP_DEBUG_ASSERT(threads_data != NULL); // Caller should check this condition 2668 2669 victim_tid = victim_thr->th.th_info.ds.ds_tid; 2670 victim_td = &threads_data[victim_tid]; 2671 2672 KA_TRACE(10, ("__kmp_steal_task(enter): T#%d try to steal from T#%d: " 2673 "task_team=%p ntasks=%d head=%u tail=%u\n", 2674 gtid, __kmp_gtid_from_thread(victim_thr), task_team, 2675 victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head, 2676 victim_td->td.td_deque_tail)); 2677 2678 if (TCR_4(victim_td->td.td_deque_ntasks) == 0) { 2679 KA_TRACE(10, ("__kmp_steal_task(exit #1): T#%d could not steal from T#%d: " 2680 "task_team=%p ntasks=%d head=%u tail=%u\n", 2681 gtid, __kmp_gtid_from_thread(victim_thr), task_team, 2682 victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head, 2683 victim_td->td.td_deque_tail)); 2684 return NULL; 2685 } 2686 2687 __kmp_acquire_bootstrap_lock(&victim_td->td.td_deque_lock); 2688 2689 int ntasks = TCR_4(victim_td->td.td_deque_ntasks); 2690 // Check again after we acquire the lock 2691 if (ntasks == 0) { 2692 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock); 2693 KA_TRACE(10, ("__kmp_steal_task(exit #2): T#%d could not steal from T#%d: " 2694 "task_team=%p ntasks=%d head=%u tail=%u\n", 2695 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks, 2696 victim_td->td.td_deque_head, victim_td->td.td_deque_tail)); 2697 return NULL; 2698 } 2699 2700 KMP_DEBUG_ASSERT(victim_td->td.td_deque != NULL); 2701 current = __kmp_threads[gtid]->th.th_current_task; 2702 taskdata = victim_td->td.td_deque[victim_td->td.td_deque_head]; 2703 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) { 2704 // Bump head pointer and Wrap. 2705 victim_td->td.td_deque_head = 2706 (victim_td->td.td_deque_head + 1) & TASK_DEQUE_MASK(victim_td->td); 2707 } else { 2708 if (!task_team->tt.tt_untied_task_encountered) { 2709 // The TSC does not allow to steal victim task 2710 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock); 2711 KA_TRACE(10, ("__kmp_steal_task(exit #3): T#%d could not steal from " 2712 "T#%d: task_team=%p ntasks=%d head=%u tail=%u\n", 2713 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks, 2714 victim_td->td.td_deque_head, victim_td->td.td_deque_tail)); 2715 return NULL; 2716 } 2717 int i; 2718 // walk through victim's deque trying to steal any task 2719 target = victim_td->td.td_deque_head; 2720 taskdata = NULL; 2721 for (i = 1; i < ntasks; ++i) { 2722 target = (target + 1) & TASK_DEQUE_MASK(victim_td->td); 2723 taskdata = victim_td->td.td_deque[target]; 2724 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) { 2725 break; // found victim task 2726 } else { 2727 taskdata = NULL; 2728 } 2729 } 2730 if (taskdata == NULL) { 2731 // No appropriate candidate to steal found 2732 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock); 2733 KA_TRACE(10, ("__kmp_steal_task(exit #4): T#%d could not steal from " 2734 "T#%d: task_team=%p ntasks=%d head=%u tail=%u\n", 2735 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks, 2736 victim_td->td.td_deque_head, victim_td->td.td_deque_tail)); 2737 return NULL; 2738 } 2739 int prev = target; 2740 for (i = i + 1; i < ntasks; ++i) { 2741 // shift remaining tasks in the deque left by 1 2742 target = (target + 1) & TASK_DEQUE_MASK(victim_td->td); 2743 victim_td->td.td_deque[prev] = victim_td->td.td_deque[target]; 2744 prev = target; 2745 } 2746 KMP_DEBUG_ASSERT( 2747 victim_td->td.td_deque_tail == 2748 (kmp_uint32)((target + 1) & TASK_DEQUE_MASK(victim_td->td))); 2749 victim_td->td.td_deque_tail = target; // tail -= 1 (wrapped)) 2750 } 2751 if (*thread_finished) { 2752 // We need to un-mark this victim as a finished victim. This must be done 2753 // before releasing the lock, or else other threads (starting with the 2754 // master victim) might be prematurely released from the barrier!!! 2755 kmp_int32 count; 2756 2757 count = KMP_ATOMIC_INC(unfinished_threads); 2758 2759 KA_TRACE( 2760 20, 2761 ("__kmp_steal_task: T#%d inc unfinished_threads to %d: task_team=%p\n", 2762 gtid, count + 1, task_team)); 2763 2764 *thread_finished = FALSE; 2765 } 2766 TCW_4(victim_td->td.td_deque_ntasks, ntasks - 1); 2767 2768 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock); 2769 2770 KMP_COUNT_BLOCK(TASK_stolen); 2771 KA_TRACE(10, 2772 ("__kmp_steal_task(exit #5): T#%d stole task %p from T#%d: " 2773 "task_team=%p ntasks=%d head=%u tail=%u\n", 2774 gtid, taskdata, __kmp_gtid_from_thread(victim_thr), task_team, 2775 ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail)); 2776 2777 task = KMP_TASKDATA_TO_TASK(taskdata); 2778 return task; 2779 } 2780 2781 // __kmp_execute_tasks_template: Choose and execute tasks until either the 2782 // condition is statisfied (return true) or there are none left (return false). 2783 // 2784 // final_spin is TRUE if this is the spin at the release barrier. 2785 // thread_finished indicates whether the thread is finished executing all 2786 // the tasks it has on its deque, and is at the release barrier. 2787 // spinner is the location on which to spin. 2788 // spinner == NULL means only execute a single task and return. 2789 // checker is the value to check to terminate the spin. 2790 template <class C> 2791 static inline int __kmp_execute_tasks_template( 2792 kmp_info_t *thread, kmp_int32 gtid, C *flag, int final_spin, 2793 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 2794 kmp_int32 is_constrained) { 2795 kmp_task_team_t *task_team = thread->th.th_task_team; 2796 kmp_thread_data_t *threads_data; 2797 kmp_task_t *task; 2798 kmp_info_t *other_thread; 2799 kmp_taskdata_t *current_task = thread->th.th_current_task; 2800 std::atomic<kmp_int32> *unfinished_threads; 2801 kmp_int32 nthreads, victim_tid = -2, use_own_tasks = 1, new_victim = 0, 2802 tid = thread->th.th_info.ds.ds_tid; 2803 2804 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 2805 KMP_DEBUG_ASSERT(thread == __kmp_threads[gtid]); 2806 2807 if (task_team == NULL || current_task == NULL) 2808 return FALSE; 2809 2810 KA_TRACE(15, ("__kmp_execute_tasks_template(enter): T#%d final_spin=%d " 2811 "*thread_finished=%d\n", 2812 gtid, final_spin, *thread_finished)); 2813 2814 thread->th.th_reap_state = KMP_NOT_SAFE_TO_REAP; 2815 threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data); 2816 KMP_DEBUG_ASSERT(threads_data != NULL); 2817 2818 nthreads = task_team->tt.tt_nproc; 2819 unfinished_threads = &(task_team->tt.tt_unfinished_threads); 2820 KMP_DEBUG_ASSERT(nthreads > 1 || task_team->tt.tt_found_proxy_tasks); 2821 KMP_DEBUG_ASSERT(*unfinished_threads >= 0); 2822 2823 while (1) { // Outer loop keeps trying to find tasks in case of single thread 2824 // getting tasks from target constructs 2825 while (1) { // Inner loop to find a task and execute it 2826 task = NULL; 2827 if (use_own_tasks) { // check on own queue first 2828 task = __kmp_remove_my_task(thread, gtid, task_team, is_constrained); 2829 } 2830 if ((task == NULL) && (nthreads > 1)) { // Steal a task 2831 int asleep = 1; 2832 use_own_tasks = 0; 2833 // Try to steal from the last place I stole from successfully. 2834 if (victim_tid == -2) { // haven't stolen anything yet 2835 victim_tid = threads_data[tid].td.td_deque_last_stolen; 2836 if (victim_tid != 2837 -1) // if we have a last stolen from victim, get the thread 2838 other_thread = threads_data[victim_tid].td.td_thr; 2839 } 2840 if (victim_tid != -1) { // found last victim 2841 asleep = 0; 2842 } else if (!new_victim) { // no recent steals and we haven't already 2843 // used a new victim; select a random thread 2844 do { // Find a different thread to steal work from. 2845 // Pick a random thread. Initial plan was to cycle through all the 2846 // threads, and only return if we tried to steal from every thread, 2847 // and failed. Arch says that's not such a great idea. 2848 victim_tid = __kmp_get_random(thread) % (nthreads - 1); 2849 if (victim_tid >= tid) { 2850 ++victim_tid; // Adjusts random distribution to exclude self 2851 } 2852 // Found a potential victim 2853 other_thread = threads_data[victim_tid].td.td_thr; 2854 // There is a slight chance that __kmp_enable_tasking() did not wake 2855 // up all threads waiting at the barrier. If victim is sleeping, 2856 // then wake it up. Since we were going to pay the cache miss 2857 // penalty for referencing another thread's kmp_info_t struct 2858 // anyway, 2859 // the check shouldn't cost too much performance at this point. In 2860 // extra barrier mode, tasks do not sleep at the separate tasking 2861 // barrier, so this isn't a problem. 2862 asleep = 0; 2863 if ((__kmp_tasking_mode == tskm_task_teams) && 2864 (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) && 2865 (TCR_PTR(CCAST(void *, other_thread->th.th_sleep_loc)) != 2866 NULL)) { 2867 asleep = 1; 2868 __kmp_null_resume_wrapper(__kmp_gtid_from_thread(other_thread), 2869 other_thread->th.th_sleep_loc); 2870 // A sleeping thread should not have any tasks on it's queue. 2871 // There is a slight possibility that it resumes, steals a task 2872 // from another thread, which spawns more tasks, all in the time 2873 // that it takes this thread to check => don't write an assertion 2874 // that the victim's queue is empty. Try stealing from a 2875 // different thread. 2876 } 2877 } while (asleep); 2878 } 2879 2880 if (!asleep) { 2881 // We have a victim to try to steal from 2882 task = __kmp_steal_task(other_thread, gtid, task_team, 2883 unfinished_threads, thread_finished, 2884 is_constrained); 2885 } 2886 if (task != NULL) { // set last stolen to victim 2887 if (threads_data[tid].td.td_deque_last_stolen != victim_tid) { 2888 threads_data[tid].td.td_deque_last_stolen = victim_tid; 2889 // The pre-refactored code did not try more than 1 successful new 2890 // vicitm, unless the last one generated more local tasks; 2891 // new_victim keeps track of this 2892 new_victim = 1; 2893 } 2894 } else { // No tasks found; unset last_stolen 2895 KMP_CHECK_UPDATE(threads_data[tid].td.td_deque_last_stolen, -1); 2896 victim_tid = -2; // no successful victim found 2897 } 2898 } 2899 2900 if (task == NULL) // break out of tasking loop 2901 break; 2902 2903 // Found a task; execute it 2904 #if USE_ITT_BUILD && USE_ITT_NOTIFY 2905 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) { 2906 if (itt_sync_obj == NULL) { // we are at fork barrier where we could not 2907 // get the object reliably 2908 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier); 2909 } 2910 __kmp_itt_task_starting(itt_sync_obj); 2911 } 2912 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */ 2913 __kmp_invoke_task(gtid, task, current_task); 2914 #if USE_ITT_BUILD 2915 if (itt_sync_obj != NULL) 2916 __kmp_itt_task_finished(itt_sync_obj); 2917 #endif /* USE_ITT_BUILD */ 2918 // If this thread is only partway through the barrier and the condition is 2919 // met, then return now, so that the barrier gather/release pattern can 2920 // proceed. If this thread is in the last spin loop in the barrier, 2921 // waiting to be released, we know that the termination condition will not 2922 // be satisfied, so don't waste any cycles checking it. 2923 if (flag == NULL || (!final_spin && flag->done_check())) { 2924 KA_TRACE( 2925 15, 2926 ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n", 2927 gtid)); 2928 return TRUE; 2929 } 2930 if (thread->th.th_task_team == NULL) { 2931 break; 2932 } 2933 KMP_YIELD(__kmp_library == library_throughput); // Yield before next task 2934 // If execution of a stolen task results in more tasks being placed on our 2935 // run queue, reset use_own_tasks 2936 if (!use_own_tasks && TCR_4(threads_data[tid].td.td_deque_ntasks) != 0) { 2937 KA_TRACE(20, ("__kmp_execute_tasks_template: T#%d stolen task spawned " 2938 "other tasks, restart\n", 2939 gtid)); 2940 use_own_tasks = 1; 2941 new_victim = 0; 2942 } 2943 } 2944 2945 // The task source has been exhausted. If in final spin loop of barrier, 2946 // check if termination condition is satisfied. The work queue may be empty 2947 // but there might be proxy tasks still executing. 2948 if (final_spin && 2949 KMP_ATOMIC_LD_ACQ(¤t_task->td_incomplete_child_tasks) == 0) { 2950 // First, decrement the #unfinished threads, if that has not already been 2951 // done. This decrement might be to the spin location, and result in the 2952 // termination condition being satisfied. 2953 if (!*thread_finished) { 2954 kmp_int32 count; 2955 2956 count = KMP_ATOMIC_DEC(unfinished_threads) - 1; 2957 KA_TRACE(20, ("__kmp_execute_tasks_template: T#%d dec " 2958 "unfinished_threads to %d task_team=%p\n", 2959 gtid, count, task_team)); 2960 *thread_finished = TRUE; 2961 } 2962 2963 // It is now unsafe to reference thread->th.th_team !!! 2964 // Decrementing task_team->tt.tt_unfinished_threads can allow the master 2965 // thread to pass through the barrier, where it might reset each thread's 2966 // th.th_team field for the next parallel region. If we can steal more 2967 // work, we know that this has not happened yet. 2968 if (flag != NULL && flag->done_check()) { 2969 KA_TRACE( 2970 15, 2971 ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n", 2972 gtid)); 2973 return TRUE; 2974 } 2975 } 2976 2977 // If this thread's task team is NULL, master has recognized that there are 2978 // no more tasks; bail out 2979 if (thread->th.th_task_team == NULL) { 2980 KA_TRACE(15, 2981 ("__kmp_execute_tasks_template: T#%d no more tasks\n", gtid)); 2982 return FALSE; 2983 } 2984 2985 // We could be getting tasks from target constructs; if this is the only 2986 // thread, keep trying to execute tasks from own queue 2987 if (nthreads == 1) 2988 use_own_tasks = 1; 2989 else { 2990 KA_TRACE(15, 2991 ("__kmp_execute_tasks_template: T#%d can't find work\n", gtid)); 2992 return FALSE; 2993 } 2994 } 2995 } 2996 2997 int __kmp_execute_tasks_32( 2998 kmp_info_t *thread, kmp_int32 gtid, kmp_flag_32 *flag, int final_spin, 2999 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 3000 kmp_int32 is_constrained) { 3001 return __kmp_execute_tasks_template( 3002 thread, gtid, flag, final_spin, 3003 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained); 3004 } 3005 3006 int __kmp_execute_tasks_64( 3007 kmp_info_t *thread, kmp_int32 gtid, kmp_flag_64 *flag, int final_spin, 3008 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 3009 kmp_int32 is_constrained) { 3010 return __kmp_execute_tasks_template( 3011 thread, gtid, flag, final_spin, 3012 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained); 3013 } 3014 3015 int __kmp_execute_tasks_oncore( 3016 kmp_info_t *thread, kmp_int32 gtid, kmp_flag_oncore *flag, int final_spin, 3017 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 3018 kmp_int32 is_constrained) { 3019 return __kmp_execute_tasks_template( 3020 thread, gtid, flag, final_spin, 3021 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained); 3022 } 3023 3024 // __kmp_enable_tasking: Allocate task team and resume threads sleeping at the 3025 // next barrier so they can assist in executing enqueued tasks. 3026 // First thread in allocates the task team atomically. 3027 static void __kmp_enable_tasking(kmp_task_team_t *task_team, 3028 kmp_info_t *this_thr) { 3029 kmp_thread_data_t *threads_data; 3030 int nthreads, i, is_init_thread; 3031 3032 KA_TRACE(10, ("__kmp_enable_tasking(enter): T#%d\n", 3033 __kmp_gtid_from_thread(this_thr))); 3034 3035 KMP_DEBUG_ASSERT(task_team != NULL); 3036 KMP_DEBUG_ASSERT(this_thr->th.th_team != NULL); 3037 3038 nthreads = task_team->tt.tt_nproc; 3039 KMP_DEBUG_ASSERT(nthreads > 0); 3040 KMP_DEBUG_ASSERT(nthreads == this_thr->th.th_team->t.t_nproc); 3041 3042 // Allocate or increase the size of threads_data if necessary 3043 is_init_thread = __kmp_realloc_task_threads_data(this_thr, task_team); 3044 3045 if (!is_init_thread) { 3046 // Some other thread already set up the array. 3047 KA_TRACE( 3048 20, 3049 ("__kmp_enable_tasking(exit): T#%d: threads array already set up.\n", 3050 __kmp_gtid_from_thread(this_thr))); 3051 return; 3052 } 3053 threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data); 3054 KMP_DEBUG_ASSERT(threads_data != NULL); 3055 3056 if (__kmp_tasking_mode == tskm_task_teams && 3057 (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME)) { 3058 // Release any threads sleeping at the barrier, so that they can steal 3059 // tasks and execute them. In extra barrier mode, tasks do not sleep 3060 // at the separate tasking barrier, so this isn't a problem. 3061 for (i = 0; i < nthreads; i++) { 3062 volatile void *sleep_loc; 3063 kmp_info_t *thread = threads_data[i].td.td_thr; 3064 3065 if (i == this_thr->th.th_info.ds.ds_tid) { 3066 continue; 3067 } 3068 // Since we haven't locked the thread's suspend mutex lock at this 3069 // point, there is a small window where a thread might be putting 3070 // itself to sleep, but hasn't set the th_sleep_loc field yet. 3071 // To work around this, __kmp_execute_tasks_template() periodically checks 3072 // see if other threads are sleeping (using the same random mechanism that 3073 // is used for task stealing) and awakens them if they are. 3074 if ((sleep_loc = TCR_PTR(CCAST(void *, thread->th.th_sleep_loc))) != 3075 NULL) { 3076 KF_TRACE(50, ("__kmp_enable_tasking: T#%d waking up thread T#%d\n", 3077 __kmp_gtid_from_thread(this_thr), 3078 __kmp_gtid_from_thread(thread))); 3079 __kmp_null_resume_wrapper(__kmp_gtid_from_thread(thread), sleep_loc); 3080 } else { 3081 KF_TRACE(50, ("__kmp_enable_tasking: T#%d don't wake up thread T#%d\n", 3082 __kmp_gtid_from_thread(this_thr), 3083 __kmp_gtid_from_thread(thread))); 3084 } 3085 } 3086 } 3087 3088 KA_TRACE(10, ("__kmp_enable_tasking(exit): T#%d\n", 3089 __kmp_gtid_from_thread(this_thr))); 3090 } 3091 3092 /* // TODO: Check the comment consistency 3093 * Utility routines for "task teams". A task team (kmp_task_t) is kind of 3094 * like a shadow of the kmp_team_t data struct, with a different lifetime. 3095 * After a child * thread checks into a barrier and calls __kmp_release() from 3096 * the particular variant of __kmp_<barrier_kind>_barrier_gather(), it can no 3097 * longer assume that the kmp_team_t structure is intact (at any moment, the 3098 * master thread may exit the barrier code and free the team data structure, 3099 * and return the threads to the thread pool). 3100 * 3101 * This does not work with the tasking code, as the thread is still 3102 * expected to participate in the execution of any tasks that may have been 3103 * spawned my a member of the team, and the thread still needs access to all 3104 * to each thread in the team, so that it can steal work from it. 3105 * 3106 * Enter the existence of the kmp_task_team_t struct. It employs a reference 3107 * counting mechanism, and is allocated by the master thread before calling 3108 * __kmp_<barrier_kind>_release, and then is release by the last thread to 3109 * exit __kmp_<barrier_kind>_release at the next barrier. I.e. the lifetimes 3110 * of the kmp_task_team_t structs for consecutive barriers can overlap 3111 * (and will, unless the master thread is the last thread to exit the barrier 3112 * release phase, which is not typical). The existence of such a struct is 3113 * useful outside the context of tasking. 3114 * 3115 * We currently use the existence of the threads array as an indicator that 3116 * tasks were spawned since the last barrier. If the structure is to be 3117 * useful outside the context of tasking, then this will have to change, but 3118 * not setting the field minimizes the performance impact of tasking on 3119 * barriers, when no explicit tasks were spawned (pushed, actually). 3120 */ 3121 3122 static kmp_task_team_t *__kmp_free_task_teams = 3123 NULL; // Free list for task_team data structures 3124 // Lock for task team data structures 3125 kmp_bootstrap_lock_t __kmp_task_team_lock = 3126 KMP_BOOTSTRAP_LOCK_INITIALIZER(__kmp_task_team_lock); 3127 3128 // __kmp_alloc_task_deque: 3129 // Allocates a task deque for a particular thread, and initialize the necessary 3130 // data structures relating to the deque. This only happens once per thread 3131 // per task team since task teams are recycled. No lock is needed during 3132 // allocation since each thread allocates its own deque. 3133 static void __kmp_alloc_task_deque(kmp_info_t *thread, 3134 kmp_thread_data_t *thread_data) { 3135 __kmp_init_bootstrap_lock(&thread_data->td.td_deque_lock); 3136 KMP_DEBUG_ASSERT(thread_data->td.td_deque == NULL); 3137 3138 // Initialize last stolen task field to "none" 3139 thread_data->td.td_deque_last_stolen = -1; 3140 3141 KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) == 0); 3142 KMP_DEBUG_ASSERT(thread_data->td.td_deque_head == 0); 3143 KMP_DEBUG_ASSERT(thread_data->td.td_deque_tail == 0); 3144 3145 KE_TRACE( 3146 10, 3147 ("__kmp_alloc_task_deque: T#%d allocating deque[%d] for thread_data %p\n", 3148 __kmp_gtid_from_thread(thread), INITIAL_TASK_DEQUE_SIZE, thread_data)); 3149 // Allocate space for task deque, and zero the deque 3150 // Cannot use __kmp_thread_calloc() because threads not around for 3151 // kmp_reap_task_team( ). 3152 thread_data->td.td_deque = (kmp_taskdata_t **)__kmp_allocate( 3153 INITIAL_TASK_DEQUE_SIZE * sizeof(kmp_taskdata_t *)); 3154 thread_data->td.td_deque_size = INITIAL_TASK_DEQUE_SIZE; 3155 } 3156 3157 // __kmp_free_task_deque: 3158 // Deallocates a task deque for a particular thread. Happens at library 3159 // deallocation so don't need to reset all thread data fields. 3160 static void __kmp_free_task_deque(kmp_thread_data_t *thread_data) { 3161 if (thread_data->td.td_deque != NULL) { 3162 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 3163 TCW_4(thread_data->td.td_deque_ntasks, 0); 3164 __kmp_free(thread_data->td.td_deque); 3165 thread_data->td.td_deque = NULL; 3166 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 3167 } 3168 3169 #ifdef BUILD_TIED_TASK_STACK 3170 // GEH: Figure out what to do here for td_susp_tied_tasks 3171 if (thread_data->td.td_susp_tied_tasks.ts_entries != TASK_STACK_EMPTY) { 3172 __kmp_free_task_stack(__kmp_thread_from_gtid(gtid), thread_data); 3173 } 3174 #endif // BUILD_TIED_TASK_STACK 3175 } 3176 3177 // __kmp_realloc_task_threads_data: 3178 // Allocates a threads_data array for a task team, either by allocating an 3179 // initial array or enlarging an existing array. Only the first thread to get 3180 // the lock allocs or enlarges the array and re-initializes the array elements. 3181 // That thread returns "TRUE", the rest return "FALSE". 3182 // Assumes that the new array size is given by task_team -> tt.tt_nproc. 3183 // The current size is given by task_team -> tt.tt_max_threads. 3184 static int __kmp_realloc_task_threads_data(kmp_info_t *thread, 3185 kmp_task_team_t *task_team) { 3186 kmp_thread_data_t **threads_data_p; 3187 kmp_int32 nthreads, maxthreads; 3188 int is_init_thread = FALSE; 3189 3190 if (TCR_4(task_team->tt.tt_found_tasks)) { 3191 // Already reallocated and initialized. 3192 return FALSE; 3193 } 3194 3195 threads_data_p = &task_team->tt.tt_threads_data; 3196 nthreads = task_team->tt.tt_nproc; 3197 maxthreads = task_team->tt.tt_max_threads; 3198 3199 // All threads must lock when they encounter the first task of the implicit 3200 // task region to make sure threads_data fields are (re)initialized before 3201 // used. 3202 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_threads_lock); 3203 3204 if (!TCR_4(task_team->tt.tt_found_tasks)) { 3205 // first thread to enable tasking 3206 kmp_team_t *team = thread->th.th_team; 3207 int i; 3208 3209 is_init_thread = TRUE; 3210 if (maxthreads < nthreads) { 3211 3212 if (*threads_data_p != NULL) { 3213 kmp_thread_data_t *old_data = *threads_data_p; 3214 kmp_thread_data_t *new_data = NULL; 3215 3216 KE_TRACE( 3217 10, 3218 ("__kmp_realloc_task_threads_data: T#%d reallocating " 3219 "threads data for task_team %p, new_size = %d, old_size = %d\n", 3220 __kmp_gtid_from_thread(thread), task_team, nthreads, maxthreads)); 3221 // Reallocate threads_data to have more elements than current array 3222 // Cannot use __kmp_thread_realloc() because threads not around for 3223 // kmp_reap_task_team( ). Note all new array entries are initialized 3224 // to zero by __kmp_allocate(). 3225 new_data = (kmp_thread_data_t *)__kmp_allocate( 3226 nthreads * sizeof(kmp_thread_data_t)); 3227 // copy old data to new data 3228 KMP_MEMCPY_S((void *)new_data, nthreads * sizeof(kmp_thread_data_t), 3229 (void *)old_data, maxthreads * sizeof(kmp_thread_data_t)); 3230 3231 #ifdef BUILD_TIED_TASK_STACK 3232 // GEH: Figure out if this is the right thing to do 3233 for (i = maxthreads; i < nthreads; i++) { 3234 kmp_thread_data_t *thread_data = &(*threads_data_p)[i]; 3235 __kmp_init_task_stack(__kmp_gtid_from_thread(thread), thread_data); 3236 } 3237 #endif // BUILD_TIED_TASK_STACK 3238 // Install the new data and free the old data 3239 (*threads_data_p) = new_data; 3240 __kmp_free(old_data); 3241 } else { 3242 KE_TRACE(10, ("__kmp_realloc_task_threads_data: T#%d allocating " 3243 "threads data for task_team %p, size = %d\n", 3244 __kmp_gtid_from_thread(thread), task_team, nthreads)); 3245 // Make the initial allocate for threads_data array, and zero entries 3246 // Cannot use __kmp_thread_calloc() because threads not around for 3247 // kmp_reap_task_team( ). 3248 ANNOTATE_IGNORE_WRITES_BEGIN(); 3249 *threads_data_p = (kmp_thread_data_t *)__kmp_allocate( 3250 nthreads * sizeof(kmp_thread_data_t)); 3251 ANNOTATE_IGNORE_WRITES_END(); 3252 #ifdef BUILD_TIED_TASK_STACK 3253 // GEH: Figure out if this is the right thing to do 3254 for (i = 0; i < nthreads; i++) { 3255 kmp_thread_data_t *thread_data = &(*threads_data_p)[i]; 3256 __kmp_init_task_stack(__kmp_gtid_from_thread(thread), thread_data); 3257 } 3258 #endif // BUILD_TIED_TASK_STACK 3259 } 3260 task_team->tt.tt_max_threads = nthreads; 3261 } else { 3262 // If array has (more than) enough elements, go ahead and use it 3263 KMP_DEBUG_ASSERT(*threads_data_p != NULL); 3264 } 3265 3266 // initialize threads_data pointers back to thread_info structures 3267 for (i = 0; i < nthreads; i++) { 3268 kmp_thread_data_t *thread_data = &(*threads_data_p)[i]; 3269 thread_data->td.td_thr = team->t.t_threads[i]; 3270 3271 if (thread_data->td.td_deque_last_stolen >= nthreads) { 3272 // The last stolen field survives across teams / barrier, and the number 3273 // of threads may have changed. It's possible (likely?) that a new 3274 // parallel region will exhibit the same behavior as previous region. 3275 thread_data->td.td_deque_last_stolen = -1; 3276 } 3277 } 3278 3279 KMP_MB(); 3280 TCW_SYNC_4(task_team->tt.tt_found_tasks, TRUE); 3281 } 3282 3283 __kmp_release_bootstrap_lock(&task_team->tt.tt_threads_lock); 3284 return is_init_thread; 3285 } 3286 3287 // __kmp_free_task_threads_data: 3288 // Deallocates a threads_data array for a task team, including any attached 3289 // tasking deques. Only occurs at library shutdown. 3290 static void __kmp_free_task_threads_data(kmp_task_team_t *task_team) { 3291 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_threads_lock); 3292 if (task_team->tt.tt_threads_data != NULL) { 3293 int i; 3294 for (i = 0; i < task_team->tt.tt_max_threads; i++) { 3295 __kmp_free_task_deque(&task_team->tt.tt_threads_data[i]); 3296 } 3297 __kmp_free(task_team->tt.tt_threads_data); 3298 task_team->tt.tt_threads_data = NULL; 3299 } 3300 __kmp_release_bootstrap_lock(&task_team->tt.tt_threads_lock); 3301 } 3302 3303 // __kmp_allocate_task_team: 3304 // Allocates a task team associated with a specific team, taking it from 3305 // the global task team free list if possible. Also initializes data 3306 // structures. 3307 static kmp_task_team_t *__kmp_allocate_task_team(kmp_info_t *thread, 3308 kmp_team_t *team) { 3309 kmp_task_team_t *task_team = NULL; 3310 int nthreads; 3311 3312 KA_TRACE(20, ("__kmp_allocate_task_team: T#%d entering; team = %p\n", 3313 (thread ? __kmp_gtid_from_thread(thread) : -1), team)); 3314 3315 if (TCR_PTR(__kmp_free_task_teams) != NULL) { 3316 // Take a task team from the task team pool 3317 __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock); 3318 if (__kmp_free_task_teams != NULL) { 3319 task_team = __kmp_free_task_teams; 3320 TCW_PTR(__kmp_free_task_teams, task_team->tt.tt_next); 3321 task_team->tt.tt_next = NULL; 3322 } 3323 __kmp_release_bootstrap_lock(&__kmp_task_team_lock); 3324 } 3325 3326 if (task_team == NULL) { 3327 KE_TRACE(10, ("__kmp_allocate_task_team: T#%d allocating " 3328 "task team for team %p\n", 3329 __kmp_gtid_from_thread(thread), team)); 3330 // Allocate a new task team if one is not available. 3331 // Cannot use __kmp_thread_malloc() because threads not around for 3332 // kmp_reap_task_team( ). 3333 task_team = (kmp_task_team_t *)__kmp_allocate(sizeof(kmp_task_team_t)); 3334 __kmp_init_bootstrap_lock(&task_team->tt.tt_threads_lock); 3335 // AC: __kmp_allocate zeroes returned memory 3336 // task_team -> tt.tt_threads_data = NULL; 3337 // task_team -> tt.tt_max_threads = 0; 3338 // task_team -> tt.tt_next = NULL; 3339 } 3340 3341 TCW_4(task_team->tt.tt_found_tasks, FALSE); 3342 TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE); 3343 task_team->tt.tt_nproc = nthreads = team->t.t_nproc; 3344 3345 KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads, nthreads); 3346 TCW_4(task_team->tt.tt_active, TRUE); 3347 3348 KA_TRACE(20, ("__kmp_allocate_task_team: T#%d exiting; task_team = %p " 3349 "unfinished_threads init'd to %d\n", 3350 (thread ? __kmp_gtid_from_thread(thread) : -1), task_team, 3351 KMP_ATOMIC_LD_RLX(&task_team->tt.tt_unfinished_threads))); 3352 return task_team; 3353 } 3354 3355 // __kmp_free_task_team: 3356 // Frees the task team associated with a specific thread, and adds it 3357 // to the global task team free list. 3358 void __kmp_free_task_team(kmp_info_t *thread, kmp_task_team_t *task_team) { 3359 KA_TRACE(20, ("__kmp_free_task_team: T#%d task_team = %p\n", 3360 thread ? __kmp_gtid_from_thread(thread) : -1, task_team)); 3361 3362 // Put task team back on free list 3363 __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock); 3364 3365 KMP_DEBUG_ASSERT(task_team->tt.tt_next == NULL); 3366 task_team->tt.tt_next = __kmp_free_task_teams; 3367 TCW_PTR(__kmp_free_task_teams, task_team); 3368 3369 __kmp_release_bootstrap_lock(&__kmp_task_team_lock); 3370 } 3371 3372 // __kmp_reap_task_teams: 3373 // Free all the task teams on the task team free list. 3374 // Should only be done during library shutdown. 3375 // Cannot do anything that needs a thread structure or gtid since they are 3376 // already gone. 3377 void __kmp_reap_task_teams(void) { 3378 kmp_task_team_t *task_team; 3379 3380 if (TCR_PTR(__kmp_free_task_teams) != NULL) { 3381 // Free all task_teams on the free list 3382 __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock); 3383 while ((task_team = __kmp_free_task_teams) != NULL) { 3384 __kmp_free_task_teams = task_team->tt.tt_next; 3385 task_team->tt.tt_next = NULL; 3386 3387 // Free threads_data if necessary 3388 if (task_team->tt.tt_threads_data != NULL) { 3389 __kmp_free_task_threads_data(task_team); 3390 } 3391 __kmp_free(task_team); 3392 } 3393 __kmp_release_bootstrap_lock(&__kmp_task_team_lock); 3394 } 3395 } 3396 3397 // __kmp_wait_to_unref_task_teams: 3398 // Some threads could still be in the fork barrier release code, possibly 3399 // trying to steal tasks. Wait for each thread to unreference its task team. 3400 void __kmp_wait_to_unref_task_teams(void) { 3401 kmp_info_t *thread; 3402 kmp_uint32 spins; 3403 int done; 3404 3405 KMP_INIT_YIELD(spins); 3406 3407 for (;;) { 3408 done = TRUE; 3409 3410 // TODO: GEH - this may be is wrong because some sync would be necessary 3411 // in case threads are added to the pool during the traversal. Need to 3412 // verify that lock for thread pool is held when calling this routine. 3413 for (thread = CCAST(kmp_info_t *, __kmp_thread_pool); thread != NULL; 3414 thread = thread->th.th_next_pool) { 3415 #if KMP_OS_WINDOWS 3416 DWORD exit_val; 3417 #endif 3418 if (TCR_PTR(thread->th.th_task_team) == NULL) { 3419 KA_TRACE(10, ("__kmp_wait_to_unref_task_team: T#%d task_team == NULL\n", 3420 __kmp_gtid_from_thread(thread))); 3421 continue; 3422 } 3423 #if KMP_OS_WINDOWS 3424 // TODO: GEH - add this check for Linux* OS / OS X* as well? 3425 if (!__kmp_is_thread_alive(thread, &exit_val)) { 3426 thread->th.th_task_team = NULL; 3427 continue; 3428 } 3429 #endif 3430 3431 done = FALSE; // Because th_task_team pointer is not NULL for this thread 3432 3433 KA_TRACE(10, ("__kmp_wait_to_unref_task_team: Waiting for T#%d to " 3434 "unreference task_team\n", 3435 __kmp_gtid_from_thread(thread))); 3436 3437 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) { 3438 volatile void *sleep_loc; 3439 // If the thread is sleeping, awaken it. 3440 if ((sleep_loc = TCR_PTR(CCAST(void *, thread->th.th_sleep_loc))) != 3441 NULL) { 3442 KA_TRACE( 3443 10, 3444 ("__kmp_wait_to_unref_task_team: T#%d waking up thread T#%d\n", 3445 __kmp_gtid_from_thread(thread), __kmp_gtid_from_thread(thread))); 3446 __kmp_null_resume_wrapper(__kmp_gtid_from_thread(thread), sleep_loc); 3447 } 3448 } 3449 } 3450 if (done) { 3451 break; 3452 } 3453 3454 // If oversubscribed or have waited a bit, yield. 3455 KMP_YIELD_OVERSUB_ELSE_SPIN(spins); 3456 } 3457 } 3458 3459 // __kmp_task_team_setup: Create a task_team for the current team, but use 3460 // an already created, unused one if it already exists. 3461 void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team, int always) { 3462 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 3463 3464 // If this task_team hasn't been created yet, allocate it. It will be used in 3465 // the region after the next. 3466 // If it exists, it is the current task team and shouldn't be touched yet as 3467 // it may still be in use. 3468 if (team->t.t_task_team[this_thr->th.th_task_state] == NULL && 3469 (always || team->t.t_nproc > 1)) { 3470 team->t.t_task_team[this_thr->th.th_task_state] = 3471 __kmp_allocate_task_team(this_thr, team); 3472 KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d created new task_team %p " 3473 "for team %d at parity=%d\n", 3474 __kmp_gtid_from_thread(this_thr), 3475 team->t.t_task_team[this_thr->th.th_task_state], 3476 ((team != NULL) ? team->t.t_id : -1), 3477 this_thr->th.th_task_state)); 3478 } 3479 3480 // After threads exit the release, they will call sync, and then point to this 3481 // other task_team; make sure it is allocated and properly initialized. As 3482 // threads spin in the barrier release phase, they will continue to use the 3483 // previous task_team struct(above), until they receive the signal to stop 3484 // checking for tasks (they can't safely reference the kmp_team_t struct, 3485 // which could be reallocated by the master thread). No task teams are formed 3486 // for serialized teams. 3487 if (team->t.t_nproc > 1) { 3488 int other_team = 1 - this_thr->th.th_task_state; 3489 if (team->t.t_task_team[other_team] == NULL) { // setup other team as well 3490 team->t.t_task_team[other_team] = 3491 __kmp_allocate_task_team(this_thr, team); 3492 KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d created second new " 3493 "task_team %p for team %d at parity=%d\n", 3494 __kmp_gtid_from_thread(this_thr), 3495 team->t.t_task_team[other_team], 3496 ((team != NULL) ? team->t.t_id : -1), other_team)); 3497 } else { // Leave the old task team struct in place for the upcoming region; 3498 // adjust as needed 3499 kmp_task_team_t *task_team = team->t.t_task_team[other_team]; 3500 if (!task_team->tt.tt_active || 3501 team->t.t_nproc != task_team->tt.tt_nproc) { 3502 TCW_4(task_team->tt.tt_nproc, team->t.t_nproc); 3503 TCW_4(task_team->tt.tt_found_tasks, FALSE); 3504 TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE); 3505 KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads, 3506 team->t.t_nproc); 3507 TCW_4(task_team->tt.tt_active, TRUE); 3508 } 3509 // if team size has changed, the first thread to enable tasking will 3510 // realloc threads_data if necessary 3511 KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d reset next task_team " 3512 "%p for team %d at parity=%d\n", 3513 __kmp_gtid_from_thread(this_thr), 3514 team->t.t_task_team[other_team], 3515 ((team != NULL) ? team->t.t_id : -1), other_team)); 3516 } 3517 } 3518 } 3519 3520 // __kmp_task_team_sync: Propagation of task team data from team to threads 3521 // which happens just after the release phase of a team barrier. This may be 3522 // called by any thread, but only for teams with # threads > 1. 3523 void __kmp_task_team_sync(kmp_info_t *this_thr, kmp_team_t *team) { 3524 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 3525 3526 // Toggle the th_task_state field, to switch which task_team this thread 3527 // refers to 3528 this_thr->th.th_task_state = 1 - this_thr->th.th_task_state; 3529 // It is now safe to propagate the task team pointer from the team struct to 3530 // the current thread. 3531 TCW_PTR(this_thr->th.th_task_team, 3532 team->t.t_task_team[this_thr->th.th_task_state]); 3533 KA_TRACE(20, 3534 ("__kmp_task_team_sync: Thread T#%d task team switched to task_team " 3535 "%p from Team #%d (parity=%d)\n", 3536 __kmp_gtid_from_thread(this_thr), this_thr->th.th_task_team, 3537 ((team != NULL) ? team->t.t_id : -1), this_thr->th.th_task_state)); 3538 } 3539 3540 // __kmp_task_team_wait: Master thread waits for outstanding tasks after the 3541 // barrier gather phase. Only called by master thread if #threads in team > 1 or 3542 // if proxy tasks were created. 3543 // 3544 // wait is a flag that defaults to 1 (see kmp.h), but waiting can be turned off 3545 // by passing in 0 optionally as the last argument. When wait is zero, master 3546 // thread does not wait for unfinished_threads to reach 0. 3547 void __kmp_task_team_wait( 3548 kmp_info_t *this_thr, 3549 kmp_team_t *team USE_ITT_BUILD_ARG(void *itt_sync_obj), int wait) { 3550 kmp_task_team_t *task_team = team->t.t_task_team[this_thr->th.th_task_state]; 3551 3552 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 3553 KMP_DEBUG_ASSERT(task_team == this_thr->th.th_task_team); 3554 3555 if ((task_team != NULL) && KMP_TASKING_ENABLED(task_team)) { 3556 if (wait) { 3557 KA_TRACE(20, ("__kmp_task_team_wait: Master T#%d waiting for all tasks " 3558 "(for unfinished_threads to reach 0) on task_team = %p\n", 3559 __kmp_gtid_from_thread(this_thr), task_team)); 3560 // Worker threads may have dropped through to release phase, but could 3561 // still be executing tasks. Wait here for tasks to complete. To avoid 3562 // memory contention, only master thread checks termination condition. 3563 kmp_flag_32 flag(RCAST(std::atomic<kmp_uint32> *, 3564 &task_team->tt.tt_unfinished_threads), 3565 0U); 3566 flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj)); 3567 } 3568 // Deactivate the old task team, so that the worker threads will stop 3569 // referencing it while spinning. 3570 KA_TRACE( 3571 20, 3572 ("__kmp_task_team_wait: Master T#%d deactivating task_team %p: " 3573 "setting active to false, setting local and team's pointer to NULL\n", 3574 __kmp_gtid_from_thread(this_thr), task_team)); 3575 KMP_DEBUG_ASSERT(task_team->tt.tt_nproc > 1 || 3576 task_team->tt.tt_found_proxy_tasks == TRUE); 3577 TCW_SYNC_4(task_team->tt.tt_found_proxy_tasks, FALSE); 3578 KMP_CHECK_UPDATE(task_team->tt.tt_untied_task_encountered, 0); 3579 TCW_SYNC_4(task_team->tt.tt_active, FALSE); 3580 KMP_MB(); 3581 3582 TCW_PTR(this_thr->th.th_task_team, NULL); 3583 } 3584 } 3585 3586 // __kmp_tasking_barrier: 3587 // This routine may only called when __kmp_tasking_mode == tskm_extra_barrier. 3588 // Internal function to execute all tasks prior to a regular barrier or a join 3589 // barrier. It is a full barrier itself, which unfortunately turns regular 3590 // barriers into double barriers and join barriers into 1 1/2 barriers. 3591 void __kmp_tasking_barrier(kmp_team_t *team, kmp_info_t *thread, int gtid) { 3592 std::atomic<kmp_uint32> *spin = RCAST( 3593 std::atomic<kmp_uint32> *, 3594 &team->t.t_task_team[thread->th.th_task_state]->tt.tt_unfinished_threads); 3595 int flag = FALSE; 3596 KMP_DEBUG_ASSERT(__kmp_tasking_mode == tskm_extra_barrier); 3597 3598 #if USE_ITT_BUILD 3599 KMP_FSYNC_SPIN_INIT(spin, NULL); 3600 #endif /* USE_ITT_BUILD */ 3601 kmp_flag_32 spin_flag(spin, 0U); 3602 while (!spin_flag.execute_tasks(thread, gtid, TRUE, 3603 &flag USE_ITT_BUILD_ARG(NULL), 0)) { 3604 #if USE_ITT_BUILD 3605 // TODO: What about itt_sync_obj?? 3606 KMP_FSYNC_SPIN_PREPARE(RCAST(void *, spin)); 3607 #endif /* USE_ITT_BUILD */ 3608 3609 if (TCR_4(__kmp_global.g.g_done)) { 3610 if (__kmp_global.g.g_abort) 3611 __kmp_abort_thread(); 3612 break; 3613 } 3614 KMP_YIELD(TRUE); 3615 } 3616 #if USE_ITT_BUILD 3617 KMP_FSYNC_SPIN_ACQUIRED(RCAST(void *, spin)); 3618 #endif /* USE_ITT_BUILD */ 3619 } 3620 3621 // __kmp_give_task puts a task into a given thread queue if: 3622 // - the queue for that thread was created 3623 // - there's space in that queue 3624 // Because of this, __kmp_push_task needs to check if there's space after 3625 // getting the lock 3626 static bool __kmp_give_task(kmp_info_t *thread, kmp_int32 tid, kmp_task_t *task, 3627 kmp_int32 pass) { 3628 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 3629 kmp_task_team_t *task_team = taskdata->td_task_team; 3630 3631 KA_TRACE(20, ("__kmp_give_task: trying to give task %p to thread %d.\n", 3632 taskdata, tid)); 3633 3634 // If task_team is NULL something went really bad... 3635 KMP_DEBUG_ASSERT(task_team != NULL); 3636 3637 bool result = false; 3638 kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid]; 3639 3640 if (thread_data->td.td_deque == NULL) { 3641 // There's no queue in this thread, go find another one 3642 // We're guaranteed that at least one thread has a queue 3643 KA_TRACE(30, 3644 ("__kmp_give_task: thread %d has no queue while giving task %p.\n", 3645 tid, taskdata)); 3646 return result; 3647 } 3648 3649 if (TCR_4(thread_data->td.td_deque_ntasks) >= 3650 TASK_DEQUE_SIZE(thread_data->td)) { 3651 KA_TRACE( 3652 30, 3653 ("__kmp_give_task: queue is full while giving task %p to thread %d.\n", 3654 taskdata, tid)); 3655 3656 // if this deque is bigger than the pass ratio give a chance to another 3657 // thread 3658 if (TASK_DEQUE_SIZE(thread_data->td) / INITIAL_TASK_DEQUE_SIZE >= pass) 3659 return result; 3660 3661 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 3662 __kmp_realloc_task_deque(thread, thread_data); 3663 3664 } else { 3665 3666 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 3667 3668 if (TCR_4(thread_data->td.td_deque_ntasks) >= 3669 TASK_DEQUE_SIZE(thread_data->td)) { 3670 KA_TRACE(30, ("__kmp_give_task: queue is full while giving task %p to " 3671 "thread %d.\n", 3672 taskdata, tid)); 3673 3674 // if this deque is bigger than the pass ratio give a chance to another 3675 // thread 3676 if (TASK_DEQUE_SIZE(thread_data->td) / INITIAL_TASK_DEQUE_SIZE >= pass) 3677 goto release_and_exit; 3678 3679 __kmp_realloc_task_deque(thread, thread_data); 3680 } 3681 } 3682 3683 // lock is held here, and there is space in the deque 3684 3685 thread_data->td.td_deque[thread_data->td.td_deque_tail] = taskdata; 3686 // Wrap index. 3687 thread_data->td.td_deque_tail = 3688 (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td); 3689 TCW_4(thread_data->td.td_deque_ntasks, 3690 TCR_4(thread_data->td.td_deque_ntasks) + 1); 3691 3692 result = true; 3693 KA_TRACE(30, ("__kmp_give_task: successfully gave task %p to thread %d.\n", 3694 taskdata, tid)); 3695 3696 release_and_exit: 3697 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 3698 3699 return result; 3700 } 3701 3702 /* The finish of the proxy tasks is divided in two pieces: 3703 - the top half is the one that can be done from a thread outside the team 3704 - the bottom half must be run from a thread within the team 3705 3706 In order to run the bottom half the task gets queued back into one of the 3707 threads of the team. Once the td_incomplete_child_task counter of the parent 3708 is decremented the threads can leave the barriers. So, the bottom half needs 3709 to be queued before the counter is decremented. The top half is therefore 3710 divided in two parts: 3711 - things that can be run before queuing the bottom half 3712 - things that must be run after queuing the bottom half 3713 3714 This creates a second race as the bottom half can free the task before the 3715 second top half is executed. To avoid this we use the 3716 td_incomplete_child_task of the proxy task to synchronize the top and bottom 3717 half. */ 3718 static void __kmp_first_top_half_finish_proxy(kmp_taskdata_t *taskdata) { 3719 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); 3720 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); 3721 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); 3722 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); 3723 3724 taskdata->td_flags.complete = 1; // mark the task as completed 3725 3726 if (taskdata->td_taskgroup) 3727 KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count); 3728 3729 // Create an imaginary children for this task so the bottom half cannot 3730 // release the task before we have completed the second top half 3731 KMP_ATOMIC_INC(&taskdata->td_incomplete_child_tasks); 3732 } 3733 3734 static void __kmp_second_top_half_finish_proxy(kmp_taskdata_t *taskdata) { 3735 kmp_int32 children = 0; 3736 3737 // Predecrement simulated by "- 1" calculation 3738 children = 3739 KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks) - 1; 3740 KMP_DEBUG_ASSERT(children >= 0); 3741 3742 // Remove the imaginary children 3743 KMP_ATOMIC_DEC(&taskdata->td_incomplete_child_tasks); 3744 } 3745 3746 static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask) { 3747 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); 3748 kmp_info_t *thread = __kmp_threads[gtid]; 3749 3750 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); 3751 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 3752 1); // top half must run before bottom half 3753 3754 // We need to wait to make sure the top half is finished 3755 // Spinning here should be ok as this should happen quickly 3756 while (KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) > 0) 3757 ; 3758 3759 __kmp_release_deps(gtid, taskdata); 3760 __kmp_free_task_and_ancestors(gtid, taskdata, thread); 3761 } 3762 3763 /*! 3764 @ingroup TASKING 3765 @param gtid Global Thread ID of encountering thread 3766 @param ptask Task which execution is completed 3767 3768 Execute the completion of a proxy task from a thread of that is part of the 3769 team. Run first and bottom halves directly. 3770 */ 3771 void __kmpc_proxy_task_completed(kmp_int32 gtid, kmp_task_t *ptask) { 3772 KMP_DEBUG_ASSERT(ptask != NULL); 3773 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); 3774 KA_TRACE( 3775 10, ("__kmp_proxy_task_completed(enter): T#%d proxy task %p completing\n", 3776 gtid, taskdata)); 3777 3778 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); 3779 3780 __kmp_first_top_half_finish_proxy(taskdata); 3781 __kmp_second_top_half_finish_proxy(taskdata); 3782 __kmp_bottom_half_finish_proxy(gtid, ptask); 3783 3784 KA_TRACE(10, 3785 ("__kmp_proxy_task_completed(exit): T#%d proxy task %p completing\n", 3786 gtid, taskdata)); 3787 } 3788 3789 /*! 3790 @ingroup TASKING 3791 @param ptask Task which execution is completed 3792 3793 Execute the completion of a proxy task from a thread that could not belong to 3794 the team. 3795 */ 3796 void __kmpc_proxy_task_completed_ooo(kmp_task_t *ptask) { 3797 KMP_DEBUG_ASSERT(ptask != NULL); 3798 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); 3799 3800 KA_TRACE( 3801 10, 3802 ("__kmp_proxy_task_completed_ooo(enter): proxy task completing ooo %p\n", 3803 taskdata)); 3804 3805 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); 3806 3807 __kmp_first_top_half_finish_proxy(taskdata); 3808 3809 // Enqueue task to complete bottom half completion from a thread within the 3810 // corresponding team 3811 kmp_team_t *team = taskdata->td_team; 3812 kmp_int32 nthreads = team->t.t_nproc; 3813 kmp_info_t *thread; 3814 3815 // This should be similar to start_k = __kmp_get_random( thread ) % nthreads 3816 // but we cannot use __kmp_get_random here 3817 kmp_int32 start_k = 0; 3818 kmp_int32 pass = 1; 3819 kmp_int32 k = start_k; 3820 3821 do { 3822 // For now we're just linearly trying to find a thread 3823 thread = team->t.t_threads[k]; 3824 k = (k + 1) % nthreads; 3825 3826 // we did a full pass through all the threads 3827 if (k == start_k) 3828 pass = pass << 1; 3829 3830 } while (!__kmp_give_task(thread, k, ptask, pass)); 3831 3832 __kmp_second_top_half_finish_proxy(taskdata); 3833 3834 KA_TRACE( 3835 10, 3836 ("__kmp_proxy_task_completed_ooo(exit): proxy task completing ooo %p\n", 3837 taskdata)); 3838 } 3839 3840 kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref, int gtid, 3841 kmp_task_t *task) { 3842 kmp_taskdata_t *td = KMP_TASK_TO_TASKDATA(task); 3843 if (td->td_allow_completion_event.type == KMP_EVENT_UNINITIALIZED) { 3844 td->td_allow_completion_event.type = KMP_EVENT_ALLOW_COMPLETION; 3845 td->td_allow_completion_event.ed.task = task; 3846 __kmp_init_tas_lock(&td->td_allow_completion_event.lock); 3847 } 3848 return &td->td_allow_completion_event; 3849 } 3850 3851 void __kmp_fulfill_event(kmp_event_t *event) { 3852 if (event->type == KMP_EVENT_ALLOW_COMPLETION) { 3853 kmp_task_t *ptask = event->ed.task; 3854 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); 3855 bool detached = false; 3856 int gtid = __kmp_get_gtid(); 3857 3858 // The associated task might have completed or could be completing at this 3859 // point. 3860 // We need to take the lock to avoid races 3861 __kmp_acquire_tas_lock(&event->lock, gtid); 3862 if (taskdata->td_flags.proxy == TASK_PROXY) 3863 detached = true; 3864 event->type = KMP_EVENT_UNINITIALIZED; 3865 __kmp_release_tas_lock(&event->lock, gtid); 3866 3867 if (detached) { 3868 // If the task detached complete the proxy task 3869 if (gtid >= 0) { 3870 kmp_team_t *team = taskdata->td_team; 3871 kmp_info_t *thread = __kmp_get_thread(); 3872 if (thread->th.th_team == team) { 3873 __kmpc_proxy_task_completed(gtid, ptask); 3874 return; 3875 } 3876 } 3877 3878 // fallback 3879 __kmpc_proxy_task_completed_ooo(ptask); 3880 } 3881 } 3882 } 3883 3884 // __kmp_task_dup_alloc: Allocate the taskdata and make a copy of source task 3885 // for taskloop 3886 // 3887 // thread: allocating thread 3888 // task_src: pointer to source task to be duplicated 3889 // returns: a pointer to the allocated kmp_task_t structure (task). 3890 kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src) { 3891 kmp_task_t *task; 3892 kmp_taskdata_t *taskdata; 3893 kmp_taskdata_t *taskdata_src; 3894 kmp_taskdata_t *parent_task = thread->th.th_current_task; 3895 size_t shareds_offset; 3896 size_t task_size; 3897 3898 KA_TRACE(10, ("__kmp_task_dup_alloc(enter): Th %p, source task %p\n", thread, 3899 task_src)); 3900 taskdata_src = KMP_TASK_TO_TASKDATA(task_src); 3901 KMP_DEBUG_ASSERT(taskdata_src->td_flags.proxy == 3902 TASK_FULL); // it should not be proxy task 3903 KMP_DEBUG_ASSERT(taskdata_src->td_flags.tasktype == TASK_EXPLICIT); 3904 task_size = taskdata_src->td_size_alloc; 3905 3906 // Allocate a kmp_taskdata_t block and a kmp_task_t block. 3907 KA_TRACE(30, ("__kmp_task_dup_alloc: Th %p, malloc size %ld\n", thread, 3908 task_size)); 3909 #if USE_FAST_MEMORY 3910 taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, task_size); 3911 #else 3912 taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, task_size); 3913 #endif /* USE_FAST_MEMORY */ 3914 KMP_MEMCPY(taskdata, taskdata_src, task_size); 3915 3916 task = KMP_TASKDATA_TO_TASK(taskdata); 3917 3918 // Initialize new task (only specific fields not affected by memcpy) 3919 taskdata->td_task_id = KMP_GEN_TASK_ID(); 3920 if (task->shareds != NULL) { // need setup shareds pointer 3921 shareds_offset = (char *)task_src->shareds - (char *)taskdata_src; 3922 task->shareds = &((char *)taskdata)[shareds_offset]; 3923 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) == 3924 0); 3925 } 3926 taskdata->td_alloc_thread = thread; 3927 taskdata->td_parent = parent_task; 3928 // task inherits the taskgroup from the parent task 3929 taskdata->td_taskgroup = parent_task->td_taskgroup; 3930 // tied task needs to initialize the td_last_tied at creation, 3931 // untied one does this when it is scheduled for execution 3932 if (taskdata->td_flags.tiedness == TASK_TIED) 3933 taskdata->td_last_tied = taskdata; 3934 3935 // Only need to keep track of child task counts if team parallel and tasking 3936 // not serialized 3937 if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) { 3938 KMP_ATOMIC_INC(&parent_task->td_incomplete_child_tasks); 3939 if (parent_task->td_taskgroup) 3940 KMP_ATOMIC_INC(&parent_task->td_taskgroup->count); 3941 // Only need to keep track of allocated child tasks for explicit tasks since 3942 // implicit not deallocated 3943 if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT) 3944 KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks); 3945 } 3946 3947 KA_TRACE(20, 3948 ("__kmp_task_dup_alloc(exit): Th %p, created task %p, parent=%p\n", 3949 thread, taskdata, taskdata->td_parent)); 3950 #if OMPT_SUPPORT 3951 if (UNLIKELY(ompt_enabled.enabled)) 3952 __ompt_task_init(taskdata, thread->th.th_info.ds.ds_gtid); 3953 #endif 3954 return task; 3955 } 3956 3957 // Routine optionally generated by the compiler for setting the lastprivate flag 3958 // and calling needed constructors for private/firstprivate objects 3959 // (used to form taskloop tasks from pattern task) 3960 // Parameters: dest task, src task, lastprivate flag. 3961 typedef void (*p_task_dup_t)(kmp_task_t *, kmp_task_t *, kmp_int32); 3962 3963 KMP_BUILD_ASSERT(sizeof(long) == 4 || sizeof(long) == 8); 3964 3965 // class to encapsulate manipulating loop bounds in a taskloop task. 3966 // this abstracts away the Intel vs GOMP taskloop interface for setting/getting 3967 // the loop bound variables. 3968 class kmp_taskloop_bounds_t { 3969 kmp_task_t *task; 3970 const kmp_taskdata_t *taskdata; 3971 size_t lower_offset; 3972 size_t upper_offset; 3973 3974 public: 3975 kmp_taskloop_bounds_t(kmp_task_t *_task, kmp_uint64 *lb, kmp_uint64 *ub) 3976 : task(_task), taskdata(KMP_TASK_TO_TASKDATA(task)), 3977 lower_offset((char *)lb - (char *)task), 3978 upper_offset((char *)ub - (char *)task) { 3979 KMP_DEBUG_ASSERT((char *)lb > (char *)_task); 3980 KMP_DEBUG_ASSERT((char *)ub > (char *)_task); 3981 } 3982 kmp_taskloop_bounds_t(kmp_task_t *_task, const kmp_taskloop_bounds_t &bounds) 3983 : task(_task), taskdata(KMP_TASK_TO_TASKDATA(_task)), 3984 lower_offset(bounds.lower_offset), upper_offset(bounds.upper_offset) {} 3985 size_t get_lower_offset() const { return lower_offset; } 3986 size_t get_upper_offset() const { return upper_offset; } 3987 kmp_uint64 get_lb() const { 3988 kmp_int64 retval; 3989 #if defined(KMP_GOMP_COMPAT) 3990 // Intel task just returns the lower bound normally 3991 if (!taskdata->td_flags.native) { 3992 retval = *(kmp_int64 *)((char *)task + lower_offset); 3993 } else { 3994 // GOMP task has to take into account the sizeof(long) 3995 if (taskdata->td_size_loop_bounds == 4) { 3996 kmp_int32 *lb = RCAST(kmp_int32 *, task->shareds); 3997 retval = (kmp_int64)*lb; 3998 } else { 3999 kmp_int64 *lb = RCAST(kmp_int64 *, task->shareds); 4000 retval = (kmp_int64)*lb; 4001 } 4002 } 4003 #else 4004 retval = *(kmp_int64 *)((char *)task + lower_offset); 4005 #endif // defined(KMP_GOMP_COMPAT) 4006 return retval; 4007 } 4008 kmp_uint64 get_ub() const { 4009 kmp_int64 retval; 4010 #if defined(KMP_GOMP_COMPAT) 4011 // Intel task just returns the upper bound normally 4012 if (!taskdata->td_flags.native) { 4013 retval = *(kmp_int64 *)((char *)task + upper_offset); 4014 } else { 4015 // GOMP task has to take into account the sizeof(long) 4016 if (taskdata->td_size_loop_bounds == 4) { 4017 kmp_int32 *ub = RCAST(kmp_int32 *, task->shareds) + 1; 4018 retval = (kmp_int64)*ub; 4019 } else { 4020 kmp_int64 *ub = RCAST(kmp_int64 *, task->shareds) + 1; 4021 retval = (kmp_int64)*ub; 4022 } 4023 } 4024 #else 4025 retval = *(kmp_int64 *)((char *)task + upper_offset); 4026 #endif // defined(KMP_GOMP_COMPAT) 4027 return retval; 4028 } 4029 void set_lb(kmp_uint64 lb) { 4030 #if defined(KMP_GOMP_COMPAT) 4031 // Intel task just sets the lower bound normally 4032 if (!taskdata->td_flags.native) { 4033 *(kmp_uint64 *)((char *)task + lower_offset) = lb; 4034 } else { 4035 // GOMP task has to take into account the sizeof(long) 4036 if (taskdata->td_size_loop_bounds == 4) { 4037 kmp_uint32 *lower = RCAST(kmp_uint32 *, task->shareds); 4038 *lower = (kmp_uint32)lb; 4039 } else { 4040 kmp_uint64 *lower = RCAST(kmp_uint64 *, task->shareds); 4041 *lower = (kmp_uint64)lb; 4042 } 4043 } 4044 #else 4045 *(kmp_uint64 *)((char *)task + lower_offset) = lb; 4046 #endif // defined(KMP_GOMP_COMPAT) 4047 } 4048 void set_ub(kmp_uint64 ub) { 4049 #if defined(KMP_GOMP_COMPAT) 4050 // Intel task just sets the upper bound normally 4051 if (!taskdata->td_flags.native) { 4052 *(kmp_uint64 *)((char *)task + upper_offset) = ub; 4053 } else { 4054 // GOMP task has to take into account the sizeof(long) 4055 if (taskdata->td_size_loop_bounds == 4) { 4056 kmp_uint32 *upper = RCAST(kmp_uint32 *, task->shareds) + 1; 4057 *upper = (kmp_uint32)ub; 4058 } else { 4059 kmp_uint64 *upper = RCAST(kmp_uint64 *, task->shareds) + 1; 4060 *upper = (kmp_uint64)ub; 4061 } 4062 } 4063 #else 4064 *(kmp_uint64 *)((char *)task + upper_offset) = ub; 4065 #endif // defined(KMP_GOMP_COMPAT) 4066 } 4067 }; 4068 4069 // __kmp_taskloop_linear: Start tasks of the taskloop linearly 4070 // 4071 // loc Source location information 4072 // gtid Global thread ID 4073 // task Pattern task, exposes the loop iteration range 4074 // lb Pointer to loop lower bound in task structure 4075 // ub Pointer to loop upper bound in task structure 4076 // st Loop stride 4077 // ub_glob Global upper bound (used for lastprivate check) 4078 // num_tasks Number of tasks to execute 4079 // grainsize Number of loop iterations per task 4080 // extras Number of chunks with grainsize+1 iterations 4081 // tc Iterations count 4082 // task_dup Tasks duplication routine 4083 // codeptr_ra Return address for OMPT events 4084 void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task, 4085 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, 4086 kmp_uint64 ub_glob, kmp_uint64 num_tasks, 4087 kmp_uint64 grainsize, kmp_uint64 extras, 4088 kmp_uint64 tc, 4089 #if OMPT_SUPPORT 4090 void *codeptr_ra, 4091 #endif 4092 void *task_dup) { 4093 KMP_COUNT_BLOCK(OMP_TASKLOOP); 4094 KMP_TIME_PARTITIONED_BLOCK(OMP_taskloop_scheduling); 4095 p_task_dup_t ptask_dup = (p_task_dup_t)task_dup; 4096 // compiler provides global bounds here 4097 kmp_taskloop_bounds_t task_bounds(task, lb, ub); 4098 kmp_uint64 lower = task_bounds.get_lb(); 4099 kmp_uint64 upper = task_bounds.get_ub(); 4100 kmp_uint64 i; 4101 kmp_info_t *thread = __kmp_threads[gtid]; 4102 kmp_taskdata_t *current_task = thread->th.th_current_task; 4103 kmp_task_t *next_task; 4104 kmp_int32 lastpriv = 0; 4105 4106 KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + extras); 4107 KMP_DEBUG_ASSERT(num_tasks > extras); 4108 KMP_DEBUG_ASSERT(num_tasks > 0); 4109 KA_TRACE(20, ("__kmp_taskloop_linear: T#%d: %lld tasks, grainsize %lld, " 4110 "extras %lld, i=%lld,%lld(%d)%lld, dup %p\n", 4111 gtid, num_tasks, grainsize, extras, lower, upper, ub_glob, st, 4112 task_dup)); 4113 4114 // Launch num_tasks tasks, assign grainsize iterations each task 4115 for (i = 0; i < num_tasks; ++i) { 4116 kmp_uint64 chunk_minus_1; 4117 if (extras == 0) { 4118 chunk_minus_1 = grainsize - 1; 4119 } else { 4120 chunk_minus_1 = grainsize; 4121 --extras; // first extras iterations get bigger chunk (grainsize+1) 4122 } 4123 upper = lower + st * chunk_minus_1; 4124 if (i == num_tasks - 1) { 4125 // schedule the last task, set lastprivate flag if needed 4126 if (st == 1) { // most common case 4127 KMP_DEBUG_ASSERT(upper == *ub); 4128 if (upper == ub_glob) 4129 lastpriv = 1; 4130 } else if (st > 0) { // positive loop stride 4131 KMP_DEBUG_ASSERT((kmp_uint64)st > *ub - upper); 4132 if ((kmp_uint64)st > ub_glob - upper) 4133 lastpriv = 1; 4134 } else { // negative loop stride 4135 KMP_DEBUG_ASSERT(upper + st < *ub); 4136 if (upper - ub_glob < (kmp_uint64)(-st)) 4137 lastpriv = 1; 4138 } 4139 } 4140 next_task = __kmp_task_dup_alloc(thread, task); // allocate new task 4141 kmp_taskdata_t *next_taskdata = KMP_TASK_TO_TASKDATA(next_task); 4142 kmp_taskloop_bounds_t next_task_bounds = 4143 kmp_taskloop_bounds_t(next_task, task_bounds); 4144 4145 // adjust task-specific bounds 4146 next_task_bounds.set_lb(lower); 4147 if (next_taskdata->td_flags.native) { 4148 next_task_bounds.set_ub(upper + (st > 0 ? 1 : -1)); 4149 } else { 4150 next_task_bounds.set_ub(upper); 4151 } 4152 if (ptask_dup != NULL) // set lastprivate flag, construct firstprivates, 4153 // etc. 4154 ptask_dup(next_task, task, lastpriv); 4155 KA_TRACE(40, 4156 ("__kmp_taskloop_linear: T#%d; task #%llu: task %p: lower %lld, " 4157 "upper %lld stride %lld, (offsets %p %p)\n", 4158 gtid, i, next_task, lower, upper, st, 4159 next_task_bounds.get_lower_offset(), 4160 next_task_bounds.get_upper_offset())); 4161 #if OMPT_SUPPORT 4162 __kmp_omp_taskloop_task(NULL, gtid, next_task, 4163 codeptr_ra); // schedule new task 4164 #else 4165 __kmp_omp_task(gtid, next_task, true); // schedule new task 4166 #endif 4167 lower = upper + st; // adjust lower bound for the next iteration 4168 } 4169 // free the pattern task and exit 4170 __kmp_task_start(gtid, task, current_task); // make internal bookkeeping 4171 // do not execute the pattern task, just do internal bookkeeping 4172 __kmp_task_finish<false>(gtid, task, current_task); 4173 } 4174 4175 // Structure to keep taskloop parameters for auxiliary task 4176 // kept in the shareds of the task structure. 4177 typedef struct __taskloop_params { 4178 kmp_task_t *task; 4179 kmp_uint64 *lb; 4180 kmp_uint64 *ub; 4181 void *task_dup; 4182 kmp_int64 st; 4183 kmp_uint64 ub_glob; 4184 kmp_uint64 num_tasks; 4185 kmp_uint64 grainsize; 4186 kmp_uint64 extras; 4187 kmp_uint64 tc; 4188 kmp_uint64 num_t_min; 4189 #if OMPT_SUPPORT 4190 void *codeptr_ra; 4191 #endif 4192 } __taskloop_params_t; 4193 4194 void __kmp_taskloop_recur(ident_t *, int, kmp_task_t *, kmp_uint64 *, 4195 kmp_uint64 *, kmp_int64, kmp_uint64, kmp_uint64, 4196 kmp_uint64, kmp_uint64, kmp_uint64, kmp_uint64, 4197 #if OMPT_SUPPORT 4198 void *, 4199 #endif 4200 void *); 4201 4202 // Execute part of the taskloop submitted as a task. 4203 int __kmp_taskloop_task(int gtid, void *ptask) { 4204 __taskloop_params_t *p = 4205 (__taskloop_params_t *)((kmp_task_t *)ptask)->shareds; 4206 kmp_task_t *task = p->task; 4207 kmp_uint64 *lb = p->lb; 4208 kmp_uint64 *ub = p->ub; 4209 void *task_dup = p->task_dup; 4210 // p_task_dup_t ptask_dup = (p_task_dup_t)task_dup; 4211 kmp_int64 st = p->st; 4212 kmp_uint64 ub_glob = p->ub_glob; 4213 kmp_uint64 num_tasks = p->num_tasks; 4214 kmp_uint64 grainsize = p->grainsize; 4215 kmp_uint64 extras = p->extras; 4216 kmp_uint64 tc = p->tc; 4217 kmp_uint64 num_t_min = p->num_t_min; 4218 #if OMPT_SUPPORT 4219 void *codeptr_ra = p->codeptr_ra; 4220 #endif 4221 #if KMP_DEBUG 4222 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 4223 KMP_DEBUG_ASSERT(task != NULL); 4224 KA_TRACE(20, ("__kmp_taskloop_task: T#%d, task %p: %lld tasks, grainsize" 4225 " %lld, extras %lld, i=%lld,%lld(%d), dup %p\n", 4226 gtid, taskdata, num_tasks, grainsize, extras, *lb, *ub, st, 4227 task_dup)); 4228 #endif 4229 KMP_DEBUG_ASSERT(num_tasks * 2 + 1 > num_t_min); 4230 if (num_tasks > num_t_min) 4231 __kmp_taskloop_recur(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks, 4232 grainsize, extras, tc, num_t_min, 4233 #if OMPT_SUPPORT 4234 codeptr_ra, 4235 #endif 4236 task_dup); 4237 else 4238 __kmp_taskloop_linear(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks, 4239 grainsize, extras, tc, 4240 #if OMPT_SUPPORT 4241 codeptr_ra, 4242 #endif 4243 task_dup); 4244 4245 KA_TRACE(40, ("__kmp_taskloop_task(exit): T#%d\n", gtid)); 4246 return 0; 4247 } 4248 4249 // Schedule part of the taskloop as a task, 4250 // execute the rest of the taskloop. 4251 // 4252 // loc Source location information 4253 // gtid Global thread ID 4254 // task Pattern task, exposes the loop iteration range 4255 // lb Pointer to loop lower bound in task structure 4256 // ub Pointer to loop upper bound in task structure 4257 // st Loop stride 4258 // ub_glob Global upper bound (used for lastprivate check) 4259 // num_tasks Number of tasks to execute 4260 // grainsize Number of loop iterations per task 4261 // extras Number of chunks with grainsize+1 iterations 4262 // tc Iterations count 4263 // num_t_min Threshold to launch tasks recursively 4264 // task_dup Tasks duplication routine 4265 // codeptr_ra Return address for OMPT events 4266 void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task, 4267 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, 4268 kmp_uint64 ub_glob, kmp_uint64 num_tasks, 4269 kmp_uint64 grainsize, kmp_uint64 extras, 4270 kmp_uint64 tc, kmp_uint64 num_t_min, 4271 #if OMPT_SUPPORT 4272 void *codeptr_ra, 4273 #endif 4274 void *task_dup) { 4275 #if KMP_DEBUG 4276 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 4277 KMP_DEBUG_ASSERT(task != NULL); 4278 KMP_DEBUG_ASSERT(num_tasks > num_t_min); 4279 KA_TRACE(20, ("__kmp_taskloop_recur: T#%d, task %p: %lld tasks, grainsize" 4280 " %lld, extras %lld, i=%lld,%lld(%d), dup %p\n", 4281 gtid, taskdata, num_tasks, grainsize, extras, *lb, *ub, st, 4282 task_dup)); 4283 #endif 4284 p_task_dup_t ptask_dup = (p_task_dup_t)task_dup; 4285 kmp_uint64 lower = *lb; 4286 kmp_info_t *thread = __kmp_threads[gtid]; 4287 // kmp_taskdata_t *current_task = thread->th.th_current_task; 4288 kmp_task_t *next_task; 4289 size_t lower_offset = 4290 (char *)lb - (char *)task; // remember offset of lb in the task structure 4291 size_t upper_offset = 4292 (char *)ub - (char *)task; // remember offset of ub in the task structure 4293 4294 KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + extras); 4295 KMP_DEBUG_ASSERT(num_tasks > extras); 4296 KMP_DEBUG_ASSERT(num_tasks > 0); 4297 4298 // split the loop in two halves 4299 kmp_uint64 lb1, ub0, tc0, tc1, ext0, ext1; 4300 kmp_uint64 gr_size0 = grainsize; 4301 kmp_uint64 n_tsk0 = num_tasks >> 1; // num_tasks/2 to execute 4302 kmp_uint64 n_tsk1 = num_tasks - n_tsk0; // to schedule as a task 4303 if (n_tsk0 <= extras) { 4304 gr_size0++; // integrate extras into grainsize 4305 ext0 = 0; // no extra iters in 1st half 4306 ext1 = extras - n_tsk0; // remaining extras 4307 tc0 = gr_size0 * n_tsk0; 4308 tc1 = tc - tc0; 4309 } else { // n_tsk0 > extras 4310 ext1 = 0; // no extra iters in 2nd half 4311 ext0 = extras; 4312 tc1 = grainsize * n_tsk1; 4313 tc0 = tc - tc1; 4314 } 4315 ub0 = lower + st * (tc0 - 1); 4316 lb1 = ub0 + st; 4317 4318 // create pattern task for 2nd half of the loop 4319 next_task = __kmp_task_dup_alloc(thread, task); // duplicate the task 4320 // adjust lower bound (upper bound is not changed) for the 2nd half 4321 *(kmp_uint64 *)((char *)next_task + lower_offset) = lb1; 4322 if (ptask_dup != NULL) // construct firstprivates, etc. 4323 ptask_dup(next_task, task, 0); 4324 *ub = ub0; // adjust upper bound for the 1st half 4325 4326 // create auxiliary task for 2nd half of the loop 4327 kmp_task_t *new_task = 4328 __kmpc_omp_task_alloc(loc, gtid, 1, 3 * sizeof(void *), 4329 sizeof(__taskloop_params_t), &__kmp_taskloop_task); 4330 __taskloop_params_t *p = (__taskloop_params_t *)new_task->shareds; 4331 p->task = next_task; 4332 p->lb = (kmp_uint64 *)((char *)next_task + lower_offset); 4333 p->ub = (kmp_uint64 *)((char *)next_task + upper_offset); 4334 p->task_dup = task_dup; 4335 p->st = st; 4336 p->ub_glob = ub_glob; 4337 p->num_tasks = n_tsk1; 4338 p->grainsize = grainsize; 4339 p->extras = ext1; 4340 p->tc = tc1; 4341 p->num_t_min = num_t_min; 4342 #if OMPT_SUPPORT 4343 p->codeptr_ra = codeptr_ra; 4344 #endif 4345 4346 #if OMPT_SUPPORT 4347 // schedule new task with correct return address for OMPT events 4348 __kmp_omp_taskloop_task(NULL, gtid, new_task, codeptr_ra); 4349 #else 4350 __kmp_omp_task(gtid, new_task, true); // schedule new task 4351 #endif 4352 4353 // execute the 1st half of current subrange 4354 if (n_tsk0 > num_t_min) 4355 __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0, gr_size0, 4356 ext0, tc0, num_t_min, 4357 #if OMPT_SUPPORT 4358 codeptr_ra, 4359 #endif 4360 task_dup); 4361 else 4362 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0, 4363 gr_size0, ext0, tc0, 4364 #if OMPT_SUPPORT 4365 codeptr_ra, 4366 #endif 4367 task_dup); 4368 4369 KA_TRACE(40, ("__kmpc_taskloop_recur(exit): T#%d\n", gtid)); 4370 } 4371 4372 /*! 4373 @ingroup TASKING 4374 @param loc Source location information 4375 @param gtid Global thread ID 4376 @param task Task structure 4377 @param if_val Value of the if clause 4378 @param lb Pointer to loop lower bound in task structure 4379 @param ub Pointer to loop upper bound in task structure 4380 @param st Loop stride 4381 @param nogroup Flag, 1 if no taskgroup needs to be added, 0 otherwise 4382 @param sched Schedule specified 0/1/2 for none/grainsize/num_tasks 4383 @param grainsize Schedule value if specified 4384 @param task_dup Tasks duplication routine 4385 4386 Execute the taskloop construct. 4387 */ 4388 void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val, 4389 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, 4390 int sched, kmp_uint64 grainsize, void *task_dup) { 4391 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 4392 KMP_DEBUG_ASSERT(task != NULL); 4393 4394 if (nogroup == 0) { 4395 #if OMPT_SUPPORT && OMPT_OPTIONAL 4396 OMPT_STORE_RETURN_ADDRESS(gtid); 4397 #endif 4398 __kmpc_taskgroup(loc, gtid); 4399 } 4400 4401 // ========================================================================= 4402 // calculate loop parameters 4403 kmp_taskloop_bounds_t task_bounds(task, lb, ub); 4404 kmp_uint64 tc; 4405 // compiler provides global bounds here 4406 kmp_uint64 lower = task_bounds.get_lb(); 4407 kmp_uint64 upper = task_bounds.get_ub(); 4408 kmp_uint64 ub_glob = upper; // global upper used to calc lastprivate flag 4409 kmp_uint64 num_tasks = 0, extras = 0; 4410 kmp_uint64 num_tasks_min = __kmp_taskloop_min_tasks; 4411 kmp_info_t *thread = __kmp_threads[gtid]; 4412 kmp_taskdata_t *current_task = thread->th.th_current_task; 4413 4414 KA_TRACE(20, ("__kmpc_taskloop: T#%d, task %p, lb %lld, ub %lld, st %lld, " 4415 "grain %llu(%d), dup %p\n", 4416 gtid, taskdata, lower, upper, st, grainsize, sched, task_dup)); 4417 4418 // compute trip count 4419 if (st == 1) { // most common case 4420 tc = upper - lower + 1; 4421 } else if (st < 0) { 4422 tc = (lower - upper) / (-st) + 1; 4423 } else { // st > 0 4424 tc = (upper - lower) / st + 1; 4425 } 4426 if (tc == 0) { 4427 KA_TRACE(20, ("__kmpc_taskloop(exit): T#%d zero-trip loop\n", gtid)); 4428 // free the pattern task and exit 4429 __kmp_task_start(gtid, task, current_task); 4430 // do not execute anything for zero-trip loop 4431 __kmp_task_finish<false>(gtid, task, current_task); 4432 return; 4433 } 4434 4435 #if OMPT_SUPPORT && OMPT_OPTIONAL 4436 ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL); 4437 ompt_task_info_t *task_info = __ompt_get_task_info_object(0); 4438 if (ompt_enabled.ompt_callback_work) { 4439 ompt_callbacks.ompt_callback(ompt_callback_work)( 4440 ompt_work_taskloop, ompt_scope_begin, &(team_info->parallel_data), 4441 &(task_info->task_data), tc, OMPT_GET_RETURN_ADDRESS(0)); 4442 } 4443 #endif 4444 4445 if (num_tasks_min == 0) 4446 // TODO: can we choose better default heuristic? 4447 num_tasks_min = 4448 KMP_MIN(thread->th.th_team_nproc * 10, INITIAL_TASK_DEQUE_SIZE); 4449 4450 // compute num_tasks/grainsize based on the input provided 4451 switch (sched) { 4452 case 0: // no schedule clause specified, we can choose the default 4453 // let's try to schedule (team_size*10) tasks 4454 grainsize = thread->th.th_team_nproc * 10; 4455 KMP_FALLTHROUGH(); 4456 case 2: // num_tasks provided 4457 if (grainsize > tc) { 4458 num_tasks = tc; // too big num_tasks requested, adjust values 4459 grainsize = 1; 4460 extras = 0; 4461 } else { 4462 num_tasks = grainsize; 4463 grainsize = tc / num_tasks; 4464 extras = tc % num_tasks; 4465 } 4466 break; 4467 case 1: // grainsize provided 4468 if (grainsize > tc) { 4469 num_tasks = 1; // too big grainsize requested, adjust values 4470 grainsize = tc; 4471 extras = 0; 4472 } else { 4473 num_tasks = tc / grainsize; 4474 // adjust grainsize for balanced distribution of iterations 4475 grainsize = tc / num_tasks; 4476 extras = tc % num_tasks; 4477 } 4478 break; 4479 default: 4480 KMP_ASSERT2(0, "unknown scheduling of taskloop"); 4481 } 4482 KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + extras); 4483 KMP_DEBUG_ASSERT(num_tasks > extras); 4484 KMP_DEBUG_ASSERT(num_tasks > 0); 4485 // ========================================================================= 4486 4487 // check if clause value first 4488 // Also require GOMP_taskloop to reduce to linear (taskdata->td_flags.native) 4489 if (if_val == 0) { // if(0) specified, mark task as serial 4490 taskdata->td_flags.task_serial = 1; 4491 taskdata->td_flags.tiedness = TASK_TIED; // AC: serial task cannot be untied 4492 // always start serial tasks linearly 4493 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks, 4494 grainsize, extras, tc, 4495 #if OMPT_SUPPORT 4496 OMPT_GET_RETURN_ADDRESS(0), 4497 #endif 4498 task_dup); 4499 // !taskdata->td_flags.native => currently force linear spawning of tasks 4500 // for GOMP_taskloop 4501 } else if (num_tasks > num_tasks_min && !taskdata->td_flags.native) { 4502 KA_TRACE(20, ("__kmpc_taskloop: T#%d, go recursive: tc %llu, #tasks %llu" 4503 "(%lld), grain %llu, extras %llu\n", 4504 gtid, tc, num_tasks, num_tasks_min, grainsize, extras)); 4505 __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, num_tasks, 4506 grainsize, extras, tc, num_tasks_min, 4507 #if OMPT_SUPPORT 4508 OMPT_GET_RETURN_ADDRESS(0), 4509 #endif 4510 task_dup); 4511 } else { 4512 KA_TRACE(20, ("__kmpc_taskloop: T#%d, go linear: tc %llu, #tasks %llu" 4513 "(%lld), grain %llu, extras %llu\n", 4514 gtid, tc, num_tasks, num_tasks_min, grainsize, extras)); 4515 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks, 4516 grainsize, extras, tc, 4517 #if OMPT_SUPPORT 4518 OMPT_GET_RETURN_ADDRESS(0), 4519 #endif 4520 task_dup); 4521 } 4522 4523 #if OMPT_SUPPORT && OMPT_OPTIONAL 4524 if (ompt_enabled.ompt_callback_work) { 4525 ompt_callbacks.ompt_callback(ompt_callback_work)( 4526 ompt_work_taskloop, ompt_scope_end, &(team_info->parallel_data), 4527 &(task_info->task_data), tc, OMPT_GET_RETURN_ADDRESS(0)); 4528 } 4529 #endif 4530 4531 if (nogroup == 0) { 4532 #if OMPT_SUPPORT && OMPT_OPTIONAL 4533 OMPT_STORE_RETURN_ADDRESS(gtid); 4534 #endif 4535 __kmpc_end_taskgroup(loc, gtid); 4536 } 4537 KA_TRACE(20, ("__kmpc_taskloop(exit): T#%d\n", gtid)); 4538 } 4539