1 /* 2 * kmp_tasking.cpp -- OpenMP 3.0 tasking support. 3 */ 4 5 //===----------------------------------------------------------------------===// 6 // 7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 8 // See https://llvm.org/LICENSE.txt for license information. 9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "kmp.h" 14 #include "kmp_i18n.h" 15 #include "kmp_itt.h" 16 #include "kmp_stats.h" 17 #include "kmp_wait_release.h" 18 #include "kmp_taskdeps.h" 19 20 #if OMPT_SUPPORT 21 #include "ompt-specific.h" 22 #endif 23 24 #include "tsan_annotations.h" 25 26 /* forward declaration */ 27 static void __kmp_enable_tasking(kmp_task_team_t *task_team, 28 kmp_info_t *this_thr); 29 static void __kmp_alloc_task_deque(kmp_info_t *thread, 30 kmp_thread_data_t *thread_data); 31 static int __kmp_realloc_task_threads_data(kmp_info_t *thread, 32 kmp_task_team_t *task_team); 33 34 #if OMP_45_ENABLED 35 static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask); 36 #endif 37 38 #ifdef BUILD_TIED_TASK_STACK 39 40 // __kmp_trace_task_stack: print the tied tasks from the task stack in order 41 // from top do bottom 42 // 43 // gtid: global thread identifier for thread containing stack 44 // thread_data: thread data for task team thread containing stack 45 // threshold: value above which the trace statement triggers 46 // location: string identifying call site of this function (for trace) 47 static void __kmp_trace_task_stack(kmp_int32 gtid, 48 kmp_thread_data_t *thread_data, 49 int threshold, char *location) { 50 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; 51 kmp_taskdata_t **stack_top = task_stack->ts_top; 52 kmp_int32 entries = task_stack->ts_entries; 53 kmp_taskdata_t *tied_task; 54 55 KA_TRACE( 56 threshold, 57 ("__kmp_trace_task_stack(start): location = %s, gtid = %d, entries = %d, " 58 "first_block = %p, stack_top = %p \n", 59 location, gtid, entries, task_stack->ts_first_block, stack_top)); 60 61 KMP_DEBUG_ASSERT(stack_top != NULL); 62 KMP_DEBUG_ASSERT(entries > 0); 63 64 while (entries != 0) { 65 KMP_DEBUG_ASSERT(stack_top != &task_stack->ts_first_block.sb_block[0]); 66 // fix up ts_top if we need to pop from previous block 67 if (entries & TASK_STACK_INDEX_MASK == 0) { 68 kmp_stack_block_t *stack_block = (kmp_stack_block_t *)(stack_top); 69 70 stack_block = stack_block->sb_prev; 71 stack_top = &stack_block->sb_block[TASK_STACK_BLOCK_SIZE]; 72 } 73 74 // finish bookkeeping 75 stack_top--; 76 entries--; 77 78 tied_task = *stack_top; 79 80 KMP_DEBUG_ASSERT(tied_task != NULL); 81 KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED); 82 83 KA_TRACE(threshold, 84 ("__kmp_trace_task_stack(%s): gtid=%d, entry=%d, " 85 "stack_top=%p, tied_task=%p\n", 86 location, gtid, entries, stack_top, tied_task)); 87 } 88 KMP_DEBUG_ASSERT(stack_top == &task_stack->ts_first_block.sb_block[0]); 89 90 KA_TRACE(threshold, 91 ("__kmp_trace_task_stack(exit): location = %s, gtid = %d\n", 92 location, gtid)); 93 } 94 95 // __kmp_init_task_stack: initialize the task stack for the first time 96 // after a thread_data structure is created. 97 // It should not be necessary to do this again (assuming the stack works). 98 // 99 // gtid: global thread identifier of calling thread 100 // thread_data: thread data for task team thread containing stack 101 static void __kmp_init_task_stack(kmp_int32 gtid, 102 kmp_thread_data_t *thread_data) { 103 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; 104 kmp_stack_block_t *first_block; 105 106 // set up the first block of the stack 107 first_block = &task_stack->ts_first_block; 108 task_stack->ts_top = (kmp_taskdata_t **)first_block; 109 memset((void *)first_block, '\0', 110 TASK_STACK_BLOCK_SIZE * sizeof(kmp_taskdata_t *)); 111 112 // initialize the stack to be empty 113 task_stack->ts_entries = TASK_STACK_EMPTY; 114 first_block->sb_next = NULL; 115 first_block->sb_prev = NULL; 116 } 117 118 // __kmp_free_task_stack: free the task stack when thread_data is destroyed. 119 // 120 // gtid: global thread identifier for calling thread 121 // thread_data: thread info for thread containing stack 122 static void __kmp_free_task_stack(kmp_int32 gtid, 123 kmp_thread_data_t *thread_data) { 124 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; 125 kmp_stack_block_t *stack_block = &task_stack->ts_first_block; 126 127 KMP_DEBUG_ASSERT(task_stack->ts_entries == TASK_STACK_EMPTY); 128 // free from the second block of the stack 129 while (stack_block != NULL) { 130 kmp_stack_block_t *next_block = (stack_block) ? stack_block->sb_next : NULL; 131 132 stack_block->sb_next = NULL; 133 stack_block->sb_prev = NULL; 134 if (stack_block != &task_stack->ts_first_block) { 135 __kmp_thread_free(thread, 136 stack_block); // free the block, if not the first 137 } 138 stack_block = next_block; 139 } 140 // initialize the stack to be empty 141 task_stack->ts_entries = 0; 142 task_stack->ts_top = NULL; 143 } 144 145 // __kmp_push_task_stack: Push the tied task onto the task stack. 146 // Grow the stack if necessary by allocating another block. 147 // 148 // gtid: global thread identifier for calling thread 149 // thread: thread info for thread containing stack 150 // tied_task: the task to push on the stack 151 static void __kmp_push_task_stack(kmp_int32 gtid, kmp_info_t *thread, 152 kmp_taskdata_t *tied_task) { 153 // GEH - need to consider what to do if tt_threads_data not allocated yet 154 kmp_thread_data_t *thread_data = 155 &thread->th.th_task_team->tt.tt_threads_data[__kmp_tid_from_gtid(gtid)]; 156 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; 157 158 if (tied_task->td_flags.team_serial || tied_task->td_flags.tasking_ser) { 159 return; // Don't push anything on stack if team or team tasks are serialized 160 } 161 162 KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED); 163 KMP_DEBUG_ASSERT(task_stack->ts_top != NULL); 164 165 KA_TRACE(20, 166 ("__kmp_push_task_stack(enter): GTID: %d; THREAD: %p; TASK: %p\n", 167 gtid, thread, tied_task)); 168 // Store entry 169 *(task_stack->ts_top) = tied_task; 170 171 // Do bookkeeping for next push 172 task_stack->ts_top++; 173 task_stack->ts_entries++; 174 175 if (task_stack->ts_entries & TASK_STACK_INDEX_MASK == 0) { 176 // Find beginning of this task block 177 kmp_stack_block_t *stack_block = 178 (kmp_stack_block_t *)(task_stack->ts_top - TASK_STACK_BLOCK_SIZE); 179 180 // Check if we already have a block 181 if (stack_block->sb_next != 182 NULL) { // reset ts_top to beginning of next block 183 task_stack->ts_top = &stack_block->sb_next->sb_block[0]; 184 } else { // Alloc new block and link it up 185 kmp_stack_block_t *new_block = (kmp_stack_block_t *)__kmp_thread_calloc( 186 thread, sizeof(kmp_stack_block_t)); 187 188 task_stack->ts_top = &new_block->sb_block[0]; 189 stack_block->sb_next = new_block; 190 new_block->sb_prev = stack_block; 191 new_block->sb_next = NULL; 192 193 KA_TRACE( 194 30, 195 ("__kmp_push_task_stack(): GTID: %d; TASK: %p; Alloc new block: %p\n", 196 gtid, tied_task, new_block)); 197 } 198 } 199 KA_TRACE(20, ("__kmp_push_task_stack(exit): GTID: %d; TASK: %p\n", gtid, 200 tied_task)); 201 } 202 203 // __kmp_pop_task_stack: Pop the tied task from the task stack. Don't return 204 // the task, just check to make sure it matches the ending task passed in. 205 // 206 // gtid: global thread identifier for the calling thread 207 // thread: thread info structure containing stack 208 // tied_task: the task popped off the stack 209 // ending_task: the task that is ending (should match popped task) 210 static void __kmp_pop_task_stack(kmp_int32 gtid, kmp_info_t *thread, 211 kmp_taskdata_t *ending_task) { 212 // GEH - need to consider what to do if tt_threads_data not allocated yet 213 kmp_thread_data_t *thread_data = 214 &thread->th.th_task_team->tt_threads_data[__kmp_tid_from_gtid(gtid)]; 215 kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks; 216 kmp_taskdata_t *tied_task; 217 218 if (ending_task->td_flags.team_serial || ending_task->td_flags.tasking_ser) { 219 // Don't pop anything from stack if team or team tasks are serialized 220 return; 221 } 222 223 KMP_DEBUG_ASSERT(task_stack->ts_top != NULL); 224 KMP_DEBUG_ASSERT(task_stack->ts_entries > 0); 225 226 KA_TRACE(20, ("__kmp_pop_task_stack(enter): GTID: %d; THREAD: %p\n", gtid, 227 thread)); 228 229 // fix up ts_top if we need to pop from previous block 230 if (task_stack->ts_entries & TASK_STACK_INDEX_MASK == 0) { 231 kmp_stack_block_t *stack_block = (kmp_stack_block_t *)(task_stack->ts_top); 232 233 stack_block = stack_block->sb_prev; 234 task_stack->ts_top = &stack_block->sb_block[TASK_STACK_BLOCK_SIZE]; 235 } 236 237 // finish bookkeeping 238 task_stack->ts_top--; 239 task_stack->ts_entries--; 240 241 tied_task = *(task_stack->ts_top); 242 243 KMP_DEBUG_ASSERT(tied_task != NULL); 244 KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED); 245 KMP_DEBUG_ASSERT(tied_task == ending_task); // If we built the stack correctly 246 247 KA_TRACE(20, ("__kmp_pop_task_stack(exit): GTID: %d; TASK: %p\n", gtid, 248 tied_task)); 249 return; 250 } 251 #endif /* BUILD_TIED_TASK_STACK */ 252 253 // returns 1 if new task is allowed to execute, 0 otherwise 254 // checks Task Scheduling constraint (if requested) and 255 // mutexinoutset dependencies if any 256 static bool __kmp_task_is_allowed(int gtid, const kmp_int32 is_constrained, 257 const kmp_taskdata_t *tasknew, 258 const kmp_taskdata_t *taskcurr) { 259 if (is_constrained && (tasknew->td_flags.tiedness == TASK_TIED)) { 260 // Check if the candidate obeys the Task Scheduling Constraints (TSC) 261 // only descendant of all deferred tied tasks can be scheduled, checking 262 // the last one is enough, as it in turn is the descendant of all others 263 kmp_taskdata_t *current = taskcurr->td_last_tied; 264 KMP_DEBUG_ASSERT(current != NULL); 265 // check if the task is not suspended on barrier 266 if (current->td_flags.tasktype == TASK_EXPLICIT || 267 current->td_taskwait_thread > 0) { // <= 0 on barrier 268 kmp_int32 level = current->td_level; 269 kmp_taskdata_t *parent = tasknew->td_parent; 270 while (parent != current && parent->td_level > level) { 271 // check generation up to the level of the current task 272 parent = parent->td_parent; 273 KMP_DEBUG_ASSERT(parent != NULL); 274 } 275 if (parent != current) 276 return false; 277 } 278 } 279 // Check mutexinoutset dependencies, acquire locks 280 kmp_depnode_t *node = tasknew->td_depnode; 281 if (node && (node->dn.mtx_num_locks > 0)) { 282 for (int i = 0; i < node->dn.mtx_num_locks; ++i) { 283 KMP_DEBUG_ASSERT(node->dn.mtx_locks[i] != NULL); 284 if (__kmp_test_lock(node->dn.mtx_locks[i], gtid)) 285 continue; 286 // could not get the lock, release previous locks 287 for (int j = i - 1; j >= 0; --j) 288 __kmp_release_lock(node->dn.mtx_locks[j], gtid); 289 return false; 290 } 291 // negative num_locks means all locks acquired successfully 292 node->dn.mtx_num_locks = -node->dn.mtx_num_locks; 293 } 294 return true; 295 } 296 297 // __kmp_realloc_task_deque: 298 // Re-allocates a task deque for a particular thread, copies the content from 299 // the old deque and adjusts the necessary data structures relating to the 300 // deque. This operation must be done with the deque_lock being held 301 static void __kmp_realloc_task_deque(kmp_info_t *thread, 302 kmp_thread_data_t *thread_data) { 303 kmp_int32 size = TASK_DEQUE_SIZE(thread_data->td); 304 kmp_int32 new_size = 2 * size; 305 306 KE_TRACE(10, ("__kmp_realloc_task_deque: T#%d reallocating deque[from %d to " 307 "%d] for thread_data %p\n", 308 __kmp_gtid_from_thread(thread), size, new_size, thread_data)); 309 310 kmp_taskdata_t **new_deque = 311 (kmp_taskdata_t **)__kmp_allocate(new_size * sizeof(kmp_taskdata_t *)); 312 313 int i, j; 314 for (i = thread_data->td.td_deque_head, j = 0; j < size; 315 i = (i + 1) & TASK_DEQUE_MASK(thread_data->td), j++) 316 new_deque[j] = thread_data->td.td_deque[i]; 317 318 __kmp_free(thread_data->td.td_deque); 319 320 thread_data->td.td_deque_head = 0; 321 thread_data->td.td_deque_tail = size; 322 thread_data->td.td_deque = new_deque; 323 thread_data->td.td_deque_size = new_size; 324 } 325 326 // __kmp_push_task: Add a task to the thread's deque 327 static kmp_int32 __kmp_push_task(kmp_int32 gtid, kmp_task_t *task) { 328 kmp_info_t *thread = __kmp_threads[gtid]; 329 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 330 kmp_task_team_t *task_team = thread->th.th_task_team; 331 kmp_int32 tid = __kmp_tid_from_gtid(gtid); 332 kmp_thread_data_t *thread_data; 333 334 KA_TRACE(20, 335 ("__kmp_push_task: T#%d trying to push task %p.\n", gtid, taskdata)); 336 337 if (taskdata->td_flags.tiedness == TASK_UNTIED) { 338 // untied task needs to increment counter so that the task structure is not 339 // freed prematurely 340 kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count); 341 KMP_DEBUG_USE_VAR(counter); 342 KA_TRACE( 343 20, 344 ("__kmp_push_task: T#%d untied_count (%d) incremented for task %p\n", 345 gtid, counter, taskdata)); 346 } 347 348 // The first check avoids building task_team thread data if serialized 349 if (taskdata->td_flags.task_serial) { 350 KA_TRACE(20, ("__kmp_push_task: T#%d team serialized; returning " 351 "TASK_NOT_PUSHED for task %p\n", 352 gtid, taskdata)); 353 return TASK_NOT_PUSHED; 354 } 355 356 // Now that serialized tasks have returned, we can assume that we are not in 357 // immediate exec mode 358 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 359 if (!KMP_TASKING_ENABLED(task_team)) { 360 __kmp_enable_tasking(task_team, thread); 361 } 362 KMP_DEBUG_ASSERT(TCR_4(task_team->tt.tt_found_tasks) == TRUE); 363 KMP_DEBUG_ASSERT(TCR_PTR(task_team->tt.tt_threads_data) != NULL); 364 365 // Find tasking deque specific to encountering thread 366 thread_data = &task_team->tt.tt_threads_data[tid]; 367 368 // No lock needed since only owner can allocate 369 if (thread_data->td.td_deque == NULL) { 370 __kmp_alloc_task_deque(thread, thread_data); 371 } 372 373 int locked = 0; 374 // Check if deque is full 375 if (TCR_4(thread_data->td.td_deque_ntasks) >= 376 TASK_DEQUE_SIZE(thread_data->td)) { 377 if (__kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata, 378 thread->th.th_current_task)) { 379 KA_TRACE(20, ("__kmp_push_task: T#%d deque is full; returning " 380 "TASK_NOT_PUSHED for task %p\n", 381 gtid, taskdata)); 382 return TASK_NOT_PUSHED; 383 } else { 384 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 385 locked = 1; 386 // expand deque to push the task which is not allowed to execute 387 __kmp_realloc_task_deque(thread, thread_data); 388 } 389 } 390 // Lock the deque for the task push operation 391 if (!locked) { 392 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 393 #if OMP_45_ENABLED 394 // Need to recheck as we can get a proxy task from thread outside of OpenMP 395 if (TCR_4(thread_data->td.td_deque_ntasks) >= 396 TASK_DEQUE_SIZE(thread_data->td)) { 397 if (__kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata, 398 thread->th.th_current_task)) { 399 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 400 KA_TRACE(20, ("__kmp_push_task: T#%d deque is full on 2nd check; " 401 "returning TASK_NOT_PUSHED for task %p\n", 402 gtid, taskdata)); 403 return TASK_NOT_PUSHED; 404 } else { 405 // expand deque to push the task which is not allowed to execute 406 __kmp_realloc_task_deque(thread, thread_data); 407 } 408 } 409 #endif 410 } 411 // Must have room since no thread can add tasks but calling thread 412 KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) < 413 TASK_DEQUE_SIZE(thread_data->td)); 414 415 thread_data->td.td_deque[thread_data->td.td_deque_tail] = 416 taskdata; // Push taskdata 417 // Wrap index. 418 thread_data->td.td_deque_tail = 419 (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td); 420 TCW_4(thread_data->td.td_deque_ntasks, 421 TCR_4(thread_data->td.td_deque_ntasks) + 1); // Adjust task count 422 423 KA_TRACE(20, ("__kmp_push_task: T#%d returning TASK_SUCCESSFULLY_PUSHED: " 424 "task=%p ntasks=%d head=%u tail=%u\n", 425 gtid, taskdata, thread_data->td.td_deque_ntasks, 426 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 427 428 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 429 430 return TASK_SUCCESSFULLY_PUSHED; 431 } 432 433 // __kmp_pop_current_task_from_thread: set up current task from called thread 434 // when team ends 435 // 436 // this_thr: thread structure to set current_task in. 437 void __kmp_pop_current_task_from_thread(kmp_info_t *this_thr) { 438 KF_TRACE(10, ("__kmp_pop_current_task_from_thread(enter): T#%d " 439 "this_thread=%p, curtask=%p, " 440 "curtask_parent=%p\n", 441 0, this_thr, this_thr->th.th_current_task, 442 this_thr->th.th_current_task->td_parent)); 443 444 this_thr->th.th_current_task = this_thr->th.th_current_task->td_parent; 445 446 KF_TRACE(10, ("__kmp_pop_current_task_from_thread(exit): T#%d " 447 "this_thread=%p, curtask=%p, " 448 "curtask_parent=%p\n", 449 0, this_thr, this_thr->th.th_current_task, 450 this_thr->th.th_current_task->td_parent)); 451 } 452 453 // __kmp_push_current_task_to_thread: set up current task in called thread for a 454 // new team 455 // 456 // this_thr: thread structure to set up 457 // team: team for implicit task data 458 // tid: thread within team to set up 459 void __kmp_push_current_task_to_thread(kmp_info_t *this_thr, kmp_team_t *team, 460 int tid) { 461 // current task of the thread is a parent of the new just created implicit 462 // tasks of new team 463 KF_TRACE(10, ("__kmp_push_current_task_to_thread(enter): T#%d this_thread=%p " 464 "curtask=%p " 465 "parent_task=%p\n", 466 tid, this_thr, this_thr->th.th_current_task, 467 team->t.t_implicit_task_taskdata[tid].td_parent)); 468 469 KMP_DEBUG_ASSERT(this_thr != NULL); 470 471 if (tid == 0) { 472 if (this_thr->th.th_current_task != &team->t.t_implicit_task_taskdata[0]) { 473 team->t.t_implicit_task_taskdata[0].td_parent = 474 this_thr->th.th_current_task; 475 this_thr->th.th_current_task = &team->t.t_implicit_task_taskdata[0]; 476 } 477 } else { 478 team->t.t_implicit_task_taskdata[tid].td_parent = 479 team->t.t_implicit_task_taskdata[0].td_parent; 480 this_thr->th.th_current_task = &team->t.t_implicit_task_taskdata[tid]; 481 } 482 483 KF_TRACE(10, ("__kmp_push_current_task_to_thread(exit): T#%d this_thread=%p " 484 "curtask=%p " 485 "parent_task=%p\n", 486 tid, this_thr, this_thr->th.th_current_task, 487 team->t.t_implicit_task_taskdata[tid].td_parent)); 488 } 489 490 // __kmp_task_start: bookkeeping for a task starting execution 491 // 492 // GTID: global thread id of calling thread 493 // task: task starting execution 494 // current_task: task suspending 495 static void __kmp_task_start(kmp_int32 gtid, kmp_task_t *task, 496 kmp_taskdata_t *current_task) { 497 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 498 kmp_info_t *thread = __kmp_threads[gtid]; 499 500 KA_TRACE(10, 501 ("__kmp_task_start(enter): T#%d starting task %p: current_task=%p\n", 502 gtid, taskdata, current_task)); 503 504 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); 505 506 // mark currently executing task as suspended 507 // TODO: GEH - make sure root team implicit task is initialized properly. 508 // KMP_DEBUG_ASSERT( current_task -> td_flags.executing == 1 ); 509 current_task->td_flags.executing = 0; 510 511 // Add task to stack if tied 512 #ifdef BUILD_TIED_TASK_STACK 513 if (taskdata->td_flags.tiedness == TASK_TIED) { 514 __kmp_push_task_stack(gtid, thread, taskdata); 515 } 516 #endif /* BUILD_TIED_TASK_STACK */ 517 518 // mark starting task as executing and as current task 519 thread->th.th_current_task = taskdata; 520 521 KMP_DEBUG_ASSERT(taskdata->td_flags.started == 0 || 522 taskdata->td_flags.tiedness == TASK_UNTIED); 523 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0 || 524 taskdata->td_flags.tiedness == TASK_UNTIED); 525 taskdata->td_flags.started = 1; 526 taskdata->td_flags.executing = 1; 527 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); 528 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); 529 530 // GEH TODO: shouldn't we pass some sort of location identifier here? 531 // APT: yes, we will pass location here. 532 // need to store current thread state (in a thread or taskdata structure) 533 // before setting work_state, otherwise wrong state is set after end of task 534 535 KA_TRACE(10, ("__kmp_task_start(exit): T#%d task=%p\n", gtid, taskdata)); 536 537 return; 538 } 539 540 #if OMPT_SUPPORT 541 //------------------------------------------------------------------------------ 542 // __ompt_task_init: 543 // Initialize OMPT fields maintained by a task. This will only be called after 544 // ompt_start_tool, so we already know whether ompt is enabled or not. 545 546 static inline void __ompt_task_init(kmp_taskdata_t *task, int tid) { 547 // The calls to __ompt_task_init already have the ompt_enabled condition. 548 task->ompt_task_info.task_data.value = 0; 549 task->ompt_task_info.frame.exit_frame = ompt_data_none; 550 task->ompt_task_info.frame.enter_frame = ompt_data_none; 551 task->ompt_task_info.frame.exit_frame_flags = ompt_frame_runtime | ompt_frame_framepointer; 552 task->ompt_task_info.frame.enter_frame_flags = ompt_frame_runtime | ompt_frame_framepointer; 553 #if OMP_40_ENABLED 554 task->ompt_task_info.ndeps = 0; 555 task->ompt_task_info.deps = NULL; 556 #endif /* OMP_40_ENABLED */ 557 } 558 559 // __ompt_task_start: 560 // Build and trigger task-begin event 561 static inline void __ompt_task_start(kmp_task_t *task, 562 kmp_taskdata_t *current_task, 563 kmp_int32 gtid) { 564 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 565 ompt_task_status_t status = ompt_task_switch; 566 if (__kmp_threads[gtid]->th.ompt_thread_info.ompt_task_yielded) { 567 status = ompt_task_yield; 568 __kmp_threads[gtid]->th.ompt_thread_info.ompt_task_yielded = 0; 569 } 570 /* let OMPT know that we're about to run this task */ 571 if (ompt_enabled.ompt_callback_task_schedule) { 572 ompt_callbacks.ompt_callback(ompt_callback_task_schedule)( 573 &(current_task->ompt_task_info.task_data), status, 574 &(taskdata->ompt_task_info.task_data)); 575 } 576 taskdata->ompt_task_info.scheduling_parent = current_task; 577 } 578 579 // __ompt_task_finish: 580 // Build and trigger final task-schedule event 581 static inline void 582 __ompt_task_finish(kmp_task_t *task, kmp_taskdata_t *resumed_task, 583 ompt_task_status_t status = ompt_task_complete) { 584 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 585 if (__kmp_omp_cancellation && taskdata->td_taskgroup && 586 taskdata->td_taskgroup->cancel_request == cancel_taskgroup) { 587 status = ompt_task_cancel; 588 } 589 590 /* let OMPT know that we're returning to the callee task */ 591 if (ompt_enabled.ompt_callback_task_schedule) { 592 ompt_callbacks.ompt_callback(ompt_callback_task_schedule)( 593 &(taskdata->ompt_task_info.task_data), status, 594 &((resumed_task ? resumed_task 595 : (taskdata->ompt_task_info.scheduling_parent 596 ? taskdata->ompt_task_info.scheduling_parent 597 : taskdata->td_parent)) 598 ->ompt_task_info.task_data)); 599 } 600 } 601 #endif 602 603 template <bool ompt> 604 static void __kmpc_omp_task_begin_if0_template(ident_t *loc_ref, kmp_int32 gtid, 605 kmp_task_t *task, 606 void *frame_address, 607 void *return_address) { 608 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 609 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task; 610 611 KA_TRACE(10, ("__kmpc_omp_task_begin_if0(enter): T#%d loc=%p task=%p " 612 "current_task=%p\n", 613 gtid, loc_ref, taskdata, current_task)); 614 615 if (taskdata->td_flags.tiedness == TASK_UNTIED) { 616 // untied task needs to increment counter so that the task structure is not 617 // freed prematurely 618 kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count); 619 KMP_DEBUG_USE_VAR(counter); 620 KA_TRACE(20, ("__kmpc_omp_task_begin_if0: T#%d untied_count (%d) " 621 "incremented for task %p\n", 622 gtid, counter, taskdata)); 623 } 624 625 taskdata->td_flags.task_serial = 626 1; // Execute this task immediately, not deferred. 627 __kmp_task_start(gtid, task, current_task); 628 629 #if OMPT_SUPPORT 630 if (ompt) { 631 if (current_task->ompt_task_info.frame.enter_frame.ptr == NULL) { 632 current_task->ompt_task_info.frame.enter_frame.ptr = 633 taskdata->ompt_task_info.frame.exit_frame.ptr = frame_address; 634 current_task->ompt_task_info.frame.enter_frame_flags = 635 taskdata->ompt_task_info.frame.exit_frame_flags = ompt_frame_application | ompt_frame_framepointer; 636 } 637 if (ompt_enabled.ompt_callback_task_create) { 638 ompt_task_info_t *parent_info = &(current_task->ompt_task_info); 639 ompt_callbacks.ompt_callback(ompt_callback_task_create)( 640 &(parent_info->task_data), &(parent_info->frame), 641 &(taskdata->ompt_task_info.task_data), 642 ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(taskdata), 0, 643 return_address); 644 } 645 __ompt_task_start(task, current_task, gtid); 646 } 647 #endif // OMPT_SUPPORT 648 649 KA_TRACE(10, ("__kmpc_omp_task_begin_if0(exit): T#%d loc=%p task=%p,\n", gtid, 650 loc_ref, taskdata)); 651 } 652 653 #if OMPT_SUPPORT 654 OMPT_NOINLINE 655 static void __kmpc_omp_task_begin_if0_ompt(ident_t *loc_ref, kmp_int32 gtid, 656 kmp_task_t *task, 657 void *frame_address, 658 void *return_address) { 659 __kmpc_omp_task_begin_if0_template<true>(loc_ref, gtid, task, frame_address, 660 return_address); 661 } 662 #endif // OMPT_SUPPORT 663 664 // __kmpc_omp_task_begin_if0: report that a given serialized task has started 665 // execution 666 // 667 // loc_ref: source location information; points to beginning of task block. 668 // gtid: global thread number. 669 // task: task thunk for the started task. 670 void __kmpc_omp_task_begin_if0(ident_t *loc_ref, kmp_int32 gtid, 671 kmp_task_t *task) { 672 #if OMPT_SUPPORT 673 if (UNLIKELY(ompt_enabled.enabled)) { 674 OMPT_STORE_RETURN_ADDRESS(gtid); 675 __kmpc_omp_task_begin_if0_ompt(loc_ref, gtid, task, 676 OMPT_GET_FRAME_ADDRESS(1), 677 OMPT_LOAD_RETURN_ADDRESS(gtid)); 678 return; 679 } 680 #endif 681 __kmpc_omp_task_begin_if0_template<false>(loc_ref, gtid, task, NULL, NULL); 682 } 683 684 #ifdef TASK_UNUSED 685 // __kmpc_omp_task_begin: report that a given task has started execution 686 // NEVER GENERATED BY COMPILER, DEPRECATED!!! 687 void __kmpc_omp_task_begin(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task) { 688 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task; 689 690 KA_TRACE( 691 10, 692 ("__kmpc_omp_task_begin(enter): T#%d loc=%p task=%p current_task=%p\n", 693 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task), current_task)); 694 695 __kmp_task_start(gtid, task, current_task); 696 697 KA_TRACE(10, ("__kmpc_omp_task_begin(exit): T#%d loc=%p task=%p,\n", gtid, 698 loc_ref, KMP_TASK_TO_TASKDATA(task))); 699 return; 700 } 701 #endif // TASK_UNUSED 702 703 // __kmp_free_task: free the current task space and the space for shareds 704 // 705 // gtid: Global thread ID of calling thread 706 // taskdata: task to free 707 // thread: thread data structure of caller 708 static void __kmp_free_task(kmp_int32 gtid, kmp_taskdata_t *taskdata, 709 kmp_info_t *thread) { 710 KA_TRACE(30, ("__kmp_free_task: T#%d freeing data from task %p\n", gtid, 711 taskdata)); 712 713 // Check to make sure all flags and counters have the correct values 714 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); 715 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0); 716 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 1); 717 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); 718 KMP_DEBUG_ASSERT(taskdata->td_allocated_child_tasks == 0 || 719 taskdata->td_flags.task_serial == 1); 720 KMP_DEBUG_ASSERT(taskdata->td_incomplete_child_tasks == 0); 721 722 taskdata->td_flags.freed = 1; 723 ANNOTATE_HAPPENS_BEFORE(taskdata); 724 // deallocate the taskdata and shared variable blocks associated with this task 725 #if USE_FAST_MEMORY 726 __kmp_fast_free(thread, taskdata); 727 #else /* ! USE_FAST_MEMORY */ 728 __kmp_thread_free(thread, taskdata); 729 #endif 730 731 KA_TRACE(20, ("__kmp_free_task: T#%d freed task %p\n", gtid, taskdata)); 732 } 733 734 // __kmp_free_task_and_ancestors: free the current task and ancestors without 735 // children 736 // 737 // gtid: Global thread ID of calling thread 738 // taskdata: task to free 739 // thread: thread data structure of caller 740 static void __kmp_free_task_and_ancestors(kmp_int32 gtid, 741 kmp_taskdata_t *taskdata, 742 kmp_info_t *thread) { 743 #if OMP_45_ENABLED 744 // Proxy tasks must always be allowed to free their parents 745 // because they can be run in background even in serial mode. 746 kmp_int32 team_serial = 747 (taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser) && 748 !taskdata->td_flags.proxy; 749 #else 750 kmp_int32 team_serial = 751 taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser; 752 #endif 753 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); 754 755 kmp_int32 children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks) - 1; 756 KMP_DEBUG_ASSERT(children >= 0); 757 758 // Now, go up the ancestor tree to see if any ancestors can now be freed. 759 while (children == 0) { 760 kmp_taskdata_t *parent_taskdata = taskdata->td_parent; 761 762 KA_TRACE(20, ("__kmp_free_task_and_ancestors(enter): T#%d task %p complete " 763 "and freeing itself\n", 764 gtid, taskdata)); 765 766 // --- Deallocate my ancestor task --- 767 __kmp_free_task(gtid, taskdata, thread); 768 769 taskdata = parent_taskdata; 770 771 if (team_serial) 772 return; 773 // Stop checking ancestors at implicit task instead of walking up ancestor 774 // tree to avoid premature deallocation of ancestors. 775 if (taskdata->td_flags.tasktype == TASK_IMPLICIT) { 776 if (taskdata->td_dephash) { // do we need to cleanup dephash? 777 int children = KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks); 778 kmp_tasking_flags_t flags_old = taskdata->td_flags; 779 if (children == 0 && flags_old.complete == 1) { 780 kmp_tasking_flags_t flags_new = flags_old; 781 flags_new.complete = 0; 782 if (KMP_COMPARE_AND_STORE_ACQ32( 783 RCAST(kmp_int32 *, &taskdata->td_flags), 784 *RCAST(kmp_int32 *, &flags_old), 785 *RCAST(kmp_int32 *, &flags_new))) { 786 KA_TRACE(100, ("__kmp_free_task_and_ancestors: T#%d cleans " 787 "dephash of implicit task %p\n", 788 gtid, taskdata)); 789 // cleanup dephash of finished implicit task 790 __kmp_dephash_free_entries(thread, taskdata->td_dephash); 791 } 792 } 793 } 794 return; 795 } 796 // Predecrement simulated by "- 1" calculation 797 children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks) - 1; 798 KMP_DEBUG_ASSERT(children >= 0); 799 } 800 801 KA_TRACE( 802 20, ("__kmp_free_task_and_ancestors(exit): T#%d task %p has %d children; " 803 "not freeing it yet\n", 804 gtid, taskdata, children)); 805 } 806 807 // __kmp_task_finish: bookkeeping to do when a task finishes execution 808 // 809 // gtid: global thread ID for calling thread 810 // task: task to be finished 811 // resumed_task: task to be resumed. (may be NULL if task is serialized) 812 template <bool ompt> 813 static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task, 814 kmp_taskdata_t *resumed_task) { 815 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 816 kmp_info_t *thread = __kmp_threads[gtid]; 817 #if OMP_45_ENABLED 818 kmp_task_team_t *task_team = 819 thread->th.th_task_team; // might be NULL for serial teams... 820 #endif // OMP_45_ENABLED 821 kmp_int32 children = 0; 822 823 KA_TRACE(10, ("__kmp_task_finish(enter): T#%d finishing task %p and resuming " 824 "task %p\n", 825 gtid, taskdata, resumed_task)); 826 827 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); 828 829 // Pop task from stack if tied 830 #ifdef BUILD_TIED_TASK_STACK 831 if (taskdata->td_flags.tiedness == TASK_TIED) { 832 __kmp_pop_task_stack(gtid, thread, taskdata); 833 } 834 #endif /* BUILD_TIED_TASK_STACK */ 835 836 if (taskdata->td_flags.tiedness == TASK_UNTIED) { 837 // untied task needs to check the counter so that the task structure is not 838 // freed prematurely 839 kmp_int32 counter = KMP_ATOMIC_DEC(&taskdata->td_untied_count) - 1; 840 KA_TRACE( 841 20, 842 ("__kmp_task_finish: T#%d untied_count (%d) decremented for task %p\n", 843 gtid, counter, taskdata)); 844 if (counter > 0) { 845 // untied task is not done, to be continued possibly by other thread, do 846 // not free it now 847 if (resumed_task == NULL) { 848 KMP_DEBUG_ASSERT(taskdata->td_flags.task_serial); 849 resumed_task = taskdata->td_parent; // In a serialized task, the resumed 850 // task is the parent 851 } 852 thread->th.th_current_task = resumed_task; // restore current_task 853 resumed_task->td_flags.executing = 1; // resume previous task 854 KA_TRACE(10, ("__kmp_task_finish(exit): T#%d partially done task %p, " 855 "resuming task %p\n", 856 gtid, taskdata, resumed_task)); 857 return; 858 } 859 } 860 #if OMPT_SUPPORT 861 if (ompt) 862 __ompt_task_finish(task, resumed_task); 863 #endif 864 865 // Check mutexinoutset dependencies, release locks 866 kmp_depnode_t *node = taskdata->td_depnode; 867 if (node && (node->dn.mtx_num_locks < 0)) { 868 // negative num_locks means all locks were acquired 869 node->dn.mtx_num_locks = -node->dn.mtx_num_locks; 870 for (int i = node->dn.mtx_num_locks - 1; i >= 0; --i) { 871 KMP_DEBUG_ASSERT(node->dn.mtx_locks[i] != NULL); 872 __kmp_release_lock(node->dn.mtx_locks[i], gtid); 873 } 874 } 875 876 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); 877 bool detach = false; 878 #if OMP_50_ENABLED 879 if (taskdata->td_flags.detachable == TASK_DETACHABLE) { 880 if (taskdata->td_allow_completion_event.type == 881 KMP_EVENT_ALLOW_COMPLETION) { 882 // event hasn't been fulfilled yet. Try to detach task. 883 __kmp_acquire_tas_lock(&taskdata->td_allow_completion_event.lock, gtid); 884 if (taskdata->td_allow_completion_event.type == 885 KMP_EVENT_ALLOW_COMPLETION) { 886 taskdata->td_flags.proxy = TASK_PROXY; // proxify! 887 detach = true; 888 } 889 __kmp_release_tas_lock(&taskdata->td_allow_completion_event.lock, gtid); 890 } 891 } 892 #endif 893 KMP_DEBUG_ASSERT(taskdata->td_flags.started == 1); 894 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); 895 896 if (!detach) { 897 taskdata->td_flags.complete = 1; // mark the task as completed 898 899 // Only need to keep track of count if team parallel and tasking not 900 // serialized 901 if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) { 902 // Predecrement simulated by "- 1" calculation 903 children = 904 KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks) - 1; 905 KMP_DEBUG_ASSERT(children >= 0); 906 #if OMP_40_ENABLED 907 if (taskdata->td_taskgroup) 908 KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count); 909 __kmp_release_deps(gtid, taskdata); 910 #if OMP_45_ENABLED 911 } else if (task_team && task_team->tt.tt_found_proxy_tasks) { 912 // if we found proxy tasks there could exist a dependency chain 913 // with the proxy task as origin 914 __kmp_release_deps(gtid, taskdata); 915 #endif // OMP_45_ENABLED 916 #endif // OMP_40_ENABLED 917 } 918 } 919 920 // td_flags.executing must be marked as 0 after __kmp_release_deps has been 921 // called. Othertwise, if a task is executed immediately from the release_deps 922 // code, the flag will be reset to 1 again by this same function 923 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1); 924 taskdata->td_flags.executing = 0; // suspend the finishing task 925 926 KA_TRACE( 927 20, ("__kmp_task_finish: T#%d finished task %p, %d incomplete children\n", 928 gtid, taskdata, children)); 929 930 #if OMP_40_ENABLED 931 /* If the tasks' destructor thunk flag has been set, we need to invoke the 932 destructor thunk that has been generated by the compiler. The code is 933 placed here, since at this point other tasks might have been released 934 hence overlapping the destructor invokations with some other work in the 935 released tasks. The OpenMP spec is not specific on when the destructors 936 are invoked, so we should be free to choose. */ 937 if (taskdata->td_flags.destructors_thunk) { 938 kmp_routine_entry_t destr_thunk = task->data1.destructors; 939 KMP_ASSERT(destr_thunk); 940 destr_thunk(gtid, task); 941 } 942 #endif // OMP_40_ENABLED 943 944 // bookkeeping for resuming task: 945 // GEH - note tasking_ser => task_serial 946 KMP_DEBUG_ASSERT( 947 (taskdata->td_flags.tasking_ser || taskdata->td_flags.task_serial) == 948 taskdata->td_flags.task_serial); 949 if (taskdata->td_flags.task_serial) { 950 if (resumed_task == NULL) { 951 resumed_task = taskdata->td_parent; // In a serialized task, the resumed 952 // task is the parent 953 } 954 } else { 955 KMP_DEBUG_ASSERT(resumed_task != 956 NULL); // verify that resumed task is passed as arguemnt 957 } 958 959 // Free this task and then ancestor tasks if they have no children. 960 // Restore th_current_task first as suggested by John: 961 // johnmc: if an asynchronous inquiry peers into the runtime system 962 // it doesn't see the freed task as the current task. 963 thread->th.th_current_task = resumed_task; 964 if (!detach) 965 __kmp_free_task_and_ancestors(gtid, taskdata, thread); 966 967 // TODO: GEH - make sure root team implicit task is initialized properly. 968 // KMP_DEBUG_ASSERT( resumed_task->td_flags.executing == 0 ); 969 resumed_task->td_flags.executing = 1; // resume previous task 970 971 KA_TRACE( 972 10, ("__kmp_task_finish(exit): T#%d finished task %p, resuming task %p\n", 973 gtid, taskdata, resumed_task)); 974 975 return; 976 } 977 978 template <bool ompt> 979 static void __kmpc_omp_task_complete_if0_template(ident_t *loc_ref, 980 kmp_int32 gtid, 981 kmp_task_t *task) { 982 KA_TRACE(10, ("__kmpc_omp_task_complete_if0(enter): T#%d loc=%p task=%p\n", 983 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task))); 984 // this routine will provide task to resume 985 __kmp_task_finish<ompt>(gtid, task, NULL); 986 987 KA_TRACE(10, ("__kmpc_omp_task_complete_if0(exit): T#%d loc=%p task=%p\n", 988 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task))); 989 990 #if OMPT_SUPPORT 991 if (ompt) { 992 ompt_frame_t *ompt_frame; 993 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 994 ompt_frame->enter_frame = ompt_data_none; 995 ompt_frame->enter_frame_flags = ompt_frame_runtime | ompt_frame_framepointer; 996 } 997 #endif 998 999 return; 1000 } 1001 1002 #if OMPT_SUPPORT 1003 OMPT_NOINLINE 1004 void __kmpc_omp_task_complete_if0_ompt(ident_t *loc_ref, kmp_int32 gtid, 1005 kmp_task_t *task) { 1006 __kmpc_omp_task_complete_if0_template<true>(loc_ref, gtid, task); 1007 } 1008 #endif // OMPT_SUPPORT 1009 1010 // __kmpc_omp_task_complete_if0: report that a task has completed execution 1011 // 1012 // loc_ref: source location information; points to end of task block. 1013 // gtid: global thread number. 1014 // task: task thunk for the completed task. 1015 void __kmpc_omp_task_complete_if0(ident_t *loc_ref, kmp_int32 gtid, 1016 kmp_task_t *task) { 1017 #if OMPT_SUPPORT 1018 if (UNLIKELY(ompt_enabled.enabled)) { 1019 __kmpc_omp_task_complete_if0_ompt(loc_ref, gtid, task); 1020 return; 1021 } 1022 #endif 1023 __kmpc_omp_task_complete_if0_template<false>(loc_ref, gtid, task); 1024 } 1025 1026 #ifdef TASK_UNUSED 1027 // __kmpc_omp_task_complete: report that a task has completed execution 1028 // NEVER GENERATED BY COMPILER, DEPRECATED!!! 1029 void __kmpc_omp_task_complete(ident_t *loc_ref, kmp_int32 gtid, 1030 kmp_task_t *task) { 1031 KA_TRACE(10, ("__kmpc_omp_task_complete(enter): T#%d loc=%p task=%p\n", gtid, 1032 loc_ref, KMP_TASK_TO_TASKDATA(task))); 1033 1034 __kmp_task_finish<false>(gtid, task, 1035 NULL); // Not sure how to find task to resume 1036 1037 KA_TRACE(10, ("__kmpc_omp_task_complete(exit): T#%d loc=%p task=%p\n", gtid, 1038 loc_ref, KMP_TASK_TO_TASKDATA(task))); 1039 return; 1040 } 1041 #endif // TASK_UNUSED 1042 1043 // __kmp_init_implicit_task: Initialize the appropriate fields in the implicit 1044 // task for a given thread 1045 // 1046 // loc_ref: reference to source location of parallel region 1047 // this_thr: thread data structure corresponding to implicit task 1048 // team: team for this_thr 1049 // tid: thread id of given thread within team 1050 // set_curr_task: TRUE if need to push current task to thread 1051 // NOTE: Routine does not set up the implicit task ICVS. This is assumed to 1052 // have already been done elsewhere. 1053 // TODO: Get better loc_ref. Value passed in may be NULL 1054 void __kmp_init_implicit_task(ident_t *loc_ref, kmp_info_t *this_thr, 1055 kmp_team_t *team, int tid, int set_curr_task) { 1056 kmp_taskdata_t *task = &team->t.t_implicit_task_taskdata[tid]; 1057 1058 KF_TRACE( 1059 10, 1060 ("__kmp_init_implicit_task(enter): T#:%d team=%p task=%p, reinit=%s\n", 1061 tid, team, task, set_curr_task ? "TRUE" : "FALSE")); 1062 1063 task->td_task_id = KMP_GEN_TASK_ID(); 1064 task->td_team = team; 1065 // task->td_parent = NULL; // fix for CQ230101 (broken parent task info 1066 // in debugger) 1067 task->td_ident = loc_ref; 1068 task->td_taskwait_ident = NULL; 1069 task->td_taskwait_counter = 0; 1070 task->td_taskwait_thread = 0; 1071 1072 task->td_flags.tiedness = TASK_TIED; 1073 task->td_flags.tasktype = TASK_IMPLICIT; 1074 #if OMP_45_ENABLED 1075 task->td_flags.proxy = TASK_FULL; 1076 #endif 1077 1078 // All implicit tasks are executed immediately, not deferred 1079 task->td_flags.task_serial = 1; 1080 task->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec); 1081 task->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0; 1082 1083 task->td_flags.started = 1; 1084 task->td_flags.executing = 1; 1085 task->td_flags.complete = 0; 1086 task->td_flags.freed = 0; 1087 1088 #if OMP_40_ENABLED 1089 task->td_depnode = NULL; 1090 #endif 1091 task->td_last_tied = task; 1092 #if OMP_50_ENABLED 1093 task->td_allow_completion_event.type = KMP_EVENT_UNINITIALIZED; 1094 #endif 1095 1096 if (set_curr_task) { // only do this init first time thread is created 1097 KMP_ATOMIC_ST_REL(&task->td_incomplete_child_tasks, 0); 1098 // Not used: don't need to deallocate implicit task 1099 KMP_ATOMIC_ST_REL(&task->td_allocated_child_tasks, 0); 1100 #if OMP_40_ENABLED 1101 task->td_taskgroup = NULL; // An implicit task does not have taskgroup 1102 task->td_dephash = NULL; 1103 #endif 1104 __kmp_push_current_task_to_thread(this_thr, team, tid); 1105 } else { 1106 KMP_DEBUG_ASSERT(task->td_incomplete_child_tasks == 0); 1107 KMP_DEBUG_ASSERT(task->td_allocated_child_tasks == 0); 1108 } 1109 1110 #if OMPT_SUPPORT 1111 if (UNLIKELY(ompt_enabled.enabled)) 1112 __ompt_task_init(task, tid); 1113 #endif 1114 1115 KF_TRACE(10, ("__kmp_init_implicit_task(exit): T#:%d team=%p task=%p\n", tid, 1116 team, task)); 1117 } 1118 1119 // __kmp_finish_implicit_task: Release resources associated to implicit tasks 1120 // at the end of parallel regions. Some resources are kept for reuse in the next 1121 // parallel region. 1122 // 1123 // thread: thread data structure corresponding to implicit task 1124 void __kmp_finish_implicit_task(kmp_info_t *thread) { 1125 kmp_taskdata_t *task = thread->th.th_current_task; 1126 if (task->td_dephash) { 1127 int children; 1128 task->td_flags.complete = 1; 1129 children = KMP_ATOMIC_LD_ACQ(&task->td_incomplete_child_tasks); 1130 kmp_tasking_flags_t flags_old = task->td_flags; 1131 if (children == 0 && flags_old.complete == 1) { 1132 kmp_tasking_flags_t flags_new = flags_old; 1133 flags_new.complete = 0; 1134 if (KMP_COMPARE_AND_STORE_ACQ32(RCAST(kmp_int32 *, &task->td_flags), 1135 *RCAST(kmp_int32 *, &flags_old), 1136 *RCAST(kmp_int32 *, &flags_new))) { 1137 KA_TRACE(100, ("__kmp_finish_implicit_task: T#%d cleans " 1138 "dephash of implicit task %p\n", 1139 thread->th.th_info.ds.ds_gtid, task)); 1140 __kmp_dephash_free_entries(thread, task->td_dephash); 1141 } 1142 } 1143 } 1144 } 1145 1146 // __kmp_free_implicit_task: Release resources associated to implicit tasks 1147 // when these are destroyed regions 1148 // 1149 // thread: thread data structure corresponding to implicit task 1150 void __kmp_free_implicit_task(kmp_info_t *thread) { 1151 kmp_taskdata_t *task = thread->th.th_current_task; 1152 if (task && task->td_dephash) { 1153 __kmp_dephash_free(thread, task->td_dephash); 1154 task->td_dephash = NULL; 1155 } 1156 } 1157 1158 // Round up a size to a power of two specified by val: Used to insert padding 1159 // between structures co-allocated using a single malloc() call 1160 static size_t __kmp_round_up_to_val(size_t size, size_t val) { 1161 if (size & (val - 1)) { 1162 size &= ~(val - 1); 1163 if (size <= KMP_SIZE_T_MAX - val) { 1164 size += val; // Round up if there is no overflow. 1165 } 1166 } 1167 return size; 1168 } // __kmp_round_up_to_va 1169 1170 // __kmp_task_alloc: Allocate the taskdata and task data structures for a task 1171 // 1172 // loc_ref: source location information 1173 // gtid: global thread number. 1174 // flags: include tiedness & task type (explicit vs. implicit) of the ''new'' 1175 // task encountered. Converted from kmp_int32 to kmp_tasking_flags_t in routine. 1176 // sizeof_kmp_task_t: Size in bytes of kmp_task_t data structure including 1177 // private vars accessed in task. 1178 // sizeof_shareds: Size in bytes of array of pointers to shared vars accessed 1179 // in task. 1180 // task_entry: Pointer to task code entry point generated by compiler. 1181 // returns: a pointer to the allocated kmp_task_t structure (task). 1182 kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid, 1183 kmp_tasking_flags_t *flags, 1184 size_t sizeof_kmp_task_t, size_t sizeof_shareds, 1185 kmp_routine_entry_t task_entry) { 1186 kmp_task_t *task; 1187 kmp_taskdata_t *taskdata; 1188 kmp_info_t *thread = __kmp_threads[gtid]; 1189 kmp_team_t *team = thread->th.th_team; 1190 kmp_taskdata_t *parent_task = thread->th.th_current_task; 1191 size_t shareds_offset; 1192 1193 if (!TCR_4(__kmp_init_middle)) 1194 __kmp_middle_initialize(); 1195 1196 KA_TRACE(10, ("__kmp_task_alloc(enter): T#%d loc=%p, flags=(0x%x) " 1197 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n", 1198 gtid, loc_ref, *((kmp_int32 *)flags), sizeof_kmp_task_t, 1199 sizeof_shareds, task_entry)); 1200 1201 if (parent_task->td_flags.final) { 1202 if (flags->merged_if0) { 1203 } 1204 flags->final = 1; 1205 } 1206 if (flags->tiedness == TASK_UNTIED && !team->t.t_serialized) { 1207 // Untied task encountered causes the TSC algorithm to check entire deque of 1208 // the victim thread. If no untied task encountered, then checking the head 1209 // of the deque should be enough. 1210 KMP_CHECK_UPDATE(thread->th.th_task_team->tt.tt_untied_task_encountered, 1); 1211 } 1212 1213 #if OMP_50_ENABLED 1214 // Detachable tasks are not proxy tasks yet but could be in the future. Doing 1215 // the tasking setup 1216 // when that happens is too late. 1217 if (flags->proxy == TASK_PROXY || flags->detachable == TASK_DETACHABLE) { 1218 #endif 1219 #if OMP_45_ENABLED 1220 if (flags->proxy == TASK_PROXY) { 1221 flags->tiedness = TASK_UNTIED; 1222 flags->merged_if0 = 1; 1223 #if OMP_50_ENABLED 1224 } 1225 #endif 1226 /* are we running in a sequential parallel or tskm_immediate_exec... we need 1227 tasking support enabled */ 1228 if ((thread->th.th_task_team) == NULL) { 1229 /* This should only happen if the team is serialized 1230 setup a task team and propagate it to the thread */ 1231 KMP_DEBUG_ASSERT(team->t.t_serialized); 1232 KA_TRACE(30, 1233 ("T#%d creating task team in __kmp_task_alloc for proxy task\n", 1234 gtid)); 1235 __kmp_task_team_setup( 1236 thread, team, 1237 1); // 1 indicates setup the current team regardless of nthreads 1238 thread->th.th_task_team = team->t.t_task_team[thread->th.th_task_state]; 1239 } 1240 kmp_task_team_t *task_team = thread->th.th_task_team; 1241 1242 /* tasking must be enabled now as the task might not be pushed */ 1243 if (!KMP_TASKING_ENABLED(task_team)) { 1244 KA_TRACE( 1245 30, 1246 ("T#%d enabling tasking in __kmp_task_alloc for proxy task\n", gtid)); 1247 __kmp_enable_tasking(task_team, thread); 1248 kmp_int32 tid = thread->th.th_info.ds.ds_tid; 1249 kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid]; 1250 // No lock needed since only owner can allocate 1251 if (thread_data->td.td_deque == NULL) { 1252 __kmp_alloc_task_deque(thread, thread_data); 1253 } 1254 } 1255 1256 if (task_team->tt.tt_found_proxy_tasks == FALSE) 1257 TCW_4(task_team->tt.tt_found_proxy_tasks, TRUE); 1258 } 1259 #endif 1260 1261 // Calculate shared structure offset including padding after kmp_task_t struct 1262 // to align pointers in shared struct 1263 shareds_offset = sizeof(kmp_taskdata_t) + sizeof_kmp_task_t; 1264 shareds_offset = __kmp_round_up_to_val(shareds_offset, sizeof(void *)); 1265 1266 // Allocate a kmp_taskdata_t block and a kmp_task_t block. 1267 KA_TRACE(30, ("__kmp_task_alloc: T#%d First malloc size: %ld\n", gtid, 1268 shareds_offset)); 1269 KA_TRACE(30, ("__kmp_task_alloc: T#%d Second malloc size: %ld\n", gtid, 1270 sizeof_shareds)); 1271 1272 // Avoid double allocation here by combining shareds with taskdata 1273 #if USE_FAST_MEMORY 1274 taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, shareds_offset + 1275 sizeof_shareds); 1276 #else /* ! USE_FAST_MEMORY */ 1277 taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, shareds_offset + 1278 sizeof_shareds); 1279 #endif /* USE_FAST_MEMORY */ 1280 ANNOTATE_HAPPENS_AFTER(taskdata); 1281 1282 task = KMP_TASKDATA_TO_TASK(taskdata); 1283 1284 // Make sure task & taskdata are aligned appropriately 1285 #if KMP_ARCH_X86 || KMP_ARCH_PPC64 || !KMP_HAVE_QUAD 1286 KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(double) - 1)) == 0); 1287 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (sizeof(double) - 1)) == 0); 1288 #else 1289 KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(_Quad) - 1)) == 0); 1290 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (sizeof(_Quad) - 1)) == 0); 1291 #endif 1292 if (sizeof_shareds > 0) { 1293 // Avoid double allocation here by combining shareds with taskdata 1294 task->shareds = &((char *)taskdata)[shareds_offset]; 1295 // Make sure shareds struct is aligned to pointer size 1296 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) == 1297 0); 1298 } else { 1299 task->shareds = NULL; 1300 } 1301 task->routine = task_entry; 1302 task->part_id = 0; // AC: Always start with 0 part id 1303 1304 taskdata->td_task_id = KMP_GEN_TASK_ID(); 1305 taskdata->td_team = team; 1306 taskdata->td_alloc_thread = thread; 1307 taskdata->td_parent = parent_task; 1308 taskdata->td_level = parent_task->td_level + 1; // increment nesting level 1309 KMP_ATOMIC_ST_RLX(&taskdata->td_untied_count, 0); 1310 taskdata->td_ident = loc_ref; 1311 taskdata->td_taskwait_ident = NULL; 1312 taskdata->td_taskwait_counter = 0; 1313 taskdata->td_taskwait_thread = 0; 1314 KMP_DEBUG_ASSERT(taskdata->td_parent != NULL); 1315 #if OMP_45_ENABLED 1316 // avoid copying icvs for proxy tasks 1317 if (flags->proxy == TASK_FULL) 1318 #endif 1319 copy_icvs(&taskdata->td_icvs, &taskdata->td_parent->td_icvs); 1320 1321 taskdata->td_flags.tiedness = flags->tiedness; 1322 taskdata->td_flags.final = flags->final; 1323 taskdata->td_flags.merged_if0 = flags->merged_if0; 1324 #if OMP_40_ENABLED 1325 taskdata->td_flags.destructors_thunk = flags->destructors_thunk; 1326 #endif // OMP_40_ENABLED 1327 #if OMP_45_ENABLED 1328 taskdata->td_flags.proxy = flags->proxy; 1329 #if OMP_50_ENABLED 1330 taskdata->td_flags.detachable = flags->detachable; 1331 #endif 1332 taskdata->td_task_team = thread->th.th_task_team; 1333 taskdata->td_size_alloc = shareds_offset + sizeof_shareds; 1334 #endif 1335 taskdata->td_flags.tasktype = TASK_EXPLICIT; 1336 1337 // GEH - TODO: fix this to copy parent task's value of tasking_ser flag 1338 taskdata->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec); 1339 1340 // GEH - TODO: fix this to copy parent task's value of team_serial flag 1341 taskdata->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0; 1342 1343 // GEH - Note we serialize the task if the team is serialized to make sure 1344 // implicit parallel region tasks are not left until program termination to 1345 // execute. Also, it helps locality to execute immediately. 1346 1347 taskdata->td_flags.task_serial = 1348 (parent_task->td_flags.final || taskdata->td_flags.team_serial || 1349 taskdata->td_flags.tasking_ser); 1350 1351 taskdata->td_flags.started = 0; 1352 taskdata->td_flags.executing = 0; 1353 taskdata->td_flags.complete = 0; 1354 taskdata->td_flags.freed = 0; 1355 1356 taskdata->td_flags.native = flags->native; 1357 1358 KMP_ATOMIC_ST_RLX(&taskdata->td_incomplete_child_tasks, 0); 1359 // start at one because counts current task and children 1360 KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 1); 1361 #if OMP_40_ENABLED 1362 taskdata->td_taskgroup = 1363 parent_task->td_taskgroup; // task inherits taskgroup from the parent task 1364 taskdata->td_dephash = NULL; 1365 taskdata->td_depnode = NULL; 1366 #endif 1367 if (flags->tiedness == TASK_UNTIED) 1368 taskdata->td_last_tied = NULL; // will be set when the task is scheduled 1369 else 1370 taskdata->td_last_tied = taskdata; 1371 #if OMP_50_ENABLED 1372 taskdata->td_allow_completion_event.type = KMP_EVENT_UNINITIALIZED; 1373 #endif 1374 #if OMPT_SUPPORT 1375 if (UNLIKELY(ompt_enabled.enabled)) 1376 __ompt_task_init(taskdata, gtid); 1377 #endif 1378 // Only need to keep track of child task counts if team parallel and tasking not 1379 // serialized or if it is a proxy or detachable task 1380 #if OMP_45_ENABLED 1381 if (flags->proxy == TASK_PROXY || 1382 #if OMP_50_ENABLED 1383 flags->detachable == TASK_DETACHABLE || 1384 #endif 1385 !(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) 1386 #else 1387 if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) 1388 #endif 1389 { 1390 KMP_ATOMIC_INC(&parent_task->td_incomplete_child_tasks); 1391 #if OMP_40_ENABLED 1392 if (parent_task->td_taskgroup) 1393 KMP_ATOMIC_INC(&parent_task->td_taskgroup->count); 1394 #endif 1395 // Only need to keep track of allocated child tasks for explicit tasks since 1396 // implicit not deallocated 1397 if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT) { 1398 KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks); 1399 } 1400 } 1401 1402 KA_TRACE(20, ("__kmp_task_alloc(exit): T#%d created task %p parent=%p\n", 1403 gtid, taskdata, taskdata->td_parent)); 1404 ANNOTATE_HAPPENS_BEFORE(task); 1405 1406 return task; 1407 } 1408 1409 kmp_task_t *__kmpc_omp_task_alloc(ident_t *loc_ref, kmp_int32 gtid, 1410 kmp_int32 flags, size_t sizeof_kmp_task_t, 1411 size_t sizeof_shareds, 1412 kmp_routine_entry_t task_entry) { 1413 kmp_task_t *retval; 1414 kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *)&flags; 1415 1416 input_flags->native = FALSE; 1417 // __kmp_task_alloc() sets up all other runtime flags 1418 1419 #if OMP_45_ENABLED 1420 #if OMP_50_ENABLED 1421 KA_TRACE(10, ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s %s %s) " 1422 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n", 1423 gtid, loc_ref, input_flags->tiedness ? "tied " : "untied", 1424 input_flags->proxy ? "proxy" : "", 1425 input_flags->detachable ? "detachable" : "", sizeof_kmp_task_t, 1426 sizeof_shareds, task_entry)); 1427 #else 1428 KA_TRACE(10, ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s %s) " 1429 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n", 1430 gtid, loc_ref, input_flags->tiedness ? "tied " : "untied", 1431 input_flags->proxy ? "proxy" : "", sizeof_kmp_task_t, 1432 sizeof_shareds, task_entry)); 1433 #endif 1434 #else 1435 KA_TRACE(10, ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s) " 1436 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n", 1437 gtid, loc_ref, input_flags->tiedness ? "tied " : "untied", 1438 sizeof_kmp_task_t, sizeof_shareds, task_entry)); 1439 #endif 1440 1441 retval = __kmp_task_alloc(loc_ref, gtid, input_flags, sizeof_kmp_task_t, 1442 sizeof_shareds, task_entry); 1443 1444 KA_TRACE(20, ("__kmpc_omp_task_alloc(exit): T#%d retval %p\n", gtid, retval)); 1445 1446 return retval; 1447 } 1448 1449 kmp_task_t *__kmpc_omp_target_task_alloc(ident_t *loc_ref, kmp_int32 gtid, 1450 kmp_int32 flags, 1451 size_t sizeof_kmp_task_t, 1452 size_t sizeof_shareds, 1453 kmp_routine_entry_t task_entry, 1454 kmp_int64 device_id) { 1455 return __kmpc_omp_task_alloc(loc_ref, gtid, flags, sizeof_kmp_task_t, 1456 sizeof_shareds, task_entry); 1457 } 1458 1459 #if OMP_50_ENABLED 1460 /*! 1461 @ingroup TASKING 1462 @param loc_ref location of the original task directive 1463 @param gtid Global Thread ID of encountering thread 1464 @param new_task task thunk allocated by __kmpc_omp_task_alloc() for the ''new 1465 task'' 1466 @param naffins Number of affinity items 1467 @param affin_list List of affinity items 1468 @return Returns non-zero if registering affinity information was not successful. 1469 Returns 0 if registration was successful 1470 This entry registers the affinity information attached to a task with the task 1471 thunk structure kmp_taskdata_t. 1472 */ 1473 kmp_int32 1474 __kmpc_omp_reg_task_with_affinity(ident_t *loc_ref, kmp_int32 gtid, 1475 kmp_task_t *new_task, kmp_int32 naffins, 1476 kmp_task_affinity_info_t *affin_list) { 1477 return 0; 1478 } 1479 #endif 1480 1481 // __kmp_invoke_task: invoke the specified task 1482 // 1483 // gtid: global thread ID of caller 1484 // task: the task to invoke 1485 // current_task: the task to resume after task invokation 1486 static void __kmp_invoke_task(kmp_int32 gtid, kmp_task_t *task, 1487 kmp_taskdata_t *current_task) { 1488 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 1489 kmp_info_t *thread; 1490 #if OMP_40_ENABLED 1491 int discard = 0 /* false */; 1492 #endif 1493 KA_TRACE( 1494 30, ("__kmp_invoke_task(enter): T#%d invoking task %p, current_task=%p\n", 1495 gtid, taskdata, current_task)); 1496 KMP_DEBUG_ASSERT(task); 1497 #if OMP_45_ENABLED 1498 if (taskdata->td_flags.proxy == TASK_PROXY && 1499 taskdata->td_flags.complete == 1) { 1500 // This is a proxy task that was already completed but it needs to run 1501 // its bottom-half finish 1502 KA_TRACE( 1503 30, 1504 ("__kmp_invoke_task: T#%d running bottom finish for proxy task %p\n", 1505 gtid, taskdata)); 1506 1507 __kmp_bottom_half_finish_proxy(gtid, task); 1508 1509 KA_TRACE(30, ("__kmp_invoke_task(exit): T#%d completed bottom finish for " 1510 "proxy task %p, resuming task %p\n", 1511 gtid, taskdata, current_task)); 1512 1513 return; 1514 } 1515 #endif 1516 1517 #if OMPT_SUPPORT 1518 // For untied tasks, the first task executed only calls __kmpc_omp_task and 1519 // does not execute code. 1520 ompt_thread_info_t oldInfo; 1521 if (UNLIKELY(ompt_enabled.enabled)) { 1522 // Store the threads states and restore them after the task 1523 thread = __kmp_threads[gtid]; 1524 oldInfo = thread->th.ompt_thread_info; 1525 thread->th.ompt_thread_info.wait_id = 0; 1526 thread->th.ompt_thread_info.state = (thread->th.th_team_serialized) 1527 ? ompt_state_work_serial 1528 : ompt_state_work_parallel; 1529 taskdata->ompt_task_info.frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1530 } 1531 #endif 1532 1533 #if OMP_45_ENABLED 1534 // Proxy tasks are not handled by the runtime 1535 if (taskdata->td_flags.proxy != TASK_PROXY) { 1536 #endif 1537 ANNOTATE_HAPPENS_AFTER(task); 1538 __kmp_task_start(gtid, task, current_task); // OMPT only if not discarded 1539 #if OMP_45_ENABLED 1540 } 1541 #endif 1542 1543 #if OMP_40_ENABLED 1544 // TODO: cancel tasks if the parallel region has also been cancelled 1545 // TODO: check if this sequence can be hoisted above __kmp_task_start 1546 // if cancellation has been enabled for this run ... 1547 if (__kmp_omp_cancellation) { 1548 thread = __kmp_threads[gtid]; 1549 kmp_team_t *this_team = thread->th.th_team; 1550 kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup; 1551 if ((taskgroup && taskgroup->cancel_request) || 1552 (this_team->t.t_cancel_request == cancel_parallel)) { 1553 #if OMPT_SUPPORT && OMPT_OPTIONAL 1554 ompt_data_t *task_data; 1555 if (UNLIKELY(ompt_enabled.ompt_callback_cancel)) { 1556 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL, NULL); 1557 ompt_callbacks.ompt_callback(ompt_callback_cancel)( 1558 task_data, 1559 ((taskgroup && taskgroup->cancel_request) ? ompt_cancel_taskgroup 1560 : ompt_cancel_parallel) | 1561 ompt_cancel_discarded_task, 1562 NULL); 1563 } 1564 #endif 1565 KMP_COUNT_BLOCK(TASK_cancelled); 1566 // this task belongs to a task group and we need to cancel it 1567 discard = 1 /* true */; 1568 } 1569 } 1570 1571 // Invoke the task routine and pass in relevant data. 1572 // Thunks generated by gcc take a different argument list. 1573 if (!discard) { 1574 if (taskdata->td_flags.tiedness == TASK_UNTIED) { 1575 taskdata->td_last_tied = current_task->td_last_tied; 1576 KMP_DEBUG_ASSERT(taskdata->td_last_tied); 1577 } 1578 #if KMP_STATS_ENABLED 1579 KMP_COUNT_BLOCK(TASK_executed); 1580 switch (KMP_GET_THREAD_STATE()) { 1581 case FORK_JOIN_BARRIER: 1582 KMP_PUSH_PARTITIONED_TIMER(OMP_task_join_bar); 1583 break; 1584 case PLAIN_BARRIER: 1585 KMP_PUSH_PARTITIONED_TIMER(OMP_task_plain_bar); 1586 break; 1587 case TASKYIELD: 1588 KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskyield); 1589 break; 1590 case TASKWAIT: 1591 KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskwait); 1592 break; 1593 case TASKGROUP: 1594 KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskgroup); 1595 break; 1596 default: 1597 KMP_PUSH_PARTITIONED_TIMER(OMP_task_immediate); 1598 break; 1599 } 1600 #endif // KMP_STATS_ENABLED 1601 #endif // OMP_40_ENABLED 1602 1603 // OMPT task begin 1604 #if OMPT_SUPPORT 1605 if (UNLIKELY(ompt_enabled.enabled)) 1606 __ompt_task_start(task, current_task, gtid); 1607 #endif 1608 1609 #if USE_ITT_BUILD && USE_ITT_NOTIFY 1610 kmp_uint64 cur_time; 1611 kmp_int32 kmp_itt_count_task = 1612 __kmp_forkjoin_frames_mode == 3 && !taskdata->td_flags.task_serial && 1613 current_task->td_flags.tasktype == TASK_IMPLICIT; 1614 if (kmp_itt_count_task) { 1615 thread = __kmp_threads[gtid]; 1616 // Time outer level explicit task on barrier for adjusting imbalance time 1617 if (thread->th.th_bar_arrive_time) 1618 cur_time = __itt_get_timestamp(); 1619 else 1620 kmp_itt_count_task = 0; // thread is not on a barrier - skip timing 1621 } 1622 #endif 1623 1624 #ifdef KMP_GOMP_COMPAT 1625 if (taskdata->td_flags.native) { 1626 ((void (*)(void *))(*(task->routine)))(task->shareds); 1627 } else 1628 #endif /* KMP_GOMP_COMPAT */ 1629 { 1630 (*(task->routine))(gtid, task); 1631 } 1632 KMP_POP_PARTITIONED_TIMER(); 1633 1634 #if USE_ITT_BUILD && USE_ITT_NOTIFY 1635 if (kmp_itt_count_task) { 1636 // Barrier imbalance - adjust arrive time with the task duration 1637 thread->th.th_bar_arrive_time += (__itt_get_timestamp() - cur_time); 1638 } 1639 #endif 1640 1641 #if OMP_40_ENABLED 1642 } 1643 #endif // OMP_40_ENABLED 1644 1645 1646 #if OMP_45_ENABLED 1647 // Proxy tasks are not handled by the runtime 1648 if (taskdata->td_flags.proxy != TASK_PROXY) { 1649 #endif 1650 ANNOTATE_HAPPENS_BEFORE(taskdata->td_parent); 1651 #if OMPT_SUPPORT 1652 if (UNLIKELY(ompt_enabled.enabled)) { 1653 thread->th.ompt_thread_info = oldInfo; 1654 if (taskdata->td_flags.tiedness == TASK_TIED) { 1655 taskdata->ompt_task_info.frame.exit_frame = ompt_data_none; 1656 } 1657 __kmp_task_finish<true>(gtid, task, current_task); 1658 } else 1659 #endif 1660 __kmp_task_finish<false>(gtid, task, current_task); 1661 #if OMP_45_ENABLED 1662 } 1663 #endif 1664 1665 KA_TRACE( 1666 30, 1667 ("__kmp_invoke_task(exit): T#%d completed task %p, resuming task %p\n", 1668 gtid, taskdata, current_task)); 1669 return; 1670 } 1671 1672 // __kmpc_omp_task_parts: Schedule a thread-switchable task for execution 1673 // 1674 // loc_ref: location of original task pragma (ignored) 1675 // gtid: Global Thread ID of encountering thread 1676 // new_task: task thunk allocated by __kmp_omp_task_alloc() for the ''new task'' 1677 // Returns: 1678 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to 1679 // be resumed later. 1680 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be 1681 // resumed later. 1682 kmp_int32 __kmpc_omp_task_parts(ident_t *loc_ref, kmp_int32 gtid, 1683 kmp_task_t *new_task) { 1684 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task); 1685 1686 KA_TRACE(10, ("__kmpc_omp_task_parts(enter): T#%d loc=%p task=%p\n", gtid, 1687 loc_ref, new_taskdata)); 1688 1689 #if OMPT_SUPPORT 1690 kmp_taskdata_t *parent; 1691 if (UNLIKELY(ompt_enabled.enabled)) { 1692 parent = new_taskdata->td_parent; 1693 if (ompt_enabled.ompt_callback_task_create) { 1694 ompt_data_t task_data = ompt_data_none; 1695 ompt_callbacks.ompt_callback(ompt_callback_task_create)( 1696 parent ? &(parent->ompt_task_info.task_data) : &task_data, 1697 parent ? &(parent->ompt_task_info.frame) : NULL, 1698 &(new_taskdata->ompt_task_info.task_data), ompt_task_explicit, 0, 1699 OMPT_GET_RETURN_ADDRESS(0)); 1700 } 1701 } 1702 #endif 1703 1704 /* Should we execute the new task or queue it? For now, let's just always try 1705 to queue it. If the queue fills up, then we'll execute it. */ 1706 1707 if (__kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED) // if cannot defer 1708 { // Execute this task immediately 1709 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task; 1710 new_taskdata->td_flags.task_serial = 1; 1711 __kmp_invoke_task(gtid, new_task, current_task); 1712 } 1713 1714 KA_TRACE( 1715 10, 1716 ("__kmpc_omp_task_parts(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: " 1717 "loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n", 1718 gtid, loc_ref, new_taskdata)); 1719 1720 ANNOTATE_HAPPENS_BEFORE(new_task); 1721 #if OMPT_SUPPORT 1722 if (UNLIKELY(ompt_enabled.enabled)) { 1723 parent->ompt_task_info.frame.enter_frame = ompt_data_none; 1724 } 1725 #endif 1726 return TASK_CURRENT_NOT_QUEUED; 1727 } 1728 1729 // __kmp_omp_task: Schedule a non-thread-switchable task for execution 1730 // 1731 // gtid: Global Thread ID of encountering thread 1732 // new_task:non-thread-switchable task thunk allocated by __kmp_omp_task_alloc() 1733 // serialize_immediate: if TRUE then if the task is executed immediately its 1734 // execution will be serialized 1735 // Returns: 1736 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to 1737 // be resumed later. 1738 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be 1739 // resumed later. 1740 kmp_int32 __kmp_omp_task(kmp_int32 gtid, kmp_task_t *new_task, 1741 bool serialize_immediate) { 1742 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task); 1743 1744 /* Should we execute the new task or queue it? For now, let's just always try to 1745 queue it. If the queue fills up, then we'll execute it. */ 1746 #if OMP_45_ENABLED 1747 if (new_taskdata->td_flags.proxy == TASK_PROXY || 1748 __kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED) // if cannot defer 1749 #else 1750 if (__kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED) // if cannot defer 1751 #endif 1752 { // Execute this task immediately 1753 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task; 1754 if (serialize_immediate) 1755 new_taskdata->td_flags.task_serial = 1; 1756 __kmp_invoke_task(gtid, new_task, current_task); 1757 } 1758 1759 ANNOTATE_HAPPENS_BEFORE(new_task); 1760 return TASK_CURRENT_NOT_QUEUED; 1761 } 1762 1763 // __kmpc_omp_task: Wrapper around __kmp_omp_task to schedule a 1764 // non-thread-switchable task from the parent thread only! 1765 // 1766 // loc_ref: location of original task pragma (ignored) 1767 // gtid: Global Thread ID of encountering thread 1768 // new_task: non-thread-switchable task thunk allocated by 1769 // __kmp_omp_task_alloc() 1770 // Returns: 1771 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to 1772 // be resumed later. 1773 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be 1774 // resumed later. 1775 kmp_int32 __kmpc_omp_task(ident_t *loc_ref, kmp_int32 gtid, 1776 kmp_task_t *new_task) { 1777 kmp_int32 res; 1778 KMP_SET_THREAD_STATE_BLOCK(EXPLICIT_TASK); 1779 1780 #if KMP_DEBUG || OMPT_SUPPORT 1781 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task); 1782 #endif 1783 KA_TRACE(10, ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref, 1784 new_taskdata)); 1785 1786 #if OMPT_SUPPORT 1787 kmp_taskdata_t *parent = NULL; 1788 if (UNLIKELY(ompt_enabled.enabled)) { 1789 if (!new_taskdata->td_flags.started) { 1790 OMPT_STORE_RETURN_ADDRESS(gtid); 1791 parent = new_taskdata->td_parent; 1792 if (!parent->ompt_task_info.frame.enter_frame.ptr) { 1793 parent->ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1794 } 1795 if (ompt_enabled.ompt_callback_task_create) { 1796 ompt_data_t task_data = ompt_data_none; 1797 ompt_callbacks.ompt_callback(ompt_callback_task_create)( 1798 parent ? &(parent->ompt_task_info.task_data) : &task_data, 1799 parent ? &(parent->ompt_task_info.frame) : NULL, 1800 &(new_taskdata->ompt_task_info.task_data), 1801 ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata), 0, 1802 OMPT_LOAD_RETURN_ADDRESS(gtid)); 1803 } 1804 } else { 1805 // We are scheduling the continuation of an UNTIED task. 1806 // Scheduling back to the parent task. 1807 __ompt_task_finish(new_task, 1808 new_taskdata->ompt_task_info.scheduling_parent, 1809 ompt_task_switch); 1810 new_taskdata->ompt_task_info.frame.exit_frame = ompt_data_none; 1811 } 1812 } 1813 #endif 1814 1815 res = __kmp_omp_task(gtid, new_task, true); 1816 1817 KA_TRACE(10, ("__kmpc_omp_task(exit): T#%d returning " 1818 "TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n", 1819 gtid, loc_ref, new_taskdata)); 1820 #if OMPT_SUPPORT 1821 if (UNLIKELY(ompt_enabled.enabled && parent != NULL)) { 1822 parent->ompt_task_info.frame.enter_frame = ompt_data_none; 1823 } 1824 #endif 1825 return res; 1826 } 1827 1828 // __kmp_omp_taskloop_task: Wrapper around __kmp_omp_task to schedule 1829 // a taskloop task with the correct OMPT return address 1830 // 1831 // loc_ref: location of original task pragma (ignored) 1832 // gtid: Global Thread ID of encountering thread 1833 // new_task: non-thread-switchable task thunk allocated by 1834 // __kmp_omp_task_alloc() 1835 // codeptr_ra: return address for OMPT callback 1836 // Returns: 1837 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to 1838 // be resumed later. 1839 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be 1840 // resumed later. 1841 kmp_int32 __kmp_omp_taskloop_task(ident_t *loc_ref, kmp_int32 gtid, 1842 kmp_task_t *new_task, void *codeptr_ra) { 1843 kmp_int32 res; 1844 KMP_SET_THREAD_STATE_BLOCK(EXPLICIT_TASK); 1845 1846 #if KMP_DEBUG || OMPT_SUPPORT 1847 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task); 1848 #endif 1849 KA_TRACE(10, ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref, 1850 new_taskdata)); 1851 1852 #if OMPT_SUPPORT 1853 kmp_taskdata_t *parent = NULL; 1854 if (UNLIKELY(ompt_enabled.enabled && !new_taskdata->td_flags.started)) { 1855 parent = new_taskdata->td_parent; 1856 if (!parent->ompt_task_info.frame.enter_frame.ptr) 1857 parent->ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1858 if (ompt_enabled.ompt_callback_task_create) { 1859 ompt_data_t task_data = ompt_data_none; 1860 ompt_callbacks.ompt_callback(ompt_callback_task_create)( 1861 parent ? &(parent->ompt_task_info.task_data) : &task_data, 1862 parent ? &(parent->ompt_task_info.frame) : NULL, 1863 &(new_taskdata->ompt_task_info.task_data), 1864 ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata), 0, 1865 codeptr_ra); 1866 } 1867 } 1868 #endif 1869 1870 res = __kmp_omp_task(gtid, new_task, true); 1871 1872 KA_TRACE(10, ("__kmpc_omp_task(exit): T#%d returning " 1873 "TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n", 1874 gtid, loc_ref, new_taskdata)); 1875 #if OMPT_SUPPORT 1876 if (UNLIKELY(ompt_enabled.enabled && parent != NULL)) { 1877 parent->ompt_task_info.frame.enter_frame = ompt_data_none; 1878 } 1879 #endif 1880 return res; 1881 } 1882 1883 template <bool ompt> 1884 static kmp_int32 __kmpc_omp_taskwait_template(ident_t *loc_ref, kmp_int32 gtid, 1885 void *frame_address, 1886 void *return_address) { 1887 kmp_taskdata_t *taskdata; 1888 kmp_info_t *thread; 1889 int thread_finished = FALSE; 1890 KMP_SET_THREAD_STATE_BLOCK(TASKWAIT); 1891 1892 KA_TRACE(10, ("__kmpc_omp_taskwait(enter): T#%d loc=%p\n", gtid, loc_ref)); 1893 1894 if (__kmp_tasking_mode != tskm_immediate_exec) { 1895 thread = __kmp_threads[gtid]; 1896 taskdata = thread->th.th_current_task; 1897 1898 #if OMPT_SUPPORT && OMPT_OPTIONAL 1899 ompt_data_t *my_task_data; 1900 ompt_data_t *my_parallel_data; 1901 1902 if (ompt) { 1903 my_task_data = &(taskdata->ompt_task_info.task_data); 1904 my_parallel_data = OMPT_CUR_TEAM_DATA(thread); 1905 1906 taskdata->ompt_task_info.frame.enter_frame.ptr = frame_address; 1907 1908 if (ompt_enabled.ompt_callback_sync_region) { 1909 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 1910 ompt_sync_region_taskwait, ompt_scope_begin, my_parallel_data, 1911 my_task_data, return_address); 1912 } 1913 1914 if (ompt_enabled.ompt_callback_sync_region_wait) { 1915 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 1916 ompt_sync_region_taskwait, ompt_scope_begin, my_parallel_data, 1917 my_task_data, return_address); 1918 } 1919 } 1920 #endif // OMPT_SUPPORT && OMPT_OPTIONAL 1921 1922 // Debugger: The taskwait is active. Store location and thread encountered the 1923 // taskwait. 1924 #if USE_ITT_BUILD 1925 // Note: These values are used by ITT events as well. 1926 #endif /* USE_ITT_BUILD */ 1927 taskdata->td_taskwait_counter += 1; 1928 taskdata->td_taskwait_ident = loc_ref; 1929 taskdata->td_taskwait_thread = gtid + 1; 1930 1931 #if USE_ITT_BUILD 1932 void *itt_sync_obj = __kmp_itt_taskwait_object(gtid); 1933 if (itt_sync_obj != NULL) 1934 __kmp_itt_taskwait_starting(gtid, itt_sync_obj); 1935 #endif /* USE_ITT_BUILD */ 1936 1937 bool must_wait = 1938 !taskdata->td_flags.team_serial && !taskdata->td_flags.final; 1939 1940 #if OMP_45_ENABLED 1941 must_wait = must_wait || (thread->th.th_task_team != NULL && 1942 thread->th.th_task_team->tt.tt_found_proxy_tasks); 1943 #endif 1944 if (must_wait) { 1945 kmp_flag_32 flag(RCAST(std::atomic<kmp_uint32> *, 1946 &(taskdata->td_incomplete_child_tasks)), 1947 0U); 1948 while (KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) != 0) { 1949 flag.execute_tasks(thread, gtid, FALSE, 1950 &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), 1951 __kmp_task_stealing_constraint); 1952 } 1953 } 1954 #if USE_ITT_BUILD 1955 if (itt_sync_obj != NULL) 1956 __kmp_itt_taskwait_finished(gtid, itt_sync_obj); 1957 #endif /* USE_ITT_BUILD */ 1958 1959 // Debugger: The taskwait is completed. Location remains, but thread is 1960 // negated. 1961 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; 1962 1963 #if OMPT_SUPPORT && OMPT_OPTIONAL 1964 if (ompt) { 1965 if (ompt_enabled.ompt_callback_sync_region_wait) { 1966 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 1967 ompt_sync_region_taskwait, ompt_scope_end, my_parallel_data, 1968 my_task_data, return_address); 1969 } 1970 if (ompt_enabled.ompt_callback_sync_region) { 1971 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 1972 ompt_sync_region_taskwait, ompt_scope_end, my_parallel_data, 1973 my_task_data, return_address); 1974 } 1975 taskdata->ompt_task_info.frame.enter_frame = ompt_data_none; 1976 } 1977 #endif // OMPT_SUPPORT && OMPT_OPTIONAL 1978 1979 ANNOTATE_HAPPENS_AFTER(taskdata); 1980 } 1981 1982 KA_TRACE(10, ("__kmpc_omp_taskwait(exit): T#%d task %p finished waiting, " 1983 "returning TASK_CURRENT_NOT_QUEUED\n", 1984 gtid, taskdata)); 1985 1986 return TASK_CURRENT_NOT_QUEUED; 1987 } 1988 1989 #if OMPT_SUPPORT && OMPT_OPTIONAL 1990 OMPT_NOINLINE 1991 static kmp_int32 __kmpc_omp_taskwait_ompt(ident_t *loc_ref, kmp_int32 gtid, 1992 void *frame_address, 1993 void *return_address) { 1994 return __kmpc_omp_taskwait_template<true>(loc_ref, gtid, frame_address, 1995 return_address); 1996 } 1997 #endif // OMPT_SUPPORT && OMPT_OPTIONAL 1998 1999 // __kmpc_omp_taskwait: Wait until all tasks generated by the current task are 2000 // complete 2001 kmp_int32 __kmpc_omp_taskwait(ident_t *loc_ref, kmp_int32 gtid) { 2002 #if OMPT_SUPPORT && OMPT_OPTIONAL 2003 if (UNLIKELY(ompt_enabled.enabled)) { 2004 OMPT_STORE_RETURN_ADDRESS(gtid); 2005 return __kmpc_omp_taskwait_ompt(loc_ref, gtid, OMPT_GET_FRAME_ADDRESS(0), 2006 OMPT_LOAD_RETURN_ADDRESS(gtid)); 2007 } 2008 #endif 2009 return __kmpc_omp_taskwait_template<false>(loc_ref, gtid, NULL, NULL); 2010 } 2011 2012 // __kmpc_omp_taskyield: switch to a different task 2013 kmp_int32 __kmpc_omp_taskyield(ident_t *loc_ref, kmp_int32 gtid, int end_part) { 2014 kmp_taskdata_t *taskdata; 2015 kmp_info_t *thread; 2016 int thread_finished = FALSE; 2017 2018 KMP_COUNT_BLOCK(OMP_TASKYIELD); 2019 KMP_SET_THREAD_STATE_BLOCK(TASKYIELD); 2020 2021 KA_TRACE(10, ("__kmpc_omp_taskyield(enter): T#%d loc=%p end_part = %d\n", 2022 gtid, loc_ref, end_part)); 2023 2024 if (__kmp_tasking_mode != tskm_immediate_exec && __kmp_init_parallel) { 2025 thread = __kmp_threads[gtid]; 2026 taskdata = thread->th.th_current_task; 2027 // Should we model this as a task wait or not? 2028 // Debugger: The taskwait is active. Store location and thread encountered the 2029 // taskwait. 2030 #if USE_ITT_BUILD 2031 // Note: These values are used by ITT events as well. 2032 #endif /* USE_ITT_BUILD */ 2033 taskdata->td_taskwait_counter += 1; 2034 taskdata->td_taskwait_ident = loc_ref; 2035 taskdata->td_taskwait_thread = gtid + 1; 2036 2037 #if USE_ITT_BUILD 2038 void *itt_sync_obj = __kmp_itt_taskwait_object(gtid); 2039 if (itt_sync_obj != NULL) 2040 __kmp_itt_taskwait_starting(gtid, itt_sync_obj); 2041 #endif /* USE_ITT_BUILD */ 2042 if (!taskdata->td_flags.team_serial) { 2043 kmp_task_team_t *task_team = thread->th.th_task_team; 2044 if (task_team != NULL) { 2045 if (KMP_TASKING_ENABLED(task_team)) { 2046 #if OMPT_SUPPORT 2047 if (UNLIKELY(ompt_enabled.enabled)) 2048 thread->th.ompt_thread_info.ompt_task_yielded = 1; 2049 #endif 2050 __kmp_execute_tasks_32( 2051 thread, gtid, NULL, FALSE, 2052 &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), 2053 __kmp_task_stealing_constraint); 2054 #if OMPT_SUPPORT 2055 if (UNLIKELY(ompt_enabled.enabled)) 2056 thread->th.ompt_thread_info.ompt_task_yielded = 0; 2057 #endif 2058 } 2059 } 2060 } 2061 #if USE_ITT_BUILD 2062 if (itt_sync_obj != NULL) 2063 __kmp_itt_taskwait_finished(gtid, itt_sync_obj); 2064 #endif /* USE_ITT_BUILD */ 2065 2066 // Debugger: The taskwait is completed. Location remains, but thread is 2067 // negated. 2068 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; 2069 } 2070 2071 KA_TRACE(10, ("__kmpc_omp_taskyield(exit): T#%d task %p resuming, " 2072 "returning TASK_CURRENT_NOT_QUEUED\n", 2073 gtid, taskdata)); 2074 2075 return TASK_CURRENT_NOT_QUEUED; 2076 } 2077 2078 #if OMP_50_ENABLED 2079 // Task Reduction implementation 2080 // 2081 // Note: initial implementation didn't take into account the possibility 2082 // to specify omp_orig for initializer of the UDR (user defined reduction). 2083 // Corrected implementation takes into account the omp_orig object. 2084 // Compiler is free to use old implementation if omp_orig is not specified. 2085 2086 /*! 2087 @ingroup BASIC_TYPES 2088 @{ 2089 */ 2090 2091 /*! 2092 Flags for special info per task reduction item. 2093 */ 2094 typedef struct kmp_taskred_flags { 2095 /*! 1 - use lazy alloc/init (e.g. big objects, #tasks < #threads) */ 2096 unsigned lazy_priv : 1; 2097 unsigned reserved31 : 31; 2098 } kmp_taskred_flags_t; 2099 2100 /*! 2101 Internal struct for reduction data item related info set up by compiler. 2102 */ 2103 typedef struct kmp_task_red_input { 2104 void *reduce_shar; /**< shared between tasks item to reduce into */ 2105 size_t reduce_size; /**< size of data item in bytes */ 2106 // three compiler-generated routines (init, fini are optional): 2107 void *reduce_init; /**< data initialization routine (single parameter) */ 2108 void *reduce_fini; /**< data finalization routine */ 2109 void *reduce_comb; /**< data combiner routine */ 2110 kmp_taskred_flags_t flags; /**< flags for additional info from compiler */ 2111 } kmp_task_red_input_t; 2112 2113 /*! 2114 Internal struct for reduction data item related info saved by the library. 2115 */ 2116 typedef struct kmp_taskred_data { 2117 void *reduce_shar; /**< shared between tasks item to reduce into */ 2118 size_t reduce_size; /**< size of data item */ 2119 kmp_taskred_flags_t flags; /**< flags for additional info from compiler */ 2120 void *reduce_priv; /**< array of thread specific items */ 2121 void *reduce_pend; /**< end of private data for faster comparison op */ 2122 // three compiler-generated routines (init, fini are optional): 2123 void *reduce_comb; /**< data combiner routine */ 2124 void *reduce_init; /**< data initialization routine (two parameters) */ 2125 void *reduce_fini; /**< data finalization routine */ 2126 void *reduce_orig; /**< original item (can be used in UDR initializer) */ 2127 } kmp_taskred_data_t; 2128 2129 /*! 2130 Internal struct for reduction data item related info set up by compiler. 2131 2132 New interface: added reduce_orig field to provide omp_orig for UDR initializer. 2133 */ 2134 typedef struct kmp_taskred_input { 2135 void *reduce_shar; /**< shared between tasks item to reduce into */ 2136 void *reduce_orig; /**< original reduction item used for initialization */ 2137 size_t reduce_size; /**< size of data item */ 2138 // three compiler-generated routines (init, fini are optional): 2139 void *reduce_init; /**< data initialization routine (two parameters) */ 2140 void *reduce_fini; /**< data finalization routine */ 2141 void *reduce_comb; /**< data combiner routine */ 2142 kmp_taskred_flags_t flags; /**< flags for additional info from compiler */ 2143 } kmp_taskred_input_t; 2144 /*! 2145 @} 2146 */ 2147 2148 template <typename T> void __kmp_assign_orig(kmp_taskred_data_t &item, T &src); 2149 template <> 2150 void __kmp_assign_orig<kmp_task_red_input_t>(kmp_taskred_data_t &item, 2151 kmp_task_red_input_t &src) { 2152 item.reduce_orig = NULL; 2153 } 2154 template <> 2155 void __kmp_assign_orig<kmp_taskred_input_t>(kmp_taskred_data_t &item, 2156 kmp_taskred_input_t &src) { 2157 if (src.reduce_orig != NULL) { 2158 item.reduce_orig = src.reduce_orig; 2159 } else { 2160 item.reduce_orig = src.reduce_shar; 2161 } // non-NULL reduce_orig means new interface used 2162 } 2163 2164 template <typename T> void __kmp_call_init(kmp_taskred_data_t &item, int j); 2165 template <> 2166 void __kmp_call_init<kmp_task_red_input_t>(kmp_taskred_data_t &item, 2167 int offset) { 2168 ((void (*)(void *))item.reduce_init)((char *)(item.reduce_priv) + offset); 2169 } 2170 template <> 2171 void __kmp_call_init<kmp_taskred_input_t>(kmp_taskred_data_t &item, 2172 int offset) { 2173 ((void (*)(void *, void *))item.reduce_init)( 2174 (char *)(item.reduce_priv) + offset, item.reduce_orig); 2175 } 2176 2177 template <typename T> 2178 void *__kmp_task_reduction_init(int gtid, int num, T *data) { 2179 kmp_info_t *thread = __kmp_threads[gtid]; 2180 kmp_taskgroup_t *tg = thread->th.th_current_task->td_taskgroup; 2181 kmp_int32 nth = thread->th.th_team_nproc; 2182 kmp_taskred_data_t *arr; 2183 2184 // check input data just in case 2185 KMP_ASSERT(tg != NULL); 2186 KMP_ASSERT(data != NULL); 2187 KMP_ASSERT(num > 0); 2188 if (nth == 1) { 2189 KA_TRACE(10, ("__kmpc_task_reduction_init: T#%d, tg %p, exiting nth=1\n", 2190 gtid, tg)); 2191 return (void *)tg; 2192 } 2193 KA_TRACE(10, ("__kmpc_task_reduction_init: T#%d, taskgroup %p, #items %d\n", 2194 gtid, tg, num)); 2195 arr = (kmp_taskred_data_t *)__kmp_thread_malloc( 2196 thread, num * sizeof(kmp_taskred_data_t)); 2197 for (int i = 0; i < num; ++i) { 2198 size_t size = data[i].reduce_size - 1; 2199 // round the size up to cache line per thread-specific item 2200 size += CACHE_LINE - size % CACHE_LINE; 2201 KMP_ASSERT(data[i].reduce_comb != NULL); // combiner is mandatory 2202 arr[i].reduce_shar = data[i].reduce_shar; 2203 arr[i].reduce_size = size; 2204 arr[i].flags = data[i].flags; 2205 arr[i].reduce_comb = data[i].reduce_comb; 2206 arr[i].reduce_init = data[i].reduce_init; 2207 arr[i].reduce_fini = data[i].reduce_fini; 2208 __kmp_assign_orig<T>(arr[i], data[i]); 2209 if (!arr[i].flags.lazy_priv) { 2210 // allocate cache-line aligned block and fill it with zeros 2211 arr[i].reduce_priv = __kmp_allocate(nth * size); 2212 arr[i].reduce_pend = (char *)(arr[i].reduce_priv) + nth * size; 2213 if (arr[i].reduce_init != NULL) { 2214 // initialize all thread-specific items 2215 for (int j = 0; j < nth; ++j) { 2216 __kmp_call_init<T>(arr[i], j * size); 2217 } 2218 } 2219 } else { 2220 // only allocate space for pointers now, 2221 // objects will be lazily allocated/initialized if/when requested 2222 // note that __kmp_allocate zeroes the allocated memory 2223 arr[i].reduce_priv = __kmp_allocate(nth * sizeof(void *)); 2224 } 2225 } 2226 tg->reduce_data = (void *)arr; 2227 tg->reduce_num_data = num; 2228 return (void *)tg; 2229 } 2230 2231 /*! 2232 @ingroup TASKING 2233 @param gtid Global thread ID 2234 @param num Number of data items to reduce 2235 @param data Array of data for reduction 2236 @return The taskgroup identifier 2237 2238 Initialize task reduction for the taskgroup. 2239 2240 Note: this entry supposes the optional compiler-generated initializer routine 2241 has single parameter - pointer to object to be initialized. That means 2242 the reduction either does not use omp_orig object, or the omp_orig is accessible 2243 without help of the runtime library. 2244 */ 2245 void *__kmpc_task_reduction_init(int gtid, int num, void *data) { 2246 return __kmp_task_reduction_init(gtid, num, (kmp_task_red_input_t *)data); 2247 } 2248 2249 /*! 2250 @ingroup TASKING 2251 @param gtid Global thread ID 2252 @param num Number of data items to reduce 2253 @param data Array of data for reduction 2254 @return The taskgroup identifier 2255 2256 Initialize task reduction for the taskgroup. 2257 2258 Note: this entry supposes the optional compiler-generated initializer routine 2259 has two parameters, pointer to object to be initialized and pointer to omp_orig 2260 */ 2261 void *__kmpc_taskred_init(int gtid, int num, void *data) { 2262 return __kmp_task_reduction_init(gtid, num, (kmp_taskred_input_t *)data); 2263 } 2264 2265 // Copy task reduction data (except for shared pointers). 2266 template <typename T> 2267 void __kmp_task_reduction_init_copy(kmp_info_t *thr, int num, T *data, 2268 kmp_taskgroup_t *tg, void *reduce_data) { 2269 kmp_taskred_data_t *arr; 2270 KA_TRACE(20, ("__kmp_task_reduction_init_copy: Th %p, init taskgroup %p," 2271 " from data %p\n", 2272 thr, tg, reduce_data)); 2273 arr = (kmp_taskred_data_t *)__kmp_thread_malloc( 2274 thr, num * sizeof(kmp_taskred_data_t)); 2275 // threads will share private copies, thunk routines, sizes, flags, etc.: 2276 KMP_MEMCPY(arr, reduce_data, num * sizeof(kmp_taskred_data_t)); 2277 for (int i = 0; i < num; ++i) { 2278 arr[i].reduce_shar = data[i].reduce_shar; // init unique shared pointers 2279 } 2280 tg->reduce_data = (void *)arr; 2281 tg->reduce_num_data = num; 2282 } 2283 2284 /*! 2285 @ingroup TASKING 2286 @param gtid Global thread ID 2287 @param tskgrp The taskgroup ID (optional) 2288 @param data Shared location of the item 2289 @return The pointer to per-thread data 2290 2291 Get thread-specific location of data item 2292 */ 2293 void *__kmpc_task_reduction_get_th_data(int gtid, void *tskgrp, void *data) { 2294 kmp_info_t *thread = __kmp_threads[gtid]; 2295 kmp_int32 nth = thread->th.th_team_nproc; 2296 if (nth == 1) 2297 return data; // nothing to do 2298 2299 kmp_taskgroup_t *tg = (kmp_taskgroup_t *)tskgrp; 2300 if (tg == NULL) 2301 tg = thread->th.th_current_task->td_taskgroup; 2302 KMP_ASSERT(tg != NULL); 2303 kmp_taskred_data_t *arr = (kmp_taskred_data_t *)(tg->reduce_data); 2304 kmp_int32 num = tg->reduce_num_data; 2305 kmp_int32 tid = thread->th.th_info.ds.ds_tid; 2306 2307 KMP_ASSERT(data != NULL); 2308 while (tg != NULL) { 2309 for (int i = 0; i < num; ++i) { 2310 if (!arr[i].flags.lazy_priv) { 2311 if (data == arr[i].reduce_shar || 2312 (data >= arr[i].reduce_priv && data < arr[i].reduce_pend)) 2313 return (char *)(arr[i].reduce_priv) + tid * arr[i].reduce_size; 2314 } else { 2315 // check shared location first 2316 void **p_priv = (void **)(arr[i].reduce_priv); 2317 if (data == arr[i].reduce_shar) 2318 goto found; 2319 // check if we get some thread specific location as parameter 2320 for (int j = 0; j < nth; ++j) 2321 if (data == p_priv[j]) 2322 goto found; 2323 continue; // not found, continue search 2324 found: 2325 if (p_priv[tid] == NULL) { 2326 // allocate thread specific object lazily 2327 p_priv[tid] = __kmp_allocate(arr[i].reduce_size); 2328 if (arr[i].reduce_init != NULL) { 2329 if (arr[i].reduce_orig != NULL) { // new interface 2330 ((void (*)(void *, void *))arr[i].reduce_init)( 2331 p_priv[tid], arr[i].reduce_orig); 2332 } else { // old interface (single parameter) 2333 ((void (*)(void *))arr[i].reduce_init)(p_priv[tid]); 2334 } 2335 } 2336 } 2337 return p_priv[tid]; 2338 } 2339 } 2340 tg = tg->parent; 2341 arr = (kmp_taskred_data_t *)(tg->reduce_data); 2342 num = tg->reduce_num_data; 2343 } 2344 KMP_ASSERT2(0, "Unknown task reduction item"); 2345 return NULL; // ERROR, this line never executed 2346 } 2347 2348 // Finalize task reduction. 2349 // Called from __kmpc_end_taskgroup() 2350 static void __kmp_task_reduction_fini(kmp_info_t *th, kmp_taskgroup_t *tg) { 2351 kmp_int32 nth = th->th.th_team_nproc; 2352 KMP_DEBUG_ASSERT(nth > 1); // should not be called if nth == 1 2353 kmp_taskred_data_t *arr = (kmp_taskred_data_t *)tg->reduce_data; 2354 kmp_int32 num = tg->reduce_num_data; 2355 for (int i = 0; i < num; ++i) { 2356 void *sh_data = arr[i].reduce_shar; 2357 void (*f_fini)(void *) = (void (*)(void *))(arr[i].reduce_fini); 2358 void (*f_comb)(void *, void *) = 2359 (void (*)(void *, void *))(arr[i].reduce_comb); 2360 if (!arr[i].flags.lazy_priv) { 2361 void *pr_data = arr[i].reduce_priv; 2362 size_t size = arr[i].reduce_size; 2363 for (int j = 0; j < nth; ++j) { 2364 void *priv_data = (char *)pr_data + j * size; 2365 f_comb(sh_data, priv_data); // combine results 2366 if (f_fini) 2367 f_fini(priv_data); // finalize if needed 2368 } 2369 } else { 2370 void **pr_data = (void **)(arr[i].reduce_priv); 2371 for (int j = 0; j < nth; ++j) { 2372 if (pr_data[j] != NULL) { 2373 f_comb(sh_data, pr_data[j]); // combine results 2374 if (f_fini) 2375 f_fini(pr_data[j]); // finalize if needed 2376 __kmp_free(pr_data[j]); 2377 } 2378 } 2379 } 2380 __kmp_free(arr[i].reduce_priv); 2381 } 2382 __kmp_thread_free(th, arr); 2383 tg->reduce_data = NULL; 2384 tg->reduce_num_data = 0; 2385 } 2386 2387 // Cleanup task reduction data for parallel or worksharing, 2388 // do not touch task private data other threads still working with. 2389 // Called from __kmpc_end_taskgroup() 2390 static void __kmp_task_reduction_clean(kmp_info_t *th, kmp_taskgroup_t *tg) { 2391 __kmp_thread_free(th, tg->reduce_data); 2392 tg->reduce_data = NULL; 2393 tg->reduce_num_data = 0; 2394 } 2395 2396 template <typename T> 2397 void *__kmp_task_reduction_modifier_init(ident_t *loc, int gtid, int is_ws, 2398 int num, T *data) { 2399 kmp_info_t *thr = __kmp_threads[gtid]; 2400 kmp_int32 nth = thr->th.th_team_nproc; 2401 __kmpc_taskgroup(loc, gtid); // form new taskgroup first 2402 if (nth == 1) { 2403 KA_TRACE(10, 2404 ("__kmpc_reduction_modifier_init: T#%d, tg %p, exiting nth=1\n", 2405 gtid, thr->th.th_current_task->td_taskgroup)); 2406 return (void *)thr->th.th_current_task->td_taskgroup; 2407 } 2408 kmp_team_t *team = thr->th.th_team; 2409 void *reduce_data; 2410 kmp_taskgroup_t *tg; 2411 reduce_data = KMP_ATOMIC_LD_RLX(&team->t.t_tg_reduce_data[is_ws]); 2412 if (reduce_data == NULL && 2413 __kmp_atomic_compare_store(&team->t.t_tg_reduce_data[is_ws], reduce_data, 2414 (void *)1)) { 2415 // single thread enters this block to initialize common reduction data 2416 KMP_DEBUG_ASSERT(reduce_data == NULL); 2417 // first initialize own data, then make a copy other threads can use 2418 tg = (kmp_taskgroup_t *)__kmp_task_reduction_init<T>(gtid, num, data); 2419 reduce_data = __kmp_thread_malloc(thr, num * sizeof(kmp_taskred_data_t)); 2420 KMP_MEMCPY(reduce_data, tg->reduce_data, num * sizeof(kmp_taskred_data_t)); 2421 // fini counters should be 0 at this point 2422 KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&team->t.t_tg_fini_counter[0]) == 0); 2423 KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&team->t.t_tg_fini_counter[1]) == 0); 2424 KMP_ATOMIC_ST_REL(&team->t.t_tg_reduce_data[is_ws], reduce_data); 2425 } else { 2426 while ( 2427 (reduce_data = KMP_ATOMIC_LD_ACQ(&team->t.t_tg_reduce_data[is_ws])) == 2428 (void *)1) { // wait for task reduction initialization 2429 KMP_CPU_PAUSE(); 2430 } 2431 KMP_DEBUG_ASSERT(reduce_data > (void *)1); // should be valid pointer here 2432 tg = thr->th.th_current_task->td_taskgroup; 2433 __kmp_task_reduction_init_copy<T>(thr, num, data, tg, reduce_data); 2434 } 2435 return tg; 2436 } 2437 2438 /*! 2439 @ingroup TASKING 2440 @param loc Source location info 2441 @param gtid Global thread ID 2442 @param is_ws Is 1 if the reduction is for worksharing, 0 otherwise 2443 @param num Number of data items to reduce 2444 @param data Array of data for reduction 2445 @return The taskgroup identifier 2446 2447 Initialize task reduction for a parallel or worksharing. 2448 2449 Note: this entry supposes the optional compiler-generated initializer routine 2450 has single parameter - pointer to object to be initialized. That means 2451 the reduction either does not use omp_orig object, or the omp_orig is accessible 2452 without help of the runtime library. 2453 */ 2454 void *__kmpc_task_reduction_modifier_init(ident_t *loc, int gtid, int is_ws, 2455 int num, void *data) { 2456 return __kmp_task_reduction_modifier_init(loc, gtid, is_ws, num, 2457 (kmp_task_red_input_t *)data); 2458 } 2459 2460 /*! 2461 @ingroup TASKING 2462 @param loc Source location info 2463 @param gtid Global thread ID 2464 @param is_ws Is 1 if the reduction is for worksharing, 0 otherwise 2465 @param num Number of data items to reduce 2466 @param data Array of data for reduction 2467 @return The taskgroup identifier 2468 2469 Initialize task reduction for a parallel or worksharing. 2470 2471 Note: this entry supposes the optional compiler-generated initializer routine 2472 has two parameters, pointer to object to be initialized and pointer to omp_orig 2473 */ 2474 void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int is_ws, int num, 2475 void *data) { 2476 return __kmp_task_reduction_modifier_init(loc, gtid, is_ws, num, 2477 (kmp_taskred_input_t *)data); 2478 } 2479 2480 /*! 2481 @ingroup TASKING 2482 @param loc Source location info 2483 @param gtid Global thread ID 2484 @param is_ws Is 1 if the reduction is for worksharing, 0 otherwise 2485 2486 Finalize task reduction for a parallel or worksharing. 2487 */ 2488 void __kmpc_task_reduction_modifier_fini(ident_t *loc, int gtid, int is_ws) { 2489 __kmpc_end_taskgroup(loc, gtid); 2490 } 2491 #endif 2492 2493 #if OMP_40_ENABLED 2494 // __kmpc_taskgroup: Start a new taskgroup 2495 void __kmpc_taskgroup(ident_t *loc, int gtid) { 2496 kmp_info_t *thread = __kmp_threads[gtid]; 2497 kmp_taskdata_t *taskdata = thread->th.th_current_task; 2498 kmp_taskgroup_t *tg_new = 2499 (kmp_taskgroup_t *)__kmp_thread_malloc(thread, sizeof(kmp_taskgroup_t)); 2500 KA_TRACE(10, ("__kmpc_taskgroup: T#%d loc=%p group=%p\n", gtid, loc, tg_new)); 2501 KMP_ATOMIC_ST_RLX(&tg_new->count, 0); 2502 KMP_ATOMIC_ST_RLX(&tg_new->cancel_request, cancel_noreq); 2503 tg_new->parent = taskdata->td_taskgroup; 2504 #if OMP_50_ENABLED 2505 tg_new->reduce_data = NULL; 2506 tg_new->reduce_num_data = 0; 2507 #endif 2508 taskdata->td_taskgroup = tg_new; 2509 2510 #if OMPT_SUPPORT && OMPT_OPTIONAL 2511 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)) { 2512 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); 2513 if (!codeptr) 2514 codeptr = OMPT_GET_RETURN_ADDRESS(0); 2515 kmp_team_t *team = thread->th.th_team; 2516 ompt_data_t my_task_data = taskdata->ompt_task_info.task_data; 2517 // FIXME: I think this is wrong for lwt! 2518 ompt_data_t my_parallel_data = team->t.ompt_team_info.parallel_data; 2519 2520 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 2521 ompt_sync_region_taskgroup, ompt_scope_begin, &(my_parallel_data), 2522 &(my_task_data), codeptr); 2523 } 2524 #endif 2525 } 2526 2527 // __kmpc_end_taskgroup: Wait until all tasks generated by the current task 2528 // and its descendants are complete 2529 void __kmpc_end_taskgroup(ident_t *loc, int gtid) { 2530 kmp_info_t *thread = __kmp_threads[gtid]; 2531 kmp_taskdata_t *taskdata = thread->th.th_current_task; 2532 kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup; 2533 int thread_finished = FALSE; 2534 2535 #if OMPT_SUPPORT && OMPT_OPTIONAL 2536 kmp_team_t *team; 2537 ompt_data_t my_task_data; 2538 ompt_data_t my_parallel_data; 2539 void *codeptr; 2540 if (UNLIKELY(ompt_enabled.enabled)) { 2541 team = thread->th.th_team; 2542 my_task_data = taskdata->ompt_task_info.task_data; 2543 // FIXME: I think this is wrong for lwt! 2544 my_parallel_data = team->t.ompt_team_info.parallel_data; 2545 codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid); 2546 if (!codeptr) 2547 codeptr = OMPT_GET_RETURN_ADDRESS(0); 2548 } 2549 #endif 2550 2551 KA_TRACE(10, ("__kmpc_end_taskgroup(enter): T#%d loc=%p\n", gtid, loc)); 2552 KMP_DEBUG_ASSERT(taskgroup != NULL); 2553 KMP_SET_THREAD_STATE_BLOCK(TASKGROUP); 2554 2555 if (__kmp_tasking_mode != tskm_immediate_exec) { 2556 // mark task as waiting not on a barrier 2557 taskdata->td_taskwait_counter += 1; 2558 taskdata->td_taskwait_ident = loc; 2559 taskdata->td_taskwait_thread = gtid + 1; 2560 #if USE_ITT_BUILD 2561 // For ITT the taskgroup wait is similar to taskwait until we need to 2562 // distinguish them 2563 void *itt_sync_obj = __kmp_itt_taskwait_object(gtid); 2564 if (itt_sync_obj != NULL) 2565 __kmp_itt_taskwait_starting(gtid, itt_sync_obj); 2566 #endif /* USE_ITT_BUILD */ 2567 2568 #if OMPT_SUPPORT && OMPT_OPTIONAL 2569 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region_wait)) { 2570 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 2571 ompt_sync_region_taskgroup, ompt_scope_begin, &(my_parallel_data), 2572 &(my_task_data), codeptr); 2573 } 2574 #endif 2575 2576 #if OMP_45_ENABLED 2577 if (!taskdata->td_flags.team_serial || 2578 (thread->th.th_task_team != NULL && 2579 thread->th.th_task_team->tt.tt_found_proxy_tasks)) 2580 #else 2581 if (!taskdata->td_flags.team_serial) 2582 #endif 2583 { 2584 kmp_flag_32 flag(RCAST(std::atomic<kmp_uint32> *, &(taskgroup->count)), 2585 0U); 2586 while (KMP_ATOMIC_LD_ACQ(&taskgroup->count) != 0) { 2587 flag.execute_tasks(thread, gtid, FALSE, 2588 &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), 2589 __kmp_task_stealing_constraint); 2590 } 2591 } 2592 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; // end waiting 2593 2594 #if OMPT_SUPPORT && OMPT_OPTIONAL 2595 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region_wait)) { 2596 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 2597 ompt_sync_region_taskgroup, ompt_scope_end, &(my_parallel_data), 2598 &(my_task_data), codeptr); 2599 } 2600 #endif 2601 2602 #if USE_ITT_BUILD 2603 if (itt_sync_obj != NULL) 2604 __kmp_itt_taskwait_finished(gtid, itt_sync_obj); 2605 #endif /* USE_ITT_BUILD */ 2606 } 2607 KMP_DEBUG_ASSERT(taskgroup->count == 0); 2608 2609 #if OMP_50_ENABLED 2610 if (taskgroup->reduce_data != NULL) { // need to reduce? 2611 int cnt; 2612 void *reduce_data; 2613 kmp_team_t *t = thread->th.th_team; 2614 kmp_taskred_data_t *arr = (kmp_taskred_data_t *)taskgroup->reduce_data; 2615 // check if <priv> data of the first reduction variable shared for the team 2616 void *priv0 = arr[0].reduce_priv; 2617 if ((reduce_data = KMP_ATOMIC_LD_ACQ(&t->t.t_tg_reduce_data[0])) != NULL && 2618 ((kmp_taskred_data_t *)reduce_data)[0].reduce_priv == priv0) { 2619 // finishing task reduction on parallel 2620 cnt = KMP_ATOMIC_INC(&t->t.t_tg_fini_counter[0]); 2621 if (cnt == thread->th.th_team_nproc - 1) { 2622 // we are the last thread passing __kmpc_reduction_modifier_fini() 2623 // finalize task reduction: 2624 __kmp_task_reduction_fini(thread, taskgroup); 2625 // cleanup fields in the team structure: 2626 // TODO: is relaxed store enough here (whole barrier should follow)? 2627 __kmp_thread_free(thread, reduce_data); 2628 KMP_ATOMIC_ST_REL(&t->t.t_tg_reduce_data[0], NULL); 2629 KMP_ATOMIC_ST_REL(&t->t.t_tg_fini_counter[0], 0); 2630 } else { 2631 // we are not the last thread passing __kmpc_reduction_modifier_fini(), 2632 // so do not finalize reduction, just clean own copy of the data 2633 __kmp_task_reduction_clean(thread, taskgroup); 2634 } 2635 } else if ((reduce_data = KMP_ATOMIC_LD_ACQ(&t->t.t_tg_reduce_data[1])) != 2636 NULL && 2637 ((kmp_taskred_data_t *)reduce_data)[0].reduce_priv == priv0) { 2638 // finishing task reduction on worksharing 2639 cnt = KMP_ATOMIC_INC(&t->t.t_tg_fini_counter[1]); 2640 if (cnt == thread->th.th_team_nproc - 1) { 2641 // we are the last thread passing __kmpc_reduction_modifier_fini() 2642 __kmp_task_reduction_fini(thread, taskgroup); 2643 // cleanup fields in team structure: 2644 // TODO: is relaxed store enough here (whole barrier should follow)? 2645 __kmp_thread_free(thread, reduce_data); 2646 KMP_ATOMIC_ST_REL(&t->t.t_tg_reduce_data[1], NULL); 2647 KMP_ATOMIC_ST_REL(&t->t.t_tg_fini_counter[1], 0); 2648 } else { 2649 // we are not the last thread passing __kmpc_reduction_modifier_fini(), 2650 // so do not finalize reduction, just clean own copy of the data 2651 __kmp_task_reduction_clean(thread, taskgroup); 2652 } 2653 } else { 2654 // finishing task reduction on taskgroup 2655 __kmp_task_reduction_fini(thread, taskgroup); 2656 } 2657 } 2658 #endif 2659 // Restore parent taskgroup for the current task 2660 taskdata->td_taskgroup = taskgroup->parent; 2661 __kmp_thread_free(thread, taskgroup); 2662 2663 KA_TRACE(10, ("__kmpc_end_taskgroup(exit): T#%d task %p finished waiting\n", 2664 gtid, taskdata)); 2665 ANNOTATE_HAPPENS_AFTER(taskdata); 2666 2667 #if OMPT_SUPPORT && OMPT_OPTIONAL 2668 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)) { 2669 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 2670 ompt_sync_region_taskgroup, ompt_scope_end, &(my_parallel_data), 2671 &(my_task_data), codeptr); 2672 } 2673 #endif 2674 } 2675 #endif 2676 2677 // __kmp_remove_my_task: remove a task from my own deque 2678 static kmp_task_t *__kmp_remove_my_task(kmp_info_t *thread, kmp_int32 gtid, 2679 kmp_task_team_t *task_team, 2680 kmp_int32 is_constrained) { 2681 kmp_task_t *task; 2682 kmp_taskdata_t *taskdata; 2683 kmp_thread_data_t *thread_data; 2684 kmp_uint32 tail; 2685 2686 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 2687 KMP_DEBUG_ASSERT(task_team->tt.tt_threads_data != 2688 NULL); // Caller should check this condition 2689 2690 thread_data = &task_team->tt.tt_threads_data[__kmp_tid_from_gtid(gtid)]; 2691 2692 KA_TRACE(10, ("__kmp_remove_my_task(enter): T#%d ntasks=%d head=%u tail=%u\n", 2693 gtid, thread_data->td.td_deque_ntasks, 2694 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 2695 2696 if (TCR_4(thread_data->td.td_deque_ntasks) == 0) { 2697 KA_TRACE(10, 2698 ("__kmp_remove_my_task(exit #1): T#%d No tasks to remove: " 2699 "ntasks=%d head=%u tail=%u\n", 2700 gtid, thread_data->td.td_deque_ntasks, 2701 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 2702 return NULL; 2703 } 2704 2705 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 2706 2707 if (TCR_4(thread_data->td.td_deque_ntasks) == 0) { 2708 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 2709 KA_TRACE(10, 2710 ("__kmp_remove_my_task(exit #2): T#%d No tasks to remove: " 2711 "ntasks=%d head=%u tail=%u\n", 2712 gtid, thread_data->td.td_deque_ntasks, 2713 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 2714 return NULL; 2715 } 2716 2717 tail = (thread_data->td.td_deque_tail - 1) & 2718 TASK_DEQUE_MASK(thread_data->td); // Wrap index. 2719 taskdata = thread_data->td.td_deque[tail]; 2720 2721 if (!__kmp_task_is_allowed(gtid, is_constrained, taskdata, 2722 thread->th.th_current_task)) { 2723 // The TSC does not allow to steal victim task 2724 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 2725 KA_TRACE(10, 2726 ("__kmp_remove_my_task(exit #3): T#%d TSC blocks tail task: " 2727 "ntasks=%d head=%u tail=%u\n", 2728 gtid, thread_data->td.td_deque_ntasks, 2729 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 2730 return NULL; 2731 } 2732 2733 thread_data->td.td_deque_tail = tail; 2734 TCW_4(thread_data->td.td_deque_ntasks, thread_data->td.td_deque_ntasks - 1); 2735 2736 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 2737 2738 KA_TRACE(10, ("__kmp_remove_my_task(exit #4): T#%d task %p removed: " 2739 "ntasks=%d head=%u tail=%u\n", 2740 gtid, taskdata, thread_data->td.td_deque_ntasks, 2741 thread_data->td.td_deque_head, thread_data->td.td_deque_tail)); 2742 2743 task = KMP_TASKDATA_TO_TASK(taskdata); 2744 return task; 2745 } 2746 2747 // __kmp_steal_task: remove a task from another thread's deque 2748 // Assume that calling thread has already checked existence of 2749 // task_team thread_data before calling this routine. 2750 static kmp_task_t *__kmp_steal_task(kmp_info_t *victim_thr, kmp_int32 gtid, 2751 kmp_task_team_t *task_team, 2752 std::atomic<kmp_int32> *unfinished_threads, 2753 int *thread_finished, 2754 kmp_int32 is_constrained) { 2755 kmp_task_t *task; 2756 kmp_taskdata_t *taskdata; 2757 kmp_taskdata_t *current; 2758 kmp_thread_data_t *victim_td, *threads_data; 2759 kmp_int32 target; 2760 kmp_int32 victim_tid; 2761 2762 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 2763 2764 threads_data = task_team->tt.tt_threads_data; 2765 KMP_DEBUG_ASSERT(threads_data != NULL); // Caller should check this condition 2766 2767 victim_tid = victim_thr->th.th_info.ds.ds_tid; 2768 victim_td = &threads_data[victim_tid]; 2769 2770 KA_TRACE(10, ("__kmp_steal_task(enter): T#%d try to steal from T#%d: " 2771 "task_team=%p ntasks=%d head=%u tail=%u\n", 2772 gtid, __kmp_gtid_from_thread(victim_thr), task_team, 2773 victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head, 2774 victim_td->td.td_deque_tail)); 2775 2776 if (TCR_4(victim_td->td.td_deque_ntasks) == 0) { 2777 KA_TRACE(10, ("__kmp_steal_task(exit #1): T#%d could not steal from T#%d: " 2778 "task_team=%p ntasks=%d head=%u tail=%u\n", 2779 gtid, __kmp_gtid_from_thread(victim_thr), task_team, 2780 victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head, 2781 victim_td->td.td_deque_tail)); 2782 return NULL; 2783 } 2784 2785 __kmp_acquire_bootstrap_lock(&victim_td->td.td_deque_lock); 2786 2787 int ntasks = TCR_4(victim_td->td.td_deque_ntasks); 2788 // Check again after we acquire the lock 2789 if (ntasks == 0) { 2790 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock); 2791 KA_TRACE(10, ("__kmp_steal_task(exit #2): T#%d could not steal from T#%d: " 2792 "task_team=%p ntasks=%d head=%u tail=%u\n", 2793 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks, 2794 victim_td->td.td_deque_head, victim_td->td.td_deque_tail)); 2795 return NULL; 2796 } 2797 2798 KMP_DEBUG_ASSERT(victim_td->td.td_deque != NULL); 2799 current = __kmp_threads[gtid]->th.th_current_task; 2800 taskdata = victim_td->td.td_deque[victim_td->td.td_deque_head]; 2801 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) { 2802 // Bump head pointer and Wrap. 2803 victim_td->td.td_deque_head = 2804 (victim_td->td.td_deque_head + 1) & TASK_DEQUE_MASK(victim_td->td); 2805 } else { 2806 if (!task_team->tt.tt_untied_task_encountered) { 2807 // The TSC does not allow to steal victim task 2808 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock); 2809 KA_TRACE(10, ("__kmp_steal_task(exit #3): T#%d could not steal from " 2810 "T#%d: task_team=%p ntasks=%d head=%u tail=%u\n", 2811 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks, 2812 victim_td->td.td_deque_head, victim_td->td.td_deque_tail)); 2813 return NULL; 2814 } 2815 int i; 2816 // walk through victim's deque trying to steal any task 2817 target = victim_td->td.td_deque_head; 2818 taskdata = NULL; 2819 for (i = 1; i < ntasks; ++i) { 2820 target = (target + 1) & TASK_DEQUE_MASK(victim_td->td); 2821 taskdata = victim_td->td.td_deque[target]; 2822 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) { 2823 break; // found victim task 2824 } else { 2825 taskdata = NULL; 2826 } 2827 } 2828 if (taskdata == NULL) { 2829 // No appropriate candidate to steal found 2830 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock); 2831 KA_TRACE(10, ("__kmp_steal_task(exit #4): T#%d could not steal from " 2832 "T#%d: task_team=%p ntasks=%d head=%u tail=%u\n", 2833 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks, 2834 victim_td->td.td_deque_head, victim_td->td.td_deque_tail)); 2835 return NULL; 2836 } 2837 int prev = target; 2838 for (i = i + 1; i < ntasks; ++i) { 2839 // shift remaining tasks in the deque left by 1 2840 target = (target + 1) & TASK_DEQUE_MASK(victim_td->td); 2841 victim_td->td.td_deque[prev] = victim_td->td.td_deque[target]; 2842 prev = target; 2843 } 2844 KMP_DEBUG_ASSERT( 2845 victim_td->td.td_deque_tail == 2846 (kmp_uint32)((target + 1) & TASK_DEQUE_MASK(victim_td->td))); 2847 victim_td->td.td_deque_tail = target; // tail -= 1 (wrapped)) 2848 } 2849 if (*thread_finished) { 2850 // We need to un-mark this victim as a finished victim. This must be done 2851 // before releasing the lock, or else other threads (starting with the 2852 // master victim) might be prematurely released from the barrier!!! 2853 kmp_int32 count; 2854 2855 count = KMP_ATOMIC_INC(unfinished_threads); 2856 2857 KA_TRACE( 2858 20, 2859 ("__kmp_steal_task: T#%d inc unfinished_threads to %d: task_team=%p\n", 2860 gtid, count + 1, task_team)); 2861 2862 *thread_finished = FALSE; 2863 } 2864 TCW_4(victim_td->td.td_deque_ntasks, ntasks - 1); 2865 2866 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock); 2867 2868 KMP_COUNT_BLOCK(TASK_stolen); 2869 KA_TRACE(10, 2870 ("__kmp_steal_task(exit #5): T#%d stole task %p from T#%d: " 2871 "task_team=%p ntasks=%d head=%u tail=%u\n", 2872 gtid, taskdata, __kmp_gtid_from_thread(victim_thr), task_team, 2873 ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail)); 2874 2875 task = KMP_TASKDATA_TO_TASK(taskdata); 2876 return task; 2877 } 2878 2879 // __kmp_execute_tasks_template: Choose and execute tasks until either the 2880 // condition is statisfied (return true) or there are none left (return false). 2881 // 2882 // final_spin is TRUE if this is the spin at the release barrier. 2883 // thread_finished indicates whether the thread is finished executing all 2884 // the tasks it has on its deque, and is at the release barrier. 2885 // spinner is the location on which to spin. 2886 // spinner == NULL means only execute a single task and return. 2887 // checker is the value to check to terminate the spin. 2888 template <class C> 2889 static inline int __kmp_execute_tasks_template( 2890 kmp_info_t *thread, kmp_int32 gtid, C *flag, int final_spin, 2891 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 2892 kmp_int32 is_constrained) { 2893 kmp_task_team_t *task_team = thread->th.th_task_team; 2894 kmp_thread_data_t *threads_data; 2895 kmp_task_t *task; 2896 kmp_info_t *other_thread; 2897 kmp_taskdata_t *current_task = thread->th.th_current_task; 2898 std::atomic<kmp_int32> *unfinished_threads; 2899 kmp_int32 nthreads, victim_tid = -2, use_own_tasks = 1, new_victim = 0, 2900 tid = thread->th.th_info.ds.ds_tid; 2901 2902 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 2903 KMP_DEBUG_ASSERT(thread == __kmp_threads[gtid]); 2904 2905 if (task_team == NULL || current_task == NULL) 2906 return FALSE; 2907 2908 KA_TRACE(15, ("__kmp_execute_tasks_template(enter): T#%d final_spin=%d " 2909 "*thread_finished=%d\n", 2910 gtid, final_spin, *thread_finished)); 2911 2912 thread->th.th_reap_state = KMP_NOT_SAFE_TO_REAP; 2913 threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data); 2914 KMP_DEBUG_ASSERT(threads_data != NULL); 2915 2916 nthreads = task_team->tt.tt_nproc; 2917 unfinished_threads = &(task_team->tt.tt_unfinished_threads); 2918 #if OMP_45_ENABLED 2919 KMP_DEBUG_ASSERT(nthreads > 1 || task_team->tt.tt_found_proxy_tasks); 2920 #else 2921 KMP_DEBUG_ASSERT(nthreads > 1); 2922 #endif 2923 KMP_DEBUG_ASSERT(*unfinished_threads >= 0); 2924 2925 while (1) { // Outer loop keeps trying to find tasks in case of single thread 2926 // getting tasks from target constructs 2927 while (1) { // Inner loop to find a task and execute it 2928 task = NULL; 2929 if (use_own_tasks) { // check on own queue first 2930 task = __kmp_remove_my_task(thread, gtid, task_team, is_constrained); 2931 } 2932 if ((task == NULL) && (nthreads > 1)) { // Steal a task 2933 int asleep = 1; 2934 use_own_tasks = 0; 2935 // Try to steal from the last place I stole from successfully. 2936 if (victim_tid == -2) { // haven't stolen anything yet 2937 victim_tid = threads_data[tid].td.td_deque_last_stolen; 2938 if (victim_tid != 2939 -1) // if we have a last stolen from victim, get the thread 2940 other_thread = threads_data[victim_tid].td.td_thr; 2941 } 2942 if (victim_tid != -1) { // found last victim 2943 asleep = 0; 2944 } else if (!new_victim) { // no recent steals and we haven't already 2945 // used a new victim; select a random thread 2946 do { // Find a different thread to steal work from. 2947 // Pick a random thread. Initial plan was to cycle through all the 2948 // threads, and only return if we tried to steal from every thread, 2949 // and failed. Arch says that's not such a great idea. 2950 victim_tid = __kmp_get_random(thread) % (nthreads - 1); 2951 if (victim_tid >= tid) { 2952 ++victim_tid; // Adjusts random distribution to exclude self 2953 } 2954 // Found a potential victim 2955 other_thread = threads_data[victim_tid].td.td_thr; 2956 // There is a slight chance that __kmp_enable_tasking() did not wake 2957 // up all threads waiting at the barrier. If victim is sleeping, 2958 // then wake it up. Since we were going to pay the cache miss 2959 // penalty for referencing another thread's kmp_info_t struct 2960 // anyway, 2961 // the check shouldn't cost too much performance at this point. In 2962 // extra barrier mode, tasks do not sleep at the separate tasking 2963 // barrier, so this isn't a problem. 2964 asleep = 0; 2965 if ((__kmp_tasking_mode == tskm_task_teams) && 2966 (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) && 2967 (TCR_PTR(CCAST(void *, other_thread->th.th_sleep_loc)) != 2968 NULL)) { 2969 asleep = 1; 2970 __kmp_null_resume_wrapper(__kmp_gtid_from_thread(other_thread), 2971 other_thread->th.th_sleep_loc); 2972 // A sleeping thread should not have any tasks on it's queue. 2973 // There is a slight possibility that it resumes, steals a task 2974 // from another thread, which spawns more tasks, all in the time 2975 // that it takes this thread to check => don't write an assertion 2976 // that the victim's queue is empty. Try stealing from a 2977 // different thread. 2978 } 2979 } while (asleep); 2980 } 2981 2982 if (!asleep) { 2983 // We have a victim to try to steal from 2984 task = __kmp_steal_task(other_thread, gtid, task_team, 2985 unfinished_threads, thread_finished, 2986 is_constrained); 2987 } 2988 if (task != NULL) { // set last stolen to victim 2989 if (threads_data[tid].td.td_deque_last_stolen != victim_tid) { 2990 threads_data[tid].td.td_deque_last_stolen = victim_tid; 2991 // The pre-refactored code did not try more than 1 successful new 2992 // vicitm, unless the last one generated more local tasks; 2993 // new_victim keeps track of this 2994 new_victim = 1; 2995 } 2996 } else { // No tasks found; unset last_stolen 2997 KMP_CHECK_UPDATE(threads_data[tid].td.td_deque_last_stolen, -1); 2998 victim_tid = -2; // no successful victim found 2999 } 3000 } 3001 3002 if (task == NULL) // break out of tasking loop 3003 break; 3004 3005 // Found a task; execute it 3006 #if USE_ITT_BUILD && USE_ITT_NOTIFY 3007 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) { 3008 if (itt_sync_obj == NULL) { // we are at fork barrier where we could not 3009 // get the object reliably 3010 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier); 3011 } 3012 __kmp_itt_task_starting(itt_sync_obj); 3013 } 3014 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */ 3015 __kmp_invoke_task(gtid, task, current_task); 3016 #if USE_ITT_BUILD 3017 if (itt_sync_obj != NULL) 3018 __kmp_itt_task_finished(itt_sync_obj); 3019 #endif /* USE_ITT_BUILD */ 3020 // If this thread is only partway through the barrier and the condition is 3021 // met, then return now, so that the barrier gather/release pattern can 3022 // proceed. If this thread is in the last spin loop in the barrier, 3023 // waiting to be released, we know that the termination condition will not 3024 // be satisified, so don't waste any cycles checking it. 3025 if (flag == NULL || (!final_spin && flag->done_check())) { 3026 KA_TRACE( 3027 15, 3028 ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n", 3029 gtid)); 3030 return TRUE; 3031 } 3032 if (thread->th.th_task_team == NULL) { 3033 break; 3034 } 3035 KMP_YIELD(__kmp_library == library_throughput); // Yield before next task 3036 // If execution of a stolen task results in more tasks being placed on our 3037 // run queue, reset use_own_tasks 3038 if (!use_own_tasks && TCR_4(threads_data[tid].td.td_deque_ntasks) != 0) { 3039 KA_TRACE(20, ("__kmp_execute_tasks_template: T#%d stolen task spawned " 3040 "other tasks, restart\n", 3041 gtid)); 3042 use_own_tasks = 1; 3043 new_victim = 0; 3044 } 3045 } 3046 3047 // The task source has been exhausted. If in final spin loop of barrier, check 3048 // if termination condition is satisfied. 3049 #if OMP_45_ENABLED 3050 // The work queue may be empty but there might be proxy tasks still 3051 // executing 3052 if (final_spin && 3053 KMP_ATOMIC_LD_ACQ(¤t_task->td_incomplete_child_tasks) == 0) 3054 #else 3055 if (final_spin) 3056 #endif 3057 { 3058 // First, decrement the #unfinished threads, if that has not already been 3059 // done. This decrement might be to the spin location, and result in the 3060 // termination condition being satisfied. 3061 if (!*thread_finished) { 3062 kmp_int32 count; 3063 3064 count = KMP_ATOMIC_DEC(unfinished_threads) - 1; 3065 KA_TRACE(20, ("__kmp_execute_tasks_template: T#%d dec " 3066 "unfinished_threads to %d task_team=%p\n", 3067 gtid, count, task_team)); 3068 *thread_finished = TRUE; 3069 } 3070 3071 // It is now unsafe to reference thread->th.th_team !!! 3072 // Decrementing task_team->tt.tt_unfinished_threads can allow the master 3073 // thread to pass through the barrier, where it might reset each thread's 3074 // th.th_team field for the next parallel region. If we can steal more 3075 // work, we know that this has not happened yet. 3076 if (flag != NULL && flag->done_check()) { 3077 KA_TRACE( 3078 15, 3079 ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n", 3080 gtid)); 3081 return TRUE; 3082 } 3083 } 3084 3085 // If this thread's task team is NULL, master has recognized that there are 3086 // no more tasks; bail out 3087 if (thread->th.th_task_team == NULL) { 3088 KA_TRACE(15, 3089 ("__kmp_execute_tasks_template: T#%d no more tasks\n", gtid)); 3090 return FALSE; 3091 } 3092 3093 #if OMP_45_ENABLED 3094 // We could be getting tasks from target constructs; if this is the only 3095 // thread, keep trying to execute tasks from own queue 3096 if (nthreads == 1) 3097 use_own_tasks = 1; 3098 else 3099 #endif 3100 { 3101 KA_TRACE(15, 3102 ("__kmp_execute_tasks_template: T#%d can't find work\n", gtid)); 3103 return FALSE; 3104 } 3105 } 3106 } 3107 3108 int __kmp_execute_tasks_32( 3109 kmp_info_t *thread, kmp_int32 gtid, kmp_flag_32 *flag, int final_spin, 3110 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 3111 kmp_int32 is_constrained) { 3112 return __kmp_execute_tasks_template( 3113 thread, gtid, flag, final_spin, 3114 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained); 3115 } 3116 3117 int __kmp_execute_tasks_64( 3118 kmp_info_t *thread, kmp_int32 gtid, kmp_flag_64 *flag, int final_spin, 3119 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 3120 kmp_int32 is_constrained) { 3121 return __kmp_execute_tasks_template( 3122 thread, gtid, flag, final_spin, 3123 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained); 3124 } 3125 3126 int __kmp_execute_tasks_oncore( 3127 kmp_info_t *thread, kmp_int32 gtid, kmp_flag_oncore *flag, int final_spin, 3128 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj), 3129 kmp_int32 is_constrained) { 3130 return __kmp_execute_tasks_template( 3131 thread, gtid, flag, final_spin, 3132 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained); 3133 } 3134 3135 // __kmp_enable_tasking: Allocate task team and resume threads sleeping at the 3136 // next barrier so they can assist in executing enqueued tasks. 3137 // First thread in allocates the task team atomically. 3138 static void __kmp_enable_tasking(kmp_task_team_t *task_team, 3139 kmp_info_t *this_thr) { 3140 kmp_thread_data_t *threads_data; 3141 int nthreads, i, is_init_thread; 3142 3143 KA_TRACE(10, ("__kmp_enable_tasking(enter): T#%d\n", 3144 __kmp_gtid_from_thread(this_thr))); 3145 3146 KMP_DEBUG_ASSERT(task_team != NULL); 3147 KMP_DEBUG_ASSERT(this_thr->th.th_team != NULL); 3148 3149 nthreads = task_team->tt.tt_nproc; 3150 KMP_DEBUG_ASSERT(nthreads > 0); 3151 KMP_DEBUG_ASSERT(nthreads == this_thr->th.th_team->t.t_nproc); 3152 3153 // Allocate or increase the size of threads_data if necessary 3154 is_init_thread = __kmp_realloc_task_threads_data(this_thr, task_team); 3155 3156 if (!is_init_thread) { 3157 // Some other thread already set up the array. 3158 KA_TRACE( 3159 20, 3160 ("__kmp_enable_tasking(exit): T#%d: threads array already set up.\n", 3161 __kmp_gtid_from_thread(this_thr))); 3162 return; 3163 } 3164 threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data); 3165 KMP_DEBUG_ASSERT(threads_data != NULL); 3166 3167 if (__kmp_tasking_mode == tskm_task_teams && 3168 (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME)) { 3169 // Release any threads sleeping at the barrier, so that they can steal 3170 // tasks and execute them. In extra barrier mode, tasks do not sleep 3171 // at the separate tasking barrier, so this isn't a problem. 3172 for (i = 0; i < nthreads; i++) { 3173 volatile void *sleep_loc; 3174 kmp_info_t *thread = threads_data[i].td.td_thr; 3175 3176 if (i == this_thr->th.th_info.ds.ds_tid) { 3177 continue; 3178 } 3179 // Since we haven't locked the thread's suspend mutex lock at this 3180 // point, there is a small window where a thread might be putting 3181 // itself to sleep, but hasn't set the th_sleep_loc field yet. 3182 // To work around this, __kmp_execute_tasks_template() periodically checks 3183 // see if other threads are sleeping (using the same random mechanism that 3184 // is used for task stealing) and awakens them if they are. 3185 if ((sleep_loc = TCR_PTR(CCAST(void *, thread->th.th_sleep_loc))) != 3186 NULL) { 3187 KF_TRACE(50, ("__kmp_enable_tasking: T#%d waking up thread T#%d\n", 3188 __kmp_gtid_from_thread(this_thr), 3189 __kmp_gtid_from_thread(thread))); 3190 __kmp_null_resume_wrapper(__kmp_gtid_from_thread(thread), sleep_loc); 3191 } else { 3192 KF_TRACE(50, ("__kmp_enable_tasking: T#%d don't wake up thread T#%d\n", 3193 __kmp_gtid_from_thread(this_thr), 3194 __kmp_gtid_from_thread(thread))); 3195 } 3196 } 3197 } 3198 3199 KA_TRACE(10, ("__kmp_enable_tasking(exit): T#%d\n", 3200 __kmp_gtid_from_thread(this_thr))); 3201 } 3202 3203 /* // TODO: Check the comment consistency 3204 * Utility routines for "task teams". A task team (kmp_task_t) is kind of 3205 * like a shadow of the kmp_team_t data struct, with a different lifetime. 3206 * After a child * thread checks into a barrier and calls __kmp_release() from 3207 * the particular variant of __kmp_<barrier_kind>_barrier_gather(), it can no 3208 * longer assume that the kmp_team_t structure is intact (at any moment, the 3209 * master thread may exit the barrier code and free the team data structure, 3210 * and return the threads to the thread pool). 3211 * 3212 * This does not work with the the tasking code, as the thread is still 3213 * expected to participate in the execution of any tasks that may have been 3214 * spawned my a member of the team, and the thread still needs access to all 3215 * to each thread in the team, so that it can steal work from it. 3216 * 3217 * Enter the existence of the kmp_task_team_t struct. It employs a reference 3218 * counting mechanims, and is allocated by the master thread before calling 3219 * __kmp_<barrier_kind>_release, and then is release by the last thread to 3220 * exit __kmp_<barrier_kind>_release at the next barrier. I.e. the lifetimes 3221 * of the kmp_task_team_t structs for consecutive barriers can overlap 3222 * (and will, unless the master thread is the last thread to exit the barrier 3223 * release phase, which is not typical). 3224 * 3225 * The existence of such a struct is useful outside the context of tasking, 3226 * but for now, I'm trying to keep it specific to the OMP_30_ENABLED macro, 3227 * so that any performance differences show up when comparing the 2.5 vs. 3.0 3228 * libraries. 3229 * 3230 * We currently use the existence of the threads array as an indicator that 3231 * tasks were spawned since the last barrier. If the structure is to be 3232 * useful outside the context of tasking, then this will have to change, but 3233 * not settting the field minimizes the performance impact of tasking on 3234 * barriers, when no explicit tasks were spawned (pushed, actually). 3235 */ 3236 3237 static kmp_task_team_t *__kmp_free_task_teams = 3238 NULL; // Free list for task_team data structures 3239 // Lock for task team data structures 3240 kmp_bootstrap_lock_t __kmp_task_team_lock = 3241 KMP_BOOTSTRAP_LOCK_INITIALIZER(__kmp_task_team_lock); 3242 3243 // __kmp_alloc_task_deque: 3244 // Allocates a task deque for a particular thread, and initialize the necessary 3245 // data structures relating to the deque. This only happens once per thread 3246 // per task team since task teams are recycled. No lock is needed during 3247 // allocation since each thread allocates its own deque. 3248 static void __kmp_alloc_task_deque(kmp_info_t *thread, 3249 kmp_thread_data_t *thread_data) { 3250 __kmp_init_bootstrap_lock(&thread_data->td.td_deque_lock); 3251 KMP_DEBUG_ASSERT(thread_data->td.td_deque == NULL); 3252 3253 // Initialize last stolen task field to "none" 3254 thread_data->td.td_deque_last_stolen = -1; 3255 3256 KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) == 0); 3257 KMP_DEBUG_ASSERT(thread_data->td.td_deque_head == 0); 3258 KMP_DEBUG_ASSERT(thread_data->td.td_deque_tail == 0); 3259 3260 KE_TRACE( 3261 10, 3262 ("__kmp_alloc_task_deque: T#%d allocating deque[%d] for thread_data %p\n", 3263 __kmp_gtid_from_thread(thread), INITIAL_TASK_DEQUE_SIZE, thread_data)); 3264 // Allocate space for task deque, and zero the deque 3265 // Cannot use __kmp_thread_calloc() because threads not around for 3266 // kmp_reap_task_team( ). 3267 thread_data->td.td_deque = (kmp_taskdata_t **)__kmp_allocate( 3268 INITIAL_TASK_DEQUE_SIZE * sizeof(kmp_taskdata_t *)); 3269 thread_data->td.td_deque_size = INITIAL_TASK_DEQUE_SIZE; 3270 } 3271 3272 // __kmp_free_task_deque: 3273 // Deallocates a task deque for a particular thread. Happens at library 3274 // deallocation so don't need to reset all thread data fields. 3275 static void __kmp_free_task_deque(kmp_thread_data_t *thread_data) { 3276 if (thread_data->td.td_deque != NULL) { 3277 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 3278 TCW_4(thread_data->td.td_deque_ntasks, 0); 3279 __kmp_free(thread_data->td.td_deque); 3280 thread_data->td.td_deque = NULL; 3281 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 3282 } 3283 3284 #ifdef BUILD_TIED_TASK_STACK 3285 // GEH: Figure out what to do here for td_susp_tied_tasks 3286 if (thread_data->td.td_susp_tied_tasks.ts_entries != TASK_STACK_EMPTY) { 3287 __kmp_free_task_stack(__kmp_thread_from_gtid(gtid), thread_data); 3288 } 3289 #endif // BUILD_TIED_TASK_STACK 3290 } 3291 3292 // __kmp_realloc_task_threads_data: 3293 // Allocates a threads_data array for a task team, either by allocating an 3294 // initial array or enlarging an existing array. Only the first thread to get 3295 // the lock allocs or enlarges the array and re-initializes the array eleemnts. 3296 // That thread returns "TRUE", the rest return "FALSE". 3297 // Assumes that the new array size is given by task_team -> tt.tt_nproc. 3298 // The current size is given by task_team -> tt.tt_max_threads. 3299 static int __kmp_realloc_task_threads_data(kmp_info_t *thread, 3300 kmp_task_team_t *task_team) { 3301 kmp_thread_data_t **threads_data_p; 3302 kmp_int32 nthreads, maxthreads; 3303 int is_init_thread = FALSE; 3304 3305 if (TCR_4(task_team->tt.tt_found_tasks)) { 3306 // Already reallocated and initialized. 3307 return FALSE; 3308 } 3309 3310 threads_data_p = &task_team->tt.tt_threads_data; 3311 nthreads = task_team->tt.tt_nproc; 3312 maxthreads = task_team->tt.tt_max_threads; 3313 3314 // All threads must lock when they encounter the first task of the implicit 3315 // task region to make sure threads_data fields are (re)initialized before 3316 // used. 3317 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_threads_lock); 3318 3319 if (!TCR_4(task_team->tt.tt_found_tasks)) { 3320 // first thread to enable tasking 3321 kmp_team_t *team = thread->th.th_team; 3322 int i; 3323 3324 is_init_thread = TRUE; 3325 if (maxthreads < nthreads) { 3326 3327 if (*threads_data_p != NULL) { 3328 kmp_thread_data_t *old_data = *threads_data_p; 3329 kmp_thread_data_t *new_data = NULL; 3330 3331 KE_TRACE( 3332 10, 3333 ("__kmp_realloc_task_threads_data: T#%d reallocating " 3334 "threads data for task_team %p, new_size = %d, old_size = %d\n", 3335 __kmp_gtid_from_thread(thread), task_team, nthreads, maxthreads)); 3336 // Reallocate threads_data to have more elements than current array 3337 // Cannot use __kmp_thread_realloc() because threads not around for 3338 // kmp_reap_task_team( ). Note all new array entries are initialized 3339 // to zero by __kmp_allocate(). 3340 new_data = (kmp_thread_data_t *)__kmp_allocate( 3341 nthreads * sizeof(kmp_thread_data_t)); 3342 // copy old data to new data 3343 KMP_MEMCPY_S((void *)new_data, nthreads * sizeof(kmp_thread_data_t), 3344 (void *)old_data, maxthreads * sizeof(kmp_thread_data_t)); 3345 3346 #ifdef BUILD_TIED_TASK_STACK 3347 // GEH: Figure out if this is the right thing to do 3348 for (i = maxthreads; i < nthreads; i++) { 3349 kmp_thread_data_t *thread_data = &(*threads_data_p)[i]; 3350 __kmp_init_task_stack(__kmp_gtid_from_thread(thread), thread_data); 3351 } 3352 #endif // BUILD_TIED_TASK_STACK 3353 // Install the new data and free the old data 3354 (*threads_data_p) = new_data; 3355 __kmp_free(old_data); 3356 } else { 3357 KE_TRACE(10, ("__kmp_realloc_task_threads_data: T#%d allocating " 3358 "threads data for task_team %p, size = %d\n", 3359 __kmp_gtid_from_thread(thread), task_team, nthreads)); 3360 // Make the initial allocate for threads_data array, and zero entries 3361 // Cannot use __kmp_thread_calloc() because threads not around for 3362 // kmp_reap_task_team( ). 3363 ANNOTATE_IGNORE_WRITES_BEGIN(); 3364 *threads_data_p = (kmp_thread_data_t *)__kmp_allocate( 3365 nthreads * sizeof(kmp_thread_data_t)); 3366 ANNOTATE_IGNORE_WRITES_END(); 3367 #ifdef BUILD_TIED_TASK_STACK 3368 // GEH: Figure out if this is the right thing to do 3369 for (i = 0; i < nthreads; i++) { 3370 kmp_thread_data_t *thread_data = &(*threads_data_p)[i]; 3371 __kmp_init_task_stack(__kmp_gtid_from_thread(thread), thread_data); 3372 } 3373 #endif // BUILD_TIED_TASK_STACK 3374 } 3375 task_team->tt.tt_max_threads = nthreads; 3376 } else { 3377 // If array has (more than) enough elements, go ahead and use it 3378 KMP_DEBUG_ASSERT(*threads_data_p != NULL); 3379 } 3380 3381 // initialize threads_data pointers back to thread_info structures 3382 for (i = 0; i < nthreads; i++) { 3383 kmp_thread_data_t *thread_data = &(*threads_data_p)[i]; 3384 thread_data->td.td_thr = team->t.t_threads[i]; 3385 3386 if (thread_data->td.td_deque_last_stolen >= nthreads) { 3387 // The last stolen field survives across teams / barrier, and the number 3388 // of threads may have changed. It's possible (likely?) that a new 3389 // parallel region will exhibit the same behavior as previous region. 3390 thread_data->td.td_deque_last_stolen = -1; 3391 } 3392 } 3393 3394 KMP_MB(); 3395 TCW_SYNC_4(task_team->tt.tt_found_tasks, TRUE); 3396 } 3397 3398 __kmp_release_bootstrap_lock(&task_team->tt.tt_threads_lock); 3399 return is_init_thread; 3400 } 3401 3402 // __kmp_free_task_threads_data: 3403 // Deallocates a threads_data array for a task team, including any attached 3404 // tasking deques. Only occurs at library shutdown. 3405 static void __kmp_free_task_threads_data(kmp_task_team_t *task_team) { 3406 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_threads_lock); 3407 if (task_team->tt.tt_threads_data != NULL) { 3408 int i; 3409 for (i = 0; i < task_team->tt.tt_max_threads; i++) { 3410 __kmp_free_task_deque(&task_team->tt.tt_threads_data[i]); 3411 } 3412 __kmp_free(task_team->tt.tt_threads_data); 3413 task_team->tt.tt_threads_data = NULL; 3414 } 3415 __kmp_release_bootstrap_lock(&task_team->tt.tt_threads_lock); 3416 } 3417 3418 // __kmp_allocate_task_team: 3419 // Allocates a task team associated with a specific team, taking it from 3420 // the global task team free list if possible. Also initializes data 3421 // structures. 3422 static kmp_task_team_t *__kmp_allocate_task_team(kmp_info_t *thread, 3423 kmp_team_t *team) { 3424 kmp_task_team_t *task_team = NULL; 3425 int nthreads; 3426 3427 KA_TRACE(20, ("__kmp_allocate_task_team: T#%d entering; team = %p\n", 3428 (thread ? __kmp_gtid_from_thread(thread) : -1), team)); 3429 3430 if (TCR_PTR(__kmp_free_task_teams) != NULL) { 3431 // Take a task team from the task team pool 3432 __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock); 3433 if (__kmp_free_task_teams != NULL) { 3434 task_team = __kmp_free_task_teams; 3435 TCW_PTR(__kmp_free_task_teams, task_team->tt.tt_next); 3436 task_team->tt.tt_next = NULL; 3437 } 3438 __kmp_release_bootstrap_lock(&__kmp_task_team_lock); 3439 } 3440 3441 if (task_team == NULL) { 3442 KE_TRACE(10, ("__kmp_allocate_task_team: T#%d allocating " 3443 "task team for team %p\n", 3444 __kmp_gtid_from_thread(thread), team)); 3445 // Allocate a new task team if one is not available. 3446 // Cannot use __kmp_thread_malloc() because threads not around for 3447 // kmp_reap_task_team( ). 3448 task_team = (kmp_task_team_t *)__kmp_allocate(sizeof(kmp_task_team_t)); 3449 __kmp_init_bootstrap_lock(&task_team->tt.tt_threads_lock); 3450 // AC: __kmp_allocate zeroes returned memory 3451 // task_team -> tt.tt_threads_data = NULL; 3452 // task_team -> tt.tt_max_threads = 0; 3453 // task_team -> tt.tt_next = NULL; 3454 } 3455 3456 TCW_4(task_team->tt.tt_found_tasks, FALSE); 3457 #if OMP_45_ENABLED 3458 TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE); 3459 #endif 3460 task_team->tt.tt_nproc = nthreads = team->t.t_nproc; 3461 3462 KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads, nthreads); 3463 TCW_4(task_team->tt.tt_active, TRUE); 3464 3465 KA_TRACE(20, ("__kmp_allocate_task_team: T#%d exiting; task_team = %p " 3466 "unfinished_threads init'd to %d\n", 3467 (thread ? __kmp_gtid_from_thread(thread) : -1), task_team, 3468 KMP_ATOMIC_LD_RLX(&task_team->tt.tt_unfinished_threads))); 3469 return task_team; 3470 } 3471 3472 // __kmp_free_task_team: 3473 // Frees the task team associated with a specific thread, and adds it 3474 // to the global task team free list. 3475 void __kmp_free_task_team(kmp_info_t *thread, kmp_task_team_t *task_team) { 3476 KA_TRACE(20, ("__kmp_free_task_team: T#%d task_team = %p\n", 3477 thread ? __kmp_gtid_from_thread(thread) : -1, task_team)); 3478 3479 // Put task team back on free list 3480 __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock); 3481 3482 KMP_DEBUG_ASSERT(task_team->tt.tt_next == NULL); 3483 task_team->tt.tt_next = __kmp_free_task_teams; 3484 TCW_PTR(__kmp_free_task_teams, task_team); 3485 3486 __kmp_release_bootstrap_lock(&__kmp_task_team_lock); 3487 } 3488 3489 // __kmp_reap_task_teams: 3490 // Free all the task teams on the task team free list. 3491 // Should only be done during library shutdown. 3492 // Cannot do anything that needs a thread structure or gtid since they are 3493 // already gone. 3494 void __kmp_reap_task_teams(void) { 3495 kmp_task_team_t *task_team; 3496 3497 if (TCR_PTR(__kmp_free_task_teams) != NULL) { 3498 // Free all task_teams on the free list 3499 __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock); 3500 while ((task_team = __kmp_free_task_teams) != NULL) { 3501 __kmp_free_task_teams = task_team->tt.tt_next; 3502 task_team->tt.tt_next = NULL; 3503 3504 // Free threads_data if necessary 3505 if (task_team->tt.tt_threads_data != NULL) { 3506 __kmp_free_task_threads_data(task_team); 3507 } 3508 __kmp_free(task_team); 3509 } 3510 __kmp_release_bootstrap_lock(&__kmp_task_team_lock); 3511 } 3512 } 3513 3514 // __kmp_wait_to_unref_task_teams: 3515 // Some threads could still be in the fork barrier release code, possibly 3516 // trying to steal tasks. Wait for each thread to unreference its task team. 3517 void __kmp_wait_to_unref_task_teams(void) { 3518 kmp_info_t *thread; 3519 kmp_uint32 spins; 3520 int done; 3521 3522 KMP_INIT_YIELD(spins); 3523 3524 for (;;) { 3525 done = TRUE; 3526 3527 // TODO: GEH - this may be is wrong because some sync would be necessary 3528 // in case threads are added to the pool during the traversal. Need to 3529 // verify that lock for thread pool is held when calling this routine. 3530 for (thread = CCAST(kmp_info_t *, __kmp_thread_pool); thread != NULL; 3531 thread = thread->th.th_next_pool) { 3532 #if KMP_OS_WINDOWS 3533 DWORD exit_val; 3534 #endif 3535 if (TCR_PTR(thread->th.th_task_team) == NULL) { 3536 KA_TRACE(10, ("__kmp_wait_to_unref_task_team: T#%d task_team == NULL\n", 3537 __kmp_gtid_from_thread(thread))); 3538 continue; 3539 } 3540 #if KMP_OS_WINDOWS 3541 // TODO: GEH - add this check for Linux* OS / OS X* as well? 3542 if (!__kmp_is_thread_alive(thread, &exit_val)) { 3543 thread->th.th_task_team = NULL; 3544 continue; 3545 } 3546 #endif 3547 3548 done = FALSE; // Because th_task_team pointer is not NULL for this thread 3549 3550 KA_TRACE(10, ("__kmp_wait_to_unref_task_team: Waiting for T#%d to " 3551 "unreference task_team\n", 3552 __kmp_gtid_from_thread(thread))); 3553 3554 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) { 3555 volatile void *sleep_loc; 3556 // If the thread is sleeping, awaken it. 3557 if ((sleep_loc = TCR_PTR(CCAST(void *, thread->th.th_sleep_loc))) != 3558 NULL) { 3559 KA_TRACE( 3560 10, 3561 ("__kmp_wait_to_unref_task_team: T#%d waking up thread T#%d\n", 3562 __kmp_gtid_from_thread(thread), __kmp_gtid_from_thread(thread))); 3563 __kmp_null_resume_wrapper(__kmp_gtid_from_thread(thread), sleep_loc); 3564 } 3565 } 3566 } 3567 if (done) { 3568 break; 3569 } 3570 3571 // If oversubscribed or have waited a bit, yield. 3572 KMP_YIELD_OVERSUB_ELSE_SPIN(spins); 3573 } 3574 } 3575 3576 // __kmp_task_team_setup: Create a task_team for the current team, but use 3577 // an already created, unused one if it already exists. 3578 void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team, int always) { 3579 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 3580 3581 // If this task_team hasn't been created yet, allocate it. It will be used in 3582 // the region after the next. 3583 // If it exists, it is the current task team and shouldn't be touched yet as 3584 // it may still be in use. 3585 if (team->t.t_task_team[this_thr->th.th_task_state] == NULL && 3586 (always || team->t.t_nproc > 1)) { 3587 team->t.t_task_team[this_thr->th.th_task_state] = 3588 __kmp_allocate_task_team(this_thr, team); 3589 KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d created new task_team %p " 3590 "for team %d at parity=%d\n", 3591 __kmp_gtid_from_thread(this_thr), 3592 team->t.t_task_team[this_thr->th.th_task_state], 3593 ((team != NULL) ? team->t.t_id : -1), 3594 this_thr->th.th_task_state)); 3595 } 3596 3597 // After threads exit the release, they will call sync, and then point to this 3598 // other task_team; make sure it is allocated and properly initialized. As 3599 // threads spin in the barrier release phase, they will continue to use the 3600 // previous task_team struct(above), until they receive the signal to stop 3601 // checking for tasks (they can't safely reference the kmp_team_t struct, 3602 // which could be reallocated by the master thread). No task teams are formed 3603 // for serialized teams. 3604 if (team->t.t_nproc > 1) { 3605 int other_team = 1 - this_thr->th.th_task_state; 3606 if (team->t.t_task_team[other_team] == NULL) { // setup other team as well 3607 team->t.t_task_team[other_team] = 3608 __kmp_allocate_task_team(this_thr, team); 3609 KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d created second new " 3610 "task_team %p for team %d at parity=%d\n", 3611 __kmp_gtid_from_thread(this_thr), 3612 team->t.t_task_team[other_team], 3613 ((team != NULL) ? team->t.t_id : -1), other_team)); 3614 } else { // Leave the old task team struct in place for the upcoming region; 3615 // adjust as needed 3616 kmp_task_team_t *task_team = team->t.t_task_team[other_team]; 3617 if (!task_team->tt.tt_active || 3618 team->t.t_nproc != task_team->tt.tt_nproc) { 3619 TCW_4(task_team->tt.tt_nproc, team->t.t_nproc); 3620 TCW_4(task_team->tt.tt_found_tasks, FALSE); 3621 #if OMP_45_ENABLED 3622 TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE); 3623 #endif 3624 KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads, 3625 team->t.t_nproc); 3626 TCW_4(task_team->tt.tt_active, TRUE); 3627 } 3628 // if team size has changed, the first thread to enable tasking will 3629 // realloc threads_data if necessary 3630 KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d reset next task_team " 3631 "%p for team %d at parity=%d\n", 3632 __kmp_gtid_from_thread(this_thr), 3633 team->t.t_task_team[other_team], 3634 ((team != NULL) ? team->t.t_id : -1), other_team)); 3635 } 3636 } 3637 } 3638 3639 // __kmp_task_team_sync: Propagation of task team data from team to threads 3640 // which happens just after the release phase of a team barrier. This may be 3641 // called by any thread, but only for teams with # threads > 1. 3642 void __kmp_task_team_sync(kmp_info_t *this_thr, kmp_team_t *team) { 3643 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 3644 3645 // Toggle the th_task_state field, to switch which task_team this thread 3646 // refers to 3647 this_thr->th.th_task_state = 1 - this_thr->th.th_task_state; 3648 // It is now safe to propagate the task team pointer from the team struct to 3649 // the current thread. 3650 TCW_PTR(this_thr->th.th_task_team, 3651 team->t.t_task_team[this_thr->th.th_task_state]); 3652 KA_TRACE(20, 3653 ("__kmp_task_team_sync: Thread T#%d task team switched to task_team " 3654 "%p from Team #%d (parity=%d)\n", 3655 __kmp_gtid_from_thread(this_thr), this_thr->th.th_task_team, 3656 ((team != NULL) ? team->t.t_id : -1), this_thr->th.th_task_state)); 3657 } 3658 3659 // __kmp_task_team_wait: Master thread waits for outstanding tasks after the 3660 // barrier gather phase. Only called by master thread if #threads in team > 1 or 3661 // if proxy tasks were created. 3662 // 3663 // wait is a flag that defaults to 1 (see kmp.h), but waiting can be turned off 3664 // by passing in 0 optionally as the last argument. When wait is zero, master 3665 // thread does not wait for unfinished_threads to reach 0. 3666 void __kmp_task_team_wait( 3667 kmp_info_t *this_thr, 3668 kmp_team_t *team USE_ITT_BUILD_ARG(void *itt_sync_obj), int wait) { 3669 kmp_task_team_t *task_team = team->t.t_task_team[this_thr->th.th_task_state]; 3670 3671 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec); 3672 KMP_DEBUG_ASSERT(task_team == this_thr->th.th_task_team); 3673 3674 if ((task_team != NULL) && KMP_TASKING_ENABLED(task_team)) { 3675 if (wait) { 3676 KA_TRACE(20, ("__kmp_task_team_wait: Master T#%d waiting for all tasks " 3677 "(for unfinished_threads to reach 0) on task_team = %p\n", 3678 __kmp_gtid_from_thread(this_thr), task_team)); 3679 // Worker threads may have dropped through to release phase, but could 3680 // still be executing tasks. Wait here for tasks to complete. To avoid 3681 // memory contention, only master thread checks termination condition. 3682 kmp_flag_32 flag(RCAST(std::atomic<kmp_uint32> *, 3683 &task_team->tt.tt_unfinished_threads), 3684 0U); 3685 flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj)); 3686 } 3687 // Deactivate the old task team, so that the worker threads will stop 3688 // referencing it while spinning. 3689 KA_TRACE( 3690 20, 3691 ("__kmp_task_team_wait: Master T#%d deactivating task_team %p: " 3692 "setting active to false, setting local and team's pointer to NULL\n", 3693 __kmp_gtid_from_thread(this_thr), task_team)); 3694 #if OMP_45_ENABLED 3695 KMP_DEBUG_ASSERT(task_team->tt.tt_nproc > 1 || 3696 task_team->tt.tt_found_proxy_tasks == TRUE); 3697 TCW_SYNC_4(task_team->tt.tt_found_proxy_tasks, FALSE); 3698 #else 3699 KMP_DEBUG_ASSERT(task_team->tt.tt_nproc > 1); 3700 #endif 3701 KMP_CHECK_UPDATE(task_team->tt.tt_untied_task_encountered, 0); 3702 TCW_SYNC_4(task_team->tt.tt_active, FALSE); 3703 KMP_MB(); 3704 3705 TCW_PTR(this_thr->th.th_task_team, NULL); 3706 } 3707 } 3708 3709 // __kmp_tasking_barrier: 3710 // This routine may only called when __kmp_tasking_mode == tskm_extra_barrier. 3711 // Internal function to execute all tasks prior to a regular barrier or a join 3712 // barrier. It is a full barrier itself, which unfortunately turns regular 3713 // barriers into double barriers and join barriers into 1 1/2 barriers. 3714 void __kmp_tasking_barrier(kmp_team_t *team, kmp_info_t *thread, int gtid) { 3715 std::atomic<kmp_uint32> *spin = RCAST( 3716 std::atomic<kmp_uint32> *, 3717 &team->t.t_task_team[thread->th.th_task_state]->tt.tt_unfinished_threads); 3718 int flag = FALSE; 3719 KMP_DEBUG_ASSERT(__kmp_tasking_mode == tskm_extra_barrier); 3720 3721 #if USE_ITT_BUILD 3722 KMP_FSYNC_SPIN_INIT(spin, NULL); 3723 #endif /* USE_ITT_BUILD */ 3724 kmp_flag_32 spin_flag(spin, 0U); 3725 while (!spin_flag.execute_tasks(thread, gtid, TRUE, 3726 &flag USE_ITT_BUILD_ARG(NULL), 0)) { 3727 #if USE_ITT_BUILD 3728 // TODO: What about itt_sync_obj?? 3729 KMP_FSYNC_SPIN_PREPARE(RCAST(void *, spin)); 3730 #endif /* USE_ITT_BUILD */ 3731 3732 if (TCR_4(__kmp_global.g.g_done)) { 3733 if (__kmp_global.g.g_abort) 3734 __kmp_abort_thread(); 3735 break; 3736 } 3737 KMP_YIELD(TRUE); 3738 } 3739 #if USE_ITT_BUILD 3740 KMP_FSYNC_SPIN_ACQUIRED(RCAST(void *, spin)); 3741 #endif /* USE_ITT_BUILD */ 3742 } 3743 3744 #if OMP_45_ENABLED 3745 3746 // __kmp_give_task puts a task into a given thread queue if: 3747 // - the queue for that thread was created 3748 // - there's space in that queue 3749 // Because of this, __kmp_push_task needs to check if there's space after 3750 // getting the lock 3751 static bool __kmp_give_task(kmp_info_t *thread, kmp_int32 tid, kmp_task_t *task, 3752 kmp_int32 pass) { 3753 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 3754 kmp_task_team_t *task_team = taskdata->td_task_team; 3755 3756 KA_TRACE(20, ("__kmp_give_task: trying to give task %p to thread %d.\n", 3757 taskdata, tid)); 3758 3759 // If task_team is NULL something went really bad... 3760 KMP_DEBUG_ASSERT(task_team != NULL); 3761 3762 bool result = false; 3763 kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid]; 3764 3765 if (thread_data->td.td_deque == NULL) { 3766 // There's no queue in this thread, go find another one 3767 // We're guaranteed that at least one thread has a queue 3768 KA_TRACE(30, 3769 ("__kmp_give_task: thread %d has no queue while giving task %p.\n", 3770 tid, taskdata)); 3771 return result; 3772 } 3773 3774 if (TCR_4(thread_data->td.td_deque_ntasks) >= 3775 TASK_DEQUE_SIZE(thread_data->td)) { 3776 KA_TRACE( 3777 30, 3778 ("__kmp_give_task: queue is full while giving task %p to thread %d.\n", 3779 taskdata, tid)); 3780 3781 // if this deque is bigger than the pass ratio give a chance to another 3782 // thread 3783 if (TASK_DEQUE_SIZE(thread_data->td) / INITIAL_TASK_DEQUE_SIZE >= pass) 3784 return result; 3785 3786 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 3787 __kmp_realloc_task_deque(thread, thread_data); 3788 3789 } else { 3790 3791 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock); 3792 3793 if (TCR_4(thread_data->td.td_deque_ntasks) >= 3794 TASK_DEQUE_SIZE(thread_data->td)) { 3795 KA_TRACE(30, ("__kmp_give_task: queue is full while giving task %p to " 3796 "thread %d.\n", 3797 taskdata, tid)); 3798 3799 // if this deque is bigger than the pass ratio give a chance to another 3800 // thread 3801 if (TASK_DEQUE_SIZE(thread_data->td) / INITIAL_TASK_DEQUE_SIZE >= pass) 3802 goto release_and_exit; 3803 3804 __kmp_realloc_task_deque(thread, thread_data); 3805 } 3806 } 3807 3808 // lock is held here, and there is space in the deque 3809 3810 thread_data->td.td_deque[thread_data->td.td_deque_tail] = taskdata; 3811 // Wrap index. 3812 thread_data->td.td_deque_tail = 3813 (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td); 3814 TCW_4(thread_data->td.td_deque_ntasks, 3815 TCR_4(thread_data->td.td_deque_ntasks) + 1); 3816 3817 result = true; 3818 KA_TRACE(30, ("__kmp_give_task: successfully gave task %p to thread %d.\n", 3819 taskdata, tid)); 3820 3821 release_and_exit: 3822 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock); 3823 3824 return result; 3825 } 3826 3827 /* The finish of the proxy tasks is divided in two pieces: 3828 - the top half is the one that can be done from a thread outside the team 3829 - the bottom half must be run from a thread within the team 3830 3831 In order to run the bottom half the task gets queued back into one of the 3832 threads of the team. Once the td_incomplete_child_task counter of the parent 3833 is decremented the threads can leave the barriers. So, the bottom half needs 3834 to be queued before the counter is decremented. The top half is therefore 3835 divided in two parts: 3836 - things that can be run before queuing the bottom half 3837 - things that must be run after queuing the bottom half 3838 3839 This creates a second race as the bottom half can free the task before the 3840 second top half is executed. To avoid this we use the 3841 td_incomplete_child_task of the proxy task to synchronize the top and bottom 3842 half. */ 3843 static void __kmp_first_top_half_finish_proxy(kmp_taskdata_t *taskdata) { 3844 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT); 3845 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); 3846 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0); 3847 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0); 3848 3849 taskdata->td_flags.complete = 1; // mark the task as completed 3850 3851 if (taskdata->td_taskgroup) 3852 KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count); 3853 3854 // Create an imaginary children for this task so the bottom half cannot 3855 // release the task before we have completed the second top half 3856 KMP_ATOMIC_INC(&taskdata->td_incomplete_child_tasks); 3857 } 3858 3859 static void __kmp_second_top_half_finish_proxy(kmp_taskdata_t *taskdata) { 3860 kmp_int32 children = 0; 3861 3862 // Predecrement simulated by "- 1" calculation 3863 children = 3864 KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks) - 1; 3865 KMP_DEBUG_ASSERT(children >= 0); 3866 3867 // Remove the imaginary children 3868 KMP_ATOMIC_DEC(&taskdata->td_incomplete_child_tasks); 3869 } 3870 3871 static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask) { 3872 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); 3873 kmp_info_t *thread = __kmp_threads[gtid]; 3874 3875 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); 3876 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 3877 1); // top half must run before bottom half 3878 3879 // We need to wait to make sure the top half is finished 3880 // Spinning here should be ok as this should happen quickly 3881 while (KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) > 0) 3882 ; 3883 3884 __kmp_release_deps(gtid, taskdata); 3885 __kmp_free_task_and_ancestors(gtid, taskdata, thread); 3886 } 3887 3888 /*! 3889 @ingroup TASKING 3890 @param gtid Global Thread ID of encountering thread 3891 @param ptask Task which execution is completed 3892 3893 Execute the completation of a proxy task from a thread of that is part of the 3894 team. Run first and bottom halves directly. 3895 */ 3896 void __kmpc_proxy_task_completed(kmp_int32 gtid, kmp_task_t *ptask) { 3897 KMP_DEBUG_ASSERT(ptask != NULL); 3898 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); 3899 KA_TRACE( 3900 10, ("__kmp_proxy_task_completed(enter): T#%d proxy task %p completing\n", 3901 gtid, taskdata)); 3902 3903 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); 3904 3905 __kmp_first_top_half_finish_proxy(taskdata); 3906 __kmp_second_top_half_finish_proxy(taskdata); 3907 __kmp_bottom_half_finish_proxy(gtid, ptask); 3908 3909 KA_TRACE(10, 3910 ("__kmp_proxy_task_completed(exit): T#%d proxy task %p completing\n", 3911 gtid, taskdata)); 3912 } 3913 3914 /*! 3915 @ingroup TASKING 3916 @param ptask Task which execution is completed 3917 3918 Execute the completation of a proxy task from a thread that could not belong to 3919 the team. 3920 */ 3921 void __kmpc_proxy_task_completed_ooo(kmp_task_t *ptask) { 3922 KMP_DEBUG_ASSERT(ptask != NULL); 3923 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); 3924 3925 KA_TRACE( 3926 10, 3927 ("__kmp_proxy_task_completed_ooo(enter): proxy task completing ooo %p\n", 3928 taskdata)); 3929 3930 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY); 3931 3932 __kmp_first_top_half_finish_proxy(taskdata); 3933 3934 // Enqueue task to complete bottom half completion from a thread within the 3935 // corresponding team 3936 kmp_team_t *team = taskdata->td_team; 3937 kmp_int32 nthreads = team->t.t_nproc; 3938 kmp_info_t *thread; 3939 3940 // This should be similar to start_k = __kmp_get_random( thread ) % nthreads 3941 // but we cannot use __kmp_get_random here 3942 kmp_int32 start_k = 0; 3943 kmp_int32 pass = 1; 3944 kmp_int32 k = start_k; 3945 3946 do { 3947 // For now we're just linearly trying to find a thread 3948 thread = team->t.t_threads[k]; 3949 k = (k + 1) % nthreads; 3950 3951 // we did a full pass through all the threads 3952 if (k == start_k) 3953 pass = pass << 1; 3954 3955 } while (!__kmp_give_task(thread, k, ptask, pass)); 3956 3957 __kmp_second_top_half_finish_proxy(taskdata); 3958 3959 KA_TRACE( 3960 10, 3961 ("__kmp_proxy_task_completed_ooo(exit): proxy task completing ooo %p\n", 3962 taskdata)); 3963 } 3964 3965 #if OMP_50_ENABLED 3966 kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref, int gtid, 3967 kmp_task_t *task) { 3968 kmp_taskdata_t *td = KMP_TASK_TO_TASKDATA(task); 3969 if (td->td_allow_completion_event.type == KMP_EVENT_UNINITIALIZED) { 3970 td->td_allow_completion_event.type = KMP_EVENT_ALLOW_COMPLETION; 3971 td->td_allow_completion_event.ed.task = task; 3972 __kmp_init_tas_lock(&td->td_allow_completion_event.lock); 3973 } 3974 return &td->td_allow_completion_event; 3975 } 3976 3977 void __kmp_fulfill_event(kmp_event_t *event) { 3978 if (event->type == KMP_EVENT_ALLOW_COMPLETION) { 3979 kmp_task_t *ptask = event->ed.task; 3980 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask); 3981 bool detached = false; 3982 int gtid = __kmp_get_gtid(); 3983 3984 if (taskdata->td_flags.proxy == TASK_PROXY) { 3985 // The associated task code completed before this call and detached. 3986 detached = true; 3987 event->type = KMP_EVENT_UNINITIALIZED; 3988 } else { 3989 // The associated task has not completed but could be completing at this 3990 // point. 3991 // We need to take the lock to avoid races 3992 __kmp_acquire_tas_lock(&event->lock, gtid); 3993 if (taskdata->td_flags.proxy == TASK_PROXY) 3994 detached = true; 3995 event->type = KMP_EVENT_UNINITIALIZED; 3996 __kmp_release_tas_lock(&event->lock, gtid); 3997 } 3998 3999 if (detached) { 4000 // If the task detached complete the proxy task 4001 if (gtid >= 0) { 4002 kmp_team_t *team = taskdata->td_team; 4003 kmp_info_t *thread = __kmp_get_thread(); 4004 if (thread->th.th_team == team) { 4005 __kmpc_proxy_task_completed(gtid, ptask); 4006 return; 4007 } 4008 } 4009 4010 // fallback 4011 __kmpc_proxy_task_completed_ooo(ptask); 4012 } 4013 } 4014 } 4015 #endif 4016 4017 // __kmp_task_dup_alloc: Allocate the taskdata and make a copy of source task 4018 // for taskloop 4019 // 4020 // thread: allocating thread 4021 // task_src: pointer to source task to be duplicated 4022 // returns: a pointer to the allocated kmp_task_t structure (task). 4023 kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src) { 4024 kmp_task_t *task; 4025 kmp_taskdata_t *taskdata; 4026 kmp_taskdata_t *taskdata_src; 4027 kmp_taskdata_t *parent_task = thread->th.th_current_task; 4028 size_t shareds_offset; 4029 size_t task_size; 4030 4031 KA_TRACE(10, ("__kmp_task_dup_alloc(enter): Th %p, source task %p\n", thread, 4032 task_src)); 4033 taskdata_src = KMP_TASK_TO_TASKDATA(task_src); 4034 KMP_DEBUG_ASSERT(taskdata_src->td_flags.proxy == 4035 TASK_FULL); // it should not be proxy task 4036 KMP_DEBUG_ASSERT(taskdata_src->td_flags.tasktype == TASK_EXPLICIT); 4037 task_size = taskdata_src->td_size_alloc; 4038 4039 // Allocate a kmp_taskdata_t block and a kmp_task_t block. 4040 KA_TRACE(30, ("__kmp_task_dup_alloc: Th %p, malloc size %ld\n", thread, 4041 task_size)); 4042 #if USE_FAST_MEMORY 4043 taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, task_size); 4044 #else 4045 taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, task_size); 4046 #endif /* USE_FAST_MEMORY */ 4047 KMP_MEMCPY(taskdata, taskdata_src, task_size); 4048 4049 task = KMP_TASKDATA_TO_TASK(taskdata); 4050 4051 // Initialize new task (only specific fields not affected by memcpy) 4052 taskdata->td_task_id = KMP_GEN_TASK_ID(); 4053 if (task->shareds != NULL) { // need setup shareds pointer 4054 shareds_offset = (char *)task_src->shareds - (char *)taskdata_src; 4055 task->shareds = &((char *)taskdata)[shareds_offset]; 4056 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) == 4057 0); 4058 } 4059 taskdata->td_alloc_thread = thread; 4060 taskdata->td_parent = parent_task; 4061 taskdata->td_taskgroup = 4062 parent_task 4063 ->td_taskgroup; // task inherits the taskgroup from the parent task 4064 4065 // Only need to keep track of child task counts if team parallel and tasking 4066 // not serialized 4067 if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) { 4068 KMP_ATOMIC_INC(&parent_task->td_incomplete_child_tasks); 4069 if (parent_task->td_taskgroup) 4070 KMP_ATOMIC_INC(&parent_task->td_taskgroup->count); 4071 // Only need to keep track of allocated child tasks for explicit tasks since 4072 // implicit not deallocated 4073 if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT) 4074 KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks); 4075 } 4076 4077 KA_TRACE(20, 4078 ("__kmp_task_dup_alloc(exit): Th %p, created task %p, parent=%p\n", 4079 thread, taskdata, taskdata->td_parent)); 4080 #if OMPT_SUPPORT 4081 if (UNLIKELY(ompt_enabled.enabled)) 4082 __ompt_task_init(taskdata, thread->th.th_info.ds.ds_gtid); 4083 #endif 4084 return task; 4085 } 4086 4087 // Routine optionally generated by the compiler for setting the lastprivate flag 4088 // and calling needed constructors for private/firstprivate objects 4089 // (used to form taskloop tasks from pattern task) 4090 // Parameters: dest task, src task, lastprivate flag. 4091 typedef void (*p_task_dup_t)(kmp_task_t *, kmp_task_t *, kmp_int32); 4092 4093 KMP_BUILD_ASSERT(sizeof(long) == 4 || sizeof(long) == 8); 4094 4095 // class to encapsulate manipulating loop bounds in a taskloop task. 4096 // this abstracts away the Intel vs GOMP taskloop interface for setting/getting 4097 // the loop bound variables. 4098 class kmp_taskloop_bounds_t { 4099 kmp_task_t *task; 4100 const kmp_taskdata_t *taskdata; 4101 size_t lower_offset; 4102 size_t upper_offset; 4103 4104 public: 4105 kmp_taskloop_bounds_t(kmp_task_t *_task, kmp_uint64 *lb, kmp_uint64 *ub) 4106 : task(_task), taskdata(KMP_TASK_TO_TASKDATA(task)), 4107 lower_offset((char *)lb - (char *)task), 4108 upper_offset((char *)ub - (char *)task) { 4109 KMP_DEBUG_ASSERT((char *)lb > (char *)_task); 4110 KMP_DEBUG_ASSERT((char *)ub > (char *)_task); 4111 } 4112 kmp_taskloop_bounds_t(kmp_task_t *_task, const kmp_taskloop_bounds_t &bounds) 4113 : task(_task), taskdata(KMP_TASK_TO_TASKDATA(_task)), 4114 lower_offset(bounds.lower_offset), upper_offset(bounds.upper_offset) {} 4115 size_t get_lower_offset() const { return lower_offset; } 4116 size_t get_upper_offset() const { return upper_offset; } 4117 kmp_uint64 get_lb() const { 4118 kmp_int64 retval; 4119 #if defined(KMP_GOMP_COMPAT) 4120 // Intel task just returns the lower bound normally 4121 if (!taskdata->td_flags.native) { 4122 retval = *(kmp_int64 *)((char *)task + lower_offset); 4123 } else { 4124 // GOMP task has to take into account the sizeof(long) 4125 if (taskdata->td_size_loop_bounds == 4) { 4126 kmp_int32 *lb = RCAST(kmp_int32 *, task->shareds); 4127 retval = (kmp_int64)*lb; 4128 } else { 4129 kmp_int64 *lb = RCAST(kmp_int64 *, task->shareds); 4130 retval = (kmp_int64)*lb; 4131 } 4132 } 4133 #else 4134 retval = *(kmp_int64 *)((char *)task + lower_offset); 4135 #endif // defined(KMP_GOMP_COMPAT) 4136 return retval; 4137 } 4138 kmp_uint64 get_ub() const { 4139 kmp_int64 retval; 4140 #if defined(KMP_GOMP_COMPAT) 4141 // Intel task just returns the upper bound normally 4142 if (!taskdata->td_flags.native) { 4143 retval = *(kmp_int64 *)((char *)task + upper_offset); 4144 } else { 4145 // GOMP task has to take into account the sizeof(long) 4146 if (taskdata->td_size_loop_bounds == 4) { 4147 kmp_int32 *ub = RCAST(kmp_int32 *, task->shareds) + 1; 4148 retval = (kmp_int64)*ub; 4149 } else { 4150 kmp_int64 *ub = RCAST(kmp_int64 *, task->shareds) + 1; 4151 retval = (kmp_int64)*ub; 4152 } 4153 } 4154 #else 4155 retval = *(kmp_int64 *)((char *)task + upper_offset); 4156 #endif // defined(KMP_GOMP_COMPAT) 4157 return retval; 4158 } 4159 void set_lb(kmp_uint64 lb) { 4160 #if defined(KMP_GOMP_COMPAT) 4161 // Intel task just sets the lower bound normally 4162 if (!taskdata->td_flags.native) { 4163 *(kmp_uint64 *)((char *)task + lower_offset) = lb; 4164 } else { 4165 // GOMP task has to take into account the sizeof(long) 4166 if (taskdata->td_size_loop_bounds == 4) { 4167 kmp_uint32 *lower = RCAST(kmp_uint32 *, task->shareds); 4168 *lower = (kmp_uint32)lb; 4169 } else { 4170 kmp_uint64 *lower = RCAST(kmp_uint64 *, task->shareds); 4171 *lower = (kmp_uint64)lb; 4172 } 4173 } 4174 #else 4175 *(kmp_uint64 *)((char *)task + lower_offset) = lb; 4176 #endif // defined(KMP_GOMP_COMPAT) 4177 } 4178 void set_ub(kmp_uint64 ub) { 4179 #if defined(KMP_GOMP_COMPAT) 4180 // Intel task just sets the upper bound normally 4181 if (!taskdata->td_flags.native) { 4182 *(kmp_uint64 *)((char *)task + upper_offset) = ub; 4183 } else { 4184 // GOMP task has to take into account the sizeof(long) 4185 if (taskdata->td_size_loop_bounds == 4) { 4186 kmp_uint32 *upper = RCAST(kmp_uint32 *, task->shareds) + 1; 4187 *upper = (kmp_uint32)ub; 4188 } else { 4189 kmp_uint64 *upper = RCAST(kmp_uint64 *, task->shareds) + 1; 4190 *upper = (kmp_uint64)ub; 4191 } 4192 } 4193 #else 4194 *(kmp_uint64 *)((char *)task + upper_offset) = ub; 4195 #endif // defined(KMP_GOMP_COMPAT) 4196 } 4197 }; 4198 4199 // __kmp_taskloop_linear: Start tasks of the taskloop linearly 4200 // 4201 // loc Source location information 4202 // gtid Global thread ID 4203 // task Pattern task, exposes the loop iteration range 4204 // lb Pointer to loop lower bound in task structure 4205 // ub Pointer to loop upper bound in task structure 4206 // st Loop stride 4207 // ub_glob Global upper bound (used for lastprivate check) 4208 // num_tasks Number of tasks to execute 4209 // grainsize Number of loop iterations per task 4210 // extras Number of chunks with grainsize+1 iterations 4211 // tc Iterations count 4212 // task_dup Tasks duplication routine 4213 // codeptr_ra Return address for OMPT events 4214 void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task, 4215 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, 4216 kmp_uint64 ub_glob, kmp_uint64 num_tasks, 4217 kmp_uint64 grainsize, kmp_uint64 extras, 4218 kmp_uint64 tc, 4219 #if OMPT_SUPPORT 4220 void *codeptr_ra, 4221 #endif 4222 void *task_dup) { 4223 KMP_COUNT_BLOCK(OMP_TASKLOOP); 4224 KMP_TIME_PARTITIONED_BLOCK(OMP_taskloop_scheduling); 4225 p_task_dup_t ptask_dup = (p_task_dup_t)task_dup; 4226 // compiler provides global bounds here 4227 kmp_taskloop_bounds_t task_bounds(task, lb, ub); 4228 kmp_uint64 lower = task_bounds.get_lb(); 4229 kmp_uint64 upper = task_bounds.get_ub(); 4230 kmp_uint64 i; 4231 kmp_info_t *thread = __kmp_threads[gtid]; 4232 kmp_taskdata_t *current_task = thread->th.th_current_task; 4233 kmp_task_t *next_task; 4234 kmp_int32 lastpriv = 0; 4235 4236 KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + extras); 4237 KMP_DEBUG_ASSERT(num_tasks > extras); 4238 KMP_DEBUG_ASSERT(num_tasks > 0); 4239 KA_TRACE(20, ("__kmp_taskloop_linear: T#%d: %lld tasks, grainsize %lld, " 4240 "extras %lld, i=%lld,%lld(%d)%lld, dup %p\n", 4241 gtid, num_tasks, grainsize, extras, lower, upper, ub_glob, st, 4242 task_dup)); 4243 4244 // Launch num_tasks tasks, assign grainsize iterations each task 4245 for (i = 0; i < num_tasks; ++i) { 4246 kmp_uint64 chunk_minus_1; 4247 if (extras == 0) { 4248 chunk_minus_1 = grainsize - 1; 4249 } else { 4250 chunk_minus_1 = grainsize; 4251 --extras; // first extras iterations get bigger chunk (grainsize+1) 4252 } 4253 upper = lower + st * chunk_minus_1; 4254 if (i == num_tasks - 1) { 4255 // schedule the last task, set lastprivate flag if needed 4256 if (st == 1) { // most common case 4257 KMP_DEBUG_ASSERT(upper == *ub); 4258 if (upper == ub_glob) 4259 lastpriv = 1; 4260 } else if (st > 0) { // positive loop stride 4261 KMP_DEBUG_ASSERT((kmp_uint64)st > *ub - upper); 4262 if ((kmp_uint64)st > ub_glob - upper) 4263 lastpriv = 1; 4264 } else { // negative loop stride 4265 KMP_DEBUG_ASSERT(upper + st < *ub); 4266 if (upper - ub_glob < (kmp_uint64)(-st)) 4267 lastpriv = 1; 4268 } 4269 } 4270 next_task = __kmp_task_dup_alloc(thread, task); // allocate new task 4271 kmp_taskdata_t *next_taskdata = KMP_TASK_TO_TASKDATA(next_task); 4272 kmp_taskloop_bounds_t next_task_bounds = 4273 kmp_taskloop_bounds_t(next_task, task_bounds); 4274 4275 // adjust task-specific bounds 4276 next_task_bounds.set_lb(lower); 4277 if (next_taskdata->td_flags.native) { 4278 next_task_bounds.set_ub(upper + (st > 0 ? 1 : -1)); 4279 } else { 4280 next_task_bounds.set_ub(upper); 4281 } 4282 if (ptask_dup != NULL) // set lastprivate flag, construct fistprivates, etc. 4283 ptask_dup(next_task, task, lastpriv); 4284 KA_TRACE(40, 4285 ("__kmp_taskloop_linear: T#%d; task #%llu: task %p: lower %lld, " 4286 "upper %lld stride %lld, (offsets %p %p)\n", 4287 gtid, i, next_task, lower, upper, st, 4288 next_task_bounds.get_lower_offset(), 4289 next_task_bounds.get_upper_offset())); 4290 #if OMPT_SUPPORT 4291 __kmp_omp_taskloop_task(NULL, gtid, next_task, 4292 codeptr_ra); // schedule new task 4293 #else 4294 __kmp_omp_task(gtid, next_task, true); // schedule new task 4295 #endif 4296 lower = upper + st; // adjust lower bound for the next iteration 4297 } 4298 // free the pattern task and exit 4299 __kmp_task_start(gtid, task, current_task); // make internal bookkeeping 4300 // do not execute the pattern task, just do internal bookkeeping 4301 __kmp_task_finish<false>(gtid, task, current_task); 4302 } 4303 4304 // Structure to keep taskloop parameters for auxiliary task 4305 // kept in the shareds of the task structure. 4306 typedef struct __taskloop_params { 4307 kmp_task_t *task; 4308 kmp_uint64 *lb; 4309 kmp_uint64 *ub; 4310 void *task_dup; 4311 kmp_int64 st; 4312 kmp_uint64 ub_glob; 4313 kmp_uint64 num_tasks; 4314 kmp_uint64 grainsize; 4315 kmp_uint64 extras; 4316 kmp_uint64 tc; 4317 kmp_uint64 num_t_min; 4318 #if OMPT_SUPPORT 4319 void *codeptr_ra; 4320 #endif 4321 } __taskloop_params_t; 4322 4323 void __kmp_taskloop_recur(ident_t *, int, kmp_task_t *, kmp_uint64 *, 4324 kmp_uint64 *, kmp_int64, kmp_uint64, kmp_uint64, 4325 kmp_uint64, kmp_uint64, kmp_uint64, kmp_uint64, 4326 #if OMPT_SUPPORT 4327 void *, 4328 #endif 4329 void *); 4330 4331 // Execute part of the the taskloop submitted as a task. 4332 int __kmp_taskloop_task(int gtid, void *ptask) { 4333 __taskloop_params_t *p = 4334 (__taskloop_params_t *)((kmp_task_t *)ptask)->shareds; 4335 kmp_task_t *task = p->task; 4336 kmp_uint64 *lb = p->lb; 4337 kmp_uint64 *ub = p->ub; 4338 void *task_dup = p->task_dup; 4339 // p_task_dup_t ptask_dup = (p_task_dup_t)task_dup; 4340 kmp_int64 st = p->st; 4341 kmp_uint64 ub_glob = p->ub_glob; 4342 kmp_uint64 num_tasks = p->num_tasks; 4343 kmp_uint64 grainsize = p->grainsize; 4344 kmp_uint64 extras = p->extras; 4345 kmp_uint64 tc = p->tc; 4346 kmp_uint64 num_t_min = p->num_t_min; 4347 #if OMPT_SUPPORT 4348 void *codeptr_ra = p->codeptr_ra; 4349 #endif 4350 #if KMP_DEBUG 4351 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 4352 KMP_DEBUG_ASSERT(task != NULL); 4353 KA_TRACE(20, ("__kmp_taskloop_task: T#%d, task %p: %lld tasks, grainsize" 4354 " %lld, extras %lld, i=%lld,%lld(%d), dup %p\n", 4355 gtid, taskdata, num_tasks, grainsize, extras, *lb, *ub, st, 4356 task_dup)); 4357 #endif 4358 KMP_DEBUG_ASSERT(num_tasks * 2 + 1 > num_t_min); 4359 if (num_tasks > num_t_min) 4360 __kmp_taskloop_recur(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks, 4361 grainsize, extras, tc, num_t_min, 4362 #if OMPT_SUPPORT 4363 codeptr_ra, 4364 #endif 4365 task_dup); 4366 else 4367 __kmp_taskloop_linear(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks, 4368 grainsize, extras, tc, 4369 #if OMPT_SUPPORT 4370 codeptr_ra, 4371 #endif 4372 task_dup); 4373 4374 KA_TRACE(40, ("__kmp_taskloop_task(exit): T#%d\n", gtid)); 4375 return 0; 4376 } 4377 4378 // Schedule part of the the taskloop as a task, 4379 // execute the rest of the the taskloop. 4380 // 4381 // loc Source location information 4382 // gtid Global thread ID 4383 // task Pattern task, exposes the loop iteration range 4384 // lb Pointer to loop lower bound in task structure 4385 // ub Pointer to loop upper bound in task structure 4386 // st Loop stride 4387 // ub_glob Global upper bound (used for lastprivate check) 4388 // num_tasks Number of tasks to execute 4389 // grainsize Number of loop iterations per task 4390 // extras Number of chunks with grainsize+1 iterations 4391 // tc Iterations count 4392 // num_t_min Threashold to launch tasks recursively 4393 // task_dup Tasks duplication routine 4394 // codeptr_ra Return address for OMPT events 4395 void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task, 4396 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, 4397 kmp_uint64 ub_glob, kmp_uint64 num_tasks, 4398 kmp_uint64 grainsize, kmp_uint64 extras, 4399 kmp_uint64 tc, kmp_uint64 num_t_min, 4400 #if OMPT_SUPPORT 4401 void *codeptr_ra, 4402 #endif 4403 void *task_dup) { 4404 #if KMP_DEBUG 4405 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 4406 KMP_DEBUG_ASSERT(task != NULL); 4407 KMP_DEBUG_ASSERT(num_tasks > num_t_min); 4408 KA_TRACE(20, ("__kmp_taskloop_recur: T#%d, task %p: %lld tasks, grainsize" 4409 " %lld, extras %lld, i=%lld,%lld(%d), dup %p\n", 4410 gtid, taskdata, num_tasks, grainsize, extras, *lb, *ub, st, 4411 task_dup)); 4412 #endif 4413 p_task_dup_t ptask_dup = (p_task_dup_t)task_dup; 4414 kmp_uint64 lower = *lb; 4415 kmp_info_t *thread = __kmp_threads[gtid]; 4416 // kmp_taskdata_t *current_task = thread->th.th_current_task; 4417 kmp_task_t *next_task; 4418 size_t lower_offset = 4419 (char *)lb - (char *)task; // remember offset of lb in the task structure 4420 size_t upper_offset = 4421 (char *)ub - (char *)task; // remember offset of ub in the task structure 4422 4423 KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + extras); 4424 KMP_DEBUG_ASSERT(num_tasks > extras); 4425 KMP_DEBUG_ASSERT(num_tasks > 0); 4426 4427 // split the loop in two halves 4428 kmp_uint64 lb1, ub0, tc0, tc1, ext0, ext1; 4429 kmp_uint64 gr_size0 = grainsize; 4430 kmp_uint64 n_tsk0 = num_tasks >> 1; // num_tasks/2 to execute 4431 kmp_uint64 n_tsk1 = num_tasks - n_tsk0; // to schedule as a task 4432 if (n_tsk0 <= extras) { 4433 gr_size0++; // integrate extras into grainsize 4434 ext0 = 0; // no extra iters in 1st half 4435 ext1 = extras - n_tsk0; // remaining extras 4436 tc0 = gr_size0 * n_tsk0; 4437 tc1 = tc - tc0; 4438 } else { // n_tsk0 > extras 4439 ext1 = 0; // no extra iters in 2nd half 4440 ext0 = extras; 4441 tc1 = grainsize * n_tsk1; 4442 tc0 = tc - tc1; 4443 } 4444 ub0 = lower + st * (tc0 - 1); 4445 lb1 = ub0 + st; 4446 4447 // create pattern task for 2nd half of the loop 4448 next_task = __kmp_task_dup_alloc(thread, task); // duplicate the task 4449 // adjust lower bound (upper bound is not changed) for the 2nd half 4450 *(kmp_uint64 *)((char *)next_task + lower_offset) = lb1; 4451 if (ptask_dup != NULL) // construct fistprivates, etc. 4452 ptask_dup(next_task, task, 0); 4453 *ub = ub0; // adjust upper bound for the 1st half 4454 4455 // create auxiliary task for 2nd half of the loop 4456 kmp_task_t *new_task = 4457 __kmpc_omp_task_alloc(loc, gtid, 1, 3 * sizeof(void *), 4458 sizeof(__taskloop_params_t), &__kmp_taskloop_task); 4459 __taskloop_params_t *p = (__taskloop_params_t *)new_task->shareds; 4460 p->task = next_task; 4461 p->lb = (kmp_uint64 *)((char *)next_task + lower_offset); 4462 p->ub = (kmp_uint64 *)((char *)next_task + upper_offset); 4463 p->task_dup = task_dup; 4464 p->st = st; 4465 p->ub_glob = ub_glob; 4466 p->num_tasks = n_tsk1; 4467 p->grainsize = grainsize; 4468 p->extras = ext1; 4469 p->tc = tc1; 4470 p->num_t_min = num_t_min; 4471 #if OMPT_SUPPORT 4472 p->codeptr_ra = codeptr_ra; 4473 #endif 4474 4475 #if OMPT_SUPPORT 4476 // schedule new task with correct return address for OMPT events 4477 __kmp_omp_taskloop_task(NULL, gtid, new_task, codeptr_ra); 4478 #else 4479 __kmp_omp_task(gtid, new_task, true); // schedule new task 4480 #endif 4481 4482 // execute the 1st half of current subrange 4483 if (n_tsk0 > num_t_min) 4484 __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0, gr_size0, 4485 ext0, tc0, num_t_min, 4486 #if OMPT_SUPPORT 4487 codeptr_ra, 4488 #endif 4489 task_dup); 4490 else 4491 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0, 4492 gr_size0, ext0, tc0, 4493 #if OMPT_SUPPORT 4494 codeptr_ra, 4495 #endif 4496 task_dup); 4497 4498 KA_TRACE(40, ("__kmpc_taskloop_recur(exit): T#%d\n", gtid)); 4499 } 4500 4501 /*! 4502 @ingroup TASKING 4503 @param loc Source location information 4504 @param gtid Global thread ID 4505 @param task Task structure 4506 @param if_val Value of the if clause 4507 @param lb Pointer to loop lower bound in task structure 4508 @param ub Pointer to loop upper bound in task structure 4509 @param st Loop stride 4510 @param nogroup Flag, 1 if no taskgroup needs to be added, 0 otherwise 4511 @param sched Schedule specified 0/1/2 for none/grainsize/num_tasks 4512 @param grainsize Schedule value if specified 4513 @param task_dup Tasks duplication routine 4514 4515 Execute the taskloop construct. 4516 */ 4517 void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val, 4518 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, 4519 int sched, kmp_uint64 grainsize, void *task_dup) { 4520 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 4521 KMP_DEBUG_ASSERT(task != NULL); 4522 4523 if (nogroup == 0) { 4524 #if OMPT_SUPPORT && OMPT_OPTIONAL 4525 OMPT_STORE_RETURN_ADDRESS(gtid); 4526 #endif 4527 __kmpc_taskgroup(loc, gtid); 4528 } 4529 4530 // ========================================================================= 4531 // calculate loop parameters 4532 kmp_taskloop_bounds_t task_bounds(task, lb, ub); 4533 kmp_uint64 tc; 4534 // compiler provides global bounds here 4535 kmp_uint64 lower = task_bounds.get_lb(); 4536 kmp_uint64 upper = task_bounds.get_ub(); 4537 kmp_uint64 ub_glob = upper; // global upper used to calc lastprivate flag 4538 kmp_uint64 num_tasks = 0, extras = 0; 4539 kmp_uint64 num_tasks_min = __kmp_taskloop_min_tasks; 4540 kmp_info_t *thread = __kmp_threads[gtid]; 4541 kmp_taskdata_t *current_task = thread->th.th_current_task; 4542 4543 KA_TRACE(20, ("__kmpc_taskloop: T#%d, task %p, lb %lld, ub %lld, st %lld, " 4544 "grain %llu(%d), dup %p\n", 4545 gtid, taskdata, lower, upper, st, grainsize, sched, task_dup)); 4546 4547 // compute trip count 4548 if (st == 1) { // most common case 4549 tc = upper - lower + 1; 4550 } else if (st < 0) { 4551 tc = (lower - upper) / (-st) + 1; 4552 } else { // st > 0 4553 tc = (upper - lower) / st + 1; 4554 } 4555 if (tc == 0) { 4556 KA_TRACE(20, ("__kmpc_taskloop(exit): T#%d zero-trip loop\n", gtid)); 4557 // free the pattern task and exit 4558 __kmp_task_start(gtid, task, current_task); 4559 // do not execute anything for zero-trip loop 4560 __kmp_task_finish<false>(gtid, task, current_task); 4561 return; 4562 } 4563 4564 #if OMPT_SUPPORT && OMPT_OPTIONAL 4565 ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL); 4566 ompt_task_info_t *task_info = __ompt_get_task_info_object(0); 4567 if (ompt_enabled.ompt_callback_work) { 4568 ompt_callbacks.ompt_callback(ompt_callback_work)( 4569 ompt_work_taskloop, ompt_scope_begin, &(team_info->parallel_data), 4570 &(task_info->task_data), tc, OMPT_GET_RETURN_ADDRESS(0)); 4571 } 4572 #endif 4573 4574 if (num_tasks_min == 0) 4575 // TODO: can we choose better default heuristic? 4576 num_tasks_min = 4577 KMP_MIN(thread->th.th_team_nproc * 10, INITIAL_TASK_DEQUE_SIZE); 4578 4579 // compute num_tasks/grainsize based on the input provided 4580 switch (sched) { 4581 case 0: // no schedule clause specified, we can choose the default 4582 // let's try to schedule (team_size*10) tasks 4583 grainsize = thread->th.th_team_nproc * 10; 4584 KMP_FALLTHROUGH(); 4585 case 2: // num_tasks provided 4586 if (grainsize > tc) { 4587 num_tasks = tc; // too big num_tasks requested, adjust values 4588 grainsize = 1; 4589 extras = 0; 4590 } else { 4591 num_tasks = grainsize; 4592 grainsize = tc / num_tasks; 4593 extras = tc % num_tasks; 4594 } 4595 break; 4596 case 1: // grainsize provided 4597 if (grainsize > tc) { 4598 num_tasks = 1; // too big grainsize requested, adjust values 4599 grainsize = tc; 4600 extras = 0; 4601 } else { 4602 num_tasks = tc / grainsize; 4603 // adjust grainsize for balanced distribution of iterations 4604 grainsize = tc / num_tasks; 4605 extras = tc % num_tasks; 4606 } 4607 break; 4608 default: 4609 KMP_ASSERT2(0, "unknown scheduling of taskloop"); 4610 } 4611 KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + extras); 4612 KMP_DEBUG_ASSERT(num_tasks > extras); 4613 KMP_DEBUG_ASSERT(num_tasks > 0); 4614 // ========================================================================= 4615 4616 // check if clause value first 4617 // Also require GOMP_taskloop to reduce to linear (taskdata->td_flags.native) 4618 if (if_val == 0) { // if(0) specified, mark task as serial 4619 taskdata->td_flags.task_serial = 1; 4620 taskdata->td_flags.tiedness = TASK_TIED; // AC: serial task cannot be untied 4621 // always start serial tasks linearly 4622 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks, 4623 grainsize, extras, tc, 4624 #if OMPT_SUPPORT 4625 OMPT_GET_RETURN_ADDRESS(0), 4626 #endif 4627 task_dup); 4628 // !taskdata->td_flags.native => currently force linear spawning of tasks 4629 // for GOMP_taskloop 4630 } else if (num_tasks > num_tasks_min && !taskdata->td_flags.native) { 4631 KA_TRACE(20, ("__kmpc_taskloop: T#%d, go recursive: tc %llu, #tasks %llu" 4632 "(%lld), grain %llu, extras %llu\n", 4633 gtid, tc, num_tasks, num_tasks_min, grainsize, extras)); 4634 __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, num_tasks, 4635 grainsize, extras, tc, num_tasks_min, 4636 #if OMPT_SUPPORT 4637 OMPT_GET_RETURN_ADDRESS(0), 4638 #endif 4639 task_dup); 4640 } else { 4641 KA_TRACE(20, ("__kmpc_taskloop: T#%d, go linear: tc %llu, #tasks %llu" 4642 "(%lld), grain %llu, extras %llu\n", 4643 gtid, tc, num_tasks, num_tasks_min, grainsize, extras)); 4644 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks, 4645 grainsize, extras, tc, 4646 #if OMPT_SUPPORT 4647 OMPT_GET_RETURN_ADDRESS(0), 4648 #endif 4649 task_dup); 4650 } 4651 4652 #if OMPT_SUPPORT && OMPT_OPTIONAL 4653 if (ompt_enabled.ompt_callback_work) { 4654 ompt_callbacks.ompt_callback(ompt_callback_work)( 4655 ompt_work_taskloop, ompt_scope_end, &(team_info->parallel_data), 4656 &(task_info->task_data), tc, OMPT_GET_RETURN_ADDRESS(0)); 4657 } 4658 #endif 4659 4660 if (nogroup == 0) { 4661 #if OMPT_SUPPORT && OMPT_OPTIONAL 4662 OMPT_STORE_RETURN_ADDRESS(gtid); 4663 #endif 4664 __kmpc_end_taskgroup(loc, gtid); 4665 } 4666 KA_TRACE(20, ("__kmpc_taskloop(exit): T#%d\n", gtid)); 4667 } 4668 4669 #endif 4670