1 /* 2 * kmp_runtime.cpp -- KPTS runtime support library 3 */ 4 5 6 //===----------------------------------------------------------------------===// 7 // 8 // The LLVM Compiler Infrastructure 9 // 10 // This file is dual licensed under the MIT and the University of Illinois Open 11 // Source Licenses. See LICENSE.txt for details. 12 // 13 //===----------------------------------------------------------------------===// 14 15 16 #include "kmp.h" 17 #include "kmp_affinity.h" 18 #include "kmp_atomic.h" 19 #include "kmp_environment.h" 20 #include "kmp_error.h" 21 #include "kmp_i18n.h" 22 #include "kmp_io.h" 23 #include "kmp_itt.h" 24 #include "kmp_settings.h" 25 #include "kmp_stats.h" 26 #include "kmp_str.h" 27 #include "kmp_wait_release.h" 28 #include "kmp_wrapper_getpid.h" 29 30 #if OMPT_SUPPORT 31 #include "ompt-specific.h" 32 #endif 33 34 /* these are temporary issues to be dealt with */ 35 #define KMP_USE_PRCTL 0 36 37 #if KMP_OS_WINDOWS 38 #include <process.h> 39 #endif 40 41 #include "tsan_annotations.h" 42 43 #if defined(KMP_GOMP_COMPAT) 44 char const __kmp_version_alt_comp[] = 45 KMP_VERSION_PREFIX "alternative compiler support: yes"; 46 #endif /* defined(KMP_GOMP_COMPAT) */ 47 48 char const __kmp_version_omp_api[] = KMP_VERSION_PREFIX "API version: " 49 #if OMP_50_ENABLED 50 "5.0 (201611)"; 51 #elif OMP_45_ENABLED 52 "4.5 (201511)"; 53 #elif OMP_40_ENABLED 54 "4.0 (201307)"; 55 #else 56 "3.1 (201107)"; 57 #endif 58 59 #ifdef KMP_DEBUG 60 char const __kmp_version_lock[] = 61 KMP_VERSION_PREFIX "lock type: run time selectable"; 62 #endif /* KMP_DEBUG */ 63 64 #define KMP_MIN(x, y) ((x) < (y) ? (x) : (y)) 65 66 /* ------------------------------------------------------------------------ */ 67 68 kmp_info_t __kmp_monitor; 69 70 /* Forward declarations */ 71 72 void __kmp_cleanup(void); 73 74 static void __kmp_initialize_info(kmp_info_t *, kmp_team_t *, int tid, 75 int gtid); 76 static void __kmp_initialize_team(kmp_team_t *team, int new_nproc, 77 kmp_internal_control_t *new_icvs, 78 ident_t *loc); 79 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 80 static void __kmp_partition_places(kmp_team_t *team, 81 int update_master_only = 0); 82 #endif 83 static void __kmp_do_serial_initialize(void); 84 void __kmp_fork_barrier(int gtid, int tid); 85 void __kmp_join_barrier(int gtid); 86 void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc, 87 kmp_internal_control_t *new_icvs, ident_t *loc); 88 89 #ifdef USE_LOAD_BALANCE 90 static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc); 91 #endif 92 93 static int __kmp_expand_threads(int nWish, int nNeed); 94 #if KMP_OS_WINDOWS 95 static int __kmp_unregister_root_other_thread(int gtid); 96 #endif 97 static void __kmp_unregister_library(void); // called by __kmp_internal_end() 98 static void __kmp_reap_thread(kmp_info_t *thread, int is_root); 99 static kmp_info_t *__kmp_thread_pool_insert_pt = NULL; 100 101 /* Calculate the identifier of the current thread */ 102 /* fast (and somewhat portable) way to get unique identifier of executing 103 thread. Returns KMP_GTID_DNE if we haven't been assigned a gtid. */ 104 int __kmp_get_global_thread_id() { 105 int i; 106 kmp_info_t **other_threads; 107 size_t stack_data; 108 char *stack_addr; 109 size_t stack_size; 110 char *stack_base; 111 112 KA_TRACE( 113 1000, 114 ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n", 115 __kmp_nth, __kmp_all_nth)); 116 117 /* JPH - to handle the case where __kmpc_end(0) is called immediately prior to 118 a parallel region, made it return KMP_GTID_DNE to force serial_initialize 119 by caller. Had to handle KMP_GTID_DNE at all call-sites, or else guarantee 120 __kmp_init_gtid for this to work. */ 121 122 if (!TCR_4(__kmp_init_gtid)) 123 return KMP_GTID_DNE; 124 125 #ifdef KMP_TDATA_GTID 126 if (TCR_4(__kmp_gtid_mode) >= 3) { 127 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using TDATA\n")); 128 return __kmp_gtid; 129 } 130 #endif 131 if (TCR_4(__kmp_gtid_mode) >= 2) { 132 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using keyed TLS\n")); 133 return __kmp_gtid_get_specific(); 134 } 135 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using internal alg.\n")); 136 137 stack_addr = (char *)&stack_data; 138 other_threads = __kmp_threads; 139 140 /* ATT: The code below is a source of potential bugs due to unsynchronized 141 access to __kmp_threads array. For example: 142 1. Current thread loads other_threads[i] to thr and checks it, it is 143 non-NULL. 144 2. Current thread is suspended by OS. 145 3. Another thread unregisters and finishes (debug versions of free() 146 may fill memory with something like 0xEF). 147 4. Current thread is resumed. 148 5. Current thread reads junk from *thr. 149 TODO: Fix it. --ln */ 150 151 for (i = 0; i < __kmp_threads_capacity; i++) { 152 153 kmp_info_t *thr = (kmp_info_t *)TCR_SYNC_PTR(other_threads[i]); 154 if (!thr) 155 continue; 156 157 stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize); 158 stack_base = (char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase); 159 160 /* stack grows down -- search through all of the active threads */ 161 162 if (stack_addr <= stack_base) { 163 size_t stack_diff = stack_base - stack_addr; 164 165 if (stack_diff <= stack_size) { 166 /* The only way we can be closer than the allocated */ 167 /* stack size is if we are running on this thread. */ 168 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == i); 169 return i; 170 } 171 } 172 } 173 174 /* get specific to try and determine our gtid */ 175 KA_TRACE(1000, 176 ("*** __kmp_get_global_thread_id: internal alg. failed to find " 177 "thread, using TLS\n")); 178 i = __kmp_gtid_get_specific(); 179 180 /*fprintf( stderr, "=== %d\n", i ); */ /* GROO */ 181 182 /* if we havn't been assigned a gtid, then return code */ 183 if (i < 0) 184 return i; 185 186 /* dynamically updated stack window for uber threads to avoid get_specific 187 call */ 188 if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)) { 189 KMP_FATAL(StackOverflow, i); 190 } 191 192 stack_base = (char *)other_threads[i]->th.th_info.ds.ds_stackbase; 193 if (stack_addr > stack_base) { 194 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr); 195 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize, 196 other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr - 197 stack_base); 198 } else { 199 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize, 200 stack_base - stack_addr); 201 } 202 203 /* Reprint stack bounds for ubermaster since they have been refined */ 204 if (__kmp_storage_map) { 205 char *stack_end = (char *)other_threads[i]->th.th_info.ds.ds_stackbase; 206 char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize; 207 __kmp_print_storage_map_gtid(i, stack_beg, stack_end, 208 other_threads[i]->th.th_info.ds.ds_stacksize, 209 "th_%d stack (refinement)", i); 210 } 211 return i; 212 } 213 214 int __kmp_get_global_thread_id_reg() { 215 int gtid; 216 217 if (!__kmp_init_serial) { 218 gtid = KMP_GTID_DNE; 219 } else 220 #ifdef KMP_TDATA_GTID 221 if (TCR_4(__kmp_gtid_mode) >= 3) { 222 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using TDATA\n")); 223 gtid = __kmp_gtid; 224 } else 225 #endif 226 if (TCR_4(__kmp_gtid_mode) >= 2) { 227 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using keyed TLS\n")); 228 gtid = __kmp_gtid_get_specific(); 229 } else { 230 KA_TRACE(1000, 231 ("*** __kmp_get_global_thread_id_reg: using internal alg.\n")); 232 gtid = __kmp_get_global_thread_id(); 233 } 234 235 /* we must be a new uber master sibling thread */ 236 if (gtid == KMP_GTID_DNE) { 237 KA_TRACE(10, 238 ("__kmp_get_global_thread_id_reg: Encountered new root thread. " 239 "Registering a new gtid.\n")); 240 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 241 if (!__kmp_init_serial) { 242 __kmp_do_serial_initialize(); 243 gtid = __kmp_gtid_get_specific(); 244 } else { 245 gtid = __kmp_register_root(FALSE); 246 } 247 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 248 /*__kmp_printf( "+++ %d\n", gtid ); */ /* GROO */ 249 } 250 251 KMP_DEBUG_ASSERT(gtid >= 0); 252 253 return gtid; 254 } 255 256 /* caller must hold forkjoin_lock */ 257 void __kmp_check_stack_overlap(kmp_info_t *th) { 258 int f; 259 char *stack_beg = NULL; 260 char *stack_end = NULL; 261 int gtid; 262 263 KA_TRACE(10, ("__kmp_check_stack_overlap: called\n")); 264 if (__kmp_storage_map) { 265 stack_end = (char *)th->th.th_info.ds.ds_stackbase; 266 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize; 267 268 gtid = __kmp_gtid_from_thread(th); 269 270 if (gtid == KMP_GTID_MONITOR) { 271 __kmp_print_storage_map_gtid( 272 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize, 273 "th_%s stack (%s)", "mon", 274 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual"); 275 } else { 276 __kmp_print_storage_map_gtid( 277 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize, 278 "th_%d stack (%s)", gtid, 279 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual"); 280 } 281 } 282 283 /* No point in checking ubermaster threads since they use refinement and 284 * cannot overlap */ 285 gtid = __kmp_gtid_from_thread(th); 286 if (__kmp_env_checks == TRUE && !KMP_UBER_GTID(gtid)) { 287 KA_TRACE(10, 288 ("__kmp_check_stack_overlap: performing extensive checking\n")); 289 if (stack_beg == NULL) { 290 stack_end = (char *)th->th.th_info.ds.ds_stackbase; 291 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize; 292 } 293 294 for (f = 0; f < __kmp_threads_capacity; f++) { 295 kmp_info_t *f_th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[f]); 296 297 if (f_th && f_th != th) { 298 char *other_stack_end = 299 (char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase); 300 char *other_stack_beg = 301 other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize); 302 if ((stack_beg > other_stack_beg && stack_beg < other_stack_end) || 303 (stack_end > other_stack_beg && stack_end < other_stack_end)) { 304 305 /* Print the other stack values before the abort */ 306 if (__kmp_storage_map) 307 __kmp_print_storage_map_gtid( 308 -1, other_stack_beg, other_stack_end, 309 (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize), 310 "th_%d stack (overlapped)", __kmp_gtid_from_thread(f_th)); 311 312 __kmp_fatal(KMP_MSG(StackOverlap), KMP_HNT(ChangeStackLimit), 313 __kmp_msg_null); 314 } 315 } 316 } 317 } 318 KA_TRACE(10, ("__kmp_check_stack_overlap: returning\n")); 319 } 320 321 /* ------------------------------------------------------------------------ */ 322 323 void __kmp_infinite_loop(void) { 324 static int done = FALSE; 325 326 while (!done) { 327 KMP_YIELD(1); 328 } 329 } 330 331 #define MAX_MESSAGE 512 332 333 void __kmp_print_storage_map_gtid(int gtid, void *p1, void *p2, size_t size, 334 char const *format, ...) { 335 char buffer[MAX_MESSAGE]; 336 va_list ap; 337 338 va_start(ap, format); 339 KMP_SNPRINTF(buffer, sizeof(buffer), "OMP storage map: %p %p%8lu %s\n", p1, 340 p2, (unsigned long)size, format); 341 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock); 342 __kmp_vprintf(kmp_err, buffer, ap); 343 #if KMP_PRINT_DATA_PLACEMENT 344 int node; 345 if (gtid >= 0) { 346 if (p1 <= p2 && (char *)p2 - (char *)p1 == size) { 347 if (__kmp_storage_map_verbose) { 348 node = __kmp_get_host_node(p1); 349 if (node < 0) /* doesn't work, so don't try this next time */ 350 __kmp_storage_map_verbose = FALSE; 351 else { 352 char *last; 353 int lastNode; 354 int localProc = __kmp_get_cpu_from_gtid(gtid); 355 356 const int page_size = KMP_GET_PAGE_SIZE(); 357 358 p1 = (void *)((size_t)p1 & ~((size_t)page_size - 1)); 359 p2 = (void *)(((size_t)p2 - 1) & ~((size_t)page_size - 1)); 360 if (localProc >= 0) 361 __kmp_printf_no_lock(" GTID %d localNode %d\n", gtid, 362 localProc >> 1); 363 else 364 __kmp_printf_no_lock(" GTID %d\n", gtid); 365 #if KMP_USE_PRCTL 366 /* The more elaborate format is disabled for now because of the prctl 367 * hanging bug. */ 368 do { 369 last = p1; 370 lastNode = node; 371 /* This loop collates adjacent pages with the same host node. */ 372 do { 373 (char *)p1 += page_size; 374 } while (p1 <= p2 && (node = __kmp_get_host_node(p1)) == lastNode); 375 __kmp_printf_no_lock(" %p-%p memNode %d\n", last, (char *)p1 - 1, 376 lastNode); 377 } while (p1 <= p2); 378 #else 379 __kmp_printf_no_lock(" %p-%p memNode %d\n", p1, 380 (char *)p1 + (page_size - 1), 381 __kmp_get_host_node(p1)); 382 if (p1 < p2) { 383 __kmp_printf_no_lock(" %p-%p memNode %d\n", p2, 384 (char *)p2 + (page_size - 1), 385 __kmp_get_host_node(p2)); 386 } 387 #endif 388 } 389 } 390 } else 391 __kmp_printf_no_lock(" %s\n", KMP_I18N_STR(StorageMapWarning)); 392 } 393 #endif /* KMP_PRINT_DATA_PLACEMENT */ 394 __kmp_release_bootstrap_lock(&__kmp_stdio_lock); 395 } 396 397 void __kmp_warn(char const *format, ...) { 398 char buffer[MAX_MESSAGE]; 399 va_list ap; 400 401 if (__kmp_generate_warnings == kmp_warnings_off) { 402 return; 403 } 404 405 va_start(ap, format); 406 407 KMP_SNPRINTF(buffer, sizeof(buffer), "OMP warning: %s\n", format); 408 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock); 409 __kmp_vprintf(kmp_err, buffer, ap); 410 __kmp_release_bootstrap_lock(&__kmp_stdio_lock); 411 412 va_end(ap); 413 } 414 415 void __kmp_abort_process() { 416 // Later threads may stall here, but that's ok because abort() will kill them. 417 __kmp_acquire_bootstrap_lock(&__kmp_exit_lock); 418 419 if (__kmp_debug_buf) { 420 __kmp_dump_debug_buffer(); 421 }; // if 422 423 if (KMP_OS_WINDOWS) { 424 // Let other threads know of abnormal termination and prevent deadlock 425 // if abort happened during library initialization or shutdown 426 __kmp_global.g.g_abort = SIGABRT; 427 428 /* On Windows* OS by default abort() causes pop-up error box, which stalls 429 nightly testing. Unfortunately, we cannot reliably suppress pop-up error 430 boxes. _set_abort_behavior() works well, but this function is not 431 available in VS7 (this is not problem for DLL, but it is a problem for 432 static OpenMP RTL). SetErrorMode (and so, timelimit utility) does not 433 help, at least in some versions of MS C RTL. 434 435 It seems following sequence is the only way to simulate abort() and 436 avoid pop-up error box. */ 437 raise(SIGABRT); 438 _exit(3); // Just in case, if signal ignored, exit anyway. 439 } else { 440 abort(); 441 }; // if 442 443 __kmp_infinite_loop(); 444 __kmp_release_bootstrap_lock(&__kmp_exit_lock); 445 446 } // __kmp_abort_process 447 448 void __kmp_abort_thread(void) { 449 // TODO: Eliminate g_abort global variable and this function. 450 // In case of abort just call abort(), it will kill all the threads. 451 __kmp_infinite_loop(); 452 } // __kmp_abort_thread 453 454 /* Print out the storage map for the major kmp_info_t thread data structures 455 that are allocated together. */ 456 457 static void __kmp_print_thread_storage_map(kmp_info_t *thr, int gtid) { 458 __kmp_print_storage_map_gtid(gtid, thr, thr + 1, sizeof(kmp_info_t), "th_%d", 459 gtid); 460 461 __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team, 462 sizeof(kmp_desc_t), "th_%d.th_info", gtid); 463 464 __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head, 465 sizeof(kmp_local_t), "th_%d.th_local", gtid); 466 467 __kmp_print_storage_map_gtid( 468 gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier], 469 sizeof(kmp_balign_t) * bs_last_barrier, "th_%d.th_bar", gtid); 470 471 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier], 472 &thr->th.th_bar[bs_plain_barrier + 1], 473 sizeof(kmp_balign_t), "th_%d.th_bar[plain]", 474 gtid); 475 476 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier], 477 &thr->th.th_bar[bs_forkjoin_barrier + 1], 478 sizeof(kmp_balign_t), "th_%d.th_bar[forkjoin]", 479 gtid); 480 481 #if KMP_FAST_REDUCTION_BARRIER 482 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier], 483 &thr->th.th_bar[bs_reduction_barrier + 1], 484 sizeof(kmp_balign_t), "th_%d.th_bar[reduction]", 485 gtid); 486 #endif // KMP_FAST_REDUCTION_BARRIER 487 } 488 489 /* Print out the storage map for the major kmp_team_t team data structures 490 that are allocated together. */ 491 492 static void __kmp_print_team_storage_map(const char *header, kmp_team_t *team, 493 int team_id, int num_thr) { 494 int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2; 495 __kmp_print_storage_map_gtid(-1, team, team + 1, sizeof(kmp_team_t), "%s_%d", 496 header, team_id); 497 498 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0], 499 &team->t.t_bar[bs_last_barrier], 500 sizeof(kmp_balign_team_t) * bs_last_barrier, 501 "%s_%d.t_bar", header, team_id); 502 503 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier], 504 &team->t.t_bar[bs_plain_barrier + 1], 505 sizeof(kmp_balign_team_t), "%s_%d.t_bar[plain]", 506 header, team_id); 507 508 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier], 509 &team->t.t_bar[bs_forkjoin_barrier + 1], 510 sizeof(kmp_balign_team_t), 511 "%s_%d.t_bar[forkjoin]", header, team_id); 512 513 #if KMP_FAST_REDUCTION_BARRIER 514 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier], 515 &team->t.t_bar[bs_reduction_barrier + 1], 516 sizeof(kmp_balign_team_t), 517 "%s_%d.t_bar[reduction]", header, team_id); 518 #endif // KMP_FAST_REDUCTION_BARRIER 519 520 __kmp_print_storage_map_gtid( 521 -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr], 522 sizeof(kmp_disp_t) * num_thr, "%s_%d.t_dispatch", header, team_id); 523 524 __kmp_print_storage_map_gtid( 525 -1, &team->t.t_threads[0], &team->t.t_threads[num_thr], 526 sizeof(kmp_info_t *) * num_thr, "%s_%d.t_threads", header, team_id); 527 528 __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0], 529 &team->t.t_disp_buffer[num_disp_buff], 530 sizeof(dispatch_shared_info_t) * num_disp_buff, 531 "%s_%d.t_disp_buffer", header, team_id); 532 533 __kmp_print_storage_map_gtid(-1, &team->t.t_taskq, &team->t.t_copypriv_data, 534 sizeof(kmp_taskq_t), "%s_%d.t_taskq", header, 535 team_id); 536 } 537 538 static void __kmp_init_allocator() {} 539 static void __kmp_fini_allocator() {} 540 541 /* ------------------------------------------------------------------------ */ 542 543 #ifdef KMP_DYNAMIC_LIB 544 #if KMP_OS_WINDOWS 545 546 static void __kmp_reset_lock(kmp_bootstrap_lock_t *lck) { 547 // TODO: Change to __kmp_break_bootstrap_lock(). 548 __kmp_init_bootstrap_lock(lck); // make the lock released 549 } 550 551 static void __kmp_reset_locks_on_process_detach(int gtid_req) { 552 int i; 553 int thread_count; 554 555 // PROCESS_DETACH is expected to be called by a thread that executes 556 // ProcessExit() or FreeLibrary(). OS terminates other threads (except the one 557 // calling ProcessExit or FreeLibrary). So, it might be safe to access the 558 // __kmp_threads[] without taking the forkjoin_lock. However, in fact, some 559 // threads can be still alive here, although being about to be terminated. The 560 // threads in the array with ds_thread==0 are most suspicious. Actually, it 561 // can be not safe to access the __kmp_threads[]. 562 563 // TODO: does it make sense to check __kmp_roots[] ? 564 565 // Let's check that there are no other alive threads registered with the OMP 566 // lib. 567 while (1) { 568 thread_count = 0; 569 for (i = 0; i < __kmp_threads_capacity; ++i) { 570 if (!__kmp_threads) 571 continue; 572 kmp_info_t *th = __kmp_threads[i]; 573 if (th == NULL) 574 continue; 575 int gtid = th->th.th_info.ds.ds_gtid; 576 if (gtid == gtid_req) 577 continue; 578 if (gtid < 0) 579 continue; 580 DWORD exit_val; 581 int alive = __kmp_is_thread_alive(th, &exit_val); 582 if (alive) { 583 ++thread_count; 584 } 585 } 586 if (thread_count == 0) 587 break; // success 588 } 589 590 // Assume that I'm alone. Now it might be safe to check and reset locks. 591 // __kmp_forkjoin_lock and __kmp_stdio_lock are expected to be reset. 592 __kmp_reset_lock(&__kmp_forkjoin_lock); 593 #ifdef KMP_DEBUG 594 __kmp_reset_lock(&__kmp_stdio_lock); 595 #endif // KMP_DEBUG 596 } 597 598 BOOL WINAPI DllMain(HINSTANCE hInstDLL, DWORD fdwReason, LPVOID lpReserved) { 599 //__kmp_acquire_bootstrap_lock( &__kmp_initz_lock ); 600 601 switch (fdwReason) { 602 603 case DLL_PROCESS_ATTACH: 604 KA_TRACE(10, ("DllMain: PROCESS_ATTACH\n")); 605 606 return TRUE; 607 608 case DLL_PROCESS_DETACH: 609 KA_TRACE(10, ("DllMain: PROCESS_DETACH T#%d\n", __kmp_gtid_get_specific())); 610 611 if (lpReserved != NULL) { 612 // lpReserved is used for telling the difference: 613 // lpReserved == NULL when FreeLibrary() was called, 614 // lpReserved != NULL when the process terminates. 615 // When FreeLibrary() is called, worker threads remain alive. So they will 616 // release the forkjoin lock by themselves. When the process terminates, 617 // worker threads disappear triggering the problem of unreleased forkjoin 618 // lock as described below. 619 620 // A worker thread can take the forkjoin lock. The problem comes up if 621 // that worker thread becomes dead before it releases the forkjoin lock. 622 // The forkjoin lock remains taken, while the thread executing 623 // DllMain()->PROCESS_DETACH->__kmp_internal_end_library() below will try 624 // to take the forkjoin lock and will always fail, so that the application 625 // will never finish [normally]. This scenario is possible if 626 // __kmpc_end() has not been executed. It looks like it's not a corner 627 // case, but common cases: 628 // - the main function was compiled by an alternative compiler; 629 // - the main function was compiled by icl but without /Qopenmp 630 // (application with plugins); 631 // - application terminates by calling C exit(), Fortran CALL EXIT() or 632 // Fortran STOP. 633 // - alive foreign thread prevented __kmpc_end from doing cleanup. 634 // 635 // This is a hack to work around the problem. 636 // TODO: !!! figure out something better. 637 __kmp_reset_locks_on_process_detach(__kmp_gtid_get_specific()); 638 } 639 640 __kmp_internal_end_library(__kmp_gtid_get_specific()); 641 642 return TRUE; 643 644 case DLL_THREAD_ATTACH: 645 KA_TRACE(10, ("DllMain: THREAD_ATTACH\n")); 646 647 /* if we want to register new siblings all the time here call 648 * __kmp_get_gtid(); */ 649 return TRUE; 650 651 case DLL_THREAD_DETACH: 652 KA_TRACE(10, ("DllMain: THREAD_DETACH T#%d\n", __kmp_gtid_get_specific())); 653 654 __kmp_internal_end_thread(__kmp_gtid_get_specific()); 655 return TRUE; 656 } 657 658 return TRUE; 659 } 660 661 #endif /* KMP_OS_WINDOWS */ 662 #endif /* KMP_DYNAMIC_LIB */ 663 664 /* Change the library type to "status" and return the old type */ 665 /* called from within initialization routines where __kmp_initz_lock is held */ 666 int __kmp_change_library(int status) { 667 int old_status; 668 669 old_status = __kmp_yield_init & 670 1; // check whether KMP_LIBRARY=throughput (even init count) 671 672 if (status) { 673 __kmp_yield_init |= 1; // throughput => turnaround (odd init count) 674 } else { 675 __kmp_yield_init &= ~1; // turnaround => throughput (even init count) 676 } 677 678 return old_status; // return previous setting of whether 679 // KMP_LIBRARY=throughput 680 } 681 682 /* __kmp_parallel_deo -- Wait until it's our turn. */ 683 void __kmp_parallel_deo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) { 684 int gtid = *gtid_ref; 685 #ifdef BUILD_PARALLEL_ORDERED 686 kmp_team_t *team = __kmp_team_from_gtid(gtid); 687 #endif /* BUILD_PARALLEL_ORDERED */ 688 689 if (__kmp_env_consistency_check) { 690 if (__kmp_threads[gtid]->th.th_root->r.r_active) 691 #if KMP_USE_DYNAMIC_LOCK 692 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL, 0); 693 #else 694 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL); 695 #endif 696 } 697 #ifdef BUILD_PARALLEL_ORDERED 698 if (!team->t.t_serialized) { 699 KMP_MB(); 700 KMP_WAIT_YIELD(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid), 701 KMP_EQ, NULL); 702 KMP_MB(); 703 } 704 #endif /* BUILD_PARALLEL_ORDERED */ 705 } 706 707 /* __kmp_parallel_dxo -- Signal the next task. */ 708 void __kmp_parallel_dxo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) { 709 int gtid = *gtid_ref; 710 #ifdef BUILD_PARALLEL_ORDERED 711 int tid = __kmp_tid_from_gtid(gtid); 712 kmp_team_t *team = __kmp_team_from_gtid(gtid); 713 #endif /* BUILD_PARALLEL_ORDERED */ 714 715 if (__kmp_env_consistency_check) { 716 if (__kmp_threads[gtid]->th.th_root->r.r_active) 717 __kmp_pop_sync(gtid, ct_ordered_in_parallel, loc_ref); 718 } 719 #ifdef BUILD_PARALLEL_ORDERED 720 if (!team->t.t_serialized) { 721 KMP_MB(); /* Flush all pending memory write invalidates. */ 722 723 /* use the tid of the next thread in this team */ 724 /* TODO replace with general release procedure */ 725 team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc); 726 727 #if OMPT_SUPPORT && OMPT_BLAME 728 if (ompt_enabled && 729 ompt_callbacks.ompt_callback(ompt_event_release_ordered)) { 730 /* accept blame for "ordered" waiting */ 731 kmp_info_t *this_thread = __kmp_threads[gtid]; 732 ompt_callbacks.ompt_callback(ompt_event_release_ordered)( 733 this_thread->th.ompt_thread_info.wait_id); 734 } 735 #endif 736 737 KMP_MB(); /* Flush all pending memory write invalidates. */ 738 } 739 #endif /* BUILD_PARALLEL_ORDERED */ 740 } 741 742 /* ------------------------------------------------------------------------ */ 743 /* The BARRIER for a SINGLE process section is always explicit */ 744 745 int __kmp_enter_single(int gtid, ident_t *id_ref, int push_ws) { 746 int status; 747 kmp_info_t *th; 748 kmp_team_t *team; 749 750 if (!TCR_4(__kmp_init_parallel)) 751 __kmp_parallel_initialize(); 752 753 th = __kmp_threads[gtid]; 754 team = th->th.th_team; 755 status = 0; 756 757 th->th.th_ident = id_ref; 758 759 if (team->t.t_serialized) { 760 status = 1; 761 } else { 762 kmp_int32 old_this = th->th.th_local.this_construct; 763 764 ++th->th.th_local.this_construct; 765 /* try to set team count to thread count--success means thread got the 766 single block */ 767 /* TODO: Should this be acquire or release? */ 768 if (team->t.t_construct == old_this) { 769 status = KMP_COMPARE_AND_STORE_ACQ32(&team->t.t_construct, old_this, 770 th->th.th_local.this_construct); 771 } 772 #if USE_ITT_BUILD 773 if (__itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 && 774 KMP_MASTER_GTID(gtid) && 775 #if OMP_40_ENABLED 776 th->th.th_teams_microtask == NULL && 777 #endif 778 team->t.t_active_level == 779 1) { // Only report metadata by master of active team at level 1 780 __kmp_itt_metadata_single(id_ref); 781 } 782 #endif /* USE_ITT_BUILD */ 783 } 784 785 if (__kmp_env_consistency_check) { 786 if (status && push_ws) { 787 __kmp_push_workshare(gtid, ct_psingle, id_ref); 788 } else { 789 __kmp_check_workshare(gtid, ct_psingle, id_ref); 790 } 791 } 792 #if USE_ITT_BUILD 793 if (status) { 794 __kmp_itt_single_start(gtid); 795 } 796 #endif /* USE_ITT_BUILD */ 797 return status; 798 } 799 800 void __kmp_exit_single(int gtid) { 801 #if USE_ITT_BUILD 802 __kmp_itt_single_end(gtid); 803 #endif /* USE_ITT_BUILD */ 804 if (__kmp_env_consistency_check) 805 __kmp_pop_workshare(gtid, ct_psingle, NULL); 806 } 807 808 /* determine if we can go parallel or must use a serialized parallel region and 809 * how many threads we can use 810 * set_nproc is the number of threads requested for the team 811 * returns 0 if we should serialize or only use one thread, 812 * otherwise the number of threads to use 813 * The forkjoin lock is held by the caller. */ 814 static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team, 815 int master_tid, int set_nthreads 816 #if OMP_40_ENABLED 817 , 818 int enter_teams 819 #endif /* OMP_40_ENABLED */ 820 ) { 821 int capacity; 822 int new_nthreads; 823 KMP_DEBUG_ASSERT(__kmp_init_serial); 824 KMP_DEBUG_ASSERT(root && parent_team); 825 826 // If dyn-var is set, dynamically adjust the number of desired threads, 827 // according to the method specified by dynamic_mode. 828 new_nthreads = set_nthreads; 829 if (!get__dynamic_2(parent_team, master_tid)) { 830 ; 831 } 832 #ifdef USE_LOAD_BALANCE 833 else if (__kmp_global.g.g_dynamic_mode == dynamic_load_balance) { 834 new_nthreads = __kmp_load_balance_nproc(root, set_nthreads); 835 if (new_nthreads == 1) { 836 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced " 837 "reservation to 1 thread\n", 838 master_tid)); 839 return 1; 840 } 841 if (new_nthreads < set_nthreads) { 842 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced " 843 "reservation to %d threads\n", 844 master_tid, new_nthreads)); 845 } 846 } 847 #endif /* USE_LOAD_BALANCE */ 848 else if (__kmp_global.g.g_dynamic_mode == dynamic_thread_limit) { 849 new_nthreads = __kmp_avail_proc - __kmp_nth + 850 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc); 851 if (new_nthreads <= 1) { 852 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced " 853 "reservation to 1 thread\n", 854 master_tid)); 855 return 1; 856 } 857 if (new_nthreads < set_nthreads) { 858 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced " 859 "reservation to %d threads\n", 860 master_tid, new_nthreads)); 861 } else { 862 new_nthreads = set_nthreads; 863 } 864 } else if (__kmp_global.g.g_dynamic_mode == dynamic_random) { 865 if (set_nthreads > 2) { 866 new_nthreads = __kmp_get_random(parent_team->t.t_threads[master_tid]); 867 new_nthreads = (new_nthreads % set_nthreads) + 1; 868 if (new_nthreads == 1) { 869 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced " 870 "reservation to 1 thread\n", 871 master_tid)); 872 return 1; 873 } 874 if (new_nthreads < set_nthreads) { 875 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced " 876 "reservation to %d threads\n", 877 master_tid, new_nthreads)); 878 } 879 } 880 } else { 881 KMP_ASSERT(0); 882 } 883 884 // Respect KMP_ALL_THREADS/KMP_DEVICE_THREAD_LIMIT. 885 if (__kmp_nth + new_nthreads - 886 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) > 887 __kmp_max_nth) { 888 int tl_nthreads = __kmp_max_nth - __kmp_nth + 889 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc); 890 if (tl_nthreads <= 0) { 891 tl_nthreads = 1; 892 } 893 894 // If dyn-var is false, emit a 1-time warning. 895 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) { 896 __kmp_reserve_warn = 1; 897 __kmp_msg(kmp_ms_warning, 898 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads), 899 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 900 } 901 if (tl_nthreads == 1) { 902 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT " 903 "reduced reservation to 1 thread\n", 904 master_tid)); 905 return 1; 906 } 907 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced " 908 "reservation to %d threads\n", 909 master_tid, tl_nthreads)); 910 new_nthreads = tl_nthreads; 911 } 912 913 // Respect OMP_THREAD_LIMIT 914 if (root->r.r_cg_nthreads + new_nthreads - 915 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) > 916 __kmp_cg_max_nth) { 917 int tl_nthreads = __kmp_cg_max_nth - root->r.r_cg_nthreads + 918 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc); 919 if (tl_nthreads <= 0) { 920 tl_nthreads = 1; 921 } 922 923 // If dyn-var is false, emit a 1-time warning. 924 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) { 925 __kmp_reserve_warn = 1; 926 __kmp_msg(kmp_ms_warning, 927 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads), 928 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 929 } 930 if (tl_nthreads == 1) { 931 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT " 932 "reduced reservation to 1 thread\n", 933 master_tid)); 934 return 1; 935 } 936 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced " 937 "reservation to %d threads\n", 938 master_tid, tl_nthreads)); 939 new_nthreads = tl_nthreads; 940 } 941 942 // Check if the threads array is large enough, or needs expanding. 943 // See comment in __kmp_register_root() about the adjustment if 944 // __kmp_threads[0] == NULL. 945 capacity = __kmp_threads_capacity; 946 if (TCR_PTR(__kmp_threads[0]) == NULL) { 947 --capacity; 948 } 949 if (__kmp_nth + new_nthreads - 950 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) > 951 capacity) { 952 // Expand the threads array. 953 int slotsRequired = __kmp_nth + new_nthreads - 954 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) - 955 capacity; 956 int slotsAdded = __kmp_expand_threads(slotsRequired, slotsRequired); 957 if (slotsAdded < slotsRequired) { 958 // The threads array was not expanded enough. 959 new_nthreads -= (slotsRequired - slotsAdded); 960 KMP_ASSERT(new_nthreads >= 1); 961 962 // If dyn-var is false, emit a 1-time warning. 963 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) { 964 __kmp_reserve_warn = 1; 965 if (__kmp_tp_cached) { 966 __kmp_msg(kmp_ms_warning, 967 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads), 968 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity), 969 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null); 970 } else { 971 __kmp_msg(kmp_ms_warning, 972 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads), 973 KMP_HNT(SystemLimitOnThreads), __kmp_msg_null); 974 } 975 } 976 } 977 } 978 979 #ifdef KMP_DEBUG 980 if (new_nthreads == 1) { 981 KC_TRACE(10, 982 ("__kmp_reserve_threads: T#%d serializing team after reclaiming " 983 "dead roots and rechecking; requested %d threads\n", 984 __kmp_get_gtid(), set_nthreads)); 985 } else { 986 KC_TRACE(10, ("__kmp_reserve_threads: T#%d allocating %d threads; requested" 987 " %d threads\n", 988 __kmp_get_gtid(), new_nthreads, set_nthreads)); 989 } 990 #endif // KMP_DEBUG 991 return new_nthreads; 992 } 993 994 /* Allocate threads from the thread pool and assign them to the new team. We are 995 assured that there are enough threads available, because we checked on that 996 earlier within critical section forkjoin */ 997 static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team, 998 kmp_info_t *master_th, int master_gtid) { 999 int i; 1000 int use_hot_team; 1001 1002 KA_TRACE(10, ("__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc)); 1003 KMP_DEBUG_ASSERT(master_gtid == __kmp_get_gtid()); 1004 KMP_MB(); 1005 1006 /* first, let's setup the master thread */ 1007 master_th->th.th_info.ds.ds_tid = 0; 1008 master_th->th.th_team = team; 1009 master_th->th.th_team_nproc = team->t.t_nproc; 1010 master_th->th.th_team_master = master_th; 1011 master_th->th.th_team_serialized = FALSE; 1012 master_th->th.th_dispatch = &team->t.t_dispatch[0]; 1013 1014 /* make sure we are not the optimized hot team */ 1015 #if KMP_NESTED_HOT_TEAMS 1016 use_hot_team = 0; 1017 kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams; 1018 if (hot_teams) { // hot teams array is not allocated if 1019 // KMP_HOT_TEAMS_MAX_LEVEL=0 1020 int level = team->t.t_active_level - 1; // index in array of hot teams 1021 if (master_th->th.th_teams_microtask) { // are we inside the teams? 1022 if (master_th->th.th_teams_size.nteams > 1) { 1023 ++level; // level was not increased in teams construct for 1024 // team_of_masters 1025 } 1026 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master && 1027 master_th->th.th_teams_level == team->t.t_level) { 1028 ++level; // level was not increased in teams construct for 1029 // team_of_workers before the parallel 1030 } // team->t.t_level will be increased inside parallel 1031 } 1032 if (level < __kmp_hot_teams_max_level) { 1033 if (hot_teams[level].hot_team) { 1034 // hot team has already been allocated for given level 1035 KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team); 1036 use_hot_team = 1; // the team is ready to use 1037 } else { 1038 use_hot_team = 0; // AC: threads are not allocated yet 1039 hot_teams[level].hot_team = team; // remember new hot team 1040 hot_teams[level].hot_team_nth = team->t.t_nproc; 1041 } 1042 } else { 1043 use_hot_team = 0; 1044 } 1045 } 1046 #else 1047 use_hot_team = team == root->r.r_hot_team; 1048 #endif 1049 if (!use_hot_team) { 1050 1051 /* install the master thread */ 1052 team->t.t_threads[0] = master_th; 1053 __kmp_initialize_info(master_th, team, 0, master_gtid); 1054 1055 /* now, install the worker threads */ 1056 for (i = 1; i < team->t.t_nproc; i++) { 1057 1058 /* fork or reallocate a new thread and install it in team */ 1059 kmp_info_t *thr = __kmp_allocate_thread(root, team, i); 1060 team->t.t_threads[i] = thr; 1061 KMP_DEBUG_ASSERT(thr); 1062 KMP_DEBUG_ASSERT(thr->th.th_team == team); 1063 /* align team and thread arrived states */ 1064 KA_TRACE(20, ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived " 1065 "T#%d(%d:%d) join =%llu, plain=%llu\n", 1066 __kmp_gtid_from_tid(0, team), team->t.t_id, 0, 1067 __kmp_gtid_from_tid(i, team), team->t.t_id, i, 1068 team->t.t_bar[bs_forkjoin_barrier].b_arrived, 1069 team->t.t_bar[bs_plain_barrier].b_arrived)); 1070 #if OMP_40_ENABLED 1071 thr->th.th_teams_microtask = master_th->th.th_teams_microtask; 1072 thr->th.th_teams_level = master_th->th.th_teams_level; 1073 thr->th.th_teams_size = master_th->th.th_teams_size; 1074 #endif 1075 { // Initialize threads' barrier data. 1076 int b; 1077 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar; 1078 for (b = 0; b < bs_last_barrier; ++b) { 1079 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 1080 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 1081 #if USE_DEBUGGER 1082 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 1083 #endif 1084 }; // for b 1085 } 1086 } 1087 1088 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 1089 __kmp_partition_places(team); 1090 #endif 1091 } 1092 1093 KMP_MB(); 1094 } 1095 1096 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1097 // Propagate any changes to the floating point control registers out to the team 1098 // We try to avoid unnecessary writes to the relevant cache line in the team 1099 // structure, so we don't make changes unless they are needed. 1100 inline static void propagateFPControl(kmp_team_t *team) { 1101 if (__kmp_inherit_fp_control) { 1102 kmp_int16 x87_fpu_control_word; 1103 kmp_uint32 mxcsr; 1104 1105 // Get master values of FPU control flags (both X87 and vector) 1106 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word); 1107 __kmp_store_mxcsr(&mxcsr); 1108 mxcsr &= KMP_X86_MXCSR_MASK; 1109 1110 // There is no point looking at t_fp_control_saved here. 1111 // If it is TRUE, we still have to update the values if they are different from 1112 // those we now have. 1113 // If it is FALSE we didn't save anything yet, but our objective is the same. We 1114 // have to ensure that the values in the team are the same as those we have. 1115 // So, this code achieves what we need whether or not t_fp_control_saved is 1116 // true. By checking whether the value needs updating we avoid unnecessary 1117 // writes that would put the cache-line into a written state, causing all 1118 // threads in the team to have to read it again. 1119 KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word); 1120 KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr); 1121 // Although we don't use this value, other code in the runtime wants to know 1122 // whether it should restore them. So we must ensure it is correct. 1123 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE); 1124 } else { 1125 // Similarly here. Don't write to this cache-line in the team structure 1126 // unless we have to. 1127 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE); 1128 } 1129 } 1130 1131 // Do the opposite, setting the hardware registers to the updated values from 1132 // the team. 1133 inline static void updateHWFPControl(kmp_team_t *team) { 1134 if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) { 1135 // Only reset the fp control regs if they have been changed in the team. 1136 // the parallel region that we are exiting. 1137 kmp_int16 x87_fpu_control_word; 1138 kmp_uint32 mxcsr; 1139 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word); 1140 __kmp_store_mxcsr(&mxcsr); 1141 mxcsr &= KMP_X86_MXCSR_MASK; 1142 1143 if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) { 1144 __kmp_clear_x87_fpu_status_word(); 1145 __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word); 1146 } 1147 1148 if (team->t.t_mxcsr != mxcsr) { 1149 __kmp_load_mxcsr(&team->t.t_mxcsr); 1150 } 1151 } 1152 } 1153 #else 1154 #define propagateFPControl(x) ((void)0) 1155 #define updateHWFPControl(x) ((void)0) 1156 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 1157 1158 static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team, 1159 int realloc); // forward declaration 1160 1161 /* Run a parallel region that has been serialized, so runs only in a team of the 1162 single master thread. */ 1163 void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) { 1164 kmp_info_t *this_thr; 1165 kmp_team_t *serial_team; 1166 1167 KC_TRACE(10, ("__kmpc_serialized_parallel: called by T#%d\n", global_tid)); 1168 1169 /* Skip all this code for autopar serialized loops since it results in 1170 unacceptable overhead */ 1171 if (loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR)) 1172 return; 1173 1174 if (!TCR_4(__kmp_init_parallel)) 1175 __kmp_parallel_initialize(); 1176 1177 this_thr = __kmp_threads[global_tid]; 1178 serial_team = this_thr->th.th_serial_team; 1179 1180 /* utilize the serialized team held by this thread */ 1181 KMP_DEBUG_ASSERT(serial_team); 1182 KMP_MB(); 1183 1184 if (__kmp_tasking_mode != tskm_immediate_exec) { 1185 KMP_DEBUG_ASSERT( 1186 this_thr->th.th_task_team == 1187 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]); 1188 KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] == 1189 NULL); 1190 KA_TRACE(20, ("__kmpc_serialized_parallel: T#%d pushing task_team %p / " 1191 "team %p, new task_team = NULL\n", 1192 global_tid, this_thr->th.th_task_team, this_thr->th.th_team)); 1193 this_thr->th.th_task_team = NULL; 1194 } 1195 1196 #if OMP_40_ENABLED 1197 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind; 1198 if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) { 1199 proc_bind = proc_bind_false; 1200 } else if (proc_bind == proc_bind_default) { 1201 // No proc_bind clause was specified, so use the current value 1202 // of proc-bind-var for this parallel region. 1203 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind; 1204 } 1205 // Reset for next parallel region 1206 this_thr->th.th_set_proc_bind = proc_bind_default; 1207 #endif /* OMP_40_ENABLED */ 1208 1209 if (this_thr->th.th_team != serial_team) { 1210 // Nested level will be an index in the nested nthreads array 1211 int level = this_thr->th.th_team->t.t_level; 1212 1213 if (serial_team->t.t_serialized) { 1214 /* this serial team was already used 1215 TODO increase performance by making this locks more specific */ 1216 kmp_team_t *new_team; 1217 1218 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 1219 1220 #if OMPT_SUPPORT 1221 ompt_parallel_id_t ompt_parallel_id = __ompt_parallel_id_new(global_tid); 1222 #endif 1223 1224 new_team = __kmp_allocate_team(this_thr->th.th_root, 1, 1, 1225 #if OMPT_SUPPORT 1226 ompt_parallel_id, 1227 #endif 1228 #if OMP_40_ENABLED 1229 proc_bind, 1230 #endif 1231 &this_thr->th.th_current_task->td_icvs, 1232 0 USE_NESTED_HOT_ARG(NULL)); 1233 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 1234 KMP_ASSERT(new_team); 1235 1236 /* setup new serialized team and install it */ 1237 new_team->t.t_threads[0] = this_thr; 1238 new_team->t.t_parent = this_thr->th.th_team; 1239 serial_team = new_team; 1240 this_thr->th.th_serial_team = serial_team; 1241 1242 KF_TRACE( 1243 10, 1244 ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n", 1245 global_tid, serial_team)); 1246 1247 /* TODO the above breaks the requirement that if we run out of resources, 1248 then we can still guarantee that serialized teams are ok, since we may 1249 need to allocate a new one */ 1250 } else { 1251 KF_TRACE( 1252 10, 1253 ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n", 1254 global_tid, serial_team)); 1255 } 1256 1257 /* we have to initialize this serial team */ 1258 KMP_DEBUG_ASSERT(serial_team->t.t_threads); 1259 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr); 1260 KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team); 1261 serial_team->t.t_ident = loc; 1262 serial_team->t.t_serialized = 1; 1263 serial_team->t.t_nproc = 1; 1264 serial_team->t.t_parent = this_thr->th.th_team; 1265 serial_team->t.t_sched = this_thr->th.th_team->t.t_sched; 1266 this_thr->th.th_team = serial_team; 1267 serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid; 1268 1269 KF_TRACE(10, ("__kmpc_serialized_parallel: T#d curtask=%p\n", global_tid, 1270 this_thr->th.th_current_task)); 1271 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1); 1272 this_thr->th.th_current_task->td_flags.executing = 0; 1273 1274 __kmp_push_current_task_to_thread(this_thr, serial_team, 0); 1275 1276 /* TODO: GEH: do ICVs work for nested serialized teams? Don't we need an 1277 implicit task for each serialized task represented by 1278 team->t.t_serialized? */ 1279 copy_icvs(&this_thr->th.th_current_task->td_icvs, 1280 &this_thr->th.th_current_task->td_parent->td_icvs); 1281 1282 // Thread value exists in the nested nthreads array for the next nested 1283 // level 1284 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) { 1285 this_thr->th.th_current_task->td_icvs.nproc = 1286 __kmp_nested_nth.nth[level + 1]; 1287 } 1288 1289 #if OMP_40_ENABLED 1290 if (__kmp_nested_proc_bind.used && 1291 (level + 1 < __kmp_nested_proc_bind.used)) { 1292 this_thr->th.th_current_task->td_icvs.proc_bind = 1293 __kmp_nested_proc_bind.bind_types[level + 1]; 1294 } 1295 #endif /* OMP_40_ENABLED */ 1296 1297 #if USE_DEBUGGER 1298 serial_team->t.t_pkfn = (microtask_t)(~0); // For the debugger. 1299 #endif 1300 this_thr->th.th_info.ds.ds_tid = 0; 1301 1302 /* set thread cache values */ 1303 this_thr->th.th_team_nproc = 1; 1304 this_thr->th.th_team_master = this_thr; 1305 this_thr->th.th_team_serialized = 1; 1306 1307 serial_team->t.t_level = serial_team->t.t_parent->t.t_level + 1; 1308 serial_team->t.t_active_level = serial_team->t.t_parent->t.t_active_level; 1309 1310 propagateFPControl(serial_team); 1311 1312 /* check if we need to allocate dispatch buffers stack */ 1313 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch); 1314 if (!serial_team->t.t_dispatch->th_disp_buffer) { 1315 serial_team->t.t_dispatch->th_disp_buffer = 1316 (dispatch_private_info_t *)__kmp_allocate( 1317 sizeof(dispatch_private_info_t)); 1318 } 1319 this_thr->th.th_dispatch = serial_team->t.t_dispatch; 1320 1321 #if OMPT_SUPPORT 1322 ompt_parallel_id_t ompt_parallel_id = __ompt_parallel_id_new(global_tid); 1323 __ompt_team_assign_id(serial_team, ompt_parallel_id); 1324 #endif 1325 1326 KMP_MB(); 1327 1328 } else { 1329 /* this serialized team is already being used, 1330 * that's fine, just add another nested level */ 1331 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team); 1332 KMP_DEBUG_ASSERT(serial_team->t.t_threads); 1333 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr); 1334 ++serial_team->t.t_serialized; 1335 this_thr->th.th_team_serialized = serial_team->t.t_serialized; 1336 1337 // Nested level will be an index in the nested nthreads array 1338 int level = this_thr->th.th_team->t.t_level; 1339 // Thread value exists in the nested nthreads array for the next nested 1340 // level 1341 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) { 1342 this_thr->th.th_current_task->td_icvs.nproc = 1343 __kmp_nested_nth.nth[level + 1]; 1344 } 1345 serial_team->t.t_level++; 1346 KF_TRACE(10, ("__kmpc_serialized_parallel: T#%d increasing nesting level " 1347 "of serial team %p to %d\n", 1348 global_tid, serial_team, serial_team->t.t_level)); 1349 1350 /* allocate/push dispatch buffers stack */ 1351 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch); 1352 { 1353 dispatch_private_info_t *disp_buffer = 1354 (dispatch_private_info_t *)__kmp_allocate( 1355 sizeof(dispatch_private_info_t)); 1356 disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer; 1357 serial_team->t.t_dispatch->th_disp_buffer = disp_buffer; 1358 } 1359 this_thr->th.th_dispatch = serial_team->t.t_dispatch; 1360 1361 KMP_MB(); 1362 } 1363 #if OMP_40_ENABLED 1364 KMP_CHECK_UPDATE(serial_team->t.t_cancel_request, cancel_noreq); 1365 #endif 1366 1367 if (__kmp_env_consistency_check) 1368 __kmp_push_parallel(global_tid, NULL); 1369 } 1370 1371 /* most of the work for a fork */ 1372 /* return true if we really went parallel, false if serialized */ 1373 int __kmp_fork_call(ident_t *loc, int gtid, 1374 enum fork_context_e call_context, // Intel, GNU, ... 1375 kmp_int32 argc, 1376 #if OMPT_SUPPORT 1377 void *unwrapped_task, 1378 #endif 1379 microtask_t microtask, launch_t invoker, 1380 /* TODO: revert workaround for Intel(R) 64 tracker #96 */ 1381 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 1382 va_list *ap 1383 #else 1384 va_list ap 1385 #endif 1386 ) { 1387 void **argv; 1388 int i; 1389 int master_tid; 1390 int master_this_cons; 1391 kmp_team_t *team; 1392 kmp_team_t *parent_team; 1393 kmp_info_t *master_th; 1394 kmp_root_t *root; 1395 int nthreads; 1396 int master_active; 1397 int master_set_numthreads; 1398 int level; 1399 #if OMP_40_ENABLED 1400 int active_level; 1401 int teams_level; 1402 #endif 1403 #if KMP_NESTED_HOT_TEAMS 1404 kmp_hot_team_ptr_t **p_hot_teams; 1405 #endif 1406 { // KMP_TIME_BLOCK 1407 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_fork_call); 1408 KMP_COUNT_VALUE(OMP_PARALLEL_args, argc); 1409 1410 KA_TRACE(20, ("__kmp_fork_call: enter T#%d\n", gtid)); 1411 if (__kmp_stkpadding > 0 && __kmp_root[gtid] != NULL) { 1412 /* Some systems prefer the stack for the root thread(s) to start with */ 1413 /* some gap from the parent stack to prevent false sharing. */ 1414 void *dummy = KMP_ALLOCA(__kmp_stkpadding); 1415 /* These 2 lines below are so this does not get optimized out */ 1416 if (__kmp_stkpadding > KMP_MAX_STKPADDING) 1417 __kmp_stkpadding += (short)((kmp_int64)dummy); 1418 } 1419 1420 /* initialize if needed */ 1421 KMP_DEBUG_ASSERT( 1422 __kmp_init_serial); // AC: potentially unsafe, not in sync with shutdown 1423 if (!TCR_4(__kmp_init_parallel)) 1424 __kmp_parallel_initialize(); 1425 1426 /* setup current data */ 1427 master_th = __kmp_threads[gtid]; // AC: potentially unsafe, not in sync with 1428 // shutdown 1429 parent_team = master_th->th.th_team; 1430 master_tid = master_th->th.th_info.ds.ds_tid; 1431 master_this_cons = master_th->th.th_local.this_construct; 1432 root = master_th->th.th_root; 1433 master_active = root->r.r_active; 1434 master_set_numthreads = master_th->th.th_set_nproc; 1435 1436 #if OMPT_SUPPORT 1437 ompt_parallel_id_t ompt_parallel_id; 1438 ompt_task_id_t ompt_task_id; 1439 ompt_frame_t *ompt_frame; 1440 ompt_task_id_t my_task_id; 1441 ompt_parallel_id_t my_parallel_id; 1442 1443 if (ompt_enabled) { 1444 ompt_parallel_id = __ompt_parallel_id_new(gtid); 1445 ompt_task_id = __ompt_get_task_id_internal(0); 1446 ompt_frame = __ompt_get_task_frame_internal(0); 1447 } 1448 #endif 1449 1450 // Nested level will be an index in the nested nthreads array 1451 level = parent_team->t.t_level; 1452 // used to launch non-serial teams even if nested is not allowed 1453 active_level = parent_team->t.t_active_level; 1454 #if OMP_40_ENABLED 1455 // needed to check nesting inside the teams 1456 teams_level = master_th->th.th_teams_level; 1457 #endif 1458 #if KMP_NESTED_HOT_TEAMS 1459 p_hot_teams = &master_th->th.th_hot_teams; 1460 if (*p_hot_teams == NULL && __kmp_hot_teams_max_level > 0) { 1461 *p_hot_teams = (kmp_hot_team_ptr_t *)__kmp_allocate( 1462 sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level); 1463 (*p_hot_teams)[0].hot_team = root->r.r_hot_team; 1464 // it is either actual or not needed (when active_level > 0) 1465 (*p_hot_teams)[0].hot_team_nth = 1; 1466 } 1467 #endif 1468 1469 #if OMPT_SUPPORT 1470 if (ompt_enabled && 1471 ompt_callbacks.ompt_callback(ompt_event_parallel_begin)) { 1472 int team_size = master_set_numthreads; 1473 1474 ompt_callbacks.ompt_callback(ompt_event_parallel_begin)( 1475 ompt_task_id, ompt_frame, ompt_parallel_id, team_size, unwrapped_task, 1476 OMPT_INVOKER(call_context)); 1477 } 1478 #endif 1479 1480 master_th->th.th_ident = loc; 1481 1482 #if OMP_40_ENABLED 1483 if (master_th->th.th_teams_microtask && ap && 1484 microtask != (microtask_t)__kmp_teams_master && level == teams_level) { 1485 // AC: This is start of parallel that is nested inside teams construct. 1486 // The team is actual (hot), all workers are ready at the fork barrier. 1487 // No lock needed to initialize the team a bit, then free workers. 1488 parent_team->t.t_ident = loc; 1489 __kmp_alloc_argv_entries(argc, parent_team, TRUE); 1490 parent_team->t.t_argc = argc; 1491 argv = (void **)parent_team->t.t_argv; 1492 for (i = argc - 1; i >= 0; --i) 1493 /* TODO: revert workaround for Intel(R) 64 tracker #96 */ 1494 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 1495 *argv++ = va_arg(*ap, void *); 1496 #else 1497 *argv++ = va_arg(ap, void *); 1498 #endif 1499 // Increment our nested depth levels, but not increase the serialization 1500 if (parent_team == master_th->th.th_serial_team) { 1501 // AC: we are in serialized parallel 1502 __kmpc_serialized_parallel(loc, gtid); 1503 KMP_DEBUG_ASSERT(parent_team->t.t_serialized > 1); 1504 // AC: need this in order enquiry functions work 1505 // correctly, will restore at join time 1506 parent_team->t.t_serialized--; 1507 #if OMPT_SUPPORT 1508 void *dummy; 1509 void **exit_runtime_p; 1510 1511 ompt_lw_taskteam_t lw_taskteam; 1512 1513 if (ompt_enabled) { 1514 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid, unwrapped_task, 1515 ompt_parallel_id); 1516 lw_taskteam.ompt_task_info.task_id = __ompt_task_id_new(gtid); 1517 exit_runtime_p = 1518 &(lw_taskteam.ompt_task_info.frame.exit_runtime_frame); 1519 1520 __ompt_lw_taskteam_link(&lw_taskteam, master_th); 1521 1522 #if OMPT_TRACE 1523 /* OMPT implicit task begin */ 1524 my_task_id = lw_taskteam.ompt_task_info.task_id; 1525 my_parallel_id = parent_team->t.ompt_team_info.parallel_id; 1526 if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)) { 1527 ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)( 1528 my_parallel_id, my_task_id); 1529 } 1530 #endif 1531 1532 /* OMPT state */ 1533 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; 1534 } else { 1535 exit_runtime_p = &dummy; 1536 } 1537 #endif 1538 1539 { 1540 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 1541 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 1542 __kmp_invoke_microtask(microtask, gtid, 0, argc, parent_team->t.t_argv 1543 #if OMPT_SUPPORT 1544 , 1545 exit_runtime_p 1546 #endif 1547 ); 1548 } 1549 1550 #if OMPT_SUPPORT 1551 *exit_runtime_p = NULL; 1552 if (ompt_enabled) { 1553 #if OMPT_TRACE 1554 lw_taskteam.ompt_task_info.frame.exit_runtime_frame = NULL; 1555 1556 if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) { 1557 ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)( 1558 ompt_parallel_id, ompt_task_id); 1559 } 1560 1561 __ompt_lw_taskteam_unlink(master_th); 1562 // reset clear the task id only after unlinking the task 1563 lw_taskteam.ompt_task_info.task_id = ompt_task_id_none; 1564 #endif 1565 1566 if (ompt_callbacks.ompt_callback(ompt_event_parallel_end)) { 1567 ompt_callbacks.ompt_callback(ompt_event_parallel_end)( 1568 ompt_parallel_id, ompt_task_id, OMPT_INVOKER(call_context)); 1569 } 1570 master_th->th.ompt_thread_info.state = ompt_state_overhead; 1571 } 1572 #endif 1573 return TRUE; 1574 } 1575 1576 parent_team->t.t_pkfn = microtask; 1577 #if OMPT_SUPPORT 1578 parent_team->t.ompt_team_info.microtask = unwrapped_task; 1579 #endif 1580 parent_team->t.t_invoke = invoker; 1581 KMP_TEST_THEN_INC32((kmp_int32 *)&root->r.r_in_parallel); 1582 parent_team->t.t_active_level++; 1583 parent_team->t.t_level++; 1584 1585 /* Change number of threads in the team if requested */ 1586 if (master_set_numthreads) { // The parallel has num_threads clause 1587 if (master_set_numthreads < master_th->th.th_teams_size.nth) { 1588 // AC: only can reduce number of threads dynamically, can't increase 1589 kmp_info_t **other_threads = parent_team->t.t_threads; 1590 parent_team->t.t_nproc = master_set_numthreads; 1591 for (i = 0; i < master_set_numthreads; ++i) { 1592 other_threads[i]->th.th_team_nproc = master_set_numthreads; 1593 } 1594 // Keep extra threads hot in the team for possible next parallels 1595 } 1596 master_th->th.th_set_nproc = 0; 1597 } 1598 1599 #if USE_DEBUGGER 1600 if (__kmp_debugging) { // Let debugger override number of threads. 1601 int nth = __kmp_omp_num_threads(loc); 1602 if (nth > 0) { // 0 means debugger doesn't want to change num threads 1603 master_set_numthreads = nth; 1604 }; // if 1605 }; // if 1606 #endif 1607 1608 KF_TRACE(10, ("__kmp_fork_call: before internal fork: root=%p, team=%p, " 1609 "master_th=%p, gtid=%d\n", 1610 root, parent_team, master_th, gtid)); 1611 __kmp_internal_fork(loc, gtid, parent_team); 1612 KF_TRACE(10, ("__kmp_fork_call: after internal fork: root=%p, team=%p, " 1613 "master_th=%p, gtid=%d\n", 1614 root, parent_team, master_th, gtid)); 1615 1616 /* Invoke microtask for MASTER thread */ 1617 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid, 1618 parent_team->t.t_id, parent_team->t.t_pkfn)); 1619 1620 { 1621 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 1622 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 1623 if (!parent_team->t.t_invoke(gtid)) { 1624 KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread"); 1625 } 1626 } 1627 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid, 1628 parent_team->t.t_id, parent_team->t.t_pkfn)); 1629 KMP_MB(); /* Flush all pending memory write invalidates. */ 1630 1631 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid)); 1632 1633 return TRUE; 1634 } // Parallel closely nested in teams construct 1635 #endif /* OMP_40_ENABLED */ 1636 1637 #if KMP_DEBUG 1638 if (__kmp_tasking_mode != tskm_immediate_exec) { 1639 KMP_DEBUG_ASSERT(master_th->th.th_task_team == 1640 parent_team->t.t_task_team[master_th->th.th_task_state]); 1641 } 1642 #endif 1643 1644 if (parent_team->t.t_active_level >= 1645 master_th->th.th_current_task->td_icvs.max_active_levels) { 1646 nthreads = 1; 1647 } else { 1648 #if OMP_40_ENABLED 1649 int enter_teams = ((ap == NULL && active_level == 0) || 1650 (ap && teams_level > 0 && teams_level == level)); 1651 #endif 1652 nthreads = 1653 master_set_numthreads 1654 ? master_set_numthreads 1655 : get__nproc_2( 1656 parent_team, 1657 master_tid); // TODO: get nproc directly from current task 1658 1659 // Check if we need to take forkjoin lock? (no need for serialized 1660 // parallel out of teams construct). This code moved here from 1661 // __kmp_reserve_threads() to speedup nested serialized parallels. 1662 if (nthreads > 1) { 1663 if ((!get__nested(master_th) && (root->r.r_in_parallel 1664 #if OMP_40_ENABLED 1665 && !enter_teams 1666 #endif /* OMP_40_ENABLED */ 1667 )) || 1668 (__kmp_library == library_serial)) { 1669 KC_TRACE(10, ("__kmp_fork_call: T#%d serializing team; requested %d" 1670 " threads\n", 1671 gtid, nthreads)); 1672 nthreads = 1; 1673 } 1674 } 1675 if (nthreads > 1) { 1676 /* determine how many new threads we can use */ 1677 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 1678 nthreads = __kmp_reserve_threads( 1679 root, parent_team, master_tid, nthreads 1680 #if OMP_40_ENABLED 1681 /* AC: If we execute teams from parallel region (on host), then 1682 teams should be created but each can only have 1 thread if 1683 nesting is disabled. If teams called from serial region, then 1684 teams and their threads should be created regardless of the 1685 nesting setting. */ 1686 , 1687 enter_teams 1688 #endif /* OMP_40_ENABLED */ 1689 ); 1690 if (nthreads == 1) { 1691 // Free lock for single thread execution here; for multi-thread 1692 // execution it will be freed later after team of threads created 1693 // and initialized 1694 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 1695 } 1696 } 1697 } 1698 KMP_DEBUG_ASSERT(nthreads > 0); 1699 1700 // If we temporarily changed the set number of threads then restore it now 1701 master_th->th.th_set_nproc = 0; 1702 1703 /* create a serialized parallel region? */ 1704 if (nthreads == 1) { 1705 /* josh todo: hypothetical question: what do we do for OS X*? */ 1706 #if KMP_OS_LINUX && \ 1707 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 1708 void *args[argc]; 1709 #else 1710 void **args = (void **)KMP_ALLOCA(argc * sizeof(void *)); 1711 #endif /* KMP_OS_LINUX && ( KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || \ 1712 KMP_ARCH_AARCH64) */ 1713 1714 KA_TRACE(20, 1715 ("__kmp_fork_call: T#%d serializing parallel region\n", gtid)); 1716 1717 __kmpc_serialized_parallel(loc, gtid); 1718 1719 if (call_context == fork_context_intel) { 1720 /* TODO this sucks, use the compiler itself to pass args! :) */ 1721 master_th->th.th_serial_team->t.t_ident = loc; 1722 #if OMP_40_ENABLED 1723 if (!ap) { 1724 // revert change made in __kmpc_serialized_parallel() 1725 master_th->th.th_serial_team->t.t_level--; 1726 // Get args from parent team for teams construct 1727 1728 #if OMPT_SUPPORT 1729 void *dummy; 1730 void **exit_runtime_p; 1731 1732 ompt_lw_taskteam_t lw_taskteam; 1733 1734 if (ompt_enabled) { 1735 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid, 1736 unwrapped_task, ompt_parallel_id); 1737 lw_taskteam.ompt_task_info.task_id = __ompt_task_id_new(gtid); 1738 exit_runtime_p = 1739 &(lw_taskteam.ompt_task_info.frame.exit_runtime_frame); 1740 1741 __ompt_lw_taskteam_link(&lw_taskteam, master_th); 1742 1743 #if OMPT_TRACE 1744 my_task_id = lw_taskteam.ompt_task_info.task_id; 1745 if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)) { 1746 ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)( 1747 ompt_parallel_id, my_task_id); 1748 } 1749 #endif 1750 1751 /* OMPT state */ 1752 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; 1753 } else { 1754 exit_runtime_p = &dummy; 1755 } 1756 #endif 1757 1758 { 1759 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 1760 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 1761 __kmp_invoke_microtask(microtask, gtid, 0, argc, 1762 parent_team->t.t_argv 1763 #if OMPT_SUPPORT 1764 , 1765 exit_runtime_p 1766 #endif 1767 ); 1768 } 1769 1770 #if OMPT_SUPPORT 1771 *exit_runtime_p = NULL; 1772 if (ompt_enabled) { 1773 lw_taskteam.ompt_task_info.frame.exit_runtime_frame = NULL; 1774 1775 #if OMPT_TRACE 1776 if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) { 1777 ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)( 1778 ompt_parallel_id, ompt_task_id); 1779 } 1780 #endif 1781 1782 __ompt_lw_taskteam_unlink(master_th); 1783 // reset clear the task id only after unlinking the task 1784 lw_taskteam.ompt_task_info.task_id = ompt_task_id_none; 1785 1786 if (ompt_callbacks.ompt_callback(ompt_event_parallel_end)) { 1787 ompt_callbacks.ompt_callback(ompt_event_parallel_end)( 1788 ompt_parallel_id, ompt_task_id, OMPT_INVOKER(call_context)); 1789 } 1790 master_th->th.ompt_thread_info.state = ompt_state_overhead; 1791 } 1792 #endif 1793 } else if (microtask == (microtask_t)__kmp_teams_master) { 1794 KMP_DEBUG_ASSERT(master_th->th.th_team == 1795 master_th->th.th_serial_team); 1796 team = master_th->th.th_team; 1797 // team->t.t_pkfn = microtask; 1798 team->t.t_invoke = invoker; 1799 __kmp_alloc_argv_entries(argc, team, TRUE); 1800 team->t.t_argc = argc; 1801 argv = (void **)team->t.t_argv; 1802 if (ap) { 1803 for (i = argc - 1; i >= 0; --i) 1804 // TODO: revert workaround for Intel(R) 64 tracker #96 1805 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 1806 *argv++ = va_arg(*ap, void *); 1807 #else 1808 *argv++ = va_arg(ap, void *); 1809 #endif 1810 } else { 1811 for (i = 0; i < argc; ++i) 1812 // Get args from parent team for teams construct 1813 argv[i] = parent_team->t.t_argv[i]; 1814 } 1815 // AC: revert change made in __kmpc_serialized_parallel() 1816 // because initial code in teams should have level=0 1817 team->t.t_level--; 1818 // AC: call special invoker for outer "parallel" of teams construct 1819 { 1820 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 1821 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 1822 invoker(gtid); 1823 } 1824 } else { 1825 #endif /* OMP_40_ENABLED */ 1826 argv = args; 1827 for (i = argc - 1; i >= 0; --i) 1828 // TODO: revert workaround for Intel(R) 64 tracker #96 1829 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 1830 *argv++ = va_arg(*ap, void *); 1831 #else 1832 *argv++ = va_arg(ap, void *); 1833 #endif 1834 KMP_MB(); 1835 1836 #if OMPT_SUPPORT 1837 void *dummy; 1838 void **exit_runtime_p; 1839 1840 ompt_lw_taskteam_t lw_taskteam; 1841 1842 if (ompt_enabled) { 1843 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid, 1844 unwrapped_task, ompt_parallel_id); 1845 lw_taskteam.ompt_task_info.task_id = __ompt_task_id_new(gtid); 1846 exit_runtime_p = 1847 &(lw_taskteam.ompt_task_info.frame.exit_runtime_frame); 1848 1849 __ompt_lw_taskteam_link(&lw_taskteam, master_th); 1850 1851 #if OMPT_TRACE 1852 /* OMPT implicit task begin */ 1853 my_task_id = lw_taskteam.ompt_task_info.task_id; 1854 my_parallel_id = ompt_parallel_id; 1855 if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)) { 1856 ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)( 1857 my_parallel_id, my_task_id); 1858 } 1859 #endif 1860 1861 /* OMPT state */ 1862 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; 1863 } else { 1864 exit_runtime_p = &dummy; 1865 } 1866 #endif 1867 1868 { 1869 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 1870 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 1871 __kmp_invoke_microtask(microtask, gtid, 0, argc, args 1872 #if OMPT_SUPPORT 1873 , 1874 exit_runtime_p 1875 #endif 1876 ); 1877 } 1878 1879 #if OMPT_SUPPORT 1880 *exit_runtime_p = NULL; 1881 if (ompt_enabled) { 1882 #if OMPT_TRACE 1883 lw_taskteam.ompt_task_info.frame.exit_runtime_frame = NULL; 1884 1885 if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) { 1886 ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)( 1887 my_parallel_id, my_task_id); 1888 } 1889 #endif 1890 1891 __ompt_lw_taskteam_unlink(master_th); 1892 // reset clear the task id only after unlinking the task 1893 lw_taskteam.ompt_task_info.task_id = ompt_task_id_none; 1894 1895 if (ompt_callbacks.ompt_callback(ompt_event_parallel_end)) { 1896 ompt_callbacks.ompt_callback(ompt_event_parallel_end)( 1897 ompt_parallel_id, ompt_task_id, OMPT_INVOKER(call_context)); 1898 } 1899 master_th->th.ompt_thread_info.state = ompt_state_overhead; 1900 } 1901 #endif 1902 #if OMP_40_ENABLED 1903 } 1904 #endif /* OMP_40_ENABLED */ 1905 } else if (call_context == fork_context_gnu) { 1906 #if OMPT_SUPPORT 1907 ompt_lw_taskteam_t *lwt = 1908 (ompt_lw_taskteam_t *)__kmp_allocate(sizeof(ompt_lw_taskteam_t)); 1909 __ompt_lw_taskteam_init(lwt, master_th, gtid, unwrapped_task, 1910 ompt_parallel_id); 1911 1912 lwt->ompt_task_info.task_id = __ompt_task_id_new(gtid); 1913 lwt->ompt_task_info.frame.exit_runtime_frame = NULL; 1914 __ompt_lw_taskteam_link(lwt, master_th); 1915 #endif 1916 1917 // we were called from GNU native code 1918 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid)); 1919 return FALSE; 1920 } else { 1921 KMP_ASSERT2(call_context < fork_context_last, 1922 "__kmp_fork_call: unknown fork_context parameter"); 1923 } 1924 1925 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid)); 1926 KMP_MB(); 1927 return FALSE; 1928 } 1929 1930 // GEH: only modify the executing flag in the case when not serialized 1931 // serialized case is handled in kmpc_serialized_parallel 1932 KF_TRACE(10, ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, " 1933 "curtask=%p, curtask_max_aclevel=%d\n", 1934 parent_team->t.t_active_level, master_th, 1935 master_th->th.th_current_task, 1936 master_th->th.th_current_task->td_icvs.max_active_levels)); 1937 // TODO: GEH - cannot do this assertion because root thread not set up as 1938 // executing 1939 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 1 ); 1940 master_th->th.th_current_task->td_flags.executing = 0; 1941 1942 #if OMP_40_ENABLED 1943 if (!master_th->th.th_teams_microtask || level > teams_level) 1944 #endif /* OMP_40_ENABLED */ 1945 { 1946 /* Increment our nested depth level */ 1947 KMP_TEST_THEN_INC32((kmp_int32 *)&root->r.r_in_parallel); 1948 } 1949 1950 // See if we need to make a copy of the ICVs. 1951 int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc; 1952 if ((level + 1 < __kmp_nested_nth.used) && 1953 (__kmp_nested_nth.nth[level + 1] != nthreads_icv)) { 1954 nthreads_icv = __kmp_nested_nth.nth[level + 1]; 1955 } else { 1956 nthreads_icv = 0; // don't update 1957 } 1958 1959 #if OMP_40_ENABLED 1960 // Figure out the proc_bind_policy for the new team. 1961 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind; 1962 kmp_proc_bind_t proc_bind_icv = 1963 proc_bind_default; // proc_bind_default means don't update 1964 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) { 1965 proc_bind = proc_bind_false; 1966 } else { 1967 if (proc_bind == proc_bind_default) { 1968 // No proc_bind clause specified; use current proc-bind-var for this 1969 // parallel region 1970 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind; 1971 } 1972 /* else: The proc_bind policy was specified explicitly on parallel clause. 1973 This overrides proc-bind-var for this parallel region, but does not 1974 change proc-bind-var. */ 1975 // Figure the value of proc-bind-var for the child threads. 1976 if ((level + 1 < __kmp_nested_proc_bind.used) && 1977 (__kmp_nested_proc_bind.bind_types[level + 1] != 1978 master_th->th.th_current_task->td_icvs.proc_bind)) { 1979 proc_bind_icv = __kmp_nested_proc_bind.bind_types[level + 1]; 1980 } 1981 } 1982 1983 // Reset for next parallel region 1984 master_th->th.th_set_proc_bind = proc_bind_default; 1985 #endif /* OMP_40_ENABLED */ 1986 1987 if ((nthreads_icv > 0) 1988 #if OMP_40_ENABLED 1989 || (proc_bind_icv != proc_bind_default) 1990 #endif /* OMP_40_ENABLED */ 1991 ) { 1992 kmp_internal_control_t new_icvs; 1993 copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs); 1994 new_icvs.next = NULL; 1995 if (nthreads_icv > 0) { 1996 new_icvs.nproc = nthreads_icv; 1997 } 1998 1999 #if OMP_40_ENABLED 2000 if (proc_bind_icv != proc_bind_default) { 2001 new_icvs.proc_bind = proc_bind_icv; 2002 } 2003 #endif /* OMP_40_ENABLED */ 2004 2005 /* allocate a new parallel team */ 2006 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n")); 2007 team = __kmp_allocate_team(root, nthreads, nthreads, 2008 #if OMPT_SUPPORT 2009 ompt_parallel_id, 2010 #endif 2011 #if OMP_40_ENABLED 2012 proc_bind, 2013 #endif 2014 &new_icvs, argc USE_NESTED_HOT_ARG(master_th)); 2015 } else { 2016 /* allocate a new parallel team */ 2017 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n")); 2018 team = __kmp_allocate_team(root, nthreads, nthreads, 2019 #if OMPT_SUPPORT 2020 ompt_parallel_id, 2021 #endif 2022 #if OMP_40_ENABLED 2023 proc_bind, 2024 #endif 2025 &master_th->th.th_current_task->td_icvs, 2026 argc USE_NESTED_HOT_ARG(master_th)); 2027 } 2028 KF_TRACE( 2029 10, ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team)); 2030 2031 /* setup the new team */ 2032 KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid); 2033 KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons); 2034 KMP_CHECK_UPDATE(team->t.t_ident, loc); 2035 KMP_CHECK_UPDATE(team->t.t_parent, parent_team); 2036 KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask); 2037 #if OMPT_SUPPORT 2038 KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.microtask, unwrapped_task); 2039 #endif 2040 KMP_CHECK_UPDATE(team->t.t_invoke, invoker); // TODO move to root, maybe 2041 // TODO: parent_team->t.t_level == INT_MAX ??? 2042 #if OMP_40_ENABLED 2043 if (!master_th->th.th_teams_microtask || level > teams_level) { 2044 #endif /* OMP_40_ENABLED */ 2045 int new_level = parent_team->t.t_level + 1; 2046 KMP_CHECK_UPDATE(team->t.t_level, new_level); 2047 new_level = parent_team->t.t_active_level + 1; 2048 KMP_CHECK_UPDATE(team->t.t_active_level, new_level); 2049 #if OMP_40_ENABLED 2050 } else { 2051 // AC: Do not increase parallel level at start of the teams construct 2052 int new_level = parent_team->t.t_level; 2053 KMP_CHECK_UPDATE(team->t.t_level, new_level); 2054 new_level = parent_team->t.t_active_level; 2055 KMP_CHECK_UPDATE(team->t.t_active_level, new_level); 2056 } 2057 #endif /* OMP_40_ENABLED */ 2058 kmp_r_sched_t new_sched = get__sched_2(parent_team, master_tid); 2059 if (team->t.t_sched.r_sched_type != new_sched.r_sched_type || 2060 team->t.t_sched.chunk != new_sched.chunk) 2061 team->t.t_sched = 2062 new_sched; // set master's schedule as new run-time schedule 2063 2064 #if OMP_40_ENABLED 2065 KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq); 2066 #endif 2067 2068 // Update the floating point rounding in the team if required. 2069 propagateFPControl(team); 2070 2071 if (__kmp_tasking_mode != tskm_immediate_exec) { 2072 // Set master's task team to team's task team. Unless this is hot team, it 2073 // should be NULL. 2074 KMP_DEBUG_ASSERT(master_th->th.th_task_team == 2075 parent_team->t.t_task_team[master_th->th.th_task_state]); 2076 KA_TRACE(20, ("__kmp_fork_call: Master T#%d pushing task_team %p / team " 2077 "%p, new task_team %p / team %p\n", 2078 __kmp_gtid_from_thread(master_th), 2079 master_th->th.th_task_team, parent_team, 2080 team->t.t_task_team[master_th->th.th_task_state], team)); 2081 2082 if (active_level || master_th->th.th_task_team) { 2083 // Take a memo of master's task_state 2084 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack); 2085 if (master_th->th.th_task_state_top >= 2086 master_th->th.th_task_state_stack_sz) { // increase size 2087 kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz; 2088 kmp_uint8 *old_stack, *new_stack; 2089 kmp_uint32 i; 2090 new_stack = (kmp_uint8 *)__kmp_allocate(new_size); 2091 for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) { 2092 new_stack[i] = master_th->th.th_task_state_memo_stack[i]; 2093 } 2094 for (i = master_th->th.th_task_state_stack_sz; i < new_size; 2095 ++i) { // zero-init rest of stack 2096 new_stack[i] = 0; 2097 } 2098 old_stack = master_th->th.th_task_state_memo_stack; 2099 master_th->th.th_task_state_memo_stack = new_stack; 2100 master_th->th.th_task_state_stack_sz = new_size; 2101 __kmp_free(old_stack); 2102 } 2103 // Store master's task_state on stack 2104 master_th->th 2105 .th_task_state_memo_stack[master_th->th.th_task_state_top] = 2106 master_th->th.th_task_state; 2107 master_th->th.th_task_state_top++; 2108 #if KMP_NESTED_HOT_TEAMS 2109 if (team == master_th->th.th_hot_teams[active_level].hot_team) { 2110 // Restore master's nested state if nested hot team 2111 master_th->th.th_task_state = 2112 master_th->th 2113 .th_task_state_memo_stack[master_th->th.th_task_state_top]; 2114 } else { 2115 #endif 2116 master_th->th.th_task_state = 0; 2117 #if KMP_NESTED_HOT_TEAMS 2118 } 2119 #endif 2120 } 2121 #if !KMP_NESTED_HOT_TEAMS 2122 KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) || 2123 (team == root->r.r_hot_team)); 2124 #endif 2125 } 2126 2127 KA_TRACE( 2128 20, 2129 ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n", 2130 gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id, 2131 team->t.t_nproc)); 2132 KMP_DEBUG_ASSERT(team != root->r.r_hot_team || 2133 (team->t.t_master_tid == 0 && 2134 (team->t.t_parent == root->r.r_root_team || 2135 team->t.t_parent->t.t_serialized))); 2136 KMP_MB(); 2137 2138 /* now, setup the arguments */ 2139 argv = (void **)team->t.t_argv; 2140 #if OMP_40_ENABLED 2141 if (ap) { 2142 #endif /* OMP_40_ENABLED */ 2143 for (i = argc - 1; i >= 0; --i) { 2144 // TODO: revert workaround for Intel(R) 64 tracker #96 2145 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 2146 void *new_argv = va_arg(*ap, void *); 2147 #else 2148 void *new_argv = va_arg(ap, void *); 2149 #endif 2150 KMP_CHECK_UPDATE(*argv, new_argv); 2151 argv++; 2152 } 2153 #if OMP_40_ENABLED 2154 } else { 2155 for (i = 0; i < argc; ++i) { 2156 // Get args from parent team for teams construct 2157 KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i]); 2158 } 2159 } 2160 #endif /* OMP_40_ENABLED */ 2161 2162 /* now actually fork the threads */ 2163 KMP_CHECK_UPDATE(team->t.t_master_active, master_active); 2164 if (!root->r.r_active) // Only do assignment if it prevents cache ping-pong 2165 root->r.r_active = TRUE; 2166 2167 __kmp_fork_team_threads(root, team, master_th, gtid); 2168 __kmp_setup_icv_copy(team, nthreads, 2169 &master_th->th.th_current_task->td_icvs, loc); 2170 2171 #if OMPT_SUPPORT 2172 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; 2173 #endif 2174 2175 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 2176 2177 #if USE_ITT_BUILD 2178 if (team->t.t_active_level == 1 // only report frames at level 1 2179 #if OMP_40_ENABLED 2180 && !master_th->th.th_teams_microtask // not in teams construct 2181 #endif /* OMP_40_ENABLED */ 2182 ) { 2183 #if USE_ITT_NOTIFY 2184 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) && 2185 (__kmp_forkjoin_frames_mode == 3 || 2186 __kmp_forkjoin_frames_mode == 1)) { 2187 kmp_uint64 tmp_time = 0; 2188 if (__itt_get_timestamp_ptr) 2189 tmp_time = __itt_get_timestamp(); 2190 // Internal fork - report frame begin 2191 master_th->th.th_frame_time = tmp_time; 2192 if (__kmp_forkjoin_frames_mode == 3) 2193 team->t.t_region_time = tmp_time; 2194 } else 2195 // only one notification scheme (either "submit" or "forking/joined", not both) 2196 #endif /* USE_ITT_NOTIFY */ 2197 if ((__itt_frame_begin_v3_ptr || KMP_ITT_DEBUG) && 2198 __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode) { 2199 // Mark start of "parallel" region for VTune. 2200 __kmp_itt_region_forking(gtid, team->t.t_nproc, 0); 2201 } 2202 } 2203 #endif /* USE_ITT_BUILD */ 2204 2205 /* now go on and do the work */ 2206 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team); 2207 KMP_MB(); 2208 KF_TRACE(10, 2209 ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n", 2210 root, team, master_th, gtid)); 2211 2212 #if USE_ITT_BUILD 2213 if (__itt_stack_caller_create_ptr) { 2214 team->t.t_stack_id = 2215 __kmp_itt_stack_caller_create(); // create new stack stitching id 2216 // before entering fork barrier 2217 } 2218 #endif /* USE_ITT_BUILD */ 2219 2220 #if OMP_40_ENABLED 2221 // AC: skip __kmp_internal_fork at teams construct, let only master 2222 // threads execute 2223 if (ap) 2224 #endif /* OMP_40_ENABLED */ 2225 { 2226 __kmp_internal_fork(loc, gtid, team); 2227 KF_TRACE(10, ("__kmp_internal_fork : after : root=%p, team=%p, " 2228 "master_th=%p, gtid=%d\n", 2229 root, team, master_th, gtid)); 2230 } 2231 2232 if (call_context == fork_context_gnu) { 2233 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid)); 2234 return TRUE; 2235 } 2236 2237 /* Invoke microtask for MASTER thread */ 2238 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid, 2239 team->t.t_id, team->t.t_pkfn)); 2240 } // END of timer KMP_fork_call block 2241 2242 { 2243 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 2244 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 2245 if (!team->t.t_invoke(gtid)) { 2246 KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread"); 2247 } 2248 } 2249 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid, 2250 team->t.t_id, team->t.t_pkfn)); 2251 KMP_MB(); /* Flush all pending memory write invalidates. */ 2252 2253 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid)); 2254 2255 #if OMPT_SUPPORT 2256 if (ompt_enabled) { 2257 master_th->th.ompt_thread_info.state = ompt_state_overhead; 2258 } 2259 #endif 2260 2261 return TRUE; 2262 } 2263 2264 #if OMPT_SUPPORT 2265 static inline void __kmp_join_restore_state(kmp_info_t *thread, 2266 kmp_team_t *team) { 2267 // restore state outside the region 2268 thread->th.ompt_thread_info.state = 2269 ((team->t.t_serialized) ? ompt_state_work_serial 2270 : ompt_state_work_parallel); 2271 } 2272 2273 static inline void __kmp_join_ompt(kmp_info_t *thread, kmp_team_t *team, 2274 ompt_parallel_id_t parallel_id, 2275 fork_context_e fork_context) { 2276 ompt_task_info_t *task_info = __ompt_get_taskinfo(0); 2277 if (ompt_callbacks.ompt_callback(ompt_event_parallel_end)) { 2278 ompt_callbacks.ompt_callback(ompt_event_parallel_end)( 2279 parallel_id, task_info->task_id, OMPT_INVOKER(fork_context)); 2280 } 2281 2282 task_info->frame.reenter_runtime_frame = NULL; 2283 __kmp_join_restore_state(thread, team); 2284 } 2285 #endif 2286 2287 void __kmp_join_call(ident_t *loc, int gtid 2288 #if OMPT_SUPPORT 2289 , 2290 enum fork_context_e fork_context 2291 #endif 2292 #if OMP_40_ENABLED 2293 , 2294 int exit_teams 2295 #endif /* OMP_40_ENABLED */ 2296 ) { 2297 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_join_call); 2298 kmp_team_t *team; 2299 kmp_team_t *parent_team; 2300 kmp_info_t *master_th; 2301 kmp_root_t *root; 2302 int master_active; 2303 int i; 2304 2305 KA_TRACE(20, ("__kmp_join_call: enter T#%d\n", gtid)); 2306 2307 /* setup current data */ 2308 master_th = __kmp_threads[gtid]; 2309 root = master_th->th.th_root; 2310 team = master_th->th.th_team; 2311 parent_team = team->t.t_parent; 2312 2313 master_th->th.th_ident = loc; 2314 2315 #if OMPT_SUPPORT 2316 if (ompt_enabled) { 2317 master_th->th.ompt_thread_info.state = ompt_state_overhead; 2318 } 2319 #endif 2320 2321 #if KMP_DEBUG 2322 if (__kmp_tasking_mode != tskm_immediate_exec && !exit_teams) { 2323 KA_TRACE(20, ("__kmp_join_call: T#%d, old team = %p old task_team = %p, " 2324 "th_task_team = %p\n", 2325 __kmp_gtid_from_thread(master_th), team, 2326 team->t.t_task_team[master_th->th.th_task_state], 2327 master_th->th.th_task_team)); 2328 KMP_DEBUG_ASSERT(master_th->th.th_task_team == 2329 team->t.t_task_team[master_th->th.th_task_state]); 2330 } 2331 #endif 2332 2333 if (team->t.t_serialized) { 2334 #if OMP_40_ENABLED 2335 if (master_th->th.th_teams_microtask) { 2336 // We are in teams construct 2337 int level = team->t.t_level; 2338 int tlevel = master_th->th.th_teams_level; 2339 if (level == tlevel) { 2340 // AC: we haven't incremented it earlier at start of teams construct, 2341 // so do it here - at the end of teams construct 2342 team->t.t_level++; 2343 } else if (level == tlevel + 1) { 2344 // AC: we are exiting parallel inside teams, need to increment 2345 // serialization in order to restore it in the next call to 2346 // __kmpc_end_serialized_parallel 2347 team->t.t_serialized++; 2348 } 2349 } 2350 #endif /* OMP_40_ENABLED */ 2351 __kmpc_end_serialized_parallel(loc, gtid); 2352 2353 #if OMPT_SUPPORT 2354 if (ompt_enabled) { 2355 __kmp_join_restore_state(master_th, parent_team); 2356 } 2357 #endif 2358 2359 return; 2360 } 2361 2362 master_active = team->t.t_master_active; 2363 2364 #if OMP_40_ENABLED 2365 if (!exit_teams) 2366 #endif /* OMP_40_ENABLED */ 2367 { 2368 // AC: No barrier for internal teams at exit from teams construct. 2369 // But there is barrier for external team (league). 2370 __kmp_internal_join(loc, gtid, team); 2371 } 2372 #if OMP_40_ENABLED 2373 else { 2374 master_th->th.th_task_state = 2375 0; // AC: no tasking in teams (out of any parallel) 2376 } 2377 #endif /* OMP_40_ENABLED */ 2378 2379 KMP_MB(); 2380 2381 #if OMPT_SUPPORT 2382 ompt_parallel_id_t parallel_id = team->t.ompt_team_info.parallel_id; 2383 #endif 2384 2385 #if USE_ITT_BUILD 2386 if (__itt_stack_caller_create_ptr) { 2387 __kmp_itt_stack_caller_destroy( 2388 (__itt_caller)team->t 2389 .t_stack_id); // destroy the stack stitching id after join barrier 2390 } 2391 2392 // Mark end of "parallel" region for VTune. 2393 if (team->t.t_active_level == 1 2394 #if OMP_40_ENABLED 2395 && !master_th->th.th_teams_microtask /* not in teams construct */ 2396 #endif /* OMP_40_ENABLED */ 2397 ) { 2398 master_th->th.th_ident = loc; 2399 // only one notification scheme (either "submit" or "forking/joined", not 2400 // both) 2401 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) && 2402 __kmp_forkjoin_frames_mode == 3) 2403 __kmp_itt_frame_submit(gtid, team->t.t_region_time, 2404 master_th->th.th_frame_time, 0, loc, 2405 master_th->th.th_team_nproc, 1); 2406 else if ((__itt_frame_end_v3_ptr || KMP_ITT_DEBUG) && 2407 !__kmp_forkjoin_frames_mode && __kmp_forkjoin_frames) 2408 __kmp_itt_region_joined(gtid); 2409 } // active_level == 1 2410 #endif /* USE_ITT_BUILD */ 2411 2412 #if OMP_40_ENABLED 2413 if (master_th->th.th_teams_microtask && !exit_teams && 2414 team->t.t_pkfn != (microtask_t)__kmp_teams_master && 2415 team->t.t_level == master_th->th.th_teams_level + 1) { 2416 // AC: We need to leave the team structure intact at the end of parallel 2417 // inside the teams construct, so that at the next parallel same (hot) team 2418 // works, only adjust nesting levels 2419 2420 /* Decrement our nested depth level */ 2421 team->t.t_level--; 2422 team->t.t_active_level--; 2423 KMP_TEST_THEN_DEC32((kmp_int32 *)&root->r.r_in_parallel); 2424 2425 /* Restore number of threads in the team if needed */ 2426 if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) { 2427 int old_num = master_th->th.th_team_nproc; 2428 int new_num = master_th->th.th_teams_size.nth; 2429 kmp_info_t **other_threads = team->t.t_threads; 2430 team->t.t_nproc = new_num; 2431 for (i = 0; i < old_num; ++i) { 2432 other_threads[i]->th.th_team_nproc = new_num; 2433 } 2434 // Adjust states of non-used threads of the team 2435 for (i = old_num; i < new_num; ++i) { 2436 // Re-initialize thread's barrier data. 2437 int b; 2438 kmp_balign_t *balign = other_threads[i]->th.th_bar; 2439 for (b = 0; b < bs_last_barrier; ++b) { 2440 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 2441 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 2442 #if USE_DEBUGGER 2443 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 2444 #endif 2445 } 2446 if (__kmp_tasking_mode != tskm_immediate_exec) { 2447 // Synchronize thread's task state 2448 other_threads[i]->th.th_task_state = master_th->th.th_task_state; 2449 } 2450 } 2451 } 2452 2453 #if OMPT_SUPPORT 2454 if (ompt_enabled) { 2455 __kmp_join_ompt(master_th, parent_team, parallel_id, fork_context); 2456 } 2457 #endif 2458 2459 return; 2460 } 2461 #endif /* OMP_40_ENABLED */ 2462 2463 /* do cleanup and restore the parent team */ 2464 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid; 2465 master_th->th.th_local.this_construct = team->t.t_master_this_cons; 2466 2467 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid]; 2468 2469 /* jc: The following lock has instructions with REL and ACQ semantics, 2470 separating the parallel user code called in this parallel region 2471 from the serial user code called after this function returns. */ 2472 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 2473 2474 #if OMP_40_ENABLED 2475 if (!master_th->th.th_teams_microtask || 2476 team->t.t_level > master_th->th.th_teams_level) 2477 #endif /* OMP_40_ENABLED */ 2478 { 2479 /* Decrement our nested depth level */ 2480 KMP_TEST_THEN_DEC32((kmp_int32 *)&root->r.r_in_parallel); 2481 } 2482 KMP_DEBUG_ASSERT(root->r.r_in_parallel >= 0); 2483 2484 #if OMPT_SUPPORT && OMPT_TRACE 2485 if (ompt_enabled) { 2486 ompt_task_info_t *task_info = __ompt_get_taskinfo(0); 2487 if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) { 2488 ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)( 2489 parallel_id, task_info->task_id); 2490 } 2491 task_info->frame.exit_runtime_frame = NULL; 2492 task_info->task_id = 0; 2493 } 2494 #endif 2495 2496 KF_TRACE(10, ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n", 0, 2497 master_th, team)); 2498 __kmp_pop_current_task_from_thread(master_th); 2499 2500 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 2501 // Restore master thread's partition. 2502 master_th->th.th_first_place = team->t.t_first_place; 2503 master_th->th.th_last_place = team->t.t_last_place; 2504 #endif /* OMP_40_ENABLED */ 2505 2506 updateHWFPControl(team); 2507 2508 if (root->r.r_active != master_active) 2509 root->r.r_active = master_active; 2510 2511 __kmp_free_team(root, team USE_NESTED_HOT_ARG( 2512 master_th)); // this will free worker threads 2513 2514 /* this race was fun to find. make sure the following is in the critical 2515 region otherwise assertions may fail occasionally since the old team may be 2516 reallocated and the hierarchy appears inconsistent. it is actually safe to 2517 run and won't cause any bugs, but will cause those assertion failures. it's 2518 only one deref&assign so might as well put this in the critical region */ 2519 master_th->th.th_team = parent_team; 2520 master_th->th.th_team_nproc = parent_team->t.t_nproc; 2521 master_th->th.th_team_master = parent_team->t.t_threads[0]; 2522 master_th->th.th_team_serialized = parent_team->t.t_serialized; 2523 2524 /* restore serialized team, if need be */ 2525 if (parent_team->t.t_serialized && 2526 parent_team != master_th->th.th_serial_team && 2527 parent_team != root->r.r_root_team) { 2528 __kmp_free_team(root, 2529 master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL)); 2530 master_th->th.th_serial_team = parent_team; 2531 } 2532 2533 if (__kmp_tasking_mode != tskm_immediate_exec) { 2534 if (master_th->th.th_task_state_top > 2535 0) { // Restore task state from memo stack 2536 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack); 2537 // Remember master's state if we re-use this nested hot team 2538 master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] = 2539 master_th->th.th_task_state; 2540 --master_th->th.th_task_state_top; // pop 2541 // Now restore state at this level 2542 master_th->th.th_task_state = 2543 master_th->th 2544 .th_task_state_memo_stack[master_th->th.th_task_state_top]; 2545 } 2546 // Copy the task team from the parent team to the master thread 2547 master_th->th.th_task_team = 2548 parent_team->t.t_task_team[master_th->th.th_task_state]; 2549 KA_TRACE(20, 2550 ("__kmp_join_call: Master T#%d restoring task_team %p / team %p\n", 2551 __kmp_gtid_from_thread(master_th), master_th->th.th_task_team, 2552 parent_team)); 2553 } 2554 2555 // TODO: GEH - cannot do this assertion because root thread not set up as 2556 // executing 2557 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 0 ); 2558 master_th->th.th_current_task->td_flags.executing = 1; 2559 2560 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 2561 2562 #if OMPT_SUPPORT 2563 if (ompt_enabled) { 2564 __kmp_join_ompt(master_th, parent_team, parallel_id, fork_context); 2565 } 2566 #endif 2567 2568 KMP_MB(); 2569 KA_TRACE(20, ("__kmp_join_call: exit T#%d\n", gtid)); 2570 } 2571 2572 /* Check whether we should push an internal control record onto the 2573 serial team stack. If so, do it. */ 2574 void __kmp_save_internal_controls(kmp_info_t *thread) { 2575 2576 if (thread->th.th_team != thread->th.th_serial_team) { 2577 return; 2578 } 2579 if (thread->th.th_team->t.t_serialized > 1) { 2580 int push = 0; 2581 2582 if (thread->th.th_team->t.t_control_stack_top == NULL) { 2583 push = 1; 2584 } else { 2585 if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level != 2586 thread->th.th_team->t.t_serialized) { 2587 push = 1; 2588 } 2589 } 2590 if (push) { /* push a record on the serial team's stack */ 2591 kmp_internal_control_t *control = 2592 (kmp_internal_control_t *)__kmp_allocate( 2593 sizeof(kmp_internal_control_t)); 2594 2595 copy_icvs(control, &thread->th.th_current_task->td_icvs); 2596 2597 control->serial_nesting_level = thread->th.th_team->t.t_serialized; 2598 2599 control->next = thread->th.th_team->t.t_control_stack_top; 2600 thread->th.th_team->t.t_control_stack_top = control; 2601 } 2602 } 2603 } 2604 2605 /* Changes set_nproc */ 2606 void __kmp_set_num_threads(int new_nth, int gtid) { 2607 kmp_info_t *thread; 2608 kmp_root_t *root; 2609 2610 KF_TRACE(10, ("__kmp_set_num_threads: new __kmp_nth = %d\n", new_nth)); 2611 KMP_DEBUG_ASSERT(__kmp_init_serial); 2612 2613 if (new_nth < 1) 2614 new_nth = 1; 2615 else if (new_nth > __kmp_max_nth) 2616 new_nth = __kmp_max_nth; 2617 2618 KMP_COUNT_VALUE(OMP_set_numthreads, new_nth); 2619 thread = __kmp_threads[gtid]; 2620 2621 __kmp_save_internal_controls(thread); 2622 2623 set__nproc(thread, new_nth); 2624 2625 // If this omp_set_num_threads() call will cause the hot team size to be 2626 // reduced (in the absence of a num_threads clause), then reduce it now, 2627 // rather than waiting for the next parallel region. 2628 root = thread->th.th_root; 2629 if (__kmp_init_parallel && (!root->r.r_active) && 2630 (root->r.r_hot_team->t.t_nproc > new_nth) 2631 #if KMP_NESTED_HOT_TEAMS 2632 && __kmp_hot_teams_max_level && !__kmp_hot_teams_mode 2633 #endif 2634 ) { 2635 kmp_team_t *hot_team = root->r.r_hot_team; 2636 int f; 2637 2638 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 2639 2640 // Release the extra threads we don't need any more. 2641 for (f = new_nth; f < hot_team->t.t_nproc; f++) { 2642 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL); 2643 if (__kmp_tasking_mode != tskm_immediate_exec) { 2644 // When decreasing team size, threads no longer in the team should unref 2645 // task team. 2646 hot_team->t.t_threads[f]->th.th_task_team = NULL; 2647 } 2648 __kmp_free_thread(hot_team->t.t_threads[f]); 2649 hot_team->t.t_threads[f] = NULL; 2650 } 2651 hot_team->t.t_nproc = new_nth; 2652 #if KMP_NESTED_HOT_TEAMS 2653 if (thread->th.th_hot_teams) { 2654 KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team); 2655 thread->th.th_hot_teams[0].hot_team_nth = new_nth; 2656 } 2657 #endif 2658 2659 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 2660 2661 // Update the t_nproc field in the threads that are still active. 2662 for (f = 0; f < new_nth; f++) { 2663 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL); 2664 hot_team->t.t_threads[f]->th.th_team_nproc = new_nth; 2665 } 2666 // Special flag in case omp_set_num_threads() call 2667 hot_team->t.t_size_changed = -1; 2668 } 2669 } 2670 2671 /* Changes max_active_levels */ 2672 void __kmp_set_max_active_levels(int gtid, int max_active_levels) { 2673 kmp_info_t *thread; 2674 2675 KF_TRACE(10, ("__kmp_set_max_active_levels: new max_active_levels for thread " 2676 "%d = (%d)\n", 2677 gtid, max_active_levels)); 2678 KMP_DEBUG_ASSERT(__kmp_init_serial); 2679 2680 // validate max_active_levels 2681 if (max_active_levels < 0) { 2682 KMP_WARNING(ActiveLevelsNegative, max_active_levels); 2683 // We ignore this call if the user has specified a negative value. 2684 // The current setting won't be changed. The last valid setting will be 2685 // used. A warning will be issued (if warnings are allowed as controlled by 2686 // the KMP_WARNINGS env var). 2687 KF_TRACE(10, ("__kmp_set_max_active_levels: the call is ignored: new " 2688 "max_active_levels for thread %d = (%d)\n", 2689 gtid, max_active_levels)); 2690 return; 2691 } 2692 if (max_active_levels <= KMP_MAX_ACTIVE_LEVELS_LIMIT) { 2693 // it's OK, the max_active_levels is within the valid range: [ 0; 2694 // KMP_MAX_ACTIVE_LEVELS_LIMIT ] 2695 // We allow a zero value. (implementation defined behavior) 2696 } else { 2697 KMP_WARNING(ActiveLevelsExceedLimit, max_active_levels, 2698 KMP_MAX_ACTIVE_LEVELS_LIMIT); 2699 max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT; 2700 // Current upper limit is MAX_INT. (implementation defined behavior) 2701 // If the input exceeds the upper limit, we correct the input to be the 2702 // upper limit. (implementation defined behavior) 2703 // Actually, the flow should never get here until we use MAX_INT limit. 2704 } 2705 KF_TRACE(10, ("__kmp_set_max_active_levels: after validation: new " 2706 "max_active_levels for thread %d = (%d)\n", 2707 gtid, max_active_levels)); 2708 2709 thread = __kmp_threads[gtid]; 2710 2711 __kmp_save_internal_controls(thread); 2712 2713 set__max_active_levels(thread, max_active_levels); 2714 } 2715 2716 /* Gets max_active_levels */ 2717 int __kmp_get_max_active_levels(int gtid) { 2718 kmp_info_t *thread; 2719 2720 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d\n", gtid)); 2721 KMP_DEBUG_ASSERT(__kmp_init_serial); 2722 2723 thread = __kmp_threads[gtid]; 2724 KMP_DEBUG_ASSERT(thread->th.th_current_task); 2725 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d, curtask=%p, " 2726 "curtask_maxaclevel=%d\n", 2727 gtid, thread->th.th_current_task, 2728 thread->th.th_current_task->td_icvs.max_active_levels)); 2729 return thread->th.th_current_task->td_icvs.max_active_levels; 2730 } 2731 2732 /* Changes def_sched_var ICV values (run-time schedule kind and chunk) */ 2733 void __kmp_set_schedule(int gtid, kmp_sched_t kind, int chunk) { 2734 kmp_info_t *thread; 2735 // kmp_team_t *team; 2736 2737 KF_TRACE(10, ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n", 2738 gtid, (int)kind, chunk)); 2739 KMP_DEBUG_ASSERT(__kmp_init_serial); 2740 2741 // Check if the kind parameter is valid, correct if needed. 2742 // Valid parameters should fit in one of two intervals - standard or extended: 2743 // <lower>, <valid>, <upper_std>, <lower_ext>, <valid>, <upper> 2744 // 2008-01-25: 0, 1 - 4, 5, 100, 101 - 102, 103 2745 if (kind <= kmp_sched_lower || kind >= kmp_sched_upper || 2746 (kind <= kmp_sched_lower_ext && kind >= kmp_sched_upper_std)) { 2747 // TODO: Hint needs attention in case we change the default schedule. 2748 __kmp_msg(kmp_ms_warning, KMP_MSG(ScheduleKindOutOfRange, kind), 2749 KMP_HNT(DefaultScheduleKindUsed, "static, no chunk"), 2750 __kmp_msg_null); 2751 kind = kmp_sched_default; 2752 chunk = 0; // ignore chunk value in case of bad kind 2753 } 2754 2755 thread = __kmp_threads[gtid]; 2756 2757 __kmp_save_internal_controls(thread); 2758 2759 if (kind < kmp_sched_upper_std) { 2760 if (kind == kmp_sched_static && chunk < KMP_DEFAULT_CHUNK) { 2761 // differ static chunked vs. unchunked: chunk should be invalid to 2762 // indicate unchunked schedule (which is the default) 2763 thread->th.th_current_task->td_icvs.sched.r_sched_type = kmp_sch_static; 2764 } else { 2765 thread->th.th_current_task->td_icvs.sched.r_sched_type = 2766 __kmp_sch_map[kind - kmp_sched_lower - 1]; 2767 } 2768 } else { 2769 // __kmp_sch_map[ kind - kmp_sched_lower_ext + kmp_sched_upper_std - 2770 // kmp_sched_lower - 2 ]; 2771 thread->th.th_current_task->td_icvs.sched.r_sched_type = 2772 __kmp_sch_map[kind - kmp_sched_lower_ext + kmp_sched_upper_std - 2773 kmp_sched_lower - 2]; 2774 } 2775 if (kind == kmp_sched_auto || chunk < 1) { 2776 // ignore parameter chunk for schedule auto 2777 thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK; 2778 } else { 2779 thread->th.th_current_task->td_icvs.sched.chunk = chunk; 2780 } 2781 } 2782 2783 /* Gets def_sched_var ICV values */ 2784 void __kmp_get_schedule(int gtid, kmp_sched_t *kind, int *chunk) { 2785 kmp_info_t *thread; 2786 enum sched_type th_type; 2787 2788 KF_TRACE(10, ("__kmp_get_schedule: thread %d\n", gtid)); 2789 KMP_DEBUG_ASSERT(__kmp_init_serial); 2790 2791 thread = __kmp_threads[gtid]; 2792 2793 th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type; 2794 2795 switch (th_type) { 2796 case kmp_sch_static: 2797 case kmp_sch_static_greedy: 2798 case kmp_sch_static_balanced: 2799 *kind = kmp_sched_static; 2800 *chunk = 0; // chunk was not set, try to show this fact via zero value 2801 return; 2802 case kmp_sch_static_chunked: 2803 *kind = kmp_sched_static; 2804 break; 2805 case kmp_sch_dynamic_chunked: 2806 *kind = kmp_sched_dynamic; 2807 break; 2808 case kmp_sch_guided_chunked: 2809 case kmp_sch_guided_iterative_chunked: 2810 case kmp_sch_guided_analytical_chunked: 2811 *kind = kmp_sched_guided; 2812 break; 2813 case kmp_sch_auto: 2814 *kind = kmp_sched_auto; 2815 break; 2816 case kmp_sch_trapezoidal: 2817 *kind = kmp_sched_trapezoidal; 2818 break; 2819 #if KMP_STATIC_STEAL_ENABLED 2820 case kmp_sch_static_steal: 2821 *kind = kmp_sched_static_steal; 2822 break; 2823 #endif 2824 default: 2825 KMP_FATAL(UnknownSchedulingType, th_type); 2826 } 2827 2828 *chunk = thread->th.th_current_task->td_icvs.sched.chunk; 2829 } 2830 2831 int __kmp_get_ancestor_thread_num(int gtid, int level) { 2832 2833 int ii, dd; 2834 kmp_team_t *team; 2835 kmp_info_t *thr; 2836 2837 KF_TRACE(10, ("__kmp_get_ancestor_thread_num: thread %d %d\n", gtid, level)); 2838 KMP_DEBUG_ASSERT(__kmp_init_serial); 2839 2840 // validate level 2841 if (level == 0) 2842 return 0; 2843 if (level < 0) 2844 return -1; 2845 thr = __kmp_threads[gtid]; 2846 team = thr->th.th_team; 2847 ii = team->t.t_level; 2848 if (level > ii) 2849 return -1; 2850 2851 #if OMP_40_ENABLED 2852 if (thr->th.th_teams_microtask) { 2853 // AC: we are in teams region where multiple nested teams have same level 2854 int tlevel = thr->th.th_teams_level; // the level of the teams construct 2855 if (level <= 2856 tlevel) { // otherwise usual algorithm works (will not touch the teams) 2857 KMP_DEBUG_ASSERT(ii >= tlevel); 2858 // AC: As we need to pass by the teams league, we need to artificially 2859 // increase ii 2860 if (ii == tlevel) { 2861 ii += 2; // three teams have same level 2862 } else { 2863 ii++; // two teams have same level 2864 } 2865 } 2866 } 2867 #endif 2868 2869 if (ii == level) 2870 return __kmp_tid_from_gtid(gtid); 2871 2872 dd = team->t.t_serialized; 2873 level++; 2874 while (ii > level) { 2875 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) { 2876 } 2877 if ((team->t.t_serialized) && (!dd)) { 2878 team = team->t.t_parent; 2879 continue; 2880 } 2881 if (ii > level) { 2882 team = team->t.t_parent; 2883 dd = team->t.t_serialized; 2884 ii--; 2885 } 2886 } 2887 2888 return (dd > 1) ? (0) : (team->t.t_master_tid); 2889 } 2890 2891 int __kmp_get_team_size(int gtid, int level) { 2892 2893 int ii, dd; 2894 kmp_team_t *team; 2895 kmp_info_t *thr; 2896 2897 KF_TRACE(10, ("__kmp_get_team_size: thread %d %d\n", gtid, level)); 2898 KMP_DEBUG_ASSERT(__kmp_init_serial); 2899 2900 // validate level 2901 if (level == 0) 2902 return 1; 2903 if (level < 0) 2904 return -1; 2905 thr = __kmp_threads[gtid]; 2906 team = thr->th.th_team; 2907 ii = team->t.t_level; 2908 if (level > ii) 2909 return -1; 2910 2911 #if OMP_40_ENABLED 2912 if (thr->th.th_teams_microtask) { 2913 // AC: we are in teams region where multiple nested teams have same level 2914 int tlevel = thr->th.th_teams_level; // the level of the teams construct 2915 if (level <= 2916 tlevel) { // otherwise usual algorithm works (will not touch the teams) 2917 KMP_DEBUG_ASSERT(ii >= tlevel); 2918 // AC: As we need to pass by the teams league, we need to artificially 2919 // increase ii 2920 if (ii == tlevel) { 2921 ii += 2; // three teams have same level 2922 } else { 2923 ii++; // two teams have same level 2924 } 2925 } 2926 } 2927 #endif 2928 2929 while (ii > level) { 2930 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) { 2931 } 2932 if (team->t.t_serialized && (!dd)) { 2933 team = team->t.t_parent; 2934 continue; 2935 } 2936 if (ii > level) { 2937 team = team->t.t_parent; 2938 ii--; 2939 } 2940 } 2941 2942 return team->t.t_nproc; 2943 } 2944 2945 kmp_r_sched_t __kmp_get_schedule_global() { 2946 // This routine created because pairs (__kmp_sched, __kmp_chunk) and 2947 // (__kmp_static, __kmp_guided) may be changed by kmp_set_defaults 2948 // independently. So one can get the updated schedule here. 2949 2950 kmp_r_sched_t r_sched; 2951 2952 // create schedule from 4 globals: __kmp_sched, __kmp_chunk, __kmp_static, 2953 // __kmp_guided. __kmp_sched should keep original value, so that user can set 2954 // KMP_SCHEDULE multiple times, and thus have different run-time schedules in 2955 // different roots (even in OMP 2.5) 2956 if (__kmp_sched == kmp_sch_static) { 2957 r_sched.r_sched_type = __kmp_static; // replace STATIC with more detailed 2958 // schedule (balanced or greedy) 2959 } else if (__kmp_sched == kmp_sch_guided_chunked) { 2960 r_sched.r_sched_type = __kmp_guided; // replace GUIDED with more detailed 2961 // schedule (iterative or analytical) 2962 } else { 2963 r_sched.r_sched_type = 2964 __kmp_sched; // (STATIC_CHUNKED), or (DYNAMIC_CHUNKED), or other 2965 } 2966 2967 if (__kmp_chunk < KMP_DEFAULT_CHUNK) { // __kmp_chunk may be wrong here (if it 2968 // was not ever set) 2969 r_sched.chunk = KMP_DEFAULT_CHUNK; 2970 } else { 2971 r_sched.chunk = __kmp_chunk; 2972 } 2973 2974 return r_sched; 2975 } 2976 2977 /* Allocate (realloc == FALSE) * or reallocate (realloc == TRUE) 2978 at least argc number of *t_argv entries for the requested team. */ 2979 static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team, int realloc) { 2980 2981 KMP_DEBUG_ASSERT(team); 2982 if (!realloc || argc > team->t.t_max_argc) { 2983 2984 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: needed entries=%d, " 2985 "current entries=%d\n", 2986 team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0)); 2987 /* if previously allocated heap space for args, free them */ 2988 if (realloc && team->t.t_argv != &team->t.t_inline_argv[0]) 2989 __kmp_free((void *)team->t.t_argv); 2990 2991 if (argc <= KMP_INLINE_ARGV_ENTRIES) { 2992 /* use unused space in the cache line for arguments */ 2993 team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES; 2994 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: inline allocate %d " 2995 "argv entries\n", 2996 team->t.t_id, team->t.t_max_argc)); 2997 team->t.t_argv = &team->t.t_inline_argv[0]; 2998 if (__kmp_storage_map) { 2999 __kmp_print_storage_map_gtid( 3000 -1, &team->t.t_inline_argv[0], 3001 &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES], 3002 (sizeof(void *) * KMP_INLINE_ARGV_ENTRIES), "team_%d.t_inline_argv", 3003 team->t.t_id); 3004 } 3005 } else { 3006 /* allocate space for arguments in the heap */ 3007 team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES >> 1)) 3008 ? KMP_MIN_MALLOC_ARGV_ENTRIES 3009 : 2 * argc; 3010 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d " 3011 "argv entries\n", 3012 team->t.t_id, team->t.t_max_argc)); 3013 team->t.t_argv = 3014 (void **)__kmp_page_allocate(sizeof(void *) * team->t.t_max_argc); 3015 if (__kmp_storage_map) { 3016 __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0], 3017 &team->t.t_argv[team->t.t_max_argc], 3018 sizeof(void *) * team->t.t_max_argc, 3019 "team_%d.t_argv", team->t.t_id); 3020 } 3021 } 3022 } 3023 } 3024 3025 static void __kmp_allocate_team_arrays(kmp_team_t *team, int max_nth) { 3026 int i; 3027 int num_disp_buff = max_nth > 1 ? __kmp_dispatch_num_buffers : 2; 3028 team->t.t_threads = 3029 (kmp_info_t **)__kmp_allocate(sizeof(kmp_info_t *) * max_nth); 3030 team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate( 3031 sizeof(dispatch_shared_info_t) * num_disp_buff); 3032 team->t.t_dispatch = 3033 (kmp_disp_t *)__kmp_allocate(sizeof(kmp_disp_t) * max_nth); 3034 team->t.t_implicit_task_taskdata = 3035 (kmp_taskdata_t *)__kmp_allocate(sizeof(kmp_taskdata_t) * max_nth); 3036 team->t.t_max_nproc = max_nth; 3037 3038 /* setup dispatch buffers */ 3039 for (i = 0; i < num_disp_buff; ++i) { 3040 team->t.t_disp_buffer[i].buffer_index = i; 3041 #if OMP_45_ENABLED 3042 team->t.t_disp_buffer[i].doacross_buf_idx = i; 3043 #endif 3044 } 3045 } 3046 3047 static void __kmp_free_team_arrays(kmp_team_t *team) { 3048 /* Note: this does not free the threads in t_threads (__kmp_free_threads) */ 3049 int i; 3050 for (i = 0; i < team->t.t_max_nproc; ++i) { 3051 if (team->t.t_dispatch[i].th_disp_buffer != NULL) { 3052 __kmp_free(team->t.t_dispatch[i].th_disp_buffer); 3053 team->t.t_dispatch[i].th_disp_buffer = NULL; 3054 }; // if 3055 }; // for 3056 __kmp_free(team->t.t_threads); 3057 __kmp_free(team->t.t_disp_buffer); 3058 __kmp_free(team->t.t_dispatch); 3059 __kmp_free(team->t.t_implicit_task_taskdata); 3060 team->t.t_threads = NULL; 3061 team->t.t_disp_buffer = NULL; 3062 team->t.t_dispatch = NULL; 3063 team->t.t_implicit_task_taskdata = 0; 3064 } 3065 3066 static void __kmp_reallocate_team_arrays(kmp_team_t *team, int max_nth) { 3067 kmp_info_t **oldThreads = team->t.t_threads; 3068 3069 __kmp_free(team->t.t_disp_buffer); 3070 __kmp_free(team->t.t_dispatch); 3071 __kmp_free(team->t.t_implicit_task_taskdata); 3072 __kmp_allocate_team_arrays(team, max_nth); 3073 3074 KMP_MEMCPY(team->t.t_threads, oldThreads, 3075 team->t.t_nproc * sizeof(kmp_info_t *)); 3076 3077 __kmp_free(oldThreads); 3078 } 3079 3080 static kmp_internal_control_t __kmp_get_global_icvs(void) { 3081 3082 kmp_r_sched_t r_sched = 3083 __kmp_get_schedule_global(); // get current state of scheduling globals 3084 3085 #if OMP_40_ENABLED 3086 KMP_DEBUG_ASSERT(__kmp_nested_proc_bind.used > 0); 3087 #endif /* OMP_40_ENABLED */ 3088 3089 kmp_internal_control_t g_icvs = { 3090 0, // int serial_nesting_level; //corresponds to value of th_team_serialized 3091 (kmp_int8)__kmp_dflt_nested, // int nested; //internal control 3092 // for nested parallelism (per thread) 3093 (kmp_int8)__kmp_global.g.g_dynamic, // internal control for dynamic 3094 // adjustment of threads (per thread) 3095 (kmp_int8)__kmp_env_blocktime, // int bt_set; //internal control for 3096 // whether blocktime is explicitly set 3097 __kmp_dflt_blocktime, // int blocktime; //internal control for blocktime 3098 #if KMP_USE_MONITOR 3099 __kmp_bt_intervals, // int bt_intervals; //internal control for blocktime 3100 // intervals 3101 #endif 3102 __kmp_dflt_team_nth, // int nproc; //internal control for # of threads for 3103 // next parallel region (per thread) 3104 // (use a max ub on value if __kmp_parallel_initialize not called yet) 3105 __kmp_dflt_max_active_levels, // int max_active_levels; //internal control 3106 // for max_active_levels 3107 r_sched, // kmp_r_sched_t sched; //internal control for runtime schedule 3108 // {sched,chunk} pair 3109 #if OMP_40_ENABLED 3110 __kmp_nested_proc_bind.bind_types[0], 3111 __kmp_default_device, 3112 #endif /* OMP_40_ENABLED */ 3113 NULL // struct kmp_internal_control *next; 3114 }; 3115 3116 return g_icvs; 3117 } 3118 3119 static kmp_internal_control_t __kmp_get_x_global_icvs(const kmp_team_t *team) { 3120 3121 kmp_internal_control_t gx_icvs; 3122 gx_icvs.serial_nesting_level = 3123 0; // probably =team->t.t_serial like in save_inter_controls 3124 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs); 3125 gx_icvs.next = NULL; 3126 3127 return gx_icvs; 3128 } 3129 3130 static void __kmp_initialize_root(kmp_root_t *root) { 3131 int f; 3132 kmp_team_t *root_team; 3133 kmp_team_t *hot_team; 3134 int hot_team_max_nth; 3135 kmp_r_sched_t r_sched = 3136 __kmp_get_schedule_global(); // get current state of scheduling globals 3137 kmp_internal_control_t r_icvs = __kmp_get_global_icvs(); 3138 KMP_DEBUG_ASSERT(root); 3139 KMP_ASSERT(!root->r.r_begin); 3140 3141 /* setup the root state structure */ 3142 __kmp_init_lock(&root->r.r_begin_lock); 3143 root->r.r_begin = FALSE; 3144 root->r.r_active = FALSE; 3145 root->r.r_in_parallel = 0; 3146 root->r.r_blocktime = __kmp_dflt_blocktime; 3147 root->r.r_nested = __kmp_dflt_nested; 3148 root->r.r_cg_nthreads = 1; 3149 3150 /* setup the root team for this task */ 3151 /* allocate the root team structure */ 3152 KF_TRACE(10, ("__kmp_initialize_root: before root_team\n")); 3153 3154 root_team = 3155 __kmp_allocate_team(root, 3156 1, // new_nproc 3157 1, // max_nproc 3158 #if OMPT_SUPPORT 3159 0, // root parallel id 3160 #endif 3161 #if OMP_40_ENABLED 3162 __kmp_nested_proc_bind.bind_types[0], 3163 #endif 3164 &r_icvs, 3165 0 // argc 3166 USE_NESTED_HOT_ARG(NULL) // master thread is unknown 3167 ); 3168 #if USE_DEBUGGER 3169 // Non-NULL value should be assigned to make the debugger display the root 3170 // team. 3171 TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)(~0)); 3172 #endif 3173 3174 KF_TRACE(10, ("__kmp_initialize_root: after root_team = %p\n", root_team)); 3175 3176 root->r.r_root_team = root_team; 3177 root_team->t.t_control_stack_top = NULL; 3178 3179 /* initialize root team */ 3180 root_team->t.t_threads[0] = NULL; 3181 root_team->t.t_nproc = 1; 3182 root_team->t.t_serialized = 1; 3183 // TODO???: root_team->t.t_max_active_levels = __kmp_dflt_max_active_levels; 3184 root_team->t.t_sched.r_sched_type = r_sched.r_sched_type; 3185 root_team->t.t_sched.chunk = r_sched.chunk; 3186 KA_TRACE( 3187 20, 3188 ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n", 3189 root_team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); 3190 3191 /* setup the hot team for this task */ 3192 /* allocate the hot team structure */ 3193 KF_TRACE(10, ("__kmp_initialize_root: before hot_team\n")); 3194 3195 hot_team = 3196 __kmp_allocate_team(root, 3197 1, // new_nproc 3198 __kmp_dflt_team_nth_ub * 2, // max_nproc 3199 #if OMPT_SUPPORT 3200 0, // root parallel id 3201 #endif 3202 #if OMP_40_ENABLED 3203 __kmp_nested_proc_bind.bind_types[0], 3204 #endif 3205 &r_icvs, 3206 0 // argc 3207 USE_NESTED_HOT_ARG(NULL) // master thread is unknown 3208 ); 3209 KF_TRACE(10, ("__kmp_initialize_root: after hot_team = %p\n", hot_team)); 3210 3211 root->r.r_hot_team = hot_team; 3212 root_team->t.t_control_stack_top = NULL; 3213 3214 /* first-time initialization */ 3215 hot_team->t.t_parent = root_team; 3216 3217 /* initialize hot team */ 3218 hot_team_max_nth = hot_team->t.t_max_nproc; 3219 for (f = 0; f < hot_team_max_nth; ++f) { 3220 hot_team->t.t_threads[f] = NULL; 3221 }; // for 3222 hot_team->t.t_nproc = 1; 3223 // TODO???: hot_team->t.t_max_active_levels = __kmp_dflt_max_active_levels; 3224 hot_team->t.t_sched.r_sched_type = r_sched.r_sched_type; 3225 hot_team->t.t_sched.chunk = r_sched.chunk; 3226 hot_team->t.t_size_changed = 0; 3227 } 3228 3229 #ifdef KMP_DEBUG 3230 3231 typedef struct kmp_team_list_item { 3232 kmp_team_p const *entry; 3233 struct kmp_team_list_item *next; 3234 } kmp_team_list_item_t; 3235 typedef kmp_team_list_item_t *kmp_team_list_t; 3236 3237 static void __kmp_print_structure_team_accum( // Add team to list of teams. 3238 kmp_team_list_t list, // List of teams. 3239 kmp_team_p const *team // Team to add. 3240 ) { 3241 3242 // List must terminate with item where both entry and next are NULL. 3243 // Team is added to the list only once. 3244 // List is sorted in ascending order by team id. 3245 // Team id is *not* a key. 3246 3247 kmp_team_list_t l; 3248 3249 KMP_DEBUG_ASSERT(list != NULL); 3250 if (team == NULL) { 3251 return; 3252 }; // if 3253 3254 __kmp_print_structure_team_accum(list, team->t.t_parent); 3255 __kmp_print_structure_team_accum(list, team->t.t_next_pool); 3256 3257 // Search list for the team. 3258 l = list; 3259 while (l->next != NULL && l->entry != team) { 3260 l = l->next; 3261 }; // while 3262 if (l->next != NULL) { 3263 return; // Team has been added before, exit. 3264 }; // if 3265 3266 // Team is not found. Search list again for insertion point. 3267 l = list; 3268 while (l->next != NULL && l->entry->t.t_id <= team->t.t_id) { 3269 l = l->next; 3270 }; // while 3271 3272 // Insert team. 3273 { 3274 kmp_team_list_item_t *item = (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC( 3275 sizeof(kmp_team_list_item_t)); 3276 *item = *l; 3277 l->entry = team; 3278 l->next = item; 3279 } 3280 } 3281 3282 static void __kmp_print_structure_team(char const *title, kmp_team_p const *team 3283 3284 ) { 3285 __kmp_printf("%s", title); 3286 if (team != NULL) { 3287 __kmp_printf("%2x %p\n", team->t.t_id, team); 3288 } else { 3289 __kmp_printf(" - (nil)\n"); 3290 }; // if 3291 } 3292 3293 static void __kmp_print_structure_thread(char const *title, 3294 kmp_info_p const *thread) { 3295 __kmp_printf("%s", title); 3296 if (thread != NULL) { 3297 __kmp_printf("%2d %p\n", thread->th.th_info.ds.ds_gtid, thread); 3298 } else { 3299 __kmp_printf(" - (nil)\n"); 3300 }; // if 3301 } 3302 3303 void __kmp_print_structure(void) { 3304 3305 kmp_team_list_t list; 3306 3307 // Initialize list of teams. 3308 list = 3309 (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(sizeof(kmp_team_list_item_t)); 3310 list->entry = NULL; 3311 list->next = NULL; 3312 3313 __kmp_printf("\n------------------------------\nGlobal Thread " 3314 "Table\n------------------------------\n"); 3315 { 3316 int gtid; 3317 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) { 3318 __kmp_printf("%2d", gtid); 3319 if (__kmp_threads != NULL) { 3320 __kmp_printf(" %p", __kmp_threads[gtid]); 3321 }; // if 3322 if (__kmp_root != NULL) { 3323 __kmp_printf(" %p", __kmp_root[gtid]); 3324 }; // if 3325 __kmp_printf("\n"); 3326 }; // for gtid 3327 } 3328 3329 // Print out __kmp_threads array. 3330 __kmp_printf("\n------------------------------\nThreads\n--------------------" 3331 "----------\n"); 3332 if (__kmp_threads != NULL) { 3333 int gtid; 3334 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) { 3335 kmp_info_t const *thread = __kmp_threads[gtid]; 3336 if (thread != NULL) { 3337 __kmp_printf("GTID %2d %p:\n", gtid, thread); 3338 __kmp_printf(" Our Root: %p\n", thread->th.th_root); 3339 __kmp_print_structure_team(" Our Team: ", thread->th.th_team); 3340 __kmp_print_structure_team(" Serial Team: ", 3341 thread->th.th_serial_team); 3342 __kmp_printf(" Threads: %2d\n", thread->th.th_team_nproc); 3343 __kmp_print_structure_thread(" Master: ", 3344 thread->th.th_team_master); 3345 __kmp_printf(" Serialized?: %2d\n", thread->th.th_team_serialized); 3346 __kmp_printf(" Set NProc: %2d\n", thread->th.th_set_nproc); 3347 #if OMP_40_ENABLED 3348 __kmp_printf(" Set Proc Bind: %2d\n", thread->th.th_set_proc_bind); 3349 #endif 3350 __kmp_print_structure_thread(" Next in pool: ", 3351 thread->th.th_next_pool); 3352 __kmp_printf("\n"); 3353 __kmp_print_structure_team_accum(list, thread->th.th_team); 3354 __kmp_print_structure_team_accum(list, thread->th.th_serial_team); 3355 }; // if 3356 }; // for gtid 3357 } else { 3358 __kmp_printf("Threads array is not allocated.\n"); 3359 }; // if 3360 3361 // Print out __kmp_root array. 3362 __kmp_printf("\n------------------------------\nUbers\n----------------------" 3363 "--------\n"); 3364 if (__kmp_root != NULL) { 3365 int gtid; 3366 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) { 3367 kmp_root_t const *root = __kmp_root[gtid]; 3368 if (root != NULL) { 3369 __kmp_printf("GTID %2d %p:\n", gtid, root); 3370 __kmp_print_structure_team(" Root Team: ", root->r.r_root_team); 3371 __kmp_print_structure_team(" Hot Team: ", root->r.r_hot_team); 3372 __kmp_print_structure_thread(" Uber Thread: ", 3373 root->r.r_uber_thread); 3374 __kmp_printf(" Active?: %2d\n", root->r.r_active); 3375 __kmp_printf(" Nested?: %2d\n", root->r.r_nested); 3376 __kmp_printf(" In Parallel: %2d\n", root->r.r_in_parallel); 3377 __kmp_printf("\n"); 3378 __kmp_print_structure_team_accum(list, root->r.r_root_team); 3379 __kmp_print_structure_team_accum(list, root->r.r_hot_team); 3380 }; // if 3381 }; // for gtid 3382 } else { 3383 __kmp_printf("Ubers array is not allocated.\n"); 3384 }; // if 3385 3386 __kmp_printf("\n------------------------------\nTeams\n----------------------" 3387 "--------\n"); 3388 while (list->next != NULL) { 3389 kmp_team_p const *team = list->entry; 3390 int i; 3391 __kmp_printf("Team %2x %p:\n", team->t.t_id, team); 3392 __kmp_print_structure_team(" Parent Team: ", team->t.t_parent); 3393 __kmp_printf(" Master TID: %2d\n", team->t.t_master_tid); 3394 __kmp_printf(" Max threads: %2d\n", team->t.t_max_nproc); 3395 __kmp_printf(" Levels of serial: %2d\n", team->t.t_serialized); 3396 __kmp_printf(" Number threads: %2d\n", team->t.t_nproc); 3397 for (i = 0; i < team->t.t_nproc; ++i) { 3398 __kmp_printf(" Thread %2d: ", i); 3399 __kmp_print_structure_thread("", team->t.t_threads[i]); 3400 }; // for i 3401 __kmp_print_structure_team(" Next in pool: ", team->t.t_next_pool); 3402 __kmp_printf("\n"); 3403 list = list->next; 3404 }; // while 3405 3406 // Print out __kmp_thread_pool and __kmp_team_pool. 3407 __kmp_printf("\n------------------------------\nPools\n----------------------" 3408 "--------\n"); 3409 __kmp_print_structure_thread("Thread pool: ", 3410 CCAST(kmp_info_t *, __kmp_thread_pool)); 3411 __kmp_print_structure_team("Team pool: ", 3412 CCAST(kmp_team_t *, __kmp_team_pool)); 3413 __kmp_printf("\n"); 3414 3415 // Free team list. 3416 while (list != NULL) { 3417 kmp_team_list_item_t *item = list; 3418 list = list->next; 3419 KMP_INTERNAL_FREE(item); 3420 }; // while 3421 } 3422 3423 #endif 3424 3425 //--------------------------------------------------------------------------- 3426 // Stuff for per-thread fast random number generator 3427 // Table of primes 3428 static const unsigned __kmp_primes[] = { 3429 0x9e3779b1, 0xffe6cc59, 0x2109f6dd, 0x43977ab5, 0xba5703f5, 0xb495a877, 3430 0xe1626741, 0x79695e6b, 0xbc98c09f, 0xd5bee2b3, 0x287488f9, 0x3af18231, 3431 0x9677cd4d, 0xbe3a6929, 0xadc6a877, 0xdcf0674b, 0xbe4d6fe9, 0x5f15e201, 3432 0x99afc3fd, 0xf3f16801, 0xe222cfff, 0x24ba5fdb, 0x0620452d, 0x79f149e3, 3433 0xc8b93f49, 0x972702cd, 0xb07dd827, 0x6c97d5ed, 0x085a3d61, 0x46eb5ea7, 3434 0x3d9910ed, 0x2e687b5b, 0x29609227, 0x6eb081f1, 0x0954c4e1, 0x9d114db9, 3435 0x542acfa9, 0xb3e6bd7b, 0x0742d917, 0xe9f3ffa7, 0x54581edb, 0xf2480f45, 3436 0x0bb9288f, 0xef1affc7, 0x85fa0ca7, 0x3ccc14db, 0xe6baf34b, 0x343377f7, 3437 0x5ca19031, 0xe6d9293b, 0xf0a9f391, 0x5d2e980b, 0xfc411073, 0xc3749363, 3438 0xb892d829, 0x3549366b, 0x629750ad, 0xb98294e5, 0x892d9483, 0xc235baf3, 3439 0x3d2402a3, 0x6bdef3c9, 0xbec333cd, 0x40c9520f}; 3440 3441 //--------------------------------------------------------------------------- 3442 // __kmp_get_random: Get a random number using a linear congruential method. 3443 unsigned short __kmp_get_random(kmp_info_t *thread) { 3444 unsigned x = thread->th.th_x; 3445 unsigned short r = x >> 16; 3446 3447 thread->th.th_x = x * thread->th.th_a + 1; 3448 3449 KA_TRACE(30, ("__kmp_get_random: THREAD: %d, RETURN: %u\n", 3450 thread->th.th_info.ds.ds_tid, r)); 3451 3452 return r; 3453 } 3454 //-------------------------------------------------------- 3455 // __kmp_init_random: Initialize a random number generator 3456 void __kmp_init_random(kmp_info_t *thread) { 3457 unsigned seed = thread->th.th_info.ds.ds_tid; 3458 3459 thread->th.th_a = 3460 __kmp_primes[seed % (sizeof(__kmp_primes) / sizeof(__kmp_primes[0]))]; 3461 thread->th.th_x = (seed + 1) * thread->th.th_a + 1; 3462 KA_TRACE(30, 3463 ("__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a)); 3464 } 3465 3466 #if KMP_OS_WINDOWS 3467 /* reclaim array entries for root threads that are already dead, returns number 3468 * reclaimed */ 3469 static int __kmp_reclaim_dead_roots(void) { 3470 int i, r = 0; 3471 3472 for (i = 0; i < __kmp_threads_capacity; ++i) { 3473 if (KMP_UBER_GTID(i) && 3474 !__kmp_still_running((kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[i])) && 3475 !__kmp_root[i] 3476 ->r.r_active) { // AC: reclaim only roots died in non-active state 3477 r += __kmp_unregister_root_other_thread(i); 3478 } 3479 } 3480 return r; 3481 } 3482 #endif 3483 3484 /* This function attempts to create free entries in __kmp_threads and 3485 __kmp_root, and returns the number of free entries generated. 3486 3487 For Windows* OS static library, the first mechanism used is to reclaim array 3488 entries for root threads that are already dead. 3489 3490 On all platforms, expansion is attempted on the arrays __kmp_threads_ and 3491 __kmp_root, with appropriate update to __kmp_threads_capacity. Array 3492 capacity is increased by doubling with clipping to __kmp_tp_capacity, if 3493 threadprivate cache array has been created. Synchronization with 3494 __kmpc_threadprivate_cached is done using __kmp_tp_cached_lock. 3495 3496 After any dead root reclamation, if the clipping value allows array expansion 3497 to result in the generation of a total of nWish free slots, the function does 3498 that expansion. If not, but the clipping value allows array expansion to 3499 result in the generation of a total of nNeed free slots, the function does 3500 that expansion. Otherwise, nothing is done beyond the possible initial root 3501 thread reclamation. However, if nNeed is zero, a best-effort attempt is made 3502 to fulfil nWish as far as possible, i.e. the function will attempt to create 3503 as many free slots as possible up to nWish. 3504 3505 If any argument is negative, the behavior is undefined. */ 3506 static int __kmp_expand_threads(int nWish, int nNeed) { 3507 int added = 0; 3508 int old_tp_cached; 3509 int __kmp_actual_max_nth; 3510 3511 if (nNeed > nWish) /* normalize the arguments */ 3512 nWish = nNeed; 3513 #if KMP_OS_WINDOWS && !defined KMP_DYNAMIC_LIB 3514 /* only for Windows static library */ 3515 /* reclaim array entries for root threads that are already dead */ 3516 added = __kmp_reclaim_dead_roots(); 3517 3518 if (nNeed) { 3519 nNeed -= added; 3520 if (nNeed < 0) 3521 nNeed = 0; 3522 } 3523 if (nWish) { 3524 nWish -= added; 3525 if (nWish < 0) 3526 nWish = 0; 3527 } 3528 #endif 3529 if (nWish <= 0) 3530 return added; 3531 3532 while (1) { 3533 int nTarget; 3534 int minimumRequiredCapacity; 3535 int newCapacity; 3536 kmp_info_t **newThreads; 3537 kmp_root_t **newRoot; 3538 3539 // Note that __kmp_threads_capacity is not bounded by __kmp_max_nth. If 3540 // __kmp_max_nth is set to some value less than __kmp_sys_max_nth by the 3541 // user via KMP_DEVICE_THREAD_LIMIT, then __kmp_threads_capacity may become 3542 // > __kmp_max_nth in one of two ways: 3543 // 3544 // 1) The initialization thread (gtid = 0) exits. __kmp_threads[0] 3545 // may not be resused by another thread, so we may need to increase 3546 // __kmp_threads_capacity to __kmp_max_nth + 1. 3547 // 3548 // 2) New foreign root(s) are encountered. We always register new foreign 3549 // roots. This may cause a smaller # of threads to be allocated at 3550 // subsequent parallel regions, but the worker threads hang around (and 3551 // eventually go to sleep) and need slots in the __kmp_threads[] array. 3552 // 3553 // Anyway, that is the reason for moving the check to see if 3554 // __kmp_max_nth was exceeded into __kmp_reserve_threads() 3555 // instead of having it performed here. -BB 3556 old_tp_cached = __kmp_tp_cached; 3557 __kmp_actual_max_nth = 3558 old_tp_cached ? __kmp_tp_capacity : __kmp_sys_max_nth; 3559 KMP_DEBUG_ASSERT(__kmp_actual_max_nth >= __kmp_threads_capacity); 3560 3561 /* compute expansion headroom to check if we can expand and whether to aim 3562 for nWish or nNeed */ 3563 nTarget = nWish; 3564 if (__kmp_actual_max_nth - __kmp_threads_capacity < nTarget) { 3565 /* can't fulfil nWish, so try nNeed */ 3566 if (nNeed) { 3567 nTarget = nNeed; 3568 if (__kmp_actual_max_nth - __kmp_threads_capacity < nTarget) { 3569 /* possible expansion too small -- give up */ 3570 break; 3571 } 3572 } else { 3573 /* best-effort */ 3574 nTarget = __kmp_actual_max_nth - __kmp_threads_capacity; 3575 if (!nTarget) { 3576 /* can expand at all -- give up */ 3577 break; 3578 } 3579 } 3580 } 3581 minimumRequiredCapacity = __kmp_threads_capacity + nTarget; 3582 3583 newCapacity = __kmp_threads_capacity; 3584 do { 3585 newCapacity = newCapacity <= (__kmp_actual_max_nth >> 1) 3586 ? (newCapacity << 1) 3587 : __kmp_actual_max_nth; 3588 } while (newCapacity < minimumRequiredCapacity); 3589 newThreads = (kmp_info_t **)__kmp_allocate( 3590 (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * newCapacity + 3591 CACHE_LINE); 3592 newRoot = (kmp_root_t **)((char *)newThreads + 3593 sizeof(kmp_info_t *) * newCapacity); 3594 KMP_MEMCPY(newThreads, __kmp_threads, 3595 __kmp_threads_capacity * sizeof(kmp_info_t *)); 3596 KMP_MEMCPY(newRoot, __kmp_root, 3597 __kmp_threads_capacity * sizeof(kmp_root_t *)); 3598 memset(newThreads + __kmp_threads_capacity, 0, 3599 (newCapacity - __kmp_threads_capacity) * sizeof(kmp_info_t *)); 3600 memset(newRoot + __kmp_threads_capacity, 0, 3601 (newCapacity - __kmp_threads_capacity) * sizeof(kmp_root_t *)); 3602 3603 if (!old_tp_cached && __kmp_tp_cached && newCapacity > __kmp_tp_capacity) { 3604 /* __kmp_tp_cached has changed, i.e. __kmpc_threadprivate_cached has 3605 allocated a threadprivate cache while we were allocating the expanded 3606 array, and our new capacity is larger than the threadprivate cache 3607 capacity, so we should deallocate the expanded arrays and try again. 3608 This is the first check of a double-check pair. */ 3609 __kmp_free(newThreads); 3610 continue; /* start over and try again */ 3611 } 3612 __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock); 3613 if (!old_tp_cached && __kmp_tp_cached && newCapacity > __kmp_tp_capacity) { 3614 /* Same check as above, but this time with the lock so we can be sure if 3615 we can succeed. */ 3616 __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock); 3617 __kmp_free(newThreads); 3618 continue; /* start over and try again */ 3619 } else { 3620 /* success */ 3621 // __kmp_free( __kmp_threads ); // ATT: It leads to crash. Need to be 3622 // investigated. 3623 *(kmp_info_t * *volatile *)&__kmp_threads = newThreads; 3624 *(kmp_root_t * *volatile *)&__kmp_root = newRoot; 3625 added += newCapacity - __kmp_threads_capacity; 3626 *(volatile int *)&__kmp_threads_capacity = newCapacity; 3627 __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock); 3628 break; /* succeeded, so we can exit the loop */ 3629 } 3630 } 3631 return added; 3632 } 3633 3634 /* Register the current thread as a root thread and obtain our gtid. We must 3635 have the __kmp_initz_lock held at this point. Argument TRUE only if are the 3636 thread that calls from __kmp_do_serial_initialize() */ 3637 int __kmp_register_root(int initial_thread) { 3638 kmp_info_t *root_thread; 3639 kmp_root_t *root; 3640 int gtid; 3641 int capacity; 3642 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 3643 KA_TRACE(20, ("__kmp_register_root: entered\n")); 3644 KMP_MB(); 3645 3646 /* 2007-03-02: 3647 If initial thread did not invoke OpenMP RTL yet, and this thread is not an 3648 initial one, "__kmp_all_nth >= __kmp_threads_capacity" condition does not 3649 work as expected -- it may return false (that means there is at least one 3650 empty slot in __kmp_threads array), but it is possible the only free slot 3651 is #0, which is reserved for initial thread and so cannot be used for this 3652 one. Following code workarounds this bug. 3653 3654 However, right solution seems to be not reserving slot #0 for initial 3655 thread because: 3656 (1) there is no magic in slot #0, 3657 (2) we cannot detect initial thread reliably (the first thread which does 3658 serial initialization may be not a real initial thread). 3659 */ 3660 capacity = __kmp_threads_capacity; 3661 if (!initial_thread && TCR_PTR(__kmp_threads[0]) == NULL) { 3662 --capacity; 3663 }; // if 3664 3665 /* see if there are too many threads */ 3666 if (__kmp_all_nth >= capacity && !__kmp_expand_threads(1, 1)) { 3667 if (__kmp_tp_cached) { 3668 __kmp_fatal(KMP_MSG(CantRegisterNewThread), 3669 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity), 3670 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null); 3671 } else { 3672 __kmp_fatal(KMP_MSG(CantRegisterNewThread), KMP_HNT(SystemLimitOnThreads), 3673 __kmp_msg_null); 3674 } 3675 }; // if 3676 3677 /* find an available thread slot */ 3678 /* Don't reassign the zero slot since we need that to only be used by initial 3679 thread */ 3680 for (gtid = (initial_thread ? 0 : 1); TCR_PTR(__kmp_threads[gtid]) != NULL; 3681 gtid++) 3682 ; 3683 KA_TRACE(1, 3684 ("__kmp_register_root: found slot in threads array: T#%d\n", gtid)); 3685 KMP_ASSERT(gtid < __kmp_threads_capacity); 3686 3687 /* update global accounting */ 3688 __kmp_all_nth++; 3689 TCW_4(__kmp_nth, __kmp_nth + 1); 3690 3691 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low 3692 // numbers of procs, and method #2 (keyed API call) for higher numbers. 3693 if (__kmp_adjust_gtid_mode) { 3694 if (__kmp_all_nth >= __kmp_tls_gtid_min) { 3695 if (TCR_4(__kmp_gtid_mode) != 2) { 3696 TCW_4(__kmp_gtid_mode, 2); 3697 } 3698 } else { 3699 if (TCR_4(__kmp_gtid_mode) != 1) { 3700 TCW_4(__kmp_gtid_mode, 1); 3701 } 3702 } 3703 } 3704 3705 #ifdef KMP_ADJUST_BLOCKTIME 3706 /* Adjust blocktime to zero if necessary */ 3707 /* Middle initialization might not have occurred yet */ 3708 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 3709 if (__kmp_nth > __kmp_avail_proc) { 3710 __kmp_zero_bt = TRUE; 3711 } 3712 } 3713 #endif /* KMP_ADJUST_BLOCKTIME */ 3714 3715 /* setup this new hierarchy */ 3716 if (!(root = __kmp_root[gtid])) { 3717 root = __kmp_root[gtid] = (kmp_root_t *)__kmp_allocate(sizeof(kmp_root_t)); 3718 KMP_DEBUG_ASSERT(!root->r.r_root_team); 3719 } 3720 3721 #if KMP_STATS_ENABLED 3722 // Initialize stats as soon as possible (right after gtid assignment). 3723 __kmp_stats_thread_ptr = __kmp_stats_list->push_back(gtid); 3724 KMP_START_EXPLICIT_TIMER(OMP_worker_thread_life); 3725 KMP_SET_THREAD_STATE(SERIAL_REGION); 3726 KMP_INIT_PARTITIONED_TIMERS(OMP_serial); 3727 #endif 3728 __kmp_initialize_root(root); 3729 3730 /* setup new root thread structure */ 3731 if (root->r.r_uber_thread) { 3732 root_thread = root->r.r_uber_thread; 3733 } else { 3734 root_thread = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t)); 3735 if (__kmp_storage_map) { 3736 __kmp_print_thread_storage_map(root_thread, gtid); 3737 } 3738 root_thread->th.th_info.ds.ds_gtid = gtid; 3739 root_thread->th.th_root = root; 3740 if (__kmp_env_consistency_check) { 3741 root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid); 3742 } 3743 #if USE_FAST_MEMORY 3744 __kmp_initialize_fast_memory(root_thread); 3745 #endif /* USE_FAST_MEMORY */ 3746 3747 #if KMP_USE_BGET 3748 KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL); 3749 __kmp_initialize_bget(root_thread); 3750 #endif 3751 __kmp_init_random(root_thread); // Initialize random number generator 3752 } 3753 3754 /* setup the serial team held in reserve by the root thread */ 3755 if (!root_thread->th.th_serial_team) { 3756 kmp_internal_control_t r_icvs = __kmp_get_global_icvs(); 3757 KF_TRACE(10, ("__kmp_register_root: before serial_team\n")); 3758 root_thread->th.th_serial_team = 3759 __kmp_allocate_team(root, 1, 1, 3760 #if OMPT_SUPPORT 3761 0, // root parallel id 3762 #endif 3763 #if OMP_40_ENABLED 3764 proc_bind_default, 3765 #endif 3766 &r_icvs, 0 USE_NESTED_HOT_ARG(NULL)); 3767 } 3768 KMP_ASSERT(root_thread->th.th_serial_team); 3769 KF_TRACE(10, ("__kmp_register_root: after serial_team = %p\n", 3770 root_thread->th.th_serial_team)); 3771 3772 /* drop root_thread into place */ 3773 TCW_SYNC_PTR(__kmp_threads[gtid], root_thread); 3774 3775 root->r.r_root_team->t.t_threads[0] = root_thread; 3776 root->r.r_hot_team->t.t_threads[0] = root_thread; 3777 root_thread->th.th_serial_team->t.t_threads[0] = root_thread; 3778 // AC: the team created in reserve, not for execution (it is unused for now). 3779 root_thread->th.th_serial_team->t.t_serialized = 0; 3780 root->r.r_uber_thread = root_thread; 3781 3782 /* initialize the thread, get it ready to go */ 3783 __kmp_initialize_info(root_thread, root->r.r_root_team, 0, gtid); 3784 TCW_4(__kmp_init_gtid, TRUE); 3785 3786 /* prepare the master thread for get_gtid() */ 3787 __kmp_gtid_set_specific(gtid); 3788 3789 #if USE_ITT_BUILD 3790 __kmp_itt_thread_name(gtid); 3791 #endif /* USE_ITT_BUILD */ 3792 3793 #ifdef KMP_TDATA_GTID 3794 __kmp_gtid = gtid; 3795 #endif 3796 __kmp_create_worker(gtid, root_thread, __kmp_stksize); 3797 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == gtid); 3798 3799 KA_TRACE(20, ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, " 3800 "plain=%u\n", 3801 gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team), 3802 root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE, 3803 KMP_INIT_BARRIER_STATE)); 3804 { // Initialize barrier data. 3805 int b; 3806 for (b = 0; b < bs_last_barrier; ++b) { 3807 root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE; 3808 #if USE_DEBUGGER 3809 root_thread->th.th_bar[b].bb.b_worker_arrived = 0; 3810 #endif 3811 }; // for 3812 } 3813 KMP_DEBUG_ASSERT(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived == 3814 KMP_INIT_BARRIER_STATE); 3815 3816 #if KMP_AFFINITY_SUPPORTED 3817 #if OMP_40_ENABLED 3818 root_thread->th.th_current_place = KMP_PLACE_UNDEFINED; 3819 root_thread->th.th_new_place = KMP_PLACE_UNDEFINED; 3820 root_thread->th.th_first_place = KMP_PLACE_UNDEFINED; 3821 root_thread->th.th_last_place = KMP_PLACE_UNDEFINED; 3822 #endif 3823 3824 if (TCR_4(__kmp_init_middle)) { 3825 __kmp_affinity_set_init_mask(gtid, TRUE); 3826 } 3827 #endif /* KMP_AFFINITY_SUPPORTED */ 3828 3829 __kmp_root_counter++; 3830 3831 KMP_MB(); 3832 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 3833 3834 return gtid; 3835 } 3836 3837 #if KMP_NESTED_HOT_TEAMS 3838 static int __kmp_free_hot_teams(kmp_root_t *root, kmp_info_t *thr, int level, 3839 const int max_level) { 3840 int i, n, nth; 3841 kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams; 3842 if (!hot_teams || !hot_teams[level].hot_team) { 3843 return 0; 3844 } 3845 KMP_DEBUG_ASSERT(level < max_level); 3846 kmp_team_t *team = hot_teams[level].hot_team; 3847 nth = hot_teams[level].hot_team_nth; 3848 n = nth - 1; // master is not freed 3849 if (level < max_level - 1) { 3850 for (i = 0; i < nth; ++i) { 3851 kmp_info_t *th = team->t.t_threads[i]; 3852 n += __kmp_free_hot_teams(root, th, level + 1, max_level); 3853 if (i > 0 && th->th.th_hot_teams) { 3854 __kmp_free(th->th.th_hot_teams); 3855 th->th.th_hot_teams = NULL; 3856 } 3857 } 3858 } 3859 __kmp_free_team(root, team, NULL); 3860 return n; 3861 } 3862 #endif 3863 3864 // Resets a root thread and clear its root and hot teams. 3865 // Returns the number of __kmp_threads entries directly and indirectly freed. 3866 static int __kmp_reset_root(int gtid, kmp_root_t *root) { 3867 kmp_team_t *root_team = root->r.r_root_team; 3868 kmp_team_t *hot_team = root->r.r_hot_team; 3869 int n = hot_team->t.t_nproc; 3870 int i; 3871 3872 KMP_DEBUG_ASSERT(!root->r.r_active); 3873 3874 root->r.r_root_team = NULL; 3875 root->r.r_hot_team = NULL; 3876 // __kmp_free_team() does not free hot teams, so we have to clear r_hot_team 3877 // before call to __kmp_free_team(). 3878 __kmp_free_team(root, root_team USE_NESTED_HOT_ARG(NULL)); 3879 #if KMP_NESTED_HOT_TEAMS 3880 if (__kmp_hot_teams_max_level > 3881 0) { // need to free nested hot teams and their threads if any 3882 for (i = 0; i < hot_team->t.t_nproc; ++i) { 3883 kmp_info_t *th = hot_team->t.t_threads[i]; 3884 if (__kmp_hot_teams_max_level > 1) { 3885 n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level); 3886 } 3887 if (th->th.th_hot_teams) { 3888 __kmp_free(th->th.th_hot_teams); 3889 th->th.th_hot_teams = NULL; 3890 } 3891 } 3892 } 3893 #endif 3894 __kmp_free_team(root, hot_team USE_NESTED_HOT_ARG(NULL)); 3895 3896 // Before we can reap the thread, we need to make certain that all other 3897 // threads in the teams that had this root as ancestor have stopped trying to 3898 // steal tasks. 3899 if (__kmp_tasking_mode != tskm_immediate_exec) { 3900 __kmp_wait_to_unref_task_teams(); 3901 } 3902 3903 #if KMP_OS_WINDOWS 3904 /* Close Handle of root duplicated in __kmp_create_worker (tr #62919) */ 3905 KA_TRACE( 3906 10, ("__kmp_reset_root: free handle, th = %p, handle = %" KMP_UINTPTR_SPEC 3907 "\n", 3908 (LPVOID) & (root->r.r_uber_thread->th), 3909 root->r.r_uber_thread->th.th_info.ds.ds_thread)); 3910 __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread); 3911 #endif /* KMP_OS_WINDOWS */ 3912 3913 #if OMPT_SUPPORT 3914 if (ompt_enabled && ompt_callbacks.ompt_callback(ompt_event_thread_end)) { 3915 int gtid = __kmp_get_gtid(); 3916 __ompt_thread_end(ompt_thread_initial, gtid); 3917 } 3918 #endif 3919 3920 TCW_4(__kmp_nth, 3921 __kmp_nth - 1); // __kmp_reap_thread will decrement __kmp_all_nth. 3922 root->r.r_cg_nthreads--; 3923 3924 __kmp_reap_thread(root->r.r_uber_thread, 1); 3925 3926 // We canot put root thread to __kmp_thread_pool, so we have to reap it istead 3927 // of freeing. 3928 root->r.r_uber_thread = NULL; 3929 /* mark root as no longer in use */ 3930 root->r.r_begin = FALSE; 3931 3932 return n; 3933 } 3934 3935 void __kmp_unregister_root_current_thread(int gtid) { 3936 KA_TRACE(1, ("__kmp_unregister_root_current_thread: enter T#%d\n", gtid)); 3937 /* this lock should be ok, since unregister_root_current_thread is never 3938 called during an abort, only during a normal close. furthermore, if you 3939 have the forkjoin lock, you should never try to get the initz lock */ 3940 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 3941 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 3942 KC_TRACE(10, ("__kmp_unregister_root_current_thread: already finished, " 3943 "exiting T#%d\n", 3944 gtid)); 3945 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 3946 return; 3947 } 3948 kmp_root_t *root = __kmp_root[gtid]; 3949 3950 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]); 3951 KMP_ASSERT(KMP_UBER_GTID(gtid)); 3952 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root); 3953 KMP_ASSERT(root->r.r_active == FALSE); 3954 3955 KMP_MB(); 3956 3957 #if OMP_45_ENABLED 3958 kmp_info_t *thread = __kmp_threads[gtid]; 3959 kmp_team_t *team = thread->th.th_team; 3960 kmp_task_team_t *task_team = thread->th.th_task_team; 3961 3962 // we need to wait for the proxy tasks before finishing the thread 3963 if (task_team != NULL && task_team->tt.tt_found_proxy_tasks) { 3964 #if OMPT_SUPPORT 3965 // the runtime is shutting down so we won't report any events 3966 thread->th.ompt_thread_info.state = ompt_state_undefined; 3967 #endif 3968 __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL)); 3969 } 3970 #endif 3971 3972 __kmp_reset_root(gtid, root); 3973 3974 /* free up this thread slot */ 3975 __kmp_gtid_set_specific(KMP_GTID_DNE); 3976 #ifdef KMP_TDATA_GTID 3977 __kmp_gtid = KMP_GTID_DNE; 3978 #endif 3979 3980 KMP_MB(); 3981 KC_TRACE(10, 3982 ("__kmp_unregister_root_current_thread: T#%d unregistered\n", gtid)); 3983 3984 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 3985 } 3986 3987 #if KMP_OS_WINDOWS 3988 /* __kmp_forkjoin_lock must be already held 3989 Unregisters a root thread that is not the current thread. Returns the number 3990 of __kmp_threads entries freed as a result. */ 3991 static int __kmp_unregister_root_other_thread(int gtid) { 3992 kmp_root_t *root = __kmp_root[gtid]; 3993 int r; 3994 3995 KA_TRACE(1, ("__kmp_unregister_root_other_thread: enter T#%d\n", gtid)); 3996 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]); 3997 KMP_ASSERT(KMP_UBER_GTID(gtid)); 3998 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root); 3999 KMP_ASSERT(root->r.r_active == FALSE); 4000 4001 r = __kmp_reset_root(gtid, root); 4002 KC_TRACE(10, 4003 ("__kmp_unregister_root_other_thread: T#%d unregistered\n", gtid)); 4004 return r; 4005 } 4006 #endif 4007 4008 #if KMP_DEBUG 4009 void __kmp_task_info() { 4010 4011 kmp_int32 gtid = __kmp_entry_gtid(); 4012 kmp_int32 tid = __kmp_tid_from_gtid(gtid); 4013 kmp_info_t *this_thr = __kmp_threads[gtid]; 4014 kmp_team_t *steam = this_thr->th.th_serial_team; 4015 kmp_team_t *team = this_thr->th.th_team; 4016 4017 __kmp_printf("__kmp_task_info: gtid=%d tid=%d t_thread=%p team=%p curtask=%p " 4018 "ptask=%p\n", 4019 gtid, tid, this_thr, team, this_thr->th.th_current_task, 4020 team->t.t_implicit_task_taskdata[tid].td_parent); 4021 } 4022 #endif // KMP_DEBUG 4023 4024 /* TODO optimize with one big memclr, take out what isn't needed, split 4025 responsibility to workers as much as possible, and delay initialization of 4026 features as much as possible */ 4027 static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team, 4028 int tid, int gtid) { 4029 /* this_thr->th.th_info.ds.ds_gtid is setup in 4030 kmp_allocate_thread/create_worker. 4031 this_thr->th.th_serial_team is setup in __kmp_allocate_thread */ 4032 kmp_info_t *master = team->t.t_threads[0]; 4033 KMP_DEBUG_ASSERT(this_thr != NULL); 4034 KMP_DEBUG_ASSERT(this_thr->th.th_serial_team); 4035 KMP_DEBUG_ASSERT(team); 4036 KMP_DEBUG_ASSERT(team->t.t_threads); 4037 KMP_DEBUG_ASSERT(team->t.t_dispatch); 4038 KMP_DEBUG_ASSERT(master); 4039 KMP_DEBUG_ASSERT(master->th.th_root); 4040 4041 KMP_MB(); 4042 4043 TCW_SYNC_PTR(this_thr->th.th_team, team); 4044 4045 this_thr->th.th_info.ds.ds_tid = tid; 4046 this_thr->th.th_set_nproc = 0; 4047 if (__kmp_tasking_mode != tskm_immediate_exec) 4048 // When tasking is possible, threads are not safe to reap until they are 4049 // done tasking; this will be set when tasking code is exited in wait 4050 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP; 4051 else // no tasking --> always safe to reap 4052 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP; 4053 #if OMP_40_ENABLED 4054 this_thr->th.th_set_proc_bind = proc_bind_default; 4055 #if KMP_AFFINITY_SUPPORTED 4056 this_thr->th.th_new_place = this_thr->th.th_current_place; 4057 #endif 4058 #endif 4059 this_thr->th.th_root = master->th.th_root; 4060 4061 /* setup the thread's cache of the team structure */ 4062 this_thr->th.th_team_nproc = team->t.t_nproc; 4063 this_thr->th.th_team_master = master; 4064 this_thr->th.th_team_serialized = team->t.t_serialized; 4065 TCW_PTR(this_thr->th.th_sleep_loc, NULL); 4066 4067 KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata); 4068 4069 KF_TRACE(10, ("__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n", 4070 tid, gtid, this_thr, this_thr->th.th_current_task)); 4071 4072 __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr, 4073 team, tid, TRUE); 4074 4075 KF_TRACE(10, ("__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n", 4076 tid, gtid, this_thr, this_thr->th.th_current_task)); 4077 // TODO: Initialize ICVs from parent; GEH - isn't that already done in 4078 // __kmp_initialize_team()? 4079 4080 /* TODO no worksharing in speculative threads */ 4081 this_thr->th.th_dispatch = &team->t.t_dispatch[tid]; 4082 4083 this_thr->th.th_local.this_construct = 0; 4084 4085 if (!this_thr->th.th_pri_common) { 4086 this_thr->th.th_pri_common = 4087 (struct common_table *)__kmp_allocate(sizeof(struct common_table)); 4088 if (__kmp_storage_map) { 4089 __kmp_print_storage_map_gtid( 4090 gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1, 4091 sizeof(struct common_table), "th_%d.th_pri_common\n", gtid); 4092 }; // if 4093 this_thr->th.th_pri_head = NULL; 4094 }; // if 4095 4096 /* Initialize dynamic dispatch */ 4097 { 4098 volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch; 4099 // Use team max_nproc since this will never change for the team. 4100 size_t disp_size = 4101 sizeof(dispatch_private_info_t) * 4102 (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers); 4103 KD_TRACE(10, ("__kmp_initialize_info: T#%d max_nproc: %d\n", gtid, 4104 team->t.t_max_nproc)); 4105 KMP_ASSERT(dispatch); 4106 KMP_DEBUG_ASSERT(team->t.t_dispatch); 4107 KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid]); 4108 4109 dispatch->th_disp_index = 0; 4110 #if OMP_45_ENABLED 4111 dispatch->th_doacross_buf_idx = 0; 4112 #endif 4113 if (!dispatch->th_disp_buffer) { 4114 dispatch->th_disp_buffer = 4115 (dispatch_private_info_t *)__kmp_allocate(disp_size); 4116 4117 if (__kmp_storage_map) { 4118 __kmp_print_storage_map_gtid( 4119 gtid, &dispatch->th_disp_buffer[0], 4120 &dispatch->th_disp_buffer[team->t.t_max_nproc == 1 4121 ? 1 4122 : __kmp_dispatch_num_buffers], 4123 disp_size, "th_%d.th_dispatch.th_disp_buffer " 4124 "(team_%d.t_dispatch[%d].th_disp_buffer)", 4125 gtid, team->t.t_id, gtid); 4126 } 4127 } else { 4128 memset(&dispatch->th_disp_buffer[0], '\0', disp_size); 4129 } 4130 4131 dispatch->th_dispatch_pr_current = 0; 4132 dispatch->th_dispatch_sh_current = 0; 4133 4134 dispatch->th_deo_fcn = 0; /* ORDERED */ 4135 dispatch->th_dxo_fcn = 0; /* END ORDERED */ 4136 } 4137 4138 this_thr->th.th_next_pool = NULL; 4139 4140 if (!this_thr->th.th_task_state_memo_stack) { 4141 size_t i; 4142 this_thr->th.th_task_state_memo_stack = 4143 (kmp_uint8 *)__kmp_allocate(4 * sizeof(kmp_uint8)); 4144 this_thr->th.th_task_state_top = 0; 4145 this_thr->th.th_task_state_stack_sz = 4; 4146 for (i = 0; i < this_thr->th.th_task_state_stack_sz; 4147 ++i) // zero init the stack 4148 this_thr->th.th_task_state_memo_stack[i] = 0; 4149 } 4150 4151 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here); 4152 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0); 4153 4154 KMP_MB(); 4155 } 4156 4157 /* allocate a new thread for the requesting team. this is only called from 4158 within a forkjoin critical section. we will first try to get an available 4159 thread from the thread pool. if none is available, we will fork a new one 4160 assuming we are able to create a new one. this should be assured, as the 4161 caller should check on this first. */ 4162 kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team, 4163 int new_tid) { 4164 kmp_team_t *serial_team; 4165 kmp_info_t *new_thr; 4166 int new_gtid; 4167 4168 KA_TRACE(20, ("__kmp_allocate_thread: T#%d\n", __kmp_get_gtid())); 4169 KMP_DEBUG_ASSERT(root && team); 4170 #if !KMP_NESTED_HOT_TEAMS 4171 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(__kmp_get_gtid())); 4172 #endif 4173 KMP_MB(); 4174 4175 /* first, try to get one from the thread pool */ 4176 if (__kmp_thread_pool) { 4177 4178 new_thr = CCAST(kmp_info_t *, __kmp_thread_pool); 4179 __kmp_thread_pool = (volatile kmp_info_t *)new_thr->th.th_next_pool; 4180 if (new_thr == __kmp_thread_pool_insert_pt) { 4181 __kmp_thread_pool_insert_pt = NULL; 4182 } 4183 TCW_4(new_thr->th.th_in_pool, FALSE); 4184 // Don't touch th_active_in_pool or th_active. 4185 // The worker thread adjusts those flags as it sleeps/awakens. 4186 __kmp_thread_pool_nth--; 4187 4188 KA_TRACE(20, ("__kmp_allocate_thread: T#%d using thread T#%d\n", 4189 __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid)); 4190 KMP_ASSERT(!new_thr->th.th_team); 4191 KMP_DEBUG_ASSERT(__kmp_nth < __kmp_threads_capacity); 4192 KMP_DEBUG_ASSERT(__kmp_thread_pool_nth >= 0); 4193 4194 /* setup the thread structure */ 4195 __kmp_initialize_info(new_thr, team, new_tid, 4196 new_thr->th.th_info.ds.ds_gtid); 4197 KMP_DEBUG_ASSERT(new_thr->th.th_serial_team); 4198 4199 TCW_4(__kmp_nth, __kmp_nth + 1); 4200 root->r.r_cg_nthreads++; 4201 4202 new_thr->th.th_task_state = 0; 4203 new_thr->th.th_task_state_top = 0; 4204 new_thr->th.th_task_state_stack_sz = 4; 4205 4206 #ifdef KMP_ADJUST_BLOCKTIME 4207 /* Adjust blocktime back to zero if necessary */ 4208 /* Middle initialization might not have occurred yet */ 4209 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 4210 if (__kmp_nth > __kmp_avail_proc) { 4211 __kmp_zero_bt = TRUE; 4212 } 4213 } 4214 #endif /* KMP_ADJUST_BLOCKTIME */ 4215 4216 #if KMP_DEBUG 4217 // If thread entered pool via __kmp_free_thread, wait_flag should != 4218 // KMP_BARRIER_PARENT_FLAG. 4219 int b; 4220 kmp_balign_t *balign = new_thr->th.th_bar; 4221 for (b = 0; b < bs_last_barrier; ++b) 4222 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 4223 #endif 4224 4225 KF_TRACE(10, ("__kmp_allocate_thread: T#%d using thread %p T#%d\n", 4226 __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid)); 4227 4228 KMP_MB(); 4229 return new_thr; 4230 } 4231 4232 /* no, well fork a new one */ 4233 KMP_ASSERT(__kmp_nth == __kmp_all_nth); 4234 KMP_ASSERT(__kmp_all_nth < __kmp_threads_capacity); 4235 4236 #if KMP_USE_MONITOR 4237 // If this is the first worker thread the RTL is creating, then also 4238 // launch the monitor thread. We try to do this as early as possible. 4239 if (!TCR_4(__kmp_init_monitor)) { 4240 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock); 4241 if (!TCR_4(__kmp_init_monitor)) { 4242 KF_TRACE(10, ("before __kmp_create_monitor\n")); 4243 TCW_4(__kmp_init_monitor, 1); 4244 __kmp_create_monitor(&__kmp_monitor); 4245 KF_TRACE(10, ("after __kmp_create_monitor\n")); 4246 #if KMP_OS_WINDOWS 4247 // AC: wait until monitor has started. This is a fix for CQ232808. 4248 // The reason is that if the library is loaded/unloaded in a loop with 4249 // small (parallel) work in between, then there is high probability that 4250 // monitor thread started after the library shutdown. At shutdown it is 4251 // too late to cope with the problem, because when the master is in 4252 // DllMain (process detach) the monitor has no chances to start (it is 4253 // blocked), and master has no means to inform the monitor that the 4254 // library has gone, because all the memory which the monitor can access 4255 // is going to be released/reset. 4256 while (TCR_4(__kmp_init_monitor) < 2) { 4257 KMP_YIELD(TRUE); 4258 } 4259 KF_TRACE(10, ("after monitor thread has started\n")); 4260 #endif 4261 } 4262 __kmp_release_bootstrap_lock(&__kmp_monitor_lock); 4263 } 4264 #endif 4265 4266 KMP_MB(); 4267 for (new_gtid = 1; TCR_PTR(__kmp_threads[new_gtid]) != NULL; ++new_gtid) { 4268 KMP_DEBUG_ASSERT(new_gtid < __kmp_threads_capacity); 4269 } 4270 4271 /* allocate space for it. */ 4272 new_thr = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t)); 4273 4274 TCW_SYNC_PTR(__kmp_threads[new_gtid], new_thr); 4275 4276 if (__kmp_storage_map) { 4277 __kmp_print_thread_storage_map(new_thr, new_gtid); 4278 } 4279 4280 // add the reserve serialized team, initialized from the team's master thread 4281 { 4282 kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team); 4283 KF_TRACE(10, ("__kmp_allocate_thread: before th_serial/serial_team\n")); 4284 new_thr->th.th_serial_team = serial_team = 4285 (kmp_team_t *)__kmp_allocate_team(root, 1, 1, 4286 #if OMPT_SUPPORT 4287 0, // root parallel id 4288 #endif 4289 #if OMP_40_ENABLED 4290 proc_bind_default, 4291 #endif 4292 &r_icvs, 0 USE_NESTED_HOT_ARG(NULL)); 4293 } 4294 KMP_ASSERT(serial_team); 4295 serial_team->t.t_serialized = 0; // AC: the team created in reserve, not for 4296 // execution (it is unused for now). 4297 serial_team->t.t_threads[0] = new_thr; 4298 KF_TRACE(10, 4299 ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n", 4300 new_thr)); 4301 4302 /* setup the thread structures */ 4303 __kmp_initialize_info(new_thr, team, new_tid, new_gtid); 4304 4305 #if USE_FAST_MEMORY 4306 __kmp_initialize_fast_memory(new_thr); 4307 #endif /* USE_FAST_MEMORY */ 4308 4309 #if KMP_USE_BGET 4310 KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL); 4311 __kmp_initialize_bget(new_thr); 4312 #endif 4313 4314 __kmp_init_random(new_thr); // Initialize random number generator 4315 4316 /* Initialize these only once when thread is grabbed for a team allocation */ 4317 KA_TRACE(20, 4318 ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n", 4319 __kmp_get_gtid(), KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); 4320 4321 int b; 4322 kmp_balign_t *balign = new_thr->th.th_bar; 4323 for (b = 0; b < bs_last_barrier; ++b) { 4324 balign[b].bb.b_go = KMP_INIT_BARRIER_STATE; 4325 balign[b].bb.team = NULL; 4326 balign[b].bb.wait_flag = KMP_BARRIER_NOT_WAITING; 4327 balign[b].bb.use_oncore_barrier = 0; 4328 } 4329 4330 new_thr->th.th_spin_here = FALSE; 4331 new_thr->th.th_next_waiting = 0; 4332 4333 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 4334 new_thr->th.th_current_place = KMP_PLACE_UNDEFINED; 4335 new_thr->th.th_new_place = KMP_PLACE_UNDEFINED; 4336 new_thr->th.th_first_place = KMP_PLACE_UNDEFINED; 4337 new_thr->th.th_last_place = KMP_PLACE_UNDEFINED; 4338 #endif 4339 4340 TCW_4(new_thr->th.th_in_pool, FALSE); 4341 new_thr->th.th_active_in_pool = FALSE; 4342 TCW_4(new_thr->th.th_active, TRUE); 4343 4344 /* adjust the global counters */ 4345 __kmp_all_nth++; 4346 __kmp_nth++; 4347 4348 root->r.r_cg_nthreads++; 4349 4350 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low 4351 // numbers of procs, and method #2 (keyed API call) for higher numbers. 4352 if (__kmp_adjust_gtid_mode) { 4353 if (__kmp_all_nth >= __kmp_tls_gtid_min) { 4354 if (TCR_4(__kmp_gtid_mode) != 2) { 4355 TCW_4(__kmp_gtid_mode, 2); 4356 } 4357 } else { 4358 if (TCR_4(__kmp_gtid_mode) != 1) { 4359 TCW_4(__kmp_gtid_mode, 1); 4360 } 4361 } 4362 } 4363 4364 #ifdef KMP_ADJUST_BLOCKTIME 4365 /* Adjust blocktime back to zero if necessary */ 4366 /* Middle initialization might not have occurred yet */ 4367 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 4368 if (__kmp_nth > __kmp_avail_proc) { 4369 __kmp_zero_bt = TRUE; 4370 } 4371 } 4372 #endif /* KMP_ADJUST_BLOCKTIME */ 4373 4374 /* actually fork it and create the new worker thread */ 4375 KF_TRACE( 4376 10, ("__kmp_allocate_thread: before __kmp_create_worker: %p\n", new_thr)); 4377 __kmp_create_worker(new_gtid, new_thr, __kmp_stksize); 4378 KF_TRACE(10, 4379 ("__kmp_allocate_thread: after __kmp_create_worker: %p\n", new_thr)); 4380 4381 KA_TRACE(20, ("__kmp_allocate_thread: T#%d forked T#%d\n", __kmp_get_gtid(), 4382 new_gtid)); 4383 KMP_MB(); 4384 return new_thr; 4385 } 4386 4387 /* Reinitialize team for reuse. 4388 The hot team code calls this case at every fork barrier, so EPCC barrier 4389 test are extremely sensitive to changes in it, esp. writes to the team 4390 struct, which cause a cache invalidation in all threads. 4391 IF YOU TOUCH THIS ROUTINE, RUN EPCC C SYNCBENCH ON A BIG-IRON MACHINE!!! */ 4392 static void __kmp_reinitialize_team(kmp_team_t *team, 4393 kmp_internal_control_t *new_icvs, 4394 ident_t *loc) { 4395 KF_TRACE(10, ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n", 4396 team->t.t_threads[0], team)); 4397 KMP_DEBUG_ASSERT(team && new_icvs); 4398 KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc); 4399 KMP_CHECK_UPDATE(team->t.t_ident, loc); 4400 4401 KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID()); 4402 // Copy ICVs to the master thread's implicit taskdata 4403 __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE); 4404 copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs); 4405 4406 KF_TRACE(10, ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n", 4407 team->t.t_threads[0], team)); 4408 } 4409 4410 /* Initialize the team data structure. 4411 This assumes the t_threads and t_max_nproc are already set. 4412 Also, we don't touch the arguments */ 4413 static void __kmp_initialize_team(kmp_team_t *team, int new_nproc, 4414 kmp_internal_control_t *new_icvs, 4415 ident_t *loc) { 4416 KF_TRACE(10, ("__kmp_initialize_team: enter: team=%p\n", team)); 4417 4418 /* verify */ 4419 KMP_DEBUG_ASSERT(team); 4420 KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc); 4421 KMP_DEBUG_ASSERT(team->t.t_threads); 4422 KMP_MB(); 4423 4424 team->t.t_master_tid = 0; /* not needed */ 4425 /* team->t.t_master_bar; not needed */ 4426 team->t.t_serialized = new_nproc > 1 ? 0 : 1; 4427 team->t.t_nproc = new_nproc; 4428 4429 /* team->t.t_parent = NULL; TODO not needed & would mess up hot team */ 4430 team->t.t_next_pool = NULL; 4431 /* memset( team->t.t_threads, 0, sizeof(kmp_info_t*)*new_nproc ); would mess 4432 * up hot team */ 4433 4434 TCW_SYNC_PTR(team->t.t_pkfn, NULL); /* not needed */ 4435 team->t.t_invoke = NULL; /* not needed */ 4436 4437 // TODO???: team->t.t_max_active_levels = new_max_active_levels; 4438 team->t.t_sched = new_icvs->sched; 4439 4440 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 4441 team->t.t_fp_control_saved = FALSE; /* not needed */ 4442 team->t.t_x87_fpu_control_word = 0; /* not needed */ 4443 team->t.t_mxcsr = 0; /* not needed */ 4444 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 4445 4446 team->t.t_construct = 0; 4447 __kmp_init_lock(&team->t.t_single_lock); 4448 4449 team->t.t_ordered.dt.t_value = 0; 4450 team->t.t_master_active = FALSE; 4451 4452 memset(&team->t.t_taskq, '\0', sizeof(kmp_taskq_t)); 4453 4454 #ifdef KMP_DEBUG 4455 team->t.t_copypriv_data = NULL; /* not necessary, but nice for debugging */ 4456 #endif 4457 team->t.t_copyin_counter = 0; /* for barrier-free copyin implementation */ 4458 4459 team->t.t_control_stack_top = NULL; 4460 4461 __kmp_reinitialize_team(team, new_icvs, loc); 4462 4463 KMP_MB(); 4464 KF_TRACE(10, ("__kmp_initialize_team: exit: team=%p\n", team)); 4465 } 4466 4467 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 4468 /* Sets full mask for thread and returns old mask, no changes to structures. */ 4469 static void 4470 __kmp_set_thread_affinity_mask_full_tmp(kmp_affin_mask_t *old_mask) { 4471 if (KMP_AFFINITY_CAPABLE()) { 4472 int status; 4473 if (old_mask != NULL) { 4474 status = __kmp_get_system_affinity(old_mask, TRUE); 4475 int error = errno; 4476 if (status != 0) { 4477 __kmp_fatal(KMP_MSG(ChangeThreadAffMaskError), KMP_ERR(error), 4478 __kmp_msg_null); 4479 } 4480 } 4481 __kmp_set_system_affinity(__kmp_affin_fullMask, TRUE); 4482 } 4483 } 4484 #endif 4485 4486 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 4487 4488 // __kmp_partition_places() is the heart of the OpenMP 4.0 affinity mechanism. 4489 // It calculats the worker + master thread's partition based upon the parent 4490 // thread's partition, and binds each worker to a thread in their partition. 4491 // The master thread's partition should already include its current binding. 4492 static void __kmp_partition_places(kmp_team_t *team, int update_master_only) { 4493 // Copy the master thread's place partion to the team struct 4494 kmp_info_t *master_th = team->t.t_threads[0]; 4495 KMP_DEBUG_ASSERT(master_th != NULL); 4496 kmp_proc_bind_t proc_bind = team->t.t_proc_bind; 4497 int first_place = master_th->th.th_first_place; 4498 int last_place = master_th->th.th_last_place; 4499 int masters_place = master_th->th.th_current_place; 4500 team->t.t_first_place = first_place; 4501 team->t.t_last_place = last_place; 4502 4503 KA_TRACE(20, ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) " 4504 "bound to place %d partition = [%d,%d]\n", 4505 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]), 4506 team->t.t_id, masters_place, first_place, last_place)); 4507 4508 switch (proc_bind) { 4509 4510 case proc_bind_default: 4511 // serial teams might have the proc_bind policy set to proc_bind_default. It 4512 // doesn't matter, as we don't rebind master thread for any proc_bind policy 4513 KMP_DEBUG_ASSERT(team->t.t_nproc == 1); 4514 break; 4515 4516 case proc_bind_master: { 4517 int f; 4518 int n_th = team->t.t_nproc; 4519 for (f = 1; f < n_th; f++) { 4520 kmp_info_t *th = team->t.t_threads[f]; 4521 KMP_DEBUG_ASSERT(th != NULL); 4522 th->th.th_first_place = first_place; 4523 th->th.th_last_place = last_place; 4524 th->th.th_new_place = masters_place; 4525 4526 KA_TRACE(100, ("__kmp_partition_places: master: T#%d(%d:%d) place %d " 4527 "partition = [%d,%d]\n", 4528 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, 4529 f, masters_place, first_place, last_place)); 4530 } 4531 } break; 4532 4533 case proc_bind_close: { 4534 int f; 4535 int n_th = team->t.t_nproc; 4536 int n_places; 4537 if (first_place <= last_place) { 4538 n_places = last_place - first_place + 1; 4539 } else { 4540 n_places = __kmp_affinity_num_masks - first_place + last_place + 1; 4541 } 4542 if (n_th <= n_places) { 4543 int place = masters_place; 4544 for (f = 1; f < n_th; f++) { 4545 kmp_info_t *th = team->t.t_threads[f]; 4546 KMP_DEBUG_ASSERT(th != NULL); 4547 4548 if (place == last_place) { 4549 place = first_place; 4550 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4551 place = 0; 4552 } else { 4553 place++; 4554 } 4555 th->th.th_first_place = first_place; 4556 th->th.th_last_place = last_place; 4557 th->th.th_new_place = place; 4558 4559 KA_TRACE(100, ("__kmp_partition_places: close: T#%d(%d:%d) place %d " 4560 "partition = [%d,%d]\n", 4561 __kmp_gtid_from_thread(team->t.t_threads[f]), 4562 team->t.t_id, f, place, first_place, last_place)); 4563 } 4564 } else { 4565 int S, rem, gap, s_count; 4566 S = n_th / n_places; 4567 s_count = 0; 4568 rem = n_th - (S * n_places); 4569 gap = rem > 0 ? n_places / rem : n_places; 4570 int place = masters_place; 4571 int gap_ct = gap; 4572 for (f = 0; f < n_th; f++) { 4573 kmp_info_t *th = team->t.t_threads[f]; 4574 KMP_DEBUG_ASSERT(th != NULL); 4575 4576 th->th.th_first_place = first_place; 4577 th->th.th_last_place = last_place; 4578 th->th.th_new_place = place; 4579 s_count++; 4580 4581 if ((s_count == S) && rem && (gap_ct == gap)) { 4582 // do nothing, add an extra thread to place on next iteration 4583 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) { 4584 // we added an extra thread to this place; move to next place 4585 if (place == last_place) { 4586 place = first_place; 4587 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4588 place = 0; 4589 } else { 4590 place++; 4591 } 4592 s_count = 0; 4593 gap_ct = 1; 4594 rem--; 4595 } else if (s_count == S) { // place full; don't add extra 4596 if (place == last_place) { 4597 place = first_place; 4598 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4599 place = 0; 4600 } else { 4601 place++; 4602 } 4603 gap_ct++; 4604 s_count = 0; 4605 } 4606 4607 KA_TRACE(100, 4608 ("__kmp_partition_places: close: T#%d(%d:%d) place %d " 4609 "partition = [%d,%d]\n", 4610 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f, 4611 th->th.th_new_place, first_place, last_place)); 4612 } 4613 KMP_DEBUG_ASSERT(place == masters_place); 4614 } 4615 } break; 4616 4617 case proc_bind_spread: { 4618 int f; 4619 int n_th = team->t.t_nproc; 4620 int n_places; 4621 int thidx; 4622 if (first_place <= last_place) { 4623 n_places = last_place - first_place + 1; 4624 } else { 4625 n_places = __kmp_affinity_num_masks - first_place + last_place + 1; 4626 } 4627 if (n_th <= n_places) { 4628 int place = -1; 4629 4630 if (n_places != static_cast<int>(__kmp_affinity_num_masks)) { 4631 int S = n_places / n_th; 4632 int s_count, rem, gap, gap_ct; 4633 4634 place = masters_place; 4635 rem = n_places - n_th * S; 4636 gap = rem ? n_th / rem : 1; 4637 gap_ct = gap; 4638 thidx = n_th; 4639 if (update_master_only == 1) 4640 thidx = 1; 4641 for (f = 0; f < thidx; f++) { 4642 kmp_info_t *th = team->t.t_threads[f]; 4643 KMP_DEBUG_ASSERT(th != NULL); 4644 4645 th->th.th_first_place = place; 4646 th->th.th_new_place = place; 4647 s_count = 1; 4648 while (s_count < S) { 4649 if (place == last_place) { 4650 place = first_place; 4651 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4652 place = 0; 4653 } else { 4654 place++; 4655 } 4656 s_count++; 4657 } 4658 if (rem && (gap_ct == gap)) { 4659 if (place == last_place) { 4660 place = first_place; 4661 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4662 place = 0; 4663 } else { 4664 place++; 4665 } 4666 rem--; 4667 gap_ct = 0; 4668 } 4669 th->th.th_last_place = place; 4670 gap_ct++; 4671 4672 if (place == last_place) { 4673 place = first_place; 4674 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4675 place = 0; 4676 } else { 4677 place++; 4678 } 4679 4680 KA_TRACE(100, ("__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4681 "partition = [%d,%d], __kmp_affinity_num_masks: %u\n", 4682 __kmp_gtid_from_thread(team->t.t_threads[f]), 4683 team->t.t_id, f, th->th.th_new_place, 4684 th->th.th_first_place, th->th.th_last_place, 4685 __kmp_affinity_num_masks)); 4686 } 4687 } else { 4688 /* Having uniform space of available computation places I can create 4689 T partitions of round(P/T) size and put threads into the first 4690 place of each partition. */ 4691 double current = static_cast<double>(masters_place); 4692 double spacing = 4693 (static_cast<double>(n_places + 1) / static_cast<double>(n_th)); 4694 int first, last; 4695 kmp_info_t *th; 4696 4697 thidx = n_th + 1; 4698 if (update_master_only == 1) 4699 thidx = 1; 4700 for (f = 0; f < thidx; f++) { 4701 first = static_cast<int>(current); 4702 last = static_cast<int>(current + spacing) - 1; 4703 KMP_DEBUG_ASSERT(last >= first); 4704 if (first >= n_places) { 4705 if (masters_place) { 4706 first -= n_places; 4707 last -= n_places; 4708 if (first == (masters_place + 1)) { 4709 KMP_DEBUG_ASSERT(f == n_th); 4710 first--; 4711 } 4712 if (last == masters_place) { 4713 KMP_DEBUG_ASSERT(f == (n_th - 1)); 4714 last--; 4715 } 4716 } else { 4717 KMP_DEBUG_ASSERT(f == n_th); 4718 first = 0; 4719 last = 0; 4720 } 4721 } 4722 if (last >= n_places) { 4723 last = (n_places - 1); 4724 } 4725 place = first; 4726 current += spacing; 4727 if (f < n_th) { 4728 KMP_DEBUG_ASSERT(0 <= first); 4729 KMP_DEBUG_ASSERT(n_places > first); 4730 KMP_DEBUG_ASSERT(0 <= last); 4731 KMP_DEBUG_ASSERT(n_places > last); 4732 KMP_DEBUG_ASSERT(last_place >= first_place); 4733 th = team->t.t_threads[f]; 4734 KMP_DEBUG_ASSERT(th); 4735 th->th.th_first_place = first; 4736 th->th.th_new_place = place; 4737 th->th.th_last_place = last; 4738 4739 KA_TRACE(100, ("__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4740 "partition = [%d,%d], spacing = %.4f\n", 4741 __kmp_gtid_from_thread(team->t.t_threads[f]), 4742 team->t.t_id, f, th->th.th_new_place, 4743 th->th.th_first_place, th->th.th_last_place, 4744 spacing)); 4745 } 4746 } 4747 } 4748 KMP_DEBUG_ASSERT(update_master_only || place == masters_place); 4749 } else { 4750 int S, rem, gap, s_count; 4751 S = n_th / n_places; 4752 s_count = 0; 4753 rem = n_th - (S * n_places); 4754 gap = rem > 0 ? n_places / rem : n_places; 4755 int place = masters_place; 4756 int gap_ct = gap; 4757 thidx = n_th; 4758 if (update_master_only == 1) 4759 thidx = 1; 4760 for (f = 0; f < thidx; f++) { 4761 kmp_info_t *th = team->t.t_threads[f]; 4762 KMP_DEBUG_ASSERT(th != NULL); 4763 4764 th->th.th_first_place = place; 4765 th->th.th_last_place = place; 4766 th->th.th_new_place = place; 4767 s_count++; 4768 4769 if ((s_count == S) && rem && (gap_ct == gap)) { 4770 // do nothing, add an extra thread to place on next iteration 4771 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) { 4772 // we added an extra thread to this place; move on to next place 4773 if (place == last_place) { 4774 place = first_place; 4775 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4776 place = 0; 4777 } else { 4778 place++; 4779 } 4780 s_count = 0; 4781 gap_ct = 1; 4782 rem--; 4783 } else if (s_count == S) { // place is full; don't add extra thread 4784 if (place == last_place) { 4785 place = first_place; 4786 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4787 place = 0; 4788 } else { 4789 place++; 4790 } 4791 gap_ct++; 4792 s_count = 0; 4793 } 4794 4795 KA_TRACE(100, ("__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4796 "partition = [%d,%d]\n", 4797 __kmp_gtid_from_thread(team->t.t_threads[f]), 4798 team->t.t_id, f, th->th.th_new_place, 4799 th->th.th_first_place, th->th.th_last_place)); 4800 } 4801 KMP_DEBUG_ASSERT(update_master_only || place == masters_place); 4802 } 4803 } break; 4804 4805 default: 4806 break; 4807 } 4808 4809 KA_TRACE(20, ("__kmp_partition_places: exit T#%d\n", team->t.t_id)); 4810 } 4811 4812 #endif /* OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED */ 4813 4814 /* allocate a new team data structure to use. take one off of the free pool if 4815 available */ 4816 kmp_team_t * 4817 __kmp_allocate_team(kmp_root_t *root, int new_nproc, int max_nproc, 4818 #if OMPT_SUPPORT 4819 ompt_parallel_id_t ompt_parallel_id, 4820 #endif 4821 #if OMP_40_ENABLED 4822 kmp_proc_bind_t new_proc_bind, 4823 #endif 4824 kmp_internal_control_t *new_icvs, 4825 int argc USE_NESTED_HOT_ARG(kmp_info_t *master)) { 4826 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_allocate_team); 4827 int f; 4828 kmp_team_t *team; 4829 int use_hot_team = !root->r.r_active; 4830 int level = 0; 4831 4832 KA_TRACE(20, ("__kmp_allocate_team: called\n")); 4833 KMP_DEBUG_ASSERT(new_nproc >= 1 && argc >= 0); 4834 KMP_DEBUG_ASSERT(max_nproc >= new_nproc); 4835 KMP_MB(); 4836 4837 #if KMP_NESTED_HOT_TEAMS 4838 kmp_hot_team_ptr_t *hot_teams; 4839 if (master) { 4840 team = master->th.th_team; 4841 level = team->t.t_active_level; 4842 if (master->th.th_teams_microtask) { // in teams construct? 4843 if (master->th.th_teams_size.nteams > 1 && 4844 ( // #teams > 1 4845 team->t.t_pkfn == 4846 (microtask_t)__kmp_teams_master || // inner fork of the teams 4847 master->th.th_teams_level < 4848 team->t.t_level)) { // or nested parallel inside the teams 4849 ++level; // not increment if #teams==1, or for outer fork of the teams; 4850 // increment otherwise 4851 } 4852 } 4853 hot_teams = master->th.th_hot_teams; 4854 if (level < __kmp_hot_teams_max_level && hot_teams && 4855 hot_teams[level] 4856 .hot_team) { // hot team has already been allocated for given level 4857 use_hot_team = 1; 4858 } else { 4859 use_hot_team = 0; 4860 } 4861 } 4862 #endif 4863 // Optimization to use a "hot" team 4864 if (use_hot_team && new_nproc > 1) { 4865 KMP_DEBUG_ASSERT(new_nproc == max_nproc); 4866 #if KMP_NESTED_HOT_TEAMS 4867 team = hot_teams[level].hot_team; 4868 #else 4869 team = root->r.r_hot_team; 4870 #endif 4871 #if KMP_DEBUG 4872 if (__kmp_tasking_mode != tskm_immediate_exec) { 4873 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p " 4874 "task_team[1] = %p before reinit\n", 4875 team->t.t_task_team[0], team->t.t_task_team[1])); 4876 } 4877 #endif 4878 4879 // Has the number of threads changed? 4880 /* Let's assume the most common case is that the number of threads is 4881 unchanged, and put that case first. */ 4882 if (team->t.t_nproc == new_nproc) { // Check changes in number of threads 4883 KA_TRACE(20, ("__kmp_allocate_team: reusing hot team\n")); 4884 // This case can mean that omp_set_num_threads() was called and the hot 4885 // team size was already reduced, so we check the special flag 4886 if (team->t.t_size_changed == -1) { 4887 team->t.t_size_changed = 1; 4888 } else { 4889 KMP_CHECK_UPDATE(team->t.t_size_changed, 0); 4890 } 4891 4892 // TODO???: team->t.t_max_active_levels = new_max_active_levels; 4893 kmp_r_sched_t new_sched = new_icvs->sched; 4894 if (team->t.t_sched.r_sched_type != new_sched.r_sched_type || 4895 team->t.t_sched.chunk != new_sched.chunk) 4896 team->t.t_sched = 4897 new_sched; // set master's schedule as new run-time schedule 4898 4899 __kmp_reinitialize_team(team, new_icvs, 4900 root->r.r_uber_thread->th.th_ident); 4901 4902 KF_TRACE(10, ("__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n", 0, 4903 team->t.t_threads[0], team)); 4904 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0); 4905 4906 #if OMP_40_ENABLED 4907 #if KMP_AFFINITY_SUPPORTED 4908 if ((team->t.t_size_changed == 0) && 4909 (team->t.t_proc_bind == new_proc_bind)) { 4910 if (new_proc_bind == proc_bind_spread) { 4911 __kmp_partition_places( 4912 team, 1); // add flag to update only master for spread 4913 } 4914 KA_TRACE(200, ("__kmp_allocate_team: reusing hot team #%d bindings: " 4915 "proc_bind = %d, partition = [%d,%d]\n", 4916 team->t.t_id, new_proc_bind, team->t.t_first_place, 4917 team->t.t_last_place)); 4918 } else { 4919 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); 4920 __kmp_partition_places(team); 4921 } 4922 #else 4923 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); 4924 #endif /* KMP_AFFINITY_SUPPORTED */ 4925 #endif /* OMP_40_ENABLED */ 4926 } else if (team->t.t_nproc > new_nproc) { 4927 KA_TRACE(20, 4928 ("__kmp_allocate_team: decreasing hot team thread count to %d\n", 4929 new_nproc)); 4930 4931 team->t.t_size_changed = 1; 4932 #if KMP_NESTED_HOT_TEAMS 4933 if (__kmp_hot_teams_mode == 0) { 4934 // AC: saved number of threads should correspond to team's value in this 4935 // mode, can be bigger in mode 1, when hot team has threads in reserve 4936 KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc); 4937 hot_teams[level].hot_team_nth = new_nproc; 4938 #endif // KMP_NESTED_HOT_TEAMS 4939 /* release the extra threads we don't need any more */ 4940 for (f = new_nproc; f < team->t.t_nproc; f++) { 4941 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 4942 if (__kmp_tasking_mode != tskm_immediate_exec) { 4943 // When decreasing team size, threads no longer in the team should 4944 // unref task team. 4945 team->t.t_threads[f]->th.th_task_team = NULL; 4946 } 4947 __kmp_free_thread(team->t.t_threads[f]); 4948 team->t.t_threads[f] = NULL; 4949 } 4950 #if KMP_NESTED_HOT_TEAMS 4951 } // (__kmp_hot_teams_mode == 0) 4952 else { 4953 // When keeping extra threads in team, switch threads to wait on own 4954 // b_go flag 4955 for (f = new_nproc; f < team->t.t_nproc; ++f) { 4956 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 4957 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar; 4958 for (int b = 0; b < bs_last_barrier; ++b) { 4959 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG) { 4960 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG; 4961 } 4962 KMP_CHECK_UPDATE(balign[b].bb.leaf_kids, 0); 4963 } 4964 } 4965 } 4966 #endif // KMP_NESTED_HOT_TEAMS 4967 team->t.t_nproc = new_nproc; 4968 // TODO???: team->t.t_max_active_levels = new_max_active_levels; 4969 if (team->t.t_sched.r_sched_type != new_icvs->sched.r_sched_type || 4970 team->t.t_sched.chunk != new_icvs->sched.chunk) 4971 team->t.t_sched = new_icvs->sched; 4972 __kmp_reinitialize_team(team, new_icvs, 4973 root->r.r_uber_thread->th.th_ident); 4974 4975 /* update the remaining threads */ 4976 for (f = 0; f < new_nproc; ++f) { 4977 team->t.t_threads[f]->th.th_team_nproc = new_nproc; 4978 } 4979 // restore the current task state of the master thread: should be the 4980 // implicit task 4981 KF_TRACE(10, ("__kmp_allocate_team: T#%d, this_thread=%p team=%p\n", 0, 4982 team->t.t_threads[0], team)); 4983 4984 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0); 4985 4986 #ifdef KMP_DEBUG 4987 for (f = 0; f < team->t.t_nproc; f++) { 4988 KMP_DEBUG_ASSERT(team->t.t_threads[f] && 4989 team->t.t_threads[f]->th.th_team_nproc == 4990 team->t.t_nproc); 4991 } 4992 #endif 4993 4994 #if OMP_40_ENABLED 4995 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); 4996 #if KMP_AFFINITY_SUPPORTED 4997 __kmp_partition_places(team); 4998 #endif 4999 #endif 5000 } else { // team->t.t_nproc < new_nproc 5001 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 5002 kmp_affin_mask_t *old_mask; 5003 if (KMP_AFFINITY_CAPABLE()) { 5004 KMP_CPU_ALLOC(old_mask); 5005 } 5006 #endif 5007 5008 KA_TRACE(20, 5009 ("__kmp_allocate_team: increasing hot team thread count to %d\n", 5010 new_nproc)); 5011 5012 team->t.t_size_changed = 1; 5013 5014 #if KMP_NESTED_HOT_TEAMS 5015 int avail_threads = hot_teams[level].hot_team_nth; 5016 if (new_nproc < avail_threads) 5017 avail_threads = new_nproc; 5018 kmp_info_t **other_threads = team->t.t_threads; 5019 for (f = team->t.t_nproc; f < avail_threads; ++f) { 5020 // Adjust barrier data of reserved threads (if any) of the team 5021 // Other data will be set in __kmp_initialize_info() below. 5022 int b; 5023 kmp_balign_t *balign = other_threads[f]->th.th_bar; 5024 for (b = 0; b < bs_last_barrier; ++b) { 5025 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 5026 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 5027 #if USE_DEBUGGER 5028 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 5029 #endif 5030 } 5031 } 5032 if (hot_teams[level].hot_team_nth >= new_nproc) { 5033 // we have all needed threads in reserve, no need to allocate any 5034 // this only possible in mode 1, cannot have reserved threads in mode 0 5035 KMP_DEBUG_ASSERT(__kmp_hot_teams_mode == 1); 5036 team->t.t_nproc = new_nproc; // just get reserved threads involved 5037 } else { 5038 // we may have some threads in reserve, but not enough 5039 team->t.t_nproc = 5040 hot_teams[level] 5041 .hot_team_nth; // get reserved threads involved if any 5042 hot_teams[level].hot_team_nth = new_nproc; // adjust hot team max size 5043 #endif // KMP_NESTED_HOT_TEAMS 5044 if (team->t.t_max_nproc < new_nproc) { 5045 /* reallocate larger arrays */ 5046 __kmp_reallocate_team_arrays(team, new_nproc); 5047 __kmp_reinitialize_team(team, new_icvs, NULL); 5048 } 5049 5050 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 5051 /* Temporarily set full mask for master thread before creation of 5052 workers. The reason is that workers inherit the affinity from master, 5053 so if a lot of workers are created on the single core quickly, they 5054 don't get a chance to set their own affinity for a long time. */ 5055 __kmp_set_thread_affinity_mask_full_tmp(old_mask); 5056 #endif 5057 5058 /* allocate new threads for the hot team */ 5059 for (f = team->t.t_nproc; f < new_nproc; f++) { 5060 kmp_info_t *new_worker = __kmp_allocate_thread(root, team, f); 5061 KMP_DEBUG_ASSERT(new_worker); 5062 team->t.t_threads[f] = new_worker; 5063 5064 KA_TRACE(20, 5065 ("__kmp_allocate_team: team %d init T#%d arrived: " 5066 "join=%llu, plain=%llu\n", 5067 team->t.t_id, __kmp_gtid_from_tid(f, team), team->t.t_id, f, 5068 team->t.t_bar[bs_forkjoin_barrier].b_arrived, 5069 team->t.t_bar[bs_plain_barrier].b_arrived)); 5070 5071 { // Initialize barrier data for new threads. 5072 int b; 5073 kmp_balign_t *balign = new_worker->th.th_bar; 5074 for (b = 0; b < bs_last_barrier; ++b) { 5075 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 5076 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != 5077 KMP_BARRIER_PARENT_FLAG); 5078 #if USE_DEBUGGER 5079 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 5080 #endif 5081 } 5082 } 5083 } 5084 5085 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 5086 if (KMP_AFFINITY_CAPABLE()) { 5087 /* Restore initial master thread's affinity mask */ 5088 __kmp_set_system_affinity(old_mask, TRUE); 5089 KMP_CPU_FREE(old_mask); 5090 } 5091 #endif 5092 #if KMP_NESTED_HOT_TEAMS 5093 } // end of check of t_nproc vs. new_nproc vs. hot_team_nth 5094 #endif // KMP_NESTED_HOT_TEAMS 5095 /* make sure everyone is syncronized */ 5096 int old_nproc = team->t.t_nproc; // save old value and use to update only 5097 // new threads below 5098 __kmp_initialize_team(team, new_nproc, new_icvs, 5099 root->r.r_uber_thread->th.th_ident); 5100 5101 /* reinitialize the threads */ 5102 KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc); 5103 for (f = 0; f < team->t.t_nproc; ++f) 5104 __kmp_initialize_info(team->t.t_threads[f], team, f, 5105 __kmp_gtid_from_tid(f, team)); 5106 if (level) { // set th_task_state for new threads in nested hot team 5107 // __kmp_initialize_info() no longer zeroes th_task_state, so we should 5108 // only need to set the th_task_state for the new threads. th_task_state 5109 // for master thread will not be accurate until after this in 5110 // __kmp_fork_call(), so we look to the master's memo_stack to get the 5111 // correct value. 5112 for (f = old_nproc; f < team->t.t_nproc; ++f) 5113 team->t.t_threads[f]->th.th_task_state = 5114 team->t.t_threads[0]->th.th_task_state_memo_stack[level]; 5115 } else { // set th_task_state for new threads in non-nested hot team 5116 int old_state = 5117 team->t.t_threads[0]->th.th_task_state; // copy master's state 5118 for (f = old_nproc; f < team->t.t_nproc; ++f) 5119 team->t.t_threads[f]->th.th_task_state = old_state; 5120 } 5121 5122 #ifdef KMP_DEBUG 5123 for (f = 0; f < team->t.t_nproc; ++f) { 5124 KMP_DEBUG_ASSERT(team->t.t_threads[f] && 5125 team->t.t_threads[f]->th.th_team_nproc == 5126 team->t.t_nproc); 5127 } 5128 #endif 5129 5130 #if OMP_40_ENABLED 5131 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); 5132 #if KMP_AFFINITY_SUPPORTED 5133 __kmp_partition_places(team); 5134 #endif 5135 #endif 5136 } // Check changes in number of threads 5137 5138 #if OMP_40_ENABLED 5139 kmp_info_t *master = team->t.t_threads[0]; 5140 if (master->th.th_teams_microtask) { 5141 for (f = 1; f < new_nproc; ++f) { 5142 // propagate teams construct specific info to workers 5143 kmp_info_t *thr = team->t.t_threads[f]; 5144 thr->th.th_teams_microtask = master->th.th_teams_microtask; 5145 thr->th.th_teams_level = master->th.th_teams_level; 5146 thr->th.th_teams_size = master->th.th_teams_size; 5147 } 5148 } 5149 #endif /* OMP_40_ENABLED */ 5150 #if KMP_NESTED_HOT_TEAMS 5151 if (level) { 5152 // Sync barrier state for nested hot teams, not needed for outermost hot 5153 // team. 5154 for (f = 1; f < new_nproc; ++f) { 5155 kmp_info_t *thr = team->t.t_threads[f]; 5156 int b; 5157 kmp_balign_t *balign = thr->th.th_bar; 5158 for (b = 0; b < bs_last_barrier; ++b) { 5159 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 5160 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 5161 #if USE_DEBUGGER 5162 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 5163 #endif 5164 } 5165 } 5166 } 5167 #endif // KMP_NESTED_HOT_TEAMS 5168 5169 /* reallocate space for arguments if necessary */ 5170 __kmp_alloc_argv_entries(argc, team, TRUE); 5171 KMP_CHECK_UPDATE(team->t.t_argc, argc); 5172 // The hot team re-uses the previous task team, 5173 // if untouched during the previous release->gather phase. 5174 5175 KF_TRACE(10, (" hot_team = %p\n", team)); 5176 5177 #if KMP_DEBUG 5178 if (__kmp_tasking_mode != tskm_immediate_exec) { 5179 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p " 5180 "task_team[1] = %p after reinit\n", 5181 team->t.t_task_team[0], team->t.t_task_team[1])); 5182 } 5183 #endif 5184 5185 #if OMPT_SUPPORT 5186 __ompt_team_assign_id(team, ompt_parallel_id); 5187 #endif 5188 5189 KMP_MB(); 5190 5191 return team; 5192 } 5193 5194 /* next, let's try to take one from the team pool */ 5195 KMP_MB(); 5196 for (team = CCAST(kmp_team_t *, __kmp_team_pool); (team);) { 5197 /* TODO: consider resizing undersized teams instead of reaping them, now 5198 that we have a resizing mechanism */ 5199 if (team->t.t_max_nproc >= max_nproc) { 5200 /* take this team from the team pool */ 5201 __kmp_team_pool = team->t.t_next_pool; 5202 5203 /* setup the team for fresh use */ 5204 __kmp_initialize_team(team, new_nproc, new_icvs, NULL); 5205 5206 KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and " 5207 "task_team[1] %p to NULL\n", 5208 &team->t.t_task_team[0], &team->t.t_task_team[1])); 5209 team->t.t_task_team[0] = NULL; 5210 team->t.t_task_team[1] = NULL; 5211 5212 /* reallocate space for arguments if necessary */ 5213 __kmp_alloc_argv_entries(argc, team, TRUE); 5214 KMP_CHECK_UPDATE(team->t.t_argc, argc); 5215 5216 KA_TRACE( 5217 20, ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n", 5218 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); 5219 { // Initialize barrier data. 5220 int b; 5221 for (b = 0; b < bs_last_barrier; ++b) { 5222 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE; 5223 #if USE_DEBUGGER 5224 team->t.t_bar[b].b_master_arrived = 0; 5225 team->t.t_bar[b].b_team_arrived = 0; 5226 #endif 5227 } 5228 } 5229 5230 #if OMP_40_ENABLED 5231 team->t.t_proc_bind = new_proc_bind; 5232 #endif 5233 5234 KA_TRACE(20, ("__kmp_allocate_team: using team from pool %d.\n", 5235 team->t.t_id)); 5236 5237 #if OMPT_SUPPORT 5238 __ompt_team_assign_id(team, ompt_parallel_id); 5239 #endif 5240 5241 KMP_MB(); 5242 5243 return team; 5244 } 5245 5246 /* reap team if it is too small, then loop back and check the next one */ 5247 // not sure if this is wise, but, will be redone during the hot-teams rewrite. 5248 /* TODO: Use technique to find the right size hot-team, don't reap them */ 5249 team = __kmp_reap_team(team); 5250 __kmp_team_pool = team; 5251 } 5252 5253 /* nothing available in the pool, no matter, make a new team! */ 5254 KMP_MB(); 5255 team = (kmp_team_t *)__kmp_allocate(sizeof(kmp_team_t)); 5256 5257 /* and set it up */ 5258 team->t.t_max_nproc = max_nproc; 5259 /* NOTE well, for some reason allocating one big buffer and dividing it up 5260 seems to really hurt performance a lot on the P4, so, let's not use this */ 5261 __kmp_allocate_team_arrays(team, max_nproc); 5262 5263 KA_TRACE(20, ("__kmp_allocate_team: making a new team\n")); 5264 __kmp_initialize_team(team, new_nproc, new_icvs, NULL); 5265 5266 KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and task_team[1] " 5267 "%p to NULL\n", 5268 &team->t.t_task_team[0], &team->t.t_task_team[1])); 5269 team->t.t_task_team[0] = NULL; // to be removed, as __kmp_allocate zeroes 5270 // memory, no need to duplicate 5271 team->t.t_task_team[1] = NULL; // to be removed, as __kmp_allocate zeroes 5272 // memory, no need to duplicate 5273 5274 if (__kmp_storage_map) { 5275 __kmp_print_team_storage_map("team", team, team->t.t_id, new_nproc); 5276 } 5277 5278 /* allocate space for arguments */ 5279 __kmp_alloc_argv_entries(argc, team, FALSE); 5280 team->t.t_argc = argc; 5281 5282 KA_TRACE(20, 5283 ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n", 5284 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); 5285 { // Initialize barrier data. 5286 int b; 5287 for (b = 0; b < bs_last_barrier; ++b) { 5288 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE; 5289 #if USE_DEBUGGER 5290 team->t.t_bar[b].b_master_arrived = 0; 5291 team->t.t_bar[b].b_team_arrived = 0; 5292 #endif 5293 } 5294 } 5295 5296 #if OMP_40_ENABLED 5297 team->t.t_proc_bind = new_proc_bind; 5298 #endif 5299 5300 #if OMPT_SUPPORT 5301 __ompt_team_assign_id(team, ompt_parallel_id); 5302 team->t.ompt_serialized_team_info = NULL; 5303 #endif 5304 5305 KMP_MB(); 5306 5307 KA_TRACE(20, ("__kmp_allocate_team: done creating a new team %d.\n", 5308 team->t.t_id)); 5309 5310 return team; 5311 } 5312 5313 /* TODO implement hot-teams at all levels */ 5314 /* TODO implement lazy thread release on demand (disband request) */ 5315 5316 /* free the team. return it to the team pool. release all the threads 5317 * associated with it */ 5318 void __kmp_free_team(kmp_root_t *root, 5319 kmp_team_t *team USE_NESTED_HOT_ARG(kmp_info_t *master)) { 5320 int f; 5321 KA_TRACE(20, ("__kmp_free_team: T#%d freeing team %d\n", __kmp_get_gtid(), 5322 team->t.t_id)); 5323 5324 /* verify state */ 5325 KMP_DEBUG_ASSERT(root); 5326 KMP_DEBUG_ASSERT(team); 5327 KMP_DEBUG_ASSERT(team->t.t_nproc <= team->t.t_max_nproc); 5328 KMP_DEBUG_ASSERT(team->t.t_threads); 5329 5330 int use_hot_team = team == root->r.r_hot_team; 5331 #if KMP_NESTED_HOT_TEAMS 5332 int level; 5333 kmp_hot_team_ptr_t *hot_teams; 5334 if (master) { 5335 level = team->t.t_active_level - 1; 5336 if (master->th.th_teams_microtask) { // in teams construct? 5337 if (master->th.th_teams_size.nteams > 1) { 5338 ++level; // level was not increased in teams construct for 5339 // team_of_masters 5340 } 5341 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master && 5342 master->th.th_teams_level == team->t.t_level) { 5343 ++level; // level was not increased in teams construct for 5344 // team_of_workers before the parallel 5345 } // team->t.t_level will be increased inside parallel 5346 } 5347 hot_teams = master->th.th_hot_teams; 5348 if (level < __kmp_hot_teams_max_level) { 5349 KMP_DEBUG_ASSERT(team == hot_teams[level].hot_team); 5350 use_hot_team = 1; 5351 } 5352 } 5353 #endif // KMP_NESTED_HOT_TEAMS 5354 5355 /* team is done working */ 5356 TCW_SYNC_PTR(team->t.t_pkfn, 5357 NULL); // Important for Debugging Support Library. 5358 team->t.t_copyin_counter = 0; // init counter for possible reuse 5359 // Do not reset pointer to parent team to NULL for hot teams. 5360 5361 /* if we are non-hot team, release our threads */ 5362 if (!use_hot_team) { 5363 if (__kmp_tasking_mode != tskm_immediate_exec) { 5364 // Wait for threads to reach reapable state 5365 for (f = 1; f < team->t.t_nproc; ++f) { 5366 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 5367 kmp_info_t *th = team->t.t_threads[f]; 5368 volatile kmp_uint32 *state = &th->th.th_reap_state; 5369 while (*state != KMP_SAFE_TO_REAP) { 5370 #if KMP_OS_WINDOWS 5371 // On Windows a thread can be killed at any time, check this 5372 DWORD ecode; 5373 if (!__kmp_is_thread_alive(th, &ecode)) { 5374 *state = KMP_SAFE_TO_REAP; // reset the flag for dead thread 5375 break; 5376 } 5377 #endif 5378 // first check if thread is sleeping 5379 kmp_flag_64 fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th); 5380 if (fl.is_sleeping()) 5381 fl.resume(__kmp_gtid_from_thread(th)); 5382 KMP_CPU_PAUSE(); 5383 } 5384 } 5385 5386 // Delete task teams 5387 int tt_idx; 5388 for (tt_idx = 0; tt_idx < 2; ++tt_idx) { 5389 kmp_task_team_t *task_team = team->t.t_task_team[tt_idx]; 5390 if (task_team != NULL) { 5391 for (f = 0; f < team->t.t_nproc; 5392 ++f) { // Have all threads unref task teams 5393 team->t.t_threads[f]->th.th_task_team = NULL; 5394 } 5395 KA_TRACE( 5396 20, 5397 ("__kmp_free_team: T#%d deactivating task_team %p on team %d\n", 5398 __kmp_get_gtid(), task_team, team->t.t_id)); 5399 #if KMP_NESTED_HOT_TEAMS 5400 __kmp_free_task_team(master, task_team); 5401 #endif 5402 team->t.t_task_team[tt_idx] = NULL; 5403 } 5404 } 5405 } 5406 5407 // Reset pointer to parent team only for non-hot teams. 5408 team->t.t_parent = NULL; 5409 team->t.t_level = 0; 5410 team->t.t_active_level = 0; 5411 5412 /* free the worker threads */ 5413 for (f = 1; f < team->t.t_nproc; ++f) { 5414 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 5415 __kmp_free_thread(team->t.t_threads[f]); 5416 team->t.t_threads[f] = NULL; 5417 } 5418 5419 /* put the team back in the team pool */ 5420 /* TODO limit size of team pool, call reap_team if pool too large */ 5421 team->t.t_next_pool = CCAST(kmp_team_t *, __kmp_team_pool); 5422 __kmp_team_pool = (volatile kmp_team_t *)team; 5423 } 5424 5425 KMP_MB(); 5426 } 5427 5428 /* reap the team. destroy it, reclaim all its resources and free its memory */ 5429 kmp_team_t *__kmp_reap_team(kmp_team_t *team) { 5430 kmp_team_t *next_pool = team->t.t_next_pool; 5431 5432 KMP_DEBUG_ASSERT(team); 5433 KMP_DEBUG_ASSERT(team->t.t_dispatch); 5434 KMP_DEBUG_ASSERT(team->t.t_disp_buffer); 5435 KMP_DEBUG_ASSERT(team->t.t_threads); 5436 KMP_DEBUG_ASSERT(team->t.t_argv); 5437 5438 /* TODO clean the threads that are a part of this? */ 5439 5440 /* free stuff */ 5441 __kmp_free_team_arrays(team); 5442 if (team->t.t_argv != &team->t.t_inline_argv[0]) 5443 __kmp_free((void *)team->t.t_argv); 5444 __kmp_free(team); 5445 5446 KMP_MB(); 5447 return next_pool; 5448 } 5449 5450 // Free the thread. Don't reap it, just place it on the pool of available 5451 // threads. 5452 // 5453 // Changes for Quad issue 527845: We need a predictable OMP tid <-> gtid 5454 // binding for the affinity mechanism to be useful. 5455 // 5456 // Now, we always keep the free list (__kmp_thread_pool) sorted by gtid. 5457 // However, we want to avoid a potential performance problem by always 5458 // scanning through the list to find the correct point at which to insert 5459 // the thread (potential N**2 behavior). To do this we keep track of the 5460 // last place a thread struct was inserted (__kmp_thread_pool_insert_pt). 5461 // With single-level parallelism, threads will always be added to the tail 5462 // of the list, kept track of by __kmp_thread_pool_insert_pt. With nested 5463 // parallelism, all bets are off and we may need to scan through the entire 5464 // free list. 5465 // 5466 // This change also has a potentially large performance benefit, for some 5467 // applications. Previously, as threads were freed from the hot team, they 5468 // would be placed back on the free list in inverse order. If the hot team 5469 // grew back to it's original size, then the freed thread would be placed 5470 // back on the hot team in reverse order. This could cause bad cache 5471 // locality problems on programs where the size of the hot team regularly 5472 // grew and shrunk. 5473 // 5474 // Now, for single-level parallelism, the OMP tid is alway == gtid. 5475 void __kmp_free_thread(kmp_info_t *this_th) { 5476 int gtid; 5477 kmp_info_t **scan; 5478 kmp_root_t *root = this_th->th.th_root; 5479 5480 KA_TRACE(20, ("__kmp_free_thread: T#%d putting T#%d back on free pool.\n", 5481 __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid)); 5482 5483 KMP_DEBUG_ASSERT(this_th); 5484 5485 // When moving thread to pool, switch thread to wait on own b_go flag, and 5486 // uninitialized (NULL team). 5487 int b; 5488 kmp_balign_t *balign = this_th->th.th_bar; 5489 for (b = 0; b < bs_last_barrier; ++b) { 5490 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG) 5491 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG; 5492 balign[b].bb.team = NULL; 5493 balign[b].bb.leaf_kids = 0; 5494 } 5495 this_th->th.th_task_state = 0; 5496 5497 /* put thread back on the free pool */ 5498 TCW_PTR(this_th->th.th_team, NULL); 5499 TCW_PTR(this_th->th.th_root, NULL); 5500 TCW_PTR(this_th->th.th_dispatch, NULL); /* NOT NEEDED */ 5501 5502 // If the __kmp_thread_pool_insert_pt is already past the new insert 5503 // point, then we need to re-scan the entire list. 5504 gtid = this_th->th.th_info.ds.ds_gtid; 5505 if (__kmp_thread_pool_insert_pt != NULL) { 5506 KMP_DEBUG_ASSERT(__kmp_thread_pool != NULL); 5507 if (__kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid) { 5508 __kmp_thread_pool_insert_pt = NULL; 5509 } 5510 } 5511 5512 // Scan down the list to find the place to insert the thread. 5513 // scan is the address of a link in the list, possibly the address of 5514 // __kmp_thread_pool itself. 5515 // 5516 // In the absence of nested parallism, the for loop will have 0 iterations. 5517 if (__kmp_thread_pool_insert_pt != NULL) { 5518 scan = &(__kmp_thread_pool_insert_pt->th.th_next_pool); 5519 } else { 5520 scan = CCAST(kmp_info_t **, &__kmp_thread_pool); 5521 } 5522 for (; (*scan != NULL) && ((*scan)->th.th_info.ds.ds_gtid < gtid); 5523 scan = &((*scan)->th.th_next_pool)) 5524 ; 5525 5526 // Insert the new element on the list, and set __kmp_thread_pool_insert_pt 5527 // to its address. 5528 TCW_PTR(this_th->th.th_next_pool, *scan); 5529 __kmp_thread_pool_insert_pt = *scan = this_th; 5530 KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) || 5531 (this_th->th.th_info.ds.ds_gtid < 5532 this_th->th.th_next_pool->th.th_info.ds.ds_gtid)); 5533 TCW_4(this_th->th.th_in_pool, TRUE); 5534 __kmp_thread_pool_nth++; 5535 5536 TCW_4(__kmp_nth, __kmp_nth - 1); 5537 root->r.r_cg_nthreads--; 5538 5539 #ifdef KMP_ADJUST_BLOCKTIME 5540 /* Adjust blocktime back to user setting or default if necessary */ 5541 /* Middle initialization might never have occurred */ 5542 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 5543 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0); 5544 if (__kmp_nth <= __kmp_avail_proc) { 5545 __kmp_zero_bt = FALSE; 5546 } 5547 } 5548 #endif /* KMP_ADJUST_BLOCKTIME */ 5549 5550 KMP_MB(); 5551 } 5552 5553 /* ------------------------------------------------------------------------ */ 5554 5555 void *__kmp_launch_thread(kmp_info_t *this_thr) { 5556 int gtid = this_thr->th.th_info.ds.ds_gtid; 5557 /* void *stack_data;*/ 5558 kmp_team_t *(*volatile pteam); 5559 5560 KMP_MB(); 5561 KA_TRACE(10, ("__kmp_launch_thread: T#%d start\n", gtid)); 5562 5563 if (__kmp_env_consistency_check) { 5564 this_thr->th.th_cons = __kmp_allocate_cons_stack(gtid); // ATT: Memory leak? 5565 } 5566 5567 #if OMPT_SUPPORT 5568 if (ompt_enabled) { 5569 this_thr->th.ompt_thread_info.state = ompt_state_overhead; 5570 this_thr->th.ompt_thread_info.wait_id = 0; 5571 this_thr->th.ompt_thread_info.idle_frame = __builtin_frame_address(0); 5572 if (ompt_callbacks.ompt_callback(ompt_event_thread_begin)) { 5573 __ompt_thread_begin(ompt_thread_worker, gtid); 5574 } 5575 } 5576 #endif 5577 5578 /* This is the place where threads wait for work */ 5579 while (!TCR_4(__kmp_global.g.g_done)) { 5580 KMP_DEBUG_ASSERT(this_thr == __kmp_threads[gtid]); 5581 KMP_MB(); 5582 5583 /* wait for work to do */ 5584 KA_TRACE(20, ("__kmp_launch_thread: T#%d waiting for work\n", gtid)); 5585 5586 #if OMPT_SUPPORT 5587 if (ompt_enabled) { 5588 this_thr->th.ompt_thread_info.state = ompt_state_idle; 5589 } 5590 #endif 5591 5592 /* No tid yet since not part of a team */ 5593 __kmp_fork_barrier(gtid, KMP_GTID_DNE); 5594 5595 #if OMPT_SUPPORT 5596 if (ompt_enabled) { 5597 this_thr->th.ompt_thread_info.state = ompt_state_overhead; 5598 } 5599 #endif 5600 5601 pteam = (kmp_team_t * (*))(&this_thr->th.th_team); 5602 5603 /* have we been allocated? */ 5604 if (TCR_SYNC_PTR(*pteam) && !TCR_4(__kmp_global.g.g_done)) { 5605 #if OMPT_SUPPORT 5606 ompt_task_info_t *task_info; 5607 ompt_parallel_id_t my_parallel_id; 5608 if (ompt_enabled) { 5609 task_info = __ompt_get_taskinfo(0); 5610 my_parallel_id = (*pteam)->t.ompt_team_info.parallel_id; 5611 } 5612 #endif 5613 /* we were just woken up, so run our new task */ 5614 if (TCR_SYNC_PTR((*pteam)->t.t_pkfn) != NULL) { 5615 int rc; 5616 KA_TRACE(20, 5617 ("__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n", 5618 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), 5619 (*pteam)->t.t_pkfn)); 5620 5621 updateHWFPControl(*pteam); 5622 5623 #if OMPT_SUPPORT 5624 if (ompt_enabled) { 5625 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel; 5626 // Initialize OMPT task id for implicit task. 5627 int tid = __kmp_tid_from_gtid(gtid); 5628 task_info->task_id = __ompt_task_id_new(tid); 5629 } 5630 #endif 5631 5632 { 5633 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 5634 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 5635 rc = (*pteam)->t.t_invoke(gtid); 5636 } 5637 KMP_ASSERT(rc); 5638 5639 #if OMPT_SUPPORT 5640 if (ompt_enabled) { 5641 /* no frame set while outside task */ 5642 task_info->frame.exit_runtime_frame = NULL; 5643 5644 this_thr->th.ompt_thread_info.state = ompt_state_overhead; 5645 } 5646 #endif 5647 KMP_MB(); 5648 KA_TRACE(20, ("__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n", 5649 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), 5650 (*pteam)->t.t_pkfn)); 5651 } 5652 /* join barrier after parallel region */ 5653 __kmp_join_barrier(gtid); 5654 #if OMPT_SUPPORT && OMPT_TRACE 5655 if (ompt_enabled) { 5656 if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) { 5657 // don't access *pteam here: it may have already been freed 5658 // by the master thread behind the barrier (possible race) 5659 ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)( 5660 my_parallel_id, task_info->task_id); 5661 } 5662 task_info->frame.exit_runtime_frame = NULL; 5663 task_info->task_id = 0; 5664 } 5665 #endif 5666 } 5667 } 5668 TCR_SYNC_PTR((intptr_t)__kmp_global.g.g_done); 5669 5670 #if OMPT_SUPPORT 5671 if (ompt_enabled && ompt_callbacks.ompt_callback(ompt_event_thread_end)) { 5672 __ompt_thread_end(ompt_thread_worker, gtid); 5673 } 5674 #endif 5675 5676 this_thr->th.th_task_team = NULL; 5677 /* run the destructors for the threadprivate data for this thread */ 5678 __kmp_common_destroy_gtid(gtid); 5679 5680 KA_TRACE(10, ("__kmp_launch_thread: T#%d done\n", gtid)); 5681 KMP_MB(); 5682 return this_thr; 5683 } 5684 5685 /* ------------------------------------------------------------------------ */ 5686 5687 void __kmp_internal_end_dest(void *specific_gtid) { 5688 #if KMP_COMPILER_ICC 5689 #pragma warning(push) 5690 #pragma warning(disable : 810) // conversion from "void *" to "int" may lose 5691 // significant bits 5692 #endif 5693 // Make sure no significant bits are lost 5694 int gtid = (kmp_intptr_t)specific_gtid - 1; 5695 #if KMP_COMPILER_ICC 5696 #pragma warning(pop) 5697 #endif 5698 5699 KA_TRACE(30, ("__kmp_internal_end_dest: T#%d\n", gtid)); 5700 /* NOTE: the gtid is stored as gitd+1 in the thread-local-storage 5701 * this is because 0 is reserved for the nothing-stored case */ 5702 5703 /* josh: One reason for setting the gtid specific data even when it is being 5704 destroyed by pthread is to allow gtid lookup through thread specific data 5705 (__kmp_gtid_get_specific). Some of the code, especially stat code, 5706 that gets executed in the call to __kmp_internal_end_thread, actually 5707 gets the gtid through the thread specific data. Setting it here seems 5708 rather inelegant and perhaps wrong, but allows __kmp_internal_end_thread 5709 to run smoothly. 5710 todo: get rid of this after we remove the dependence on 5711 __kmp_gtid_get_specific */ 5712 if (gtid >= 0 && KMP_UBER_GTID(gtid)) 5713 __kmp_gtid_set_specific(gtid); 5714 #ifdef KMP_TDATA_GTID 5715 __kmp_gtid = gtid; 5716 #endif 5717 __kmp_internal_end_thread(gtid); 5718 } 5719 5720 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB 5721 5722 // 2009-09-08 (lev): It looks the destructor does not work. In simple test cases 5723 // destructors work perfectly, but in real libomp.so I have no evidence it is 5724 // ever called. However, -fini linker option in makefile.mk works fine. 5725 5726 __attribute__((destructor)) void __kmp_internal_end_dtor(void) { 5727 __kmp_internal_end_atexit(); 5728 } 5729 5730 void __kmp_internal_end_fini(void) { __kmp_internal_end_atexit(); } 5731 5732 #endif 5733 5734 /* [Windows] josh: when the atexit handler is called, there may still be more 5735 than one thread alive */ 5736 void __kmp_internal_end_atexit(void) { 5737 KA_TRACE(30, ("__kmp_internal_end_atexit\n")); 5738 /* [Windows] 5739 josh: ideally, we want to completely shutdown the library in this atexit 5740 handler, but stat code that depends on thread specific data for gtid fails 5741 because that data becomes unavailable at some point during the shutdown, so 5742 we call __kmp_internal_end_thread instead. We should eventually remove the 5743 dependency on __kmp_get_specific_gtid in the stat code and use 5744 __kmp_internal_end_library to cleanly shutdown the library. 5745 5746 // TODO: Can some of this comment about GVS be removed? 5747 I suspect that the offending stat code is executed when the calling thread 5748 tries to clean up a dead root thread's data structures, resulting in GVS 5749 code trying to close the GVS structures for that thread, but since the stat 5750 code uses __kmp_get_specific_gtid to get the gtid with the assumption that 5751 the calling thread is cleaning up itself instead of another thread, it get 5752 confused. This happens because allowing a thread to unregister and cleanup 5753 another thread is a recent modification for addressing an issue. 5754 Based on the current design (20050722), a thread may end up 5755 trying to unregister another thread only if thread death does not trigger 5756 the calling of __kmp_internal_end_thread. For Linux* OS, there is the 5757 thread specific data destructor function to detect thread death. For 5758 Windows dynamic, there is DllMain(THREAD_DETACH). For Windows static, there 5759 is nothing. Thus, the workaround is applicable only for Windows static 5760 stat library. */ 5761 __kmp_internal_end_library(-1); 5762 #if KMP_OS_WINDOWS 5763 __kmp_close_console(); 5764 #endif 5765 } 5766 5767 static void __kmp_reap_thread(kmp_info_t *thread, int is_root) { 5768 // It is assumed __kmp_forkjoin_lock is acquired. 5769 5770 int gtid; 5771 5772 KMP_DEBUG_ASSERT(thread != NULL); 5773 5774 gtid = thread->th.th_info.ds.ds_gtid; 5775 5776 if (!is_root) { 5777 5778 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) { 5779 /* Assume the threads are at the fork barrier here */ 5780 KA_TRACE( 5781 20, ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n", 5782 gtid)); 5783 /* Need release fence here to prevent seg faults for tree forkjoin barrier 5784 * (GEH) */ 5785 ANNOTATE_HAPPENS_BEFORE(thread); 5786 kmp_flag_64 flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, thread); 5787 __kmp_release_64(&flag); 5788 }; // if 5789 5790 // Terminate OS thread. 5791 __kmp_reap_worker(thread); 5792 5793 // The thread was killed asynchronously. If it was actively 5794 // spinning in the thread pool, decrement the global count. 5795 // 5796 // There is a small timing hole here - if the worker thread was just waking 5797 // up after sleeping in the pool, had reset it's th_active_in_pool flag but 5798 // not decremented the global counter __kmp_thread_pool_active_nth yet, then 5799 // the global counter might not get updated. 5800 // 5801 // Currently, this can only happen as the library is unloaded, 5802 // so there are no harmful side effects. 5803 if (thread->th.th_active_in_pool) { 5804 thread->th.th_active_in_pool = FALSE; 5805 KMP_TEST_THEN_DEC32(&__kmp_thread_pool_active_nth); 5806 KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0); 5807 } 5808 5809 // Decrement # of [worker] threads in the pool. 5810 KMP_DEBUG_ASSERT(__kmp_thread_pool_nth > 0); 5811 --__kmp_thread_pool_nth; 5812 }; // if 5813 5814 __kmp_free_implicit_task(thread); 5815 5816 // Free the fast memory for tasking 5817 #if USE_FAST_MEMORY 5818 __kmp_free_fast_memory(thread); 5819 #endif /* USE_FAST_MEMORY */ 5820 5821 __kmp_suspend_uninitialize_thread(thread); 5822 5823 KMP_DEBUG_ASSERT(__kmp_threads[gtid] == thread); 5824 TCW_SYNC_PTR(__kmp_threads[gtid], NULL); 5825 5826 --__kmp_all_nth; 5827 // __kmp_nth was decremented when thread is added to the pool. 5828 5829 #ifdef KMP_ADJUST_BLOCKTIME 5830 /* Adjust blocktime back to user setting or default if necessary */ 5831 /* Middle initialization might never have occurred */ 5832 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 5833 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0); 5834 if (__kmp_nth <= __kmp_avail_proc) { 5835 __kmp_zero_bt = FALSE; 5836 } 5837 } 5838 #endif /* KMP_ADJUST_BLOCKTIME */ 5839 5840 /* free the memory being used */ 5841 if (__kmp_env_consistency_check) { 5842 if (thread->th.th_cons) { 5843 __kmp_free_cons_stack(thread->th.th_cons); 5844 thread->th.th_cons = NULL; 5845 }; // if 5846 } 5847 5848 if (thread->th.th_pri_common != NULL) { 5849 __kmp_free(thread->th.th_pri_common); 5850 thread->th.th_pri_common = NULL; 5851 }; // if 5852 5853 if (thread->th.th_task_state_memo_stack != NULL) { 5854 __kmp_free(thread->th.th_task_state_memo_stack); 5855 thread->th.th_task_state_memo_stack = NULL; 5856 } 5857 5858 #if KMP_USE_BGET 5859 if (thread->th.th_local.bget_data != NULL) { 5860 __kmp_finalize_bget(thread); 5861 }; // if 5862 #endif 5863 5864 #if KMP_AFFINITY_SUPPORTED 5865 if (thread->th.th_affin_mask != NULL) { 5866 KMP_CPU_FREE(thread->th.th_affin_mask); 5867 thread->th.th_affin_mask = NULL; 5868 }; // if 5869 #endif /* KMP_AFFINITY_SUPPORTED */ 5870 5871 __kmp_reap_team(thread->th.th_serial_team); 5872 thread->th.th_serial_team = NULL; 5873 __kmp_free(thread); 5874 5875 KMP_MB(); 5876 5877 } // __kmp_reap_thread 5878 5879 static void __kmp_internal_end(void) { 5880 int i; 5881 5882 /* First, unregister the library */ 5883 __kmp_unregister_library(); 5884 5885 #if KMP_OS_WINDOWS 5886 /* In Win static library, we can't tell when a root actually dies, so we 5887 reclaim the data structures for any root threads that have died but not 5888 unregistered themselves, in order to shut down cleanly. 5889 In Win dynamic library we also can't tell when a thread dies. */ 5890 __kmp_reclaim_dead_roots(); // AC: moved here to always clean resources of 5891 // dead roots 5892 #endif 5893 5894 for (i = 0; i < __kmp_threads_capacity; i++) 5895 if (__kmp_root[i]) 5896 if (__kmp_root[i]->r.r_active) 5897 break; 5898 KMP_MB(); /* Flush all pending memory write invalidates. */ 5899 TCW_SYNC_4(__kmp_global.g.g_done, TRUE); 5900 5901 if (i < __kmp_threads_capacity) { 5902 #if KMP_USE_MONITOR 5903 // 2009-09-08 (lev): Other alive roots found. Why do we kill the monitor?? 5904 KMP_MB(); /* Flush all pending memory write invalidates. */ 5905 5906 // Need to check that monitor was initialized before reaping it. If we are 5907 // called form __kmp_atfork_child (which sets __kmp_init_parallel = 0), then 5908 // __kmp_monitor will appear to contain valid data, but it is only valid in the 5909 // parent process, not the child. 5910 // New behavior (201008): instead of keying off of the flag 5911 // __kmp_init_parallel, the monitor thread creation is keyed off 5912 // of the new flag __kmp_init_monitor. 5913 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock); 5914 if (TCR_4(__kmp_init_monitor)) { 5915 __kmp_reap_monitor(&__kmp_monitor); 5916 TCW_4(__kmp_init_monitor, 0); 5917 } 5918 __kmp_release_bootstrap_lock(&__kmp_monitor_lock); 5919 KA_TRACE(10, ("__kmp_internal_end: monitor reaped\n")); 5920 #endif // KMP_USE_MONITOR 5921 } else { 5922 /* TODO move this to cleanup code */ 5923 #ifdef KMP_DEBUG 5924 /* make sure that everything has properly ended */ 5925 for (i = 0; i < __kmp_threads_capacity; i++) { 5926 if (__kmp_root[i]) { 5927 // KMP_ASSERT( ! KMP_UBER_GTID( i ) ); // AC: 5928 // there can be uber threads alive here 5929 KMP_ASSERT(!__kmp_root[i]->r.r_active); // TODO: can they be active? 5930 } 5931 } 5932 #endif 5933 5934 KMP_MB(); 5935 5936 // Reap the worker threads. 5937 // This is valid for now, but be careful if threads are reaped sooner. 5938 while (__kmp_thread_pool != NULL) { // Loop thru all the thread in the pool. 5939 // Get the next thread from the pool. 5940 kmp_info_t *thread = CCAST(kmp_info_t *, __kmp_thread_pool); 5941 __kmp_thread_pool = thread->th.th_next_pool; 5942 // Reap it. 5943 KMP_DEBUG_ASSERT(thread->th.th_reap_state == KMP_SAFE_TO_REAP); 5944 thread->th.th_next_pool = NULL; 5945 thread->th.th_in_pool = FALSE; 5946 __kmp_reap_thread(thread, 0); 5947 }; // while 5948 __kmp_thread_pool_insert_pt = NULL; 5949 5950 // Reap teams. 5951 while (__kmp_team_pool != NULL) { // Loop thru all the teams in the pool. 5952 // Get the next team from the pool. 5953 kmp_team_t *team = CCAST(kmp_team_t *, __kmp_team_pool); 5954 __kmp_team_pool = team->t.t_next_pool; 5955 // Reap it. 5956 team->t.t_next_pool = NULL; 5957 __kmp_reap_team(team); 5958 }; // while 5959 5960 __kmp_reap_task_teams(); 5961 5962 for (i = 0; i < __kmp_threads_capacity; ++i) { 5963 // TBD: Add some checking... 5964 // Something like KMP_DEBUG_ASSERT( __kmp_thread[ i ] == NULL ); 5965 } 5966 5967 /* Make sure all threadprivate destructors get run by joining with all 5968 worker threads before resetting this flag */ 5969 TCW_SYNC_4(__kmp_init_common, FALSE); 5970 5971 KA_TRACE(10, ("__kmp_internal_end: all workers reaped\n")); 5972 KMP_MB(); 5973 5974 #if KMP_USE_MONITOR 5975 // See note above: One of the possible fixes for CQ138434 / CQ140126 5976 // 5977 // FIXME: push both code fragments down and CSE them? 5978 // push them into __kmp_cleanup() ? 5979 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock); 5980 if (TCR_4(__kmp_init_monitor)) { 5981 __kmp_reap_monitor(&__kmp_monitor); 5982 TCW_4(__kmp_init_monitor, 0); 5983 } 5984 __kmp_release_bootstrap_lock(&__kmp_monitor_lock); 5985 KA_TRACE(10, ("__kmp_internal_end: monitor reaped\n")); 5986 #endif 5987 } /* else !__kmp_global.t_active */ 5988 TCW_4(__kmp_init_gtid, FALSE); 5989 KMP_MB(); /* Flush all pending memory write invalidates. */ 5990 5991 __kmp_cleanup(); 5992 #if OMPT_SUPPORT 5993 ompt_fini(); 5994 #endif 5995 } 5996 5997 void __kmp_internal_end_library(int gtid_req) { 5998 /* if we have already cleaned up, don't try again, it wouldn't be pretty */ 5999 /* this shouldn't be a race condition because __kmp_internal_end() is the 6000 only place to clear __kmp_serial_init */ 6001 /* we'll check this later too, after we get the lock */ 6002 // 2009-09-06: We do not set g_abort without setting g_done. This check looks 6003 // redundaant, because the next check will work in any case. 6004 if (__kmp_global.g.g_abort) { 6005 KA_TRACE(11, ("__kmp_internal_end_library: abort, exiting\n")); 6006 /* TODO abort? */ 6007 return; 6008 } 6009 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 6010 KA_TRACE(10, ("__kmp_internal_end_library: already finished\n")); 6011 return; 6012 } 6013 6014 KMP_MB(); /* Flush all pending memory write invalidates. */ 6015 6016 /* find out who we are and what we should do */ 6017 { 6018 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific(); 6019 KA_TRACE( 6020 10, ("__kmp_internal_end_library: enter T#%d (%d)\n", gtid, gtid_req)); 6021 if (gtid == KMP_GTID_SHUTDOWN) { 6022 KA_TRACE(10, ("__kmp_internal_end_library: !__kmp_init_runtime, system " 6023 "already shutdown\n")); 6024 return; 6025 } else if (gtid == KMP_GTID_MONITOR) { 6026 KA_TRACE(10, ("__kmp_internal_end_library: monitor thread, gtid not " 6027 "registered, or system shutdown\n")); 6028 return; 6029 } else if (gtid == KMP_GTID_DNE) { 6030 KA_TRACE(10, ("__kmp_internal_end_library: gtid not registered or system " 6031 "shutdown\n")); 6032 /* we don't know who we are, but we may still shutdown the library */ 6033 } else if (KMP_UBER_GTID(gtid)) { 6034 /* unregister ourselves as an uber thread. gtid is no longer valid */ 6035 if (__kmp_root[gtid]->r.r_active) { 6036 __kmp_global.g.g_abort = -1; 6037 TCW_SYNC_4(__kmp_global.g.g_done, TRUE); 6038 KA_TRACE(10, 6039 ("__kmp_internal_end_library: root still active, abort T#%d\n", 6040 gtid)); 6041 return; 6042 } else { 6043 KA_TRACE( 6044 10, 6045 ("__kmp_internal_end_library: unregistering sibling T#%d\n", gtid)); 6046 __kmp_unregister_root_current_thread(gtid); 6047 } 6048 } else { 6049 /* worker threads may call this function through the atexit handler, if they 6050 * call exit() */ 6051 /* For now, skip the usual subsequent processing and just dump the debug buffer. 6052 TODO: do a thorough shutdown instead */ 6053 #ifdef DUMP_DEBUG_ON_EXIT 6054 if (__kmp_debug_buf) 6055 __kmp_dump_debug_buffer(); 6056 #endif 6057 return; 6058 } 6059 } 6060 /* synchronize the termination process */ 6061 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6062 6063 /* have we already finished */ 6064 if (__kmp_global.g.g_abort) { 6065 KA_TRACE(10, ("__kmp_internal_end_library: abort, exiting\n")); 6066 /* TODO abort? */ 6067 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6068 return; 6069 } 6070 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 6071 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6072 return; 6073 } 6074 6075 /* We need this lock to enforce mutex between this reading of 6076 __kmp_threads_capacity and the writing by __kmp_register_root. 6077 Alternatively, we can use a counter of roots that is atomically updated by 6078 __kmp_get_global_thread_id_reg, __kmp_do_serial_initialize and 6079 __kmp_internal_end_*. */ 6080 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 6081 6082 /* now we can safely conduct the actual termination */ 6083 __kmp_internal_end(); 6084 6085 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 6086 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6087 6088 KA_TRACE(10, ("__kmp_internal_end_library: exit\n")); 6089 6090 #ifdef DUMP_DEBUG_ON_EXIT 6091 if (__kmp_debug_buf) 6092 __kmp_dump_debug_buffer(); 6093 #endif 6094 6095 #if KMP_OS_WINDOWS 6096 __kmp_close_console(); 6097 #endif 6098 6099 __kmp_fini_allocator(); 6100 6101 } // __kmp_internal_end_library 6102 6103 void __kmp_internal_end_thread(int gtid_req) { 6104 int i; 6105 6106 /* if we have already cleaned up, don't try again, it wouldn't be pretty */ 6107 /* this shouldn't be a race condition because __kmp_internal_end() is the 6108 * only place to clear __kmp_serial_init */ 6109 /* we'll check this later too, after we get the lock */ 6110 // 2009-09-06: We do not set g_abort without setting g_done. This check looks 6111 // redundant, because the next check will work in any case. 6112 if (__kmp_global.g.g_abort) { 6113 KA_TRACE(11, ("__kmp_internal_end_thread: abort, exiting\n")); 6114 /* TODO abort? */ 6115 return; 6116 } 6117 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 6118 KA_TRACE(10, ("__kmp_internal_end_thread: already finished\n")); 6119 return; 6120 } 6121 6122 KMP_MB(); /* Flush all pending memory write invalidates. */ 6123 6124 /* find out who we are and what we should do */ 6125 { 6126 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific(); 6127 KA_TRACE(10, 6128 ("__kmp_internal_end_thread: enter T#%d (%d)\n", gtid, gtid_req)); 6129 if (gtid == KMP_GTID_SHUTDOWN) { 6130 KA_TRACE(10, ("__kmp_internal_end_thread: !__kmp_init_runtime, system " 6131 "already shutdown\n")); 6132 return; 6133 } else if (gtid == KMP_GTID_MONITOR) { 6134 KA_TRACE(10, ("__kmp_internal_end_thread: monitor thread, gtid not " 6135 "registered, or system shutdown\n")); 6136 return; 6137 } else if (gtid == KMP_GTID_DNE) { 6138 KA_TRACE(10, ("__kmp_internal_end_thread: gtid not registered or system " 6139 "shutdown\n")); 6140 return; 6141 /* we don't know who we are */ 6142 } else if (KMP_UBER_GTID(gtid)) { 6143 /* unregister ourselves as an uber thread. gtid is no longer valid */ 6144 if (__kmp_root[gtid]->r.r_active) { 6145 __kmp_global.g.g_abort = -1; 6146 TCW_SYNC_4(__kmp_global.g.g_done, TRUE); 6147 KA_TRACE(10, 6148 ("__kmp_internal_end_thread: root still active, abort T#%d\n", 6149 gtid)); 6150 return; 6151 } else { 6152 KA_TRACE(10, ("__kmp_internal_end_thread: unregistering sibling T#%d\n", 6153 gtid)); 6154 __kmp_unregister_root_current_thread(gtid); 6155 } 6156 } else { 6157 /* just a worker thread, let's leave */ 6158 KA_TRACE(10, ("__kmp_internal_end_thread: worker thread T#%d\n", gtid)); 6159 6160 if (gtid >= 0) { 6161 __kmp_threads[gtid]->th.th_task_team = NULL; 6162 } 6163 6164 KA_TRACE(10, 6165 ("__kmp_internal_end_thread: worker thread done, exiting T#%d\n", 6166 gtid)); 6167 return; 6168 } 6169 } 6170 #if defined KMP_DYNAMIC_LIB 6171 // AC: lets not shutdown the Linux* OS dynamic library at the exit of uber 6172 // thread, because we will better shutdown later in the library destructor. 6173 // The reason of this change is performance problem when non-openmp thread in 6174 // a loop forks and joins many openmp threads. We can save a lot of time 6175 // keeping worker threads alive until the program shutdown. 6176 // OM: Removed Linux* OS restriction to fix the crash on OS X* (DPD200239966) 6177 // and Windows(DPD200287443) that occurs when using critical sections from 6178 // foreign threads. 6179 KA_TRACE(10, ("__kmp_internal_end_thread: exiting T#%d\n", gtid_req)); 6180 return; 6181 #endif 6182 /* synchronize the termination process */ 6183 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6184 6185 /* have we already finished */ 6186 if (__kmp_global.g.g_abort) { 6187 KA_TRACE(10, ("__kmp_internal_end_thread: abort, exiting\n")); 6188 /* TODO abort? */ 6189 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6190 return; 6191 } 6192 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 6193 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6194 return; 6195 } 6196 6197 /* We need this lock to enforce mutex between this reading of 6198 __kmp_threads_capacity and the writing by __kmp_register_root. 6199 Alternatively, we can use a counter of roots that is atomically updated by 6200 __kmp_get_global_thread_id_reg, __kmp_do_serial_initialize and 6201 __kmp_internal_end_*. */ 6202 6203 /* should we finish the run-time? are all siblings done? */ 6204 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 6205 6206 for (i = 0; i < __kmp_threads_capacity; ++i) { 6207 if (KMP_UBER_GTID(i)) { 6208 KA_TRACE( 6209 10, 6210 ("__kmp_internal_end_thread: remaining sibling task: gtid==%d\n", i)); 6211 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 6212 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6213 return; 6214 }; 6215 } 6216 6217 /* now we can safely conduct the actual termination */ 6218 6219 __kmp_internal_end(); 6220 6221 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 6222 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6223 6224 KA_TRACE(10, ("__kmp_internal_end_thread: exit T#%d\n", gtid_req)); 6225 6226 #ifdef DUMP_DEBUG_ON_EXIT 6227 if (__kmp_debug_buf) 6228 __kmp_dump_debug_buffer(); 6229 #endif 6230 } // __kmp_internal_end_thread 6231 6232 // ----------------------------------------------------------------------------- 6233 // Library registration stuff. 6234 6235 static long __kmp_registration_flag = 0; 6236 // Random value used to indicate library initialization. 6237 static char *__kmp_registration_str = NULL; 6238 // Value to be saved in env var __KMP_REGISTERED_LIB_<pid>. 6239 6240 static inline char *__kmp_reg_status_name() { 6241 /* On RHEL 3u5 if linked statically, getpid() returns different values in 6242 each thread. If registration and unregistration go in different threads 6243 (omp_misc_other_root_exit.cpp test case), the name of registered_lib_env 6244 env var can not be found, because the name will contain different pid. */ 6245 return __kmp_str_format("__KMP_REGISTERED_LIB_%d", (int)getpid()); 6246 } // __kmp_reg_status_get 6247 6248 void __kmp_register_library_startup(void) { 6249 6250 char *name = __kmp_reg_status_name(); // Name of the environment variable. 6251 int done = 0; 6252 union { 6253 double dtime; 6254 long ltime; 6255 } time; 6256 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 6257 __kmp_initialize_system_tick(); 6258 #endif 6259 __kmp_read_system_time(&time.dtime); 6260 __kmp_registration_flag = 0xCAFE0000L | (time.ltime & 0x0000FFFFL); 6261 __kmp_registration_str = 6262 __kmp_str_format("%p-%lx-%s", &__kmp_registration_flag, 6263 __kmp_registration_flag, KMP_LIBRARY_FILE); 6264 6265 KA_TRACE(50, ("__kmp_register_library_startup: %s=\"%s\"\n", name, 6266 __kmp_registration_str)); 6267 6268 while (!done) { 6269 6270 char *value = NULL; // Actual value of the environment variable. 6271 6272 // Set environment variable, but do not overwrite if it is exist. 6273 __kmp_env_set(name, __kmp_registration_str, 0); 6274 // Check the variable is written. 6275 value = __kmp_env_get(name); 6276 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) { 6277 6278 done = 1; // Ok, environment variable set successfully, exit the loop. 6279 6280 } else { 6281 6282 // Oops. Write failed. Another copy of OpenMP RTL is in memory. 6283 // Check whether it alive or dead. 6284 int neighbor = 0; // 0 -- unknown status, 1 -- alive, 2 -- dead. 6285 char *tail = value; 6286 char *flag_addr_str = NULL; 6287 char *flag_val_str = NULL; 6288 char const *file_name = NULL; 6289 __kmp_str_split(tail, '-', &flag_addr_str, &tail); 6290 __kmp_str_split(tail, '-', &flag_val_str, &tail); 6291 file_name = tail; 6292 if (tail != NULL) { 6293 long *flag_addr = 0; 6294 long flag_val = 0; 6295 KMP_SSCANF(flag_addr_str, "%p", &flag_addr); 6296 KMP_SSCANF(flag_val_str, "%lx", &flag_val); 6297 if (flag_addr != 0 && flag_val != 0 && strcmp(file_name, "") != 0) { 6298 // First, check whether environment-encoded address is mapped into 6299 // addr space. 6300 // If so, dereference it to see if it still has the right value. 6301 if (__kmp_is_address_mapped(flag_addr) && *flag_addr == flag_val) { 6302 neighbor = 1; 6303 } else { 6304 // If not, then we know the other copy of the library is no longer 6305 // running. 6306 neighbor = 2; 6307 }; // if 6308 }; // if 6309 }; // if 6310 switch (neighbor) { 6311 case 0: // Cannot parse environment variable -- neighbor status unknown. 6312 // Assume it is the incompatible format of future version of the 6313 // library. Assume the other library is alive. 6314 // WARN( ... ); // TODO: Issue a warning. 6315 file_name = "unknown library"; 6316 // Attention! Falling to the next case. That's intentional. 6317 case 1: { // Neighbor is alive. 6318 // Check it is allowed. 6319 char *duplicate_ok = __kmp_env_get("KMP_DUPLICATE_LIB_OK"); 6320 if (!__kmp_str_match_true(duplicate_ok)) { 6321 // That's not allowed. Issue fatal error. 6322 __kmp_fatal(KMP_MSG(DuplicateLibrary, KMP_LIBRARY_FILE, file_name), 6323 KMP_HNT(DuplicateLibrary), __kmp_msg_null); 6324 }; // if 6325 KMP_INTERNAL_FREE(duplicate_ok); 6326 __kmp_duplicate_library_ok = 1; 6327 done = 1; // Exit the loop. 6328 } break; 6329 case 2: { // Neighbor is dead. 6330 // Clear the variable and try to register library again. 6331 __kmp_env_unset(name); 6332 } break; 6333 default: { KMP_DEBUG_ASSERT(0); } break; 6334 }; // switch 6335 6336 }; // if 6337 KMP_INTERNAL_FREE((void *)value); 6338 6339 }; // while 6340 KMP_INTERNAL_FREE((void *)name); 6341 6342 } // func __kmp_register_library_startup 6343 6344 void __kmp_unregister_library(void) { 6345 6346 char *name = __kmp_reg_status_name(); 6347 char *value = __kmp_env_get(name); 6348 6349 KMP_DEBUG_ASSERT(__kmp_registration_flag != 0); 6350 KMP_DEBUG_ASSERT(__kmp_registration_str != NULL); 6351 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) { 6352 // Ok, this is our variable. Delete it. 6353 __kmp_env_unset(name); 6354 }; // if 6355 6356 KMP_INTERNAL_FREE(__kmp_registration_str); 6357 KMP_INTERNAL_FREE(value); 6358 KMP_INTERNAL_FREE(name); 6359 6360 __kmp_registration_flag = 0; 6361 __kmp_registration_str = NULL; 6362 6363 } // __kmp_unregister_library 6364 6365 // End of Library registration stuff. 6366 // ----------------------------------------------------------------------------- 6367 6368 #if KMP_MIC_SUPPORTED 6369 6370 static void __kmp_check_mic_type() { 6371 kmp_cpuid_t cpuid_state = {0}; 6372 kmp_cpuid_t *cs_p = &cpuid_state; 6373 __kmp_x86_cpuid(1, 0, cs_p); 6374 // We don't support mic1 at the moment 6375 if ((cs_p->eax & 0xff0) == 0xB10) { 6376 __kmp_mic_type = mic2; 6377 } else if ((cs_p->eax & 0xf0ff0) == 0x50670) { 6378 __kmp_mic_type = mic3; 6379 } else { 6380 __kmp_mic_type = non_mic; 6381 } 6382 } 6383 6384 #endif /* KMP_MIC_SUPPORTED */ 6385 6386 static void __kmp_do_serial_initialize(void) { 6387 int i, gtid; 6388 int size; 6389 6390 KA_TRACE(10, ("__kmp_do_serial_initialize: enter\n")); 6391 6392 KMP_DEBUG_ASSERT(sizeof(kmp_int32) == 4); 6393 KMP_DEBUG_ASSERT(sizeof(kmp_uint32) == 4); 6394 KMP_DEBUG_ASSERT(sizeof(kmp_int64) == 8); 6395 KMP_DEBUG_ASSERT(sizeof(kmp_uint64) == 8); 6396 KMP_DEBUG_ASSERT(sizeof(kmp_intptr_t) == sizeof(void *)); 6397 6398 #if OMPT_SUPPORT 6399 ompt_pre_init(); 6400 #endif 6401 6402 __kmp_validate_locks(); 6403 6404 /* Initialize internal memory allocator */ 6405 __kmp_init_allocator(); 6406 6407 /* Register the library startup via an environment variable and check to see 6408 whether another copy of the library is already registered. */ 6409 6410 __kmp_register_library_startup(); 6411 6412 /* TODO reinitialization of library */ 6413 if (TCR_4(__kmp_global.g.g_done)) { 6414 KA_TRACE(10, ("__kmp_do_serial_initialize: reinitialization of library\n")); 6415 } 6416 6417 __kmp_global.g.g_abort = 0; 6418 TCW_SYNC_4(__kmp_global.g.g_done, FALSE); 6419 6420 /* initialize the locks */ 6421 #if KMP_USE_ADAPTIVE_LOCKS 6422 #if KMP_DEBUG_ADAPTIVE_LOCKS 6423 __kmp_init_speculative_stats(); 6424 #endif 6425 #endif 6426 #if KMP_STATS_ENABLED 6427 __kmp_stats_init(); 6428 #endif 6429 __kmp_init_lock(&__kmp_global_lock); 6430 __kmp_init_queuing_lock(&__kmp_dispatch_lock); 6431 __kmp_init_lock(&__kmp_debug_lock); 6432 __kmp_init_atomic_lock(&__kmp_atomic_lock); 6433 __kmp_init_atomic_lock(&__kmp_atomic_lock_1i); 6434 __kmp_init_atomic_lock(&__kmp_atomic_lock_2i); 6435 __kmp_init_atomic_lock(&__kmp_atomic_lock_4i); 6436 __kmp_init_atomic_lock(&__kmp_atomic_lock_4r); 6437 __kmp_init_atomic_lock(&__kmp_atomic_lock_8i); 6438 __kmp_init_atomic_lock(&__kmp_atomic_lock_8r); 6439 __kmp_init_atomic_lock(&__kmp_atomic_lock_8c); 6440 __kmp_init_atomic_lock(&__kmp_atomic_lock_10r); 6441 __kmp_init_atomic_lock(&__kmp_atomic_lock_16r); 6442 __kmp_init_atomic_lock(&__kmp_atomic_lock_16c); 6443 __kmp_init_atomic_lock(&__kmp_atomic_lock_20c); 6444 __kmp_init_atomic_lock(&__kmp_atomic_lock_32c); 6445 __kmp_init_bootstrap_lock(&__kmp_forkjoin_lock); 6446 __kmp_init_bootstrap_lock(&__kmp_exit_lock); 6447 #if KMP_USE_MONITOR 6448 __kmp_init_bootstrap_lock(&__kmp_monitor_lock); 6449 #endif 6450 __kmp_init_bootstrap_lock(&__kmp_tp_cached_lock); 6451 6452 /* conduct initialization and initial setup of configuration */ 6453 6454 __kmp_runtime_initialize(); 6455 6456 #if KMP_MIC_SUPPORTED 6457 __kmp_check_mic_type(); 6458 #endif 6459 6460 // Some global variable initialization moved here from kmp_env_initialize() 6461 #ifdef KMP_DEBUG 6462 kmp_diag = 0; 6463 #endif 6464 __kmp_abort_delay = 0; 6465 6466 // From __kmp_init_dflt_team_nth() 6467 /* assume the entire machine will be used */ 6468 __kmp_dflt_team_nth_ub = __kmp_xproc; 6469 if (__kmp_dflt_team_nth_ub < KMP_MIN_NTH) { 6470 __kmp_dflt_team_nth_ub = KMP_MIN_NTH; 6471 } 6472 if (__kmp_dflt_team_nth_ub > __kmp_sys_max_nth) { 6473 __kmp_dflt_team_nth_ub = __kmp_sys_max_nth; 6474 } 6475 __kmp_max_nth = __kmp_sys_max_nth; 6476 __kmp_cg_max_nth = __kmp_sys_max_nth; 6477 __kmp_teams_max_nth = __kmp_xproc; // set a "reasonable" default 6478 if (__kmp_teams_max_nth > __kmp_sys_max_nth) { 6479 __kmp_teams_max_nth = __kmp_sys_max_nth; 6480 } 6481 6482 // Three vars below moved here from __kmp_env_initialize() "KMP_BLOCKTIME" 6483 // part 6484 __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME; 6485 #if KMP_USE_MONITOR 6486 __kmp_monitor_wakeups = 6487 KMP_WAKEUPS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups); 6488 __kmp_bt_intervals = 6489 KMP_INTERVALS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups); 6490 #endif 6491 // From "KMP_LIBRARY" part of __kmp_env_initialize() 6492 __kmp_library = library_throughput; 6493 // From KMP_SCHEDULE initialization 6494 __kmp_static = kmp_sch_static_balanced; 6495 // AC: do not use analytical here, because it is non-monotonous 6496 //__kmp_guided = kmp_sch_guided_iterative_chunked; 6497 //__kmp_auto = kmp_sch_guided_analytical_chunked; // AC: it is the default, no 6498 // need to repeat assignment 6499 // Barrier initialization. Moved here from __kmp_env_initialize() Barrier branch 6500 // bit control and barrier method control parts 6501 #if KMP_FAST_REDUCTION_BARRIER 6502 #define kmp_reduction_barrier_gather_bb ((int)1) 6503 #define kmp_reduction_barrier_release_bb ((int)1) 6504 #define kmp_reduction_barrier_gather_pat bp_hyper_bar 6505 #define kmp_reduction_barrier_release_pat bp_hyper_bar 6506 #endif // KMP_FAST_REDUCTION_BARRIER 6507 for (i = bs_plain_barrier; i < bs_last_barrier; i++) { 6508 __kmp_barrier_gather_branch_bits[i] = __kmp_barrier_gather_bb_dflt; 6509 __kmp_barrier_release_branch_bits[i] = __kmp_barrier_release_bb_dflt; 6510 __kmp_barrier_gather_pattern[i] = __kmp_barrier_gather_pat_dflt; 6511 __kmp_barrier_release_pattern[i] = __kmp_barrier_release_pat_dflt; 6512 #if KMP_FAST_REDUCTION_BARRIER 6513 if (i == bs_reduction_barrier) { // tested and confirmed on ALTIX only ( 6514 // lin_64 ): hyper,1 6515 __kmp_barrier_gather_branch_bits[i] = kmp_reduction_barrier_gather_bb; 6516 __kmp_barrier_release_branch_bits[i] = kmp_reduction_barrier_release_bb; 6517 __kmp_barrier_gather_pattern[i] = kmp_reduction_barrier_gather_pat; 6518 __kmp_barrier_release_pattern[i] = kmp_reduction_barrier_release_pat; 6519 } 6520 #endif // KMP_FAST_REDUCTION_BARRIER 6521 } 6522 #if KMP_FAST_REDUCTION_BARRIER 6523 #undef kmp_reduction_barrier_release_pat 6524 #undef kmp_reduction_barrier_gather_pat 6525 #undef kmp_reduction_barrier_release_bb 6526 #undef kmp_reduction_barrier_gather_bb 6527 #endif // KMP_FAST_REDUCTION_BARRIER 6528 #if KMP_MIC_SUPPORTED 6529 if (__kmp_mic_type == mic2) { // KNC 6530 // AC: plane=3,2, forkjoin=2,1 are optimal for 240 threads on KNC 6531 __kmp_barrier_gather_branch_bits[bs_plain_barrier] = 3; // plain gather 6532 __kmp_barrier_release_branch_bits[bs_forkjoin_barrier] = 6533 1; // forkjoin release 6534 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar; 6535 __kmp_barrier_release_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar; 6536 } 6537 #if KMP_FAST_REDUCTION_BARRIER 6538 if (__kmp_mic_type == mic2) { // KNC 6539 __kmp_barrier_gather_pattern[bs_reduction_barrier] = bp_hierarchical_bar; 6540 __kmp_barrier_release_pattern[bs_reduction_barrier] = bp_hierarchical_bar; 6541 } 6542 #endif // KMP_FAST_REDUCTION_BARRIER 6543 #endif // KMP_MIC_SUPPORTED 6544 6545 // From KMP_CHECKS initialization 6546 #ifdef KMP_DEBUG 6547 __kmp_env_checks = TRUE; /* development versions have the extra checks */ 6548 #else 6549 __kmp_env_checks = FALSE; /* port versions do not have the extra checks */ 6550 #endif 6551 6552 // From "KMP_FOREIGN_THREADS_THREADPRIVATE" initialization 6553 __kmp_foreign_tp = TRUE; 6554 6555 __kmp_global.g.g_dynamic = FALSE; 6556 __kmp_global.g.g_dynamic_mode = dynamic_default; 6557 6558 __kmp_env_initialize(NULL); 6559 6560 // Print all messages in message catalog for testing purposes. 6561 #ifdef KMP_DEBUG 6562 char const *val = __kmp_env_get("KMP_DUMP_CATALOG"); 6563 if (__kmp_str_match_true(val)) { 6564 kmp_str_buf_t buffer; 6565 __kmp_str_buf_init(&buffer); 6566 __kmp_i18n_dump_catalog(&buffer); 6567 __kmp_printf("%s", buffer.str); 6568 __kmp_str_buf_free(&buffer); 6569 }; // if 6570 __kmp_env_free(&val); 6571 #endif 6572 6573 __kmp_threads_capacity = 6574 __kmp_initial_threads_capacity(__kmp_dflt_team_nth_ub); 6575 // Moved here from __kmp_env_initialize() "KMP_ALL_THREADPRIVATE" part 6576 __kmp_tp_capacity = __kmp_default_tp_capacity( 6577 __kmp_dflt_team_nth_ub, __kmp_max_nth, __kmp_allThreadsSpecified); 6578 6579 // If the library is shut down properly, both pools must be NULL. Just in 6580 // case, set them to NULL -- some memory may leak, but subsequent code will 6581 // work even if pools are not freed. 6582 KMP_DEBUG_ASSERT(__kmp_thread_pool == NULL); 6583 KMP_DEBUG_ASSERT(__kmp_thread_pool_insert_pt == NULL); 6584 KMP_DEBUG_ASSERT(__kmp_team_pool == NULL); 6585 __kmp_thread_pool = NULL; 6586 __kmp_thread_pool_insert_pt = NULL; 6587 __kmp_team_pool = NULL; 6588 6589 /* Allocate all of the variable sized records */ 6590 /* NOTE: __kmp_threads_capacity entries are allocated, but the arrays are 6591 * expandable */ 6592 /* Since allocation is cache-aligned, just add extra padding at the end */ 6593 size = 6594 (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * __kmp_threads_capacity + 6595 CACHE_LINE; 6596 __kmp_threads = (kmp_info_t **)__kmp_allocate(size); 6597 __kmp_root = (kmp_root_t **)((char *)__kmp_threads + 6598 sizeof(kmp_info_t *) * __kmp_threads_capacity); 6599 6600 /* init thread counts */ 6601 KMP_DEBUG_ASSERT(__kmp_all_nth == 6602 0); // Asserts fail if the library is reinitializing and 6603 KMP_DEBUG_ASSERT(__kmp_nth == 0); // something was wrong in termination. 6604 __kmp_all_nth = 0; 6605 __kmp_nth = 0; 6606 6607 /* setup the uber master thread and hierarchy */ 6608 gtid = __kmp_register_root(TRUE); 6609 KA_TRACE(10, ("__kmp_do_serial_initialize T#%d\n", gtid)); 6610 KMP_ASSERT(KMP_UBER_GTID(gtid)); 6611 KMP_ASSERT(KMP_INITIAL_GTID(gtid)); 6612 6613 KMP_MB(); /* Flush all pending memory write invalidates. */ 6614 6615 __kmp_common_initialize(); 6616 6617 #if KMP_OS_UNIX 6618 /* invoke the child fork handler */ 6619 __kmp_register_atfork(); 6620 #endif 6621 6622 #if !defined KMP_DYNAMIC_LIB 6623 { 6624 /* Invoke the exit handler when the program finishes, only for static 6625 library. For dynamic library, we already have _fini and DllMain. */ 6626 int rc = atexit(__kmp_internal_end_atexit); 6627 if (rc != 0) { 6628 __kmp_fatal(KMP_MSG(FunctionError, "atexit()"), KMP_ERR(rc), 6629 __kmp_msg_null); 6630 }; // if 6631 } 6632 #endif 6633 6634 #if KMP_HANDLE_SIGNALS 6635 #if KMP_OS_UNIX 6636 /* NOTE: make sure that this is called before the user installs their own 6637 signal handlers so that the user handlers are called first. this way they 6638 can return false, not call our handler, avoid terminating the library, and 6639 continue execution where they left off. */ 6640 __kmp_install_signals(FALSE); 6641 #endif /* KMP_OS_UNIX */ 6642 #if KMP_OS_WINDOWS 6643 __kmp_install_signals(TRUE); 6644 #endif /* KMP_OS_WINDOWS */ 6645 #endif 6646 6647 /* we have finished the serial initialization */ 6648 __kmp_init_counter++; 6649 6650 __kmp_init_serial = TRUE; 6651 6652 if (__kmp_settings) { 6653 __kmp_env_print(); 6654 } 6655 6656 #if OMP_40_ENABLED 6657 if (__kmp_display_env || __kmp_display_env_verbose) { 6658 __kmp_env_print_2(); 6659 } 6660 #endif // OMP_40_ENABLED 6661 6662 #if OMPT_SUPPORT 6663 ompt_post_init(); 6664 #endif 6665 6666 KMP_MB(); 6667 6668 KA_TRACE(10, ("__kmp_do_serial_initialize: exit\n")); 6669 } 6670 6671 void __kmp_serial_initialize(void) { 6672 if (__kmp_init_serial) { 6673 return; 6674 } 6675 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6676 if (__kmp_init_serial) { 6677 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6678 return; 6679 } 6680 __kmp_do_serial_initialize(); 6681 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6682 } 6683 6684 static void __kmp_do_middle_initialize(void) { 6685 int i, j; 6686 int prev_dflt_team_nth; 6687 6688 if (!__kmp_init_serial) { 6689 __kmp_do_serial_initialize(); 6690 } 6691 6692 KA_TRACE(10, ("__kmp_middle_initialize: enter\n")); 6693 6694 // Save the previous value for the __kmp_dflt_team_nth so that 6695 // we can avoid some reinitialization if it hasn't changed. 6696 prev_dflt_team_nth = __kmp_dflt_team_nth; 6697 6698 #if KMP_AFFINITY_SUPPORTED 6699 // __kmp_affinity_initialize() will try to set __kmp_ncores to the 6700 // number of cores on the machine. 6701 __kmp_affinity_initialize(); 6702 6703 // Run through the __kmp_threads array and set the affinity mask 6704 // for each root thread that is currently registered with the RTL. 6705 for (i = 0; i < __kmp_threads_capacity; i++) { 6706 if (TCR_PTR(__kmp_threads[i]) != NULL) { 6707 __kmp_affinity_set_init_mask(i, TRUE); 6708 } 6709 } 6710 #endif /* KMP_AFFINITY_SUPPORTED */ 6711 6712 KMP_ASSERT(__kmp_xproc > 0); 6713 if (__kmp_avail_proc == 0) { 6714 __kmp_avail_proc = __kmp_xproc; 6715 } 6716 6717 // If there were empty places in num_threads list (OMP_NUM_THREADS=,,2,3), 6718 // correct them now 6719 j = 0; 6720 while ((j < __kmp_nested_nth.used) && !__kmp_nested_nth.nth[j]) { 6721 __kmp_nested_nth.nth[j] = __kmp_dflt_team_nth = __kmp_dflt_team_nth_ub = 6722 __kmp_avail_proc; 6723 j++; 6724 } 6725 6726 if (__kmp_dflt_team_nth == 0) { 6727 #ifdef KMP_DFLT_NTH_CORES 6728 // Default #threads = #cores 6729 __kmp_dflt_team_nth = __kmp_ncores; 6730 KA_TRACE(20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = " 6731 "__kmp_ncores (%d)\n", 6732 __kmp_dflt_team_nth)); 6733 #else 6734 // Default #threads = #available OS procs 6735 __kmp_dflt_team_nth = __kmp_avail_proc; 6736 KA_TRACE(20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = " 6737 "__kmp_avail_proc(%d)\n", 6738 __kmp_dflt_team_nth)); 6739 #endif /* KMP_DFLT_NTH_CORES */ 6740 } 6741 6742 if (__kmp_dflt_team_nth < KMP_MIN_NTH) { 6743 __kmp_dflt_team_nth = KMP_MIN_NTH; 6744 } 6745 if (__kmp_dflt_team_nth > __kmp_sys_max_nth) { 6746 __kmp_dflt_team_nth = __kmp_sys_max_nth; 6747 } 6748 6749 // There's no harm in continuing if the following check fails, 6750 // but it indicates an error in the previous logic. 6751 KMP_DEBUG_ASSERT(__kmp_dflt_team_nth <= __kmp_dflt_team_nth_ub); 6752 6753 if (__kmp_dflt_team_nth != prev_dflt_team_nth) { 6754 // Run through the __kmp_threads array and set the num threads icv for each 6755 // root thread that is currently registered with the RTL (which has not 6756 // already explicitly set its nthreads-var with a call to 6757 // omp_set_num_threads()). 6758 for (i = 0; i < __kmp_threads_capacity; i++) { 6759 kmp_info_t *thread = __kmp_threads[i]; 6760 if (thread == NULL) 6761 continue; 6762 if (thread->th.th_current_task->td_icvs.nproc != 0) 6763 continue; 6764 6765 set__nproc(__kmp_threads[i], __kmp_dflt_team_nth); 6766 } 6767 } 6768 KA_TRACE( 6769 20, 6770 ("__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n", 6771 __kmp_dflt_team_nth)); 6772 6773 #ifdef KMP_ADJUST_BLOCKTIME 6774 /* Adjust blocktime to zero if necessary now that __kmp_avail_proc is set */ 6775 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 6776 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0); 6777 if (__kmp_nth > __kmp_avail_proc) { 6778 __kmp_zero_bt = TRUE; 6779 } 6780 } 6781 #endif /* KMP_ADJUST_BLOCKTIME */ 6782 6783 /* we have finished middle initialization */ 6784 TCW_SYNC_4(__kmp_init_middle, TRUE); 6785 6786 KA_TRACE(10, ("__kmp_do_middle_initialize: exit\n")); 6787 } 6788 6789 void __kmp_middle_initialize(void) { 6790 if (__kmp_init_middle) { 6791 return; 6792 } 6793 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6794 if (__kmp_init_middle) { 6795 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6796 return; 6797 } 6798 __kmp_do_middle_initialize(); 6799 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6800 } 6801 6802 void __kmp_parallel_initialize(void) { 6803 int gtid = __kmp_entry_gtid(); // this might be a new root 6804 6805 /* synchronize parallel initialization (for sibling) */ 6806 if (TCR_4(__kmp_init_parallel)) 6807 return; 6808 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6809 if (TCR_4(__kmp_init_parallel)) { 6810 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6811 return; 6812 } 6813 6814 /* TODO reinitialization after we have already shut down */ 6815 if (TCR_4(__kmp_global.g.g_done)) { 6816 KA_TRACE( 6817 10, 6818 ("__kmp_parallel_initialize: attempt to init while shutting down\n")); 6819 __kmp_infinite_loop(); 6820 } 6821 6822 /* jc: The lock __kmp_initz_lock is already held, so calling 6823 __kmp_serial_initialize would cause a deadlock. So we call 6824 __kmp_do_serial_initialize directly. */ 6825 if (!__kmp_init_middle) { 6826 __kmp_do_middle_initialize(); 6827 } 6828 6829 /* begin initialization */ 6830 KA_TRACE(10, ("__kmp_parallel_initialize: enter\n")); 6831 KMP_ASSERT(KMP_UBER_GTID(gtid)); 6832 6833 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 6834 // Save the FP control regs. 6835 // Worker threads will set theirs to these values at thread startup. 6836 __kmp_store_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word); 6837 __kmp_store_mxcsr(&__kmp_init_mxcsr); 6838 __kmp_init_mxcsr &= KMP_X86_MXCSR_MASK; 6839 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 6840 6841 #if KMP_OS_UNIX 6842 #if KMP_HANDLE_SIGNALS 6843 /* must be after __kmp_serial_initialize */ 6844 __kmp_install_signals(TRUE); 6845 #endif 6846 #endif 6847 6848 __kmp_suspend_initialize(); 6849 6850 #if defined(USE_LOAD_BALANCE) 6851 if (__kmp_global.g.g_dynamic_mode == dynamic_default) { 6852 __kmp_global.g.g_dynamic_mode = dynamic_load_balance; 6853 } 6854 #else 6855 if (__kmp_global.g.g_dynamic_mode == dynamic_default) { 6856 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit; 6857 } 6858 #endif 6859 6860 if (__kmp_version) { 6861 __kmp_print_version_2(); 6862 } 6863 6864 /* we have finished parallel initialization */ 6865 TCW_SYNC_4(__kmp_init_parallel, TRUE); 6866 6867 KMP_MB(); 6868 KA_TRACE(10, ("__kmp_parallel_initialize: exit\n")); 6869 6870 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6871 } 6872 6873 /* ------------------------------------------------------------------------ */ 6874 6875 void __kmp_run_before_invoked_task(int gtid, int tid, kmp_info_t *this_thr, 6876 kmp_team_t *team) { 6877 kmp_disp_t *dispatch; 6878 6879 KMP_MB(); 6880 6881 /* none of the threads have encountered any constructs, yet. */ 6882 this_thr->th.th_local.this_construct = 0; 6883 #if KMP_CACHE_MANAGE 6884 KMP_CACHE_PREFETCH(&this_thr->th.th_bar[bs_forkjoin_barrier].bb.b_arrived); 6885 #endif /* KMP_CACHE_MANAGE */ 6886 dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch); 6887 KMP_DEBUG_ASSERT(dispatch); 6888 KMP_DEBUG_ASSERT(team->t.t_dispatch); 6889 // KMP_DEBUG_ASSERT( this_thr->th.th_dispatch == &team->t.t_dispatch[ 6890 // this_thr->th.th_info.ds.ds_tid ] ); 6891 6892 dispatch->th_disp_index = 0; /* reset the dispatch buffer counter */ 6893 #if OMP_45_ENABLED 6894 dispatch->th_doacross_buf_idx = 6895 0; /* reset the doacross dispatch buffer counter */ 6896 #endif 6897 if (__kmp_env_consistency_check) 6898 __kmp_push_parallel(gtid, team->t.t_ident); 6899 6900 KMP_MB(); /* Flush all pending memory write invalidates. */ 6901 } 6902 6903 void __kmp_run_after_invoked_task(int gtid, int tid, kmp_info_t *this_thr, 6904 kmp_team_t *team) { 6905 if (__kmp_env_consistency_check) 6906 __kmp_pop_parallel(gtid, team->t.t_ident); 6907 6908 __kmp_finish_implicit_task(this_thr); 6909 } 6910 6911 int __kmp_invoke_task_func(int gtid) { 6912 int rc; 6913 int tid = __kmp_tid_from_gtid(gtid); 6914 kmp_info_t *this_thr = __kmp_threads[gtid]; 6915 kmp_team_t *team = this_thr->th.th_team; 6916 6917 __kmp_run_before_invoked_task(gtid, tid, this_thr, team); 6918 #if USE_ITT_BUILD 6919 if (__itt_stack_caller_create_ptr) { 6920 __kmp_itt_stack_callee_enter( 6921 (__itt_caller) 6922 team->t.t_stack_id); // inform ittnotify about entering user's code 6923 } 6924 #endif /* USE_ITT_BUILD */ 6925 #if INCLUDE_SSC_MARKS 6926 SSC_MARK_INVOKING(); 6927 #endif 6928 6929 #if OMPT_SUPPORT 6930 void *dummy; 6931 void **exit_runtime_p; 6932 ompt_task_id_t my_task_id; 6933 ompt_parallel_id_t my_parallel_id; 6934 6935 if (ompt_enabled) { 6936 exit_runtime_p = &(team->t.t_implicit_task_taskdata[tid] 6937 .ompt_task_info.frame.exit_runtime_frame); 6938 } else { 6939 exit_runtime_p = &dummy; 6940 } 6941 6942 #if OMPT_TRACE 6943 my_task_id = team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id; 6944 my_parallel_id = team->t.ompt_team_info.parallel_id; 6945 if (ompt_enabled && 6946 ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)) { 6947 ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)(my_parallel_id, 6948 my_task_id); 6949 } 6950 #endif 6951 #endif 6952 6953 { 6954 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 6955 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 6956 rc = 6957 __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn), gtid, 6958 tid, (int)team->t.t_argc, (void **)team->t.t_argv 6959 #if OMPT_SUPPORT 6960 , 6961 exit_runtime_p 6962 #endif 6963 ); 6964 #if OMPT_SUPPORT 6965 *exit_runtime_p = NULL; 6966 #endif 6967 } 6968 6969 #if USE_ITT_BUILD 6970 if (__itt_stack_caller_create_ptr) { 6971 __kmp_itt_stack_callee_leave( 6972 (__itt_caller) 6973 team->t.t_stack_id); // inform ittnotify about leaving user's code 6974 } 6975 #endif /* USE_ITT_BUILD */ 6976 __kmp_run_after_invoked_task(gtid, tid, this_thr, team); 6977 6978 return rc; 6979 } 6980 6981 #if OMP_40_ENABLED 6982 void __kmp_teams_master(int gtid) { 6983 // This routine is called by all master threads in teams construct 6984 kmp_info_t *thr = __kmp_threads[gtid]; 6985 kmp_team_t *team = thr->th.th_team; 6986 ident_t *loc = team->t.t_ident; 6987 thr->th.th_set_nproc = thr->th.th_teams_size.nth; 6988 KMP_DEBUG_ASSERT(thr->th.th_teams_microtask); 6989 KMP_DEBUG_ASSERT(thr->th.th_set_nproc); 6990 KA_TRACE(20, ("__kmp_teams_master: T#%d, Tid %d, microtask %p\n", gtid, 6991 __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask)); 6992 // Launch league of teams now, but not let workers execute 6993 // (they hang on fork barrier until next parallel) 6994 #if INCLUDE_SSC_MARKS 6995 SSC_MARK_FORKING(); 6996 #endif 6997 __kmp_fork_call(loc, gtid, fork_context_intel, team->t.t_argc, 6998 #if OMPT_SUPPORT 6999 (void *)thr->th.th_teams_microtask, // "unwrapped" task 7000 #endif 7001 (microtask_t)thr->th.th_teams_microtask, // "wrapped" task 7002 VOLATILE_CAST(launch_t) __kmp_invoke_task_func, NULL); 7003 #if INCLUDE_SSC_MARKS 7004 SSC_MARK_JOINING(); 7005 #endif 7006 7007 // AC: last parameter "1" eliminates join barrier which won't work because 7008 // worker threads are in a fork barrier waiting for more parallel regions 7009 __kmp_join_call(loc, gtid 7010 #if OMPT_SUPPORT 7011 , 7012 fork_context_intel 7013 #endif 7014 , 7015 1); 7016 } 7017 7018 int __kmp_invoke_teams_master(int gtid) { 7019 kmp_info_t *this_thr = __kmp_threads[gtid]; 7020 kmp_team_t *team = this_thr->th.th_team; 7021 #if KMP_DEBUG 7022 if (!__kmp_threads[gtid]->th.th_team->t.t_serialized) 7023 KMP_DEBUG_ASSERT((void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn == 7024 (void *)__kmp_teams_master); 7025 #endif 7026 __kmp_run_before_invoked_task(gtid, 0, this_thr, team); 7027 __kmp_teams_master(gtid); 7028 __kmp_run_after_invoked_task(gtid, 0, this_thr, team); 7029 return 1; 7030 } 7031 #endif /* OMP_40_ENABLED */ 7032 7033 /* this sets the requested number of threads for the next parallel region 7034 encountered by this team. since this should be enclosed in the forkjoin 7035 critical section it should avoid race conditions with assymmetrical nested 7036 parallelism */ 7037 7038 void __kmp_push_num_threads(ident_t *id, int gtid, int num_threads) { 7039 kmp_info_t *thr = __kmp_threads[gtid]; 7040 7041 if (num_threads > 0) 7042 thr->th.th_set_nproc = num_threads; 7043 } 7044 7045 #if OMP_40_ENABLED 7046 7047 /* this sets the requested number of teams for the teams region and/or 7048 the number of threads for the next parallel region encountered */ 7049 void __kmp_push_num_teams(ident_t *id, int gtid, int num_teams, 7050 int num_threads) { 7051 kmp_info_t *thr = __kmp_threads[gtid]; 7052 KMP_DEBUG_ASSERT(num_teams >= 0); 7053 KMP_DEBUG_ASSERT(num_threads >= 0); 7054 7055 if (num_teams == 0) 7056 num_teams = 1; // default number of teams is 1. 7057 if (num_teams > __kmp_teams_max_nth) { // if too many teams requested? 7058 if (!__kmp_reserve_warn) { 7059 __kmp_reserve_warn = 1; 7060 __kmp_msg(kmp_ms_warning, 7061 KMP_MSG(CantFormThrTeam, num_teams, __kmp_teams_max_nth), 7062 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 7063 } 7064 num_teams = __kmp_teams_max_nth; 7065 } 7066 // Set number of teams (number of threads in the outer "parallel" of the 7067 // teams) 7068 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams; 7069 7070 // Remember the number of threads for inner parallel regions 7071 if (num_threads == 0) { 7072 if (!TCR_4(__kmp_init_middle)) 7073 __kmp_middle_initialize(); // get __kmp_avail_proc calculated 7074 num_threads = __kmp_avail_proc / num_teams; 7075 if (num_teams * num_threads > __kmp_teams_max_nth) { 7076 // adjust num_threads w/o warning as it is not user setting 7077 num_threads = __kmp_teams_max_nth / num_teams; 7078 } 7079 } else { 7080 if (num_teams * num_threads > __kmp_teams_max_nth) { 7081 int new_threads = __kmp_teams_max_nth / num_teams; 7082 if (!__kmp_reserve_warn) { // user asked for too many threads 7083 __kmp_reserve_warn = 1; // that conflicts with KMP_TEAMS_THREAD_LIMIT 7084 __kmp_msg(kmp_ms_warning, 7085 KMP_MSG(CantFormThrTeam, num_threads, new_threads), 7086 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 7087 } 7088 num_threads = new_threads; 7089 } 7090 } 7091 thr->th.th_teams_size.nth = num_threads; 7092 } 7093 7094 // Set the proc_bind var to use in the following parallel region. 7095 void __kmp_push_proc_bind(ident_t *id, int gtid, kmp_proc_bind_t proc_bind) { 7096 kmp_info_t *thr = __kmp_threads[gtid]; 7097 thr->th.th_set_proc_bind = proc_bind; 7098 } 7099 7100 #endif /* OMP_40_ENABLED */ 7101 7102 /* Launch the worker threads into the microtask. */ 7103 7104 void __kmp_internal_fork(ident_t *id, int gtid, kmp_team_t *team) { 7105 kmp_info_t *this_thr = __kmp_threads[gtid]; 7106 7107 #ifdef KMP_DEBUG 7108 int f; 7109 #endif /* KMP_DEBUG */ 7110 7111 KMP_DEBUG_ASSERT(team); 7112 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); 7113 KMP_ASSERT(KMP_MASTER_GTID(gtid)); 7114 KMP_MB(); /* Flush all pending memory write invalidates. */ 7115 7116 team->t.t_construct = 0; /* no single directives seen yet */ 7117 team->t.t_ordered.dt.t_value = 7118 0; /* thread 0 enters the ordered section first */ 7119 7120 /* Reset the identifiers on the dispatch buffer */ 7121 KMP_DEBUG_ASSERT(team->t.t_disp_buffer); 7122 if (team->t.t_max_nproc > 1) { 7123 int i; 7124 for (i = 0; i < __kmp_dispatch_num_buffers; ++i) { 7125 team->t.t_disp_buffer[i].buffer_index = i; 7126 #if OMP_45_ENABLED 7127 team->t.t_disp_buffer[i].doacross_buf_idx = i; 7128 #endif 7129 } 7130 } else { 7131 team->t.t_disp_buffer[0].buffer_index = 0; 7132 #if OMP_45_ENABLED 7133 team->t.t_disp_buffer[0].doacross_buf_idx = 0; 7134 #endif 7135 } 7136 7137 KMP_MB(); /* Flush all pending memory write invalidates. */ 7138 KMP_ASSERT(this_thr->th.th_team == team); 7139 7140 #ifdef KMP_DEBUG 7141 for (f = 0; f < team->t.t_nproc; f++) { 7142 KMP_DEBUG_ASSERT(team->t.t_threads[f] && 7143 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc); 7144 } 7145 #endif /* KMP_DEBUG */ 7146 7147 /* release the worker threads so they may begin working */ 7148 __kmp_fork_barrier(gtid, 0); 7149 } 7150 7151 void __kmp_internal_join(ident_t *id, int gtid, kmp_team_t *team) { 7152 kmp_info_t *this_thr = __kmp_threads[gtid]; 7153 7154 KMP_DEBUG_ASSERT(team); 7155 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); 7156 KMP_ASSERT(KMP_MASTER_GTID(gtid)); 7157 KMP_MB(); /* Flush all pending memory write invalidates. */ 7158 7159 /* Join barrier after fork */ 7160 7161 #ifdef KMP_DEBUG 7162 if (__kmp_threads[gtid] && 7163 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) { 7164 __kmp_printf("GTID: %d, __kmp_threads[%d]=%p\n", gtid, gtid, 7165 __kmp_threads[gtid]); 7166 __kmp_printf("__kmp_threads[%d]->th.th_team_nproc=%d, TEAM: %p, " 7167 "team->t.t_nproc=%d\n", 7168 gtid, __kmp_threads[gtid]->th.th_team_nproc, team, 7169 team->t.t_nproc); 7170 __kmp_print_structure(); 7171 } 7172 KMP_DEBUG_ASSERT(__kmp_threads[gtid] && 7173 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc); 7174 #endif /* KMP_DEBUG */ 7175 7176 __kmp_join_barrier(gtid); /* wait for everyone */ 7177 7178 KMP_MB(); /* Flush all pending memory write invalidates. */ 7179 KMP_ASSERT(this_thr->th.th_team == team); 7180 } 7181 7182 /* ------------------------------------------------------------------------ */ 7183 7184 #ifdef USE_LOAD_BALANCE 7185 7186 // Return the worker threads actively spinning in the hot team, if we 7187 // are at the outermost level of parallelism. Otherwise, return 0. 7188 static int __kmp_active_hot_team_nproc(kmp_root_t *root) { 7189 int i; 7190 int retval; 7191 kmp_team_t *hot_team; 7192 7193 if (root->r.r_active) { 7194 return 0; 7195 } 7196 hot_team = root->r.r_hot_team; 7197 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) { 7198 return hot_team->t.t_nproc - 1; // Don't count master thread 7199 } 7200 7201 // Skip the master thread - it is accounted for elsewhere. 7202 retval = 0; 7203 for (i = 1; i < hot_team->t.t_nproc; i++) { 7204 if (hot_team->t.t_threads[i]->th.th_active) { 7205 retval++; 7206 } 7207 } 7208 return retval; 7209 } 7210 7211 // Perform an automatic adjustment to the number of 7212 // threads used by the next parallel region. 7213 static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc) { 7214 int retval; 7215 int pool_active; 7216 int hot_team_active; 7217 int team_curr_active; 7218 int system_active; 7219 7220 KB_TRACE(20, ("__kmp_load_balance_nproc: called root:%p set_nproc:%d\n", root, 7221 set_nproc)); 7222 KMP_DEBUG_ASSERT(root); 7223 KMP_DEBUG_ASSERT(root->r.r_root_team->t.t_threads[0] 7224 ->th.th_current_task->td_icvs.dynamic == TRUE); 7225 KMP_DEBUG_ASSERT(set_nproc > 1); 7226 7227 if (set_nproc == 1) { 7228 KB_TRACE(20, ("__kmp_load_balance_nproc: serial execution.\n")); 7229 return 1; 7230 } 7231 7232 // Threads that are active in the thread pool, active in the hot team for this 7233 // particular root (if we are at the outer par level), and the currently 7234 // executing thread (to become the master) are available to add to the new 7235 // team, but are currently contributing to the system load, and must be 7236 // accounted for. 7237 pool_active = TCR_4(__kmp_thread_pool_active_nth); 7238 hot_team_active = __kmp_active_hot_team_nproc(root); 7239 team_curr_active = pool_active + hot_team_active + 1; 7240 7241 // Check the system load. 7242 system_active = __kmp_get_load_balance(__kmp_avail_proc + team_curr_active); 7243 KB_TRACE(30, ("__kmp_load_balance_nproc: system active = %d pool active = %d " 7244 "hot team active = %d\n", 7245 system_active, pool_active, hot_team_active)); 7246 7247 if (system_active < 0) { 7248 // There was an error reading the necessary info from /proc, so use the 7249 // thread limit algorithm instead. Once we set __kmp_global.g.g_dynamic_mode 7250 // = dynamic_thread_limit, we shouldn't wind up getting back here. 7251 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit; 7252 KMP_WARNING(CantLoadBalUsing, "KMP_DYNAMIC_MODE=thread limit"); 7253 7254 // Make this call behave like the thread limit algorithm. 7255 retval = __kmp_avail_proc - __kmp_nth + 7256 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc); 7257 if (retval > set_nproc) { 7258 retval = set_nproc; 7259 } 7260 if (retval < KMP_MIN_NTH) { 7261 retval = KMP_MIN_NTH; 7262 } 7263 7264 KB_TRACE(20, ("__kmp_load_balance_nproc: thread limit exit. retval:%d\n", 7265 retval)); 7266 return retval; 7267 } 7268 7269 // There is a slight delay in the load balance algorithm in detecting new 7270 // running procs. The real system load at this instant should be at least as 7271 // large as the #active omp thread that are available to add to the team. 7272 if (system_active < team_curr_active) { 7273 system_active = team_curr_active; 7274 } 7275 retval = __kmp_avail_proc - system_active + team_curr_active; 7276 if (retval > set_nproc) { 7277 retval = set_nproc; 7278 } 7279 if (retval < KMP_MIN_NTH) { 7280 retval = KMP_MIN_NTH; 7281 } 7282 7283 KB_TRACE(20, ("__kmp_load_balance_nproc: exit. retval:%d\n", retval)); 7284 return retval; 7285 } // __kmp_load_balance_nproc() 7286 7287 #endif /* USE_LOAD_BALANCE */ 7288 7289 /* ------------------------------------------------------------------------ */ 7290 7291 /* NOTE: this is called with the __kmp_init_lock held */ 7292 void __kmp_cleanup(void) { 7293 int f; 7294 7295 KA_TRACE(10, ("__kmp_cleanup: enter\n")); 7296 7297 if (TCR_4(__kmp_init_parallel)) { 7298 #if KMP_HANDLE_SIGNALS 7299 __kmp_remove_signals(); 7300 #endif 7301 TCW_4(__kmp_init_parallel, FALSE); 7302 } 7303 7304 if (TCR_4(__kmp_init_middle)) { 7305 #if KMP_AFFINITY_SUPPORTED 7306 __kmp_affinity_uninitialize(); 7307 #endif /* KMP_AFFINITY_SUPPORTED */ 7308 __kmp_cleanup_hierarchy(); 7309 TCW_4(__kmp_init_middle, FALSE); 7310 } 7311 7312 KA_TRACE(10, ("__kmp_cleanup: go serial cleanup\n")); 7313 7314 if (__kmp_init_serial) { 7315 __kmp_runtime_destroy(); 7316 __kmp_init_serial = FALSE; 7317 } 7318 7319 for (f = 0; f < __kmp_threads_capacity; f++) { 7320 if (__kmp_root[f] != NULL) { 7321 __kmp_free(__kmp_root[f]); 7322 __kmp_root[f] = NULL; 7323 } 7324 } 7325 __kmp_free(__kmp_threads); 7326 // __kmp_threads and __kmp_root were allocated at once, as single block, so 7327 // there is no need in freeing __kmp_root. 7328 __kmp_threads = NULL; 7329 __kmp_root = NULL; 7330 __kmp_threads_capacity = 0; 7331 7332 #if KMP_USE_DYNAMIC_LOCK 7333 __kmp_cleanup_indirect_user_locks(); 7334 #else 7335 __kmp_cleanup_user_locks(); 7336 #endif 7337 7338 #if KMP_AFFINITY_SUPPORTED 7339 KMP_INTERNAL_FREE(CCAST(char *, __kmp_cpuinfo_file)); 7340 __kmp_cpuinfo_file = NULL; 7341 #endif /* KMP_AFFINITY_SUPPORTED */ 7342 7343 #if KMP_USE_ADAPTIVE_LOCKS 7344 #if KMP_DEBUG_ADAPTIVE_LOCKS 7345 __kmp_print_speculative_stats(); 7346 #endif 7347 #endif 7348 KMP_INTERNAL_FREE(__kmp_nested_nth.nth); 7349 __kmp_nested_nth.nth = NULL; 7350 __kmp_nested_nth.size = 0; 7351 __kmp_nested_nth.used = 0; 7352 KMP_INTERNAL_FREE(__kmp_nested_proc_bind.bind_types); 7353 __kmp_nested_proc_bind.bind_types = NULL; 7354 __kmp_nested_proc_bind.size = 0; 7355 __kmp_nested_proc_bind.used = 0; 7356 7357 __kmp_i18n_catclose(); 7358 7359 #if KMP_STATS_ENABLED 7360 __kmp_stats_fini(); 7361 #endif 7362 7363 KA_TRACE(10, ("__kmp_cleanup: exit\n")); 7364 } 7365 7366 /* ------------------------------------------------------------------------ */ 7367 7368 int __kmp_ignore_mppbeg(void) { 7369 char *env; 7370 7371 if ((env = getenv("KMP_IGNORE_MPPBEG")) != NULL) { 7372 if (__kmp_str_match_false(env)) 7373 return FALSE; 7374 } 7375 // By default __kmpc_begin() is no-op. 7376 return TRUE; 7377 } 7378 7379 int __kmp_ignore_mppend(void) { 7380 char *env; 7381 7382 if ((env = getenv("KMP_IGNORE_MPPEND")) != NULL) { 7383 if (__kmp_str_match_false(env)) 7384 return FALSE; 7385 } 7386 // By default __kmpc_end() is no-op. 7387 return TRUE; 7388 } 7389 7390 void __kmp_internal_begin(void) { 7391 int gtid; 7392 kmp_root_t *root; 7393 7394 /* this is a very important step as it will register new sibling threads 7395 and assign these new uber threads a new gtid */ 7396 gtid = __kmp_entry_gtid(); 7397 root = __kmp_threads[gtid]->th.th_root; 7398 KMP_ASSERT(KMP_UBER_GTID(gtid)); 7399 7400 if (root->r.r_begin) 7401 return; 7402 __kmp_acquire_lock(&root->r.r_begin_lock, gtid); 7403 if (root->r.r_begin) { 7404 __kmp_release_lock(&root->r.r_begin_lock, gtid); 7405 return; 7406 } 7407 7408 root->r.r_begin = TRUE; 7409 7410 __kmp_release_lock(&root->r.r_begin_lock, gtid); 7411 } 7412 7413 /* ------------------------------------------------------------------------ */ 7414 7415 void __kmp_user_set_library(enum library_type arg) { 7416 int gtid; 7417 kmp_root_t *root; 7418 kmp_info_t *thread; 7419 7420 /* first, make sure we are initialized so we can get our gtid */ 7421 7422 gtid = __kmp_entry_gtid(); 7423 thread = __kmp_threads[gtid]; 7424 7425 root = thread->th.th_root; 7426 7427 KA_TRACE(20, ("__kmp_user_set_library: enter T#%d, arg: %d, %d\n", gtid, arg, 7428 library_serial)); 7429 if (root->r.r_in_parallel) { /* Must be called in serial section of top-level 7430 thread */ 7431 KMP_WARNING(SetLibraryIncorrectCall); 7432 return; 7433 } 7434 7435 switch (arg) { 7436 case library_serial: 7437 thread->th.th_set_nproc = 0; 7438 set__nproc(thread, 1); 7439 break; 7440 case library_turnaround: 7441 thread->th.th_set_nproc = 0; 7442 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth 7443 : __kmp_dflt_team_nth_ub); 7444 break; 7445 case library_throughput: 7446 thread->th.th_set_nproc = 0; 7447 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth 7448 : __kmp_dflt_team_nth_ub); 7449 break; 7450 default: 7451 KMP_FATAL(UnknownLibraryType, arg); 7452 } 7453 7454 __kmp_aux_set_library(arg); 7455 } 7456 7457 void __kmp_aux_set_stacksize(size_t arg) { 7458 if (!__kmp_init_serial) 7459 __kmp_serial_initialize(); 7460 7461 #if KMP_OS_DARWIN 7462 if (arg & (0x1000 - 1)) { 7463 arg &= ~(0x1000 - 1); 7464 if (arg + 0x1000) /* check for overflow if we round up */ 7465 arg += 0x1000; 7466 } 7467 #endif 7468 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 7469 7470 /* only change the default stacksize before the first parallel region */ 7471 if (!TCR_4(__kmp_init_parallel)) { 7472 size_t value = arg; /* argument is in bytes */ 7473 7474 if (value < __kmp_sys_min_stksize) 7475 value = __kmp_sys_min_stksize; 7476 else if (value > KMP_MAX_STKSIZE) 7477 value = KMP_MAX_STKSIZE; 7478 7479 __kmp_stksize = value; 7480 7481 __kmp_env_stksize = TRUE; /* was KMP_STACKSIZE specified? */ 7482 } 7483 7484 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 7485 } 7486 7487 /* set the behaviour of the runtime library */ 7488 /* TODO this can cause some odd behaviour with sibling parallelism... */ 7489 void __kmp_aux_set_library(enum library_type arg) { 7490 __kmp_library = arg; 7491 7492 switch (__kmp_library) { 7493 case library_serial: { 7494 KMP_INFORM(LibraryIsSerial); 7495 (void)__kmp_change_library(TRUE); 7496 } break; 7497 case library_turnaround: 7498 (void)__kmp_change_library(TRUE); 7499 break; 7500 case library_throughput: 7501 (void)__kmp_change_library(FALSE); 7502 break; 7503 default: 7504 KMP_FATAL(UnknownLibraryType, arg); 7505 } 7506 } 7507 7508 /* ------------------------------------------------------------------------ */ 7509 7510 void __kmp_aux_set_blocktime(int arg, kmp_info_t *thread, int tid) { 7511 int blocktime = arg; /* argument is in milliseconds */ 7512 #if KMP_USE_MONITOR 7513 int bt_intervals; 7514 #endif 7515 int bt_set; 7516 7517 __kmp_save_internal_controls(thread); 7518 7519 /* Normalize and set blocktime for the teams */ 7520 if (blocktime < KMP_MIN_BLOCKTIME) 7521 blocktime = KMP_MIN_BLOCKTIME; 7522 else if (blocktime > KMP_MAX_BLOCKTIME) 7523 blocktime = KMP_MAX_BLOCKTIME; 7524 7525 set__blocktime_team(thread->th.th_team, tid, blocktime); 7526 set__blocktime_team(thread->th.th_serial_team, 0, blocktime); 7527 7528 #if KMP_USE_MONITOR 7529 /* Calculate and set blocktime intervals for the teams */ 7530 bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME(blocktime, __kmp_monitor_wakeups); 7531 7532 set__bt_intervals_team(thread->th.th_team, tid, bt_intervals); 7533 set__bt_intervals_team(thread->th.th_serial_team, 0, bt_intervals); 7534 #endif 7535 7536 /* Set whether blocktime has been set to "TRUE" */ 7537 bt_set = TRUE; 7538 7539 set__bt_set_team(thread->th.th_team, tid, bt_set); 7540 set__bt_set_team(thread->th.th_serial_team, 0, bt_set); 7541 #if KMP_USE_MONITOR 7542 KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, " 7543 "bt_intervals=%d, monitor_updates=%d\n", 7544 __kmp_gtid_from_tid(tid, thread->th.th_team), 7545 thread->th.th_team->t.t_id, tid, blocktime, bt_intervals, 7546 __kmp_monitor_wakeups)); 7547 #else 7548 KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n", 7549 __kmp_gtid_from_tid(tid, thread->th.th_team), 7550 thread->th.th_team->t.t_id, tid, blocktime)); 7551 #endif 7552 } 7553 7554 void __kmp_aux_set_defaults(char const *str, int len) { 7555 if (!__kmp_init_serial) { 7556 __kmp_serial_initialize(); 7557 }; 7558 __kmp_env_initialize(str); 7559 7560 if (__kmp_settings 7561 #if OMP_40_ENABLED 7562 || __kmp_display_env || __kmp_display_env_verbose 7563 #endif // OMP_40_ENABLED 7564 ) { 7565 __kmp_env_print(); 7566 } 7567 } // __kmp_aux_set_defaults 7568 7569 /* ------------------------------------------------------------------------ */ 7570 /* internal fast reduction routines */ 7571 7572 PACKED_REDUCTION_METHOD_T 7573 __kmp_determine_reduction_method( 7574 ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, 7575 void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data), 7576 kmp_critical_name *lck) { 7577 7578 // Default reduction method: critical construct ( lck != NULL, like in current 7579 // PAROPT ) 7580 // If ( reduce_data!=NULL && reduce_func!=NULL ): the tree-reduction method 7581 // can be selected by RTL 7582 // If loc->flags contains KMP_IDENT_ATOMIC_REDUCE, the atomic reduce method 7583 // can be selected by RTL 7584 // Finally, it's up to OpenMP RTL to make a decision on which method to select 7585 // among generated by PAROPT. 7586 7587 PACKED_REDUCTION_METHOD_T retval; 7588 7589 int team_size; 7590 7591 KMP_DEBUG_ASSERT(loc); // it would be nice to test ( loc != 0 ) 7592 KMP_DEBUG_ASSERT(lck); // it would be nice to test ( lck != 0 ) 7593 7594 #define FAST_REDUCTION_ATOMIC_METHOD_GENERATED \ 7595 ((loc->flags & (KMP_IDENT_ATOMIC_REDUCE)) == (KMP_IDENT_ATOMIC_REDUCE)) 7596 #define FAST_REDUCTION_TREE_METHOD_GENERATED ((reduce_data) && (reduce_func)) 7597 7598 retval = critical_reduce_block; 7599 7600 // another choice of getting a team size (with 1 dynamic deference) is slower 7601 team_size = __kmp_get_team_num_threads(global_tid); 7602 if (team_size == 1) { 7603 7604 retval = empty_reduce_block; 7605 7606 } else { 7607 7608 int atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED; 7609 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED; 7610 7611 #if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS64 7612 7613 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_WINDOWS || \ 7614 KMP_OS_DARWIN 7615 7616 int teamsize_cutoff = 4; 7617 7618 #if KMP_MIC_SUPPORTED 7619 if (__kmp_mic_type != non_mic) { 7620 teamsize_cutoff = 8; 7621 } 7622 #endif 7623 if (tree_available) { 7624 if (team_size <= teamsize_cutoff) { 7625 if (atomic_available) { 7626 retval = atomic_reduce_block; 7627 } 7628 } else { 7629 retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER; 7630 } 7631 } else if (atomic_available) { 7632 retval = atomic_reduce_block; 7633 } 7634 #else 7635 #error "Unknown or unsupported OS" 7636 #endif // KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_WINDOWS || 7637 // KMP_OS_DARWIN 7638 7639 #elif KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS 7640 7641 #if KMP_OS_LINUX || KMP_OS_WINDOWS 7642 7643 // basic tuning 7644 7645 if (atomic_available) { 7646 if (num_vars <= 2) { // && ( team_size <= 8 ) due to false-sharing ??? 7647 retval = atomic_reduce_block; 7648 } 7649 } // otherwise: use critical section 7650 7651 #elif KMP_OS_DARWIN 7652 7653 if (atomic_available && (num_vars <= 3)) { 7654 retval = atomic_reduce_block; 7655 } else if (tree_available) { 7656 if ((reduce_size > (9 * sizeof(kmp_real64))) && 7657 (reduce_size < (2000 * sizeof(kmp_real64)))) { 7658 retval = TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER; 7659 } 7660 } // otherwise: use critical section 7661 7662 #else 7663 #error "Unknown or unsupported OS" 7664 #endif 7665 7666 #else 7667 #error "Unknown or unsupported architecture" 7668 #endif 7669 } 7670 7671 // KMP_FORCE_REDUCTION 7672 7673 // If the team is serialized (team_size == 1), ignore the forced reduction 7674 // method and stay with the unsynchronized method (empty_reduce_block) 7675 if (__kmp_force_reduction_method != reduction_method_not_defined && 7676 team_size != 1) { 7677 7678 PACKED_REDUCTION_METHOD_T forced_retval = critical_reduce_block; 7679 7680 int atomic_available, tree_available; 7681 7682 switch ((forced_retval = __kmp_force_reduction_method)) { 7683 case critical_reduce_block: 7684 KMP_ASSERT(lck); // lck should be != 0 7685 break; 7686 7687 case atomic_reduce_block: 7688 atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED; 7689 if (!atomic_available) { 7690 KMP_WARNING(RedMethodNotSupported, "atomic"); 7691 forced_retval = critical_reduce_block; 7692 } 7693 break; 7694 7695 case tree_reduce_block: 7696 tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED; 7697 if (!tree_available) { 7698 KMP_WARNING(RedMethodNotSupported, "tree"); 7699 forced_retval = critical_reduce_block; 7700 } else { 7701 #if KMP_FAST_REDUCTION_BARRIER 7702 forced_retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER; 7703 #endif 7704 } 7705 break; 7706 7707 default: 7708 KMP_ASSERT(0); // "unsupported method specified" 7709 } 7710 7711 retval = forced_retval; 7712 } 7713 7714 KA_TRACE(10, ("reduction method selected=%08x\n", retval)); 7715 7716 #undef FAST_REDUCTION_TREE_METHOD_GENERATED 7717 #undef FAST_REDUCTION_ATOMIC_METHOD_GENERATED 7718 7719 return (retval); 7720 } 7721 7722 // this function is for testing set/get/determine reduce method 7723 kmp_int32 __kmp_get_reduce_method(void) { 7724 return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8); 7725 } 7726