1 /* 2 * kmp_runtime.cpp -- KPTS runtime support library 3 */ 4 5 //===----------------------------------------------------------------------===// 6 // 7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 8 // See https://llvm.org/LICENSE.txt for license information. 9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "kmp.h" 14 #include "kmp_affinity.h" 15 #include "kmp_atomic.h" 16 #include "kmp_environment.h" 17 #include "kmp_error.h" 18 #include "kmp_i18n.h" 19 #include "kmp_io.h" 20 #include "kmp_itt.h" 21 #include "kmp_settings.h" 22 #include "kmp_stats.h" 23 #include "kmp_str.h" 24 #include "kmp_wait_release.h" 25 #include "kmp_wrapper_getpid.h" 26 #include "kmp_dispatch.h" 27 #if KMP_USE_HIER_SCHED 28 #include "kmp_dispatch_hier.h" 29 #endif 30 31 #if OMPT_SUPPORT 32 #include "ompt-specific.h" 33 #endif 34 #if OMPD_SUPPORT 35 #include "ompd-specific.h" 36 #endif 37 38 #if OMP_PROFILING_SUPPORT 39 #include "llvm/Support/TimeProfiler.h" 40 static char *ProfileTraceFile = nullptr; 41 #endif 42 43 /* these are temporary issues to be dealt with */ 44 #define KMP_USE_PRCTL 0 45 46 #if KMP_OS_WINDOWS 47 #include <process.h> 48 #endif 49 50 #if KMP_OS_WINDOWS 51 // windows does not need include files as it doesn't use shared memory 52 #else 53 #include <sys/mman.h> 54 #include <sys/stat.h> 55 #include <fcntl.h> 56 #define SHM_SIZE 1024 57 #endif 58 59 #if defined(KMP_GOMP_COMPAT) 60 char const __kmp_version_alt_comp[] = 61 KMP_VERSION_PREFIX "alternative compiler support: yes"; 62 #endif /* defined(KMP_GOMP_COMPAT) */ 63 64 char const __kmp_version_omp_api[] = 65 KMP_VERSION_PREFIX "API version: 5.0 (201611)"; 66 67 #ifdef KMP_DEBUG 68 char const __kmp_version_lock[] = 69 KMP_VERSION_PREFIX "lock type: run time selectable"; 70 #endif /* KMP_DEBUG */ 71 72 #define KMP_MIN(x, y) ((x) < (y) ? (x) : (y)) 73 74 /* ------------------------------------------------------------------------ */ 75 76 #if KMP_USE_MONITOR 77 kmp_info_t __kmp_monitor; 78 #endif 79 80 /* Forward declarations */ 81 82 void __kmp_cleanup(void); 83 84 static void __kmp_initialize_info(kmp_info_t *, kmp_team_t *, int tid, 85 int gtid); 86 static void __kmp_initialize_team(kmp_team_t *team, int new_nproc, 87 kmp_internal_control_t *new_icvs, 88 ident_t *loc); 89 #if KMP_AFFINITY_SUPPORTED 90 static void __kmp_partition_places(kmp_team_t *team, 91 int update_master_only = 0); 92 #endif 93 static void __kmp_do_serial_initialize(void); 94 void __kmp_fork_barrier(int gtid, int tid); 95 void __kmp_join_barrier(int gtid); 96 void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc, 97 kmp_internal_control_t *new_icvs, ident_t *loc); 98 99 #ifdef USE_LOAD_BALANCE 100 static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc); 101 #endif 102 103 static int __kmp_expand_threads(int nNeed); 104 #if KMP_OS_WINDOWS 105 static int __kmp_unregister_root_other_thread(int gtid); 106 #endif 107 static void __kmp_reap_thread(kmp_info_t *thread, int is_root); 108 kmp_info_t *__kmp_thread_pool_insert_pt = NULL; 109 110 void __kmp_resize_dist_barrier(kmp_team_t *team, int old_nthreads, 111 int new_nthreads); 112 void __kmp_add_threads_to_team(kmp_team_t *team, int new_nthreads); 113 114 /* Calculate the identifier of the current thread */ 115 /* fast (and somewhat portable) way to get unique identifier of executing 116 thread. Returns KMP_GTID_DNE if we haven't been assigned a gtid. */ 117 int __kmp_get_global_thread_id() { 118 int i; 119 kmp_info_t **other_threads; 120 size_t stack_data; 121 char *stack_addr; 122 size_t stack_size; 123 char *stack_base; 124 125 KA_TRACE( 126 1000, 127 ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n", 128 __kmp_nth, __kmp_all_nth)); 129 130 /* JPH - to handle the case where __kmpc_end(0) is called immediately prior to 131 a parallel region, made it return KMP_GTID_DNE to force serial_initialize 132 by caller. Had to handle KMP_GTID_DNE at all call-sites, or else guarantee 133 __kmp_init_gtid for this to work. */ 134 135 if (!TCR_4(__kmp_init_gtid)) 136 return KMP_GTID_DNE; 137 138 #ifdef KMP_TDATA_GTID 139 if (TCR_4(__kmp_gtid_mode) >= 3) { 140 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using TDATA\n")); 141 return __kmp_gtid; 142 } 143 #endif 144 if (TCR_4(__kmp_gtid_mode) >= 2) { 145 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using keyed TLS\n")); 146 return __kmp_gtid_get_specific(); 147 } 148 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using internal alg.\n")); 149 150 stack_addr = (char *)&stack_data; 151 other_threads = __kmp_threads; 152 153 /* ATT: The code below is a source of potential bugs due to unsynchronized 154 access to __kmp_threads array. For example: 155 1. Current thread loads other_threads[i] to thr and checks it, it is 156 non-NULL. 157 2. Current thread is suspended by OS. 158 3. Another thread unregisters and finishes (debug versions of free() 159 may fill memory with something like 0xEF). 160 4. Current thread is resumed. 161 5. Current thread reads junk from *thr. 162 TODO: Fix it. --ln */ 163 164 for (i = 0; i < __kmp_threads_capacity; i++) { 165 166 kmp_info_t *thr = (kmp_info_t *)TCR_SYNC_PTR(other_threads[i]); 167 if (!thr) 168 continue; 169 170 stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize); 171 stack_base = (char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase); 172 173 /* stack grows down -- search through all of the active threads */ 174 175 if (stack_addr <= stack_base) { 176 size_t stack_diff = stack_base - stack_addr; 177 178 if (stack_diff <= stack_size) { 179 /* The only way we can be closer than the allocated */ 180 /* stack size is if we are running on this thread. */ 181 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == i); 182 return i; 183 } 184 } 185 } 186 187 /* get specific to try and determine our gtid */ 188 KA_TRACE(1000, 189 ("*** __kmp_get_global_thread_id: internal alg. failed to find " 190 "thread, using TLS\n")); 191 i = __kmp_gtid_get_specific(); 192 193 /*fprintf( stderr, "=== %d\n", i ); */ /* GROO */ 194 195 /* if we havn't been assigned a gtid, then return code */ 196 if (i < 0) 197 return i; 198 199 /* dynamically updated stack window for uber threads to avoid get_specific 200 call */ 201 if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)) { 202 KMP_FATAL(StackOverflow, i); 203 } 204 205 stack_base = (char *)other_threads[i]->th.th_info.ds.ds_stackbase; 206 if (stack_addr > stack_base) { 207 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr); 208 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize, 209 other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr - 210 stack_base); 211 } else { 212 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize, 213 stack_base - stack_addr); 214 } 215 216 /* Reprint stack bounds for ubermaster since they have been refined */ 217 if (__kmp_storage_map) { 218 char *stack_end = (char *)other_threads[i]->th.th_info.ds.ds_stackbase; 219 char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize; 220 __kmp_print_storage_map_gtid(i, stack_beg, stack_end, 221 other_threads[i]->th.th_info.ds.ds_stacksize, 222 "th_%d stack (refinement)", i); 223 } 224 return i; 225 } 226 227 int __kmp_get_global_thread_id_reg() { 228 int gtid; 229 230 if (!__kmp_init_serial) { 231 gtid = KMP_GTID_DNE; 232 } else 233 #ifdef KMP_TDATA_GTID 234 if (TCR_4(__kmp_gtid_mode) >= 3) { 235 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using TDATA\n")); 236 gtid = __kmp_gtid; 237 } else 238 #endif 239 if (TCR_4(__kmp_gtid_mode) >= 2) { 240 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using keyed TLS\n")); 241 gtid = __kmp_gtid_get_specific(); 242 } else { 243 KA_TRACE(1000, 244 ("*** __kmp_get_global_thread_id_reg: using internal alg.\n")); 245 gtid = __kmp_get_global_thread_id(); 246 } 247 248 /* we must be a new uber master sibling thread */ 249 if (gtid == KMP_GTID_DNE) { 250 KA_TRACE(10, 251 ("__kmp_get_global_thread_id_reg: Encountered new root thread. " 252 "Registering a new gtid.\n")); 253 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 254 if (!__kmp_init_serial) { 255 __kmp_do_serial_initialize(); 256 gtid = __kmp_gtid_get_specific(); 257 } else { 258 gtid = __kmp_register_root(FALSE); 259 } 260 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 261 /*__kmp_printf( "+++ %d\n", gtid ); */ /* GROO */ 262 } 263 264 KMP_DEBUG_ASSERT(gtid >= 0); 265 266 return gtid; 267 } 268 269 /* caller must hold forkjoin_lock */ 270 void __kmp_check_stack_overlap(kmp_info_t *th) { 271 int f; 272 char *stack_beg = NULL; 273 char *stack_end = NULL; 274 int gtid; 275 276 KA_TRACE(10, ("__kmp_check_stack_overlap: called\n")); 277 if (__kmp_storage_map) { 278 stack_end = (char *)th->th.th_info.ds.ds_stackbase; 279 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize; 280 281 gtid = __kmp_gtid_from_thread(th); 282 283 if (gtid == KMP_GTID_MONITOR) { 284 __kmp_print_storage_map_gtid( 285 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize, 286 "th_%s stack (%s)", "mon", 287 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual"); 288 } else { 289 __kmp_print_storage_map_gtid( 290 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize, 291 "th_%d stack (%s)", gtid, 292 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual"); 293 } 294 } 295 296 /* No point in checking ubermaster threads since they use refinement and 297 * cannot overlap */ 298 gtid = __kmp_gtid_from_thread(th); 299 if (__kmp_env_checks == TRUE && !KMP_UBER_GTID(gtid)) { 300 KA_TRACE(10, 301 ("__kmp_check_stack_overlap: performing extensive checking\n")); 302 if (stack_beg == NULL) { 303 stack_end = (char *)th->th.th_info.ds.ds_stackbase; 304 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize; 305 } 306 307 for (f = 0; f < __kmp_threads_capacity; f++) { 308 kmp_info_t *f_th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[f]); 309 310 if (f_th && f_th != th) { 311 char *other_stack_end = 312 (char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase); 313 char *other_stack_beg = 314 other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize); 315 if ((stack_beg > other_stack_beg && stack_beg < other_stack_end) || 316 (stack_end > other_stack_beg && stack_end < other_stack_end)) { 317 318 /* Print the other stack values before the abort */ 319 if (__kmp_storage_map) 320 __kmp_print_storage_map_gtid( 321 -1, other_stack_beg, other_stack_end, 322 (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize), 323 "th_%d stack (overlapped)", __kmp_gtid_from_thread(f_th)); 324 325 __kmp_fatal(KMP_MSG(StackOverlap), KMP_HNT(ChangeStackLimit), 326 __kmp_msg_null); 327 } 328 } 329 } 330 } 331 KA_TRACE(10, ("__kmp_check_stack_overlap: returning\n")); 332 } 333 334 /* ------------------------------------------------------------------------ */ 335 336 void __kmp_infinite_loop(void) { 337 static int done = FALSE; 338 339 while (!done) { 340 KMP_YIELD(TRUE); 341 } 342 } 343 344 #define MAX_MESSAGE 512 345 346 void __kmp_print_storage_map_gtid(int gtid, void *p1, void *p2, size_t size, 347 char const *format, ...) { 348 char buffer[MAX_MESSAGE]; 349 va_list ap; 350 351 va_start(ap, format); 352 KMP_SNPRINTF(buffer, sizeof(buffer), "OMP storage map: %p %p%8lu %s\n", p1, 353 p2, (unsigned long)size, format); 354 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock); 355 __kmp_vprintf(kmp_err, buffer, ap); 356 #if KMP_PRINT_DATA_PLACEMENT 357 int node; 358 if (gtid >= 0) { 359 if (p1 <= p2 && (char *)p2 - (char *)p1 == size) { 360 if (__kmp_storage_map_verbose) { 361 node = __kmp_get_host_node(p1); 362 if (node < 0) /* doesn't work, so don't try this next time */ 363 __kmp_storage_map_verbose = FALSE; 364 else { 365 char *last; 366 int lastNode; 367 int localProc = __kmp_get_cpu_from_gtid(gtid); 368 369 const int page_size = KMP_GET_PAGE_SIZE(); 370 371 p1 = (void *)((size_t)p1 & ~((size_t)page_size - 1)); 372 p2 = (void *)(((size_t)p2 - 1) & ~((size_t)page_size - 1)); 373 if (localProc >= 0) 374 __kmp_printf_no_lock(" GTID %d localNode %d\n", gtid, 375 localProc >> 1); 376 else 377 __kmp_printf_no_lock(" GTID %d\n", gtid); 378 #if KMP_USE_PRCTL 379 /* The more elaborate format is disabled for now because of the prctl 380 * hanging bug. */ 381 do { 382 last = p1; 383 lastNode = node; 384 /* This loop collates adjacent pages with the same host node. */ 385 do { 386 (char *)p1 += page_size; 387 } while (p1 <= p2 && (node = __kmp_get_host_node(p1)) == lastNode); 388 __kmp_printf_no_lock(" %p-%p memNode %d\n", last, (char *)p1 - 1, 389 lastNode); 390 } while (p1 <= p2); 391 #else 392 __kmp_printf_no_lock(" %p-%p memNode %d\n", p1, 393 (char *)p1 + (page_size - 1), 394 __kmp_get_host_node(p1)); 395 if (p1 < p2) { 396 __kmp_printf_no_lock(" %p-%p memNode %d\n", p2, 397 (char *)p2 + (page_size - 1), 398 __kmp_get_host_node(p2)); 399 } 400 #endif 401 } 402 } 403 } else 404 __kmp_printf_no_lock(" %s\n", KMP_I18N_STR(StorageMapWarning)); 405 } 406 #endif /* KMP_PRINT_DATA_PLACEMENT */ 407 __kmp_release_bootstrap_lock(&__kmp_stdio_lock); 408 } 409 410 void __kmp_warn(char const *format, ...) { 411 char buffer[MAX_MESSAGE]; 412 va_list ap; 413 414 if (__kmp_generate_warnings == kmp_warnings_off) { 415 return; 416 } 417 418 va_start(ap, format); 419 420 KMP_SNPRINTF(buffer, sizeof(buffer), "OMP warning: %s\n", format); 421 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock); 422 __kmp_vprintf(kmp_err, buffer, ap); 423 __kmp_release_bootstrap_lock(&__kmp_stdio_lock); 424 425 va_end(ap); 426 } 427 428 void __kmp_abort_process() { 429 // Later threads may stall here, but that's ok because abort() will kill them. 430 __kmp_acquire_bootstrap_lock(&__kmp_exit_lock); 431 432 if (__kmp_debug_buf) { 433 __kmp_dump_debug_buffer(); 434 } 435 436 if (KMP_OS_WINDOWS) { 437 // Let other threads know of abnormal termination and prevent deadlock 438 // if abort happened during library initialization or shutdown 439 __kmp_global.g.g_abort = SIGABRT; 440 441 /* On Windows* OS by default abort() causes pop-up error box, which stalls 442 nightly testing. Unfortunately, we cannot reliably suppress pop-up error 443 boxes. _set_abort_behavior() works well, but this function is not 444 available in VS7 (this is not problem for DLL, but it is a problem for 445 static OpenMP RTL). SetErrorMode (and so, timelimit utility) does not 446 help, at least in some versions of MS C RTL. 447 448 It seems following sequence is the only way to simulate abort() and 449 avoid pop-up error box. */ 450 raise(SIGABRT); 451 _exit(3); // Just in case, if signal ignored, exit anyway. 452 } else { 453 __kmp_unregister_library(); 454 abort(); 455 } 456 457 __kmp_infinite_loop(); 458 __kmp_release_bootstrap_lock(&__kmp_exit_lock); 459 460 } // __kmp_abort_process 461 462 void __kmp_abort_thread(void) { 463 // TODO: Eliminate g_abort global variable and this function. 464 // In case of abort just call abort(), it will kill all the threads. 465 __kmp_infinite_loop(); 466 } // __kmp_abort_thread 467 468 /* Print out the storage map for the major kmp_info_t thread data structures 469 that are allocated together. */ 470 471 static void __kmp_print_thread_storage_map(kmp_info_t *thr, int gtid) { 472 __kmp_print_storage_map_gtid(gtid, thr, thr + 1, sizeof(kmp_info_t), "th_%d", 473 gtid); 474 475 __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team, 476 sizeof(kmp_desc_t), "th_%d.th_info", gtid); 477 478 __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head, 479 sizeof(kmp_local_t), "th_%d.th_local", gtid); 480 481 __kmp_print_storage_map_gtid( 482 gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier], 483 sizeof(kmp_balign_t) * bs_last_barrier, "th_%d.th_bar", gtid); 484 485 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier], 486 &thr->th.th_bar[bs_plain_barrier + 1], 487 sizeof(kmp_balign_t), "th_%d.th_bar[plain]", 488 gtid); 489 490 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier], 491 &thr->th.th_bar[bs_forkjoin_barrier + 1], 492 sizeof(kmp_balign_t), "th_%d.th_bar[forkjoin]", 493 gtid); 494 495 #if KMP_FAST_REDUCTION_BARRIER 496 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier], 497 &thr->th.th_bar[bs_reduction_barrier + 1], 498 sizeof(kmp_balign_t), "th_%d.th_bar[reduction]", 499 gtid); 500 #endif // KMP_FAST_REDUCTION_BARRIER 501 } 502 503 /* Print out the storage map for the major kmp_team_t team data structures 504 that are allocated together. */ 505 506 static void __kmp_print_team_storage_map(const char *header, kmp_team_t *team, 507 int team_id, int num_thr) { 508 int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2; 509 __kmp_print_storage_map_gtid(-1, team, team + 1, sizeof(kmp_team_t), "%s_%d", 510 header, team_id); 511 512 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0], 513 &team->t.t_bar[bs_last_barrier], 514 sizeof(kmp_balign_team_t) * bs_last_barrier, 515 "%s_%d.t_bar", header, team_id); 516 517 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier], 518 &team->t.t_bar[bs_plain_barrier + 1], 519 sizeof(kmp_balign_team_t), "%s_%d.t_bar[plain]", 520 header, team_id); 521 522 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier], 523 &team->t.t_bar[bs_forkjoin_barrier + 1], 524 sizeof(kmp_balign_team_t), 525 "%s_%d.t_bar[forkjoin]", header, team_id); 526 527 #if KMP_FAST_REDUCTION_BARRIER 528 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier], 529 &team->t.t_bar[bs_reduction_barrier + 1], 530 sizeof(kmp_balign_team_t), 531 "%s_%d.t_bar[reduction]", header, team_id); 532 #endif // KMP_FAST_REDUCTION_BARRIER 533 534 __kmp_print_storage_map_gtid( 535 -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr], 536 sizeof(kmp_disp_t) * num_thr, "%s_%d.t_dispatch", header, team_id); 537 538 __kmp_print_storage_map_gtid( 539 -1, &team->t.t_threads[0], &team->t.t_threads[num_thr], 540 sizeof(kmp_info_t *) * num_thr, "%s_%d.t_threads", header, team_id); 541 542 __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0], 543 &team->t.t_disp_buffer[num_disp_buff], 544 sizeof(dispatch_shared_info_t) * num_disp_buff, 545 "%s_%d.t_disp_buffer", header, team_id); 546 } 547 548 static void __kmp_init_allocator() { 549 __kmp_init_memkind(); 550 __kmp_init_target_mem(); 551 } 552 static void __kmp_fini_allocator() { __kmp_fini_memkind(); } 553 554 /* ------------------------------------------------------------------------ */ 555 556 #if KMP_DYNAMIC_LIB 557 #if KMP_OS_WINDOWS 558 559 BOOL WINAPI DllMain(HINSTANCE hInstDLL, DWORD fdwReason, LPVOID lpReserved) { 560 //__kmp_acquire_bootstrap_lock( &__kmp_initz_lock ); 561 562 switch (fdwReason) { 563 564 case DLL_PROCESS_ATTACH: 565 KA_TRACE(10, ("DllMain: PROCESS_ATTACH\n")); 566 567 return TRUE; 568 569 case DLL_PROCESS_DETACH: 570 KA_TRACE(10, ("DllMain: PROCESS_DETACH T#%d\n", __kmp_gtid_get_specific())); 571 572 // According to Windows* documentation for DllMain entry point: 573 // for DLL_PROCESS_DETACH, lpReserved is used for telling the difference: 574 // lpReserved == NULL when FreeLibrary() is called, 575 // lpReserved != NULL when the process is terminated. 576 // When FreeLibrary() is called, worker threads remain alive. So the 577 // runtime's state is consistent and executing proper shutdown is OK. 578 // When the process is terminated, worker threads have exited or been 579 // forcefully terminated by the OS and only the shutdown thread remains. 580 // This can leave the runtime in an inconsistent state. 581 // Hence, only attempt proper cleanup when FreeLibrary() is called. 582 // Otherwise, rely on OS to reclaim resources. 583 if (lpReserved == NULL) 584 __kmp_internal_end_library(__kmp_gtid_get_specific()); 585 586 return TRUE; 587 588 case DLL_THREAD_ATTACH: 589 KA_TRACE(10, ("DllMain: THREAD_ATTACH\n")); 590 591 /* if we want to register new siblings all the time here call 592 * __kmp_get_gtid(); */ 593 return TRUE; 594 595 case DLL_THREAD_DETACH: 596 KA_TRACE(10, ("DllMain: THREAD_DETACH T#%d\n", __kmp_gtid_get_specific())); 597 598 __kmp_internal_end_thread(__kmp_gtid_get_specific()); 599 return TRUE; 600 } 601 602 return TRUE; 603 } 604 605 #endif /* KMP_OS_WINDOWS */ 606 #endif /* KMP_DYNAMIC_LIB */ 607 608 /* __kmp_parallel_deo -- Wait until it's our turn. */ 609 void __kmp_parallel_deo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) { 610 int gtid = *gtid_ref; 611 #ifdef BUILD_PARALLEL_ORDERED 612 kmp_team_t *team = __kmp_team_from_gtid(gtid); 613 #endif /* BUILD_PARALLEL_ORDERED */ 614 615 if (__kmp_env_consistency_check) { 616 if (__kmp_threads[gtid]->th.th_root->r.r_active) 617 #if KMP_USE_DYNAMIC_LOCK 618 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL, 0); 619 #else 620 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL); 621 #endif 622 } 623 #ifdef BUILD_PARALLEL_ORDERED 624 if (!team->t.t_serialized) { 625 KMP_MB(); 626 KMP_WAIT(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid), KMP_EQ, 627 NULL); 628 KMP_MB(); 629 } 630 #endif /* BUILD_PARALLEL_ORDERED */ 631 } 632 633 /* __kmp_parallel_dxo -- Signal the next task. */ 634 void __kmp_parallel_dxo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) { 635 int gtid = *gtid_ref; 636 #ifdef BUILD_PARALLEL_ORDERED 637 int tid = __kmp_tid_from_gtid(gtid); 638 kmp_team_t *team = __kmp_team_from_gtid(gtid); 639 #endif /* BUILD_PARALLEL_ORDERED */ 640 641 if (__kmp_env_consistency_check) { 642 if (__kmp_threads[gtid]->th.th_root->r.r_active) 643 __kmp_pop_sync(gtid, ct_ordered_in_parallel, loc_ref); 644 } 645 #ifdef BUILD_PARALLEL_ORDERED 646 if (!team->t.t_serialized) { 647 KMP_MB(); /* Flush all pending memory write invalidates. */ 648 649 /* use the tid of the next thread in this team */ 650 /* TODO replace with general release procedure */ 651 team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc); 652 653 KMP_MB(); /* Flush all pending memory write invalidates. */ 654 } 655 #endif /* BUILD_PARALLEL_ORDERED */ 656 } 657 658 /* ------------------------------------------------------------------------ */ 659 /* The BARRIER for a SINGLE process section is always explicit */ 660 661 int __kmp_enter_single(int gtid, ident_t *id_ref, int push_ws) { 662 int status; 663 kmp_info_t *th; 664 kmp_team_t *team; 665 666 if (!TCR_4(__kmp_init_parallel)) 667 __kmp_parallel_initialize(); 668 __kmp_resume_if_soft_paused(); 669 670 th = __kmp_threads[gtid]; 671 team = th->th.th_team; 672 status = 0; 673 674 th->th.th_ident = id_ref; 675 676 if (team->t.t_serialized) { 677 status = 1; 678 } else { 679 kmp_int32 old_this = th->th.th_local.this_construct; 680 681 ++th->th.th_local.this_construct; 682 /* try to set team count to thread count--success means thread got the 683 single block */ 684 /* TODO: Should this be acquire or release? */ 685 if (team->t.t_construct == old_this) { 686 status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this, 687 th->th.th_local.this_construct); 688 } 689 #if USE_ITT_BUILD 690 if (__itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 && 691 KMP_MASTER_GTID(gtid) && th->th.th_teams_microtask == NULL && 692 team->t.t_active_level == 1) { 693 // Only report metadata by primary thread of active team at level 1 694 __kmp_itt_metadata_single(id_ref); 695 } 696 #endif /* USE_ITT_BUILD */ 697 } 698 699 if (__kmp_env_consistency_check) { 700 if (status && push_ws) { 701 __kmp_push_workshare(gtid, ct_psingle, id_ref); 702 } else { 703 __kmp_check_workshare(gtid, ct_psingle, id_ref); 704 } 705 } 706 #if USE_ITT_BUILD 707 if (status) { 708 __kmp_itt_single_start(gtid); 709 } 710 #endif /* USE_ITT_BUILD */ 711 return status; 712 } 713 714 void __kmp_exit_single(int gtid) { 715 #if USE_ITT_BUILD 716 __kmp_itt_single_end(gtid); 717 #endif /* USE_ITT_BUILD */ 718 if (__kmp_env_consistency_check) 719 __kmp_pop_workshare(gtid, ct_psingle, NULL); 720 } 721 722 /* determine if we can go parallel or must use a serialized parallel region and 723 * how many threads we can use 724 * set_nproc is the number of threads requested for the team 725 * returns 0 if we should serialize or only use one thread, 726 * otherwise the number of threads to use 727 * The forkjoin lock is held by the caller. */ 728 static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team, 729 int master_tid, int set_nthreads, 730 int enter_teams) { 731 int capacity; 732 int new_nthreads; 733 KMP_DEBUG_ASSERT(__kmp_init_serial); 734 KMP_DEBUG_ASSERT(root && parent_team); 735 kmp_info_t *this_thr = parent_team->t.t_threads[master_tid]; 736 737 // If dyn-var is set, dynamically adjust the number of desired threads, 738 // according to the method specified by dynamic_mode. 739 new_nthreads = set_nthreads; 740 if (!get__dynamic_2(parent_team, master_tid)) { 741 ; 742 } 743 #ifdef USE_LOAD_BALANCE 744 else if (__kmp_global.g.g_dynamic_mode == dynamic_load_balance) { 745 new_nthreads = __kmp_load_balance_nproc(root, set_nthreads); 746 if (new_nthreads == 1) { 747 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced " 748 "reservation to 1 thread\n", 749 master_tid)); 750 return 1; 751 } 752 if (new_nthreads < set_nthreads) { 753 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced " 754 "reservation to %d threads\n", 755 master_tid, new_nthreads)); 756 } 757 } 758 #endif /* USE_LOAD_BALANCE */ 759 else if (__kmp_global.g.g_dynamic_mode == dynamic_thread_limit) { 760 new_nthreads = __kmp_avail_proc - __kmp_nth + 761 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc); 762 if (new_nthreads <= 1) { 763 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced " 764 "reservation to 1 thread\n", 765 master_tid)); 766 return 1; 767 } 768 if (new_nthreads < set_nthreads) { 769 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced " 770 "reservation to %d threads\n", 771 master_tid, new_nthreads)); 772 } else { 773 new_nthreads = set_nthreads; 774 } 775 } else if (__kmp_global.g.g_dynamic_mode == dynamic_random) { 776 if (set_nthreads > 2) { 777 new_nthreads = __kmp_get_random(parent_team->t.t_threads[master_tid]); 778 new_nthreads = (new_nthreads % set_nthreads) + 1; 779 if (new_nthreads == 1) { 780 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced " 781 "reservation to 1 thread\n", 782 master_tid)); 783 return 1; 784 } 785 if (new_nthreads < set_nthreads) { 786 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced " 787 "reservation to %d threads\n", 788 master_tid, new_nthreads)); 789 } 790 } 791 } else { 792 KMP_ASSERT(0); 793 } 794 795 // Respect KMP_ALL_THREADS/KMP_DEVICE_THREAD_LIMIT. 796 if (__kmp_nth + new_nthreads - 797 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) > 798 __kmp_max_nth) { 799 int tl_nthreads = __kmp_max_nth - __kmp_nth + 800 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc); 801 if (tl_nthreads <= 0) { 802 tl_nthreads = 1; 803 } 804 805 // If dyn-var is false, emit a 1-time warning. 806 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) { 807 __kmp_reserve_warn = 1; 808 __kmp_msg(kmp_ms_warning, 809 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads), 810 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 811 } 812 if (tl_nthreads == 1) { 813 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT " 814 "reduced reservation to 1 thread\n", 815 master_tid)); 816 return 1; 817 } 818 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced " 819 "reservation to %d threads\n", 820 master_tid, tl_nthreads)); 821 new_nthreads = tl_nthreads; 822 } 823 824 // Respect OMP_THREAD_LIMIT 825 int cg_nthreads = this_thr->th.th_cg_roots->cg_nthreads; 826 int max_cg_threads = this_thr->th.th_cg_roots->cg_thread_limit; 827 if (cg_nthreads + new_nthreads - 828 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) > 829 max_cg_threads) { 830 int tl_nthreads = max_cg_threads - cg_nthreads + 831 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc); 832 if (tl_nthreads <= 0) { 833 tl_nthreads = 1; 834 } 835 836 // If dyn-var is false, emit a 1-time warning. 837 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) { 838 __kmp_reserve_warn = 1; 839 __kmp_msg(kmp_ms_warning, 840 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads), 841 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 842 } 843 if (tl_nthreads == 1) { 844 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT " 845 "reduced reservation to 1 thread\n", 846 master_tid)); 847 return 1; 848 } 849 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced " 850 "reservation to %d threads\n", 851 master_tid, tl_nthreads)); 852 new_nthreads = tl_nthreads; 853 } 854 855 // Check if the threads array is large enough, or needs expanding. 856 // See comment in __kmp_register_root() about the adjustment if 857 // __kmp_threads[0] == NULL. 858 capacity = __kmp_threads_capacity; 859 if (TCR_PTR(__kmp_threads[0]) == NULL) { 860 --capacity; 861 } 862 // If it is not for initializing the hidden helper team, we need to take 863 // __kmp_hidden_helper_threads_num out of the capacity because it is included 864 // in __kmp_threads_capacity. 865 if (__kmp_enable_hidden_helper && !TCR_4(__kmp_init_hidden_helper_threads)) { 866 capacity -= __kmp_hidden_helper_threads_num; 867 } 868 if (__kmp_nth + new_nthreads - 869 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) > 870 capacity) { 871 // Expand the threads array. 872 int slotsRequired = __kmp_nth + new_nthreads - 873 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) - 874 capacity; 875 int slotsAdded = __kmp_expand_threads(slotsRequired); 876 if (slotsAdded < slotsRequired) { 877 // The threads array was not expanded enough. 878 new_nthreads -= (slotsRequired - slotsAdded); 879 KMP_ASSERT(new_nthreads >= 1); 880 881 // If dyn-var is false, emit a 1-time warning. 882 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) { 883 __kmp_reserve_warn = 1; 884 if (__kmp_tp_cached) { 885 __kmp_msg(kmp_ms_warning, 886 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads), 887 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity), 888 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null); 889 } else { 890 __kmp_msg(kmp_ms_warning, 891 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads), 892 KMP_HNT(SystemLimitOnThreads), __kmp_msg_null); 893 } 894 } 895 } 896 } 897 898 #ifdef KMP_DEBUG 899 if (new_nthreads == 1) { 900 KC_TRACE(10, 901 ("__kmp_reserve_threads: T#%d serializing team after reclaiming " 902 "dead roots and rechecking; requested %d threads\n", 903 __kmp_get_gtid(), set_nthreads)); 904 } else { 905 KC_TRACE(10, ("__kmp_reserve_threads: T#%d allocating %d threads; requested" 906 " %d threads\n", 907 __kmp_get_gtid(), new_nthreads, set_nthreads)); 908 } 909 #endif // KMP_DEBUG 910 return new_nthreads; 911 } 912 913 /* Allocate threads from the thread pool and assign them to the new team. We are 914 assured that there are enough threads available, because we checked on that 915 earlier within critical section forkjoin */ 916 static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team, 917 kmp_info_t *master_th, int master_gtid) { 918 int i; 919 int use_hot_team; 920 921 KA_TRACE(10, ("__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc)); 922 KMP_DEBUG_ASSERT(master_gtid == __kmp_get_gtid()); 923 KMP_MB(); 924 925 /* first, let's setup the primary thread */ 926 master_th->th.th_info.ds.ds_tid = 0; 927 master_th->th.th_team = team; 928 master_th->th.th_team_nproc = team->t.t_nproc; 929 master_th->th.th_team_master = master_th; 930 master_th->th.th_team_serialized = FALSE; 931 master_th->th.th_dispatch = &team->t.t_dispatch[0]; 932 933 /* make sure we are not the optimized hot team */ 934 #if KMP_NESTED_HOT_TEAMS 935 use_hot_team = 0; 936 kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams; 937 if (hot_teams) { // hot teams array is not allocated if 938 // KMP_HOT_TEAMS_MAX_LEVEL=0 939 int level = team->t.t_active_level - 1; // index in array of hot teams 940 if (master_th->th.th_teams_microtask) { // are we inside the teams? 941 if (master_th->th.th_teams_size.nteams > 1) { 942 ++level; // level was not increased in teams construct for 943 // team_of_masters 944 } 945 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master && 946 master_th->th.th_teams_level == team->t.t_level) { 947 ++level; // level was not increased in teams construct for 948 // team_of_workers before the parallel 949 } // team->t.t_level will be increased inside parallel 950 } 951 if (level < __kmp_hot_teams_max_level) { 952 if (hot_teams[level].hot_team) { 953 // hot team has already been allocated for given level 954 KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team); 955 use_hot_team = 1; // the team is ready to use 956 } else { 957 use_hot_team = 0; // AC: threads are not allocated yet 958 hot_teams[level].hot_team = team; // remember new hot team 959 hot_teams[level].hot_team_nth = team->t.t_nproc; 960 } 961 } else { 962 use_hot_team = 0; 963 } 964 } 965 #else 966 use_hot_team = team == root->r.r_hot_team; 967 #endif 968 if (!use_hot_team) { 969 970 /* install the primary thread */ 971 team->t.t_threads[0] = master_th; 972 __kmp_initialize_info(master_th, team, 0, master_gtid); 973 974 /* now, install the worker threads */ 975 for (i = 1; i < team->t.t_nproc; i++) { 976 977 /* fork or reallocate a new thread and install it in team */ 978 kmp_info_t *thr = __kmp_allocate_thread(root, team, i); 979 team->t.t_threads[i] = thr; 980 KMP_DEBUG_ASSERT(thr); 981 KMP_DEBUG_ASSERT(thr->th.th_team == team); 982 /* align team and thread arrived states */ 983 KA_TRACE(20, ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived " 984 "T#%d(%d:%d) join =%llu, plain=%llu\n", 985 __kmp_gtid_from_tid(0, team), team->t.t_id, 0, 986 __kmp_gtid_from_tid(i, team), team->t.t_id, i, 987 team->t.t_bar[bs_forkjoin_barrier].b_arrived, 988 team->t.t_bar[bs_plain_barrier].b_arrived)); 989 thr->th.th_teams_microtask = master_th->th.th_teams_microtask; 990 thr->th.th_teams_level = master_th->th.th_teams_level; 991 thr->th.th_teams_size = master_th->th.th_teams_size; 992 { // Initialize threads' barrier data. 993 int b; 994 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar; 995 for (b = 0; b < bs_last_barrier; ++b) { 996 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 997 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 998 #if USE_DEBUGGER 999 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 1000 #endif 1001 } 1002 } 1003 } 1004 1005 #if KMP_AFFINITY_SUPPORTED 1006 __kmp_partition_places(team); 1007 #endif 1008 } 1009 1010 if (__kmp_display_affinity && team->t.t_display_affinity != 1) { 1011 for (i = 0; i < team->t.t_nproc; i++) { 1012 kmp_info_t *thr = team->t.t_threads[i]; 1013 if (thr->th.th_prev_num_threads != team->t.t_nproc || 1014 thr->th.th_prev_level != team->t.t_level) { 1015 team->t.t_display_affinity = 1; 1016 break; 1017 } 1018 } 1019 } 1020 1021 KMP_MB(); 1022 } 1023 1024 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1025 // Propagate any changes to the floating point control registers out to the team 1026 // We try to avoid unnecessary writes to the relevant cache line in the team 1027 // structure, so we don't make changes unless they are needed. 1028 inline static void propagateFPControl(kmp_team_t *team) { 1029 if (__kmp_inherit_fp_control) { 1030 kmp_int16 x87_fpu_control_word; 1031 kmp_uint32 mxcsr; 1032 1033 // Get primary thread's values of FPU control flags (both X87 and vector) 1034 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word); 1035 __kmp_store_mxcsr(&mxcsr); 1036 mxcsr &= KMP_X86_MXCSR_MASK; 1037 1038 // There is no point looking at t_fp_control_saved here. 1039 // If it is TRUE, we still have to update the values if they are different 1040 // from those we now have. If it is FALSE we didn't save anything yet, but 1041 // our objective is the same. We have to ensure that the values in the team 1042 // are the same as those we have. 1043 // So, this code achieves what we need whether or not t_fp_control_saved is 1044 // true. By checking whether the value needs updating we avoid unnecessary 1045 // writes that would put the cache-line into a written state, causing all 1046 // threads in the team to have to read it again. 1047 KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word); 1048 KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr); 1049 // Although we don't use this value, other code in the runtime wants to know 1050 // whether it should restore them. So we must ensure it is correct. 1051 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE); 1052 } else { 1053 // Similarly here. Don't write to this cache-line in the team structure 1054 // unless we have to. 1055 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE); 1056 } 1057 } 1058 1059 // Do the opposite, setting the hardware registers to the updated values from 1060 // the team. 1061 inline static void updateHWFPControl(kmp_team_t *team) { 1062 if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) { 1063 // Only reset the fp control regs if they have been changed in the team. 1064 // the parallel region that we are exiting. 1065 kmp_int16 x87_fpu_control_word; 1066 kmp_uint32 mxcsr; 1067 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word); 1068 __kmp_store_mxcsr(&mxcsr); 1069 mxcsr &= KMP_X86_MXCSR_MASK; 1070 1071 if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) { 1072 __kmp_clear_x87_fpu_status_word(); 1073 __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word); 1074 } 1075 1076 if (team->t.t_mxcsr != mxcsr) { 1077 __kmp_load_mxcsr(&team->t.t_mxcsr); 1078 } 1079 } 1080 } 1081 #else 1082 #define propagateFPControl(x) ((void)0) 1083 #define updateHWFPControl(x) ((void)0) 1084 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 1085 1086 static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team, 1087 int realloc); // forward declaration 1088 1089 /* Run a parallel region that has been serialized, so runs only in a team of the 1090 single primary thread. */ 1091 void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) { 1092 kmp_info_t *this_thr; 1093 kmp_team_t *serial_team; 1094 1095 KC_TRACE(10, ("__kmpc_serialized_parallel: called by T#%d\n", global_tid)); 1096 1097 /* Skip all this code for autopar serialized loops since it results in 1098 unacceptable overhead */ 1099 if (loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR)) 1100 return; 1101 1102 if (!TCR_4(__kmp_init_parallel)) 1103 __kmp_parallel_initialize(); 1104 __kmp_resume_if_soft_paused(); 1105 1106 this_thr = __kmp_threads[global_tid]; 1107 serial_team = this_thr->th.th_serial_team; 1108 1109 /* utilize the serialized team held by this thread */ 1110 KMP_DEBUG_ASSERT(serial_team); 1111 KMP_MB(); 1112 1113 if (__kmp_tasking_mode != tskm_immediate_exec) { 1114 KMP_DEBUG_ASSERT( 1115 this_thr->th.th_task_team == 1116 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]); 1117 KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] == 1118 NULL); 1119 KA_TRACE(20, ("__kmpc_serialized_parallel: T#%d pushing task_team %p / " 1120 "team %p, new task_team = NULL\n", 1121 global_tid, this_thr->th.th_task_team, this_thr->th.th_team)); 1122 this_thr->th.th_task_team = NULL; 1123 } 1124 1125 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind; 1126 if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) { 1127 proc_bind = proc_bind_false; 1128 } else if (proc_bind == proc_bind_default) { 1129 // No proc_bind clause was specified, so use the current value 1130 // of proc-bind-var for this parallel region. 1131 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind; 1132 } 1133 // Reset for next parallel region 1134 this_thr->th.th_set_proc_bind = proc_bind_default; 1135 1136 #if OMPT_SUPPORT 1137 ompt_data_t ompt_parallel_data = ompt_data_none; 1138 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid); 1139 if (ompt_enabled.enabled && 1140 this_thr->th.ompt_thread_info.state != ompt_state_overhead) { 1141 1142 ompt_task_info_t *parent_task_info; 1143 parent_task_info = OMPT_CUR_TASK_INFO(this_thr); 1144 1145 parent_task_info->frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1146 if (ompt_enabled.ompt_callback_parallel_begin) { 1147 int team_size = 1; 1148 1149 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)( 1150 &(parent_task_info->task_data), &(parent_task_info->frame), 1151 &ompt_parallel_data, team_size, 1152 ompt_parallel_invoker_program | ompt_parallel_team, codeptr); 1153 } 1154 } 1155 #endif // OMPT_SUPPORT 1156 1157 if (this_thr->th.th_team != serial_team) { 1158 // Nested level will be an index in the nested nthreads array 1159 int level = this_thr->th.th_team->t.t_level; 1160 1161 if (serial_team->t.t_serialized) { 1162 /* this serial team was already used 1163 TODO increase performance by making this locks more specific */ 1164 kmp_team_t *new_team; 1165 1166 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 1167 1168 new_team = 1169 __kmp_allocate_team(this_thr->th.th_root, 1, 1, 1170 #if OMPT_SUPPORT 1171 ompt_parallel_data, 1172 #endif 1173 proc_bind, &this_thr->th.th_current_task->td_icvs, 1174 0 USE_NESTED_HOT_ARG(NULL)); 1175 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 1176 KMP_ASSERT(new_team); 1177 1178 /* setup new serialized team and install it */ 1179 new_team->t.t_threads[0] = this_thr; 1180 new_team->t.t_parent = this_thr->th.th_team; 1181 serial_team = new_team; 1182 this_thr->th.th_serial_team = serial_team; 1183 1184 KF_TRACE( 1185 10, 1186 ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n", 1187 global_tid, serial_team)); 1188 1189 /* TODO the above breaks the requirement that if we run out of resources, 1190 then we can still guarantee that serialized teams are ok, since we may 1191 need to allocate a new one */ 1192 } else { 1193 KF_TRACE( 1194 10, 1195 ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n", 1196 global_tid, serial_team)); 1197 } 1198 1199 /* we have to initialize this serial team */ 1200 KMP_DEBUG_ASSERT(serial_team->t.t_threads); 1201 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr); 1202 KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team); 1203 serial_team->t.t_ident = loc; 1204 serial_team->t.t_serialized = 1; 1205 serial_team->t.t_nproc = 1; 1206 serial_team->t.t_parent = this_thr->th.th_team; 1207 serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched; 1208 this_thr->th.th_team = serial_team; 1209 serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid; 1210 1211 KF_TRACE(10, ("__kmpc_serialized_parallel: T#%d curtask=%p\n", global_tid, 1212 this_thr->th.th_current_task)); 1213 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1); 1214 this_thr->th.th_current_task->td_flags.executing = 0; 1215 1216 __kmp_push_current_task_to_thread(this_thr, serial_team, 0); 1217 1218 /* TODO: GEH: do ICVs work for nested serialized teams? Don't we need an 1219 implicit task for each serialized task represented by 1220 team->t.t_serialized? */ 1221 copy_icvs(&this_thr->th.th_current_task->td_icvs, 1222 &this_thr->th.th_current_task->td_parent->td_icvs); 1223 1224 // Thread value exists in the nested nthreads array for the next nested 1225 // level 1226 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) { 1227 this_thr->th.th_current_task->td_icvs.nproc = 1228 __kmp_nested_nth.nth[level + 1]; 1229 } 1230 1231 if (__kmp_nested_proc_bind.used && 1232 (level + 1 < __kmp_nested_proc_bind.used)) { 1233 this_thr->th.th_current_task->td_icvs.proc_bind = 1234 __kmp_nested_proc_bind.bind_types[level + 1]; 1235 } 1236 1237 #if USE_DEBUGGER 1238 serial_team->t.t_pkfn = (microtask_t)(~0); // For the debugger. 1239 #endif 1240 this_thr->th.th_info.ds.ds_tid = 0; 1241 1242 /* set thread cache values */ 1243 this_thr->th.th_team_nproc = 1; 1244 this_thr->th.th_team_master = this_thr; 1245 this_thr->th.th_team_serialized = 1; 1246 1247 serial_team->t.t_level = serial_team->t.t_parent->t.t_level + 1; 1248 serial_team->t.t_active_level = serial_team->t.t_parent->t.t_active_level; 1249 serial_team->t.t_def_allocator = this_thr->th.th_def_allocator; // save 1250 1251 propagateFPControl(serial_team); 1252 1253 /* check if we need to allocate dispatch buffers stack */ 1254 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch); 1255 if (!serial_team->t.t_dispatch->th_disp_buffer) { 1256 serial_team->t.t_dispatch->th_disp_buffer = 1257 (dispatch_private_info_t *)__kmp_allocate( 1258 sizeof(dispatch_private_info_t)); 1259 } 1260 this_thr->th.th_dispatch = serial_team->t.t_dispatch; 1261 1262 KMP_MB(); 1263 1264 } else { 1265 /* this serialized team is already being used, 1266 * that's fine, just add another nested level */ 1267 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team); 1268 KMP_DEBUG_ASSERT(serial_team->t.t_threads); 1269 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr); 1270 ++serial_team->t.t_serialized; 1271 this_thr->th.th_team_serialized = serial_team->t.t_serialized; 1272 1273 // Nested level will be an index in the nested nthreads array 1274 int level = this_thr->th.th_team->t.t_level; 1275 // Thread value exists in the nested nthreads array for the next nested 1276 // level 1277 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) { 1278 this_thr->th.th_current_task->td_icvs.nproc = 1279 __kmp_nested_nth.nth[level + 1]; 1280 } 1281 serial_team->t.t_level++; 1282 KF_TRACE(10, ("__kmpc_serialized_parallel: T#%d increasing nesting level " 1283 "of serial team %p to %d\n", 1284 global_tid, serial_team, serial_team->t.t_level)); 1285 1286 /* allocate/push dispatch buffers stack */ 1287 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch); 1288 { 1289 dispatch_private_info_t *disp_buffer = 1290 (dispatch_private_info_t *)__kmp_allocate( 1291 sizeof(dispatch_private_info_t)); 1292 disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer; 1293 serial_team->t.t_dispatch->th_disp_buffer = disp_buffer; 1294 } 1295 this_thr->th.th_dispatch = serial_team->t.t_dispatch; 1296 1297 KMP_MB(); 1298 } 1299 KMP_CHECK_UPDATE(serial_team->t.t_cancel_request, cancel_noreq); 1300 1301 // Perform the display affinity functionality for 1302 // serialized parallel regions 1303 if (__kmp_display_affinity) { 1304 if (this_thr->th.th_prev_level != serial_team->t.t_level || 1305 this_thr->th.th_prev_num_threads != 1) { 1306 // NULL means use the affinity-format-var ICV 1307 __kmp_aux_display_affinity(global_tid, NULL); 1308 this_thr->th.th_prev_level = serial_team->t.t_level; 1309 this_thr->th.th_prev_num_threads = 1; 1310 } 1311 } 1312 1313 if (__kmp_env_consistency_check) 1314 __kmp_push_parallel(global_tid, NULL); 1315 #if OMPT_SUPPORT 1316 serial_team->t.ompt_team_info.master_return_address = codeptr; 1317 if (ompt_enabled.enabled && 1318 this_thr->th.ompt_thread_info.state != ompt_state_overhead) { 1319 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr = 1320 OMPT_GET_FRAME_ADDRESS(0); 1321 1322 ompt_lw_taskteam_t lw_taskteam; 1323 __ompt_lw_taskteam_init(&lw_taskteam, this_thr, global_tid, 1324 &ompt_parallel_data, codeptr); 1325 1326 __ompt_lw_taskteam_link(&lw_taskteam, this_thr, 1); 1327 // don't use lw_taskteam after linking. content was swaped 1328 1329 /* OMPT implicit task begin */ 1330 if (ompt_enabled.ompt_callback_implicit_task) { 1331 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1332 ompt_scope_begin, OMPT_CUR_TEAM_DATA(this_thr), 1333 OMPT_CUR_TASK_DATA(this_thr), 1, __kmp_tid_from_gtid(global_tid), 1334 ompt_task_implicit); // TODO: Can this be ompt_task_initial? 1335 OMPT_CUR_TASK_INFO(this_thr)->thread_num = 1336 __kmp_tid_from_gtid(global_tid); 1337 } 1338 1339 /* OMPT state */ 1340 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel; 1341 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr = 1342 OMPT_GET_FRAME_ADDRESS(0); 1343 } 1344 #endif 1345 } 1346 1347 /* most of the work for a fork */ 1348 /* return true if we really went parallel, false if serialized */ 1349 int __kmp_fork_call(ident_t *loc, int gtid, 1350 enum fork_context_e call_context, // Intel, GNU, ... 1351 kmp_int32 argc, microtask_t microtask, launch_t invoker, 1352 kmp_va_list ap) { 1353 void **argv; 1354 int i; 1355 int master_tid; 1356 int master_this_cons; 1357 kmp_team_t *team; 1358 kmp_team_t *parent_team; 1359 kmp_info_t *master_th; 1360 kmp_root_t *root; 1361 int nthreads; 1362 int master_active; 1363 int master_set_numthreads; 1364 int level; 1365 int active_level; 1366 int teams_level; 1367 #if KMP_NESTED_HOT_TEAMS 1368 kmp_hot_team_ptr_t **p_hot_teams; 1369 #endif 1370 { // KMP_TIME_BLOCK 1371 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_fork_call); 1372 KMP_COUNT_VALUE(OMP_PARALLEL_args, argc); 1373 1374 KA_TRACE(20, ("__kmp_fork_call: enter T#%d\n", gtid)); 1375 if (__kmp_stkpadding > 0 && __kmp_root[gtid] != NULL) { 1376 /* Some systems prefer the stack for the root thread(s) to start with */ 1377 /* some gap from the parent stack to prevent false sharing. */ 1378 void *dummy = KMP_ALLOCA(__kmp_stkpadding); 1379 /* These 2 lines below are so this does not get optimized out */ 1380 if (__kmp_stkpadding > KMP_MAX_STKPADDING) 1381 __kmp_stkpadding += (short)((kmp_int64)dummy); 1382 } 1383 1384 /* initialize if needed */ 1385 KMP_DEBUG_ASSERT( 1386 __kmp_init_serial); // AC: potentially unsafe, not in sync with shutdown 1387 if (!TCR_4(__kmp_init_parallel)) 1388 __kmp_parallel_initialize(); 1389 __kmp_resume_if_soft_paused(); 1390 1391 /* setup current data */ 1392 master_th = __kmp_threads[gtid]; // AC: potentially unsafe, not in sync with 1393 // shutdown 1394 parent_team = master_th->th.th_team; 1395 master_tid = master_th->th.th_info.ds.ds_tid; 1396 master_this_cons = master_th->th.th_local.this_construct; 1397 root = master_th->th.th_root; 1398 master_active = root->r.r_active; 1399 master_set_numthreads = master_th->th.th_set_nproc; 1400 1401 #if OMPT_SUPPORT 1402 ompt_data_t ompt_parallel_data = ompt_data_none; 1403 ompt_data_t *parent_task_data; 1404 ompt_frame_t *ompt_frame; 1405 ompt_data_t *implicit_task_data; 1406 void *return_address = NULL; 1407 1408 if (ompt_enabled.enabled) { 1409 __ompt_get_task_info_internal(0, NULL, &parent_task_data, &ompt_frame, 1410 NULL, NULL); 1411 return_address = OMPT_LOAD_RETURN_ADDRESS(gtid); 1412 } 1413 #endif 1414 1415 // Assign affinity to root thread if it hasn't happened yet 1416 __kmp_assign_root_init_mask(); 1417 1418 // Nested level will be an index in the nested nthreads array 1419 level = parent_team->t.t_level; 1420 // used to launch non-serial teams even if nested is not allowed 1421 active_level = parent_team->t.t_active_level; 1422 // needed to check nesting inside the teams 1423 teams_level = master_th->th.th_teams_level; 1424 #if KMP_NESTED_HOT_TEAMS 1425 p_hot_teams = &master_th->th.th_hot_teams; 1426 if (*p_hot_teams == NULL && __kmp_hot_teams_max_level > 0) { 1427 *p_hot_teams = (kmp_hot_team_ptr_t *)__kmp_allocate( 1428 sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level); 1429 (*p_hot_teams)[0].hot_team = root->r.r_hot_team; 1430 // it is either actual or not needed (when active_level > 0) 1431 (*p_hot_teams)[0].hot_team_nth = 1; 1432 } 1433 #endif 1434 1435 #if OMPT_SUPPORT 1436 if (ompt_enabled.enabled) { 1437 if (ompt_enabled.ompt_callback_parallel_begin) { 1438 int team_size = master_set_numthreads 1439 ? master_set_numthreads 1440 : get__nproc_2(parent_team, master_tid); 1441 int flags = OMPT_INVOKER(call_context) | 1442 ((microtask == (microtask_t)__kmp_teams_master) 1443 ? ompt_parallel_league 1444 : ompt_parallel_team); 1445 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)( 1446 parent_task_data, ompt_frame, &ompt_parallel_data, team_size, flags, 1447 return_address); 1448 } 1449 master_th->th.ompt_thread_info.state = ompt_state_overhead; 1450 } 1451 #endif 1452 1453 master_th->th.th_ident = loc; 1454 1455 if (master_th->th.th_teams_microtask && ap && 1456 microtask != (microtask_t)__kmp_teams_master && level == teams_level) { 1457 // AC: This is start of parallel that is nested inside teams construct. 1458 // The team is actual (hot), all workers are ready at the fork barrier. 1459 // No lock needed to initialize the team a bit, then free workers. 1460 parent_team->t.t_ident = loc; 1461 __kmp_alloc_argv_entries(argc, parent_team, TRUE); 1462 parent_team->t.t_argc = argc; 1463 argv = (void **)parent_team->t.t_argv; 1464 for (i = argc - 1; i >= 0; --i) 1465 *argv++ = va_arg(kmp_va_deref(ap), void *); 1466 // Increment our nested depth levels, but not increase the serialization 1467 if (parent_team == master_th->th.th_serial_team) { 1468 // AC: we are in serialized parallel 1469 __kmpc_serialized_parallel(loc, gtid); 1470 KMP_DEBUG_ASSERT(parent_team->t.t_serialized > 1); 1471 1472 if (call_context == fork_context_gnu) { 1473 // AC: need to decrement t_serialized for enquiry functions to work 1474 // correctly, will restore at join time 1475 parent_team->t.t_serialized--; 1476 return TRUE; 1477 } 1478 1479 #if OMPD_SUPPORT 1480 parent_team->t.t_pkfn = microtask; 1481 #endif 1482 1483 #if OMPT_SUPPORT 1484 void *dummy; 1485 void **exit_frame_p; 1486 1487 ompt_lw_taskteam_t lw_taskteam; 1488 1489 if (ompt_enabled.enabled) { 1490 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid, 1491 &ompt_parallel_data, return_address); 1492 exit_frame_p = &(lw_taskteam.ompt_task_info.frame.exit_frame.ptr); 1493 1494 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0); 1495 // don't use lw_taskteam after linking. content was swaped 1496 1497 /* OMPT implicit task begin */ 1498 implicit_task_data = OMPT_CUR_TASK_DATA(master_th); 1499 if (ompt_enabled.ompt_callback_implicit_task) { 1500 OMPT_CUR_TASK_INFO(master_th)->thread_num = 1501 __kmp_tid_from_gtid(gtid); 1502 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1503 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th), 1504 implicit_task_data, 1, 1505 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit); 1506 } 1507 1508 /* OMPT state */ 1509 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; 1510 } else { 1511 exit_frame_p = &dummy; 1512 } 1513 #endif 1514 // AC: need to decrement t_serialized for enquiry functions to work 1515 // correctly, will restore at join time 1516 parent_team->t.t_serialized--; 1517 1518 { 1519 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 1520 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 1521 __kmp_invoke_microtask(microtask, gtid, 0, argc, parent_team->t.t_argv 1522 #if OMPT_SUPPORT 1523 , 1524 exit_frame_p 1525 #endif 1526 ); 1527 } 1528 1529 #if OMPT_SUPPORT 1530 if (ompt_enabled.enabled) { 1531 *exit_frame_p = NULL; 1532 OMPT_CUR_TASK_INFO(master_th)->frame.exit_frame = ompt_data_none; 1533 if (ompt_enabled.ompt_callback_implicit_task) { 1534 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1535 ompt_scope_end, NULL, implicit_task_data, 1, 1536 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit); 1537 } 1538 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th); 1539 __ompt_lw_taskteam_unlink(master_th); 1540 if (ompt_enabled.ompt_callback_parallel_end) { 1541 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)( 1542 &ompt_parallel_data, OMPT_CUR_TASK_DATA(master_th), 1543 OMPT_INVOKER(call_context) | ompt_parallel_team, 1544 return_address); 1545 } 1546 master_th->th.ompt_thread_info.state = ompt_state_overhead; 1547 } 1548 #endif 1549 return TRUE; 1550 } 1551 1552 parent_team->t.t_pkfn = microtask; 1553 parent_team->t.t_invoke = invoker; 1554 KMP_ATOMIC_INC(&root->r.r_in_parallel); 1555 parent_team->t.t_active_level++; 1556 parent_team->t.t_level++; 1557 parent_team->t.t_def_allocator = master_th->th.th_def_allocator; // save 1558 1559 #if OMPT_SUPPORT 1560 if (ompt_enabled.enabled) { 1561 ompt_lw_taskteam_t lw_taskteam; 1562 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid, 1563 &ompt_parallel_data, return_address); 1564 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 1, true); 1565 } 1566 #endif 1567 1568 /* Change number of threads in the team if requested */ 1569 if (master_set_numthreads) { // The parallel has num_threads clause 1570 if (master_set_numthreads <= master_th->th.th_teams_size.nth) { 1571 // AC: only can reduce number of threads dynamically, can't increase 1572 kmp_info_t **other_threads = parent_team->t.t_threads; 1573 // NOTE: if using distributed barrier, we need to run this code block 1574 // even when the team size appears not to have changed from the max. 1575 int old_proc = master_th->th.th_teams_size.nth; 1576 if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == 1577 bp_dist_bar) { 1578 __kmp_resize_dist_barrier(parent_team, old_proc, 1579 master_set_numthreads); 1580 __kmp_add_threads_to_team(parent_team, master_set_numthreads); 1581 } 1582 parent_team->t.t_nproc = master_set_numthreads; 1583 for (i = 0; i < master_set_numthreads; ++i) { 1584 other_threads[i]->th.th_team_nproc = master_set_numthreads; 1585 } 1586 } 1587 // Keep extra threads hot in the team for possible next parallels 1588 master_th->th.th_set_nproc = 0; 1589 } 1590 1591 #if USE_DEBUGGER 1592 if (__kmp_debugging) { // Let debugger override number of threads. 1593 int nth = __kmp_omp_num_threads(loc); 1594 if (nth > 0) { // 0 means debugger doesn't want to change num threads 1595 master_set_numthreads = nth; 1596 } 1597 } 1598 #endif 1599 1600 #if USE_ITT_BUILD && USE_ITT_NOTIFY 1601 if (((__itt_frame_submit_v3_ptr && __itt_get_timestamp_ptr) || 1602 KMP_ITT_DEBUG) && 1603 __kmp_forkjoin_frames_mode == 3 && 1604 parent_team->t.t_active_level == 1 // only report frames at level 1 1605 && master_th->th.th_teams_size.nteams == 1) { 1606 kmp_uint64 tmp_time = __itt_get_timestamp(); 1607 master_th->th.th_frame_time = tmp_time; 1608 parent_team->t.t_region_time = tmp_time; 1609 } 1610 if (__itt_stack_caller_create_ptr) { 1611 KMP_DEBUG_ASSERT(parent_team->t.t_stack_id == NULL); 1612 // create new stack stitching id before entering fork barrier 1613 parent_team->t.t_stack_id = __kmp_itt_stack_caller_create(); 1614 } 1615 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */ 1616 1617 KF_TRACE(10, ("__kmp_fork_call: before internal fork: root=%p, team=%p, " 1618 "master_th=%p, gtid=%d\n", 1619 root, parent_team, master_th, gtid)); 1620 __kmp_internal_fork(loc, gtid, parent_team); 1621 KF_TRACE(10, ("__kmp_fork_call: after internal fork: root=%p, team=%p, " 1622 "master_th=%p, gtid=%d\n", 1623 root, parent_team, master_th, gtid)); 1624 1625 if (call_context == fork_context_gnu) 1626 return TRUE; 1627 1628 /* Invoke microtask for PRIMARY thread */ 1629 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid, 1630 parent_team->t.t_id, parent_team->t.t_pkfn)); 1631 1632 if (!parent_team->t.t_invoke(gtid)) { 1633 KMP_ASSERT2(0, "cannot invoke microtask for PRIMARY thread"); 1634 } 1635 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid, 1636 parent_team->t.t_id, parent_team->t.t_pkfn)); 1637 KMP_MB(); /* Flush all pending memory write invalidates. */ 1638 1639 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid)); 1640 1641 return TRUE; 1642 } // Parallel closely nested in teams construct 1643 1644 #if KMP_DEBUG 1645 if (__kmp_tasking_mode != tskm_immediate_exec) { 1646 KMP_DEBUG_ASSERT(master_th->th.th_task_team == 1647 parent_team->t.t_task_team[master_th->th.th_task_state]); 1648 } 1649 #endif 1650 1651 // Need this to happen before we determine the number of threads, not while 1652 // we are allocating the team 1653 //__kmp_push_current_task_to_thread(master_th, parent_team, 0); 1654 int enter_teams = 0; 1655 if (parent_team->t.t_active_level >= 1656 master_th->th.th_current_task->td_icvs.max_active_levels) { 1657 nthreads = 1; 1658 } else { 1659 enter_teams = ((ap == NULL && active_level == 0) || 1660 (ap && teams_level > 0 && teams_level == level)); 1661 nthreads = master_set_numthreads 1662 ? master_set_numthreads 1663 // TODO: get nproc directly from current task 1664 : get__nproc_2(parent_team, master_tid); 1665 // Check if we need to take forkjoin lock? (no need for serialized 1666 // parallel out of teams construct). This code moved here from 1667 // __kmp_reserve_threads() to speedup nested serialized parallels. 1668 if (nthreads > 1) { 1669 if ((get__max_active_levels(master_th) == 1 && 1670 (root->r.r_in_parallel && !enter_teams)) || 1671 (__kmp_library == library_serial)) { 1672 KC_TRACE(10, ("__kmp_fork_call: T#%d serializing team; requested %d" 1673 " threads\n", 1674 gtid, nthreads)); 1675 nthreads = 1; 1676 } 1677 } 1678 if (nthreads > 1) { 1679 /* determine how many new threads we can use */ 1680 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 1681 /* AC: If we execute teams from parallel region (on host), then teams 1682 should be created but each can only have 1 thread if nesting is 1683 disabled. If teams called from serial region, then teams and their 1684 threads should be created regardless of the nesting setting. */ 1685 nthreads = __kmp_reserve_threads(root, parent_team, master_tid, 1686 nthreads, enter_teams); 1687 if (nthreads == 1) { 1688 // Free lock for single thread execution here; for multi-thread 1689 // execution it will be freed later after team of threads created 1690 // and initialized 1691 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 1692 } 1693 } 1694 } 1695 KMP_DEBUG_ASSERT(nthreads > 0); 1696 1697 // If we temporarily changed the set number of threads then restore it now 1698 master_th->th.th_set_nproc = 0; 1699 1700 /* create a serialized parallel region? */ 1701 if (nthreads == 1) { 1702 /* josh todo: hypothetical question: what do we do for OS X*? */ 1703 #if KMP_OS_LINUX && \ 1704 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 1705 void *args[argc]; 1706 #else 1707 void **args = (void **)KMP_ALLOCA(argc * sizeof(void *)); 1708 #endif /* KMP_OS_LINUX && ( KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || \ 1709 KMP_ARCH_AARCH64) */ 1710 1711 KA_TRACE(20, 1712 ("__kmp_fork_call: T#%d serializing parallel region\n", gtid)); 1713 1714 __kmpc_serialized_parallel(loc, gtid); 1715 1716 #if OMPD_SUPPORT 1717 master_th->th.th_serial_team->t.t_pkfn = microtask; 1718 #endif 1719 1720 if (call_context == fork_context_intel) { 1721 /* TODO this sucks, use the compiler itself to pass args! :) */ 1722 master_th->th.th_serial_team->t.t_ident = loc; 1723 if (!ap) { 1724 // revert change made in __kmpc_serialized_parallel() 1725 master_th->th.th_serial_team->t.t_level--; 1726 // Get args from parent team for teams construct 1727 1728 #if OMPT_SUPPORT 1729 void *dummy; 1730 void **exit_frame_p; 1731 ompt_task_info_t *task_info; 1732 1733 ompt_lw_taskteam_t lw_taskteam; 1734 1735 if (ompt_enabled.enabled) { 1736 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid, 1737 &ompt_parallel_data, return_address); 1738 1739 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0); 1740 // don't use lw_taskteam after linking. content was swaped 1741 1742 task_info = OMPT_CUR_TASK_INFO(master_th); 1743 exit_frame_p = &(task_info->frame.exit_frame.ptr); 1744 if (ompt_enabled.ompt_callback_implicit_task) { 1745 OMPT_CUR_TASK_INFO(master_th)->thread_num = 1746 __kmp_tid_from_gtid(gtid); 1747 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1748 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th), 1749 &(task_info->task_data), 1, 1750 OMPT_CUR_TASK_INFO(master_th)->thread_num, 1751 ompt_task_implicit); 1752 } 1753 1754 /* OMPT state */ 1755 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; 1756 } else { 1757 exit_frame_p = &dummy; 1758 } 1759 #endif 1760 1761 { 1762 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 1763 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 1764 __kmp_invoke_microtask(microtask, gtid, 0, argc, 1765 parent_team->t.t_argv 1766 #if OMPT_SUPPORT 1767 , 1768 exit_frame_p 1769 #endif 1770 ); 1771 } 1772 1773 #if OMPT_SUPPORT 1774 if (ompt_enabled.enabled) { 1775 *exit_frame_p = NULL; 1776 if (ompt_enabled.ompt_callback_implicit_task) { 1777 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1778 ompt_scope_end, NULL, &(task_info->task_data), 1, 1779 OMPT_CUR_TASK_INFO(master_th)->thread_num, 1780 ompt_task_implicit); 1781 } 1782 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th); 1783 __ompt_lw_taskteam_unlink(master_th); 1784 if (ompt_enabled.ompt_callback_parallel_end) { 1785 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)( 1786 &ompt_parallel_data, parent_task_data, 1787 OMPT_INVOKER(call_context) | ompt_parallel_team, 1788 return_address); 1789 } 1790 master_th->th.ompt_thread_info.state = ompt_state_overhead; 1791 } 1792 #endif 1793 } else if (microtask == (microtask_t)__kmp_teams_master) { 1794 KMP_DEBUG_ASSERT(master_th->th.th_team == 1795 master_th->th.th_serial_team); 1796 team = master_th->th.th_team; 1797 // team->t.t_pkfn = microtask; 1798 team->t.t_invoke = invoker; 1799 __kmp_alloc_argv_entries(argc, team, TRUE); 1800 team->t.t_argc = argc; 1801 argv = (void **)team->t.t_argv; 1802 if (ap) { 1803 for (i = argc - 1; i >= 0; --i) 1804 *argv++ = va_arg(kmp_va_deref(ap), void *); 1805 } else { 1806 for (i = 0; i < argc; ++i) 1807 // Get args from parent team for teams construct 1808 argv[i] = parent_team->t.t_argv[i]; 1809 } 1810 // AC: revert change made in __kmpc_serialized_parallel() 1811 // because initial code in teams should have level=0 1812 team->t.t_level--; 1813 // AC: call special invoker for outer "parallel" of teams construct 1814 invoker(gtid); 1815 #if OMPT_SUPPORT 1816 if (ompt_enabled.enabled) { 1817 ompt_task_info_t *task_info = OMPT_CUR_TASK_INFO(master_th); 1818 if (ompt_enabled.ompt_callback_implicit_task) { 1819 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1820 ompt_scope_end, NULL, &(task_info->task_data), 0, 1821 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_initial); 1822 } 1823 if (ompt_enabled.ompt_callback_parallel_end) { 1824 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)( 1825 &ompt_parallel_data, parent_task_data, 1826 OMPT_INVOKER(call_context) | ompt_parallel_league, 1827 return_address); 1828 } 1829 master_th->th.ompt_thread_info.state = ompt_state_overhead; 1830 } 1831 #endif 1832 } else { 1833 argv = args; 1834 for (i = argc - 1; i >= 0; --i) 1835 *argv++ = va_arg(kmp_va_deref(ap), void *); 1836 KMP_MB(); 1837 1838 #if OMPT_SUPPORT 1839 void *dummy; 1840 void **exit_frame_p; 1841 ompt_task_info_t *task_info; 1842 1843 ompt_lw_taskteam_t lw_taskteam; 1844 1845 if (ompt_enabled.enabled) { 1846 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid, 1847 &ompt_parallel_data, return_address); 1848 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0); 1849 // don't use lw_taskteam after linking. content was swaped 1850 task_info = OMPT_CUR_TASK_INFO(master_th); 1851 exit_frame_p = &(task_info->frame.exit_frame.ptr); 1852 1853 /* OMPT implicit task begin */ 1854 implicit_task_data = OMPT_CUR_TASK_DATA(master_th); 1855 if (ompt_enabled.ompt_callback_implicit_task) { 1856 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1857 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th), 1858 implicit_task_data, 1, __kmp_tid_from_gtid(gtid), 1859 ompt_task_implicit); 1860 OMPT_CUR_TASK_INFO(master_th)->thread_num = 1861 __kmp_tid_from_gtid(gtid); 1862 } 1863 1864 /* OMPT state */ 1865 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; 1866 } else { 1867 exit_frame_p = &dummy; 1868 } 1869 #endif 1870 1871 { 1872 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 1873 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 1874 __kmp_invoke_microtask(microtask, gtid, 0, argc, args 1875 #if OMPT_SUPPORT 1876 , 1877 exit_frame_p 1878 #endif 1879 ); 1880 } 1881 1882 #if OMPT_SUPPORT 1883 if (ompt_enabled.enabled) { 1884 *exit_frame_p = NULL; 1885 if (ompt_enabled.ompt_callback_implicit_task) { 1886 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1887 ompt_scope_end, NULL, &(task_info->task_data), 1, 1888 OMPT_CUR_TASK_INFO(master_th)->thread_num, 1889 ompt_task_implicit); 1890 } 1891 1892 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th); 1893 __ompt_lw_taskteam_unlink(master_th); 1894 if (ompt_enabled.ompt_callback_parallel_end) { 1895 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)( 1896 &ompt_parallel_data, parent_task_data, 1897 OMPT_INVOKER(call_context) | ompt_parallel_team, 1898 return_address); 1899 } 1900 master_th->th.ompt_thread_info.state = ompt_state_overhead; 1901 } 1902 #endif 1903 } 1904 } else if (call_context == fork_context_gnu) { 1905 #if OMPT_SUPPORT 1906 ompt_lw_taskteam_t lwt; 1907 __ompt_lw_taskteam_init(&lwt, master_th, gtid, &ompt_parallel_data, 1908 return_address); 1909 1910 lwt.ompt_task_info.frame.exit_frame = ompt_data_none; 1911 __ompt_lw_taskteam_link(&lwt, master_th, 1); 1912 // don't use lw_taskteam after linking. content was swaped 1913 #endif 1914 1915 // we were called from GNU native code 1916 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid)); 1917 return FALSE; 1918 } else { 1919 KMP_ASSERT2(call_context < fork_context_last, 1920 "__kmp_fork_call: unknown fork_context parameter"); 1921 } 1922 1923 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid)); 1924 KMP_MB(); 1925 return FALSE; 1926 } // if (nthreads == 1) 1927 1928 // GEH: only modify the executing flag in the case when not serialized 1929 // serialized case is handled in kmpc_serialized_parallel 1930 KF_TRACE(10, ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, " 1931 "curtask=%p, curtask_max_aclevel=%d\n", 1932 parent_team->t.t_active_level, master_th, 1933 master_th->th.th_current_task, 1934 master_th->th.th_current_task->td_icvs.max_active_levels)); 1935 // TODO: GEH - cannot do this assertion because root thread not set up as 1936 // executing 1937 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 1 ); 1938 master_th->th.th_current_task->td_flags.executing = 0; 1939 1940 if (!master_th->th.th_teams_microtask || level > teams_level) { 1941 /* Increment our nested depth level */ 1942 KMP_ATOMIC_INC(&root->r.r_in_parallel); 1943 } 1944 1945 // See if we need to make a copy of the ICVs. 1946 int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc; 1947 if ((level + 1 < __kmp_nested_nth.used) && 1948 (__kmp_nested_nth.nth[level + 1] != nthreads_icv)) { 1949 nthreads_icv = __kmp_nested_nth.nth[level + 1]; 1950 } else { 1951 nthreads_icv = 0; // don't update 1952 } 1953 1954 // Figure out the proc_bind_policy for the new team. 1955 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind; 1956 kmp_proc_bind_t proc_bind_icv = 1957 proc_bind_default; // proc_bind_default means don't update 1958 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) { 1959 proc_bind = proc_bind_false; 1960 } else { 1961 if (proc_bind == proc_bind_default) { 1962 // No proc_bind clause specified; use current proc-bind-var for this 1963 // parallel region 1964 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind; 1965 } 1966 /* else: The proc_bind policy was specified explicitly on parallel clause. 1967 This overrides proc-bind-var for this parallel region, but does not 1968 change proc-bind-var. */ 1969 // Figure the value of proc-bind-var for the child threads. 1970 if ((level + 1 < __kmp_nested_proc_bind.used) && 1971 (__kmp_nested_proc_bind.bind_types[level + 1] != 1972 master_th->th.th_current_task->td_icvs.proc_bind)) { 1973 proc_bind_icv = __kmp_nested_proc_bind.bind_types[level + 1]; 1974 } 1975 } 1976 1977 // Reset for next parallel region 1978 master_th->th.th_set_proc_bind = proc_bind_default; 1979 1980 if ((nthreads_icv > 0) || (proc_bind_icv != proc_bind_default)) { 1981 kmp_internal_control_t new_icvs; 1982 copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs); 1983 new_icvs.next = NULL; 1984 if (nthreads_icv > 0) { 1985 new_icvs.nproc = nthreads_icv; 1986 } 1987 if (proc_bind_icv != proc_bind_default) { 1988 new_icvs.proc_bind = proc_bind_icv; 1989 } 1990 1991 /* allocate a new parallel team */ 1992 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n")); 1993 team = __kmp_allocate_team(root, nthreads, nthreads, 1994 #if OMPT_SUPPORT 1995 ompt_parallel_data, 1996 #endif 1997 proc_bind, &new_icvs, 1998 argc USE_NESTED_HOT_ARG(master_th)); 1999 if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar) 2000 copy_icvs((kmp_internal_control_t *)team->t.b->team_icvs, &new_icvs); 2001 } else { 2002 /* allocate a new parallel team */ 2003 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n")); 2004 team = __kmp_allocate_team(root, nthreads, nthreads, 2005 #if OMPT_SUPPORT 2006 ompt_parallel_data, 2007 #endif 2008 proc_bind, 2009 &master_th->th.th_current_task->td_icvs, 2010 argc USE_NESTED_HOT_ARG(master_th)); 2011 if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar) 2012 copy_icvs((kmp_internal_control_t *)team->t.b->team_icvs, 2013 &master_th->th.th_current_task->td_icvs); 2014 } 2015 KF_TRACE( 2016 10, ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team)); 2017 2018 /* setup the new team */ 2019 KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid); 2020 KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons); 2021 KMP_CHECK_UPDATE(team->t.t_ident, loc); 2022 KMP_CHECK_UPDATE(team->t.t_parent, parent_team); 2023 KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask); 2024 #if OMPT_SUPPORT 2025 KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.master_return_address, 2026 return_address); 2027 #endif 2028 KMP_CHECK_UPDATE(team->t.t_invoke, invoker); // TODO move to root, maybe 2029 // TODO: parent_team->t.t_level == INT_MAX ??? 2030 if (!master_th->th.th_teams_microtask || level > teams_level) { 2031 int new_level = parent_team->t.t_level + 1; 2032 KMP_CHECK_UPDATE(team->t.t_level, new_level); 2033 new_level = parent_team->t.t_active_level + 1; 2034 KMP_CHECK_UPDATE(team->t.t_active_level, new_level); 2035 } else { 2036 // AC: Do not increase parallel level at start of the teams construct 2037 int new_level = parent_team->t.t_level; 2038 KMP_CHECK_UPDATE(team->t.t_level, new_level); 2039 new_level = parent_team->t.t_active_level; 2040 KMP_CHECK_UPDATE(team->t.t_active_level, new_level); 2041 } 2042 kmp_r_sched_t new_sched = get__sched_2(parent_team, master_tid); 2043 // set primary thread's schedule as new run-time schedule 2044 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched); 2045 2046 KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq); 2047 KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator); 2048 2049 // Update the floating point rounding in the team if required. 2050 propagateFPControl(team); 2051 #if OMPD_SUPPORT 2052 if (ompd_state & OMPD_ENABLE_BP) 2053 ompd_bp_parallel_begin(); 2054 #endif 2055 2056 if (__kmp_tasking_mode != tskm_immediate_exec) { 2057 // Set primary thread's task team to team's task team. Unless this is hot 2058 // team, it should be NULL. 2059 KMP_DEBUG_ASSERT(master_th->th.th_task_team == 2060 parent_team->t.t_task_team[master_th->th.th_task_state]); 2061 KA_TRACE(20, ("__kmp_fork_call: Primary T#%d pushing task_team %p / team " 2062 "%p, new task_team %p / team %p\n", 2063 __kmp_gtid_from_thread(master_th), 2064 master_th->th.th_task_team, parent_team, 2065 team->t.t_task_team[master_th->th.th_task_state], team)); 2066 2067 if (active_level || master_th->th.th_task_team) { 2068 // Take a memo of primary thread's task_state 2069 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack); 2070 if (master_th->th.th_task_state_top >= 2071 master_th->th.th_task_state_stack_sz) { // increase size 2072 kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz; 2073 kmp_uint8 *old_stack, *new_stack; 2074 kmp_uint32 i; 2075 new_stack = (kmp_uint8 *)__kmp_allocate(new_size); 2076 for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) { 2077 new_stack[i] = master_th->th.th_task_state_memo_stack[i]; 2078 } 2079 for (i = master_th->th.th_task_state_stack_sz; i < new_size; 2080 ++i) { // zero-init rest of stack 2081 new_stack[i] = 0; 2082 } 2083 old_stack = master_th->th.th_task_state_memo_stack; 2084 master_th->th.th_task_state_memo_stack = new_stack; 2085 master_th->th.th_task_state_stack_sz = new_size; 2086 __kmp_free(old_stack); 2087 } 2088 // Store primary thread's task_state on stack 2089 master_th->th 2090 .th_task_state_memo_stack[master_th->th.th_task_state_top] = 2091 master_th->th.th_task_state; 2092 master_th->th.th_task_state_top++; 2093 #if KMP_NESTED_HOT_TEAMS 2094 if (master_th->th.th_hot_teams && 2095 active_level < __kmp_hot_teams_max_level && 2096 team == master_th->th.th_hot_teams[active_level].hot_team) { 2097 // Restore primary thread's nested state if nested hot team 2098 master_th->th.th_task_state = 2099 master_th->th 2100 .th_task_state_memo_stack[master_th->th.th_task_state_top]; 2101 } else { 2102 #endif 2103 master_th->th.th_task_state = 0; 2104 #if KMP_NESTED_HOT_TEAMS 2105 } 2106 #endif 2107 } 2108 #if !KMP_NESTED_HOT_TEAMS 2109 KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) || 2110 (team == root->r.r_hot_team)); 2111 #endif 2112 } 2113 2114 KA_TRACE( 2115 20, 2116 ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n", 2117 gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id, 2118 team->t.t_nproc)); 2119 KMP_DEBUG_ASSERT(team != root->r.r_hot_team || 2120 (team->t.t_master_tid == 0 && 2121 (team->t.t_parent == root->r.r_root_team || 2122 team->t.t_parent->t.t_serialized))); 2123 KMP_MB(); 2124 2125 /* now, setup the arguments */ 2126 argv = (void **)team->t.t_argv; 2127 if (ap) { 2128 for (i = argc - 1; i >= 0; --i) { 2129 void *new_argv = va_arg(kmp_va_deref(ap), void *); 2130 KMP_CHECK_UPDATE(*argv, new_argv); 2131 argv++; 2132 } 2133 } else { 2134 for (i = 0; i < argc; ++i) { 2135 // Get args from parent team for teams construct 2136 KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i]); 2137 } 2138 } 2139 2140 /* now actually fork the threads */ 2141 KMP_CHECK_UPDATE(team->t.t_master_active, master_active); 2142 if (!root->r.r_active) // Only do assignment if it prevents cache ping-pong 2143 root->r.r_active = TRUE; 2144 2145 __kmp_fork_team_threads(root, team, master_th, gtid); 2146 __kmp_setup_icv_copy(team, nthreads, 2147 &master_th->th.th_current_task->td_icvs, loc); 2148 2149 #if OMPT_SUPPORT 2150 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; 2151 #endif 2152 2153 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 2154 2155 #if USE_ITT_BUILD 2156 if (team->t.t_active_level == 1 // only report frames at level 1 2157 && !master_th->th.th_teams_microtask) { // not in teams construct 2158 #if USE_ITT_NOTIFY 2159 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) && 2160 (__kmp_forkjoin_frames_mode == 3 || 2161 __kmp_forkjoin_frames_mode == 1)) { 2162 kmp_uint64 tmp_time = 0; 2163 if (__itt_get_timestamp_ptr) 2164 tmp_time = __itt_get_timestamp(); 2165 // Internal fork - report frame begin 2166 master_th->th.th_frame_time = tmp_time; 2167 if (__kmp_forkjoin_frames_mode == 3) 2168 team->t.t_region_time = tmp_time; 2169 } else 2170 // only one notification scheme (either "submit" or "forking/joined", not both) 2171 #endif /* USE_ITT_NOTIFY */ 2172 if ((__itt_frame_begin_v3_ptr || KMP_ITT_DEBUG) && 2173 __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode) { 2174 // Mark start of "parallel" region for Intel(R) VTune(TM) analyzer. 2175 __kmp_itt_region_forking(gtid, team->t.t_nproc, 0); 2176 } 2177 } 2178 #endif /* USE_ITT_BUILD */ 2179 2180 /* now go on and do the work */ 2181 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team); 2182 KMP_MB(); 2183 KF_TRACE(10, 2184 ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n", 2185 root, team, master_th, gtid)); 2186 2187 #if USE_ITT_BUILD 2188 if (__itt_stack_caller_create_ptr) { 2189 // create new stack stitching id before entering fork barrier 2190 if (!enter_teams) { 2191 KMP_DEBUG_ASSERT(team->t.t_stack_id == NULL); 2192 team->t.t_stack_id = __kmp_itt_stack_caller_create(); 2193 } else if (parent_team->t.t_serialized) { 2194 // keep stack stitching id in the serialized parent_team; 2195 // current team will be used for parallel inside the teams; 2196 // if parent_team is active, then it already keeps stack stitching id 2197 // for the league of teams 2198 KMP_DEBUG_ASSERT(parent_team->t.t_stack_id == NULL); 2199 parent_team->t.t_stack_id = __kmp_itt_stack_caller_create(); 2200 } 2201 } 2202 #endif /* USE_ITT_BUILD */ 2203 2204 // AC: skip __kmp_internal_fork at teams construct, let only primary 2205 // threads execute 2206 if (ap) { 2207 __kmp_internal_fork(loc, gtid, team); 2208 KF_TRACE(10, ("__kmp_internal_fork : after : root=%p, team=%p, " 2209 "master_th=%p, gtid=%d\n", 2210 root, team, master_th, gtid)); 2211 } 2212 2213 if (call_context == fork_context_gnu) { 2214 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid)); 2215 return TRUE; 2216 } 2217 2218 /* Invoke microtask for PRIMARY thread */ 2219 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid, 2220 team->t.t_id, team->t.t_pkfn)); 2221 } // END of timer KMP_fork_call block 2222 2223 #if KMP_STATS_ENABLED 2224 // If beginning a teams construct, then change thread state 2225 stats_state_e previous_state = KMP_GET_THREAD_STATE(); 2226 if (!ap) { 2227 KMP_SET_THREAD_STATE(stats_state_e::TEAMS_REGION); 2228 } 2229 #endif 2230 2231 if (!team->t.t_invoke(gtid)) { 2232 KMP_ASSERT2(0, "cannot invoke microtask for PRIMARY thread"); 2233 } 2234 2235 #if KMP_STATS_ENABLED 2236 // If was beginning of a teams construct, then reset thread state 2237 if (!ap) { 2238 KMP_SET_THREAD_STATE(previous_state); 2239 } 2240 #endif 2241 2242 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid, 2243 team->t.t_id, team->t.t_pkfn)); 2244 KMP_MB(); /* Flush all pending memory write invalidates. */ 2245 2246 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid)); 2247 #if OMPT_SUPPORT 2248 if (ompt_enabled.enabled) { 2249 master_th->th.ompt_thread_info.state = ompt_state_overhead; 2250 } 2251 #endif 2252 2253 return TRUE; 2254 } 2255 2256 #if OMPT_SUPPORT 2257 static inline void __kmp_join_restore_state(kmp_info_t *thread, 2258 kmp_team_t *team) { 2259 // restore state outside the region 2260 thread->th.ompt_thread_info.state = 2261 ((team->t.t_serialized) ? ompt_state_work_serial 2262 : ompt_state_work_parallel); 2263 } 2264 2265 static inline void __kmp_join_ompt(int gtid, kmp_info_t *thread, 2266 kmp_team_t *team, ompt_data_t *parallel_data, 2267 int flags, void *codeptr) { 2268 ompt_task_info_t *task_info = __ompt_get_task_info_object(0); 2269 if (ompt_enabled.ompt_callback_parallel_end) { 2270 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)( 2271 parallel_data, &(task_info->task_data), flags, codeptr); 2272 } 2273 2274 task_info->frame.enter_frame = ompt_data_none; 2275 __kmp_join_restore_state(thread, team); 2276 } 2277 #endif 2278 2279 void __kmp_join_call(ident_t *loc, int gtid 2280 #if OMPT_SUPPORT 2281 , 2282 enum fork_context_e fork_context 2283 #endif 2284 , 2285 int exit_teams) { 2286 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_join_call); 2287 kmp_team_t *team; 2288 kmp_team_t *parent_team; 2289 kmp_info_t *master_th; 2290 kmp_root_t *root; 2291 int master_active; 2292 2293 KA_TRACE(20, ("__kmp_join_call: enter T#%d\n", gtid)); 2294 2295 /* setup current data */ 2296 master_th = __kmp_threads[gtid]; 2297 root = master_th->th.th_root; 2298 team = master_th->th.th_team; 2299 parent_team = team->t.t_parent; 2300 2301 master_th->th.th_ident = loc; 2302 2303 #if OMPT_SUPPORT 2304 void *team_microtask = (void *)team->t.t_pkfn; 2305 // For GOMP interface with serialized parallel, need the 2306 // __kmpc_end_serialized_parallel to call hooks for OMPT end-implicit-task 2307 // and end-parallel events. 2308 if (ompt_enabled.enabled && 2309 !(team->t.t_serialized && fork_context == fork_context_gnu)) { 2310 master_th->th.ompt_thread_info.state = ompt_state_overhead; 2311 } 2312 #endif 2313 2314 #if KMP_DEBUG 2315 if (__kmp_tasking_mode != tskm_immediate_exec && !exit_teams) { 2316 KA_TRACE(20, ("__kmp_join_call: T#%d, old team = %p old task_team = %p, " 2317 "th_task_team = %p\n", 2318 __kmp_gtid_from_thread(master_th), team, 2319 team->t.t_task_team[master_th->th.th_task_state], 2320 master_th->th.th_task_team)); 2321 KMP_DEBUG_ASSERT(master_th->th.th_task_team == 2322 team->t.t_task_team[master_th->th.th_task_state]); 2323 } 2324 #endif 2325 2326 if (team->t.t_serialized) { 2327 if (master_th->th.th_teams_microtask) { 2328 // We are in teams construct 2329 int level = team->t.t_level; 2330 int tlevel = master_th->th.th_teams_level; 2331 if (level == tlevel) { 2332 // AC: we haven't incremented it earlier at start of teams construct, 2333 // so do it here - at the end of teams construct 2334 team->t.t_level++; 2335 } else if (level == tlevel + 1) { 2336 // AC: we are exiting parallel inside teams, need to increment 2337 // serialization in order to restore it in the next call to 2338 // __kmpc_end_serialized_parallel 2339 team->t.t_serialized++; 2340 } 2341 } 2342 __kmpc_end_serialized_parallel(loc, gtid); 2343 2344 #if OMPT_SUPPORT 2345 if (ompt_enabled.enabled) { 2346 __kmp_join_restore_state(master_th, parent_team); 2347 } 2348 #endif 2349 2350 return; 2351 } 2352 2353 master_active = team->t.t_master_active; 2354 2355 if (!exit_teams) { 2356 // AC: No barrier for internal teams at exit from teams construct. 2357 // But there is barrier for external team (league). 2358 __kmp_internal_join(loc, gtid, team); 2359 #if USE_ITT_BUILD 2360 if (__itt_stack_caller_create_ptr) { 2361 KMP_DEBUG_ASSERT(team->t.t_stack_id != NULL); 2362 // destroy the stack stitching id after join barrier 2363 __kmp_itt_stack_caller_destroy((__itt_caller)team->t.t_stack_id); 2364 team->t.t_stack_id = NULL; 2365 } 2366 #endif 2367 } else { 2368 master_th->th.th_task_state = 2369 0; // AC: no tasking in teams (out of any parallel) 2370 #if USE_ITT_BUILD 2371 if (__itt_stack_caller_create_ptr && parent_team->t.t_serialized) { 2372 KMP_DEBUG_ASSERT(parent_team->t.t_stack_id != NULL); 2373 // destroy the stack stitching id on exit from the teams construct 2374 // if parent_team is active, then the id will be destroyed later on 2375 // by master of the league of teams 2376 __kmp_itt_stack_caller_destroy((__itt_caller)parent_team->t.t_stack_id); 2377 parent_team->t.t_stack_id = NULL; 2378 } 2379 #endif 2380 2381 if (team->t.t_nproc > 1 && 2382 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) { 2383 team->t.b->update_num_threads(team->t.t_nproc); 2384 __kmp_add_threads_to_team(team, team->t.t_nproc); 2385 } 2386 } 2387 2388 KMP_MB(); 2389 2390 #if OMPT_SUPPORT 2391 ompt_data_t *parallel_data = &(team->t.ompt_team_info.parallel_data); 2392 void *codeptr = team->t.ompt_team_info.master_return_address; 2393 #endif 2394 2395 #if USE_ITT_BUILD 2396 // Mark end of "parallel" region for Intel(R) VTune(TM) analyzer. 2397 if (team->t.t_active_level == 1 && 2398 (!master_th->th.th_teams_microtask || /* not in teams construct */ 2399 master_th->th.th_teams_size.nteams == 1)) { 2400 master_th->th.th_ident = loc; 2401 // only one notification scheme (either "submit" or "forking/joined", not 2402 // both) 2403 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) && 2404 __kmp_forkjoin_frames_mode == 3) 2405 __kmp_itt_frame_submit(gtid, team->t.t_region_time, 2406 master_th->th.th_frame_time, 0, loc, 2407 master_th->th.th_team_nproc, 1); 2408 else if ((__itt_frame_end_v3_ptr || KMP_ITT_DEBUG) && 2409 !__kmp_forkjoin_frames_mode && __kmp_forkjoin_frames) 2410 __kmp_itt_region_joined(gtid); 2411 } // active_level == 1 2412 #endif /* USE_ITT_BUILD */ 2413 2414 if (master_th->th.th_teams_microtask && !exit_teams && 2415 team->t.t_pkfn != (microtask_t)__kmp_teams_master && 2416 team->t.t_level == master_th->th.th_teams_level + 1) { 2417 // AC: We need to leave the team structure intact at the end of parallel 2418 // inside the teams construct, so that at the next parallel same (hot) team 2419 // works, only adjust nesting levels 2420 #if OMPT_SUPPORT 2421 ompt_data_t ompt_parallel_data = ompt_data_none; 2422 if (ompt_enabled.enabled) { 2423 ompt_task_info_t *task_info = __ompt_get_task_info_object(0); 2424 if (ompt_enabled.ompt_callback_implicit_task) { 2425 int ompt_team_size = team->t.t_nproc; 2426 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 2427 ompt_scope_end, NULL, &(task_info->task_data), ompt_team_size, 2428 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit); 2429 } 2430 task_info->frame.exit_frame = ompt_data_none; 2431 task_info->task_data = ompt_data_none; 2432 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th); 2433 __ompt_lw_taskteam_unlink(master_th); 2434 } 2435 #endif 2436 /* Decrement our nested depth level */ 2437 team->t.t_level--; 2438 team->t.t_active_level--; 2439 KMP_ATOMIC_DEC(&root->r.r_in_parallel); 2440 2441 // Restore number of threads in the team if needed. This code relies on 2442 // the proper adjustment of th_teams_size.nth after the fork in 2443 // __kmp_teams_master on each teams primary thread in the case that 2444 // __kmp_reserve_threads reduced it. 2445 if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) { 2446 int old_num = master_th->th.th_team_nproc; 2447 int new_num = master_th->th.th_teams_size.nth; 2448 kmp_info_t **other_threads = team->t.t_threads; 2449 team->t.t_nproc = new_num; 2450 for (int i = 0; i < old_num; ++i) { 2451 other_threads[i]->th.th_team_nproc = new_num; 2452 } 2453 // Adjust states of non-used threads of the team 2454 for (int i = old_num; i < new_num; ++i) { 2455 // Re-initialize thread's barrier data. 2456 KMP_DEBUG_ASSERT(other_threads[i]); 2457 kmp_balign_t *balign = other_threads[i]->th.th_bar; 2458 for (int b = 0; b < bs_last_barrier; ++b) { 2459 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 2460 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 2461 #if USE_DEBUGGER 2462 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 2463 #endif 2464 } 2465 if (__kmp_tasking_mode != tskm_immediate_exec) { 2466 // Synchronize thread's task state 2467 other_threads[i]->th.th_task_state = master_th->th.th_task_state; 2468 } 2469 } 2470 } 2471 2472 #if OMPT_SUPPORT 2473 if (ompt_enabled.enabled) { 2474 __kmp_join_ompt(gtid, master_th, parent_team, &ompt_parallel_data, 2475 OMPT_INVOKER(fork_context) | ompt_parallel_team, codeptr); 2476 } 2477 #endif 2478 2479 return; 2480 } 2481 2482 /* do cleanup and restore the parent team */ 2483 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid; 2484 master_th->th.th_local.this_construct = team->t.t_master_this_cons; 2485 2486 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid]; 2487 2488 /* jc: The following lock has instructions with REL and ACQ semantics, 2489 separating the parallel user code called in this parallel region 2490 from the serial user code called after this function returns. */ 2491 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 2492 2493 if (!master_th->th.th_teams_microtask || 2494 team->t.t_level > master_th->th.th_teams_level) { 2495 /* Decrement our nested depth level */ 2496 KMP_ATOMIC_DEC(&root->r.r_in_parallel); 2497 } 2498 KMP_DEBUG_ASSERT(root->r.r_in_parallel >= 0); 2499 2500 #if OMPT_SUPPORT 2501 if (ompt_enabled.enabled) { 2502 ompt_task_info_t *task_info = __ompt_get_task_info_object(0); 2503 if (ompt_enabled.ompt_callback_implicit_task) { 2504 int flags = (team_microtask == (void *)__kmp_teams_master) 2505 ? ompt_task_initial 2506 : ompt_task_implicit; 2507 int ompt_team_size = (flags == ompt_task_initial) ? 0 : team->t.t_nproc; 2508 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 2509 ompt_scope_end, NULL, &(task_info->task_data), ompt_team_size, 2510 OMPT_CUR_TASK_INFO(master_th)->thread_num, flags); 2511 } 2512 task_info->frame.exit_frame = ompt_data_none; 2513 task_info->task_data = ompt_data_none; 2514 } 2515 #endif 2516 2517 KF_TRACE(10, ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n", 0, 2518 master_th, team)); 2519 __kmp_pop_current_task_from_thread(master_th); 2520 2521 #if KMP_AFFINITY_SUPPORTED 2522 // Restore master thread's partition. 2523 master_th->th.th_first_place = team->t.t_first_place; 2524 master_th->th.th_last_place = team->t.t_last_place; 2525 #endif // KMP_AFFINITY_SUPPORTED 2526 master_th->th.th_def_allocator = team->t.t_def_allocator; 2527 2528 #if OMPD_SUPPORT 2529 if (ompd_state & OMPD_ENABLE_BP) 2530 ompd_bp_parallel_end(); 2531 #endif 2532 updateHWFPControl(team); 2533 2534 if (root->r.r_active != master_active) 2535 root->r.r_active = master_active; 2536 2537 __kmp_free_team(root, team USE_NESTED_HOT_ARG( 2538 master_th)); // this will free worker threads 2539 2540 /* this race was fun to find. make sure the following is in the critical 2541 region otherwise assertions may fail occasionally since the old team may be 2542 reallocated and the hierarchy appears inconsistent. it is actually safe to 2543 run and won't cause any bugs, but will cause those assertion failures. it's 2544 only one deref&assign so might as well put this in the critical region */ 2545 master_th->th.th_team = parent_team; 2546 master_th->th.th_team_nproc = parent_team->t.t_nproc; 2547 master_th->th.th_team_master = parent_team->t.t_threads[0]; 2548 master_th->th.th_team_serialized = parent_team->t.t_serialized; 2549 2550 /* restore serialized team, if need be */ 2551 if (parent_team->t.t_serialized && 2552 parent_team != master_th->th.th_serial_team && 2553 parent_team != root->r.r_root_team) { 2554 __kmp_free_team(root, 2555 master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL)); 2556 master_th->th.th_serial_team = parent_team; 2557 } 2558 2559 if (__kmp_tasking_mode != tskm_immediate_exec) { 2560 if (master_th->th.th_task_state_top > 2561 0) { // Restore task state from memo stack 2562 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack); 2563 // Remember primary thread's state if we re-use this nested hot team 2564 master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] = 2565 master_th->th.th_task_state; 2566 --master_th->th.th_task_state_top; // pop 2567 // Now restore state at this level 2568 master_th->th.th_task_state = 2569 master_th->th 2570 .th_task_state_memo_stack[master_th->th.th_task_state_top]; 2571 } 2572 // Copy the task team from the parent team to the primary thread 2573 master_th->th.th_task_team = 2574 parent_team->t.t_task_team[master_th->th.th_task_state]; 2575 KA_TRACE(20, 2576 ("__kmp_join_call: Primary T#%d restoring task_team %p, team %p\n", 2577 __kmp_gtid_from_thread(master_th), master_th->th.th_task_team, 2578 parent_team)); 2579 } 2580 2581 // TODO: GEH - cannot do this assertion because root thread not set up as 2582 // executing 2583 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 0 ); 2584 master_th->th.th_current_task->td_flags.executing = 1; 2585 2586 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 2587 2588 #if OMPT_SUPPORT 2589 int flags = 2590 OMPT_INVOKER(fork_context) | 2591 ((team_microtask == (void *)__kmp_teams_master) ? ompt_parallel_league 2592 : ompt_parallel_team); 2593 if (ompt_enabled.enabled) { 2594 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, flags, 2595 codeptr); 2596 } 2597 #endif 2598 2599 KMP_MB(); 2600 KA_TRACE(20, ("__kmp_join_call: exit T#%d\n", gtid)); 2601 } 2602 2603 /* Check whether we should push an internal control record onto the 2604 serial team stack. If so, do it. */ 2605 void __kmp_save_internal_controls(kmp_info_t *thread) { 2606 2607 if (thread->th.th_team != thread->th.th_serial_team) { 2608 return; 2609 } 2610 if (thread->th.th_team->t.t_serialized > 1) { 2611 int push = 0; 2612 2613 if (thread->th.th_team->t.t_control_stack_top == NULL) { 2614 push = 1; 2615 } else { 2616 if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level != 2617 thread->th.th_team->t.t_serialized) { 2618 push = 1; 2619 } 2620 } 2621 if (push) { /* push a record on the serial team's stack */ 2622 kmp_internal_control_t *control = 2623 (kmp_internal_control_t *)__kmp_allocate( 2624 sizeof(kmp_internal_control_t)); 2625 2626 copy_icvs(control, &thread->th.th_current_task->td_icvs); 2627 2628 control->serial_nesting_level = thread->th.th_team->t.t_serialized; 2629 2630 control->next = thread->th.th_team->t.t_control_stack_top; 2631 thread->th.th_team->t.t_control_stack_top = control; 2632 } 2633 } 2634 } 2635 2636 /* Changes set_nproc */ 2637 void __kmp_set_num_threads(int new_nth, int gtid) { 2638 kmp_info_t *thread; 2639 kmp_root_t *root; 2640 2641 KF_TRACE(10, ("__kmp_set_num_threads: new __kmp_nth = %d\n", new_nth)); 2642 KMP_DEBUG_ASSERT(__kmp_init_serial); 2643 2644 if (new_nth < 1) 2645 new_nth = 1; 2646 else if (new_nth > __kmp_max_nth) 2647 new_nth = __kmp_max_nth; 2648 2649 KMP_COUNT_VALUE(OMP_set_numthreads, new_nth); 2650 thread = __kmp_threads[gtid]; 2651 if (thread->th.th_current_task->td_icvs.nproc == new_nth) 2652 return; // nothing to do 2653 2654 __kmp_save_internal_controls(thread); 2655 2656 set__nproc(thread, new_nth); 2657 2658 // If this omp_set_num_threads() call will cause the hot team size to be 2659 // reduced (in the absence of a num_threads clause), then reduce it now, 2660 // rather than waiting for the next parallel region. 2661 root = thread->th.th_root; 2662 if (__kmp_init_parallel && (!root->r.r_active) && 2663 (root->r.r_hot_team->t.t_nproc > new_nth) 2664 #if KMP_NESTED_HOT_TEAMS 2665 && __kmp_hot_teams_max_level && !__kmp_hot_teams_mode 2666 #endif 2667 ) { 2668 kmp_team_t *hot_team = root->r.r_hot_team; 2669 int f; 2670 2671 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 2672 2673 if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar) { 2674 __kmp_resize_dist_barrier(hot_team, hot_team->t.t_nproc, new_nth); 2675 } 2676 // Release the extra threads we don't need any more. 2677 for (f = new_nth; f < hot_team->t.t_nproc; f++) { 2678 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL); 2679 if (__kmp_tasking_mode != tskm_immediate_exec) { 2680 // When decreasing team size, threads no longer in the team should unref 2681 // task team. 2682 hot_team->t.t_threads[f]->th.th_task_team = NULL; 2683 } 2684 __kmp_free_thread(hot_team->t.t_threads[f]); 2685 hot_team->t.t_threads[f] = NULL; 2686 } 2687 hot_team->t.t_nproc = new_nth; 2688 #if KMP_NESTED_HOT_TEAMS 2689 if (thread->th.th_hot_teams) { 2690 KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team); 2691 thread->th.th_hot_teams[0].hot_team_nth = new_nth; 2692 } 2693 #endif 2694 2695 if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar) { 2696 hot_team->t.b->update_num_threads(new_nth); 2697 __kmp_add_threads_to_team(hot_team, new_nth); 2698 } 2699 2700 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 2701 2702 // Update the t_nproc field in the threads that are still active. 2703 for (f = 0; f < new_nth; f++) { 2704 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL); 2705 hot_team->t.t_threads[f]->th.th_team_nproc = new_nth; 2706 } 2707 // Special flag in case omp_set_num_threads() call 2708 hot_team->t.t_size_changed = -1; 2709 } 2710 } 2711 2712 /* Changes max_active_levels */ 2713 void __kmp_set_max_active_levels(int gtid, int max_active_levels) { 2714 kmp_info_t *thread; 2715 2716 KF_TRACE(10, ("__kmp_set_max_active_levels: new max_active_levels for thread " 2717 "%d = (%d)\n", 2718 gtid, max_active_levels)); 2719 KMP_DEBUG_ASSERT(__kmp_init_serial); 2720 2721 // validate max_active_levels 2722 if (max_active_levels < 0) { 2723 KMP_WARNING(ActiveLevelsNegative, max_active_levels); 2724 // We ignore this call if the user has specified a negative value. 2725 // The current setting won't be changed. The last valid setting will be 2726 // used. A warning will be issued (if warnings are allowed as controlled by 2727 // the KMP_WARNINGS env var). 2728 KF_TRACE(10, ("__kmp_set_max_active_levels: the call is ignored: new " 2729 "max_active_levels for thread %d = (%d)\n", 2730 gtid, max_active_levels)); 2731 return; 2732 } 2733 if (max_active_levels <= KMP_MAX_ACTIVE_LEVELS_LIMIT) { 2734 // it's OK, the max_active_levels is within the valid range: [ 0; 2735 // KMP_MAX_ACTIVE_LEVELS_LIMIT ] 2736 // We allow a zero value. (implementation defined behavior) 2737 } else { 2738 KMP_WARNING(ActiveLevelsExceedLimit, max_active_levels, 2739 KMP_MAX_ACTIVE_LEVELS_LIMIT); 2740 max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT; 2741 // Current upper limit is MAX_INT. (implementation defined behavior) 2742 // If the input exceeds the upper limit, we correct the input to be the 2743 // upper limit. (implementation defined behavior) 2744 // Actually, the flow should never get here until we use MAX_INT limit. 2745 } 2746 KF_TRACE(10, ("__kmp_set_max_active_levels: after validation: new " 2747 "max_active_levels for thread %d = (%d)\n", 2748 gtid, max_active_levels)); 2749 2750 thread = __kmp_threads[gtid]; 2751 2752 __kmp_save_internal_controls(thread); 2753 2754 set__max_active_levels(thread, max_active_levels); 2755 } 2756 2757 /* Gets max_active_levels */ 2758 int __kmp_get_max_active_levels(int gtid) { 2759 kmp_info_t *thread; 2760 2761 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d\n", gtid)); 2762 KMP_DEBUG_ASSERT(__kmp_init_serial); 2763 2764 thread = __kmp_threads[gtid]; 2765 KMP_DEBUG_ASSERT(thread->th.th_current_task); 2766 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d, curtask=%p, " 2767 "curtask_maxaclevel=%d\n", 2768 gtid, thread->th.th_current_task, 2769 thread->th.th_current_task->td_icvs.max_active_levels)); 2770 return thread->th.th_current_task->td_icvs.max_active_levels; 2771 } 2772 2773 // nteams-var per-device ICV 2774 void __kmp_set_num_teams(int num_teams) { 2775 if (num_teams > 0) 2776 __kmp_nteams = num_teams; 2777 } 2778 int __kmp_get_max_teams(void) { return __kmp_nteams; } 2779 // teams-thread-limit-var per-device ICV 2780 void __kmp_set_teams_thread_limit(int limit) { 2781 if (limit > 0) 2782 __kmp_teams_thread_limit = limit; 2783 } 2784 int __kmp_get_teams_thread_limit(void) { return __kmp_teams_thread_limit; } 2785 2786 KMP_BUILD_ASSERT(sizeof(kmp_sched_t) == sizeof(int)); 2787 KMP_BUILD_ASSERT(sizeof(enum sched_type) == sizeof(int)); 2788 2789 /* Changes def_sched_var ICV values (run-time schedule kind and chunk) */ 2790 void __kmp_set_schedule(int gtid, kmp_sched_t kind, int chunk) { 2791 kmp_info_t *thread; 2792 kmp_sched_t orig_kind; 2793 // kmp_team_t *team; 2794 2795 KF_TRACE(10, ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n", 2796 gtid, (int)kind, chunk)); 2797 KMP_DEBUG_ASSERT(__kmp_init_serial); 2798 2799 // Check if the kind parameter is valid, correct if needed. 2800 // Valid parameters should fit in one of two intervals - standard or extended: 2801 // <lower>, <valid>, <upper_std>, <lower_ext>, <valid>, <upper> 2802 // 2008-01-25: 0, 1 - 4, 5, 100, 101 - 102, 103 2803 orig_kind = kind; 2804 kind = __kmp_sched_without_mods(kind); 2805 2806 if (kind <= kmp_sched_lower || kind >= kmp_sched_upper || 2807 (kind <= kmp_sched_lower_ext && kind >= kmp_sched_upper_std)) { 2808 // TODO: Hint needs attention in case we change the default schedule. 2809 __kmp_msg(kmp_ms_warning, KMP_MSG(ScheduleKindOutOfRange, kind), 2810 KMP_HNT(DefaultScheduleKindUsed, "static, no chunk"), 2811 __kmp_msg_null); 2812 kind = kmp_sched_default; 2813 chunk = 0; // ignore chunk value in case of bad kind 2814 } 2815 2816 thread = __kmp_threads[gtid]; 2817 2818 __kmp_save_internal_controls(thread); 2819 2820 if (kind < kmp_sched_upper_std) { 2821 if (kind == kmp_sched_static && chunk < KMP_DEFAULT_CHUNK) { 2822 // differ static chunked vs. unchunked: chunk should be invalid to 2823 // indicate unchunked schedule (which is the default) 2824 thread->th.th_current_task->td_icvs.sched.r_sched_type = kmp_sch_static; 2825 } else { 2826 thread->th.th_current_task->td_icvs.sched.r_sched_type = 2827 __kmp_sch_map[kind - kmp_sched_lower - 1]; 2828 } 2829 } else { 2830 // __kmp_sch_map[ kind - kmp_sched_lower_ext + kmp_sched_upper_std - 2831 // kmp_sched_lower - 2 ]; 2832 thread->th.th_current_task->td_icvs.sched.r_sched_type = 2833 __kmp_sch_map[kind - kmp_sched_lower_ext + kmp_sched_upper_std - 2834 kmp_sched_lower - 2]; 2835 } 2836 __kmp_sched_apply_mods_intkind( 2837 orig_kind, &(thread->th.th_current_task->td_icvs.sched.r_sched_type)); 2838 if (kind == kmp_sched_auto || chunk < 1) { 2839 // ignore parameter chunk for schedule auto 2840 thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK; 2841 } else { 2842 thread->th.th_current_task->td_icvs.sched.chunk = chunk; 2843 } 2844 } 2845 2846 /* Gets def_sched_var ICV values */ 2847 void __kmp_get_schedule(int gtid, kmp_sched_t *kind, int *chunk) { 2848 kmp_info_t *thread; 2849 enum sched_type th_type; 2850 2851 KF_TRACE(10, ("__kmp_get_schedule: thread %d\n", gtid)); 2852 KMP_DEBUG_ASSERT(__kmp_init_serial); 2853 2854 thread = __kmp_threads[gtid]; 2855 2856 th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type; 2857 switch (SCHEDULE_WITHOUT_MODIFIERS(th_type)) { 2858 case kmp_sch_static: 2859 case kmp_sch_static_greedy: 2860 case kmp_sch_static_balanced: 2861 *kind = kmp_sched_static; 2862 __kmp_sched_apply_mods_stdkind(kind, th_type); 2863 *chunk = 0; // chunk was not set, try to show this fact via zero value 2864 return; 2865 case kmp_sch_static_chunked: 2866 *kind = kmp_sched_static; 2867 break; 2868 case kmp_sch_dynamic_chunked: 2869 *kind = kmp_sched_dynamic; 2870 break; 2871 case kmp_sch_guided_chunked: 2872 case kmp_sch_guided_iterative_chunked: 2873 case kmp_sch_guided_analytical_chunked: 2874 *kind = kmp_sched_guided; 2875 break; 2876 case kmp_sch_auto: 2877 *kind = kmp_sched_auto; 2878 break; 2879 case kmp_sch_trapezoidal: 2880 *kind = kmp_sched_trapezoidal; 2881 break; 2882 #if KMP_STATIC_STEAL_ENABLED 2883 case kmp_sch_static_steal: 2884 *kind = kmp_sched_static_steal; 2885 break; 2886 #endif 2887 default: 2888 KMP_FATAL(UnknownSchedulingType, th_type); 2889 } 2890 2891 __kmp_sched_apply_mods_stdkind(kind, th_type); 2892 *chunk = thread->th.th_current_task->td_icvs.sched.chunk; 2893 } 2894 2895 int __kmp_get_ancestor_thread_num(int gtid, int level) { 2896 2897 int ii, dd; 2898 kmp_team_t *team; 2899 kmp_info_t *thr; 2900 2901 KF_TRACE(10, ("__kmp_get_ancestor_thread_num: thread %d %d\n", gtid, level)); 2902 KMP_DEBUG_ASSERT(__kmp_init_serial); 2903 2904 // validate level 2905 if (level == 0) 2906 return 0; 2907 if (level < 0) 2908 return -1; 2909 thr = __kmp_threads[gtid]; 2910 team = thr->th.th_team; 2911 ii = team->t.t_level; 2912 if (level > ii) 2913 return -1; 2914 2915 if (thr->th.th_teams_microtask) { 2916 // AC: we are in teams region where multiple nested teams have same level 2917 int tlevel = thr->th.th_teams_level; // the level of the teams construct 2918 if (level <= 2919 tlevel) { // otherwise usual algorithm works (will not touch the teams) 2920 KMP_DEBUG_ASSERT(ii >= tlevel); 2921 // AC: As we need to pass by the teams league, we need to artificially 2922 // increase ii 2923 if (ii == tlevel) { 2924 ii += 2; // three teams have same level 2925 } else { 2926 ii++; // two teams have same level 2927 } 2928 } 2929 } 2930 2931 if (ii == level) 2932 return __kmp_tid_from_gtid(gtid); 2933 2934 dd = team->t.t_serialized; 2935 level++; 2936 while (ii > level) { 2937 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) { 2938 } 2939 if ((team->t.t_serialized) && (!dd)) { 2940 team = team->t.t_parent; 2941 continue; 2942 } 2943 if (ii > level) { 2944 team = team->t.t_parent; 2945 dd = team->t.t_serialized; 2946 ii--; 2947 } 2948 } 2949 2950 return (dd > 1) ? (0) : (team->t.t_master_tid); 2951 } 2952 2953 int __kmp_get_team_size(int gtid, int level) { 2954 2955 int ii, dd; 2956 kmp_team_t *team; 2957 kmp_info_t *thr; 2958 2959 KF_TRACE(10, ("__kmp_get_team_size: thread %d %d\n", gtid, level)); 2960 KMP_DEBUG_ASSERT(__kmp_init_serial); 2961 2962 // validate level 2963 if (level == 0) 2964 return 1; 2965 if (level < 0) 2966 return -1; 2967 thr = __kmp_threads[gtid]; 2968 team = thr->th.th_team; 2969 ii = team->t.t_level; 2970 if (level > ii) 2971 return -1; 2972 2973 if (thr->th.th_teams_microtask) { 2974 // AC: we are in teams region where multiple nested teams have same level 2975 int tlevel = thr->th.th_teams_level; // the level of the teams construct 2976 if (level <= 2977 tlevel) { // otherwise usual algorithm works (will not touch the teams) 2978 KMP_DEBUG_ASSERT(ii >= tlevel); 2979 // AC: As we need to pass by the teams league, we need to artificially 2980 // increase ii 2981 if (ii == tlevel) { 2982 ii += 2; // three teams have same level 2983 } else { 2984 ii++; // two teams have same level 2985 } 2986 } 2987 } 2988 2989 while (ii > level) { 2990 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) { 2991 } 2992 if (team->t.t_serialized && (!dd)) { 2993 team = team->t.t_parent; 2994 continue; 2995 } 2996 if (ii > level) { 2997 team = team->t.t_parent; 2998 ii--; 2999 } 3000 } 3001 3002 return team->t.t_nproc; 3003 } 3004 3005 kmp_r_sched_t __kmp_get_schedule_global() { 3006 // This routine created because pairs (__kmp_sched, __kmp_chunk) and 3007 // (__kmp_static, __kmp_guided) may be changed by kmp_set_defaults 3008 // independently. So one can get the updated schedule here. 3009 3010 kmp_r_sched_t r_sched; 3011 3012 // create schedule from 4 globals: __kmp_sched, __kmp_chunk, __kmp_static, 3013 // __kmp_guided. __kmp_sched should keep original value, so that user can set 3014 // KMP_SCHEDULE multiple times, and thus have different run-time schedules in 3015 // different roots (even in OMP 2.5) 3016 enum sched_type s = SCHEDULE_WITHOUT_MODIFIERS(__kmp_sched); 3017 enum sched_type sched_modifiers = SCHEDULE_GET_MODIFIERS(__kmp_sched); 3018 if (s == kmp_sch_static) { 3019 // replace STATIC with more detailed schedule (balanced or greedy) 3020 r_sched.r_sched_type = __kmp_static; 3021 } else if (s == kmp_sch_guided_chunked) { 3022 // replace GUIDED with more detailed schedule (iterative or analytical) 3023 r_sched.r_sched_type = __kmp_guided; 3024 } else { // (STATIC_CHUNKED), or (DYNAMIC_CHUNKED), or other 3025 r_sched.r_sched_type = __kmp_sched; 3026 } 3027 SCHEDULE_SET_MODIFIERS(r_sched.r_sched_type, sched_modifiers); 3028 3029 if (__kmp_chunk < KMP_DEFAULT_CHUNK) { 3030 // __kmp_chunk may be wrong here (if it was not ever set) 3031 r_sched.chunk = KMP_DEFAULT_CHUNK; 3032 } else { 3033 r_sched.chunk = __kmp_chunk; 3034 } 3035 3036 return r_sched; 3037 } 3038 3039 /* Allocate (realloc == FALSE) * or reallocate (realloc == TRUE) 3040 at least argc number of *t_argv entries for the requested team. */ 3041 static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team, int realloc) { 3042 3043 KMP_DEBUG_ASSERT(team); 3044 if (!realloc || argc > team->t.t_max_argc) { 3045 3046 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: needed entries=%d, " 3047 "current entries=%d\n", 3048 team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0)); 3049 /* if previously allocated heap space for args, free them */ 3050 if (realloc && team->t.t_argv != &team->t.t_inline_argv[0]) 3051 __kmp_free((void *)team->t.t_argv); 3052 3053 if (argc <= KMP_INLINE_ARGV_ENTRIES) { 3054 /* use unused space in the cache line for arguments */ 3055 team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES; 3056 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: inline allocate %d " 3057 "argv entries\n", 3058 team->t.t_id, team->t.t_max_argc)); 3059 team->t.t_argv = &team->t.t_inline_argv[0]; 3060 if (__kmp_storage_map) { 3061 __kmp_print_storage_map_gtid( 3062 -1, &team->t.t_inline_argv[0], 3063 &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES], 3064 (sizeof(void *) * KMP_INLINE_ARGV_ENTRIES), "team_%d.t_inline_argv", 3065 team->t.t_id); 3066 } 3067 } else { 3068 /* allocate space for arguments in the heap */ 3069 team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES >> 1)) 3070 ? KMP_MIN_MALLOC_ARGV_ENTRIES 3071 : 2 * argc; 3072 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d " 3073 "argv entries\n", 3074 team->t.t_id, team->t.t_max_argc)); 3075 team->t.t_argv = 3076 (void **)__kmp_page_allocate(sizeof(void *) * team->t.t_max_argc); 3077 if (__kmp_storage_map) { 3078 __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0], 3079 &team->t.t_argv[team->t.t_max_argc], 3080 sizeof(void *) * team->t.t_max_argc, 3081 "team_%d.t_argv", team->t.t_id); 3082 } 3083 } 3084 } 3085 } 3086 3087 static void __kmp_allocate_team_arrays(kmp_team_t *team, int max_nth) { 3088 int i; 3089 int num_disp_buff = max_nth > 1 ? __kmp_dispatch_num_buffers : 2; 3090 team->t.t_threads = 3091 (kmp_info_t **)__kmp_allocate(sizeof(kmp_info_t *) * max_nth); 3092 team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate( 3093 sizeof(dispatch_shared_info_t) * num_disp_buff); 3094 team->t.t_dispatch = 3095 (kmp_disp_t *)__kmp_allocate(sizeof(kmp_disp_t) * max_nth); 3096 team->t.t_implicit_task_taskdata = 3097 (kmp_taskdata_t *)__kmp_allocate(sizeof(kmp_taskdata_t) * max_nth); 3098 team->t.t_max_nproc = max_nth; 3099 3100 /* setup dispatch buffers */ 3101 for (i = 0; i < num_disp_buff; ++i) { 3102 team->t.t_disp_buffer[i].buffer_index = i; 3103 team->t.t_disp_buffer[i].doacross_buf_idx = i; 3104 } 3105 } 3106 3107 static void __kmp_free_team_arrays(kmp_team_t *team) { 3108 /* Note: this does not free the threads in t_threads (__kmp_free_threads) */ 3109 int i; 3110 for (i = 0; i < team->t.t_max_nproc; ++i) { 3111 if (team->t.t_dispatch[i].th_disp_buffer != NULL) { 3112 __kmp_free(team->t.t_dispatch[i].th_disp_buffer); 3113 team->t.t_dispatch[i].th_disp_buffer = NULL; 3114 } 3115 } 3116 #if KMP_USE_HIER_SCHED 3117 __kmp_dispatch_free_hierarchies(team); 3118 #endif 3119 __kmp_free(team->t.t_threads); 3120 __kmp_free(team->t.t_disp_buffer); 3121 __kmp_free(team->t.t_dispatch); 3122 __kmp_free(team->t.t_implicit_task_taskdata); 3123 team->t.t_threads = NULL; 3124 team->t.t_disp_buffer = NULL; 3125 team->t.t_dispatch = NULL; 3126 team->t.t_implicit_task_taskdata = 0; 3127 } 3128 3129 static void __kmp_reallocate_team_arrays(kmp_team_t *team, int max_nth) { 3130 kmp_info_t **oldThreads = team->t.t_threads; 3131 3132 __kmp_free(team->t.t_disp_buffer); 3133 __kmp_free(team->t.t_dispatch); 3134 __kmp_free(team->t.t_implicit_task_taskdata); 3135 __kmp_allocate_team_arrays(team, max_nth); 3136 3137 KMP_MEMCPY(team->t.t_threads, oldThreads, 3138 team->t.t_nproc * sizeof(kmp_info_t *)); 3139 3140 __kmp_free(oldThreads); 3141 } 3142 3143 static kmp_internal_control_t __kmp_get_global_icvs(void) { 3144 3145 kmp_r_sched_t r_sched = 3146 __kmp_get_schedule_global(); // get current state of scheduling globals 3147 3148 KMP_DEBUG_ASSERT(__kmp_nested_proc_bind.used > 0); 3149 3150 kmp_internal_control_t g_icvs = { 3151 0, // int serial_nesting_level; //corresponds to value of th_team_serialized 3152 (kmp_int8)__kmp_global.g.g_dynamic, // internal control for dynamic 3153 // adjustment of threads (per thread) 3154 (kmp_int8)__kmp_env_blocktime, // int bt_set; //internal control for 3155 // whether blocktime is explicitly set 3156 __kmp_dflt_blocktime, // int blocktime; //internal control for blocktime 3157 #if KMP_USE_MONITOR 3158 __kmp_bt_intervals, // int bt_intervals; //internal control for blocktime 3159 // intervals 3160 #endif 3161 __kmp_dflt_team_nth, // int nproc; //internal control for # of threads for 3162 // next parallel region (per thread) 3163 // (use a max ub on value if __kmp_parallel_initialize not called yet) 3164 __kmp_cg_max_nth, // int thread_limit; 3165 __kmp_dflt_max_active_levels, // int max_active_levels; //internal control 3166 // for max_active_levels 3167 r_sched, // kmp_r_sched_t sched; //internal control for runtime schedule 3168 // {sched,chunk} pair 3169 __kmp_nested_proc_bind.bind_types[0], 3170 __kmp_default_device, 3171 NULL // struct kmp_internal_control *next; 3172 }; 3173 3174 return g_icvs; 3175 } 3176 3177 static kmp_internal_control_t __kmp_get_x_global_icvs(const kmp_team_t *team) { 3178 3179 kmp_internal_control_t gx_icvs; 3180 gx_icvs.serial_nesting_level = 3181 0; // probably =team->t.t_serial like in save_inter_controls 3182 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs); 3183 gx_icvs.next = NULL; 3184 3185 return gx_icvs; 3186 } 3187 3188 static void __kmp_initialize_root(kmp_root_t *root) { 3189 int f; 3190 kmp_team_t *root_team; 3191 kmp_team_t *hot_team; 3192 int hot_team_max_nth; 3193 kmp_r_sched_t r_sched = 3194 __kmp_get_schedule_global(); // get current state of scheduling globals 3195 kmp_internal_control_t r_icvs = __kmp_get_global_icvs(); 3196 KMP_DEBUG_ASSERT(root); 3197 KMP_ASSERT(!root->r.r_begin); 3198 3199 /* setup the root state structure */ 3200 __kmp_init_lock(&root->r.r_begin_lock); 3201 root->r.r_begin = FALSE; 3202 root->r.r_active = FALSE; 3203 root->r.r_in_parallel = 0; 3204 root->r.r_blocktime = __kmp_dflt_blocktime; 3205 #if KMP_AFFINITY_SUPPORTED 3206 root->r.r_affinity_assigned = FALSE; 3207 #endif 3208 3209 /* setup the root team for this task */ 3210 /* allocate the root team structure */ 3211 KF_TRACE(10, ("__kmp_initialize_root: before root_team\n")); 3212 3213 root_team = 3214 __kmp_allocate_team(root, 3215 1, // new_nproc 3216 1, // max_nproc 3217 #if OMPT_SUPPORT 3218 ompt_data_none, // root parallel id 3219 #endif 3220 __kmp_nested_proc_bind.bind_types[0], &r_icvs, 3221 0 // argc 3222 USE_NESTED_HOT_ARG(NULL) // primary thread is unknown 3223 ); 3224 #if USE_DEBUGGER 3225 // Non-NULL value should be assigned to make the debugger display the root 3226 // team. 3227 TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)(~0)); 3228 #endif 3229 3230 KF_TRACE(10, ("__kmp_initialize_root: after root_team = %p\n", root_team)); 3231 3232 root->r.r_root_team = root_team; 3233 root_team->t.t_control_stack_top = NULL; 3234 3235 /* initialize root team */ 3236 root_team->t.t_threads[0] = NULL; 3237 root_team->t.t_nproc = 1; 3238 root_team->t.t_serialized = 1; 3239 // TODO???: root_team->t.t_max_active_levels = __kmp_dflt_max_active_levels; 3240 root_team->t.t_sched.sched = r_sched.sched; 3241 KA_TRACE( 3242 20, 3243 ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n", 3244 root_team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); 3245 3246 /* setup the hot team for this task */ 3247 /* allocate the hot team structure */ 3248 KF_TRACE(10, ("__kmp_initialize_root: before hot_team\n")); 3249 3250 hot_team = 3251 __kmp_allocate_team(root, 3252 1, // new_nproc 3253 __kmp_dflt_team_nth_ub * 2, // max_nproc 3254 #if OMPT_SUPPORT 3255 ompt_data_none, // root parallel id 3256 #endif 3257 __kmp_nested_proc_bind.bind_types[0], &r_icvs, 3258 0 // argc 3259 USE_NESTED_HOT_ARG(NULL) // primary thread is unknown 3260 ); 3261 KF_TRACE(10, ("__kmp_initialize_root: after hot_team = %p\n", hot_team)); 3262 3263 root->r.r_hot_team = hot_team; 3264 root_team->t.t_control_stack_top = NULL; 3265 3266 /* first-time initialization */ 3267 hot_team->t.t_parent = root_team; 3268 3269 /* initialize hot team */ 3270 hot_team_max_nth = hot_team->t.t_max_nproc; 3271 for (f = 0; f < hot_team_max_nth; ++f) { 3272 hot_team->t.t_threads[f] = NULL; 3273 } 3274 hot_team->t.t_nproc = 1; 3275 // TODO???: hot_team->t.t_max_active_levels = __kmp_dflt_max_active_levels; 3276 hot_team->t.t_sched.sched = r_sched.sched; 3277 hot_team->t.t_size_changed = 0; 3278 } 3279 3280 #ifdef KMP_DEBUG 3281 3282 typedef struct kmp_team_list_item { 3283 kmp_team_p const *entry; 3284 struct kmp_team_list_item *next; 3285 } kmp_team_list_item_t; 3286 typedef kmp_team_list_item_t *kmp_team_list_t; 3287 3288 static void __kmp_print_structure_team_accum( // Add team to list of teams. 3289 kmp_team_list_t list, // List of teams. 3290 kmp_team_p const *team // Team to add. 3291 ) { 3292 3293 // List must terminate with item where both entry and next are NULL. 3294 // Team is added to the list only once. 3295 // List is sorted in ascending order by team id. 3296 // Team id is *not* a key. 3297 3298 kmp_team_list_t l; 3299 3300 KMP_DEBUG_ASSERT(list != NULL); 3301 if (team == NULL) { 3302 return; 3303 } 3304 3305 __kmp_print_structure_team_accum(list, team->t.t_parent); 3306 __kmp_print_structure_team_accum(list, team->t.t_next_pool); 3307 3308 // Search list for the team. 3309 l = list; 3310 while (l->next != NULL && l->entry != team) { 3311 l = l->next; 3312 } 3313 if (l->next != NULL) { 3314 return; // Team has been added before, exit. 3315 } 3316 3317 // Team is not found. Search list again for insertion point. 3318 l = list; 3319 while (l->next != NULL && l->entry->t.t_id <= team->t.t_id) { 3320 l = l->next; 3321 } 3322 3323 // Insert team. 3324 { 3325 kmp_team_list_item_t *item = (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC( 3326 sizeof(kmp_team_list_item_t)); 3327 *item = *l; 3328 l->entry = team; 3329 l->next = item; 3330 } 3331 } 3332 3333 static void __kmp_print_structure_team(char const *title, kmp_team_p const *team 3334 3335 ) { 3336 __kmp_printf("%s", title); 3337 if (team != NULL) { 3338 __kmp_printf("%2x %p\n", team->t.t_id, team); 3339 } else { 3340 __kmp_printf(" - (nil)\n"); 3341 } 3342 } 3343 3344 static void __kmp_print_structure_thread(char const *title, 3345 kmp_info_p const *thread) { 3346 __kmp_printf("%s", title); 3347 if (thread != NULL) { 3348 __kmp_printf("%2d %p\n", thread->th.th_info.ds.ds_gtid, thread); 3349 } else { 3350 __kmp_printf(" - (nil)\n"); 3351 } 3352 } 3353 3354 void __kmp_print_structure(void) { 3355 3356 kmp_team_list_t list; 3357 3358 // Initialize list of teams. 3359 list = 3360 (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(sizeof(kmp_team_list_item_t)); 3361 list->entry = NULL; 3362 list->next = NULL; 3363 3364 __kmp_printf("\n------------------------------\nGlobal Thread " 3365 "Table\n------------------------------\n"); 3366 { 3367 int gtid; 3368 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) { 3369 __kmp_printf("%2d", gtid); 3370 if (__kmp_threads != NULL) { 3371 __kmp_printf(" %p", __kmp_threads[gtid]); 3372 } 3373 if (__kmp_root != NULL) { 3374 __kmp_printf(" %p", __kmp_root[gtid]); 3375 } 3376 __kmp_printf("\n"); 3377 } 3378 } 3379 3380 // Print out __kmp_threads array. 3381 __kmp_printf("\n------------------------------\nThreads\n--------------------" 3382 "----------\n"); 3383 if (__kmp_threads != NULL) { 3384 int gtid; 3385 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) { 3386 kmp_info_t const *thread = __kmp_threads[gtid]; 3387 if (thread != NULL) { 3388 __kmp_printf("GTID %2d %p:\n", gtid, thread); 3389 __kmp_printf(" Our Root: %p\n", thread->th.th_root); 3390 __kmp_print_structure_team(" Our Team: ", thread->th.th_team); 3391 __kmp_print_structure_team(" Serial Team: ", 3392 thread->th.th_serial_team); 3393 __kmp_printf(" Threads: %2d\n", thread->th.th_team_nproc); 3394 __kmp_print_structure_thread(" Primary: ", 3395 thread->th.th_team_master); 3396 __kmp_printf(" Serialized?: %2d\n", thread->th.th_team_serialized); 3397 __kmp_printf(" Set NProc: %2d\n", thread->th.th_set_nproc); 3398 __kmp_printf(" Set Proc Bind: %2d\n", thread->th.th_set_proc_bind); 3399 __kmp_print_structure_thread(" Next in pool: ", 3400 thread->th.th_next_pool); 3401 __kmp_printf("\n"); 3402 __kmp_print_structure_team_accum(list, thread->th.th_team); 3403 __kmp_print_structure_team_accum(list, thread->th.th_serial_team); 3404 } 3405 } 3406 } else { 3407 __kmp_printf("Threads array is not allocated.\n"); 3408 } 3409 3410 // Print out __kmp_root array. 3411 __kmp_printf("\n------------------------------\nUbers\n----------------------" 3412 "--------\n"); 3413 if (__kmp_root != NULL) { 3414 int gtid; 3415 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) { 3416 kmp_root_t const *root = __kmp_root[gtid]; 3417 if (root != NULL) { 3418 __kmp_printf("GTID %2d %p:\n", gtid, root); 3419 __kmp_print_structure_team(" Root Team: ", root->r.r_root_team); 3420 __kmp_print_structure_team(" Hot Team: ", root->r.r_hot_team); 3421 __kmp_print_structure_thread(" Uber Thread: ", 3422 root->r.r_uber_thread); 3423 __kmp_printf(" Active?: %2d\n", root->r.r_active); 3424 __kmp_printf(" In Parallel: %2d\n", 3425 KMP_ATOMIC_LD_RLX(&root->r.r_in_parallel)); 3426 __kmp_printf("\n"); 3427 __kmp_print_structure_team_accum(list, root->r.r_root_team); 3428 __kmp_print_structure_team_accum(list, root->r.r_hot_team); 3429 } 3430 } 3431 } else { 3432 __kmp_printf("Ubers array is not allocated.\n"); 3433 } 3434 3435 __kmp_printf("\n------------------------------\nTeams\n----------------------" 3436 "--------\n"); 3437 while (list->next != NULL) { 3438 kmp_team_p const *team = list->entry; 3439 int i; 3440 __kmp_printf("Team %2x %p:\n", team->t.t_id, team); 3441 __kmp_print_structure_team(" Parent Team: ", team->t.t_parent); 3442 __kmp_printf(" Primary TID: %2d\n", team->t.t_master_tid); 3443 __kmp_printf(" Max threads: %2d\n", team->t.t_max_nproc); 3444 __kmp_printf(" Levels of serial: %2d\n", team->t.t_serialized); 3445 __kmp_printf(" Number threads: %2d\n", team->t.t_nproc); 3446 for (i = 0; i < team->t.t_nproc; ++i) { 3447 __kmp_printf(" Thread %2d: ", i); 3448 __kmp_print_structure_thread("", team->t.t_threads[i]); 3449 } 3450 __kmp_print_structure_team(" Next in pool: ", team->t.t_next_pool); 3451 __kmp_printf("\n"); 3452 list = list->next; 3453 } 3454 3455 // Print out __kmp_thread_pool and __kmp_team_pool. 3456 __kmp_printf("\n------------------------------\nPools\n----------------------" 3457 "--------\n"); 3458 __kmp_print_structure_thread("Thread pool: ", 3459 CCAST(kmp_info_t *, __kmp_thread_pool)); 3460 __kmp_print_structure_team("Team pool: ", 3461 CCAST(kmp_team_t *, __kmp_team_pool)); 3462 __kmp_printf("\n"); 3463 3464 // Free team list. 3465 while (list != NULL) { 3466 kmp_team_list_item_t *item = list; 3467 list = list->next; 3468 KMP_INTERNAL_FREE(item); 3469 } 3470 } 3471 3472 #endif 3473 3474 //--------------------------------------------------------------------------- 3475 // Stuff for per-thread fast random number generator 3476 // Table of primes 3477 static const unsigned __kmp_primes[] = { 3478 0x9e3779b1, 0xffe6cc59, 0x2109f6dd, 0x43977ab5, 0xba5703f5, 0xb495a877, 3479 0xe1626741, 0x79695e6b, 0xbc98c09f, 0xd5bee2b3, 0x287488f9, 0x3af18231, 3480 0x9677cd4d, 0xbe3a6929, 0xadc6a877, 0xdcf0674b, 0xbe4d6fe9, 0x5f15e201, 3481 0x99afc3fd, 0xf3f16801, 0xe222cfff, 0x24ba5fdb, 0x0620452d, 0x79f149e3, 3482 0xc8b93f49, 0x972702cd, 0xb07dd827, 0x6c97d5ed, 0x085a3d61, 0x46eb5ea7, 3483 0x3d9910ed, 0x2e687b5b, 0x29609227, 0x6eb081f1, 0x0954c4e1, 0x9d114db9, 3484 0x542acfa9, 0xb3e6bd7b, 0x0742d917, 0xe9f3ffa7, 0x54581edb, 0xf2480f45, 3485 0x0bb9288f, 0xef1affc7, 0x85fa0ca7, 0x3ccc14db, 0xe6baf34b, 0x343377f7, 3486 0x5ca19031, 0xe6d9293b, 0xf0a9f391, 0x5d2e980b, 0xfc411073, 0xc3749363, 3487 0xb892d829, 0x3549366b, 0x629750ad, 0xb98294e5, 0x892d9483, 0xc235baf3, 3488 0x3d2402a3, 0x6bdef3c9, 0xbec333cd, 0x40c9520f}; 3489 3490 //--------------------------------------------------------------------------- 3491 // __kmp_get_random: Get a random number using a linear congruential method. 3492 unsigned short __kmp_get_random(kmp_info_t *thread) { 3493 unsigned x = thread->th.th_x; 3494 unsigned short r = (unsigned short)(x >> 16); 3495 3496 thread->th.th_x = x * thread->th.th_a + 1; 3497 3498 KA_TRACE(30, ("__kmp_get_random: THREAD: %d, RETURN: %u\n", 3499 thread->th.th_info.ds.ds_tid, r)); 3500 3501 return r; 3502 } 3503 //-------------------------------------------------------- 3504 // __kmp_init_random: Initialize a random number generator 3505 void __kmp_init_random(kmp_info_t *thread) { 3506 unsigned seed = thread->th.th_info.ds.ds_tid; 3507 3508 thread->th.th_a = 3509 __kmp_primes[seed % (sizeof(__kmp_primes) / sizeof(__kmp_primes[0]))]; 3510 thread->th.th_x = (seed + 1) * thread->th.th_a + 1; 3511 KA_TRACE(30, 3512 ("__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a)); 3513 } 3514 3515 #if KMP_OS_WINDOWS 3516 /* reclaim array entries for root threads that are already dead, returns number 3517 * reclaimed */ 3518 static int __kmp_reclaim_dead_roots(void) { 3519 int i, r = 0; 3520 3521 for (i = 0; i < __kmp_threads_capacity; ++i) { 3522 if (KMP_UBER_GTID(i) && 3523 !__kmp_still_running((kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[i])) && 3524 !__kmp_root[i] 3525 ->r.r_active) { // AC: reclaim only roots died in non-active state 3526 r += __kmp_unregister_root_other_thread(i); 3527 } 3528 } 3529 return r; 3530 } 3531 #endif 3532 3533 /* This function attempts to create free entries in __kmp_threads and 3534 __kmp_root, and returns the number of free entries generated. 3535 3536 For Windows* OS static library, the first mechanism used is to reclaim array 3537 entries for root threads that are already dead. 3538 3539 On all platforms, expansion is attempted on the arrays __kmp_threads_ and 3540 __kmp_root, with appropriate update to __kmp_threads_capacity. Array 3541 capacity is increased by doubling with clipping to __kmp_tp_capacity, if 3542 threadprivate cache array has been created. Synchronization with 3543 __kmpc_threadprivate_cached is done using __kmp_tp_cached_lock. 3544 3545 After any dead root reclamation, if the clipping value allows array expansion 3546 to result in the generation of a total of nNeed free slots, the function does 3547 that expansion. If not, nothing is done beyond the possible initial root 3548 thread reclamation. 3549 3550 If any argument is negative, the behavior is undefined. */ 3551 static int __kmp_expand_threads(int nNeed) { 3552 int added = 0; 3553 int minimumRequiredCapacity; 3554 int newCapacity; 3555 kmp_info_t **newThreads; 3556 kmp_root_t **newRoot; 3557 3558 // All calls to __kmp_expand_threads should be under __kmp_forkjoin_lock, so 3559 // resizing __kmp_threads does not need additional protection if foreign 3560 // threads are present 3561 3562 #if KMP_OS_WINDOWS && !KMP_DYNAMIC_LIB 3563 /* only for Windows static library */ 3564 /* reclaim array entries for root threads that are already dead */ 3565 added = __kmp_reclaim_dead_roots(); 3566 3567 if (nNeed) { 3568 nNeed -= added; 3569 if (nNeed < 0) 3570 nNeed = 0; 3571 } 3572 #endif 3573 if (nNeed <= 0) 3574 return added; 3575 3576 // Note that __kmp_threads_capacity is not bounded by __kmp_max_nth. If 3577 // __kmp_max_nth is set to some value less than __kmp_sys_max_nth by the 3578 // user via KMP_DEVICE_THREAD_LIMIT, then __kmp_threads_capacity may become 3579 // > __kmp_max_nth in one of two ways: 3580 // 3581 // 1) The initialization thread (gtid = 0) exits. __kmp_threads[0] 3582 // may not be reused by another thread, so we may need to increase 3583 // __kmp_threads_capacity to __kmp_max_nth + 1. 3584 // 3585 // 2) New foreign root(s) are encountered. We always register new foreign 3586 // roots. This may cause a smaller # of threads to be allocated at 3587 // subsequent parallel regions, but the worker threads hang around (and 3588 // eventually go to sleep) and need slots in the __kmp_threads[] array. 3589 // 3590 // Anyway, that is the reason for moving the check to see if 3591 // __kmp_max_nth was exceeded into __kmp_reserve_threads() 3592 // instead of having it performed here. -BB 3593 3594 KMP_DEBUG_ASSERT(__kmp_sys_max_nth >= __kmp_threads_capacity); 3595 3596 /* compute expansion headroom to check if we can expand */ 3597 if (__kmp_sys_max_nth - __kmp_threads_capacity < nNeed) { 3598 /* possible expansion too small -- give up */ 3599 return added; 3600 } 3601 minimumRequiredCapacity = __kmp_threads_capacity + nNeed; 3602 3603 newCapacity = __kmp_threads_capacity; 3604 do { 3605 newCapacity = newCapacity <= (__kmp_sys_max_nth >> 1) ? (newCapacity << 1) 3606 : __kmp_sys_max_nth; 3607 } while (newCapacity < minimumRequiredCapacity); 3608 newThreads = (kmp_info_t **)__kmp_allocate( 3609 (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * newCapacity + CACHE_LINE); 3610 newRoot = 3611 (kmp_root_t **)((char *)newThreads + sizeof(kmp_info_t *) * newCapacity); 3612 KMP_MEMCPY(newThreads, __kmp_threads, 3613 __kmp_threads_capacity * sizeof(kmp_info_t *)); 3614 KMP_MEMCPY(newRoot, __kmp_root, 3615 __kmp_threads_capacity * sizeof(kmp_root_t *)); 3616 3617 kmp_info_t **temp_threads = __kmp_threads; 3618 *(kmp_info_t * *volatile *)&__kmp_threads = newThreads; 3619 *(kmp_root_t * *volatile *)&__kmp_root = newRoot; 3620 __kmp_free(temp_threads); 3621 added += newCapacity - __kmp_threads_capacity; 3622 *(volatile int *)&__kmp_threads_capacity = newCapacity; 3623 3624 if (newCapacity > __kmp_tp_capacity) { 3625 __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock); 3626 if (__kmp_tp_cached && newCapacity > __kmp_tp_capacity) { 3627 __kmp_threadprivate_resize_cache(newCapacity); 3628 } else { // increase __kmp_tp_capacity to correspond with kmp_threads size 3629 *(volatile int *)&__kmp_tp_capacity = newCapacity; 3630 } 3631 __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock); 3632 } 3633 3634 return added; 3635 } 3636 3637 /* Register the current thread as a root thread and obtain our gtid. We must 3638 have the __kmp_initz_lock held at this point. Argument TRUE only if are the 3639 thread that calls from __kmp_do_serial_initialize() */ 3640 int __kmp_register_root(int initial_thread) { 3641 kmp_info_t *root_thread; 3642 kmp_root_t *root; 3643 int gtid; 3644 int capacity; 3645 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 3646 KA_TRACE(20, ("__kmp_register_root: entered\n")); 3647 KMP_MB(); 3648 3649 /* 2007-03-02: 3650 If initial thread did not invoke OpenMP RTL yet, and this thread is not an 3651 initial one, "__kmp_all_nth >= __kmp_threads_capacity" condition does not 3652 work as expected -- it may return false (that means there is at least one 3653 empty slot in __kmp_threads array), but it is possible the only free slot 3654 is #0, which is reserved for initial thread and so cannot be used for this 3655 one. Following code workarounds this bug. 3656 3657 However, right solution seems to be not reserving slot #0 for initial 3658 thread because: 3659 (1) there is no magic in slot #0, 3660 (2) we cannot detect initial thread reliably (the first thread which does 3661 serial initialization may be not a real initial thread). 3662 */ 3663 capacity = __kmp_threads_capacity; 3664 if (!initial_thread && TCR_PTR(__kmp_threads[0]) == NULL) { 3665 --capacity; 3666 } 3667 3668 // If it is not for initializing the hidden helper team, we need to take 3669 // __kmp_hidden_helper_threads_num out of the capacity because it is included 3670 // in __kmp_threads_capacity. 3671 if (__kmp_enable_hidden_helper && !TCR_4(__kmp_init_hidden_helper_threads)) { 3672 capacity -= __kmp_hidden_helper_threads_num; 3673 } 3674 3675 /* see if there are too many threads */ 3676 if (__kmp_all_nth >= capacity && !__kmp_expand_threads(1)) { 3677 if (__kmp_tp_cached) { 3678 __kmp_fatal(KMP_MSG(CantRegisterNewThread), 3679 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity), 3680 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null); 3681 } else { 3682 __kmp_fatal(KMP_MSG(CantRegisterNewThread), KMP_HNT(SystemLimitOnThreads), 3683 __kmp_msg_null); 3684 } 3685 } 3686 3687 // When hidden helper task is enabled, __kmp_threads is organized as follows: 3688 // 0: initial thread, also a regular OpenMP thread. 3689 // [1, __kmp_hidden_helper_threads_num]: slots for hidden helper threads. 3690 // [__kmp_hidden_helper_threads_num + 1, __kmp_threads_capacity): slots for 3691 // regular OpenMP threads. 3692 if (TCR_4(__kmp_init_hidden_helper_threads)) { 3693 // Find an available thread slot for hidden helper thread. Slots for hidden 3694 // helper threads start from 1 to __kmp_hidden_helper_threads_num. 3695 for (gtid = 1; TCR_PTR(__kmp_threads[gtid]) != NULL && 3696 gtid <= __kmp_hidden_helper_threads_num; 3697 gtid++) 3698 ; 3699 KMP_ASSERT(gtid <= __kmp_hidden_helper_threads_num); 3700 KA_TRACE(1, ("__kmp_register_root: found slot in threads array for " 3701 "hidden helper thread: T#%d\n", 3702 gtid)); 3703 } else { 3704 /* find an available thread slot */ 3705 // Don't reassign the zero slot since we need that to only be used by 3706 // initial thread. Slots for hidden helper threads should also be skipped. 3707 if (initial_thread && TCR_PTR(__kmp_threads[0]) == NULL) { 3708 gtid = 0; 3709 } else { 3710 for (gtid = __kmp_hidden_helper_threads_num + 1; 3711 TCR_PTR(__kmp_threads[gtid]) != NULL; gtid++) 3712 ; 3713 } 3714 KA_TRACE( 3715 1, ("__kmp_register_root: found slot in threads array: T#%d\n", gtid)); 3716 KMP_ASSERT(gtid < __kmp_threads_capacity); 3717 } 3718 3719 /* update global accounting */ 3720 __kmp_all_nth++; 3721 TCW_4(__kmp_nth, __kmp_nth + 1); 3722 3723 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low 3724 // numbers of procs, and method #2 (keyed API call) for higher numbers. 3725 if (__kmp_adjust_gtid_mode) { 3726 if (__kmp_all_nth >= __kmp_tls_gtid_min) { 3727 if (TCR_4(__kmp_gtid_mode) != 2) { 3728 TCW_4(__kmp_gtid_mode, 2); 3729 } 3730 } else { 3731 if (TCR_4(__kmp_gtid_mode) != 1) { 3732 TCW_4(__kmp_gtid_mode, 1); 3733 } 3734 } 3735 } 3736 3737 #ifdef KMP_ADJUST_BLOCKTIME 3738 /* Adjust blocktime to zero if necessary */ 3739 /* Middle initialization might not have occurred yet */ 3740 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 3741 if (__kmp_nth > __kmp_avail_proc) { 3742 __kmp_zero_bt = TRUE; 3743 } 3744 } 3745 #endif /* KMP_ADJUST_BLOCKTIME */ 3746 3747 /* setup this new hierarchy */ 3748 if (!(root = __kmp_root[gtid])) { 3749 root = __kmp_root[gtid] = (kmp_root_t *)__kmp_allocate(sizeof(kmp_root_t)); 3750 KMP_DEBUG_ASSERT(!root->r.r_root_team); 3751 } 3752 3753 #if KMP_STATS_ENABLED 3754 // Initialize stats as soon as possible (right after gtid assignment). 3755 __kmp_stats_thread_ptr = __kmp_stats_list->push_back(gtid); 3756 __kmp_stats_thread_ptr->startLife(); 3757 KMP_SET_THREAD_STATE(SERIAL_REGION); 3758 KMP_INIT_PARTITIONED_TIMERS(OMP_serial); 3759 #endif 3760 __kmp_initialize_root(root); 3761 3762 /* setup new root thread structure */ 3763 if (root->r.r_uber_thread) { 3764 root_thread = root->r.r_uber_thread; 3765 } else { 3766 root_thread = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t)); 3767 if (__kmp_storage_map) { 3768 __kmp_print_thread_storage_map(root_thread, gtid); 3769 } 3770 root_thread->th.th_info.ds.ds_gtid = gtid; 3771 #if OMPT_SUPPORT 3772 root_thread->th.ompt_thread_info.thread_data = ompt_data_none; 3773 #endif 3774 root_thread->th.th_root = root; 3775 if (__kmp_env_consistency_check) { 3776 root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid); 3777 } 3778 #if USE_FAST_MEMORY 3779 __kmp_initialize_fast_memory(root_thread); 3780 #endif /* USE_FAST_MEMORY */ 3781 3782 #if KMP_USE_BGET 3783 KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL); 3784 __kmp_initialize_bget(root_thread); 3785 #endif 3786 __kmp_init_random(root_thread); // Initialize random number generator 3787 } 3788 3789 /* setup the serial team held in reserve by the root thread */ 3790 if (!root_thread->th.th_serial_team) { 3791 kmp_internal_control_t r_icvs = __kmp_get_global_icvs(); 3792 KF_TRACE(10, ("__kmp_register_root: before serial_team\n")); 3793 root_thread->th.th_serial_team = __kmp_allocate_team( 3794 root, 1, 1, 3795 #if OMPT_SUPPORT 3796 ompt_data_none, // root parallel id 3797 #endif 3798 proc_bind_default, &r_icvs, 0 USE_NESTED_HOT_ARG(NULL)); 3799 } 3800 KMP_ASSERT(root_thread->th.th_serial_team); 3801 KF_TRACE(10, ("__kmp_register_root: after serial_team = %p\n", 3802 root_thread->th.th_serial_team)); 3803 3804 /* drop root_thread into place */ 3805 TCW_SYNC_PTR(__kmp_threads[gtid], root_thread); 3806 3807 root->r.r_root_team->t.t_threads[0] = root_thread; 3808 root->r.r_hot_team->t.t_threads[0] = root_thread; 3809 root_thread->th.th_serial_team->t.t_threads[0] = root_thread; 3810 // AC: the team created in reserve, not for execution (it is unused for now). 3811 root_thread->th.th_serial_team->t.t_serialized = 0; 3812 root->r.r_uber_thread = root_thread; 3813 3814 /* initialize the thread, get it ready to go */ 3815 __kmp_initialize_info(root_thread, root->r.r_root_team, 0, gtid); 3816 TCW_4(__kmp_init_gtid, TRUE); 3817 3818 /* prepare the primary thread for get_gtid() */ 3819 __kmp_gtid_set_specific(gtid); 3820 3821 #if USE_ITT_BUILD 3822 __kmp_itt_thread_name(gtid); 3823 #endif /* USE_ITT_BUILD */ 3824 3825 #ifdef KMP_TDATA_GTID 3826 __kmp_gtid = gtid; 3827 #endif 3828 __kmp_create_worker(gtid, root_thread, __kmp_stksize); 3829 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == gtid); 3830 3831 KA_TRACE(20, ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, " 3832 "plain=%u\n", 3833 gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team), 3834 root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE, 3835 KMP_INIT_BARRIER_STATE)); 3836 { // Initialize barrier data. 3837 int b; 3838 for (b = 0; b < bs_last_barrier; ++b) { 3839 root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE; 3840 #if USE_DEBUGGER 3841 root_thread->th.th_bar[b].bb.b_worker_arrived = 0; 3842 #endif 3843 } 3844 } 3845 KMP_DEBUG_ASSERT(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived == 3846 KMP_INIT_BARRIER_STATE); 3847 3848 #if KMP_AFFINITY_SUPPORTED 3849 root_thread->th.th_current_place = KMP_PLACE_UNDEFINED; 3850 root_thread->th.th_new_place = KMP_PLACE_UNDEFINED; 3851 root_thread->th.th_first_place = KMP_PLACE_UNDEFINED; 3852 root_thread->th.th_last_place = KMP_PLACE_UNDEFINED; 3853 #endif /* KMP_AFFINITY_SUPPORTED */ 3854 root_thread->th.th_def_allocator = __kmp_def_allocator; 3855 root_thread->th.th_prev_level = 0; 3856 root_thread->th.th_prev_num_threads = 1; 3857 3858 kmp_cg_root_t *tmp = (kmp_cg_root_t *)__kmp_allocate(sizeof(kmp_cg_root_t)); 3859 tmp->cg_root = root_thread; 3860 tmp->cg_thread_limit = __kmp_cg_max_nth; 3861 tmp->cg_nthreads = 1; 3862 KA_TRACE(100, ("__kmp_register_root: Thread %p created node %p with" 3863 " cg_nthreads init to 1\n", 3864 root_thread, tmp)); 3865 tmp->up = NULL; 3866 root_thread->th.th_cg_roots = tmp; 3867 3868 __kmp_root_counter++; 3869 3870 #if OMPT_SUPPORT 3871 if (!initial_thread && ompt_enabled.enabled) { 3872 3873 kmp_info_t *root_thread = ompt_get_thread(); 3874 3875 ompt_set_thread_state(root_thread, ompt_state_overhead); 3876 3877 if (ompt_enabled.ompt_callback_thread_begin) { 3878 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)( 3879 ompt_thread_initial, __ompt_get_thread_data_internal()); 3880 } 3881 ompt_data_t *task_data; 3882 ompt_data_t *parallel_data; 3883 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, ¶llel_data, 3884 NULL); 3885 if (ompt_enabled.ompt_callback_implicit_task) { 3886 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 3887 ompt_scope_begin, parallel_data, task_data, 1, 1, ompt_task_initial); 3888 } 3889 3890 ompt_set_thread_state(root_thread, ompt_state_work_serial); 3891 } 3892 #endif 3893 #if OMPD_SUPPORT 3894 if (ompd_state & OMPD_ENABLE_BP) 3895 ompd_bp_thread_begin(); 3896 #endif 3897 3898 KMP_MB(); 3899 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 3900 3901 return gtid; 3902 } 3903 3904 #if KMP_NESTED_HOT_TEAMS 3905 static int __kmp_free_hot_teams(kmp_root_t *root, kmp_info_t *thr, int level, 3906 const int max_level) { 3907 int i, n, nth; 3908 kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams; 3909 if (!hot_teams || !hot_teams[level].hot_team) { 3910 return 0; 3911 } 3912 KMP_DEBUG_ASSERT(level < max_level); 3913 kmp_team_t *team = hot_teams[level].hot_team; 3914 nth = hot_teams[level].hot_team_nth; 3915 n = nth - 1; // primary thread is not freed 3916 if (level < max_level - 1) { 3917 for (i = 0; i < nth; ++i) { 3918 kmp_info_t *th = team->t.t_threads[i]; 3919 n += __kmp_free_hot_teams(root, th, level + 1, max_level); 3920 if (i > 0 && th->th.th_hot_teams) { 3921 __kmp_free(th->th.th_hot_teams); 3922 th->th.th_hot_teams = NULL; 3923 } 3924 } 3925 } 3926 __kmp_free_team(root, team, NULL); 3927 return n; 3928 } 3929 #endif 3930 3931 // Resets a root thread and clear its root and hot teams. 3932 // Returns the number of __kmp_threads entries directly and indirectly freed. 3933 static int __kmp_reset_root(int gtid, kmp_root_t *root) { 3934 kmp_team_t *root_team = root->r.r_root_team; 3935 kmp_team_t *hot_team = root->r.r_hot_team; 3936 int n = hot_team->t.t_nproc; 3937 int i; 3938 3939 KMP_DEBUG_ASSERT(!root->r.r_active); 3940 3941 root->r.r_root_team = NULL; 3942 root->r.r_hot_team = NULL; 3943 // __kmp_free_team() does not free hot teams, so we have to clear r_hot_team 3944 // before call to __kmp_free_team(). 3945 __kmp_free_team(root, root_team USE_NESTED_HOT_ARG(NULL)); 3946 #if KMP_NESTED_HOT_TEAMS 3947 if (__kmp_hot_teams_max_level > 3948 0) { // need to free nested hot teams and their threads if any 3949 for (i = 0; i < hot_team->t.t_nproc; ++i) { 3950 kmp_info_t *th = hot_team->t.t_threads[i]; 3951 if (__kmp_hot_teams_max_level > 1) { 3952 n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level); 3953 } 3954 if (th->th.th_hot_teams) { 3955 __kmp_free(th->th.th_hot_teams); 3956 th->th.th_hot_teams = NULL; 3957 } 3958 } 3959 } 3960 #endif 3961 __kmp_free_team(root, hot_team USE_NESTED_HOT_ARG(NULL)); 3962 3963 // Before we can reap the thread, we need to make certain that all other 3964 // threads in the teams that had this root as ancestor have stopped trying to 3965 // steal tasks. 3966 if (__kmp_tasking_mode != tskm_immediate_exec) { 3967 __kmp_wait_to_unref_task_teams(); 3968 } 3969 3970 #if KMP_OS_WINDOWS 3971 /* Close Handle of root duplicated in __kmp_create_worker (tr #62919) */ 3972 KA_TRACE( 3973 10, ("__kmp_reset_root: free handle, th = %p, handle = %" KMP_UINTPTR_SPEC 3974 "\n", 3975 (LPVOID) & (root->r.r_uber_thread->th), 3976 root->r.r_uber_thread->th.th_info.ds.ds_thread)); 3977 __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread); 3978 #endif /* KMP_OS_WINDOWS */ 3979 3980 #if OMPD_SUPPORT 3981 if (ompd_state & OMPD_ENABLE_BP) 3982 ompd_bp_thread_end(); 3983 #endif 3984 3985 #if OMPT_SUPPORT 3986 ompt_data_t *task_data; 3987 ompt_data_t *parallel_data; 3988 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, ¶llel_data, 3989 NULL); 3990 if (ompt_enabled.ompt_callback_implicit_task) { 3991 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 3992 ompt_scope_end, parallel_data, task_data, 0, 1, ompt_task_initial); 3993 } 3994 if (ompt_enabled.ompt_callback_thread_end) { 3995 ompt_callbacks.ompt_callback(ompt_callback_thread_end)( 3996 &(root->r.r_uber_thread->th.ompt_thread_info.thread_data)); 3997 } 3998 #endif 3999 4000 TCW_4(__kmp_nth, 4001 __kmp_nth - 1); // __kmp_reap_thread will decrement __kmp_all_nth. 4002 i = root->r.r_uber_thread->th.th_cg_roots->cg_nthreads--; 4003 KA_TRACE(100, ("__kmp_reset_root: Thread %p decrement cg_nthreads on node %p" 4004 " to %d\n", 4005 root->r.r_uber_thread, root->r.r_uber_thread->th.th_cg_roots, 4006 root->r.r_uber_thread->th.th_cg_roots->cg_nthreads)); 4007 if (i == 1) { 4008 // need to free contention group structure 4009 KMP_DEBUG_ASSERT(root->r.r_uber_thread == 4010 root->r.r_uber_thread->th.th_cg_roots->cg_root); 4011 KMP_DEBUG_ASSERT(root->r.r_uber_thread->th.th_cg_roots->up == NULL); 4012 __kmp_free(root->r.r_uber_thread->th.th_cg_roots); 4013 root->r.r_uber_thread->th.th_cg_roots = NULL; 4014 } 4015 __kmp_reap_thread(root->r.r_uber_thread, 1); 4016 4017 // We canot put root thread to __kmp_thread_pool, so we have to reap it 4018 // instead of freeing. 4019 root->r.r_uber_thread = NULL; 4020 /* mark root as no longer in use */ 4021 root->r.r_begin = FALSE; 4022 4023 return n; 4024 } 4025 4026 void __kmp_unregister_root_current_thread(int gtid) { 4027 KA_TRACE(1, ("__kmp_unregister_root_current_thread: enter T#%d\n", gtid)); 4028 /* this lock should be ok, since unregister_root_current_thread is never 4029 called during an abort, only during a normal close. furthermore, if you 4030 have the forkjoin lock, you should never try to get the initz lock */ 4031 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 4032 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 4033 KC_TRACE(10, ("__kmp_unregister_root_current_thread: already finished, " 4034 "exiting T#%d\n", 4035 gtid)); 4036 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 4037 return; 4038 } 4039 kmp_root_t *root = __kmp_root[gtid]; 4040 4041 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]); 4042 KMP_ASSERT(KMP_UBER_GTID(gtid)); 4043 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root); 4044 KMP_ASSERT(root->r.r_active == FALSE); 4045 4046 KMP_MB(); 4047 4048 kmp_info_t *thread = __kmp_threads[gtid]; 4049 kmp_team_t *team = thread->th.th_team; 4050 kmp_task_team_t *task_team = thread->th.th_task_team; 4051 4052 // we need to wait for the proxy tasks before finishing the thread 4053 if (task_team != NULL && task_team->tt.tt_found_proxy_tasks) { 4054 #if OMPT_SUPPORT 4055 // the runtime is shutting down so we won't report any events 4056 thread->th.ompt_thread_info.state = ompt_state_undefined; 4057 #endif 4058 __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL)); 4059 } 4060 4061 __kmp_reset_root(gtid, root); 4062 4063 KMP_MB(); 4064 KC_TRACE(10, 4065 ("__kmp_unregister_root_current_thread: T#%d unregistered\n", gtid)); 4066 4067 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 4068 } 4069 4070 #if KMP_OS_WINDOWS 4071 /* __kmp_forkjoin_lock must be already held 4072 Unregisters a root thread that is not the current thread. Returns the number 4073 of __kmp_threads entries freed as a result. */ 4074 static int __kmp_unregister_root_other_thread(int gtid) { 4075 kmp_root_t *root = __kmp_root[gtid]; 4076 int r; 4077 4078 KA_TRACE(1, ("__kmp_unregister_root_other_thread: enter T#%d\n", gtid)); 4079 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]); 4080 KMP_ASSERT(KMP_UBER_GTID(gtid)); 4081 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root); 4082 KMP_ASSERT(root->r.r_active == FALSE); 4083 4084 r = __kmp_reset_root(gtid, root); 4085 KC_TRACE(10, 4086 ("__kmp_unregister_root_other_thread: T#%d unregistered\n", gtid)); 4087 return r; 4088 } 4089 #endif 4090 4091 #if KMP_DEBUG 4092 void __kmp_task_info() { 4093 4094 kmp_int32 gtid = __kmp_entry_gtid(); 4095 kmp_int32 tid = __kmp_tid_from_gtid(gtid); 4096 kmp_info_t *this_thr = __kmp_threads[gtid]; 4097 kmp_team_t *steam = this_thr->th.th_serial_team; 4098 kmp_team_t *team = this_thr->th.th_team; 4099 4100 __kmp_printf( 4101 "__kmp_task_info: gtid=%d tid=%d t_thread=%p team=%p steam=%p curtask=%p " 4102 "ptask=%p\n", 4103 gtid, tid, this_thr, team, steam, this_thr->th.th_current_task, 4104 team->t.t_implicit_task_taskdata[tid].td_parent); 4105 } 4106 #endif // KMP_DEBUG 4107 4108 /* TODO optimize with one big memclr, take out what isn't needed, split 4109 responsibility to workers as much as possible, and delay initialization of 4110 features as much as possible */ 4111 static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team, 4112 int tid, int gtid) { 4113 /* this_thr->th.th_info.ds.ds_gtid is setup in 4114 kmp_allocate_thread/create_worker. 4115 this_thr->th.th_serial_team is setup in __kmp_allocate_thread */ 4116 KMP_DEBUG_ASSERT(this_thr != NULL); 4117 KMP_DEBUG_ASSERT(this_thr->th.th_serial_team); 4118 KMP_DEBUG_ASSERT(team); 4119 KMP_DEBUG_ASSERT(team->t.t_threads); 4120 KMP_DEBUG_ASSERT(team->t.t_dispatch); 4121 kmp_info_t *master = team->t.t_threads[0]; 4122 KMP_DEBUG_ASSERT(master); 4123 KMP_DEBUG_ASSERT(master->th.th_root); 4124 4125 KMP_MB(); 4126 4127 TCW_SYNC_PTR(this_thr->th.th_team, team); 4128 4129 this_thr->th.th_info.ds.ds_tid = tid; 4130 this_thr->th.th_set_nproc = 0; 4131 if (__kmp_tasking_mode != tskm_immediate_exec) 4132 // When tasking is possible, threads are not safe to reap until they are 4133 // done tasking; this will be set when tasking code is exited in wait 4134 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP; 4135 else // no tasking --> always safe to reap 4136 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP; 4137 this_thr->th.th_set_proc_bind = proc_bind_default; 4138 #if KMP_AFFINITY_SUPPORTED 4139 this_thr->th.th_new_place = this_thr->th.th_current_place; 4140 #endif 4141 this_thr->th.th_root = master->th.th_root; 4142 4143 /* setup the thread's cache of the team structure */ 4144 this_thr->th.th_team_nproc = team->t.t_nproc; 4145 this_thr->th.th_team_master = master; 4146 this_thr->th.th_team_serialized = team->t.t_serialized; 4147 4148 KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata); 4149 4150 KF_TRACE(10, ("__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n", 4151 tid, gtid, this_thr, this_thr->th.th_current_task)); 4152 4153 __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr, 4154 team, tid, TRUE); 4155 4156 KF_TRACE(10, ("__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n", 4157 tid, gtid, this_thr, this_thr->th.th_current_task)); 4158 // TODO: Initialize ICVs from parent; GEH - isn't that already done in 4159 // __kmp_initialize_team()? 4160 4161 /* TODO no worksharing in speculative threads */ 4162 this_thr->th.th_dispatch = &team->t.t_dispatch[tid]; 4163 4164 this_thr->th.th_local.this_construct = 0; 4165 4166 if (!this_thr->th.th_pri_common) { 4167 this_thr->th.th_pri_common = 4168 (struct common_table *)__kmp_allocate(sizeof(struct common_table)); 4169 if (__kmp_storage_map) { 4170 __kmp_print_storage_map_gtid( 4171 gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1, 4172 sizeof(struct common_table), "th_%d.th_pri_common\n", gtid); 4173 } 4174 this_thr->th.th_pri_head = NULL; 4175 } 4176 4177 if (this_thr != master && // Primary thread's CG root is initialized elsewhere 4178 this_thr->th.th_cg_roots != master->th.th_cg_roots) { // CG root not set 4179 // Make new thread's CG root same as primary thread's 4180 KMP_DEBUG_ASSERT(master->th.th_cg_roots); 4181 kmp_cg_root_t *tmp = this_thr->th.th_cg_roots; 4182 if (tmp) { 4183 // worker changes CG, need to check if old CG should be freed 4184 int i = tmp->cg_nthreads--; 4185 KA_TRACE(100, ("__kmp_initialize_info: Thread %p decrement cg_nthreads" 4186 " on node %p of thread %p to %d\n", 4187 this_thr, tmp, tmp->cg_root, tmp->cg_nthreads)); 4188 if (i == 1) { 4189 __kmp_free(tmp); // last thread left CG --> free it 4190 } 4191 } 4192 this_thr->th.th_cg_roots = master->th.th_cg_roots; 4193 // Increment new thread's CG root's counter to add the new thread 4194 this_thr->th.th_cg_roots->cg_nthreads++; 4195 KA_TRACE(100, ("__kmp_initialize_info: Thread %p increment cg_nthreads on" 4196 " node %p of thread %p to %d\n", 4197 this_thr, this_thr->th.th_cg_roots, 4198 this_thr->th.th_cg_roots->cg_root, 4199 this_thr->th.th_cg_roots->cg_nthreads)); 4200 this_thr->th.th_current_task->td_icvs.thread_limit = 4201 this_thr->th.th_cg_roots->cg_thread_limit; 4202 } 4203 4204 /* Initialize dynamic dispatch */ 4205 { 4206 volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch; 4207 // Use team max_nproc since this will never change for the team. 4208 size_t disp_size = 4209 sizeof(dispatch_private_info_t) * 4210 (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers); 4211 KD_TRACE(10, ("__kmp_initialize_info: T#%d max_nproc: %d\n", gtid, 4212 team->t.t_max_nproc)); 4213 KMP_ASSERT(dispatch); 4214 KMP_DEBUG_ASSERT(team->t.t_dispatch); 4215 KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid]); 4216 4217 dispatch->th_disp_index = 0; 4218 dispatch->th_doacross_buf_idx = 0; 4219 if (!dispatch->th_disp_buffer) { 4220 dispatch->th_disp_buffer = 4221 (dispatch_private_info_t *)__kmp_allocate(disp_size); 4222 4223 if (__kmp_storage_map) { 4224 __kmp_print_storage_map_gtid( 4225 gtid, &dispatch->th_disp_buffer[0], 4226 &dispatch->th_disp_buffer[team->t.t_max_nproc == 1 4227 ? 1 4228 : __kmp_dispatch_num_buffers], 4229 disp_size, 4230 "th_%d.th_dispatch.th_disp_buffer " 4231 "(team_%d.t_dispatch[%d].th_disp_buffer)", 4232 gtid, team->t.t_id, gtid); 4233 } 4234 } else { 4235 memset(&dispatch->th_disp_buffer[0], '\0', disp_size); 4236 } 4237 4238 dispatch->th_dispatch_pr_current = 0; 4239 dispatch->th_dispatch_sh_current = 0; 4240 4241 dispatch->th_deo_fcn = 0; /* ORDERED */ 4242 dispatch->th_dxo_fcn = 0; /* END ORDERED */ 4243 } 4244 4245 this_thr->th.th_next_pool = NULL; 4246 4247 if (!this_thr->th.th_task_state_memo_stack) { 4248 size_t i; 4249 this_thr->th.th_task_state_memo_stack = 4250 (kmp_uint8 *)__kmp_allocate(4 * sizeof(kmp_uint8)); 4251 this_thr->th.th_task_state_top = 0; 4252 this_thr->th.th_task_state_stack_sz = 4; 4253 for (i = 0; i < this_thr->th.th_task_state_stack_sz; 4254 ++i) // zero init the stack 4255 this_thr->th.th_task_state_memo_stack[i] = 0; 4256 } 4257 4258 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here); 4259 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0); 4260 4261 KMP_MB(); 4262 } 4263 4264 /* allocate a new thread for the requesting team. this is only called from 4265 within a forkjoin critical section. we will first try to get an available 4266 thread from the thread pool. if none is available, we will fork a new one 4267 assuming we are able to create a new one. this should be assured, as the 4268 caller should check on this first. */ 4269 kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team, 4270 int new_tid) { 4271 kmp_team_t *serial_team; 4272 kmp_info_t *new_thr; 4273 int new_gtid; 4274 4275 KA_TRACE(20, ("__kmp_allocate_thread: T#%d\n", __kmp_get_gtid())); 4276 KMP_DEBUG_ASSERT(root && team); 4277 #if !KMP_NESTED_HOT_TEAMS 4278 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(__kmp_get_gtid())); 4279 #endif 4280 KMP_MB(); 4281 4282 /* first, try to get one from the thread pool */ 4283 if (__kmp_thread_pool) { 4284 new_thr = CCAST(kmp_info_t *, __kmp_thread_pool); 4285 __kmp_thread_pool = (volatile kmp_info_t *)new_thr->th.th_next_pool; 4286 if (new_thr == __kmp_thread_pool_insert_pt) { 4287 __kmp_thread_pool_insert_pt = NULL; 4288 } 4289 TCW_4(new_thr->th.th_in_pool, FALSE); 4290 __kmp_suspend_initialize_thread(new_thr); 4291 __kmp_lock_suspend_mx(new_thr); 4292 if (new_thr->th.th_active_in_pool == TRUE) { 4293 KMP_DEBUG_ASSERT(new_thr->th.th_active == TRUE); 4294 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth); 4295 new_thr->th.th_active_in_pool = FALSE; 4296 } 4297 __kmp_unlock_suspend_mx(new_thr); 4298 4299 KA_TRACE(20, ("__kmp_allocate_thread: T#%d using thread T#%d\n", 4300 __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid)); 4301 KMP_ASSERT(!new_thr->th.th_team); 4302 KMP_DEBUG_ASSERT(__kmp_nth < __kmp_threads_capacity); 4303 4304 /* setup the thread structure */ 4305 __kmp_initialize_info(new_thr, team, new_tid, 4306 new_thr->th.th_info.ds.ds_gtid); 4307 KMP_DEBUG_ASSERT(new_thr->th.th_serial_team); 4308 4309 TCW_4(__kmp_nth, __kmp_nth + 1); 4310 4311 new_thr->th.th_task_state = 0; 4312 new_thr->th.th_task_state_top = 0; 4313 new_thr->th.th_task_state_stack_sz = 4; 4314 4315 if (__kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) { 4316 // Make sure pool thread has transitioned to waiting on own thread struct 4317 KMP_DEBUG_ASSERT(new_thr->th.th_used_in_team.load() == 0); 4318 // Thread activated in __kmp_allocate_team when increasing team size 4319 } 4320 4321 #ifdef KMP_ADJUST_BLOCKTIME 4322 /* Adjust blocktime back to zero if necessary */ 4323 /* Middle initialization might not have occurred yet */ 4324 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 4325 if (__kmp_nth > __kmp_avail_proc) { 4326 __kmp_zero_bt = TRUE; 4327 } 4328 } 4329 #endif /* KMP_ADJUST_BLOCKTIME */ 4330 4331 #if KMP_DEBUG 4332 // If thread entered pool via __kmp_free_thread, wait_flag should != 4333 // KMP_BARRIER_PARENT_FLAG. 4334 int b; 4335 kmp_balign_t *balign = new_thr->th.th_bar; 4336 for (b = 0; b < bs_last_barrier; ++b) 4337 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 4338 #endif 4339 4340 KF_TRACE(10, ("__kmp_allocate_thread: T#%d using thread %p T#%d\n", 4341 __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid)); 4342 4343 KMP_MB(); 4344 return new_thr; 4345 } 4346 4347 /* no, well fork a new one */ 4348 KMP_ASSERT(__kmp_nth == __kmp_all_nth); 4349 KMP_ASSERT(__kmp_all_nth < __kmp_threads_capacity); 4350 4351 #if KMP_USE_MONITOR 4352 // If this is the first worker thread the RTL is creating, then also 4353 // launch the monitor thread. We try to do this as early as possible. 4354 if (!TCR_4(__kmp_init_monitor)) { 4355 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock); 4356 if (!TCR_4(__kmp_init_monitor)) { 4357 KF_TRACE(10, ("before __kmp_create_monitor\n")); 4358 TCW_4(__kmp_init_monitor, 1); 4359 __kmp_create_monitor(&__kmp_monitor); 4360 KF_TRACE(10, ("after __kmp_create_monitor\n")); 4361 #if KMP_OS_WINDOWS 4362 // AC: wait until monitor has started. This is a fix for CQ232808. 4363 // The reason is that if the library is loaded/unloaded in a loop with 4364 // small (parallel) work in between, then there is high probability that 4365 // monitor thread started after the library shutdown. At shutdown it is 4366 // too late to cope with the problem, because when the primary thread is 4367 // in DllMain (process detach) the monitor has no chances to start (it is 4368 // blocked), and primary thread has no means to inform the monitor that 4369 // the library has gone, because all the memory which the monitor can 4370 // access is going to be released/reset. 4371 while (TCR_4(__kmp_init_monitor) < 2) { 4372 KMP_YIELD(TRUE); 4373 } 4374 KF_TRACE(10, ("after monitor thread has started\n")); 4375 #endif 4376 } 4377 __kmp_release_bootstrap_lock(&__kmp_monitor_lock); 4378 } 4379 #endif 4380 4381 KMP_MB(); 4382 4383 { 4384 int new_start_gtid = TCR_4(__kmp_init_hidden_helper_threads) 4385 ? 1 4386 : __kmp_hidden_helper_threads_num + 1; 4387 4388 for (new_gtid = new_start_gtid; TCR_PTR(__kmp_threads[new_gtid]) != NULL; 4389 ++new_gtid) { 4390 KMP_DEBUG_ASSERT(new_gtid < __kmp_threads_capacity); 4391 } 4392 4393 if (TCR_4(__kmp_init_hidden_helper_threads)) { 4394 KMP_DEBUG_ASSERT(new_gtid <= __kmp_hidden_helper_threads_num); 4395 } 4396 } 4397 4398 /* allocate space for it. */ 4399 new_thr = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t)); 4400 4401 TCW_SYNC_PTR(__kmp_threads[new_gtid], new_thr); 4402 4403 #if USE_ITT_BUILD && USE_ITT_NOTIFY && KMP_DEBUG 4404 // suppress race conditions detection on synchronization flags in debug mode 4405 // this helps to analyze library internals eliminating false positives 4406 __itt_suppress_mark_range( 4407 __itt_suppress_range, __itt_suppress_threading_errors, 4408 &new_thr->th.th_sleep_loc, sizeof(new_thr->th.th_sleep_loc)); 4409 __itt_suppress_mark_range( 4410 __itt_suppress_range, __itt_suppress_threading_errors, 4411 &new_thr->th.th_reap_state, sizeof(new_thr->th.th_reap_state)); 4412 #if KMP_OS_WINDOWS 4413 __itt_suppress_mark_range( 4414 __itt_suppress_range, __itt_suppress_threading_errors, 4415 &new_thr->th.th_suspend_init, sizeof(new_thr->th.th_suspend_init)); 4416 #else 4417 __itt_suppress_mark_range(__itt_suppress_range, 4418 __itt_suppress_threading_errors, 4419 &new_thr->th.th_suspend_init_count, 4420 sizeof(new_thr->th.th_suspend_init_count)); 4421 #endif 4422 // TODO: check if we need to also suppress b_arrived flags 4423 __itt_suppress_mark_range(__itt_suppress_range, 4424 __itt_suppress_threading_errors, 4425 CCAST(kmp_uint64 *, &new_thr->th.th_bar[0].bb.b_go), 4426 sizeof(new_thr->th.th_bar[0].bb.b_go)); 4427 __itt_suppress_mark_range(__itt_suppress_range, 4428 __itt_suppress_threading_errors, 4429 CCAST(kmp_uint64 *, &new_thr->th.th_bar[1].bb.b_go), 4430 sizeof(new_thr->th.th_bar[1].bb.b_go)); 4431 __itt_suppress_mark_range(__itt_suppress_range, 4432 __itt_suppress_threading_errors, 4433 CCAST(kmp_uint64 *, &new_thr->th.th_bar[2].bb.b_go), 4434 sizeof(new_thr->th.th_bar[2].bb.b_go)); 4435 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY && KMP_DEBUG */ 4436 if (__kmp_storage_map) { 4437 __kmp_print_thread_storage_map(new_thr, new_gtid); 4438 } 4439 4440 // add the reserve serialized team, initialized from the team's primary thread 4441 { 4442 kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team); 4443 KF_TRACE(10, ("__kmp_allocate_thread: before th_serial/serial_team\n")); 4444 new_thr->th.th_serial_team = serial_team = 4445 (kmp_team_t *)__kmp_allocate_team(root, 1, 1, 4446 #if OMPT_SUPPORT 4447 ompt_data_none, // root parallel id 4448 #endif 4449 proc_bind_default, &r_icvs, 4450 0 USE_NESTED_HOT_ARG(NULL)); 4451 } 4452 KMP_ASSERT(serial_team); 4453 serial_team->t.t_serialized = 0; // AC: the team created in reserve, not for 4454 // execution (it is unused for now). 4455 serial_team->t.t_threads[0] = new_thr; 4456 KF_TRACE(10, 4457 ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n", 4458 new_thr)); 4459 4460 /* setup the thread structures */ 4461 __kmp_initialize_info(new_thr, team, new_tid, new_gtid); 4462 4463 #if USE_FAST_MEMORY 4464 __kmp_initialize_fast_memory(new_thr); 4465 #endif /* USE_FAST_MEMORY */ 4466 4467 #if KMP_USE_BGET 4468 KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL); 4469 __kmp_initialize_bget(new_thr); 4470 #endif 4471 4472 __kmp_init_random(new_thr); // Initialize random number generator 4473 4474 /* Initialize these only once when thread is grabbed for a team allocation */ 4475 KA_TRACE(20, 4476 ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n", 4477 __kmp_get_gtid(), KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); 4478 4479 int b; 4480 kmp_balign_t *balign = new_thr->th.th_bar; 4481 for (b = 0; b < bs_last_barrier; ++b) { 4482 balign[b].bb.b_go = KMP_INIT_BARRIER_STATE; 4483 balign[b].bb.team = NULL; 4484 balign[b].bb.wait_flag = KMP_BARRIER_NOT_WAITING; 4485 balign[b].bb.use_oncore_barrier = 0; 4486 } 4487 4488 TCW_PTR(new_thr->th.th_sleep_loc, NULL); 4489 new_thr->th.th_sleep_loc_type = flag_unset; 4490 4491 new_thr->th.th_spin_here = FALSE; 4492 new_thr->th.th_next_waiting = 0; 4493 #if KMP_OS_UNIX 4494 new_thr->th.th_blocking = false; 4495 #endif 4496 4497 #if KMP_AFFINITY_SUPPORTED 4498 new_thr->th.th_current_place = KMP_PLACE_UNDEFINED; 4499 new_thr->th.th_new_place = KMP_PLACE_UNDEFINED; 4500 new_thr->th.th_first_place = KMP_PLACE_UNDEFINED; 4501 new_thr->th.th_last_place = KMP_PLACE_UNDEFINED; 4502 #endif 4503 new_thr->th.th_def_allocator = __kmp_def_allocator; 4504 new_thr->th.th_prev_level = 0; 4505 new_thr->th.th_prev_num_threads = 1; 4506 4507 TCW_4(new_thr->th.th_in_pool, FALSE); 4508 new_thr->th.th_active_in_pool = FALSE; 4509 TCW_4(new_thr->th.th_active, TRUE); 4510 4511 /* adjust the global counters */ 4512 __kmp_all_nth++; 4513 __kmp_nth++; 4514 4515 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low 4516 // numbers of procs, and method #2 (keyed API call) for higher numbers. 4517 if (__kmp_adjust_gtid_mode) { 4518 if (__kmp_all_nth >= __kmp_tls_gtid_min) { 4519 if (TCR_4(__kmp_gtid_mode) != 2) { 4520 TCW_4(__kmp_gtid_mode, 2); 4521 } 4522 } else { 4523 if (TCR_4(__kmp_gtid_mode) != 1) { 4524 TCW_4(__kmp_gtid_mode, 1); 4525 } 4526 } 4527 } 4528 4529 #ifdef KMP_ADJUST_BLOCKTIME 4530 /* Adjust blocktime back to zero if necessary */ 4531 /* Middle initialization might not have occurred yet */ 4532 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 4533 if (__kmp_nth > __kmp_avail_proc) { 4534 __kmp_zero_bt = TRUE; 4535 } 4536 } 4537 #endif /* KMP_ADJUST_BLOCKTIME */ 4538 4539 /* actually fork it and create the new worker thread */ 4540 KF_TRACE( 4541 10, ("__kmp_allocate_thread: before __kmp_create_worker: %p\n", new_thr)); 4542 __kmp_create_worker(new_gtid, new_thr, __kmp_stksize); 4543 KF_TRACE(10, 4544 ("__kmp_allocate_thread: after __kmp_create_worker: %p\n", new_thr)); 4545 4546 KA_TRACE(20, ("__kmp_allocate_thread: T#%d forked T#%d\n", __kmp_get_gtid(), 4547 new_gtid)); 4548 KMP_MB(); 4549 return new_thr; 4550 } 4551 4552 /* Reinitialize team for reuse. 4553 The hot team code calls this case at every fork barrier, so EPCC barrier 4554 test are extremely sensitive to changes in it, esp. writes to the team 4555 struct, which cause a cache invalidation in all threads. 4556 IF YOU TOUCH THIS ROUTINE, RUN EPCC C SYNCBENCH ON A BIG-IRON MACHINE!!! */ 4557 static void __kmp_reinitialize_team(kmp_team_t *team, 4558 kmp_internal_control_t *new_icvs, 4559 ident_t *loc) { 4560 KF_TRACE(10, ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n", 4561 team->t.t_threads[0], team)); 4562 KMP_DEBUG_ASSERT(team && new_icvs); 4563 KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc); 4564 KMP_CHECK_UPDATE(team->t.t_ident, loc); 4565 4566 KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID()); 4567 // Copy ICVs to the primary thread's implicit taskdata 4568 __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE); 4569 copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs); 4570 4571 KF_TRACE(10, ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n", 4572 team->t.t_threads[0], team)); 4573 } 4574 4575 /* Initialize the team data structure. 4576 This assumes the t_threads and t_max_nproc are already set. 4577 Also, we don't touch the arguments */ 4578 static void __kmp_initialize_team(kmp_team_t *team, int new_nproc, 4579 kmp_internal_control_t *new_icvs, 4580 ident_t *loc) { 4581 KF_TRACE(10, ("__kmp_initialize_team: enter: team=%p\n", team)); 4582 4583 /* verify */ 4584 KMP_DEBUG_ASSERT(team); 4585 KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc); 4586 KMP_DEBUG_ASSERT(team->t.t_threads); 4587 KMP_MB(); 4588 4589 team->t.t_master_tid = 0; /* not needed */ 4590 /* team->t.t_master_bar; not needed */ 4591 team->t.t_serialized = new_nproc > 1 ? 0 : 1; 4592 team->t.t_nproc = new_nproc; 4593 4594 /* team->t.t_parent = NULL; TODO not needed & would mess up hot team */ 4595 team->t.t_next_pool = NULL; 4596 /* memset( team->t.t_threads, 0, sizeof(kmp_info_t*)*new_nproc ); would mess 4597 * up hot team */ 4598 4599 TCW_SYNC_PTR(team->t.t_pkfn, NULL); /* not needed */ 4600 team->t.t_invoke = NULL; /* not needed */ 4601 4602 // TODO???: team->t.t_max_active_levels = new_max_active_levels; 4603 team->t.t_sched.sched = new_icvs->sched.sched; 4604 4605 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 4606 team->t.t_fp_control_saved = FALSE; /* not needed */ 4607 team->t.t_x87_fpu_control_word = 0; /* not needed */ 4608 team->t.t_mxcsr = 0; /* not needed */ 4609 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 4610 4611 team->t.t_construct = 0; 4612 4613 team->t.t_ordered.dt.t_value = 0; 4614 team->t.t_master_active = FALSE; 4615 4616 #ifdef KMP_DEBUG 4617 team->t.t_copypriv_data = NULL; /* not necessary, but nice for debugging */ 4618 #endif 4619 #if KMP_OS_WINDOWS 4620 team->t.t_copyin_counter = 0; /* for barrier-free copyin implementation */ 4621 #endif 4622 4623 team->t.t_control_stack_top = NULL; 4624 4625 __kmp_reinitialize_team(team, new_icvs, loc); 4626 4627 KMP_MB(); 4628 KF_TRACE(10, ("__kmp_initialize_team: exit: team=%p\n", team)); 4629 } 4630 4631 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED 4632 /* Sets full mask for thread and returns old mask, no changes to structures. */ 4633 static void 4634 __kmp_set_thread_affinity_mask_full_tmp(kmp_affin_mask_t *old_mask) { 4635 if (KMP_AFFINITY_CAPABLE()) { 4636 int status; 4637 if (old_mask != NULL) { 4638 status = __kmp_get_system_affinity(old_mask, TRUE); 4639 int error = errno; 4640 if (status != 0) { 4641 __kmp_fatal(KMP_MSG(ChangeThreadAffMaskError), KMP_ERR(error), 4642 __kmp_msg_null); 4643 } 4644 } 4645 __kmp_set_system_affinity(__kmp_affin_fullMask, TRUE); 4646 } 4647 } 4648 #endif 4649 4650 #if KMP_AFFINITY_SUPPORTED 4651 4652 // __kmp_partition_places() is the heart of the OpenMP 4.0 affinity mechanism. 4653 // It calculates the worker + primary thread's partition based upon the parent 4654 // thread's partition, and binds each worker to a thread in their partition. 4655 // The primary thread's partition should already include its current binding. 4656 static void __kmp_partition_places(kmp_team_t *team, int update_master_only) { 4657 // Do not partition places for the hidden helper team 4658 if (KMP_HIDDEN_HELPER_TEAM(team)) 4659 return; 4660 // Copy the primary thread's place partition to the team struct 4661 kmp_info_t *master_th = team->t.t_threads[0]; 4662 KMP_DEBUG_ASSERT(master_th != NULL); 4663 kmp_proc_bind_t proc_bind = team->t.t_proc_bind; 4664 int first_place = master_th->th.th_first_place; 4665 int last_place = master_th->th.th_last_place; 4666 int masters_place = master_th->th.th_current_place; 4667 team->t.t_first_place = first_place; 4668 team->t.t_last_place = last_place; 4669 4670 KA_TRACE(20, ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) " 4671 "bound to place %d partition = [%d,%d]\n", 4672 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]), 4673 team->t.t_id, masters_place, first_place, last_place)); 4674 4675 switch (proc_bind) { 4676 4677 case proc_bind_default: 4678 // Serial teams might have the proc_bind policy set to proc_bind_default. 4679 // Not an issue -- we don't rebind primary thread for any proc_bind policy. 4680 KMP_DEBUG_ASSERT(team->t.t_nproc == 1); 4681 break; 4682 4683 case proc_bind_primary: { 4684 int f; 4685 int n_th = team->t.t_nproc; 4686 for (f = 1; f < n_th; f++) { 4687 kmp_info_t *th = team->t.t_threads[f]; 4688 KMP_DEBUG_ASSERT(th != NULL); 4689 th->th.th_first_place = first_place; 4690 th->th.th_last_place = last_place; 4691 th->th.th_new_place = masters_place; 4692 if (__kmp_display_affinity && masters_place != th->th.th_current_place && 4693 team->t.t_display_affinity != 1) { 4694 team->t.t_display_affinity = 1; 4695 } 4696 4697 KA_TRACE(100, ("__kmp_partition_places: primary: T#%d(%d:%d) place %d " 4698 "partition = [%d,%d]\n", 4699 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, 4700 f, masters_place, first_place, last_place)); 4701 } 4702 } break; 4703 4704 case proc_bind_close: { 4705 int f; 4706 int n_th = team->t.t_nproc; 4707 int n_places; 4708 if (first_place <= last_place) { 4709 n_places = last_place - first_place + 1; 4710 } else { 4711 n_places = __kmp_affinity_num_masks - first_place + last_place + 1; 4712 } 4713 if (n_th <= n_places) { 4714 int place = masters_place; 4715 for (f = 1; f < n_th; f++) { 4716 kmp_info_t *th = team->t.t_threads[f]; 4717 KMP_DEBUG_ASSERT(th != NULL); 4718 4719 if (place == last_place) { 4720 place = first_place; 4721 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4722 place = 0; 4723 } else { 4724 place++; 4725 } 4726 th->th.th_first_place = first_place; 4727 th->th.th_last_place = last_place; 4728 th->th.th_new_place = place; 4729 if (__kmp_display_affinity && place != th->th.th_current_place && 4730 team->t.t_display_affinity != 1) { 4731 team->t.t_display_affinity = 1; 4732 } 4733 4734 KA_TRACE(100, ("__kmp_partition_places: close: T#%d(%d:%d) place %d " 4735 "partition = [%d,%d]\n", 4736 __kmp_gtid_from_thread(team->t.t_threads[f]), 4737 team->t.t_id, f, place, first_place, last_place)); 4738 } 4739 } else { 4740 int S, rem, gap, s_count; 4741 S = n_th / n_places; 4742 s_count = 0; 4743 rem = n_th - (S * n_places); 4744 gap = rem > 0 ? n_places / rem : n_places; 4745 int place = masters_place; 4746 int gap_ct = gap; 4747 for (f = 0; f < n_th; f++) { 4748 kmp_info_t *th = team->t.t_threads[f]; 4749 KMP_DEBUG_ASSERT(th != NULL); 4750 4751 th->th.th_first_place = first_place; 4752 th->th.th_last_place = last_place; 4753 th->th.th_new_place = place; 4754 if (__kmp_display_affinity && place != th->th.th_current_place && 4755 team->t.t_display_affinity != 1) { 4756 team->t.t_display_affinity = 1; 4757 } 4758 s_count++; 4759 4760 if ((s_count == S) && rem && (gap_ct == gap)) { 4761 // do nothing, add an extra thread to place on next iteration 4762 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) { 4763 // we added an extra thread to this place; move to next place 4764 if (place == last_place) { 4765 place = first_place; 4766 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4767 place = 0; 4768 } else { 4769 place++; 4770 } 4771 s_count = 0; 4772 gap_ct = 1; 4773 rem--; 4774 } else if (s_count == S) { // place full; don't add extra 4775 if (place == last_place) { 4776 place = first_place; 4777 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4778 place = 0; 4779 } else { 4780 place++; 4781 } 4782 gap_ct++; 4783 s_count = 0; 4784 } 4785 4786 KA_TRACE(100, 4787 ("__kmp_partition_places: close: T#%d(%d:%d) place %d " 4788 "partition = [%d,%d]\n", 4789 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f, 4790 th->th.th_new_place, first_place, last_place)); 4791 } 4792 KMP_DEBUG_ASSERT(place == masters_place); 4793 } 4794 } break; 4795 4796 case proc_bind_spread: { 4797 int f; 4798 int n_th = team->t.t_nproc; 4799 int n_places; 4800 int thidx; 4801 if (first_place <= last_place) { 4802 n_places = last_place - first_place + 1; 4803 } else { 4804 n_places = __kmp_affinity_num_masks - first_place + last_place + 1; 4805 } 4806 if (n_th <= n_places) { 4807 int place = -1; 4808 4809 if (n_places != static_cast<int>(__kmp_affinity_num_masks)) { 4810 int S = n_places / n_th; 4811 int s_count, rem, gap, gap_ct; 4812 4813 place = masters_place; 4814 rem = n_places - n_th * S; 4815 gap = rem ? n_th / rem : 1; 4816 gap_ct = gap; 4817 thidx = n_th; 4818 if (update_master_only == 1) 4819 thidx = 1; 4820 for (f = 0; f < thidx; f++) { 4821 kmp_info_t *th = team->t.t_threads[f]; 4822 KMP_DEBUG_ASSERT(th != NULL); 4823 4824 th->th.th_first_place = place; 4825 th->th.th_new_place = place; 4826 if (__kmp_display_affinity && place != th->th.th_current_place && 4827 team->t.t_display_affinity != 1) { 4828 team->t.t_display_affinity = 1; 4829 } 4830 s_count = 1; 4831 while (s_count < S) { 4832 if (place == last_place) { 4833 place = first_place; 4834 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4835 place = 0; 4836 } else { 4837 place++; 4838 } 4839 s_count++; 4840 } 4841 if (rem && (gap_ct == gap)) { 4842 if (place == last_place) { 4843 place = first_place; 4844 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4845 place = 0; 4846 } else { 4847 place++; 4848 } 4849 rem--; 4850 gap_ct = 0; 4851 } 4852 th->th.th_last_place = place; 4853 gap_ct++; 4854 4855 if (place == last_place) { 4856 place = first_place; 4857 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4858 place = 0; 4859 } else { 4860 place++; 4861 } 4862 4863 KA_TRACE(100, 4864 ("__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4865 "partition = [%d,%d], __kmp_affinity_num_masks: %u\n", 4866 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, 4867 f, th->th.th_new_place, th->th.th_first_place, 4868 th->th.th_last_place, __kmp_affinity_num_masks)); 4869 } 4870 } else { 4871 /* Having uniform space of available computation places I can create 4872 T partitions of round(P/T) size and put threads into the first 4873 place of each partition. */ 4874 double current = static_cast<double>(masters_place); 4875 double spacing = 4876 (static_cast<double>(n_places + 1) / static_cast<double>(n_th)); 4877 int first, last; 4878 kmp_info_t *th; 4879 4880 thidx = n_th + 1; 4881 if (update_master_only == 1) 4882 thidx = 1; 4883 for (f = 0; f < thidx; f++) { 4884 first = static_cast<int>(current); 4885 last = static_cast<int>(current + spacing) - 1; 4886 KMP_DEBUG_ASSERT(last >= first); 4887 if (first >= n_places) { 4888 if (masters_place) { 4889 first -= n_places; 4890 last -= n_places; 4891 if (first == (masters_place + 1)) { 4892 KMP_DEBUG_ASSERT(f == n_th); 4893 first--; 4894 } 4895 if (last == masters_place) { 4896 KMP_DEBUG_ASSERT(f == (n_th - 1)); 4897 last--; 4898 } 4899 } else { 4900 KMP_DEBUG_ASSERT(f == n_th); 4901 first = 0; 4902 last = 0; 4903 } 4904 } 4905 if (last >= n_places) { 4906 last = (n_places - 1); 4907 } 4908 place = first; 4909 current += spacing; 4910 if (f < n_th) { 4911 KMP_DEBUG_ASSERT(0 <= first); 4912 KMP_DEBUG_ASSERT(n_places > first); 4913 KMP_DEBUG_ASSERT(0 <= last); 4914 KMP_DEBUG_ASSERT(n_places > last); 4915 KMP_DEBUG_ASSERT(last_place >= first_place); 4916 th = team->t.t_threads[f]; 4917 KMP_DEBUG_ASSERT(th); 4918 th->th.th_first_place = first; 4919 th->th.th_new_place = place; 4920 th->th.th_last_place = last; 4921 if (__kmp_display_affinity && place != th->th.th_current_place && 4922 team->t.t_display_affinity != 1) { 4923 team->t.t_display_affinity = 1; 4924 } 4925 KA_TRACE(100, 4926 ("__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4927 "partition = [%d,%d], spacing = %.4f\n", 4928 __kmp_gtid_from_thread(team->t.t_threads[f]), 4929 team->t.t_id, f, th->th.th_new_place, 4930 th->th.th_first_place, th->th.th_last_place, spacing)); 4931 } 4932 } 4933 } 4934 KMP_DEBUG_ASSERT(update_master_only || place == masters_place); 4935 } else { 4936 int S, rem, gap, s_count; 4937 S = n_th / n_places; 4938 s_count = 0; 4939 rem = n_th - (S * n_places); 4940 gap = rem > 0 ? n_places / rem : n_places; 4941 int place = masters_place; 4942 int gap_ct = gap; 4943 thidx = n_th; 4944 if (update_master_only == 1) 4945 thidx = 1; 4946 for (f = 0; f < thidx; f++) { 4947 kmp_info_t *th = team->t.t_threads[f]; 4948 KMP_DEBUG_ASSERT(th != NULL); 4949 4950 th->th.th_first_place = place; 4951 th->th.th_last_place = place; 4952 th->th.th_new_place = place; 4953 if (__kmp_display_affinity && place != th->th.th_current_place && 4954 team->t.t_display_affinity != 1) { 4955 team->t.t_display_affinity = 1; 4956 } 4957 s_count++; 4958 4959 if ((s_count == S) && rem && (gap_ct == gap)) { 4960 // do nothing, add an extra thread to place on next iteration 4961 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) { 4962 // we added an extra thread to this place; move on to next place 4963 if (place == last_place) { 4964 place = first_place; 4965 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4966 place = 0; 4967 } else { 4968 place++; 4969 } 4970 s_count = 0; 4971 gap_ct = 1; 4972 rem--; 4973 } else if (s_count == S) { // place is full; don't add extra thread 4974 if (place == last_place) { 4975 place = first_place; 4976 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4977 place = 0; 4978 } else { 4979 place++; 4980 } 4981 gap_ct++; 4982 s_count = 0; 4983 } 4984 4985 KA_TRACE(100, ("__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4986 "partition = [%d,%d]\n", 4987 __kmp_gtid_from_thread(team->t.t_threads[f]), 4988 team->t.t_id, f, th->th.th_new_place, 4989 th->th.th_first_place, th->th.th_last_place)); 4990 } 4991 KMP_DEBUG_ASSERT(update_master_only || place == masters_place); 4992 } 4993 } break; 4994 4995 default: 4996 break; 4997 } 4998 4999 KA_TRACE(20, ("__kmp_partition_places: exit T#%d\n", team->t.t_id)); 5000 } 5001 5002 #endif // KMP_AFFINITY_SUPPORTED 5003 5004 /* allocate a new team data structure to use. take one off of the free pool if 5005 available */ 5006 kmp_team_t * 5007 __kmp_allocate_team(kmp_root_t *root, int new_nproc, int max_nproc, 5008 #if OMPT_SUPPORT 5009 ompt_data_t ompt_parallel_data, 5010 #endif 5011 kmp_proc_bind_t new_proc_bind, 5012 kmp_internal_control_t *new_icvs, 5013 int argc USE_NESTED_HOT_ARG(kmp_info_t *master)) { 5014 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_allocate_team); 5015 int f; 5016 kmp_team_t *team; 5017 int use_hot_team = !root->r.r_active; 5018 int level = 0; 5019 5020 KA_TRACE(20, ("__kmp_allocate_team: called\n")); 5021 KMP_DEBUG_ASSERT(new_nproc >= 1 && argc >= 0); 5022 KMP_DEBUG_ASSERT(max_nproc >= new_nproc); 5023 KMP_MB(); 5024 5025 #if KMP_NESTED_HOT_TEAMS 5026 kmp_hot_team_ptr_t *hot_teams; 5027 if (master) { 5028 team = master->th.th_team; 5029 level = team->t.t_active_level; 5030 if (master->th.th_teams_microtask) { // in teams construct? 5031 if (master->th.th_teams_size.nteams > 1 && 5032 ( // #teams > 1 5033 team->t.t_pkfn == 5034 (microtask_t)__kmp_teams_master || // inner fork of the teams 5035 master->th.th_teams_level < 5036 team->t.t_level)) { // or nested parallel inside the teams 5037 ++level; // not increment if #teams==1, or for outer fork of the teams; 5038 // increment otherwise 5039 } 5040 } 5041 hot_teams = master->th.th_hot_teams; 5042 if (level < __kmp_hot_teams_max_level && hot_teams && 5043 hot_teams[level].hot_team) { 5044 // hot team has already been allocated for given level 5045 use_hot_team = 1; 5046 } else { 5047 use_hot_team = 0; 5048 } 5049 } else { 5050 // check we won't access uninitialized hot_teams, just in case 5051 KMP_DEBUG_ASSERT(new_nproc == 1); 5052 } 5053 #endif 5054 // Optimization to use a "hot" team 5055 if (use_hot_team && new_nproc > 1) { 5056 KMP_DEBUG_ASSERT(new_nproc <= max_nproc); 5057 #if KMP_NESTED_HOT_TEAMS 5058 team = hot_teams[level].hot_team; 5059 #else 5060 team = root->r.r_hot_team; 5061 #endif 5062 #if KMP_DEBUG 5063 if (__kmp_tasking_mode != tskm_immediate_exec) { 5064 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p " 5065 "task_team[1] = %p before reinit\n", 5066 team->t.t_task_team[0], team->t.t_task_team[1])); 5067 } 5068 #endif 5069 5070 if (team->t.t_nproc != new_nproc && 5071 __kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar) { 5072 // Distributed barrier may need a resize 5073 int old_nthr = team->t.t_nproc; 5074 __kmp_resize_dist_barrier(team, old_nthr, new_nproc); 5075 } 5076 5077 // Has the number of threads changed? 5078 /* Let's assume the most common case is that the number of threads is 5079 unchanged, and put that case first. */ 5080 if (team->t.t_nproc == new_nproc) { // Check changes in number of threads 5081 KA_TRACE(20, ("__kmp_allocate_team: reusing hot team\n")); 5082 // This case can mean that omp_set_num_threads() was called and the hot 5083 // team size was already reduced, so we check the special flag 5084 if (team->t.t_size_changed == -1) { 5085 team->t.t_size_changed = 1; 5086 } else { 5087 KMP_CHECK_UPDATE(team->t.t_size_changed, 0); 5088 } 5089 5090 // TODO???: team->t.t_max_active_levels = new_max_active_levels; 5091 kmp_r_sched_t new_sched = new_icvs->sched; 5092 // set primary thread's schedule as new run-time schedule 5093 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched); 5094 5095 __kmp_reinitialize_team(team, new_icvs, 5096 root->r.r_uber_thread->th.th_ident); 5097 5098 KF_TRACE(10, ("__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n", 0, 5099 team->t.t_threads[0], team)); 5100 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0); 5101 5102 #if KMP_AFFINITY_SUPPORTED 5103 if ((team->t.t_size_changed == 0) && 5104 (team->t.t_proc_bind == new_proc_bind)) { 5105 if (new_proc_bind == proc_bind_spread) { 5106 __kmp_partition_places( 5107 team, 1); // add flag to update only master for spread 5108 } 5109 KA_TRACE(200, ("__kmp_allocate_team: reusing hot team #%d bindings: " 5110 "proc_bind = %d, partition = [%d,%d]\n", 5111 team->t.t_id, new_proc_bind, team->t.t_first_place, 5112 team->t.t_last_place)); 5113 } else { 5114 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); 5115 __kmp_partition_places(team); 5116 } 5117 #else 5118 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); 5119 #endif /* KMP_AFFINITY_SUPPORTED */ 5120 } else if (team->t.t_nproc > new_nproc) { 5121 KA_TRACE(20, 5122 ("__kmp_allocate_team: decreasing hot team thread count to %d\n", 5123 new_nproc)); 5124 5125 team->t.t_size_changed = 1; 5126 if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar) { 5127 // Barrier size already reduced earlier in this function 5128 // Activate team threads via th_used_in_team 5129 __kmp_add_threads_to_team(team, new_nproc); 5130 } 5131 #if KMP_NESTED_HOT_TEAMS 5132 if (__kmp_hot_teams_mode == 0) { 5133 // AC: saved number of threads should correspond to team's value in this 5134 // mode, can be bigger in mode 1, when hot team has threads in reserve 5135 KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc); 5136 hot_teams[level].hot_team_nth = new_nproc; 5137 #endif // KMP_NESTED_HOT_TEAMS 5138 /* release the extra threads we don't need any more */ 5139 for (f = new_nproc; f < team->t.t_nproc; f++) { 5140 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 5141 if (__kmp_tasking_mode != tskm_immediate_exec) { 5142 // When decreasing team size, threads no longer in the team should 5143 // unref task team. 5144 team->t.t_threads[f]->th.th_task_team = NULL; 5145 } 5146 __kmp_free_thread(team->t.t_threads[f]); 5147 team->t.t_threads[f] = NULL; 5148 } 5149 #if KMP_NESTED_HOT_TEAMS 5150 } // (__kmp_hot_teams_mode == 0) 5151 else { 5152 // When keeping extra threads in team, switch threads to wait on own 5153 // b_go flag 5154 for (f = new_nproc; f < team->t.t_nproc; ++f) { 5155 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 5156 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar; 5157 for (int b = 0; b < bs_last_barrier; ++b) { 5158 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG) { 5159 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG; 5160 } 5161 KMP_CHECK_UPDATE(balign[b].bb.leaf_kids, 0); 5162 } 5163 } 5164 } 5165 #endif // KMP_NESTED_HOT_TEAMS 5166 team->t.t_nproc = new_nproc; 5167 // TODO???: team->t.t_max_active_levels = new_max_active_levels; 5168 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_icvs->sched.sched); 5169 __kmp_reinitialize_team(team, new_icvs, 5170 root->r.r_uber_thread->th.th_ident); 5171 5172 // Update remaining threads 5173 for (f = 0; f < new_nproc; ++f) { 5174 team->t.t_threads[f]->th.th_team_nproc = new_nproc; 5175 } 5176 5177 // restore the current task state of the primary thread: should be the 5178 // implicit task 5179 KF_TRACE(10, ("__kmp_allocate_team: T#%d, this_thread=%p team=%p\n", 0, 5180 team->t.t_threads[0], team)); 5181 5182 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0); 5183 5184 #ifdef KMP_DEBUG 5185 for (f = 0; f < team->t.t_nproc; f++) { 5186 KMP_DEBUG_ASSERT(team->t.t_threads[f] && 5187 team->t.t_threads[f]->th.th_team_nproc == 5188 team->t.t_nproc); 5189 } 5190 #endif 5191 5192 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); 5193 #if KMP_AFFINITY_SUPPORTED 5194 __kmp_partition_places(team); 5195 #endif 5196 } else { // team->t.t_nproc < new_nproc 5197 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED 5198 kmp_affin_mask_t *old_mask; 5199 if (KMP_AFFINITY_CAPABLE()) { 5200 KMP_CPU_ALLOC(old_mask); 5201 } 5202 #endif 5203 5204 KA_TRACE(20, 5205 ("__kmp_allocate_team: increasing hot team thread count to %d\n", 5206 new_nproc)); 5207 int old_nproc = team->t.t_nproc; // save old value and use to update only 5208 team->t.t_size_changed = 1; 5209 5210 #if KMP_NESTED_HOT_TEAMS 5211 int avail_threads = hot_teams[level].hot_team_nth; 5212 if (new_nproc < avail_threads) 5213 avail_threads = new_nproc; 5214 kmp_info_t **other_threads = team->t.t_threads; 5215 for (f = team->t.t_nproc; f < avail_threads; ++f) { 5216 // Adjust barrier data of reserved threads (if any) of the team 5217 // Other data will be set in __kmp_initialize_info() below. 5218 int b; 5219 kmp_balign_t *balign = other_threads[f]->th.th_bar; 5220 for (b = 0; b < bs_last_barrier; ++b) { 5221 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 5222 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 5223 #if USE_DEBUGGER 5224 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 5225 #endif 5226 } 5227 } 5228 if (hot_teams[level].hot_team_nth >= new_nproc) { 5229 // we have all needed threads in reserve, no need to allocate any 5230 // this only possible in mode 1, cannot have reserved threads in mode 0 5231 KMP_DEBUG_ASSERT(__kmp_hot_teams_mode == 1); 5232 team->t.t_nproc = new_nproc; // just get reserved threads involved 5233 } else { 5234 // We may have some threads in reserve, but not enough; 5235 // get reserved threads involved if any. 5236 team->t.t_nproc = hot_teams[level].hot_team_nth; 5237 hot_teams[level].hot_team_nth = new_nproc; // adjust hot team max size 5238 #endif // KMP_NESTED_HOT_TEAMS 5239 if (team->t.t_max_nproc < new_nproc) { 5240 /* reallocate larger arrays */ 5241 __kmp_reallocate_team_arrays(team, new_nproc); 5242 __kmp_reinitialize_team(team, new_icvs, NULL); 5243 } 5244 5245 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED 5246 /* Temporarily set full mask for primary thread before creation of 5247 workers. The reason is that workers inherit the affinity from the 5248 primary thread, so if a lot of workers are created on the single 5249 core quickly, they don't get a chance to set their own affinity for 5250 a long time. */ 5251 __kmp_set_thread_affinity_mask_full_tmp(old_mask); 5252 #endif 5253 5254 /* allocate new threads for the hot team */ 5255 for (f = team->t.t_nproc; f < new_nproc; f++) { 5256 kmp_info_t *new_worker = __kmp_allocate_thread(root, team, f); 5257 KMP_DEBUG_ASSERT(new_worker); 5258 team->t.t_threads[f] = new_worker; 5259 5260 KA_TRACE(20, 5261 ("__kmp_allocate_team: team %d init T#%d arrived: " 5262 "join=%llu, plain=%llu\n", 5263 team->t.t_id, __kmp_gtid_from_tid(f, team), team->t.t_id, f, 5264 team->t.t_bar[bs_forkjoin_barrier].b_arrived, 5265 team->t.t_bar[bs_plain_barrier].b_arrived)); 5266 5267 { // Initialize barrier data for new threads. 5268 int b; 5269 kmp_balign_t *balign = new_worker->th.th_bar; 5270 for (b = 0; b < bs_last_barrier; ++b) { 5271 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 5272 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != 5273 KMP_BARRIER_PARENT_FLAG); 5274 #if USE_DEBUGGER 5275 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 5276 #endif 5277 } 5278 } 5279 } 5280 5281 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED 5282 if (KMP_AFFINITY_CAPABLE()) { 5283 /* Restore initial primary thread's affinity mask */ 5284 __kmp_set_system_affinity(old_mask, TRUE); 5285 KMP_CPU_FREE(old_mask); 5286 } 5287 #endif 5288 #if KMP_NESTED_HOT_TEAMS 5289 } // end of check of t_nproc vs. new_nproc vs. hot_team_nth 5290 #endif // KMP_NESTED_HOT_TEAMS 5291 if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar) { 5292 // Barrier size already increased earlier in this function 5293 // Activate team threads via th_used_in_team 5294 __kmp_add_threads_to_team(team, new_nproc); 5295 } 5296 /* make sure everyone is syncronized */ 5297 // new threads below 5298 __kmp_initialize_team(team, new_nproc, new_icvs, 5299 root->r.r_uber_thread->th.th_ident); 5300 5301 /* reinitialize the threads */ 5302 KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc); 5303 for (f = 0; f < team->t.t_nproc; ++f) 5304 __kmp_initialize_info(team->t.t_threads[f], team, f, 5305 __kmp_gtid_from_tid(f, team)); 5306 5307 if (level) { // set th_task_state for new threads in nested hot team 5308 // __kmp_initialize_info() no longer zeroes th_task_state, so we should 5309 // only need to set the th_task_state for the new threads. th_task_state 5310 // for primary thread will not be accurate until after this in 5311 // __kmp_fork_call(), so we look to the primary thread's memo_stack to 5312 // get the correct value. 5313 for (f = old_nproc; f < team->t.t_nproc; ++f) 5314 team->t.t_threads[f]->th.th_task_state = 5315 team->t.t_threads[0]->th.th_task_state_memo_stack[level]; 5316 } else { // set th_task_state for new threads in non-nested hot team 5317 // copy primary thread's state 5318 kmp_uint8 old_state = team->t.t_threads[0]->th.th_task_state; 5319 for (f = old_nproc; f < team->t.t_nproc; ++f) 5320 team->t.t_threads[f]->th.th_task_state = old_state; 5321 } 5322 5323 #ifdef KMP_DEBUG 5324 for (f = 0; f < team->t.t_nproc; ++f) { 5325 KMP_DEBUG_ASSERT(team->t.t_threads[f] && 5326 team->t.t_threads[f]->th.th_team_nproc == 5327 team->t.t_nproc); 5328 } 5329 #endif 5330 5331 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); 5332 #if KMP_AFFINITY_SUPPORTED 5333 __kmp_partition_places(team); 5334 #endif 5335 } // Check changes in number of threads 5336 5337 kmp_info_t *master = team->t.t_threads[0]; 5338 if (master->th.th_teams_microtask) { 5339 for (f = 1; f < new_nproc; ++f) { 5340 // propagate teams construct specific info to workers 5341 kmp_info_t *thr = team->t.t_threads[f]; 5342 thr->th.th_teams_microtask = master->th.th_teams_microtask; 5343 thr->th.th_teams_level = master->th.th_teams_level; 5344 thr->th.th_teams_size = master->th.th_teams_size; 5345 } 5346 } 5347 #if KMP_NESTED_HOT_TEAMS 5348 if (level) { 5349 // Sync barrier state for nested hot teams, not needed for outermost hot 5350 // team. 5351 for (f = 1; f < new_nproc; ++f) { 5352 kmp_info_t *thr = team->t.t_threads[f]; 5353 int b; 5354 kmp_balign_t *balign = thr->th.th_bar; 5355 for (b = 0; b < bs_last_barrier; ++b) { 5356 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 5357 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 5358 #if USE_DEBUGGER 5359 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 5360 #endif 5361 } 5362 } 5363 } 5364 #endif // KMP_NESTED_HOT_TEAMS 5365 5366 /* reallocate space for arguments if necessary */ 5367 __kmp_alloc_argv_entries(argc, team, TRUE); 5368 KMP_CHECK_UPDATE(team->t.t_argc, argc); 5369 // The hot team re-uses the previous task team, 5370 // if untouched during the previous release->gather phase. 5371 5372 KF_TRACE(10, (" hot_team = %p\n", team)); 5373 5374 #if KMP_DEBUG 5375 if (__kmp_tasking_mode != tskm_immediate_exec) { 5376 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p " 5377 "task_team[1] = %p after reinit\n", 5378 team->t.t_task_team[0], team->t.t_task_team[1])); 5379 } 5380 #endif 5381 5382 #if OMPT_SUPPORT 5383 __ompt_team_assign_id(team, ompt_parallel_data); 5384 #endif 5385 5386 KMP_MB(); 5387 5388 return team; 5389 } 5390 5391 /* next, let's try to take one from the team pool */ 5392 KMP_MB(); 5393 for (team = CCAST(kmp_team_t *, __kmp_team_pool); (team);) { 5394 /* TODO: consider resizing undersized teams instead of reaping them, now 5395 that we have a resizing mechanism */ 5396 if (team->t.t_max_nproc >= max_nproc) { 5397 /* take this team from the team pool */ 5398 __kmp_team_pool = team->t.t_next_pool; 5399 5400 if (max_nproc > 1 && 5401 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) { 5402 if (!team->t.b) { // Allocate barrier structure 5403 team->t.b = distributedBarrier::allocate(__kmp_dflt_team_nth_ub); 5404 } 5405 } 5406 5407 /* setup the team for fresh use */ 5408 __kmp_initialize_team(team, new_nproc, new_icvs, NULL); 5409 5410 KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and " 5411 "task_team[1] %p to NULL\n", 5412 &team->t.t_task_team[0], &team->t.t_task_team[1])); 5413 team->t.t_task_team[0] = NULL; 5414 team->t.t_task_team[1] = NULL; 5415 5416 /* reallocate space for arguments if necessary */ 5417 __kmp_alloc_argv_entries(argc, team, TRUE); 5418 KMP_CHECK_UPDATE(team->t.t_argc, argc); 5419 5420 KA_TRACE( 5421 20, ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n", 5422 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); 5423 { // Initialize barrier data. 5424 int b; 5425 for (b = 0; b < bs_last_barrier; ++b) { 5426 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE; 5427 #if USE_DEBUGGER 5428 team->t.t_bar[b].b_master_arrived = 0; 5429 team->t.t_bar[b].b_team_arrived = 0; 5430 #endif 5431 } 5432 } 5433 5434 team->t.t_proc_bind = new_proc_bind; 5435 5436 KA_TRACE(20, ("__kmp_allocate_team: using team from pool %d.\n", 5437 team->t.t_id)); 5438 5439 #if OMPT_SUPPORT 5440 __ompt_team_assign_id(team, ompt_parallel_data); 5441 #endif 5442 5443 KMP_MB(); 5444 5445 return team; 5446 } 5447 5448 /* reap team if it is too small, then loop back and check the next one */ 5449 // not sure if this is wise, but, will be redone during the hot-teams 5450 // rewrite. 5451 /* TODO: Use technique to find the right size hot-team, don't reap them */ 5452 team = __kmp_reap_team(team); 5453 __kmp_team_pool = team; 5454 } 5455 5456 /* nothing available in the pool, no matter, make a new team! */ 5457 KMP_MB(); 5458 team = (kmp_team_t *)__kmp_allocate(sizeof(kmp_team_t)); 5459 5460 /* and set it up */ 5461 team->t.t_max_nproc = max_nproc; 5462 if (max_nproc > 1 && 5463 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) { 5464 // Allocate barrier structure 5465 team->t.b = distributedBarrier::allocate(__kmp_dflt_team_nth_ub); 5466 } 5467 5468 /* NOTE well, for some reason allocating one big buffer and dividing it up 5469 seems to really hurt performance a lot on the P4, so, let's not use this */ 5470 __kmp_allocate_team_arrays(team, max_nproc); 5471 5472 KA_TRACE(20, ("__kmp_allocate_team: making a new team\n")); 5473 __kmp_initialize_team(team, new_nproc, new_icvs, NULL); 5474 5475 KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and task_team[1] " 5476 "%p to NULL\n", 5477 &team->t.t_task_team[0], &team->t.t_task_team[1])); 5478 team->t.t_task_team[0] = NULL; // to be removed, as __kmp_allocate zeroes 5479 // memory, no need to duplicate 5480 team->t.t_task_team[1] = NULL; // to be removed, as __kmp_allocate zeroes 5481 // memory, no need to duplicate 5482 5483 if (__kmp_storage_map) { 5484 __kmp_print_team_storage_map("team", team, team->t.t_id, new_nproc); 5485 } 5486 5487 /* allocate space for arguments */ 5488 __kmp_alloc_argv_entries(argc, team, FALSE); 5489 team->t.t_argc = argc; 5490 5491 KA_TRACE(20, 5492 ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n", 5493 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); 5494 { // Initialize barrier data. 5495 int b; 5496 for (b = 0; b < bs_last_barrier; ++b) { 5497 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE; 5498 #if USE_DEBUGGER 5499 team->t.t_bar[b].b_master_arrived = 0; 5500 team->t.t_bar[b].b_team_arrived = 0; 5501 #endif 5502 } 5503 } 5504 5505 team->t.t_proc_bind = new_proc_bind; 5506 5507 #if OMPT_SUPPORT 5508 __ompt_team_assign_id(team, ompt_parallel_data); 5509 team->t.ompt_serialized_team_info = NULL; 5510 #endif 5511 5512 KMP_MB(); 5513 5514 KA_TRACE(20, ("__kmp_allocate_team: done creating a new team %d.\n", 5515 team->t.t_id)); 5516 5517 return team; 5518 } 5519 5520 /* TODO implement hot-teams at all levels */ 5521 /* TODO implement lazy thread release on demand (disband request) */ 5522 5523 /* free the team. return it to the team pool. release all the threads 5524 * associated with it */ 5525 void __kmp_free_team(kmp_root_t *root, 5526 kmp_team_t *team USE_NESTED_HOT_ARG(kmp_info_t *master)) { 5527 int f; 5528 KA_TRACE(20, ("__kmp_free_team: T#%d freeing team %d\n", __kmp_get_gtid(), 5529 team->t.t_id)); 5530 5531 /* verify state */ 5532 KMP_DEBUG_ASSERT(root); 5533 KMP_DEBUG_ASSERT(team); 5534 KMP_DEBUG_ASSERT(team->t.t_nproc <= team->t.t_max_nproc); 5535 KMP_DEBUG_ASSERT(team->t.t_threads); 5536 5537 int use_hot_team = team == root->r.r_hot_team; 5538 #if KMP_NESTED_HOT_TEAMS 5539 int level; 5540 kmp_hot_team_ptr_t *hot_teams; 5541 if (master) { 5542 level = team->t.t_active_level - 1; 5543 if (master->th.th_teams_microtask) { // in teams construct? 5544 if (master->th.th_teams_size.nteams > 1) { 5545 ++level; // level was not increased in teams construct for 5546 // team_of_masters 5547 } 5548 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master && 5549 master->th.th_teams_level == team->t.t_level) { 5550 ++level; // level was not increased in teams construct for 5551 // team_of_workers before the parallel 5552 } // team->t.t_level will be increased inside parallel 5553 } 5554 hot_teams = master->th.th_hot_teams; 5555 if (level < __kmp_hot_teams_max_level) { 5556 KMP_DEBUG_ASSERT(team == hot_teams[level].hot_team); 5557 use_hot_team = 1; 5558 } 5559 } 5560 #endif // KMP_NESTED_HOT_TEAMS 5561 5562 /* team is done working */ 5563 TCW_SYNC_PTR(team->t.t_pkfn, 5564 NULL); // Important for Debugging Support Library. 5565 #if KMP_OS_WINDOWS 5566 team->t.t_copyin_counter = 0; // init counter for possible reuse 5567 #endif 5568 // Do not reset pointer to parent team to NULL for hot teams. 5569 5570 /* if we are non-hot team, release our threads */ 5571 if (!use_hot_team) { 5572 if (__kmp_tasking_mode != tskm_immediate_exec) { 5573 // Wait for threads to reach reapable state 5574 for (f = 1; f < team->t.t_nproc; ++f) { 5575 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 5576 kmp_info_t *th = team->t.t_threads[f]; 5577 volatile kmp_uint32 *state = &th->th.th_reap_state; 5578 while (*state != KMP_SAFE_TO_REAP) { 5579 #if KMP_OS_WINDOWS 5580 // On Windows a thread can be killed at any time, check this 5581 DWORD ecode; 5582 if (!__kmp_is_thread_alive(th, &ecode)) { 5583 *state = KMP_SAFE_TO_REAP; // reset the flag for dead thread 5584 break; 5585 } 5586 #endif 5587 // first check if thread is sleeping 5588 kmp_flag_64<> fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th); 5589 if (fl.is_sleeping()) 5590 fl.resume(__kmp_gtid_from_thread(th)); 5591 KMP_CPU_PAUSE(); 5592 } 5593 } 5594 5595 // Delete task teams 5596 int tt_idx; 5597 for (tt_idx = 0; tt_idx < 2; ++tt_idx) { 5598 kmp_task_team_t *task_team = team->t.t_task_team[tt_idx]; 5599 if (task_team != NULL) { 5600 for (f = 0; f < team->t.t_nproc; ++f) { // threads unref task teams 5601 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 5602 team->t.t_threads[f]->th.th_task_team = NULL; 5603 } 5604 KA_TRACE( 5605 20, 5606 ("__kmp_free_team: T#%d deactivating task_team %p on team %d\n", 5607 __kmp_get_gtid(), task_team, team->t.t_id)); 5608 #if KMP_NESTED_HOT_TEAMS 5609 __kmp_free_task_team(master, task_team); 5610 #endif 5611 team->t.t_task_team[tt_idx] = NULL; 5612 } 5613 } 5614 } 5615 5616 // Reset pointer to parent team only for non-hot teams. 5617 team->t.t_parent = NULL; 5618 team->t.t_level = 0; 5619 team->t.t_active_level = 0; 5620 5621 /* free the worker threads */ 5622 for (f = 1; f < team->t.t_nproc; ++f) { 5623 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 5624 if (__kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) { 5625 KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team), 5626 1, 2); 5627 } 5628 __kmp_free_thread(team->t.t_threads[f]); 5629 } 5630 5631 if (__kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) { 5632 if (team->t.b) { 5633 // wake up thread at old location 5634 team->t.b->go_release(); 5635 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) { 5636 for (f = 1; f < team->t.t_nproc; ++f) { 5637 if (team->t.b->sleep[f].sleep) { 5638 __kmp_atomic_resume_64( 5639 team->t.t_threads[f]->th.th_info.ds.ds_gtid, 5640 (kmp_atomic_flag_64<> *)NULL); 5641 } 5642 } 5643 } 5644 // Wait for threads to be removed from team 5645 for (int f = 1; f < team->t.t_nproc; ++f) { 5646 while (team->t.t_threads[f]->th.th_used_in_team.load() != 0) 5647 KMP_CPU_PAUSE(); 5648 } 5649 } 5650 } 5651 5652 for (f = 1; f < team->t.t_nproc; ++f) { 5653 team->t.t_threads[f] = NULL; 5654 } 5655 5656 if (team->t.t_max_nproc > 1 && 5657 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) { 5658 distributedBarrier::deallocate(team->t.b); 5659 team->t.b = NULL; 5660 } 5661 /* put the team back in the team pool */ 5662 /* TODO limit size of team pool, call reap_team if pool too large */ 5663 team->t.t_next_pool = CCAST(kmp_team_t *, __kmp_team_pool); 5664 __kmp_team_pool = (volatile kmp_team_t *)team; 5665 } else { // Check if team was created for primary threads in teams construct 5666 // See if first worker is a CG root 5667 KMP_DEBUG_ASSERT(team->t.t_threads[1] && 5668 team->t.t_threads[1]->th.th_cg_roots); 5669 if (team->t.t_threads[1]->th.th_cg_roots->cg_root == team->t.t_threads[1]) { 5670 // Clean up the CG root nodes on workers so that this team can be re-used 5671 for (f = 1; f < team->t.t_nproc; ++f) { 5672 kmp_info_t *thr = team->t.t_threads[f]; 5673 KMP_DEBUG_ASSERT(thr && thr->th.th_cg_roots && 5674 thr->th.th_cg_roots->cg_root == thr); 5675 // Pop current CG root off list 5676 kmp_cg_root_t *tmp = thr->th.th_cg_roots; 5677 thr->th.th_cg_roots = tmp->up; 5678 KA_TRACE(100, ("__kmp_free_team: Thread %p popping node %p and moving" 5679 " up to node %p. cg_nthreads was %d\n", 5680 thr, tmp, thr->th.th_cg_roots, tmp->cg_nthreads)); 5681 int i = tmp->cg_nthreads--; 5682 if (i == 1) { 5683 __kmp_free(tmp); // free CG if we are the last thread in it 5684 } 5685 // Restore current task's thread_limit from CG root 5686 if (thr->th.th_cg_roots) 5687 thr->th.th_current_task->td_icvs.thread_limit = 5688 thr->th.th_cg_roots->cg_thread_limit; 5689 } 5690 } 5691 } 5692 5693 KMP_MB(); 5694 } 5695 5696 /* reap the team. destroy it, reclaim all its resources and free its memory */ 5697 kmp_team_t *__kmp_reap_team(kmp_team_t *team) { 5698 kmp_team_t *next_pool = team->t.t_next_pool; 5699 5700 KMP_DEBUG_ASSERT(team); 5701 KMP_DEBUG_ASSERT(team->t.t_dispatch); 5702 KMP_DEBUG_ASSERT(team->t.t_disp_buffer); 5703 KMP_DEBUG_ASSERT(team->t.t_threads); 5704 KMP_DEBUG_ASSERT(team->t.t_argv); 5705 5706 /* TODO clean the threads that are a part of this? */ 5707 5708 /* free stuff */ 5709 __kmp_free_team_arrays(team); 5710 if (team->t.t_argv != &team->t.t_inline_argv[0]) 5711 __kmp_free((void *)team->t.t_argv); 5712 __kmp_free(team); 5713 5714 KMP_MB(); 5715 return next_pool; 5716 } 5717 5718 // Free the thread. Don't reap it, just place it on the pool of available 5719 // threads. 5720 // 5721 // Changes for Quad issue 527845: We need a predictable OMP tid <-> gtid 5722 // binding for the affinity mechanism to be useful. 5723 // 5724 // Now, we always keep the free list (__kmp_thread_pool) sorted by gtid. 5725 // However, we want to avoid a potential performance problem by always 5726 // scanning through the list to find the correct point at which to insert 5727 // the thread (potential N**2 behavior). To do this we keep track of the 5728 // last place a thread struct was inserted (__kmp_thread_pool_insert_pt). 5729 // With single-level parallelism, threads will always be added to the tail 5730 // of the list, kept track of by __kmp_thread_pool_insert_pt. With nested 5731 // parallelism, all bets are off and we may need to scan through the entire 5732 // free list. 5733 // 5734 // This change also has a potentially large performance benefit, for some 5735 // applications. Previously, as threads were freed from the hot team, they 5736 // would be placed back on the free list in inverse order. If the hot team 5737 // grew back to it's original size, then the freed thread would be placed 5738 // back on the hot team in reverse order. This could cause bad cache 5739 // locality problems on programs where the size of the hot team regularly 5740 // grew and shrunk. 5741 // 5742 // Now, for single-level parallelism, the OMP tid is always == gtid. 5743 void __kmp_free_thread(kmp_info_t *this_th) { 5744 int gtid; 5745 kmp_info_t **scan; 5746 5747 KA_TRACE(20, ("__kmp_free_thread: T#%d putting T#%d back on free pool.\n", 5748 __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid)); 5749 5750 KMP_DEBUG_ASSERT(this_th); 5751 5752 // When moving thread to pool, switch thread to wait on own b_go flag, and 5753 // uninitialized (NULL team). 5754 int b; 5755 kmp_balign_t *balign = this_th->th.th_bar; 5756 for (b = 0; b < bs_last_barrier; ++b) { 5757 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG) 5758 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG; 5759 balign[b].bb.team = NULL; 5760 balign[b].bb.leaf_kids = 0; 5761 } 5762 this_th->th.th_task_state = 0; 5763 this_th->th.th_reap_state = KMP_SAFE_TO_REAP; 5764 5765 /* put thread back on the free pool */ 5766 TCW_PTR(this_th->th.th_team, NULL); 5767 TCW_PTR(this_th->th.th_root, NULL); 5768 TCW_PTR(this_th->th.th_dispatch, NULL); /* NOT NEEDED */ 5769 5770 while (this_th->th.th_cg_roots) { 5771 this_th->th.th_cg_roots->cg_nthreads--; 5772 KA_TRACE(100, ("__kmp_free_thread: Thread %p decrement cg_nthreads on node" 5773 " %p of thread %p to %d\n", 5774 this_th, this_th->th.th_cg_roots, 5775 this_th->th.th_cg_roots->cg_root, 5776 this_th->th.th_cg_roots->cg_nthreads)); 5777 kmp_cg_root_t *tmp = this_th->th.th_cg_roots; 5778 if (tmp->cg_root == this_th) { // Thread is a cg_root 5779 KMP_DEBUG_ASSERT(tmp->cg_nthreads == 0); 5780 KA_TRACE( 5781 5, ("__kmp_free_thread: Thread %p freeing node %p\n", this_th, tmp)); 5782 this_th->th.th_cg_roots = tmp->up; 5783 __kmp_free(tmp); 5784 } else { // Worker thread 5785 if (tmp->cg_nthreads == 0) { // last thread leaves contention group 5786 __kmp_free(tmp); 5787 } 5788 this_th->th.th_cg_roots = NULL; 5789 break; 5790 } 5791 } 5792 5793 /* If the implicit task assigned to this thread can be used by other threads 5794 * -> multiple threads can share the data and try to free the task at 5795 * __kmp_reap_thread at exit. This duplicate use of the task data can happen 5796 * with higher probability when hot team is disabled but can occurs even when 5797 * the hot team is enabled */ 5798 __kmp_free_implicit_task(this_th); 5799 this_th->th.th_current_task = NULL; 5800 5801 // If the __kmp_thread_pool_insert_pt is already past the new insert 5802 // point, then we need to re-scan the entire list. 5803 gtid = this_th->th.th_info.ds.ds_gtid; 5804 if (__kmp_thread_pool_insert_pt != NULL) { 5805 KMP_DEBUG_ASSERT(__kmp_thread_pool != NULL); 5806 if (__kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid) { 5807 __kmp_thread_pool_insert_pt = NULL; 5808 } 5809 } 5810 5811 // Scan down the list to find the place to insert the thread. 5812 // scan is the address of a link in the list, possibly the address of 5813 // __kmp_thread_pool itself. 5814 // 5815 // In the absence of nested parallelism, the for loop will have 0 iterations. 5816 if (__kmp_thread_pool_insert_pt != NULL) { 5817 scan = &(__kmp_thread_pool_insert_pt->th.th_next_pool); 5818 } else { 5819 scan = CCAST(kmp_info_t **, &__kmp_thread_pool); 5820 } 5821 for (; (*scan != NULL) && ((*scan)->th.th_info.ds.ds_gtid < gtid); 5822 scan = &((*scan)->th.th_next_pool)) 5823 ; 5824 5825 // Insert the new element on the list, and set __kmp_thread_pool_insert_pt 5826 // to its address. 5827 TCW_PTR(this_th->th.th_next_pool, *scan); 5828 __kmp_thread_pool_insert_pt = *scan = this_th; 5829 KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) || 5830 (this_th->th.th_info.ds.ds_gtid < 5831 this_th->th.th_next_pool->th.th_info.ds.ds_gtid)); 5832 TCW_4(this_th->th.th_in_pool, TRUE); 5833 __kmp_suspend_initialize_thread(this_th); 5834 __kmp_lock_suspend_mx(this_th); 5835 if (this_th->th.th_active == TRUE) { 5836 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth); 5837 this_th->th.th_active_in_pool = TRUE; 5838 } 5839 #if KMP_DEBUG 5840 else { 5841 KMP_DEBUG_ASSERT(this_th->th.th_active_in_pool == FALSE); 5842 } 5843 #endif 5844 __kmp_unlock_suspend_mx(this_th); 5845 5846 TCW_4(__kmp_nth, __kmp_nth - 1); 5847 5848 #ifdef KMP_ADJUST_BLOCKTIME 5849 /* Adjust blocktime back to user setting or default if necessary */ 5850 /* Middle initialization might never have occurred */ 5851 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 5852 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0); 5853 if (__kmp_nth <= __kmp_avail_proc) { 5854 __kmp_zero_bt = FALSE; 5855 } 5856 } 5857 #endif /* KMP_ADJUST_BLOCKTIME */ 5858 5859 KMP_MB(); 5860 } 5861 5862 /* ------------------------------------------------------------------------ */ 5863 5864 void *__kmp_launch_thread(kmp_info_t *this_thr) { 5865 #if OMP_PROFILING_SUPPORT 5866 ProfileTraceFile = getenv("LIBOMPTARGET_PROFILE"); 5867 // TODO: add a configuration option for time granularity 5868 if (ProfileTraceFile) 5869 llvm::timeTraceProfilerInitialize(500 /* us */, "libomptarget"); 5870 #endif 5871 5872 int gtid = this_thr->th.th_info.ds.ds_gtid; 5873 /* void *stack_data;*/ 5874 kmp_team_t **volatile pteam; 5875 5876 KMP_MB(); 5877 KA_TRACE(10, ("__kmp_launch_thread: T#%d start\n", gtid)); 5878 5879 if (__kmp_env_consistency_check) { 5880 this_thr->th.th_cons = __kmp_allocate_cons_stack(gtid); // ATT: Memory leak? 5881 } 5882 5883 #if OMPD_SUPPORT 5884 if (ompd_state & OMPD_ENABLE_BP) 5885 ompd_bp_thread_begin(); 5886 #endif 5887 5888 #if OMPT_SUPPORT 5889 ompt_data_t *thread_data = nullptr; 5890 if (ompt_enabled.enabled) { 5891 thread_data = &(this_thr->th.ompt_thread_info.thread_data); 5892 *thread_data = ompt_data_none; 5893 5894 this_thr->th.ompt_thread_info.state = ompt_state_overhead; 5895 this_thr->th.ompt_thread_info.wait_id = 0; 5896 this_thr->th.ompt_thread_info.idle_frame = OMPT_GET_FRAME_ADDRESS(0); 5897 this_thr->th.ompt_thread_info.parallel_flags = 0; 5898 if (ompt_enabled.ompt_callback_thread_begin) { 5899 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)( 5900 ompt_thread_worker, thread_data); 5901 } 5902 this_thr->th.ompt_thread_info.state = ompt_state_idle; 5903 } 5904 #endif 5905 5906 /* This is the place where threads wait for work */ 5907 while (!TCR_4(__kmp_global.g.g_done)) { 5908 KMP_DEBUG_ASSERT(this_thr == __kmp_threads[gtid]); 5909 KMP_MB(); 5910 5911 /* wait for work to do */ 5912 KA_TRACE(20, ("__kmp_launch_thread: T#%d waiting for work\n", gtid)); 5913 5914 /* No tid yet since not part of a team */ 5915 __kmp_fork_barrier(gtid, KMP_GTID_DNE); 5916 5917 #if OMPT_SUPPORT 5918 if (ompt_enabled.enabled) { 5919 this_thr->th.ompt_thread_info.state = ompt_state_overhead; 5920 } 5921 #endif 5922 5923 pteam = &this_thr->th.th_team; 5924 5925 /* have we been allocated? */ 5926 if (TCR_SYNC_PTR(*pteam) && !TCR_4(__kmp_global.g.g_done)) { 5927 /* we were just woken up, so run our new task */ 5928 if (TCR_SYNC_PTR((*pteam)->t.t_pkfn) != NULL) { 5929 int rc; 5930 KA_TRACE(20, 5931 ("__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n", 5932 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), 5933 (*pteam)->t.t_pkfn)); 5934 5935 updateHWFPControl(*pteam); 5936 5937 #if OMPT_SUPPORT 5938 if (ompt_enabled.enabled) { 5939 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel; 5940 } 5941 #endif 5942 5943 rc = (*pteam)->t.t_invoke(gtid); 5944 KMP_ASSERT(rc); 5945 5946 KMP_MB(); 5947 KA_TRACE(20, ("__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n", 5948 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), 5949 (*pteam)->t.t_pkfn)); 5950 } 5951 #if OMPT_SUPPORT 5952 if (ompt_enabled.enabled) { 5953 /* no frame set while outside task */ 5954 __ompt_get_task_info_object(0)->frame.exit_frame = ompt_data_none; 5955 5956 this_thr->th.ompt_thread_info.state = ompt_state_overhead; 5957 } 5958 #endif 5959 /* join barrier after parallel region */ 5960 __kmp_join_barrier(gtid); 5961 } 5962 } 5963 TCR_SYNC_PTR((intptr_t)__kmp_global.g.g_done); 5964 5965 #if OMPD_SUPPORT 5966 if (ompd_state & OMPD_ENABLE_BP) 5967 ompd_bp_thread_end(); 5968 #endif 5969 5970 #if OMPT_SUPPORT 5971 if (ompt_enabled.ompt_callback_thread_end) { 5972 ompt_callbacks.ompt_callback(ompt_callback_thread_end)(thread_data); 5973 } 5974 #endif 5975 5976 this_thr->th.th_task_team = NULL; 5977 /* run the destructors for the threadprivate data for this thread */ 5978 __kmp_common_destroy_gtid(gtid); 5979 5980 KA_TRACE(10, ("__kmp_launch_thread: T#%d done\n", gtid)); 5981 KMP_MB(); 5982 5983 #if OMP_PROFILING_SUPPORT 5984 llvm::timeTraceProfilerFinishThread(); 5985 #endif 5986 return this_thr; 5987 } 5988 5989 /* ------------------------------------------------------------------------ */ 5990 5991 void __kmp_internal_end_dest(void *specific_gtid) { 5992 // Make sure no significant bits are lost 5993 int gtid; 5994 __kmp_type_convert((kmp_intptr_t)specific_gtid - 1, >id); 5995 5996 KA_TRACE(30, ("__kmp_internal_end_dest: T#%d\n", gtid)); 5997 /* NOTE: the gtid is stored as gitd+1 in the thread-local-storage 5998 * this is because 0 is reserved for the nothing-stored case */ 5999 6000 __kmp_internal_end_thread(gtid); 6001 } 6002 6003 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB 6004 6005 __attribute__((destructor)) void __kmp_internal_end_dtor(void) { 6006 __kmp_internal_end_atexit(); 6007 } 6008 6009 #endif 6010 6011 /* [Windows] josh: when the atexit handler is called, there may still be more 6012 than one thread alive */ 6013 void __kmp_internal_end_atexit(void) { 6014 KA_TRACE(30, ("__kmp_internal_end_atexit\n")); 6015 /* [Windows] 6016 josh: ideally, we want to completely shutdown the library in this atexit 6017 handler, but stat code that depends on thread specific data for gtid fails 6018 because that data becomes unavailable at some point during the shutdown, so 6019 we call __kmp_internal_end_thread instead. We should eventually remove the 6020 dependency on __kmp_get_specific_gtid in the stat code and use 6021 __kmp_internal_end_library to cleanly shutdown the library. 6022 6023 // TODO: Can some of this comment about GVS be removed? 6024 I suspect that the offending stat code is executed when the calling thread 6025 tries to clean up a dead root thread's data structures, resulting in GVS 6026 code trying to close the GVS structures for that thread, but since the stat 6027 code uses __kmp_get_specific_gtid to get the gtid with the assumption that 6028 the calling thread is cleaning up itself instead of another thread, it get 6029 confused. This happens because allowing a thread to unregister and cleanup 6030 another thread is a recent modification for addressing an issue. 6031 Based on the current design (20050722), a thread may end up 6032 trying to unregister another thread only if thread death does not trigger 6033 the calling of __kmp_internal_end_thread. For Linux* OS, there is the 6034 thread specific data destructor function to detect thread death. For 6035 Windows dynamic, there is DllMain(THREAD_DETACH). For Windows static, there 6036 is nothing. Thus, the workaround is applicable only for Windows static 6037 stat library. */ 6038 __kmp_internal_end_library(-1); 6039 #if KMP_OS_WINDOWS 6040 __kmp_close_console(); 6041 #endif 6042 } 6043 6044 static void __kmp_reap_thread(kmp_info_t *thread, int is_root) { 6045 // It is assumed __kmp_forkjoin_lock is acquired. 6046 6047 int gtid; 6048 6049 KMP_DEBUG_ASSERT(thread != NULL); 6050 6051 gtid = thread->th.th_info.ds.ds_gtid; 6052 6053 if (!is_root) { 6054 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) { 6055 /* Assume the threads are at the fork barrier here */ 6056 KA_TRACE( 6057 20, ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n", 6058 gtid)); 6059 if (__kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) { 6060 while ( 6061 !KMP_COMPARE_AND_STORE_ACQ32(&(thread->th.th_used_in_team), 0, 3)) 6062 KMP_CPU_PAUSE(); 6063 __kmp_resume_32(gtid, (kmp_flag_32<false, false> *)NULL); 6064 } else { 6065 /* Need release fence here to prevent seg faults for tree forkjoin 6066 barrier (GEH) */ 6067 kmp_flag_64<> flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, 6068 thread); 6069 __kmp_release_64(&flag); 6070 } 6071 } 6072 6073 // Terminate OS thread. 6074 __kmp_reap_worker(thread); 6075 6076 // The thread was killed asynchronously. If it was actively 6077 // spinning in the thread pool, decrement the global count. 6078 // 6079 // There is a small timing hole here - if the worker thread was just waking 6080 // up after sleeping in the pool, had reset it's th_active_in_pool flag but 6081 // not decremented the global counter __kmp_thread_pool_active_nth yet, then 6082 // the global counter might not get updated. 6083 // 6084 // Currently, this can only happen as the library is unloaded, 6085 // so there are no harmful side effects. 6086 if (thread->th.th_active_in_pool) { 6087 thread->th.th_active_in_pool = FALSE; 6088 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth); 6089 KMP_DEBUG_ASSERT(__kmp_thread_pool_active_nth >= 0); 6090 } 6091 } 6092 6093 __kmp_free_implicit_task(thread); 6094 6095 // Free the fast memory for tasking 6096 #if USE_FAST_MEMORY 6097 __kmp_free_fast_memory(thread); 6098 #endif /* USE_FAST_MEMORY */ 6099 6100 __kmp_suspend_uninitialize_thread(thread); 6101 6102 KMP_DEBUG_ASSERT(__kmp_threads[gtid] == thread); 6103 TCW_SYNC_PTR(__kmp_threads[gtid], NULL); 6104 6105 --__kmp_all_nth; 6106 // __kmp_nth was decremented when thread is added to the pool. 6107 6108 #ifdef KMP_ADJUST_BLOCKTIME 6109 /* Adjust blocktime back to user setting or default if necessary */ 6110 /* Middle initialization might never have occurred */ 6111 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 6112 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0); 6113 if (__kmp_nth <= __kmp_avail_proc) { 6114 __kmp_zero_bt = FALSE; 6115 } 6116 } 6117 #endif /* KMP_ADJUST_BLOCKTIME */ 6118 6119 /* free the memory being used */ 6120 if (__kmp_env_consistency_check) { 6121 if (thread->th.th_cons) { 6122 __kmp_free_cons_stack(thread->th.th_cons); 6123 thread->th.th_cons = NULL; 6124 } 6125 } 6126 6127 if (thread->th.th_pri_common != NULL) { 6128 __kmp_free(thread->th.th_pri_common); 6129 thread->th.th_pri_common = NULL; 6130 } 6131 6132 if (thread->th.th_task_state_memo_stack != NULL) { 6133 __kmp_free(thread->th.th_task_state_memo_stack); 6134 thread->th.th_task_state_memo_stack = NULL; 6135 } 6136 6137 #if KMP_USE_BGET 6138 if (thread->th.th_local.bget_data != NULL) { 6139 __kmp_finalize_bget(thread); 6140 } 6141 #endif 6142 6143 #if KMP_AFFINITY_SUPPORTED 6144 if (thread->th.th_affin_mask != NULL) { 6145 KMP_CPU_FREE(thread->th.th_affin_mask); 6146 thread->th.th_affin_mask = NULL; 6147 } 6148 #endif /* KMP_AFFINITY_SUPPORTED */ 6149 6150 #if KMP_USE_HIER_SCHED 6151 if (thread->th.th_hier_bar_data != NULL) { 6152 __kmp_free(thread->th.th_hier_bar_data); 6153 thread->th.th_hier_bar_data = NULL; 6154 } 6155 #endif 6156 6157 __kmp_reap_team(thread->th.th_serial_team); 6158 thread->th.th_serial_team = NULL; 6159 __kmp_free(thread); 6160 6161 KMP_MB(); 6162 6163 } // __kmp_reap_thread 6164 6165 static void __kmp_internal_end(void) { 6166 int i; 6167 6168 /* First, unregister the library */ 6169 __kmp_unregister_library(); 6170 6171 #if KMP_OS_WINDOWS 6172 /* In Win static library, we can't tell when a root actually dies, so we 6173 reclaim the data structures for any root threads that have died but not 6174 unregistered themselves, in order to shut down cleanly. 6175 In Win dynamic library we also can't tell when a thread dies. */ 6176 __kmp_reclaim_dead_roots(); // AC: moved here to always clean resources of 6177 // dead roots 6178 #endif 6179 6180 for (i = 0; i < __kmp_threads_capacity; i++) 6181 if (__kmp_root[i]) 6182 if (__kmp_root[i]->r.r_active) 6183 break; 6184 KMP_MB(); /* Flush all pending memory write invalidates. */ 6185 TCW_SYNC_4(__kmp_global.g.g_done, TRUE); 6186 6187 if (i < __kmp_threads_capacity) { 6188 #if KMP_USE_MONITOR 6189 // 2009-09-08 (lev): Other alive roots found. Why do we kill the monitor?? 6190 KMP_MB(); /* Flush all pending memory write invalidates. */ 6191 6192 // Need to check that monitor was initialized before reaping it. If we are 6193 // called form __kmp_atfork_child (which sets __kmp_init_parallel = 0), then 6194 // __kmp_monitor will appear to contain valid data, but it is only valid in 6195 // the parent process, not the child. 6196 // New behavior (201008): instead of keying off of the flag 6197 // __kmp_init_parallel, the monitor thread creation is keyed off 6198 // of the new flag __kmp_init_monitor. 6199 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock); 6200 if (TCR_4(__kmp_init_monitor)) { 6201 __kmp_reap_monitor(&__kmp_monitor); 6202 TCW_4(__kmp_init_monitor, 0); 6203 } 6204 __kmp_release_bootstrap_lock(&__kmp_monitor_lock); 6205 KA_TRACE(10, ("__kmp_internal_end: monitor reaped\n")); 6206 #endif // KMP_USE_MONITOR 6207 } else { 6208 /* TODO move this to cleanup code */ 6209 #ifdef KMP_DEBUG 6210 /* make sure that everything has properly ended */ 6211 for (i = 0; i < __kmp_threads_capacity; i++) { 6212 if (__kmp_root[i]) { 6213 // KMP_ASSERT( ! KMP_UBER_GTID( i ) ); // AC: 6214 // there can be uber threads alive here 6215 KMP_ASSERT(!__kmp_root[i]->r.r_active); // TODO: can they be active? 6216 } 6217 } 6218 #endif 6219 6220 KMP_MB(); 6221 6222 // Reap the worker threads. 6223 // This is valid for now, but be careful if threads are reaped sooner. 6224 while (__kmp_thread_pool != NULL) { // Loop thru all the thread in the pool. 6225 // Get the next thread from the pool. 6226 kmp_info_t *thread = CCAST(kmp_info_t *, __kmp_thread_pool); 6227 __kmp_thread_pool = thread->th.th_next_pool; 6228 // Reap it. 6229 KMP_DEBUG_ASSERT(thread->th.th_reap_state == KMP_SAFE_TO_REAP); 6230 thread->th.th_next_pool = NULL; 6231 thread->th.th_in_pool = FALSE; 6232 __kmp_reap_thread(thread, 0); 6233 } 6234 __kmp_thread_pool_insert_pt = NULL; 6235 6236 // Reap teams. 6237 while (__kmp_team_pool != NULL) { // Loop thru all the teams in the pool. 6238 // Get the next team from the pool. 6239 kmp_team_t *team = CCAST(kmp_team_t *, __kmp_team_pool); 6240 __kmp_team_pool = team->t.t_next_pool; 6241 // Reap it. 6242 team->t.t_next_pool = NULL; 6243 __kmp_reap_team(team); 6244 } 6245 6246 __kmp_reap_task_teams(); 6247 6248 #if KMP_OS_UNIX 6249 // Threads that are not reaped should not access any resources since they 6250 // are going to be deallocated soon, so the shutdown sequence should wait 6251 // until all threads either exit the final spin-waiting loop or begin 6252 // sleeping after the given blocktime. 6253 for (i = 0; i < __kmp_threads_capacity; i++) { 6254 kmp_info_t *thr = __kmp_threads[i]; 6255 while (thr && KMP_ATOMIC_LD_ACQ(&thr->th.th_blocking)) 6256 KMP_CPU_PAUSE(); 6257 } 6258 #endif 6259 6260 for (i = 0; i < __kmp_threads_capacity; ++i) { 6261 // TBD: Add some checking... 6262 // Something like KMP_DEBUG_ASSERT( __kmp_thread[ i ] == NULL ); 6263 } 6264 6265 /* Make sure all threadprivate destructors get run by joining with all 6266 worker threads before resetting this flag */ 6267 TCW_SYNC_4(__kmp_init_common, FALSE); 6268 6269 KA_TRACE(10, ("__kmp_internal_end: all workers reaped\n")); 6270 KMP_MB(); 6271 6272 #if KMP_USE_MONITOR 6273 // See note above: One of the possible fixes for CQ138434 / CQ140126 6274 // 6275 // FIXME: push both code fragments down and CSE them? 6276 // push them into __kmp_cleanup() ? 6277 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock); 6278 if (TCR_4(__kmp_init_monitor)) { 6279 __kmp_reap_monitor(&__kmp_monitor); 6280 TCW_4(__kmp_init_monitor, 0); 6281 } 6282 __kmp_release_bootstrap_lock(&__kmp_monitor_lock); 6283 KA_TRACE(10, ("__kmp_internal_end: monitor reaped\n")); 6284 #endif 6285 } /* else !__kmp_global.t_active */ 6286 TCW_4(__kmp_init_gtid, FALSE); 6287 KMP_MB(); /* Flush all pending memory write invalidates. */ 6288 6289 __kmp_cleanup(); 6290 #if OMPT_SUPPORT 6291 ompt_fini(); 6292 #endif 6293 } 6294 6295 void __kmp_internal_end_library(int gtid_req) { 6296 /* if we have already cleaned up, don't try again, it wouldn't be pretty */ 6297 /* this shouldn't be a race condition because __kmp_internal_end() is the 6298 only place to clear __kmp_serial_init */ 6299 /* we'll check this later too, after we get the lock */ 6300 // 2009-09-06: We do not set g_abort without setting g_done. This check looks 6301 // redundant, because the next check will work in any case. 6302 if (__kmp_global.g.g_abort) { 6303 KA_TRACE(11, ("__kmp_internal_end_library: abort, exiting\n")); 6304 /* TODO abort? */ 6305 return; 6306 } 6307 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 6308 KA_TRACE(10, ("__kmp_internal_end_library: already finished\n")); 6309 return; 6310 } 6311 6312 // If hidden helper team has been initialized, we need to deinit it 6313 if (TCR_4(__kmp_init_hidden_helper) && 6314 !TCR_4(__kmp_hidden_helper_team_done)) { 6315 TCW_SYNC_4(__kmp_hidden_helper_team_done, TRUE); 6316 // First release the main thread to let it continue its work 6317 __kmp_hidden_helper_main_thread_release(); 6318 // Wait until the hidden helper team has been destroyed 6319 __kmp_hidden_helper_threads_deinitz_wait(); 6320 } 6321 6322 KMP_MB(); /* Flush all pending memory write invalidates. */ 6323 /* find out who we are and what we should do */ 6324 { 6325 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific(); 6326 KA_TRACE( 6327 10, ("__kmp_internal_end_library: enter T#%d (%d)\n", gtid, gtid_req)); 6328 if (gtid == KMP_GTID_SHUTDOWN) { 6329 KA_TRACE(10, ("__kmp_internal_end_library: !__kmp_init_runtime, system " 6330 "already shutdown\n")); 6331 return; 6332 } else if (gtid == KMP_GTID_MONITOR) { 6333 KA_TRACE(10, ("__kmp_internal_end_library: monitor thread, gtid not " 6334 "registered, or system shutdown\n")); 6335 return; 6336 } else if (gtid == KMP_GTID_DNE) { 6337 KA_TRACE(10, ("__kmp_internal_end_library: gtid not registered or system " 6338 "shutdown\n")); 6339 /* we don't know who we are, but we may still shutdown the library */ 6340 } else if (KMP_UBER_GTID(gtid)) { 6341 /* unregister ourselves as an uber thread. gtid is no longer valid */ 6342 if (__kmp_root[gtid]->r.r_active) { 6343 __kmp_global.g.g_abort = -1; 6344 TCW_SYNC_4(__kmp_global.g.g_done, TRUE); 6345 __kmp_unregister_library(); 6346 KA_TRACE(10, 6347 ("__kmp_internal_end_library: root still active, abort T#%d\n", 6348 gtid)); 6349 return; 6350 } else { 6351 KA_TRACE( 6352 10, 6353 ("__kmp_internal_end_library: unregistering sibling T#%d\n", gtid)); 6354 __kmp_unregister_root_current_thread(gtid); 6355 } 6356 } else { 6357 /* worker threads may call this function through the atexit handler, if they 6358 * call exit() */ 6359 /* For now, skip the usual subsequent processing and just dump the debug buffer. 6360 TODO: do a thorough shutdown instead */ 6361 #ifdef DUMP_DEBUG_ON_EXIT 6362 if (__kmp_debug_buf) 6363 __kmp_dump_debug_buffer(); 6364 #endif 6365 // added unregister library call here when we switch to shm linux 6366 // if we don't, it will leave lots of files in /dev/shm 6367 // cleanup shared memory file before exiting. 6368 __kmp_unregister_library(); 6369 return; 6370 } 6371 } 6372 /* synchronize the termination process */ 6373 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6374 6375 /* have we already finished */ 6376 if (__kmp_global.g.g_abort) { 6377 KA_TRACE(10, ("__kmp_internal_end_library: abort, exiting\n")); 6378 /* TODO abort? */ 6379 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6380 return; 6381 } 6382 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 6383 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6384 return; 6385 } 6386 6387 /* We need this lock to enforce mutex between this reading of 6388 __kmp_threads_capacity and the writing by __kmp_register_root. 6389 Alternatively, we can use a counter of roots that is atomically updated by 6390 __kmp_get_global_thread_id_reg, __kmp_do_serial_initialize and 6391 __kmp_internal_end_*. */ 6392 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 6393 6394 /* now we can safely conduct the actual termination */ 6395 __kmp_internal_end(); 6396 6397 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 6398 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6399 6400 KA_TRACE(10, ("__kmp_internal_end_library: exit\n")); 6401 6402 #ifdef DUMP_DEBUG_ON_EXIT 6403 if (__kmp_debug_buf) 6404 __kmp_dump_debug_buffer(); 6405 #endif 6406 6407 #if KMP_OS_WINDOWS 6408 __kmp_close_console(); 6409 #endif 6410 6411 __kmp_fini_allocator(); 6412 6413 } // __kmp_internal_end_library 6414 6415 void __kmp_internal_end_thread(int gtid_req) { 6416 int i; 6417 6418 /* if we have already cleaned up, don't try again, it wouldn't be pretty */ 6419 /* this shouldn't be a race condition because __kmp_internal_end() is the 6420 * only place to clear __kmp_serial_init */ 6421 /* we'll check this later too, after we get the lock */ 6422 // 2009-09-06: We do not set g_abort without setting g_done. This check looks 6423 // redundant, because the next check will work in any case. 6424 if (__kmp_global.g.g_abort) { 6425 KA_TRACE(11, ("__kmp_internal_end_thread: abort, exiting\n")); 6426 /* TODO abort? */ 6427 return; 6428 } 6429 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 6430 KA_TRACE(10, ("__kmp_internal_end_thread: already finished\n")); 6431 return; 6432 } 6433 6434 // If hidden helper team has been initialized, we need to deinit it 6435 if (TCR_4(__kmp_init_hidden_helper) && 6436 !TCR_4(__kmp_hidden_helper_team_done)) { 6437 TCW_SYNC_4(__kmp_hidden_helper_team_done, TRUE); 6438 // First release the main thread to let it continue its work 6439 __kmp_hidden_helper_main_thread_release(); 6440 // Wait until the hidden helper team has been destroyed 6441 __kmp_hidden_helper_threads_deinitz_wait(); 6442 } 6443 6444 KMP_MB(); /* Flush all pending memory write invalidates. */ 6445 6446 /* find out who we are and what we should do */ 6447 { 6448 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific(); 6449 KA_TRACE(10, 6450 ("__kmp_internal_end_thread: enter T#%d (%d)\n", gtid, gtid_req)); 6451 if (gtid == KMP_GTID_SHUTDOWN) { 6452 KA_TRACE(10, ("__kmp_internal_end_thread: !__kmp_init_runtime, system " 6453 "already shutdown\n")); 6454 return; 6455 } else if (gtid == KMP_GTID_MONITOR) { 6456 KA_TRACE(10, ("__kmp_internal_end_thread: monitor thread, gtid not " 6457 "registered, or system shutdown\n")); 6458 return; 6459 } else if (gtid == KMP_GTID_DNE) { 6460 KA_TRACE(10, ("__kmp_internal_end_thread: gtid not registered or system " 6461 "shutdown\n")); 6462 return; 6463 /* we don't know who we are */ 6464 } else if (KMP_UBER_GTID(gtid)) { 6465 /* unregister ourselves as an uber thread. gtid is no longer valid */ 6466 if (__kmp_root[gtid]->r.r_active) { 6467 __kmp_global.g.g_abort = -1; 6468 TCW_SYNC_4(__kmp_global.g.g_done, TRUE); 6469 KA_TRACE(10, 6470 ("__kmp_internal_end_thread: root still active, abort T#%d\n", 6471 gtid)); 6472 return; 6473 } else { 6474 KA_TRACE(10, ("__kmp_internal_end_thread: unregistering sibling T#%d\n", 6475 gtid)); 6476 __kmp_unregister_root_current_thread(gtid); 6477 } 6478 } else { 6479 /* just a worker thread, let's leave */ 6480 KA_TRACE(10, ("__kmp_internal_end_thread: worker thread T#%d\n", gtid)); 6481 6482 if (gtid >= 0) { 6483 __kmp_threads[gtid]->th.th_task_team = NULL; 6484 } 6485 6486 KA_TRACE(10, 6487 ("__kmp_internal_end_thread: worker thread done, exiting T#%d\n", 6488 gtid)); 6489 return; 6490 } 6491 } 6492 #if KMP_DYNAMIC_LIB 6493 if (__kmp_pause_status != kmp_hard_paused) 6494 // AC: lets not shutdown the dynamic library at the exit of uber thread, 6495 // because we will better shutdown later in the library destructor. 6496 { 6497 KA_TRACE(10, ("__kmp_internal_end_thread: exiting T#%d\n", gtid_req)); 6498 return; 6499 } 6500 #endif 6501 /* synchronize the termination process */ 6502 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6503 6504 /* have we already finished */ 6505 if (__kmp_global.g.g_abort) { 6506 KA_TRACE(10, ("__kmp_internal_end_thread: abort, exiting\n")); 6507 /* TODO abort? */ 6508 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6509 return; 6510 } 6511 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 6512 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6513 return; 6514 } 6515 6516 /* We need this lock to enforce mutex between this reading of 6517 __kmp_threads_capacity and the writing by __kmp_register_root. 6518 Alternatively, we can use a counter of roots that is atomically updated by 6519 __kmp_get_global_thread_id_reg, __kmp_do_serial_initialize and 6520 __kmp_internal_end_*. */ 6521 6522 /* should we finish the run-time? are all siblings done? */ 6523 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 6524 6525 for (i = 0; i < __kmp_threads_capacity; ++i) { 6526 if (KMP_UBER_GTID(i)) { 6527 KA_TRACE( 6528 10, 6529 ("__kmp_internal_end_thread: remaining sibling task: gtid==%d\n", i)); 6530 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 6531 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6532 return; 6533 } 6534 } 6535 6536 /* now we can safely conduct the actual termination */ 6537 6538 __kmp_internal_end(); 6539 6540 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 6541 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6542 6543 KA_TRACE(10, ("__kmp_internal_end_thread: exit T#%d\n", gtid_req)); 6544 6545 #ifdef DUMP_DEBUG_ON_EXIT 6546 if (__kmp_debug_buf) 6547 __kmp_dump_debug_buffer(); 6548 #endif 6549 } // __kmp_internal_end_thread 6550 6551 // ----------------------------------------------------------------------------- 6552 // Library registration stuff. 6553 6554 static long __kmp_registration_flag = 0; 6555 // Random value used to indicate library initialization. 6556 static char *__kmp_registration_str = NULL; 6557 // Value to be saved in env var __KMP_REGISTERED_LIB_<pid>. 6558 6559 static inline char *__kmp_reg_status_name() { 6560 /* On RHEL 3u5 if linked statically, getpid() returns different values in 6561 each thread. If registration and unregistration go in different threads 6562 (omp_misc_other_root_exit.cpp test case), the name of registered_lib_env 6563 env var can not be found, because the name will contain different pid. */ 6564 // macOS* complains about name being too long with additional getuid() 6565 #if KMP_OS_UNIX && !KMP_OS_DARWIN && KMP_DYNAMIC_LIB 6566 return __kmp_str_format("__KMP_REGISTERED_LIB_%d_%d", (int)getpid(), 6567 (int)getuid()); 6568 #else 6569 return __kmp_str_format("__KMP_REGISTERED_LIB_%d", (int)getpid()); 6570 #endif 6571 } // __kmp_reg_status_get 6572 6573 void __kmp_register_library_startup(void) { 6574 6575 char *name = __kmp_reg_status_name(); // Name of the environment variable. 6576 int done = 0; 6577 union { 6578 double dtime; 6579 long ltime; 6580 } time; 6581 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 6582 __kmp_initialize_system_tick(); 6583 #endif 6584 __kmp_read_system_time(&time.dtime); 6585 __kmp_registration_flag = 0xCAFE0000L | (time.ltime & 0x0000FFFFL); 6586 __kmp_registration_str = 6587 __kmp_str_format("%p-%lx-%s", &__kmp_registration_flag, 6588 __kmp_registration_flag, KMP_LIBRARY_FILE); 6589 6590 KA_TRACE(50, ("__kmp_register_library_startup: %s=\"%s\"\n", name, 6591 __kmp_registration_str)); 6592 6593 while (!done) { 6594 6595 char *value = NULL; // Actual value of the environment variable. 6596 6597 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB // shared memory is with dynamic library 6598 char *shm_name = __kmp_str_format("/%s", name); 6599 int shm_preexist = 0; 6600 char *data1; 6601 int fd1 = shm_open(shm_name, O_CREAT | O_EXCL | O_RDWR, 0666); 6602 if ((fd1 == -1) && (errno == EEXIST)) { 6603 // file didn't open because it already exists. 6604 // try opening existing file 6605 fd1 = shm_open(shm_name, O_RDWR, 0666); 6606 if (fd1 == -1) { // file didn't open 6607 // error out here 6608 __kmp_fatal(KMP_MSG(FunctionError, "Can't open SHM"), KMP_ERR(0), 6609 __kmp_msg_null); 6610 } else { 6611 // able to open existing file 6612 shm_preexist = 1; 6613 } 6614 } else if (fd1 == -1) { // SHM didn't open; it was due to error other than 6615 // already exists. 6616 // error out here. 6617 __kmp_fatal(KMP_MSG(FunctionError, "Can't open SHM2"), KMP_ERR(errno), 6618 __kmp_msg_null); 6619 } 6620 if (shm_preexist == 0) { 6621 // we created SHM now set size 6622 if (ftruncate(fd1, SHM_SIZE) == -1) { 6623 // error occured setting size; 6624 __kmp_fatal(KMP_MSG(FunctionError, "Can't set size of SHM"), 6625 KMP_ERR(errno), __kmp_msg_null); 6626 } 6627 } 6628 data1 = 6629 (char *)mmap(0, SHM_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd1, 0); 6630 if (data1 == MAP_FAILED) { 6631 // failed to map shared memory 6632 __kmp_fatal(KMP_MSG(FunctionError, "Can't map SHM"), KMP_ERR(errno), 6633 __kmp_msg_null); 6634 } 6635 if (shm_preexist == 0) { // set data to SHM, set value 6636 KMP_STRCPY_S(data1, SHM_SIZE, __kmp_registration_str); 6637 } 6638 // Read value from either what we just wrote or existing file. 6639 value = __kmp_str_format("%s", data1); // read value from SHM 6640 munmap(data1, SHM_SIZE); 6641 close(fd1); 6642 #else // Windows and unix with static library 6643 // Set environment variable, but do not overwrite if it is exist. 6644 __kmp_env_set(name, __kmp_registration_str, 0); 6645 // read value to see if it got set 6646 value = __kmp_env_get(name); 6647 #endif 6648 6649 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) { 6650 done = 1; // Ok, environment variable set successfully, exit the loop. 6651 } else { 6652 // Oops. Write failed. Another copy of OpenMP RTL is in memory. 6653 // Check whether it alive or dead. 6654 int neighbor = 0; // 0 -- unknown status, 1 -- alive, 2 -- dead. 6655 char *tail = value; 6656 char *flag_addr_str = NULL; 6657 char *flag_val_str = NULL; 6658 char const *file_name = NULL; 6659 __kmp_str_split(tail, '-', &flag_addr_str, &tail); 6660 __kmp_str_split(tail, '-', &flag_val_str, &tail); 6661 file_name = tail; 6662 if (tail != NULL) { 6663 unsigned long *flag_addr = 0; 6664 unsigned long flag_val = 0; 6665 KMP_SSCANF(flag_addr_str, "%p", RCAST(void **, &flag_addr)); 6666 KMP_SSCANF(flag_val_str, "%lx", &flag_val); 6667 if (flag_addr != 0 && flag_val != 0 && strcmp(file_name, "") != 0) { 6668 // First, check whether environment-encoded address is mapped into 6669 // addr space. 6670 // If so, dereference it to see if it still has the right value. 6671 if (__kmp_is_address_mapped(flag_addr) && *flag_addr == flag_val) { 6672 neighbor = 1; 6673 } else { 6674 // If not, then we know the other copy of the library is no longer 6675 // running. 6676 neighbor = 2; 6677 } 6678 } 6679 } 6680 switch (neighbor) { 6681 case 0: // Cannot parse environment variable -- neighbor status unknown. 6682 // Assume it is the incompatible format of future version of the 6683 // library. Assume the other library is alive. 6684 // WARN( ... ); // TODO: Issue a warning. 6685 file_name = "unknown library"; 6686 KMP_FALLTHROUGH(); 6687 // Attention! Falling to the next case. That's intentional. 6688 case 1: { // Neighbor is alive. 6689 // Check it is allowed. 6690 char *duplicate_ok = __kmp_env_get("KMP_DUPLICATE_LIB_OK"); 6691 if (!__kmp_str_match_true(duplicate_ok)) { 6692 // That's not allowed. Issue fatal error. 6693 __kmp_fatal(KMP_MSG(DuplicateLibrary, KMP_LIBRARY_FILE, file_name), 6694 KMP_HNT(DuplicateLibrary), __kmp_msg_null); 6695 } 6696 KMP_INTERNAL_FREE(duplicate_ok); 6697 __kmp_duplicate_library_ok = 1; 6698 done = 1; // Exit the loop. 6699 } break; 6700 case 2: { // Neighbor is dead. 6701 6702 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB // shared memory is with dynamic library 6703 // close shared memory. 6704 shm_unlink(shm_name); // this removes file in /dev/shm 6705 #else 6706 // Clear the variable and try to register library again. 6707 __kmp_env_unset(name); 6708 #endif 6709 } break; 6710 default: { 6711 KMP_DEBUG_ASSERT(0); 6712 } break; 6713 } 6714 } 6715 KMP_INTERNAL_FREE((void *)value); 6716 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB // shared memory is with dynamic library 6717 KMP_INTERNAL_FREE((void *)shm_name); 6718 #endif 6719 } // while 6720 KMP_INTERNAL_FREE((void *)name); 6721 6722 } // func __kmp_register_library_startup 6723 6724 void __kmp_unregister_library(void) { 6725 6726 char *name = __kmp_reg_status_name(); 6727 char *value = NULL; 6728 6729 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB // shared memory is with dynamic library 6730 char *shm_name = __kmp_str_format("/%s", name); 6731 int fd1 = shm_open(shm_name, O_RDONLY, 0666); 6732 if (fd1 == -1) { 6733 // file did not open. return. 6734 return; 6735 } 6736 char *data1 = (char *)mmap(0, SHM_SIZE, PROT_READ, MAP_SHARED, fd1, 0); 6737 if (data1 != MAP_FAILED) { 6738 value = __kmp_str_format("%s", data1); // read value from SHM 6739 munmap(data1, SHM_SIZE); 6740 } 6741 close(fd1); 6742 #else 6743 value = __kmp_env_get(name); 6744 #endif 6745 6746 KMP_DEBUG_ASSERT(__kmp_registration_flag != 0); 6747 KMP_DEBUG_ASSERT(__kmp_registration_str != NULL); 6748 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) { 6749 // Ok, this is our variable. Delete it. 6750 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB // shared memory is with dynamic library 6751 shm_unlink(shm_name); // this removes file in /dev/shm 6752 #else 6753 __kmp_env_unset(name); 6754 #endif 6755 } 6756 6757 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB // shared memory is with dynamic library 6758 KMP_INTERNAL_FREE(shm_name); 6759 #endif 6760 6761 KMP_INTERNAL_FREE(__kmp_registration_str); 6762 KMP_INTERNAL_FREE(value); 6763 KMP_INTERNAL_FREE(name); 6764 6765 __kmp_registration_flag = 0; 6766 __kmp_registration_str = NULL; 6767 6768 } // __kmp_unregister_library 6769 6770 // End of Library registration stuff. 6771 // ----------------------------------------------------------------------------- 6772 6773 #if KMP_MIC_SUPPORTED 6774 6775 static void __kmp_check_mic_type() { 6776 kmp_cpuid_t cpuid_state = {0}; 6777 kmp_cpuid_t *cs_p = &cpuid_state; 6778 __kmp_x86_cpuid(1, 0, cs_p); 6779 // We don't support mic1 at the moment 6780 if ((cs_p->eax & 0xff0) == 0xB10) { 6781 __kmp_mic_type = mic2; 6782 } else if ((cs_p->eax & 0xf0ff0) == 0x50670) { 6783 __kmp_mic_type = mic3; 6784 } else { 6785 __kmp_mic_type = non_mic; 6786 } 6787 } 6788 6789 #endif /* KMP_MIC_SUPPORTED */ 6790 6791 #if KMP_HAVE_UMWAIT 6792 static void __kmp_user_level_mwait_init() { 6793 struct kmp_cpuid buf; 6794 __kmp_x86_cpuid(7, 0, &buf); 6795 __kmp_umwait_enabled = ((buf.ecx >> 5) & 1) && __kmp_user_level_mwait; 6796 KF_TRACE(30, ("__kmp_user_level_mwait_init: __kmp_umwait_enabled = %d\n", 6797 __kmp_umwait_enabled)); 6798 } 6799 #elif KMP_HAVE_MWAIT 6800 #ifndef AT_INTELPHIUSERMWAIT 6801 // Spurious, non-existent value that should always fail to return anything. 6802 // Will be replaced with the correct value when we know that. 6803 #define AT_INTELPHIUSERMWAIT 10000 6804 #endif 6805 // getauxval() function is available in RHEL7 and SLES12. If a system with an 6806 // earlier OS is used to build the RTL, we'll use the following internal 6807 // function when the entry is not found. 6808 unsigned long getauxval(unsigned long) KMP_WEAK_ATTRIBUTE_EXTERNAL; 6809 unsigned long getauxval(unsigned long) { return 0; } 6810 6811 static void __kmp_user_level_mwait_init() { 6812 // When getauxval() and correct value of AT_INTELPHIUSERMWAIT are available 6813 // use them to find if the user-level mwait is enabled. Otherwise, forcibly 6814 // set __kmp_mwait_enabled=TRUE on Intel MIC if the environment variable 6815 // KMP_USER_LEVEL_MWAIT was set to TRUE. 6816 if (__kmp_mic_type == mic3) { 6817 unsigned long res = getauxval(AT_INTELPHIUSERMWAIT); 6818 if ((res & 0x1) || __kmp_user_level_mwait) { 6819 __kmp_mwait_enabled = TRUE; 6820 if (__kmp_user_level_mwait) { 6821 KMP_INFORM(EnvMwaitWarn); 6822 } 6823 } else { 6824 __kmp_mwait_enabled = FALSE; 6825 } 6826 } 6827 KF_TRACE(30, ("__kmp_user_level_mwait_init: __kmp_mic_type = %d, " 6828 "__kmp_mwait_enabled = %d\n", 6829 __kmp_mic_type, __kmp_mwait_enabled)); 6830 } 6831 #endif /* KMP_HAVE_UMWAIT */ 6832 6833 static void __kmp_do_serial_initialize(void) { 6834 int i, gtid; 6835 size_t size; 6836 6837 KA_TRACE(10, ("__kmp_do_serial_initialize: enter\n")); 6838 6839 KMP_DEBUG_ASSERT(sizeof(kmp_int32) == 4); 6840 KMP_DEBUG_ASSERT(sizeof(kmp_uint32) == 4); 6841 KMP_DEBUG_ASSERT(sizeof(kmp_int64) == 8); 6842 KMP_DEBUG_ASSERT(sizeof(kmp_uint64) == 8); 6843 KMP_DEBUG_ASSERT(sizeof(kmp_intptr_t) == sizeof(void *)); 6844 6845 #if OMPT_SUPPORT 6846 ompt_pre_init(); 6847 #endif 6848 #if OMPD_SUPPORT 6849 __kmp_env_dump(); 6850 ompd_init(); 6851 #endif 6852 6853 __kmp_validate_locks(); 6854 6855 /* Initialize internal memory allocator */ 6856 __kmp_init_allocator(); 6857 6858 /* Register the library startup via an environment variable and check to see 6859 whether another copy of the library is already registered. */ 6860 6861 __kmp_register_library_startup(); 6862 6863 /* TODO reinitialization of library */ 6864 if (TCR_4(__kmp_global.g.g_done)) { 6865 KA_TRACE(10, ("__kmp_do_serial_initialize: reinitialization of library\n")); 6866 } 6867 6868 __kmp_global.g.g_abort = 0; 6869 TCW_SYNC_4(__kmp_global.g.g_done, FALSE); 6870 6871 /* initialize the locks */ 6872 #if KMP_USE_ADAPTIVE_LOCKS 6873 #if KMP_DEBUG_ADAPTIVE_LOCKS 6874 __kmp_init_speculative_stats(); 6875 #endif 6876 #endif 6877 #if KMP_STATS_ENABLED 6878 __kmp_stats_init(); 6879 #endif 6880 __kmp_init_lock(&__kmp_global_lock); 6881 __kmp_init_queuing_lock(&__kmp_dispatch_lock); 6882 __kmp_init_lock(&__kmp_debug_lock); 6883 __kmp_init_atomic_lock(&__kmp_atomic_lock); 6884 __kmp_init_atomic_lock(&__kmp_atomic_lock_1i); 6885 __kmp_init_atomic_lock(&__kmp_atomic_lock_2i); 6886 __kmp_init_atomic_lock(&__kmp_atomic_lock_4i); 6887 __kmp_init_atomic_lock(&__kmp_atomic_lock_4r); 6888 __kmp_init_atomic_lock(&__kmp_atomic_lock_8i); 6889 __kmp_init_atomic_lock(&__kmp_atomic_lock_8r); 6890 __kmp_init_atomic_lock(&__kmp_atomic_lock_8c); 6891 __kmp_init_atomic_lock(&__kmp_atomic_lock_10r); 6892 __kmp_init_atomic_lock(&__kmp_atomic_lock_16r); 6893 __kmp_init_atomic_lock(&__kmp_atomic_lock_16c); 6894 __kmp_init_atomic_lock(&__kmp_atomic_lock_20c); 6895 __kmp_init_atomic_lock(&__kmp_atomic_lock_32c); 6896 __kmp_init_bootstrap_lock(&__kmp_forkjoin_lock); 6897 __kmp_init_bootstrap_lock(&__kmp_exit_lock); 6898 #if KMP_USE_MONITOR 6899 __kmp_init_bootstrap_lock(&__kmp_monitor_lock); 6900 #endif 6901 __kmp_init_bootstrap_lock(&__kmp_tp_cached_lock); 6902 6903 /* conduct initialization and initial setup of configuration */ 6904 6905 __kmp_runtime_initialize(); 6906 6907 #if KMP_MIC_SUPPORTED 6908 __kmp_check_mic_type(); 6909 #endif 6910 6911 // Some global variable initialization moved here from kmp_env_initialize() 6912 #ifdef KMP_DEBUG 6913 kmp_diag = 0; 6914 #endif 6915 __kmp_abort_delay = 0; 6916 6917 // From __kmp_init_dflt_team_nth() 6918 /* assume the entire machine will be used */ 6919 __kmp_dflt_team_nth_ub = __kmp_xproc; 6920 if (__kmp_dflt_team_nth_ub < KMP_MIN_NTH) { 6921 __kmp_dflt_team_nth_ub = KMP_MIN_NTH; 6922 } 6923 if (__kmp_dflt_team_nth_ub > __kmp_sys_max_nth) { 6924 __kmp_dflt_team_nth_ub = __kmp_sys_max_nth; 6925 } 6926 __kmp_max_nth = __kmp_sys_max_nth; 6927 __kmp_cg_max_nth = __kmp_sys_max_nth; 6928 __kmp_teams_max_nth = __kmp_xproc; // set a "reasonable" default 6929 if (__kmp_teams_max_nth > __kmp_sys_max_nth) { 6930 __kmp_teams_max_nth = __kmp_sys_max_nth; 6931 } 6932 6933 // Three vars below moved here from __kmp_env_initialize() "KMP_BLOCKTIME" 6934 // part 6935 __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME; 6936 #if KMP_USE_MONITOR 6937 __kmp_monitor_wakeups = 6938 KMP_WAKEUPS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups); 6939 __kmp_bt_intervals = 6940 KMP_INTERVALS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups); 6941 #endif 6942 // From "KMP_LIBRARY" part of __kmp_env_initialize() 6943 __kmp_library = library_throughput; 6944 // From KMP_SCHEDULE initialization 6945 __kmp_static = kmp_sch_static_balanced; 6946 // AC: do not use analytical here, because it is non-monotonous 6947 //__kmp_guided = kmp_sch_guided_iterative_chunked; 6948 //__kmp_auto = kmp_sch_guided_analytical_chunked; // AC: it is the default, no 6949 // need to repeat assignment 6950 // Barrier initialization. Moved here from __kmp_env_initialize() Barrier branch 6951 // bit control and barrier method control parts 6952 #if KMP_FAST_REDUCTION_BARRIER 6953 #define kmp_reduction_barrier_gather_bb ((int)1) 6954 #define kmp_reduction_barrier_release_bb ((int)1) 6955 #define kmp_reduction_barrier_gather_pat __kmp_barrier_gather_pat_dflt 6956 #define kmp_reduction_barrier_release_pat __kmp_barrier_release_pat_dflt 6957 #endif // KMP_FAST_REDUCTION_BARRIER 6958 for (i = bs_plain_barrier; i < bs_last_barrier; i++) { 6959 __kmp_barrier_gather_branch_bits[i] = __kmp_barrier_gather_bb_dflt; 6960 __kmp_barrier_release_branch_bits[i] = __kmp_barrier_release_bb_dflt; 6961 __kmp_barrier_gather_pattern[i] = __kmp_barrier_gather_pat_dflt; 6962 __kmp_barrier_release_pattern[i] = __kmp_barrier_release_pat_dflt; 6963 #if KMP_FAST_REDUCTION_BARRIER 6964 if (i == bs_reduction_barrier) { // tested and confirmed on ALTIX only ( 6965 // lin_64 ): hyper,1 6966 __kmp_barrier_gather_branch_bits[i] = kmp_reduction_barrier_gather_bb; 6967 __kmp_barrier_release_branch_bits[i] = kmp_reduction_barrier_release_bb; 6968 __kmp_barrier_gather_pattern[i] = kmp_reduction_barrier_gather_pat; 6969 __kmp_barrier_release_pattern[i] = kmp_reduction_barrier_release_pat; 6970 } 6971 #endif // KMP_FAST_REDUCTION_BARRIER 6972 } 6973 #if KMP_FAST_REDUCTION_BARRIER 6974 #undef kmp_reduction_barrier_release_pat 6975 #undef kmp_reduction_barrier_gather_pat 6976 #undef kmp_reduction_barrier_release_bb 6977 #undef kmp_reduction_barrier_gather_bb 6978 #endif // KMP_FAST_REDUCTION_BARRIER 6979 #if KMP_MIC_SUPPORTED 6980 if (__kmp_mic_type == mic2) { // KNC 6981 // AC: plane=3,2, forkjoin=2,1 are optimal for 240 threads on KNC 6982 __kmp_barrier_gather_branch_bits[bs_plain_barrier] = 3; // plain gather 6983 __kmp_barrier_release_branch_bits[bs_forkjoin_barrier] = 6984 1; // forkjoin release 6985 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar; 6986 __kmp_barrier_release_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar; 6987 } 6988 #if KMP_FAST_REDUCTION_BARRIER 6989 if (__kmp_mic_type == mic2) { // KNC 6990 __kmp_barrier_gather_pattern[bs_reduction_barrier] = bp_hierarchical_bar; 6991 __kmp_barrier_release_pattern[bs_reduction_barrier] = bp_hierarchical_bar; 6992 } 6993 #endif // KMP_FAST_REDUCTION_BARRIER 6994 #endif // KMP_MIC_SUPPORTED 6995 6996 // From KMP_CHECKS initialization 6997 #ifdef KMP_DEBUG 6998 __kmp_env_checks = TRUE; /* development versions have the extra checks */ 6999 #else 7000 __kmp_env_checks = FALSE; /* port versions do not have the extra checks */ 7001 #endif 7002 7003 // From "KMP_FOREIGN_THREADS_THREADPRIVATE" initialization 7004 __kmp_foreign_tp = TRUE; 7005 7006 __kmp_global.g.g_dynamic = FALSE; 7007 __kmp_global.g.g_dynamic_mode = dynamic_default; 7008 7009 __kmp_init_nesting_mode(); 7010 7011 __kmp_env_initialize(NULL); 7012 7013 #if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT 7014 __kmp_user_level_mwait_init(); 7015 #endif 7016 // Print all messages in message catalog for testing purposes. 7017 #ifdef KMP_DEBUG 7018 char const *val = __kmp_env_get("KMP_DUMP_CATALOG"); 7019 if (__kmp_str_match_true(val)) { 7020 kmp_str_buf_t buffer; 7021 __kmp_str_buf_init(&buffer); 7022 __kmp_i18n_dump_catalog(&buffer); 7023 __kmp_printf("%s", buffer.str); 7024 __kmp_str_buf_free(&buffer); 7025 } 7026 __kmp_env_free(&val); 7027 #endif 7028 7029 __kmp_threads_capacity = 7030 __kmp_initial_threads_capacity(__kmp_dflt_team_nth_ub); 7031 // Moved here from __kmp_env_initialize() "KMP_ALL_THREADPRIVATE" part 7032 __kmp_tp_capacity = __kmp_default_tp_capacity( 7033 __kmp_dflt_team_nth_ub, __kmp_max_nth, __kmp_allThreadsSpecified); 7034 7035 // If the library is shut down properly, both pools must be NULL. Just in 7036 // case, set them to NULL -- some memory may leak, but subsequent code will 7037 // work even if pools are not freed. 7038 KMP_DEBUG_ASSERT(__kmp_thread_pool == NULL); 7039 KMP_DEBUG_ASSERT(__kmp_thread_pool_insert_pt == NULL); 7040 KMP_DEBUG_ASSERT(__kmp_team_pool == NULL); 7041 __kmp_thread_pool = NULL; 7042 __kmp_thread_pool_insert_pt = NULL; 7043 __kmp_team_pool = NULL; 7044 7045 /* Allocate all of the variable sized records */ 7046 /* NOTE: __kmp_threads_capacity entries are allocated, but the arrays are 7047 * expandable */ 7048 /* Since allocation is cache-aligned, just add extra padding at the end */ 7049 size = 7050 (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * __kmp_threads_capacity + 7051 CACHE_LINE; 7052 __kmp_threads = (kmp_info_t **)__kmp_allocate(size); 7053 __kmp_root = (kmp_root_t **)((char *)__kmp_threads + 7054 sizeof(kmp_info_t *) * __kmp_threads_capacity); 7055 7056 /* init thread counts */ 7057 KMP_DEBUG_ASSERT(__kmp_all_nth == 7058 0); // Asserts fail if the library is reinitializing and 7059 KMP_DEBUG_ASSERT(__kmp_nth == 0); // something was wrong in termination. 7060 __kmp_all_nth = 0; 7061 __kmp_nth = 0; 7062 7063 /* setup the uber master thread and hierarchy */ 7064 gtid = __kmp_register_root(TRUE); 7065 KA_TRACE(10, ("__kmp_do_serial_initialize T#%d\n", gtid)); 7066 KMP_ASSERT(KMP_UBER_GTID(gtid)); 7067 KMP_ASSERT(KMP_INITIAL_GTID(gtid)); 7068 7069 KMP_MB(); /* Flush all pending memory write invalidates. */ 7070 7071 __kmp_common_initialize(); 7072 7073 #if KMP_OS_UNIX 7074 /* invoke the child fork handler */ 7075 __kmp_register_atfork(); 7076 #endif 7077 7078 #if !KMP_DYNAMIC_LIB 7079 { 7080 /* Invoke the exit handler when the program finishes, only for static 7081 library. For dynamic library, we already have _fini and DllMain. */ 7082 int rc = atexit(__kmp_internal_end_atexit); 7083 if (rc != 0) { 7084 __kmp_fatal(KMP_MSG(FunctionError, "atexit()"), KMP_ERR(rc), 7085 __kmp_msg_null); 7086 } 7087 } 7088 #endif 7089 7090 #if KMP_HANDLE_SIGNALS 7091 #if KMP_OS_UNIX 7092 /* NOTE: make sure that this is called before the user installs their own 7093 signal handlers so that the user handlers are called first. this way they 7094 can return false, not call our handler, avoid terminating the library, and 7095 continue execution where they left off. */ 7096 __kmp_install_signals(FALSE); 7097 #endif /* KMP_OS_UNIX */ 7098 #if KMP_OS_WINDOWS 7099 __kmp_install_signals(TRUE); 7100 #endif /* KMP_OS_WINDOWS */ 7101 #endif 7102 7103 /* we have finished the serial initialization */ 7104 __kmp_init_counter++; 7105 7106 __kmp_init_serial = TRUE; 7107 7108 if (__kmp_settings) { 7109 __kmp_env_print(); 7110 } 7111 7112 if (__kmp_display_env || __kmp_display_env_verbose) { 7113 __kmp_env_print_2(); 7114 } 7115 7116 #if OMPT_SUPPORT 7117 ompt_post_init(); 7118 #endif 7119 7120 KMP_MB(); 7121 7122 KA_TRACE(10, ("__kmp_do_serial_initialize: exit\n")); 7123 } 7124 7125 void __kmp_serial_initialize(void) { 7126 if (__kmp_init_serial) { 7127 return; 7128 } 7129 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 7130 if (__kmp_init_serial) { 7131 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 7132 return; 7133 } 7134 __kmp_do_serial_initialize(); 7135 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 7136 } 7137 7138 static void __kmp_do_middle_initialize(void) { 7139 int i, j; 7140 int prev_dflt_team_nth; 7141 7142 if (!__kmp_init_serial) { 7143 __kmp_do_serial_initialize(); 7144 } 7145 7146 KA_TRACE(10, ("__kmp_middle_initialize: enter\n")); 7147 7148 // Save the previous value for the __kmp_dflt_team_nth so that 7149 // we can avoid some reinitialization if it hasn't changed. 7150 prev_dflt_team_nth = __kmp_dflt_team_nth; 7151 7152 #if KMP_AFFINITY_SUPPORTED 7153 // __kmp_affinity_initialize() will try to set __kmp_ncores to the 7154 // number of cores on the machine. 7155 __kmp_affinity_initialize(); 7156 7157 #endif /* KMP_AFFINITY_SUPPORTED */ 7158 7159 KMP_ASSERT(__kmp_xproc > 0); 7160 if (__kmp_avail_proc == 0) { 7161 __kmp_avail_proc = __kmp_xproc; 7162 } 7163 7164 // If there were empty places in num_threads list (OMP_NUM_THREADS=,,2,3), 7165 // correct them now 7166 j = 0; 7167 while ((j < __kmp_nested_nth.used) && !__kmp_nested_nth.nth[j]) { 7168 __kmp_nested_nth.nth[j] = __kmp_dflt_team_nth = __kmp_dflt_team_nth_ub = 7169 __kmp_avail_proc; 7170 j++; 7171 } 7172 7173 if (__kmp_dflt_team_nth == 0) { 7174 #ifdef KMP_DFLT_NTH_CORES 7175 // Default #threads = #cores 7176 __kmp_dflt_team_nth = __kmp_ncores; 7177 KA_TRACE(20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = " 7178 "__kmp_ncores (%d)\n", 7179 __kmp_dflt_team_nth)); 7180 #else 7181 // Default #threads = #available OS procs 7182 __kmp_dflt_team_nth = __kmp_avail_proc; 7183 KA_TRACE(20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = " 7184 "__kmp_avail_proc(%d)\n", 7185 __kmp_dflt_team_nth)); 7186 #endif /* KMP_DFLT_NTH_CORES */ 7187 } 7188 7189 if (__kmp_dflt_team_nth < KMP_MIN_NTH) { 7190 __kmp_dflt_team_nth = KMP_MIN_NTH; 7191 } 7192 if (__kmp_dflt_team_nth > __kmp_sys_max_nth) { 7193 __kmp_dflt_team_nth = __kmp_sys_max_nth; 7194 } 7195 7196 if (__kmp_nesting_mode > 0) 7197 __kmp_set_nesting_mode_threads(); 7198 7199 // There's no harm in continuing if the following check fails, 7200 // but it indicates an error in the previous logic. 7201 KMP_DEBUG_ASSERT(__kmp_dflt_team_nth <= __kmp_dflt_team_nth_ub); 7202 7203 if (__kmp_dflt_team_nth != prev_dflt_team_nth) { 7204 // Run through the __kmp_threads array and set the num threads icv for each 7205 // root thread that is currently registered with the RTL (which has not 7206 // already explicitly set its nthreads-var with a call to 7207 // omp_set_num_threads()). 7208 for (i = 0; i < __kmp_threads_capacity; i++) { 7209 kmp_info_t *thread = __kmp_threads[i]; 7210 if (thread == NULL) 7211 continue; 7212 if (thread->th.th_current_task->td_icvs.nproc != 0) 7213 continue; 7214 7215 set__nproc(__kmp_threads[i], __kmp_dflt_team_nth); 7216 } 7217 } 7218 KA_TRACE( 7219 20, 7220 ("__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n", 7221 __kmp_dflt_team_nth)); 7222 7223 #ifdef KMP_ADJUST_BLOCKTIME 7224 /* Adjust blocktime to zero if necessary now that __kmp_avail_proc is set */ 7225 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 7226 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0); 7227 if (__kmp_nth > __kmp_avail_proc) { 7228 __kmp_zero_bt = TRUE; 7229 } 7230 } 7231 #endif /* KMP_ADJUST_BLOCKTIME */ 7232 7233 /* we have finished middle initialization */ 7234 TCW_SYNC_4(__kmp_init_middle, TRUE); 7235 7236 KA_TRACE(10, ("__kmp_do_middle_initialize: exit\n")); 7237 } 7238 7239 void __kmp_middle_initialize(void) { 7240 if (__kmp_init_middle) { 7241 return; 7242 } 7243 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 7244 if (__kmp_init_middle) { 7245 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 7246 return; 7247 } 7248 __kmp_do_middle_initialize(); 7249 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 7250 } 7251 7252 void __kmp_parallel_initialize(void) { 7253 int gtid = __kmp_entry_gtid(); // this might be a new root 7254 7255 /* synchronize parallel initialization (for sibling) */ 7256 if (TCR_4(__kmp_init_parallel)) 7257 return; 7258 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 7259 if (TCR_4(__kmp_init_parallel)) { 7260 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 7261 return; 7262 } 7263 7264 /* TODO reinitialization after we have already shut down */ 7265 if (TCR_4(__kmp_global.g.g_done)) { 7266 KA_TRACE( 7267 10, 7268 ("__kmp_parallel_initialize: attempt to init while shutting down\n")); 7269 __kmp_infinite_loop(); 7270 } 7271 7272 /* jc: The lock __kmp_initz_lock is already held, so calling 7273 __kmp_serial_initialize would cause a deadlock. So we call 7274 __kmp_do_serial_initialize directly. */ 7275 if (!__kmp_init_middle) { 7276 __kmp_do_middle_initialize(); 7277 } 7278 __kmp_assign_root_init_mask(); 7279 __kmp_resume_if_hard_paused(); 7280 7281 /* begin initialization */ 7282 KA_TRACE(10, ("__kmp_parallel_initialize: enter\n")); 7283 KMP_ASSERT(KMP_UBER_GTID(gtid)); 7284 7285 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 7286 // Save the FP control regs. 7287 // Worker threads will set theirs to these values at thread startup. 7288 __kmp_store_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word); 7289 __kmp_store_mxcsr(&__kmp_init_mxcsr); 7290 __kmp_init_mxcsr &= KMP_X86_MXCSR_MASK; 7291 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 7292 7293 #if KMP_OS_UNIX 7294 #if KMP_HANDLE_SIGNALS 7295 /* must be after __kmp_serial_initialize */ 7296 __kmp_install_signals(TRUE); 7297 #endif 7298 #endif 7299 7300 __kmp_suspend_initialize(); 7301 7302 #if defined(USE_LOAD_BALANCE) 7303 if (__kmp_global.g.g_dynamic_mode == dynamic_default) { 7304 __kmp_global.g.g_dynamic_mode = dynamic_load_balance; 7305 } 7306 #else 7307 if (__kmp_global.g.g_dynamic_mode == dynamic_default) { 7308 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit; 7309 } 7310 #endif 7311 7312 if (__kmp_version) { 7313 __kmp_print_version_2(); 7314 } 7315 7316 /* we have finished parallel initialization */ 7317 TCW_SYNC_4(__kmp_init_parallel, TRUE); 7318 7319 KMP_MB(); 7320 KA_TRACE(10, ("__kmp_parallel_initialize: exit\n")); 7321 7322 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 7323 } 7324 7325 void __kmp_hidden_helper_initialize() { 7326 if (TCR_4(__kmp_init_hidden_helper)) 7327 return; 7328 7329 // __kmp_parallel_initialize is required before we initialize hidden helper 7330 if (!TCR_4(__kmp_init_parallel)) 7331 __kmp_parallel_initialize(); 7332 7333 // Double check. Note that this double check should not be placed before 7334 // __kmp_parallel_initialize as it will cause dead lock. 7335 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 7336 if (TCR_4(__kmp_init_hidden_helper)) { 7337 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 7338 return; 7339 } 7340 7341 // Set the count of hidden helper tasks to be executed to zero 7342 KMP_ATOMIC_ST_REL(&__kmp_unexecuted_hidden_helper_tasks, 0); 7343 7344 // Set the global variable indicating that we're initializing hidden helper 7345 // team/threads 7346 TCW_SYNC_4(__kmp_init_hidden_helper_threads, TRUE); 7347 7348 // Platform independent initialization 7349 __kmp_do_initialize_hidden_helper_threads(); 7350 7351 // Wait here for the finish of initialization of hidden helper teams 7352 __kmp_hidden_helper_threads_initz_wait(); 7353 7354 // We have finished hidden helper initialization 7355 TCW_SYNC_4(__kmp_init_hidden_helper, TRUE); 7356 7357 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 7358 } 7359 7360 /* ------------------------------------------------------------------------ */ 7361 7362 void __kmp_run_before_invoked_task(int gtid, int tid, kmp_info_t *this_thr, 7363 kmp_team_t *team) { 7364 kmp_disp_t *dispatch; 7365 7366 KMP_MB(); 7367 7368 /* none of the threads have encountered any constructs, yet. */ 7369 this_thr->th.th_local.this_construct = 0; 7370 #if KMP_CACHE_MANAGE 7371 KMP_CACHE_PREFETCH(&this_thr->th.th_bar[bs_forkjoin_barrier].bb.b_arrived); 7372 #endif /* KMP_CACHE_MANAGE */ 7373 dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch); 7374 KMP_DEBUG_ASSERT(dispatch); 7375 KMP_DEBUG_ASSERT(team->t.t_dispatch); 7376 // KMP_DEBUG_ASSERT( this_thr->th.th_dispatch == &team->t.t_dispatch[ 7377 // this_thr->th.th_info.ds.ds_tid ] ); 7378 7379 dispatch->th_disp_index = 0; /* reset the dispatch buffer counter */ 7380 dispatch->th_doacross_buf_idx = 0; // reset doacross dispatch buffer counter 7381 if (__kmp_env_consistency_check) 7382 __kmp_push_parallel(gtid, team->t.t_ident); 7383 7384 KMP_MB(); /* Flush all pending memory write invalidates. */ 7385 } 7386 7387 void __kmp_run_after_invoked_task(int gtid, int tid, kmp_info_t *this_thr, 7388 kmp_team_t *team) { 7389 if (__kmp_env_consistency_check) 7390 __kmp_pop_parallel(gtid, team->t.t_ident); 7391 7392 __kmp_finish_implicit_task(this_thr); 7393 } 7394 7395 int __kmp_invoke_task_func(int gtid) { 7396 int rc; 7397 int tid = __kmp_tid_from_gtid(gtid); 7398 kmp_info_t *this_thr = __kmp_threads[gtid]; 7399 kmp_team_t *team = this_thr->th.th_team; 7400 7401 __kmp_run_before_invoked_task(gtid, tid, this_thr, team); 7402 #if USE_ITT_BUILD 7403 if (__itt_stack_caller_create_ptr) { 7404 // inform ittnotify about entering user's code 7405 if (team->t.t_stack_id != NULL) { 7406 __kmp_itt_stack_callee_enter((__itt_caller)team->t.t_stack_id); 7407 } else { 7408 KMP_DEBUG_ASSERT(team->t.t_parent->t.t_stack_id != NULL); 7409 __kmp_itt_stack_callee_enter( 7410 (__itt_caller)team->t.t_parent->t.t_stack_id); 7411 } 7412 } 7413 #endif /* USE_ITT_BUILD */ 7414 #if INCLUDE_SSC_MARKS 7415 SSC_MARK_INVOKING(); 7416 #endif 7417 7418 #if OMPT_SUPPORT 7419 void *dummy; 7420 void **exit_frame_p; 7421 ompt_data_t *my_task_data; 7422 ompt_data_t *my_parallel_data; 7423 int ompt_team_size; 7424 7425 if (ompt_enabled.enabled) { 7426 exit_frame_p = &(team->t.t_implicit_task_taskdata[tid] 7427 .ompt_task_info.frame.exit_frame.ptr); 7428 } else { 7429 exit_frame_p = &dummy; 7430 } 7431 7432 my_task_data = 7433 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data); 7434 my_parallel_data = &(team->t.ompt_team_info.parallel_data); 7435 if (ompt_enabled.ompt_callback_implicit_task) { 7436 ompt_team_size = team->t.t_nproc; 7437 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 7438 ompt_scope_begin, my_parallel_data, my_task_data, ompt_team_size, 7439 __kmp_tid_from_gtid(gtid), ompt_task_implicit); 7440 OMPT_CUR_TASK_INFO(this_thr)->thread_num = __kmp_tid_from_gtid(gtid); 7441 } 7442 #endif 7443 7444 #if KMP_STATS_ENABLED 7445 stats_state_e previous_state = KMP_GET_THREAD_STATE(); 7446 if (previous_state == stats_state_e::TEAMS_REGION) { 7447 KMP_PUSH_PARTITIONED_TIMER(OMP_teams); 7448 } else { 7449 KMP_PUSH_PARTITIONED_TIMER(OMP_parallel); 7450 } 7451 KMP_SET_THREAD_STATE(IMPLICIT_TASK); 7452 #endif 7453 7454 rc = __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn), gtid, 7455 tid, (int)team->t.t_argc, (void **)team->t.t_argv 7456 #if OMPT_SUPPORT 7457 , 7458 exit_frame_p 7459 #endif 7460 ); 7461 #if OMPT_SUPPORT 7462 *exit_frame_p = NULL; 7463 this_thr->th.ompt_thread_info.parallel_flags |= ompt_parallel_team; 7464 #endif 7465 7466 #if KMP_STATS_ENABLED 7467 if (previous_state == stats_state_e::TEAMS_REGION) { 7468 KMP_SET_THREAD_STATE(previous_state); 7469 } 7470 KMP_POP_PARTITIONED_TIMER(); 7471 #endif 7472 7473 #if USE_ITT_BUILD 7474 if (__itt_stack_caller_create_ptr) { 7475 // inform ittnotify about leaving user's code 7476 if (team->t.t_stack_id != NULL) { 7477 __kmp_itt_stack_callee_leave((__itt_caller)team->t.t_stack_id); 7478 } else { 7479 KMP_DEBUG_ASSERT(team->t.t_parent->t.t_stack_id != NULL); 7480 __kmp_itt_stack_callee_leave( 7481 (__itt_caller)team->t.t_parent->t.t_stack_id); 7482 } 7483 } 7484 #endif /* USE_ITT_BUILD */ 7485 __kmp_run_after_invoked_task(gtid, tid, this_thr, team); 7486 7487 return rc; 7488 } 7489 7490 void __kmp_teams_master(int gtid) { 7491 // This routine is called by all primary threads in teams construct 7492 kmp_info_t *thr = __kmp_threads[gtid]; 7493 kmp_team_t *team = thr->th.th_team; 7494 ident_t *loc = team->t.t_ident; 7495 thr->th.th_set_nproc = thr->th.th_teams_size.nth; 7496 KMP_DEBUG_ASSERT(thr->th.th_teams_microtask); 7497 KMP_DEBUG_ASSERT(thr->th.th_set_nproc); 7498 KA_TRACE(20, ("__kmp_teams_master: T#%d, Tid %d, microtask %p\n", gtid, 7499 __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask)); 7500 7501 // This thread is a new CG root. Set up the proper variables. 7502 kmp_cg_root_t *tmp = (kmp_cg_root_t *)__kmp_allocate(sizeof(kmp_cg_root_t)); 7503 tmp->cg_root = thr; // Make thr the CG root 7504 // Init to thread limit stored when league primary threads were forked 7505 tmp->cg_thread_limit = thr->th.th_current_task->td_icvs.thread_limit; 7506 tmp->cg_nthreads = 1; // Init counter to one active thread, this one 7507 KA_TRACE(100, ("__kmp_teams_master: Thread %p created node %p and init" 7508 " cg_nthreads to 1\n", 7509 thr, tmp)); 7510 tmp->up = thr->th.th_cg_roots; 7511 thr->th.th_cg_roots = tmp; 7512 7513 // Launch league of teams now, but not let workers execute 7514 // (they hang on fork barrier until next parallel) 7515 #if INCLUDE_SSC_MARKS 7516 SSC_MARK_FORKING(); 7517 #endif 7518 __kmp_fork_call(loc, gtid, fork_context_intel, team->t.t_argc, 7519 (microtask_t)thr->th.th_teams_microtask, // "wrapped" task 7520 VOLATILE_CAST(launch_t) __kmp_invoke_task_func, NULL); 7521 #if INCLUDE_SSC_MARKS 7522 SSC_MARK_JOINING(); 7523 #endif 7524 // If the team size was reduced from the limit, set it to the new size 7525 if (thr->th.th_team_nproc < thr->th.th_teams_size.nth) 7526 thr->th.th_teams_size.nth = thr->th.th_team_nproc; 7527 // AC: last parameter "1" eliminates join barrier which won't work because 7528 // worker threads are in a fork barrier waiting for more parallel regions 7529 __kmp_join_call(loc, gtid 7530 #if OMPT_SUPPORT 7531 , 7532 fork_context_intel 7533 #endif 7534 , 7535 1); 7536 } 7537 7538 int __kmp_invoke_teams_master(int gtid) { 7539 kmp_info_t *this_thr = __kmp_threads[gtid]; 7540 kmp_team_t *team = this_thr->th.th_team; 7541 #if KMP_DEBUG 7542 if (!__kmp_threads[gtid]->th.th_team->t.t_serialized) 7543 KMP_DEBUG_ASSERT((void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn == 7544 (void *)__kmp_teams_master); 7545 #endif 7546 __kmp_run_before_invoked_task(gtid, 0, this_thr, team); 7547 #if OMPT_SUPPORT 7548 int tid = __kmp_tid_from_gtid(gtid); 7549 ompt_data_t *task_data = 7550 &team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data; 7551 ompt_data_t *parallel_data = &team->t.ompt_team_info.parallel_data; 7552 if (ompt_enabled.ompt_callback_implicit_task) { 7553 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 7554 ompt_scope_begin, parallel_data, task_data, team->t.t_nproc, tid, 7555 ompt_task_initial); 7556 OMPT_CUR_TASK_INFO(this_thr)->thread_num = tid; 7557 } 7558 #endif 7559 __kmp_teams_master(gtid); 7560 #if OMPT_SUPPORT 7561 this_thr->th.ompt_thread_info.parallel_flags |= ompt_parallel_league; 7562 #endif 7563 __kmp_run_after_invoked_task(gtid, 0, this_thr, team); 7564 return 1; 7565 } 7566 7567 /* this sets the requested number of threads for the next parallel region 7568 encountered by this team. since this should be enclosed in the forkjoin 7569 critical section it should avoid race conditions with asymmetrical nested 7570 parallelism */ 7571 7572 void __kmp_push_num_threads(ident_t *id, int gtid, int num_threads) { 7573 kmp_info_t *thr = __kmp_threads[gtid]; 7574 7575 if (num_threads > 0) 7576 thr->th.th_set_nproc = num_threads; 7577 } 7578 7579 static void __kmp_push_thread_limit(kmp_info_t *thr, int num_teams, 7580 int num_threads) { 7581 KMP_DEBUG_ASSERT(thr); 7582 // Remember the number of threads for inner parallel regions 7583 if (!TCR_4(__kmp_init_middle)) 7584 __kmp_middle_initialize(); // get internal globals calculated 7585 __kmp_assign_root_init_mask(); 7586 KMP_DEBUG_ASSERT(__kmp_avail_proc); 7587 KMP_DEBUG_ASSERT(__kmp_dflt_team_nth); 7588 7589 if (num_threads == 0) { 7590 if (__kmp_teams_thread_limit > 0) { 7591 num_threads = __kmp_teams_thread_limit; 7592 } else { 7593 num_threads = __kmp_avail_proc / num_teams; 7594 } 7595 // adjust num_threads w/o warning as it is not user setting 7596 // num_threads = min(num_threads, nthreads-var, thread-limit-var) 7597 // no thread_limit clause specified - do not change thread-limit-var ICV 7598 if (num_threads > __kmp_dflt_team_nth) { 7599 num_threads = __kmp_dflt_team_nth; // honor nthreads-var ICV 7600 } 7601 if (num_threads > thr->th.th_current_task->td_icvs.thread_limit) { 7602 num_threads = thr->th.th_current_task->td_icvs.thread_limit; 7603 } // prevent team size to exceed thread-limit-var 7604 if (num_teams * num_threads > __kmp_teams_max_nth) { 7605 num_threads = __kmp_teams_max_nth / num_teams; 7606 } 7607 if (num_threads == 0) { 7608 num_threads = 1; 7609 } 7610 } else { 7611 // This thread will be the primary thread of the league primary threads 7612 // Store new thread limit; old limit is saved in th_cg_roots list 7613 thr->th.th_current_task->td_icvs.thread_limit = num_threads; 7614 // num_threads = min(num_threads, nthreads-var) 7615 if (num_threads > __kmp_dflt_team_nth) { 7616 num_threads = __kmp_dflt_team_nth; // honor nthreads-var ICV 7617 } 7618 if (num_teams * num_threads > __kmp_teams_max_nth) { 7619 int new_threads = __kmp_teams_max_nth / num_teams; 7620 if (new_threads == 0) { 7621 new_threads = 1; 7622 } 7623 if (new_threads != num_threads) { 7624 if (!__kmp_reserve_warn) { // user asked for too many threads 7625 __kmp_reserve_warn = 1; // conflicts with KMP_TEAMS_THREAD_LIMIT 7626 __kmp_msg(kmp_ms_warning, 7627 KMP_MSG(CantFormThrTeam, num_threads, new_threads), 7628 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 7629 } 7630 } 7631 num_threads = new_threads; 7632 } 7633 } 7634 thr->th.th_teams_size.nth = num_threads; 7635 } 7636 7637 /* this sets the requested number of teams for the teams region and/or 7638 the number of threads for the next parallel region encountered */ 7639 void __kmp_push_num_teams(ident_t *id, int gtid, int num_teams, 7640 int num_threads) { 7641 kmp_info_t *thr = __kmp_threads[gtid]; 7642 KMP_DEBUG_ASSERT(num_teams >= 0); 7643 KMP_DEBUG_ASSERT(num_threads >= 0); 7644 7645 if (num_teams == 0) { 7646 if (__kmp_nteams > 0) { 7647 num_teams = __kmp_nteams; 7648 } else { 7649 num_teams = 1; // default number of teams is 1. 7650 } 7651 } 7652 if (num_teams > __kmp_teams_max_nth) { // if too many teams requested? 7653 if (!__kmp_reserve_warn) { 7654 __kmp_reserve_warn = 1; 7655 __kmp_msg(kmp_ms_warning, 7656 KMP_MSG(CantFormThrTeam, num_teams, __kmp_teams_max_nth), 7657 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 7658 } 7659 num_teams = __kmp_teams_max_nth; 7660 } 7661 // Set number of teams (number of threads in the outer "parallel" of the 7662 // teams) 7663 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams; 7664 7665 __kmp_push_thread_limit(thr, num_teams, num_threads); 7666 } 7667 7668 /* This sets the requested number of teams for the teams region and/or 7669 the number of threads for the next parallel region encountered */ 7670 void __kmp_push_num_teams_51(ident_t *id, int gtid, int num_teams_lb, 7671 int num_teams_ub, int num_threads) { 7672 kmp_info_t *thr = __kmp_threads[gtid]; 7673 KMP_DEBUG_ASSERT(num_teams_lb >= 0 && num_teams_ub >= 0); 7674 KMP_DEBUG_ASSERT(num_teams_ub >= num_teams_lb); 7675 KMP_DEBUG_ASSERT(num_threads >= 0); 7676 7677 if (num_teams_lb > num_teams_ub) { 7678 __kmp_fatal(KMP_MSG(FailedToCreateTeam, num_teams_lb, num_teams_ub), 7679 KMP_HNT(SetNewBound, __kmp_teams_max_nth), __kmp_msg_null); 7680 } 7681 7682 int num_teams = 1; // defalt number of teams is 1. 7683 7684 if (num_teams_lb == 0 && num_teams_ub > 0) 7685 num_teams_lb = num_teams_ub; 7686 7687 if (num_teams_lb == 0 && num_teams_ub == 0) { // no num_teams clause 7688 num_teams = (__kmp_nteams > 0) ? __kmp_nteams : num_teams; 7689 if (num_teams > __kmp_teams_max_nth) { 7690 if (!__kmp_reserve_warn) { 7691 __kmp_reserve_warn = 1; 7692 __kmp_msg(kmp_ms_warning, 7693 KMP_MSG(CantFormThrTeam, num_teams, __kmp_teams_max_nth), 7694 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 7695 } 7696 num_teams = __kmp_teams_max_nth; 7697 } 7698 } else if (num_teams_lb == num_teams_ub) { // requires exact number of teams 7699 num_teams = num_teams_ub; 7700 } else { // num_teams_lb <= num_teams <= num_teams_ub 7701 if (num_threads == 0) { 7702 if (num_teams_ub > __kmp_teams_max_nth) { 7703 num_teams = num_teams_lb; 7704 } else { 7705 num_teams = num_teams_ub; 7706 } 7707 } else { 7708 num_teams = (num_threads > __kmp_teams_max_nth) 7709 ? num_teams 7710 : __kmp_teams_max_nth / num_threads; 7711 if (num_teams < num_teams_lb) { 7712 num_teams = num_teams_lb; 7713 } else if (num_teams > num_teams_ub) { 7714 num_teams = num_teams_ub; 7715 } 7716 } 7717 } 7718 // Set number of teams (number of threads in the outer "parallel" of the 7719 // teams) 7720 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams; 7721 7722 __kmp_push_thread_limit(thr, num_teams, num_threads); 7723 } 7724 7725 // Set the proc_bind var to use in the following parallel region. 7726 void __kmp_push_proc_bind(ident_t *id, int gtid, kmp_proc_bind_t proc_bind) { 7727 kmp_info_t *thr = __kmp_threads[gtid]; 7728 thr->th.th_set_proc_bind = proc_bind; 7729 } 7730 7731 /* Launch the worker threads into the microtask. */ 7732 7733 void __kmp_internal_fork(ident_t *id, int gtid, kmp_team_t *team) { 7734 kmp_info_t *this_thr = __kmp_threads[gtid]; 7735 7736 #ifdef KMP_DEBUG 7737 int f; 7738 #endif /* KMP_DEBUG */ 7739 7740 KMP_DEBUG_ASSERT(team); 7741 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); 7742 KMP_ASSERT(KMP_MASTER_GTID(gtid)); 7743 KMP_MB(); /* Flush all pending memory write invalidates. */ 7744 7745 team->t.t_construct = 0; /* no single directives seen yet */ 7746 team->t.t_ordered.dt.t_value = 7747 0; /* thread 0 enters the ordered section first */ 7748 7749 /* Reset the identifiers on the dispatch buffer */ 7750 KMP_DEBUG_ASSERT(team->t.t_disp_buffer); 7751 if (team->t.t_max_nproc > 1) { 7752 int i; 7753 for (i = 0; i < __kmp_dispatch_num_buffers; ++i) { 7754 team->t.t_disp_buffer[i].buffer_index = i; 7755 team->t.t_disp_buffer[i].doacross_buf_idx = i; 7756 } 7757 } else { 7758 team->t.t_disp_buffer[0].buffer_index = 0; 7759 team->t.t_disp_buffer[0].doacross_buf_idx = 0; 7760 } 7761 7762 KMP_MB(); /* Flush all pending memory write invalidates. */ 7763 KMP_ASSERT(this_thr->th.th_team == team); 7764 7765 #ifdef KMP_DEBUG 7766 for (f = 0; f < team->t.t_nproc; f++) { 7767 KMP_DEBUG_ASSERT(team->t.t_threads[f] && 7768 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc); 7769 } 7770 #endif /* KMP_DEBUG */ 7771 7772 /* release the worker threads so they may begin working */ 7773 __kmp_fork_barrier(gtid, 0); 7774 } 7775 7776 void __kmp_internal_join(ident_t *id, int gtid, kmp_team_t *team) { 7777 kmp_info_t *this_thr = __kmp_threads[gtid]; 7778 7779 KMP_DEBUG_ASSERT(team); 7780 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); 7781 KMP_ASSERT(KMP_MASTER_GTID(gtid)); 7782 KMP_MB(); /* Flush all pending memory write invalidates. */ 7783 7784 /* Join barrier after fork */ 7785 7786 #ifdef KMP_DEBUG 7787 if (__kmp_threads[gtid] && 7788 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) { 7789 __kmp_printf("GTID: %d, __kmp_threads[%d]=%p\n", gtid, gtid, 7790 __kmp_threads[gtid]); 7791 __kmp_printf("__kmp_threads[%d]->th.th_team_nproc=%d, TEAM: %p, " 7792 "team->t.t_nproc=%d\n", 7793 gtid, __kmp_threads[gtid]->th.th_team_nproc, team, 7794 team->t.t_nproc); 7795 __kmp_print_structure(); 7796 } 7797 KMP_DEBUG_ASSERT(__kmp_threads[gtid] && 7798 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc); 7799 #endif /* KMP_DEBUG */ 7800 7801 __kmp_join_barrier(gtid); /* wait for everyone */ 7802 #if OMPT_SUPPORT 7803 if (ompt_enabled.enabled && 7804 this_thr->th.ompt_thread_info.state == ompt_state_wait_barrier_implicit) { 7805 int ds_tid = this_thr->th.th_info.ds.ds_tid; 7806 ompt_data_t *task_data = OMPT_CUR_TASK_DATA(this_thr); 7807 this_thr->th.ompt_thread_info.state = ompt_state_overhead; 7808 #if OMPT_OPTIONAL 7809 void *codeptr = NULL; 7810 if (KMP_MASTER_TID(ds_tid) && 7811 (ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait) || 7812 ompt_callbacks.ompt_callback(ompt_callback_sync_region))) 7813 codeptr = OMPT_CUR_TEAM_INFO(this_thr)->master_return_address; 7814 7815 if (ompt_enabled.ompt_callback_sync_region_wait) { 7816 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 7817 ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data, 7818 codeptr); 7819 } 7820 if (ompt_enabled.ompt_callback_sync_region) { 7821 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 7822 ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data, 7823 codeptr); 7824 } 7825 #endif 7826 if (!KMP_MASTER_TID(ds_tid) && ompt_enabled.ompt_callback_implicit_task) { 7827 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 7828 ompt_scope_end, NULL, task_data, 0, ds_tid, 7829 ompt_task_implicit); // TODO: Can this be ompt_task_initial? 7830 } 7831 } 7832 #endif 7833 7834 KMP_MB(); /* Flush all pending memory write invalidates. */ 7835 KMP_ASSERT(this_thr->th.th_team == team); 7836 } 7837 7838 /* ------------------------------------------------------------------------ */ 7839 7840 #ifdef USE_LOAD_BALANCE 7841 7842 // Return the worker threads actively spinning in the hot team, if we 7843 // are at the outermost level of parallelism. Otherwise, return 0. 7844 static int __kmp_active_hot_team_nproc(kmp_root_t *root) { 7845 int i; 7846 int retval; 7847 kmp_team_t *hot_team; 7848 7849 if (root->r.r_active) { 7850 return 0; 7851 } 7852 hot_team = root->r.r_hot_team; 7853 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) { 7854 return hot_team->t.t_nproc - 1; // Don't count primary thread 7855 } 7856 7857 // Skip the primary thread - it is accounted for elsewhere. 7858 retval = 0; 7859 for (i = 1; i < hot_team->t.t_nproc; i++) { 7860 if (hot_team->t.t_threads[i]->th.th_active) { 7861 retval++; 7862 } 7863 } 7864 return retval; 7865 } 7866 7867 // Perform an automatic adjustment to the number of 7868 // threads used by the next parallel region. 7869 static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc) { 7870 int retval; 7871 int pool_active; 7872 int hot_team_active; 7873 int team_curr_active; 7874 int system_active; 7875 7876 KB_TRACE(20, ("__kmp_load_balance_nproc: called root:%p set_nproc:%d\n", root, 7877 set_nproc)); 7878 KMP_DEBUG_ASSERT(root); 7879 KMP_DEBUG_ASSERT(root->r.r_root_team->t.t_threads[0] 7880 ->th.th_current_task->td_icvs.dynamic == TRUE); 7881 KMP_DEBUG_ASSERT(set_nproc > 1); 7882 7883 if (set_nproc == 1) { 7884 KB_TRACE(20, ("__kmp_load_balance_nproc: serial execution.\n")); 7885 return 1; 7886 } 7887 7888 // Threads that are active in the thread pool, active in the hot team for this 7889 // particular root (if we are at the outer par level), and the currently 7890 // executing thread (to become the primary thread) are available to add to the 7891 // new team, but are currently contributing to the system load, and must be 7892 // accounted for. 7893 pool_active = __kmp_thread_pool_active_nth; 7894 hot_team_active = __kmp_active_hot_team_nproc(root); 7895 team_curr_active = pool_active + hot_team_active + 1; 7896 7897 // Check the system load. 7898 system_active = __kmp_get_load_balance(__kmp_avail_proc + team_curr_active); 7899 KB_TRACE(30, ("__kmp_load_balance_nproc: system active = %d pool active = %d " 7900 "hot team active = %d\n", 7901 system_active, pool_active, hot_team_active)); 7902 7903 if (system_active < 0) { 7904 // There was an error reading the necessary info from /proc, so use the 7905 // thread limit algorithm instead. Once we set __kmp_global.g.g_dynamic_mode 7906 // = dynamic_thread_limit, we shouldn't wind up getting back here. 7907 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit; 7908 KMP_WARNING(CantLoadBalUsing, "KMP_DYNAMIC_MODE=thread limit"); 7909 7910 // Make this call behave like the thread limit algorithm. 7911 retval = __kmp_avail_proc - __kmp_nth + 7912 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc); 7913 if (retval > set_nproc) { 7914 retval = set_nproc; 7915 } 7916 if (retval < KMP_MIN_NTH) { 7917 retval = KMP_MIN_NTH; 7918 } 7919 7920 KB_TRACE(20, ("__kmp_load_balance_nproc: thread limit exit. retval:%d\n", 7921 retval)); 7922 return retval; 7923 } 7924 7925 // There is a slight delay in the load balance algorithm in detecting new 7926 // running procs. The real system load at this instant should be at least as 7927 // large as the #active omp thread that are available to add to the team. 7928 if (system_active < team_curr_active) { 7929 system_active = team_curr_active; 7930 } 7931 retval = __kmp_avail_proc - system_active + team_curr_active; 7932 if (retval > set_nproc) { 7933 retval = set_nproc; 7934 } 7935 if (retval < KMP_MIN_NTH) { 7936 retval = KMP_MIN_NTH; 7937 } 7938 7939 KB_TRACE(20, ("__kmp_load_balance_nproc: exit. retval:%d\n", retval)); 7940 return retval; 7941 } // __kmp_load_balance_nproc() 7942 7943 #endif /* USE_LOAD_BALANCE */ 7944 7945 /* ------------------------------------------------------------------------ */ 7946 7947 /* NOTE: this is called with the __kmp_init_lock held */ 7948 void __kmp_cleanup(void) { 7949 int f; 7950 7951 KA_TRACE(10, ("__kmp_cleanup: enter\n")); 7952 7953 if (TCR_4(__kmp_init_parallel)) { 7954 #if KMP_HANDLE_SIGNALS 7955 __kmp_remove_signals(); 7956 #endif 7957 TCW_4(__kmp_init_parallel, FALSE); 7958 } 7959 7960 if (TCR_4(__kmp_init_middle)) { 7961 #if KMP_AFFINITY_SUPPORTED 7962 __kmp_affinity_uninitialize(); 7963 #endif /* KMP_AFFINITY_SUPPORTED */ 7964 __kmp_cleanup_hierarchy(); 7965 TCW_4(__kmp_init_middle, FALSE); 7966 } 7967 7968 KA_TRACE(10, ("__kmp_cleanup: go serial cleanup\n")); 7969 7970 if (__kmp_init_serial) { 7971 __kmp_runtime_destroy(); 7972 __kmp_init_serial = FALSE; 7973 } 7974 7975 __kmp_cleanup_threadprivate_caches(); 7976 7977 for (f = 0; f < __kmp_threads_capacity; f++) { 7978 if (__kmp_root[f] != NULL) { 7979 __kmp_free(__kmp_root[f]); 7980 __kmp_root[f] = NULL; 7981 } 7982 } 7983 __kmp_free(__kmp_threads); 7984 // __kmp_threads and __kmp_root were allocated at once, as single block, so 7985 // there is no need in freeing __kmp_root. 7986 __kmp_threads = NULL; 7987 __kmp_root = NULL; 7988 __kmp_threads_capacity = 0; 7989 7990 #if KMP_USE_DYNAMIC_LOCK 7991 __kmp_cleanup_indirect_user_locks(); 7992 #else 7993 __kmp_cleanup_user_locks(); 7994 #endif 7995 #if OMPD_SUPPORT 7996 if (ompd_state) { 7997 __kmp_free(ompd_env_block); 7998 ompd_env_block = NULL; 7999 ompd_env_block_size = 0; 8000 } 8001 #endif 8002 8003 #if KMP_AFFINITY_SUPPORTED 8004 KMP_INTERNAL_FREE(CCAST(char *, __kmp_cpuinfo_file)); 8005 __kmp_cpuinfo_file = NULL; 8006 #endif /* KMP_AFFINITY_SUPPORTED */ 8007 8008 #if KMP_USE_ADAPTIVE_LOCKS 8009 #if KMP_DEBUG_ADAPTIVE_LOCKS 8010 __kmp_print_speculative_stats(); 8011 #endif 8012 #endif 8013 KMP_INTERNAL_FREE(__kmp_nested_nth.nth); 8014 __kmp_nested_nth.nth = NULL; 8015 __kmp_nested_nth.size = 0; 8016 __kmp_nested_nth.used = 0; 8017 KMP_INTERNAL_FREE(__kmp_nested_proc_bind.bind_types); 8018 __kmp_nested_proc_bind.bind_types = NULL; 8019 __kmp_nested_proc_bind.size = 0; 8020 __kmp_nested_proc_bind.used = 0; 8021 if (__kmp_affinity_format) { 8022 KMP_INTERNAL_FREE(__kmp_affinity_format); 8023 __kmp_affinity_format = NULL; 8024 } 8025 8026 __kmp_i18n_catclose(); 8027 8028 #if KMP_USE_HIER_SCHED 8029 __kmp_hier_scheds.deallocate(); 8030 #endif 8031 8032 #if KMP_STATS_ENABLED 8033 __kmp_stats_fini(); 8034 #endif 8035 8036 KA_TRACE(10, ("__kmp_cleanup: exit\n")); 8037 } 8038 8039 /* ------------------------------------------------------------------------ */ 8040 8041 int __kmp_ignore_mppbeg(void) { 8042 char *env; 8043 8044 if ((env = getenv("KMP_IGNORE_MPPBEG")) != NULL) { 8045 if (__kmp_str_match_false(env)) 8046 return FALSE; 8047 } 8048 // By default __kmpc_begin() is no-op. 8049 return TRUE; 8050 } 8051 8052 int __kmp_ignore_mppend(void) { 8053 char *env; 8054 8055 if ((env = getenv("KMP_IGNORE_MPPEND")) != NULL) { 8056 if (__kmp_str_match_false(env)) 8057 return FALSE; 8058 } 8059 // By default __kmpc_end() is no-op. 8060 return TRUE; 8061 } 8062 8063 void __kmp_internal_begin(void) { 8064 int gtid; 8065 kmp_root_t *root; 8066 8067 /* this is a very important step as it will register new sibling threads 8068 and assign these new uber threads a new gtid */ 8069 gtid = __kmp_entry_gtid(); 8070 root = __kmp_threads[gtid]->th.th_root; 8071 KMP_ASSERT(KMP_UBER_GTID(gtid)); 8072 8073 if (root->r.r_begin) 8074 return; 8075 __kmp_acquire_lock(&root->r.r_begin_lock, gtid); 8076 if (root->r.r_begin) { 8077 __kmp_release_lock(&root->r.r_begin_lock, gtid); 8078 return; 8079 } 8080 8081 root->r.r_begin = TRUE; 8082 8083 __kmp_release_lock(&root->r.r_begin_lock, gtid); 8084 } 8085 8086 /* ------------------------------------------------------------------------ */ 8087 8088 void __kmp_user_set_library(enum library_type arg) { 8089 int gtid; 8090 kmp_root_t *root; 8091 kmp_info_t *thread; 8092 8093 /* first, make sure we are initialized so we can get our gtid */ 8094 8095 gtid = __kmp_entry_gtid(); 8096 thread = __kmp_threads[gtid]; 8097 8098 root = thread->th.th_root; 8099 8100 KA_TRACE(20, ("__kmp_user_set_library: enter T#%d, arg: %d, %d\n", gtid, arg, 8101 library_serial)); 8102 if (root->r.r_in_parallel) { /* Must be called in serial section of top-level 8103 thread */ 8104 KMP_WARNING(SetLibraryIncorrectCall); 8105 return; 8106 } 8107 8108 switch (arg) { 8109 case library_serial: 8110 thread->th.th_set_nproc = 0; 8111 set__nproc(thread, 1); 8112 break; 8113 case library_turnaround: 8114 thread->th.th_set_nproc = 0; 8115 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth 8116 : __kmp_dflt_team_nth_ub); 8117 break; 8118 case library_throughput: 8119 thread->th.th_set_nproc = 0; 8120 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth 8121 : __kmp_dflt_team_nth_ub); 8122 break; 8123 default: 8124 KMP_FATAL(UnknownLibraryType, arg); 8125 } 8126 8127 __kmp_aux_set_library(arg); 8128 } 8129 8130 void __kmp_aux_set_stacksize(size_t arg) { 8131 if (!__kmp_init_serial) 8132 __kmp_serial_initialize(); 8133 8134 #if KMP_OS_DARWIN 8135 if (arg & (0x1000 - 1)) { 8136 arg &= ~(0x1000 - 1); 8137 if (arg + 0x1000) /* check for overflow if we round up */ 8138 arg += 0x1000; 8139 } 8140 #endif 8141 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 8142 8143 /* only change the default stacksize before the first parallel region */ 8144 if (!TCR_4(__kmp_init_parallel)) { 8145 size_t value = arg; /* argument is in bytes */ 8146 8147 if (value < __kmp_sys_min_stksize) 8148 value = __kmp_sys_min_stksize; 8149 else if (value > KMP_MAX_STKSIZE) 8150 value = KMP_MAX_STKSIZE; 8151 8152 __kmp_stksize = value; 8153 8154 __kmp_env_stksize = TRUE; /* was KMP_STACKSIZE specified? */ 8155 } 8156 8157 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 8158 } 8159 8160 /* set the behaviour of the runtime library */ 8161 /* TODO this can cause some odd behaviour with sibling parallelism... */ 8162 void __kmp_aux_set_library(enum library_type arg) { 8163 __kmp_library = arg; 8164 8165 switch (__kmp_library) { 8166 case library_serial: { 8167 KMP_INFORM(LibraryIsSerial); 8168 } break; 8169 case library_turnaround: 8170 if (__kmp_use_yield == 1 && !__kmp_use_yield_exp_set) 8171 __kmp_use_yield = 2; // only yield when oversubscribed 8172 break; 8173 case library_throughput: 8174 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) 8175 __kmp_dflt_blocktime = 200; 8176 break; 8177 default: 8178 KMP_FATAL(UnknownLibraryType, arg); 8179 } 8180 } 8181 8182 /* Getting team information common for all team API */ 8183 // Returns NULL if not in teams construct 8184 static kmp_team_t *__kmp_aux_get_team_info(int &teams_serialized) { 8185 kmp_info_t *thr = __kmp_entry_thread(); 8186 teams_serialized = 0; 8187 if (thr->th.th_teams_microtask) { 8188 kmp_team_t *team = thr->th.th_team; 8189 int tlevel = thr->th.th_teams_level; // the level of the teams construct 8190 int ii = team->t.t_level; 8191 teams_serialized = team->t.t_serialized; 8192 int level = tlevel + 1; 8193 KMP_DEBUG_ASSERT(ii >= tlevel); 8194 while (ii > level) { 8195 for (teams_serialized = team->t.t_serialized; 8196 (teams_serialized > 0) && (ii > level); teams_serialized--, ii--) { 8197 } 8198 if (team->t.t_serialized && (!teams_serialized)) { 8199 team = team->t.t_parent; 8200 continue; 8201 } 8202 if (ii > level) { 8203 team = team->t.t_parent; 8204 ii--; 8205 } 8206 } 8207 return team; 8208 } 8209 return NULL; 8210 } 8211 8212 int __kmp_aux_get_team_num() { 8213 int serialized; 8214 kmp_team_t *team = __kmp_aux_get_team_info(serialized); 8215 if (team) { 8216 if (serialized > 1) { 8217 return 0; // teams region is serialized ( 1 team of 1 thread ). 8218 } else { 8219 return team->t.t_master_tid; 8220 } 8221 } 8222 return 0; 8223 } 8224 8225 int __kmp_aux_get_num_teams() { 8226 int serialized; 8227 kmp_team_t *team = __kmp_aux_get_team_info(serialized); 8228 if (team) { 8229 if (serialized > 1) { 8230 return 1; 8231 } else { 8232 return team->t.t_parent->t.t_nproc; 8233 } 8234 } 8235 return 1; 8236 } 8237 8238 /* ------------------------------------------------------------------------ */ 8239 8240 /* 8241 * Affinity Format Parser 8242 * 8243 * Field is in form of: %[[[0].]size]type 8244 * % and type are required (%% means print a literal '%') 8245 * type is either single char or long name surrounded by {}, 8246 * e.g., N or {num_threads} 8247 * 0 => leading zeros 8248 * . => right justified when size is specified 8249 * by default output is left justified 8250 * size is the *minimum* field length 8251 * All other characters are printed as is 8252 * 8253 * Available field types: 8254 * L {thread_level} - omp_get_level() 8255 * n {thread_num} - omp_get_thread_num() 8256 * h {host} - name of host machine 8257 * P {process_id} - process id (integer) 8258 * T {thread_identifier} - native thread identifier (integer) 8259 * N {num_threads} - omp_get_num_threads() 8260 * A {ancestor_tnum} - omp_get_ancestor_thread_num(omp_get_level()-1) 8261 * a {thread_affinity} - comma separated list of integers or integer ranges 8262 * (values of affinity mask) 8263 * 8264 * Implementation-specific field types can be added 8265 * If a type is unknown, print "undefined" 8266 */ 8267 8268 // Structure holding the short name, long name, and corresponding data type 8269 // for snprintf. A table of these will represent the entire valid keyword 8270 // field types. 8271 typedef struct kmp_affinity_format_field_t { 8272 char short_name; // from spec e.g., L -> thread level 8273 const char *long_name; // from spec thread_level -> thread level 8274 char field_format; // data type for snprintf (typically 'd' or 's' 8275 // for integer or string) 8276 } kmp_affinity_format_field_t; 8277 8278 static const kmp_affinity_format_field_t __kmp_affinity_format_table[] = { 8279 #if KMP_AFFINITY_SUPPORTED 8280 {'A', "thread_affinity", 's'}, 8281 #endif 8282 {'t', "team_num", 'd'}, 8283 {'T', "num_teams", 'd'}, 8284 {'L', "nesting_level", 'd'}, 8285 {'n', "thread_num", 'd'}, 8286 {'N', "num_threads", 'd'}, 8287 {'a', "ancestor_tnum", 'd'}, 8288 {'H', "host", 's'}, 8289 {'P', "process_id", 'd'}, 8290 {'i', "native_thread_id", 'd'}}; 8291 8292 // Return the number of characters it takes to hold field 8293 static int __kmp_aux_capture_affinity_field(int gtid, const kmp_info_t *th, 8294 const char **ptr, 8295 kmp_str_buf_t *field_buffer) { 8296 int rc, format_index, field_value; 8297 const char *width_left, *width_right; 8298 bool pad_zeros, right_justify, parse_long_name, found_valid_name; 8299 static const int FORMAT_SIZE = 20; 8300 char format[FORMAT_SIZE] = {0}; 8301 char absolute_short_name = 0; 8302 8303 KMP_DEBUG_ASSERT(gtid >= 0); 8304 KMP_DEBUG_ASSERT(th); 8305 KMP_DEBUG_ASSERT(**ptr == '%'); 8306 KMP_DEBUG_ASSERT(field_buffer); 8307 8308 __kmp_str_buf_clear(field_buffer); 8309 8310 // Skip the initial % 8311 (*ptr)++; 8312 8313 // Check for %% first 8314 if (**ptr == '%') { 8315 __kmp_str_buf_cat(field_buffer, "%", 1); 8316 (*ptr)++; // skip over the second % 8317 return 1; 8318 } 8319 8320 // Parse field modifiers if they are present 8321 pad_zeros = false; 8322 if (**ptr == '0') { 8323 pad_zeros = true; 8324 (*ptr)++; // skip over 0 8325 } 8326 right_justify = false; 8327 if (**ptr == '.') { 8328 right_justify = true; 8329 (*ptr)++; // skip over . 8330 } 8331 // Parse width of field: [width_left, width_right) 8332 width_left = width_right = NULL; 8333 if (**ptr >= '0' && **ptr <= '9') { 8334 width_left = *ptr; 8335 SKIP_DIGITS(*ptr); 8336 width_right = *ptr; 8337 } 8338 8339 // Create the format for KMP_SNPRINTF based on flags parsed above 8340 format_index = 0; 8341 format[format_index++] = '%'; 8342 if (!right_justify) 8343 format[format_index++] = '-'; 8344 if (pad_zeros) 8345 format[format_index++] = '0'; 8346 if (width_left && width_right) { 8347 int i = 0; 8348 // Only allow 8 digit number widths. 8349 // This also prevents overflowing format variable 8350 while (i < 8 && width_left < width_right) { 8351 format[format_index++] = *width_left; 8352 width_left++; 8353 i++; 8354 } 8355 } 8356 8357 // Parse a name (long or short) 8358 // Canonicalize the name into absolute_short_name 8359 found_valid_name = false; 8360 parse_long_name = (**ptr == '{'); 8361 if (parse_long_name) 8362 (*ptr)++; // skip initial left brace 8363 for (size_t i = 0; i < sizeof(__kmp_affinity_format_table) / 8364 sizeof(__kmp_affinity_format_table[0]); 8365 ++i) { 8366 char short_name = __kmp_affinity_format_table[i].short_name; 8367 const char *long_name = __kmp_affinity_format_table[i].long_name; 8368 char field_format = __kmp_affinity_format_table[i].field_format; 8369 if (parse_long_name) { 8370 size_t length = KMP_STRLEN(long_name); 8371 if (strncmp(*ptr, long_name, length) == 0) { 8372 found_valid_name = true; 8373 (*ptr) += length; // skip the long name 8374 } 8375 } else if (**ptr == short_name) { 8376 found_valid_name = true; 8377 (*ptr)++; // skip the short name 8378 } 8379 if (found_valid_name) { 8380 format[format_index++] = field_format; 8381 format[format_index++] = '\0'; 8382 absolute_short_name = short_name; 8383 break; 8384 } 8385 } 8386 if (parse_long_name) { 8387 if (**ptr != '}') { 8388 absolute_short_name = 0; 8389 } else { 8390 (*ptr)++; // skip over the right brace 8391 } 8392 } 8393 8394 // Attempt to fill the buffer with the requested 8395 // value using snprintf within __kmp_str_buf_print() 8396 switch (absolute_short_name) { 8397 case 't': 8398 rc = __kmp_str_buf_print(field_buffer, format, __kmp_aux_get_team_num()); 8399 break; 8400 case 'T': 8401 rc = __kmp_str_buf_print(field_buffer, format, __kmp_aux_get_num_teams()); 8402 break; 8403 case 'L': 8404 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_level); 8405 break; 8406 case 'n': 8407 rc = __kmp_str_buf_print(field_buffer, format, __kmp_tid_from_gtid(gtid)); 8408 break; 8409 case 'H': { 8410 static const int BUFFER_SIZE = 256; 8411 char buf[BUFFER_SIZE]; 8412 __kmp_expand_host_name(buf, BUFFER_SIZE); 8413 rc = __kmp_str_buf_print(field_buffer, format, buf); 8414 } break; 8415 case 'P': 8416 rc = __kmp_str_buf_print(field_buffer, format, getpid()); 8417 break; 8418 case 'i': 8419 rc = __kmp_str_buf_print(field_buffer, format, __kmp_gettid()); 8420 break; 8421 case 'N': 8422 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_nproc); 8423 break; 8424 case 'a': 8425 field_value = 8426 __kmp_get_ancestor_thread_num(gtid, th->th.th_team->t.t_level - 1); 8427 rc = __kmp_str_buf_print(field_buffer, format, field_value); 8428 break; 8429 #if KMP_AFFINITY_SUPPORTED 8430 case 'A': { 8431 kmp_str_buf_t buf; 8432 __kmp_str_buf_init(&buf); 8433 __kmp_affinity_str_buf_mask(&buf, th->th.th_affin_mask); 8434 rc = __kmp_str_buf_print(field_buffer, format, buf.str); 8435 __kmp_str_buf_free(&buf); 8436 } break; 8437 #endif 8438 default: 8439 // According to spec, If an implementation does not have info for field 8440 // type, then "undefined" is printed 8441 rc = __kmp_str_buf_print(field_buffer, "%s", "undefined"); 8442 // Skip the field 8443 if (parse_long_name) { 8444 SKIP_TOKEN(*ptr); 8445 if (**ptr == '}') 8446 (*ptr)++; 8447 } else { 8448 (*ptr)++; 8449 } 8450 } 8451 8452 KMP_ASSERT(format_index <= FORMAT_SIZE); 8453 return rc; 8454 } 8455 8456 /* 8457 * Return number of characters needed to hold the affinity string 8458 * (not including null byte character) 8459 * The resultant string is printed to buffer, which the caller can then 8460 * handle afterwards 8461 */ 8462 size_t __kmp_aux_capture_affinity(int gtid, const char *format, 8463 kmp_str_buf_t *buffer) { 8464 const char *parse_ptr; 8465 size_t retval; 8466 const kmp_info_t *th; 8467 kmp_str_buf_t field; 8468 8469 KMP_DEBUG_ASSERT(buffer); 8470 KMP_DEBUG_ASSERT(gtid >= 0); 8471 8472 __kmp_str_buf_init(&field); 8473 __kmp_str_buf_clear(buffer); 8474 8475 th = __kmp_threads[gtid]; 8476 retval = 0; 8477 8478 // If format is NULL or zero-length string, then we use 8479 // affinity-format-var ICV 8480 parse_ptr = format; 8481 if (parse_ptr == NULL || *parse_ptr == '\0') { 8482 parse_ptr = __kmp_affinity_format; 8483 } 8484 KMP_DEBUG_ASSERT(parse_ptr); 8485 8486 while (*parse_ptr != '\0') { 8487 // Parse a field 8488 if (*parse_ptr == '%') { 8489 // Put field in the buffer 8490 int rc = __kmp_aux_capture_affinity_field(gtid, th, &parse_ptr, &field); 8491 __kmp_str_buf_catbuf(buffer, &field); 8492 retval += rc; 8493 } else { 8494 // Put literal character in buffer 8495 __kmp_str_buf_cat(buffer, parse_ptr, 1); 8496 retval++; 8497 parse_ptr++; 8498 } 8499 } 8500 __kmp_str_buf_free(&field); 8501 return retval; 8502 } 8503 8504 // Displays the affinity string to stdout 8505 void __kmp_aux_display_affinity(int gtid, const char *format) { 8506 kmp_str_buf_t buf; 8507 __kmp_str_buf_init(&buf); 8508 __kmp_aux_capture_affinity(gtid, format, &buf); 8509 __kmp_fprintf(kmp_out, "%s" KMP_END_OF_LINE, buf.str); 8510 __kmp_str_buf_free(&buf); 8511 } 8512 8513 /* ------------------------------------------------------------------------ */ 8514 8515 void __kmp_aux_set_blocktime(int arg, kmp_info_t *thread, int tid) { 8516 int blocktime = arg; /* argument is in milliseconds */ 8517 #if KMP_USE_MONITOR 8518 int bt_intervals; 8519 #endif 8520 kmp_int8 bt_set; 8521 8522 __kmp_save_internal_controls(thread); 8523 8524 /* Normalize and set blocktime for the teams */ 8525 if (blocktime < KMP_MIN_BLOCKTIME) 8526 blocktime = KMP_MIN_BLOCKTIME; 8527 else if (blocktime > KMP_MAX_BLOCKTIME) 8528 blocktime = KMP_MAX_BLOCKTIME; 8529 8530 set__blocktime_team(thread->th.th_team, tid, blocktime); 8531 set__blocktime_team(thread->th.th_serial_team, 0, blocktime); 8532 8533 #if KMP_USE_MONITOR 8534 /* Calculate and set blocktime intervals for the teams */ 8535 bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME(blocktime, __kmp_monitor_wakeups); 8536 8537 set__bt_intervals_team(thread->th.th_team, tid, bt_intervals); 8538 set__bt_intervals_team(thread->th.th_serial_team, 0, bt_intervals); 8539 #endif 8540 8541 /* Set whether blocktime has been set to "TRUE" */ 8542 bt_set = TRUE; 8543 8544 set__bt_set_team(thread->th.th_team, tid, bt_set); 8545 set__bt_set_team(thread->th.th_serial_team, 0, bt_set); 8546 #if KMP_USE_MONITOR 8547 KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, " 8548 "bt_intervals=%d, monitor_updates=%d\n", 8549 __kmp_gtid_from_tid(tid, thread->th.th_team), 8550 thread->th.th_team->t.t_id, tid, blocktime, bt_intervals, 8551 __kmp_monitor_wakeups)); 8552 #else 8553 KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n", 8554 __kmp_gtid_from_tid(tid, thread->th.th_team), 8555 thread->th.th_team->t.t_id, tid, blocktime)); 8556 #endif 8557 } 8558 8559 void __kmp_aux_set_defaults(char const *str, size_t len) { 8560 if (!__kmp_init_serial) { 8561 __kmp_serial_initialize(); 8562 } 8563 __kmp_env_initialize(str); 8564 8565 if (__kmp_settings || __kmp_display_env || __kmp_display_env_verbose) { 8566 __kmp_env_print(); 8567 } 8568 } // __kmp_aux_set_defaults 8569 8570 /* ------------------------------------------------------------------------ */ 8571 /* internal fast reduction routines */ 8572 8573 PACKED_REDUCTION_METHOD_T 8574 __kmp_determine_reduction_method( 8575 ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, 8576 void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data), 8577 kmp_critical_name *lck) { 8578 8579 // Default reduction method: critical construct ( lck != NULL, like in current 8580 // PAROPT ) 8581 // If ( reduce_data!=NULL && reduce_func!=NULL ): the tree-reduction method 8582 // can be selected by RTL 8583 // If loc->flags contains KMP_IDENT_ATOMIC_REDUCE, the atomic reduce method 8584 // can be selected by RTL 8585 // Finally, it's up to OpenMP RTL to make a decision on which method to select 8586 // among generated by PAROPT. 8587 8588 PACKED_REDUCTION_METHOD_T retval; 8589 8590 int team_size; 8591 8592 KMP_DEBUG_ASSERT(loc); // it would be nice to test ( loc != 0 ) 8593 KMP_DEBUG_ASSERT(lck); // it would be nice to test ( lck != 0 ) 8594 8595 #define FAST_REDUCTION_ATOMIC_METHOD_GENERATED \ 8596 ((loc->flags & (KMP_IDENT_ATOMIC_REDUCE)) == (KMP_IDENT_ATOMIC_REDUCE)) 8597 #define FAST_REDUCTION_TREE_METHOD_GENERATED ((reduce_data) && (reduce_func)) 8598 8599 retval = critical_reduce_block; 8600 8601 // another choice of getting a team size (with 1 dynamic deference) is slower 8602 team_size = __kmp_get_team_num_threads(global_tid); 8603 if (team_size == 1) { 8604 8605 retval = empty_reduce_block; 8606 8607 } else { 8608 8609 int atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED; 8610 8611 #if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || \ 8612 KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64 8613 8614 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ 8615 KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HURD 8616 8617 int teamsize_cutoff = 4; 8618 8619 #if KMP_MIC_SUPPORTED 8620 if (__kmp_mic_type != non_mic) { 8621 teamsize_cutoff = 8; 8622 } 8623 #endif 8624 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED; 8625 if (tree_available) { 8626 if (team_size <= teamsize_cutoff) { 8627 if (atomic_available) { 8628 retval = atomic_reduce_block; 8629 } 8630 } else { 8631 retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER; 8632 } 8633 } else if (atomic_available) { 8634 retval = atomic_reduce_block; 8635 } 8636 #else 8637 #error "Unknown or unsupported OS" 8638 #endif // KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || 8639 // KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HURD 8640 8641 #elif KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS 8642 8643 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_WINDOWS || KMP_OS_HURD 8644 8645 // basic tuning 8646 8647 if (atomic_available) { 8648 if (num_vars <= 2) { // && ( team_size <= 8 ) due to false-sharing ??? 8649 retval = atomic_reduce_block; 8650 } 8651 } // otherwise: use critical section 8652 8653 #elif KMP_OS_DARWIN 8654 8655 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED; 8656 if (atomic_available && (num_vars <= 3)) { 8657 retval = atomic_reduce_block; 8658 } else if (tree_available) { 8659 if ((reduce_size > (9 * sizeof(kmp_real64))) && 8660 (reduce_size < (2000 * sizeof(kmp_real64)))) { 8661 retval = TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER; 8662 } 8663 } // otherwise: use critical section 8664 8665 #else 8666 #error "Unknown or unsupported OS" 8667 #endif 8668 8669 #else 8670 #error "Unknown or unsupported architecture" 8671 #endif 8672 } 8673 8674 // KMP_FORCE_REDUCTION 8675 8676 // If the team is serialized (team_size == 1), ignore the forced reduction 8677 // method and stay with the unsynchronized method (empty_reduce_block) 8678 if (__kmp_force_reduction_method != reduction_method_not_defined && 8679 team_size != 1) { 8680 8681 PACKED_REDUCTION_METHOD_T forced_retval = critical_reduce_block; 8682 8683 int atomic_available, tree_available; 8684 8685 switch ((forced_retval = __kmp_force_reduction_method)) { 8686 case critical_reduce_block: 8687 KMP_ASSERT(lck); // lck should be != 0 8688 break; 8689 8690 case atomic_reduce_block: 8691 atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED; 8692 if (!atomic_available) { 8693 KMP_WARNING(RedMethodNotSupported, "atomic"); 8694 forced_retval = critical_reduce_block; 8695 } 8696 break; 8697 8698 case tree_reduce_block: 8699 tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED; 8700 if (!tree_available) { 8701 KMP_WARNING(RedMethodNotSupported, "tree"); 8702 forced_retval = critical_reduce_block; 8703 } else { 8704 #if KMP_FAST_REDUCTION_BARRIER 8705 forced_retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER; 8706 #endif 8707 } 8708 break; 8709 8710 default: 8711 KMP_ASSERT(0); // "unsupported method specified" 8712 } 8713 8714 retval = forced_retval; 8715 } 8716 8717 KA_TRACE(10, ("reduction method selected=%08x\n", retval)); 8718 8719 #undef FAST_REDUCTION_TREE_METHOD_GENERATED 8720 #undef FAST_REDUCTION_ATOMIC_METHOD_GENERATED 8721 8722 return (retval); 8723 } 8724 // this function is for testing set/get/determine reduce method 8725 kmp_int32 __kmp_get_reduce_method(void) { 8726 return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8); 8727 } 8728 8729 // Soft pause sets up threads to ignore blocktime and just go to sleep. 8730 // Spin-wait code checks __kmp_pause_status and reacts accordingly. 8731 void __kmp_soft_pause() { __kmp_pause_status = kmp_soft_paused; } 8732 8733 // Hard pause shuts down the runtime completely. Resume happens naturally when 8734 // OpenMP is used subsequently. 8735 void __kmp_hard_pause() { 8736 __kmp_pause_status = kmp_hard_paused; 8737 __kmp_internal_end_thread(-1); 8738 } 8739 8740 // Soft resume sets __kmp_pause_status, and wakes up all threads. 8741 void __kmp_resume_if_soft_paused() { 8742 if (__kmp_pause_status == kmp_soft_paused) { 8743 __kmp_pause_status = kmp_not_paused; 8744 8745 for (int gtid = 1; gtid < __kmp_threads_capacity; ++gtid) { 8746 kmp_info_t *thread = __kmp_threads[gtid]; 8747 if (thread) { // Wake it if sleeping 8748 kmp_flag_64<> fl(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, 8749 thread); 8750 if (fl.is_sleeping()) 8751 fl.resume(gtid); 8752 else if (__kmp_try_suspend_mx(thread)) { // got suspend lock 8753 __kmp_unlock_suspend_mx(thread); // unlock it; it won't sleep 8754 } else { // thread holds the lock and may sleep soon 8755 do { // until either the thread sleeps, or we can get the lock 8756 if (fl.is_sleeping()) { 8757 fl.resume(gtid); 8758 break; 8759 } else if (__kmp_try_suspend_mx(thread)) { 8760 __kmp_unlock_suspend_mx(thread); 8761 break; 8762 } 8763 } while (1); 8764 } 8765 } 8766 } 8767 } 8768 } 8769 8770 // This function is called via __kmpc_pause_resource. Returns 0 if successful. 8771 // TODO: add warning messages 8772 int __kmp_pause_resource(kmp_pause_status_t level) { 8773 if (level == kmp_not_paused) { // requesting resume 8774 if (__kmp_pause_status == kmp_not_paused) { 8775 // error message about runtime not being paused, so can't resume 8776 return 1; 8777 } else { 8778 KMP_DEBUG_ASSERT(__kmp_pause_status == kmp_soft_paused || 8779 __kmp_pause_status == kmp_hard_paused); 8780 __kmp_pause_status = kmp_not_paused; 8781 return 0; 8782 } 8783 } else if (level == kmp_soft_paused) { // requesting soft pause 8784 if (__kmp_pause_status != kmp_not_paused) { 8785 // error message about already being paused 8786 return 1; 8787 } else { 8788 __kmp_soft_pause(); 8789 return 0; 8790 } 8791 } else if (level == kmp_hard_paused) { // requesting hard pause 8792 if (__kmp_pause_status != kmp_not_paused) { 8793 // error message about already being paused 8794 return 1; 8795 } else { 8796 __kmp_hard_pause(); 8797 return 0; 8798 } 8799 } else { 8800 // error message about invalid level 8801 return 1; 8802 } 8803 } 8804 8805 void __kmp_omp_display_env(int verbose) { 8806 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 8807 if (__kmp_init_serial == 0) 8808 __kmp_do_serial_initialize(); 8809 __kmp_display_env_impl(!verbose, verbose); 8810 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 8811 } 8812 8813 // The team size is changing, so distributed barrier must be modified 8814 void __kmp_resize_dist_barrier(kmp_team_t *team, int old_nthreads, 8815 int new_nthreads) { 8816 KMP_DEBUG_ASSERT(__kmp_barrier_release_pattern[bs_forkjoin_barrier] == 8817 bp_dist_bar); 8818 kmp_info_t **other_threads = team->t.t_threads; 8819 8820 // We want all the workers to stop waiting on the barrier while we adjust the 8821 // size of the team. 8822 for (int f = 1; f < old_nthreads; ++f) { 8823 KMP_DEBUG_ASSERT(other_threads[f] != NULL); 8824 // Ignore threads that are already inactive or not present in the team 8825 if (team->t.t_threads[f]->th.th_used_in_team.load() == 0) { 8826 // teams construct causes thread_limit to get passed in, and some of 8827 // those could be inactive; just ignore them 8828 continue; 8829 } 8830 // If thread is transitioning still to in_use state, wait for it 8831 if (team->t.t_threads[f]->th.th_used_in_team.load() == 3) { 8832 while (team->t.t_threads[f]->th.th_used_in_team.load() == 3) 8833 KMP_CPU_PAUSE(); 8834 } 8835 // The thread should be in_use now 8836 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 1); 8837 // Transition to unused state 8838 team->t.t_threads[f]->th.th_used_in_team.store(2); 8839 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 2); 8840 } 8841 // Release all the workers 8842 kmp_uint64 new_value; // new value for go 8843 new_value = team->t.b->go_release(); 8844 8845 KMP_MFENCE(); 8846 8847 // Workers should see transition status 2 and move to 0; but may need to be 8848 // woken up first 8849 size_t my_go_index; 8850 int count = old_nthreads - 1; 8851 while (count > 0) { 8852 count = old_nthreads - 1; 8853 for (int f = 1; f < old_nthreads; ++f) { 8854 my_go_index = f / team->t.b->threads_per_go; 8855 if (other_threads[f]->th.th_used_in_team.load() != 0) { 8856 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) { // Wake up the workers 8857 kmp_atomic_flag_64<> *flag = (kmp_atomic_flag_64<> *)CCAST( 8858 void *, other_threads[f]->th.th_sleep_loc); 8859 __kmp_atomic_resume_64(other_threads[f]->th.th_info.ds.ds_gtid, flag); 8860 } 8861 } else { 8862 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 0); 8863 count--; 8864 } 8865 } 8866 } 8867 // Now update the barrier size 8868 team->t.b->update_num_threads(new_nthreads); 8869 team->t.b->go_reset(); 8870 } 8871 8872 void __kmp_add_threads_to_team(kmp_team_t *team, int new_nthreads) { 8873 // Add the threads back to the team 8874 KMP_DEBUG_ASSERT(team); 8875 // Threads were paused and pointed at th_used_in_team temporarily during a 8876 // resize of the team. We're going to set th_used_in_team to 3 to indicate to 8877 // the thread that it should transition itself back into the team. Then, if 8878 // blocktime isn't infinite, the thread could be sleeping, so we send a resume 8879 // to wake it up. 8880 for (int f = 1; f < new_nthreads; ++f) { 8881 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 8882 KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team), 0, 8883 3); 8884 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) { // Wake up sleeping threads 8885 __kmp_resume_32(team->t.t_threads[f]->th.th_info.ds.ds_gtid, 8886 (kmp_flag_32<false, false> *)NULL); 8887 } 8888 } 8889 // The threads should be transitioning to the team; when they are done, they 8890 // should have set th_used_in_team to 1. This loop forces master to wait until 8891 // all threads have moved into the team and are waiting in the barrier. 8892 int count = new_nthreads - 1; 8893 while (count > 0) { 8894 count = new_nthreads - 1; 8895 for (int f = 1; f < new_nthreads; ++f) { 8896 if (team->t.t_threads[f]->th.th_used_in_team.load() == 1) { 8897 count--; 8898 } 8899 } 8900 } 8901 } 8902 8903 // Globals and functions for hidden helper task 8904 kmp_info_t **__kmp_hidden_helper_threads; 8905 kmp_info_t *__kmp_hidden_helper_main_thread; 8906 std::atomic<kmp_int32> __kmp_unexecuted_hidden_helper_tasks; 8907 #if KMP_OS_LINUX 8908 kmp_int32 __kmp_hidden_helper_threads_num = 8; 8909 kmp_int32 __kmp_enable_hidden_helper = TRUE; 8910 #else 8911 kmp_int32 __kmp_hidden_helper_threads_num = 0; 8912 kmp_int32 __kmp_enable_hidden_helper = FALSE; 8913 #endif 8914 8915 namespace { 8916 std::atomic<kmp_int32> __kmp_hit_hidden_helper_threads_num; 8917 8918 void __kmp_hidden_helper_wrapper_fn(int *gtid, int *, ...) { 8919 // This is an explicit synchronization on all hidden helper threads in case 8920 // that when a regular thread pushes a hidden helper task to one hidden 8921 // helper thread, the thread has not been awaken once since they're released 8922 // by the main thread after creating the team. 8923 KMP_ATOMIC_INC(&__kmp_hit_hidden_helper_threads_num); 8924 while (KMP_ATOMIC_LD_ACQ(&__kmp_hit_hidden_helper_threads_num) != 8925 __kmp_hidden_helper_threads_num) 8926 ; 8927 8928 // If main thread, then wait for signal 8929 if (__kmpc_master(nullptr, *gtid)) { 8930 // First, unset the initial state and release the initial thread 8931 TCW_4(__kmp_init_hidden_helper_threads, FALSE); 8932 __kmp_hidden_helper_initz_release(); 8933 __kmp_hidden_helper_main_thread_wait(); 8934 // Now wake up all worker threads 8935 for (int i = 1; i < __kmp_hit_hidden_helper_threads_num; ++i) { 8936 __kmp_hidden_helper_worker_thread_signal(); 8937 } 8938 } 8939 } 8940 } // namespace 8941 8942 void __kmp_hidden_helper_threads_initz_routine() { 8943 // Create a new root for hidden helper team/threads 8944 const int gtid = __kmp_register_root(TRUE); 8945 __kmp_hidden_helper_main_thread = __kmp_threads[gtid]; 8946 __kmp_hidden_helper_threads = &__kmp_threads[gtid]; 8947 __kmp_hidden_helper_main_thread->th.th_set_nproc = 8948 __kmp_hidden_helper_threads_num; 8949 8950 KMP_ATOMIC_ST_REL(&__kmp_hit_hidden_helper_threads_num, 0); 8951 8952 __kmpc_fork_call(nullptr, 0, __kmp_hidden_helper_wrapper_fn); 8953 8954 // Set the initialization flag to FALSE 8955 TCW_SYNC_4(__kmp_init_hidden_helper, FALSE); 8956 8957 __kmp_hidden_helper_threads_deinitz_release(); 8958 } 8959 8960 /* Nesting Mode: 8961 Set via KMP_NESTING_MODE, which takes an integer. 8962 Note: we skip duplicate topology levels, and skip levels with only 8963 one entity. 8964 KMP_NESTING_MODE=0 is the default, and doesn't use nesting mode. 8965 KMP_NESTING_MODE=1 sets as many nesting levels as there are distinct levels 8966 in the topology, and initializes the number of threads at each of those 8967 levels to the number of entities at each level, respectively, below the 8968 entity at the parent level. 8969 KMP_NESTING_MODE=N, where N>1, attempts to create up to N nesting levels, 8970 but starts with nesting OFF -- max-active-levels-var is 1 -- and requires 8971 the user to turn nesting on explicitly. This is an even more experimental 8972 option to this experimental feature, and may change or go away in the 8973 future. 8974 */ 8975 8976 // Allocate space to store nesting levels 8977 void __kmp_init_nesting_mode() { 8978 int levels = KMP_HW_LAST; 8979 __kmp_nesting_mode_nlevels = levels; 8980 __kmp_nesting_nth_level = (int *)KMP_INTERNAL_MALLOC(levels * sizeof(int)); 8981 for (int i = 0; i < levels; ++i) 8982 __kmp_nesting_nth_level[i] = 0; 8983 if (__kmp_nested_nth.size < levels) { 8984 __kmp_nested_nth.nth = 8985 (int *)KMP_INTERNAL_REALLOC(__kmp_nested_nth.nth, levels * sizeof(int)); 8986 __kmp_nested_nth.size = levels; 8987 } 8988 } 8989 8990 // Set # threads for top levels of nesting; must be called after topology set 8991 void __kmp_set_nesting_mode_threads() { 8992 kmp_info_t *thread = __kmp_threads[__kmp_entry_gtid()]; 8993 8994 if (__kmp_nesting_mode == 1) 8995 __kmp_nesting_mode_nlevels = KMP_MAX_ACTIVE_LEVELS_LIMIT; 8996 else if (__kmp_nesting_mode > 1) 8997 __kmp_nesting_mode_nlevels = __kmp_nesting_mode; 8998 8999 if (__kmp_topology) { // use topology info 9000 int loc, hw_level; 9001 for (loc = 0, hw_level = 0; hw_level < __kmp_topology->get_depth() && 9002 loc < __kmp_nesting_mode_nlevels; 9003 loc++, hw_level++) { 9004 __kmp_nesting_nth_level[loc] = __kmp_topology->get_ratio(hw_level); 9005 if (__kmp_nesting_nth_level[loc] == 1) 9006 loc--; 9007 } 9008 // Make sure all cores are used 9009 if (__kmp_nesting_mode > 1 && loc > 1) { 9010 int core_level = __kmp_topology->get_level(KMP_HW_CORE); 9011 int num_cores = __kmp_topology->get_count(core_level); 9012 int upper_levels = 1; 9013 for (int level = 0; level < loc - 1; ++level) 9014 upper_levels *= __kmp_nesting_nth_level[level]; 9015 if (upper_levels * __kmp_nesting_nth_level[loc - 1] < num_cores) 9016 __kmp_nesting_nth_level[loc - 1] = 9017 num_cores / __kmp_nesting_nth_level[loc - 2]; 9018 } 9019 __kmp_nesting_mode_nlevels = loc; 9020 __kmp_nested_nth.used = __kmp_nesting_mode_nlevels; 9021 } else { // no topology info available; provide a reasonable guesstimation 9022 if (__kmp_avail_proc >= 4) { 9023 __kmp_nesting_nth_level[0] = __kmp_avail_proc / 2; 9024 __kmp_nesting_nth_level[1] = 2; 9025 __kmp_nesting_mode_nlevels = 2; 9026 } else { 9027 __kmp_nesting_nth_level[0] = __kmp_avail_proc; 9028 __kmp_nesting_mode_nlevels = 1; 9029 } 9030 __kmp_nested_nth.used = __kmp_nesting_mode_nlevels; 9031 } 9032 for (int i = 0; i < __kmp_nesting_mode_nlevels; ++i) { 9033 __kmp_nested_nth.nth[i] = __kmp_nesting_nth_level[i]; 9034 } 9035 set__nproc(thread, __kmp_nesting_nth_level[0]); 9036 if (__kmp_nesting_mode > 1 && __kmp_nesting_mode_nlevels > __kmp_nesting_mode) 9037 __kmp_nesting_mode_nlevels = __kmp_nesting_mode; 9038 if (get__max_active_levels(thread) > 1) { 9039 // if max levels was set, set nesting mode levels to same 9040 __kmp_nesting_mode_nlevels = get__max_active_levels(thread); 9041 } 9042 if (__kmp_nesting_mode == 1) // turn on nesting for this case only 9043 set__max_active_levels(thread, __kmp_nesting_mode_nlevels); 9044 } 9045