1 /* 2 * kmp_runtime.cpp -- KPTS runtime support library 3 */ 4 5 //===----------------------------------------------------------------------===// 6 // 7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 8 // See https://llvm.org/LICENSE.txt for license information. 9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "kmp.h" 14 #include "kmp_affinity.h" 15 #include "kmp_atomic.h" 16 #include "kmp_environment.h" 17 #include "kmp_error.h" 18 #include "kmp_i18n.h" 19 #include "kmp_io.h" 20 #include "kmp_itt.h" 21 #include "kmp_settings.h" 22 #include "kmp_stats.h" 23 #include "kmp_str.h" 24 #include "kmp_wait_release.h" 25 #include "kmp_wrapper_getpid.h" 26 #include "kmp_dispatch.h" 27 #if KMP_USE_HIER_SCHED 28 #include "kmp_dispatch_hier.h" 29 #endif 30 31 #if OMPT_SUPPORT 32 #include "ompt-specific.h" 33 #endif 34 35 /* these are temporary issues to be dealt with */ 36 #define KMP_USE_PRCTL 0 37 38 #if KMP_OS_WINDOWS 39 #include <process.h> 40 #endif 41 42 #include "tsan_annotations.h" 43 44 #if defined(KMP_GOMP_COMPAT) 45 char const __kmp_version_alt_comp[] = 46 KMP_VERSION_PREFIX "alternative compiler support: yes"; 47 #endif /* defined(KMP_GOMP_COMPAT) */ 48 49 char const __kmp_version_omp_api[] = 50 KMP_VERSION_PREFIX "API version: 5.0 (201611)"; 51 52 #ifdef KMP_DEBUG 53 char const __kmp_version_lock[] = 54 KMP_VERSION_PREFIX "lock type: run time selectable"; 55 #endif /* KMP_DEBUG */ 56 57 #define KMP_MIN(x, y) ((x) < (y) ? (x) : (y)) 58 59 /* ------------------------------------------------------------------------ */ 60 61 #if KMP_USE_MONITOR 62 kmp_info_t __kmp_monitor; 63 #endif 64 65 /* Forward declarations */ 66 67 void __kmp_cleanup(void); 68 69 static void __kmp_initialize_info(kmp_info_t *, kmp_team_t *, int tid, 70 int gtid); 71 static void __kmp_initialize_team(kmp_team_t *team, int new_nproc, 72 kmp_internal_control_t *new_icvs, 73 ident_t *loc); 74 #if KMP_AFFINITY_SUPPORTED 75 static void __kmp_partition_places(kmp_team_t *team, 76 int update_master_only = 0); 77 #endif 78 static void __kmp_do_serial_initialize(void); 79 void __kmp_fork_barrier(int gtid, int tid); 80 void __kmp_join_barrier(int gtid); 81 void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc, 82 kmp_internal_control_t *new_icvs, ident_t *loc); 83 84 #ifdef USE_LOAD_BALANCE 85 static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc); 86 #endif 87 88 static int __kmp_expand_threads(int nNeed); 89 #if KMP_OS_WINDOWS 90 static int __kmp_unregister_root_other_thread(int gtid); 91 #endif 92 static void __kmp_unregister_library(void); // called by __kmp_internal_end() 93 static void __kmp_reap_thread(kmp_info_t *thread, int is_root); 94 kmp_info_t *__kmp_thread_pool_insert_pt = NULL; 95 96 /* Calculate the identifier of the current thread */ 97 /* fast (and somewhat portable) way to get unique identifier of executing 98 thread. Returns KMP_GTID_DNE if we haven't been assigned a gtid. */ 99 int __kmp_get_global_thread_id() { 100 int i; 101 kmp_info_t **other_threads; 102 size_t stack_data; 103 char *stack_addr; 104 size_t stack_size; 105 char *stack_base; 106 107 KA_TRACE( 108 1000, 109 ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n", 110 __kmp_nth, __kmp_all_nth)); 111 112 /* JPH - to handle the case where __kmpc_end(0) is called immediately prior to 113 a parallel region, made it return KMP_GTID_DNE to force serial_initialize 114 by caller. Had to handle KMP_GTID_DNE at all call-sites, or else guarantee 115 __kmp_init_gtid for this to work. */ 116 117 if (!TCR_4(__kmp_init_gtid)) 118 return KMP_GTID_DNE; 119 120 #ifdef KMP_TDATA_GTID 121 if (TCR_4(__kmp_gtid_mode) >= 3) { 122 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using TDATA\n")); 123 return __kmp_gtid; 124 } 125 #endif 126 if (TCR_4(__kmp_gtid_mode) >= 2) { 127 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using keyed TLS\n")); 128 return __kmp_gtid_get_specific(); 129 } 130 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using internal alg.\n")); 131 132 stack_addr = (char *)&stack_data; 133 other_threads = __kmp_threads; 134 135 /* ATT: The code below is a source of potential bugs due to unsynchronized 136 access to __kmp_threads array. For example: 137 1. Current thread loads other_threads[i] to thr and checks it, it is 138 non-NULL. 139 2. Current thread is suspended by OS. 140 3. Another thread unregisters and finishes (debug versions of free() 141 may fill memory with something like 0xEF). 142 4. Current thread is resumed. 143 5. Current thread reads junk from *thr. 144 TODO: Fix it. --ln */ 145 146 for (i = 0; i < __kmp_threads_capacity; i++) { 147 148 kmp_info_t *thr = (kmp_info_t *)TCR_SYNC_PTR(other_threads[i]); 149 if (!thr) 150 continue; 151 152 stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize); 153 stack_base = (char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase); 154 155 /* stack grows down -- search through all of the active threads */ 156 157 if (stack_addr <= stack_base) { 158 size_t stack_diff = stack_base - stack_addr; 159 160 if (stack_diff <= stack_size) { 161 /* The only way we can be closer than the allocated */ 162 /* stack size is if we are running on this thread. */ 163 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == i); 164 return i; 165 } 166 } 167 } 168 169 /* get specific to try and determine our gtid */ 170 KA_TRACE(1000, 171 ("*** __kmp_get_global_thread_id: internal alg. failed to find " 172 "thread, using TLS\n")); 173 i = __kmp_gtid_get_specific(); 174 175 /*fprintf( stderr, "=== %d\n", i ); */ /* GROO */ 176 177 /* if we havn't been assigned a gtid, then return code */ 178 if (i < 0) 179 return i; 180 181 /* dynamically updated stack window for uber threads to avoid get_specific 182 call */ 183 if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)) { 184 KMP_FATAL(StackOverflow, i); 185 } 186 187 stack_base = (char *)other_threads[i]->th.th_info.ds.ds_stackbase; 188 if (stack_addr > stack_base) { 189 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr); 190 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize, 191 other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr - 192 stack_base); 193 } else { 194 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize, 195 stack_base - stack_addr); 196 } 197 198 /* Reprint stack bounds for ubermaster since they have been refined */ 199 if (__kmp_storage_map) { 200 char *stack_end = (char *)other_threads[i]->th.th_info.ds.ds_stackbase; 201 char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize; 202 __kmp_print_storage_map_gtid(i, stack_beg, stack_end, 203 other_threads[i]->th.th_info.ds.ds_stacksize, 204 "th_%d stack (refinement)", i); 205 } 206 return i; 207 } 208 209 int __kmp_get_global_thread_id_reg() { 210 int gtid; 211 212 if (!__kmp_init_serial) { 213 gtid = KMP_GTID_DNE; 214 } else 215 #ifdef KMP_TDATA_GTID 216 if (TCR_4(__kmp_gtid_mode) >= 3) { 217 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using TDATA\n")); 218 gtid = __kmp_gtid; 219 } else 220 #endif 221 if (TCR_4(__kmp_gtid_mode) >= 2) { 222 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using keyed TLS\n")); 223 gtid = __kmp_gtid_get_specific(); 224 } else { 225 KA_TRACE(1000, 226 ("*** __kmp_get_global_thread_id_reg: using internal alg.\n")); 227 gtid = __kmp_get_global_thread_id(); 228 } 229 230 /* we must be a new uber master sibling thread */ 231 if (gtid == KMP_GTID_DNE) { 232 KA_TRACE(10, 233 ("__kmp_get_global_thread_id_reg: Encountered new root thread. " 234 "Registering a new gtid.\n")); 235 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 236 if (!__kmp_init_serial) { 237 __kmp_do_serial_initialize(); 238 gtid = __kmp_gtid_get_specific(); 239 } else { 240 gtid = __kmp_register_root(FALSE); 241 } 242 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 243 /*__kmp_printf( "+++ %d\n", gtid ); */ /* GROO */ 244 } 245 246 KMP_DEBUG_ASSERT(gtid >= 0); 247 248 return gtid; 249 } 250 251 /* caller must hold forkjoin_lock */ 252 void __kmp_check_stack_overlap(kmp_info_t *th) { 253 int f; 254 char *stack_beg = NULL; 255 char *stack_end = NULL; 256 int gtid; 257 258 KA_TRACE(10, ("__kmp_check_stack_overlap: called\n")); 259 if (__kmp_storage_map) { 260 stack_end = (char *)th->th.th_info.ds.ds_stackbase; 261 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize; 262 263 gtid = __kmp_gtid_from_thread(th); 264 265 if (gtid == KMP_GTID_MONITOR) { 266 __kmp_print_storage_map_gtid( 267 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize, 268 "th_%s stack (%s)", "mon", 269 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual"); 270 } else { 271 __kmp_print_storage_map_gtid( 272 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize, 273 "th_%d stack (%s)", gtid, 274 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual"); 275 } 276 } 277 278 /* No point in checking ubermaster threads since they use refinement and 279 * cannot overlap */ 280 gtid = __kmp_gtid_from_thread(th); 281 if (__kmp_env_checks == TRUE && !KMP_UBER_GTID(gtid)) { 282 KA_TRACE(10, 283 ("__kmp_check_stack_overlap: performing extensive checking\n")); 284 if (stack_beg == NULL) { 285 stack_end = (char *)th->th.th_info.ds.ds_stackbase; 286 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize; 287 } 288 289 for (f = 0; f < __kmp_threads_capacity; f++) { 290 kmp_info_t *f_th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[f]); 291 292 if (f_th && f_th != th) { 293 char *other_stack_end = 294 (char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase); 295 char *other_stack_beg = 296 other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize); 297 if ((stack_beg > other_stack_beg && stack_beg < other_stack_end) || 298 (stack_end > other_stack_beg && stack_end < other_stack_end)) { 299 300 /* Print the other stack values before the abort */ 301 if (__kmp_storage_map) 302 __kmp_print_storage_map_gtid( 303 -1, other_stack_beg, other_stack_end, 304 (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize), 305 "th_%d stack (overlapped)", __kmp_gtid_from_thread(f_th)); 306 307 __kmp_fatal(KMP_MSG(StackOverlap), KMP_HNT(ChangeStackLimit), 308 __kmp_msg_null); 309 } 310 } 311 } 312 } 313 KA_TRACE(10, ("__kmp_check_stack_overlap: returning\n")); 314 } 315 316 /* ------------------------------------------------------------------------ */ 317 318 void __kmp_infinite_loop(void) { 319 static int done = FALSE; 320 321 while (!done) { 322 KMP_YIELD(TRUE); 323 } 324 } 325 326 #define MAX_MESSAGE 512 327 328 void __kmp_print_storage_map_gtid(int gtid, void *p1, void *p2, size_t size, 329 char const *format, ...) { 330 char buffer[MAX_MESSAGE]; 331 va_list ap; 332 333 va_start(ap, format); 334 KMP_SNPRINTF(buffer, sizeof(buffer), "OMP storage map: %p %p%8lu %s\n", p1, 335 p2, (unsigned long)size, format); 336 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock); 337 __kmp_vprintf(kmp_err, buffer, ap); 338 #if KMP_PRINT_DATA_PLACEMENT 339 int node; 340 if (gtid >= 0) { 341 if (p1 <= p2 && (char *)p2 - (char *)p1 == size) { 342 if (__kmp_storage_map_verbose) { 343 node = __kmp_get_host_node(p1); 344 if (node < 0) /* doesn't work, so don't try this next time */ 345 __kmp_storage_map_verbose = FALSE; 346 else { 347 char *last; 348 int lastNode; 349 int localProc = __kmp_get_cpu_from_gtid(gtid); 350 351 const int page_size = KMP_GET_PAGE_SIZE(); 352 353 p1 = (void *)((size_t)p1 & ~((size_t)page_size - 1)); 354 p2 = (void *)(((size_t)p2 - 1) & ~((size_t)page_size - 1)); 355 if (localProc >= 0) 356 __kmp_printf_no_lock(" GTID %d localNode %d\n", gtid, 357 localProc >> 1); 358 else 359 __kmp_printf_no_lock(" GTID %d\n", gtid); 360 #if KMP_USE_PRCTL 361 /* The more elaborate format is disabled for now because of the prctl 362 * hanging bug. */ 363 do { 364 last = p1; 365 lastNode = node; 366 /* This loop collates adjacent pages with the same host node. */ 367 do { 368 (char *)p1 += page_size; 369 } while (p1 <= p2 && (node = __kmp_get_host_node(p1)) == lastNode); 370 __kmp_printf_no_lock(" %p-%p memNode %d\n", last, (char *)p1 - 1, 371 lastNode); 372 } while (p1 <= p2); 373 #else 374 __kmp_printf_no_lock(" %p-%p memNode %d\n", p1, 375 (char *)p1 + (page_size - 1), 376 __kmp_get_host_node(p1)); 377 if (p1 < p2) { 378 __kmp_printf_no_lock(" %p-%p memNode %d\n", p2, 379 (char *)p2 + (page_size - 1), 380 __kmp_get_host_node(p2)); 381 } 382 #endif 383 } 384 } 385 } else 386 __kmp_printf_no_lock(" %s\n", KMP_I18N_STR(StorageMapWarning)); 387 } 388 #endif /* KMP_PRINT_DATA_PLACEMENT */ 389 __kmp_release_bootstrap_lock(&__kmp_stdio_lock); 390 } 391 392 void __kmp_warn(char const *format, ...) { 393 char buffer[MAX_MESSAGE]; 394 va_list ap; 395 396 if (__kmp_generate_warnings == kmp_warnings_off) { 397 return; 398 } 399 400 va_start(ap, format); 401 402 KMP_SNPRINTF(buffer, sizeof(buffer), "OMP warning: %s\n", format); 403 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock); 404 __kmp_vprintf(kmp_err, buffer, ap); 405 __kmp_release_bootstrap_lock(&__kmp_stdio_lock); 406 407 va_end(ap); 408 } 409 410 void __kmp_abort_process() { 411 // Later threads may stall here, but that's ok because abort() will kill them. 412 __kmp_acquire_bootstrap_lock(&__kmp_exit_lock); 413 414 if (__kmp_debug_buf) { 415 __kmp_dump_debug_buffer(); 416 } 417 418 if (KMP_OS_WINDOWS) { 419 // Let other threads know of abnormal termination and prevent deadlock 420 // if abort happened during library initialization or shutdown 421 __kmp_global.g.g_abort = SIGABRT; 422 423 /* On Windows* OS by default abort() causes pop-up error box, which stalls 424 nightly testing. Unfortunately, we cannot reliably suppress pop-up error 425 boxes. _set_abort_behavior() works well, but this function is not 426 available in VS7 (this is not problem for DLL, but it is a problem for 427 static OpenMP RTL). SetErrorMode (and so, timelimit utility) does not 428 help, at least in some versions of MS C RTL. 429 430 It seems following sequence is the only way to simulate abort() and 431 avoid pop-up error box. */ 432 raise(SIGABRT); 433 _exit(3); // Just in case, if signal ignored, exit anyway. 434 } else { 435 abort(); 436 } 437 438 __kmp_infinite_loop(); 439 __kmp_release_bootstrap_lock(&__kmp_exit_lock); 440 441 } // __kmp_abort_process 442 443 void __kmp_abort_thread(void) { 444 // TODO: Eliminate g_abort global variable and this function. 445 // In case of abort just call abort(), it will kill all the threads. 446 __kmp_infinite_loop(); 447 } // __kmp_abort_thread 448 449 /* Print out the storage map for the major kmp_info_t thread data structures 450 that are allocated together. */ 451 452 static void __kmp_print_thread_storage_map(kmp_info_t *thr, int gtid) { 453 __kmp_print_storage_map_gtid(gtid, thr, thr + 1, sizeof(kmp_info_t), "th_%d", 454 gtid); 455 456 __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team, 457 sizeof(kmp_desc_t), "th_%d.th_info", gtid); 458 459 __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head, 460 sizeof(kmp_local_t), "th_%d.th_local", gtid); 461 462 __kmp_print_storage_map_gtid( 463 gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier], 464 sizeof(kmp_balign_t) * bs_last_barrier, "th_%d.th_bar", gtid); 465 466 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier], 467 &thr->th.th_bar[bs_plain_barrier + 1], 468 sizeof(kmp_balign_t), "th_%d.th_bar[plain]", 469 gtid); 470 471 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier], 472 &thr->th.th_bar[bs_forkjoin_barrier + 1], 473 sizeof(kmp_balign_t), "th_%d.th_bar[forkjoin]", 474 gtid); 475 476 #if KMP_FAST_REDUCTION_BARRIER 477 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier], 478 &thr->th.th_bar[bs_reduction_barrier + 1], 479 sizeof(kmp_balign_t), "th_%d.th_bar[reduction]", 480 gtid); 481 #endif // KMP_FAST_REDUCTION_BARRIER 482 } 483 484 /* Print out the storage map for the major kmp_team_t team data structures 485 that are allocated together. */ 486 487 static void __kmp_print_team_storage_map(const char *header, kmp_team_t *team, 488 int team_id, int num_thr) { 489 int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2; 490 __kmp_print_storage_map_gtid(-1, team, team + 1, sizeof(kmp_team_t), "%s_%d", 491 header, team_id); 492 493 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0], 494 &team->t.t_bar[bs_last_barrier], 495 sizeof(kmp_balign_team_t) * bs_last_barrier, 496 "%s_%d.t_bar", header, team_id); 497 498 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier], 499 &team->t.t_bar[bs_plain_barrier + 1], 500 sizeof(kmp_balign_team_t), "%s_%d.t_bar[plain]", 501 header, team_id); 502 503 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier], 504 &team->t.t_bar[bs_forkjoin_barrier + 1], 505 sizeof(kmp_balign_team_t), 506 "%s_%d.t_bar[forkjoin]", header, team_id); 507 508 #if KMP_FAST_REDUCTION_BARRIER 509 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier], 510 &team->t.t_bar[bs_reduction_barrier + 1], 511 sizeof(kmp_balign_team_t), 512 "%s_%d.t_bar[reduction]", header, team_id); 513 #endif // KMP_FAST_REDUCTION_BARRIER 514 515 __kmp_print_storage_map_gtid( 516 -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr], 517 sizeof(kmp_disp_t) * num_thr, "%s_%d.t_dispatch", header, team_id); 518 519 __kmp_print_storage_map_gtid( 520 -1, &team->t.t_threads[0], &team->t.t_threads[num_thr], 521 sizeof(kmp_info_t *) * num_thr, "%s_%d.t_threads", header, team_id); 522 523 __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0], 524 &team->t.t_disp_buffer[num_disp_buff], 525 sizeof(dispatch_shared_info_t) * num_disp_buff, 526 "%s_%d.t_disp_buffer", header, team_id); 527 } 528 529 static void __kmp_init_allocator() { __kmp_init_memkind(); } 530 static void __kmp_fini_allocator() { __kmp_fini_memkind(); } 531 532 /* ------------------------------------------------------------------------ */ 533 534 #if KMP_DYNAMIC_LIB 535 #if KMP_OS_WINDOWS 536 537 static void __kmp_reset_lock(kmp_bootstrap_lock_t *lck) { 538 // TODO: Change to __kmp_break_bootstrap_lock(). 539 __kmp_init_bootstrap_lock(lck); // make the lock released 540 } 541 542 static void __kmp_reset_locks_on_process_detach(int gtid_req) { 543 int i; 544 int thread_count; 545 546 // PROCESS_DETACH is expected to be called by a thread that executes 547 // ProcessExit() or FreeLibrary(). OS terminates other threads (except the one 548 // calling ProcessExit or FreeLibrary). So, it might be safe to access the 549 // __kmp_threads[] without taking the forkjoin_lock. However, in fact, some 550 // threads can be still alive here, although being about to be terminated. The 551 // threads in the array with ds_thread==0 are most suspicious. Actually, it 552 // can be not safe to access the __kmp_threads[]. 553 554 // TODO: does it make sense to check __kmp_roots[] ? 555 556 // Let's check that there are no other alive threads registered with the OMP 557 // lib. 558 while (1) { 559 thread_count = 0; 560 for (i = 0; i < __kmp_threads_capacity; ++i) { 561 if (!__kmp_threads) 562 continue; 563 kmp_info_t *th = __kmp_threads[i]; 564 if (th == NULL) 565 continue; 566 int gtid = th->th.th_info.ds.ds_gtid; 567 if (gtid == gtid_req) 568 continue; 569 if (gtid < 0) 570 continue; 571 DWORD exit_val; 572 int alive = __kmp_is_thread_alive(th, &exit_val); 573 if (alive) { 574 ++thread_count; 575 } 576 } 577 if (thread_count == 0) 578 break; // success 579 } 580 581 // Assume that I'm alone. Now it might be safe to check and reset locks. 582 // __kmp_forkjoin_lock and __kmp_stdio_lock are expected to be reset. 583 __kmp_reset_lock(&__kmp_forkjoin_lock); 584 #ifdef KMP_DEBUG 585 __kmp_reset_lock(&__kmp_stdio_lock); 586 #endif // KMP_DEBUG 587 } 588 589 BOOL WINAPI DllMain(HINSTANCE hInstDLL, DWORD fdwReason, LPVOID lpReserved) { 590 //__kmp_acquire_bootstrap_lock( &__kmp_initz_lock ); 591 592 switch (fdwReason) { 593 594 case DLL_PROCESS_ATTACH: 595 KA_TRACE(10, ("DllMain: PROCESS_ATTACH\n")); 596 597 return TRUE; 598 599 case DLL_PROCESS_DETACH: 600 KA_TRACE(10, ("DllMain: PROCESS_DETACH T#%d\n", __kmp_gtid_get_specific())); 601 602 if (lpReserved != NULL) { 603 // lpReserved is used for telling the difference: 604 // lpReserved == NULL when FreeLibrary() was called, 605 // lpReserved != NULL when the process terminates. 606 // When FreeLibrary() is called, worker threads remain alive. So they will 607 // release the forkjoin lock by themselves. When the process terminates, 608 // worker threads disappear triggering the problem of unreleased forkjoin 609 // lock as described below. 610 611 // A worker thread can take the forkjoin lock. The problem comes up if 612 // that worker thread becomes dead before it releases the forkjoin lock. 613 // The forkjoin lock remains taken, while the thread executing 614 // DllMain()->PROCESS_DETACH->__kmp_internal_end_library() below will try 615 // to take the forkjoin lock and will always fail, so that the application 616 // will never finish [normally]. This scenario is possible if 617 // __kmpc_end() has not been executed. It looks like it's not a corner 618 // case, but common cases: 619 // - the main function was compiled by an alternative compiler; 620 // - the main function was compiled by icl but without /Qopenmp 621 // (application with plugins); 622 // - application terminates by calling C exit(), Fortran CALL EXIT() or 623 // Fortran STOP. 624 // - alive foreign thread prevented __kmpc_end from doing cleanup. 625 // 626 // This is a hack to work around the problem. 627 // TODO: !!! figure out something better. 628 __kmp_reset_locks_on_process_detach(__kmp_gtid_get_specific()); 629 } 630 631 __kmp_internal_end_library(__kmp_gtid_get_specific()); 632 633 return TRUE; 634 635 case DLL_THREAD_ATTACH: 636 KA_TRACE(10, ("DllMain: THREAD_ATTACH\n")); 637 638 /* if we want to register new siblings all the time here call 639 * __kmp_get_gtid(); */ 640 return TRUE; 641 642 case DLL_THREAD_DETACH: 643 KA_TRACE(10, ("DllMain: THREAD_DETACH T#%d\n", __kmp_gtid_get_specific())); 644 645 __kmp_internal_end_thread(__kmp_gtid_get_specific()); 646 return TRUE; 647 } 648 649 return TRUE; 650 } 651 652 #endif /* KMP_OS_WINDOWS */ 653 #endif /* KMP_DYNAMIC_LIB */ 654 655 /* __kmp_parallel_deo -- Wait until it's our turn. */ 656 void __kmp_parallel_deo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) { 657 int gtid = *gtid_ref; 658 #ifdef BUILD_PARALLEL_ORDERED 659 kmp_team_t *team = __kmp_team_from_gtid(gtid); 660 #endif /* BUILD_PARALLEL_ORDERED */ 661 662 if (__kmp_env_consistency_check) { 663 if (__kmp_threads[gtid]->th.th_root->r.r_active) 664 #if KMP_USE_DYNAMIC_LOCK 665 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL, 0); 666 #else 667 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL); 668 #endif 669 } 670 #ifdef BUILD_PARALLEL_ORDERED 671 if (!team->t.t_serialized) { 672 KMP_MB(); 673 KMP_WAIT(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid), KMP_EQ, 674 NULL); 675 KMP_MB(); 676 } 677 #endif /* BUILD_PARALLEL_ORDERED */ 678 } 679 680 /* __kmp_parallel_dxo -- Signal the next task. */ 681 void __kmp_parallel_dxo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) { 682 int gtid = *gtid_ref; 683 #ifdef BUILD_PARALLEL_ORDERED 684 int tid = __kmp_tid_from_gtid(gtid); 685 kmp_team_t *team = __kmp_team_from_gtid(gtid); 686 #endif /* BUILD_PARALLEL_ORDERED */ 687 688 if (__kmp_env_consistency_check) { 689 if (__kmp_threads[gtid]->th.th_root->r.r_active) 690 __kmp_pop_sync(gtid, ct_ordered_in_parallel, loc_ref); 691 } 692 #ifdef BUILD_PARALLEL_ORDERED 693 if (!team->t.t_serialized) { 694 KMP_MB(); /* Flush all pending memory write invalidates. */ 695 696 /* use the tid of the next thread in this team */ 697 /* TODO replace with general release procedure */ 698 team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc); 699 700 KMP_MB(); /* Flush all pending memory write invalidates. */ 701 } 702 #endif /* BUILD_PARALLEL_ORDERED */ 703 } 704 705 /* ------------------------------------------------------------------------ */ 706 /* The BARRIER for a SINGLE process section is always explicit */ 707 708 int __kmp_enter_single(int gtid, ident_t *id_ref, int push_ws) { 709 int status; 710 kmp_info_t *th; 711 kmp_team_t *team; 712 713 if (!TCR_4(__kmp_init_parallel)) 714 __kmp_parallel_initialize(); 715 __kmp_resume_if_soft_paused(); 716 717 th = __kmp_threads[gtid]; 718 team = th->th.th_team; 719 status = 0; 720 721 th->th.th_ident = id_ref; 722 723 if (team->t.t_serialized) { 724 status = 1; 725 } else { 726 kmp_int32 old_this = th->th.th_local.this_construct; 727 728 ++th->th.th_local.this_construct; 729 /* try to set team count to thread count--success means thread got the 730 single block */ 731 /* TODO: Should this be acquire or release? */ 732 if (team->t.t_construct == old_this) { 733 status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this, 734 th->th.th_local.this_construct); 735 } 736 #if USE_ITT_BUILD 737 if (__itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 && 738 KMP_MASTER_GTID(gtid) && th->th.th_teams_microtask == NULL && 739 team->t.t_active_level == 740 1) { // Only report metadata by master of active team at level 1 741 __kmp_itt_metadata_single(id_ref); 742 } 743 #endif /* USE_ITT_BUILD */ 744 } 745 746 if (__kmp_env_consistency_check) { 747 if (status && push_ws) { 748 __kmp_push_workshare(gtid, ct_psingle, id_ref); 749 } else { 750 __kmp_check_workshare(gtid, ct_psingle, id_ref); 751 } 752 } 753 #if USE_ITT_BUILD 754 if (status) { 755 __kmp_itt_single_start(gtid); 756 } 757 #endif /* USE_ITT_BUILD */ 758 return status; 759 } 760 761 void __kmp_exit_single(int gtid) { 762 #if USE_ITT_BUILD 763 __kmp_itt_single_end(gtid); 764 #endif /* USE_ITT_BUILD */ 765 if (__kmp_env_consistency_check) 766 __kmp_pop_workshare(gtid, ct_psingle, NULL); 767 } 768 769 /* determine if we can go parallel or must use a serialized parallel region and 770 * how many threads we can use 771 * set_nproc is the number of threads requested for the team 772 * returns 0 if we should serialize or only use one thread, 773 * otherwise the number of threads to use 774 * The forkjoin lock is held by the caller. */ 775 static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team, 776 int master_tid, int set_nthreads, 777 int enter_teams) { 778 int capacity; 779 int new_nthreads; 780 KMP_DEBUG_ASSERT(__kmp_init_serial); 781 KMP_DEBUG_ASSERT(root && parent_team); 782 kmp_info_t *this_thr = parent_team->t.t_threads[master_tid]; 783 784 // If dyn-var is set, dynamically adjust the number of desired threads, 785 // according to the method specified by dynamic_mode. 786 new_nthreads = set_nthreads; 787 if (!get__dynamic_2(parent_team, master_tid)) { 788 ; 789 } 790 #ifdef USE_LOAD_BALANCE 791 else if (__kmp_global.g.g_dynamic_mode == dynamic_load_balance) { 792 new_nthreads = __kmp_load_balance_nproc(root, set_nthreads); 793 if (new_nthreads == 1) { 794 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced " 795 "reservation to 1 thread\n", 796 master_tid)); 797 return 1; 798 } 799 if (new_nthreads < set_nthreads) { 800 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced " 801 "reservation to %d threads\n", 802 master_tid, new_nthreads)); 803 } 804 } 805 #endif /* USE_LOAD_BALANCE */ 806 else if (__kmp_global.g.g_dynamic_mode == dynamic_thread_limit) { 807 new_nthreads = __kmp_avail_proc - __kmp_nth + 808 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc); 809 if (new_nthreads <= 1) { 810 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced " 811 "reservation to 1 thread\n", 812 master_tid)); 813 return 1; 814 } 815 if (new_nthreads < set_nthreads) { 816 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced " 817 "reservation to %d threads\n", 818 master_tid, new_nthreads)); 819 } else { 820 new_nthreads = set_nthreads; 821 } 822 } else if (__kmp_global.g.g_dynamic_mode == dynamic_random) { 823 if (set_nthreads > 2) { 824 new_nthreads = __kmp_get_random(parent_team->t.t_threads[master_tid]); 825 new_nthreads = (new_nthreads % set_nthreads) + 1; 826 if (new_nthreads == 1) { 827 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced " 828 "reservation to 1 thread\n", 829 master_tid)); 830 return 1; 831 } 832 if (new_nthreads < set_nthreads) { 833 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced " 834 "reservation to %d threads\n", 835 master_tid, new_nthreads)); 836 } 837 } 838 } else { 839 KMP_ASSERT(0); 840 } 841 842 // Respect KMP_ALL_THREADS/KMP_DEVICE_THREAD_LIMIT. 843 if (__kmp_nth + new_nthreads - 844 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) > 845 __kmp_max_nth) { 846 int tl_nthreads = __kmp_max_nth - __kmp_nth + 847 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc); 848 if (tl_nthreads <= 0) { 849 tl_nthreads = 1; 850 } 851 852 // If dyn-var is false, emit a 1-time warning. 853 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) { 854 __kmp_reserve_warn = 1; 855 __kmp_msg(kmp_ms_warning, 856 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads), 857 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 858 } 859 if (tl_nthreads == 1) { 860 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT " 861 "reduced reservation to 1 thread\n", 862 master_tid)); 863 return 1; 864 } 865 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced " 866 "reservation to %d threads\n", 867 master_tid, tl_nthreads)); 868 new_nthreads = tl_nthreads; 869 } 870 871 // Respect OMP_THREAD_LIMIT 872 int cg_nthreads = this_thr->th.th_cg_roots->cg_nthreads; 873 int max_cg_threads = this_thr->th.th_cg_roots->cg_thread_limit; 874 if (cg_nthreads + new_nthreads - 875 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) > 876 max_cg_threads) { 877 int tl_nthreads = max_cg_threads - cg_nthreads + 878 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc); 879 if (tl_nthreads <= 0) { 880 tl_nthreads = 1; 881 } 882 883 // If dyn-var is false, emit a 1-time warning. 884 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) { 885 __kmp_reserve_warn = 1; 886 __kmp_msg(kmp_ms_warning, 887 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads), 888 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 889 } 890 if (tl_nthreads == 1) { 891 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT " 892 "reduced reservation to 1 thread\n", 893 master_tid)); 894 return 1; 895 } 896 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced " 897 "reservation to %d threads\n", 898 master_tid, tl_nthreads)); 899 new_nthreads = tl_nthreads; 900 } 901 902 // Check if the threads array is large enough, or needs expanding. 903 // See comment in __kmp_register_root() about the adjustment if 904 // __kmp_threads[0] == NULL. 905 capacity = __kmp_threads_capacity; 906 if (TCR_PTR(__kmp_threads[0]) == NULL) { 907 --capacity; 908 } 909 if (__kmp_nth + new_nthreads - 910 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) > 911 capacity) { 912 // Expand the threads array. 913 int slotsRequired = __kmp_nth + new_nthreads - 914 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) - 915 capacity; 916 int slotsAdded = __kmp_expand_threads(slotsRequired); 917 if (slotsAdded < slotsRequired) { 918 // The threads array was not expanded enough. 919 new_nthreads -= (slotsRequired - slotsAdded); 920 KMP_ASSERT(new_nthreads >= 1); 921 922 // If dyn-var is false, emit a 1-time warning. 923 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) { 924 __kmp_reserve_warn = 1; 925 if (__kmp_tp_cached) { 926 __kmp_msg(kmp_ms_warning, 927 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads), 928 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity), 929 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null); 930 } else { 931 __kmp_msg(kmp_ms_warning, 932 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads), 933 KMP_HNT(SystemLimitOnThreads), __kmp_msg_null); 934 } 935 } 936 } 937 } 938 939 #ifdef KMP_DEBUG 940 if (new_nthreads == 1) { 941 KC_TRACE(10, 942 ("__kmp_reserve_threads: T#%d serializing team after reclaiming " 943 "dead roots and rechecking; requested %d threads\n", 944 __kmp_get_gtid(), set_nthreads)); 945 } else { 946 KC_TRACE(10, ("__kmp_reserve_threads: T#%d allocating %d threads; requested" 947 " %d threads\n", 948 __kmp_get_gtid(), new_nthreads, set_nthreads)); 949 } 950 #endif // KMP_DEBUG 951 return new_nthreads; 952 } 953 954 /* Allocate threads from the thread pool and assign them to the new team. We are 955 assured that there are enough threads available, because we checked on that 956 earlier within critical section forkjoin */ 957 static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team, 958 kmp_info_t *master_th, int master_gtid) { 959 int i; 960 int use_hot_team; 961 962 KA_TRACE(10, ("__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc)); 963 KMP_DEBUG_ASSERT(master_gtid == __kmp_get_gtid()); 964 KMP_MB(); 965 966 /* first, let's setup the master thread */ 967 master_th->th.th_info.ds.ds_tid = 0; 968 master_th->th.th_team = team; 969 master_th->th.th_team_nproc = team->t.t_nproc; 970 master_th->th.th_team_master = master_th; 971 master_th->th.th_team_serialized = FALSE; 972 master_th->th.th_dispatch = &team->t.t_dispatch[0]; 973 974 /* make sure we are not the optimized hot team */ 975 #if KMP_NESTED_HOT_TEAMS 976 use_hot_team = 0; 977 kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams; 978 if (hot_teams) { // hot teams array is not allocated if 979 // KMP_HOT_TEAMS_MAX_LEVEL=0 980 int level = team->t.t_active_level - 1; // index in array of hot teams 981 if (master_th->th.th_teams_microtask) { // are we inside the teams? 982 if (master_th->th.th_teams_size.nteams > 1) { 983 ++level; // level was not increased in teams construct for 984 // team_of_masters 985 } 986 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master && 987 master_th->th.th_teams_level == team->t.t_level) { 988 ++level; // level was not increased in teams construct for 989 // team_of_workers before the parallel 990 } // team->t.t_level will be increased inside parallel 991 } 992 if (level < __kmp_hot_teams_max_level) { 993 if (hot_teams[level].hot_team) { 994 // hot team has already been allocated for given level 995 KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team); 996 use_hot_team = 1; // the team is ready to use 997 } else { 998 use_hot_team = 0; // AC: threads are not allocated yet 999 hot_teams[level].hot_team = team; // remember new hot team 1000 hot_teams[level].hot_team_nth = team->t.t_nproc; 1001 } 1002 } else { 1003 use_hot_team = 0; 1004 } 1005 } 1006 #else 1007 use_hot_team = team == root->r.r_hot_team; 1008 #endif 1009 if (!use_hot_team) { 1010 1011 /* install the master thread */ 1012 team->t.t_threads[0] = master_th; 1013 __kmp_initialize_info(master_th, team, 0, master_gtid); 1014 1015 /* now, install the worker threads */ 1016 for (i = 1; i < team->t.t_nproc; i++) { 1017 1018 /* fork or reallocate a new thread and install it in team */ 1019 kmp_info_t *thr = __kmp_allocate_thread(root, team, i); 1020 team->t.t_threads[i] = thr; 1021 KMP_DEBUG_ASSERT(thr); 1022 KMP_DEBUG_ASSERT(thr->th.th_team == team); 1023 /* align team and thread arrived states */ 1024 KA_TRACE(20, ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived " 1025 "T#%d(%d:%d) join =%llu, plain=%llu\n", 1026 __kmp_gtid_from_tid(0, team), team->t.t_id, 0, 1027 __kmp_gtid_from_tid(i, team), team->t.t_id, i, 1028 team->t.t_bar[bs_forkjoin_barrier].b_arrived, 1029 team->t.t_bar[bs_plain_barrier].b_arrived)); 1030 thr->th.th_teams_microtask = master_th->th.th_teams_microtask; 1031 thr->th.th_teams_level = master_th->th.th_teams_level; 1032 thr->th.th_teams_size = master_th->th.th_teams_size; 1033 { // Initialize threads' barrier data. 1034 int b; 1035 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar; 1036 for (b = 0; b < bs_last_barrier; ++b) { 1037 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 1038 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 1039 #if USE_DEBUGGER 1040 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 1041 #endif 1042 } 1043 } 1044 } 1045 1046 #if KMP_AFFINITY_SUPPORTED 1047 __kmp_partition_places(team); 1048 #endif 1049 } 1050 1051 if (__kmp_display_affinity && team->t.t_display_affinity != 1) { 1052 for (i = 0; i < team->t.t_nproc; i++) { 1053 kmp_info_t *thr = team->t.t_threads[i]; 1054 if (thr->th.th_prev_num_threads != team->t.t_nproc || 1055 thr->th.th_prev_level != team->t.t_level) { 1056 team->t.t_display_affinity = 1; 1057 break; 1058 } 1059 } 1060 } 1061 1062 KMP_MB(); 1063 } 1064 1065 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1066 // Propagate any changes to the floating point control registers out to the team 1067 // We try to avoid unnecessary writes to the relevant cache line in the team 1068 // structure, so we don't make changes unless they are needed. 1069 inline static void propagateFPControl(kmp_team_t *team) { 1070 if (__kmp_inherit_fp_control) { 1071 kmp_int16 x87_fpu_control_word; 1072 kmp_uint32 mxcsr; 1073 1074 // Get master values of FPU control flags (both X87 and vector) 1075 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word); 1076 __kmp_store_mxcsr(&mxcsr); 1077 mxcsr &= KMP_X86_MXCSR_MASK; 1078 1079 // There is no point looking at t_fp_control_saved here. 1080 // If it is TRUE, we still have to update the values if they are different 1081 // from those we now have. If it is FALSE we didn't save anything yet, but 1082 // our objective is the same. We have to ensure that the values in the team 1083 // are the same as those we have. 1084 // So, this code achieves what we need whether or not t_fp_control_saved is 1085 // true. By checking whether the value needs updating we avoid unnecessary 1086 // writes that would put the cache-line into a written state, causing all 1087 // threads in the team to have to read it again. 1088 KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word); 1089 KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr); 1090 // Although we don't use this value, other code in the runtime wants to know 1091 // whether it should restore them. So we must ensure it is correct. 1092 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE); 1093 } else { 1094 // Similarly here. Don't write to this cache-line in the team structure 1095 // unless we have to. 1096 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE); 1097 } 1098 } 1099 1100 // Do the opposite, setting the hardware registers to the updated values from 1101 // the team. 1102 inline static void updateHWFPControl(kmp_team_t *team) { 1103 if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) { 1104 // Only reset the fp control regs if they have been changed in the team. 1105 // the parallel region that we are exiting. 1106 kmp_int16 x87_fpu_control_word; 1107 kmp_uint32 mxcsr; 1108 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word); 1109 __kmp_store_mxcsr(&mxcsr); 1110 mxcsr &= KMP_X86_MXCSR_MASK; 1111 1112 if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) { 1113 __kmp_clear_x87_fpu_status_word(); 1114 __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word); 1115 } 1116 1117 if (team->t.t_mxcsr != mxcsr) { 1118 __kmp_load_mxcsr(&team->t.t_mxcsr); 1119 } 1120 } 1121 } 1122 #else 1123 #define propagateFPControl(x) ((void)0) 1124 #define updateHWFPControl(x) ((void)0) 1125 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 1126 1127 static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team, 1128 int realloc); // forward declaration 1129 1130 /* Run a parallel region that has been serialized, so runs only in a team of the 1131 single master thread. */ 1132 void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) { 1133 kmp_info_t *this_thr; 1134 kmp_team_t *serial_team; 1135 1136 KC_TRACE(10, ("__kmpc_serialized_parallel: called by T#%d\n", global_tid)); 1137 1138 /* Skip all this code for autopar serialized loops since it results in 1139 unacceptable overhead */ 1140 if (loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR)) 1141 return; 1142 1143 if (!TCR_4(__kmp_init_parallel)) 1144 __kmp_parallel_initialize(); 1145 __kmp_resume_if_soft_paused(); 1146 1147 this_thr = __kmp_threads[global_tid]; 1148 serial_team = this_thr->th.th_serial_team; 1149 1150 /* utilize the serialized team held by this thread */ 1151 KMP_DEBUG_ASSERT(serial_team); 1152 KMP_MB(); 1153 1154 if (__kmp_tasking_mode != tskm_immediate_exec) { 1155 KMP_DEBUG_ASSERT( 1156 this_thr->th.th_task_team == 1157 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]); 1158 KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] == 1159 NULL); 1160 KA_TRACE(20, ("__kmpc_serialized_parallel: T#%d pushing task_team %p / " 1161 "team %p, new task_team = NULL\n", 1162 global_tid, this_thr->th.th_task_team, this_thr->th.th_team)); 1163 this_thr->th.th_task_team = NULL; 1164 } 1165 1166 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind; 1167 if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) { 1168 proc_bind = proc_bind_false; 1169 } else if (proc_bind == proc_bind_default) { 1170 // No proc_bind clause was specified, so use the current value 1171 // of proc-bind-var for this parallel region. 1172 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind; 1173 } 1174 // Reset for next parallel region 1175 this_thr->th.th_set_proc_bind = proc_bind_default; 1176 1177 #if OMPT_SUPPORT 1178 ompt_data_t ompt_parallel_data = ompt_data_none; 1179 ompt_data_t *implicit_task_data; 1180 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid); 1181 if (ompt_enabled.enabled && 1182 this_thr->th.ompt_thread_info.state != ompt_state_overhead) { 1183 1184 ompt_task_info_t *parent_task_info; 1185 parent_task_info = OMPT_CUR_TASK_INFO(this_thr); 1186 1187 parent_task_info->frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1188 if (ompt_enabled.ompt_callback_parallel_begin) { 1189 int team_size = 1; 1190 1191 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)( 1192 &(parent_task_info->task_data), &(parent_task_info->frame), 1193 &ompt_parallel_data, team_size, 1194 ompt_parallel_invoker_program | ompt_parallel_team, codeptr); 1195 } 1196 } 1197 #endif // OMPT_SUPPORT 1198 1199 if (this_thr->th.th_team != serial_team) { 1200 // Nested level will be an index in the nested nthreads array 1201 int level = this_thr->th.th_team->t.t_level; 1202 1203 if (serial_team->t.t_serialized) { 1204 /* this serial team was already used 1205 TODO increase performance by making this locks more specific */ 1206 kmp_team_t *new_team; 1207 1208 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 1209 1210 new_team = 1211 __kmp_allocate_team(this_thr->th.th_root, 1, 1, 1212 #if OMPT_SUPPORT 1213 ompt_parallel_data, 1214 #endif 1215 proc_bind, &this_thr->th.th_current_task->td_icvs, 1216 0 USE_NESTED_HOT_ARG(NULL)); 1217 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 1218 KMP_ASSERT(new_team); 1219 1220 /* setup new serialized team and install it */ 1221 new_team->t.t_threads[0] = this_thr; 1222 new_team->t.t_parent = this_thr->th.th_team; 1223 serial_team = new_team; 1224 this_thr->th.th_serial_team = serial_team; 1225 1226 KF_TRACE( 1227 10, 1228 ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n", 1229 global_tid, serial_team)); 1230 1231 /* TODO the above breaks the requirement that if we run out of resources, 1232 then we can still guarantee that serialized teams are ok, since we may 1233 need to allocate a new one */ 1234 } else { 1235 KF_TRACE( 1236 10, 1237 ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n", 1238 global_tid, serial_team)); 1239 } 1240 1241 /* we have to initialize this serial team */ 1242 KMP_DEBUG_ASSERT(serial_team->t.t_threads); 1243 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr); 1244 KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team); 1245 serial_team->t.t_ident = loc; 1246 serial_team->t.t_serialized = 1; 1247 serial_team->t.t_nproc = 1; 1248 serial_team->t.t_parent = this_thr->th.th_team; 1249 serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched; 1250 this_thr->th.th_team = serial_team; 1251 serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid; 1252 1253 KF_TRACE(10, ("__kmpc_serialized_parallel: T#d curtask=%p\n", global_tid, 1254 this_thr->th.th_current_task)); 1255 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1); 1256 this_thr->th.th_current_task->td_flags.executing = 0; 1257 1258 __kmp_push_current_task_to_thread(this_thr, serial_team, 0); 1259 1260 /* TODO: GEH: do ICVs work for nested serialized teams? Don't we need an 1261 implicit task for each serialized task represented by 1262 team->t.t_serialized? */ 1263 copy_icvs(&this_thr->th.th_current_task->td_icvs, 1264 &this_thr->th.th_current_task->td_parent->td_icvs); 1265 1266 // Thread value exists in the nested nthreads array for the next nested 1267 // level 1268 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) { 1269 this_thr->th.th_current_task->td_icvs.nproc = 1270 __kmp_nested_nth.nth[level + 1]; 1271 } 1272 1273 if (__kmp_nested_proc_bind.used && 1274 (level + 1 < __kmp_nested_proc_bind.used)) { 1275 this_thr->th.th_current_task->td_icvs.proc_bind = 1276 __kmp_nested_proc_bind.bind_types[level + 1]; 1277 } 1278 1279 #if USE_DEBUGGER 1280 serial_team->t.t_pkfn = (microtask_t)(~0); // For the debugger. 1281 #endif 1282 this_thr->th.th_info.ds.ds_tid = 0; 1283 1284 /* set thread cache values */ 1285 this_thr->th.th_team_nproc = 1; 1286 this_thr->th.th_team_master = this_thr; 1287 this_thr->th.th_team_serialized = 1; 1288 1289 serial_team->t.t_level = serial_team->t.t_parent->t.t_level + 1; 1290 serial_team->t.t_active_level = serial_team->t.t_parent->t.t_active_level; 1291 serial_team->t.t_def_allocator = this_thr->th.th_def_allocator; // save 1292 1293 propagateFPControl(serial_team); 1294 1295 /* check if we need to allocate dispatch buffers stack */ 1296 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch); 1297 if (!serial_team->t.t_dispatch->th_disp_buffer) { 1298 serial_team->t.t_dispatch->th_disp_buffer = 1299 (dispatch_private_info_t *)__kmp_allocate( 1300 sizeof(dispatch_private_info_t)); 1301 } 1302 this_thr->th.th_dispatch = serial_team->t.t_dispatch; 1303 1304 KMP_MB(); 1305 1306 } else { 1307 /* this serialized team is already being used, 1308 * that's fine, just add another nested level */ 1309 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team); 1310 KMP_DEBUG_ASSERT(serial_team->t.t_threads); 1311 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr); 1312 ++serial_team->t.t_serialized; 1313 this_thr->th.th_team_serialized = serial_team->t.t_serialized; 1314 1315 // Nested level will be an index in the nested nthreads array 1316 int level = this_thr->th.th_team->t.t_level; 1317 // Thread value exists in the nested nthreads array for the next nested 1318 // level 1319 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) { 1320 this_thr->th.th_current_task->td_icvs.nproc = 1321 __kmp_nested_nth.nth[level + 1]; 1322 } 1323 serial_team->t.t_level++; 1324 KF_TRACE(10, ("__kmpc_serialized_parallel: T#%d increasing nesting level " 1325 "of serial team %p to %d\n", 1326 global_tid, serial_team, serial_team->t.t_level)); 1327 1328 /* allocate/push dispatch buffers stack */ 1329 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch); 1330 { 1331 dispatch_private_info_t *disp_buffer = 1332 (dispatch_private_info_t *)__kmp_allocate( 1333 sizeof(dispatch_private_info_t)); 1334 disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer; 1335 serial_team->t.t_dispatch->th_disp_buffer = disp_buffer; 1336 } 1337 this_thr->th.th_dispatch = serial_team->t.t_dispatch; 1338 1339 KMP_MB(); 1340 } 1341 KMP_CHECK_UPDATE(serial_team->t.t_cancel_request, cancel_noreq); 1342 1343 // Perform the display affinity functionality for 1344 // serialized parallel regions 1345 if (__kmp_display_affinity) { 1346 if (this_thr->th.th_prev_level != serial_team->t.t_level || 1347 this_thr->th.th_prev_num_threads != 1) { 1348 // NULL means use the affinity-format-var ICV 1349 __kmp_aux_display_affinity(global_tid, NULL); 1350 this_thr->th.th_prev_level = serial_team->t.t_level; 1351 this_thr->th.th_prev_num_threads = 1; 1352 } 1353 } 1354 1355 if (__kmp_env_consistency_check) 1356 __kmp_push_parallel(global_tid, NULL); 1357 #if OMPT_SUPPORT 1358 serial_team->t.ompt_team_info.master_return_address = codeptr; 1359 if (ompt_enabled.enabled && 1360 this_thr->th.ompt_thread_info.state != ompt_state_overhead) { 1361 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1362 1363 ompt_lw_taskteam_t lw_taskteam; 1364 __ompt_lw_taskteam_init(&lw_taskteam, this_thr, global_tid, 1365 &ompt_parallel_data, codeptr); 1366 1367 __ompt_lw_taskteam_link(&lw_taskteam, this_thr, 1); 1368 // don't use lw_taskteam after linking. content was swaped 1369 1370 /* OMPT implicit task begin */ 1371 implicit_task_data = OMPT_CUR_TASK_DATA(this_thr); 1372 if (ompt_enabled.ompt_callback_implicit_task) { 1373 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1374 ompt_scope_begin, OMPT_CUR_TEAM_DATA(this_thr), 1375 OMPT_CUR_TASK_DATA(this_thr), 1, __kmp_tid_from_gtid(global_tid), ompt_task_implicit); // TODO: Can this be ompt_task_initial? 1376 OMPT_CUR_TASK_INFO(this_thr) 1377 ->thread_num = __kmp_tid_from_gtid(global_tid); 1378 } 1379 1380 /* OMPT state */ 1381 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel; 1382 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1383 } 1384 #endif 1385 } 1386 1387 /* most of the work for a fork */ 1388 /* return true if we really went parallel, false if serialized */ 1389 int __kmp_fork_call(ident_t *loc, int gtid, 1390 enum fork_context_e call_context, // Intel, GNU, ... 1391 kmp_int32 argc, microtask_t microtask, launch_t invoker, 1392 /* TODO: revert workaround for Intel(R) 64 tracker #96 */ 1393 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 1394 va_list *ap 1395 #else 1396 va_list ap 1397 #endif 1398 ) { 1399 void **argv; 1400 int i; 1401 int master_tid; 1402 int master_this_cons; 1403 kmp_team_t *team; 1404 kmp_team_t *parent_team; 1405 kmp_info_t *master_th; 1406 kmp_root_t *root; 1407 int nthreads; 1408 int master_active; 1409 int master_set_numthreads; 1410 int level; 1411 int active_level; 1412 int teams_level; 1413 #if KMP_NESTED_HOT_TEAMS 1414 kmp_hot_team_ptr_t **p_hot_teams; 1415 #endif 1416 { // KMP_TIME_BLOCK 1417 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_fork_call); 1418 KMP_COUNT_VALUE(OMP_PARALLEL_args, argc); 1419 1420 KA_TRACE(20, ("__kmp_fork_call: enter T#%d\n", gtid)); 1421 if (__kmp_stkpadding > 0 && __kmp_root[gtid] != NULL) { 1422 /* Some systems prefer the stack for the root thread(s) to start with */ 1423 /* some gap from the parent stack to prevent false sharing. */ 1424 void *dummy = KMP_ALLOCA(__kmp_stkpadding); 1425 /* These 2 lines below are so this does not get optimized out */ 1426 if (__kmp_stkpadding > KMP_MAX_STKPADDING) 1427 __kmp_stkpadding += (short)((kmp_int64)dummy); 1428 } 1429 1430 /* initialize if needed */ 1431 KMP_DEBUG_ASSERT( 1432 __kmp_init_serial); // AC: potentially unsafe, not in sync with shutdown 1433 if (!TCR_4(__kmp_init_parallel)) 1434 __kmp_parallel_initialize(); 1435 __kmp_resume_if_soft_paused(); 1436 1437 /* setup current data */ 1438 master_th = __kmp_threads[gtid]; // AC: potentially unsafe, not in sync with 1439 // shutdown 1440 parent_team = master_th->th.th_team; 1441 master_tid = master_th->th.th_info.ds.ds_tid; 1442 master_this_cons = master_th->th.th_local.this_construct; 1443 root = master_th->th.th_root; 1444 master_active = root->r.r_active; 1445 master_set_numthreads = master_th->th.th_set_nproc; 1446 1447 #if OMPT_SUPPORT 1448 ompt_data_t ompt_parallel_data = ompt_data_none; 1449 ompt_data_t *parent_task_data; 1450 ompt_frame_t *ompt_frame; 1451 ompt_data_t *implicit_task_data; 1452 void *return_address = NULL; 1453 1454 if (ompt_enabled.enabled) { 1455 __ompt_get_task_info_internal(0, NULL, &parent_task_data, &ompt_frame, 1456 NULL, NULL); 1457 return_address = OMPT_LOAD_RETURN_ADDRESS(gtid); 1458 } 1459 #endif 1460 1461 // Nested level will be an index in the nested nthreads array 1462 level = parent_team->t.t_level; 1463 // used to launch non-serial teams even if nested is not allowed 1464 active_level = parent_team->t.t_active_level; 1465 // needed to check nesting inside the teams 1466 teams_level = master_th->th.th_teams_level; 1467 #if KMP_NESTED_HOT_TEAMS 1468 p_hot_teams = &master_th->th.th_hot_teams; 1469 if (*p_hot_teams == NULL && __kmp_hot_teams_max_level > 0) { 1470 *p_hot_teams = (kmp_hot_team_ptr_t *)__kmp_allocate( 1471 sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level); 1472 (*p_hot_teams)[0].hot_team = root->r.r_hot_team; 1473 // it is either actual or not needed (when active_level > 0) 1474 (*p_hot_teams)[0].hot_team_nth = 1; 1475 } 1476 #endif 1477 1478 #if OMPT_SUPPORT 1479 if (ompt_enabled.enabled) { 1480 if (ompt_enabled.ompt_callback_parallel_begin) { 1481 int team_size = master_set_numthreads 1482 ? master_set_numthreads 1483 : get__nproc_2(parent_team, master_tid); 1484 int flags = OMPT_INVOKER(call_context) | 1485 ((microtask == (microtask_t)__kmp_teams_master) 1486 ? ompt_parallel_league 1487 : ompt_parallel_team); 1488 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)( 1489 parent_task_data, ompt_frame, &ompt_parallel_data, team_size, flags, 1490 return_address); 1491 } 1492 master_th->th.ompt_thread_info.state = ompt_state_overhead; 1493 } 1494 #endif 1495 1496 master_th->th.th_ident = loc; 1497 1498 if (master_th->th.th_teams_microtask && ap && 1499 microtask != (microtask_t)__kmp_teams_master && level == teams_level) { 1500 // AC: This is start of parallel that is nested inside teams construct. 1501 // The team is actual (hot), all workers are ready at the fork barrier. 1502 // No lock needed to initialize the team a bit, then free workers. 1503 parent_team->t.t_ident = loc; 1504 __kmp_alloc_argv_entries(argc, parent_team, TRUE); 1505 parent_team->t.t_argc = argc; 1506 argv = (void **)parent_team->t.t_argv; 1507 for (i = argc - 1; i >= 0; --i) 1508 /* TODO: revert workaround for Intel(R) 64 tracker #96 */ 1509 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 1510 *argv++ = va_arg(*ap, void *); 1511 #else 1512 *argv++ = va_arg(ap, void *); 1513 #endif 1514 // Increment our nested depth levels, but not increase the serialization 1515 if (parent_team == master_th->th.th_serial_team) { 1516 // AC: we are in serialized parallel 1517 __kmpc_serialized_parallel(loc, gtid); 1518 KMP_DEBUG_ASSERT(parent_team->t.t_serialized > 1); 1519 1520 #if OMPT_SUPPORT 1521 void *dummy; 1522 void **exit_frame_p; 1523 1524 ompt_lw_taskteam_t lw_taskteam; 1525 1526 if (ompt_enabled.enabled) { 1527 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid, 1528 &ompt_parallel_data, return_address); 1529 exit_frame_p = &(lw_taskteam.ompt_task_info.frame.exit_frame.ptr); 1530 1531 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0); 1532 // don't use lw_taskteam after linking. content was swaped 1533 1534 /* OMPT implicit task begin */ 1535 implicit_task_data = OMPT_CUR_TASK_DATA(master_th); 1536 if (ompt_enabled.ompt_callback_implicit_task) { 1537 OMPT_CUR_TASK_INFO(master_th) 1538 ->thread_num = __kmp_tid_from_gtid(gtid); 1539 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1540 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th), 1541 implicit_task_data, 1, 1542 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit); 1543 } 1544 1545 /* OMPT state */ 1546 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; 1547 } else { 1548 exit_frame_p = &dummy; 1549 } 1550 #endif 1551 // AC: need to decrement t_serialized for enquiry functions to work 1552 // correctly, will restore at join time 1553 parent_team->t.t_serialized--; 1554 1555 { 1556 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 1557 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 1558 __kmp_invoke_microtask(microtask, gtid, 0, argc, parent_team->t.t_argv 1559 #if OMPT_SUPPORT 1560 , 1561 exit_frame_p 1562 #endif 1563 ); 1564 } 1565 1566 #if OMPT_SUPPORT 1567 if (ompt_enabled.enabled) { 1568 *exit_frame_p = NULL; 1569 OMPT_CUR_TASK_INFO(master_th)->frame.exit_frame = ompt_data_none; 1570 if (ompt_enabled.ompt_callback_implicit_task) { 1571 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1572 ompt_scope_end, NULL, implicit_task_data, 1, 1573 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit); 1574 } 1575 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th); 1576 __ompt_lw_taskteam_unlink(master_th); 1577 if (ompt_enabled.ompt_callback_parallel_end) { 1578 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)( 1579 &ompt_parallel_data, OMPT_CUR_TASK_DATA(master_th), 1580 OMPT_INVOKER(call_context) | ompt_parallel_team, 1581 return_address); 1582 } 1583 master_th->th.ompt_thread_info.state = ompt_state_overhead; 1584 } 1585 #endif 1586 return TRUE; 1587 } 1588 1589 parent_team->t.t_pkfn = microtask; 1590 parent_team->t.t_invoke = invoker; 1591 KMP_ATOMIC_INC(&root->r.r_in_parallel); 1592 parent_team->t.t_active_level++; 1593 parent_team->t.t_level++; 1594 parent_team->t.t_def_allocator = master_th->th.th_def_allocator; // save 1595 1596 #if OMPT_SUPPORT 1597 if (ompt_enabled.enabled) { 1598 ompt_lw_taskteam_t lw_taskteam; 1599 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid, 1600 &ompt_parallel_data, return_address); 1601 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 1, true); 1602 } 1603 #endif 1604 1605 /* Change number of threads in the team if requested */ 1606 if (master_set_numthreads) { // The parallel has num_threads clause 1607 if (master_set_numthreads < master_th->th.th_teams_size.nth) { 1608 // AC: only can reduce number of threads dynamically, can't increase 1609 kmp_info_t **other_threads = parent_team->t.t_threads; 1610 parent_team->t.t_nproc = master_set_numthreads; 1611 for (i = 0; i < master_set_numthreads; ++i) { 1612 other_threads[i]->th.th_team_nproc = master_set_numthreads; 1613 } 1614 // Keep extra threads hot in the team for possible next parallels 1615 } 1616 master_th->th.th_set_nproc = 0; 1617 } 1618 1619 #if USE_DEBUGGER 1620 if (__kmp_debugging) { // Let debugger override number of threads. 1621 int nth = __kmp_omp_num_threads(loc); 1622 if (nth > 0) { // 0 means debugger doesn't want to change num threads 1623 master_set_numthreads = nth; 1624 } 1625 } 1626 #endif 1627 1628 #if USE_ITT_BUILD 1629 if (((__itt_frame_submit_v3_ptr && __itt_get_timestamp_ptr) || 1630 KMP_ITT_DEBUG) && 1631 __kmp_forkjoin_frames_mode == 3 && 1632 parent_team->t.t_active_level == 1 // only report frames at level 1 1633 && master_th->th.th_teams_size.nteams == 1) { 1634 kmp_uint64 tmp_time = __itt_get_timestamp(); 1635 master_th->th.th_frame_time = tmp_time; 1636 parent_team->t.t_region_time = tmp_time; 1637 } 1638 if (__itt_stack_caller_create_ptr) { 1639 // create new stack stitching id before entering fork barrier 1640 parent_team->t.t_stack_id = __kmp_itt_stack_caller_create(); 1641 } 1642 #endif /* USE_ITT_BUILD */ 1643 1644 KF_TRACE(10, ("__kmp_fork_call: before internal fork: root=%p, team=%p, " 1645 "master_th=%p, gtid=%d\n", 1646 root, parent_team, master_th, gtid)); 1647 __kmp_internal_fork(loc, gtid, parent_team); 1648 KF_TRACE(10, ("__kmp_fork_call: after internal fork: root=%p, team=%p, " 1649 "master_th=%p, gtid=%d\n", 1650 root, parent_team, master_th, gtid)); 1651 1652 /* Invoke microtask for MASTER thread */ 1653 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid, 1654 parent_team->t.t_id, parent_team->t.t_pkfn)); 1655 1656 if (!parent_team->t.t_invoke(gtid)) { 1657 KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread"); 1658 } 1659 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid, 1660 parent_team->t.t_id, parent_team->t.t_pkfn)); 1661 KMP_MB(); /* Flush all pending memory write invalidates. */ 1662 1663 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid)); 1664 1665 return TRUE; 1666 } // Parallel closely nested in teams construct 1667 1668 #if KMP_DEBUG 1669 if (__kmp_tasking_mode != tskm_immediate_exec) { 1670 KMP_DEBUG_ASSERT(master_th->th.th_task_team == 1671 parent_team->t.t_task_team[master_th->th.th_task_state]); 1672 } 1673 #endif 1674 1675 if (parent_team->t.t_active_level >= 1676 master_th->th.th_current_task->td_icvs.max_active_levels) { 1677 nthreads = 1; 1678 } else { 1679 int enter_teams = ((ap == NULL && active_level == 0) || 1680 (ap && teams_level > 0 && teams_level == level)); 1681 nthreads = 1682 master_set_numthreads 1683 ? master_set_numthreads 1684 : get__nproc_2( 1685 parent_team, 1686 master_tid); // TODO: get nproc directly from current task 1687 1688 // Check if we need to take forkjoin lock? (no need for serialized 1689 // parallel out of teams construct). This code moved here from 1690 // __kmp_reserve_threads() to speedup nested serialized parallels. 1691 if (nthreads > 1) { 1692 if ((get__max_active_levels(master_th) == 1 && 1693 (root->r.r_in_parallel && !enter_teams)) || 1694 (__kmp_library == library_serial)) { 1695 KC_TRACE(10, ("__kmp_fork_call: T#%d serializing team; requested %d" 1696 " threads\n", 1697 gtid, nthreads)); 1698 nthreads = 1; 1699 } 1700 } 1701 if (nthreads > 1) { 1702 /* determine how many new threads we can use */ 1703 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 1704 /* AC: If we execute teams from parallel region (on host), then teams 1705 should be created but each can only have 1 thread if nesting is 1706 disabled. If teams called from serial region, then teams and their 1707 threads should be created regardless of the nesting setting. */ 1708 nthreads = __kmp_reserve_threads(root, parent_team, master_tid, 1709 nthreads, enter_teams); 1710 if (nthreads == 1) { 1711 // Free lock for single thread execution here; for multi-thread 1712 // execution it will be freed later after team of threads created 1713 // and initialized 1714 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 1715 } 1716 } 1717 } 1718 KMP_DEBUG_ASSERT(nthreads > 0); 1719 1720 // If we temporarily changed the set number of threads then restore it now 1721 master_th->th.th_set_nproc = 0; 1722 1723 /* create a serialized parallel region? */ 1724 if (nthreads == 1) { 1725 /* josh todo: hypothetical question: what do we do for OS X*? */ 1726 #if KMP_OS_LINUX && \ 1727 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 1728 void *args[argc]; 1729 #else 1730 void **args = (void **)KMP_ALLOCA(argc * sizeof(void *)); 1731 #endif /* KMP_OS_LINUX && ( KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || \ 1732 KMP_ARCH_AARCH64) */ 1733 1734 KA_TRACE(20, 1735 ("__kmp_fork_call: T#%d serializing parallel region\n", gtid)); 1736 1737 __kmpc_serialized_parallel(loc, gtid); 1738 1739 if (call_context == fork_context_intel) { 1740 /* TODO this sucks, use the compiler itself to pass args! :) */ 1741 master_th->th.th_serial_team->t.t_ident = loc; 1742 if (!ap) { 1743 // revert change made in __kmpc_serialized_parallel() 1744 master_th->th.th_serial_team->t.t_level--; 1745 // Get args from parent team for teams construct 1746 1747 #if OMPT_SUPPORT 1748 void *dummy; 1749 void **exit_frame_p; 1750 ompt_task_info_t *task_info; 1751 1752 ompt_lw_taskteam_t lw_taskteam; 1753 1754 if (ompt_enabled.enabled) { 1755 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid, 1756 &ompt_parallel_data, return_address); 1757 1758 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0); 1759 // don't use lw_taskteam after linking. content was swaped 1760 1761 task_info = OMPT_CUR_TASK_INFO(master_th); 1762 exit_frame_p = &(task_info->frame.exit_frame.ptr); 1763 if (ompt_enabled.ompt_callback_implicit_task) { 1764 OMPT_CUR_TASK_INFO(master_th) 1765 ->thread_num = __kmp_tid_from_gtid(gtid); 1766 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1767 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th), 1768 &(task_info->task_data), 1, 1769 OMPT_CUR_TASK_INFO(master_th)->thread_num, 1770 ompt_task_implicit); 1771 } 1772 1773 /* OMPT state */ 1774 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; 1775 } else { 1776 exit_frame_p = &dummy; 1777 } 1778 #endif 1779 1780 { 1781 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 1782 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 1783 __kmp_invoke_microtask(microtask, gtid, 0, argc, 1784 parent_team->t.t_argv 1785 #if OMPT_SUPPORT 1786 , 1787 exit_frame_p 1788 #endif 1789 ); 1790 } 1791 1792 #if OMPT_SUPPORT 1793 if (ompt_enabled.enabled) { 1794 *exit_frame_p = NULL; 1795 if (ompt_enabled.ompt_callback_implicit_task) { 1796 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1797 ompt_scope_end, NULL, &(task_info->task_data), 1, 1798 OMPT_CUR_TASK_INFO(master_th)->thread_num, 1799 ompt_task_implicit); 1800 } 1801 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th); 1802 __ompt_lw_taskteam_unlink(master_th); 1803 if (ompt_enabled.ompt_callback_parallel_end) { 1804 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)( 1805 &ompt_parallel_data, parent_task_data, 1806 OMPT_INVOKER(call_context) | ompt_parallel_team, 1807 return_address); 1808 } 1809 master_th->th.ompt_thread_info.state = ompt_state_overhead; 1810 } 1811 #endif 1812 } else if (microtask == (microtask_t)__kmp_teams_master) { 1813 KMP_DEBUG_ASSERT(master_th->th.th_team == 1814 master_th->th.th_serial_team); 1815 team = master_th->th.th_team; 1816 // team->t.t_pkfn = microtask; 1817 team->t.t_invoke = invoker; 1818 __kmp_alloc_argv_entries(argc, team, TRUE); 1819 team->t.t_argc = argc; 1820 argv = (void **)team->t.t_argv; 1821 if (ap) { 1822 for (i = argc - 1; i >= 0; --i) 1823 // TODO: revert workaround for Intel(R) 64 tracker #96 1824 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 1825 *argv++ = va_arg(*ap, void *); 1826 #else 1827 *argv++ = va_arg(ap, void *); 1828 #endif 1829 } else { 1830 for (i = 0; i < argc; ++i) 1831 // Get args from parent team for teams construct 1832 argv[i] = parent_team->t.t_argv[i]; 1833 } 1834 // AC: revert change made in __kmpc_serialized_parallel() 1835 // because initial code in teams should have level=0 1836 team->t.t_level--; 1837 // AC: call special invoker for outer "parallel" of teams construct 1838 invoker(gtid); 1839 #if OMPT_SUPPORT 1840 if (ompt_enabled.enabled) { 1841 ompt_task_info_t *task_info = OMPT_CUR_TASK_INFO(master_th); 1842 if (ompt_enabled.ompt_callback_implicit_task) { 1843 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1844 ompt_scope_end, NULL, &(task_info->task_data), 0, 1845 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_initial); 1846 } 1847 if (ompt_enabled.ompt_callback_parallel_end) { 1848 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)( 1849 &ompt_parallel_data, parent_task_data, 1850 OMPT_INVOKER(call_context) | ompt_parallel_league, 1851 return_address); 1852 } 1853 master_th->th.ompt_thread_info.state = ompt_state_overhead; 1854 } 1855 #endif 1856 } else { 1857 argv = args; 1858 for (i = argc - 1; i >= 0; --i) 1859 // TODO: revert workaround for Intel(R) 64 tracker #96 1860 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 1861 *argv++ = va_arg(*ap, void *); 1862 #else 1863 *argv++ = va_arg(ap, void *); 1864 #endif 1865 KMP_MB(); 1866 1867 #if OMPT_SUPPORT 1868 void *dummy; 1869 void **exit_frame_p; 1870 ompt_task_info_t *task_info; 1871 1872 ompt_lw_taskteam_t lw_taskteam; 1873 1874 if (ompt_enabled.enabled) { 1875 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid, 1876 &ompt_parallel_data, return_address); 1877 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0); 1878 // don't use lw_taskteam after linking. content was swaped 1879 task_info = OMPT_CUR_TASK_INFO(master_th); 1880 exit_frame_p = &(task_info->frame.exit_frame.ptr); 1881 1882 /* OMPT implicit task begin */ 1883 implicit_task_data = OMPT_CUR_TASK_DATA(master_th); 1884 if (ompt_enabled.ompt_callback_implicit_task) { 1885 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1886 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th), 1887 implicit_task_data, 1, __kmp_tid_from_gtid(gtid), 1888 ompt_task_implicit); 1889 OMPT_CUR_TASK_INFO(master_th) 1890 ->thread_num = __kmp_tid_from_gtid(gtid); 1891 } 1892 1893 /* OMPT state */ 1894 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; 1895 } else { 1896 exit_frame_p = &dummy; 1897 } 1898 #endif 1899 1900 { 1901 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 1902 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 1903 __kmp_invoke_microtask(microtask, gtid, 0, argc, args 1904 #if OMPT_SUPPORT 1905 , 1906 exit_frame_p 1907 #endif 1908 ); 1909 } 1910 1911 #if OMPT_SUPPORT 1912 if (ompt_enabled.enabled) { 1913 *exit_frame_p = NULL; 1914 if (ompt_enabled.ompt_callback_implicit_task) { 1915 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1916 ompt_scope_end, NULL, &(task_info->task_data), 1, 1917 OMPT_CUR_TASK_INFO(master_th)->thread_num, 1918 ompt_task_implicit); 1919 } 1920 1921 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th); 1922 __ompt_lw_taskteam_unlink(master_th); 1923 if (ompt_enabled.ompt_callback_parallel_end) { 1924 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)( 1925 &ompt_parallel_data, parent_task_data, 1926 OMPT_INVOKER(call_context) | ompt_parallel_team, 1927 return_address); 1928 } 1929 master_th->th.ompt_thread_info.state = ompt_state_overhead; 1930 } 1931 #endif 1932 } 1933 } else if (call_context == fork_context_gnu) { 1934 #if OMPT_SUPPORT 1935 ompt_lw_taskteam_t lwt; 1936 __ompt_lw_taskteam_init(&lwt, master_th, gtid, &ompt_parallel_data, 1937 return_address); 1938 1939 lwt.ompt_task_info.frame.exit_frame = ompt_data_none; 1940 __ompt_lw_taskteam_link(&lwt, master_th, 1); 1941 // don't use lw_taskteam after linking. content was swaped 1942 #endif 1943 1944 // we were called from GNU native code 1945 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid)); 1946 return FALSE; 1947 } else { 1948 KMP_ASSERT2(call_context < fork_context_last, 1949 "__kmp_fork_call: unknown fork_context parameter"); 1950 } 1951 1952 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid)); 1953 KMP_MB(); 1954 return FALSE; 1955 } // if (nthreads == 1) 1956 1957 // GEH: only modify the executing flag in the case when not serialized 1958 // serialized case is handled in kmpc_serialized_parallel 1959 KF_TRACE(10, ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, " 1960 "curtask=%p, curtask_max_aclevel=%d\n", 1961 parent_team->t.t_active_level, master_th, 1962 master_th->th.th_current_task, 1963 master_th->th.th_current_task->td_icvs.max_active_levels)); 1964 // TODO: GEH - cannot do this assertion because root thread not set up as 1965 // executing 1966 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 1 ); 1967 master_th->th.th_current_task->td_flags.executing = 0; 1968 1969 if (!master_th->th.th_teams_microtask || level > teams_level) { 1970 /* Increment our nested depth level */ 1971 KMP_ATOMIC_INC(&root->r.r_in_parallel); 1972 } 1973 1974 // See if we need to make a copy of the ICVs. 1975 int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc; 1976 if ((level + 1 < __kmp_nested_nth.used) && 1977 (__kmp_nested_nth.nth[level + 1] != nthreads_icv)) { 1978 nthreads_icv = __kmp_nested_nth.nth[level + 1]; 1979 } else { 1980 nthreads_icv = 0; // don't update 1981 } 1982 1983 // Figure out the proc_bind_policy for the new team. 1984 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind; 1985 kmp_proc_bind_t proc_bind_icv = 1986 proc_bind_default; // proc_bind_default means don't update 1987 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) { 1988 proc_bind = proc_bind_false; 1989 } else { 1990 if (proc_bind == proc_bind_default) { 1991 // No proc_bind clause specified; use current proc-bind-var for this 1992 // parallel region 1993 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind; 1994 } 1995 /* else: The proc_bind policy was specified explicitly on parallel clause. 1996 This overrides proc-bind-var for this parallel region, but does not 1997 change proc-bind-var. */ 1998 // Figure the value of proc-bind-var for the child threads. 1999 if ((level + 1 < __kmp_nested_proc_bind.used) && 2000 (__kmp_nested_proc_bind.bind_types[level + 1] != 2001 master_th->th.th_current_task->td_icvs.proc_bind)) { 2002 proc_bind_icv = __kmp_nested_proc_bind.bind_types[level + 1]; 2003 } 2004 } 2005 2006 // Reset for next parallel region 2007 master_th->th.th_set_proc_bind = proc_bind_default; 2008 2009 if ((nthreads_icv > 0) || (proc_bind_icv != proc_bind_default)) { 2010 kmp_internal_control_t new_icvs; 2011 copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs); 2012 new_icvs.next = NULL; 2013 if (nthreads_icv > 0) { 2014 new_icvs.nproc = nthreads_icv; 2015 } 2016 if (proc_bind_icv != proc_bind_default) { 2017 new_icvs.proc_bind = proc_bind_icv; 2018 } 2019 2020 /* allocate a new parallel team */ 2021 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n")); 2022 team = __kmp_allocate_team(root, nthreads, nthreads, 2023 #if OMPT_SUPPORT 2024 ompt_parallel_data, 2025 #endif 2026 proc_bind, &new_icvs, 2027 argc USE_NESTED_HOT_ARG(master_th)); 2028 } else { 2029 /* allocate a new parallel team */ 2030 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n")); 2031 team = __kmp_allocate_team(root, nthreads, nthreads, 2032 #if OMPT_SUPPORT 2033 ompt_parallel_data, 2034 #endif 2035 proc_bind, 2036 &master_th->th.th_current_task->td_icvs, 2037 argc USE_NESTED_HOT_ARG(master_th)); 2038 } 2039 KF_TRACE( 2040 10, ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team)); 2041 2042 /* setup the new team */ 2043 KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid); 2044 KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons); 2045 KMP_CHECK_UPDATE(team->t.t_ident, loc); 2046 KMP_CHECK_UPDATE(team->t.t_parent, parent_team); 2047 KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask); 2048 #if OMPT_SUPPORT 2049 KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.master_return_address, 2050 return_address); 2051 #endif 2052 KMP_CHECK_UPDATE(team->t.t_invoke, invoker); // TODO move to root, maybe 2053 // TODO: parent_team->t.t_level == INT_MAX ??? 2054 if (!master_th->th.th_teams_microtask || level > teams_level) { 2055 int new_level = parent_team->t.t_level + 1; 2056 KMP_CHECK_UPDATE(team->t.t_level, new_level); 2057 new_level = parent_team->t.t_active_level + 1; 2058 KMP_CHECK_UPDATE(team->t.t_active_level, new_level); 2059 } else { 2060 // AC: Do not increase parallel level at start of the teams construct 2061 int new_level = parent_team->t.t_level; 2062 KMP_CHECK_UPDATE(team->t.t_level, new_level); 2063 new_level = parent_team->t.t_active_level; 2064 KMP_CHECK_UPDATE(team->t.t_active_level, new_level); 2065 } 2066 kmp_r_sched_t new_sched = get__sched_2(parent_team, master_tid); 2067 // set master's schedule as new run-time schedule 2068 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched); 2069 2070 KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq); 2071 KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator); 2072 2073 // Update the floating point rounding in the team if required. 2074 propagateFPControl(team); 2075 2076 if (__kmp_tasking_mode != tskm_immediate_exec) { 2077 // Set master's task team to team's task team. Unless this is hot team, it 2078 // should be NULL. 2079 KMP_DEBUG_ASSERT(master_th->th.th_task_team == 2080 parent_team->t.t_task_team[master_th->th.th_task_state]); 2081 KA_TRACE(20, ("__kmp_fork_call: Master T#%d pushing task_team %p / team " 2082 "%p, new task_team %p / team %p\n", 2083 __kmp_gtid_from_thread(master_th), 2084 master_th->th.th_task_team, parent_team, 2085 team->t.t_task_team[master_th->th.th_task_state], team)); 2086 2087 if (active_level || master_th->th.th_task_team) { 2088 // Take a memo of master's task_state 2089 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack); 2090 if (master_th->th.th_task_state_top >= 2091 master_th->th.th_task_state_stack_sz) { // increase size 2092 kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz; 2093 kmp_uint8 *old_stack, *new_stack; 2094 kmp_uint32 i; 2095 new_stack = (kmp_uint8 *)__kmp_allocate(new_size); 2096 for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) { 2097 new_stack[i] = master_th->th.th_task_state_memo_stack[i]; 2098 } 2099 for (i = master_th->th.th_task_state_stack_sz; i < new_size; 2100 ++i) { // zero-init rest of stack 2101 new_stack[i] = 0; 2102 } 2103 old_stack = master_th->th.th_task_state_memo_stack; 2104 master_th->th.th_task_state_memo_stack = new_stack; 2105 master_th->th.th_task_state_stack_sz = new_size; 2106 __kmp_free(old_stack); 2107 } 2108 // Store master's task_state on stack 2109 master_th->th 2110 .th_task_state_memo_stack[master_th->th.th_task_state_top] = 2111 master_th->th.th_task_state; 2112 master_th->th.th_task_state_top++; 2113 #if KMP_NESTED_HOT_TEAMS 2114 if (master_th->th.th_hot_teams && 2115 active_level < __kmp_hot_teams_max_level && 2116 team == master_th->th.th_hot_teams[active_level].hot_team) { 2117 // Restore master's nested state if nested hot team 2118 master_th->th.th_task_state = 2119 master_th->th 2120 .th_task_state_memo_stack[master_th->th.th_task_state_top]; 2121 } else { 2122 #endif 2123 master_th->th.th_task_state = 0; 2124 #if KMP_NESTED_HOT_TEAMS 2125 } 2126 #endif 2127 } 2128 #if !KMP_NESTED_HOT_TEAMS 2129 KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) || 2130 (team == root->r.r_hot_team)); 2131 #endif 2132 } 2133 2134 KA_TRACE( 2135 20, 2136 ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n", 2137 gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id, 2138 team->t.t_nproc)); 2139 KMP_DEBUG_ASSERT(team != root->r.r_hot_team || 2140 (team->t.t_master_tid == 0 && 2141 (team->t.t_parent == root->r.r_root_team || 2142 team->t.t_parent->t.t_serialized))); 2143 KMP_MB(); 2144 2145 /* now, setup the arguments */ 2146 argv = (void **)team->t.t_argv; 2147 if (ap) { 2148 for (i = argc - 1; i >= 0; --i) { 2149 // TODO: revert workaround for Intel(R) 64 tracker #96 2150 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 2151 void *new_argv = va_arg(*ap, void *); 2152 #else 2153 void *new_argv = va_arg(ap, void *); 2154 #endif 2155 KMP_CHECK_UPDATE(*argv, new_argv); 2156 argv++; 2157 } 2158 } else { 2159 for (i = 0; i < argc; ++i) { 2160 // Get args from parent team for teams construct 2161 KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i]); 2162 } 2163 } 2164 2165 /* now actually fork the threads */ 2166 KMP_CHECK_UPDATE(team->t.t_master_active, master_active); 2167 if (!root->r.r_active) // Only do assignment if it prevents cache ping-pong 2168 root->r.r_active = TRUE; 2169 2170 __kmp_fork_team_threads(root, team, master_th, gtid); 2171 __kmp_setup_icv_copy(team, nthreads, 2172 &master_th->th.th_current_task->td_icvs, loc); 2173 2174 #if OMPT_SUPPORT 2175 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; 2176 #endif 2177 2178 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 2179 2180 #if USE_ITT_BUILD 2181 if (team->t.t_active_level == 1 // only report frames at level 1 2182 && !master_th->th.th_teams_microtask) { // not in teams construct 2183 #if USE_ITT_NOTIFY 2184 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) && 2185 (__kmp_forkjoin_frames_mode == 3 || 2186 __kmp_forkjoin_frames_mode == 1)) { 2187 kmp_uint64 tmp_time = 0; 2188 if (__itt_get_timestamp_ptr) 2189 tmp_time = __itt_get_timestamp(); 2190 // Internal fork - report frame begin 2191 master_th->th.th_frame_time = tmp_time; 2192 if (__kmp_forkjoin_frames_mode == 3) 2193 team->t.t_region_time = tmp_time; 2194 } else 2195 // only one notification scheme (either "submit" or "forking/joined", not both) 2196 #endif /* USE_ITT_NOTIFY */ 2197 if ((__itt_frame_begin_v3_ptr || KMP_ITT_DEBUG) && 2198 __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode) { 2199 // Mark start of "parallel" region for Intel(R) VTune(TM) analyzer. 2200 __kmp_itt_region_forking(gtid, team->t.t_nproc, 0); 2201 } 2202 } 2203 #endif /* USE_ITT_BUILD */ 2204 2205 /* now go on and do the work */ 2206 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team); 2207 KMP_MB(); 2208 KF_TRACE(10, 2209 ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n", 2210 root, team, master_th, gtid)); 2211 2212 #if USE_ITT_BUILD 2213 if (__itt_stack_caller_create_ptr) { 2214 team->t.t_stack_id = 2215 __kmp_itt_stack_caller_create(); // create new stack stitching id 2216 // before entering fork barrier 2217 } 2218 #endif /* USE_ITT_BUILD */ 2219 2220 // AC: skip __kmp_internal_fork at teams construct, let only master 2221 // threads execute 2222 if (ap) { 2223 __kmp_internal_fork(loc, gtid, team); 2224 KF_TRACE(10, ("__kmp_internal_fork : after : root=%p, team=%p, " 2225 "master_th=%p, gtid=%d\n", 2226 root, team, master_th, gtid)); 2227 } 2228 2229 if (call_context == fork_context_gnu) { 2230 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid)); 2231 return TRUE; 2232 } 2233 2234 /* Invoke microtask for MASTER thread */ 2235 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid, 2236 team->t.t_id, team->t.t_pkfn)); 2237 } // END of timer KMP_fork_call block 2238 2239 #if KMP_STATS_ENABLED 2240 // If beginning a teams construct, then change thread state 2241 stats_state_e previous_state = KMP_GET_THREAD_STATE(); 2242 if (!ap) { 2243 KMP_SET_THREAD_STATE(stats_state_e::TEAMS_REGION); 2244 } 2245 #endif 2246 2247 if (!team->t.t_invoke(gtid)) { 2248 KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread"); 2249 } 2250 2251 #if KMP_STATS_ENABLED 2252 // If was beginning of a teams construct, then reset thread state 2253 if (!ap) { 2254 KMP_SET_THREAD_STATE(previous_state); 2255 } 2256 #endif 2257 2258 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid, 2259 team->t.t_id, team->t.t_pkfn)); 2260 KMP_MB(); /* Flush all pending memory write invalidates. */ 2261 2262 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid)); 2263 2264 #if OMPT_SUPPORT 2265 if (ompt_enabled.enabled) { 2266 master_th->th.ompt_thread_info.state = ompt_state_overhead; 2267 } 2268 #endif 2269 2270 return TRUE; 2271 } 2272 2273 #if OMPT_SUPPORT 2274 static inline void __kmp_join_restore_state(kmp_info_t *thread, 2275 kmp_team_t *team) { 2276 // restore state outside the region 2277 thread->th.ompt_thread_info.state = 2278 ((team->t.t_serialized) ? ompt_state_work_serial 2279 : ompt_state_work_parallel); 2280 } 2281 2282 static inline void __kmp_join_ompt(int gtid, kmp_info_t *thread, 2283 kmp_team_t *team, ompt_data_t *parallel_data, 2284 int flags, void *codeptr) { 2285 ompt_task_info_t *task_info = __ompt_get_task_info_object(0); 2286 if (ompt_enabled.ompt_callback_parallel_end) { 2287 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)( 2288 parallel_data, &(task_info->task_data), flags, codeptr); 2289 } 2290 2291 task_info->frame.enter_frame = ompt_data_none; 2292 __kmp_join_restore_state(thread, team); 2293 } 2294 #endif 2295 2296 void __kmp_join_call(ident_t *loc, int gtid 2297 #if OMPT_SUPPORT 2298 , 2299 enum fork_context_e fork_context 2300 #endif 2301 , 2302 int exit_teams) { 2303 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_join_call); 2304 kmp_team_t *team; 2305 kmp_team_t *parent_team; 2306 kmp_info_t *master_th; 2307 kmp_root_t *root; 2308 int master_active; 2309 2310 KA_TRACE(20, ("__kmp_join_call: enter T#%d\n", gtid)); 2311 2312 /* setup current data */ 2313 master_th = __kmp_threads[gtid]; 2314 root = master_th->th.th_root; 2315 team = master_th->th.th_team; 2316 parent_team = team->t.t_parent; 2317 2318 master_th->th.th_ident = loc; 2319 2320 #if OMPT_SUPPORT 2321 void *team_microtask = (void *)team->t.t_pkfn; 2322 if (ompt_enabled.enabled) { 2323 master_th->th.ompt_thread_info.state = ompt_state_overhead; 2324 } 2325 #endif 2326 2327 #if KMP_DEBUG 2328 if (__kmp_tasking_mode != tskm_immediate_exec && !exit_teams) { 2329 KA_TRACE(20, ("__kmp_join_call: T#%d, old team = %p old task_team = %p, " 2330 "th_task_team = %p\n", 2331 __kmp_gtid_from_thread(master_th), team, 2332 team->t.t_task_team[master_th->th.th_task_state], 2333 master_th->th.th_task_team)); 2334 KMP_DEBUG_ASSERT(master_th->th.th_task_team == 2335 team->t.t_task_team[master_th->th.th_task_state]); 2336 } 2337 #endif 2338 2339 if (team->t.t_serialized) { 2340 if (master_th->th.th_teams_microtask) { 2341 // We are in teams construct 2342 int level = team->t.t_level; 2343 int tlevel = master_th->th.th_teams_level; 2344 if (level == tlevel) { 2345 // AC: we haven't incremented it earlier at start of teams construct, 2346 // so do it here - at the end of teams construct 2347 team->t.t_level++; 2348 } else if (level == tlevel + 1) { 2349 // AC: we are exiting parallel inside teams, need to increment 2350 // serialization in order to restore it in the next call to 2351 // __kmpc_end_serialized_parallel 2352 team->t.t_serialized++; 2353 } 2354 } 2355 __kmpc_end_serialized_parallel(loc, gtid); 2356 2357 #if OMPT_SUPPORT 2358 if (ompt_enabled.enabled) { 2359 __kmp_join_restore_state(master_th, parent_team); 2360 } 2361 #endif 2362 2363 return; 2364 } 2365 2366 master_active = team->t.t_master_active; 2367 2368 if (!exit_teams) { 2369 // AC: No barrier for internal teams at exit from teams construct. 2370 // But there is barrier for external team (league). 2371 __kmp_internal_join(loc, gtid, team); 2372 } else { 2373 master_th->th.th_task_state = 2374 0; // AC: no tasking in teams (out of any parallel) 2375 } 2376 2377 KMP_MB(); 2378 2379 #if OMPT_SUPPORT 2380 ompt_data_t *parallel_data = &(team->t.ompt_team_info.parallel_data); 2381 void *codeptr = team->t.ompt_team_info.master_return_address; 2382 #endif 2383 2384 #if USE_ITT_BUILD 2385 if (__itt_stack_caller_create_ptr) { 2386 // destroy the stack stitching id after join barrier 2387 __kmp_itt_stack_caller_destroy((__itt_caller)team->t.t_stack_id); 2388 } 2389 // Mark end of "parallel" region for Intel(R) VTune(TM) analyzer. 2390 if (team->t.t_active_level == 1 && 2391 (!master_th->th.th_teams_microtask || /* not in teams construct */ 2392 master_th->th.th_teams_size.nteams == 1)) { 2393 master_th->th.th_ident = loc; 2394 // only one notification scheme (either "submit" or "forking/joined", not 2395 // both) 2396 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) && 2397 __kmp_forkjoin_frames_mode == 3) 2398 __kmp_itt_frame_submit(gtid, team->t.t_region_time, 2399 master_th->th.th_frame_time, 0, loc, 2400 master_th->th.th_team_nproc, 1); 2401 else if ((__itt_frame_end_v3_ptr || KMP_ITT_DEBUG) && 2402 !__kmp_forkjoin_frames_mode && __kmp_forkjoin_frames) 2403 __kmp_itt_region_joined(gtid); 2404 } // active_level == 1 2405 #endif /* USE_ITT_BUILD */ 2406 2407 if (master_th->th.th_teams_microtask && !exit_teams && 2408 team->t.t_pkfn != (microtask_t)__kmp_teams_master && 2409 team->t.t_level == master_th->th.th_teams_level + 1) { 2410 // AC: We need to leave the team structure intact at the end of parallel 2411 // inside the teams construct, so that at the next parallel same (hot) team 2412 // works, only adjust nesting levels 2413 #if OMPT_SUPPORT 2414 ompt_data_t ompt_parallel_data = ompt_data_none; 2415 if (ompt_enabled.enabled) { 2416 ompt_task_info_t *task_info = __ompt_get_task_info_object(0); 2417 if (ompt_enabled.ompt_callback_implicit_task) { 2418 int ompt_team_size = team->t.t_nproc; 2419 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 2420 ompt_scope_end, NULL, &(task_info->task_data), ompt_team_size, 2421 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit); 2422 } 2423 task_info->frame.exit_frame = ompt_data_none; 2424 task_info->task_data = ompt_data_none; 2425 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th); 2426 __ompt_lw_taskteam_unlink(master_th); 2427 } 2428 #endif 2429 /* Decrement our nested depth level */ 2430 team->t.t_level--; 2431 team->t.t_active_level--; 2432 KMP_ATOMIC_DEC(&root->r.r_in_parallel); 2433 2434 // Restore number of threads in the team if needed. This code relies on 2435 // the proper adjustment of th_teams_size.nth after the fork in 2436 // __kmp_teams_master on each teams master in the case that 2437 // __kmp_reserve_threads reduced it. 2438 if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) { 2439 int old_num = master_th->th.th_team_nproc; 2440 int new_num = master_th->th.th_teams_size.nth; 2441 kmp_info_t **other_threads = team->t.t_threads; 2442 team->t.t_nproc = new_num; 2443 for (int i = 0; i < old_num; ++i) { 2444 other_threads[i]->th.th_team_nproc = new_num; 2445 } 2446 // Adjust states of non-used threads of the team 2447 for (int i = old_num; i < new_num; ++i) { 2448 // Re-initialize thread's barrier data. 2449 KMP_DEBUG_ASSERT(other_threads[i]); 2450 kmp_balign_t *balign = other_threads[i]->th.th_bar; 2451 for (int b = 0; b < bs_last_barrier; ++b) { 2452 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 2453 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 2454 #if USE_DEBUGGER 2455 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 2456 #endif 2457 } 2458 if (__kmp_tasking_mode != tskm_immediate_exec) { 2459 // Synchronize thread's task state 2460 other_threads[i]->th.th_task_state = master_th->th.th_task_state; 2461 } 2462 } 2463 } 2464 2465 #if OMPT_SUPPORT 2466 if (ompt_enabled.enabled) { 2467 __kmp_join_ompt(gtid, master_th, parent_team, &ompt_parallel_data, 2468 OMPT_INVOKER(fork_context) | ompt_parallel_team, codeptr); 2469 } 2470 #endif 2471 2472 return; 2473 } 2474 2475 /* do cleanup and restore the parent team */ 2476 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid; 2477 master_th->th.th_local.this_construct = team->t.t_master_this_cons; 2478 2479 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid]; 2480 2481 /* jc: The following lock has instructions with REL and ACQ semantics, 2482 separating the parallel user code called in this parallel region 2483 from the serial user code called after this function returns. */ 2484 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 2485 2486 if (!master_th->th.th_teams_microtask || 2487 team->t.t_level > master_th->th.th_teams_level) { 2488 /* Decrement our nested depth level */ 2489 KMP_ATOMIC_DEC(&root->r.r_in_parallel); 2490 } 2491 KMP_DEBUG_ASSERT(root->r.r_in_parallel >= 0); 2492 2493 #if OMPT_SUPPORT 2494 if (ompt_enabled.enabled) { 2495 ompt_task_info_t *task_info = __ompt_get_task_info_object(0); 2496 if (ompt_enabled.ompt_callback_implicit_task) { 2497 int flags = (team_microtask == (void *)__kmp_teams_master) 2498 ? ompt_task_initial 2499 : ompt_task_implicit; 2500 int ompt_team_size = (flags == ompt_task_initial) ? 0 : team->t.t_nproc; 2501 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 2502 ompt_scope_end, NULL, &(task_info->task_data), ompt_team_size, 2503 OMPT_CUR_TASK_INFO(master_th)->thread_num, flags); 2504 } 2505 task_info->frame.exit_frame = ompt_data_none; 2506 task_info->task_data = ompt_data_none; 2507 } 2508 #endif 2509 2510 KF_TRACE(10, ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n", 0, 2511 master_th, team)); 2512 __kmp_pop_current_task_from_thread(master_th); 2513 2514 #if KMP_AFFINITY_SUPPORTED 2515 // Restore master thread's partition. 2516 master_th->th.th_first_place = team->t.t_first_place; 2517 master_th->th.th_last_place = team->t.t_last_place; 2518 #endif // KMP_AFFINITY_SUPPORTED 2519 master_th->th.th_def_allocator = team->t.t_def_allocator; 2520 2521 updateHWFPControl(team); 2522 2523 if (root->r.r_active != master_active) 2524 root->r.r_active = master_active; 2525 2526 __kmp_free_team(root, team USE_NESTED_HOT_ARG( 2527 master_th)); // this will free worker threads 2528 2529 /* this race was fun to find. make sure the following is in the critical 2530 region otherwise assertions may fail occasionally since the old team may be 2531 reallocated and the hierarchy appears inconsistent. it is actually safe to 2532 run and won't cause any bugs, but will cause those assertion failures. it's 2533 only one deref&assign so might as well put this in the critical region */ 2534 master_th->th.th_team = parent_team; 2535 master_th->th.th_team_nproc = parent_team->t.t_nproc; 2536 master_th->th.th_team_master = parent_team->t.t_threads[0]; 2537 master_th->th.th_team_serialized = parent_team->t.t_serialized; 2538 2539 /* restore serialized team, if need be */ 2540 if (parent_team->t.t_serialized && 2541 parent_team != master_th->th.th_serial_team && 2542 parent_team != root->r.r_root_team) { 2543 __kmp_free_team(root, 2544 master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL)); 2545 master_th->th.th_serial_team = parent_team; 2546 } 2547 2548 if (__kmp_tasking_mode != tskm_immediate_exec) { 2549 if (master_th->th.th_task_state_top > 2550 0) { // Restore task state from memo stack 2551 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack); 2552 // Remember master's state if we re-use this nested hot team 2553 master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] = 2554 master_th->th.th_task_state; 2555 --master_th->th.th_task_state_top; // pop 2556 // Now restore state at this level 2557 master_th->th.th_task_state = 2558 master_th->th 2559 .th_task_state_memo_stack[master_th->th.th_task_state_top]; 2560 } 2561 // Copy the task team from the parent team to the master thread 2562 master_th->th.th_task_team = 2563 parent_team->t.t_task_team[master_th->th.th_task_state]; 2564 KA_TRACE(20, 2565 ("__kmp_join_call: Master T#%d restoring task_team %p / team %p\n", 2566 __kmp_gtid_from_thread(master_th), master_th->th.th_task_team, 2567 parent_team)); 2568 } 2569 2570 // TODO: GEH - cannot do this assertion because root thread not set up as 2571 // executing 2572 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 0 ); 2573 master_th->th.th_current_task->td_flags.executing = 1; 2574 2575 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 2576 2577 #if OMPT_SUPPORT 2578 int flags = 2579 OMPT_INVOKER(fork_context) | 2580 ((team_microtask == (void *)__kmp_teams_master) ? ompt_parallel_league 2581 : ompt_parallel_team); 2582 if (ompt_enabled.enabled) { 2583 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, flags, 2584 codeptr); 2585 } 2586 #endif 2587 2588 KMP_MB(); 2589 KA_TRACE(20, ("__kmp_join_call: exit T#%d\n", gtid)); 2590 } 2591 2592 /* Check whether we should push an internal control record onto the 2593 serial team stack. If so, do it. */ 2594 void __kmp_save_internal_controls(kmp_info_t *thread) { 2595 2596 if (thread->th.th_team != thread->th.th_serial_team) { 2597 return; 2598 } 2599 if (thread->th.th_team->t.t_serialized > 1) { 2600 int push = 0; 2601 2602 if (thread->th.th_team->t.t_control_stack_top == NULL) { 2603 push = 1; 2604 } else { 2605 if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level != 2606 thread->th.th_team->t.t_serialized) { 2607 push = 1; 2608 } 2609 } 2610 if (push) { /* push a record on the serial team's stack */ 2611 kmp_internal_control_t *control = 2612 (kmp_internal_control_t *)__kmp_allocate( 2613 sizeof(kmp_internal_control_t)); 2614 2615 copy_icvs(control, &thread->th.th_current_task->td_icvs); 2616 2617 control->serial_nesting_level = thread->th.th_team->t.t_serialized; 2618 2619 control->next = thread->th.th_team->t.t_control_stack_top; 2620 thread->th.th_team->t.t_control_stack_top = control; 2621 } 2622 } 2623 } 2624 2625 /* Changes set_nproc */ 2626 void __kmp_set_num_threads(int new_nth, int gtid) { 2627 kmp_info_t *thread; 2628 kmp_root_t *root; 2629 2630 KF_TRACE(10, ("__kmp_set_num_threads: new __kmp_nth = %d\n", new_nth)); 2631 KMP_DEBUG_ASSERT(__kmp_init_serial); 2632 2633 if (new_nth < 1) 2634 new_nth = 1; 2635 else if (new_nth > __kmp_max_nth) 2636 new_nth = __kmp_max_nth; 2637 2638 KMP_COUNT_VALUE(OMP_set_numthreads, new_nth); 2639 thread = __kmp_threads[gtid]; 2640 if (thread->th.th_current_task->td_icvs.nproc == new_nth) 2641 return; // nothing to do 2642 2643 __kmp_save_internal_controls(thread); 2644 2645 set__nproc(thread, new_nth); 2646 2647 // If this omp_set_num_threads() call will cause the hot team size to be 2648 // reduced (in the absence of a num_threads clause), then reduce it now, 2649 // rather than waiting for the next parallel region. 2650 root = thread->th.th_root; 2651 if (__kmp_init_parallel && (!root->r.r_active) && 2652 (root->r.r_hot_team->t.t_nproc > new_nth) 2653 #if KMP_NESTED_HOT_TEAMS 2654 && __kmp_hot_teams_max_level && !__kmp_hot_teams_mode 2655 #endif 2656 ) { 2657 kmp_team_t *hot_team = root->r.r_hot_team; 2658 int f; 2659 2660 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 2661 2662 // Release the extra threads we don't need any more. 2663 for (f = new_nth; f < hot_team->t.t_nproc; f++) { 2664 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL); 2665 if (__kmp_tasking_mode != tskm_immediate_exec) { 2666 // When decreasing team size, threads no longer in the team should unref 2667 // task team. 2668 hot_team->t.t_threads[f]->th.th_task_team = NULL; 2669 } 2670 __kmp_free_thread(hot_team->t.t_threads[f]); 2671 hot_team->t.t_threads[f] = NULL; 2672 } 2673 hot_team->t.t_nproc = new_nth; 2674 #if KMP_NESTED_HOT_TEAMS 2675 if (thread->th.th_hot_teams) { 2676 KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team); 2677 thread->th.th_hot_teams[0].hot_team_nth = new_nth; 2678 } 2679 #endif 2680 2681 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 2682 2683 // Update the t_nproc field in the threads that are still active. 2684 for (f = 0; f < new_nth; f++) { 2685 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL); 2686 hot_team->t.t_threads[f]->th.th_team_nproc = new_nth; 2687 } 2688 // Special flag in case omp_set_num_threads() call 2689 hot_team->t.t_size_changed = -1; 2690 } 2691 } 2692 2693 /* Changes max_active_levels */ 2694 void __kmp_set_max_active_levels(int gtid, int max_active_levels) { 2695 kmp_info_t *thread; 2696 2697 KF_TRACE(10, ("__kmp_set_max_active_levels: new max_active_levels for thread " 2698 "%d = (%d)\n", 2699 gtid, max_active_levels)); 2700 KMP_DEBUG_ASSERT(__kmp_init_serial); 2701 2702 // validate max_active_levels 2703 if (max_active_levels < 0) { 2704 KMP_WARNING(ActiveLevelsNegative, max_active_levels); 2705 // We ignore this call if the user has specified a negative value. 2706 // The current setting won't be changed. The last valid setting will be 2707 // used. A warning will be issued (if warnings are allowed as controlled by 2708 // the KMP_WARNINGS env var). 2709 KF_TRACE(10, ("__kmp_set_max_active_levels: the call is ignored: new " 2710 "max_active_levels for thread %d = (%d)\n", 2711 gtid, max_active_levels)); 2712 return; 2713 } 2714 if (max_active_levels <= KMP_MAX_ACTIVE_LEVELS_LIMIT) { 2715 // it's OK, the max_active_levels is within the valid range: [ 0; 2716 // KMP_MAX_ACTIVE_LEVELS_LIMIT ] 2717 // We allow a zero value. (implementation defined behavior) 2718 } else { 2719 KMP_WARNING(ActiveLevelsExceedLimit, max_active_levels, 2720 KMP_MAX_ACTIVE_LEVELS_LIMIT); 2721 max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT; 2722 // Current upper limit is MAX_INT. (implementation defined behavior) 2723 // If the input exceeds the upper limit, we correct the input to be the 2724 // upper limit. (implementation defined behavior) 2725 // Actually, the flow should never get here until we use MAX_INT limit. 2726 } 2727 KF_TRACE(10, ("__kmp_set_max_active_levels: after validation: new " 2728 "max_active_levels for thread %d = (%d)\n", 2729 gtid, max_active_levels)); 2730 2731 thread = __kmp_threads[gtid]; 2732 2733 __kmp_save_internal_controls(thread); 2734 2735 set__max_active_levels(thread, max_active_levels); 2736 } 2737 2738 /* Gets max_active_levels */ 2739 int __kmp_get_max_active_levels(int gtid) { 2740 kmp_info_t *thread; 2741 2742 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d\n", gtid)); 2743 KMP_DEBUG_ASSERT(__kmp_init_serial); 2744 2745 thread = __kmp_threads[gtid]; 2746 KMP_DEBUG_ASSERT(thread->th.th_current_task); 2747 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d, curtask=%p, " 2748 "curtask_maxaclevel=%d\n", 2749 gtid, thread->th.th_current_task, 2750 thread->th.th_current_task->td_icvs.max_active_levels)); 2751 return thread->th.th_current_task->td_icvs.max_active_levels; 2752 } 2753 2754 KMP_BUILD_ASSERT(sizeof(kmp_sched_t) == sizeof(int)); 2755 KMP_BUILD_ASSERT(sizeof(enum sched_type) == sizeof(int)); 2756 2757 /* Changes def_sched_var ICV values (run-time schedule kind and chunk) */ 2758 void __kmp_set_schedule(int gtid, kmp_sched_t kind, int chunk) { 2759 kmp_info_t *thread; 2760 kmp_sched_t orig_kind; 2761 // kmp_team_t *team; 2762 2763 KF_TRACE(10, ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n", 2764 gtid, (int)kind, chunk)); 2765 KMP_DEBUG_ASSERT(__kmp_init_serial); 2766 2767 // Check if the kind parameter is valid, correct if needed. 2768 // Valid parameters should fit in one of two intervals - standard or extended: 2769 // <lower>, <valid>, <upper_std>, <lower_ext>, <valid>, <upper> 2770 // 2008-01-25: 0, 1 - 4, 5, 100, 101 - 102, 103 2771 orig_kind = kind; 2772 kind = __kmp_sched_without_mods(kind); 2773 2774 if (kind <= kmp_sched_lower || kind >= kmp_sched_upper || 2775 (kind <= kmp_sched_lower_ext && kind >= kmp_sched_upper_std)) { 2776 // TODO: Hint needs attention in case we change the default schedule. 2777 __kmp_msg(kmp_ms_warning, KMP_MSG(ScheduleKindOutOfRange, kind), 2778 KMP_HNT(DefaultScheduleKindUsed, "static, no chunk"), 2779 __kmp_msg_null); 2780 kind = kmp_sched_default; 2781 chunk = 0; // ignore chunk value in case of bad kind 2782 } 2783 2784 thread = __kmp_threads[gtid]; 2785 2786 __kmp_save_internal_controls(thread); 2787 2788 if (kind < kmp_sched_upper_std) { 2789 if (kind == kmp_sched_static && chunk < KMP_DEFAULT_CHUNK) { 2790 // differ static chunked vs. unchunked: chunk should be invalid to 2791 // indicate unchunked schedule (which is the default) 2792 thread->th.th_current_task->td_icvs.sched.r_sched_type = kmp_sch_static; 2793 } else { 2794 thread->th.th_current_task->td_icvs.sched.r_sched_type = 2795 __kmp_sch_map[kind - kmp_sched_lower - 1]; 2796 } 2797 } else { 2798 // __kmp_sch_map[ kind - kmp_sched_lower_ext + kmp_sched_upper_std - 2799 // kmp_sched_lower - 2 ]; 2800 thread->th.th_current_task->td_icvs.sched.r_sched_type = 2801 __kmp_sch_map[kind - kmp_sched_lower_ext + kmp_sched_upper_std - 2802 kmp_sched_lower - 2]; 2803 } 2804 __kmp_sched_apply_mods_intkind( 2805 orig_kind, &(thread->th.th_current_task->td_icvs.sched.r_sched_type)); 2806 if (kind == kmp_sched_auto || chunk < 1) { 2807 // ignore parameter chunk for schedule auto 2808 thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK; 2809 } else { 2810 thread->th.th_current_task->td_icvs.sched.chunk = chunk; 2811 } 2812 } 2813 2814 /* Gets def_sched_var ICV values */ 2815 void __kmp_get_schedule(int gtid, kmp_sched_t *kind, int *chunk) { 2816 kmp_info_t *thread; 2817 enum sched_type th_type; 2818 2819 KF_TRACE(10, ("__kmp_get_schedule: thread %d\n", gtid)); 2820 KMP_DEBUG_ASSERT(__kmp_init_serial); 2821 2822 thread = __kmp_threads[gtid]; 2823 2824 th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type; 2825 switch (SCHEDULE_WITHOUT_MODIFIERS(th_type)) { 2826 case kmp_sch_static: 2827 case kmp_sch_static_greedy: 2828 case kmp_sch_static_balanced: 2829 *kind = kmp_sched_static; 2830 __kmp_sched_apply_mods_stdkind(kind, th_type); 2831 *chunk = 0; // chunk was not set, try to show this fact via zero value 2832 return; 2833 case kmp_sch_static_chunked: 2834 *kind = kmp_sched_static; 2835 break; 2836 case kmp_sch_dynamic_chunked: 2837 *kind = kmp_sched_dynamic; 2838 break; 2839 case kmp_sch_guided_chunked: 2840 case kmp_sch_guided_iterative_chunked: 2841 case kmp_sch_guided_analytical_chunked: 2842 *kind = kmp_sched_guided; 2843 break; 2844 case kmp_sch_auto: 2845 *kind = kmp_sched_auto; 2846 break; 2847 case kmp_sch_trapezoidal: 2848 *kind = kmp_sched_trapezoidal; 2849 break; 2850 #if KMP_STATIC_STEAL_ENABLED 2851 case kmp_sch_static_steal: 2852 *kind = kmp_sched_static_steal; 2853 break; 2854 #endif 2855 default: 2856 KMP_FATAL(UnknownSchedulingType, th_type); 2857 } 2858 2859 __kmp_sched_apply_mods_stdkind(kind, th_type); 2860 *chunk = thread->th.th_current_task->td_icvs.sched.chunk; 2861 } 2862 2863 int __kmp_get_ancestor_thread_num(int gtid, int level) { 2864 2865 int ii, dd; 2866 kmp_team_t *team; 2867 kmp_info_t *thr; 2868 2869 KF_TRACE(10, ("__kmp_get_ancestor_thread_num: thread %d %d\n", gtid, level)); 2870 KMP_DEBUG_ASSERT(__kmp_init_serial); 2871 2872 // validate level 2873 if (level == 0) 2874 return 0; 2875 if (level < 0) 2876 return -1; 2877 thr = __kmp_threads[gtid]; 2878 team = thr->th.th_team; 2879 ii = team->t.t_level; 2880 if (level > ii) 2881 return -1; 2882 2883 if (thr->th.th_teams_microtask) { 2884 // AC: we are in teams region where multiple nested teams have same level 2885 int tlevel = thr->th.th_teams_level; // the level of the teams construct 2886 if (level <= 2887 tlevel) { // otherwise usual algorithm works (will not touch the teams) 2888 KMP_DEBUG_ASSERT(ii >= tlevel); 2889 // AC: As we need to pass by the teams league, we need to artificially 2890 // increase ii 2891 if (ii == tlevel) { 2892 ii += 2; // three teams have same level 2893 } else { 2894 ii++; // two teams have same level 2895 } 2896 } 2897 } 2898 2899 if (ii == level) 2900 return __kmp_tid_from_gtid(gtid); 2901 2902 dd = team->t.t_serialized; 2903 level++; 2904 while (ii > level) { 2905 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) { 2906 } 2907 if ((team->t.t_serialized) && (!dd)) { 2908 team = team->t.t_parent; 2909 continue; 2910 } 2911 if (ii > level) { 2912 team = team->t.t_parent; 2913 dd = team->t.t_serialized; 2914 ii--; 2915 } 2916 } 2917 2918 return (dd > 1) ? (0) : (team->t.t_master_tid); 2919 } 2920 2921 int __kmp_get_team_size(int gtid, int level) { 2922 2923 int ii, dd; 2924 kmp_team_t *team; 2925 kmp_info_t *thr; 2926 2927 KF_TRACE(10, ("__kmp_get_team_size: thread %d %d\n", gtid, level)); 2928 KMP_DEBUG_ASSERT(__kmp_init_serial); 2929 2930 // validate level 2931 if (level == 0) 2932 return 1; 2933 if (level < 0) 2934 return -1; 2935 thr = __kmp_threads[gtid]; 2936 team = thr->th.th_team; 2937 ii = team->t.t_level; 2938 if (level > ii) 2939 return -1; 2940 2941 if (thr->th.th_teams_microtask) { 2942 // AC: we are in teams region where multiple nested teams have same level 2943 int tlevel = thr->th.th_teams_level; // the level of the teams construct 2944 if (level <= 2945 tlevel) { // otherwise usual algorithm works (will not touch the teams) 2946 KMP_DEBUG_ASSERT(ii >= tlevel); 2947 // AC: As we need to pass by the teams league, we need to artificially 2948 // increase ii 2949 if (ii == tlevel) { 2950 ii += 2; // three teams have same level 2951 } else { 2952 ii++; // two teams have same level 2953 } 2954 } 2955 } 2956 2957 while (ii > level) { 2958 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) { 2959 } 2960 if (team->t.t_serialized && (!dd)) { 2961 team = team->t.t_parent; 2962 continue; 2963 } 2964 if (ii > level) { 2965 team = team->t.t_parent; 2966 ii--; 2967 } 2968 } 2969 2970 return team->t.t_nproc; 2971 } 2972 2973 kmp_r_sched_t __kmp_get_schedule_global() { 2974 // This routine created because pairs (__kmp_sched, __kmp_chunk) and 2975 // (__kmp_static, __kmp_guided) may be changed by kmp_set_defaults 2976 // independently. So one can get the updated schedule here. 2977 2978 kmp_r_sched_t r_sched; 2979 2980 // create schedule from 4 globals: __kmp_sched, __kmp_chunk, __kmp_static, 2981 // __kmp_guided. __kmp_sched should keep original value, so that user can set 2982 // KMP_SCHEDULE multiple times, and thus have different run-time schedules in 2983 // different roots (even in OMP 2.5) 2984 enum sched_type s = SCHEDULE_WITHOUT_MODIFIERS(__kmp_sched); 2985 enum sched_type sched_modifiers = SCHEDULE_GET_MODIFIERS(__kmp_sched); 2986 if (s == kmp_sch_static) { 2987 // replace STATIC with more detailed schedule (balanced or greedy) 2988 r_sched.r_sched_type = __kmp_static; 2989 } else if (s == kmp_sch_guided_chunked) { 2990 // replace GUIDED with more detailed schedule (iterative or analytical) 2991 r_sched.r_sched_type = __kmp_guided; 2992 } else { // (STATIC_CHUNKED), or (DYNAMIC_CHUNKED), or other 2993 r_sched.r_sched_type = __kmp_sched; 2994 } 2995 SCHEDULE_SET_MODIFIERS(r_sched.r_sched_type, sched_modifiers); 2996 2997 if (__kmp_chunk < KMP_DEFAULT_CHUNK) { 2998 // __kmp_chunk may be wrong here (if it was not ever set) 2999 r_sched.chunk = KMP_DEFAULT_CHUNK; 3000 } else { 3001 r_sched.chunk = __kmp_chunk; 3002 } 3003 3004 return r_sched; 3005 } 3006 3007 /* Allocate (realloc == FALSE) * or reallocate (realloc == TRUE) 3008 at least argc number of *t_argv entries for the requested team. */ 3009 static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team, int realloc) { 3010 3011 KMP_DEBUG_ASSERT(team); 3012 if (!realloc || argc > team->t.t_max_argc) { 3013 3014 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: needed entries=%d, " 3015 "current entries=%d\n", 3016 team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0)); 3017 /* if previously allocated heap space for args, free them */ 3018 if (realloc && team->t.t_argv != &team->t.t_inline_argv[0]) 3019 __kmp_free((void *)team->t.t_argv); 3020 3021 if (argc <= KMP_INLINE_ARGV_ENTRIES) { 3022 /* use unused space in the cache line for arguments */ 3023 team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES; 3024 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: inline allocate %d " 3025 "argv entries\n", 3026 team->t.t_id, team->t.t_max_argc)); 3027 team->t.t_argv = &team->t.t_inline_argv[0]; 3028 if (__kmp_storage_map) { 3029 __kmp_print_storage_map_gtid( 3030 -1, &team->t.t_inline_argv[0], 3031 &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES], 3032 (sizeof(void *) * KMP_INLINE_ARGV_ENTRIES), "team_%d.t_inline_argv", 3033 team->t.t_id); 3034 } 3035 } else { 3036 /* allocate space for arguments in the heap */ 3037 team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES >> 1)) 3038 ? KMP_MIN_MALLOC_ARGV_ENTRIES 3039 : 2 * argc; 3040 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d " 3041 "argv entries\n", 3042 team->t.t_id, team->t.t_max_argc)); 3043 team->t.t_argv = 3044 (void **)__kmp_page_allocate(sizeof(void *) * team->t.t_max_argc); 3045 if (__kmp_storage_map) { 3046 __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0], 3047 &team->t.t_argv[team->t.t_max_argc], 3048 sizeof(void *) * team->t.t_max_argc, 3049 "team_%d.t_argv", team->t.t_id); 3050 } 3051 } 3052 } 3053 } 3054 3055 static void __kmp_allocate_team_arrays(kmp_team_t *team, int max_nth) { 3056 int i; 3057 int num_disp_buff = max_nth > 1 ? __kmp_dispatch_num_buffers : 2; 3058 team->t.t_threads = 3059 (kmp_info_t **)__kmp_allocate(sizeof(kmp_info_t *) * max_nth); 3060 team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate( 3061 sizeof(dispatch_shared_info_t) * num_disp_buff); 3062 team->t.t_dispatch = 3063 (kmp_disp_t *)__kmp_allocate(sizeof(kmp_disp_t) * max_nth); 3064 team->t.t_implicit_task_taskdata = 3065 (kmp_taskdata_t *)__kmp_allocate(sizeof(kmp_taskdata_t) * max_nth); 3066 team->t.t_max_nproc = max_nth; 3067 3068 /* setup dispatch buffers */ 3069 for (i = 0; i < num_disp_buff; ++i) { 3070 team->t.t_disp_buffer[i].buffer_index = i; 3071 team->t.t_disp_buffer[i].doacross_buf_idx = i; 3072 } 3073 } 3074 3075 static void __kmp_free_team_arrays(kmp_team_t *team) { 3076 /* Note: this does not free the threads in t_threads (__kmp_free_threads) */ 3077 int i; 3078 for (i = 0; i < team->t.t_max_nproc; ++i) { 3079 if (team->t.t_dispatch[i].th_disp_buffer != NULL) { 3080 __kmp_free(team->t.t_dispatch[i].th_disp_buffer); 3081 team->t.t_dispatch[i].th_disp_buffer = NULL; 3082 } 3083 } 3084 #if KMP_USE_HIER_SCHED 3085 __kmp_dispatch_free_hierarchies(team); 3086 #endif 3087 __kmp_free(team->t.t_threads); 3088 __kmp_free(team->t.t_disp_buffer); 3089 __kmp_free(team->t.t_dispatch); 3090 __kmp_free(team->t.t_implicit_task_taskdata); 3091 team->t.t_threads = NULL; 3092 team->t.t_disp_buffer = NULL; 3093 team->t.t_dispatch = NULL; 3094 team->t.t_implicit_task_taskdata = 0; 3095 } 3096 3097 static void __kmp_reallocate_team_arrays(kmp_team_t *team, int max_nth) { 3098 kmp_info_t **oldThreads = team->t.t_threads; 3099 3100 __kmp_free(team->t.t_disp_buffer); 3101 __kmp_free(team->t.t_dispatch); 3102 __kmp_free(team->t.t_implicit_task_taskdata); 3103 __kmp_allocate_team_arrays(team, max_nth); 3104 3105 KMP_MEMCPY(team->t.t_threads, oldThreads, 3106 team->t.t_nproc * sizeof(kmp_info_t *)); 3107 3108 __kmp_free(oldThreads); 3109 } 3110 3111 static kmp_internal_control_t __kmp_get_global_icvs(void) { 3112 3113 kmp_r_sched_t r_sched = 3114 __kmp_get_schedule_global(); // get current state of scheduling globals 3115 3116 KMP_DEBUG_ASSERT(__kmp_nested_proc_bind.used > 0); 3117 3118 kmp_internal_control_t g_icvs = { 3119 0, // int serial_nesting_level; //corresponds to value of th_team_serialized 3120 (kmp_int8)__kmp_global.g.g_dynamic, // internal control for dynamic 3121 // adjustment of threads (per thread) 3122 (kmp_int8)__kmp_env_blocktime, // int bt_set; //internal control for 3123 // whether blocktime is explicitly set 3124 __kmp_dflt_blocktime, // int blocktime; //internal control for blocktime 3125 #if KMP_USE_MONITOR 3126 __kmp_bt_intervals, // int bt_intervals; //internal control for blocktime 3127 // intervals 3128 #endif 3129 __kmp_dflt_team_nth, // int nproc; //internal control for # of threads for 3130 // next parallel region (per thread) 3131 // (use a max ub on value if __kmp_parallel_initialize not called yet) 3132 __kmp_cg_max_nth, // int thread_limit; 3133 __kmp_dflt_max_active_levels, // int max_active_levels; //internal control 3134 // for max_active_levels 3135 r_sched, // kmp_r_sched_t sched; //internal control for runtime schedule 3136 // {sched,chunk} pair 3137 __kmp_nested_proc_bind.bind_types[0], 3138 __kmp_default_device, 3139 NULL // struct kmp_internal_control *next; 3140 }; 3141 3142 return g_icvs; 3143 } 3144 3145 static kmp_internal_control_t __kmp_get_x_global_icvs(const kmp_team_t *team) { 3146 3147 kmp_internal_control_t gx_icvs; 3148 gx_icvs.serial_nesting_level = 3149 0; // probably =team->t.t_serial like in save_inter_controls 3150 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs); 3151 gx_icvs.next = NULL; 3152 3153 return gx_icvs; 3154 } 3155 3156 static void __kmp_initialize_root(kmp_root_t *root) { 3157 int f; 3158 kmp_team_t *root_team; 3159 kmp_team_t *hot_team; 3160 int hot_team_max_nth; 3161 kmp_r_sched_t r_sched = 3162 __kmp_get_schedule_global(); // get current state of scheduling globals 3163 kmp_internal_control_t r_icvs = __kmp_get_global_icvs(); 3164 KMP_DEBUG_ASSERT(root); 3165 KMP_ASSERT(!root->r.r_begin); 3166 3167 /* setup the root state structure */ 3168 __kmp_init_lock(&root->r.r_begin_lock); 3169 root->r.r_begin = FALSE; 3170 root->r.r_active = FALSE; 3171 root->r.r_in_parallel = 0; 3172 root->r.r_blocktime = __kmp_dflt_blocktime; 3173 3174 /* setup the root team for this task */ 3175 /* allocate the root team structure */ 3176 KF_TRACE(10, ("__kmp_initialize_root: before root_team\n")); 3177 3178 root_team = 3179 __kmp_allocate_team(root, 3180 1, // new_nproc 3181 1, // max_nproc 3182 #if OMPT_SUPPORT 3183 ompt_data_none, // root parallel id 3184 #endif 3185 __kmp_nested_proc_bind.bind_types[0], &r_icvs, 3186 0 // argc 3187 USE_NESTED_HOT_ARG(NULL) // master thread is unknown 3188 ); 3189 #if USE_DEBUGGER 3190 // Non-NULL value should be assigned to make the debugger display the root 3191 // team. 3192 TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)(~0)); 3193 #endif 3194 3195 KF_TRACE(10, ("__kmp_initialize_root: after root_team = %p\n", root_team)); 3196 3197 root->r.r_root_team = root_team; 3198 root_team->t.t_control_stack_top = NULL; 3199 3200 /* initialize root team */ 3201 root_team->t.t_threads[0] = NULL; 3202 root_team->t.t_nproc = 1; 3203 root_team->t.t_serialized = 1; 3204 // TODO???: root_team->t.t_max_active_levels = __kmp_dflt_max_active_levels; 3205 root_team->t.t_sched.sched = r_sched.sched; 3206 KA_TRACE( 3207 20, 3208 ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n", 3209 root_team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); 3210 3211 /* setup the hot team for this task */ 3212 /* allocate the hot team structure */ 3213 KF_TRACE(10, ("__kmp_initialize_root: before hot_team\n")); 3214 3215 hot_team = 3216 __kmp_allocate_team(root, 3217 1, // new_nproc 3218 __kmp_dflt_team_nth_ub * 2, // max_nproc 3219 #if OMPT_SUPPORT 3220 ompt_data_none, // root parallel id 3221 #endif 3222 __kmp_nested_proc_bind.bind_types[0], &r_icvs, 3223 0 // argc 3224 USE_NESTED_HOT_ARG(NULL) // master thread is unknown 3225 ); 3226 KF_TRACE(10, ("__kmp_initialize_root: after hot_team = %p\n", hot_team)); 3227 3228 root->r.r_hot_team = hot_team; 3229 root_team->t.t_control_stack_top = NULL; 3230 3231 /* first-time initialization */ 3232 hot_team->t.t_parent = root_team; 3233 3234 /* initialize hot team */ 3235 hot_team_max_nth = hot_team->t.t_max_nproc; 3236 for (f = 0; f < hot_team_max_nth; ++f) { 3237 hot_team->t.t_threads[f] = NULL; 3238 } 3239 hot_team->t.t_nproc = 1; 3240 // TODO???: hot_team->t.t_max_active_levels = __kmp_dflt_max_active_levels; 3241 hot_team->t.t_sched.sched = r_sched.sched; 3242 hot_team->t.t_size_changed = 0; 3243 } 3244 3245 #ifdef KMP_DEBUG 3246 3247 typedef struct kmp_team_list_item { 3248 kmp_team_p const *entry; 3249 struct kmp_team_list_item *next; 3250 } kmp_team_list_item_t; 3251 typedef kmp_team_list_item_t *kmp_team_list_t; 3252 3253 static void __kmp_print_structure_team_accum( // Add team to list of teams. 3254 kmp_team_list_t list, // List of teams. 3255 kmp_team_p const *team // Team to add. 3256 ) { 3257 3258 // List must terminate with item where both entry and next are NULL. 3259 // Team is added to the list only once. 3260 // List is sorted in ascending order by team id. 3261 // Team id is *not* a key. 3262 3263 kmp_team_list_t l; 3264 3265 KMP_DEBUG_ASSERT(list != NULL); 3266 if (team == NULL) { 3267 return; 3268 } 3269 3270 __kmp_print_structure_team_accum(list, team->t.t_parent); 3271 __kmp_print_structure_team_accum(list, team->t.t_next_pool); 3272 3273 // Search list for the team. 3274 l = list; 3275 while (l->next != NULL && l->entry != team) { 3276 l = l->next; 3277 } 3278 if (l->next != NULL) { 3279 return; // Team has been added before, exit. 3280 } 3281 3282 // Team is not found. Search list again for insertion point. 3283 l = list; 3284 while (l->next != NULL && l->entry->t.t_id <= team->t.t_id) { 3285 l = l->next; 3286 } 3287 3288 // Insert team. 3289 { 3290 kmp_team_list_item_t *item = (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC( 3291 sizeof(kmp_team_list_item_t)); 3292 *item = *l; 3293 l->entry = team; 3294 l->next = item; 3295 } 3296 } 3297 3298 static void __kmp_print_structure_team(char const *title, kmp_team_p const *team 3299 3300 ) { 3301 __kmp_printf("%s", title); 3302 if (team != NULL) { 3303 __kmp_printf("%2x %p\n", team->t.t_id, team); 3304 } else { 3305 __kmp_printf(" - (nil)\n"); 3306 } 3307 } 3308 3309 static void __kmp_print_structure_thread(char const *title, 3310 kmp_info_p const *thread) { 3311 __kmp_printf("%s", title); 3312 if (thread != NULL) { 3313 __kmp_printf("%2d %p\n", thread->th.th_info.ds.ds_gtid, thread); 3314 } else { 3315 __kmp_printf(" - (nil)\n"); 3316 } 3317 } 3318 3319 void __kmp_print_structure(void) { 3320 3321 kmp_team_list_t list; 3322 3323 // Initialize list of teams. 3324 list = 3325 (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(sizeof(kmp_team_list_item_t)); 3326 list->entry = NULL; 3327 list->next = NULL; 3328 3329 __kmp_printf("\n------------------------------\nGlobal Thread " 3330 "Table\n------------------------------\n"); 3331 { 3332 int gtid; 3333 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) { 3334 __kmp_printf("%2d", gtid); 3335 if (__kmp_threads != NULL) { 3336 __kmp_printf(" %p", __kmp_threads[gtid]); 3337 } 3338 if (__kmp_root != NULL) { 3339 __kmp_printf(" %p", __kmp_root[gtid]); 3340 } 3341 __kmp_printf("\n"); 3342 } 3343 } 3344 3345 // Print out __kmp_threads array. 3346 __kmp_printf("\n------------------------------\nThreads\n--------------------" 3347 "----------\n"); 3348 if (__kmp_threads != NULL) { 3349 int gtid; 3350 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) { 3351 kmp_info_t const *thread = __kmp_threads[gtid]; 3352 if (thread != NULL) { 3353 __kmp_printf("GTID %2d %p:\n", gtid, thread); 3354 __kmp_printf(" Our Root: %p\n", thread->th.th_root); 3355 __kmp_print_structure_team(" Our Team: ", thread->th.th_team); 3356 __kmp_print_structure_team(" Serial Team: ", 3357 thread->th.th_serial_team); 3358 __kmp_printf(" Threads: %2d\n", thread->th.th_team_nproc); 3359 __kmp_print_structure_thread(" Master: ", 3360 thread->th.th_team_master); 3361 __kmp_printf(" Serialized?: %2d\n", thread->th.th_team_serialized); 3362 __kmp_printf(" Set NProc: %2d\n", thread->th.th_set_nproc); 3363 __kmp_printf(" Set Proc Bind: %2d\n", thread->th.th_set_proc_bind); 3364 __kmp_print_structure_thread(" Next in pool: ", 3365 thread->th.th_next_pool); 3366 __kmp_printf("\n"); 3367 __kmp_print_structure_team_accum(list, thread->th.th_team); 3368 __kmp_print_structure_team_accum(list, thread->th.th_serial_team); 3369 } 3370 } 3371 } else { 3372 __kmp_printf("Threads array is not allocated.\n"); 3373 } 3374 3375 // Print out __kmp_root array. 3376 __kmp_printf("\n------------------------------\nUbers\n----------------------" 3377 "--------\n"); 3378 if (__kmp_root != NULL) { 3379 int gtid; 3380 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) { 3381 kmp_root_t const *root = __kmp_root[gtid]; 3382 if (root != NULL) { 3383 __kmp_printf("GTID %2d %p:\n", gtid, root); 3384 __kmp_print_structure_team(" Root Team: ", root->r.r_root_team); 3385 __kmp_print_structure_team(" Hot Team: ", root->r.r_hot_team); 3386 __kmp_print_structure_thread(" Uber Thread: ", 3387 root->r.r_uber_thread); 3388 __kmp_printf(" Active?: %2d\n", root->r.r_active); 3389 __kmp_printf(" In Parallel: %2d\n", 3390 KMP_ATOMIC_LD_RLX(&root->r.r_in_parallel)); 3391 __kmp_printf("\n"); 3392 __kmp_print_structure_team_accum(list, root->r.r_root_team); 3393 __kmp_print_structure_team_accum(list, root->r.r_hot_team); 3394 } 3395 } 3396 } else { 3397 __kmp_printf("Ubers array is not allocated.\n"); 3398 } 3399 3400 __kmp_printf("\n------------------------------\nTeams\n----------------------" 3401 "--------\n"); 3402 while (list->next != NULL) { 3403 kmp_team_p const *team = list->entry; 3404 int i; 3405 __kmp_printf("Team %2x %p:\n", team->t.t_id, team); 3406 __kmp_print_structure_team(" Parent Team: ", team->t.t_parent); 3407 __kmp_printf(" Master TID: %2d\n", team->t.t_master_tid); 3408 __kmp_printf(" Max threads: %2d\n", team->t.t_max_nproc); 3409 __kmp_printf(" Levels of serial: %2d\n", team->t.t_serialized); 3410 __kmp_printf(" Number threads: %2d\n", team->t.t_nproc); 3411 for (i = 0; i < team->t.t_nproc; ++i) { 3412 __kmp_printf(" Thread %2d: ", i); 3413 __kmp_print_structure_thread("", team->t.t_threads[i]); 3414 } 3415 __kmp_print_structure_team(" Next in pool: ", team->t.t_next_pool); 3416 __kmp_printf("\n"); 3417 list = list->next; 3418 } 3419 3420 // Print out __kmp_thread_pool and __kmp_team_pool. 3421 __kmp_printf("\n------------------------------\nPools\n----------------------" 3422 "--------\n"); 3423 __kmp_print_structure_thread("Thread pool: ", 3424 CCAST(kmp_info_t *, __kmp_thread_pool)); 3425 __kmp_print_structure_team("Team pool: ", 3426 CCAST(kmp_team_t *, __kmp_team_pool)); 3427 __kmp_printf("\n"); 3428 3429 // Free team list. 3430 while (list != NULL) { 3431 kmp_team_list_item_t *item = list; 3432 list = list->next; 3433 KMP_INTERNAL_FREE(item); 3434 } 3435 } 3436 3437 #endif 3438 3439 //--------------------------------------------------------------------------- 3440 // Stuff for per-thread fast random number generator 3441 // Table of primes 3442 static const unsigned __kmp_primes[] = { 3443 0x9e3779b1, 0xffe6cc59, 0x2109f6dd, 0x43977ab5, 0xba5703f5, 0xb495a877, 3444 0xe1626741, 0x79695e6b, 0xbc98c09f, 0xd5bee2b3, 0x287488f9, 0x3af18231, 3445 0x9677cd4d, 0xbe3a6929, 0xadc6a877, 0xdcf0674b, 0xbe4d6fe9, 0x5f15e201, 3446 0x99afc3fd, 0xf3f16801, 0xe222cfff, 0x24ba5fdb, 0x0620452d, 0x79f149e3, 3447 0xc8b93f49, 0x972702cd, 0xb07dd827, 0x6c97d5ed, 0x085a3d61, 0x46eb5ea7, 3448 0x3d9910ed, 0x2e687b5b, 0x29609227, 0x6eb081f1, 0x0954c4e1, 0x9d114db9, 3449 0x542acfa9, 0xb3e6bd7b, 0x0742d917, 0xe9f3ffa7, 0x54581edb, 0xf2480f45, 3450 0x0bb9288f, 0xef1affc7, 0x85fa0ca7, 0x3ccc14db, 0xe6baf34b, 0x343377f7, 3451 0x5ca19031, 0xe6d9293b, 0xf0a9f391, 0x5d2e980b, 0xfc411073, 0xc3749363, 3452 0xb892d829, 0x3549366b, 0x629750ad, 0xb98294e5, 0x892d9483, 0xc235baf3, 3453 0x3d2402a3, 0x6bdef3c9, 0xbec333cd, 0x40c9520f}; 3454 3455 //--------------------------------------------------------------------------- 3456 // __kmp_get_random: Get a random number using a linear congruential method. 3457 unsigned short __kmp_get_random(kmp_info_t *thread) { 3458 unsigned x = thread->th.th_x; 3459 unsigned short r = x >> 16; 3460 3461 thread->th.th_x = x * thread->th.th_a + 1; 3462 3463 KA_TRACE(30, ("__kmp_get_random: THREAD: %d, RETURN: %u\n", 3464 thread->th.th_info.ds.ds_tid, r)); 3465 3466 return r; 3467 } 3468 //-------------------------------------------------------- 3469 // __kmp_init_random: Initialize a random number generator 3470 void __kmp_init_random(kmp_info_t *thread) { 3471 unsigned seed = thread->th.th_info.ds.ds_tid; 3472 3473 thread->th.th_a = 3474 __kmp_primes[seed % (sizeof(__kmp_primes) / sizeof(__kmp_primes[0]))]; 3475 thread->th.th_x = (seed + 1) * thread->th.th_a + 1; 3476 KA_TRACE(30, 3477 ("__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a)); 3478 } 3479 3480 #if KMP_OS_WINDOWS 3481 /* reclaim array entries for root threads that are already dead, returns number 3482 * reclaimed */ 3483 static int __kmp_reclaim_dead_roots(void) { 3484 int i, r = 0; 3485 3486 for (i = 0; i < __kmp_threads_capacity; ++i) { 3487 if (KMP_UBER_GTID(i) && 3488 !__kmp_still_running((kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[i])) && 3489 !__kmp_root[i] 3490 ->r.r_active) { // AC: reclaim only roots died in non-active state 3491 r += __kmp_unregister_root_other_thread(i); 3492 } 3493 } 3494 return r; 3495 } 3496 #endif 3497 3498 /* This function attempts to create free entries in __kmp_threads and 3499 __kmp_root, and returns the number of free entries generated. 3500 3501 For Windows* OS static library, the first mechanism used is to reclaim array 3502 entries for root threads that are already dead. 3503 3504 On all platforms, expansion is attempted on the arrays __kmp_threads_ and 3505 __kmp_root, with appropriate update to __kmp_threads_capacity. Array 3506 capacity is increased by doubling with clipping to __kmp_tp_capacity, if 3507 threadprivate cache array has been created. Synchronization with 3508 __kmpc_threadprivate_cached is done using __kmp_tp_cached_lock. 3509 3510 After any dead root reclamation, if the clipping value allows array expansion 3511 to result in the generation of a total of nNeed free slots, the function does 3512 that expansion. If not, nothing is done beyond the possible initial root 3513 thread reclamation. 3514 3515 If any argument is negative, the behavior is undefined. */ 3516 static int __kmp_expand_threads(int nNeed) { 3517 int added = 0; 3518 int minimumRequiredCapacity; 3519 int newCapacity; 3520 kmp_info_t **newThreads; 3521 kmp_root_t **newRoot; 3522 3523 // All calls to __kmp_expand_threads should be under __kmp_forkjoin_lock, so 3524 // resizing __kmp_threads does not need additional protection if foreign 3525 // threads are present 3526 3527 #if KMP_OS_WINDOWS && !KMP_DYNAMIC_LIB 3528 /* only for Windows static library */ 3529 /* reclaim array entries for root threads that are already dead */ 3530 added = __kmp_reclaim_dead_roots(); 3531 3532 if (nNeed) { 3533 nNeed -= added; 3534 if (nNeed < 0) 3535 nNeed = 0; 3536 } 3537 #endif 3538 if (nNeed <= 0) 3539 return added; 3540 3541 // Note that __kmp_threads_capacity is not bounded by __kmp_max_nth. If 3542 // __kmp_max_nth is set to some value less than __kmp_sys_max_nth by the 3543 // user via KMP_DEVICE_THREAD_LIMIT, then __kmp_threads_capacity may become 3544 // > __kmp_max_nth in one of two ways: 3545 // 3546 // 1) The initialization thread (gtid = 0) exits. __kmp_threads[0] 3547 // may not be reused by another thread, so we may need to increase 3548 // __kmp_threads_capacity to __kmp_max_nth + 1. 3549 // 3550 // 2) New foreign root(s) are encountered. We always register new foreign 3551 // roots. This may cause a smaller # of threads to be allocated at 3552 // subsequent parallel regions, but the worker threads hang around (and 3553 // eventually go to sleep) and need slots in the __kmp_threads[] array. 3554 // 3555 // Anyway, that is the reason for moving the check to see if 3556 // __kmp_max_nth was exceeded into __kmp_reserve_threads() 3557 // instead of having it performed here. -BB 3558 3559 KMP_DEBUG_ASSERT(__kmp_sys_max_nth >= __kmp_threads_capacity); 3560 3561 /* compute expansion headroom to check if we can expand */ 3562 if (__kmp_sys_max_nth - __kmp_threads_capacity < nNeed) { 3563 /* possible expansion too small -- give up */ 3564 return added; 3565 } 3566 minimumRequiredCapacity = __kmp_threads_capacity + nNeed; 3567 3568 newCapacity = __kmp_threads_capacity; 3569 do { 3570 newCapacity = newCapacity <= (__kmp_sys_max_nth >> 1) ? (newCapacity << 1) 3571 : __kmp_sys_max_nth; 3572 } while (newCapacity < minimumRequiredCapacity); 3573 newThreads = (kmp_info_t **)__kmp_allocate( 3574 (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * newCapacity + CACHE_LINE); 3575 newRoot = 3576 (kmp_root_t **)((char *)newThreads + sizeof(kmp_info_t *) * newCapacity); 3577 KMP_MEMCPY(newThreads, __kmp_threads, 3578 __kmp_threads_capacity * sizeof(kmp_info_t *)); 3579 KMP_MEMCPY(newRoot, __kmp_root, 3580 __kmp_threads_capacity * sizeof(kmp_root_t *)); 3581 3582 kmp_info_t **temp_threads = __kmp_threads; 3583 *(kmp_info_t * *volatile *)&__kmp_threads = newThreads; 3584 *(kmp_root_t * *volatile *)&__kmp_root = newRoot; 3585 __kmp_free(temp_threads); 3586 added += newCapacity - __kmp_threads_capacity; 3587 *(volatile int *)&__kmp_threads_capacity = newCapacity; 3588 3589 if (newCapacity > __kmp_tp_capacity) { 3590 __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock); 3591 if (__kmp_tp_cached && newCapacity > __kmp_tp_capacity) { 3592 __kmp_threadprivate_resize_cache(newCapacity); 3593 } else { // increase __kmp_tp_capacity to correspond with kmp_threads size 3594 *(volatile int *)&__kmp_tp_capacity = newCapacity; 3595 } 3596 __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock); 3597 } 3598 3599 return added; 3600 } 3601 3602 /* Register the current thread as a root thread and obtain our gtid. We must 3603 have the __kmp_initz_lock held at this point. Argument TRUE only if are the 3604 thread that calls from __kmp_do_serial_initialize() */ 3605 int __kmp_register_root(int initial_thread) { 3606 kmp_info_t *root_thread; 3607 kmp_root_t *root; 3608 int gtid; 3609 int capacity; 3610 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 3611 KA_TRACE(20, ("__kmp_register_root: entered\n")); 3612 KMP_MB(); 3613 3614 /* 2007-03-02: 3615 If initial thread did not invoke OpenMP RTL yet, and this thread is not an 3616 initial one, "__kmp_all_nth >= __kmp_threads_capacity" condition does not 3617 work as expected -- it may return false (that means there is at least one 3618 empty slot in __kmp_threads array), but it is possible the only free slot 3619 is #0, which is reserved for initial thread and so cannot be used for this 3620 one. Following code workarounds this bug. 3621 3622 However, right solution seems to be not reserving slot #0 for initial 3623 thread because: 3624 (1) there is no magic in slot #0, 3625 (2) we cannot detect initial thread reliably (the first thread which does 3626 serial initialization may be not a real initial thread). 3627 */ 3628 capacity = __kmp_threads_capacity; 3629 if (!initial_thread && TCR_PTR(__kmp_threads[0]) == NULL) { 3630 --capacity; 3631 } 3632 3633 /* see if there are too many threads */ 3634 if (__kmp_all_nth >= capacity && !__kmp_expand_threads(1)) { 3635 if (__kmp_tp_cached) { 3636 __kmp_fatal(KMP_MSG(CantRegisterNewThread), 3637 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity), 3638 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null); 3639 } else { 3640 __kmp_fatal(KMP_MSG(CantRegisterNewThread), KMP_HNT(SystemLimitOnThreads), 3641 __kmp_msg_null); 3642 } 3643 } 3644 3645 /* find an available thread slot */ 3646 /* Don't reassign the zero slot since we need that to only be used by initial 3647 thread */ 3648 for (gtid = (initial_thread ? 0 : 1); TCR_PTR(__kmp_threads[gtid]) != NULL; 3649 gtid++) 3650 ; 3651 KA_TRACE(1, 3652 ("__kmp_register_root: found slot in threads array: T#%d\n", gtid)); 3653 KMP_ASSERT(gtid < __kmp_threads_capacity); 3654 3655 /* update global accounting */ 3656 __kmp_all_nth++; 3657 TCW_4(__kmp_nth, __kmp_nth + 1); 3658 3659 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low 3660 // numbers of procs, and method #2 (keyed API call) for higher numbers. 3661 if (__kmp_adjust_gtid_mode) { 3662 if (__kmp_all_nth >= __kmp_tls_gtid_min) { 3663 if (TCR_4(__kmp_gtid_mode) != 2) { 3664 TCW_4(__kmp_gtid_mode, 2); 3665 } 3666 } else { 3667 if (TCR_4(__kmp_gtid_mode) != 1) { 3668 TCW_4(__kmp_gtid_mode, 1); 3669 } 3670 } 3671 } 3672 3673 #ifdef KMP_ADJUST_BLOCKTIME 3674 /* Adjust blocktime to zero if necessary */ 3675 /* Middle initialization might not have occurred yet */ 3676 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 3677 if (__kmp_nth > __kmp_avail_proc) { 3678 __kmp_zero_bt = TRUE; 3679 } 3680 } 3681 #endif /* KMP_ADJUST_BLOCKTIME */ 3682 3683 /* setup this new hierarchy */ 3684 if (!(root = __kmp_root[gtid])) { 3685 root = __kmp_root[gtid] = (kmp_root_t *)__kmp_allocate(sizeof(kmp_root_t)); 3686 KMP_DEBUG_ASSERT(!root->r.r_root_team); 3687 } 3688 3689 #if KMP_STATS_ENABLED 3690 // Initialize stats as soon as possible (right after gtid assignment). 3691 __kmp_stats_thread_ptr = __kmp_stats_list->push_back(gtid); 3692 __kmp_stats_thread_ptr->startLife(); 3693 KMP_SET_THREAD_STATE(SERIAL_REGION); 3694 KMP_INIT_PARTITIONED_TIMERS(OMP_serial); 3695 #endif 3696 __kmp_initialize_root(root); 3697 3698 /* setup new root thread structure */ 3699 if (root->r.r_uber_thread) { 3700 root_thread = root->r.r_uber_thread; 3701 } else { 3702 root_thread = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t)); 3703 if (__kmp_storage_map) { 3704 __kmp_print_thread_storage_map(root_thread, gtid); 3705 } 3706 root_thread->th.th_info.ds.ds_gtid = gtid; 3707 #if OMPT_SUPPORT 3708 root_thread->th.ompt_thread_info.thread_data = ompt_data_none; 3709 #endif 3710 root_thread->th.th_root = root; 3711 if (__kmp_env_consistency_check) { 3712 root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid); 3713 } 3714 #if USE_FAST_MEMORY 3715 __kmp_initialize_fast_memory(root_thread); 3716 #endif /* USE_FAST_MEMORY */ 3717 3718 #if KMP_USE_BGET 3719 KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL); 3720 __kmp_initialize_bget(root_thread); 3721 #endif 3722 __kmp_init_random(root_thread); // Initialize random number generator 3723 } 3724 3725 /* setup the serial team held in reserve by the root thread */ 3726 if (!root_thread->th.th_serial_team) { 3727 kmp_internal_control_t r_icvs = __kmp_get_global_icvs(); 3728 KF_TRACE(10, ("__kmp_register_root: before serial_team\n")); 3729 root_thread->th.th_serial_team = __kmp_allocate_team( 3730 root, 1, 1, 3731 #if OMPT_SUPPORT 3732 ompt_data_none, // root parallel id 3733 #endif 3734 proc_bind_default, &r_icvs, 0 USE_NESTED_HOT_ARG(NULL)); 3735 } 3736 KMP_ASSERT(root_thread->th.th_serial_team); 3737 KF_TRACE(10, ("__kmp_register_root: after serial_team = %p\n", 3738 root_thread->th.th_serial_team)); 3739 3740 /* drop root_thread into place */ 3741 TCW_SYNC_PTR(__kmp_threads[gtid], root_thread); 3742 3743 root->r.r_root_team->t.t_threads[0] = root_thread; 3744 root->r.r_hot_team->t.t_threads[0] = root_thread; 3745 root_thread->th.th_serial_team->t.t_threads[0] = root_thread; 3746 // AC: the team created in reserve, not for execution (it is unused for now). 3747 root_thread->th.th_serial_team->t.t_serialized = 0; 3748 root->r.r_uber_thread = root_thread; 3749 3750 /* initialize the thread, get it ready to go */ 3751 __kmp_initialize_info(root_thread, root->r.r_root_team, 0, gtid); 3752 TCW_4(__kmp_init_gtid, TRUE); 3753 3754 /* prepare the master thread for get_gtid() */ 3755 __kmp_gtid_set_specific(gtid); 3756 3757 #if USE_ITT_BUILD 3758 __kmp_itt_thread_name(gtid); 3759 #endif /* USE_ITT_BUILD */ 3760 3761 #ifdef KMP_TDATA_GTID 3762 __kmp_gtid = gtid; 3763 #endif 3764 __kmp_create_worker(gtid, root_thread, __kmp_stksize); 3765 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == gtid); 3766 3767 KA_TRACE(20, ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, " 3768 "plain=%u\n", 3769 gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team), 3770 root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE, 3771 KMP_INIT_BARRIER_STATE)); 3772 { // Initialize barrier data. 3773 int b; 3774 for (b = 0; b < bs_last_barrier; ++b) { 3775 root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE; 3776 #if USE_DEBUGGER 3777 root_thread->th.th_bar[b].bb.b_worker_arrived = 0; 3778 #endif 3779 } 3780 } 3781 KMP_DEBUG_ASSERT(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived == 3782 KMP_INIT_BARRIER_STATE); 3783 3784 #if KMP_AFFINITY_SUPPORTED 3785 root_thread->th.th_current_place = KMP_PLACE_UNDEFINED; 3786 root_thread->th.th_new_place = KMP_PLACE_UNDEFINED; 3787 root_thread->th.th_first_place = KMP_PLACE_UNDEFINED; 3788 root_thread->th.th_last_place = KMP_PLACE_UNDEFINED; 3789 if (TCR_4(__kmp_init_middle)) { 3790 __kmp_affinity_set_init_mask(gtid, TRUE); 3791 } 3792 #endif /* KMP_AFFINITY_SUPPORTED */ 3793 root_thread->th.th_def_allocator = __kmp_def_allocator; 3794 root_thread->th.th_prev_level = 0; 3795 root_thread->th.th_prev_num_threads = 1; 3796 3797 kmp_cg_root_t *tmp = (kmp_cg_root_t *)__kmp_allocate(sizeof(kmp_cg_root_t)); 3798 tmp->cg_root = root_thread; 3799 tmp->cg_thread_limit = __kmp_cg_max_nth; 3800 tmp->cg_nthreads = 1; 3801 KA_TRACE(100, ("__kmp_register_root: Thread %p created node %p with" 3802 " cg_nthreads init to 1\n", 3803 root_thread, tmp)); 3804 tmp->up = NULL; 3805 root_thread->th.th_cg_roots = tmp; 3806 3807 __kmp_root_counter++; 3808 3809 #if OMPT_SUPPORT 3810 if (!initial_thread && ompt_enabled.enabled) { 3811 3812 kmp_info_t *root_thread = ompt_get_thread(); 3813 3814 ompt_set_thread_state(root_thread, ompt_state_overhead); 3815 3816 if (ompt_enabled.ompt_callback_thread_begin) { 3817 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)( 3818 ompt_thread_initial, __ompt_get_thread_data_internal()); 3819 } 3820 ompt_data_t *task_data; 3821 ompt_data_t *parallel_data; 3822 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, ¶llel_data, NULL); 3823 if (ompt_enabled.ompt_callback_implicit_task) { 3824 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 3825 ompt_scope_begin, parallel_data, task_data, 1, 1, ompt_task_initial); 3826 } 3827 3828 ompt_set_thread_state(root_thread, ompt_state_work_serial); 3829 } 3830 #endif 3831 3832 KMP_MB(); 3833 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 3834 3835 return gtid; 3836 } 3837 3838 #if KMP_NESTED_HOT_TEAMS 3839 static int __kmp_free_hot_teams(kmp_root_t *root, kmp_info_t *thr, int level, 3840 const int max_level) { 3841 int i, n, nth; 3842 kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams; 3843 if (!hot_teams || !hot_teams[level].hot_team) { 3844 return 0; 3845 } 3846 KMP_DEBUG_ASSERT(level < max_level); 3847 kmp_team_t *team = hot_teams[level].hot_team; 3848 nth = hot_teams[level].hot_team_nth; 3849 n = nth - 1; // master is not freed 3850 if (level < max_level - 1) { 3851 for (i = 0; i < nth; ++i) { 3852 kmp_info_t *th = team->t.t_threads[i]; 3853 n += __kmp_free_hot_teams(root, th, level + 1, max_level); 3854 if (i > 0 && th->th.th_hot_teams) { 3855 __kmp_free(th->th.th_hot_teams); 3856 th->th.th_hot_teams = NULL; 3857 } 3858 } 3859 } 3860 __kmp_free_team(root, team, NULL); 3861 return n; 3862 } 3863 #endif 3864 3865 // Resets a root thread and clear its root and hot teams. 3866 // Returns the number of __kmp_threads entries directly and indirectly freed. 3867 static int __kmp_reset_root(int gtid, kmp_root_t *root) { 3868 kmp_team_t *root_team = root->r.r_root_team; 3869 kmp_team_t *hot_team = root->r.r_hot_team; 3870 int n = hot_team->t.t_nproc; 3871 int i; 3872 3873 KMP_DEBUG_ASSERT(!root->r.r_active); 3874 3875 root->r.r_root_team = NULL; 3876 root->r.r_hot_team = NULL; 3877 // __kmp_free_team() does not free hot teams, so we have to clear r_hot_team 3878 // before call to __kmp_free_team(). 3879 __kmp_free_team(root, root_team USE_NESTED_HOT_ARG(NULL)); 3880 #if KMP_NESTED_HOT_TEAMS 3881 if (__kmp_hot_teams_max_level > 3882 0) { // need to free nested hot teams and their threads if any 3883 for (i = 0; i < hot_team->t.t_nproc; ++i) { 3884 kmp_info_t *th = hot_team->t.t_threads[i]; 3885 if (__kmp_hot_teams_max_level > 1) { 3886 n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level); 3887 } 3888 if (th->th.th_hot_teams) { 3889 __kmp_free(th->th.th_hot_teams); 3890 th->th.th_hot_teams = NULL; 3891 } 3892 } 3893 } 3894 #endif 3895 __kmp_free_team(root, hot_team USE_NESTED_HOT_ARG(NULL)); 3896 3897 // Before we can reap the thread, we need to make certain that all other 3898 // threads in the teams that had this root as ancestor have stopped trying to 3899 // steal tasks. 3900 if (__kmp_tasking_mode != tskm_immediate_exec) { 3901 __kmp_wait_to_unref_task_teams(); 3902 } 3903 3904 #if KMP_OS_WINDOWS 3905 /* Close Handle of root duplicated in __kmp_create_worker (tr #62919) */ 3906 KA_TRACE( 3907 10, ("__kmp_reset_root: free handle, th = %p, handle = %" KMP_UINTPTR_SPEC 3908 "\n", 3909 (LPVOID) & (root->r.r_uber_thread->th), 3910 root->r.r_uber_thread->th.th_info.ds.ds_thread)); 3911 __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread); 3912 #endif /* KMP_OS_WINDOWS */ 3913 3914 #if OMPT_SUPPORT 3915 ompt_data_t *task_data; 3916 ompt_data_t *parallel_data; 3917 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, ¶llel_data, NULL); 3918 if (ompt_enabled.ompt_callback_implicit_task) { 3919 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 3920 ompt_scope_end, parallel_data, task_data, 0, 1, ompt_task_initial); 3921 } 3922 if (ompt_enabled.ompt_callback_thread_end) { 3923 ompt_callbacks.ompt_callback(ompt_callback_thread_end)( 3924 &(root->r.r_uber_thread->th.ompt_thread_info.thread_data)); 3925 } 3926 #endif 3927 3928 TCW_4(__kmp_nth, 3929 __kmp_nth - 1); // __kmp_reap_thread will decrement __kmp_all_nth. 3930 i = root->r.r_uber_thread->th.th_cg_roots->cg_nthreads--; 3931 KA_TRACE(100, ("__kmp_reset_root: Thread %p decrement cg_nthreads on node %p" 3932 " to %d\n", 3933 root->r.r_uber_thread, root->r.r_uber_thread->th.th_cg_roots, 3934 root->r.r_uber_thread->th.th_cg_roots->cg_nthreads)); 3935 if (i == 1) { 3936 // need to free contention group structure 3937 KMP_DEBUG_ASSERT(root->r.r_uber_thread == 3938 root->r.r_uber_thread->th.th_cg_roots->cg_root); 3939 KMP_DEBUG_ASSERT(root->r.r_uber_thread->th.th_cg_roots->up == NULL); 3940 __kmp_free(root->r.r_uber_thread->th.th_cg_roots); 3941 root->r.r_uber_thread->th.th_cg_roots = NULL; 3942 } 3943 __kmp_reap_thread(root->r.r_uber_thread, 1); 3944 3945 // We canot put root thread to __kmp_thread_pool, so we have to reap it 3946 // instead of freeing. 3947 root->r.r_uber_thread = NULL; 3948 /* mark root as no longer in use */ 3949 root->r.r_begin = FALSE; 3950 3951 return n; 3952 } 3953 3954 void __kmp_unregister_root_current_thread(int gtid) { 3955 KA_TRACE(1, ("__kmp_unregister_root_current_thread: enter T#%d\n", gtid)); 3956 /* this lock should be ok, since unregister_root_current_thread is never 3957 called during an abort, only during a normal close. furthermore, if you 3958 have the forkjoin lock, you should never try to get the initz lock */ 3959 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 3960 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 3961 KC_TRACE(10, ("__kmp_unregister_root_current_thread: already finished, " 3962 "exiting T#%d\n", 3963 gtid)); 3964 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 3965 return; 3966 } 3967 kmp_root_t *root = __kmp_root[gtid]; 3968 3969 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]); 3970 KMP_ASSERT(KMP_UBER_GTID(gtid)); 3971 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root); 3972 KMP_ASSERT(root->r.r_active == FALSE); 3973 3974 KMP_MB(); 3975 3976 kmp_info_t *thread = __kmp_threads[gtid]; 3977 kmp_team_t *team = thread->th.th_team; 3978 kmp_task_team_t *task_team = thread->th.th_task_team; 3979 3980 // we need to wait for the proxy tasks before finishing the thread 3981 if (task_team != NULL && task_team->tt.tt_found_proxy_tasks) { 3982 #if OMPT_SUPPORT 3983 // the runtime is shutting down so we won't report any events 3984 thread->th.ompt_thread_info.state = ompt_state_undefined; 3985 #endif 3986 __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL)); 3987 } 3988 3989 __kmp_reset_root(gtid, root); 3990 3991 /* free up this thread slot */ 3992 __kmp_gtid_set_specific(KMP_GTID_DNE); 3993 #ifdef KMP_TDATA_GTID 3994 __kmp_gtid = KMP_GTID_DNE; 3995 #endif 3996 3997 KMP_MB(); 3998 KC_TRACE(10, 3999 ("__kmp_unregister_root_current_thread: T#%d unregistered\n", gtid)); 4000 4001 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 4002 } 4003 4004 #if KMP_OS_WINDOWS 4005 /* __kmp_forkjoin_lock must be already held 4006 Unregisters a root thread that is not the current thread. Returns the number 4007 of __kmp_threads entries freed as a result. */ 4008 static int __kmp_unregister_root_other_thread(int gtid) { 4009 kmp_root_t *root = __kmp_root[gtid]; 4010 int r; 4011 4012 KA_TRACE(1, ("__kmp_unregister_root_other_thread: enter T#%d\n", gtid)); 4013 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]); 4014 KMP_ASSERT(KMP_UBER_GTID(gtid)); 4015 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root); 4016 KMP_ASSERT(root->r.r_active == FALSE); 4017 4018 r = __kmp_reset_root(gtid, root); 4019 KC_TRACE(10, 4020 ("__kmp_unregister_root_other_thread: T#%d unregistered\n", gtid)); 4021 return r; 4022 } 4023 #endif 4024 4025 #if KMP_DEBUG 4026 void __kmp_task_info() { 4027 4028 kmp_int32 gtid = __kmp_entry_gtid(); 4029 kmp_int32 tid = __kmp_tid_from_gtid(gtid); 4030 kmp_info_t *this_thr = __kmp_threads[gtid]; 4031 kmp_team_t *steam = this_thr->th.th_serial_team; 4032 kmp_team_t *team = this_thr->th.th_team; 4033 4034 __kmp_printf( 4035 "__kmp_task_info: gtid=%d tid=%d t_thread=%p team=%p steam=%p curtask=%p " 4036 "ptask=%p\n", 4037 gtid, tid, this_thr, team, steam, this_thr->th.th_current_task, 4038 team->t.t_implicit_task_taskdata[tid].td_parent); 4039 } 4040 #endif // KMP_DEBUG 4041 4042 /* TODO optimize with one big memclr, take out what isn't needed, split 4043 responsibility to workers as much as possible, and delay initialization of 4044 features as much as possible */ 4045 static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team, 4046 int tid, int gtid) { 4047 /* this_thr->th.th_info.ds.ds_gtid is setup in 4048 kmp_allocate_thread/create_worker. 4049 this_thr->th.th_serial_team is setup in __kmp_allocate_thread */ 4050 kmp_info_t *master = team->t.t_threads[0]; 4051 KMP_DEBUG_ASSERT(this_thr != NULL); 4052 KMP_DEBUG_ASSERT(this_thr->th.th_serial_team); 4053 KMP_DEBUG_ASSERT(team); 4054 KMP_DEBUG_ASSERT(team->t.t_threads); 4055 KMP_DEBUG_ASSERT(team->t.t_dispatch); 4056 KMP_DEBUG_ASSERT(master); 4057 KMP_DEBUG_ASSERT(master->th.th_root); 4058 4059 KMP_MB(); 4060 4061 TCW_SYNC_PTR(this_thr->th.th_team, team); 4062 4063 this_thr->th.th_info.ds.ds_tid = tid; 4064 this_thr->th.th_set_nproc = 0; 4065 if (__kmp_tasking_mode != tskm_immediate_exec) 4066 // When tasking is possible, threads are not safe to reap until they are 4067 // done tasking; this will be set when tasking code is exited in wait 4068 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP; 4069 else // no tasking --> always safe to reap 4070 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP; 4071 this_thr->th.th_set_proc_bind = proc_bind_default; 4072 #if KMP_AFFINITY_SUPPORTED 4073 this_thr->th.th_new_place = this_thr->th.th_current_place; 4074 #endif 4075 this_thr->th.th_root = master->th.th_root; 4076 4077 /* setup the thread's cache of the team structure */ 4078 this_thr->th.th_team_nproc = team->t.t_nproc; 4079 this_thr->th.th_team_master = master; 4080 this_thr->th.th_team_serialized = team->t.t_serialized; 4081 TCW_PTR(this_thr->th.th_sleep_loc, NULL); 4082 4083 KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata); 4084 4085 KF_TRACE(10, ("__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n", 4086 tid, gtid, this_thr, this_thr->th.th_current_task)); 4087 4088 __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr, 4089 team, tid, TRUE); 4090 4091 KF_TRACE(10, ("__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n", 4092 tid, gtid, this_thr, this_thr->th.th_current_task)); 4093 // TODO: Initialize ICVs from parent; GEH - isn't that already done in 4094 // __kmp_initialize_team()? 4095 4096 /* TODO no worksharing in speculative threads */ 4097 this_thr->th.th_dispatch = &team->t.t_dispatch[tid]; 4098 4099 this_thr->th.th_local.this_construct = 0; 4100 4101 if (!this_thr->th.th_pri_common) { 4102 this_thr->th.th_pri_common = 4103 (struct common_table *)__kmp_allocate(sizeof(struct common_table)); 4104 if (__kmp_storage_map) { 4105 __kmp_print_storage_map_gtid( 4106 gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1, 4107 sizeof(struct common_table), "th_%d.th_pri_common\n", gtid); 4108 } 4109 this_thr->th.th_pri_head = NULL; 4110 } 4111 4112 if (this_thr != master && // Master's CG root is initialized elsewhere 4113 this_thr->th.th_cg_roots != master->th.th_cg_roots) { // CG root not set 4114 // Make new thread's CG root same as master's 4115 KMP_DEBUG_ASSERT(master->th.th_cg_roots); 4116 kmp_cg_root_t *tmp = this_thr->th.th_cg_roots; 4117 if (tmp) { 4118 // worker changes CG, need to check if old CG should be freed 4119 int i = tmp->cg_nthreads--; 4120 KA_TRACE(100, ("__kmp_initialize_info: Thread %p decrement cg_nthreads" 4121 " on node %p of thread %p to %d\n", 4122 this_thr, tmp, tmp->cg_root, tmp->cg_nthreads)); 4123 if (i == 1) { 4124 __kmp_free(tmp); // last thread left CG --> free it 4125 } 4126 } 4127 this_thr->th.th_cg_roots = master->th.th_cg_roots; 4128 // Increment new thread's CG root's counter to add the new thread 4129 this_thr->th.th_cg_roots->cg_nthreads++; 4130 KA_TRACE(100, ("__kmp_initialize_info: Thread %p increment cg_nthreads on" 4131 " node %p of thread %p to %d\n", 4132 this_thr, this_thr->th.th_cg_roots, 4133 this_thr->th.th_cg_roots->cg_root, 4134 this_thr->th.th_cg_roots->cg_nthreads)); 4135 this_thr->th.th_current_task->td_icvs.thread_limit = 4136 this_thr->th.th_cg_roots->cg_thread_limit; 4137 } 4138 4139 /* Initialize dynamic dispatch */ 4140 { 4141 volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch; 4142 // Use team max_nproc since this will never change for the team. 4143 size_t disp_size = 4144 sizeof(dispatch_private_info_t) * 4145 (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers); 4146 KD_TRACE(10, ("__kmp_initialize_info: T#%d max_nproc: %d\n", gtid, 4147 team->t.t_max_nproc)); 4148 KMP_ASSERT(dispatch); 4149 KMP_DEBUG_ASSERT(team->t.t_dispatch); 4150 KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid]); 4151 4152 dispatch->th_disp_index = 0; 4153 dispatch->th_doacross_buf_idx = 0; 4154 if (!dispatch->th_disp_buffer) { 4155 dispatch->th_disp_buffer = 4156 (dispatch_private_info_t *)__kmp_allocate(disp_size); 4157 4158 if (__kmp_storage_map) { 4159 __kmp_print_storage_map_gtid( 4160 gtid, &dispatch->th_disp_buffer[0], 4161 &dispatch->th_disp_buffer[team->t.t_max_nproc == 1 4162 ? 1 4163 : __kmp_dispatch_num_buffers], 4164 disp_size, "th_%d.th_dispatch.th_disp_buffer " 4165 "(team_%d.t_dispatch[%d].th_disp_buffer)", 4166 gtid, team->t.t_id, gtid); 4167 } 4168 } else { 4169 memset(&dispatch->th_disp_buffer[0], '\0', disp_size); 4170 } 4171 4172 dispatch->th_dispatch_pr_current = 0; 4173 dispatch->th_dispatch_sh_current = 0; 4174 4175 dispatch->th_deo_fcn = 0; /* ORDERED */ 4176 dispatch->th_dxo_fcn = 0; /* END ORDERED */ 4177 } 4178 4179 this_thr->th.th_next_pool = NULL; 4180 4181 if (!this_thr->th.th_task_state_memo_stack) { 4182 size_t i; 4183 this_thr->th.th_task_state_memo_stack = 4184 (kmp_uint8 *)__kmp_allocate(4 * sizeof(kmp_uint8)); 4185 this_thr->th.th_task_state_top = 0; 4186 this_thr->th.th_task_state_stack_sz = 4; 4187 for (i = 0; i < this_thr->th.th_task_state_stack_sz; 4188 ++i) // zero init the stack 4189 this_thr->th.th_task_state_memo_stack[i] = 0; 4190 } 4191 4192 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here); 4193 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0); 4194 4195 KMP_MB(); 4196 } 4197 4198 /* allocate a new thread for the requesting team. this is only called from 4199 within a forkjoin critical section. we will first try to get an available 4200 thread from the thread pool. if none is available, we will fork a new one 4201 assuming we are able to create a new one. this should be assured, as the 4202 caller should check on this first. */ 4203 kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team, 4204 int new_tid) { 4205 kmp_team_t *serial_team; 4206 kmp_info_t *new_thr; 4207 int new_gtid; 4208 4209 KA_TRACE(20, ("__kmp_allocate_thread: T#%d\n", __kmp_get_gtid())); 4210 KMP_DEBUG_ASSERT(root && team); 4211 #if !KMP_NESTED_HOT_TEAMS 4212 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(__kmp_get_gtid())); 4213 #endif 4214 KMP_MB(); 4215 4216 /* first, try to get one from the thread pool */ 4217 if (__kmp_thread_pool) { 4218 new_thr = CCAST(kmp_info_t *, __kmp_thread_pool); 4219 __kmp_thread_pool = (volatile kmp_info_t *)new_thr->th.th_next_pool; 4220 if (new_thr == __kmp_thread_pool_insert_pt) { 4221 __kmp_thread_pool_insert_pt = NULL; 4222 } 4223 TCW_4(new_thr->th.th_in_pool, FALSE); 4224 __kmp_suspend_initialize_thread(new_thr); 4225 __kmp_lock_suspend_mx(new_thr); 4226 if (new_thr->th.th_active_in_pool == TRUE) { 4227 KMP_DEBUG_ASSERT(new_thr->th.th_active == TRUE); 4228 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth); 4229 new_thr->th.th_active_in_pool = FALSE; 4230 } 4231 __kmp_unlock_suspend_mx(new_thr); 4232 4233 KA_TRACE(20, ("__kmp_allocate_thread: T#%d using thread T#%d\n", 4234 __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid)); 4235 KMP_ASSERT(!new_thr->th.th_team); 4236 KMP_DEBUG_ASSERT(__kmp_nth < __kmp_threads_capacity); 4237 4238 /* setup the thread structure */ 4239 __kmp_initialize_info(new_thr, team, new_tid, 4240 new_thr->th.th_info.ds.ds_gtid); 4241 KMP_DEBUG_ASSERT(new_thr->th.th_serial_team); 4242 4243 TCW_4(__kmp_nth, __kmp_nth + 1); 4244 4245 new_thr->th.th_task_state = 0; 4246 new_thr->th.th_task_state_top = 0; 4247 new_thr->th.th_task_state_stack_sz = 4; 4248 4249 #ifdef KMP_ADJUST_BLOCKTIME 4250 /* Adjust blocktime back to zero if necessary */ 4251 /* Middle initialization might not have occurred yet */ 4252 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 4253 if (__kmp_nth > __kmp_avail_proc) { 4254 __kmp_zero_bt = TRUE; 4255 } 4256 } 4257 #endif /* KMP_ADJUST_BLOCKTIME */ 4258 4259 #if KMP_DEBUG 4260 // If thread entered pool via __kmp_free_thread, wait_flag should != 4261 // KMP_BARRIER_PARENT_FLAG. 4262 int b; 4263 kmp_balign_t *balign = new_thr->th.th_bar; 4264 for (b = 0; b < bs_last_barrier; ++b) 4265 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 4266 #endif 4267 4268 KF_TRACE(10, ("__kmp_allocate_thread: T#%d using thread %p T#%d\n", 4269 __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid)); 4270 4271 KMP_MB(); 4272 return new_thr; 4273 } 4274 4275 /* no, well fork a new one */ 4276 KMP_ASSERT(__kmp_nth == __kmp_all_nth); 4277 KMP_ASSERT(__kmp_all_nth < __kmp_threads_capacity); 4278 4279 #if KMP_USE_MONITOR 4280 // If this is the first worker thread the RTL is creating, then also 4281 // launch the monitor thread. We try to do this as early as possible. 4282 if (!TCR_4(__kmp_init_monitor)) { 4283 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock); 4284 if (!TCR_4(__kmp_init_monitor)) { 4285 KF_TRACE(10, ("before __kmp_create_monitor\n")); 4286 TCW_4(__kmp_init_monitor, 1); 4287 __kmp_create_monitor(&__kmp_monitor); 4288 KF_TRACE(10, ("after __kmp_create_monitor\n")); 4289 #if KMP_OS_WINDOWS 4290 // AC: wait until monitor has started. This is a fix for CQ232808. 4291 // The reason is that if the library is loaded/unloaded in a loop with 4292 // small (parallel) work in between, then there is high probability that 4293 // monitor thread started after the library shutdown. At shutdown it is 4294 // too late to cope with the problem, because when the master is in 4295 // DllMain (process detach) the monitor has no chances to start (it is 4296 // blocked), and master has no means to inform the monitor that the 4297 // library has gone, because all the memory which the monitor can access 4298 // is going to be released/reset. 4299 while (TCR_4(__kmp_init_monitor) < 2) { 4300 KMP_YIELD(TRUE); 4301 } 4302 KF_TRACE(10, ("after monitor thread has started\n")); 4303 #endif 4304 } 4305 __kmp_release_bootstrap_lock(&__kmp_monitor_lock); 4306 } 4307 #endif 4308 4309 KMP_MB(); 4310 for (new_gtid = 1; TCR_PTR(__kmp_threads[new_gtid]) != NULL; ++new_gtid) { 4311 KMP_DEBUG_ASSERT(new_gtid < __kmp_threads_capacity); 4312 } 4313 4314 /* allocate space for it. */ 4315 new_thr = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t)); 4316 4317 TCW_SYNC_PTR(__kmp_threads[new_gtid], new_thr); 4318 4319 #if USE_ITT_BUILD && USE_ITT_NOTIFY && KMP_DEBUG 4320 // suppress race conditions detection on synchronization flags in debug mode 4321 // this helps to analyze library internals eliminating false positives 4322 __itt_suppress_mark_range( 4323 __itt_suppress_range, __itt_suppress_threading_errors, 4324 &new_thr->th.th_sleep_loc, sizeof(new_thr->th.th_sleep_loc)); 4325 __itt_suppress_mark_range( 4326 __itt_suppress_range, __itt_suppress_threading_errors, 4327 &new_thr->th.th_reap_state, sizeof(new_thr->th.th_reap_state)); 4328 #if KMP_OS_WINDOWS 4329 __itt_suppress_mark_range( 4330 __itt_suppress_range, __itt_suppress_threading_errors, 4331 &new_thr->th.th_suspend_init, sizeof(new_thr->th.th_suspend_init)); 4332 #else 4333 __itt_suppress_mark_range(__itt_suppress_range, 4334 __itt_suppress_threading_errors, 4335 &new_thr->th.th_suspend_init_count, 4336 sizeof(new_thr->th.th_suspend_init_count)); 4337 #endif 4338 // TODO: check if we need to also suppress b_arrived flags 4339 __itt_suppress_mark_range(__itt_suppress_range, 4340 __itt_suppress_threading_errors, 4341 CCAST(kmp_uint64 *, &new_thr->th.th_bar[0].bb.b_go), 4342 sizeof(new_thr->th.th_bar[0].bb.b_go)); 4343 __itt_suppress_mark_range(__itt_suppress_range, 4344 __itt_suppress_threading_errors, 4345 CCAST(kmp_uint64 *, &new_thr->th.th_bar[1].bb.b_go), 4346 sizeof(new_thr->th.th_bar[1].bb.b_go)); 4347 __itt_suppress_mark_range(__itt_suppress_range, 4348 __itt_suppress_threading_errors, 4349 CCAST(kmp_uint64 *, &new_thr->th.th_bar[2].bb.b_go), 4350 sizeof(new_thr->th.th_bar[2].bb.b_go)); 4351 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY && KMP_DEBUG */ 4352 if (__kmp_storage_map) { 4353 __kmp_print_thread_storage_map(new_thr, new_gtid); 4354 } 4355 4356 // add the reserve serialized team, initialized from the team's master thread 4357 { 4358 kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team); 4359 KF_TRACE(10, ("__kmp_allocate_thread: before th_serial/serial_team\n")); 4360 new_thr->th.th_serial_team = serial_team = 4361 (kmp_team_t *)__kmp_allocate_team(root, 1, 1, 4362 #if OMPT_SUPPORT 4363 ompt_data_none, // root parallel id 4364 #endif 4365 proc_bind_default, &r_icvs, 4366 0 USE_NESTED_HOT_ARG(NULL)); 4367 } 4368 KMP_ASSERT(serial_team); 4369 serial_team->t.t_serialized = 0; // AC: the team created in reserve, not for 4370 // execution (it is unused for now). 4371 serial_team->t.t_threads[0] = new_thr; 4372 KF_TRACE(10, 4373 ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n", 4374 new_thr)); 4375 4376 /* setup the thread structures */ 4377 __kmp_initialize_info(new_thr, team, new_tid, new_gtid); 4378 4379 #if USE_FAST_MEMORY 4380 __kmp_initialize_fast_memory(new_thr); 4381 #endif /* USE_FAST_MEMORY */ 4382 4383 #if KMP_USE_BGET 4384 KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL); 4385 __kmp_initialize_bget(new_thr); 4386 #endif 4387 4388 __kmp_init_random(new_thr); // Initialize random number generator 4389 4390 /* Initialize these only once when thread is grabbed for a team allocation */ 4391 KA_TRACE(20, 4392 ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n", 4393 __kmp_get_gtid(), KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); 4394 4395 int b; 4396 kmp_balign_t *balign = new_thr->th.th_bar; 4397 for (b = 0; b < bs_last_barrier; ++b) { 4398 balign[b].bb.b_go = KMP_INIT_BARRIER_STATE; 4399 balign[b].bb.team = NULL; 4400 balign[b].bb.wait_flag = KMP_BARRIER_NOT_WAITING; 4401 balign[b].bb.use_oncore_barrier = 0; 4402 } 4403 4404 new_thr->th.th_spin_here = FALSE; 4405 new_thr->th.th_next_waiting = 0; 4406 #if KMP_OS_UNIX 4407 new_thr->th.th_blocking = false; 4408 #endif 4409 4410 #if KMP_AFFINITY_SUPPORTED 4411 new_thr->th.th_current_place = KMP_PLACE_UNDEFINED; 4412 new_thr->th.th_new_place = KMP_PLACE_UNDEFINED; 4413 new_thr->th.th_first_place = KMP_PLACE_UNDEFINED; 4414 new_thr->th.th_last_place = KMP_PLACE_UNDEFINED; 4415 #endif 4416 new_thr->th.th_def_allocator = __kmp_def_allocator; 4417 new_thr->th.th_prev_level = 0; 4418 new_thr->th.th_prev_num_threads = 1; 4419 4420 TCW_4(new_thr->th.th_in_pool, FALSE); 4421 new_thr->th.th_active_in_pool = FALSE; 4422 TCW_4(new_thr->th.th_active, TRUE); 4423 4424 /* adjust the global counters */ 4425 __kmp_all_nth++; 4426 __kmp_nth++; 4427 4428 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low 4429 // numbers of procs, and method #2 (keyed API call) for higher numbers. 4430 if (__kmp_adjust_gtid_mode) { 4431 if (__kmp_all_nth >= __kmp_tls_gtid_min) { 4432 if (TCR_4(__kmp_gtid_mode) != 2) { 4433 TCW_4(__kmp_gtid_mode, 2); 4434 } 4435 } else { 4436 if (TCR_4(__kmp_gtid_mode) != 1) { 4437 TCW_4(__kmp_gtid_mode, 1); 4438 } 4439 } 4440 } 4441 4442 #ifdef KMP_ADJUST_BLOCKTIME 4443 /* Adjust blocktime back to zero if necessary */ 4444 /* Middle initialization might not have occurred yet */ 4445 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 4446 if (__kmp_nth > __kmp_avail_proc) { 4447 __kmp_zero_bt = TRUE; 4448 } 4449 } 4450 #endif /* KMP_ADJUST_BLOCKTIME */ 4451 4452 /* actually fork it and create the new worker thread */ 4453 KF_TRACE( 4454 10, ("__kmp_allocate_thread: before __kmp_create_worker: %p\n", new_thr)); 4455 __kmp_create_worker(new_gtid, new_thr, __kmp_stksize); 4456 KF_TRACE(10, 4457 ("__kmp_allocate_thread: after __kmp_create_worker: %p\n", new_thr)); 4458 4459 KA_TRACE(20, ("__kmp_allocate_thread: T#%d forked T#%d\n", __kmp_get_gtid(), 4460 new_gtid)); 4461 KMP_MB(); 4462 return new_thr; 4463 } 4464 4465 /* Reinitialize team for reuse. 4466 The hot team code calls this case at every fork barrier, so EPCC barrier 4467 test are extremely sensitive to changes in it, esp. writes to the team 4468 struct, which cause a cache invalidation in all threads. 4469 IF YOU TOUCH THIS ROUTINE, RUN EPCC C SYNCBENCH ON A BIG-IRON MACHINE!!! */ 4470 static void __kmp_reinitialize_team(kmp_team_t *team, 4471 kmp_internal_control_t *new_icvs, 4472 ident_t *loc) { 4473 KF_TRACE(10, ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n", 4474 team->t.t_threads[0], team)); 4475 KMP_DEBUG_ASSERT(team && new_icvs); 4476 KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc); 4477 KMP_CHECK_UPDATE(team->t.t_ident, loc); 4478 4479 KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID()); 4480 // Copy ICVs to the master thread's implicit taskdata 4481 __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE); 4482 copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs); 4483 4484 KF_TRACE(10, ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n", 4485 team->t.t_threads[0], team)); 4486 } 4487 4488 /* Initialize the team data structure. 4489 This assumes the t_threads and t_max_nproc are already set. 4490 Also, we don't touch the arguments */ 4491 static void __kmp_initialize_team(kmp_team_t *team, int new_nproc, 4492 kmp_internal_control_t *new_icvs, 4493 ident_t *loc) { 4494 KF_TRACE(10, ("__kmp_initialize_team: enter: team=%p\n", team)); 4495 4496 /* verify */ 4497 KMP_DEBUG_ASSERT(team); 4498 KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc); 4499 KMP_DEBUG_ASSERT(team->t.t_threads); 4500 KMP_MB(); 4501 4502 team->t.t_master_tid = 0; /* not needed */ 4503 /* team->t.t_master_bar; not needed */ 4504 team->t.t_serialized = new_nproc > 1 ? 0 : 1; 4505 team->t.t_nproc = new_nproc; 4506 4507 /* team->t.t_parent = NULL; TODO not needed & would mess up hot team */ 4508 team->t.t_next_pool = NULL; 4509 /* memset( team->t.t_threads, 0, sizeof(kmp_info_t*)*new_nproc ); would mess 4510 * up hot team */ 4511 4512 TCW_SYNC_PTR(team->t.t_pkfn, NULL); /* not needed */ 4513 team->t.t_invoke = NULL; /* not needed */ 4514 4515 // TODO???: team->t.t_max_active_levels = new_max_active_levels; 4516 team->t.t_sched.sched = new_icvs->sched.sched; 4517 4518 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 4519 team->t.t_fp_control_saved = FALSE; /* not needed */ 4520 team->t.t_x87_fpu_control_word = 0; /* not needed */ 4521 team->t.t_mxcsr = 0; /* not needed */ 4522 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 4523 4524 team->t.t_construct = 0; 4525 4526 team->t.t_ordered.dt.t_value = 0; 4527 team->t.t_master_active = FALSE; 4528 4529 #ifdef KMP_DEBUG 4530 team->t.t_copypriv_data = NULL; /* not necessary, but nice for debugging */ 4531 #endif 4532 #if KMP_OS_WINDOWS 4533 team->t.t_copyin_counter = 0; /* for barrier-free copyin implementation */ 4534 #endif 4535 4536 team->t.t_control_stack_top = NULL; 4537 4538 __kmp_reinitialize_team(team, new_icvs, loc); 4539 4540 KMP_MB(); 4541 KF_TRACE(10, ("__kmp_initialize_team: exit: team=%p\n", team)); 4542 } 4543 4544 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED 4545 /* Sets full mask for thread and returns old mask, no changes to structures. */ 4546 static void 4547 __kmp_set_thread_affinity_mask_full_tmp(kmp_affin_mask_t *old_mask) { 4548 if (KMP_AFFINITY_CAPABLE()) { 4549 int status; 4550 if (old_mask != NULL) { 4551 status = __kmp_get_system_affinity(old_mask, TRUE); 4552 int error = errno; 4553 if (status != 0) { 4554 __kmp_fatal(KMP_MSG(ChangeThreadAffMaskError), KMP_ERR(error), 4555 __kmp_msg_null); 4556 } 4557 } 4558 __kmp_set_system_affinity(__kmp_affin_fullMask, TRUE); 4559 } 4560 } 4561 #endif 4562 4563 #if KMP_AFFINITY_SUPPORTED 4564 4565 // __kmp_partition_places() is the heart of the OpenMP 4.0 affinity mechanism. 4566 // It calculates the worker + master thread's partition based upon the parent 4567 // thread's partition, and binds each worker to a thread in their partition. 4568 // The master thread's partition should already include its current binding. 4569 static void __kmp_partition_places(kmp_team_t *team, int update_master_only) { 4570 // Copy the master thread's place partition to the team struct 4571 kmp_info_t *master_th = team->t.t_threads[0]; 4572 KMP_DEBUG_ASSERT(master_th != NULL); 4573 kmp_proc_bind_t proc_bind = team->t.t_proc_bind; 4574 int first_place = master_th->th.th_first_place; 4575 int last_place = master_th->th.th_last_place; 4576 int masters_place = master_th->th.th_current_place; 4577 team->t.t_first_place = first_place; 4578 team->t.t_last_place = last_place; 4579 4580 KA_TRACE(20, ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) " 4581 "bound to place %d partition = [%d,%d]\n", 4582 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]), 4583 team->t.t_id, masters_place, first_place, last_place)); 4584 4585 switch (proc_bind) { 4586 4587 case proc_bind_default: 4588 // serial teams might have the proc_bind policy set to proc_bind_default. It 4589 // doesn't matter, as we don't rebind master thread for any proc_bind policy 4590 KMP_DEBUG_ASSERT(team->t.t_nproc == 1); 4591 break; 4592 4593 case proc_bind_master: { 4594 int f; 4595 int n_th = team->t.t_nproc; 4596 for (f = 1; f < n_th; f++) { 4597 kmp_info_t *th = team->t.t_threads[f]; 4598 KMP_DEBUG_ASSERT(th != NULL); 4599 th->th.th_first_place = first_place; 4600 th->th.th_last_place = last_place; 4601 th->th.th_new_place = masters_place; 4602 if (__kmp_display_affinity && masters_place != th->th.th_current_place && 4603 team->t.t_display_affinity != 1) { 4604 team->t.t_display_affinity = 1; 4605 } 4606 4607 KA_TRACE(100, ("__kmp_partition_places: master: T#%d(%d:%d) place %d " 4608 "partition = [%d,%d]\n", 4609 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, 4610 f, masters_place, first_place, last_place)); 4611 } 4612 } break; 4613 4614 case proc_bind_close: { 4615 int f; 4616 int n_th = team->t.t_nproc; 4617 int n_places; 4618 if (first_place <= last_place) { 4619 n_places = last_place - first_place + 1; 4620 } else { 4621 n_places = __kmp_affinity_num_masks - first_place + last_place + 1; 4622 } 4623 if (n_th <= n_places) { 4624 int place = masters_place; 4625 for (f = 1; f < n_th; f++) { 4626 kmp_info_t *th = team->t.t_threads[f]; 4627 KMP_DEBUG_ASSERT(th != NULL); 4628 4629 if (place == last_place) { 4630 place = first_place; 4631 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4632 place = 0; 4633 } else { 4634 place++; 4635 } 4636 th->th.th_first_place = first_place; 4637 th->th.th_last_place = last_place; 4638 th->th.th_new_place = place; 4639 if (__kmp_display_affinity && place != th->th.th_current_place && 4640 team->t.t_display_affinity != 1) { 4641 team->t.t_display_affinity = 1; 4642 } 4643 4644 KA_TRACE(100, ("__kmp_partition_places: close: T#%d(%d:%d) place %d " 4645 "partition = [%d,%d]\n", 4646 __kmp_gtid_from_thread(team->t.t_threads[f]), 4647 team->t.t_id, f, place, first_place, last_place)); 4648 } 4649 } else { 4650 int S, rem, gap, s_count; 4651 S = n_th / n_places; 4652 s_count = 0; 4653 rem = n_th - (S * n_places); 4654 gap = rem > 0 ? n_places / rem : n_places; 4655 int place = masters_place; 4656 int gap_ct = gap; 4657 for (f = 0; f < n_th; f++) { 4658 kmp_info_t *th = team->t.t_threads[f]; 4659 KMP_DEBUG_ASSERT(th != NULL); 4660 4661 th->th.th_first_place = first_place; 4662 th->th.th_last_place = last_place; 4663 th->th.th_new_place = place; 4664 if (__kmp_display_affinity && place != th->th.th_current_place && 4665 team->t.t_display_affinity != 1) { 4666 team->t.t_display_affinity = 1; 4667 } 4668 s_count++; 4669 4670 if ((s_count == S) && rem && (gap_ct == gap)) { 4671 // do nothing, add an extra thread to place on next iteration 4672 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) { 4673 // we added an extra thread to this place; move to next place 4674 if (place == last_place) { 4675 place = first_place; 4676 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4677 place = 0; 4678 } else { 4679 place++; 4680 } 4681 s_count = 0; 4682 gap_ct = 1; 4683 rem--; 4684 } else if (s_count == S) { // place full; don't add extra 4685 if (place == last_place) { 4686 place = first_place; 4687 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4688 place = 0; 4689 } else { 4690 place++; 4691 } 4692 gap_ct++; 4693 s_count = 0; 4694 } 4695 4696 KA_TRACE(100, 4697 ("__kmp_partition_places: close: T#%d(%d:%d) place %d " 4698 "partition = [%d,%d]\n", 4699 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f, 4700 th->th.th_new_place, first_place, last_place)); 4701 } 4702 KMP_DEBUG_ASSERT(place == masters_place); 4703 } 4704 } break; 4705 4706 case proc_bind_spread: { 4707 int f; 4708 int n_th = team->t.t_nproc; 4709 int n_places; 4710 int thidx; 4711 if (first_place <= last_place) { 4712 n_places = last_place - first_place + 1; 4713 } else { 4714 n_places = __kmp_affinity_num_masks - first_place + last_place + 1; 4715 } 4716 if (n_th <= n_places) { 4717 int place = -1; 4718 4719 if (n_places != static_cast<int>(__kmp_affinity_num_masks)) { 4720 int S = n_places / n_th; 4721 int s_count, rem, gap, gap_ct; 4722 4723 place = masters_place; 4724 rem = n_places - n_th * S; 4725 gap = rem ? n_th / rem : 1; 4726 gap_ct = gap; 4727 thidx = n_th; 4728 if (update_master_only == 1) 4729 thidx = 1; 4730 for (f = 0; f < thidx; f++) { 4731 kmp_info_t *th = team->t.t_threads[f]; 4732 KMP_DEBUG_ASSERT(th != NULL); 4733 4734 th->th.th_first_place = place; 4735 th->th.th_new_place = place; 4736 if (__kmp_display_affinity && place != th->th.th_current_place && 4737 team->t.t_display_affinity != 1) { 4738 team->t.t_display_affinity = 1; 4739 } 4740 s_count = 1; 4741 while (s_count < S) { 4742 if (place == last_place) { 4743 place = first_place; 4744 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4745 place = 0; 4746 } else { 4747 place++; 4748 } 4749 s_count++; 4750 } 4751 if (rem && (gap_ct == gap)) { 4752 if (place == last_place) { 4753 place = first_place; 4754 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4755 place = 0; 4756 } else { 4757 place++; 4758 } 4759 rem--; 4760 gap_ct = 0; 4761 } 4762 th->th.th_last_place = place; 4763 gap_ct++; 4764 4765 if (place == last_place) { 4766 place = first_place; 4767 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4768 place = 0; 4769 } else { 4770 place++; 4771 } 4772 4773 KA_TRACE(100, 4774 ("__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4775 "partition = [%d,%d], __kmp_affinity_num_masks: %u\n", 4776 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, 4777 f, th->th.th_new_place, th->th.th_first_place, 4778 th->th.th_last_place, __kmp_affinity_num_masks)); 4779 } 4780 } else { 4781 /* Having uniform space of available computation places I can create 4782 T partitions of round(P/T) size and put threads into the first 4783 place of each partition. */ 4784 double current = static_cast<double>(masters_place); 4785 double spacing = 4786 (static_cast<double>(n_places + 1) / static_cast<double>(n_th)); 4787 int first, last; 4788 kmp_info_t *th; 4789 4790 thidx = n_th + 1; 4791 if (update_master_only == 1) 4792 thidx = 1; 4793 for (f = 0; f < thidx; f++) { 4794 first = static_cast<int>(current); 4795 last = static_cast<int>(current + spacing) - 1; 4796 KMP_DEBUG_ASSERT(last >= first); 4797 if (first >= n_places) { 4798 if (masters_place) { 4799 first -= n_places; 4800 last -= n_places; 4801 if (first == (masters_place + 1)) { 4802 KMP_DEBUG_ASSERT(f == n_th); 4803 first--; 4804 } 4805 if (last == masters_place) { 4806 KMP_DEBUG_ASSERT(f == (n_th - 1)); 4807 last--; 4808 } 4809 } else { 4810 KMP_DEBUG_ASSERT(f == n_th); 4811 first = 0; 4812 last = 0; 4813 } 4814 } 4815 if (last >= n_places) { 4816 last = (n_places - 1); 4817 } 4818 place = first; 4819 current += spacing; 4820 if (f < n_th) { 4821 KMP_DEBUG_ASSERT(0 <= first); 4822 KMP_DEBUG_ASSERT(n_places > first); 4823 KMP_DEBUG_ASSERT(0 <= last); 4824 KMP_DEBUG_ASSERT(n_places > last); 4825 KMP_DEBUG_ASSERT(last_place >= first_place); 4826 th = team->t.t_threads[f]; 4827 KMP_DEBUG_ASSERT(th); 4828 th->th.th_first_place = first; 4829 th->th.th_new_place = place; 4830 th->th.th_last_place = last; 4831 if (__kmp_display_affinity && place != th->th.th_current_place && 4832 team->t.t_display_affinity != 1) { 4833 team->t.t_display_affinity = 1; 4834 } 4835 KA_TRACE(100, 4836 ("__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4837 "partition = [%d,%d], spacing = %.4f\n", 4838 __kmp_gtid_from_thread(team->t.t_threads[f]), 4839 team->t.t_id, f, th->th.th_new_place, 4840 th->th.th_first_place, th->th.th_last_place, spacing)); 4841 } 4842 } 4843 } 4844 KMP_DEBUG_ASSERT(update_master_only || place == masters_place); 4845 } else { 4846 int S, rem, gap, s_count; 4847 S = n_th / n_places; 4848 s_count = 0; 4849 rem = n_th - (S * n_places); 4850 gap = rem > 0 ? n_places / rem : n_places; 4851 int place = masters_place; 4852 int gap_ct = gap; 4853 thidx = n_th; 4854 if (update_master_only == 1) 4855 thidx = 1; 4856 for (f = 0; f < thidx; f++) { 4857 kmp_info_t *th = team->t.t_threads[f]; 4858 KMP_DEBUG_ASSERT(th != NULL); 4859 4860 th->th.th_first_place = place; 4861 th->th.th_last_place = place; 4862 th->th.th_new_place = place; 4863 if (__kmp_display_affinity && place != th->th.th_current_place && 4864 team->t.t_display_affinity != 1) { 4865 team->t.t_display_affinity = 1; 4866 } 4867 s_count++; 4868 4869 if ((s_count == S) && rem && (gap_ct == gap)) { 4870 // do nothing, add an extra thread to place on next iteration 4871 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) { 4872 // we added an extra thread to this place; move on to next place 4873 if (place == last_place) { 4874 place = first_place; 4875 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4876 place = 0; 4877 } else { 4878 place++; 4879 } 4880 s_count = 0; 4881 gap_ct = 1; 4882 rem--; 4883 } else if (s_count == S) { // place is full; don't add extra thread 4884 if (place == last_place) { 4885 place = first_place; 4886 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4887 place = 0; 4888 } else { 4889 place++; 4890 } 4891 gap_ct++; 4892 s_count = 0; 4893 } 4894 4895 KA_TRACE(100, ("__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4896 "partition = [%d,%d]\n", 4897 __kmp_gtid_from_thread(team->t.t_threads[f]), 4898 team->t.t_id, f, th->th.th_new_place, 4899 th->th.th_first_place, th->th.th_last_place)); 4900 } 4901 KMP_DEBUG_ASSERT(update_master_only || place == masters_place); 4902 } 4903 } break; 4904 4905 default: 4906 break; 4907 } 4908 4909 KA_TRACE(20, ("__kmp_partition_places: exit T#%d\n", team->t.t_id)); 4910 } 4911 4912 #endif // KMP_AFFINITY_SUPPORTED 4913 4914 /* allocate a new team data structure to use. take one off of the free pool if 4915 available */ 4916 kmp_team_t * 4917 __kmp_allocate_team(kmp_root_t *root, int new_nproc, int max_nproc, 4918 #if OMPT_SUPPORT 4919 ompt_data_t ompt_parallel_data, 4920 #endif 4921 kmp_proc_bind_t new_proc_bind, 4922 kmp_internal_control_t *new_icvs, 4923 int argc USE_NESTED_HOT_ARG(kmp_info_t *master)) { 4924 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_allocate_team); 4925 int f; 4926 kmp_team_t *team; 4927 int use_hot_team = !root->r.r_active; 4928 int level = 0; 4929 4930 KA_TRACE(20, ("__kmp_allocate_team: called\n")); 4931 KMP_DEBUG_ASSERT(new_nproc >= 1 && argc >= 0); 4932 KMP_DEBUG_ASSERT(max_nproc >= new_nproc); 4933 KMP_MB(); 4934 4935 #if KMP_NESTED_HOT_TEAMS 4936 kmp_hot_team_ptr_t *hot_teams; 4937 if (master) { 4938 team = master->th.th_team; 4939 level = team->t.t_active_level; 4940 if (master->th.th_teams_microtask) { // in teams construct? 4941 if (master->th.th_teams_size.nteams > 1 && 4942 ( // #teams > 1 4943 team->t.t_pkfn == 4944 (microtask_t)__kmp_teams_master || // inner fork of the teams 4945 master->th.th_teams_level < 4946 team->t.t_level)) { // or nested parallel inside the teams 4947 ++level; // not increment if #teams==1, or for outer fork of the teams; 4948 // increment otherwise 4949 } 4950 } 4951 hot_teams = master->th.th_hot_teams; 4952 if (level < __kmp_hot_teams_max_level && hot_teams && 4953 hot_teams[level].hot_team) { 4954 // hot team has already been allocated for given level 4955 use_hot_team = 1; 4956 } else { 4957 use_hot_team = 0; 4958 } 4959 } else { 4960 // check we won't access uninitialized hot_teams, just in case 4961 KMP_DEBUG_ASSERT(new_nproc == 1); 4962 } 4963 #endif 4964 // Optimization to use a "hot" team 4965 if (use_hot_team && new_nproc > 1) { 4966 KMP_DEBUG_ASSERT(new_nproc <= max_nproc); 4967 #if KMP_NESTED_HOT_TEAMS 4968 team = hot_teams[level].hot_team; 4969 #else 4970 team = root->r.r_hot_team; 4971 #endif 4972 #if KMP_DEBUG 4973 if (__kmp_tasking_mode != tskm_immediate_exec) { 4974 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p " 4975 "task_team[1] = %p before reinit\n", 4976 team->t.t_task_team[0], team->t.t_task_team[1])); 4977 } 4978 #endif 4979 4980 // Has the number of threads changed? 4981 /* Let's assume the most common case is that the number of threads is 4982 unchanged, and put that case first. */ 4983 if (team->t.t_nproc == new_nproc) { // Check changes in number of threads 4984 KA_TRACE(20, ("__kmp_allocate_team: reusing hot team\n")); 4985 // This case can mean that omp_set_num_threads() was called and the hot 4986 // team size was already reduced, so we check the special flag 4987 if (team->t.t_size_changed == -1) { 4988 team->t.t_size_changed = 1; 4989 } else { 4990 KMP_CHECK_UPDATE(team->t.t_size_changed, 0); 4991 } 4992 4993 // TODO???: team->t.t_max_active_levels = new_max_active_levels; 4994 kmp_r_sched_t new_sched = new_icvs->sched; 4995 // set master's schedule as new run-time schedule 4996 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched); 4997 4998 __kmp_reinitialize_team(team, new_icvs, 4999 root->r.r_uber_thread->th.th_ident); 5000 5001 KF_TRACE(10, ("__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n", 0, 5002 team->t.t_threads[0], team)); 5003 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0); 5004 5005 #if KMP_AFFINITY_SUPPORTED 5006 if ((team->t.t_size_changed == 0) && 5007 (team->t.t_proc_bind == new_proc_bind)) { 5008 if (new_proc_bind == proc_bind_spread) { 5009 __kmp_partition_places( 5010 team, 1); // add flag to update only master for spread 5011 } 5012 KA_TRACE(200, ("__kmp_allocate_team: reusing hot team #%d bindings: " 5013 "proc_bind = %d, partition = [%d,%d]\n", 5014 team->t.t_id, new_proc_bind, team->t.t_first_place, 5015 team->t.t_last_place)); 5016 } else { 5017 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); 5018 __kmp_partition_places(team); 5019 } 5020 #else 5021 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); 5022 #endif /* KMP_AFFINITY_SUPPORTED */ 5023 } else if (team->t.t_nproc > new_nproc) { 5024 KA_TRACE(20, 5025 ("__kmp_allocate_team: decreasing hot team thread count to %d\n", 5026 new_nproc)); 5027 5028 team->t.t_size_changed = 1; 5029 #if KMP_NESTED_HOT_TEAMS 5030 if (__kmp_hot_teams_mode == 0) { 5031 // AC: saved number of threads should correspond to team's value in this 5032 // mode, can be bigger in mode 1, when hot team has threads in reserve 5033 KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc); 5034 hot_teams[level].hot_team_nth = new_nproc; 5035 #endif // KMP_NESTED_HOT_TEAMS 5036 /* release the extra threads we don't need any more */ 5037 for (f = new_nproc; f < team->t.t_nproc; f++) { 5038 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 5039 if (__kmp_tasking_mode != tskm_immediate_exec) { 5040 // When decreasing team size, threads no longer in the team should 5041 // unref task team. 5042 team->t.t_threads[f]->th.th_task_team = NULL; 5043 } 5044 __kmp_free_thread(team->t.t_threads[f]); 5045 team->t.t_threads[f] = NULL; 5046 } 5047 #if KMP_NESTED_HOT_TEAMS 5048 } // (__kmp_hot_teams_mode == 0) 5049 else { 5050 // When keeping extra threads in team, switch threads to wait on own 5051 // b_go flag 5052 for (f = new_nproc; f < team->t.t_nproc; ++f) { 5053 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 5054 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar; 5055 for (int b = 0; b < bs_last_barrier; ++b) { 5056 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG) { 5057 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG; 5058 } 5059 KMP_CHECK_UPDATE(balign[b].bb.leaf_kids, 0); 5060 } 5061 } 5062 } 5063 #endif // KMP_NESTED_HOT_TEAMS 5064 team->t.t_nproc = new_nproc; 5065 // TODO???: team->t.t_max_active_levels = new_max_active_levels; 5066 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_icvs->sched.sched); 5067 __kmp_reinitialize_team(team, new_icvs, 5068 root->r.r_uber_thread->th.th_ident); 5069 5070 // Update remaining threads 5071 for (f = 0; f < new_nproc; ++f) { 5072 team->t.t_threads[f]->th.th_team_nproc = new_nproc; 5073 } 5074 5075 // restore the current task state of the master thread: should be the 5076 // implicit task 5077 KF_TRACE(10, ("__kmp_allocate_team: T#%d, this_thread=%p team=%p\n", 0, 5078 team->t.t_threads[0], team)); 5079 5080 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0); 5081 5082 #ifdef KMP_DEBUG 5083 for (f = 0; f < team->t.t_nproc; f++) { 5084 KMP_DEBUG_ASSERT(team->t.t_threads[f] && 5085 team->t.t_threads[f]->th.th_team_nproc == 5086 team->t.t_nproc); 5087 } 5088 #endif 5089 5090 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); 5091 #if KMP_AFFINITY_SUPPORTED 5092 __kmp_partition_places(team); 5093 #endif 5094 } else { // team->t.t_nproc < new_nproc 5095 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED 5096 kmp_affin_mask_t *old_mask; 5097 if (KMP_AFFINITY_CAPABLE()) { 5098 KMP_CPU_ALLOC(old_mask); 5099 } 5100 #endif 5101 5102 KA_TRACE(20, 5103 ("__kmp_allocate_team: increasing hot team thread count to %d\n", 5104 new_nproc)); 5105 5106 team->t.t_size_changed = 1; 5107 5108 #if KMP_NESTED_HOT_TEAMS 5109 int avail_threads = hot_teams[level].hot_team_nth; 5110 if (new_nproc < avail_threads) 5111 avail_threads = new_nproc; 5112 kmp_info_t **other_threads = team->t.t_threads; 5113 for (f = team->t.t_nproc; f < avail_threads; ++f) { 5114 // Adjust barrier data of reserved threads (if any) of the team 5115 // Other data will be set in __kmp_initialize_info() below. 5116 int b; 5117 kmp_balign_t *balign = other_threads[f]->th.th_bar; 5118 for (b = 0; b < bs_last_barrier; ++b) { 5119 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 5120 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 5121 #if USE_DEBUGGER 5122 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 5123 #endif 5124 } 5125 } 5126 if (hot_teams[level].hot_team_nth >= new_nproc) { 5127 // we have all needed threads in reserve, no need to allocate any 5128 // this only possible in mode 1, cannot have reserved threads in mode 0 5129 KMP_DEBUG_ASSERT(__kmp_hot_teams_mode == 1); 5130 team->t.t_nproc = new_nproc; // just get reserved threads involved 5131 } else { 5132 // we may have some threads in reserve, but not enough 5133 team->t.t_nproc = 5134 hot_teams[level] 5135 .hot_team_nth; // get reserved threads involved if any 5136 hot_teams[level].hot_team_nth = new_nproc; // adjust hot team max size 5137 #endif // KMP_NESTED_HOT_TEAMS 5138 if (team->t.t_max_nproc < new_nproc) { 5139 /* reallocate larger arrays */ 5140 __kmp_reallocate_team_arrays(team, new_nproc); 5141 __kmp_reinitialize_team(team, new_icvs, NULL); 5142 } 5143 5144 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED 5145 /* Temporarily set full mask for master thread before creation of 5146 workers. The reason is that workers inherit the affinity from master, 5147 so if a lot of workers are created on the single core quickly, they 5148 don't get a chance to set their own affinity for a long time. */ 5149 __kmp_set_thread_affinity_mask_full_tmp(old_mask); 5150 #endif 5151 5152 /* allocate new threads for the hot team */ 5153 for (f = team->t.t_nproc; f < new_nproc; f++) { 5154 kmp_info_t *new_worker = __kmp_allocate_thread(root, team, f); 5155 KMP_DEBUG_ASSERT(new_worker); 5156 team->t.t_threads[f] = new_worker; 5157 5158 KA_TRACE(20, 5159 ("__kmp_allocate_team: team %d init T#%d arrived: " 5160 "join=%llu, plain=%llu\n", 5161 team->t.t_id, __kmp_gtid_from_tid(f, team), team->t.t_id, f, 5162 team->t.t_bar[bs_forkjoin_barrier].b_arrived, 5163 team->t.t_bar[bs_plain_barrier].b_arrived)); 5164 5165 { // Initialize barrier data for new threads. 5166 int b; 5167 kmp_balign_t *balign = new_worker->th.th_bar; 5168 for (b = 0; b < bs_last_barrier; ++b) { 5169 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 5170 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != 5171 KMP_BARRIER_PARENT_FLAG); 5172 #if USE_DEBUGGER 5173 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 5174 #endif 5175 } 5176 } 5177 } 5178 5179 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED 5180 if (KMP_AFFINITY_CAPABLE()) { 5181 /* Restore initial master thread's affinity mask */ 5182 __kmp_set_system_affinity(old_mask, TRUE); 5183 KMP_CPU_FREE(old_mask); 5184 } 5185 #endif 5186 #if KMP_NESTED_HOT_TEAMS 5187 } // end of check of t_nproc vs. new_nproc vs. hot_team_nth 5188 #endif // KMP_NESTED_HOT_TEAMS 5189 /* make sure everyone is syncronized */ 5190 int old_nproc = team->t.t_nproc; // save old value and use to update only 5191 // new threads below 5192 __kmp_initialize_team(team, new_nproc, new_icvs, 5193 root->r.r_uber_thread->th.th_ident); 5194 5195 /* reinitialize the threads */ 5196 KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc); 5197 for (f = 0; f < team->t.t_nproc; ++f) 5198 __kmp_initialize_info(team->t.t_threads[f], team, f, 5199 __kmp_gtid_from_tid(f, team)); 5200 5201 if (level) { // set th_task_state for new threads in nested hot team 5202 // __kmp_initialize_info() no longer zeroes th_task_state, so we should 5203 // only need to set the th_task_state for the new threads. th_task_state 5204 // for master thread will not be accurate until after this in 5205 // __kmp_fork_call(), so we look to the master's memo_stack to get the 5206 // correct value. 5207 for (f = old_nproc; f < team->t.t_nproc; ++f) 5208 team->t.t_threads[f]->th.th_task_state = 5209 team->t.t_threads[0]->th.th_task_state_memo_stack[level]; 5210 } else { // set th_task_state for new threads in non-nested hot team 5211 int old_state = 5212 team->t.t_threads[0]->th.th_task_state; // copy master's state 5213 for (f = old_nproc; f < team->t.t_nproc; ++f) 5214 team->t.t_threads[f]->th.th_task_state = old_state; 5215 } 5216 5217 #ifdef KMP_DEBUG 5218 for (f = 0; f < team->t.t_nproc; ++f) { 5219 KMP_DEBUG_ASSERT(team->t.t_threads[f] && 5220 team->t.t_threads[f]->th.th_team_nproc == 5221 team->t.t_nproc); 5222 } 5223 #endif 5224 5225 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); 5226 #if KMP_AFFINITY_SUPPORTED 5227 __kmp_partition_places(team); 5228 #endif 5229 } // Check changes in number of threads 5230 5231 kmp_info_t *master = team->t.t_threads[0]; 5232 if (master->th.th_teams_microtask) { 5233 for (f = 1; f < new_nproc; ++f) { 5234 // propagate teams construct specific info to workers 5235 kmp_info_t *thr = team->t.t_threads[f]; 5236 thr->th.th_teams_microtask = master->th.th_teams_microtask; 5237 thr->th.th_teams_level = master->th.th_teams_level; 5238 thr->th.th_teams_size = master->th.th_teams_size; 5239 } 5240 } 5241 #if KMP_NESTED_HOT_TEAMS 5242 if (level) { 5243 // Sync barrier state for nested hot teams, not needed for outermost hot 5244 // team. 5245 for (f = 1; f < new_nproc; ++f) { 5246 kmp_info_t *thr = team->t.t_threads[f]; 5247 int b; 5248 kmp_balign_t *balign = thr->th.th_bar; 5249 for (b = 0; b < bs_last_barrier; ++b) { 5250 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 5251 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 5252 #if USE_DEBUGGER 5253 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 5254 #endif 5255 } 5256 } 5257 } 5258 #endif // KMP_NESTED_HOT_TEAMS 5259 5260 /* reallocate space for arguments if necessary */ 5261 __kmp_alloc_argv_entries(argc, team, TRUE); 5262 KMP_CHECK_UPDATE(team->t.t_argc, argc); 5263 // The hot team re-uses the previous task team, 5264 // if untouched during the previous release->gather phase. 5265 5266 KF_TRACE(10, (" hot_team = %p\n", team)); 5267 5268 #if KMP_DEBUG 5269 if (__kmp_tasking_mode != tskm_immediate_exec) { 5270 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p " 5271 "task_team[1] = %p after reinit\n", 5272 team->t.t_task_team[0], team->t.t_task_team[1])); 5273 } 5274 #endif 5275 5276 #if OMPT_SUPPORT 5277 __ompt_team_assign_id(team, ompt_parallel_data); 5278 #endif 5279 5280 KMP_MB(); 5281 5282 return team; 5283 } 5284 5285 /* next, let's try to take one from the team pool */ 5286 KMP_MB(); 5287 for (team = CCAST(kmp_team_t *, __kmp_team_pool); (team);) { 5288 /* TODO: consider resizing undersized teams instead of reaping them, now 5289 that we have a resizing mechanism */ 5290 if (team->t.t_max_nproc >= max_nproc) { 5291 /* take this team from the team pool */ 5292 __kmp_team_pool = team->t.t_next_pool; 5293 5294 /* setup the team for fresh use */ 5295 __kmp_initialize_team(team, new_nproc, new_icvs, NULL); 5296 5297 KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and " 5298 "task_team[1] %p to NULL\n", 5299 &team->t.t_task_team[0], &team->t.t_task_team[1])); 5300 team->t.t_task_team[0] = NULL; 5301 team->t.t_task_team[1] = NULL; 5302 5303 /* reallocate space for arguments if necessary */ 5304 __kmp_alloc_argv_entries(argc, team, TRUE); 5305 KMP_CHECK_UPDATE(team->t.t_argc, argc); 5306 5307 KA_TRACE( 5308 20, ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n", 5309 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); 5310 { // Initialize barrier data. 5311 int b; 5312 for (b = 0; b < bs_last_barrier; ++b) { 5313 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE; 5314 #if USE_DEBUGGER 5315 team->t.t_bar[b].b_master_arrived = 0; 5316 team->t.t_bar[b].b_team_arrived = 0; 5317 #endif 5318 } 5319 } 5320 5321 team->t.t_proc_bind = new_proc_bind; 5322 5323 KA_TRACE(20, ("__kmp_allocate_team: using team from pool %d.\n", 5324 team->t.t_id)); 5325 5326 #if OMPT_SUPPORT 5327 __ompt_team_assign_id(team, ompt_parallel_data); 5328 #endif 5329 5330 KMP_MB(); 5331 5332 return team; 5333 } 5334 5335 /* reap team if it is too small, then loop back and check the next one */ 5336 // not sure if this is wise, but, will be redone during the hot-teams 5337 // rewrite. 5338 /* TODO: Use technique to find the right size hot-team, don't reap them */ 5339 team = __kmp_reap_team(team); 5340 __kmp_team_pool = team; 5341 } 5342 5343 /* nothing available in the pool, no matter, make a new team! */ 5344 KMP_MB(); 5345 team = (kmp_team_t *)__kmp_allocate(sizeof(kmp_team_t)); 5346 5347 /* and set it up */ 5348 team->t.t_max_nproc = max_nproc; 5349 /* NOTE well, for some reason allocating one big buffer and dividing it up 5350 seems to really hurt performance a lot on the P4, so, let's not use this */ 5351 __kmp_allocate_team_arrays(team, max_nproc); 5352 5353 KA_TRACE(20, ("__kmp_allocate_team: making a new team\n")); 5354 __kmp_initialize_team(team, new_nproc, new_icvs, NULL); 5355 5356 KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and task_team[1] " 5357 "%p to NULL\n", 5358 &team->t.t_task_team[0], &team->t.t_task_team[1])); 5359 team->t.t_task_team[0] = NULL; // to be removed, as __kmp_allocate zeroes 5360 // memory, no need to duplicate 5361 team->t.t_task_team[1] = NULL; // to be removed, as __kmp_allocate zeroes 5362 // memory, no need to duplicate 5363 5364 if (__kmp_storage_map) { 5365 __kmp_print_team_storage_map("team", team, team->t.t_id, new_nproc); 5366 } 5367 5368 /* allocate space for arguments */ 5369 __kmp_alloc_argv_entries(argc, team, FALSE); 5370 team->t.t_argc = argc; 5371 5372 KA_TRACE(20, 5373 ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n", 5374 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); 5375 { // Initialize barrier data. 5376 int b; 5377 for (b = 0; b < bs_last_barrier; ++b) { 5378 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE; 5379 #if USE_DEBUGGER 5380 team->t.t_bar[b].b_master_arrived = 0; 5381 team->t.t_bar[b].b_team_arrived = 0; 5382 #endif 5383 } 5384 } 5385 5386 team->t.t_proc_bind = new_proc_bind; 5387 5388 #if OMPT_SUPPORT 5389 __ompt_team_assign_id(team, ompt_parallel_data); 5390 team->t.ompt_serialized_team_info = NULL; 5391 #endif 5392 5393 KMP_MB(); 5394 5395 KA_TRACE(20, ("__kmp_allocate_team: done creating a new team %d.\n", 5396 team->t.t_id)); 5397 5398 return team; 5399 } 5400 5401 /* TODO implement hot-teams at all levels */ 5402 /* TODO implement lazy thread release on demand (disband request) */ 5403 5404 /* free the team. return it to the team pool. release all the threads 5405 * associated with it */ 5406 void __kmp_free_team(kmp_root_t *root, 5407 kmp_team_t *team USE_NESTED_HOT_ARG(kmp_info_t *master)) { 5408 int f; 5409 KA_TRACE(20, ("__kmp_free_team: T#%d freeing team %d\n", __kmp_get_gtid(), 5410 team->t.t_id)); 5411 5412 /* verify state */ 5413 KMP_DEBUG_ASSERT(root); 5414 KMP_DEBUG_ASSERT(team); 5415 KMP_DEBUG_ASSERT(team->t.t_nproc <= team->t.t_max_nproc); 5416 KMP_DEBUG_ASSERT(team->t.t_threads); 5417 5418 int use_hot_team = team == root->r.r_hot_team; 5419 #if KMP_NESTED_HOT_TEAMS 5420 int level; 5421 kmp_hot_team_ptr_t *hot_teams; 5422 if (master) { 5423 level = team->t.t_active_level - 1; 5424 if (master->th.th_teams_microtask) { // in teams construct? 5425 if (master->th.th_teams_size.nteams > 1) { 5426 ++level; // level was not increased in teams construct for 5427 // team_of_masters 5428 } 5429 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master && 5430 master->th.th_teams_level == team->t.t_level) { 5431 ++level; // level was not increased in teams construct for 5432 // team_of_workers before the parallel 5433 } // team->t.t_level will be increased inside parallel 5434 } 5435 hot_teams = master->th.th_hot_teams; 5436 if (level < __kmp_hot_teams_max_level) { 5437 KMP_DEBUG_ASSERT(team == hot_teams[level].hot_team); 5438 use_hot_team = 1; 5439 } 5440 } 5441 #endif // KMP_NESTED_HOT_TEAMS 5442 5443 /* team is done working */ 5444 TCW_SYNC_PTR(team->t.t_pkfn, 5445 NULL); // Important for Debugging Support Library. 5446 #if KMP_OS_WINDOWS 5447 team->t.t_copyin_counter = 0; // init counter for possible reuse 5448 #endif 5449 // Do not reset pointer to parent team to NULL for hot teams. 5450 5451 /* if we are non-hot team, release our threads */ 5452 if (!use_hot_team) { 5453 if (__kmp_tasking_mode != tskm_immediate_exec) { 5454 // Wait for threads to reach reapable state 5455 for (f = 1; f < team->t.t_nproc; ++f) { 5456 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 5457 kmp_info_t *th = team->t.t_threads[f]; 5458 volatile kmp_uint32 *state = &th->th.th_reap_state; 5459 while (*state != KMP_SAFE_TO_REAP) { 5460 #if KMP_OS_WINDOWS 5461 // On Windows a thread can be killed at any time, check this 5462 DWORD ecode; 5463 if (!__kmp_is_thread_alive(th, &ecode)) { 5464 *state = KMP_SAFE_TO_REAP; // reset the flag for dead thread 5465 break; 5466 } 5467 #endif 5468 // first check if thread is sleeping 5469 kmp_flag_64 fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th); 5470 if (fl.is_sleeping()) 5471 fl.resume(__kmp_gtid_from_thread(th)); 5472 KMP_CPU_PAUSE(); 5473 } 5474 } 5475 5476 // Delete task teams 5477 int tt_idx; 5478 for (tt_idx = 0; tt_idx < 2; ++tt_idx) { 5479 kmp_task_team_t *task_team = team->t.t_task_team[tt_idx]; 5480 if (task_team != NULL) { 5481 for (f = 0; f < team->t.t_nproc; ++f) { // threads unref task teams 5482 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 5483 team->t.t_threads[f]->th.th_task_team = NULL; 5484 } 5485 KA_TRACE( 5486 20, 5487 ("__kmp_free_team: T#%d deactivating task_team %p on team %d\n", 5488 __kmp_get_gtid(), task_team, team->t.t_id)); 5489 #if KMP_NESTED_HOT_TEAMS 5490 __kmp_free_task_team(master, task_team); 5491 #endif 5492 team->t.t_task_team[tt_idx] = NULL; 5493 } 5494 } 5495 } 5496 5497 // Reset pointer to parent team only for non-hot teams. 5498 team->t.t_parent = NULL; 5499 team->t.t_level = 0; 5500 team->t.t_active_level = 0; 5501 5502 /* free the worker threads */ 5503 for (f = 1; f < team->t.t_nproc; ++f) { 5504 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 5505 __kmp_free_thread(team->t.t_threads[f]); 5506 team->t.t_threads[f] = NULL; 5507 } 5508 5509 /* put the team back in the team pool */ 5510 /* TODO limit size of team pool, call reap_team if pool too large */ 5511 team->t.t_next_pool = CCAST(kmp_team_t *, __kmp_team_pool); 5512 __kmp_team_pool = (volatile kmp_team_t *)team; 5513 } else { // Check if team was created for the masters in a teams construct 5514 // See if first worker is a CG root 5515 KMP_DEBUG_ASSERT(team->t.t_threads[1] && 5516 team->t.t_threads[1]->th.th_cg_roots); 5517 if (team->t.t_threads[1]->th.th_cg_roots->cg_root == team->t.t_threads[1]) { 5518 // Clean up the CG root nodes on workers so that this team can be re-used 5519 for (f = 1; f < team->t.t_nproc; ++f) { 5520 kmp_info_t *thr = team->t.t_threads[f]; 5521 KMP_DEBUG_ASSERT(thr && thr->th.th_cg_roots && 5522 thr->th.th_cg_roots->cg_root == thr); 5523 // Pop current CG root off list 5524 kmp_cg_root_t *tmp = thr->th.th_cg_roots; 5525 thr->th.th_cg_roots = tmp->up; 5526 KA_TRACE(100, ("__kmp_free_team: Thread %p popping node %p and moving" 5527 " up to node %p. cg_nthreads was %d\n", 5528 thr, tmp, thr->th.th_cg_roots, tmp->cg_nthreads)); 5529 int i = tmp->cg_nthreads--; 5530 if (i == 1) { 5531 __kmp_free(tmp); // free CG if we are the last thread in it 5532 } 5533 // Restore current task's thread_limit from CG root 5534 if (thr->th.th_cg_roots) 5535 thr->th.th_current_task->td_icvs.thread_limit = 5536 thr->th.th_cg_roots->cg_thread_limit; 5537 } 5538 } 5539 } 5540 5541 KMP_MB(); 5542 } 5543 5544 /* reap the team. destroy it, reclaim all its resources and free its memory */ 5545 kmp_team_t *__kmp_reap_team(kmp_team_t *team) { 5546 kmp_team_t *next_pool = team->t.t_next_pool; 5547 5548 KMP_DEBUG_ASSERT(team); 5549 KMP_DEBUG_ASSERT(team->t.t_dispatch); 5550 KMP_DEBUG_ASSERT(team->t.t_disp_buffer); 5551 KMP_DEBUG_ASSERT(team->t.t_threads); 5552 KMP_DEBUG_ASSERT(team->t.t_argv); 5553 5554 /* TODO clean the threads that are a part of this? */ 5555 5556 /* free stuff */ 5557 __kmp_free_team_arrays(team); 5558 if (team->t.t_argv != &team->t.t_inline_argv[0]) 5559 __kmp_free((void *)team->t.t_argv); 5560 __kmp_free(team); 5561 5562 KMP_MB(); 5563 return next_pool; 5564 } 5565 5566 // Free the thread. Don't reap it, just place it on the pool of available 5567 // threads. 5568 // 5569 // Changes for Quad issue 527845: We need a predictable OMP tid <-> gtid 5570 // binding for the affinity mechanism to be useful. 5571 // 5572 // Now, we always keep the free list (__kmp_thread_pool) sorted by gtid. 5573 // However, we want to avoid a potential performance problem by always 5574 // scanning through the list to find the correct point at which to insert 5575 // the thread (potential N**2 behavior). To do this we keep track of the 5576 // last place a thread struct was inserted (__kmp_thread_pool_insert_pt). 5577 // With single-level parallelism, threads will always be added to the tail 5578 // of the list, kept track of by __kmp_thread_pool_insert_pt. With nested 5579 // parallelism, all bets are off and we may need to scan through the entire 5580 // free list. 5581 // 5582 // This change also has a potentially large performance benefit, for some 5583 // applications. Previously, as threads were freed from the hot team, they 5584 // would be placed back on the free list in inverse order. If the hot team 5585 // grew back to it's original size, then the freed thread would be placed 5586 // back on the hot team in reverse order. This could cause bad cache 5587 // locality problems on programs where the size of the hot team regularly 5588 // grew and shrunk. 5589 // 5590 // Now, for single-level parallelism, the OMP tid is always == gtid. 5591 void __kmp_free_thread(kmp_info_t *this_th) { 5592 int gtid; 5593 kmp_info_t **scan; 5594 5595 KA_TRACE(20, ("__kmp_free_thread: T#%d putting T#%d back on free pool.\n", 5596 __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid)); 5597 5598 KMP_DEBUG_ASSERT(this_th); 5599 5600 // When moving thread to pool, switch thread to wait on own b_go flag, and 5601 // uninitialized (NULL team). 5602 int b; 5603 kmp_balign_t *balign = this_th->th.th_bar; 5604 for (b = 0; b < bs_last_barrier; ++b) { 5605 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG) 5606 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG; 5607 balign[b].bb.team = NULL; 5608 balign[b].bb.leaf_kids = 0; 5609 } 5610 this_th->th.th_task_state = 0; 5611 this_th->th.th_reap_state = KMP_SAFE_TO_REAP; 5612 5613 /* put thread back on the free pool */ 5614 TCW_PTR(this_th->th.th_team, NULL); 5615 TCW_PTR(this_th->th.th_root, NULL); 5616 TCW_PTR(this_th->th.th_dispatch, NULL); /* NOT NEEDED */ 5617 5618 while (this_th->th.th_cg_roots) { 5619 this_th->th.th_cg_roots->cg_nthreads--; 5620 KA_TRACE(100, ("__kmp_free_thread: Thread %p decrement cg_nthreads on node" 5621 " %p of thread %p to %d\n", 5622 this_th, this_th->th.th_cg_roots, 5623 this_th->th.th_cg_roots->cg_root, 5624 this_th->th.th_cg_roots->cg_nthreads)); 5625 kmp_cg_root_t *tmp = this_th->th.th_cg_roots; 5626 if (tmp->cg_root == this_th) { // Thread is a cg_root 5627 KMP_DEBUG_ASSERT(tmp->cg_nthreads == 0); 5628 KA_TRACE( 5629 5, ("__kmp_free_thread: Thread %p freeing node %p\n", this_th, tmp)); 5630 this_th->th.th_cg_roots = tmp->up; 5631 __kmp_free(tmp); 5632 } else { // Worker thread 5633 if (tmp->cg_nthreads == 0) { // last thread leaves contention group 5634 __kmp_free(tmp); 5635 } 5636 this_th->th.th_cg_roots = NULL; 5637 break; 5638 } 5639 } 5640 5641 /* If the implicit task assigned to this thread can be used by other threads 5642 * -> multiple threads can share the data and try to free the task at 5643 * __kmp_reap_thread at exit. This duplicate use of the task data can happen 5644 * with higher probability when hot team is disabled but can occurs even when 5645 * the hot team is enabled */ 5646 __kmp_free_implicit_task(this_th); 5647 this_th->th.th_current_task = NULL; 5648 5649 // If the __kmp_thread_pool_insert_pt is already past the new insert 5650 // point, then we need to re-scan the entire list. 5651 gtid = this_th->th.th_info.ds.ds_gtid; 5652 if (__kmp_thread_pool_insert_pt != NULL) { 5653 KMP_DEBUG_ASSERT(__kmp_thread_pool != NULL); 5654 if (__kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid) { 5655 __kmp_thread_pool_insert_pt = NULL; 5656 } 5657 } 5658 5659 // Scan down the list to find the place to insert the thread. 5660 // scan is the address of a link in the list, possibly the address of 5661 // __kmp_thread_pool itself. 5662 // 5663 // In the absence of nested parallelism, the for loop will have 0 iterations. 5664 if (__kmp_thread_pool_insert_pt != NULL) { 5665 scan = &(__kmp_thread_pool_insert_pt->th.th_next_pool); 5666 } else { 5667 scan = CCAST(kmp_info_t **, &__kmp_thread_pool); 5668 } 5669 for (; (*scan != NULL) && ((*scan)->th.th_info.ds.ds_gtid < gtid); 5670 scan = &((*scan)->th.th_next_pool)) 5671 ; 5672 5673 // Insert the new element on the list, and set __kmp_thread_pool_insert_pt 5674 // to its address. 5675 TCW_PTR(this_th->th.th_next_pool, *scan); 5676 __kmp_thread_pool_insert_pt = *scan = this_th; 5677 KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) || 5678 (this_th->th.th_info.ds.ds_gtid < 5679 this_th->th.th_next_pool->th.th_info.ds.ds_gtid)); 5680 TCW_4(this_th->th.th_in_pool, TRUE); 5681 __kmp_suspend_initialize_thread(this_th); 5682 __kmp_lock_suspend_mx(this_th); 5683 if (this_th->th.th_active == TRUE) { 5684 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth); 5685 this_th->th.th_active_in_pool = TRUE; 5686 } 5687 #if KMP_DEBUG 5688 else { 5689 KMP_DEBUG_ASSERT(this_th->th.th_active_in_pool == FALSE); 5690 } 5691 #endif 5692 __kmp_unlock_suspend_mx(this_th); 5693 5694 TCW_4(__kmp_nth, __kmp_nth - 1); 5695 5696 #ifdef KMP_ADJUST_BLOCKTIME 5697 /* Adjust blocktime back to user setting or default if necessary */ 5698 /* Middle initialization might never have occurred */ 5699 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 5700 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0); 5701 if (__kmp_nth <= __kmp_avail_proc) { 5702 __kmp_zero_bt = FALSE; 5703 } 5704 } 5705 #endif /* KMP_ADJUST_BLOCKTIME */ 5706 5707 KMP_MB(); 5708 } 5709 5710 /* ------------------------------------------------------------------------ */ 5711 5712 void *__kmp_launch_thread(kmp_info_t *this_thr) { 5713 int gtid = this_thr->th.th_info.ds.ds_gtid; 5714 /* void *stack_data;*/ 5715 kmp_team_t **volatile pteam; 5716 5717 KMP_MB(); 5718 KA_TRACE(10, ("__kmp_launch_thread: T#%d start\n", gtid)); 5719 5720 if (__kmp_env_consistency_check) { 5721 this_thr->th.th_cons = __kmp_allocate_cons_stack(gtid); // ATT: Memory leak? 5722 } 5723 5724 #if OMPT_SUPPORT 5725 ompt_data_t *thread_data; 5726 if (ompt_enabled.enabled) { 5727 thread_data = &(this_thr->th.ompt_thread_info.thread_data); 5728 *thread_data = ompt_data_none; 5729 5730 this_thr->th.ompt_thread_info.state = ompt_state_overhead; 5731 this_thr->th.ompt_thread_info.wait_id = 0; 5732 this_thr->th.ompt_thread_info.idle_frame = OMPT_GET_FRAME_ADDRESS(0); 5733 this_thr->th.ompt_thread_info.parallel_flags = 0; 5734 if (ompt_enabled.ompt_callback_thread_begin) { 5735 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)( 5736 ompt_thread_worker, thread_data); 5737 } 5738 this_thr->th.ompt_thread_info.state = ompt_state_idle; 5739 } 5740 #endif 5741 5742 /* This is the place where threads wait for work */ 5743 while (!TCR_4(__kmp_global.g.g_done)) { 5744 KMP_DEBUG_ASSERT(this_thr == __kmp_threads[gtid]); 5745 KMP_MB(); 5746 5747 /* wait for work to do */ 5748 KA_TRACE(20, ("__kmp_launch_thread: T#%d waiting for work\n", gtid)); 5749 5750 /* No tid yet since not part of a team */ 5751 __kmp_fork_barrier(gtid, KMP_GTID_DNE); 5752 5753 #if OMPT_SUPPORT 5754 if (ompt_enabled.enabled) { 5755 this_thr->th.ompt_thread_info.state = ompt_state_overhead; 5756 } 5757 #endif 5758 5759 pteam = &this_thr->th.th_team; 5760 5761 /* have we been allocated? */ 5762 if (TCR_SYNC_PTR(*pteam) && !TCR_4(__kmp_global.g.g_done)) { 5763 /* we were just woken up, so run our new task */ 5764 if (TCR_SYNC_PTR((*pteam)->t.t_pkfn) != NULL) { 5765 int rc; 5766 KA_TRACE(20, 5767 ("__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n", 5768 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), 5769 (*pteam)->t.t_pkfn)); 5770 5771 updateHWFPControl(*pteam); 5772 5773 #if OMPT_SUPPORT 5774 if (ompt_enabled.enabled) { 5775 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel; 5776 } 5777 #endif 5778 5779 rc = (*pteam)->t.t_invoke(gtid); 5780 KMP_ASSERT(rc); 5781 5782 KMP_MB(); 5783 KA_TRACE(20, ("__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n", 5784 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), 5785 (*pteam)->t.t_pkfn)); 5786 } 5787 #if OMPT_SUPPORT 5788 if (ompt_enabled.enabled) { 5789 /* no frame set while outside task */ 5790 __ompt_get_task_info_object(0)->frame.exit_frame = ompt_data_none; 5791 5792 this_thr->th.ompt_thread_info.state = ompt_state_overhead; 5793 } 5794 #endif 5795 /* join barrier after parallel region */ 5796 __kmp_join_barrier(gtid); 5797 } 5798 } 5799 TCR_SYNC_PTR((intptr_t)__kmp_global.g.g_done); 5800 5801 #if OMPT_SUPPORT 5802 if (ompt_enabled.ompt_callback_thread_end) { 5803 ompt_callbacks.ompt_callback(ompt_callback_thread_end)(thread_data); 5804 } 5805 #endif 5806 5807 this_thr->th.th_task_team = NULL; 5808 /* run the destructors for the threadprivate data for this thread */ 5809 __kmp_common_destroy_gtid(gtid); 5810 5811 KA_TRACE(10, ("__kmp_launch_thread: T#%d done\n", gtid)); 5812 KMP_MB(); 5813 return this_thr; 5814 } 5815 5816 /* ------------------------------------------------------------------------ */ 5817 5818 void __kmp_internal_end_dest(void *specific_gtid) { 5819 #if KMP_COMPILER_ICC 5820 #pragma warning(push) 5821 #pragma warning(disable : 810) // conversion from "void *" to "int" may lose 5822 // significant bits 5823 #endif 5824 // Make sure no significant bits are lost 5825 int gtid = (kmp_intptr_t)specific_gtid - 1; 5826 #if KMP_COMPILER_ICC 5827 #pragma warning(pop) 5828 #endif 5829 5830 KA_TRACE(30, ("__kmp_internal_end_dest: T#%d\n", gtid)); 5831 /* NOTE: the gtid is stored as gitd+1 in the thread-local-storage 5832 * this is because 0 is reserved for the nothing-stored case */ 5833 5834 /* josh: One reason for setting the gtid specific data even when it is being 5835 destroyed by pthread is to allow gtid lookup through thread specific data 5836 (__kmp_gtid_get_specific). Some of the code, especially stat code, 5837 that gets executed in the call to __kmp_internal_end_thread, actually 5838 gets the gtid through the thread specific data. Setting it here seems 5839 rather inelegant and perhaps wrong, but allows __kmp_internal_end_thread 5840 to run smoothly. 5841 todo: get rid of this after we remove the dependence on 5842 __kmp_gtid_get_specific */ 5843 if (gtid >= 0 && KMP_UBER_GTID(gtid)) 5844 __kmp_gtid_set_specific(gtid); 5845 #ifdef KMP_TDATA_GTID 5846 __kmp_gtid = gtid; 5847 #endif 5848 __kmp_internal_end_thread(gtid); 5849 } 5850 5851 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB 5852 5853 __attribute__((destructor)) void __kmp_internal_end_dtor(void) { 5854 __kmp_internal_end_atexit(); 5855 } 5856 5857 #endif 5858 5859 /* [Windows] josh: when the atexit handler is called, there may still be more 5860 than one thread alive */ 5861 void __kmp_internal_end_atexit(void) { 5862 KA_TRACE(30, ("__kmp_internal_end_atexit\n")); 5863 /* [Windows] 5864 josh: ideally, we want to completely shutdown the library in this atexit 5865 handler, but stat code that depends on thread specific data for gtid fails 5866 because that data becomes unavailable at some point during the shutdown, so 5867 we call __kmp_internal_end_thread instead. We should eventually remove the 5868 dependency on __kmp_get_specific_gtid in the stat code and use 5869 __kmp_internal_end_library to cleanly shutdown the library. 5870 5871 // TODO: Can some of this comment about GVS be removed? 5872 I suspect that the offending stat code is executed when the calling thread 5873 tries to clean up a dead root thread's data structures, resulting in GVS 5874 code trying to close the GVS structures for that thread, but since the stat 5875 code uses __kmp_get_specific_gtid to get the gtid with the assumption that 5876 the calling thread is cleaning up itself instead of another thread, it get 5877 confused. This happens because allowing a thread to unregister and cleanup 5878 another thread is a recent modification for addressing an issue. 5879 Based on the current design (20050722), a thread may end up 5880 trying to unregister another thread only if thread death does not trigger 5881 the calling of __kmp_internal_end_thread. For Linux* OS, there is the 5882 thread specific data destructor function to detect thread death. For 5883 Windows dynamic, there is DllMain(THREAD_DETACH). For Windows static, there 5884 is nothing. Thus, the workaround is applicable only for Windows static 5885 stat library. */ 5886 __kmp_internal_end_library(-1); 5887 #if KMP_OS_WINDOWS 5888 __kmp_close_console(); 5889 #endif 5890 } 5891 5892 static void __kmp_reap_thread(kmp_info_t *thread, int is_root) { 5893 // It is assumed __kmp_forkjoin_lock is acquired. 5894 5895 int gtid; 5896 5897 KMP_DEBUG_ASSERT(thread != NULL); 5898 5899 gtid = thread->th.th_info.ds.ds_gtid; 5900 5901 if (!is_root) { 5902 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) { 5903 /* Assume the threads are at the fork barrier here */ 5904 KA_TRACE( 5905 20, ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n", 5906 gtid)); 5907 /* Need release fence here to prevent seg faults for tree forkjoin barrier 5908 * (GEH) */ 5909 ANNOTATE_HAPPENS_BEFORE(thread); 5910 kmp_flag_64 flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, thread); 5911 __kmp_release_64(&flag); 5912 } 5913 5914 // Terminate OS thread. 5915 __kmp_reap_worker(thread); 5916 5917 // The thread was killed asynchronously. If it was actively 5918 // spinning in the thread pool, decrement the global count. 5919 // 5920 // There is a small timing hole here - if the worker thread was just waking 5921 // up after sleeping in the pool, had reset it's th_active_in_pool flag but 5922 // not decremented the global counter __kmp_thread_pool_active_nth yet, then 5923 // the global counter might not get updated. 5924 // 5925 // Currently, this can only happen as the library is unloaded, 5926 // so there are no harmful side effects. 5927 if (thread->th.th_active_in_pool) { 5928 thread->th.th_active_in_pool = FALSE; 5929 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth); 5930 KMP_DEBUG_ASSERT(__kmp_thread_pool_active_nth >= 0); 5931 } 5932 } 5933 5934 __kmp_free_implicit_task(thread); 5935 5936 // Free the fast memory for tasking 5937 #if USE_FAST_MEMORY 5938 __kmp_free_fast_memory(thread); 5939 #endif /* USE_FAST_MEMORY */ 5940 5941 __kmp_suspend_uninitialize_thread(thread); 5942 5943 KMP_DEBUG_ASSERT(__kmp_threads[gtid] == thread); 5944 TCW_SYNC_PTR(__kmp_threads[gtid], NULL); 5945 5946 --__kmp_all_nth; 5947 // __kmp_nth was decremented when thread is added to the pool. 5948 5949 #ifdef KMP_ADJUST_BLOCKTIME 5950 /* Adjust blocktime back to user setting or default if necessary */ 5951 /* Middle initialization might never have occurred */ 5952 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 5953 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0); 5954 if (__kmp_nth <= __kmp_avail_proc) { 5955 __kmp_zero_bt = FALSE; 5956 } 5957 } 5958 #endif /* KMP_ADJUST_BLOCKTIME */ 5959 5960 /* free the memory being used */ 5961 if (__kmp_env_consistency_check) { 5962 if (thread->th.th_cons) { 5963 __kmp_free_cons_stack(thread->th.th_cons); 5964 thread->th.th_cons = NULL; 5965 } 5966 } 5967 5968 if (thread->th.th_pri_common != NULL) { 5969 __kmp_free(thread->th.th_pri_common); 5970 thread->th.th_pri_common = NULL; 5971 } 5972 5973 if (thread->th.th_task_state_memo_stack != NULL) { 5974 __kmp_free(thread->th.th_task_state_memo_stack); 5975 thread->th.th_task_state_memo_stack = NULL; 5976 } 5977 5978 #if KMP_USE_BGET 5979 if (thread->th.th_local.bget_data != NULL) { 5980 __kmp_finalize_bget(thread); 5981 } 5982 #endif 5983 5984 #if KMP_AFFINITY_SUPPORTED 5985 if (thread->th.th_affin_mask != NULL) { 5986 KMP_CPU_FREE(thread->th.th_affin_mask); 5987 thread->th.th_affin_mask = NULL; 5988 } 5989 #endif /* KMP_AFFINITY_SUPPORTED */ 5990 5991 #if KMP_USE_HIER_SCHED 5992 if (thread->th.th_hier_bar_data != NULL) { 5993 __kmp_free(thread->th.th_hier_bar_data); 5994 thread->th.th_hier_bar_data = NULL; 5995 } 5996 #endif 5997 5998 __kmp_reap_team(thread->th.th_serial_team); 5999 thread->th.th_serial_team = NULL; 6000 __kmp_free(thread); 6001 6002 KMP_MB(); 6003 6004 } // __kmp_reap_thread 6005 6006 static void __kmp_internal_end(void) { 6007 int i; 6008 6009 /* First, unregister the library */ 6010 __kmp_unregister_library(); 6011 6012 #if KMP_OS_WINDOWS 6013 /* In Win static library, we can't tell when a root actually dies, so we 6014 reclaim the data structures for any root threads that have died but not 6015 unregistered themselves, in order to shut down cleanly. 6016 In Win dynamic library we also can't tell when a thread dies. */ 6017 __kmp_reclaim_dead_roots(); // AC: moved here to always clean resources of 6018 // dead roots 6019 #endif 6020 6021 for (i = 0; i < __kmp_threads_capacity; i++) 6022 if (__kmp_root[i]) 6023 if (__kmp_root[i]->r.r_active) 6024 break; 6025 KMP_MB(); /* Flush all pending memory write invalidates. */ 6026 TCW_SYNC_4(__kmp_global.g.g_done, TRUE); 6027 6028 if (i < __kmp_threads_capacity) { 6029 #if KMP_USE_MONITOR 6030 // 2009-09-08 (lev): Other alive roots found. Why do we kill the monitor?? 6031 KMP_MB(); /* Flush all pending memory write invalidates. */ 6032 6033 // Need to check that monitor was initialized before reaping it. If we are 6034 // called form __kmp_atfork_child (which sets __kmp_init_parallel = 0), then 6035 // __kmp_monitor will appear to contain valid data, but it is only valid in 6036 // the parent process, not the child. 6037 // New behavior (201008): instead of keying off of the flag 6038 // __kmp_init_parallel, the monitor thread creation is keyed off 6039 // of the new flag __kmp_init_monitor. 6040 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock); 6041 if (TCR_4(__kmp_init_monitor)) { 6042 __kmp_reap_monitor(&__kmp_monitor); 6043 TCW_4(__kmp_init_monitor, 0); 6044 } 6045 __kmp_release_bootstrap_lock(&__kmp_monitor_lock); 6046 KA_TRACE(10, ("__kmp_internal_end: monitor reaped\n")); 6047 #endif // KMP_USE_MONITOR 6048 } else { 6049 /* TODO move this to cleanup code */ 6050 #ifdef KMP_DEBUG 6051 /* make sure that everything has properly ended */ 6052 for (i = 0; i < __kmp_threads_capacity; i++) { 6053 if (__kmp_root[i]) { 6054 // KMP_ASSERT( ! KMP_UBER_GTID( i ) ); // AC: 6055 // there can be uber threads alive here 6056 KMP_ASSERT(!__kmp_root[i]->r.r_active); // TODO: can they be active? 6057 } 6058 } 6059 #endif 6060 6061 KMP_MB(); 6062 6063 // Reap the worker threads. 6064 // This is valid for now, but be careful if threads are reaped sooner. 6065 while (__kmp_thread_pool != NULL) { // Loop thru all the thread in the pool. 6066 // Get the next thread from the pool. 6067 kmp_info_t *thread = CCAST(kmp_info_t *, __kmp_thread_pool); 6068 __kmp_thread_pool = thread->th.th_next_pool; 6069 // Reap it. 6070 KMP_DEBUG_ASSERT(thread->th.th_reap_state == KMP_SAFE_TO_REAP); 6071 thread->th.th_next_pool = NULL; 6072 thread->th.th_in_pool = FALSE; 6073 __kmp_reap_thread(thread, 0); 6074 } 6075 __kmp_thread_pool_insert_pt = NULL; 6076 6077 // Reap teams. 6078 while (__kmp_team_pool != NULL) { // Loop thru all the teams in the pool. 6079 // Get the next team from the pool. 6080 kmp_team_t *team = CCAST(kmp_team_t *, __kmp_team_pool); 6081 __kmp_team_pool = team->t.t_next_pool; 6082 // Reap it. 6083 team->t.t_next_pool = NULL; 6084 __kmp_reap_team(team); 6085 } 6086 6087 __kmp_reap_task_teams(); 6088 6089 #if KMP_OS_UNIX 6090 // Threads that are not reaped should not access any resources since they 6091 // are going to be deallocated soon, so the shutdown sequence should wait 6092 // until all threads either exit the final spin-waiting loop or begin 6093 // sleeping after the given blocktime. 6094 for (i = 0; i < __kmp_threads_capacity; i++) { 6095 kmp_info_t *thr = __kmp_threads[i]; 6096 while (thr && KMP_ATOMIC_LD_ACQ(&thr->th.th_blocking)) 6097 KMP_CPU_PAUSE(); 6098 } 6099 #endif 6100 6101 for (i = 0; i < __kmp_threads_capacity; ++i) { 6102 // TBD: Add some checking... 6103 // Something like KMP_DEBUG_ASSERT( __kmp_thread[ i ] == NULL ); 6104 } 6105 6106 /* Make sure all threadprivate destructors get run by joining with all 6107 worker threads before resetting this flag */ 6108 TCW_SYNC_4(__kmp_init_common, FALSE); 6109 6110 KA_TRACE(10, ("__kmp_internal_end: all workers reaped\n")); 6111 KMP_MB(); 6112 6113 #if KMP_USE_MONITOR 6114 // See note above: One of the possible fixes for CQ138434 / CQ140126 6115 // 6116 // FIXME: push both code fragments down and CSE them? 6117 // push them into __kmp_cleanup() ? 6118 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock); 6119 if (TCR_4(__kmp_init_monitor)) { 6120 __kmp_reap_monitor(&__kmp_monitor); 6121 TCW_4(__kmp_init_monitor, 0); 6122 } 6123 __kmp_release_bootstrap_lock(&__kmp_monitor_lock); 6124 KA_TRACE(10, ("__kmp_internal_end: monitor reaped\n")); 6125 #endif 6126 } /* else !__kmp_global.t_active */ 6127 TCW_4(__kmp_init_gtid, FALSE); 6128 KMP_MB(); /* Flush all pending memory write invalidates. */ 6129 6130 __kmp_cleanup(); 6131 #if OMPT_SUPPORT 6132 ompt_fini(); 6133 #endif 6134 } 6135 6136 void __kmp_internal_end_library(int gtid_req) { 6137 /* if we have already cleaned up, don't try again, it wouldn't be pretty */ 6138 /* this shouldn't be a race condition because __kmp_internal_end() is the 6139 only place to clear __kmp_serial_init */ 6140 /* we'll check this later too, after we get the lock */ 6141 // 2009-09-06: We do not set g_abort without setting g_done. This check looks 6142 // redundant, because the next check will work in any case. 6143 if (__kmp_global.g.g_abort) { 6144 KA_TRACE(11, ("__kmp_internal_end_library: abort, exiting\n")); 6145 /* TODO abort? */ 6146 return; 6147 } 6148 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 6149 KA_TRACE(10, ("__kmp_internal_end_library: already finished\n")); 6150 return; 6151 } 6152 6153 KMP_MB(); /* Flush all pending memory write invalidates. */ 6154 6155 /* find out who we are and what we should do */ 6156 { 6157 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific(); 6158 KA_TRACE( 6159 10, ("__kmp_internal_end_library: enter T#%d (%d)\n", gtid, gtid_req)); 6160 if (gtid == KMP_GTID_SHUTDOWN) { 6161 KA_TRACE(10, ("__kmp_internal_end_library: !__kmp_init_runtime, system " 6162 "already shutdown\n")); 6163 return; 6164 } else if (gtid == KMP_GTID_MONITOR) { 6165 KA_TRACE(10, ("__kmp_internal_end_library: monitor thread, gtid not " 6166 "registered, or system shutdown\n")); 6167 return; 6168 } else if (gtid == KMP_GTID_DNE) { 6169 KA_TRACE(10, ("__kmp_internal_end_library: gtid not registered or system " 6170 "shutdown\n")); 6171 /* we don't know who we are, but we may still shutdown the library */ 6172 } else if (KMP_UBER_GTID(gtid)) { 6173 /* unregister ourselves as an uber thread. gtid is no longer valid */ 6174 if (__kmp_root[gtid]->r.r_active) { 6175 __kmp_global.g.g_abort = -1; 6176 TCW_SYNC_4(__kmp_global.g.g_done, TRUE); 6177 KA_TRACE(10, 6178 ("__kmp_internal_end_library: root still active, abort T#%d\n", 6179 gtid)); 6180 return; 6181 } else { 6182 KA_TRACE( 6183 10, 6184 ("__kmp_internal_end_library: unregistering sibling T#%d\n", gtid)); 6185 __kmp_unregister_root_current_thread(gtid); 6186 } 6187 } else { 6188 /* worker threads may call this function through the atexit handler, if they 6189 * call exit() */ 6190 /* For now, skip the usual subsequent processing and just dump the debug buffer. 6191 TODO: do a thorough shutdown instead */ 6192 #ifdef DUMP_DEBUG_ON_EXIT 6193 if (__kmp_debug_buf) 6194 __kmp_dump_debug_buffer(); 6195 #endif 6196 return; 6197 } 6198 } 6199 /* synchronize the termination process */ 6200 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6201 6202 /* have we already finished */ 6203 if (__kmp_global.g.g_abort) { 6204 KA_TRACE(10, ("__kmp_internal_end_library: abort, exiting\n")); 6205 /* TODO abort? */ 6206 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6207 return; 6208 } 6209 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 6210 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6211 return; 6212 } 6213 6214 /* We need this lock to enforce mutex between this reading of 6215 __kmp_threads_capacity and the writing by __kmp_register_root. 6216 Alternatively, we can use a counter of roots that is atomically updated by 6217 __kmp_get_global_thread_id_reg, __kmp_do_serial_initialize and 6218 __kmp_internal_end_*. */ 6219 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 6220 6221 /* now we can safely conduct the actual termination */ 6222 __kmp_internal_end(); 6223 6224 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 6225 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6226 6227 KA_TRACE(10, ("__kmp_internal_end_library: exit\n")); 6228 6229 #ifdef DUMP_DEBUG_ON_EXIT 6230 if (__kmp_debug_buf) 6231 __kmp_dump_debug_buffer(); 6232 #endif 6233 6234 #if KMP_OS_WINDOWS 6235 __kmp_close_console(); 6236 #endif 6237 6238 __kmp_fini_allocator(); 6239 6240 } // __kmp_internal_end_library 6241 6242 void __kmp_internal_end_thread(int gtid_req) { 6243 int i; 6244 6245 /* if we have already cleaned up, don't try again, it wouldn't be pretty */ 6246 /* this shouldn't be a race condition because __kmp_internal_end() is the 6247 * only place to clear __kmp_serial_init */ 6248 /* we'll check this later too, after we get the lock */ 6249 // 2009-09-06: We do not set g_abort without setting g_done. This check looks 6250 // redundant, because the next check will work in any case. 6251 if (__kmp_global.g.g_abort) { 6252 KA_TRACE(11, ("__kmp_internal_end_thread: abort, exiting\n")); 6253 /* TODO abort? */ 6254 return; 6255 } 6256 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 6257 KA_TRACE(10, ("__kmp_internal_end_thread: already finished\n")); 6258 return; 6259 } 6260 6261 KMP_MB(); /* Flush all pending memory write invalidates. */ 6262 6263 /* find out who we are and what we should do */ 6264 { 6265 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific(); 6266 KA_TRACE(10, 6267 ("__kmp_internal_end_thread: enter T#%d (%d)\n", gtid, gtid_req)); 6268 if (gtid == KMP_GTID_SHUTDOWN) { 6269 KA_TRACE(10, ("__kmp_internal_end_thread: !__kmp_init_runtime, system " 6270 "already shutdown\n")); 6271 return; 6272 } else if (gtid == KMP_GTID_MONITOR) { 6273 KA_TRACE(10, ("__kmp_internal_end_thread: monitor thread, gtid not " 6274 "registered, or system shutdown\n")); 6275 return; 6276 } else if (gtid == KMP_GTID_DNE) { 6277 KA_TRACE(10, ("__kmp_internal_end_thread: gtid not registered or system " 6278 "shutdown\n")); 6279 return; 6280 /* we don't know who we are */ 6281 } else if (KMP_UBER_GTID(gtid)) { 6282 /* unregister ourselves as an uber thread. gtid is no longer valid */ 6283 if (__kmp_root[gtid]->r.r_active) { 6284 __kmp_global.g.g_abort = -1; 6285 TCW_SYNC_4(__kmp_global.g.g_done, TRUE); 6286 KA_TRACE(10, 6287 ("__kmp_internal_end_thread: root still active, abort T#%d\n", 6288 gtid)); 6289 return; 6290 } else { 6291 KA_TRACE(10, ("__kmp_internal_end_thread: unregistering sibling T#%d\n", 6292 gtid)); 6293 __kmp_unregister_root_current_thread(gtid); 6294 } 6295 } else { 6296 /* just a worker thread, let's leave */ 6297 KA_TRACE(10, ("__kmp_internal_end_thread: worker thread T#%d\n", gtid)); 6298 6299 if (gtid >= 0) { 6300 __kmp_threads[gtid]->th.th_task_team = NULL; 6301 } 6302 6303 KA_TRACE(10, 6304 ("__kmp_internal_end_thread: worker thread done, exiting T#%d\n", 6305 gtid)); 6306 return; 6307 } 6308 } 6309 #if KMP_DYNAMIC_LIB 6310 if (__kmp_pause_status != kmp_hard_paused) 6311 // AC: lets not shutdown the dynamic library at the exit of uber thread, 6312 // because we will better shutdown later in the library destructor. 6313 { 6314 KA_TRACE(10, ("__kmp_internal_end_thread: exiting T#%d\n", gtid_req)); 6315 return; 6316 } 6317 #endif 6318 /* synchronize the termination process */ 6319 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6320 6321 /* have we already finished */ 6322 if (__kmp_global.g.g_abort) { 6323 KA_TRACE(10, ("__kmp_internal_end_thread: abort, exiting\n")); 6324 /* TODO abort? */ 6325 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6326 return; 6327 } 6328 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 6329 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6330 return; 6331 } 6332 6333 /* We need this lock to enforce mutex between this reading of 6334 __kmp_threads_capacity and the writing by __kmp_register_root. 6335 Alternatively, we can use a counter of roots that is atomically updated by 6336 __kmp_get_global_thread_id_reg, __kmp_do_serial_initialize and 6337 __kmp_internal_end_*. */ 6338 6339 /* should we finish the run-time? are all siblings done? */ 6340 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 6341 6342 for (i = 0; i < __kmp_threads_capacity; ++i) { 6343 if (KMP_UBER_GTID(i)) { 6344 KA_TRACE( 6345 10, 6346 ("__kmp_internal_end_thread: remaining sibling task: gtid==%d\n", i)); 6347 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 6348 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6349 return; 6350 } 6351 } 6352 6353 /* now we can safely conduct the actual termination */ 6354 6355 __kmp_internal_end(); 6356 6357 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 6358 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6359 6360 KA_TRACE(10, ("__kmp_internal_end_thread: exit T#%d\n", gtid_req)); 6361 6362 #ifdef DUMP_DEBUG_ON_EXIT 6363 if (__kmp_debug_buf) 6364 __kmp_dump_debug_buffer(); 6365 #endif 6366 } // __kmp_internal_end_thread 6367 6368 // ----------------------------------------------------------------------------- 6369 // Library registration stuff. 6370 6371 static long __kmp_registration_flag = 0; 6372 // Random value used to indicate library initialization. 6373 static char *__kmp_registration_str = NULL; 6374 // Value to be saved in env var __KMP_REGISTERED_LIB_<pid>. 6375 6376 static inline char *__kmp_reg_status_name() { 6377 /* On RHEL 3u5 if linked statically, getpid() returns different values in 6378 each thread. If registration and unregistration go in different threads 6379 (omp_misc_other_root_exit.cpp test case), the name of registered_lib_env 6380 env var can not be found, because the name will contain different pid. */ 6381 return __kmp_str_format("__KMP_REGISTERED_LIB_%d", (int)getpid()); 6382 } // __kmp_reg_status_get 6383 6384 void __kmp_register_library_startup(void) { 6385 6386 char *name = __kmp_reg_status_name(); // Name of the environment variable. 6387 int done = 0; 6388 union { 6389 double dtime; 6390 long ltime; 6391 } time; 6392 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 6393 __kmp_initialize_system_tick(); 6394 #endif 6395 __kmp_read_system_time(&time.dtime); 6396 __kmp_registration_flag = 0xCAFE0000L | (time.ltime & 0x0000FFFFL); 6397 __kmp_registration_str = 6398 __kmp_str_format("%p-%lx-%s", &__kmp_registration_flag, 6399 __kmp_registration_flag, KMP_LIBRARY_FILE); 6400 6401 KA_TRACE(50, ("__kmp_register_library_startup: %s=\"%s\"\n", name, 6402 __kmp_registration_str)); 6403 6404 while (!done) { 6405 6406 char *value = NULL; // Actual value of the environment variable. 6407 6408 // Set environment variable, but do not overwrite if it is exist. 6409 __kmp_env_set(name, __kmp_registration_str, 0); 6410 // Check the variable is written. 6411 value = __kmp_env_get(name); 6412 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) { 6413 6414 done = 1; // Ok, environment variable set successfully, exit the loop. 6415 6416 } else { 6417 6418 // Oops. Write failed. Another copy of OpenMP RTL is in memory. 6419 // Check whether it alive or dead. 6420 int neighbor = 0; // 0 -- unknown status, 1 -- alive, 2 -- dead. 6421 char *tail = value; 6422 char *flag_addr_str = NULL; 6423 char *flag_val_str = NULL; 6424 char const *file_name = NULL; 6425 __kmp_str_split(tail, '-', &flag_addr_str, &tail); 6426 __kmp_str_split(tail, '-', &flag_val_str, &tail); 6427 file_name = tail; 6428 if (tail != NULL) { 6429 long *flag_addr = 0; 6430 long flag_val = 0; 6431 KMP_SSCANF(flag_addr_str, "%p", RCAST(void**, &flag_addr)); 6432 KMP_SSCANF(flag_val_str, "%lx", &flag_val); 6433 if (flag_addr != 0 && flag_val != 0 && strcmp(file_name, "") != 0) { 6434 // First, check whether environment-encoded address is mapped into 6435 // addr space. 6436 // If so, dereference it to see if it still has the right value. 6437 if (__kmp_is_address_mapped(flag_addr) && *flag_addr == flag_val) { 6438 neighbor = 1; 6439 } else { 6440 // If not, then we know the other copy of the library is no longer 6441 // running. 6442 neighbor = 2; 6443 } 6444 } 6445 } 6446 switch (neighbor) { 6447 case 0: // Cannot parse environment variable -- neighbor status unknown. 6448 // Assume it is the incompatible format of future version of the 6449 // library. Assume the other library is alive. 6450 // WARN( ... ); // TODO: Issue a warning. 6451 file_name = "unknown library"; 6452 KMP_FALLTHROUGH(); 6453 // Attention! Falling to the next case. That's intentional. 6454 case 1: { // Neighbor is alive. 6455 // Check it is allowed. 6456 char *duplicate_ok = __kmp_env_get("KMP_DUPLICATE_LIB_OK"); 6457 if (!__kmp_str_match_true(duplicate_ok)) { 6458 // That's not allowed. Issue fatal error. 6459 __kmp_fatal(KMP_MSG(DuplicateLibrary, KMP_LIBRARY_FILE, file_name), 6460 KMP_HNT(DuplicateLibrary), __kmp_msg_null); 6461 } 6462 KMP_INTERNAL_FREE(duplicate_ok); 6463 __kmp_duplicate_library_ok = 1; 6464 done = 1; // Exit the loop. 6465 } break; 6466 case 2: { // Neighbor is dead. 6467 // Clear the variable and try to register library again. 6468 __kmp_env_unset(name); 6469 } break; 6470 default: { KMP_DEBUG_ASSERT(0); } break; 6471 } 6472 } 6473 KMP_INTERNAL_FREE((void *)value); 6474 } 6475 KMP_INTERNAL_FREE((void *)name); 6476 6477 } // func __kmp_register_library_startup 6478 6479 void __kmp_unregister_library(void) { 6480 6481 char *name = __kmp_reg_status_name(); 6482 char *value = __kmp_env_get(name); 6483 6484 KMP_DEBUG_ASSERT(__kmp_registration_flag != 0); 6485 KMP_DEBUG_ASSERT(__kmp_registration_str != NULL); 6486 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) { 6487 // Ok, this is our variable. Delete it. 6488 __kmp_env_unset(name); 6489 } 6490 6491 KMP_INTERNAL_FREE(__kmp_registration_str); 6492 KMP_INTERNAL_FREE(value); 6493 KMP_INTERNAL_FREE(name); 6494 6495 __kmp_registration_flag = 0; 6496 __kmp_registration_str = NULL; 6497 6498 } // __kmp_unregister_library 6499 6500 // End of Library registration stuff. 6501 // ----------------------------------------------------------------------------- 6502 6503 #if KMP_MIC_SUPPORTED 6504 6505 static void __kmp_check_mic_type() { 6506 kmp_cpuid_t cpuid_state = {0}; 6507 kmp_cpuid_t *cs_p = &cpuid_state; 6508 __kmp_x86_cpuid(1, 0, cs_p); 6509 // We don't support mic1 at the moment 6510 if ((cs_p->eax & 0xff0) == 0xB10) { 6511 __kmp_mic_type = mic2; 6512 } else if ((cs_p->eax & 0xf0ff0) == 0x50670) { 6513 __kmp_mic_type = mic3; 6514 } else { 6515 __kmp_mic_type = non_mic; 6516 } 6517 } 6518 6519 #endif /* KMP_MIC_SUPPORTED */ 6520 6521 static void __kmp_do_serial_initialize(void) { 6522 int i, gtid; 6523 int size; 6524 6525 KA_TRACE(10, ("__kmp_do_serial_initialize: enter\n")); 6526 6527 KMP_DEBUG_ASSERT(sizeof(kmp_int32) == 4); 6528 KMP_DEBUG_ASSERT(sizeof(kmp_uint32) == 4); 6529 KMP_DEBUG_ASSERT(sizeof(kmp_int64) == 8); 6530 KMP_DEBUG_ASSERT(sizeof(kmp_uint64) == 8); 6531 KMP_DEBUG_ASSERT(sizeof(kmp_intptr_t) == sizeof(void *)); 6532 6533 #if OMPT_SUPPORT 6534 ompt_pre_init(); 6535 #endif 6536 6537 __kmp_validate_locks(); 6538 6539 /* Initialize internal memory allocator */ 6540 __kmp_init_allocator(); 6541 6542 /* Register the library startup via an environment variable and check to see 6543 whether another copy of the library is already registered. */ 6544 6545 __kmp_register_library_startup(); 6546 6547 /* TODO reinitialization of library */ 6548 if (TCR_4(__kmp_global.g.g_done)) { 6549 KA_TRACE(10, ("__kmp_do_serial_initialize: reinitialization of library\n")); 6550 } 6551 6552 __kmp_global.g.g_abort = 0; 6553 TCW_SYNC_4(__kmp_global.g.g_done, FALSE); 6554 6555 /* initialize the locks */ 6556 #if KMP_USE_ADAPTIVE_LOCKS 6557 #if KMP_DEBUG_ADAPTIVE_LOCKS 6558 __kmp_init_speculative_stats(); 6559 #endif 6560 #endif 6561 #if KMP_STATS_ENABLED 6562 __kmp_stats_init(); 6563 #endif 6564 __kmp_init_lock(&__kmp_global_lock); 6565 __kmp_init_queuing_lock(&__kmp_dispatch_lock); 6566 __kmp_init_lock(&__kmp_debug_lock); 6567 __kmp_init_atomic_lock(&__kmp_atomic_lock); 6568 __kmp_init_atomic_lock(&__kmp_atomic_lock_1i); 6569 __kmp_init_atomic_lock(&__kmp_atomic_lock_2i); 6570 __kmp_init_atomic_lock(&__kmp_atomic_lock_4i); 6571 __kmp_init_atomic_lock(&__kmp_atomic_lock_4r); 6572 __kmp_init_atomic_lock(&__kmp_atomic_lock_8i); 6573 __kmp_init_atomic_lock(&__kmp_atomic_lock_8r); 6574 __kmp_init_atomic_lock(&__kmp_atomic_lock_8c); 6575 __kmp_init_atomic_lock(&__kmp_atomic_lock_10r); 6576 __kmp_init_atomic_lock(&__kmp_atomic_lock_16r); 6577 __kmp_init_atomic_lock(&__kmp_atomic_lock_16c); 6578 __kmp_init_atomic_lock(&__kmp_atomic_lock_20c); 6579 __kmp_init_atomic_lock(&__kmp_atomic_lock_32c); 6580 __kmp_init_bootstrap_lock(&__kmp_forkjoin_lock); 6581 __kmp_init_bootstrap_lock(&__kmp_exit_lock); 6582 #if KMP_USE_MONITOR 6583 __kmp_init_bootstrap_lock(&__kmp_monitor_lock); 6584 #endif 6585 __kmp_init_bootstrap_lock(&__kmp_tp_cached_lock); 6586 6587 /* conduct initialization and initial setup of configuration */ 6588 6589 __kmp_runtime_initialize(); 6590 6591 #if KMP_MIC_SUPPORTED 6592 __kmp_check_mic_type(); 6593 #endif 6594 6595 // Some global variable initialization moved here from kmp_env_initialize() 6596 #ifdef KMP_DEBUG 6597 kmp_diag = 0; 6598 #endif 6599 __kmp_abort_delay = 0; 6600 6601 // From __kmp_init_dflt_team_nth() 6602 /* assume the entire machine will be used */ 6603 __kmp_dflt_team_nth_ub = __kmp_xproc; 6604 if (__kmp_dflt_team_nth_ub < KMP_MIN_NTH) { 6605 __kmp_dflt_team_nth_ub = KMP_MIN_NTH; 6606 } 6607 if (__kmp_dflt_team_nth_ub > __kmp_sys_max_nth) { 6608 __kmp_dflt_team_nth_ub = __kmp_sys_max_nth; 6609 } 6610 __kmp_max_nth = __kmp_sys_max_nth; 6611 __kmp_cg_max_nth = __kmp_sys_max_nth; 6612 __kmp_teams_max_nth = __kmp_xproc; // set a "reasonable" default 6613 if (__kmp_teams_max_nth > __kmp_sys_max_nth) { 6614 __kmp_teams_max_nth = __kmp_sys_max_nth; 6615 } 6616 6617 // Three vars below moved here from __kmp_env_initialize() "KMP_BLOCKTIME" 6618 // part 6619 __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME; 6620 #if KMP_USE_MONITOR 6621 __kmp_monitor_wakeups = 6622 KMP_WAKEUPS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups); 6623 __kmp_bt_intervals = 6624 KMP_INTERVALS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups); 6625 #endif 6626 // From "KMP_LIBRARY" part of __kmp_env_initialize() 6627 __kmp_library = library_throughput; 6628 // From KMP_SCHEDULE initialization 6629 __kmp_static = kmp_sch_static_balanced; 6630 // AC: do not use analytical here, because it is non-monotonous 6631 //__kmp_guided = kmp_sch_guided_iterative_chunked; 6632 //__kmp_auto = kmp_sch_guided_analytical_chunked; // AC: it is the default, no 6633 // need to repeat assignment 6634 // Barrier initialization. Moved here from __kmp_env_initialize() Barrier branch 6635 // bit control and barrier method control parts 6636 #if KMP_FAST_REDUCTION_BARRIER 6637 #define kmp_reduction_barrier_gather_bb ((int)1) 6638 #define kmp_reduction_barrier_release_bb ((int)1) 6639 #define kmp_reduction_barrier_gather_pat bp_hyper_bar 6640 #define kmp_reduction_barrier_release_pat bp_hyper_bar 6641 #endif // KMP_FAST_REDUCTION_BARRIER 6642 for (i = bs_plain_barrier; i < bs_last_barrier; i++) { 6643 __kmp_barrier_gather_branch_bits[i] = __kmp_barrier_gather_bb_dflt; 6644 __kmp_barrier_release_branch_bits[i] = __kmp_barrier_release_bb_dflt; 6645 __kmp_barrier_gather_pattern[i] = __kmp_barrier_gather_pat_dflt; 6646 __kmp_barrier_release_pattern[i] = __kmp_barrier_release_pat_dflt; 6647 #if KMP_FAST_REDUCTION_BARRIER 6648 if (i == bs_reduction_barrier) { // tested and confirmed on ALTIX only ( 6649 // lin_64 ): hyper,1 6650 __kmp_barrier_gather_branch_bits[i] = kmp_reduction_barrier_gather_bb; 6651 __kmp_barrier_release_branch_bits[i] = kmp_reduction_barrier_release_bb; 6652 __kmp_barrier_gather_pattern[i] = kmp_reduction_barrier_gather_pat; 6653 __kmp_barrier_release_pattern[i] = kmp_reduction_barrier_release_pat; 6654 } 6655 #endif // KMP_FAST_REDUCTION_BARRIER 6656 } 6657 #if KMP_FAST_REDUCTION_BARRIER 6658 #undef kmp_reduction_barrier_release_pat 6659 #undef kmp_reduction_barrier_gather_pat 6660 #undef kmp_reduction_barrier_release_bb 6661 #undef kmp_reduction_barrier_gather_bb 6662 #endif // KMP_FAST_REDUCTION_BARRIER 6663 #if KMP_MIC_SUPPORTED 6664 if (__kmp_mic_type == mic2) { // KNC 6665 // AC: plane=3,2, forkjoin=2,1 are optimal for 240 threads on KNC 6666 __kmp_barrier_gather_branch_bits[bs_plain_barrier] = 3; // plain gather 6667 __kmp_barrier_release_branch_bits[bs_forkjoin_barrier] = 6668 1; // forkjoin release 6669 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar; 6670 __kmp_barrier_release_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar; 6671 } 6672 #if KMP_FAST_REDUCTION_BARRIER 6673 if (__kmp_mic_type == mic2) { // KNC 6674 __kmp_barrier_gather_pattern[bs_reduction_barrier] = bp_hierarchical_bar; 6675 __kmp_barrier_release_pattern[bs_reduction_barrier] = bp_hierarchical_bar; 6676 } 6677 #endif // KMP_FAST_REDUCTION_BARRIER 6678 #endif // KMP_MIC_SUPPORTED 6679 6680 // From KMP_CHECKS initialization 6681 #ifdef KMP_DEBUG 6682 __kmp_env_checks = TRUE; /* development versions have the extra checks */ 6683 #else 6684 __kmp_env_checks = FALSE; /* port versions do not have the extra checks */ 6685 #endif 6686 6687 // From "KMP_FOREIGN_THREADS_THREADPRIVATE" initialization 6688 __kmp_foreign_tp = TRUE; 6689 6690 __kmp_global.g.g_dynamic = FALSE; 6691 __kmp_global.g.g_dynamic_mode = dynamic_default; 6692 6693 __kmp_env_initialize(NULL); 6694 6695 // Print all messages in message catalog for testing purposes. 6696 #ifdef KMP_DEBUG 6697 char const *val = __kmp_env_get("KMP_DUMP_CATALOG"); 6698 if (__kmp_str_match_true(val)) { 6699 kmp_str_buf_t buffer; 6700 __kmp_str_buf_init(&buffer); 6701 __kmp_i18n_dump_catalog(&buffer); 6702 __kmp_printf("%s", buffer.str); 6703 __kmp_str_buf_free(&buffer); 6704 } 6705 __kmp_env_free(&val); 6706 #endif 6707 6708 __kmp_threads_capacity = 6709 __kmp_initial_threads_capacity(__kmp_dflt_team_nth_ub); 6710 // Moved here from __kmp_env_initialize() "KMP_ALL_THREADPRIVATE" part 6711 __kmp_tp_capacity = __kmp_default_tp_capacity( 6712 __kmp_dflt_team_nth_ub, __kmp_max_nth, __kmp_allThreadsSpecified); 6713 6714 // If the library is shut down properly, both pools must be NULL. Just in 6715 // case, set them to NULL -- some memory may leak, but subsequent code will 6716 // work even if pools are not freed. 6717 KMP_DEBUG_ASSERT(__kmp_thread_pool == NULL); 6718 KMP_DEBUG_ASSERT(__kmp_thread_pool_insert_pt == NULL); 6719 KMP_DEBUG_ASSERT(__kmp_team_pool == NULL); 6720 __kmp_thread_pool = NULL; 6721 __kmp_thread_pool_insert_pt = NULL; 6722 __kmp_team_pool = NULL; 6723 6724 /* Allocate all of the variable sized records */ 6725 /* NOTE: __kmp_threads_capacity entries are allocated, but the arrays are 6726 * expandable */ 6727 /* Since allocation is cache-aligned, just add extra padding at the end */ 6728 size = 6729 (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * __kmp_threads_capacity + 6730 CACHE_LINE; 6731 __kmp_threads = (kmp_info_t **)__kmp_allocate(size); 6732 __kmp_root = (kmp_root_t **)((char *)__kmp_threads + 6733 sizeof(kmp_info_t *) * __kmp_threads_capacity); 6734 6735 /* init thread counts */ 6736 KMP_DEBUG_ASSERT(__kmp_all_nth == 6737 0); // Asserts fail if the library is reinitializing and 6738 KMP_DEBUG_ASSERT(__kmp_nth == 0); // something was wrong in termination. 6739 __kmp_all_nth = 0; 6740 __kmp_nth = 0; 6741 6742 /* setup the uber master thread and hierarchy */ 6743 gtid = __kmp_register_root(TRUE); 6744 KA_TRACE(10, ("__kmp_do_serial_initialize T#%d\n", gtid)); 6745 KMP_ASSERT(KMP_UBER_GTID(gtid)); 6746 KMP_ASSERT(KMP_INITIAL_GTID(gtid)); 6747 6748 KMP_MB(); /* Flush all pending memory write invalidates. */ 6749 6750 __kmp_common_initialize(); 6751 6752 #if KMP_OS_UNIX 6753 /* invoke the child fork handler */ 6754 __kmp_register_atfork(); 6755 #endif 6756 6757 #if !KMP_DYNAMIC_LIB 6758 { 6759 /* Invoke the exit handler when the program finishes, only for static 6760 library. For dynamic library, we already have _fini and DllMain. */ 6761 int rc = atexit(__kmp_internal_end_atexit); 6762 if (rc != 0) { 6763 __kmp_fatal(KMP_MSG(FunctionError, "atexit()"), KMP_ERR(rc), 6764 __kmp_msg_null); 6765 } 6766 } 6767 #endif 6768 6769 #if KMP_HANDLE_SIGNALS 6770 #if KMP_OS_UNIX 6771 /* NOTE: make sure that this is called before the user installs their own 6772 signal handlers so that the user handlers are called first. this way they 6773 can return false, not call our handler, avoid terminating the library, and 6774 continue execution where they left off. */ 6775 __kmp_install_signals(FALSE); 6776 #endif /* KMP_OS_UNIX */ 6777 #if KMP_OS_WINDOWS 6778 __kmp_install_signals(TRUE); 6779 #endif /* KMP_OS_WINDOWS */ 6780 #endif 6781 6782 /* we have finished the serial initialization */ 6783 __kmp_init_counter++; 6784 6785 __kmp_init_serial = TRUE; 6786 6787 if (__kmp_settings) { 6788 __kmp_env_print(); 6789 } 6790 6791 if (__kmp_display_env || __kmp_display_env_verbose) { 6792 __kmp_env_print_2(); 6793 } 6794 6795 #if OMPT_SUPPORT 6796 ompt_post_init(); 6797 #endif 6798 6799 KMP_MB(); 6800 6801 KA_TRACE(10, ("__kmp_do_serial_initialize: exit\n")); 6802 } 6803 6804 void __kmp_serial_initialize(void) { 6805 if (__kmp_init_serial) { 6806 return; 6807 } 6808 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6809 if (__kmp_init_serial) { 6810 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6811 return; 6812 } 6813 __kmp_do_serial_initialize(); 6814 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6815 } 6816 6817 static void __kmp_do_middle_initialize(void) { 6818 int i, j; 6819 int prev_dflt_team_nth; 6820 6821 if (!__kmp_init_serial) { 6822 __kmp_do_serial_initialize(); 6823 } 6824 6825 KA_TRACE(10, ("__kmp_middle_initialize: enter\n")); 6826 6827 // Save the previous value for the __kmp_dflt_team_nth so that 6828 // we can avoid some reinitialization if it hasn't changed. 6829 prev_dflt_team_nth = __kmp_dflt_team_nth; 6830 6831 #if KMP_AFFINITY_SUPPORTED 6832 // __kmp_affinity_initialize() will try to set __kmp_ncores to the 6833 // number of cores on the machine. 6834 __kmp_affinity_initialize(); 6835 6836 // Run through the __kmp_threads array and set the affinity mask 6837 // for each root thread that is currently registered with the RTL. 6838 for (i = 0; i < __kmp_threads_capacity; i++) { 6839 if (TCR_PTR(__kmp_threads[i]) != NULL) { 6840 __kmp_affinity_set_init_mask(i, TRUE); 6841 } 6842 } 6843 #endif /* KMP_AFFINITY_SUPPORTED */ 6844 6845 KMP_ASSERT(__kmp_xproc > 0); 6846 if (__kmp_avail_proc == 0) { 6847 __kmp_avail_proc = __kmp_xproc; 6848 } 6849 6850 // If there were empty places in num_threads list (OMP_NUM_THREADS=,,2,3), 6851 // correct them now 6852 j = 0; 6853 while ((j < __kmp_nested_nth.used) && !__kmp_nested_nth.nth[j]) { 6854 __kmp_nested_nth.nth[j] = __kmp_dflt_team_nth = __kmp_dflt_team_nth_ub = 6855 __kmp_avail_proc; 6856 j++; 6857 } 6858 6859 if (__kmp_dflt_team_nth == 0) { 6860 #ifdef KMP_DFLT_NTH_CORES 6861 // Default #threads = #cores 6862 __kmp_dflt_team_nth = __kmp_ncores; 6863 KA_TRACE(20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = " 6864 "__kmp_ncores (%d)\n", 6865 __kmp_dflt_team_nth)); 6866 #else 6867 // Default #threads = #available OS procs 6868 __kmp_dflt_team_nth = __kmp_avail_proc; 6869 KA_TRACE(20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = " 6870 "__kmp_avail_proc(%d)\n", 6871 __kmp_dflt_team_nth)); 6872 #endif /* KMP_DFLT_NTH_CORES */ 6873 } 6874 6875 if (__kmp_dflt_team_nth < KMP_MIN_NTH) { 6876 __kmp_dflt_team_nth = KMP_MIN_NTH; 6877 } 6878 if (__kmp_dflt_team_nth > __kmp_sys_max_nth) { 6879 __kmp_dflt_team_nth = __kmp_sys_max_nth; 6880 } 6881 6882 // There's no harm in continuing if the following check fails, 6883 // but it indicates an error in the previous logic. 6884 KMP_DEBUG_ASSERT(__kmp_dflt_team_nth <= __kmp_dflt_team_nth_ub); 6885 6886 if (__kmp_dflt_team_nth != prev_dflt_team_nth) { 6887 // Run through the __kmp_threads array and set the num threads icv for each 6888 // root thread that is currently registered with the RTL (which has not 6889 // already explicitly set its nthreads-var with a call to 6890 // omp_set_num_threads()). 6891 for (i = 0; i < __kmp_threads_capacity; i++) { 6892 kmp_info_t *thread = __kmp_threads[i]; 6893 if (thread == NULL) 6894 continue; 6895 if (thread->th.th_current_task->td_icvs.nproc != 0) 6896 continue; 6897 6898 set__nproc(__kmp_threads[i], __kmp_dflt_team_nth); 6899 } 6900 } 6901 KA_TRACE( 6902 20, 6903 ("__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n", 6904 __kmp_dflt_team_nth)); 6905 6906 #ifdef KMP_ADJUST_BLOCKTIME 6907 /* Adjust blocktime to zero if necessary now that __kmp_avail_proc is set */ 6908 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 6909 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0); 6910 if (__kmp_nth > __kmp_avail_proc) { 6911 __kmp_zero_bt = TRUE; 6912 } 6913 } 6914 #endif /* KMP_ADJUST_BLOCKTIME */ 6915 6916 /* we have finished middle initialization */ 6917 TCW_SYNC_4(__kmp_init_middle, TRUE); 6918 6919 KA_TRACE(10, ("__kmp_do_middle_initialize: exit\n")); 6920 } 6921 6922 void __kmp_middle_initialize(void) { 6923 if (__kmp_init_middle) { 6924 return; 6925 } 6926 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6927 if (__kmp_init_middle) { 6928 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6929 return; 6930 } 6931 __kmp_do_middle_initialize(); 6932 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6933 } 6934 6935 void __kmp_parallel_initialize(void) { 6936 int gtid = __kmp_entry_gtid(); // this might be a new root 6937 6938 /* synchronize parallel initialization (for sibling) */ 6939 if (TCR_4(__kmp_init_parallel)) 6940 return; 6941 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6942 if (TCR_4(__kmp_init_parallel)) { 6943 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6944 return; 6945 } 6946 6947 /* TODO reinitialization after we have already shut down */ 6948 if (TCR_4(__kmp_global.g.g_done)) { 6949 KA_TRACE( 6950 10, 6951 ("__kmp_parallel_initialize: attempt to init while shutting down\n")); 6952 __kmp_infinite_loop(); 6953 } 6954 6955 /* jc: The lock __kmp_initz_lock is already held, so calling 6956 __kmp_serial_initialize would cause a deadlock. So we call 6957 __kmp_do_serial_initialize directly. */ 6958 if (!__kmp_init_middle) { 6959 __kmp_do_middle_initialize(); 6960 } 6961 __kmp_resume_if_hard_paused(); 6962 6963 /* begin initialization */ 6964 KA_TRACE(10, ("__kmp_parallel_initialize: enter\n")); 6965 KMP_ASSERT(KMP_UBER_GTID(gtid)); 6966 6967 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 6968 // Save the FP control regs. 6969 // Worker threads will set theirs to these values at thread startup. 6970 __kmp_store_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word); 6971 __kmp_store_mxcsr(&__kmp_init_mxcsr); 6972 __kmp_init_mxcsr &= KMP_X86_MXCSR_MASK; 6973 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 6974 6975 #if KMP_OS_UNIX 6976 #if KMP_HANDLE_SIGNALS 6977 /* must be after __kmp_serial_initialize */ 6978 __kmp_install_signals(TRUE); 6979 #endif 6980 #endif 6981 6982 __kmp_suspend_initialize(); 6983 6984 #if defined(USE_LOAD_BALANCE) 6985 if (__kmp_global.g.g_dynamic_mode == dynamic_default) { 6986 __kmp_global.g.g_dynamic_mode = dynamic_load_balance; 6987 } 6988 #else 6989 if (__kmp_global.g.g_dynamic_mode == dynamic_default) { 6990 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit; 6991 } 6992 #endif 6993 6994 if (__kmp_version) { 6995 __kmp_print_version_2(); 6996 } 6997 6998 /* we have finished parallel initialization */ 6999 TCW_SYNC_4(__kmp_init_parallel, TRUE); 7000 7001 KMP_MB(); 7002 KA_TRACE(10, ("__kmp_parallel_initialize: exit\n")); 7003 7004 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 7005 } 7006 7007 /* ------------------------------------------------------------------------ */ 7008 7009 void __kmp_run_before_invoked_task(int gtid, int tid, kmp_info_t *this_thr, 7010 kmp_team_t *team) { 7011 kmp_disp_t *dispatch; 7012 7013 KMP_MB(); 7014 7015 /* none of the threads have encountered any constructs, yet. */ 7016 this_thr->th.th_local.this_construct = 0; 7017 #if KMP_CACHE_MANAGE 7018 KMP_CACHE_PREFETCH(&this_thr->th.th_bar[bs_forkjoin_barrier].bb.b_arrived); 7019 #endif /* KMP_CACHE_MANAGE */ 7020 dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch); 7021 KMP_DEBUG_ASSERT(dispatch); 7022 KMP_DEBUG_ASSERT(team->t.t_dispatch); 7023 // KMP_DEBUG_ASSERT( this_thr->th.th_dispatch == &team->t.t_dispatch[ 7024 // this_thr->th.th_info.ds.ds_tid ] ); 7025 7026 dispatch->th_disp_index = 0; /* reset the dispatch buffer counter */ 7027 dispatch->th_doacross_buf_idx = 0; // reset doacross dispatch buffer counter 7028 if (__kmp_env_consistency_check) 7029 __kmp_push_parallel(gtid, team->t.t_ident); 7030 7031 KMP_MB(); /* Flush all pending memory write invalidates. */ 7032 } 7033 7034 void __kmp_run_after_invoked_task(int gtid, int tid, kmp_info_t *this_thr, 7035 kmp_team_t *team) { 7036 if (__kmp_env_consistency_check) 7037 __kmp_pop_parallel(gtid, team->t.t_ident); 7038 7039 __kmp_finish_implicit_task(this_thr); 7040 } 7041 7042 int __kmp_invoke_task_func(int gtid) { 7043 int rc; 7044 int tid = __kmp_tid_from_gtid(gtid); 7045 kmp_info_t *this_thr = __kmp_threads[gtid]; 7046 kmp_team_t *team = this_thr->th.th_team; 7047 7048 __kmp_run_before_invoked_task(gtid, tid, this_thr, team); 7049 #if USE_ITT_BUILD 7050 if (__itt_stack_caller_create_ptr) { 7051 __kmp_itt_stack_callee_enter( 7052 (__itt_caller) 7053 team->t.t_stack_id); // inform ittnotify about entering user's code 7054 } 7055 #endif /* USE_ITT_BUILD */ 7056 #if INCLUDE_SSC_MARKS 7057 SSC_MARK_INVOKING(); 7058 #endif 7059 7060 #if OMPT_SUPPORT 7061 void *dummy; 7062 void **exit_frame_p; 7063 ompt_data_t *my_task_data; 7064 ompt_data_t *my_parallel_data; 7065 int ompt_team_size; 7066 7067 if (ompt_enabled.enabled) { 7068 exit_frame_p = &( 7069 team->t.t_implicit_task_taskdata[tid].ompt_task_info.frame.exit_frame.ptr); 7070 } else { 7071 exit_frame_p = &dummy; 7072 } 7073 7074 my_task_data = 7075 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data); 7076 my_parallel_data = &(team->t.ompt_team_info.parallel_data); 7077 if (ompt_enabled.ompt_callback_implicit_task) { 7078 ompt_team_size = team->t.t_nproc; 7079 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 7080 ompt_scope_begin, my_parallel_data, my_task_data, ompt_team_size, 7081 __kmp_tid_from_gtid(gtid), ompt_task_implicit); 7082 OMPT_CUR_TASK_INFO(this_thr)->thread_num = __kmp_tid_from_gtid(gtid); 7083 } 7084 #endif 7085 7086 #if KMP_STATS_ENABLED 7087 stats_state_e previous_state = KMP_GET_THREAD_STATE(); 7088 if (previous_state == stats_state_e::TEAMS_REGION) { 7089 KMP_PUSH_PARTITIONED_TIMER(OMP_teams); 7090 } else { 7091 KMP_PUSH_PARTITIONED_TIMER(OMP_parallel); 7092 } 7093 KMP_SET_THREAD_STATE(IMPLICIT_TASK); 7094 #endif 7095 7096 rc = __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn), gtid, 7097 tid, (int)team->t.t_argc, (void **)team->t.t_argv 7098 #if OMPT_SUPPORT 7099 , 7100 exit_frame_p 7101 #endif 7102 ); 7103 #if OMPT_SUPPORT 7104 *exit_frame_p = NULL; 7105 this_thr->th.ompt_thread_info.parallel_flags |= ompt_parallel_team; 7106 #endif 7107 7108 #if KMP_STATS_ENABLED 7109 if (previous_state == stats_state_e::TEAMS_REGION) { 7110 KMP_SET_THREAD_STATE(previous_state); 7111 } 7112 KMP_POP_PARTITIONED_TIMER(); 7113 #endif 7114 7115 #if USE_ITT_BUILD 7116 if (__itt_stack_caller_create_ptr) { 7117 __kmp_itt_stack_callee_leave( 7118 (__itt_caller) 7119 team->t.t_stack_id); // inform ittnotify about leaving user's code 7120 } 7121 #endif /* USE_ITT_BUILD */ 7122 __kmp_run_after_invoked_task(gtid, tid, this_thr, team); 7123 7124 return rc; 7125 } 7126 7127 void __kmp_teams_master(int gtid) { 7128 // This routine is called by all master threads in teams construct 7129 kmp_info_t *thr = __kmp_threads[gtid]; 7130 kmp_team_t *team = thr->th.th_team; 7131 ident_t *loc = team->t.t_ident; 7132 thr->th.th_set_nproc = thr->th.th_teams_size.nth; 7133 KMP_DEBUG_ASSERT(thr->th.th_teams_microtask); 7134 KMP_DEBUG_ASSERT(thr->th.th_set_nproc); 7135 KA_TRACE(20, ("__kmp_teams_master: T#%d, Tid %d, microtask %p\n", gtid, 7136 __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask)); 7137 7138 // This thread is a new CG root. Set up the proper variables. 7139 kmp_cg_root_t *tmp = (kmp_cg_root_t *)__kmp_allocate(sizeof(kmp_cg_root_t)); 7140 tmp->cg_root = thr; // Make thr the CG root 7141 // Init to thread limit that was stored when league masters were forked 7142 tmp->cg_thread_limit = thr->th.th_current_task->td_icvs.thread_limit; 7143 tmp->cg_nthreads = 1; // Init counter to one active thread, this one 7144 KA_TRACE(100, ("__kmp_teams_master: Thread %p created node %p and init" 7145 " cg_nthreads to 1\n", 7146 thr, tmp)); 7147 tmp->up = thr->th.th_cg_roots; 7148 thr->th.th_cg_roots = tmp; 7149 7150 // Launch league of teams now, but not let workers execute 7151 // (they hang on fork barrier until next parallel) 7152 #if INCLUDE_SSC_MARKS 7153 SSC_MARK_FORKING(); 7154 #endif 7155 __kmp_fork_call(loc, gtid, fork_context_intel, team->t.t_argc, 7156 (microtask_t)thr->th.th_teams_microtask, // "wrapped" task 7157 VOLATILE_CAST(launch_t) __kmp_invoke_task_func, NULL); 7158 #if INCLUDE_SSC_MARKS 7159 SSC_MARK_JOINING(); 7160 #endif 7161 // If the team size was reduced from the limit, set it to the new size 7162 if (thr->th.th_team_nproc < thr->th.th_teams_size.nth) 7163 thr->th.th_teams_size.nth = thr->th.th_team_nproc; 7164 // AC: last parameter "1" eliminates join barrier which won't work because 7165 // worker threads are in a fork barrier waiting for more parallel regions 7166 __kmp_join_call(loc, gtid 7167 #if OMPT_SUPPORT 7168 , 7169 fork_context_intel 7170 #endif 7171 , 7172 1); 7173 } 7174 7175 int __kmp_invoke_teams_master(int gtid) { 7176 kmp_info_t *this_thr = __kmp_threads[gtid]; 7177 kmp_team_t *team = this_thr->th.th_team; 7178 #if KMP_DEBUG 7179 if (!__kmp_threads[gtid]->th.th_team->t.t_serialized) 7180 KMP_DEBUG_ASSERT((void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn == 7181 (void *)__kmp_teams_master); 7182 #endif 7183 __kmp_run_before_invoked_task(gtid, 0, this_thr, team); 7184 #if OMPT_SUPPORT 7185 int tid = __kmp_tid_from_gtid(gtid); 7186 ompt_data_t *task_data = 7187 &team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data; 7188 ompt_data_t *parallel_data = &team->t.ompt_team_info.parallel_data; 7189 if (ompt_enabled.ompt_callback_implicit_task) { 7190 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 7191 ompt_scope_begin, parallel_data, task_data, team->t.t_nproc, tid, 7192 ompt_task_initial); 7193 OMPT_CUR_TASK_INFO(this_thr)->thread_num = tid; 7194 } 7195 #endif 7196 __kmp_teams_master(gtid); 7197 #if OMPT_SUPPORT 7198 this_thr->th.ompt_thread_info.parallel_flags |= ompt_parallel_league; 7199 #endif 7200 __kmp_run_after_invoked_task(gtid, 0, this_thr, team); 7201 return 1; 7202 } 7203 7204 /* this sets the requested number of threads for the next parallel region 7205 encountered by this team. since this should be enclosed in the forkjoin 7206 critical section it should avoid race conditions with asymmetrical nested 7207 parallelism */ 7208 7209 void __kmp_push_num_threads(ident_t *id, int gtid, int num_threads) { 7210 kmp_info_t *thr = __kmp_threads[gtid]; 7211 7212 if (num_threads > 0) 7213 thr->th.th_set_nproc = num_threads; 7214 } 7215 7216 /* this sets the requested number of teams for the teams region and/or 7217 the number of threads for the next parallel region encountered */ 7218 void __kmp_push_num_teams(ident_t *id, int gtid, int num_teams, 7219 int num_threads) { 7220 kmp_info_t *thr = __kmp_threads[gtid]; 7221 KMP_DEBUG_ASSERT(num_teams >= 0); 7222 KMP_DEBUG_ASSERT(num_threads >= 0); 7223 7224 if (num_teams == 0) 7225 num_teams = 1; // default number of teams is 1. 7226 if (num_teams > __kmp_teams_max_nth) { // if too many teams requested? 7227 if (!__kmp_reserve_warn) { 7228 __kmp_reserve_warn = 1; 7229 __kmp_msg(kmp_ms_warning, 7230 KMP_MSG(CantFormThrTeam, num_teams, __kmp_teams_max_nth), 7231 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 7232 } 7233 num_teams = __kmp_teams_max_nth; 7234 } 7235 // Set number of teams (number of threads in the outer "parallel" of the 7236 // teams) 7237 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams; 7238 7239 // Remember the number of threads for inner parallel regions 7240 if (!TCR_4(__kmp_init_middle)) 7241 __kmp_middle_initialize(); // get internal globals calculated 7242 KMP_DEBUG_ASSERT(__kmp_avail_proc); 7243 KMP_DEBUG_ASSERT(__kmp_dflt_team_nth); 7244 if (num_threads == 0) { 7245 num_threads = __kmp_avail_proc / num_teams; 7246 // adjust num_threads w/o warning as it is not user setting 7247 // num_threads = min(num_threads, nthreads-var, thread-limit-var) 7248 // no thread_limit clause specified - do not change thread-limit-var ICV 7249 if (num_threads > __kmp_dflt_team_nth) { 7250 num_threads = __kmp_dflt_team_nth; // honor nthreads-var ICV 7251 } 7252 if (num_threads > thr->th.th_current_task->td_icvs.thread_limit) { 7253 num_threads = thr->th.th_current_task->td_icvs.thread_limit; 7254 } // prevent team size to exceed thread-limit-var 7255 if (num_teams * num_threads > __kmp_teams_max_nth) { 7256 num_threads = __kmp_teams_max_nth / num_teams; 7257 } 7258 } else { 7259 // This thread will be the master of the league masters 7260 // Store new thread limit; old limit is saved in th_cg_roots list 7261 thr->th.th_current_task->td_icvs.thread_limit = num_threads; 7262 // num_threads = min(num_threads, nthreads-var) 7263 if (num_threads > __kmp_dflt_team_nth) { 7264 num_threads = __kmp_dflt_team_nth; // honor nthreads-var ICV 7265 } 7266 if (num_teams * num_threads > __kmp_teams_max_nth) { 7267 int new_threads = __kmp_teams_max_nth / num_teams; 7268 if (!__kmp_reserve_warn) { // user asked for too many threads 7269 __kmp_reserve_warn = 1; // conflicts with KMP_TEAMS_THREAD_LIMIT 7270 __kmp_msg(kmp_ms_warning, 7271 KMP_MSG(CantFormThrTeam, num_threads, new_threads), 7272 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 7273 } 7274 num_threads = new_threads; 7275 } 7276 } 7277 thr->th.th_teams_size.nth = num_threads; 7278 } 7279 7280 // Set the proc_bind var to use in the following parallel region. 7281 void __kmp_push_proc_bind(ident_t *id, int gtid, kmp_proc_bind_t proc_bind) { 7282 kmp_info_t *thr = __kmp_threads[gtid]; 7283 thr->th.th_set_proc_bind = proc_bind; 7284 } 7285 7286 /* Launch the worker threads into the microtask. */ 7287 7288 void __kmp_internal_fork(ident_t *id, int gtid, kmp_team_t *team) { 7289 kmp_info_t *this_thr = __kmp_threads[gtid]; 7290 7291 #ifdef KMP_DEBUG 7292 int f; 7293 #endif /* KMP_DEBUG */ 7294 7295 KMP_DEBUG_ASSERT(team); 7296 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); 7297 KMP_ASSERT(KMP_MASTER_GTID(gtid)); 7298 KMP_MB(); /* Flush all pending memory write invalidates. */ 7299 7300 team->t.t_construct = 0; /* no single directives seen yet */ 7301 team->t.t_ordered.dt.t_value = 7302 0; /* thread 0 enters the ordered section first */ 7303 7304 /* Reset the identifiers on the dispatch buffer */ 7305 KMP_DEBUG_ASSERT(team->t.t_disp_buffer); 7306 if (team->t.t_max_nproc > 1) { 7307 int i; 7308 for (i = 0; i < __kmp_dispatch_num_buffers; ++i) { 7309 team->t.t_disp_buffer[i].buffer_index = i; 7310 team->t.t_disp_buffer[i].doacross_buf_idx = i; 7311 } 7312 } else { 7313 team->t.t_disp_buffer[0].buffer_index = 0; 7314 team->t.t_disp_buffer[0].doacross_buf_idx = 0; 7315 } 7316 7317 KMP_MB(); /* Flush all pending memory write invalidates. */ 7318 KMP_ASSERT(this_thr->th.th_team == team); 7319 7320 #ifdef KMP_DEBUG 7321 for (f = 0; f < team->t.t_nproc; f++) { 7322 KMP_DEBUG_ASSERT(team->t.t_threads[f] && 7323 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc); 7324 } 7325 #endif /* KMP_DEBUG */ 7326 7327 /* release the worker threads so they may begin working */ 7328 __kmp_fork_barrier(gtid, 0); 7329 } 7330 7331 void __kmp_internal_join(ident_t *id, int gtid, kmp_team_t *team) { 7332 kmp_info_t *this_thr = __kmp_threads[gtid]; 7333 7334 KMP_DEBUG_ASSERT(team); 7335 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); 7336 KMP_ASSERT(KMP_MASTER_GTID(gtid)); 7337 KMP_MB(); /* Flush all pending memory write invalidates. */ 7338 7339 /* Join barrier after fork */ 7340 7341 #ifdef KMP_DEBUG 7342 if (__kmp_threads[gtid] && 7343 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) { 7344 __kmp_printf("GTID: %d, __kmp_threads[%d]=%p\n", gtid, gtid, 7345 __kmp_threads[gtid]); 7346 __kmp_printf("__kmp_threads[%d]->th.th_team_nproc=%d, TEAM: %p, " 7347 "team->t.t_nproc=%d\n", 7348 gtid, __kmp_threads[gtid]->th.th_team_nproc, team, 7349 team->t.t_nproc); 7350 __kmp_print_structure(); 7351 } 7352 KMP_DEBUG_ASSERT(__kmp_threads[gtid] && 7353 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc); 7354 #endif /* KMP_DEBUG */ 7355 7356 __kmp_join_barrier(gtid); /* wait for everyone */ 7357 #if OMPT_SUPPORT 7358 if (ompt_enabled.enabled && 7359 this_thr->th.ompt_thread_info.state == ompt_state_wait_barrier_implicit) { 7360 int ds_tid = this_thr->th.th_info.ds.ds_tid; 7361 ompt_data_t *task_data = OMPT_CUR_TASK_DATA(this_thr); 7362 this_thr->th.ompt_thread_info.state = ompt_state_overhead; 7363 #if OMPT_OPTIONAL 7364 void *codeptr = NULL; 7365 if (KMP_MASTER_TID(ds_tid) && 7366 (ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait) || 7367 ompt_callbacks.ompt_callback(ompt_callback_sync_region))) 7368 codeptr = OMPT_CUR_TEAM_INFO(this_thr)->master_return_address; 7369 7370 if (ompt_enabled.ompt_callback_sync_region_wait) { 7371 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 7372 ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data, 7373 codeptr); 7374 } 7375 if (ompt_enabled.ompt_callback_sync_region) { 7376 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 7377 ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data, 7378 codeptr); 7379 } 7380 #endif 7381 if (!KMP_MASTER_TID(ds_tid) && ompt_enabled.ompt_callback_implicit_task) { 7382 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 7383 ompt_scope_end, NULL, task_data, 0, ds_tid, ompt_task_implicit); // TODO: Can this be ompt_task_initial? 7384 } 7385 } 7386 #endif 7387 7388 KMP_MB(); /* Flush all pending memory write invalidates. */ 7389 KMP_ASSERT(this_thr->th.th_team == team); 7390 } 7391 7392 /* ------------------------------------------------------------------------ */ 7393 7394 #ifdef USE_LOAD_BALANCE 7395 7396 // Return the worker threads actively spinning in the hot team, if we 7397 // are at the outermost level of parallelism. Otherwise, return 0. 7398 static int __kmp_active_hot_team_nproc(kmp_root_t *root) { 7399 int i; 7400 int retval; 7401 kmp_team_t *hot_team; 7402 7403 if (root->r.r_active) { 7404 return 0; 7405 } 7406 hot_team = root->r.r_hot_team; 7407 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) { 7408 return hot_team->t.t_nproc - 1; // Don't count master thread 7409 } 7410 7411 // Skip the master thread - it is accounted for elsewhere. 7412 retval = 0; 7413 for (i = 1; i < hot_team->t.t_nproc; i++) { 7414 if (hot_team->t.t_threads[i]->th.th_active) { 7415 retval++; 7416 } 7417 } 7418 return retval; 7419 } 7420 7421 // Perform an automatic adjustment to the number of 7422 // threads used by the next parallel region. 7423 static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc) { 7424 int retval; 7425 int pool_active; 7426 int hot_team_active; 7427 int team_curr_active; 7428 int system_active; 7429 7430 KB_TRACE(20, ("__kmp_load_balance_nproc: called root:%p set_nproc:%d\n", root, 7431 set_nproc)); 7432 KMP_DEBUG_ASSERT(root); 7433 KMP_DEBUG_ASSERT(root->r.r_root_team->t.t_threads[0] 7434 ->th.th_current_task->td_icvs.dynamic == TRUE); 7435 KMP_DEBUG_ASSERT(set_nproc > 1); 7436 7437 if (set_nproc == 1) { 7438 KB_TRACE(20, ("__kmp_load_balance_nproc: serial execution.\n")); 7439 return 1; 7440 } 7441 7442 // Threads that are active in the thread pool, active in the hot team for this 7443 // particular root (if we are at the outer par level), and the currently 7444 // executing thread (to become the master) are available to add to the new 7445 // team, but are currently contributing to the system load, and must be 7446 // accounted for. 7447 pool_active = __kmp_thread_pool_active_nth; 7448 hot_team_active = __kmp_active_hot_team_nproc(root); 7449 team_curr_active = pool_active + hot_team_active + 1; 7450 7451 // Check the system load. 7452 system_active = __kmp_get_load_balance(__kmp_avail_proc + team_curr_active); 7453 KB_TRACE(30, ("__kmp_load_balance_nproc: system active = %d pool active = %d " 7454 "hot team active = %d\n", 7455 system_active, pool_active, hot_team_active)); 7456 7457 if (system_active < 0) { 7458 // There was an error reading the necessary info from /proc, so use the 7459 // thread limit algorithm instead. Once we set __kmp_global.g.g_dynamic_mode 7460 // = dynamic_thread_limit, we shouldn't wind up getting back here. 7461 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit; 7462 KMP_WARNING(CantLoadBalUsing, "KMP_DYNAMIC_MODE=thread limit"); 7463 7464 // Make this call behave like the thread limit algorithm. 7465 retval = __kmp_avail_proc - __kmp_nth + 7466 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc); 7467 if (retval > set_nproc) { 7468 retval = set_nproc; 7469 } 7470 if (retval < KMP_MIN_NTH) { 7471 retval = KMP_MIN_NTH; 7472 } 7473 7474 KB_TRACE(20, ("__kmp_load_balance_nproc: thread limit exit. retval:%d\n", 7475 retval)); 7476 return retval; 7477 } 7478 7479 // There is a slight delay in the load balance algorithm in detecting new 7480 // running procs. The real system load at this instant should be at least as 7481 // large as the #active omp thread that are available to add to the team. 7482 if (system_active < team_curr_active) { 7483 system_active = team_curr_active; 7484 } 7485 retval = __kmp_avail_proc - system_active + team_curr_active; 7486 if (retval > set_nproc) { 7487 retval = set_nproc; 7488 } 7489 if (retval < KMP_MIN_NTH) { 7490 retval = KMP_MIN_NTH; 7491 } 7492 7493 KB_TRACE(20, ("__kmp_load_balance_nproc: exit. retval:%d\n", retval)); 7494 return retval; 7495 } // __kmp_load_balance_nproc() 7496 7497 #endif /* USE_LOAD_BALANCE */ 7498 7499 /* ------------------------------------------------------------------------ */ 7500 7501 /* NOTE: this is called with the __kmp_init_lock held */ 7502 void __kmp_cleanup(void) { 7503 int f; 7504 7505 KA_TRACE(10, ("__kmp_cleanup: enter\n")); 7506 7507 if (TCR_4(__kmp_init_parallel)) { 7508 #if KMP_HANDLE_SIGNALS 7509 __kmp_remove_signals(); 7510 #endif 7511 TCW_4(__kmp_init_parallel, FALSE); 7512 } 7513 7514 if (TCR_4(__kmp_init_middle)) { 7515 #if KMP_AFFINITY_SUPPORTED 7516 __kmp_affinity_uninitialize(); 7517 #endif /* KMP_AFFINITY_SUPPORTED */ 7518 __kmp_cleanup_hierarchy(); 7519 TCW_4(__kmp_init_middle, FALSE); 7520 } 7521 7522 KA_TRACE(10, ("__kmp_cleanup: go serial cleanup\n")); 7523 7524 if (__kmp_init_serial) { 7525 __kmp_runtime_destroy(); 7526 __kmp_init_serial = FALSE; 7527 } 7528 7529 __kmp_cleanup_threadprivate_caches(); 7530 7531 for (f = 0; f < __kmp_threads_capacity; f++) { 7532 if (__kmp_root[f] != NULL) { 7533 __kmp_free(__kmp_root[f]); 7534 __kmp_root[f] = NULL; 7535 } 7536 } 7537 __kmp_free(__kmp_threads); 7538 // __kmp_threads and __kmp_root were allocated at once, as single block, so 7539 // there is no need in freeing __kmp_root. 7540 __kmp_threads = NULL; 7541 __kmp_root = NULL; 7542 __kmp_threads_capacity = 0; 7543 7544 #if KMP_USE_DYNAMIC_LOCK 7545 __kmp_cleanup_indirect_user_locks(); 7546 #else 7547 __kmp_cleanup_user_locks(); 7548 #endif 7549 7550 #if KMP_AFFINITY_SUPPORTED 7551 KMP_INTERNAL_FREE(CCAST(char *, __kmp_cpuinfo_file)); 7552 __kmp_cpuinfo_file = NULL; 7553 #endif /* KMP_AFFINITY_SUPPORTED */ 7554 7555 #if KMP_USE_ADAPTIVE_LOCKS 7556 #if KMP_DEBUG_ADAPTIVE_LOCKS 7557 __kmp_print_speculative_stats(); 7558 #endif 7559 #endif 7560 KMP_INTERNAL_FREE(__kmp_nested_nth.nth); 7561 __kmp_nested_nth.nth = NULL; 7562 __kmp_nested_nth.size = 0; 7563 __kmp_nested_nth.used = 0; 7564 KMP_INTERNAL_FREE(__kmp_nested_proc_bind.bind_types); 7565 __kmp_nested_proc_bind.bind_types = NULL; 7566 __kmp_nested_proc_bind.size = 0; 7567 __kmp_nested_proc_bind.used = 0; 7568 if (__kmp_affinity_format) { 7569 KMP_INTERNAL_FREE(__kmp_affinity_format); 7570 __kmp_affinity_format = NULL; 7571 } 7572 7573 __kmp_i18n_catclose(); 7574 7575 #if KMP_USE_HIER_SCHED 7576 __kmp_hier_scheds.deallocate(); 7577 #endif 7578 7579 #if KMP_STATS_ENABLED 7580 __kmp_stats_fini(); 7581 #endif 7582 7583 KA_TRACE(10, ("__kmp_cleanup: exit\n")); 7584 } 7585 7586 /* ------------------------------------------------------------------------ */ 7587 7588 int __kmp_ignore_mppbeg(void) { 7589 char *env; 7590 7591 if ((env = getenv("KMP_IGNORE_MPPBEG")) != NULL) { 7592 if (__kmp_str_match_false(env)) 7593 return FALSE; 7594 } 7595 // By default __kmpc_begin() is no-op. 7596 return TRUE; 7597 } 7598 7599 int __kmp_ignore_mppend(void) { 7600 char *env; 7601 7602 if ((env = getenv("KMP_IGNORE_MPPEND")) != NULL) { 7603 if (__kmp_str_match_false(env)) 7604 return FALSE; 7605 } 7606 // By default __kmpc_end() is no-op. 7607 return TRUE; 7608 } 7609 7610 void __kmp_internal_begin(void) { 7611 int gtid; 7612 kmp_root_t *root; 7613 7614 /* this is a very important step as it will register new sibling threads 7615 and assign these new uber threads a new gtid */ 7616 gtid = __kmp_entry_gtid(); 7617 root = __kmp_threads[gtid]->th.th_root; 7618 KMP_ASSERT(KMP_UBER_GTID(gtid)); 7619 7620 if (root->r.r_begin) 7621 return; 7622 __kmp_acquire_lock(&root->r.r_begin_lock, gtid); 7623 if (root->r.r_begin) { 7624 __kmp_release_lock(&root->r.r_begin_lock, gtid); 7625 return; 7626 } 7627 7628 root->r.r_begin = TRUE; 7629 7630 __kmp_release_lock(&root->r.r_begin_lock, gtid); 7631 } 7632 7633 /* ------------------------------------------------------------------------ */ 7634 7635 void __kmp_user_set_library(enum library_type arg) { 7636 int gtid; 7637 kmp_root_t *root; 7638 kmp_info_t *thread; 7639 7640 /* first, make sure we are initialized so we can get our gtid */ 7641 7642 gtid = __kmp_entry_gtid(); 7643 thread = __kmp_threads[gtid]; 7644 7645 root = thread->th.th_root; 7646 7647 KA_TRACE(20, ("__kmp_user_set_library: enter T#%d, arg: %d, %d\n", gtid, arg, 7648 library_serial)); 7649 if (root->r.r_in_parallel) { /* Must be called in serial section of top-level 7650 thread */ 7651 KMP_WARNING(SetLibraryIncorrectCall); 7652 return; 7653 } 7654 7655 switch (arg) { 7656 case library_serial: 7657 thread->th.th_set_nproc = 0; 7658 set__nproc(thread, 1); 7659 break; 7660 case library_turnaround: 7661 thread->th.th_set_nproc = 0; 7662 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth 7663 : __kmp_dflt_team_nth_ub); 7664 break; 7665 case library_throughput: 7666 thread->th.th_set_nproc = 0; 7667 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth 7668 : __kmp_dflt_team_nth_ub); 7669 break; 7670 default: 7671 KMP_FATAL(UnknownLibraryType, arg); 7672 } 7673 7674 __kmp_aux_set_library(arg); 7675 } 7676 7677 void __kmp_aux_set_stacksize(size_t arg) { 7678 if (!__kmp_init_serial) 7679 __kmp_serial_initialize(); 7680 7681 #if KMP_OS_DARWIN 7682 if (arg & (0x1000 - 1)) { 7683 arg &= ~(0x1000 - 1); 7684 if (arg + 0x1000) /* check for overflow if we round up */ 7685 arg += 0x1000; 7686 } 7687 #endif 7688 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 7689 7690 /* only change the default stacksize before the first parallel region */ 7691 if (!TCR_4(__kmp_init_parallel)) { 7692 size_t value = arg; /* argument is in bytes */ 7693 7694 if (value < __kmp_sys_min_stksize) 7695 value = __kmp_sys_min_stksize; 7696 else if (value > KMP_MAX_STKSIZE) 7697 value = KMP_MAX_STKSIZE; 7698 7699 __kmp_stksize = value; 7700 7701 __kmp_env_stksize = TRUE; /* was KMP_STACKSIZE specified? */ 7702 } 7703 7704 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 7705 } 7706 7707 /* set the behaviour of the runtime library */ 7708 /* TODO this can cause some odd behaviour with sibling parallelism... */ 7709 void __kmp_aux_set_library(enum library_type arg) { 7710 __kmp_library = arg; 7711 7712 switch (__kmp_library) { 7713 case library_serial: { 7714 KMP_INFORM(LibraryIsSerial); 7715 } break; 7716 case library_turnaround: 7717 if (__kmp_use_yield == 1 && !__kmp_use_yield_exp_set) 7718 __kmp_use_yield = 2; // only yield when oversubscribed 7719 break; 7720 case library_throughput: 7721 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) 7722 __kmp_dflt_blocktime = 200; 7723 break; 7724 default: 7725 KMP_FATAL(UnknownLibraryType, arg); 7726 } 7727 } 7728 7729 /* Getting team information common for all team API */ 7730 // Returns NULL if not in teams construct 7731 static kmp_team_t *__kmp_aux_get_team_info(int &teams_serialized) { 7732 kmp_info_t *thr = __kmp_entry_thread(); 7733 teams_serialized = 0; 7734 if (thr->th.th_teams_microtask) { 7735 kmp_team_t *team = thr->th.th_team; 7736 int tlevel = thr->th.th_teams_level; // the level of the teams construct 7737 int ii = team->t.t_level; 7738 teams_serialized = team->t.t_serialized; 7739 int level = tlevel + 1; 7740 KMP_DEBUG_ASSERT(ii >= tlevel); 7741 while (ii > level) { 7742 for (teams_serialized = team->t.t_serialized; 7743 (teams_serialized > 0) && (ii > level); teams_serialized--, ii--) { 7744 } 7745 if (team->t.t_serialized && (!teams_serialized)) { 7746 team = team->t.t_parent; 7747 continue; 7748 } 7749 if (ii > level) { 7750 team = team->t.t_parent; 7751 ii--; 7752 } 7753 } 7754 return team; 7755 } 7756 return NULL; 7757 } 7758 7759 int __kmp_aux_get_team_num() { 7760 int serialized; 7761 kmp_team_t *team = __kmp_aux_get_team_info(serialized); 7762 if (team) { 7763 if (serialized > 1) { 7764 return 0; // teams region is serialized ( 1 team of 1 thread ). 7765 } else { 7766 return team->t.t_master_tid; 7767 } 7768 } 7769 return 0; 7770 } 7771 7772 int __kmp_aux_get_num_teams() { 7773 int serialized; 7774 kmp_team_t *team = __kmp_aux_get_team_info(serialized); 7775 if (team) { 7776 if (serialized > 1) { 7777 return 1; 7778 } else { 7779 return team->t.t_parent->t.t_nproc; 7780 } 7781 } 7782 return 1; 7783 } 7784 7785 /* ------------------------------------------------------------------------ */ 7786 7787 /* 7788 * Affinity Format Parser 7789 * 7790 * Field is in form of: %[[[0].]size]type 7791 * % and type are required (%% means print a literal '%') 7792 * type is either single char or long name surrounded by {}, 7793 * e.g., N or {num_threads} 7794 * 0 => leading zeros 7795 * . => right justified when size is specified 7796 * by default output is left justified 7797 * size is the *minimum* field length 7798 * All other characters are printed as is 7799 * 7800 * Available field types: 7801 * L {thread_level} - omp_get_level() 7802 * n {thread_num} - omp_get_thread_num() 7803 * h {host} - name of host machine 7804 * P {process_id} - process id (integer) 7805 * T {thread_identifier} - native thread identifier (integer) 7806 * N {num_threads} - omp_get_num_threads() 7807 * A {ancestor_tnum} - omp_get_ancestor_thread_num(omp_get_level()-1) 7808 * a {thread_affinity} - comma separated list of integers or integer ranges 7809 * (values of affinity mask) 7810 * 7811 * Implementation-specific field types can be added 7812 * If a type is unknown, print "undefined" 7813 */ 7814 7815 // Structure holding the short name, long name, and corresponding data type 7816 // for snprintf. A table of these will represent the entire valid keyword 7817 // field types. 7818 typedef struct kmp_affinity_format_field_t { 7819 char short_name; // from spec e.g., L -> thread level 7820 const char *long_name; // from spec thread_level -> thread level 7821 char field_format; // data type for snprintf (typically 'd' or 's' 7822 // for integer or string) 7823 } kmp_affinity_format_field_t; 7824 7825 static const kmp_affinity_format_field_t __kmp_affinity_format_table[] = { 7826 #if KMP_AFFINITY_SUPPORTED 7827 {'A', "thread_affinity", 's'}, 7828 #endif 7829 {'t', "team_num", 'd'}, 7830 {'T', "num_teams", 'd'}, 7831 {'L', "nesting_level", 'd'}, 7832 {'n', "thread_num", 'd'}, 7833 {'N', "num_threads", 'd'}, 7834 {'a', "ancestor_tnum", 'd'}, 7835 {'H', "host", 's'}, 7836 {'P', "process_id", 'd'}, 7837 {'i', "native_thread_id", 'd'}}; 7838 7839 // Return the number of characters it takes to hold field 7840 static int __kmp_aux_capture_affinity_field(int gtid, const kmp_info_t *th, 7841 const char **ptr, 7842 kmp_str_buf_t *field_buffer) { 7843 int rc, format_index, field_value; 7844 const char *width_left, *width_right; 7845 bool pad_zeros, right_justify, parse_long_name, found_valid_name; 7846 static const int FORMAT_SIZE = 20; 7847 char format[FORMAT_SIZE] = {0}; 7848 char absolute_short_name = 0; 7849 7850 KMP_DEBUG_ASSERT(gtid >= 0); 7851 KMP_DEBUG_ASSERT(th); 7852 KMP_DEBUG_ASSERT(**ptr == '%'); 7853 KMP_DEBUG_ASSERT(field_buffer); 7854 7855 __kmp_str_buf_clear(field_buffer); 7856 7857 // Skip the initial % 7858 (*ptr)++; 7859 7860 // Check for %% first 7861 if (**ptr == '%') { 7862 __kmp_str_buf_cat(field_buffer, "%", 1); 7863 (*ptr)++; // skip over the second % 7864 return 1; 7865 } 7866 7867 // Parse field modifiers if they are present 7868 pad_zeros = false; 7869 if (**ptr == '0') { 7870 pad_zeros = true; 7871 (*ptr)++; // skip over 0 7872 } 7873 right_justify = false; 7874 if (**ptr == '.') { 7875 right_justify = true; 7876 (*ptr)++; // skip over . 7877 } 7878 // Parse width of field: [width_left, width_right) 7879 width_left = width_right = NULL; 7880 if (**ptr >= '0' && **ptr <= '9') { 7881 width_left = *ptr; 7882 SKIP_DIGITS(*ptr); 7883 width_right = *ptr; 7884 } 7885 7886 // Create the format for KMP_SNPRINTF based on flags parsed above 7887 format_index = 0; 7888 format[format_index++] = '%'; 7889 if (!right_justify) 7890 format[format_index++] = '-'; 7891 if (pad_zeros) 7892 format[format_index++] = '0'; 7893 if (width_left && width_right) { 7894 int i = 0; 7895 // Only allow 8 digit number widths. 7896 // This also prevents overflowing format variable 7897 while (i < 8 && width_left < width_right) { 7898 format[format_index++] = *width_left; 7899 width_left++; 7900 i++; 7901 } 7902 } 7903 7904 // Parse a name (long or short) 7905 // Canonicalize the name into absolute_short_name 7906 found_valid_name = false; 7907 parse_long_name = (**ptr == '{'); 7908 if (parse_long_name) 7909 (*ptr)++; // skip initial left brace 7910 for (size_t i = 0; i < sizeof(__kmp_affinity_format_table) / 7911 sizeof(__kmp_affinity_format_table[0]); 7912 ++i) { 7913 char short_name = __kmp_affinity_format_table[i].short_name; 7914 const char *long_name = __kmp_affinity_format_table[i].long_name; 7915 char field_format = __kmp_affinity_format_table[i].field_format; 7916 if (parse_long_name) { 7917 int length = KMP_STRLEN(long_name); 7918 if (strncmp(*ptr, long_name, length) == 0) { 7919 found_valid_name = true; 7920 (*ptr) += length; // skip the long name 7921 } 7922 } else if (**ptr == short_name) { 7923 found_valid_name = true; 7924 (*ptr)++; // skip the short name 7925 } 7926 if (found_valid_name) { 7927 format[format_index++] = field_format; 7928 format[format_index++] = '\0'; 7929 absolute_short_name = short_name; 7930 break; 7931 } 7932 } 7933 if (parse_long_name) { 7934 if (**ptr != '}') { 7935 absolute_short_name = 0; 7936 } else { 7937 (*ptr)++; // skip over the right brace 7938 } 7939 } 7940 7941 // Attempt to fill the buffer with the requested 7942 // value using snprintf within __kmp_str_buf_print() 7943 switch (absolute_short_name) { 7944 case 't': 7945 rc = __kmp_str_buf_print(field_buffer, format, __kmp_aux_get_team_num()); 7946 break; 7947 case 'T': 7948 rc = __kmp_str_buf_print(field_buffer, format, __kmp_aux_get_num_teams()); 7949 break; 7950 case 'L': 7951 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_level); 7952 break; 7953 case 'n': 7954 rc = __kmp_str_buf_print(field_buffer, format, __kmp_tid_from_gtid(gtid)); 7955 break; 7956 case 'H': { 7957 static const int BUFFER_SIZE = 256; 7958 char buf[BUFFER_SIZE]; 7959 __kmp_expand_host_name(buf, BUFFER_SIZE); 7960 rc = __kmp_str_buf_print(field_buffer, format, buf); 7961 } break; 7962 case 'P': 7963 rc = __kmp_str_buf_print(field_buffer, format, getpid()); 7964 break; 7965 case 'i': 7966 rc = __kmp_str_buf_print(field_buffer, format, __kmp_gettid()); 7967 break; 7968 case 'N': 7969 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_nproc); 7970 break; 7971 case 'a': 7972 field_value = 7973 __kmp_get_ancestor_thread_num(gtid, th->th.th_team->t.t_level - 1); 7974 rc = __kmp_str_buf_print(field_buffer, format, field_value); 7975 break; 7976 #if KMP_AFFINITY_SUPPORTED 7977 case 'A': { 7978 kmp_str_buf_t buf; 7979 __kmp_str_buf_init(&buf); 7980 __kmp_affinity_str_buf_mask(&buf, th->th.th_affin_mask); 7981 rc = __kmp_str_buf_print(field_buffer, format, buf.str); 7982 __kmp_str_buf_free(&buf); 7983 } break; 7984 #endif 7985 default: 7986 // According to spec, If an implementation does not have info for field 7987 // type, then "undefined" is printed 7988 rc = __kmp_str_buf_print(field_buffer, "%s", "undefined"); 7989 // Skip the field 7990 if (parse_long_name) { 7991 SKIP_TOKEN(*ptr); 7992 if (**ptr == '}') 7993 (*ptr)++; 7994 } else { 7995 (*ptr)++; 7996 } 7997 } 7998 7999 KMP_ASSERT(format_index <= FORMAT_SIZE); 8000 return rc; 8001 } 8002 8003 /* 8004 * Return number of characters needed to hold the affinity string 8005 * (not including null byte character) 8006 * The resultant string is printed to buffer, which the caller can then 8007 * handle afterwards 8008 */ 8009 size_t __kmp_aux_capture_affinity(int gtid, const char *format, 8010 kmp_str_buf_t *buffer) { 8011 const char *parse_ptr; 8012 size_t retval; 8013 const kmp_info_t *th; 8014 kmp_str_buf_t field; 8015 8016 KMP_DEBUG_ASSERT(buffer); 8017 KMP_DEBUG_ASSERT(gtid >= 0); 8018 8019 __kmp_str_buf_init(&field); 8020 __kmp_str_buf_clear(buffer); 8021 8022 th = __kmp_threads[gtid]; 8023 retval = 0; 8024 8025 // If format is NULL or zero-length string, then we use 8026 // affinity-format-var ICV 8027 parse_ptr = format; 8028 if (parse_ptr == NULL || *parse_ptr == '\0') { 8029 parse_ptr = __kmp_affinity_format; 8030 } 8031 KMP_DEBUG_ASSERT(parse_ptr); 8032 8033 while (*parse_ptr != '\0') { 8034 // Parse a field 8035 if (*parse_ptr == '%') { 8036 // Put field in the buffer 8037 int rc = __kmp_aux_capture_affinity_field(gtid, th, &parse_ptr, &field); 8038 __kmp_str_buf_catbuf(buffer, &field); 8039 retval += rc; 8040 } else { 8041 // Put literal character in buffer 8042 __kmp_str_buf_cat(buffer, parse_ptr, 1); 8043 retval++; 8044 parse_ptr++; 8045 } 8046 } 8047 __kmp_str_buf_free(&field); 8048 return retval; 8049 } 8050 8051 // Displays the affinity string to stdout 8052 void __kmp_aux_display_affinity(int gtid, const char *format) { 8053 kmp_str_buf_t buf; 8054 __kmp_str_buf_init(&buf); 8055 __kmp_aux_capture_affinity(gtid, format, &buf); 8056 __kmp_fprintf(kmp_out, "%s" KMP_END_OF_LINE, buf.str); 8057 __kmp_str_buf_free(&buf); 8058 } 8059 8060 /* ------------------------------------------------------------------------ */ 8061 8062 void __kmp_aux_set_blocktime(int arg, kmp_info_t *thread, int tid) { 8063 int blocktime = arg; /* argument is in milliseconds */ 8064 #if KMP_USE_MONITOR 8065 int bt_intervals; 8066 #endif 8067 int bt_set; 8068 8069 __kmp_save_internal_controls(thread); 8070 8071 /* Normalize and set blocktime for the teams */ 8072 if (blocktime < KMP_MIN_BLOCKTIME) 8073 blocktime = KMP_MIN_BLOCKTIME; 8074 else if (blocktime > KMP_MAX_BLOCKTIME) 8075 blocktime = KMP_MAX_BLOCKTIME; 8076 8077 set__blocktime_team(thread->th.th_team, tid, blocktime); 8078 set__blocktime_team(thread->th.th_serial_team, 0, blocktime); 8079 8080 #if KMP_USE_MONITOR 8081 /* Calculate and set blocktime intervals for the teams */ 8082 bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME(blocktime, __kmp_monitor_wakeups); 8083 8084 set__bt_intervals_team(thread->th.th_team, tid, bt_intervals); 8085 set__bt_intervals_team(thread->th.th_serial_team, 0, bt_intervals); 8086 #endif 8087 8088 /* Set whether blocktime has been set to "TRUE" */ 8089 bt_set = TRUE; 8090 8091 set__bt_set_team(thread->th.th_team, tid, bt_set); 8092 set__bt_set_team(thread->th.th_serial_team, 0, bt_set); 8093 #if KMP_USE_MONITOR 8094 KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, " 8095 "bt_intervals=%d, monitor_updates=%d\n", 8096 __kmp_gtid_from_tid(tid, thread->th.th_team), 8097 thread->th.th_team->t.t_id, tid, blocktime, bt_intervals, 8098 __kmp_monitor_wakeups)); 8099 #else 8100 KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n", 8101 __kmp_gtid_from_tid(tid, thread->th.th_team), 8102 thread->th.th_team->t.t_id, tid, blocktime)); 8103 #endif 8104 } 8105 8106 void __kmp_aux_set_defaults(char const *str, int len) { 8107 if (!__kmp_init_serial) { 8108 __kmp_serial_initialize(); 8109 } 8110 __kmp_env_initialize(str); 8111 8112 if (__kmp_settings || __kmp_display_env || __kmp_display_env_verbose) { 8113 __kmp_env_print(); 8114 } 8115 } // __kmp_aux_set_defaults 8116 8117 /* ------------------------------------------------------------------------ */ 8118 /* internal fast reduction routines */ 8119 8120 PACKED_REDUCTION_METHOD_T 8121 __kmp_determine_reduction_method( 8122 ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, 8123 void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data), 8124 kmp_critical_name *lck) { 8125 8126 // Default reduction method: critical construct ( lck != NULL, like in current 8127 // PAROPT ) 8128 // If ( reduce_data!=NULL && reduce_func!=NULL ): the tree-reduction method 8129 // can be selected by RTL 8130 // If loc->flags contains KMP_IDENT_ATOMIC_REDUCE, the atomic reduce method 8131 // can be selected by RTL 8132 // Finally, it's up to OpenMP RTL to make a decision on which method to select 8133 // among generated by PAROPT. 8134 8135 PACKED_REDUCTION_METHOD_T retval; 8136 8137 int team_size; 8138 8139 KMP_DEBUG_ASSERT(loc); // it would be nice to test ( loc != 0 ) 8140 KMP_DEBUG_ASSERT(lck); // it would be nice to test ( lck != 0 ) 8141 8142 #define FAST_REDUCTION_ATOMIC_METHOD_GENERATED \ 8143 ((loc->flags & (KMP_IDENT_ATOMIC_REDUCE)) == (KMP_IDENT_ATOMIC_REDUCE)) 8144 #define FAST_REDUCTION_TREE_METHOD_GENERATED ((reduce_data) && (reduce_func)) 8145 8146 retval = critical_reduce_block; 8147 8148 // another choice of getting a team size (with 1 dynamic deference) is slower 8149 team_size = __kmp_get_team_num_threads(global_tid); 8150 if (team_size == 1) { 8151 8152 retval = empty_reduce_block; 8153 8154 } else { 8155 8156 int atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED; 8157 8158 #if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || \ 8159 KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64 8160 8161 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ 8162 KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HURD 8163 8164 int teamsize_cutoff = 4; 8165 8166 #if KMP_MIC_SUPPORTED 8167 if (__kmp_mic_type != non_mic) { 8168 teamsize_cutoff = 8; 8169 } 8170 #endif 8171 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED; 8172 if (tree_available) { 8173 if (team_size <= teamsize_cutoff) { 8174 if (atomic_available) { 8175 retval = atomic_reduce_block; 8176 } 8177 } else { 8178 retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER; 8179 } 8180 } else if (atomic_available) { 8181 retval = atomic_reduce_block; 8182 } 8183 #else 8184 #error "Unknown or unsupported OS" 8185 #endif // KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || 8186 // KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HURD 8187 8188 #elif KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS 8189 8190 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_WINDOWS || KMP_OS_HURD 8191 8192 // basic tuning 8193 8194 if (atomic_available) { 8195 if (num_vars <= 2) { // && ( team_size <= 8 ) due to false-sharing ??? 8196 retval = atomic_reduce_block; 8197 } 8198 } // otherwise: use critical section 8199 8200 #elif KMP_OS_DARWIN 8201 8202 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED; 8203 if (atomic_available && (num_vars <= 3)) { 8204 retval = atomic_reduce_block; 8205 } else if (tree_available) { 8206 if ((reduce_size > (9 * sizeof(kmp_real64))) && 8207 (reduce_size < (2000 * sizeof(kmp_real64)))) { 8208 retval = TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER; 8209 } 8210 } // otherwise: use critical section 8211 8212 #else 8213 #error "Unknown or unsupported OS" 8214 #endif 8215 8216 #else 8217 #error "Unknown or unsupported architecture" 8218 #endif 8219 } 8220 8221 // KMP_FORCE_REDUCTION 8222 8223 // If the team is serialized (team_size == 1), ignore the forced reduction 8224 // method and stay with the unsynchronized method (empty_reduce_block) 8225 if (__kmp_force_reduction_method != reduction_method_not_defined && 8226 team_size != 1) { 8227 8228 PACKED_REDUCTION_METHOD_T forced_retval = critical_reduce_block; 8229 8230 int atomic_available, tree_available; 8231 8232 switch ((forced_retval = __kmp_force_reduction_method)) { 8233 case critical_reduce_block: 8234 KMP_ASSERT(lck); // lck should be != 0 8235 break; 8236 8237 case atomic_reduce_block: 8238 atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED; 8239 if (!atomic_available) { 8240 KMP_WARNING(RedMethodNotSupported, "atomic"); 8241 forced_retval = critical_reduce_block; 8242 } 8243 break; 8244 8245 case tree_reduce_block: 8246 tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED; 8247 if (!tree_available) { 8248 KMP_WARNING(RedMethodNotSupported, "tree"); 8249 forced_retval = critical_reduce_block; 8250 } else { 8251 #if KMP_FAST_REDUCTION_BARRIER 8252 forced_retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER; 8253 #endif 8254 } 8255 break; 8256 8257 default: 8258 KMP_ASSERT(0); // "unsupported method specified" 8259 } 8260 8261 retval = forced_retval; 8262 } 8263 8264 KA_TRACE(10, ("reduction method selected=%08x\n", retval)); 8265 8266 #undef FAST_REDUCTION_TREE_METHOD_GENERATED 8267 #undef FAST_REDUCTION_ATOMIC_METHOD_GENERATED 8268 8269 return (retval); 8270 } 8271 // this function is for testing set/get/determine reduce method 8272 kmp_int32 __kmp_get_reduce_method(void) { 8273 return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8); 8274 } 8275 8276 // Soft pause sets up threads to ignore blocktime and just go to sleep. 8277 // Spin-wait code checks __kmp_pause_status and reacts accordingly. 8278 void __kmp_soft_pause() { __kmp_pause_status = kmp_soft_paused; } 8279 8280 // Hard pause shuts down the runtime completely. Resume happens naturally when 8281 // OpenMP is used subsequently. 8282 void __kmp_hard_pause() { 8283 __kmp_pause_status = kmp_hard_paused; 8284 __kmp_internal_end_thread(-1); 8285 } 8286 8287 // Soft resume sets __kmp_pause_status, and wakes up all threads. 8288 void __kmp_resume_if_soft_paused() { 8289 if (__kmp_pause_status == kmp_soft_paused) { 8290 __kmp_pause_status = kmp_not_paused; 8291 8292 for (int gtid = 1; gtid < __kmp_threads_capacity; ++gtid) { 8293 kmp_info_t *thread = __kmp_threads[gtid]; 8294 if (thread) { // Wake it if sleeping 8295 kmp_flag_64 fl(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, thread); 8296 if (fl.is_sleeping()) 8297 fl.resume(gtid); 8298 else if (__kmp_try_suspend_mx(thread)) { // got suspend lock 8299 __kmp_unlock_suspend_mx(thread); // unlock it; it won't sleep 8300 } else { // thread holds the lock and may sleep soon 8301 do { // until either the thread sleeps, or we can get the lock 8302 if (fl.is_sleeping()) { 8303 fl.resume(gtid); 8304 break; 8305 } else if (__kmp_try_suspend_mx(thread)) { 8306 __kmp_unlock_suspend_mx(thread); 8307 break; 8308 } 8309 } while (1); 8310 } 8311 } 8312 } 8313 } 8314 } 8315 8316 // This function is called via __kmpc_pause_resource. Returns 0 if successful. 8317 // TODO: add warning messages 8318 int __kmp_pause_resource(kmp_pause_status_t level) { 8319 if (level == kmp_not_paused) { // requesting resume 8320 if (__kmp_pause_status == kmp_not_paused) { 8321 // error message about runtime not being paused, so can't resume 8322 return 1; 8323 } else { 8324 KMP_DEBUG_ASSERT(__kmp_pause_status == kmp_soft_paused || 8325 __kmp_pause_status == kmp_hard_paused); 8326 __kmp_pause_status = kmp_not_paused; 8327 return 0; 8328 } 8329 } else if (level == kmp_soft_paused) { // requesting soft pause 8330 if (__kmp_pause_status != kmp_not_paused) { 8331 // error message about already being paused 8332 return 1; 8333 } else { 8334 __kmp_soft_pause(); 8335 return 0; 8336 } 8337 } else if (level == kmp_hard_paused) { // requesting hard pause 8338 if (__kmp_pause_status != kmp_not_paused) { 8339 // error message about already being paused 8340 return 1; 8341 } else { 8342 __kmp_hard_pause(); 8343 return 0; 8344 } 8345 } else { 8346 // error message about invalid level 8347 return 1; 8348 } 8349 } 8350 8351 8352 void __kmp_omp_display_env(int verbose) { 8353 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 8354 if (__kmp_init_serial == 0) 8355 __kmp_do_serial_initialize(); 8356 __kmp_display_env_impl(!verbose, verbose); 8357 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 8358 } 8359