1 /* 2 * kmp_runtime.cpp -- KPTS runtime support library 3 */ 4 5 //===----------------------------------------------------------------------===// 6 // 7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 8 // See https://llvm.org/LICENSE.txt for license information. 9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "kmp.h" 14 #include "kmp_affinity.h" 15 #include "kmp_atomic.h" 16 #include "kmp_environment.h" 17 #include "kmp_error.h" 18 #include "kmp_i18n.h" 19 #include "kmp_io.h" 20 #include "kmp_itt.h" 21 #include "kmp_settings.h" 22 #include "kmp_stats.h" 23 #include "kmp_str.h" 24 #include "kmp_wait_release.h" 25 #include "kmp_wrapper_getpid.h" 26 #include "kmp_dispatch.h" 27 #if KMP_USE_HIER_SCHED 28 #include "kmp_dispatch_hier.h" 29 #endif 30 31 #if OMPT_SUPPORT 32 #include "ompt-specific.h" 33 #endif 34 35 /* these are temporary issues to be dealt with */ 36 #define KMP_USE_PRCTL 0 37 38 #if KMP_OS_WINDOWS 39 #include <process.h> 40 #endif 41 42 #include "tsan_annotations.h" 43 44 #if defined(KMP_GOMP_COMPAT) 45 char const __kmp_version_alt_comp[] = 46 KMP_VERSION_PREFIX "alternative compiler support: yes"; 47 #endif /* defined(KMP_GOMP_COMPAT) */ 48 49 char const __kmp_version_omp_api[] = KMP_VERSION_PREFIX "API version: " 50 #if OMP_50_ENABLED 51 "5.0 (201611)"; 52 #elif OMP_45_ENABLED 53 "4.5 (201511)"; 54 #elif OMP_40_ENABLED 55 "4.0 (201307)"; 56 #else 57 "3.1 (201107)"; 58 #endif 59 60 #ifdef KMP_DEBUG 61 char const __kmp_version_lock[] = 62 KMP_VERSION_PREFIX "lock type: run time selectable"; 63 #endif /* KMP_DEBUG */ 64 65 #define KMP_MIN(x, y) ((x) < (y) ? (x) : (y)) 66 67 /* ------------------------------------------------------------------------ */ 68 69 #if KMP_USE_MONITOR 70 kmp_info_t __kmp_monitor; 71 #endif 72 73 /* Forward declarations */ 74 75 void __kmp_cleanup(void); 76 77 static void __kmp_initialize_info(kmp_info_t *, kmp_team_t *, int tid, 78 int gtid); 79 static void __kmp_initialize_team(kmp_team_t *team, int new_nproc, 80 kmp_internal_control_t *new_icvs, 81 ident_t *loc); 82 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 83 static void __kmp_partition_places(kmp_team_t *team, 84 int update_master_only = 0); 85 #endif 86 static void __kmp_do_serial_initialize(void); 87 void __kmp_fork_barrier(int gtid, int tid); 88 void __kmp_join_barrier(int gtid); 89 void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc, 90 kmp_internal_control_t *new_icvs, ident_t *loc); 91 92 #ifdef USE_LOAD_BALANCE 93 static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc); 94 #endif 95 96 static int __kmp_expand_threads(int nNeed); 97 #if KMP_OS_WINDOWS 98 static int __kmp_unregister_root_other_thread(int gtid); 99 #endif 100 static void __kmp_unregister_library(void); // called by __kmp_internal_end() 101 static void __kmp_reap_thread(kmp_info_t *thread, int is_root); 102 kmp_info_t *__kmp_thread_pool_insert_pt = NULL; 103 104 /* Calculate the identifier of the current thread */ 105 /* fast (and somewhat portable) way to get unique identifier of executing 106 thread. Returns KMP_GTID_DNE if we haven't been assigned a gtid. */ 107 int __kmp_get_global_thread_id() { 108 int i; 109 kmp_info_t **other_threads; 110 size_t stack_data; 111 char *stack_addr; 112 size_t stack_size; 113 char *stack_base; 114 115 KA_TRACE( 116 1000, 117 ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n", 118 __kmp_nth, __kmp_all_nth)); 119 120 /* JPH - to handle the case where __kmpc_end(0) is called immediately prior to 121 a parallel region, made it return KMP_GTID_DNE to force serial_initialize 122 by caller. Had to handle KMP_GTID_DNE at all call-sites, or else guarantee 123 __kmp_init_gtid for this to work. */ 124 125 if (!TCR_4(__kmp_init_gtid)) 126 return KMP_GTID_DNE; 127 128 #ifdef KMP_TDATA_GTID 129 if (TCR_4(__kmp_gtid_mode) >= 3) { 130 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using TDATA\n")); 131 return __kmp_gtid; 132 } 133 #endif 134 if (TCR_4(__kmp_gtid_mode) >= 2) { 135 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using keyed TLS\n")); 136 return __kmp_gtid_get_specific(); 137 } 138 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using internal alg.\n")); 139 140 stack_addr = (char *)&stack_data; 141 other_threads = __kmp_threads; 142 143 /* ATT: The code below is a source of potential bugs due to unsynchronized 144 access to __kmp_threads array. For example: 145 1. Current thread loads other_threads[i] to thr and checks it, it is 146 non-NULL. 147 2. Current thread is suspended by OS. 148 3. Another thread unregisters and finishes (debug versions of free() 149 may fill memory with something like 0xEF). 150 4. Current thread is resumed. 151 5. Current thread reads junk from *thr. 152 TODO: Fix it. --ln */ 153 154 for (i = 0; i < __kmp_threads_capacity; i++) { 155 156 kmp_info_t *thr = (kmp_info_t *)TCR_SYNC_PTR(other_threads[i]); 157 if (!thr) 158 continue; 159 160 stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize); 161 stack_base = (char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase); 162 163 /* stack grows down -- search through all of the active threads */ 164 165 if (stack_addr <= stack_base) { 166 size_t stack_diff = stack_base - stack_addr; 167 168 if (stack_diff <= stack_size) { 169 /* The only way we can be closer than the allocated */ 170 /* stack size is if we are running on this thread. */ 171 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == i); 172 return i; 173 } 174 } 175 } 176 177 /* get specific to try and determine our gtid */ 178 KA_TRACE(1000, 179 ("*** __kmp_get_global_thread_id: internal alg. failed to find " 180 "thread, using TLS\n")); 181 i = __kmp_gtid_get_specific(); 182 183 /*fprintf( stderr, "=== %d\n", i ); */ /* GROO */ 184 185 /* if we havn't been assigned a gtid, then return code */ 186 if (i < 0) 187 return i; 188 189 /* dynamically updated stack window for uber threads to avoid get_specific 190 call */ 191 if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)) { 192 KMP_FATAL(StackOverflow, i); 193 } 194 195 stack_base = (char *)other_threads[i]->th.th_info.ds.ds_stackbase; 196 if (stack_addr > stack_base) { 197 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr); 198 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize, 199 other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr - 200 stack_base); 201 } else { 202 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize, 203 stack_base - stack_addr); 204 } 205 206 /* Reprint stack bounds for ubermaster since they have been refined */ 207 if (__kmp_storage_map) { 208 char *stack_end = (char *)other_threads[i]->th.th_info.ds.ds_stackbase; 209 char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize; 210 __kmp_print_storage_map_gtid(i, stack_beg, stack_end, 211 other_threads[i]->th.th_info.ds.ds_stacksize, 212 "th_%d stack (refinement)", i); 213 } 214 return i; 215 } 216 217 int __kmp_get_global_thread_id_reg() { 218 int gtid; 219 220 if (!__kmp_init_serial) { 221 gtid = KMP_GTID_DNE; 222 } else 223 #ifdef KMP_TDATA_GTID 224 if (TCR_4(__kmp_gtid_mode) >= 3) { 225 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using TDATA\n")); 226 gtid = __kmp_gtid; 227 } else 228 #endif 229 if (TCR_4(__kmp_gtid_mode) >= 2) { 230 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using keyed TLS\n")); 231 gtid = __kmp_gtid_get_specific(); 232 } else { 233 KA_TRACE(1000, 234 ("*** __kmp_get_global_thread_id_reg: using internal alg.\n")); 235 gtid = __kmp_get_global_thread_id(); 236 } 237 238 /* we must be a new uber master sibling thread */ 239 if (gtid == KMP_GTID_DNE) { 240 KA_TRACE(10, 241 ("__kmp_get_global_thread_id_reg: Encountered new root thread. " 242 "Registering a new gtid.\n")); 243 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 244 if (!__kmp_init_serial) { 245 __kmp_do_serial_initialize(); 246 gtid = __kmp_gtid_get_specific(); 247 } else { 248 gtid = __kmp_register_root(FALSE); 249 } 250 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 251 /*__kmp_printf( "+++ %d\n", gtid ); */ /* GROO */ 252 } 253 254 KMP_DEBUG_ASSERT(gtid >= 0); 255 256 return gtid; 257 } 258 259 /* caller must hold forkjoin_lock */ 260 void __kmp_check_stack_overlap(kmp_info_t *th) { 261 int f; 262 char *stack_beg = NULL; 263 char *stack_end = NULL; 264 int gtid; 265 266 KA_TRACE(10, ("__kmp_check_stack_overlap: called\n")); 267 if (__kmp_storage_map) { 268 stack_end = (char *)th->th.th_info.ds.ds_stackbase; 269 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize; 270 271 gtid = __kmp_gtid_from_thread(th); 272 273 if (gtid == KMP_GTID_MONITOR) { 274 __kmp_print_storage_map_gtid( 275 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize, 276 "th_%s stack (%s)", "mon", 277 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual"); 278 } else { 279 __kmp_print_storage_map_gtid( 280 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize, 281 "th_%d stack (%s)", gtid, 282 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual"); 283 } 284 } 285 286 /* No point in checking ubermaster threads since they use refinement and 287 * cannot overlap */ 288 gtid = __kmp_gtid_from_thread(th); 289 if (__kmp_env_checks == TRUE && !KMP_UBER_GTID(gtid)) { 290 KA_TRACE(10, 291 ("__kmp_check_stack_overlap: performing extensive checking\n")); 292 if (stack_beg == NULL) { 293 stack_end = (char *)th->th.th_info.ds.ds_stackbase; 294 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize; 295 } 296 297 for (f = 0; f < __kmp_threads_capacity; f++) { 298 kmp_info_t *f_th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[f]); 299 300 if (f_th && f_th != th) { 301 char *other_stack_end = 302 (char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase); 303 char *other_stack_beg = 304 other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize); 305 if ((stack_beg > other_stack_beg && stack_beg < other_stack_end) || 306 (stack_end > other_stack_beg && stack_end < other_stack_end)) { 307 308 /* Print the other stack values before the abort */ 309 if (__kmp_storage_map) 310 __kmp_print_storage_map_gtid( 311 -1, other_stack_beg, other_stack_end, 312 (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize), 313 "th_%d stack (overlapped)", __kmp_gtid_from_thread(f_th)); 314 315 __kmp_fatal(KMP_MSG(StackOverlap), KMP_HNT(ChangeStackLimit), 316 __kmp_msg_null); 317 } 318 } 319 } 320 } 321 KA_TRACE(10, ("__kmp_check_stack_overlap: returning\n")); 322 } 323 324 /* ------------------------------------------------------------------------ */ 325 326 void __kmp_infinite_loop(void) { 327 static int done = FALSE; 328 329 while (!done) { 330 KMP_YIELD(TRUE); 331 } 332 } 333 334 #define MAX_MESSAGE 512 335 336 void __kmp_print_storage_map_gtid(int gtid, void *p1, void *p2, size_t size, 337 char const *format, ...) { 338 char buffer[MAX_MESSAGE]; 339 va_list ap; 340 341 va_start(ap, format); 342 KMP_SNPRINTF(buffer, sizeof(buffer), "OMP storage map: %p %p%8lu %s\n", p1, 343 p2, (unsigned long)size, format); 344 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock); 345 __kmp_vprintf(kmp_err, buffer, ap); 346 #if KMP_PRINT_DATA_PLACEMENT 347 int node; 348 if (gtid >= 0) { 349 if (p1 <= p2 && (char *)p2 - (char *)p1 == size) { 350 if (__kmp_storage_map_verbose) { 351 node = __kmp_get_host_node(p1); 352 if (node < 0) /* doesn't work, so don't try this next time */ 353 __kmp_storage_map_verbose = FALSE; 354 else { 355 char *last; 356 int lastNode; 357 int localProc = __kmp_get_cpu_from_gtid(gtid); 358 359 const int page_size = KMP_GET_PAGE_SIZE(); 360 361 p1 = (void *)((size_t)p1 & ~((size_t)page_size - 1)); 362 p2 = (void *)(((size_t)p2 - 1) & ~((size_t)page_size - 1)); 363 if (localProc >= 0) 364 __kmp_printf_no_lock(" GTID %d localNode %d\n", gtid, 365 localProc >> 1); 366 else 367 __kmp_printf_no_lock(" GTID %d\n", gtid); 368 #if KMP_USE_PRCTL 369 /* The more elaborate format is disabled for now because of the prctl 370 * hanging bug. */ 371 do { 372 last = p1; 373 lastNode = node; 374 /* This loop collates adjacent pages with the same host node. */ 375 do { 376 (char *)p1 += page_size; 377 } while (p1 <= p2 && (node = __kmp_get_host_node(p1)) == lastNode); 378 __kmp_printf_no_lock(" %p-%p memNode %d\n", last, (char *)p1 - 1, 379 lastNode); 380 } while (p1 <= p2); 381 #else 382 __kmp_printf_no_lock(" %p-%p memNode %d\n", p1, 383 (char *)p1 + (page_size - 1), 384 __kmp_get_host_node(p1)); 385 if (p1 < p2) { 386 __kmp_printf_no_lock(" %p-%p memNode %d\n", p2, 387 (char *)p2 + (page_size - 1), 388 __kmp_get_host_node(p2)); 389 } 390 #endif 391 } 392 } 393 } else 394 __kmp_printf_no_lock(" %s\n", KMP_I18N_STR(StorageMapWarning)); 395 } 396 #endif /* KMP_PRINT_DATA_PLACEMENT */ 397 __kmp_release_bootstrap_lock(&__kmp_stdio_lock); 398 } 399 400 void __kmp_warn(char const *format, ...) { 401 char buffer[MAX_MESSAGE]; 402 va_list ap; 403 404 if (__kmp_generate_warnings == kmp_warnings_off) { 405 return; 406 } 407 408 va_start(ap, format); 409 410 KMP_SNPRINTF(buffer, sizeof(buffer), "OMP warning: %s\n", format); 411 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock); 412 __kmp_vprintf(kmp_err, buffer, ap); 413 __kmp_release_bootstrap_lock(&__kmp_stdio_lock); 414 415 va_end(ap); 416 } 417 418 void __kmp_abort_process() { 419 // Later threads may stall here, but that's ok because abort() will kill them. 420 __kmp_acquire_bootstrap_lock(&__kmp_exit_lock); 421 422 if (__kmp_debug_buf) { 423 __kmp_dump_debug_buffer(); 424 } 425 426 if (KMP_OS_WINDOWS) { 427 // Let other threads know of abnormal termination and prevent deadlock 428 // if abort happened during library initialization or shutdown 429 __kmp_global.g.g_abort = SIGABRT; 430 431 /* On Windows* OS by default abort() causes pop-up error box, which stalls 432 nightly testing. Unfortunately, we cannot reliably suppress pop-up error 433 boxes. _set_abort_behavior() works well, but this function is not 434 available in VS7 (this is not problem for DLL, but it is a problem for 435 static OpenMP RTL). SetErrorMode (and so, timelimit utility) does not 436 help, at least in some versions of MS C RTL. 437 438 It seems following sequence is the only way to simulate abort() and 439 avoid pop-up error box. */ 440 raise(SIGABRT); 441 _exit(3); // Just in case, if signal ignored, exit anyway. 442 } else { 443 abort(); 444 } 445 446 __kmp_infinite_loop(); 447 __kmp_release_bootstrap_lock(&__kmp_exit_lock); 448 449 } // __kmp_abort_process 450 451 void __kmp_abort_thread(void) { 452 // TODO: Eliminate g_abort global variable and this function. 453 // In case of abort just call abort(), it will kill all the threads. 454 __kmp_infinite_loop(); 455 } // __kmp_abort_thread 456 457 /* Print out the storage map for the major kmp_info_t thread data structures 458 that are allocated together. */ 459 460 static void __kmp_print_thread_storage_map(kmp_info_t *thr, int gtid) { 461 __kmp_print_storage_map_gtid(gtid, thr, thr + 1, sizeof(kmp_info_t), "th_%d", 462 gtid); 463 464 __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team, 465 sizeof(kmp_desc_t), "th_%d.th_info", gtid); 466 467 __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head, 468 sizeof(kmp_local_t), "th_%d.th_local", gtid); 469 470 __kmp_print_storage_map_gtid( 471 gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier], 472 sizeof(kmp_balign_t) * bs_last_barrier, "th_%d.th_bar", gtid); 473 474 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier], 475 &thr->th.th_bar[bs_plain_barrier + 1], 476 sizeof(kmp_balign_t), "th_%d.th_bar[plain]", 477 gtid); 478 479 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier], 480 &thr->th.th_bar[bs_forkjoin_barrier + 1], 481 sizeof(kmp_balign_t), "th_%d.th_bar[forkjoin]", 482 gtid); 483 484 #if KMP_FAST_REDUCTION_BARRIER 485 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier], 486 &thr->th.th_bar[bs_reduction_barrier + 1], 487 sizeof(kmp_balign_t), "th_%d.th_bar[reduction]", 488 gtid); 489 #endif // KMP_FAST_REDUCTION_BARRIER 490 } 491 492 /* Print out the storage map for the major kmp_team_t team data structures 493 that are allocated together. */ 494 495 static void __kmp_print_team_storage_map(const char *header, kmp_team_t *team, 496 int team_id, int num_thr) { 497 int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2; 498 __kmp_print_storage_map_gtid(-1, team, team + 1, sizeof(kmp_team_t), "%s_%d", 499 header, team_id); 500 501 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0], 502 &team->t.t_bar[bs_last_barrier], 503 sizeof(kmp_balign_team_t) * bs_last_barrier, 504 "%s_%d.t_bar", header, team_id); 505 506 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier], 507 &team->t.t_bar[bs_plain_barrier + 1], 508 sizeof(kmp_balign_team_t), "%s_%d.t_bar[plain]", 509 header, team_id); 510 511 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier], 512 &team->t.t_bar[bs_forkjoin_barrier + 1], 513 sizeof(kmp_balign_team_t), 514 "%s_%d.t_bar[forkjoin]", header, team_id); 515 516 #if KMP_FAST_REDUCTION_BARRIER 517 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier], 518 &team->t.t_bar[bs_reduction_barrier + 1], 519 sizeof(kmp_balign_team_t), 520 "%s_%d.t_bar[reduction]", header, team_id); 521 #endif // KMP_FAST_REDUCTION_BARRIER 522 523 __kmp_print_storage_map_gtid( 524 -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr], 525 sizeof(kmp_disp_t) * num_thr, "%s_%d.t_dispatch", header, team_id); 526 527 __kmp_print_storage_map_gtid( 528 -1, &team->t.t_threads[0], &team->t.t_threads[num_thr], 529 sizeof(kmp_info_t *) * num_thr, "%s_%d.t_threads", header, team_id); 530 531 __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0], 532 &team->t.t_disp_buffer[num_disp_buff], 533 sizeof(dispatch_shared_info_t) * num_disp_buff, 534 "%s_%d.t_disp_buffer", header, team_id); 535 } 536 537 static void __kmp_init_allocator() { 538 #if OMP_50_ENABLED 539 __kmp_init_memkind(); 540 #endif 541 } 542 static void __kmp_fini_allocator() { 543 #if OMP_50_ENABLED 544 __kmp_fini_memkind(); 545 #endif 546 } 547 548 /* ------------------------------------------------------------------------ */ 549 550 #if KMP_DYNAMIC_LIB 551 #if KMP_OS_WINDOWS 552 553 static void __kmp_reset_lock(kmp_bootstrap_lock_t *lck) { 554 // TODO: Change to __kmp_break_bootstrap_lock(). 555 __kmp_init_bootstrap_lock(lck); // make the lock released 556 } 557 558 static void __kmp_reset_locks_on_process_detach(int gtid_req) { 559 int i; 560 int thread_count; 561 562 // PROCESS_DETACH is expected to be called by a thread that executes 563 // ProcessExit() or FreeLibrary(). OS terminates other threads (except the one 564 // calling ProcessExit or FreeLibrary). So, it might be safe to access the 565 // __kmp_threads[] without taking the forkjoin_lock. However, in fact, some 566 // threads can be still alive here, although being about to be terminated. The 567 // threads in the array with ds_thread==0 are most suspicious. Actually, it 568 // can be not safe to access the __kmp_threads[]. 569 570 // TODO: does it make sense to check __kmp_roots[] ? 571 572 // Let's check that there are no other alive threads registered with the OMP 573 // lib. 574 while (1) { 575 thread_count = 0; 576 for (i = 0; i < __kmp_threads_capacity; ++i) { 577 if (!__kmp_threads) 578 continue; 579 kmp_info_t *th = __kmp_threads[i]; 580 if (th == NULL) 581 continue; 582 int gtid = th->th.th_info.ds.ds_gtid; 583 if (gtid == gtid_req) 584 continue; 585 if (gtid < 0) 586 continue; 587 DWORD exit_val; 588 int alive = __kmp_is_thread_alive(th, &exit_val); 589 if (alive) { 590 ++thread_count; 591 } 592 } 593 if (thread_count == 0) 594 break; // success 595 } 596 597 // Assume that I'm alone. Now it might be safe to check and reset locks. 598 // __kmp_forkjoin_lock and __kmp_stdio_lock are expected to be reset. 599 __kmp_reset_lock(&__kmp_forkjoin_lock); 600 #ifdef KMP_DEBUG 601 __kmp_reset_lock(&__kmp_stdio_lock); 602 #endif // KMP_DEBUG 603 } 604 605 BOOL WINAPI DllMain(HINSTANCE hInstDLL, DWORD fdwReason, LPVOID lpReserved) { 606 //__kmp_acquire_bootstrap_lock( &__kmp_initz_lock ); 607 608 switch (fdwReason) { 609 610 case DLL_PROCESS_ATTACH: 611 KA_TRACE(10, ("DllMain: PROCESS_ATTACH\n")); 612 613 return TRUE; 614 615 case DLL_PROCESS_DETACH: 616 KA_TRACE(10, ("DllMain: PROCESS_DETACH T#%d\n", __kmp_gtid_get_specific())); 617 618 if (lpReserved != NULL) { 619 // lpReserved is used for telling the difference: 620 // lpReserved == NULL when FreeLibrary() was called, 621 // lpReserved != NULL when the process terminates. 622 // When FreeLibrary() is called, worker threads remain alive. So they will 623 // release the forkjoin lock by themselves. When the process terminates, 624 // worker threads disappear triggering the problem of unreleased forkjoin 625 // lock as described below. 626 627 // A worker thread can take the forkjoin lock. The problem comes up if 628 // that worker thread becomes dead before it releases the forkjoin lock. 629 // The forkjoin lock remains taken, while the thread executing 630 // DllMain()->PROCESS_DETACH->__kmp_internal_end_library() below will try 631 // to take the forkjoin lock and will always fail, so that the application 632 // will never finish [normally]. This scenario is possible if 633 // __kmpc_end() has not been executed. It looks like it's not a corner 634 // case, but common cases: 635 // - the main function was compiled by an alternative compiler; 636 // - the main function was compiled by icl but without /Qopenmp 637 // (application with plugins); 638 // - application terminates by calling C exit(), Fortran CALL EXIT() or 639 // Fortran STOP. 640 // - alive foreign thread prevented __kmpc_end from doing cleanup. 641 // 642 // This is a hack to work around the problem. 643 // TODO: !!! figure out something better. 644 __kmp_reset_locks_on_process_detach(__kmp_gtid_get_specific()); 645 } 646 647 __kmp_internal_end_library(__kmp_gtid_get_specific()); 648 649 return TRUE; 650 651 case DLL_THREAD_ATTACH: 652 KA_TRACE(10, ("DllMain: THREAD_ATTACH\n")); 653 654 /* if we want to register new siblings all the time here call 655 * __kmp_get_gtid(); */ 656 return TRUE; 657 658 case DLL_THREAD_DETACH: 659 KA_TRACE(10, ("DllMain: THREAD_DETACH T#%d\n", __kmp_gtid_get_specific())); 660 661 __kmp_internal_end_thread(__kmp_gtid_get_specific()); 662 return TRUE; 663 } 664 665 return TRUE; 666 } 667 668 #endif /* KMP_OS_WINDOWS */ 669 #endif /* KMP_DYNAMIC_LIB */ 670 671 /* __kmp_parallel_deo -- Wait until it's our turn. */ 672 void __kmp_parallel_deo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) { 673 int gtid = *gtid_ref; 674 #ifdef BUILD_PARALLEL_ORDERED 675 kmp_team_t *team = __kmp_team_from_gtid(gtid); 676 #endif /* BUILD_PARALLEL_ORDERED */ 677 678 if (__kmp_env_consistency_check) { 679 if (__kmp_threads[gtid]->th.th_root->r.r_active) 680 #if KMP_USE_DYNAMIC_LOCK 681 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL, 0); 682 #else 683 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL); 684 #endif 685 } 686 #ifdef BUILD_PARALLEL_ORDERED 687 if (!team->t.t_serialized) { 688 KMP_MB(); 689 KMP_WAIT(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid), KMP_EQ, 690 NULL); 691 KMP_MB(); 692 } 693 #endif /* BUILD_PARALLEL_ORDERED */ 694 } 695 696 /* __kmp_parallel_dxo -- Signal the next task. */ 697 void __kmp_parallel_dxo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) { 698 int gtid = *gtid_ref; 699 #ifdef BUILD_PARALLEL_ORDERED 700 int tid = __kmp_tid_from_gtid(gtid); 701 kmp_team_t *team = __kmp_team_from_gtid(gtid); 702 #endif /* BUILD_PARALLEL_ORDERED */ 703 704 if (__kmp_env_consistency_check) { 705 if (__kmp_threads[gtid]->th.th_root->r.r_active) 706 __kmp_pop_sync(gtid, ct_ordered_in_parallel, loc_ref); 707 } 708 #ifdef BUILD_PARALLEL_ORDERED 709 if (!team->t.t_serialized) { 710 KMP_MB(); /* Flush all pending memory write invalidates. */ 711 712 /* use the tid of the next thread in this team */ 713 /* TODO replace with general release procedure */ 714 team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc); 715 716 KMP_MB(); /* Flush all pending memory write invalidates. */ 717 } 718 #endif /* BUILD_PARALLEL_ORDERED */ 719 } 720 721 /* ------------------------------------------------------------------------ */ 722 /* The BARRIER for a SINGLE process section is always explicit */ 723 724 int __kmp_enter_single(int gtid, ident_t *id_ref, int push_ws) { 725 int status; 726 kmp_info_t *th; 727 kmp_team_t *team; 728 729 if (!TCR_4(__kmp_init_parallel)) 730 __kmp_parallel_initialize(); 731 732 #if OMP_50_ENABLED 733 __kmp_resume_if_soft_paused(); 734 #endif 735 736 th = __kmp_threads[gtid]; 737 team = th->th.th_team; 738 status = 0; 739 740 th->th.th_ident = id_ref; 741 742 if (team->t.t_serialized) { 743 status = 1; 744 } else { 745 kmp_int32 old_this = th->th.th_local.this_construct; 746 747 ++th->th.th_local.this_construct; 748 /* try to set team count to thread count--success means thread got the 749 single block */ 750 /* TODO: Should this be acquire or release? */ 751 if (team->t.t_construct == old_this) { 752 status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this, 753 th->th.th_local.this_construct); 754 } 755 #if USE_ITT_BUILD 756 if (__itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 && 757 KMP_MASTER_GTID(gtid) && 758 #if OMP_40_ENABLED 759 th->th.th_teams_microtask == NULL && 760 #endif 761 team->t.t_active_level == 762 1) { // Only report metadata by master of active team at level 1 763 __kmp_itt_metadata_single(id_ref); 764 } 765 #endif /* USE_ITT_BUILD */ 766 } 767 768 if (__kmp_env_consistency_check) { 769 if (status && push_ws) { 770 __kmp_push_workshare(gtid, ct_psingle, id_ref); 771 } else { 772 __kmp_check_workshare(gtid, ct_psingle, id_ref); 773 } 774 } 775 #if USE_ITT_BUILD 776 if (status) { 777 __kmp_itt_single_start(gtid); 778 } 779 #endif /* USE_ITT_BUILD */ 780 return status; 781 } 782 783 void __kmp_exit_single(int gtid) { 784 #if USE_ITT_BUILD 785 __kmp_itt_single_end(gtid); 786 #endif /* USE_ITT_BUILD */ 787 if (__kmp_env_consistency_check) 788 __kmp_pop_workshare(gtid, ct_psingle, NULL); 789 } 790 791 /* determine if we can go parallel or must use a serialized parallel region and 792 * how many threads we can use 793 * set_nproc is the number of threads requested for the team 794 * returns 0 if we should serialize or only use one thread, 795 * otherwise the number of threads to use 796 * The forkjoin lock is held by the caller. */ 797 static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team, 798 int master_tid, int set_nthreads 799 #if OMP_40_ENABLED 800 , 801 int enter_teams 802 #endif /* OMP_40_ENABLED */ 803 ) { 804 int capacity; 805 int new_nthreads; 806 KMP_DEBUG_ASSERT(__kmp_init_serial); 807 KMP_DEBUG_ASSERT(root && parent_team); 808 kmp_info_t *this_thr = parent_team->t.t_threads[master_tid]; 809 810 // If dyn-var is set, dynamically adjust the number of desired threads, 811 // according to the method specified by dynamic_mode. 812 new_nthreads = set_nthreads; 813 if (!get__dynamic_2(parent_team, master_tid)) { 814 ; 815 } 816 #ifdef USE_LOAD_BALANCE 817 else if (__kmp_global.g.g_dynamic_mode == dynamic_load_balance) { 818 new_nthreads = __kmp_load_balance_nproc(root, set_nthreads); 819 if (new_nthreads == 1) { 820 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced " 821 "reservation to 1 thread\n", 822 master_tid)); 823 return 1; 824 } 825 if (new_nthreads < set_nthreads) { 826 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced " 827 "reservation to %d threads\n", 828 master_tid, new_nthreads)); 829 } 830 } 831 #endif /* USE_LOAD_BALANCE */ 832 else if (__kmp_global.g.g_dynamic_mode == dynamic_thread_limit) { 833 new_nthreads = __kmp_avail_proc - __kmp_nth + 834 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc); 835 if (new_nthreads <= 1) { 836 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced " 837 "reservation to 1 thread\n", 838 master_tid)); 839 return 1; 840 } 841 if (new_nthreads < set_nthreads) { 842 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced " 843 "reservation to %d threads\n", 844 master_tid, new_nthreads)); 845 } else { 846 new_nthreads = set_nthreads; 847 } 848 } else if (__kmp_global.g.g_dynamic_mode == dynamic_random) { 849 if (set_nthreads > 2) { 850 new_nthreads = __kmp_get_random(parent_team->t.t_threads[master_tid]); 851 new_nthreads = (new_nthreads % set_nthreads) + 1; 852 if (new_nthreads == 1) { 853 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced " 854 "reservation to 1 thread\n", 855 master_tid)); 856 return 1; 857 } 858 if (new_nthreads < set_nthreads) { 859 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced " 860 "reservation to %d threads\n", 861 master_tid, new_nthreads)); 862 } 863 } 864 } else { 865 KMP_ASSERT(0); 866 } 867 868 // Respect KMP_ALL_THREADS/KMP_DEVICE_THREAD_LIMIT. 869 if (__kmp_nth + new_nthreads - 870 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) > 871 __kmp_max_nth) { 872 int tl_nthreads = __kmp_max_nth - __kmp_nth + 873 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc); 874 if (tl_nthreads <= 0) { 875 tl_nthreads = 1; 876 } 877 878 // If dyn-var is false, emit a 1-time warning. 879 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) { 880 __kmp_reserve_warn = 1; 881 __kmp_msg(kmp_ms_warning, 882 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads), 883 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 884 } 885 if (tl_nthreads == 1) { 886 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT " 887 "reduced reservation to 1 thread\n", 888 master_tid)); 889 return 1; 890 } 891 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced " 892 "reservation to %d threads\n", 893 master_tid, tl_nthreads)); 894 new_nthreads = tl_nthreads; 895 } 896 897 // Respect OMP_THREAD_LIMIT 898 int cg_nthreads = this_thr->th.th_cg_roots->cg_nthreads; 899 int max_cg_threads = this_thr->th.th_cg_roots->cg_thread_limit; 900 if (cg_nthreads + new_nthreads - 901 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) > 902 max_cg_threads) { 903 int tl_nthreads = max_cg_threads - cg_nthreads + 904 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc); 905 if (tl_nthreads <= 0) { 906 tl_nthreads = 1; 907 } 908 909 // If dyn-var is false, emit a 1-time warning. 910 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) { 911 __kmp_reserve_warn = 1; 912 __kmp_msg(kmp_ms_warning, 913 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads), 914 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 915 } 916 if (tl_nthreads == 1) { 917 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT " 918 "reduced reservation to 1 thread\n", 919 master_tid)); 920 return 1; 921 } 922 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced " 923 "reservation to %d threads\n", 924 master_tid, tl_nthreads)); 925 new_nthreads = tl_nthreads; 926 } 927 928 // Check if the threads array is large enough, or needs expanding. 929 // See comment in __kmp_register_root() about the adjustment if 930 // __kmp_threads[0] == NULL. 931 capacity = __kmp_threads_capacity; 932 if (TCR_PTR(__kmp_threads[0]) == NULL) { 933 --capacity; 934 } 935 if (__kmp_nth + new_nthreads - 936 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) > 937 capacity) { 938 // Expand the threads array. 939 int slotsRequired = __kmp_nth + new_nthreads - 940 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) - 941 capacity; 942 int slotsAdded = __kmp_expand_threads(slotsRequired); 943 if (slotsAdded < slotsRequired) { 944 // The threads array was not expanded enough. 945 new_nthreads -= (slotsRequired - slotsAdded); 946 KMP_ASSERT(new_nthreads >= 1); 947 948 // If dyn-var is false, emit a 1-time warning. 949 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) { 950 __kmp_reserve_warn = 1; 951 if (__kmp_tp_cached) { 952 __kmp_msg(kmp_ms_warning, 953 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads), 954 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity), 955 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null); 956 } else { 957 __kmp_msg(kmp_ms_warning, 958 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads), 959 KMP_HNT(SystemLimitOnThreads), __kmp_msg_null); 960 } 961 } 962 } 963 } 964 965 #ifdef KMP_DEBUG 966 if (new_nthreads == 1) { 967 KC_TRACE(10, 968 ("__kmp_reserve_threads: T#%d serializing team after reclaiming " 969 "dead roots and rechecking; requested %d threads\n", 970 __kmp_get_gtid(), set_nthreads)); 971 } else { 972 KC_TRACE(10, ("__kmp_reserve_threads: T#%d allocating %d threads; requested" 973 " %d threads\n", 974 __kmp_get_gtid(), new_nthreads, set_nthreads)); 975 } 976 #endif // KMP_DEBUG 977 return new_nthreads; 978 } 979 980 /* Allocate threads from the thread pool and assign them to the new team. We are 981 assured that there are enough threads available, because we checked on that 982 earlier within critical section forkjoin */ 983 static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team, 984 kmp_info_t *master_th, int master_gtid) { 985 int i; 986 int use_hot_team; 987 988 KA_TRACE(10, ("__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc)); 989 KMP_DEBUG_ASSERT(master_gtid == __kmp_get_gtid()); 990 KMP_MB(); 991 992 /* first, let's setup the master thread */ 993 master_th->th.th_info.ds.ds_tid = 0; 994 master_th->th.th_team = team; 995 master_th->th.th_team_nproc = team->t.t_nproc; 996 master_th->th.th_team_master = master_th; 997 master_th->th.th_team_serialized = FALSE; 998 master_th->th.th_dispatch = &team->t.t_dispatch[0]; 999 1000 /* make sure we are not the optimized hot team */ 1001 #if KMP_NESTED_HOT_TEAMS 1002 use_hot_team = 0; 1003 kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams; 1004 if (hot_teams) { // hot teams array is not allocated if 1005 // KMP_HOT_TEAMS_MAX_LEVEL=0 1006 int level = team->t.t_active_level - 1; // index in array of hot teams 1007 if (master_th->th.th_teams_microtask) { // are we inside the teams? 1008 if (master_th->th.th_teams_size.nteams > 1) { 1009 ++level; // level was not increased in teams construct for 1010 // team_of_masters 1011 } 1012 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master && 1013 master_th->th.th_teams_level == team->t.t_level) { 1014 ++level; // level was not increased in teams construct for 1015 // team_of_workers before the parallel 1016 } // team->t.t_level will be increased inside parallel 1017 } 1018 if (level < __kmp_hot_teams_max_level) { 1019 if (hot_teams[level].hot_team) { 1020 // hot team has already been allocated for given level 1021 KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team); 1022 use_hot_team = 1; // the team is ready to use 1023 } else { 1024 use_hot_team = 0; // AC: threads are not allocated yet 1025 hot_teams[level].hot_team = team; // remember new hot team 1026 hot_teams[level].hot_team_nth = team->t.t_nproc; 1027 } 1028 } else { 1029 use_hot_team = 0; 1030 } 1031 } 1032 #else 1033 use_hot_team = team == root->r.r_hot_team; 1034 #endif 1035 if (!use_hot_team) { 1036 1037 /* install the master thread */ 1038 team->t.t_threads[0] = master_th; 1039 __kmp_initialize_info(master_th, team, 0, master_gtid); 1040 1041 /* now, install the worker threads */ 1042 for (i = 1; i < team->t.t_nproc; i++) { 1043 1044 /* fork or reallocate a new thread and install it in team */ 1045 kmp_info_t *thr = __kmp_allocate_thread(root, team, i); 1046 team->t.t_threads[i] = thr; 1047 KMP_DEBUG_ASSERT(thr); 1048 KMP_DEBUG_ASSERT(thr->th.th_team == team); 1049 /* align team and thread arrived states */ 1050 KA_TRACE(20, ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived " 1051 "T#%d(%d:%d) join =%llu, plain=%llu\n", 1052 __kmp_gtid_from_tid(0, team), team->t.t_id, 0, 1053 __kmp_gtid_from_tid(i, team), team->t.t_id, i, 1054 team->t.t_bar[bs_forkjoin_barrier].b_arrived, 1055 team->t.t_bar[bs_plain_barrier].b_arrived)); 1056 #if OMP_40_ENABLED 1057 thr->th.th_teams_microtask = master_th->th.th_teams_microtask; 1058 thr->th.th_teams_level = master_th->th.th_teams_level; 1059 thr->th.th_teams_size = master_th->th.th_teams_size; 1060 #endif 1061 { // Initialize threads' barrier data. 1062 int b; 1063 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar; 1064 for (b = 0; b < bs_last_barrier; ++b) { 1065 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 1066 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 1067 #if USE_DEBUGGER 1068 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 1069 #endif 1070 } 1071 } 1072 } 1073 1074 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 1075 __kmp_partition_places(team); 1076 #endif 1077 } 1078 1079 #if OMP_50_ENABLED 1080 if (__kmp_display_affinity && team->t.t_display_affinity != 1) { 1081 for (i = 0; i < team->t.t_nproc; i++) { 1082 kmp_info_t *thr = team->t.t_threads[i]; 1083 if (thr->th.th_prev_num_threads != team->t.t_nproc || 1084 thr->th.th_prev_level != team->t.t_level) { 1085 team->t.t_display_affinity = 1; 1086 break; 1087 } 1088 } 1089 } 1090 #endif 1091 1092 KMP_MB(); 1093 } 1094 1095 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1096 // Propagate any changes to the floating point control registers out to the team 1097 // We try to avoid unnecessary writes to the relevant cache line in the team 1098 // structure, so we don't make changes unless they are needed. 1099 inline static void propagateFPControl(kmp_team_t *team) { 1100 if (__kmp_inherit_fp_control) { 1101 kmp_int16 x87_fpu_control_word; 1102 kmp_uint32 mxcsr; 1103 1104 // Get master values of FPU control flags (both X87 and vector) 1105 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word); 1106 __kmp_store_mxcsr(&mxcsr); 1107 mxcsr &= KMP_X86_MXCSR_MASK; 1108 1109 // There is no point looking at t_fp_control_saved here. 1110 // If it is TRUE, we still have to update the values if they are different 1111 // from those we now have. If it is FALSE we didn't save anything yet, but 1112 // our objective is the same. We have to ensure that the values in the team 1113 // are the same as those we have. 1114 // So, this code achieves what we need whether or not t_fp_control_saved is 1115 // true. By checking whether the value needs updating we avoid unnecessary 1116 // writes that would put the cache-line into a written state, causing all 1117 // threads in the team to have to read it again. 1118 KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word); 1119 KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr); 1120 // Although we don't use this value, other code in the runtime wants to know 1121 // whether it should restore them. So we must ensure it is correct. 1122 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE); 1123 } else { 1124 // Similarly here. Don't write to this cache-line in the team structure 1125 // unless we have to. 1126 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE); 1127 } 1128 } 1129 1130 // Do the opposite, setting the hardware registers to the updated values from 1131 // the team. 1132 inline static void updateHWFPControl(kmp_team_t *team) { 1133 if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) { 1134 // Only reset the fp control regs if they have been changed in the team. 1135 // the parallel region that we are exiting. 1136 kmp_int16 x87_fpu_control_word; 1137 kmp_uint32 mxcsr; 1138 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word); 1139 __kmp_store_mxcsr(&mxcsr); 1140 mxcsr &= KMP_X86_MXCSR_MASK; 1141 1142 if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) { 1143 __kmp_clear_x87_fpu_status_word(); 1144 __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word); 1145 } 1146 1147 if (team->t.t_mxcsr != mxcsr) { 1148 __kmp_load_mxcsr(&team->t.t_mxcsr); 1149 } 1150 } 1151 } 1152 #else 1153 #define propagateFPControl(x) ((void)0) 1154 #define updateHWFPControl(x) ((void)0) 1155 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 1156 1157 static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team, 1158 int realloc); // forward declaration 1159 1160 /* Run a parallel region that has been serialized, so runs only in a team of the 1161 single master thread. */ 1162 void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) { 1163 kmp_info_t *this_thr; 1164 kmp_team_t *serial_team; 1165 1166 KC_TRACE(10, ("__kmpc_serialized_parallel: called by T#%d\n", global_tid)); 1167 1168 /* Skip all this code for autopar serialized loops since it results in 1169 unacceptable overhead */ 1170 if (loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR)) 1171 return; 1172 1173 if (!TCR_4(__kmp_init_parallel)) 1174 __kmp_parallel_initialize(); 1175 1176 #if OMP_50_ENABLED 1177 __kmp_resume_if_soft_paused(); 1178 #endif 1179 1180 this_thr = __kmp_threads[global_tid]; 1181 serial_team = this_thr->th.th_serial_team; 1182 1183 /* utilize the serialized team held by this thread */ 1184 KMP_DEBUG_ASSERT(serial_team); 1185 KMP_MB(); 1186 1187 if (__kmp_tasking_mode != tskm_immediate_exec) { 1188 KMP_DEBUG_ASSERT( 1189 this_thr->th.th_task_team == 1190 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]); 1191 KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] == 1192 NULL); 1193 KA_TRACE(20, ("__kmpc_serialized_parallel: T#%d pushing task_team %p / " 1194 "team %p, new task_team = NULL\n", 1195 global_tid, this_thr->th.th_task_team, this_thr->th.th_team)); 1196 this_thr->th.th_task_team = NULL; 1197 } 1198 1199 #if OMP_40_ENABLED 1200 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind; 1201 if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) { 1202 proc_bind = proc_bind_false; 1203 } else if (proc_bind == proc_bind_default) { 1204 // No proc_bind clause was specified, so use the current value 1205 // of proc-bind-var for this parallel region. 1206 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind; 1207 } 1208 // Reset for next parallel region 1209 this_thr->th.th_set_proc_bind = proc_bind_default; 1210 #endif /* OMP_40_ENABLED */ 1211 1212 #if OMPT_SUPPORT 1213 ompt_data_t ompt_parallel_data = ompt_data_none; 1214 ompt_data_t *implicit_task_data; 1215 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid); 1216 if (ompt_enabled.enabled && 1217 this_thr->th.ompt_thread_info.state != ompt_state_overhead) { 1218 1219 ompt_task_info_t *parent_task_info; 1220 parent_task_info = OMPT_CUR_TASK_INFO(this_thr); 1221 1222 parent_task_info->frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1223 if (ompt_enabled.ompt_callback_parallel_begin) { 1224 int team_size = 1; 1225 1226 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)( 1227 &(parent_task_info->task_data), &(parent_task_info->frame), 1228 &ompt_parallel_data, team_size, ompt_parallel_invoker_program, 1229 codeptr); 1230 } 1231 } 1232 #endif // OMPT_SUPPORT 1233 1234 if (this_thr->th.th_team != serial_team) { 1235 // Nested level will be an index in the nested nthreads array 1236 int level = this_thr->th.th_team->t.t_level; 1237 1238 if (serial_team->t.t_serialized) { 1239 /* this serial team was already used 1240 TODO increase performance by making this locks more specific */ 1241 kmp_team_t *new_team; 1242 1243 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 1244 1245 new_team = __kmp_allocate_team(this_thr->th.th_root, 1, 1, 1246 #if OMPT_SUPPORT 1247 ompt_parallel_data, 1248 #endif 1249 #if OMP_40_ENABLED 1250 proc_bind, 1251 #endif 1252 &this_thr->th.th_current_task->td_icvs, 1253 0 USE_NESTED_HOT_ARG(NULL)); 1254 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 1255 KMP_ASSERT(new_team); 1256 1257 /* setup new serialized team and install it */ 1258 new_team->t.t_threads[0] = this_thr; 1259 new_team->t.t_parent = this_thr->th.th_team; 1260 serial_team = new_team; 1261 this_thr->th.th_serial_team = serial_team; 1262 1263 KF_TRACE( 1264 10, 1265 ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n", 1266 global_tid, serial_team)); 1267 1268 /* TODO the above breaks the requirement that if we run out of resources, 1269 then we can still guarantee that serialized teams are ok, since we may 1270 need to allocate a new one */ 1271 } else { 1272 KF_TRACE( 1273 10, 1274 ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n", 1275 global_tid, serial_team)); 1276 } 1277 1278 /* we have to initialize this serial team */ 1279 KMP_DEBUG_ASSERT(serial_team->t.t_threads); 1280 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr); 1281 KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team); 1282 serial_team->t.t_ident = loc; 1283 serial_team->t.t_serialized = 1; 1284 serial_team->t.t_nproc = 1; 1285 serial_team->t.t_parent = this_thr->th.th_team; 1286 serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched; 1287 this_thr->th.th_team = serial_team; 1288 serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid; 1289 1290 KF_TRACE(10, ("__kmpc_serialized_parallel: T#d curtask=%p\n", global_tid, 1291 this_thr->th.th_current_task)); 1292 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1); 1293 this_thr->th.th_current_task->td_flags.executing = 0; 1294 1295 __kmp_push_current_task_to_thread(this_thr, serial_team, 0); 1296 1297 /* TODO: GEH: do ICVs work for nested serialized teams? Don't we need an 1298 implicit task for each serialized task represented by 1299 team->t.t_serialized? */ 1300 copy_icvs(&this_thr->th.th_current_task->td_icvs, 1301 &this_thr->th.th_current_task->td_parent->td_icvs); 1302 1303 // Thread value exists in the nested nthreads array for the next nested 1304 // level 1305 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) { 1306 this_thr->th.th_current_task->td_icvs.nproc = 1307 __kmp_nested_nth.nth[level + 1]; 1308 } 1309 1310 #if OMP_40_ENABLED 1311 if (__kmp_nested_proc_bind.used && 1312 (level + 1 < __kmp_nested_proc_bind.used)) { 1313 this_thr->th.th_current_task->td_icvs.proc_bind = 1314 __kmp_nested_proc_bind.bind_types[level + 1]; 1315 } 1316 #endif /* OMP_40_ENABLED */ 1317 1318 #if USE_DEBUGGER 1319 serial_team->t.t_pkfn = (microtask_t)(~0); // For the debugger. 1320 #endif 1321 this_thr->th.th_info.ds.ds_tid = 0; 1322 1323 /* set thread cache values */ 1324 this_thr->th.th_team_nproc = 1; 1325 this_thr->th.th_team_master = this_thr; 1326 this_thr->th.th_team_serialized = 1; 1327 1328 serial_team->t.t_level = serial_team->t.t_parent->t.t_level + 1; 1329 serial_team->t.t_active_level = serial_team->t.t_parent->t.t_active_level; 1330 #if OMP_50_ENABLED 1331 serial_team->t.t_def_allocator = this_thr->th.th_def_allocator; // save 1332 #endif 1333 1334 propagateFPControl(serial_team); 1335 1336 /* check if we need to allocate dispatch buffers stack */ 1337 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch); 1338 if (!serial_team->t.t_dispatch->th_disp_buffer) { 1339 serial_team->t.t_dispatch->th_disp_buffer = 1340 (dispatch_private_info_t *)__kmp_allocate( 1341 sizeof(dispatch_private_info_t)); 1342 } 1343 this_thr->th.th_dispatch = serial_team->t.t_dispatch; 1344 1345 KMP_MB(); 1346 1347 } else { 1348 /* this serialized team is already being used, 1349 * that's fine, just add another nested level */ 1350 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team); 1351 KMP_DEBUG_ASSERT(serial_team->t.t_threads); 1352 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr); 1353 ++serial_team->t.t_serialized; 1354 this_thr->th.th_team_serialized = serial_team->t.t_serialized; 1355 1356 // Nested level will be an index in the nested nthreads array 1357 int level = this_thr->th.th_team->t.t_level; 1358 // Thread value exists in the nested nthreads array for the next nested 1359 // level 1360 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) { 1361 this_thr->th.th_current_task->td_icvs.nproc = 1362 __kmp_nested_nth.nth[level + 1]; 1363 } 1364 serial_team->t.t_level++; 1365 KF_TRACE(10, ("__kmpc_serialized_parallel: T#%d increasing nesting level " 1366 "of serial team %p to %d\n", 1367 global_tid, serial_team, serial_team->t.t_level)); 1368 1369 /* allocate/push dispatch buffers stack */ 1370 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch); 1371 { 1372 dispatch_private_info_t *disp_buffer = 1373 (dispatch_private_info_t *)__kmp_allocate( 1374 sizeof(dispatch_private_info_t)); 1375 disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer; 1376 serial_team->t.t_dispatch->th_disp_buffer = disp_buffer; 1377 } 1378 this_thr->th.th_dispatch = serial_team->t.t_dispatch; 1379 1380 KMP_MB(); 1381 } 1382 #if OMP_40_ENABLED 1383 KMP_CHECK_UPDATE(serial_team->t.t_cancel_request, cancel_noreq); 1384 #endif 1385 1386 #if OMP_50_ENABLED 1387 // Perform the display affinity functionality for 1388 // serialized parallel regions 1389 if (__kmp_display_affinity) { 1390 if (this_thr->th.th_prev_level != serial_team->t.t_level || 1391 this_thr->th.th_prev_num_threads != 1) { 1392 // NULL means use the affinity-format-var ICV 1393 __kmp_aux_display_affinity(global_tid, NULL); 1394 this_thr->th.th_prev_level = serial_team->t.t_level; 1395 this_thr->th.th_prev_num_threads = 1; 1396 } 1397 } 1398 #endif 1399 1400 if (__kmp_env_consistency_check) 1401 __kmp_push_parallel(global_tid, NULL); 1402 #if OMPT_SUPPORT 1403 serial_team->t.ompt_team_info.master_return_address = codeptr; 1404 if (ompt_enabled.enabled && 1405 this_thr->th.ompt_thread_info.state != ompt_state_overhead) { 1406 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1407 1408 ompt_lw_taskteam_t lw_taskteam; 1409 __ompt_lw_taskteam_init(&lw_taskteam, this_thr, global_tid, 1410 &ompt_parallel_data, codeptr); 1411 1412 __ompt_lw_taskteam_link(&lw_taskteam, this_thr, 1); 1413 // don't use lw_taskteam after linking. content was swaped 1414 1415 /* OMPT implicit task begin */ 1416 implicit_task_data = OMPT_CUR_TASK_DATA(this_thr); 1417 if (ompt_enabled.ompt_callback_implicit_task) { 1418 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1419 ompt_scope_begin, OMPT_CUR_TEAM_DATA(this_thr), 1420 OMPT_CUR_TASK_DATA(this_thr), 1, __kmp_tid_from_gtid(global_tid), ompt_task_implicit); // TODO: Can this be ompt_task_initial? 1421 OMPT_CUR_TASK_INFO(this_thr) 1422 ->thread_num = __kmp_tid_from_gtid(global_tid); 1423 } 1424 1425 /* OMPT state */ 1426 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel; 1427 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1428 } 1429 #endif 1430 } 1431 1432 /* most of the work for a fork */ 1433 /* return true if we really went parallel, false if serialized */ 1434 int __kmp_fork_call(ident_t *loc, int gtid, 1435 enum fork_context_e call_context, // Intel, GNU, ... 1436 kmp_int32 argc, microtask_t microtask, launch_t invoker, 1437 /* TODO: revert workaround for Intel(R) 64 tracker #96 */ 1438 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 1439 va_list *ap 1440 #else 1441 va_list ap 1442 #endif 1443 ) { 1444 void **argv; 1445 int i; 1446 int master_tid; 1447 int master_this_cons; 1448 kmp_team_t *team; 1449 kmp_team_t *parent_team; 1450 kmp_info_t *master_th; 1451 kmp_root_t *root; 1452 int nthreads; 1453 int master_active; 1454 int master_set_numthreads; 1455 int level; 1456 #if OMP_40_ENABLED 1457 int active_level; 1458 int teams_level; 1459 #endif 1460 #if KMP_NESTED_HOT_TEAMS 1461 kmp_hot_team_ptr_t **p_hot_teams; 1462 #endif 1463 { // KMP_TIME_BLOCK 1464 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_fork_call); 1465 KMP_COUNT_VALUE(OMP_PARALLEL_args, argc); 1466 1467 KA_TRACE(20, ("__kmp_fork_call: enter T#%d\n", gtid)); 1468 if (__kmp_stkpadding > 0 && __kmp_root[gtid] != NULL) { 1469 /* Some systems prefer the stack for the root thread(s) to start with */ 1470 /* some gap from the parent stack to prevent false sharing. */ 1471 void *dummy = KMP_ALLOCA(__kmp_stkpadding); 1472 /* These 2 lines below are so this does not get optimized out */ 1473 if (__kmp_stkpadding > KMP_MAX_STKPADDING) 1474 __kmp_stkpadding += (short)((kmp_int64)dummy); 1475 } 1476 1477 /* initialize if needed */ 1478 KMP_DEBUG_ASSERT( 1479 __kmp_init_serial); // AC: potentially unsafe, not in sync with shutdown 1480 if (!TCR_4(__kmp_init_parallel)) 1481 __kmp_parallel_initialize(); 1482 1483 #if OMP_50_ENABLED 1484 __kmp_resume_if_soft_paused(); 1485 #endif 1486 1487 /* setup current data */ 1488 master_th = __kmp_threads[gtid]; // AC: potentially unsafe, not in sync with 1489 // shutdown 1490 parent_team = master_th->th.th_team; 1491 master_tid = master_th->th.th_info.ds.ds_tid; 1492 master_this_cons = master_th->th.th_local.this_construct; 1493 root = master_th->th.th_root; 1494 master_active = root->r.r_active; 1495 master_set_numthreads = master_th->th.th_set_nproc; 1496 1497 #if OMPT_SUPPORT 1498 ompt_data_t ompt_parallel_data = ompt_data_none; 1499 ompt_data_t *parent_task_data; 1500 ompt_frame_t *ompt_frame; 1501 ompt_data_t *implicit_task_data; 1502 void *return_address = NULL; 1503 1504 if (ompt_enabled.enabled) { 1505 __ompt_get_task_info_internal(0, NULL, &parent_task_data, &ompt_frame, 1506 NULL, NULL); 1507 return_address = OMPT_LOAD_RETURN_ADDRESS(gtid); 1508 } 1509 #endif 1510 1511 // Nested level will be an index in the nested nthreads array 1512 level = parent_team->t.t_level; 1513 // used to launch non-serial teams even if nested is not allowed 1514 active_level = parent_team->t.t_active_level; 1515 #if OMP_40_ENABLED 1516 // needed to check nesting inside the teams 1517 teams_level = master_th->th.th_teams_level; 1518 #endif 1519 #if KMP_NESTED_HOT_TEAMS 1520 p_hot_teams = &master_th->th.th_hot_teams; 1521 if (*p_hot_teams == NULL && __kmp_hot_teams_max_level > 0) { 1522 *p_hot_teams = (kmp_hot_team_ptr_t *)__kmp_allocate( 1523 sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level); 1524 (*p_hot_teams)[0].hot_team = root->r.r_hot_team; 1525 // it is either actual or not needed (when active_level > 0) 1526 (*p_hot_teams)[0].hot_team_nth = 1; 1527 } 1528 #endif 1529 1530 #if OMPT_SUPPORT 1531 if (ompt_enabled.enabled) { 1532 if (ompt_enabled.ompt_callback_parallel_begin) { 1533 int team_size = master_set_numthreads 1534 ? master_set_numthreads 1535 : get__nproc_2(parent_team, master_tid); 1536 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)( 1537 parent_task_data, ompt_frame, &ompt_parallel_data, team_size, 1538 OMPT_INVOKER(call_context), return_address); 1539 } 1540 master_th->th.ompt_thread_info.state = ompt_state_overhead; 1541 } 1542 #endif 1543 1544 master_th->th.th_ident = loc; 1545 1546 #if OMP_40_ENABLED 1547 if (master_th->th.th_teams_microtask && ap && 1548 microtask != (microtask_t)__kmp_teams_master && level == teams_level) { 1549 // AC: This is start of parallel that is nested inside teams construct. 1550 // The team is actual (hot), all workers are ready at the fork barrier. 1551 // No lock needed to initialize the team a bit, then free workers. 1552 parent_team->t.t_ident = loc; 1553 __kmp_alloc_argv_entries(argc, parent_team, TRUE); 1554 parent_team->t.t_argc = argc; 1555 argv = (void **)parent_team->t.t_argv; 1556 for (i = argc - 1; i >= 0; --i) 1557 /* TODO: revert workaround for Intel(R) 64 tracker #96 */ 1558 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 1559 *argv++ = va_arg(*ap, void *); 1560 #else 1561 *argv++ = va_arg(ap, void *); 1562 #endif 1563 // Increment our nested depth levels, but not increase the serialization 1564 if (parent_team == master_th->th.th_serial_team) { 1565 // AC: we are in serialized parallel 1566 __kmpc_serialized_parallel(loc, gtid); 1567 KMP_DEBUG_ASSERT(parent_team->t.t_serialized > 1); 1568 // AC: need this in order enquiry functions work 1569 // correctly, will restore at join time 1570 parent_team->t.t_serialized--; 1571 #if OMPT_SUPPORT 1572 void *dummy; 1573 void **exit_runtime_p; 1574 1575 ompt_lw_taskteam_t lw_taskteam; 1576 1577 if (ompt_enabled.enabled) { 1578 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid, 1579 &ompt_parallel_data, return_address); 1580 exit_runtime_p = &(lw_taskteam.ompt_task_info.frame.exit_frame.ptr); 1581 1582 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0); 1583 // don't use lw_taskteam after linking. content was swaped 1584 1585 /* OMPT implicit task begin */ 1586 implicit_task_data = OMPT_CUR_TASK_DATA(master_th); 1587 if (ompt_enabled.ompt_callback_implicit_task) { 1588 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1589 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th), 1590 implicit_task_data, 1, __kmp_tid_from_gtid(gtid), ompt_task_implicit); // TODO: Can this be ompt_task_initial? 1591 OMPT_CUR_TASK_INFO(master_th) 1592 ->thread_num = __kmp_tid_from_gtid(gtid); 1593 } 1594 1595 /* OMPT state */ 1596 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; 1597 } else { 1598 exit_runtime_p = &dummy; 1599 } 1600 #endif 1601 1602 { 1603 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 1604 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 1605 __kmp_invoke_microtask(microtask, gtid, 0, argc, parent_team->t.t_argv 1606 #if OMPT_SUPPORT 1607 , 1608 exit_runtime_p 1609 #endif 1610 ); 1611 } 1612 1613 #if OMPT_SUPPORT 1614 *exit_runtime_p = NULL; 1615 if (ompt_enabled.enabled) { 1616 OMPT_CUR_TASK_INFO(master_th)->frame.exit_frame = ompt_data_none; 1617 if (ompt_enabled.ompt_callback_implicit_task) { 1618 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1619 ompt_scope_end, NULL, implicit_task_data, 1, 1620 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit); // TODO: Can this be ompt_task_initial? 1621 } 1622 __ompt_lw_taskteam_unlink(master_th); 1623 1624 if (ompt_enabled.ompt_callback_parallel_end) { 1625 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)( 1626 OMPT_CUR_TEAM_DATA(master_th), OMPT_CUR_TASK_DATA(master_th), 1627 OMPT_INVOKER(call_context), return_address); 1628 } 1629 master_th->th.ompt_thread_info.state = ompt_state_overhead; 1630 } 1631 #endif 1632 return TRUE; 1633 } 1634 1635 parent_team->t.t_pkfn = microtask; 1636 parent_team->t.t_invoke = invoker; 1637 KMP_ATOMIC_INC(&root->r.r_in_parallel); 1638 parent_team->t.t_active_level++; 1639 parent_team->t.t_level++; 1640 #if OMP_50_ENABLED 1641 parent_team->t.t_def_allocator = master_th->th.th_def_allocator; // save 1642 #endif 1643 1644 /* Change number of threads in the team if requested */ 1645 if (master_set_numthreads) { // The parallel has num_threads clause 1646 if (master_set_numthreads < master_th->th.th_teams_size.nth) { 1647 // AC: only can reduce number of threads dynamically, can't increase 1648 kmp_info_t **other_threads = parent_team->t.t_threads; 1649 parent_team->t.t_nproc = master_set_numthreads; 1650 for (i = 0; i < master_set_numthreads; ++i) { 1651 other_threads[i]->th.th_team_nproc = master_set_numthreads; 1652 } 1653 // Keep extra threads hot in the team for possible next parallels 1654 } 1655 master_th->th.th_set_nproc = 0; 1656 } 1657 1658 #if USE_DEBUGGER 1659 if (__kmp_debugging) { // Let debugger override number of threads. 1660 int nth = __kmp_omp_num_threads(loc); 1661 if (nth > 0) { // 0 means debugger doesn't want to change num threads 1662 master_set_numthreads = nth; 1663 } 1664 } 1665 #endif 1666 1667 KF_TRACE(10, ("__kmp_fork_call: before internal fork: root=%p, team=%p, " 1668 "master_th=%p, gtid=%d\n", 1669 root, parent_team, master_th, gtid)); 1670 __kmp_internal_fork(loc, gtid, parent_team); 1671 KF_TRACE(10, ("__kmp_fork_call: after internal fork: root=%p, team=%p, " 1672 "master_th=%p, gtid=%d\n", 1673 root, parent_team, master_th, gtid)); 1674 1675 /* Invoke microtask for MASTER thread */ 1676 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid, 1677 parent_team->t.t_id, parent_team->t.t_pkfn)); 1678 1679 if (!parent_team->t.t_invoke(gtid)) { 1680 KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread"); 1681 } 1682 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid, 1683 parent_team->t.t_id, parent_team->t.t_pkfn)); 1684 KMP_MB(); /* Flush all pending memory write invalidates. */ 1685 1686 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid)); 1687 1688 return TRUE; 1689 } // Parallel closely nested in teams construct 1690 #endif /* OMP_40_ENABLED */ 1691 1692 #if KMP_DEBUG 1693 if (__kmp_tasking_mode != tskm_immediate_exec) { 1694 KMP_DEBUG_ASSERT(master_th->th.th_task_team == 1695 parent_team->t.t_task_team[master_th->th.th_task_state]); 1696 } 1697 #endif 1698 1699 if (parent_team->t.t_active_level >= 1700 master_th->th.th_current_task->td_icvs.max_active_levels) { 1701 nthreads = 1; 1702 } else { 1703 #if OMP_40_ENABLED 1704 int enter_teams = ((ap == NULL && active_level == 0) || 1705 (ap && teams_level > 0 && teams_level == level)); 1706 #endif 1707 nthreads = 1708 master_set_numthreads 1709 ? master_set_numthreads 1710 : get__nproc_2( 1711 parent_team, 1712 master_tid); // TODO: get nproc directly from current task 1713 1714 // Check if we need to take forkjoin lock? (no need for serialized 1715 // parallel out of teams construct). This code moved here from 1716 // __kmp_reserve_threads() to speedup nested serialized parallels. 1717 if (nthreads > 1) { 1718 if ((get__max_active_levels(master_th) == 1 && (root->r.r_in_parallel 1719 #if OMP_40_ENABLED 1720 && !enter_teams 1721 #endif /* OMP_40_ENABLED */ 1722 )) || 1723 (__kmp_library == library_serial)) { 1724 KC_TRACE(10, ("__kmp_fork_call: T#%d serializing team; requested %d" 1725 " threads\n", 1726 gtid, nthreads)); 1727 nthreads = 1; 1728 } 1729 } 1730 if (nthreads > 1) { 1731 /* determine how many new threads we can use */ 1732 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 1733 nthreads = __kmp_reserve_threads( 1734 root, parent_team, master_tid, nthreads 1735 #if OMP_40_ENABLED 1736 /* AC: If we execute teams from parallel region (on host), then 1737 teams should be created but each can only have 1 thread if 1738 nesting is disabled. If teams called from serial region, then 1739 teams and their threads should be created regardless of the 1740 nesting setting. */ 1741 , 1742 enter_teams 1743 #endif /* OMP_40_ENABLED */ 1744 ); 1745 if (nthreads == 1) { 1746 // Free lock for single thread execution here; for multi-thread 1747 // execution it will be freed later after team of threads created 1748 // and initialized 1749 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 1750 } 1751 } 1752 } 1753 KMP_DEBUG_ASSERT(nthreads > 0); 1754 1755 // If we temporarily changed the set number of threads then restore it now 1756 master_th->th.th_set_nproc = 0; 1757 1758 /* create a serialized parallel region? */ 1759 if (nthreads == 1) { 1760 /* josh todo: hypothetical question: what do we do for OS X*? */ 1761 #if KMP_OS_LINUX && \ 1762 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 1763 void *args[argc]; 1764 #else 1765 void **args = (void **)KMP_ALLOCA(argc * sizeof(void *)); 1766 #endif /* KMP_OS_LINUX && ( KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || \ 1767 KMP_ARCH_AARCH64) */ 1768 1769 KA_TRACE(20, 1770 ("__kmp_fork_call: T#%d serializing parallel region\n", gtid)); 1771 1772 __kmpc_serialized_parallel(loc, gtid); 1773 1774 if (call_context == fork_context_intel) { 1775 /* TODO this sucks, use the compiler itself to pass args! :) */ 1776 master_th->th.th_serial_team->t.t_ident = loc; 1777 #if OMP_40_ENABLED 1778 if (!ap) { 1779 // revert change made in __kmpc_serialized_parallel() 1780 master_th->th.th_serial_team->t.t_level--; 1781 // Get args from parent team for teams construct 1782 1783 #if OMPT_SUPPORT 1784 void *dummy; 1785 void **exit_runtime_p; 1786 ompt_task_info_t *task_info; 1787 1788 ompt_lw_taskteam_t lw_taskteam; 1789 1790 if (ompt_enabled.enabled) { 1791 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid, 1792 &ompt_parallel_data, return_address); 1793 1794 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0); 1795 // don't use lw_taskteam after linking. content was swaped 1796 1797 task_info = OMPT_CUR_TASK_INFO(master_th); 1798 exit_runtime_p = &(task_info->frame.exit_frame.ptr); 1799 if (ompt_enabled.ompt_callback_implicit_task) { 1800 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1801 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th), 1802 &(task_info->task_data), 1, __kmp_tid_from_gtid(gtid), ompt_task_implicit); // TODO: Can this be ompt_task_initial? 1803 OMPT_CUR_TASK_INFO(master_th) 1804 ->thread_num = __kmp_tid_from_gtid(gtid); 1805 } 1806 1807 /* OMPT state */ 1808 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; 1809 } else { 1810 exit_runtime_p = &dummy; 1811 } 1812 #endif 1813 1814 { 1815 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 1816 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 1817 __kmp_invoke_microtask(microtask, gtid, 0, argc, 1818 parent_team->t.t_argv 1819 #if OMPT_SUPPORT 1820 , 1821 exit_runtime_p 1822 #endif 1823 ); 1824 } 1825 1826 #if OMPT_SUPPORT 1827 if (ompt_enabled.enabled) { 1828 exit_runtime_p = NULL; 1829 if (ompt_enabled.ompt_callback_implicit_task) { 1830 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1831 ompt_scope_end, NULL, &(task_info->task_data), 1, 1832 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit); // TODO: Can this be ompt_task_initial? 1833 } 1834 1835 __ompt_lw_taskteam_unlink(master_th); 1836 if (ompt_enabled.ompt_callback_parallel_end) { 1837 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)( 1838 OMPT_CUR_TEAM_DATA(master_th), parent_task_data, 1839 OMPT_INVOKER(call_context), return_address); 1840 } 1841 master_th->th.ompt_thread_info.state = ompt_state_overhead; 1842 } 1843 #endif 1844 } else if (microtask == (microtask_t)__kmp_teams_master) { 1845 KMP_DEBUG_ASSERT(master_th->th.th_team == 1846 master_th->th.th_serial_team); 1847 team = master_th->th.th_team; 1848 // team->t.t_pkfn = microtask; 1849 team->t.t_invoke = invoker; 1850 __kmp_alloc_argv_entries(argc, team, TRUE); 1851 team->t.t_argc = argc; 1852 argv = (void **)team->t.t_argv; 1853 if (ap) { 1854 for (i = argc - 1; i >= 0; --i) 1855 // TODO: revert workaround for Intel(R) 64 tracker #96 1856 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 1857 *argv++ = va_arg(*ap, void *); 1858 #else 1859 *argv++ = va_arg(ap, void *); 1860 #endif 1861 } else { 1862 for (i = 0; i < argc; ++i) 1863 // Get args from parent team for teams construct 1864 argv[i] = parent_team->t.t_argv[i]; 1865 } 1866 // AC: revert change made in __kmpc_serialized_parallel() 1867 // because initial code in teams should have level=0 1868 team->t.t_level--; 1869 // AC: call special invoker for outer "parallel" of teams construct 1870 invoker(gtid); 1871 } else { 1872 #endif /* OMP_40_ENABLED */ 1873 argv = args; 1874 for (i = argc - 1; i >= 0; --i) 1875 // TODO: revert workaround for Intel(R) 64 tracker #96 1876 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 1877 *argv++ = va_arg(*ap, void *); 1878 #else 1879 *argv++ = va_arg(ap, void *); 1880 #endif 1881 KMP_MB(); 1882 1883 #if OMPT_SUPPORT 1884 void *dummy; 1885 void **exit_runtime_p; 1886 ompt_task_info_t *task_info; 1887 1888 ompt_lw_taskteam_t lw_taskteam; 1889 1890 if (ompt_enabled.enabled) { 1891 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid, 1892 &ompt_parallel_data, return_address); 1893 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0); 1894 // don't use lw_taskteam after linking. content was swaped 1895 task_info = OMPT_CUR_TASK_INFO(master_th); 1896 exit_runtime_p = &(task_info->frame.exit_frame.ptr); 1897 1898 /* OMPT implicit task begin */ 1899 implicit_task_data = OMPT_CUR_TASK_DATA(master_th); 1900 if (ompt_enabled.ompt_callback_implicit_task) { 1901 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1902 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th), 1903 implicit_task_data, 1, __kmp_tid_from_gtid(gtid), ompt_task_implicit); // TODO: Can this be ompt_task_initial? 1904 OMPT_CUR_TASK_INFO(master_th) 1905 ->thread_num = __kmp_tid_from_gtid(gtid); 1906 } 1907 1908 /* OMPT state */ 1909 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; 1910 } else { 1911 exit_runtime_p = &dummy; 1912 } 1913 #endif 1914 1915 { 1916 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 1917 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 1918 __kmp_invoke_microtask(microtask, gtid, 0, argc, args 1919 #if OMPT_SUPPORT 1920 , 1921 exit_runtime_p 1922 #endif 1923 ); 1924 } 1925 1926 #if OMPT_SUPPORT 1927 if (ompt_enabled.enabled) { 1928 *exit_runtime_p = NULL; 1929 if (ompt_enabled.ompt_callback_implicit_task) { 1930 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1931 ompt_scope_end, NULL, &(task_info->task_data), 1, 1932 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit); // TODO: Can this be ompt_task_initial? 1933 } 1934 1935 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th); 1936 __ompt_lw_taskteam_unlink(master_th); 1937 if (ompt_enabled.ompt_callback_parallel_end) { 1938 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)( 1939 &ompt_parallel_data, parent_task_data, 1940 OMPT_INVOKER(call_context), return_address); 1941 } 1942 master_th->th.ompt_thread_info.state = ompt_state_overhead; 1943 } 1944 #endif 1945 #if OMP_40_ENABLED 1946 } 1947 #endif /* OMP_40_ENABLED */ 1948 } else if (call_context == fork_context_gnu) { 1949 #if OMPT_SUPPORT 1950 ompt_lw_taskteam_t lwt; 1951 __ompt_lw_taskteam_init(&lwt, master_th, gtid, &ompt_parallel_data, 1952 return_address); 1953 1954 lwt.ompt_task_info.frame.exit_frame = ompt_data_none; 1955 __ompt_lw_taskteam_link(&lwt, master_th, 1); 1956 // don't use lw_taskteam after linking. content was swaped 1957 #endif 1958 1959 // we were called from GNU native code 1960 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid)); 1961 return FALSE; 1962 } else { 1963 KMP_ASSERT2(call_context < fork_context_last, 1964 "__kmp_fork_call: unknown fork_context parameter"); 1965 } 1966 1967 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid)); 1968 KMP_MB(); 1969 return FALSE; 1970 } // if (nthreads == 1) 1971 1972 // GEH: only modify the executing flag in the case when not serialized 1973 // serialized case is handled in kmpc_serialized_parallel 1974 KF_TRACE(10, ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, " 1975 "curtask=%p, curtask_max_aclevel=%d\n", 1976 parent_team->t.t_active_level, master_th, 1977 master_th->th.th_current_task, 1978 master_th->th.th_current_task->td_icvs.max_active_levels)); 1979 // TODO: GEH - cannot do this assertion because root thread not set up as 1980 // executing 1981 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 1 ); 1982 master_th->th.th_current_task->td_flags.executing = 0; 1983 1984 #if OMP_40_ENABLED 1985 if (!master_th->th.th_teams_microtask || level > teams_level) 1986 #endif /* OMP_40_ENABLED */ 1987 { 1988 /* Increment our nested depth level */ 1989 KMP_ATOMIC_INC(&root->r.r_in_parallel); 1990 } 1991 1992 // See if we need to make a copy of the ICVs. 1993 int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc; 1994 if ((level + 1 < __kmp_nested_nth.used) && 1995 (__kmp_nested_nth.nth[level + 1] != nthreads_icv)) { 1996 nthreads_icv = __kmp_nested_nth.nth[level + 1]; 1997 } else { 1998 nthreads_icv = 0; // don't update 1999 } 2000 2001 #if OMP_40_ENABLED 2002 // Figure out the proc_bind_policy for the new team. 2003 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind; 2004 kmp_proc_bind_t proc_bind_icv = 2005 proc_bind_default; // proc_bind_default means don't update 2006 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) { 2007 proc_bind = proc_bind_false; 2008 } else { 2009 if (proc_bind == proc_bind_default) { 2010 // No proc_bind clause specified; use current proc-bind-var for this 2011 // parallel region 2012 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind; 2013 } 2014 /* else: The proc_bind policy was specified explicitly on parallel clause. 2015 This overrides proc-bind-var for this parallel region, but does not 2016 change proc-bind-var. */ 2017 // Figure the value of proc-bind-var for the child threads. 2018 if ((level + 1 < __kmp_nested_proc_bind.used) && 2019 (__kmp_nested_proc_bind.bind_types[level + 1] != 2020 master_th->th.th_current_task->td_icvs.proc_bind)) { 2021 proc_bind_icv = __kmp_nested_proc_bind.bind_types[level + 1]; 2022 } 2023 } 2024 2025 // Reset for next parallel region 2026 master_th->th.th_set_proc_bind = proc_bind_default; 2027 #endif /* OMP_40_ENABLED */ 2028 2029 if ((nthreads_icv > 0) 2030 #if OMP_40_ENABLED 2031 || (proc_bind_icv != proc_bind_default) 2032 #endif /* OMP_40_ENABLED */ 2033 ) { 2034 kmp_internal_control_t new_icvs; 2035 copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs); 2036 new_icvs.next = NULL; 2037 if (nthreads_icv > 0) { 2038 new_icvs.nproc = nthreads_icv; 2039 } 2040 2041 #if OMP_40_ENABLED 2042 if (proc_bind_icv != proc_bind_default) { 2043 new_icvs.proc_bind = proc_bind_icv; 2044 } 2045 #endif /* OMP_40_ENABLED */ 2046 2047 /* allocate a new parallel team */ 2048 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n")); 2049 team = __kmp_allocate_team(root, nthreads, nthreads, 2050 #if OMPT_SUPPORT 2051 ompt_parallel_data, 2052 #endif 2053 #if OMP_40_ENABLED 2054 proc_bind, 2055 #endif 2056 &new_icvs, argc USE_NESTED_HOT_ARG(master_th)); 2057 } else { 2058 /* allocate a new parallel team */ 2059 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n")); 2060 team = __kmp_allocate_team(root, nthreads, nthreads, 2061 #if OMPT_SUPPORT 2062 ompt_parallel_data, 2063 #endif 2064 #if OMP_40_ENABLED 2065 proc_bind, 2066 #endif 2067 &master_th->th.th_current_task->td_icvs, 2068 argc USE_NESTED_HOT_ARG(master_th)); 2069 } 2070 KF_TRACE( 2071 10, ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team)); 2072 2073 /* setup the new team */ 2074 KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid); 2075 KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons); 2076 KMP_CHECK_UPDATE(team->t.t_ident, loc); 2077 KMP_CHECK_UPDATE(team->t.t_parent, parent_team); 2078 KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask); 2079 #if OMPT_SUPPORT 2080 KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.master_return_address, 2081 return_address); 2082 #endif 2083 KMP_CHECK_UPDATE(team->t.t_invoke, invoker); // TODO move to root, maybe 2084 // TODO: parent_team->t.t_level == INT_MAX ??? 2085 #if OMP_40_ENABLED 2086 if (!master_th->th.th_teams_microtask || level > teams_level) { 2087 #endif /* OMP_40_ENABLED */ 2088 int new_level = parent_team->t.t_level + 1; 2089 KMP_CHECK_UPDATE(team->t.t_level, new_level); 2090 new_level = parent_team->t.t_active_level + 1; 2091 KMP_CHECK_UPDATE(team->t.t_active_level, new_level); 2092 #if OMP_40_ENABLED 2093 } else { 2094 // AC: Do not increase parallel level at start of the teams construct 2095 int new_level = parent_team->t.t_level; 2096 KMP_CHECK_UPDATE(team->t.t_level, new_level); 2097 new_level = parent_team->t.t_active_level; 2098 KMP_CHECK_UPDATE(team->t.t_active_level, new_level); 2099 } 2100 #endif /* OMP_40_ENABLED */ 2101 kmp_r_sched_t new_sched = get__sched_2(parent_team, master_tid); 2102 // set master's schedule as new run-time schedule 2103 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched); 2104 2105 #if OMP_40_ENABLED 2106 KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq); 2107 #endif 2108 #if OMP_50_ENABLED 2109 KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator); 2110 #endif 2111 2112 // Update the floating point rounding in the team if required. 2113 propagateFPControl(team); 2114 2115 if (__kmp_tasking_mode != tskm_immediate_exec) { 2116 // Set master's task team to team's task team. Unless this is hot team, it 2117 // should be NULL. 2118 KMP_DEBUG_ASSERT(master_th->th.th_task_team == 2119 parent_team->t.t_task_team[master_th->th.th_task_state]); 2120 KA_TRACE(20, ("__kmp_fork_call: Master T#%d pushing task_team %p / team " 2121 "%p, new task_team %p / team %p\n", 2122 __kmp_gtid_from_thread(master_th), 2123 master_th->th.th_task_team, parent_team, 2124 team->t.t_task_team[master_th->th.th_task_state], team)); 2125 2126 if (active_level || master_th->th.th_task_team) { 2127 // Take a memo of master's task_state 2128 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack); 2129 if (master_th->th.th_task_state_top >= 2130 master_th->th.th_task_state_stack_sz) { // increase size 2131 kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz; 2132 kmp_uint8 *old_stack, *new_stack; 2133 kmp_uint32 i; 2134 new_stack = (kmp_uint8 *)__kmp_allocate(new_size); 2135 for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) { 2136 new_stack[i] = master_th->th.th_task_state_memo_stack[i]; 2137 } 2138 for (i = master_th->th.th_task_state_stack_sz; i < new_size; 2139 ++i) { // zero-init rest of stack 2140 new_stack[i] = 0; 2141 } 2142 old_stack = master_th->th.th_task_state_memo_stack; 2143 master_th->th.th_task_state_memo_stack = new_stack; 2144 master_th->th.th_task_state_stack_sz = new_size; 2145 __kmp_free(old_stack); 2146 } 2147 // Store master's task_state on stack 2148 master_th->th 2149 .th_task_state_memo_stack[master_th->th.th_task_state_top] = 2150 master_th->th.th_task_state; 2151 master_th->th.th_task_state_top++; 2152 #if KMP_NESTED_HOT_TEAMS 2153 if (master_th->th.th_hot_teams && 2154 active_level < __kmp_hot_teams_max_level && 2155 team == master_th->th.th_hot_teams[active_level].hot_team) { 2156 // Restore master's nested state if nested hot team 2157 master_th->th.th_task_state = 2158 master_th->th 2159 .th_task_state_memo_stack[master_th->th.th_task_state_top]; 2160 } else { 2161 #endif 2162 master_th->th.th_task_state = 0; 2163 #if KMP_NESTED_HOT_TEAMS 2164 } 2165 #endif 2166 } 2167 #if !KMP_NESTED_HOT_TEAMS 2168 KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) || 2169 (team == root->r.r_hot_team)); 2170 #endif 2171 } 2172 2173 KA_TRACE( 2174 20, 2175 ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n", 2176 gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id, 2177 team->t.t_nproc)); 2178 KMP_DEBUG_ASSERT(team != root->r.r_hot_team || 2179 (team->t.t_master_tid == 0 && 2180 (team->t.t_parent == root->r.r_root_team || 2181 team->t.t_parent->t.t_serialized))); 2182 KMP_MB(); 2183 2184 /* now, setup the arguments */ 2185 argv = (void **)team->t.t_argv; 2186 #if OMP_40_ENABLED 2187 if (ap) { 2188 #endif /* OMP_40_ENABLED */ 2189 for (i = argc - 1; i >= 0; --i) { 2190 // TODO: revert workaround for Intel(R) 64 tracker #96 2191 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 2192 void *new_argv = va_arg(*ap, void *); 2193 #else 2194 void *new_argv = va_arg(ap, void *); 2195 #endif 2196 KMP_CHECK_UPDATE(*argv, new_argv); 2197 argv++; 2198 } 2199 #if OMP_40_ENABLED 2200 } else { 2201 for (i = 0; i < argc; ++i) { 2202 // Get args from parent team for teams construct 2203 KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i]); 2204 } 2205 } 2206 #endif /* OMP_40_ENABLED */ 2207 2208 /* now actually fork the threads */ 2209 KMP_CHECK_UPDATE(team->t.t_master_active, master_active); 2210 if (!root->r.r_active) // Only do assignment if it prevents cache ping-pong 2211 root->r.r_active = TRUE; 2212 2213 __kmp_fork_team_threads(root, team, master_th, gtid); 2214 __kmp_setup_icv_copy(team, nthreads, 2215 &master_th->th.th_current_task->td_icvs, loc); 2216 2217 #if OMPT_SUPPORT 2218 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; 2219 #endif 2220 2221 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 2222 2223 #if USE_ITT_BUILD 2224 if (team->t.t_active_level == 1 // only report frames at level 1 2225 #if OMP_40_ENABLED 2226 && !master_th->th.th_teams_microtask // not in teams construct 2227 #endif /* OMP_40_ENABLED */ 2228 ) { 2229 #if USE_ITT_NOTIFY 2230 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) && 2231 (__kmp_forkjoin_frames_mode == 3 || 2232 __kmp_forkjoin_frames_mode == 1)) { 2233 kmp_uint64 tmp_time = 0; 2234 if (__itt_get_timestamp_ptr) 2235 tmp_time = __itt_get_timestamp(); 2236 // Internal fork - report frame begin 2237 master_th->th.th_frame_time = tmp_time; 2238 if (__kmp_forkjoin_frames_mode == 3) 2239 team->t.t_region_time = tmp_time; 2240 } else 2241 // only one notification scheme (either "submit" or "forking/joined", not both) 2242 #endif /* USE_ITT_NOTIFY */ 2243 if ((__itt_frame_begin_v3_ptr || KMP_ITT_DEBUG) && 2244 __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode) { 2245 // Mark start of "parallel" region for Intel(R) VTune(TM) analyzer. 2246 __kmp_itt_region_forking(gtid, team->t.t_nproc, 0); 2247 } 2248 } 2249 #endif /* USE_ITT_BUILD */ 2250 2251 /* now go on and do the work */ 2252 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team); 2253 KMP_MB(); 2254 KF_TRACE(10, 2255 ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n", 2256 root, team, master_th, gtid)); 2257 2258 #if USE_ITT_BUILD 2259 if (__itt_stack_caller_create_ptr) { 2260 team->t.t_stack_id = 2261 __kmp_itt_stack_caller_create(); // create new stack stitching id 2262 // before entering fork barrier 2263 } 2264 #endif /* USE_ITT_BUILD */ 2265 2266 #if OMP_40_ENABLED 2267 // AC: skip __kmp_internal_fork at teams construct, let only master 2268 // threads execute 2269 if (ap) 2270 #endif /* OMP_40_ENABLED */ 2271 { 2272 __kmp_internal_fork(loc, gtid, team); 2273 KF_TRACE(10, ("__kmp_internal_fork : after : root=%p, team=%p, " 2274 "master_th=%p, gtid=%d\n", 2275 root, team, master_th, gtid)); 2276 } 2277 2278 if (call_context == fork_context_gnu) { 2279 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid)); 2280 return TRUE; 2281 } 2282 2283 /* Invoke microtask for MASTER thread */ 2284 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid, 2285 team->t.t_id, team->t.t_pkfn)); 2286 } // END of timer KMP_fork_call block 2287 2288 if (!team->t.t_invoke(gtid)) { 2289 KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread"); 2290 } 2291 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid, 2292 team->t.t_id, team->t.t_pkfn)); 2293 KMP_MB(); /* Flush all pending memory write invalidates. */ 2294 2295 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid)); 2296 2297 #if OMPT_SUPPORT 2298 if (ompt_enabled.enabled) { 2299 master_th->th.ompt_thread_info.state = ompt_state_overhead; 2300 } 2301 #endif 2302 2303 return TRUE; 2304 } 2305 2306 #if OMPT_SUPPORT 2307 static inline void __kmp_join_restore_state(kmp_info_t *thread, 2308 kmp_team_t *team) { 2309 // restore state outside the region 2310 thread->th.ompt_thread_info.state = 2311 ((team->t.t_serialized) ? ompt_state_work_serial 2312 : ompt_state_work_parallel); 2313 } 2314 2315 static inline void __kmp_join_ompt(int gtid, kmp_info_t *thread, 2316 kmp_team_t *team, ompt_data_t *parallel_data, 2317 fork_context_e fork_context, void *codeptr) { 2318 ompt_task_info_t *task_info = __ompt_get_task_info_object(0); 2319 if (ompt_enabled.ompt_callback_parallel_end) { 2320 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)( 2321 parallel_data, &(task_info->task_data), OMPT_INVOKER(fork_context), 2322 codeptr); 2323 } 2324 2325 task_info->frame.enter_frame = ompt_data_none; 2326 __kmp_join_restore_state(thread, team); 2327 } 2328 #endif 2329 2330 void __kmp_join_call(ident_t *loc, int gtid 2331 #if OMPT_SUPPORT 2332 , 2333 enum fork_context_e fork_context 2334 #endif 2335 #if OMP_40_ENABLED 2336 , 2337 int exit_teams 2338 #endif /* OMP_40_ENABLED */ 2339 ) { 2340 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_join_call); 2341 kmp_team_t *team; 2342 kmp_team_t *parent_team; 2343 kmp_info_t *master_th; 2344 kmp_root_t *root; 2345 int master_active; 2346 2347 KA_TRACE(20, ("__kmp_join_call: enter T#%d\n", gtid)); 2348 2349 /* setup current data */ 2350 master_th = __kmp_threads[gtid]; 2351 root = master_th->th.th_root; 2352 team = master_th->th.th_team; 2353 parent_team = team->t.t_parent; 2354 2355 master_th->th.th_ident = loc; 2356 2357 #if OMPT_SUPPORT 2358 if (ompt_enabled.enabled) { 2359 master_th->th.ompt_thread_info.state = ompt_state_overhead; 2360 } 2361 #endif 2362 2363 #if KMP_DEBUG 2364 if (__kmp_tasking_mode != tskm_immediate_exec && !exit_teams) { 2365 KA_TRACE(20, ("__kmp_join_call: T#%d, old team = %p old task_team = %p, " 2366 "th_task_team = %p\n", 2367 __kmp_gtid_from_thread(master_th), team, 2368 team->t.t_task_team[master_th->th.th_task_state], 2369 master_th->th.th_task_team)); 2370 KMP_DEBUG_ASSERT(master_th->th.th_task_team == 2371 team->t.t_task_team[master_th->th.th_task_state]); 2372 } 2373 #endif 2374 2375 if (team->t.t_serialized) { 2376 #if OMP_40_ENABLED 2377 if (master_th->th.th_teams_microtask) { 2378 // We are in teams construct 2379 int level = team->t.t_level; 2380 int tlevel = master_th->th.th_teams_level; 2381 if (level == tlevel) { 2382 // AC: we haven't incremented it earlier at start of teams construct, 2383 // so do it here - at the end of teams construct 2384 team->t.t_level++; 2385 } else if (level == tlevel + 1) { 2386 // AC: we are exiting parallel inside teams, need to increment 2387 // serialization in order to restore it in the next call to 2388 // __kmpc_end_serialized_parallel 2389 team->t.t_serialized++; 2390 } 2391 } 2392 #endif /* OMP_40_ENABLED */ 2393 __kmpc_end_serialized_parallel(loc, gtid); 2394 2395 #if OMPT_SUPPORT 2396 if (ompt_enabled.enabled) { 2397 __kmp_join_restore_state(master_th, parent_team); 2398 } 2399 #endif 2400 2401 return; 2402 } 2403 2404 master_active = team->t.t_master_active; 2405 2406 #if OMP_40_ENABLED 2407 if (!exit_teams) 2408 #endif /* OMP_40_ENABLED */ 2409 { 2410 // AC: No barrier for internal teams at exit from teams construct. 2411 // But there is barrier for external team (league). 2412 __kmp_internal_join(loc, gtid, team); 2413 } 2414 #if OMP_40_ENABLED 2415 else { 2416 master_th->th.th_task_state = 2417 0; // AC: no tasking in teams (out of any parallel) 2418 } 2419 #endif /* OMP_40_ENABLED */ 2420 2421 KMP_MB(); 2422 2423 #if OMPT_SUPPORT 2424 ompt_data_t *parallel_data = &(team->t.ompt_team_info.parallel_data); 2425 void *codeptr = team->t.ompt_team_info.master_return_address; 2426 #endif 2427 2428 #if USE_ITT_BUILD 2429 if (__itt_stack_caller_create_ptr) { 2430 __kmp_itt_stack_caller_destroy( 2431 (__itt_caller)team->t 2432 .t_stack_id); // destroy the stack stitching id after join barrier 2433 } 2434 2435 // Mark end of "parallel" region for Intel(R) VTune(TM) analyzer. 2436 if (team->t.t_active_level == 1 2437 #if OMP_40_ENABLED 2438 && !master_th->th.th_teams_microtask /* not in teams construct */ 2439 #endif /* OMP_40_ENABLED */ 2440 ) { 2441 master_th->th.th_ident = loc; 2442 // only one notification scheme (either "submit" or "forking/joined", not 2443 // both) 2444 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) && 2445 __kmp_forkjoin_frames_mode == 3) 2446 __kmp_itt_frame_submit(gtid, team->t.t_region_time, 2447 master_th->th.th_frame_time, 0, loc, 2448 master_th->th.th_team_nproc, 1); 2449 else if ((__itt_frame_end_v3_ptr || KMP_ITT_DEBUG) && 2450 !__kmp_forkjoin_frames_mode && __kmp_forkjoin_frames) 2451 __kmp_itt_region_joined(gtid); 2452 } // active_level == 1 2453 #endif /* USE_ITT_BUILD */ 2454 2455 #if OMP_40_ENABLED 2456 if (master_th->th.th_teams_microtask && !exit_teams && 2457 team->t.t_pkfn != (microtask_t)__kmp_teams_master && 2458 team->t.t_level == master_th->th.th_teams_level + 1) { 2459 // AC: We need to leave the team structure intact at the end of parallel 2460 // inside the teams construct, so that at the next parallel same (hot) team 2461 // works, only adjust nesting levels 2462 2463 /* Decrement our nested depth level */ 2464 team->t.t_level--; 2465 team->t.t_active_level--; 2466 KMP_ATOMIC_DEC(&root->r.r_in_parallel); 2467 2468 // Restore number of threads in the team if needed. This code relies on 2469 // the proper adjustment of th_teams_size.nth after the fork in 2470 // __kmp_teams_master on each teams master in the case that 2471 // __kmp_reserve_threads reduced it. 2472 if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) { 2473 int old_num = master_th->th.th_team_nproc; 2474 int new_num = master_th->th.th_teams_size.nth; 2475 kmp_info_t **other_threads = team->t.t_threads; 2476 team->t.t_nproc = new_num; 2477 for (int i = 0; i < old_num; ++i) { 2478 other_threads[i]->th.th_team_nproc = new_num; 2479 } 2480 // Adjust states of non-used threads of the team 2481 for (int i = old_num; i < new_num; ++i) { 2482 // Re-initialize thread's barrier data. 2483 KMP_DEBUG_ASSERT(other_threads[i]); 2484 kmp_balign_t *balign = other_threads[i]->th.th_bar; 2485 for (int b = 0; b < bs_last_barrier; ++b) { 2486 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 2487 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 2488 #if USE_DEBUGGER 2489 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 2490 #endif 2491 } 2492 if (__kmp_tasking_mode != tskm_immediate_exec) { 2493 // Synchronize thread's task state 2494 other_threads[i]->th.th_task_state = master_th->th.th_task_state; 2495 } 2496 } 2497 } 2498 2499 #if OMPT_SUPPORT 2500 if (ompt_enabled.enabled) { 2501 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, fork_context, 2502 codeptr); 2503 } 2504 #endif 2505 2506 return; 2507 } 2508 #endif /* OMP_40_ENABLED */ 2509 2510 /* do cleanup and restore the parent team */ 2511 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid; 2512 master_th->th.th_local.this_construct = team->t.t_master_this_cons; 2513 2514 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid]; 2515 2516 /* jc: The following lock has instructions with REL and ACQ semantics, 2517 separating the parallel user code called in this parallel region 2518 from the serial user code called after this function returns. */ 2519 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 2520 2521 #if OMP_40_ENABLED 2522 if (!master_th->th.th_teams_microtask || 2523 team->t.t_level > master_th->th.th_teams_level) 2524 #endif /* OMP_40_ENABLED */ 2525 { 2526 /* Decrement our nested depth level */ 2527 KMP_ATOMIC_DEC(&root->r.r_in_parallel); 2528 } 2529 KMP_DEBUG_ASSERT(root->r.r_in_parallel >= 0); 2530 2531 #if OMPT_SUPPORT 2532 if (ompt_enabled.enabled) { 2533 ompt_task_info_t *task_info = __ompt_get_task_info_object(0); 2534 if (ompt_enabled.ompt_callback_implicit_task) { 2535 int ompt_team_size = team->t.t_nproc; 2536 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 2537 ompt_scope_end, NULL, &(task_info->task_data), ompt_team_size, 2538 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit); // TODO: Can this be ompt_task_initial? 2539 } 2540 2541 task_info->frame.exit_frame = ompt_data_none; 2542 task_info->task_data = ompt_data_none; 2543 } 2544 #endif 2545 2546 KF_TRACE(10, ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n", 0, 2547 master_th, team)); 2548 __kmp_pop_current_task_from_thread(master_th); 2549 2550 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 2551 // Restore master thread's partition. 2552 master_th->th.th_first_place = team->t.t_first_place; 2553 master_th->th.th_last_place = team->t.t_last_place; 2554 #endif /* OMP_40_ENABLED */ 2555 #if OMP_50_ENABLED 2556 master_th->th.th_def_allocator = team->t.t_def_allocator; 2557 #endif 2558 2559 updateHWFPControl(team); 2560 2561 if (root->r.r_active != master_active) 2562 root->r.r_active = master_active; 2563 2564 __kmp_free_team(root, team USE_NESTED_HOT_ARG( 2565 master_th)); // this will free worker threads 2566 2567 /* this race was fun to find. make sure the following is in the critical 2568 region otherwise assertions may fail occasionally since the old team may be 2569 reallocated and the hierarchy appears inconsistent. it is actually safe to 2570 run and won't cause any bugs, but will cause those assertion failures. it's 2571 only one deref&assign so might as well put this in the critical region */ 2572 master_th->th.th_team = parent_team; 2573 master_th->th.th_team_nproc = parent_team->t.t_nproc; 2574 master_th->th.th_team_master = parent_team->t.t_threads[0]; 2575 master_th->th.th_team_serialized = parent_team->t.t_serialized; 2576 2577 /* restore serialized team, if need be */ 2578 if (parent_team->t.t_serialized && 2579 parent_team != master_th->th.th_serial_team && 2580 parent_team != root->r.r_root_team) { 2581 __kmp_free_team(root, 2582 master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL)); 2583 master_th->th.th_serial_team = parent_team; 2584 } 2585 2586 if (__kmp_tasking_mode != tskm_immediate_exec) { 2587 if (master_th->th.th_task_state_top > 2588 0) { // Restore task state from memo stack 2589 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack); 2590 // Remember master's state if we re-use this nested hot team 2591 master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] = 2592 master_th->th.th_task_state; 2593 --master_th->th.th_task_state_top; // pop 2594 // Now restore state at this level 2595 master_th->th.th_task_state = 2596 master_th->th 2597 .th_task_state_memo_stack[master_th->th.th_task_state_top]; 2598 } 2599 // Copy the task team from the parent team to the master thread 2600 master_th->th.th_task_team = 2601 parent_team->t.t_task_team[master_th->th.th_task_state]; 2602 KA_TRACE(20, 2603 ("__kmp_join_call: Master T#%d restoring task_team %p / team %p\n", 2604 __kmp_gtid_from_thread(master_th), master_th->th.th_task_team, 2605 parent_team)); 2606 } 2607 2608 // TODO: GEH - cannot do this assertion because root thread not set up as 2609 // executing 2610 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 0 ); 2611 master_th->th.th_current_task->td_flags.executing = 1; 2612 2613 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 2614 2615 #if OMPT_SUPPORT 2616 if (ompt_enabled.enabled) { 2617 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, fork_context, 2618 codeptr); 2619 } 2620 #endif 2621 2622 KMP_MB(); 2623 KA_TRACE(20, ("__kmp_join_call: exit T#%d\n", gtid)); 2624 } 2625 2626 /* Check whether we should push an internal control record onto the 2627 serial team stack. If so, do it. */ 2628 void __kmp_save_internal_controls(kmp_info_t *thread) { 2629 2630 if (thread->th.th_team != thread->th.th_serial_team) { 2631 return; 2632 } 2633 if (thread->th.th_team->t.t_serialized > 1) { 2634 int push = 0; 2635 2636 if (thread->th.th_team->t.t_control_stack_top == NULL) { 2637 push = 1; 2638 } else { 2639 if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level != 2640 thread->th.th_team->t.t_serialized) { 2641 push = 1; 2642 } 2643 } 2644 if (push) { /* push a record on the serial team's stack */ 2645 kmp_internal_control_t *control = 2646 (kmp_internal_control_t *)__kmp_allocate( 2647 sizeof(kmp_internal_control_t)); 2648 2649 copy_icvs(control, &thread->th.th_current_task->td_icvs); 2650 2651 control->serial_nesting_level = thread->th.th_team->t.t_serialized; 2652 2653 control->next = thread->th.th_team->t.t_control_stack_top; 2654 thread->th.th_team->t.t_control_stack_top = control; 2655 } 2656 } 2657 } 2658 2659 /* Changes set_nproc */ 2660 void __kmp_set_num_threads(int new_nth, int gtid) { 2661 kmp_info_t *thread; 2662 kmp_root_t *root; 2663 2664 KF_TRACE(10, ("__kmp_set_num_threads: new __kmp_nth = %d\n", new_nth)); 2665 KMP_DEBUG_ASSERT(__kmp_init_serial); 2666 2667 if (new_nth < 1) 2668 new_nth = 1; 2669 else if (new_nth > __kmp_max_nth) 2670 new_nth = __kmp_max_nth; 2671 2672 KMP_COUNT_VALUE(OMP_set_numthreads, new_nth); 2673 thread = __kmp_threads[gtid]; 2674 if (thread->th.th_current_task->td_icvs.nproc == new_nth) 2675 return; // nothing to do 2676 2677 __kmp_save_internal_controls(thread); 2678 2679 set__nproc(thread, new_nth); 2680 2681 // If this omp_set_num_threads() call will cause the hot team size to be 2682 // reduced (in the absence of a num_threads clause), then reduce it now, 2683 // rather than waiting for the next parallel region. 2684 root = thread->th.th_root; 2685 if (__kmp_init_parallel && (!root->r.r_active) && 2686 (root->r.r_hot_team->t.t_nproc > new_nth) 2687 #if KMP_NESTED_HOT_TEAMS 2688 && __kmp_hot_teams_max_level && !__kmp_hot_teams_mode 2689 #endif 2690 ) { 2691 kmp_team_t *hot_team = root->r.r_hot_team; 2692 int f; 2693 2694 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 2695 2696 // Release the extra threads we don't need any more. 2697 for (f = new_nth; f < hot_team->t.t_nproc; f++) { 2698 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL); 2699 if (__kmp_tasking_mode != tskm_immediate_exec) { 2700 // When decreasing team size, threads no longer in the team should unref 2701 // task team. 2702 hot_team->t.t_threads[f]->th.th_task_team = NULL; 2703 } 2704 __kmp_free_thread(hot_team->t.t_threads[f]); 2705 hot_team->t.t_threads[f] = NULL; 2706 } 2707 hot_team->t.t_nproc = new_nth; 2708 #if KMP_NESTED_HOT_TEAMS 2709 if (thread->th.th_hot_teams) { 2710 KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team); 2711 thread->th.th_hot_teams[0].hot_team_nth = new_nth; 2712 } 2713 #endif 2714 2715 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 2716 2717 // Update the t_nproc field in the threads that are still active. 2718 for (f = 0; f < new_nth; f++) { 2719 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL); 2720 hot_team->t.t_threads[f]->th.th_team_nproc = new_nth; 2721 } 2722 // Special flag in case omp_set_num_threads() call 2723 hot_team->t.t_size_changed = -1; 2724 } 2725 } 2726 2727 /* Changes max_active_levels */ 2728 void __kmp_set_max_active_levels(int gtid, int max_active_levels) { 2729 kmp_info_t *thread; 2730 2731 KF_TRACE(10, ("__kmp_set_max_active_levels: new max_active_levels for thread " 2732 "%d = (%d)\n", 2733 gtid, max_active_levels)); 2734 KMP_DEBUG_ASSERT(__kmp_init_serial); 2735 2736 // validate max_active_levels 2737 if (max_active_levels < 0) { 2738 KMP_WARNING(ActiveLevelsNegative, max_active_levels); 2739 // We ignore this call if the user has specified a negative value. 2740 // The current setting won't be changed. The last valid setting will be 2741 // used. A warning will be issued (if warnings are allowed as controlled by 2742 // the KMP_WARNINGS env var). 2743 KF_TRACE(10, ("__kmp_set_max_active_levels: the call is ignored: new " 2744 "max_active_levels for thread %d = (%d)\n", 2745 gtid, max_active_levels)); 2746 return; 2747 } 2748 if (max_active_levels <= KMP_MAX_ACTIVE_LEVELS_LIMIT) { 2749 // it's OK, the max_active_levels is within the valid range: [ 0; 2750 // KMP_MAX_ACTIVE_LEVELS_LIMIT ] 2751 // We allow a zero value. (implementation defined behavior) 2752 } else { 2753 KMP_WARNING(ActiveLevelsExceedLimit, max_active_levels, 2754 KMP_MAX_ACTIVE_LEVELS_LIMIT); 2755 max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT; 2756 // Current upper limit is MAX_INT. (implementation defined behavior) 2757 // If the input exceeds the upper limit, we correct the input to be the 2758 // upper limit. (implementation defined behavior) 2759 // Actually, the flow should never get here until we use MAX_INT limit. 2760 } 2761 KF_TRACE(10, ("__kmp_set_max_active_levels: after validation: new " 2762 "max_active_levels for thread %d = (%d)\n", 2763 gtid, max_active_levels)); 2764 2765 thread = __kmp_threads[gtid]; 2766 2767 __kmp_save_internal_controls(thread); 2768 2769 set__max_active_levels(thread, max_active_levels); 2770 } 2771 2772 /* Gets max_active_levels */ 2773 int __kmp_get_max_active_levels(int gtid) { 2774 kmp_info_t *thread; 2775 2776 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d\n", gtid)); 2777 KMP_DEBUG_ASSERT(__kmp_init_serial); 2778 2779 thread = __kmp_threads[gtid]; 2780 KMP_DEBUG_ASSERT(thread->th.th_current_task); 2781 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d, curtask=%p, " 2782 "curtask_maxaclevel=%d\n", 2783 gtid, thread->th.th_current_task, 2784 thread->th.th_current_task->td_icvs.max_active_levels)); 2785 return thread->th.th_current_task->td_icvs.max_active_levels; 2786 } 2787 2788 /* Changes def_sched_var ICV values (run-time schedule kind and chunk) */ 2789 void __kmp_set_schedule(int gtid, kmp_sched_t kind, int chunk) { 2790 kmp_info_t *thread; 2791 // kmp_team_t *team; 2792 2793 KF_TRACE(10, ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n", 2794 gtid, (int)kind, chunk)); 2795 KMP_DEBUG_ASSERT(__kmp_init_serial); 2796 2797 // Check if the kind parameter is valid, correct if needed. 2798 // Valid parameters should fit in one of two intervals - standard or extended: 2799 // <lower>, <valid>, <upper_std>, <lower_ext>, <valid>, <upper> 2800 // 2008-01-25: 0, 1 - 4, 5, 100, 101 - 102, 103 2801 if (kind <= kmp_sched_lower || kind >= kmp_sched_upper || 2802 (kind <= kmp_sched_lower_ext && kind >= kmp_sched_upper_std)) { 2803 // TODO: Hint needs attention in case we change the default schedule. 2804 __kmp_msg(kmp_ms_warning, KMP_MSG(ScheduleKindOutOfRange, kind), 2805 KMP_HNT(DefaultScheduleKindUsed, "static, no chunk"), 2806 __kmp_msg_null); 2807 kind = kmp_sched_default; 2808 chunk = 0; // ignore chunk value in case of bad kind 2809 } 2810 2811 thread = __kmp_threads[gtid]; 2812 2813 __kmp_save_internal_controls(thread); 2814 2815 if (kind < kmp_sched_upper_std) { 2816 if (kind == kmp_sched_static && chunk < KMP_DEFAULT_CHUNK) { 2817 // differ static chunked vs. unchunked: chunk should be invalid to 2818 // indicate unchunked schedule (which is the default) 2819 thread->th.th_current_task->td_icvs.sched.r_sched_type = kmp_sch_static; 2820 } else { 2821 thread->th.th_current_task->td_icvs.sched.r_sched_type = 2822 __kmp_sch_map[kind - kmp_sched_lower - 1]; 2823 } 2824 } else { 2825 // __kmp_sch_map[ kind - kmp_sched_lower_ext + kmp_sched_upper_std - 2826 // kmp_sched_lower - 2 ]; 2827 thread->th.th_current_task->td_icvs.sched.r_sched_type = 2828 __kmp_sch_map[kind - kmp_sched_lower_ext + kmp_sched_upper_std - 2829 kmp_sched_lower - 2]; 2830 } 2831 if (kind == kmp_sched_auto || chunk < 1) { 2832 // ignore parameter chunk for schedule auto 2833 thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK; 2834 } else { 2835 thread->th.th_current_task->td_icvs.sched.chunk = chunk; 2836 } 2837 } 2838 2839 /* Gets def_sched_var ICV values */ 2840 void __kmp_get_schedule(int gtid, kmp_sched_t *kind, int *chunk) { 2841 kmp_info_t *thread; 2842 enum sched_type th_type; 2843 2844 KF_TRACE(10, ("__kmp_get_schedule: thread %d\n", gtid)); 2845 KMP_DEBUG_ASSERT(__kmp_init_serial); 2846 2847 thread = __kmp_threads[gtid]; 2848 2849 th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type; 2850 2851 switch (th_type) { 2852 case kmp_sch_static: 2853 case kmp_sch_static_greedy: 2854 case kmp_sch_static_balanced: 2855 *kind = kmp_sched_static; 2856 *chunk = 0; // chunk was not set, try to show this fact via zero value 2857 return; 2858 case kmp_sch_static_chunked: 2859 *kind = kmp_sched_static; 2860 break; 2861 case kmp_sch_dynamic_chunked: 2862 *kind = kmp_sched_dynamic; 2863 break; 2864 case kmp_sch_guided_chunked: 2865 case kmp_sch_guided_iterative_chunked: 2866 case kmp_sch_guided_analytical_chunked: 2867 *kind = kmp_sched_guided; 2868 break; 2869 case kmp_sch_auto: 2870 *kind = kmp_sched_auto; 2871 break; 2872 case kmp_sch_trapezoidal: 2873 *kind = kmp_sched_trapezoidal; 2874 break; 2875 #if KMP_STATIC_STEAL_ENABLED 2876 case kmp_sch_static_steal: 2877 *kind = kmp_sched_static_steal; 2878 break; 2879 #endif 2880 default: 2881 KMP_FATAL(UnknownSchedulingType, th_type); 2882 } 2883 2884 *chunk = thread->th.th_current_task->td_icvs.sched.chunk; 2885 } 2886 2887 int __kmp_get_ancestor_thread_num(int gtid, int level) { 2888 2889 int ii, dd; 2890 kmp_team_t *team; 2891 kmp_info_t *thr; 2892 2893 KF_TRACE(10, ("__kmp_get_ancestor_thread_num: thread %d %d\n", gtid, level)); 2894 KMP_DEBUG_ASSERT(__kmp_init_serial); 2895 2896 // validate level 2897 if (level == 0) 2898 return 0; 2899 if (level < 0) 2900 return -1; 2901 thr = __kmp_threads[gtid]; 2902 team = thr->th.th_team; 2903 ii = team->t.t_level; 2904 if (level > ii) 2905 return -1; 2906 2907 #if OMP_40_ENABLED 2908 if (thr->th.th_teams_microtask) { 2909 // AC: we are in teams region where multiple nested teams have same level 2910 int tlevel = thr->th.th_teams_level; // the level of the teams construct 2911 if (level <= 2912 tlevel) { // otherwise usual algorithm works (will not touch the teams) 2913 KMP_DEBUG_ASSERT(ii >= tlevel); 2914 // AC: As we need to pass by the teams league, we need to artificially 2915 // increase ii 2916 if (ii == tlevel) { 2917 ii += 2; // three teams have same level 2918 } else { 2919 ii++; // two teams have same level 2920 } 2921 } 2922 } 2923 #endif 2924 2925 if (ii == level) 2926 return __kmp_tid_from_gtid(gtid); 2927 2928 dd = team->t.t_serialized; 2929 level++; 2930 while (ii > level) { 2931 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) { 2932 } 2933 if ((team->t.t_serialized) && (!dd)) { 2934 team = team->t.t_parent; 2935 continue; 2936 } 2937 if (ii > level) { 2938 team = team->t.t_parent; 2939 dd = team->t.t_serialized; 2940 ii--; 2941 } 2942 } 2943 2944 return (dd > 1) ? (0) : (team->t.t_master_tid); 2945 } 2946 2947 int __kmp_get_team_size(int gtid, int level) { 2948 2949 int ii, dd; 2950 kmp_team_t *team; 2951 kmp_info_t *thr; 2952 2953 KF_TRACE(10, ("__kmp_get_team_size: thread %d %d\n", gtid, level)); 2954 KMP_DEBUG_ASSERT(__kmp_init_serial); 2955 2956 // validate level 2957 if (level == 0) 2958 return 1; 2959 if (level < 0) 2960 return -1; 2961 thr = __kmp_threads[gtid]; 2962 team = thr->th.th_team; 2963 ii = team->t.t_level; 2964 if (level > ii) 2965 return -1; 2966 2967 #if OMP_40_ENABLED 2968 if (thr->th.th_teams_microtask) { 2969 // AC: we are in teams region where multiple nested teams have same level 2970 int tlevel = thr->th.th_teams_level; // the level of the teams construct 2971 if (level <= 2972 tlevel) { // otherwise usual algorithm works (will not touch the teams) 2973 KMP_DEBUG_ASSERT(ii >= tlevel); 2974 // AC: As we need to pass by the teams league, we need to artificially 2975 // increase ii 2976 if (ii == tlevel) { 2977 ii += 2; // three teams have same level 2978 } else { 2979 ii++; // two teams have same level 2980 } 2981 } 2982 } 2983 #endif 2984 2985 while (ii > level) { 2986 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) { 2987 } 2988 if (team->t.t_serialized && (!dd)) { 2989 team = team->t.t_parent; 2990 continue; 2991 } 2992 if (ii > level) { 2993 team = team->t.t_parent; 2994 ii--; 2995 } 2996 } 2997 2998 return team->t.t_nproc; 2999 } 3000 3001 kmp_r_sched_t __kmp_get_schedule_global() { 3002 // This routine created because pairs (__kmp_sched, __kmp_chunk) and 3003 // (__kmp_static, __kmp_guided) may be changed by kmp_set_defaults 3004 // independently. So one can get the updated schedule here. 3005 3006 kmp_r_sched_t r_sched; 3007 3008 // create schedule from 4 globals: __kmp_sched, __kmp_chunk, __kmp_static, 3009 // __kmp_guided. __kmp_sched should keep original value, so that user can set 3010 // KMP_SCHEDULE multiple times, and thus have different run-time schedules in 3011 // different roots (even in OMP 2.5) 3012 if (__kmp_sched == kmp_sch_static) { 3013 // replace STATIC with more detailed schedule (balanced or greedy) 3014 r_sched.r_sched_type = __kmp_static; 3015 } else if (__kmp_sched == kmp_sch_guided_chunked) { 3016 // replace GUIDED with more detailed schedule (iterative or analytical) 3017 r_sched.r_sched_type = __kmp_guided; 3018 } else { // (STATIC_CHUNKED), or (DYNAMIC_CHUNKED), or other 3019 r_sched.r_sched_type = __kmp_sched; 3020 } 3021 3022 if (__kmp_chunk < KMP_DEFAULT_CHUNK) { 3023 // __kmp_chunk may be wrong here (if it was not ever set) 3024 r_sched.chunk = KMP_DEFAULT_CHUNK; 3025 } else { 3026 r_sched.chunk = __kmp_chunk; 3027 } 3028 3029 return r_sched; 3030 } 3031 3032 /* Allocate (realloc == FALSE) * or reallocate (realloc == TRUE) 3033 at least argc number of *t_argv entries for the requested team. */ 3034 static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team, int realloc) { 3035 3036 KMP_DEBUG_ASSERT(team); 3037 if (!realloc || argc > team->t.t_max_argc) { 3038 3039 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: needed entries=%d, " 3040 "current entries=%d\n", 3041 team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0)); 3042 /* if previously allocated heap space for args, free them */ 3043 if (realloc && team->t.t_argv != &team->t.t_inline_argv[0]) 3044 __kmp_free((void *)team->t.t_argv); 3045 3046 if (argc <= KMP_INLINE_ARGV_ENTRIES) { 3047 /* use unused space in the cache line for arguments */ 3048 team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES; 3049 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: inline allocate %d " 3050 "argv entries\n", 3051 team->t.t_id, team->t.t_max_argc)); 3052 team->t.t_argv = &team->t.t_inline_argv[0]; 3053 if (__kmp_storage_map) { 3054 __kmp_print_storage_map_gtid( 3055 -1, &team->t.t_inline_argv[0], 3056 &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES], 3057 (sizeof(void *) * KMP_INLINE_ARGV_ENTRIES), "team_%d.t_inline_argv", 3058 team->t.t_id); 3059 } 3060 } else { 3061 /* allocate space for arguments in the heap */ 3062 team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES >> 1)) 3063 ? KMP_MIN_MALLOC_ARGV_ENTRIES 3064 : 2 * argc; 3065 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d " 3066 "argv entries\n", 3067 team->t.t_id, team->t.t_max_argc)); 3068 team->t.t_argv = 3069 (void **)__kmp_page_allocate(sizeof(void *) * team->t.t_max_argc); 3070 if (__kmp_storage_map) { 3071 __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0], 3072 &team->t.t_argv[team->t.t_max_argc], 3073 sizeof(void *) * team->t.t_max_argc, 3074 "team_%d.t_argv", team->t.t_id); 3075 } 3076 } 3077 } 3078 } 3079 3080 static void __kmp_allocate_team_arrays(kmp_team_t *team, int max_nth) { 3081 int i; 3082 int num_disp_buff = max_nth > 1 ? __kmp_dispatch_num_buffers : 2; 3083 team->t.t_threads = 3084 (kmp_info_t **)__kmp_allocate(sizeof(kmp_info_t *) * max_nth); 3085 team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate( 3086 sizeof(dispatch_shared_info_t) * num_disp_buff); 3087 team->t.t_dispatch = 3088 (kmp_disp_t *)__kmp_allocate(sizeof(kmp_disp_t) * max_nth); 3089 team->t.t_implicit_task_taskdata = 3090 (kmp_taskdata_t *)__kmp_allocate(sizeof(kmp_taskdata_t) * max_nth); 3091 team->t.t_max_nproc = max_nth; 3092 3093 /* setup dispatch buffers */ 3094 for (i = 0; i < num_disp_buff; ++i) { 3095 team->t.t_disp_buffer[i].buffer_index = i; 3096 #if OMP_45_ENABLED 3097 team->t.t_disp_buffer[i].doacross_buf_idx = i; 3098 #endif 3099 } 3100 } 3101 3102 static void __kmp_free_team_arrays(kmp_team_t *team) { 3103 /* Note: this does not free the threads in t_threads (__kmp_free_threads) */ 3104 int i; 3105 for (i = 0; i < team->t.t_max_nproc; ++i) { 3106 if (team->t.t_dispatch[i].th_disp_buffer != NULL) { 3107 __kmp_free(team->t.t_dispatch[i].th_disp_buffer); 3108 team->t.t_dispatch[i].th_disp_buffer = NULL; 3109 } 3110 } 3111 #if KMP_USE_HIER_SCHED 3112 __kmp_dispatch_free_hierarchies(team); 3113 #endif 3114 __kmp_free(team->t.t_threads); 3115 __kmp_free(team->t.t_disp_buffer); 3116 __kmp_free(team->t.t_dispatch); 3117 __kmp_free(team->t.t_implicit_task_taskdata); 3118 team->t.t_threads = NULL; 3119 team->t.t_disp_buffer = NULL; 3120 team->t.t_dispatch = NULL; 3121 team->t.t_implicit_task_taskdata = 0; 3122 } 3123 3124 static void __kmp_reallocate_team_arrays(kmp_team_t *team, int max_nth) { 3125 kmp_info_t **oldThreads = team->t.t_threads; 3126 3127 __kmp_free(team->t.t_disp_buffer); 3128 __kmp_free(team->t.t_dispatch); 3129 __kmp_free(team->t.t_implicit_task_taskdata); 3130 __kmp_allocate_team_arrays(team, max_nth); 3131 3132 KMP_MEMCPY(team->t.t_threads, oldThreads, 3133 team->t.t_nproc * sizeof(kmp_info_t *)); 3134 3135 __kmp_free(oldThreads); 3136 } 3137 3138 static kmp_internal_control_t __kmp_get_global_icvs(void) { 3139 3140 kmp_r_sched_t r_sched = 3141 __kmp_get_schedule_global(); // get current state of scheduling globals 3142 3143 #if OMP_40_ENABLED 3144 KMP_DEBUG_ASSERT(__kmp_nested_proc_bind.used > 0); 3145 #endif /* OMP_40_ENABLED */ 3146 3147 kmp_internal_control_t g_icvs = { 3148 0, // int serial_nesting_level; //corresponds to value of th_team_serialized 3149 (kmp_int8)__kmp_global.g.g_dynamic, // internal control for dynamic 3150 // adjustment of threads (per thread) 3151 (kmp_int8)__kmp_env_blocktime, // int bt_set; //internal control for 3152 // whether blocktime is explicitly set 3153 __kmp_dflt_blocktime, // int blocktime; //internal control for blocktime 3154 #if KMP_USE_MONITOR 3155 __kmp_bt_intervals, // int bt_intervals; //internal control for blocktime 3156 // intervals 3157 #endif 3158 __kmp_dflt_team_nth, // int nproc; //internal control for # of threads for 3159 // next parallel region (per thread) 3160 // (use a max ub on value if __kmp_parallel_initialize not called yet) 3161 __kmp_cg_max_nth, // int thread_limit; 3162 __kmp_dflt_max_active_levels, // int max_active_levels; //internal control 3163 // for max_active_levels 3164 r_sched, // kmp_r_sched_t sched; //internal control for runtime schedule 3165 // {sched,chunk} pair 3166 #if OMP_40_ENABLED 3167 __kmp_nested_proc_bind.bind_types[0], 3168 __kmp_default_device, 3169 #endif /* OMP_40_ENABLED */ 3170 NULL // struct kmp_internal_control *next; 3171 }; 3172 3173 return g_icvs; 3174 } 3175 3176 static kmp_internal_control_t __kmp_get_x_global_icvs(const kmp_team_t *team) { 3177 3178 kmp_internal_control_t gx_icvs; 3179 gx_icvs.serial_nesting_level = 3180 0; // probably =team->t.t_serial like in save_inter_controls 3181 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs); 3182 gx_icvs.next = NULL; 3183 3184 return gx_icvs; 3185 } 3186 3187 static void __kmp_initialize_root(kmp_root_t *root) { 3188 int f; 3189 kmp_team_t *root_team; 3190 kmp_team_t *hot_team; 3191 int hot_team_max_nth; 3192 kmp_r_sched_t r_sched = 3193 __kmp_get_schedule_global(); // get current state of scheduling globals 3194 kmp_internal_control_t r_icvs = __kmp_get_global_icvs(); 3195 KMP_DEBUG_ASSERT(root); 3196 KMP_ASSERT(!root->r.r_begin); 3197 3198 /* setup the root state structure */ 3199 __kmp_init_lock(&root->r.r_begin_lock); 3200 root->r.r_begin = FALSE; 3201 root->r.r_active = FALSE; 3202 root->r.r_in_parallel = 0; 3203 root->r.r_blocktime = __kmp_dflt_blocktime; 3204 3205 /* setup the root team for this task */ 3206 /* allocate the root team structure */ 3207 KF_TRACE(10, ("__kmp_initialize_root: before root_team\n")); 3208 3209 root_team = 3210 __kmp_allocate_team(root, 3211 1, // new_nproc 3212 1, // max_nproc 3213 #if OMPT_SUPPORT 3214 ompt_data_none, // root parallel id 3215 #endif 3216 #if OMP_40_ENABLED 3217 __kmp_nested_proc_bind.bind_types[0], 3218 #endif 3219 &r_icvs, 3220 0 // argc 3221 USE_NESTED_HOT_ARG(NULL) // master thread is unknown 3222 ); 3223 #if USE_DEBUGGER 3224 // Non-NULL value should be assigned to make the debugger display the root 3225 // team. 3226 TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)(~0)); 3227 #endif 3228 3229 KF_TRACE(10, ("__kmp_initialize_root: after root_team = %p\n", root_team)); 3230 3231 root->r.r_root_team = root_team; 3232 root_team->t.t_control_stack_top = NULL; 3233 3234 /* initialize root team */ 3235 root_team->t.t_threads[0] = NULL; 3236 root_team->t.t_nproc = 1; 3237 root_team->t.t_serialized = 1; 3238 // TODO???: root_team->t.t_max_active_levels = __kmp_dflt_max_active_levels; 3239 root_team->t.t_sched.sched = r_sched.sched; 3240 KA_TRACE( 3241 20, 3242 ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n", 3243 root_team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); 3244 3245 /* setup the hot team for this task */ 3246 /* allocate the hot team structure */ 3247 KF_TRACE(10, ("__kmp_initialize_root: before hot_team\n")); 3248 3249 hot_team = 3250 __kmp_allocate_team(root, 3251 1, // new_nproc 3252 __kmp_dflt_team_nth_ub * 2, // max_nproc 3253 #if OMPT_SUPPORT 3254 ompt_data_none, // root parallel id 3255 #endif 3256 #if OMP_40_ENABLED 3257 __kmp_nested_proc_bind.bind_types[0], 3258 #endif 3259 &r_icvs, 3260 0 // argc 3261 USE_NESTED_HOT_ARG(NULL) // master thread is unknown 3262 ); 3263 KF_TRACE(10, ("__kmp_initialize_root: after hot_team = %p\n", hot_team)); 3264 3265 root->r.r_hot_team = hot_team; 3266 root_team->t.t_control_stack_top = NULL; 3267 3268 /* first-time initialization */ 3269 hot_team->t.t_parent = root_team; 3270 3271 /* initialize hot team */ 3272 hot_team_max_nth = hot_team->t.t_max_nproc; 3273 for (f = 0; f < hot_team_max_nth; ++f) { 3274 hot_team->t.t_threads[f] = NULL; 3275 } 3276 hot_team->t.t_nproc = 1; 3277 // TODO???: hot_team->t.t_max_active_levels = __kmp_dflt_max_active_levels; 3278 hot_team->t.t_sched.sched = r_sched.sched; 3279 hot_team->t.t_size_changed = 0; 3280 } 3281 3282 #ifdef KMP_DEBUG 3283 3284 typedef struct kmp_team_list_item { 3285 kmp_team_p const *entry; 3286 struct kmp_team_list_item *next; 3287 } kmp_team_list_item_t; 3288 typedef kmp_team_list_item_t *kmp_team_list_t; 3289 3290 static void __kmp_print_structure_team_accum( // Add team to list of teams. 3291 kmp_team_list_t list, // List of teams. 3292 kmp_team_p const *team // Team to add. 3293 ) { 3294 3295 // List must terminate with item where both entry and next are NULL. 3296 // Team is added to the list only once. 3297 // List is sorted in ascending order by team id. 3298 // Team id is *not* a key. 3299 3300 kmp_team_list_t l; 3301 3302 KMP_DEBUG_ASSERT(list != NULL); 3303 if (team == NULL) { 3304 return; 3305 } 3306 3307 __kmp_print_structure_team_accum(list, team->t.t_parent); 3308 __kmp_print_structure_team_accum(list, team->t.t_next_pool); 3309 3310 // Search list for the team. 3311 l = list; 3312 while (l->next != NULL && l->entry != team) { 3313 l = l->next; 3314 } 3315 if (l->next != NULL) { 3316 return; // Team has been added before, exit. 3317 } 3318 3319 // Team is not found. Search list again for insertion point. 3320 l = list; 3321 while (l->next != NULL && l->entry->t.t_id <= team->t.t_id) { 3322 l = l->next; 3323 } 3324 3325 // Insert team. 3326 { 3327 kmp_team_list_item_t *item = (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC( 3328 sizeof(kmp_team_list_item_t)); 3329 *item = *l; 3330 l->entry = team; 3331 l->next = item; 3332 } 3333 } 3334 3335 static void __kmp_print_structure_team(char const *title, kmp_team_p const *team 3336 3337 ) { 3338 __kmp_printf("%s", title); 3339 if (team != NULL) { 3340 __kmp_printf("%2x %p\n", team->t.t_id, team); 3341 } else { 3342 __kmp_printf(" - (nil)\n"); 3343 } 3344 } 3345 3346 static void __kmp_print_structure_thread(char const *title, 3347 kmp_info_p const *thread) { 3348 __kmp_printf("%s", title); 3349 if (thread != NULL) { 3350 __kmp_printf("%2d %p\n", thread->th.th_info.ds.ds_gtid, thread); 3351 } else { 3352 __kmp_printf(" - (nil)\n"); 3353 } 3354 } 3355 3356 void __kmp_print_structure(void) { 3357 3358 kmp_team_list_t list; 3359 3360 // Initialize list of teams. 3361 list = 3362 (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(sizeof(kmp_team_list_item_t)); 3363 list->entry = NULL; 3364 list->next = NULL; 3365 3366 __kmp_printf("\n------------------------------\nGlobal Thread " 3367 "Table\n------------------------------\n"); 3368 { 3369 int gtid; 3370 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) { 3371 __kmp_printf("%2d", gtid); 3372 if (__kmp_threads != NULL) { 3373 __kmp_printf(" %p", __kmp_threads[gtid]); 3374 } 3375 if (__kmp_root != NULL) { 3376 __kmp_printf(" %p", __kmp_root[gtid]); 3377 } 3378 __kmp_printf("\n"); 3379 } 3380 } 3381 3382 // Print out __kmp_threads array. 3383 __kmp_printf("\n------------------------------\nThreads\n--------------------" 3384 "----------\n"); 3385 if (__kmp_threads != NULL) { 3386 int gtid; 3387 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) { 3388 kmp_info_t const *thread = __kmp_threads[gtid]; 3389 if (thread != NULL) { 3390 __kmp_printf("GTID %2d %p:\n", gtid, thread); 3391 __kmp_printf(" Our Root: %p\n", thread->th.th_root); 3392 __kmp_print_structure_team(" Our Team: ", thread->th.th_team); 3393 __kmp_print_structure_team(" Serial Team: ", 3394 thread->th.th_serial_team); 3395 __kmp_printf(" Threads: %2d\n", thread->th.th_team_nproc); 3396 __kmp_print_structure_thread(" Master: ", 3397 thread->th.th_team_master); 3398 __kmp_printf(" Serialized?: %2d\n", thread->th.th_team_serialized); 3399 __kmp_printf(" Set NProc: %2d\n", thread->th.th_set_nproc); 3400 #if OMP_40_ENABLED 3401 __kmp_printf(" Set Proc Bind: %2d\n", thread->th.th_set_proc_bind); 3402 #endif 3403 __kmp_print_structure_thread(" Next in pool: ", 3404 thread->th.th_next_pool); 3405 __kmp_printf("\n"); 3406 __kmp_print_structure_team_accum(list, thread->th.th_team); 3407 __kmp_print_structure_team_accum(list, thread->th.th_serial_team); 3408 } 3409 } 3410 } else { 3411 __kmp_printf("Threads array is not allocated.\n"); 3412 } 3413 3414 // Print out __kmp_root array. 3415 __kmp_printf("\n------------------------------\nUbers\n----------------------" 3416 "--------\n"); 3417 if (__kmp_root != NULL) { 3418 int gtid; 3419 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) { 3420 kmp_root_t const *root = __kmp_root[gtid]; 3421 if (root != NULL) { 3422 __kmp_printf("GTID %2d %p:\n", gtid, root); 3423 __kmp_print_structure_team(" Root Team: ", root->r.r_root_team); 3424 __kmp_print_structure_team(" Hot Team: ", root->r.r_hot_team); 3425 __kmp_print_structure_thread(" Uber Thread: ", 3426 root->r.r_uber_thread); 3427 __kmp_printf(" Active?: %2d\n", root->r.r_active); 3428 __kmp_printf(" In Parallel: %2d\n", 3429 KMP_ATOMIC_LD_RLX(&root->r.r_in_parallel)); 3430 __kmp_printf("\n"); 3431 __kmp_print_structure_team_accum(list, root->r.r_root_team); 3432 __kmp_print_structure_team_accum(list, root->r.r_hot_team); 3433 } 3434 } 3435 } else { 3436 __kmp_printf("Ubers array is not allocated.\n"); 3437 } 3438 3439 __kmp_printf("\n------------------------------\nTeams\n----------------------" 3440 "--------\n"); 3441 while (list->next != NULL) { 3442 kmp_team_p const *team = list->entry; 3443 int i; 3444 __kmp_printf("Team %2x %p:\n", team->t.t_id, team); 3445 __kmp_print_structure_team(" Parent Team: ", team->t.t_parent); 3446 __kmp_printf(" Master TID: %2d\n", team->t.t_master_tid); 3447 __kmp_printf(" Max threads: %2d\n", team->t.t_max_nproc); 3448 __kmp_printf(" Levels of serial: %2d\n", team->t.t_serialized); 3449 __kmp_printf(" Number threads: %2d\n", team->t.t_nproc); 3450 for (i = 0; i < team->t.t_nproc; ++i) { 3451 __kmp_printf(" Thread %2d: ", i); 3452 __kmp_print_structure_thread("", team->t.t_threads[i]); 3453 } 3454 __kmp_print_structure_team(" Next in pool: ", team->t.t_next_pool); 3455 __kmp_printf("\n"); 3456 list = list->next; 3457 } 3458 3459 // Print out __kmp_thread_pool and __kmp_team_pool. 3460 __kmp_printf("\n------------------------------\nPools\n----------------------" 3461 "--------\n"); 3462 __kmp_print_structure_thread("Thread pool: ", 3463 CCAST(kmp_info_t *, __kmp_thread_pool)); 3464 __kmp_print_structure_team("Team pool: ", 3465 CCAST(kmp_team_t *, __kmp_team_pool)); 3466 __kmp_printf("\n"); 3467 3468 // Free team list. 3469 while (list != NULL) { 3470 kmp_team_list_item_t *item = list; 3471 list = list->next; 3472 KMP_INTERNAL_FREE(item); 3473 } 3474 } 3475 3476 #endif 3477 3478 //--------------------------------------------------------------------------- 3479 // Stuff for per-thread fast random number generator 3480 // Table of primes 3481 static const unsigned __kmp_primes[] = { 3482 0x9e3779b1, 0xffe6cc59, 0x2109f6dd, 0x43977ab5, 0xba5703f5, 0xb495a877, 3483 0xe1626741, 0x79695e6b, 0xbc98c09f, 0xd5bee2b3, 0x287488f9, 0x3af18231, 3484 0x9677cd4d, 0xbe3a6929, 0xadc6a877, 0xdcf0674b, 0xbe4d6fe9, 0x5f15e201, 3485 0x99afc3fd, 0xf3f16801, 0xe222cfff, 0x24ba5fdb, 0x0620452d, 0x79f149e3, 3486 0xc8b93f49, 0x972702cd, 0xb07dd827, 0x6c97d5ed, 0x085a3d61, 0x46eb5ea7, 3487 0x3d9910ed, 0x2e687b5b, 0x29609227, 0x6eb081f1, 0x0954c4e1, 0x9d114db9, 3488 0x542acfa9, 0xb3e6bd7b, 0x0742d917, 0xe9f3ffa7, 0x54581edb, 0xf2480f45, 3489 0x0bb9288f, 0xef1affc7, 0x85fa0ca7, 0x3ccc14db, 0xe6baf34b, 0x343377f7, 3490 0x5ca19031, 0xe6d9293b, 0xf0a9f391, 0x5d2e980b, 0xfc411073, 0xc3749363, 3491 0xb892d829, 0x3549366b, 0x629750ad, 0xb98294e5, 0x892d9483, 0xc235baf3, 3492 0x3d2402a3, 0x6bdef3c9, 0xbec333cd, 0x40c9520f}; 3493 3494 //--------------------------------------------------------------------------- 3495 // __kmp_get_random: Get a random number using a linear congruential method. 3496 unsigned short __kmp_get_random(kmp_info_t *thread) { 3497 unsigned x = thread->th.th_x; 3498 unsigned short r = x >> 16; 3499 3500 thread->th.th_x = x * thread->th.th_a + 1; 3501 3502 KA_TRACE(30, ("__kmp_get_random: THREAD: %d, RETURN: %u\n", 3503 thread->th.th_info.ds.ds_tid, r)); 3504 3505 return r; 3506 } 3507 //-------------------------------------------------------- 3508 // __kmp_init_random: Initialize a random number generator 3509 void __kmp_init_random(kmp_info_t *thread) { 3510 unsigned seed = thread->th.th_info.ds.ds_tid; 3511 3512 thread->th.th_a = 3513 __kmp_primes[seed % (sizeof(__kmp_primes) / sizeof(__kmp_primes[0]))]; 3514 thread->th.th_x = (seed + 1) * thread->th.th_a + 1; 3515 KA_TRACE(30, 3516 ("__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a)); 3517 } 3518 3519 #if KMP_OS_WINDOWS 3520 /* reclaim array entries for root threads that are already dead, returns number 3521 * reclaimed */ 3522 static int __kmp_reclaim_dead_roots(void) { 3523 int i, r = 0; 3524 3525 for (i = 0; i < __kmp_threads_capacity; ++i) { 3526 if (KMP_UBER_GTID(i) && 3527 !__kmp_still_running((kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[i])) && 3528 !__kmp_root[i] 3529 ->r.r_active) { // AC: reclaim only roots died in non-active state 3530 r += __kmp_unregister_root_other_thread(i); 3531 } 3532 } 3533 return r; 3534 } 3535 #endif 3536 3537 /* This function attempts to create free entries in __kmp_threads and 3538 __kmp_root, and returns the number of free entries generated. 3539 3540 For Windows* OS static library, the first mechanism used is to reclaim array 3541 entries for root threads that are already dead. 3542 3543 On all platforms, expansion is attempted on the arrays __kmp_threads_ and 3544 __kmp_root, with appropriate update to __kmp_threads_capacity. Array 3545 capacity is increased by doubling with clipping to __kmp_tp_capacity, if 3546 threadprivate cache array has been created. Synchronization with 3547 __kmpc_threadprivate_cached is done using __kmp_tp_cached_lock. 3548 3549 After any dead root reclamation, if the clipping value allows array expansion 3550 to result in the generation of a total of nNeed free slots, the function does 3551 that expansion. If not, nothing is done beyond the possible initial root 3552 thread reclamation. 3553 3554 If any argument is negative, the behavior is undefined. */ 3555 static int __kmp_expand_threads(int nNeed) { 3556 int added = 0; 3557 int minimumRequiredCapacity; 3558 int newCapacity; 3559 kmp_info_t **newThreads; 3560 kmp_root_t **newRoot; 3561 3562 // All calls to __kmp_expand_threads should be under __kmp_forkjoin_lock, so 3563 // resizing __kmp_threads does not need additional protection if foreign 3564 // threads are present 3565 3566 #if KMP_OS_WINDOWS && !KMP_DYNAMIC_LIB 3567 /* only for Windows static library */ 3568 /* reclaim array entries for root threads that are already dead */ 3569 added = __kmp_reclaim_dead_roots(); 3570 3571 if (nNeed) { 3572 nNeed -= added; 3573 if (nNeed < 0) 3574 nNeed = 0; 3575 } 3576 #endif 3577 if (nNeed <= 0) 3578 return added; 3579 3580 // Note that __kmp_threads_capacity is not bounded by __kmp_max_nth. If 3581 // __kmp_max_nth is set to some value less than __kmp_sys_max_nth by the 3582 // user via KMP_DEVICE_THREAD_LIMIT, then __kmp_threads_capacity may become 3583 // > __kmp_max_nth in one of two ways: 3584 // 3585 // 1) The initialization thread (gtid = 0) exits. __kmp_threads[0] 3586 // may not be resused by another thread, so we may need to increase 3587 // __kmp_threads_capacity to __kmp_max_nth + 1. 3588 // 3589 // 2) New foreign root(s) are encountered. We always register new foreign 3590 // roots. This may cause a smaller # of threads to be allocated at 3591 // subsequent parallel regions, but the worker threads hang around (and 3592 // eventually go to sleep) and need slots in the __kmp_threads[] array. 3593 // 3594 // Anyway, that is the reason for moving the check to see if 3595 // __kmp_max_nth was exceeded into __kmp_reserve_threads() 3596 // instead of having it performed here. -BB 3597 3598 KMP_DEBUG_ASSERT(__kmp_sys_max_nth >= __kmp_threads_capacity); 3599 3600 /* compute expansion headroom to check if we can expand */ 3601 if (__kmp_sys_max_nth - __kmp_threads_capacity < nNeed) { 3602 /* possible expansion too small -- give up */ 3603 return added; 3604 } 3605 minimumRequiredCapacity = __kmp_threads_capacity + nNeed; 3606 3607 newCapacity = __kmp_threads_capacity; 3608 do { 3609 newCapacity = newCapacity <= (__kmp_sys_max_nth >> 1) ? (newCapacity << 1) 3610 : __kmp_sys_max_nth; 3611 } while (newCapacity < minimumRequiredCapacity); 3612 newThreads = (kmp_info_t **)__kmp_allocate( 3613 (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * newCapacity + CACHE_LINE); 3614 newRoot = 3615 (kmp_root_t **)((char *)newThreads + sizeof(kmp_info_t *) * newCapacity); 3616 KMP_MEMCPY(newThreads, __kmp_threads, 3617 __kmp_threads_capacity * sizeof(kmp_info_t *)); 3618 KMP_MEMCPY(newRoot, __kmp_root, 3619 __kmp_threads_capacity * sizeof(kmp_root_t *)); 3620 3621 kmp_info_t **temp_threads = __kmp_threads; 3622 *(kmp_info_t * *volatile *)&__kmp_threads = newThreads; 3623 *(kmp_root_t * *volatile *)&__kmp_root = newRoot; 3624 __kmp_free(temp_threads); 3625 added += newCapacity - __kmp_threads_capacity; 3626 *(volatile int *)&__kmp_threads_capacity = newCapacity; 3627 3628 if (newCapacity > __kmp_tp_capacity) { 3629 __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock); 3630 if (__kmp_tp_cached && newCapacity > __kmp_tp_capacity) { 3631 __kmp_threadprivate_resize_cache(newCapacity); 3632 } else { // increase __kmp_tp_capacity to correspond with kmp_threads size 3633 *(volatile int *)&__kmp_tp_capacity = newCapacity; 3634 } 3635 __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock); 3636 } 3637 3638 return added; 3639 } 3640 3641 /* Register the current thread as a root thread and obtain our gtid. We must 3642 have the __kmp_initz_lock held at this point. Argument TRUE only if are the 3643 thread that calls from __kmp_do_serial_initialize() */ 3644 int __kmp_register_root(int initial_thread) { 3645 kmp_info_t *root_thread; 3646 kmp_root_t *root; 3647 int gtid; 3648 int capacity; 3649 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 3650 KA_TRACE(20, ("__kmp_register_root: entered\n")); 3651 KMP_MB(); 3652 3653 /* 2007-03-02: 3654 If initial thread did not invoke OpenMP RTL yet, and this thread is not an 3655 initial one, "__kmp_all_nth >= __kmp_threads_capacity" condition does not 3656 work as expected -- it may return false (that means there is at least one 3657 empty slot in __kmp_threads array), but it is possible the only free slot 3658 is #0, which is reserved for initial thread and so cannot be used for this 3659 one. Following code workarounds this bug. 3660 3661 However, right solution seems to be not reserving slot #0 for initial 3662 thread because: 3663 (1) there is no magic in slot #0, 3664 (2) we cannot detect initial thread reliably (the first thread which does 3665 serial initialization may be not a real initial thread). 3666 */ 3667 capacity = __kmp_threads_capacity; 3668 if (!initial_thread && TCR_PTR(__kmp_threads[0]) == NULL) { 3669 --capacity; 3670 } 3671 3672 /* see if there are too many threads */ 3673 if (__kmp_all_nth >= capacity && !__kmp_expand_threads(1)) { 3674 if (__kmp_tp_cached) { 3675 __kmp_fatal(KMP_MSG(CantRegisterNewThread), 3676 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity), 3677 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null); 3678 } else { 3679 __kmp_fatal(KMP_MSG(CantRegisterNewThread), KMP_HNT(SystemLimitOnThreads), 3680 __kmp_msg_null); 3681 } 3682 } 3683 3684 /* find an available thread slot */ 3685 /* Don't reassign the zero slot since we need that to only be used by initial 3686 thread */ 3687 for (gtid = (initial_thread ? 0 : 1); TCR_PTR(__kmp_threads[gtid]) != NULL; 3688 gtid++) 3689 ; 3690 KA_TRACE(1, 3691 ("__kmp_register_root: found slot in threads array: T#%d\n", gtid)); 3692 KMP_ASSERT(gtid < __kmp_threads_capacity); 3693 3694 /* update global accounting */ 3695 __kmp_all_nth++; 3696 TCW_4(__kmp_nth, __kmp_nth + 1); 3697 3698 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low 3699 // numbers of procs, and method #2 (keyed API call) for higher numbers. 3700 if (__kmp_adjust_gtid_mode) { 3701 if (__kmp_all_nth >= __kmp_tls_gtid_min) { 3702 if (TCR_4(__kmp_gtid_mode) != 2) { 3703 TCW_4(__kmp_gtid_mode, 2); 3704 } 3705 } else { 3706 if (TCR_4(__kmp_gtid_mode) != 1) { 3707 TCW_4(__kmp_gtid_mode, 1); 3708 } 3709 } 3710 } 3711 3712 #ifdef KMP_ADJUST_BLOCKTIME 3713 /* Adjust blocktime to zero if necessary */ 3714 /* Middle initialization might not have occurred yet */ 3715 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 3716 if (__kmp_nth > __kmp_avail_proc) { 3717 __kmp_zero_bt = TRUE; 3718 } 3719 } 3720 #endif /* KMP_ADJUST_BLOCKTIME */ 3721 3722 /* setup this new hierarchy */ 3723 if (!(root = __kmp_root[gtid])) { 3724 root = __kmp_root[gtid] = (kmp_root_t *)__kmp_allocate(sizeof(kmp_root_t)); 3725 KMP_DEBUG_ASSERT(!root->r.r_root_team); 3726 } 3727 3728 #if KMP_STATS_ENABLED 3729 // Initialize stats as soon as possible (right after gtid assignment). 3730 __kmp_stats_thread_ptr = __kmp_stats_list->push_back(gtid); 3731 __kmp_stats_thread_ptr->startLife(); 3732 KMP_SET_THREAD_STATE(SERIAL_REGION); 3733 KMP_INIT_PARTITIONED_TIMERS(OMP_serial); 3734 #endif 3735 __kmp_initialize_root(root); 3736 3737 /* setup new root thread structure */ 3738 if (root->r.r_uber_thread) { 3739 root_thread = root->r.r_uber_thread; 3740 } else { 3741 root_thread = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t)); 3742 if (__kmp_storage_map) { 3743 __kmp_print_thread_storage_map(root_thread, gtid); 3744 } 3745 root_thread->th.th_info.ds.ds_gtid = gtid; 3746 #if OMPT_SUPPORT 3747 root_thread->th.ompt_thread_info.thread_data = ompt_data_none; 3748 #endif 3749 root_thread->th.th_root = root; 3750 if (__kmp_env_consistency_check) { 3751 root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid); 3752 } 3753 #if USE_FAST_MEMORY 3754 __kmp_initialize_fast_memory(root_thread); 3755 #endif /* USE_FAST_MEMORY */ 3756 3757 #if KMP_USE_BGET 3758 KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL); 3759 __kmp_initialize_bget(root_thread); 3760 #endif 3761 __kmp_init_random(root_thread); // Initialize random number generator 3762 } 3763 3764 /* setup the serial team held in reserve by the root thread */ 3765 if (!root_thread->th.th_serial_team) { 3766 kmp_internal_control_t r_icvs = __kmp_get_global_icvs(); 3767 KF_TRACE(10, ("__kmp_register_root: before serial_team\n")); 3768 root_thread->th.th_serial_team = 3769 __kmp_allocate_team(root, 1, 1, 3770 #if OMPT_SUPPORT 3771 ompt_data_none, // root parallel id 3772 #endif 3773 #if OMP_40_ENABLED 3774 proc_bind_default, 3775 #endif 3776 &r_icvs, 0 USE_NESTED_HOT_ARG(NULL)); 3777 } 3778 KMP_ASSERT(root_thread->th.th_serial_team); 3779 KF_TRACE(10, ("__kmp_register_root: after serial_team = %p\n", 3780 root_thread->th.th_serial_team)); 3781 3782 /* drop root_thread into place */ 3783 TCW_SYNC_PTR(__kmp_threads[gtid], root_thread); 3784 3785 root->r.r_root_team->t.t_threads[0] = root_thread; 3786 root->r.r_hot_team->t.t_threads[0] = root_thread; 3787 root_thread->th.th_serial_team->t.t_threads[0] = root_thread; 3788 // AC: the team created in reserve, not for execution (it is unused for now). 3789 root_thread->th.th_serial_team->t.t_serialized = 0; 3790 root->r.r_uber_thread = root_thread; 3791 3792 /* initialize the thread, get it ready to go */ 3793 __kmp_initialize_info(root_thread, root->r.r_root_team, 0, gtid); 3794 TCW_4(__kmp_init_gtid, TRUE); 3795 3796 /* prepare the master thread for get_gtid() */ 3797 __kmp_gtid_set_specific(gtid); 3798 3799 #if USE_ITT_BUILD 3800 __kmp_itt_thread_name(gtid); 3801 #endif /* USE_ITT_BUILD */ 3802 3803 #ifdef KMP_TDATA_GTID 3804 __kmp_gtid = gtid; 3805 #endif 3806 __kmp_create_worker(gtid, root_thread, __kmp_stksize); 3807 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == gtid); 3808 3809 KA_TRACE(20, ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, " 3810 "plain=%u\n", 3811 gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team), 3812 root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE, 3813 KMP_INIT_BARRIER_STATE)); 3814 { // Initialize barrier data. 3815 int b; 3816 for (b = 0; b < bs_last_barrier; ++b) { 3817 root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE; 3818 #if USE_DEBUGGER 3819 root_thread->th.th_bar[b].bb.b_worker_arrived = 0; 3820 #endif 3821 } 3822 } 3823 KMP_DEBUG_ASSERT(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived == 3824 KMP_INIT_BARRIER_STATE); 3825 3826 #if KMP_AFFINITY_SUPPORTED 3827 #if OMP_40_ENABLED 3828 root_thread->th.th_current_place = KMP_PLACE_UNDEFINED; 3829 root_thread->th.th_new_place = KMP_PLACE_UNDEFINED; 3830 root_thread->th.th_first_place = KMP_PLACE_UNDEFINED; 3831 root_thread->th.th_last_place = KMP_PLACE_UNDEFINED; 3832 #endif 3833 if (TCR_4(__kmp_init_middle)) { 3834 __kmp_affinity_set_init_mask(gtid, TRUE); 3835 } 3836 #endif /* KMP_AFFINITY_SUPPORTED */ 3837 #if OMP_50_ENABLED 3838 root_thread->th.th_def_allocator = __kmp_def_allocator; 3839 root_thread->th.th_prev_level = 0; 3840 root_thread->th.th_prev_num_threads = 1; 3841 #endif 3842 3843 kmp_cg_root_t *tmp = (kmp_cg_root_t *)__kmp_allocate(sizeof(kmp_cg_root_t)); 3844 tmp->cg_root = root_thread; 3845 tmp->cg_thread_limit = __kmp_cg_max_nth; 3846 tmp->cg_nthreads = 1; 3847 KA_TRACE(100, ("__kmp_register_root: Thread %p created node %p with" 3848 " cg_nthreads init to 1\n", 3849 root_thread, tmp)); 3850 tmp->up = NULL; 3851 root_thread->th.th_cg_roots = tmp; 3852 3853 __kmp_root_counter++; 3854 3855 #if OMPT_SUPPORT 3856 if (!initial_thread && ompt_enabled.enabled) { 3857 3858 kmp_info_t *root_thread = ompt_get_thread(); 3859 3860 ompt_set_thread_state(root_thread, ompt_state_overhead); 3861 3862 if (ompt_enabled.ompt_callback_thread_begin) { 3863 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)( 3864 ompt_thread_initial, __ompt_get_thread_data_internal()); 3865 } 3866 ompt_data_t *task_data; 3867 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL, NULL); 3868 if (ompt_enabled.ompt_callback_task_create) { 3869 ompt_callbacks.ompt_callback(ompt_callback_task_create)( 3870 NULL, NULL, task_data, ompt_task_initial, 0, NULL); 3871 // initial task has nothing to return to 3872 } 3873 3874 ompt_set_thread_state(root_thread, ompt_state_work_serial); 3875 } 3876 #endif 3877 3878 KMP_MB(); 3879 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 3880 3881 return gtid; 3882 } 3883 3884 #if KMP_NESTED_HOT_TEAMS 3885 static int __kmp_free_hot_teams(kmp_root_t *root, kmp_info_t *thr, int level, 3886 const int max_level) { 3887 int i, n, nth; 3888 kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams; 3889 if (!hot_teams || !hot_teams[level].hot_team) { 3890 return 0; 3891 } 3892 KMP_DEBUG_ASSERT(level < max_level); 3893 kmp_team_t *team = hot_teams[level].hot_team; 3894 nth = hot_teams[level].hot_team_nth; 3895 n = nth - 1; // master is not freed 3896 if (level < max_level - 1) { 3897 for (i = 0; i < nth; ++i) { 3898 kmp_info_t *th = team->t.t_threads[i]; 3899 n += __kmp_free_hot_teams(root, th, level + 1, max_level); 3900 if (i > 0 && th->th.th_hot_teams) { 3901 __kmp_free(th->th.th_hot_teams); 3902 th->th.th_hot_teams = NULL; 3903 } 3904 } 3905 } 3906 __kmp_free_team(root, team, NULL); 3907 return n; 3908 } 3909 #endif 3910 3911 // Resets a root thread and clear its root and hot teams. 3912 // Returns the number of __kmp_threads entries directly and indirectly freed. 3913 static int __kmp_reset_root(int gtid, kmp_root_t *root) { 3914 kmp_team_t *root_team = root->r.r_root_team; 3915 kmp_team_t *hot_team = root->r.r_hot_team; 3916 int n = hot_team->t.t_nproc; 3917 int i; 3918 3919 KMP_DEBUG_ASSERT(!root->r.r_active); 3920 3921 root->r.r_root_team = NULL; 3922 root->r.r_hot_team = NULL; 3923 // __kmp_free_team() does not free hot teams, so we have to clear r_hot_team 3924 // before call to __kmp_free_team(). 3925 __kmp_free_team(root, root_team USE_NESTED_HOT_ARG(NULL)); 3926 #if KMP_NESTED_HOT_TEAMS 3927 if (__kmp_hot_teams_max_level > 3928 0) { // need to free nested hot teams and their threads if any 3929 for (i = 0; i < hot_team->t.t_nproc; ++i) { 3930 kmp_info_t *th = hot_team->t.t_threads[i]; 3931 if (__kmp_hot_teams_max_level > 1) { 3932 n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level); 3933 } 3934 if (th->th.th_hot_teams) { 3935 __kmp_free(th->th.th_hot_teams); 3936 th->th.th_hot_teams = NULL; 3937 } 3938 } 3939 } 3940 #endif 3941 __kmp_free_team(root, hot_team USE_NESTED_HOT_ARG(NULL)); 3942 3943 // Before we can reap the thread, we need to make certain that all other 3944 // threads in the teams that had this root as ancestor have stopped trying to 3945 // steal tasks. 3946 if (__kmp_tasking_mode != tskm_immediate_exec) { 3947 __kmp_wait_to_unref_task_teams(); 3948 } 3949 3950 #if KMP_OS_WINDOWS 3951 /* Close Handle of root duplicated in __kmp_create_worker (tr #62919) */ 3952 KA_TRACE( 3953 10, ("__kmp_reset_root: free handle, th = %p, handle = %" KMP_UINTPTR_SPEC 3954 "\n", 3955 (LPVOID) & (root->r.r_uber_thread->th), 3956 root->r.r_uber_thread->th.th_info.ds.ds_thread)); 3957 __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread); 3958 #endif /* KMP_OS_WINDOWS */ 3959 3960 #if OMPT_SUPPORT 3961 if (ompt_enabled.ompt_callback_thread_end) { 3962 ompt_callbacks.ompt_callback(ompt_callback_thread_end)( 3963 &(root->r.r_uber_thread->th.ompt_thread_info.thread_data)); 3964 } 3965 #endif 3966 3967 TCW_4(__kmp_nth, 3968 __kmp_nth - 1); // __kmp_reap_thread will decrement __kmp_all_nth. 3969 root->r.r_uber_thread->th.th_cg_roots->cg_nthreads--; 3970 KA_TRACE(100, ("__kmp_reset_root: Thread %p decrement cg_nthreads on node %p" 3971 " to %d\n", 3972 root->r.r_uber_thread, root->r.r_uber_thread->th.th_cg_roots, 3973 root->r.r_uber_thread->th.th_cg_roots->cg_nthreads)); 3974 3975 __kmp_reap_thread(root->r.r_uber_thread, 1); 3976 3977 // We canot put root thread to __kmp_thread_pool, so we have to reap it istead 3978 // of freeing. 3979 root->r.r_uber_thread = NULL; 3980 /* mark root as no longer in use */ 3981 root->r.r_begin = FALSE; 3982 3983 return n; 3984 } 3985 3986 void __kmp_unregister_root_current_thread(int gtid) { 3987 KA_TRACE(1, ("__kmp_unregister_root_current_thread: enter T#%d\n", gtid)); 3988 /* this lock should be ok, since unregister_root_current_thread is never 3989 called during an abort, only during a normal close. furthermore, if you 3990 have the forkjoin lock, you should never try to get the initz lock */ 3991 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 3992 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 3993 KC_TRACE(10, ("__kmp_unregister_root_current_thread: already finished, " 3994 "exiting T#%d\n", 3995 gtid)); 3996 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 3997 return; 3998 } 3999 kmp_root_t *root = __kmp_root[gtid]; 4000 4001 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]); 4002 KMP_ASSERT(KMP_UBER_GTID(gtid)); 4003 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root); 4004 KMP_ASSERT(root->r.r_active == FALSE); 4005 4006 KMP_MB(); 4007 4008 #if OMP_45_ENABLED 4009 kmp_info_t *thread = __kmp_threads[gtid]; 4010 kmp_team_t *team = thread->th.th_team; 4011 kmp_task_team_t *task_team = thread->th.th_task_team; 4012 4013 // we need to wait for the proxy tasks before finishing the thread 4014 if (task_team != NULL && task_team->tt.tt_found_proxy_tasks) { 4015 #if OMPT_SUPPORT 4016 // the runtime is shutting down so we won't report any events 4017 thread->th.ompt_thread_info.state = ompt_state_undefined; 4018 #endif 4019 __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL)); 4020 } 4021 #endif 4022 4023 __kmp_reset_root(gtid, root); 4024 4025 /* free up this thread slot */ 4026 __kmp_gtid_set_specific(KMP_GTID_DNE); 4027 #ifdef KMP_TDATA_GTID 4028 __kmp_gtid = KMP_GTID_DNE; 4029 #endif 4030 4031 KMP_MB(); 4032 KC_TRACE(10, 4033 ("__kmp_unregister_root_current_thread: T#%d unregistered\n", gtid)); 4034 4035 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 4036 } 4037 4038 #if KMP_OS_WINDOWS 4039 /* __kmp_forkjoin_lock must be already held 4040 Unregisters a root thread that is not the current thread. Returns the number 4041 of __kmp_threads entries freed as a result. */ 4042 static int __kmp_unregister_root_other_thread(int gtid) { 4043 kmp_root_t *root = __kmp_root[gtid]; 4044 int r; 4045 4046 KA_TRACE(1, ("__kmp_unregister_root_other_thread: enter T#%d\n", gtid)); 4047 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]); 4048 KMP_ASSERT(KMP_UBER_GTID(gtid)); 4049 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root); 4050 KMP_ASSERT(root->r.r_active == FALSE); 4051 4052 r = __kmp_reset_root(gtid, root); 4053 KC_TRACE(10, 4054 ("__kmp_unregister_root_other_thread: T#%d unregistered\n", gtid)); 4055 return r; 4056 } 4057 #endif 4058 4059 #if KMP_DEBUG 4060 void __kmp_task_info() { 4061 4062 kmp_int32 gtid = __kmp_entry_gtid(); 4063 kmp_int32 tid = __kmp_tid_from_gtid(gtid); 4064 kmp_info_t *this_thr = __kmp_threads[gtid]; 4065 kmp_team_t *steam = this_thr->th.th_serial_team; 4066 kmp_team_t *team = this_thr->th.th_team; 4067 4068 __kmp_printf( 4069 "__kmp_task_info: gtid=%d tid=%d t_thread=%p team=%p steam=%p curtask=%p " 4070 "ptask=%p\n", 4071 gtid, tid, this_thr, team, steam, this_thr->th.th_current_task, 4072 team->t.t_implicit_task_taskdata[tid].td_parent); 4073 } 4074 #endif // KMP_DEBUG 4075 4076 /* TODO optimize with one big memclr, take out what isn't needed, split 4077 responsibility to workers as much as possible, and delay initialization of 4078 features as much as possible */ 4079 static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team, 4080 int tid, int gtid) { 4081 /* this_thr->th.th_info.ds.ds_gtid is setup in 4082 kmp_allocate_thread/create_worker. 4083 this_thr->th.th_serial_team is setup in __kmp_allocate_thread */ 4084 kmp_info_t *master = team->t.t_threads[0]; 4085 KMP_DEBUG_ASSERT(this_thr != NULL); 4086 KMP_DEBUG_ASSERT(this_thr->th.th_serial_team); 4087 KMP_DEBUG_ASSERT(team); 4088 KMP_DEBUG_ASSERT(team->t.t_threads); 4089 KMP_DEBUG_ASSERT(team->t.t_dispatch); 4090 KMP_DEBUG_ASSERT(master); 4091 KMP_DEBUG_ASSERT(master->th.th_root); 4092 4093 KMP_MB(); 4094 4095 TCW_SYNC_PTR(this_thr->th.th_team, team); 4096 4097 this_thr->th.th_info.ds.ds_tid = tid; 4098 this_thr->th.th_set_nproc = 0; 4099 if (__kmp_tasking_mode != tskm_immediate_exec) 4100 // When tasking is possible, threads are not safe to reap until they are 4101 // done tasking; this will be set when tasking code is exited in wait 4102 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP; 4103 else // no tasking --> always safe to reap 4104 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP; 4105 #if OMP_40_ENABLED 4106 this_thr->th.th_set_proc_bind = proc_bind_default; 4107 #if KMP_AFFINITY_SUPPORTED 4108 this_thr->th.th_new_place = this_thr->th.th_current_place; 4109 #endif 4110 #endif 4111 this_thr->th.th_root = master->th.th_root; 4112 4113 /* setup the thread's cache of the team structure */ 4114 this_thr->th.th_team_nproc = team->t.t_nproc; 4115 this_thr->th.th_team_master = master; 4116 this_thr->th.th_team_serialized = team->t.t_serialized; 4117 TCW_PTR(this_thr->th.th_sleep_loc, NULL); 4118 4119 KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata); 4120 4121 KF_TRACE(10, ("__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n", 4122 tid, gtid, this_thr, this_thr->th.th_current_task)); 4123 4124 __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr, 4125 team, tid, TRUE); 4126 4127 KF_TRACE(10, ("__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n", 4128 tid, gtid, this_thr, this_thr->th.th_current_task)); 4129 // TODO: Initialize ICVs from parent; GEH - isn't that already done in 4130 // __kmp_initialize_team()? 4131 4132 /* TODO no worksharing in speculative threads */ 4133 this_thr->th.th_dispatch = &team->t.t_dispatch[tid]; 4134 4135 this_thr->th.th_local.this_construct = 0; 4136 4137 if (!this_thr->th.th_pri_common) { 4138 this_thr->th.th_pri_common = 4139 (struct common_table *)__kmp_allocate(sizeof(struct common_table)); 4140 if (__kmp_storage_map) { 4141 __kmp_print_storage_map_gtid( 4142 gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1, 4143 sizeof(struct common_table), "th_%d.th_pri_common\n", gtid); 4144 } 4145 this_thr->th.th_pri_head = NULL; 4146 } 4147 4148 if (this_thr != master && // Master's CG root is initialized elsewhere 4149 this_thr->th.th_cg_roots != master->th.th_cg_roots) { // CG root not set 4150 // Make new thread's CG root same as master's 4151 KMP_DEBUG_ASSERT(master->th.th_cg_roots); 4152 this_thr->th.th_cg_roots = master->th.th_cg_roots; 4153 // Increment new thread's CG root's counter to add the new thread 4154 this_thr->th.th_cg_roots->cg_nthreads++; 4155 KA_TRACE(100, ("__kmp_initialize_info: Thread %p increment cg_nthreads on" 4156 " node %p of thread %p to %d\n", 4157 this_thr, this_thr->th.th_cg_roots, 4158 this_thr->th.th_cg_roots->cg_root, 4159 this_thr->th.th_cg_roots->cg_nthreads)); 4160 this_thr->th.th_current_task->td_icvs.thread_limit = 4161 this_thr->th.th_cg_roots->cg_thread_limit; 4162 } 4163 4164 /* Initialize dynamic dispatch */ 4165 { 4166 volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch; 4167 // Use team max_nproc since this will never change for the team. 4168 size_t disp_size = 4169 sizeof(dispatch_private_info_t) * 4170 (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers); 4171 KD_TRACE(10, ("__kmp_initialize_info: T#%d max_nproc: %d\n", gtid, 4172 team->t.t_max_nproc)); 4173 KMP_ASSERT(dispatch); 4174 KMP_DEBUG_ASSERT(team->t.t_dispatch); 4175 KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid]); 4176 4177 dispatch->th_disp_index = 0; 4178 #if OMP_45_ENABLED 4179 dispatch->th_doacross_buf_idx = 0; 4180 #endif 4181 if (!dispatch->th_disp_buffer) { 4182 dispatch->th_disp_buffer = 4183 (dispatch_private_info_t *)__kmp_allocate(disp_size); 4184 4185 if (__kmp_storage_map) { 4186 __kmp_print_storage_map_gtid( 4187 gtid, &dispatch->th_disp_buffer[0], 4188 &dispatch->th_disp_buffer[team->t.t_max_nproc == 1 4189 ? 1 4190 : __kmp_dispatch_num_buffers], 4191 disp_size, "th_%d.th_dispatch.th_disp_buffer " 4192 "(team_%d.t_dispatch[%d].th_disp_buffer)", 4193 gtid, team->t.t_id, gtid); 4194 } 4195 } else { 4196 memset(&dispatch->th_disp_buffer[0], '\0', disp_size); 4197 } 4198 4199 dispatch->th_dispatch_pr_current = 0; 4200 dispatch->th_dispatch_sh_current = 0; 4201 4202 dispatch->th_deo_fcn = 0; /* ORDERED */ 4203 dispatch->th_dxo_fcn = 0; /* END ORDERED */ 4204 } 4205 4206 this_thr->th.th_next_pool = NULL; 4207 4208 if (!this_thr->th.th_task_state_memo_stack) { 4209 size_t i; 4210 this_thr->th.th_task_state_memo_stack = 4211 (kmp_uint8 *)__kmp_allocate(4 * sizeof(kmp_uint8)); 4212 this_thr->th.th_task_state_top = 0; 4213 this_thr->th.th_task_state_stack_sz = 4; 4214 for (i = 0; i < this_thr->th.th_task_state_stack_sz; 4215 ++i) // zero init the stack 4216 this_thr->th.th_task_state_memo_stack[i] = 0; 4217 } 4218 4219 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here); 4220 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0); 4221 4222 KMP_MB(); 4223 } 4224 4225 /* allocate a new thread for the requesting team. this is only called from 4226 within a forkjoin critical section. we will first try to get an available 4227 thread from the thread pool. if none is available, we will fork a new one 4228 assuming we are able to create a new one. this should be assured, as the 4229 caller should check on this first. */ 4230 kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team, 4231 int new_tid) { 4232 kmp_team_t *serial_team; 4233 kmp_info_t *new_thr; 4234 int new_gtid; 4235 4236 KA_TRACE(20, ("__kmp_allocate_thread: T#%d\n", __kmp_get_gtid())); 4237 KMP_DEBUG_ASSERT(root && team); 4238 #if !KMP_NESTED_HOT_TEAMS 4239 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(__kmp_get_gtid())); 4240 #endif 4241 KMP_MB(); 4242 4243 /* first, try to get one from the thread pool */ 4244 if (__kmp_thread_pool) { 4245 new_thr = CCAST(kmp_info_t *, __kmp_thread_pool); 4246 __kmp_thread_pool = (volatile kmp_info_t *)new_thr->th.th_next_pool; 4247 if (new_thr == __kmp_thread_pool_insert_pt) { 4248 __kmp_thread_pool_insert_pt = NULL; 4249 } 4250 TCW_4(new_thr->th.th_in_pool, FALSE); 4251 // Don't touch th_active_in_pool or th_active. 4252 // The worker thread adjusts those flags as it sleeps/awakens. 4253 __kmp_thread_pool_nth--; 4254 4255 KA_TRACE(20, ("__kmp_allocate_thread: T#%d using thread T#%d\n", 4256 __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid)); 4257 KMP_ASSERT(!new_thr->th.th_team); 4258 KMP_DEBUG_ASSERT(__kmp_nth < __kmp_threads_capacity); 4259 KMP_DEBUG_ASSERT(__kmp_thread_pool_nth >= 0); 4260 4261 /* setup the thread structure */ 4262 __kmp_initialize_info(new_thr, team, new_tid, 4263 new_thr->th.th_info.ds.ds_gtid); 4264 KMP_DEBUG_ASSERT(new_thr->th.th_serial_team); 4265 4266 TCW_4(__kmp_nth, __kmp_nth + 1); 4267 4268 new_thr->th.th_task_state = 0; 4269 new_thr->th.th_task_state_top = 0; 4270 new_thr->th.th_task_state_stack_sz = 4; 4271 4272 #ifdef KMP_ADJUST_BLOCKTIME 4273 /* Adjust blocktime back to zero if necessary */ 4274 /* Middle initialization might not have occurred yet */ 4275 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 4276 if (__kmp_nth > __kmp_avail_proc) { 4277 __kmp_zero_bt = TRUE; 4278 } 4279 } 4280 #endif /* KMP_ADJUST_BLOCKTIME */ 4281 4282 #if KMP_DEBUG 4283 // If thread entered pool via __kmp_free_thread, wait_flag should != 4284 // KMP_BARRIER_PARENT_FLAG. 4285 int b; 4286 kmp_balign_t *balign = new_thr->th.th_bar; 4287 for (b = 0; b < bs_last_barrier; ++b) 4288 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 4289 #endif 4290 4291 KF_TRACE(10, ("__kmp_allocate_thread: T#%d using thread %p T#%d\n", 4292 __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid)); 4293 4294 KMP_MB(); 4295 return new_thr; 4296 } 4297 4298 /* no, well fork a new one */ 4299 KMP_ASSERT(__kmp_nth == __kmp_all_nth); 4300 KMP_ASSERT(__kmp_all_nth < __kmp_threads_capacity); 4301 4302 #if KMP_USE_MONITOR 4303 // If this is the first worker thread the RTL is creating, then also 4304 // launch the monitor thread. We try to do this as early as possible. 4305 if (!TCR_4(__kmp_init_monitor)) { 4306 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock); 4307 if (!TCR_4(__kmp_init_monitor)) { 4308 KF_TRACE(10, ("before __kmp_create_monitor\n")); 4309 TCW_4(__kmp_init_monitor, 1); 4310 __kmp_create_monitor(&__kmp_monitor); 4311 KF_TRACE(10, ("after __kmp_create_monitor\n")); 4312 #if KMP_OS_WINDOWS 4313 // AC: wait until monitor has started. This is a fix for CQ232808. 4314 // The reason is that if the library is loaded/unloaded in a loop with 4315 // small (parallel) work in between, then there is high probability that 4316 // monitor thread started after the library shutdown. At shutdown it is 4317 // too late to cope with the problem, because when the master is in 4318 // DllMain (process detach) the monitor has no chances to start (it is 4319 // blocked), and master has no means to inform the monitor that the 4320 // library has gone, because all the memory which the monitor can access 4321 // is going to be released/reset. 4322 while (TCR_4(__kmp_init_monitor) < 2) { 4323 KMP_YIELD(TRUE); 4324 } 4325 KF_TRACE(10, ("after monitor thread has started\n")); 4326 #endif 4327 } 4328 __kmp_release_bootstrap_lock(&__kmp_monitor_lock); 4329 } 4330 #endif 4331 4332 KMP_MB(); 4333 for (new_gtid = 1; TCR_PTR(__kmp_threads[new_gtid]) != NULL; ++new_gtid) { 4334 KMP_DEBUG_ASSERT(new_gtid < __kmp_threads_capacity); 4335 } 4336 4337 /* allocate space for it. */ 4338 new_thr = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t)); 4339 4340 TCW_SYNC_PTR(__kmp_threads[new_gtid], new_thr); 4341 4342 if (__kmp_storage_map) { 4343 __kmp_print_thread_storage_map(new_thr, new_gtid); 4344 } 4345 4346 // add the reserve serialized team, initialized from the team's master thread 4347 { 4348 kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team); 4349 KF_TRACE(10, ("__kmp_allocate_thread: before th_serial/serial_team\n")); 4350 new_thr->th.th_serial_team = serial_team = 4351 (kmp_team_t *)__kmp_allocate_team(root, 1, 1, 4352 #if OMPT_SUPPORT 4353 ompt_data_none, // root parallel id 4354 #endif 4355 #if OMP_40_ENABLED 4356 proc_bind_default, 4357 #endif 4358 &r_icvs, 0 USE_NESTED_HOT_ARG(NULL)); 4359 } 4360 KMP_ASSERT(serial_team); 4361 serial_team->t.t_serialized = 0; // AC: the team created in reserve, not for 4362 // execution (it is unused for now). 4363 serial_team->t.t_threads[0] = new_thr; 4364 KF_TRACE(10, 4365 ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n", 4366 new_thr)); 4367 4368 /* setup the thread structures */ 4369 __kmp_initialize_info(new_thr, team, new_tid, new_gtid); 4370 4371 #if USE_FAST_MEMORY 4372 __kmp_initialize_fast_memory(new_thr); 4373 #endif /* USE_FAST_MEMORY */ 4374 4375 #if KMP_USE_BGET 4376 KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL); 4377 __kmp_initialize_bget(new_thr); 4378 #endif 4379 4380 __kmp_init_random(new_thr); // Initialize random number generator 4381 4382 /* Initialize these only once when thread is grabbed for a team allocation */ 4383 KA_TRACE(20, 4384 ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n", 4385 __kmp_get_gtid(), KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); 4386 4387 int b; 4388 kmp_balign_t *balign = new_thr->th.th_bar; 4389 for (b = 0; b < bs_last_barrier; ++b) { 4390 balign[b].bb.b_go = KMP_INIT_BARRIER_STATE; 4391 balign[b].bb.team = NULL; 4392 balign[b].bb.wait_flag = KMP_BARRIER_NOT_WAITING; 4393 balign[b].bb.use_oncore_barrier = 0; 4394 } 4395 4396 new_thr->th.th_spin_here = FALSE; 4397 new_thr->th.th_next_waiting = 0; 4398 #if KMP_OS_UNIX 4399 new_thr->th.th_blocking = false; 4400 #endif 4401 4402 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 4403 new_thr->th.th_current_place = KMP_PLACE_UNDEFINED; 4404 new_thr->th.th_new_place = KMP_PLACE_UNDEFINED; 4405 new_thr->th.th_first_place = KMP_PLACE_UNDEFINED; 4406 new_thr->th.th_last_place = KMP_PLACE_UNDEFINED; 4407 #endif 4408 #if OMP_50_ENABLED 4409 new_thr->th.th_def_allocator = __kmp_def_allocator; 4410 new_thr->th.th_prev_level = 0; 4411 new_thr->th.th_prev_num_threads = 1; 4412 #endif 4413 4414 TCW_4(new_thr->th.th_in_pool, FALSE); 4415 new_thr->th.th_active_in_pool = FALSE; 4416 TCW_4(new_thr->th.th_active, TRUE); 4417 4418 /* adjust the global counters */ 4419 __kmp_all_nth++; 4420 __kmp_nth++; 4421 4422 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low 4423 // numbers of procs, and method #2 (keyed API call) for higher numbers. 4424 if (__kmp_adjust_gtid_mode) { 4425 if (__kmp_all_nth >= __kmp_tls_gtid_min) { 4426 if (TCR_4(__kmp_gtid_mode) != 2) { 4427 TCW_4(__kmp_gtid_mode, 2); 4428 } 4429 } else { 4430 if (TCR_4(__kmp_gtid_mode) != 1) { 4431 TCW_4(__kmp_gtid_mode, 1); 4432 } 4433 } 4434 } 4435 4436 #ifdef KMP_ADJUST_BLOCKTIME 4437 /* Adjust blocktime back to zero if necessary */ 4438 /* Middle initialization might not have occurred yet */ 4439 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 4440 if (__kmp_nth > __kmp_avail_proc) { 4441 __kmp_zero_bt = TRUE; 4442 } 4443 } 4444 #endif /* KMP_ADJUST_BLOCKTIME */ 4445 4446 /* actually fork it and create the new worker thread */ 4447 KF_TRACE( 4448 10, ("__kmp_allocate_thread: before __kmp_create_worker: %p\n", new_thr)); 4449 __kmp_create_worker(new_gtid, new_thr, __kmp_stksize); 4450 KF_TRACE(10, 4451 ("__kmp_allocate_thread: after __kmp_create_worker: %p\n", new_thr)); 4452 4453 KA_TRACE(20, ("__kmp_allocate_thread: T#%d forked T#%d\n", __kmp_get_gtid(), 4454 new_gtid)); 4455 KMP_MB(); 4456 return new_thr; 4457 } 4458 4459 /* Reinitialize team for reuse. 4460 The hot team code calls this case at every fork barrier, so EPCC barrier 4461 test are extremely sensitive to changes in it, esp. writes to the team 4462 struct, which cause a cache invalidation in all threads. 4463 IF YOU TOUCH THIS ROUTINE, RUN EPCC C SYNCBENCH ON A BIG-IRON MACHINE!!! */ 4464 static void __kmp_reinitialize_team(kmp_team_t *team, 4465 kmp_internal_control_t *new_icvs, 4466 ident_t *loc) { 4467 KF_TRACE(10, ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n", 4468 team->t.t_threads[0], team)); 4469 KMP_DEBUG_ASSERT(team && new_icvs); 4470 KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc); 4471 KMP_CHECK_UPDATE(team->t.t_ident, loc); 4472 4473 KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID()); 4474 // Copy ICVs to the master thread's implicit taskdata 4475 __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE); 4476 copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs); 4477 4478 KF_TRACE(10, ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n", 4479 team->t.t_threads[0], team)); 4480 } 4481 4482 /* Initialize the team data structure. 4483 This assumes the t_threads and t_max_nproc are already set. 4484 Also, we don't touch the arguments */ 4485 static void __kmp_initialize_team(kmp_team_t *team, int new_nproc, 4486 kmp_internal_control_t *new_icvs, 4487 ident_t *loc) { 4488 KF_TRACE(10, ("__kmp_initialize_team: enter: team=%p\n", team)); 4489 4490 /* verify */ 4491 KMP_DEBUG_ASSERT(team); 4492 KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc); 4493 KMP_DEBUG_ASSERT(team->t.t_threads); 4494 KMP_MB(); 4495 4496 team->t.t_master_tid = 0; /* not needed */ 4497 /* team->t.t_master_bar; not needed */ 4498 team->t.t_serialized = new_nproc > 1 ? 0 : 1; 4499 team->t.t_nproc = new_nproc; 4500 4501 /* team->t.t_parent = NULL; TODO not needed & would mess up hot team */ 4502 team->t.t_next_pool = NULL; 4503 /* memset( team->t.t_threads, 0, sizeof(kmp_info_t*)*new_nproc ); would mess 4504 * up hot team */ 4505 4506 TCW_SYNC_PTR(team->t.t_pkfn, NULL); /* not needed */ 4507 team->t.t_invoke = NULL; /* not needed */ 4508 4509 // TODO???: team->t.t_max_active_levels = new_max_active_levels; 4510 team->t.t_sched.sched = new_icvs->sched.sched; 4511 4512 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 4513 team->t.t_fp_control_saved = FALSE; /* not needed */ 4514 team->t.t_x87_fpu_control_word = 0; /* not needed */ 4515 team->t.t_mxcsr = 0; /* not needed */ 4516 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 4517 4518 team->t.t_construct = 0; 4519 4520 team->t.t_ordered.dt.t_value = 0; 4521 team->t.t_master_active = FALSE; 4522 4523 #ifdef KMP_DEBUG 4524 team->t.t_copypriv_data = NULL; /* not necessary, but nice for debugging */ 4525 #endif 4526 #if KMP_OS_WINDOWS 4527 team->t.t_copyin_counter = 0; /* for barrier-free copyin implementation */ 4528 #endif 4529 4530 team->t.t_control_stack_top = NULL; 4531 4532 __kmp_reinitialize_team(team, new_icvs, loc); 4533 4534 KMP_MB(); 4535 KF_TRACE(10, ("__kmp_initialize_team: exit: team=%p\n", team)); 4536 } 4537 4538 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 4539 /* Sets full mask for thread and returns old mask, no changes to structures. */ 4540 static void 4541 __kmp_set_thread_affinity_mask_full_tmp(kmp_affin_mask_t *old_mask) { 4542 if (KMP_AFFINITY_CAPABLE()) { 4543 int status; 4544 if (old_mask != NULL) { 4545 status = __kmp_get_system_affinity(old_mask, TRUE); 4546 int error = errno; 4547 if (status != 0) { 4548 __kmp_fatal(KMP_MSG(ChangeThreadAffMaskError), KMP_ERR(error), 4549 __kmp_msg_null); 4550 } 4551 } 4552 __kmp_set_system_affinity(__kmp_affin_fullMask, TRUE); 4553 } 4554 } 4555 #endif 4556 4557 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 4558 4559 // __kmp_partition_places() is the heart of the OpenMP 4.0 affinity mechanism. 4560 // It calculats the worker + master thread's partition based upon the parent 4561 // thread's partition, and binds each worker to a thread in their partition. 4562 // The master thread's partition should already include its current binding. 4563 static void __kmp_partition_places(kmp_team_t *team, int update_master_only) { 4564 // Copy the master thread's place partion to the team struct 4565 kmp_info_t *master_th = team->t.t_threads[0]; 4566 KMP_DEBUG_ASSERT(master_th != NULL); 4567 kmp_proc_bind_t proc_bind = team->t.t_proc_bind; 4568 int first_place = master_th->th.th_first_place; 4569 int last_place = master_th->th.th_last_place; 4570 int masters_place = master_th->th.th_current_place; 4571 team->t.t_first_place = first_place; 4572 team->t.t_last_place = last_place; 4573 4574 KA_TRACE(20, ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) " 4575 "bound to place %d partition = [%d,%d]\n", 4576 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]), 4577 team->t.t_id, masters_place, first_place, last_place)); 4578 4579 switch (proc_bind) { 4580 4581 case proc_bind_default: 4582 // serial teams might have the proc_bind policy set to proc_bind_default. It 4583 // doesn't matter, as we don't rebind master thread for any proc_bind policy 4584 KMP_DEBUG_ASSERT(team->t.t_nproc == 1); 4585 break; 4586 4587 case proc_bind_master: { 4588 int f; 4589 int n_th = team->t.t_nproc; 4590 for (f = 1; f < n_th; f++) { 4591 kmp_info_t *th = team->t.t_threads[f]; 4592 KMP_DEBUG_ASSERT(th != NULL); 4593 th->th.th_first_place = first_place; 4594 th->th.th_last_place = last_place; 4595 th->th.th_new_place = masters_place; 4596 #if OMP_50_ENABLED 4597 if (__kmp_display_affinity && masters_place != th->th.th_current_place && 4598 team->t.t_display_affinity != 1) { 4599 team->t.t_display_affinity = 1; 4600 } 4601 #endif 4602 4603 KA_TRACE(100, ("__kmp_partition_places: master: T#%d(%d:%d) place %d " 4604 "partition = [%d,%d]\n", 4605 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, 4606 f, masters_place, first_place, last_place)); 4607 } 4608 } break; 4609 4610 case proc_bind_close: { 4611 int f; 4612 int n_th = team->t.t_nproc; 4613 int n_places; 4614 if (first_place <= last_place) { 4615 n_places = last_place - first_place + 1; 4616 } else { 4617 n_places = __kmp_affinity_num_masks - first_place + last_place + 1; 4618 } 4619 if (n_th <= n_places) { 4620 int place = masters_place; 4621 for (f = 1; f < n_th; f++) { 4622 kmp_info_t *th = team->t.t_threads[f]; 4623 KMP_DEBUG_ASSERT(th != NULL); 4624 4625 if (place == last_place) { 4626 place = first_place; 4627 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4628 place = 0; 4629 } else { 4630 place++; 4631 } 4632 th->th.th_first_place = first_place; 4633 th->th.th_last_place = last_place; 4634 th->th.th_new_place = place; 4635 #if OMP_50_ENABLED 4636 if (__kmp_display_affinity && place != th->th.th_current_place && 4637 team->t.t_display_affinity != 1) { 4638 team->t.t_display_affinity = 1; 4639 } 4640 #endif 4641 4642 KA_TRACE(100, ("__kmp_partition_places: close: T#%d(%d:%d) place %d " 4643 "partition = [%d,%d]\n", 4644 __kmp_gtid_from_thread(team->t.t_threads[f]), 4645 team->t.t_id, f, place, first_place, last_place)); 4646 } 4647 } else { 4648 int S, rem, gap, s_count; 4649 S = n_th / n_places; 4650 s_count = 0; 4651 rem = n_th - (S * n_places); 4652 gap = rem > 0 ? n_places / rem : n_places; 4653 int place = masters_place; 4654 int gap_ct = gap; 4655 for (f = 0; f < n_th; f++) { 4656 kmp_info_t *th = team->t.t_threads[f]; 4657 KMP_DEBUG_ASSERT(th != NULL); 4658 4659 th->th.th_first_place = first_place; 4660 th->th.th_last_place = last_place; 4661 th->th.th_new_place = place; 4662 #if OMP_50_ENABLED 4663 if (__kmp_display_affinity && place != th->th.th_current_place && 4664 team->t.t_display_affinity != 1) { 4665 team->t.t_display_affinity = 1; 4666 } 4667 #endif 4668 s_count++; 4669 4670 if ((s_count == S) && rem && (gap_ct == gap)) { 4671 // do nothing, add an extra thread to place on next iteration 4672 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) { 4673 // we added an extra thread to this place; move to next place 4674 if (place == last_place) { 4675 place = first_place; 4676 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4677 place = 0; 4678 } else { 4679 place++; 4680 } 4681 s_count = 0; 4682 gap_ct = 1; 4683 rem--; 4684 } else if (s_count == S) { // place full; don't add extra 4685 if (place == last_place) { 4686 place = first_place; 4687 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4688 place = 0; 4689 } else { 4690 place++; 4691 } 4692 gap_ct++; 4693 s_count = 0; 4694 } 4695 4696 KA_TRACE(100, 4697 ("__kmp_partition_places: close: T#%d(%d:%d) place %d " 4698 "partition = [%d,%d]\n", 4699 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f, 4700 th->th.th_new_place, first_place, last_place)); 4701 } 4702 KMP_DEBUG_ASSERT(place == masters_place); 4703 } 4704 } break; 4705 4706 case proc_bind_spread: { 4707 int f; 4708 int n_th = team->t.t_nproc; 4709 int n_places; 4710 int thidx; 4711 if (first_place <= last_place) { 4712 n_places = last_place - first_place + 1; 4713 } else { 4714 n_places = __kmp_affinity_num_masks - first_place + last_place + 1; 4715 } 4716 if (n_th <= n_places) { 4717 int place = -1; 4718 4719 if (n_places != static_cast<int>(__kmp_affinity_num_masks)) { 4720 int S = n_places / n_th; 4721 int s_count, rem, gap, gap_ct; 4722 4723 place = masters_place; 4724 rem = n_places - n_th * S; 4725 gap = rem ? n_th / rem : 1; 4726 gap_ct = gap; 4727 thidx = n_th; 4728 if (update_master_only == 1) 4729 thidx = 1; 4730 for (f = 0; f < thidx; f++) { 4731 kmp_info_t *th = team->t.t_threads[f]; 4732 KMP_DEBUG_ASSERT(th != NULL); 4733 4734 th->th.th_first_place = place; 4735 th->th.th_new_place = place; 4736 #if OMP_50_ENABLED 4737 if (__kmp_display_affinity && place != th->th.th_current_place && 4738 team->t.t_display_affinity != 1) { 4739 team->t.t_display_affinity = 1; 4740 } 4741 #endif 4742 s_count = 1; 4743 while (s_count < S) { 4744 if (place == last_place) { 4745 place = first_place; 4746 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4747 place = 0; 4748 } else { 4749 place++; 4750 } 4751 s_count++; 4752 } 4753 if (rem && (gap_ct == gap)) { 4754 if (place == last_place) { 4755 place = first_place; 4756 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4757 place = 0; 4758 } else { 4759 place++; 4760 } 4761 rem--; 4762 gap_ct = 0; 4763 } 4764 th->th.th_last_place = place; 4765 gap_ct++; 4766 4767 if (place == last_place) { 4768 place = first_place; 4769 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4770 place = 0; 4771 } else { 4772 place++; 4773 } 4774 4775 KA_TRACE(100, 4776 ("__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4777 "partition = [%d,%d], __kmp_affinity_num_masks: %u\n", 4778 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, 4779 f, th->th.th_new_place, th->th.th_first_place, 4780 th->th.th_last_place, __kmp_affinity_num_masks)); 4781 } 4782 } else { 4783 /* Having uniform space of available computation places I can create 4784 T partitions of round(P/T) size and put threads into the first 4785 place of each partition. */ 4786 double current = static_cast<double>(masters_place); 4787 double spacing = 4788 (static_cast<double>(n_places + 1) / static_cast<double>(n_th)); 4789 int first, last; 4790 kmp_info_t *th; 4791 4792 thidx = n_th + 1; 4793 if (update_master_only == 1) 4794 thidx = 1; 4795 for (f = 0; f < thidx; f++) { 4796 first = static_cast<int>(current); 4797 last = static_cast<int>(current + spacing) - 1; 4798 KMP_DEBUG_ASSERT(last >= first); 4799 if (first >= n_places) { 4800 if (masters_place) { 4801 first -= n_places; 4802 last -= n_places; 4803 if (first == (masters_place + 1)) { 4804 KMP_DEBUG_ASSERT(f == n_th); 4805 first--; 4806 } 4807 if (last == masters_place) { 4808 KMP_DEBUG_ASSERT(f == (n_th - 1)); 4809 last--; 4810 } 4811 } else { 4812 KMP_DEBUG_ASSERT(f == n_th); 4813 first = 0; 4814 last = 0; 4815 } 4816 } 4817 if (last >= n_places) { 4818 last = (n_places - 1); 4819 } 4820 place = first; 4821 current += spacing; 4822 if (f < n_th) { 4823 KMP_DEBUG_ASSERT(0 <= first); 4824 KMP_DEBUG_ASSERT(n_places > first); 4825 KMP_DEBUG_ASSERT(0 <= last); 4826 KMP_DEBUG_ASSERT(n_places > last); 4827 KMP_DEBUG_ASSERT(last_place >= first_place); 4828 th = team->t.t_threads[f]; 4829 KMP_DEBUG_ASSERT(th); 4830 th->th.th_first_place = first; 4831 th->th.th_new_place = place; 4832 th->th.th_last_place = last; 4833 #if OMP_50_ENABLED 4834 if (__kmp_display_affinity && place != th->th.th_current_place && 4835 team->t.t_display_affinity != 1) { 4836 team->t.t_display_affinity = 1; 4837 } 4838 #endif 4839 KA_TRACE(100, 4840 ("__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4841 "partition = [%d,%d], spacing = %.4f\n", 4842 __kmp_gtid_from_thread(team->t.t_threads[f]), 4843 team->t.t_id, f, th->th.th_new_place, 4844 th->th.th_first_place, th->th.th_last_place, spacing)); 4845 } 4846 } 4847 } 4848 KMP_DEBUG_ASSERT(update_master_only || place == masters_place); 4849 } else { 4850 int S, rem, gap, s_count; 4851 S = n_th / n_places; 4852 s_count = 0; 4853 rem = n_th - (S * n_places); 4854 gap = rem > 0 ? n_places / rem : n_places; 4855 int place = masters_place; 4856 int gap_ct = gap; 4857 thidx = n_th; 4858 if (update_master_only == 1) 4859 thidx = 1; 4860 for (f = 0; f < thidx; f++) { 4861 kmp_info_t *th = team->t.t_threads[f]; 4862 KMP_DEBUG_ASSERT(th != NULL); 4863 4864 th->th.th_first_place = place; 4865 th->th.th_last_place = place; 4866 th->th.th_new_place = place; 4867 #if OMP_50_ENABLED 4868 if (__kmp_display_affinity && place != th->th.th_current_place && 4869 team->t.t_display_affinity != 1) { 4870 team->t.t_display_affinity = 1; 4871 } 4872 #endif 4873 s_count++; 4874 4875 if ((s_count == S) && rem && (gap_ct == gap)) { 4876 // do nothing, add an extra thread to place on next iteration 4877 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) { 4878 // we added an extra thread to this place; move on to next place 4879 if (place == last_place) { 4880 place = first_place; 4881 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4882 place = 0; 4883 } else { 4884 place++; 4885 } 4886 s_count = 0; 4887 gap_ct = 1; 4888 rem--; 4889 } else if (s_count == S) { // place is full; don't add extra thread 4890 if (place == last_place) { 4891 place = first_place; 4892 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4893 place = 0; 4894 } else { 4895 place++; 4896 } 4897 gap_ct++; 4898 s_count = 0; 4899 } 4900 4901 KA_TRACE(100, ("__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4902 "partition = [%d,%d]\n", 4903 __kmp_gtid_from_thread(team->t.t_threads[f]), 4904 team->t.t_id, f, th->th.th_new_place, 4905 th->th.th_first_place, th->th.th_last_place)); 4906 } 4907 KMP_DEBUG_ASSERT(update_master_only || place == masters_place); 4908 } 4909 } break; 4910 4911 default: 4912 break; 4913 } 4914 4915 KA_TRACE(20, ("__kmp_partition_places: exit T#%d\n", team->t.t_id)); 4916 } 4917 4918 #endif /* OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED */ 4919 4920 /* allocate a new team data structure to use. take one off of the free pool if 4921 available */ 4922 kmp_team_t * 4923 __kmp_allocate_team(kmp_root_t *root, int new_nproc, int max_nproc, 4924 #if OMPT_SUPPORT 4925 ompt_data_t ompt_parallel_data, 4926 #endif 4927 #if OMP_40_ENABLED 4928 kmp_proc_bind_t new_proc_bind, 4929 #endif 4930 kmp_internal_control_t *new_icvs, 4931 int argc USE_NESTED_HOT_ARG(kmp_info_t *master)) { 4932 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_allocate_team); 4933 int f; 4934 kmp_team_t *team; 4935 int use_hot_team = !root->r.r_active; 4936 int level = 0; 4937 4938 KA_TRACE(20, ("__kmp_allocate_team: called\n")); 4939 KMP_DEBUG_ASSERT(new_nproc >= 1 && argc >= 0); 4940 KMP_DEBUG_ASSERT(max_nproc >= new_nproc); 4941 KMP_MB(); 4942 4943 #if KMP_NESTED_HOT_TEAMS 4944 kmp_hot_team_ptr_t *hot_teams; 4945 if (master) { 4946 team = master->th.th_team; 4947 level = team->t.t_active_level; 4948 if (master->th.th_teams_microtask) { // in teams construct? 4949 if (master->th.th_teams_size.nteams > 1 && 4950 ( // #teams > 1 4951 team->t.t_pkfn == 4952 (microtask_t)__kmp_teams_master || // inner fork of the teams 4953 master->th.th_teams_level < 4954 team->t.t_level)) { // or nested parallel inside the teams 4955 ++level; // not increment if #teams==1, or for outer fork of the teams; 4956 // increment otherwise 4957 } 4958 } 4959 hot_teams = master->th.th_hot_teams; 4960 if (level < __kmp_hot_teams_max_level && hot_teams && 4961 hot_teams[level] 4962 .hot_team) { // hot team has already been allocated for given level 4963 use_hot_team = 1; 4964 } else { 4965 use_hot_team = 0; 4966 } 4967 } 4968 #endif 4969 // Optimization to use a "hot" team 4970 if (use_hot_team && new_nproc > 1) { 4971 KMP_DEBUG_ASSERT(new_nproc <= max_nproc); 4972 #if KMP_NESTED_HOT_TEAMS 4973 team = hot_teams[level].hot_team; 4974 #else 4975 team = root->r.r_hot_team; 4976 #endif 4977 #if KMP_DEBUG 4978 if (__kmp_tasking_mode != tskm_immediate_exec) { 4979 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p " 4980 "task_team[1] = %p before reinit\n", 4981 team->t.t_task_team[0], team->t.t_task_team[1])); 4982 } 4983 #endif 4984 4985 // Has the number of threads changed? 4986 /* Let's assume the most common case is that the number of threads is 4987 unchanged, and put that case first. */ 4988 if (team->t.t_nproc == new_nproc) { // Check changes in number of threads 4989 KA_TRACE(20, ("__kmp_allocate_team: reusing hot team\n")); 4990 // This case can mean that omp_set_num_threads() was called and the hot 4991 // team size was already reduced, so we check the special flag 4992 if (team->t.t_size_changed == -1) { 4993 team->t.t_size_changed = 1; 4994 } else { 4995 KMP_CHECK_UPDATE(team->t.t_size_changed, 0); 4996 } 4997 4998 // TODO???: team->t.t_max_active_levels = new_max_active_levels; 4999 kmp_r_sched_t new_sched = new_icvs->sched; 5000 // set master's schedule as new run-time schedule 5001 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched); 5002 5003 __kmp_reinitialize_team(team, new_icvs, 5004 root->r.r_uber_thread->th.th_ident); 5005 5006 KF_TRACE(10, ("__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n", 0, 5007 team->t.t_threads[0], team)); 5008 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0); 5009 5010 #if OMP_40_ENABLED 5011 #if KMP_AFFINITY_SUPPORTED 5012 if ((team->t.t_size_changed == 0) && 5013 (team->t.t_proc_bind == new_proc_bind)) { 5014 if (new_proc_bind == proc_bind_spread) { 5015 __kmp_partition_places( 5016 team, 1); // add flag to update only master for spread 5017 } 5018 KA_TRACE(200, ("__kmp_allocate_team: reusing hot team #%d bindings: " 5019 "proc_bind = %d, partition = [%d,%d]\n", 5020 team->t.t_id, new_proc_bind, team->t.t_first_place, 5021 team->t.t_last_place)); 5022 } else { 5023 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); 5024 __kmp_partition_places(team); 5025 } 5026 #else 5027 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); 5028 #endif /* KMP_AFFINITY_SUPPORTED */ 5029 #endif /* OMP_40_ENABLED */ 5030 } else if (team->t.t_nproc > new_nproc) { 5031 KA_TRACE(20, 5032 ("__kmp_allocate_team: decreasing hot team thread count to %d\n", 5033 new_nproc)); 5034 5035 team->t.t_size_changed = 1; 5036 #if KMP_NESTED_HOT_TEAMS 5037 if (__kmp_hot_teams_mode == 0) { 5038 // AC: saved number of threads should correspond to team's value in this 5039 // mode, can be bigger in mode 1, when hot team has threads in reserve 5040 KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc); 5041 hot_teams[level].hot_team_nth = new_nproc; 5042 #endif // KMP_NESTED_HOT_TEAMS 5043 /* release the extra threads we don't need any more */ 5044 for (f = new_nproc; f < team->t.t_nproc; f++) { 5045 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 5046 if (__kmp_tasking_mode != tskm_immediate_exec) { 5047 // When decreasing team size, threads no longer in the team should 5048 // unref task team. 5049 team->t.t_threads[f]->th.th_task_team = NULL; 5050 } 5051 __kmp_free_thread(team->t.t_threads[f]); 5052 team->t.t_threads[f] = NULL; 5053 } 5054 #if KMP_NESTED_HOT_TEAMS 5055 } // (__kmp_hot_teams_mode == 0) 5056 else { 5057 // When keeping extra threads in team, switch threads to wait on own 5058 // b_go flag 5059 for (f = new_nproc; f < team->t.t_nproc; ++f) { 5060 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 5061 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar; 5062 for (int b = 0; b < bs_last_barrier; ++b) { 5063 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG) { 5064 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG; 5065 } 5066 KMP_CHECK_UPDATE(balign[b].bb.leaf_kids, 0); 5067 } 5068 } 5069 } 5070 #endif // KMP_NESTED_HOT_TEAMS 5071 team->t.t_nproc = new_nproc; 5072 // TODO???: team->t.t_max_active_levels = new_max_active_levels; 5073 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_icvs->sched.sched); 5074 __kmp_reinitialize_team(team, new_icvs, 5075 root->r.r_uber_thread->th.th_ident); 5076 5077 // Update remaining threads 5078 for (f = 0; f < new_nproc; ++f) { 5079 team->t.t_threads[f]->th.th_team_nproc = new_nproc; 5080 } 5081 5082 // restore the current task state of the master thread: should be the 5083 // implicit task 5084 KF_TRACE(10, ("__kmp_allocate_team: T#%d, this_thread=%p team=%p\n", 0, 5085 team->t.t_threads[0], team)); 5086 5087 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0); 5088 5089 #ifdef KMP_DEBUG 5090 for (f = 0; f < team->t.t_nproc; f++) { 5091 KMP_DEBUG_ASSERT(team->t.t_threads[f] && 5092 team->t.t_threads[f]->th.th_team_nproc == 5093 team->t.t_nproc); 5094 } 5095 #endif 5096 5097 #if OMP_40_ENABLED 5098 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); 5099 #if KMP_AFFINITY_SUPPORTED 5100 __kmp_partition_places(team); 5101 #endif 5102 #endif 5103 } else { // team->t.t_nproc < new_nproc 5104 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 5105 kmp_affin_mask_t *old_mask; 5106 if (KMP_AFFINITY_CAPABLE()) { 5107 KMP_CPU_ALLOC(old_mask); 5108 } 5109 #endif 5110 5111 KA_TRACE(20, 5112 ("__kmp_allocate_team: increasing hot team thread count to %d\n", 5113 new_nproc)); 5114 5115 team->t.t_size_changed = 1; 5116 5117 #if KMP_NESTED_HOT_TEAMS 5118 int avail_threads = hot_teams[level].hot_team_nth; 5119 if (new_nproc < avail_threads) 5120 avail_threads = new_nproc; 5121 kmp_info_t **other_threads = team->t.t_threads; 5122 for (f = team->t.t_nproc; f < avail_threads; ++f) { 5123 // Adjust barrier data of reserved threads (if any) of the team 5124 // Other data will be set in __kmp_initialize_info() below. 5125 int b; 5126 kmp_balign_t *balign = other_threads[f]->th.th_bar; 5127 for (b = 0; b < bs_last_barrier; ++b) { 5128 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 5129 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 5130 #if USE_DEBUGGER 5131 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 5132 #endif 5133 } 5134 } 5135 if (hot_teams[level].hot_team_nth >= new_nproc) { 5136 // we have all needed threads in reserve, no need to allocate any 5137 // this only possible in mode 1, cannot have reserved threads in mode 0 5138 KMP_DEBUG_ASSERT(__kmp_hot_teams_mode == 1); 5139 team->t.t_nproc = new_nproc; // just get reserved threads involved 5140 } else { 5141 // we may have some threads in reserve, but not enough 5142 team->t.t_nproc = 5143 hot_teams[level] 5144 .hot_team_nth; // get reserved threads involved if any 5145 hot_teams[level].hot_team_nth = new_nproc; // adjust hot team max size 5146 #endif // KMP_NESTED_HOT_TEAMS 5147 if (team->t.t_max_nproc < new_nproc) { 5148 /* reallocate larger arrays */ 5149 __kmp_reallocate_team_arrays(team, new_nproc); 5150 __kmp_reinitialize_team(team, new_icvs, NULL); 5151 } 5152 5153 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 5154 /* Temporarily set full mask for master thread before creation of 5155 workers. The reason is that workers inherit the affinity from master, 5156 so if a lot of workers are created on the single core quickly, they 5157 don't get a chance to set their own affinity for a long time. */ 5158 __kmp_set_thread_affinity_mask_full_tmp(old_mask); 5159 #endif 5160 5161 /* allocate new threads for the hot team */ 5162 for (f = team->t.t_nproc; f < new_nproc; f++) { 5163 kmp_info_t *new_worker = __kmp_allocate_thread(root, team, f); 5164 KMP_DEBUG_ASSERT(new_worker); 5165 team->t.t_threads[f] = new_worker; 5166 5167 KA_TRACE(20, 5168 ("__kmp_allocate_team: team %d init T#%d arrived: " 5169 "join=%llu, plain=%llu\n", 5170 team->t.t_id, __kmp_gtid_from_tid(f, team), team->t.t_id, f, 5171 team->t.t_bar[bs_forkjoin_barrier].b_arrived, 5172 team->t.t_bar[bs_plain_barrier].b_arrived)); 5173 5174 { // Initialize barrier data for new threads. 5175 int b; 5176 kmp_balign_t *balign = new_worker->th.th_bar; 5177 for (b = 0; b < bs_last_barrier; ++b) { 5178 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 5179 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != 5180 KMP_BARRIER_PARENT_FLAG); 5181 #if USE_DEBUGGER 5182 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 5183 #endif 5184 } 5185 } 5186 } 5187 5188 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 5189 if (KMP_AFFINITY_CAPABLE()) { 5190 /* Restore initial master thread's affinity mask */ 5191 __kmp_set_system_affinity(old_mask, TRUE); 5192 KMP_CPU_FREE(old_mask); 5193 } 5194 #endif 5195 #if KMP_NESTED_HOT_TEAMS 5196 } // end of check of t_nproc vs. new_nproc vs. hot_team_nth 5197 #endif // KMP_NESTED_HOT_TEAMS 5198 /* make sure everyone is syncronized */ 5199 int old_nproc = team->t.t_nproc; // save old value and use to update only 5200 // new threads below 5201 __kmp_initialize_team(team, new_nproc, new_icvs, 5202 root->r.r_uber_thread->th.th_ident); 5203 5204 /* reinitialize the threads */ 5205 KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc); 5206 for (f = 0; f < team->t.t_nproc; ++f) 5207 __kmp_initialize_info(team->t.t_threads[f], team, f, 5208 __kmp_gtid_from_tid(f, team)); 5209 5210 if (level) { // set th_task_state for new threads in nested hot team 5211 // __kmp_initialize_info() no longer zeroes th_task_state, so we should 5212 // only need to set the th_task_state for the new threads. th_task_state 5213 // for master thread will not be accurate until after this in 5214 // __kmp_fork_call(), so we look to the master's memo_stack to get the 5215 // correct value. 5216 for (f = old_nproc; f < team->t.t_nproc; ++f) 5217 team->t.t_threads[f]->th.th_task_state = 5218 team->t.t_threads[0]->th.th_task_state_memo_stack[level]; 5219 } else { // set th_task_state for new threads in non-nested hot team 5220 int old_state = 5221 team->t.t_threads[0]->th.th_task_state; // copy master's state 5222 for (f = old_nproc; f < team->t.t_nproc; ++f) 5223 team->t.t_threads[f]->th.th_task_state = old_state; 5224 } 5225 5226 #ifdef KMP_DEBUG 5227 for (f = 0; f < team->t.t_nproc; ++f) { 5228 KMP_DEBUG_ASSERT(team->t.t_threads[f] && 5229 team->t.t_threads[f]->th.th_team_nproc == 5230 team->t.t_nproc); 5231 } 5232 #endif 5233 5234 #if OMP_40_ENABLED 5235 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); 5236 #if KMP_AFFINITY_SUPPORTED 5237 __kmp_partition_places(team); 5238 #endif 5239 #endif 5240 } // Check changes in number of threads 5241 5242 #if OMP_40_ENABLED 5243 kmp_info_t *master = team->t.t_threads[0]; 5244 if (master->th.th_teams_microtask) { 5245 for (f = 1; f < new_nproc; ++f) { 5246 // propagate teams construct specific info to workers 5247 kmp_info_t *thr = team->t.t_threads[f]; 5248 thr->th.th_teams_microtask = master->th.th_teams_microtask; 5249 thr->th.th_teams_level = master->th.th_teams_level; 5250 thr->th.th_teams_size = master->th.th_teams_size; 5251 } 5252 } 5253 #endif /* OMP_40_ENABLED */ 5254 #if KMP_NESTED_HOT_TEAMS 5255 if (level) { 5256 // Sync barrier state for nested hot teams, not needed for outermost hot 5257 // team. 5258 for (f = 1; f < new_nproc; ++f) { 5259 kmp_info_t *thr = team->t.t_threads[f]; 5260 int b; 5261 kmp_balign_t *balign = thr->th.th_bar; 5262 for (b = 0; b < bs_last_barrier; ++b) { 5263 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 5264 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 5265 #if USE_DEBUGGER 5266 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 5267 #endif 5268 } 5269 } 5270 } 5271 #endif // KMP_NESTED_HOT_TEAMS 5272 5273 /* reallocate space for arguments if necessary */ 5274 __kmp_alloc_argv_entries(argc, team, TRUE); 5275 KMP_CHECK_UPDATE(team->t.t_argc, argc); 5276 // The hot team re-uses the previous task team, 5277 // if untouched during the previous release->gather phase. 5278 5279 KF_TRACE(10, (" hot_team = %p\n", team)); 5280 5281 #if KMP_DEBUG 5282 if (__kmp_tasking_mode != tskm_immediate_exec) { 5283 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p " 5284 "task_team[1] = %p after reinit\n", 5285 team->t.t_task_team[0], team->t.t_task_team[1])); 5286 } 5287 #endif 5288 5289 #if OMPT_SUPPORT 5290 __ompt_team_assign_id(team, ompt_parallel_data); 5291 #endif 5292 5293 KMP_MB(); 5294 5295 return team; 5296 } 5297 5298 /* next, let's try to take one from the team pool */ 5299 KMP_MB(); 5300 for (team = CCAST(kmp_team_t *, __kmp_team_pool); (team);) { 5301 /* TODO: consider resizing undersized teams instead of reaping them, now 5302 that we have a resizing mechanism */ 5303 if (team->t.t_max_nproc >= max_nproc) { 5304 /* take this team from the team pool */ 5305 __kmp_team_pool = team->t.t_next_pool; 5306 5307 /* setup the team for fresh use */ 5308 __kmp_initialize_team(team, new_nproc, new_icvs, NULL); 5309 5310 KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and " 5311 "task_team[1] %p to NULL\n", 5312 &team->t.t_task_team[0], &team->t.t_task_team[1])); 5313 team->t.t_task_team[0] = NULL; 5314 team->t.t_task_team[1] = NULL; 5315 5316 /* reallocate space for arguments if necessary */ 5317 __kmp_alloc_argv_entries(argc, team, TRUE); 5318 KMP_CHECK_UPDATE(team->t.t_argc, argc); 5319 5320 KA_TRACE( 5321 20, ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n", 5322 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); 5323 { // Initialize barrier data. 5324 int b; 5325 for (b = 0; b < bs_last_barrier; ++b) { 5326 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE; 5327 #if USE_DEBUGGER 5328 team->t.t_bar[b].b_master_arrived = 0; 5329 team->t.t_bar[b].b_team_arrived = 0; 5330 #endif 5331 } 5332 } 5333 5334 #if OMP_40_ENABLED 5335 team->t.t_proc_bind = new_proc_bind; 5336 #endif 5337 5338 KA_TRACE(20, ("__kmp_allocate_team: using team from pool %d.\n", 5339 team->t.t_id)); 5340 5341 #if OMPT_SUPPORT 5342 __ompt_team_assign_id(team, ompt_parallel_data); 5343 #endif 5344 5345 KMP_MB(); 5346 5347 return team; 5348 } 5349 5350 /* reap team if it is too small, then loop back and check the next one */ 5351 // not sure if this is wise, but, will be redone during the hot-teams 5352 // rewrite. 5353 /* TODO: Use technique to find the right size hot-team, don't reap them */ 5354 team = __kmp_reap_team(team); 5355 __kmp_team_pool = team; 5356 } 5357 5358 /* nothing available in the pool, no matter, make a new team! */ 5359 KMP_MB(); 5360 team = (kmp_team_t *)__kmp_allocate(sizeof(kmp_team_t)); 5361 5362 /* and set it up */ 5363 team->t.t_max_nproc = max_nproc; 5364 /* NOTE well, for some reason allocating one big buffer and dividing it up 5365 seems to really hurt performance a lot on the P4, so, let's not use this */ 5366 __kmp_allocate_team_arrays(team, max_nproc); 5367 5368 KA_TRACE(20, ("__kmp_allocate_team: making a new team\n")); 5369 __kmp_initialize_team(team, new_nproc, new_icvs, NULL); 5370 5371 KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and task_team[1] " 5372 "%p to NULL\n", 5373 &team->t.t_task_team[0], &team->t.t_task_team[1])); 5374 team->t.t_task_team[0] = NULL; // to be removed, as __kmp_allocate zeroes 5375 // memory, no need to duplicate 5376 team->t.t_task_team[1] = NULL; // to be removed, as __kmp_allocate zeroes 5377 // memory, no need to duplicate 5378 5379 if (__kmp_storage_map) { 5380 __kmp_print_team_storage_map("team", team, team->t.t_id, new_nproc); 5381 } 5382 5383 /* allocate space for arguments */ 5384 __kmp_alloc_argv_entries(argc, team, FALSE); 5385 team->t.t_argc = argc; 5386 5387 KA_TRACE(20, 5388 ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n", 5389 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); 5390 { // Initialize barrier data. 5391 int b; 5392 for (b = 0; b < bs_last_barrier; ++b) { 5393 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE; 5394 #if USE_DEBUGGER 5395 team->t.t_bar[b].b_master_arrived = 0; 5396 team->t.t_bar[b].b_team_arrived = 0; 5397 #endif 5398 } 5399 } 5400 5401 #if OMP_40_ENABLED 5402 team->t.t_proc_bind = new_proc_bind; 5403 #endif 5404 5405 #if OMPT_SUPPORT 5406 __ompt_team_assign_id(team, ompt_parallel_data); 5407 team->t.ompt_serialized_team_info = NULL; 5408 #endif 5409 5410 KMP_MB(); 5411 5412 KA_TRACE(20, ("__kmp_allocate_team: done creating a new team %d.\n", 5413 team->t.t_id)); 5414 5415 return team; 5416 } 5417 5418 /* TODO implement hot-teams at all levels */ 5419 /* TODO implement lazy thread release on demand (disband request) */ 5420 5421 /* free the team. return it to the team pool. release all the threads 5422 * associated with it */ 5423 void __kmp_free_team(kmp_root_t *root, 5424 kmp_team_t *team USE_NESTED_HOT_ARG(kmp_info_t *master)) { 5425 int f; 5426 KA_TRACE(20, ("__kmp_free_team: T#%d freeing team %d\n", __kmp_get_gtid(), 5427 team->t.t_id)); 5428 5429 /* verify state */ 5430 KMP_DEBUG_ASSERT(root); 5431 KMP_DEBUG_ASSERT(team); 5432 KMP_DEBUG_ASSERT(team->t.t_nproc <= team->t.t_max_nproc); 5433 KMP_DEBUG_ASSERT(team->t.t_threads); 5434 5435 int use_hot_team = team == root->r.r_hot_team; 5436 #if KMP_NESTED_HOT_TEAMS 5437 int level; 5438 kmp_hot_team_ptr_t *hot_teams; 5439 if (master) { 5440 level = team->t.t_active_level - 1; 5441 if (master->th.th_teams_microtask) { // in teams construct? 5442 if (master->th.th_teams_size.nteams > 1) { 5443 ++level; // level was not increased in teams construct for 5444 // team_of_masters 5445 } 5446 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master && 5447 master->th.th_teams_level == team->t.t_level) { 5448 ++level; // level was not increased in teams construct for 5449 // team_of_workers before the parallel 5450 } // team->t.t_level will be increased inside parallel 5451 } 5452 hot_teams = master->th.th_hot_teams; 5453 if (level < __kmp_hot_teams_max_level) { 5454 KMP_DEBUG_ASSERT(team == hot_teams[level].hot_team); 5455 use_hot_team = 1; 5456 } 5457 } 5458 #endif // KMP_NESTED_HOT_TEAMS 5459 5460 /* team is done working */ 5461 TCW_SYNC_PTR(team->t.t_pkfn, 5462 NULL); // Important for Debugging Support Library. 5463 #if KMP_OS_WINDOWS 5464 team->t.t_copyin_counter = 0; // init counter for possible reuse 5465 #endif 5466 // Do not reset pointer to parent team to NULL for hot teams. 5467 5468 /* if we are non-hot team, release our threads */ 5469 if (!use_hot_team) { 5470 if (__kmp_tasking_mode != tskm_immediate_exec) { 5471 // Wait for threads to reach reapable state 5472 for (f = 1; f < team->t.t_nproc; ++f) { 5473 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 5474 kmp_info_t *th = team->t.t_threads[f]; 5475 volatile kmp_uint32 *state = &th->th.th_reap_state; 5476 while (*state != KMP_SAFE_TO_REAP) { 5477 #if KMP_OS_WINDOWS 5478 // On Windows a thread can be killed at any time, check this 5479 DWORD ecode; 5480 if (!__kmp_is_thread_alive(th, &ecode)) { 5481 *state = KMP_SAFE_TO_REAP; // reset the flag for dead thread 5482 break; 5483 } 5484 #endif 5485 // first check if thread is sleeping 5486 kmp_flag_64 fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th); 5487 if (fl.is_sleeping()) 5488 fl.resume(__kmp_gtid_from_thread(th)); 5489 KMP_CPU_PAUSE(); 5490 } 5491 } 5492 5493 // Delete task teams 5494 int tt_idx; 5495 for (tt_idx = 0; tt_idx < 2; ++tt_idx) { 5496 kmp_task_team_t *task_team = team->t.t_task_team[tt_idx]; 5497 if (task_team != NULL) { 5498 for (f = 0; f < team->t.t_nproc; ++f) { // threads unref task teams 5499 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 5500 team->t.t_threads[f]->th.th_task_team = NULL; 5501 } 5502 KA_TRACE( 5503 20, 5504 ("__kmp_free_team: T#%d deactivating task_team %p on team %d\n", 5505 __kmp_get_gtid(), task_team, team->t.t_id)); 5506 #if KMP_NESTED_HOT_TEAMS 5507 __kmp_free_task_team(master, task_team); 5508 #endif 5509 team->t.t_task_team[tt_idx] = NULL; 5510 } 5511 } 5512 } 5513 5514 // Reset pointer to parent team only for non-hot teams. 5515 team->t.t_parent = NULL; 5516 team->t.t_level = 0; 5517 team->t.t_active_level = 0; 5518 5519 /* free the worker threads */ 5520 for (f = 1; f < team->t.t_nproc; ++f) { 5521 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 5522 __kmp_free_thread(team->t.t_threads[f]); 5523 team->t.t_threads[f] = NULL; 5524 } 5525 5526 /* put the team back in the team pool */ 5527 /* TODO limit size of team pool, call reap_team if pool too large */ 5528 team->t.t_next_pool = CCAST(kmp_team_t *, __kmp_team_pool); 5529 __kmp_team_pool = (volatile kmp_team_t *)team; 5530 } else { // Check if team was created for the masters in a teams construct 5531 // See if first worker is a CG root 5532 KMP_DEBUG_ASSERT(team->t.t_threads[1] && 5533 team->t.t_threads[1]->th.th_cg_roots); 5534 if (team->t.t_threads[1]->th.th_cg_roots->cg_root == team->t.t_threads[1]) { 5535 // Clean up the CG root nodes on workers so that this team can be re-used 5536 for (f = 1; f < team->t.t_nproc; ++f) { 5537 kmp_info_t *thr = team->t.t_threads[f]; 5538 KMP_DEBUG_ASSERT(thr && thr->th.th_cg_roots && 5539 thr->th.th_cg_roots->cg_root == thr); 5540 // Pop current CG root off list 5541 kmp_cg_root_t *tmp = thr->th.th_cg_roots; 5542 thr->th.th_cg_roots = tmp->up; 5543 KA_TRACE(100, ("__kmp_free_team: Thread %p popping node %p and moving" 5544 " up to node %p. cg_nthreads was %d\n", 5545 thr, tmp, thr->th.th_cg_roots, tmp->cg_nthreads)); 5546 __kmp_free(tmp); 5547 // Restore current task's thread_limit from CG root 5548 if (thr->th.th_cg_roots) 5549 thr->th.th_current_task->td_icvs.thread_limit = 5550 thr->th.th_cg_roots->cg_thread_limit; 5551 } 5552 } 5553 } 5554 5555 KMP_MB(); 5556 } 5557 5558 /* reap the team. destroy it, reclaim all its resources and free its memory */ 5559 kmp_team_t *__kmp_reap_team(kmp_team_t *team) { 5560 kmp_team_t *next_pool = team->t.t_next_pool; 5561 5562 KMP_DEBUG_ASSERT(team); 5563 KMP_DEBUG_ASSERT(team->t.t_dispatch); 5564 KMP_DEBUG_ASSERT(team->t.t_disp_buffer); 5565 KMP_DEBUG_ASSERT(team->t.t_threads); 5566 KMP_DEBUG_ASSERT(team->t.t_argv); 5567 5568 /* TODO clean the threads that are a part of this? */ 5569 5570 /* free stuff */ 5571 __kmp_free_team_arrays(team); 5572 if (team->t.t_argv != &team->t.t_inline_argv[0]) 5573 __kmp_free((void *)team->t.t_argv); 5574 __kmp_free(team); 5575 5576 KMP_MB(); 5577 return next_pool; 5578 } 5579 5580 // Free the thread. Don't reap it, just place it on the pool of available 5581 // threads. 5582 // 5583 // Changes for Quad issue 527845: We need a predictable OMP tid <-> gtid 5584 // binding for the affinity mechanism to be useful. 5585 // 5586 // Now, we always keep the free list (__kmp_thread_pool) sorted by gtid. 5587 // However, we want to avoid a potential performance problem by always 5588 // scanning through the list to find the correct point at which to insert 5589 // the thread (potential N**2 behavior). To do this we keep track of the 5590 // last place a thread struct was inserted (__kmp_thread_pool_insert_pt). 5591 // With single-level parallelism, threads will always be added to the tail 5592 // of the list, kept track of by __kmp_thread_pool_insert_pt. With nested 5593 // parallelism, all bets are off and we may need to scan through the entire 5594 // free list. 5595 // 5596 // This change also has a potentially large performance benefit, for some 5597 // applications. Previously, as threads were freed from the hot team, they 5598 // would be placed back on the free list in inverse order. If the hot team 5599 // grew back to it's original size, then the freed thread would be placed 5600 // back on the hot team in reverse order. This could cause bad cache 5601 // locality problems on programs where the size of the hot team regularly 5602 // grew and shrunk. 5603 // 5604 // Now, for single-level parallelism, the OMP tid is alway == gtid. 5605 void __kmp_free_thread(kmp_info_t *this_th) { 5606 int gtid; 5607 kmp_info_t **scan; 5608 5609 KA_TRACE(20, ("__kmp_free_thread: T#%d putting T#%d back on free pool.\n", 5610 __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid)); 5611 5612 KMP_DEBUG_ASSERT(this_th); 5613 5614 // When moving thread to pool, switch thread to wait on own b_go flag, and 5615 // uninitialized (NULL team). 5616 int b; 5617 kmp_balign_t *balign = this_th->th.th_bar; 5618 for (b = 0; b < bs_last_barrier; ++b) { 5619 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG) 5620 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG; 5621 balign[b].bb.team = NULL; 5622 balign[b].bb.leaf_kids = 0; 5623 } 5624 this_th->th.th_task_state = 0; 5625 this_th->th.th_reap_state = KMP_SAFE_TO_REAP; 5626 5627 /* put thread back on the free pool */ 5628 TCW_PTR(this_th->th.th_team, NULL); 5629 TCW_PTR(this_th->th.th_root, NULL); 5630 TCW_PTR(this_th->th.th_dispatch, NULL); /* NOT NEEDED */ 5631 5632 while (this_th->th.th_cg_roots) { 5633 this_th->th.th_cg_roots->cg_nthreads--; 5634 KA_TRACE(100, ("__kmp_free_thread: Thread %p decrement cg_nthreads on node" 5635 " %p of thread %p to %d\n", 5636 this_th, this_th->th.th_cg_roots, 5637 this_th->th.th_cg_roots->cg_root, 5638 this_th->th.th_cg_roots->cg_nthreads)); 5639 kmp_cg_root_t *tmp = this_th->th.th_cg_roots; 5640 if (tmp->cg_root == this_th) { // Thread is a cg_root 5641 KMP_DEBUG_ASSERT(tmp->cg_nthreads == 0); 5642 KA_TRACE( 5643 5, ("__kmp_free_thread: Thread %p freeing node %p\n", this_th, tmp)); 5644 this_th->th.th_cg_roots = tmp->up; 5645 __kmp_free(tmp); 5646 } else { // Worker thread 5647 this_th->th.th_cg_roots = NULL; 5648 break; 5649 } 5650 } 5651 5652 /* If the implicit task assigned to this thread can be used by other threads 5653 * -> multiple threads can share the data and try to free the task at 5654 * __kmp_reap_thread at exit. This duplicate use of the task data can happen 5655 * with higher probability when hot team is disabled but can occurs even when 5656 * the hot team is enabled */ 5657 __kmp_free_implicit_task(this_th); 5658 this_th->th.th_current_task = NULL; 5659 5660 // If the __kmp_thread_pool_insert_pt is already past the new insert 5661 // point, then we need to re-scan the entire list. 5662 gtid = this_th->th.th_info.ds.ds_gtid; 5663 if (__kmp_thread_pool_insert_pt != NULL) { 5664 KMP_DEBUG_ASSERT(__kmp_thread_pool != NULL); 5665 if (__kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid) { 5666 __kmp_thread_pool_insert_pt = NULL; 5667 } 5668 } 5669 5670 // Scan down the list to find the place to insert the thread. 5671 // scan is the address of a link in the list, possibly the address of 5672 // __kmp_thread_pool itself. 5673 // 5674 // In the absence of nested parallism, the for loop will have 0 iterations. 5675 if (__kmp_thread_pool_insert_pt != NULL) { 5676 scan = &(__kmp_thread_pool_insert_pt->th.th_next_pool); 5677 } else { 5678 scan = CCAST(kmp_info_t **, &__kmp_thread_pool); 5679 } 5680 for (; (*scan != NULL) && ((*scan)->th.th_info.ds.ds_gtid < gtid); 5681 scan = &((*scan)->th.th_next_pool)) 5682 ; 5683 5684 // Insert the new element on the list, and set __kmp_thread_pool_insert_pt 5685 // to its address. 5686 TCW_PTR(this_th->th.th_next_pool, *scan); 5687 __kmp_thread_pool_insert_pt = *scan = this_th; 5688 KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) || 5689 (this_th->th.th_info.ds.ds_gtid < 5690 this_th->th.th_next_pool->th.th_info.ds.ds_gtid)); 5691 TCW_4(this_th->th.th_in_pool, TRUE); 5692 __kmp_thread_pool_nth++; 5693 5694 TCW_4(__kmp_nth, __kmp_nth - 1); 5695 5696 #ifdef KMP_ADJUST_BLOCKTIME 5697 /* Adjust blocktime back to user setting or default if necessary */ 5698 /* Middle initialization might never have occurred */ 5699 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 5700 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0); 5701 if (__kmp_nth <= __kmp_avail_proc) { 5702 __kmp_zero_bt = FALSE; 5703 } 5704 } 5705 #endif /* KMP_ADJUST_BLOCKTIME */ 5706 5707 KMP_MB(); 5708 } 5709 5710 /* ------------------------------------------------------------------------ */ 5711 5712 void *__kmp_launch_thread(kmp_info_t *this_thr) { 5713 int gtid = this_thr->th.th_info.ds.ds_gtid; 5714 /* void *stack_data;*/ 5715 kmp_team_t *(*volatile pteam); 5716 5717 KMP_MB(); 5718 KA_TRACE(10, ("__kmp_launch_thread: T#%d start\n", gtid)); 5719 5720 if (__kmp_env_consistency_check) { 5721 this_thr->th.th_cons = __kmp_allocate_cons_stack(gtid); // ATT: Memory leak? 5722 } 5723 5724 #if OMPT_SUPPORT 5725 ompt_data_t *thread_data; 5726 if (ompt_enabled.enabled) { 5727 thread_data = &(this_thr->th.ompt_thread_info.thread_data); 5728 *thread_data = ompt_data_none; 5729 5730 this_thr->th.ompt_thread_info.state = ompt_state_overhead; 5731 this_thr->th.ompt_thread_info.wait_id = 0; 5732 this_thr->th.ompt_thread_info.idle_frame = OMPT_GET_FRAME_ADDRESS(0); 5733 if (ompt_enabled.ompt_callback_thread_begin) { 5734 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)( 5735 ompt_thread_worker, thread_data); 5736 } 5737 } 5738 #endif 5739 5740 #if OMPT_SUPPORT 5741 if (ompt_enabled.enabled) { 5742 this_thr->th.ompt_thread_info.state = ompt_state_idle; 5743 } 5744 #endif 5745 /* This is the place where threads wait for work */ 5746 while (!TCR_4(__kmp_global.g.g_done)) { 5747 KMP_DEBUG_ASSERT(this_thr == __kmp_threads[gtid]); 5748 KMP_MB(); 5749 5750 /* wait for work to do */ 5751 KA_TRACE(20, ("__kmp_launch_thread: T#%d waiting for work\n", gtid)); 5752 5753 /* No tid yet since not part of a team */ 5754 __kmp_fork_barrier(gtid, KMP_GTID_DNE); 5755 5756 #if OMPT_SUPPORT 5757 if (ompt_enabled.enabled) { 5758 this_thr->th.ompt_thread_info.state = ompt_state_overhead; 5759 } 5760 #endif 5761 5762 pteam = (kmp_team_t * (*))(&this_thr->th.th_team); 5763 5764 /* have we been allocated? */ 5765 if (TCR_SYNC_PTR(*pteam) && !TCR_4(__kmp_global.g.g_done)) { 5766 /* we were just woken up, so run our new task */ 5767 if (TCR_SYNC_PTR((*pteam)->t.t_pkfn) != NULL) { 5768 int rc; 5769 KA_TRACE(20, 5770 ("__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n", 5771 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), 5772 (*pteam)->t.t_pkfn)); 5773 5774 updateHWFPControl(*pteam); 5775 5776 #if OMPT_SUPPORT 5777 if (ompt_enabled.enabled) { 5778 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel; 5779 } 5780 #endif 5781 5782 rc = (*pteam)->t.t_invoke(gtid); 5783 KMP_ASSERT(rc); 5784 5785 KMP_MB(); 5786 KA_TRACE(20, ("__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n", 5787 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), 5788 (*pteam)->t.t_pkfn)); 5789 } 5790 #if OMPT_SUPPORT 5791 if (ompt_enabled.enabled) { 5792 /* no frame set while outside task */ 5793 __ompt_get_task_info_object(0)->frame.exit_frame = ompt_data_none; 5794 5795 this_thr->th.ompt_thread_info.state = ompt_state_overhead; 5796 } 5797 #endif 5798 /* join barrier after parallel region */ 5799 __kmp_join_barrier(gtid); 5800 } 5801 } 5802 TCR_SYNC_PTR((intptr_t)__kmp_global.g.g_done); 5803 5804 #if OMPT_SUPPORT 5805 if (ompt_enabled.ompt_callback_thread_end) { 5806 ompt_callbacks.ompt_callback(ompt_callback_thread_end)(thread_data); 5807 } 5808 #endif 5809 5810 this_thr->th.th_task_team = NULL; 5811 /* run the destructors for the threadprivate data for this thread */ 5812 __kmp_common_destroy_gtid(gtid); 5813 5814 KA_TRACE(10, ("__kmp_launch_thread: T#%d done\n", gtid)); 5815 KMP_MB(); 5816 return this_thr; 5817 } 5818 5819 /* ------------------------------------------------------------------------ */ 5820 5821 void __kmp_internal_end_dest(void *specific_gtid) { 5822 #if KMP_COMPILER_ICC 5823 #pragma warning(push) 5824 #pragma warning(disable : 810) // conversion from "void *" to "int" may lose 5825 // significant bits 5826 #endif 5827 // Make sure no significant bits are lost 5828 int gtid = (kmp_intptr_t)specific_gtid - 1; 5829 #if KMP_COMPILER_ICC 5830 #pragma warning(pop) 5831 #endif 5832 5833 KA_TRACE(30, ("__kmp_internal_end_dest: T#%d\n", gtid)); 5834 /* NOTE: the gtid is stored as gitd+1 in the thread-local-storage 5835 * this is because 0 is reserved for the nothing-stored case */ 5836 5837 /* josh: One reason for setting the gtid specific data even when it is being 5838 destroyed by pthread is to allow gtid lookup through thread specific data 5839 (__kmp_gtid_get_specific). Some of the code, especially stat code, 5840 that gets executed in the call to __kmp_internal_end_thread, actually 5841 gets the gtid through the thread specific data. Setting it here seems 5842 rather inelegant and perhaps wrong, but allows __kmp_internal_end_thread 5843 to run smoothly. 5844 todo: get rid of this after we remove the dependence on 5845 __kmp_gtid_get_specific */ 5846 if (gtid >= 0 && KMP_UBER_GTID(gtid)) 5847 __kmp_gtid_set_specific(gtid); 5848 #ifdef KMP_TDATA_GTID 5849 __kmp_gtid = gtid; 5850 #endif 5851 __kmp_internal_end_thread(gtid); 5852 } 5853 5854 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB 5855 5856 // 2009-09-08 (lev): It looks the destructor does not work. In simple test cases 5857 // destructors work perfectly, but in real libomp.so I have no evidence it is 5858 // ever called. However, -fini linker option in makefile.mk works fine. 5859 5860 __attribute__((destructor)) void __kmp_internal_end_dtor(void) { 5861 __kmp_internal_end_atexit(); 5862 } 5863 5864 void __kmp_internal_end_fini(void) { __kmp_internal_end_atexit(); } 5865 5866 #endif 5867 5868 /* [Windows] josh: when the atexit handler is called, there may still be more 5869 than one thread alive */ 5870 void __kmp_internal_end_atexit(void) { 5871 KA_TRACE(30, ("__kmp_internal_end_atexit\n")); 5872 /* [Windows] 5873 josh: ideally, we want to completely shutdown the library in this atexit 5874 handler, but stat code that depends on thread specific data for gtid fails 5875 because that data becomes unavailable at some point during the shutdown, so 5876 we call __kmp_internal_end_thread instead. We should eventually remove the 5877 dependency on __kmp_get_specific_gtid in the stat code and use 5878 __kmp_internal_end_library to cleanly shutdown the library. 5879 5880 // TODO: Can some of this comment about GVS be removed? 5881 I suspect that the offending stat code is executed when the calling thread 5882 tries to clean up a dead root thread's data structures, resulting in GVS 5883 code trying to close the GVS structures for that thread, but since the stat 5884 code uses __kmp_get_specific_gtid to get the gtid with the assumption that 5885 the calling thread is cleaning up itself instead of another thread, it get 5886 confused. This happens because allowing a thread to unregister and cleanup 5887 another thread is a recent modification for addressing an issue. 5888 Based on the current design (20050722), a thread may end up 5889 trying to unregister another thread only if thread death does not trigger 5890 the calling of __kmp_internal_end_thread. For Linux* OS, there is the 5891 thread specific data destructor function to detect thread death. For 5892 Windows dynamic, there is DllMain(THREAD_DETACH). For Windows static, there 5893 is nothing. Thus, the workaround is applicable only for Windows static 5894 stat library. */ 5895 __kmp_internal_end_library(-1); 5896 #if KMP_OS_WINDOWS 5897 __kmp_close_console(); 5898 #endif 5899 } 5900 5901 static void __kmp_reap_thread(kmp_info_t *thread, int is_root) { 5902 // It is assumed __kmp_forkjoin_lock is acquired. 5903 5904 int gtid; 5905 5906 KMP_DEBUG_ASSERT(thread != NULL); 5907 5908 gtid = thread->th.th_info.ds.ds_gtid; 5909 5910 if (!is_root) { 5911 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) { 5912 /* Assume the threads are at the fork barrier here */ 5913 KA_TRACE( 5914 20, ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n", 5915 gtid)); 5916 /* Need release fence here to prevent seg faults for tree forkjoin barrier 5917 * (GEH) */ 5918 ANNOTATE_HAPPENS_BEFORE(thread); 5919 kmp_flag_64 flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, thread); 5920 __kmp_release_64(&flag); 5921 } 5922 5923 // Terminate OS thread. 5924 __kmp_reap_worker(thread); 5925 5926 // The thread was killed asynchronously. If it was actively 5927 // spinning in the thread pool, decrement the global count. 5928 // 5929 // There is a small timing hole here - if the worker thread was just waking 5930 // up after sleeping in the pool, had reset it's th_active_in_pool flag but 5931 // not decremented the global counter __kmp_thread_pool_active_nth yet, then 5932 // the global counter might not get updated. 5933 // 5934 // Currently, this can only happen as the library is unloaded, 5935 // so there are no harmful side effects. 5936 if (thread->th.th_active_in_pool) { 5937 thread->th.th_active_in_pool = FALSE; 5938 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth); 5939 KMP_DEBUG_ASSERT(__kmp_thread_pool_active_nth >= 0); 5940 } 5941 5942 // Decrement # of [worker] threads in the pool. 5943 KMP_DEBUG_ASSERT(__kmp_thread_pool_nth > 0); 5944 --__kmp_thread_pool_nth; 5945 } 5946 5947 __kmp_free_implicit_task(thread); 5948 5949 // Free the fast memory for tasking 5950 #if USE_FAST_MEMORY 5951 __kmp_free_fast_memory(thread); 5952 #endif /* USE_FAST_MEMORY */ 5953 5954 __kmp_suspend_uninitialize_thread(thread); 5955 5956 KMP_DEBUG_ASSERT(__kmp_threads[gtid] == thread); 5957 TCW_SYNC_PTR(__kmp_threads[gtid], NULL); 5958 5959 --__kmp_all_nth; 5960 // __kmp_nth was decremented when thread is added to the pool. 5961 5962 #ifdef KMP_ADJUST_BLOCKTIME 5963 /* Adjust blocktime back to user setting or default if necessary */ 5964 /* Middle initialization might never have occurred */ 5965 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 5966 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0); 5967 if (__kmp_nth <= __kmp_avail_proc) { 5968 __kmp_zero_bt = FALSE; 5969 } 5970 } 5971 #endif /* KMP_ADJUST_BLOCKTIME */ 5972 5973 /* free the memory being used */ 5974 if (__kmp_env_consistency_check) { 5975 if (thread->th.th_cons) { 5976 __kmp_free_cons_stack(thread->th.th_cons); 5977 thread->th.th_cons = NULL; 5978 } 5979 } 5980 5981 if (thread->th.th_pri_common != NULL) { 5982 __kmp_free(thread->th.th_pri_common); 5983 thread->th.th_pri_common = NULL; 5984 } 5985 5986 if (thread->th.th_task_state_memo_stack != NULL) { 5987 __kmp_free(thread->th.th_task_state_memo_stack); 5988 thread->th.th_task_state_memo_stack = NULL; 5989 } 5990 5991 #if KMP_USE_BGET 5992 if (thread->th.th_local.bget_data != NULL) { 5993 __kmp_finalize_bget(thread); 5994 } 5995 #endif 5996 5997 #if KMP_AFFINITY_SUPPORTED 5998 if (thread->th.th_affin_mask != NULL) { 5999 KMP_CPU_FREE(thread->th.th_affin_mask); 6000 thread->th.th_affin_mask = NULL; 6001 } 6002 #endif /* KMP_AFFINITY_SUPPORTED */ 6003 6004 #if KMP_USE_HIER_SCHED 6005 if (thread->th.th_hier_bar_data != NULL) { 6006 __kmp_free(thread->th.th_hier_bar_data); 6007 thread->th.th_hier_bar_data = NULL; 6008 } 6009 #endif 6010 6011 __kmp_reap_team(thread->th.th_serial_team); 6012 thread->th.th_serial_team = NULL; 6013 __kmp_free(thread); 6014 6015 KMP_MB(); 6016 6017 } // __kmp_reap_thread 6018 6019 static void __kmp_internal_end(void) { 6020 int i; 6021 6022 /* First, unregister the library */ 6023 __kmp_unregister_library(); 6024 6025 #if KMP_OS_WINDOWS 6026 /* In Win static library, we can't tell when a root actually dies, so we 6027 reclaim the data structures for any root threads that have died but not 6028 unregistered themselves, in order to shut down cleanly. 6029 In Win dynamic library we also can't tell when a thread dies. */ 6030 __kmp_reclaim_dead_roots(); // AC: moved here to always clean resources of 6031 // dead roots 6032 #endif 6033 6034 for (i = 0; i < __kmp_threads_capacity; i++) 6035 if (__kmp_root[i]) 6036 if (__kmp_root[i]->r.r_active) 6037 break; 6038 KMP_MB(); /* Flush all pending memory write invalidates. */ 6039 TCW_SYNC_4(__kmp_global.g.g_done, TRUE); 6040 6041 if (i < __kmp_threads_capacity) { 6042 #if KMP_USE_MONITOR 6043 // 2009-09-08 (lev): Other alive roots found. Why do we kill the monitor?? 6044 KMP_MB(); /* Flush all pending memory write invalidates. */ 6045 6046 // Need to check that monitor was initialized before reaping it. If we are 6047 // called form __kmp_atfork_child (which sets __kmp_init_parallel = 0), then 6048 // __kmp_monitor will appear to contain valid data, but it is only valid in 6049 // the parent process, not the child. 6050 // New behavior (201008): instead of keying off of the flag 6051 // __kmp_init_parallel, the monitor thread creation is keyed off 6052 // of the new flag __kmp_init_monitor. 6053 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock); 6054 if (TCR_4(__kmp_init_monitor)) { 6055 __kmp_reap_monitor(&__kmp_monitor); 6056 TCW_4(__kmp_init_monitor, 0); 6057 } 6058 __kmp_release_bootstrap_lock(&__kmp_monitor_lock); 6059 KA_TRACE(10, ("__kmp_internal_end: monitor reaped\n")); 6060 #endif // KMP_USE_MONITOR 6061 } else { 6062 /* TODO move this to cleanup code */ 6063 #ifdef KMP_DEBUG 6064 /* make sure that everything has properly ended */ 6065 for (i = 0; i < __kmp_threads_capacity; i++) { 6066 if (__kmp_root[i]) { 6067 // KMP_ASSERT( ! KMP_UBER_GTID( i ) ); // AC: 6068 // there can be uber threads alive here 6069 KMP_ASSERT(!__kmp_root[i]->r.r_active); // TODO: can they be active? 6070 } 6071 } 6072 #endif 6073 6074 KMP_MB(); 6075 6076 // Reap the worker threads. 6077 // This is valid for now, but be careful if threads are reaped sooner. 6078 while (__kmp_thread_pool != NULL) { // Loop thru all the thread in the pool. 6079 // Get the next thread from the pool. 6080 kmp_info_t *thread = CCAST(kmp_info_t *, __kmp_thread_pool); 6081 __kmp_thread_pool = thread->th.th_next_pool; 6082 // Reap it. 6083 KMP_DEBUG_ASSERT(thread->th.th_reap_state == KMP_SAFE_TO_REAP); 6084 thread->th.th_next_pool = NULL; 6085 thread->th.th_in_pool = FALSE; 6086 __kmp_reap_thread(thread, 0); 6087 } 6088 __kmp_thread_pool_insert_pt = NULL; 6089 6090 // Reap teams. 6091 while (__kmp_team_pool != NULL) { // Loop thru all the teams in the pool. 6092 // Get the next team from the pool. 6093 kmp_team_t *team = CCAST(kmp_team_t *, __kmp_team_pool); 6094 __kmp_team_pool = team->t.t_next_pool; 6095 // Reap it. 6096 team->t.t_next_pool = NULL; 6097 __kmp_reap_team(team); 6098 } 6099 6100 __kmp_reap_task_teams(); 6101 6102 #if KMP_OS_UNIX 6103 // Threads that are not reaped should not access any resources since they 6104 // are going to be deallocated soon, so the shutdown sequence should wait 6105 // until all threads either exit the final spin-waiting loop or begin 6106 // sleeping after the given blocktime. 6107 for (i = 0; i < __kmp_threads_capacity; i++) { 6108 kmp_info_t *thr = __kmp_threads[i]; 6109 while (thr && KMP_ATOMIC_LD_ACQ(&thr->th.th_blocking)) 6110 KMP_CPU_PAUSE(); 6111 } 6112 #endif 6113 6114 for (i = 0; i < __kmp_threads_capacity; ++i) { 6115 // TBD: Add some checking... 6116 // Something like KMP_DEBUG_ASSERT( __kmp_thread[ i ] == NULL ); 6117 } 6118 6119 /* Make sure all threadprivate destructors get run by joining with all 6120 worker threads before resetting this flag */ 6121 TCW_SYNC_4(__kmp_init_common, FALSE); 6122 6123 KA_TRACE(10, ("__kmp_internal_end: all workers reaped\n")); 6124 KMP_MB(); 6125 6126 #if KMP_USE_MONITOR 6127 // See note above: One of the possible fixes for CQ138434 / CQ140126 6128 // 6129 // FIXME: push both code fragments down and CSE them? 6130 // push them into __kmp_cleanup() ? 6131 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock); 6132 if (TCR_4(__kmp_init_monitor)) { 6133 __kmp_reap_monitor(&__kmp_monitor); 6134 TCW_4(__kmp_init_monitor, 0); 6135 } 6136 __kmp_release_bootstrap_lock(&__kmp_monitor_lock); 6137 KA_TRACE(10, ("__kmp_internal_end: monitor reaped\n")); 6138 #endif 6139 } /* else !__kmp_global.t_active */ 6140 TCW_4(__kmp_init_gtid, FALSE); 6141 KMP_MB(); /* Flush all pending memory write invalidates. */ 6142 6143 __kmp_cleanup(); 6144 #if OMPT_SUPPORT 6145 ompt_fini(); 6146 #endif 6147 } 6148 6149 void __kmp_internal_end_library(int gtid_req) { 6150 /* if we have already cleaned up, don't try again, it wouldn't be pretty */ 6151 /* this shouldn't be a race condition because __kmp_internal_end() is the 6152 only place to clear __kmp_serial_init */ 6153 /* we'll check this later too, after we get the lock */ 6154 // 2009-09-06: We do not set g_abort without setting g_done. This check looks 6155 // redundaant, because the next check will work in any case. 6156 if (__kmp_global.g.g_abort) { 6157 KA_TRACE(11, ("__kmp_internal_end_library: abort, exiting\n")); 6158 /* TODO abort? */ 6159 return; 6160 } 6161 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 6162 KA_TRACE(10, ("__kmp_internal_end_library: already finished\n")); 6163 return; 6164 } 6165 6166 KMP_MB(); /* Flush all pending memory write invalidates. */ 6167 6168 /* find out who we are and what we should do */ 6169 { 6170 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific(); 6171 KA_TRACE( 6172 10, ("__kmp_internal_end_library: enter T#%d (%d)\n", gtid, gtid_req)); 6173 if (gtid == KMP_GTID_SHUTDOWN) { 6174 KA_TRACE(10, ("__kmp_internal_end_library: !__kmp_init_runtime, system " 6175 "already shutdown\n")); 6176 return; 6177 } else if (gtid == KMP_GTID_MONITOR) { 6178 KA_TRACE(10, ("__kmp_internal_end_library: monitor thread, gtid not " 6179 "registered, or system shutdown\n")); 6180 return; 6181 } else if (gtid == KMP_GTID_DNE) { 6182 KA_TRACE(10, ("__kmp_internal_end_library: gtid not registered or system " 6183 "shutdown\n")); 6184 /* we don't know who we are, but we may still shutdown the library */ 6185 } else if (KMP_UBER_GTID(gtid)) { 6186 /* unregister ourselves as an uber thread. gtid is no longer valid */ 6187 if (__kmp_root[gtid]->r.r_active) { 6188 __kmp_global.g.g_abort = -1; 6189 TCW_SYNC_4(__kmp_global.g.g_done, TRUE); 6190 KA_TRACE(10, 6191 ("__kmp_internal_end_library: root still active, abort T#%d\n", 6192 gtid)); 6193 return; 6194 } else { 6195 KA_TRACE( 6196 10, 6197 ("__kmp_internal_end_library: unregistering sibling T#%d\n", gtid)); 6198 __kmp_unregister_root_current_thread(gtid); 6199 } 6200 } else { 6201 /* worker threads may call this function through the atexit handler, if they 6202 * call exit() */ 6203 /* For now, skip the usual subsequent processing and just dump the debug buffer. 6204 TODO: do a thorough shutdown instead */ 6205 #ifdef DUMP_DEBUG_ON_EXIT 6206 if (__kmp_debug_buf) 6207 __kmp_dump_debug_buffer(); 6208 #endif 6209 return; 6210 } 6211 } 6212 /* synchronize the termination process */ 6213 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6214 6215 /* have we already finished */ 6216 if (__kmp_global.g.g_abort) { 6217 KA_TRACE(10, ("__kmp_internal_end_library: abort, exiting\n")); 6218 /* TODO abort? */ 6219 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6220 return; 6221 } 6222 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 6223 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6224 return; 6225 } 6226 6227 /* We need this lock to enforce mutex between this reading of 6228 __kmp_threads_capacity and the writing by __kmp_register_root. 6229 Alternatively, we can use a counter of roots that is atomically updated by 6230 __kmp_get_global_thread_id_reg, __kmp_do_serial_initialize and 6231 __kmp_internal_end_*. */ 6232 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 6233 6234 /* now we can safely conduct the actual termination */ 6235 __kmp_internal_end(); 6236 6237 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 6238 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6239 6240 KA_TRACE(10, ("__kmp_internal_end_library: exit\n")); 6241 6242 #ifdef DUMP_DEBUG_ON_EXIT 6243 if (__kmp_debug_buf) 6244 __kmp_dump_debug_buffer(); 6245 #endif 6246 6247 #if KMP_OS_WINDOWS 6248 __kmp_close_console(); 6249 #endif 6250 6251 __kmp_fini_allocator(); 6252 6253 } // __kmp_internal_end_library 6254 6255 void __kmp_internal_end_thread(int gtid_req) { 6256 int i; 6257 6258 /* if we have already cleaned up, don't try again, it wouldn't be pretty */ 6259 /* this shouldn't be a race condition because __kmp_internal_end() is the 6260 * only place to clear __kmp_serial_init */ 6261 /* we'll check this later too, after we get the lock */ 6262 // 2009-09-06: We do not set g_abort without setting g_done. This check looks 6263 // redundant, because the next check will work in any case. 6264 if (__kmp_global.g.g_abort) { 6265 KA_TRACE(11, ("__kmp_internal_end_thread: abort, exiting\n")); 6266 /* TODO abort? */ 6267 return; 6268 } 6269 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 6270 KA_TRACE(10, ("__kmp_internal_end_thread: already finished\n")); 6271 return; 6272 } 6273 6274 KMP_MB(); /* Flush all pending memory write invalidates. */ 6275 6276 /* find out who we are and what we should do */ 6277 { 6278 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific(); 6279 KA_TRACE(10, 6280 ("__kmp_internal_end_thread: enter T#%d (%d)\n", gtid, gtid_req)); 6281 if (gtid == KMP_GTID_SHUTDOWN) { 6282 KA_TRACE(10, ("__kmp_internal_end_thread: !__kmp_init_runtime, system " 6283 "already shutdown\n")); 6284 return; 6285 } else if (gtid == KMP_GTID_MONITOR) { 6286 KA_TRACE(10, ("__kmp_internal_end_thread: monitor thread, gtid not " 6287 "registered, or system shutdown\n")); 6288 return; 6289 } else if (gtid == KMP_GTID_DNE) { 6290 KA_TRACE(10, ("__kmp_internal_end_thread: gtid not registered or system " 6291 "shutdown\n")); 6292 return; 6293 /* we don't know who we are */ 6294 } else if (KMP_UBER_GTID(gtid)) { 6295 /* unregister ourselves as an uber thread. gtid is no longer valid */ 6296 if (__kmp_root[gtid]->r.r_active) { 6297 __kmp_global.g.g_abort = -1; 6298 TCW_SYNC_4(__kmp_global.g.g_done, TRUE); 6299 KA_TRACE(10, 6300 ("__kmp_internal_end_thread: root still active, abort T#%d\n", 6301 gtid)); 6302 return; 6303 } else { 6304 KA_TRACE(10, ("__kmp_internal_end_thread: unregistering sibling T#%d\n", 6305 gtid)); 6306 __kmp_unregister_root_current_thread(gtid); 6307 } 6308 } else { 6309 /* just a worker thread, let's leave */ 6310 KA_TRACE(10, ("__kmp_internal_end_thread: worker thread T#%d\n", gtid)); 6311 6312 if (gtid >= 0) { 6313 __kmp_threads[gtid]->th.th_task_team = NULL; 6314 } 6315 6316 KA_TRACE(10, 6317 ("__kmp_internal_end_thread: worker thread done, exiting T#%d\n", 6318 gtid)); 6319 return; 6320 } 6321 } 6322 #if KMP_DYNAMIC_LIB 6323 #if OMP_50_ENABLED 6324 if (__kmp_pause_status != kmp_hard_paused) 6325 #endif 6326 // AC: lets not shutdown the dynamic library at the exit of uber thread, 6327 // because we will better shutdown later in the library destructor. 6328 { 6329 KA_TRACE(10, ("__kmp_internal_end_thread: exiting T#%d\n", gtid_req)); 6330 return; 6331 } 6332 #endif 6333 /* synchronize the termination process */ 6334 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6335 6336 /* have we already finished */ 6337 if (__kmp_global.g.g_abort) { 6338 KA_TRACE(10, ("__kmp_internal_end_thread: abort, exiting\n")); 6339 /* TODO abort? */ 6340 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6341 return; 6342 } 6343 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 6344 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6345 return; 6346 } 6347 6348 /* We need this lock to enforce mutex between this reading of 6349 __kmp_threads_capacity and the writing by __kmp_register_root. 6350 Alternatively, we can use a counter of roots that is atomically updated by 6351 __kmp_get_global_thread_id_reg, __kmp_do_serial_initialize and 6352 __kmp_internal_end_*. */ 6353 6354 /* should we finish the run-time? are all siblings done? */ 6355 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 6356 6357 for (i = 0; i < __kmp_threads_capacity; ++i) { 6358 if (KMP_UBER_GTID(i)) { 6359 KA_TRACE( 6360 10, 6361 ("__kmp_internal_end_thread: remaining sibling task: gtid==%d\n", i)); 6362 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 6363 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6364 return; 6365 } 6366 } 6367 6368 /* now we can safely conduct the actual termination */ 6369 6370 __kmp_internal_end(); 6371 6372 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 6373 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6374 6375 KA_TRACE(10, ("__kmp_internal_end_thread: exit T#%d\n", gtid_req)); 6376 6377 #ifdef DUMP_DEBUG_ON_EXIT 6378 if (__kmp_debug_buf) 6379 __kmp_dump_debug_buffer(); 6380 #endif 6381 } // __kmp_internal_end_thread 6382 6383 // ----------------------------------------------------------------------------- 6384 // Library registration stuff. 6385 6386 static long __kmp_registration_flag = 0; 6387 // Random value used to indicate library initialization. 6388 static char *__kmp_registration_str = NULL; 6389 // Value to be saved in env var __KMP_REGISTERED_LIB_<pid>. 6390 6391 static inline char *__kmp_reg_status_name() { 6392 /* On RHEL 3u5 if linked statically, getpid() returns different values in 6393 each thread. If registration and unregistration go in different threads 6394 (omp_misc_other_root_exit.cpp test case), the name of registered_lib_env 6395 env var can not be found, because the name will contain different pid. */ 6396 return __kmp_str_format("__KMP_REGISTERED_LIB_%d", (int)getpid()); 6397 } // __kmp_reg_status_get 6398 6399 void __kmp_register_library_startup(void) { 6400 6401 char *name = __kmp_reg_status_name(); // Name of the environment variable. 6402 int done = 0; 6403 union { 6404 double dtime; 6405 long ltime; 6406 } time; 6407 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 6408 __kmp_initialize_system_tick(); 6409 #endif 6410 __kmp_read_system_time(&time.dtime); 6411 __kmp_registration_flag = 0xCAFE0000L | (time.ltime & 0x0000FFFFL); 6412 __kmp_registration_str = 6413 __kmp_str_format("%p-%lx-%s", &__kmp_registration_flag, 6414 __kmp_registration_flag, KMP_LIBRARY_FILE); 6415 6416 KA_TRACE(50, ("__kmp_register_library_startup: %s=\"%s\"\n", name, 6417 __kmp_registration_str)); 6418 6419 while (!done) { 6420 6421 char *value = NULL; // Actual value of the environment variable. 6422 6423 // Set environment variable, but do not overwrite if it is exist. 6424 __kmp_env_set(name, __kmp_registration_str, 0); 6425 // Check the variable is written. 6426 value = __kmp_env_get(name); 6427 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) { 6428 6429 done = 1; // Ok, environment variable set successfully, exit the loop. 6430 6431 } else { 6432 6433 // Oops. Write failed. Another copy of OpenMP RTL is in memory. 6434 // Check whether it alive or dead. 6435 int neighbor = 0; // 0 -- unknown status, 1 -- alive, 2 -- dead. 6436 char *tail = value; 6437 char *flag_addr_str = NULL; 6438 char *flag_val_str = NULL; 6439 char const *file_name = NULL; 6440 __kmp_str_split(tail, '-', &flag_addr_str, &tail); 6441 __kmp_str_split(tail, '-', &flag_val_str, &tail); 6442 file_name = tail; 6443 if (tail != NULL) { 6444 long *flag_addr = 0; 6445 long flag_val = 0; 6446 KMP_SSCANF(flag_addr_str, "%p", RCAST(void**, &flag_addr)); 6447 KMP_SSCANF(flag_val_str, "%lx", &flag_val); 6448 if (flag_addr != 0 && flag_val != 0 && strcmp(file_name, "") != 0) { 6449 // First, check whether environment-encoded address is mapped into 6450 // addr space. 6451 // If so, dereference it to see if it still has the right value. 6452 if (__kmp_is_address_mapped(flag_addr) && *flag_addr == flag_val) { 6453 neighbor = 1; 6454 } else { 6455 // If not, then we know the other copy of the library is no longer 6456 // running. 6457 neighbor = 2; 6458 } 6459 } 6460 } 6461 switch (neighbor) { 6462 case 0: // Cannot parse environment variable -- neighbor status unknown. 6463 // Assume it is the incompatible format of future version of the 6464 // library. Assume the other library is alive. 6465 // WARN( ... ); // TODO: Issue a warning. 6466 file_name = "unknown library"; 6467 KMP_FALLTHROUGH(); 6468 // Attention! Falling to the next case. That's intentional. 6469 case 1: { // Neighbor is alive. 6470 // Check it is allowed. 6471 char *duplicate_ok = __kmp_env_get("KMP_DUPLICATE_LIB_OK"); 6472 if (!__kmp_str_match_true(duplicate_ok)) { 6473 // That's not allowed. Issue fatal error. 6474 __kmp_fatal(KMP_MSG(DuplicateLibrary, KMP_LIBRARY_FILE, file_name), 6475 KMP_HNT(DuplicateLibrary), __kmp_msg_null); 6476 } 6477 KMP_INTERNAL_FREE(duplicate_ok); 6478 __kmp_duplicate_library_ok = 1; 6479 done = 1; // Exit the loop. 6480 } break; 6481 case 2: { // Neighbor is dead. 6482 // Clear the variable and try to register library again. 6483 __kmp_env_unset(name); 6484 } break; 6485 default: { KMP_DEBUG_ASSERT(0); } break; 6486 } 6487 } 6488 KMP_INTERNAL_FREE((void *)value); 6489 } 6490 KMP_INTERNAL_FREE((void *)name); 6491 6492 } // func __kmp_register_library_startup 6493 6494 void __kmp_unregister_library(void) { 6495 6496 char *name = __kmp_reg_status_name(); 6497 char *value = __kmp_env_get(name); 6498 6499 KMP_DEBUG_ASSERT(__kmp_registration_flag != 0); 6500 KMP_DEBUG_ASSERT(__kmp_registration_str != NULL); 6501 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) { 6502 // Ok, this is our variable. Delete it. 6503 __kmp_env_unset(name); 6504 } 6505 6506 KMP_INTERNAL_FREE(__kmp_registration_str); 6507 KMP_INTERNAL_FREE(value); 6508 KMP_INTERNAL_FREE(name); 6509 6510 __kmp_registration_flag = 0; 6511 __kmp_registration_str = NULL; 6512 6513 } // __kmp_unregister_library 6514 6515 // End of Library registration stuff. 6516 // ----------------------------------------------------------------------------- 6517 6518 #if KMP_MIC_SUPPORTED 6519 6520 static void __kmp_check_mic_type() { 6521 kmp_cpuid_t cpuid_state = {0}; 6522 kmp_cpuid_t *cs_p = &cpuid_state; 6523 __kmp_x86_cpuid(1, 0, cs_p); 6524 // We don't support mic1 at the moment 6525 if ((cs_p->eax & 0xff0) == 0xB10) { 6526 __kmp_mic_type = mic2; 6527 } else if ((cs_p->eax & 0xf0ff0) == 0x50670) { 6528 __kmp_mic_type = mic3; 6529 } else { 6530 __kmp_mic_type = non_mic; 6531 } 6532 } 6533 6534 #endif /* KMP_MIC_SUPPORTED */ 6535 6536 static void __kmp_do_serial_initialize(void) { 6537 int i, gtid; 6538 int size; 6539 6540 KA_TRACE(10, ("__kmp_do_serial_initialize: enter\n")); 6541 6542 KMP_DEBUG_ASSERT(sizeof(kmp_int32) == 4); 6543 KMP_DEBUG_ASSERT(sizeof(kmp_uint32) == 4); 6544 KMP_DEBUG_ASSERT(sizeof(kmp_int64) == 8); 6545 KMP_DEBUG_ASSERT(sizeof(kmp_uint64) == 8); 6546 KMP_DEBUG_ASSERT(sizeof(kmp_intptr_t) == sizeof(void *)); 6547 6548 #if OMPT_SUPPORT 6549 ompt_pre_init(); 6550 #endif 6551 6552 __kmp_validate_locks(); 6553 6554 /* Initialize internal memory allocator */ 6555 __kmp_init_allocator(); 6556 6557 /* Register the library startup via an environment variable and check to see 6558 whether another copy of the library is already registered. */ 6559 6560 __kmp_register_library_startup(); 6561 6562 /* TODO reinitialization of library */ 6563 if (TCR_4(__kmp_global.g.g_done)) { 6564 KA_TRACE(10, ("__kmp_do_serial_initialize: reinitialization of library\n")); 6565 } 6566 6567 __kmp_global.g.g_abort = 0; 6568 TCW_SYNC_4(__kmp_global.g.g_done, FALSE); 6569 6570 /* initialize the locks */ 6571 #if KMP_USE_ADAPTIVE_LOCKS 6572 #if KMP_DEBUG_ADAPTIVE_LOCKS 6573 __kmp_init_speculative_stats(); 6574 #endif 6575 #endif 6576 #if KMP_STATS_ENABLED 6577 __kmp_stats_init(); 6578 #endif 6579 __kmp_init_lock(&__kmp_global_lock); 6580 __kmp_init_queuing_lock(&__kmp_dispatch_lock); 6581 __kmp_init_lock(&__kmp_debug_lock); 6582 __kmp_init_atomic_lock(&__kmp_atomic_lock); 6583 __kmp_init_atomic_lock(&__kmp_atomic_lock_1i); 6584 __kmp_init_atomic_lock(&__kmp_atomic_lock_2i); 6585 __kmp_init_atomic_lock(&__kmp_atomic_lock_4i); 6586 __kmp_init_atomic_lock(&__kmp_atomic_lock_4r); 6587 __kmp_init_atomic_lock(&__kmp_atomic_lock_8i); 6588 __kmp_init_atomic_lock(&__kmp_atomic_lock_8r); 6589 __kmp_init_atomic_lock(&__kmp_atomic_lock_8c); 6590 __kmp_init_atomic_lock(&__kmp_atomic_lock_10r); 6591 __kmp_init_atomic_lock(&__kmp_atomic_lock_16r); 6592 __kmp_init_atomic_lock(&__kmp_atomic_lock_16c); 6593 __kmp_init_atomic_lock(&__kmp_atomic_lock_20c); 6594 __kmp_init_atomic_lock(&__kmp_atomic_lock_32c); 6595 __kmp_init_bootstrap_lock(&__kmp_forkjoin_lock); 6596 __kmp_init_bootstrap_lock(&__kmp_exit_lock); 6597 #if KMP_USE_MONITOR 6598 __kmp_init_bootstrap_lock(&__kmp_monitor_lock); 6599 #endif 6600 __kmp_init_bootstrap_lock(&__kmp_tp_cached_lock); 6601 6602 /* conduct initialization and initial setup of configuration */ 6603 6604 __kmp_runtime_initialize(); 6605 6606 #if KMP_MIC_SUPPORTED 6607 __kmp_check_mic_type(); 6608 #endif 6609 6610 // Some global variable initialization moved here from kmp_env_initialize() 6611 #ifdef KMP_DEBUG 6612 kmp_diag = 0; 6613 #endif 6614 __kmp_abort_delay = 0; 6615 6616 // From __kmp_init_dflt_team_nth() 6617 /* assume the entire machine will be used */ 6618 __kmp_dflt_team_nth_ub = __kmp_xproc; 6619 if (__kmp_dflt_team_nth_ub < KMP_MIN_NTH) { 6620 __kmp_dflt_team_nth_ub = KMP_MIN_NTH; 6621 } 6622 if (__kmp_dflt_team_nth_ub > __kmp_sys_max_nth) { 6623 __kmp_dflt_team_nth_ub = __kmp_sys_max_nth; 6624 } 6625 __kmp_max_nth = __kmp_sys_max_nth; 6626 __kmp_cg_max_nth = __kmp_sys_max_nth; 6627 __kmp_teams_max_nth = __kmp_xproc; // set a "reasonable" default 6628 if (__kmp_teams_max_nth > __kmp_sys_max_nth) { 6629 __kmp_teams_max_nth = __kmp_sys_max_nth; 6630 } 6631 6632 // Three vars below moved here from __kmp_env_initialize() "KMP_BLOCKTIME" 6633 // part 6634 __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME; 6635 #if KMP_USE_MONITOR 6636 __kmp_monitor_wakeups = 6637 KMP_WAKEUPS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups); 6638 __kmp_bt_intervals = 6639 KMP_INTERVALS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups); 6640 #endif 6641 // From "KMP_LIBRARY" part of __kmp_env_initialize() 6642 __kmp_library = library_throughput; 6643 // From KMP_SCHEDULE initialization 6644 __kmp_static = kmp_sch_static_balanced; 6645 // AC: do not use analytical here, because it is non-monotonous 6646 //__kmp_guided = kmp_sch_guided_iterative_chunked; 6647 //__kmp_auto = kmp_sch_guided_analytical_chunked; // AC: it is the default, no 6648 // need to repeat assignment 6649 // Barrier initialization. Moved here from __kmp_env_initialize() Barrier branch 6650 // bit control and barrier method control parts 6651 #if KMP_FAST_REDUCTION_BARRIER 6652 #define kmp_reduction_barrier_gather_bb ((int)1) 6653 #define kmp_reduction_barrier_release_bb ((int)1) 6654 #define kmp_reduction_barrier_gather_pat bp_hyper_bar 6655 #define kmp_reduction_barrier_release_pat bp_hyper_bar 6656 #endif // KMP_FAST_REDUCTION_BARRIER 6657 for (i = bs_plain_barrier; i < bs_last_barrier; i++) { 6658 __kmp_barrier_gather_branch_bits[i] = __kmp_barrier_gather_bb_dflt; 6659 __kmp_barrier_release_branch_bits[i] = __kmp_barrier_release_bb_dflt; 6660 __kmp_barrier_gather_pattern[i] = __kmp_barrier_gather_pat_dflt; 6661 __kmp_barrier_release_pattern[i] = __kmp_barrier_release_pat_dflt; 6662 #if KMP_FAST_REDUCTION_BARRIER 6663 if (i == bs_reduction_barrier) { // tested and confirmed on ALTIX only ( 6664 // lin_64 ): hyper,1 6665 __kmp_barrier_gather_branch_bits[i] = kmp_reduction_barrier_gather_bb; 6666 __kmp_barrier_release_branch_bits[i] = kmp_reduction_barrier_release_bb; 6667 __kmp_barrier_gather_pattern[i] = kmp_reduction_barrier_gather_pat; 6668 __kmp_barrier_release_pattern[i] = kmp_reduction_barrier_release_pat; 6669 } 6670 #endif // KMP_FAST_REDUCTION_BARRIER 6671 } 6672 #if KMP_FAST_REDUCTION_BARRIER 6673 #undef kmp_reduction_barrier_release_pat 6674 #undef kmp_reduction_barrier_gather_pat 6675 #undef kmp_reduction_barrier_release_bb 6676 #undef kmp_reduction_barrier_gather_bb 6677 #endif // KMP_FAST_REDUCTION_BARRIER 6678 #if KMP_MIC_SUPPORTED 6679 if (__kmp_mic_type == mic2) { // KNC 6680 // AC: plane=3,2, forkjoin=2,1 are optimal for 240 threads on KNC 6681 __kmp_barrier_gather_branch_bits[bs_plain_barrier] = 3; // plain gather 6682 __kmp_barrier_release_branch_bits[bs_forkjoin_barrier] = 6683 1; // forkjoin release 6684 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar; 6685 __kmp_barrier_release_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar; 6686 } 6687 #if KMP_FAST_REDUCTION_BARRIER 6688 if (__kmp_mic_type == mic2) { // KNC 6689 __kmp_barrier_gather_pattern[bs_reduction_barrier] = bp_hierarchical_bar; 6690 __kmp_barrier_release_pattern[bs_reduction_barrier] = bp_hierarchical_bar; 6691 } 6692 #endif // KMP_FAST_REDUCTION_BARRIER 6693 #endif // KMP_MIC_SUPPORTED 6694 6695 // From KMP_CHECKS initialization 6696 #ifdef KMP_DEBUG 6697 __kmp_env_checks = TRUE; /* development versions have the extra checks */ 6698 #else 6699 __kmp_env_checks = FALSE; /* port versions do not have the extra checks */ 6700 #endif 6701 6702 // From "KMP_FOREIGN_THREADS_THREADPRIVATE" initialization 6703 __kmp_foreign_tp = TRUE; 6704 6705 __kmp_global.g.g_dynamic = FALSE; 6706 __kmp_global.g.g_dynamic_mode = dynamic_default; 6707 6708 __kmp_env_initialize(NULL); 6709 6710 // Print all messages in message catalog for testing purposes. 6711 #ifdef KMP_DEBUG 6712 char const *val = __kmp_env_get("KMP_DUMP_CATALOG"); 6713 if (__kmp_str_match_true(val)) { 6714 kmp_str_buf_t buffer; 6715 __kmp_str_buf_init(&buffer); 6716 __kmp_i18n_dump_catalog(&buffer); 6717 __kmp_printf("%s", buffer.str); 6718 __kmp_str_buf_free(&buffer); 6719 } 6720 __kmp_env_free(&val); 6721 #endif 6722 6723 __kmp_threads_capacity = 6724 __kmp_initial_threads_capacity(__kmp_dflt_team_nth_ub); 6725 // Moved here from __kmp_env_initialize() "KMP_ALL_THREADPRIVATE" part 6726 __kmp_tp_capacity = __kmp_default_tp_capacity( 6727 __kmp_dflt_team_nth_ub, __kmp_max_nth, __kmp_allThreadsSpecified); 6728 6729 // If the library is shut down properly, both pools must be NULL. Just in 6730 // case, set them to NULL -- some memory may leak, but subsequent code will 6731 // work even if pools are not freed. 6732 KMP_DEBUG_ASSERT(__kmp_thread_pool == NULL); 6733 KMP_DEBUG_ASSERT(__kmp_thread_pool_insert_pt == NULL); 6734 KMP_DEBUG_ASSERT(__kmp_team_pool == NULL); 6735 __kmp_thread_pool = NULL; 6736 __kmp_thread_pool_insert_pt = NULL; 6737 __kmp_team_pool = NULL; 6738 6739 /* Allocate all of the variable sized records */ 6740 /* NOTE: __kmp_threads_capacity entries are allocated, but the arrays are 6741 * expandable */ 6742 /* Since allocation is cache-aligned, just add extra padding at the end */ 6743 size = 6744 (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * __kmp_threads_capacity + 6745 CACHE_LINE; 6746 __kmp_threads = (kmp_info_t **)__kmp_allocate(size); 6747 __kmp_root = (kmp_root_t **)((char *)__kmp_threads + 6748 sizeof(kmp_info_t *) * __kmp_threads_capacity); 6749 6750 /* init thread counts */ 6751 KMP_DEBUG_ASSERT(__kmp_all_nth == 6752 0); // Asserts fail if the library is reinitializing and 6753 KMP_DEBUG_ASSERT(__kmp_nth == 0); // something was wrong in termination. 6754 __kmp_all_nth = 0; 6755 __kmp_nth = 0; 6756 6757 /* setup the uber master thread and hierarchy */ 6758 gtid = __kmp_register_root(TRUE); 6759 KA_TRACE(10, ("__kmp_do_serial_initialize T#%d\n", gtid)); 6760 KMP_ASSERT(KMP_UBER_GTID(gtid)); 6761 KMP_ASSERT(KMP_INITIAL_GTID(gtid)); 6762 6763 KMP_MB(); /* Flush all pending memory write invalidates. */ 6764 6765 __kmp_common_initialize(); 6766 6767 #if KMP_OS_UNIX 6768 /* invoke the child fork handler */ 6769 __kmp_register_atfork(); 6770 #endif 6771 6772 #if !KMP_DYNAMIC_LIB 6773 { 6774 /* Invoke the exit handler when the program finishes, only for static 6775 library. For dynamic library, we already have _fini and DllMain. */ 6776 int rc = atexit(__kmp_internal_end_atexit); 6777 if (rc != 0) { 6778 __kmp_fatal(KMP_MSG(FunctionError, "atexit()"), KMP_ERR(rc), 6779 __kmp_msg_null); 6780 } 6781 } 6782 #endif 6783 6784 #if KMP_HANDLE_SIGNALS 6785 #if KMP_OS_UNIX 6786 /* NOTE: make sure that this is called before the user installs their own 6787 signal handlers so that the user handlers are called first. this way they 6788 can return false, not call our handler, avoid terminating the library, and 6789 continue execution where they left off. */ 6790 __kmp_install_signals(FALSE); 6791 #endif /* KMP_OS_UNIX */ 6792 #if KMP_OS_WINDOWS 6793 __kmp_install_signals(TRUE); 6794 #endif /* KMP_OS_WINDOWS */ 6795 #endif 6796 6797 /* we have finished the serial initialization */ 6798 __kmp_init_counter++; 6799 6800 __kmp_init_serial = TRUE; 6801 6802 if (__kmp_settings) { 6803 __kmp_env_print(); 6804 } 6805 6806 #if OMP_40_ENABLED 6807 if (__kmp_display_env || __kmp_display_env_verbose) { 6808 __kmp_env_print_2(); 6809 } 6810 #endif // OMP_40_ENABLED 6811 6812 #if OMPT_SUPPORT 6813 ompt_post_init(); 6814 #endif 6815 6816 KMP_MB(); 6817 6818 KA_TRACE(10, ("__kmp_do_serial_initialize: exit\n")); 6819 } 6820 6821 void __kmp_serial_initialize(void) { 6822 if (__kmp_init_serial) { 6823 return; 6824 } 6825 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6826 if (__kmp_init_serial) { 6827 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6828 return; 6829 } 6830 __kmp_do_serial_initialize(); 6831 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6832 } 6833 6834 static void __kmp_do_middle_initialize(void) { 6835 int i, j; 6836 int prev_dflt_team_nth; 6837 6838 if (!__kmp_init_serial) { 6839 __kmp_do_serial_initialize(); 6840 } 6841 6842 KA_TRACE(10, ("__kmp_middle_initialize: enter\n")); 6843 6844 // Save the previous value for the __kmp_dflt_team_nth so that 6845 // we can avoid some reinitialization if it hasn't changed. 6846 prev_dflt_team_nth = __kmp_dflt_team_nth; 6847 6848 #if KMP_AFFINITY_SUPPORTED 6849 // __kmp_affinity_initialize() will try to set __kmp_ncores to the 6850 // number of cores on the machine. 6851 __kmp_affinity_initialize(); 6852 6853 // Run through the __kmp_threads array and set the affinity mask 6854 // for each root thread that is currently registered with the RTL. 6855 for (i = 0; i < __kmp_threads_capacity; i++) { 6856 if (TCR_PTR(__kmp_threads[i]) != NULL) { 6857 __kmp_affinity_set_init_mask(i, TRUE); 6858 } 6859 } 6860 #endif /* KMP_AFFINITY_SUPPORTED */ 6861 6862 KMP_ASSERT(__kmp_xproc > 0); 6863 if (__kmp_avail_proc == 0) { 6864 __kmp_avail_proc = __kmp_xproc; 6865 } 6866 6867 // If there were empty places in num_threads list (OMP_NUM_THREADS=,,2,3), 6868 // correct them now 6869 j = 0; 6870 while ((j < __kmp_nested_nth.used) && !__kmp_nested_nth.nth[j]) { 6871 __kmp_nested_nth.nth[j] = __kmp_dflt_team_nth = __kmp_dflt_team_nth_ub = 6872 __kmp_avail_proc; 6873 j++; 6874 } 6875 6876 if (__kmp_dflt_team_nth == 0) { 6877 #ifdef KMP_DFLT_NTH_CORES 6878 // Default #threads = #cores 6879 __kmp_dflt_team_nth = __kmp_ncores; 6880 KA_TRACE(20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = " 6881 "__kmp_ncores (%d)\n", 6882 __kmp_dflt_team_nth)); 6883 #else 6884 // Default #threads = #available OS procs 6885 __kmp_dflt_team_nth = __kmp_avail_proc; 6886 KA_TRACE(20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = " 6887 "__kmp_avail_proc(%d)\n", 6888 __kmp_dflt_team_nth)); 6889 #endif /* KMP_DFLT_NTH_CORES */ 6890 } 6891 6892 if (__kmp_dflt_team_nth < KMP_MIN_NTH) { 6893 __kmp_dflt_team_nth = KMP_MIN_NTH; 6894 } 6895 if (__kmp_dflt_team_nth > __kmp_sys_max_nth) { 6896 __kmp_dflt_team_nth = __kmp_sys_max_nth; 6897 } 6898 6899 // There's no harm in continuing if the following check fails, 6900 // but it indicates an error in the previous logic. 6901 KMP_DEBUG_ASSERT(__kmp_dflt_team_nth <= __kmp_dflt_team_nth_ub); 6902 6903 if (__kmp_dflt_team_nth != prev_dflt_team_nth) { 6904 // Run through the __kmp_threads array and set the num threads icv for each 6905 // root thread that is currently registered with the RTL (which has not 6906 // already explicitly set its nthreads-var with a call to 6907 // omp_set_num_threads()). 6908 for (i = 0; i < __kmp_threads_capacity; i++) { 6909 kmp_info_t *thread = __kmp_threads[i]; 6910 if (thread == NULL) 6911 continue; 6912 if (thread->th.th_current_task->td_icvs.nproc != 0) 6913 continue; 6914 6915 set__nproc(__kmp_threads[i], __kmp_dflt_team_nth); 6916 } 6917 } 6918 KA_TRACE( 6919 20, 6920 ("__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n", 6921 __kmp_dflt_team_nth)); 6922 6923 #ifdef KMP_ADJUST_BLOCKTIME 6924 /* Adjust blocktime to zero if necessary now that __kmp_avail_proc is set */ 6925 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 6926 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0); 6927 if (__kmp_nth > __kmp_avail_proc) { 6928 __kmp_zero_bt = TRUE; 6929 } 6930 } 6931 #endif /* KMP_ADJUST_BLOCKTIME */ 6932 6933 /* we have finished middle initialization */ 6934 TCW_SYNC_4(__kmp_init_middle, TRUE); 6935 6936 KA_TRACE(10, ("__kmp_do_middle_initialize: exit\n")); 6937 } 6938 6939 void __kmp_middle_initialize(void) { 6940 if (__kmp_init_middle) { 6941 return; 6942 } 6943 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6944 if (__kmp_init_middle) { 6945 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6946 return; 6947 } 6948 __kmp_do_middle_initialize(); 6949 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6950 } 6951 6952 void __kmp_parallel_initialize(void) { 6953 int gtid = __kmp_entry_gtid(); // this might be a new root 6954 6955 /* synchronize parallel initialization (for sibling) */ 6956 if (TCR_4(__kmp_init_parallel)) 6957 return; 6958 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6959 if (TCR_4(__kmp_init_parallel)) { 6960 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6961 return; 6962 } 6963 6964 /* TODO reinitialization after we have already shut down */ 6965 if (TCR_4(__kmp_global.g.g_done)) { 6966 KA_TRACE( 6967 10, 6968 ("__kmp_parallel_initialize: attempt to init while shutting down\n")); 6969 __kmp_infinite_loop(); 6970 } 6971 6972 /* jc: The lock __kmp_initz_lock is already held, so calling 6973 __kmp_serial_initialize would cause a deadlock. So we call 6974 __kmp_do_serial_initialize directly. */ 6975 if (!__kmp_init_middle) { 6976 __kmp_do_middle_initialize(); 6977 } 6978 6979 #if OMP_50_ENABLED 6980 __kmp_resume_if_hard_paused(); 6981 #endif 6982 6983 /* begin initialization */ 6984 KA_TRACE(10, ("__kmp_parallel_initialize: enter\n")); 6985 KMP_ASSERT(KMP_UBER_GTID(gtid)); 6986 6987 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 6988 // Save the FP control regs. 6989 // Worker threads will set theirs to these values at thread startup. 6990 __kmp_store_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word); 6991 __kmp_store_mxcsr(&__kmp_init_mxcsr); 6992 __kmp_init_mxcsr &= KMP_X86_MXCSR_MASK; 6993 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 6994 6995 #if KMP_OS_UNIX 6996 #if KMP_HANDLE_SIGNALS 6997 /* must be after __kmp_serial_initialize */ 6998 __kmp_install_signals(TRUE); 6999 #endif 7000 #endif 7001 7002 __kmp_suspend_initialize(); 7003 7004 #if defined(USE_LOAD_BALANCE) 7005 if (__kmp_global.g.g_dynamic_mode == dynamic_default) { 7006 __kmp_global.g.g_dynamic_mode = dynamic_load_balance; 7007 } 7008 #else 7009 if (__kmp_global.g.g_dynamic_mode == dynamic_default) { 7010 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit; 7011 } 7012 #endif 7013 7014 if (__kmp_version) { 7015 __kmp_print_version_2(); 7016 } 7017 7018 /* we have finished parallel initialization */ 7019 TCW_SYNC_4(__kmp_init_parallel, TRUE); 7020 7021 KMP_MB(); 7022 KA_TRACE(10, ("__kmp_parallel_initialize: exit\n")); 7023 7024 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 7025 } 7026 7027 /* ------------------------------------------------------------------------ */ 7028 7029 void __kmp_run_before_invoked_task(int gtid, int tid, kmp_info_t *this_thr, 7030 kmp_team_t *team) { 7031 kmp_disp_t *dispatch; 7032 7033 KMP_MB(); 7034 7035 /* none of the threads have encountered any constructs, yet. */ 7036 this_thr->th.th_local.this_construct = 0; 7037 #if KMP_CACHE_MANAGE 7038 KMP_CACHE_PREFETCH(&this_thr->th.th_bar[bs_forkjoin_barrier].bb.b_arrived); 7039 #endif /* KMP_CACHE_MANAGE */ 7040 dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch); 7041 KMP_DEBUG_ASSERT(dispatch); 7042 KMP_DEBUG_ASSERT(team->t.t_dispatch); 7043 // KMP_DEBUG_ASSERT( this_thr->th.th_dispatch == &team->t.t_dispatch[ 7044 // this_thr->th.th_info.ds.ds_tid ] ); 7045 7046 dispatch->th_disp_index = 0; /* reset the dispatch buffer counter */ 7047 #if OMP_45_ENABLED 7048 dispatch->th_doacross_buf_idx = 7049 0; /* reset the doacross dispatch buffer counter */ 7050 #endif 7051 if (__kmp_env_consistency_check) 7052 __kmp_push_parallel(gtid, team->t.t_ident); 7053 7054 KMP_MB(); /* Flush all pending memory write invalidates. */ 7055 } 7056 7057 void __kmp_run_after_invoked_task(int gtid, int tid, kmp_info_t *this_thr, 7058 kmp_team_t *team) { 7059 if (__kmp_env_consistency_check) 7060 __kmp_pop_parallel(gtid, team->t.t_ident); 7061 7062 __kmp_finish_implicit_task(this_thr); 7063 } 7064 7065 int __kmp_invoke_task_func(int gtid) { 7066 int rc; 7067 int tid = __kmp_tid_from_gtid(gtid); 7068 kmp_info_t *this_thr = __kmp_threads[gtid]; 7069 kmp_team_t *team = this_thr->th.th_team; 7070 7071 __kmp_run_before_invoked_task(gtid, tid, this_thr, team); 7072 #if USE_ITT_BUILD 7073 if (__itt_stack_caller_create_ptr) { 7074 __kmp_itt_stack_callee_enter( 7075 (__itt_caller) 7076 team->t.t_stack_id); // inform ittnotify about entering user's code 7077 } 7078 #endif /* USE_ITT_BUILD */ 7079 #if INCLUDE_SSC_MARKS 7080 SSC_MARK_INVOKING(); 7081 #endif 7082 7083 #if OMPT_SUPPORT 7084 void *dummy; 7085 void **exit_runtime_p; 7086 ompt_data_t *my_task_data; 7087 ompt_data_t *my_parallel_data; 7088 int ompt_team_size; 7089 7090 if (ompt_enabled.enabled) { 7091 exit_runtime_p = &( 7092 team->t.t_implicit_task_taskdata[tid].ompt_task_info.frame.exit_frame.ptr); 7093 } else { 7094 exit_runtime_p = &dummy; 7095 } 7096 7097 my_task_data = 7098 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data); 7099 my_parallel_data = &(team->t.ompt_team_info.parallel_data); 7100 if (ompt_enabled.ompt_callback_implicit_task) { 7101 ompt_team_size = team->t.t_nproc; 7102 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 7103 ompt_scope_begin, my_parallel_data, my_task_data, ompt_team_size, 7104 __kmp_tid_from_gtid(gtid), ompt_task_implicit); // TODO: Can this be ompt_task_initial? 7105 OMPT_CUR_TASK_INFO(this_thr)->thread_num = __kmp_tid_from_gtid(gtid); 7106 } 7107 #endif 7108 7109 { 7110 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 7111 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 7112 rc = 7113 __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn), gtid, 7114 tid, (int)team->t.t_argc, (void **)team->t.t_argv 7115 #if OMPT_SUPPORT 7116 , 7117 exit_runtime_p 7118 #endif 7119 ); 7120 #if OMPT_SUPPORT 7121 *exit_runtime_p = NULL; 7122 #endif 7123 } 7124 7125 #if USE_ITT_BUILD 7126 if (__itt_stack_caller_create_ptr) { 7127 __kmp_itt_stack_callee_leave( 7128 (__itt_caller) 7129 team->t.t_stack_id); // inform ittnotify about leaving user's code 7130 } 7131 #endif /* USE_ITT_BUILD */ 7132 __kmp_run_after_invoked_task(gtid, tid, this_thr, team); 7133 7134 return rc; 7135 } 7136 7137 #if OMP_40_ENABLED 7138 void __kmp_teams_master(int gtid) { 7139 // This routine is called by all master threads in teams construct 7140 kmp_info_t *thr = __kmp_threads[gtid]; 7141 kmp_team_t *team = thr->th.th_team; 7142 ident_t *loc = team->t.t_ident; 7143 thr->th.th_set_nproc = thr->th.th_teams_size.nth; 7144 KMP_DEBUG_ASSERT(thr->th.th_teams_microtask); 7145 KMP_DEBUG_ASSERT(thr->th.th_set_nproc); 7146 KA_TRACE(20, ("__kmp_teams_master: T#%d, Tid %d, microtask %p\n", gtid, 7147 __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask)); 7148 7149 // This thread is a new CG root. Set up the proper variables. 7150 kmp_cg_root_t *tmp = (kmp_cg_root_t *)__kmp_allocate(sizeof(kmp_cg_root_t)); 7151 tmp->cg_root = thr; // Make thr the CG root 7152 // Init to thread limit that was stored when league masters were forked 7153 tmp->cg_thread_limit = thr->th.th_current_task->td_icvs.thread_limit; 7154 tmp->cg_nthreads = 1; // Init counter to one active thread, this one 7155 KA_TRACE(100, ("__kmp_teams_master: Thread %p created node %p and init" 7156 " cg_threads to 1\n", 7157 thr, tmp)); 7158 tmp->up = thr->th.th_cg_roots; 7159 thr->th.th_cg_roots = tmp; 7160 7161 // Launch league of teams now, but not let workers execute 7162 // (they hang on fork barrier until next parallel) 7163 #if INCLUDE_SSC_MARKS 7164 SSC_MARK_FORKING(); 7165 #endif 7166 __kmp_fork_call(loc, gtid, fork_context_intel, team->t.t_argc, 7167 (microtask_t)thr->th.th_teams_microtask, // "wrapped" task 7168 VOLATILE_CAST(launch_t) __kmp_invoke_task_func, NULL); 7169 #if INCLUDE_SSC_MARKS 7170 SSC_MARK_JOINING(); 7171 #endif 7172 // If the team size was reduced from the limit, set it to the new size 7173 if (thr->th.th_team_nproc < thr->th.th_teams_size.nth) 7174 thr->th.th_teams_size.nth = thr->th.th_team_nproc; 7175 // AC: last parameter "1" eliminates join barrier which won't work because 7176 // worker threads are in a fork barrier waiting for more parallel regions 7177 __kmp_join_call(loc, gtid 7178 #if OMPT_SUPPORT 7179 , 7180 fork_context_intel 7181 #endif 7182 , 7183 1); 7184 } 7185 7186 int __kmp_invoke_teams_master(int gtid) { 7187 kmp_info_t *this_thr = __kmp_threads[gtid]; 7188 kmp_team_t *team = this_thr->th.th_team; 7189 #if KMP_DEBUG 7190 if (!__kmp_threads[gtid]->th.th_team->t.t_serialized) 7191 KMP_DEBUG_ASSERT((void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn == 7192 (void *)__kmp_teams_master); 7193 #endif 7194 __kmp_run_before_invoked_task(gtid, 0, this_thr, team); 7195 __kmp_teams_master(gtid); 7196 __kmp_run_after_invoked_task(gtid, 0, this_thr, team); 7197 return 1; 7198 } 7199 #endif /* OMP_40_ENABLED */ 7200 7201 /* this sets the requested number of threads for the next parallel region 7202 encountered by this team. since this should be enclosed in the forkjoin 7203 critical section it should avoid race conditions with assymmetrical nested 7204 parallelism */ 7205 7206 void __kmp_push_num_threads(ident_t *id, int gtid, int num_threads) { 7207 kmp_info_t *thr = __kmp_threads[gtid]; 7208 7209 if (num_threads > 0) 7210 thr->th.th_set_nproc = num_threads; 7211 } 7212 7213 #if OMP_40_ENABLED 7214 7215 /* this sets the requested number of teams for the teams region and/or 7216 the number of threads for the next parallel region encountered */ 7217 void __kmp_push_num_teams(ident_t *id, int gtid, int num_teams, 7218 int num_threads) { 7219 kmp_info_t *thr = __kmp_threads[gtid]; 7220 KMP_DEBUG_ASSERT(num_teams >= 0); 7221 KMP_DEBUG_ASSERT(num_threads >= 0); 7222 7223 if (num_teams == 0) 7224 num_teams = 1; // default number of teams is 1. 7225 if (num_teams > __kmp_teams_max_nth) { // if too many teams requested? 7226 if (!__kmp_reserve_warn) { 7227 __kmp_reserve_warn = 1; 7228 __kmp_msg(kmp_ms_warning, 7229 KMP_MSG(CantFormThrTeam, num_teams, __kmp_teams_max_nth), 7230 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 7231 } 7232 num_teams = __kmp_teams_max_nth; 7233 } 7234 // Set number of teams (number of threads in the outer "parallel" of the 7235 // teams) 7236 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams; 7237 7238 // Remember the number of threads for inner parallel regions 7239 if (num_threads == 0) { 7240 if (!TCR_4(__kmp_init_middle)) 7241 __kmp_middle_initialize(); // get __kmp_avail_proc calculated 7242 num_threads = __kmp_avail_proc / num_teams; 7243 if (num_teams * num_threads > __kmp_teams_max_nth) { 7244 // adjust num_threads w/o warning as it is not user setting 7245 num_threads = __kmp_teams_max_nth / num_teams; 7246 } 7247 } else { 7248 // This thread will be the master of the league masters 7249 // Store new thread limit; old limit is saved in th_cg_roots list 7250 thr->th.th_current_task->td_icvs.thread_limit = num_threads; 7251 7252 if (num_teams * num_threads > __kmp_teams_max_nth) { 7253 int new_threads = __kmp_teams_max_nth / num_teams; 7254 if (!__kmp_reserve_warn) { // user asked for too many threads 7255 __kmp_reserve_warn = 1; // conflicts with KMP_TEAMS_THREAD_LIMIT 7256 __kmp_msg(kmp_ms_warning, 7257 KMP_MSG(CantFormThrTeam, num_threads, new_threads), 7258 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 7259 } 7260 num_threads = new_threads; 7261 } 7262 } 7263 thr->th.th_teams_size.nth = num_threads; 7264 } 7265 7266 // Set the proc_bind var to use in the following parallel region. 7267 void __kmp_push_proc_bind(ident_t *id, int gtid, kmp_proc_bind_t proc_bind) { 7268 kmp_info_t *thr = __kmp_threads[gtid]; 7269 thr->th.th_set_proc_bind = proc_bind; 7270 } 7271 7272 #endif /* OMP_40_ENABLED */ 7273 7274 /* Launch the worker threads into the microtask. */ 7275 7276 void __kmp_internal_fork(ident_t *id, int gtid, kmp_team_t *team) { 7277 kmp_info_t *this_thr = __kmp_threads[gtid]; 7278 7279 #ifdef KMP_DEBUG 7280 int f; 7281 #endif /* KMP_DEBUG */ 7282 7283 KMP_DEBUG_ASSERT(team); 7284 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); 7285 KMP_ASSERT(KMP_MASTER_GTID(gtid)); 7286 KMP_MB(); /* Flush all pending memory write invalidates. */ 7287 7288 team->t.t_construct = 0; /* no single directives seen yet */ 7289 team->t.t_ordered.dt.t_value = 7290 0; /* thread 0 enters the ordered section first */ 7291 7292 /* Reset the identifiers on the dispatch buffer */ 7293 KMP_DEBUG_ASSERT(team->t.t_disp_buffer); 7294 if (team->t.t_max_nproc > 1) { 7295 int i; 7296 for (i = 0; i < __kmp_dispatch_num_buffers; ++i) { 7297 team->t.t_disp_buffer[i].buffer_index = i; 7298 #if OMP_45_ENABLED 7299 team->t.t_disp_buffer[i].doacross_buf_idx = i; 7300 #endif 7301 } 7302 } else { 7303 team->t.t_disp_buffer[0].buffer_index = 0; 7304 #if OMP_45_ENABLED 7305 team->t.t_disp_buffer[0].doacross_buf_idx = 0; 7306 #endif 7307 } 7308 7309 KMP_MB(); /* Flush all pending memory write invalidates. */ 7310 KMP_ASSERT(this_thr->th.th_team == team); 7311 7312 #ifdef KMP_DEBUG 7313 for (f = 0; f < team->t.t_nproc; f++) { 7314 KMP_DEBUG_ASSERT(team->t.t_threads[f] && 7315 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc); 7316 } 7317 #endif /* KMP_DEBUG */ 7318 7319 /* release the worker threads so they may begin working */ 7320 __kmp_fork_barrier(gtid, 0); 7321 } 7322 7323 void __kmp_internal_join(ident_t *id, int gtid, kmp_team_t *team) { 7324 kmp_info_t *this_thr = __kmp_threads[gtid]; 7325 7326 KMP_DEBUG_ASSERT(team); 7327 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); 7328 KMP_ASSERT(KMP_MASTER_GTID(gtid)); 7329 KMP_MB(); /* Flush all pending memory write invalidates. */ 7330 7331 /* Join barrier after fork */ 7332 7333 #ifdef KMP_DEBUG 7334 if (__kmp_threads[gtid] && 7335 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) { 7336 __kmp_printf("GTID: %d, __kmp_threads[%d]=%p\n", gtid, gtid, 7337 __kmp_threads[gtid]); 7338 __kmp_printf("__kmp_threads[%d]->th.th_team_nproc=%d, TEAM: %p, " 7339 "team->t.t_nproc=%d\n", 7340 gtid, __kmp_threads[gtid]->th.th_team_nproc, team, 7341 team->t.t_nproc); 7342 __kmp_print_structure(); 7343 } 7344 KMP_DEBUG_ASSERT(__kmp_threads[gtid] && 7345 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc); 7346 #endif /* KMP_DEBUG */ 7347 7348 __kmp_join_barrier(gtid); /* wait for everyone */ 7349 #if OMPT_SUPPORT 7350 if (ompt_enabled.enabled && 7351 this_thr->th.ompt_thread_info.state == ompt_state_wait_barrier_implicit) { 7352 int ds_tid = this_thr->th.th_info.ds.ds_tid; 7353 ompt_data_t *task_data = OMPT_CUR_TASK_DATA(this_thr); 7354 this_thr->th.ompt_thread_info.state = ompt_state_overhead; 7355 #if OMPT_OPTIONAL 7356 void *codeptr = NULL; 7357 if (KMP_MASTER_TID(ds_tid) && 7358 (ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait) || 7359 ompt_callbacks.ompt_callback(ompt_callback_sync_region))) 7360 codeptr = OMPT_CUR_TEAM_INFO(this_thr)->master_return_address; 7361 7362 if (ompt_enabled.ompt_callback_sync_region_wait) { 7363 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 7364 ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data, 7365 codeptr); 7366 } 7367 if (ompt_enabled.ompt_callback_sync_region) { 7368 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 7369 ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data, 7370 codeptr); 7371 } 7372 #endif 7373 if (!KMP_MASTER_TID(ds_tid) && ompt_enabled.ompt_callback_implicit_task) { 7374 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 7375 ompt_scope_end, NULL, task_data, 0, ds_tid, ompt_task_implicit); // TODO: Can this be ompt_task_initial? 7376 } 7377 } 7378 #endif 7379 7380 KMP_MB(); /* Flush all pending memory write invalidates. */ 7381 KMP_ASSERT(this_thr->th.th_team == team); 7382 } 7383 7384 /* ------------------------------------------------------------------------ */ 7385 7386 #ifdef USE_LOAD_BALANCE 7387 7388 // Return the worker threads actively spinning in the hot team, if we 7389 // are at the outermost level of parallelism. Otherwise, return 0. 7390 static int __kmp_active_hot_team_nproc(kmp_root_t *root) { 7391 int i; 7392 int retval; 7393 kmp_team_t *hot_team; 7394 7395 if (root->r.r_active) { 7396 return 0; 7397 } 7398 hot_team = root->r.r_hot_team; 7399 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) { 7400 return hot_team->t.t_nproc - 1; // Don't count master thread 7401 } 7402 7403 // Skip the master thread - it is accounted for elsewhere. 7404 retval = 0; 7405 for (i = 1; i < hot_team->t.t_nproc; i++) { 7406 if (hot_team->t.t_threads[i]->th.th_active) { 7407 retval++; 7408 } 7409 } 7410 return retval; 7411 } 7412 7413 // Perform an automatic adjustment to the number of 7414 // threads used by the next parallel region. 7415 static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc) { 7416 int retval; 7417 int pool_active; 7418 int hot_team_active; 7419 int team_curr_active; 7420 int system_active; 7421 7422 KB_TRACE(20, ("__kmp_load_balance_nproc: called root:%p set_nproc:%d\n", root, 7423 set_nproc)); 7424 KMP_DEBUG_ASSERT(root); 7425 KMP_DEBUG_ASSERT(root->r.r_root_team->t.t_threads[0] 7426 ->th.th_current_task->td_icvs.dynamic == TRUE); 7427 KMP_DEBUG_ASSERT(set_nproc > 1); 7428 7429 if (set_nproc == 1) { 7430 KB_TRACE(20, ("__kmp_load_balance_nproc: serial execution.\n")); 7431 return 1; 7432 } 7433 7434 // Threads that are active in the thread pool, active in the hot team for this 7435 // particular root (if we are at the outer par level), and the currently 7436 // executing thread (to become the master) are available to add to the new 7437 // team, but are currently contributing to the system load, and must be 7438 // accounted for. 7439 pool_active = __kmp_thread_pool_active_nth; 7440 hot_team_active = __kmp_active_hot_team_nproc(root); 7441 team_curr_active = pool_active + hot_team_active + 1; 7442 7443 // Check the system load. 7444 system_active = __kmp_get_load_balance(__kmp_avail_proc + team_curr_active); 7445 KB_TRACE(30, ("__kmp_load_balance_nproc: system active = %d pool active = %d " 7446 "hot team active = %d\n", 7447 system_active, pool_active, hot_team_active)); 7448 7449 if (system_active < 0) { 7450 // There was an error reading the necessary info from /proc, so use the 7451 // thread limit algorithm instead. Once we set __kmp_global.g.g_dynamic_mode 7452 // = dynamic_thread_limit, we shouldn't wind up getting back here. 7453 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit; 7454 KMP_WARNING(CantLoadBalUsing, "KMP_DYNAMIC_MODE=thread limit"); 7455 7456 // Make this call behave like the thread limit algorithm. 7457 retval = __kmp_avail_proc - __kmp_nth + 7458 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc); 7459 if (retval > set_nproc) { 7460 retval = set_nproc; 7461 } 7462 if (retval < KMP_MIN_NTH) { 7463 retval = KMP_MIN_NTH; 7464 } 7465 7466 KB_TRACE(20, ("__kmp_load_balance_nproc: thread limit exit. retval:%d\n", 7467 retval)); 7468 return retval; 7469 } 7470 7471 // There is a slight delay in the load balance algorithm in detecting new 7472 // running procs. The real system load at this instant should be at least as 7473 // large as the #active omp thread that are available to add to the team. 7474 if (system_active < team_curr_active) { 7475 system_active = team_curr_active; 7476 } 7477 retval = __kmp_avail_proc - system_active + team_curr_active; 7478 if (retval > set_nproc) { 7479 retval = set_nproc; 7480 } 7481 if (retval < KMP_MIN_NTH) { 7482 retval = KMP_MIN_NTH; 7483 } 7484 7485 KB_TRACE(20, ("__kmp_load_balance_nproc: exit. retval:%d\n", retval)); 7486 return retval; 7487 } // __kmp_load_balance_nproc() 7488 7489 #endif /* USE_LOAD_BALANCE */ 7490 7491 /* ------------------------------------------------------------------------ */ 7492 7493 /* NOTE: this is called with the __kmp_init_lock held */ 7494 void __kmp_cleanup(void) { 7495 int f; 7496 7497 KA_TRACE(10, ("__kmp_cleanup: enter\n")); 7498 7499 if (TCR_4(__kmp_init_parallel)) { 7500 #if KMP_HANDLE_SIGNALS 7501 __kmp_remove_signals(); 7502 #endif 7503 TCW_4(__kmp_init_parallel, FALSE); 7504 } 7505 7506 if (TCR_4(__kmp_init_middle)) { 7507 #if KMP_AFFINITY_SUPPORTED 7508 __kmp_affinity_uninitialize(); 7509 #endif /* KMP_AFFINITY_SUPPORTED */ 7510 __kmp_cleanup_hierarchy(); 7511 TCW_4(__kmp_init_middle, FALSE); 7512 } 7513 7514 KA_TRACE(10, ("__kmp_cleanup: go serial cleanup\n")); 7515 7516 if (__kmp_init_serial) { 7517 __kmp_runtime_destroy(); 7518 __kmp_init_serial = FALSE; 7519 } 7520 7521 __kmp_cleanup_threadprivate_caches(); 7522 7523 for (f = 0; f < __kmp_threads_capacity; f++) { 7524 if (__kmp_root[f] != NULL) { 7525 __kmp_free(__kmp_root[f]); 7526 __kmp_root[f] = NULL; 7527 } 7528 } 7529 __kmp_free(__kmp_threads); 7530 // __kmp_threads and __kmp_root were allocated at once, as single block, so 7531 // there is no need in freeing __kmp_root. 7532 __kmp_threads = NULL; 7533 __kmp_root = NULL; 7534 __kmp_threads_capacity = 0; 7535 7536 #if KMP_USE_DYNAMIC_LOCK 7537 __kmp_cleanup_indirect_user_locks(); 7538 #else 7539 __kmp_cleanup_user_locks(); 7540 #endif 7541 7542 #if KMP_AFFINITY_SUPPORTED 7543 KMP_INTERNAL_FREE(CCAST(char *, __kmp_cpuinfo_file)); 7544 __kmp_cpuinfo_file = NULL; 7545 #endif /* KMP_AFFINITY_SUPPORTED */ 7546 7547 #if KMP_USE_ADAPTIVE_LOCKS 7548 #if KMP_DEBUG_ADAPTIVE_LOCKS 7549 __kmp_print_speculative_stats(); 7550 #endif 7551 #endif 7552 KMP_INTERNAL_FREE(__kmp_nested_nth.nth); 7553 __kmp_nested_nth.nth = NULL; 7554 __kmp_nested_nth.size = 0; 7555 __kmp_nested_nth.used = 0; 7556 KMP_INTERNAL_FREE(__kmp_nested_proc_bind.bind_types); 7557 __kmp_nested_proc_bind.bind_types = NULL; 7558 __kmp_nested_proc_bind.size = 0; 7559 __kmp_nested_proc_bind.used = 0; 7560 #if OMP_50_ENABLED 7561 if (__kmp_affinity_format) { 7562 KMP_INTERNAL_FREE(__kmp_affinity_format); 7563 __kmp_affinity_format = NULL; 7564 } 7565 #endif 7566 7567 __kmp_i18n_catclose(); 7568 7569 #if KMP_USE_HIER_SCHED 7570 __kmp_hier_scheds.deallocate(); 7571 #endif 7572 7573 #if KMP_STATS_ENABLED 7574 __kmp_stats_fini(); 7575 #endif 7576 7577 KA_TRACE(10, ("__kmp_cleanup: exit\n")); 7578 } 7579 7580 /* ------------------------------------------------------------------------ */ 7581 7582 int __kmp_ignore_mppbeg(void) { 7583 char *env; 7584 7585 if ((env = getenv("KMP_IGNORE_MPPBEG")) != NULL) { 7586 if (__kmp_str_match_false(env)) 7587 return FALSE; 7588 } 7589 // By default __kmpc_begin() is no-op. 7590 return TRUE; 7591 } 7592 7593 int __kmp_ignore_mppend(void) { 7594 char *env; 7595 7596 if ((env = getenv("KMP_IGNORE_MPPEND")) != NULL) { 7597 if (__kmp_str_match_false(env)) 7598 return FALSE; 7599 } 7600 // By default __kmpc_end() is no-op. 7601 return TRUE; 7602 } 7603 7604 void __kmp_internal_begin(void) { 7605 int gtid; 7606 kmp_root_t *root; 7607 7608 /* this is a very important step as it will register new sibling threads 7609 and assign these new uber threads a new gtid */ 7610 gtid = __kmp_entry_gtid(); 7611 root = __kmp_threads[gtid]->th.th_root; 7612 KMP_ASSERT(KMP_UBER_GTID(gtid)); 7613 7614 if (root->r.r_begin) 7615 return; 7616 __kmp_acquire_lock(&root->r.r_begin_lock, gtid); 7617 if (root->r.r_begin) { 7618 __kmp_release_lock(&root->r.r_begin_lock, gtid); 7619 return; 7620 } 7621 7622 root->r.r_begin = TRUE; 7623 7624 __kmp_release_lock(&root->r.r_begin_lock, gtid); 7625 } 7626 7627 /* ------------------------------------------------------------------------ */ 7628 7629 void __kmp_user_set_library(enum library_type arg) { 7630 int gtid; 7631 kmp_root_t *root; 7632 kmp_info_t *thread; 7633 7634 /* first, make sure we are initialized so we can get our gtid */ 7635 7636 gtid = __kmp_entry_gtid(); 7637 thread = __kmp_threads[gtid]; 7638 7639 root = thread->th.th_root; 7640 7641 KA_TRACE(20, ("__kmp_user_set_library: enter T#%d, arg: %d, %d\n", gtid, arg, 7642 library_serial)); 7643 if (root->r.r_in_parallel) { /* Must be called in serial section of top-level 7644 thread */ 7645 KMP_WARNING(SetLibraryIncorrectCall); 7646 return; 7647 } 7648 7649 switch (arg) { 7650 case library_serial: 7651 thread->th.th_set_nproc = 0; 7652 set__nproc(thread, 1); 7653 break; 7654 case library_turnaround: 7655 thread->th.th_set_nproc = 0; 7656 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth 7657 : __kmp_dflt_team_nth_ub); 7658 break; 7659 case library_throughput: 7660 thread->th.th_set_nproc = 0; 7661 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth 7662 : __kmp_dflt_team_nth_ub); 7663 break; 7664 default: 7665 KMP_FATAL(UnknownLibraryType, arg); 7666 } 7667 7668 __kmp_aux_set_library(arg); 7669 } 7670 7671 void __kmp_aux_set_stacksize(size_t arg) { 7672 if (!__kmp_init_serial) 7673 __kmp_serial_initialize(); 7674 7675 #if KMP_OS_DARWIN 7676 if (arg & (0x1000 - 1)) { 7677 arg &= ~(0x1000 - 1); 7678 if (arg + 0x1000) /* check for overflow if we round up */ 7679 arg += 0x1000; 7680 } 7681 #endif 7682 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 7683 7684 /* only change the default stacksize before the first parallel region */ 7685 if (!TCR_4(__kmp_init_parallel)) { 7686 size_t value = arg; /* argument is in bytes */ 7687 7688 if (value < __kmp_sys_min_stksize) 7689 value = __kmp_sys_min_stksize; 7690 else if (value > KMP_MAX_STKSIZE) 7691 value = KMP_MAX_STKSIZE; 7692 7693 __kmp_stksize = value; 7694 7695 __kmp_env_stksize = TRUE; /* was KMP_STACKSIZE specified? */ 7696 } 7697 7698 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 7699 } 7700 7701 /* set the behaviour of the runtime library */ 7702 /* TODO this can cause some odd behaviour with sibling parallelism... */ 7703 void __kmp_aux_set_library(enum library_type arg) { 7704 __kmp_library = arg; 7705 7706 switch (__kmp_library) { 7707 case library_serial: { 7708 KMP_INFORM(LibraryIsSerial); 7709 } break; 7710 case library_turnaround: 7711 if (__kmp_use_yield == 1 && !__kmp_use_yield_exp_set) 7712 __kmp_use_yield = 2; // only yield when oversubscribed 7713 break; 7714 case library_throughput: 7715 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) 7716 __kmp_dflt_blocktime = 200; 7717 break; 7718 default: 7719 KMP_FATAL(UnknownLibraryType, arg); 7720 } 7721 } 7722 7723 /* Getting team information common for all team API */ 7724 // Returns NULL if not in teams construct 7725 static kmp_team_t *__kmp_aux_get_team_info(int &teams_serialized) { 7726 kmp_info_t *thr = __kmp_entry_thread(); 7727 teams_serialized = 0; 7728 if (thr->th.th_teams_microtask) { 7729 kmp_team_t *team = thr->th.th_team; 7730 int tlevel = thr->th.th_teams_level; // the level of the teams construct 7731 int ii = team->t.t_level; 7732 teams_serialized = team->t.t_serialized; 7733 int level = tlevel + 1; 7734 KMP_DEBUG_ASSERT(ii >= tlevel); 7735 while (ii > level) { 7736 for (teams_serialized = team->t.t_serialized; 7737 (teams_serialized > 0) && (ii > level); teams_serialized--, ii--) { 7738 } 7739 if (team->t.t_serialized && (!teams_serialized)) { 7740 team = team->t.t_parent; 7741 continue; 7742 } 7743 if (ii > level) { 7744 team = team->t.t_parent; 7745 ii--; 7746 } 7747 } 7748 return team; 7749 } 7750 return NULL; 7751 } 7752 7753 int __kmp_aux_get_team_num() { 7754 int serialized; 7755 kmp_team_t *team = __kmp_aux_get_team_info(serialized); 7756 if (team) { 7757 if (serialized > 1) { 7758 return 0; // teams region is serialized ( 1 team of 1 thread ). 7759 } else { 7760 return team->t.t_master_tid; 7761 } 7762 } 7763 return 0; 7764 } 7765 7766 int __kmp_aux_get_num_teams() { 7767 int serialized; 7768 kmp_team_t *team = __kmp_aux_get_team_info(serialized); 7769 if (team) { 7770 if (serialized > 1) { 7771 return 1; 7772 } else { 7773 return team->t.t_parent->t.t_nproc; 7774 } 7775 } 7776 return 1; 7777 } 7778 7779 /* ------------------------------------------------------------------------ */ 7780 7781 #if OMP_50_ENABLED 7782 /* 7783 * Affinity Format Parser 7784 * 7785 * Field is in form of: %[[[0].]size]type 7786 * % and type are required (%% means print a literal '%') 7787 * type is either single char or long name surrounded by {}, 7788 * e.g., N or {num_threads} 7789 * 0 => leading zeros 7790 * . => right justified when size is specified 7791 * by default output is left justified 7792 * size is the *minimum* field length 7793 * All other characters are printed as is 7794 * 7795 * Available field types: 7796 * L {thread_level} - omp_get_level() 7797 * n {thread_num} - omp_get_thread_num() 7798 * h {host} - name of host machine 7799 * P {process_id} - process id (integer) 7800 * T {thread_identifier} - native thread identifier (integer) 7801 * N {num_threads} - omp_get_num_threads() 7802 * A {ancestor_tnum} - omp_get_ancestor_thread_num(omp_get_level()-1) 7803 * a {thread_affinity} - comma separated list of integers or integer ranges 7804 * (values of affinity mask) 7805 * 7806 * Implementation-specific field types can be added 7807 * If a type is unknown, print "undefined" 7808 */ 7809 7810 // Structure holding the short name, long name, and corresponding data type 7811 // for snprintf. A table of these will represent the entire valid keyword 7812 // field types. 7813 typedef struct kmp_affinity_format_field_t { 7814 char short_name; // from spec e.g., L -> thread level 7815 const char *long_name; // from spec thread_level -> thread level 7816 char field_format; // data type for snprintf (typically 'd' or 's' 7817 // for integer or string) 7818 } kmp_affinity_format_field_t; 7819 7820 static const kmp_affinity_format_field_t __kmp_affinity_format_table[] = { 7821 #if KMP_AFFINITY_SUPPORTED 7822 {'A', "thread_affinity", 's'}, 7823 #endif 7824 {'t', "team_num", 'd'}, 7825 {'T', "num_teams", 'd'}, 7826 {'L', "nesting_level", 'd'}, 7827 {'n', "thread_num", 'd'}, 7828 {'N', "num_threads", 'd'}, 7829 {'a', "ancestor_tnum", 'd'}, 7830 {'H', "host", 's'}, 7831 {'P', "process_id", 'd'}, 7832 {'i', "native_thread_id", 'd'}}; 7833 7834 // Return the number of characters it takes to hold field 7835 static int __kmp_aux_capture_affinity_field(int gtid, const kmp_info_t *th, 7836 const char **ptr, 7837 kmp_str_buf_t *field_buffer) { 7838 int rc, format_index, field_value; 7839 const char *width_left, *width_right; 7840 bool pad_zeros, right_justify, parse_long_name, found_valid_name; 7841 static const int FORMAT_SIZE = 20; 7842 char format[FORMAT_SIZE] = {0}; 7843 char absolute_short_name = 0; 7844 7845 KMP_DEBUG_ASSERT(gtid >= 0); 7846 KMP_DEBUG_ASSERT(th); 7847 KMP_DEBUG_ASSERT(**ptr == '%'); 7848 KMP_DEBUG_ASSERT(field_buffer); 7849 7850 __kmp_str_buf_clear(field_buffer); 7851 7852 // Skip the initial % 7853 (*ptr)++; 7854 7855 // Check for %% first 7856 if (**ptr == '%') { 7857 __kmp_str_buf_cat(field_buffer, "%", 1); 7858 (*ptr)++; // skip over the second % 7859 return 1; 7860 } 7861 7862 // Parse field modifiers if they are present 7863 pad_zeros = false; 7864 if (**ptr == '0') { 7865 pad_zeros = true; 7866 (*ptr)++; // skip over 0 7867 } 7868 right_justify = false; 7869 if (**ptr == '.') { 7870 right_justify = true; 7871 (*ptr)++; // skip over . 7872 } 7873 // Parse width of field: [width_left, width_right) 7874 width_left = width_right = NULL; 7875 if (**ptr >= '0' && **ptr <= '9') { 7876 width_left = *ptr; 7877 SKIP_DIGITS(*ptr); 7878 width_right = *ptr; 7879 } 7880 7881 // Create the format for KMP_SNPRINTF based on flags parsed above 7882 format_index = 0; 7883 format[format_index++] = '%'; 7884 if (!right_justify) 7885 format[format_index++] = '-'; 7886 if (pad_zeros) 7887 format[format_index++] = '0'; 7888 if (width_left && width_right) { 7889 int i = 0; 7890 // Only allow 8 digit number widths. 7891 // This also prevents overflowing format variable 7892 while (i < 8 && width_left < width_right) { 7893 format[format_index++] = *width_left; 7894 width_left++; 7895 i++; 7896 } 7897 } 7898 7899 // Parse a name (long or short) 7900 // Canonicalize the name into absolute_short_name 7901 found_valid_name = false; 7902 parse_long_name = (**ptr == '{'); 7903 if (parse_long_name) 7904 (*ptr)++; // skip initial left brace 7905 for (size_t i = 0; i < sizeof(__kmp_affinity_format_table) / 7906 sizeof(__kmp_affinity_format_table[0]); 7907 ++i) { 7908 char short_name = __kmp_affinity_format_table[i].short_name; 7909 const char *long_name = __kmp_affinity_format_table[i].long_name; 7910 char field_format = __kmp_affinity_format_table[i].field_format; 7911 if (parse_long_name) { 7912 int length = KMP_STRLEN(long_name); 7913 if (strncmp(*ptr, long_name, length) == 0) { 7914 found_valid_name = true; 7915 (*ptr) += length; // skip the long name 7916 } 7917 } else if (**ptr == short_name) { 7918 found_valid_name = true; 7919 (*ptr)++; // skip the short name 7920 } 7921 if (found_valid_name) { 7922 format[format_index++] = field_format; 7923 format[format_index++] = '\0'; 7924 absolute_short_name = short_name; 7925 break; 7926 } 7927 } 7928 if (parse_long_name) { 7929 if (**ptr != '}') { 7930 absolute_short_name = 0; 7931 } else { 7932 (*ptr)++; // skip over the right brace 7933 } 7934 } 7935 7936 // Attempt to fill the buffer with the requested 7937 // value using snprintf within __kmp_str_buf_print() 7938 switch (absolute_short_name) { 7939 case 't': 7940 rc = __kmp_str_buf_print(field_buffer, format, __kmp_aux_get_team_num()); 7941 break; 7942 case 'T': 7943 rc = __kmp_str_buf_print(field_buffer, format, __kmp_aux_get_num_teams()); 7944 break; 7945 case 'L': 7946 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_level); 7947 break; 7948 case 'n': 7949 rc = __kmp_str_buf_print(field_buffer, format, __kmp_tid_from_gtid(gtid)); 7950 break; 7951 case 'H': { 7952 static const int BUFFER_SIZE = 256; 7953 char buf[BUFFER_SIZE]; 7954 __kmp_expand_host_name(buf, BUFFER_SIZE); 7955 rc = __kmp_str_buf_print(field_buffer, format, buf); 7956 } break; 7957 case 'P': 7958 rc = __kmp_str_buf_print(field_buffer, format, getpid()); 7959 break; 7960 case 'i': 7961 rc = __kmp_str_buf_print(field_buffer, format, __kmp_gettid()); 7962 break; 7963 case 'N': 7964 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_nproc); 7965 break; 7966 case 'a': 7967 field_value = 7968 __kmp_get_ancestor_thread_num(gtid, th->th.th_team->t.t_level - 1); 7969 rc = __kmp_str_buf_print(field_buffer, format, field_value); 7970 break; 7971 #if KMP_AFFINITY_SUPPORTED 7972 case 'A': { 7973 kmp_str_buf_t buf; 7974 __kmp_str_buf_init(&buf); 7975 __kmp_affinity_str_buf_mask(&buf, th->th.th_affin_mask); 7976 rc = __kmp_str_buf_print(field_buffer, format, buf.str); 7977 __kmp_str_buf_free(&buf); 7978 } break; 7979 #endif 7980 default: 7981 // According to spec, If an implementation does not have info for field 7982 // type, then "undefined" is printed 7983 rc = __kmp_str_buf_print(field_buffer, "%s", "undefined"); 7984 // Skip the field 7985 if (parse_long_name) { 7986 SKIP_TOKEN(*ptr); 7987 if (**ptr == '}') 7988 (*ptr)++; 7989 } else { 7990 (*ptr)++; 7991 } 7992 } 7993 7994 KMP_ASSERT(format_index <= FORMAT_SIZE); 7995 return rc; 7996 } 7997 7998 /* 7999 * Return number of characters needed to hold the affinity string 8000 * (not including null byte character) 8001 * The resultant string is printed to buffer, which the caller can then 8002 * handle afterwards 8003 */ 8004 size_t __kmp_aux_capture_affinity(int gtid, const char *format, 8005 kmp_str_buf_t *buffer) { 8006 const char *parse_ptr; 8007 size_t retval; 8008 const kmp_info_t *th; 8009 kmp_str_buf_t field; 8010 8011 KMP_DEBUG_ASSERT(buffer); 8012 KMP_DEBUG_ASSERT(gtid >= 0); 8013 8014 __kmp_str_buf_init(&field); 8015 __kmp_str_buf_clear(buffer); 8016 8017 th = __kmp_threads[gtid]; 8018 retval = 0; 8019 8020 // If format is NULL or zero-length string, then we use 8021 // affinity-format-var ICV 8022 parse_ptr = format; 8023 if (parse_ptr == NULL || *parse_ptr == '\0') { 8024 parse_ptr = __kmp_affinity_format; 8025 } 8026 KMP_DEBUG_ASSERT(parse_ptr); 8027 8028 while (*parse_ptr != '\0') { 8029 // Parse a field 8030 if (*parse_ptr == '%') { 8031 // Put field in the buffer 8032 int rc = __kmp_aux_capture_affinity_field(gtid, th, &parse_ptr, &field); 8033 __kmp_str_buf_catbuf(buffer, &field); 8034 retval += rc; 8035 } else { 8036 // Put literal character in buffer 8037 __kmp_str_buf_cat(buffer, parse_ptr, 1); 8038 retval++; 8039 parse_ptr++; 8040 } 8041 } 8042 __kmp_str_buf_free(&field); 8043 return retval; 8044 } 8045 8046 // Displays the affinity string to stdout 8047 void __kmp_aux_display_affinity(int gtid, const char *format) { 8048 kmp_str_buf_t buf; 8049 __kmp_str_buf_init(&buf); 8050 __kmp_aux_capture_affinity(gtid, format, &buf); 8051 __kmp_fprintf(kmp_out, "%s" KMP_END_OF_LINE, buf.str); 8052 __kmp_str_buf_free(&buf); 8053 } 8054 #endif // OMP_50_ENABLED 8055 8056 /* ------------------------------------------------------------------------ */ 8057 8058 void __kmp_aux_set_blocktime(int arg, kmp_info_t *thread, int tid) { 8059 int blocktime = arg; /* argument is in milliseconds */ 8060 #if KMP_USE_MONITOR 8061 int bt_intervals; 8062 #endif 8063 int bt_set; 8064 8065 __kmp_save_internal_controls(thread); 8066 8067 /* Normalize and set blocktime for the teams */ 8068 if (blocktime < KMP_MIN_BLOCKTIME) 8069 blocktime = KMP_MIN_BLOCKTIME; 8070 else if (blocktime > KMP_MAX_BLOCKTIME) 8071 blocktime = KMP_MAX_BLOCKTIME; 8072 8073 set__blocktime_team(thread->th.th_team, tid, blocktime); 8074 set__blocktime_team(thread->th.th_serial_team, 0, blocktime); 8075 8076 #if KMP_USE_MONITOR 8077 /* Calculate and set blocktime intervals for the teams */ 8078 bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME(blocktime, __kmp_monitor_wakeups); 8079 8080 set__bt_intervals_team(thread->th.th_team, tid, bt_intervals); 8081 set__bt_intervals_team(thread->th.th_serial_team, 0, bt_intervals); 8082 #endif 8083 8084 /* Set whether blocktime has been set to "TRUE" */ 8085 bt_set = TRUE; 8086 8087 set__bt_set_team(thread->th.th_team, tid, bt_set); 8088 set__bt_set_team(thread->th.th_serial_team, 0, bt_set); 8089 #if KMP_USE_MONITOR 8090 KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, " 8091 "bt_intervals=%d, monitor_updates=%d\n", 8092 __kmp_gtid_from_tid(tid, thread->th.th_team), 8093 thread->th.th_team->t.t_id, tid, blocktime, bt_intervals, 8094 __kmp_monitor_wakeups)); 8095 #else 8096 KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n", 8097 __kmp_gtid_from_tid(tid, thread->th.th_team), 8098 thread->th.th_team->t.t_id, tid, blocktime)); 8099 #endif 8100 } 8101 8102 void __kmp_aux_set_defaults(char const *str, int len) { 8103 if (!__kmp_init_serial) { 8104 __kmp_serial_initialize(); 8105 } 8106 __kmp_env_initialize(str); 8107 8108 if (__kmp_settings 8109 #if OMP_40_ENABLED 8110 || __kmp_display_env || __kmp_display_env_verbose 8111 #endif // OMP_40_ENABLED 8112 ) { 8113 __kmp_env_print(); 8114 } 8115 } // __kmp_aux_set_defaults 8116 8117 /* ------------------------------------------------------------------------ */ 8118 /* internal fast reduction routines */ 8119 8120 PACKED_REDUCTION_METHOD_T 8121 __kmp_determine_reduction_method( 8122 ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, 8123 void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data), 8124 kmp_critical_name *lck) { 8125 8126 // Default reduction method: critical construct ( lck != NULL, like in current 8127 // PAROPT ) 8128 // If ( reduce_data!=NULL && reduce_func!=NULL ): the tree-reduction method 8129 // can be selected by RTL 8130 // If loc->flags contains KMP_IDENT_ATOMIC_REDUCE, the atomic reduce method 8131 // can be selected by RTL 8132 // Finally, it's up to OpenMP RTL to make a decision on which method to select 8133 // among generated by PAROPT. 8134 8135 PACKED_REDUCTION_METHOD_T retval; 8136 8137 int team_size; 8138 8139 KMP_DEBUG_ASSERT(loc); // it would be nice to test ( loc != 0 ) 8140 KMP_DEBUG_ASSERT(lck); // it would be nice to test ( lck != 0 ) 8141 8142 #define FAST_REDUCTION_ATOMIC_METHOD_GENERATED \ 8143 ((loc->flags & (KMP_IDENT_ATOMIC_REDUCE)) == (KMP_IDENT_ATOMIC_REDUCE)) 8144 #define FAST_REDUCTION_TREE_METHOD_GENERATED ((reduce_data) && (reduce_func)) 8145 8146 retval = critical_reduce_block; 8147 8148 // another choice of getting a team size (with 1 dynamic deference) is slower 8149 team_size = __kmp_get_team_num_threads(global_tid); 8150 if (team_size == 1) { 8151 8152 retval = empty_reduce_block; 8153 8154 } else { 8155 8156 int atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED; 8157 8158 #if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS64 8159 8160 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ 8161 KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HURD 8162 8163 int teamsize_cutoff = 4; 8164 8165 #if KMP_MIC_SUPPORTED 8166 if (__kmp_mic_type != non_mic) { 8167 teamsize_cutoff = 8; 8168 } 8169 #endif 8170 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED; 8171 if (tree_available) { 8172 if (team_size <= teamsize_cutoff) { 8173 if (atomic_available) { 8174 retval = atomic_reduce_block; 8175 } 8176 } else { 8177 retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER; 8178 } 8179 } else if (atomic_available) { 8180 retval = atomic_reduce_block; 8181 } 8182 #else 8183 #error "Unknown or unsupported OS" 8184 #endif // KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || 8185 // KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HURD 8186 8187 #elif KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS 8188 8189 #if KMP_OS_LINUX || KMP_OS_WINDOWS || KMP_OS_HURD 8190 8191 // basic tuning 8192 8193 if (atomic_available) { 8194 if (num_vars <= 2) { // && ( team_size <= 8 ) due to false-sharing ??? 8195 retval = atomic_reduce_block; 8196 } 8197 } // otherwise: use critical section 8198 8199 #elif KMP_OS_DARWIN 8200 8201 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED; 8202 if (atomic_available && (num_vars <= 3)) { 8203 retval = atomic_reduce_block; 8204 } else if (tree_available) { 8205 if ((reduce_size > (9 * sizeof(kmp_real64))) && 8206 (reduce_size < (2000 * sizeof(kmp_real64)))) { 8207 retval = TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER; 8208 } 8209 } // otherwise: use critical section 8210 8211 #else 8212 #error "Unknown or unsupported OS" 8213 #endif 8214 8215 #else 8216 #error "Unknown or unsupported architecture" 8217 #endif 8218 } 8219 8220 // KMP_FORCE_REDUCTION 8221 8222 // If the team is serialized (team_size == 1), ignore the forced reduction 8223 // method and stay with the unsynchronized method (empty_reduce_block) 8224 if (__kmp_force_reduction_method != reduction_method_not_defined && 8225 team_size != 1) { 8226 8227 PACKED_REDUCTION_METHOD_T forced_retval = critical_reduce_block; 8228 8229 int atomic_available, tree_available; 8230 8231 switch ((forced_retval = __kmp_force_reduction_method)) { 8232 case critical_reduce_block: 8233 KMP_ASSERT(lck); // lck should be != 0 8234 break; 8235 8236 case atomic_reduce_block: 8237 atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED; 8238 if (!atomic_available) { 8239 KMP_WARNING(RedMethodNotSupported, "atomic"); 8240 forced_retval = critical_reduce_block; 8241 } 8242 break; 8243 8244 case tree_reduce_block: 8245 tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED; 8246 if (!tree_available) { 8247 KMP_WARNING(RedMethodNotSupported, "tree"); 8248 forced_retval = critical_reduce_block; 8249 } else { 8250 #if KMP_FAST_REDUCTION_BARRIER 8251 forced_retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER; 8252 #endif 8253 } 8254 break; 8255 8256 default: 8257 KMP_ASSERT(0); // "unsupported method specified" 8258 } 8259 8260 retval = forced_retval; 8261 } 8262 8263 KA_TRACE(10, ("reduction method selected=%08x\n", retval)); 8264 8265 #undef FAST_REDUCTION_TREE_METHOD_GENERATED 8266 #undef FAST_REDUCTION_ATOMIC_METHOD_GENERATED 8267 8268 return (retval); 8269 } 8270 8271 // this function is for testing set/get/determine reduce method 8272 kmp_int32 __kmp_get_reduce_method(void) { 8273 return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8); 8274 } 8275 8276 #if OMP_50_ENABLED 8277 8278 // Soft pause sets up threads to ignore blocktime and just go to sleep. 8279 // Spin-wait code checks __kmp_pause_status and reacts accordingly. 8280 void __kmp_soft_pause() { __kmp_pause_status = kmp_soft_paused; } 8281 8282 // Hard pause shuts down the runtime completely. Resume happens naturally when 8283 // OpenMP is used subsequently. 8284 void __kmp_hard_pause() { 8285 __kmp_pause_status = kmp_hard_paused; 8286 __kmp_internal_end_thread(-1); 8287 } 8288 8289 // Soft resume sets __kmp_pause_status, and wakes up all threads. 8290 void __kmp_resume_if_soft_paused() { 8291 if (__kmp_pause_status == kmp_soft_paused) { 8292 __kmp_pause_status = kmp_not_paused; 8293 8294 for (int gtid = 1; gtid < __kmp_threads_capacity; ++gtid) { 8295 kmp_info_t *thread = __kmp_threads[gtid]; 8296 if (thread) { // Wake it if sleeping 8297 kmp_flag_64 fl(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, thread); 8298 if (fl.is_sleeping()) 8299 fl.resume(gtid); 8300 else if (__kmp_try_suspend_mx(thread)) { // got suspend lock 8301 __kmp_unlock_suspend_mx(thread); // unlock it; it won't sleep 8302 } else { // thread holds the lock and may sleep soon 8303 do { // until either the thread sleeps, or we can get the lock 8304 if (fl.is_sleeping()) { 8305 fl.resume(gtid); 8306 break; 8307 } else if (__kmp_try_suspend_mx(thread)) { 8308 __kmp_unlock_suspend_mx(thread); 8309 break; 8310 } 8311 } while (1); 8312 } 8313 } 8314 } 8315 } 8316 } 8317 8318 // This function is called via __kmpc_pause_resource. Returns 0 if successful. 8319 // TODO: add warning messages 8320 int __kmp_pause_resource(kmp_pause_status_t level) { 8321 if (level == kmp_not_paused) { // requesting resume 8322 if (__kmp_pause_status == kmp_not_paused) { 8323 // error message about runtime not being paused, so can't resume 8324 return 1; 8325 } else { 8326 KMP_DEBUG_ASSERT(__kmp_pause_status == kmp_soft_paused || 8327 __kmp_pause_status == kmp_hard_paused); 8328 __kmp_pause_status = kmp_not_paused; 8329 return 0; 8330 } 8331 } else if (level == kmp_soft_paused) { // requesting soft pause 8332 if (__kmp_pause_status != kmp_not_paused) { 8333 // error message about already being paused 8334 return 1; 8335 } else { 8336 __kmp_soft_pause(); 8337 return 0; 8338 } 8339 } else if (level == kmp_hard_paused) { // requesting hard pause 8340 if (__kmp_pause_status != kmp_not_paused) { 8341 // error message about already being paused 8342 return 1; 8343 } else { 8344 __kmp_hard_pause(); 8345 return 0; 8346 } 8347 } else { 8348 // error message about invalid level 8349 return 1; 8350 } 8351 } 8352 8353 #endif // OMP_50_ENABLED 8354