1 /* 2 * kmp_runtime.cpp -- KPTS runtime support library 3 */ 4 5 //===----------------------------------------------------------------------===// 6 // 7 // The LLVM Compiler Infrastructure 8 // 9 // This file is dual licensed under the MIT and the University of Illinois Open 10 // Source Licenses. See LICENSE.txt for details. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "kmp.h" 15 #include "kmp_affinity.h" 16 #include "kmp_atomic.h" 17 #include "kmp_environment.h" 18 #include "kmp_error.h" 19 #include "kmp_i18n.h" 20 #include "kmp_io.h" 21 #include "kmp_itt.h" 22 #include "kmp_settings.h" 23 #include "kmp_stats.h" 24 #include "kmp_str.h" 25 #include "kmp_wait_release.h" 26 #include "kmp_wrapper_getpid.h" 27 #include "kmp_dispatch.h" 28 #if KMP_USE_HIER_SCHED 29 #include "kmp_dispatch_hier.h" 30 #endif 31 32 #if OMPT_SUPPORT 33 #include "ompt-specific.h" 34 #endif 35 36 /* these are temporary issues to be dealt with */ 37 #define KMP_USE_PRCTL 0 38 39 #if KMP_OS_WINDOWS 40 #include <process.h> 41 #endif 42 43 #include "tsan_annotations.h" 44 45 #if defined(KMP_GOMP_COMPAT) 46 char const __kmp_version_alt_comp[] = 47 KMP_VERSION_PREFIX "alternative compiler support: yes"; 48 #endif /* defined(KMP_GOMP_COMPAT) */ 49 50 char const __kmp_version_omp_api[] = KMP_VERSION_PREFIX "API version: " 51 #if OMP_50_ENABLED 52 "5.0 (201611)"; 53 #elif OMP_45_ENABLED 54 "4.5 (201511)"; 55 #elif OMP_40_ENABLED 56 "4.0 (201307)"; 57 #else 58 "3.1 (201107)"; 59 #endif 60 61 #ifdef KMP_DEBUG 62 char const __kmp_version_lock[] = 63 KMP_VERSION_PREFIX "lock type: run time selectable"; 64 #endif /* KMP_DEBUG */ 65 66 #define KMP_MIN(x, y) ((x) < (y) ? (x) : (y)) 67 68 /* ------------------------------------------------------------------------ */ 69 70 #if KMP_USE_MONITOR 71 kmp_info_t __kmp_monitor; 72 #endif 73 74 /* Forward declarations */ 75 76 void __kmp_cleanup(void); 77 78 static void __kmp_initialize_info(kmp_info_t *, kmp_team_t *, int tid, 79 int gtid); 80 static void __kmp_initialize_team(kmp_team_t *team, int new_nproc, 81 kmp_internal_control_t *new_icvs, 82 ident_t *loc); 83 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 84 static void __kmp_partition_places(kmp_team_t *team, 85 int update_master_only = 0); 86 #endif 87 static void __kmp_do_serial_initialize(void); 88 void __kmp_fork_barrier(int gtid, int tid); 89 void __kmp_join_barrier(int gtid); 90 void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc, 91 kmp_internal_control_t *new_icvs, ident_t *loc); 92 93 #ifdef USE_LOAD_BALANCE 94 static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc); 95 #endif 96 97 static int __kmp_expand_threads(int nNeed); 98 #if KMP_OS_WINDOWS 99 static int __kmp_unregister_root_other_thread(int gtid); 100 #endif 101 static void __kmp_unregister_library(void); // called by __kmp_internal_end() 102 static void __kmp_reap_thread(kmp_info_t *thread, int is_root); 103 kmp_info_t *__kmp_thread_pool_insert_pt = NULL; 104 105 /* Calculate the identifier of the current thread */ 106 /* fast (and somewhat portable) way to get unique identifier of executing 107 thread. Returns KMP_GTID_DNE if we haven't been assigned a gtid. */ 108 int __kmp_get_global_thread_id() { 109 int i; 110 kmp_info_t **other_threads; 111 size_t stack_data; 112 char *stack_addr; 113 size_t stack_size; 114 char *stack_base; 115 116 KA_TRACE( 117 1000, 118 ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n", 119 __kmp_nth, __kmp_all_nth)); 120 121 /* JPH - to handle the case where __kmpc_end(0) is called immediately prior to 122 a parallel region, made it return KMP_GTID_DNE to force serial_initialize 123 by caller. Had to handle KMP_GTID_DNE at all call-sites, or else guarantee 124 __kmp_init_gtid for this to work. */ 125 126 if (!TCR_4(__kmp_init_gtid)) 127 return KMP_GTID_DNE; 128 129 #ifdef KMP_TDATA_GTID 130 if (TCR_4(__kmp_gtid_mode) >= 3) { 131 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using TDATA\n")); 132 return __kmp_gtid; 133 } 134 #endif 135 if (TCR_4(__kmp_gtid_mode) >= 2) { 136 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using keyed TLS\n")); 137 return __kmp_gtid_get_specific(); 138 } 139 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using internal alg.\n")); 140 141 stack_addr = (char *)&stack_data; 142 other_threads = __kmp_threads; 143 144 /* ATT: The code below is a source of potential bugs due to unsynchronized 145 access to __kmp_threads array. For example: 146 1. Current thread loads other_threads[i] to thr and checks it, it is 147 non-NULL. 148 2. Current thread is suspended by OS. 149 3. Another thread unregisters and finishes (debug versions of free() 150 may fill memory with something like 0xEF). 151 4. Current thread is resumed. 152 5. Current thread reads junk from *thr. 153 TODO: Fix it. --ln */ 154 155 for (i = 0; i < __kmp_threads_capacity; i++) { 156 157 kmp_info_t *thr = (kmp_info_t *)TCR_SYNC_PTR(other_threads[i]); 158 if (!thr) 159 continue; 160 161 stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize); 162 stack_base = (char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase); 163 164 /* stack grows down -- search through all of the active threads */ 165 166 if (stack_addr <= stack_base) { 167 size_t stack_diff = stack_base - stack_addr; 168 169 if (stack_diff <= stack_size) { 170 /* The only way we can be closer than the allocated */ 171 /* stack size is if we are running on this thread. */ 172 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == i); 173 return i; 174 } 175 } 176 } 177 178 /* get specific to try and determine our gtid */ 179 KA_TRACE(1000, 180 ("*** __kmp_get_global_thread_id: internal alg. failed to find " 181 "thread, using TLS\n")); 182 i = __kmp_gtid_get_specific(); 183 184 /*fprintf( stderr, "=== %d\n", i ); */ /* GROO */ 185 186 /* if we havn't been assigned a gtid, then return code */ 187 if (i < 0) 188 return i; 189 190 /* dynamically updated stack window for uber threads to avoid get_specific 191 call */ 192 if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)) { 193 KMP_FATAL(StackOverflow, i); 194 } 195 196 stack_base = (char *)other_threads[i]->th.th_info.ds.ds_stackbase; 197 if (stack_addr > stack_base) { 198 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr); 199 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize, 200 other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr - 201 stack_base); 202 } else { 203 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize, 204 stack_base - stack_addr); 205 } 206 207 /* Reprint stack bounds for ubermaster since they have been refined */ 208 if (__kmp_storage_map) { 209 char *stack_end = (char *)other_threads[i]->th.th_info.ds.ds_stackbase; 210 char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize; 211 __kmp_print_storage_map_gtid(i, stack_beg, stack_end, 212 other_threads[i]->th.th_info.ds.ds_stacksize, 213 "th_%d stack (refinement)", i); 214 } 215 return i; 216 } 217 218 int __kmp_get_global_thread_id_reg() { 219 int gtid; 220 221 if (!__kmp_init_serial) { 222 gtid = KMP_GTID_DNE; 223 } else 224 #ifdef KMP_TDATA_GTID 225 if (TCR_4(__kmp_gtid_mode) >= 3) { 226 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using TDATA\n")); 227 gtid = __kmp_gtid; 228 } else 229 #endif 230 if (TCR_4(__kmp_gtid_mode) >= 2) { 231 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using keyed TLS\n")); 232 gtid = __kmp_gtid_get_specific(); 233 } else { 234 KA_TRACE(1000, 235 ("*** __kmp_get_global_thread_id_reg: using internal alg.\n")); 236 gtid = __kmp_get_global_thread_id(); 237 } 238 239 /* we must be a new uber master sibling thread */ 240 if (gtid == KMP_GTID_DNE) { 241 KA_TRACE(10, 242 ("__kmp_get_global_thread_id_reg: Encountered new root thread. " 243 "Registering a new gtid.\n")); 244 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 245 if (!__kmp_init_serial) { 246 __kmp_do_serial_initialize(); 247 gtid = __kmp_gtid_get_specific(); 248 } else { 249 gtid = __kmp_register_root(FALSE); 250 } 251 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 252 /*__kmp_printf( "+++ %d\n", gtid ); */ /* GROO */ 253 } 254 255 KMP_DEBUG_ASSERT(gtid >= 0); 256 257 return gtid; 258 } 259 260 /* caller must hold forkjoin_lock */ 261 void __kmp_check_stack_overlap(kmp_info_t *th) { 262 int f; 263 char *stack_beg = NULL; 264 char *stack_end = NULL; 265 int gtid; 266 267 KA_TRACE(10, ("__kmp_check_stack_overlap: called\n")); 268 if (__kmp_storage_map) { 269 stack_end = (char *)th->th.th_info.ds.ds_stackbase; 270 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize; 271 272 gtid = __kmp_gtid_from_thread(th); 273 274 if (gtid == KMP_GTID_MONITOR) { 275 __kmp_print_storage_map_gtid( 276 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize, 277 "th_%s stack (%s)", "mon", 278 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual"); 279 } else { 280 __kmp_print_storage_map_gtid( 281 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize, 282 "th_%d stack (%s)", gtid, 283 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual"); 284 } 285 } 286 287 /* No point in checking ubermaster threads since they use refinement and 288 * cannot overlap */ 289 gtid = __kmp_gtid_from_thread(th); 290 if (__kmp_env_checks == TRUE && !KMP_UBER_GTID(gtid)) { 291 KA_TRACE(10, 292 ("__kmp_check_stack_overlap: performing extensive checking\n")); 293 if (stack_beg == NULL) { 294 stack_end = (char *)th->th.th_info.ds.ds_stackbase; 295 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize; 296 } 297 298 for (f = 0; f < __kmp_threads_capacity; f++) { 299 kmp_info_t *f_th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[f]); 300 301 if (f_th && f_th != th) { 302 char *other_stack_end = 303 (char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase); 304 char *other_stack_beg = 305 other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize); 306 if ((stack_beg > other_stack_beg && stack_beg < other_stack_end) || 307 (stack_end > other_stack_beg && stack_end < other_stack_end)) { 308 309 /* Print the other stack values before the abort */ 310 if (__kmp_storage_map) 311 __kmp_print_storage_map_gtid( 312 -1, other_stack_beg, other_stack_end, 313 (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize), 314 "th_%d stack (overlapped)", __kmp_gtid_from_thread(f_th)); 315 316 __kmp_fatal(KMP_MSG(StackOverlap), KMP_HNT(ChangeStackLimit), 317 __kmp_msg_null); 318 } 319 } 320 } 321 } 322 KA_TRACE(10, ("__kmp_check_stack_overlap: returning\n")); 323 } 324 325 /* ------------------------------------------------------------------------ */ 326 327 void __kmp_infinite_loop(void) { 328 static int done = FALSE; 329 330 while (!done) { 331 KMP_YIELD(1); 332 } 333 } 334 335 #define MAX_MESSAGE 512 336 337 void __kmp_print_storage_map_gtid(int gtid, void *p1, void *p2, size_t size, 338 char const *format, ...) { 339 char buffer[MAX_MESSAGE]; 340 va_list ap; 341 342 va_start(ap, format); 343 KMP_SNPRINTF(buffer, sizeof(buffer), "OMP storage map: %p %p%8lu %s\n", p1, 344 p2, (unsigned long)size, format); 345 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock); 346 __kmp_vprintf(kmp_err, buffer, ap); 347 #if KMP_PRINT_DATA_PLACEMENT 348 int node; 349 if (gtid >= 0) { 350 if (p1 <= p2 && (char *)p2 - (char *)p1 == size) { 351 if (__kmp_storage_map_verbose) { 352 node = __kmp_get_host_node(p1); 353 if (node < 0) /* doesn't work, so don't try this next time */ 354 __kmp_storage_map_verbose = FALSE; 355 else { 356 char *last; 357 int lastNode; 358 int localProc = __kmp_get_cpu_from_gtid(gtid); 359 360 const int page_size = KMP_GET_PAGE_SIZE(); 361 362 p1 = (void *)((size_t)p1 & ~((size_t)page_size - 1)); 363 p2 = (void *)(((size_t)p2 - 1) & ~((size_t)page_size - 1)); 364 if (localProc >= 0) 365 __kmp_printf_no_lock(" GTID %d localNode %d\n", gtid, 366 localProc >> 1); 367 else 368 __kmp_printf_no_lock(" GTID %d\n", gtid); 369 #if KMP_USE_PRCTL 370 /* The more elaborate format is disabled for now because of the prctl 371 * hanging bug. */ 372 do { 373 last = p1; 374 lastNode = node; 375 /* This loop collates adjacent pages with the same host node. */ 376 do { 377 (char *)p1 += page_size; 378 } while (p1 <= p2 && (node = __kmp_get_host_node(p1)) == lastNode); 379 __kmp_printf_no_lock(" %p-%p memNode %d\n", last, (char *)p1 - 1, 380 lastNode); 381 } while (p1 <= p2); 382 #else 383 __kmp_printf_no_lock(" %p-%p memNode %d\n", p1, 384 (char *)p1 + (page_size - 1), 385 __kmp_get_host_node(p1)); 386 if (p1 < p2) { 387 __kmp_printf_no_lock(" %p-%p memNode %d\n", p2, 388 (char *)p2 + (page_size - 1), 389 __kmp_get_host_node(p2)); 390 } 391 #endif 392 } 393 } 394 } else 395 __kmp_printf_no_lock(" %s\n", KMP_I18N_STR(StorageMapWarning)); 396 } 397 #endif /* KMP_PRINT_DATA_PLACEMENT */ 398 __kmp_release_bootstrap_lock(&__kmp_stdio_lock); 399 } 400 401 void __kmp_warn(char const *format, ...) { 402 char buffer[MAX_MESSAGE]; 403 va_list ap; 404 405 if (__kmp_generate_warnings == kmp_warnings_off) { 406 return; 407 } 408 409 va_start(ap, format); 410 411 KMP_SNPRINTF(buffer, sizeof(buffer), "OMP warning: %s\n", format); 412 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock); 413 __kmp_vprintf(kmp_err, buffer, ap); 414 __kmp_release_bootstrap_lock(&__kmp_stdio_lock); 415 416 va_end(ap); 417 } 418 419 void __kmp_abort_process() { 420 // Later threads may stall here, but that's ok because abort() will kill them. 421 __kmp_acquire_bootstrap_lock(&__kmp_exit_lock); 422 423 if (__kmp_debug_buf) { 424 __kmp_dump_debug_buffer(); 425 } 426 427 if (KMP_OS_WINDOWS) { 428 // Let other threads know of abnormal termination and prevent deadlock 429 // if abort happened during library initialization or shutdown 430 __kmp_global.g.g_abort = SIGABRT; 431 432 /* On Windows* OS by default abort() causes pop-up error box, which stalls 433 nightly testing. Unfortunately, we cannot reliably suppress pop-up error 434 boxes. _set_abort_behavior() works well, but this function is not 435 available in VS7 (this is not problem for DLL, but it is a problem for 436 static OpenMP RTL). SetErrorMode (and so, timelimit utility) does not 437 help, at least in some versions of MS C RTL. 438 439 It seems following sequence is the only way to simulate abort() and 440 avoid pop-up error box. */ 441 raise(SIGABRT); 442 _exit(3); // Just in case, if signal ignored, exit anyway. 443 } else { 444 abort(); 445 } 446 447 __kmp_infinite_loop(); 448 __kmp_release_bootstrap_lock(&__kmp_exit_lock); 449 450 } // __kmp_abort_process 451 452 void __kmp_abort_thread(void) { 453 // TODO: Eliminate g_abort global variable and this function. 454 // In case of abort just call abort(), it will kill all the threads. 455 __kmp_infinite_loop(); 456 } // __kmp_abort_thread 457 458 /* Print out the storage map for the major kmp_info_t thread data structures 459 that are allocated together. */ 460 461 static void __kmp_print_thread_storage_map(kmp_info_t *thr, int gtid) { 462 __kmp_print_storage_map_gtid(gtid, thr, thr + 1, sizeof(kmp_info_t), "th_%d", 463 gtid); 464 465 __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team, 466 sizeof(kmp_desc_t), "th_%d.th_info", gtid); 467 468 __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head, 469 sizeof(kmp_local_t), "th_%d.th_local", gtid); 470 471 __kmp_print_storage_map_gtid( 472 gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier], 473 sizeof(kmp_balign_t) * bs_last_barrier, "th_%d.th_bar", gtid); 474 475 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier], 476 &thr->th.th_bar[bs_plain_barrier + 1], 477 sizeof(kmp_balign_t), "th_%d.th_bar[plain]", 478 gtid); 479 480 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier], 481 &thr->th.th_bar[bs_forkjoin_barrier + 1], 482 sizeof(kmp_balign_t), "th_%d.th_bar[forkjoin]", 483 gtid); 484 485 #if KMP_FAST_REDUCTION_BARRIER 486 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier], 487 &thr->th.th_bar[bs_reduction_barrier + 1], 488 sizeof(kmp_balign_t), "th_%d.th_bar[reduction]", 489 gtid); 490 #endif // KMP_FAST_REDUCTION_BARRIER 491 } 492 493 /* Print out the storage map for the major kmp_team_t team data structures 494 that are allocated together. */ 495 496 static void __kmp_print_team_storage_map(const char *header, kmp_team_t *team, 497 int team_id, int num_thr) { 498 int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2; 499 __kmp_print_storage_map_gtid(-1, team, team + 1, sizeof(kmp_team_t), "%s_%d", 500 header, team_id); 501 502 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0], 503 &team->t.t_bar[bs_last_barrier], 504 sizeof(kmp_balign_team_t) * bs_last_barrier, 505 "%s_%d.t_bar", header, team_id); 506 507 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier], 508 &team->t.t_bar[bs_plain_barrier + 1], 509 sizeof(kmp_balign_team_t), "%s_%d.t_bar[plain]", 510 header, team_id); 511 512 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier], 513 &team->t.t_bar[bs_forkjoin_barrier + 1], 514 sizeof(kmp_balign_team_t), 515 "%s_%d.t_bar[forkjoin]", header, team_id); 516 517 #if KMP_FAST_REDUCTION_BARRIER 518 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier], 519 &team->t.t_bar[bs_reduction_barrier + 1], 520 sizeof(kmp_balign_team_t), 521 "%s_%d.t_bar[reduction]", header, team_id); 522 #endif // KMP_FAST_REDUCTION_BARRIER 523 524 __kmp_print_storage_map_gtid( 525 -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr], 526 sizeof(kmp_disp_t) * num_thr, "%s_%d.t_dispatch", header, team_id); 527 528 __kmp_print_storage_map_gtid( 529 -1, &team->t.t_threads[0], &team->t.t_threads[num_thr], 530 sizeof(kmp_info_t *) * num_thr, "%s_%d.t_threads", header, team_id); 531 532 __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0], 533 &team->t.t_disp_buffer[num_disp_buff], 534 sizeof(dispatch_shared_info_t) * num_disp_buff, 535 "%s_%d.t_disp_buffer", header, team_id); 536 537 __kmp_print_storage_map_gtid(-1, &team->t.t_taskq, &team->t.t_copypriv_data, 538 sizeof(kmp_taskq_t), "%s_%d.t_taskq", header, 539 team_id); 540 } 541 542 static void __kmp_init_allocator() {} 543 static void __kmp_fini_allocator() {} 544 545 /* ------------------------------------------------------------------------ */ 546 547 #ifdef KMP_DYNAMIC_LIB 548 #if KMP_OS_WINDOWS 549 550 static void __kmp_reset_lock(kmp_bootstrap_lock_t *lck) { 551 // TODO: Change to __kmp_break_bootstrap_lock(). 552 __kmp_init_bootstrap_lock(lck); // make the lock released 553 } 554 555 static void __kmp_reset_locks_on_process_detach(int gtid_req) { 556 int i; 557 int thread_count; 558 559 // PROCESS_DETACH is expected to be called by a thread that executes 560 // ProcessExit() or FreeLibrary(). OS terminates other threads (except the one 561 // calling ProcessExit or FreeLibrary). So, it might be safe to access the 562 // __kmp_threads[] without taking the forkjoin_lock. However, in fact, some 563 // threads can be still alive here, although being about to be terminated. The 564 // threads in the array with ds_thread==0 are most suspicious. Actually, it 565 // can be not safe to access the __kmp_threads[]. 566 567 // TODO: does it make sense to check __kmp_roots[] ? 568 569 // Let's check that there are no other alive threads registered with the OMP 570 // lib. 571 while (1) { 572 thread_count = 0; 573 for (i = 0; i < __kmp_threads_capacity; ++i) { 574 if (!__kmp_threads) 575 continue; 576 kmp_info_t *th = __kmp_threads[i]; 577 if (th == NULL) 578 continue; 579 int gtid = th->th.th_info.ds.ds_gtid; 580 if (gtid == gtid_req) 581 continue; 582 if (gtid < 0) 583 continue; 584 DWORD exit_val; 585 int alive = __kmp_is_thread_alive(th, &exit_val); 586 if (alive) { 587 ++thread_count; 588 } 589 } 590 if (thread_count == 0) 591 break; // success 592 } 593 594 // Assume that I'm alone. Now it might be safe to check and reset locks. 595 // __kmp_forkjoin_lock and __kmp_stdio_lock are expected to be reset. 596 __kmp_reset_lock(&__kmp_forkjoin_lock); 597 #ifdef KMP_DEBUG 598 __kmp_reset_lock(&__kmp_stdio_lock); 599 #endif // KMP_DEBUG 600 } 601 602 BOOL WINAPI DllMain(HINSTANCE hInstDLL, DWORD fdwReason, LPVOID lpReserved) { 603 //__kmp_acquire_bootstrap_lock( &__kmp_initz_lock ); 604 605 switch (fdwReason) { 606 607 case DLL_PROCESS_ATTACH: 608 KA_TRACE(10, ("DllMain: PROCESS_ATTACH\n")); 609 610 return TRUE; 611 612 case DLL_PROCESS_DETACH: 613 KA_TRACE(10, ("DllMain: PROCESS_DETACH T#%d\n", __kmp_gtid_get_specific())); 614 615 if (lpReserved != NULL) { 616 // lpReserved is used for telling the difference: 617 // lpReserved == NULL when FreeLibrary() was called, 618 // lpReserved != NULL when the process terminates. 619 // When FreeLibrary() is called, worker threads remain alive. So they will 620 // release the forkjoin lock by themselves. When the process terminates, 621 // worker threads disappear triggering the problem of unreleased forkjoin 622 // lock as described below. 623 624 // A worker thread can take the forkjoin lock. The problem comes up if 625 // that worker thread becomes dead before it releases the forkjoin lock. 626 // The forkjoin lock remains taken, while the thread executing 627 // DllMain()->PROCESS_DETACH->__kmp_internal_end_library() below will try 628 // to take the forkjoin lock and will always fail, so that the application 629 // will never finish [normally]. This scenario is possible if 630 // __kmpc_end() has not been executed. It looks like it's not a corner 631 // case, but common cases: 632 // - the main function was compiled by an alternative compiler; 633 // - the main function was compiled by icl but without /Qopenmp 634 // (application with plugins); 635 // - application terminates by calling C exit(), Fortran CALL EXIT() or 636 // Fortran STOP. 637 // - alive foreign thread prevented __kmpc_end from doing cleanup. 638 // 639 // This is a hack to work around the problem. 640 // TODO: !!! figure out something better. 641 __kmp_reset_locks_on_process_detach(__kmp_gtid_get_specific()); 642 } 643 644 __kmp_internal_end_library(__kmp_gtid_get_specific()); 645 646 return TRUE; 647 648 case DLL_THREAD_ATTACH: 649 KA_TRACE(10, ("DllMain: THREAD_ATTACH\n")); 650 651 /* if we want to register new siblings all the time here call 652 * __kmp_get_gtid(); */ 653 return TRUE; 654 655 case DLL_THREAD_DETACH: 656 KA_TRACE(10, ("DllMain: THREAD_DETACH T#%d\n", __kmp_gtid_get_specific())); 657 658 __kmp_internal_end_thread(__kmp_gtid_get_specific()); 659 return TRUE; 660 } 661 662 return TRUE; 663 } 664 665 #endif /* KMP_OS_WINDOWS */ 666 #endif /* KMP_DYNAMIC_LIB */ 667 668 /* Change the library type to "status" and return the old type */ 669 /* called from within initialization routines where __kmp_initz_lock is held */ 670 int __kmp_change_library(int status) { 671 int old_status; 672 673 old_status = __kmp_yield_init & 674 1; // check whether KMP_LIBRARY=throughput (even init count) 675 676 if (status) { 677 __kmp_yield_init |= 1; // throughput => turnaround (odd init count) 678 } else { 679 __kmp_yield_init &= ~1; // turnaround => throughput (even init count) 680 } 681 682 return old_status; // return previous setting of whether 683 // KMP_LIBRARY=throughput 684 } 685 686 /* __kmp_parallel_deo -- Wait until it's our turn. */ 687 void __kmp_parallel_deo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) { 688 int gtid = *gtid_ref; 689 #ifdef BUILD_PARALLEL_ORDERED 690 kmp_team_t *team = __kmp_team_from_gtid(gtid); 691 #endif /* BUILD_PARALLEL_ORDERED */ 692 693 if (__kmp_env_consistency_check) { 694 if (__kmp_threads[gtid]->th.th_root->r.r_active) 695 #if KMP_USE_DYNAMIC_LOCK 696 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL, 0); 697 #else 698 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL); 699 #endif 700 } 701 #ifdef BUILD_PARALLEL_ORDERED 702 if (!team->t.t_serialized) { 703 KMP_MB(); 704 KMP_WAIT_YIELD(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid), 705 KMP_EQ, NULL); 706 KMP_MB(); 707 } 708 #endif /* BUILD_PARALLEL_ORDERED */ 709 } 710 711 /* __kmp_parallel_dxo -- Signal the next task. */ 712 void __kmp_parallel_dxo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) { 713 int gtid = *gtid_ref; 714 #ifdef BUILD_PARALLEL_ORDERED 715 int tid = __kmp_tid_from_gtid(gtid); 716 kmp_team_t *team = __kmp_team_from_gtid(gtid); 717 #endif /* BUILD_PARALLEL_ORDERED */ 718 719 if (__kmp_env_consistency_check) { 720 if (__kmp_threads[gtid]->th.th_root->r.r_active) 721 __kmp_pop_sync(gtid, ct_ordered_in_parallel, loc_ref); 722 } 723 #ifdef BUILD_PARALLEL_ORDERED 724 if (!team->t.t_serialized) { 725 KMP_MB(); /* Flush all pending memory write invalidates. */ 726 727 /* use the tid of the next thread in this team */ 728 /* TODO replace with general release procedure */ 729 team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc); 730 731 KMP_MB(); /* Flush all pending memory write invalidates. */ 732 } 733 #endif /* BUILD_PARALLEL_ORDERED */ 734 } 735 736 /* ------------------------------------------------------------------------ */ 737 /* The BARRIER for a SINGLE process section is always explicit */ 738 739 int __kmp_enter_single(int gtid, ident_t *id_ref, int push_ws) { 740 int status; 741 kmp_info_t *th; 742 kmp_team_t *team; 743 744 if (!TCR_4(__kmp_init_parallel)) 745 __kmp_parallel_initialize(); 746 747 th = __kmp_threads[gtid]; 748 team = th->th.th_team; 749 status = 0; 750 751 th->th.th_ident = id_ref; 752 753 if (team->t.t_serialized) { 754 status = 1; 755 } else { 756 kmp_int32 old_this = th->th.th_local.this_construct; 757 758 ++th->th.th_local.this_construct; 759 /* try to set team count to thread count--success means thread got the 760 single block */ 761 /* TODO: Should this be acquire or release? */ 762 if (team->t.t_construct == old_this) { 763 status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this, 764 th->th.th_local.this_construct); 765 } 766 #if USE_ITT_BUILD 767 if (__itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 && 768 KMP_MASTER_GTID(gtid) && 769 #if OMP_40_ENABLED 770 th->th.th_teams_microtask == NULL && 771 #endif 772 team->t.t_active_level == 773 1) { // Only report metadata by master of active team at level 1 774 __kmp_itt_metadata_single(id_ref); 775 } 776 #endif /* USE_ITT_BUILD */ 777 } 778 779 if (__kmp_env_consistency_check) { 780 if (status && push_ws) { 781 __kmp_push_workshare(gtid, ct_psingle, id_ref); 782 } else { 783 __kmp_check_workshare(gtid, ct_psingle, id_ref); 784 } 785 } 786 #if USE_ITT_BUILD 787 if (status) { 788 __kmp_itt_single_start(gtid); 789 } 790 #endif /* USE_ITT_BUILD */ 791 return status; 792 } 793 794 void __kmp_exit_single(int gtid) { 795 #if USE_ITT_BUILD 796 __kmp_itt_single_end(gtid); 797 #endif /* USE_ITT_BUILD */ 798 if (__kmp_env_consistency_check) 799 __kmp_pop_workshare(gtid, ct_psingle, NULL); 800 } 801 802 /* determine if we can go parallel or must use a serialized parallel region and 803 * how many threads we can use 804 * set_nproc is the number of threads requested for the team 805 * returns 0 if we should serialize or only use one thread, 806 * otherwise the number of threads to use 807 * The forkjoin lock is held by the caller. */ 808 static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team, 809 int master_tid, int set_nthreads 810 #if OMP_40_ENABLED 811 , 812 int enter_teams 813 #endif /* OMP_40_ENABLED */ 814 ) { 815 int capacity; 816 int new_nthreads; 817 KMP_DEBUG_ASSERT(__kmp_init_serial); 818 KMP_DEBUG_ASSERT(root && parent_team); 819 820 // If dyn-var is set, dynamically adjust the number of desired threads, 821 // according to the method specified by dynamic_mode. 822 new_nthreads = set_nthreads; 823 if (!get__dynamic_2(parent_team, master_tid)) { 824 ; 825 } 826 #ifdef USE_LOAD_BALANCE 827 else if (__kmp_global.g.g_dynamic_mode == dynamic_load_balance) { 828 new_nthreads = __kmp_load_balance_nproc(root, set_nthreads); 829 if (new_nthreads == 1) { 830 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced " 831 "reservation to 1 thread\n", 832 master_tid)); 833 return 1; 834 } 835 if (new_nthreads < set_nthreads) { 836 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced " 837 "reservation to %d threads\n", 838 master_tid, new_nthreads)); 839 } 840 } 841 #endif /* USE_LOAD_BALANCE */ 842 else if (__kmp_global.g.g_dynamic_mode == dynamic_thread_limit) { 843 new_nthreads = __kmp_avail_proc - __kmp_nth + 844 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc); 845 if (new_nthreads <= 1) { 846 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced " 847 "reservation to 1 thread\n", 848 master_tid)); 849 return 1; 850 } 851 if (new_nthreads < set_nthreads) { 852 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced " 853 "reservation to %d threads\n", 854 master_tid, new_nthreads)); 855 } else { 856 new_nthreads = set_nthreads; 857 } 858 } else if (__kmp_global.g.g_dynamic_mode == dynamic_random) { 859 if (set_nthreads > 2) { 860 new_nthreads = __kmp_get_random(parent_team->t.t_threads[master_tid]); 861 new_nthreads = (new_nthreads % set_nthreads) + 1; 862 if (new_nthreads == 1) { 863 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced " 864 "reservation to 1 thread\n", 865 master_tid)); 866 return 1; 867 } 868 if (new_nthreads < set_nthreads) { 869 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced " 870 "reservation to %d threads\n", 871 master_tid, new_nthreads)); 872 } 873 } 874 } else { 875 KMP_ASSERT(0); 876 } 877 878 // Respect KMP_ALL_THREADS/KMP_DEVICE_THREAD_LIMIT. 879 if (__kmp_nth + new_nthreads - 880 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) > 881 __kmp_max_nth) { 882 int tl_nthreads = __kmp_max_nth - __kmp_nth + 883 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc); 884 if (tl_nthreads <= 0) { 885 tl_nthreads = 1; 886 } 887 888 // If dyn-var is false, emit a 1-time warning. 889 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) { 890 __kmp_reserve_warn = 1; 891 __kmp_msg(kmp_ms_warning, 892 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads), 893 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 894 } 895 if (tl_nthreads == 1) { 896 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT " 897 "reduced reservation to 1 thread\n", 898 master_tid)); 899 return 1; 900 } 901 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced " 902 "reservation to %d threads\n", 903 master_tid, tl_nthreads)); 904 new_nthreads = tl_nthreads; 905 } 906 907 // Respect OMP_THREAD_LIMIT 908 if (root->r.r_cg_nthreads + new_nthreads - 909 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) > 910 __kmp_cg_max_nth) { 911 int tl_nthreads = __kmp_cg_max_nth - root->r.r_cg_nthreads + 912 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc); 913 if (tl_nthreads <= 0) { 914 tl_nthreads = 1; 915 } 916 917 // If dyn-var is false, emit a 1-time warning. 918 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) { 919 __kmp_reserve_warn = 1; 920 __kmp_msg(kmp_ms_warning, 921 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads), 922 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 923 } 924 if (tl_nthreads == 1) { 925 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT " 926 "reduced reservation to 1 thread\n", 927 master_tid)); 928 return 1; 929 } 930 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced " 931 "reservation to %d threads\n", 932 master_tid, tl_nthreads)); 933 new_nthreads = tl_nthreads; 934 } 935 936 // Check if the threads array is large enough, or needs expanding. 937 // See comment in __kmp_register_root() about the adjustment if 938 // __kmp_threads[0] == NULL. 939 capacity = __kmp_threads_capacity; 940 if (TCR_PTR(__kmp_threads[0]) == NULL) { 941 --capacity; 942 } 943 if (__kmp_nth + new_nthreads - 944 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) > 945 capacity) { 946 // Expand the threads array. 947 int slotsRequired = __kmp_nth + new_nthreads - 948 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) - 949 capacity; 950 int slotsAdded = __kmp_expand_threads(slotsRequired); 951 if (slotsAdded < slotsRequired) { 952 // The threads array was not expanded enough. 953 new_nthreads -= (slotsRequired - slotsAdded); 954 KMP_ASSERT(new_nthreads >= 1); 955 956 // If dyn-var is false, emit a 1-time warning. 957 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) { 958 __kmp_reserve_warn = 1; 959 if (__kmp_tp_cached) { 960 __kmp_msg(kmp_ms_warning, 961 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads), 962 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity), 963 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null); 964 } else { 965 __kmp_msg(kmp_ms_warning, 966 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads), 967 KMP_HNT(SystemLimitOnThreads), __kmp_msg_null); 968 } 969 } 970 } 971 } 972 973 #ifdef KMP_DEBUG 974 if (new_nthreads == 1) { 975 KC_TRACE(10, 976 ("__kmp_reserve_threads: T#%d serializing team after reclaiming " 977 "dead roots and rechecking; requested %d threads\n", 978 __kmp_get_gtid(), set_nthreads)); 979 } else { 980 KC_TRACE(10, ("__kmp_reserve_threads: T#%d allocating %d threads; requested" 981 " %d threads\n", 982 __kmp_get_gtid(), new_nthreads, set_nthreads)); 983 } 984 #endif // KMP_DEBUG 985 return new_nthreads; 986 } 987 988 /* Allocate threads from the thread pool and assign them to the new team. We are 989 assured that there are enough threads available, because we checked on that 990 earlier within critical section forkjoin */ 991 static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team, 992 kmp_info_t *master_th, int master_gtid) { 993 int i; 994 int use_hot_team; 995 996 KA_TRACE(10, ("__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc)); 997 KMP_DEBUG_ASSERT(master_gtid == __kmp_get_gtid()); 998 KMP_MB(); 999 1000 /* first, let's setup the master thread */ 1001 master_th->th.th_info.ds.ds_tid = 0; 1002 master_th->th.th_team = team; 1003 master_th->th.th_team_nproc = team->t.t_nproc; 1004 master_th->th.th_team_master = master_th; 1005 master_th->th.th_team_serialized = FALSE; 1006 master_th->th.th_dispatch = &team->t.t_dispatch[0]; 1007 1008 /* make sure we are not the optimized hot team */ 1009 #if KMP_NESTED_HOT_TEAMS 1010 use_hot_team = 0; 1011 kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams; 1012 if (hot_teams) { // hot teams array is not allocated if 1013 // KMP_HOT_TEAMS_MAX_LEVEL=0 1014 int level = team->t.t_active_level - 1; // index in array of hot teams 1015 if (master_th->th.th_teams_microtask) { // are we inside the teams? 1016 if (master_th->th.th_teams_size.nteams > 1) { 1017 ++level; // level was not increased in teams construct for 1018 // team_of_masters 1019 } 1020 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master && 1021 master_th->th.th_teams_level == team->t.t_level) { 1022 ++level; // level was not increased in teams construct for 1023 // team_of_workers before the parallel 1024 } // team->t.t_level will be increased inside parallel 1025 } 1026 if (level < __kmp_hot_teams_max_level) { 1027 if (hot_teams[level].hot_team) { 1028 // hot team has already been allocated for given level 1029 KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team); 1030 use_hot_team = 1; // the team is ready to use 1031 } else { 1032 use_hot_team = 0; // AC: threads are not allocated yet 1033 hot_teams[level].hot_team = team; // remember new hot team 1034 hot_teams[level].hot_team_nth = team->t.t_nproc; 1035 } 1036 } else { 1037 use_hot_team = 0; 1038 } 1039 } 1040 #else 1041 use_hot_team = team == root->r.r_hot_team; 1042 #endif 1043 if (!use_hot_team) { 1044 1045 /* install the master thread */ 1046 team->t.t_threads[0] = master_th; 1047 __kmp_initialize_info(master_th, team, 0, master_gtid); 1048 1049 /* now, install the worker threads */ 1050 for (i = 1; i < team->t.t_nproc; i++) { 1051 1052 /* fork or reallocate a new thread and install it in team */ 1053 kmp_info_t *thr = __kmp_allocate_thread(root, team, i); 1054 team->t.t_threads[i] = thr; 1055 KMP_DEBUG_ASSERT(thr); 1056 KMP_DEBUG_ASSERT(thr->th.th_team == team); 1057 /* align team and thread arrived states */ 1058 KA_TRACE(20, ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived " 1059 "T#%d(%d:%d) join =%llu, plain=%llu\n", 1060 __kmp_gtid_from_tid(0, team), team->t.t_id, 0, 1061 __kmp_gtid_from_tid(i, team), team->t.t_id, i, 1062 team->t.t_bar[bs_forkjoin_barrier].b_arrived, 1063 team->t.t_bar[bs_plain_barrier].b_arrived)); 1064 #if OMP_40_ENABLED 1065 thr->th.th_teams_microtask = master_th->th.th_teams_microtask; 1066 thr->th.th_teams_level = master_th->th.th_teams_level; 1067 thr->th.th_teams_size = master_th->th.th_teams_size; 1068 #endif 1069 { // Initialize threads' barrier data. 1070 int b; 1071 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar; 1072 for (b = 0; b < bs_last_barrier; ++b) { 1073 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 1074 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 1075 #if USE_DEBUGGER 1076 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 1077 #endif 1078 } 1079 } 1080 } 1081 1082 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 1083 __kmp_partition_places(team); 1084 #endif 1085 } 1086 1087 KMP_MB(); 1088 } 1089 1090 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1091 // Propagate any changes to the floating point control registers out to the team 1092 // We try to avoid unnecessary writes to the relevant cache line in the team 1093 // structure, so we don't make changes unless they are needed. 1094 inline static void propagateFPControl(kmp_team_t *team) { 1095 if (__kmp_inherit_fp_control) { 1096 kmp_int16 x87_fpu_control_word; 1097 kmp_uint32 mxcsr; 1098 1099 // Get master values of FPU control flags (both X87 and vector) 1100 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word); 1101 __kmp_store_mxcsr(&mxcsr); 1102 mxcsr &= KMP_X86_MXCSR_MASK; 1103 1104 // There is no point looking at t_fp_control_saved here. 1105 // If it is TRUE, we still have to update the values if they are different 1106 // from those we now have. If it is FALSE we didn't save anything yet, but 1107 // our objective is the same. We have to ensure that the values in the team 1108 // are the same as those we have. 1109 // So, this code achieves what we need whether or not t_fp_control_saved is 1110 // true. By checking whether the value needs updating we avoid unnecessary 1111 // writes that would put the cache-line into a written state, causing all 1112 // threads in the team to have to read it again. 1113 KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word); 1114 KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr); 1115 // Although we don't use this value, other code in the runtime wants to know 1116 // whether it should restore them. So we must ensure it is correct. 1117 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE); 1118 } else { 1119 // Similarly here. Don't write to this cache-line in the team structure 1120 // unless we have to. 1121 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE); 1122 } 1123 } 1124 1125 // Do the opposite, setting the hardware registers to the updated values from 1126 // the team. 1127 inline static void updateHWFPControl(kmp_team_t *team) { 1128 if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) { 1129 // Only reset the fp control regs if they have been changed in the team. 1130 // the parallel region that we are exiting. 1131 kmp_int16 x87_fpu_control_word; 1132 kmp_uint32 mxcsr; 1133 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word); 1134 __kmp_store_mxcsr(&mxcsr); 1135 mxcsr &= KMP_X86_MXCSR_MASK; 1136 1137 if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) { 1138 __kmp_clear_x87_fpu_status_word(); 1139 __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word); 1140 } 1141 1142 if (team->t.t_mxcsr != mxcsr) { 1143 __kmp_load_mxcsr(&team->t.t_mxcsr); 1144 } 1145 } 1146 } 1147 #else 1148 #define propagateFPControl(x) ((void)0) 1149 #define updateHWFPControl(x) ((void)0) 1150 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 1151 1152 static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team, 1153 int realloc); // forward declaration 1154 1155 /* Run a parallel region that has been serialized, so runs only in a team of the 1156 single master thread. */ 1157 void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) { 1158 kmp_info_t *this_thr; 1159 kmp_team_t *serial_team; 1160 1161 KC_TRACE(10, ("__kmpc_serialized_parallel: called by T#%d\n", global_tid)); 1162 1163 /* Skip all this code for autopar serialized loops since it results in 1164 unacceptable overhead */ 1165 if (loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR)) 1166 return; 1167 1168 if (!TCR_4(__kmp_init_parallel)) 1169 __kmp_parallel_initialize(); 1170 1171 this_thr = __kmp_threads[global_tid]; 1172 serial_team = this_thr->th.th_serial_team; 1173 1174 /* utilize the serialized team held by this thread */ 1175 KMP_DEBUG_ASSERT(serial_team); 1176 KMP_MB(); 1177 1178 if (__kmp_tasking_mode != tskm_immediate_exec) { 1179 KMP_DEBUG_ASSERT( 1180 this_thr->th.th_task_team == 1181 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]); 1182 KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] == 1183 NULL); 1184 KA_TRACE(20, ("__kmpc_serialized_parallel: T#%d pushing task_team %p / " 1185 "team %p, new task_team = NULL\n", 1186 global_tid, this_thr->th.th_task_team, this_thr->th.th_team)); 1187 this_thr->th.th_task_team = NULL; 1188 } 1189 1190 #if OMP_40_ENABLED 1191 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind; 1192 if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) { 1193 proc_bind = proc_bind_false; 1194 } else if (proc_bind == proc_bind_default) { 1195 // No proc_bind clause was specified, so use the current value 1196 // of proc-bind-var for this parallel region. 1197 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind; 1198 } 1199 // Reset for next parallel region 1200 this_thr->th.th_set_proc_bind = proc_bind_default; 1201 #endif /* OMP_40_ENABLED */ 1202 1203 #if OMPT_SUPPORT 1204 ompt_data_t ompt_parallel_data; 1205 ompt_parallel_data.ptr = NULL; 1206 ompt_data_t *implicit_task_data; 1207 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid); 1208 if (ompt_enabled.enabled && 1209 this_thr->th.ompt_thread_info.state != omp_state_overhead) { 1210 1211 ompt_task_info_t *parent_task_info; 1212 parent_task_info = OMPT_CUR_TASK_INFO(this_thr); 1213 1214 parent_task_info->frame.enter_frame = OMPT_GET_FRAME_ADDRESS(1); 1215 if (ompt_enabled.ompt_callback_parallel_begin) { 1216 int team_size = 1; 1217 1218 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)( 1219 &(parent_task_info->task_data), &(parent_task_info->frame), 1220 &ompt_parallel_data, team_size, ompt_invoker_program, codeptr); 1221 } 1222 } 1223 #endif // OMPT_SUPPORT 1224 1225 if (this_thr->th.th_team != serial_team) { 1226 // Nested level will be an index in the nested nthreads array 1227 int level = this_thr->th.th_team->t.t_level; 1228 1229 if (serial_team->t.t_serialized) { 1230 /* this serial team was already used 1231 TODO increase performance by making this locks more specific */ 1232 kmp_team_t *new_team; 1233 1234 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 1235 1236 new_team = __kmp_allocate_team(this_thr->th.th_root, 1, 1, 1237 #if OMPT_SUPPORT 1238 ompt_parallel_data, 1239 #endif 1240 #if OMP_40_ENABLED 1241 proc_bind, 1242 #endif 1243 &this_thr->th.th_current_task->td_icvs, 1244 0 USE_NESTED_HOT_ARG(NULL)); 1245 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 1246 KMP_ASSERT(new_team); 1247 1248 /* setup new serialized team and install it */ 1249 new_team->t.t_threads[0] = this_thr; 1250 new_team->t.t_parent = this_thr->th.th_team; 1251 serial_team = new_team; 1252 this_thr->th.th_serial_team = serial_team; 1253 1254 KF_TRACE( 1255 10, 1256 ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n", 1257 global_tid, serial_team)); 1258 1259 /* TODO the above breaks the requirement that if we run out of resources, 1260 then we can still guarantee that serialized teams are ok, since we may 1261 need to allocate a new one */ 1262 } else { 1263 KF_TRACE( 1264 10, 1265 ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n", 1266 global_tid, serial_team)); 1267 } 1268 1269 /* we have to initialize this serial team */ 1270 KMP_DEBUG_ASSERT(serial_team->t.t_threads); 1271 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr); 1272 KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team); 1273 serial_team->t.t_ident = loc; 1274 serial_team->t.t_serialized = 1; 1275 serial_team->t.t_nproc = 1; 1276 serial_team->t.t_parent = this_thr->th.th_team; 1277 serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched; 1278 this_thr->th.th_team = serial_team; 1279 serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid; 1280 1281 KF_TRACE(10, ("__kmpc_serialized_parallel: T#d curtask=%p\n", global_tid, 1282 this_thr->th.th_current_task)); 1283 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1); 1284 this_thr->th.th_current_task->td_flags.executing = 0; 1285 1286 __kmp_push_current_task_to_thread(this_thr, serial_team, 0); 1287 1288 /* TODO: GEH: do ICVs work for nested serialized teams? Don't we need an 1289 implicit task for each serialized task represented by 1290 team->t.t_serialized? */ 1291 copy_icvs(&this_thr->th.th_current_task->td_icvs, 1292 &this_thr->th.th_current_task->td_parent->td_icvs); 1293 1294 // Thread value exists in the nested nthreads array for the next nested 1295 // level 1296 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) { 1297 this_thr->th.th_current_task->td_icvs.nproc = 1298 __kmp_nested_nth.nth[level + 1]; 1299 } 1300 1301 #if OMP_40_ENABLED 1302 if (__kmp_nested_proc_bind.used && 1303 (level + 1 < __kmp_nested_proc_bind.used)) { 1304 this_thr->th.th_current_task->td_icvs.proc_bind = 1305 __kmp_nested_proc_bind.bind_types[level + 1]; 1306 } 1307 #endif /* OMP_40_ENABLED */ 1308 1309 #if USE_DEBUGGER 1310 serial_team->t.t_pkfn = (microtask_t)(~0); // For the debugger. 1311 #endif 1312 this_thr->th.th_info.ds.ds_tid = 0; 1313 1314 /* set thread cache values */ 1315 this_thr->th.th_team_nproc = 1; 1316 this_thr->th.th_team_master = this_thr; 1317 this_thr->th.th_team_serialized = 1; 1318 1319 serial_team->t.t_level = serial_team->t.t_parent->t.t_level + 1; 1320 serial_team->t.t_active_level = serial_team->t.t_parent->t.t_active_level; 1321 1322 propagateFPControl(serial_team); 1323 1324 /* check if we need to allocate dispatch buffers stack */ 1325 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch); 1326 if (!serial_team->t.t_dispatch->th_disp_buffer) { 1327 serial_team->t.t_dispatch->th_disp_buffer = 1328 (dispatch_private_info_t *)__kmp_allocate( 1329 sizeof(dispatch_private_info_t)); 1330 } 1331 this_thr->th.th_dispatch = serial_team->t.t_dispatch; 1332 1333 KMP_MB(); 1334 1335 } else { 1336 /* this serialized team is already being used, 1337 * that's fine, just add another nested level */ 1338 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team); 1339 KMP_DEBUG_ASSERT(serial_team->t.t_threads); 1340 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr); 1341 ++serial_team->t.t_serialized; 1342 this_thr->th.th_team_serialized = serial_team->t.t_serialized; 1343 1344 // Nested level will be an index in the nested nthreads array 1345 int level = this_thr->th.th_team->t.t_level; 1346 // Thread value exists in the nested nthreads array for the next nested 1347 // level 1348 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) { 1349 this_thr->th.th_current_task->td_icvs.nproc = 1350 __kmp_nested_nth.nth[level + 1]; 1351 } 1352 serial_team->t.t_level++; 1353 KF_TRACE(10, ("__kmpc_serialized_parallel: T#%d increasing nesting level " 1354 "of serial team %p to %d\n", 1355 global_tid, serial_team, serial_team->t.t_level)); 1356 1357 /* allocate/push dispatch buffers stack */ 1358 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch); 1359 { 1360 dispatch_private_info_t *disp_buffer = 1361 (dispatch_private_info_t *)__kmp_allocate( 1362 sizeof(dispatch_private_info_t)); 1363 disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer; 1364 serial_team->t.t_dispatch->th_disp_buffer = disp_buffer; 1365 } 1366 this_thr->th.th_dispatch = serial_team->t.t_dispatch; 1367 1368 KMP_MB(); 1369 } 1370 #if OMP_40_ENABLED 1371 KMP_CHECK_UPDATE(serial_team->t.t_cancel_request, cancel_noreq); 1372 #endif 1373 1374 if (__kmp_env_consistency_check) 1375 __kmp_push_parallel(global_tid, NULL); 1376 #if OMPT_SUPPORT 1377 serial_team->t.ompt_team_info.master_return_address = codeptr; 1378 if (ompt_enabled.enabled && 1379 this_thr->th.ompt_thread_info.state != omp_state_overhead) { 1380 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame = OMPT_GET_FRAME_ADDRESS(1); 1381 1382 ompt_lw_taskteam_t lw_taskteam; 1383 __ompt_lw_taskteam_init(&lw_taskteam, this_thr, global_tid, 1384 &ompt_parallel_data, codeptr); 1385 1386 __ompt_lw_taskteam_link(&lw_taskteam, this_thr, 1); 1387 // don't use lw_taskteam after linking. content was swaped 1388 1389 /* OMPT implicit task begin */ 1390 implicit_task_data = OMPT_CUR_TASK_DATA(this_thr); 1391 if (ompt_enabled.ompt_callback_implicit_task) { 1392 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1393 ompt_scope_begin, OMPT_CUR_TEAM_DATA(this_thr), 1394 OMPT_CUR_TASK_DATA(this_thr), 1, __kmp_tid_from_gtid(global_tid)); 1395 OMPT_CUR_TASK_INFO(this_thr) 1396 ->thread_num = __kmp_tid_from_gtid(global_tid); 1397 } 1398 1399 /* OMPT state */ 1400 this_thr->th.ompt_thread_info.state = omp_state_work_parallel; 1401 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame = OMPT_GET_FRAME_ADDRESS(1); 1402 } 1403 #endif 1404 } 1405 1406 /* most of the work for a fork */ 1407 /* return true if we really went parallel, false if serialized */ 1408 int __kmp_fork_call(ident_t *loc, int gtid, 1409 enum fork_context_e call_context, // Intel, GNU, ... 1410 kmp_int32 argc, microtask_t microtask, launch_t invoker, 1411 /* TODO: revert workaround for Intel(R) 64 tracker #96 */ 1412 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 1413 va_list *ap 1414 #else 1415 va_list ap 1416 #endif 1417 ) { 1418 void **argv; 1419 int i; 1420 int master_tid; 1421 int master_this_cons; 1422 kmp_team_t *team; 1423 kmp_team_t *parent_team; 1424 kmp_info_t *master_th; 1425 kmp_root_t *root; 1426 int nthreads; 1427 int master_active; 1428 int master_set_numthreads; 1429 int level; 1430 #if OMP_40_ENABLED 1431 int active_level; 1432 int teams_level; 1433 #endif 1434 #if KMP_NESTED_HOT_TEAMS 1435 kmp_hot_team_ptr_t **p_hot_teams; 1436 #endif 1437 { // KMP_TIME_BLOCK 1438 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_fork_call); 1439 KMP_COUNT_VALUE(OMP_PARALLEL_args, argc); 1440 1441 KA_TRACE(20, ("__kmp_fork_call: enter T#%d\n", gtid)); 1442 if (__kmp_stkpadding > 0 && __kmp_root[gtid] != NULL) { 1443 /* Some systems prefer the stack for the root thread(s) to start with */ 1444 /* some gap from the parent stack to prevent false sharing. */ 1445 void *dummy = KMP_ALLOCA(__kmp_stkpadding); 1446 /* These 2 lines below are so this does not get optimized out */ 1447 if (__kmp_stkpadding > KMP_MAX_STKPADDING) 1448 __kmp_stkpadding += (short)((kmp_int64)dummy); 1449 } 1450 1451 /* initialize if needed */ 1452 KMP_DEBUG_ASSERT( 1453 __kmp_init_serial); // AC: potentially unsafe, not in sync with shutdown 1454 if (!TCR_4(__kmp_init_parallel)) 1455 __kmp_parallel_initialize(); 1456 1457 /* setup current data */ 1458 master_th = __kmp_threads[gtid]; // AC: potentially unsafe, not in sync with 1459 // shutdown 1460 parent_team = master_th->th.th_team; 1461 master_tid = master_th->th.th_info.ds.ds_tid; 1462 master_this_cons = master_th->th.th_local.this_construct; 1463 root = master_th->th.th_root; 1464 master_active = root->r.r_active; 1465 master_set_numthreads = master_th->th.th_set_nproc; 1466 1467 #if OMPT_SUPPORT 1468 ompt_data_t ompt_parallel_data; 1469 ompt_parallel_data.ptr = NULL; 1470 ompt_data_t *parent_task_data; 1471 omp_frame_t *ompt_frame; 1472 ompt_data_t *implicit_task_data; 1473 void *return_address = NULL; 1474 1475 if (ompt_enabled.enabled) { 1476 __ompt_get_task_info_internal(0, NULL, &parent_task_data, &ompt_frame, 1477 NULL, NULL); 1478 return_address = OMPT_LOAD_RETURN_ADDRESS(gtid); 1479 } 1480 #endif 1481 1482 // Nested level will be an index in the nested nthreads array 1483 level = parent_team->t.t_level; 1484 // used to launch non-serial teams even if nested is not allowed 1485 active_level = parent_team->t.t_active_level; 1486 #if OMP_40_ENABLED 1487 // needed to check nesting inside the teams 1488 teams_level = master_th->th.th_teams_level; 1489 #endif 1490 #if KMP_NESTED_HOT_TEAMS 1491 p_hot_teams = &master_th->th.th_hot_teams; 1492 if (*p_hot_teams == NULL && __kmp_hot_teams_max_level > 0) { 1493 *p_hot_teams = (kmp_hot_team_ptr_t *)__kmp_allocate( 1494 sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level); 1495 (*p_hot_teams)[0].hot_team = root->r.r_hot_team; 1496 // it is either actual or not needed (when active_level > 0) 1497 (*p_hot_teams)[0].hot_team_nth = 1; 1498 } 1499 #endif 1500 1501 #if OMPT_SUPPORT 1502 if (ompt_enabled.enabled) { 1503 if (ompt_enabled.ompt_callback_parallel_begin) { 1504 int team_size = master_set_numthreads 1505 ? master_set_numthreads 1506 : get__nproc_2(parent_team, master_tid); 1507 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)( 1508 parent_task_data, ompt_frame, &ompt_parallel_data, team_size, 1509 OMPT_INVOKER(call_context), return_address); 1510 } 1511 master_th->th.ompt_thread_info.state = omp_state_overhead; 1512 } 1513 #endif 1514 1515 master_th->th.th_ident = loc; 1516 1517 #if OMP_40_ENABLED 1518 if (master_th->th.th_teams_microtask && ap && 1519 microtask != (microtask_t)__kmp_teams_master && level == teams_level) { 1520 // AC: This is start of parallel that is nested inside teams construct. 1521 // The team is actual (hot), all workers are ready at the fork barrier. 1522 // No lock needed to initialize the team a bit, then free workers. 1523 parent_team->t.t_ident = loc; 1524 __kmp_alloc_argv_entries(argc, parent_team, TRUE); 1525 parent_team->t.t_argc = argc; 1526 argv = (void **)parent_team->t.t_argv; 1527 for (i = argc - 1; i >= 0; --i) 1528 /* TODO: revert workaround for Intel(R) 64 tracker #96 */ 1529 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 1530 *argv++ = va_arg(*ap, void *); 1531 #else 1532 *argv++ = va_arg(ap, void *); 1533 #endif 1534 // Increment our nested depth levels, but not increase the serialization 1535 if (parent_team == master_th->th.th_serial_team) { 1536 // AC: we are in serialized parallel 1537 __kmpc_serialized_parallel(loc, gtid); 1538 KMP_DEBUG_ASSERT(parent_team->t.t_serialized > 1); 1539 // AC: need this in order enquiry functions work 1540 // correctly, will restore at join time 1541 parent_team->t.t_serialized--; 1542 #if OMPT_SUPPORT 1543 void *dummy; 1544 void **exit_runtime_p; 1545 1546 ompt_lw_taskteam_t lw_taskteam; 1547 1548 if (ompt_enabled.enabled) { 1549 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid, 1550 &ompt_parallel_data, return_address); 1551 exit_runtime_p = &(lw_taskteam.ompt_task_info.frame.exit_frame); 1552 1553 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0); 1554 // don't use lw_taskteam after linking. content was swaped 1555 1556 /* OMPT implicit task begin */ 1557 implicit_task_data = OMPT_CUR_TASK_DATA(master_th); 1558 if (ompt_enabled.ompt_callback_implicit_task) { 1559 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1560 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th), 1561 implicit_task_data, 1, __kmp_tid_from_gtid(gtid)); 1562 OMPT_CUR_TASK_INFO(master_th) 1563 ->thread_num = __kmp_tid_from_gtid(gtid); 1564 } 1565 1566 /* OMPT state */ 1567 master_th->th.ompt_thread_info.state = omp_state_work_parallel; 1568 } else { 1569 exit_runtime_p = &dummy; 1570 } 1571 #endif 1572 1573 { 1574 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 1575 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 1576 __kmp_invoke_microtask(microtask, gtid, 0, argc, parent_team->t.t_argv 1577 #if OMPT_SUPPORT 1578 , 1579 exit_runtime_p 1580 #endif 1581 ); 1582 } 1583 1584 #if OMPT_SUPPORT 1585 *exit_runtime_p = NULL; 1586 if (ompt_enabled.enabled) { 1587 OMPT_CUR_TASK_INFO(master_th)->frame.exit_frame = NULL; 1588 if (ompt_enabled.ompt_callback_implicit_task) { 1589 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1590 ompt_scope_end, NULL, implicit_task_data, 1, 1591 OMPT_CUR_TASK_INFO(master_th)->thread_num); 1592 } 1593 __ompt_lw_taskteam_unlink(master_th); 1594 1595 if (ompt_enabled.ompt_callback_parallel_end) { 1596 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)( 1597 OMPT_CUR_TEAM_DATA(master_th), OMPT_CUR_TASK_DATA(master_th), 1598 OMPT_INVOKER(call_context), return_address); 1599 } 1600 master_th->th.ompt_thread_info.state = omp_state_overhead; 1601 } 1602 #endif 1603 return TRUE; 1604 } 1605 1606 parent_team->t.t_pkfn = microtask; 1607 parent_team->t.t_invoke = invoker; 1608 KMP_ATOMIC_INC(&root->r.r_in_parallel); 1609 parent_team->t.t_active_level++; 1610 parent_team->t.t_level++; 1611 1612 /* Change number of threads in the team if requested */ 1613 if (master_set_numthreads) { // The parallel has num_threads clause 1614 if (master_set_numthreads < master_th->th.th_teams_size.nth) { 1615 // AC: only can reduce number of threads dynamically, can't increase 1616 kmp_info_t **other_threads = parent_team->t.t_threads; 1617 parent_team->t.t_nproc = master_set_numthreads; 1618 for (i = 0; i < master_set_numthreads; ++i) { 1619 other_threads[i]->th.th_team_nproc = master_set_numthreads; 1620 } 1621 // Keep extra threads hot in the team for possible next parallels 1622 } 1623 master_th->th.th_set_nproc = 0; 1624 } 1625 1626 #if USE_DEBUGGER 1627 if (__kmp_debugging) { // Let debugger override number of threads. 1628 int nth = __kmp_omp_num_threads(loc); 1629 if (nth > 0) { // 0 means debugger doesn't want to change num threads 1630 master_set_numthreads = nth; 1631 } 1632 } 1633 #endif 1634 1635 KF_TRACE(10, ("__kmp_fork_call: before internal fork: root=%p, team=%p, " 1636 "master_th=%p, gtid=%d\n", 1637 root, parent_team, master_th, gtid)); 1638 __kmp_internal_fork(loc, gtid, parent_team); 1639 KF_TRACE(10, ("__kmp_fork_call: after internal fork: root=%p, team=%p, " 1640 "master_th=%p, gtid=%d\n", 1641 root, parent_team, master_th, gtid)); 1642 1643 /* Invoke microtask for MASTER thread */ 1644 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid, 1645 parent_team->t.t_id, parent_team->t.t_pkfn)); 1646 1647 if (!parent_team->t.t_invoke(gtid)) { 1648 KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread"); 1649 } 1650 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid, 1651 parent_team->t.t_id, parent_team->t.t_pkfn)); 1652 KMP_MB(); /* Flush all pending memory write invalidates. */ 1653 1654 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid)); 1655 1656 return TRUE; 1657 } // Parallel closely nested in teams construct 1658 #endif /* OMP_40_ENABLED */ 1659 1660 #if KMP_DEBUG 1661 if (__kmp_tasking_mode != tskm_immediate_exec) { 1662 KMP_DEBUG_ASSERT(master_th->th.th_task_team == 1663 parent_team->t.t_task_team[master_th->th.th_task_state]); 1664 } 1665 #endif 1666 1667 if (parent_team->t.t_active_level >= 1668 master_th->th.th_current_task->td_icvs.max_active_levels) { 1669 nthreads = 1; 1670 } else { 1671 #if OMP_40_ENABLED 1672 int enter_teams = ((ap == NULL && active_level == 0) || 1673 (ap && teams_level > 0 && teams_level == level)); 1674 #endif 1675 nthreads = 1676 master_set_numthreads 1677 ? master_set_numthreads 1678 : get__nproc_2( 1679 parent_team, 1680 master_tid); // TODO: get nproc directly from current task 1681 1682 // Check if we need to take forkjoin lock? (no need for serialized 1683 // parallel out of teams construct). This code moved here from 1684 // __kmp_reserve_threads() to speedup nested serialized parallels. 1685 if (nthreads > 1) { 1686 if ((!get__nested(master_th) && (root->r.r_in_parallel 1687 #if OMP_40_ENABLED 1688 && !enter_teams 1689 #endif /* OMP_40_ENABLED */ 1690 )) || 1691 (__kmp_library == library_serial)) { 1692 KC_TRACE(10, ("__kmp_fork_call: T#%d serializing team; requested %d" 1693 " threads\n", 1694 gtid, nthreads)); 1695 nthreads = 1; 1696 } 1697 } 1698 if (nthreads > 1) { 1699 /* determine how many new threads we can use */ 1700 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 1701 nthreads = __kmp_reserve_threads( 1702 root, parent_team, master_tid, nthreads 1703 #if OMP_40_ENABLED 1704 /* AC: If we execute teams from parallel region (on host), then 1705 teams should be created but each can only have 1 thread if 1706 nesting is disabled. If teams called from serial region, then 1707 teams and their threads should be created regardless of the 1708 nesting setting. */ 1709 , 1710 enter_teams 1711 #endif /* OMP_40_ENABLED */ 1712 ); 1713 if (nthreads == 1) { 1714 // Free lock for single thread execution here; for multi-thread 1715 // execution it will be freed later after team of threads created 1716 // and initialized 1717 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 1718 } 1719 } 1720 } 1721 KMP_DEBUG_ASSERT(nthreads > 0); 1722 1723 // If we temporarily changed the set number of threads then restore it now 1724 master_th->th.th_set_nproc = 0; 1725 1726 /* create a serialized parallel region? */ 1727 if (nthreads == 1) { 1728 /* josh todo: hypothetical question: what do we do for OS X*? */ 1729 #if KMP_OS_LINUX && \ 1730 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 1731 void *args[argc]; 1732 #else 1733 void **args = (void **)KMP_ALLOCA(argc * sizeof(void *)); 1734 #endif /* KMP_OS_LINUX && ( KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || \ 1735 KMP_ARCH_AARCH64) */ 1736 1737 KA_TRACE(20, 1738 ("__kmp_fork_call: T#%d serializing parallel region\n", gtid)); 1739 1740 __kmpc_serialized_parallel(loc, gtid); 1741 1742 if (call_context == fork_context_intel) { 1743 /* TODO this sucks, use the compiler itself to pass args! :) */ 1744 master_th->th.th_serial_team->t.t_ident = loc; 1745 #if OMP_40_ENABLED 1746 if (!ap) { 1747 // revert change made in __kmpc_serialized_parallel() 1748 master_th->th.th_serial_team->t.t_level--; 1749 // Get args from parent team for teams construct 1750 1751 #if OMPT_SUPPORT 1752 void *dummy; 1753 void **exit_runtime_p; 1754 ompt_task_info_t *task_info; 1755 1756 ompt_lw_taskteam_t lw_taskteam; 1757 1758 if (ompt_enabled.enabled) { 1759 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid, 1760 &ompt_parallel_data, return_address); 1761 1762 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0); 1763 // don't use lw_taskteam after linking. content was swaped 1764 1765 task_info = OMPT_CUR_TASK_INFO(master_th); 1766 exit_runtime_p = &(task_info->frame.exit_frame); 1767 if (ompt_enabled.ompt_callback_implicit_task) { 1768 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1769 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th), 1770 &(task_info->task_data), 1, __kmp_tid_from_gtid(gtid)); 1771 OMPT_CUR_TASK_INFO(master_th) 1772 ->thread_num = __kmp_tid_from_gtid(gtid); 1773 } 1774 1775 /* OMPT state */ 1776 master_th->th.ompt_thread_info.state = omp_state_work_parallel; 1777 } else { 1778 exit_runtime_p = &dummy; 1779 } 1780 #endif 1781 1782 { 1783 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 1784 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 1785 __kmp_invoke_microtask(microtask, gtid, 0, argc, 1786 parent_team->t.t_argv 1787 #if OMPT_SUPPORT 1788 , 1789 exit_runtime_p 1790 #endif 1791 ); 1792 } 1793 1794 #if OMPT_SUPPORT 1795 if (ompt_enabled.enabled) { 1796 exit_runtime_p = NULL; 1797 if (ompt_enabled.ompt_callback_implicit_task) { 1798 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1799 ompt_scope_end, NULL, &(task_info->task_data), 1, 1800 OMPT_CUR_TASK_INFO(master_th)->thread_num); 1801 } 1802 1803 __ompt_lw_taskteam_unlink(master_th); 1804 if (ompt_enabled.ompt_callback_parallel_end) { 1805 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)( 1806 OMPT_CUR_TEAM_DATA(master_th), parent_task_data, 1807 OMPT_INVOKER(call_context), return_address); 1808 } 1809 master_th->th.ompt_thread_info.state = omp_state_overhead; 1810 } 1811 #endif 1812 } else if (microtask == (microtask_t)__kmp_teams_master) { 1813 KMP_DEBUG_ASSERT(master_th->th.th_team == 1814 master_th->th.th_serial_team); 1815 team = master_th->th.th_team; 1816 // team->t.t_pkfn = microtask; 1817 team->t.t_invoke = invoker; 1818 __kmp_alloc_argv_entries(argc, team, TRUE); 1819 team->t.t_argc = argc; 1820 argv = (void **)team->t.t_argv; 1821 if (ap) { 1822 for (i = argc - 1; i >= 0; --i) 1823 // TODO: revert workaround for Intel(R) 64 tracker #96 1824 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 1825 *argv++ = va_arg(*ap, void *); 1826 #else 1827 *argv++ = va_arg(ap, void *); 1828 #endif 1829 } else { 1830 for (i = 0; i < argc; ++i) 1831 // Get args from parent team for teams construct 1832 argv[i] = parent_team->t.t_argv[i]; 1833 } 1834 // AC: revert change made in __kmpc_serialized_parallel() 1835 // because initial code in teams should have level=0 1836 team->t.t_level--; 1837 // AC: call special invoker for outer "parallel" of teams construct 1838 invoker(gtid); 1839 } else { 1840 #endif /* OMP_40_ENABLED */ 1841 argv = args; 1842 for (i = argc - 1; i >= 0; --i) 1843 // TODO: revert workaround for Intel(R) 64 tracker #96 1844 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 1845 *argv++ = va_arg(*ap, void *); 1846 #else 1847 *argv++ = va_arg(ap, void *); 1848 #endif 1849 KMP_MB(); 1850 1851 #if OMPT_SUPPORT 1852 void *dummy; 1853 void **exit_runtime_p; 1854 ompt_task_info_t *task_info; 1855 1856 ompt_lw_taskteam_t lw_taskteam; 1857 1858 if (ompt_enabled.enabled) { 1859 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid, 1860 &ompt_parallel_data, return_address); 1861 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0); 1862 // don't use lw_taskteam after linking. content was swaped 1863 task_info = OMPT_CUR_TASK_INFO(master_th); 1864 exit_runtime_p = &(task_info->frame.exit_frame); 1865 1866 /* OMPT implicit task begin */ 1867 implicit_task_data = OMPT_CUR_TASK_DATA(master_th); 1868 if (ompt_enabled.ompt_callback_implicit_task) { 1869 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1870 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th), 1871 implicit_task_data, 1, __kmp_tid_from_gtid(gtid)); 1872 OMPT_CUR_TASK_INFO(master_th) 1873 ->thread_num = __kmp_tid_from_gtid(gtid); 1874 } 1875 1876 /* OMPT state */ 1877 master_th->th.ompt_thread_info.state = omp_state_work_parallel; 1878 } else { 1879 exit_runtime_p = &dummy; 1880 } 1881 #endif 1882 1883 { 1884 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 1885 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 1886 __kmp_invoke_microtask(microtask, gtid, 0, argc, args 1887 #if OMPT_SUPPORT 1888 , 1889 exit_runtime_p 1890 #endif 1891 ); 1892 } 1893 1894 #if OMPT_SUPPORT 1895 if (ompt_enabled.enabled) { 1896 *exit_runtime_p = NULL; 1897 if (ompt_enabled.ompt_callback_implicit_task) { 1898 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 1899 ompt_scope_end, NULL, &(task_info->task_data), 1, 1900 OMPT_CUR_TASK_INFO(master_th)->thread_num); 1901 } 1902 1903 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th); 1904 __ompt_lw_taskteam_unlink(master_th); 1905 if (ompt_enabled.ompt_callback_parallel_end) { 1906 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)( 1907 &ompt_parallel_data, parent_task_data, 1908 OMPT_INVOKER(call_context), return_address); 1909 } 1910 master_th->th.ompt_thread_info.state = omp_state_overhead; 1911 } 1912 #endif 1913 #if OMP_40_ENABLED 1914 } 1915 #endif /* OMP_40_ENABLED */ 1916 } else if (call_context == fork_context_gnu) { 1917 #if OMPT_SUPPORT 1918 ompt_lw_taskteam_t lwt; 1919 __ompt_lw_taskteam_init(&lwt, master_th, gtid, &ompt_parallel_data, 1920 return_address); 1921 1922 lwt.ompt_task_info.frame.exit_frame = NULL; 1923 __ompt_lw_taskteam_link(&lwt, master_th, 1); 1924 // don't use lw_taskteam after linking. content was swaped 1925 #endif 1926 1927 // we were called from GNU native code 1928 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid)); 1929 return FALSE; 1930 } else { 1931 KMP_ASSERT2(call_context < fork_context_last, 1932 "__kmp_fork_call: unknown fork_context parameter"); 1933 } 1934 1935 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid)); 1936 KMP_MB(); 1937 return FALSE; 1938 } 1939 1940 // GEH: only modify the executing flag in the case when not serialized 1941 // serialized case is handled in kmpc_serialized_parallel 1942 KF_TRACE(10, ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, " 1943 "curtask=%p, curtask_max_aclevel=%d\n", 1944 parent_team->t.t_active_level, master_th, 1945 master_th->th.th_current_task, 1946 master_th->th.th_current_task->td_icvs.max_active_levels)); 1947 // TODO: GEH - cannot do this assertion because root thread not set up as 1948 // executing 1949 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 1 ); 1950 master_th->th.th_current_task->td_flags.executing = 0; 1951 1952 #if OMP_40_ENABLED 1953 if (!master_th->th.th_teams_microtask || level > teams_level) 1954 #endif /* OMP_40_ENABLED */ 1955 { 1956 /* Increment our nested depth level */ 1957 KMP_ATOMIC_INC(&root->r.r_in_parallel); 1958 } 1959 1960 // See if we need to make a copy of the ICVs. 1961 int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc; 1962 if ((level + 1 < __kmp_nested_nth.used) && 1963 (__kmp_nested_nth.nth[level + 1] != nthreads_icv)) { 1964 nthreads_icv = __kmp_nested_nth.nth[level + 1]; 1965 } else { 1966 nthreads_icv = 0; // don't update 1967 } 1968 1969 #if OMP_40_ENABLED 1970 // Figure out the proc_bind_policy for the new team. 1971 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind; 1972 kmp_proc_bind_t proc_bind_icv = 1973 proc_bind_default; // proc_bind_default means don't update 1974 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) { 1975 proc_bind = proc_bind_false; 1976 } else { 1977 if (proc_bind == proc_bind_default) { 1978 // No proc_bind clause specified; use current proc-bind-var for this 1979 // parallel region 1980 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind; 1981 } 1982 /* else: The proc_bind policy was specified explicitly on parallel clause. 1983 This overrides proc-bind-var for this parallel region, but does not 1984 change proc-bind-var. */ 1985 // Figure the value of proc-bind-var for the child threads. 1986 if ((level + 1 < __kmp_nested_proc_bind.used) && 1987 (__kmp_nested_proc_bind.bind_types[level + 1] != 1988 master_th->th.th_current_task->td_icvs.proc_bind)) { 1989 proc_bind_icv = __kmp_nested_proc_bind.bind_types[level + 1]; 1990 } 1991 } 1992 1993 // Reset for next parallel region 1994 master_th->th.th_set_proc_bind = proc_bind_default; 1995 #endif /* OMP_40_ENABLED */ 1996 1997 if ((nthreads_icv > 0) 1998 #if OMP_40_ENABLED 1999 || (proc_bind_icv != proc_bind_default) 2000 #endif /* OMP_40_ENABLED */ 2001 ) { 2002 kmp_internal_control_t new_icvs; 2003 copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs); 2004 new_icvs.next = NULL; 2005 if (nthreads_icv > 0) { 2006 new_icvs.nproc = nthreads_icv; 2007 } 2008 2009 #if OMP_40_ENABLED 2010 if (proc_bind_icv != proc_bind_default) { 2011 new_icvs.proc_bind = proc_bind_icv; 2012 } 2013 #endif /* OMP_40_ENABLED */ 2014 2015 /* allocate a new parallel team */ 2016 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n")); 2017 team = __kmp_allocate_team(root, nthreads, nthreads, 2018 #if OMPT_SUPPORT 2019 ompt_parallel_data, 2020 #endif 2021 #if OMP_40_ENABLED 2022 proc_bind, 2023 #endif 2024 &new_icvs, argc USE_NESTED_HOT_ARG(master_th)); 2025 } else { 2026 /* allocate a new parallel team */ 2027 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n")); 2028 team = __kmp_allocate_team(root, nthreads, nthreads, 2029 #if OMPT_SUPPORT 2030 ompt_parallel_data, 2031 #endif 2032 #if OMP_40_ENABLED 2033 proc_bind, 2034 #endif 2035 &master_th->th.th_current_task->td_icvs, 2036 argc USE_NESTED_HOT_ARG(master_th)); 2037 } 2038 KF_TRACE( 2039 10, ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team)); 2040 2041 /* setup the new team */ 2042 KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid); 2043 KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons); 2044 KMP_CHECK_UPDATE(team->t.t_ident, loc); 2045 KMP_CHECK_UPDATE(team->t.t_parent, parent_team); 2046 KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask); 2047 #if OMPT_SUPPORT 2048 KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.master_return_address, 2049 return_address); 2050 #endif 2051 KMP_CHECK_UPDATE(team->t.t_invoke, invoker); // TODO move to root, maybe 2052 // TODO: parent_team->t.t_level == INT_MAX ??? 2053 #if OMP_40_ENABLED 2054 if (!master_th->th.th_teams_microtask || level > teams_level) { 2055 #endif /* OMP_40_ENABLED */ 2056 int new_level = parent_team->t.t_level + 1; 2057 KMP_CHECK_UPDATE(team->t.t_level, new_level); 2058 new_level = parent_team->t.t_active_level + 1; 2059 KMP_CHECK_UPDATE(team->t.t_active_level, new_level); 2060 #if OMP_40_ENABLED 2061 } else { 2062 // AC: Do not increase parallel level at start of the teams construct 2063 int new_level = parent_team->t.t_level; 2064 KMP_CHECK_UPDATE(team->t.t_level, new_level); 2065 new_level = parent_team->t.t_active_level; 2066 KMP_CHECK_UPDATE(team->t.t_active_level, new_level); 2067 } 2068 #endif /* OMP_40_ENABLED */ 2069 kmp_r_sched_t new_sched = get__sched_2(parent_team, master_tid); 2070 // set master's schedule as new run-time schedule 2071 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched); 2072 2073 #if OMP_40_ENABLED 2074 KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq); 2075 #endif 2076 2077 // Update the floating point rounding in the team if required. 2078 propagateFPControl(team); 2079 2080 if (__kmp_tasking_mode != tskm_immediate_exec) { 2081 // Set master's task team to team's task team. Unless this is hot team, it 2082 // should be NULL. 2083 KMP_DEBUG_ASSERT(master_th->th.th_task_team == 2084 parent_team->t.t_task_team[master_th->th.th_task_state]); 2085 KA_TRACE(20, ("__kmp_fork_call: Master T#%d pushing task_team %p / team " 2086 "%p, new task_team %p / team %p\n", 2087 __kmp_gtid_from_thread(master_th), 2088 master_th->th.th_task_team, parent_team, 2089 team->t.t_task_team[master_th->th.th_task_state], team)); 2090 2091 if (active_level || master_th->th.th_task_team) { 2092 // Take a memo of master's task_state 2093 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack); 2094 if (master_th->th.th_task_state_top >= 2095 master_th->th.th_task_state_stack_sz) { // increase size 2096 kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz; 2097 kmp_uint8 *old_stack, *new_stack; 2098 kmp_uint32 i; 2099 new_stack = (kmp_uint8 *)__kmp_allocate(new_size); 2100 for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) { 2101 new_stack[i] = master_th->th.th_task_state_memo_stack[i]; 2102 } 2103 for (i = master_th->th.th_task_state_stack_sz; i < new_size; 2104 ++i) { // zero-init rest of stack 2105 new_stack[i] = 0; 2106 } 2107 old_stack = master_th->th.th_task_state_memo_stack; 2108 master_th->th.th_task_state_memo_stack = new_stack; 2109 master_th->th.th_task_state_stack_sz = new_size; 2110 __kmp_free(old_stack); 2111 } 2112 // Store master's task_state on stack 2113 master_th->th 2114 .th_task_state_memo_stack[master_th->th.th_task_state_top] = 2115 master_th->th.th_task_state; 2116 master_th->th.th_task_state_top++; 2117 #if KMP_NESTED_HOT_TEAMS 2118 if (master_th->th.th_hot_teams && 2119 team == master_th->th.th_hot_teams[active_level].hot_team) { 2120 // Restore master's nested state if nested hot team 2121 master_th->th.th_task_state = 2122 master_th->th 2123 .th_task_state_memo_stack[master_th->th.th_task_state_top]; 2124 } else { 2125 #endif 2126 master_th->th.th_task_state = 0; 2127 #if KMP_NESTED_HOT_TEAMS 2128 } 2129 #endif 2130 } 2131 #if !KMP_NESTED_HOT_TEAMS 2132 KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) || 2133 (team == root->r.r_hot_team)); 2134 #endif 2135 } 2136 2137 KA_TRACE( 2138 20, 2139 ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n", 2140 gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id, 2141 team->t.t_nproc)); 2142 KMP_DEBUG_ASSERT(team != root->r.r_hot_team || 2143 (team->t.t_master_tid == 0 && 2144 (team->t.t_parent == root->r.r_root_team || 2145 team->t.t_parent->t.t_serialized))); 2146 KMP_MB(); 2147 2148 /* now, setup the arguments */ 2149 argv = (void **)team->t.t_argv; 2150 #if OMP_40_ENABLED 2151 if (ap) { 2152 #endif /* OMP_40_ENABLED */ 2153 for (i = argc - 1; i >= 0; --i) { 2154 // TODO: revert workaround for Intel(R) 64 tracker #96 2155 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX 2156 void *new_argv = va_arg(*ap, void *); 2157 #else 2158 void *new_argv = va_arg(ap, void *); 2159 #endif 2160 KMP_CHECK_UPDATE(*argv, new_argv); 2161 argv++; 2162 } 2163 #if OMP_40_ENABLED 2164 } else { 2165 for (i = 0; i < argc; ++i) { 2166 // Get args from parent team for teams construct 2167 KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i]); 2168 } 2169 } 2170 #endif /* OMP_40_ENABLED */ 2171 2172 /* now actually fork the threads */ 2173 KMP_CHECK_UPDATE(team->t.t_master_active, master_active); 2174 if (!root->r.r_active) // Only do assignment if it prevents cache ping-pong 2175 root->r.r_active = TRUE; 2176 2177 __kmp_fork_team_threads(root, team, master_th, gtid); 2178 __kmp_setup_icv_copy(team, nthreads, 2179 &master_th->th.th_current_task->td_icvs, loc); 2180 2181 #if OMPT_SUPPORT 2182 master_th->th.ompt_thread_info.state = omp_state_work_parallel; 2183 #endif 2184 2185 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 2186 2187 #if USE_ITT_BUILD 2188 if (team->t.t_active_level == 1 // only report frames at level 1 2189 #if OMP_40_ENABLED 2190 && !master_th->th.th_teams_microtask // not in teams construct 2191 #endif /* OMP_40_ENABLED */ 2192 ) { 2193 #if USE_ITT_NOTIFY 2194 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) && 2195 (__kmp_forkjoin_frames_mode == 3 || 2196 __kmp_forkjoin_frames_mode == 1)) { 2197 kmp_uint64 tmp_time = 0; 2198 if (__itt_get_timestamp_ptr) 2199 tmp_time = __itt_get_timestamp(); 2200 // Internal fork - report frame begin 2201 master_th->th.th_frame_time = tmp_time; 2202 if (__kmp_forkjoin_frames_mode == 3) 2203 team->t.t_region_time = tmp_time; 2204 } else 2205 // only one notification scheme (either "submit" or "forking/joined", not both) 2206 #endif /* USE_ITT_NOTIFY */ 2207 if ((__itt_frame_begin_v3_ptr || KMP_ITT_DEBUG) && 2208 __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode) { 2209 // Mark start of "parallel" region for Intel(R) VTune(TM) analyzer. 2210 __kmp_itt_region_forking(gtid, team->t.t_nproc, 0); 2211 } 2212 } 2213 #endif /* USE_ITT_BUILD */ 2214 2215 /* now go on and do the work */ 2216 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team); 2217 KMP_MB(); 2218 KF_TRACE(10, 2219 ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n", 2220 root, team, master_th, gtid)); 2221 2222 #if USE_ITT_BUILD 2223 if (__itt_stack_caller_create_ptr) { 2224 team->t.t_stack_id = 2225 __kmp_itt_stack_caller_create(); // create new stack stitching id 2226 // before entering fork barrier 2227 } 2228 #endif /* USE_ITT_BUILD */ 2229 2230 #if OMP_40_ENABLED 2231 // AC: skip __kmp_internal_fork at teams construct, let only master 2232 // threads execute 2233 if (ap) 2234 #endif /* OMP_40_ENABLED */ 2235 { 2236 __kmp_internal_fork(loc, gtid, team); 2237 KF_TRACE(10, ("__kmp_internal_fork : after : root=%p, team=%p, " 2238 "master_th=%p, gtid=%d\n", 2239 root, team, master_th, gtid)); 2240 } 2241 2242 if (call_context == fork_context_gnu) { 2243 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid)); 2244 return TRUE; 2245 } 2246 2247 /* Invoke microtask for MASTER thread */ 2248 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid, 2249 team->t.t_id, team->t.t_pkfn)); 2250 } // END of timer KMP_fork_call block 2251 2252 if (!team->t.t_invoke(gtid)) { 2253 KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread"); 2254 } 2255 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid, 2256 team->t.t_id, team->t.t_pkfn)); 2257 KMP_MB(); /* Flush all pending memory write invalidates. */ 2258 2259 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid)); 2260 2261 #if OMPT_SUPPORT 2262 if (ompt_enabled.enabled) { 2263 master_th->th.ompt_thread_info.state = omp_state_overhead; 2264 } 2265 #endif 2266 2267 return TRUE; 2268 } 2269 2270 #if OMPT_SUPPORT 2271 static inline void __kmp_join_restore_state(kmp_info_t *thread, 2272 kmp_team_t *team) { 2273 // restore state outside the region 2274 thread->th.ompt_thread_info.state = 2275 ((team->t.t_serialized) ? omp_state_work_serial 2276 : omp_state_work_parallel); 2277 } 2278 2279 static inline void __kmp_join_ompt(int gtid, kmp_info_t *thread, 2280 kmp_team_t *team, ompt_data_t *parallel_data, 2281 fork_context_e fork_context, void *codeptr) { 2282 ompt_task_info_t *task_info = __ompt_get_task_info_object(0); 2283 if (ompt_enabled.ompt_callback_parallel_end) { 2284 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)( 2285 parallel_data, &(task_info->task_data), OMPT_INVOKER(fork_context), 2286 codeptr); 2287 } 2288 2289 task_info->frame.enter_frame = NULL; 2290 __kmp_join_restore_state(thread, team); 2291 } 2292 #endif 2293 2294 void __kmp_join_call(ident_t *loc, int gtid 2295 #if OMPT_SUPPORT 2296 , 2297 enum fork_context_e fork_context 2298 #endif 2299 #if OMP_40_ENABLED 2300 , 2301 int exit_teams 2302 #endif /* OMP_40_ENABLED */ 2303 ) { 2304 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_join_call); 2305 kmp_team_t *team; 2306 kmp_team_t *parent_team; 2307 kmp_info_t *master_th; 2308 kmp_root_t *root; 2309 int master_active; 2310 int i; 2311 2312 KA_TRACE(20, ("__kmp_join_call: enter T#%d\n", gtid)); 2313 2314 /* setup current data */ 2315 master_th = __kmp_threads[gtid]; 2316 root = master_th->th.th_root; 2317 team = master_th->th.th_team; 2318 parent_team = team->t.t_parent; 2319 2320 master_th->th.th_ident = loc; 2321 2322 #if OMPT_SUPPORT 2323 if (ompt_enabled.enabled) { 2324 master_th->th.ompt_thread_info.state = omp_state_overhead; 2325 } 2326 #endif 2327 2328 #if KMP_DEBUG 2329 if (__kmp_tasking_mode != tskm_immediate_exec && !exit_teams) { 2330 KA_TRACE(20, ("__kmp_join_call: T#%d, old team = %p old task_team = %p, " 2331 "th_task_team = %p\n", 2332 __kmp_gtid_from_thread(master_th), team, 2333 team->t.t_task_team[master_th->th.th_task_state], 2334 master_th->th.th_task_team)); 2335 KMP_DEBUG_ASSERT(master_th->th.th_task_team == 2336 team->t.t_task_team[master_th->th.th_task_state]); 2337 } 2338 #endif 2339 2340 if (team->t.t_serialized) { 2341 #if OMP_40_ENABLED 2342 if (master_th->th.th_teams_microtask) { 2343 // We are in teams construct 2344 int level = team->t.t_level; 2345 int tlevel = master_th->th.th_teams_level; 2346 if (level == tlevel) { 2347 // AC: we haven't incremented it earlier at start of teams construct, 2348 // so do it here - at the end of teams construct 2349 team->t.t_level++; 2350 } else if (level == tlevel + 1) { 2351 // AC: we are exiting parallel inside teams, need to increment 2352 // serialization in order to restore it in the next call to 2353 // __kmpc_end_serialized_parallel 2354 team->t.t_serialized++; 2355 } 2356 } 2357 #endif /* OMP_40_ENABLED */ 2358 __kmpc_end_serialized_parallel(loc, gtid); 2359 2360 #if OMPT_SUPPORT 2361 if (ompt_enabled.enabled) { 2362 __kmp_join_restore_state(master_th, parent_team); 2363 } 2364 #endif 2365 2366 return; 2367 } 2368 2369 master_active = team->t.t_master_active; 2370 2371 #if OMP_40_ENABLED 2372 if (!exit_teams) 2373 #endif /* OMP_40_ENABLED */ 2374 { 2375 // AC: No barrier for internal teams at exit from teams construct. 2376 // But there is barrier for external team (league). 2377 __kmp_internal_join(loc, gtid, team); 2378 } 2379 #if OMP_40_ENABLED 2380 else { 2381 master_th->th.th_task_state = 2382 0; // AC: no tasking in teams (out of any parallel) 2383 } 2384 #endif /* OMP_40_ENABLED */ 2385 2386 KMP_MB(); 2387 2388 #if OMPT_SUPPORT 2389 ompt_data_t *parallel_data = &(team->t.ompt_team_info.parallel_data); 2390 void *codeptr = team->t.ompt_team_info.master_return_address; 2391 #endif 2392 2393 #if USE_ITT_BUILD 2394 if (__itt_stack_caller_create_ptr) { 2395 __kmp_itt_stack_caller_destroy( 2396 (__itt_caller)team->t 2397 .t_stack_id); // destroy the stack stitching id after join barrier 2398 } 2399 2400 // Mark end of "parallel" region for Intel(R) VTune(TM) analyzer. 2401 if (team->t.t_active_level == 1 2402 #if OMP_40_ENABLED 2403 && !master_th->th.th_teams_microtask /* not in teams construct */ 2404 #endif /* OMP_40_ENABLED */ 2405 ) { 2406 master_th->th.th_ident = loc; 2407 // only one notification scheme (either "submit" or "forking/joined", not 2408 // both) 2409 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) && 2410 __kmp_forkjoin_frames_mode == 3) 2411 __kmp_itt_frame_submit(gtid, team->t.t_region_time, 2412 master_th->th.th_frame_time, 0, loc, 2413 master_th->th.th_team_nproc, 1); 2414 else if ((__itt_frame_end_v3_ptr || KMP_ITT_DEBUG) && 2415 !__kmp_forkjoin_frames_mode && __kmp_forkjoin_frames) 2416 __kmp_itt_region_joined(gtid); 2417 } // active_level == 1 2418 #endif /* USE_ITT_BUILD */ 2419 2420 #if OMP_40_ENABLED 2421 if (master_th->th.th_teams_microtask && !exit_teams && 2422 team->t.t_pkfn != (microtask_t)__kmp_teams_master && 2423 team->t.t_level == master_th->th.th_teams_level + 1) { 2424 // AC: We need to leave the team structure intact at the end of parallel 2425 // inside the teams construct, so that at the next parallel same (hot) team 2426 // works, only adjust nesting levels 2427 2428 /* Decrement our nested depth level */ 2429 team->t.t_level--; 2430 team->t.t_active_level--; 2431 KMP_ATOMIC_DEC(&root->r.r_in_parallel); 2432 2433 /* Restore number of threads in the team if needed */ 2434 if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) { 2435 int old_num = master_th->th.th_team_nproc; 2436 int new_num = master_th->th.th_teams_size.nth; 2437 kmp_info_t **other_threads = team->t.t_threads; 2438 team->t.t_nproc = new_num; 2439 for (i = 0; i < old_num; ++i) { 2440 other_threads[i]->th.th_team_nproc = new_num; 2441 } 2442 // Adjust states of non-used threads of the team 2443 for (i = old_num; i < new_num; ++i) { 2444 // Re-initialize thread's barrier data. 2445 int b; 2446 kmp_balign_t *balign = other_threads[i]->th.th_bar; 2447 for (b = 0; b < bs_last_barrier; ++b) { 2448 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 2449 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 2450 #if USE_DEBUGGER 2451 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 2452 #endif 2453 } 2454 if (__kmp_tasking_mode != tskm_immediate_exec) { 2455 // Synchronize thread's task state 2456 other_threads[i]->th.th_task_state = master_th->th.th_task_state; 2457 } 2458 } 2459 } 2460 2461 #if OMPT_SUPPORT 2462 if (ompt_enabled.enabled) { 2463 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, fork_context, 2464 codeptr); 2465 } 2466 #endif 2467 2468 return; 2469 } 2470 #endif /* OMP_40_ENABLED */ 2471 2472 /* do cleanup and restore the parent team */ 2473 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid; 2474 master_th->th.th_local.this_construct = team->t.t_master_this_cons; 2475 2476 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid]; 2477 2478 /* jc: The following lock has instructions with REL and ACQ semantics, 2479 separating the parallel user code called in this parallel region 2480 from the serial user code called after this function returns. */ 2481 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 2482 2483 #if OMP_40_ENABLED 2484 if (!master_th->th.th_teams_microtask || 2485 team->t.t_level > master_th->th.th_teams_level) 2486 #endif /* OMP_40_ENABLED */ 2487 { 2488 /* Decrement our nested depth level */ 2489 KMP_ATOMIC_DEC(&root->r.r_in_parallel); 2490 } 2491 KMP_DEBUG_ASSERT(root->r.r_in_parallel >= 0); 2492 2493 #if OMPT_SUPPORT 2494 if (ompt_enabled.enabled) { 2495 ompt_task_info_t *task_info = __ompt_get_task_info_object(0); 2496 if (ompt_enabled.ompt_callback_implicit_task) { 2497 int ompt_team_size = team->t.t_nproc; 2498 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 2499 ompt_scope_end, NULL, &(task_info->task_data), ompt_team_size, 2500 OMPT_CUR_TASK_INFO(master_th)->thread_num); 2501 } 2502 2503 task_info->frame.exit_frame = NULL; 2504 task_info->task_data = ompt_data_none; 2505 } 2506 #endif 2507 2508 KF_TRACE(10, ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n", 0, 2509 master_th, team)); 2510 __kmp_pop_current_task_from_thread(master_th); 2511 2512 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 2513 // Restore master thread's partition. 2514 master_th->th.th_first_place = team->t.t_first_place; 2515 master_th->th.th_last_place = team->t.t_last_place; 2516 #endif /* OMP_40_ENABLED */ 2517 2518 updateHWFPControl(team); 2519 2520 if (root->r.r_active != master_active) 2521 root->r.r_active = master_active; 2522 2523 __kmp_free_team(root, team USE_NESTED_HOT_ARG( 2524 master_th)); // this will free worker threads 2525 2526 /* this race was fun to find. make sure the following is in the critical 2527 region otherwise assertions may fail occasionally since the old team may be 2528 reallocated and the hierarchy appears inconsistent. it is actually safe to 2529 run and won't cause any bugs, but will cause those assertion failures. it's 2530 only one deref&assign so might as well put this in the critical region */ 2531 master_th->th.th_team = parent_team; 2532 master_th->th.th_team_nproc = parent_team->t.t_nproc; 2533 master_th->th.th_team_master = parent_team->t.t_threads[0]; 2534 master_th->th.th_team_serialized = parent_team->t.t_serialized; 2535 2536 /* restore serialized team, if need be */ 2537 if (parent_team->t.t_serialized && 2538 parent_team != master_th->th.th_serial_team && 2539 parent_team != root->r.r_root_team) { 2540 __kmp_free_team(root, 2541 master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL)); 2542 master_th->th.th_serial_team = parent_team; 2543 } 2544 2545 if (__kmp_tasking_mode != tskm_immediate_exec) { 2546 if (master_th->th.th_task_state_top > 2547 0) { // Restore task state from memo stack 2548 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack); 2549 // Remember master's state if we re-use this nested hot team 2550 master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] = 2551 master_th->th.th_task_state; 2552 --master_th->th.th_task_state_top; // pop 2553 // Now restore state at this level 2554 master_th->th.th_task_state = 2555 master_th->th 2556 .th_task_state_memo_stack[master_th->th.th_task_state_top]; 2557 } 2558 // Copy the task team from the parent team to the master thread 2559 master_th->th.th_task_team = 2560 parent_team->t.t_task_team[master_th->th.th_task_state]; 2561 KA_TRACE(20, 2562 ("__kmp_join_call: Master T#%d restoring task_team %p / team %p\n", 2563 __kmp_gtid_from_thread(master_th), master_th->th.th_task_team, 2564 parent_team)); 2565 } 2566 2567 // TODO: GEH - cannot do this assertion because root thread not set up as 2568 // executing 2569 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 0 ); 2570 master_th->th.th_current_task->td_flags.executing = 1; 2571 2572 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 2573 2574 #if OMPT_SUPPORT 2575 if (ompt_enabled.enabled) { 2576 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, fork_context, 2577 codeptr); 2578 } 2579 #endif 2580 2581 KMP_MB(); 2582 KA_TRACE(20, ("__kmp_join_call: exit T#%d\n", gtid)); 2583 } 2584 2585 /* Check whether we should push an internal control record onto the 2586 serial team stack. If so, do it. */ 2587 void __kmp_save_internal_controls(kmp_info_t *thread) { 2588 2589 if (thread->th.th_team != thread->th.th_serial_team) { 2590 return; 2591 } 2592 if (thread->th.th_team->t.t_serialized > 1) { 2593 int push = 0; 2594 2595 if (thread->th.th_team->t.t_control_stack_top == NULL) { 2596 push = 1; 2597 } else { 2598 if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level != 2599 thread->th.th_team->t.t_serialized) { 2600 push = 1; 2601 } 2602 } 2603 if (push) { /* push a record on the serial team's stack */ 2604 kmp_internal_control_t *control = 2605 (kmp_internal_control_t *)__kmp_allocate( 2606 sizeof(kmp_internal_control_t)); 2607 2608 copy_icvs(control, &thread->th.th_current_task->td_icvs); 2609 2610 control->serial_nesting_level = thread->th.th_team->t.t_serialized; 2611 2612 control->next = thread->th.th_team->t.t_control_stack_top; 2613 thread->th.th_team->t.t_control_stack_top = control; 2614 } 2615 } 2616 } 2617 2618 /* Changes set_nproc */ 2619 void __kmp_set_num_threads(int new_nth, int gtid) { 2620 kmp_info_t *thread; 2621 kmp_root_t *root; 2622 2623 KF_TRACE(10, ("__kmp_set_num_threads: new __kmp_nth = %d\n", new_nth)); 2624 KMP_DEBUG_ASSERT(__kmp_init_serial); 2625 2626 if (new_nth < 1) 2627 new_nth = 1; 2628 else if (new_nth > __kmp_max_nth) 2629 new_nth = __kmp_max_nth; 2630 2631 KMP_COUNT_VALUE(OMP_set_numthreads, new_nth); 2632 thread = __kmp_threads[gtid]; 2633 2634 __kmp_save_internal_controls(thread); 2635 2636 set__nproc(thread, new_nth); 2637 2638 // If this omp_set_num_threads() call will cause the hot team size to be 2639 // reduced (in the absence of a num_threads clause), then reduce it now, 2640 // rather than waiting for the next parallel region. 2641 root = thread->th.th_root; 2642 if (__kmp_init_parallel && (!root->r.r_active) && 2643 (root->r.r_hot_team->t.t_nproc > new_nth) 2644 #if KMP_NESTED_HOT_TEAMS 2645 && __kmp_hot_teams_max_level && !__kmp_hot_teams_mode 2646 #endif 2647 ) { 2648 kmp_team_t *hot_team = root->r.r_hot_team; 2649 int f; 2650 2651 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 2652 2653 // Release the extra threads we don't need any more. 2654 for (f = new_nth; f < hot_team->t.t_nproc; f++) { 2655 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL); 2656 if (__kmp_tasking_mode != tskm_immediate_exec) { 2657 // When decreasing team size, threads no longer in the team should unref 2658 // task team. 2659 hot_team->t.t_threads[f]->th.th_task_team = NULL; 2660 } 2661 __kmp_free_thread(hot_team->t.t_threads[f]); 2662 hot_team->t.t_threads[f] = NULL; 2663 } 2664 hot_team->t.t_nproc = new_nth; 2665 #if KMP_NESTED_HOT_TEAMS 2666 if (thread->th.th_hot_teams) { 2667 KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team); 2668 thread->th.th_hot_teams[0].hot_team_nth = new_nth; 2669 } 2670 #endif 2671 2672 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 2673 2674 // Update the t_nproc field in the threads that are still active. 2675 for (f = 0; f < new_nth; f++) { 2676 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL); 2677 hot_team->t.t_threads[f]->th.th_team_nproc = new_nth; 2678 } 2679 // Special flag in case omp_set_num_threads() call 2680 hot_team->t.t_size_changed = -1; 2681 } 2682 } 2683 2684 /* Changes max_active_levels */ 2685 void __kmp_set_max_active_levels(int gtid, int max_active_levels) { 2686 kmp_info_t *thread; 2687 2688 KF_TRACE(10, ("__kmp_set_max_active_levels: new max_active_levels for thread " 2689 "%d = (%d)\n", 2690 gtid, max_active_levels)); 2691 KMP_DEBUG_ASSERT(__kmp_init_serial); 2692 2693 // validate max_active_levels 2694 if (max_active_levels < 0) { 2695 KMP_WARNING(ActiveLevelsNegative, max_active_levels); 2696 // We ignore this call if the user has specified a negative value. 2697 // The current setting won't be changed. The last valid setting will be 2698 // used. A warning will be issued (if warnings are allowed as controlled by 2699 // the KMP_WARNINGS env var). 2700 KF_TRACE(10, ("__kmp_set_max_active_levels: the call is ignored: new " 2701 "max_active_levels for thread %d = (%d)\n", 2702 gtid, max_active_levels)); 2703 return; 2704 } 2705 if (max_active_levels <= KMP_MAX_ACTIVE_LEVELS_LIMIT) { 2706 // it's OK, the max_active_levels is within the valid range: [ 0; 2707 // KMP_MAX_ACTIVE_LEVELS_LIMIT ] 2708 // We allow a zero value. (implementation defined behavior) 2709 } else { 2710 KMP_WARNING(ActiveLevelsExceedLimit, max_active_levels, 2711 KMP_MAX_ACTIVE_LEVELS_LIMIT); 2712 max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT; 2713 // Current upper limit is MAX_INT. (implementation defined behavior) 2714 // If the input exceeds the upper limit, we correct the input to be the 2715 // upper limit. (implementation defined behavior) 2716 // Actually, the flow should never get here until we use MAX_INT limit. 2717 } 2718 KF_TRACE(10, ("__kmp_set_max_active_levels: after validation: new " 2719 "max_active_levels for thread %d = (%d)\n", 2720 gtid, max_active_levels)); 2721 2722 thread = __kmp_threads[gtid]; 2723 2724 __kmp_save_internal_controls(thread); 2725 2726 set__max_active_levels(thread, max_active_levels); 2727 } 2728 2729 /* Gets max_active_levels */ 2730 int __kmp_get_max_active_levels(int gtid) { 2731 kmp_info_t *thread; 2732 2733 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d\n", gtid)); 2734 KMP_DEBUG_ASSERT(__kmp_init_serial); 2735 2736 thread = __kmp_threads[gtid]; 2737 KMP_DEBUG_ASSERT(thread->th.th_current_task); 2738 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d, curtask=%p, " 2739 "curtask_maxaclevel=%d\n", 2740 gtid, thread->th.th_current_task, 2741 thread->th.th_current_task->td_icvs.max_active_levels)); 2742 return thread->th.th_current_task->td_icvs.max_active_levels; 2743 } 2744 2745 /* Changes def_sched_var ICV values (run-time schedule kind and chunk) */ 2746 void __kmp_set_schedule(int gtid, kmp_sched_t kind, int chunk) { 2747 kmp_info_t *thread; 2748 // kmp_team_t *team; 2749 2750 KF_TRACE(10, ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n", 2751 gtid, (int)kind, chunk)); 2752 KMP_DEBUG_ASSERT(__kmp_init_serial); 2753 2754 // Check if the kind parameter is valid, correct if needed. 2755 // Valid parameters should fit in one of two intervals - standard or extended: 2756 // <lower>, <valid>, <upper_std>, <lower_ext>, <valid>, <upper> 2757 // 2008-01-25: 0, 1 - 4, 5, 100, 101 - 102, 103 2758 if (kind <= kmp_sched_lower || kind >= kmp_sched_upper || 2759 (kind <= kmp_sched_lower_ext && kind >= kmp_sched_upper_std)) { 2760 // TODO: Hint needs attention in case we change the default schedule. 2761 __kmp_msg(kmp_ms_warning, KMP_MSG(ScheduleKindOutOfRange, kind), 2762 KMP_HNT(DefaultScheduleKindUsed, "static, no chunk"), 2763 __kmp_msg_null); 2764 kind = kmp_sched_default; 2765 chunk = 0; // ignore chunk value in case of bad kind 2766 } 2767 2768 thread = __kmp_threads[gtid]; 2769 2770 __kmp_save_internal_controls(thread); 2771 2772 if (kind < kmp_sched_upper_std) { 2773 if (kind == kmp_sched_static && chunk < KMP_DEFAULT_CHUNK) { 2774 // differ static chunked vs. unchunked: chunk should be invalid to 2775 // indicate unchunked schedule (which is the default) 2776 thread->th.th_current_task->td_icvs.sched.r_sched_type = kmp_sch_static; 2777 } else { 2778 thread->th.th_current_task->td_icvs.sched.r_sched_type = 2779 __kmp_sch_map[kind - kmp_sched_lower - 1]; 2780 } 2781 } else { 2782 // __kmp_sch_map[ kind - kmp_sched_lower_ext + kmp_sched_upper_std - 2783 // kmp_sched_lower - 2 ]; 2784 thread->th.th_current_task->td_icvs.sched.r_sched_type = 2785 __kmp_sch_map[kind - kmp_sched_lower_ext + kmp_sched_upper_std - 2786 kmp_sched_lower - 2]; 2787 } 2788 if (kind == kmp_sched_auto || chunk < 1) { 2789 // ignore parameter chunk for schedule auto 2790 thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK; 2791 } else { 2792 thread->th.th_current_task->td_icvs.sched.chunk = chunk; 2793 } 2794 } 2795 2796 /* Gets def_sched_var ICV values */ 2797 void __kmp_get_schedule(int gtid, kmp_sched_t *kind, int *chunk) { 2798 kmp_info_t *thread; 2799 enum sched_type th_type; 2800 2801 KF_TRACE(10, ("__kmp_get_schedule: thread %d\n", gtid)); 2802 KMP_DEBUG_ASSERT(__kmp_init_serial); 2803 2804 thread = __kmp_threads[gtid]; 2805 2806 th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type; 2807 2808 switch (th_type) { 2809 case kmp_sch_static: 2810 case kmp_sch_static_greedy: 2811 case kmp_sch_static_balanced: 2812 *kind = kmp_sched_static; 2813 *chunk = 0; // chunk was not set, try to show this fact via zero value 2814 return; 2815 case kmp_sch_static_chunked: 2816 *kind = kmp_sched_static; 2817 break; 2818 case kmp_sch_dynamic_chunked: 2819 *kind = kmp_sched_dynamic; 2820 break; 2821 case kmp_sch_guided_chunked: 2822 case kmp_sch_guided_iterative_chunked: 2823 case kmp_sch_guided_analytical_chunked: 2824 *kind = kmp_sched_guided; 2825 break; 2826 case kmp_sch_auto: 2827 *kind = kmp_sched_auto; 2828 break; 2829 case kmp_sch_trapezoidal: 2830 *kind = kmp_sched_trapezoidal; 2831 break; 2832 #if KMP_STATIC_STEAL_ENABLED 2833 case kmp_sch_static_steal: 2834 *kind = kmp_sched_static_steal; 2835 break; 2836 #endif 2837 default: 2838 KMP_FATAL(UnknownSchedulingType, th_type); 2839 } 2840 2841 *chunk = thread->th.th_current_task->td_icvs.sched.chunk; 2842 } 2843 2844 int __kmp_get_ancestor_thread_num(int gtid, int level) { 2845 2846 int ii, dd; 2847 kmp_team_t *team; 2848 kmp_info_t *thr; 2849 2850 KF_TRACE(10, ("__kmp_get_ancestor_thread_num: thread %d %d\n", gtid, level)); 2851 KMP_DEBUG_ASSERT(__kmp_init_serial); 2852 2853 // validate level 2854 if (level == 0) 2855 return 0; 2856 if (level < 0) 2857 return -1; 2858 thr = __kmp_threads[gtid]; 2859 team = thr->th.th_team; 2860 ii = team->t.t_level; 2861 if (level > ii) 2862 return -1; 2863 2864 #if OMP_40_ENABLED 2865 if (thr->th.th_teams_microtask) { 2866 // AC: we are in teams region where multiple nested teams have same level 2867 int tlevel = thr->th.th_teams_level; // the level of the teams construct 2868 if (level <= 2869 tlevel) { // otherwise usual algorithm works (will not touch the teams) 2870 KMP_DEBUG_ASSERT(ii >= tlevel); 2871 // AC: As we need to pass by the teams league, we need to artificially 2872 // increase ii 2873 if (ii == tlevel) { 2874 ii += 2; // three teams have same level 2875 } else { 2876 ii++; // two teams have same level 2877 } 2878 } 2879 } 2880 #endif 2881 2882 if (ii == level) 2883 return __kmp_tid_from_gtid(gtid); 2884 2885 dd = team->t.t_serialized; 2886 level++; 2887 while (ii > level) { 2888 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) { 2889 } 2890 if ((team->t.t_serialized) && (!dd)) { 2891 team = team->t.t_parent; 2892 continue; 2893 } 2894 if (ii > level) { 2895 team = team->t.t_parent; 2896 dd = team->t.t_serialized; 2897 ii--; 2898 } 2899 } 2900 2901 return (dd > 1) ? (0) : (team->t.t_master_tid); 2902 } 2903 2904 int __kmp_get_team_size(int gtid, int level) { 2905 2906 int ii, dd; 2907 kmp_team_t *team; 2908 kmp_info_t *thr; 2909 2910 KF_TRACE(10, ("__kmp_get_team_size: thread %d %d\n", gtid, level)); 2911 KMP_DEBUG_ASSERT(__kmp_init_serial); 2912 2913 // validate level 2914 if (level == 0) 2915 return 1; 2916 if (level < 0) 2917 return -1; 2918 thr = __kmp_threads[gtid]; 2919 team = thr->th.th_team; 2920 ii = team->t.t_level; 2921 if (level > ii) 2922 return -1; 2923 2924 #if OMP_40_ENABLED 2925 if (thr->th.th_teams_microtask) { 2926 // AC: we are in teams region where multiple nested teams have same level 2927 int tlevel = thr->th.th_teams_level; // the level of the teams construct 2928 if (level <= 2929 tlevel) { // otherwise usual algorithm works (will not touch the teams) 2930 KMP_DEBUG_ASSERT(ii >= tlevel); 2931 // AC: As we need to pass by the teams league, we need to artificially 2932 // increase ii 2933 if (ii == tlevel) { 2934 ii += 2; // three teams have same level 2935 } else { 2936 ii++; // two teams have same level 2937 } 2938 } 2939 } 2940 #endif 2941 2942 while (ii > level) { 2943 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) { 2944 } 2945 if (team->t.t_serialized && (!dd)) { 2946 team = team->t.t_parent; 2947 continue; 2948 } 2949 if (ii > level) { 2950 team = team->t.t_parent; 2951 ii--; 2952 } 2953 } 2954 2955 return team->t.t_nproc; 2956 } 2957 2958 kmp_r_sched_t __kmp_get_schedule_global() { 2959 // This routine created because pairs (__kmp_sched, __kmp_chunk) and 2960 // (__kmp_static, __kmp_guided) may be changed by kmp_set_defaults 2961 // independently. So one can get the updated schedule here. 2962 2963 kmp_r_sched_t r_sched; 2964 2965 // create schedule from 4 globals: __kmp_sched, __kmp_chunk, __kmp_static, 2966 // __kmp_guided. __kmp_sched should keep original value, so that user can set 2967 // KMP_SCHEDULE multiple times, and thus have different run-time schedules in 2968 // different roots (even in OMP 2.5) 2969 if (__kmp_sched == kmp_sch_static) { 2970 // replace STATIC with more detailed schedule (balanced or greedy) 2971 r_sched.r_sched_type = __kmp_static; 2972 } else if (__kmp_sched == kmp_sch_guided_chunked) { 2973 // replace GUIDED with more detailed schedule (iterative or analytical) 2974 r_sched.r_sched_type = __kmp_guided; 2975 } else { // (STATIC_CHUNKED), or (DYNAMIC_CHUNKED), or other 2976 r_sched.r_sched_type = __kmp_sched; 2977 } 2978 2979 if (__kmp_chunk < KMP_DEFAULT_CHUNK) { 2980 // __kmp_chunk may be wrong here (if it was not ever set) 2981 r_sched.chunk = KMP_DEFAULT_CHUNK; 2982 } else { 2983 r_sched.chunk = __kmp_chunk; 2984 } 2985 2986 return r_sched; 2987 } 2988 2989 /* Allocate (realloc == FALSE) * or reallocate (realloc == TRUE) 2990 at least argc number of *t_argv entries for the requested team. */ 2991 static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team, int realloc) { 2992 2993 KMP_DEBUG_ASSERT(team); 2994 if (!realloc || argc > team->t.t_max_argc) { 2995 2996 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: needed entries=%d, " 2997 "current entries=%d\n", 2998 team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0)); 2999 /* if previously allocated heap space for args, free them */ 3000 if (realloc && team->t.t_argv != &team->t.t_inline_argv[0]) 3001 __kmp_free((void *)team->t.t_argv); 3002 3003 if (argc <= KMP_INLINE_ARGV_ENTRIES) { 3004 /* use unused space in the cache line for arguments */ 3005 team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES; 3006 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: inline allocate %d " 3007 "argv entries\n", 3008 team->t.t_id, team->t.t_max_argc)); 3009 team->t.t_argv = &team->t.t_inline_argv[0]; 3010 if (__kmp_storage_map) { 3011 __kmp_print_storage_map_gtid( 3012 -1, &team->t.t_inline_argv[0], 3013 &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES], 3014 (sizeof(void *) * KMP_INLINE_ARGV_ENTRIES), "team_%d.t_inline_argv", 3015 team->t.t_id); 3016 } 3017 } else { 3018 /* allocate space for arguments in the heap */ 3019 team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES >> 1)) 3020 ? KMP_MIN_MALLOC_ARGV_ENTRIES 3021 : 2 * argc; 3022 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d " 3023 "argv entries\n", 3024 team->t.t_id, team->t.t_max_argc)); 3025 team->t.t_argv = 3026 (void **)__kmp_page_allocate(sizeof(void *) * team->t.t_max_argc); 3027 if (__kmp_storage_map) { 3028 __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0], 3029 &team->t.t_argv[team->t.t_max_argc], 3030 sizeof(void *) * team->t.t_max_argc, 3031 "team_%d.t_argv", team->t.t_id); 3032 } 3033 } 3034 } 3035 } 3036 3037 static void __kmp_allocate_team_arrays(kmp_team_t *team, int max_nth) { 3038 int i; 3039 int num_disp_buff = max_nth > 1 ? __kmp_dispatch_num_buffers : 2; 3040 team->t.t_threads = 3041 (kmp_info_t **)__kmp_allocate(sizeof(kmp_info_t *) * max_nth); 3042 team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate( 3043 sizeof(dispatch_shared_info_t) * num_disp_buff); 3044 team->t.t_dispatch = 3045 (kmp_disp_t *)__kmp_allocate(sizeof(kmp_disp_t) * max_nth); 3046 team->t.t_implicit_task_taskdata = 3047 (kmp_taskdata_t *)__kmp_allocate(sizeof(kmp_taskdata_t) * max_nth); 3048 team->t.t_max_nproc = max_nth; 3049 3050 /* setup dispatch buffers */ 3051 for (i = 0; i < num_disp_buff; ++i) { 3052 team->t.t_disp_buffer[i].buffer_index = i; 3053 #if OMP_45_ENABLED 3054 team->t.t_disp_buffer[i].doacross_buf_idx = i; 3055 #endif 3056 } 3057 } 3058 3059 static void __kmp_free_team_arrays(kmp_team_t *team) { 3060 /* Note: this does not free the threads in t_threads (__kmp_free_threads) */ 3061 int i; 3062 for (i = 0; i < team->t.t_max_nproc; ++i) { 3063 if (team->t.t_dispatch[i].th_disp_buffer != NULL) { 3064 __kmp_free(team->t.t_dispatch[i].th_disp_buffer); 3065 team->t.t_dispatch[i].th_disp_buffer = NULL; 3066 } 3067 } 3068 #if KMP_USE_HIER_SCHED 3069 __kmp_dispatch_free_hierarchies(team); 3070 #endif 3071 __kmp_free(team->t.t_threads); 3072 __kmp_free(team->t.t_disp_buffer); 3073 __kmp_free(team->t.t_dispatch); 3074 __kmp_free(team->t.t_implicit_task_taskdata); 3075 team->t.t_threads = NULL; 3076 team->t.t_disp_buffer = NULL; 3077 team->t.t_dispatch = NULL; 3078 team->t.t_implicit_task_taskdata = 0; 3079 } 3080 3081 static void __kmp_reallocate_team_arrays(kmp_team_t *team, int max_nth) { 3082 kmp_info_t **oldThreads = team->t.t_threads; 3083 3084 __kmp_free(team->t.t_disp_buffer); 3085 __kmp_free(team->t.t_dispatch); 3086 __kmp_free(team->t.t_implicit_task_taskdata); 3087 __kmp_allocate_team_arrays(team, max_nth); 3088 3089 KMP_MEMCPY(team->t.t_threads, oldThreads, 3090 team->t.t_nproc * sizeof(kmp_info_t *)); 3091 3092 __kmp_free(oldThreads); 3093 } 3094 3095 static kmp_internal_control_t __kmp_get_global_icvs(void) { 3096 3097 kmp_r_sched_t r_sched = 3098 __kmp_get_schedule_global(); // get current state of scheduling globals 3099 3100 #if OMP_40_ENABLED 3101 KMP_DEBUG_ASSERT(__kmp_nested_proc_bind.used > 0); 3102 #endif /* OMP_40_ENABLED */ 3103 3104 kmp_internal_control_t g_icvs = { 3105 0, // int serial_nesting_level; //corresponds to value of th_team_serialized 3106 (kmp_int8)__kmp_dflt_nested, // int nested; //internal control 3107 // for nested parallelism (per thread) 3108 (kmp_int8)__kmp_global.g.g_dynamic, // internal control for dynamic 3109 // adjustment of threads (per thread) 3110 (kmp_int8)__kmp_env_blocktime, // int bt_set; //internal control for 3111 // whether blocktime is explicitly set 3112 __kmp_dflt_blocktime, // int blocktime; //internal control for blocktime 3113 #if KMP_USE_MONITOR 3114 __kmp_bt_intervals, // int bt_intervals; //internal control for blocktime 3115 // intervals 3116 #endif 3117 __kmp_dflt_team_nth, // int nproc; //internal control for # of threads for 3118 // next parallel region (per thread) 3119 // (use a max ub on value if __kmp_parallel_initialize not called yet) 3120 __kmp_dflt_max_active_levels, // int max_active_levels; //internal control 3121 // for max_active_levels 3122 r_sched, // kmp_r_sched_t sched; //internal control for runtime schedule 3123 // {sched,chunk} pair 3124 #if OMP_40_ENABLED 3125 __kmp_nested_proc_bind.bind_types[0], 3126 __kmp_default_device, 3127 #endif /* OMP_40_ENABLED */ 3128 NULL // struct kmp_internal_control *next; 3129 }; 3130 3131 return g_icvs; 3132 } 3133 3134 static kmp_internal_control_t __kmp_get_x_global_icvs(const kmp_team_t *team) { 3135 3136 kmp_internal_control_t gx_icvs; 3137 gx_icvs.serial_nesting_level = 3138 0; // probably =team->t.t_serial like in save_inter_controls 3139 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs); 3140 gx_icvs.next = NULL; 3141 3142 return gx_icvs; 3143 } 3144 3145 static void __kmp_initialize_root(kmp_root_t *root) { 3146 int f; 3147 kmp_team_t *root_team; 3148 kmp_team_t *hot_team; 3149 int hot_team_max_nth; 3150 kmp_r_sched_t r_sched = 3151 __kmp_get_schedule_global(); // get current state of scheduling globals 3152 kmp_internal_control_t r_icvs = __kmp_get_global_icvs(); 3153 KMP_DEBUG_ASSERT(root); 3154 KMP_ASSERT(!root->r.r_begin); 3155 3156 /* setup the root state structure */ 3157 __kmp_init_lock(&root->r.r_begin_lock); 3158 root->r.r_begin = FALSE; 3159 root->r.r_active = FALSE; 3160 root->r.r_in_parallel = 0; 3161 root->r.r_blocktime = __kmp_dflt_blocktime; 3162 root->r.r_nested = __kmp_dflt_nested; 3163 root->r.r_cg_nthreads = 1; 3164 3165 /* setup the root team for this task */ 3166 /* allocate the root team structure */ 3167 KF_TRACE(10, ("__kmp_initialize_root: before root_team\n")); 3168 3169 root_team = 3170 __kmp_allocate_team(root, 3171 1, // new_nproc 3172 1, // max_nproc 3173 #if OMPT_SUPPORT 3174 ompt_data_none, // root parallel id 3175 #endif 3176 #if OMP_40_ENABLED 3177 __kmp_nested_proc_bind.bind_types[0], 3178 #endif 3179 &r_icvs, 3180 0 // argc 3181 USE_NESTED_HOT_ARG(NULL) // master thread is unknown 3182 ); 3183 #if USE_DEBUGGER 3184 // Non-NULL value should be assigned to make the debugger display the root 3185 // team. 3186 TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)(~0)); 3187 #endif 3188 3189 KF_TRACE(10, ("__kmp_initialize_root: after root_team = %p\n", root_team)); 3190 3191 root->r.r_root_team = root_team; 3192 root_team->t.t_control_stack_top = NULL; 3193 3194 /* initialize root team */ 3195 root_team->t.t_threads[0] = NULL; 3196 root_team->t.t_nproc = 1; 3197 root_team->t.t_serialized = 1; 3198 // TODO???: root_team->t.t_max_active_levels = __kmp_dflt_max_active_levels; 3199 root_team->t.t_sched.sched = r_sched.sched; 3200 KA_TRACE( 3201 20, 3202 ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n", 3203 root_team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); 3204 3205 /* setup the hot team for this task */ 3206 /* allocate the hot team structure */ 3207 KF_TRACE(10, ("__kmp_initialize_root: before hot_team\n")); 3208 3209 hot_team = 3210 __kmp_allocate_team(root, 3211 1, // new_nproc 3212 __kmp_dflt_team_nth_ub * 2, // max_nproc 3213 #if OMPT_SUPPORT 3214 ompt_data_none, // root parallel id 3215 #endif 3216 #if OMP_40_ENABLED 3217 __kmp_nested_proc_bind.bind_types[0], 3218 #endif 3219 &r_icvs, 3220 0 // argc 3221 USE_NESTED_HOT_ARG(NULL) // master thread is unknown 3222 ); 3223 KF_TRACE(10, ("__kmp_initialize_root: after hot_team = %p\n", hot_team)); 3224 3225 root->r.r_hot_team = hot_team; 3226 root_team->t.t_control_stack_top = NULL; 3227 3228 /* first-time initialization */ 3229 hot_team->t.t_parent = root_team; 3230 3231 /* initialize hot team */ 3232 hot_team_max_nth = hot_team->t.t_max_nproc; 3233 for (f = 0; f < hot_team_max_nth; ++f) { 3234 hot_team->t.t_threads[f] = NULL; 3235 } 3236 hot_team->t.t_nproc = 1; 3237 // TODO???: hot_team->t.t_max_active_levels = __kmp_dflt_max_active_levels; 3238 hot_team->t.t_sched.sched = r_sched.sched; 3239 hot_team->t.t_size_changed = 0; 3240 } 3241 3242 #ifdef KMP_DEBUG 3243 3244 typedef struct kmp_team_list_item { 3245 kmp_team_p const *entry; 3246 struct kmp_team_list_item *next; 3247 } kmp_team_list_item_t; 3248 typedef kmp_team_list_item_t *kmp_team_list_t; 3249 3250 static void __kmp_print_structure_team_accum( // Add team to list of teams. 3251 kmp_team_list_t list, // List of teams. 3252 kmp_team_p const *team // Team to add. 3253 ) { 3254 3255 // List must terminate with item where both entry and next are NULL. 3256 // Team is added to the list only once. 3257 // List is sorted in ascending order by team id. 3258 // Team id is *not* a key. 3259 3260 kmp_team_list_t l; 3261 3262 KMP_DEBUG_ASSERT(list != NULL); 3263 if (team == NULL) { 3264 return; 3265 } 3266 3267 __kmp_print_structure_team_accum(list, team->t.t_parent); 3268 __kmp_print_structure_team_accum(list, team->t.t_next_pool); 3269 3270 // Search list for the team. 3271 l = list; 3272 while (l->next != NULL && l->entry != team) { 3273 l = l->next; 3274 } 3275 if (l->next != NULL) { 3276 return; // Team has been added before, exit. 3277 } 3278 3279 // Team is not found. Search list again for insertion point. 3280 l = list; 3281 while (l->next != NULL && l->entry->t.t_id <= team->t.t_id) { 3282 l = l->next; 3283 } 3284 3285 // Insert team. 3286 { 3287 kmp_team_list_item_t *item = (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC( 3288 sizeof(kmp_team_list_item_t)); 3289 *item = *l; 3290 l->entry = team; 3291 l->next = item; 3292 } 3293 } 3294 3295 static void __kmp_print_structure_team(char const *title, kmp_team_p const *team 3296 3297 ) { 3298 __kmp_printf("%s", title); 3299 if (team != NULL) { 3300 __kmp_printf("%2x %p\n", team->t.t_id, team); 3301 } else { 3302 __kmp_printf(" - (nil)\n"); 3303 } 3304 } 3305 3306 static void __kmp_print_structure_thread(char const *title, 3307 kmp_info_p const *thread) { 3308 __kmp_printf("%s", title); 3309 if (thread != NULL) { 3310 __kmp_printf("%2d %p\n", thread->th.th_info.ds.ds_gtid, thread); 3311 } else { 3312 __kmp_printf(" - (nil)\n"); 3313 } 3314 } 3315 3316 void __kmp_print_structure(void) { 3317 3318 kmp_team_list_t list; 3319 3320 // Initialize list of teams. 3321 list = 3322 (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(sizeof(kmp_team_list_item_t)); 3323 list->entry = NULL; 3324 list->next = NULL; 3325 3326 __kmp_printf("\n------------------------------\nGlobal Thread " 3327 "Table\n------------------------------\n"); 3328 { 3329 int gtid; 3330 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) { 3331 __kmp_printf("%2d", gtid); 3332 if (__kmp_threads != NULL) { 3333 __kmp_printf(" %p", __kmp_threads[gtid]); 3334 } 3335 if (__kmp_root != NULL) { 3336 __kmp_printf(" %p", __kmp_root[gtid]); 3337 } 3338 __kmp_printf("\n"); 3339 } 3340 } 3341 3342 // Print out __kmp_threads array. 3343 __kmp_printf("\n------------------------------\nThreads\n--------------------" 3344 "----------\n"); 3345 if (__kmp_threads != NULL) { 3346 int gtid; 3347 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) { 3348 kmp_info_t const *thread = __kmp_threads[gtid]; 3349 if (thread != NULL) { 3350 __kmp_printf("GTID %2d %p:\n", gtid, thread); 3351 __kmp_printf(" Our Root: %p\n", thread->th.th_root); 3352 __kmp_print_structure_team(" Our Team: ", thread->th.th_team); 3353 __kmp_print_structure_team(" Serial Team: ", 3354 thread->th.th_serial_team); 3355 __kmp_printf(" Threads: %2d\n", thread->th.th_team_nproc); 3356 __kmp_print_structure_thread(" Master: ", 3357 thread->th.th_team_master); 3358 __kmp_printf(" Serialized?: %2d\n", thread->th.th_team_serialized); 3359 __kmp_printf(" Set NProc: %2d\n", thread->th.th_set_nproc); 3360 #if OMP_40_ENABLED 3361 __kmp_printf(" Set Proc Bind: %2d\n", thread->th.th_set_proc_bind); 3362 #endif 3363 __kmp_print_structure_thread(" Next in pool: ", 3364 thread->th.th_next_pool); 3365 __kmp_printf("\n"); 3366 __kmp_print_structure_team_accum(list, thread->th.th_team); 3367 __kmp_print_structure_team_accum(list, thread->th.th_serial_team); 3368 } 3369 } 3370 } else { 3371 __kmp_printf("Threads array is not allocated.\n"); 3372 } 3373 3374 // Print out __kmp_root array. 3375 __kmp_printf("\n------------------------------\nUbers\n----------------------" 3376 "--------\n"); 3377 if (__kmp_root != NULL) { 3378 int gtid; 3379 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) { 3380 kmp_root_t const *root = __kmp_root[gtid]; 3381 if (root != NULL) { 3382 __kmp_printf("GTID %2d %p:\n", gtid, root); 3383 __kmp_print_structure_team(" Root Team: ", root->r.r_root_team); 3384 __kmp_print_structure_team(" Hot Team: ", root->r.r_hot_team); 3385 __kmp_print_structure_thread(" Uber Thread: ", 3386 root->r.r_uber_thread); 3387 __kmp_printf(" Active?: %2d\n", root->r.r_active); 3388 __kmp_printf(" Nested?: %2d\n", root->r.r_nested); 3389 __kmp_printf(" In Parallel: %2d\n", 3390 KMP_ATOMIC_LD_RLX(&root->r.r_in_parallel)); 3391 __kmp_printf("\n"); 3392 __kmp_print_structure_team_accum(list, root->r.r_root_team); 3393 __kmp_print_structure_team_accum(list, root->r.r_hot_team); 3394 } 3395 } 3396 } else { 3397 __kmp_printf("Ubers array is not allocated.\n"); 3398 } 3399 3400 __kmp_printf("\n------------------------------\nTeams\n----------------------" 3401 "--------\n"); 3402 while (list->next != NULL) { 3403 kmp_team_p const *team = list->entry; 3404 int i; 3405 __kmp_printf("Team %2x %p:\n", team->t.t_id, team); 3406 __kmp_print_structure_team(" Parent Team: ", team->t.t_parent); 3407 __kmp_printf(" Master TID: %2d\n", team->t.t_master_tid); 3408 __kmp_printf(" Max threads: %2d\n", team->t.t_max_nproc); 3409 __kmp_printf(" Levels of serial: %2d\n", team->t.t_serialized); 3410 __kmp_printf(" Number threads: %2d\n", team->t.t_nproc); 3411 for (i = 0; i < team->t.t_nproc; ++i) { 3412 __kmp_printf(" Thread %2d: ", i); 3413 __kmp_print_structure_thread("", team->t.t_threads[i]); 3414 } 3415 __kmp_print_structure_team(" Next in pool: ", team->t.t_next_pool); 3416 __kmp_printf("\n"); 3417 list = list->next; 3418 } 3419 3420 // Print out __kmp_thread_pool and __kmp_team_pool. 3421 __kmp_printf("\n------------------------------\nPools\n----------------------" 3422 "--------\n"); 3423 __kmp_print_structure_thread("Thread pool: ", 3424 CCAST(kmp_info_t *, __kmp_thread_pool)); 3425 __kmp_print_structure_team("Team pool: ", 3426 CCAST(kmp_team_t *, __kmp_team_pool)); 3427 __kmp_printf("\n"); 3428 3429 // Free team list. 3430 while (list != NULL) { 3431 kmp_team_list_item_t *item = list; 3432 list = list->next; 3433 KMP_INTERNAL_FREE(item); 3434 } 3435 } 3436 3437 #endif 3438 3439 //--------------------------------------------------------------------------- 3440 // Stuff for per-thread fast random number generator 3441 // Table of primes 3442 static const unsigned __kmp_primes[] = { 3443 0x9e3779b1, 0xffe6cc59, 0x2109f6dd, 0x43977ab5, 0xba5703f5, 0xb495a877, 3444 0xe1626741, 0x79695e6b, 0xbc98c09f, 0xd5bee2b3, 0x287488f9, 0x3af18231, 3445 0x9677cd4d, 0xbe3a6929, 0xadc6a877, 0xdcf0674b, 0xbe4d6fe9, 0x5f15e201, 3446 0x99afc3fd, 0xf3f16801, 0xe222cfff, 0x24ba5fdb, 0x0620452d, 0x79f149e3, 3447 0xc8b93f49, 0x972702cd, 0xb07dd827, 0x6c97d5ed, 0x085a3d61, 0x46eb5ea7, 3448 0x3d9910ed, 0x2e687b5b, 0x29609227, 0x6eb081f1, 0x0954c4e1, 0x9d114db9, 3449 0x542acfa9, 0xb3e6bd7b, 0x0742d917, 0xe9f3ffa7, 0x54581edb, 0xf2480f45, 3450 0x0bb9288f, 0xef1affc7, 0x85fa0ca7, 0x3ccc14db, 0xe6baf34b, 0x343377f7, 3451 0x5ca19031, 0xe6d9293b, 0xf0a9f391, 0x5d2e980b, 0xfc411073, 0xc3749363, 3452 0xb892d829, 0x3549366b, 0x629750ad, 0xb98294e5, 0x892d9483, 0xc235baf3, 3453 0x3d2402a3, 0x6bdef3c9, 0xbec333cd, 0x40c9520f}; 3454 3455 //--------------------------------------------------------------------------- 3456 // __kmp_get_random: Get a random number using a linear congruential method. 3457 unsigned short __kmp_get_random(kmp_info_t *thread) { 3458 unsigned x = thread->th.th_x; 3459 unsigned short r = x >> 16; 3460 3461 thread->th.th_x = x * thread->th.th_a + 1; 3462 3463 KA_TRACE(30, ("__kmp_get_random: THREAD: %d, RETURN: %u\n", 3464 thread->th.th_info.ds.ds_tid, r)); 3465 3466 return r; 3467 } 3468 //-------------------------------------------------------- 3469 // __kmp_init_random: Initialize a random number generator 3470 void __kmp_init_random(kmp_info_t *thread) { 3471 unsigned seed = thread->th.th_info.ds.ds_tid; 3472 3473 thread->th.th_a = 3474 __kmp_primes[seed % (sizeof(__kmp_primes) / sizeof(__kmp_primes[0]))]; 3475 thread->th.th_x = (seed + 1) * thread->th.th_a + 1; 3476 KA_TRACE(30, 3477 ("__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a)); 3478 } 3479 3480 #if KMP_OS_WINDOWS 3481 /* reclaim array entries for root threads that are already dead, returns number 3482 * reclaimed */ 3483 static int __kmp_reclaim_dead_roots(void) { 3484 int i, r = 0; 3485 3486 for (i = 0; i < __kmp_threads_capacity; ++i) { 3487 if (KMP_UBER_GTID(i) && 3488 !__kmp_still_running((kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[i])) && 3489 !__kmp_root[i] 3490 ->r.r_active) { // AC: reclaim only roots died in non-active state 3491 r += __kmp_unregister_root_other_thread(i); 3492 } 3493 } 3494 return r; 3495 } 3496 #endif 3497 3498 /* This function attempts to create free entries in __kmp_threads and 3499 __kmp_root, and returns the number of free entries generated. 3500 3501 For Windows* OS static library, the first mechanism used is to reclaim array 3502 entries for root threads that are already dead. 3503 3504 On all platforms, expansion is attempted on the arrays __kmp_threads_ and 3505 __kmp_root, with appropriate update to __kmp_threads_capacity. Array 3506 capacity is increased by doubling with clipping to __kmp_tp_capacity, if 3507 threadprivate cache array has been created. Synchronization with 3508 __kmpc_threadprivate_cached is done using __kmp_tp_cached_lock. 3509 3510 After any dead root reclamation, if the clipping value allows array expansion 3511 to result in the generation of a total of nNeed free slots, the function does 3512 that expansion. If not, nothing is done beyond the possible initial root 3513 thread reclamation. 3514 3515 If any argument is negative, the behavior is undefined. */ 3516 static int __kmp_expand_threads(int nNeed) { 3517 int added = 0; 3518 int minimumRequiredCapacity; 3519 int newCapacity; 3520 kmp_info_t **newThreads; 3521 kmp_root_t **newRoot; 3522 3523 // All calls to __kmp_expand_threads should be under __kmp_forkjoin_lock, so 3524 // resizing __kmp_threads does not need additional protection if foreign 3525 // threads are present 3526 3527 #if KMP_OS_WINDOWS && !defined KMP_DYNAMIC_LIB 3528 /* only for Windows static library */ 3529 /* reclaim array entries for root threads that are already dead */ 3530 added = __kmp_reclaim_dead_roots(); 3531 3532 if (nNeed) { 3533 nNeed -= added; 3534 if (nNeed < 0) 3535 nNeed = 0; 3536 } 3537 #endif 3538 if (nNeed <= 0) 3539 return added; 3540 3541 // Note that __kmp_threads_capacity is not bounded by __kmp_max_nth. If 3542 // __kmp_max_nth is set to some value less than __kmp_sys_max_nth by the 3543 // user via KMP_DEVICE_THREAD_LIMIT, then __kmp_threads_capacity may become 3544 // > __kmp_max_nth in one of two ways: 3545 // 3546 // 1) The initialization thread (gtid = 0) exits. __kmp_threads[0] 3547 // may not be resused by another thread, so we may need to increase 3548 // __kmp_threads_capacity to __kmp_max_nth + 1. 3549 // 3550 // 2) New foreign root(s) are encountered. We always register new foreign 3551 // roots. This may cause a smaller # of threads to be allocated at 3552 // subsequent parallel regions, but the worker threads hang around (and 3553 // eventually go to sleep) and need slots in the __kmp_threads[] array. 3554 // 3555 // Anyway, that is the reason for moving the check to see if 3556 // __kmp_max_nth was exceeded into __kmp_reserve_threads() 3557 // instead of having it performed here. -BB 3558 3559 KMP_DEBUG_ASSERT(__kmp_sys_max_nth >= __kmp_threads_capacity); 3560 3561 /* compute expansion headroom to check if we can expand */ 3562 if (__kmp_sys_max_nth - __kmp_threads_capacity < nNeed) { 3563 /* possible expansion too small -- give up */ 3564 return added; 3565 } 3566 minimumRequiredCapacity = __kmp_threads_capacity + nNeed; 3567 3568 newCapacity = __kmp_threads_capacity; 3569 do { 3570 newCapacity = newCapacity <= (__kmp_sys_max_nth >> 1) ? (newCapacity << 1) 3571 : __kmp_sys_max_nth; 3572 } while (newCapacity < minimumRequiredCapacity); 3573 newThreads = (kmp_info_t **)__kmp_allocate( 3574 (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * newCapacity + CACHE_LINE); 3575 newRoot = 3576 (kmp_root_t **)((char *)newThreads + sizeof(kmp_info_t *) * newCapacity); 3577 KMP_MEMCPY(newThreads, __kmp_threads, 3578 __kmp_threads_capacity * sizeof(kmp_info_t *)); 3579 KMP_MEMCPY(newRoot, __kmp_root, 3580 __kmp_threads_capacity * sizeof(kmp_root_t *)); 3581 3582 kmp_info_t **temp_threads = __kmp_threads; 3583 *(kmp_info_t * *volatile *)&__kmp_threads = newThreads; 3584 *(kmp_root_t * *volatile *)&__kmp_root = newRoot; 3585 __kmp_free(temp_threads); 3586 added += newCapacity - __kmp_threads_capacity; 3587 *(volatile int *)&__kmp_threads_capacity = newCapacity; 3588 3589 if (newCapacity > __kmp_tp_capacity) { 3590 __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock); 3591 if (__kmp_tp_cached && newCapacity > __kmp_tp_capacity) { 3592 __kmp_threadprivate_resize_cache(newCapacity); 3593 } else { // increase __kmp_tp_capacity to correspond with kmp_threads size 3594 *(volatile int *)&__kmp_tp_capacity = newCapacity; 3595 } 3596 __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock); 3597 } 3598 3599 return added; 3600 } 3601 3602 /* Register the current thread as a root thread and obtain our gtid. We must 3603 have the __kmp_initz_lock held at this point. Argument TRUE only if are the 3604 thread that calls from __kmp_do_serial_initialize() */ 3605 int __kmp_register_root(int initial_thread) { 3606 kmp_info_t *root_thread; 3607 kmp_root_t *root; 3608 int gtid; 3609 int capacity; 3610 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 3611 KA_TRACE(20, ("__kmp_register_root: entered\n")); 3612 KMP_MB(); 3613 3614 /* 2007-03-02: 3615 If initial thread did not invoke OpenMP RTL yet, and this thread is not an 3616 initial one, "__kmp_all_nth >= __kmp_threads_capacity" condition does not 3617 work as expected -- it may return false (that means there is at least one 3618 empty slot in __kmp_threads array), but it is possible the only free slot 3619 is #0, which is reserved for initial thread and so cannot be used for this 3620 one. Following code workarounds this bug. 3621 3622 However, right solution seems to be not reserving slot #0 for initial 3623 thread because: 3624 (1) there is no magic in slot #0, 3625 (2) we cannot detect initial thread reliably (the first thread which does 3626 serial initialization may be not a real initial thread). 3627 */ 3628 capacity = __kmp_threads_capacity; 3629 if (!initial_thread && TCR_PTR(__kmp_threads[0]) == NULL) { 3630 --capacity; 3631 } 3632 3633 /* see if there are too many threads */ 3634 if (__kmp_all_nth >= capacity && !__kmp_expand_threads(1)) { 3635 if (__kmp_tp_cached) { 3636 __kmp_fatal(KMP_MSG(CantRegisterNewThread), 3637 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity), 3638 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null); 3639 } else { 3640 __kmp_fatal(KMP_MSG(CantRegisterNewThread), KMP_HNT(SystemLimitOnThreads), 3641 __kmp_msg_null); 3642 } 3643 } 3644 3645 /* find an available thread slot */ 3646 /* Don't reassign the zero slot since we need that to only be used by initial 3647 thread */ 3648 for (gtid = (initial_thread ? 0 : 1); TCR_PTR(__kmp_threads[gtid]) != NULL; 3649 gtid++) 3650 ; 3651 KA_TRACE(1, 3652 ("__kmp_register_root: found slot in threads array: T#%d\n", gtid)); 3653 KMP_ASSERT(gtid < __kmp_threads_capacity); 3654 3655 /* update global accounting */ 3656 __kmp_all_nth++; 3657 TCW_4(__kmp_nth, __kmp_nth + 1); 3658 3659 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low 3660 // numbers of procs, and method #2 (keyed API call) for higher numbers. 3661 if (__kmp_adjust_gtid_mode) { 3662 if (__kmp_all_nth >= __kmp_tls_gtid_min) { 3663 if (TCR_4(__kmp_gtid_mode) != 2) { 3664 TCW_4(__kmp_gtid_mode, 2); 3665 } 3666 } else { 3667 if (TCR_4(__kmp_gtid_mode) != 1) { 3668 TCW_4(__kmp_gtid_mode, 1); 3669 } 3670 } 3671 } 3672 3673 #ifdef KMP_ADJUST_BLOCKTIME 3674 /* Adjust blocktime to zero if necessary */ 3675 /* Middle initialization might not have occurred yet */ 3676 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 3677 if (__kmp_nth > __kmp_avail_proc) { 3678 __kmp_zero_bt = TRUE; 3679 } 3680 } 3681 #endif /* KMP_ADJUST_BLOCKTIME */ 3682 3683 /* setup this new hierarchy */ 3684 if (!(root = __kmp_root[gtid])) { 3685 root = __kmp_root[gtid] = (kmp_root_t *)__kmp_allocate(sizeof(kmp_root_t)); 3686 KMP_DEBUG_ASSERT(!root->r.r_root_team); 3687 } 3688 3689 #if KMP_STATS_ENABLED 3690 // Initialize stats as soon as possible (right after gtid assignment). 3691 __kmp_stats_thread_ptr = __kmp_stats_list->push_back(gtid); 3692 __kmp_stats_thread_ptr->startLife(); 3693 KMP_SET_THREAD_STATE(SERIAL_REGION); 3694 KMP_INIT_PARTITIONED_TIMERS(OMP_serial); 3695 #endif 3696 __kmp_initialize_root(root); 3697 3698 /* setup new root thread structure */ 3699 if (root->r.r_uber_thread) { 3700 root_thread = root->r.r_uber_thread; 3701 } else { 3702 root_thread = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t)); 3703 if (__kmp_storage_map) { 3704 __kmp_print_thread_storage_map(root_thread, gtid); 3705 } 3706 root_thread->th.th_info.ds.ds_gtid = gtid; 3707 #if OMPT_SUPPORT 3708 root_thread->th.ompt_thread_info.thread_data.ptr = NULL; 3709 #endif 3710 root_thread->th.th_root = root; 3711 if (__kmp_env_consistency_check) { 3712 root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid); 3713 } 3714 #if USE_FAST_MEMORY 3715 __kmp_initialize_fast_memory(root_thread); 3716 #endif /* USE_FAST_MEMORY */ 3717 3718 #if KMP_USE_BGET 3719 KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL); 3720 __kmp_initialize_bget(root_thread); 3721 #endif 3722 __kmp_init_random(root_thread); // Initialize random number generator 3723 } 3724 3725 /* setup the serial team held in reserve by the root thread */ 3726 if (!root_thread->th.th_serial_team) { 3727 kmp_internal_control_t r_icvs = __kmp_get_global_icvs(); 3728 KF_TRACE(10, ("__kmp_register_root: before serial_team\n")); 3729 root_thread->th.th_serial_team = 3730 __kmp_allocate_team(root, 1, 1, 3731 #if OMPT_SUPPORT 3732 ompt_data_none, // root parallel id 3733 #endif 3734 #if OMP_40_ENABLED 3735 proc_bind_default, 3736 #endif 3737 &r_icvs, 0 USE_NESTED_HOT_ARG(NULL)); 3738 } 3739 KMP_ASSERT(root_thread->th.th_serial_team); 3740 KF_TRACE(10, ("__kmp_register_root: after serial_team = %p\n", 3741 root_thread->th.th_serial_team)); 3742 3743 /* drop root_thread into place */ 3744 TCW_SYNC_PTR(__kmp_threads[gtid], root_thread); 3745 3746 root->r.r_root_team->t.t_threads[0] = root_thread; 3747 root->r.r_hot_team->t.t_threads[0] = root_thread; 3748 root_thread->th.th_serial_team->t.t_threads[0] = root_thread; 3749 // AC: the team created in reserve, not for execution (it is unused for now). 3750 root_thread->th.th_serial_team->t.t_serialized = 0; 3751 root->r.r_uber_thread = root_thread; 3752 3753 /* initialize the thread, get it ready to go */ 3754 __kmp_initialize_info(root_thread, root->r.r_root_team, 0, gtid); 3755 TCW_4(__kmp_init_gtid, TRUE); 3756 3757 /* prepare the master thread for get_gtid() */ 3758 __kmp_gtid_set_specific(gtid); 3759 3760 #if USE_ITT_BUILD 3761 __kmp_itt_thread_name(gtid); 3762 #endif /* USE_ITT_BUILD */ 3763 3764 #ifdef KMP_TDATA_GTID 3765 __kmp_gtid = gtid; 3766 #endif 3767 __kmp_create_worker(gtid, root_thread, __kmp_stksize); 3768 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == gtid); 3769 3770 KA_TRACE(20, ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, " 3771 "plain=%u\n", 3772 gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team), 3773 root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE, 3774 KMP_INIT_BARRIER_STATE)); 3775 { // Initialize barrier data. 3776 int b; 3777 for (b = 0; b < bs_last_barrier; ++b) { 3778 root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE; 3779 #if USE_DEBUGGER 3780 root_thread->th.th_bar[b].bb.b_worker_arrived = 0; 3781 #endif 3782 } 3783 } 3784 KMP_DEBUG_ASSERT(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived == 3785 KMP_INIT_BARRIER_STATE); 3786 3787 #if KMP_AFFINITY_SUPPORTED 3788 #if OMP_40_ENABLED 3789 root_thread->th.th_current_place = KMP_PLACE_UNDEFINED; 3790 root_thread->th.th_new_place = KMP_PLACE_UNDEFINED; 3791 root_thread->th.th_first_place = KMP_PLACE_UNDEFINED; 3792 root_thread->th.th_last_place = KMP_PLACE_UNDEFINED; 3793 #endif 3794 3795 if (TCR_4(__kmp_init_middle)) { 3796 __kmp_affinity_set_init_mask(gtid, TRUE); 3797 } 3798 #endif /* KMP_AFFINITY_SUPPORTED */ 3799 3800 __kmp_root_counter++; 3801 3802 #if OMPT_SUPPORT 3803 if (!initial_thread && ompt_enabled.enabled) { 3804 3805 ompt_thread_t *root_thread = ompt_get_thread(); 3806 3807 ompt_set_thread_state(root_thread, omp_state_overhead); 3808 3809 if (ompt_enabled.ompt_callback_thread_begin) { 3810 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)( 3811 ompt_thread_initial, __ompt_get_thread_data_internal()); 3812 } 3813 ompt_data_t *task_data; 3814 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL, NULL); 3815 if (ompt_enabled.ompt_callback_task_create) { 3816 ompt_callbacks.ompt_callback(ompt_callback_task_create)( 3817 NULL, NULL, task_data, ompt_task_initial, 0, NULL); 3818 // initial task has nothing to return to 3819 } 3820 3821 ompt_set_thread_state(root_thread, omp_state_work_serial); 3822 } 3823 #endif 3824 3825 KMP_MB(); 3826 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 3827 3828 return gtid; 3829 } 3830 3831 #if KMP_NESTED_HOT_TEAMS 3832 static int __kmp_free_hot_teams(kmp_root_t *root, kmp_info_t *thr, int level, 3833 const int max_level) { 3834 int i, n, nth; 3835 kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams; 3836 if (!hot_teams || !hot_teams[level].hot_team) { 3837 return 0; 3838 } 3839 KMP_DEBUG_ASSERT(level < max_level); 3840 kmp_team_t *team = hot_teams[level].hot_team; 3841 nth = hot_teams[level].hot_team_nth; 3842 n = nth - 1; // master is not freed 3843 if (level < max_level - 1) { 3844 for (i = 0; i < nth; ++i) { 3845 kmp_info_t *th = team->t.t_threads[i]; 3846 n += __kmp_free_hot_teams(root, th, level + 1, max_level); 3847 if (i > 0 && th->th.th_hot_teams) { 3848 __kmp_free(th->th.th_hot_teams); 3849 th->th.th_hot_teams = NULL; 3850 } 3851 } 3852 } 3853 __kmp_free_team(root, team, NULL); 3854 return n; 3855 } 3856 #endif 3857 3858 // Resets a root thread and clear its root and hot teams. 3859 // Returns the number of __kmp_threads entries directly and indirectly freed. 3860 static int __kmp_reset_root(int gtid, kmp_root_t *root) { 3861 kmp_team_t *root_team = root->r.r_root_team; 3862 kmp_team_t *hot_team = root->r.r_hot_team; 3863 int n = hot_team->t.t_nproc; 3864 int i; 3865 3866 KMP_DEBUG_ASSERT(!root->r.r_active); 3867 3868 root->r.r_root_team = NULL; 3869 root->r.r_hot_team = NULL; 3870 // __kmp_free_team() does not free hot teams, so we have to clear r_hot_team 3871 // before call to __kmp_free_team(). 3872 __kmp_free_team(root, root_team USE_NESTED_HOT_ARG(NULL)); 3873 #if KMP_NESTED_HOT_TEAMS 3874 if (__kmp_hot_teams_max_level > 3875 0) { // need to free nested hot teams and their threads if any 3876 for (i = 0; i < hot_team->t.t_nproc; ++i) { 3877 kmp_info_t *th = hot_team->t.t_threads[i]; 3878 if (__kmp_hot_teams_max_level > 1) { 3879 n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level); 3880 } 3881 if (th->th.th_hot_teams) { 3882 __kmp_free(th->th.th_hot_teams); 3883 th->th.th_hot_teams = NULL; 3884 } 3885 } 3886 } 3887 #endif 3888 __kmp_free_team(root, hot_team USE_NESTED_HOT_ARG(NULL)); 3889 3890 // Before we can reap the thread, we need to make certain that all other 3891 // threads in the teams that had this root as ancestor have stopped trying to 3892 // steal tasks. 3893 if (__kmp_tasking_mode != tskm_immediate_exec) { 3894 __kmp_wait_to_unref_task_teams(); 3895 } 3896 3897 #if KMP_OS_WINDOWS 3898 /* Close Handle of root duplicated in __kmp_create_worker (tr #62919) */ 3899 KA_TRACE( 3900 10, ("__kmp_reset_root: free handle, th = %p, handle = %" KMP_UINTPTR_SPEC 3901 "\n", 3902 (LPVOID) & (root->r.r_uber_thread->th), 3903 root->r.r_uber_thread->th.th_info.ds.ds_thread)); 3904 __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread); 3905 #endif /* KMP_OS_WINDOWS */ 3906 3907 #if OMPT_SUPPORT 3908 if (ompt_enabled.ompt_callback_thread_end) { 3909 ompt_callbacks.ompt_callback(ompt_callback_thread_end)( 3910 &(root->r.r_uber_thread->th.ompt_thread_info.thread_data)); 3911 } 3912 #endif 3913 3914 TCW_4(__kmp_nth, 3915 __kmp_nth - 1); // __kmp_reap_thread will decrement __kmp_all_nth. 3916 root->r.r_cg_nthreads--; 3917 3918 __kmp_reap_thread(root->r.r_uber_thread, 1); 3919 3920 // We canot put root thread to __kmp_thread_pool, so we have to reap it istead 3921 // of freeing. 3922 root->r.r_uber_thread = NULL; 3923 /* mark root as no longer in use */ 3924 root->r.r_begin = FALSE; 3925 3926 return n; 3927 } 3928 3929 void __kmp_unregister_root_current_thread(int gtid) { 3930 KA_TRACE(1, ("__kmp_unregister_root_current_thread: enter T#%d\n", gtid)); 3931 /* this lock should be ok, since unregister_root_current_thread is never 3932 called during an abort, only during a normal close. furthermore, if you 3933 have the forkjoin lock, you should never try to get the initz lock */ 3934 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 3935 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 3936 KC_TRACE(10, ("__kmp_unregister_root_current_thread: already finished, " 3937 "exiting T#%d\n", 3938 gtid)); 3939 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 3940 return; 3941 } 3942 kmp_root_t *root = __kmp_root[gtid]; 3943 3944 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]); 3945 KMP_ASSERT(KMP_UBER_GTID(gtid)); 3946 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root); 3947 KMP_ASSERT(root->r.r_active == FALSE); 3948 3949 KMP_MB(); 3950 3951 #if OMP_45_ENABLED 3952 kmp_info_t *thread = __kmp_threads[gtid]; 3953 kmp_team_t *team = thread->th.th_team; 3954 kmp_task_team_t *task_team = thread->th.th_task_team; 3955 3956 // we need to wait for the proxy tasks before finishing the thread 3957 if (task_team != NULL && task_team->tt.tt_found_proxy_tasks) { 3958 #if OMPT_SUPPORT 3959 // the runtime is shutting down so we won't report any events 3960 thread->th.ompt_thread_info.state = omp_state_undefined; 3961 #endif 3962 __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL)); 3963 } 3964 #endif 3965 3966 __kmp_reset_root(gtid, root); 3967 3968 /* free up this thread slot */ 3969 __kmp_gtid_set_specific(KMP_GTID_DNE); 3970 #ifdef KMP_TDATA_GTID 3971 __kmp_gtid = KMP_GTID_DNE; 3972 #endif 3973 3974 KMP_MB(); 3975 KC_TRACE(10, 3976 ("__kmp_unregister_root_current_thread: T#%d unregistered\n", gtid)); 3977 3978 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 3979 } 3980 3981 #if KMP_OS_WINDOWS 3982 /* __kmp_forkjoin_lock must be already held 3983 Unregisters a root thread that is not the current thread. Returns the number 3984 of __kmp_threads entries freed as a result. */ 3985 static int __kmp_unregister_root_other_thread(int gtid) { 3986 kmp_root_t *root = __kmp_root[gtid]; 3987 int r; 3988 3989 KA_TRACE(1, ("__kmp_unregister_root_other_thread: enter T#%d\n", gtid)); 3990 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]); 3991 KMP_ASSERT(KMP_UBER_GTID(gtid)); 3992 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root); 3993 KMP_ASSERT(root->r.r_active == FALSE); 3994 3995 r = __kmp_reset_root(gtid, root); 3996 KC_TRACE(10, 3997 ("__kmp_unregister_root_other_thread: T#%d unregistered\n", gtid)); 3998 return r; 3999 } 4000 #endif 4001 4002 #if KMP_DEBUG 4003 void __kmp_task_info() { 4004 4005 kmp_int32 gtid = __kmp_entry_gtid(); 4006 kmp_int32 tid = __kmp_tid_from_gtid(gtid); 4007 kmp_info_t *this_thr = __kmp_threads[gtid]; 4008 kmp_team_t *steam = this_thr->th.th_serial_team; 4009 kmp_team_t *team = this_thr->th.th_team; 4010 4011 __kmp_printf( 4012 "__kmp_task_info: gtid=%d tid=%d t_thread=%p team=%p steam=%p curtask=%p " 4013 "ptask=%p\n", 4014 gtid, tid, this_thr, team, steam, this_thr->th.th_current_task, 4015 team->t.t_implicit_task_taskdata[tid].td_parent); 4016 } 4017 #endif // KMP_DEBUG 4018 4019 /* TODO optimize with one big memclr, take out what isn't needed, split 4020 responsibility to workers as much as possible, and delay initialization of 4021 features as much as possible */ 4022 static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team, 4023 int tid, int gtid) { 4024 /* this_thr->th.th_info.ds.ds_gtid is setup in 4025 kmp_allocate_thread/create_worker. 4026 this_thr->th.th_serial_team is setup in __kmp_allocate_thread */ 4027 kmp_info_t *master = team->t.t_threads[0]; 4028 KMP_DEBUG_ASSERT(this_thr != NULL); 4029 KMP_DEBUG_ASSERT(this_thr->th.th_serial_team); 4030 KMP_DEBUG_ASSERT(team); 4031 KMP_DEBUG_ASSERT(team->t.t_threads); 4032 KMP_DEBUG_ASSERT(team->t.t_dispatch); 4033 KMP_DEBUG_ASSERT(master); 4034 KMP_DEBUG_ASSERT(master->th.th_root); 4035 4036 KMP_MB(); 4037 4038 TCW_SYNC_PTR(this_thr->th.th_team, team); 4039 4040 this_thr->th.th_info.ds.ds_tid = tid; 4041 this_thr->th.th_set_nproc = 0; 4042 if (__kmp_tasking_mode != tskm_immediate_exec) 4043 // When tasking is possible, threads are not safe to reap until they are 4044 // done tasking; this will be set when tasking code is exited in wait 4045 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP; 4046 else // no tasking --> always safe to reap 4047 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP; 4048 #if OMP_40_ENABLED 4049 this_thr->th.th_set_proc_bind = proc_bind_default; 4050 #if KMP_AFFINITY_SUPPORTED 4051 this_thr->th.th_new_place = this_thr->th.th_current_place; 4052 #endif 4053 #endif 4054 this_thr->th.th_root = master->th.th_root; 4055 4056 /* setup the thread's cache of the team structure */ 4057 this_thr->th.th_team_nproc = team->t.t_nproc; 4058 this_thr->th.th_team_master = master; 4059 this_thr->th.th_team_serialized = team->t.t_serialized; 4060 TCW_PTR(this_thr->th.th_sleep_loc, NULL); 4061 4062 KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata); 4063 4064 KF_TRACE(10, ("__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n", 4065 tid, gtid, this_thr, this_thr->th.th_current_task)); 4066 4067 __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr, 4068 team, tid, TRUE); 4069 4070 KF_TRACE(10, ("__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n", 4071 tid, gtid, this_thr, this_thr->th.th_current_task)); 4072 // TODO: Initialize ICVs from parent; GEH - isn't that already done in 4073 // __kmp_initialize_team()? 4074 4075 /* TODO no worksharing in speculative threads */ 4076 this_thr->th.th_dispatch = &team->t.t_dispatch[tid]; 4077 4078 this_thr->th.th_local.this_construct = 0; 4079 4080 if (!this_thr->th.th_pri_common) { 4081 this_thr->th.th_pri_common = 4082 (struct common_table *)__kmp_allocate(sizeof(struct common_table)); 4083 if (__kmp_storage_map) { 4084 __kmp_print_storage_map_gtid( 4085 gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1, 4086 sizeof(struct common_table), "th_%d.th_pri_common\n", gtid); 4087 } 4088 this_thr->th.th_pri_head = NULL; 4089 } 4090 4091 /* Initialize dynamic dispatch */ 4092 { 4093 volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch; 4094 // Use team max_nproc since this will never change for the team. 4095 size_t disp_size = 4096 sizeof(dispatch_private_info_t) * 4097 (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers); 4098 KD_TRACE(10, ("__kmp_initialize_info: T#%d max_nproc: %d\n", gtid, 4099 team->t.t_max_nproc)); 4100 KMP_ASSERT(dispatch); 4101 KMP_DEBUG_ASSERT(team->t.t_dispatch); 4102 KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid]); 4103 4104 dispatch->th_disp_index = 0; 4105 #if OMP_45_ENABLED 4106 dispatch->th_doacross_buf_idx = 0; 4107 #endif 4108 if (!dispatch->th_disp_buffer) { 4109 dispatch->th_disp_buffer = 4110 (dispatch_private_info_t *)__kmp_allocate(disp_size); 4111 4112 if (__kmp_storage_map) { 4113 __kmp_print_storage_map_gtid( 4114 gtid, &dispatch->th_disp_buffer[0], 4115 &dispatch->th_disp_buffer[team->t.t_max_nproc == 1 4116 ? 1 4117 : __kmp_dispatch_num_buffers], 4118 disp_size, "th_%d.th_dispatch.th_disp_buffer " 4119 "(team_%d.t_dispatch[%d].th_disp_buffer)", 4120 gtid, team->t.t_id, gtid); 4121 } 4122 } else { 4123 memset(&dispatch->th_disp_buffer[0], '\0', disp_size); 4124 } 4125 4126 dispatch->th_dispatch_pr_current = 0; 4127 dispatch->th_dispatch_sh_current = 0; 4128 4129 dispatch->th_deo_fcn = 0; /* ORDERED */ 4130 dispatch->th_dxo_fcn = 0; /* END ORDERED */ 4131 } 4132 4133 this_thr->th.th_next_pool = NULL; 4134 4135 if (!this_thr->th.th_task_state_memo_stack) { 4136 size_t i; 4137 this_thr->th.th_task_state_memo_stack = 4138 (kmp_uint8 *)__kmp_allocate(4 * sizeof(kmp_uint8)); 4139 this_thr->th.th_task_state_top = 0; 4140 this_thr->th.th_task_state_stack_sz = 4; 4141 for (i = 0; i < this_thr->th.th_task_state_stack_sz; 4142 ++i) // zero init the stack 4143 this_thr->th.th_task_state_memo_stack[i] = 0; 4144 } 4145 4146 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here); 4147 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0); 4148 4149 KMP_MB(); 4150 } 4151 4152 /* allocate a new thread for the requesting team. this is only called from 4153 within a forkjoin critical section. we will first try to get an available 4154 thread from the thread pool. if none is available, we will fork a new one 4155 assuming we are able to create a new one. this should be assured, as the 4156 caller should check on this first. */ 4157 kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team, 4158 int new_tid) { 4159 kmp_team_t *serial_team; 4160 kmp_info_t *new_thr; 4161 int new_gtid; 4162 4163 KA_TRACE(20, ("__kmp_allocate_thread: T#%d\n", __kmp_get_gtid())); 4164 KMP_DEBUG_ASSERT(root && team); 4165 #if !KMP_NESTED_HOT_TEAMS 4166 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(__kmp_get_gtid())); 4167 #endif 4168 KMP_MB(); 4169 4170 /* first, try to get one from the thread pool */ 4171 if (__kmp_thread_pool) { 4172 4173 new_thr = CCAST(kmp_info_t *, __kmp_thread_pool); 4174 __kmp_thread_pool = (volatile kmp_info_t *)new_thr->th.th_next_pool; 4175 if (new_thr == __kmp_thread_pool_insert_pt) { 4176 __kmp_thread_pool_insert_pt = NULL; 4177 } 4178 TCW_4(new_thr->th.th_in_pool, FALSE); 4179 // Don't touch th_active_in_pool or th_active. 4180 // The worker thread adjusts those flags as it sleeps/awakens. 4181 __kmp_thread_pool_nth--; 4182 4183 KA_TRACE(20, ("__kmp_allocate_thread: T#%d using thread T#%d\n", 4184 __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid)); 4185 KMP_ASSERT(!new_thr->th.th_team); 4186 KMP_DEBUG_ASSERT(__kmp_nth < __kmp_threads_capacity); 4187 KMP_DEBUG_ASSERT(__kmp_thread_pool_nth >= 0); 4188 4189 /* setup the thread structure */ 4190 __kmp_initialize_info(new_thr, team, new_tid, 4191 new_thr->th.th_info.ds.ds_gtid); 4192 KMP_DEBUG_ASSERT(new_thr->th.th_serial_team); 4193 4194 TCW_4(__kmp_nth, __kmp_nth + 1); 4195 root->r.r_cg_nthreads++; 4196 4197 new_thr->th.th_task_state = 0; 4198 new_thr->th.th_task_state_top = 0; 4199 new_thr->th.th_task_state_stack_sz = 4; 4200 4201 #ifdef KMP_ADJUST_BLOCKTIME 4202 /* Adjust blocktime back to zero if necessary */ 4203 /* Middle initialization might not have occurred yet */ 4204 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 4205 if (__kmp_nth > __kmp_avail_proc) { 4206 __kmp_zero_bt = TRUE; 4207 } 4208 } 4209 #endif /* KMP_ADJUST_BLOCKTIME */ 4210 4211 #if KMP_DEBUG 4212 // If thread entered pool via __kmp_free_thread, wait_flag should != 4213 // KMP_BARRIER_PARENT_FLAG. 4214 int b; 4215 kmp_balign_t *balign = new_thr->th.th_bar; 4216 for (b = 0; b < bs_last_barrier; ++b) 4217 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 4218 #endif 4219 4220 KF_TRACE(10, ("__kmp_allocate_thread: T#%d using thread %p T#%d\n", 4221 __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid)); 4222 4223 KMP_MB(); 4224 return new_thr; 4225 } 4226 4227 /* no, well fork a new one */ 4228 KMP_ASSERT(__kmp_nth == __kmp_all_nth); 4229 KMP_ASSERT(__kmp_all_nth < __kmp_threads_capacity); 4230 4231 #if KMP_USE_MONITOR 4232 // If this is the first worker thread the RTL is creating, then also 4233 // launch the monitor thread. We try to do this as early as possible. 4234 if (!TCR_4(__kmp_init_monitor)) { 4235 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock); 4236 if (!TCR_4(__kmp_init_monitor)) { 4237 KF_TRACE(10, ("before __kmp_create_monitor\n")); 4238 TCW_4(__kmp_init_monitor, 1); 4239 __kmp_create_monitor(&__kmp_monitor); 4240 KF_TRACE(10, ("after __kmp_create_monitor\n")); 4241 #if KMP_OS_WINDOWS 4242 // AC: wait until monitor has started. This is a fix for CQ232808. 4243 // The reason is that if the library is loaded/unloaded in a loop with 4244 // small (parallel) work in between, then there is high probability that 4245 // monitor thread started after the library shutdown. At shutdown it is 4246 // too late to cope with the problem, because when the master is in 4247 // DllMain (process detach) the monitor has no chances to start (it is 4248 // blocked), and master has no means to inform the monitor that the 4249 // library has gone, because all the memory which the monitor can access 4250 // is going to be released/reset. 4251 while (TCR_4(__kmp_init_monitor) < 2) { 4252 KMP_YIELD(TRUE); 4253 } 4254 KF_TRACE(10, ("after monitor thread has started\n")); 4255 #endif 4256 } 4257 __kmp_release_bootstrap_lock(&__kmp_monitor_lock); 4258 } 4259 #endif 4260 4261 KMP_MB(); 4262 for (new_gtid = 1; TCR_PTR(__kmp_threads[new_gtid]) != NULL; ++new_gtid) { 4263 KMP_DEBUG_ASSERT(new_gtid < __kmp_threads_capacity); 4264 } 4265 4266 /* allocate space for it. */ 4267 new_thr = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t)); 4268 4269 TCW_SYNC_PTR(__kmp_threads[new_gtid], new_thr); 4270 4271 if (__kmp_storage_map) { 4272 __kmp_print_thread_storage_map(new_thr, new_gtid); 4273 } 4274 4275 // add the reserve serialized team, initialized from the team's master thread 4276 { 4277 kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team); 4278 KF_TRACE(10, ("__kmp_allocate_thread: before th_serial/serial_team\n")); 4279 new_thr->th.th_serial_team = serial_team = 4280 (kmp_team_t *)__kmp_allocate_team(root, 1, 1, 4281 #if OMPT_SUPPORT 4282 ompt_data_none, // root parallel id 4283 #endif 4284 #if OMP_40_ENABLED 4285 proc_bind_default, 4286 #endif 4287 &r_icvs, 0 USE_NESTED_HOT_ARG(NULL)); 4288 } 4289 KMP_ASSERT(serial_team); 4290 serial_team->t.t_serialized = 0; // AC: the team created in reserve, not for 4291 // execution (it is unused for now). 4292 serial_team->t.t_threads[0] = new_thr; 4293 KF_TRACE(10, 4294 ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n", 4295 new_thr)); 4296 4297 /* setup the thread structures */ 4298 __kmp_initialize_info(new_thr, team, new_tid, new_gtid); 4299 4300 #if USE_FAST_MEMORY 4301 __kmp_initialize_fast_memory(new_thr); 4302 #endif /* USE_FAST_MEMORY */ 4303 4304 #if KMP_USE_BGET 4305 KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL); 4306 __kmp_initialize_bget(new_thr); 4307 #endif 4308 4309 __kmp_init_random(new_thr); // Initialize random number generator 4310 4311 /* Initialize these only once when thread is grabbed for a team allocation */ 4312 KA_TRACE(20, 4313 ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n", 4314 __kmp_get_gtid(), KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); 4315 4316 int b; 4317 kmp_balign_t *balign = new_thr->th.th_bar; 4318 for (b = 0; b < bs_last_barrier; ++b) { 4319 balign[b].bb.b_go = KMP_INIT_BARRIER_STATE; 4320 balign[b].bb.team = NULL; 4321 balign[b].bb.wait_flag = KMP_BARRIER_NOT_WAITING; 4322 balign[b].bb.use_oncore_barrier = 0; 4323 } 4324 4325 new_thr->th.th_spin_here = FALSE; 4326 new_thr->th.th_next_waiting = 0; 4327 #if KMP_OS_UNIX 4328 new_thr->th.th_blocking = false; 4329 #endif 4330 4331 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 4332 new_thr->th.th_current_place = KMP_PLACE_UNDEFINED; 4333 new_thr->th.th_new_place = KMP_PLACE_UNDEFINED; 4334 new_thr->th.th_first_place = KMP_PLACE_UNDEFINED; 4335 new_thr->th.th_last_place = KMP_PLACE_UNDEFINED; 4336 #endif 4337 4338 TCW_4(new_thr->th.th_in_pool, FALSE); 4339 new_thr->th.th_active_in_pool = FALSE; 4340 TCW_4(new_thr->th.th_active, TRUE); 4341 4342 /* adjust the global counters */ 4343 __kmp_all_nth++; 4344 __kmp_nth++; 4345 4346 root->r.r_cg_nthreads++; 4347 4348 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low 4349 // numbers of procs, and method #2 (keyed API call) for higher numbers. 4350 if (__kmp_adjust_gtid_mode) { 4351 if (__kmp_all_nth >= __kmp_tls_gtid_min) { 4352 if (TCR_4(__kmp_gtid_mode) != 2) { 4353 TCW_4(__kmp_gtid_mode, 2); 4354 } 4355 } else { 4356 if (TCR_4(__kmp_gtid_mode) != 1) { 4357 TCW_4(__kmp_gtid_mode, 1); 4358 } 4359 } 4360 } 4361 4362 #ifdef KMP_ADJUST_BLOCKTIME 4363 /* Adjust blocktime back to zero if necessary */ 4364 /* Middle initialization might not have occurred yet */ 4365 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 4366 if (__kmp_nth > __kmp_avail_proc) { 4367 __kmp_zero_bt = TRUE; 4368 } 4369 } 4370 #endif /* KMP_ADJUST_BLOCKTIME */ 4371 4372 /* actually fork it and create the new worker thread */ 4373 KF_TRACE( 4374 10, ("__kmp_allocate_thread: before __kmp_create_worker: %p\n", new_thr)); 4375 __kmp_create_worker(new_gtid, new_thr, __kmp_stksize); 4376 KF_TRACE(10, 4377 ("__kmp_allocate_thread: after __kmp_create_worker: %p\n", new_thr)); 4378 4379 KA_TRACE(20, ("__kmp_allocate_thread: T#%d forked T#%d\n", __kmp_get_gtid(), 4380 new_gtid)); 4381 KMP_MB(); 4382 return new_thr; 4383 } 4384 4385 /* Reinitialize team for reuse. 4386 The hot team code calls this case at every fork barrier, so EPCC barrier 4387 test are extremely sensitive to changes in it, esp. writes to the team 4388 struct, which cause a cache invalidation in all threads. 4389 IF YOU TOUCH THIS ROUTINE, RUN EPCC C SYNCBENCH ON A BIG-IRON MACHINE!!! */ 4390 static void __kmp_reinitialize_team(kmp_team_t *team, 4391 kmp_internal_control_t *new_icvs, 4392 ident_t *loc) { 4393 KF_TRACE(10, ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n", 4394 team->t.t_threads[0], team)); 4395 KMP_DEBUG_ASSERT(team && new_icvs); 4396 KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc); 4397 KMP_CHECK_UPDATE(team->t.t_ident, loc); 4398 4399 KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID()); 4400 // Copy ICVs to the master thread's implicit taskdata 4401 __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE); 4402 copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs); 4403 4404 KF_TRACE(10, ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n", 4405 team->t.t_threads[0], team)); 4406 } 4407 4408 /* Initialize the team data structure. 4409 This assumes the t_threads and t_max_nproc are already set. 4410 Also, we don't touch the arguments */ 4411 static void __kmp_initialize_team(kmp_team_t *team, int new_nproc, 4412 kmp_internal_control_t *new_icvs, 4413 ident_t *loc) { 4414 KF_TRACE(10, ("__kmp_initialize_team: enter: team=%p\n", team)); 4415 4416 /* verify */ 4417 KMP_DEBUG_ASSERT(team); 4418 KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc); 4419 KMP_DEBUG_ASSERT(team->t.t_threads); 4420 KMP_MB(); 4421 4422 team->t.t_master_tid = 0; /* not needed */ 4423 /* team->t.t_master_bar; not needed */ 4424 team->t.t_serialized = new_nproc > 1 ? 0 : 1; 4425 team->t.t_nproc = new_nproc; 4426 4427 /* team->t.t_parent = NULL; TODO not needed & would mess up hot team */ 4428 team->t.t_next_pool = NULL; 4429 /* memset( team->t.t_threads, 0, sizeof(kmp_info_t*)*new_nproc ); would mess 4430 * up hot team */ 4431 4432 TCW_SYNC_PTR(team->t.t_pkfn, NULL); /* not needed */ 4433 team->t.t_invoke = NULL; /* not needed */ 4434 4435 // TODO???: team->t.t_max_active_levels = new_max_active_levels; 4436 team->t.t_sched.sched = new_icvs->sched.sched; 4437 4438 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 4439 team->t.t_fp_control_saved = FALSE; /* not needed */ 4440 team->t.t_x87_fpu_control_word = 0; /* not needed */ 4441 team->t.t_mxcsr = 0; /* not needed */ 4442 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 4443 4444 team->t.t_construct = 0; 4445 4446 team->t.t_ordered.dt.t_value = 0; 4447 team->t.t_master_active = FALSE; 4448 4449 memset(&team->t.t_taskq, '\0', sizeof(kmp_taskq_t)); 4450 4451 #ifdef KMP_DEBUG 4452 team->t.t_copypriv_data = NULL; /* not necessary, but nice for debugging */ 4453 #endif 4454 #if KMP_OS_WINDOWS 4455 team->t.t_copyin_counter = 0; /* for barrier-free copyin implementation */ 4456 #endif 4457 4458 team->t.t_control_stack_top = NULL; 4459 4460 __kmp_reinitialize_team(team, new_icvs, loc); 4461 4462 KMP_MB(); 4463 KF_TRACE(10, ("__kmp_initialize_team: exit: team=%p\n", team)); 4464 } 4465 4466 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 4467 /* Sets full mask for thread and returns old mask, no changes to structures. */ 4468 static void 4469 __kmp_set_thread_affinity_mask_full_tmp(kmp_affin_mask_t *old_mask) { 4470 if (KMP_AFFINITY_CAPABLE()) { 4471 int status; 4472 if (old_mask != NULL) { 4473 status = __kmp_get_system_affinity(old_mask, TRUE); 4474 int error = errno; 4475 if (status != 0) { 4476 __kmp_fatal(KMP_MSG(ChangeThreadAffMaskError), KMP_ERR(error), 4477 __kmp_msg_null); 4478 } 4479 } 4480 __kmp_set_system_affinity(__kmp_affin_fullMask, TRUE); 4481 } 4482 } 4483 #endif 4484 4485 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 4486 4487 // __kmp_partition_places() is the heart of the OpenMP 4.0 affinity mechanism. 4488 // It calculats the worker + master thread's partition based upon the parent 4489 // thread's partition, and binds each worker to a thread in their partition. 4490 // The master thread's partition should already include its current binding. 4491 static void __kmp_partition_places(kmp_team_t *team, int update_master_only) { 4492 // Copy the master thread's place partion to the team struct 4493 kmp_info_t *master_th = team->t.t_threads[0]; 4494 KMP_DEBUG_ASSERT(master_th != NULL); 4495 kmp_proc_bind_t proc_bind = team->t.t_proc_bind; 4496 int first_place = master_th->th.th_first_place; 4497 int last_place = master_th->th.th_last_place; 4498 int masters_place = master_th->th.th_current_place; 4499 team->t.t_first_place = first_place; 4500 team->t.t_last_place = last_place; 4501 4502 KA_TRACE(20, ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) " 4503 "bound to place %d partition = [%d,%d]\n", 4504 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]), 4505 team->t.t_id, masters_place, first_place, last_place)); 4506 4507 switch (proc_bind) { 4508 4509 case proc_bind_default: 4510 // serial teams might have the proc_bind policy set to proc_bind_default. It 4511 // doesn't matter, as we don't rebind master thread for any proc_bind policy 4512 KMP_DEBUG_ASSERT(team->t.t_nproc == 1); 4513 break; 4514 4515 case proc_bind_master: { 4516 int f; 4517 int n_th = team->t.t_nproc; 4518 for (f = 1; f < n_th; f++) { 4519 kmp_info_t *th = team->t.t_threads[f]; 4520 KMP_DEBUG_ASSERT(th != NULL); 4521 th->th.th_first_place = first_place; 4522 th->th.th_last_place = last_place; 4523 th->th.th_new_place = masters_place; 4524 4525 KA_TRACE(100, ("__kmp_partition_places: master: T#%d(%d:%d) place %d " 4526 "partition = [%d,%d]\n", 4527 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, 4528 f, masters_place, first_place, last_place)); 4529 } 4530 } break; 4531 4532 case proc_bind_close: { 4533 int f; 4534 int n_th = team->t.t_nproc; 4535 int n_places; 4536 if (first_place <= last_place) { 4537 n_places = last_place - first_place + 1; 4538 } else { 4539 n_places = __kmp_affinity_num_masks - first_place + last_place + 1; 4540 } 4541 if (n_th <= n_places) { 4542 int place = masters_place; 4543 for (f = 1; f < n_th; f++) { 4544 kmp_info_t *th = team->t.t_threads[f]; 4545 KMP_DEBUG_ASSERT(th != NULL); 4546 4547 if (place == last_place) { 4548 place = first_place; 4549 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4550 place = 0; 4551 } else { 4552 place++; 4553 } 4554 th->th.th_first_place = first_place; 4555 th->th.th_last_place = last_place; 4556 th->th.th_new_place = place; 4557 4558 KA_TRACE(100, ("__kmp_partition_places: close: T#%d(%d:%d) place %d " 4559 "partition = [%d,%d]\n", 4560 __kmp_gtid_from_thread(team->t.t_threads[f]), 4561 team->t.t_id, f, place, first_place, last_place)); 4562 } 4563 } else { 4564 int S, rem, gap, s_count; 4565 S = n_th / n_places; 4566 s_count = 0; 4567 rem = n_th - (S * n_places); 4568 gap = rem > 0 ? n_places / rem : n_places; 4569 int place = masters_place; 4570 int gap_ct = gap; 4571 for (f = 0; f < n_th; f++) { 4572 kmp_info_t *th = team->t.t_threads[f]; 4573 KMP_DEBUG_ASSERT(th != NULL); 4574 4575 th->th.th_first_place = first_place; 4576 th->th.th_last_place = last_place; 4577 th->th.th_new_place = place; 4578 s_count++; 4579 4580 if ((s_count == S) && rem && (gap_ct == gap)) { 4581 // do nothing, add an extra thread to place on next iteration 4582 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) { 4583 // we added an extra thread to this place; move to next place 4584 if (place == last_place) { 4585 place = first_place; 4586 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4587 place = 0; 4588 } else { 4589 place++; 4590 } 4591 s_count = 0; 4592 gap_ct = 1; 4593 rem--; 4594 } else if (s_count == S) { // place full; don't add extra 4595 if (place == last_place) { 4596 place = first_place; 4597 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4598 place = 0; 4599 } else { 4600 place++; 4601 } 4602 gap_ct++; 4603 s_count = 0; 4604 } 4605 4606 KA_TRACE(100, 4607 ("__kmp_partition_places: close: T#%d(%d:%d) place %d " 4608 "partition = [%d,%d]\n", 4609 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f, 4610 th->th.th_new_place, first_place, last_place)); 4611 } 4612 KMP_DEBUG_ASSERT(place == masters_place); 4613 } 4614 } break; 4615 4616 case proc_bind_spread: { 4617 int f; 4618 int n_th = team->t.t_nproc; 4619 int n_places; 4620 int thidx; 4621 if (first_place <= last_place) { 4622 n_places = last_place - first_place + 1; 4623 } else { 4624 n_places = __kmp_affinity_num_masks - first_place + last_place + 1; 4625 } 4626 if (n_th <= n_places) { 4627 int place = -1; 4628 4629 if (n_places != static_cast<int>(__kmp_affinity_num_masks)) { 4630 int S = n_places / n_th; 4631 int s_count, rem, gap, gap_ct; 4632 4633 place = masters_place; 4634 rem = n_places - n_th * S; 4635 gap = rem ? n_th / rem : 1; 4636 gap_ct = gap; 4637 thidx = n_th; 4638 if (update_master_only == 1) 4639 thidx = 1; 4640 for (f = 0; f < thidx; f++) { 4641 kmp_info_t *th = team->t.t_threads[f]; 4642 KMP_DEBUG_ASSERT(th != NULL); 4643 4644 th->th.th_first_place = place; 4645 th->th.th_new_place = place; 4646 s_count = 1; 4647 while (s_count < S) { 4648 if (place == last_place) { 4649 place = first_place; 4650 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4651 place = 0; 4652 } else { 4653 place++; 4654 } 4655 s_count++; 4656 } 4657 if (rem && (gap_ct == gap)) { 4658 if (place == last_place) { 4659 place = first_place; 4660 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4661 place = 0; 4662 } else { 4663 place++; 4664 } 4665 rem--; 4666 gap_ct = 0; 4667 } 4668 th->th.th_last_place = place; 4669 gap_ct++; 4670 4671 if (place == last_place) { 4672 place = first_place; 4673 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4674 place = 0; 4675 } else { 4676 place++; 4677 } 4678 4679 KA_TRACE(100, 4680 ("__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4681 "partition = [%d,%d], __kmp_affinity_num_masks: %u\n", 4682 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, 4683 f, th->th.th_new_place, th->th.th_first_place, 4684 th->th.th_last_place, __kmp_affinity_num_masks)); 4685 } 4686 } else { 4687 /* Having uniform space of available computation places I can create 4688 T partitions of round(P/T) size and put threads into the first 4689 place of each partition. */ 4690 double current = static_cast<double>(masters_place); 4691 double spacing = 4692 (static_cast<double>(n_places + 1) / static_cast<double>(n_th)); 4693 int first, last; 4694 kmp_info_t *th; 4695 4696 thidx = n_th + 1; 4697 if (update_master_only == 1) 4698 thidx = 1; 4699 for (f = 0; f < thidx; f++) { 4700 first = static_cast<int>(current); 4701 last = static_cast<int>(current + spacing) - 1; 4702 KMP_DEBUG_ASSERT(last >= first); 4703 if (first >= n_places) { 4704 if (masters_place) { 4705 first -= n_places; 4706 last -= n_places; 4707 if (first == (masters_place + 1)) { 4708 KMP_DEBUG_ASSERT(f == n_th); 4709 first--; 4710 } 4711 if (last == masters_place) { 4712 KMP_DEBUG_ASSERT(f == (n_th - 1)); 4713 last--; 4714 } 4715 } else { 4716 KMP_DEBUG_ASSERT(f == n_th); 4717 first = 0; 4718 last = 0; 4719 } 4720 } 4721 if (last >= n_places) { 4722 last = (n_places - 1); 4723 } 4724 place = first; 4725 current += spacing; 4726 if (f < n_th) { 4727 KMP_DEBUG_ASSERT(0 <= first); 4728 KMP_DEBUG_ASSERT(n_places > first); 4729 KMP_DEBUG_ASSERT(0 <= last); 4730 KMP_DEBUG_ASSERT(n_places > last); 4731 KMP_DEBUG_ASSERT(last_place >= first_place); 4732 th = team->t.t_threads[f]; 4733 KMP_DEBUG_ASSERT(th); 4734 th->th.th_first_place = first; 4735 th->th.th_new_place = place; 4736 th->th.th_last_place = last; 4737 4738 KA_TRACE(100, 4739 ("__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4740 "partition = [%d,%d], spacing = %.4f\n", 4741 __kmp_gtid_from_thread(team->t.t_threads[f]), 4742 team->t.t_id, f, th->th.th_new_place, 4743 th->th.th_first_place, th->th.th_last_place, spacing)); 4744 } 4745 } 4746 } 4747 KMP_DEBUG_ASSERT(update_master_only || place == masters_place); 4748 } else { 4749 int S, rem, gap, s_count; 4750 S = n_th / n_places; 4751 s_count = 0; 4752 rem = n_th - (S * n_places); 4753 gap = rem > 0 ? n_places / rem : n_places; 4754 int place = masters_place; 4755 int gap_ct = gap; 4756 thidx = n_th; 4757 if (update_master_only == 1) 4758 thidx = 1; 4759 for (f = 0; f < thidx; f++) { 4760 kmp_info_t *th = team->t.t_threads[f]; 4761 KMP_DEBUG_ASSERT(th != NULL); 4762 4763 th->th.th_first_place = place; 4764 th->th.th_last_place = place; 4765 th->th.th_new_place = place; 4766 s_count++; 4767 4768 if ((s_count == S) && rem && (gap_ct == gap)) { 4769 // do nothing, add an extra thread to place on next iteration 4770 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) { 4771 // we added an extra thread to this place; move on to next place 4772 if (place == last_place) { 4773 place = first_place; 4774 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4775 place = 0; 4776 } else { 4777 place++; 4778 } 4779 s_count = 0; 4780 gap_ct = 1; 4781 rem--; 4782 } else if (s_count == S) { // place is full; don't add extra thread 4783 if (place == last_place) { 4784 place = first_place; 4785 } else if (place == (int)(__kmp_affinity_num_masks - 1)) { 4786 place = 0; 4787 } else { 4788 place++; 4789 } 4790 gap_ct++; 4791 s_count = 0; 4792 } 4793 4794 KA_TRACE(100, ("__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4795 "partition = [%d,%d]\n", 4796 __kmp_gtid_from_thread(team->t.t_threads[f]), 4797 team->t.t_id, f, th->th.th_new_place, 4798 th->th.th_first_place, th->th.th_last_place)); 4799 } 4800 KMP_DEBUG_ASSERT(update_master_only || place == masters_place); 4801 } 4802 } break; 4803 4804 default: 4805 break; 4806 } 4807 4808 KA_TRACE(20, ("__kmp_partition_places: exit T#%d\n", team->t.t_id)); 4809 } 4810 4811 #endif /* OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED */ 4812 4813 /* allocate a new team data structure to use. take one off of the free pool if 4814 available */ 4815 kmp_team_t * 4816 __kmp_allocate_team(kmp_root_t *root, int new_nproc, int max_nproc, 4817 #if OMPT_SUPPORT 4818 ompt_data_t ompt_parallel_data, 4819 #endif 4820 #if OMP_40_ENABLED 4821 kmp_proc_bind_t new_proc_bind, 4822 #endif 4823 kmp_internal_control_t *new_icvs, 4824 int argc USE_NESTED_HOT_ARG(kmp_info_t *master)) { 4825 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_allocate_team); 4826 int f; 4827 kmp_team_t *team; 4828 int use_hot_team = !root->r.r_active; 4829 int level = 0; 4830 4831 KA_TRACE(20, ("__kmp_allocate_team: called\n")); 4832 KMP_DEBUG_ASSERT(new_nproc >= 1 && argc >= 0); 4833 KMP_DEBUG_ASSERT(max_nproc >= new_nproc); 4834 KMP_MB(); 4835 4836 #if KMP_NESTED_HOT_TEAMS 4837 kmp_hot_team_ptr_t *hot_teams; 4838 if (master) { 4839 team = master->th.th_team; 4840 level = team->t.t_active_level; 4841 if (master->th.th_teams_microtask) { // in teams construct? 4842 if (master->th.th_teams_size.nteams > 1 && 4843 ( // #teams > 1 4844 team->t.t_pkfn == 4845 (microtask_t)__kmp_teams_master || // inner fork of the teams 4846 master->th.th_teams_level < 4847 team->t.t_level)) { // or nested parallel inside the teams 4848 ++level; // not increment if #teams==1, or for outer fork of the teams; 4849 // increment otherwise 4850 } 4851 } 4852 hot_teams = master->th.th_hot_teams; 4853 if (level < __kmp_hot_teams_max_level && hot_teams && 4854 hot_teams[level] 4855 .hot_team) { // hot team has already been allocated for given level 4856 use_hot_team = 1; 4857 } else { 4858 use_hot_team = 0; 4859 } 4860 } 4861 #endif 4862 // Optimization to use a "hot" team 4863 if (use_hot_team && new_nproc > 1) { 4864 KMP_DEBUG_ASSERT(new_nproc == max_nproc); 4865 #if KMP_NESTED_HOT_TEAMS 4866 team = hot_teams[level].hot_team; 4867 #else 4868 team = root->r.r_hot_team; 4869 #endif 4870 #if KMP_DEBUG 4871 if (__kmp_tasking_mode != tskm_immediate_exec) { 4872 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p " 4873 "task_team[1] = %p before reinit\n", 4874 team->t.t_task_team[0], team->t.t_task_team[1])); 4875 } 4876 #endif 4877 4878 // Has the number of threads changed? 4879 /* Let's assume the most common case is that the number of threads is 4880 unchanged, and put that case first. */ 4881 if (team->t.t_nproc == new_nproc) { // Check changes in number of threads 4882 KA_TRACE(20, ("__kmp_allocate_team: reusing hot team\n")); 4883 // This case can mean that omp_set_num_threads() was called and the hot 4884 // team size was already reduced, so we check the special flag 4885 if (team->t.t_size_changed == -1) { 4886 team->t.t_size_changed = 1; 4887 } else { 4888 KMP_CHECK_UPDATE(team->t.t_size_changed, 0); 4889 } 4890 4891 // TODO???: team->t.t_max_active_levels = new_max_active_levels; 4892 kmp_r_sched_t new_sched = new_icvs->sched; 4893 // set master's schedule as new run-time schedule 4894 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched); 4895 4896 __kmp_reinitialize_team(team, new_icvs, 4897 root->r.r_uber_thread->th.th_ident); 4898 4899 KF_TRACE(10, ("__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n", 0, 4900 team->t.t_threads[0], team)); 4901 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0); 4902 4903 #if OMP_40_ENABLED 4904 #if KMP_AFFINITY_SUPPORTED 4905 if ((team->t.t_size_changed == 0) && 4906 (team->t.t_proc_bind == new_proc_bind)) { 4907 if (new_proc_bind == proc_bind_spread) { 4908 __kmp_partition_places( 4909 team, 1); // add flag to update only master for spread 4910 } 4911 KA_TRACE(200, ("__kmp_allocate_team: reusing hot team #%d bindings: " 4912 "proc_bind = %d, partition = [%d,%d]\n", 4913 team->t.t_id, new_proc_bind, team->t.t_first_place, 4914 team->t.t_last_place)); 4915 } else { 4916 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); 4917 __kmp_partition_places(team); 4918 } 4919 #else 4920 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); 4921 #endif /* KMP_AFFINITY_SUPPORTED */ 4922 #endif /* OMP_40_ENABLED */ 4923 } else if (team->t.t_nproc > new_nproc) { 4924 KA_TRACE(20, 4925 ("__kmp_allocate_team: decreasing hot team thread count to %d\n", 4926 new_nproc)); 4927 4928 team->t.t_size_changed = 1; 4929 #if KMP_NESTED_HOT_TEAMS 4930 if (__kmp_hot_teams_mode == 0) { 4931 // AC: saved number of threads should correspond to team's value in this 4932 // mode, can be bigger in mode 1, when hot team has threads in reserve 4933 KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc); 4934 hot_teams[level].hot_team_nth = new_nproc; 4935 #endif // KMP_NESTED_HOT_TEAMS 4936 /* release the extra threads we don't need any more */ 4937 for (f = new_nproc; f < team->t.t_nproc; f++) { 4938 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 4939 if (__kmp_tasking_mode != tskm_immediate_exec) { 4940 // When decreasing team size, threads no longer in the team should 4941 // unref task team. 4942 team->t.t_threads[f]->th.th_task_team = NULL; 4943 } 4944 __kmp_free_thread(team->t.t_threads[f]); 4945 team->t.t_threads[f] = NULL; 4946 } 4947 #if KMP_NESTED_HOT_TEAMS 4948 } // (__kmp_hot_teams_mode == 0) 4949 else { 4950 // When keeping extra threads in team, switch threads to wait on own 4951 // b_go flag 4952 for (f = new_nproc; f < team->t.t_nproc; ++f) { 4953 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 4954 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar; 4955 for (int b = 0; b < bs_last_barrier; ++b) { 4956 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG) { 4957 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG; 4958 } 4959 KMP_CHECK_UPDATE(balign[b].bb.leaf_kids, 0); 4960 } 4961 } 4962 } 4963 #endif // KMP_NESTED_HOT_TEAMS 4964 team->t.t_nproc = new_nproc; 4965 // TODO???: team->t.t_max_active_levels = new_max_active_levels; 4966 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_icvs->sched.sched); 4967 __kmp_reinitialize_team(team, new_icvs, 4968 root->r.r_uber_thread->th.th_ident); 4969 4970 /* update the remaining threads */ 4971 for (f = 0; f < new_nproc; ++f) { 4972 team->t.t_threads[f]->th.th_team_nproc = new_nproc; 4973 } 4974 // restore the current task state of the master thread: should be the 4975 // implicit task 4976 KF_TRACE(10, ("__kmp_allocate_team: T#%d, this_thread=%p team=%p\n", 0, 4977 team->t.t_threads[0], team)); 4978 4979 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0); 4980 4981 #ifdef KMP_DEBUG 4982 for (f = 0; f < team->t.t_nproc; f++) { 4983 KMP_DEBUG_ASSERT(team->t.t_threads[f] && 4984 team->t.t_threads[f]->th.th_team_nproc == 4985 team->t.t_nproc); 4986 } 4987 #endif 4988 4989 #if OMP_40_ENABLED 4990 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); 4991 #if KMP_AFFINITY_SUPPORTED 4992 __kmp_partition_places(team); 4993 #endif 4994 #endif 4995 } else { // team->t.t_nproc < new_nproc 4996 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 4997 kmp_affin_mask_t *old_mask; 4998 if (KMP_AFFINITY_CAPABLE()) { 4999 KMP_CPU_ALLOC(old_mask); 5000 } 5001 #endif 5002 5003 KA_TRACE(20, 5004 ("__kmp_allocate_team: increasing hot team thread count to %d\n", 5005 new_nproc)); 5006 5007 team->t.t_size_changed = 1; 5008 5009 #if KMP_NESTED_HOT_TEAMS 5010 int avail_threads = hot_teams[level].hot_team_nth; 5011 if (new_nproc < avail_threads) 5012 avail_threads = new_nproc; 5013 kmp_info_t **other_threads = team->t.t_threads; 5014 for (f = team->t.t_nproc; f < avail_threads; ++f) { 5015 // Adjust barrier data of reserved threads (if any) of the team 5016 // Other data will be set in __kmp_initialize_info() below. 5017 int b; 5018 kmp_balign_t *balign = other_threads[f]->th.th_bar; 5019 for (b = 0; b < bs_last_barrier; ++b) { 5020 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 5021 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 5022 #if USE_DEBUGGER 5023 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 5024 #endif 5025 } 5026 } 5027 if (hot_teams[level].hot_team_nth >= new_nproc) { 5028 // we have all needed threads in reserve, no need to allocate any 5029 // this only possible in mode 1, cannot have reserved threads in mode 0 5030 KMP_DEBUG_ASSERT(__kmp_hot_teams_mode == 1); 5031 team->t.t_nproc = new_nproc; // just get reserved threads involved 5032 } else { 5033 // we may have some threads in reserve, but not enough 5034 team->t.t_nproc = 5035 hot_teams[level] 5036 .hot_team_nth; // get reserved threads involved if any 5037 hot_teams[level].hot_team_nth = new_nproc; // adjust hot team max size 5038 #endif // KMP_NESTED_HOT_TEAMS 5039 if (team->t.t_max_nproc < new_nproc) { 5040 /* reallocate larger arrays */ 5041 __kmp_reallocate_team_arrays(team, new_nproc); 5042 __kmp_reinitialize_team(team, new_icvs, NULL); 5043 } 5044 5045 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 5046 /* Temporarily set full mask for master thread before creation of 5047 workers. The reason is that workers inherit the affinity from master, 5048 so if a lot of workers are created on the single core quickly, they 5049 don't get a chance to set their own affinity for a long time. */ 5050 __kmp_set_thread_affinity_mask_full_tmp(old_mask); 5051 #endif 5052 5053 /* allocate new threads for the hot team */ 5054 for (f = team->t.t_nproc; f < new_nproc; f++) { 5055 kmp_info_t *new_worker = __kmp_allocate_thread(root, team, f); 5056 KMP_DEBUG_ASSERT(new_worker); 5057 team->t.t_threads[f] = new_worker; 5058 5059 KA_TRACE(20, 5060 ("__kmp_allocate_team: team %d init T#%d arrived: " 5061 "join=%llu, plain=%llu\n", 5062 team->t.t_id, __kmp_gtid_from_tid(f, team), team->t.t_id, f, 5063 team->t.t_bar[bs_forkjoin_barrier].b_arrived, 5064 team->t.t_bar[bs_plain_barrier].b_arrived)); 5065 5066 { // Initialize barrier data for new threads. 5067 int b; 5068 kmp_balign_t *balign = new_worker->th.th_bar; 5069 for (b = 0; b < bs_last_barrier; ++b) { 5070 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 5071 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != 5072 KMP_BARRIER_PARENT_FLAG); 5073 #if USE_DEBUGGER 5074 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 5075 #endif 5076 } 5077 } 5078 } 5079 5080 #if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 5081 if (KMP_AFFINITY_CAPABLE()) { 5082 /* Restore initial master thread's affinity mask */ 5083 __kmp_set_system_affinity(old_mask, TRUE); 5084 KMP_CPU_FREE(old_mask); 5085 } 5086 #endif 5087 #if KMP_NESTED_HOT_TEAMS 5088 } // end of check of t_nproc vs. new_nproc vs. hot_team_nth 5089 #endif // KMP_NESTED_HOT_TEAMS 5090 /* make sure everyone is syncronized */ 5091 int old_nproc = team->t.t_nproc; // save old value and use to update only 5092 // new threads below 5093 __kmp_initialize_team(team, new_nproc, new_icvs, 5094 root->r.r_uber_thread->th.th_ident); 5095 5096 /* reinitialize the threads */ 5097 KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc); 5098 for (f = 0; f < team->t.t_nproc; ++f) 5099 __kmp_initialize_info(team->t.t_threads[f], team, f, 5100 __kmp_gtid_from_tid(f, team)); 5101 if (level) { // set th_task_state for new threads in nested hot team 5102 // __kmp_initialize_info() no longer zeroes th_task_state, so we should 5103 // only need to set the th_task_state for the new threads. th_task_state 5104 // for master thread will not be accurate until after this in 5105 // __kmp_fork_call(), so we look to the master's memo_stack to get the 5106 // correct value. 5107 for (f = old_nproc; f < team->t.t_nproc; ++f) 5108 team->t.t_threads[f]->th.th_task_state = 5109 team->t.t_threads[0]->th.th_task_state_memo_stack[level]; 5110 } else { // set th_task_state for new threads in non-nested hot team 5111 int old_state = 5112 team->t.t_threads[0]->th.th_task_state; // copy master's state 5113 for (f = old_nproc; f < team->t.t_nproc; ++f) 5114 team->t.t_threads[f]->th.th_task_state = old_state; 5115 } 5116 5117 #ifdef KMP_DEBUG 5118 for (f = 0; f < team->t.t_nproc; ++f) { 5119 KMP_DEBUG_ASSERT(team->t.t_threads[f] && 5120 team->t.t_threads[f]->th.th_team_nproc == 5121 team->t.t_nproc); 5122 } 5123 #endif 5124 5125 #if OMP_40_ENABLED 5126 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind); 5127 #if KMP_AFFINITY_SUPPORTED 5128 __kmp_partition_places(team); 5129 #endif 5130 #endif 5131 } // Check changes in number of threads 5132 5133 #if OMP_40_ENABLED 5134 kmp_info_t *master = team->t.t_threads[0]; 5135 if (master->th.th_teams_microtask) { 5136 for (f = 1; f < new_nproc; ++f) { 5137 // propagate teams construct specific info to workers 5138 kmp_info_t *thr = team->t.t_threads[f]; 5139 thr->th.th_teams_microtask = master->th.th_teams_microtask; 5140 thr->th.th_teams_level = master->th.th_teams_level; 5141 thr->th.th_teams_size = master->th.th_teams_size; 5142 } 5143 } 5144 #endif /* OMP_40_ENABLED */ 5145 #if KMP_NESTED_HOT_TEAMS 5146 if (level) { 5147 // Sync barrier state for nested hot teams, not needed for outermost hot 5148 // team. 5149 for (f = 1; f < new_nproc; ++f) { 5150 kmp_info_t *thr = team->t.t_threads[f]; 5151 int b; 5152 kmp_balign_t *balign = thr->th.th_bar; 5153 for (b = 0; b < bs_last_barrier; ++b) { 5154 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived; 5155 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG); 5156 #if USE_DEBUGGER 5157 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived; 5158 #endif 5159 } 5160 } 5161 } 5162 #endif // KMP_NESTED_HOT_TEAMS 5163 5164 /* reallocate space for arguments if necessary */ 5165 __kmp_alloc_argv_entries(argc, team, TRUE); 5166 KMP_CHECK_UPDATE(team->t.t_argc, argc); 5167 // The hot team re-uses the previous task team, 5168 // if untouched during the previous release->gather phase. 5169 5170 KF_TRACE(10, (" hot_team = %p\n", team)); 5171 5172 #if KMP_DEBUG 5173 if (__kmp_tasking_mode != tskm_immediate_exec) { 5174 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p " 5175 "task_team[1] = %p after reinit\n", 5176 team->t.t_task_team[0], team->t.t_task_team[1])); 5177 } 5178 #endif 5179 5180 #if OMPT_SUPPORT 5181 __ompt_team_assign_id(team, ompt_parallel_data); 5182 #endif 5183 5184 KMP_MB(); 5185 5186 return team; 5187 } 5188 5189 /* next, let's try to take one from the team pool */ 5190 KMP_MB(); 5191 for (team = CCAST(kmp_team_t *, __kmp_team_pool); (team);) { 5192 /* TODO: consider resizing undersized teams instead of reaping them, now 5193 that we have a resizing mechanism */ 5194 if (team->t.t_max_nproc >= max_nproc) { 5195 /* take this team from the team pool */ 5196 __kmp_team_pool = team->t.t_next_pool; 5197 5198 /* setup the team for fresh use */ 5199 __kmp_initialize_team(team, new_nproc, new_icvs, NULL); 5200 5201 KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and " 5202 "task_team[1] %p to NULL\n", 5203 &team->t.t_task_team[0], &team->t.t_task_team[1])); 5204 team->t.t_task_team[0] = NULL; 5205 team->t.t_task_team[1] = NULL; 5206 5207 /* reallocate space for arguments if necessary */ 5208 __kmp_alloc_argv_entries(argc, team, TRUE); 5209 KMP_CHECK_UPDATE(team->t.t_argc, argc); 5210 5211 KA_TRACE( 5212 20, ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n", 5213 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); 5214 { // Initialize barrier data. 5215 int b; 5216 for (b = 0; b < bs_last_barrier; ++b) { 5217 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE; 5218 #if USE_DEBUGGER 5219 team->t.t_bar[b].b_master_arrived = 0; 5220 team->t.t_bar[b].b_team_arrived = 0; 5221 #endif 5222 } 5223 } 5224 5225 #if OMP_40_ENABLED 5226 team->t.t_proc_bind = new_proc_bind; 5227 #endif 5228 5229 KA_TRACE(20, ("__kmp_allocate_team: using team from pool %d.\n", 5230 team->t.t_id)); 5231 5232 #if OMPT_SUPPORT 5233 __ompt_team_assign_id(team, ompt_parallel_data); 5234 #endif 5235 5236 KMP_MB(); 5237 5238 return team; 5239 } 5240 5241 /* reap team if it is too small, then loop back and check the next one */ 5242 // not sure if this is wise, but, will be redone during the hot-teams 5243 // rewrite. 5244 /* TODO: Use technique to find the right size hot-team, don't reap them */ 5245 team = __kmp_reap_team(team); 5246 __kmp_team_pool = team; 5247 } 5248 5249 /* nothing available in the pool, no matter, make a new team! */ 5250 KMP_MB(); 5251 team = (kmp_team_t *)__kmp_allocate(sizeof(kmp_team_t)); 5252 5253 /* and set it up */ 5254 team->t.t_max_nproc = max_nproc; 5255 /* NOTE well, for some reason allocating one big buffer and dividing it up 5256 seems to really hurt performance a lot on the P4, so, let's not use this */ 5257 __kmp_allocate_team_arrays(team, max_nproc); 5258 5259 KA_TRACE(20, ("__kmp_allocate_team: making a new team\n")); 5260 __kmp_initialize_team(team, new_nproc, new_icvs, NULL); 5261 5262 KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and task_team[1] " 5263 "%p to NULL\n", 5264 &team->t.t_task_team[0], &team->t.t_task_team[1])); 5265 team->t.t_task_team[0] = NULL; // to be removed, as __kmp_allocate zeroes 5266 // memory, no need to duplicate 5267 team->t.t_task_team[1] = NULL; // to be removed, as __kmp_allocate zeroes 5268 // memory, no need to duplicate 5269 5270 if (__kmp_storage_map) { 5271 __kmp_print_team_storage_map("team", team, team->t.t_id, new_nproc); 5272 } 5273 5274 /* allocate space for arguments */ 5275 __kmp_alloc_argv_entries(argc, team, FALSE); 5276 team->t.t_argc = argc; 5277 5278 KA_TRACE(20, 5279 ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n", 5280 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE)); 5281 { // Initialize barrier data. 5282 int b; 5283 for (b = 0; b < bs_last_barrier; ++b) { 5284 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE; 5285 #if USE_DEBUGGER 5286 team->t.t_bar[b].b_master_arrived = 0; 5287 team->t.t_bar[b].b_team_arrived = 0; 5288 #endif 5289 } 5290 } 5291 5292 #if OMP_40_ENABLED 5293 team->t.t_proc_bind = new_proc_bind; 5294 #endif 5295 5296 #if OMPT_SUPPORT 5297 __ompt_team_assign_id(team, ompt_parallel_data); 5298 team->t.ompt_serialized_team_info = NULL; 5299 #endif 5300 5301 KMP_MB(); 5302 5303 KA_TRACE(20, ("__kmp_allocate_team: done creating a new team %d.\n", 5304 team->t.t_id)); 5305 5306 return team; 5307 } 5308 5309 /* TODO implement hot-teams at all levels */ 5310 /* TODO implement lazy thread release on demand (disband request) */ 5311 5312 /* free the team. return it to the team pool. release all the threads 5313 * associated with it */ 5314 void __kmp_free_team(kmp_root_t *root, 5315 kmp_team_t *team USE_NESTED_HOT_ARG(kmp_info_t *master)) { 5316 int f; 5317 KA_TRACE(20, ("__kmp_free_team: T#%d freeing team %d\n", __kmp_get_gtid(), 5318 team->t.t_id)); 5319 5320 /* verify state */ 5321 KMP_DEBUG_ASSERT(root); 5322 KMP_DEBUG_ASSERT(team); 5323 KMP_DEBUG_ASSERT(team->t.t_nproc <= team->t.t_max_nproc); 5324 KMP_DEBUG_ASSERT(team->t.t_threads); 5325 5326 int use_hot_team = team == root->r.r_hot_team; 5327 #if KMP_NESTED_HOT_TEAMS 5328 int level; 5329 kmp_hot_team_ptr_t *hot_teams; 5330 if (master) { 5331 level = team->t.t_active_level - 1; 5332 if (master->th.th_teams_microtask) { // in teams construct? 5333 if (master->th.th_teams_size.nteams > 1) { 5334 ++level; // level was not increased in teams construct for 5335 // team_of_masters 5336 } 5337 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master && 5338 master->th.th_teams_level == team->t.t_level) { 5339 ++level; // level was not increased in teams construct for 5340 // team_of_workers before the parallel 5341 } // team->t.t_level will be increased inside parallel 5342 } 5343 hot_teams = master->th.th_hot_teams; 5344 if (level < __kmp_hot_teams_max_level) { 5345 KMP_DEBUG_ASSERT(team == hot_teams[level].hot_team); 5346 use_hot_team = 1; 5347 } 5348 } 5349 #endif // KMP_NESTED_HOT_TEAMS 5350 5351 /* team is done working */ 5352 TCW_SYNC_PTR(team->t.t_pkfn, 5353 NULL); // Important for Debugging Support Library. 5354 #if KMP_OS_WINDOWS 5355 team->t.t_copyin_counter = 0; // init counter for possible reuse 5356 #endif 5357 // Do not reset pointer to parent team to NULL for hot teams. 5358 5359 /* if we are non-hot team, release our threads */ 5360 if (!use_hot_team) { 5361 if (__kmp_tasking_mode != tskm_immediate_exec) { 5362 // Wait for threads to reach reapable state 5363 for (f = 1; f < team->t.t_nproc; ++f) { 5364 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 5365 kmp_info_t *th = team->t.t_threads[f]; 5366 volatile kmp_uint32 *state = &th->th.th_reap_state; 5367 while (*state != KMP_SAFE_TO_REAP) { 5368 #if KMP_OS_WINDOWS 5369 // On Windows a thread can be killed at any time, check this 5370 DWORD ecode; 5371 if (!__kmp_is_thread_alive(th, &ecode)) { 5372 *state = KMP_SAFE_TO_REAP; // reset the flag for dead thread 5373 break; 5374 } 5375 #endif 5376 // first check if thread is sleeping 5377 kmp_flag_64 fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th); 5378 if (fl.is_sleeping()) 5379 fl.resume(__kmp_gtid_from_thread(th)); 5380 KMP_CPU_PAUSE(); 5381 } 5382 } 5383 5384 // Delete task teams 5385 int tt_idx; 5386 for (tt_idx = 0; tt_idx < 2; ++tt_idx) { 5387 kmp_task_team_t *task_team = team->t.t_task_team[tt_idx]; 5388 if (task_team != NULL) { 5389 for (f = 0; f < team->t.t_nproc; 5390 ++f) { // Have all threads unref task teams 5391 team->t.t_threads[f]->th.th_task_team = NULL; 5392 } 5393 KA_TRACE( 5394 20, 5395 ("__kmp_free_team: T#%d deactivating task_team %p on team %d\n", 5396 __kmp_get_gtid(), task_team, team->t.t_id)); 5397 #if KMP_NESTED_HOT_TEAMS 5398 __kmp_free_task_team(master, task_team); 5399 #endif 5400 team->t.t_task_team[tt_idx] = NULL; 5401 } 5402 } 5403 } 5404 5405 // Reset pointer to parent team only for non-hot teams. 5406 team->t.t_parent = NULL; 5407 team->t.t_level = 0; 5408 team->t.t_active_level = 0; 5409 5410 /* free the worker threads */ 5411 for (f = 1; f < team->t.t_nproc; ++f) { 5412 KMP_DEBUG_ASSERT(team->t.t_threads[f]); 5413 __kmp_free_thread(team->t.t_threads[f]); 5414 team->t.t_threads[f] = NULL; 5415 } 5416 5417 /* put the team back in the team pool */ 5418 /* TODO limit size of team pool, call reap_team if pool too large */ 5419 team->t.t_next_pool = CCAST(kmp_team_t *, __kmp_team_pool); 5420 __kmp_team_pool = (volatile kmp_team_t *)team; 5421 } 5422 5423 KMP_MB(); 5424 } 5425 5426 /* reap the team. destroy it, reclaim all its resources and free its memory */ 5427 kmp_team_t *__kmp_reap_team(kmp_team_t *team) { 5428 kmp_team_t *next_pool = team->t.t_next_pool; 5429 5430 KMP_DEBUG_ASSERT(team); 5431 KMP_DEBUG_ASSERT(team->t.t_dispatch); 5432 KMP_DEBUG_ASSERT(team->t.t_disp_buffer); 5433 KMP_DEBUG_ASSERT(team->t.t_threads); 5434 KMP_DEBUG_ASSERT(team->t.t_argv); 5435 5436 /* TODO clean the threads that are a part of this? */ 5437 5438 /* free stuff */ 5439 __kmp_free_team_arrays(team); 5440 if (team->t.t_argv != &team->t.t_inline_argv[0]) 5441 __kmp_free((void *)team->t.t_argv); 5442 __kmp_free(team); 5443 5444 KMP_MB(); 5445 return next_pool; 5446 } 5447 5448 // Free the thread. Don't reap it, just place it on the pool of available 5449 // threads. 5450 // 5451 // Changes for Quad issue 527845: We need a predictable OMP tid <-> gtid 5452 // binding for the affinity mechanism to be useful. 5453 // 5454 // Now, we always keep the free list (__kmp_thread_pool) sorted by gtid. 5455 // However, we want to avoid a potential performance problem by always 5456 // scanning through the list to find the correct point at which to insert 5457 // the thread (potential N**2 behavior). To do this we keep track of the 5458 // last place a thread struct was inserted (__kmp_thread_pool_insert_pt). 5459 // With single-level parallelism, threads will always be added to the tail 5460 // of the list, kept track of by __kmp_thread_pool_insert_pt. With nested 5461 // parallelism, all bets are off and we may need to scan through the entire 5462 // free list. 5463 // 5464 // This change also has a potentially large performance benefit, for some 5465 // applications. Previously, as threads were freed from the hot team, they 5466 // would be placed back on the free list in inverse order. If the hot team 5467 // grew back to it's original size, then the freed thread would be placed 5468 // back on the hot team in reverse order. This could cause bad cache 5469 // locality problems on programs where the size of the hot team regularly 5470 // grew and shrunk. 5471 // 5472 // Now, for single-level parallelism, the OMP tid is alway == gtid. 5473 void __kmp_free_thread(kmp_info_t *this_th) { 5474 int gtid; 5475 kmp_info_t **scan; 5476 kmp_root_t *root = this_th->th.th_root; 5477 5478 KA_TRACE(20, ("__kmp_free_thread: T#%d putting T#%d back on free pool.\n", 5479 __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid)); 5480 5481 KMP_DEBUG_ASSERT(this_th); 5482 5483 // When moving thread to pool, switch thread to wait on own b_go flag, and 5484 // uninitialized (NULL team). 5485 int b; 5486 kmp_balign_t *balign = this_th->th.th_bar; 5487 for (b = 0; b < bs_last_barrier; ++b) { 5488 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG) 5489 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG; 5490 balign[b].bb.team = NULL; 5491 balign[b].bb.leaf_kids = 0; 5492 } 5493 this_th->th.th_task_state = 0; 5494 this_th->th.th_reap_state = KMP_SAFE_TO_REAP; 5495 5496 /* put thread back on the free pool */ 5497 TCW_PTR(this_th->th.th_team, NULL); 5498 TCW_PTR(this_th->th.th_root, NULL); 5499 TCW_PTR(this_th->th.th_dispatch, NULL); /* NOT NEEDED */ 5500 5501 /* If the implicit task assigned to this thread can be used by other threads 5502 * -> multiple threads can share the data and try to free the task at 5503 * __kmp_reap_thread at exit. This duplicate use of the task data can happen 5504 * with higher probability when hot team is disabled but can occurs even when 5505 * the hot team is enabled */ 5506 __kmp_free_implicit_task(this_th); 5507 this_th->th.th_current_task = NULL; 5508 5509 // If the __kmp_thread_pool_insert_pt is already past the new insert 5510 // point, then we need to re-scan the entire list. 5511 gtid = this_th->th.th_info.ds.ds_gtid; 5512 if (__kmp_thread_pool_insert_pt != NULL) { 5513 KMP_DEBUG_ASSERT(__kmp_thread_pool != NULL); 5514 if (__kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid) { 5515 __kmp_thread_pool_insert_pt = NULL; 5516 } 5517 } 5518 5519 // Scan down the list to find the place to insert the thread. 5520 // scan is the address of a link in the list, possibly the address of 5521 // __kmp_thread_pool itself. 5522 // 5523 // In the absence of nested parallism, the for loop will have 0 iterations. 5524 if (__kmp_thread_pool_insert_pt != NULL) { 5525 scan = &(__kmp_thread_pool_insert_pt->th.th_next_pool); 5526 } else { 5527 scan = CCAST(kmp_info_t **, &__kmp_thread_pool); 5528 } 5529 for (; (*scan != NULL) && ((*scan)->th.th_info.ds.ds_gtid < gtid); 5530 scan = &((*scan)->th.th_next_pool)) 5531 ; 5532 5533 // Insert the new element on the list, and set __kmp_thread_pool_insert_pt 5534 // to its address. 5535 TCW_PTR(this_th->th.th_next_pool, *scan); 5536 __kmp_thread_pool_insert_pt = *scan = this_th; 5537 KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) || 5538 (this_th->th.th_info.ds.ds_gtid < 5539 this_th->th.th_next_pool->th.th_info.ds.ds_gtid)); 5540 TCW_4(this_th->th.th_in_pool, TRUE); 5541 __kmp_thread_pool_nth++; 5542 5543 TCW_4(__kmp_nth, __kmp_nth - 1); 5544 root->r.r_cg_nthreads--; 5545 5546 #ifdef KMP_ADJUST_BLOCKTIME 5547 /* Adjust blocktime back to user setting or default if necessary */ 5548 /* Middle initialization might never have occurred */ 5549 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 5550 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0); 5551 if (__kmp_nth <= __kmp_avail_proc) { 5552 __kmp_zero_bt = FALSE; 5553 } 5554 } 5555 #endif /* KMP_ADJUST_BLOCKTIME */ 5556 5557 KMP_MB(); 5558 } 5559 5560 /* ------------------------------------------------------------------------ */ 5561 5562 void *__kmp_launch_thread(kmp_info_t *this_thr) { 5563 int gtid = this_thr->th.th_info.ds.ds_gtid; 5564 /* void *stack_data;*/ 5565 kmp_team_t *(*volatile pteam); 5566 5567 KMP_MB(); 5568 KA_TRACE(10, ("__kmp_launch_thread: T#%d start\n", gtid)); 5569 5570 if (__kmp_env_consistency_check) { 5571 this_thr->th.th_cons = __kmp_allocate_cons_stack(gtid); // ATT: Memory leak? 5572 } 5573 5574 #if OMPT_SUPPORT 5575 ompt_data_t *thread_data; 5576 if (ompt_enabled.enabled) { 5577 thread_data = &(this_thr->th.ompt_thread_info.thread_data); 5578 thread_data->ptr = NULL; 5579 5580 this_thr->th.ompt_thread_info.state = omp_state_overhead; 5581 this_thr->th.ompt_thread_info.wait_id = 0; 5582 this_thr->th.ompt_thread_info.idle_frame = OMPT_GET_FRAME_ADDRESS(0); 5583 if (ompt_enabled.ompt_callback_thread_begin) { 5584 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)( 5585 ompt_thread_worker, thread_data); 5586 } 5587 } 5588 #endif 5589 5590 #if OMPT_SUPPORT 5591 if (ompt_enabled.enabled) { 5592 this_thr->th.ompt_thread_info.state = omp_state_idle; 5593 } 5594 #endif 5595 /* This is the place where threads wait for work */ 5596 while (!TCR_4(__kmp_global.g.g_done)) { 5597 KMP_DEBUG_ASSERT(this_thr == __kmp_threads[gtid]); 5598 KMP_MB(); 5599 5600 /* wait for work to do */ 5601 KA_TRACE(20, ("__kmp_launch_thread: T#%d waiting for work\n", gtid)); 5602 5603 /* No tid yet since not part of a team */ 5604 __kmp_fork_barrier(gtid, KMP_GTID_DNE); 5605 5606 #if OMPT_SUPPORT 5607 if (ompt_enabled.enabled) { 5608 this_thr->th.ompt_thread_info.state = omp_state_overhead; 5609 } 5610 #endif 5611 5612 pteam = (kmp_team_t * (*))(&this_thr->th.th_team); 5613 5614 /* have we been allocated? */ 5615 if (TCR_SYNC_PTR(*pteam) && !TCR_4(__kmp_global.g.g_done)) { 5616 /* we were just woken up, so run our new task */ 5617 if (TCR_SYNC_PTR((*pteam)->t.t_pkfn) != NULL) { 5618 int rc; 5619 KA_TRACE(20, 5620 ("__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n", 5621 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), 5622 (*pteam)->t.t_pkfn)); 5623 5624 updateHWFPControl(*pteam); 5625 5626 #if OMPT_SUPPORT 5627 if (ompt_enabled.enabled) { 5628 this_thr->th.ompt_thread_info.state = omp_state_work_parallel; 5629 } 5630 #endif 5631 5632 rc = (*pteam)->t.t_invoke(gtid); 5633 KMP_ASSERT(rc); 5634 5635 KMP_MB(); 5636 KA_TRACE(20, ("__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n", 5637 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), 5638 (*pteam)->t.t_pkfn)); 5639 } 5640 #if OMPT_SUPPORT 5641 if (ompt_enabled.enabled) { 5642 /* no frame set while outside task */ 5643 __ompt_get_task_info_object(0)->frame.exit_frame = NULL; 5644 5645 this_thr->th.ompt_thread_info.state = omp_state_overhead; 5646 } 5647 #endif 5648 /* join barrier after parallel region */ 5649 __kmp_join_barrier(gtid); 5650 } 5651 } 5652 TCR_SYNC_PTR((intptr_t)__kmp_global.g.g_done); 5653 5654 #if OMPT_SUPPORT 5655 if (ompt_enabled.ompt_callback_thread_end) { 5656 ompt_callbacks.ompt_callback(ompt_callback_thread_end)(thread_data); 5657 } 5658 #endif 5659 5660 this_thr->th.th_task_team = NULL; 5661 /* run the destructors for the threadprivate data for this thread */ 5662 __kmp_common_destroy_gtid(gtid); 5663 5664 KA_TRACE(10, ("__kmp_launch_thread: T#%d done\n", gtid)); 5665 KMP_MB(); 5666 return this_thr; 5667 } 5668 5669 /* ------------------------------------------------------------------------ */ 5670 5671 void __kmp_internal_end_dest(void *specific_gtid) { 5672 #if KMP_COMPILER_ICC 5673 #pragma warning(push) 5674 #pragma warning(disable : 810) // conversion from "void *" to "int" may lose 5675 // significant bits 5676 #endif 5677 // Make sure no significant bits are lost 5678 int gtid = (kmp_intptr_t)specific_gtid - 1; 5679 #if KMP_COMPILER_ICC 5680 #pragma warning(pop) 5681 #endif 5682 5683 KA_TRACE(30, ("__kmp_internal_end_dest: T#%d\n", gtid)); 5684 /* NOTE: the gtid is stored as gitd+1 in the thread-local-storage 5685 * this is because 0 is reserved for the nothing-stored case */ 5686 5687 /* josh: One reason for setting the gtid specific data even when it is being 5688 destroyed by pthread is to allow gtid lookup through thread specific data 5689 (__kmp_gtid_get_specific). Some of the code, especially stat code, 5690 that gets executed in the call to __kmp_internal_end_thread, actually 5691 gets the gtid through the thread specific data. Setting it here seems 5692 rather inelegant and perhaps wrong, but allows __kmp_internal_end_thread 5693 to run smoothly. 5694 todo: get rid of this after we remove the dependence on 5695 __kmp_gtid_get_specific */ 5696 if (gtid >= 0 && KMP_UBER_GTID(gtid)) 5697 __kmp_gtid_set_specific(gtid); 5698 #ifdef KMP_TDATA_GTID 5699 __kmp_gtid = gtid; 5700 #endif 5701 __kmp_internal_end_thread(gtid); 5702 } 5703 5704 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB 5705 5706 // 2009-09-08 (lev): It looks the destructor does not work. In simple test cases 5707 // destructors work perfectly, but in real libomp.so I have no evidence it is 5708 // ever called. However, -fini linker option in makefile.mk works fine. 5709 5710 __attribute__((destructor)) void __kmp_internal_end_dtor(void) { 5711 __kmp_internal_end_atexit(); 5712 } 5713 5714 void __kmp_internal_end_fini(void) { __kmp_internal_end_atexit(); } 5715 5716 #endif 5717 5718 /* [Windows] josh: when the atexit handler is called, there may still be more 5719 than one thread alive */ 5720 void __kmp_internal_end_atexit(void) { 5721 KA_TRACE(30, ("__kmp_internal_end_atexit\n")); 5722 /* [Windows] 5723 josh: ideally, we want to completely shutdown the library in this atexit 5724 handler, but stat code that depends on thread specific data for gtid fails 5725 because that data becomes unavailable at some point during the shutdown, so 5726 we call __kmp_internal_end_thread instead. We should eventually remove the 5727 dependency on __kmp_get_specific_gtid in the stat code and use 5728 __kmp_internal_end_library to cleanly shutdown the library. 5729 5730 // TODO: Can some of this comment about GVS be removed? 5731 I suspect that the offending stat code is executed when the calling thread 5732 tries to clean up a dead root thread's data structures, resulting in GVS 5733 code trying to close the GVS structures for that thread, but since the stat 5734 code uses __kmp_get_specific_gtid to get the gtid with the assumption that 5735 the calling thread is cleaning up itself instead of another thread, it get 5736 confused. This happens because allowing a thread to unregister and cleanup 5737 another thread is a recent modification for addressing an issue. 5738 Based on the current design (20050722), a thread may end up 5739 trying to unregister another thread only if thread death does not trigger 5740 the calling of __kmp_internal_end_thread. For Linux* OS, there is the 5741 thread specific data destructor function to detect thread death. For 5742 Windows dynamic, there is DllMain(THREAD_DETACH). For Windows static, there 5743 is nothing. Thus, the workaround is applicable only for Windows static 5744 stat library. */ 5745 __kmp_internal_end_library(-1); 5746 #if KMP_OS_WINDOWS 5747 __kmp_close_console(); 5748 #endif 5749 } 5750 5751 static void __kmp_reap_thread(kmp_info_t *thread, int is_root) { 5752 // It is assumed __kmp_forkjoin_lock is acquired. 5753 5754 int gtid; 5755 5756 KMP_DEBUG_ASSERT(thread != NULL); 5757 5758 gtid = thread->th.th_info.ds.ds_gtid; 5759 5760 if (!is_root) { 5761 5762 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) { 5763 /* Assume the threads are at the fork barrier here */ 5764 KA_TRACE( 5765 20, ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n", 5766 gtid)); 5767 /* Need release fence here to prevent seg faults for tree forkjoin barrier 5768 * (GEH) */ 5769 ANNOTATE_HAPPENS_BEFORE(thread); 5770 kmp_flag_64 flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, thread); 5771 __kmp_release_64(&flag); 5772 } 5773 5774 // Terminate OS thread. 5775 __kmp_reap_worker(thread); 5776 5777 // The thread was killed asynchronously. If it was actively 5778 // spinning in the thread pool, decrement the global count. 5779 // 5780 // There is a small timing hole here - if the worker thread was just waking 5781 // up after sleeping in the pool, had reset it's th_active_in_pool flag but 5782 // not decremented the global counter __kmp_thread_pool_active_nth yet, then 5783 // the global counter might not get updated. 5784 // 5785 // Currently, this can only happen as the library is unloaded, 5786 // so there are no harmful side effects. 5787 if (thread->th.th_active_in_pool) { 5788 thread->th.th_active_in_pool = FALSE; 5789 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth); 5790 KMP_DEBUG_ASSERT(__kmp_thread_pool_active_nth >= 0); 5791 } 5792 5793 // Decrement # of [worker] threads in the pool. 5794 KMP_DEBUG_ASSERT(__kmp_thread_pool_nth > 0); 5795 --__kmp_thread_pool_nth; 5796 } 5797 5798 __kmp_free_implicit_task(thread); 5799 5800 // Free the fast memory for tasking 5801 #if USE_FAST_MEMORY 5802 __kmp_free_fast_memory(thread); 5803 #endif /* USE_FAST_MEMORY */ 5804 5805 __kmp_suspend_uninitialize_thread(thread); 5806 5807 KMP_DEBUG_ASSERT(__kmp_threads[gtid] == thread); 5808 TCW_SYNC_PTR(__kmp_threads[gtid], NULL); 5809 5810 --__kmp_all_nth; 5811 // __kmp_nth was decremented when thread is added to the pool. 5812 5813 #ifdef KMP_ADJUST_BLOCKTIME 5814 /* Adjust blocktime back to user setting or default if necessary */ 5815 /* Middle initialization might never have occurred */ 5816 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 5817 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0); 5818 if (__kmp_nth <= __kmp_avail_proc) { 5819 __kmp_zero_bt = FALSE; 5820 } 5821 } 5822 #endif /* KMP_ADJUST_BLOCKTIME */ 5823 5824 /* free the memory being used */ 5825 if (__kmp_env_consistency_check) { 5826 if (thread->th.th_cons) { 5827 __kmp_free_cons_stack(thread->th.th_cons); 5828 thread->th.th_cons = NULL; 5829 } 5830 } 5831 5832 if (thread->th.th_pri_common != NULL) { 5833 __kmp_free(thread->th.th_pri_common); 5834 thread->th.th_pri_common = NULL; 5835 } 5836 5837 if (thread->th.th_task_state_memo_stack != NULL) { 5838 __kmp_free(thread->th.th_task_state_memo_stack); 5839 thread->th.th_task_state_memo_stack = NULL; 5840 } 5841 5842 #if KMP_USE_BGET 5843 if (thread->th.th_local.bget_data != NULL) { 5844 __kmp_finalize_bget(thread); 5845 } 5846 #endif 5847 5848 #if KMP_AFFINITY_SUPPORTED 5849 if (thread->th.th_affin_mask != NULL) { 5850 KMP_CPU_FREE(thread->th.th_affin_mask); 5851 thread->th.th_affin_mask = NULL; 5852 } 5853 #endif /* KMP_AFFINITY_SUPPORTED */ 5854 5855 #if KMP_USE_HIER_SCHED 5856 if (thread->th.th_hier_bar_data != NULL) { 5857 __kmp_free(thread->th.th_hier_bar_data); 5858 thread->th.th_hier_bar_data = NULL; 5859 } 5860 #endif 5861 5862 __kmp_reap_team(thread->th.th_serial_team); 5863 thread->th.th_serial_team = NULL; 5864 __kmp_free(thread); 5865 5866 KMP_MB(); 5867 5868 } // __kmp_reap_thread 5869 5870 static void __kmp_internal_end(void) { 5871 int i; 5872 5873 /* First, unregister the library */ 5874 __kmp_unregister_library(); 5875 5876 #if KMP_OS_WINDOWS 5877 /* In Win static library, we can't tell when a root actually dies, so we 5878 reclaim the data structures for any root threads that have died but not 5879 unregistered themselves, in order to shut down cleanly. 5880 In Win dynamic library we also can't tell when a thread dies. */ 5881 __kmp_reclaim_dead_roots(); // AC: moved here to always clean resources of 5882 // dead roots 5883 #endif 5884 5885 for (i = 0; i < __kmp_threads_capacity; i++) 5886 if (__kmp_root[i]) 5887 if (__kmp_root[i]->r.r_active) 5888 break; 5889 KMP_MB(); /* Flush all pending memory write invalidates. */ 5890 TCW_SYNC_4(__kmp_global.g.g_done, TRUE); 5891 5892 if (i < __kmp_threads_capacity) { 5893 #if KMP_USE_MONITOR 5894 // 2009-09-08 (lev): Other alive roots found. Why do we kill the monitor?? 5895 KMP_MB(); /* Flush all pending memory write invalidates. */ 5896 5897 // Need to check that monitor was initialized before reaping it. If we are 5898 // called form __kmp_atfork_child (which sets __kmp_init_parallel = 0), then 5899 // __kmp_monitor will appear to contain valid data, but it is only valid in 5900 // the parent process, not the child. 5901 // New behavior (201008): instead of keying off of the flag 5902 // __kmp_init_parallel, the monitor thread creation is keyed off 5903 // of the new flag __kmp_init_monitor. 5904 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock); 5905 if (TCR_4(__kmp_init_monitor)) { 5906 __kmp_reap_monitor(&__kmp_monitor); 5907 TCW_4(__kmp_init_monitor, 0); 5908 } 5909 __kmp_release_bootstrap_lock(&__kmp_monitor_lock); 5910 KA_TRACE(10, ("__kmp_internal_end: monitor reaped\n")); 5911 #endif // KMP_USE_MONITOR 5912 } else { 5913 /* TODO move this to cleanup code */ 5914 #ifdef KMP_DEBUG 5915 /* make sure that everything has properly ended */ 5916 for (i = 0; i < __kmp_threads_capacity; i++) { 5917 if (__kmp_root[i]) { 5918 // KMP_ASSERT( ! KMP_UBER_GTID( i ) ); // AC: 5919 // there can be uber threads alive here 5920 KMP_ASSERT(!__kmp_root[i]->r.r_active); // TODO: can they be active? 5921 } 5922 } 5923 #endif 5924 5925 KMP_MB(); 5926 5927 // Reap the worker threads. 5928 // This is valid for now, but be careful if threads are reaped sooner. 5929 while (__kmp_thread_pool != NULL) { // Loop thru all the thread in the pool. 5930 // Get the next thread from the pool. 5931 kmp_info_t *thread = CCAST(kmp_info_t *, __kmp_thread_pool); 5932 __kmp_thread_pool = thread->th.th_next_pool; 5933 // Reap it. 5934 KMP_DEBUG_ASSERT(thread->th.th_reap_state == KMP_SAFE_TO_REAP); 5935 thread->th.th_next_pool = NULL; 5936 thread->th.th_in_pool = FALSE; 5937 __kmp_reap_thread(thread, 0); 5938 } 5939 __kmp_thread_pool_insert_pt = NULL; 5940 5941 // Reap teams. 5942 while (__kmp_team_pool != NULL) { // Loop thru all the teams in the pool. 5943 // Get the next team from the pool. 5944 kmp_team_t *team = CCAST(kmp_team_t *, __kmp_team_pool); 5945 __kmp_team_pool = team->t.t_next_pool; 5946 // Reap it. 5947 team->t.t_next_pool = NULL; 5948 __kmp_reap_team(team); 5949 } 5950 5951 __kmp_reap_task_teams(); 5952 5953 #if KMP_OS_UNIX 5954 // Threads that are not reaped should not access any resources since they 5955 // are going to be deallocated soon, so the shutdown sequence should wait 5956 // until all threads either exit the final spin-waiting loop or begin 5957 // sleeping after the given blocktime. 5958 for (i = 0; i < __kmp_threads_capacity; i++) { 5959 kmp_info_t *thr = __kmp_threads[i]; 5960 while (thr && KMP_ATOMIC_LD_ACQ(&thr->th.th_blocking)) 5961 KMP_CPU_PAUSE(); 5962 } 5963 #endif 5964 5965 for (i = 0; i < __kmp_threads_capacity; ++i) { 5966 // TBD: Add some checking... 5967 // Something like KMP_DEBUG_ASSERT( __kmp_thread[ i ] == NULL ); 5968 } 5969 5970 /* Make sure all threadprivate destructors get run by joining with all 5971 worker threads before resetting this flag */ 5972 TCW_SYNC_4(__kmp_init_common, FALSE); 5973 5974 KA_TRACE(10, ("__kmp_internal_end: all workers reaped\n")); 5975 KMP_MB(); 5976 5977 #if KMP_USE_MONITOR 5978 // See note above: One of the possible fixes for CQ138434 / CQ140126 5979 // 5980 // FIXME: push both code fragments down and CSE them? 5981 // push them into __kmp_cleanup() ? 5982 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock); 5983 if (TCR_4(__kmp_init_monitor)) { 5984 __kmp_reap_monitor(&__kmp_monitor); 5985 TCW_4(__kmp_init_monitor, 0); 5986 } 5987 __kmp_release_bootstrap_lock(&__kmp_monitor_lock); 5988 KA_TRACE(10, ("__kmp_internal_end: monitor reaped\n")); 5989 #endif 5990 } /* else !__kmp_global.t_active */ 5991 TCW_4(__kmp_init_gtid, FALSE); 5992 KMP_MB(); /* Flush all pending memory write invalidates. */ 5993 5994 __kmp_cleanup(); 5995 #if OMPT_SUPPORT 5996 ompt_fini(); 5997 #endif 5998 } 5999 6000 void __kmp_internal_end_library(int gtid_req) { 6001 /* if we have already cleaned up, don't try again, it wouldn't be pretty */ 6002 /* this shouldn't be a race condition because __kmp_internal_end() is the 6003 only place to clear __kmp_serial_init */ 6004 /* we'll check this later too, after we get the lock */ 6005 // 2009-09-06: We do not set g_abort without setting g_done. This check looks 6006 // redundaant, because the next check will work in any case. 6007 if (__kmp_global.g.g_abort) { 6008 KA_TRACE(11, ("__kmp_internal_end_library: abort, exiting\n")); 6009 /* TODO abort? */ 6010 return; 6011 } 6012 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 6013 KA_TRACE(10, ("__kmp_internal_end_library: already finished\n")); 6014 return; 6015 } 6016 6017 KMP_MB(); /* Flush all pending memory write invalidates. */ 6018 6019 /* find out who we are and what we should do */ 6020 { 6021 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific(); 6022 KA_TRACE( 6023 10, ("__kmp_internal_end_library: enter T#%d (%d)\n", gtid, gtid_req)); 6024 if (gtid == KMP_GTID_SHUTDOWN) { 6025 KA_TRACE(10, ("__kmp_internal_end_library: !__kmp_init_runtime, system " 6026 "already shutdown\n")); 6027 return; 6028 } else if (gtid == KMP_GTID_MONITOR) { 6029 KA_TRACE(10, ("__kmp_internal_end_library: monitor thread, gtid not " 6030 "registered, or system shutdown\n")); 6031 return; 6032 } else if (gtid == KMP_GTID_DNE) { 6033 KA_TRACE(10, ("__kmp_internal_end_library: gtid not registered or system " 6034 "shutdown\n")); 6035 /* we don't know who we are, but we may still shutdown the library */ 6036 } else if (KMP_UBER_GTID(gtid)) { 6037 /* unregister ourselves as an uber thread. gtid is no longer valid */ 6038 if (__kmp_root[gtid]->r.r_active) { 6039 __kmp_global.g.g_abort = -1; 6040 TCW_SYNC_4(__kmp_global.g.g_done, TRUE); 6041 KA_TRACE(10, 6042 ("__kmp_internal_end_library: root still active, abort T#%d\n", 6043 gtid)); 6044 return; 6045 } else { 6046 KA_TRACE( 6047 10, 6048 ("__kmp_internal_end_library: unregistering sibling T#%d\n", gtid)); 6049 __kmp_unregister_root_current_thread(gtid); 6050 } 6051 } else { 6052 /* worker threads may call this function through the atexit handler, if they 6053 * call exit() */ 6054 /* For now, skip the usual subsequent processing and just dump the debug buffer. 6055 TODO: do a thorough shutdown instead */ 6056 #ifdef DUMP_DEBUG_ON_EXIT 6057 if (__kmp_debug_buf) 6058 __kmp_dump_debug_buffer(); 6059 #endif 6060 return; 6061 } 6062 } 6063 /* synchronize the termination process */ 6064 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6065 6066 /* have we already finished */ 6067 if (__kmp_global.g.g_abort) { 6068 KA_TRACE(10, ("__kmp_internal_end_library: abort, exiting\n")); 6069 /* TODO abort? */ 6070 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6071 return; 6072 } 6073 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 6074 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6075 return; 6076 } 6077 6078 /* We need this lock to enforce mutex between this reading of 6079 __kmp_threads_capacity and the writing by __kmp_register_root. 6080 Alternatively, we can use a counter of roots that is atomically updated by 6081 __kmp_get_global_thread_id_reg, __kmp_do_serial_initialize and 6082 __kmp_internal_end_*. */ 6083 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 6084 6085 /* now we can safely conduct the actual termination */ 6086 __kmp_internal_end(); 6087 6088 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 6089 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6090 6091 KA_TRACE(10, ("__kmp_internal_end_library: exit\n")); 6092 6093 #ifdef DUMP_DEBUG_ON_EXIT 6094 if (__kmp_debug_buf) 6095 __kmp_dump_debug_buffer(); 6096 #endif 6097 6098 #if KMP_OS_WINDOWS 6099 __kmp_close_console(); 6100 #endif 6101 6102 __kmp_fini_allocator(); 6103 6104 } // __kmp_internal_end_library 6105 6106 void __kmp_internal_end_thread(int gtid_req) { 6107 int i; 6108 6109 /* if we have already cleaned up, don't try again, it wouldn't be pretty */ 6110 /* this shouldn't be a race condition because __kmp_internal_end() is the 6111 * only place to clear __kmp_serial_init */ 6112 /* we'll check this later too, after we get the lock */ 6113 // 2009-09-06: We do not set g_abort without setting g_done. This check looks 6114 // redundant, because the next check will work in any case. 6115 if (__kmp_global.g.g_abort) { 6116 KA_TRACE(11, ("__kmp_internal_end_thread: abort, exiting\n")); 6117 /* TODO abort? */ 6118 return; 6119 } 6120 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 6121 KA_TRACE(10, ("__kmp_internal_end_thread: already finished\n")); 6122 return; 6123 } 6124 6125 KMP_MB(); /* Flush all pending memory write invalidates. */ 6126 6127 /* find out who we are and what we should do */ 6128 { 6129 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific(); 6130 KA_TRACE(10, 6131 ("__kmp_internal_end_thread: enter T#%d (%d)\n", gtid, gtid_req)); 6132 if (gtid == KMP_GTID_SHUTDOWN) { 6133 KA_TRACE(10, ("__kmp_internal_end_thread: !__kmp_init_runtime, system " 6134 "already shutdown\n")); 6135 return; 6136 } else if (gtid == KMP_GTID_MONITOR) { 6137 KA_TRACE(10, ("__kmp_internal_end_thread: monitor thread, gtid not " 6138 "registered, or system shutdown\n")); 6139 return; 6140 } else if (gtid == KMP_GTID_DNE) { 6141 KA_TRACE(10, ("__kmp_internal_end_thread: gtid not registered or system " 6142 "shutdown\n")); 6143 return; 6144 /* we don't know who we are */ 6145 } else if (KMP_UBER_GTID(gtid)) { 6146 /* unregister ourselves as an uber thread. gtid is no longer valid */ 6147 if (__kmp_root[gtid]->r.r_active) { 6148 __kmp_global.g.g_abort = -1; 6149 TCW_SYNC_4(__kmp_global.g.g_done, TRUE); 6150 KA_TRACE(10, 6151 ("__kmp_internal_end_thread: root still active, abort T#%d\n", 6152 gtid)); 6153 return; 6154 } else { 6155 KA_TRACE(10, ("__kmp_internal_end_thread: unregistering sibling T#%d\n", 6156 gtid)); 6157 __kmp_unregister_root_current_thread(gtid); 6158 } 6159 } else { 6160 /* just a worker thread, let's leave */ 6161 KA_TRACE(10, ("__kmp_internal_end_thread: worker thread T#%d\n", gtid)); 6162 6163 if (gtid >= 0) { 6164 __kmp_threads[gtid]->th.th_task_team = NULL; 6165 } 6166 6167 KA_TRACE(10, 6168 ("__kmp_internal_end_thread: worker thread done, exiting T#%d\n", 6169 gtid)); 6170 return; 6171 } 6172 } 6173 #if defined KMP_DYNAMIC_LIB 6174 // AC: lets not shutdown the Linux* OS dynamic library at the exit of uber 6175 // thread, because we will better shutdown later in the library destructor. 6176 // The reason of this change is performance problem when non-openmp thread in 6177 // a loop forks and joins many openmp threads. We can save a lot of time 6178 // keeping worker threads alive until the program shutdown. 6179 // OM: Removed Linux* OS restriction to fix the crash on OS X* (DPD200239966) 6180 // and Windows(DPD200287443) that occurs when using critical sections from 6181 // foreign threads. 6182 KA_TRACE(10, ("__kmp_internal_end_thread: exiting T#%d\n", gtid_req)); 6183 return; 6184 #endif 6185 /* synchronize the termination process */ 6186 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6187 6188 /* have we already finished */ 6189 if (__kmp_global.g.g_abort) { 6190 KA_TRACE(10, ("__kmp_internal_end_thread: abort, exiting\n")); 6191 /* TODO abort? */ 6192 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6193 return; 6194 } 6195 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) { 6196 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6197 return; 6198 } 6199 6200 /* We need this lock to enforce mutex between this reading of 6201 __kmp_threads_capacity and the writing by __kmp_register_root. 6202 Alternatively, we can use a counter of roots that is atomically updated by 6203 __kmp_get_global_thread_id_reg, __kmp_do_serial_initialize and 6204 __kmp_internal_end_*. */ 6205 6206 /* should we finish the run-time? are all siblings done? */ 6207 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 6208 6209 for (i = 0; i < __kmp_threads_capacity; ++i) { 6210 if (KMP_UBER_GTID(i)) { 6211 KA_TRACE( 6212 10, 6213 ("__kmp_internal_end_thread: remaining sibling task: gtid==%d\n", i)); 6214 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 6215 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6216 return; 6217 } 6218 } 6219 6220 /* now we can safely conduct the actual termination */ 6221 6222 __kmp_internal_end(); 6223 6224 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 6225 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6226 6227 KA_TRACE(10, ("__kmp_internal_end_thread: exit T#%d\n", gtid_req)); 6228 6229 #ifdef DUMP_DEBUG_ON_EXIT 6230 if (__kmp_debug_buf) 6231 __kmp_dump_debug_buffer(); 6232 #endif 6233 } // __kmp_internal_end_thread 6234 6235 // ----------------------------------------------------------------------------- 6236 // Library registration stuff. 6237 6238 static long __kmp_registration_flag = 0; 6239 // Random value used to indicate library initialization. 6240 static char *__kmp_registration_str = NULL; 6241 // Value to be saved in env var __KMP_REGISTERED_LIB_<pid>. 6242 6243 static inline char *__kmp_reg_status_name() { 6244 /* On RHEL 3u5 if linked statically, getpid() returns different values in 6245 each thread. If registration and unregistration go in different threads 6246 (omp_misc_other_root_exit.cpp test case), the name of registered_lib_env 6247 env var can not be found, because the name will contain different pid. */ 6248 return __kmp_str_format("__KMP_REGISTERED_LIB_%d", (int)getpid()); 6249 } // __kmp_reg_status_get 6250 6251 void __kmp_register_library_startup(void) { 6252 6253 char *name = __kmp_reg_status_name(); // Name of the environment variable. 6254 int done = 0; 6255 union { 6256 double dtime; 6257 long ltime; 6258 } time; 6259 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 6260 __kmp_initialize_system_tick(); 6261 #endif 6262 __kmp_read_system_time(&time.dtime); 6263 __kmp_registration_flag = 0xCAFE0000L | (time.ltime & 0x0000FFFFL); 6264 __kmp_registration_str = 6265 __kmp_str_format("%p-%lx-%s", &__kmp_registration_flag, 6266 __kmp_registration_flag, KMP_LIBRARY_FILE); 6267 6268 KA_TRACE(50, ("__kmp_register_library_startup: %s=\"%s\"\n", name, 6269 __kmp_registration_str)); 6270 6271 while (!done) { 6272 6273 char *value = NULL; // Actual value of the environment variable. 6274 6275 // Set environment variable, but do not overwrite if it is exist. 6276 __kmp_env_set(name, __kmp_registration_str, 0); 6277 // Check the variable is written. 6278 value = __kmp_env_get(name); 6279 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) { 6280 6281 done = 1; // Ok, environment variable set successfully, exit the loop. 6282 6283 } else { 6284 6285 // Oops. Write failed. Another copy of OpenMP RTL is in memory. 6286 // Check whether it alive or dead. 6287 int neighbor = 0; // 0 -- unknown status, 1 -- alive, 2 -- dead. 6288 char *tail = value; 6289 char *flag_addr_str = NULL; 6290 char *flag_val_str = NULL; 6291 char const *file_name = NULL; 6292 __kmp_str_split(tail, '-', &flag_addr_str, &tail); 6293 __kmp_str_split(tail, '-', &flag_val_str, &tail); 6294 file_name = tail; 6295 if (tail != NULL) { 6296 long *flag_addr = 0; 6297 long flag_val = 0; 6298 KMP_SSCANF(flag_addr_str, "%p", RCAST(void**, &flag_addr)); 6299 KMP_SSCANF(flag_val_str, "%lx", &flag_val); 6300 if (flag_addr != 0 && flag_val != 0 && strcmp(file_name, "") != 0) { 6301 // First, check whether environment-encoded address is mapped into 6302 // addr space. 6303 // If so, dereference it to see if it still has the right value. 6304 if (__kmp_is_address_mapped(flag_addr) && *flag_addr == flag_val) { 6305 neighbor = 1; 6306 } else { 6307 // If not, then we know the other copy of the library is no longer 6308 // running. 6309 neighbor = 2; 6310 } 6311 } 6312 } 6313 switch (neighbor) { 6314 case 0: // Cannot parse environment variable -- neighbor status unknown. 6315 // Assume it is the incompatible format of future version of the 6316 // library. Assume the other library is alive. 6317 // WARN( ... ); // TODO: Issue a warning. 6318 file_name = "unknown library"; 6319 // Attention! Falling to the next case. That's intentional. 6320 case 1: { // Neighbor is alive. 6321 // Check it is allowed. 6322 char *duplicate_ok = __kmp_env_get("KMP_DUPLICATE_LIB_OK"); 6323 if (!__kmp_str_match_true(duplicate_ok)) { 6324 // That's not allowed. Issue fatal error. 6325 __kmp_fatal(KMP_MSG(DuplicateLibrary, KMP_LIBRARY_FILE, file_name), 6326 KMP_HNT(DuplicateLibrary), __kmp_msg_null); 6327 } 6328 KMP_INTERNAL_FREE(duplicate_ok); 6329 __kmp_duplicate_library_ok = 1; 6330 done = 1; // Exit the loop. 6331 } break; 6332 case 2: { // Neighbor is dead. 6333 // Clear the variable and try to register library again. 6334 __kmp_env_unset(name); 6335 } break; 6336 default: { KMP_DEBUG_ASSERT(0); } break; 6337 } 6338 } 6339 KMP_INTERNAL_FREE((void *)value); 6340 } 6341 KMP_INTERNAL_FREE((void *)name); 6342 6343 } // func __kmp_register_library_startup 6344 6345 void __kmp_unregister_library(void) { 6346 6347 char *name = __kmp_reg_status_name(); 6348 char *value = __kmp_env_get(name); 6349 6350 KMP_DEBUG_ASSERT(__kmp_registration_flag != 0); 6351 KMP_DEBUG_ASSERT(__kmp_registration_str != NULL); 6352 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) { 6353 // Ok, this is our variable. Delete it. 6354 __kmp_env_unset(name); 6355 } 6356 6357 KMP_INTERNAL_FREE(__kmp_registration_str); 6358 KMP_INTERNAL_FREE(value); 6359 KMP_INTERNAL_FREE(name); 6360 6361 __kmp_registration_flag = 0; 6362 __kmp_registration_str = NULL; 6363 6364 } // __kmp_unregister_library 6365 6366 // End of Library registration stuff. 6367 // ----------------------------------------------------------------------------- 6368 6369 #if KMP_MIC_SUPPORTED 6370 6371 static void __kmp_check_mic_type() { 6372 kmp_cpuid_t cpuid_state = {0}; 6373 kmp_cpuid_t *cs_p = &cpuid_state; 6374 __kmp_x86_cpuid(1, 0, cs_p); 6375 // We don't support mic1 at the moment 6376 if ((cs_p->eax & 0xff0) == 0xB10) { 6377 __kmp_mic_type = mic2; 6378 } else if ((cs_p->eax & 0xf0ff0) == 0x50670) { 6379 __kmp_mic_type = mic3; 6380 } else { 6381 __kmp_mic_type = non_mic; 6382 } 6383 } 6384 6385 #endif /* KMP_MIC_SUPPORTED */ 6386 6387 static void __kmp_do_serial_initialize(void) { 6388 int i, gtid; 6389 int size; 6390 6391 KA_TRACE(10, ("__kmp_do_serial_initialize: enter\n")); 6392 6393 KMP_DEBUG_ASSERT(sizeof(kmp_int32) == 4); 6394 KMP_DEBUG_ASSERT(sizeof(kmp_uint32) == 4); 6395 KMP_DEBUG_ASSERT(sizeof(kmp_int64) == 8); 6396 KMP_DEBUG_ASSERT(sizeof(kmp_uint64) == 8); 6397 KMP_DEBUG_ASSERT(sizeof(kmp_intptr_t) == sizeof(void *)); 6398 6399 #if OMPT_SUPPORT 6400 ompt_pre_init(); 6401 #endif 6402 6403 __kmp_validate_locks(); 6404 6405 /* Initialize internal memory allocator */ 6406 __kmp_init_allocator(); 6407 6408 /* Register the library startup via an environment variable and check to see 6409 whether another copy of the library is already registered. */ 6410 6411 __kmp_register_library_startup(); 6412 6413 /* TODO reinitialization of library */ 6414 if (TCR_4(__kmp_global.g.g_done)) { 6415 KA_TRACE(10, ("__kmp_do_serial_initialize: reinitialization of library\n")); 6416 } 6417 6418 __kmp_global.g.g_abort = 0; 6419 TCW_SYNC_4(__kmp_global.g.g_done, FALSE); 6420 6421 /* initialize the locks */ 6422 #if KMP_USE_ADAPTIVE_LOCKS 6423 #if KMP_DEBUG_ADAPTIVE_LOCKS 6424 __kmp_init_speculative_stats(); 6425 #endif 6426 #endif 6427 #if KMP_STATS_ENABLED 6428 __kmp_stats_init(); 6429 #endif 6430 __kmp_init_lock(&__kmp_global_lock); 6431 __kmp_init_queuing_lock(&__kmp_dispatch_lock); 6432 __kmp_init_lock(&__kmp_debug_lock); 6433 __kmp_init_atomic_lock(&__kmp_atomic_lock); 6434 __kmp_init_atomic_lock(&__kmp_atomic_lock_1i); 6435 __kmp_init_atomic_lock(&__kmp_atomic_lock_2i); 6436 __kmp_init_atomic_lock(&__kmp_atomic_lock_4i); 6437 __kmp_init_atomic_lock(&__kmp_atomic_lock_4r); 6438 __kmp_init_atomic_lock(&__kmp_atomic_lock_8i); 6439 __kmp_init_atomic_lock(&__kmp_atomic_lock_8r); 6440 __kmp_init_atomic_lock(&__kmp_atomic_lock_8c); 6441 __kmp_init_atomic_lock(&__kmp_atomic_lock_10r); 6442 __kmp_init_atomic_lock(&__kmp_atomic_lock_16r); 6443 __kmp_init_atomic_lock(&__kmp_atomic_lock_16c); 6444 __kmp_init_atomic_lock(&__kmp_atomic_lock_20c); 6445 __kmp_init_atomic_lock(&__kmp_atomic_lock_32c); 6446 __kmp_init_bootstrap_lock(&__kmp_forkjoin_lock); 6447 __kmp_init_bootstrap_lock(&__kmp_exit_lock); 6448 #if KMP_USE_MONITOR 6449 __kmp_init_bootstrap_lock(&__kmp_monitor_lock); 6450 #endif 6451 __kmp_init_bootstrap_lock(&__kmp_tp_cached_lock); 6452 6453 /* conduct initialization and initial setup of configuration */ 6454 6455 __kmp_runtime_initialize(); 6456 6457 #if KMP_MIC_SUPPORTED 6458 __kmp_check_mic_type(); 6459 #endif 6460 6461 // Some global variable initialization moved here from kmp_env_initialize() 6462 #ifdef KMP_DEBUG 6463 kmp_diag = 0; 6464 #endif 6465 __kmp_abort_delay = 0; 6466 6467 // From __kmp_init_dflt_team_nth() 6468 /* assume the entire machine will be used */ 6469 __kmp_dflt_team_nth_ub = __kmp_xproc; 6470 if (__kmp_dflt_team_nth_ub < KMP_MIN_NTH) { 6471 __kmp_dflt_team_nth_ub = KMP_MIN_NTH; 6472 } 6473 if (__kmp_dflt_team_nth_ub > __kmp_sys_max_nth) { 6474 __kmp_dflt_team_nth_ub = __kmp_sys_max_nth; 6475 } 6476 __kmp_max_nth = __kmp_sys_max_nth; 6477 __kmp_cg_max_nth = __kmp_sys_max_nth; 6478 __kmp_teams_max_nth = __kmp_xproc; // set a "reasonable" default 6479 if (__kmp_teams_max_nth > __kmp_sys_max_nth) { 6480 __kmp_teams_max_nth = __kmp_sys_max_nth; 6481 } 6482 6483 // Three vars below moved here from __kmp_env_initialize() "KMP_BLOCKTIME" 6484 // part 6485 __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME; 6486 #if KMP_USE_MONITOR 6487 __kmp_monitor_wakeups = 6488 KMP_WAKEUPS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups); 6489 __kmp_bt_intervals = 6490 KMP_INTERVALS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups); 6491 #endif 6492 // From "KMP_LIBRARY" part of __kmp_env_initialize() 6493 __kmp_library = library_throughput; 6494 // From KMP_SCHEDULE initialization 6495 __kmp_static = kmp_sch_static_balanced; 6496 // AC: do not use analytical here, because it is non-monotonous 6497 //__kmp_guided = kmp_sch_guided_iterative_chunked; 6498 //__kmp_auto = kmp_sch_guided_analytical_chunked; // AC: it is the default, no 6499 // need to repeat assignment 6500 // Barrier initialization. Moved here from __kmp_env_initialize() Barrier branch 6501 // bit control and barrier method control parts 6502 #if KMP_FAST_REDUCTION_BARRIER 6503 #define kmp_reduction_barrier_gather_bb ((int)1) 6504 #define kmp_reduction_barrier_release_bb ((int)1) 6505 #define kmp_reduction_barrier_gather_pat bp_hyper_bar 6506 #define kmp_reduction_barrier_release_pat bp_hyper_bar 6507 #endif // KMP_FAST_REDUCTION_BARRIER 6508 for (i = bs_plain_barrier; i < bs_last_barrier; i++) { 6509 __kmp_barrier_gather_branch_bits[i] = __kmp_barrier_gather_bb_dflt; 6510 __kmp_barrier_release_branch_bits[i] = __kmp_barrier_release_bb_dflt; 6511 __kmp_barrier_gather_pattern[i] = __kmp_barrier_gather_pat_dflt; 6512 __kmp_barrier_release_pattern[i] = __kmp_barrier_release_pat_dflt; 6513 #if KMP_FAST_REDUCTION_BARRIER 6514 if (i == bs_reduction_barrier) { // tested and confirmed on ALTIX only ( 6515 // lin_64 ): hyper,1 6516 __kmp_barrier_gather_branch_bits[i] = kmp_reduction_barrier_gather_bb; 6517 __kmp_barrier_release_branch_bits[i] = kmp_reduction_barrier_release_bb; 6518 __kmp_barrier_gather_pattern[i] = kmp_reduction_barrier_gather_pat; 6519 __kmp_barrier_release_pattern[i] = kmp_reduction_barrier_release_pat; 6520 } 6521 #endif // KMP_FAST_REDUCTION_BARRIER 6522 } 6523 #if KMP_FAST_REDUCTION_BARRIER 6524 #undef kmp_reduction_barrier_release_pat 6525 #undef kmp_reduction_barrier_gather_pat 6526 #undef kmp_reduction_barrier_release_bb 6527 #undef kmp_reduction_barrier_gather_bb 6528 #endif // KMP_FAST_REDUCTION_BARRIER 6529 #if KMP_MIC_SUPPORTED 6530 if (__kmp_mic_type == mic2) { // KNC 6531 // AC: plane=3,2, forkjoin=2,1 are optimal for 240 threads on KNC 6532 __kmp_barrier_gather_branch_bits[bs_plain_barrier] = 3; // plain gather 6533 __kmp_barrier_release_branch_bits[bs_forkjoin_barrier] = 6534 1; // forkjoin release 6535 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar; 6536 __kmp_barrier_release_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar; 6537 } 6538 #if KMP_FAST_REDUCTION_BARRIER 6539 if (__kmp_mic_type == mic2) { // KNC 6540 __kmp_barrier_gather_pattern[bs_reduction_barrier] = bp_hierarchical_bar; 6541 __kmp_barrier_release_pattern[bs_reduction_barrier] = bp_hierarchical_bar; 6542 } 6543 #endif // KMP_FAST_REDUCTION_BARRIER 6544 #endif // KMP_MIC_SUPPORTED 6545 6546 // From KMP_CHECKS initialization 6547 #ifdef KMP_DEBUG 6548 __kmp_env_checks = TRUE; /* development versions have the extra checks */ 6549 #else 6550 __kmp_env_checks = FALSE; /* port versions do not have the extra checks */ 6551 #endif 6552 6553 // From "KMP_FOREIGN_THREADS_THREADPRIVATE" initialization 6554 __kmp_foreign_tp = TRUE; 6555 6556 __kmp_global.g.g_dynamic = FALSE; 6557 __kmp_global.g.g_dynamic_mode = dynamic_default; 6558 6559 __kmp_env_initialize(NULL); 6560 6561 // Print all messages in message catalog for testing purposes. 6562 #ifdef KMP_DEBUG 6563 char const *val = __kmp_env_get("KMP_DUMP_CATALOG"); 6564 if (__kmp_str_match_true(val)) { 6565 kmp_str_buf_t buffer; 6566 __kmp_str_buf_init(&buffer); 6567 __kmp_i18n_dump_catalog(&buffer); 6568 __kmp_printf("%s", buffer.str); 6569 __kmp_str_buf_free(&buffer); 6570 } 6571 __kmp_env_free(&val); 6572 #endif 6573 6574 __kmp_threads_capacity = 6575 __kmp_initial_threads_capacity(__kmp_dflt_team_nth_ub); 6576 // Moved here from __kmp_env_initialize() "KMP_ALL_THREADPRIVATE" part 6577 __kmp_tp_capacity = __kmp_default_tp_capacity( 6578 __kmp_dflt_team_nth_ub, __kmp_max_nth, __kmp_allThreadsSpecified); 6579 6580 // If the library is shut down properly, both pools must be NULL. Just in 6581 // case, set them to NULL -- some memory may leak, but subsequent code will 6582 // work even if pools are not freed. 6583 KMP_DEBUG_ASSERT(__kmp_thread_pool == NULL); 6584 KMP_DEBUG_ASSERT(__kmp_thread_pool_insert_pt == NULL); 6585 KMP_DEBUG_ASSERT(__kmp_team_pool == NULL); 6586 __kmp_thread_pool = NULL; 6587 __kmp_thread_pool_insert_pt = NULL; 6588 __kmp_team_pool = NULL; 6589 6590 /* Allocate all of the variable sized records */ 6591 /* NOTE: __kmp_threads_capacity entries are allocated, but the arrays are 6592 * expandable */ 6593 /* Since allocation is cache-aligned, just add extra padding at the end */ 6594 size = 6595 (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * __kmp_threads_capacity + 6596 CACHE_LINE; 6597 __kmp_threads = (kmp_info_t **)__kmp_allocate(size); 6598 __kmp_root = (kmp_root_t **)((char *)__kmp_threads + 6599 sizeof(kmp_info_t *) * __kmp_threads_capacity); 6600 6601 /* init thread counts */ 6602 KMP_DEBUG_ASSERT(__kmp_all_nth == 6603 0); // Asserts fail if the library is reinitializing and 6604 KMP_DEBUG_ASSERT(__kmp_nth == 0); // something was wrong in termination. 6605 __kmp_all_nth = 0; 6606 __kmp_nth = 0; 6607 6608 /* setup the uber master thread and hierarchy */ 6609 gtid = __kmp_register_root(TRUE); 6610 KA_TRACE(10, ("__kmp_do_serial_initialize T#%d\n", gtid)); 6611 KMP_ASSERT(KMP_UBER_GTID(gtid)); 6612 KMP_ASSERT(KMP_INITIAL_GTID(gtid)); 6613 6614 KMP_MB(); /* Flush all pending memory write invalidates. */ 6615 6616 __kmp_common_initialize(); 6617 6618 #if KMP_OS_UNIX 6619 /* invoke the child fork handler */ 6620 __kmp_register_atfork(); 6621 #endif 6622 6623 #if !defined KMP_DYNAMIC_LIB 6624 { 6625 /* Invoke the exit handler when the program finishes, only for static 6626 library. For dynamic library, we already have _fini and DllMain. */ 6627 int rc = atexit(__kmp_internal_end_atexit); 6628 if (rc != 0) { 6629 __kmp_fatal(KMP_MSG(FunctionError, "atexit()"), KMP_ERR(rc), 6630 __kmp_msg_null); 6631 } 6632 } 6633 #endif 6634 6635 #if KMP_HANDLE_SIGNALS 6636 #if KMP_OS_UNIX 6637 /* NOTE: make sure that this is called before the user installs their own 6638 signal handlers so that the user handlers are called first. this way they 6639 can return false, not call our handler, avoid terminating the library, and 6640 continue execution where they left off. */ 6641 __kmp_install_signals(FALSE); 6642 #endif /* KMP_OS_UNIX */ 6643 #if KMP_OS_WINDOWS 6644 __kmp_install_signals(TRUE); 6645 #endif /* KMP_OS_WINDOWS */ 6646 #endif 6647 6648 /* we have finished the serial initialization */ 6649 __kmp_init_counter++; 6650 6651 __kmp_init_serial = TRUE; 6652 6653 if (__kmp_settings) { 6654 __kmp_env_print(); 6655 } 6656 6657 #if OMP_40_ENABLED 6658 if (__kmp_display_env || __kmp_display_env_verbose) { 6659 __kmp_env_print_2(); 6660 } 6661 #endif // OMP_40_ENABLED 6662 6663 #if OMPT_SUPPORT 6664 ompt_post_init(); 6665 #endif 6666 6667 KMP_MB(); 6668 6669 KA_TRACE(10, ("__kmp_do_serial_initialize: exit\n")); 6670 } 6671 6672 void __kmp_serial_initialize(void) { 6673 if (__kmp_init_serial) { 6674 return; 6675 } 6676 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6677 if (__kmp_init_serial) { 6678 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6679 return; 6680 } 6681 __kmp_do_serial_initialize(); 6682 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6683 } 6684 6685 static void __kmp_do_middle_initialize(void) { 6686 int i, j; 6687 int prev_dflt_team_nth; 6688 6689 if (!__kmp_init_serial) { 6690 __kmp_do_serial_initialize(); 6691 } 6692 6693 KA_TRACE(10, ("__kmp_middle_initialize: enter\n")); 6694 6695 // Save the previous value for the __kmp_dflt_team_nth so that 6696 // we can avoid some reinitialization if it hasn't changed. 6697 prev_dflt_team_nth = __kmp_dflt_team_nth; 6698 6699 #if KMP_AFFINITY_SUPPORTED 6700 // __kmp_affinity_initialize() will try to set __kmp_ncores to the 6701 // number of cores on the machine. 6702 __kmp_affinity_initialize(); 6703 6704 // Run through the __kmp_threads array and set the affinity mask 6705 // for each root thread that is currently registered with the RTL. 6706 for (i = 0; i < __kmp_threads_capacity; i++) { 6707 if (TCR_PTR(__kmp_threads[i]) != NULL) { 6708 __kmp_affinity_set_init_mask(i, TRUE); 6709 } 6710 } 6711 #endif /* KMP_AFFINITY_SUPPORTED */ 6712 6713 KMP_ASSERT(__kmp_xproc > 0); 6714 if (__kmp_avail_proc == 0) { 6715 __kmp_avail_proc = __kmp_xproc; 6716 } 6717 6718 // If there were empty places in num_threads list (OMP_NUM_THREADS=,,2,3), 6719 // correct them now 6720 j = 0; 6721 while ((j < __kmp_nested_nth.used) && !__kmp_nested_nth.nth[j]) { 6722 __kmp_nested_nth.nth[j] = __kmp_dflt_team_nth = __kmp_dflt_team_nth_ub = 6723 __kmp_avail_proc; 6724 j++; 6725 } 6726 6727 if (__kmp_dflt_team_nth == 0) { 6728 #ifdef KMP_DFLT_NTH_CORES 6729 // Default #threads = #cores 6730 __kmp_dflt_team_nth = __kmp_ncores; 6731 KA_TRACE(20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = " 6732 "__kmp_ncores (%d)\n", 6733 __kmp_dflt_team_nth)); 6734 #else 6735 // Default #threads = #available OS procs 6736 __kmp_dflt_team_nth = __kmp_avail_proc; 6737 KA_TRACE(20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = " 6738 "__kmp_avail_proc(%d)\n", 6739 __kmp_dflt_team_nth)); 6740 #endif /* KMP_DFLT_NTH_CORES */ 6741 } 6742 6743 if (__kmp_dflt_team_nth < KMP_MIN_NTH) { 6744 __kmp_dflt_team_nth = KMP_MIN_NTH; 6745 } 6746 if (__kmp_dflt_team_nth > __kmp_sys_max_nth) { 6747 __kmp_dflt_team_nth = __kmp_sys_max_nth; 6748 } 6749 6750 // There's no harm in continuing if the following check fails, 6751 // but it indicates an error in the previous logic. 6752 KMP_DEBUG_ASSERT(__kmp_dflt_team_nth <= __kmp_dflt_team_nth_ub); 6753 6754 if (__kmp_dflt_team_nth != prev_dflt_team_nth) { 6755 // Run through the __kmp_threads array and set the num threads icv for each 6756 // root thread that is currently registered with the RTL (which has not 6757 // already explicitly set its nthreads-var with a call to 6758 // omp_set_num_threads()). 6759 for (i = 0; i < __kmp_threads_capacity; i++) { 6760 kmp_info_t *thread = __kmp_threads[i]; 6761 if (thread == NULL) 6762 continue; 6763 if (thread->th.th_current_task->td_icvs.nproc != 0) 6764 continue; 6765 6766 set__nproc(__kmp_threads[i], __kmp_dflt_team_nth); 6767 } 6768 } 6769 KA_TRACE( 6770 20, 6771 ("__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n", 6772 __kmp_dflt_team_nth)); 6773 6774 #ifdef KMP_ADJUST_BLOCKTIME 6775 /* Adjust blocktime to zero if necessary now that __kmp_avail_proc is set */ 6776 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) { 6777 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0); 6778 if (__kmp_nth > __kmp_avail_proc) { 6779 __kmp_zero_bt = TRUE; 6780 } 6781 } 6782 #endif /* KMP_ADJUST_BLOCKTIME */ 6783 6784 /* we have finished middle initialization */ 6785 TCW_SYNC_4(__kmp_init_middle, TRUE); 6786 6787 KA_TRACE(10, ("__kmp_do_middle_initialize: exit\n")); 6788 } 6789 6790 void __kmp_middle_initialize(void) { 6791 if (__kmp_init_middle) { 6792 return; 6793 } 6794 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6795 if (__kmp_init_middle) { 6796 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6797 return; 6798 } 6799 __kmp_do_middle_initialize(); 6800 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6801 } 6802 6803 void __kmp_parallel_initialize(void) { 6804 int gtid = __kmp_entry_gtid(); // this might be a new root 6805 6806 /* synchronize parallel initialization (for sibling) */ 6807 if (TCR_4(__kmp_init_parallel)) 6808 return; 6809 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 6810 if (TCR_4(__kmp_init_parallel)) { 6811 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6812 return; 6813 } 6814 6815 /* TODO reinitialization after we have already shut down */ 6816 if (TCR_4(__kmp_global.g.g_done)) { 6817 KA_TRACE( 6818 10, 6819 ("__kmp_parallel_initialize: attempt to init while shutting down\n")); 6820 __kmp_infinite_loop(); 6821 } 6822 6823 /* jc: The lock __kmp_initz_lock is already held, so calling 6824 __kmp_serial_initialize would cause a deadlock. So we call 6825 __kmp_do_serial_initialize directly. */ 6826 if (!__kmp_init_middle) { 6827 __kmp_do_middle_initialize(); 6828 } 6829 6830 /* begin initialization */ 6831 KA_TRACE(10, ("__kmp_parallel_initialize: enter\n")); 6832 KMP_ASSERT(KMP_UBER_GTID(gtid)); 6833 6834 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 6835 // Save the FP control regs. 6836 // Worker threads will set theirs to these values at thread startup. 6837 __kmp_store_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word); 6838 __kmp_store_mxcsr(&__kmp_init_mxcsr); 6839 __kmp_init_mxcsr &= KMP_X86_MXCSR_MASK; 6840 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 6841 6842 #if KMP_OS_UNIX 6843 #if KMP_HANDLE_SIGNALS 6844 /* must be after __kmp_serial_initialize */ 6845 __kmp_install_signals(TRUE); 6846 #endif 6847 #endif 6848 6849 __kmp_suspend_initialize(); 6850 6851 #if defined(USE_LOAD_BALANCE) 6852 if (__kmp_global.g.g_dynamic_mode == dynamic_default) { 6853 __kmp_global.g.g_dynamic_mode = dynamic_load_balance; 6854 } 6855 #else 6856 if (__kmp_global.g.g_dynamic_mode == dynamic_default) { 6857 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit; 6858 } 6859 #endif 6860 6861 if (__kmp_version) { 6862 __kmp_print_version_2(); 6863 } 6864 6865 /* we have finished parallel initialization */ 6866 TCW_SYNC_4(__kmp_init_parallel, TRUE); 6867 6868 KMP_MB(); 6869 KA_TRACE(10, ("__kmp_parallel_initialize: exit\n")); 6870 6871 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 6872 } 6873 6874 /* ------------------------------------------------------------------------ */ 6875 6876 void __kmp_run_before_invoked_task(int gtid, int tid, kmp_info_t *this_thr, 6877 kmp_team_t *team) { 6878 kmp_disp_t *dispatch; 6879 6880 KMP_MB(); 6881 6882 /* none of the threads have encountered any constructs, yet. */ 6883 this_thr->th.th_local.this_construct = 0; 6884 #if KMP_CACHE_MANAGE 6885 KMP_CACHE_PREFETCH(&this_thr->th.th_bar[bs_forkjoin_barrier].bb.b_arrived); 6886 #endif /* KMP_CACHE_MANAGE */ 6887 dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch); 6888 KMP_DEBUG_ASSERT(dispatch); 6889 KMP_DEBUG_ASSERT(team->t.t_dispatch); 6890 // KMP_DEBUG_ASSERT( this_thr->th.th_dispatch == &team->t.t_dispatch[ 6891 // this_thr->th.th_info.ds.ds_tid ] ); 6892 6893 dispatch->th_disp_index = 0; /* reset the dispatch buffer counter */ 6894 #if OMP_45_ENABLED 6895 dispatch->th_doacross_buf_idx = 6896 0; /* reset the doacross dispatch buffer counter */ 6897 #endif 6898 if (__kmp_env_consistency_check) 6899 __kmp_push_parallel(gtid, team->t.t_ident); 6900 6901 KMP_MB(); /* Flush all pending memory write invalidates. */ 6902 } 6903 6904 void __kmp_run_after_invoked_task(int gtid, int tid, kmp_info_t *this_thr, 6905 kmp_team_t *team) { 6906 if (__kmp_env_consistency_check) 6907 __kmp_pop_parallel(gtid, team->t.t_ident); 6908 6909 __kmp_finish_implicit_task(this_thr); 6910 } 6911 6912 int __kmp_invoke_task_func(int gtid) { 6913 int rc; 6914 int tid = __kmp_tid_from_gtid(gtid); 6915 kmp_info_t *this_thr = __kmp_threads[gtid]; 6916 kmp_team_t *team = this_thr->th.th_team; 6917 6918 __kmp_run_before_invoked_task(gtid, tid, this_thr, team); 6919 #if USE_ITT_BUILD 6920 if (__itt_stack_caller_create_ptr) { 6921 __kmp_itt_stack_callee_enter( 6922 (__itt_caller) 6923 team->t.t_stack_id); // inform ittnotify about entering user's code 6924 } 6925 #endif /* USE_ITT_BUILD */ 6926 #if INCLUDE_SSC_MARKS 6927 SSC_MARK_INVOKING(); 6928 #endif 6929 6930 #if OMPT_SUPPORT 6931 void *dummy; 6932 void **exit_runtime_p; 6933 ompt_data_t *my_task_data; 6934 ompt_data_t *my_parallel_data; 6935 int ompt_team_size; 6936 6937 if (ompt_enabled.enabled) { 6938 exit_runtime_p = &( 6939 team->t.t_implicit_task_taskdata[tid].ompt_task_info.frame.exit_frame); 6940 } else { 6941 exit_runtime_p = &dummy; 6942 } 6943 6944 my_task_data = 6945 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data); 6946 my_parallel_data = &(team->t.ompt_team_info.parallel_data); 6947 if (ompt_enabled.ompt_callback_implicit_task) { 6948 ompt_team_size = team->t.t_nproc; 6949 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 6950 ompt_scope_begin, my_parallel_data, my_task_data, ompt_team_size, 6951 __kmp_tid_from_gtid(gtid)); 6952 OMPT_CUR_TASK_INFO(this_thr)->thread_num = __kmp_tid_from_gtid(gtid); 6953 } 6954 #endif 6955 6956 { 6957 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel); 6958 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK); 6959 rc = 6960 __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn), gtid, 6961 tid, (int)team->t.t_argc, (void **)team->t.t_argv 6962 #if OMPT_SUPPORT 6963 , 6964 exit_runtime_p 6965 #endif 6966 ); 6967 #if OMPT_SUPPORT 6968 *exit_runtime_p = NULL; 6969 #endif 6970 } 6971 6972 #if USE_ITT_BUILD 6973 if (__itt_stack_caller_create_ptr) { 6974 __kmp_itt_stack_callee_leave( 6975 (__itt_caller) 6976 team->t.t_stack_id); // inform ittnotify about leaving user's code 6977 } 6978 #endif /* USE_ITT_BUILD */ 6979 __kmp_run_after_invoked_task(gtid, tid, this_thr, team); 6980 6981 return rc; 6982 } 6983 6984 #if OMP_40_ENABLED 6985 void __kmp_teams_master(int gtid) { 6986 // This routine is called by all master threads in teams construct 6987 kmp_info_t *thr = __kmp_threads[gtid]; 6988 kmp_team_t *team = thr->th.th_team; 6989 ident_t *loc = team->t.t_ident; 6990 thr->th.th_set_nproc = thr->th.th_teams_size.nth; 6991 KMP_DEBUG_ASSERT(thr->th.th_teams_microtask); 6992 KMP_DEBUG_ASSERT(thr->th.th_set_nproc); 6993 KA_TRACE(20, ("__kmp_teams_master: T#%d, Tid %d, microtask %p\n", gtid, 6994 __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask)); 6995 // Launch league of teams now, but not let workers execute 6996 // (they hang on fork barrier until next parallel) 6997 #if INCLUDE_SSC_MARKS 6998 SSC_MARK_FORKING(); 6999 #endif 7000 __kmp_fork_call(loc, gtid, fork_context_intel, team->t.t_argc, 7001 (microtask_t)thr->th.th_teams_microtask, // "wrapped" task 7002 VOLATILE_CAST(launch_t) __kmp_invoke_task_func, NULL); 7003 #if INCLUDE_SSC_MARKS 7004 SSC_MARK_JOINING(); 7005 #endif 7006 7007 // AC: last parameter "1" eliminates join barrier which won't work because 7008 // worker threads are in a fork barrier waiting for more parallel regions 7009 __kmp_join_call(loc, gtid 7010 #if OMPT_SUPPORT 7011 , 7012 fork_context_intel 7013 #endif 7014 , 7015 1); 7016 } 7017 7018 int __kmp_invoke_teams_master(int gtid) { 7019 kmp_info_t *this_thr = __kmp_threads[gtid]; 7020 kmp_team_t *team = this_thr->th.th_team; 7021 #if KMP_DEBUG 7022 if (!__kmp_threads[gtid]->th.th_team->t.t_serialized) 7023 KMP_DEBUG_ASSERT((void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn == 7024 (void *)__kmp_teams_master); 7025 #endif 7026 __kmp_run_before_invoked_task(gtid, 0, this_thr, team); 7027 __kmp_teams_master(gtid); 7028 __kmp_run_after_invoked_task(gtid, 0, this_thr, team); 7029 return 1; 7030 } 7031 #endif /* OMP_40_ENABLED */ 7032 7033 /* this sets the requested number of threads for the next parallel region 7034 encountered by this team. since this should be enclosed in the forkjoin 7035 critical section it should avoid race conditions with assymmetrical nested 7036 parallelism */ 7037 7038 void __kmp_push_num_threads(ident_t *id, int gtid, int num_threads) { 7039 kmp_info_t *thr = __kmp_threads[gtid]; 7040 7041 if (num_threads > 0) 7042 thr->th.th_set_nproc = num_threads; 7043 } 7044 7045 #if OMP_40_ENABLED 7046 7047 /* this sets the requested number of teams for the teams region and/or 7048 the number of threads for the next parallel region encountered */ 7049 void __kmp_push_num_teams(ident_t *id, int gtid, int num_teams, 7050 int num_threads) { 7051 kmp_info_t *thr = __kmp_threads[gtid]; 7052 KMP_DEBUG_ASSERT(num_teams >= 0); 7053 KMP_DEBUG_ASSERT(num_threads >= 0); 7054 7055 if (num_teams == 0) 7056 num_teams = 1; // default number of teams is 1. 7057 if (num_teams > __kmp_teams_max_nth) { // if too many teams requested? 7058 if (!__kmp_reserve_warn) { 7059 __kmp_reserve_warn = 1; 7060 __kmp_msg(kmp_ms_warning, 7061 KMP_MSG(CantFormThrTeam, num_teams, __kmp_teams_max_nth), 7062 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 7063 } 7064 num_teams = __kmp_teams_max_nth; 7065 } 7066 // Set number of teams (number of threads in the outer "parallel" of the 7067 // teams) 7068 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams; 7069 7070 // Remember the number of threads for inner parallel regions 7071 if (num_threads == 0) { 7072 if (!TCR_4(__kmp_init_middle)) 7073 __kmp_middle_initialize(); // get __kmp_avail_proc calculated 7074 num_threads = __kmp_avail_proc / num_teams; 7075 if (num_teams * num_threads > __kmp_teams_max_nth) { 7076 // adjust num_threads w/o warning as it is not user setting 7077 num_threads = __kmp_teams_max_nth / num_teams; 7078 } 7079 } else { 7080 if (num_teams * num_threads > __kmp_teams_max_nth) { 7081 int new_threads = __kmp_teams_max_nth / num_teams; 7082 if (!__kmp_reserve_warn) { // user asked for too many threads 7083 __kmp_reserve_warn = 1; // that conflicts with KMP_TEAMS_THREAD_LIMIT 7084 __kmp_msg(kmp_ms_warning, 7085 KMP_MSG(CantFormThrTeam, num_threads, new_threads), 7086 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null); 7087 } 7088 num_threads = new_threads; 7089 } 7090 } 7091 thr->th.th_teams_size.nth = num_threads; 7092 } 7093 7094 // Set the proc_bind var to use in the following parallel region. 7095 void __kmp_push_proc_bind(ident_t *id, int gtid, kmp_proc_bind_t proc_bind) { 7096 kmp_info_t *thr = __kmp_threads[gtid]; 7097 thr->th.th_set_proc_bind = proc_bind; 7098 } 7099 7100 #endif /* OMP_40_ENABLED */ 7101 7102 /* Launch the worker threads into the microtask. */ 7103 7104 void __kmp_internal_fork(ident_t *id, int gtid, kmp_team_t *team) { 7105 kmp_info_t *this_thr = __kmp_threads[gtid]; 7106 7107 #ifdef KMP_DEBUG 7108 int f; 7109 #endif /* KMP_DEBUG */ 7110 7111 KMP_DEBUG_ASSERT(team); 7112 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); 7113 KMP_ASSERT(KMP_MASTER_GTID(gtid)); 7114 KMP_MB(); /* Flush all pending memory write invalidates. */ 7115 7116 team->t.t_construct = 0; /* no single directives seen yet */ 7117 team->t.t_ordered.dt.t_value = 7118 0; /* thread 0 enters the ordered section first */ 7119 7120 /* Reset the identifiers on the dispatch buffer */ 7121 KMP_DEBUG_ASSERT(team->t.t_disp_buffer); 7122 if (team->t.t_max_nproc > 1) { 7123 int i; 7124 for (i = 0; i < __kmp_dispatch_num_buffers; ++i) { 7125 team->t.t_disp_buffer[i].buffer_index = i; 7126 #if OMP_45_ENABLED 7127 team->t.t_disp_buffer[i].doacross_buf_idx = i; 7128 #endif 7129 } 7130 } else { 7131 team->t.t_disp_buffer[0].buffer_index = 0; 7132 #if OMP_45_ENABLED 7133 team->t.t_disp_buffer[0].doacross_buf_idx = 0; 7134 #endif 7135 } 7136 7137 KMP_MB(); /* Flush all pending memory write invalidates. */ 7138 KMP_ASSERT(this_thr->th.th_team == team); 7139 7140 #ifdef KMP_DEBUG 7141 for (f = 0; f < team->t.t_nproc; f++) { 7142 KMP_DEBUG_ASSERT(team->t.t_threads[f] && 7143 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc); 7144 } 7145 #endif /* KMP_DEBUG */ 7146 7147 /* release the worker threads so they may begin working */ 7148 __kmp_fork_barrier(gtid, 0); 7149 } 7150 7151 void __kmp_internal_join(ident_t *id, int gtid, kmp_team_t *team) { 7152 kmp_info_t *this_thr = __kmp_threads[gtid]; 7153 7154 KMP_DEBUG_ASSERT(team); 7155 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); 7156 KMP_ASSERT(KMP_MASTER_GTID(gtid)); 7157 KMP_MB(); /* Flush all pending memory write invalidates. */ 7158 7159 /* Join barrier after fork */ 7160 7161 #ifdef KMP_DEBUG 7162 if (__kmp_threads[gtid] && 7163 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) { 7164 __kmp_printf("GTID: %d, __kmp_threads[%d]=%p\n", gtid, gtid, 7165 __kmp_threads[gtid]); 7166 __kmp_printf("__kmp_threads[%d]->th.th_team_nproc=%d, TEAM: %p, " 7167 "team->t.t_nproc=%d\n", 7168 gtid, __kmp_threads[gtid]->th.th_team_nproc, team, 7169 team->t.t_nproc); 7170 __kmp_print_structure(); 7171 } 7172 KMP_DEBUG_ASSERT(__kmp_threads[gtid] && 7173 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc); 7174 #endif /* KMP_DEBUG */ 7175 7176 __kmp_join_barrier(gtid); /* wait for everyone */ 7177 #if OMPT_SUPPORT 7178 if (ompt_enabled.enabled && 7179 this_thr->th.ompt_thread_info.state == omp_state_wait_barrier_implicit) { 7180 int ds_tid = this_thr->th.th_info.ds.ds_tid; 7181 ompt_data_t *task_data = OMPT_CUR_TASK_DATA(this_thr); 7182 this_thr->th.ompt_thread_info.state = omp_state_overhead; 7183 #if OMPT_OPTIONAL 7184 void *codeptr = NULL; 7185 if (KMP_MASTER_TID(ds_tid) && 7186 (ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait) || 7187 ompt_callbacks.ompt_callback(ompt_callback_sync_region))) 7188 codeptr = OMPT_CUR_TEAM_INFO(this_thr)->master_return_address; 7189 7190 if (ompt_enabled.ompt_callback_sync_region_wait) { 7191 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)( 7192 ompt_sync_region_barrier, ompt_scope_end, NULL, task_data, codeptr); 7193 } 7194 if (ompt_enabled.ompt_callback_sync_region) { 7195 ompt_callbacks.ompt_callback(ompt_callback_sync_region)( 7196 ompt_sync_region_barrier, ompt_scope_end, NULL, task_data, codeptr); 7197 } 7198 #endif 7199 if (!KMP_MASTER_TID(ds_tid) && ompt_enabled.ompt_callback_implicit_task) { 7200 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 7201 ompt_scope_end, NULL, task_data, 0, ds_tid); 7202 } 7203 } 7204 #endif 7205 7206 KMP_MB(); /* Flush all pending memory write invalidates. */ 7207 KMP_ASSERT(this_thr->th.th_team == team); 7208 } 7209 7210 /* ------------------------------------------------------------------------ */ 7211 7212 #ifdef USE_LOAD_BALANCE 7213 7214 // Return the worker threads actively spinning in the hot team, if we 7215 // are at the outermost level of parallelism. Otherwise, return 0. 7216 static int __kmp_active_hot_team_nproc(kmp_root_t *root) { 7217 int i; 7218 int retval; 7219 kmp_team_t *hot_team; 7220 7221 if (root->r.r_active) { 7222 return 0; 7223 } 7224 hot_team = root->r.r_hot_team; 7225 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) { 7226 return hot_team->t.t_nproc - 1; // Don't count master thread 7227 } 7228 7229 // Skip the master thread - it is accounted for elsewhere. 7230 retval = 0; 7231 for (i = 1; i < hot_team->t.t_nproc; i++) { 7232 if (hot_team->t.t_threads[i]->th.th_active) { 7233 retval++; 7234 } 7235 } 7236 return retval; 7237 } 7238 7239 // Perform an automatic adjustment to the number of 7240 // threads used by the next parallel region. 7241 static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc) { 7242 int retval; 7243 int pool_active; 7244 int hot_team_active; 7245 int team_curr_active; 7246 int system_active; 7247 7248 KB_TRACE(20, ("__kmp_load_balance_nproc: called root:%p set_nproc:%d\n", root, 7249 set_nproc)); 7250 KMP_DEBUG_ASSERT(root); 7251 KMP_DEBUG_ASSERT(root->r.r_root_team->t.t_threads[0] 7252 ->th.th_current_task->td_icvs.dynamic == TRUE); 7253 KMP_DEBUG_ASSERT(set_nproc > 1); 7254 7255 if (set_nproc == 1) { 7256 KB_TRACE(20, ("__kmp_load_balance_nproc: serial execution.\n")); 7257 return 1; 7258 } 7259 7260 // Threads that are active in the thread pool, active in the hot team for this 7261 // particular root (if we are at the outer par level), and the currently 7262 // executing thread (to become the master) are available to add to the new 7263 // team, but are currently contributing to the system load, and must be 7264 // accounted for. 7265 pool_active = __kmp_thread_pool_active_nth; 7266 hot_team_active = __kmp_active_hot_team_nproc(root); 7267 team_curr_active = pool_active + hot_team_active + 1; 7268 7269 // Check the system load. 7270 system_active = __kmp_get_load_balance(__kmp_avail_proc + team_curr_active); 7271 KB_TRACE(30, ("__kmp_load_balance_nproc: system active = %d pool active = %d " 7272 "hot team active = %d\n", 7273 system_active, pool_active, hot_team_active)); 7274 7275 if (system_active < 0) { 7276 // There was an error reading the necessary info from /proc, so use the 7277 // thread limit algorithm instead. Once we set __kmp_global.g.g_dynamic_mode 7278 // = dynamic_thread_limit, we shouldn't wind up getting back here. 7279 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit; 7280 KMP_WARNING(CantLoadBalUsing, "KMP_DYNAMIC_MODE=thread limit"); 7281 7282 // Make this call behave like the thread limit algorithm. 7283 retval = __kmp_avail_proc - __kmp_nth + 7284 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc); 7285 if (retval > set_nproc) { 7286 retval = set_nproc; 7287 } 7288 if (retval < KMP_MIN_NTH) { 7289 retval = KMP_MIN_NTH; 7290 } 7291 7292 KB_TRACE(20, ("__kmp_load_balance_nproc: thread limit exit. retval:%d\n", 7293 retval)); 7294 return retval; 7295 } 7296 7297 // There is a slight delay in the load balance algorithm in detecting new 7298 // running procs. The real system load at this instant should be at least as 7299 // large as the #active omp thread that are available to add to the team. 7300 if (system_active < team_curr_active) { 7301 system_active = team_curr_active; 7302 } 7303 retval = __kmp_avail_proc - system_active + team_curr_active; 7304 if (retval > set_nproc) { 7305 retval = set_nproc; 7306 } 7307 if (retval < KMP_MIN_NTH) { 7308 retval = KMP_MIN_NTH; 7309 } 7310 7311 KB_TRACE(20, ("__kmp_load_balance_nproc: exit. retval:%d\n", retval)); 7312 return retval; 7313 } // __kmp_load_balance_nproc() 7314 7315 #endif /* USE_LOAD_BALANCE */ 7316 7317 /* ------------------------------------------------------------------------ */ 7318 7319 /* NOTE: this is called with the __kmp_init_lock held */ 7320 void __kmp_cleanup(void) { 7321 int f; 7322 7323 KA_TRACE(10, ("__kmp_cleanup: enter\n")); 7324 7325 if (TCR_4(__kmp_init_parallel)) { 7326 #if KMP_HANDLE_SIGNALS 7327 __kmp_remove_signals(); 7328 #endif 7329 TCW_4(__kmp_init_parallel, FALSE); 7330 } 7331 7332 if (TCR_4(__kmp_init_middle)) { 7333 #if KMP_AFFINITY_SUPPORTED 7334 __kmp_affinity_uninitialize(); 7335 #endif /* KMP_AFFINITY_SUPPORTED */ 7336 __kmp_cleanup_hierarchy(); 7337 TCW_4(__kmp_init_middle, FALSE); 7338 } 7339 7340 KA_TRACE(10, ("__kmp_cleanup: go serial cleanup\n")); 7341 7342 if (__kmp_init_serial) { 7343 __kmp_runtime_destroy(); 7344 __kmp_init_serial = FALSE; 7345 } 7346 7347 __kmp_cleanup_threadprivate_caches(); 7348 7349 for (f = 0; f < __kmp_threads_capacity; f++) { 7350 if (__kmp_root[f] != NULL) { 7351 __kmp_free(__kmp_root[f]); 7352 __kmp_root[f] = NULL; 7353 } 7354 } 7355 __kmp_free(__kmp_threads); 7356 // __kmp_threads and __kmp_root were allocated at once, as single block, so 7357 // there is no need in freeing __kmp_root. 7358 __kmp_threads = NULL; 7359 __kmp_root = NULL; 7360 __kmp_threads_capacity = 0; 7361 7362 #if KMP_USE_DYNAMIC_LOCK 7363 __kmp_cleanup_indirect_user_locks(); 7364 #else 7365 __kmp_cleanup_user_locks(); 7366 #endif 7367 7368 #if KMP_AFFINITY_SUPPORTED 7369 KMP_INTERNAL_FREE(CCAST(char *, __kmp_cpuinfo_file)); 7370 __kmp_cpuinfo_file = NULL; 7371 #endif /* KMP_AFFINITY_SUPPORTED */ 7372 7373 #if KMP_USE_ADAPTIVE_LOCKS 7374 #if KMP_DEBUG_ADAPTIVE_LOCKS 7375 __kmp_print_speculative_stats(); 7376 #endif 7377 #endif 7378 KMP_INTERNAL_FREE(__kmp_nested_nth.nth); 7379 __kmp_nested_nth.nth = NULL; 7380 __kmp_nested_nth.size = 0; 7381 __kmp_nested_nth.used = 0; 7382 KMP_INTERNAL_FREE(__kmp_nested_proc_bind.bind_types); 7383 __kmp_nested_proc_bind.bind_types = NULL; 7384 __kmp_nested_proc_bind.size = 0; 7385 __kmp_nested_proc_bind.used = 0; 7386 7387 __kmp_i18n_catclose(); 7388 7389 #if KMP_USE_HIER_SCHED 7390 __kmp_hier_scheds.deallocate(); 7391 #endif 7392 7393 #if KMP_STATS_ENABLED 7394 __kmp_stats_fini(); 7395 #endif 7396 7397 KA_TRACE(10, ("__kmp_cleanup: exit\n")); 7398 } 7399 7400 /* ------------------------------------------------------------------------ */ 7401 7402 int __kmp_ignore_mppbeg(void) { 7403 char *env; 7404 7405 if ((env = getenv("KMP_IGNORE_MPPBEG")) != NULL) { 7406 if (__kmp_str_match_false(env)) 7407 return FALSE; 7408 } 7409 // By default __kmpc_begin() is no-op. 7410 return TRUE; 7411 } 7412 7413 int __kmp_ignore_mppend(void) { 7414 char *env; 7415 7416 if ((env = getenv("KMP_IGNORE_MPPEND")) != NULL) { 7417 if (__kmp_str_match_false(env)) 7418 return FALSE; 7419 } 7420 // By default __kmpc_end() is no-op. 7421 return TRUE; 7422 } 7423 7424 void __kmp_internal_begin(void) { 7425 int gtid; 7426 kmp_root_t *root; 7427 7428 /* this is a very important step as it will register new sibling threads 7429 and assign these new uber threads a new gtid */ 7430 gtid = __kmp_entry_gtid(); 7431 root = __kmp_threads[gtid]->th.th_root; 7432 KMP_ASSERT(KMP_UBER_GTID(gtid)); 7433 7434 if (root->r.r_begin) 7435 return; 7436 __kmp_acquire_lock(&root->r.r_begin_lock, gtid); 7437 if (root->r.r_begin) { 7438 __kmp_release_lock(&root->r.r_begin_lock, gtid); 7439 return; 7440 } 7441 7442 root->r.r_begin = TRUE; 7443 7444 __kmp_release_lock(&root->r.r_begin_lock, gtid); 7445 } 7446 7447 /* ------------------------------------------------------------------------ */ 7448 7449 void __kmp_user_set_library(enum library_type arg) { 7450 int gtid; 7451 kmp_root_t *root; 7452 kmp_info_t *thread; 7453 7454 /* first, make sure we are initialized so we can get our gtid */ 7455 7456 gtid = __kmp_entry_gtid(); 7457 thread = __kmp_threads[gtid]; 7458 7459 root = thread->th.th_root; 7460 7461 KA_TRACE(20, ("__kmp_user_set_library: enter T#%d, arg: %d, %d\n", gtid, arg, 7462 library_serial)); 7463 if (root->r.r_in_parallel) { /* Must be called in serial section of top-level 7464 thread */ 7465 KMP_WARNING(SetLibraryIncorrectCall); 7466 return; 7467 } 7468 7469 switch (arg) { 7470 case library_serial: 7471 thread->th.th_set_nproc = 0; 7472 set__nproc(thread, 1); 7473 break; 7474 case library_turnaround: 7475 thread->th.th_set_nproc = 0; 7476 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth 7477 : __kmp_dflt_team_nth_ub); 7478 break; 7479 case library_throughput: 7480 thread->th.th_set_nproc = 0; 7481 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth 7482 : __kmp_dflt_team_nth_ub); 7483 break; 7484 default: 7485 KMP_FATAL(UnknownLibraryType, arg); 7486 } 7487 7488 __kmp_aux_set_library(arg); 7489 } 7490 7491 void __kmp_aux_set_stacksize(size_t arg) { 7492 if (!__kmp_init_serial) 7493 __kmp_serial_initialize(); 7494 7495 #if KMP_OS_DARWIN 7496 if (arg & (0x1000 - 1)) { 7497 arg &= ~(0x1000 - 1); 7498 if (arg + 0x1000) /* check for overflow if we round up */ 7499 arg += 0x1000; 7500 } 7501 #endif 7502 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 7503 7504 /* only change the default stacksize before the first parallel region */ 7505 if (!TCR_4(__kmp_init_parallel)) { 7506 size_t value = arg; /* argument is in bytes */ 7507 7508 if (value < __kmp_sys_min_stksize) 7509 value = __kmp_sys_min_stksize; 7510 else if (value > KMP_MAX_STKSIZE) 7511 value = KMP_MAX_STKSIZE; 7512 7513 __kmp_stksize = value; 7514 7515 __kmp_env_stksize = TRUE; /* was KMP_STACKSIZE specified? */ 7516 } 7517 7518 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 7519 } 7520 7521 /* set the behaviour of the runtime library */ 7522 /* TODO this can cause some odd behaviour with sibling parallelism... */ 7523 void __kmp_aux_set_library(enum library_type arg) { 7524 __kmp_library = arg; 7525 7526 switch (__kmp_library) { 7527 case library_serial: { 7528 KMP_INFORM(LibraryIsSerial); 7529 (void)__kmp_change_library(TRUE); 7530 } break; 7531 case library_turnaround: 7532 (void)__kmp_change_library(TRUE); 7533 break; 7534 case library_throughput: 7535 (void)__kmp_change_library(FALSE); 7536 break; 7537 default: 7538 KMP_FATAL(UnknownLibraryType, arg); 7539 } 7540 } 7541 7542 /* ------------------------------------------------------------------------ */ 7543 7544 void __kmp_aux_set_blocktime(int arg, kmp_info_t *thread, int tid) { 7545 int blocktime = arg; /* argument is in milliseconds */ 7546 #if KMP_USE_MONITOR 7547 int bt_intervals; 7548 #endif 7549 int bt_set; 7550 7551 __kmp_save_internal_controls(thread); 7552 7553 /* Normalize and set blocktime for the teams */ 7554 if (blocktime < KMP_MIN_BLOCKTIME) 7555 blocktime = KMP_MIN_BLOCKTIME; 7556 else if (blocktime > KMP_MAX_BLOCKTIME) 7557 blocktime = KMP_MAX_BLOCKTIME; 7558 7559 set__blocktime_team(thread->th.th_team, tid, blocktime); 7560 set__blocktime_team(thread->th.th_serial_team, 0, blocktime); 7561 7562 #if KMP_USE_MONITOR 7563 /* Calculate and set blocktime intervals for the teams */ 7564 bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME(blocktime, __kmp_monitor_wakeups); 7565 7566 set__bt_intervals_team(thread->th.th_team, tid, bt_intervals); 7567 set__bt_intervals_team(thread->th.th_serial_team, 0, bt_intervals); 7568 #endif 7569 7570 /* Set whether blocktime has been set to "TRUE" */ 7571 bt_set = TRUE; 7572 7573 set__bt_set_team(thread->th.th_team, tid, bt_set); 7574 set__bt_set_team(thread->th.th_serial_team, 0, bt_set); 7575 #if KMP_USE_MONITOR 7576 KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, " 7577 "bt_intervals=%d, monitor_updates=%d\n", 7578 __kmp_gtid_from_tid(tid, thread->th.th_team), 7579 thread->th.th_team->t.t_id, tid, blocktime, bt_intervals, 7580 __kmp_monitor_wakeups)); 7581 #else 7582 KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n", 7583 __kmp_gtid_from_tid(tid, thread->th.th_team), 7584 thread->th.th_team->t.t_id, tid, blocktime)); 7585 #endif 7586 } 7587 7588 void __kmp_aux_set_defaults(char const *str, int len) { 7589 if (!__kmp_init_serial) { 7590 __kmp_serial_initialize(); 7591 } 7592 __kmp_env_initialize(str); 7593 7594 if (__kmp_settings 7595 #if OMP_40_ENABLED 7596 || __kmp_display_env || __kmp_display_env_verbose 7597 #endif // OMP_40_ENABLED 7598 ) { 7599 __kmp_env_print(); 7600 } 7601 } // __kmp_aux_set_defaults 7602 7603 /* ------------------------------------------------------------------------ */ 7604 /* internal fast reduction routines */ 7605 7606 PACKED_REDUCTION_METHOD_T 7607 __kmp_determine_reduction_method( 7608 ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, 7609 void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data), 7610 kmp_critical_name *lck) { 7611 7612 // Default reduction method: critical construct ( lck != NULL, like in current 7613 // PAROPT ) 7614 // If ( reduce_data!=NULL && reduce_func!=NULL ): the tree-reduction method 7615 // can be selected by RTL 7616 // If loc->flags contains KMP_IDENT_ATOMIC_REDUCE, the atomic reduce method 7617 // can be selected by RTL 7618 // Finally, it's up to OpenMP RTL to make a decision on which method to select 7619 // among generated by PAROPT. 7620 7621 PACKED_REDUCTION_METHOD_T retval; 7622 7623 int team_size; 7624 7625 KMP_DEBUG_ASSERT(loc); // it would be nice to test ( loc != 0 ) 7626 KMP_DEBUG_ASSERT(lck); // it would be nice to test ( lck != 0 ) 7627 7628 #define FAST_REDUCTION_ATOMIC_METHOD_GENERATED \ 7629 ((loc->flags & (KMP_IDENT_ATOMIC_REDUCE)) == (KMP_IDENT_ATOMIC_REDUCE)) 7630 #define FAST_REDUCTION_TREE_METHOD_GENERATED ((reduce_data) && (reduce_func)) 7631 7632 retval = critical_reduce_block; 7633 7634 // another choice of getting a team size (with 1 dynamic deference) is slower 7635 team_size = __kmp_get_team_num_threads(global_tid); 7636 if (team_size == 1) { 7637 7638 retval = empty_reduce_block; 7639 7640 } else { 7641 7642 int atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED; 7643 7644 #if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS64 7645 7646 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_WINDOWS || \ 7647 KMP_OS_DARWIN 7648 7649 int teamsize_cutoff = 4; 7650 7651 #if KMP_MIC_SUPPORTED 7652 if (__kmp_mic_type != non_mic) { 7653 teamsize_cutoff = 8; 7654 } 7655 #endif 7656 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED; 7657 if (tree_available) { 7658 if (team_size <= teamsize_cutoff) { 7659 if (atomic_available) { 7660 retval = atomic_reduce_block; 7661 } 7662 } else { 7663 retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER; 7664 } 7665 } else if (atomic_available) { 7666 retval = atomic_reduce_block; 7667 } 7668 #else 7669 #error "Unknown or unsupported OS" 7670 #endif // KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_WINDOWS || 7671 // KMP_OS_DARWIN 7672 7673 #elif KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS 7674 7675 #if KMP_OS_LINUX || KMP_OS_WINDOWS 7676 7677 // basic tuning 7678 7679 if (atomic_available) { 7680 if (num_vars <= 2) { // && ( team_size <= 8 ) due to false-sharing ??? 7681 retval = atomic_reduce_block; 7682 } 7683 } // otherwise: use critical section 7684 7685 #elif KMP_OS_DARWIN 7686 7687 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED; 7688 if (atomic_available && (num_vars <= 3)) { 7689 retval = atomic_reduce_block; 7690 } else if (tree_available) { 7691 if ((reduce_size > (9 * sizeof(kmp_real64))) && 7692 (reduce_size < (2000 * sizeof(kmp_real64)))) { 7693 retval = TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER; 7694 } 7695 } // otherwise: use critical section 7696 7697 #else 7698 #error "Unknown or unsupported OS" 7699 #endif 7700 7701 #else 7702 #error "Unknown or unsupported architecture" 7703 #endif 7704 } 7705 7706 // KMP_FORCE_REDUCTION 7707 7708 // If the team is serialized (team_size == 1), ignore the forced reduction 7709 // method and stay with the unsynchronized method (empty_reduce_block) 7710 if (__kmp_force_reduction_method != reduction_method_not_defined && 7711 team_size != 1) { 7712 7713 PACKED_REDUCTION_METHOD_T forced_retval = critical_reduce_block; 7714 7715 int atomic_available, tree_available; 7716 7717 switch ((forced_retval = __kmp_force_reduction_method)) { 7718 case critical_reduce_block: 7719 KMP_ASSERT(lck); // lck should be != 0 7720 break; 7721 7722 case atomic_reduce_block: 7723 atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED; 7724 if (!atomic_available) { 7725 KMP_WARNING(RedMethodNotSupported, "atomic"); 7726 forced_retval = critical_reduce_block; 7727 } 7728 break; 7729 7730 case tree_reduce_block: 7731 tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED; 7732 if (!tree_available) { 7733 KMP_WARNING(RedMethodNotSupported, "tree"); 7734 forced_retval = critical_reduce_block; 7735 } else { 7736 #if KMP_FAST_REDUCTION_BARRIER 7737 forced_retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER; 7738 #endif 7739 } 7740 break; 7741 7742 default: 7743 KMP_ASSERT(0); // "unsupported method specified" 7744 } 7745 7746 retval = forced_retval; 7747 } 7748 7749 KA_TRACE(10, ("reduction method selected=%08x\n", retval)); 7750 7751 #undef FAST_REDUCTION_TREE_METHOD_GENERATED 7752 #undef FAST_REDUCTION_ATOMIC_METHOD_GENERATED 7753 7754 return (retval); 7755 } 7756 7757 // this function is for testing set/get/determine reduce method 7758 kmp_int32 __kmp_get_reduce_method(void) { 7759 return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8); 7760 } 7761