1 /* 2 * z_Linux_util.cpp -- platform specific routines. 3 */ 4 5 //===----------------------------------------------------------------------===// 6 // 7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 8 // See https://llvm.org/LICENSE.txt for license information. 9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "kmp.h" 14 #include "kmp_affinity.h" 15 #include "kmp_i18n.h" 16 #include "kmp_io.h" 17 #include "kmp_itt.h" 18 #include "kmp_lock.h" 19 #include "kmp_stats.h" 20 #include "kmp_str.h" 21 #include "kmp_wait_release.h" 22 #include "kmp_wrapper_getpid.h" 23 24 #if !KMP_OS_DRAGONFLY && !KMP_OS_FREEBSD && !KMP_OS_NETBSD && !KMP_OS_OPENBSD 25 #include <alloca.h> 26 #endif 27 #include <math.h> // HUGE_VAL. 28 #include <semaphore.h> 29 #include <sys/resource.h> 30 #include <sys/syscall.h> 31 #include <sys/time.h> 32 #include <sys/times.h> 33 #include <unistd.h> 34 35 #if KMP_OS_LINUX 36 #include <sys/sysinfo.h> 37 #if KMP_USE_FUTEX 38 // We should really include <futex.h>, but that causes compatibility problems on 39 // different Linux* OS distributions that either require that you include (or 40 // break when you try to include) <pci/types.h>. Since all we need is the two 41 // macros below (which are part of the kernel ABI, so can't change) we just 42 // define the constants here and don't include <futex.h> 43 #ifndef FUTEX_WAIT 44 #define FUTEX_WAIT 0 45 #endif 46 #ifndef FUTEX_WAKE 47 #define FUTEX_WAKE 1 48 #endif 49 #endif 50 #elif KMP_OS_DARWIN 51 #include <mach/mach.h> 52 #include <sys/sysctl.h> 53 #elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD 54 #include <sys/types.h> 55 #include <sys/sysctl.h> 56 #include <sys/user.h> 57 #include <pthread_np.h> 58 #elif KMP_OS_NETBSD || KMP_OS_OPENBSD 59 #include <sys/types.h> 60 #include <sys/sysctl.h> 61 #endif 62 63 #include <ctype.h> 64 #include <dirent.h> 65 #include <fcntl.h> 66 67 #include "tsan_annotations.h" 68 69 struct kmp_sys_timer { 70 struct timespec start; 71 }; 72 73 // Convert timespec to nanoseconds. 74 #define TS2NS(timespec) \ 75 (((timespec).tv_sec * (long int)1e9) + (timespec).tv_nsec) 76 77 static struct kmp_sys_timer __kmp_sys_timer_data; 78 79 #if KMP_HANDLE_SIGNALS 80 typedef void (*sig_func_t)(int); 81 STATIC_EFI2_WORKAROUND struct sigaction __kmp_sighldrs[NSIG]; 82 static sigset_t __kmp_sigset; 83 #endif 84 85 static int __kmp_init_runtime = FALSE; 86 87 static int __kmp_fork_count = 0; 88 89 static pthread_condattr_t __kmp_suspend_cond_attr; 90 static pthread_mutexattr_t __kmp_suspend_mutex_attr; 91 92 static kmp_cond_align_t __kmp_wait_cv; 93 static kmp_mutex_align_t __kmp_wait_mx; 94 95 kmp_uint64 __kmp_ticks_per_msec = 1000000; 96 97 #ifdef DEBUG_SUSPEND 98 static void __kmp_print_cond(char *buffer, kmp_cond_align_t *cond) { 99 KMP_SNPRINTF(buffer, 128, "(cond (lock (%ld, %d)), (descr (%p)))", 100 cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock, 101 cond->c_cond.__c_waiting); 102 } 103 #endif 104 105 #if ((KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED) 106 107 /* Affinity support */ 108 109 void __kmp_affinity_bind_thread(int which) { 110 KMP_ASSERT2(KMP_AFFINITY_CAPABLE(), 111 "Illegal set affinity operation when not capable"); 112 113 kmp_affin_mask_t *mask; 114 KMP_CPU_ALLOC_ON_STACK(mask); 115 KMP_CPU_ZERO(mask); 116 KMP_CPU_SET(which, mask); 117 __kmp_set_system_affinity(mask, TRUE); 118 KMP_CPU_FREE_FROM_STACK(mask); 119 } 120 121 /* Determine if we can access affinity functionality on this version of 122 * Linux* OS by checking __NR_sched_{get,set}affinity system calls, and set 123 * __kmp_affin_mask_size to the appropriate value (0 means not capable). */ 124 void __kmp_affinity_determine_capable(const char *env_var) { 125 // Check and see if the OS supports thread affinity. 126 127 #if KMP_OS_LINUX 128 #define KMP_CPU_SET_SIZE_LIMIT (1024 * 1024) 129 #define KMP_CPU_SET_TRY_SIZE CACHE_LINE 130 #elif KMP_OS_FREEBSD 131 #define KMP_CPU_SET_SIZE_LIMIT (sizeof(cpuset_t)) 132 #endif 133 134 #if KMP_OS_LINUX 135 long gCode; 136 unsigned char *buf; 137 buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT); 138 139 // If the syscall returns a suggestion for the size, 140 // then we don't have to search for an appropriate size. 141 gCode = syscall(__NR_sched_getaffinity, 0, KMP_CPU_SET_TRY_SIZE, buf); 142 KA_TRACE(30, ("__kmp_affinity_determine_capable: " 143 "initial getaffinity call returned %ld errno = %d\n", 144 gCode, errno)); 145 146 if (gCode < 0 && errno != EINVAL) { 147 // System call not supported 148 if (__kmp_affinity_verbose || 149 (__kmp_affinity_warnings && (__kmp_affinity_type != affinity_none) && 150 (__kmp_affinity_type != affinity_default) && 151 (__kmp_affinity_type != affinity_disabled))) { 152 int error = errno; 153 kmp_msg_t err_code = KMP_ERR(error); 154 __kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var), 155 err_code, __kmp_msg_null); 156 if (__kmp_generate_warnings == kmp_warnings_off) { 157 __kmp_str_free(&err_code.str); 158 } 159 } 160 KMP_AFFINITY_DISABLE(); 161 KMP_INTERNAL_FREE(buf); 162 return; 163 } else if (gCode > 0) { 164 // The optimal situation: the OS returns the size of the buffer it expects. 165 KMP_AFFINITY_ENABLE(gCode); 166 KA_TRACE(10, ("__kmp_affinity_determine_capable: " 167 "affinity supported (mask size %d)\n", 168 (int)__kmp_affin_mask_size)); 169 KMP_INTERNAL_FREE(buf); 170 return; 171 } 172 173 // Call the getaffinity system call repeatedly with increasing set sizes 174 // until we succeed, or reach an upper bound on the search. 175 KA_TRACE(30, ("__kmp_affinity_determine_capable: " 176 "searching for proper set size\n")); 177 int size; 178 for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) { 179 gCode = syscall(__NR_sched_getaffinity, 0, size, buf); 180 KA_TRACE(30, ("__kmp_affinity_determine_capable: " 181 "getaffinity for mask size %ld returned %ld errno = %d\n", 182 size, gCode, errno)); 183 184 if (gCode < 0) { 185 if (errno == ENOSYS) { 186 // We shouldn't get here 187 KA_TRACE(30, ("__kmp_affinity_determine_capable: " 188 "inconsistent OS call behavior: errno == ENOSYS for mask " 189 "size %d\n", 190 size)); 191 if (__kmp_affinity_verbose || 192 (__kmp_affinity_warnings && 193 (__kmp_affinity_type != affinity_none) && 194 (__kmp_affinity_type != affinity_default) && 195 (__kmp_affinity_type != affinity_disabled))) { 196 int error = errno; 197 kmp_msg_t err_code = KMP_ERR(error); 198 __kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var), 199 err_code, __kmp_msg_null); 200 if (__kmp_generate_warnings == kmp_warnings_off) { 201 __kmp_str_free(&err_code.str); 202 } 203 } 204 KMP_AFFINITY_DISABLE(); 205 KMP_INTERNAL_FREE(buf); 206 return; 207 } 208 continue; 209 } 210 211 KMP_AFFINITY_ENABLE(gCode); 212 KA_TRACE(10, ("__kmp_affinity_determine_capable: " 213 "affinity supported (mask size %d)\n", 214 (int)__kmp_affin_mask_size)); 215 KMP_INTERNAL_FREE(buf); 216 return; 217 } 218 #elif KMP_OS_FREEBSD 219 long gCode; 220 unsigned char *buf; 221 buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT); 222 gCode = pthread_getaffinity_np(pthread_self(), KMP_CPU_SET_SIZE_LIMIT, 223 reinterpret_cast<cpuset_t *>(buf)); 224 KA_TRACE(30, ("__kmp_affinity_determine_capable: " 225 "initial getaffinity call returned %d errno = %d\n", 226 gCode, errno)); 227 if (gCode == 0) { 228 KMP_AFFINITY_ENABLE(KMP_CPU_SET_SIZE_LIMIT); 229 KA_TRACE(10, ("__kmp_affinity_determine_capable: " 230 "affinity supported (mask size %d)\n", 231 (int)__kmp_affin_mask_size)); 232 KMP_INTERNAL_FREE(buf); 233 return; 234 } 235 #endif 236 KMP_INTERNAL_FREE(buf); 237 238 // Affinity is not supported 239 KMP_AFFINITY_DISABLE(); 240 KA_TRACE(10, ("__kmp_affinity_determine_capable: " 241 "cannot determine mask size - affinity not supported\n")); 242 if (__kmp_affinity_verbose || 243 (__kmp_affinity_warnings && (__kmp_affinity_type != affinity_none) && 244 (__kmp_affinity_type != affinity_default) && 245 (__kmp_affinity_type != affinity_disabled))) { 246 KMP_WARNING(AffCantGetMaskSize, env_var); 247 } 248 } 249 250 #endif // KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 251 252 #if KMP_USE_FUTEX 253 254 int __kmp_futex_determine_capable() { 255 int loc = 0; 256 long rc = syscall(__NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0); 257 int retval = (rc == 0) || (errno != ENOSYS); 258 259 KA_TRACE(10, 260 ("__kmp_futex_determine_capable: rc = %d errno = %d\n", rc, errno)); 261 KA_TRACE(10, ("__kmp_futex_determine_capable: futex syscall%s supported\n", 262 retval ? "" : " not")); 263 264 return retval; 265 } 266 267 #endif // KMP_USE_FUTEX 268 269 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (!KMP_ASM_INTRINS) 270 /* Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to 271 use compare_and_store for these routines */ 272 273 kmp_int8 __kmp_test_then_or8(volatile kmp_int8 *p, kmp_int8 d) { 274 kmp_int8 old_value, new_value; 275 276 old_value = TCR_1(*p); 277 new_value = old_value | d; 278 279 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) { 280 KMP_CPU_PAUSE(); 281 old_value = TCR_1(*p); 282 new_value = old_value | d; 283 } 284 return old_value; 285 } 286 287 kmp_int8 __kmp_test_then_and8(volatile kmp_int8 *p, kmp_int8 d) { 288 kmp_int8 old_value, new_value; 289 290 old_value = TCR_1(*p); 291 new_value = old_value & d; 292 293 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) { 294 KMP_CPU_PAUSE(); 295 old_value = TCR_1(*p); 296 new_value = old_value & d; 297 } 298 return old_value; 299 } 300 301 kmp_uint32 __kmp_test_then_or32(volatile kmp_uint32 *p, kmp_uint32 d) { 302 kmp_uint32 old_value, new_value; 303 304 old_value = TCR_4(*p); 305 new_value = old_value | d; 306 307 while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) { 308 KMP_CPU_PAUSE(); 309 old_value = TCR_4(*p); 310 new_value = old_value | d; 311 } 312 return old_value; 313 } 314 315 kmp_uint32 __kmp_test_then_and32(volatile kmp_uint32 *p, kmp_uint32 d) { 316 kmp_uint32 old_value, new_value; 317 318 old_value = TCR_4(*p); 319 new_value = old_value & d; 320 321 while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) { 322 KMP_CPU_PAUSE(); 323 old_value = TCR_4(*p); 324 new_value = old_value & d; 325 } 326 return old_value; 327 } 328 329 #if KMP_ARCH_X86 330 kmp_int8 __kmp_test_then_add8(volatile kmp_int8 *p, kmp_int8 d) { 331 kmp_int8 old_value, new_value; 332 333 old_value = TCR_1(*p); 334 new_value = old_value + d; 335 336 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) { 337 KMP_CPU_PAUSE(); 338 old_value = TCR_1(*p); 339 new_value = old_value + d; 340 } 341 return old_value; 342 } 343 344 kmp_int64 __kmp_test_then_add64(volatile kmp_int64 *p, kmp_int64 d) { 345 kmp_int64 old_value, new_value; 346 347 old_value = TCR_8(*p); 348 new_value = old_value + d; 349 350 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) { 351 KMP_CPU_PAUSE(); 352 old_value = TCR_8(*p); 353 new_value = old_value + d; 354 } 355 return old_value; 356 } 357 #endif /* KMP_ARCH_X86 */ 358 359 kmp_uint64 __kmp_test_then_or64(volatile kmp_uint64 *p, kmp_uint64 d) { 360 kmp_uint64 old_value, new_value; 361 362 old_value = TCR_8(*p); 363 new_value = old_value | d; 364 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) { 365 KMP_CPU_PAUSE(); 366 old_value = TCR_8(*p); 367 new_value = old_value | d; 368 } 369 return old_value; 370 } 371 372 kmp_uint64 __kmp_test_then_and64(volatile kmp_uint64 *p, kmp_uint64 d) { 373 kmp_uint64 old_value, new_value; 374 375 old_value = TCR_8(*p); 376 new_value = old_value & d; 377 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) { 378 KMP_CPU_PAUSE(); 379 old_value = TCR_8(*p); 380 new_value = old_value & d; 381 } 382 return old_value; 383 } 384 385 #endif /* (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS) */ 386 387 void __kmp_terminate_thread(int gtid) { 388 int status; 389 kmp_info_t *th = __kmp_threads[gtid]; 390 391 if (!th) 392 return; 393 394 #ifdef KMP_CANCEL_THREADS 395 KA_TRACE(10, ("__kmp_terminate_thread: kill (%d)\n", gtid)); 396 status = pthread_cancel(th->th.th_info.ds.ds_thread); 397 if (status != 0 && status != ESRCH) { 398 __kmp_fatal(KMP_MSG(CantTerminateWorkerThread), KMP_ERR(status), 399 __kmp_msg_null); 400 } 401 #endif 402 KMP_YIELD(TRUE); 403 } // 404 405 /* Set thread stack info according to values returned by pthread_getattr_np(). 406 If values are unreasonable, assume call failed and use incremental stack 407 refinement method instead. Returns TRUE if the stack parameters could be 408 determined exactly, FALSE if incremental refinement is necessary. */ 409 static kmp_int32 __kmp_set_stack_info(int gtid, kmp_info_t *th) { 410 int stack_data; 411 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ 412 KMP_OS_HURD 413 pthread_attr_t attr; 414 int status; 415 size_t size = 0; 416 void *addr = 0; 417 418 /* Always do incremental stack refinement for ubermaster threads since the 419 initial thread stack range can be reduced by sibling thread creation so 420 pthread_attr_getstack may cause thread gtid aliasing */ 421 if (!KMP_UBER_GTID(gtid)) { 422 423 /* Fetch the real thread attributes */ 424 status = pthread_attr_init(&attr); 425 KMP_CHECK_SYSFAIL("pthread_attr_init", status); 426 #if KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD 427 status = pthread_attr_get_np(pthread_self(), &attr); 428 KMP_CHECK_SYSFAIL("pthread_attr_get_np", status); 429 #else 430 status = pthread_getattr_np(pthread_self(), &attr); 431 KMP_CHECK_SYSFAIL("pthread_getattr_np", status); 432 #endif 433 status = pthread_attr_getstack(&attr, &addr, &size); 434 KMP_CHECK_SYSFAIL("pthread_attr_getstack", status); 435 KA_TRACE(60, 436 ("__kmp_set_stack_info: T#%d pthread_attr_getstack returned size:" 437 " %lu, low addr: %p\n", 438 gtid, size, addr)); 439 status = pthread_attr_destroy(&attr); 440 KMP_CHECK_SYSFAIL("pthread_attr_destroy", status); 441 } 442 443 if (size != 0 && addr != 0) { // was stack parameter determination successful? 444 /* Store the correct base and size */ 445 TCW_PTR(th->th.th_info.ds.ds_stackbase, (((char *)addr) + size)); 446 TCW_PTR(th->th.th_info.ds.ds_stacksize, size); 447 TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE); 448 return TRUE; 449 } 450 #endif /* KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD \ 451 || KMP_OS_HURD */ 452 /* Use incremental refinement starting from initial conservative estimate */ 453 TCW_PTR(th->th.th_info.ds.ds_stacksize, 0); 454 TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data); 455 TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE); 456 return FALSE; 457 } 458 459 static void *__kmp_launch_worker(void *thr) { 460 int status, old_type, old_state; 461 #ifdef KMP_BLOCK_SIGNALS 462 sigset_t new_set, old_set; 463 #endif /* KMP_BLOCK_SIGNALS */ 464 void *exit_val; 465 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ 466 KMP_OS_OPENBSD || KMP_OS_HURD 467 void *volatile padding = 0; 468 #endif 469 int gtid; 470 471 gtid = ((kmp_info_t *)thr)->th.th_info.ds.ds_gtid; 472 __kmp_gtid_set_specific(gtid); 473 #ifdef KMP_TDATA_GTID 474 __kmp_gtid = gtid; 475 #endif 476 #if KMP_STATS_ENABLED 477 // set thread local index to point to thread-specific stats 478 __kmp_stats_thread_ptr = ((kmp_info_t *)thr)->th.th_stats; 479 __kmp_stats_thread_ptr->startLife(); 480 KMP_SET_THREAD_STATE(IDLE); 481 KMP_INIT_PARTITIONED_TIMERS(OMP_idle); 482 #endif 483 484 #if USE_ITT_BUILD 485 __kmp_itt_thread_name(gtid); 486 #endif /* USE_ITT_BUILD */ 487 488 #if KMP_AFFINITY_SUPPORTED 489 __kmp_affinity_set_init_mask(gtid, FALSE); 490 #endif 491 492 #ifdef KMP_CANCEL_THREADS 493 status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type); 494 KMP_CHECK_SYSFAIL("pthread_setcanceltype", status); 495 // josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads? 496 status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state); 497 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status); 498 #endif 499 500 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 501 // Set FP control regs to be a copy of the parallel initialization thread's. 502 __kmp_clear_x87_fpu_status_word(); 503 __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word); 504 __kmp_load_mxcsr(&__kmp_init_mxcsr); 505 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 506 507 #ifdef KMP_BLOCK_SIGNALS 508 status = sigfillset(&new_set); 509 KMP_CHECK_SYSFAIL_ERRNO("sigfillset", status); 510 status = pthread_sigmask(SIG_BLOCK, &new_set, &old_set); 511 KMP_CHECK_SYSFAIL("pthread_sigmask", status); 512 #endif /* KMP_BLOCK_SIGNALS */ 513 514 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ 515 KMP_OS_OPENBSD 516 if (__kmp_stkoffset > 0 && gtid > 0) { 517 padding = KMP_ALLOCA(gtid * __kmp_stkoffset); 518 } 519 #endif 520 521 KMP_MB(); 522 __kmp_set_stack_info(gtid, (kmp_info_t *)thr); 523 524 __kmp_check_stack_overlap((kmp_info_t *)thr); 525 526 exit_val = __kmp_launch_thread((kmp_info_t *)thr); 527 528 #ifdef KMP_BLOCK_SIGNALS 529 status = pthread_sigmask(SIG_SETMASK, &old_set, NULL); 530 KMP_CHECK_SYSFAIL("pthread_sigmask", status); 531 #endif /* KMP_BLOCK_SIGNALS */ 532 533 return exit_val; 534 } 535 536 #if KMP_USE_MONITOR 537 /* The monitor thread controls all of the threads in the complex */ 538 539 static void *__kmp_launch_monitor(void *thr) { 540 int status, old_type, old_state; 541 #ifdef KMP_BLOCK_SIGNALS 542 sigset_t new_set; 543 #endif /* KMP_BLOCK_SIGNALS */ 544 struct timespec interval; 545 546 KMP_MB(); /* Flush all pending memory write invalidates. */ 547 548 KA_TRACE(10, ("__kmp_launch_monitor: #1 launched\n")); 549 550 /* register us as the monitor thread */ 551 __kmp_gtid_set_specific(KMP_GTID_MONITOR); 552 #ifdef KMP_TDATA_GTID 553 __kmp_gtid = KMP_GTID_MONITOR; 554 #endif 555 556 KMP_MB(); 557 558 #if USE_ITT_BUILD 559 // Instruct Intel(R) Threading Tools to ignore monitor thread. 560 __kmp_itt_thread_ignore(); 561 #endif /* USE_ITT_BUILD */ 562 563 __kmp_set_stack_info(((kmp_info_t *)thr)->th.th_info.ds.ds_gtid, 564 (kmp_info_t *)thr); 565 566 __kmp_check_stack_overlap((kmp_info_t *)thr); 567 568 #ifdef KMP_CANCEL_THREADS 569 status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type); 570 KMP_CHECK_SYSFAIL("pthread_setcanceltype", status); 571 // josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads? 572 status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state); 573 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status); 574 #endif 575 576 #if KMP_REAL_TIME_FIX 577 // This is a potential fix which allows application with real-time scheduling 578 // policy work. However, decision about the fix is not made yet, so it is 579 // disabled by default. 580 { // Are program started with real-time scheduling policy? 581 int sched = sched_getscheduler(0); 582 if (sched == SCHED_FIFO || sched == SCHED_RR) { 583 // Yes, we are a part of real-time application. Try to increase the 584 // priority of the monitor. 585 struct sched_param param; 586 int max_priority = sched_get_priority_max(sched); 587 int rc; 588 KMP_WARNING(RealTimeSchedNotSupported); 589 sched_getparam(0, ¶m); 590 if (param.sched_priority < max_priority) { 591 param.sched_priority += 1; 592 rc = sched_setscheduler(0, sched, ¶m); 593 if (rc != 0) { 594 int error = errno; 595 kmp_msg_t err_code = KMP_ERR(error); 596 __kmp_msg(kmp_ms_warning, KMP_MSG(CantChangeMonitorPriority), 597 err_code, KMP_MSG(MonitorWillStarve), __kmp_msg_null); 598 if (__kmp_generate_warnings == kmp_warnings_off) { 599 __kmp_str_free(&err_code.str); 600 } 601 } 602 } else { 603 // We cannot abort here, because number of CPUs may be enough for all 604 // the threads, including the monitor thread, so application could 605 // potentially work... 606 __kmp_msg(kmp_ms_warning, KMP_MSG(RunningAtMaxPriority), 607 KMP_MSG(MonitorWillStarve), KMP_HNT(RunningAtMaxPriority), 608 __kmp_msg_null); 609 } 610 } 611 // AC: free thread that waits for monitor started 612 TCW_4(__kmp_global.g.g_time.dt.t_value, 0); 613 } 614 #endif // KMP_REAL_TIME_FIX 615 616 KMP_MB(); /* Flush all pending memory write invalidates. */ 617 618 if (__kmp_monitor_wakeups == 1) { 619 interval.tv_sec = 1; 620 interval.tv_nsec = 0; 621 } else { 622 interval.tv_sec = 0; 623 interval.tv_nsec = (KMP_NSEC_PER_SEC / __kmp_monitor_wakeups); 624 } 625 626 KA_TRACE(10, ("__kmp_launch_monitor: #2 monitor\n")); 627 628 while (!TCR_4(__kmp_global.g.g_done)) { 629 struct timespec now; 630 struct timeval tval; 631 632 /* This thread monitors the state of the system */ 633 634 KA_TRACE(15, ("__kmp_launch_monitor: update\n")); 635 636 status = gettimeofday(&tval, NULL); 637 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status); 638 TIMEVAL_TO_TIMESPEC(&tval, &now); 639 640 now.tv_sec += interval.tv_sec; 641 now.tv_nsec += interval.tv_nsec; 642 643 if (now.tv_nsec >= KMP_NSEC_PER_SEC) { 644 now.tv_sec += 1; 645 now.tv_nsec -= KMP_NSEC_PER_SEC; 646 } 647 648 status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex); 649 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 650 // AC: the monitor should not fall asleep if g_done has been set 651 if (!TCR_4(__kmp_global.g.g_done)) { // check once more under mutex 652 status = pthread_cond_timedwait(&__kmp_wait_cv.c_cond, 653 &__kmp_wait_mx.m_mutex, &now); 654 if (status != 0) { 655 if (status != ETIMEDOUT && status != EINTR) { 656 KMP_SYSFAIL("pthread_cond_timedwait", status); 657 } 658 } 659 } 660 status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex); 661 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 662 663 TCW_4(__kmp_global.g.g_time.dt.t_value, 664 TCR_4(__kmp_global.g.g_time.dt.t_value) + 1); 665 666 KMP_MB(); /* Flush all pending memory write invalidates. */ 667 } 668 669 KA_TRACE(10, ("__kmp_launch_monitor: #3 cleanup\n")); 670 671 #ifdef KMP_BLOCK_SIGNALS 672 status = sigfillset(&new_set); 673 KMP_CHECK_SYSFAIL_ERRNO("sigfillset", status); 674 status = pthread_sigmask(SIG_UNBLOCK, &new_set, NULL); 675 KMP_CHECK_SYSFAIL("pthread_sigmask", status); 676 #endif /* KMP_BLOCK_SIGNALS */ 677 678 KA_TRACE(10, ("__kmp_launch_monitor: #4 finished\n")); 679 680 if (__kmp_global.g.g_abort != 0) { 681 /* now we need to terminate the worker threads */ 682 /* the value of t_abort is the signal we caught */ 683 684 int gtid; 685 686 KA_TRACE(10, ("__kmp_launch_monitor: #5 terminate sig=%d\n", 687 __kmp_global.g.g_abort)); 688 689 /* terminate the OpenMP worker threads */ 690 /* TODO this is not valid for sibling threads!! 691 * the uber master might not be 0 anymore.. */ 692 for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid) 693 __kmp_terminate_thread(gtid); 694 695 __kmp_cleanup(); 696 697 KA_TRACE(10, ("__kmp_launch_monitor: #6 raise sig=%d\n", 698 __kmp_global.g.g_abort)); 699 700 if (__kmp_global.g.g_abort > 0) 701 raise(__kmp_global.g.g_abort); 702 } 703 704 KA_TRACE(10, ("__kmp_launch_monitor: #7 exit\n")); 705 706 return thr; 707 } 708 #endif // KMP_USE_MONITOR 709 710 void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size) { 711 pthread_t handle; 712 pthread_attr_t thread_attr; 713 int status; 714 715 th->th.th_info.ds.ds_gtid = gtid; 716 717 #if KMP_STATS_ENABLED 718 // sets up worker thread stats 719 __kmp_acquire_tas_lock(&__kmp_stats_lock, gtid); 720 721 // th->th.th_stats is used to transfer thread-specific stats-pointer to 722 // __kmp_launch_worker. So when thread is created (goes into 723 // __kmp_launch_worker) it will set its thread local pointer to 724 // th->th.th_stats 725 if (!KMP_UBER_GTID(gtid)) { 726 th->th.th_stats = __kmp_stats_list->push_back(gtid); 727 } else { 728 // For root threads, __kmp_stats_thread_ptr is set in __kmp_register_root(), 729 // so set the th->th.th_stats field to it. 730 th->th.th_stats = __kmp_stats_thread_ptr; 731 } 732 __kmp_release_tas_lock(&__kmp_stats_lock, gtid); 733 734 #endif // KMP_STATS_ENABLED 735 736 if (KMP_UBER_GTID(gtid)) { 737 KA_TRACE(10, ("__kmp_create_worker: uber thread (%d)\n", gtid)); 738 th->th.th_info.ds.ds_thread = pthread_self(); 739 __kmp_set_stack_info(gtid, th); 740 __kmp_check_stack_overlap(th); 741 return; 742 } 743 744 KA_TRACE(10, ("__kmp_create_worker: try to create thread (%d)\n", gtid)); 745 746 KMP_MB(); /* Flush all pending memory write invalidates. */ 747 748 #ifdef KMP_THREAD_ATTR 749 status = pthread_attr_init(&thread_attr); 750 if (status != 0) { 751 __kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null); 752 } 753 status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE); 754 if (status != 0) { 755 __kmp_fatal(KMP_MSG(CantSetWorkerState), KMP_ERR(status), __kmp_msg_null); 756 } 757 758 /* Set stack size for this thread now. 759 The multiple of 2 is there because on some machines, requesting an unusual 760 stacksize causes the thread to have an offset before the dummy alloca() 761 takes place to create the offset. Since we want the user to have a 762 sufficient stacksize AND support a stack offset, we alloca() twice the 763 offset so that the upcoming alloca() does not eliminate any premade offset, 764 and also gives the user the stack space they requested for all threads */ 765 stack_size += gtid * __kmp_stkoffset * 2; 766 767 #if defined(__ANDROID__) && __ANDROID_API__ < 19 768 // Round the stack size to a multiple of the page size. Older versions of 769 // Android (until KitKat) would fail pthread_attr_setstacksize with EINVAL 770 // if the stack size was not a multiple of the page size. 771 stack_size = (stack_size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); 772 #endif 773 774 KA_TRACE(10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, " 775 "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n", 776 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size)); 777 778 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 779 status = pthread_attr_setstacksize(&thread_attr, stack_size); 780 #ifdef KMP_BACKUP_STKSIZE 781 if (status != 0) { 782 if (!__kmp_env_stksize) { 783 stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset; 784 __kmp_stksize = KMP_BACKUP_STKSIZE; 785 KA_TRACE(10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, " 786 "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu " 787 "bytes\n", 788 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size)); 789 status = pthread_attr_setstacksize(&thread_attr, stack_size); 790 } 791 } 792 #endif /* KMP_BACKUP_STKSIZE */ 793 if (status != 0) { 794 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status), 795 KMP_HNT(ChangeWorkerStackSize), __kmp_msg_null); 796 } 797 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */ 798 799 #endif /* KMP_THREAD_ATTR */ 800 801 status = 802 pthread_create(&handle, &thread_attr, __kmp_launch_worker, (void *)th); 803 if (status != 0 || !handle) { // ??? Why do we check handle?? 804 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 805 if (status == EINVAL) { 806 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status), 807 KMP_HNT(IncreaseWorkerStackSize), __kmp_msg_null); 808 } 809 if (status == ENOMEM) { 810 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status), 811 KMP_HNT(DecreaseWorkerStackSize), __kmp_msg_null); 812 } 813 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */ 814 if (status == EAGAIN) { 815 __kmp_fatal(KMP_MSG(NoResourcesForWorkerThread), KMP_ERR(status), 816 KMP_HNT(Decrease_NUM_THREADS), __kmp_msg_null); 817 } 818 KMP_SYSFAIL("pthread_create", status); 819 } 820 821 th->th.th_info.ds.ds_thread = handle; 822 823 #ifdef KMP_THREAD_ATTR 824 status = pthread_attr_destroy(&thread_attr); 825 if (status) { 826 kmp_msg_t err_code = KMP_ERR(status); 827 __kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code, 828 __kmp_msg_null); 829 if (__kmp_generate_warnings == kmp_warnings_off) { 830 __kmp_str_free(&err_code.str); 831 } 832 } 833 #endif /* KMP_THREAD_ATTR */ 834 835 KMP_MB(); /* Flush all pending memory write invalidates. */ 836 837 KA_TRACE(10, ("__kmp_create_worker: done creating thread (%d)\n", gtid)); 838 839 } // __kmp_create_worker 840 841 #if KMP_USE_MONITOR 842 void __kmp_create_monitor(kmp_info_t *th) { 843 pthread_t handle; 844 pthread_attr_t thread_attr; 845 size_t size; 846 int status; 847 int auto_adj_size = FALSE; 848 849 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) { 850 // We don't need monitor thread in case of MAX_BLOCKTIME 851 KA_TRACE(10, ("__kmp_create_monitor: skipping monitor thread because of " 852 "MAX blocktime\n")); 853 th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op 854 th->th.th_info.ds.ds_gtid = 0; 855 return; 856 } 857 KA_TRACE(10, ("__kmp_create_monitor: try to create monitor\n")); 858 859 KMP_MB(); /* Flush all pending memory write invalidates. */ 860 861 th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR; 862 th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR; 863 #if KMP_REAL_TIME_FIX 864 TCW_4(__kmp_global.g.g_time.dt.t_value, 865 -1); // Will use it for synchronization a bit later. 866 #else 867 TCW_4(__kmp_global.g.g_time.dt.t_value, 0); 868 #endif // KMP_REAL_TIME_FIX 869 870 #ifdef KMP_THREAD_ATTR 871 if (__kmp_monitor_stksize == 0) { 872 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE; 873 auto_adj_size = TRUE; 874 } 875 status = pthread_attr_init(&thread_attr); 876 if (status != 0) { 877 __kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null); 878 } 879 status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE); 880 if (status != 0) { 881 __kmp_fatal(KMP_MSG(CantSetMonitorState), KMP_ERR(status), __kmp_msg_null); 882 } 883 884 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 885 status = pthread_attr_getstacksize(&thread_attr, &size); 886 KMP_CHECK_SYSFAIL("pthread_attr_getstacksize", status); 887 #else 888 size = __kmp_sys_min_stksize; 889 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */ 890 #endif /* KMP_THREAD_ATTR */ 891 892 if (__kmp_monitor_stksize == 0) { 893 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE; 894 } 895 if (__kmp_monitor_stksize < __kmp_sys_min_stksize) { 896 __kmp_monitor_stksize = __kmp_sys_min_stksize; 897 } 898 899 KA_TRACE(10, ("__kmp_create_monitor: default stacksize = %lu bytes," 900 "requested stacksize = %lu bytes\n", 901 size, __kmp_monitor_stksize)); 902 903 retry: 904 905 /* Set stack size for this thread now. */ 906 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 907 KA_TRACE(10, ("__kmp_create_monitor: setting stacksize = %lu bytes,", 908 __kmp_monitor_stksize)); 909 status = pthread_attr_setstacksize(&thread_attr, __kmp_monitor_stksize); 910 if (status != 0) { 911 if (auto_adj_size) { 912 __kmp_monitor_stksize *= 2; 913 goto retry; 914 } 915 kmp_msg_t err_code = KMP_ERR(status); 916 __kmp_msg(kmp_ms_warning, // should this be fatal? BB 917 KMP_MSG(CantSetMonitorStackSize, (long int)__kmp_monitor_stksize), 918 err_code, KMP_HNT(ChangeMonitorStackSize), __kmp_msg_null); 919 if (__kmp_generate_warnings == kmp_warnings_off) { 920 __kmp_str_free(&err_code.str); 921 } 922 } 923 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */ 924 925 status = 926 pthread_create(&handle, &thread_attr, __kmp_launch_monitor, (void *)th); 927 928 if (status != 0) { 929 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 930 if (status == EINVAL) { 931 if (auto_adj_size && (__kmp_monitor_stksize < (size_t)0x40000000)) { 932 __kmp_monitor_stksize *= 2; 933 goto retry; 934 } 935 __kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize), 936 KMP_ERR(status), KMP_HNT(IncreaseMonitorStackSize), 937 __kmp_msg_null); 938 } 939 if (status == ENOMEM) { 940 __kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize), 941 KMP_ERR(status), KMP_HNT(DecreaseMonitorStackSize), 942 __kmp_msg_null); 943 } 944 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */ 945 if (status == EAGAIN) { 946 __kmp_fatal(KMP_MSG(NoResourcesForMonitorThread), KMP_ERR(status), 947 KMP_HNT(DecreaseNumberOfThreadsInUse), __kmp_msg_null); 948 } 949 KMP_SYSFAIL("pthread_create", status); 950 } 951 952 th->th.th_info.ds.ds_thread = handle; 953 954 #if KMP_REAL_TIME_FIX 955 // Wait for the monitor thread is really started and set its *priority*. 956 KMP_DEBUG_ASSERT(sizeof(kmp_uint32) == 957 sizeof(__kmp_global.g.g_time.dt.t_value)); 958 __kmp_wait_4((kmp_uint32 volatile *)&__kmp_global.g.g_time.dt.t_value, -1, 959 &__kmp_neq_4, NULL); 960 #endif // KMP_REAL_TIME_FIX 961 962 #ifdef KMP_THREAD_ATTR 963 status = pthread_attr_destroy(&thread_attr); 964 if (status != 0) { 965 kmp_msg_t err_code = KMP_ERR(status); 966 __kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code, 967 __kmp_msg_null); 968 if (__kmp_generate_warnings == kmp_warnings_off) { 969 __kmp_str_free(&err_code.str); 970 } 971 } 972 #endif 973 974 KMP_MB(); /* Flush all pending memory write invalidates. */ 975 976 KA_TRACE(10, ("__kmp_create_monitor: monitor created %#.8lx\n", 977 th->th.th_info.ds.ds_thread)); 978 979 } // __kmp_create_monitor 980 #endif // KMP_USE_MONITOR 981 982 void __kmp_exit_thread(int exit_status) { 983 pthread_exit((void *)(intptr_t)exit_status); 984 } // __kmp_exit_thread 985 986 #if KMP_USE_MONITOR 987 void __kmp_resume_monitor(); 988 989 void __kmp_reap_monitor(kmp_info_t *th) { 990 int status; 991 void *exit_val; 992 993 KA_TRACE(10, ("__kmp_reap_monitor: try to reap monitor thread with handle" 994 " %#.8lx\n", 995 th->th.th_info.ds.ds_thread)); 996 997 // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR. 998 // If both tid and gtid are 0, it means the monitor did not ever start. 999 // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down. 1000 KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid); 1001 if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) { 1002 KA_TRACE(10, ("__kmp_reap_monitor: monitor did not start, returning\n")); 1003 return; 1004 } 1005 1006 KMP_MB(); /* Flush all pending memory write invalidates. */ 1007 1008 /* First, check to see whether the monitor thread exists to wake it up. This 1009 is to avoid performance problem when the monitor sleeps during 1010 blocktime-size interval */ 1011 1012 status = pthread_kill(th->th.th_info.ds.ds_thread, 0); 1013 if (status != ESRCH) { 1014 __kmp_resume_monitor(); // Wake up the monitor thread 1015 } 1016 KA_TRACE(10, ("__kmp_reap_monitor: try to join with monitor\n")); 1017 status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val); 1018 if (exit_val != th) { 1019 __kmp_fatal(KMP_MSG(ReapMonitorError), KMP_ERR(status), __kmp_msg_null); 1020 } 1021 1022 th->th.th_info.ds.ds_tid = KMP_GTID_DNE; 1023 th->th.th_info.ds.ds_gtid = KMP_GTID_DNE; 1024 1025 KA_TRACE(10, ("__kmp_reap_monitor: done reaping monitor thread with handle" 1026 " %#.8lx\n", 1027 th->th.th_info.ds.ds_thread)); 1028 1029 KMP_MB(); /* Flush all pending memory write invalidates. */ 1030 } 1031 #endif // KMP_USE_MONITOR 1032 1033 void __kmp_reap_worker(kmp_info_t *th) { 1034 int status; 1035 void *exit_val; 1036 1037 KMP_MB(); /* Flush all pending memory write invalidates. */ 1038 1039 KA_TRACE( 1040 10, ("__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid)); 1041 1042 status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val); 1043 #ifdef KMP_DEBUG 1044 /* Don't expose these to the user until we understand when they trigger */ 1045 if (status != 0) { 1046 __kmp_fatal(KMP_MSG(ReapWorkerError), KMP_ERR(status), __kmp_msg_null); 1047 } 1048 if (exit_val != th) { 1049 KA_TRACE(10, ("__kmp_reap_worker: worker T#%d did not reap properly, " 1050 "exit_val = %p\n", 1051 th->th.th_info.ds.ds_gtid, exit_val)); 1052 } 1053 #endif /* KMP_DEBUG */ 1054 1055 KA_TRACE(10, ("__kmp_reap_worker: done reaping T#%d\n", 1056 th->th.th_info.ds.ds_gtid)); 1057 1058 KMP_MB(); /* Flush all pending memory write invalidates. */ 1059 } 1060 1061 #if KMP_HANDLE_SIGNALS 1062 1063 static void __kmp_null_handler(int signo) { 1064 // Do nothing, for doing SIG_IGN-type actions. 1065 } // __kmp_null_handler 1066 1067 static void __kmp_team_handler(int signo) { 1068 if (__kmp_global.g.g_abort == 0) { 1069 /* Stage 1 signal handler, let's shut down all of the threads */ 1070 #ifdef KMP_DEBUG 1071 __kmp_debug_printf("__kmp_team_handler: caught signal = %d\n", signo); 1072 #endif 1073 switch (signo) { 1074 case SIGHUP: 1075 case SIGINT: 1076 case SIGQUIT: 1077 case SIGILL: 1078 case SIGABRT: 1079 case SIGFPE: 1080 case SIGBUS: 1081 case SIGSEGV: 1082 #ifdef SIGSYS 1083 case SIGSYS: 1084 #endif 1085 case SIGTERM: 1086 if (__kmp_debug_buf) { 1087 __kmp_dump_debug_buffer(); 1088 } 1089 __kmp_unregister_library(); // cleanup shared memory 1090 KMP_MB(); // Flush all pending memory write invalidates. 1091 TCW_4(__kmp_global.g.g_abort, signo); 1092 KMP_MB(); // Flush all pending memory write invalidates. 1093 TCW_4(__kmp_global.g.g_done, TRUE); 1094 KMP_MB(); // Flush all pending memory write invalidates. 1095 break; 1096 default: 1097 #ifdef KMP_DEBUG 1098 __kmp_debug_printf("__kmp_team_handler: unknown signal type"); 1099 #endif 1100 break; 1101 } 1102 } 1103 } // __kmp_team_handler 1104 1105 static void __kmp_sigaction(int signum, const struct sigaction *act, 1106 struct sigaction *oldact) { 1107 int rc = sigaction(signum, act, oldact); 1108 KMP_CHECK_SYSFAIL_ERRNO("sigaction", rc); 1109 } 1110 1111 static void __kmp_install_one_handler(int sig, sig_func_t handler_func, 1112 int parallel_init) { 1113 KMP_MB(); // Flush all pending memory write invalidates. 1114 KB_TRACE(60, 1115 ("__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init)); 1116 if (parallel_init) { 1117 struct sigaction new_action; 1118 struct sigaction old_action; 1119 new_action.sa_handler = handler_func; 1120 new_action.sa_flags = 0; 1121 sigfillset(&new_action.sa_mask); 1122 __kmp_sigaction(sig, &new_action, &old_action); 1123 if (old_action.sa_handler == __kmp_sighldrs[sig].sa_handler) { 1124 sigaddset(&__kmp_sigset, sig); 1125 } else { 1126 // Restore/keep user's handler if one previously installed. 1127 __kmp_sigaction(sig, &old_action, NULL); 1128 } 1129 } else { 1130 // Save initial/system signal handlers to see if user handlers installed. 1131 __kmp_sigaction(sig, NULL, &__kmp_sighldrs[sig]); 1132 } 1133 KMP_MB(); // Flush all pending memory write invalidates. 1134 } // __kmp_install_one_handler 1135 1136 static void __kmp_remove_one_handler(int sig) { 1137 KB_TRACE(60, ("__kmp_remove_one_handler( %d )\n", sig)); 1138 if (sigismember(&__kmp_sigset, sig)) { 1139 struct sigaction old; 1140 KMP_MB(); // Flush all pending memory write invalidates. 1141 __kmp_sigaction(sig, &__kmp_sighldrs[sig], &old); 1142 if ((old.sa_handler != __kmp_team_handler) && 1143 (old.sa_handler != __kmp_null_handler)) { 1144 // Restore the users signal handler. 1145 KB_TRACE(10, ("__kmp_remove_one_handler: oops, not our handler, " 1146 "restoring: sig=%d\n", 1147 sig)); 1148 __kmp_sigaction(sig, &old, NULL); 1149 } 1150 sigdelset(&__kmp_sigset, sig); 1151 KMP_MB(); // Flush all pending memory write invalidates. 1152 } 1153 } // __kmp_remove_one_handler 1154 1155 void __kmp_install_signals(int parallel_init) { 1156 KB_TRACE(10, ("__kmp_install_signals( %d )\n", parallel_init)); 1157 if (__kmp_handle_signals || !parallel_init) { 1158 // If ! parallel_init, we do not install handlers, just save original 1159 // handlers. Let us do it even __handle_signals is 0. 1160 sigemptyset(&__kmp_sigset); 1161 __kmp_install_one_handler(SIGHUP, __kmp_team_handler, parallel_init); 1162 __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init); 1163 __kmp_install_one_handler(SIGQUIT, __kmp_team_handler, parallel_init); 1164 __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init); 1165 __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init); 1166 __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init); 1167 __kmp_install_one_handler(SIGBUS, __kmp_team_handler, parallel_init); 1168 __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init); 1169 #ifdef SIGSYS 1170 __kmp_install_one_handler(SIGSYS, __kmp_team_handler, parallel_init); 1171 #endif // SIGSYS 1172 __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init); 1173 #ifdef SIGPIPE 1174 __kmp_install_one_handler(SIGPIPE, __kmp_team_handler, parallel_init); 1175 #endif // SIGPIPE 1176 } 1177 } // __kmp_install_signals 1178 1179 void __kmp_remove_signals(void) { 1180 int sig; 1181 KB_TRACE(10, ("__kmp_remove_signals()\n")); 1182 for (sig = 1; sig < NSIG; ++sig) { 1183 __kmp_remove_one_handler(sig); 1184 } 1185 } // __kmp_remove_signals 1186 1187 #endif // KMP_HANDLE_SIGNALS 1188 1189 void __kmp_enable(int new_state) { 1190 #ifdef KMP_CANCEL_THREADS 1191 int status, old_state; 1192 status = pthread_setcancelstate(new_state, &old_state); 1193 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status); 1194 KMP_DEBUG_ASSERT(old_state == PTHREAD_CANCEL_DISABLE); 1195 #endif 1196 } 1197 1198 void __kmp_disable(int *old_state) { 1199 #ifdef KMP_CANCEL_THREADS 1200 int status; 1201 status = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, old_state); 1202 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status); 1203 #endif 1204 } 1205 1206 static void __kmp_atfork_prepare(void) { 1207 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 1208 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 1209 } 1210 1211 static void __kmp_atfork_parent(void) { 1212 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 1213 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 1214 } 1215 1216 /* Reset the library so execution in the child starts "all over again" with 1217 clean data structures in initial states. Don't worry about freeing memory 1218 allocated by parent, just abandon it to be safe. */ 1219 static void __kmp_atfork_child(void) { 1220 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 1221 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 1222 /* TODO make sure this is done right for nested/sibling */ 1223 // ATT: Memory leaks are here? TODO: Check it and fix. 1224 /* KMP_ASSERT( 0 ); */ 1225 1226 ++__kmp_fork_count; 1227 1228 #if KMP_AFFINITY_SUPPORTED 1229 #if KMP_OS_LINUX || KMP_OS_FREEBSD 1230 // reset the affinity in the child to the initial thread 1231 // affinity in the parent 1232 kmp_set_thread_affinity_mask_initial(); 1233 #endif 1234 // Set default not to bind threads tightly in the child (we’re expecting 1235 // over-subscription after the fork and this can improve things for 1236 // scripting languages that use OpenMP inside process-parallel code). 1237 __kmp_affinity_type = affinity_none; 1238 if (__kmp_nested_proc_bind.bind_types != NULL) { 1239 __kmp_nested_proc_bind.bind_types[0] = proc_bind_false; 1240 } 1241 __kmp_affinity_masks = NULL; 1242 __kmp_affinity_num_masks = 0; 1243 #endif // KMP_AFFINITY_SUPPORTED 1244 1245 #if KMP_USE_MONITOR 1246 __kmp_init_monitor = 0; 1247 #endif 1248 __kmp_init_parallel = FALSE; 1249 __kmp_init_middle = FALSE; 1250 __kmp_init_serial = FALSE; 1251 TCW_4(__kmp_init_gtid, FALSE); 1252 __kmp_init_common = FALSE; 1253 1254 TCW_4(__kmp_init_user_locks, FALSE); 1255 #if !KMP_USE_DYNAMIC_LOCK 1256 __kmp_user_lock_table.used = 1; 1257 __kmp_user_lock_table.allocated = 0; 1258 __kmp_user_lock_table.table = NULL; 1259 __kmp_lock_blocks = NULL; 1260 #endif 1261 1262 __kmp_all_nth = 0; 1263 TCW_4(__kmp_nth, 0); 1264 1265 __kmp_thread_pool = NULL; 1266 __kmp_thread_pool_insert_pt = NULL; 1267 __kmp_team_pool = NULL; 1268 1269 /* Must actually zero all the *cache arguments passed to __kmpc_threadprivate 1270 here so threadprivate doesn't use stale data */ 1271 KA_TRACE(10, ("__kmp_atfork_child: checking cache address list %p\n", 1272 __kmp_threadpriv_cache_list)); 1273 1274 while (__kmp_threadpriv_cache_list != NULL) { 1275 1276 if (*__kmp_threadpriv_cache_list->addr != NULL) { 1277 KC_TRACE(50, ("__kmp_atfork_child: zeroing cache at address %p\n", 1278 &(*__kmp_threadpriv_cache_list->addr))); 1279 1280 *__kmp_threadpriv_cache_list->addr = NULL; 1281 } 1282 __kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list->next; 1283 } 1284 1285 __kmp_init_runtime = FALSE; 1286 1287 /* reset statically initialized locks */ 1288 __kmp_init_bootstrap_lock(&__kmp_initz_lock); 1289 __kmp_init_bootstrap_lock(&__kmp_stdio_lock); 1290 __kmp_init_bootstrap_lock(&__kmp_console_lock); 1291 __kmp_init_bootstrap_lock(&__kmp_task_team_lock); 1292 1293 #if USE_ITT_BUILD 1294 __kmp_itt_reset(); // reset ITT's global state 1295 #endif /* USE_ITT_BUILD */ 1296 1297 __kmp_serial_initialize(); 1298 1299 /* This is necessary to make sure no stale data is left around */ 1300 /* AC: customers complain that we use unsafe routines in the atfork 1301 handler. Mathworks: dlsym() is unsafe. We call dlsym and dlopen 1302 in dynamic_link when check the presence of shared tbbmalloc library. 1303 Suggestion is to make the library initialization lazier, similar 1304 to what done for __kmpc_begin(). */ 1305 // TODO: synchronize all static initializations with regular library 1306 // startup; look at kmp_global.cpp and etc. 1307 //__kmp_internal_begin (); 1308 } 1309 1310 void __kmp_register_atfork(void) { 1311 if (__kmp_need_register_atfork) { 1312 int status = pthread_atfork(__kmp_atfork_prepare, __kmp_atfork_parent, 1313 __kmp_atfork_child); 1314 KMP_CHECK_SYSFAIL("pthread_atfork", status); 1315 __kmp_need_register_atfork = FALSE; 1316 } 1317 } 1318 1319 void __kmp_suspend_initialize(void) { 1320 int status; 1321 status = pthread_mutexattr_init(&__kmp_suspend_mutex_attr); 1322 KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status); 1323 status = pthread_condattr_init(&__kmp_suspend_cond_attr); 1324 KMP_CHECK_SYSFAIL("pthread_condattr_init", status); 1325 } 1326 1327 void __kmp_suspend_initialize_thread(kmp_info_t *th) { 1328 ANNOTATE_HAPPENS_AFTER(&th->th.th_suspend_init_count); 1329 int old_value = KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count); 1330 int new_value = __kmp_fork_count + 1; 1331 // Return if already initialized 1332 if (old_value == new_value) 1333 return; 1334 // Wait, then return if being initialized 1335 if (old_value == -1 || !__kmp_atomic_compare_store( 1336 &th->th.th_suspend_init_count, old_value, -1)) { 1337 while (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) != new_value) { 1338 KMP_CPU_PAUSE(); 1339 } 1340 } else { 1341 // Claim to be the initializer and do initializations 1342 int status; 1343 status = pthread_cond_init(&th->th.th_suspend_cv.c_cond, 1344 &__kmp_suspend_cond_attr); 1345 KMP_CHECK_SYSFAIL("pthread_cond_init", status); 1346 status = pthread_mutex_init(&th->th.th_suspend_mx.m_mutex, 1347 &__kmp_suspend_mutex_attr); 1348 KMP_CHECK_SYSFAIL("pthread_mutex_init", status); 1349 KMP_ATOMIC_ST_REL(&th->th.th_suspend_init_count, new_value); 1350 ANNOTATE_HAPPENS_BEFORE(&th->th.th_suspend_init_count); 1351 } 1352 } 1353 1354 void __kmp_suspend_uninitialize_thread(kmp_info_t *th) { 1355 if (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) > __kmp_fork_count) { 1356 /* this means we have initialize the suspension pthread objects for this 1357 thread in this instance of the process */ 1358 int status; 1359 1360 status = pthread_cond_destroy(&th->th.th_suspend_cv.c_cond); 1361 if (status != 0 && status != EBUSY) { 1362 KMP_SYSFAIL("pthread_cond_destroy", status); 1363 } 1364 status = pthread_mutex_destroy(&th->th.th_suspend_mx.m_mutex); 1365 if (status != 0 && status != EBUSY) { 1366 KMP_SYSFAIL("pthread_mutex_destroy", status); 1367 } 1368 --th->th.th_suspend_init_count; 1369 KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count) == 1370 __kmp_fork_count); 1371 } 1372 } 1373 1374 // return true if lock obtained, false otherwise 1375 int __kmp_try_suspend_mx(kmp_info_t *th) { 1376 return (pthread_mutex_trylock(&th->th.th_suspend_mx.m_mutex) == 0); 1377 } 1378 1379 void __kmp_lock_suspend_mx(kmp_info_t *th) { 1380 int status = pthread_mutex_lock(&th->th.th_suspend_mx.m_mutex); 1381 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 1382 } 1383 1384 void __kmp_unlock_suspend_mx(kmp_info_t *th) { 1385 int status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex); 1386 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 1387 } 1388 1389 /* This routine puts the calling thread to sleep after setting the 1390 sleep bit for the indicated flag variable to true. */ 1391 template <class C> 1392 static inline void __kmp_suspend_template(int th_gtid, C *flag) { 1393 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_suspend); 1394 kmp_info_t *th = __kmp_threads[th_gtid]; 1395 int status; 1396 typename C::flag_t old_spin; 1397 1398 KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid, 1399 flag->get())); 1400 1401 __kmp_suspend_initialize_thread(th); 1402 1403 __kmp_lock_suspend_mx(th); 1404 1405 KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n", 1406 th_gtid, flag->get())); 1407 1408 /* TODO: shouldn't this use release semantics to ensure that 1409 __kmp_suspend_initialize_thread gets called first? */ 1410 old_spin = flag->set_sleeping(); 1411 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME && 1412 __kmp_pause_status != kmp_soft_paused) { 1413 flag->unset_sleeping(); 1414 __kmp_unlock_suspend_mx(th); 1415 return; 1416 } 1417 KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x," 1418 " was %x\n", 1419 th_gtid, flag->get(), flag->load(), old_spin)); 1420 1421 if (flag->done_check_val(old_spin)) { 1422 old_spin = flag->unset_sleeping(); 1423 KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit " 1424 "for spin(%p)\n", 1425 th_gtid, flag->get())); 1426 } else { 1427 /* Encapsulate in a loop as the documentation states that this may 1428 "with low probability" return when the condition variable has 1429 not been signaled or broadcast */ 1430 int deactivated = FALSE; 1431 TCW_PTR(th->th.th_sleep_loc, (void *)flag); 1432 1433 while (flag->is_sleeping()) { 1434 #ifdef DEBUG_SUSPEND 1435 char buffer[128]; 1436 __kmp_suspend_count++; 1437 __kmp_print_cond(buffer, &th->th.th_suspend_cv); 1438 __kmp_printf("__kmp_suspend_template: suspending T#%d: %s\n", th_gtid, 1439 buffer); 1440 #endif 1441 // Mark the thread as no longer active (only in the first iteration of the 1442 // loop). 1443 if (!deactivated) { 1444 th->th.th_active = FALSE; 1445 if (th->th.th_active_in_pool) { 1446 th->th.th_active_in_pool = FALSE; 1447 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth); 1448 KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0); 1449 } 1450 deactivated = TRUE; 1451 } 1452 1453 #if USE_SUSPEND_TIMEOUT 1454 struct timespec now; 1455 struct timeval tval; 1456 int msecs; 1457 1458 status = gettimeofday(&tval, NULL); 1459 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status); 1460 TIMEVAL_TO_TIMESPEC(&tval, &now); 1461 1462 msecs = (4 * __kmp_dflt_blocktime) + 200; 1463 now.tv_sec += msecs / 1000; 1464 now.tv_nsec += (msecs % 1000) * 1000; 1465 1466 KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform " 1467 "pthread_cond_timedwait\n", 1468 th_gtid)); 1469 status = pthread_cond_timedwait(&th->th.th_suspend_cv.c_cond, 1470 &th->th.th_suspend_mx.m_mutex, &now); 1471 #else 1472 KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform" 1473 " pthread_cond_wait\n", 1474 th_gtid)); 1475 status = pthread_cond_wait(&th->th.th_suspend_cv.c_cond, 1476 &th->th.th_suspend_mx.m_mutex); 1477 #endif // USE_SUSPEND_TIMEOUT 1478 1479 if ((status != 0) && (status != EINTR) && (status != ETIMEDOUT)) { 1480 KMP_SYSFAIL("pthread_cond_wait", status); 1481 } 1482 #ifdef KMP_DEBUG 1483 if (status == ETIMEDOUT) { 1484 if (flag->is_sleeping()) { 1485 KF_TRACE(100, 1486 ("__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid)); 1487 } else { 1488 KF_TRACE(2, ("__kmp_suspend_template: T#%d timeout wakeup, sleep bit " 1489 "not set!\n", 1490 th_gtid)); 1491 } 1492 } else if (flag->is_sleeping()) { 1493 KF_TRACE(100, 1494 ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid)); 1495 } 1496 #endif 1497 } // while 1498 1499 // Mark the thread as active again (if it was previous marked as inactive) 1500 if (deactivated) { 1501 th->th.th_active = TRUE; 1502 if (TCR_4(th->th.th_in_pool)) { 1503 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth); 1504 th->th.th_active_in_pool = TRUE; 1505 } 1506 } 1507 } 1508 #ifdef DEBUG_SUSPEND 1509 { 1510 char buffer[128]; 1511 __kmp_print_cond(buffer, &th->th.th_suspend_cv); 1512 __kmp_printf("__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid, 1513 buffer); 1514 } 1515 #endif 1516 1517 __kmp_unlock_suspend_mx(th); 1518 KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid)); 1519 } 1520 1521 template <bool C, bool S> 1522 void __kmp_suspend_32(int th_gtid, kmp_flag_32<C, S> *flag) { 1523 __kmp_suspend_template(th_gtid, flag); 1524 } 1525 template <bool C, bool S> 1526 void __kmp_suspend_64(int th_gtid, kmp_flag_64<C, S> *flag) { 1527 __kmp_suspend_template(th_gtid, flag); 1528 } 1529 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) { 1530 __kmp_suspend_template(th_gtid, flag); 1531 } 1532 1533 template void __kmp_suspend_32<false, false>(int, kmp_flag_32<false, false> *); 1534 template void __kmp_suspend_64<false, true>(int, kmp_flag_64<false, true> *); 1535 template void __kmp_suspend_64<true, false>(int, kmp_flag_64<true, false> *); 1536 1537 /* This routine signals the thread specified by target_gtid to wake up 1538 after setting the sleep bit indicated by the flag argument to FALSE. 1539 The target thread must already have called __kmp_suspend_template() */ 1540 template <class C> 1541 static inline void __kmp_resume_template(int target_gtid, C *flag) { 1542 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume); 1543 kmp_info_t *th = __kmp_threads[target_gtid]; 1544 int status; 1545 1546 #ifdef KMP_DEBUG 1547 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1; 1548 #endif 1549 1550 KF_TRACE(30, ("__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", 1551 gtid, target_gtid)); 1552 KMP_DEBUG_ASSERT(gtid != target_gtid); 1553 1554 __kmp_suspend_initialize_thread(th); 1555 1556 __kmp_lock_suspend_mx(th); 1557 1558 if (!flag) { // coming from __kmp_null_resume_wrapper 1559 flag = (C *)CCAST(void *, th->th.th_sleep_loc); 1560 } 1561 1562 // First, check if the flag is null or its type has changed. If so, someone 1563 // else woke it up. 1564 if (!flag || flag->get_type() != flag->get_ptr_type()) { // get_ptr_type 1565 // simply shows what flag was cast to 1566 KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already " 1567 "awake: flag(%p)\n", 1568 gtid, target_gtid, NULL)); 1569 __kmp_unlock_suspend_mx(th); 1570 return; 1571 } else { // if multiple threads are sleeping, flag should be internally 1572 // referring to a specific thread here 1573 typename C::flag_t old_spin = flag->unset_sleeping(); 1574 if (!flag->is_sleeping_val(old_spin)) { 1575 KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already " 1576 "awake: flag(%p): " 1577 "%u => %u\n", 1578 gtid, target_gtid, flag->get(), old_spin, flag->load())); 1579 __kmp_unlock_suspend_mx(th); 1580 return; 1581 } 1582 KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset " 1583 "sleep bit for flag's loc(%p): " 1584 "%u => %u\n", 1585 gtid, target_gtid, flag->get(), old_spin, flag->load())); 1586 } 1587 TCW_PTR(th->th.th_sleep_loc, NULL); 1588 1589 #ifdef DEBUG_SUSPEND 1590 { 1591 char buffer[128]; 1592 __kmp_print_cond(buffer, &th->th.th_suspend_cv); 1593 __kmp_printf("__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid, 1594 target_gtid, buffer); 1595 } 1596 #endif 1597 status = pthread_cond_signal(&th->th.th_suspend_cv.c_cond); 1598 KMP_CHECK_SYSFAIL("pthread_cond_signal", status); 1599 __kmp_unlock_suspend_mx(th); 1600 KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up" 1601 " for T#%d\n", 1602 gtid, target_gtid)); 1603 } 1604 1605 template <bool C, bool S> 1606 void __kmp_resume_32(int target_gtid, kmp_flag_32<C, S> *flag) { 1607 __kmp_resume_template(target_gtid, flag); 1608 } 1609 template <bool C, bool S> 1610 void __kmp_resume_64(int target_gtid, kmp_flag_64<C, S> *flag) { 1611 __kmp_resume_template(target_gtid, flag); 1612 } 1613 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) { 1614 __kmp_resume_template(target_gtid, flag); 1615 } 1616 1617 template void __kmp_resume_32<false, true>(int, kmp_flag_32<false, true> *); 1618 template void __kmp_resume_64<false, true>(int, kmp_flag_64<false, true> *); 1619 1620 #if KMP_USE_MONITOR 1621 void __kmp_resume_monitor() { 1622 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume); 1623 int status; 1624 #ifdef KMP_DEBUG 1625 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1; 1626 KF_TRACE(30, ("__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n", gtid, 1627 KMP_GTID_MONITOR)); 1628 KMP_DEBUG_ASSERT(gtid != KMP_GTID_MONITOR); 1629 #endif 1630 status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex); 1631 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 1632 #ifdef DEBUG_SUSPEND 1633 { 1634 char buffer[128]; 1635 __kmp_print_cond(buffer, &__kmp_wait_cv.c_cond); 1636 __kmp_printf("__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid, 1637 KMP_GTID_MONITOR, buffer); 1638 } 1639 #endif 1640 status = pthread_cond_signal(&__kmp_wait_cv.c_cond); 1641 KMP_CHECK_SYSFAIL("pthread_cond_signal", status); 1642 status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex); 1643 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 1644 KF_TRACE(30, ("__kmp_resume_monitor: T#%d exiting after signaling wake up" 1645 " for T#%d\n", 1646 gtid, KMP_GTID_MONITOR)); 1647 } 1648 #endif // KMP_USE_MONITOR 1649 1650 void __kmp_yield() { sched_yield(); } 1651 1652 void __kmp_gtid_set_specific(int gtid) { 1653 if (__kmp_init_gtid) { 1654 int status; 1655 status = pthread_setspecific(__kmp_gtid_threadprivate_key, 1656 (void *)(intptr_t)(gtid + 1)); 1657 KMP_CHECK_SYSFAIL("pthread_setspecific", status); 1658 } else { 1659 KA_TRACE(50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n")); 1660 } 1661 } 1662 1663 int __kmp_gtid_get_specific() { 1664 int gtid; 1665 if (!__kmp_init_gtid) { 1666 KA_TRACE(50, ("__kmp_gtid_get_specific: runtime shutdown, returning " 1667 "KMP_GTID_SHUTDOWN\n")); 1668 return KMP_GTID_SHUTDOWN; 1669 } 1670 gtid = (int)(size_t)pthread_getspecific(__kmp_gtid_threadprivate_key); 1671 if (gtid == 0) { 1672 gtid = KMP_GTID_DNE; 1673 } else { 1674 gtid--; 1675 } 1676 KA_TRACE(50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n", 1677 __kmp_gtid_threadprivate_key, gtid)); 1678 return gtid; 1679 } 1680 1681 double __kmp_read_cpu_time(void) { 1682 /*clock_t t;*/ 1683 struct tms buffer; 1684 1685 /*t =*/times(&buffer); 1686 1687 return (double)(buffer.tms_utime + buffer.tms_cutime) / 1688 (double)CLOCKS_PER_SEC; 1689 } 1690 1691 int __kmp_read_system_info(struct kmp_sys_info *info) { 1692 int status; 1693 struct rusage r_usage; 1694 1695 memset(info, 0, sizeof(*info)); 1696 1697 status = getrusage(RUSAGE_SELF, &r_usage); 1698 KMP_CHECK_SYSFAIL_ERRNO("getrusage", status); 1699 1700 // The maximum resident set size utilized (in kilobytes) 1701 info->maxrss = r_usage.ru_maxrss; 1702 // The number of page faults serviced without any I/O 1703 info->minflt = r_usage.ru_minflt; 1704 // The number of page faults serviced that required I/O 1705 info->majflt = r_usage.ru_majflt; 1706 // The number of times a process was "swapped" out of memory 1707 info->nswap = r_usage.ru_nswap; 1708 // The number of times the file system had to perform input 1709 info->inblock = r_usage.ru_inblock; 1710 // The number of times the file system had to perform output 1711 info->oublock = r_usage.ru_oublock; 1712 // The number of times a context switch was voluntarily 1713 info->nvcsw = r_usage.ru_nvcsw; 1714 // The number of times a context switch was forced 1715 info->nivcsw = r_usage.ru_nivcsw; 1716 1717 return (status != 0); 1718 } 1719 1720 void __kmp_read_system_time(double *delta) { 1721 double t_ns; 1722 struct timeval tval; 1723 struct timespec stop; 1724 int status; 1725 1726 status = gettimeofday(&tval, NULL); 1727 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status); 1728 TIMEVAL_TO_TIMESPEC(&tval, &stop); 1729 t_ns = (double)(TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start)); 1730 *delta = (t_ns * 1e-9); 1731 } 1732 1733 void __kmp_clear_system_time(void) { 1734 struct timeval tval; 1735 int status; 1736 status = gettimeofday(&tval, NULL); 1737 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status); 1738 TIMEVAL_TO_TIMESPEC(&tval, &__kmp_sys_timer_data.start); 1739 } 1740 1741 static int __kmp_get_xproc(void) { 1742 1743 int r = 0; 1744 1745 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ 1746 KMP_OS_OPENBSD || KMP_OS_HURD 1747 1748 __kmp_type_convert(sysconf(_SC_NPROCESSORS_ONLN), &(r)); 1749 1750 #elif KMP_OS_DARWIN 1751 1752 // Bug C77011 High "OpenMP Threads and number of active cores". 1753 1754 // Find the number of available CPUs. 1755 kern_return_t rc; 1756 host_basic_info_data_t info; 1757 mach_msg_type_number_t num = HOST_BASIC_INFO_COUNT; 1758 rc = host_info(mach_host_self(), HOST_BASIC_INFO, (host_info_t)&info, &num); 1759 if (rc == 0 && num == HOST_BASIC_INFO_COUNT) { 1760 // Cannot use KA_TRACE() here because this code works before trace support 1761 // is initialized. 1762 r = info.avail_cpus; 1763 } else { 1764 KMP_WARNING(CantGetNumAvailCPU); 1765 KMP_INFORM(AssumedNumCPU); 1766 } 1767 1768 #else 1769 1770 #error "Unknown or unsupported OS." 1771 1772 #endif 1773 1774 return r > 0 ? r : 2; /* guess value of 2 if OS told us 0 */ 1775 1776 } // __kmp_get_xproc 1777 1778 int __kmp_read_from_file(char const *path, char const *format, ...) { 1779 int result; 1780 va_list args; 1781 1782 va_start(args, format); 1783 FILE *f = fopen(path, "rb"); 1784 if (f == NULL) 1785 return 0; 1786 result = vfscanf(f, format, args); 1787 fclose(f); 1788 1789 return result; 1790 } 1791 1792 void __kmp_runtime_initialize(void) { 1793 int status; 1794 pthread_mutexattr_t mutex_attr; 1795 pthread_condattr_t cond_attr; 1796 1797 if (__kmp_init_runtime) { 1798 return; 1799 } 1800 1801 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) 1802 if (!__kmp_cpuinfo.initialized) { 1803 __kmp_query_cpuid(&__kmp_cpuinfo); 1804 } 1805 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 1806 1807 __kmp_xproc = __kmp_get_xproc(); 1808 1809 #if !KMP_32_BIT_ARCH 1810 struct rlimit rlim; 1811 // read stack size of calling thread, save it as default for worker threads; 1812 // this should be done before reading environment variables 1813 status = getrlimit(RLIMIT_STACK, &rlim); 1814 if (status == 0) { // success? 1815 __kmp_stksize = rlim.rlim_cur; 1816 __kmp_check_stksize(&__kmp_stksize); // check value and adjust if needed 1817 } 1818 #endif /* KMP_32_BIT_ARCH */ 1819 1820 if (sysconf(_SC_THREADS)) { 1821 1822 /* Query the maximum number of threads */ 1823 __kmp_type_convert(sysconf(_SC_THREAD_THREADS_MAX), &(__kmp_sys_max_nth)); 1824 if (__kmp_sys_max_nth == -1) { 1825 /* Unlimited threads for NPTL */ 1826 __kmp_sys_max_nth = INT_MAX; 1827 } else if (__kmp_sys_max_nth <= 1) { 1828 /* Can't tell, just use PTHREAD_THREADS_MAX */ 1829 __kmp_sys_max_nth = KMP_MAX_NTH; 1830 } 1831 1832 /* Query the minimum stack size */ 1833 __kmp_sys_min_stksize = sysconf(_SC_THREAD_STACK_MIN); 1834 if (__kmp_sys_min_stksize <= 1) { 1835 __kmp_sys_min_stksize = KMP_MIN_STKSIZE; 1836 } 1837 } 1838 1839 /* Set up minimum number of threads to switch to TLS gtid */ 1840 __kmp_tls_gtid_min = KMP_TLS_GTID_MIN; 1841 1842 status = pthread_key_create(&__kmp_gtid_threadprivate_key, 1843 __kmp_internal_end_dest); 1844 KMP_CHECK_SYSFAIL("pthread_key_create", status); 1845 status = pthread_mutexattr_init(&mutex_attr); 1846 KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status); 1847 status = pthread_mutex_init(&__kmp_wait_mx.m_mutex, &mutex_attr); 1848 KMP_CHECK_SYSFAIL("pthread_mutex_init", status); 1849 status = pthread_mutexattr_destroy(&mutex_attr); 1850 KMP_CHECK_SYSFAIL("pthread_mutexattr_destroy", status); 1851 status = pthread_condattr_init(&cond_attr); 1852 KMP_CHECK_SYSFAIL("pthread_condattr_init", status); 1853 status = pthread_cond_init(&__kmp_wait_cv.c_cond, &cond_attr); 1854 KMP_CHECK_SYSFAIL("pthread_cond_init", status); 1855 status = pthread_condattr_destroy(&cond_attr); 1856 KMP_CHECK_SYSFAIL("pthread_condattr_destroy", status); 1857 #if USE_ITT_BUILD 1858 __kmp_itt_initialize(); 1859 #endif /* USE_ITT_BUILD */ 1860 1861 __kmp_init_runtime = TRUE; 1862 } 1863 1864 void __kmp_runtime_destroy(void) { 1865 int status; 1866 1867 if (!__kmp_init_runtime) { 1868 return; // Nothing to do. 1869 } 1870 1871 #if USE_ITT_BUILD 1872 __kmp_itt_destroy(); 1873 #endif /* USE_ITT_BUILD */ 1874 1875 status = pthread_key_delete(__kmp_gtid_threadprivate_key); 1876 KMP_CHECK_SYSFAIL("pthread_key_delete", status); 1877 1878 status = pthread_mutex_destroy(&__kmp_wait_mx.m_mutex); 1879 if (status != 0 && status != EBUSY) { 1880 KMP_SYSFAIL("pthread_mutex_destroy", status); 1881 } 1882 status = pthread_cond_destroy(&__kmp_wait_cv.c_cond); 1883 if (status != 0 && status != EBUSY) { 1884 KMP_SYSFAIL("pthread_cond_destroy", status); 1885 } 1886 #if KMP_AFFINITY_SUPPORTED 1887 __kmp_affinity_uninitialize(); 1888 #endif 1889 1890 __kmp_init_runtime = FALSE; 1891 } 1892 1893 /* Put the thread to sleep for a time period */ 1894 /* NOTE: not currently used anywhere */ 1895 void __kmp_thread_sleep(int millis) { sleep((millis + 500) / 1000); } 1896 1897 /* Calculate the elapsed wall clock time for the user */ 1898 void __kmp_elapsed(double *t) { 1899 int status; 1900 #ifdef FIX_SGI_CLOCK 1901 struct timespec ts; 1902 1903 status = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts); 1904 KMP_CHECK_SYSFAIL_ERRNO("clock_gettime", status); 1905 *t = 1906 (double)ts.tv_nsec * (1.0 / (double)KMP_NSEC_PER_SEC) + (double)ts.tv_sec; 1907 #else 1908 struct timeval tv; 1909 1910 status = gettimeofday(&tv, NULL); 1911 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status); 1912 *t = 1913 (double)tv.tv_usec * (1.0 / (double)KMP_USEC_PER_SEC) + (double)tv.tv_sec; 1914 #endif 1915 } 1916 1917 /* Calculate the elapsed wall clock tick for the user */ 1918 void __kmp_elapsed_tick(double *t) { *t = 1 / (double)CLOCKS_PER_SEC; } 1919 1920 /* Return the current time stamp in nsec */ 1921 kmp_uint64 __kmp_now_nsec() { 1922 struct timeval t; 1923 gettimeofday(&t, NULL); 1924 kmp_uint64 nsec = (kmp_uint64)KMP_NSEC_PER_SEC * (kmp_uint64)t.tv_sec + 1925 (kmp_uint64)1000 * (kmp_uint64)t.tv_usec; 1926 return nsec; 1927 } 1928 1929 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1930 /* Measure clock ticks per millisecond */ 1931 void __kmp_initialize_system_tick() { 1932 kmp_uint64 now, nsec2, diff; 1933 kmp_uint64 delay = 100000; // 50~100 usec on most machines. 1934 kmp_uint64 nsec = __kmp_now_nsec(); 1935 kmp_uint64 goal = __kmp_hardware_timestamp() + delay; 1936 while ((now = __kmp_hardware_timestamp()) < goal) 1937 ; 1938 nsec2 = __kmp_now_nsec(); 1939 diff = nsec2 - nsec; 1940 if (diff > 0) { 1941 kmp_uint64 tpms = ((kmp_uint64)1e6 * (delay + (now - goal)) / diff); 1942 if (tpms > 0) 1943 __kmp_ticks_per_msec = tpms; 1944 } 1945 } 1946 #endif 1947 1948 /* Determine whether the given address is mapped into the current address 1949 space. */ 1950 1951 int __kmp_is_address_mapped(void *addr) { 1952 1953 int found = 0; 1954 int rc; 1955 1956 #if KMP_OS_LINUX || KMP_OS_HURD 1957 1958 /* On GNUish OSes, read the /proc/<pid>/maps pseudo-file to get all the 1959 address ranges mapped into the address space. */ 1960 1961 char *name = __kmp_str_format("/proc/%d/maps", getpid()); 1962 FILE *file = NULL; 1963 1964 file = fopen(name, "r"); 1965 KMP_ASSERT(file != NULL); 1966 1967 for (;;) { 1968 1969 void *beginning = NULL; 1970 void *ending = NULL; 1971 char perms[5]; 1972 1973 rc = fscanf(file, "%p-%p %4s %*[^\n]\n", &beginning, &ending, perms); 1974 if (rc == EOF) { 1975 break; 1976 } 1977 KMP_ASSERT(rc == 3 && 1978 KMP_STRLEN(perms) == 4); // Make sure all fields are read. 1979 1980 // Ending address is not included in the region, but beginning is. 1981 if ((addr >= beginning) && (addr < ending)) { 1982 perms[2] = 0; // 3th and 4th character does not matter. 1983 if (strcmp(perms, "rw") == 0) { 1984 // Memory we are looking for should be readable and writable. 1985 found = 1; 1986 } 1987 break; 1988 } 1989 } 1990 1991 // Free resources. 1992 fclose(file); 1993 KMP_INTERNAL_FREE(name); 1994 #elif KMP_OS_FREEBSD 1995 char *buf; 1996 size_t lstsz; 1997 int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid()}; 1998 rc = sysctl(mib, 4, NULL, &lstsz, NULL, 0); 1999 if (rc < 0) 2000 return 0; 2001 // We pass from number of vm entry's semantic 2002 // to size of whole entry map list. 2003 lstsz = lstsz * 4 / 3; 2004 buf = reinterpret_cast<char *>(kmpc_malloc(lstsz)); 2005 rc = sysctl(mib, 4, buf, &lstsz, NULL, 0); 2006 if (rc < 0) { 2007 kmpc_free(buf); 2008 return 0; 2009 } 2010 2011 char *lw = buf; 2012 char *up = buf + lstsz; 2013 2014 while (lw < up) { 2015 struct kinfo_vmentry *cur = reinterpret_cast<struct kinfo_vmentry *>(lw); 2016 size_t cursz = cur->kve_structsize; 2017 if (cursz == 0) 2018 break; 2019 void *start = reinterpret_cast<void *>(cur->kve_start); 2020 void *end = reinterpret_cast<void *>(cur->kve_end); 2021 // Readable/Writable addresses within current map entry 2022 if ((addr >= start) && (addr < end)) { 2023 if ((cur->kve_protection & KVME_PROT_READ) != 0 && 2024 (cur->kve_protection & KVME_PROT_WRITE) != 0) { 2025 found = 1; 2026 break; 2027 } 2028 } 2029 lw += cursz; 2030 } 2031 kmpc_free(buf); 2032 2033 #elif KMP_OS_DARWIN 2034 2035 /* On OS X*, /proc pseudo filesystem is not available. Try to read memory 2036 using vm interface. */ 2037 2038 int buffer; 2039 vm_size_t count; 2040 rc = vm_read_overwrite( 2041 mach_task_self(), // Task to read memory of. 2042 (vm_address_t)(addr), // Address to read from. 2043 1, // Number of bytes to be read. 2044 (vm_address_t)(&buffer), // Address of buffer to save read bytes in. 2045 &count // Address of var to save number of read bytes in. 2046 ); 2047 if (rc == 0) { 2048 // Memory successfully read. 2049 found = 1; 2050 } 2051 2052 #elif KMP_OS_NETBSD 2053 2054 int mib[5]; 2055 mib[0] = CTL_VM; 2056 mib[1] = VM_PROC; 2057 mib[2] = VM_PROC_MAP; 2058 mib[3] = getpid(); 2059 mib[4] = sizeof(struct kinfo_vmentry); 2060 2061 size_t size; 2062 rc = sysctl(mib, __arraycount(mib), NULL, &size, NULL, 0); 2063 KMP_ASSERT(!rc); 2064 KMP_ASSERT(size); 2065 2066 size = size * 4 / 3; 2067 struct kinfo_vmentry *kiv = (struct kinfo_vmentry *)KMP_INTERNAL_MALLOC(size); 2068 KMP_ASSERT(kiv); 2069 2070 rc = sysctl(mib, __arraycount(mib), kiv, &size, NULL, 0); 2071 KMP_ASSERT(!rc); 2072 KMP_ASSERT(size); 2073 2074 for (size_t i = 0; i < size; i++) { 2075 if (kiv[i].kve_start >= (uint64_t)addr && 2076 kiv[i].kve_end <= (uint64_t)addr) { 2077 found = 1; 2078 break; 2079 } 2080 } 2081 KMP_INTERNAL_FREE(kiv); 2082 #elif KMP_OS_OPENBSD 2083 2084 int mib[3]; 2085 mib[0] = CTL_KERN; 2086 mib[1] = KERN_PROC_VMMAP; 2087 mib[2] = getpid(); 2088 2089 size_t size; 2090 uint64_t end; 2091 rc = sysctl(mib, 3, NULL, &size, NULL, 0); 2092 KMP_ASSERT(!rc); 2093 KMP_ASSERT(size); 2094 end = size; 2095 2096 struct kinfo_vmentry kiv = {.kve_start = 0}; 2097 2098 while ((rc = sysctl(mib, 3, &kiv, &size, NULL, 0)) == 0) { 2099 KMP_ASSERT(size); 2100 if (kiv.kve_end == end) 2101 break; 2102 2103 if (kiv.kve_start >= (uint64_t)addr && kiv.kve_end <= (uint64_t)addr) { 2104 found = 1; 2105 break; 2106 } 2107 kiv.kve_start += 1; 2108 } 2109 #elif KMP_OS_DRAGONFLY 2110 2111 // FIXME(DragonFly): Implement this 2112 found = 1; 2113 2114 #else 2115 2116 #error "Unknown or unsupported OS" 2117 2118 #endif 2119 2120 return found; 2121 2122 } // __kmp_is_address_mapped 2123 2124 #ifdef USE_LOAD_BALANCE 2125 2126 #if KMP_OS_DARWIN || KMP_OS_NETBSD 2127 2128 // The function returns the rounded value of the system load average 2129 // during given time interval which depends on the value of 2130 // __kmp_load_balance_interval variable (default is 60 sec, other values 2131 // may be 300 sec or 900 sec). 2132 // It returns -1 in case of error. 2133 int __kmp_get_load_balance(int max) { 2134 double averages[3]; 2135 int ret_avg = 0; 2136 2137 int res = getloadavg(averages, 3); 2138 2139 // Check __kmp_load_balance_interval to determine which of averages to use. 2140 // getloadavg() may return the number of samples less than requested that is 2141 // less than 3. 2142 if (__kmp_load_balance_interval < 180 && (res >= 1)) { 2143 ret_avg = (int)averages[0]; // 1 min 2144 } else if ((__kmp_load_balance_interval >= 180 && 2145 __kmp_load_balance_interval < 600) && 2146 (res >= 2)) { 2147 ret_avg = (int)averages[1]; // 5 min 2148 } else if ((__kmp_load_balance_interval >= 600) && (res == 3)) { 2149 ret_avg = (int)averages[2]; // 15 min 2150 } else { // Error occurred 2151 return -1; 2152 } 2153 2154 return ret_avg; 2155 } 2156 2157 #else // Linux* OS 2158 2159 // The function returns number of running (not sleeping) threads, or -1 in case 2160 // of error. Error could be reported if Linux* OS kernel too old (without 2161 // "/proc" support). Counting running threads stops if max running threads 2162 // encountered. 2163 int __kmp_get_load_balance(int max) { 2164 static int permanent_error = 0; 2165 static int glb_running_threads = 0; // Saved count of the running threads for 2166 // the thread balance algorithm 2167 static double glb_call_time = 0; /* Thread balance algorithm call time */ 2168 2169 int running_threads = 0; // Number of running threads in the system. 2170 2171 DIR *proc_dir = NULL; // Handle of "/proc/" directory. 2172 struct dirent *proc_entry = NULL; 2173 2174 kmp_str_buf_t task_path; // "/proc/<pid>/task/<tid>/" path. 2175 DIR *task_dir = NULL; // Handle of "/proc/<pid>/task/<tid>/" directory. 2176 struct dirent *task_entry = NULL; 2177 int task_path_fixed_len; 2178 2179 kmp_str_buf_t stat_path; // "/proc/<pid>/task/<tid>/stat" path. 2180 int stat_file = -1; 2181 int stat_path_fixed_len; 2182 2183 int total_processes = 0; // Total number of processes in system. 2184 int total_threads = 0; // Total number of threads in system. 2185 2186 double call_time = 0.0; 2187 2188 __kmp_str_buf_init(&task_path); 2189 __kmp_str_buf_init(&stat_path); 2190 2191 __kmp_elapsed(&call_time); 2192 2193 if (glb_call_time && 2194 (call_time - glb_call_time < __kmp_load_balance_interval)) { 2195 running_threads = glb_running_threads; 2196 goto finish; 2197 } 2198 2199 glb_call_time = call_time; 2200 2201 // Do not spend time on scanning "/proc/" if we have a permanent error. 2202 if (permanent_error) { 2203 running_threads = -1; 2204 goto finish; 2205 } 2206 2207 if (max <= 0) { 2208 max = INT_MAX; 2209 } 2210 2211 // Open "/proc/" directory. 2212 proc_dir = opendir("/proc"); 2213 if (proc_dir == NULL) { 2214 // Cannot open "/prroc/". Probably the kernel does not support it. Return an 2215 // error now and in subsequent calls. 2216 running_threads = -1; 2217 permanent_error = 1; 2218 goto finish; 2219 } 2220 2221 // Initialize fixed part of task_path. This part will not change. 2222 __kmp_str_buf_cat(&task_path, "/proc/", 6); 2223 task_path_fixed_len = task_path.used; // Remember number of used characters. 2224 2225 proc_entry = readdir(proc_dir); 2226 while (proc_entry != NULL) { 2227 // Proc entry is a directory and name starts with a digit. Assume it is a 2228 // process' directory. 2229 if (proc_entry->d_type == DT_DIR && isdigit(proc_entry->d_name[0])) { 2230 2231 ++total_processes; 2232 // Make sure init process is the very first in "/proc", so we can replace 2233 // strcmp( proc_entry->d_name, "1" ) == 0 with simpler total_processes == 2234 // 1. We are going to check that total_processes == 1 => d_name == "1" is 2235 // true (where "=>" is implication). Since C++ does not have => operator, 2236 // let us replace it with its equivalent: a => b == ! a || b. 2237 KMP_DEBUG_ASSERT(total_processes != 1 || 2238 strcmp(proc_entry->d_name, "1") == 0); 2239 2240 // Construct task_path. 2241 task_path.used = task_path_fixed_len; // Reset task_path to "/proc/". 2242 __kmp_str_buf_cat(&task_path, proc_entry->d_name, 2243 KMP_STRLEN(proc_entry->d_name)); 2244 __kmp_str_buf_cat(&task_path, "/task", 5); 2245 2246 task_dir = opendir(task_path.str); 2247 if (task_dir == NULL) { 2248 // Process can finish between reading "/proc/" directory entry and 2249 // opening process' "task/" directory. So, in general case we should not 2250 // complain, but have to skip this process and read the next one. But on 2251 // systems with no "task/" support we will spend lot of time to scan 2252 // "/proc/" tree again and again without any benefit. "init" process 2253 // (its pid is 1) should exist always, so, if we cannot open 2254 // "/proc/1/task/" directory, it means "task/" is not supported by 2255 // kernel. Report an error now and in the future. 2256 if (strcmp(proc_entry->d_name, "1") == 0) { 2257 running_threads = -1; 2258 permanent_error = 1; 2259 goto finish; 2260 } 2261 } else { 2262 // Construct fixed part of stat file path. 2263 __kmp_str_buf_clear(&stat_path); 2264 __kmp_str_buf_cat(&stat_path, task_path.str, task_path.used); 2265 __kmp_str_buf_cat(&stat_path, "/", 1); 2266 stat_path_fixed_len = stat_path.used; 2267 2268 task_entry = readdir(task_dir); 2269 while (task_entry != NULL) { 2270 // It is a directory and name starts with a digit. 2271 if (proc_entry->d_type == DT_DIR && isdigit(task_entry->d_name[0])) { 2272 ++total_threads; 2273 2274 // Construct complete stat file path. Easiest way would be: 2275 // __kmp_str_buf_print( & stat_path, "%s/%s/stat", task_path.str, 2276 // task_entry->d_name ); 2277 // but seriae of __kmp_str_buf_cat works a bit faster. 2278 stat_path.used = 2279 stat_path_fixed_len; // Reset stat path to its fixed part. 2280 __kmp_str_buf_cat(&stat_path, task_entry->d_name, 2281 KMP_STRLEN(task_entry->d_name)); 2282 __kmp_str_buf_cat(&stat_path, "/stat", 5); 2283 2284 // Note: Low-level API (open/read/close) is used. High-level API 2285 // (fopen/fclose) works ~ 30 % slower. 2286 stat_file = open(stat_path.str, O_RDONLY); 2287 if (stat_file == -1) { 2288 // We cannot report an error because task (thread) can terminate 2289 // just before reading this file. 2290 } else { 2291 /* Content of "stat" file looks like: 2292 24285 (program) S ... 2293 2294 It is a single line (if program name does not include funny 2295 symbols). First number is a thread id, then name of executable 2296 file name in paretheses, then state of the thread. We need just 2297 thread state. 2298 2299 Good news: Length of program name is 15 characters max. Longer 2300 names are truncated. 2301 2302 Thus, we need rather short buffer: 15 chars for program name + 2303 2 parenthesis, + 3 spaces + ~7 digits of pid = 37. 2304 2305 Bad news: Program name may contain special symbols like space, 2306 closing parenthesis, or even new line. This makes parsing 2307 "stat" file not 100 % reliable. In case of fanny program names 2308 parsing may fail (report incorrect thread state). 2309 2310 Parsing "status" file looks more promissing (due to different 2311 file structure and escaping special symbols) but reading and 2312 parsing of "status" file works slower. 2313 -- ln 2314 */ 2315 char buffer[65]; 2316 ssize_t len; 2317 len = read(stat_file, buffer, sizeof(buffer) - 1); 2318 if (len >= 0) { 2319 buffer[len] = 0; 2320 // Using scanf: 2321 // sscanf( buffer, "%*d (%*s) %c ", & state ); 2322 // looks very nice, but searching for a closing parenthesis 2323 // works a bit faster. 2324 char *close_parent = strstr(buffer, ") "); 2325 if (close_parent != NULL) { 2326 char state = *(close_parent + 2); 2327 if (state == 'R') { 2328 ++running_threads; 2329 if (running_threads >= max) { 2330 goto finish; 2331 } 2332 } 2333 } 2334 } 2335 close(stat_file); 2336 stat_file = -1; 2337 } 2338 } 2339 task_entry = readdir(task_dir); 2340 } 2341 closedir(task_dir); 2342 task_dir = NULL; 2343 } 2344 } 2345 proc_entry = readdir(proc_dir); 2346 } 2347 2348 // There _might_ be a timing hole where the thread executing this 2349 // code get skipped in the load balance, and running_threads is 0. 2350 // Assert in the debug builds only!!! 2351 KMP_DEBUG_ASSERT(running_threads > 0); 2352 if (running_threads <= 0) { 2353 running_threads = 1; 2354 } 2355 2356 finish: // Clean up and exit. 2357 if (proc_dir != NULL) { 2358 closedir(proc_dir); 2359 } 2360 __kmp_str_buf_free(&task_path); 2361 if (task_dir != NULL) { 2362 closedir(task_dir); 2363 } 2364 __kmp_str_buf_free(&stat_path); 2365 if (stat_file != -1) { 2366 close(stat_file); 2367 } 2368 2369 glb_running_threads = running_threads; 2370 2371 return running_threads; 2372 2373 } // __kmp_get_load_balance 2374 2375 #endif // KMP_OS_DARWIN 2376 2377 #endif // USE_LOAD_BALANCE 2378 2379 #if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || \ 2380 ((KMP_OS_LINUX || KMP_OS_DARWIN) && KMP_ARCH_AARCH64) || \ 2381 KMP_ARCH_PPC64 || KMP_ARCH_RISCV64) 2382 2383 // we really only need the case with 1 argument, because CLANG always build 2384 // a struct of pointers to shared variables referenced in the outlined function 2385 int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc, 2386 void *p_argv[] 2387 #if OMPT_SUPPORT 2388 , 2389 void **exit_frame_ptr 2390 #endif 2391 ) { 2392 #if OMPT_SUPPORT 2393 *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0); 2394 #endif 2395 2396 switch (argc) { 2397 default: 2398 fprintf(stderr, "Too many args to microtask: %d!\n", argc); 2399 fflush(stderr); 2400 exit(-1); 2401 case 0: 2402 (*pkfn)(>id, &tid); 2403 break; 2404 case 1: 2405 (*pkfn)(>id, &tid, p_argv[0]); 2406 break; 2407 case 2: 2408 (*pkfn)(>id, &tid, p_argv[0], p_argv[1]); 2409 break; 2410 case 3: 2411 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2]); 2412 break; 2413 case 4: 2414 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3]); 2415 break; 2416 case 5: 2417 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4]); 2418 break; 2419 case 6: 2420 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4], 2421 p_argv[5]); 2422 break; 2423 case 7: 2424 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4], 2425 p_argv[5], p_argv[6]); 2426 break; 2427 case 8: 2428 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4], 2429 p_argv[5], p_argv[6], p_argv[7]); 2430 break; 2431 case 9: 2432 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4], 2433 p_argv[5], p_argv[6], p_argv[7], p_argv[8]); 2434 break; 2435 case 10: 2436 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4], 2437 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9]); 2438 break; 2439 case 11: 2440 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4], 2441 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10]); 2442 break; 2443 case 12: 2444 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4], 2445 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10], 2446 p_argv[11]); 2447 break; 2448 case 13: 2449 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4], 2450 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10], 2451 p_argv[11], p_argv[12]); 2452 break; 2453 case 14: 2454 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4], 2455 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10], 2456 p_argv[11], p_argv[12], p_argv[13]); 2457 break; 2458 case 15: 2459 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4], 2460 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10], 2461 p_argv[11], p_argv[12], p_argv[13], p_argv[14]); 2462 break; 2463 } 2464 2465 return 1; 2466 } 2467 2468 #endif 2469 2470 // Functions for hidden helper task 2471 namespace { 2472 // Condition variable for initializing hidden helper team 2473 pthread_cond_t hidden_helper_threads_initz_cond_var; 2474 pthread_mutex_t hidden_helper_threads_initz_lock; 2475 volatile int hidden_helper_initz_signaled = FALSE; 2476 2477 // Condition variable for deinitializing hidden helper team 2478 pthread_cond_t hidden_helper_threads_deinitz_cond_var; 2479 pthread_mutex_t hidden_helper_threads_deinitz_lock; 2480 volatile int hidden_helper_deinitz_signaled = FALSE; 2481 2482 // Condition variable for the wrapper function of main thread 2483 pthread_cond_t hidden_helper_main_thread_cond_var; 2484 pthread_mutex_t hidden_helper_main_thread_lock; 2485 volatile int hidden_helper_main_thread_signaled = FALSE; 2486 2487 // Semaphore for worker threads. We don't use condition variable here in case 2488 // that when multiple signals are sent at the same time, only one thread might 2489 // be waken. 2490 sem_t hidden_helper_task_sem; 2491 } // namespace 2492 2493 void __kmp_hidden_helper_worker_thread_wait() { 2494 int status = sem_wait(&hidden_helper_task_sem); 2495 KMP_CHECK_SYSFAIL("sem_wait", status); 2496 } 2497 2498 void __kmp_do_initialize_hidden_helper_threads() { 2499 // Initialize condition variable 2500 int status = 2501 pthread_cond_init(&hidden_helper_threads_initz_cond_var, nullptr); 2502 KMP_CHECK_SYSFAIL("pthread_cond_init", status); 2503 2504 status = pthread_cond_init(&hidden_helper_threads_deinitz_cond_var, nullptr); 2505 KMP_CHECK_SYSFAIL("pthread_cond_init", status); 2506 2507 status = pthread_cond_init(&hidden_helper_main_thread_cond_var, nullptr); 2508 KMP_CHECK_SYSFAIL("pthread_cond_init", status); 2509 2510 status = pthread_mutex_init(&hidden_helper_threads_initz_lock, nullptr); 2511 KMP_CHECK_SYSFAIL("pthread_mutex_init", status); 2512 2513 status = pthread_mutex_init(&hidden_helper_threads_deinitz_lock, nullptr); 2514 KMP_CHECK_SYSFAIL("pthread_mutex_init", status); 2515 2516 status = pthread_mutex_init(&hidden_helper_main_thread_lock, nullptr); 2517 KMP_CHECK_SYSFAIL("pthread_mutex_init", status); 2518 2519 // Initialize the semaphore 2520 status = sem_init(&hidden_helper_task_sem, 0, 0); 2521 KMP_CHECK_SYSFAIL("sem_init", status); 2522 2523 // Create a new thread to finish initialization 2524 pthread_t handle; 2525 status = pthread_create( 2526 &handle, nullptr, 2527 [](void *) -> void * { 2528 __kmp_hidden_helper_threads_initz_routine(); 2529 return nullptr; 2530 }, 2531 nullptr); 2532 KMP_CHECK_SYSFAIL("pthread_create", status); 2533 } 2534 2535 void __kmp_hidden_helper_threads_initz_wait() { 2536 // Initial thread waits here for the completion of the initialization. The 2537 // condition variable will be notified by main thread of hidden helper teams. 2538 int status = pthread_mutex_lock(&hidden_helper_threads_initz_lock); 2539 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 2540 2541 if (!TCR_4(hidden_helper_initz_signaled)) { 2542 status = pthread_cond_wait(&hidden_helper_threads_initz_cond_var, 2543 &hidden_helper_threads_initz_lock); 2544 KMP_CHECK_SYSFAIL("pthread_cond_wait", status); 2545 } 2546 2547 status = pthread_mutex_unlock(&hidden_helper_threads_initz_lock); 2548 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 2549 } 2550 2551 void __kmp_hidden_helper_initz_release() { 2552 // After all initialization, reset __kmp_init_hidden_helper_threads to false. 2553 int status = pthread_mutex_lock(&hidden_helper_threads_initz_lock); 2554 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 2555 2556 status = pthread_cond_signal(&hidden_helper_threads_initz_cond_var); 2557 KMP_CHECK_SYSFAIL("pthread_cond_wait", status); 2558 2559 TCW_SYNC_4(hidden_helper_initz_signaled, TRUE); 2560 2561 status = pthread_mutex_unlock(&hidden_helper_threads_initz_lock); 2562 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 2563 } 2564 2565 void __kmp_hidden_helper_main_thread_wait() { 2566 // The main thread of hidden helper team will be blocked here. The 2567 // condition variable can only be signal in the destructor of RTL. 2568 int status = pthread_mutex_lock(&hidden_helper_main_thread_lock); 2569 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 2570 2571 if (!TCR_4(hidden_helper_main_thread_signaled)) { 2572 status = pthread_cond_wait(&hidden_helper_main_thread_cond_var, 2573 &hidden_helper_main_thread_lock); 2574 KMP_CHECK_SYSFAIL("pthread_cond_wait", status); 2575 } 2576 2577 status = pthread_mutex_unlock(&hidden_helper_main_thread_lock); 2578 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 2579 } 2580 2581 void __kmp_hidden_helper_main_thread_release() { 2582 // The initial thread of OpenMP RTL should call this function to wake up the 2583 // main thread of hidden helper team. 2584 int status = pthread_mutex_lock(&hidden_helper_main_thread_lock); 2585 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 2586 2587 status = pthread_cond_signal(&hidden_helper_main_thread_cond_var); 2588 KMP_CHECK_SYSFAIL("pthread_cond_signal", status); 2589 2590 // The hidden helper team is done here 2591 TCW_SYNC_4(hidden_helper_main_thread_signaled, TRUE); 2592 2593 status = pthread_mutex_unlock(&hidden_helper_main_thread_lock); 2594 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 2595 } 2596 2597 void __kmp_hidden_helper_worker_thread_signal() { 2598 int status = sem_post(&hidden_helper_task_sem); 2599 KMP_CHECK_SYSFAIL("sem_post", status); 2600 } 2601 2602 void __kmp_hidden_helper_threads_deinitz_wait() { 2603 // Initial thread waits here for the completion of the deinitialization. The 2604 // condition variable will be notified by main thread of hidden helper teams. 2605 int status = pthread_mutex_lock(&hidden_helper_threads_deinitz_lock); 2606 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 2607 2608 if (!TCR_4(hidden_helper_deinitz_signaled)) { 2609 status = pthread_cond_wait(&hidden_helper_threads_deinitz_cond_var, 2610 &hidden_helper_threads_deinitz_lock); 2611 KMP_CHECK_SYSFAIL("pthread_cond_wait", status); 2612 } 2613 2614 status = pthread_mutex_unlock(&hidden_helper_threads_deinitz_lock); 2615 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 2616 } 2617 2618 void __kmp_hidden_helper_threads_deinitz_release() { 2619 int status = pthread_mutex_lock(&hidden_helper_threads_deinitz_lock); 2620 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 2621 2622 status = pthread_cond_signal(&hidden_helper_threads_deinitz_cond_var); 2623 KMP_CHECK_SYSFAIL("pthread_cond_wait", status); 2624 2625 TCW_SYNC_4(hidden_helper_deinitz_signaled, TRUE); 2626 2627 status = pthread_mutex_unlock(&hidden_helper_threads_deinitz_lock); 2628 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 2629 } 2630 2631 // end of file // 2632