1 /* 2 * z_Windows_NT_util.cpp -- platform specific routines. 3 */ 4 5 //===----------------------------------------------------------------------===// 6 // 7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 8 // See https://llvm.org/LICENSE.txt for license information. 9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "kmp.h" 14 #include "kmp_affinity.h" 15 #include "kmp_i18n.h" 16 #include "kmp_io.h" 17 #include "kmp_itt.h" 18 #include "kmp_wait_release.h" 19 20 /* This code is related to NtQuerySystemInformation() function. This function 21 is used in the Load balance algorithm for OMP_DYNAMIC=true to find the 22 number of running threads in the system. */ 23 24 #include <ntsecapi.h> // UNICODE_STRING 25 #include <ntstatus.h> 26 27 enum SYSTEM_INFORMATION_CLASS { 28 SystemProcessInformation = 5 29 }; // SYSTEM_INFORMATION_CLASS 30 31 struct CLIENT_ID { 32 HANDLE UniqueProcess; 33 HANDLE UniqueThread; 34 }; // struct CLIENT_ID 35 36 enum THREAD_STATE { 37 StateInitialized, 38 StateReady, 39 StateRunning, 40 StateStandby, 41 StateTerminated, 42 StateWait, 43 StateTransition, 44 StateUnknown 45 }; // enum THREAD_STATE 46 47 struct VM_COUNTERS { 48 SIZE_T PeakVirtualSize; 49 SIZE_T VirtualSize; 50 ULONG PageFaultCount; 51 SIZE_T PeakWorkingSetSize; 52 SIZE_T WorkingSetSize; 53 SIZE_T QuotaPeakPagedPoolUsage; 54 SIZE_T QuotaPagedPoolUsage; 55 SIZE_T QuotaPeakNonPagedPoolUsage; 56 SIZE_T QuotaNonPagedPoolUsage; 57 SIZE_T PagefileUsage; 58 SIZE_T PeakPagefileUsage; 59 SIZE_T PrivatePageCount; 60 }; // struct VM_COUNTERS 61 62 struct SYSTEM_THREAD { 63 LARGE_INTEGER KernelTime; 64 LARGE_INTEGER UserTime; 65 LARGE_INTEGER CreateTime; 66 ULONG WaitTime; 67 LPVOID StartAddress; 68 CLIENT_ID ClientId; 69 DWORD Priority; 70 LONG BasePriority; 71 ULONG ContextSwitchCount; 72 THREAD_STATE State; 73 ULONG WaitReason; 74 }; // SYSTEM_THREAD 75 76 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, KernelTime) == 0); 77 #if KMP_ARCH_X86 78 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 28); 79 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 52); 80 #else 81 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 32); 82 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 68); 83 #endif 84 85 struct SYSTEM_PROCESS_INFORMATION { 86 ULONG NextEntryOffset; 87 ULONG NumberOfThreads; 88 LARGE_INTEGER Reserved[3]; 89 LARGE_INTEGER CreateTime; 90 LARGE_INTEGER UserTime; 91 LARGE_INTEGER KernelTime; 92 UNICODE_STRING ImageName; 93 DWORD BasePriority; 94 HANDLE ProcessId; 95 HANDLE ParentProcessId; 96 ULONG HandleCount; 97 ULONG Reserved2[2]; 98 VM_COUNTERS VMCounters; 99 IO_COUNTERS IOCounters; 100 SYSTEM_THREAD Threads[1]; 101 }; // SYSTEM_PROCESS_INFORMATION 102 typedef SYSTEM_PROCESS_INFORMATION *PSYSTEM_PROCESS_INFORMATION; 103 104 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, NextEntryOffset) == 0); 105 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, CreateTime) == 32); 106 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ImageName) == 56); 107 #if KMP_ARCH_X86 108 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 68); 109 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 76); 110 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 88); 111 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 136); 112 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 184); 113 #else 114 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 80); 115 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 96); 116 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 112); 117 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 208); 118 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 256); 119 #endif 120 121 typedef NTSTATUS(NTAPI *NtQuerySystemInformation_t)(SYSTEM_INFORMATION_CLASS, 122 PVOID, ULONG, PULONG); 123 NtQuerySystemInformation_t NtQuerySystemInformation = NULL; 124 125 HMODULE ntdll = NULL; 126 127 /* End of NtQuerySystemInformation()-related code */ 128 129 static HMODULE kernel32 = NULL; 130 131 #if KMP_HANDLE_SIGNALS 132 typedef void (*sig_func_t)(int); 133 static sig_func_t __kmp_sighldrs[NSIG]; 134 static int __kmp_siginstalled[NSIG]; 135 #endif 136 137 #if KMP_USE_MONITOR 138 static HANDLE __kmp_monitor_ev; 139 #endif 140 static kmp_int64 __kmp_win32_time; 141 double __kmp_win32_tick; 142 143 int __kmp_init_runtime = FALSE; 144 CRITICAL_SECTION __kmp_win32_section; 145 146 void __kmp_win32_mutex_init(kmp_win32_mutex_t *mx) { 147 InitializeCriticalSection(&mx->cs); 148 #if USE_ITT_BUILD 149 __kmp_itt_system_object_created(&mx->cs, "Critical Section"); 150 #endif /* USE_ITT_BUILD */ 151 } 152 153 void __kmp_win32_mutex_destroy(kmp_win32_mutex_t *mx) { 154 DeleteCriticalSection(&mx->cs); 155 } 156 157 void __kmp_win32_mutex_lock(kmp_win32_mutex_t *mx) { 158 EnterCriticalSection(&mx->cs); 159 } 160 161 int __kmp_win32_mutex_trylock(kmp_win32_mutex_t *mx) { 162 return TryEnterCriticalSection(&mx->cs); 163 } 164 165 void __kmp_win32_mutex_unlock(kmp_win32_mutex_t *mx) { 166 LeaveCriticalSection(&mx->cs); 167 } 168 169 void __kmp_win32_cond_init(kmp_win32_cond_t *cv) { 170 cv->waiters_count_ = 0; 171 cv->wait_generation_count_ = 0; 172 cv->release_count_ = 0; 173 174 /* Initialize the critical section */ 175 __kmp_win32_mutex_init(&cv->waiters_count_lock_); 176 177 /* Create a manual-reset event. */ 178 cv->event_ = CreateEvent(NULL, // no security 179 TRUE, // manual-reset 180 FALSE, // non-signaled initially 181 NULL); // unnamed 182 #if USE_ITT_BUILD 183 __kmp_itt_system_object_created(cv->event_, "Event"); 184 #endif /* USE_ITT_BUILD */ 185 } 186 187 void __kmp_win32_cond_destroy(kmp_win32_cond_t *cv) { 188 __kmp_win32_mutex_destroy(&cv->waiters_count_lock_); 189 __kmp_free_handle(cv->event_); 190 memset(cv, '\0', sizeof(*cv)); 191 } 192 193 /* TODO associate cv with a team instead of a thread so as to optimize 194 the case where we wake up a whole team */ 195 196 template <class C> 197 static void __kmp_win32_cond_wait(kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx, 198 kmp_info_t *th, C *flag) { 199 int my_generation; 200 int last_waiter; 201 202 /* Avoid race conditions */ 203 __kmp_win32_mutex_lock(&cv->waiters_count_lock_); 204 205 /* Increment count of waiters */ 206 cv->waiters_count_++; 207 208 /* Store current generation in our activation record. */ 209 my_generation = cv->wait_generation_count_; 210 211 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_); 212 __kmp_win32_mutex_unlock(mx); 213 214 for (;;) { 215 int wait_done = 0; 216 DWORD res, timeout = 5000; // just tried to quess an appropriate number 217 /* Wait until the event is signaled */ 218 res = WaitForSingleObject(cv->event_, timeout); 219 220 if (res == WAIT_OBJECT_0) { 221 // event signaled 222 __kmp_win32_mutex_lock(&cv->waiters_count_lock_); 223 /* Exit the loop when the <cv->event_> is signaled and there are still 224 waiting threads from this <wait_generation> that haven't been released 225 from this wait yet. */ 226 wait_done = (cv->release_count_ > 0) && 227 (cv->wait_generation_count_ != my_generation); 228 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_); 229 } else if (res == WAIT_TIMEOUT || res == WAIT_FAILED) { 230 // check if the flag and cv counters are in consistent state 231 // as MS sent us debug dump whith inconsistent state of data 232 __kmp_win32_mutex_lock(mx); 233 typename C::flag_t old_f = flag->set_sleeping(); 234 if (!flag->done_check_val(old_f & ~KMP_BARRIER_SLEEP_STATE)) { 235 __kmp_win32_mutex_unlock(mx); 236 continue; 237 } 238 // condition fulfilled, exiting 239 old_f = flag->unset_sleeping(); 240 KMP_DEBUG_ASSERT(old_f & KMP_BARRIER_SLEEP_STATE); 241 TCW_PTR(th->th.th_sleep_loc, NULL); 242 KF_TRACE(50, ("__kmp_win32_cond_wait: exiting, condition " 243 "fulfilled: flag's loc(%p): %u => %u\n", 244 flag->get(), old_f, *(flag->get()))); 245 246 __kmp_win32_mutex_lock(&cv->waiters_count_lock_); 247 KMP_DEBUG_ASSERT(cv->waiters_count_ > 0); 248 cv->release_count_ = cv->waiters_count_; 249 cv->wait_generation_count_++; 250 wait_done = 1; 251 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_); 252 253 __kmp_win32_mutex_unlock(mx); 254 } 255 /* there used to be a semicolon after the if statement, it looked like a 256 bug, so i removed it */ 257 if (wait_done) 258 break; 259 } 260 261 __kmp_win32_mutex_lock(mx); 262 __kmp_win32_mutex_lock(&cv->waiters_count_lock_); 263 264 cv->waiters_count_--; 265 cv->release_count_--; 266 267 last_waiter = (cv->release_count_ == 0); 268 269 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_); 270 271 if (last_waiter) { 272 /* We're the last waiter to be notified, so reset the manual event. */ 273 ResetEvent(cv->event_); 274 } 275 } 276 277 void __kmp_win32_cond_broadcast(kmp_win32_cond_t *cv) { 278 __kmp_win32_mutex_lock(&cv->waiters_count_lock_); 279 280 if (cv->waiters_count_ > 0) { 281 SetEvent(cv->event_); 282 /* Release all the threads in this generation. */ 283 284 cv->release_count_ = cv->waiters_count_; 285 286 /* Start a new generation. */ 287 cv->wait_generation_count_++; 288 } 289 290 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_); 291 } 292 293 void __kmp_win32_cond_signal(kmp_win32_cond_t *cv) { 294 __kmp_win32_cond_broadcast(cv); 295 } 296 297 void __kmp_enable(int new_state) { 298 if (__kmp_init_runtime) 299 LeaveCriticalSection(&__kmp_win32_section); 300 } 301 302 void __kmp_disable(int *old_state) { 303 *old_state = 0; 304 305 if (__kmp_init_runtime) 306 EnterCriticalSection(&__kmp_win32_section); 307 } 308 309 void __kmp_suspend_initialize(void) { /* do nothing */ 310 } 311 312 void __kmp_suspend_initialize_thread(kmp_info_t *th) { 313 int old_value = KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init); 314 int new_value = TRUE; 315 // Return if already initialized 316 if (old_value == new_value) 317 return; 318 // Wait, then return if being initialized 319 if (old_value == -1 || 320 !__kmp_atomic_compare_store(&th->th.th_suspend_init, old_value, -1)) { 321 while (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init) != new_value) { 322 KMP_CPU_PAUSE(); 323 } 324 } else { 325 // Claim to be the initializer and do initializations 326 __kmp_win32_cond_init(&th->th.th_suspend_cv); 327 __kmp_win32_mutex_init(&th->th.th_suspend_mx); 328 KMP_ATOMIC_ST_REL(&th->th.th_suspend_init, new_value); 329 } 330 } 331 332 void __kmp_suspend_uninitialize_thread(kmp_info_t *th) { 333 if (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init)) { 334 /* this means we have initialize the suspension pthread objects for this 335 thread in this instance of the process */ 336 __kmp_win32_cond_destroy(&th->th.th_suspend_cv); 337 __kmp_win32_mutex_destroy(&th->th.th_suspend_mx); 338 KMP_ATOMIC_ST_REL(&th->th.th_suspend_init, FALSE); 339 } 340 } 341 342 int __kmp_try_suspend_mx(kmp_info_t *th) { 343 return __kmp_win32_mutex_trylock(&th->th.th_suspend_mx); 344 } 345 346 void __kmp_lock_suspend_mx(kmp_info_t *th) { 347 __kmp_win32_mutex_lock(&th->th.th_suspend_mx); 348 } 349 350 void __kmp_unlock_suspend_mx(kmp_info_t *th) { 351 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx); 352 } 353 354 /* This routine puts the calling thread to sleep after setting the 355 sleep bit for the indicated flag variable to true. */ 356 template <class C> 357 static inline void __kmp_suspend_template(int th_gtid, C *flag) { 358 kmp_info_t *th = __kmp_threads[th_gtid]; 359 int status; 360 typename C::flag_t old_spin; 361 362 KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag's loc(%p)\n", 363 th_gtid, flag->get())); 364 365 __kmp_suspend_initialize_thread(th); 366 __kmp_win32_mutex_lock(&th->th.th_suspend_mx); 367 368 KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for flag's" 369 " loc(%p)\n", 370 th_gtid, flag->get())); 371 372 /* TODO: shouldn't this use release semantics to ensure that 373 __kmp_suspend_initialize_thread gets called first? */ 374 old_spin = flag->set_sleeping(); 375 #if OMP_50_ENABLED 376 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME && 377 __kmp_pause_status != kmp_soft_paused) { 378 flag->unset_sleeping(); 379 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx); 380 return; 381 } 382 #endif 383 384 KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for flag's" 385 " loc(%p)==%d\n", 386 th_gtid, flag->get(), *(flag->get()))); 387 388 if (flag->done_check_val(old_spin)) { 389 old_spin = flag->unset_sleeping(); 390 KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit " 391 "for flag's loc(%p)\n", 392 th_gtid, flag->get())); 393 } else { 394 #ifdef DEBUG_SUSPEND 395 __kmp_suspend_count++; 396 #endif 397 /* Encapsulate in a loop as the documentation states that this may "with 398 low probability" return when the condition variable has not been signaled 399 or broadcast */ 400 int deactivated = FALSE; 401 TCW_PTR(th->th.th_sleep_loc, (void *)flag); 402 while (flag->is_sleeping()) { 403 KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform " 404 "kmp_win32_cond_wait()\n", 405 th_gtid)); 406 // Mark the thread as no longer active (only in the first iteration of the 407 // loop). 408 if (!deactivated) { 409 th->th.th_active = FALSE; 410 if (th->th.th_active_in_pool) { 411 th->th.th_active_in_pool = FALSE; 412 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth); 413 KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0); 414 } 415 deactivated = TRUE; 416 __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, th, 417 flag); 418 } else { 419 __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, th, 420 flag); 421 } 422 423 #ifdef KMP_DEBUG 424 if (flag->is_sleeping()) { 425 KF_TRACE(100, 426 ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid)); 427 } 428 #endif /* KMP_DEBUG */ 429 430 } // while 431 432 // Mark the thread as active again (if it was previous marked as inactive) 433 if (deactivated) { 434 th->th.th_active = TRUE; 435 if (TCR_4(th->th.th_in_pool)) { 436 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth); 437 th->th.th_active_in_pool = TRUE; 438 } 439 } 440 } 441 442 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx); 443 444 KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid)); 445 } 446 447 void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) { 448 __kmp_suspend_template(th_gtid, flag); 449 } 450 void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) { 451 __kmp_suspend_template(th_gtid, flag); 452 } 453 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) { 454 __kmp_suspend_template(th_gtid, flag); 455 } 456 457 /* This routine signals the thread specified by target_gtid to wake up 458 after setting the sleep bit indicated by the flag argument to FALSE */ 459 template <class C> 460 static inline void __kmp_resume_template(int target_gtid, C *flag) { 461 kmp_info_t *th = __kmp_threads[target_gtid]; 462 int status; 463 464 #ifdef KMP_DEBUG 465 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1; 466 #endif 467 468 KF_TRACE(30, ("__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", 469 gtid, target_gtid)); 470 471 __kmp_suspend_initialize_thread(th); 472 __kmp_win32_mutex_lock(&th->th.th_suspend_mx); 473 474 if (!flag) { // coming from __kmp_null_resume_wrapper 475 flag = (C *)th->th.th_sleep_loc; 476 } 477 478 // First, check if the flag is null or its type has changed. If so, someone 479 // else woke it up. 480 if (!flag || flag->get_type() != flag->get_ptr_type()) { // get_ptr_type 481 // simply shows what 482 // flag was cast to 483 KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already " 484 "awake: flag's loc(%p)\n", 485 gtid, target_gtid, NULL)); 486 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx); 487 return; 488 } else { 489 typename C::flag_t old_spin = flag->unset_sleeping(); 490 if (!flag->is_sleeping_val(old_spin)) { 491 KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already " 492 "awake: flag's loc(%p): %u => %u\n", 493 gtid, target_gtid, flag->get(), old_spin, *(flag->get()))); 494 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx); 495 return; 496 } 497 } 498 TCW_PTR(th->th.th_sleep_loc, NULL); 499 KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep " 500 "bit for flag's loc(%p)\n", 501 gtid, target_gtid, flag->get())); 502 503 __kmp_win32_cond_signal(&th->th.th_suspend_cv); 504 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx); 505 506 KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up" 507 " for T#%d\n", 508 gtid, target_gtid)); 509 } 510 511 void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) { 512 __kmp_resume_template(target_gtid, flag); 513 } 514 void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) { 515 __kmp_resume_template(target_gtid, flag); 516 } 517 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) { 518 __kmp_resume_template(target_gtid, flag); 519 } 520 521 void __kmp_yield() { Sleep(0); } 522 523 void __kmp_gtid_set_specific(int gtid) { 524 if (__kmp_init_gtid) { 525 KA_TRACE(50, ("__kmp_gtid_set_specific: T#%d key:%d\n", gtid, 526 __kmp_gtid_threadprivate_key)); 527 if (!TlsSetValue(__kmp_gtid_threadprivate_key, (LPVOID)(gtid + 1))) 528 KMP_FATAL(TLSSetValueFailed); 529 } else { 530 KA_TRACE(50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n")); 531 } 532 } 533 534 int __kmp_gtid_get_specific() { 535 int gtid; 536 if (!__kmp_init_gtid) { 537 KA_TRACE(50, ("__kmp_gtid_get_specific: runtime shutdown, returning " 538 "KMP_GTID_SHUTDOWN\n")); 539 return KMP_GTID_SHUTDOWN; 540 } 541 gtid = (int)(kmp_intptr_t)TlsGetValue(__kmp_gtid_threadprivate_key); 542 if (gtid == 0) { 543 gtid = KMP_GTID_DNE; 544 } else { 545 gtid--; 546 } 547 KA_TRACE(50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n", 548 __kmp_gtid_threadprivate_key, gtid)); 549 return gtid; 550 } 551 552 void __kmp_affinity_bind_thread(int proc) { 553 if (__kmp_num_proc_groups > 1) { 554 // Form the GROUP_AFFINITY struct directly, rather than filling 555 // out a bit vector and calling __kmp_set_system_affinity(). 556 GROUP_AFFINITY ga; 557 KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups * CHAR_BIT * 558 sizeof(DWORD_PTR)))); 559 ga.Group = proc / (CHAR_BIT * sizeof(DWORD_PTR)); 560 ga.Mask = (unsigned long long)1 << (proc % (CHAR_BIT * sizeof(DWORD_PTR))); 561 ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0; 562 563 KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL); 564 if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) { 565 DWORD error = GetLastError(); 566 if (__kmp_affinity_verbose) { // AC: continue silently if not verbose 567 kmp_msg_t err_code = KMP_ERR(error); 568 __kmp_msg(kmp_ms_warning, KMP_MSG(CantSetThreadAffMask), err_code, 569 __kmp_msg_null); 570 if (__kmp_generate_warnings == kmp_warnings_off) { 571 __kmp_str_free(&err_code.str); 572 } 573 } 574 } 575 } else { 576 kmp_affin_mask_t *mask; 577 KMP_CPU_ALLOC_ON_STACK(mask); 578 KMP_CPU_ZERO(mask); 579 KMP_CPU_SET(proc, mask); 580 __kmp_set_system_affinity(mask, TRUE); 581 KMP_CPU_FREE_FROM_STACK(mask); 582 } 583 } 584 585 void __kmp_affinity_determine_capable(const char *env_var) { 586 // All versions of Windows* OS (since Win '95) support SetThreadAffinityMask(). 587 588 #if KMP_GROUP_AFFINITY 589 KMP_AFFINITY_ENABLE(__kmp_num_proc_groups * sizeof(DWORD_PTR)); 590 #else 591 KMP_AFFINITY_ENABLE(sizeof(DWORD_PTR)); 592 #endif 593 594 KA_TRACE(10, ("__kmp_affinity_determine_capable: " 595 "Windows* OS affinity interface functional (mask size = " 596 "%" KMP_SIZE_T_SPEC ").\n", 597 __kmp_affin_mask_size)); 598 } 599 600 double __kmp_read_cpu_time(void) { 601 FILETIME CreationTime, ExitTime, KernelTime, UserTime; 602 int status; 603 double cpu_time; 604 605 cpu_time = 0; 606 607 status = GetProcessTimes(GetCurrentProcess(), &CreationTime, &ExitTime, 608 &KernelTime, &UserTime); 609 610 if (status) { 611 double sec = 0; 612 613 sec += KernelTime.dwHighDateTime; 614 sec += UserTime.dwHighDateTime; 615 616 /* Shift left by 32 bits */ 617 sec *= (double)(1 << 16) * (double)(1 << 16); 618 619 sec += KernelTime.dwLowDateTime; 620 sec += UserTime.dwLowDateTime; 621 622 cpu_time += (sec * 100.0) / KMP_NSEC_PER_SEC; 623 } 624 625 return cpu_time; 626 } 627 628 int __kmp_read_system_info(struct kmp_sys_info *info) { 629 info->maxrss = 0; /* the maximum resident set size utilized (in kilobytes) */ 630 info->minflt = 0; /* the number of page faults serviced without any I/O */ 631 info->majflt = 0; /* the number of page faults serviced that required I/O */ 632 info->nswap = 0; // the number of times a process was "swapped" out of memory 633 info->inblock = 0; // the number of times the file system had to perform input 634 info->oublock = 0; // number of times the file system had to perform output 635 info->nvcsw = 0; /* the number of times a context switch was voluntarily */ 636 info->nivcsw = 0; /* the number of times a context switch was forced */ 637 638 return 1; 639 } 640 641 void __kmp_runtime_initialize(void) { 642 SYSTEM_INFO info; 643 kmp_str_buf_t path; 644 UINT path_size; 645 646 if (__kmp_init_runtime) { 647 return; 648 } 649 650 #if KMP_DYNAMIC_LIB 651 /* Pin dynamic library for the lifetime of application */ 652 { 653 // First, turn off error message boxes 654 UINT err_mode = SetErrorMode(SEM_FAILCRITICALERRORS); 655 HMODULE h; 656 BOOL ret = GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | 657 GET_MODULE_HANDLE_EX_FLAG_PIN, 658 (LPCTSTR)&__kmp_serial_initialize, &h); 659 KMP_DEBUG_ASSERT2(h && ret, "OpenMP RTL cannot find itself loaded"); 660 SetErrorMode(err_mode); // Restore error mode 661 KA_TRACE(10, ("__kmp_runtime_initialize: dynamic library pinned\n")); 662 } 663 #endif 664 665 InitializeCriticalSection(&__kmp_win32_section); 666 #if USE_ITT_BUILD 667 __kmp_itt_system_object_created(&__kmp_win32_section, "Critical Section"); 668 #endif /* USE_ITT_BUILD */ 669 __kmp_initialize_system_tick(); 670 671 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) 672 if (!__kmp_cpuinfo.initialized) { 673 __kmp_query_cpuid(&__kmp_cpuinfo); 674 } 675 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 676 677 /* Set up minimum number of threads to switch to TLS gtid */ 678 #if KMP_OS_WINDOWS && !KMP_DYNAMIC_LIB 679 // Windows* OS, static library. 680 /* New thread may use stack space previously used by another thread, 681 currently terminated. On Windows* OS, in case of static linking, we do not 682 know the moment of thread termination, and our structures (__kmp_threads 683 and __kmp_root arrays) are still keep info about dead threads. This leads 684 to problem in __kmp_get_global_thread_id() function: it wrongly finds gtid 685 (by searching through stack addresses of all known threads) for 686 unregistered foreign tread. 687 688 Setting __kmp_tls_gtid_min to 0 workarounds this problem: 689 __kmp_get_global_thread_id() does not search through stacks, but get gtid 690 from TLS immediately. 691 --ln 692 */ 693 __kmp_tls_gtid_min = 0; 694 #else 695 __kmp_tls_gtid_min = KMP_TLS_GTID_MIN; 696 #endif 697 698 /* for the static library */ 699 if (!__kmp_gtid_threadprivate_key) { 700 __kmp_gtid_threadprivate_key = TlsAlloc(); 701 if (__kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES) { 702 KMP_FATAL(TLSOutOfIndexes); 703 } 704 } 705 706 // Load ntdll.dll. 707 /* Simple GetModuleHandle( "ntdll.dl" ) is not suitable due to security issue 708 (see http://www.microsoft.com/technet/security/advisory/2269637.mspx). We 709 have to specify full path to the library. */ 710 __kmp_str_buf_init(&path); 711 path_size = GetSystemDirectory(path.str, path.size); 712 KMP_DEBUG_ASSERT(path_size > 0); 713 if (path_size >= path.size) { 714 // Buffer is too short. Expand the buffer and try again. 715 __kmp_str_buf_reserve(&path, path_size); 716 path_size = GetSystemDirectory(path.str, path.size); 717 KMP_DEBUG_ASSERT(path_size > 0); 718 } 719 if (path_size > 0 && path_size < path.size) { 720 // Now we have system directory name in the buffer. 721 // Append backslash and name of dll to form full path, 722 path.used = path_size; 723 __kmp_str_buf_print(&path, "\\%s", "ntdll.dll"); 724 725 // Now load ntdll using full path. 726 ntdll = GetModuleHandle(path.str); 727 } 728 729 KMP_DEBUG_ASSERT(ntdll != NULL); 730 if (ntdll != NULL) { 731 NtQuerySystemInformation = (NtQuerySystemInformation_t)GetProcAddress( 732 ntdll, "NtQuerySystemInformation"); 733 } 734 KMP_DEBUG_ASSERT(NtQuerySystemInformation != NULL); 735 736 #if KMP_GROUP_AFFINITY 737 // Load kernel32.dll. 738 // Same caveat - must use full system path name. 739 if (path_size > 0 && path_size < path.size) { 740 // Truncate the buffer back to just the system path length, 741 // discarding "\\ntdll.dll", and replacing it with "kernel32.dll". 742 path.used = path_size; 743 __kmp_str_buf_print(&path, "\\%s", "kernel32.dll"); 744 745 // Load kernel32.dll using full path. 746 kernel32 = GetModuleHandle(path.str); 747 KA_TRACE(10, ("__kmp_runtime_initialize: kernel32.dll = %s\n", path.str)); 748 749 // Load the function pointers to kernel32.dll routines 750 // that may or may not exist on this system. 751 if (kernel32 != NULL) { 752 __kmp_GetActiveProcessorCount = 753 (kmp_GetActiveProcessorCount_t)GetProcAddress( 754 kernel32, "GetActiveProcessorCount"); 755 __kmp_GetActiveProcessorGroupCount = 756 (kmp_GetActiveProcessorGroupCount_t)GetProcAddress( 757 kernel32, "GetActiveProcessorGroupCount"); 758 __kmp_GetThreadGroupAffinity = 759 (kmp_GetThreadGroupAffinity_t)GetProcAddress( 760 kernel32, "GetThreadGroupAffinity"); 761 __kmp_SetThreadGroupAffinity = 762 (kmp_SetThreadGroupAffinity_t)GetProcAddress( 763 kernel32, "SetThreadGroupAffinity"); 764 765 KA_TRACE(10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorCount" 766 " = %p\n", 767 __kmp_GetActiveProcessorCount)); 768 KA_TRACE(10, ("__kmp_runtime_initialize: " 769 "__kmp_GetActiveProcessorGroupCount = %p\n", 770 __kmp_GetActiveProcessorGroupCount)); 771 KA_TRACE(10, ("__kmp_runtime_initialize:__kmp_GetThreadGroupAffinity" 772 " = %p\n", 773 __kmp_GetThreadGroupAffinity)); 774 KA_TRACE(10, ("__kmp_runtime_initialize: __kmp_SetThreadGroupAffinity" 775 " = %p\n", 776 __kmp_SetThreadGroupAffinity)); 777 KA_TRACE(10, ("__kmp_runtime_initialize: sizeof(kmp_affin_mask_t) = %d\n", 778 sizeof(kmp_affin_mask_t))); 779 780 // See if group affinity is supported on this system. 781 // If so, calculate the #groups and #procs. 782 // 783 // Group affinity was introduced with Windows* 7 OS and 784 // Windows* Server 2008 R2 OS. 785 if ((__kmp_GetActiveProcessorCount != NULL) && 786 (__kmp_GetActiveProcessorGroupCount != NULL) && 787 (__kmp_GetThreadGroupAffinity != NULL) && 788 (__kmp_SetThreadGroupAffinity != NULL) && 789 ((__kmp_num_proc_groups = __kmp_GetActiveProcessorGroupCount()) > 790 1)) { 791 // Calculate the total number of active OS procs. 792 int i; 793 794 KA_TRACE(10, ("__kmp_runtime_initialize: %d processor groups" 795 " detected\n", 796 __kmp_num_proc_groups)); 797 798 __kmp_xproc = 0; 799 800 for (i = 0; i < __kmp_num_proc_groups; i++) { 801 DWORD size = __kmp_GetActiveProcessorCount(i); 802 __kmp_xproc += size; 803 KA_TRACE(10, ("__kmp_runtime_initialize: proc group %d size = %d\n", 804 i, size)); 805 } 806 } else { 807 KA_TRACE(10, ("__kmp_runtime_initialize: %d processor groups" 808 " detected\n", 809 __kmp_num_proc_groups)); 810 } 811 } 812 } 813 if (__kmp_num_proc_groups <= 1) { 814 GetSystemInfo(&info); 815 __kmp_xproc = info.dwNumberOfProcessors; 816 } 817 #else 818 GetSystemInfo(&info); 819 __kmp_xproc = info.dwNumberOfProcessors; 820 #endif /* KMP_GROUP_AFFINITY */ 821 822 // If the OS said there were 0 procs, take a guess and use a value of 2. 823 // This is done for Linux* OS, also. Do we need error / warning? 824 if (__kmp_xproc <= 0) { 825 __kmp_xproc = 2; 826 } 827 828 KA_TRACE(5, 829 ("__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc)); 830 831 __kmp_str_buf_free(&path); 832 833 #if USE_ITT_BUILD 834 __kmp_itt_initialize(); 835 #endif /* USE_ITT_BUILD */ 836 837 __kmp_init_runtime = TRUE; 838 } // __kmp_runtime_initialize 839 840 void __kmp_runtime_destroy(void) { 841 if (!__kmp_init_runtime) { 842 return; 843 } 844 845 #if USE_ITT_BUILD 846 __kmp_itt_destroy(); 847 #endif /* USE_ITT_BUILD */ 848 849 /* we can't DeleteCriticalsection( & __kmp_win32_section ); */ 850 /* due to the KX_TRACE() commands */ 851 KA_TRACE(40, ("__kmp_runtime_destroy\n")); 852 853 if (__kmp_gtid_threadprivate_key) { 854 TlsFree(__kmp_gtid_threadprivate_key); 855 __kmp_gtid_threadprivate_key = 0; 856 } 857 858 __kmp_affinity_uninitialize(); 859 DeleteCriticalSection(&__kmp_win32_section); 860 861 ntdll = NULL; 862 NtQuerySystemInformation = NULL; 863 864 #if KMP_ARCH_X86_64 865 kernel32 = NULL; 866 __kmp_GetActiveProcessorCount = NULL; 867 __kmp_GetActiveProcessorGroupCount = NULL; 868 __kmp_GetThreadGroupAffinity = NULL; 869 __kmp_SetThreadGroupAffinity = NULL; 870 #endif // KMP_ARCH_X86_64 871 872 __kmp_init_runtime = FALSE; 873 } 874 875 void __kmp_terminate_thread(int gtid) { 876 kmp_info_t *th = __kmp_threads[gtid]; 877 878 if (!th) 879 return; 880 881 KA_TRACE(10, ("__kmp_terminate_thread: kill (%d)\n", gtid)); 882 883 if (TerminateThread(th->th.th_info.ds.ds_thread, (DWORD)-1) == FALSE) { 884 /* It's OK, the thread may have exited already */ 885 } 886 __kmp_free_handle(th->th.th_info.ds.ds_thread); 887 } 888 889 void __kmp_clear_system_time(void) { 890 BOOL status; 891 LARGE_INTEGER time; 892 status = QueryPerformanceCounter(&time); 893 __kmp_win32_time = (kmp_int64)time.QuadPart; 894 } 895 896 void __kmp_initialize_system_tick(void) { 897 { 898 BOOL status; 899 LARGE_INTEGER freq; 900 901 status = QueryPerformanceFrequency(&freq); 902 if (!status) { 903 DWORD error = GetLastError(); 904 __kmp_fatal(KMP_MSG(FunctionError, "QueryPerformanceFrequency()"), 905 KMP_ERR(error), __kmp_msg_null); 906 907 } else { 908 __kmp_win32_tick = ((double)1.0) / (double)freq.QuadPart; 909 } 910 } 911 } 912 913 /* Calculate the elapsed wall clock time for the user */ 914 915 void __kmp_elapsed(double *t) { 916 BOOL status; 917 LARGE_INTEGER now; 918 status = QueryPerformanceCounter(&now); 919 *t = ((double)now.QuadPart) * __kmp_win32_tick; 920 } 921 922 /* Calculate the elapsed wall clock tick for the user */ 923 924 void __kmp_elapsed_tick(double *t) { *t = __kmp_win32_tick; } 925 926 void __kmp_read_system_time(double *delta) { 927 if (delta != NULL) { 928 BOOL status; 929 LARGE_INTEGER now; 930 931 status = QueryPerformanceCounter(&now); 932 933 *delta = ((double)(((kmp_int64)now.QuadPart) - __kmp_win32_time)) * 934 __kmp_win32_tick; 935 } 936 } 937 938 /* Return the current time stamp in nsec */ 939 kmp_uint64 __kmp_now_nsec() { 940 LARGE_INTEGER now; 941 QueryPerformanceCounter(&now); 942 return 1e9 * __kmp_win32_tick * now.QuadPart; 943 } 944 945 extern "C" 946 void *__stdcall __kmp_launch_worker(void *arg) { 947 volatile void *stack_data; 948 void *exit_val; 949 void *padding = 0; 950 kmp_info_t *this_thr = (kmp_info_t *)arg; 951 int gtid; 952 953 gtid = this_thr->th.th_info.ds.ds_gtid; 954 __kmp_gtid_set_specific(gtid); 955 #ifdef KMP_TDATA_GTID 956 #error "This define causes problems with LoadLibrary() + declspec(thread) " \ 957 "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \ 958 "reference: http://support.microsoft.com/kb/118816" 959 //__kmp_gtid = gtid; 960 #endif 961 962 #if USE_ITT_BUILD 963 __kmp_itt_thread_name(gtid); 964 #endif /* USE_ITT_BUILD */ 965 966 __kmp_affinity_set_init_mask(gtid, FALSE); 967 968 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 969 // Set FP control regs to be a copy of the parallel initialization thread's. 970 __kmp_clear_x87_fpu_status_word(); 971 __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word); 972 __kmp_load_mxcsr(&__kmp_init_mxcsr); 973 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 974 975 if (__kmp_stkoffset > 0 && gtid > 0) { 976 padding = KMP_ALLOCA(gtid * __kmp_stkoffset); 977 } 978 979 KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive); 980 this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId(); 981 TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE); 982 983 if (TCR_4(__kmp_gtid_mode) < 984 2) { // check stack only if it is used to get gtid 985 TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data); 986 KMP_ASSERT(this_thr->th.th_info.ds.ds_stackgrow == FALSE); 987 __kmp_check_stack_overlap(this_thr); 988 } 989 KMP_MB(); 990 exit_val = __kmp_launch_thread(this_thr); 991 KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive); 992 TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE); 993 KMP_MB(); 994 return exit_val; 995 } 996 997 #if KMP_USE_MONITOR 998 /* The monitor thread controls all of the threads in the complex */ 999 1000 void *__stdcall __kmp_launch_monitor(void *arg) { 1001 DWORD wait_status; 1002 kmp_thread_t monitor; 1003 int status; 1004 int interval; 1005 kmp_info_t *this_thr = (kmp_info_t *)arg; 1006 1007 KMP_DEBUG_ASSERT(__kmp_init_monitor); 1008 TCW_4(__kmp_init_monitor, 2); // AC: Signal library that monitor has started 1009 // TODO: hide "2" in enum (like {true,false,started}) 1010 this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId(); 1011 TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE); 1012 1013 KMP_MB(); /* Flush all pending memory write invalidates. */ 1014 KA_TRACE(10, ("__kmp_launch_monitor: launched\n")); 1015 1016 monitor = GetCurrentThread(); 1017 1018 /* set thread priority */ 1019 status = SetThreadPriority(monitor, THREAD_PRIORITY_HIGHEST); 1020 if (!status) { 1021 DWORD error = GetLastError(); 1022 __kmp_fatal(KMP_MSG(CantSetThreadPriority), KMP_ERR(error), __kmp_msg_null); 1023 } 1024 1025 /* register us as monitor */ 1026 __kmp_gtid_set_specific(KMP_GTID_MONITOR); 1027 #ifdef KMP_TDATA_GTID 1028 #error "This define causes problems with LoadLibrary() + declspec(thread) " \ 1029 "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \ 1030 "reference: http://support.microsoft.com/kb/118816" 1031 //__kmp_gtid = KMP_GTID_MONITOR; 1032 #endif 1033 1034 #if USE_ITT_BUILD 1035 __kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore 1036 // monitor thread. 1037 #endif /* USE_ITT_BUILD */ 1038 1039 KMP_MB(); /* Flush all pending memory write invalidates. */ 1040 1041 interval = (1000 / __kmp_monitor_wakeups); /* in milliseconds */ 1042 1043 while (!TCR_4(__kmp_global.g.g_done)) { 1044 /* This thread monitors the state of the system */ 1045 1046 KA_TRACE(15, ("__kmp_launch_monitor: update\n")); 1047 1048 wait_status = WaitForSingleObject(__kmp_monitor_ev, interval); 1049 1050 if (wait_status == WAIT_TIMEOUT) { 1051 TCW_4(__kmp_global.g.g_time.dt.t_value, 1052 TCR_4(__kmp_global.g.g_time.dt.t_value) + 1); 1053 } 1054 1055 KMP_MB(); /* Flush all pending memory write invalidates. */ 1056 } 1057 1058 KA_TRACE(10, ("__kmp_launch_monitor: finished\n")); 1059 1060 status = SetThreadPriority(monitor, THREAD_PRIORITY_NORMAL); 1061 if (!status) { 1062 DWORD error = GetLastError(); 1063 __kmp_fatal(KMP_MSG(CantSetThreadPriority), KMP_ERR(error), __kmp_msg_null); 1064 } 1065 1066 if (__kmp_global.g.g_abort != 0) { 1067 /* now we need to terminate the worker threads */ 1068 /* the value of t_abort is the signal we caught */ 1069 int gtid; 1070 1071 KA_TRACE(10, ("__kmp_launch_monitor: terminate sig=%d\n", 1072 (__kmp_global.g.g_abort))); 1073 1074 /* terminate the OpenMP worker threads */ 1075 /* TODO this is not valid for sibling threads!! 1076 * the uber master might not be 0 anymore.. */ 1077 for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid) 1078 __kmp_terminate_thread(gtid); 1079 1080 __kmp_cleanup(); 1081 1082 Sleep(0); 1083 1084 KA_TRACE(10, 1085 ("__kmp_launch_monitor: raise sig=%d\n", __kmp_global.g.g_abort)); 1086 1087 if (__kmp_global.g.g_abort > 0) { 1088 raise(__kmp_global.g.g_abort); 1089 } 1090 } 1091 1092 TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE); 1093 1094 KMP_MB(); 1095 return arg; 1096 } 1097 #endif 1098 1099 void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size) { 1100 kmp_thread_t handle; 1101 DWORD idThread; 1102 1103 KA_TRACE(10, ("__kmp_create_worker: try to create thread (%d)\n", gtid)); 1104 1105 th->th.th_info.ds.ds_gtid = gtid; 1106 1107 if (KMP_UBER_GTID(gtid)) { 1108 int stack_data; 1109 1110 /* TODO: GetCurrentThread() returns a pseudo-handle that is unsuitable for 1111 other threads to use. Is it appropriate to just use GetCurrentThread? 1112 When should we close this handle? When unregistering the root? */ 1113 { 1114 BOOL rc; 1115 rc = DuplicateHandle(GetCurrentProcess(), GetCurrentThread(), 1116 GetCurrentProcess(), &th->th.th_info.ds.ds_thread, 0, 1117 FALSE, DUPLICATE_SAME_ACCESS); 1118 KMP_ASSERT(rc); 1119 KA_TRACE(10, (" __kmp_create_worker: ROOT Handle duplicated, th = %p, " 1120 "handle = %" KMP_UINTPTR_SPEC "\n", 1121 (LPVOID)th, th->th.th_info.ds.ds_thread)); 1122 th->th.th_info.ds.ds_thread_id = GetCurrentThreadId(); 1123 } 1124 if (TCR_4(__kmp_gtid_mode) < 2) { // check stack only if used to get gtid 1125 /* we will dynamically update the stack range if gtid_mode == 1 */ 1126 TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data); 1127 TCW_PTR(th->th.th_info.ds.ds_stacksize, 0); 1128 TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE); 1129 __kmp_check_stack_overlap(th); 1130 } 1131 } else { 1132 KMP_MB(); /* Flush all pending memory write invalidates. */ 1133 1134 /* Set stack size for this thread now. */ 1135 KA_TRACE(10, 1136 ("__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC " bytes\n", 1137 stack_size)); 1138 1139 stack_size += gtid * __kmp_stkoffset; 1140 1141 TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size); 1142 TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE); 1143 1144 KA_TRACE(10, 1145 ("__kmp_create_worker: (before) stack_size = %" KMP_SIZE_T_SPEC 1146 " bytes, &__kmp_launch_worker = %p, th = %p, &idThread = %p\n", 1147 (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker, 1148 (LPVOID)th, &idThread)); 1149 1150 handle = CreateThread( 1151 NULL, (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)__kmp_launch_worker, 1152 (LPVOID)th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread); 1153 1154 KA_TRACE(10, 1155 ("__kmp_create_worker: (after) stack_size = %" KMP_SIZE_T_SPEC 1156 " bytes, &__kmp_launch_worker = %p, th = %p, " 1157 "idThread = %u, handle = %" KMP_UINTPTR_SPEC "\n", 1158 (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker, 1159 (LPVOID)th, idThread, handle)); 1160 1161 if (handle == 0) { 1162 DWORD error = GetLastError(); 1163 __kmp_fatal(KMP_MSG(CantCreateThread), KMP_ERR(error), __kmp_msg_null); 1164 } else { 1165 th->th.th_info.ds.ds_thread = handle; 1166 } 1167 1168 KMP_MB(); /* Flush all pending memory write invalidates. */ 1169 } 1170 1171 KA_TRACE(10, ("__kmp_create_worker: done creating thread (%d)\n", gtid)); 1172 } 1173 1174 int __kmp_still_running(kmp_info_t *th) { 1175 return (WAIT_TIMEOUT == WaitForSingleObject(th->th.th_info.ds.ds_thread, 0)); 1176 } 1177 1178 #if KMP_USE_MONITOR 1179 void __kmp_create_monitor(kmp_info_t *th) { 1180 kmp_thread_t handle; 1181 DWORD idThread; 1182 int ideal, new_ideal; 1183 1184 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) { 1185 // We don't need monitor thread in case of MAX_BLOCKTIME 1186 KA_TRACE(10, ("__kmp_create_monitor: skipping monitor thread because of " 1187 "MAX blocktime\n")); 1188 th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op 1189 th->th.th_info.ds.ds_gtid = 0; 1190 TCW_4(__kmp_init_monitor, 2); // Signal to stop waiting for monitor creation 1191 return; 1192 } 1193 KA_TRACE(10, ("__kmp_create_monitor: try to create monitor\n")); 1194 1195 KMP_MB(); /* Flush all pending memory write invalidates. */ 1196 1197 __kmp_monitor_ev = CreateEvent(NULL, TRUE, FALSE, NULL); 1198 if (__kmp_monitor_ev == NULL) { 1199 DWORD error = GetLastError(); 1200 __kmp_fatal(KMP_MSG(CantCreateEvent), KMP_ERR(error), __kmp_msg_null); 1201 } 1202 #if USE_ITT_BUILD 1203 __kmp_itt_system_object_created(__kmp_monitor_ev, "Event"); 1204 #endif /* USE_ITT_BUILD */ 1205 1206 th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR; 1207 th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR; 1208 1209 // FIXME - on Windows* OS, if __kmp_monitor_stksize = 0, figure out how 1210 // to automatically expand stacksize based on CreateThread error code. 1211 if (__kmp_monitor_stksize == 0) { 1212 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE; 1213 } 1214 if (__kmp_monitor_stksize < __kmp_sys_min_stksize) { 1215 __kmp_monitor_stksize = __kmp_sys_min_stksize; 1216 } 1217 1218 KA_TRACE(10, ("__kmp_create_monitor: requested stacksize = %d bytes\n", 1219 (int)__kmp_monitor_stksize)); 1220 1221 TCW_4(__kmp_global.g.g_time.dt.t_value, 0); 1222 1223 handle = 1224 CreateThread(NULL, (SIZE_T)__kmp_monitor_stksize, 1225 (LPTHREAD_START_ROUTINE)__kmp_launch_monitor, (LPVOID)th, 1226 STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread); 1227 if (handle == 0) { 1228 DWORD error = GetLastError(); 1229 __kmp_fatal(KMP_MSG(CantCreateThread), KMP_ERR(error), __kmp_msg_null); 1230 } else 1231 th->th.th_info.ds.ds_thread = handle; 1232 1233 KMP_MB(); /* Flush all pending memory write invalidates. */ 1234 1235 KA_TRACE(10, ("__kmp_create_monitor: monitor created %p\n", 1236 (void *)th->th.th_info.ds.ds_thread)); 1237 } 1238 #endif 1239 1240 /* Check to see if thread is still alive. 1241 NOTE: The ExitProcess(code) system call causes all threads to Terminate 1242 with a exit_val = code. Because of this we can not rely on exit_val having 1243 any particular value. So this routine may return STILL_ALIVE in exit_val 1244 even after the thread is dead. */ 1245 1246 int __kmp_is_thread_alive(kmp_info_t *th, DWORD *exit_val) { 1247 DWORD rc; 1248 rc = GetExitCodeThread(th->th.th_info.ds.ds_thread, exit_val); 1249 if (rc == 0) { 1250 DWORD error = GetLastError(); 1251 __kmp_fatal(KMP_MSG(FunctionError, "GetExitCodeThread()"), KMP_ERR(error), 1252 __kmp_msg_null); 1253 } 1254 return (*exit_val == STILL_ACTIVE); 1255 } 1256 1257 void __kmp_exit_thread(int exit_status) { 1258 ExitThread(exit_status); 1259 } // __kmp_exit_thread 1260 1261 // This is a common part for both __kmp_reap_worker() and __kmp_reap_monitor(). 1262 static void __kmp_reap_common(kmp_info_t *th) { 1263 DWORD exit_val; 1264 1265 KMP_MB(); /* Flush all pending memory write invalidates. */ 1266 1267 KA_TRACE( 1268 10, ("__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid)); 1269 1270 /* 2006-10-19: 1271 There are two opposite situations: 1272 1. Windows* OS keep thread alive after it resets ds_alive flag and 1273 exits from thread function. (For example, see C70770/Q394281 "unloading of 1274 dll based on OMP is very slow".) 1275 2. Windows* OS may kill thread before it resets ds_alive flag. 1276 1277 Right solution seems to be waiting for *either* thread termination *or* 1278 ds_alive resetting. */ 1279 { 1280 // TODO: This code is very similar to KMP_WAIT. Need to generalize 1281 // KMP_WAIT to cover this usage also. 1282 void *obj = NULL; 1283 kmp_uint32 spins; 1284 #if USE_ITT_BUILD 1285 KMP_FSYNC_SPIN_INIT(obj, (void *)&th->th.th_info.ds.ds_alive); 1286 #endif /* USE_ITT_BUILD */ 1287 KMP_INIT_YIELD(spins); 1288 do { 1289 #if USE_ITT_BUILD 1290 KMP_FSYNC_SPIN_PREPARE(obj); 1291 #endif /* USE_ITT_BUILD */ 1292 __kmp_is_thread_alive(th, &exit_val); 1293 KMP_YIELD_OVERSUB_ELSE_SPIN(spins); 1294 } while (exit_val == STILL_ACTIVE && TCR_4(th->th.th_info.ds.ds_alive)); 1295 #if USE_ITT_BUILD 1296 if (exit_val == STILL_ACTIVE) { 1297 KMP_FSYNC_CANCEL(obj); 1298 } else { 1299 KMP_FSYNC_SPIN_ACQUIRED(obj); 1300 } 1301 #endif /* USE_ITT_BUILD */ 1302 } 1303 1304 __kmp_free_handle(th->th.th_info.ds.ds_thread); 1305 1306 /* NOTE: The ExitProcess(code) system call causes all threads to Terminate 1307 with a exit_val = code. Because of this we can not rely on exit_val having 1308 any particular value. */ 1309 if (exit_val == STILL_ACTIVE) { 1310 KA_TRACE(1, ("__kmp_reap_common: thread still active.\n")); 1311 } else if ((void *)exit_val != (void *)th) { 1312 KA_TRACE(1, ("__kmp_reap_common: ExitProcess / TerminateThread used?\n")); 1313 } 1314 1315 KA_TRACE(10, 1316 ("__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC 1317 "\n", 1318 th->th.th_info.ds.ds_gtid, th->th.th_info.ds.ds_thread)); 1319 1320 th->th.th_info.ds.ds_thread = 0; 1321 th->th.th_info.ds.ds_tid = KMP_GTID_DNE; 1322 th->th.th_info.ds.ds_gtid = KMP_GTID_DNE; 1323 th->th.th_info.ds.ds_thread_id = 0; 1324 1325 KMP_MB(); /* Flush all pending memory write invalidates. */ 1326 } 1327 1328 #if KMP_USE_MONITOR 1329 void __kmp_reap_monitor(kmp_info_t *th) { 1330 int status; 1331 1332 KA_TRACE(10, ("__kmp_reap_monitor: try to reap %p\n", 1333 (void *)th->th.th_info.ds.ds_thread)); 1334 1335 // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR. 1336 // If both tid and gtid are 0, it means the monitor did not ever start. 1337 // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down. 1338 KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid); 1339 if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) { 1340 KA_TRACE(10, ("__kmp_reap_monitor: monitor did not start, returning\n")); 1341 return; 1342 } 1343 1344 KMP_MB(); /* Flush all pending memory write invalidates. */ 1345 1346 status = SetEvent(__kmp_monitor_ev); 1347 if (status == FALSE) { 1348 DWORD error = GetLastError(); 1349 __kmp_fatal(KMP_MSG(CantSetEvent), KMP_ERR(error), __kmp_msg_null); 1350 } 1351 KA_TRACE(10, ("__kmp_reap_monitor: reaping thread (%d)\n", 1352 th->th.th_info.ds.ds_gtid)); 1353 __kmp_reap_common(th); 1354 1355 __kmp_free_handle(__kmp_monitor_ev); 1356 1357 KMP_MB(); /* Flush all pending memory write invalidates. */ 1358 } 1359 #endif 1360 1361 void __kmp_reap_worker(kmp_info_t *th) { 1362 KA_TRACE(10, ("__kmp_reap_worker: reaping thread (%d)\n", 1363 th->th.th_info.ds.ds_gtid)); 1364 __kmp_reap_common(th); 1365 } 1366 1367 #if KMP_HANDLE_SIGNALS 1368 1369 static void __kmp_team_handler(int signo) { 1370 if (__kmp_global.g.g_abort == 0) { 1371 // Stage 1 signal handler, let's shut down all of the threads. 1372 if (__kmp_debug_buf) { 1373 __kmp_dump_debug_buffer(); 1374 } 1375 KMP_MB(); // Flush all pending memory write invalidates. 1376 TCW_4(__kmp_global.g.g_abort, signo); 1377 KMP_MB(); // Flush all pending memory write invalidates. 1378 TCW_4(__kmp_global.g.g_done, TRUE); 1379 KMP_MB(); // Flush all pending memory write invalidates. 1380 } 1381 } // __kmp_team_handler 1382 1383 static sig_func_t __kmp_signal(int signum, sig_func_t handler) { 1384 sig_func_t old = signal(signum, handler); 1385 if (old == SIG_ERR) { 1386 int error = errno; 1387 __kmp_fatal(KMP_MSG(FunctionError, "signal"), KMP_ERR(error), 1388 __kmp_msg_null); 1389 } 1390 return old; 1391 } 1392 1393 static void __kmp_install_one_handler(int sig, sig_func_t handler, 1394 int parallel_init) { 1395 sig_func_t old; 1396 KMP_MB(); /* Flush all pending memory write invalidates. */ 1397 KB_TRACE(60, ("__kmp_install_one_handler: called: sig=%d\n", sig)); 1398 if (parallel_init) { 1399 old = __kmp_signal(sig, handler); 1400 // SIG_DFL on Windows* OS in NULL or 0. 1401 if (old == __kmp_sighldrs[sig]) { 1402 __kmp_siginstalled[sig] = 1; 1403 } else { // Restore/keep user's handler if one previously installed. 1404 old = __kmp_signal(sig, old); 1405 } 1406 } else { 1407 // Save initial/system signal handlers to see if user handlers installed. 1408 // 2009-09-23: It is a dead code. On Windows* OS __kmp_install_signals 1409 // called once with parallel_init == TRUE. 1410 old = __kmp_signal(sig, SIG_DFL); 1411 __kmp_sighldrs[sig] = old; 1412 __kmp_signal(sig, old); 1413 } 1414 KMP_MB(); /* Flush all pending memory write invalidates. */ 1415 } // __kmp_install_one_handler 1416 1417 static void __kmp_remove_one_handler(int sig) { 1418 if (__kmp_siginstalled[sig]) { 1419 sig_func_t old; 1420 KMP_MB(); // Flush all pending memory write invalidates. 1421 KB_TRACE(60, ("__kmp_remove_one_handler: called: sig=%d\n", sig)); 1422 old = __kmp_signal(sig, __kmp_sighldrs[sig]); 1423 if (old != __kmp_team_handler) { 1424 KB_TRACE(10, ("__kmp_remove_one_handler: oops, not our handler, " 1425 "restoring: sig=%d\n", 1426 sig)); 1427 old = __kmp_signal(sig, old); 1428 } 1429 __kmp_sighldrs[sig] = NULL; 1430 __kmp_siginstalled[sig] = 0; 1431 KMP_MB(); // Flush all pending memory write invalidates. 1432 } 1433 } // __kmp_remove_one_handler 1434 1435 void __kmp_install_signals(int parallel_init) { 1436 KB_TRACE(10, ("__kmp_install_signals: called\n")); 1437 if (!__kmp_handle_signals) { 1438 KB_TRACE(10, ("__kmp_install_signals: KMP_HANDLE_SIGNALS is false - " 1439 "handlers not installed\n")); 1440 return; 1441 } 1442 __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init); 1443 __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init); 1444 __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init); 1445 __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init); 1446 __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init); 1447 __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init); 1448 } // __kmp_install_signals 1449 1450 void __kmp_remove_signals(void) { 1451 int sig; 1452 KB_TRACE(10, ("__kmp_remove_signals: called\n")); 1453 for (sig = 1; sig < NSIG; ++sig) { 1454 __kmp_remove_one_handler(sig); 1455 } 1456 } // __kmp_remove_signals 1457 1458 #endif // KMP_HANDLE_SIGNALS 1459 1460 /* Put the thread to sleep for a time period */ 1461 void __kmp_thread_sleep(int millis) { 1462 DWORD status; 1463 1464 status = SleepEx((DWORD)millis, FALSE); 1465 if (status) { 1466 DWORD error = GetLastError(); 1467 __kmp_fatal(KMP_MSG(FunctionError, "SleepEx()"), KMP_ERR(error), 1468 __kmp_msg_null); 1469 } 1470 } 1471 1472 // Determine whether the given address is mapped into the current address space. 1473 int __kmp_is_address_mapped(void *addr) { 1474 DWORD status; 1475 MEMORY_BASIC_INFORMATION lpBuffer; 1476 SIZE_T dwLength; 1477 1478 dwLength = sizeof(MEMORY_BASIC_INFORMATION); 1479 1480 status = VirtualQuery(addr, &lpBuffer, dwLength); 1481 1482 return !(((lpBuffer.State == MEM_RESERVE) || (lpBuffer.State == MEM_FREE)) || 1483 ((lpBuffer.Protect == PAGE_NOACCESS) || 1484 (lpBuffer.Protect == PAGE_EXECUTE))); 1485 } 1486 1487 kmp_uint64 __kmp_hardware_timestamp(void) { 1488 kmp_uint64 r = 0; 1489 1490 QueryPerformanceCounter((LARGE_INTEGER *)&r); 1491 return r; 1492 } 1493 1494 /* Free handle and check the error code */ 1495 void __kmp_free_handle(kmp_thread_t tHandle) { 1496 /* called with parameter type HANDLE also, thus suppose kmp_thread_t defined 1497 * as HANDLE */ 1498 BOOL rc; 1499 rc = CloseHandle(tHandle); 1500 if (!rc) { 1501 DWORD error = GetLastError(); 1502 __kmp_fatal(KMP_MSG(CantCloseHandle), KMP_ERR(error), __kmp_msg_null); 1503 } 1504 } 1505 1506 int __kmp_get_load_balance(int max) { 1507 static ULONG glb_buff_size = 100 * 1024; 1508 1509 // Saved count of the running threads for the thread balance algortihm 1510 static int glb_running_threads = 0; 1511 static double glb_call_time = 0; /* Thread balance algorithm call time */ 1512 1513 int running_threads = 0; // Number of running threads in the system. 1514 NTSTATUS status = 0; 1515 ULONG buff_size = 0; 1516 ULONG info_size = 0; 1517 void *buffer = NULL; 1518 PSYSTEM_PROCESS_INFORMATION spi = NULL; 1519 int first_time = 1; 1520 1521 double call_time = 0.0; // start, finish; 1522 1523 __kmp_elapsed(&call_time); 1524 1525 if (glb_call_time && 1526 (call_time - glb_call_time < __kmp_load_balance_interval)) { 1527 running_threads = glb_running_threads; 1528 goto finish; 1529 } 1530 glb_call_time = call_time; 1531 1532 // Do not spend time on running algorithm if we have a permanent error. 1533 if (NtQuerySystemInformation == NULL) { 1534 running_threads = -1; 1535 goto finish; 1536 } 1537 1538 if (max <= 0) { 1539 max = INT_MAX; 1540 } 1541 1542 do { 1543 1544 if (first_time) { 1545 buff_size = glb_buff_size; 1546 } else { 1547 buff_size = 2 * buff_size; 1548 } 1549 1550 buffer = KMP_INTERNAL_REALLOC(buffer, buff_size); 1551 if (buffer == NULL) { 1552 running_threads = -1; 1553 goto finish; 1554 } 1555 status = NtQuerySystemInformation(SystemProcessInformation, buffer, 1556 buff_size, &info_size); 1557 first_time = 0; 1558 1559 } while (status == STATUS_INFO_LENGTH_MISMATCH); 1560 glb_buff_size = buff_size; 1561 1562 #define CHECK(cond) \ 1563 { \ 1564 KMP_DEBUG_ASSERT(cond); \ 1565 if (!(cond)) { \ 1566 running_threads = -1; \ 1567 goto finish; \ 1568 } \ 1569 } 1570 1571 CHECK(buff_size >= info_size); 1572 spi = PSYSTEM_PROCESS_INFORMATION(buffer); 1573 for (;;) { 1574 ptrdiff_t offset = uintptr_t(spi) - uintptr_t(buffer); 1575 CHECK(0 <= offset && 1576 offset + sizeof(SYSTEM_PROCESS_INFORMATION) < info_size); 1577 HANDLE pid = spi->ProcessId; 1578 ULONG num = spi->NumberOfThreads; 1579 CHECK(num >= 1); 1580 size_t spi_size = 1581 sizeof(SYSTEM_PROCESS_INFORMATION) + sizeof(SYSTEM_THREAD) * (num - 1); 1582 CHECK(offset + spi_size < 1583 info_size); // Make sure process info record fits the buffer. 1584 if (spi->NextEntryOffset != 0) { 1585 CHECK(spi_size <= 1586 spi->NextEntryOffset); // And do not overlap with the next record. 1587 } 1588 // pid == 0 corresponds to the System Idle Process. It always has running 1589 // threads on all cores. So, we don't consider the running threads of this 1590 // process. 1591 if (pid != 0) { 1592 for (int i = 0; i < num; ++i) { 1593 THREAD_STATE state = spi->Threads[i].State; 1594 // Count threads that have Ready or Running state. 1595 // !!! TODO: Why comment does not match the code??? 1596 if (state == StateRunning) { 1597 ++running_threads; 1598 // Stop counting running threads if the number is already greater than 1599 // the number of available cores 1600 if (running_threads >= max) { 1601 goto finish; 1602 } 1603 } 1604 } 1605 } 1606 if (spi->NextEntryOffset == 0) { 1607 break; 1608 } 1609 spi = PSYSTEM_PROCESS_INFORMATION(uintptr_t(spi) + spi->NextEntryOffset); 1610 } 1611 1612 #undef CHECK 1613 1614 finish: // Clean up and exit. 1615 1616 if (buffer != NULL) { 1617 KMP_INTERNAL_FREE(buffer); 1618 } 1619 1620 glb_running_threads = running_threads; 1621 1622 return running_threads; 1623 } //__kmp_get_load_balance() 1624