1 /* 2 * z_Windows_NT_util.cpp -- platform specific routines. 3 */ 4 5 6 //===----------------------------------------------------------------------===// 7 // 8 // The LLVM Compiler Infrastructure 9 // 10 // This file is dual licensed under the MIT and the University of Illinois Open 11 // Source Licenses. See LICENSE.txt for details. 12 // 13 //===----------------------------------------------------------------------===// 14 15 16 #include "kmp.h" 17 #include "kmp_itt.h" 18 #include "kmp_i18n.h" 19 #include "kmp_io.h" 20 #include "kmp_wait_release.h" 21 #include "kmp_affinity.h" 22 23 /* This code is related to NtQuerySystemInformation() function. This function 24 is used in the Load balance algorithm for OMP_DYNAMIC=true to find the 25 number of running threads in the system. */ 26 27 #include <ntstatus.h> 28 #include <ntsecapi.h> // UNICODE_STRING 29 30 enum SYSTEM_INFORMATION_CLASS { 31 SystemProcessInformation = 5 32 }; // SYSTEM_INFORMATION_CLASS 33 34 struct CLIENT_ID { 35 HANDLE UniqueProcess; 36 HANDLE UniqueThread; 37 }; // struct CLIENT_ID 38 39 enum THREAD_STATE { 40 StateInitialized, 41 StateReady, 42 StateRunning, 43 StateStandby, 44 StateTerminated, 45 StateWait, 46 StateTransition, 47 StateUnknown 48 }; // enum THREAD_STATE 49 50 struct VM_COUNTERS { 51 SIZE_T PeakVirtualSize; 52 SIZE_T VirtualSize; 53 ULONG PageFaultCount; 54 SIZE_T PeakWorkingSetSize; 55 SIZE_T WorkingSetSize; 56 SIZE_T QuotaPeakPagedPoolUsage; 57 SIZE_T QuotaPagedPoolUsage; 58 SIZE_T QuotaPeakNonPagedPoolUsage; 59 SIZE_T QuotaNonPagedPoolUsage; 60 SIZE_T PagefileUsage; 61 SIZE_T PeakPagefileUsage; 62 SIZE_T PrivatePageCount; 63 }; // struct VM_COUNTERS 64 65 struct SYSTEM_THREAD { 66 LARGE_INTEGER KernelTime; 67 LARGE_INTEGER UserTime; 68 LARGE_INTEGER CreateTime; 69 ULONG WaitTime; 70 LPVOID StartAddress; 71 CLIENT_ID ClientId; 72 DWORD Priority; 73 LONG BasePriority; 74 ULONG ContextSwitchCount; 75 THREAD_STATE State; 76 ULONG WaitReason; 77 }; // SYSTEM_THREAD 78 79 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, KernelTime ) == 0 ); 80 #if KMP_ARCH_X86 81 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, StartAddress ) == 28 ); 82 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, State ) == 52 ); 83 #else 84 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, StartAddress ) == 32 ); 85 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, State ) == 68 ); 86 #endif 87 88 struct SYSTEM_PROCESS_INFORMATION { 89 ULONG NextEntryOffset; 90 ULONG NumberOfThreads; 91 LARGE_INTEGER Reserved[ 3 ]; 92 LARGE_INTEGER CreateTime; 93 LARGE_INTEGER UserTime; 94 LARGE_INTEGER KernelTime; 95 UNICODE_STRING ImageName; 96 DWORD BasePriority; 97 HANDLE ProcessId; 98 HANDLE ParentProcessId; 99 ULONG HandleCount; 100 ULONG Reserved2[ 2 ]; 101 VM_COUNTERS VMCounters; 102 IO_COUNTERS IOCounters; 103 SYSTEM_THREAD Threads[ 1 ]; 104 }; // SYSTEM_PROCESS_INFORMATION 105 typedef SYSTEM_PROCESS_INFORMATION * PSYSTEM_PROCESS_INFORMATION; 106 107 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, NextEntryOffset ) == 0 ); 108 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, CreateTime ) == 32 ); 109 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ImageName ) == 56 ); 110 #if KMP_ARCH_X86 111 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ProcessId ) == 68 ); 112 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, HandleCount ) == 76 ); 113 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, VMCounters ) == 88 ); 114 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, IOCounters ) == 136 ); 115 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, Threads ) == 184 ); 116 #else 117 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ProcessId ) == 80 ); 118 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, HandleCount ) == 96 ); 119 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, VMCounters ) == 112 ); 120 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, IOCounters ) == 208 ); 121 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, Threads ) == 256 ); 122 #endif 123 124 typedef NTSTATUS (NTAPI *NtQuerySystemInformation_t)( SYSTEM_INFORMATION_CLASS, PVOID, ULONG, PULONG ); 125 NtQuerySystemInformation_t NtQuerySystemInformation = NULL; 126 127 HMODULE ntdll = NULL; 128 129 /* End of NtQuerySystemInformation()-related code */ 130 131 static HMODULE kernel32 = NULL; 132 133 /* ----------------------------------------------------------------------------------- */ 134 /* ----------------------------------------------------------------------------------- */ 135 136 #if KMP_HANDLE_SIGNALS 137 typedef void (* sig_func_t )( int ); 138 static sig_func_t __kmp_sighldrs[ NSIG ]; 139 static int __kmp_siginstalled[ NSIG ]; 140 #endif 141 142 #if KMP_USE_MONITOR 143 static HANDLE __kmp_monitor_ev; 144 #endif 145 static kmp_int64 __kmp_win32_time; 146 double __kmp_win32_tick; 147 148 int __kmp_init_runtime = FALSE; 149 CRITICAL_SECTION __kmp_win32_section; 150 151 void 152 __kmp_win32_mutex_init( kmp_win32_mutex_t *mx ) 153 { 154 InitializeCriticalSection( & mx->cs ); 155 #if USE_ITT_BUILD 156 __kmp_itt_system_object_created( & mx->cs, "Critical Section" ); 157 #endif /* USE_ITT_BUILD */ 158 } 159 160 void 161 __kmp_win32_mutex_destroy( kmp_win32_mutex_t *mx ) 162 { 163 DeleteCriticalSection( & mx->cs ); 164 } 165 166 void 167 __kmp_win32_mutex_lock( kmp_win32_mutex_t *mx ) 168 { 169 EnterCriticalSection( & mx->cs ); 170 } 171 172 void 173 __kmp_win32_mutex_unlock( kmp_win32_mutex_t *mx ) 174 { 175 LeaveCriticalSection( & mx->cs ); 176 } 177 178 void 179 __kmp_win32_cond_init( kmp_win32_cond_t *cv ) 180 { 181 cv->waiters_count_ = 0; 182 cv->wait_generation_count_ = 0; 183 cv->release_count_ = 0; 184 185 /* Initialize the critical section */ 186 __kmp_win32_mutex_init( & cv->waiters_count_lock_ ); 187 188 /* Create a manual-reset event. */ 189 cv->event_ = CreateEvent( NULL, // no security 190 TRUE, // manual-reset 191 FALSE, // non-signaled initially 192 NULL ); // unnamed 193 #if USE_ITT_BUILD 194 __kmp_itt_system_object_created( cv->event_, "Event" ); 195 #endif /* USE_ITT_BUILD */ 196 } 197 198 void 199 __kmp_win32_cond_destroy( kmp_win32_cond_t *cv ) 200 { 201 __kmp_win32_mutex_destroy( & cv->waiters_count_lock_ ); 202 __kmp_free_handle( cv->event_ ); 203 memset( cv, '\0', sizeof( *cv ) ); 204 } 205 206 /* TODO associate cv with a team instead of a thread so as to optimize 207 * the case where we wake up a whole team */ 208 209 void 210 __kmp_win32_cond_wait( kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx, kmp_info_t *th, int need_decrease_load ) 211 { 212 int my_generation; 213 int last_waiter; 214 215 /* Avoid race conditions */ 216 __kmp_win32_mutex_lock( &cv->waiters_count_lock_ ); 217 218 /* Increment count of waiters */ 219 cv->waiters_count_++; 220 221 /* Store current generation in our activation record. */ 222 my_generation = cv->wait_generation_count_; 223 224 __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ ); 225 __kmp_win32_mutex_unlock( mx ); 226 227 for (;;) { 228 int wait_done; 229 230 /* Wait until the event is signaled */ 231 WaitForSingleObject( cv->event_, INFINITE ); 232 233 __kmp_win32_mutex_lock( &cv->waiters_count_lock_ ); 234 235 /* Exit the loop when the <cv->event_> is signaled and 236 * there are still waiting threads from this <wait_generation> 237 * that haven't been released from this wait yet. */ 238 wait_done = ( cv->release_count_ > 0 ) && 239 ( cv->wait_generation_count_ != my_generation ); 240 241 __kmp_win32_mutex_unlock( &cv->waiters_count_lock_); 242 243 /* there used to be a semicolon after the if statement, 244 * it looked like a bug, so i removed it */ 245 if( wait_done ) 246 break; 247 } 248 249 __kmp_win32_mutex_lock( mx ); 250 __kmp_win32_mutex_lock( &cv->waiters_count_lock_ ); 251 252 cv->waiters_count_--; 253 cv->release_count_--; 254 255 last_waiter = ( cv->release_count_ == 0 ); 256 257 __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ ); 258 259 if( last_waiter ) { 260 /* We're the last waiter to be notified, so reset the manual event. */ 261 ResetEvent( cv->event_ ); 262 } 263 } 264 265 void 266 __kmp_win32_cond_broadcast( kmp_win32_cond_t *cv ) 267 { 268 __kmp_win32_mutex_lock( &cv->waiters_count_lock_ ); 269 270 if( cv->waiters_count_ > 0 ) { 271 SetEvent( cv->event_ ); 272 /* Release all the threads in this generation. */ 273 274 cv->release_count_ = cv->waiters_count_; 275 276 /* Start a new generation. */ 277 cv->wait_generation_count_++; 278 } 279 280 __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ ); 281 } 282 283 void 284 __kmp_win32_cond_signal( kmp_win32_cond_t *cv ) 285 { 286 __kmp_win32_cond_broadcast( cv ); 287 } 288 289 /* ------------------------------------------------------------------------ */ 290 /* ------------------------------------------------------------------------ */ 291 292 void 293 __kmp_enable( int new_state ) 294 { 295 if (__kmp_init_runtime) 296 LeaveCriticalSection( & __kmp_win32_section ); 297 } 298 299 void 300 __kmp_disable( int *old_state ) 301 { 302 *old_state = 0; 303 304 if (__kmp_init_runtime) 305 EnterCriticalSection( & __kmp_win32_section ); 306 } 307 308 void 309 __kmp_suspend_initialize( void ) 310 { 311 /* do nothing */ 312 } 313 314 static void 315 __kmp_suspend_initialize_thread( kmp_info_t *th ) 316 { 317 if ( ! TCR_4( th->th.th_suspend_init ) ) { 318 /* this means we haven't initialized the suspension pthread objects for this thread 319 in this instance of the process */ 320 __kmp_win32_cond_init( &th->th.th_suspend_cv ); 321 __kmp_win32_mutex_init( &th->th.th_suspend_mx ); 322 TCW_4( th->th.th_suspend_init, TRUE ); 323 } 324 } 325 326 void 327 __kmp_suspend_uninitialize_thread( kmp_info_t *th ) 328 { 329 if ( TCR_4( th->th.th_suspend_init ) ) { 330 /* this means we have initialize the suspension pthread objects for this thread 331 in this instance of the process */ 332 __kmp_win32_cond_destroy( & th->th.th_suspend_cv ); 333 __kmp_win32_mutex_destroy( & th->th.th_suspend_mx ); 334 TCW_4( th->th.th_suspend_init, FALSE ); 335 } 336 } 337 338 /* This routine puts the calling thread to sleep after setting the 339 * sleep bit for the indicated flag variable to true. 340 */ 341 template <class C> 342 static inline void __kmp_suspend_template( int th_gtid, C *flag ) 343 { 344 kmp_info_t *th = __kmp_threads[th_gtid]; 345 int status; 346 typename C::flag_t old_spin; 347 348 KF_TRACE( 30, ("__kmp_suspend_template: T#%d enter for flag's loc(%p)\n", th_gtid, flag->get() ) ); 349 350 __kmp_suspend_initialize_thread( th ); 351 __kmp_win32_mutex_lock( &th->th.th_suspend_mx ); 352 353 KF_TRACE( 10, ( "__kmp_suspend_template: T#%d setting sleep bit for flag's loc(%p)\n", 354 th_gtid, flag->get() ) ); 355 356 /* TODO: shouldn't this use release semantics to ensure that __kmp_suspend_initialize_thread 357 gets called first? 358 */ 359 old_spin = flag->set_sleeping(); 360 361 KF_TRACE( 5, ( "__kmp_suspend_template: T#%d set sleep bit for flag's loc(%p)==%d\n", 362 th_gtid, flag->get(), *(flag->get()) ) ); 363 364 if ( flag->done_check_val(old_spin) ) { 365 old_spin = flag->unset_sleeping(); 366 KF_TRACE( 5, ( "__kmp_suspend_template: T#%d false alarm, reset sleep bit for flag's loc(%p)\n", 367 th_gtid, flag->get()) ); 368 } else { 369 #ifdef DEBUG_SUSPEND 370 __kmp_suspend_count++; 371 #endif 372 /* Encapsulate in a loop as the documentation states that this may 373 * "with low probability" return when the condition variable has 374 * not been signaled or broadcast 375 */ 376 int deactivated = FALSE; 377 TCW_PTR(th->th.th_sleep_loc, (void *)flag); 378 while ( flag->is_sleeping() ) { 379 KF_TRACE( 15, ("__kmp_suspend_template: T#%d about to perform kmp_win32_cond_wait()\n", 380 th_gtid ) ); 381 // Mark the thread as no longer active (only in the first iteration of the loop). 382 if ( ! deactivated ) { 383 th->th.th_active = FALSE; 384 if ( th->th.th_active_in_pool ) { 385 th->th.th_active_in_pool = FALSE; 386 KMP_TEST_THEN_DEC32( 387 (kmp_int32 *) &__kmp_thread_pool_active_nth ); 388 KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 ); 389 } 390 deactivated = TRUE; 391 392 __kmp_win32_cond_wait( &th->th.th_suspend_cv, &th->th.th_suspend_mx, 0, 0 ); 393 } 394 else { 395 __kmp_win32_cond_wait( &th->th.th_suspend_cv, &th->th.th_suspend_mx, 0, 0 ); 396 } 397 398 #ifdef KMP_DEBUG 399 if( flag->is_sleeping() ) { 400 KF_TRACE( 100, ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid )); 401 } 402 #endif /* KMP_DEBUG */ 403 404 } // while 405 406 // Mark the thread as active again (if it was previous marked as inactive) 407 if ( deactivated ) { 408 th->th.th_active = TRUE; 409 if ( TCR_4(th->th.th_in_pool) ) { 410 KMP_TEST_THEN_INC32( 411 (kmp_int32 *) &__kmp_thread_pool_active_nth ); 412 th->th.th_active_in_pool = TRUE; 413 } 414 } 415 } 416 417 __kmp_win32_mutex_unlock( &th->th.th_suspend_mx ); 418 419 KF_TRACE( 30, ("__kmp_suspend_template: T#%d exit\n", th_gtid ) ); 420 } 421 422 void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) { 423 __kmp_suspend_template(th_gtid, flag); 424 } 425 void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) { 426 __kmp_suspend_template(th_gtid, flag); 427 } 428 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) { 429 __kmp_suspend_template(th_gtid, flag); 430 } 431 432 433 /* This routine signals the thread specified by target_gtid to wake up 434 * after setting the sleep bit indicated by the flag argument to FALSE 435 */ 436 template <class C> 437 static inline void __kmp_resume_template( int target_gtid, C *flag ) 438 { 439 kmp_info_t *th = __kmp_threads[target_gtid]; 440 int status; 441 442 #ifdef KMP_DEBUG 443 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1; 444 #endif 445 446 KF_TRACE( 30, ( "__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", gtid, target_gtid ) ); 447 448 __kmp_suspend_initialize_thread( th ); 449 __kmp_win32_mutex_lock( &th->th.th_suspend_mx ); 450 451 if (!flag) { // coming from __kmp_null_resume_wrapper 452 flag = (C *)th->th.th_sleep_loc; 453 } 454 455 // First, check if the flag is null or its type has changed. If so, someone else woke it up. 456 if (!flag || flag->get_type() != flag->get_ptr_type()) { // get_ptr_type simply shows what flag was cast to 457 KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag's loc(%p)\n", 458 gtid, target_gtid, NULL ) ); 459 __kmp_win32_mutex_unlock( &th->th.th_suspend_mx ); 460 return; 461 } 462 else { 463 typename C::flag_t old_spin = flag->unset_sleeping(); 464 if ( !flag->is_sleeping_val(old_spin) ) { 465 KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag's loc(%p): " 466 "%u => %u\n", 467 gtid, target_gtid, flag->get(), old_spin, *(flag->get()) ) ); 468 __kmp_win32_mutex_unlock( &th->th.th_suspend_mx ); 469 return; 470 } 471 } 472 TCW_PTR(th->th.th_sleep_loc, NULL); 473 474 KF_TRACE( 5, ( "__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep bit for flag's loc(%p)\n", 475 gtid, target_gtid, flag->get() ) ); 476 477 __kmp_win32_cond_signal( &th->th.th_suspend_cv ); 478 __kmp_win32_mutex_unlock( &th->th.th_suspend_mx ); 479 480 KF_TRACE( 30, ( "__kmp_resume_template: T#%d exiting after signaling wake up for T#%d\n", 481 gtid, target_gtid ) ); 482 } 483 484 void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) { 485 __kmp_resume_template(target_gtid, flag); 486 } 487 void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) { 488 __kmp_resume_template(target_gtid, flag); 489 } 490 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) { 491 __kmp_resume_template(target_gtid, flag); 492 } 493 494 495 /* ------------------------------------------------------------------------ */ 496 /* ------------------------------------------------------------------------ */ 497 498 void 499 __kmp_yield( int cond ) 500 { 501 if (cond) 502 Sleep(0); 503 } 504 505 /* ------------------------------------------------------------------------ */ 506 /* ------------------------------------------------------------------------ */ 507 508 void 509 __kmp_gtid_set_specific( int gtid ) 510 { 511 if( __kmp_init_gtid ) { 512 KA_TRACE( 50, ("__kmp_gtid_set_specific: T#%d key:%d\n", 513 gtid, __kmp_gtid_threadprivate_key )); 514 if( ! TlsSetValue( __kmp_gtid_threadprivate_key, (LPVOID)(gtid+1)) ) 515 KMP_FATAL( TLSSetValueFailed ); 516 } else { 517 KA_TRACE( 50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n" ) ); 518 } 519 } 520 521 int 522 __kmp_gtid_get_specific() 523 { 524 int gtid; 525 if( !__kmp_init_gtid ) { 526 KA_TRACE( 50, ("__kmp_gtid_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) ); 527 return KMP_GTID_SHUTDOWN; 528 } 529 gtid = (int)(kmp_intptr_t)TlsGetValue( __kmp_gtid_threadprivate_key ); 530 if ( gtid == 0 ) { 531 gtid = KMP_GTID_DNE; 532 } 533 else { 534 gtid--; 535 } 536 KA_TRACE( 50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n", 537 __kmp_gtid_threadprivate_key, gtid )); 538 return gtid; 539 } 540 541 /* ------------------------------------------------------------------------ */ 542 /* ------------------------------------------------------------------------ */ 543 544 void 545 __kmp_affinity_bind_thread( int proc ) 546 { 547 if (__kmp_num_proc_groups > 1) { 548 // 549 // Form the GROUP_AFFINITY struct directly, rather than filling 550 // out a bit vector and calling __kmp_set_system_affinity(). 551 // 552 GROUP_AFFINITY ga; 553 KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups 554 * CHAR_BIT * sizeof(DWORD_PTR)))); 555 ga.Group = proc / (CHAR_BIT * sizeof(DWORD_PTR)); 556 ga.Mask = (unsigned long long)1 << (proc % (CHAR_BIT * sizeof(DWORD_PTR))); 557 ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0; 558 559 KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL); 560 if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) { 561 DWORD error = GetLastError(); 562 if (__kmp_affinity_verbose) { // AC: continue silently if not verbose 563 kmp_msg_t err_code = KMP_ERR( error ); 564 __kmp_msg( 565 kmp_ms_warning, 566 KMP_MSG( CantSetThreadAffMask ), 567 err_code, 568 __kmp_msg_null 569 ); 570 if (__kmp_generate_warnings == kmp_warnings_off) { 571 __kmp_str_free(&err_code.str); 572 } 573 } 574 } 575 } else { 576 kmp_affin_mask_t *mask; 577 KMP_CPU_ALLOC_ON_STACK(mask); 578 KMP_CPU_ZERO(mask); 579 KMP_CPU_SET(proc, mask); 580 __kmp_set_system_affinity(mask, TRUE); 581 KMP_CPU_FREE_FROM_STACK(mask); 582 } 583 } 584 585 void 586 __kmp_affinity_determine_capable( const char *env_var ) 587 { 588 // 589 // All versions of Windows* OS (since Win '95) support SetThreadAffinityMask(). 590 // 591 592 #if KMP_GROUP_AFFINITY 593 KMP_AFFINITY_ENABLE(__kmp_num_proc_groups*sizeof(DWORD_PTR)); 594 #else 595 KMP_AFFINITY_ENABLE(sizeof(DWORD_PTR)); 596 #endif 597 598 KA_TRACE( 10, ( 599 "__kmp_affinity_determine_capable: " 600 "Windows* OS affinity interface functional (mask size = %" KMP_SIZE_T_SPEC ").\n", 601 __kmp_affin_mask_size 602 ) ); 603 } 604 605 double 606 __kmp_read_cpu_time( void ) 607 { 608 FILETIME CreationTime, ExitTime, KernelTime, UserTime; 609 int status; 610 double cpu_time; 611 612 cpu_time = 0; 613 614 status = GetProcessTimes( GetCurrentProcess(), &CreationTime, 615 &ExitTime, &KernelTime, &UserTime ); 616 617 if (status) { 618 double sec = 0; 619 620 sec += KernelTime.dwHighDateTime; 621 sec += UserTime.dwHighDateTime; 622 623 /* Shift left by 32 bits */ 624 sec *= (double) (1 << 16) * (double) (1 << 16); 625 626 sec += KernelTime.dwLowDateTime; 627 sec += UserTime.dwLowDateTime; 628 629 cpu_time += (sec * 100.0) / KMP_NSEC_PER_SEC; 630 } 631 632 return cpu_time; 633 } 634 635 int 636 __kmp_read_system_info( struct kmp_sys_info *info ) 637 { 638 info->maxrss = 0; /* the maximum resident set size utilized (in kilobytes) */ 639 info->minflt = 0; /* the number of page faults serviced without any I/O */ 640 info->majflt = 0; /* the number of page faults serviced that required I/O */ 641 info->nswap = 0; /* the number of times a process was "swapped" out of memory */ 642 info->inblock = 0; /* the number of times the file system had to perform input */ 643 info->oublock = 0; /* the number of times the file system had to perform output */ 644 info->nvcsw = 0; /* the number of times a context switch was voluntarily */ 645 info->nivcsw = 0; /* the number of times a context switch was forced */ 646 647 return 1; 648 } 649 650 /* ------------------------------------------------------------------------ */ 651 /* ------------------------------------------------------------------------ */ 652 653 654 void 655 __kmp_runtime_initialize( void ) 656 { 657 SYSTEM_INFO info; 658 kmp_str_buf_t path; 659 UINT path_size; 660 661 if ( __kmp_init_runtime ) { 662 return; 663 }; 664 665 #if KMP_DYNAMIC_LIB 666 /* Pin dynamic library for the lifetime of application */ 667 { 668 // First, turn off error message boxes 669 UINT err_mode = SetErrorMode (SEM_FAILCRITICALERRORS); 670 HMODULE h; 671 BOOL ret = GetModuleHandleEx( GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS 672 |GET_MODULE_HANDLE_EX_FLAG_PIN, 673 (LPCTSTR)&__kmp_serial_initialize, &h); 674 KMP_DEBUG_ASSERT2(h && ret, "OpenMP RTL cannot find itself loaded"); 675 SetErrorMode (err_mode); // Restore error mode 676 KA_TRACE( 10, ("__kmp_runtime_initialize: dynamic library pinned\n") ); 677 } 678 #endif 679 680 InitializeCriticalSection( & __kmp_win32_section ); 681 #if USE_ITT_BUILD 682 __kmp_itt_system_object_created( & __kmp_win32_section, "Critical Section" ); 683 #endif /* USE_ITT_BUILD */ 684 __kmp_initialize_system_tick(); 685 686 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) 687 if ( ! __kmp_cpuinfo.initialized ) { 688 __kmp_query_cpuid( & __kmp_cpuinfo ); 689 }; // if 690 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 691 692 /* Set up minimum number of threads to switch to TLS gtid */ 693 #if KMP_OS_WINDOWS && ! defined KMP_DYNAMIC_LIB 694 // Windows* OS, static library. 695 /* 696 New thread may use stack space previously used by another thread, currently terminated. 697 On Windows* OS, in case of static linking, we do not know the moment of thread termination, 698 and our structures (__kmp_threads and __kmp_root arrays) are still keep info about dead 699 threads. This leads to problem in __kmp_get_global_thread_id() function: it wrongly 700 finds gtid (by searching through stack addresses of all known threads) for unregistered 701 foreign tread. 702 703 Setting __kmp_tls_gtid_min to 0 workarounds this problem: __kmp_get_global_thread_id() 704 does not search through stacks, but get gtid from TLS immediately. 705 706 --ln 707 */ 708 __kmp_tls_gtid_min = 0; 709 #else 710 __kmp_tls_gtid_min = KMP_TLS_GTID_MIN; 711 #endif 712 713 /* for the static library */ 714 if ( !__kmp_gtid_threadprivate_key ) { 715 __kmp_gtid_threadprivate_key = TlsAlloc(); 716 if( __kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES ) { 717 KMP_FATAL( TLSOutOfIndexes ); 718 } 719 } 720 721 722 // 723 // Load ntdll.dll. 724 // 725 /* 726 Simple 727 GetModuleHandle( "ntdll.dl" ) 728 is not suitable due to security issue (see 729 http://www.microsoft.com/technet/security/advisory/2269637.mspx). We have to specify full 730 path to the library. 731 */ 732 __kmp_str_buf_init( & path ); 733 path_size = GetSystemDirectory( path.str, path.size ); 734 KMP_DEBUG_ASSERT( path_size > 0 ); 735 if ( path_size >= path.size ) { 736 // 737 // Buffer is too short. Expand the buffer and try again. 738 // 739 __kmp_str_buf_reserve( & path, path_size ); 740 path_size = GetSystemDirectory( path.str, path.size ); 741 KMP_DEBUG_ASSERT( path_size > 0 ); 742 }; // if 743 if ( path_size > 0 && path_size < path.size ) { 744 // 745 // Now we have system directory name in the buffer. 746 // Append backslash and name of dll to form full path, 747 // 748 path.used = path_size; 749 __kmp_str_buf_print( & path, "\\%s", "ntdll.dll" ); 750 751 // 752 // Now load ntdll using full path. 753 // 754 ntdll = GetModuleHandle( path.str ); 755 } 756 757 KMP_DEBUG_ASSERT( ntdll != NULL ); 758 if ( ntdll != NULL ) { 759 NtQuerySystemInformation = (NtQuerySystemInformation_t) GetProcAddress( ntdll, "NtQuerySystemInformation" ); 760 } 761 KMP_DEBUG_ASSERT( NtQuerySystemInformation != NULL ); 762 763 #if KMP_GROUP_AFFINITY 764 // 765 // Load kernel32.dll. 766 // Same caveat - must use full system path name. 767 // 768 if ( path_size > 0 && path_size < path.size ) { 769 // 770 // Truncate the buffer back to just the system path length, 771 // discarding "\\ntdll.dll", and replacing it with "kernel32.dll". 772 // 773 path.used = path_size; 774 __kmp_str_buf_print( & path, "\\%s", "kernel32.dll" ); 775 776 // 777 // Load kernel32.dll using full path. 778 // 779 kernel32 = GetModuleHandle( path.str ); 780 KA_TRACE( 10, ("__kmp_runtime_initialize: kernel32.dll = %s\n", path.str ) ); 781 782 // 783 // Load the function pointers to kernel32.dll routines 784 // that may or may not exist on this system. 785 // 786 if ( kernel32 != NULL ) { 787 __kmp_GetActiveProcessorCount = (kmp_GetActiveProcessorCount_t) GetProcAddress( kernel32, "GetActiveProcessorCount" ); 788 __kmp_GetActiveProcessorGroupCount = (kmp_GetActiveProcessorGroupCount_t) GetProcAddress( kernel32, "GetActiveProcessorGroupCount" ); 789 __kmp_GetThreadGroupAffinity = (kmp_GetThreadGroupAffinity_t) GetProcAddress( kernel32, "GetThreadGroupAffinity" ); 790 __kmp_SetThreadGroupAffinity = (kmp_SetThreadGroupAffinity_t) GetProcAddress( kernel32, "SetThreadGroupAffinity" ); 791 792 KA_TRACE( 10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorCount = %p\n", __kmp_GetActiveProcessorCount ) ); 793 KA_TRACE( 10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorGroupCount = %p\n", __kmp_GetActiveProcessorGroupCount ) ); 794 KA_TRACE( 10, ("__kmp_runtime_initialize:__kmp_GetThreadGroupAffinity = %p\n", __kmp_GetThreadGroupAffinity ) ); 795 KA_TRACE( 10, ("__kmp_runtime_initialize: __kmp_SetThreadGroupAffinity = %p\n", __kmp_SetThreadGroupAffinity ) ); 796 KA_TRACE( 10, ("__kmp_runtime_initialize: sizeof(kmp_affin_mask_t) = %d\n", sizeof(kmp_affin_mask_t) ) ); 797 798 // 799 // See if group affinity is supported on this system. 800 // If so, calculate the #groups and #procs. 801 // 802 // Group affinity was introduced with Windows* 7 OS and 803 // Windows* Server 2008 R2 OS. 804 // 805 if ( ( __kmp_GetActiveProcessorCount != NULL ) 806 && ( __kmp_GetActiveProcessorGroupCount != NULL ) 807 && ( __kmp_GetThreadGroupAffinity != NULL ) 808 && ( __kmp_SetThreadGroupAffinity != NULL ) 809 && ( ( __kmp_num_proc_groups 810 = __kmp_GetActiveProcessorGroupCount() ) > 1 ) ) { 811 // 812 // Calculate the total number of active OS procs. 813 // 814 int i; 815 816 KA_TRACE( 10, ("__kmp_runtime_initialize: %d processor groups detected\n", __kmp_num_proc_groups ) ); 817 818 __kmp_xproc = 0; 819 820 for ( i = 0; i < __kmp_num_proc_groups; i++ ) { 821 DWORD size = __kmp_GetActiveProcessorCount( i ); 822 __kmp_xproc += size; 823 KA_TRACE( 10, ("__kmp_runtime_initialize: proc group %d size = %d\n", i, size ) ); 824 } 825 } 826 else { 827 KA_TRACE( 10, ("__kmp_runtime_initialize: %d processor groups detected\n", __kmp_num_proc_groups ) ); 828 } 829 } 830 } 831 if ( __kmp_num_proc_groups <= 1 ) { 832 GetSystemInfo( & info ); 833 __kmp_xproc = info.dwNumberOfProcessors; 834 } 835 #else 836 GetSystemInfo( & info ); 837 __kmp_xproc = info.dwNumberOfProcessors; 838 #endif /* KMP_GROUP_AFFINITY */ 839 840 // 841 // If the OS said there were 0 procs, take a guess and use a value of 2. 842 // This is done for Linux* OS, also. Do we need error / warning? 843 // 844 if ( __kmp_xproc <= 0 ) { 845 __kmp_xproc = 2; 846 } 847 848 KA_TRACE( 5, ("__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc) ); 849 850 __kmp_str_buf_free( & path ); 851 852 #if USE_ITT_BUILD 853 __kmp_itt_initialize(); 854 #endif /* USE_ITT_BUILD */ 855 856 __kmp_init_runtime = TRUE; 857 } // __kmp_runtime_initialize 858 859 void 860 __kmp_runtime_destroy( void ) 861 { 862 if ( ! __kmp_init_runtime ) { 863 return; 864 } 865 866 #if USE_ITT_BUILD 867 __kmp_itt_destroy(); 868 #endif /* USE_ITT_BUILD */ 869 870 /* we can't DeleteCriticalsection( & __kmp_win32_section ); */ 871 /* due to the KX_TRACE() commands */ 872 KA_TRACE( 40, ("__kmp_runtime_destroy\n" )); 873 874 if( __kmp_gtid_threadprivate_key ) { 875 TlsFree( __kmp_gtid_threadprivate_key ); 876 __kmp_gtid_threadprivate_key = 0; 877 } 878 879 __kmp_affinity_uninitialize(); 880 DeleteCriticalSection( & __kmp_win32_section ); 881 882 ntdll = NULL; 883 NtQuerySystemInformation = NULL; 884 885 #if KMP_ARCH_X86_64 886 kernel32 = NULL; 887 __kmp_GetActiveProcessorCount = NULL; 888 __kmp_GetActiveProcessorGroupCount = NULL; 889 __kmp_GetThreadGroupAffinity = NULL; 890 __kmp_SetThreadGroupAffinity = NULL; 891 #endif // KMP_ARCH_X86_64 892 893 __kmp_init_runtime = FALSE; 894 } 895 896 897 void 898 __kmp_terminate_thread( int gtid ) 899 { 900 kmp_info_t *th = __kmp_threads[ gtid ]; 901 902 if( !th ) return; 903 904 KA_TRACE( 10, ("__kmp_terminate_thread: kill (%d)\n", gtid ) ); 905 906 if (TerminateThread( th->th.th_info.ds.ds_thread, (DWORD) -1) == FALSE) { 907 /* It's OK, the thread may have exited already */ 908 } 909 __kmp_free_handle( th->th.th_info.ds.ds_thread ); 910 } 911 912 /* ------------------------------------------------------------------------ */ 913 /* ------------------------------------------------------------------------ */ 914 915 void 916 __kmp_clear_system_time( void ) 917 { 918 BOOL status; 919 LARGE_INTEGER time; 920 status = QueryPerformanceCounter( & time ); 921 __kmp_win32_time = (kmp_int64) time.QuadPart; 922 } 923 924 void 925 __kmp_initialize_system_tick( void ) 926 { 927 { 928 BOOL status; 929 LARGE_INTEGER freq; 930 931 status = QueryPerformanceFrequency( & freq ); 932 if (! status) { 933 DWORD error = GetLastError(); 934 __kmp_msg( 935 kmp_ms_fatal, 936 KMP_MSG( FunctionError, "QueryPerformanceFrequency()" ), 937 KMP_ERR( error ), 938 __kmp_msg_null 939 ); 940 941 } 942 else { 943 __kmp_win32_tick = ((double) 1.0) / (double) freq.QuadPart; 944 } 945 } 946 } 947 948 /* Calculate the elapsed wall clock time for the user */ 949 950 void 951 __kmp_elapsed( double *t ) 952 { 953 BOOL status; 954 LARGE_INTEGER now; 955 status = QueryPerformanceCounter( & now ); 956 *t = ((double) now.QuadPart) * __kmp_win32_tick; 957 } 958 959 /* Calculate the elapsed wall clock tick for the user */ 960 961 void 962 __kmp_elapsed_tick( double *t ) 963 { 964 *t = __kmp_win32_tick; 965 } 966 967 void 968 __kmp_read_system_time( double *delta ) 969 { 970 if (delta != NULL) { 971 BOOL status; 972 LARGE_INTEGER now; 973 974 status = QueryPerformanceCounter( & now ); 975 976 *delta = ((double) (((kmp_int64) now.QuadPart) - __kmp_win32_time)) 977 * __kmp_win32_tick; 978 } 979 } 980 981 /* Return the current time stamp in nsec */ 982 kmp_uint64 983 __kmp_now_nsec() 984 { 985 LARGE_INTEGER now; 986 QueryPerformanceCounter(&now); 987 return 1e9 * __kmp_win32_tick * now.QuadPart; 988 } 989 990 /* ------------------------------------------------------------------------ */ 991 /* ------------------------------------------------------------------------ */ 992 993 void * __stdcall 994 __kmp_launch_worker( void *arg ) 995 { 996 volatile void *stack_data; 997 void *exit_val; 998 void *padding = 0; 999 kmp_info_t *this_thr = (kmp_info_t *) arg; 1000 int gtid; 1001 1002 gtid = this_thr->th.th_info.ds.ds_gtid; 1003 __kmp_gtid_set_specific( gtid ); 1004 #ifdef KMP_TDATA_GTID 1005 #error "This define causes problems with LoadLibrary() + declspec(thread) " \ 1006 "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \ 1007 "reference: http://support.microsoft.com/kb/118816" 1008 //__kmp_gtid = gtid; 1009 #endif 1010 1011 #if USE_ITT_BUILD 1012 __kmp_itt_thread_name( gtid ); 1013 #endif /* USE_ITT_BUILD */ 1014 1015 __kmp_affinity_set_init_mask( gtid, FALSE ); 1016 1017 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1018 // 1019 // Set the FP control regs to be a copy of 1020 // the parallel initialization thread's. 1021 // 1022 __kmp_clear_x87_fpu_status_word(); 1023 __kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word ); 1024 __kmp_load_mxcsr( &__kmp_init_mxcsr ); 1025 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 1026 1027 if ( __kmp_stkoffset > 0 && gtid > 0 ) { 1028 padding = KMP_ALLOCA( gtid * __kmp_stkoffset ); 1029 } 1030 1031 KMP_FSYNC_RELEASING( &this_thr -> th.th_info.ds.ds_alive ); 1032 this_thr -> th.th_info.ds.ds_thread_id = GetCurrentThreadId(); 1033 TCW_4( this_thr -> th.th_info.ds.ds_alive, TRUE ); 1034 1035 if ( TCR_4(__kmp_gtid_mode) < 2 ) { // check stack only if it is used to get gtid 1036 TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data); 1037 KMP_ASSERT( this_thr -> th.th_info.ds.ds_stackgrow == FALSE ); 1038 __kmp_check_stack_overlap( this_thr ); 1039 } 1040 KMP_MB(); 1041 exit_val = __kmp_launch_thread( this_thr ); 1042 KMP_FSYNC_RELEASING( &this_thr -> th.th_info.ds.ds_alive ); 1043 TCW_4( this_thr -> th.th_info.ds.ds_alive, FALSE ); 1044 KMP_MB(); 1045 return exit_val; 1046 } 1047 1048 #if KMP_USE_MONITOR 1049 /* The monitor thread controls all of the threads in the complex */ 1050 1051 void * __stdcall 1052 __kmp_launch_monitor( void *arg ) 1053 { 1054 DWORD wait_status; 1055 kmp_thread_t monitor; 1056 int status; 1057 int interval; 1058 kmp_info_t *this_thr = (kmp_info_t *) arg; 1059 1060 KMP_DEBUG_ASSERT(__kmp_init_monitor); 1061 TCW_4( __kmp_init_monitor, 2 ); // AC: Signal the library that monitor has started 1062 // TODO: hide "2" in enum (like {true,false,started}) 1063 this_thr -> th.th_info.ds.ds_thread_id = GetCurrentThreadId(); 1064 TCW_4( this_thr -> th.th_info.ds.ds_alive, TRUE ); 1065 1066 KMP_MB(); /* Flush all pending memory write invalidates. */ 1067 KA_TRACE( 10, ("__kmp_launch_monitor: launched\n" ) ); 1068 1069 monitor = GetCurrentThread(); 1070 1071 /* set thread priority */ 1072 status = SetThreadPriority( monitor, THREAD_PRIORITY_HIGHEST ); 1073 if (! status) { 1074 DWORD error = GetLastError(); 1075 __kmp_msg( 1076 kmp_ms_fatal, 1077 KMP_MSG( CantSetThreadPriority ), 1078 KMP_ERR( error ), 1079 __kmp_msg_null 1080 ); 1081 } 1082 1083 /* register us as monitor */ 1084 __kmp_gtid_set_specific( KMP_GTID_MONITOR ); 1085 #ifdef KMP_TDATA_GTID 1086 #error "This define causes problems with LoadLibrary() + declspec(thread) " \ 1087 "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \ 1088 "reference: http://support.microsoft.com/kb/118816" 1089 //__kmp_gtid = KMP_GTID_MONITOR; 1090 #endif 1091 1092 #if USE_ITT_BUILD 1093 __kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore monitor thread. 1094 #endif /* USE_ITT_BUILD */ 1095 1096 KMP_MB(); /* Flush all pending memory write invalidates. */ 1097 1098 interval = ( 1000 / __kmp_monitor_wakeups ); /* in milliseconds */ 1099 1100 while (! TCR_4(__kmp_global.g.g_done)) { 1101 /* This thread monitors the state of the system */ 1102 1103 KA_TRACE( 15, ( "__kmp_launch_monitor: update\n" ) ); 1104 1105 wait_status = WaitForSingleObject( __kmp_monitor_ev, interval ); 1106 1107 if (wait_status == WAIT_TIMEOUT) { 1108 TCW_4( __kmp_global.g.g_time.dt.t_value, 1109 TCR_4( __kmp_global.g.g_time.dt.t_value ) + 1 ); 1110 } 1111 1112 KMP_MB(); /* Flush all pending memory write invalidates. */ 1113 } 1114 1115 KA_TRACE( 10, ("__kmp_launch_monitor: finished\n" ) ); 1116 1117 status = SetThreadPriority( monitor, THREAD_PRIORITY_NORMAL ); 1118 if (! status) { 1119 DWORD error = GetLastError(); 1120 __kmp_msg( 1121 kmp_ms_fatal, 1122 KMP_MSG( CantSetThreadPriority ), 1123 KMP_ERR( error ), 1124 __kmp_msg_null 1125 ); 1126 } 1127 1128 if (__kmp_global.g.g_abort != 0) { 1129 /* now we need to terminate the worker threads */ 1130 /* the value of t_abort is the signal we caught */ 1131 1132 int gtid; 1133 1134 KA_TRACE( 10, ("__kmp_launch_monitor: terminate sig=%d\n", (__kmp_global.g.g_abort) ) ); 1135 1136 /* terminate the OpenMP worker threads */ 1137 /* TODO this is not valid for sibling threads!! 1138 * the uber master might not be 0 anymore.. */ 1139 for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid) 1140 __kmp_terminate_thread( gtid ); 1141 1142 __kmp_cleanup(); 1143 1144 Sleep( 0 ); 1145 1146 KA_TRACE( 10, ("__kmp_launch_monitor: raise sig=%d\n", (__kmp_global.g.g_abort) ) ); 1147 1148 if (__kmp_global.g.g_abort > 0) { 1149 raise( __kmp_global.g.g_abort ); 1150 } 1151 } 1152 1153 TCW_4( this_thr -> th.th_info.ds.ds_alive, FALSE ); 1154 1155 KMP_MB(); 1156 return arg; 1157 } 1158 #endif 1159 1160 void 1161 __kmp_create_worker( int gtid, kmp_info_t *th, size_t stack_size ) 1162 { 1163 kmp_thread_t handle; 1164 DWORD idThread; 1165 1166 KA_TRACE( 10, ("__kmp_create_worker: try to create thread (%d)\n", gtid ) ); 1167 1168 th->th.th_info.ds.ds_gtid = gtid; 1169 1170 if ( KMP_UBER_GTID(gtid) ) { 1171 int stack_data; 1172 1173 /* TODO: GetCurrentThread() returns a pseudo-handle that is unsuitable for other threads to use. 1174 Is it appropriate to just use GetCurrentThread? When should we close this handle? When 1175 unregistering the root? 1176 */ 1177 { 1178 BOOL rc; 1179 rc = DuplicateHandle( 1180 GetCurrentProcess(), 1181 GetCurrentThread(), 1182 GetCurrentProcess(), 1183 &th->th.th_info.ds.ds_thread, 1184 0, 1185 FALSE, 1186 DUPLICATE_SAME_ACCESS 1187 ); 1188 KMP_ASSERT( rc ); 1189 KA_TRACE( 10, (" __kmp_create_worker: ROOT Handle duplicated, th = %p, handle = %" KMP_UINTPTR_SPEC "\n", 1190 (LPVOID)th, 1191 th->th.th_info.ds.ds_thread ) ); 1192 th->th.th_info.ds.ds_thread_id = GetCurrentThreadId(); 1193 } 1194 if ( TCR_4(__kmp_gtid_mode) < 2 ) { // check stack only if it is used to get gtid 1195 /* we will dynamically update the stack range if gtid_mode == 1 */ 1196 TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data); 1197 TCW_PTR(th->th.th_info.ds.ds_stacksize, 0); 1198 TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE); 1199 __kmp_check_stack_overlap( th ); 1200 } 1201 } 1202 else { 1203 KMP_MB(); /* Flush all pending memory write invalidates. */ 1204 1205 /* Set stack size for this thread now. */ 1206 KA_TRACE( 10, ( "__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC 1207 " bytes\n", stack_size ) ); 1208 1209 stack_size += gtid * __kmp_stkoffset; 1210 1211 TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size); 1212 TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE); 1213 1214 KA_TRACE( 10, ( "__kmp_create_worker: (before) stack_size = %" 1215 KMP_SIZE_T_SPEC 1216 " bytes, &__kmp_launch_worker = %p, th = %p, " 1217 "&idThread = %p\n", 1218 (SIZE_T) stack_size, 1219 (LPTHREAD_START_ROUTINE) & __kmp_launch_worker, 1220 (LPVOID) th, &idThread ) ); 1221 1222 handle = CreateThread( NULL, (SIZE_T) stack_size, 1223 (LPTHREAD_START_ROUTINE) __kmp_launch_worker, 1224 (LPVOID) th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread ); 1225 1226 KA_TRACE( 10, ( "__kmp_create_worker: (after) stack_size = %" 1227 KMP_SIZE_T_SPEC 1228 " bytes, &__kmp_launch_worker = %p, th = %p, " 1229 "idThread = %u, handle = %" KMP_UINTPTR_SPEC "\n", 1230 (SIZE_T) stack_size, 1231 (LPTHREAD_START_ROUTINE) & __kmp_launch_worker, 1232 (LPVOID) th, idThread, handle ) ); 1233 1234 if ( handle == 0 ) { 1235 DWORD error = GetLastError(); 1236 __kmp_msg(kmp_ms_fatal, KMP_MSG( CantCreateThread ), KMP_ERR( error ), __kmp_msg_null); 1237 } else { 1238 th->th.th_info.ds.ds_thread = handle; 1239 } 1240 1241 KMP_MB(); /* Flush all pending memory write invalidates. */ 1242 } 1243 1244 KA_TRACE( 10, ("__kmp_create_worker: done creating thread (%d)\n", gtid ) ); 1245 } 1246 1247 int 1248 __kmp_still_running(kmp_info_t *th) { 1249 return (WAIT_TIMEOUT == WaitForSingleObject( th->th.th_info.ds.ds_thread, 0)); 1250 } 1251 1252 #if KMP_USE_MONITOR 1253 void 1254 __kmp_create_monitor( kmp_info_t *th ) 1255 { 1256 kmp_thread_t handle; 1257 DWORD idThread; 1258 int ideal, new_ideal; 1259 1260 if( __kmp_dflt_blocktime == KMP_MAX_BLOCKTIME ) { 1261 // We don't need monitor thread in case of MAX_BLOCKTIME 1262 KA_TRACE( 10, ("__kmp_create_monitor: skipping monitor thread because of MAX blocktime\n" ) ); 1263 th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op 1264 th->th.th_info.ds.ds_gtid = 0; 1265 TCW_4( __kmp_init_monitor, 2 ); // Signal to stop waiting for monitor creation 1266 return; 1267 } 1268 KA_TRACE( 10, ("__kmp_create_monitor: try to create monitor\n" ) ); 1269 1270 KMP_MB(); /* Flush all pending memory write invalidates. */ 1271 1272 __kmp_monitor_ev = CreateEvent( NULL, TRUE, FALSE, NULL ); 1273 if ( __kmp_monitor_ev == NULL ) { 1274 DWORD error = GetLastError(); 1275 __kmp_msg( 1276 kmp_ms_fatal, 1277 KMP_MSG( CantCreateEvent ), 1278 KMP_ERR( error ), 1279 __kmp_msg_null 1280 ); 1281 }; // if 1282 #if USE_ITT_BUILD 1283 __kmp_itt_system_object_created( __kmp_monitor_ev, "Event" ); 1284 #endif /* USE_ITT_BUILD */ 1285 1286 th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR; 1287 th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR; 1288 1289 // FIXME - on Windows* OS, if __kmp_monitor_stksize = 0, figure out how 1290 // to automatically expand stacksize based on CreateThread error code. 1291 if ( __kmp_monitor_stksize == 0 ) { 1292 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE; 1293 } 1294 if ( __kmp_monitor_stksize < __kmp_sys_min_stksize ) { 1295 __kmp_monitor_stksize = __kmp_sys_min_stksize; 1296 } 1297 1298 KA_TRACE( 10, ("__kmp_create_monitor: requested stacksize = %d bytes\n", 1299 (int) __kmp_monitor_stksize ) ); 1300 1301 TCW_4( __kmp_global.g.g_time.dt.t_value, 0 ); 1302 1303 handle = CreateThread( NULL, (SIZE_T) __kmp_monitor_stksize, 1304 (LPTHREAD_START_ROUTINE) __kmp_launch_monitor, 1305 (LPVOID) th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread ); 1306 if (handle == 0) { 1307 DWORD error = GetLastError(); 1308 __kmp_msg( 1309 kmp_ms_fatal, 1310 KMP_MSG( CantCreateThread ), 1311 KMP_ERR( error ), 1312 __kmp_msg_null 1313 ); 1314 } 1315 else 1316 th->th.th_info.ds.ds_thread = handle; 1317 1318 KMP_MB(); /* Flush all pending memory write invalidates. */ 1319 1320 KA_TRACE( 10, ("__kmp_create_monitor: monitor created %p\n", 1321 (void *) th->th.th_info.ds.ds_thread ) ); 1322 } 1323 #endif 1324 1325 /* 1326 Check to see if thread is still alive. 1327 1328 NOTE: The ExitProcess(code) system call causes all threads to Terminate 1329 with a exit_val = code. Because of this we can not rely on 1330 exit_val having any particular value. So this routine may 1331 return STILL_ALIVE in exit_val even after the thread is dead. 1332 */ 1333 1334 int 1335 __kmp_is_thread_alive( kmp_info_t * th, DWORD *exit_val ) 1336 { 1337 DWORD rc; 1338 rc = GetExitCodeThread( th->th.th_info.ds.ds_thread, exit_val ); 1339 if ( rc == 0 ) { 1340 DWORD error = GetLastError(); 1341 __kmp_msg( 1342 kmp_ms_fatal, 1343 KMP_MSG( FunctionError, "GetExitCodeThread()" ), 1344 KMP_ERR( error ), 1345 __kmp_msg_null 1346 ); 1347 }; // if 1348 return ( *exit_val == STILL_ACTIVE ); 1349 } 1350 1351 1352 void 1353 __kmp_exit_thread( 1354 int exit_status 1355 ) { 1356 ExitThread( exit_status ); 1357 } // __kmp_exit_thread 1358 1359 /* 1360 This is a common part for both __kmp_reap_worker() and __kmp_reap_monitor(). 1361 */ 1362 static void 1363 __kmp_reap_common( kmp_info_t * th ) 1364 { 1365 DWORD exit_val; 1366 1367 KMP_MB(); /* Flush all pending memory write invalidates. */ 1368 1369 KA_TRACE( 10, ( "__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid ) ); 1370 1371 /* 1372 2006-10-19: 1373 1374 There are two opposite situations: 1375 1376 1. Windows* OS keep thread alive after it resets ds_alive flag and exits from thread 1377 function. (For example, see C70770/Q394281 "unloading of dll based on OMP is very 1378 slow".) 1379 2. Windows* OS may kill thread before it resets ds_alive flag. 1380 1381 Right solution seems to be waiting for *either* thread termination *or* ds_alive resetting. 1382 1383 */ 1384 1385 { 1386 // TODO: This code is very similar to KMP_WAIT_YIELD. Need to generalize KMP_WAIT_YIELD to 1387 // cover this usage also. 1388 void * obj = NULL; 1389 register kmp_uint32 spins; 1390 #if USE_ITT_BUILD 1391 KMP_FSYNC_SPIN_INIT( obj, (void*) & th->th.th_info.ds.ds_alive ); 1392 #endif /* USE_ITT_BUILD */ 1393 KMP_INIT_YIELD( spins ); 1394 do { 1395 #if USE_ITT_BUILD 1396 KMP_FSYNC_SPIN_PREPARE( obj ); 1397 #endif /* USE_ITT_BUILD */ 1398 __kmp_is_thread_alive( th, &exit_val ); 1399 KMP_YIELD( TCR_4(__kmp_nth) > __kmp_avail_proc ); 1400 KMP_YIELD_SPIN( spins ); 1401 } while ( exit_val == STILL_ACTIVE && TCR_4( th->th.th_info.ds.ds_alive ) ); 1402 #if USE_ITT_BUILD 1403 if ( exit_val == STILL_ACTIVE ) { 1404 KMP_FSYNC_CANCEL( obj ); 1405 } else { 1406 KMP_FSYNC_SPIN_ACQUIRED( obj ); 1407 }; // if 1408 #endif /* USE_ITT_BUILD */ 1409 } 1410 1411 __kmp_free_handle( th->th.th_info.ds.ds_thread ); 1412 1413 /* 1414 * NOTE: The ExitProcess(code) system call causes all threads to Terminate 1415 * with a exit_val = code. Because of this we can not rely on 1416 * exit_val having any particular value. 1417 */ 1418 if ( exit_val == STILL_ACTIVE ) { 1419 KA_TRACE( 1, ( "__kmp_reap_common: thread still active.\n" ) ); 1420 } else if ( (void *) exit_val != (void *) th) { 1421 KA_TRACE( 1, ( "__kmp_reap_common: ExitProcess / TerminateThread used?\n" ) ); 1422 }; // if 1423 1424 KA_TRACE( 10, 1425 ( 1426 "__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC "\n", 1427 th->th.th_info.ds.ds_gtid, 1428 th->th.th_info.ds.ds_thread 1429 ) 1430 ); 1431 1432 th->th.th_info.ds.ds_thread = 0; 1433 th->th.th_info.ds.ds_tid = KMP_GTID_DNE; 1434 th->th.th_info.ds.ds_gtid = KMP_GTID_DNE; 1435 th->th.th_info.ds.ds_thread_id = 0; 1436 1437 KMP_MB(); /* Flush all pending memory write invalidates. */ 1438 } 1439 1440 #if KMP_USE_MONITOR 1441 void 1442 __kmp_reap_monitor( kmp_info_t *th ) 1443 { 1444 int status; 1445 1446 KA_TRACE( 10, ("__kmp_reap_monitor: try to reap %p\n", 1447 (void *) th->th.th_info.ds.ds_thread ) ); 1448 1449 // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR. 1450 // If both tid and gtid are 0, it means the monitor did not ever start. 1451 // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down. 1452 KMP_DEBUG_ASSERT( th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid ); 1453 if ( th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR ) { 1454 KA_TRACE( 10, ("__kmp_reap_monitor: monitor did not start, returning\n") ); 1455 return; 1456 }; // if 1457 1458 KMP_MB(); /* Flush all pending memory write invalidates. */ 1459 1460 status = SetEvent( __kmp_monitor_ev ); 1461 if ( status == FALSE ) { 1462 DWORD error = GetLastError(); 1463 __kmp_msg( 1464 kmp_ms_fatal, 1465 KMP_MSG( CantSetEvent ), 1466 KMP_ERR( error ), 1467 __kmp_msg_null 1468 ); 1469 } 1470 KA_TRACE( 10, ( "__kmp_reap_monitor: reaping thread (%d)\n", th->th.th_info.ds.ds_gtid ) ); 1471 __kmp_reap_common( th ); 1472 1473 __kmp_free_handle( __kmp_monitor_ev ); 1474 1475 KMP_MB(); /* Flush all pending memory write invalidates. */ 1476 } 1477 #endif 1478 1479 void 1480 __kmp_reap_worker( kmp_info_t * th ) 1481 { 1482 KA_TRACE( 10, ( "__kmp_reap_worker: reaping thread (%d)\n", th->th.th_info.ds.ds_gtid ) ); 1483 __kmp_reap_common( th ); 1484 } 1485 1486 /* ------------------------------------------------------------------------ */ 1487 /* ------------------------------------------------------------------------ */ 1488 1489 #if KMP_HANDLE_SIGNALS 1490 1491 1492 static void 1493 __kmp_team_handler( int signo ) 1494 { 1495 if ( __kmp_global.g.g_abort == 0 ) { 1496 // Stage 1 signal handler, let's shut down all of the threads. 1497 if ( __kmp_debug_buf ) { 1498 __kmp_dump_debug_buffer(); 1499 }; // if 1500 KMP_MB(); // Flush all pending memory write invalidates. 1501 TCW_4( __kmp_global.g.g_abort, signo ); 1502 KMP_MB(); // Flush all pending memory write invalidates. 1503 TCW_4( __kmp_global.g.g_done, TRUE ); 1504 KMP_MB(); // Flush all pending memory write invalidates. 1505 } 1506 } // __kmp_team_handler 1507 1508 1509 1510 static 1511 sig_func_t __kmp_signal( int signum, sig_func_t handler ) { 1512 sig_func_t old = signal( signum, handler ); 1513 if ( old == SIG_ERR ) { 1514 int error = errno; 1515 __kmp_msg( kmp_ms_fatal, KMP_MSG( FunctionError, "signal" ), KMP_ERR( error ), __kmp_msg_null ); 1516 }; // if 1517 return old; 1518 } 1519 1520 static void 1521 __kmp_install_one_handler( 1522 int sig, 1523 sig_func_t handler, 1524 int parallel_init 1525 ) { 1526 sig_func_t old; 1527 KMP_MB(); /* Flush all pending memory write invalidates. */ 1528 KB_TRACE( 60, ("__kmp_install_one_handler: called: sig=%d\n", sig ) ); 1529 if ( parallel_init ) { 1530 old = __kmp_signal( sig, handler ); 1531 // SIG_DFL on Windows* OS in NULL or 0. 1532 if ( old == __kmp_sighldrs[ sig ] ) { 1533 __kmp_siginstalled[ sig ] = 1; 1534 } else { 1535 // Restore/keep user's handler if one previously installed. 1536 old = __kmp_signal( sig, old ); 1537 }; // if 1538 } else { 1539 // Save initial/system signal handlers to see if user handlers installed. 1540 // 2009-09-23: It is a dead code. On Windows* OS __kmp_install_signals called once with 1541 // parallel_init == TRUE. 1542 old = __kmp_signal( sig, SIG_DFL ); 1543 __kmp_sighldrs[ sig ] = old; 1544 __kmp_signal( sig, old ); 1545 }; // if 1546 KMP_MB(); /* Flush all pending memory write invalidates. */ 1547 } // __kmp_install_one_handler 1548 1549 static void 1550 __kmp_remove_one_handler( int sig ) { 1551 if ( __kmp_siginstalled[ sig ] ) { 1552 sig_func_t old; 1553 KMP_MB(); // Flush all pending memory write invalidates. 1554 KB_TRACE( 60, ( "__kmp_remove_one_handler: called: sig=%d\n", sig ) ); 1555 old = __kmp_signal( sig, __kmp_sighldrs[ sig ] ); 1556 if ( old != __kmp_team_handler ) { 1557 KB_TRACE( 10, ( "__kmp_remove_one_handler: oops, not our handler, restoring: sig=%d\n", sig ) ); 1558 old = __kmp_signal( sig, old ); 1559 }; // if 1560 __kmp_sighldrs[ sig ] = NULL; 1561 __kmp_siginstalled[ sig ] = 0; 1562 KMP_MB(); // Flush all pending memory write invalidates. 1563 }; // if 1564 } // __kmp_remove_one_handler 1565 1566 1567 void 1568 __kmp_install_signals( int parallel_init ) 1569 { 1570 KB_TRACE( 10, ( "__kmp_install_signals: called\n" ) ); 1571 if ( ! __kmp_handle_signals ) { 1572 KB_TRACE( 10, ( "__kmp_install_signals: KMP_HANDLE_SIGNALS is false - handlers not installed\n" ) ); 1573 return; 1574 }; // if 1575 __kmp_install_one_handler( SIGINT, __kmp_team_handler, parallel_init ); 1576 __kmp_install_one_handler( SIGILL, __kmp_team_handler, parallel_init ); 1577 __kmp_install_one_handler( SIGABRT, __kmp_team_handler, parallel_init ); 1578 __kmp_install_one_handler( SIGFPE, __kmp_team_handler, parallel_init ); 1579 __kmp_install_one_handler( SIGSEGV, __kmp_team_handler, parallel_init ); 1580 __kmp_install_one_handler( SIGTERM, __kmp_team_handler, parallel_init ); 1581 } // __kmp_install_signals 1582 1583 1584 void 1585 __kmp_remove_signals( void ) 1586 { 1587 int sig; 1588 KB_TRACE( 10, ("__kmp_remove_signals: called\n" ) ); 1589 for ( sig = 1; sig < NSIG; ++ sig ) { 1590 __kmp_remove_one_handler( sig ); 1591 }; // for sig 1592 } // __kmp_remove_signals 1593 1594 1595 #endif // KMP_HANDLE_SIGNALS 1596 1597 /* Put the thread to sleep for a time period */ 1598 void 1599 __kmp_thread_sleep( int millis ) 1600 { 1601 DWORD status; 1602 1603 status = SleepEx( (DWORD) millis, FALSE ); 1604 if ( status ) { 1605 DWORD error = GetLastError(); 1606 __kmp_msg( 1607 kmp_ms_fatal, 1608 KMP_MSG( FunctionError, "SleepEx()" ), 1609 KMP_ERR( error ), 1610 __kmp_msg_null 1611 ); 1612 } 1613 } 1614 1615 /* Determine whether the given address is mapped into the current address space. */ 1616 int 1617 __kmp_is_address_mapped( void * addr ) 1618 { 1619 DWORD status; 1620 MEMORY_BASIC_INFORMATION lpBuffer; 1621 SIZE_T dwLength; 1622 1623 dwLength = sizeof(MEMORY_BASIC_INFORMATION); 1624 1625 status = VirtualQuery( addr, &lpBuffer, dwLength ); 1626 1627 return !((( lpBuffer.State == MEM_RESERVE) || ( lpBuffer.State == MEM_FREE )) || 1628 (( lpBuffer.Protect == PAGE_NOACCESS ) || ( lpBuffer.Protect == PAGE_EXECUTE ))); 1629 } 1630 1631 kmp_uint64 1632 __kmp_hardware_timestamp(void) 1633 { 1634 kmp_uint64 r = 0; 1635 1636 QueryPerformanceCounter((LARGE_INTEGER*) &r); 1637 return r; 1638 } 1639 1640 /* Free handle and check the error code */ 1641 void 1642 __kmp_free_handle( kmp_thread_t tHandle ) 1643 { 1644 /* called with parameter type HANDLE also, thus suppose kmp_thread_t defined as HANDLE */ 1645 BOOL rc; 1646 rc = CloseHandle( tHandle ); 1647 if ( !rc ) { 1648 DWORD error = GetLastError(); 1649 __kmp_msg( 1650 kmp_ms_fatal, 1651 KMP_MSG( CantCloseHandle ), 1652 KMP_ERR( error ), 1653 __kmp_msg_null 1654 ); 1655 } 1656 } 1657 1658 int 1659 __kmp_get_load_balance( int max ) { 1660 1661 static ULONG glb_buff_size = 100 * 1024; 1662 1663 static int glb_running_threads = 0; /* Saved count of the running threads for the thread balance algortihm */ 1664 static double glb_call_time = 0; /* Thread balance algorithm call time */ 1665 1666 int running_threads = 0; // Number of running threads in the system. 1667 NTSTATUS status = 0; 1668 ULONG buff_size = 0; 1669 ULONG info_size = 0; 1670 void * buffer = NULL; 1671 PSYSTEM_PROCESS_INFORMATION spi = NULL; 1672 int first_time = 1; 1673 1674 double call_time = 0.0; //start, finish; 1675 1676 __kmp_elapsed( & call_time ); 1677 1678 if ( glb_call_time && 1679 ( call_time - glb_call_time < __kmp_load_balance_interval ) ) { 1680 running_threads = glb_running_threads; 1681 goto finish; 1682 } 1683 glb_call_time = call_time; 1684 1685 // Do not spend time on running algorithm if we have a permanent error. 1686 if ( NtQuerySystemInformation == NULL ) { 1687 running_threads = -1; 1688 goto finish; 1689 }; // if 1690 1691 if ( max <= 0 ) { 1692 max = INT_MAX; 1693 }; // if 1694 1695 do { 1696 1697 if ( first_time ) { 1698 buff_size = glb_buff_size; 1699 } else { 1700 buff_size = 2 * buff_size; 1701 } 1702 1703 buffer = KMP_INTERNAL_REALLOC( buffer, buff_size ); 1704 if ( buffer == NULL ) { 1705 running_threads = -1; 1706 goto finish; 1707 }; // if 1708 status = NtQuerySystemInformation( SystemProcessInformation, buffer, buff_size, & info_size ); 1709 first_time = 0; 1710 1711 } while ( status == STATUS_INFO_LENGTH_MISMATCH ); 1712 glb_buff_size = buff_size; 1713 1714 #define CHECK( cond ) \ 1715 { \ 1716 KMP_DEBUG_ASSERT( cond ); \ 1717 if ( ! ( cond ) ) { \ 1718 running_threads = -1; \ 1719 goto finish; \ 1720 } \ 1721 } 1722 1723 CHECK( buff_size >= info_size ); 1724 spi = PSYSTEM_PROCESS_INFORMATION( buffer ); 1725 for ( ; ; ) { 1726 ptrdiff_t offset = uintptr_t( spi ) - uintptr_t( buffer ); 1727 CHECK( 0 <= offset && offset + sizeof( SYSTEM_PROCESS_INFORMATION ) < info_size ); 1728 HANDLE pid = spi->ProcessId; 1729 ULONG num = spi->NumberOfThreads; 1730 CHECK( num >= 1 ); 1731 size_t spi_size = sizeof( SYSTEM_PROCESS_INFORMATION ) + sizeof( SYSTEM_THREAD ) * ( num - 1 ); 1732 CHECK( offset + spi_size < info_size ); // Make sure process info record fits the buffer. 1733 if ( spi->NextEntryOffset != 0 ) { 1734 CHECK( spi_size <= spi->NextEntryOffset ); // And do not overlap with the next record. 1735 }; // if 1736 // pid == 0 corresponds to the System Idle Process. It always has running threads 1737 // on all cores. So, we don't consider the running threads of this process. 1738 if ( pid != 0 ) { 1739 for ( int i = 0; i < num; ++ i ) { 1740 THREAD_STATE state = spi->Threads[ i ].State; 1741 // Count threads that have Ready or Running state. 1742 // !!! TODO: Why comment does not match the code??? 1743 if ( state == StateRunning ) { 1744 ++ running_threads; 1745 // Stop counting running threads if the number is already greater than 1746 // the number of available cores 1747 if ( running_threads >= max ) { 1748 goto finish; 1749 } 1750 } // if 1751 }; // for i 1752 } // if 1753 if ( spi->NextEntryOffset == 0 ) { 1754 break; 1755 }; // if 1756 spi = PSYSTEM_PROCESS_INFORMATION( uintptr_t( spi ) + spi->NextEntryOffset ); 1757 }; // forever 1758 1759 #undef CHECK 1760 1761 finish: // Clean up and exit. 1762 1763 if ( buffer != NULL ) { 1764 KMP_INTERNAL_FREE( buffer ); 1765 }; // if 1766 1767 glb_running_threads = running_threads; 1768 1769 return running_threads; 1770 1771 } //__kmp_get_load_balance() 1772 1773