1 /*
2 * z_Windows_NT_util.cpp -- platform specific routines.
3 */
4
5 //===----------------------------------------------------------------------===//
6 //
7 // The LLVM Compiler Infrastructure
8 //
9 // This file is dual licensed under the MIT and the University of Illinois Open
10 // Source Licenses. See LICENSE.txt for details.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "kmp.h"
15 #include "kmp_affinity.h"
16 #include "kmp_i18n.h"
17 #include "kmp_io.h"
18 #include "kmp_itt.h"
19 #include "kmp_wait_release.h"
20
21 /* This code is related to NtQuerySystemInformation() function. This function
22 is used in the Load balance algorithm for OMP_DYNAMIC=true to find the
23 number of running threads in the system. */
24
25 #include <ntsecapi.h> // UNICODE_STRING
26 #include <ntstatus.h>
27
28 enum SYSTEM_INFORMATION_CLASS {
29 SystemProcessInformation = 5
30 }; // SYSTEM_INFORMATION_CLASS
31
32 struct CLIENT_ID {
33 HANDLE UniqueProcess;
34 HANDLE UniqueThread;
35 }; // struct CLIENT_ID
36
37 enum THREAD_STATE {
38 StateInitialized,
39 StateReady,
40 StateRunning,
41 StateStandby,
42 StateTerminated,
43 StateWait,
44 StateTransition,
45 StateUnknown
46 }; // enum THREAD_STATE
47
48 struct VM_COUNTERS {
49 SIZE_T PeakVirtualSize;
50 SIZE_T VirtualSize;
51 ULONG PageFaultCount;
52 SIZE_T PeakWorkingSetSize;
53 SIZE_T WorkingSetSize;
54 SIZE_T QuotaPeakPagedPoolUsage;
55 SIZE_T QuotaPagedPoolUsage;
56 SIZE_T QuotaPeakNonPagedPoolUsage;
57 SIZE_T QuotaNonPagedPoolUsage;
58 SIZE_T PagefileUsage;
59 SIZE_T PeakPagefileUsage;
60 SIZE_T PrivatePageCount;
61 }; // struct VM_COUNTERS
62
63 struct SYSTEM_THREAD {
64 LARGE_INTEGER KernelTime;
65 LARGE_INTEGER UserTime;
66 LARGE_INTEGER CreateTime;
67 ULONG WaitTime;
68 LPVOID StartAddress;
69 CLIENT_ID ClientId;
70 DWORD Priority;
71 LONG BasePriority;
72 ULONG ContextSwitchCount;
73 THREAD_STATE State;
74 ULONG WaitReason;
75 }; // SYSTEM_THREAD
76
77 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, KernelTime) == 0);
78 #if KMP_ARCH_X86
79 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 28);
80 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 52);
81 #else
82 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 32);
83 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 68);
84 #endif
85
86 struct SYSTEM_PROCESS_INFORMATION {
87 ULONG NextEntryOffset;
88 ULONG NumberOfThreads;
89 LARGE_INTEGER Reserved[3];
90 LARGE_INTEGER CreateTime;
91 LARGE_INTEGER UserTime;
92 LARGE_INTEGER KernelTime;
93 UNICODE_STRING ImageName;
94 DWORD BasePriority;
95 HANDLE ProcessId;
96 HANDLE ParentProcessId;
97 ULONG HandleCount;
98 ULONG Reserved2[2];
99 VM_COUNTERS VMCounters;
100 IO_COUNTERS IOCounters;
101 SYSTEM_THREAD Threads[1];
102 }; // SYSTEM_PROCESS_INFORMATION
103 typedef SYSTEM_PROCESS_INFORMATION *PSYSTEM_PROCESS_INFORMATION;
104
105 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, NextEntryOffset) == 0);
106 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, CreateTime) == 32);
107 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ImageName) == 56);
108 #if KMP_ARCH_X86
109 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 68);
110 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 76);
111 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 88);
112 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 136);
113 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 184);
114 #else
115 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 80);
116 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 96);
117 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 112);
118 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 208);
119 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 256);
120 #endif
121
122 typedef NTSTATUS(NTAPI *NtQuerySystemInformation_t)(SYSTEM_INFORMATION_CLASS,
123 PVOID, ULONG, PULONG);
124 NtQuerySystemInformation_t NtQuerySystemInformation = NULL;
125
126 HMODULE ntdll = NULL;
127
128 /* End of NtQuerySystemInformation()-related code */
129
130 static HMODULE kernel32 = NULL;
131
132 #if KMP_HANDLE_SIGNALS
133 typedef void (*sig_func_t)(int);
134 static sig_func_t __kmp_sighldrs[NSIG];
135 static int __kmp_siginstalled[NSIG];
136 #endif
137
138 #if KMP_USE_MONITOR
139 static HANDLE __kmp_monitor_ev;
140 #endif
141 static kmp_int64 __kmp_win32_time;
142 double __kmp_win32_tick;
143
144 int __kmp_init_runtime = FALSE;
145 CRITICAL_SECTION __kmp_win32_section;
146
__kmp_win32_mutex_init(kmp_win32_mutex_t * mx)147 void __kmp_win32_mutex_init(kmp_win32_mutex_t *mx) {
148 InitializeCriticalSection(&mx->cs);
149 #if USE_ITT_BUILD
150 __kmp_itt_system_object_created(&mx->cs, "Critical Section");
151 #endif /* USE_ITT_BUILD */
152 }
153
__kmp_win32_mutex_destroy(kmp_win32_mutex_t * mx)154 void __kmp_win32_mutex_destroy(kmp_win32_mutex_t *mx) {
155 DeleteCriticalSection(&mx->cs);
156 }
157
__kmp_win32_mutex_lock(kmp_win32_mutex_t * mx)158 void __kmp_win32_mutex_lock(kmp_win32_mutex_t *mx) {
159 EnterCriticalSection(&mx->cs);
160 }
161
__kmp_win32_mutex_unlock(kmp_win32_mutex_t * mx)162 void __kmp_win32_mutex_unlock(kmp_win32_mutex_t *mx) {
163 LeaveCriticalSection(&mx->cs);
164 }
165
__kmp_win32_cond_init(kmp_win32_cond_t * cv)166 void __kmp_win32_cond_init(kmp_win32_cond_t *cv) {
167 cv->waiters_count_ = 0;
168 cv->wait_generation_count_ = 0;
169 cv->release_count_ = 0;
170
171 /* Initialize the critical section */
172 __kmp_win32_mutex_init(&cv->waiters_count_lock_);
173
174 /* Create a manual-reset event. */
175 cv->event_ = CreateEvent(NULL, // no security
176 TRUE, // manual-reset
177 FALSE, // non-signaled initially
178 NULL); // unnamed
179 #if USE_ITT_BUILD
180 __kmp_itt_system_object_created(cv->event_, "Event");
181 #endif /* USE_ITT_BUILD */
182 }
183
__kmp_win32_cond_destroy(kmp_win32_cond_t * cv)184 void __kmp_win32_cond_destroy(kmp_win32_cond_t *cv) {
185 __kmp_win32_mutex_destroy(&cv->waiters_count_lock_);
186 __kmp_free_handle(cv->event_);
187 memset(cv, '\0', sizeof(*cv));
188 }
189
190 /* TODO associate cv with a team instead of a thread so as to optimize
191 the case where we wake up a whole team */
192
__kmp_win32_cond_wait(kmp_win32_cond_t * cv,kmp_win32_mutex_t * mx,kmp_info_t * th,int need_decrease_load)193 void __kmp_win32_cond_wait(kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx,
194 kmp_info_t *th, int need_decrease_load) {
195 int my_generation;
196 int last_waiter;
197
198 /* Avoid race conditions */
199 __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
200
201 /* Increment count of waiters */
202 cv->waiters_count_++;
203
204 /* Store current generation in our activation record. */
205 my_generation = cv->wait_generation_count_;
206
207 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
208 __kmp_win32_mutex_unlock(mx);
209
210 for (;;) {
211 int wait_done;
212
213 /* Wait until the event is signaled */
214 WaitForSingleObject(cv->event_, INFINITE);
215
216 __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
217
218 /* Exit the loop when the <cv->event_> is signaled and there are still
219 waiting threads from this <wait_generation> that haven't been released
220 from this wait yet. */
221 wait_done = (cv->release_count_ > 0) &&
222 (cv->wait_generation_count_ != my_generation);
223
224 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
225
226 /* there used to be a semicolon after the if statement, it looked like a
227 bug, so i removed it */
228 if (wait_done)
229 break;
230 }
231
232 __kmp_win32_mutex_lock(mx);
233 __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
234
235 cv->waiters_count_--;
236 cv->release_count_--;
237
238 last_waiter = (cv->release_count_ == 0);
239
240 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
241
242 if (last_waiter) {
243 /* We're the last waiter to be notified, so reset the manual event. */
244 ResetEvent(cv->event_);
245 }
246 }
247
__kmp_win32_cond_broadcast(kmp_win32_cond_t * cv)248 void __kmp_win32_cond_broadcast(kmp_win32_cond_t *cv) {
249 __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
250
251 if (cv->waiters_count_ > 0) {
252 SetEvent(cv->event_);
253 /* Release all the threads in this generation. */
254
255 cv->release_count_ = cv->waiters_count_;
256
257 /* Start a new generation. */
258 cv->wait_generation_count_++;
259 }
260
261 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
262 }
263
__kmp_win32_cond_signal(kmp_win32_cond_t * cv)264 void __kmp_win32_cond_signal(kmp_win32_cond_t *cv) {
265 __kmp_win32_cond_broadcast(cv);
266 }
267
__kmp_enable(int new_state)268 void __kmp_enable(int new_state) {
269 if (__kmp_init_runtime)
270 LeaveCriticalSection(&__kmp_win32_section);
271 }
272
__kmp_disable(int * old_state)273 void __kmp_disable(int *old_state) {
274 *old_state = 0;
275
276 if (__kmp_init_runtime)
277 EnterCriticalSection(&__kmp_win32_section);
278 }
279
__kmp_suspend_initialize(void)280 void __kmp_suspend_initialize(void) { /* do nothing */
281 }
282
__kmp_suspend_initialize_thread(kmp_info_t * th)283 static void __kmp_suspend_initialize_thread(kmp_info_t *th) {
284 if (!TCR_4(th->th.th_suspend_init)) {
285 /* this means we haven't initialized the suspension pthread objects for this
286 thread in this instance of the process */
287 __kmp_win32_cond_init(&th->th.th_suspend_cv);
288 __kmp_win32_mutex_init(&th->th.th_suspend_mx);
289 TCW_4(th->th.th_suspend_init, TRUE);
290 }
291 }
292
__kmp_suspend_uninitialize_thread(kmp_info_t * th)293 void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
294 if (TCR_4(th->th.th_suspend_init)) {
295 /* this means we have initialize the suspension pthread objects for this
296 thread in this instance of the process */
297 __kmp_win32_cond_destroy(&th->th.th_suspend_cv);
298 __kmp_win32_mutex_destroy(&th->th.th_suspend_mx);
299 TCW_4(th->th.th_suspend_init, FALSE);
300 }
301 }
302
303 /* This routine puts the calling thread to sleep after setting the
304 sleep bit for the indicated flag variable to true. */
305 template <class C>
__kmp_suspend_template(int th_gtid,C * flag)306 static inline void __kmp_suspend_template(int th_gtid, C *flag) {
307 kmp_info_t *th = __kmp_threads[th_gtid];
308 int status;
309 typename C::flag_t old_spin;
310
311 KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag's loc(%p)\n",
312 th_gtid, flag->get()));
313
314 __kmp_suspend_initialize_thread(th);
315 __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
316
317 KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for flag's"
318 " loc(%p)\n",
319 th_gtid, flag->get()));
320
321 /* TODO: shouldn't this use release semantics to ensure that
322 __kmp_suspend_initialize_thread gets called first? */
323 old_spin = flag->set_sleeping();
324
325 KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for flag's"
326 " loc(%p)==%d\n",
327 th_gtid, flag->get(), *(flag->get())));
328
329 if (flag->done_check_val(old_spin)) {
330 old_spin = flag->unset_sleeping();
331 KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit "
332 "for flag's loc(%p)\n",
333 th_gtid, flag->get()));
334 } else {
335 #ifdef DEBUG_SUSPEND
336 __kmp_suspend_count++;
337 #endif
338 /* Encapsulate in a loop as the documentation states that this may "with
339 low probability" return when the condition variable has not been signaled
340 or broadcast */
341 int deactivated = FALSE;
342 TCW_PTR(th->th.th_sleep_loc, (void *)flag);
343 while (flag->is_sleeping()) {
344 KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform "
345 "kmp_win32_cond_wait()\n",
346 th_gtid));
347 // Mark the thread as no longer active (only in the first iteration of the
348 // loop).
349 if (!deactivated) {
350 th->th.th_active = FALSE;
351 if (th->th.th_active_in_pool) {
352 th->th.th_active_in_pool = FALSE;
353 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
354 KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
355 }
356 deactivated = TRUE;
357
358 __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, 0,
359 0);
360 } else {
361 __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, 0,
362 0);
363 }
364
365 #ifdef KMP_DEBUG
366 if (flag->is_sleeping()) {
367 KF_TRACE(100,
368 ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid));
369 }
370 #endif /* KMP_DEBUG */
371
372 } // while
373
374 // Mark the thread as active again (if it was previous marked as inactive)
375 if (deactivated) {
376 th->th.th_active = TRUE;
377 if (TCR_4(th->th.th_in_pool)) {
378 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
379 th->th.th_active_in_pool = TRUE;
380 }
381 }
382 }
383
384 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
385
386 KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid));
387 }
388
__kmp_suspend_32(int th_gtid,kmp_flag_32 * flag)389 void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) {
390 __kmp_suspend_template(th_gtid, flag);
391 }
__kmp_suspend_64(int th_gtid,kmp_flag_64 * flag)392 void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) {
393 __kmp_suspend_template(th_gtid, flag);
394 }
__kmp_suspend_oncore(int th_gtid,kmp_flag_oncore * flag)395 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
396 __kmp_suspend_template(th_gtid, flag);
397 }
398
399 /* This routine signals the thread specified by target_gtid to wake up
400 after setting the sleep bit indicated by the flag argument to FALSE */
401 template <class C>
__kmp_resume_template(int target_gtid,C * flag)402 static inline void __kmp_resume_template(int target_gtid, C *flag) {
403 kmp_info_t *th = __kmp_threads[target_gtid];
404 int status;
405
406 #ifdef KMP_DEBUG
407 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
408 #endif
409
410 KF_TRACE(30, ("__kmp_resume_template: T#%d wants to wakeup T#%d enter\n",
411 gtid, target_gtid));
412
413 __kmp_suspend_initialize_thread(th);
414 __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
415
416 if (!flag) { // coming from __kmp_null_resume_wrapper
417 flag = (C *)th->th.th_sleep_loc;
418 }
419
420 // First, check if the flag is null or its type has changed. If so, someone
421 // else woke it up.
422 if (!flag || flag->get_type() != flag->get_ptr_type()) { // get_ptr_type
423 // simply shows what
424 // flag was cast to
425 KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
426 "awake: flag's loc(%p)\n",
427 gtid, target_gtid, NULL));
428 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
429 return;
430 } else {
431 typename C::flag_t old_spin = flag->unset_sleeping();
432 if (!flag->is_sleeping_val(old_spin)) {
433 KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
434 "awake: flag's loc(%p): %u => %u\n",
435 gtid, target_gtid, flag->get(), old_spin, *(flag->get())));
436 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
437 return;
438 }
439 }
440 TCW_PTR(th->th.th_sleep_loc, NULL);
441 KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep "
442 "bit for flag's loc(%p)\n",
443 gtid, target_gtid, flag->get()));
444
445 __kmp_win32_cond_signal(&th->th.th_suspend_cv);
446 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
447
448 KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up"
449 " for T#%d\n",
450 gtid, target_gtid));
451 }
452
__kmp_resume_32(int target_gtid,kmp_flag_32 * flag)453 void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) {
454 __kmp_resume_template(target_gtid, flag);
455 }
__kmp_resume_64(int target_gtid,kmp_flag_64 * flag)456 void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) {
457 __kmp_resume_template(target_gtid, flag);
458 }
__kmp_resume_oncore(int target_gtid,kmp_flag_oncore * flag)459 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
460 __kmp_resume_template(target_gtid, flag);
461 }
462
__kmp_yield(int cond)463 void __kmp_yield(int cond) {
464 if (cond)
465 Sleep(0);
466 }
467
__kmp_gtid_set_specific(int gtid)468 void __kmp_gtid_set_specific(int gtid) {
469 if (__kmp_init_gtid) {
470 KA_TRACE(50, ("__kmp_gtid_set_specific: T#%d key:%d\n", gtid,
471 __kmp_gtid_threadprivate_key));
472 if (!TlsSetValue(__kmp_gtid_threadprivate_key, (LPVOID)(gtid + 1)))
473 KMP_FATAL(TLSSetValueFailed);
474 } else {
475 KA_TRACE(50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n"));
476 }
477 }
478
__kmp_gtid_get_specific()479 int __kmp_gtid_get_specific() {
480 int gtid;
481 if (!__kmp_init_gtid) {
482 KA_TRACE(50, ("__kmp_gtid_get_specific: runtime shutdown, returning "
483 "KMP_GTID_SHUTDOWN\n"));
484 return KMP_GTID_SHUTDOWN;
485 }
486 gtid = (int)(kmp_intptr_t)TlsGetValue(__kmp_gtid_threadprivate_key);
487 if (gtid == 0) {
488 gtid = KMP_GTID_DNE;
489 } else {
490 gtid--;
491 }
492 KA_TRACE(50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
493 __kmp_gtid_threadprivate_key, gtid));
494 return gtid;
495 }
496
__kmp_affinity_bind_thread(int proc)497 void __kmp_affinity_bind_thread(int proc) {
498 if (__kmp_num_proc_groups > 1) {
499 // Form the GROUP_AFFINITY struct directly, rather than filling
500 // out a bit vector and calling __kmp_set_system_affinity().
501 GROUP_AFFINITY ga;
502 KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups * CHAR_BIT *
503 sizeof(DWORD_PTR))));
504 ga.Group = proc / (CHAR_BIT * sizeof(DWORD_PTR));
505 ga.Mask = (unsigned long long)1 << (proc % (CHAR_BIT * sizeof(DWORD_PTR)));
506 ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
507
508 KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
509 if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
510 DWORD error = GetLastError();
511 if (__kmp_affinity_verbose) { // AC: continue silently if not verbose
512 kmp_msg_t err_code = KMP_ERR(error);
513 __kmp_msg(kmp_ms_warning, KMP_MSG(CantSetThreadAffMask), err_code,
514 __kmp_msg_null);
515 if (__kmp_generate_warnings == kmp_warnings_off) {
516 __kmp_str_free(&err_code.str);
517 }
518 }
519 }
520 } else {
521 kmp_affin_mask_t *mask;
522 KMP_CPU_ALLOC_ON_STACK(mask);
523 KMP_CPU_ZERO(mask);
524 KMP_CPU_SET(proc, mask);
525 __kmp_set_system_affinity(mask, TRUE);
526 KMP_CPU_FREE_FROM_STACK(mask);
527 }
528 }
529
__kmp_affinity_determine_capable(const char * env_var)530 void __kmp_affinity_determine_capable(const char *env_var) {
531 // All versions of Windows* OS (since Win '95) support SetThreadAffinityMask().
532
533 #if KMP_GROUP_AFFINITY
534 KMP_AFFINITY_ENABLE(__kmp_num_proc_groups * sizeof(DWORD_PTR));
535 #else
536 KMP_AFFINITY_ENABLE(sizeof(DWORD_PTR));
537 #endif
538
539 KA_TRACE(10, ("__kmp_affinity_determine_capable: "
540 "Windows* OS affinity interface functional (mask size = "
541 "%" KMP_SIZE_T_SPEC ").\n",
542 __kmp_affin_mask_size));
543 }
544
__kmp_read_cpu_time(void)545 double __kmp_read_cpu_time(void) {
546 FILETIME CreationTime, ExitTime, KernelTime, UserTime;
547 int status;
548 double cpu_time;
549
550 cpu_time = 0;
551
552 status = GetProcessTimes(GetCurrentProcess(), &CreationTime, &ExitTime,
553 &KernelTime, &UserTime);
554
555 if (status) {
556 double sec = 0;
557
558 sec += KernelTime.dwHighDateTime;
559 sec += UserTime.dwHighDateTime;
560
561 /* Shift left by 32 bits */
562 sec *= (double)(1 << 16) * (double)(1 << 16);
563
564 sec += KernelTime.dwLowDateTime;
565 sec += UserTime.dwLowDateTime;
566
567 cpu_time += (sec * 100.0) / KMP_NSEC_PER_SEC;
568 }
569
570 return cpu_time;
571 }
572
__kmp_read_system_info(struct kmp_sys_info * info)573 int __kmp_read_system_info(struct kmp_sys_info *info) {
574 info->maxrss = 0; /* the maximum resident set size utilized (in kilobytes) */
575 info->minflt = 0; /* the number of page faults serviced without any I/O */
576 info->majflt = 0; /* the number of page faults serviced that required I/O */
577 info->nswap = 0; // the number of times a process was "swapped" out of memory
578 info->inblock = 0; // the number of times the file system had to perform input
579 info->oublock = 0; // number of times the file system had to perform output
580 info->nvcsw = 0; /* the number of times a context switch was voluntarily */
581 info->nivcsw = 0; /* the number of times a context switch was forced */
582
583 return 1;
584 }
585
__kmp_runtime_initialize(void)586 void __kmp_runtime_initialize(void) {
587 SYSTEM_INFO info;
588 kmp_str_buf_t path;
589 UINT path_size;
590
591 if (__kmp_init_runtime) {
592 return;
593 }
594
595 #if KMP_DYNAMIC_LIB
596 /* Pin dynamic library for the lifetime of application */
597 {
598 // First, turn off error message boxes
599 UINT err_mode = SetErrorMode(SEM_FAILCRITICALERRORS);
600 HMODULE h;
601 BOOL ret = GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
602 GET_MODULE_HANDLE_EX_FLAG_PIN,
603 (LPCTSTR)&__kmp_serial_initialize, &h);
604 KMP_DEBUG_ASSERT2(h && ret, "OpenMP RTL cannot find itself loaded");
605 SetErrorMode(err_mode); // Restore error mode
606 KA_TRACE(10, ("__kmp_runtime_initialize: dynamic library pinned\n"));
607 }
608 #endif
609
610 InitializeCriticalSection(&__kmp_win32_section);
611 #if USE_ITT_BUILD
612 __kmp_itt_system_object_created(&__kmp_win32_section, "Critical Section");
613 #endif /* USE_ITT_BUILD */
614 __kmp_initialize_system_tick();
615
616 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
617 if (!__kmp_cpuinfo.initialized) {
618 __kmp_query_cpuid(&__kmp_cpuinfo);
619 }
620 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
621
622 /* Set up minimum number of threads to switch to TLS gtid */
623 #if KMP_OS_WINDOWS && !KMP_DYNAMIC_LIB
624 // Windows* OS, static library.
625 /* New thread may use stack space previously used by another thread,
626 currently terminated. On Windows* OS, in case of static linking, we do not
627 know the moment of thread termination, and our structures (__kmp_threads
628 and __kmp_root arrays) are still keep info about dead threads. This leads
629 to problem in __kmp_get_global_thread_id() function: it wrongly finds gtid
630 (by searching through stack addresses of all known threads) for
631 unregistered foreign tread.
632
633 Setting __kmp_tls_gtid_min to 0 workarounds this problem:
634 __kmp_get_global_thread_id() does not search through stacks, but get gtid
635 from TLS immediately.
636 --ln
637 */
638 __kmp_tls_gtid_min = 0;
639 #else
640 __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
641 #endif
642
643 /* for the static library */
644 if (!__kmp_gtid_threadprivate_key) {
645 __kmp_gtid_threadprivate_key = TlsAlloc();
646 if (__kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES) {
647 KMP_FATAL(TLSOutOfIndexes);
648 }
649 }
650
651 // Load ntdll.dll.
652 /* Simple GetModuleHandle( "ntdll.dl" ) is not suitable due to security issue
653 (see http://www.microsoft.com/technet/security/advisory/2269637.mspx). We
654 have to specify full path to the library. */
655 __kmp_str_buf_init(&path);
656 path_size = GetSystemDirectory(path.str, path.size);
657 KMP_DEBUG_ASSERT(path_size > 0);
658 if (path_size >= path.size) {
659 // Buffer is too short. Expand the buffer and try again.
660 __kmp_str_buf_reserve(&path, path_size);
661 path_size = GetSystemDirectory(path.str, path.size);
662 KMP_DEBUG_ASSERT(path_size > 0);
663 }
664 if (path_size > 0 && path_size < path.size) {
665 // Now we have system directory name in the buffer.
666 // Append backslash and name of dll to form full path,
667 path.used = path_size;
668 __kmp_str_buf_print(&path, "\\%s", "ntdll.dll");
669
670 // Now load ntdll using full path.
671 ntdll = GetModuleHandle(path.str);
672 }
673
674 KMP_DEBUG_ASSERT(ntdll != NULL);
675 if (ntdll != NULL) {
676 NtQuerySystemInformation = (NtQuerySystemInformation_t)GetProcAddress(
677 ntdll, "NtQuerySystemInformation");
678 }
679 KMP_DEBUG_ASSERT(NtQuerySystemInformation != NULL);
680
681 #if KMP_GROUP_AFFINITY
682 // Load kernel32.dll.
683 // Same caveat - must use full system path name.
684 if (path_size > 0 && path_size < path.size) {
685 // Truncate the buffer back to just the system path length,
686 // discarding "\\ntdll.dll", and replacing it with "kernel32.dll".
687 path.used = path_size;
688 __kmp_str_buf_print(&path, "\\%s", "kernel32.dll");
689
690 // Load kernel32.dll using full path.
691 kernel32 = GetModuleHandle(path.str);
692 KA_TRACE(10, ("__kmp_runtime_initialize: kernel32.dll = %s\n", path.str));
693
694 // Load the function pointers to kernel32.dll routines
695 // that may or may not exist on this system.
696 if (kernel32 != NULL) {
697 __kmp_GetActiveProcessorCount =
698 (kmp_GetActiveProcessorCount_t)GetProcAddress(
699 kernel32, "GetActiveProcessorCount");
700 __kmp_GetActiveProcessorGroupCount =
701 (kmp_GetActiveProcessorGroupCount_t)GetProcAddress(
702 kernel32, "GetActiveProcessorGroupCount");
703 __kmp_GetThreadGroupAffinity =
704 (kmp_GetThreadGroupAffinity_t)GetProcAddress(
705 kernel32, "GetThreadGroupAffinity");
706 __kmp_SetThreadGroupAffinity =
707 (kmp_SetThreadGroupAffinity_t)GetProcAddress(
708 kernel32, "SetThreadGroupAffinity");
709
710 KA_TRACE(10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorCount"
711 " = %p\n",
712 __kmp_GetActiveProcessorCount));
713 KA_TRACE(10, ("__kmp_runtime_initialize: "
714 "__kmp_GetActiveProcessorGroupCount = %p\n",
715 __kmp_GetActiveProcessorGroupCount));
716 KA_TRACE(10, ("__kmp_runtime_initialize:__kmp_GetThreadGroupAffinity"
717 " = %p\n",
718 __kmp_GetThreadGroupAffinity));
719 KA_TRACE(10, ("__kmp_runtime_initialize: __kmp_SetThreadGroupAffinity"
720 " = %p\n",
721 __kmp_SetThreadGroupAffinity));
722 KA_TRACE(10, ("__kmp_runtime_initialize: sizeof(kmp_affin_mask_t) = %d\n",
723 sizeof(kmp_affin_mask_t)));
724
725 // See if group affinity is supported on this system.
726 // If so, calculate the #groups and #procs.
727 //
728 // Group affinity was introduced with Windows* 7 OS and
729 // Windows* Server 2008 R2 OS.
730 if ((__kmp_GetActiveProcessorCount != NULL) &&
731 (__kmp_GetActiveProcessorGroupCount != NULL) &&
732 (__kmp_GetThreadGroupAffinity != NULL) &&
733 (__kmp_SetThreadGroupAffinity != NULL) &&
734 ((__kmp_num_proc_groups = __kmp_GetActiveProcessorGroupCount()) >
735 1)) {
736 // Calculate the total number of active OS procs.
737 int i;
738
739 KA_TRACE(10, ("__kmp_runtime_initialize: %d processor groups"
740 " detected\n",
741 __kmp_num_proc_groups));
742
743 __kmp_xproc = 0;
744
745 for (i = 0; i < __kmp_num_proc_groups; i++) {
746 DWORD size = __kmp_GetActiveProcessorCount(i);
747 __kmp_xproc += size;
748 KA_TRACE(10, ("__kmp_runtime_initialize: proc group %d size = %d\n",
749 i, size));
750 }
751 } else {
752 KA_TRACE(10, ("__kmp_runtime_initialize: %d processor groups"
753 " detected\n",
754 __kmp_num_proc_groups));
755 }
756 }
757 }
758 if (__kmp_num_proc_groups <= 1) {
759 GetSystemInfo(&info);
760 __kmp_xproc = info.dwNumberOfProcessors;
761 }
762 #else
763 GetSystemInfo(&info);
764 __kmp_xproc = info.dwNumberOfProcessors;
765 #endif /* KMP_GROUP_AFFINITY */
766
767 // If the OS said there were 0 procs, take a guess and use a value of 2.
768 // This is done for Linux* OS, also. Do we need error / warning?
769 if (__kmp_xproc <= 0) {
770 __kmp_xproc = 2;
771 }
772
773 KA_TRACE(5,
774 ("__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc));
775
776 __kmp_str_buf_free(&path);
777
778 #if USE_ITT_BUILD
779 __kmp_itt_initialize();
780 #endif /* USE_ITT_BUILD */
781
782 __kmp_init_runtime = TRUE;
783 } // __kmp_runtime_initialize
784
__kmp_runtime_destroy(void)785 void __kmp_runtime_destroy(void) {
786 if (!__kmp_init_runtime) {
787 return;
788 }
789
790 #if USE_ITT_BUILD
791 __kmp_itt_destroy();
792 #endif /* USE_ITT_BUILD */
793
794 /* we can't DeleteCriticalsection( & __kmp_win32_section ); */
795 /* due to the KX_TRACE() commands */
796 KA_TRACE(40, ("__kmp_runtime_destroy\n"));
797
798 if (__kmp_gtid_threadprivate_key) {
799 TlsFree(__kmp_gtid_threadprivate_key);
800 __kmp_gtid_threadprivate_key = 0;
801 }
802
803 __kmp_affinity_uninitialize();
804 DeleteCriticalSection(&__kmp_win32_section);
805
806 ntdll = NULL;
807 NtQuerySystemInformation = NULL;
808
809 #if KMP_ARCH_X86_64
810 kernel32 = NULL;
811 __kmp_GetActiveProcessorCount = NULL;
812 __kmp_GetActiveProcessorGroupCount = NULL;
813 __kmp_GetThreadGroupAffinity = NULL;
814 __kmp_SetThreadGroupAffinity = NULL;
815 #endif // KMP_ARCH_X86_64
816
817 __kmp_init_runtime = FALSE;
818 }
819
__kmp_terminate_thread(int gtid)820 void __kmp_terminate_thread(int gtid) {
821 kmp_info_t *th = __kmp_threads[gtid];
822
823 if (!th)
824 return;
825
826 KA_TRACE(10, ("__kmp_terminate_thread: kill (%d)\n", gtid));
827
828 if (TerminateThread(th->th.th_info.ds.ds_thread, (DWORD)-1) == FALSE) {
829 /* It's OK, the thread may have exited already */
830 }
831 __kmp_free_handle(th->th.th_info.ds.ds_thread);
832 }
833
__kmp_clear_system_time(void)834 void __kmp_clear_system_time(void) {
835 BOOL status;
836 LARGE_INTEGER time;
837 status = QueryPerformanceCounter(&time);
838 __kmp_win32_time = (kmp_int64)time.QuadPart;
839 }
840
__kmp_initialize_system_tick(void)841 void __kmp_initialize_system_tick(void) {
842 {
843 BOOL status;
844 LARGE_INTEGER freq;
845
846 status = QueryPerformanceFrequency(&freq);
847 if (!status) {
848 DWORD error = GetLastError();
849 __kmp_fatal(KMP_MSG(FunctionError, "QueryPerformanceFrequency()"),
850 KMP_ERR(error), __kmp_msg_null);
851
852 } else {
853 __kmp_win32_tick = ((double)1.0) / (double)freq.QuadPart;
854 }
855 }
856 }
857
858 /* Calculate the elapsed wall clock time for the user */
859
__kmp_elapsed(double * t)860 void __kmp_elapsed(double *t) {
861 BOOL status;
862 LARGE_INTEGER now;
863 status = QueryPerformanceCounter(&now);
864 *t = ((double)now.QuadPart) * __kmp_win32_tick;
865 }
866
867 /* Calculate the elapsed wall clock tick for the user */
868
__kmp_elapsed_tick(double * t)869 void __kmp_elapsed_tick(double *t) { *t = __kmp_win32_tick; }
870
__kmp_read_system_time(double * delta)871 void __kmp_read_system_time(double *delta) {
872 if (delta != NULL) {
873 BOOL status;
874 LARGE_INTEGER now;
875
876 status = QueryPerformanceCounter(&now);
877
878 *delta = ((double)(((kmp_int64)now.QuadPart) - __kmp_win32_time)) *
879 __kmp_win32_tick;
880 }
881 }
882
883 /* Return the current time stamp in nsec */
__kmp_now_nsec()884 kmp_uint64 __kmp_now_nsec() {
885 LARGE_INTEGER now;
886 QueryPerformanceCounter(&now);
887 return 1e9 * __kmp_win32_tick * now.QuadPart;
888 }
889
890 extern "C"
__kmp_launch_worker(void * arg)891 void *__stdcall __kmp_launch_worker(void *arg) {
892 volatile void *stack_data;
893 void *exit_val;
894 void *padding = 0;
895 kmp_info_t *this_thr = (kmp_info_t *)arg;
896 int gtid;
897
898 gtid = this_thr->th.th_info.ds.ds_gtid;
899 __kmp_gtid_set_specific(gtid);
900 #ifdef KMP_TDATA_GTID
901 #error "This define causes problems with LoadLibrary() + declspec(thread) " \
902 "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
903 "reference: http://support.microsoft.com/kb/118816"
904 //__kmp_gtid = gtid;
905 #endif
906
907 #if USE_ITT_BUILD
908 __kmp_itt_thread_name(gtid);
909 #endif /* USE_ITT_BUILD */
910
911 __kmp_affinity_set_init_mask(gtid, FALSE);
912
913 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
914 // Set FP control regs to be a copy of the parallel initialization thread's.
915 __kmp_clear_x87_fpu_status_word();
916 __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
917 __kmp_load_mxcsr(&__kmp_init_mxcsr);
918 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
919
920 if (__kmp_stkoffset > 0 && gtid > 0) {
921 padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
922 }
923
924 KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
925 this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
926 TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
927
928 if (TCR_4(__kmp_gtid_mode) <
929 2) { // check stack only if it is used to get gtid
930 TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data);
931 KMP_ASSERT(this_thr->th.th_info.ds.ds_stackgrow == FALSE);
932 __kmp_check_stack_overlap(this_thr);
933 }
934 KMP_MB();
935 exit_val = __kmp_launch_thread(this_thr);
936 KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
937 TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
938 KMP_MB();
939 return exit_val;
940 }
941
942 #if KMP_USE_MONITOR
943 /* The monitor thread controls all of the threads in the complex */
944
__kmp_launch_monitor(void * arg)945 void *__stdcall __kmp_launch_monitor(void *arg) {
946 DWORD wait_status;
947 kmp_thread_t monitor;
948 int status;
949 int interval;
950 kmp_info_t *this_thr = (kmp_info_t *)arg;
951
952 KMP_DEBUG_ASSERT(__kmp_init_monitor);
953 TCW_4(__kmp_init_monitor, 2); // AC: Signal library that monitor has started
954 // TODO: hide "2" in enum (like {true,false,started})
955 this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
956 TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
957
958 KMP_MB(); /* Flush all pending memory write invalidates. */
959 KA_TRACE(10, ("__kmp_launch_monitor: launched\n"));
960
961 monitor = GetCurrentThread();
962
963 /* set thread priority */
964 status = SetThreadPriority(monitor, THREAD_PRIORITY_HIGHEST);
965 if (!status) {
966 DWORD error = GetLastError();
967 __kmp_fatal(KMP_MSG(CantSetThreadPriority), KMP_ERR(error), __kmp_msg_null);
968 }
969
970 /* register us as monitor */
971 __kmp_gtid_set_specific(KMP_GTID_MONITOR);
972 #ifdef KMP_TDATA_GTID
973 #error "This define causes problems with LoadLibrary() + declspec(thread) " \
974 "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
975 "reference: http://support.microsoft.com/kb/118816"
976 //__kmp_gtid = KMP_GTID_MONITOR;
977 #endif
978
979 #if USE_ITT_BUILD
980 __kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore
981 // monitor thread.
982 #endif /* USE_ITT_BUILD */
983
984 KMP_MB(); /* Flush all pending memory write invalidates. */
985
986 interval = (1000 / __kmp_monitor_wakeups); /* in milliseconds */
987
988 while (!TCR_4(__kmp_global.g.g_done)) {
989 /* This thread monitors the state of the system */
990
991 KA_TRACE(15, ("__kmp_launch_monitor: update\n"));
992
993 wait_status = WaitForSingleObject(__kmp_monitor_ev, interval);
994
995 if (wait_status == WAIT_TIMEOUT) {
996 TCW_4(__kmp_global.g.g_time.dt.t_value,
997 TCR_4(__kmp_global.g.g_time.dt.t_value) + 1);
998 }
999
1000 KMP_MB(); /* Flush all pending memory write invalidates. */
1001 }
1002
1003 KA_TRACE(10, ("__kmp_launch_monitor: finished\n"));
1004
1005 status = SetThreadPriority(monitor, THREAD_PRIORITY_NORMAL);
1006 if (!status) {
1007 DWORD error = GetLastError();
1008 __kmp_fatal(KMP_MSG(CantSetThreadPriority), KMP_ERR(error), __kmp_msg_null);
1009 }
1010
1011 if (__kmp_global.g.g_abort != 0) {
1012 /* now we need to terminate the worker threads */
1013 /* the value of t_abort is the signal we caught */
1014 int gtid;
1015
1016 KA_TRACE(10, ("__kmp_launch_monitor: terminate sig=%d\n",
1017 (__kmp_global.g.g_abort)));
1018
1019 /* terminate the OpenMP worker threads */
1020 /* TODO this is not valid for sibling threads!!
1021 * the uber master might not be 0 anymore.. */
1022 for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
1023 __kmp_terminate_thread(gtid);
1024
1025 __kmp_cleanup();
1026
1027 Sleep(0);
1028
1029 KA_TRACE(10,
1030 ("__kmp_launch_monitor: raise sig=%d\n", __kmp_global.g.g_abort));
1031
1032 if (__kmp_global.g.g_abort > 0) {
1033 raise(__kmp_global.g.g_abort);
1034 }
1035 }
1036
1037 TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
1038
1039 KMP_MB();
1040 return arg;
1041 }
1042 #endif
1043
__kmp_create_worker(int gtid,kmp_info_t * th,size_t stack_size)1044 void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size) {
1045 kmp_thread_t handle;
1046 DWORD idThread;
1047
1048 KA_TRACE(10, ("__kmp_create_worker: try to create thread (%d)\n", gtid));
1049
1050 th->th.th_info.ds.ds_gtid = gtid;
1051
1052 if (KMP_UBER_GTID(gtid)) {
1053 int stack_data;
1054
1055 /* TODO: GetCurrentThread() returns a pseudo-handle that is unsuitable for
1056 other threads to use. Is it appropriate to just use GetCurrentThread?
1057 When should we close this handle? When unregistering the root? */
1058 {
1059 BOOL rc;
1060 rc = DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
1061 GetCurrentProcess(), &th->th.th_info.ds.ds_thread, 0,
1062 FALSE, DUPLICATE_SAME_ACCESS);
1063 KMP_ASSERT(rc);
1064 KA_TRACE(10, (" __kmp_create_worker: ROOT Handle duplicated, th = %p, "
1065 "handle = %" KMP_UINTPTR_SPEC "\n",
1066 (LPVOID)th, th->th.th_info.ds.ds_thread));
1067 th->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1068 }
1069 if (TCR_4(__kmp_gtid_mode) < 2) { // check stack only if used to get gtid
1070 /* we will dynamically update the stack range if gtid_mode == 1 */
1071 TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
1072 TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
1073 TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
1074 __kmp_check_stack_overlap(th);
1075 }
1076 } else {
1077 KMP_MB(); /* Flush all pending memory write invalidates. */
1078
1079 /* Set stack size for this thread now. */
1080 KA_TRACE(10,
1081 ("__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC " bytes\n",
1082 stack_size));
1083
1084 stack_size += gtid * __kmp_stkoffset;
1085
1086 TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size);
1087 TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
1088
1089 KA_TRACE(10,
1090 ("__kmp_create_worker: (before) stack_size = %" KMP_SIZE_T_SPEC
1091 " bytes, &__kmp_launch_worker = %p, th = %p, &idThread = %p\n",
1092 (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
1093 (LPVOID)th, &idThread));
1094
1095 handle = CreateThread(
1096 NULL, (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)__kmp_launch_worker,
1097 (LPVOID)th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
1098
1099 KA_TRACE(10,
1100 ("__kmp_create_worker: (after) stack_size = %" KMP_SIZE_T_SPEC
1101 " bytes, &__kmp_launch_worker = %p, th = %p, "
1102 "idThread = %u, handle = %" KMP_UINTPTR_SPEC "\n",
1103 (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
1104 (LPVOID)th, idThread, handle));
1105
1106 if (handle == 0) {
1107 DWORD error = GetLastError();
1108 __kmp_fatal(KMP_MSG(CantCreateThread), KMP_ERR(error), __kmp_msg_null);
1109 } else {
1110 th->th.th_info.ds.ds_thread = handle;
1111 }
1112
1113 KMP_MB(); /* Flush all pending memory write invalidates. */
1114 }
1115
1116 KA_TRACE(10, ("__kmp_create_worker: done creating thread (%d)\n", gtid));
1117 }
1118
__kmp_still_running(kmp_info_t * th)1119 int __kmp_still_running(kmp_info_t *th) {
1120 return (WAIT_TIMEOUT == WaitForSingleObject(th->th.th_info.ds.ds_thread, 0));
1121 }
1122
1123 #if KMP_USE_MONITOR
__kmp_create_monitor(kmp_info_t * th)1124 void __kmp_create_monitor(kmp_info_t *th) {
1125 kmp_thread_t handle;
1126 DWORD idThread;
1127 int ideal, new_ideal;
1128
1129 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
1130 // We don't need monitor thread in case of MAX_BLOCKTIME
1131 KA_TRACE(10, ("__kmp_create_monitor: skipping monitor thread because of "
1132 "MAX blocktime\n"));
1133 th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
1134 th->th.th_info.ds.ds_gtid = 0;
1135 TCW_4(__kmp_init_monitor, 2); // Signal to stop waiting for monitor creation
1136 return;
1137 }
1138 KA_TRACE(10, ("__kmp_create_monitor: try to create monitor\n"));
1139
1140 KMP_MB(); /* Flush all pending memory write invalidates. */
1141
1142 __kmp_monitor_ev = CreateEvent(NULL, TRUE, FALSE, NULL);
1143 if (__kmp_monitor_ev == NULL) {
1144 DWORD error = GetLastError();
1145 __kmp_fatal(KMP_MSG(CantCreateEvent), KMP_ERR(error), __kmp_msg_null);
1146 }
1147 #if USE_ITT_BUILD
1148 __kmp_itt_system_object_created(__kmp_monitor_ev, "Event");
1149 #endif /* USE_ITT_BUILD */
1150
1151 th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1152 th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1153
1154 // FIXME - on Windows* OS, if __kmp_monitor_stksize = 0, figure out how
1155 // to automatically expand stacksize based on CreateThread error code.
1156 if (__kmp_monitor_stksize == 0) {
1157 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1158 }
1159 if (__kmp_monitor_stksize < __kmp_sys_min_stksize) {
1160 __kmp_monitor_stksize = __kmp_sys_min_stksize;
1161 }
1162
1163 KA_TRACE(10, ("__kmp_create_monitor: requested stacksize = %d bytes\n",
1164 (int)__kmp_monitor_stksize));
1165
1166 TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
1167
1168 handle =
1169 CreateThread(NULL, (SIZE_T)__kmp_monitor_stksize,
1170 (LPTHREAD_START_ROUTINE)__kmp_launch_monitor, (LPVOID)th,
1171 STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
1172 if (handle == 0) {
1173 DWORD error = GetLastError();
1174 __kmp_fatal(KMP_MSG(CantCreateThread), KMP_ERR(error), __kmp_msg_null);
1175 } else
1176 th->th.th_info.ds.ds_thread = handle;
1177
1178 KMP_MB(); /* Flush all pending memory write invalidates. */
1179
1180 KA_TRACE(10, ("__kmp_create_monitor: monitor created %p\n",
1181 (void *)th->th.th_info.ds.ds_thread));
1182 }
1183 #endif
1184
1185 /* Check to see if thread is still alive.
1186 NOTE: The ExitProcess(code) system call causes all threads to Terminate
1187 with a exit_val = code. Because of this we can not rely on exit_val having
1188 any particular value. So this routine may return STILL_ALIVE in exit_val
1189 even after the thread is dead. */
1190
__kmp_is_thread_alive(kmp_info_t * th,DWORD * exit_val)1191 int __kmp_is_thread_alive(kmp_info_t *th, DWORD *exit_val) {
1192 DWORD rc;
1193 rc = GetExitCodeThread(th->th.th_info.ds.ds_thread, exit_val);
1194 if (rc == 0) {
1195 DWORD error = GetLastError();
1196 __kmp_fatal(KMP_MSG(FunctionError, "GetExitCodeThread()"), KMP_ERR(error),
1197 __kmp_msg_null);
1198 }
1199 return (*exit_val == STILL_ACTIVE);
1200 }
1201
__kmp_exit_thread(int exit_status)1202 void __kmp_exit_thread(int exit_status) {
1203 ExitThread(exit_status);
1204 } // __kmp_exit_thread
1205
1206 // This is a common part for both __kmp_reap_worker() and __kmp_reap_monitor().
__kmp_reap_common(kmp_info_t * th)1207 static void __kmp_reap_common(kmp_info_t *th) {
1208 DWORD exit_val;
1209
1210 KMP_MB(); /* Flush all pending memory write invalidates. */
1211
1212 KA_TRACE(
1213 10, ("__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid));
1214
1215 /* 2006-10-19:
1216 There are two opposite situations:
1217 1. Windows* OS keep thread alive after it resets ds_alive flag and
1218 exits from thread function. (For example, see C70770/Q394281 "unloading of
1219 dll based on OMP is very slow".)
1220 2. Windows* OS may kill thread before it resets ds_alive flag.
1221
1222 Right solution seems to be waiting for *either* thread termination *or*
1223 ds_alive resetting. */
1224 {
1225 // TODO: This code is very similar to KMP_WAIT_YIELD. Need to generalize
1226 // KMP_WAIT_YIELD to cover this usage also.
1227 void *obj = NULL;
1228 kmp_uint32 spins;
1229 #if USE_ITT_BUILD
1230 KMP_FSYNC_SPIN_INIT(obj, (void *)&th->th.th_info.ds.ds_alive);
1231 #endif /* USE_ITT_BUILD */
1232 KMP_INIT_YIELD(spins);
1233 do {
1234 #if USE_ITT_BUILD
1235 KMP_FSYNC_SPIN_PREPARE(obj);
1236 #endif /* USE_ITT_BUILD */
1237 __kmp_is_thread_alive(th, &exit_val);
1238 KMP_YIELD(TCR_4(__kmp_nth) > __kmp_avail_proc);
1239 KMP_YIELD_SPIN(spins);
1240 } while (exit_val == STILL_ACTIVE && TCR_4(th->th.th_info.ds.ds_alive));
1241 #if USE_ITT_BUILD
1242 if (exit_val == STILL_ACTIVE) {
1243 KMP_FSYNC_CANCEL(obj);
1244 } else {
1245 KMP_FSYNC_SPIN_ACQUIRED(obj);
1246 }
1247 #endif /* USE_ITT_BUILD */
1248 }
1249
1250 __kmp_free_handle(th->th.th_info.ds.ds_thread);
1251
1252 /* NOTE: The ExitProcess(code) system call causes all threads to Terminate
1253 with a exit_val = code. Because of this we can not rely on exit_val having
1254 any particular value. */
1255 if (exit_val == STILL_ACTIVE) {
1256 KA_TRACE(1, ("__kmp_reap_common: thread still active.\n"));
1257 } else if ((void *)exit_val != (void *)th) {
1258 KA_TRACE(1, ("__kmp_reap_common: ExitProcess / TerminateThread used?\n"));
1259 }
1260
1261 KA_TRACE(10,
1262 ("__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC
1263 "\n",
1264 th->th.th_info.ds.ds_gtid, th->th.th_info.ds.ds_thread));
1265
1266 th->th.th_info.ds.ds_thread = 0;
1267 th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1268 th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1269 th->th.th_info.ds.ds_thread_id = 0;
1270
1271 KMP_MB(); /* Flush all pending memory write invalidates. */
1272 }
1273
1274 #if KMP_USE_MONITOR
__kmp_reap_monitor(kmp_info_t * th)1275 void __kmp_reap_monitor(kmp_info_t *th) {
1276 int status;
1277
1278 KA_TRACE(10, ("__kmp_reap_monitor: try to reap %p\n",
1279 (void *)th->th.th_info.ds.ds_thread));
1280
1281 // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1282 // If both tid and gtid are 0, it means the monitor did not ever start.
1283 // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1284 KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid);
1285 if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) {
1286 KA_TRACE(10, ("__kmp_reap_monitor: monitor did not start, returning\n"));
1287 return;
1288 }
1289
1290 KMP_MB(); /* Flush all pending memory write invalidates. */
1291
1292 status = SetEvent(__kmp_monitor_ev);
1293 if (status == FALSE) {
1294 DWORD error = GetLastError();
1295 __kmp_fatal(KMP_MSG(CantSetEvent), KMP_ERR(error), __kmp_msg_null);
1296 }
1297 KA_TRACE(10, ("__kmp_reap_monitor: reaping thread (%d)\n",
1298 th->th.th_info.ds.ds_gtid));
1299 __kmp_reap_common(th);
1300
1301 __kmp_free_handle(__kmp_monitor_ev);
1302
1303 KMP_MB(); /* Flush all pending memory write invalidates. */
1304 }
1305 #endif
1306
__kmp_reap_worker(kmp_info_t * th)1307 void __kmp_reap_worker(kmp_info_t *th) {
1308 KA_TRACE(10, ("__kmp_reap_worker: reaping thread (%d)\n",
1309 th->th.th_info.ds.ds_gtid));
1310 __kmp_reap_common(th);
1311 }
1312
1313 #if KMP_HANDLE_SIGNALS
1314
__kmp_team_handler(int signo)1315 static void __kmp_team_handler(int signo) {
1316 if (__kmp_global.g.g_abort == 0) {
1317 // Stage 1 signal handler, let's shut down all of the threads.
1318 if (__kmp_debug_buf) {
1319 __kmp_dump_debug_buffer();
1320 }
1321 KMP_MB(); // Flush all pending memory write invalidates.
1322 TCW_4(__kmp_global.g.g_abort, signo);
1323 KMP_MB(); // Flush all pending memory write invalidates.
1324 TCW_4(__kmp_global.g.g_done, TRUE);
1325 KMP_MB(); // Flush all pending memory write invalidates.
1326 }
1327 } // __kmp_team_handler
1328
__kmp_signal(int signum,sig_func_t handler)1329 static sig_func_t __kmp_signal(int signum, sig_func_t handler) {
1330 sig_func_t old = signal(signum, handler);
1331 if (old == SIG_ERR) {
1332 int error = errno;
1333 __kmp_fatal(KMP_MSG(FunctionError, "signal"), KMP_ERR(error),
1334 __kmp_msg_null);
1335 }
1336 return old;
1337 }
1338
__kmp_install_one_handler(int sig,sig_func_t handler,int parallel_init)1339 static void __kmp_install_one_handler(int sig, sig_func_t handler,
1340 int parallel_init) {
1341 sig_func_t old;
1342 KMP_MB(); /* Flush all pending memory write invalidates. */
1343 KB_TRACE(60, ("__kmp_install_one_handler: called: sig=%d\n", sig));
1344 if (parallel_init) {
1345 old = __kmp_signal(sig, handler);
1346 // SIG_DFL on Windows* OS in NULL or 0.
1347 if (old == __kmp_sighldrs[sig]) {
1348 __kmp_siginstalled[sig] = 1;
1349 } else { // Restore/keep user's handler if one previously installed.
1350 old = __kmp_signal(sig, old);
1351 }
1352 } else {
1353 // Save initial/system signal handlers to see if user handlers installed.
1354 // 2009-09-23: It is a dead code. On Windows* OS __kmp_install_signals
1355 // called once with parallel_init == TRUE.
1356 old = __kmp_signal(sig, SIG_DFL);
1357 __kmp_sighldrs[sig] = old;
1358 __kmp_signal(sig, old);
1359 }
1360 KMP_MB(); /* Flush all pending memory write invalidates. */
1361 } // __kmp_install_one_handler
1362
__kmp_remove_one_handler(int sig)1363 static void __kmp_remove_one_handler(int sig) {
1364 if (__kmp_siginstalled[sig]) {
1365 sig_func_t old;
1366 KMP_MB(); // Flush all pending memory write invalidates.
1367 KB_TRACE(60, ("__kmp_remove_one_handler: called: sig=%d\n", sig));
1368 old = __kmp_signal(sig, __kmp_sighldrs[sig]);
1369 if (old != __kmp_team_handler) {
1370 KB_TRACE(10, ("__kmp_remove_one_handler: oops, not our handler, "
1371 "restoring: sig=%d\n",
1372 sig));
1373 old = __kmp_signal(sig, old);
1374 }
1375 __kmp_sighldrs[sig] = NULL;
1376 __kmp_siginstalled[sig] = 0;
1377 KMP_MB(); // Flush all pending memory write invalidates.
1378 }
1379 } // __kmp_remove_one_handler
1380
__kmp_install_signals(int parallel_init)1381 void __kmp_install_signals(int parallel_init) {
1382 KB_TRACE(10, ("__kmp_install_signals: called\n"));
1383 if (!__kmp_handle_signals) {
1384 KB_TRACE(10, ("__kmp_install_signals: KMP_HANDLE_SIGNALS is false - "
1385 "handlers not installed\n"));
1386 return;
1387 }
1388 __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init);
1389 __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init);
1390 __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init);
1391 __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init);
1392 __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init);
1393 __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init);
1394 } // __kmp_install_signals
1395
__kmp_remove_signals(void)1396 void __kmp_remove_signals(void) {
1397 int sig;
1398 KB_TRACE(10, ("__kmp_remove_signals: called\n"));
1399 for (sig = 1; sig < NSIG; ++sig) {
1400 __kmp_remove_one_handler(sig);
1401 }
1402 } // __kmp_remove_signals
1403
1404 #endif // KMP_HANDLE_SIGNALS
1405
1406 /* Put the thread to sleep for a time period */
__kmp_thread_sleep(int millis)1407 void __kmp_thread_sleep(int millis) {
1408 DWORD status;
1409
1410 status = SleepEx((DWORD)millis, FALSE);
1411 if (status) {
1412 DWORD error = GetLastError();
1413 __kmp_fatal(KMP_MSG(FunctionError, "SleepEx()"), KMP_ERR(error),
1414 __kmp_msg_null);
1415 }
1416 }
1417
1418 // Determine whether the given address is mapped into the current address space.
__kmp_is_address_mapped(void * addr)1419 int __kmp_is_address_mapped(void *addr) {
1420 DWORD status;
1421 MEMORY_BASIC_INFORMATION lpBuffer;
1422 SIZE_T dwLength;
1423
1424 dwLength = sizeof(MEMORY_BASIC_INFORMATION);
1425
1426 status = VirtualQuery(addr, &lpBuffer, dwLength);
1427
1428 return !(((lpBuffer.State == MEM_RESERVE) || (lpBuffer.State == MEM_FREE)) ||
1429 ((lpBuffer.Protect == PAGE_NOACCESS) ||
1430 (lpBuffer.Protect == PAGE_EXECUTE)));
1431 }
1432
__kmp_hardware_timestamp(void)1433 kmp_uint64 __kmp_hardware_timestamp(void) {
1434 kmp_uint64 r = 0;
1435
1436 QueryPerformanceCounter((LARGE_INTEGER *)&r);
1437 return r;
1438 }
1439
1440 /* Free handle and check the error code */
__kmp_free_handle(kmp_thread_t tHandle)1441 void __kmp_free_handle(kmp_thread_t tHandle) {
1442 /* called with parameter type HANDLE also, thus suppose kmp_thread_t defined
1443 * as HANDLE */
1444 BOOL rc;
1445 rc = CloseHandle(tHandle);
1446 if (!rc) {
1447 DWORD error = GetLastError();
1448 __kmp_fatal(KMP_MSG(CantCloseHandle), KMP_ERR(error), __kmp_msg_null);
1449 }
1450 }
1451
__kmp_get_load_balance(int max)1452 int __kmp_get_load_balance(int max) {
1453 static ULONG glb_buff_size = 100 * 1024;
1454
1455 // Saved count of the running threads for the thread balance algortihm
1456 static int glb_running_threads = 0;
1457 static double glb_call_time = 0; /* Thread balance algorithm call time */
1458
1459 int running_threads = 0; // Number of running threads in the system.
1460 NTSTATUS status = 0;
1461 ULONG buff_size = 0;
1462 ULONG info_size = 0;
1463 void *buffer = NULL;
1464 PSYSTEM_PROCESS_INFORMATION spi = NULL;
1465 int first_time = 1;
1466
1467 double call_time = 0.0; // start, finish;
1468
1469 __kmp_elapsed(&call_time);
1470
1471 if (glb_call_time &&
1472 (call_time - glb_call_time < __kmp_load_balance_interval)) {
1473 running_threads = glb_running_threads;
1474 goto finish;
1475 }
1476 glb_call_time = call_time;
1477
1478 // Do not spend time on running algorithm if we have a permanent error.
1479 if (NtQuerySystemInformation == NULL) {
1480 running_threads = -1;
1481 goto finish;
1482 }
1483
1484 if (max <= 0) {
1485 max = INT_MAX;
1486 }
1487
1488 do {
1489
1490 if (first_time) {
1491 buff_size = glb_buff_size;
1492 } else {
1493 buff_size = 2 * buff_size;
1494 }
1495
1496 buffer = KMP_INTERNAL_REALLOC(buffer, buff_size);
1497 if (buffer == NULL) {
1498 running_threads = -1;
1499 goto finish;
1500 }
1501 status = NtQuerySystemInformation(SystemProcessInformation, buffer,
1502 buff_size, &info_size);
1503 first_time = 0;
1504
1505 } while (status == STATUS_INFO_LENGTH_MISMATCH);
1506 glb_buff_size = buff_size;
1507
1508 #define CHECK(cond) \
1509 { \
1510 KMP_DEBUG_ASSERT(cond); \
1511 if (!(cond)) { \
1512 running_threads = -1; \
1513 goto finish; \
1514 } \
1515 }
1516
1517 CHECK(buff_size >= info_size);
1518 spi = PSYSTEM_PROCESS_INFORMATION(buffer);
1519 for (;;) {
1520 ptrdiff_t offset = uintptr_t(spi) - uintptr_t(buffer);
1521 CHECK(0 <= offset &&
1522 offset + sizeof(SYSTEM_PROCESS_INFORMATION) < info_size);
1523 HANDLE pid = spi->ProcessId;
1524 ULONG num = spi->NumberOfThreads;
1525 CHECK(num >= 1);
1526 size_t spi_size =
1527 sizeof(SYSTEM_PROCESS_INFORMATION) + sizeof(SYSTEM_THREAD) * (num - 1);
1528 CHECK(offset + spi_size <
1529 info_size); // Make sure process info record fits the buffer.
1530 if (spi->NextEntryOffset != 0) {
1531 CHECK(spi_size <=
1532 spi->NextEntryOffset); // And do not overlap with the next record.
1533 }
1534 // pid == 0 corresponds to the System Idle Process. It always has running
1535 // threads on all cores. So, we don't consider the running threads of this
1536 // process.
1537 if (pid != 0) {
1538 for (int i = 0; i < num; ++i) {
1539 THREAD_STATE state = spi->Threads[i].State;
1540 // Count threads that have Ready or Running state.
1541 // !!! TODO: Why comment does not match the code???
1542 if (state == StateRunning) {
1543 ++running_threads;
1544 // Stop counting running threads if the number is already greater than
1545 // the number of available cores
1546 if (running_threads >= max) {
1547 goto finish;
1548 }
1549 }
1550 }
1551 }
1552 if (spi->NextEntryOffset == 0) {
1553 break;
1554 }
1555 spi = PSYSTEM_PROCESS_INFORMATION(uintptr_t(spi) + spi->NextEntryOffset);
1556 }
1557
1558 #undef CHECK
1559
1560 finish: // Clean up and exit.
1561
1562 if (buffer != NULL) {
1563 KMP_INTERNAL_FREE(buffer);
1564 }
1565
1566 glb_running_threads = running_threads;
1567
1568 return running_threads;
1569 } //__kmp_get_load_balance()
1570