1 /*! \file */ 2 /* 3 * kmp.h -- KPTS runtime header file. 4 */ 5 6 7 //===----------------------------------------------------------------------===// 8 // 9 // The LLVM Compiler Infrastructure 10 // 11 // This file is dual licensed under the MIT and the University of Illinois Open 12 // Source Licenses. See LICENSE.txt for details. 13 // 14 //===----------------------------------------------------------------------===// 15 16 17 #ifndef KMP_H 18 #define KMP_H 19 20 #include "kmp_config.h" 21 22 /* #define BUILD_PARALLEL_ORDERED 1 */ 23 24 /* This fix replaces gettimeofday with clock_gettime for better scalability on 25 the Altix. Requires user code to be linked with -lrt. 26 */ 27 //#define FIX_SGI_CLOCK 28 29 /* Defines for OpenMP 3.0 tasking and auto scheduling */ 30 31 # ifndef KMP_STATIC_STEAL_ENABLED 32 # define KMP_STATIC_STEAL_ENABLED 1 33 # endif 34 35 #define TASK_CURRENT_NOT_QUEUED 0 36 #define TASK_CURRENT_QUEUED 1 37 38 #define TASK_DEQUE_BITS 8 // Used solely to define TASK_DEQUE_SIZE and TASK_DEQUE_MASK. 39 #define TASK_DEQUE_SIZE ( 1 << TASK_DEQUE_BITS ) 40 #define TASK_DEQUE_MASK ( TASK_DEQUE_SIZE - 1 ) 41 42 #ifdef BUILD_TIED_TASK_STACK 43 #define TASK_STACK_EMPTY 0 // entries when the stack is empty 44 45 #define TASK_STACK_BLOCK_BITS 5 // Used to define TASK_STACK_SIZE and TASK_STACK_MASK 46 #define TASK_STACK_BLOCK_SIZE ( 1 << TASK_STACK_BLOCK_BITS ) // Number of entries in each task stack array 47 #define TASK_STACK_INDEX_MASK ( TASK_STACK_BLOCK_SIZE - 1 ) // Mask for determining index into stack block 48 #endif // BUILD_TIED_TASK_STACK 49 50 #define TASK_NOT_PUSHED 1 51 #define TASK_SUCCESSFULLY_PUSHED 0 52 #define TASK_TIED 1 53 #define TASK_UNTIED 0 54 #define TASK_EXPLICIT 1 55 #define TASK_IMPLICIT 0 56 #define TASK_PROXY 1 57 #define TASK_FULL 0 58 59 #define KMP_CANCEL_THREADS 60 #define KMP_THREAD_ATTR 61 62 #include <stdio.h> 63 #include <stdlib.h> 64 #include <stddef.h> 65 #include <stdarg.h> 66 #include <string.h> 67 #include <signal.h> 68 /* include <ctype.h> don't use; problems with /MD on Windows* OS NT due to bad Microsoft library */ 69 /* some macros provided below to replace some of these functions */ 70 #ifndef __ABSOFT_WIN 71 #include <sys/types.h> 72 #endif 73 #include <limits.h> 74 #include <time.h> 75 76 #include <errno.h> 77 78 #include "kmp_os.h" 79 80 #include "kmp_safe_c_api.h" 81 82 #if KMP_STATS_ENABLED 83 class kmp_stats_list; 84 #endif 85 86 #if KMP_USE_HWLOC 87 #include "hwloc.h" 88 extern hwloc_topology_t __kmp_hwloc_topology; 89 extern int __kmp_hwloc_error; 90 #endif 91 92 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 93 #include <xmmintrin.h> 94 #endif 95 96 #include "kmp_version.h" 97 #include "kmp_debug.h" 98 #include "kmp_lock.h" 99 #if USE_DEBUGGER 100 #include "kmp_debugger.h" 101 #endif 102 #include "kmp_i18n.h" 103 104 #define KMP_HANDLE_SIGNALS (KMP_OS_UNIX || KMP_OS_WINDOWS) 105 106 #include "kmp_wrapper_malloc.h" 107 #if KMP_OS_UNIX 108 # include <unistd.h> 109 # if !defined NSIG && defined _NSIG 110 # define NSIG _NSIG 111 # endif 112 #endif 113 114 #if KMP_OS_LINUX 115 # pragma weak clock_gettime 116 #endif 117 118 #if OMPT_SUPPORT 119 #include "ompt-internal.h" 120 #endif 121 122 /*Select data placement in NUMA memory */ 123 #define NO_FIRST_TOUCH 0 124 #define FIRST_TOUCH 1 /* Exploit SGI's first touch page placement algo */ 125 126 /* If not specified on compile command line, assume no first touch */ 127 #ifndef BUILD_MEMORY 128 #define BUILD_MEMORY NO_FIRST_TOUCH 129 #endif 130 131 // 0 - no fast memory allocation, alignment: 8-byte on x86, 16-byte on x64. 132 // 3 - fast allocation using sync, non-sync free lists of any size, non-self free lists of limited size. 133 #ifndef USE_FAST_MEMORY 134 #define USE_FAST_MEMORY 3 135 #endif 136 137 #ifndef KMP_NESTED_HOT_TEAMS 138 # define KMP_NESTED_HOT_TEAMS 0 139 # define USE_NESTED_HOT_ARG(x) 140 #else 141 # if KMP_NESTED_HOT_TEAMS 142 # if OMP_40_ENABLED 143 # define USE_NESTED_HOT_ARG(x) ,x 144 # else 145 // Nested hot teams feature depends on omp 4.0, disable it for earlier versions 146 # undef KMP_NESTED_HOT_TEAMS 147 # define KMP_NESTED_HOT_TEAMS 0 148 # define USE_NESTED_HOT_ARG(x) 149 # endif 150 # else 151 # define USE_NESTED_HOT_ARG(x) 152 # endif 153 #endif 154 155 // Assume using BGET compare_exchange instruction instead of lock by default. 156 #ifndef USE_CMP_XCHG_FOR_BGET 157 #define USE_CMP_XCHG_FOR_BGET 1 158 #endif 159 160 // Test to see if queuing lock is better than bootstrap lock for bget 161 // #ifndef USE_QUEUING_LOCK_FOR_BGET 162 // #define USE_QUEUING_LOCK_FOR_BGET 163 // #endif 164 165 #define KMP_NSEC_PER_SEC 1000000000L 166 #define KMP_USEC_PER_SEC 1000000L 167 168 /*! 169 @ingroup BASIC_TYPES 170 @{ 171 */ 172 173 // FIXME DOXYGEN... need to group these flags somehow (Making them an anonymous enum would do it...) 174 /*! 175 Values for bit flags used in the ident_t to describe the fields. 176 */ 177 /*! Use trampoline for internal microtasks */ 178 #define KMP_IDENT_IMB 0x01 179 /*! Use c-style ident structure */ 180 #define KMP_IDENT_KMPC 0x02 181 /* 0x04 is no longer used */ 182 /*! Entry point generated by auto-parallelization */ 183 #define KMP_IDENT_AUTOPAR 0x08 184 /*! Compiler generates atomic reduction option for kmpc_reduce* */ 185 #define KMP_IDENT_ATOMIC_REDUCE 0x10 186 /*! To mark a 'barrier' directive in user code */ 187 #define KMP_IDENT_BARRIER_EXPL 0x20 188 /*! To Mark implicit barriers. */ 189 #define KMP_IDENT_BARRIER_IMPL 0x0040 190 #define KMP_IDENT_BARRIER_IMPL_MASK 0x01C0 191 #define KMP_IDENT_BARRIER_IMPL_FOR 0x0040 192 #define KMP_IDENT_BARRIER_IMPL_SECTIONS 0x00C0 193 194 #define KMP_IDENT_BARRIER_IMPL_SINGLE 0x0140 195 #define KMP_IDENT_BARRIER_IMPL_WORKSHARE 0x01C0 196 197 /*! 198 * The ident structure that describes a source location. 199 */ 200 typedef struct ident { 201 kmp_int32 reserved_1; /**< might be used in Fortran; see above */ 202 kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags; KMP_IDENT_KMPC identifies this union member */ 203 kmp_int32 reserved_2; /**< not really used in Fortran any more; see above */ 204 #if USE_ITT_BUILD 205 /* but currently used for storing region-specific ITT */ 206 /* contextual information. */ 207 #endif /* USE_ITT_BUILD */ 208 kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for C++ */ 209 char const *psource; /**< String describing the source location. 210 The string is composed of semi-colon separated fields which describe the source file, 211 the function and a pair of line numbers that delimit the construct. 212 */ 213 } ident_t; 214 /*! 215 @} 216 */ 217 218 // Some forward declarations. 219 220 typedef union kmp_team kmp_team_t; 221 typedef struct kmp_taskdata kmp_taskdata_t; 222 typedef union kmp_task_team kmp_task_team_t; 223 typedef union kmp_team kmp_team_p; 224 typedef union kmp_info kmp_info_p; 225 typedef union kmp_root kmp_root_p; 226 227 #ifdef __cplusplus 228 extern "C" { 229 #endif 230 231 /* ------------------------------------------------------------------------ */ 232 /* ------------------------------------------------------------------------ */ 233 234 /* Pack two 32-bit signed integers into a 64-bit signed integer */ 235 /* ToDo: Fix word ordering for big-endian machines. */ 236 #define KMP_PACK_64(HIGH_32,LOW_32) \ 237 ( (kmp_int64) ((((kmp_uint64)(HIGH_32))<<32) | (kmp_uint64)(LOW_32)) ) 238 239 240 /* 241 * Generic string manipulation macros. 242 * Assume that _x is of type char * 243 */ 244 #define SKIP_WS(_x) { while (*(_x) == ' ' || *(_x) == '\t') (_x)++; } 245 #define SKIP_DIGITS(_x) { while (*(_x) >= '0' && *(_x) <= '9') (_x)++; } 246 #define SKIP_TO(_x,_c) { while (*(_x) != '\0' && *(_x) != (_c)) (_x)++; } 247 248 /* ------------------------------------------------------------------------ */ 249 /* ------------------------------------------------------------------------ */ 250 251 #define KMP_MAX( x, y ) ( (x) > (y) ? (x) : (y) ) 252 #define KMP_MIN( x, y ) ( (x) < (y) ? (x) : (y) ) 253 254 /* ------------------------------------------------------------------------ */ 255 /* ------------------------------------------------------------------------ */ 256 257 258 /* Enumeration types */ 259 260 enum kmp_state_timer { 261 ts_stop, 262 ts_start, 263 ts_pause, 264 265 ts_last_state 266 }; 267 268 enum dynamic_mode { 269 dynamic_default, 270 #ifdef USE_LOAD_BALANCE 271 dynamic_load_balance, 272 #endif /* USE_LOAD_BALANCE */ 273 dynamic_random, 274 dynamic_thread_limit, 275 dynamic_max 276 }; 277 278 /* external schedule constants, duplicate enum omp_sched in omp.h in order to not include it here */ 279 #ifndef KMP_SCHED_TYPE_DEFINED 280 #define KMP_SCHED_TYPE_DEFINED 281 typedef enum kmp_sched { 282 kmp_sched_lower = 0, // lower and upper bounds are for routine parameter check 283 // Note: need to adjust __kmp_sch_map global array in case this enum is changed 284 kmp_sched_static = 1, // mapped to kmp_sch_static_chunked (33) 285 kmp_sched_dynamic = 2, // mapped to kmp_sch_dynamic_chunked (35) 286 kmp_sched_guided = 3, // mapped to kmp_sch_guided_chunked (36) 287 kmp_sched_auto = 4, // mapped to kmp_sch_auto (38) 288 kmp_sched_upper_std = 5, // upper bound for standard schedules 289 kmp_sched_lower_ext = 100, // lower bound of Intel extension schedules 290 kmp_sched_trapezoidal = 101, // mapped to kmp_sch_trapezoidal (39) 291 // kmp_sched_static_steal = 102, // mapped to kmp_sch_static_steal (44) 292 kmp_sched_upper = 102, 293 kmp_sched_default = kmp_sched_static // default scheduling 294 } kmp_sched_t; 295 #endif 296 297 /*! 298 @ingroup WORK_SHARING 299 * Describes the loop schedule to be used for a parallel for loop. 300 */ 301 enum sched_type { 302 kmp_sch_lower = 32, /**< lower bound for unordered values */ 303 kmp_sch_static_chunked = 33, 304 kmp_sch_static = 34, /**< static unspecialized */ 305 kmp_sch_dynamic_chunked = 35, 306 kmp_sch_guided_chunked = 36, /**< guided unspecialized */ 307 kmp_sch_runtime = 37, 308 kmp_sch_auto = 38, /**< auto */ 309 kmp_sch_trapezoidal = 39, 310 311 /* accessible only through KMP_SCHEDULE environment variable */ 312 kmp_sch_static_greedy = 40, 313 kmp_sch_static_balanced = 41, 314 /* accessible only through KMP_SCHEDULE environment variable */ 315 kmp_sch_guided_iterative_chunked = 42, 316 kmp_sch_guided_analytical_chunked = 43, 317 318 kmp_sch_static_steal = 44, /**< accessible only through KMP_SCHEDULE environment variable */ 319 320 /* accessible only through KMP_SCHEDULE environment variable */ 321 kmp_sch_upper = 45, /**< upper bound for unordered values */ 322 323 kmp_ord_lower = 64, /**< lower bound for ordered values, must be power of 2 */ 324 kmp_ord_static_chunked = 65, 325 kmp_ord_static = 66, /**< ordered static unspecialized */ 326 kmp_ord_dynamic_chunked = 67, 327 kmp_ord_guided_chunked = 68, 328 kmp_ord_runtime = 69, 329 kmp_ord_auto = 70, /**< ordered auto */ 330 kmp_ord_trapezoidal = 71, 331 kmp_ord_upper = 72, /**< upper bound for ordered values */ 332 333 #if OMP_40_ENABLED 334 /* Schedules for Distribute construct */ 335 kmp_distribute_static_chunked = 91, /**< distribute static chunked */ 336 kmp_distribute_static = 92, /**< distribute static unspecialized */ 337 #endif 338 339 /* 340 * For the "nomerge" versions, kmp_dispatch_next*() will always return 341 * a single iteration/chunk, even if the loop is serialized. For the 342 * schedule types listed above, the entire iteration vector is returned 343 * if the loop is serialized. This doesn't work for gcc/gcomp sections. 344 */ 345 kmp_nm_lower = 160, /**< lower bound for nomerge values */ 346 347 kmp_nm_static_chunked = (kmp_sch_static_chunked - kmp_sch_lower + kmp_nm_lower), 348 kmp_nm_static = 162, /**< static unspecialized */ 349 kmp_nm_dynamic_chunked = 163, 350 kmp_nm_guided_chunked = 164, /**< guided unspecialized */ 351 kmp_nm_runtime = 165, 352 kmp_nm_auto = 166, /**< auto */ 353 kmp_nm_trapezoidal = 167, 354 355 /* accessible only through KMP_SCHEDULE environment variable */ 356 kmp_nm_static_greedy = 168, 357 kmp_nm_static_balanced = 169, 358 /* accessible only through KMP_SCHEDULE environment variable */ 359 kmp_nm_guided_iterative_chunked = 170, 360 kmp_nm_guided_analytical_chunked = 171, 361 kmp_nm_static_steal = 172, /* accessible only through OMP_SCHEDULE environment variable */ 362 363 kmp_nm_ord_static_chunked = 193, 364 kmp_nm_ord_static = 194, /**< ordered static unspecialized */ 365 kmp_nm_ord_dynamic_chunked = 195, 366 kmp_nm_ord_guided_chunked = 196, 367 kmp_nm_ord_runtime = 197, 368 kmp_nm_ord_auto = 198, /**< auto */ 369 kmp_nm_ord_trapezoidal = 199, 370 kmp_nm_upper = 200, /**< upper bound for nomerge values */ 371 372 #if OMP_41_ENABLED 373 /* Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers. 374 * Since we need to distinguish the three possible cases (no modifier, monotonic modifier, 375 * nonmonotonic modifier), we need separate bits for each modifier. 376 * The absence of monotonic does not imply nonmonotonic, especially since 4.5 says 377 * that the behaviour of the "no modifier" case is implementation defined in 4.5, 378 * but will become "nonmonotonic" in 5.0. 379 * 380 * Since we're passing a full 32 bit value, we can use a couple of high bits for these 381 * flags; out of paranoia we avoid the sign bit. 382 * 383 * These modifiers can be or-ed into non-static schedules by the compiler to pass 384 * the additional information. 385 * They will be stripped early in the processing in __kmp_dispatch_init when setting up schedules, so 386 * most of the code won't ever see schedules with these bits set. 387 */ 388 kmp_sch_modifier_monotonic = (1<<29), /**< Set if the monotonic schedule modifier was present */ 389 kmp_sch_modifier_nonmonotonic = (1<<30), /**< Set if the nonmonotonic schedule modifier was present */ 390 391 # define SCHEDULE_WITHOUT_MODIFIERS(s) (enum sched_type)((s) & ~ (kmp_sch_modifier_nonmonotonic | kmp_sch_modifier_monotonic)) 392 # define SCHEDULE_HAS_MONOTONIC(s) (((s) & kmp_sch_modifier_monotonic) != 0) 393 # define SCHEDULE_HAS_NONMONOTONIC(s) (((s) & kmp_sch_modifier_nonmonotonic) != 0) 394 # define SCHEDULE_HAS_NO_MODIFIERS(s) (((s) & (kmp_sch_modifier_nonmonotonic | kmp_sch_modifier_monotonic)) == 0) 395 #else 396 /* By doing this we hope to avoid multiple tests on OMP_41_ENABLED. Compilers can now eliminate tests on compile time 397 * constants and dead code that results from them, so we can leave code guarded by such an if in place. 398 */ 399 # define SCHEDULE_WITHOUT_MODIFIERS(s) (s) 400 # define SCHEDULE_HAS_MONOTONIC(s) false 401 # define SCHEDULE_HAS_NONMONOTONIC(s) false 402 # define SCHEDULE_HAS_NO_MODIFIERS(s) true 403 #endif 404 405 kmp_sch_default = kmp_sch_static /**< default scheduling algorithm */ 406 }; 407 408 /* Type to keep runtime schedule set via OMP_SCHEDULE or omp_set_schedule() */ 409 typedef struct kmp_r_sched { 410 enum sched_type r_sched_type; 411 int chunk; 412 } kmp_r_sched_t; 413 414 extern enum sched_type __kmp_sch_map[]; // map OMP 3.0 schedule types with our internal schedule types 415 416 enum library_type { 417 library_none, 418 library_serial, 419 library_turnaround, 420 library_throughput 421 }; 422 423 #if KMP_OS_LINUX 424 enum clock_function_type { 425 clock_function_gettimeofday, 426 clock_function_clock_gettime 427 }; 428 #endif /* KMP_OS_LINUX */ 429 430 #if KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_WINDOWS) 431 enum mic_type { 432 non_mic, 433 mic1, 434 mic2, 435 mic3, 436 dummy 437 }; 438 #endif 439 440 /* ------------------------------------------------------------------------ */ 441 /* -- fast reduction stuff ------------------------------------------------ */ 442 443 #undef KMP_FAST_REDUCTION_BARRIER 444 #define KMP_FAST_REDUCTION_BARRIER 1 445 446 #undef KMP_FAST_REDUCTION_CORE_DUO 447 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 448 #define KMP_FAST_REDUCTION_CORE_DUO 1 449 #endif 450 451 enum _reduction_method { 452 reduction_method_not_defined = 0, 453 critical_reduce_block = ( 1 << 8 ), 454 atomic_reduce_block = ( 2 << 8 ), 455 tree_reduce_block = ( 3 << 8 ), 456 empty_reduce_block = ( 4 << 8 ) 457 }; 458 459 // description of the packed_reduction_method variable 460 // the packed_reduction_method variable consists of two enum types variables that are packed together into 0-th byte and 1-st byte: 461 // 0: ( packed_reduction_method & 0x000000FF ) is a 'enum barrier_type' value of barrier that will be used in fast reduction: bs_plain_barrier or bs_reduction_barrier 462 // 1: ( packed_reduction_method & 0x0000FF00 ) is a reduction method that will be used in fast reduction; 463 // reduction method is of 'enum _reduction_method' type and it's defined the way so that the bits of 0-th byte are empty, 464 // so no need to execute a shift instruction while packing/unpacking 465 466 #if KMP_FAST_REDUCTION_BARRIER 467 #define PACK_REDUCTION_METHOD_AND_BARRIER(reduction_method,barrier_type) \ 468 ( ( reduction_method ) | ( barrier_type ) ) 469 470 #define UNPACK_REDUCTION_METHOD(packed_reduction_method) \ 471 ( ( enum _reduction_method )( ( packed_reduction_method ) & ( 0x0000FF00 ) ) ) 472 473 #define UNPACK_REDUCTION_BARRIER(packed_reduction_method) \ 474 ( ( enum barrier_type )( ( packed_reduction_method ) & ( 0x000000FF ) ) ) 475 #else 476 #define PACK_REDUCTION_METHOD_AND_BARRIER(reduction_method,barrier_type) \ 477 ( reduction_method ) 478 479 #define UNPACK_REDUCTION_METHOD(packed_reduction_method) \ 480 ( packed_reduction_method ) 481 482 #define UNPACK_REDUCTION_BARRIER(packed_reduction_method) \ 483 ( bs_plain_barrier ) 484 #endif 485 486 #define TEST_REDUCTION_METHOD(packed_reduction_method,which_reduction_block) \ 487 ( ( UNPACK_REDUCTION_METHOD( packed_reduction_method ) ) == ( which_reduction_block ) ) 488 489 #if KMP_FAST_REDUCTION_BARRIER 490 #define TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER \ 491 ( PACK_REDUCTION_METHOD_AND_BARRIER( tree_reduce_block, bs_reduction_barrier ) ) 492 493 #define TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER \ 494 ( PACK_REDUCTION_METHOD_AND_BARRIER( tree_reduce_block, bs_plain_barrier ) ) 495 #endif 496 497 typedef int PACKED_REDUCTION_METHOD_T; 498 499 /* -- end of fast reduction stuff ----------------------------------------- */ 500 501 /* ------------------------------------------------------------------------ */ 502 /* ------------------------------------------------------------------------ */ 503 504 #if KMP_OS_WINDOWS 505 # define USE_CBLKDATA 506 # pragma warning( push ) 507 # pragma warning( disable: 271 310 ) 508 # include <windows.h> 509 # pragma warning( pop ) 510 #endif 511 512 #if KMP_OS_UNIX 513 # include <pthread.h> 514 # include <dlfcn.h> 515 #endif 516 517 /* ------------------------------------------------------------------------ */ 518 /* ------------------------------------------------------------------------ */ 519 520 /* 521 * Only Linux* OS and Windows* OS support thread affinity. 522 */ 523 #if KMP_AFFINITY_SUPPORTED 524 525 extern size_t __kmp_affin_mask_size; 526 # define KMP_AFFINITY_CAPABLE() (__kmp_affin_mask_size > 0) 527 # define KMP_AFFINITY_DISABLE() (__kmp_affin_mask_size = 0) 528 # define KMP_AFFINITY_ENABLE(mask_size) (__kmp_affin_mask_size = mask_size) 529 # define KMP_CPU_SETSIZE (__kmp_affin_mask_size * CHAR_BIT) 530 531 #if KMP_USE_HWLOC 532 533 typedef hwloc_cpuset_t kmp_affin_mask_t; 534 # define KMP_CPU_SET(i,mask) hwloc_bitmap_set((hwloc_cpuset_t)mask, (unsigned)i) 535 # define KMP_CPU_ISSET(i,mask) hwloc_bitmap_isset((hwloc_cpuset_t)mask, (unsigned)i) 536 # define KMP_CPU_CLR(i,mask) hwloc_bitmap_clr((hwloc_cpuset_t)mask, (unsigned)i) 537 # define KMP_CPU_ZERO(mask) hwloc_bitmap_zero((hwloc_cpuset_t)mask) 538 # define KMP_CPU_COPY(dest, src) hwloc_bitmap_copy((hwloc_cpuset_t)dest, (hwloc_cpuset_t)src) 539 # define KMP_CPU_COMPLEMENT(max_bit_number, mask) \ 540 { \ 541 unsigned i; \ 542 for(i=0;i<(unsigned)max_bit_number+1;i++) { \ 543 if(hwloc_bitmap_isset((hwloc_cpuset_t)mask, i)) { \ 544 hwloc_bitmap_clr((hwloc_cpuset_t)mask, i); \ 545 } else { \ 546 hwloc_bitmap_set((hwloc_cpuset_t)mask, i); \ 547 } \ 548 } \ 549 } \ 550 551 # define KMP_CPU_UNION(dest, src) hwloc_bitmap_or((hwloc_cpuset_t)dest, (hwloc_cpuset_t)dest, (hwloc_cpuset_t)src) 552 # define KMP_CPU_SET_ITERATE(i,mask) \ 553 for(i = hwloc_bitmap_first((hwloc_cpuset_t)mask); (int)i != -1; i = hwloc_bitmap_next((hwloc_cpuset_t)mask, i)) 554 555 # define KMP_CPU_ALLOC(ptr) ptr = (kmp_affin_mask_t*)hwloc_bitmap_alloc() 556 # define KMP_CPU_FREE(ptr) hwloc_bitmap_free((hwloc_bitmap_t)ptr); 557 # define KMP_CPU_ALLOC_ON_STACK(ptr) KMP_CPU_ALLOC(ptr) 558 # define KMP_CPU_FREE_FROM_STACK(ptr) KMP_CPU_FREE(ptr) 559 # define KMP_CPU_INTERNAL_ALLOC(ptr) KMP_CPU_ALLOC(ptr) 560 # define KMP_CPU_INTERNAL_FREE(ptr) KMP_CPU_FREE(ptr) 561 562 // 563 // The following macro should be used to index an array of masks. 564 // The array should be declared as "kmp_affinity_t *" and allocated with 565 // size "__kmp_affinity_mask_size * len". The macro takes care of the fact 566 // that on Windows* OS, sizeof(kmp_affin_t) is really the size of the mask, but 567 // on Linux* OS, sizeof(kmp_affin_t) is 1. 568 // 569 # define KMP_CPU_INDEX(array,i) ((kmp_affin_mask_t*)(array[i])) 570 # define KMP_CPU_ALLOC_ARRAY(arr, n) { \ 571 arr = (kmp_affin_mask_t *)__kmp_allocate(n*sizeof(kmp_affin_mask_t)); \ 572 unsigned i; \ 573 for(i=0;i<(unsigned)n;i++) { \ 574 arr[i] = hwloc_bitmap_alloc(); \ 575 } \ 576 } 577 # define KMP_CPU_FREE_ARRAY(arr, n) { \ 578 unsigned i; \ 579 for(i=0;i<(unsigned)n;i++) { \ 580 hwloc_bitmap_free(arr[i]); \ 581 } \ 582 __kmp_free(arr); \ 583 } 584 # define KMP_CPU_INTERNAL_ALLOC_ARRAY(arr, n) { \ 585 arr = (kmp_affin_mask_t *)KMP_INTERNAL_MALLOC(n*sizeof(kmp_affin_mask_t)); \ 586 unsigned i; \ 587 for(i=0;i<(unsigned)n;i++) { \ 588 arr[i] = hwloc_bitmap_alloc(); \ 589 } \ 590 } 591 # define KMP_CPU_INTERNAL_FREE_ARRAY(arr, n) { \ 592 unsigned i; \ 593 for(i=0;i<(unsigned)n;i++) { \ 594 hwloc_bitmap_free(arr[i]); \ 595 } \ 596 KMP_INTERNAL_FREE(arr); \ 597 } 598 599 #else /* KMP_USE_HWLOC */ 600 # define KMP_CPU_SET_ITERATE(i,mask) \ 601 for(i = 0; (size_t)i < KMP_CPU_SETSIZE; ++i) 602 603 # if KMP_OS_LINUX 604 // 605 // On Linux* OS, the mask is actually a vector of length __kmp_affin_mask_size 606 // (in bytes). It should be allocated on a word boundary. 607 // 608 // WARNING!!! We have made the base type of the affinity mask unsigned char, 609 // in order to eliminate a lot of checks that the true system mask size is 610 // really a multiple of 4 bytes (on Linux* OS). 611 // 612 // THESE MACROS WON'T WORK PROPERLY ON BIG ENDIAN MACHINES!!! 613 // 614 615 typedef unsigned char kmp_affin_mask_t; 616 617 # define _KMP_CPU_SET(i,mask) (mask[i/CHAR_BIT] |= (((kmp_affin_mask_t)1) << (i % CHAR_BIT))) 618 # define KMP_CPU_SET(i,mask) _KMP_CPU_SET((i), ((kmp_affin_mask_t *)(mask))) 619 # define _KMP_CPU_ISSET(i,mask) (!!(mask[i/CHAR_BIT] & (((kmp_affin_mask_t)1) << (i % CHAR_BIT)))) 620 # define KMP_CPU_ISSET(i,mask) _KMP_CPU_ISSET((i), ((kmp_affin_mask_t *)(mask))) 621 # define _KMP_CPU_CLR(i,mask) (mask[i/CHAR_BIT] &= ~(((kmp_affin_mask_t)1) << (i % CHAR_BIT))) 622 # define KMP_CPU_CLR(i,mask) _KMP_CPU_CLR((i), ((kmp_affin_mask_t *)(mask))) 623 624 # define KMP_CPU_ZERO(mask) \ 625 { \ 626 size_t __i; \ 627 for (__i = 0; __i < __kmp_affin_mask_size; __i++) { \ 628 ((kmp_affin_mask_t *)(mask))[__i] = 0; \ 629 } \ 630 } 631 632 # define KMP_CPU_COPY(dest, src) \ 633 { \ 634 size_t __i; \ 635 for (__i = 0; __i < __kmp_affin_mask_size; __i++) { \ 636 ((kmp_affin_mask_t *)(dest))[__i] \ 637 = ((kmp_affin_mask_t *)(src))[__i]; \ 638 } \ 639 } 640 641 # define KMP_CPU_COMPLEMENT(max_bit_number, mask) \ 642 { \ 643 size_t __i; \ 644 for (__i = 0; __i < __kmp_affin_mask_size; __i++) { \ 645 ((kmp_affin_mask_t *)(mask))[__i] \ 646 = ~((kmp_affin_mask_t *)(mask))[__i]; \ 647 } \ 648 } 649 650 # define KMP_CPU_UNION(dest, src) \ 651 { \ 652 size_t __i; \ 653 for (__i = 0; __i < __kmp_affin_mask_size; __i++) { \ 654 ((kmp_affin_mask_t *)(dest))[__i] \ 655 |= ((kmp_affin_mask_t *)(src))[__i]; \ 656 } \ 657 } 658 659 # endif /* KMP_OS_LINUX */ 660 661 # if KMP_OS_WINDOWS 662 // 663 // On Windows* OS, the mask size is 4 bytes for IA-32 architecture, and on 664 // Intel(R) 64 it is 8 bytes times the number of processor groups. 665 // 666 667 # if KMP_GROUP_AFFINITY 668 669 // GROUP_AFFINITY is already defined for _MSC_VER>=1600 (VS2010 and later). 670 # if _MSC_VER < 1600 671 typedef struct GROUP_AFFINITY { 672 KAFFINITY Mask; 673 WORD Group; 674 WORD Reserved[3]; 675 } GROUP_AFFINITY; 676 # endif 677 678 typedef DWORD_PTR kmp_affin_mask_t; 679 680 extern int __kmp_num_proc_groups; 681 682 # define _KMP_CPU_SET(i,mask) \ 683 (mask[i/(CHAR_BIT * sizeof(kmp_affin_mask_t))] |= \ 684 (((kmp_affin_mask_t)1) << (i % (CHAR_BIT * sizeof(kmp_affin_mask_t))))) 685 686 # define KMP_CPU_SET(i,mask) \ 687 _KMP_CPU_SET((i), ((kmp_affin_mask_t *)(mask))) 688 689 # define _KMP_CPU_ISSET(i,mask) \ 690 (!!(mask[i/(CHAR_BIT * sizeof(kmp_affin_mask_t))] & \ 691 (((kmp_affin_mask_t)1) << (i % (CHAR_BIT * sizeof(kmp_affin_mask_t)))))) 692 693 # define KMP_CPU_ISSET(i,mask) \ 694 _KMP_CPU_ISSET((i), ((kmp_affin_mask_t *)(mask))) 695 696 # define _KMP_CPU_CLR(i,mask) \ 697 (mask[i/(CHAR_BIT * sizeof(kmp_affin_mask_t))] &= \ 698 ~(((kmp_affin_mask_t)1) << (i % (CHAR_BIT * sizeof(kmp_affin_mask_t))))) 699 700 # define KMP_CPU_CLR(i,mask) \ 701 _KMP_CPU_CLR((i), ((kmp_affin_mask_t *)(mask))) 702 703 # define KMP_CPU_ZERO(mask) \ 704 { \ 705 int __i; \ 706 for (__i = 0; __i < __kmp_num_proc_groups; __i++) { \ 707 ((kmp_affin_mask_t *)(mask))[__i] = 0; \ 708 } \ 709 } 710 711 # define KMP_CPU_COPY(dest, src) \ 712 { \ 713 int __i; \ 714 for (__i = 0; __i < __kmp_num_proc_groups; __i++) { \ 715 ((kmp_affin_mask_t *)(dest))[__i] \ 716 = ((kmp_affin_mask_t *)(src))[__i]; \ 717 } \ 718 } 719 720 # define KMP_CPU_COMPLEMENT(max_bit_number, mask) \ 721 { \ 722 int __i; \ 723 for (__i = 0; __i < __kmp_num_proc_groups; __i++) { \ 724 ((kmp_affin_mask_t *)(mask))[__i] \ 725 = ~((kmp_affin_mask_t *)(mask))[__i]; \ 726 } \ 727 } 728 729 # define KMP_CPU_UNION(dest, src) \ 730 { \ 731 int __i; \ 732 for (__i = 0; __i < __kmp_num_proc_groups; __i++) { \ 733 ((kmp_affin_mask_t *)(dest))[__i] \ 734 |= ((kmp_affin_mask_t *)(src))[__i]; \ 735 } \ 736 } 737 738 typedef DWORD (*kmp_GetActiveProcessorCount_t)(WORD); 739 extern kmp_GetActiveProcessorCount_t __kmp_GetActiveProcessorCount; 740 741 typedef WORD (*kmp_GetActiveProcessorGroupCount_t)(void); 742 extern kmp_GetActiveProcessorGroupCount_t __kmp_GetActiveProcessorGroupCount; 743 744 typedef BOOL (*kmp_GetThreadGroupAffinity_t)(HANDLE, GROUP_AFFINITY *); 745 extern kmp_GetThreadGroupAffinity_t __kmp_GetThreadGroupAffinity; 746 747 typedef BOOL (*kmp_SetThreadGroupAffinity_t)(HANDLE, const GROUP_AFFINITY *, GROUP_AFFINITY *); 748 extern kmp_SetThreadGroupAffinity_t __kmp_SetThreadGroupAffinity; 749 750 extern int __kmp_get_proc_group(kmp_affin_mask_t const *mask); 751 752 # else /* KMP_GROUP_AFFINITY */ 753 754 typedef DWORD kmp_affin_mask_t; /* for compatibility with older winbase.h */ 755 756 # define KMP_CPU_SET(i,mask) (*(mask) |= (((kmp_affin_mask_t)1) << (i))) 757 # define KMP_CPU_ISSET(i,mask) (!!(*(mask) & (((kmp_affin_mask_t)1) << (i)))) 758 # define KMP_CPU_CLR(i,mask) (*(mask) &= ~(((kmp_affin_mask_t)1) << (i))) 759 # define KMP_CPU_ZERO(mask) (*(mask) = 0) 760 # define KMP_CPU_COPY(dest, src) (*(dest) = *(src)) 761 # define KMP_CPU_COMPLEMENT(max_bit_number, mask) (*(mask) = ~*(mask)) 762 # define KMP_CPU_UNION(dest, src) (*(dest) |= *(src)) 763 764 # endif /* KMP_GROUP_AFFINITY */ 765 766 # endif /* KMP_OS_WINDOWS */ 767 768 // 769 // __kmp_allocate() will return memory allocated on a 4-bytes boundary. 770 // after zeroing it - it takes care of those assumptions stated above. 771 // 772 # define KMP_CPU_ALLOC(ptr) \ 773 (ptr = ((kmp_affin_mask_t *)__kmp_allocate(__kmp_affin_mask_size))) 774 # define KMP_CPU_FREE(ptr) __kmp_free(ptr) 775 # define KMP_CPU_ALLOC_ON_STACK(ptr) (ptr = ((kmp_affin_mask_t *)KMP_ALLOCA(__kmp_affin_mask_size))) 776 # define KMP_CPU_FREE_FROM_STACK(ptr) /* Nothing */ 777 # define KMP_CPU_INTERNAL_ALLOC(ptr) (ptr = ((kmp_affin_mask_t *)KMP_INTERNAL_MALLOC(__kmp_affin_mask_size))) 778 # define KMP_CPU_INTERNAL_FREE(ptr) KMP_INTERNAL_FREE(ptr) 779 780 // 781 // The following macro should be used to index an array of masks. 782 // The array should be declared as "kmp_affinity_t *" and allocated with 783 // size "__kmp_affinity_mask_size * len". The macro takes care of the fact 784 // that on Windows* OS, sizeof(kmp_affin_t) is really the size of the mask, but 785 // on Linux* OS, sizeof(kmp_affin_t) is 1. 786 // 787 # define KMP_CPU_INDEX(array,i) \ 788 ((kmp_affin_mask_t *)(((char *)(array)) + (i) * __kmp_affin_mask_size)) 789 # define KMP_CPU_ALLOC_ARRAY(arr, n) arr = (kmp_affin_mask_t *)__kmp_allocate(n * __kmp_affin_mask_size) 790 # define KMP_CPU_FREE_ARRAY(arr, n) __kmp_free(arr); 791 # define KMP_CPU_INTERNAL_ALLOC_ARRAY(arr, n) arr = (kmp_affin_mask_t *)KMP_INTERNAL_MALLOC(n * __kmp_affin_mask_size) 792 # define KMP_CPU_INTERNAL_FREE_ARRAY(arr, n) KMP_INTERNAL_FREE(arr); 793 794 #endif /* KMP_USE_HWLOC */ 795 796 // 797 // Declare local char buffers with this size for printing debug and info 798 // messages, using __kmp_affinity_print_mask(). 799 // 800 #define KMP_AFFIN_MASK_PRINT_LEN 1024 801 802 enum affinity_type { 803 affinity_none = 0, 804 affinity_physical, 805 affinity_logical, 806 affinity_compact, 807 affinity_scatter, 808 affinity_explicit, 809 affinity_balanced, 810 affinity_disabled, // not used outsize the env var parser 811 affinity_default 812 }; 813 814 enum affinity_gran { 815 affinity_gran_fine = 0, 816 affinity_gran_thread, 817 affinity_gran_core, 818 affinity_gran_package, 819 affinity_gran_node, 820 #if KMP_GROUP_AFFINITY 821 // 822 // The "group" granularity isn't necesssarily coarser than all of the 823 // other levels, but we put it last in the enum. 824 // 825 affinity_gran_group, 826 #endif /* KMP_GROUP_AFFINITY */ 827 affinity_gran_default 828 }; 829 830 enum affinity_top_method { 831 affinity_top_method_all = 0, // try all (supported) methods, in order 832 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 833 affinity_top_method_apicid, 834 affinity_top_method_x2apicid, 835 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 836 affinity_top_method_cpuinfo, // KMP_CPUINFO_FILE is usable on Windows* OS, too 837 #if KMP_GROUP_AFFINITY 838 affinity_top_method_group, 839 #endif /* KMP_GROUP_AFFINITY */ 840 affinity_top_method_flat, 841 #if KMP_USE_HWLOC 842 affinity_top_method_hwloc, 843 #endif 844 affinity_top_method_default 845 }; 846 847 #define affinity_respect_mask_default (-1) 848 849 extern enum affinity_type __kmp_affinity_type; /* Affinity type */ 850 extern enum affinity_gran __kmp_affinity_gran; /* Affinity granularity */ 851 extern int __kmp_affinity_gran_levels; /* corresponding int value */ 852 extern int __kmp_affinity_dups; /* Affinity duplicate masks */ 853 extern enum affinity_top_method __kmp_affinity_top_method; 854 extern int __kmp_affinity_compact; /* Affinity 'compact' value */ 855 extern int __kmp_affinity_offset; /* Affinity offset value */ 856 extern int __kmp_affinity_verbose; /* Was verbose specified for KMP_AFFINITY? */ 857 extern int __kmp_affinity_warnings; /* KMP_AFFINITY warnings enabled ? */ 858 extern int __kmp_affinity_respect_mask; /* Respect process' initial affinity mask? */ 859 extern char * __kmp_affinity_proclist; /* proc ID list */ 860 extern kmp_affin_mask_t *__kmp_affinity_masks; 861 extern unsigned __kmp_affinity_num_masks; 862 extern int __kmp_get_system_affinity(kmp_affin_mask_t *mask, int abort_on_error); 863 extern int __kmp_set_system_affinity(kmp_affin_mask_t const *mask, int abort_on_error); 864 extern void __kmp_affinity_bind_thread(int which); 865 866 # if KMP_OS_LINUX 867 extern kmp_affin_mask_t *__kmp_affinity_get_fullMask(); 868 # endif /* KMP_OS_LINUX */ 869 extern char const * __kmp_cpuinfo_file; 870 871 #endif /* KMP_AFFINITY_SUPPORTED */ 872 873 #if OMP_40_ENABLED 874 875 // 876 // This needs to be kept in sync with the values in omp.h !!! 877 // 878 typedef enum kmp_proc_bind_t { 879 proc_bind_false = 0, 880 proc_bind_true, 881 proc_bind_master, 882 proc_bind_close, 883 proc_bind_spread, 884 proc_bind_intel, // use KMP_AFFINITY interface 885 proc_bind_default 886 } kmp_proc_bind_t; 887 888 typedef struct kmp_nested_proc_bind_t { 889 kmp_proc_bind_t *bind_types; 890 int size; 891 int used; 892 } kmp_nested_proc_bind_t; 893 894 extern kmp_nested_proc_bind_t __kmp_nested_proc_bind; 895 896 #endif /* OMP_40_ENABLED */ 897 898 # if KMP_AFFINITY_SUPPORTED 899 # define KMP_PLACE_ALL (-1) 900 # define KMP_PLACE_UNDEFINED (-2) 901 # endif /* KMP_AFFINITY_SUPPORTED */ 902 903 extern int __kmp_affinity_num_places; 904 905 906 #if OMP_40_ENABLED 907 typedef enum kmp_cancel_kind_t { 908 cancel_noreq = 0, 909 cancel_parallel = 1, 910 cancel_loop = 2, 911 cancel_sections = 3, 912 cancel_taskgroup = 4 913 } kmp_cancel_kind_t; 914 #endif // OMP_40_ENABLED 915 916 extern int __kmp_place_num_sockets; 917 extern int __kmp_place_socket_offset; 918 extern int __kmp_place_num_cores; 919 extern int __kmp_place_core_offset; 920 extern int __kmp_place_num_threads_per_core; 921 922 /* ------------------------------------------------------------------------ */ 923 /* ------------------------------------------------------------------------ */ 924 925 #define KMP_PAD(type, sz) (sizeof(type) + (sz - ((sizeof(type) - 1) % (sz)) - 1)) 926 927 // 928 // We need to avoid using -1 as a GTID as +1 is added to the gtid 929 // when storing it in a lock, and the value 0 is reserved. 930 // 931 #define KMP_GTID_DNE (-2) /* Does not exist */ 932 #define KMP_GTID_SHUTDOWN (-3) /* Library is shutting down */ 933 #define KMP_GTID_MONITOR (-4) /* Monitor thread ID */ 934 #define KMP_GTID_UNKNOWN (-5) /* Is not known */ 935 #define KMP_GTID_MIN (-6) /* Minimal gtid for low bound check in DEBUG */ 936 937 #define __kmp_get_gtid() __kmp_get_global_thread_id() 938 #define __kmp_entry_gtid() __kmp_get_global_thread_id_reg() 939 940 #define __kmp_tid_from_gtid(gtid) ( KMP_DEBUG_ASSERT( (gtid) >= 0 ), \ 941 __kmp_threads[ (gtid) ]->th.th_info.ds.ds_tid ) 942 943 #define __kmp_get_tid() ( __kmp_tid_from_gtid( __kmp_get_gtid() ) ) 944 #define __kmp_gtid_from_tid(tid,team) ( KMP_DEBUG_ASSERT( (tid) >= 0 && (team) != NULL ), \ 945 team -> t.t_threads[ (tid) ] -> th.th_info .ds.ds_gtid ) 946 947 #define __kmp_get_team() ( __kmp_threads[ (__kmp_get_gtid()) ]-> th.th_team ) 948 #define __kmp_team_from_gtid(gtid) ( KMP_DEBUG_ASSERT( (gtid) >= 0 ), \ 949 __kmp_threads[ (gtid) ]-> th.th_team ) 950 951 #define __kmp_thread_from_gtid(gtid) ( KMP_DEBUG_ASSERT( (gtid) >= 0 ), __kmp_threads[ (gtid) ] ) 952 #define __kmp_get_thread() ( __kmp_thread_from_gtid( __kmp_get_gtid() ) ) 953 954 // Returns current thread (pointer to kmp_info_t). In contrast to __kmp_get_thread(), it works 955 // with registered and not-yet-registered threads. 956 #define __kmp_gtid_from_thread(thr) ( KMP_DEBUG_ASSERT( (thr) != NULL ), \ 957 (thr)->th.th_info.ds.ds_gtid ) 958 959 // AT: Which way is correct? 960 // AT: 1. nproc = __kmp_threads[ ( gtid ) ] -> th.th_team -> t.t_nproc; 961 // AT: 2. nproc = __kmp_threads[ ( gtid ) ] -> th.th_team_nproc; 962 #define __kmp_get_team_num_threads(gtid) ( __kmp_threads[ ( gtid ) ] -> th.th_team -> t.t_nproc ) 963 964 965 /* ------------------------------------------------------------------------ */ 966 /* ------------------------------------------------------------------------ */ 967 968 #define KMP_UINT64_MAX (~((kmp_uint64)1<<((sizeof(kmp_uint64)*(1<<3))-1))) 969 970 #define KMP_MIN_NTH 1 971 972 #ifndef KMP_MAX_NTH 973 # if defined(PTHREAD_THREADS_MAX) && PTHREAD_THREADS_MAX < INT_MAX 974 # define KMP_MAX_NTH PTHREAD_THREADS_MAX 975 # else 976 # define KMP_MAX_NTH INT_MAX 977 # endif 978 #endif /* KMP_MAX_NTH */ 979 980 #ifdef PTHREAD_STACK_MIN 981 # define KMP_MIN_STKSIZE PTHREAD_STACK_MIN 982 #else 983 # define KMP_MIN_STKSIZE ((size_t)(32 * 1024)) 984 #endif 985 986 #define KMP_MAX_STKSIZE (~((size_t)1<<((sizeof(size_t)*(1<<3))-1))) 987 988 #if KMP_ARCH_X86 989 # define KMP_DEFAULT_STKSIZE ((size_t)(2 * 1024 * 1024)) 990 #elif KMP_ARCH_X86_64 991 # define KMP_DEFAULT_STKSIZE ((size_t)(4 * 1024 * 1024)) 992 # define KMP_BACKUP_STKSIZE ((size_t)(2 * 1024 * 1024)) 993 #else 994 # define KMP_DEFAULT_STKSIZE ((size_t)(1024 * 1024)) 995 #endif 996 997 #define KMP_DEFAULT_MONITOR_STKSIZE ((size_t)(64 * 1024)) 998 999 #define KMP_DEFAULT_MALLOC_POOL_INCR ((size_t) (1024 * 1024)) 1000 #define KMP_MIN_MALLOC_POOL_INCR ((size_t) (4 * 1024)) 1001 #define KMP_MAX_MALLOC_POOL_INCR (~((size_t)1<<((sizeof(size_t)*(1<<3))-1))) 1002 1003 #define KMP_MIN_STKOFFSET (0) 1004 #define KMP_MAX_STKOFFSET KMP_MAX_STKSIZE 1005 #if KMP_OS_DARWIN 1006 # define KMP_DEFAULT_STKOFFSET KMP_MIN_STKOFFSET 1007 #else 1008 # define KMP_DEFAULT_STKOFFSET CACHE_LINE 1009 #endif 1010 1011 #define KMP_MIN_STKPADDING (0) 1012 #define KMP_MAX_STKPADDING (2 * 1024 * 1024) 1013 1014 #define KMP_MIN_MONITOR_WAKEUPS (1) /* min number of times monitor wakes up per second */ 1015 #define KMP_MAX_MONITOR_WAKEUPS (1000) /* maximum number of times monitor can wake up per second */ 1016 #define KMP_BLOCKTIME_MULTIPLIER (1000) /* number of blocktime units per second */ 1017 #define KMP_MIN_BLOCKTIME (0) 1018 #define KMP_MAX_BLOCKTIME (INT_MAX) /* Must be this for "infinite" setting the work */ 1019 #define KMP_DEFAULT_BLOCKTIME (200) /* __kmp_blocktime is in milliseconds */ 1020 /* Calculate new number of monitor wakeups for a specific block time based on previous monitor_wakeups */ 1021 /* Only allow increasing number of wakeups */ 1022 #define KMP_WAKEUPS_FROM_BLOCKTIME(blocktime, monitor_wakeups) \ 1023 ( ((blocktime) == KMP_MAX_BLOCKTIME) ? (monitor_wakeups) : \ 1024 ((blocktime) == KMP_MIN_BLOCKTIME) ? KMP_MAX_MONITOR_WAKEUPS : \ 1025 ((monitor_wakeups) > (KMP_BLOCKTIME_MULTIPLIER / (blocktime))) ? (monitor_wakeups) : \ 1026 (KMP_BLOCKTIME_MULTIPLIER) / (blocktime) ) 1027 1028 /* Calculate number of intervals for a specific block time based on monitor_wakeups */ 1029 #define KMP_INTERVALS_FROM_BLOCKTIME(blocktime, monitor_wakeups) \ 1030 ( ( (blocktime) + (KMP_BLOCKTIME_MULTIPLIER / (monitor_wakeups)) - 1 ) / \ 1031 (KMP_BLOCKTIME_MULTIPLIER / (monitor_wakeups)) ) 1032 1033 #define KMP_MIN_STATSCOLS 40 1034 #define KMP_MAX_STATSCOLS 4096 1035 #define KMP_DEFAULT_STATSCOLS 80 1036 1037 #define KMP_MIN_INTERVAL 0 1038 #define KMP_MAX_INTERVAL (INT_MAX-1) 1039 #define KMP_DEFAULT_INTERVAL 0 1040 1041 #define KMP_MIN_CHUNK 1 1042 #define KMP_MAX_CHUNK (INT_MAX-1) 1043 #define KMP_DEFAULT_CHUNK 1 1044 1045 #define KMP_MIN_INIT_WAIT 1 1046 #define KMP_MAX_INIT_WAIT (INT_MAX/2) 1047 #define KMP_DEFAULT_INIT_WAIT 2048U 1048 1049 #define KMP_MIN_NEXT_WAIT 1 1050 #define KMP_MAX_NEXT_WAIT (INT_MAX/2) 1051 #define KMP_DEFAULT_NEXT_WAIT 1024U 1052 1053 // max possible dynamic loops in concurrent execution per team 1054 #define KMP_MAX_DISP_BUF 7 1055 #define KMP_MAX_ORDERED 8 1056 1057 #define KMP_MAX_FIELDS 32 1058 1059 #define KMP_MAX_BRANCH_BITS 31 1060 1061 #define KMP_MAX_ACTIVE_LEVELS_LIMIT INT_MAX 1062 1063 #define KMP_MAX_TASK_PRIORITY_LIMIT INT_MAX 1064 1065 /* Minimum number of threads before switch to TLS gtid (experimentally determined) */ 1066 /* josh TODO: what about OS X* tuning? */ 1067 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1068 # define KMP_TLS_GTID_MIN 5 1069 #else 1070 # define KMP_TLS_GTID_MIN INT_MAX 1071 #endif 1072 1073 #define KMP_MASTER_TID(tid) ( (tid) == 0 ) 1074 #define KMP_WORKER_TID(tid) ( (tid) != 0 ) 1075 1076 #define KMP_MASTER_GTID(gtid) ( __kmp_tid_from_gtid((gtid)) == 0 ) 1077 #define KMP_WORKER_GTID(gtid) ( __kmp_tid_from_gtid((gtid)) != 0 ) 1078 #define KMP_UBER_GTID(gtid) \ 1079 ( \ 1080 KMP_DEBUG_ASSERT( (gtid) >= KMP_GTID_MIN ), \ 1081 KMP_DEBUG_ASSERT( (gtid) < __kmp_threads_capacity ), \ 1082 (gtid) >= 0 && __kmp_root[(gtid)] && __kmp_threads[(gtid)] && \ 1083 (__kmp_threads[(gtid)] == __kmp_root[(gtid)]->r.r_uber_thread)\ 1084 ) 1085 #define KMP_INITIAL_GTID(gtid) ( (gtid) == 0 ) 1086 1087 #ifndef TRUE 1088 #define FALSE 0 1089 #define TRUE (! FALSE) 1090 #endif 1091 1092 /* NOTE: all of the following constants must be even */ 1093 1094 #if KMP_OS_WINDOWS 1095 # define KMP_INIT_WAIT 64U /* initial number of spin-tests */ 1096 # define KMP_NEXT_WAIT 32U /* susequent number of spin-tests */ 1097 #elif KMP_OS_CNK 1098 # define KMP_INIT_WAIT 16U /* initial number of spin-tests */ 1099 # define KMP_NEXT_WAIT 8U /* susequent number of spin-tests */ 1100 #elif KMP_OS_LINUX 1101 # define KMP_INIT_WAIT 1024U /* initial number of spin-tests */ 1102 # define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */ 1103 #elif KMP_OS_DARWIN 1104 /* TODO: tune for KMP_OS_DARWIN */ 1105 # define KMP_INIT_WAIT 1024U /* initial number of spin-tests */ 1106 # define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */ 1107 #elif KMP_OS_FREEBSD 1108 /* TODO: tune for KMP_OS_FREEBSD */ 1109 # define KMP_INIT_WAIT 1024U /* initial number of spin-tests */ 1110 # define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */ 1111 #elif KMP_OS_NETBSD 1112 /* TODO: tune for KMP_OS_NETBSD */ 1113 # define KMP_INIT_WAIT 1024U /* initial number of spin-tests */ 1114 # define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */ 1115 #endif 1116 1117 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1118 typedef struct kmp_cpuid { 1119 kmp_uint32 eax; 1120 kmp_uint32 ebx; 1121 kmp_uint32 ecx; 1122 kmp_uint32 edx; 1123 } kmp_cpuid_t; 1124 extern void __kmp_x86_cpuid( int mode, int mode2, struct kmp_cpuid *p ); 1125 # if KMP_ARCH_X86 1126 extern void __kmp_x86_pause( void ); 1127 # elif KMP_MIC 1128 static void __kmp_x86_pause( void ) { _mm_delay_32( 100 ); } 1129 # else 1130 static void __kmp_x86_pause( void ) { _mm_pause(); } 1131 # endif 1132 # define KMP_CPU_PAUSE() __kmp_x86_pause() 1133 #elif KMP_ARCH_PPC64 1134 # define KMP_PPC64_PRI_LOW() __asm__ volatile ("or 1, 1, 1") 1135 # define KMP_PPC64_PRI_MED() __asm__ volatile ("or 2, 2, 2") 1136 # define KMP_PPC64_PRI_LOC_MB() __asm__ volatile ("" : : : "memory") 1137 # define KMP_CPU_PAUSE() do { KMP_PPC64_PRI_LOW(); KMP_PPC64_PRI_MED(); KMP_PPC64_PRI_LOC_MB(); } while (0) 1138 #else 1139 # define KMP_CPU_PAUSE() /* nothing to do */ 1140 #endif 1141 1142 #define KMP_INIT_YIELD(count) { (count) = __kmp_yield_init; } 1143 1144 #define KMP_YIELD(cond) { KMP_CPU_PAUSE(); __kmp_yield( (cond) ); } 1145 1146 // Note the decrement of 2 in the following Macros. With KMP_LIBRARY=turnaround, 1147 // there should be no yielding since the starting value from KMP_INIT_YIELD() is odd. 1148 1149 #define KMP_YIELD_WHEN(cond,count) { KMP_CPU_PAUSE(); (count) -= 2; \ 1150 if (!(count)) { KMP_YIELD(cond); (count) = __kmp_yield_next; } } 1151 #define KMP_YIELD_SPIN(count) { KMP_CPU_PAUSE(); (count) -=2; \ 1152 if (!(count)) { KMP_YIELD(1); (count) = __kmp_yield_next; } } 1153 1154 /* ------------------------------------------------------------------------ */ 1155 /* Support datatypes for the orphaned construct nesting checks. */ 1156 /* ------------------------------------------------------------------------ */ 1157 1158 enum cons_type { 1159 ct_none, 1160 ct_parallel, 1161 ct_pdo, 1162 ct_pdo_ordered, 1163 ct_psections, 1164 ct_psingle, 1165 1166 /* the following must be left in order and not split up */ 1167 ct_taskq, 1168 ct_task, /* really task inside non-ordered taskq, considered a worksharing type */ 1169 ct_task_ordered, /* really task inside ordered taskq, considered a worksharing type */ 1170 /* the preceding must be left in order and not split up */ 1171 1172 ct_critical, 1173 ct_ordered_in_parallel, 1174 ct_ordered_in_pdo, 1175 ct_ordered_in_taskq, 1176 ct_master, 1177 ct_reduce, 1178 ct_barrier 1179 }; 1180 1181 /* test to see if we are in a taskq construct */ 1182 # define IS_CONS_TYPE_TASKQ( ct ) ( ((int)(ct)) >= ((int)ct_taskq) && ((int)(ct)) <= ((int)ct_task_ordered) ) 1183 # define IS_CONS_TYPE_ORDERED( ct ) ((ct) == ct_pdo_ordered || (ct) == ct_task_ordered) 1184 1185 struct cons_data { 1186 ident_t const *ident; 1187 enum cons_type type; 1188 int prev; 1189 kmp_user_lock_p name; /* address exclusively for critical section name comparison */ 1190 }; 1191 1192 struct cons_header { 1193 int p_top, w_top, s_top; 1194 int stack_size, stack_top; 1195 struct cons_data *stack_data; 1196 }; 1197 1198 struct kmp_region_info { 1199 char *text; 1200 int offset[KMP_MAX_FIELDS]; 1201 int length[KMP_MAX_FIELDS]; 1202 }; 1203 1204 1205 /* ---------------------------------------------------------------------- */ 1206 /* ---------------------------------------------------------------------- */ 1207 1208 #if KMP_OS_WINDOWS 1209 typedef HANDLE kmp_thread_t; 1210 typedef DWORD kmp_key_t; 1211 #endif /* KMP_OS_WINDOWS */ 1212 1213 #if KMP_OS_UNIX 1214 typedef pthread_t kmp_thread_t; 1215 typedef pthread_key_t kmp_key_t; 1216 #endif 1217 1218 extern kmp_key_t __kmp_gtid_threadprivate_key; 1219 1220 typedef struct kmp_sys_info { 1221 long maxrss; /* the maximum resident set size utilized (in kilobytes) */ 1222 long minflt; /* the number of page faults serviced without any I/O */ 1223 long majflt; /* the number of page faults serviced that required I/O */ 1224 long nswap; /* the number of times a process was "swapped" out of memory */ 1225 long inblock; /* the number of times the file system had to perform input */ 1226 long oublock; /* the number of times the file system had to perform output */ 1227 long nvcsw; /* the number of times a context switch was voluntarily */ 1228 long nivcsw; /* the number of times a context switch was forced */ 1229 } kmp_sys_info_t; 1230 1231 typedef struct kmp_cpuinfo { 1232 int initialized; // If 0, other fields are not initialized. 1233 int signature; // CPUID(1).EAX 1234 int family; // CPUID(1).EAX[27:20] + CPUID(1).EAX[11:8] ( Extended Family + Family ) 1235 int model; // ( CPUID(1).EAX[19:16] << 4 ) + CPUID(1).EAX[7:4] ( ( Extended Model << 4 ) + Model) 1236 int stepping; // CPUID(1).EAX[3:0] ( Stepping ) 1237 int sse2; // 0 if SSE2 instructions are not supported, 1 otherwise. 1238 int rtm; // 0 if RTM instructions are not supported, 1 otherwise. 1239 int cpu_stackoffset; 1240 int apic_id; 1241 int physical_id; 1242 int logical_id; 1243 kmp_uint64 frequency; // Nominal CPU frequency in Hz. 1244 } kmp_cpuinfo_t; 1245 1246 1247 #ifdef BUILD_TV 1248 1249 struct tv_threadprivate { 1250 /* Record type #1 */ 1251 void *global_addr; 1252 void *thread_addr; 1253 }; 1254 1255 struct tv_data { 1256 struct tv_data *next; 1257 void *type; 1258 union tv_union { 1259 struct tv_threadprivate tp; 1260 } u; 1261 }; 1262 1263 extern kmp_key_t __kmp_tv_key; 1264 1265 #endif /* BUILD_TV */ 1266 1267 /* ------------------------------------------------------------------------ */ 1268 1269 #if USE_ITT_BUILD 1270 // We cannot include "kmp_itt.h" due to circular dependency. Declare the only required type here. 1271 // Later we will check the type meets requirements. 1272 typedef int kmp_itt_mark_t; 1273 #define KMP_ITT_DEBUG 0 1274 #endif /* USE_ITT_BUILD */ 1275 1276 /* ------------------------------------------------------------------------ */ 1277 1278 /* 1279 * Taskq data structures 1280 */ 1281 1282 #define HIGH_WATER_MARK(nslots) (((nslots) * 3) / 4) 1283 #define __KMP_TASKQ_THUNKS_PER_TH 1 /* num thunks that each thread can simultaneously execute from a task queue */ 1284 1285 /* flags for taskq_global_flags, kmp_task_queue_t tq_flags, kmpc_thunk_t th_flags */ 1286 1287 #define TQF_IS_ORDERED 0x0001 /* __kmpc_taskq interface, taskq ordered */ 1288 #define TQF_IS_LASTPRIVATE 0x0002 /* __kmpc_taskq interface, taskq with lastprivate list */ 1289 #define TQF_IS_NOWAIT 0x0004 /* __kmpc_taskq interface, end taskq nowait */ 1290 #define TQF_HEURISTICS 0x0008 /* __kmpc_taskq interface, use heuristics to decide task queue size */ 1291 #define TQF_INTERFACE_RESERVED1 0x0010 /* __kmpc_taskq interface, reserved for future use */ 1292 #define TQF_INTERFACE_RESERVED2 0x0020 /* __kmpc_taskq interface, reserved for future use */ 1293 #define TQF_INTERFACE_RESERVED3 0x0040 /* __kmpc_taskq interface, reserved for future use */ 1294 #define TQF_INTERFACE_RESERVED4 0x0080 /* __kmpc_taskq interface, reserved for future use */ 1295 1296 #define TQF_INTERFACE_FLAGS 0x00ff /* all the __kmpc_taskq interface flags */ 1297 1298 #define TQF_IS_LAST_TASK 0x0100 /* internal/read by instrumentation; only used with TQF_IS_LASTPRIVATE */ 1299 #define TQF_TASKQ_TASK 0x0200 /* internal use only; this thunk->th_task is the taskq_task */ 1300 #define TQF_RELEASE_WORKERS 0x0400 /* internal use only; must release worker threads once ANY queued task exists (global) */ 1301 #define TQF_ALL_TASKS_QUEUED 0x0800 /* internal use only; notify workers that master has finished enqueuing tasks */ 1302 #define TQF_PARALLEL_CONTEXT 0x1000 /* internal use only: this queue encountered in a parallel context: not serialized */ 1303 #define TQF_DEALLOCATED 0x2000 /* internal use only; this queue is on the freelist and not in use */ 1304 1305 #define TQF_INTERNAL_FLAGS 0x3f00 /* all the internal use only flags */ 1306 1307 typedef struct KMP_ALIGN_CACHE kmpc_aligned_int32_t { 1308 kmp_int32 ai_data; 1309 } kmpc_aligned_int32_t; 1310 1311 typedef struct KMP_ALIGN_CACHE kmpc_aligned_queue_slot_t { 1312 struct kmpc_thunk_t *qs_thunk; 1313 } kmpc_aligned_queue_slot_t; 1314 1315 typedef struct kmpc_task_queue_t { 1316 /* task queue linkage fields for n-ary tree of queues (locked with global taskq_tree_lck) */ 1317 kmp_lock_t tq_link_lck; /* lock for child link, child next/prev links and child ref counts */ 1318 union { 1319 struct kmpc_task_queue_t *tq_parent; /* pointer to parent taskq, not locked */ 1320 struct kmpc_task_queue_t *tq_next_free; /* for taskq internal freelists, locked with global taskq_freelist_lck */ 1321 } tq; 1322 volatile struct kmpc_task_queue_t *tq_first_child; /* pointer to linked-list of children, locked by tq's tq_link_lck */ 1323 struct kmpc_task_queue_t *tq_next_child; /* next child in linked-list, locked by parent tq's tq_link_lck */ 1324 struct kmpc_task_queue_t *tq_prev_child; /* previous child in linked-list, locked by parent tq's tq_link_lck */ 1325 volatile kmp_int32 tq_ref_count; /* reference count of threads with access to this task queue */ 1326 /* (other than the thread executing the kmpc_end_taskq call) */ 1327 /* locked by parent tq's tq_link_lck */ 1328 1329 /* shared data for task queue */ 1330 struct kmpc_aligned_shared_vars_t *tq_shareds; /* per-thread array of pointers to shared variable structures */ 1331 /* only one array element exists for all but outermost taskq */ 1332 1333 /* bookkeeping for ordered task queue */ 1334 kmp_uint32 tq_tasknum_queuing; /* ordered task number assigned while queuing tasks */ 1335 volatile kmp_uint32 tq_tasknum_serving; /* ordered number of next task to be served (executed) */ 1336 1337 /* thunk storage management for task queue */ 1338 kmp_lock_t tq_free_thunks_lck; /* lock for thunk freelist manipulation */ 1339 struct kmpc_thunk_t *tq_free_thunks; /* thunk freelist, chained via th.th_next_free */ 1340 struct kmpc_thunk_t *tq_thunk_space; /* space allocated for thunks for this task queue */ 1341 1342 /* data fields for queue itself */ 1343 kmp_lock_t tq_queue_lck; /* lock for [de]enqueue operations: tq_queue, tq_head, tq_tail, tq_nfull */ 1344 kmpc_aligned_queue_slot_t *tq_queue; /* array of queue slots to hold thunks for tasks */ 1345 volatile struct kmpc_thunk_t *tq_taskq_slot; /* special slot for taskq task thunk, occupied if not NULL */ 1346 kmp_int32 tq_nslots; /* # of tq_thunk_space thunks alloc'd (not incl. tq_taskq_slot space) */ 1347 kmp_int32 tq_head; /* enqueue puts next item in here (index into tq_queue array) */ 1348 kmp_int32 tq_tail; /* dequeue takes next item out of here (index into tq_queue array) */ 1349 volatile kmp_int32 tq_nfull; /* # of occupied entries in task queue right now */ 1350 kmp_int32 tq_hiwat; /* high-water mark for tq_nfull and queue scheduling */ 1351 volatile kmp_int32 tq_flags; /* TQF_xxx */ 1352 1353 /* bookkeeping for outstanding thunks */ 1354 struct kmpc_aligned_int32_t *tq_th_thunks; /* per-thread array for # of regular thunks currently being executed */ 1355 kmp_int32 tq_nproc; /* number of thunks in the th_thunks array */ 1356 1357 /* statistics library bookkeeping */ 1358 ident_t *tq_loc; /* source location information for taskq directive */ 1359 } kmpc_task_queue_t; 1360 1361 typedef void (*kmpc_task_t) (kmp_int32 global_tid, struct kmpc_thunk_t *thunk); 1362 1363 /* sizeof_shareds passed as arg to __kmpc_taskq call */ 1364 typedef struct kmpc_shared_vars_t { /* aligned during dynamic allocation */ 1365 kmpc_task_queue_t *sv_queue; 1366 /* (pointers to) shared vars */ 1367 } kmpc_shared_vars_t; 1368 1369 typedef struct KMP_ALIGN_CACHE kmpc_aligned_shared_vars_t { 1370 volatile struct kmpc_shared_vars_t *ai_data; 1371 } kmpc_aligned_shared_vars_t; 1372 1373 /* sizeof_thunk passed as arg to kmpc_taskq call */ 1374 typedef struct kmpc_thunk_t { /* aligned during dynamic allocation */ 1375 union { /* field used for internal freelists too */ 1376 kmpc_shared_vars_t *th_shareds; 1377 struct kmpc_thunk_t *th_next_free; /* freelist of individual thunks within queue, head at tq_free_thunks */ 1378 } th; 1379 kmpc_task_t th_task; /* taskq_task if flags & TQF_TASKQ_TASK */ 1380 struct kmpc_thunk_t *th_encl_thunk; /* pointer to dynamically enclosing thunk on this thread's call stack */ 1381 kmp_int32 th_flags; /* TQF_xxx (tq_flags interface plus possible internal flags) */ 1382 kmp_int32 th_status; 1383 kmp_uint32 th_tasknum; /* task number assigned in order of queuing, used for ordered sections */ 1384 /* private vars */ 1385 } kmpc_thunk_t; 1386 1387 typedef struct KMP_ALIGN_CACHE kmp_taskq { 1388 int tq_curr_thunk_capacity; 1389 1390 kmpc_task_queue_t *tq_root; 1391 kmp_int32 tq_global_flags; 1392 1393 kmp_lock_t tq_freelist_lck; 1394 kmpc_task_queue_t *tq_freelist; 1395 1396 kmpc_thunk_t **tq_curr_thunk; 1397 } kmp_taskq_t; 1398 1399 /* END Taskq data structures */ 1400 /* --------------------------------------------------------------------------- */ 1401 1402 typedef kmp_int32 kmp_critical_name[8]; 1403 1404 /*! 1405 @ingroup PARALLEL 1406 The type for a microtask which gets passed to @ref __kmpc_fork_call(). 1407 The arguments to the outlined function are 1408 @param global_tid the global thread identity of the thread executing the function. 1409 @param bound_tid the local identitiy of the thread executing the function 1410 @param ... pointers to shared variables accessed by the function. 1411 */ 1412 typedef void (*kmpc_micro) ( kmp_int32 * global_tid, kmp_int32 * bound_tid, ... ); 1413 typedef void (*kmpc_micro_bound) ( kmp_int32 * bound_tid, kmp_int32 * bound_nth, ... ); 1414 1415 /*! 1416 @ingroup THREADPRIVATE 1417 @{ 1418 */ 1419 /* --------------------------------------------------------------------------- */ 1420 /* Threadprivate initialization/finalization function declarations */ 1421 1422 /* for non-array objects: __kmpc_threadprivate_register() */ 1423 1424 /*! 1425 Pointer to the constructor function. 1426 The first argument is the <tt>this</tt> pointer 1427 */ 1428 typedef void *(*kmpc_ctor) (void *); 1429 1430 /*! 1431 Pointer to the destructor function. 1432 The first argument is the <tt>this</tt> pointer 1433 */ 1434 typedef void (*kmpc_dtor) (void * /*, size_t */); /* 2nd arg: magic number for KCC unused by Intel compiler */ 1435 /*! 1436 Pointer to an alternate constructor. 1437 The first argument is the <tt>this</tt> pointer. 1438 */ 1439 typedef void *(*kmpc_cctor) (void *, void *); 1440 1441 /* for array objects: __kmpc_threadprivate_register_vec() */ 1442 /* First arg: "this" pointer */ 1443 /* Last arg: number of array elements */ 1444 /*! 1445 Array constructor. 1446 First argument is the <tt>this</tt> pointer 1447 Second argument the number of array elements. 1448 */ 1449 typedef void *(*kmpc_ctor_vec) (void *, size_t); 1450 /*! 1451 Pointer to the array destructor function. 1452 The first argument is the <tt>this</tt> pointer 1453 Second argument the number of array elements. 1454 */ 1455 typedef void (*kmpc_dtor_vec) (void *, size_t); 1456 /*! 1457 Array constructor. 1458 First argument is the <tt>this</tt> pointer 1459 Third argument the number of array elements. 1460 */ 1461 typedef void *(*kmpc_cctor_vec) (void *, void *, size_t); /* function unused by compiler */ 1462 1463 /*! 1464 @} 1465 */ 1466 1467 1468 /* ------------------------------------------------------------------------ */ 1469 1470 /* keeps tracked of threadprivate cache allocations for cleanup later */ 1471 typedef struct kmp_cached_addr { 1472 void **addr; /* address of allocated cache */ 1473 struct kmp_cached_addr *next; /* pointer to next cached address */ 1474 } kmp_cached_addr_t; 1475 1476 struct private_data { 1477 struct private_data *next; /* The next descriptor in the list */ 1478 void *data; /* The data buffer for this descriptor */ 1479 int more; /* The repeat count for this descriptor */ 1480 size_t size; /* The data size for this descriptor */ 1481 }; 1482 1483 struct private_common { 1484 struct private_common *next; 1485 struct private_common *link; 1486 void *gbl_addr; 1487 void *par_addr; /* par_addr == gbl_addr for MASTER thread */ 1488 size_t cmn_size; 1489 }; 1490 1491 struct shared_common 1492 { 1493 struct shared_common *next; 1494 struct private_data *pod_init; 1495 void *obj_init; 1496 void *gbl_addr; 1497 union { 1498 kmpc_ctor ctor; 1499 kmpc_ctor_vec ctorv; 1500 } ct; 1501 union { 1502 kmpc_cctor cctor; 1503 kmpc_cctor_vec cctorv; 1504 } cct; 1505 union { 1506 kmpc_dtor dtor; 1507 kmpc_dtor_vec dtorv; 1508 } dt; 1509 size_t vec_len; 1510 int is_vec; 1511 size_t cmn_size; 1512 }; 1513 1514 #define KMP_HASH_TABLE_LOG2 9 /* log2 of the hash table size */ 1515 #define KMP_HASH_TABLE_SIZE (1 << KMP_HASH_TABLE_LOG2) /* size of the hash table */ 1516 #define KMP_HASH_SHIFT 3 /* throw away this many low bits from the address */ 1517 #define KMP_HASH(x) ((((kmp_uintptr_t) x) >> KMP_HASH_SHIFT) & (KMP_HASH_TABLE_SIZE-1)) 1518 1519 struct common_table { 1520 struct private_common *data[ KMP_HASH_TABLE_SIZE ]; 1521 }; 1522 1523 struct shared_table { 1524 struct shared_common *data[ KMP_HASH_TABLE_SIZE ]; 1525 }; 1526 /* ------------------------------------------------------------------------ */ 1527 /* ------------------------------------------------------------------------ */ 1528 1529 #ifdef KMP_STATIC_STEAL_ENABLED 1530 typedef struct KMP_ALIGN_CACHE dispatch_private_info32 { 1531 kmp_int32 count; 1532 kmp_int32 ub; 1533 /* Adding KMP_ALIGN_CACHE here doesn't help / can hurt performance */ 1534 kmp_int32 lb; 1535 kmp_int32 st; 1536 kmp_int32 tc; 1537 kmp_int32 static_steal_counter; /* for static_steal only; maybe better to put after ub */ 1538 1539 // KMP_ALIGN( 16 ) ensures ( if the KMP_ALIGN macro is turned on ) 1540 // a) parm3 is properly aligned and 1541 // b) all parm1-4 are in the same cache line. 1542 // Because of parm1-4 are used together, performance seems to be better 1543 // if they are in the same line (not measured though). 1544 1545 struct KMP_ALIGN( 32 ) { // AC: changed 16 to 32 in order to simplify template 1546 kmp_int32 parm1; // structures in kmp_dispatch.cpp. This should 1547 kmp_int32 parm2; // make no real change at least while padding is off. 1548 kmp_int32 parm3; 1549 kmp_int32 parm4; 1550 }; 1551 1552 kmp_uint32 ordered_lower; 1553 kmp_uint32 ordered_upper; 1554 #if KMP_OS_WINDOWS 1555 // This var can be placed in the hole between 'tc' and 'parm1', instead of 'static_steal_counter'. 1556 // It would be nice to measure execution times. 1557 // Conditional if/endif can be removed at all. 1558 kmp_int32 last_upper; 1559 #endif /* KMP_OS_WINDOWS */ 1560 } dispatch_private_info32_t; 1561 1562 typedef struct KMP_ALIGN_CACHE dispatch_private_info64 { 1563 kmp_int64 count; /* current chunk number for static and static-steal scheduling*/ 1564 kmp_int64 ub; /* upper-bound */ 1565 /* Adding KMP_ALIGN_CACHE here doesn't help / can hurt performance */ 1566 kmp_int64 lb; /* lower-bound */ 1567 kmp_int64 st; /* stride */ 1568 kmp_int64 tc; /* trip count (number of iterations) */ 1569 kmp_int64 static_steal_counter; /* for static_steal only; maybe better to put after ub */ 1570 1571 /* parm[1-4] are used in different ways by different scheduling algorithms */ 1572 1573 // KMP_ALIGN( 32 ) ensures ( if the KMP_ALIGN macro is turned on ) 1574 // a) parm3 is properly aligned and 1575 // b) all parm1-4 are in the same cache line. 1576 // Because of parm1-4 are used together, performance seems to be better 1577 // if they are in the same line (not measured though). 1578 1579 struct KMP_ALIGN( 32 ) { 1580 kmp_int64 parm1; 1581 kmp_int64 parm2; 1582 kmp_int64 parm3; 1583 kmp_int64 parm4; 1584 }; 1585 1586 kmp_uint64 ordered_lower; 1587 kmp_uint64 ordered_upper; 1588 #if KMP_OS_WINDOWS 1589 // This var can be placed in the hole between 'tc' and 'parm1', instead of 'static_steal_counter'. 1590 // It would be nice to measure execution times. 1591 // Conditional if/endif can be removed at all. 1592 kmp_int64 last_upper; 1593 #endif /* KMP_OS_WINDOWS */ 1594 } dispatch_private_info64_t; 1595 #else /* KMP_STATIC_STEAL_ENABLED */ 1596 typedef struct KMP_ALIGN_CACHE dispatch_private_info32 { 1597 kmp_int32 lb; 1598 kmp_int32 ub; 1599 kmp_int32 st; 1600 kmp_int32 tc; 1601 1602 kmp_int32 parm1; 1603 kmp_int32 parm2; 1604 kmp_int32 parm3; 1605 kmp_int32 parm4; 1606 1607 kmp_int32 count; 1608 1609 kmp_uint32 ordered_lower; 1610 kmp_uint32 ordered_upper; 1611 #if KMP_OS_WINDOWS 1612 kmp_int32 last_upper; 1613 #endif /* KMP_OS_WINDOWS */ 1614 } dispatch_private_info32_t; 1615 1616 typedef struct KMP_ALIGN_CACHE dispatch_private_info64 { 1617 kmp_int64 lb; /* lower-bound */ 1618 kmp_int64 ub; /* upper-bound */ 1619 kmp_int64 st; /* stride */ 1620 kmp_int64 tc; /* trip count (number of iterations) */ 1621 1622 /* parm[1-4] are used in different ways by different scheduling algorithms */ 1623 kmp_int64 parm1; 1624 kmp_int64 parm2; 1625 kmp_int64 parm3; 1626 kmp_int64 parm4; 1627 1628 kmp_int64 count; /* current chunk number for static scheduling */ 1629 1630 kmp_uint64 ordered_lower; 1631 kmp_uint64 ordered_upper; 1632 #if KMP_OS_WINDOWS 1633 kmp_int64 last_upper; 1634 #endif /* KMP_OS_WINDOWS */ 1635 } dispatch_private_info64_t; 1636 #endif /* KMP_STATIC_STEAL_ENABLED */ 1637 1638 typedef struct KMP_ALIGN_CACHE dispatch_private_info { 1639 union private_info { 1640 dispatch_private_info32_t p32; 1641 dispatch_private_info64_t p64; 1642 } u; 1643 enum sched_type schedule; /* scheduling algorithm */ 1644 kmp_int32 ordered; /* ordered clause specified */ 1645 kmp_int32 ordered_bumped; 1646 kmp_int32 ordered_dummy[KMP_MAX_ORDERED-3]; // to retain the structure size after making ordered_iteration scalar 1647 struct dispatch_private_info * next; /* stack of buffers for nest of serial regions */ 1648 kmp_int32 nomerge; /* don't merge iters if serialized */ 1649 kmp_int32 type_size; /* the size of types in private_info */ 1650 enum cons_type pushed_ws; 1651 } dispatch_private_info_t; 1652 1653 typedef struct dispatch_shared_info32 { 1654 /* chunk index under dynamic, number of idle threads under static-steal; 1655 iteration index otherwise */ 1656 volatile kmp_uint32 iteration; 1657 volatile kmp_uint32 num_done; 1658 volatile kmp_uint32 ordered_iteration; 1659 kmp_int32 ordered_dummy[KMP_MAX_ORDERED-1]; // to retain the structure size after making ordered_iteration scalar 1660 } dispatch_shared_info32_t; 1661 1662 typedef struct dispatch_shared_info64 { 1663 /* chunk index under dynamic, number of idle threads under static-steal; 1664 iteration index otherwise */ 1665 volatile kmp_uint64 iteration; 1666 volatile kmp_uint64 num_done; 1667 volatile kmp_uint64 ordered_iteration; 1668 kmp_int64 ordered_dummy[KMP_MAX_ORDERED-1]; // to retain the structure size after making ordered_iteration scalar 1669 } dispatch_shared_info64_t; 1670 1671 typedef struct dispatch_shared_info { 1672 union shared_info { 1673 dispatch_shared_info32_t s32; 1674 dispatch_shared_info64_t s64; 1675 } u; 1676 /* volatile kmp_int32 dispatch_abort; depricated */ 1677 volatile kmp_uint32 buffer_index; 1678 } dispatch_shared_info_t; 1679 1680 typedef struct kmp_disp { 1681 /* Vector for ORDERED SECTION */ 1682 void (*th_deo_fcn)( int * gtid, int * cid, ident_t *); 1683 /* Vector for END ORDERED SECTION */ 1684 void (*th_dxo_fcn)( int * gtid, int * cid, ident_t *); 1685 1686 dispatch_shared_info_t *th_dispatch_sh_current; 1687 dispatch_private_info_t *th_dispatch_pr_current; 1688 1689 dispatch_private_info_t *th_disp_buffer; 1690 kmp_int32 th_disp_index; 1691 void* dummy_padding[2]; // make it 64 bytes on Intel(R) 64 1692 #if KMP_USE_INTERNODE_ALIGNMENT 1693 char more_padding[INTERNODE_CACHE_LINE]; 1694 #endif 1695 } kmp_disp_t; 1696 1697 /* ------------------------------------------------------------------------ */ 1698 /* ------------------------------------------------------------------------ */ 1699 1700 /* Barrier stuff */ 1701 1702 /* constants for barrier state update */ 1703 #define KMP_INIT_BARRIER_STATE 0 /* should probably start from zero */ 1704 #define KMP_BARRIER_SLEEP_BIT 0 /* bit used for suspend/sleep part of state */ 1705 #define KMP_BARRIER_UNUSED_BIT 1 /* bit that must never be set for valid state */ 1706 #define KMP_BARRIER_BUMP_BIT 2 /* lsb used for bump of go/arrived state */ 1707 1708 #define KMP_BARRIER_SLEEP_STATE (1 << KMP_BARRIER_SLEEP_BIT) 1709 #define KMP_BARRIER_UNUSED_STATE (1 << KMP_BARRIER_UNUSED_BIT) 1710 #define KMP_BARRIER_STATE_BUMP (1 << KMP_BARRIER_BUMP_BIT) 1711 1712 #if (KMP_BARRIER_SLEEP_BIT >= KMP_BARRIER_BUMP_BIT) 1713 # error "Barrier sleep bit must be smaller than barrier bump bit" 1714 #endif 1715 #if (KMP_BARRIER_UNUSED_BIT >= KMP_BARRIER_BUMP_BIT) 1716 # error "Barrier unused bit must be smaller than barrier bump bit" 1717 #endif 1718 1719 // Constants for release barrier wait state: currently, hierarchical only 1720 #define KMP_BARRIER_NOT_WAITING 0 // Normal state; worker not in wait_sleep 1721 #define KMP_BARRIER_OWN_FLAG 1 // Normal state; worker waiting on own b_go flag in release 1722 #define KMP_BARRIER_PARENT_FLAG 2 // Special state; worker waiting on parent's b_go flag in release 1723 #define KMP_BARRIER_SWITCH_TO_OWN_FLAG 3 // Special state; tells worker to shift from parent to own b_go 1724 #define KMP_BARRIER_SWITCHING 4 // Special state; worker resets appropriate flag on wake-up 1725 1726 enum barrier_type { 1727 bs_plain_barrier = 0, /* 0, All non-fork/join barriers (except reduction barriers if enabled) */ 1728 bs_forkjoin_barrier, /* 1, All fork/join (parallel region) barriers */ 1729 #if KMP_FAST_REDUCTION_BARRIER 1730 bs_reduction_barrier, /* 2, All barriers that are used in reduction */ 1731 #endif // KMP_FAST_REDUCTION_BARRIER 1732 bs_last_barrier /* Just a placeholder to mark the end */ 1733 }; 1734 1735 // to work with reduction barriers just like with plain barriers 1736 #if !KMP_FAST_REDUCTION_BARRIER 1737 #define bs_reduction_barrier bs_plain_barrier 1738 #endif // KMP_FAST_REDUCTION_BARRIER 1739 1740 typedef enum kmp_bar_pat { /* Barrier communication patterns */ 1741 bp_linear_bar = 0, /* Single level (degenerate) tree */ 1742 bp_tree_bar = 1, /* Balanced tree with branching factor 2^n */ 1743 bp_hyper_bar = 2, /* Hypercube-embedded tree with min branching factor 2^n */ 1744 bp_hierarchical_bar = 3, /* Machine hierarchy tree */ 1745 bp_last_bar = 4 /* Placeholder to mark the end */ 1746 } kmp_bar_pat_e; 1747 1748 # define KMP_BARRIER_ICV_PUSH 1 1749 1750 /* Record for holding the values of the internal controls stack records */ 1751 typedef struct kmp_internal_control { 1752 int serial_nesting_level; /* corresponds to the value of the th_team_serialized field */ 1753 kmp_int8 nested; /* internal control for nested parallelism (per thread) */ 1754 kmp_int8 dynamic; /* internal control for dynamic adjustment of threads (per thread) */ 1755 kmp_int8 bt_set; /* internal control for whether blocktime is explicitly set */ 1756 int blocktime; /* internal control for blocktime */ 1757 int bt_intervals; /* internal control for blocktime intervals */ 1758 int nproc; /* internal control for #threads for next parallel region (per thread) */ 1759 int max_active_levels; /* internal control for max_active_levels */ 1760 kmp_r_sched_t sched; /* internal control for runtime schedule {sched,chunk} pair */ 1761 #if OMP_40_ENABLED 1762 kmp_proc_bind_t proc_bind; /* internal control for affinity */ 1763 #endif // OMP_40_ENABLED 1764 struct kmp_internal_control *next; 1765 } kmp_internal_control_t; 1766 1767 static inline void 1768 copy_icvs( kmp_internal_control_t *dst, kmp_internal_control_t *src ) { 1769 *dst = *src; 1770 } 1771 1772 /* Thread barrier needs volatile barrier fields */ 1773 typedef struct KMP_ALIGN_CACHE kmp_bstate { 1774 // th_fixed_icvs is aligned by virtue of kmp_bstate being aligned (and all uses of it). 1775 // It is not explicitly aligned below, because we *don't* want it to be padded -- instead, 1776 // we fit b_go into the same cache line with th_fixed_icvs, enabling NGO cache lines 1777 // stores in the hierarchical barrier. 1778 kmp_internal_control_t th_fixed_icvs; // Initial ICVs for the thread 1779 // Tuck b_go into end of th_fixed_icvs cache line, so it can be stored with same NGO store 1780 volatile kmp_uint64 b_go; // STATE => task should proceed (hierarchical) 1781 KMP_ALIGN_CACHE volatile kmp_uint64 b_arrived; // STATE => task reached synch point. 1782 kmp_uint32 *skip_per_level; 1783 kmp_uint32 my_level; 1784 kmp_int32 parent_tid; 1785 kmp_int32 old_tid; 1786 kmp_uint32 depth; 1787 struct kmp_bstate *parent_bar; 1788 kmp_team_t *team; 1789 kmp_uint64 leaf_state; 1790 kmp_uint32 nproc; 1791 kmp_uint8 base_leaf_kids; 1792 kmp_uint8 leaf_kids; 1793 kmp_uint8 offset; 1794 kmp_uint8 wait_flag; 1795 kmp_uint8 use_oncore_barrier; 1796 #if USE_DEBUGGER 1797 // The following field is intended for the debugger solely. Only the worker thread itself accesses this 1798 // field: the worker increases it by 1 when it arrives to a barrier. 1799 KMP_ALIGN_CACHE kmp_uint b_worker_arrived; 1800 #endif /* USE_DEBUGGER */ 1801 } kmp_bstate_t; 1802 1803 union KMP_ALIGN_CACHE kmp_barrier_union { 1804 double b_align; /* use worst case alignment */ 1805 char b_pad[ KMP_PAD(kmp_bstate_t, CACHE_LINE) ]; 1806 kmp_bstate_t bb; 1807 }; 1808 1809 typedef union kmp_barrier_union kmp_balign_t; 1810 1811 /* Team barrier needs only non-volatile arrived counter */ 1812 union KMP_ALIGN_CACHE kmp_barrier_team_union { 1813 double b_align; /* use worst case alignment */ 1814 char b_pad[ CACHE_LINE ]; 1815 struct { 1816 kmp_uint64 b_arrived; /* STATE => task reached synch point. */ 1817 #if USE_DEBUGGER 1818 // The following two fields are indended for the debugger solely. Only master of the team accesses 1819 // these fields: the first one is increased by 1 when master arrives to a barrier, the 1820 // second one is increased by one when all the threads arrived. 1821 kmp_uint b_master_arrived; 1822 kmp_uint b_team_arrived; 1823 #endif 1824 }; 1825 }; 1826 1827 typedef union kmp_barrier_team_union kmp_balign_team_t; 1828 1829 /* 1830 * Padding for Linux* OS pthreads condition variables and mutexes used to signal 1831 * threads when a condition changes. This is to workaround an NPTL bug 1832 * where padding was added to pthread_cond_t which caused the initialization 1833 * routine to write outside of the structure if compiled on pre-NPTL threads. 1834 */ 1835 1836 #if KMP_OS_WINDOWS 1837 typedef struct kmp_win32_mutex 1838 { 1839 /* The Lock */ 1840 CRITICAL_SECTION cs; 1841 } kmp_win32_mutex_t; 1842 1843 typedef struct kmp_win32_cond 1844 { 1845 /* Count of the number of waiters. */ 1846 int waiters_count_; 1847 1848 /* Serialize access to <waiters_count_> */ 1849 kmp_win32_mutex_t waiters_count_lock_; 1850 1851 /* Number of threads to release via a <cond_broadcast> or a */ 1852 /* <cond_signal> */ 1853 int release_count_; 1854 1855 /* Keeps track of the current "generation" so that we don't allow */ 1856 /* one thread to steal all the "releases" from the broadcast. */ 1857 int wait_generation_count_; 1858 1859 /* A manual-reset event that's used to block and release waiting */ 1860 /* threads. */ 1861 HANDLE event_; 1862 } kmp_win32_cond_t; 1863 #endif 1864 1865 #if KMP_OS_UNIX 1866 1867 union KMP_ALIGN_CACHE kmp_cond_union { 1868 double c_align; 1869 char c_pad[ CACHE_LINE ]; 1870 pthread_cond_t c_cond; 1871 }; 1872 1873 typedef union kmp_cond_union kmp_cond_align_t; 1874 1875 union KMP_ALIGN_CACHE kmp_mutex_union { 1876 double m_align; 1877 char m_pad[ CACHE_LINE ]; 1878 pthread_mutex_t m_mutex; 1879 }; 1880 1881 typedef union kmp_mutex_union kmp_mutex_align_t; 1882 1883 #endif /* KMP_OS_UNIX */ 1884 1885 typedef struct kmp_desc_base { 1886 void *ds_stackbase; 1887 size_t ds_stacksize; 1888 int ds_stackgrow; 1889 kmp_thread_t ds_thread; 1890 volatile int ds_tid; 1891 int ds_gtid; 1892 #if KMP_OS_WINDOWS 1893 volatile int ds_alive; 1894 DWORD ds_thread_id; 1895 /* 1896 ds_thread keeps thread handle on Windows* OS. It is enough for RTL purposes. However, 1897 debugger support (libomp_db) cannot work with handles, because they uncomparable. For 1898 example, debugger requests info about thread with handle h. h is valid within debugger 1899 process, and meaningless within debugee process. Even if h is duped by call to 1900 DuplicateHandle(), so the result h' is valid within debugee process, but it is a *new* 1901 handle which does *not* equal to any other handle in debugee... The only way to 1902 compare handles is convert them to system-wide ids. GetThreadId() function is 1903 available only in Longhorn and Server 2003. :-( In contrast, GetCurrentThreadId() is 1904 available on all Windows* OS flavours (including Windows* 95). Thus, we have to get thread id by 1905 call to GetCurrentThreadId() from within the thread and save it to let libomp_db 1906 identify threads. 1907 */ 1908 #endif /* KMP_OS_WINDOWS */ 1909 } kmp_desc_base_t; 1910 1911 typedef union KMP_ALIGN_CACHE kmp_desc { 1912 double ds_align; /* use worst case alignment */ 1913 char ds_pad[ KMP_PAD(kmp_desc_base_t, CACHE_LINE) ]; 1914 kmp_desc_base_t ds; 1915 } kmp_desc_t; 1916 1917 1918 typedef struct kmp_local { 1919 volatile int this_construct; /* count of single's encountered by thread */ 1920 void *reduce_data; 1921 #if KMP_USE_BGET 1922 void *bget_data; 1923 void *bget_list; 1924 #if ! USE_CMP_XCHG_FOR_BGET 1925 #ifdef USE_QUEUING_LOCK_FOR_BGET 1926 kmp_lock_t bget_lock; /* Lock for accessing bget free list */ 1927 #else 1928 kmp_bootstrap_lock_t bget_lock; /* Lock for accessing bget free list */ 1929 /* Must be bootstrap lock so we can use it at library shutdown */ 1930 #endif /* USE_LOCK_FOR_BGET */ 1931 #endif /* ! USE_CMP_XCHG_FOR_BGET */ 1932 #endif /* KMP_USE_BGET */ 1933 1934 #ifdef BUILD_TV 1935 struct tv_data *tv_data; 1936 #endif 1937 1938 PACKED_REDUCTION_METHOD_T packed_reduction_method; /* stored by __kmpc_reduce*(), used by __kmpc_end_reduce*() */ 1939 1940 } kmp_local_t; 1941 1942 #define get__blocktime( xteam, xtid ) ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.blocktime) 1943 #define get__bt_set( xteam, xtid ) ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_set) 1944 #define get__bt_intervals( xteam, xtid ) ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_intervals) 1945 1946 #define get__nested_2(xteam,xtid) ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.nested) 1947 #define get__dynamic_2(xteam,xtid) ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.dynamic) 1948 #define get__nproc_2(xteam,xtid) ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.nproc) 1949 #define get__sched_2(xteam,xtid) ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.sched) 1950 1951 #define set__blocktime_team( xteam, xtid, xval ) \ 1952 ( ( (xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.blocktime ) = (xval) ) 1953 1954 #define set__bt_intervals_team( xteam, xtid, xval ) \ 1955 ( ( (xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_intervals ) = (xval) ) 1956 1957 #define set__bt_set_team( xteam, xtid, xval ) \ 1958 ( ( (xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_set ) = (xval) ) 1959 1960 1961 #define set__nested( xthread, xval ) \ 1962 ( ( (xthread)->th.th_current_task->td_icvs.nested ) = (xval) ) 1963 #define get__nested( xthread ) \ 1964 ( ( (xthread)->th.th_current_task->td_icvs.nested ) ? (FTN_TRUE) : (FTN_FALSE) ) 1965 1966 #define set__dynamic( xthread, xval ) \ 1967 ( ( (xthread)->th.th_current_task->td_icvs.dynamic ) = (xval) ) 1968 #define get__dynamic( xthread ) \ 1969 ( ( (xthread)->th.th_current_task->td_icvs.dynamic ) ? (FTN_TRUE) : (FTN_FALSE) ) 1970 1971 #define set__nproc( xthread, xval ) \ 1972 ( ( (xthread)->th.th_current_task->td_icvs.nproc ) = (xval) ) 1973 1974 #define set__max_active_levels( xthread, xval ) \ 1975 ( ( (xthread)->th.th_current_task->td_icvs.max_active_levels ) = (xval) ) 1976 1977 #define set__sched( xthread, xval ) \ 1978 ( ( (xthread)->th.th_current_task->td_icvs.sched ) = (xval) ) 1979 1980 #if OMP_40_ENABLED 1981 1982 #define set__proc_bind( xthread, xval ) \ 1983 ( ( (xthread)->th.th_current_task->td_icvs.proc_bind ) = (xval) ) 1984 #define get__proc_bind( xthread ) \ 1985 ( (xthread)->th.th_current_task->td_icvs.proc_bind ) 1986 1987 #endif /* OMP_40_ENABLED */ 1988 1989 1990 /* ------------------------------------------------------------------------ */ 1991 // OpenMP tasking data structures 1992 // 1993 1994 typedef enum kmp_tasking_mode { 1995 tskm_immediate_exec = 0, 1996 tskm_extra_barrier = 1, 1997 tskm_task_teams = 2, 1998 tskm_max = 2 1999 } kmp_tasking_mode_t; 2000 2001 extern kmp_tasking_mode_t __kmp_tasking_mode; /* determines how/when to execute tasks */ 2002 extern kmp_int32 __kmp_task_stealing_constraint; 2003 #if OMP_41_ENABLED 2004 extern kmp_int32 __kmp_max_task_priority; // Set via OMP_MAX_TASK_PRIORITY if specified, defaults to 0 otherwise 2005 #endif 2006 2007 /* NOTE: kmp_taskdata_t and kmp_task_t structures allocated in single block with taskdata first */ 2008 #define KMP_TASK_TO_TASKDATA(task) (((kmp_taskdata_t *) task) - 1) 2009 #define KMP_TASKDATA_TO_TASK(taskdata) (kmp_task_t *) (taskdata + 1) 2010 2011 // The tt_found_tasks flag is a signal to all threads in the team that tasks were spawned and 2012 // queued since the previous barrier release. 2013 #define KMP_TASKING_ENABLED(task_team) \ 2014 (TCR_SYNC_4((task_team)->tt.tt_found_tasks) == TRUE) 2015 /*! 2016 @ingroup BASIC_TYPES 2017 @{ 2018 */ 2019 2020 /*! 2021 */ 2022 typedef kmp_int32 (* kmp_routine_entry_t)( kmp_int32, void * ); 2023 2024 #if OMP_40_ENABLED || OMP_41_ENABLED 2025 typedef union kmp_cmplrdata { 2026 #if OMP_41_ENABLED 2027 kmp_int32 priority; /**< priority specified by user for the task */ 2028 #endif // OMP_41_ENABLED 2029 #if OMP_40_ENABLED 2030 kmp_routine_entry_t destructors; /* pointer to function to invoke deconstructors of firstprivate C++ objects */ 2031 #endif // OMP_40_ENABLED 2032 /* future data */ 2033 } kmp_cmplrdata_t; 2034 #endif 2035 2036 /* sizeof_kmp_task_t passed as arg to kmpc_omp_task call */ 2037 /*! 2038 */ 2039 typedef struct kmp_task { /* GEH: Shouldn't this be aligned somehow? */ 2040 void * shareds; /**< pointer to block of pointers to shared vars */ 2041 kmp_routine_entry_t routine; /**< pointer to routine to call for executing task */ 2042 kmp_int32 part_id; /**< part id for the task */ 2043 #if OMP_40_ENABLED || OMP_41_ENABLED 2044 kmp_cmplrdata_t data1; /* Two known optional additions: destructors and priority */ 2045 kmp_cmplrdata_t data2; /* Process destructors first, priority second */ 2046 /* future data */ 2047 #endif 2048 /* private vars */ 2049 } kmp_task_t; 2050 2051 /*! 2052 @} 2053 */ 2054 2055 #if OMP_40_ENABLED 2056 typedef struct kmp_taskgroup { 2057 kmp_uint32 count; // number of allocated and not yet complete tasks 2058 kmp_int32 cancel_request; // request for cancellation of this taskgroup 2059 struct kmp_taskgroup *parent; // parent taskgroup 2060 } kmp_taskgroup_t; 2061 2062 2063 // forward declarations 2064 typedef union kmp_depnode kmp_depnode_t; 2065 typedef struct kmp_depnode_list kmp_depnode_list_t; 2066 typedef struct kmp_dephash_entry kmp_dephash_entry_t; 2067 2068 typedef struct kmp_depend_info { 2069 kmp_intptr_t base_addr; 2070 size_t len; 2071 struct { 2072 bool in:1; 2073 bool out:1; 2074 } flags; 2075 } kmp_depend_info_t; 2076 2077 struct kmp_depnode_list { 2078 kmp_depnode_t * node; 2079 kmp_depnode_list_t * next; 2080 }; 2081 2082 typedef struct kmp_base_depnode { 2083 kmp_depnode_list_t * successors; 2084 kmp_task_t * task; 2085 2086 kmp_lock_t lock; 2087 2088 #if KMP_SUPPORT_GRAPH_OUTPUT 2089 kmp_uint32 id; 2090 #endif 2091 2092 volatile kmp_int32 npredecessors; 2093 volatile kmp_int32 nrefs; 2094 } kmp_base_depnode_t; 2095 2096 union KMP_ALIGN_CACHE kmp_depnode { 2097 double dn_align; /* use worst case alignment */ 2098 char dn_pad[ KMP_PAD(kmp_base_depnode_t, CACHE_LINE) ]; 2099 kmp_base_depnode_t dn; 2100 }; 2101 2102 struct kmp_dephash_entry { 2103 kmp_intptr_t addr; 2104 kmp_depnode_t * last_out; 2105 kmp_depnode_list_t * last_ins; 2106 kmp_dephash_entry_t * next_in_bucket; 2107 }; 2108 2109 typedef struct kmp_dephash { 2110 kmp_dephash_entry_t ** buckets; 2111 size_t size; 2112 #ifdef KMP_DEBUG 2113 kmp_uint32 nelements; 2114 kmp_uint32 nconflicts; 2115 #endif 2116 } kmp_dephash_t; 2117 2118 #endif 2119 2120 #ifdef BUILD_TIED_TASK_STACK 2121 2122 /* Tied Task stack definitions */ 2123 typedef struct kmp_stack_block { 2124 kmp_taskdata_t * sb_block[ TASK_STACK_BLOCK_SIZE ]; 2125 struct kmp_stack_block * sb_next; 2126 struct kmp_stack_block * sb_prev; 2127 } kmp_stack_block_t; 2128 2129 typedef struct kmp_task_stack { 2130 kmp_stack_block_t ts_first_block; // first block of stack entries 2131 kmp_taskdata_t ** ts_top; // pointer to the top of stack 2132 kmp_int32 ts_entries; // number of entries on the stack 2133 } kmp_task_stack_t; 2134 2135 #endif // BUILD_TIED_TASK_STACK 2136 2137 typedef struct kmp_tasking_flags { /* Total struct must be exactly 32 bits */ 2138 /* Compiler flags */ /* Total compiler flags must be 16 bits */ 2139 unsigned tiedness : 1; /* task is either tied (1) or untied (0) */ 2140 unsigned final : 1; /* task is final(1) so execute immediately */ 2141 unsigned merged_if0 : 1; /* no __kmpc_task_{begin/complete}_if0 calls in if0 code path */ 2142 #if OMP_40_ENABLED 2143 unsigned destructors_thunk : 1; /* set if the compiler creates a thunk to invoke destructors from the runtime */ 2144 #if OMP_41_ENABLED 2145 unsigned proxy : 1; /* task is a proxy task (it will be executed outside the context of the RTL) */ 2146 unsigned priority_specified :1; /* set if the compiler provides priority setting for the task */ 2147 unsigned reserved : 10; /* reserved for compiler use */ 2148 #else 2149 unsigned reserved : 12; /* reserved for compiler use */ 2150 #endif 2151 #else // OMP_40_ENABLED 2152 unsigned reserved : 13; /* reserved for compiler use */ 2153 #endif // OMP_40_ENABLED 2154 2155 /* Library flags */ /* Total library flags must be 16 bits */ 2156 unsigned tasktype : 1; /* task is either explicit(1) or implicit (0) */ 2157 unsigned task_serial : 1; /* this task is executed immediately (1) or deferred (0) */ 2158 unsigned tasking_ser : 1; /* all tasks in team are either executed immediately (1) or may be deferred (0) */ 2159 unsigned team_serial : 1; /* entire team is serial (1) [1 thread] or parallel (0) [>= 2 threads] */ 2160 /* If either team_serial or tasking_ser is set, task team may be NULL */ 2161 /* Task State Flags: */ 2162 unsigned started : 1; /* 1==started, 0==not started */ 2163 unsigned executing : 1; /* 1==executing, 0==not executing */ 2164 unsigned complete : 1; /* 1==complete, 0==not complete */ 2165 unsigned freed : 1; /* 1==freed, 0==allocateed */ 2166 unsigned native : 1; /* 1==gcc-compiled task, 0==intel */ 2167 unsigned reserved31 : 7; /* reserved for library use */ 2168 2169 } kmp_tasking_flags_t; 2170 2171 2172 struct kmp_taskdata { /* aligned during dynamic allocation */ 2173 kmp_int32 td_task_id; /* id, assigned by debugger */ 2174 kmp_tasking_flags_t td_flags; /* task flags */ 2175 kmp_team_t * td_team; /* team for this task */ 2176 kmp_info_p * td_alloc_thread; /* thread that allocated data structures */ 2177 /* Currently not used except for perhaps IDB */ 2178 kmp_taskdata_t * td_parent; /* parent task */ 2179 kmp_int32 td_level; /* task nesting level */ 2180 ident_t * td_ident; /* task identifier */ 2181 // Taskwait data. 2182 ident_t * td_taskwait_ident; 2183 kmp_uint32 td_taskwait_counter; 2184 kmp_int32 td_taskwait_thread; /* gtid + 1 of thread encountered taskwait */ 2185 KMP_ALIGN_CACHE kmp_internal_control_t td_icvs; /* Internal control variables for the task */ 2186 volatile kmp_uint32 td_allocated_child_tasks; /* Child tasks (+ current task) not yet deallocated */ 2187 volatile kmp_uint32 td_incomplete_child_tasks; /* Child tasks not yet complete */ 2188 #if OMP_40_ENABLED 2189 kmp_taskgroup_t * td_taskgroup; // Each task keeps pointer to its current taskgroup 2190 kmp_dephash_t * td_dephash; // Dependencies for children tasks are tracked from here 2191 kmp_depnode_t * td_depnode; // Pointer to graph node if this task has dependencies 2192 #endif 2193 #if OMPT_SUPPORT 2194 ompt_task_info_t ompt_task_info; 2195 #endif 2196 #if OMP_41_ENABLED 2197 kmp_task_team_t * td_task_team; 2198 #endif 2199 #if KMP_HAVE_QUAD 2200 _Quad td_dummy; // Align structure 16-byte size since allocated just before kmp_task_t 2201 #else 2202 kmp_uint32 td_dummy[2]; 2203 #endif 2204 }; // struct kmp_taskdata 2205 2206 // Make sure padding above worked 2207 KMP_BUILD_ASSERT( sizeof(kmp_taskdata_t) % sizeof(void *) == 0 ); 2208 2209 // Data for task team but per thread 2210 typedef struct kmp_base_thread_data { 2211 kmp_info_p * td_thr; // Pointer back to thread info 2212 // Used only in __kmp_execute_tasks_template, maybe not avail until task is queued? 2213 kmp_bootstrap_lock_t td_deque_lock; // Lock for accessing deque 2214 kmp_taskdata_t ** td_deque; // Deque of tasks encountered by td_thr, dynamically allocated 2215 kmp_uint32 td_deque_head; // Head of deque (will wrap) 2216 kmp_uint32 td_deque_tail; // Tail of deque (will wrap) 2217 kmp_int32 td_deque_ntasks; // Number of tasks in deque 2218 // GEH: shouldn't this be volatile since used in while-spin? 2219 kmp_int32 td_deque_last_stolen; // Thread number of last successful steal 2220 #ifdef BUILD_TIED_TASK_STACK 2221 kmp_task_stack_t td_susp_tied_tasks; // Stack of suspended tied tasks for task scheduling constraint 2222 #endif // BUILD_TIED_TASK_STACK 2223 } kmp_base_thread_data_t; 2224 2225 typedef union KMP_ALIGN_CACHE kmp_thread_data { 2226 kmp_base_thread_data_t td; 2227 double td_align; /* use worst case alignment */ 2228 char td_pad[ KMP_PAD(kmp_base_thread_data_t, CACHE_LINE) ]; 2229 } kmp_thread_data_t; 2230 2231 2232 // Data for task teams which are used when tasking is enabled for the team 2233 typedef struct kmp_base_task_team { 2234 kmp_bootstrap_lock_t tt_threads_lock; /* Lock used to allocate per-thread part of task team */ 2235 /* must be bootstrap lock since used at library shutdown*/ 2236 kmp_task_team_t * tt_next; /* For linking the task team free list */ 2237 kmp_thread_data_t * tt_threads_data; /* Array of per-thread structures for task team */ 2238 /* Data survives task team deallocation */ 2239 kmp_int32 tt_found_tasks; /* Have we found tasks and queued them while executing this team? */ 2240 /* TRUE means tt_threads_data is set up and initialized */ 2241 kmp_int32 tt_nproc; /* #threads in team */ 2242 kmp_int32 tt_max_threads; /* number of entries allocated for threads_data array */ 2243 #if OMP_41_ENABLED 2244 kmp_int32 tt_found_proxy_tasks; /* Have we found proxy tasks since last barrier */ 2245 #endif 2246 2247 KMP_ALIGN_CACHE 2248 volatile kmp_uint32 tt_unfinished_threads; /* #threads still active */ 2249 2250 KMP_ALIGN_CACHE 2251 volatile kmp_uint32 tt_active; /* is the team still actively executing tasks */ 2252 } kmp_base_task_team_t; 2253 2254 union KMP_ALIGN_CACHE kmp_task_team { 2255 kmp_base_task_team_t tt; 2256 double tt_align; /* use worst case alignment */ 2257 char tt_pad[ KMP_PAD(kmp_base_task_team_t, CACHE_LINE) ]; 2258 }; 2259 2260 #if ( USE_FAST_MEMORY == 3 ) || ( USE_FAST_MEMORY == 5 ) 2261 // Free lists keep same-size free memory slots for fast memory allocation routines 2262 typedef struct kmp_free_list { 2263 void *th_free_list_self; // Self-allocated tasks free list 2264 void *th_free_list_sync; // Self-allocated tasks stolen/returned by other threads 2265 void *th_free_list_other; // Non-self free list (to be returned to owner's sync list) 2266 } kmp_free_list_t; 2267 #endif 2268 #if KMP_NESTED_HOT_TEAMS 2269 // Hot teams array keeps hot teams and their sizes for given thread. 2270 // Hot teams are not put in teams pool, and they don't put threads in threads pool. 2271 typedef struct kmp_hot_team_ptr { 2272 kmp_team_p *hot_team; // pointer to hot_team of given nesting level 2273 kmp_int32 hot_team_nth; // number of threads allocated for the hot_team 2274 } kmp_hot_team_ptr_t; 2275 #endif 2276 #if OMP_40_ENABLED 2277 typedef struct kmp_teams_size { 2278 kmp_int32 nteams; // number of teams in a league 2279 kmp_int32 nth; // number of threads in each team of the league 2280 } kmp_teams_size_t; 2281 #endif 2282 2283 /* ------------------------------------------------------------------------ */ 2284 // OpenMP thread data structures 2285 // 2286 2287 typedef struct KMP_ALIGN_CACHE kmp_base_info { 2288 /* 2289 * Start with the readonly data which is cache aligned and padded. 2290 * this is written before the thread starts working by the master. 2291 * (uber masters may update themselves later) 2292 * (usage does not consider serialized regions) 2293 */ 2294 kmp_desc_t th_info; 2295 kmp_team_p *th_team; /* team we belong to */ 2296 kmp_root_p *th_root; /* pointer to root of task hierarchy */ 2297 kmp_info_p *th_next_pool; /* next available thread in the pool */ 2298 kmp_disp_t *th_dispatch; /* thread's dispatch data */ 2299 int th_in_pool; /* in thread pool (32 bits for TCR/TCW) */ 2300 2301 /* The following are cached from the team info structure */ 2302 /* TODO use these in more places as determined to be needed via profiling */ 2303 int th_team_nproc; /* number of threads in a team */ 2304 kmp_info_p *th_team_master; /* the team's master thread */ 2305 int th_team_serialized; /* team is serialized */ 2306 #if OMP_40_ENABLED 2307 microtask_t th_teams_microtask; /* save entry address for teams construct */ 2308 int th_teams_level; /* save initial level of teams construct */ 2309 /* it is 0 on device but may be any on host */ 2310 #endif 2311 2312 /* The blocktime info is copied from the team struct to the thread sruct */ 2313 /* at the start of a barrier, and the values stored in the team are used */ 2314 /* at points in the code where the team struct is no longer guaranteed */ 2315 /* to exist (from the POV of worker threads). */ 2316 int th_team_bt_intervals; 2317 int th_team_bt_set; 2318 2319 2320 #if KMP_AFFINITY_SUPPORTED 2321 kmp_affin_mask_t *th_affin_mask; /* thread's current affinity mask */ 2322 #endif 2323 2324 /* 2325 * The data set by the master at reinit, then R/W by the worker 2326 */ 2327 KMP_ALIGN_CACHE int th_set_nproc; /* if > 0, then only use this request for the next fork */ 2328 #if KMP_NESTED_HOT_TEAMS 2329 kmp_hot_team_ptr_t *th_hot_teams; /* array of hot teams */ 2330 #endif 2331 #if OMP_40_ENABLED 2332 kmp_proc_bind_t th_set_proc_bind; /* if != proc_bind_default, use request for next fork */ 2333 kmp_teams_size_t th_teams_size; /* number of teams/threads in teams construct */ 2334 # if KMP_AFFINITY_SUPPORTED 2335 int th_current_place; /* place currently bound to */ 2336 int th_new_place; /* place to bind to in par reg */ 2337 int th_first_place; /* first place in partition */ 2338 int th_last_place; /* last place in partition */ 2339 # endif 2340 #endif 2341 #if USE_ITT_BUILD 2342 kmp_uint64 th_bar_arrive_time; /* arrival to barrier timestamp */ 2343 kmp_uint64 th_bar_min_time; /* minimum arrival time at the barrier */ 2344 kmp_uint64 th_frame_time; /* frame timestamp */ 2345 kmp_uint64 th_frame_time_serialized; /* frame timestamp in serialized parallel */ 2346 #endif /* USE_ITT_BUILD */ 2347 kmp_local_t th_local; 2348 struct private_common *th_pri_head; 2349 2350 /* 2351 * Now the data only used by the worker (after initial allocation) 2352 */ 2353 /* TODO the first serial team should actually be stored in the info_t 2354 * structure. this will help reduce initial allocation overhead */ 2355 KMP_ALIGN_CACHE kmp_team_p *th_serial_team; /*serialized team held in reserve*/ 2356 2357 #if OMPT_SUPPORT 2358 ompt_thread_info_t ompt_thread_info; 2359 #endif 2360 2361 /* The following are also read by the master during reinit */ 2362 struct common_table *th_pri_common; 2363 2364 volatile kmp_uint32 th_spin_here; /* thread-local location for spinning */ 2365 /* while awaiting queuing lock acquire */ 2366 2367 volatile void *th_sleep_loc; // this points at a kmp_flag<T> 2368 2369 ident_t *th_ident; 2370 unsigned th_x; // Random number generator data 2371 unsigned th_a; // Random number generator data 2372 2373 /* 2374 * Tasking-related data for the thread 2375 */ 2376 kmp_task_team_t * th_task_team; // Task team struct 2377 kmp_taskdata_t * th_current_task; // Innermost Task being executed 2378 kmp_uint8 th_task_state; // alternating 0/1 for task team identification 2379 kmp_uint8 * th_task_state_memo_stack; // Stack holding memos of th_task_state at nested levels 2380 kmp_uint32 th_task_state_top; // Top element of th_task_state_memo_stack 2381 kmp_uint32 th_task_state_stack_sz; // Size of th_task_state_memo_stack 2382 2383 /* 2384 * More stuff for keeping track of active/sleeping threads 2385 * (this part is written by the worker thread) 2386 */ 2387 kmp_uint8 th_active_in_pool; // included in count of 2388 // #active threads in pool 2389 int th_active; // ! sleeping 2390 // 32 bits for TCR/TCW 2391 2392 struct cons_header * th_cons; // used for consistency check 2393 2394 /* 2395 * Add the syncronizing data which is cache aligned and padded. 2396 */ 2397 KMP_ALIGN_CACHE kmp_balign_t th_bar[ bs_last_barrier ]; 2398 2399 KMP_ALIGN_CACHE volatile kmp_int32 th_next_waiting; /* gtid+1 of next thread on lock wait queue, 0 if none */ 2400 2401 #if ( USE_FAST_MEMORY == 3 ) || ( USE_FAST_MEMORY == 5 ) 2402 #define NUM_LISTS 4 2403 kmp_free_list_t th_free_lists[NUM_LISTS]; // Free lists for fast memory allocation routines 2404 #endif 2405 2406 #if KMP_OS_WINDOWS 2407 kmp_win32_cond_t th_suspend_cv; 2408 kmp_win32_mutex_t th_suspend_mx; 2409 int th_suspend_init; 2410 #endif 2411 #if KMP_OS_UNIX 2412 kmp_cond_align_t th_suspend_cv; 2413 kmp_mutex_align_t th_suspend_mx; 2414 int th_suspend_init_count; 2415 #endif 2416 2417 #if USE_ITT_BUILD 2418 kmp_itt_mark_t th_itt_mark_single; 2419 // alignment ??? 2420 #endif /* USE_ITT_BUILD */ 2421 #if KMP_STATS_ENABLED 2422 kmp_stats_list* th_stats; 2423 #endif 2424 } kmp_base_info_t; 2425 2426 typedef union KMP_ALIGN_CACHE kmp_info { 2427 double th_align; /* use worst case alignment */ 2428 char th_pad[ KMP_PAD(kmp_base_info_t, CACHE_LINE) ]; 2429 kmp_base_info_t th; 2430 } kmp_info_t; 2431 2432 /* ------------------------------------------------------------------------ */ 2433 // OpenMP thread team data structures 2434 // 2435 typedef struct kmp_base_data { 2436 volatile kmp_uint32 t_value; 2437 } kmp_base_data_t; 2438 2439 typedef union KMP_ALIGN_CACHE kmp_sleep_team { 2440 double dt_align; /* use worst case alignment */ 2441 char dt_pad[ KMP_PAD(kmp_base_data_t, CACHE_LINE) ]; 2442 kmp_base_data_t dt; 2443 } kmp_sleep_team_t; 2444 2445 typedef union KMP_ALIGN_CACHE kmp_ordered_team { 2446 double dt_align; /* use worst case alignment */ 2447 char dt_pad[ KMP_PAD(kmp_base_data_t, CACHE_LINE) ]; 2448 kmp_base_data_t dt; 2449 } kmp_ordered_team_t; 2450 2451 typedef int (*launch_t)( int gtid ); 2452 2453 /* Minimum number of ARGV entries to malloc if necessary */ 2454 #define KMP_MIN_MALLOC_ARGV_ENTRIES 100 2455 2456 // Set up how many argv pointers will fit in cache lines containing t_inline_argv. Historically, we 2457 // have supported at least 96 bytes. Using a larger value for more space between the master write/worker 2458 // read section and read/write by all section seems to buy more performance on EPCC PARALLEL. 2459 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 2460 # define KMP_INLINE_ARGV_BYTES ( 4 * CACHE_LINE - ( ( 3 * KMP_PTR_SKIP + 2 * sizeof(int) + 2 * sizeof(kmp_int8) + sizeof(kmp_int16) + sizeof(kmp_uint32) ) % CACHE_LINE ) ) 2461 #else 2462 # define KMP_INLINE_ARGV_BYTES ( 2 * CACHE_LINE - ( ( 3 * KMP_PTR_SKIP + 2 * sizeof(int) ) % CACHE_LINE ) ) 2463 #endif 2464 #define KMP_INLINE_ARGV_ENTRIES (int)( KMP_INLINE_ARGV_BYTES / KMP_PTR_SKIP ) 2465 2466 typedef struct KMP_ALIGN_CACHE kmp_base_team { 2467 // Synchronization Data --------------------------------------------------------------------------------- 2468 KMP_ALIGN_CACHE kmp_ordered_team_t t_ordered; 2469 kmp_balign_team_t t_bar[ bs_last_barrier ]; 2470 volatile int t_construct; // count of single directive encountered by team 2471 kmp_lock_t t_single_lock; // team specific lock 2472 2473 // Master only ----------------------------------------------------------------------------------------- 2474 KMP_ALIGN_CACHE int t_master_tid; // tid of master in parent team 2475 int t_master_this_cons; // "this_construct" single counter of master in parent team 2476 ident_t *t_ident; // if volatile, have to change too much other crud to volatile too 2477 kmp_team_p *t_parent; // parent team 2478 kmp_team_p *t_next_pool; // next free team in the team pool 2479 kmp_disp_t *t_dispatch; // thread's dispatch data 2480 kmp_task_team_t *t_task_team[2]; // Task team struct; switch between 2 2481 #if OMP_40_ENABLED 2482 kmp_proc_bind_t t_proc_bind; // bind type for par region 2483 #endif // OMP_40_ENABLED 2484 #if USE_ITT_BUILD 2485 kmp_uint64 t_region_time; // region begin timestamp 2486 #endif /* USE_ITT_BUILD */ 2487 2488 // Master write, workers read -------------------------------------------------------------------------- 2489 KMP_ALIGN_CACHE void **t_argv; 2490 int t_argc; 2491 int t_nproc; // number of threads in team 2492 microtask_t t_pkfn; 2493 launch_t t_invoke; // procedure to launch the microtask 2494 2495 #if OMPT_SUPPORT 2496 ompt_team_info_t ompt_team_info; 2497 ompt_lw_taskteam_t *ompt_serialized_team_info; 2498 #endif 2499 2500 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 2501 kmp_int8 t_fp_control_saved; 2502 kmp_int8 t_pad2b; 2503 kmp_int16 t_x87_fpu_control_word; // FP control regs 2504 kmp_uint32 t_mxcsr; 2505 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 2506 2507 void *t_inline_argv[ KMP_INLINE_ARGV_ENTRIES ]; 2508 2509 KMP_ALIGN_CACHE kmp_info_t **t_threads; 2510 int t_max_argc; 2511 int t_max_nproc; // maximum threads this team can handle (dynamicly expandable) 2512 int t_serialized; // levels deep of serialized teams 2513 dispatch_shared_info_t *t_disp_buffer; // buffers for dispatch system 2514 int t_id; // team's id, assigned by debugger. 2515 int t_level; // nested parallel level 2516 int t_active_level; // nested active parallel level 2517 kmp_r_sched_t t_sched; // run-time schedule for the team 2518 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 2519 int t_first_place; // first & last place in parent thread's partition. 2520 int t_last_place; // Restore these values to master after par region. 2521 #endif // OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 2522 int t_size_changed; // team size was changed?: 0: no, 1: yes, -1: changed via omp_set_num_threads() call 2523 2524 // Read/write by workers as well ----------------------------------------------------------------------- 2525 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 2526 // Using CACHE_LINE=64 reduces memory footprint, but causes a big perf regression of epcc 'parallel' 2527 // and 'barrier' on fxe256lin01. This extra padding serves to fix the performance of epcc 'parallel' 2528 // and 'barrier' when CACHE_LINE=64. TODO: investigate more and get rid if this padding. 2529 char dummy_padding[1024]; 2530 #endif 2531 KMP_ALIGN_CACHE kmp_taskdata_t *t_implicit_task_taskdata; // Taskdata for the thread's implicit task 2532 kmp_internal_control_t *t_control_stack_top; // internal control stack for additional nested teams. 2533 // for SERIALIZED teams nested 2 or more levels deep 2534 #if OMP_40_ENABLED 2535 kmp_int32 t_cancel_request; // typed flag to store request state of cancellation 2536 #endif 2537 int t_master_active; // save on fork, restore on join 2538 kmp_taskq_t t_taskq; // this team's task queue 2539 void *t_copypriv_data; // team specific pointer to copyprivate data array 2540 kmp_uint32 t_copyin_counter; 2541 #if USE_ITT_BUILD 2542 void *t_stack_id; // team specific stack stitching id (for ittnotify) 2543 #endif /* USE_ITT_BUILD */ 2544 } kmp_base_team_t; 2545 2546 union KMP_ALIGN_CACHE kmp_team { 2547 kmp_base_team_t t; 2548 double t_align; /* use worst case alignment */ 2549 char t_pad[ KMP_PAD(kmp_base_team_t, CACHE_LINE) ]; 2550 }; 2551 2552 2553 typedef union KMP_ALIGN_CACHE kmp_time_global { 2554 double dt_align; /* use worst case alignment */ 2555 char dt_pad[ KMP_PAD(kmp_base_data_t, CACHE_LINE) ]; 2556 kmp_base_data_t dt; 2557 } kmp_time_global_t; 2558 2559 typedef struct kmp_base_global { 2560 /* cache-aligned */ 2561 kmp_time_global_t g_time; 2562 2563 /* non cache-aligned */ 2564 volatile int g_abort; 2565 volatile int g_done; 2566 2567 int g_dynamic; 2568 enum dynamic_mode g_dynamic_mode; 2569 } kmp_base_global_t; 2570 2571 typedef union KMP_ALIGN_CACHE kmp_global { 2572 kmp_base_global_t g; 2573 double g_align; /* use worst case alignment */ 2574 char g_pad[ KMP_PAD(kmp_base_global_t, CACHE_LINE) ]; 2575 } kmp_global_t; 2576 2577 2578 typedef struct kmp_base_root { 2579 // TODO: GEH - combine r_active with r_in_parallel then r_active == (r_in_parallel>= 0) 2580 // TODO: GEH - then replace r_active with t_active_levels if we can to reduce the synch 2581 // overhead or keeping r_active 2582 2583 volatile int r_active; /* TRUE if some region in a nest has > 1 thread */ 2584 // GEH: This is misnamed, should be r_in_parallel 2585 volatile int r_nested; // TODO: GEH - This is unused, just remove it entirely. 2586 int r_in_parallel; /* keeps a count of active parallel regions per root */ 2587 // GEH: This is misnamed, should be r_active_levels 2588 kmp_team_t *r_root_team; 2589 kmp_team_t *r_hot_team; 2590 kmp_info_t *r_uber_thread; 2591 kmp_lock_t r_begin_lock; 2592 volatile int r_begin; 2593 int r_blocktime; /* blocktime for this root and descendants */ 2594 } kmp_base_root_t; 2595 2596 typedef union KMP_ALIGN_CACHE kmp_root { 2597 kmp_base_root_t r; 2598 double r_align; /* use worst case alignment */ 2599 char r_pad[ KMP_PAD(kmp_base_root_t, CACHE_LINE) ]; 2600 } kmp_root_t; 2601 2602 struct fortran_inx_info { 2603 kmp_int32 data; 2604 }; 2605 2606 /* ------------------------------------------------------------------------ */ 2607 2608 /* ------------------------------------------------------------------------ */ 2609 /* ------------------------------------------------------------------------ */ 2610 2611 extern int __kmp_settings; 2612 extern int __kmp_duplicate_library_ok; 2613 #if USE_ITT_BUILD 2614 extern int __kmp_forkjoin_frames; 2615 extern int __kmp_forkjoin_frames_mode; 2616 #endif 2617 extern PACKED_REDUCTION_METHOD_T __kmp_force_reduction_method; 2618 extern int __kmp_determ_red; 2619 2620 #ifdef KMP_DEBUG 2621 extern int kmp_a_debug; 2622 extern int kmp_b_debug; 2623 extern int kmp_c_debug; 2624 extern int kmp_d_debug; 2625 extern int kmp_e_debug; 2626 extern int kmp_f_debug; 2627 #endif /* KMP_DEBUG */ 2628 2629 /* For debug information logging using rotating buffer */ 2630 #define KMP_DEBUG_BUF_LINES_INIT 512 2631 #define KMP_DEBUG_BUF_LINES_MIN 1 2632 2633 #define KMP_DEBUG_BUF_CHARS_INIT 128 2634 #define KMP_DEBUG_BUF_CHARS_MIN 2 2635 2636 extern int __kmp_debug_buf; /* TRUE means use buffer, FALSE means print to stderr */ 2637 extern int __kmp_debug_buf_lines; /* How many lines of debug stored in buffer */ 2638 extern int __kmp_debug_buf_chars; /* How many characters allowed per line in buffer */ 2639 extern int __kmp_debug_buf_atomic; /* TRUE means use atomic update of buffer entry pointer */ 2640 2641 extern char *__kmp_debug_buffer; /* Debug buffer itself */ 2642 extern int __kmp_debug_count; /* Counter for number of lines printed in buffer so far */ 2643 extern int __kmp_debug_buf_warn_chars; /* Keep track of char increase recommended in warnings */ 2644 /* end rotating debug buffer */ 2645 2646 #ifdef KMP_DEBUG 2647 extern int __kmp_par_range; /* +1 => only go par for constructs in range */ 2648 2649 #define KMP_PAR_RANGE_ROUTINE_LEN 1024 2650 extern char __kmp_par_range_routine[KMP_PAR_RANGE_ROUTINE_LEN]; 2651 #define KMP_PAR_RANGE_FILENAME_LEN 1024 2652 extern char __kmp_par_range_filename[KMP_PAR_RANGE_FILENAME_LEN]; 2653 extern int __kmp_par_range_lb; 2654 extern int __kmp_par_range_ub; 2655 #endif 2656 2657 /* For printing out dynamic storage map for threads and teams */ 2658 extern int __kmp_storage_map; /* True means print storage map for threads and teams */ 2659 extern int __kmp_storage_map_verbose; /* True means storage map includes placement info */ 2660 extern int __kmp_storage_map_verbose_specified; 2661 2662 extern kmp_cpuinfo_t __kmp_cpuinfo; 2663 2664 extern volatile int __kmp_init_serial; 2665 extern volatile int __kmp_init_gtid; 2666 extern volatile int __kmp_init_common; 2667 extern volatile int __kmp_init_middle; 2668 extern volatile int __kmp_init_parallel; 2669 extern volatile int __kmp_init_monitor; 2670 extern volatile int __kmp_init_user_locks; 2671 extern int __kmp_init_counter; 2672 extern int __kmp_root_counter; 2673 extern int __kmp_version; 2674 2675 /* list of address of allocated caches for commons */ 2676 extern kmp_cached_addr_t *__kmp_threadpriv_cache_list; 2677 2678 /* Barrier algorithm types and options */ 2679 extern kmp_uint32 __kmp_barrier_gather_bb_dflt; 2680 extern kmp_uint32 __kmp_barrier_release_bb_dflt; 2681 extern kmp_bar_pat_e __kmp_barrier_gather_pat_dflt; 2682 extern kmp_bar_pat_e __kmp_barrier_release_pat_dflt; 2683 extern kmp_uint32 __kmp_barrier_gather_branch_bits [ bs_last_barrier ]; 2684 extern kmp_uint32 __kmp_barrier_release_branch_bits [ bs_last_barrier ]; 2685 extern kmp_bar_pat_e __kmp_barrier_gather_pattern [ bs_last_barrier ]; 2686 extern kmp_bar_pat_e __kmp_barrier_release_pattern [ bs_last_barrier ]; 2687 extern char const *__kmp_barrier_branch_bit_env_name [ bs_last_barrier ]; 2688 extern char const *__kmp_barrier_pattern_env_name [ bs_last_barrier ]; 2689 extern char const *__kmp_barrier_type_name [ bs_last_barrier ]; 2690 extern char const *__kmp_barrier_pattern_name [ bp_last_bar ]; 2691 2692 /* Global Locks */ 2693 extern kmp_bootstrap_lock_t __kmp_initz_lock; /* control initialization */ 2694 extern kmp_bootstrap_lock_t __kmp_forkjoin_lock; /* control fork/join access */ 2695 extern kmp_bootstrap_lock_t __kmp_exit_lock; /* exit() is not always thread-safe */ 2696 extern kmp_bootstrap_lock_t __kmp_monitor_lock; /* control monitor thread creation */ 2697 extern kmp_bootstrap_lock_t __kmp_tp_cached_lock; /* used for the hack to allow threadprivate cache and __kmp_threads expansion to co-exist */ 2698 2699 extern kmp_lock_t __kmp_global_lock; /* control OS/global access */ 2700 extern kmp_queuing_lock_t __kmp_dispatch_lock; /* control dispatch access */ 2701 extern kmp_lock_t __kmp_debug_lock; /* control I/O access for KMP_DEBUG */ 2702 2703 /* used for yielding spin-waits */ 2704 extern unsigned int __kmp_init_wait; /* initial number of spin-tests */ 2705 extern unsigned int __kmp_next_wait; /* susequent number of spin-tests */ 2706 2707 extern enum library_type __kmp_library; 2708 2709 extern enum sched_type __kmp_sched; /* default runtime scheduling */ 2710 extern enum sched_type __kmp_static; /* default static scheduling method */ 2711 extern enum sched_type __kmp_guided; /* default guided scheduling method */ 2712 extern enum sched_type __kmp_auto; /* default auto scheduling method */ 2713 extern int __kmp_chunk; /* default runtime chunk size */ 2714 2715 extern size_t __kmp_stksize; /* stack size per thread */ 2716 extern size_t __kmp_monitor_stksize;/* stack size for monitor thread */ 2717 extern size_t __kmp_stkoffset; /* stack offset per thread */ 2718 extern int __kmp_stkpadding; /* Should we pad root thread(s) stack */ 2719 2720 extern size_t __kmp_malloc_pool_incr; /* incremental size of pool for kmp_malloc() */ 2721 extern int __kmp_env_chunk; /* was KMP_CHUNK specified? */ 2722 extern int __kmp_env_stksize; /* was KMP_STACKSIZE specified? */ 2723 extern int __kmp_env_omp_stksize;/* was OMP_STACKSIZE specified? */ 2724 extern int __kmp_env_all_threads; /* was KMP_ALL_THREADS or KMP_MAX_THREADS specified? */ 2725 extern int __kmp_env_omp_all_threads;/* was OMP_THREAD_LIMIT specified? */ 2726 extern int __kmp_env_blocktime; /* was KMP_BLOCKTIME specified? */ 2727 extern int __kmp_env_checks; /* was KMP_CHECKS specified? */ 2728 extern int __kmp_env_consistency_check; /* was KMP_CONSISTENCY_CHECK specified? */ 2729 extern int __kmp_generate_warnings; /* should we issue warnings? */ 2730 extern int __kmp_reserve_warn; /* have we issued reserve_threads warning? */ 2731 2732 #ifdef DEBUG_SUSPEND 2733 extern int __kmp_suspend_count; /* count inside __kmp_suspend_template() */ 2734 #endif 2735 2736 extern kmp_uint32 __kmp_yield_init; 2737 extern kmp_uint32 __kmp_yield_next; 2738 extern kmp_uint32 __kmp_yielding_on; 2739 extern kmp_uint32 __kmp_yield_cycle; 2740 extern kmp_int32 __kmp_yield_on_count; 2741 extern kmp_int32 __kmp_yield_off_count; 2742 2743 /* ------------------------------------------------------------------------- */ 2744 extern int __kmp_allThreadsSpecified; 2745 2746 extern size_t __kmp_align_alloc; 2747 /* following data protected by initialization routines */ 2748 extern int __kmp_xproc; /* number of processors in the system */ 2749 extern int __kmp_avail_proc; /* number of processors available to the process */ 2750 extern size_t __kmp_sys_min_stksize; /* system-defined minimum stack size */ 2751 extern int __kmp_sys_max_nth; /* system-imposed maximum number of threads */ 2752 extern int __kmp_max_nth; /* maximum total number of concurrently-existing threads */ 2753 extern int __kmp_threads_capacity; /* capacity of the arrays __kmp_threads and __kmp_root */ 2754 extern int __kmp_dflt_team_nth; /* default number of threads in a parallel region a la OMP_NUM_THREADS */ 2755 extern int __kmp_dflt_team_nth_ub; /* upper bound on "" determined at serial initialization */ 2756 extern int __kmp_tp_capacity; /* capacity of __kmp_threads if threadprivate is used (fixed) */ 2757 extern int __kmp_tp_cached; /* whether threadprivate cache has been created (__kmpc_threadprivate_cached()) */ 2758 extern int __kmp_dflt_nested; /* nested parallelism enabled by default a la OMP_NESTED */ 2759 extern int __kmp_dflt_blocktime; /* number of milliseconds to wait before blocking (env setting) */ 2760 extern int __kmp_monitor_wakeups;/* number of times monitor wakes up per second */ 2761 extern int __kmp_bt_intervals; /* number of monitor timestamp intervals before blocking */ 2762 #ifdef KMP_ADJUST_BLOCKTIME 2763 extern int __kmp_zero_bt; /* whether blocktime has been forced to zero */ 2764 #endif /* KMP_ADJUST_BLOCKTIME */ 2765 #ifdef KMP_DFLT_NTH_CORES 2766 extern int __kmp_ncores; /* Total number of cores for threads placement */ 2767 #endif 2768 extern int __kmp_abort_delay; /* Number of millisecs to delay on abort for VTune */ 2769 2770 extern int __kmp_need_register_atfork_specified; 2771 extern int __kmp_need_register_atfork;/* At initialization, call pthread_atfork to install fork handler */ 2772 extern int __kmp_gtid_mode; /* Method of getting gtid, values: 2773 0 - not set, will be set at runtime 2774 1 - using stack search 2775 2 - dynamic TLS (pthread_getspecific(Linux* OS/OS X*) or TlsGetValue(Windows* OS)) 2776 3 - static TLS (__declspec(thread) __kmp_gtid), Linux* OS .so only. 2777 */ 2778 extern int __kmp_adjust_gtid_mode; /* If true, adjust method based on #threads */ 2779 #ifdef KMP_TDATA_GTID 2780 #if KMP_OS_WINDOWS 2781 extern __declspec(thread) int __kmp_gtid; /* This thread's gtid, if __kmp_gtid_mode == 3 */ 2782 #else 2783 extern __thread int __kmp_gtid; 2784 #endif /* KMP_OS_WINDOWS - workaround because Intel(R) Many Integrated Core compiler 20110316 doesn't accept __declspec */ 2785 #endif 2786 extern int __kmp_tls_gtid_min; /* #threads below which use sp search for gtid */ 2787 extern int __kmp_foreign_tp; /* If true, separate TP var for each foreign thread */ 2788 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 2789 extern int __kmp_inherit_fp_control; /* copy fp creg(s) parent->workers at fork */ 2790 extern kmp_int16 __kmp_init_x87_fpu_control_word; /* init thread's FP control reg */ 2791 extern kmp_uint32 __kmp_init_mxcsr; /* init thread's mxscr */ 2792 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 2793 2794 extern int __kmp_dflt_max_active_levels; /* max_active_levels for nested parallelism enabled by default a la OMP_MAX_ACTIVE_LEVELS */ 2795 #if KMP_NESTED_HOT_TEAMS 2796 extern int __kmp_hot_teams_mode; 2797 extern int __kmp_hot_teams_max_level; 2798 #endif 2799 2800 # if KMP_OS_LINUX 2801 extern enum clock_function_type __kmp_clock_function; 2802 extern int __kmp_clock_function_param; 2803 # endif /* KMP_OS_LINUX */ 2804 2805 #if KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_WINDOWS) 2806 extern enum mic_type __kmp_mic_type; 2807 #endif 2808 2809 # ifdef USE_LOAD_BALANCE 2810 extern double __kmp_load_balance_interval; /* Interval for the load balance algorithm */ 2811 # endif /* USE_LOAD_BALANCE */ 2812 2813 // OpenMP 3.1 - Nested num threads array 2814 typedef struct kmp_nested_nthreads_t { 2815 int * nth; 2816 int size; 2817 int used; 2818 } kmp_nested_nthreads_t; 2819 2820 extern kmp_nested_nthreads_t __kmp_nested_nth; 2821 2822 #if KMP_USE_ADAPTIVE_LOCKS 2823 2824 // Parameters for the speculative lock backoff system. 2825 struct kmp_adaptive_backoff_params_t { 2826 // Number of soft retries before it counts as a hard retry. 2827 kmp_uint32 max_soft_retries; 2828 // Badness is a bit mask : 0,1,3,7,15,... on each hard failure we move one to the right 2829 kmp_uint32 max_badness; 2830 }; 2831 2832 extern kmp_adaptive_backoff_params_t __kmp_adaptive_backoff_params; 2833 2834 #if KMP_DEBUG_ADAPTIVE_LOCKS 2835 extern char * __kmp_speculative_statsfile; 2836 #endif 2837 2838 #endif // KMP_USE_ADAPTIVE_LOCKS 2839 2840 #if OMP_40_ENABLED 2841 extern int __kmp_display_env; /* TRUE or FALSE */ 2842 extern int __kmp_display_env_verbose; /* TRUE if OMP_DISPLAY_ENV=VERBOSE */ 2843 extern int __kmp_omp_cancellation; /* TRUE or FALSE */ 2844 #endif 2845 2846 /* ------------------------------------------------------------------------- */ 2847 2848 /* --------------------------------------------------------------------------- */ 2849 /* the following are protected by the fork/join lock */ 2850 /* write: lock read: anytime */ 2851 extern kmp_info_t **__kmp_threads; /* Descriptors for the threads */ 2852 /* read/write: lock */ 2853 extern volatile kmp_team_t * __kmp_team_pool; 2854 extern volatile kmp_info_t * __kmp_thread_pool; 2855 2856 /* total number of threads reachable from some root thread including all root threads*/ 2857 extern volatile int __kmp_nth; 2858 /* total number of threads reachable from some root thread including all root threads, 2859 and those in the thread pool */ 2860 extern volatile int __kmp_all_nth; 2861 extern int __kmp_thread_pool_nth; 2862 extern volatile int __kmp_thread_pool_active_nth; 2863 2864 extern kmp_root_t **__kmp_root; /* root of thread hierarchy */ 2865 /* end data protected by fork/join lock */ 2866 /* --------------------------------------------------------------------------- */ 2867 2868 extern kmp_global_t __kmp_global; /* global status */ 2869 2870 extern kmp_info_t __kmp_monitor; 2871 extern volatile kmp_uint32 __kmp_team_counter; // Used by Debugging Support Library. 2872 extern volatile kmp_uint32 __kmp_task_counter; // Used by Debugging Support Library. 2873 2874 #if USE_DEBUGGER 2875 2876 #define _KMP_GEN_ID( counter ) \ 2877 ( \ 2878 __kmp_debugging \ 2879 ? \ 2880 KMP_TEST_THEN_INC32( (volatile kmp_int32 *) & counter ) + 1 \ 2881 : \ 2882 ~ 0 \ 2883 ) 2884 #else 2885 #define _KMP_GEN_ID( counter ) \ 2886 ( \ 2887 ~ 0 \ 2888 ) 2889 #endif /* USE_DEBUGGER */ 2890 2891 #define KMP_GEN_TASK_ID() _KMP_GEN_ID( __kmp_task_counter ) 2892 #define KMP_GEN_TEAM_ID() _KMP_GEN_ID( __kmp_team_counter ) 2893 2894 /* ------------------------------------------------------------------------ */ 2895 /* ------------------------------------------------------------------------ */ 2896 2897 extern void __kmp_print_storage_map_gtid( int gtid, void *p1, void* p2, size_t size, char const *format, ... ); 2898 2899 extern void __kmp_serial_initialize( void ); 2900 extern void __kmp_middle_initialize( void ); 2901 extern void __kmp_parallel_initialize( void ); 2902 2903 extern void __kmp_internal_begin( void ); 2904 extern void __kmp_internal_end_library( int gtid ); 2905 extern void __kmp_internal_end_thread( int gtid ); 2906 extern void __kmp_internal_end_atexit( void ); 2907 extern void __kmp_internal_end_fini( void ); 2908 extern void __kmp_internal_end_dtor( void ); 2909 extern void __kmp_internal_end_dest( void* ); 2910 2911 extern int __kmp_register_root( int initial_thread ); 2912 extern void __kmp_unregister_root( int gtid ); 2913 2914 extern int __kmp_ignore_mppbeg( void ); 2915 extern int __kmp_ignore_mppend( void ); 2916 2917 extern int __kmp_enter_single( int gtid, ident_t *id_ref, int push_ws ); 2918 extern void __kmp_exit_single( int gtid ); 2919 2920 extern void __kmp_parallel_deo( int *gtid_ref, int *cid_ref, ident_t *loc_ref ); 2921 extern void __kmp_parallel_dxo( int *gtid_ref, int *cid_ref, ident_t *loc_ref ); 2922 2923 #ifdef USE_LOAD_BALANCE 2924 extern int __kmp_get_load_balance( int ); 2925 #endif 2926 2927 #ifdef BUILD_TV 2928 extern void __kmp_tv_threadprivate_store( kmp_info_t *th, void *global_addr, void *thread_addr ); 2929 #endif 2930 2931 extern int __kmp_get_global_thread_id( void ); 2932 extern int __kmp_get_global_thread_id_reg( void ); 2933 extern void __kmp_exit_thread( int exit_status ); 2934 extern void __kmp_abort( char const * format, ... ); 2935 extern void __kmp_abort_thread( void ); 2936 extern void __kmp_abort_process( void ); 2937 extern void __kmp_warn( char const * format, ... ); 2938 2939 extern void __kmp_set_num_threads( int new_nth, int gtid ); 2940 2941 // Returns current thread (pointer to kmp_info_t). Current thread *must* be registered. 2942 static inline kmp_info_t * __kmp_entry_thread() 2943 { 2944 int gtid = __kmp_entry_gtid(); 2945 2946 return __kmp_threads[gtid]; 2947 } 2948 2949 extern void __kmp_set_max_active_levels( int gtid, int new_max_active_levels ); 2950 extern int __kmp_get_max_active_levels( int gtid ); 2951 extern int __kmp_get_ancestor_thread_num( int gtid, int level ); 2952 extern int __kmp_get_team_size( int gtid, int level ); 2953 extern void __kmp_set_schedule( int gtid, kmp_sched_t new_sched, int chunk ); 2954 extern void __kmp_get_schedule( int gtid, kmp_sched_t * sched, int * chunk ); 2955 2956 extern unsigned short __kmp_get_random( kmp_info_t * thread ); 2957 extern void __kmp_init_random( kmp_info_t * thread ); 2958 2959 extern kmp_r_sched_t __kmp_get_schedule_global( void ); 2960 extern void __kmp_adjust_num_threads( int new_nproc ); 2961 2962 extern void * ___kmp_allocate( size_t size KMP_SRC_LOC_DECL ); 2963 extern void * ___kmp_page_allocate( size_t size KMP_SRC_LOC_DECL ); 2964 extern void ___kmp_free( void * ptr KMP_SRC_LOC_DECL ); 2965 #define __kmp_allocate( size ) ___kmp_allocate( (size) KMP_SRC_LOC_CURR ) 2966 #define __kmp_page_allocate( size ) ___kmp_page_allocate( (size) KMP_SRC_LOC_CURR ) 2967 #define __kmp_free( ptr ) ___kmp_free( (ptr) KMP_SRC_LOC_CURR ) 2968 2969 #if USE_FAST_MEMORY 2970 extern void * ___kmp_fast_allocate( kmp_info_t *this_thr, size_t size KMP_SRC_LOC_DECL ); 2971 extern void ___kmp_fast_free( kmp_info_t *this_thr, void *ptr KMP_SRC_LOC_DECL ); 2972 extern void __kmp_free_fast_memory( kmp_info_t *this_thr ); 2973 extern void __kmp_initialize_fast_memory( kmp_info_t *this_thr ); 2974 #define __kmp_fast_allocate( this_thr, size ) ___kmp_fast_allocate( (this_thr), (size) KMP_SRC_LOC_CURR ) 2975 #define __kmp_fast_free( this_thr, ptr ) ___kmp_fast_free( (this_thr), (ptr) KMP_SRC_LOC_CURR ) 2976 #endif 2977 2978 extern void * ___kmp_thread_malloc( kmp_info_t *th, size_t size KMP_SRC_LOC_DECL ); 2979 extern void * ___kmp_thread_calloc( kmp_info_t *th, size_t nelem, size_t elsize KMP_SRC_LOC_DECL ); 2980 extern void * ___kmp_thread_realloc( kmp_info_t *th, void *ptr, size_t size KMP_SRC_LOC_DECL ); 2981 extern void ___kmp_thread_free( kmp_info_t *th, void *ptr KMP_SRC_LOC_DECL ); 2982 #define __kmp_thread_malloc( th, size ) ___kmp_thread_malloc( (th), (size) KMP_SRC_LOC_CURR ) 2983 #define __kmp_thread_calloc( th, nelem, elsize ) ___kmp_thread_calloc( (th), (nelem), (elsize) KMP_SRC_LOC_CURR ) 2984 #define __kmp_thread_realloc( th, ptr, size ) ___kmp_thread_realloc( (th), (ptr), (size) KMP_SRC_LOC_CURR ) 2985 #define __kmp_thread_free( th, ptr ) ___kmp_thread_free( (th), (ptr) KMP_SRC_LOC_CURR ) 2986 2987 #define KMP_INTERNAL_MALLOC(sz) malloc(sz) 2988 #define KMP_INTERNAL_FREE(p) free(p) 2989 #define KMP_INTERNAL_REALLOC(p,sz) realloc((p),(sz)) 2990 #define KMP_INTERNAL_CALLOC(n,sz) calloc((n),(sz)) 2991 2992 extern void __kmp_push_num_threads( ident_t *loc, int gtid, int num_threads ); 2993 2994 #if OMP_40_ENABLED 2995 extern void __kmp_push_proc_bind( ident_t *loc, int gtid, kmp_proc_bind_t proc_bind ); 2996 extern void __kmp_push_num_teams( ident_t *loc, int gtid, int num_teams, int num_threads ); 2997 #endif 2998 2999 extern void __kmp_yield( int cond ); 3000 3001 extern void __kmpc_dispatch_init_4( ident_t *loc, kmp_int32 gtid, 3002 enum sched_type schedule, kmp_int32 lb, kmp_int32 ub, kmp_int32 st, 3003 kmp_int32 chunk ); 3004 extern void __kmpc_dispatch_init_4u( ident_t *loc, kmp_int32 gtid, 3005 enum sched_type schedule, kmp_uint32 lb, kmp_uint32 ub, kmp_int32 st, 3006 kmp_int32 chunk ); 3007 extern void __kmpc_dispatch_init_8( ident_t *loc, kmp_int32 gtid, 3008 enum sched_type schedule, kmp_int64 lb, kmp_int64 ub, kmp_int64 st, 3009 kmp_int64 chunk ); 3010 extern void __kmpc_dispatch_init_8u( ident_t *loc, kmp_int32 gtid, 3011 enum sched_type schedule, kmp_uint64 lb, kmp_uint64 ub, kmp_int64 st, 3012 kmp_int64 chunk ); 3013 3014 extern int __kmpc_dispatch_next_4( ident_t *loc, kmp_int32 gtid, 3015 kmp_int32 *p_last, kmp_int32 *p_lb, kmp_int32 *p_ub, kmp_int32 *p_st ); 3016 extern int __kmpc_dispatch_next_4u( ident_t *loc, kmp_int32 gtid, 3017 kmp_int32 *p_last, kmp_uint32 *p_lb, kmp_uint32 *p_ub, kmp_int32 *p_st ); 3018 extern int __kmpc_dispatch_next_8( ident_t *loc, kmp_int32 gtid, 3019 kmp_int32 *p_last, kmp_int64 *p_lb, kmp_int64 *p_ub, kmp_int64 *p_st ); 3020 extern int __kmpc_dispatch_next_8u( ident_t *loc, kmp_int32 gtid, 3021 kmp_int32 *p_last, kmp_uint64 *p_lb, kmp_uint64 *p_ub, kmp_int64 *p_st ); 3022 3023 extern void __kmpc_dispatch_fini_4( ident_t *loc, kmp_int32 gtid ); 3024 extern void __kmpc_dispatch_fini_8( ident_t *loc, kmp_int32 gtid ); 3025 extern void __kmpc_dispatch_fini_4u( ident_t *loc, kmp_int32 gtid ); 3026 extern void __kmpc_dispatch_fini_8u( ident_t *loc, kmp_int32 gtid ); 3027 3028 3029 #ifdef KMP_GOMP_COMPAT 3030 3031 extern void __kmp_aux_dispatch_init_4( ident_t *loc, kmp_int32 gtid, 3032 enum sched_type schedule, kmp_int32 lb, kmp_int32 ub, kmp_int32 st, 3033 kmp_int32 chunk, int push_ws ); 3034 extern void __kmp_aux_dispatch_init_4u( ident_t *loc, kmp_int32 gtid, 3035 enum sched_type schedule, kmp_uint32 lb, kmp_uint32 ub, kmp_int32 st, 3036 kmp_int32 chunk, int push_ws ); 3037 extern void __kmp_aux_dispatch_init_8( ident_t *loc, kmp_int32 gtid, 3038 enum sched_type schedule, kmp_int64 lb, kmp_int64 ub, kmp_int64 st, 3039 kmp_int64 chunk, int push_ws ); 3040 extern void __kmp_aux_dispatch_init_8u( ident_t *loc, kmp_int32 gtid, 3041 enum sched_type schedule, kmp_uint64 lb, kmp_uint64 ub, kmp_int64 st, 3042 kmp_int64 chunk, int push_ws ); 3043 extern void __kmp_aux_dispatch_fini_chunk_4( ident_t *loc, kmp_int32 gtid ); 3044 extern void __kmp_aux_dispatch_fini_chunk_8( ident_t *loc, kmp_int32 gtid ); 3045 extern void __kmp_aux_dispatch_fini_chunk_4u( ident_t *loc, kmp_int32 gtid ); 3046 extern void __kmp_aux_dispatch_fini_chunk_8u( ident_t *loc, kmp_int32 gtid ); 3047 3048 #endif /* KMP_GOMP_COMPAT */ 3049 3050 3051 extern kmp_uint32 __kmp_eq_4( kmp_uint32 value, kmp_uint32 checker ); 3052 extern kmp_uint32 __kmp_neq_4( kmp_uint32 value, kmp_uint32 checker ); 3053 extern kmp_uint32 __kmp_lt_4( kmp_uint32 value, kmp_uint32 checker ); 3054 extern kmp_uint32 __kmp_ge_4( kmp_uint32 value, kmp_uint32 checker ); 3055 extern kmp_uint32 __kmp_le_4( kmp_uint32 value, kmp_uint32 checker ); 3056 extern kmp_uint32 __kmp_wait_yield_4( kmp_uint32 volatile * spinner, kmp_uint32 checker, kmp_uint32 (*pred) (kmp_uint32, kmp_uint32), void * obj ); 3057 3058 class kmp_flag_32; 3059 class kmp_flag_64; 3060 class kmp_flag_oncore; 3061 extern void __kmp_wait_32(kmp_info_t *this_thr, kmp_flag_32 *flag, int final_spin 3062 #if USE_ITT_BUILD 3063 , void * itt_sync_obj 3064 #endif 3065 ); 3066 extern void __kmp_release_32(kmp_flag_32 *flag); 3067 extern void __kmp_wait_64(kmp_info_t *this_thr, kmp_flag_64 *flag, int final_spin 3068 #if USE_ITT_BUILD 3069 , void * itt_sync_obj 3070 #endif 3071 ); 3072 extern void __kmp_release_64(kmp_flag_64 *flag); 3073 extern void __kmp_wait_oncore(kmp_info_t *this_thr, kmp_flag_oncore *flag, int final_spin 3074 #if USE_ITT_BUILD 3075 , void * itt_sync_obj 3076 #endif 3077 ); 3078 extern void __kmp_release_oncore(kmp_flag_oncore *flag); 3079 3080 extern void __kmp_infinite_loop( void ); 3081 3082 extern void __kmp_cleanup( void ); 3083 3084 #if KMP_HANDLE_SIGNALS 3085 extern int __kmp_handle_signals; 3086 extern void __kmp_install_signals( int parallel_init ); 3087 extern void __kmp_remove_signals( void ); 3088 #endif 3089 3090 extern void __kmp_clear_system_time( void ); 3091 extern void __kmp_read_system_time( double *delta ); 3092 3093 extern void __kmp_check_stack_overlap( kmp_info_t *thr ); 3094 3095 extern void __kmp_expand_host_name( char *buffer, size_t size ); 3096 extern void __kmp_expand_file_name( char *result, size_t rlen, char *pattern ); 3097 3098 #if KMP_OS_WINDOWS 3099 extern void __kmp_initialize_system_tick( void ); /* Initialize timer tick value */ 3100 #endif 3101 3102 extern void __kmp_runtime_initialize( void ); /* machine specific initialization */ 3103 extern void __kmp_runtime_destroy( void ); 3104 3105 #if KMP_AFFINITY_SUPPORTED 3106 extern char *__kmp_affinity_print_mask(char *buf, int buf_len, kmp_affin_mask_t *mask); 3107 extern void __kmp_affinity_initialize(void); 3108 extern void __kmp_affinity_uninitialize(void); 3109 extern void __kmp_affinity_set_init_mask(int gtid, int isa_root); /* set affinity according to KMP_AFFINITY */ 3110 #if OMP_40_ENABLED 3111 extern void __kmp_affinity_set_place(int gtid); 3112 #endif 3113 extern void __kmp_affinity_determine_capable( const char *env_var ); 3114 extern int __kmp_aux_set_affinity(void **mask); 3115 extern int __kmp_aux_get_affinity(void **mask); 3116 extern int __kmp_aux_set_affinity_mask_proc(int proc, void **mask); 3117 extern int __kmp_aux_unset_affinity_mask_proc(int proc, void **mask); 3118 extern int __kmp_aux_get_affinity_mask_proc(int proc, void **mask); 3119 extern void __kmp_balanced_affinity( int tid, int team_size ); 3120 #endif /* KMP_AFFINITY_SUPPORTED */ 3121 3122 extern void __kmp_cleanup_hierarchy(); 3123 extern void __kmp_get_hierarchy(kmp_uint32 nproc, kmp_bstate_t *thr_bar); 3124 3125 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 3126 3127 extern int __kmp_futex_determine_capable( void ); 3128 3129 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 3130 3131 extern void __kmp_gtid_set_specific( int gtid ); 3132 extern int __kmp_gtid_get_specific( void ); 3133 3134 extern double __kmp_read_cpu_time( void ); 3135 3136 extern int __kmp_read_system_info( struct kmp_sys_info *info ); 3137 3138 extern void __kmp_create_monitor( kmp_info_t *th ); 3139 3140 extern void *__kmp_launch_thread( kmp_info_t *thr ); 3141 3142 extern void __kmp_create_worker( int gtid, kmp_info_t *th, size_t stack_size ); 3143 3144 #if KMP_OS_WINDOWS 3145 extern int __kmp_still_running(kmp_info_t *th); 3146 extern int __kmp_is_thread_alive( kmp_info_t * th, DWORD *exit_val ); 3147 extern void __kmp_free_handle( kmp_thread_t tHandle ); 3148 #endif 3149 3150 extern void __kmp_reap_monitor( kmp_info_t *th ); 3151 extern void __kmp_reap_worker( kmp_info_t *th ); 3152 extern void __kmp_terminate_thread( int gtid ); 3153 3154 extern void __kmp_suspend_32( int th_gtid, kmp_flag_32 *flag ); 3155 extern void __kmp_suspend_64( int th_gtid, kmp_flag_64 *flag ); 3156 extern void __kmp_suspend_oncore( int th_gtid, kmp_flag_oncore *flag ); 3157 extern void __kmp_resume_32( int target_gtid, kmp_flag_32 *flag ); 3158 extern void __kmp_resume_64( int target_gtid, kmp_flag_64 *flag ); 3159 extern void __kmp_resume_oncore( int target_gtid, kmp_flag_oncore *flag ); 3160 3161 extern void __kmp_elapsed( double * ); 3162 extern void __kmp_elapsed_tick( double * ); 3163 3164 extern void __kmp_enable( int old_state ); 3165 extern void __kmp_disable( int *old_state ); 3166 3167 extern void __kmp_thread_sleep( int millis ); 3168 3169 extern void __kmp_common_initialize( void ); 3170 extern void __kmp_common_destroy( void ); 3171 extern void __kmp_common_destroy_gtid( int gtid ); 3172 3173 #if KMP_OS_UNIX 3174 extern void __kmp_register_atfork( void ); 3175 #endif 3176 extern void __kmp_suspend_initialize( void ); 3177 extern void __kmp_suspend_uninitialize_thread( kmp_info_t *th ); 3178 3179 extern kmp_info_t * __kmp_allocate_thread( kmp_root_t *root, 3180 kmp_team_t *team, int tid); 3181 #if OMP_40_ENABLED 3182 extern kmp_team_t * __kmp_allocate_team( kmp_root_t *root, int new_nproc, int max_nproc, 3183 #if OMPT_SUPPORT 3184 ompt_parallel_id_t ompt_parallel_id, 3185 #endif 3186 kmp_proc_bind_t proc_bind, 3187 kmp_internal_control_t *new_icvs, 3188 int argc USE_NESTED_HOT_ARG(kmp_info_t *thr) ); 3189 #else 3190 extern kmp_team_t * __kmp_allocate_team( kmp_root_t *root, int new_nproc, int max_nproc, 3191 #if OMPT_SUPPORT 3192 ompt_parallel_id_t ompt_parallel_id, 3193 #endif 3194 kmp_internal_control_t *new_icvs, 3195 int argc USE_NESTED_HOT_ARG(kmp_info_t *thr) ); 3196 #endif // OMP_40_ENABLED 3197 extern void __kmp_free_thread( kmp_info_t * ); 3198 extern void __kmp_free_team( kmp_root_t *, kmp_team_t * USE_NESTED_HOT_ARG(kmp_info_t *) ); 3199 extern kmp_team_t * __kmp_reap_team( kmp_team_t * ); 3200 3201 /* ------------------------------------------------------------------------ */ 3202 3203 extern void __kmp_initialize_bget( kmp_info_t *th ); 3204 extern void __kmp_finalize_bget( kmp_info_t *th ); 3205 3206 KMP_EXPORT void *kmpc_malloc( size_t size ); 3207 KMP_EXPORT void *kmpc_calloc( size_t nelem, size_t elsize ); 3208 KMP_EXPORT void *kmpc_realloc( void *ptr, size_t size ); 3209 KMP_EXPORT void kmpc_free( void *ptr ); 3210 3211 /* ------------------------------------------------------------------------ */ 3212 /* declarations for internal use */ 3213 3214 extern int __kmp_barrier( enum barrier_type bt, int gtid, int is_split, 3215 size_t reduce_size, void *reduce_data, void (*reduce)(void *, void *) ); 3216 extern void __kmp_end_split_barrier ( enum barrier_type bt, int gtid ); 3217 3218 /*! 3219 * Tell the fork call which compiler generated the fork call, and therefore how to deal with the call. 3220 */ 3221 enum fork_context_e 3222 { 3223 fork_context_gnu, /**< Called from GNU generated code, so must not invoke the microtask internally. */ 3224 fork_context_intel, /**< Called from Intel generated code. */ 3225 fork_context_last 3226 }; 3227 extern int __kmp_fork_call( ident_t *loc, int gtid, enum fork_context_e fork_context, 3228 kmp_int32 argc, 3229 #if OMPT_SUPPORT 3230 void *unwrapped_task, 3231 #endif 3232 microtask_t microtask, launch_t invoker, 3233 /* TODO: revert workaround for Intel(R) 64 tracker #96 */ 3234 #if (KMP_ARCH_ARM || KMP_ARCH_X86_64 || KMP_ARCH_AARCH64) && KMP_OS_LINUX 3235 va_list *ap 3236 #else 3237 va_list ap 3238 #endif 3239 ); 3240 3241 extern void __kmp_join_call( ident_t *loc, int gtid 3242 #if OMPT_SUPPORT 3243 , enum fork_context_e fork_context 3244 #endif 3245 #if OMP_40_ENABLED 3246 , int exit_teams = 0 3247 #endif 3248 ); 3249 3250 extern void __kmp_serialized_parallel(ident_t *id, kmp_int32 gtid); 3251 extern void __kmp_internal_fork( ident_t *id, int gtid, kmp_team_t *team ); 3252 extern void __kmp_internal_join( ident_t *id, int gtid, kmp_team_t *team ); 3253 extern int __kmp_invoke_task_func( int gtid ); 3254 extern void __kmp_run_before_invoked_task( int gtid, int tid, kmp_info_t *this_thr, kmp_team_t *team ); 3255 extern void __kmp_run_after_invoked_task( int gtid, int tid, kmp_info_t *this_thr, kmp_team_t *team ); 3256 3257 // should never have been exported 3258 KMP_EXPORT int __kmpc_invoke_task_func( int gtid ); 3259 #if OMP_40_ENABLED 3260 extern int __kmp_invoke_teams_master( int gtid ); 3261 extern void __kmp_teams_master( int gtid ); 3262 #endif 3263 extern void __kmp_save_internal_controls( kmp_info_t * thread ); 3264 extern void __kmp_user_set_library (enum library_type arg); 3265 extern void __kmp_aux_set_library (enum library_type arg); 3266 extern void __kmp_aux_set_stacksize( size_t arg); 3267 extern void __kmp_aux_set_blocktime (int arg, kmp_info_t *thread, int tid); 3268 extern void __kmp_aux_set_defaults( char const * str, int len ); 3269 3270 /* Functions below put here to call them from __kmp_aux_env_initialize() in kmp_settings.c */ 3271 void kmpc_set_blocktime (int arg); 3272 void ompc_set_nested( int flag ); 3273 void ompc_set_dynamic( int flag ); 3274 void ompc_set_num_threads( int arg ); 3275 3276 extern void __kmp_push_current_task_to_thread( kmp_info_t *this_thr, 3277 kmp_team_t *team, int tid ); 3278 extern void __kmp_pop_current_task_from_thread( kmp_info_t *this_thr ); 3279 extern kmp_task_t* __kmp_task_alloc( ident_t *loc_ref, kmp_int32 gtid, 3280 kmp_tasking_flags_t *flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, 3281 kmp_routine_entry_t task_entry ); 3282 extern void __kmp_init_implicit_task( ident_t *loc_ref, kmp_info_t *this_thr, 3283 kmp_team_t *team, int tid, int set_curr_task ); 3284 3285 int __kmp_execute_tasks_32(kmp_info_t *thread, kmp_int32 gtid, kmp_flag_32 *flag, int final_spin, 3286 int *thread_finished, 3287 #if USE_ITT_BUILD 3288 void * itt_sync_obj, 3289 #endif /* USE_ITT_BUILD */ 3290 kmp_int32 is_constrained); 3291 int __kmp_execute_tasks_64(kmp_info_t *thread, kmp_int32 gtid, kmp_flag_64 *flag, int final_spin, 3292 int *thread_finished, 3293 #if USE_ITT_BUILD 3294 void * itt_sync_obj, 3295 #endif /* USE_ITT_BUILD */ 3296 kmp_int32 is_constrained); 3297 int __kmp_execute_tasks_oncore(kmp_info_t *thread, kmp_int32 gtid, kmp_flag_oncore *flag, int final_spin, 3298 int *thread_finished, 3299 #if USE_ITT_BUILD 3300 void * itt_sync_obj, 3301 #endif /* USE_ITT_BUILD */ 3302 kmp_int32 is_constrained); 3303 3304 extern void __kmp_free_task_team( kmp_info_t *thread, kmp_task_team_t *task_team ); 3305 extern void __kmp_reap_task_teams( void ); 3306 extern void __kmp_wait_to_unref_task_teams( void ); 3307 extern void __kmp_task_team_setup ( kmp_info_t *this_thr, kmp_team_t *team, int always ); 3308 extern void __kmp_task_team_sync ( kmp_info_t *this_thr, kmp_team_t *team ); 3309 extern void __kmp_task_team_wait ( kmp_info_t *this_thr, kmp_team_t *team 3310 #if USE_ITT_BUILD 3311 , void * itt_sync_obj 3312 #endif /* USE_ITT_BUILD */ 3313 , int wait=1 3314 ); 3315 extern void __kmp_tasking_barrier( kmp_team_t *team, kmp_info_t *thread, int gtid ); 3316 3317 extern int __kmp_is_address_mapped( void *addr ); 3318 extern kmp_uint64 __kmp_hardware_timestamp(void); 3319 3320 #if KMP_OS_UNIX 3321 extern int __kmp_read_from_file( char const *path, char const *format, ... ); 3322 #endif 3323 3324 /* ------------------------------------------------------------------------ */ 3325 // 3326 // Assembly routines that have no compiler intrinsic replacement 3327 // 3328 3329 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 3330 3331 extern void __kmp_query_cpuid( kmp_cpuinfo_t *p ); 3332 3333 #define __kmp_load_mxcsr(p) _mm_setcsr(*(p)) 3334 static inline void __kmp_store_mxcsr( kmp_uint32 *p ) { *p = _mm_getcsr(); } 3335 3336 extern void __kmp_load_x87_fpu_control_word( kmp_int16 *p ); 3337 extern void __kmp_store_x87_fpu_control_word( kmp_int16 *p ); 3338 extern void __kmp_clear_x87_fpu_status_word(); 3339 # define KMP_X86_MXCSR_MASK 0xffffffc0 /* ignore status flags (6 lsb) */ 3340 3341 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 3342 3343 extern int __kmp_invoke_microtask( microtask_t pkfn, int gtid, int npr, int argc, void *argv[] 3344 #if OMPT_SUPPORT 3345 , void **exit_frame_ptr 3346 #endif 3347 ); 3348 3349 3350 /* ------------------------------------------------------------------------ */ 3351 3352 KMP_EXPORT void __kmpc_begin ( ident_t *, kmp_int32 flags ); 3353 KMP_EXPORT void __kmpc_end ( ident_t * ); 3354 3355 KMP_EXPORT void __kmpc_threadprivate_register_vec ( ident_t *, void * data, kmpc_ctor_vec ctor, 3356 kmpc_cctor_vec cctor, kmpc_dtor_vec dtor, size_t vector_length ); 3357 KMP_EXPORT void __kmpc_threadprivate_register ( ident_t *, void * data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor ); 3358 KMP_EXPORT void * __kmpc_threadprivate ( ident_t *, kmp_int32 global_tid, void * data, size_t size ); 3359 3360 KMP_EXPORT kmp_int32 __kmpc_global_thread_num ( ident_t * ); 3361 KMP_EXPORT kmp_int32 __kmpc_global_num_threads ( ident_t * ); 3362 KMP_EXPORT kmp_int32 __kmpc_bound_thread_num ( ident_t * ); 3363 KMP_EXPORT kmp_int32 __kmpc_bound_num_threads ( ident_t * ); 3364 3365 KMP_EXPORT kmp_int32 __kmpc_ok_to_fork ( ident_t * ); 3366 KMP_EXPORT void __kmpc_fork_call ( ident_t *, kmp_int32 nargs, kmpc_micro microtask, ... ); 3367 3368 KMP_EXPORT void __kmpc_serialized_parallel ( ident_t *, kmp_int32 global_tid ); 3369 KMP_EXPORT void __kmpc_end_serialized_parallel ( ident_t *, kmp_int32 global_tid ); 3370 3371 KMP_EXPORT void __kmpc_flush ( ident_t *); 3372 KMP_EXPORT void __kmpc_barrier ( ident_t *, kmp_int32 global_tid ); 3373 KMP_EXPORT kmp_int32 __kmpc_master ( ident_t *, kmp_int32 global_tid ); 3374 KMP_EXPORT void __kmpc_end_master ( ident_t *, kmp_int32 global_tid ); 3375 KMP_EXPORT void __kmpc_ordered ( ident_t *, kmp_int32 global_tid ); 3376 KMP_EXPORT void __kmpc_end_ordered ( ident_t *, kmp_int32 global_tid ); 3377 KMP_EXPORT void __kmpc_critical ( ident_t *, kmp_int32 global_tid, kmp_critical_name * ); 3378 KMP_EXPORT void __kmpc_end_critical ( ident_t *, kmp_int32 global_tid, kmp_critical_name * ); 3379 3380 #if OMP_41_ENABLED 3381 KMP_EXPORT void __kmpc_critical_with_hint ( ident_t *, kmp_int32 global_tid, kmp_critical_name *, uintptr_t hint ); 3382 #endif 3383 3384 KMP_EXPORT kmp_int32 __kmpc_barrier_master ( ident_t *, kmp_int32 global_tid ); 3385 KMP_EXPORT void __kmpc_end_barrier_master ( ident_t *, kmp_int32 global_tid ); 3386 3387 KMP_EXPORT kmp_int32 __kmpc_barrier_master_nowait ( ident_t *, kmp_int32 global_tid ); 3388 3389 KMP_EXPORT kmp_int32 __kmpc_single ( ident_t *, kmp_int32 global_tid ); 3390 KMP_EXPORT void __kmpc_end_single ( ident_t *, kmp_int32 global_tid ); 3391 3392 KMP_EXPORT void KMPC_FOR_STATIC_INIT ( ident_t *loc, kmp_int32 global_tid, kmp_int32 schedtype, kmp_int32 *plastiter, 3393 kmp_int *plower, kmp_int *pupper, kmp_int *pstride, kmp_int incr, kmp_int chunk ); 3394 3395 KMP_EXPORT void __kmpc_for_static_fini ( ident_t *loc, kmp_int32 global_tid ); 3396 3397 KMP_EXPORT void __kmpc_copyprivate( ident_t *loc, kmp_int32 global_tid, size_t cpy_size, void *cpy_data, void(*cpy_func)(void*,void*), kmp_int32 didit ); 3398 3399 extern void KMPC_SET_NUM_THREADS ( int arg ); 3400 extern void KMPC_SET_DYNAMIC ( int flag ); 3401 extern void KMPC_SET_NESTED ( int flag ); 3402 3403 /* --------------------------------------------------------------------------- */ 3404 3405 /* 3406 * Taskq interface routines 3407 */ 3408 3409 KMP_EXPORT kmpc_thunk_t * __kmpc_taskq (ident_t *loc, kmp_int32 global_tid, kmpc_task_t taskq_task, size_t sizeof_thunk, 3410 size_t sizeof_shareds, kmp_int32 flags, kmpc_shared_vars_t **shareds); 3411 KMP_EXPORT void __kmpc_end_taskq (ident_t *loc, kmp_int32 global_tid, kmpc_thunk_t *thunk); 3412 KMP_EXPORT kmp_int32 __kmpc_task (ident_t *loc, kmp_int32 global_tid, kmpc_thunk_t *thunk); 3413 KMP_EXPORT void __kmpc_taskq_task (ident_t *loc, kmp_int32 global_tid, kmpc_thunk_t *thunk, kmp_int32 status); 3414 KMP_EXPORT void __kmpc_end_taskq_task (ident_t *loc, kmp_int32 global_tid, kmpc_thunk_t *thunk); 3415 KMP_EXPORT kmpc_thunk_t * __kmpc_task_buffer (ident_t *loc, kmp_int32 global_tid, kmpc_thunk_t *taskq_thunk, kmpc_task_t task); 3416 3417 /* ------------------------------------------------------------------------ */ 3418 3419 /* 3420 * OMP 3.0 tasking interface routines 3421 */ 3422 3423 KMP_EXPORT kmp_int32 3424 __kmpc_omp_task( ident_t *loc_ref, kmp_int32 gtid, kmp_task_t * new_task ); 3425 KMP_EXPORT kmp_task_t* 3426 __kmpc_omp_task_alloc( ident_t *loc_ref, kmp_int32 gtid, kmp_int32 flags, 3427 size_t sizeof_kmp_task_t, size_t sizeof_shareds, 3428 kmp_routine_entry_t task_entry ); 3429 KMP_EXPORT void 3430 __kmpc_omp_task_begin_if0( ident_t *loc_ref, kmp_int32 gtid, kmp_task_t * task ); 3431 KMP_EXPORT void 3432 __kmpc_omp_task_complete_if0( ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task ); 3433 KMP_EXPORT kmp_int32 3434 __kmpc_omp_task_parts( ident_t *loc_ref, kmp_int32 gtid, kmp_task_t * new_task ); 3435 KMP_EXPORT kmp_int32 3436 __kmpc_omp_taskwait( ident_t *loc_ref, kmp_int32 gtid ); 3437 3438 KMP_EXPORT kmp_int32 3439 __kmpc_omp_taskyield( ident_t *loc_ref, kmp_int32 gtid, int end_part ); 3440 3441 #if TASK_UNUSED 3442 void __kmpc_omp_task_begin( ident_t *loc_ref, kmp_int32 gtid, kmp_task_t * task ); 3443 void __kmpc_omp_task_complete( ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task ); 3444 #endif // TASK_UNUSED 3445 3446 /* ------------------------------------------------------------------------ */ 3447 3448 #if OMP_40_ENABLED 3449 3450 KMP_EXPORT void __kmpc_taskgroup( ident_t * loc, int gtid ); 3451 KMP_EXPORT void __kmpc_end_taskgroup( ident_t * loc, int gtid ); 3452 3453 KMP_EXPORT kmp_int32 __kmpc_omp_task_with_deps ( ident_t *loc_ref, kmp_int32 gtid, kmp_task_t * new_task, 3454 kmp_int32 ndeps, kmp_depend_info_t *dep_list, 3455 kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list ); 3456 KMP_EXPORT void __kmpc_omp_wait_deps ( ident_t *loc_ref, kmp_int32 gtid, kmp_int32 ndeps, kmp_depend_info_t *dep_list, 3457 kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list ); 3458 extern void __kmp_release_deps ( kmp_int32 gtid, kmp_taskdata_t *task ); 3459 3460 extern kmp_int32 __kmp_omp_task( kmp_int32 gtid, kmp_task_t * new_task, bool serialize_immediate ); 3461 3462 KMP_EXPORT kmp_int32 __kmpc_cancel(ident_t* loc_ref, kmp_int32 gtid, kmp_int32 cncl_kind); 3463 KMP_EXPORT kmp_int32 __kmpc_cancellationpoint(ident_t* loc_ref, kmp_int32 gtid, kmp_int32 cncl_kind); 3464 KMP_EXPORT kmp_int32 __kmpc_cancel_barrier(ident_t* loc_ref, kmp_int32 gtid); 3465 KMP_EXPORT int __kmp_get_cancellation_status(int cancel_kind); 3466 3467 #if OMP_41_ENABLED 3468 3469 KMP_EXPORT void __kmpc_proxy_task_completed( kmp_int32 gtid, kmp_task_t *ptask ); 3470 KMP_EXPORT void __kmpc_proxy_task_completed_ooo ( kmp_task_t *ptask ); 3471 3472 #endif 3473 3474 #endif 3475 3476 3477 /* 3478 * Lock interface routines (fast versions with gtid passed in) 3479 */ 3480 KMP_EXPORT void __kmpc_init_lock( ident_t *loc, kmp_int32 gtid, void **user_lock ); 3481 KMP_EXPORT void __kmpc_init_nest_lock( ident_t *loc, kmp_int32 gtid, void **user_lock ); 3482 KMP_EXPORT void __kmpc_destroy_lock( ident_t *loc, kmp_int32 gtid, void **user_lock ); 3483 KMP_EXPORT void __kmpc_destroy_nest_lock( ident_t *loc, kmp_int32 gtid, void **user_lock ); 3484 KMP_EXPORT void __kmpc_set_lock( ident_t *loc, kmp_int32 gtid, void **user_lock ); 3485 KMP_EXPORT void __kmpc_set_nest_lock( ident_t *loc, kmp_int32 gtid, void **user_lock ); 3486 KMP_EXPORT void __kmpc_unset_lock( ident_t *loc, kmp_int32 gtid, void **user_lock ); 3487 KMP_EXPORT void __kmpc_unset_nest_lock( ident_t *loc, kmp_int32 gtid, void **user_lock ); 3488 KMP_EXPORT int __kmpc_test_lock( ident_t *loc, kmp_int32 gtid, void **user_lock ); 3489 KMP_EXPORT int __kmpc_test_nest_lock( ident_t *loc, kmp_int32 gtid, void **user_lock ); 3490 3491 #if OMP_41_ENABLED 3492 KMP_EXPORT void __kmpc_init_lock_with_hint( ident_t *loc, kmp_int32 gtid, void **user_lock, uintptr_t hint ); 3493 KMP_EXPORT void __kmpc_init_nest_lock_with_hint( ident_t *loc, kmp_int32 gtid, void **user_lock, uintptr_t hint ); 3494 #endif 3495 3496 /* ------------------------------------------------------------------------ */ 3497 3498 /* 3499 * Interface to fast scalable reduce methods routines 3500 */ 3501 3502 KMP_EXPORT kmp_int32 __kmpc_reduce_nowait( ident_t *loc, kmp_int32 global_tid, 3503 kmp_int32 num_vars, size_t reduce_size, 3504 void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data), 3505 kmp_critical_name *lck ); 3506 KMP_EXPORT void __kmpc_end_reduce_nowait( ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck ); 3507 KMP_EXPORT kmp_int32 __kmpc_reduce( ident_t *loc, kmp_int32 global_tid, 3508 kmp_int32 num_vars, size_t reduce_size, 3509 void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data), 3510 kmp_critical_name *lck ); 3511 KMP_EXPORT void __kmpc_end_reduce( ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck ); 3512 3513 /* 3514 * internal fast reduction routines 3515 */ 3516 3517 extern PACKED_REDUCTION_METHOD_T 3518 __kmp_determine_reduction_method( ident_t *loc, kmp_int32 global_tid, 3519 kmp_int32 num_vars, size_t reduce_size, 3520 void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data), 3521 kmp_critical_name *lck ); 3522 3523 // this function is for testing set/get/determine reduce method 3524 KMP_EXPORT kmp_int32 __kmp_get_reduce_method( void ); 3525 3526 KMP_EXPORT kmp_uint64 __kmpc_get_taskid(); 3527 KMP_EXPORT kmp_uint64 __kmpc_get_parent_taskid(); 3528 3529 // this function exported for testing of KMP_PLACE_THREADS functionality 3530 KMP_EXPORT void __kmpc_place_threads(int,int,int,int,int); 3531 3532 /* ------------------------------------------------------------------------ */ 3533 /* ------------------------------------------------------------------------ */ 3534 3535 // C++ port 3536 // missing 'extern "C"' declarations 3537 3538 KMP_EXPORT kmp_int32 __kmpc_in_parallel( ident_t *loc ); 3539 KMP_EXPORT void __kmpc_pop_num_threads( ident_t *loc, kmp_int32 global_tid ); 3540 KMP_EXPORT void __kmpc_push_num_threads( ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads ); 3541 3542 #if OMP_40_ENABLED 3543 KMP_EXPORT void __kmpc_push_proc_bind( ident_t *loc, kmp_int32 global_tid, int proc_bind ); 3544 KMP_EXPORT void __kmpc_push_num_teams( ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams, kmp_int32 num_threads ); 3545 KMP_EXPORT void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...); 3546 3547 #endif 3548 3549 KMP_EXPORT void* 3550 __kmpc_threadprivate_cached( ident_t * loc, kmp_int32 global_tid, 3551 void * data, size_t size, void *** cache ); 3552 3553 // Symbols for MS mutual detection. 3554 extern int _You_must_link_with_exactly_one_OpenMP_library; 3555 extern int _You_must_link_with_Intel_OpenMP_library; 3556 #if KMP_OS_WINDOWS && ( KMP_VERSION_MAJOR > 4 ) 3557 extern int _You_must_link_with_Microsoft_OpenMP_library; 3558 #endif 3559 3560 3561 // The routines below are not exported. 3562 // Consider making them 'static' in corresponding source files. 3563 void 3564 kmp_threadprivate_insert_private_data( int gtid, void *pc_addr, void *data_addr, size_t pc_size ); 3565 struct private_common * 3566 kmp_threadprivate_insert( int gtid, void *pc_addr, void *data_addr, size_t pc_size ); 3567 3568 // 3569 // ompc_, kmpc_ entries moved from omp.h. 3570 // 3571 #if KMP_OS_WINDOWS 3572 # define KMPC_CONVENTION __cdecl 3573 #else 3574 # define KMPC_CONVENTION 3575 #endif 3576 3577 #ifndef __OMP_H 3578 typedef enum omp_sched_t { 3579 omp_sched_static = 1, 3580 omp_sched_dynamic = 2, 3581 omp_sched_guided = 3, 3582 omp_sched_auto = 4 3583 } omp_sched_t; 3584 typedef void * kmp_affinity_mask_t; 3585 #endif 3586 3587 KMP_EXPORT void KMPC_CONVENTION ompc_set_max_active_levels(int); 3588 KMP_EXPORT void KMPC_CONVENTION ompc_set_schedule(omp_sched_t, int); 3589 KMP_EXPORT int KMPC_CONVENTION ompc_get_ancestor_thread_num(int); 3590 KMP_EXPORT int KMPC_CONVENTION ompc_get_team_size(int); 3591 KMP_EXPORT int KMPC_CONVENTION kmpc_set_affinity_mask_proc(int, kmp_affinity_mask_t *); 3592 KMP_EXPORT int KMPC_CONVENTION kmpc_unset_affinity_mask_proc(int, kmp_affinity_mask_t *); 3593 KMP_EXPORT int KMPC_CONVENTION kmpc_get_affinity_mask_proc(int, kmp_affinity_mask_t *); 3594 3595 KMP_EXPORT void KMPC_CONVENTION kmpc_set_stacksize(int); 3596 KMP_EXPORT void KMPC_CONVENTION kmpc_set_stacksize_s(size_t); 3597 KMP_EXPORT void KMPC_CONVENTION kmpc_set_library(int); 3598 KMP_EXPORT void KMPC_CONVENTION kmpc_set_defaults(char const *); 3599 3600 #ifdef __cplusplus 3601 } 3602 #endif 3603 3604 #endif /* KMP_H */ 3605 3606