1 /* 2 * kmp_gsupport.cpp 3 */ 4 5 //===----------------------------------------------------------------------===// 6 // 7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 8 // See https://llvm.org/LICENSE.txt for license information. 9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "kmp.h" 14 #include "kmp_atomic.h" 15 16 #if OMPT_SUPPORT 17 #include "ompt-specific.h" 18 #endif 19 20 enum { 21 KMP_GOMP_TASK_UNTIED_FLAG = 1, 22 KMP_GOMP_TASK_FINAL_FLAG = 2, 23 KMP_GOMP_TASK_DEPENDS_FLAG = 8 24 }; 25 26 // This class helps convert gomp dependency info into 27 // kmp_depend_info_t structures 28 class kmp_gomp_depends_info_t { 29 void **depend; 30 kmp_int32 num_deps; 31 size_t num_out, num_mutexinout, num_in; 32 size_t offset; 33 34 public: 35 kmp_gomp_depends_info_t(void **depend) : depend(depend) { 36 size_t ndeps = (kmp_intptr_t)depend[0]; 37 size_t num_doable; 38 // GOMP taskdep structure: 39 // if depend[0] != 0: 40 // depend = [ ndeps | nout | &out | ... | &out | &in | ... | &in ] 41 // 42 // if depend[0] == 0: 43 // depend = [ 0 | ndeps | nout | nmtx | nin | &out | ... | &out | &mtx | 44 // ... | &mtx | &in | ... | &in | &depobj | ... | &depobj ] 45 if (ndeps) { 46 num_out = (kmp_intptr_t)depend[1]; 47 num_in = ndeps - num_out; 48 num_mutexinout = 0; 49 num_doable = ndeps; 50 offset = 2; 51 } else { 52 ndeps = (kmp_intptr_t)depend[1]; 53 num_out = (kmp_intptr_t)depend[2]; 54 num_mutexinout = (kmp_intptr_t)depend[3]; 55 num_in = (kmp_intptr_t)depend[4]; 56 num_doable = num_out + num_mutexinout + num_in; 57 offset = 5; 58 } 59 // TODO: Support gomp depobj 60 if (ndeps != num_doable) { 61 KMP_FATAL(GompFeatureNotSupported, "depobj"); 62 } 63 num_deps = static_cast<kmp_int32>(ndeps); 64 } 65 kmp_int32 get_num_deps() const { return num_deps; } 66 kmp_depend_info_t get_kmp_depend(size_t index) const { 67 kmp_depend_info_t retval; 68 memset(&retval, '\0', sizeof(retval)); 69 KMP_ASSERT(index < (size_t)num_deps); 70 retval.base_addr = (kmp_intptr_t)depend[offset + index]; 71 retval.len = 0; 72 // Because inout and out are logically equivalent, 73 // use inout and in dependency flags. GOMP does not provide a 74 // way to distinguish if user specified out vs. inout. 75 if (index < num_out) { 76 retval.flags.in = 1; 77 retval.flags.out = 1; 78 } else if (index >= num_out && index < (num_out + num_mutexinout)) { 79 retval.flags.mtx = 1; 80 } else { 81 retval.flags.in = 1; 82 } 83 return retval; 84 } 85 }; 86 87 #ifdef __cplusplus 88 extern "C" { 89 #endif // __cplusplus 90 91 #define MKLOC(loc, routine) \ 92 static ident_t loc = {0, KMP_IDENT_KMPC, 0, 0, ";unknown;unknown;0;0;;"}; 93 94 #include "kmp_ftn_os.h" 95 96 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_BARRIER)(void) { 97 int gtid = __kmp_entry_gtid(); 98 MKLOC(loc, "GOMP_barrier"); 99 KA_TRACE(20, ("GOMP_barrier: T#%d\n", gtid)); 100 #if OMPT_SUPPORT && OMPT_OPTIONAL 101 ompt_frame_t *ompt_frame; 102 if (ompt_enabled.enabled) { 103 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 104 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 105 } 106 OMPT_STORE_RETURN_ADDRESS(gtid); 107 #endif 108 __kmpc_barrier(&loc, gtid); 109 #if OMPT_SUPPORT && OMPT_OPTIONAL 110 if (ompt_enabled.enabled) { 111 ompt_frame->enter_frame = ompt_data_none; 112 } 113 #endif 114 } 115 116 // Mutual exclusion 117 118 // The symbol that icc/ifort generates for unnamed for unnamed critical sections 119 // - .gomp_critical_user_ - is defined using .comm in any objects reference it. 120 // We can't reference it directly here in C code, as the symbol contains a ".". 121 // 122 // The RTL contains an assembly language definition of .gomp_critical_user_ 123 // with another symbol __kmp_unnamed_critical_addr initialized with it's 124 // address. 125 extern kmp_critical_name *__kmp_unnamed_critical_addr; 126 127 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CRITICAL_START)(void) { 128 int gtid = __kmp_entry_gtid(); 129 MKLOC(loc, "GOMP_critical_start"); 130 KA_TRACE(20, ("GOMP_critical_start: T#%d\n", gtid)); 131 #if OMPT_SUPPORT && OMPT_OPTIONAL 132 OMPT_STORE_RETURN_ADDRESS(gtid); 133 #endif 134 __kmpc_critical(&loc, gtid, __kmp_unnamed_critical_addr); 135 } 136 137 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CRITICAL_END)(void) { 138 int gtid = __kmp_get_gtid(); 139 MKLOC(loc, "GOMP_critical_end"); 140 KA_TRACE(20, ("GOMP_critical_end: T#%d\n", gtid)); 141 #if OMPT_SUPPORT && OMPT_OPTIONAL 142 OMPT_STORE_RETURN_ADDRESS(gtid); 143 #endif 144 __kmpc_end_critical(&loc, gtid, __kmp_unnamed_critical_addr); 145 } 146 147 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CRITICAL_NAME_START)(void **pptr) { 148 int gtid = __kmp_entry_gtid(); 149 MKLOC(loc, "GOMP_critical_name_start"); 150 KA_TRACE(20, ("GOMP_critical_name_start: T#%d\n", gtid)); 151 __kmpc_critical(&loc, gtid, (kmp_critical_name *)pptr); 152 } 153 154 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CRITICAL_NAME_END)(void **pptr) { 155 int gtid = __kmp_get_gtid(); 156 MKLOC(loc, "GOMP_critical_name_end"); 157 KA_TRACE(20, ("GOMP_critical_name_end: T#%d\n", gtid)); 158 __kmpc_end_critical(&loc, gtid, (kmp_critical_name *)pptr); 159 } 160 161 // The Gnu codegen tries to use locked operations to perform atomic updates 162 // inline. If it can't, then it calls GOMP_atomic_start() before performing 163 // the update and GOMP_atomic_end() afterward, regardless of the data type. 164 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ATOMIC_START)(void) { 165 int gtid = __kmp_entry_gtid(); 166 KA_TRACE(20, ("GOMP_atomic_start: T#%d\n", gtid)); 167 168 #if OMPT_SUPPORT 169 __ompt_thread_assign_wait_id(0); 170 #endif 171 172 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid); 173 } 174 175 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ATOMIC_END)(void) { 176 int gtid = __kmp_get_gtid(); 177 KA_TRACE(20, ("GOMP_atomic_end: T#%d\n", gtid)); 178 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid); 179 } 180 181 int KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SINGLE_START)(void) { 182 int gtid = __kmp_entry_gtid(); 183 MKLOC(loc, "GOMP_single_start"); 184 KA_TRACE(20, ("GOMP_single_start: T#%d\n", gtid)); 185 186 if (!TCR_4(__kmp_init_parallel)) 187 __kmp_parallel_initialize(); 188 __kmp_resume_if_soft_paused(); 189 190 // 3rd parameter == FALSE prevents kmp_enter_single from pushing a 191 // workshare when USE_CHECKS is defined. We need to avoid the push, 192 // as there is no corresponding GOMP_single_end() call. 193 kmp_int32 rc = __kmp_enter_single(gtid, &loc, FALSE); 194 195 #if OMPT_SUPPORT && OMPT_OPTIONAL 196 kmp_info_t *this_thr = __kmp_threads[gtid]; 197 kmp_team_t *team = this_thr->th.th_team; 198 int tid = __kmp_tid_from_gtid(gtid); 199 200 if (ompt_enabled.enabled) { 201 if (rc) { 202 if (ompt_enabled.ompt_callback_work) { 203 ompt_callbacks.ompt_callback(ompt_callback_work)( 204 ompt_work_single_executor, ompt_scope_begin, 205 &(team->t.ompt_team_info.parallel_data), 206 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 207 1, OMPT_GET_RETURN_ADDRESS(0)); 208 } 209 } else { 210 if (ompt_enabled.ompt_callback_work) { 211 ompt_callbacks.ompt_callback(ompt_callback_work)( 212 ompt_work_single_other, ompt_scope_begin, 213 &(team->t.ompt_team_info.parallel_data), 214 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 215 1, OMPT_GET_RETURN_ADDRESS(0)); 216 ompt_callbacks.ompt_callback(ompt_callback_work)( 217 ompt_work_single_other, ompt_scope_end, 218 &(team->t.ompt_team_info.parallel_data), 219 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 220 1, OMPT_GET_RETURN_ADDRESS(0)); 221 } 222 } 223 } 224 #endif 225 226 return rc; 227 } 228 229 void *KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SINGLE_COPY_START)(void) { 230 void *retval; 231 int gtid = __kmp_entry_gtid(); 232 MKLOC(loc, "GOMP_single_copy_start"); 233 KA_TRACE(20, ("GOMP_single_copy_start: T#%d\n", gtid)); 234 235 if (!TCR_4(__kmp_init_parallel)) 236 __kmp_parallel_initialize(); 237 __kmp_resume_if_soft_paused(); 238 239 // If this is the first thread to enter, return NULL. The generated code will 240 // then call GOMP_single_copy_end() for this thread only, with the 241 // copyprivate data pointer as an argument. 242 if (__kmp_enter_single(gtid, &loc, FALSE)) 243 return NULL; 244 245 // Wait for the first thread to set the copyprivate data pointer, 246 // and for all other threads to reach this point. 247 248 #if OMPT_SUPPORT && OMPT_OPTIONAL 249 ompt_frame_t *ompt_frame; 250 if (ompt_enabled.enabled) { 251 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 252 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 253 } 254 OMPT_STORE_RETURN_ADDRESS(gtid); 255 #endif 256 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 257 258 // Retrieve the value of the copyprivate data point, and wait for all 259 // threads to do likewise, then return. 260 retval = __kmp_team_from_gtid(gtid)->t.t_copypriv_data; 261 { 262 #if OMPT_SUPPORT && OMPT_OPTIONAL 263 OMPT_STORE_RETURN_ADDRESS(gtid); 264 #endif 265 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 266 } 267 #if OMPT_SUPPORT && OMPT_OPTIONAL 268 if (ompt_enabled.enabled) { 269 ompt_frame->enter_frame = ompt_data_none; 270 } 271 #endif 272 return retval; 273 } 274 275 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SINGLE_COPY_END)(void *data) { 276 int gtid = __kmp_get_gtid(); 277 KA_TRACE(20, ("GOMP_single_copy_end: T#%d\n", gtid)); 278 279 // Set the copyprivate data pointer fo the team, then hit the barrier so that 280 // the other threads will continue on and read it. Hit another barrier before 281 // continuing, so that the know that the copyprivate data pointer has been 282 // propagated to all threads before trying to reuse the t_copypriv_data field. 283 __kmp_team_from_gtid(gtid)->t.t_copypriv_data = data; 284 #if OMPT_SUPPORT && OMPT_OPTIONAL 285 ompt_frame_t *ompt_frame; 286 if (ompt_enabled.enabled) { 287 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 288 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 289 } 290 OMPT_STORE_RETURN_ADDRESS(gtid); 291 #endif 292 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 293 { 294 #if OMPT_SUPPORT && OMPT_OPTIONAL 295 OMPT_STORE_RETURN_ADDRESS(gtid); 296 #endif 297 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 298 } 299 #if OMPT_SUPPORT && OMPT_OPTIONAL 300 if (ompt_enabled.enabled) { 301 ompt_frame->enter_frame = ompt_data_none; 302 } 303 #endif 304 } 305 306 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ORDERED_START)(void) { 307 int gtid = __kmp_entry_gtid(); 308 MKLOC(loc, "GOMP_ordered_start"); 309 KA_TRACE(20, ("GOMP_ordered_start: T#%d\n", gtid)); 310 #if OMPT_SUPPORT && OMPT_OPTIONAL 311 OMPT_STORE_RETURN_ADDRESS(gtid); 312 #endif 313 __kmpc_ordered(&loc, gtid); 314 } 315 316 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ORDERED_END)(void) { 317 int gtid = __kmp_get_gtid(); 318 MKLOC(loc, "GOMP_ordered_end"); 319 KA_TRACE(20, ("GOMP_ordered_start: T#%d\n", gtid)); 320 #if OMPT_SUPPORT && OMPT_OPTIONAL 321 OMPT_STORE_RETURN_ADDRESS(gtid); 322 #endif 323 __kmpc_end_ordered(&loc, gtid); 324 } 325 326 // Dispatch macro defs 327 // 328 // They come in two flavors: 64-bit unsigned, and either 32-bit signed 329 // (IA-32 architecture) or 64-bit signed (Intel(R) 64). 330 331 #if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS 332 #define KMP_DISPATCH_INIT __kmp_aux_dispatch_init_4 333 #define KMP_DISPATCH_FINI_CHUNK __kmp_aux_dispatch_fini_chunk_4 334 #define KMP_DISPATCH_NEXT __kmpc_dispatch_next_4 335 #else 336 #define KMP_DISPATCH_INIT __kmp_aux_dispatch_init_8 337 #define KMP_DISPATCH_FINI_CHUNK __kmp_aux_dispatch_fini_chunk_8 338 #define KMP_DISPATCH_NEXT __kmpc_dispatch_next_8 339 #endif /* KMP_ARCH_X86 */ 340 341 #define KMP_DISPATCH_INIT_ULL __kmp_aux_dispatch_init_8u 342 #define KMP_DISPATCH_FINI_CHUNK_ULL __kmp_aux_dispatch_fini_chunk_8u 343 #define KMP_DISPATCH_NEXT_ULL __kmpc_dispatch_next_8u 344 345 // The parallel construct 346 347 #ifndef KMP_DEBUG 348 static 349 #endif /* KMP_DEBUG */ 350 void 351 __kmp_GOMP_microtask_wrapper(int *gtid, int *npr, void (*task)(void *), 352 void *data) { 353 #if OMPT_SUPPORT 354 kmp_info_t *thr; 355 ompt_frame_t *ompt_frame; 356 ompt_state_t enclosing_state; 357 358 if (ompt_enabled.enabled) { 359 // get pointer to thread data structure 360 thr = __kmp_threads[*gtid]; 361 362 // save enclosing task state; set current state for task 363 enclosing_state = thr->th.ompt_thread_info.state; 364 thr->th.ompt_thread_info.state = ompt_state_work_parallel; 365 366 // set task frame 367 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 368 ompt_frame->exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 369 } 370 #endif 371 372 task(data); 373 374 #if OMPT_SUPPORT 375 if (ompt_enabled.enabled) { 376 // clear task frame 377 ompt_frame->exit_frame = ompt_data_none; 378 379 // restore enclosing state 380 thr->th.ompt_thread_info.state = enclosing_state; 381 } 382 #endif 383 } 384 385 #ifndef KMP_DEBUG 386 static 387 #endif /* KMP_DEBUG */ 388 void 389 __kmp_GOMP_parallel_microtask_wrapper(int *gtid, int *npr, 390 void (*task)(void *), void *data, 391 unsigned num_threads, ident_t *loc, 392 enum sched_type schedule, long start, 393 long end, long incr, 394 long chunk_size) { 395 // Initialize the loop worksharing construct. 396 397 KMP_DISPATCH_INIT(loc, *gtid, schedule, start, end, incr, chunk_size, 398 schedule != kmp_sch_static); 399 400 #if OMPT_SUPPORT 401 kmp_info_t *thr; 402 ompt_frame_t *ompt_frame; 403 ompt_state_t enclosing_state; 404 405 if (ompt_enabled.enabled) { 406 thr = __kmp_threads[*gtid]; 407 // save enclosing task state; set current state for task 408 enclosing_state = thr->th.ompt_thread_info.state; 409 thr->th.ompt_thread_info.state = ompt_state_work_parallel; 410 411 // set task frame 412 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 413 ompt_frame->exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 414 } 415 #endif 416 417 // Now invoke the microtask. 418 task(data); 419 420 #if OMPT_SUPPORT 421 if (ompt_enabled.enabled) { 422 // clear task frame 423 ompt_frame->exit_frame = ompt_data_none; 424 425 // reset enclosing state 426 thr->th.ompt_thread_info.state = enclosing_state; 427 } 428 #endif 429 } 430 431 static void __kmp_GOMP_fork_call(ident_t *loc, int gtid, unsigned num_threads, 432 unsigned flags, void (*unwrapped_task)(void *), 433 microtask_t wrapper, int argc, ...) { 434 int rc; 435 kmp_info_t *thr = __kmp_threads[gtid]; 436 kmp_team_t *team = thr->th.th_team; 437 int tid = __kmp_tid_from_gtid(gtid); 438 439 va_list ap; 440 va_start(ap, argc); 441 442 if (num_threads != 0) 443 __kmp_push_num_threads(loc, gtid, num_threads); 444 if (flags != 0) 445 __kmp_push_proc_bind(loc, gtid, (kmp_proc_bind_t)flags); 446 rc = __kmp_fork_call(loc, gtid, fork_context_gnu, argc, wrapper, 447 __kmp_invoke_task_func, kmp_va_addr_of(ap)); 448 449 va_end(ap); 450 451 if (rc) { 452 __kmp_run_before_invoked_task(gtid, tid, thr, team); 453 } 454 455 #if OMPT_SUPPORT 456 int ompt_team_size; 457 if (ompt_enabled.enabled) { 458 ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL); 459 ompt_task_info_t *task_info = __ompt_get_task_info_object(0); 460 461 // implicit task callback 462 if (ompt_enabled.ompt_callback_implicit_task) { 463 ompt_team_size = __kmp_team_from_gtid(gtid)->t.t_nproc; 464 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 465 ompt_scope_begin, &(team_info->parallel_data), 466 &(task_info->task_data), ompt_team_size, __kmp_tid_from_gtid(gtid), ompt_task_implicit); // TODO: Can this be ompt_task_initial? 467 task_info->thread_num = __kmp_tid_from_gtid(gtid); 468 } 469 thr->th.ompt_thread_info.state = ompt_state_work_parallel; 470 } 471 #endif 472 } 473 474 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_START)(void (*task)(void *), 475 void *data, 476 unsigned num_threads) { 477 int gtid = __kmp_entry_gtid(); 478 479 #if OMPT_SUPPORT 480 ompt_frame_t *parent_frame, *frame; 481 482 if (ompt_enabled.enabled) { 483 __ompt_get_task_info_internal(0, NULL, NULL, &parent_frame, NULL, NULL); 484 parent_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 485 } 486 OMPT_STORE_RETURN_ADDRESS(gtid); 487 #endif 488 489 MKLOC(loc, "GOMP_parallel_start"); 490 KA_TRACE(20, ("GOMP_parallel_start: T#%d\n", gtid)); 491 __kmp_GOMP_fork_call(&loc, gtid, num_threads, 0u, task, 492 (microtask_t)__kmp_GOMP_microtask_wrapper, 2, task, 493 data); 494 #if OMPT_SUPPORT 495 if (ompt_enabled.enabled) { 496 __ompt_get_task_info_internal(0, NULL, NULL, &frame, NULL, NULL); 497 frame->exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 498 } 499 #endif 500 } 501 502 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)(void) { 503 int gtid = __kmp_get_gtid(); 504 kmp_info_t *thr; 505 506 thr = __kmp_threads[gtid]; 507 508 MKLOC(loc, "GOMP_parallel_end"); 509 KA_TRACE(20, ("GOMP_parallel_end: T#%d\n", gtid)); 510 511 if (!thr->th.th_team->t.t_serialized) { 512 __kmp_run_after_invoked_task(gtid, __kmp_tid_from_gtid(gtid), thr, 513 thr->th.th_team); 514 } 515 #if OMPT_SUPPORT 516 if (ompt_enabled.enabled) { 517 // Implicit task is finished here, in the barrier we might schedule 518 // deferred tasks, 519 // these don't see the implicit task on the stack 520 OMPT_CUR_TASK_INFO(thr)->frame.exit_frame = ompt_data_none; 521 } 522 #endif 523 524 __kmp_join_call(&loc, gtid 525 #if OMPT_SUPPORT 526 , 527 fork_context_gnu 528 #endif 529 ); 530 } 531 532 // Loop worksharing constructs 533 534 // The Gnu codegen passes in an exclusive upper bound for the overall range, 535 // but the libguide dispatch code expects an inclusive upper bound, hence the 536 // "end - incr" 5th argument to KMP_DISPATCH_INIT (and the " ub - str" 11th 537 // argument to __kmp_GOMP_fork_call). 538 // 539 // Conversely, KMP_DISPATCH_NEXT returns and inclusive upper bound in *p_ub, 540 // but the Gnu codegen expects an exclusive upper bound, so the adjustment 541 // "*p_ub += stride" compensates for the discrepancy. 542 // 543 // Correction: the gnu codegen always adjusts the upper bound by +-1, not the 544 // stride value. We adjust the dispatch parameters accordingly (by +-1), but 545 // we still adjust p_ub by the actual stride value. 546 // 547 // The "runtime" versions do not take a chunk_sz parameter. 548 // 549 // The profile lib cannot support construct checking of unordered loops that 550 // are predetermined by the compiler to be statically scheduled, as the gcc 551 // codegen will not always emit calls to GOMP_loop_static_next() to get the 552 // next iteration. Instead, it emits inline code to call omp_get_thread_num() 553 // num and calculate the iteration space using the result. It doesn't do this 554 // with ordered static loop, so they can be checked. 555 556 #if OMPT_SUPPORT 557 #define IF_OMPT_SUPPORT(code) code 558 #else 559 #define IF_OMPT_SUPPORT(code) 560 #endif 561 562 #define LOOP_START(func, schedule) \ 563 int func(long lb, long ub, long str, long chunk_sz, long *p_lb, \ 564 long *p_ub) { \ 565 int status; \ 566 long stride; \ 567 int gtid = __kmp_entry_gtid(); \ 568 MKLOC(loc, KMP_STR(func)); \ 569 KA_TRACE( \ 570 20, \ 571 (KMP_STR( \ 572 func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz 0x%lx\n", \ 573 gtid, lb, ub, str, chunk_sz)); \ 574 \ 575 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 576 { \ 577 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);) \ 578 KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \ 579 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \ 580 (schedule) != kmp_sch_static); \ 581 } \ 582 { \ 583 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);) \ 584 status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb, \ 585 (kmp_int *)p_ub, (kmp_int *)&stride); \ 586 } \ 587 if (status) { \ 588 KMP_DEBUG_ASSERT(stride == str); \ 589 *p_ub += (str > 0) ? 1 : -1; \ 590 } \ 591 } else { \ 592 status = 0; \ 593 } \ 594 \ 595 KA_TRACE( \ 596 20, \ 597 (KMP_STR( \ 598 func) " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, returning %d\n", \ 599 gtid, *p_lb, *p_ub, status)); \ 600 return status; \ 601 } 602 603 #define LOOP_RUNTIME_START(func, schedule) \ 604 int func(long lb, long ub, long str, long *p_lb, long *p_ub) { \ 605 int status; \ 606 long stride; \ 607 long chunk_sz = 0; \ 608 int gtid = __kmp_entry_gtid(); \ 609 MKLOC(loc, KMP_STR(func)); \ 610 KA_TRACE( \ 611 20, \ 612 (KMP_STR(func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz %d\n", \ 613 gtid, lb, ub, str, chunk_sz)); \ 614 \ 615 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 616 { \ 617 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);) \ 618 KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \ 619 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \ 620 TRUE); \ 621 } \ 622 { \ 623 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);) \ 624 status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb, \ 625 (kmp_int *)p_ub, (kmp_int *)&stride); \ 626 } \ 627 if (status) { \ 628 KMP_DEBUG_ASSERT(stride == str); \ 629 *p_ub += (str > 0) ? 1 : -1; \ 630 } \ 631 } else { \ 632 status = 0; \ 633 } \ 634 \ 635 KA_TRACE( \ 636 20, \ 637 (KMP_STR( \ 638 func) " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, returning %d\n", \ 639 gtid, *p_lb, *p_ub, status)); \ 640 return status; \ 641 } 642 643 #define KMP_DOACROSS_FINI(status, gtid) \ 644 if (!status && __kmp_threads[gtid]->th.th_dispatch->th_doacross_flags) { \ 645 __kmpc_doacross_fini(NULL, gtid); \ 646 } 647 648 #define LOOP_NEXT(func, fini_code) \ 649 int func(long *p_lb, long *p_ub) { \ 650 int status; \ 651 long stride; \ 652 int gtid = __kmp_get_gtid(); \ 653 MKLOC(loc, KMP_STR(func)); \ 654 KA_TRACE(20, (KMP_STR(func) ": T#%d\n", gtid)); \ 655 \ 656 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);) \ 657 fini_code status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb, \ 658 (kmp_int *)p_ub, (kmp_int *)&stride); \ 659 if (status) { \ 660 *p_ub += (stride > 0) ? 1 : -1; \ 661 } \ 662 KMP_DOACROSS_FINI(status, gtid) \ 663 \ 664 KA_TRACE( \ 665 20, \ 666 (KMP_STR(func) " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, stride 0x%lx, " \ 667 "returning %d\n", \ 668 gtid, *p_lb, *p_ub, stride, status)); \ 669 return status; \ 670 } 671 672 LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_STATIC_START), kmp_sch_static) 673 LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_STATIC_NEXT), {}) 674 LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DYNAMIC_START), 675 kmp_sch_dynamic_chunked) 676 LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_DYNAMIC_START), 677 kmp_sch_dynamic_chunked) 678 LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DYNAMIC_NEXT), {}) 679 LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_DYNAMIC_NEXT), {}) 680 LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_GUIDED_START), 681 kmp_sch_guided_chunked) 682 LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_GUIDED_START), 683 kmp_sch_guided_chunked) 684 LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_GUIDED_NEXT), {}) 685 LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_GUIDED_NEXT), {}) 686 LOOP_RUNTIME_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_RUNTIME_START), 687 kmp_sch_runtime) 688 LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_RUNTIME_NEXT), {}) 689 LOOP_RUNTIME_START( 690 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_MAYBE_NONMONOTONIC_RUNTIME_START), 691 kmp_sch_runtime) 692 LOOP_RUNTIME_START( 693 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_RUNTIME_START), 694 kmp_sch_runtime) 695 LOOP_NEXT( 696 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_MAYBE_NONMONOTONIC_RUNTIME_NEXT), {}) 697 LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_RUNTIME_NEXT), {}) 698 699 LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_START), 700 kmp_ord_static) 701 LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_NEXT), 702 { KMP_DISPATCH_FINI_CHUNK(&loc, gtid); }) 703 LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_START), 704 kmp_ord_dynamic_chunked) 705 LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_NEXT), 706 { KMP_DISPATCH_FINI_CHUNK(&loc, gtid); }) 707 LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_START), 708 kmp_ord_guided_chunked) 709 LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_NEXT), 710 { KMP_DISPATCH_FINI_CHUNK(&loc, gtid); }) 711 LOOP_RUNTIME_START( 712 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_START), 713 kmp_ord_runtime) 714 LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_NEXT), 715 { KMP_DISPATCH_FINI_CHUNK(&loc, gtid); }) 716 717 #define LOOP_DOACROSS_START(func, schedule) \ 718 bool func(unsigned ncounts, long *counts, long chunk_sz, long *p_lb, \ 719 long *p_ub) { \ 720 int status; \ 721 long stride, lb, ub, str; \ 722 int gtid = __kmp_entry_gtid(); \ 723 struct kmp_dim *dims = \ 724 (struct kmp_dim *)__kmp_allocate(sizeof(struct kmp_dim) * ncounts); \ 725 MKLOC(loc, KMP_STR(func)); \ 726 for (unsigned i = 0; i < ncounts; ++i) { \ 727 dims[i].lo = 0; \ 728 dims[i].up = counts[i] - 1; \ 729 dims[i].st = 1; \ 730 } \ 731 __kmpc_doacross_init(&loc, gtid, (int)ncounts, dims); \ 732 lb = 0; \ 733 ub = counts[0]; \ 734 str = 1; \ 735 KA_TRACE(20, (KMP_STR(func) ": T#%d, ncounts %u, lb 0x%lx, ub 0x%lx, str " \ 736 "0x%lx, chunk_sz " \ 737 "0x%lx\n", \ 738 gtid, ncounts, lb, ub, str, chunk_sz)); \ 739 \ 740 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 741 KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \ 742 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \ 743 (schedule) != kmp_sch_static); \ 744 status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb, \ 745 (kmp_int *)p_ub, (kmp_int *)&stride); \ 746 if (status) { \ 747 KMP_DEBUG_ASSERT(stride == str); \ 748 *p_ub += (str > 0) ? 1 : -1; \ 749 } \ 750 } else { \ 751 status = 0; \ 752 } \ 753 KMP_DOACROSS_FINI(status, gtid); \ 754 \ 755 KA_TRACE( \ 756 20, \ 757 (KMP_STR( \ 758 func) " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, returning %d\n", \ 759 gtid, *p_lb, *p_ub, status)); \ 760 __kmp_free(dims); \ 761 return status; \ 762 } 763 764 #define LOOP_DOACROSS_RUNTIME_START(func, schedule) \ 765 int func(unsigned ncounts, long *counts, long *p_lb, long *p_ub) { \ 766 int status; \ 767 long stride, lb, ub, str; \ 768 long chunk_sz = 0; \ 769 int gtid = __kmp_entry_gtid(); \ 770 struct kmp_dim *dims = \ 771 (struct kmp_dim *)__kmp_allocate(sizeof(struct kmp_dim) * ncounts); \ 772 MKLOC(loc, KMP_STR(func)); \ 773 for (unsigned i = 0; i < ncounts; ++i) { \ 774 dims[i].lo = 0; \ 775 dims[i].up = counts[i] - 1; \ 776 dims[i].st = 1; \ 777 } \ 778 __kmpc_doacross_init(&loc, gtid, (int)ncounts, dims); \ 779 lb = 0; \ 780 ub = counts[0]; \ 781 str = 1; \ 782 KA_TRACE( \ 783 20, \ 784 (KMP_STR(func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz %d\n", \ 785 gtid, lb, ub, str, chunk_sz)); \ 786 \ 787 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 788 KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \ 789 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, TRUE); \ 790 status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb, \ 791 (kmp_int *)p_ub, (kmp_int *)&stride); \ 792 if (status) { \ 793 KMP_DEBUG_ASSERT(stride == str); \ 794 *p_ub += (str > 0) ? 1 : -1; \ 795 } \ 796 } else { \ 797 status = 0; \ 798 } \ 799 KMP_DOACROSS_FINI(status, gtid); \ 800 \ 801 KA_TRACE( \ 802 20, \ 803 (KMP_STR( \ 804 func) " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, returning %d\n", \ 805 gtid, *p_lb, *p_ub, status)); \ 806 __kmp_free(dims); \ 807 return status; \ 808 } 809 810 LOOP_DOACROSS_START( 811 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DOACROSS_STATIC_START), 812 kmp_sch_static) 813 LOOP_DOACROSS_START( 814 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DOACROSS_DYNAMIC_START), 815 kmp_sch_dynamic_chunked) 816 LOOP_DOACROSS_START( 817 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DOACROSS_GUIDED_START), 818 kmp_sch_guided_chunked) 819 LOOP_DOACROSS_RUNTIME_START( 820 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DOACROSS_RUNTIME_START), 821 kmp_sch_runtime) 822 823 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_END)(void) { 824 int gtid = __kmp_get_gtid(); 825 KA_TRACE(20, ("GOMP_loop_end: T#%d\n", gtid)) 826 827 #if OMPT_SUPPORT && OMPT_OPTIONAL 828 ompt_frame_t *ompt_frame; 829 if (ompt_enabled.enabled) { 830 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 831 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 832 OMPT_STORE_RETURN_ADDRESS(gtid); 833 } 834 #endif 835 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 836 #if OMPT_SUPPORT && OMPT_OPTIONAL 837 if (ompt_enabled.enabled) { 838 ompt_frame->enter_frame = ompt_data_none; 839 } 840 #endif 841 842 KA_TRACE(20, ("GOMP_loop_end exit: T#%d\n", gtid)) 843 } 844 845 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_END_NOWAIT)(void) { 846 KA_TRACE(20, ("GOMP_loop_end_nowait: T#%d\n", __kmp_get_gtid())) 847 } 848 849 // Unsigned long long loop worksharing constructs 850 // 851 // These are new with gcc 4.4 852 853 #define LOOP_START_ULL(func, schedule) \ 854 int func(int up, unsigned long long lb, unsigned long long ub, \ 855 unsigned long long str, unsigned long long chunk_sz, \ 856 unsigned long long *p_lb, unsigned long long *p_ub) { \ 857 int status; \ 858 long long str2 = up ? ((long long)str) : -((long long)str); \ 859 long long stride; \ 860 int gtid = __kmp_entry_gtid(); \ 861 MKLOC(loc, KMP_STR(func)); \ 862 \ 863 KA_TRACE(20, (KMP_STR(func) ": T#%d, up %d, lb 0x%llx, ub 0x%llx, str " \ 864 "0x%llx, chunk_sz 0x%llx\n", \ 865 gtid, up, lb, ub, str, chunk_sz)); \ 866 \ 867 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 868 KMP_DISPATCH_INIT_ULL(&loc, gtid, (schedule), lb, \ 869 (str2 > 0) ? (ub - 1) : (ub + 1), str2, chunk_sz, \ 870 (schedule) != kmp_sch_static); \ 871 status = \ 872 KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb, \ 873 (kmp_uint64 *)p_ub, (kmp_int64 *)&stride); \ 874 if (status) { \ 875 KMP_DEBUG_ASSERT(stride == str2); \ 876 *p_ub += (str > 0) ? 1 : -1; \ 877 } \ 878 } else { \ 879 status = 0; \ 880 } \ 881 \ 882 KA_TRACE( \ 883 20, \ 884 (KMP_STR( \ 885 func) " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, returning %d\n", \ 886 gtid, *p_lb, *p_ub, status)); \ 887 return status; \ 888 } 889 890 #define LOOP_RUNTIME_START_ULL(func, schedule) \ 891 int func(int up, unsigned long long lb, unsigned long long ub, \ 892 unsigned long long str, unsigned long long *p_lb, \ 893 unsigned long long *p_ub) { \ 894 int status; \ 895 long long str2 = up ? ((long long)str) : -((long long)str); \ 896 unsigned long long stride; \ 897 unsigned long long chunk_sz = 0; \ 898 int gtid = __kmp_entry_gtid(); \ 899 MKLOC(loc, KMP_STR(func)); \ 900 \ 901 KA_TRACE(20, (KMP_STR(func) ": T#%d, up %d, lb 0x%llx, ub 0x%llx, str " \ 902 "0x%llx, chunk_sz 0x%llx\n", \ 903 gtid, up, lb, ub, str, chunk_sz)); \ 904 \ 905 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 906 KMP_DISPATCH_INIT_ULL(&loc, gtid, (schedule), lb, \ 907 (str2 > 0) ? (ub - 1) : (ub + 1), str2, chunk_sz, \ 908 TRUE); \ 909 status = \ 910 KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb, \ 911 (kmp_uint64 *)p_ub, (kmp_int64 *)&stride); \ 912 if (status) { \ 913 KMP_DEBUG_ASSERT((long long)stride == str2); \ 914 *p_ub += (str > 0) ? 1 : -1; \ 915 } \ 916 } else { \ 917 status = 0; \ 918 } \ 919 \ 920 KA_TRACE( \ 921 20, \ 922 (KMP_STR( \ 923 func) " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, returning %d\n", \ 924 gtid, *p_lb, *p_ub, status)); \ 925 return status; \ 926 } 927 928 #define LOOP_NEXT_ULL(func, fini_code) \ 929 int func(unsigned long long *p_lb, unsigned long long *p_ub) { \ 930 int status; \ 931 long long stride; \ 932 int gtid = __kmp_get_gtid(); \ 933 MKLOC(loc, KMP_STR(func)); \ 934 KA_TRACE(20, (KMP_STR(func) ": T#%d\n", gtid)); \ 935 \ 936 fini_code status = \ 937 KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb, \ 938 (kmp_uint64 *)p_ub, (kmp_int64 *)&stride); \ 939 if (status) { \ 940 *p_ub += (stride > 0) ? 1 : -1; \ 941 } \ 942 \ 943 KA_TRACE( \ 944 20, \ 945 (KMP_STR( \ 946 func) " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, stride 0x%llx, " \ 947 "returning %d\n", \ 948 gtid, *p_lb, *p_ub, stride, status)); \ 949 return status; \ 950 } 951 952 LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_START), 953 kmp_sch_static) 954 LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_NEXT), {}) 955 LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_START), 956 kmp_sch_dynamic_chunked) 957 LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_NEXT), {}) 958 LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_START), 959 kmp_sch_guided_chunked) 960 LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_NEXT), {}) 961 LOOP_START_ULL( 962 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_DYNAMIC_START), 963 kmp_sch_dynamic_chunked) 964 LOOP_NEXT_ULL( 965 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_DYNAMIC_NEXT), {}) 966 LOOP_START_ULL( 967 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_GUIDED_START), 968 kmp_sch_guided_chunked) 969 LOOP_NEXT_ULL( 970 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_GUIDED_NEXT), {}) 971 LOOP_RUNTIME_START_ULL( 972 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_START), kmp_sch_runtime) 973 LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_NEXT), {}) 974 LOOP_RUNTIME_START_ULL( 975 KMP_EXPAND_NAME( 976 KMP_API_NAME_GOMP_LOOP_ULL_MAYBE_NONMONOTONIC_RUNTIME_START), 977 kmp_sch_runtime) 978 LOOP_RUNTIME_START_ULL( 979 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_RUNTIME_START), 980 kmp_sch_runtime) 981 LOOP_NEXT_ULL( 982 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_MAYBE_NONMONOTONIC_RUNTIME_NEXT), 983 {}) 984 LOOP_NEXT_ULL( 985 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_RUNTIME_NEXT), {}) 986 987 LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_START), 988 kmp_ord_static) 989 LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_NEXT), 990 { KMP_DISPATCH_FINI_CHUNK_ULL(&loc, gtid); }) 991 LOOP_START_ULL( 992 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_START), 993 kmp_ord_dynamic_chunked) 994 LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_NEXT), 995 { KMP_DISPATCH_FINI_CHUNK_ULL(&loc, gtid); }) 996 LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_START), 997 kmp_ord_guided_chunked) 998 LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_NEXT), 999 { KMP_DISPATCH_FINI_CHUNK_ULL(&loc, gtid); }) 1000 LOOP_RUNTIME_START_ULL( 1001 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_START), 1002 kmp_ord_runtime) 1003 LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_NEXT), 1004 { KMP_DISPATCH_FINI_CHUNK_ULL(&loc, gtid); }) 1005 1006 #define LOOP_DOACROSS_START_ULL(func, schedule) \ 1007 int func(unsigned ncounts, unsigned long long *counts, \ 1008 unsigned long long chunk_sz, unsigned long long *p_lb, \ 1009 unsigned long long *p_ub) { \ 1010 int status; \ 1011 long long stride, str, lb, ub; \ 1012 int gtid = __kmp_entry_gtid(); \ 1013 struct kmp_dim *dims = \ 1014 (struct kmp_dim *)__kmp_allocate(sizeof(struct kmp_dim) * ncounts); \ 1015 MKLOC(loc, KMP_STR(func)); \ 1016 for (unsigned i = 0; i < ncounts; ++i) { \ 1017 dims[i].lo = 0; \ 1018 dims[i].up = counts[i] - 1; \ 1019 dims[i].st = 1; \ 1020 } \ 1021 __kmpc_doacross_init(&loc, gtid, (int)ncounts, dims); \ 1022 lb = 0; \ 1023 ub = counts[0]; \ 1024 str = 1; \ 1025 \ 1026 KA_TRACE(20, (KMP_STR(func) ": T#%d, lb 0x%llx, ub 0x%llx, str " \ 1027 "0x%llx, chunk_sz 0x%llx\n", \ 1028 gtid, lb, ub, str, chunk_sz)); \ 1029 \ 1030 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 1031 KMP_DISPATCH_INIT_ULL(&loc, gtid, (schedule), lb, \ 1032 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \ 1033 (schedule) != kmp_sch_static); \ 1034 status = \ 1035 KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb, \ 1036 (kmp_uint64 *)p_ub, (kmp_int64 *)&stride); \ 1037 if (status) { \ 1038 KMP_DEBUG_ASSERT(stride == str); \ 1039 *p_ub += (str > 0) ? 1 : -1; \ 1040 } \ 1041 } else { \ 1042 status = 0; \ 1043 } \ 1044 KMP_DOACROSS_FINI(status, gtid); \ 1045 \ 1046 KA_TRACE( \ 1047 20, \ 1048 (KMP_STR( \ 1049 func) " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, returning %d\n", \ 1050 gtid, *p_lb, *p_ub, status)); \ 1051 __kmp_free(dims); \ 1052 return status; \ 1053 } 1054 1055 #define LOOP_DOACROSS_RUNTIME_START_ULL(func, schedule) \ 1056 int func(unsigned ncounts, unsigned long long *counts, \ 1057 unsigned long long *p_lb, unsigned long long *p_ub) { \ 1058 int status; \ 1059 unsigned long long stride, str, lb, ub; \ 1060 unsigned long long chunk_sz = 0; \ 1061 int gtid = __kmp_entry_gtid(); \ 1062 struct kmp_dim *dims = \ 1063 (struct kmp_dim *)__kmp_allocate(sizeof(struct kmp_dim) * ncounts); \ 1064 MKLOC(loc, KMP_STR(func)); \ 1065 for (unsigned i = 0; i < ncounts; ++i) { \ 1066 dims[i].lo = 0; \ 1067 dims[i].up = counts[i] - 1; \ 1068 dims[i].st = 1; \ 1069 } \ 1070 __kmpc_doacross_init(&loc, gtid, (int)ncounts, dims); \ 1071 lb = 0; \ 1072 ub = counts[0]; \ 1073 str = 1; \ 1074 KA_TRACE(20, (KMP_STR(func) ": T#%d, lb 0x%llx, ub 0x%llx, str " \ 1075 "0x%llx, chunk_sz 0x%llx\n", \ 1076 gtid, lb, ub, str, chunk_sz)); \ 1077 \ 1078 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 1079 KMP_DISPATCH_INIT_ULL(&loc, gtid, (schedule), lb, \ 1080 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \ 1081 TRUE); \ 1082 status = \ 1083 KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb, \ 1084 (kmp_uint64 *)p_ub, (kmp_int64 *)&stride); \ 1085 if (status) { \ 1086 KMP_DEBUG_ASSERT(stride == str); \ 1087 *p_ub += (str > 0) ? 1 : -1; \ 1088 } \ 1089 } else { \ 1090 status = 0; \ 1091 } \ 1092 KMP_DOACROSS_FINI(status, gtid); \ 1093 \ 1094 KA_TRACE( \ 1095 20, \ 1096 (KMP_STR( \ 1097 func) " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, returning %d\n", \ 1098 gtid, *p_lb, *p_ub, status)); \ 1099 __kmp_free(dims); \ 1100 return status; \ 1101 } 1102 1103 LOOP_DOACROSS_START_ULL( 1104 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_STATIC_START), 1105 kmp_sch_static) 1106 LOOP_DOACROSS_START_ULL( 1107 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_DYNAMIC_START), 1108 kmp_sch_dynamic_chunked) 1109 LOOP_DOACROSS_START_ULL( 1110 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_GUIDED_START), 1111 kmp_sch_guided_chunked) 1112 LOOP_DOACROSS_RUNTIME_START_ULL( 1113 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_RUNTIME_START), 1114 kmp_sch_runtime) 1115 1116 // Combined parallel / loop worksharing constructs 1117 // 1118 // There are no ull versions (yet). 1119 1120 #define PARALLEL_LOOP_START(func, schedule, ompt_pre, ompt_post) \ 1121 void func(void (*task)(void *), void *data, unsigned num_threads, long lb, \ 1122 long ub, long str, long chunk_sz) { \ 1123 int gtid = __kmp_entry_gtid(); \ 1124 MKLOC(loc, KMP_STR(func)); \ 1125 KA_TRACE( \ 1126 20, \ 1127 (KMP_STR( \ 1128 func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz 0x%lx\n", \ 1129 gtid, lb, ub, str, chunk_sz)); \ 1130 \ 1131 ompt_pre(); \ 1132 \ 1133 __kmp_GOMP_fork_call(&loc, gtid, num_threads, 0u, task, \ 1134 (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, \ 1135 9, task, data, num_threads, &loc, (schedule), lb, \ 1136 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz); \ 1137 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid)); \ 1138 \ 1139 KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \ 1140 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \ 1141 (schedule) != kmp_sch_static); \ 1142 \ 1143 ompt_post(); \ 1144 \ 1145 KA_TRACE(20, (KMP_STR(func) " exit: T#%d\n", gtid)); \ 1146 } 1147 1148 #if OMPT_SUPPORT && OMPT_OPTIONAL 1149 1150 #define OMPT_LOOP_PRE() \ 1151 ompt_frame_t *parent_frame; \ 1152 if (ompt_enabled.enabled) { \ 1153 __ompt_get_task_info_internal(0, NULL, NULL, &parent_frame, NULL, NULL); \ 1154 parent_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); \ 1155 OMPT_STORE_RETURN_ADDRESS(gtid); \ 1156 } 1157 1158 #define OMPT_LOOP_POST() \ 1159 if (ompt_enabled.enabled) { \ 1160 parent_frame->enter_frame = ompt_data_none; \ 1161 } 1162 1163 #else 1164 1165 #define OMPT_LOOP_PRE() 1166 1167 #define OMPT_LOOP_POST() 1168 1169 #endif 1170 1171 PARALLEL_LOOP_START( 1172 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC_START), 1173 kmp_sch_static, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1174 PARALLEL_LOOP_START( 1175 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC_START), 1176 kmp_sch_dynamic_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1177 PARALLEL_LOOP_START( 1178 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED_START), 1179 kmp_sch_guided_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1180 PARALLEL_LOOP_START( 1181 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME_START), 1182 kmp_sch_runtime, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1183 1184 // Tasking constructs 1185 1186 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASK)(void (*func)(void *), void *data, 1187 void (*copy_func)(void *, void *), 1188 long arg_size, long arg_align, 1189 bool if_cond, unsigned gomp_flags, 1190 void **depend) { 1191 MKLOC(loc, "GOMP_task"); 1192 int gtid = __kmp_entry_gtid(); 1193 kmp_int32 flags = 0; 1194 kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *)&flags; 1195 1196 KA_TRACE(20, ("GOMP_task: T#%d\n", gtid)); 1197 1198 // The low-order bit is the "untied" flag 1199 if (!(gomp_flags & KMP_GOMP_TASK_UNTIED_FLAG)) { 1200 input_flags->tiedness = 1; 1201 } 1202 // The second low-order bit is the "final" flag 1203 if (gomp_flags & KMP_GOMP_TASK_FINAL_FLAG) { 1204 input_flags->final = 1; 1205 } 1206 input_flags->native = 1; 1207 // __kmp_task_alloc() sets up all other flags 1208 1209 if (!if_cond) { 1210 arg_size = 0; 1211 } 1212 1213 kmp_task_t *task = __kmp_task_alloc( 1214 &loc, gtid, input_flags, sizeof(kmp_task_t), 1215 arg_size ? arg_size + arg_align - 1 : 0, (kmp_routine_entry_t)func); 1216 1217 if (arg_size > 0) { 1218 if (arg_align > 0) { 1219 task->shareds = (void *)((((size_t)task->shareds) + arg_align - 1) / 1220 arg_align * arg_align); 1221 } 1222 // else error?? 1223 1224 if (copy_func) { 1225 (*copy_func)(task->shareds, data); 1226 } else { 1227 KMP_MEMCPY(task->shareds, data, arg_size); 1228 } 1229 } 1230 1231 #if OMPT_SUPPORT 1232 kmp_taskdata_t *current_task; 1233 if (ompt_enabled.enabled) { 1234 current_task = __kmp_threads[gtid]->th.th_current_task; 1235 current_task->ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1236 } 1237 OMPT_STORE_RETURN_ADDRESS(gtid); 1238 #endif 1239 1240 if (if_cond) { 1241 if (gomp_flags & KMP_GOMP_TASK_DEPENDS_FLAG) { 1242 KMP_ASSERT(depend); 1243 kmp_gomp_depends_info_t gomp_depends(depend); 1244 kmp_int32 ndeps = gomp_depends.get_num_deps(); 1245 kmp_depend_info_t dep_list[ndeps]; 1246 for (kmp_int32 i = 0; i < ndeps; i++) 1247 dep_list[i] = gomp_depends.get_kmp_depend(i); 1248 __kmpc_omp_task_with_deps(&loc, gtid, task, ndeps, dep_list, 0, NULL); 1249 } else { 1250 __kmpc_omp_task(&loc, gtid, task); 1251 } 1252 } else { 1253 #if OMPT_SUPPORT 1254 ompt_thread_info_t oldInfo; 1255 kmp_info_t *thread; 1256 kmp_taskdata_t *taskdata; 1257 if (ompt_enabled.enabled) { 1258 // Store the threads states and restore them after the task 1259 thread = __kmp_threads[gtid]; 1260 taskdata = KMP_TASK_TO_TASKDATA(task); 1261 oldInfo = thread->th.ompt_thread_info; 1262 thread->th.ompt_thread_info.wait_id = 0; 1263 thread->th.ompt_thread_info.state = ompt_state_work_parallel; 1264 taskdata->ompt_task_info.frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1265 } 1266 OMPT_STORE_RETURN_ADDRESS(gtid); 1267 #endif 1268 if (gomp_flags & KMP_GOMP_TASK_DEPENDS_FLAG) { 1269 KMP_ASSERT(depend); 1270 kmp_gomp_depends_info_t gomp_depends(depend); 1271 kmp_int32 ndeps = gomp_depends.get_num_deps(); 1272 kmp_depend_info_t dep_list[ndeps]; 1273 for (kmp_int32 i = 0; i < ndeps; i++) 1274 dep_list[i] = gomp_depends.get_kmp_depend(i); 1275 __kmpc_omp_wait_deps(&loc, gtid, ndeps, dep_list, 0, NULL); 1276 } 1277 1278 __kmpc_omp_task_begin_if0(&loc, gtid, task); 1279 func(data); 1280 __kmpc_omp_task_complete_if0(&loc, gtid, task); 1281 1282 #if OMPT_SUPPORT 1283 if (ompt_enabled.enabled) { 1284 thread->th.ompt_thread_info = oldInfo; 1285 taskdata->ompt_task_info.frame.exit_frame = ompt_data_none; 1286 } 1287 #endif 1288 } 1289 #if OMPT_SUPPORT 1290 if (ompt_enabled.enabled) { 1291 current_task->ompt_task_info.frame.enter_frame = ompt_data_none; 1292 } 1293 #endif 1294 1295 KA_TRACE(20, ("GOMP_task exit: T#%d\n", gtid)); 1296 } 1297 1298 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKWAIT)(void) { 1299 MKLOC(loc, "GOMP_taskwait"); 1300 int gtid = __kmp_entry_gtid(); 1301 1302 #if OMPT_SUPPORT 1303 OMPT_STORE_RETURN_ADDRESS(gtid); 1304 #endif 1305 1306 KA_TRACE(20, ("GOMP_taskwait: T#%d\n", gtid)); 1307 1308 __kmpc_omp_taskwait(&loc, gtid); 1309 1310 KA_TRACE(20, ("GOMP_taskwait exit: T#%d\n", gtid)); 1311 } 1312 1313 // Sections worksharing constructs 1314 // 1315 // For the sections construct, we initialize a dynamically scheduled loop 1316 // worksharing construct with lb 1 and stride 1, and use the iteration #'s 1317 // that its returns as sections ids. 1318 // 1319 // There are no special entry points for ordered sections, so we always use 1320 // the dynamically scheduled workshare, even if the sections aren't ordered. 1321 1322 unsigned KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_START)(unsigned count) { 1323 int status; 1324 kmp_int lb, ub, stride; 1325 int gtid = __kmp_entry_gtid(); 1326 MKLOC(loc, "GOMP_sections_start"); 1327 KA_TRACE(20, ("GOMP_sections_start: T#%d\n", gtid)); 1328 1329 KMP_DISPATCH_INIT(&loc, gtid, kmp_nm_dynamic_chunked, 1, count, 1, 1, TRUE); 1330 1331 status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, &lb, &ub, &stride); 1332 if (status) { 1333 KMP_DEBUG_ASSERT(stride == 1); 1334 KMP_DEBUG_ASSERT(lb > 0); 1335 KMP_ASSERT(lb == ub); 1336 } else { 1337 lb = 0; 1338 } 1339 1340 KA_TRACE(20, ("GOMP_sections_start exit: T#%d returning %u\n", gtid, 1341 (unsigned)lb)); 1342 return (unsigned)lb; 1343 } 1344 1345 unsigned KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_NEXT)(void) { 1346 int status; 1347 kmp_int lb, ub, stride; 1348 int gtid = __kmp_get_gtid(); 1349 MKLOC(loc, "GOMP_sections_next"); 1350 KA_TRACE(20, ("GOMP_sections_next: T#%d\n", gtid)); 1351 1352 #if OMPT_SUPPORT 1353 OMPT_STORE_RETURN_ADDRESS(gtid); 1354 #endif 1355 1356 status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, &lb, &ub, &stride); 1357 if (status) { 1358 KMP_DEBUG_ASSERT(stride == 1); 1359 KMP_DEBUG_ASSERT(lb > 0); 1360 KMP_ASSERT(lb == ub); 1361 } else { 1362 lb = 0; 1363 } 1364 1365 KA_TRACE( 1366 20, ("GOMP_sections_next exit: T#%d returning %u\n", gtid, (unsigned)lb)); 1367 return (unsigned)lb; 1368 } 1369 1370 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_SECTIONS_START)( 1371 void (*task)(void *), void *data, unsigned num_threads, unsigned count) { 1372 int gtid = __kmp_entry_gtid(); 1373 1374 #if OMPT_SUPPORT 1375 ompt_frame_t *parent_frame; 1376 1377 if (ompt_enabled.enabled) { 1378 __ompt_get_task_info_internal(0, NULL, NULL, &parent_frame, NULL, NULL); 1379 parent_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1380 } 1381 OMPT_STORE_RETURN_ADDRESS(gtid); 1382 #endif 1383 1384 MKLOC(loc, "GOMP_parallel_sections_start"); 1385 KA_TRACE(20, ("GOMP_parallel_sections_start: T#%d\n", gtid)); 1386 1387 __kmp_GOMP_fork_call(&loc, gtid, num_threads, 0u, task, 1388 (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, 9, 1389 task, data, num_threads, &loc, kmp_nm_dynamic_chunked, 1390 (kmp_int)1, (kmp_int)count, (kmp_int)1, (kmp_int)1); 1391 1392 #if OMPT_SUPPORT 1393 if (ompt_enabled.enabled) { 1394 parent_frame->enter_frame = ompt_data_none; 1395 } 1396 #endif 1397 1398 KMP_DISPATCH_INIT(&loc, gtid, kmp_nm_dynamic_chunked, 1, count, 1, 1, TRUE); 1399 1400 KA_TRACE(20, ("GOMP_parallel_sections_start exit: T#%d\n", gtid)); 1401 } 1402 1403 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_END)(void) { 1404 int gtid = __kmp_get_gtid(); 1405 KA_TRACE(20, ("GOMP_sections_end: T#%d\n", gtid)) 1406 1407 #if OMPT_SUPPORT 1408 ompt_frame_t *ompt_frame; 1409 if (ompt_enabled.enabled) { 1410 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 1411 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1412 } 1413 OMPT_STORE_RETURN_ADDRESS(gtid); 1414 #endif 1415 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 1416 #if OMPT_SUPPORT 1417 if (ompt_enabled.enabled) { 1418 ompt_frame->enter_frame = ompt_data_none; 1419 } 1420 #endif 1421 1422 KA_TRACE(20, ("GOMP_sections_end exit: T#%d\n", gtid)) 1423 } 1424 1425 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_END_NOWAIT)(void) { 1426 KA_TRACE(20, ("GOMP_sections_end_nowait: T#%d\n", __kmp_get_gtid())) 1427 } 1428 1429 // libgomp has an empty function for GOMP_taskyield as of 2013-10-10 1430 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKYIELD)(void) { 1431 KA_TRACE(20, ("GOMP_taskyield: T#%d\n", __kmp_get_gtid())) 1432 return; 1433 } 1434 1435 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL)(void (*task)(void *), 1436 void *data, 1437 unsigned num_threads, 1438 unsigned int flags) { 1439 int gtid = __kmp_entry_gtid(); 1440 MKLOC(loc, "GOMP_parallel"); 1441 KA_TRACE(20, ("GOMP_parallel: T#%d\n", gtid)); 1442 1443 #if OMPT_SUPPORT 1444 ompt_task_info_t *parent_task_info, *task_info; 1445 if (ompt_enabled.enabled) { 1446 parent_task_info = __ompt_get_task_info_object(0); 1447 parent_task_info->frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1448 } 1449 OMPT_STORE_RETURN_ADDRESS(gtid); 1450 #endif 1451 __kmp_GOMP_fork_call(&loc, gtid, num_threads, flags, task, 1452 (microtask_t)__kmp_GOMP_microtask_wrapper, 2, task, 1453 data); 1454 #if OMPT_SUPPORT 1455 if (ompt_enabled.enabled) { 1456 task_info = __ompt_get_task_info_object(0); 1457 task_info->frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1458 } 1459 #endif 1460 task(data); 1461 { 1462 #if OMPT_SUPPORT 1463 OMPT_STORE_RETURN_ADDRESS(gtid); 1464 #endif 1465 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)(); 1466 } 1467 #if OMPT_SUPPORT 1468 if (ompt_enabled.enabled) { 1469 task_info->frame.exit_frame = ompt_data_none; 1470 parent_task_info->frame.enter_frame = ompt_data_none; 1471 } 1472 #endif 1473 } 1474 1475 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_SECTIONS)(void (*task)(void *), 1476 void *data, 1477 unsigned num_threads, 1478 unsigned count, 1479 unsigned flags) { 1480 int gtid = __kmp_entry_gtid(); 1481 MKLOC(loc, "GOMP_parallel_sections"); 1482 KA_TRACE(20, ("GOMP_parallel_sections: T#%d\n", gtid)); 1483 1484 #if OMPT_SUPPORT 1485 OMPT_STORE_RETURN_ADDRESS(gtid); 1486 #endif 1487 1488 __kmp_GOMP_fork_call(&loc, gtid, num_threads, flags, task, 1489 (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, 9, 1490 task, data, num_threads, &loc, kmp_nm_dynamic_chunked, 1491 (kmp_int)1, (kmp_int)count, (kmp_int)1, (kmp_int)1); 1492 1493 { 1494 #if OMPT_SUPPORT 1495 OMPT_STORE_RETURN_ADDRESS(gtid); 1496 #endif 1497 1498 KMP_DISPATCH_INIT(&loc, gtid, kmp_nm_dynamic_chunked, 1, count, 1, 1, TRUE); 1499 } 1500 task(data); 1501 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)(); 1502 KA_TRACE(20, ("GOMP_parallel_sections exit: T#%d\n", gtid)); 1503 } 1504 1505 #define PARALLEL_LOOP(func, schedule, ompt_pre, ompt_post) \ 1506 void func(void (*task)(void *), void *data, unsigned num_threads, long lb, \ 1507 long ub, long str, long chunk_sz, unsigned flags) { \ 1508 int gtid = __kmp_entry_gtid(); \ 1509 MKLOC(loc, KMP_STR(func)); \ 1510 KA_TRACE( \ 1511 20, \ 1512 (KMP_STR( \ 1513 func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz 0x%lx\n", \ 1514 gtid, lb, ub, str, chunk_sz)); \ 1515 \ 1516 ompt_pre(); \ 1517 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);) \ 1518 __kmp_GOMP_fork_call(&loc, gtid, num_threads, flags, task, \ 1519 (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, \ 1520 9, task, data, num_threads, &loc, (schedule), lb, \ 1521 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz); \ 1522 \ 1523 { \ 1524 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);) \ 1525 KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \ 1526 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \ 1527 (schedule) != kmp_sch_static); \ 1528 } \ 1529 task(data); \ 1530 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)(); \ 1531 ompt_post(); \ 1532 \ 1533 KA_TRACE(20, (KMP_STR(func) " exit: T#%d\n", gtid)); \ 1534 } 1535 1536 PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC), 1537 kmp_sch_static, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1538 PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC), 1539 kmp_sch_dynamic_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1540 PARALLEL_LOOP( 1541 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_GUIDED), 1542 kmp_sch_guided_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1543 PARALLEL_LOOP( 1544 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_DYNAMIC), 1545 kmp_sch_dynamic_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1546 PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED), 1547 kmp_sch_guided_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1548 PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME), 1549 kmp_sch_runtime, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1550 PARALLEL_LOOP( 1551 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_MAYBE_NONMONOTONIC_RUNTIME), 1552 kmp_sch_runtime, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1553 PARALLEL_LOOP( 1554 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_RUNTIME), 1555 kmp_sch_runtime, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1556 1557 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKGROUP_START)(void) { 1558 int gtid = __kmp_entry_gtid(); 1559 MKLOC(loc, "GOMP_taskgroup_start"); 1560 KA_TRACE(20, ("GOMP_taskgroup_start: T#%d\n", gtid)); 1561 1562 #if OMPT_SUPPORT 1563 OMPT_STORE_RETURN_ADDRESS(gtid); 1564 #endif 1565 1566 __kmpc_taskgroup(&loc, gtid); 1567 1568 return; 1569 } 1570 1571 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKGROUP_END)(void) { 1572 int gtid = __kmp_get_gtid(); 1573 MKLOC(loc, "GOMP_taskgroup_end"); 1574 KA_TRACE(20, ("GOMP_taskgroup_end: T#%d\n", gtid)); 1575 1576 #if OMPT_SUPPORT 1577 OMPT_STORE_RETURN_ADDRESS(gtid); 1578 #endif 1579 1580 __kmpc_end_taskgroup(&loc, gtid); 1581 1582 return; 1583 } 1584 1585 static kmp_int32 __kmp_gomp_to_omp_cancellation_kind(int gomp_kind) { 1586 kmp_int32 cncl_kind = 0; 1587 switch (gomp_kind) { 1588 case 1: 1589 cncl_kind = cancel_parallel; 1590 break; 1591 case 2: 1592 cncl_kind = cancel_loop; 1593 break; 1594 case 4: 1595 cncl_kind = cancel_sections; 1596 break; 1597 case 8: 1598 cncl_kind = cancel_taskgroup; 1599 break; 1600 } 1601 return cncl_kind; 1602 } 1603 1604 // Return true if cancellation should take place, false otherwise 1605 bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CANCELLATION_POINT)(int which) { 1606 int gtid = __kmp_get_gtid(); 1607 MKLOC(loc, "GOMP_cancellation_point"); 1608 KA_TRACE(20, ("GOMP_cancellation_point: T#%d which:%d\n", gtid, which)); 1609 kmp_int32 cncl_kind = __kmp_gomp_to_omp_cancellation_kind(which); 1610 return __kmpc_cancellationpoint(&loc, gtid, cncl_kind); 1611 } 1612 1613 // Return true if cancellation should take place, false otherwise 1614 bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CANCEL)(int which, bool do_cancel) { 1615 int gtid = __kmp_get_gtid(); 1616 MKLOC(loc, "GOMP_cancel"); 1617 KA_TRACE(20, ("GOMP_cancel: T#%d which:%d do_cancel:%d\n", gtid, which, 1618 (int)do_cancel)); 1619 kmp_int32 cncl_kind = __kmp_gomp_to_omp_cancellation_kind(which); 1620 1621 if (do_cancel == FALSE) { 1622 return __kmpc_cancellationpoint(&loc, gtid, cncl_kind); 1623 } else { 1624 return __kmpc_cancel(&loc, gtid, cncl_kind); 1625 } 1626 } 1627 1628 // Return true if cancellation should take place, false otherwise 1629 bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_BARRIER_CANCEL)(void) { 1630 int gtid = __kmp_get_gtid(); 1631 KA_TRACE(20, ("GOMP_barrier_cancel: T#%d\n", gtid)); 1632 return __kmp_barrier_gomp_cancel(gtid); 1633 } 1634 1635 // Return true if cancellation should take place, false otherwise 1636 bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_END_CANCEL)(void) { 1637 int gtid = __kmp_get_gtid(); 1638 KA_TRACE(20, ("GOMP_sections_end_cancel: T#%d\n", gtid)); 1639 return __kmp_barrier_gomp_cancel(gtid); 1640 } 1641 1642 // Return true if cancellation should take place, false otherwise 1643 bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_END_CANCEL)(void) { 1644 int gtid = __kmp_get_gtid(); 1645 KA_TRACE(20, ("GOMP_loop_end_cancel: T#%d\n", gtid)); 1646 return __kmp_barrier_gomp_cancel(gtid); 1647 } 1648 1649 // All target functions are empty as of 2014-05-29 1650 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TARGET)(int device, void (*fn)(void *), 1651 const void *openmp_target, 1652 size_t mapnum, void **hostaddrs, 1653 size_t *sizes, 1654 unsigned char *kinds) { 1655 return; 1656 } 1657 1658 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TARGET_DATA)( 1659 int device, const void *openmp_target, size_t mapnum, void **hostaddrs, 1660 size_t *sizes, unsigned char *kinds) { 1661 return; 1662 } 1663 1664 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TARGET_END_DATA)(void) { return; } 1665 1666 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TARGET_UPDATE)( 1667 int device, const void *openmp_target, size_t mapnum, void **hostaddrs, 1668 size_t *sizes, unsigned char *kinds) { 1669 return; 1670 } 1671 1672 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TEAMS)(unsigned int num_teams, 1673 unsigned int thread_limit) { 1674 return; 1675 } 1676 1677 // Task duplication function which copies src to dest (both are 1678 // preallocated task structures) 1679 static void __kmp_gomp_task_dup(kmp_task_t *dest, kmp_task_t *src, 1680 kmp_int32 last_private) { 1681 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(src); 1682 if (taskdata->td_copy_func) { 1683 (taskdata->td_copy_func)(dest->shareds, src->shareds); 1684 } 1685 } 1686 1687 #ifdef __cplusplus 1688 } // extern "C" 1689 #endif 1690 1691 template <typename T> 1692 void __GOMP_taskloop(void (*func)(void *), void *data, 1693 void (*copy_func)(void *, void *), long arg_size, 1694 long arg_align, unsigned gomp_flags, 1695 unsigned long num_tasks, int priority, T start, T end, 1696 T step) { 1697 typedef void (*p_task_dup_t)(kmp_task_t *, kmp_task_t *, kmp_int32); 1698 MKLOC(loc, "GOMP_taskloop"); 1699 int sched; 1700 T *loop_bounds; 1701 int gtid = __kmp_entry_gtid(); 1702 kmp_int32 flags = 0; 1703 int if_val = gomp_flags & (1u << 10); 1704 int nogroup = gomp_flags & (1u << 11); 1705 int up = gomp_flags & (1u << 8); 1706 p_task_dup_t task_dup = NULL; 1707 kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *)&flags; 1708 #ifdef KMP_DEBUG 1709 { 1710 char *buff; 1711 buff = __kmp_str_format( 1712 "GOMP_taskloop: T#%%d: func:%%p data:%%p copy_func:%%p " 1713 "arg_size:%%ld arg_align:%%ld gomp_flags:0x%%x num_tasks:%%lu " 1714 "priority:%%d start:%%%s end:%%%s step:%%%s\n", 1715 traits_t<T>::spec, traits_t<T>::spec, traits_t<T>::spec); 1716 KA_TRACE(20, (buff, gtid, func, data, copy_func, arg_size, arg_align, 1717 gomp_flags, num_tasks, priority, start, end, step)); 1718 __kmp_str_free(&buff); 1719 } 1720 #endif 1721 KMP_ASSERT((size_t)arg_size >= 2 * sizeof(T)); 1722 KMP_ASSERT(arg_align > 0); 1723 // The low-order bit is the "untied" flag 1724 if (!(gomp_flags & 1)) { 1725 input_flags->tiedness = 1; 1726 } 1727 // The second low-order bit is the "final" flag 1728 if (gomp_flags & 2) { 1729 input_flags->final = 1; 1730 } 1731 // Negative step flag 1732 if (!up) { 1733 // If step is flagged as negative, but isn't properly sign extended 1734 // Then manually sign extend it. Could be a short, int, char embedded 1735 // in a long. So cannot assume any cast. 1736 if (step > 0) { 1737 for (int i = sizeof(T) * CHAR_BIT - 1; i >= 0L; --i) { 1738 // break at the first 1 bit 1739 if (step & ((T)1 << i)) 1740 break; 1741 step |= ((T)1 << i); 1742 } 1743 } 1744 } 1745 input_flags->native = 1; 1746 // Figure out if none/grainsize/num_tasks clause specified 1747 if (num_tasks > 0) { 1748 if (gomp_flags & (1u << 9)) 1749 sched = 1; // grainsize specified 1750 else 1751 sched = 2; // num_tasks specified 1752 // neither grainsize nor num_tasks specified 1753 } else { 1754 sched = 0; 1755 } 1756 1757 // __kmp_task_alloc() sets up all other flags 1758 kmp_task_t *task = 1759 __kmp_task_alloc(&loc, gtid, input_flags, sizeof(kmp_task_t), 1760 arg_size + arg_align - 1, (kmp_routine_entry_t)func); 1761 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 1762 taskdata->td_copy_func = copy_func; 1763 taskdata->td_size_loop_bounds = sizeof(T); 1764 1765 // re-align shareds if needed and setup firstprivate copy constructors 1766 // through the task_dup mechanism 1767 task->shareds = (void *)((((size_t)task->shareds) + arg_align - 1) / 1768 arg_align * arg_align); 1769 if (copy_func) { 1770 task_dup = __kmp_gomp_task_dup; 1771 } 1772 KMP_MEMCPY(task->shareds, data, arg_size); 1773 1774 loop_bounds = (T *)task->shareds; 1775 loop_bounds[0] = start; 1776 loop_bounds[1] = end + (up ? -1 : 1); 1777 __kmpc_taskloop(&loc, gtid, task, if_val, (kmp_uint64 *)&(loop_bounds[0]), 1778 (kmp_uint64 *)&(loop_bounds[1]), (kmp_int64)step, nogroup, 1779 sched, (kmp_uint64)num_tasks, (void *)task_dup); 1780 } 1781 1782 // 4 byte version of GOMP_doacross_post 1783 // This verison needs to create a temporary array which converts 4 byte 1784 // integers into 8 byte integers 1785 template <typename T, bool need_conversion = (sizeof(long) == 4)> 1786 void __kmp_GOMP_doacross_post(T *count); 1787 1788 template <> void __kmp_GOMP_doacross_post<long, true>(long *count) { 1789 int gtid = __kmp_entry_gtid(); 1790 kmp_info_t *th = __kmp_threads[gtid]; 1791 MKLOC(loc, "GOMP_doacross_post"); 1792 kmp_int64 num_dims = th->th.th_dispatch->th_doacross_info[0]; 1793 kmp_int64 *vec = 1794 (kmp_int64 *)__kmp_thread_malloc(th, sizeof(kmp_int64) * num_dims); 1795 for (kmp_int64 i = 0; i < num_dims; ++i) { 1796 vec[i] = (kmp_int64)count[i]; 1797 } 1798 __kmpc_doacross_post(&loc, gtid, vec); 1799 __kmp_thread_free(th, vec); 1800 } 1801 1802 // 8 byte versions of GOMP_doacross_post 1803 // This version can just pass in the count array directly instead of creating 1804 // a temporary array 1805 template <> void __kmp_GOMP_doacross_post<long, false>(long *count) { 1806 int gtid = __kmp_entry_gtid(); 1807 MKLOC(loc, "GOMP_doacross_post"); 1808 __kmpc_doacross_post(&loc, gtid, RCAST(kmp_int64 *, count)); 1809 } 1810 1811 template <typename T> void __kmp_GOMP_doacross_wait(T first, va_list args) { 1812 int gtid = __kmp_entry_gtid(); 1813 kmp_info_t *th = __kmp_threads[gtid]; 1814 MKLOC(loc, "GOMP_doacross_wait"); 1815 kmp_int64 num_dims = th->th.th_dispatch->th_doacross_info[0]; 1816 kmp_int64 *vec = 1817 (kmp_int64 *)__kmp_thread_malloc(th, sizeof(kmp_int64) * num_dims); 1818 vec[0] = (kmp_int64)first; 1819 for (kmp_int64 i = 1; i < num_dims; ++i) { 1820 T item = va_arg(args, T); 1821 vec[i] = (kmp_int64)item; 1822 } 1823 __kmpc_doacross_wait(&loc, gtid, vec); 1824 __kmp_thread_free(th, vec); 1825 return; 1826 } 1827 1828 #ifdef __cplusplus 1829 extern "C" { 1830 #endif // __cplusplus 1831 1832 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKLOOP)( 1833 void (*func)(void *), void *data, void (*copy_func)(void *, void *), 1834 long arg_size, long arg_align, unsigned gomp_flags, unsigned long num_tasks, 1835 int priority, long start, long end, long step) { 1836 __GOMP_taskloop<long>(func, data, copy_func, arg_size, arg_align, gomp_flags, 1837 num_tasks, priority, start, end, step); 1838 } 1839 1840 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKLOOP_ULL)( 1841 void (*func)(void *), void *data, void (*copy_func)(void *, void *), 1842 long arg_size, long arg_align, unsigned gomp_flags, unsigned long num_tasks, 1843 int priority, unsigned long long start, unsigned long long end, 1844 unsigned long long step) { 1845 __GOMP_taskloop<unsigned long long>(func, data, copy_func, arg_size, 1846 arg_align, gomp_flags, num_tasks, 1847 priority, start, end, step); 1848 } 1849 1850 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_DOACROSS_POST)(long *count) { 1851 __kmp_GOMP_doacross_post(count); 1852 } 1853 1854 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_DOACROSS_WAIT)(long first, ...) { 1855 va_list args; 1856 va_start(args, first); 1857 __kmp_GOMP_doacross_wait<long>(first, args); 1858 va_end(args); 1859 } 1860 1861 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_DOACROSS_ULL_POST)( 1862 unsigned long long *count) { 1863 int gtid = __kmp_entry_gtid(); 1864 MKLOC(loc, "GOMP_doacross_ull_post"); 1865 __kmpc_doacross_post(&loc, gtid, RCAST(kmp_int64 *, count)); 1866 } 1867 1868 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_DOACROSS_ULL_WAIT)( 1869 unsigned long long first, ...) { 1870 va_list args; 1871 va_start(args, first); 1872 __kmp_GOMP_doacross_wait<unsigned long long>(first, args); 1873 va_end(args); 1874 } 1875 1876 // fn: the function each master thread of new team will call 1877 // data: argument to fn 1878 // num_teams, thread_limit: max bounds on respective ICV 1879 // flags: unused 1880 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TEAMS_REG)(void (*fn)(void *), 1881 void *data, 1882 unsigned num_teams, 1883 unsigned thread_limit, 1884 unsigned flags) { 1885 MKLOC(loc, "GOMP_teams_reg"); 1886 int gtid = __kmp_entry_gtid(); 1887 KA_TRACE(20, ("GOMP_teams_reg: T#%d num_teams=%u thread_limit=%u flag=%u\n", 1888 gtid, num_teams, thread_limit, flags)); 1889 __kmpc_push_num_teams(&loc, gtid, num_teams, thread_limit); 1890 __kmpc_fork_teams(&loc, 2, (microtask_t)__kmp_GOMP_microtask_wrapper, fn, 1891 data); 1892 KA_TRACE(20, ("GOMP_teams_reg exit: T#%d\n", gtid)); 1893 } 1894 1895 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKWAIT_DEPEND)(void **depend) { 1896 MKLOC(loc, "GOMP_taskwait_depend"); 1897 int gtid = __kmp_entry_gtid(); 1898 KA_TRACE(20, ("GOMP_taskwait_depend: T#%d\n", gtid)); 1899 kmp_gomp_depends_info_t gomp_depends(depend); 1900 kmp_int32 ndeps = gomp_depends.get_num_deps(); 1901 kmp_depend_info_t dep_list[ndeps]; 1902 for (kmp_int32 i = 0; i < ndeps; i++) 1903 dep_list[i] = gomp_depends.get_kmp_depend(i); 1904 #if OMPT_SUPPORT 1905 OMPT_STORE_RETURN_ADDRESS(gtid); 1906 #endif 1907 __kmpc_omp_wait_deps(&loc, gtid, ndeps, dep_list, 0, NULL); 1908 KA_TRACE(20, ("GOMP_taskwait_depend exit: T#%d\n", gtid)); 1909 } 1910 1911 /* The following sections of code create aliases for the GOMP_* functions, then 1912 create versioned symbols using the assembler directive .symver. This is only 1913 pertinent for ELF .so library. The KMP_VERSION_SYMBOL macro is defined in 1914 kmp_os.h */ 1915 1916 #ifdef KMP_USE_VERSION_SYMBOLS 1917 // GOMP_1.0 versioned symbols 1918 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_ATOMIC_END, 10, "GOMP_1.0"); 1919 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_ATOMIC_START, 10, "GOMP_1.0"); 1920 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_BARRIER, 10, "GOMP_1.0"); 1921 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CRITICAL_END, 10, "GOMP_1.0"); 1922 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CRITICAL_NAME_END, 10, "GOMP_1.0"); 1923 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CRITICAL_NAME_START, 10, "GOMP_1.0"); 1924 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CRITICAL_START, 10, "GOMP_1.0"); 1925 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DYNAMIC_NEXT, 10, "GOMP_1.0"); 1926 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DYNAMIC_START, 10, "GOMP_1.0"); 1927 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_END, 10, "GOMP_1.0"); 1928 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_END_NOWAIT, 10, "GOMP_1.0"); 1929 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_GUIDED_NEXT, 10, "GOMP_1.0"); 1930 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_GUIDED_START, 10, "GOMP_1.0"); 1931 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_NEXT, 10, "GOMP_1.0"); 1932 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_START, 10, 1933 "GOMP_1.0"); 1934 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_NEXT, 10, "GOMP_1.0"); 1935 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_START, 10, "GOMP_1.0"); 1936 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_NEXT, 10, "GOMP_1.0"); 1937 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_START, 10, 1938 "GOMP_1.0"); 1939 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_NEXT, 10, "GOMP_1.0"); 1940 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_START, 10, "GOMP_1.0"); 1941 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_RUNTIME_NEXT, 10, "GOMP_1.0"); 1942 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_RUNTIME_START, 10, "GOMP_1.0"); 1943 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_STATIC_NEXT, 10, "GOMP_1.0"); 1944 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_STATIC_START, 10, "GOMP_1.0"); 1945 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_ORDERED_END, 10, "GOMP_1.0"); 1946 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_ORDERED_START, 10, "GOMP_1.0"); 1947 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_END, 10, "GOMP_1.0"); 1948 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC_START, 10, 1949 "GOMP_1.0"); 1950 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED_START, 10, 1951 "GOMP_1.0"); 1952 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME_START, 10, 1953 "GOMP_1.0"); 1954 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC_START, 10, 1955 "GOMP_1.0"); 1956 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_SECTIONS_START, 10, "GOMP_1.0"); 1957 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_START, 10, "GOMP_1.0"); 1958 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SECTIONS_END, 10, "GOMP_1.0"); 1959 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SECTIONS_END_NOWAIT, 10, "GOMP_1.0"); 1960 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SECTIONS_NEXT, 10, "GOMP_1.0"); 1961 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SECTIONS_START, 10, "GOMP_1.0"); 1962 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SINGLE_COPY_END, 10, "GOMP_1.0"); 1963 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SINGLE_COPY_START, 10, "GOMP_1.0"); 1964 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SINGLE_START, 10, "GOMP_1.0"); 1965 1966 // GOMP_2.0 versioned symbols 1967 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASK, 20, "GOMP_2.0"); 1968 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKWAIT, 20, "GOMP_2.0"); 1969 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_NEXT, 20, "GOMP_2.0"); 1970 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_START, 20, "GOMP_2.0"); 1971 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_NEXT, 20, "GOMP_2.0"); 1972 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_START, 20, "GOMP_2.0"); 1973 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_NEXT, 20, 1974 "GOMP_2.0"); 1975 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_START, 20, 1976 "GOMP_2.0"); 1977 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_NEXT, 20, 1978 "GOMP_2.0"); 1979 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_START, 20, 1980 "GOMP_2.0"); 1981 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_NEXT, 20, 1982 "GOMP_2.0"); 1983 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_START, 20, 1984 "GOMP_2.0"); 1985 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_NEXT, 20, 1986 "GOMP_2.0"); 1987 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_START, 20, 1988 "GOMP_2.0"); 1989 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_NEXT, 20, "GOMP_2.0"); 1990 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_START, 20, "GOMP_2.0"); 1991 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_NEXT, 20, "GOMP_2.0"); 1992 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_START, 20, "GOMP_2.0"); 1993 1994 // GOMP_3.0 versioned symbols 1995 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKYIELD, 30, "GOMP_3.0"); 1996 1997 // GOMP_4.0 versioned symbols 1998 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL, 40, "GOMP_4.0"); 1999 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_SECTIONS, 40, "GOMP_4.0"); 2000 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC, 40, "GOMP_4.0"); 2001 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED, 40, "GOMP_4.0"); 2002 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME, 40, "GOMP_4.0"); 2003 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC, 40, "GOMP_4.0"); 2004 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKGROUP_START, 40, "GOMP_4.0"); 2005 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKGROUP_END, 40, "GOMP_4.0"); 2006 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_BARRIER_CANCEL, 40, "GOMP_4.0"); 2007 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CANCEL, 40, "GOMP_4.0"); 2008 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CANCELLATION_POINT, 40, "GOMP_4.0"); 2009 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_END_CANCEL, 40, "GOMP_4.0"); 2010 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SECTIONS_END_CANCEL, 40, "GOMP_4.0"); 2011 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TARGET, 40, "GOMP_4.0"); 2012 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TARGET_DATA, 40, "GOMP_4.0"); 2013 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TARGET_END_DATA, 40, "GOMP_4.0"); 2014 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TARGET_UPDATE, 40, "GOMP_4.0"); 2015 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TEAMS, 40, "GOMP_4.0"); 2016 2017 // GOMP_4.5 versioned symbols 2018 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKLOOP, 45, "GOMP_4.5"); 2019 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKLOOP_ULL, 45, "GOMP_4.5"); 2020 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_DOACROSS_POST, 45, "GOMP_4.5"); 2021 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_DOACROSS_WAIT, 45, "GOMP_4.5"); 2022 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DOACROSS_STATIC_START, 45, 2023 "GOMP_4.5"); 2024 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DOACROSS_DYNAMIC_START, 45, 2025 "GOMP_4.5"); 2026 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DOACROSS_GUIDED_START, 45, 2027 "GOMP_4.5"); 2028 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DOACROSS_RUNTIME_START, 45, 2029 "GOMP_4.5"); 2030 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_DOACROSS_ULL_POST, 45, "GOMP_4.5"); 2031 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_DOACROSS_ULL_WAIT, 45, "GOMP_4.5"); 2032 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_STATIC_START, 45, 2033 "GOMP_4.5"); 2034 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_DYNAMIC_START, 45, 2035 "GOMP_4.5"); 2036 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_GUIDED_START, 45, 2037 "GOMP_4.5"); 2038 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_RUNTIME_START, 45, 2039 "GOMP_4.5"); 2040 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_DYNAMIC_START, 45, 2041 "GOMP_4.5"); 2042 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_DYNAMIC_NEXT, 45, 2043 "GOMP_4.5"); 2044 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_GUIDED_START, 45, 2045 "GOMP_4.5"); 2046 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_GUIDED_NEXT, 45, 2047 "GOMP_4.5"); 2048 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_DYNAMIC_START, 45, 2049 "GOMP_4.5"); 2050 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_DYNAMIC_NEXT, 45, 2051 "GOMP_4.5"); 2052 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_GUIDED_START, 45, 2053 "GOMP_4.5"); 2054 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_GUIDED_NEXT, 45, 2055 "GOMP_4.5"); 2056 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_DYNAMIC, 45, 2057 "GOMP_4.5"); 2058 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_GUIDED, 45, 2059 "GOMP_4.5"); 2060 2061 // GOMP_5.0 versioned symbols 2062 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_MAYBE_NONMONOTONIC_RUNTIME_NEXT, 50, 2063 "GOMP_5.0"); 2064 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_MAYBE_NONMONOTONIC_RUNTIME_START, 50, 2065 "GOMP_5.0"); 2066 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_RUNTIME_NEXT, 50, 2067 "GOMP_5.0"); 2068 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_RUNTIME_START, 50, 2069 "GOMP_5.0"); 2070 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_MAYBE_NONMONOTONIC_RUNTIME_NEXT, 2071 50, "GOMP_5.0"); 2072 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_MAYBE_NONMONOTONIC_RUNTIME_START, 2073 50, "GOMP_5.0"); 2074 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_RUNTIME_NEXT, 50, 2075 "GOMP_5.0"); 2076 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_RUNTIME_START, 50, 2077 "GOMP_5.0"); 2078 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_RUNTIME, 50, 2079 "GOMP_5.0"); 2080 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_MAYBE_NONMONOTONIC_RUNTIME, 2081 50, "GOMP_5.0"); 2082 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TEAMS_REG, 50, "GOMP_5.0"); 2083 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKWAIT_DEPEND, 50, "GOMP_5.0"); 2084 2085 #endif // KMP_USE_VERSION_SYMBOLS 2086 2087 #ifdef __cplusplus 2088 } // extern "C" 2089 #endif // __cplusplus 2090