1 /* 2 * kmp_dispatch.cpp: dynamic scheduling - iteration initialization and dispatch. 3 */ 4 5 6 //===----------------------------------------------------------------------===// 7 // 8 // The LLVM Compiler Infrastructure 9 // 10 // This file is dual licensed under the MIT and the University of Illinois Open 11 // Source Licenses. See LICENSE.txt for details. 12 // 13 //===----------------------------------------------------------------------===// 14 15 16 /* 17 * Dynamic scheduling initialization and dispatch. 18 * 19 * NOTE: __kmp_nth is a constant inside of any dispatch loop, however 20 * it may change values between parallel regions. __kmp_max_nth 21 * is the largest value __kmp_nth may take, 1 is the smallest. 22 * 23 */ 24 25 /* ------------------------------------------------------------------------ */ 26 /* ------------------------------------------------------------------------ */ 27 28 #include "kmp.h" 29 #include "kmp_i18n.h" 30 #include "kmp_itt.h" 31 #include "kmp_str.h" 32 #include "kmp_error.h" 33 #include "kmp_stats.h" 34 #if KMP_OS_WINDOWS && KMP_ARCH_X86 35 #include <float.h> 36 #endif 37 38 #if OMPT_SUPPORT 39 #include "ompt-internal.h" 40 #include "ompt-specific.h" 41 #endif 42 43 /* ------------------------------------------------------------------------ */ 44 /* ------------------------------------------------------------------------ */ 45 46 // template for type limits 47 template< typename T > 48 struct i_maxmin { 49 static const T mx; 50 static const T mn; 51 }; 52 template<> 53 struct i_maxmin< int > { 54 static const int mx = 0x7fffffff; 55 static const int mn = 0x80000000; 56 }; 57 template<> 58 struct i_maxmin< unsigned int > { 59 static const unsigned int mx = 0xffffffff; 60 static const unsigned int mn = 0x00000000; 61 }; 62 template<> 63 struct i_maxmin< long long > { 64 static const long long mx = 0x7fffffffffffffffLL; 65 static const long long mn = 0x8000000000000000LL; 66 }; 67 template<> 68 struct i_maxmin< unsigned long long > { 69 static const unsigned long long mx = 0xffffffffffffffffLL; 70 static const unsigned long long mn = 0x0000000000000000LL; 71 }; 72 //------------------------------------------------------------------------- 73 74 #ifdef KMP_STATIC_STEAL_ENABLED 75 76 // replaces dispatch_private_info{32,64} structures and dispatch_private_info{32,64}_t types 77 template< typename T > 78 struct dispatch_private_infoXX_template { 79 typedef typename traits_t< T >::unsigned_t UT; 80 typedef typename traits_t< T >::signed_t ST; 81 UT count; // unsigned 82 T ub; 83 /* Adding KMP_ALIGN_CACHE here doesn't help / can hurt performance */ 84 T lb; 85 ST st; // signed 86 UT tc; // unsigned 87 T static_steal_counter; // for static_steal only; maybe better to put after ub 88 89 /* parm[1-4] are used in different ways by different scheduling algorithms */ 90 91 // KMP_ALIGN( 32 ) ensures ( if the KMP_ALIGN macro is turned on ) 92 // a) parm3 is properly aligned and 93 // b) all parm1-4 are in the same cache line. 94 // Because of parm1-4 are used together, performance seems to be better 95 // if they are in the same line (not measured though). 96 97 struct KMP_ALIGN( 32 ) { // compiler does not accept sizeof(T)*4 98 T parm1; 99 T parm2; 100 T parm3; 101 T parm4; 102 }; 103 104 UT ordered_lower; // unsigned 105 UT ordered_upper; // unsigned 106 #if KMP_OS_WINDOWS 107 T last_upper; 108 #endif /* KMP_OS_WINDOWS */ 109 }; 110 111 #else /* KMP_STATIC_STEAL_ENABLED */ 112 113 // replaces dispatch_private_info{32,64} structures and dispatch_private_info{32,64}_t types 114 template< typename T > 115 struct dispatch_private_infoXX_template { 116 typedef typename traits_t< T >::unsigned_t UT; 117 typedef typename traits_t< T >::signed_t ST; 118 T lb; 119 T ub; 120 ST st; // signed 121 UT tc; // unsigned 122 123 T parm1; 124 T parm2; 125 T parm3; 126 T parm4; 127 128 UT count; // unsigned 129 130 UT ordered_lower; // unsigned 131 UT ordered_upper; // unsigned 132 #if KMP_OS_WINDOWS 133 T last_upper; 134 #endif /* KMP_OS_WINDOWS */ 135 }; 136 137 #endif /* KMP_STATIC_STEAL_ENABLED */ 138 139 // replaces dispatch_private_info structure and dispatch_private_info_t type 140 template< typename T > 141 struct KMP_ALIGN_CACHE dispatch_private_info_template { 142 // duplicate alignment here, otherwise size of structure is not correct in our compiler 143 union KMP_ALIGN_CACHE private_info_tmpl { 144 dispatch_private_infoXX_template< T > p; 145 dispatch_private_info64_t p64; 146 } u; 147 enum sched_type schedule; /* scheduling algorithm */ 148 kmp_uint32 ordered; /* ordered clause specified */ 149 kmp_uint32 ordered_bumped; 150 kmp_int32 ordered_dummy[KMP_MAX_ORDERED-3]; // to retain the structure size after making order 151 dispatch_private_info * next; /* stack of buffers for nest of serial regions */ 152 kmp_uint32 nomerge; /* don't merge iters if serialized */ 153 kmp_uint32 type_size; 154 enum cons_type pushed_ws; 155 }; 156 157 158 // replaces dispatch_shared_info{32,64} structures and dispatch_shared_info{32,64}_t types 159 template< typename UT > 160 struct dispatch_shared_infoXX_template { 161 /* chunk index under dynamic, number of idle threads under static-steal; 162 iteration index otherwise */ 163 volatile UT iteration; 164 volatile UT num_done; 165 volatile UT ordered_iteration; 166 UT ordered_dummy[KMP_MAX_ORDERED-1]; // to retain the structure size making ordered_iteration scalar 167 }; 168 169 // replaces dispatch_shared_info structure and dispatch_shared_info_t type 170 template< typename UT > 171 struct dispatch_shared_info_template { 172 // we need union here to keep the structure size 173 union shared_info_tmpl { 174 dispatch_shared_infoXX_template< UT > s; 175 dispatch_shared_info64_t s64; 176 } u; 177 volatile kmp_uint32 buffer_index; 178 }; 179 180 /* ------------------------------------------------------------------------ */ 181 /* ------------------------------------------------------------------------ */ 182 183 #undef USE_TEST_LOCKS 184 185 // test_then_add template (general template should NOT be used) 186 template< typename T > 187 static __forceinline T 188 test_then_add( volatile T *p, T d ) { KMP_ASSERT(0); }; 189 190 template<> 191 __forceinline kmp_int32 192 test_then_add< kmp_int32 >( volatile kmp_int32 *p, kmp_int32 d ) 193 { 194 kmp_int32 r; 195 r = KMP_TEST_THEN_ADD32( p, d ); 196 return r; 197 } 198 199 template<> 200 __forceinline kmp_int64 201 test_then_add< kmp_int64 >( volatile kmp_int64 *p, kmp_int64 d ) 202 { 203 kmp_int64 r; 204 r = KMP_TEST_THEN_ADD64( p, d ); 205 return r; 206 } 207 208 // test_then_inc_acq template (general template should NOT be used) 209 template< typename T > 210 static __forceinline T 211 test_then_inc_acq( volatile T *p ) { KMP_ASSERT(0); }; 212 213 template<> 214 __forceinline kmp_int32 215 test_then_inc_acq< kmp_int32 >( volatile kmp_int32 *p ) 216 { 217 kmp_int32 r; 218 r = KMP_TEST_THEN_INC_ACQ32( p ); 219 return r; 220 } 221 222 template<> 223 __forceinline kmp_int64 224 test_then_inc_acq< kmp_int64 >( volatile kmp_int64 *p ) 225 { 226 kmp_int64 r; 227 r = KMP_TEST_THEN_INC_ACQ64( p ); 228 return r; 229 } 230 231 // test_then_inc template (general template should NOT be used) 232 template< typename T > 233 static __forceinline T 234 test_then_inc( volatile T *p ) { KMP_ASSERT(0); }; 235 236 template<> 237 __forceinline kmp_int32 238 test_then_inc< kmp_int32 >( volatile kmp_int32 *p ) 239 { 240 kmp_int32 r; 241 r = KMP_TEST_THEN_INC32( p ); 242 return r; 243 } 244 245 template<> 246 __forceinline kmp_int64 247 test_then_inc< kmp_int64 >( volatile kmp_int64 *p ) 248 { 249 kmp_int64 r; 250 r = KMP_TEST_THEN_INC64( p ); 251 return r; 252 } 253 254 // compare_and_swap template (general template should NOT be used) 255 template< typename T > 256 static __forceinline kmp_int32 257 compare_and_swap( volatile T *p, T c, T s ) { KMP_ASSERT(0); }; 258 259 template<> 260 __forceinline kmp_int32 261 compare_and_swap< kmp_int32 >( volatile kmp_int32 *p, kmp_int32 c, kmp_int32 s ) 262 { 263 return KMP_COMPARE_AND_STORE_REL32( p, c, s ); 264 } 265 266 template<> 267 __forceinline kmp_int32 268 compare_and_swap< kmp_int64 >( volatile kmp_int64 *p, kmp_int64 c, kmp_int64 s ) 269 { 270 return KMP_COMPARE_AND_STORE_REL64( p, c, s ); 271 } 272 273 /* 274 Spin wait loop that first does pause, then yield. 275 Waits until function returns non-zero when called with *spinner and check. 276 Does NOT put threads to sleep. 277 #if USE_ITT_BUILD 278 Arguments: 279 obj -- is higher-level synchronization object to report to ittnotify. It is used to report 280 locks consistently. For example, if lock is acquired immediately, its address is 281 reported to ittnotify via KMP_FSYNC_ACQUIRED(). However, it lock cannot be acquired 282 immediately and lock routine calls to KMP_WAIT_YIELD(), the later should report the same 283 address, not an address of low-level spinner. 284 #endif // USE_ITT_BUILD 285 */ 286 template< typename UT > 287 // ToDo: make inline function (move to header file for icl) 288 static UT // unsigned 4- or 8-byte type 289 __kmp_wait_yield( volatile UT * spinner, 290 UT checker, 291 kmp_uint32 (* pred)( UT, UT ) 292 USE_ITT_BUILD_ARG(void * obj) // Higher-level synchronization object, or NULL. 293 ) 294 { 295 // note: we may not belong to a team at this point 296 register volatile UT * spin = spinner; 297 register UT check = checker; 298 register kmp_uint32 spins; 299 register kmp_uint32 (*f) ( UT, UT ) = pred; 300 register UT r; 301 302 KMP_FSYNC_SPIN_INIT( obj, (void*) spin ); 303 KMP_INIT_YIELD( spins ); 304 // main wait spin loop 305 while(!f(r = *spin, check)) 306 { 307 KMP_FSYNC_SPIN_PREPARE( obj ); 308 /* GEH - remove this since it was accidentally introduced when kmp_wait was split. 309 It causes problems with infinite recursion because of exit lock */ 310 /* if ( TCR_4(__kmp_global.g.g_done) && __kmp_global.g.g_abort) 311 __kmp_abort_thread(); */ 312 313 // if we are oversubscribed, 314 // or have waited a bit (and KMP_LIBRARY=throughput, then yield 315 // pause is in the following code 316 KMP_YIELD( TCR_4(__kmp_nth) > __kmp_avail_proc ); 317 KMP_YIELD_SPIN( spins ); 318 } 319 KMP_FSYNC_SPIN_ACQUIRED( obj ); 320 return r; 321 } 322 323 template< typename UT > 324 static kmp_uint32 __kmp_eq( UT value, UT checker) { 325 return value == checker; 326 } 327 328 template< typename UT > 329 static kmp_uint32 __kmp_neq( UT value, UT checker) { 330 return value != checker; 331 } 332 333 template< typename UT > 334 static kmp_uint32 __kmp_lt( UT value, UT checker) { 335 return value < checker; 336 } 337 338 template< typename UT > 339 static kmp_uint32 __kmp_ge( UT value, UT checker) { 340 return value >= checker; 341 } 342 343 template< typename UT > 344 static kmp_uint32 __kmp_le( UT value, UT checker) { 345 return value <= checker; 346 } 347 348 349 /* ------------------------------------------------------------------------ */ 350 /* ------------------------------------------------------------------------ */ 351 352 static void 353 __kmp_dispatch_deo_error( int *gtid_ref, int *cid_ref, ident_t *loc_ref ) 354 { 355 kmp_info_t *th; 356 357 KMP_DEBUG_ASSERT( gtid_ref ); 358 359 if ( __kmp_env_consistency_check ) { 360 th = __kmp_threads[*gtid_ref]; 361 if ( th -> th.th_root -> r.r_active 362 && ( th -> th.th_dispatch -> th_dispatch_pr_current -> pushed_ws != ct_none ) ) { 363 #if KMP_USE_DYNAMIC_LOCK 364 __kmp_push_sync( *gtid_ref, ct_ordered_in_pdo, loc_ref, NULL, 0 ); 365 #else 366 __kmp_push_sync( *gtid_ref, ct_ordered_in_pdo, loc_ref, NULL ); 367 #endif 368 } 369 } 370 } 371 372 template< typename UT > 373 static void 374 __kmp_dispatch_deo( int *gtid_ref, int *cid_ref, ident_t *loc_ref ) 375 { 376 typedef typename traits_t< UT >::signed_t ST; 377 dispatch_private_info_template< UT > * pr; 378 379 int gtid = *gtid_ref; 380 // int cid = *cid_ref; 381 kmp_info_t *th = __kmp_threads[ gtid ]; 382 KMP_DEBUG_ASSERT( th -> th.th_dispatch ); 383 384 KD_TRACE(100, ("__kmp_dispatch_deo: T#%d called\n", gtid ) ); 385 if ( __kmp_env_consistency_check ) { 386 pr = reinterpret_cast< dispatch_private_info_template< UT >* > 387 ( th -> th.th_dispatch -> th_dispatch_pr_current ); 388 if ( pr -> pushed_ws != ct_none ) { 389 #if KMP_USE_DYNAMIC_LOCK 390 __kmp_push_sync( gtid, ct_ordered_in_pdo, loc_ref, NULL, 0 ); 391 #else 392 __kmp_push_sync( gtid, ct_ordered_in_pdo, loc_ref, NULL ); 393 #endif 394 } 395 } 396 397 if ( ! th -> th.th_team -> t.t_serialized ) { 398 dispatch_shared_info_template< UT > * sh = reinterpret_cast< dispatch_shared_info_template< UT >* > 399 ( th -> th.th_dispatch -> th_dispatch_sh_current ); 400 UT lower; 401 402 if ( ! __kmp_env_consistency_check ) { 403 pr = reinterpret_cast< dispatch_private_info_template< UT >* > 404 ( th -> th.th_dispatch -> th_dispatch_pr_current ); 405 } 406 lower = pr->u.p.ordered_lower; 407 408 #if ! defined( KMP_GOMP_COMPAT ) 409 if ( __kmp_env_consistency_check ) { 410 if ( pr->ordered_bumped ) { 411 struct cons_header *p = __kmp_threads[ gtid ]->th.th_cons; 412 __kmp_error_construct2( 413 kmp_i18n_msg_CnsMultipleNesting, 414 ct_ordered_in_pdo, loc_ref, 415 & p->stack_data[ p->w_top ] 416 ); 417 } 418 } 419 #endif /* !defined(KMP_GOMP_COMPAT) */ 420 421 KMP_MB(); 422 #ifdef KMP_DEBUG 423 { 424 const char * buff; 425 // create format specifiers before the debug output 426 buff = __kmp_str_format( 427 "__kmp_dispatch_deo: T#%%d before wait: ordered_iter:%%%s lower:%%%s\n", 428 traits_t< UT >::spec, traits_t< UT >::spec ); 429 KD_TRACE(1000, ( buff, gtid, sh->u.s.ordered_iteration, lower ) ); 430 __kmp_str_free( &buff ); 431 } 432 #endif 433 434 __kmp_wait_yield< UT >( &sh->u.s.ordered_iteration, lower, __kmp_ge< UT > 435 USE_ITT_BUILD_ARG( NULL ) 436 ); 437 KMP_MB(); /* is this necessary? */ 438 #ifdef KMP_DEBUG 439 { 440 const char * buff; 441 // create format specifiers before the debug output 442 buff = __kmp_str_format( 443 "__kmp_dispatch_deo: T#%%d after wait: ordered_iter:%%%s lower:%%%s\n", 444 traits_t< UT >::spec, traits_t< UT >::spec ); 445 KD_TRACE(1000, ( buff, gtid, sh->u.s.ordered_iteration, lower ) ); 446 __kmp_str_free( &buff ); 447 } 448 #endif 449 } 450 KD_TRACE(100, ("__kmp_dispatch_deo: T#%d returned\n", gtid ) ); 451 } 452 453 static void 454 __kmp_dispatch_dxo_error( int *gtid_ref, int *cid_ref, ident_t *loc_ref ) 455 { 456 kmp_info_t *th; 457 458 if ( __kmp_env_consistency_check ) { 459 th = __kmp_threads[*gtid_ref]; 460 if ( th -> th.th_dispatch -> th_dispatch_pr_current -> pushed_ws != ct_none ) { 461 __kmp_pop_sync( *gtid_ref, ct_ordered_in_pdo, loc_ref ); 462 } 463 } 464 } 465 466 template< typename UT > 467 static void 468 __kmp_dispatch_dxo( int *gtid_ref, int *cid_ref, ident_t *loc_ref ) 469 { 470 typedef typename traits_t< UT >::signed_t ST; 471 dispatch_private_info_template< UT > * pr; 472 473 int gtid = *gtid_ref; 474 // int cid = *cid_ref; 475 kmp_info_t *th = __kmp_threads[ gtid ]; 476 KMP_DEBUG_ASSERT( th -> th.th_dispatch ); 477 478 KD_TRACE(100, ("__kmp_dispatch_dxo: T#%d called\n", gtid ) ); 479 if ( __kmp_env_consistency_check ) { 480 pr = reinterpret_cast< dispatch_private_info_template< UT >* > 481 ( th -> th.th_dispatch -> th_dispatch_pr_current ); 482 if ( pr -> pushed_ws != ct_none ) { 483 __kmp_pop_sync( gtid, ct_ordered_in_pdo, loc_ref ); 484 } 485 } 486 487 if ( ! th -> th.th_team -> t.t_serialized ) { 488 dispatch_shared_info_template< UT > * sh = reinterpret_cast< dispatch_shared_info_template< UT >* > 489 ( th -> th.th_dispatch -> th_dispatch_sh_current ); 490 491 if ( ! __kmp_env_consistency_check ) { 492 pr = reinterpret_cast< dispatch_private_info_template< UT >* > 493 ( th -> th.th_dispatch -> th_dispatch_pr_current ); 494 } 495 496 KMP_FSYNC_RELEASING( & sh->u.s.ordered_iteration ); 497 #if ! defined( KMP_GOMP_COMPAT ) 498 if ( __kmp_env_consistency_check ) { 499 if ( pr->ordered_bumped != 0 ) { 500 struct cons_header *p = __kmp_threads[ gtid ]->th.th_cons; 501 /* How to test it? - OM */ 502 __kmp_error_construct2( 503 kmp_i18n_msg_CnsMultipleNesting, 504 ct_ordered_in_pdo, loc_ref, 505 & p->stack_data[ p->w_top ] 506 ); 507 } 508 } 509 #endif /* !defined(KMP_GOMP_COMPAT) */ 510 511 KMP_MB(); /* Flush all pending memory write invalidates. */ 512 513 pr->ordered_bumped += 1; 514 515 KD_TRACE(1000, ("__kmp_dispatch_dxo: T#%d bumping ordered ordered_bumped=%d\n", 516 gtid, pr->ordered_bumped ) ); 517 518 KMP_MB(); /* Flush all pending memory write invalidates. */ 519 520 /* TODO use general release procedure? */ 521 test_then_inc< ST >( (volatile ST *) & sh->u.s.ordered_iteration ); 522 523 KMP_MB(); /* Flush all pending memory write invalidates. */ 524 } 525 KD_TRACE(100, ("__kmp_dispatch_dxo: T#%d returned\n", gtid ) ); 526 } 527 528 /* Computes and returns x to the power of y, where y must a non-negative integer */ 529 template< typename UT > 530 static __forceinline long double 531 __kmp_pow(long double x, UT y) { 532 long double s=1.0L; 533 534 KMP_DEBUG_ASSERT(x > 0.0 && x < 1.0); 535 //KMP_DEBUG_ASSERT(y >= 0); // y is unsigned 536 while(y) { 537 if ( y & 1 ) 538 s *= x; 539 x *= x; 540 y >>= 1; 541 } 542 return s; 543 } 544 545 /* Computes and returns the number of unassigned iterations after idx chunks have been assigned 546 (the total number of unassigned iterations in chunks with index greater than or equal to idx). 547 __forceinline seems to be broken so that if we __forceinline this function, the behavior is wrong 548 (one of the unit tests, sch_guided_analytical_basic.cpp, fails) 549 */ 550 template< typename T > 551 static __inline typename traits_t< T >::unsigned_t 552 __kmp_dispatch_guided_remaining( 553 T tc, 554 typename traits_t< T >::floating_t base, 555 typename traits_t< T >::unsigned_t idx 556 ) { 557 /* Note: On Windows* OS on IA-32 architecture and Intel(R) 64, at 558 least for ICL 8.1, long double arithmetic may not really have 559 long double precision, even with /Qlong_double. Currently, we 560 workaround that in the caller code, by manipulating the FPCW for 561 Windows* OS on IA-32 architecture. The lack of precision is not 562 expected to be a correctness issue, though. 563 */ 564 typedef typename traits_t< T >::unsigned_t UT; 565 566 long double x = tc * __kmp_pow< UT >(base, idx); 567 UT r = (UT) x; 568 if ( x == r ) 569 return r; 570 return r + 1; 571 } 572 573 // Parameters of the guided-iterative algorithm: 574 // p2 = n * nproc * ( chunk + 1 ) // point of switching to dynamic 575 // p3 = 1 / ( n * nproc ) // remaining iterations multiplier 576 // by default n = 2. For example with n = 3 the chunks distribution will be more flat. 577 // With n = 1 first chunk is the same as for static schedule, e.g. trip / nproc. 578 static int guided_int_param = 2; 579 static double guided_flt_param = 0.5;// = 1.0 / guided_int_param; 580 581 // UT - unsigned flavor of T, ST - signed flavor of T, 582 // DBL - double if sizeof(T)==4, or long double if sizeof(T)==8 583 template< typename T > 584 static void 585 __kmp_dispatch_init( 586 ident_t * loc, 587 int gtid, 588 enum sched_type schedule, 589 T lb, 590 T ub, 591 typename traits_t< T >::signed_t st, 592 typename traits_t< T >::signed_t chunk, 593 int push_ws 594 ) { 595 typedef typename traits_t< T >::unsigned_t UT; 596 typedef typename traits_t< T >::signed_t ST; 597 typedef typename traits_t< T >::floating_t DBL; 598 static const int ___kmp_size_type = sizeof( UT ); 599 600 int active; 601 T tc; 602 kmp_info_t * th; 603 kmp_team_t * team; 604 kmp_uint32 my_buffer_index; 605 dispatch_private_info_template< T > * pr; 606 dispatch_shared_info_template< UT > volatile * sh; 607 608 KMP_BUILD_ASSERT( sizeof( dispatch_private_info_template< T > ) == sizeof( dispatch_private_info ) ); 609 KMP_BUILD_ASSERT( sizeof( dispatch_shared_info_template< UT > ) == sizeof( dispatch_shared_info ) ); 610 611 if ( ! TCR_4( __kmp_init_parallel ) ) 612 __kmp_parallel_initialize(); 613 614 #if INCLUDE_SSC_MARKS 615 SSC_MARK_DISPATCH_INIT(); 616 #endif 617 #ifdef KMP_DEBUG 618 { 619 const char * buff; 620 // create format specifiers before the debug output 621 buff = __kmp_str_format( 622 "__kmp_dispatch_init: T#%%d called: schedule:%%d chunk:%%%s lb:%%%s ub:%%%s st:%%%s\n", 623 traits_t< ST >::spec, traits_t< T >::spec, traits_t< T >::spec, traits_t< ST >::spec ); 624 KD_TRACE(10, ( buff, gtid, schedule, chunk, lb, ub, st ) ); 625 __kmp_str_free( &buff ); 626 } 627 #endif 628 /* setup data */ 629 th = __kmp_threads[ gtid ]; 630 team = th -> th.th_team; 631 active = ! team -> t.t_serialized; 632 th->th.th_ident = loc; 633 634 #if USE_ITT_BUILD 635 kmp_uint64 cur_chunk = chunk; 636 int itt_need_metadata_reporting = __itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 && 637 KMP_MASTER_GTID(gtid) && 638 #if OMP_40_ENABLED 639 th->th.th_teams_microtask == NULL && 640 #endif 641 team->t.t_active_level == 1; 642 #endif 643 if ( ! active ) { 644 pr = reinterpret_cast< dispatch_private_info_template< T >* > 645 ( th -> th.th_dispatch -> th_disp_buffer ); /* top of the stack */ 646 } else { 647 KMP_DEBUG_ASSERT( th->th.th_dispatch == 648 &th->th.th_team->t.t_dispatch[th->th.th_info.ds.ds_tid] ); 649 650 my_buffer_index = th->th.th_dispatch->th_disp_index ++; 651 652 /* What happens when number of threads changes, need to resize buffer? */ 653 pr = reinterpret_cast< dispatch_private_info_template< T > * > 654 ( &th -> th.th_dispatch -> th_disp_buffer[ my_buffer_index % KMP_MAX_DISP_BUF ] ); 655 sh = reinterpret_cast< dispatch_shared_info_template< UT > volatile * > 656 ( &team -> t.t_disp_buffer[ my_buffer_index % KMP_MAX_DISP_BUF ] ); 657 } 658 659 /* Currently just ignore the monotonic and non-monotonic modifiers (the compiler isn't producing them 660 * yet anyway). 661 * When it is we'll want to look at them somewhere here and use that information to add to our 662 * schedule choice. We shouldn't need to pass them on, they merely affect which schedule we can 663 * legally choose for various dynamic cases. (In paritcular, whether or not a stealing scheme is legal). 664 */ 665 schedule = SCHEDULE_WITHOUT_MODIFIERS(schedule); 666 667 /* Pick up the nomerge/ordered bits from the scheduling type */ 668 if ( (schedule >= kmp_nm_lower) && (schedule < kmp_nm_upper) ) { 669 pr->nomerge = TRUE; 670 schedule = (enum sched_type)(((int)schedule) - (kmp_nm_lower - kmp_sch_lower)); 671 } else { 672 pr->nomerge = FALSE; 673 } 674 pr->type_size = ___kmp_size_type; // remember the size of variables 675 if ( kmp_ord_lower & schedule ) { 676 pr->ordered = TRUE; 677 schedule = (enum sched_type)(((int)schedule) - (kmp_ord_lower - kmp_sch_lower)); 678 } else { 679 pr->ordered = FALSE; 680 } 681 682 if ( schedule == kmp_sch_static ) { 683 schedule = __kmp_static; 684 } else { 685 if ( schedule == kmp_sch_runtime ) { 686 // Use the scheduling specified by OMP_SCHEDULE (or __kmp_sch_default if not specified) 687 schedule = team -> t.t_sched.r_sched_type; 688 // Detail the schedule if needed (global controls are differentiated appropriately) 689 if ( schedule == kmp_sch_guided_chunked ) { 690 schedule = __kmp_guided; 691 } else if ( schedule == kmp_sch_static ) { 692 schedule = __kmp_static; 693 } 694 // Use the chunk size specified by OMP_SCHEDULE (or default if not specified) 695 chunk = team -> t.t_sched.chunk; 696 #if USE_ITT_BUILD 697 cur_chunk = chunk; 698 #endif 699 #ifdef KMP_DEBUG 700 { 701 const char * buff; 702 // create format specifiers before the debug output 703 buff = __kmp_str_format( 704 "__kmp_dispatch_init: T#%%d new: schedule:%%d chunk:%%%s\n", 705 traits_t< ST >::spec ); 706 KD_TRACE(10, ( buff, gtid, schedule, chunk ) ); 707 __kmp_str_free( &buff ); 708 } 709 #endif 710 } else { 711 if ( schedule == kmp_sch_guided_chunked ) { 712 schedule = __kmp_guided; 713 } 714 if ( chunk <= 0 ) { 715 chunk = KMP_DEFAULT_CHUNK; 716 } 717 } 718 719 if ( schedule == kmp_sch_auto ) { 720 // mapping and differentiation: in the __kmp_do_serial_initialize() 721 schedule = __kmp_auto; 722 #ifdef KMP_DEBUG 723 { 724 const char * buff; 725 // create format specifiers before the debug output 726 buff = __kmp_str_format( 727 "__kmp_dispatch_init: kmp_sch_auto: T#%%d new: schedule:%%d chunk:%%%s\n", 728 traits_t< ST >::spec ); 729 KD_TRACE(10, ( buff, gtid, schedule, chunk ) ); 730 __kmp_str_free( &buff ); 731 } 732 #endif 733 } 734 735 /* guided analytical not safe for too many threads */ 736 if ( team->t.t_nproc > 1<<20 && schedule == kmp_sch_guided_analytical_chunked ) { 737 schedule = kmp_sch_guided_iterative_chunked; 738 KMP_WARNING( DispatchManyThreads ); 739 } 740 pr->u.p.parm1 = chunk; 741 } 742 KMP_ASSERT2( (kmp_sch_lower < schedule && schedule < kmp_sch_upper), 743 "unknown scheduling type" ); 744 745 pr->u.p.count = 0; 746 747 if ( __kmp_env_consistency_check ) { 748 if ( st == 0 ) { 749 __kmp_error_construct( 750 kmp_i18n_msg_CnsLoopIncrZeroProhibited, 751 ( pr->ordered ? ct_pdo_ordered : ct_pdo ), loc 752 ); 753 } 754 } 755 756 tc = ( ub - lb + st ); 757 if ( st != 1 ) { 758 if ( st < 0 ) { 759 if ( lb < ub ) { 760 tc = 0; // zero-trip 761 } else { // lb >= ub 762 tc = (ST)tc / st; // convert to signed division 763 } 764 } else { // st > 0 765 if ( ub < lb ) { 766 tc = 0; // zero-trip 767 } else { // lb >= ub 768 tc /= st; 769 } 770 } 771 } else if ( ub < lb ) { // st == 1 772 tc = 0; // zero-trip 773 } 774 775 // Any half-decent optimizer will remove this test when the blocks are empty since the macros expand to nothing 776 // when statistics are disabled. 777 if (schedule == __kmp_static) 778 { 779 KMP_COUNT_BLOCK(OMP_FOR_static); 780 KMP_COUNT_VALUE(FOR_static_iterations, tc); 781 } 782 else 783 { 784 KMP_COUNT_BLOCK(OMP_FOR_dynamic); 785 KMP_COUNT_VALUE(FOR_dynamic_iterations, tc); 786 } 787 788 pr->u.p.lb = lb; 789 pr->u.p.ub = ub; 790 pr->u.p.st = st; 791 pr->u.p.tc = tc; 792 793 #if KMP_OS_WINDOWS 794 pr->u.p.last_upper = ub + st; 795 #endif /* KMP_OS_WINDOWS */ 796 797 /* NOTE: only the active parallel region(s) has active ordered sections */ 798 799 if ( active ) { 800 if ( pr->ordered == 0 ) { 801 th -> th.th_dispatch -> th_deo_fcn = __kmp_dispatch_deo_error; 802 th -> th.th_dispatch -> th_dxo_fcn = __kmp_dispatch_dxo_error; 803 } else { 804 pr->ordered_bumped = 0; 805 806 pr->u.p.ordered_lower = 1; 807 pr->u.p.ordered_upper = 0; 808 809 th -> th.th_dispatch -> th_deo_fcn = __kmp_dispatch_deo< UT >; 810 th -> th.th_dispatch -> th_dxo_fcn = __kmp_dispatch_dxo< UT >; 811 } 812 } 813 814 if ( __kmp_env_consistency_check ) { 815 enum cons_type ws = pr->ordered ? ct_pdo_ordered : ct_pdo; 816 if ( push_ws ) { 817 __kmp_push_workshare( gtid, ws, loc ); 818 pr->pushed_ws = ws; 819 } else { 820 __kmp_check_workshare( gtid, ws, loc ); 821 pr->pushed_ws = ct_none; 822 } 823 } 824 825 switch ( schedule ) { 826 #if ( KMP_STATIC_STEAL_ENABLED && KMP_ARCH_X86_64 ) 827 case kmp_sch_static_steal: 828 { 829 T nproc = team->t.t_nproc; 830 T ntc, init; 831 832 KD_TRACE(100, ("__kmp_dispatch_init: T#%d kmp_sch_static_steal case\n", gtid ) ); 833 834 ntc = (tc % chunk ? 1 : 0) + tc / chunk; 835 if ( nproc > 1 && ntc >= nproc ) { 836 T id = __kmp_tid_from_gtid(gtid); 837 T small_chunk, extras; 838 839 small_chunk = ntc / nproc; 840 extras = ntc % nproc; 841 842 init = id * small_chunk + ( id < extras ? id : extras ); 843 pr->u.p.count = init; 844 pr->u.p.ub = init + small_chunk + ( id < extras ? 1 : 0 ); 845 846 pr->u.p.parm2 = lb; 847 //pr->pfields.parm3 = 0; // it's not used in static_steal 848 pr->u.p.parm4 = id; 849 pr->u.p.st = st; 850 break; 851 } else { 852 KD_TRACE(100, ("__kmp_dispatch_init: T#%d falling-through to kmp_sch_static_balanced\n", 853 gtid ) ); 854 schedule = kmp_sch_static_balanced; 855 /* too few iterations: fall-through to kmp_sch_static_balanced */ 856 } // if 857 /* FALL-THROUGH to static balanced */ 858 } // case 859 #endif 860 case kmp_sch_static_balanced: 861 { 862 T nproc = team->t.t_nproc; 863 T init, limit; 864 865 KD_TRACE(100, ("__kmp_dispatch_init: T#%d kmp_sch_static_balanced case\n", 866 gtid ) ); 867 868 if ( nproc > 1 ) { 869 T id = __kmp_tid_from_gtid(gtid); 870 871 if ( tc < nproc ) { 872 if ( id < tc ) { 873 init = id; 874 limit = id; 875 pr->u.p.parm1 = (id == tc - 1); /* parm1 stores *plastiter */ 876 } else { 877 pr->u.p.count = 1; /* means no more chunks to execute */ 878 pr->u.p.parm1 = FALSE; 879 break; 880 } 881 } else { 882 T small_chunk = tc / nproc; 883 T extras = tc % nproc; 884 init = id * small_chunk + (id < extras ? id : extras); 885 limit = init + small_chunk - (id < extras ? 0 : 1); 886 pr->u.p.parm1 = (id == nproc - 1); 887 } 888 } else { 889 if ( tc > 0 ) { 890 init = 0; 891 limit = tc - 1; 892 pr->u.p.parm1 = TRUE; 893 } else { 894 // zero trip count 895 pr->u.p.count = 1; /* means no more chunks to execute */ 896 pr->u.p.parm1 = FALSE; 897 break; 898 } 899 } 900 #if USE_ITT_BUILD 901 // Calculate chunk for metadata report 902 if ( itt_need_metadata_reporting ) 903 cur_chunk = limit - init + 1; 904 #endif 905 if ( st == 1 ) { 906 pr->u.p.lb = lb + init; 907 pr->u.p.ub = lb + limit; 908 } else { 909 T ub_tmp = lb + limit * st; // calculated upper bound, "ub" is user-defined upper bound 910 pr->u.p.lb = lb + init * st; 911 // adjust upper bound to "ub" if needed, so that MS lastprivate will match it exactly 912 if ( st > 0 ) { 913 pr->u.p.ub = ( ub_tmp + st > ub ? ub : ub_tmp ); 914 } else { 915 pr->u.p.ub = ( ub_tmp + st < ub ? ub : ub_tmp ); 916 } 917 } 918 if ( pr->ordered ) { 919 pr->u.p.ordered_lower = init; 920 pr->u.p.ordered_upper = limit; 921 } 922 break; 923 } // case 924 case kmp_sch_guided_iterative_chunked : 925 { 926 T nproc = team->t.t_nproc; 927 KD_TRACE(100,("__kmp_dispatch_init: T#%d kmp_sch_guided_iterative_chunked case\n",gtid)); 928 929 if ( nproc > 1 ) { 930 if ( (2L * chunk + 1 ) * nproc >= tc ) { 931 /* chunk size too large, switch to dynamic */ 932 schedule = kmp_sch_dynamic_chunked; 933 } else { 934 // when remaining iters become less than parm2 - switch to dynamic 935 pr->u.p.parm2 = guided_int_param * nproc * ( chunk + 1 ); 936 *(double*)&pr->u.p.parm3 = guided_flt_param / nproc; // may occupy parm3 and parm4 937 } 938 } else { 939 KD_TRACE(100,("__kmp_dispatch_init: T#%d falling-through to kmp_sch_static_greedy\n",gtid)); 940 schedule = kmp_sch_static_greedy; 941 /* team->t.t_nproc == 1: fall-through to kmp_sch_static_greedy */ 942 KD_TRACE(100,("__kmp_dispatch_init: T#%d kmp_sch_static_greedy case\n",gtid)); 943 pr->u.p.parm1 = tc; 944 } // if 945 } // case 946 break; 947 case kmp_sch_guided_analytical_chunked: 948 { 949 T nproc = team->t.t_nproc; 950 KD_TRACE(100, ("__kmp_dispatch_init: T#%d kmp_sch_guided_analytical_chunked case\n", gtid)); 951 952 if ( nproc > 1 ) { 953 if ( (2L * chunk + 1 ) * nproc >= tc ) { 954 /* chunk size too large, switch to dynamic */ 955 schedule = kmp_sch_dynamic_chunked; 956 } else { 957 /* commonly used term: (2 nproc - 1)/(2 nproc) */ 958 DBL x; 959 960 #if KMP_OS_WINDOWS && KMP_ARCH_X86 961 /* Linux* OS already has 64-bit computation by default for 962 long double, and on Windows* OS on Intel(R) 64, 963 /Qlong_double doesn't work. On Windows* OS 964 on IA-32 architecture, we need to set precision to 965 64-bit instead of the default 53-bit. Even though long 966 double doesn't work on Windows* OS on Intel(R) 64, the 967 resulting lack of precision is not expected to impact 968 the correctness of the algorithm, but this has not been 969 mathematically proven. 970 */ 971 // save original FPCW and set precision to 64-bit, as 972 // Windows* OS on IA-32 architecture defaults to 53-bit 973 unsigned int oldFpcw = _control87(0,0); 974 _control87(_PC_64,_MCW_PC); // 0,0x30000 975 #endif 976 /* value used for comparison in solver for cross-over point */ 977 long double target = ((long double)chunk * 2 + 1) * nproc / tc; 978 979 /* crossover point--chunk indexes equal to or greater than 980 this point switch to dynamic-style scheduling */ 981 UT cross; 982 983 /* commonly used term: (2 nproc - 1)/(2 nproc) */ 984 x = (long double)1.0 - (long double)0.5 / nproc; 985 986 #ifdef KMP_DEBUG 987 { // test natural alignment 988 struct _test_a { 989 char a; 990 union { 991 char b; 992 DBL d; 993 }; 994 } t; 995 ptrdiff_t natural_alignment = (ptrdiff_t)&t.b - (ptrdiff_t)&t - (ptrdiff_t)1; 996 //__kmp_warn( " %llx %llx %lld", (long long)&t.d, (long long)&t, (long long)natural_alignment ); 997 KMP_DEBUG_ASSERT( ( ( (ptrdiff_t)&pr->u.p.parm3 ) & ( natural_alignment ) ) == 0 ); 998 } 999 #endif // KMP_DEBUG 1000 1001 /* save the term in thread private dispatch structure */ 1002 *(DBL*)&pr->u.p.parm3 = x; 1003 1004 /* solve for the crossover point to the nearest integer i for which C_i <= chunk */ 1005 { 1006 UT left, right, mid; 1007 long double p; 1008 1009 /* estimate initial upper and lower bound */ 1010 1011 /* doesn't matter what value right is as long as it is positive, but 1012 it affects performance of the solver 1013 */ 1014 right = 229; 1015 p = __kmp_pow< UT >(x,right); 1016 if ( p > target ) { 1017 do{ 1018 p *= p; 1019 right <<= 1; 1020 } while(p>target && right < (1<<27)); 1021 left = right >> 1; /* lower bound is previous (failed) estimate of upper bound */ 1022 } else { 1023 left = 0; 1024 } 1025 1026 /* bisection root-finding method */ 1027 while ( left + 1 < right ) { 1028 mid = (left + right) / 2; 1029 if ( __kmp_pow< UT >(x,mid) > target ) { 1030 left = mid; 1031 } else { 1032 right = mid; 1033 } 1034 } // while 1035 cross = right; 1036 } 1037 /* assert sanity of computed crossover point */ 1038 KMP_ASSERT(cross && __kmp_pow< UT >(x, cross - 1) > target && __kmp_pow< UT >(x, cross) <= target); 1039 1040 /* save the crossover point in thread private dispatch structure */ 1041 pr->u.p.parm2 = cross; 1042 1043 // C75803 1044 #if ( ( KMP_OS_LINUX || KMP_OS_WINDOWS ) && KMP_ARCH_X86 ) && ( ! defined( KMP_I8 ) ) 1045 #define GUIDED_ANALYTICAL_WORKAROUND (*( DBL * )&pr->u.p.parm3) 1046 #else 1047 #define GUIDED_ANALYTICAL_WORKAROUND (x) 1048 #endif 1049 /* dynamic-style scheduling offset */ 1050 pr->u.p.count = tc - __kmp_dispatch_guided_remaining(tc, GUIDED_ANALYTICAL_WORKAROUND, cross) - cross * chunk; 1051 #if KMP_OS_WINDOWS && KMP_ARCH_X86 1052 // restore FPCW 1053 _control87(oldFpcw,_MCW_PC); 1054 #endif 1055 } // if 1056 } else { 1057 KD_TRACE(100, ("__kmp_dispatch_init: T#%d falling-through to kmp_sch_static_greedy\n", 1058 gtid ) ); 1059 schedule = kmp_sch_static_greedy; 1060 /* team->t.t_nproc == 1: fall-through to kmp_sch_static_greedy */ 1061 pr->u.p.parm1 = tc; 1062 } // if 1063 } // case 1064 break; 1065 case kmp_sch_static_greedy: 1066 KD_TRACE(100,("__kmp_dispatch_init: T#%d kmp_sch_static_greedy case\n",gtid)); 1067 pr->u.p.parm1 = ( team -> t.t_nproc > 1 ) ? 1068 ( tc + team->t.t_nproc - 1 ) / team->t.t_nproc : 1069 tc; 1070 break; 1071 case kmp_sch_static_chunked : 1072 case kmp_sch_dynamic_chunked : 1073 if ( pr->u.p.parm1 <= 0 ) { 1074 pr->u.p.parm1 = KMP_DEFAULT_CHUNK; 1075 } 1076 KD_TRACE(100,("__kmp_dispatch_init: T#%d kmp_sch_static_chunked/kmp_sch_dynamic_chunked cases\n", gtid)); 1077 break; 1078 case kmp_sch_trapezoidal : 1079 { 1080 /* TSS: trapezoid self-scheduling, minimum chunk_size = parm1 */ 1081 1082 T parm1, parm2, parm3, parm4; 1083 KD_TRACE(100, ("__kmp_dispatch_init: T#%d kmp_sch_trapezoidal case\n", gtid ) ); 1084 1085 parm1 = chunk; 1086 1087 /* F : size of the first cycle */ 1088 parm2 = ( tc / (2 * team->t.t_nproc) ); 1089 1090 if ( parm2 < 1 ) { 1091 parm2 = 1; 1092 } 1093 1094 /* L : size of the last cycle. Make sure the last cycle 1095 * is not larger than the first cycle. 1096 */ 1097 if ( parm1 < 1 ) { 1098 parm1 = 1; 1099 } else if ( parm1 > parm2 ) { 1100 parm1 = parm2; 1101 } 1102 1103 /* N : number of cycles */ 1104 parm3 = ( parm2 + parm1 ); 1105 parm3 = ( 2 * tc + parm3 - 1) / parm3; 1106 1107 if ( parm3 < 2 ) { 1108 parm3 = 2; 1109 } 1110 1111 /* sigma : decreasing incr of the trapezoid */ 1112 parm4 = ( parm3 - 1 ); 1113 parm4 = ( parm2 - parm1 ) / parm4; 1114 1115 // pointless check, because parm4 >= 0 always 1116 //if ( parm4 < 0 ) { 1117 // parm4 = 0; 1118 //} 1119 1120 pr->u.p.parm1 = parm1; 1121 pr->u.p.parm2 = parm2; 1122 pr->u.p.parm3 = parm3; 1123 pr->u.p.parm4 = parm4; 1124 } // case 1125 break; 1126 1127 default: 1128 { 1129 __kmp_msg( 1130 kmp_ms_fatal, // Severity 1131 KMP_MSG( UnknownSchedTypeDetected ), // Primary message 1132 KMP_HNT( GetNewerLibrary ), // Hint 1133 __kmp_msg_null // Variadic argument list terminator 1134 ); 1135 } 1136 break; 1137 } // switch 1138 pr->schedule = schedule; 1139 if ( active ) { 1140 /* The name of this buffer should be my_buffer_index when it's free to use it */ 1141 1142 KD_TRACE(100, ("__kmp_dispatch_init: T#%d before wait: my_buffer_index:%d sh->buffer_index:%d\n", 1143 gtid, my_buffer_index, sh->buffer_index) ); 1144 __kmp_wait_yield< kmp_uint32 >( & sh->buffer_index, my_buffer_index, __kmp_eq< kmp_uint32 > 1145 USE_ITT_BUILD_ARG( NULL ) 1146 ); 1147 // Note: KMP_WAIT_YIELD() cannot be used there: buffer index and my_buffer_index are 1148 // *always* 32-bit integers. 1149 KMP_MB(); /* is this necessary? */ 1150 KD_TRACE(100, ("__kmp_dispatch_init: T#%d after wait: my_buffer_index:%d sh->buffer_index:%d\n", 1151 gtid, my_buffer_index, sh->buffer_index) ); 1152 1153 th -> th.th_dispatch -> th_dispatch_pr_current = (dispatch_private_info_t*) pr; 1154 th -> th.th_dispatch -> th_dispatch_sh_current = (dispatch_shared_info_t*) sh; 1155 #if USE_ITT_BUILD 1156 if ( pr->ordered ) { 1157 __kmp_itt_ordered_init( gtid ); 1158 }; // if 1159 // Report loop metadata 1160 if ( itt_need_metadata_reporting ) { 1161 // Only report metadata by master of active team at level 1 1162 kmp_uint64 schedtype = 0; 1163 switch ( schedule ) { 1164 case kmp_sch_static_chunked: 1165 case kmp_sch_static_balanced:// Chunk is calculated in the switch above 1166 break; 1167 case kmp_sch_static_greedy: 1168 cur_chunk = pr->u.p.parm1; 1169 break; 1170 case kmp_sch_dynamic_chunked: 1171 schedtype = 1; 1172 break; 1173 case kmp_sch_guided_iterative_chunked: 1174 case kmp_sch_guided_analytical_chunked: 1175 schedtype = 2; 1176 break; 1177 default: 1178 // Should we put this case under "static"? 1179 // case kmp_sch_static_steal: 1180 schedtype = 3; 1181 break; 1182 } 1183 __kmp_itt_metadata_loop(loc, schedtype, tc, cur_chunk); 1184 } 1185 #endif /* USE_ITT_BUILD */ 1186 }; // if 1187 1188 #ifdef KMP_DEBUG 1189 { 1190 const char * buff; 1191 // create format specifiers before the debug output 1192 buff = __kmp_str_format( 1193 "__kmp_dispatch_init: T#%%d returning: schedule:%%d ordered:%%%s lb:%%%s ub:%%%s" \ 1194 " st:%%%s tc:%%%s count:%%%s\n\tordered_lower:%%%s ordered_upper:%%%s" \ 1195 " parm1:%%%s parm2:%%%s parm3:%%%s parm4:%%%s\n", 1196 traits_t< UT >::spec, traits_t< T >::spec, traits_t< T >::spec, 1197 traits_t< ST >::spec, traits_t< UT >::spec, traits_t< UT >::spec, 1198 traits_t< UT >::spec, traits_t< UT >::spec, traits_t< T >::spec, 1199 traits_t< T >::spec, traits_t< T >::spec, traits_t< T >::spec ); 1200 KD_TRACE(10, ( buff, 1201 gtid, pr->schedule, pr->ordered, pr->u.p.lb, pr->u.p.ub, 1202 pr->u.p.st, pr->u.p.tc, pr->u.p.count, 1203 pr->u.p.ordered_lower, pr->u.p.ordered_upper, pr->u.p.parm1, 1204 pr->u.p.parm2, pr->u.p.parm3, pr->u.p.parm4 ) ); 1205 __kmp_str_free( &buff ); 1206 } 1207 #endif 1208 #if ( KMP_STATIC_STEAL_ENABLED ) 1209 if ( ___kmp_size_type < 8 ) { 1210 // It cannot be guaranteed that after execution of a loop with some other schedule kind 1211 // all the parm3 variables will contain the same value. 1212 // Even if all parm3 will be the same, it still exists a bad case like using 0 and 1 1213 // rather than program life-time increment. 1214 // So the dedicated variable is required. The 'static_steal_counter' is used. 1215 if( schedule == kmp_sch_static_steal ) { 1216 // Other threads will inspect this variable when searching for a victim. 1217 // This is a flag showing that other threads may steal from this thread since then. 1218 volatile T * p = &pr->u.p.static_steal_counter; 1219 *p = *p + 1; 1220 } 1221 } 1222 #endif // ( KMP_STATIC_STEAL_ENABLED && USE_STEALING ) 1223 1224 #if OMPT_SUPPORT && OMPT_TRACE 1225 if (ompt_enabled && 1226 ompt_callbacks.ompt_callback(ompt_event_loop_begin)) { 1227 ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL); 1228 ompt_task_info_t *task_info = __ompt_get_taskinfo(0); 1229 ompt_callbacks.ompt_callback(ompt_event_loop_begin)( 1230 team_info->parallel_id, task_info->task_id, team_info->microtask); 1231 } 1232 #endif 1233 } 1234 1235 /* 1236 * For ordered loops, either __kmp_dispatch_finish() should be called after 1237 * every iteration, or __kmp_dispatch_finish_chunk() should be called after 1238 * every chunk of iterations. If the ordered section(s) were not executed 1239 * for this iteration (or every iteration in this chunk), we need to set the 1240 * ordered iteration counters so that the next thread can proceed. 1241 */ 1242 template< typename UT > 1243 static void 1244 __kmp_dispatch_finish( int gtid, ident_t *loc ) 1245 { 1246 typedef typename traits_t< UT >::signed_t ST; 1247 kmp_info_t *th = __kmp_threads[ gtid ]; 1248 1249 KD_TRACE(100, ("__kmp_dispatch_finish: T#%d called\n", gtid ) ); 1250 if ( ! th -> th.th_team -> t.t_serialized ) { 1251 1252 dispatch_private_info_template< UT > * pr = 1253 reinterpret_cast< dispatch_private_info_template< UT >* > 1254 ( th->th.th_dispatch->th_dispatch_pr_current ); 1255 dispatch_shared_info_template< UT > volatile * sh = 1256 reinterpret_cast< dispatch_shared_info_template< UT >volatile* > 1257 ( th->th.th_dispatch->th_dispatch_sh_current ); 1258 KMP_DEBUG_ASSERT( pr ); 1259 KMP_DEBUG_ASSERT( sh ); 1260 KMP_DEBUG_ASSERT( th->th.th_dispatch == 1261 &th->th.th_team->t.t_dispatch[th->th.th_info.ds.ds_tid] ); 1262 1263 if ( pr->ordered_bumped ) { 1264 KD_TRACE(1000, ("__kmp_dispatch_finish: T#%d resetting ordered_bumped to zero\n", 1265 gtid ) ); 1266 pr->ordered_bumped = 0; 1267 } else { 1268 UT lower = pr->u.p.ordered_lower; 1269 1270 #ifdef KMP_DEBUG 1271 { 1272 const char * buff; 1273 // create format specifiers before the debug output 1274 buff = __kmp_str_format( 1275 "__kmp_dispatch_finish: T#%%d before wait: ordered_iteration:%%%s lower:%%%s\n", 1276 traits_t< UT >::spec, traits_t< UT >::spec ); 1277 KD_TRACE(1000, ( buff, gtid, sh->u.s.ordered_iteration, lower ) ); 1278 __kmp_str_free( &buff ); 1279 } 1280 #endif 1281 1282 __kmp_wait_yield< UT >(&sh->u.s.ordered_iteration, lower, __kmp_ge< UT > 1283 USE_ITT_BUILD_ARG(NULL) 1284 ); 1285 KMP_MB(); /* is this necessary? */ 1286 #ifdef KMP_DEBUG 1287 { 1288 const char * buff; 1289 // create format specifiers before the debug output 1290 buff = __kmp_str_format( 1291 "__kmp_dispatch_finish: T#%%d after wait: ordered_iteration:%%%s lower:%%%s\n", 1292 traits_t< UT >::spec, traits_t< UT >::spec ); 1293 KD_TRACE(1000, ( buff, gtid, sh->u.s.ordered_iteration, lower ) ); 1294 __kmp_str_free( &buff ); 1295 } 1296 #endif 1297 1298 test_then_inc< ST >( (volatile ST *) & sh->u.s.ordered_iteration ); 1299 } // if 1300 } // if 1301 KD_TRACE(100, ("__kmp_dispatch_finish: T#%d returned\n", gtid ) ); 1302 } 1303 1304 #ifdef KMP_GOMP_COMPAT 1305 1306 template< typename UT > 1307 static void 1308 __kmp_dispatch_finish_chunk( int gtid, ident_t *loc ) 1309 { 1310 typedef typename traits_t< UT >::signed_t ST; 1311 kmp_info_t *th = __kmp_threads[ gtid ]; 1312 1313 KD_TRACE(100, ("__kmp_dispatch_finish_chunk: T#%d called\n", gtid ) ); 1314 if ( ! th -> th.th_team -> t.t_serialized ) { 1315 // int cid; 1316 dispatch_private_info_template< UT > * pr = 1317 reinterpret_cast< dispatch_private_info_template< UT >* > 1318 ( th->th.th_dispatch->th_dispatch_pr_current ); 1319 dispatch_shared_info_template< UT > volatile * sh = 1320 reinterpret_cast< dispatch_shared_info_template< UT >volatile* > 1321 ( th->th.th_dispatch->th_dispatch_sh_current ); 1322 KMP_DEBUG_ASSERT( pr ); 1323 KMP_DEBUG_ASSERT( sh ); 1324 KMP_DEBUG_ASSERT( th->th.th_dispatch == 1325 &th->th.th_team->t.t_dispatch[th->th.th_info.ds.ds_tid] ); 1326 1327 // for (cid = 0; cid < KMP_MAX_ORDERED; ++cid) { 1328 UT lower = pr->u.p.ordered_lower; 1329 UT upper = pr->u.p.ordered_upper; 1330 UT inc = upper - lower + 1; 1331 1332 if ( pr->ordered_bumped == inc ) { 1333 KD_TRACE(1000, ("__kmp_dispatch_finish: T#%d resetting ordered_bumped to zero\n", 1334 gtid ) ); 1335 pr->ordered_bumped = 0; 1336 } else { 1337 inc -= pr->ordered_bumped; 1338 1339 #ifdef KMP_DEBUG 1340 { 1341 const char * buff; 1342 // create format specifiers before the debug output 1343 buff = __kmp_str_format( 1344 "__kmp_dispatch_finish_chunk: T#%%d before wait: " \ 1345 "ordered_iteration:%%%s lower:%%%s upper:%%%s\n", 1346 traits_t< UT >::spec, traits_t< UT >::spec, traits_t< UT >::spec ); 1347 KD_TRACE(1000, ( buff, gtid, sh->u.s.ordered_iteration, lower, upper ) ); 1348 __kmp_str_free( &buff ); 1349 } 1350 #endif 1351 1352 __kmp_wait_yield< UT >(&sh->u.s.ordered_iteration, lower, __kmp_ge< UT > 1353 USE_ITT_BUILD_ARG(NULL) 1354 ); 1355 1356 KMP_MB(); /* is this necessary? */ 1357 KD_TRACE(1000, ("__kmp_dispatch_finish_chunk: T#%d resetting ordered_bumped to zero\n", 1358 gtid ) ); 1359 pr->ordered_bumped = 0; 1360 //!!!!! TODO check if the inc should be unsigned, or signed??? 1361 #ifdef KMP_DEBUG 1362 { 1363 const char * buff; 1364 // create format specifiers before the debug output 1365 buff = __kmp_str_format( 1366 "__kmp_dispatch_finish_chunk: T#%%d after wait: " \ 1367 "ordered_iteration:%%%s inc:%%%s lower:%%%s upper:%%%s\n", 1368 traits_t< UT >::spec, traits_t< UT >::spec, traits_t< UT >::spec, traits_t< UT >::spec ); 1369 KD_TRACE(1000, ( buff, gtid, sh->u.s.ordered_iteration, inc, lower, upper ) ); 1370 __kmp_str_free( &buff ); 1371 } 1372 #endif 1373 1374 test_then_add< ST >( (volatile ST *) & sh->u.s.ordered_iteration, inc); 1375 } 1376 // } 1377 } 1378 KD_TRACE(100, ("__kmp_dispatch_finish_chunk: T#%d returned\n", gtid ) ); 1379 } 1380 1381 #endif /* KMP_GOMP_COMPAT */ 1382 1383 /* Define a macro for exiting __kmp_dispatch_next(). If status is 0 1384 * (no more work), then tell OMPT the loop is over. In some cases 1385 * kmp_dispatch_fini() is not called. */ 1386 #if OMPT_SUPPORT && OMPT_TRACE 1387 #define OMPT_LOOP_END \ 1388 if (status == 0) { \ 1389 if (ompt_enabled && \ 1390 ompt_callbacks.ompt_callback(ompt_event_loop_end)) { \ 1391 ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL); \ 1392 ompt_task_info_t *task_info = __ompt_get_taskinfo(0); \ 1393 ompt_callbacks.ompt_callback(ompt_event_loop_end)( \ 1394 team_info->parallel_id, task_info->task_id); \ 1395 } \ 1396 } 1397 #else 1398 #define OMPT_LOOP_END // no-op 1399 #endif 1400 1401 template< typename T > 1402 static int 1403 __kmp_dispatch_next( 1404 ident_t *loc, int gtid, kmp_int32 *p_last, T *p_lb, T *p_ub, typename traits_t< T >::signed_t *p_st 1405 ) { 1406 1407 typedef typename traits_t< T >::unsigned_t UT; 1408 typedef typename traits_t< T >::signed_t ST; 1409 typedef typename traits_t< T >::floating_t DBL; 1410 #if ( KMP_STATIC_STEAL_ENABLED && KMP_ARCH_X86_64 ) 1411 static const int ___kmp_size_type = sizeof( UT ); 1412 #endif 1413 1414 // This is potentially slightly misleading, schedule(runtime) will appear here even if the actual runtme schedule 1415 // is static. (Which points out a disadavantage of schedule(runtime): even when static scheduling is used it costs 1416 // more than a compile time choice to use static scheduling would.) 1417 KMP_TIME_BLOCK(FOR_dynamic_scheduling); 1418 1419 int status; 1420 dispatch_private_info_template< T > * pr; 1421 kmp_info_t * th = __kmp_threads[ gtid ]; 1422 kmp_team_t * team = th -> th.th_team; 1423 1424 KMP_DEBUG_ASSERT( p_lb && p_ub && p_st ); // AC: these cannot be NULL 1425 #ifdef KMP_DEBUG 1426 { 1427 const char * buff; 1428 // create format specifiers before the debug output 1429 buff = __kmp_str_format( 1430 "__kmp_dispatch_next: T#%%d called p_lb:%%%s p_ub:%%%s p_st:%%%s p_last: %%p\n", 1431 traits_t< T >::spec, traits_t< T >::spec, traits_t< ST >::spec ); 1432 KD_TRACE(1000, ( buff, gtid, *p_lb, *p_ub, p_st ? *p_st : 0, p_last ) ); 1433 __kmp_str_free( &buff ); 1434 } 1435 #endif 1436 1437 if ( team -> t.t_serialized ) { 1438 /* NOTE: serialize this dispatch becase we are not at the active level */ 1439 pr = reinterpret_cast< dispatch_private_info_template< T >* > 1440 ( th -> th.th_dispatch -> th_disp_buffer ); /* top of the stack */ 1441 KMP_DEBUG_ASSERT( pr ); 1442 1443 if ( (status = (pr->u.p.tc != 0)) == 0 ) { 1444 *p_lb = 0; 1445 *p_ub = 0; 1446 // if ( p_last != NULL ) 1447 // *p_last = 0; 1448 if ( p_st != NULL ) 1449 *p_st = 0; 1450 if ( __kmp_env_consistency_check ) { 1451 if ( pr->pushed_ws != ct_none ) { 1452 pr->pushed_ws = __kmp_pop_workshare( gtid, pr->pushed_ws, loc ); 1453 } 1454 } 1455 } else if ( pr->nomerge ) { 1456 kmp_int32 last; 1457 T start; 1458 UT limit, trip, init; 1459 ST incr; 1460 T chunk = pr->u.p.parm1; 1461 1462 KD_TRACE(100, ("__kmp_dispatch_next: T#%d kmp_sch_dynamic_chunked case\n", gtid ) ); 1463 1464 init = chunk * pr->u.p.count++; 1465 trip = pr->u.p.tc - 1; 1466 1467 if ( (status = (init <= trip)) == 0 ) { 1468 *p_lb = 0; 1469 *p_ub = 0; 1470 // if ( p_last != NULL ) 1471 // *p_last = 0; 1472 if ( p_st != NULL ) 1473 *p_st = 0; 1474 if ( __kmp_env_consistency_check ) { 1475 if ( pr->pushed_ws != ct_none ) { 1476 pr->pushed_ws = __kmp_pop_workshare( gtid, pr->pushed_ws, loc ); 1477 } 1478 } 1479 } else { 1480 start = pr->u.p.lb; 1481 limit = chunk + init - 1; 1482 incr = pr->u.p.st; 1483 1484 if ( (last = (limit >= trip)) != 0 ) { 1485 limit = trip; 1486 #if KMP_OS_WINDOWS 1487 pr->u.p.last_upper = pr->u.p.ub; 1488 #endif /* KMP_OS_WINDOWS */ 1489 } 1490 if ( p_last != NULL ) 1491 *p_last = last; 1492 if ( p_st != NULL ) 1493 *p_st = incr; 1494 if ( incr == 1 ) { 1495 *p_lb = start + init; 1496 *p_ub = start + limit; 1497 } else { 1498 *p_lb = start + init * incr; 1499 *p_ub = start + limit * incr; 1500 } 1501 1502 if ( pr->ordered ) { 1503 pr->u.p.ordered_lower = init; 1504 pr->u.p.ordered_upper = limit; 1505 #ifdef KMP_DEBUG 1506 { 1507 const char * buff; 1508 // create format specifiers before the debug output 1509 buff = __kmp_str_format( 1510 "__kmp_dispatch_next: T#%%d ordered_lower:%%%s ordered_upper:%%%s\n", 1511 traits_t< UT >::spec, traits_t< UT >::spec ); 1512 KD_TRACE(1000, ( buff, gtid, pr->u.p.ordered_lower, pr->u.p.ordered_upper ) ); 1513 __kmp_str_free( &buff ); 1514 } 1515 #endif 1516 } // if 1517 } // if 1518 } else { 1519 pr->u.p.tc = 0; 1520 *p_lb = pr->u.p.lb; 1521 *p_ub = pr->u.p.ub; 1522 #if KMP_OS_WINDOWS 1523 pr->u.p.last_upper = *p_ub; 1524 #endif /* KMP_OS_WINDOWS */ 1525 if ( p_last != NULL ) 1526 *p_last = TRUE; 1527 if ( p_st != NULL ) 1528 *p_st = pr->u.p.st; 1529 } // if 1530 #ifdef KMP_DEBUG 1531 { 1532 const char * buff; 1533 // create format specifiers before the debug output 1534 buff = __kmp_str_format( 1535 "__kmp_dispatch_next: T#%%d serialized case: p_lb:%%%s " \ 1536 "p_ub:%%%s p_st:%%%s p_last:%%p %%d returning:%%d\n", 1537 traits_t< T >::spec, traits_t< T >::spec, traits_t< ST >::spec ); 1538 KD_TRACE(10, ( buff, gtid, *p_lb, *p_ub, *p_st, p_last, *p_last, status) ); 1539 __kmp_str_free( &buff ); 1540 } 1541 #endif 1542 #if INCLUDE_SSC_MARKS 1543 SSC_MARK_DISPATCH_NEXT(); 1544 #endif 1545 OMPT_LOOP_END; 1546 return status; 1547 } else { 1548 kmp_int32 last = 0; 1549 dispatch_shared_info_template< UT > *sh; 1550 T start; 1551 ST incr; 1552 UT limit, trip, init; 1553 1554 KMP_DEBUG_ASSERT( th->th.th_dispatch == 1555 &th->th.th_team->t.t_dispatch[th->th.th_info.ds.ds_tid] ); 1556 1557 pr = reinterpret_cast< dispatch_private_info_template< T >* > 1558 ( th->th.th_dispatch->th_dispatch_pr_current ); 1559 KMP_DEBUG_ASSERT( pr ); 1560 sh = reinterpret_cast< dispatch_shared_info_template< UT >* > 1561 ( th->th.th_dispatch->th_dispatch_sh_current ); 1562 KMP_DEBUG_ASSERT( sh ); 1563 1564 if ( pr->u.p.tc == 0 ) { 1565 // zero trip count 1566 status = 0; 1567 } else { 1568 switch (pr->schedule) { 1569 #if ( KMP_STATIC_STEAL_ENABLED && KMP_ARCH_X86_64 ) 1570 case kmp_sch_static_steal: 1571 { 1572 T chunk = pr->u.p.parm1; 1573 1574 KD_TRACE(100, ("__kmp_dispatch_next: T#%d kmp_sch_static_steal case\n", gtid) ); 1575 1576 trip = pr->u.p.tc - 1; 1577 1578 if ( ___kmp_size_type > 4 ) { 1579 // Other threads do not look into the data of this thread, 1580 // so it's not necessary to make volatile casting. 1581 init = ( pr->u.p.count )++; 1582 status = ( init < (UT)pr->u.p.ub ); 1583 } else { 1584 typedef union { 1585 struct { 1586 UT count; 1587 T ub; 1588 } p; 1589 kmp_int64 b; 1590 } union_i4; 1591 // All operations on 'count' or 'ub' must be combined atomically together. 1592 // stealing implemented only for 4-byte indexes 1593 { 1594 union_i4 vold, vnew; 1595 vold.b = *( volatile kmp_int64 * )(&pr->u.p.count); 1596 vnew = vold; 1597 vnew.p.count++; 1598 while( ! KMP_COMPARE_AND_STORE_ACQ64( 1599 ( volatile kmp_int64* )&pr->u.p.count, 1600 *VOLATILE_CAST(kmp_int64 *)&vold.b, 1601 *VOLATILE_CAST(kmp_int64 *)&vnew.b ) ) { 1602 KMP_CPU_PAUSE(); 1603 vold.b = *( volatile kmp_int64 * )(&pr->u.p.count); 1604 vnew = vold; 1605 vnew.p.count++; 1606 } 1607 vnew = vold; 1608 init = vnew.p.count; 1609 status = ( init < (UT)vnew.p.ub ) ; 1610 } 1611 1612 if( !status ) { 1613 kmp_info_t **other_threads = team->t.t_threads; 1614 int while_limit = 10; 1615 int while_index = 0; 1616 1617 // TODO: algorithm of searching for a victim 1618 // should be cleaned up and measured 1619 while ( ( !status ) && ( while_limit != ++while_index ) ) { 1620 union_i4 vold, vnew; 1621 kmp_int32 remaining; // kmp_int32 because KMP_I4 only 1622 T victimIdx = pr->u.p.parm4; 1623 T oldVictimIdx = victimIdx; 1624 dispatch_private_info_template< T > * victim; 1625 1626 do { 1627 if( !victimIdx ) { 1628 victimIdx = team->t.t_nproc - 1; 1629 } else { 1630 --victimIdx; 1631 } 1632 victim = reinterpret_cast< dispatch_private_info_template< T >* > 1633 ( other_threads[victimIdx]->th.th_dispatch->th_dispatch_pr_current ); 1634 } while ( (victim == NULL || victim == pr) && oldVictimIdx != victimIdx ); 1635 // TODO: think about a proper place of this test 1636 if ( ( !victim ) || 1637 ( (*( volatile T * )&victim->u.p.static_steal_counter) != 1638 (*( volatile T * )&pr->u.p.static_steal_counter) ) ) { 1639 // TODO: delay would be nice 1640 continue; 1641 // the victim is not ready yet to participate in stealing 1642 // because the victim is still in kmp_init_dispatch 1643 } 1644 if ( oldVictimIdx == victimIdx ) { 1645 break; 1646 } 1647 pr->u.p.parm4 = victimIdx; 1648 1649 while( 1 ) { 1650 vold.b = *( volatile kmp_int64 * )( &victim->u.p.count ); 1651 vnew = vold; 1652 1653 KMP_DEBUG_ASSERT( (vnew.p.ub - 1) * (UT)chunk <= trip ); 1654 if ( vnew.p.count >= (UT)vnew.p.ub || (remaining = vnew.p.ub - vnew.p.count) < 4 ) { 1655 break; 1656 } 1657 vnew.p.ub -= (remaining >> 2); 1658 KMP_DEBUG_ASSERT((vnew.p.ub - 1) * (UT)chunk <= trip); 1659 #pragma warning( push ) 1660 // disable warning on pointless comparison of unsigned with 0 1661 #pragma warning( disable: 186 ) 1662 KMP_DEBUG_ASSERT(vnew.p.ub >= 0); 1663 #pragma warning( pop ) 1664 // TODO: Should this be acquire or release? 1665 if ( KMP_COMPARE_AND_STORE_ACQ64( 1666 ( volatile kmp_int64 * )&victim->u.p.count, 1667 *VOLATILE_CAST(kmp_int64 *)&vold.b, 1668 *VOLATILE_CAST(kmp_int64 *)&vnew.b ) ) { 1669 status = 1; 1670 while_index = 0; 1671 // now update own count and ub 1672 #if KMP_ARCH_X86 1673 // stealing executed on non-KMP_ARCH_X86 only 1674 // Atomic 64-bit write on ia32 is 1675 // unavailable, so we do this in steps. 1676 // This code is not tested. 1677 init = vold.p.count; 1678 pr->u.p.ub = 0; 1679 pr->u.p.count = init + 1; 1680 pr->u.p.ub = vnew.p.count; 1681 #else 1682 init = vnew.p.ub; 1683 vold.p.count = init + 1; 1684 // TODO: is it safe and enough? 1685 *( volatile kmp_int64 * )(&pr->u.p.count) = vold.b; 1686 #endif // KMP_ARCH_X86 1687 break; 1688 } // if 1689 KMP_CPU_PAUSE(); 1690 } // while (1) 1691 } // while 1692 } // if 1693 } // if 1694 if ( !status ) { 1695 *p_lb = 0; 1696 *p_ub = 0; 1697 if ( p_st != NULL ) *p_st = 0; 1698 } else { 1699 start = pr->u.p.parm2; 1700 init *= chunk; 1701 limit = chunk + init - 1; 1702 incr = pr->u.p.st; 1703 1704 KMP_DEBUG_ASSERT(init <= trip); 1705 if ( (last = (limit >= trip)) != 0 ) 1706 limit = trip; 1707 if ( p_st != NULL ) *p_st = incr; 1708 1709 if ( incr == 1 ) { 1710 *p_lb = start + init; 1711 *p_ub = start + limit; 1712 } else { 1713 *p_lb = start + init * incr; 1714 *p_ub = start + limit * incr; 1715 } 1716 1717 if ( pr->ordered ) { 1718 pr->u.p.ordered_lower = init; 1719 pr->u.p.ordered_upper = limit; 1720 #ifdef KMP_DEBUG 1721 { 1722 const char * buff; 1723 // create format specifiers before the debug output 1724 buff = __kmp_str_format( 1725 "__kmp_dispatch_next: T#%%d ordered_lower:%%%s ordered_upper:%%%s\n", 1726 traits_t< UT >::spec, traits_t< UT >::spec ); 1727 KD_TRACE(1000, ( buff, gtid, pr->u.p.ordered_lower, pr->u.p.ordered_upper ) ); 1728 __kmp_str_free( &buff ); 1729 } 1730 #endif 1731 } // if 1732 } // if 1733 break; 1734 } // case 1735 #endif // ( KMP_STATIC_STEAL_ENABLED && KMP_ARCH_X86_64 ) 1736 case kmp_sch_static_balanced: 1737 { 1738 KD_TRACE(100, ("__kmp_dispatch_next: T#%d kmp_sch_static_balanced case\n", gtid) ); 1739 if ( (status = !pr->u.p.count) != 0 ) { /* check if thread has any iteration to do */ 1740 pr->u.p.count = 1; 1741 *p_lb = pr->u.p.lb; 1742 *p_ub = pr->u.p.ub; 1743 last = pr->u.p.parm1; 1744 if ( p_st != NULL ) 1745 *p_st = pr->u.p.st; 1746 } else { /* no iterations to do */ 1747 pr->u.p.lb = pr->u.p.ub + pr->u.p.st; 1748 } 1749 if ( pr->ordered ) { 1750 #ifdef KMP_DEBUG 1751 { 1752 const char * buff; 1753 // create format specifiers before the debug output 1754 buff = __kmp_str_format( 1755 "__kmp_dispatch_next: T#%%d ordered_lower:%%%s ordered_upper:%%%s\n", 1756 traits_t< UT >::spec, traits_t< UT >::spec ); 1757 KD_TRACE(1000, ( buff, gtid, pr->u.p.ordered_lower, pr->u.p.ordered_upper ) ); 1758 __kmp_str_free( &buff ); 1759 } 1760 #endif 1761 } // if 1762 } // case 1763 break; 1764 case kmp_sch_static_greedy: /* original code for kmp_sch_static_greedy was merged here */ 1765 case kmp_sch_static_chunked: 1766 { 1767 T parm1; 1768 1769 KD_TRACE(100, ("__kmp_dispatch_next: T#%d kmp_sch_static_[affinity|chunked] case\n", 1770 gtid ) ); 1771 parm1 = pr->u.p.parm1; 1772 1773 trip = pr->u.p.tc - 1; 1774 init = parm1 * (pr->u.p.count + __kmp_tid_from_gtid(gtid)); 1775 1776 if ( (status = (init <= trip)) != 0 ) { 1777 start = pr->u.p.lb; 1778 incr = pr->u.p.st; 1779 limit = parm1 + init - 1; 1780 1781 if ( (last = (limit >= trip)) != 0 ) 1782 limit = trip; 1783 1784 if ( p_st != NULL ) *p_st = incr; 1785 1786 pr->u.p.count += team->t.t_nproc; 1787 1788 if ( incr == 1 ) { 1789 *p_lb = start + init; 1790 *p_ub = start + limit; 1791 } 1792 else { 1793 *p_lb = start + init * incr; 1794 *p_ub = start + limit * incr; 1795 } 1796 1797 if ( pr->ordered ) { 1798 pr->u.p.ordered_lower = init; 1799 pr->u.p.ordered_upper = limit; 1800 #ifdef KMP_DEBUG 1801 { 1802 const char * buff; 1803 // create format specifiers before the debug output 1804 buff = __kmp_str_format( 1805 "__kmp_dispatch_next: T#%%d ordered_lower:%%%s ordered_upper:%%%s\n", 1806 traits_t< UT >::spec, traits_t< UT >::spec ); 1807 KD_TRACE(1000, ( buff, gtid, pr->u.p.ordered_lower, pr->u.p.ordered_upper ) ); 1808 __kmp_str_free( &buff ); 1809 } 1810 #endif 1811 } // if 1812 } // if 1813 } // case 1814 break; 1815 1816 case kmp_sch_dynamic_chunked: 1817 { 1818 T chunk = pr->u.p.parm1; 1819 1820 KD_TRACE(100, ("__kmp_dispatch_next: T#%d kmp_sch_dynamic_chunked case\n", 1821 gtid ) ); 1822 1823 init = chunk * test_then_inc_acq< ST >((volatile ST *) & sh->u.s.iteration ); 1824 trip = pr->u.p.tc - 1; 1825 1826 if ( (status = (init <= trip)) == 0 ) { 1827 *p_lb = 0; 1828 *p_ub = 0; 1829 if ( p_st != NULL ) *p_st = 0; 1830 } else { 1831 start = pr->u.p.lb; 1832 limit = chunk + init - 1; 1833 incr = pr->u.p.st; 1834 1835 if ( (last = (limit >= trip)) != 0 ) 1836 limit = trip; 1837 1838 if ( p_st != NULL ) *p_st = incr; 1839 1840 if ( incr == 1 ) { 1841 *p_lb = start + init; 1842 *p_ub = start + limit; 1843 } else { 1844 *p_lb = start + init * incr; 1845 *p_ub = start + limit * incr; 1846 } 1847 1848 if ( pr->ordered ) { 1849 pr->u.p.ordered_lower = init; 1850 pr->u.p.ordered_upper = limit; 1851 #ifdef KMP_DEBUG 1852 { 1853 const char * buff; 1854 // create format specifiers before the debug output 1855 buff = __kmp_str_format( 1856 "__kmp_dispatch_next: T#%%d ordered_lower:%%%s ordered_upper:%%%s\n", 1857 traits_t< UT >::spec, traits_t< UT >::spec ); 1858 KD_TRACE(1000, ( buff, gtid, pr->u.p.ordered_lower, pr->u.p.ordered_upper ) ); 1859 __kmp_str_free( &buff ); 1860 } 1861 #endif 1862 } // if 1863 } // if 1864 } // case 1865 break; 1866 1867 case kmp_sch_guided_iterative_chunked: 1868 { 1869 T chunkspec = pr->u.p.parm1; 1870 KD_TRACE(100, 1871 ("__kmp_dispatch_next: T#%d kmp_sch_guided_chunked iterative case\n",gtid)); 1872 trip = pr->u.p.tc; 1873 // Start atomic part of calculations 1874 while(1) { 1875 ST remaining; // signed, because can be < 0 1876 init = sh->u.s.iteration; // shared value 1877 remaining = trip - init; 1878 if ( remaining <= 0 ) { // AC: need to compare with 0 first 1879 // nothing to do, don't try atomic op 1880 status = 0; 1881 break; 1882 } 1883 if ( (T)remaining < pr->u.p.parm2 ) { // compare with K*nproc*(chunk+1), K=2 by default 1884 // use dynamic-style shcedule 1885 // atomically inrement iterations, get old value 1886 init = test_then_add<ST>( (ST*)&sh->u.s.iteration, (ST)chunkspec ); 1887 remaining = trip - init; 1888 if (remaining <= 0) { 1889 status = 0; // all iterations got by other threads 1890 } else { 1891 // got some iterations to work on 1892 status = 1; 1893 if ( (T)remaining > chunkspec ) { 1894 limit = init + chunkspec - 1; 1895 } else { 1896 last = 1; // the last chunk 1897 limit = init + remaining - 1; 1898 } // if 1899 } // if 1900 break; 1901 } // if 1902 limit = init + (UT)( remaining * *(double*)&pr->u.p.parm3 ); // divide by K*nproc 1903 if ( compare_and_swap<ST>( (ST*)&sh->u.s.iteration, (ST)init, (ST)limit ) ) { 1904 // CAS was successful, chunk obtained 1905 status = 1; 1906 --limit; 1907 break; 1908 } // if 1909 } // while 1910 if ( status != 0 ) { 1911 start = pr->u.p.lb; 1912 incr = pr->u.p.st; 1913 if ( p_st != NULL ) 1914 *p_st = incr; 1915 *p_lb = start + init * incr; 1916 *p_ub = start + limit * incr; 1917 if ( pr->ordered ) { 1918 pr->u.p.ordered_lower = init; 1919 pr->u.p.ordered_upper = limit; 1920 #ifdef KMP_DEBUG 1921 { 1922 const char * buff; 1923 // create format specifiers before the debug output 1924 buff = __kmp_str_format( 1925 "__kmp_dispatch_next: T#%%d ordered_lower:%%%s ordered_upper:%%%s\n", 1926 traits_t< UT >::spec, traits_t< UT >::spec ); 1927 KD_TRACE(1000, ( buff, gtid, pr->u.p.ordered_lower, pr->u.p.ordered_upper ) ); 1928 __kmp_str_free( &buff ); 1929 } 1930 #endif 1931 } // if 1932 } else { 1933 *p_lb = 0; 1934 *p_ub = 0; 1935 if ( p_st != NULL ) 1936 *p_st = 0; 1937 } // if 1938 } // case 1939 break; 1940 1941 case kmp_sch_guided_analytical_chunked: 1942 { 1943 T chunkspec = pr->u.p.parm1; 1944 UT chunkIdx; 1945 #if KMP_OS_WINDOWS && KMP_ARCH_X86 1946 /* for storing original FPCW value for Windows* OS on 1947 IA-32 architecture 8-byte version */ 1948 unsigned int oldFpcw; 1949 unsigned int fpcwSet = 0; 1950 #endif 1951 KD_TRACE(100, ("__kmp_dispatch_next: T#%d kmp_sch_guided_chunked analytical case\n", 1952 gtid ) ); 1953 1954 trip = pr->u.p.tc; 1955 1956 KMP_DEBUG_ASSERT(team->t.t_nproc > 1); 1957 KMP_DEBUG_ASSERT((2UL * chunkspec + 1) * (UT)team->t.t_nproc < trip); 1958 1959 while(1) { /* this while loop is a safeguard against unexpected zero chunk sizes */ 1960 chunkIdx = test_then_inc_acq< ST >((volatile ST *) & sh->u.s.iteration ); 1961 if ( chunkIdx >= (UT)pr->u.p.parm2 ) { 1962 --trip; 1963 /* use dynamic-style scheduling */ 1964 init = chunkIdx * chunkspec + pr->u.p.count; 1965 /* need to verify init > 0 in case of overflow in the above calculation */ 1966 if ( (status = (init > 0 && init <= trip)) != 0 ) { 1967 limit = init + chunkspec -1; 1968 1969 if ( (last = (limit >= trip)) != 0 ) 1970 limit = trip; 1971 } 1972 break; 1973 } else { 1974 /* use exponential-style scheduling */ 1975 /* The following check is to workaround the lack of long double precision on Windows* OS. 1976 This check works around the possible effect that init != 0 for chunkIdx == 0. 1977 */ 1978 #if KMP_OS_WINDOWS && KMP_ARCH_X86 1979 /* If we haven't already done so, save original 1980 FPCW and set precision to 64-bit, as Windows* OS 1981 on IA-32 architecture defaults to 53-bit */ 1982 if ( !fpcwSet ) { 1983 oldFpcw = _control87(0,0); 1984 _control87(_PC_64,_MCW_PC); 1985 fpcwSet = 0x30000; 1986 } 1987 #endif 1988 if ( chunkIdx ) { 1989 init = __kmp_dispatch_guided_remaining< T >( 1990 trip, *( DBL * )&pr->u.p.parm3, chunkIdx ); 1991 KMP_DEBUG_ASSERT(init); 1992 init = trip - init; 1993 } else 1994 init = 0; 1995 limit = trip - __kmp_dispatch_guided_remaining< T >( 1996 trip, *( DBL * )&pr->u.p.parm3, chunkIdx + 1 ); 1997 KMP_ASSERT(init <= limit); 1998 if ( init < limit ) { 1999 KMP_DEBUG_ASSERT(limit <= trip); 2000 --limit; 2001 status = 1; 2002 break; 2003 } // if 2004 } // if 2005 } // while (1) 2006 #if KMP_OS_WINDOWS && KMP_ARCH_X86 2007 /* restore FPCW if necessary 2008 AC: check fpcwSet flag first because oldFpcw can be uninitialized here 2009 */ 2010 if ( fpcwSet && ( oldFpcw & fpcwSet ) ) 2011 _control87(oldFpcw,_MCW_PC); 2012 #endif 2013 if ( status != 0 ) { 2014 start = pr->u.p.lb; 2015 incr = pr->u.p.st; 2016 if ( p_st != NULL ) 2017 *p_st = incr; 2018 *p_lb = start + init * incr; 2019 *p_ub = start + limit * incr; 2020 if ( pr->ordered ) { 2021 pr->u.p.ordered_lower = init; 2022 pr->u.p.ordered_upper = limit; 2023 #ifdef KMP_DEBUG 2024 { 2025 const char * buff; 2026 // create format specifiers before the debug output 2027 buff = __kmp_str_format( 2028 "__kmp_dispatch_next: T#%%d ordered_lower:%%%s ordered_upper:%%%s\n", 2029 traits_t< UT >::spec, traits_t< UT >::spec ); 2030 KD_TRACE(1000, ( buff, gtid, pr->u.p.ordered_lower, pr->u.p.ordered_upper ) ); 2031 __kmp_str_free( &buff ); 2032 } 2033 #endif 2034 } 2035 } else { 2036 *p_lb = 0; 2037 *p_ub = 0; 2038 if ( p_st != NULL ) 2039 *p_st = 0; 2040 } 2041 } // case 2042 break; 2043 2044 case kmp_sch_trapezoidal: 2045 { 2046 UT index; 2047 T parm2 = pr->u.p.parm2; 2048 T parm3 = pr->u.p.parm3; 2049 T parm4 = pr->u.p.parm4; 2050 KD_TRACE(100, ("__kmp_dispatch_next: T#%d kmp_sch_trapezoidal case\n", 2051 gtid ) ); 2052 2053 index = test_then_inc< ST >( (volatile ST *) & sh->u.s.iteration ); 2054 2055 init = ( index * ( (2*parm2) - (index-1)*parm4 ) ) / 2; 2056 trip = pr->u.p.tc - 1; 2057 2058 if ( (status = ((T)index < parm3 && init <= trip)) == 0 ) { 2059 *p_lb = 0; 2060 *p_ub = 0; 2061 if ( p_st != NULL ) *p_st = 0; 2062 } else { 2063 start = pr->u.p.lb; 2064 limit = ( (index+1) * ( 2*parm2 - index*parm4 ) ) / 2 - 1; 2065 incr = pr->u.p.st; 2066 2067 if ( (last = (limit >= trip)) != 0 ) 2068 limit = trip; 2069 2070 if ( p_st != NULL ) *p_st = incr; 2071 2072 if ( incr == 1 ) { 2073 *p_lb = start + init; 2074 *p_ub = start + limit; 2075 } else { 2076 *p_lb = start + init * incr; 2077 *p_ub = start + limit * incr; 2078 } 2079 2080 if ( pr->ordered ) { 2081 pr->u.p.ordered_lower = init; 2082 pr->u.p.ordered_upper = limit; 2083 #ifdef KMP_DEBUG 2084 { 2085 const char * buff; 2086 // create format specifiers before the debug output 2087 buff = __kmp_str_format( 2088 "__kmp_dispatch_next: T#%%d ordered_lower:%%%s ordered_upper:%%%s\n", 2089 traits_t< UT >::spec, traits_t< UT >::spec ); 2090 KD_TRACE(1000, ( buff, gtid, pr->u.p.ordered_lower, pr->u.p.ordered_upper ) ); 2091 __kmp_str_free( &buff ); 2092 } 2093 #endif 2094 } // if 2095 } // if 2096 } // case 2097 break; 2098 default: 2099 { 2100 status = 0; // to avoid complaints on uninitialized variable use 2101 __kmp_msg( 2102 kmp_ms_fatal, // Severity 2103 KMP_MSG( UnknownSchedTypeDetected ), // Primary message 2104 KMP_HNT( GetNewerLibrary ), // Hint 2105 __kmp_msg_null // Variadic argument list terminator 2106 ); 2107 } 2108 break; 2109 } // switch 2110 } // if tc == 0; 2111 2112 if ( status == 0 ) { 2113 UT num_done; 2114 2115 num_done = test_then_inc< ST >( (volatile ST *) & sh->u.s.num_done ); 2116 #ifdef KMP_DEBUG 2117 { 2118 const char * buff; 2119 // create format specifiers before the debug output 2120 buff = __kmp_str_format( 2121 "__kmp_dispatch_next: T#%%d increment num_done:%%%s\n", 2122 traits_t< UT >::spec ); 2123 KD_TRACE(100, ( buff, gtid, sh->u.s.num_done ) ); 2124 __kmp_str_free( &buff ); 2125 } 2126 #endif 2127 2128 if ( (ST)num_done == team->t.t_nproc-1 ) { 2129 /* NOTE: release this buffer to be reused */ 2130 2131 KMP_MB(); /* Flush all pending memory write invalidates. */ 2132 2133 sh->u.s.num_done = 0; 2134 sh->u.s.iteration = 0; 2135 2136 /* TODO replace with general release procedure? */ 2137 if ( pr->ordered ) { 2138 sh->u.s.ordered_iteration = 0; 2139 } 2140 2141 KMP_MB(); /* Flush all pending memory write invalidates. */ 2142 2143 sh -> buffer_index += KMP_MAX_DISP_BUF; 2144 KD_TRACE(100, ("__kmp_dispatch_next: T#%d change buffer_index:%d\n", 2145 gtid, sh->buffer_index) ); 2146 2147 KMP_MB(); /* Flush all pending memory write invalidates. */ 2148 2149 } // if 2150 if ( __kmp_env_consistency_check ) { 2151 if ( pr->pushed_ws != ct_none ) { 2152 pr->pushed_ws = __kmp_pop_workshare( gtid, pr->pushed_ws, loc ); 2153 } 2154 } 2155 2156 th -> th.th_dispatch -> th_deo_fcn = NULL; 2157 th -> th.th_dispatch -> th_dxo_fcn = NULL; 2158 th -> th.th_dispatch -> th_dispatch_sh_current = NULL; 2159 th -> th.th_dispatch -> th_dispatch_pr_current = NULL; 2160 } // if (status == 0) 2161 #if KMP_OS_WINDOWS 2162 else if ( last ) { 2163 pr->u.p.last_upper = pr->u.p.ub; 2164 } 2165 #endif /* KMP_OS_WINDOWS */ 2166 if ( p_last != NULL && status != 0 ) 2167 *p_last = last; 2168 } // if 2169 2170 #ifdef KMP_DEBUG 2171 { 2172 const char * buff; 2173 // create format specifiers before the debug output 2174 buff = __kmp_str_format( 2175 "__kmp_dispatch_next: T#%%d normal case: " \ 2176 "p_lb:%%%s p_ub:%%%s p_st:%%%s p_last:%%p returning:%%d\n", 2177 traits_t< T >::spec, traits_t< T >::spec, traits_t< ST >::spec ); 2178 KD_TRACE(10, ( buff, gtid, *p_lb, *p_ub, p_st ? *p_st : 0, p_last, status ) ); 2179 __kmp_str_free( &buff ); 2180 } 2181 #endif 2182 #if INCLUDE_SSC_MARKS 2183 SSC_MARK_DISPATCH_NEXT(); 2184 #endif 2185 OMPT_LOOP_END; 2186 return status; 2187 } 2188 2189 template< typename T > 2190 static void 2191 __kmp_dist_get_bounds( 2192 ident_t *loc, 2193 kmp_int32 gtid, 2194 kmp_int32 *plastiter, 2195 T *plower, 2196 T *pupper, 2197 typename traits_t< T >::signed_t incr 2198 ) { 2199 typedef typename traits_t< T >::unsigned_t UT; 2200 typedef typename traits_t< T >::signed_t ST; 2201 register kmp_uint32 team_id; 2202 register kmp_uint32 nteams; 2203 register UT trip_count; 2204 register kmp_team_t *team; 2205 kmp_info_t * th; 2206 2207 KMP_DEBUG_ASSERT( plastiter && plower && pupper ); 2208 KE_TRACE( 10, ("__kmpc_dist_get_bounds called (%d)\n", gtid)); 2209 #ifdef KMP_DEBUG 2210 { 2211 const char * buff; 2212 // create format specifiers before the debug output 2213 buff = __kmp_str_format( "__kmpc_dist_get_bounds: T#%%d liter=%%d "\ 2214 "iter=(%%%s, %%%s, %%%s) signed?<%s>\n", 2215 traits_t< T >::spec, traits_t< T >::spec, traits_t< ST >::spec, 2216 traits_t< T >::spec ); 2217 KD_TRACE(100, ( buff, gtid, *plastiter, *plower, *pupper, incr ) ); 2218 __kmp_str_free( &buff ); 2219 } 2220 #endif 2221 2222 if( __kmp_env_consistency_check ) { 2223 if( incr == 0 ) { 2224 __kmp_error_construct( kmp_i18n_msg_CnsLoopIncrZeroProhibited, ct_pdo, loc ); 2225 } 2226 if( incr > 0 ? (*pupper < *plower) : (*plower < *pupper) ) { 2227 // The loop is illegal. 2228 // Some zero-trip loops maintained by compiler, e.g.: 2229 // for(i=10;i<0;++i) // lower >= upper - run-time check 2230 // for(i=0;i>10;--i) // lower <= upper - run-time check 2231 // for(i=0;i>10;++i) // incr > 0 - compile-time check 2232 // for(i=10;i<0;--i) // incr < 0 - compile-time check 2233 // Compiler does not check the following illegal loops: 2234 // for(i=0;i<10;i+=incr) // where incr<0 2235 // for(i=10;i>0;i-=incr) // where incr<0 2236 __kmp_error_construct( kmp_i18n_msg_CnsLoopIncrIllegal, ct_pdo, loc ); 2237 } 2238 } 2239 th = __kmp_threads[gtid]; 2240 team = th->th.th_team; 2241 #if OMP_40_ENABLED 2242 KMP_DEBUG_ASSERT(th->th.th_teams_microtask); // we are in the teams construct 2243 nteams = th->th.th_teams_size.nteams; 2244 #endif 2245 team_id = team->t.t_master_tid; 2246 KMP_DEBUG_ASSERT(nteams == team->t.t_parent->t.t_nproc); 2247 2248 // compute global trip count 2249 if( incr == 1 ) { 2250 trip_count = *pupper - *plower + 1; 2251 } else if(incr == -1) { 2252 trip_count = *plower - *pupper + 1; 2253 } else { 2254 trip_count = (ST)(*pupper - *plower) / incr + 1; // cast to signed to cover incr<0 case 2255 } 2256 2257 if( trip_count <= nteams ) { 2258 KMP_DEBUG_ASSERT( 2259 __kmp_static == kmp_sch_static_greedy || \ 2260 __kmp_static == kmp_sch_static_balanced 2261 ); // Unknown static scheduling type. 2262 // only some teams get single iteration, others get nothing 2263 if( team_id < trip_count ) { 2264 *pupper = *plower = *plower + team_id * incr; 2265 } else { 2266 *plower = *pupper + incr; // zero-trip loop 2267 } 2268 if( plastiter != NULL ) 2269 *plastiter = ( team_id == trip_count - 1 ); 2270 } else { 2271 if( __kmp_static == kmp_sch_static_balanced ) { 2272 register UT chunk = trip_count / nteams; 2273 register UT extras = trip_count % nteams; 2274 *plower += incr * ( team_id * chunk + ( team_id < extras ? team_id : extras ) ); 2275 *pupper = *plower + chunk * incr - ( team_id < extras ? 0 : incr ); 2276 if( plastiter != NULL ) 2277 *plastiter = ( team_id == nteams - 1 ); 2278 } else { 2279 register T chunk_inc_count = 2280 ( trip_count / nteams + ( ( trip_count % nteams ) ? 1 : 0) ) * incr; 2281 register T upper = *pupper; 2282 KMP_DEBUG_ASSERT( __kmp_static == kmp_sch_static_greedy ); 2283 // Unknown static scheduling type. 2284 *plower += team_id * chunk_inc_count; 2285 *pupper = *plower + chunk_inc_count - incr; 2286 // Check/correct bounds if needed 2287 if( incr > 0 ) { 2288 if( *pupper < *plower ) 2289 *pupper = i_maxmin< T >::mx; 2290 if( plastiter != NULL ) 2291 *plastiter = *plower <= upper && *pupper > upper - incr; 2292 if( *pupper > upper ) 2293 *pupper = upper; // tracker C73258 2294 } else { 2295 if( *pupper > *plower ) 2296 *pupper = i_maxmin< T >::mn; 2297 if( plastiter != NULL ) 2298 *plastiter = *plower >= upper && *pupper < upper - incr; 2299 if( *pupper < upper ) 2300 *pupper = upper; // tracker C73258 2301 } 2302 } 2303 } 2304 } 2305 2306 //----------------------------------------------------------------------------------------- 2307 // Dispatch routines 2308 // Transfer call to template< type T > 2309 // __kmp_dispatch_init( ident_t *loc, int gtid, enum sched_type schedule, 2310 // T lb, T ub, ST st, ST chunk ) 2311 extern "C" { 2312 2313 /*! 2314 @ingroup WORK_SHARING 2315 @{ 2316 @param loc Source location 2317 @param gtid Global thread id 2318 @param schedule Schedule type 2319 @param lb Lower bound 2320 @param ub Upper bound 2321 @param st Step (or increment if you prefer) 2322 @param chunk The chunk size to block with 2323 2324 This function prepares the runtime to start a dynamically scheduled for loop, saving the loop arguments. 2325 These functions are all identical apart from the types of the arguments. 2326 */ 2327 2328 void 2329 __kmpc_dispatch_init_4( ident_t *loc, kmp_int32 gtid, enum sched_type schedule, 2330 kmp_int32 lb, kmp_int32 ub, kmp_int32 st, kmp_int32 chunk ) 2331 { 2332 KMP_DEBUG_ASSERT( __kmp_init_serial ); 2333 __kmp_dispatch_init< kmp_int32 >( loc, gtid, schedule, lb, ub, st, chunk, true ); 2334 } 2335 /*! 2336 See @ref __kmpc_dispatch_init_4 2337 */ 2338 void 2339 __kmpc_dispatch_init_4u( ident_t *loc, kmp_int32 gtid, enum sched_type schedule, 2340 kmp_uint32 lb, kmp_uint32 ub, kmp_int32 st, kmp_int32 chunk ) 2341 { 2342 KMP_DEBUG_ASSERT( __kmp_init_serial ); 2343 __kmp_dispatch_init< kmp_uint32 >( loc, gtid, schedule, lb, ub, st, chunk, true ); 2344 } 2345 2346 /*! 2347 See @ref __kmpc_dispatch_init_4 2348 */ 2349 void 2350 __kmpc_dispatch_init_8( ident_t *loc, kmp_int32 gtid, enum sched_type schedule, 2351 kmp_int64 lb, kmp_int64 ub, 2352 kmp_int64 st, kmp_int64 chunk ) 2353 { 2354 KMP_DEBUG_ASSERT( __kmp_init_serial ); 2355 __kmp_dispatch_init< kmp_int64 >( loc, gtid, schedule, lb, ub, st, chunk, true ); 2356 } 2357 2358 /*! 2359 See @ref __kmpc_dispatch_init_4 2360 */ 2361 void 2362 __kmpc_dispatch_init_8u( ident_t *loc, kmp_int32 gtid, enum sched_type schedule, 2363 kmp_uint64 lb, kmp_uint64 ub, 2364 kmp_int64 st, kmp_int64 chunk ) 2365 { 2366 KMP_DEBUG_ASSERT( __kmp_init_serial ); 2367 __kmp_dispatch_init< kmp_uint64 >( loc, gtid, schedule, lb, ub, st, chunk, true ); 2368 } 2369 2370 /*! 2371 See @ref __kmpc_dispatch_init_4 2372 2373 Difference from __kmpc_dispatch_init set of functions is these functions 2374 are called for composite distribute parallel for construct. Thus before 2375 regular iterations dispatching we need to calc per-team iteration space. 2376 2377 These functions are all identical apart from the types of the arguments. 2378 */ 2379 void 2380 __kmpc_dist_dispatch_init_4( ident_t *loc, kmp_int32 gtid, enum sched_type schedule, 2381 kmp_int32 *p_last, kmp_int32 lb, kmp_int32 ub, kmp_int32 st, kmp_int32 chunk ) 2382 { 2383 KMP_DEBUG_ASSERT( __kmp_init_serial ); 2384 __kmp_dist_get_bounds< kmp_int32 >( loc, gtid, p_last, &lb, &ub, st ); 2385 __kmp_dispatch_init< kmp_int32 >( loc, gtid, schedule, lb, ub, st, chunk, true ); 2386 } 2387 2388 void 2389 __kmpc_dist_dispatch_init_4u( ident_t *loc, kmp_int32 gtid, enum sched_type schedule, 2390 kmp_int32 *p_last, kmp_uint32 lb, kmp_uint32 ub, kmp_int32 st, kmp_int32 chunk ) 2391 { 2392 KMP_DEBUG_ASSERT( __kmp_init_serial ); 2393 __kmp_dist_get_bounds< kmp_uint32 >( loc, gtid, p_last, &lb, &ub, st ); 2394 __kmp_dispatch_init< kmp_uint32 >( loc, gtid, schedule, lb, ub, st, chunk, true ); 2395 } 2396 2397 void 2398 __kmpc_dist_dispatch_init_8( ident_t *loc, kmp_int32 gtid, enum sched_type schedule, 2399 kmp_int32 *p_last, kmp_int64 lb, kmp_int64 ub, kmp_int64 st, kmp_int64 chunk ) 2400 { 2401 KMP_DEBUG_ASSERT( __kmp_init_serial ); 2402 __kmp_dist_get_bounds< kmp_int64 >( loc, gtid, p_last, &lb, &ub, st ); 2403 __kmp_dispatch_init< kmp_int64 >( loc, gtid, schedule, lb, ub, st, chunk, true ); 2404 } 2405 2406 void 2407 __kmpc_dist_dispatch_init_8u( ident_t *loc, kmp_int32 gtid, enum sched_type schedule, 2408 kmp_int32 *p_last, kmp_uint64 lb, kmp_uint64 ub, kmp_int64 st, kmp_int64 chunk ) 2409 { 2410 KMP_DEBUG_ASSERT( __kmp_init_serial ); 2411 __kmp_dist_get_bounds< kmp_uint64 >( loc, gtid, p_last, &lb, &ub, st ); 2412 __kmp_dispatch_init< kmp_uint64 >( loc, gtid, schedule, lb, ub, st, chunk, true ); 2413 } 2414 2415 /*! 2416 @param loc Source code location 2417 @param gtid Global thread id 2418 @param p_last Pointer to a flag set to one if this is the last chunk or zero otherwise 2419 @param p_lb Pointer to the lower bound for the next chunk of work 2420 @param p_ub Pointer to the upper bound for the next chunk of work 2421 @param p_st Pointer to the stride for the next chunk of work 2422 @return one if there is work to be done, zero otherwise 2423 2424 Get the next dynamically allocated chunk of work for this thread. 2425 If there is no more work, then the lb,ub and stride need not be modified. 2426 */ 2427 int 2428 __kmpc_dispatch_next_4( ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last, 2429 kmp_int32 *p_lb, kmp_int32 *p_ub, kmp_int32 *p_st ) 2430 { 2431 return __kmp_dispatch_next< kmp_int32 >( loc, gtid, p_last, p_lb, p_ub, p_st ); 2432 } 2433 2434 /*! 2435 See @ref __kmpc_dispatch_next_4 2436 */ 2437 int 2438 __kmpc_dispatch_next_4u( ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last, 2439 kmp_uint32 *p_lb, kmp_uint32 *p_ub, kmp_int32 *p_st ) 2440 { 2441 return __kmp_dispatch_next< kmp_uint32 >( loc, gtid, p_last, p_lb, p_ub, p_st ); 2442 } 2443 2444 /*! 2445 See @ref __kmpc_dispatch_next_4 2446 */ 2447 int 2448 __kmpc_dispatch_next_8( ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last, 2449 kmp_int64 *p_lb, kmp_int64 *p_ub, kmp_int64 *p_st ) 2450 { 2451 return __kmp_dispatch_next< kmp_int64 >( loc, gtid, p_last, p_lb, p_ub, p_st ); 2452 } 2453 2454 /*! 2455 See @ref __kmpc_dispatch_next_4 2456 */ 2457 int 2458 __kmpc_dispatch_next_8u( ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last, 2459 kmp_uint64 *p_lb, kmp_uint64 *p_ub, kmp_int64 *p_st ) 2460 { 2461 return __kmp_dispatch_next< kmp_uint64 >( loc, gtid, p_last, p_lb, p_ub, p_st ); 2462 } 2463 2464 /*! 2465 @param loc Source code location 2466 @param gtid Global thread id 2467 2468 Mark the end of a dynamic loop. 2469 */ 2470 void 2471 __kmpc_dispatch_fini_4( ident_t *loc, kmp_int32 gtid ) 2472 { 2473 __kmp_dispatch_finish< kmp_uint32 >( gtid, loc ); 2474 } 2475 2476 /*! 2477 See @ref __kmpc_dispatch_fini_4 2478 */ 2479 void 2480 __kmpc_dispatch_fini_8( ident_t *loc, kmp_int32 gtid ) 2481 { 2482 __kmp_dispatch_finish< kmp_uint64 >( gtid, loc ); 2483 } 2484 2485 /*! 2486 See @ref __kmpc_dispatch_fini_4 2487 */ 2488 void 2489 __kmpc_dispatch_fini_4u( ident_t *loc, kmp_int32 gtid ) 2490 { 2491 __kmp_dispatch_finish< kmp_uint32 >( gtid, loc ); 2492 } 2493 2494 /*! 2495 See @ref __kmpc_dispatch_fini_4 2496 */ 2497 void 2498 __kmpc_dispatch_fini_8u( ident_t *loc, kmp_int32 gtid ) 2499 { 2500 __kmp_dispatch_finish< kmp_uint64 >( gtid, loc ); 2501 } 2502 /*! @} */ 2503 2504 //----------------------------------------------------------------------------------------- 2505 //Non-template routines from kmp_dispatch.c used in other sources 2506 2507 kmp_uint32 __kmp_eq_4( kmp_uint32 value, kmp_uint32 checker) { 2508 return value == checker; 2509 } 2510 2511 kmp_uint32 __kmp_neq_4( kmp_uint32 value, kmp_uint32 checker) { 2512 return value != checker; 2513 } 2514 2515 kmp_uint32 __kmp_lt_4( kmp_uint32 value, kmp_uint32 checker) { 2516 return value < checker; 2517 } 2518 2519 kmp_uint32 __kmp_ge_4( kmp_uint32 value, kmp_uint32 checker) { 2520 return value >= checker; 2521 } 2522 2523 kmp_uint32 __kmp_le_4( kmp_uint32 value, kmp_uint32 checker) { 2524 return value <= checker; 2525 } 2526 2527 kmp_uint32 2528 __kmp_wait_yield_4(volatile kmp_uint32 * spinner, 2529 kmp_uint32 checker, 2530 kmp_uint32 (* pred)( kmp_uint32, kmp_uint32 ) 2531 , void * obj // Higher-level synchronization object, or NULL. 2532 ) 2533 { 2534 // note: we may not belong to a team at this point 2535 register volatile kmp_uint32 * spin = spinner; 2536 register kmp_uint32 check = checker; 2537 register kmp_uint32 spins; 2538 register kmp_uint32 (*f) ( kmp_uint32, kmp_uint32 ) = pred; 2539 register kmp_uint32 r; 2540 2541 KMP_FSYNC_SPIN_INIT( obj, (void*) spin ); 2542 KMP_INIT_YIELD( spins ); 2543 // main wait spin loop 2544 while(!f(r = TCR_4(*spin), check)) { 2545 KMP_FSYNC_SPIN_PREPARE( obj ); 2546 /* GEH - remove this since it was accidentally introduced when kmp_wait was split. 2547 It causes problems with infinite recursion because of exit lock */ 2548 /* if ( TCR_4(__kmp_global.g.g_done) && __kmp_global.g.g_abort) 2549 __kmp_abort_thread(); */ 2550 2551 /* if we have waited a bit, or are oversubscribed, yield */ 2552 /* pause is in the following code */ 2553 KMP_YIELD( TCR_4(__kmp_nth) > __kmp_avail_proc ); 2554 KMP_YIELD_SPIN( spins ); 2555 } 2556 KMP_FSYNC_SPIN_ACQUIRED( obj ); 2557 return r; 2558 } 2559 2560 } // extern "C" 2561 2562 #ifdef KMP_GOMP_COMPAT 2563 2564 void 2565 __kmp_aux_dispatch_init_4( ident_t *loc, kmp_int32 gtid, enum sched_type schedule, 2566 kmp_int32 lb, kmp_int32 ub, kmp_int32 st, 2567 kmp_int32 chunk, int push_ws ) 2568 { 2569 __kmp_dispatch_init< kmp_int32 >( loc, gtid, schedule, lb, ub, st, chunk, 2570 push_ws ); 2571 } 2572 2573 void 2574 __kmp_aux_dispatch_init_4u( ident_t *loc, kmp_int32 gtid, enum sched_type schedule, 2575 kmp_uint32 lb, kmp_uint32 ub, kmp_int32 st, 2576 kmp_int32 chunk, int push_ws ) 2577 { 2578 __kmp_dispatch_init< kmp_uint32 >( loc, gtid, schedule, lb, ub, st, chunk, 2579 push_ws ); 2580 } 2581 2582 void 2583 __kmp_aux_dispatch_init_8( ident_t *loc, kmp_int32 gtid, enum sched_type schedule, 2584 kmp_int64 lb, kmp_int64 ub, kmp_int64 st, 2585 kmp_int64 chunk, int push_ws ) 2586 { 2587 __kmp_dispatch_init< kmp_int64 >( loc, gtid, schedule, lb, ub, st, chunk, 2588 push_ws ); 2589 } 2590 2591 void 2592 __kmp_aux_dispatch_init_8u( ident_t *loc, kmp_int32 gtid, enum sched_type schedule, 2593 kmp_uint64 lb, kmp_uint64 ub, kmp_int64 st, 2594 kmp_int64 chunk, int push_ws ) 2595 { 2596 __kmp_dispatch_init< kmp_uint64 >( loc, gtid, schedule, lb, ub, st, chunk, 2597 push_ws ); 2598 } 2599 2600 void 2601 __kmp_aux_dispatch_fini_chunk_4( ident_t *loc, kmp_int32 gtid ) 2602 { 2603 __kmp_dispatch_finish_chunk< kmp_uint32 >( gtid, loc ); 2604 } 2605 2606 void 2607 __kmp_aux_dispatch_fini_chunk_8( ident_t *loc, kmp_int32 gtid ) 2608 { 2609 __kmp_dispatch_finish_chunk< kmp_uint64 >( gtid, loc ); 2610 } 2611 2612 void 2613 __kmp_aux_dispatch_fini_chunk_4u( ident_t *loc, kmp_int32 gtid ) 2614 { 2615 __kmp_dispatch_finish_chunk< kmp_uint32 >( gtid, loc ); 2616 } 2617 2618 void 2619 __kmp_aux_dispatch_fini_chunk_8u( ident_t *loc, kmp_int32 gtid ) 2620 { 2621 __kmp_dispatch_finish_chunk< kmp_uint64 >( gtid, loc ); 2622 } 2623 2624 #endif /* KMP_GOMP_COMPAT */ 2625 2626 /* ------------------------------------------------------------------------ */ 2627 /* ------------------------------------------------------------------------ */ 2628 2629