1 /* 2 * kmp_sched.cpp -- static scheduling -- iteration initialization 3 */ 4 5 //===----------------------------------------------------------------------===// 6 // 7 // The LLVM Compiler Infrastructure 8 // 9 // This file is dual licensed under the MIT and the University of Illinois Open 10 // Source Licenses. See LICENSE.txt for details. 11 // 12 //===----------------------------------------------------------------------===// 13 14 /* Static scheduling initialization. 15 16 NOTE: team->t.t_nproc is a constant inside of any dispatch loop, however 17 it may change values between parallel regions. __kmp_max_nth 18 is the largest value __kmp_nth may take, 1 is the smallest. */ 19 20 #include "kmp.h" 21 #include "kmp_error.h" 22 #include "kmp_i18n.h" 23 #include "kmp_itt.h" 24 #include "kmp_stats.h" 25 #include "kmp_str.h" 26 27 #if OMPT_SUPPORT 28 #include "ompt-specific.h" 29 #endif 30 31 #ifdef KMP_DEBUG 32 //------------------------------------------------------------------------- 33 // template for debug prints specification ( d, u, lld, llu ) 34 char const *traits_t<int>::spec = "d"; 35 char const *traits_t<unsigned int>::spec = "u"; 36 char const *traits_t<long long>::spec = "lld"; 37 char const *traits_t<unsigned long long>::spec = "llu"; 38 //------------------------------------------------------------------------- 39 #endif 40 41 template <typename T> 42 static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid, 43 kmp_int32 schedtype, kmp_int32 *plastiter, 44 T *plower, T *pupper, 45 typename traits_t<T>::signed_t *pstride, 46 typename traits_t<T>::signed_t incr, 47 typename traits_t<T>::signed_t chunk 48 #if OMPT_SUPPORT && OMPT_OPTIONAL 49 , 50 void *codeptr 51 #endif 52 ) { 53 KMP_COUNT_BLOCK(OMP_FOR_static); 54 KMP_TIME_PARTITIONED_BLOCK(FOR_static_scheduling); 55 56 typedef typename traits_t<T>::unsigned_t UT; 57 typedef typename traits_t<T>::signed_t ST; 58 /* this all has to be changed back to TID and such.. */ 59 kmp_int32 gtid = global_tid; 60 kmp_uint32 tid; 61 kmp_uint32 nth; 62 UT trip_count; 63 kmp_team_t *team; 64 kmp_info_t *th = __kmp_threads[gtid]; 65 66 #if OMPT_SUPPORT && OMPT_OPTIONAL 67 ompt_team_info_t *team_info = NULL; 68 ompt_task_info_t *task_info = NULL; 69 ompt_work_type_t ompt_work_type; 70 71 if (ompt_enabled.enabled) { 72 // Only fully initialize variables needed by OMPT if OMPT is enabled. 73 team_info = __ompt_get_teaminfo(0, NULL); 74 task_info = __ompt_get_task_info_object(0); 75 // Determine workshare type 76 if (loc != NULL) { 77 if ((loc->flags & KMP_IDENT_WORK_LOOP) != 0) { 78 ompt_work_type = ompt_work_loop; 79 } else if ((loc->flags & KMP_IDENT_WORK_SECTIONS) != 0) { 80 ompt_work_type = ompt_work_sections; 81 } else if ((loc->flags & KMP_IDENT_WORK_DISTRIBUTE) != 0) { 82 ompt_work_type = ompt_work_distribute; 83 } else { 84 KMP_ASSERT2(0, 85 "__kmpc_for_static_init: can't determine workshare type"); 86 } 87 KMP_DEBUG_ASSERT(ompt_work_type); 88 } 89 } 90 #endif 91 92 KMP_DEBUG_ASSERT(plastiter && plower && pupper && pstride); 93 KE_TRACE(10, ("__kmpc_for_static_init called (%d)\n", global_tid)); 94 #ifdef KMP_DEBUG 95 { 96 const char *buff; 97 // create format specifiers before the debug output 98 buff = __kmp_str_format( 99 "__kmpc_for_static_init: T#%%d sched=%%d liter=%%d iter=(%%%s," 100 " %%%s, %%%s) incr=%%%s chunk=%%%s signed?<%s>\n", 101 traits_t<T>::spec, traits_t<T>::spec, traits_t<ST>::spec, 102 traits_t<ST>::spec, traits_t<ST>::spec, traits_t<T>::spec); 103 KD_TRACE(100, (buff, global_tid, schedtype, *plastiter, *plower, *pupper, 104 *pstride, incr, chunk)); 105 __kmp_str_free(&buff); 106 } 107 #endif 108 109 if (__kmp_env_consistency_check) { 110 __kmp_push_workshare(global_tid, ct_pdo, loc); 111 if (incr == 0) { 112 __kmp_error_construct(kmp_i18n_msg_CnsLoopIncrZeroProhibited, ct_pdo, 113 loc); 114 } 115 } 116 /* special handling for zero-trip loops */ 117 if (incr > 0 ? (*pupper < *plower) : (*plower < *pupper)) { 118 if (plastiter != NULL) 119 *plastiter = FALSE; 120 /* leave pupper and plower set to entire iteration space */ 121 *pstride = incr; /* value should never be used */ 122 // *plower = *pupper - incr; 123 // let compiler bypass the illegal loop (like for(i=1;i<10;i--)) 124 // THE LINE COMMENTED ABOVE CAUSED shape2F/h_tests_1.f TO HAVE A FAILURE 125 // ON A ZERO-TRIP LOOP (lower=1, upper=0,stride=1) - JPH June 23, 2009. 126 #ifdef KMP_DEBUG 127 { 128 const char *buff; 129 // create format specifiers before the debug output 130 buff = __kmp_str_format("__kmpc_for_static_init:(ZERO TRIP) liter=%%d " 131 "lower=%%%s upper=%%%s stride = %%%s " 132 "signed?<%s>, loc = %%s\n", 133 traits_t<T>::spec, traits_t<T>::spec, 134 traits_t<ST>::spec, traits_t<T>::spec); 135 KD_TRACE(100, 136 (buff, *plastiter, *plower, *pupper, *pstride, loc->psource)); 137 __kmp_str_free(&buff); 138 } 139 #endif 140 KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid)); 141 142 #if OMPT_SUPPORT && OMPT_OPTIONAL 143 if (ompt_enabled.ompt_callback_work) { 144 ompt_callbacks.ompt_callback(ompt_callback_work)( 145 ompt_work_type, ompt_scope_begin, &(team_info->parallel_data), 146 &(task_info->task_data), 0, codeptr); 147 } 148 #endif 149 KMP_COUNT_VALUE(FOR_static_iterations, 0); 150 return; 151 } 152 153 #if OMP_40_ENABLED 154 // Although there are schedule enumerations above kmp_ord_upper which are not 155 // schedules for "distribute", the only ones which are useful are dynamic, so 156 // cannot be seen here, since this codepath is only executed for static 157 // schedules. 158 if (schedtype > kmp_ord_upper) { 159 // we are in DISTRIBUTE construct 160 schedtype += kmp_sch_static - 161 kmp_distribute_static; // AC: convert to usual schedule type 162 tid = th->th.th_team->t.t_master_tid; 163 team = th->th.th_team->t.t_parent; 164 } else 165 #endif 166 { 167 tid = __kmp_tid_from_gtid(global_tid); 168 team = th->th.th_team; 169 } 170 171 /* determine if "for" loop is an active worksharing construct */ 172 if (team->t.t_serialized) { 173 /* serialized parallel, each thread executes whole iteration space */ 174 if (plastiter != NULL) 175 *plastiter = TRUE; 176 /* leave pupper and plower set to entire iteration space */ 177 *pstride = 178 (incr > 0) ? (*pupper - *plower + 1) : (-(*plower - *pupper + 1)); 179 180 #ifdef KMP_DEBUG 181 { 182 const char *buff; 183 // create format specifiers before the debug output 184 buff = __kmp_str_format("__kmpc_for_static_init: (serial) liter=%%d " 185 "lower=%%%s upper=%%%s stride = %%%s\n", 186 traits_t<T>::spec, traits_t<T>::spec, 187 traits_t<ST>::spec); 188 KD_TRACE(100, (buff, *plastiter, *plower, *pupper, *pstride)); 189 __kmp_str_free(&buff); 190 } 191 #endif 192 KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid)); 193 194 #if OMPT_SUPPORT && OMPT_OPTIONAL 195 if (ompt_enabled.ompt_callback_work) { 196 ompt_callbacks.ompt_callback(ompt_callback_work)( 197 ompt_work_type, ompt_scope_begin, &(team_info->parallel_data), 198 &(task_info->task_data), *pstride, codeptr); 199 } 200 #endif 201 return; 202 } 203 nth = team->t.t_nproc; 204 if (nth == 1) { 205 if (plastiter != NULL) 206 *plastiter = TRUE; 207 *pstride = 208 (incr > 0) ? (*pupper - *plower + 1) : (-(*plower - *pupper + 1)); 209 #ifdef KMP_DEBUG 210 { 211 const char *buff; 212 // create format specifiers before the debug output 213 buff = __kmp_str_format("__kmpc_for_static_init: (serial) liter=%%d " 214 "lower=%%%s upper=%%%s stride = %%%s\n", 215 traits_t<T>::spec, traits_t<T>::spec, 216 traits_t<ST>::spec); 217 KD_TRACE(100, (buff, *plastiter, *plower, *pupper, *pstride)); 218 __kmp_str_free(&buff); 219 } 220 #endif 221 KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid)); 222 223 #if OMPT_SUPPORT && OMPT_OPTIONAL 224 if (ompt_enabled.ompt_callback_work) { 225 ompt_callbacks.ompt_callback(ompt_callback_work)( 226 ompt_work_type, ompt_scope_begin, &(team_info->parallel_data), 227 &(task_info->task_data), *pstride, codeptr); 228 } 229 #endif 230 return; 231 } 232 233 /* compute trip count */ 234 if (incr == 1) { 235 trip_count = *pupper - *plower + 1; 236 } else if (incr == -1) { 237 trip_count = *plower - *pupper + 1; 238 } else if (incr > 0) { 239 // upper-lower can exceed the limit of signed type 240 trip_count = (UT)(*pupper - *plower) / incr + 1; 241 } else { 242 trip_count = (UT)(*plower - *pupper) / (-incr) + 1; 243 } 244 245 if (__kmp_env_consistency_check) { 246 /* tripcount overflow? */ 247 if (trip_count == 0 && *pupper != *plower) { 248 __kmp_error_construct(kmp_i18n_msg_CnsIterationRangeTooLarge, ct_pdo, 249 loc); 250 } 251 } 252 KMP_COUNT_VALUE(FOR_static_iterations, trip_count); 253 254 /* compute remaining parameters */ 255 switch (schedtype) { 256 case kmp_sch_static: { 257 if (trip_count < nth) { 258 KMP_DEBUG_ASSERT( 259 __kmp_static == kmp_sch_static_greedy || 260 __kmp_static == 261 kmp_sch_static_balanced); // Unknown static scheduling type. 262 if (tid < trip_count) { 263 *pupper = *plower = *plower + tid * incr; 264 } else { 265 *plower = *pupper + incr; 266 } 267 if (plastiter != NULL) 268 *plastiter = (tid == trip_count - 1); 269 } else { 270 if (__kmp_static == kmp_sch_static_balanced) { 271 UT small_chunk = trip_count / nth; 272 UT extras = trip_count % nth; 273 *plower += incr * (tid * small_chunk + (tid < extras ? tid : extras)); 274 *pupper = *plower + small_chunk * incr - (tid < extras ? 0 : incr); 275 if (plastiter != NULL) 276 *plastiter = (tid == nth - 1); 277 } else { 278 T big_chunk_inc_count = 279 (trip_count / nth + ((trip_count % nth) ? 1 : 0)) * incr; 280 T old_upper = *pupper; 281 282 KMP_DEBUG_ASSERT(__kmp_static == kmp_sch_static_greedy); 283 // Unknown static scheduling type. 284 285 *plower += tid * big_chunk_inc_count; 286 *pupper = *plower + big_chunk_inc_count - incr; 287 if (incr > 0) { 288 if (*pupper < *plower) 289 *pupper = traits_t<T>::max_value; 290 if (plastiter != NULL) 291 *plastiter = *plower <= old_upper && *pupper > old_upper - incr; 292 if (*pupper > old_upper) 293 *pupper = old_upper; // tracker C73258 294 } else { 295 if (*pupper > *plower) 296 *pupper = traits_t<T>::min_value; 297 if (plastiter != NULL) 298 *plastiter = *plower >= old_upper && *pupper < old_upper - incr; 299 if (*pupper < old_upper) 300 *pupper = old_upper; // tracker C73258 301 } 302 } 303 } 304 *pstride = trip_count; 305 break; 306 } 307 case kmp_sch_static_chunked: { 308 ST span; 309 if (chunk < 1) { 310 chunk = 1; 311 } 312 span = chunk * incr; 313 *pstride = span * nth; 314 *plower = *plower + (span * tid); 315 *pupper = *plower + span - incr; 316 if (plastiter != NULL) 317 *plastiter = (tid == ((trip_count - 1) / (UT)chunk) % nth); 318 break; 319 } 320 #if OMP_45_ENABLED 321 case kmp_sch_static_balanced_chunked: { 322 T old_upper = *pupper; 323 // round up to make sure the chunk is enough to cover all iterations 324 UT span = (trip_count + nth - 1) / nth; 325 326 // perform chunk adjustment 327 chunk = (span + chunk - 1) & ~(chunk - 1); 328 329 span = chunk * incr; 330 *plower = *plower + (span * tid); 331 *pupper = *plower + span - incr; 332 if (incr > 0) { 333 if (*pupper > old_upper) 334 *pupper = old_upper; 335 } else if (*pupper < old_upper) 336 *pupper = old_upper; 337 338 if (plastiter != NULL) 339 *plastiter = (tid == ((trip_count - 1) / (UT)chunk)); 340 break; 341 } 342 #endif 343 default: 344 KMP_ASSERT2(0, "__kmpc_for_static_init: unknown scheduling type"); 345 break; 346 } 347 348 #if USE_ITT_BUILD 349 // Report loop metadata 350 if (KMP_MASTER_TID(tid) && __itt_metadata_add_ptr && 351 __kmp_forkjoin_frames_mode == 3 && 352 #if OMP_40_ENABLED 353 th->th.th_teams_microtask == NULL && 354 #endif 355 team->t.t_active_level == 1) { 356 kmp_uint64 cur_chunk = chunk; 357 // Calculate chunk in case it was not specified; it is specified for 358 // kmp_sch_static_chunked 359 if (schedtype == kmp_sch_static) { 360 cur_chunk = trip_count / nth + ((trip_count % nth) ? 1 : 0); 361 } 362 // 0 - "static" schedule 363 __kmp_itt_metadata_loop(loc, 0, trip_count, cur_chunk); 364 } 365 #endif 366 #ifdef KMP_DEBUG 367 { 368 const char *buff; 369 // create format specifiers before the debug output 370 buff = __kmp_str_format("__kmpc_for_static_init: liter=%%d lower=%%%s " 371 "upper=%%%s stride = %%%s signed?<%s>\n", 372 traits_t<T>::spec, traits_t<T>::spec, 373 traits_t<ST>::spec, traits_t<T>::spec); 374 KD_TRACE(100, (buff, *plastiter, *plower, *pupper, *pstride)); 375 __kmp_str_free(&buff); 376 } 377 #endif 378 KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid)); 379 380 #if OMPT_SUPPORT && OMPT_OPTIONAL 381 if (ompt_enabled.ompt_callback_work) { 382 ompt_callbacks.ompt_callback(ompt_callback_work)( 383 ompt_work_type, ompt_scope_begin, &(team_info->parallel_data), 384 &(task_info->task_data), trip_count, codeptr); 385 } 386 #endif 387 388 return; 389 } 390 391 template <typename T> 392 static void __kmp_dist_for_static_init(ident_t *loc, kmp_int32 gtid, 393 kmp_int32 schedule, kmp_int32 *plastiter, 394 T *plower, T *pupper, T *pupperDist, 395 typename traits_t<T>::signed_t *pstride, 396 typename traits_t<T>::signed_t incr, 397 typename traits_t<T>::signed_t chunk) { 398 KMP_COUNT_BLOCK(OMP_DISTRIBUTE); 399 typedef typename traits_t<T>::unsigned_t UT; 400 typedef typename traits_t<T>::signed_t ST; 401 kmp_uint32 tid; 402 kmp_uint32 nth; 403 kmp_uint32 team_id; 404 kmp_uint32 nteams; 405 UT trip_count; 406 kmp_team_t *team; 407 kmp_info_t *th; 408 409 KMP_DEBUG_ASSERT(plastiter && plower && pupper && pupperDist && pstride); 410 KE_TRACE(10, ("__kmpc_dist_for_static_init called (%d)\n", gtid)); 411 #ifdef KMP_DEBUG 412 { 413 const char *buff; 414 // create format specifiers before the debug output 415 buff = __kmp_str_format( 416 "__kmpc_dist_for_static_init: T#%%d schedLoop=%%d liter=%%d " 417 "iter=(%%%s, %%%s, %%%s) chunk=%%%s signed?<%s>\n", 418 traits_t<T>::spec, traits_t<T>::spec, traits_t<ST>::spec, 419 traits_t<ST>::spec, traits_t<T>::spec); 420 KD_TRACE(100, 421 (buff, gtid, schedule, *plastiter, *plower, *pupper, incr, chunk)); 422 __kmp_str_free(&buff); 423 } 424 #endif 425 426 if (__kmp_env_consistency_check) { 427 __kmp_push_workshare(gtid, ct_pdo, loc); 428 if (incr == 0) { 429 __kmp_error_construct(kmp_i18n_msg_CnsLoopIncrZeroProhibited, ct_pdo, 430 loc); 431 } 432 if (incr > 0 ? (*pupper < *plower) : (*plower < *pupper)) { 433 // The loop is illegal. 434 // Some zero-trip loops maintained by compiler, e.g.: 435 // for(i=10;i<0;++i) // lower >= upper - run-time check 436 // for(i=0;i>10;--i) // lower <= upper - run-time check 437 // for(i=0;i>10;++i) // incr > 0 - compile-time check 438 // for(i=10;i<0;--i) // incr < 0 - compile-time check 439 // Compiler does not check the following illegal loops: 440 // for(i=0;i<10;i+=incr) // where incr<0 441 // for(i=10;i>0;i-=incr) // where incr<0 442 __kmp_error_construct(kmp_i18n_msg_CnsLoopIncrIllegal, ct_pdo, loc); 443 } 444 } 445 tid = __kmp_tid_from_gtid(gtid); 446 th = __kmp_threads[gtid]; 447 nth = th->th.th_team_nproc; 448 team = th->th.th_team; 449 #if OMP_40_ENABLED 450 KMP_DEBUG_ASSERT(th->th.th_teams_microtask); // we are in the teams construct 451 nteams = th->th.th_teams_size.nteams; 452 #endif 453 team_id = team->t.t_master_tid; 454 KMP_DEBUG_ASSERT(nteams == team->t.t_parent->t.t_nproc); 455 456 // compute global trip count 457 if (incr == 1) { 458 trip_count = *pupper - *plower + 1; 459 } else if (incr == -1) { 460 trip_count = *plower - *pupper + 1; 461 } else if (incr > 0) { 462 // upper-lower can exceed the limit of signed type 463 trip_count = (UT)(*pupper - *plower) / incr + 1; 464 } else { 465 trip_count = (UT)(*plower - *pupper) / (-incr) + 1; 466 } 467 468 *pstride = *pupper - *plower; // just in case (can be unused) 469 if (trip_count <= nteams) { 470 KMP_DEBUG_ASSERT( 471 __kmp_static == kmp_sch_static_greedy || 472 __kmp_static == 473 kmp_sch_static_balanced); // Unknown static scheduling type. 474 // only masters of some teams get single iteration, other threads get 475 // nothing 476 if (team_id < trip_count && tid == 0) { 477 *pupper = *pupperDist = *plower = *plower + team_id * incr; 478 } else { 479 *pupperDist = *pupper; 480 *plower = *pupper + incr; // compiler should skip loop body 481 } 482 if (plastiter != NULL) 483 *plastiter = (tid == 0 && team_id == trip_count - 1); 484 } else { 485 // Get the team's chunk first (each team gets at most one chunk) 486 if (__kmp_static == kmp_sch_static_balanced) { 487 UT chunkD = trip_count / nteams; 488 UT extras = trip_count % nteams; 489 *plower += 490 incr * (team_id * chunkD + (team_id < extras ? team_id : extras)); 491 *pupperDist = *plower + chunkD * incr - (team_id < extras ? 0 : incr); 492 if (plastiter != NULL) 493 *plastiter = (team_id == nteams - 1); 494 } else { 495 T chunk_inc_count = 496 (trip_count / nteams + ((trip_count % nteams) ? 1 : 0)) * incr; 497 T upper = *pupper; 498 KMP_DEBUG_ASSERT(__kmp_static == kmp_sch_static_greedy); 499 // Unknown static scheduling type. 500 *plower += team_id * chunk_inc_count; 501 *pupperDist = *plower + chunk_inc_count - incr; 502 // Check/correct bounds if needed 503 if (incr > 0) { 504 if (*pupperDist < *plower) 505 *pupperDist = traits_t<T>::max_value; 506 if (plastiter != NULL) 507 *plastiter = *plower <= upper && *pupperDist > upper - incr; 508 if (*pupperDist > upper) 509 *pupperDist = upper; // tracker C73258 510 if (*plower > *pupperDist) { 511 *pupper = *pupperDist; // no iterations available for the team 512 goto end; 513 } 514 } else { 515 if (*pupperDist > *plower) 516 *pupperDist = traits_t<T>::min_value; 517 if (plastiter != NULL) 518 *plastiter = *plower >= upper && *pupperDist < upper - incr; 519 if (*pupperDist < upper) 520 *pupperDist = upper; // tracker C73258 521 if (*plower < *pupperDist) { 522 *pupper = *pupperDist; // no iterations available for the team 523 goto end; 524 } 525 } 526 } 527 // Get the parallel loop chunk now (for thread) 528 // compute trip count for team's chunk 529 if (incr == 1) { 530 trip_count = *pupperDist - *plower + 1; 531 } else if (incr == -1) { 532 trip_count = *plower - *pupperDist + 1; 533 } else if (incr > 1) { 534 // upper-lower can exceed the limit of signed type 535 trip_count = (UT)(*pupperDist - *plower) / incr + 1; 536 } else { 537 trip_count = (UT)(*plower - *pupperDist) / (-incr) + 1; 538 } 539 KMP_DEBUG_ASSERT(trip_count); 540 switch (schedule) { 541 case kmp_sch_static: { 542 if (trip_count <= nth) { 543 KMP_DEBUG_ASSERT( 544 __kmp_static == kmp_sch_static_greedy || 545 __kmp_static == 546 kmp_sch_static_balanced); // Unknown static scheduling type. 547 if (tid < trip_count) 548 *pupper = *plower = *plower + tid * incr; 549 else 550 *plower = *pupper + incr; // no iterations available 551 if (plastiter != NULL) 552 if (*plastiter != 0 && !(tid == trip_count - 1)) 553 *plastiter = 0; 554 } else { 555 if (__kmp_static == kmp_sch_static_balanced) { 556 UT chunkL = trip_count / nth; 557 UT extras = trip_count % nth; 558 *plower += incr * (tid * chunkL + (tid < extras ? tid : extras)); 559 *pupper = *plower + chunkL * incr - (tid < extras ? 0 : incr); 560 if (plastiter != NULL) 561 if (*plastiter != 0 && !(tid == nth - 1)) 562 *plastiter = 0; 563 } else { 564 T chunk_inc_count = 565 (trip_count / nth + ((trip_count % nth) ? 1 : 0)) * incr; 566 T upper = *pupperDist; 567 KMP_DEBUG_ASSERT(__kmp_static == kmp_sch_static_greedy); 568 // Unknown static scheduling type. 569 *plower += tid * chunk_inc_count; 570 *pupper = *plower + chunk_inc_count - incr; 571 if (incr > 0) { 572 if (*pupper < *plower) 573 *pupper = traits_t<T>::max_value; 574 if (plastiter != NULL) 575 if (*plastiter != 0 && 576 !(*plower <= upper && *pupper > upper - incr)) 577 *plastiter = 0; 578 if (*pupper > upper) 579 *pupper = upper; // tracker C73258 580 } else { 581 if (*pupper > *plower) 582 *pupper = traits_t<T>::min_value; 583 if (plastiter != NULL) 584 if (*plastiter != 0 && 585 !(*plower >= upper && *pupper < upper - incr)) 586 *plastiter = 0; 587 if (*pupper < upper) 588 *pupper = upper; // tracker C73258 589 } 590 } 591 } 592 break; 593 } 594 case kmp_sch_static_chunked: { 595 ST span; 596 if (chunk < 1) 597 chunk = 1; 598 span = chunk * incr; 599 *pstride = span * nth; 600 *plower = *plower + (span * tid); 601 *pupper = *plower + span - incr; 602 if (plastiter != NULL) 603 if (*plastiter != 0 && !(tid == ((trip_count - 1) / (UT)chunk) % nth)) 604 *plastiter = 0; 605 break; 606 } 607 default: 608 KMP_ASSERT2(0, 609 "__kmpc_dist_for_static_init: unknown loop scheduling type"); 610 break; 611 } 612 } 613 end:; 614 #ifdef KMP_DEBUG 615 { 616 const char *buff; 617 // create format specifiers before the debug output 618 buff = __kmp_str_format( 619 "__kmpc_dist_for_static_init: last=%%d lo=%%%s up=%%%s upDist=%%%s " 620 "stride=%%%s signed?<%s>\n", 621 traits_t<T>::spec, traits_t<T>::spec, traits_t<T>::spec, 622 traits_t<ST>::spec, traits_t<T>::spec); 623 KD_TRACE(100, (buff, *plastiter, *plower, *pupper, *pupperDist, *pstride)); 624 __kmp_str_free(&buff); 625 } 626 #endif 627 KE_TRACE(10, ("__kmpc_dist_for_static_init: T#%d return\n", gtid)); 628 return; 629 } 630 631 template <typename T> 632 static void __kmp_team_static_init(ident_t *loc, kmp_int32 gtid, 633 kmp_int32 *p_last, T *p_lb, T *p_ub, 634 typename traits_t<T>::signed_t *p_st, 635 typename traits_t<T>::signed_t incr, 636 typename traits_t<T>::signed_t chunk) { 637 // The routine returns the first chunk distributed to the team and 638 // stride for next chunks calculation. 639 // Last iteration flag set for the team that will execute 640 // the last iteration of the loop. 641 // The routine is called for dist_schedue(static,chunk) only. 642 typedef typename traits_t<T>::unsigned_t UT; 643 typedef typename traits_t<T>::signed_t ST; 644 kmp_uint32 team_id; 645 kmp_uint32 nteams; 646 UT trip_count; 647 T lower; 648 T upper; 649 ST span; 650 kmp_team_t *team; 651 kmp_info_t *th; 652 653 KMP_DEBUG_ASSERT(p_last && p_lb && p_ub && p_st); 654 KE_TRACE(10, ("__kmp_team_static_init called (%d)\n", gtid)); 655 #ifdef KMP_DEBUG 656 { 657 const char *buff; 658 // create format specifiers before the debug output 659 buff = __kmp_str_format("__kmp_team_static_init enter: T#%%d liter=%%d " 660 "iter=(%%%s, %%%s, %%%s) chunk %%%s; signed?<%s>\n", 661 traits_t<T>::spec, traits_t<T>::spec, 662 traits_t<ST>::spec, traits_t<ST>::spec, 663 traits_t<T>::spec); 664 KD_TRACE(100, (buff, gtid, *p_last, *p_lb, *p_ub, *p_st, chunk)); 665 __kmp_str_free(&buff); 666 } 667 #endif 668 669 lower = *p_lb; 670 upper = *p_ub; 671 if (__kmp_env_consistency_check) { 672 if (incr == 0) { 673 __kmp_error_construct(kmp_i18n_msg_CnsLoopIncrZeroProhibited, ct_pdo, 674 loc); 675 } 676 if (incr > 0 ? (upper < lower) : (lower < upper)) { 677 // The loop is illegal. 678 // Some zero-trip loops maintained by compiler, e.g.: 679 // for(i=10;i<0;++i) // lower >= upper - run-time check 680 // for(i=0;i>10;--i) // lower <= upper - run-time check 681 // for(i=0;i>10;++i) // incr > 0 - compile-time check 682 // for(i=10;i<0;--i) // incr < 0 - compile-time check 683 // Compiler does not check the following illegal loops: 684 // for(i=0;i<10;i+=incr) // where incr<0 685 // for(i=10;i>0;i-=incr) // where incr<0 686 __kmp_error_construct(kmp_i18n_msg_CnsLoopIncrIllegal, ct_pdo, loc); 687 } 688 } 689 th = __kmp_threads[gtid]; 690 team = th->th.th_team; 691 #if OMP_40_ENABLED 692 KMP_DEBUG_ASSERT(th->th.th_teams_microtask); // we are in the teams construct 693 nteams = th->th.th_teams_size.nteams; 694 #endif 695 team_id = team->t.t_master_tid; 696 KMP_DEBUG_ASSERT(nteams == team->t.t_parent->t.t_nproc); 697 698 // compute trip count 699 if (incr == 1) { 700 trip_count = upper - lower + 1; 701 } else if (incr == -1) { 702 trip_count = lower - upper + 1; 703 } else if (incr > 0) { 704 // upper-lower can exceed the limit of signed type 705 trip_count = (UT)(upper - lower) / incr + 1; 706 } else { 707 trip_count = (UT)(lower - upper) / (-incr) + 1; 708 } 709 if (chunk < 1) 710 chunk = 1; 711 span = chunk * incr; 712 *p_st = span * nteams; 713 *p_lb = lower + (span * team_id); 714 *p_ub = *p_lb + span - incr; 715 if (p_last != NULL) 716 *p_last = (team_id == ((trip_count - 1) / (UT)chunk) % nteams); 717 // Correct upper bound if needed 718 if (incr > 0) { 719 if (*p_ub < *p_lb) // overflow? 720 *p_ub = traits_t<T>::max_value; 721 if (*p_ub > upper) 722 *p_ub = upper; // tracker C73258 723 } else { // incr < 0 724 if (*p_ub > *p_lb) 725 *p_ub = traits_t<T>::min_value; 726 if (*p_ub < upper) 727 *p_ub = upper; // tracker C73258 728 } 729 #ifdef KMP_DEBUG 730 { 731 const char *buff; 732 // create format specifiers before the debug output 733 buff = 734 __kmp_str_format("__kmp_team_static_init exit: T#%%d team%%u liter=%%d " 735 "iter=(%%%s, %%%s, %%%s) chunk %%%s\n", 736 traits_t<T>::spec, traits_t<T>::spec, 737 traits_t<ST>::spec, traits_t<ST>::spec); 738 KD_TRACE(100, (buff, gtid, team_id, *p_last, *p_lb, *p_ub, *p_st, chunk)); 739 __kmp_str_free(&buff); 740 } 741 #endif 742 } 743 744 //------------------------------------------------------------------------------ 745 extern "C" { 746 /*! 747 @ingroup WORK_SHARING 748 @param loc Source code location 749 @param gtid Global thread id of this thread 750 @param schedtype Scheduling type 751 @param plastiter Pointer to the "last iteration" flag 752 @param plower Pointer to the lower bound 753 @param pupper Pointer to the upper bound 754 @param pstride Pointer to the stride 755 @param incr Loop increment 756 @param chunk The chunk size 757 758 Each of the four functions here are identical apart from the argument types. 759 760 The functions compute the upper and lower bounds and stride to be used for the 761 set of iterations to be executed by the current thread from the statically 762 scheduled loop that is described by the initial values of the bounds, stride, 763 increment and chunk size. 764 765 @{ 766 */ 767 void __kmpc_for_static_init_4(ident_t *loc, kmp_int32 gtid, kmp_int32 schedtype, 768 kmp_int32 *plastiter, kmp_int32 *plower, 769 kmp_int32 *pupper, kmp_int32 *pstride, 770 kmp_int32 incr, kmp_int32 chunk) { 771 __kmp_for_static_init<kmp_int32>(loc, gtid, schedtype, plastiter, plower, 772 pupper, pstride, incr, chunk 773 #if OMPT_SUPPORT && OMPT_OPTIONAL 774 , 775 OMPT_GET_RETURN_ADDRESS(0) 776 #endif 777 ); 778 } 779 780 /*! 781 See @ref __kmpc_for_static_init_4 782 */ 783 void __kmpc_for_static_init_4u(ident_t *loc, kmp_int32 gtid, 784 kmp_int32 schedtype, kmp_int32 *plastiter, 785 kmp_uint32 *plower, kmp_uint32 *pupper, 786 kmp_int32 *pstride, kmp_int32 incr, 787 kmp_int32 chunk) { 788 __kmp_for_static_init<kmp_uint32>(loc, gtid, schedtype, plastiter, plower, 789 pupper, pstride, incr, chunk 790 #if OMPT_SUPPORT && OMPT_OPTIONAL 791 , 792 OMPT_GET_RETURN_ADDRESS(0) 793 #endif 794 ); 795 } 796 797 /*! 798 See @ref __kmpc_for_static_init_4 799 */ 800 void __kmpc_for_static_init_8(ident_t *loc, kmp_int32 gtid, kmp_int32 schedtype, 801 kmp_int32 *plastiter, kmp_int64 *plower, 802 kmp_int64 *pupper, kmp_int64 *pstride, 803 kmp_int64 incr, kmp_int64 chunk) { 804 __kmp_for_static_init<kmp_int64>(loc, gtid, schedtype, plastiter, plower, 805 pupper, pstride, incr, chunk 806 #if OMPT_SUPPORT && OMPT_OPTIONAL 807 , 808 OMPT_GET_RETURN_ADDRESS(0) 809 #endif 810 ); 811 } 812 813 /*! 814 See @ref __kmpc_for_static_init_4 815 */ 816 void __kmpc_for_static_init_8u(ident_t *loc, kmp_int32 gtid, 817 kmp_int32 schedtype, kmp_int32 *plastiter, 818 kmp_uint64 *plower, kmp_uint64 *pupper, 819 kmp_int64 *pstride, kmp_int64 incr, 820 kmp_int64 chunk) { 821 __kmp_for_static_init<kmp_uint64>(loc, gtid, schedtype, plastiter, plower, 822 pupper, pstride, incr, chunk 823 #if OMPT_SUPPORT && OMPT_OPTIONAL 824 , 825 OMPT_GET_RETURN_ADDRESS(0) 826 #endif 827 ); 828 } 829 /*! 830 @} 831 */ 832 833 /*! 834 @ingroup WORK_SHARING 835 @param loc Source code location 836 @param gtid Global thread id of this thread 837 @param schedule Scheduling type for the parallel loop 838 @param plastiter Pointer to the "last iteration" flag 839 @param plower Pointer to the lower bound 840 @param pupper Pointer to the upper bound of loop chunk 841 @param pupperD Pointer to the upper bound of dist_chunk 842 @param pstride Pointer to the stride for parallel loop 843 @param incr Loop increment 844 @param chunk The chunk size for the parallel loop 845 846 Each of the four functions here are identical apart from the argument types. 847 848 The functions compute the upper and lower bounds and strides to be used for the 849 set of iterations to be executed by the current thread from the statically 850 scheduled loop that is described by the initial values of the bounds, strides, 851 increment and chunks for parallel loop and distribute constructs. 852 853 @{ 854 */ 855 void __kmpc_dist_for_static_init_4(ident_t *loc, kmp_int32 gtid, 856 kmp_int32 schedule, kmp_int32 *plastiter, 857 kmp_int32 *plower, kmp_int32 *pupper, 858 kmp_int32 *pupperD, kmp_int32 *pstride, 859 kmp_int32 incr, kmp_int32 chunk) { 860 __kmp_dist_for_static_init<kmp_int32>(loc, gtid, schedule, plastiter, plower, 861 pupper, pupperD, pstride, incr, chunk); 862 } 863 864 /*! 865 See @ref __kmpc_dist_for_static_init_4 866 */ 867 void __kmpc_dist_for_static_init_4u(ident_t *loc, kmp_int32 gtid, 868 kmp_int32 schedule, kmp_int32 *plastiter, 869 kmp_uint32 *plower, kmp_uint32 *pupper, 870 kmp_uint32 *pupperD, kmp_int32 *pstride, 871 kmp_int32 incr, kmp_int32 chunk) { 872 __kmp_dist_for_static_init<kmp_uint32>(loc, gtid, schedule, plastiter, plower, 873 pupper, pupperD, pstride, incr, chunk); 874 } 875 876 /*! 877 See @ref __kmpc_dist_for_static_init_4 878 */ 879 void __kmpc_dist_for_static_init_8(ident_t *loc, kmp_int32 gtid, 880 kmp_int32 schedule, kmp_int32 *plastiter, 881 kmp_int64 *plower, kmp_int64 *pupper, 882 kmp_int64 *pupperD, kmp_int64 *pstride, 883 kmp_int64 incr, kmp_int64 chunk) { 884 __kmp_dist_for_static_init<kmp_int64>(loc, gtid, schedule, plastiter, plower, 885 pupper, pupperD, pstride, incr, chunk); 886 } 887 888 /*! 889 See @ref __kmpc_dist_for_static_init_4 890 */ 891 void __kmpc_dist_for_static_init_8u(ident_t *loc, kmp_int32 gtid, 892 kmp_int32 schedule, kmp_int32 *plastiter, 893 kmp_uint64 *plower, kmp_uint64 *pupper, 894 kmp_uint64 *pupperD, kmp_int64 *pstride, 895 kmp_int64 incr, kmp_int64 chunk) { 896 __kmp_dist_for_static_init<kmp_uint64>(loc, gtid, schedule, plastiter, plower, 897 pupper, pupperD, pstride, incr, chunk); 898 } 899 /*! 900 @} 901 */ 902 903 //------------------------------------------------------------------------------ 904 // Auxiliary routines for Distribute Parallel Loop construct implementation 905 // Transfer call to template< type T > 906 // __kmp_team_static_init( ident_t *loc, int gtid, 907 // int *p_last, T *lb, T *ub, ST *st, ST incr, ST chunk ) 908 909 /*! 910 @ingroup WORK_SHARING 911 @{ 912 @param loc Source location 913 @param gtid Global thread id 914 @param p_last pointer to last iteration flag 915 @param p_lb pointer to Lower bound 916 @param p_ub pointer to Upper bound 917 @param p_st Step (or increment if you prefer) 918 @param incr Loop increment 919 @param chunk The chunk size to block with 920 921 The functions compute the upper and lower bounds and stride to be used for the 922 set of iterations to be executed by the current team from the statically 923 scheduled loop that is described by the initial values of the bounds, stride, 924 increment and chunk for the distribute construct as part of composite distribute 925 parallel loop construct. These functions are all identical apart from the types 926 of the arguments. 927 */ 928 929 void __kmpc_team_static_init_4(ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last, 930 kmp_int32 *p_lb, kmp_int32 *p_ub, 931 kmp_int32 *p_st, kmp_int32 incr, 932 kmp_int32 chunk) { 933 KMP_DEBUG_ASSERT(__kmp_init_serial); 934 __kmp_team_static_init<kmp_int32>(loc, gtid, p_last, p_lb, p_ub, p_st, incr, 935 chunk); 936 } 937 938 /*! 939 See @ref __kmpc_team_static_init_4 940 */ 941 void __kmpc_team_static_init_4u(ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last, 942 kmp_uint32 *p_lb, kmp_uint32 *p_ub, 943 kmp_int32 *p_st, kmp_int32 incr, 944 kmp_int32 chunk) { 945 KMP_DEBUG_ASSERT(__kmp_init_serial); 946 __kmp_team_static_init<kmp_uint32>(loc, gtid, p_last, p_lb, p_ub, p_st, incr, 947 chunk); 948 } 949 950 /*! 951 See @ref __kmpc_team_static_init_4 952 */ 953 void __kmpc_team_static_init_8(ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last, 954 kmp_int64 *p_lb, kmp_int64 *p_ub, 955 kmp_int64 *p_st, kmp_int64 incr, 956 kmp_int64 chunk) { 957 KMP_DEBUG_ASSERT(__kmp_init_serial); 958 __kmp_team_static_init<kmp_int64>(loc, gtid, p_last, p_lb, p_ub, p_st, incr, 959 chunk); 960 } 961 962 /*! 963 See @ref __kmpc_team_static_init_4 964 */ 965 void __kmpc_team_static_init_8u(ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last, 966 kmp_uint64 *p_lb, kmp_uint64 *p_ub, 967 kmp_int64 *p_st, kmp_int64 incr, 968 kmp_int64 chunk) { 969 KMP_DEBUG_ASSERT(__kmp_init_serial); 970 __kmp_team_static_init<kmp_uint64>(loc, gtid, p_last, p_lb, p_ub, p_st, incr, 971 chunk); 972 } 973 /*! 974 @} 975 */ 976 977 } // extern "C" 978