Lines Matching refs:pr
121 dispatch_private_info_template<T> *pr, in __kmp_dispatch_init_algorithm() argument
150 KD_TRACE(10, (buff, gtid, pr, lb, ub, st, schedule, chunk, nproc, tid)); in __kmp_dispatch_init_algorithm()
167 use_hier = pr->flags.use_hier; in __kmp_dispatch_init_algorithm()
178 pr->flags.nomerge = TRUE; in __kmp_dispatch_init_algorithm()
182 pr->flags.nomerge = FALSE; in __kmp_dispatch_init_algorithm()
184 pr->type_size = traits_t<T>::type_size; // remember the size of variables in __kmp_dispatch_init_algorithm()
186 pr->flags.ordered = TRUE; in __kmp_dispatch_init_algorithm()
190 pr->flags.ordered = FALSE; in __kmp_dispatch_init_algorithm()
193 if (pr->flags.ordered) { in __kmp_dispatch_init_algorithm()
206 if (pr->flags.ordered) // correct monotonicity for ordered loop if needed in __kmp_dispatch_init_algorithm()
303 pr->u.p.parm1 = chunk; in __kmp_dispatch_init_algorithm()
308 pr->u.p.count = 0; in __kmp_dispatch_init_algorithm()
313 (pr->flags.ordered ? ct_pdo_ordered : ct_pdo), loc); in __kmp_dispatch_init_algorithm()
347 pr->u.p.lb = lb; in __kmp_dispatch_init_algorithm()
348 pr->u.p.ub = ub; in __kmp_dispatch_init_algorithm()
349 pr->u.p.st = st; in __kmp_dispatch_init_algorithm()
350 pr->u.p.tc = tc; in __kmp_dispatch_init_algorithm()
353 pr->u.p.last_upper = ub + st; in __kmp_dispatch_init_algorithm()
359 if (pr->flags.ordered) { in __kmp_dispatch_init_algorithm()
360 pr->ordered_bumped = 0; in __kmp_dispatch_init_algorithm()
361 pr->u.p.ordered_lower = 1; in __kmp_dispatch_init_algorithm()
362 pr->u.p.ordered_upper = 0; in __kmp_dispatch_init_algorithm()
381 int claimed = pr->steal_flag.compare_exchange_strong(old, CLAIMED); in __kmp_dispatch_init_algorithm()
388 pr->u.p.steal_lock = (kmp_lock_t *)__kmp_allocate(sizeof(kmp_lock_t)); in __kmp_dispatch_init_algorithm()
389 __kmp_init_lock(pr->u.p.steal_lock); in __kmp_dispatch_init_algorithm()
395 pr->u.p.count = init; in __kmp_dispatch_init_algorithm()
397 pr->u.p.ub = init + small_chunk + (id < extras ? 1 : 0); in __kmp_dispatch_init_algorithm()
400 KMP_ATOMIC_ST_REL(&pr->steal_flag, READY); in __kmp_dispatch_init_algorithm()
403 KMP_DEBUG_ASSERT(pr->steal_flag == THIEF); in __kmp_dispatch_init_algorithm()
404 pr->u.p.ub = init; // mark there is no iterations to work on in __kmp_dispatch_init_algorithm()
406 pr->u.p.parm2 = ntc; // save number of chunks in __kmp_dispatch_init_algorithm()
409 pr->u.p.parm3 = nproc; in __kmp_dispatch_init_algorithm()
410 pr->u.p.parm4 = (id + 1) % nproc; // remember neighbour tid in __kmp_dispatch_init_algorithm()
438 pr->u.p.parm1 = (id == tc - 1); /* parm1 stores *plastiter */ in __kmp_dispatch_init_algorithm()
440 pr->u.p.count = 1; /* means no more chunks to execute */ in __kmp_dispatch_init_algorithm()
441 pr->u.p.parm1 = FALSE; in __kmp_dispatch_init_algorithm()
449 pr->u.p.parm1 = (id == nproc - 1); in __kmp_dispatch_init_algorithm()
455 pr->u.p.parm1 = TRUE; in __kmp_dispatch_init_algorithm()
458 pr->u.p.count = 1; /* means no more chunks to execute */ in __kmp_dispatch_init_algorithm()
459 pr->u.p.parm1 = FALSE; in __kmp_dispatch_init_algorithm()
470 pr->u.p.lb = lb + init; in __kmp_dispatch_init_algorithm()
471 pr->u.p.ub = lb + limit; in __kmp_dispatch_init_algorithm()
475 pr->u.p.lb = lb + init * st; in __kmp_dispatch_init_algorithm()
479 pr->u.p.ub = (ub_tmp + st > ub ? ub : ub_tmp); in __kmp_dispatch_init_algorithm()
481 pr->u.p.ub = (ub_tmp + st < ub ? ub : ub_tmp); in __kmp_dispatch_init_algorithm()
484 if (pr->flags.ordered) { in __kmp_dispatch_init_algorithm()
485 pr->u.p.ordered_lower = init; in __kmp_dispatch_init_algorithm()
486 pr->u.p.ordered_upper = limit; in __kmp_dispatch_init_algorithm()
498 pr->u.p.parm1 = ((tc + nth - 1) / nth + chunk - 1) & ~(chunk - 1); in __kmp_dispatch_init_algorithm()
500 pr->u.p.parm1 = tc; in __kmp_dispatch_init_algorithm()
518 pr->u.p.parm2 = guided_int_param * nproc * (chunk + 1); in __kmp_dispatch_init_algorithm()
519 *(double *)&pr->u.p.parm3 = in __kmp_dispatch_init_algorithm()
532 pr->u.p.parm1 = tc; in __kmp_dispatch_init_algorithm()
588 (((ptrdiff_t)&pr->u.p.parm3) & (natural_alignment)) == 0); in __kmp_dispatch_init_algorithm()
593 *(DBL *)&pr->u.p.parm3 = x; in __kmp_dispatch_init_algorithm()
634 pr->u.p.parm2 = cross; in __kmp_dispatch_init_algorithm()
638 #define GUIDED_ANALYTICAL_WORKAROUND (*(DBL *)&pr->u.p.parm3) in __kmp_dispatch_init_algorithm()
643 pr->u.p.count = tc - in __kmp_dispatch_init_algorithm()
658 pr->u.p.parm1 = tc; in __kmp_dispatch_init_algorithm()
667 pr->u.p.parm1 = (nproc > 1) ? (tc + nproc - 1) / nproc : tc; in __kmp_dispatch_init_algorithm()
674 if (pr->u.p.parm1 <= 0) in __kmp_dispatch_init_algorithm()
675 pr->u.p.parm1 = KMP_DEFAULT_CHUNK; in __kmp_dispatch_init_algorithm()
676 else if (pr->u.p.parm1 > tc) in __kmp_dispatch_init_algorithm()
677 pr->u.p.parm1 = tc; in __kmp_dispatch_init_algorithm()
680 pr->u.p.parm2 = (tc / pr->u.p.parm1) + (tc % pr->u.p.parm1 ? 1 : 0); in __kmp_dispatch_init_algorithm()
727 pr->u.p.parm1 = parm1; in __kmp_dispatch_init_algorithm()
728 pr->u.p.parm2 = parm2; in __kmp_dispatch_init_algorithm()
729 pr->u.p.parm3 = parm3; in __kmp_dispatch_init_algorithm()
730 pr->u.p.parm4 = parm4; in __kmp_dispatch_init_algorithm()
741 pr->schedule = schedule; in __kmp_dispatch_init_algorithm()
810 dispatch_private_info_template<T> *pr; in __kmp_dispatch_init() local
862 pr = reinterpret_cast<dispatch_private_info_template<T> *>( in __kmp_dispatch_init()
870 if (pr->flags.use_hier) { in __kmp_dispatch_init()
875 pr->flags.use_hier = FALSE; in __kmp_dispatch_init()
881 if (!ordered && !pr->flags.use_hier) in __kmp_dispatch_init()
894 pr = reinterpret_cast<dispatch_private_info_template<T> *>( in __kmp_dispatch_init()
903 pr = reinterpret_cast<dispatch_private_info_template<T> *>( in __kmp_dispatch_init()
924 __kmp_dispatch_init_algorithm(loc, gtid, pr, schedule, lb, ub, st, in __kmp_dispatch_init()
931 if (pr->flags.ordered == 0) { in __kmp_dispatch_init()
938 th->th.th_dispatch->th_dispatch_pr_current = (dispatch_private_info_t *)pr; in __kmp_dispatch_init()
942 if (pr->flags.ordered) { in __kmp_dispatch_init()
954 cur_chunk = pr->u.p.parm1; in __kmp_dispatch_init()
970 __kmp_itt_metadata_loop(loc, schedtype, pr->u.p.tc, cur_chunk); in __kmp_dispatch_init()
973 if (pr->flags.use_hier) { in __kmp_dispatch_init()
974 pr->u.p.count = 0; in __kmp_dispatch_init()
975 pr->u.p.ub = pr->u.p.lb = pr->u.p.st = pr->u.p.tc = 0; in __kmp_dispatch_init()
994 KD_TRACE(10, (buff, gtid, pr->schedule, pr->flags.ordered, pr->u.p.lb, in __kmp_dispatch_init()
995 pr->u.p.ub, pr->u.p.st, pr->u.p.tc, pr->u.p.count, in __kmp_dispatch_init()
996 pr->u.p.ordered_lower, pr->u.p.ordered_upper, pr->u.p.parm1, in __kmp_dispatch_init()
997 pr->u.p.parm2, pr->u.p.parm3, pr->u.p.parm4)); in __kmp_dispatch_init()
1007 &(task_info->task_data), pr->u.p.tc, OMPT_LOAD_RETURN_ADDRESS(gtid)); in __kmp_dispatch_init()
1027 dispatch_private_info_template<UT> *pr = in __kmp_dispatch_finish() local
1033 KMP_DEBUG_ASSERT(pr); in __kmp_dispatch_finish()
1038 if (pr->ordered_bumped) { in __kmp_dispatch_finish()
1043 pr->ordered_bumped = 0; in __kmp_dispatch_finish()
1045 UT lower = pr->u.p.ordered_lower; in __kmp_dispatch_finish()
1090 dispatch_private_info_template<UT> *pr = in __kmp_dispatch_finish_chunk() local
1096 KMP_DEBUG_ASSERT(pr); in __kmp_dispatch_finish_chunk()
1101 UT lower = pr->u.p.ordered_lower; in __kmp_dispatch_finish_chunk()
1102 UT upper = pr->u.p.ordered_upper; in __kmp_dispatch_finish_chunk()
1105 if (pr->ordered_bumped == inc) { in __kmp_dispatch_finish_chunk()
1110 pr->ordered_bumped = 0; in __kmp_dispatch_finish_chunk()
1112 inc -= pr->ordered_bumped; in __kmp_dispatch_finish_chunk()
1134 pr->ordered_bumped = 0; in __kmp_dispatch_finish_chunk()
1162 dispatch_private_info_template<T> *pr, in __kmp_dispatch_next_algorithm() argument
1180 KMP_DEBUG_ASSERT(pr); in __kmp_dispatch_next_algorithm()
1191 KD_TRACE(10, (buff, gtid, pr, sh, nproc, tid)); in __kmp_dispatch_next_algorithm()
1197 if (pr->u.p.tc == 0) { in __kmp_dispatch_next_algorithm()
1205 switch (pr->schedule) { in __kmp_dispatch_next_algorithm()
1208 T chunk = pr->u.p.parm1; in __kmp_dispatch_next_algorithm()
1209 UT nchunks = pr->u.p.parm2; in __kmp_dispatch_next_algorithm()
1214 trip = pr->u.p.tc - 1; in __kmp_dispatch_next_algorithm()
1219 kmp_lock_t *lck = pr->u.p.steal_lock; in __kmp_dispatch_next_algorithm()
1221 if (pr->u.p.count < (UT)pr->u.p.ub) { in __kmp_dispatch_next_algorithm()
1222 KMP_DEBUG_ASSERT(pr->steal_flag == READY); in __kmp_dispatch_next_algorithm()
1225 init = (pr->u.p.count)++; in __kmp_dispatch_next_algorithm()
1226 status = (init < (UT)pr->u.p.ub); in __kmp_dispatch_next_algorithm()
1233 T while_limit = pr->u.p.parm3; in __kmp_dispatch_next_algorithm()
1238 KMP_ATOMIC_ST_REL(&pr->steal_flag, THIEF); // mark self buffer inactive in __kmp_dispatch_next_algorithm()
1242 T victimId = pr->u.p.parm4; in __kmp_dispatch_next_algorithm()
1247 while ((v == pr || KMP_ATOMIC_LD_RLX(&v->steal_flag) == THIEF) && in __kmp_dispatch_next_algorithm()
1254 if (v == pr || KMP_ATOMIC_LD_RLX(&v->steal_flag) == THIEF) { in __kmp_dispatch_next_algorithm()
1269 pr->u.p.count = init + 1; // exclude one we execute immediately in __kmp_dispatch_next_algorithm()
1270 pr->u.p.ub = init + small_chunk + (id < extras ? 1 : 0); in __kmp_dispatch_next_algorithm()
1272 pr->u.p.parm4 = (id + 1) % nproc; // remember neighbour tid in __kmp_dispatch_next_algorithm()
1282 KD_TRACE(10, (buff, gtid, id, pr->u.p.count, pr->u.p.ub)); in __kmp_dispatch_next_algorithm()
1287 if (pr->u.p.count < (UT)pr->u.p.ub) in __kmp_dispatch_next_algorithm()
1288 KMP_ATOMIC_ST_REL(&pr->steal_flag, READY); in __kmp_dispatch_next_algorithm()
1294 pr->u.p.parm4 = (victimId + 1) % nproc; // shift start victim tid in __kmp_dispatch_next_algorithm()
1303 pr->u.p.parm4 = (victimId + 1) % nproc; // shift start victim tid in __kmp_dispatch_next_algorithm()
1333 pr->u.p.parm4 = victimId; // remember victim to steal from in __kmp_dispatch_next_algorithm()
1337 pr->u.p.count = init + 1; in __kmp_dispatch_next_algorithm()
1338 pr->u.p.ub = limit; in __kmp_dispatch_next_algorithm()
1342 KMP_ATOMIC_ST_REL(&pr->steal_flag, READY); in __kmp_dispatch_next_algorithm()
1356 if (pr->u.p.count < (UT)pr->u.p.ub) { in __kmp_dispatch_next_algorithm()
1357 KMP_DEBUG_ASSERT(pr->steal_flag == READY); in __kmp_dispatch_next_algorithm()
1358 vold.b = *(volatile kmp_int64 *)(&pr->u.p.count); in __kmp_dispatch_next_algorithm()
1362 (volatile kmp_int64 *)&pr->u.p.count, in __kmp_dispatch_next_algorithm()
1366 vold.b = *(volatile kmp_int64 *)(&pr->u.p.count); in __kmp_dispatch_next_algorithm()
1376 T while_limit = pr->u.p.parm3; in __kmp_dispatch_next_algorithm()
1381 KMP_ATOMIC_ST_REL(&pr->steal_flag, THIEF); // mark self buffer inactive in __kmp_dispatch_next_algorithm()
1385 T victimId = pr->u.p.parm4; in __kmp_dispatch_next_algorithm()
1390 while ((v == pr || KMP_ATOMIC_LD_RLX(&v->steal_flag) == THIEF) && in __kmp_dispatch_next_algorithm()
1397 if (v == pr || KMP_ATOMIC_LD_RLX(&v->steal_flag) == THIEF) { in __kmp_dispatch_next_algorithm()
1415 KMP_XCHG_FIXED64((volatile kmp_int64 *)(&pr->u.p.count), vnew.b); in __kmp_dispatch_next_algorithm()
1417 *(volatile kmp_int64 *)(&pr->u.p.count) = vnew.b; in __kmp_dispatch_next_algorithm()
1419 pr->u.p.parm4 = (id + 1) % nproc; // remember neighbour tid in __kmp_dispatch_next_algorithm()
1429 KD_TRACE(10, (buff, gtid, id, pr->u.p.count, pr->u.p.ub)); in __kmp_dispatch_next_algorithm()
1434 if (pr->u.p.count < (UT)pr->u.p.ub) in __kmp_dispatch_next_algorithm()
1435 KMP_ATOMIC_ST_REL(&pr->steal_flag, READY); in __kmp_dispatch_next_algorithm()
1444 pr->u.p.parm4 = (victimId + 1) % nproc; // shift start victim id in __kmp_dispatch_next_algorithm()
1477 pr->u.p.parm4 = victimId; // keep victim id in __kmp_dispatch_next_algorithm()
1482 KMP_XCHG_FIXED64((volatile kmp_int64 *)(&pr->u.p.count), vold.b); in __kmp_dispatch_next_algorithm()
1484 *(volatile kmp_int64 *)(&pr->u.p.count) = vold.b; in __kmp_dispatch_next_algorithm()
1488 KMP_ATOMIC_ST_REL(&pr->steal_flag, READY); in __kmp_dispatch_next_algorithm()
1502 start = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1505 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1534 if ((status = !pr->u.p.count) != 0) { in __kmp_dispatch_next_algorithm()
1535 pr->u.p.count = 1; in __kmp_dispatch_next_algorithm()
1536 *p_lb = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1537 *p_ub = pr->u.p.ub; in __kmp_dispatch_next_algorithm()
1538 last = (pr->u.p.parm1 != 0); in __kmp_dispatch_next_algorithm()
1540 *p_st = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1542 pr->u.p.lb = pr->u.p.ub + pr->u.p.st; in __kmp_dispatch_next_algorithm()
1554 parm1 = pr->u.p.parm1; in __kmp_dispatch_next_algorithm()
1556 trip = pr->u.p.tc - 1; in __kmp_dispatch_next_algorithm()
1557 init = parm1 * (pr->u.p.count + tid); in __kmp_dispatch_next_algorithm()
1560 start = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1561 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1570 pr->u.p.count += nproc; in __kmp_dispatch_next_algorithm()
1580 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1581 pr->u.p.ordered_lower = init; in __kmp_dispatch_next_algorithm()
1582 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next_algorithm()
1590 UT chunk_size = pr->u.p.parm1; in __kmp_dispatch_next_algorithm()
1591 UT nchunks = pr->u.p.parm2; in __kmp_dispatch_next_algorithm()
1607 trip = pr->u.p.tc - 1; in __kmp_dispatch_next_algorithm()
1608 start = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1609 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1627 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1628 pr->u.p.ordered_lower = init; in __kmp_dispatch_next_algorithm()
1629 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next_algorithm()
1636 T chunkspec = pr->u.p.parm1; in __kmp_dispatch_next_algorithm()
1640 trip = pr->u.p.tc; in __kmp_dispatch_next_algorithm()
1652 pr->u.p.parm2) { // compare with K*nproc*(chunk+1), K=2 by default in __kmp_dispatch_next_algorithm()
1673 *(double *)&pr->u.p.parm3); // divide by K*nproc in __kmp_dispatch_next_algorithm()
1683 start = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1684 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1689 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1690 pr->u.p.ordered_lower = init; in __kmp_dispatch_next_algorithm()
1691 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next_algorithm()
1705 T chunk = pr->u.p.parm1; in __kmp_dispatch_next_algorithm()
1709 trip = pr->u.p.tc; in __kmp_dispatch_next_algorithm()
1721 if ((T)remaining < pr->u.p.parm2) { in __kmp_dispatch_next_algorithm()
1743 __kmp_type_convert((double)remaining * (*(double *)&pr->u.p.parm3), in __kmp_dispatch_next_algorithm()
1758 start = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1759 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1764 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1765 pr->u.p.ordered_lower = init; in __kmp_dispatch_next_algorithm()
1766 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next_algorithm()
1778 T chunkspec = pr->u.p.parm1; in __kmp_dispatch_next_algorithm()
1790 trip = pr->u.p.tc; in __kmp_dispatch_next_algorithm()
1798 if (chunkIdx >= (UT)pr->u.p.parm2) { in __kmp_dispatch_next_algorithm()
1801 init = chunkIdx * chunkspec + pr->u.p.count; in __kmp_dispatch_next_algorithm()
1829 trip, *(DBL *)&pr->u.p.parm3, chunkIdx); in __kmp_dispatch_next_algorithm()
1835 trip, *(DBL *)&pr->u.p.parm3, chunkIdx + 1); in __kmp_dispatch_next_algorithm()
1853 start = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1854 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1859 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1860 pr->u.p.ordered_lower = init; in __kmp_dispatch_next_algorithm()
1861 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next_algorithm()
1874 T parm2 = pr->u.p.parm2; in __kmp_dispatch_next_algorithm()
1875 T parm3 = pr->u.p.parm3; in __kmp_dispatch_next_algorithm()
1876 T parm4 = pr->u.p.parm4; in __kmp_dispatch_next_algorithm()
1884 trip = pr->u.p.tc - 1; in __kmp_dispatch_next_algorithm()
1892 start = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1894 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1910 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1911 pr->u.p.ordered_lower = init; in __kmp_dispatch_next_algorithm()
1912 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next_algorithm()
1928 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1934 KD_TRACE(1000, (buff, gtid, pr->u.p.ordered_lower, pr->u.p.ordered_upper)); in __kmp_dispatch_next_algorithm()
1991 i = (kmp_int64)(pr->u.p.st); \
2036 dispatch_private_info_template<T> *pr; in __kmp_dispatch_next() local
2049 pr = reinterpret_cast<dispatch_private_info_template<T> *>( in __kmp_dispatch_next()
2051 KMP_DEBUG_ASSERT(pr); in __kmp_dispatch_next()
2053 if ((status = (pr->u.p.tc != 0)) == 0) { in __kmp_dispatch_next()
2061 if (pr->pushed_ws != ct_none) { in __kmp_dispatch_next()
2062 pr->pushed_ws = __kmp_pop_workshare(gtid, pr->pushed_ws, loc); in __kmp_dispatch_next()
2065 } else if (pr->flags.nomerge) { in __kmp_dispatch_next()
2070 T chunk = pr->u.p.parm1; in __kmp_dispatch_next()
2075 init = chunk * pr->u.p.count++; in __kmp_dispatch_next()
2076 trip = pr->u.p.tc - 1; in __kmp_dispatch_next()
2086 if (pr->pushed_ws != ct_none) { in __kmp_dispatch_next()
2087 pr->pushed_ws = __kmp_pop_workshare(gtid, pr->pushed_ws, loc); in __kmp_dispatch_next()
2091 start = pr->u.p.lb; in __kmp_dispatch_next()
2093 incr = pr->u.p.st; in __kmp_dispatch_next()
2098 pr->u.p.last_upper = pr->u.p.ub; in __kmp_dispatch_next()
2113 if (pr->flags.ordered) { in __kmp_dispatch_next()
2114 pr->u.p.ordered_lower = init; in __kmp_dispatch_next()
2115 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next()
2123 KD_TRACE(1000, (buff, gtid, pr->u.p.ordered_lower, in __kmp_dispatch_next()
2124 pr->u.p.ordered_upper)); in __kmp_dispatch_next()
2131 pr->u.p.tc = 0; in __kmp_dispatch_next()
2132 *p_lb = pr->u.p.lb; in __kmp_dispatch_next()
2133 *p_ub = pr->u.p.ub; in __kmp_dispatch_next()
2135 pr->u.p.last_upper = *p_ub; in __kmp_dispatch_next()
2140 *p_st = pr->u.p.st; in __kmp_dispatch_next()
2158 OMPT_LOOP_DISPATCH(*p_lb, *p_ub, pr->u.p.st, status); in __kmp_dispatch_next()
2169 pr = reinterpret_cast<dispatch_private_info_template<T> *>( in __kmp_dispatch_next()
2171 KMP_DEBUG_ASSERT(pr); in __kmp_dispatch_next()
2177 if (pr->flags.use_hier) in __kmp_dispatch_next()
2178 status = sh->hier->next(loc, gtid, pr, &last, p_lb, p_ub, p_st); in __kmp_dispatch_next()
2181 status = __kmp_dispatch_next_algorithm<T>(gtid, pr, sh, &last, p_lb, p_ub, in __kmp_dispatch_next()
2201 pr->flags.use_hier = FALSE; in __kmp_dispatch_next()
2205 if (pr->schedule == kmp_sch_static_steal) { in __kmp_dispatch_next()
2235 if (pr->flags.ordered) { in __kmp_dispatch_next()
2247 if (pr->pushed_ws != ct_none) { in __kmp_dispatch_next()
2248 pr->pushed_ws = __kmp_pop_workshare(gtid, pr->pushed_ws, loc); in __kmp_dispatch_next()
2259 pr->u.p.last_upper = pr->u.p.ub; in __kmp_dispatch_next()
2282 OMPT_LOOP_DISPATCH(*p_lb, *p_ub, pr->u.p.st, status); in __kmp_dispatch_next()