Lines Matching refs:pr
121 dispatch_private_info_template<T> *pr, in __kmp_dispatch_init_algorithm() argument
150 KD_TRACE(10, (buff, gtid, pr, lb, ub, st, schedule, chunk, nproc, tid)); in __kmp_dispatch_init_algorithm()
167 use_hier = pr->flags.use_hier; in __kmp_dispatch_init_algorithm()
178 pr->flags.nomerge = TRUE; in __kmp_dispatch_init_algorithm()
182 pr->flags.nomerge = FALSE; in __kmp_dispatch_init_algorithm()
184 pr->type_size = traits_t<T>::type_size; // remember the size of variables in __kmp_dispatch_init_algorithm()
186 pr->flags.ordered = TRUE; in __kmp_dispatch_init_algorithm()
190 pr->flags.ordered = FALSE; in __kmp_dispatch_init_algorithm()
193 if (pr->flags.ordered) { in __kmp_dispatch_init_algorithm()
206 if (pr->flags.ordered) // correct monotonicity for ordered loop if needed in __kmp_dispatch_init_algorithm()
303 pr->u.p.parm1 = chunk; in __kmp_dispatch_init_algorithm()
308 pr->u.p.count = 0; in __kmp_dispatch_init_algorithm()
313 (pr->flags.ordered ? ct_pdo_ordered : ct_pdo), loc); in __kmp_dispatch_init_algorithm()
347 pr->u.p.lb = lb; in __kmp_dispatch_init_algorithm()
348 pr->u.p.ub = ub; in __kmp_dispatch_init_algorithm()
349 pr->u.p.st = st; in __kmp_dispatch_init_algorithm()
350 pr->u.p.tc = tc; in __kmp_dispatch_init_algorithm()
353 pr->u.p.last_upper = ub + st; in __kmp_dispatch_init_algorithm()
359 if (pr->flags.ordered) { in __kmp_dispatch_init_algorithm()
360 pr->ordered_bumped = 0; in __kmp_dispatch_init_algorithm()
361 pr->u.p.ordered_lower = 1; in __kmp_dispatch_init_algorithm()
362 pr->u.p.ordered_upper = 0; in __kmp_dispatch_init_algorithm()
381 int claimed = pr->steal_flag.compare_exchange_strong(old, CLAIMED); in __kmp_dispatch_init_algorithm()
388 pr->u.p.steal_lock = (kmp_lock_t *)__kmp_allocate(sizeof(kmp_lock_t)); in __kmp_dispatch_init_algorithm()
389 __kmp_init_lock(pr->u.p.steal_lock); in __kmp_dispatch_init_algorithm()
395 pr->u.p.count = init; in __kmp_dispatch_init_algorithm()
397 pr->u.p.ub = init + small_chunk + (id < extras ? 1 : 0); in __kmp_dispatch_init_algorithm()
400 KMP_ATOMIC_ST_REL(&pr->steal_flag, READY); in __kmp_dispatch_init_algorithm()
403 KMP_DEBUG_ASSERT(pr->steal_flag == THIEF); in __kmp_dispatch_init_algorithm()
404 pr->u.p.ub = init; // mark there is no iterations to work on in __kmp_dispatch_init_algorithm()
406 pr->u.p.parm2 = ntc; // save number of chunks in __kmp_dispatch_init_algorithm()
409 pr->u.p.parm3 = nproc; in __kmp_dispatch_init_algorithm()
410 pr->u.p.parm4 = (id + 1) % nproc; // remember neighbour tid in __kmp_dispatch_init_algorithm()
438 pr->u.p.parm1 = (id == tc - 1); /* parm1 stores *plastiter */ in __kmp_dispatch_init_algorithm()
440 pr->u.p.count = 1; /* means no more chunks to execute */ in __kmp_dispatch_init_algorithm()
441 pr->u.p.parm1 = FALSE; in __kmp_dispatch_init_algorithm()
449 pr->u.p.parm1 = (id == nproc - 1); in __kmp_dispatch_init_algorithm()
455 pr->u.p.parm1 = TRUE; in __kmp_dispatch_init_algorithm()
458 pr->u.p.count = 1; /* means no more chunks to execute */ in __kmp_dispatch_init_algorithm()
459 pr->u.p.parm1 = FALSE; in __kmp_dispatch_init_algorithm()
470 pr->u.p.lb = lb + init; in __kmp_dispatch_init_algorithm()
471 pr->u.p.ub = lb + limit; in __kmp_dispatch_init_algorithm()
475 pr->u.p.lb = lb + init * st; in __kmp_dispatch_init_algorithm()
479 pr->u.p.ub = (ub_tmp + st > ub ? ub : ub_tmp); in __kmp_dispatch_init_algorithm()
481 pr->u.p.ub = (ub_tmp + st < ub ? ub : ub_tmp); in __kmp_dispatch_init_algorithm()
484 if (pr->flags.ordered) { in __kmp_dispatch_init_algorithm()
485 pr->u.p.ordered_lower = init; in __kmp_dispatch_init_algorithm()
486 pr->u.p.ordered_upper = limit; in __kmp_dispatch_init_algorithm()
498 pr->u.p.parm1 = ((tc + nth - 1) / nth + chunk - 1) & ~(chunk - 1); in __kmp_dispatch_init_algorithm()
500 pr->u.p.parm1 = tc; in __kmp_dispatch_init_algorithm()
518 pr->u.p.parm2 = guided_int_param * nproc * (chunk + 1); in __kmp_dispatch_init_algorithm()
519 *(double *)&pr->u.p.parm3 = in __kmp_dispatch_init_algorithm()
532 pr->u.p.parm1 = tc; in __kmp_dispatch_init_algorithm()
587 (((ptrdiff_t)&pr->u.p.parm3) & (natural_alignment)) == 0); in __kmp_dispatch_init_algorithm()
592 *(DBL *)&pr->u.p.parm3 = x; in __kmp_dispatch_init_algorithm()
633 pr->u.p.parm2 = cross; in __kmp_dispatch_init_algorithm()
637 #define GUIDED_ANALYTICAL_WORKAROUND (*(DBL *)&pr->u.p.parm3) in __kmp_dispatch_init_algorithm()
642 pr->u.p.count = tc - in __kmp_dispatch_init_algorithm()
657 pr->u.p.parm1 = tc; in __kmp_dispatch_init_algorithm()
666 pr->u.p.parm1 = (nproc > 1) ? (tc + nproc - 1) / nproc : tc; in __kmp_dispatch_init_algorithm()
671 if (pr->u.p.parm1 <= 0) in __kmp_dispatch_init_algorithm()
672 pr->u.p.parm1 = KMP_DEFAULT_CHUNK; in __kmp_dispatch_init_algorithm()
673 else if (pr->u.p.parm1 > tc) in __kmp_dispatch_init_algorithm()
674 pr->u.p.parm1 = tc; in __kmp_dispatch_init_algorithm()
677 pr->u.p.parm2 = (tc / pr->u.p.parm1) + (tc % pr->u.p.parm1 ? 1 : 0); in __kmp_dispatch_init_algorithm()
724 pr->u.p.parm1 = parm1; in __kmp_dispatch_init_algorithm()
725 pr->u.p.parm2 = parm2; in __kmp_dispatch_init_algorithm()
726 pr->u.p.parm3 = parm3; in __kmp_dispatch_init_algorithm()
727 pr->u.p.parm4 = parm4; in __kmp_dispatch_init_algorithm()
738 pr->schedule = schedule; in __kmp_dispatch_init_algorithm()
807 dispatch_private_info_template<T> *pr; in __kmp_dispatch_init() local
859 pr = reinterpret_cast<dispatch_private_info_template<T> *>( in __kmp_dispatch_init()
867 if (pr->flags.use_hier) { in __kmp_dispatch_init()
872 pr->flags.use_hier = FALSE; in __kmp_dispatch_init()
878 if (!ordered && !pr->flags.use_hier) in __kmp_dispatch_init()
891 pr = reinterpret_cast<dispatch_private_info_template<T> *>( in __kmp_dispatch_init()
900 pr = reinterpret_cast<dispatch_private_info_template<T> *>( in __kmp_dispatch_init()
921 __kmp_dispatch_init_algorithm(loc, gtid, pr, schedule, lb, ub, st, in __kmp_dispatch_init()
928 if (pr->flags.ordered == 0) { in __kmp_dispatch_init()
935 th->th.th_dispatch->th_dispatch_pr_current = (dispatch_private_info_t *)pr; in __kmp_dispatch_init()
939 if (pr->flags.ordered) { in __kmp_dispatch_init()
951 cur_chunk = pr->u.p.parm1; in __kmp_dispatch_init()
967 __kmp_itt_metadata_loop(loc, schedtype, pr->u.p.tc, cur_chunk); in __kmp_dispatch_init()
970 if (pr->flags.use_hier) { in __kmp_dispatch_init()
971 pr->u.p.count = 0; in __kmp_dispatch_init()
972 pr->u.p.ub = pr->u.p.lb = pr->u.p.st = pr->u.p.tc = 0; in __kmp_dispatch_init()
991 KD_TRACE(10, (buff, gtid, pr->schedule, pr->flags.ordered, pr->u.p.lb, in __kmp_dispatch_init()
992 pr->u.p.ub, pr->u.p.st, pr->u.p.tc, pr->u.p.count, in __kmp_dispatch_init()
993 pr->u.p.ordered_lower, pr->u.p.ordered_upper, pr->u.p.parm1, in __kmp_dispatch_init()
994 pr->u.p.parm2, pr->u.p.parm3, pr->u.p.parm4)); in __kmp_dispatch_init()
1004 &(task_info->task_data), pr->u.p.tc, OMPT_LOAD_RETURN_ADDRESS(gtid)); in __kmp_dispatch_init()
1024 dispatch_private_info_template<UT> *pr = in __kmp_dispatch_finish() local
1030 KMP_DEBUG_ASSERT(pr); in __kmp_dispatch_finish()
1035 if (pr->ordered_bumped) { in __kmp_dispatch_finish()
1040 pr->ordered_bumped = 0; in __kmp_dispatch_finish()
1042 UT lower = pr->u.p.ordered_lower; in __kmp_dispatch_finish()
1087 dispatch_private_info_template<UT> *pr = in __kmp_dispatch_finish_chunk() local
1093 KMP_DEBUG_ASSERT(pr); in __kmp_dispatch_finish_chunk()
1098 UT lower = pr->u.p.ordered_lower; in __kmp_dispatch_finish_chunk()
1099 UT upper = pr->u.p.ordered_upper; in __kmp_dispatch_finish_chunk()
1102 if (pr->ordered_bumped == inc) { in __kmp_dispatch_finish_chunk()
1107 pr->ordered_bumped = 0; in __kmp_dispatch_finish_chunk()
1109 inc -= pr->ordered_bumped; in __kmp_dispatch_finish_chunk()
1131 pr->ordered_bumped = 0; in __kmp_dispatch_finish_chunk()
1159 dispatch_private_info_template<T> *pr, in __kmp_dispatch_next_algorithm() argument
1177 KMP_DEBUG_ASSERT(pr); in __kmp_dispatch_next_algorithm()
1188 KD_TRACE(10, (buff, gtid, pr, sh, nproc, tid)); in __kmp_dispatch_next_algorithm()
1194 if (pr->u.p.tc == 0) { in __kmp_dispatch_next_algorithm()
1202 switch (pr->schedule) { in __kmp_dispatch_next_algorithm()
1205 T chunk = pr->u.p.parm1; in __kmp_dispatch_next_algorithm()
1206 UT nchunks = pr->u.p.parm2; in __kmp_dispatch_next_algorithm()
1211 trip = pr->u.p.tc - 1; in __kmp_dispatch_next_algorithm()
1216 kmp_lock_t *lck = pr->u.p.steal_lock; in __kmp_dispatch_next_algorithm()
1218 if (pr->u.p.count < (UT)pr->u.p.ub) { in __kmp_dispatch_next_algorithm()
1219 KMP_DEBUG_ASSERT(pr->steal_flag == READY); in __kmp_dispatch_next_algorithm()
1222 init = (pr->u.p.count)++; in __kmp_dispatch_next_algorithm()
1223 status = (init < (UT)pr->u.p.ub); in __kmp_dispatch_next_algorithm()
1230 T while_limit = pr->u.p.parm3; in __kmp_dispatch_next_algorithm()
1235 KMP_ATOMIC_ST_REL(&pr->steal_flag, THIEF); // mark self buffer inactive in __kmp_dispatch_next_algorithm()
1239 T victimId = pr->u.p.parm4; in __kmp_dispatch_next_algorithm()
1244 while ((v == pr || KMP_ATOMIC_LD_RLX(&v->steal_flag) == THIEF) && in __kmp_dispatch_next_algorithm()
1251 if (v == pr || KMP_ATOMIC_LD_RLX(&v->steal_flag) == THIEF) { in __kmp_dispatch_next_algorithm()
1266 pr->u.p.count = init + 1; // exclude one we execute immediately in __kmp_dispatch_next_algorithm()
1267 pr->u.p.ub = init + small_chunk + (id < extras ? 1 : 0); in __kmp_dispatch_next_algorithm()
1269 pr->u.p.parm4 = (id + 1) % nproc; // remember neighbour tid in __kmp_dispatch_next_algorithm()
1279 KD_TRACE(10, (buff, gtid, id, pr->u.p.count, pr->u.p.ub)); in __kmp_dispatch_next_algorithm()
1284 if (pr->u.p.count < (UT)pr->u.p.ub) in __kmp_dispatch_next_algorithm()
1285 KMP_ATOMIC_ST_REL(&pr->steal_flag, READY); in __kmp_dispatch_next_algorithm()
1291 pr->u.p.parm4 = (victimId + 1) % nproc; // shift start victim tid in __kmp_dispatch_next_algorithm()
1300 pr->u.p.parm4 = (victimId + 1) % nproc; // shift start victim tid in __kmp_dispatch_next_algorithm()
1330 pr->u.p.parm4 = victimId; // remember victim to steal from in __kmp_dispatch_next_algorithm()
1334 pr->u.p.count = init + 1; in __kmp_dispatch_next_algorithm()
1335 pr->u.p.ub = limit; in __kmp_dispatch_next_algorithm()
1339 KMP_ATOMIC_ST_REL(&pr->steal_flag, READY); in __kmp_dispatch_next_algorithm()
1353 if (pr->u.p.count < (UT)pr->u.p.ub) { in __kmp_dispatch_next_algorithm()
1354 KMP_DEBUG_ASSERT(pr->steal_flag == READY); in __kmp_dispatch_next_algorithm()
1355 vold.b = *(volatile kmp_int64 *)(&pr->u.p.count); in __kmp_dispatch_next_algorithm()
1359 (volatile kmp_int64 *)&pr->u.p.count, in __kmp_dispatch_next_algorithm()
1363 vold.b = *(volatile kmp_int64 *)(&pr->u.p.count); in __kmp_dispatch_next_algorithm()
1373 T while_limit = pr->u.p.parm3; in __kmp_dispatch_next_algorithm()
1378 KMP_ATOMIC_ST_REL(&pr->steal_flag, THIEF); // mark self buffer inactive in __kmp_dispatch_next_algorithm()
1382 T victimId = pr->u.p.parm4; in __kmp_dispatch_next_algorithm()
1387 while ((v == pr || KMP_ATOMIC_LD_RLX(&v->steal_flag) == THIEF) && in __kmp_dispatch_next_algorithm()
1394 if (v == pr || KMP_ATOMIC_LD_RLX(&v->steal_flag) == THIEF) { in __kmp_dispatch_next_algorithm()
1412 KMP_XCHG_FIXED64((volatile kmp_int64 *)(&pr->u.p.count), vnew.b); in __kmp_dispatch_next_algorithm()
1414 *(volatile kmp_int64 *)(&pr->u.p.count) = vnew.b; in __kmp_dispatch_next_algorithm()
1416 pr->u.p.parm4 = (id + 1) % nproc; // remember neighbour tid in __kmp_dispatch_next_algorithm()
1426 KD_TRACE(10, (buff, gtid, id, pr->u.p.count, pr->u.p.ub)); in __kmp_dispatch_next_algorithm()
1431 if (pr->u.p.count < (UT)pr->u.p.ub) in __kmp_dispatch_next_algorithm()
1432 KMP_ATOMIC_ST_REL(&pr->steal_flag, READY); in __kmp_dispatch_next_algorithm()
1441 pr->u.p.parm4 = (victimId + 1) % nproc; // shift start victim id in __kmp_dispatch_next_algorithm()
1474 pr->u.p.parm4 = victimId; // keep victim id in __kmp_dispatch_next_algorithm()
1479 KMP_XCHG_FIXED64((volatile kmp_int64 *)(&pr->u.p.count), vold.b); in __kmp_dispatch_next_algorithm()
1481 *(volatile kmp_int64 *)(&pr->u.p.count) = vold.b; in __kmp_dispatch_next_algorithm()
1485 KMP_ATOMIC_ST_REL(&pr->steal_flag, READY); in __kmp_dispatch_next_algorithm()
1499 start = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1502 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1531 if ((status = !pr->u.p.count) != 0) { in __kmp_dispatch_next_algorithm()
1532 pr->u.p.count = 1; in __kmp_dispatch_next_algorithm()
1533 *p_lb = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1534 *p_ub = pr->u.p.ub; in __kmp_dispatch_next_algorithm()
1535 last = (pr->u.p.parm1 != 0); in __kmp_dispatch_next_algorithm()
1537 *p_st = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1539 pr->u.p.lb = pr->u.p.ub + pr->u.p.st; in __kmp_dispatch_next_algorithm()
1551 parm1 = pr->u.p.parm1; in __kmp_dispatch_next_algorithm()
1553 trip = pr->u.p.tc - 1; in __kmp_dispatch_next_algorithm()
1554 init = parm1 * (pr->u.p.count + tid); in __kmp_dispatch_next_algorithm()
1557 start = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1558 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1567 pr->u.p.count += nproc; in __kmp_dispatch_next_algorithm()
1577 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1578 pr->u.p.ordered_lower = init; in __kmp_dispatch_next_algorithm()
1579 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next_algorithm()
1587 UT chunk_size = pr->u.p.parm1; in __kmp_dispatch_next_algorithm()
1588 UT nchunks = pr->u.p.parm2; in __kmp_dispatch_next_algorithm()
1604 trip = pr->u.p.tc - 1; in __kmp_dispatch_next_algorithm()
1605 start = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1606 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1624 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1625 pr->u.p.ordered_lower = init; in __kmp_dispatch_next_algorithm()
1626 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next_algorithm()
1633 T chunkspec = pr->u.p.parm1; in __kmp_dispatch_next_algorithm()
1637 trip = pr->u.p.tc; in __kmp_dispatch_next_algorithm()
1649 pr->u.p.parm2) { // compare with K*nproc*(chunk+1), K=2 by default in __kmp_dispatch_next_algorithm()
1670 *(double *)&pr->u.p.parm3); // divide by K*nproc in __kmp_dispatch_next_algorithm()
1680 start = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1681 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1686 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1687 pr->u.p.ordered_lower = init; in __kmp_dispatch_next_algorithm()
1688 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next_algorithm()
1702 T chunk = pr->u.p.parm1; in __kmp_dispatch_next_algorithm()
1706 trip = pr->u.p.tc; in __kmp_dispatch_next_algorithm()
1718 if ((T)remaining < pr->u.p.parm2) { in __kmp_dispatch_next_algorithm()
1740 __kmp_type_convert((double)remaining * (*(double *)&pr->u.p.parm3), in __kmp_dispatch_next_algorithm()
1755 start = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1756 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1761 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1762 pr->u.p.ordered_lower = init; in __kmp_dispatch_next_algorithm()
1763 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next_algorithm()
1775 T chunkspec = pr->u.p.parm1; in __kmp_dispatch_next_algorithm()
1787 trip = pr->u.p.tc; in __kmp_dispatch_next_algorithm()
1795 if (chunkIdx >= (UT)pr->u.p.parm2) { in __kmp_dispatch_next_algorithm()
1798 init = chunkIdx * chunkspec + pr->u.p.count; in __kmp_dispatch_next_algorithm()
1826 trip, *(DBL *)&pr->u.p.parm3, chunkIdx); in __kmp_dispatch_next_algorithm()
1832 trip, *(DBL *)&pr->u.p.parm3, chunkIdx + 1); in __kmp_dispatch_next_algorithm()
1850 start = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1851 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1856 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1857 pr->u.p.ordered_lower = init; in __kmp_dispatch_next_algorithm()
1858 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next_algorithm()
1871 T parm2 = pr->u.p.parm2; in __kmp_dispatch_next_algorithm()
1872 T parm3 = pr->u.p.parm3; in __kmp_dispatch_next_algorithm()
1873 T parm4 = pr->u.p.parm4; in __kmp_dispatch_next_algorithm()
1881 trip = pr->u.p.tc - 1; in __kmp_dispatch_next_algorithm()
1889 start = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1891 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1907 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1908 pr->u.p.ordered_lower = init; in __kmp_dispatch_next_algorithm()
1909 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next_algorithm()
1925 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1931 KD_TRACE(1000, (buff, gtid, pr->u.p.ordered_lower, pr->u.p.ordered_upper)); in __kmp_dispatch_next_algorithm()
1975 i = (kmp_int64)(pr->u.p.st); \
2020 dispatch_private_info_template<T> *pr; in __kmp_dispatch_next() local
2033 pr = reinterpret_cast<dispatch_private_info_template<T> *>( in __kmp_dispatch_next()
2035 KMP_DEBUG_ASSERT(pr); in __kmp_dispatch_next()
2037 if ((status = (pr->u.p.tc != 0)) == 0) { in __kmp_dispatch_next()
2045 if (pr->pushed_ws != ct_none) { in __kmp_dispatch_next()
2046 pr->pushed_ws = __kmp_pop_workshare(gtid, pr->pushed_ws, loc); in __kmp_dispatch_next()
2049 } else if (pr->flags.nomerge) { in __kmp_dispatch_next()
2054 T chunk = pr->u.p.parm1; in __kmp_dispatch_next()
2059 init = chunk * pr->u.p.count++; in __kmp_dispatch_next()
2060 trip = pr->u.p.tc - 1; in __kmp_dispatch_next()
2070 if (pr->pushed_ws != ct_none) { in __kmp_dispatch_next()
2071 pr->pushed_ws = __kmp_pop_workshare(gtid, pr->pushed_ws, loc); in __kmp_dispatch_next()
2075 start = pr->u.p.lb; in __kmp_dispatch_next()
2077 incr = pr->u.p.st; in __kmp_dispatch_next()
2082 pr->u.p.last_upper = pr->u.p.ub; in __kmp_dispatch_next()
2097 if (pr->flags.ordered) { in __kmp_dispatch_next()
2098 pr->u.p.ordered_lower = init; in __kmp_dispatch_next()
2099 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next()
2107 KD_TRACE(1000, (buff, gtid, pr->u.p.ordered_lower, in __kmp_dispatch_next()
2108 pr->u.p.ordered_upper)); in __kmp_dispatch_next()
2115 pr->u.p.tc = 0; in __kmp_dispatch_next()
2116 *p_lb = pr->u.p.lb; in __kmp_dispatch_next()
2117 *p_ub = pr->u.p.ub; in __kmp_dispatch_next()
2119 pr->u.p.last_upper = *p_ub; in __kmp_dispatch_next()
2124 *p_st = pr->u.p.st; in __kmp_dispatch_next()
2152 pr = reinterpret_cast<dispatch_private_info_template<T> *>( in __kmp_dispatch_next()
2154 KMP_DEBUG_ASSERT(pr); in __kmp_dispatch_next()
2160 if (pr->flags.use_hier) in __kmp_dispatch_next()
2161 status = sh->hier->next(loc, gtid, pr, &last, p_lb, p_ub, p_st); in __kmp_dispatch_next()
2164 status = __kmp_dispatch_next_algorithm<T>(gtid, pr, sh, &last, p_lb, p_ub, in __kmp_dispatch_next()
2184 pr->flags.use_hier = FALSE; in __kmp_dispatch_next()
2188 if (pr->schedule == kmp_sch_static_steal) { in __kmp_dispatch_next()
2218 if (pr->flags.ordered) { in __kmp_dispatch_next()
2230 if (pr->pushed_ws != ct_none) { in __kmp_dispatch_next()
2231 pr->pushed_ws = __kmp_pop_workshare(gtid, pr->pushed_ws, loc); in __kmp_dispatch_next()
2242 pr->u.p.last_upper = pr->u.p.ub; in __kmp_dispatch_next()