Lines Matching refs:pr
84 dispatch_private_info_template<T> *pr, in __kmp_dispatch_init_algorithm() argument
111 KD_TRACE(10, (buff, gtid, pr, lb, ub, st, schedule, chunk, nproc, tid)); in __kmp_dispatch_init_algorithm()
139 pr->flags.nomerge = TRUE; in __kmp_dispatch_init_algorithm()
143 pr->flags.nomerge = FALSE; in __kmp_dispatch_init_algorithm()
145 pr->type_size = traits_t<T>::type_size; // remember the size of variables in __kmp_dispatch_init_algorithm()
147 pr->flags.ordered = TRUE; in __kmp_dispatch_init_algorithm()
151 pr->flags.ordered = FALSE; in __kmp_dispatch_init_algorithm()
249 pr->u.p.parm1 = chunk; in __kmp_dispatch_init_algorithm()
254 pr->u.p.count = 0; in __kmp_dispatch_init_algorithm()
259 (pr->flags.ordered ? ct_pdo_ordered : ct_pdo), loc); in __kmp_dispatch_init_algorithm()
287 pr->u.p.lb = lb; in __kmp_dispatch_init_algorithm()
288 pr->u.p.ub = ub; in __kmp_dispatch_init_algorithm()
289 pr->u.p.st = st; in __kmp_dispatch_init_algorithm()
290 pr->u.p.tc = tc; in __kmp_dispatch_init_algorithm()
293 pr->u.p.last_upper = ub + st; in __kmp_dispatch_init_algorithm()
299 if (pr->flags.ordered) { in __kmp_dispatch_init_algorithm()
300 pr->ordered_bumped = 0; in __kmp_dispatch_init_algorithm()
301 pr->u.p.ordered_lower = 1; in __kmp_dispatch_init_algorithm()
302 pr->u.p.ordered_upper = 0; in __kmp_dispatch_init_algorithm()
325 pr->u.p.count = init; in __kmp_dispatch_init_algorithm()
326 pr->u.p.ub = init + small_chunk + (id < extras ? 1 : 0); in __kmp_dispatch_init_algorithm()
328 pr->u.p.parm2 = lb; in __kmp_dispatch_init_algorithm()
330 pr->u.p.parm4 = (id + 1) % nproc; // remember neighbour tid in __kmp_dispatch_init_algorithm()
331 pr->u.p.st = st; in __kmp_dispatch_init_algorithm()
369 pr->u.p.parm1 = (id == tc - 1); /* parm1 stores *plastiter */ in __kmp_dispatch_init_algorithm()
371 pr->u.p.count = 1; /* means no more chunks to execute */ in __kmp_dispatch_init_algorithm()
372 pr->u.p.parm1 = FALSE; in __kmp_dispatch_init_algorithm()
380 pr->u.p.parm1 = (id == nproc - 1); in __kmp_dispatch_init_algorithm()
386 pr->u.p.parm1 = TRUE; in __kmp_dispatch_init_algorithm()
389 pr->u.p.count = 1; /* means no more chunks to execute */ in __kmp_dispatch_init_algorithm()
390 pr->u.p.parm1 = FALSE; in __kmp_dispatch_init_algorithm()
401 pr->u.p.lb = lb + init; in __kmp_dispatch_init_algorithm()
402 pr->u.p.ub = lb + limit; in __kmp_dispatch_init_algorithm()
406 pr->u.p.lb = lb + init * st; in __kmp_dispatch_init_algorithm()
410 pr->u.p.ub = (ub_tmp + st > ub ? ub : ub_tmp); in __kmp_dispatch_init_algorithm()
412 pr->u.p.ub = (ub_tmp + st < ub ? ub : ub_tmp); in __kmp_dispatch_init_algorithm()
415 if (pr->flags.ordered) { in __kmp_dispatch_init_algorithm()
416 pr->u.p.ordered_lower = init; in __kmp_dispatch_init_algorithm()
417 pr->u.p.ordered_upper = limit; in __kmp_dispatch_init_algorithm()
430 pr->u.p.parm1 = ((tc + nth - 1) / nth + chunk - 1) & ~(chunk - 1); in __kmp_dispatch_init_algorithm()
432 pr->u.p.parm1 = tc; in __kmp_dispatch_init_algorithm()
450 pr->u.p.parm2 = guided_int_param * nproc * (chunk + 1); in __kmp_dispatch_init_algorithm()
451 *(double *)&pr->u.p.parm3 = in __kmp_dispatch_init_algorithm()
464 pr->u.p.parm1 = tc; in __kmp_dispatch_init_algorithm()
518 (((ptrdiff_t)&pr->u.p.parm3) & (natural_alignment)) == 0); in __kmp_dispatch_init_algorithm()
523 *(DBL *)&pr->u.p.parm3 = x; in __kmp_dispatch_init_algorithm()
564 pr->u.p.parm2 = cross; in __kmp_dispatch_init_algorithm()
568 #define GUIDED_ANALYTICAL_WORKAROUND (*(DBL *)&pr->u.p.parm3) in __kmp_dispatch_init_algorithm()
573 pr->u.p.count = tc - __kmp_dispatch_guided_remaining( in __kmp_dispatch_init_algorithm()
587 pr->u.p.parm1 = tc; in __kmp_dispatch_init_algorithm()
596 pr->u.p.parm1 = (nproc > 1) ? (tc + nproc - 1) / nproc : tc; in __kmp_dispatch_init_algorithm()
600 if (pr->u.p.parm1 <= 0) { in __kmp_dispatch_init_algorithm()
601 pr->u.p.parm1 = KMP_DEFAULT_CHUNK; in __kmp_dispatch_init_algorithm()
649 pr->u.p.parm1 = parm1; in __kmp_dispatch_init_algorithm()
650 pr->u.p.parm2 = parm2; in __kmp_dispatch_init_algorithm()
651 pr->u.p.parm3 = parm3; in __kmp_dispatch_init_algorithm()
652 pr->u.p.parm4 = parm4; in __kmp_dispatch_init_algorithm()
663 pr->schedule = schedule; in __kmp_dispatch_init_algorithm()
732 dispatch_private_info_template<T> *pr; in __kmp_dispatch_init() local
781 pr = reinterpret_cast<dispatch_private_info_template<T> *>( in __kmp_dispatch_init()
789 if (pr->flags.use_hier) { in __kmp_dispatch_init()
794 pr->flags.use_hier = FALSE; in __kmp_dispatch_init()
800 if (!ordered && !pr->flags.use_hier) in __kmp_dispatch_init()
816 pr = reinterpret_cast<dispatch_private_info_template<T> *>( in __kmp_dispatch_init()
825 pr = reinterpret_cast<dispatch_private_info_template<T> *>( in __kmp_dispatch_init()
834 __kmp_dispatch_init_algorithm(loc, gtid, pr, schedule, lb, ub, st, in __kmp_dispatch_init()
841 if (pr->flags.ordered == 0) { in __kmp_dispatch_init()
866 th->th.th_dispatch->th_dispatch_pr_current = (dispatch_private_info_t *)pr; in __kmp_dispatch_init()
870 if (pr->flags.ordered) { in __kmp_dispatch_init()
882 cur_chunk = pr->u.p.parm1; in __kmp_dispatch_init()
900 __kmp_itt_metadata_loop(loc, schedtype, pr->u.p.tc, cur_chunk); in __kmp_dispatch_init()
903 if (pr->flags.use_hier) { in __kmp_dispatch_init()
904 pr->u.p.count = 0; in __kmp_dispatch_init()
905 pr->u.p.ub = pr->u.p.lb = pr->u.p.st = pr->u.p.tc = 0; in __kmp_dispatch_init()
924 KD_TRACE(10, (buff, gtid, pr->schedule, pr->flags.ordered, pr->u.p.lb, in __kmp_dispatch_init()
925 pr->u.p.ub, pr->u.p.st, pr->u.p.tc, pr->u.p.count, in __kmp_dispatch_init()
926 pr->u.p.ordered_lower, pr->u.p.ordered_upper, pr->u.p.parm1, in __kmp_dispatch_init()
927 pr->u.p.parm2, pr->u.p.parm3, pr->u.p.parm4)); in __kmp_dispatch_init()
941 volatile T *p = &pr->u.p.static_steal_counter; in __kmp_dispatch_init()
952 &(task_info->task_data), pr->u.p.tc, OMPT_LOAD_RETURN_ADDRESS(gtid)); in __kmp_dispatch_init()
971 dispatch_private_info_template<UT> *pr = in __kmp_dispatch_finish() local
977 KMP_DEBUG_ASSERT(pr); in __kmp_dispatch_finish()
982 if (pr->ordered_bumped) { in __kmp_dispatch_finish()
987 pr->ordered_bumped = 0; in __kmp_dispatch_finish()
989 UT lower = pr->u.p.ordered_lower; in __kmp_dispatch_finish()
1034 dispatch_private_info_template<UT> *pr = in __kmp_dispatch_finish_chunk() local
1040 KMP_DEBUG_ASSERT(pr); in __kmp_dispatch_finish_chunk()
1046 UT lower = pr->u.p.ordered_lower; in __kmp_dispatch_finish_chunk()
1047 UT upper = pr->u.p.ordered_upper; in __kmp_dispatch_finish_chunk()
1050 if (pr->ordered_bumped == inc) { in __kmp_dispatch_finish_chunk()
1055 pr->ordered_bumped = 0; in __kmp_dispatch_finish_chunk()
1057 inc -= pr->ordered_bumped; in __kmp_dispatch_finish_chunk()
1079 pr->ordered_bumped = 0; in __kmp_dispatch_finish_chunk()
1107 dispatch_private_info_template<T> *pr, in __kmp_dispatch_next_algorithm() argument
1125 KMP_DEBUG_ASSERT(pr); in __kmp_dispatch_next_algorithm()
1136 KD_TRACE(10, (buff, gtid, pr, sh, nproc, tid)); in __kmp_dispatch_next_algorithm()
1142 if (pr->u.p.tc == 0) { in __kmp_dispatch_next_algorithm()
1150 switch (pr->schedule) { in __kmp_dispatch_next_algorithm()
1153 T chunk = pr->u.p.parm1; in __kmp_dispatch_next_algorithm()
1159 trip = pr->u.p.tc - 1; in __kmp_dispatch_next_algorithm()
1166 if (pr->u.p.count < (UT)pr->u.p.ub) { in __kmp_dispatch_next_algorithm()
1169 init = (pr->u.p.count)++; in __kmp_dispatch_next_algorithm()
1170 status = (init < (UT)pr->u.p.ub); in __kmp_dispatch_next_algorithm()
1183 T victimIdx = pr->u.p.parm4; in __kmp_dispatch_next_algorithm()
1189 while ((victim == NULL || victim == pr || in __kmp_dispatch_next_algorithm()
1191 *(volatile T *)&pr->u.p.static_steal_counter)) && in __kmp_dispatch_next_algorithm()
1199 *(volatile T *)&pr->u.p.static_steal_counter)) { in __kmp_dispatch_next_algorithm()
1205 pr->u.p.parm4 = (victimIdx + 1) % nproc; // shift start tid in __kmp_dispatch_next_algorithm()
1216 pr->u.p.parm4 = (victimIdx + 1) % nproc; // next victim in __kmp_dispatch_next_algorithm()
1233 pr->u.p.parm4 = victimIdx; // remember victim to steal from in __kmp_dispatch_next_algorithm()
1238 pr->u.p.count = init + 1; in __kmp_dispatch_next_algorithm()
1239 pr->u.p.ub = limit; in __kmp_dispatch_next_algorithm()
1256 vold.b = *(volatile kmp_int64 *)(&pr->u.p.count); in __kmp_dispatch_next_algorithm()
1260 (volatile kmp_int64 *)&pr->u.p.count, in __kmp_dispatch_next_algorithm()
1264 vold.b = *(volatile kmp_int64 *)(&pr->u.p.count); in __kmp_dispatch_next_algorithm()
1283 T victimIdx = pr->u.p.parm4; in __kmp_dispatch_next_algorithm()
1289 while ((victim == NULL || victim == pr || in __kmp_dispatch_next_algorithm()
1291 *(volatile T *)&pr->u.p.static_steal_counter)) && in __kmp_dispatch_next_algorithm()
1299 *(volatile T *)&pr->u.p.static_steal_counter)) { in __kmp_dispatch_next_algorithm()
1304 pr->u.p.parm4 = victimIdx; // new victim found in __kmp_dispatch_next_algorithm()
1312 pr->u.p.parm4 = (victimIdx + 1) % nproc; // shift start victim id in __kmp_dispatch_next_algorithm()
1335 KMP_XCHG_FIXED64((volatile kmp_int64 *)(&pr->u.p.count), vold.b); in __kmp_dispatch_next_algorithm()
1337 *(volatile kmp_int64 *)(&pr->u.p.count) = vold.b; in __kmp_dispatch_next_algorithm()
1352 start = pr->u.p.parm2; in __kmp_dispatch_next_algorithm()
1355 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1372 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1373 pr->u.p.ordered_lower = init; in __kmp_dispatch_next_algorithm()
1374 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next_algorithm()
1386 if ((status = !pr->u.p.count) != 0) { in __kmp_dispatch_next_algorithm()
1387 pr->u.p.count = 1; in __kmp_dispatch_next_algorithm()
1388 *p_lb = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1389 *p_ub = pr->u.p.ub; in __kmp_dispatch_next_algorithm()
1390 last = pr->u.p.parm1; in __kmp_dispatch_next_algorithm()
1392 *p_st = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1394 pr->u.p.lb = pr->u.p.ub + pr->u.p.st; in __kmp_dispatch_next_algorithm()
1406 parm1 = pr->u.p.parm1; in __kmp_dispatch_next_algorithm()
1408 trip = pr->u.p.tc - 1; in __kmp_dispatch_next_algorithm()
1409 init = parm1 * (pr->u.p.count + tid); in __kmp_dispatch_next_algorithm()
1412 start = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1413 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1422 pr->u.p.count += nproc; in __kmp_dispatch_next_algorithm()
1432 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1433 pr->u.p.ordered_lower = init; in __kmp_dispatch_next_algorithm()
1434 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next_algorithm()
1441 T chunk = pr->u.p.parm1; in __kmp_dispatch_next_algorithm()
1449 trip = pr->u.p.tc - 1; in __kmp_dispatch_next_algorithm()
1457 start = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1459 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1475 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1476 pr->u.p.ordered_lower = init; in __kmp_dispatch_next_algorithm()
1477 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next_algorithm()
1484 T chunkspec = pr->u.p.parm1; in __kmp_dispatch_next_algorithm()
1488 trip = pr->u.p.tc; in __kmp_dispatch_next_algorithm()
1500 pr->u.p.parm2) { // compare with K*nproc*(chunk+1), K=2 by default in __kmp_dispatch_next_algorithm()
1521 (UT)(remaining * *(double *)&pr->u.p.parm3); // divide by K*nproc in __kmp_dispatch_next_algorithm()
1531 start = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1532 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1537 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1538 pr->u.p.ordered_lower = init; in __kmp_dispatch_next_algorithm()
1539 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next_algorithm()
1554 T chunk = pr->u.p.parm1; in __kmp_dispatch_next_algorithm()
1558 trip = pr->u.p.tc; in __kmp_dispatch_next_algorithm()
1570 if ((T)remaining < pr->u.p.parm2) { in __kmp_dispatch_next_algorithm()
1591 UT span = remaining * (*(double *)&pr->u.p.parm3); in __kmp_dispatch_next_algorithm()
1605 start = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1606 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1611 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1612 pr->u.p.ordered_lower = init; in __kmp_dispatch_next_algorithm()
1613 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next_algorithm()
1626 T chunkspec = pr->u.p.parm1; in __kmp_dispatch_next_algorithm()
1638 trip = pr->u.p.tc; in __kmp_dispatch_next_algorithm()
1646 if (chunkIdx >= (UT)pr->u.p.parm2) { in __kmp_dispatch_next_algorithm()
1649 init = chunkIdx * chunkspec + pr->u.p.count; in __kmp_dispatch_next_algorithm()
1677 trip, *(DBL *)&pr->u.p.parm3, chunkIdx); in __kmp_dispatch_next_algorithm()
1683 trip, *(DBL *)&pr->u.p.parm3, chunkIdx + 1); in __kmp_dispatch_next_algorithm()
1701 start = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1702 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1707 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1708 pr->u.p.ordered_lower = init; in __kmp_dispatch_next_algorithm()
1709 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next_algorithm()
1722 T parm2 = pr->u.p.parm2; in __kmp_dispatch_next_algorithm()
1723 T parm3 = pr->u.p.parm3; in __kmp_dispatch_next_algorithm()
1724 T parm4 = pr->u.p.parm4; in __kmp_dispatch_next_algorithm()
1732 trip = pr->u.p.tc - 1; in __kmp_dispatch_next_algorithm()
1740 start = pr->u.p.lb; in __kmp_dispatch_next_algorithm()
1742 incr = pr->u.p.st; in __kmp_dispatch_next_algorithm()
1758 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1759 pr->u.p.ordered_lower = init; in __kmp_dispatch_next_algorithm()
1760 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next_algorithm()
1776 if (pr->flags.ordered) { in __kmp_dispatch_next_algorithm()
1782 KD_TRACE(1000, (buff, gtid, pr->u.p.ordered_lower, pr->u.p.ordered_upper)); in __kmp_dispatch_next_algorithm()
1824 i = (kmp_int64)(pr->u.p.st); \
1869 dispatch_private_info_template<T> *pr; in __kmp_dispatch_next() local
1881 pr = reinterpret_cast<dispatch_private_info_template<T> *>( in __kmp_dispatch_next()
1883 KMP_DEBUG_ASSERT(pr); in __kmp_dispatch_next()
1885 if ((status = (pr->u.p.tc != 0)) == 0) { in __kmp_dispatch_next()
1893 if (pr->pushed_ws != ct_none) { in __kmp_dispatch_next()
1894 pr->pushed_ws = __kmp_pop_workshare(gtid, pr->pushed_ws, loc); in __kmp_dispatch_next()
1897 } else if (pr->flags.nomerge) { in __kmp_dispatch_next()
1902 T chunk = pr->u.p.parm1; in __kmp_dispatch_next()
1907 init = chunk * pr->u.p.count++; in __kmp_dispatch_next()
1908 trip = pr->u.p.tc - 1; in __kmp_dispatch_next()
1918 if (pr->pushed_ws != ct_none) { in __kmp_dispatch_next()
1919 pr->pushed_ws = __kmp_pop_workshare(gtid, pr->pushed_ws, loc); in __kmp_dispatch_next()
1923 start = pr->u.p.lb; in __kmp_dispatch_next()
1925 incr = pr->u.p.st; in __kmp_dispatch_next()
1930 pr->u.p.last_upper = pr->u.p.ub; in __kmp_dispatch_next()
1945 if (pr->flags.ordered) { in __kmp_dispatch_next()
1946 pr->u.p.ordered_lower = init; in __kmp_dispatch_next()
1947 pr->u.p.ordered_upper = limit; in __kmp_dispatch_next()
1955 KD_TRACE(1000, (buff, gtid, pr->u.p.ordered_lower, in __kmp_dispatch_next()
1956 pr->u.p.ordered_upper)); in __kmp_dispatch_next()
1963 pr->u.p.tc = 0; in __kmp_dispatch_next()
1964 *p_lb = pr->u.p.lb; in __kmp_dispatch_next()
1965 *p_ub = pr->u.p.ub; in __kmp_dispatch_next()
1967 pr->u.p.last_upper = *p_ub; in __kmp_dispatch_next()
1972 *p_st = pr->u.p.st; in __kmp_dispatch_next()
1999 pr = reinterpret_cast<dispatch_private_info_template<T> *>( in __kmp_dispatch_next()
2001 KMP_DEBUG_ASSERT(pr); in __kmp_dispatch_next()
2007 if (pr->flags.use_hier) in __kmp_dispatch_next()
2008 status = sh->hier->next(loc, gtid, pr, &last, p_lb, p_ub, p_st); in __kmp_dispatch_next()
2011 status = __kmp_dispatch_next_algorithm<T>(gtid, pr, sh, &last, p_lb, p_ub, in __kmp_dispatch_next()
2032 pr->flags.use_hier = FALSE; in __kmp_dispatch_next()
2036 if (pr->schedule == kmp_sch_static_steal && in __kmp_dispatch_next()
2058 if (pr->flags.ordered) { in __kmp_dispatch_next()
2072 if (pr->pushed_ws != ct_none) { in __kmp_dispatch_next()
2073 pr->pushed_ws = __kmp_pop_workshare(gtid, pr->pushed_ws, loc); in __kmp_dispatch_next()
2084 pr->u.p.last_upper = pr->u.p.ub; in __kmp_dispatch_next()