1 /* 2 * kmp_barrier.cpp 3 */ 4 5 //===----------------------------------------------------------------------===// 6 // 7 // The LLVM Compiler Infrastructure 8 // 9 // This file is dual licensed under the MIT and the University of Illinois Open 10 // Source Licenses. See LICENSE.txt for details. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "kmp.h" 15 #include "kmp_wait_release.h" 16 #include "kmp_itt.h" 17 #include "kmp_os.h" 18 #include "kmp_stats.h" 19 20 #if KMP_MIC 21 #include <immintrin.h> 22 #define USE_NGO_STORES 1 23 #endif // KMP_MIC 24 25 #include "tsan_annotations.h" 26 27 #if KMP_MIC && USE_NGO_STORES 28 // ICV copying 29 #define ngo_load(src) __m512d Vt = _mm512_load_pd((void *)(src)) 30 #define ngo_store_icvs(dst, src) _mm512_storenrngo_pd((void *)(dst), Vt) 31 #define ngo_store_go(dst, src) _mm512_storenrngo_pd((void *)(dst), Vt) 32 #define ngo_sync() __asm__ volatile("lock; addl $0,0(%%rsp)" ::: "memory") 33 #else 34 #define ngo_load(src) ((void)0) 35 #define ngo_store_icvs(dst, src) copy_icvs((dst), (src)) 36 #define ngo_store_go(dst, src) KMP_MEMCPY((dst), (src), CACHE_LINE) 37 #define ngo_sync() ((void)0) 38 #endif /* KMP_MIC && USE_NGO_STORES */ 39 40 void __kmp_print_structure(void); // Forward declaration 41 42 // ---------------------------- Barrier Algorithms ---------------------------- 43 44 // Linear Barrier 45 static void __kmp_linear_barrier_gather( 46 enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid, 47 void (*reduce)(void *, void *) USE_ITT_BUILD_ARG(void *itt_sync_obj)) { 48 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_linear_gather); 49 kmp_team_t *team = this_thr->th.th_team; 50 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; 51 kmp_info_t **other_threads = team->t.t_threads; 52 53 KA_TRACE( 54 20, 55 ("__kmp_linear_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n", 56 gtid, team->t.t_id, tid, bt)); 57 KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]); 58 59 #if USE_ITT_BUILD && USE_ITT_NOTIFY 60 // Barrier imbalance - save arrive time to the thread 61 if (__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) { 62 this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time = 63 __itt_get_timestamp(); 64 } 65 #endif 66 // We now perform a linear reduction to signal that all of the threads have 67 // arrived. 68 if (!KMP_MASTER_TID(tid)) { 69 KA_TRACE(20, 70 ("__kmp_linear_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d)" 71 "arrived(%p): %llu => %llu\n", 72 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(0, team), 73 team->t.t_id, 0, &thr_bar->b_arrived, thr_bar->b_arrived, 74 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP)); 75 // Mark arrival to master thread 76 /* After performing this write, a worker thread may not assume that the team 77 is valid any more - it could be deallocated by the master thread at any 78 time. */ 79 ANNOTATE_BARRIER_BEGIN(this_thr); 80 kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[0]); 81 flag.release(); 82 } else { 83 kmp_balign_team_t *team_bar = &team->t.t_bar[bt]; 84 int nproc = this_thr->th.th_team_nproc; 85 int i; 86 // Don't have to worry about sleep bit here or atomic since team setting 87 kmp_uint64 new_state = team_bar->b_arrived + KMP_BARRIER_STATE_BUMP; 88 89 // Collect all the worker team member threads. 90 for (i = 1; i < nproc; ++i) { 91 #if KMP_CACHE_MANAGE 92 // Prefetch next thread's arrived count 93 if (i + 1 < nproc) 94 KMP_CACHE_PREFETCH(&other_threads[i + 1]->th.th_bar[bt].bb.b_arrived); 95 #endif /* KMP_CACHE_MANAGE */ 96 KA_TRACE(20, ("__kmp_linear_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%d) " 97 "arrived(%p) == %llu\n", 98 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(i, team), 99 team->t.t_id, i, 100 &other_threads[i]->th.th_bar[bt].bb.b_arrived, new_state)); 101 102 // Wait for worker thread to arrive 103 kmp_flag_64 flag(&other_threads[i]->th.th_bar[bt].bb.b_arrived, 104 new_state); 105 flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj)); 106 ANNOTATE_BARRIER_END(other_threads[i]); 107 #if USE_ITT_BUILD && USE_ITT_NOTIFY 108 // Barrier imbalance - write min of the thread time and the other thread 109 // time to the thread. 110 if (__kmp_forkjoin_frames_mode == 2) { 111 this_thr->th.th_bar_min_time = KMP_MIN( 112 this_thr->th.th_bar_min_time, other_threads[i]->th.th_bar_min_time); 113 } 114 #endif 115 if (reduce) { 116 KA_TRACE(100, 117 ("__kmp_linear_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n", 118 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(i, team), 119 team->t.t_id, i)); 120 ANNOTATE_REDUCE_AFTER(reduce); 121 (*reduce)(this_thr->th.th_local.reduce_data, 122 other_threads[i]->th.th_local.reduce_data); 123 ANNOTATE_REDUCE_BEFORE(reduce); 124 ANNOTATE_REDUCE_BEFORE(&team->t.t_bar); 125 } 126 } 127 // Don't have to worry about sleep bit here or atomic since team setting 128 team_bar->b_arrived = new_state; 129 KA_TRACE(20, ("__kmp_linear_barrier_gather: T#%d(%d:%d) set team %d " 130 "arrived(%p) = %llu\n", 131 gtid, team->t.t_id, tid, team->t.t_id, &team_bar->b_arrived, 132 new_state)); 133 } 134 KA_TRACE( 135 20, 136 ("__kmp_linear_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n", 137 gtid, team->t.t_id, tid, bt)); 138 } 139 140 static void __kmp_linear_barrier_release( 141 enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid, 142 int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj)) { 143 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_linear_release); 144 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; 145 kmp_team_t *team; 146 147 if (KMP_MASTER_TID(tid)) { 148 unsigned int i; 149 kmp_uint32 nproc = this_thr->th.th_team_nproc; 150 kmp_info_t **other_threads; 151 152 team = __kmp_threads[gtid]->th.th_team; 153 KMP_DEBUG_ASSERT(team != NULL); 154 other_threads = team->t.t_threads; 155 156 KA_TRACE(20, ("__kmp_linear_barrier_release: T#%d(%d:%d) master enter for " 157 "barrier type %d\n", 158 gtid, team->t.t_id, tid, bt)); 159 160 if (nproc > 1) { 161 #if KMP_BARRIER_ICV_PUSH 162 { 163 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_icv_copy); 164 if (propagate_icvs) { 165 ngo_load(&team->t.t_implicit_task_taskdata[0].td_icvs); 166 for (i = 1; i < nproc; ++i) { 167 __kmp_init_implicit_task(team->t.t_ident, team->t.t_threads[i], 168 team, i, FALSE); 169 ngo_store_icvs(&team->t.t_implicit_task_taskdata[i].td_icvs, 170 &team->t.t_implicit_task_taskdata[0].td_icvs); 171 } 172 ngo_sync(); 173 } 174 } 175 #endif // KMP_BARRIER_ICV_PUSH 176 177 // Now, release all of the worker threads 178 for (i = 1; i < nproc; ++i) { 179 #if KMP_CACHE_MANAGE 180 // Prefetch next thread's go flag 181 if (i + 1 < nproc) 182 KMP_CACHE_PREFETCH(&other_threads[i + 1]->th.th_bar[bt].bb.b_go); 183 #endif /* KMP_CACHE_MANAGE */ 184 KA_TRACE( 185 20, 186 ("__kmp_linear_barrier_release: T#%d(%d:%d) releasing T#%d(%d:%d) " 187 "go(%p): %u => %u\n", 188 gtid, team->t.t_id, tid, other_threads[i]->th.th_info.ds.ds_gtid, 189 team->t.t_id, i, &other_threads[i]->th.th_bar[bt].bb.b_go, 190 other_threads[i]->th.th_bar[bt].bb.b_go, 191 other_threads[i]->th.th_bar[bt].bb.b_go + KMP_BARRIER_STATE_BUMP)); 192 ANNOTATE_BARRIER_BEGIN(other_threads[i]); 193 kmp_flag_64 flag(&other_threads[i]->th.th_bar[bt].bb.b_go, 194 other_threads[i]); 195 flag.release(); 196 } 197 } 198 } else { // Wait for the MASTER thread to release us 199 KA_TRACE(20, ("__kmp_linear_barrier_release: T#%d wait go(%p) == %u\n", 200 gtid, &thr_bar->b_go, KMP_BARRIER_STATE_BUMP)); 201 kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP); 202 flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj)); 203 ANNOTATE_BARRIER_END(this_thr); 204 #if USE_ITT_BUILD && USE_ITT_NOTIFY 205 if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) { 206 // In a fork barrier; cannot get the object reliably (or ITTNOTIFY is 207 // disabled) 208 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier, 0, -1); 209 // Cancel wait on previous parallel region... 210 __kmp_itt_task_starting(itt_sync_obj); 211 212 if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done)) 213 return; 214 215 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier); 216 if (itt_sync_obj != NULL) 217 // Call prepare as early as possible for "new" barrier 218 __kmp_itt_task_finished(itt_sync_obj); 219 } else 220 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */ 221 // Early exit for reaping threads releasing forkjoin barrier 222 if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done)) 223 return; 224 // The worker thread may now assume that the team is valid. 225 #ifdef KMP_DEBUG 226 tid = __kmp_tid_from_gtid(gtid); 227 team = __kmp_threads[gtid]->th.th_team; 228 #endif 229 KMP_DEBUG_ASSERT(team != NULL); 230 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE); 231 KA_TRACE(20, 232 ("__kmp_linear_barrier_release: T#%d(%d:%d) set go(%p) = %u\n", 233 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE)); 234 KMP_MB(); // Flush all pending memory write invalidates. 235 } 236 KA_TRACE( 237 20, 238 ("__kmp_linear_barrier_release: T#%d(%d:%d) exit for barrier type %d\n", 239 gtid, team->t.t_id, tid, bt)); 240 } 241 242 // Tree barrier 243 static void 244 __kmp_tree_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid, 245 int tid, void (*reduce)(void *, void *) 246 USE_ITT_BUILD_ARG(void *itt_sync_obj)) { 247 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_tree_gather); 248 kmp_team_t *team = this_thr->th.th_team; 249 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; 250 kmp_info_t **other_threads = team->t.t_threads; 251 kmp_uint32 nproc = this_thr->th.th_team_nproc; 252 kmp_uint32 branch_bits = __kmp_barrier_gather_branch_bits[bt]; 253 kmp_uint32 branch_factor = 1 << branch_bits; 254 kmp_uint32 child; 255 kmp_uint32 child_tid; 256 kmp_uint64 new_state; 257 258 KA_TRACE( 259 20, ("__kmp_tree_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n", 260 gtid, team->t.t_id, tid, bt)); 261 KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]); 262 263 #if USE_ITT_BUILD && USE_ITT_NOTIFY 264 // Barrier imbalance - save arrive time to the thread 265 if (__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) { 266 this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time = 267 __itt_get_timestamp(); 268 } 269 #endif 270 // Perform tree gather to wait until all threads have arrived; reduce any 271 // required data as we go 272 child_tid = (tid << branch_bits) + 1; 273 if (child_tid < nproc) { 274 // Parent threads wait for all their children to arrive 275 new_state = team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP; 276 child = 1; 277 do { 278 kmp_info_t *child_thr = other_threads[child_tid]; 279 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb; 280 #if KMP_CACHE_MANAGE 281 // Prefetch next thread's arrived count 282 if (child + 1 <= branch_factor && child_tid + 1 < nproc) 283 KMP_CACHE_PREFETCH( 284 &other_threads[child_tid + 1]->th.th_bar[bt].bb.b_arrived); 285 #endif /* KMP_CACHE_MANAGE */ 286 KA_TRACE(20, 287 ("__kmp_tree_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%u) " 288 "arrived(%p) == %llu\n", 289 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team), 290 team->t.t_id, child_tid, &child_bar->b_arrived, new_state)); 291 // Wait for child to arrive 292 kmp_flag_64 flag(&child_bar->b_arrived, new_state); 293 flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj)); 294 ANNOTATE_BARRIER_END(child_thr); 295 #if USE_ITT_BUILD && USE_ITT_NOTIFY 296 // Barrier imbalance - write min of the thread time and a child time to 297 // the thread. 298 if (__kmp_forkjoin_frames_mode == 2) { 299 this_thr->th.th_bar_min_time = KMP_MIN(this_thr->th.th_bar_min_time, 300 child_thr->th.th_bar_min_time); 301 } 302 #endif 303 if (reduce) { 304 KA_TRACE(100, 305 ("__kmp_tree_barrier_gather: T#%d(%d:%d) += T#%d(%d:%u)\n", 306 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team), 307 team->t.t_id, child_tid)); 308 ANNOTATE_REDUCE_AFTER(reduce); 309 (*reduce)(this_thr->th.th_local.reduce_data, 310 child_thr->th.th_local.reduce_data); 311 ANNOTATE_REDUCE_BEFORE(reduce); 312 ANNOTATE_REDUCE_BEFORE(&team->t.t_bar); 313 } 314 child++; 315 child_tid++; 316 } while (child <= branch_factor && child_tid < nproc); 317 } 318 319 if (!KMP_MASTER_TID(tid)) { // Worker threads 320 kmp_int32 parent_tid = (tid - 1) >> branch_bits; 321 322 KA_TRACE(20, 323 ("__kmp_tree_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d) " 324 "arrived(%p): %llu => %llu\n", 325 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(parent_tid, team), 326 team->t.t_id, parent_tid, &thr_bar->b_arrived, thr_bar->b_arrived, 327 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP)); 328 329 // Mark arrival to parent thread 330 /* After performing this write, a worker thread may not assume that the team 331 is valid any more - it could be deallocated by the master thread at any 332 time. */ 333 ANNOTATE_BARRIER_BEGIN(this_thr); 334 kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[parent_tid]); 335 flag.release(); 336 } else { 337 // Need to update the team arrived pointer if we are the master thread 338 if (nproc > 1) // New value was already computed above 339 team->t.t_bar[bt].b_arrived = new_state; 340 else 341 team->t.t_bar[bt].b_arrived += KMP_BARRIER_STATE_BUMP; 342 KA_TRACE(20, ("__kmp_tree_barrier_gather: T#%d(%d:%d) set team %d " 343 "arrived(%p) = %llu\n", 344 gtid, team->t.t_id, tid, team->t.t_id, 345 &team->t.t_bar[bt].b_arrived, team->t.t_bar[bt].b_arrived)); 346 } 347 KA_TRACE(20, 348 ("__kmp_tree_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n", 349 gtid, team->t.t_id, tid, bt)); 350 } 351 352 static void __kmp_tree_barrier_release( 353 enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid, 354 int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj)) { 355 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_tree_release); 356 kmp_team_t *team; 357 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; 358 kmp_uint32 nproc; 359 kmp_uint32 branch_bits = __kmp_barrier_release_branch_bits[bt]; 360 kmp_uint32 branch_factor = 1 << branch_bits; 361 kmp_uint32 child; 362 kmp_uint32 child_tid; 363 364 // Perform a tree release for all of the threads that have been gathered 365 if (!KMP_MASTER_TID( 366 tid)) { // Handle fork barrier workers who aren't part of a team yet 367 KA_TRACE(20, ("__kmp_tree_barrier_release: T#%d wait go(%p) == %u\n", gtid, 368 &thr_bar->b_go, KMP_BARRIER_STATE_BUMP)); 369 // Wait for parent thread to release us 370 kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP); 371 flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj)); 372 ANNOTATE_BARRIER_END(this_thr); 373 #if USE_ITT_BUILD && USE_ITT_NOTIFY 374 if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) { 375 // In fork barrier where we could not get the object reliably (or 376 // ITTNOTIFY is disabled) 377 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier, 0, -1); 378 // Cancel wait on previous parallel region... 379 __kmp_itt_task_starting(itt_sync_obj); 380 381 if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done)) 382 return; 383 384 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier); 385 if (itt_sync_obj != NULL) 386 // Call prepare as early as possible for "new" barrier 387 __kmp_itt_task_finished(itt_sync_obj); 388 } else 389 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */ 390 // Early exit for reaping threads releasing forkjoin barrier 391 if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done)) 392 return; 393 394 // The worker thread may now assume that the team is valid. 395 team = __kmp_threads[gtid]->th.th_team; 396 KMP_DEBUG_ASSERT(team != NULL); 397 tid = __kmp_tid_from_gtid(gtid); 398 399 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE); 400 KA_TRACE(20, 401 ("__kmp_tree_barrier_release: T#%d(%d:%d) set go(%p) = %u\n", gtid, 402 team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE)); 403 KMP_MB(); // Flush all pending memory write invalidates. 404 } else { 405 team = __kmp_threads[gtid]->th.th_team; 406 KMP_DEBUG_ASSERT(team != NULL); 407 KA_TRACE(20, ("__kmp_tree_barrier_release: T#%d(%d:%d) master enter for " 408 "barrier type %d\n", 409 gtid, team->t.t_id, tid, bt)); 410 } 411 nproc = this_thr->th.th_team_nproc; 412 child_tid = (tid << branch_bits) + 1; 413 414 if (child_tid < nproc) { 415 kmp_info_t **other_threads = team->t.t_threads; 416 child = 1; 417 // Parent threads release all their children 418 do { 419 kmp_info_t *child_thr = other_threads[child_tid]; 420 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb; 421 #if KMP_CACHE_MANAGE 422 // Prefetch next thread's go count 423 if (child + 1 <= branch_factor && child_tid + 1 < nproc) 424 KMP_CACHE_PREFETCH( 425 &other_threads[child_tid + 1]->th.th_bar[bt].bb.b_go); 426 #endif /* KMP_CACHE_MANAGE */ 427 428 #if KMP_BARRIER_ICV_PUSH 429 { 430 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_icv_copy); 431 if (propagate_icvs) { 432 __kmp_init_implicit_task(team->t.t_ident, 433 team->t.t_threads[child_tid], team, 434 child_tid, FALSE); 435 copy_icvs(&team->t.t_implicit_task_taskdata[child_tid].td_icvs, 436 &team->t.t_implicit_task_taskdata[0].td_icvs); 437 } 438 } 439 #endif // KMP_BARRIER_ICV_PUSH 440 KA_TRACE(20, 441 ("__kmp_tree_barrier_release: T#%d(%d:%d) releasing T#%d(%d:%u)" 442 "go(%p): %u => %u\n", 443 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team), 444 team->t.t_id, child_tid, &child_bar->b_go, child_bar->b_go, 445 child_bar->b_go + KMP_BARRIER_STATE_BUMP)); 446 // Release child from barrier 447 ANNOTATE_BARRIER_BEGIN(child_thr); 448 kmp_flag_64 flag(&child_bar->b_go, child_thr); 449 flag.release(); 450 child++; 451 child_tid++; 452 } while (child <= branch_factor && child_tid < nproc); 453 } 454 KA_TRACE( 455 20, ("__kmp_tree_barrier_release: T#%d(%d:%d) exit for barrier type %d\n", 456 gtid, team->t.t_id, tid, bt)); 457 } 458 459 // Hyper Barrier 460 static void 461 __kmp_hyper_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid, 462 int tid, void (*reduce)(void *, void *) 463 USE_ITT_BUILD_ARG(void *itt_sync_obj)) { 464 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_hyper_gather); 465 kmp_team_t *team = this_thr->th.th_team; 466 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; 467 kmp_info_t **other_threads = team->t.t_threads; 468 kmp_uint64 new_state = KMP_BARRIER_UNUSED_STATE; 469 kmp_uint32 num_threads = this_thr->th.th_team_nproc; 470 kmp_uint32 branch_bits = __kmp_barrier_gather_branch_bits[bt]; 471 kmp_uint32 branch_factor = 1 << branch_bits; 472 kmp_uint32 offset; 473 kmp_uint32 level; 474 475 KA_TRACE( 476 20, 477 ("__kmp_hyper_barrier_gather: T#%d(%d:%d) enter for barrier type %d\n", 478 gtid, team->t.t_id, tid, bt)); 479 KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]); 480 481 #if USE_ITT_BUILD && USE_ITT_NOTIFY 482 // Barrier imbalance - save arrive time to the thread 483 if (__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) { 484 this_thr->th.th_bar_arrive_time = this_thr->th.th_bar_min_time = 485 __itt_get_timestamp(); 486 } 487 #endif 488 /* Perform a hypercube-embedded tree gather to wait until all of the threads 489 have arrived, and reduce any required data as we go. */ 490 kmp_flag_64 p_flag(&thr_bar->b_arrived); 491 for (level = 0, offset = 1; offset < num_threads; 492 level += branch_bits, offset <<= branch_bits) { 493 kmp_uint32 child; 494 kmp_uint32 child_tid; 495 496 if (((tid >> level) & (branch_factor - 1)) != 0) { 497 kmp_int32 parent_tid = tid & ~((1 << (level + branch_bits)) - 1); 498 499 KA_TRACE(20, 500 ("__kmp_hyper_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d) " 501 "arrived(%p): %llu => %llu\n", 502 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(parent_tid, team), 503 team->t.t_id, parent_tid, &thr_bar->b_arrived, 504 thr_bar->b_arrived, 505 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP)); 506 // Mark arrival to parent thread 507 /* After performing this write (in the last iteration of the enclosing for 508 loop), a worker thread may not assume that the team is valid any more 509 - it could be deallocated by the master thread at any time. */ 510 ANNOTATE_BARRIER_BEGIN(this_thr); 511 p_flag.set_waiter(other_threads[parent_tid]); 512 p_flag.release(); 513 break; 514 } 515 516 // Parent threads wait for children to arrive 517 if (new_state == KMP_BARRIER_UNUSED_STATE) 518 new_state = team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP; 519 for (child = 1, child_tid = tid + (1 << level); 520 child < branch_factor && child_tid < num_threads; 521 child++, child_tid += (1 << level)) { 522 kmp_info_t *child_thr = other_threads[child_tid]; 523 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb; 524 #if KMP_CACHE_MANAGE 525 kmp_uint32 next_child_tid = child_tid + (1 << level); 526 // Prefetch next thread's arrived count 527 if (child + 1 < branch_factor && next_child_tid < num_threads) 528 KMP_CACHE_PREFETCH( 529 &other_threads[next_child_tid]->th.th_bar[bt].bb.b_arrived); 530 #endif /* KMP_CACHE_MANAGE */ 531 KA_TRACE(20, 532 ("__kmp_hyper_barrier_gather: T#%d(%d:%d) wait T#%d(%d:%u) " 533 "arrived(%p) == %llu\n", 534 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team), 535 team->t.t_id, child_tid, &child_bar->b_arrived, new_state)); 536 // Wait for child to arrive 537 kmp_flag_64 c_flag(&child_bar->b_arrived, new_state); 538 c_flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj)); 539 ANNOTATE_BARRIER_END(child_thr); 540 #if USE_ITT_BUILD && USE_ITT_NOTIFY 541 // Barrier imbalance - write min of the thread time and a child time to 542 // the thread. 543 if (__kmp_forkjoin_frames_mode == 2) { 544 this_thr->th.th_bar_min_time = KMP_MIN(this_thr->th.th_bar_min_time, 545 child_thr->th.th_bar_min_time); 546 } 547 #endif 548 if (reduce) { 549 KA_TRACE(100, 550 ("__kmp_hyper_barrier_gather: T#%d(%d:%d) += T#%d(%d:%u)\n", 551 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team), 552 team->t.t_id, child_tid)); 553 ANNOTATE_REDUCE_AFTER(reduce); 554 (*reduce)(this_thr->th.th_local.reduce_data, 555 child_thr->th.th_local.reduce_data); 556 ANNOTATE_REDUCE_BEFORE(reduce); 557 ANNOTATE_REDUCE_BEFORE(&team->t.t_bar); 558 } 559 } 560 } 561 562 if (KMP_MASTER_TID(tid)) { 563 // Need to update the team arrived pointer if we are the master thread 564 if (new_state == KMP_BARRIER_UNUSED_STATE) 565 team->t.t_bar[bt].b_arrived += KMP_BARRIER_STATE_BUMP; 566 else 567 team->t.t_bar[bt].b_arrived = new_state; 568 KA_TRACE(20, ("__kmp_hyper_barrier_gather: T#%d(%d:%d) set team %d " 569 "arrived(%p) = %llu\n", 570 gtid, team->t.t_id, tid, team->t.t_id, 571 &team->t.t_bar[bt].b_arrived, team->t.t_bar[bt].b_arrived)); 572 } 573 KA_TRACE( 574 20, ("__kmp_hyper_barrier_gather: T#%d(%d:%d) exit for barrier type %d\n", 575 gtid, team->t.t_id, tid, bt)); 576 } 577 578 // The reverse versions seem to beat the forward versions overall 579 #define KMP_REVERSE_HYPER_BAR 580 static void __kmp_hyper_barrier_release( 581 enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid, 582 int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj)) { 583 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_hyper_release); 584 kmp_team_t *team; 585 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; 586 kmp_info_t **other_threads; 587 kmp_uint32 num_threads; 588 kmp_uint32 branch_bits = __kmp_barrier_release_branch_bits[bt]; 589 kmp_uint32 branch_factor = 1 << branch_bits; 590 kmp_uint32 child; 591 kmp_uint32 child_tid; 592 kmp_uint32 offset; 593 kmp_uint32 level; 594 595 /* Perform a hypercube-embedded tree release for all of the threads that have 596 been gathered. If KMP_REVERSE_HYPER_BAR is defined (default) the threads 597 are released in the reverse order of the corresponding gather, otherwise 598 threads are released in the same order. */ 599 if (KMP_MASTER_TID(tid)) { // master 600 team = __kmp_threads[gtid]->th.th_team; 601 KMP_DEBUG_ASSERT(team != NULL); 602 KA_TRACE(20, ("__kmp_hyper_barrier_release: T#%d(%d:%d) master enter for " 603 "barrier type %d\n", 604 gtid, team->t.t_id, tid, bt)); 605 #if KMP_BARRIER_ICV_PUSH 606 if (propagate_icvs) { // master already has ICVs in final destination; copy 607 copy_icvs(&thr_bar->th_fixed_icvs, 608 &team->t.t_implicit_task_taskdata[tid].td_icvs); 609 } 610 #endif 611 } else { // Handle fork barrier workers who aren't part of a team yet 612 KA_TRACE(20, ("__kmp_hyper_barrier_release: T#%d wait go(%p) == %u\n", gtid, 613 &thr_bar->b_go, KMP_BARRIER_STATE_BUMP)); 614 // Wait for parent thread to release us 615 kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP); 616 flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj)); 617 ANNOTATE_BARRIER_END(this_thr); 618 #if USE_ITT_BUILD && USE_ITT_NOTIFY 619 if ((__itt_sync_create_ptr && itt_sync_obj == NULL) || KMP_ITT_DEBUG) { 620 // In fork barrier where we could not get the object reliably 621 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier, 0, -1); 622 // Cancel wait on previous parallel region... 623 __kmp_itt_task_starting(itt_sync_obj); 624 625 if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done)) 626 return; 627 628 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier); 629 if (itt_sync_obj != NULL) 630 // Call prepare as early as possible for "new" barrier 631 __kmp_itt_task_finished(itt_sync_obj); 632 } else 633 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */ 634 // Early exit for reaping threads releasing forkjoin barrier 635 if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done)) 636 return; 637 638 // The worker thread may now assume that the team is valid. 639 team = __kmp_threads[gtid]->th.th_team; 640 KMP_DEBUG_ASSERT(team != NULL); 641 tid = __kmp_tid_from_gtid(gtid); 642 643 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE); 644 KA_TRACE(20, 645 ("__kmp_hyper_barrier_release: T#%d(%d:%d) set go(%p) = %u\n", 646 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE)); 647 KMP_MB(); // Flush all pending memory write invalidates. 648 } 649 num_threads = this_thr->th.th_team_nproc; 650 other_threads = team->t.t_threads; 651 652 #ifdef KMP_REVERSE_HYPER_BAR 653 // Count up to correct level for parent 654 for (level = 0, offset = 1; 655 offset < num_threads && (((tid >> level) & (branch_factor - 1)) == 0); 656 level += branch_bits, offset <<= branch_bits) 657 ; 658 659 // Now go down from there 660 for (level -= branch_bits, offset >>= branch_bits; offset != 0; 661 level -= branch_bits, offset >>= branch_bits) 662 #else 663 // Go down the tree, level by level 664 for (level = 0, offset = 1; offset < num_threads; 665 level += branch_bits, offset <<= branch_bits) 666 #endif // KMP_REVERSE_HYPER_BAR 667 { 668 #ifdef KMP_REVERSE_HYPER_BAR 669 /* Now go in reverse order through the children, highest to lowest. 670 Initial setting of child is conservative here. */ 671 child = num_threads >> ((level == 0) ? level : level - 1); 672 for (child = (child < branch_factor - 1) ? child : branch_factor - 1, 673 child_tid = tid + (child << level); 674 child >= 1; child--, child_tid -= (1 << level)) 675 #else 676 if (((tid >> level) & (branch_factor - 1)) != 0) 677 // No need to go lower than this, since this is the level parent would be 678 // notified 679 break; 680 // Iterate through children on this level of the tree 681 for (child = 1, child_tid = tid + (1 << level); 682 child < branch_factor && child_tid < num_threads; 683 child++, child_tid += (1 << level)) 684 #endif // KMP_REVERSE_HYPER_BAR 685 { 686 if (child_tid >= num_threads) 687 continue; // Child doesn't exist so keep going 688 else { 689 kmp_info_t *child_thr = other_threads[child_tid]; 690 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb; 691 #if KMP_CACHE_MANAGE 692 kmp_uint32 next_child_tid = child_tid - (1 << level); 693 // Prefetch next thread's go count 694 #ifdef KMP_REVERSE_HYPER_BAR 695 if (child - 1 >= 1 && next_child_tid < num_threads) 696 #else 697 if (child + 1 < branch_factor && next_child_tid < num_threads) 698 #endif // KMP_REVERSE_HYPER_BAR 699 KMP_CACHE_PREFETCH( 700 &other_threads[next_child_tid]->th.th_bar[bt].bb.b_go); 701 #endif /* KMP_CACHE_MANAGE */ 702 703 #if KMP_BARRIER_ICV_PUSH 704 if (propagate_icvs) // push my fixed ICVs to my child 705 copy_icvs(&child_bar->th_fixed_icvs, &thr_bar->th_fixed_icvs); 706 #endif // KMP_BARRIER_ICV_PUSH 707 708 KA_TRACE( 709 20, 710 ("__kmp_hyper_barrier_release: T#%d(%d:%d) releasing T#%d(%d:%u)" 711 "go(%p): %u => %u\n", 712 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team), 713 team->t.t_id, child_tid, &child_bar->b_go, child_bar->b_go, 714 child_bar->b_go + KMP_BARRIER_STATE_BUMP)); 715 // Release child from barrier 716 ANNOTATE_BARRIER_BEGIN(child_thr); 717 kmp_flag_64 flag(&child_bar->b_go, child_thr); 718 flag.release(); 719 } 720 } 721 } 722 #if KMP_BARRIER_ICV_PUSH 723 if (propagate_icvs && 724 !KMP_MASTER_TID(tid)) { // copy ICVs locally to final dest 725 __kmp_init_implicit_task(team->t.t_ident, team->t.t_threads[tid], team, tid, 726 FALSE); 727 copy_icvs(&team->t.t_implicit_task_taskdata[tid].td_icvs, 728 &thr_bar->th_fixed_icvs); 729 } 730 #endif 731 KA_TRACE( 732 20, 733 ("__kmp_hyper_barrier_release: T#%d(%d:%d) exit for barrier type %d\n", 734 gtid, team->t.t_id, tid, bt)); 735 } 736 737 // Hierarchical Barrier 738 739 // Initialize thread barrier data 740 /* Initializes/re-initializes the hierarchical barrier data stored on a thread. 741 Performs the minimum amount of initialization required based on how the team 742 has changed. Returns true if leaf children will require both on-core and 743 traditional wake-up mechanisms. For example, if the team size increases, 744 threads already in the team will respond to on-core wakeup on their parent 745 thread, but threads newly added to the team will only be listening on the 746 their local b_go. */ 747 static bool __kmp_init_hierarchical_barrier_thread(enum barrier_type bt, 748 kmp_bstate_t *thr_bar, 749 kmp_uint32 nproc, int gtid, 750 int tid, kmp_team_t *team) { 751 // Checks to determine if (re-)initialization is needed 752 bool uninitialized = thr_bar->team == NULL; 753 bool team_changed = team != thr_bar->team; 754 bool team_sz_changed = nproc != thr_bar->nproc; 755 bool tid_changed = tid != thr_bar->old_tid; 756 bool retval = false; 757 758 if (uninitialized || team_sz_changed) { 759 __kmp_get_hierarchy(nproc, thr_bar); 760 } 761 762 if (uninitialized || team_sz_changed || tid_changed) { 763 thr_bar->my_level = thr_bar->depth - 1; // default for master 764 thr_bar->parent_tid = -1; // default for master 765 if (!KMP_MASTER_TID( 766 tid)) { // if not master, find parent thread in hierarchy 767 kmp_uint32 d = 0; 768 while (d < thr_bar->depth) { // find parent based on level of thread in 769 // hierarchy, and note level 770 kmp_uint32 rem; 771 if (d == thr_bar->depth - 2) { // reached level right below the master 772 thr_bar->parent_tid = 0; 773 thr_bar->my_level = d; 774 break; 775 } else if ((rem = tid % thr_bar->skip_per_level[d + 1]) != 776 0) { // TODO: can we make this op faster? 777 // thread is not a subtree root at next level, so this is max 778 thr_bar->parent_tid = tid - rem; 779 thr_bar->my_level = d; 780 break; 781 } 782 ++d; 783 } 784 } 785 thr_bar->offset = 7 - (tid - thr_bar->parent_tid - 1); 786 thr_bar->old_tid = tid; 787 thr_bar->wait_flag = KMP_BARRIER_NOT_WAITING; 788 thr_bar->team = team; 789 thr_bar->parent_bar = 790 &team->t.t_threads[thr_bar->parent_tid]->th.th_bar[bt].bb; 791 } 792 if (uninitialized || team_changed || tid_changed) { 793 thr_bar->team = team; 794 thr_bar->parent_bar = 795 &team->t.t_threads[thr_bar->parent_tid]->th.th_bar[bt].bb; 796 retval = true; 797 } 798 if (uninitialized || team_sz_changed || tid_changed) { 799 thr_bar->nproc = nproc; 800 thr_bar->leaf_kids = thr_bar->base_leaf_kids; 801 if (thr_bar->my_level == 0) 802 thr_bar->leaf_kids = 0; 803 if (thr_bar->leaf_kids && (kmp_uint32)tid + thr_bar->leaf_kids + 1 > nproc) 804 thr_bar->leaf_kids = nproc - tid - 1; 805 thr_bar->leaf_state = 0; 806 for (int i = 0; i < thr_bar->leaf_kids; ++i) 807 ((char *)&(thr_bar->leaf_state))[7 - i] = 1; 808 } 809 return retval; 810 } 811 812 static void __kmp_hierarchical_barrier_gather( 813 enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid, 814 void (*reduce)(void *, void *) USE_ITT_BUILD_ARG(void *itt_sync_obj)) { 815 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_hier_gather); 816 kmp_team_t *team = this_thr->th.th_team; 817 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; 818 kmp_uint32 nproc = this_thr->th.th_team_nproc; 819 kmp_info_t **other_threads = team->t.t_threads; 820 kmp_uint64 new_state; 821 822 int level = team->t.t_level; 823 #if OMP_40_ENABLED 824 if (other_threads[0] 825 ->th.th_teams_microtask) // are we inside the teams construct? 826 if (this_thr->th.th_teams_size.nteams > 1) 827 ++level; // level was not increased in teams construct for team_of_masters 828 #endif 829 if (level == 1) 830 thr_bar->use_oncore_barrier = 1; 831 else 832 thr_bar->use_oncore_barrier = 0; // Do not use oncore barrier when nested 833 834 KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) enter for " 835 "barrier type %d\n", 836 gtid, team->t.t_id, tid, bt)); 837 KMP_DEBUG_ASSERT(this_thr == other_threads[this_thr->th.th_info.ds.ds_tid]); 838 839 #if USE_ITT_BUILD && USE_ITT_NOTIFY 840 // Barrier imbalance - save arrive time to the thread 841 if (__kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 2) { 842 this_thr->th.th_bar_arrive_time = __itt_get_timestamp(); 843 } 844 #endif 845 846 (void)__kmp_init_hierarchical_barrier_thread(bt, thr_bar, nproc, gtid, tid, 847 team); 848 849 if (thr_bar->my_level) { // not a leaf (my_level==0 means leaf) 850 kmp_int32 child_tid; 851 new_state = 852 (kmp_uint64)team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP; 853 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME && 854 thr_bar->use_oncore_barrier) { 855 if (thr_bar->leaf_kids) { 856 // First, wait for leaf children to check-in on my b_arrived flag 857 kmp_uint64 leaf_state = 858 KMP_MASTER_TID(tid) 859 ? thr_bar->b_arrived | thr_bar->leaf_state 860 : team->t.t_bar[bt].b_arrived | thr_bar->leaf_state; 861 KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) waiting " 862 "for leaf kids\n", 863 gtid, team->t.t_id, tid)); 864 kmp_flag_64 flag(&thr_bar->b_arrived, leaf_state); 865 flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj)); 866 if (reduce) { 867 ANNOTATE_REDUCE_AFTER(reduce); 868 for (child_tid = tid + 1; child_tid <= tid + thr_bar->leaf_kids; 869 ++child_tid) { 870 KA_TRACE(100, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += " 871 "T#%d(%d:%d)\n", 872 gtid, team->t.t_id, tid, 873 __kmp_gtid_from_tid(child_tid, team), team->t.t_id, 874 child_tid)); 875 ANNOTATE_BARRIER_END(other_threads[child_tid]); 876 (*reduce)(this_thr->th.th_local.reduce_data, 877 other_threads[child_tid]->th.th_local.reduce_data); 878 } 879 ANNOTATE_REDUCE_BEFORE(reduce); 880 ANNOTATE_REDUCE_BEFORE(&team->t.t_bar); 881 } 882 // clear leaf_state bits 883 KMP_TEST_THEN_AND64(&thr_bar->b_arrived, ~(thr_bar->leaf_state)); 884 } 885 // Next, wait for higher level children on each child's b_arrived flag 886 for (kmp_uint32 d = 1; d < thr_bar->my_level; 887 ++d) { // gather lowest level threads first, but skip 0 888 kmp_uint32 last = tid + thr_bar->skip_per_level[d + 1], 889 skip = thr_bar->skip_per_level[d]; 890 if (last > nproc) 891 last = nproc; 892 for (child_tid = tid + skip; child_tid < (int)last; child_tid += skip) { 893 kmp_info_t *child_thr = other_threads[child_tid]; 894 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb; 895 KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) wait " 896 "T#%d(%d:%d) " 897 "arrived(%p) == %llu\n", 898 gtid, team->t.t_id, tid, 899 __kmp_gtid_from_tid(child_tid, team), team->t.t_id, 900 child_tid, &child_bar->b_arrived, new_state)); 901 kmp_flag_64 flag(&child_bar->b_arrived, new_state); 902 flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj)); 903 ANNOTATE_BARRIER_END(child_thr); 904 if (reduce) { 905 KA_TRACE(100, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += " 906 "T#%d(%d:%d)\n", 907 gtid, team->t.t_id, tid, 908 __kmp_gtid_from_tid(child_tid, team), team->t.t_id, 909 child_tid)); 910 ANNOTATE_REDUCE_AFTER(reduce); 911 (*reduce)(this_thr->th.th_local.reduce_data, 912 child_thr->th.th_local.reduce_data); 913 ANNOTATE_REDUCE_BEFORE(reduce); 914 ANNOTATE_REDUCE_BEFORE(&team->t.t_bar); 915 } 916 } 917 } 918 } else { // Blocktime is not infinite 919 for (kmp_uint32 d = 0; d < thr_bar->my_level; 920 ++d) { // Gather lowest level threads first 921 kmp_uint32 last = tid + thr_bar->skip_per_level[d + 1], 922 skip = thr_bar->skip_per_level[d]; 923 if (last > nproc) 924 last = nproc; 925 for (child_tid = tid + skip; child_tid < (int)last; child_tid += skip) { 926 kmp_info_t *child_thr = other_threads[child_tid]; 927 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb; 928 KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) wait " 929 "T#%d(%d:%d) " 930 "arrived(%p) == %llu\n", 931 gtid, team->t.t_id, tid, 932 __kmp_gtid_from_tid(child_tid, team), team->t.t_id, 933 child_tid, &child_bar->b_arrived, new_state)); 934 kmp_flag_64 flag(&child_bar->b_arrived, new_state); 935 flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj)); 936 ANNOTATE_BARRIER_END(child_thr); 937 if (reduce) { 938 KA_TRACE(100, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += " 939 "T#%d(%d:%d)\n", 940 gtid, team->t.t_id, tid, 941 __kmp_gtid_from_tid(child_tid, team), team->t.t_id, 942 child_tid)); 943 ANNOTATE_REDUCE_AFTER(reduce); 944 (*reduce)(this_thr->th.th_local.reduce_data, 945 child_thr->th.th_local.reduce_data); 946 ANNOTATE_REDUCE_BEFORE(reduce); 947 ANNOTATE_REDUCE_BEFORE(&team->t.t_bar); 948 } 949 } 950 } 951 } 952 } 953 // All subordinates are gathered; now release parent if not master thread 954 955 if (!KMP_MASTER_TID(tid)) { // worker threads release parent in hierarchy 956 KA_TRACE( 957 20, 958 ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) releasing T#%d(%d:%d) " 959 "arrived(%p): %llu => %llu\n", 960 gtid, team->t.t_id, tid, 961 __kmp_gtid_from_tid(thr_bar->parent_tid, team), team->t.t_id, 962 thr_bar->parent_tid, &thr_bar->b_arrived, thr_bar->b_arrived, 963 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP)); 964 /* Mark arrival to parent: After performing this write, a worker thread may 965 not assume that the team is valid any more - it could be deallocated by 966 the master thread at any time. */ 967 if (thr_bar->my_level || __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME || 968 !thr_bar->use_oncore_barrier) { // Parent is waiting on my b_arrived 969 // flag; release it 970 ANNOTATE_BARRIER_BEGIN(this_thr); 971 kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[thr_bar->parent_tid]); 972 flag.release(); 973 } else { // Leaf does special release on the "offset" bits of parent's 974 // b_arrived flag 975 thr_bar->b_arrived = team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP; 976 kmp_flag_oncore flag(&thr_bar->parent_bar->b_arrived, thr_bar->offset); 977 flag.set_waiter(other_threads[thr_bar->parent_tid]); 978 flag.release(); 979 } 980 } else { // Master thread needs to update the team's b_arrived value 981 team->t.t_bar[bt].b_arrived = new_state; 982 KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) set team %d " 983 "arrived(%p) = %llu\n", 984 gtid, team->t.t_id, tid, team->t.t_id, 985 &team->t.t_bar[bt].b_arrived, team->t.t_bar[bt].b_arrived)); 986 } 987 // Is the team access below unsafe or just technically invalid? 988 KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) exit for " 989 "barrier type %d\n", 990 gtid, team->t.t_id, tid, bt)); 991 } 992 993 static void __kmp_hierarchical_barrier_release( 994 enum barrier_type bt, kmp_info_t *this_thr, int gtid, int tid, 995 int propagate_icvs USE_ITT_BUILD_ARG(void *itt_sync_obj)) { 996 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_hier_release); 997 kmp_team_t *team; 998 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb; 999 kmp_uint32 nproc; 1000 bool team_change = false; // indicates on-core barrier shouldn't be used 1001 1002 if (KMP_MASTER_TID(tid)) { 1003 team = __kmp_threads[gtid]->th.th_team; 1004 KMP_DEBUG_ASSERT(team != NULL); 1005 KA_TRACE(20, ("__kmp_hierarchical_barrier_release: T#%d(%d:%d) master " 1006 "entered barrier type %d\n", 1007 gtid, team->t.t_id, tid, bt)); 1008 } else { // Worker threads 1009 // Wait for parent thread to release me 1010 if (!thr_bar->use_oncore_barrier || 1011 __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME || thr_bar->my_level != 0 || 1012 thr_bar->team == NULL) { 1013 // Use traditional method of waiting on my own b_go flag 1014 thr_bar->wait_flag = KMP_BARRIER_OWN_FLAG; 1015 kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP); 1016 flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj)); 1017 ANNOTATE_BARRIER_END(this_thr); 1018 TCW_8(thr_bar->b_go, 1019 KMP_INIT_BARRIER_STATE); // Reset my b_go flag for next time 1020 } else { // Thread barrier data is initialized, this is a leaf, blocktime is 1021 // infinite, not nested 1022 // Wait on my "offset" bits on parent's b_go flag 1023 thr_bar->wait_flag = KMP_BARRIER_PARENT_FLAG; 1024 kmp_flag_oncore flag(&thr_bar->parent_bar->b_go, KMP_BARRIER_STATE_BUMP, 1025 thr_bar->offset, bt, 1026 this_thr USE_ITT_BUILD_ARG(itt_sync_obj)); 1027 flag.wait(this_thr, TRUE); 1028 if (thr_bar->wait_flag == 1029 KMP_BARRIER_SWITCHING) { // Thread was switched to own b_go 1030 TCW_8(thr_bar->b_go, 1031 KMP_INIT_BARRIER_STATE); // Reset my b_go flag for next time 1032 } else { // Reset my bits on parent's b_go flag 1033 (RCAST(volatile char *, 1034 &(thr_bar->parent_bar->b_go)))[thr_bar->offset] = 0; 1035 } 1036 } 1037 thr_bar->wait_flag = KMP_BARRIER_NOT_WAITING; 1038 // Early exit for reaping threads releasing forkjoin barrier 1039 if (bt == bs_forkjoin_barrier && TCR_4(__kmp_global.g.g_done)) 1040 return; 1041 // The worker thread may now assume that the team is valid. 1042 team = __kmp_threads[gtid]->th.th_team; 1043 KMP_DEBUG_ASSERT(team != NULL); 1044 tid = __kmp_tid_from_gtid(gtid); 1045 1046 KA_TRACE( 1047 20, 1048 ("__kmp_hierarchical_barrier_release: T#%d(%d:%d) set go(%p) = %u\n", 1049 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE)); 1050 KMP_MB(); // Flush all pending memory write invalidates. 1051 } 1052 1053 nproc = this_thr->th.th_team_nproc; 1054 int level = team->t.t_level; 1055 #if OMP_40_ENABLED 1056 if (team->t.t_threads[0] 1057 ->th.th_teams_microtask) { // are we inside the teams construct? 1058 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master && 1059 this_thr->th.th_teams_level == level) 1060 ++level; // level was not increased in teams construct for team_of_workers 1061 if (this_thr->th.th_teams_size.nteams > 1) 1062 ++level; // level was not increased in teams construct for team_of_masters 1063 } 1064 #endif 1065 if (level == 1) 1066 thr_bar->use_oncore_barrier = 1; 1067 else 1068 thr_bar->use_oncore_barrier = 0; // Do not use oncore barrier when nested 1069 1070 // If the team size has increased, we still communicate with old leaves via 1071 // oncore barrier. 1072 unsigned short int old_leaf_kids = thr_bar->leaf_kids; 1073 kmp_uint64 old_leaf_state = thr_bar->leaf_state; 1074 team_change = __kmp_init_hierarchical_barrier_thread(bt, thr_bar, nproc, gtid, 1075 tid, team); 1076 // But if the entire team changes, we won't use oncore barrier at all 1077 if (team_change) 1078 old_leaf_kids = 0; 1079 1080 #if KMP_BARRIER_ICV_PUSH 1081 if (propagate_icvs) { 1082 __kmp_init_implicit_task(team->t.t_ident, team->t.t_threads[tid], team, tid, 1083 FALSE); 1084 if (KMP_MASTER_TID( 1085 tid)) { // master already has copy in final destination; copy 1086 copy_icvs(&thr_bar->th_fixed_icvs, 1087 &team->t.t_implicit_task_taskdata[tid].td_icvs); 1088 } else if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME && 1089 thr_bar->use_oncore_barrier) { // optimization for inf blocktime 1090 if (!thr_bar->my_level) // I'm a leaf in the hierarchy (my_level==0) 1091 // leaves (on-core children) pull parent's fixed ICVs directly to local 1092 // ICV store 1093 copy_icvs(&team->t.t_implicit_task_taskdata[tid].td_icvs, 1094 &thr_bar->parent_bar->th_fixed_icvs); 1095 // non-leaves will get ICVs piggybacked with b_go via NGO store 1096 } else { // blocktime is not infinite; pull ICVs from parent's fixed ICVs 1097 if (thr_bar->my_level) // not a leaf; copy ICVs to my fixed ICVs child can 1098 // access 1099 copy_icvs(&thr_bar->th_fixed_icvs, &thr_bar->parent_bar->th_fixed_icvs); 1100 else // leaves copy parent's fixed ICVs directly to local ICV store 1101 copy_icvs(&team->t.t_implicit_task_taskdata[tid].td_icvs, 1102 &thr_bar->parent_bar->th_fixed_icvs); 1103 } 1104 } 1105 #endif // KMP_BARRIER_ICV_PUSH 1106 1107 // Now, release my children 1108 if (thr_bar->my_level) { // not a leaf 1109 kmp_int32 child_tid; 1110 kmp_uint32 last; 1111 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME && 1112 thr_bar->use_oncore_barrier) { 1113 if (KMP_MASTER_TID(tid)) { // do a flat release 1114 // Set local b_go to bump children via NGO store of the cache line 1115 // containing IVCs and b_go. 1116 thr_bar->b_go = KMP_BARRIER_STATE_BUMP; 1117 // Use ngo stores if available; b_go piggybacks in the last 8 bytes of 1118 // the cache line 1119 ngo_load(&thr_bar->th_fixed_icvs); 1120 // This loops over all the threads skipping only the leaf nodes in the 1121 // hierarchy 1122 for (child_tid = thr_bar->skip_per_level[1]; child_tid < (int)nproc; 1123 child_tid += thr_bar->skip_per_level[1]) { 1124 kmp_bstate_t *child_bar = 1125 &team->t.t_threads[child_tid]->th.th_bar[bt].bb; 1126 KA_TRACE(20, ("__kmp_hierarchical_barrier_release: T#%d(%d:%d) " 1127 "releasing T#%d(%d:%d)" 1128 " go(%p): %u => %u\n", 1129 gtid, team->t.t_id, tid, 1130 __kmp_gtid_from_tid(child_tid, team), team->t.t_id, 1131 child_tid, &child_bar->b_go, child_bar->b_go, 1132 child_bar->b_go + KMP_BARRIER_STATE_BUMP)); 1133 // Use ngo store (if available) to both store ICVs and release child 1134 // via child's b_go 1135 ngo_store_go(&child_bar->th_fixed_icvs, &thr_bar->th_fixed_icvs); 1136 } 1137 ngo_sync(); 1138 } 1139 TCW_8(thr_bar->b_go, 1140 KMP_INIT_BARRIER_STATE); // Reset my b_go flag for next time 1141 // Now, release leaf children 1142 if (thr_bar->leaf_kids) { // if there are any 1143 // We test team_change on the off-chance that the level 1 team changed. 1144 if (team_change || 1145 old_leaf_kids < thr_bar->leaf_kids) { // some old, some new 1146 if (old_leaf_kids) { // release old leaf kids 1147 thr_bar->b_go |= old_leaf_state; 1148 } 1149 // Release new leaf kids 1150 last = tid + thr_bar->skip_per_level[1]; 1151 if (last > nproc) 1152 last = nproc; 1153 for (child_tid = tid + 1 + old_leaf_kids; child_tid < (int)last; 1154 ++child_tid) { // skip_per_level[0]=1 1155 kmp_info_t *child_thr = team->t.t_threads[child_tid]; 1156 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb; 1157 KA_TRACE( 1158 20, 1159 ("__kmp_hierarchical_barrier_release: T#%d(%d:%d) releasing" 1160 " T#%d(%d:%d) go(%p): %u => %u\n", 1161 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team), 1162 team->t.t_id, child_tid, &child_bar->b_go, child_bar->b_go, 1163 child_bar->b_go + KMP_BARRIER_STATE_BUMP)); 1164 // Release child using child's b_go flag 1165 ANNOTATE_BARRIER_BEGIN(child_thr); 1166 kmp_flag_64 flag(&child_bar->b_go, child_thr); 1167 flag.release(); 1168 } 1169 } else { // Release all children at once with leaf_state bits on my own 1170 // b_go flag 1171 thr_bar->b_go |= thr_bar->leaf_state; 1172 } 1173 } 1174 } else { // Blocktime is not infinite; do a simple hierarchical release 1175 for (int d = thr_bar->my_level - 1; d >= 0; 1176 --d) { // Release highest level threads first 1177 last = tid + thr_bar->skip_per_level[d + 1]; 1178 kmp_uint32 skip = thr_bar->skip_per_level[d]; 1179 if (last > nproc) 1180 last = nproc; 1181 for (child_tid = tid + skip; child_tid < (int)last; child_tid += skip) { 1182 kmp_info_t *child_thr = team->t.t_threads[child_tid]; 1183 kmp_bstate_t *child_bar = &child_thr->th.th_bar[bt].bb; 1184 KA_TRACE(20, ("__kmp_hierarchical_barrier_release: T#%d(%d:%d) " 1185 "releasing T#%d(%d:%d) go(%p): %u => %u\n", 1186 gtid, team->t.t_id, tid, 1187 __kmp_gtid_from_tid(child_tid, team), team->t.t_id, 1188 child_tid, &child_bar->b_go, child_bar->b_go, 1189 child_bar->b_go + KMP_BARRIER_STATE_BUMP)); 1190 // Release child using child's b_go flag 1191 ANNOTATE_BARRIER_BEGIN(child_thr); 1192 kmp_flag_64 flag(&child_bar->b_go, child_thr); 1193 flag.release(); 1194 } 1195 } 1196 } 1197 #if KMP_BARRIER_ICV_PUSH 1198 if (propagate_icvs && !KMP_MASTER_TID(tid)) 1199 // non-leaves copy ICVs from fixed ICVs to local dest 1200 copy_icvs(&team->t.t_implicit_task_taskdata[tid].td_icvs, 1201 &thr_bar->th_fixed_icvs); 1202 #endif // KMP_BARRIER_ICV_PUSH 1203 } 1204 KA_TRACE(20, ("__kmp_hierarchical_barrier_release: T#%d(%d:%d) exit for " 1205 "barrier type %d\n", 1206 gtid, team->t.t_id, tid, bt)); 1207 } 1208 1209 // End of Barrier Algorithms 1210 1211 // Internal function to do a barrier. 1212 /* If is_split is true, do a split barrier, otherwise, do a plain barrier 1213 If reduce is non-NULL, do a split reduction barrier, otherwise, do a split 1214 barrier 1215 Returns 0 if master thread, 1 if worker thread. */ 1216 int __kmp_barrier(enum barrier_type bt, int gtid, int is_split, 1217 size_t reduce_size, void *reduce_data, 1218 void (*reduce)(void *, void *)) { 1219 KMP_TIME_PARTITIONED_BLOCK(OMP_plain_barrier); 1220 KMP_SET_THREAD_STATE_BLOCK(PLAIN_BARRIER); 1221 int tid = __kmp_tid_from_gtid(gtid); 1222 kmp_info_t *this_thr = __kmp_threads[gtid]; 1223 kmp_team_t *team = this_thr->th.th_team; 1224 int status = 0; 1225 ident_t *loc = __kmp_threads[gtid]->th.th_ident; 1226 #if OMPT_SUPPORT 1227 ompt_task_id_t my_task_id; 1228 ompt_parallel_id_t my_parallel_id; 1229 #endif 1230 1231 KA_TRACE(15, ("__kmp_barrier: T#%d(%d:%d) has arrived\n", gtid, 1232 __kmp_team_from_gtid(gtid)->t.t_id, __kmp_tid_from_gtid(gtid))); 1233 1234 ANNOTATE_BARRIER_BEGIN(&team->t.t_bar); 1235 #if OMPT_SUPPORT 1236 if (ompt_enabled) { 1237 #if OMPT_BLAME 1238 my_task_id = team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id; 1239 my_parallel_id = team->t.ompt_team_info.parallel_id; 1240 1241 #if OMPT_TRACE 1242 if (this_thr->th.ompt_thread_info.state == ompt_state_wait_single) { 1243 if (ompt_callbacks.ompt_callback(ompt_event_single_others_end)) { 1244 ompt_callbacks.ompt_callback(ompt_event_single_others_end)( 1245 my_parallel_id, my_task_id); 1246 } 1247 } 1248 #endif 1249 if (ompt_callbacks.ompt_callback(ompt_event_barrier_begin)) { 1250 ompt_callbacks.ompt_callback(ompt_event_barrier_begin)(my_parallel_id, 1251 my_task_id); 1252 } 1253 #endif 1254 // It is OK to report the barrier state after the barrier begin callback. 1255 // According to the OMPT specification, a compliant implementation may 1256 // even delay reporting this state until the barrier begins to wait. 1257 this_thr->th.ompt_thread_info.state = ompt_state_wait_barrier; 1258 } 1259 #endif 1260 1261 if (!team->t.t_serialized) { 1262 #if USE_ITT_BUILD 1263 // This value will be used in itt notify events below. 1264 void *itt_sync_obj = NULL; 1265 #if USE_ITT_NOTIFY 1266 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) 1267 itt_sync_obj = __kmp_itt_barrier_object(gtid, bt, 1); 1268 #endif 1269 #endif /* USE_ITT_BUILD */ 1270 if (__kmp_tasking_mode == tskm_extra_barrier) { 1271 __kmp_tasking_barrier(team, this_thr, gtid); 1272 KA_TRACE(15, 1273 ("__kmp_barrier: T#%d(%d:%d) past tasking barrier\n", gtid, 1274 __kmp_team_from_gtid(gtid)->t.t_id, __kmp_tid_from_gtid(gtid))); 1275 } 1276 1277 /* Copy the blocktime info to the thread, where __kmp_wait_template() can 1278 access it when the team struct is not guaranteed to exist. */ 1279 // See note about the corresponding code in __kmp_join_barrier() being 1280 // performance-critical. 1281 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) { 1282 #if KMP_USE_MONITOR 1283 this_thr->th.th_team_bt_intervals = 1284 team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals; 1285 this_thr->th.th_team_bt_set = 1286 team->t.t_implicit_task_taskdata[tid].td_icvs.bt_set; 1287 #else 1288 this_thr->th.th_team_bt_intervals = KMP_BLOCKTIME_INTERVAL(team, tid); 1289 #endif 1290 } 1291 1292 #if USE_ITT_BUILD 1293 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) 1294 __kmp_itt_barrier_starting(gtid, itt_sync_obj); 1295 #endif /* USE_ITT_BUILD */ 1296 #if USE_DEBUGGER 1297 // Let the debugger know: the thread arrived to the barrier and waiting. 1298 if (KMP_MASTER_TID(tid)) { // Master counter is stored in team structure. 1299 team->t.t_bar[bt].b_master_arrived += 1; 1300 } else { 1301 this_thr->th.th_bar[bt].bb.b_worker_arrived += 1; 1302 } // if 1303 #endif /* USE_DEBUGGER */ 1304 if (reduce != NULL) { 1305 // KMP_DEBUG_ASSERT( is_split == TRUE ); // #C69956 1306 this_thr->th.th_local.reduce_data = reduce_data; 1307 } 1308 1309 if (KMP_MASTER_TID(tid) && __kmp_tasking_mode != tskm_immediate_exec) 1310 __kmp_task_team_setup( 1311 this_thr, team, 1312 0); // use 0 to only setup the current team if nthreads > 1 1313 1314 switch (__kmp_barrier_gather_pattern[bt]) { 1315 case bp_hyper_bar: { 1316 KMP_ASSERT(__kmp_barrier_gather_branch_bits[bt]); // don't set branch bits 1317 // to 0; use linear 1318 __kmp_hyper_barrier_gather(bt, this_thr, gtid, tid, 1319 reduce USE_ITT_BUILD_ARG(itt_sync_obj)); 1320 break; 1321 } 1322 case bp_hierarchical_bar: { 1323 __kmp_hierarchical_barrier_gather(bt, this_thr, gtid, tid, 1324 reduce USE_ITT_BUILD_ARG(itt_sync_obj)); 1325 break; 1326 } 1327 case bp_tree_bar: { 1328 KMP_ASSERT(__kmp_barrier_gather_branch_bits[bt]); // don't set branch bits 1329 // to 0; use linear 1330 __kmp_tree_barrier_gather(bt, this_thr, gtid, tid, 1331 reduce USE_ITT_BUILD_ARG(itt_sync_obj)); 1332 break; 1333 } 1334 default: { 1335 __kmp_linear_barrier_gather(bt, this_thr, gtid, tid, 1336 reduce USE_ITT_BUILD_ARG(itt_sync_obj)); 1337 } 1338 } 1339 1340 KMP_MB(); 1341 1342 if (KMP_MASTER_TID(tid)) { 1343 status = 0; 1344 if (__kmp_tasking_mode != tskm_immediate_exec) { 1345 __kmp_task_team_wait(this_thr, team USE_ITT_BUILD_ARG(itt_sync_obj)); 1346 } 1347 #if USE_DEBUGGER 1348 // Let the debugger know: All threads are arrived and starting leaving the 1349 // barrier. 1350 team->t.t_bar[bt].b_team_arrived += 1; 1351 #endif 1352 1353 #if OMP_40_ENABLED 1354 // Reset cancellation flag for worksharing constructs 1355 if (team->t.t_cancel_request == cancel_loop || 1356 team->t.t_cancel_request == cancel_sections) { 1357 team->t.t_cancel_request = cancel_noreq; 1358 } 1359 #endif 1360 #if USE_ITT_BUILD 1361 /* TODO: In case of split reduction barrier, master thread may send 1362 acquired event early, before the final summation into the shared 1363 variable is done (final summation can be a long operation for array 1364 reductions). */ 1365 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) 1366 __kmp_itt_barrier_middle(gtid, itt_sync_obj); 1367 #endif /* USE_ITT_BUILD */ 1368 #if USE_ITT_BUILD && USE_ITT_NOTIFY 1369 // Barrier - report frame end (only if active_level == 1) 1370 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) && 1371 __kmp_forkjoin_frames_mode && 1372 #if OMP_40_ENABLED 1373 this_thr->th.th_teams_microtask == NULL && 1374 #endif 1375 team->t.t_active_level == 1) { 1376 kmp_uint64 cur_time = __itt_get_timestamp(); 1377 kmp_info_t **other_threads = team->t.t_threads; 1378 int nproc = this_thr->th.th_team_nproc; 1379 int i; 1380 switch (__kmp_forkjoin_frames_mode) { 1381 case 1: 1382 __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0, 1383 loc, nproc); 1384 this_thr->th.th_frame_time = cur_time; 1385 break; 1386 case 2: // AC 2015-01-19: currently does not work for hierarchical (to 1387 // be fixed) 1388 __kmp_itt_frame_submit(gtid, this_thr->th.th_bar_min_time, cur_time, 1389 1, loc, nproc); 1390 break; 1391 case 3: 1392 if (__itt_metadata_add_ptr) { 1393 // Initialize with master's wait time 1394 kmp_uint64 delta = cur_time - this_thr->th.th_bar_arrive_time; 1395 // Set arrive time to zero to be able to check it in 1396 // __kmp_invoke_task(); the same is done inside the loop below 1397 this_thr->th.th_bar_arrive_time = 0; 1398 for (i = 1; i < nproc; ++i) { 1399 delta += (cur_time - other_threads[i]->th.th_bar_arrive_time); 1400 other_threads[i]->th.th_bar_arrive_time = 0; 1401 } 1402 __kmp_itt_metadata_imbalance(gtid, this_thr->th.th_frame_time, 1403 cur_time, delta, 1404 (kmp_uint64)(reduce != NULL)); 1405 } 1406 __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0, 1407 loc, nproc); 1408 this_thr->th.th_frame_time = cur_time; 1409 break; 1410 } 1411 } 1412 #endif /* USE_ITT_BUILD */ 1413 } else { 1414 status = 1; 1415 #if USE_ITT_BUILD 1416 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) 1417 __kmp_itt_barrier_middle(gtid, itt_sync_obj); 1418 #endif /* USE_ITT_BUILD */ 1419 } 1420 if (status == 1 || !is_split) { 1421 switch (__kmp_barrier_release_pattern[bt]) { 1422 case bp_hyper_bar: { 1423 KMP_ASSERT(__kmp_barrier_release_branch_bits[bt]); 1424 __kmp_hyper_barrier_release(bt, this_thr, gtid, tid, 1425 FALSE USE_ITT_BUILD_ARG(itt_sync_obj)); 1426 break; 1427 } 1428 case bp_hierarchical_bar: { 1429 __kmp_hierarchical_barrier_release( 1430 bt, this_thr, gtid, tid, FALSE USE_ITT_BUILD_ARG(itt_sync_obj)); 1431 break; 1432 } 1433 case bp_tree_bar: { 1434 KMP_ASSERT(__kmp_barrier_release_branch_bits[bt]); 1435 __kmp_tree_barrier_release(bt, this_thr, gtid, tid, 1436 FALSE USE_ITT_BUILD_ARG(itt_sync_obj)); 1437 break; 1438 } 1439 default: { 1440 __kmp_linear_barrier_release(bt, this_thr, gtid, tid, 1441 FALSE USE_ITT_BUILD_ARG(itt_sync_obj)); 1442 } 1443 } 1444 if (__kmp_tasking_mode != tskm_immediate_exec) { 1445 __kmp_task_team_sync(this_thr, team); 1446 } 1447 } 1448 1449 #if USE_ITT_BUILD 1450 /* GEH: TODO: Move this under if-condition above and also include in 1451 __kmp_end_split_barrier(). This will more accurately represent the actual 1452 release time of the threads for split barriers. */ 1453 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) 1454 __kmp_itt_barrier_finished(gtid, itt_sync_obj); 1455 #endif /* USE_ITT_BUILD */ 1456 } else { // Team is serialized. 1457 status = 0; 1458 if (__kmp_tasking_mode != tskm_immediate_exec) { 1459 #if OMP_45_ENABLED 1460 if (this_thr->th.th_task_team != NULL) { 1461 void *itt_sync_obj = NULL; 1462 #if USE_ITT_NOTIFY 1463 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) { 1464 itt_sync_obj = __kmp_itt_barrier_object(gtid, bt, 1); 1465 __kmp_itt_barrier_starting(gtid, itt_sync_obj); 1466 } 1467 #endif 1468 1469 KMP_DEBUG_ASSERT(this_thr->th.th_task_team->tt.tt_found_proxy_tasks == 1470 TRUE); 1471 __kmp_task_team_wait(this_thr, team USE_ITT_BUILD_ARG(itt_sync_obj)); 1472 __kmp_task_team_setup(this_thr, team, 0); 1473 1474 #if USE_ITT_BUILD 1475 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) 1476 __kmp_itt_barrier_finished(gtid, itt_sync_obj); 1477 #endif /* USE_ITT_BUILD */ 1478 } 1479 #else 1480 // The task team should be NULL for serialized code (tasks will be 1481 // executed immediately) 1482 KMP_DEBUG_ASSERT(team->t.t_task_team[this_thr->th.th_task_state] == NULL); 1483 KMP_DEBUG_ASSERT(this_thr->th.th_task_team == NULL); 1484 #endif 1485 } 1486 } 1487 KA_TRACE(15, ("__kmp_barrier: T#%d(%d:%d) is leaving with return value %d\n", 1488 gtid, __kmp_team_from_gtid(gtid)->t.t_id, 1489 __kmp_tid_from_gtid(gtid), status)); 1490 1491 #if OMPT_SUPPORT 1492 if (ompt_enabled) { 1493 #if OMPT_BLAME 1494 if (ompt_callbacks.ompt_callback(ompt_event_barrier_end)) { 1495 ompt_callbacks.ompt_callback(ompt_event_barrier_end)(my_parallel_id, 1496 my_task_id); 1497 } 1498 #endif 1499 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel; 1500 } 1501 #endif 1502 ANNOTATE_BARRIER_END(&team->t.t_bar); 1503 1504 return status; 1505 } 1506 1507 void __kmp_end_split_barrier(enum barrier_type bt, int gtid) { 1508 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_end_split_barrier); 1509 KMP_SET_THREAD_STATE_BLOCK(PLAIN_BARRIER); 1510 int tid = __kmp_tid_from_gtid(gtid); 1511 kmp_info_t *this_thr = __kmp_threads[gtid]; 1512 kmp_team_t *team = this_thr->th.th_team; 1513 1514 ANNOTATE_BARRIER_BEGIN(&team->t.t_bar); 1515 if (!team->t.t_serialized) { 1516 if (KMP_MASTER_GTID(gtid)) { 1517 switch (__kmp_barrier_release_pattern[bt]) { 1518 case bp_hyper_bar: { 1519 KMP_ASSERT(__kmp_barrier_release_branch_bits[bt]); 1520 __kmp_hyper_barrier_release(bt, this_thr, gtid, tid, 1521 FALSE USE_ITT_BUILD_ARG(NULL)); 1522 break; 1523 } 1524 case bp_hierarchical_bar: { 1525 __kmp_hierarchical_barrier_release(bt, this_thr, gtid, tid, 1526 FALSE USE_ITT_BUILD_ARG(NULL)); 1527 break; 1528 } 1529 case bp_tree_bar: { 1530 KMP_ASSERT(__kmp_barrier_release_branch_bits[bt]); 1531 __kmp_tree_barrier_release(bt, this_thr, gtid, tid, 1532 FALSE USE_ITT_BUILD_ARG(NULL)); 1533 break; 1534 } 1535 default: { 1536 __kmp_linear_barrier_release(bt, this_thr, gtid, tid, 1537 FALSE USE_ITT_BUILD_ARG(NULL)); 1538 } 1539 } 1540 if (__kmp_tasking_mode != tskm_immediate_exec) { 1541 __kmp_task_team_sync(this_thr, team); 1542 } // if 1543 } 1544 } 1545 ANNOTATE_BARRIER_END(&team->t.t_bar); 1546 } 1547 1548 void __kmp_join_barrier(int gtid) { 1549 KMP_TIME_PARTITIONED_BLOCK(OMP_join_barrier); 1550 KMP_SET_THREAD_STATE_BLOCK(FORK_JOIN_BARRIER); 1551 kmp_info_t *this_thr = __kmp_threads[gtid]; 1552 kmp_team_t *team; 1553 kmp_uint nproc; 1554 kmp_info_t *master_thread; 1555 int tid; 1556 #ifdef KMP_DEBUG 1557 int team_id; 1558 #endif /* KMP_DEBUG */ 1559 #if USE_ITT_BUILD 1560 void *itt_sync_obj = NULL; 1561 #if USE_ITT_NOTIFY 1562 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) // Don't call routine without need 1563 // Get object created at fork_barrier 1564 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier); 1565 #endif 1566 #endif /* USE_ITT_BUILD */ 1567 KMP_MB(); 1568 1569 // Get current info 1570 team = this_thr->th.th_team; 1571 nproc = this_thr->th.th_team_nproc; 1572 KMP_DEBUG_ASSERT((int)nproc == team->t.t_nproc); 1573 tid = __kmp_tid_from_gtid(gtid); 1574 #ifdef KMP_DEBUG 1575 team_id = team->t.t_id; 1576 #endif /* KMP_DEBUG */ 1577 master_thread = this_thr->th.th_team_master; 1578 #ifdef KMP_DEBUG 1579 if (master_thread != team->t.t_threads[0]) { 1580 __kmp_print_structure(); 1581 } 1582 #endif /* KMP_DEBUG */ 1583 KMP_DEBUG_ASSERT(master_thread == team->t.t_threads[0]); 1584 KMP_MB(); 1585 1586 // Verify state 1587 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]); 1588 KMP_DEBUG_ASSERT(TCR_PTR(this_thr->th.th_team)); 1589 KMP_DEBUG_ASSERT(TCR_PTR(this_thr->th.th_root)); 1590 KMP_DEBUG_ASSERT(this_thr == team->t.t_threads[tid]); 1591 KA_TRACE(10, ("__kmp_join_barrier: T#%d(%d:%d) arrived at join barrier\n", 1592 gtid, team_id, tid)); 1593 1594 ANNOTATE_BARRIER_BEGIN(&team->t.t_bar); 1595 #if OMPT_SUPPORT 1596 #if OMPT_TRACE 1597 if (ompt_enabled && ompt_callbacks.ompt_callback(ompt_event_barrier_begin)) { 1598 ompt_callbacks.ompt_callback(ompt_event_barrier_begin)( 1599 team->t.ompt_team_info.parallel_id, 1600 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id); 1601 } 1602 #endif 1603 this_thr->th.ompt_thread_info.state = ompt_state_wait_barrier; 1604 #endif 1605 1606 if (__kmp_tasking_mode == tskm_extra_barrier) { 1607 __kmp_tasking_barrier(team, this_thr, gtid); 1608 KA_TRACE(10, ("__kmp_join_barrier: T#%d(%d:%d) past taking barrier\n", gtid, 1609 team_id, tid)); 1610 } 1611 #ifdef KMP_DEBUG 1612 if (__kmp_tasking_mode != tskm_immediate_exec) { 1613 KA_TRACE(20, ("__kmp_join_barrier: T#%d, old team = %d, old task_team = " 1614 "%p, th_task_team = %p\n", 1615 __kmp_gtid_from_thread(this_thr), team_id, 1616 team->t.t_task_team[this_thr->th.th_task_state], 1617 this_thr->th.th_task_team)); 1618 KMP_DEBUG_ASSERT(this_thr->th.th_task_team == 1619 team->t.t_task_team[this_thr->th.th_task_state]); 1620 } 1621 #endif /* KMP_DEBUG */ 1622 1623 /* Copy the blocktime info to the thread, where __kmp_wait_template() can 1624 access it when the team struct is not guaranteed to exist. Doing these 1625 loads causes a cache miss slows down EPCC parallel by 2x. As a workaround, 1626 we do not perform the copy if blocktime=infinite, since the values are not 1627 used by __kmp_wait_template() in that case. */ 1628 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) { 1629 #if KMP_USE_MONITOR 1630 this_thr->th.th_team_bt_intervals = 1631 team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals; 1632 this_thr->th.th_team_bt_set = 1633 team->t.t_implicit_task_taskdata[tid].td_icvs.bt_set; 1634 #else 1635 this_thr->th.th_team_bt_intervals = KMP_BLOCKTIME_INTERVAL(team, tid); 1636 #endif 1637 } 1638 1639 #if USE_ITT_BUILD 1640 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) 1641 __kmp_itt_barrier_starting(gtid, itt_sync_obj); 1642 #endif /* USE_ITT_BUILD */ 1643 1644 switch (__kmp_barrier_gather_pattern[bs_forkjoin_barrier]) { 1645 case bp_hyper_bar: { 1646 KMP_ASSERT(__kmp_barrier_gather_branch_bits[bs_forkjoin_barrier]); 1647 __kmp_hyper_barrier_gather(bs_forkjoin_barrier, this_thr, gtid, tid, 1648 NULL USE_ITT_BUILD_ARG(itt_sync_obj)); 1649 break; 1650 } 1651 case bp_hierarchical_bar: { 1652 __kmp_hierarchical_barrier_gather(bs_forkjoin_barrier, this_thr, gtid, tid, 1653 NULL USE_ITT_BUILD_ARG(itt_sync_obj)); 1654 break; 1655 } 1656 case bp_tree_bar: { 1657 KMP_ASSERT(__kmp_barrier_gather_branch_bits[bs_forkjoin_barrier]); 1658 __kmp_tree_barrier_gather(bs_forkjoin_barrier, this_thr, gtid, tid, 1659 NULL USE_ITT_BUILD_ARG(itt_sync_obj)); 1660 break; 1661 } 1662 default: { 1663 __kmp_linear_barrier_gather(bs_forkjoin_barrier, this_thr, gtid, tid, 1664 NULL USE_ITT_BUILD_ARG(itt_sync_obj)); 1665 } 1666 } 1667 1668 /* From this point on, the team data structure may be deallocated at any time 1669 by the master thread - it is unsafe to reference it in any of the worker 1670 threads. Any per-team data items that need to be referenced before the 1671 end of the barrier should be moved to the kmp_task_team_t structs. */ 1672 if (KMP_MASTER_TID(tid)) { 1673 if (__kmp_tasking_mode != tskm_immediate_exec) { 1674 __kmp_task_team_wait(this_thr, team USE_ITT_BUILD_ARG(itt_sync_obj)); 1675 } 1676 #if KMP_STATS_ENABLED 1677 // Have master thread flag the workers to indicate they are now waiting for 1678 // next parallel region, Also wake them up so they switch their timers to 1679 // idle. 1680 for (int i = 0; i < team->t.t_nproc; ++i) { 1681 kmp_info_t *team_thread = team->t.t_threads[i]; 1682 if (team_thread == this_thr) 1683 continue; 1684 team_thread->th.th_stats->setIdleFlag(); 1685 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME && 1686 team_thread->th.th_sleep_loc != NULL) 1687 __kmp_null_resume_wrapper(__kmp_gtid_from_thread(team_thread), 1688 team_thread->th.th_sleep_loc); 1689 } 1690 #endif 1691 #if USE_ITT_BUILD 1692 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) 1693 __kmp_itt_barrier_middle(gtid, itt_sync_obj); 1694 #endif /* USE_ITT_BUILD */ 1695 1696 #if USE_ITT_BUILD && USE_ITT_NOTIFY 1697 // Join barrier - report frame end 1698 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) && 1699 __kmp_forkjoin_frames_mode && 1700 #if OMP_40_ENABLED 1701 this_thr->th.th_teams_microtask == NULL && 1702 #endif 1703 team->t.t_active_level == 1) { 1704 kmp_uint64 cur_time = __itt_get_timestamp(); 1705 ident_t *loc = team->t.t_ident; 1706 kmp_info_t **other_threads = team->t.t_threads; 1707 int nproc = this_thr->th.th_team_nproc; 1708 int i; 1709 switch (__kmp_forkjoin_frames_mode) { 1710 case 1: 1711 __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0, 1712 loc, nproc); 1713 break; 1714 case 2: 1715 __kmp_itt_frame_submit(gtid, this_thr->th.th_bar_min_time, cur_time, 1, 1716 loc, nproc); 1717 break; 1718 case 3: 1719 if (__itt_metadata_add_ptr) { 1720 // Initialize with master's wait time 1721 kmp_uint64 delta = cur_time - this_thr->th.th_bar_arrive_time; 1722 // Set arrive time to zero to be able to check it in 1723 // __kmp_invoke_task(); the same is done inside the loop below 1724 this_thr->th.th_bar_arrive_time = 0; 1725 for (i = 1; i < nproc; ++i) { 1726 delta += (cur_time - other_threads[i]->th.th_bar_arrive_time); 1727 other_threads[i]->th.th_bar_arrive_time = 0; 1728 } 1729 __kmp_itt_metadata_imbalance(gtid, this_thr->th.th_frame_time, 1730 cur_time, delta, 0); 1731 } 1732 __kmp_itt_frame_submit(gtid, this_thr->th.th_frame_time, cur_time, 0, 1733 loc, nproc); 1734 this_thr->th.th_frame_time = cur_time; 1735 break; 1736 } 1737 } 1738 #endif /* USE_ITT_BUILD */ 1739 } 1740 #if USE_ITT_BUILD 1741 else { 1742 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) 1743 __kmp_itt_barrier_middle(gtid, itt_sync_obj); 1744 } 1745 #endif /* USE_ITT_BUILD */ 1746 1747 #if KMP_DEBUG 1748 if (KMP_MASTER_TID(tid)) { 1749 KA_TRACE( 1750 15, 1751 ("__kmp_join_barrier: T#%d(%d:%d) says all %d team threads arrived\n", 1752 gtid, team_id, tid, nproc)); 1753 } 1754 #endif /* KMP_DEBUG */ 1755 1756 // TODO now, mark worker threads as done so they may be disbanded 1757 KMP_MB(); // Flush all pending memory write invalidates. 1758 KA_TRACE(10, 1759 ("__kmp_join_barrier: T#%d(%d:%d) leaving\n", gtid, team_id, tid)); 1760 1761 #if OMPT_SUPPORT 1762 if (ompt_enabled) { 1763 #if OMPT_BLAME 1764 if (ompt_callbacks.ompt_callback(ompt_event_barrier_end)) { 1765 ompt_callbacks.ompt_callback(ompt_event_barrier_end)( 1766 team->t.ompt_team_info.parallel_id, 1767 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id); 1768 } 1769 #endif 1770 1771 // return to default state 1772 this_thr->th.ompt_thread_info.state = ompt_state_overhead; 1773 } 1774 #endif 1775 ANNOTATE_BARRIER_END(&team->t.t_bar); 1776 } 1777 1778 // TODO release worker threads' fork barriers as we are ready instead of all at 1779 // once 1780 void __kmp_fork_barrier(int gtid, int tid) { 1781 KMP_TIME_PARTITIONED_BLOCK(OMP_fork_barrier); 1782 KMP_SET_THREAD_STATE_BLOCK(FORK_JOIN_BARRIER); 1783 kmp_info_t *this_thr = __kmp_threads[gtid]; 1784 kmp_team_t *team = (tid == 0) ? this_thr->th.th_team : NULL; 1785 #if USE_ITT_BUILD 1786 void *itt_sync_obj = NULL; 1787 #endif /* USE_ITT_BUILD */ 1788 if (team) 1789 ANNOTATE_BARRIER_END(&team->t.t_bar); 1790 1791 KA_TRACE(10, ("__kmp_fork_barrier: T#%d(%d:%d) has arrived\n", gtid, 1792 (team != NULL) ? team->t.t_id : -1, tid)); 1793 1794 // th_team pointer only valid for master thread here 1795 if (KMP_MASTER_TID(tid)) { 1796 #if USE_ITT_BUILD && USE_ITT_NOTIFY 1797 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) { 1798 // Create itt barrier object 1799 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier, 1); 1800 __kmp_itt_barrier_middle(gtid, itt_sync_obj); // Call acquired/releasing 1801 } 1802 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */ 1803 1804 #ifdef KMP_DEBUG 1805 kmp_info_t **other_threads = team->t.t_threads; 1806 int i; 1807 1808 // Verify state 1809 KMP_MB(); 1810 1811 for (i = 1; i < team->t.t_nproc; ++i) { 1812 KA_TRACE(500, 1813 ("__kmp_fork_barrier: T#%d(%d:0) checking T#%d(%d:%d) fork go " 1814 "== %u.\n", 1815 gtid, team->t.t_id, other_threads[i]->th.th_info.ds.ds_gtid, 1816 team->t.t_id, other_threads[i]->th.th_info.ds.ds_tid, 1817 other_threads[i]->th.th_bar[bs_forkjoin_barrier].bb.b_go)); 1818 KMP_DEBUG_ASSERT( 1819 (TCR_4(other_threads[i]->th.th_bar[bs_forkjoin_barrier].bb.b_go) & 1820 ~(KMP_BARRIER_SLEEP_STATE)) == KMP_INIT_BARRIER_STATE); 1821 KMP_DEBUG_ASSERT(other_threads[i]->th.th_team == team); 1822 } 1823 #endif 1824 1825 if (__kmp_tasking_mode != tskm_immediate_exec) { 1826 // 0 indicates setup current task team if nthreads > 1 1827 __kmp_task_team_setup(this_thr, team, 0); 1828 } 1829 1830 /* The master thread may have changed its blocktime between the join barrier 1831 and the fork barrier. Copy the blocktime info to the thread, where 1832 __kmp_wait_template() can access it when the team struct is not 1833 guaranteed to exist. */ 1834 // See note about the corresponding code in __kmp_join_barrier() being 1835 // performance-critical 1836 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) { 1837 #if KMP_USE_MONITOR 1838 this_thr->th.th_team_bt_intervals = 1839 team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals; 1840 this_thr->th.th_team_bt_set = 1841 team->t.t_implicit_task_taskdata[tid].td_icvs.bt_set; 1842 #else 1843 this_thr->th.th_team_bt_intervals = KMP_BLOCKTIME_INTERVAL(team, tid); 1844 #endif 1845 } 1846 } // master 1847 1848 switch (__kmp_barrier_release_pattern[bs_forkjoin_barrier]) { 1849 case bp_hyper_bar: { 1850 KMP_ASSERT(__kmp_barrier_release_branch_bits[bs_forkjoin_barrier]); 1851 __kmp_hyper_barrier_release(bs_forkjoin_barrier, this_thr, gtid, tid, 1852 TRUE USE_ITT_BUILD_ARG(itt_sync_obj)); 1853 break; 1854 } 1855 case bp_hierarchical_bar: { 1856 __kmp_hierarchical_barrier_release(bs_forkjoin_barrier, this_thr, gtid, tid, 1857 TRUE USE_ITT_BUILD_ARG(itt_sync_obj)); 1858 break; 1859 } 1860 case bp_tree_bar: { 1861 KMP_ASSERT(__kmp_barrier_release_branch_bits[bs_forkjoin_barrier]); 1862 __kmp_tree_barrier_release(bs_forkjoin_barrier, this_thr, gtid, tid, 1863 TRUE USE_ITT_BUILD_ARG(itt_sync_obj)); 1864 break; 1865 } 1866 default: { 1867 __kmp_linear_barrier_release(bs_forkjoin_barrier, this_thr, gtid, tid, 1868 TRUE USE_ITT_BUILD_ARG(itt_sync_obj)); 1869 } 1870 } 1871 1872 // Early exit for reaping threads releasing forkjoin barrier 1873 if (TCR_4(__kmp_global.g.g_done)) { 1874 this_thr->th.th_task_team = NULL; 1875 1876 #if USE_ITT_BUILD && USE_ITT_NOTIFY 1877 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) { 1878 if (!KMP_MASTER_TID(tid)) { 1879 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier); 1880 if (itt_sync_obj) 1881 __kmp_itt_barrier_finished(gtid, itt_sync_obj); 1882 } 1883 } 1884 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */ 1885 KA_TRACE(10, ("__kmp_fork_barrier: T#%d is leaving early\n", gtid)); 1886 return; 1887 } 1888 1889 /* We can now assume that a valid team structure has been allocated by the 1890 master and propagated to all worker threads. The current thread, however, 1891 may not be part of the team, so we can't blindly assume that the team 1892 pointer is non-null. */ 1893 team = (kmp_team_t *)TCR_PTR(this_thr->th.th_team); 1894 KMP_DEBUG_ASSERT(team != NULL); 1895 tid = __kmp_tid_from_gtid(gtid); 1896 1897 #if KMP_BARRIER_ICV_PULL 1898 /* Master thread's copy of the ICVs was set up on the implicit taskdata in 1899 __kmp_reinitialize_team. __kmp_fork_call() assumes the master thread's 1900 implicit task has this data before this function is called. We cannot 1901 modify __kmp_fork_call() to look at the fixed ICVs in the master's thread 1902 struct, because it is not always the case that the threads arrays have 1903 been allocated when __kmp_fork_call() is executed. */ 1904 { 1905 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_icv_copy); 1906 if (!KMP_MASTER_TID(tid)) { // master thread already has ICVs 1907 // Copy the initial ICVs from the master's thread struct to the implicit 1908 // task for this tid. 1909 KA_TRACE(10, 1910 ("__kmp_fork_barrier: T#%d(%d) is PULLing ICVs\n", gtid, tid)); 1911 __kmp_init_implicit_task(team->t.t_ident, team->t.t_threads[tid], team, 1912 tid, FALSE); 1913 copy_icvs(&team->t.t_implicit_task_taskdata[tid].td_icvs, 1914 &team->t.t_threads[0] 1915 ->th.th_bar[bs_forkjoin_barrier] 1916 .bb.th_fixed_icvs); 1917 } 1918 } 1919 #endif // KMP_BARRIER_ICV_PULL 1920 1921 if (__kmp_tasking_mode != tskm_immediate_exec) { 1922 __kmp_task_team_sync(this_thr, team); 1923 } 1924 1925 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 1926 kmp_proc_bind_t proc_bind = team->t.t_proc_bind; 1927 if (proc_bind == proc_bind_intel) { 1928 #endif 1929 #if KMP_AFFINITY_SUPPORTED 1930 // Call dynamic affinity settings 1931 if (__kmp_affinity_type == affinity_balanced && team->t.t_size_changed) { 1932 __kmp_balanced_affinity(tid, team->t.t_nproc); 1933 } 1934 #endif // KMP_AFFINITY_SUPPORTED 1935 #if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED 1936 } else if (proc_bind != proc_bind_false) { 1937 if (this_thr->th.th_new_place == this_thr->th.th_current_place) { 1938 KA_TRACE(100, ("__kmp_fork_barrier: T#%d already in correct place %d\n", 1939 __kmp_gtid_from_thread(this_thr), 1940 this_thr->th.th_current_place)); 1941 } else { 1942 __kmp_affinity_set_place(gtid); 1943 } 1944 } 1945 #endif 1946 1947 #if USE_ITT_BUILD && USE_ITT_NOTIFY 1948 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) { 1949 if (!KMP_MASTER_TID(tid)) { 1950 // Get correct barrier object 1951 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier); 1952 __kmp_itt_barrier_finished(gtid, itt_sync_obj); // Workers call acquired 1953 } // (prepare called inside barrier_release) 1954 } 1955 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */ 1956 ANNOTATE_BARRIER_END(&team->t.t_bar); 1957 KA_TRACE(10, ("__kmp_fork_barrier: T#%d(%d:%d) is leaving\n", gtid, 1958 team->t.t_id, tid)); 1959 } 1960 1961 void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc, 1962 kmp_internal_control_t *new_icvs, ident_t *loc) { 1963 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_setup_icv_copy); 1964 1965 KMP_DEBUG_ASSERT(team && new_nproc && new_icvs); 1966 KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc); 1967 1968 /* Master thread's copy of the ICVs was set up on the implicit taskdata in 1969 __kmp_reinitialize_team. __kmp_fork_call() assumes the master thread's 1970 implicit task has this data before this function is called. */ 1971 #if KMP_BARRIER_ICV_PULL 1972 /* Copy ICVs to master's thread structure into th_fixed_icvs (which remains 1973 untouched), where all of the worker threads can access them and make their 1974 own copies after the barrier. */ 1975 KMP_DEBUG_ASSERT(team->t.t_threads[0]); // The threads arrays should be 1976 // allocated at this point 1977 copy_icvs( 1978 &team->t.t_threads[0]->th.th_bar[bs_forkjoin_barrier].bb.th_fixed_icvs, 1979 new_icvs); 1980 KF_TRACE(10, ("__kmp_setup_icv_copy: PULL: T#%d this_thread=%p team=%p\n", 0, 1981 team->t.t_threads[0], team)); 1982 #elif KMP_BARRIER_ICV_PUSH 1983 // The ICVs will be propagated in the fork barrier, so nothing needs to be 1984 // done here. 1985 KF_TRACE(10, ("__kmp_setup_icv_copy: PUSH: T#%d this_thread=%p team=%p\n", 0, 1986 team->t.t_threads[0], team)); 1987 #else 1988 // Copy the ICVs to each of the non-master threads. This takes O(nthreads) 1989 // time. 1990 ngo_load(new_icvs); 1991 KMP_DEBUG_ASSERT(team->t.t_threads[0]); // The threads arrays should be 1992 // allocated at this point 1993 for (int f = 1; f < new_nproc; ++f) { // Skip the master thread 1994 // TODO: GEH - pass in better source location info since usually NULL here 1995 KF_TRACE(10, ("__kmp_setup_icv_copy: LINEAR: T#%d this_thread=%p team=%p\n", 1996 f, team->t.t_threads[f], team)); 1997 __kmp_init_implicit_task(loc, team->t.t_threads[f], team, f, FALSE); 1998 ngo_store_icvs(&team->t.t_implicit_task_taskdata[f].td_icvs, new_icvs); 1999 KF_TRACE(10, ("__kmp_setup_icv_copy: LINEAR: T#%d this_thread=%p team=%p\n", 2000 f, team->t.t_threads[f], team)); 2001 } 2002 ngo_sync(); 2003 #endif // KMP_BARRIER_ICV_PULL 2004 } 2005